From ef6b4508ed2de26a55ac24bac00f0ba125744638 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 22:15:59 +0800 Subject: [PATCH 01/20] feat: a source for every tool a build uses, declared, programmable and observable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A build uses a toolchain, the payloads its plugins declare, and the tools those plugins run. Each of them now has one source that a project can state, a build program can decide, and anyone can read back; a project that writes none of the new keys builds exactly as before, with the same output. - `[xlings.overrides]`, `MCPP_XLINGS_OVERRIDE__` and config.toml state where a declared payload comes from. An overridden payload is not provisioned and does not reach the offline gate; `xpkg_program` and `xpkg_source` answer the program it named and `override`. A stated version is checked against every requirement the graph made, and a dependency that writes the table is refused. - `provision = "on-request"` installs a payload when a build program asks for it with `xpkg_request`: one batch per invocation, only the programs that asked run again, and planning records MCPP_BUILD_DATABASE_PAYLOAD_DEFERRED instead. - `[toolchain] = { path = ... }` and `MCPP_TOOLCHAIN=path:` name a toolchain this machine already has; mcpp probes it, drives it with its own link model, writes nothing into the tree, keys the fingerprint and the fast paths on its programs' content, and records it in mcpp.lock as local. `bootstrap` names the toolchain that builds build programs, and `{ configure = "build.mcpp" }` hands the build toolchain to the root program's toolchain phase. - A build reports a source that is not the ecosystem's on its own line, sums them on the `Finished` line, and writes the record to resolution.json; `mcpp why sources|tool|payload` and the `mcpp.why.sources` kind read it, and `--managed-only` refuses a build that uses any. - Protocol 15: xpkg_source, xpkg_program, xpkg_request, xpkg_pending, phase, decision, toolchain. Specifications and both documentation trees are updated: SPEC-004 §4.7, SPEC-006 §2.2.1 and §3.3, SPEC-007 R6.2/R6.5/R6.6/R9.9 and the protocol table, docs/09, 20, 23, 30, 31, 32, 50 and their 简体中文 mirrors. Closes #755 Test plan - `mcpp test`: 144 passed, including the new tests/unit/test_sources.cpp (15). - New e2e 873 (overrides), 874 (on-request), 875 (a toolchain by path), 876 (the toolchain phase), 877 (`mcpp why` and its machine output), each with MCPP_NO_AUTO_INSTALL=1 as the criterion and a control that must refuse. - 62 existing e2e cases selected by keyword pass; 219 fails identically on the released 2026.10.1.2 on this machine, and 658 needs an attached Android device. - mcpp-plugins 0.19.0 against this engine: 11 consumer fixtures and 27 plugin-logic cases pass; tests/cmake-consumer builds under MCPP_NO_AUTO_INSTALL=1 when its build program names its own cmake. --- ...10-01-tool-and-toolchain-sources-design.md | 141 ++++ .agents/docs/README.md | 3 +- CHANGELOG.md | 54 ++ docs/09-commands-by-scenario.md | 21 +- docs/20-toolchains.md | 150 ++++ docs/23-the-project-environment.md | 61 ++ docs/30-build-mcpp.md | 51 ++ docs/31-authoring-a-rule-package.md | 21 + docs/32-authoring-a-payload.md | 6 +- docs/50-machine-output.md | 44 ++ docs/specs/build-database.md | 1 + docs/specs/build-plugins.md | 36 +- docs/specs/manifest-semantics.md | 29 + docs/specs/toolchain-management.md | 36 +- docs/zh/09-commands-by-scenario.md | 19 +- docs/zh/20-toolchains.md | 133 ++++ docs/zh/23-the-project-environment.md | 54 ++ docs/zh/30-build-mcpp.md | 46 ++ docs/zh/31-authoring-a-rule-package.md | 14 + docs/zh/32-authoring-a-payload.md | 4 +- docs/zh/50-machine-output.md | 44 ++ mcpp.toml | 2 +- modules/buildmcpp/src/directives.cppm | 29 +- modules/buildmcpp/src/program_protocol.cppm | 9 +- modules/manifest/src/toml.cppm | 215 +++++- modules/manifest/src/types.cppm | 89 ++- modules/toolchain-model/src/linkmodel.cppm | 11 + modules/toolchain-model/src/model.cppm | 19 + modules/versioning/src/version.cppm | 2 +- src/build/build_program.cppm | 72 +- src/build/execute.cppm | 46 ++ src/build/flags.cppm | 14 + src/build/hostprogram.cppm | 84 ++- src/build/ninja_backend.cppm | 18 +- src/build/prepare.cppm | 86 ++- src/build/prepare/config.cpp | 6 + src/build/prepare/driver.cpp | 50 ++ src/build/prepare/features.cpp | 15 +- src/build/prepare/graph.cpp | 72 +- src/build/prepare/graph_load.cpp | 21 +- src/build/prepare/local_toolchain.cpp | 359 ++++++++++ src/build/prepare/plan.cpp | 11 +- src/build/prepare/records.cpp | 46 ++ src/build/prepare/sources.cpp | 644 ++++++++++++++++++ src/build/prepare/state.cppm | 96 +++ src/build/prepare/target_side.cpp | 18 +- src/build/prepare/toolchain.cpp | 8 + src/build/prepare/xlings.cpp | 89 ++- src/build/progress.cppm | 49 ++ src/build/refusal.cppm | 13 + src/cli.cppm | 17 +- src/cli/cmd_self.cppm | 11 +- src/config.cppm | 43 ++ src/doctor.cppm | 108 ++- src/project.cppm | 10 +- src/toolchain/gcc.cppm | 8 + src/toolchain/registry.cppm | 63 ++ src/ui.cppm | 17 + src/wire.cppm | 6 +- src/xlings/address_set.cppm | 45 ++ tests/e2e/873_payload_overrides.sh | 122 ++++ tests/e2e/874_payload_on_request.sh | 92 +++ tests/e2e/875_toolchain_by_path.sh | 91 +++ tests/e2e/876_toolchain_phase.sh | 80 +++ tests/e2e/877_why_sources.sh | 46 ++ tests/unit/test_build_directives.cpp | 8 +- tests/unit/test_sources.cpp | 252 +++++++ 67 files changed, 4017 insertions(+), 163 deletions(-) create mode 100644 .agents/docs/2026-10-01-tool-and-toolchain-sources-design.md create mode 100644 src/build/prepare/local_toolchain.cpp create mode 100644 src/build/prepare/sources.cpp create mode 100755 tests/e2e/873_payload_overrides.sh create mode 100755 tests/e2e/874_payload_on_request.sh create mode 100755 tests/e2e/875_toolchain_by_path.sh create mode 100755 tests/e2e/876_toolchain_phase.sh create mode 100755 tests/e2e/877_why_sources.sh create mode 100644 tests/unit/test_sources.cpp diff --git a/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md b/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md new file mode 100644 index 000000000..6d61cf601 --- /dev/null +++ b/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md @@ -0,0 +1,141 @@ +# 工具与工具链的来源：声明、编程决定、可观察 + +日期：2026-10-01。状态：已落地（mcpp 2026.10.1.3,mcpp#755)。 + +生态侧的设计记录在 mcpp-plugins `.agents/docs/2026-10-01-ecosystem-build-plugin-framework-design.md`(v3)。本记录只写引擎这一侧:为什么是这些机制,以及每个决定的理由。 + +## 1. 问题 + +一次构建用到三类东西,它们的来源此前各有一套规则,而其中两类根本无法由使用者陈述。 + +**插件声明的载荷与是否使用无关,都在构建程序运行前下载。** 这是时序的后果: +`[feature-xlings.]` 的条目在特性打开、target 选择器为真时由 prepare 供给 +(features.cpp 的 `step6_xlings_workspace_from_graph`),而构建程序在那之后才运行。于是 +`build.mcpp` 里写 `o.cmake = "/usr/bin/cmake"` 也照样下载 `xim:cmake`;离线时整个构建被拒, +构建程序根本没有运行的机会。实测记录在 mcpp-plugins 的 +`.agents/docs/2026-10-01-payload-source-verify.sh`:一个替身插件在 `MCPP_NO_AUTO_INSTALL=1` +下三种配置的读数。 + +**下载由「声明了」触发,而不是「用到了」。** `dist-apk` 的 `bundletool` 只在 +`--format aab` 时用到,却每次 Android 构建都装;`dist-appimage` 的工具每次 Linux 构建都装。 +插件清单自己记下了这件事的代价:「provisioning runs before the build program learns +`--format`」。 + +**主工具链只能是托管载荷。** PATH 上的编译器被拒,理由是无法识别、无法复现(docs/20), +而这条理由对「一棵被命名、被识别、被记录的树」并不成立——`msvc@system` 就是反例,它定位 +机器上的 VS 并照常驱动。自建 trunk、厂商交叉工具链因此只能写 xim 配方,还要改核心的 +`to_xim_package`。 + +**输出不区分来源。** 一次构建说不出「这次用的 cmake 是谁的」,出错时也无从归因。 + +## 2. 决定与理由 + +### 2.1 一个「来源」概念,五个类 + +主体是 `toolchain.build`、`toolchain.bootstrap`、`payload::`、 +`tool::`;类是 `managed`、`pinned`、`custom`、`program`、`host`。 + +类划出的线只有一条:**是生态选的,还是人或机器选的**。`managed` 与 `pinned` 都是生态的, +所以它们合起来是「默认」,而默认的输出必须与本机制存在之前逐字相同——这是无感升级的判据, +e2e 与 framework-lab 的 golden 都按它断言。 + +`host` 单独成类,不与 `custom` 合并:它把产物与机器状态绑在一起,而其他几类不会。判据是 +「版本有没有被陈述」,不是「路径像不像系统目录」——后者是猜测。 + +### 2.2 记录只有一份 + +`SourceDecision` 一个主体一条,写进已有的 `resolution.json`(新增 `sources` 键,没有读者 +按 `schema_version` 分支),并由输出、`mcpp why`、机器输出、`--managed-only` 共同读取。 + +**理由是这个代码库已经付过的代价。** 同一个答案被两处分别推导,失败形态是「装了 A、答了 +B,而且什么都没说」——`mcpp.xlings.address_set` 的文件头记着这件事。所以 `fillXpkgDirs` +与供给集合共用一次统一的结果,来源记录也只有一处写入。 + +### 2.3 覆盖:环境变量 > 清单 > 全局配置 + +与 `[indices]` 的「项目 > 全局」方向一致,但环境变量排在清单之前。理由是 CI 与发行版打包 +要在不改清单的前提下换工具,而 `[tools.overrides]` 的文档已经把环境变量定位成这件事的出口。 +风险(环境意外覆盖项目的明确决定)由「每条覆盖都显示并记录」抵掉: +`Using xim:cmake ← … [custom · env MCPP_XLINGS_OVERRIDE_XIM_CMAKE]`。 + +覆盖只认根。依赖替消费方决定来源,就是替使用者做决定;`[tools.overrides]` 已经立下同一条 +规矩。 + +**覆盖参与版本校验,但不参与裁决。** 它陈述的是「从哪里来」,不是「要哪一版」,所以键不带 +版本;写了 `version` 时按 `addrset::override_violation` 与每条落败的要求比较,没写时记一条 +note 点出未被校验的要求。引擎**不**运行任意程序去问版本:每个工具的 `--version` 格式不同, +那是插件的知识。 + +### 2.4 按需供给:请求 + 重跑,而不是「构建程序之后再供给」 + +`rules-cuda` 读工具包头文件里的版本,`rules-qt` 读 SDK 文件,`dist-apk` 从 platform 目录读 +API level,`dist-wix` 检查 payload 里的文件是否存在——这些成员在**规划时**就需要载荷。把 +供给整体移到构建程序之后会让它们规划失败。 + +所以是:程序请求 → 引擎批量安装 → 只重跑请求过的程序。请求的那次运行被**丢弃** +(`run_build_program` 在 `dirs::apply` 与 `write_cache` 之前返回),所以没有要撤销的状态, +也没有半应用的指令集。最多三轮,第三轮仍有新请求就报错并点名——终止条件不依赖被调用方 +改变状态,这是 `resolve_target_toolchain` 的递归曾经付过的代价。 + +稳态零开销:下一次构建载荷已安装,第一轮就能拿到答案,`contract_hash` 因此与第二轮相同, +缓存命中。 + +### 2.5 工具链:bootstrap 与 build 两段,`path:` 进入 spec + +这两个角色**本来就存在**:交叉构建时 `build.mcpp` 由一个宿主工具链编译 +(`xlings.cpp` 的 G3 分支)。本次只是给它们命名,并让 build 这一侧可以独立配置。 + +`path:` 做成 `ToolchainSpec` 的一种拼法,而不是第二条解析路径。理由是 +`parse_toolchain_spec` 有十几个调用点,第二条路径意味着十几处都要学会它;做成一种拼法后, +不认识它的调用点自然报错而不是静默走错。族由 `/bin` 里有哪个驱动决定——驱动是事实, +清单里的 `family` 只用于核对。 + +**不写入那棵树。** 托管载荷的 `clang++.cfg` 是这套机制的**产物**(docs/91 §5.1),写给直接 +调用 clang 的人;mcpp 自己的调用绕过它。一棵不属于 mcpp 的树因此不该被写入, +`resolve_clang_driver` 改为在 `localRoot` 非空时按「驱动旁有 libc++」开启模型,而不是按 +cfg 文件是否存在。 + +**身份按内容,不按版本。** 一个 trunk 驱动可以在版本不变时被重建,所以驱动与每个 +`tools` 程序的路径、大小、修改时间进入 `driverIdent`(于是进入指纹),并写入 +`local-toolchain.stamp` 供三条快速路径比较。不读字节:一个驱动几百 MB,而每次 prepare 都要 +读它。 + +### 2.6 工具链阶段:两遍 prepare,第一遍不说话 + +构建程序需要它的宿主模块,宿主模块来自依赖图;而依赖图的解析需要工具链 +(`cfg(compiler = ...)`、`requires`)。这是一个真实的环,所以它被切成两遍: +第一遍用 bootstrap 走到宿主模块注册,运行工具链阶段,然后以一个内部信号结束; +第二遍从头用陈述的工具链。 + +第一遍**不叙述**:它要说的每一句,第二遍都会就真正发生的那次构建再说一次。实现是 +`driver.cpp` 的 `QuietPass`,而不是在每个输出点加条件——后者是会漏的那种。 + +工具链阶段**只能**陈述工具链:它运行在依赖图之前,那里的一条 flag、一个源文件或一个 +action 描述的是一次还不存在的构建;一个载荷请求也无法回答,因为声明它的图还没读。引擎 +按名拒绝,而不是静默丢弃。 + +它有自己的 `artifactsDir`(`target/.build-mcpp/toolchain-phase`):与构建阶段共用一份缓存 +记录,会让两者每次构建互相失效。 + +## 3. 验证 + +- 单元:`tests/unit/test_sources.cpp`(15 例)——清单键的解析与拒绝、协议 15 的指令、 + 覆盖的版本校验、`path:` 的族判定。 +- e2e:873(覆盖)、874(按需)、875(按路径命名的工具链)、876(工具链阶段)、 + 877(`mcpp why` 与机器输出)。判据统一是 `MCPP_NO_AUTO_INSTALL=1`:构建成功即「没有要求 + 下载」,而拒绝会点名它本要装的东西。 +- 既有 e2e:以关键词选出的 62 个用例通过;219 在**已发布的 2026.10.1.2** 上同样失败(本机 + glibc 顺序),658 需要连接 Android 设备。 +- 生态:mcpp-plugins 0.19.0 的 11 个 consumer fixture 与 27 例 `plugin-logic` 通过; + `tests/cmake-consumer` 在 `MCPP_NO_AUTO_INSTALL=1` 下,由 `build.mcpp` 点名 cmake 即可 + 构建——这正是本次要做成的那件事。 + +## 4. 已知边界 + +- `mcpp build` 没有 `--format json`,所以来源的机器形态走 `mcpp why sources`,而不是构建 + 事件流。docs/50 §3 把 `ndjson` 记为保留,本次不动它。 +- 覆盖与按需供给都只作用于 xlings 载荷。依赖包 `kind = "bin"` 的宿主工具仍走 + `[tools.overrides]`:两者的键空间与语义不同(一个是 `:` 的程序,一个是 + `ns:name` 的目录),合并会让一张表有两种键。 +- 工具链描述数据化(核心读描述文件、不再硬编码 `to_xim_package`)不在本次范围; + `[toolchain] { path }` 的字段已经与那份描述同形,所以它是后续的第三种载体,而不是改写。 diff --git a/.agents/docs/README.md b/.agents/docs/README.md index 813883387..4263aa9d2 100644 --- a/.agents/docs/README.md +++ b/.agents/docs/README.md @@ -18,7 +18,7 @@ superseded_by: 2026-09-07-....md # when status is superseded --- ``` -322 records. +323 records. ## By subject @@ -113,6 +113,7 @@ Records that declare one. Everything else is listed by date below. ### 2026-10 +- [工具与工具链的来源：声明、编程决定、可观察](2026-10-01-tool-and-toolchain-sources-design.md) - [A pack's build reported as a build, and a unit's compile independent of the member selection: triage and design (#753, #751)](2026-10-01-pack-drive-and-selection-independent-compile-design.md) — landed ### 2026-09 diff --git a/CHANGELOG.md b/CHANGELOG.md index ed61d083b..f0d55e130 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,60 @@ > Each `## []` section is that release's notes. Entries are written in English > from 2026.9.28.3 on; earlier entries remain as written. +## [2026.10.1.3] - 2026-10-01 + +This release gives every tool a build uses a source that can be declared, +decided by a build program, and read back (mcpp#755; +`.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md`). A project that +writes none of the new keys builds exactly as before, and its output is +unchanged. + +### Added + +- **`[xlings.overrides]` states where a declared payload comes from**, in the + root manifest (also under `[target.'cfg(..)']`), as + `MCPP_XLINGS_OVERRIDE__`, or in `~/.mcpp/config.toml`. An + overridden payload is not provisioned and does not reach the offline gate; + `mcpp::xpkg_dir` answers the root it implies, and the new + `mcpp::xpkg_program` and `mcpp::xpkg_source` answer the program it named and + `override`. A stated `version` is checked against every requirement a package + of the graph made. A dependency that writes the table is refused: which + payloads a package needs is its own statement, where they come from is the + project's. +- **`provision = "on-request"` installs a payload when a build program asks for + it**, with `mcpp::xpkg_request`. Every request of one invocation is installed + together and only the programs that asked run again, so a build whose program + names its own tool downloads nothing. `mcpp emit build-database` installs + nothing and records `MCPP_BUILD_DATABASE_PAYLOAD_DEFERRED`. +- **A toolchain named by path**: `[toolchain] = { path = "", prefix, + sysroot, family, launcher, tools }`, or `MCPP_TOOLCHAIN=path:`. mcpp + probes the drivers in the tree, identifies them, drives them with its own + link model, and writes nothing into the tree. The driver and each stated tool + enter the fingerprint by content, the fast paths decline when one changed, + and `mcpp.lock` records the toolchain as `local`. +- **`[toolchain] bootstrap`** names the toolchain that compiles and runs build + programs when it should not be the one building the project. +- **`[toolchain] = { configure = "build.mcpp" }`** hands the build + toolchain to the root build program: it runs once in a toolchain phase, where + `mcpp::phase()` is `"toolchain"`, and states the toolchain with + `mcpp::toolchain(key, value)`. That phase may state nothing else. +- **A build reports its sources.** A source that is not the ecosystem's gets a + line of its own (`Using … [custom · mcpp.toml:22]`, `Bootstrap …`), the + `Finished` line summarises them, and the record is written to + `resolution.json`. `mcpp why sources`, `mcpp why tool ` and + `mcpp why payload ` report it, including as `mcpp.why.sources` under + `--format json`. +- **`--managed-only` / `MCPP_MANAGED_ONLY=1`** refuses a build whose toolchain, + payload or plugin tool came from anywhere but the ecosystem, naming each. +- **Protocol 15** for build programs: `xpkg_source`, `xpkg_program`, + `xpkg_request`, `xpkg_pending`, `phase`, `decision` and `toolchain`. + +### Changed + +- `mcpp why toolchain` states the source of the toolchain and the origin the + resolution recorded, in place of a sentence listing every way one can be + chosen. + ## [2026.10.1.2] - 2026-10-01 This release implements the design for a pack's build and a compile that does diff --git a/docs/09-commands-by-scenario.md b/docs/09-commands-by-scenario.md index 70bd75666..86c19fce0 100644 --- a/docs/09-commands-by-scenario.md +++ b/docs/09-commands-by-scenario.md @@ -131,7 +131,21 @@ recorded build is replayed only for the toolchain request that recorded it. $ mcpp why toolchain toolchain: gcc 16.1.0 (x86_64-linux-gnu) abi(libc)=glibc cxxstdlib=libstdc++ arch=x86_64 os=linux triple=x86_64-linux-gnu - reason: [toolchain] in mcpp.toml if set, else platform-native default + source: pinned · [toolchain] + reason: [toolchain] in mcpp.toml +``` + +`mcpp why sources` reports where each tool a build uses came from +(2026.10.1.3+): the toolchain, every payload, and every tool a plugin runs, with +what was consulted for it. `mcpp why tool ` and +`mcpp why payload ` narrow it to one: + +``` +$ mcpp why payload cmake +sources: + payload:xim:cmake /usr/bin/cmake + custom · mcpp.toml:22 for mcpp:plugins + considered: payload xim:cmake@>=3.31 (not installed: overridden) ``` `mcpp why deps` lists the resolved dependency graph before the lines of @@ -633,8 +647,9 @@ mcpp is involved and its documentation will not help. ## Current limitations -- `mcpp why --format json` is defined for the `toolchain` topic only. The other - topics report `'' has no machine-readable shape yet` and exit non-zero. +- `mcpp why --format json` is defined for the `toolchain`, `sources`, `tool` and + `payload` topics. The other topics report `'' has no machine-readable + shape yet` and exit non-zero. - `mcpp search` matches a substring; there is no field selector, and no way to restrict a search to one namespace. - `mcpp clean --stale` reads `target/.build_cache`, which holds a bounded number diff --git a/docs/20-toolchains.md b/docs/20-toolchains.md index 6be8a7179..3a22432de 100644 --- a/docs/20-toolchains.md +++ b/docs/20-toolchains.md @@ -369,6 +369,106 @@ A build that provably *cannot* run stays an error on either axis: a runtime closure that cannot be satisfied is refused, because the artifact will not start. See [binary distribution](12-binary-distribution.md). +### `{ path = … }` — a toolchain this machine already has (2026.10.1.3+) + +A toolchain mcpp did not install — a self-built LLVM, a vendor cross toolchain, +an extracted release package — is named by path: + +```toml +[toolchain] +default = { path = "/opt/llvm-trunk" } + +# A cross toolchain whose drivers and tools carry a prefix, with its own sysroot +# and a tool the tree does not keep where the layout expects it: +[toolchain.linux] +path = "/opt/acme-gcc" +prefix = "aarch64-none-linux-gnu-" +sysroot = "/opt/acme-sysroot" +family = "gcc" # checked against the drivers +launcher = "ccache" # prefixes every compile +tools = { ld = "/opt/lld-dev/bin/ld.lld" } # cc, cxx, ld, ar, ranlib, nm, objcopy, strip, as +``` + +Or for one build, without editing the manifest: + +```bash +MCPP_TOOLCHAIN=path:/opt/llvm-trunk mcpp build +``` + +**The layout.** The drivers live in `/bin`: `clang++` (llvm) or `g++` +(gcc), under `prefix` when there is one. Their family is read from which of the +two is present, and the version, the target triple, the standard library and +whether `import std` is available are read from the driver itself. The tools +beside them are found as ``, then `llvm-`, then ``; +`tools` names any the tree does not have. + +**What mcpp does with it.** The same as with a payload it installed: its own +link line, its own hermetic check, its own `import std` decision. It writes +nothing into the tree — the generated `clang++.cfg` of a managed payload is +mcpp's own file, and a tree mcpp does not own does not get one. + +**Identity.** The driver and each tool named by `tools` enter the build's +fingerprint by path, size and modification time, so rebuilding the toolchain in +place rebuilds what it produced. A build that used one records it in +`mcpp.lock` as `local`: a machine without that toolchain is told so by name +rather than building with another one. + +This is not the `system` compiler of the section above. `system` is whatever +`PATH` happens to hold; this is a tree the project names, that mcpp identifies +and records, in the shape `msvc@system` has always had. + +### `bootstrap` — the toolchain that builds build programs (2026.10.1.3+) + +```toml +[toolchain] +default = { path = "/opt/llvm-trunk" } +bootstrap = "llvm@22.1.8" +``` + +Build programs (`build.mcpp`), host tools and host modules are compiled and run +on the machine doing the build. `bootstrap` names the toolchain that does that +when it should not be the one building the project — a toolchain under +development, say, which has to be able to fail without stopping the program that +chose it. With no `bootstrap` the behaviour is unchanged: a native build uses +its own toolchain, and a cross build resolves a host one. + +### `{ configure = "build.mcpp" }` — the build program states the toolchain (2026.10.1.3+) + +A toolchain a manifest cannot express — parts from several places, a vendor +SDK's environment script, a version picked by looking at the machine — is +stated by the root build program: + +```toml +[toolchain] +default = { configure = "build.mcpp" } +``` + +```cpp +import mcpp; +import mcpp.plugins.toolchain; // mcpp:plugins, feature plugins-toolchain +namespace tc = mcpp::plugins::toolchain; + +int main() { + if (tc::configure([] { + auto d = tc::layout(tc::env("ACME_LLVM", "/opt/acme-llvm")); + tc::use(tc::with_launcher(d, "ccache")); + })) + return 0; + // the ordinary build phase +} +``` + +The program runs twice. Its **toolchain phase** runs first, compiled with the +bootstrap toolchain, before the dependency graph is resolved; `mcpp::phase()` is +`"toolchain"` there, and the only thing it may state is the toolchain — a flag, +a source or an action in that phase is refused, naming the directive. Its +**build phase** then runs as any build program does, with the toolchain it +stated. + +The keys are the ones a `[toolchain]` table takes (`spec` for a managed one, +or `path`, `prefix`, `sysroot`, `family`, `launcher`, `tool.`), so what a +manifest can say and what a program can say are one vocabulary. + ### `msvc@system` — the machine's own Visual Studio mcpp locates and identifies an installed Visual Studio / Build Tools; it never @@ -713,6 +813,53 @@ mcpp build --target aarch64-ios # resolves llvm@22.1.8 + the iPhoneOS SDK mcpp build --target aarch64-ios-sim # resolves llvm@22.1.8 + the Simulator SDK ``` +## The source of each tool (2026.10.1.3+) + +A build uses a toolchain, the payloads its plugins declare, and the tools those +plugins run. Each of them has one source, and a build whose sources are all the +ecosystem's prints exactly what it printed before this existed. + +| class | meaning | +|---|---| +| `managed` | the ecosystem's default: nothing was written | +| `pinned` | a managed payload or toolchain at a version a manifest or the machine default chose | +| `custom` | a path stated in `mcpp.toml`, an environment variable, or `config.toml` | +| `program` | decided by a build program | +| `host` | found on `PATH`, with its version stated by nobody | + +Anything but the first two gets a line of its own, naming what it is, where it +came from, and the statement that chose it: + +``` + Resolving toolchain + Bootstrap llvm@22.1.8 → @mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/bin/clang++ + Using toolchain clang 23.0.0git ← /opt/acme-llvm [program · build.mcpp:9] + Target x86_64-unknown-linux-gnu + Using xim:cmake ← /usr/bin/cmake [custom · mcpp.toml:22] + Compiling app v0.1.0 (.) + Finished dev [unoptimized + debuginfo] in 2.51s · custom: xim:cmake; program: toolchain +``` + +The tag carries the whole statement, so the line reads the same without colour. +A tool a plugin took from an overridden payload is the same statement as that +override, and is reported once. + +`mcpp why sources` lists every source with what was consulted for it; +`mcpp why tool ` and `mcpp why payload ` narrow it to one. +`--format json` answers the same as `mcpp.why.sources`, and every build writes +the record into `target///resolution.json`. + +### `--managed-only` — refuse anything but the ecosystem's + +```bash +mcpp build --managed-only # or MCPP_MANAGED_ONLY=1 +``` + +A build whose toolchain, payload or plugin tool came from anywhere else is +refused, naming each one and where it was stated. It is what a release build or +a reproducibility audit asks for; the fast paths decline under it, because the +sources are decided by the resolution it skips. + ## The host surface mcpp keeps, and the reason for each The rule this engine is arranged around: **a build is reproducible only if the @@ -989,6 +1136,9 @@ mcpp's runtime behavior can be adjusted with the following environment variables | `MCPP_OFFLINE=1` | Never touch the network; equivalent to global `--offline` | | `MCPP_NO_COLOR=1` / `NO_COLOR=1` | Disable colored output | | `MCPP_LOG_LEVEL=debug\|info\|warn\|error\|off` | Log level | +| `MCPP_TOOLCHAIN=path:` | Build with the toolchain in that directory, for this build | +| `MCPP_XLINGS_OVERRIDE__` | Where that payload comes from (`path:` looks it up on PATH) | +| `MCPP_MANAGED_ONLY=1` | Refuse a build whose sources are not the ecosystem's | When `MCPP_HOME` is not set explicitly, mcpp locates the sandbox automatically based on the parent directory of the binary (after a release tarball is extracted to `~/.mcpp/`, `~/.mcpp/` is the home), so the release build runs without any environment variable configuration. diff --git a/docs/23-the-project-environment.md b/docs/23-the-project-environment.md index 19ea23d2f..bdd74ca01 100644 --- a/docs/23-the-project-environment.md +++ b/docs/23-the-project-environment.md @@ -406,6 +406,65 @@ A feature name no `[features]` table declares is reported as a schema warning: it activates for nobody and installs nothing, and a tool whose absence is only visible as *"the device is never reachable"* is the hardest kind to diagnose. +### `provision = "on-request"` — installed when a build program asks (2026.10.1.3+) + +An entry may say when it is installed, beside the version and the tier: + +```toml +[feature-xlings.deps-cmake] +"xim:cmake" = { version = ">=3.31", provision = "on-request" } +``` + +`eager`, the default, installs the package before any build program runs. +`on-request` installs it when a build program asks for it with +`mcpp::xpkg_request` — so a build that names its own cmake, or one whose plan +never reaches the tool, downloads nothing. Every request of one mcpp invocation +is installed together, and only the programs that asked run again. + +A package is deferred only when every declaration of it says so: one manifest +that needs it eagerly installs it eagerly. + +`mcpp emit build-database` installs nothing on request. It records a note +(`MCPP_BUILD_DATABASE_PAYLOAD_DEFERRED`) naming the program and the packages, +and describes the plan without them. + +### `[xlings.overrides]` — the source of a payload (2026.10.1.3+) + +A declared payload may come from somewhere else on this machine: + +```toml +[xlings.overrides] +"xim:cmake" = "/usr/bin/cmake" # a file: the program +"xim:vcpkg" = "/opt/vcpkg" # a directory: a root laid out like the payload +"xim:slang" = { program = "slangc" } # a name: found on PATH +"xim:glslang" = { program = "/usr/bin/glslangValidator", version = "15.1.0" } +``` + +An overridden package is **not installed**: it leaves the provisioning list, so +`--offline` does not refuse it either. `mcpp::xpkg_dir` answers the root the +entry implies, `mcpp::xpkg_program` the program it named, and +`mcpp::xpkg_source` answers `override`. + +Three places may state an override, highest first: + +| | place | purpose | +|---|---|---| +| 1 | `MCPP_XLINGS_OVERRIDE__` | CI and distribution packaging, without editing the manifest. `path:cmake` looks the name up on PATH | +| 2 | `[xlings.overrides]` in the root manifest, also under `[target.'cfg(..)']` | the project's own statement | +| 3 | `[xlings.overrides]` in `~/.mcpp/config.toml` | a fact about this machine | + +A `version` in the entry is checked against every requirement a package of the +graph stated, and a version below one is refused naming both sides. Without a +`version` nothing can be compared, and the build says which requirement went +unchecked. + +Only the project being built states an override. A dependency that writes the +table is refused, naming the package: which payloads a package needs is its own +statement, where they come from is the project's. + +A build reports every override it used, and `--managed-only` refuses a build +that uses any: see [20 — Toolchain Management](20-toolchains.md). + ### A rule package brings its own environment (2026.9.6.6+) The table above is what a project writes when it has an opinion. Most projects @@ -529,6 +588,8 @@ used it. ## Current limitations +- An override states where a payload comes from, not which one: the key names + no version, and the package's identity stays what the declarations say. - **A tool cannot be conditioned on the accelerator.** The accelerator is resolved after the dependency graph, so such a tool would be declared and never installed — a build that succeeds with the tool simply absent. A manifest that diff --git a/docs/30-build-mcpp.md b/docs/30-build-mcpp.md index 9886c0854..d571e5c98 100644 --- a/docs/30-build-mcpp.md +++ b/docs/30-build-mcpp.md @@ -110,6 +110,9 @@ is ignored, so diagnostics may be logged freely. | `mcpp:warning=` *(2026.8.21.2+)* | say something to the user and **keep going**. The one directive that changes no compile line, no link line and no source set. Survives the build cache — see below | | `mcpp:fact==` *(2026.9.5.2+)* | state something the program **established about the machine** (`cuda.driver=12.4`). Compared against floors before anything is compiled; see below | | `mcpp:floor= >= ` *(2026.9.5.2+)* | state what this package **needs** of that quantity. Unmet ⇒ the build is refused with both values (`version-floor-unmet`); a floor nobody stated a fact for is silent | +| `mcpp:decision=\t\t\t\t\t` *(protocol 15)* | state which tool this plugin runs and where it came from; it joins the build's record of sources and changes nothing the build does | +| `mcpp:xpkg-request=:` *(protocol 15)* | ask for a payload declared `provision = "on-request"`. Every request of one invocation is installed together and the program runs again; the run that asked is discarded, so it must configure nothing else | +| `mcpp:toolchain==` *(protocol 15)* | state the build toolchain, in the toolchain phase of a root program whose manifest says `configure = "build.mcpp"`. The keys are a `[toolchain]` table's (`spec`, `path`, `prefix`, `sysroot`, `family`, `launcher`, `tool.`, `origin`) | | `mcpp:rerun-if-changed=` | re-run `build.mcpp` when this file changes | | `mcpp:rerun-if-env-changed=` | re-run `build.mcpp` when this env var changes | @@ -178,6 +181,11 @@ int main() { | `mcpp::link_script(p)` *(2026.8.19+)* | `mcpp:link-script=` | | `mcpp::runner(tok)` *(2026.8.19.2+)* | `mcpp:runner=` — see below | | `mcpp::xpkg_dir(ns, name)` / `mcpp::xpkg_dir(name)` *(2026.8.19+)* | the payload directory of a package declared in `[xlings.workspace]` — by this manifest, or by a dependency compiled into this build program *(2026.9.6.6+)*; `""` when it was not declared or is not installed (see below) | +| `mcpp::xpkg_source(ns, name)` / `mcpp::xpkg_program(ns, name)` *(protocol 15)* | where that payload comes from — `payload`, `override`, `pending`, or `""` — and the program an override named | +| `mcpp::xpkg_request(ns, name)` / `mcpp::xpkg_pending()` *(protocol 15)* | the directory of a payload declared `provision = "on-request"`, asking for it when it is not installed yet; `xpkg_pending()` is then true and the program should return (see below) | +| `mcpp::phase()` *(protocol 15)* | `"toolchain"` while a root program states the build toolchain, `"build"` otherwise | +| `mcpp::decision(subject, from, value, file, line, payload)` *(protocol 15)* | `mcpp:decision=` — record where a tool came from. `mcpp.plugins.tool` states it for every member | +| `mcpp::toolchain(key, value)` *(protocol 15)* | `mcpp:toolchain=` — state the build toolchain in the toolchain phase. `mcpp.plugins.toolchain` builds the statement | | `mcpp::warning(text)` *(2026.8.21.2+)* | `mcpp:warning=` — see below | | `mcpp::action{…}.submit()` *(2026.8.5.1+)* | `mcpp:action=` — declares a **build-graph node** instead of doing the work here (see below) | @@ -450,6 +458,49 @@ filled from `[xlings.workspace]` alone, so `xpkg_dir` returned `""` for a payload that was on disk. The only sensible thing a program can print then is "declare this package", naming a declaration its author had already written. +### A payload installed when the program asks: `xpkg_request` (protocol 15) + +A payload a plugin needs for only some builds is declared +`provision = "on-request"` (docs/23) and asked for here: + +```cpp +const char* dir = mcpp::xpkg_request("xim", "cmake"); +if (mcpp::xpkg_pending()) return 0; // the engine installs it and runs this again +``` + +`xpkg_request` answers like `xpkg_dir` when the payload is installed or +overridden. When it is not, it asks the engine for it and answers `""`: every +request of that invocation is installed in one batch, and the programs that +asked run again with the directories. The run that asked is **discarded** — so a +program must return without configuring what needs the tool, and anything it +printed is not applied. + +A program that names its own tool never calls this, which is what makes a build +that states its own cmake download none. + +Under `mcpp emit build-database` nothing is installed: the plan records a +`MCPP_BUILD_DATABASE_PAYLOAD_DEFERRED` note and describes the rest. + +### Stating the build toolchain: the toolchain phase (protocol 15) + +A root program whose manifest says `[toolchain] default = { configure = +"build.mcpp" }` runs twice. In the first run `mcpp::phase()` is `"toolchain"`, +and the program states the toolchain the project is built with: + +```cpp +if (std::string_view(mcpp::phase()) == "toolchain") { + mcpp::toolchain("path", "/opt/acme-llvm"); + mcpp::toolchain("origin", "build.mcpp:7"); // the line a build reports + return 0; +} +``` + +That phase runs before the dependency graph is resolved, so the toolchain is +the only thing it may state: a flag, a source or an action there is refused by +name. `mcpp.plugins.toolchain` (mcpp:plugins, feature `plugins-toolchain`) +builds the statement, including from a vendor SDK's environment script. See +[20 — Toolchain Management](20-toolchains.md). + ### Host tools from a dependency (2026.8.5.1+) Declare the need in `mcpp.toml`, then call it: diff --git a/docs/31-authoring-a-rule-package.md b/docs/31-authoring-a-rule-package.md index d4f925d55..8fbf521af 100644 --- a/docs/31-authoring-a-rule-package.md +++ b/docs/31-authoring-a-rule-package.md @@ -371,6 +371,27 @@ target-conditional declaration does: "xim:android-build-tools" = "" ``` +**One order for where a tool comes from** (2026.10.1.3+). A rule's tool is +resolved in the order every member uses: the build program's own option, the +variable that member has always read, the engine's override +(`[xlings.overrides]`), then the declared payload. A tool the build program +named must not make the payload be installed, so a rule asks for a payload only +in the last step — and a payload only some builds need is declared +`provision = "on-request"` and asked for with `mcpp::xpkg_request` (docs/23, +docs/30). + +`mcpp.plugins.tool` (mcpp:plugins, in `plugins-core`) implements that order, +including the one refusal text that lists every way to name the tool, and +records the answer with `mcpp::decision`, so a build reports the source and +`mcpp why tool ` can answer. A rule that resolves its tool itself owes the +same order and the same record. + +A rule **must not** search `PATH` unless someone said so: a tool found there is +reachable through the build program's own choice +(`mcpp::plugins::tool::on_path()`) or an override, and a rule that falls back to +it on its own has to say which program it used and how to state it (SPEC-007 +R6.2). + **A build must not reach the network, and a wrapped tool may.** Measured on `appimagetool` 1.9.1: it downloads its type-2 runtime stub from a GitHub release on every invocation unless `--runtime-file` names a local copy. A member diff --git a/docs/32-authoring-a-payload.md b/docs/32-authoring-a-payload.md index 657862589..ef89af28d 100644 --- a/docs/32-authoring-a-payload.md +++ b/docs/32-authoring-a-payload.md @@ -22,7 +22,11 @@ rules declare the payloads they drive. After: Everything mcpp installs and does not compile: a compiler, a shader compiler, a device toolkit, an emulator, a probe driver, a prebuilt C library. A project names one in `[xlings.workspace]`, or a rule package names it in -`[feature-xlings.]`, and mcpp provisions it before the build runs. +`[feature-xlings.]`, and mcpp provisions it before the build runs — or, with +`provision = "on-request"`, when a build program asks for it (docs/23). A +project may also state that a payload comes from somewhere else on this machine +(`[xlings.overrides]`), and then it is not installed at all; a recipe is +unaffected either way. A payload lives in `xim-pkgindex` as one Lua file: a `package` table that describes it, and two functions that place and register it. diff --git a/docs/50-machine-output.md b/docs/50-machine-output.md index b8fa80b47..fead53fdd 100644 --- a/docs/50-machine-output.md +++ b/docs/50-machine-output.md @@ -350,6 +350,45 @@ mcpp cache list --format json `data` is `{root, entries[]}`, the same document `--json` prints bare. +### `mcpp.why.sources` — the source of each tool *(mcpp 2026.10.1.3+)* + +``` +mcpp why sources|tool |payload --format json +``` + +It resolves and reports; it does not build. `data` is: + +| field | | +|---|---| +| `topic` | `sources`, `tool` or `payload` | +| `subject` | the name the question narrowed to, empty for `sources` | +| `status` | `ok` or `refused` | +| `reason` | a refusal token, or `none` | +| `sources[]` | one entry per subject | + +Each entry is `{subject, value, class, origin, decidedFor, considered[]}`: + +- `subject` is `toolchain.build`, `toolchain.bootstrap`, `payload::` + or `tool::`; +- `class` is `managed`, `pinned`, `custom`, `program` or `host` — the first two + are the ecosystem's, the rest are a person's or a machine's; +- `origin` is `{kind, file, line, key}`, where `kind` is `default`, `manifest`, + `env`, `config`, `build-program` or `graph`; +- `decidedFor` names the package whose declaration it answers; +- `considered` lists what was consulted and what each answered. + +```jsonc +"sources": [ { "subject": "payload:xim:cmake", "value": "/usr/bin/cmake", + "class": "custom", + "origin": { "kind": "manifest", "file": "mcpp.toml", "line": 22, + "key": "[xlings.overrides]" }, + "decidedFor": "mcpp:plugins", + "considered": ["payload xim:cmake@>=3.31 (not installed: overridden)"] } ] +``` + +Every build writes the same record into +`target///resolution.json` under `sources`. + ### `mcpp.toolchain.list` — the installed toolchains and the targets this host serves ``` @@ -444,6 +483,10 @@ a program classifying the outcome reads `reason`: | `tier-planned` | the row exists in the vocabulary; nothing is wired yet | | `host-cannot-serve` | no payload here, and no dependency supplied the system | | `capability-pin` | the row's toolchain is a capability, not a preference | +| `managed-only` | `--managed-only` met a source that is not the ecosystem's: the message names each one and where it was stated *(2026.10.1.3+)* | +| `payload-override` | an `[xlings.overrides]` entry names a path that does not exist, a version a requirement refuses, or is stated by a dependency *(2026.10.1.3+)* | +| `payload-request` | a build program asked for a payload no manifest of this build declares `provision = "on-request"`, or asked again in three consecutive runs *(2026.10.1.3+)* | +| `local-toolchain` | a toolchain named by path, or stated by a build program's toolchain phase, cannot be used: no driver, a contradicting family, a missing tool or sysroot *(2026.10.1.3+)* | | `convention-unreplaced` | the convention was overridden and nothing replaced it | | `os-mismatch` | the requested and resolved triples name different systems | | `layer-requirement` | a package requires a layer the resolution did not give it | @@ -561,6 +604,7 @@ fails there still fails the build. | `MCPP_GENERATED_FILE_NOT_MATERIALIZED` | warning | a root `[build] generated_files` entry is missing or stale on disk, and the command does not write it | | `MCPP_BUILD_DATABASE_STD_UNIT_UNDESCRIBED` | warning | no standard-library build command names its module source, so that unit is not listed | | `MCPP_BUILD_DATABASE_HOST_TOOL_DEFERRED` | note | a requested host tool is not in the tool store and is not built by the command; the plan names the path it will be published at (2026.9.27.1+; replaces the 2026.9.26.2 warning `MCPP_BUILD_DATABASE_HOST_TOOL_UNBUILT`) | +| `MCPP_BUILD_DATABASE_PAYLOAD_DEFERRED` | note | a build program asked for a payload declared `provision = "on-request"`; planning installs nothing, names the program and the packages, and describes the plan without them (2026.10.1.3+) | | `MCPP_BUILD_DATABASE_PROGRAM_FAILED` | error | a build program failed; its package is described without its directives | | `MCPP_INDEX_REQUIRES_NEWER_MCPP` | note | an index refreshed by this run requires a newer mcpp; the previous copy was kept or restored, or none is usable (2026.9.28.1+; the same notice a terminal run prints as its closing `tip:` line) | diff --git a/docs/specs/build-database.md b/docs/specs/build-database.md index 130bb1661..99a52cc16 100644 --- a/docs/specs/build-database.md +++ b/docs/specs/build-database.md @@ -242,6 +242,7 @@ mcpp 输出的 S1 文档满足 S1 等级 2,不输出 `ide.options`。等级 3 | 1.1 | 2026-09-16 | R5.2 增加离线诊断码 `MCPP_OFFLINE_DOWNLOAD_REQUIRED`;R5.3 的 `network` 按观测列出;新增 R5.4(子进程不继承调用方描述符,xlings 子进程有期限并随 mcpp 结束)(#648)。 | | 1.2 | 2026-09-17 | R3.7 陈述 `arguments` 的每一项是编译器收到的参数,单元 flag 按 SPEC-004 §8 的词列出(#655)。 | | 1.3 | 2026-09-26 | R2.5:`emit` 下构建失败的宿主工具是警告。R3.7:`work-directory` 是输出目录,模块接口单元的 `arguments` 带语言 flag。R3.8:标准库单元的 `provides` 指向 std 缓存中的 BMI,工具链带 `build-id`。R4.1:compile-commands 文档包含标准库单元(S1-12-1)。R5.2:成员各自规划,构建程序失败的包不带其指令地被描述(#699,#702)。 | +| 1.5 | 2026-10-01 | R2.5 的同一规则适用于载荷:构建程序请求了 `provision = "on-request"` 的载荷时,命令不安装,记 note `MCPP_BUILD_DATABASE_PAYLOAD_DEFERRED`,点名程序与包(mcpp#755)。 | | 1.4 | 2026-09-26 | R2.5:命令不构建宿主工具;工具库中没有的工具被推迟,输出说明 `MCPP_BUILD_DATABASE_HOST_TOOL_DEFERRED`,取代 1.3 的警告 `MCPP_BUILD_DATABASE_HOST_TOOL_UNBUILT`(#707)。 | | 1.5 | 2026-09-28 | R3.7:规则声明的设备源不是编译单元,不进入 S1 与 `compile_commands.json`(#724)。新增 R3.12:集合的 `ide.generated` 列出规则生成的文件与目录,给出构建写入的路径与生成它的步骤,S1 0.3.0(#724,Sunrisepeak/mcpp-language-server#28)。R5.1:S1 版本为 0.3.0。R5.2:以构建程序的指令为前提的检查不对其构建程序已失败的包运行,失败路径保留已记录的说明(#724)。 | | 1.6 | 2026-09-29 | 工作区按配置规划,与 `mcpp build` 相同(R2.1、R3.3、R3.4、R5.2):成员共用的包在一个配置中只描述一次;集合名的前缀由 `<成员>/` 改为只在文档描述多个配置时出现的 `<配置>/`;一个配置的规划失败时逐成员规划。R3.5:被选成员的集合按其目标给出 `ide.kind`。R4.1:同一文件与输出一条条目。 | diff --git a/docs/specs/build-plugins.md b/docs/specs/build-plugins.md index 586e31377..417e4421d 100644 --- a/docs/specs/build-plugins.md +++ b/docs/specs/build-plugins.md @@ -4,12 +4,12 @@ |---|---| | 规范编号 | SPEC-007 | | 标题 | 构建插件:配置、施工与校验的分工,运行时与规划期的义务 | -| 状态 | 草案 v0.6 | -| 版本 | 0.6 | -| 最后修改 | 2026-09-28 | +| 状态 | 草案 v0.7 | +| 版本 | 0.7 | +| 最后修改 | 2026-10-01 | | 对应实现 | 逐条标注。未注明版本的「已实现」条款对应 mcpp >= 2026.9.26.1;注明 mcpp#702 的条款对应 mcpp >= 2026.9.26.2;注明 mcpp#707、#708、#709、#711 的条款对应 mcpp >= 2026.9.27.1;注明 mcpp#723 的条款对应 mcpp >= 2026.9.28.1;注明 mcpp 2026.9.28.2 的条款对应该版本;§9 与注明 mcpp#734 的条款对应 mcpp >= 2026.9.28.3 | | 相关设计文档 | `.agents/docs/2026-09-26-compile-database-and-issue-699-design.md`(§5)、`.agents/docs/2026-09-28-ecosystem-design-and-optimisation-plan.md`(WS1、WS3)、`.agents/docs/2026-09-28-build-cost-foreign-toolsets-and-library-surface-design.md`(§3、§4) | -| 相关 issue | mcpp#699、mcpp#701、mcpp#702、mcpp#703、mcpp#707、mcpp#708、mcpp#709、mcpp#711、mcpp#734 | +| 相关 issue | mcpp#699、mcpp#701、mcpp#702、mcpp#703、mcpp#707、mcpp#708、mcpp#709、mcpp#711、mcpp#734、mcpp#755 | | 使用文档 | [docs/30 - build.mcpp](../30-build-mcpp.md)、[docs/31 - 编写规则包](../31-authoring-a-rule-package.md) | 本规范规定构建插件对引擎和对消费方承担的义务,以及引擎为此提供的机制。docs/31 说明怎样编写 @@ -184,7 +184,23 @@ 时以目标轴选择器门控,并在构建程序中用 `mcpp::xpkg_dir` 取得路径。声明**必须**放在查询发生 的包上:`xpkg_dir` 为正在构建的包回答;`host-module` 的声明对编入它的每个构建程序可见 (docs/31)。(**已实现**) -- **R6.2** 插件**禁止**探测宿主路径来寻找工具或 SDK;未声明的依赖不可复现。(作者义务) +- **R6.2** 插件**禁止****未经使用者指定**探测宿主路径来寻找工具或 SDK:未声明且未被指定的 + 依赖不可复现。使用者指定的来源(构建程序的选项、`[xlings.overrides]`、 + `MCPP_XLINGS_OVERRIDE__`、`config.toml`)**可以**是 `PATH` 上的程序,因为那是 + 一次被陈述并被记录的选择。无人指定时回落到 `PATH` 的插件**必须**报告它在用哪一个程序, + 并指出使用者如何陈述它;这种回落**应当**被移除。(作者义务;0.19.0 起 mcpp-plugins 的 + `rules-spirv`、`rules-slang` 以警告保留该回落至 2027-04-01) +- **R6.5** 一个工具的来源**必须**按同一顺序决定,并且**必须**被记录(mcpp#755): + 1. 构建程序陈述的选择;2. 该成员历来读取的环境变量;3. 引擎的覆盖 + (`[xlings.overrides]`);4. 声明的载荷。构建程序陈述了选择时,**禁止**请求该载荷——这 + 正是「构建程序自带工具即不下载」的含义。每一次解析**必须**以 `mcpp::decision` 陈述 + 主体、来源与值,官方通用库 `mcpp.plugins.tool` 实现本条,插件**应当**使用它而不是各自 + 实现。(**已实现**,mcpp 2026.10.1.3;`mcpp.plugins.tool` 为 mcpp-plugins 0.19.0) +- **R6.6** 只有部分构建需要的工具,其载荷**应当**声明 `provision = "on-request"`,并在 + 构建程序中以 `mcpp::xpkg_request` 请求:不需要它的构建因此不下载它。请求了载荷的那次 + 运行**必须**在不配置依赖该工具的任何东西的情况下返回(`mcpp::xpkg_pending()`,或 + `mcpp.plugins.tool` 的 `found::pending()`);引擎安装后会再次运行该程序。(**已实现**, + mcpp 2026.10.1.3) - **R6.3** 插件的某个特性需要本包的程序在构建机器上运行时,**应当**在该特性上声明 `[features.] tools = [""]`,而不是要求每个消费方在依赖边上重复写 `tools`。启用该 特性的消费方得到该工具,与边上写了 `tools` 相同(SPEC-004 §10.2)。(**已实现**,mcpp#709) @@ -228,6 +244,7 @@ | 12 | 2026.9.26.2 | `prepare` 角色、`runtime_search_dir` | | 13 | 2026.9.27.1 | action 的 `env` 与 `cwd` | | 14 | 2026.9.28.3 | `mcpp.core`、构建信息(R9.3)、`mcpp::report`(R9.4) | + | 15 | 2026.10.1.3 | 来源(R6.5、R6.6、R9.9):`xpkg_source`、`xpkg_program`、`xpkg_request`、`xpkg_pending`、`phase`、`decision`、`toolchain` | - **R9.3** 构建信息以事实陈述解析出的工具链,不针对任何外部构建系统:`tool(role)`(这一行的 工具)、`abi_tool(role)`(目标 ABI 的原生工具,MSVC ABI 上为工具集的 `cl`、`link`、`lib`、 @@ -251,6 +268,14 @@ `[workspace.package] mcpp` 为全部成员声明。低于下限的引擎在其他工作之前停止,写出包名、下限、 自身版本与升级命令;只接受 `>=` 形式。(**已实现**) +- **R9.9** 构建程序**可以**陈述构建工具链,仅当该工程的 `[toolchain]` 写了 + `configure = "build.mcpp"`,且仅在工具链阶段(`mcpp::phase()` 为 `"toolchain"`)。该阶段 + 在目标依赖图解析之前运行,**禁止**陈述工具链以外的任何指令:引擎拒绝并点名第一条不属于 + 该阶段的指令。陈述的键与 `[toolchain]` 表的键相同(`spec`、`path`、`prefix`、`sysroot`、 + `family`、`launcher`、`tool.`、`origin`),官方通用库 `mcpp.plugins.toolchain` + 提供构造这些陈述的函数。编译并运行构建程序的是 bootstrap 工具链,因此一个无法使用的 + 自定义工具链**必须**仍能让构建程序运行并报告原因。(**已实现**,mcpp 2026.10.1.3) + ## 10. 变更记录 | 版本 | 日期 | 变更 | @@ -259,5 +284,6 @@ | 0.3 | 2026-09-27 | 随 mcpp 2026.9.27.1:新增 R3.8(action 的 `env` 与 `cwd`,协议 13,mcpp#708);R5.3 改为规划不构建宿主工具、缺失的工具以 note 推迟(mcpp#707);新增 R6.3(特性的 `tools`,mcpp#709)与 R6.4(`artifacts` 与 `${mcpp.artifact:}`,mcpp#711)。 | | 0.4 | 2026-09-28 | 随 mcpp 2026.9.28.1:R4.2 同一目标的多个来源在放置时按内容核对,相同则放置一份,不同则失败并点名全部来源;R4.3 一个目标一个写入者,链接后的放置不覆盖另一写入者放在程序旁的文件(mcpp#723)。 | | 0.5 | 2026-09-28 | 随 mcpp 2026.9.28.2:R4.3 的部署清单是运行时放置解析器的答案(SPEC-006 §3.7.1),MSVC C++ 运行时按集合规则决定,`prepare` 填充的目录中的运行时名字由同一解析器决定;新增 R4.5,构建边在成功时的通告,构建后报告一次(2026-09-28 设计 WS1、WS3)。 | +| 0.7 | 2026-10-01 | 随 mcpp 2026.10.1.3(mcpp#755):R6.2 改为禁止**未经指定**的宿主探测并要求报告回落;新增 R6.5(工具来源的顺序与记录)、R6.6(`provision = "on-request"` 与 `xpkg_request`)、R9.9(构建程序在工具链阶段陈述构建工具链);R9.2 的协议表新增第 15 行。 | | 0.6 | 2026-09-28 | 随 mcpp 2026.9.28.3:新增 §9(mcpp#734),即 `mcpp.core` 与 `mcpp` 的永久等价、接口的稳定性与协议表、构建信息、结构化诊断、批量放置、插件模块的名字、缺失模块指出 feature、包的版本下限;R7.1 的版本写在清单中;变更记录移为 §10。 | | 0.2 | 2026-09-26 | 随 mcpp 2026.9.26.2 落地:R1.3 的警告、R2.1 的 `runtime_search_dir`、R2.4、R3.3 的 `prepare`(目录须含文件;链接边等待所有 `prepare`)、R3.5、R3.6、R4.1、R4.3、R5.2、R5.3 标为已实现。 | diff --git a/docs/specs/manifest-semantics.md b/docs/specs/manifest-semantics.md index 0606bbffa..785af838b 100644 --- a/docs/specs/manifest-semantics.md +++ b/docs/specs/manifest-semantics.md @@ -301,6 +301,35 @@ feature-deps feature-xlings ← 限定词是门 **状态:已实现**(mcpp 2026.9.27.1,mcpp#704)。 +### 4.7 载荷的来源与供给时机 + +一条载荷声明陈述**要哪个包**;它从哪里来,以及什么时候装,是另外两个问题。 + +**`[xlings.overrides]` 陈述来源。** 键是 §4.5 的身份 `[:]`,**禁止**带版本; +值是一个路径,或一个表,表中**必须**恰好写 `program`(一个文件,或一个在 `PATH` 上查找 +的程序名)或 `root`(与载荷同布局的目录)之一,并可写 `version`。 + +- 被覆盖的包**禁止**被供给:它不进入安装列表,不参与离线判定。 +- 它仍参与 §4.5 的校验:写了 `version` 时,实现**必须**按该版本校验每一条落败的要求, + 不满足则拒绝并点出两侧;没写时**必须**记一条 note,点出未被校验的要求。 +- `mcpp::xpkg_dir` **必须**回答覆盖所指的 root,`mcpp::xpkg_program` 回答它所指的程序, + `mcpp::xpkg_source` 回答 `override`。 +- 覆盖**只**由一次构建的**根**陈述:根清单、环境变量 `MCPP_XLINGS_OVERRIDE__` + 或 `config.toml`,优先级依此顺序由高到低。依赖写它**必须**被拒绝,拒绝消息点出该包与 + 正确的位置。 + +**`provision = "on-request"` 陈述供给时机。** 条目表接受它与 `version`、`when` 并列; +缺省为 `eager`,即构建程序运行前供给。标为 `on-request` 的包: + +- 在所有声明它的 manifest 都这样写时,构建程序运行前**禁止**被供给; +- 构建程序以 `mcpp::xpkg_request` 请求它;实现**必须**把一次运行中的全部请求合为一次 + 安装,并只重跑请求过的那些程序,丢弃它们该次运行的其余陈述; +- 规划(`mcpp emit build-database`)**禁止**因此安装任何东西,**必须**记一条 + `MCPP_BUILD_DATABASE_PAYLOAD_DEFERRED` note; +- 请求一个没有任何 manifest 如此声明的包**必须**被拒绝。 + +**状态:已实现**(mcpp 2026.10.1.3,mcpp#755)。 + ## 5. 命名规约 ### 5.1 两种 case,按面划分 diff --git a/docs/specs/toolchain-management.md b/docs/specs/toolchain-management.md index e74a3fb19..d90e3bf2a 100644 --- a/docs/specs/toolchain-management.md +++ b/docs/specs/toolchain-management.md @@ -4,11 +4,11 @@ |---|---| | 规范编号 | SPEC-006 | | 标题 | 工具链管理:身份、来源、选择与载荷契约 | -| 状态 | 草案 v0.4 | -| 最后修改 | 2026-09-28 | +| 状态 | 草案 v0.5 | +| 最后修改 | 2026-10-01 | | 对应实现 | 逐条标注;标为「已实现」的条款对应 mcpp >= 2026.9.24.1。标为「未实现」的条款计划与下一批 LLVM 工具链一同落地,届时按实测修订本规范 | | 相关设计文档 | `.agents/docs/2026-09-24-toolchain-selection-and-payload-trust-design.md`、`.agents/docs/2026-09-24-685-687-msvc-stl-and-toolchain-payloads.md`、`.agents/docs/2026-09-28-ecosystem-design-and-optimisation-plan.md` | -| 相关 issue | mcpp#685、mcpp#687、mcpp#718 | +| 相关 issue | mcpp#685、mcpp#687、mcpp#718、mcpp#755 | | 使用文档 | [docs/20 - 工具链](../zh/20-toolchains.md)、[docs/32 - 编写载荷](../zh/32-authoring-a-payload.md)、[docs/91 - 工具链内部](../zh/91-toolchain-internals.md) | 本规范定义 mcpp 对工具链的命名、选择和使用方式,以及一个工具链载荷在发布前必须满足的条件。 @@ -42,6 +42,29 @@ 只有 `msvc` 有系统来源,写作 `msvc@system`。其他族写 `@system`,**必须**在读取处被拒绝。 不带族的 `system`(PATH 上的编译器)**必须**被拒绝,拒绝信息给出可用的写法。 +### 2.2.1 按路径命名 已实现 + +工具链**可以**写成一个表,由路径命名本机已有的一份: +`[toolchain] <键> = { path = "<目录>", prefix, sysroot, family, launcher, tools }`, +或 `MCPP_TOOLCHAIN=path:<目录>`。`<目录>/bin` 中的驱动决定族(`clang++` 为 llvm, +`g++` 为 gcc);其余属性由探测该驱动得到。 + +这与 §2.2 拒绝 `system` 并不矛盾:`system` 是「`PATH` 上碰巧有什么就用什么」,而本条是 +一次被命名、被识别、被记录的选择,与 `msvc@system` 同形。实现**必须**: + +- 以与托管载荷相同的 link model、hermetic 检查与 `import std` 能力判定驱动它; +- **禁止**写入该目录树(它不是 mcpp 管理的); +- 把驱动与 `tools` 所列每个程序的身份(路径、大小、修改时间)纳入指纹与快速路径的判定, + 因为这样的工具链可以原地改变; +- 在 `mcpp.lock` 中记为 `local`,即不可移植; +- 在构建输出、`resolution.json` 与 `mcpp why toolchain` 中陈述其来源(§3.3)。 + +`[toolchain] bootstrap = "<族>@<版本>"` 命名编译并运行构建程序的工具链;未写时按 §3 的 +规则决定。`[toolchain] <键> = { configure = "build.mcpp" }` 把构建工具链的选择交给根构建 +程序的工具链阶段(SPEC-007 R9.9)。 + +**状态:已实现**(mcpp 2026.10.1.3,mcpp#755)。 + ### 2.3 生态包前缀 已实现 `xim:<族>@<版本>` 表示只取生态包。对没有系统来源的族,它与不带前缀的写法等价,规范写法去掉前缀; @@ -81,6 +104,12 @@ clang 以 MSVC ABI 为目标时,所选 toolset 与 SDK 以 `-Xmicrosoft-visualc- ### 3.3 结果可见 已实现 凡由探测得到的选择,其来源、版本与 SDK **必须**打印在构建输出中,写入 `resolution.json`,并进入缓存键。 + +不是生态缺省的来源(按路径命名的工具链、被覆盖的载荷、使用者指定的工具、宿主上找到的 +程序)**必须**各自以一行陈述,写出它是什么、从哪里来、以及陈述它的位置;`Finished` 行 +**必须**汇总它们;`resolution.json` 的 `sources` 记录每一条。全部来源都是生态缺省时,输出 +**必须**与没有本机制时相同。`--managed-only`(或 `MCPP_MANAGED_ONLY`)**必须**拒绝任何 +非缺省来源,并逐条点名(mcpp#755)。 MSVC ABI 目标上:SDK 以 `ucrt@<版本>` 进入运行时身份;clang 行的 toolset 与 SDK 写入 `resolution.json` 的 `msvc_toolset` 与 `windows_sdk`,toolset 目录与 SDK 版本进入缓存键,`stdlibVersion` 记为 toolset 的版本。 @@ -286,4 +315,5 @@ xim-pkgindex 的准入脚本 `verify-toolchain.sh` 对一个载荷归档做一 | v0.1 | 2026-09-24 | 初版草案:身份与写法、来源与选择(含 MSVC ABI 目标的 sysroot)、载荷契约、构建、验收、发布顺序 | | v0.2 | 2026-09-24 | 随 mcpp 2026.9.24.1 更新实现状态:§2.3、§2.4、§3.1 至 §3.6 已实现;§4.2、§6.4 部分实现;§2.2 更正:不带族的 `system` 被拒绝 | | v0.3 | 2026-09-28 | 随 mcpp 2026.9.28.1:新增 §3.7,MSVC ABI 的 CRT 模型是目标 ABI 的性质,cl 与 clang++ 同样收到,默认 `toolchain-coupled`(mcpp#718)。 | +| v0.5 | 2026-10-01 | 随 mcpp 2026.10.1.3(mcpp#755):新增 §2.2.1,工具链可由路径命名,并说明 `bootstrap` 与 `configure = "build.mcpp"`;§3.3 增加非缺省来源的陈述、汇总、记录与 `--managed-only`。 | | v0.4 | 2026-09-28 | 随 mcpp 2026.9.28.2:新增 §3.7.1,程序旁的文件由一个解析器决定;MSVC C++ 运行时是一个带版本的集合;契约决定种类;声明的运行时文件与 toolset 的版本比较;读不出的版本不作决定;action 的 `PATH` 首位是 toolset 的运行时目录(2026-09-28 设计 WS1)。 | diff --git a/docs/zh/09-commands-by-scenario.md b/docs/zh/09-commands-by-scenario.md index a47dc3e24..dec07775f 100644 --- a/docs/zh/09-commands-by-scenario.md +++ b/docs/zh/09-commands-by-scenario.md @@ -121,7 +121,20 @@ mcpp pack --toolchain llvm@22.1.8 --format dir $ mcpp why toolchain toolchain: gcc 16.1.0 (x86_64-linux-gnu) abi(libc)=glibc cxxstdlib=libstdc++ arch=x86_64 os=linux triple=x86_64-linux-gnu - reason: [toolchain] in mcpp.toml if set, else platform-native default + source: pinned · [toolchain] + reason: [toolchain] in mcpp.toml +``` + +`mcpp why sources` 报告一次构建用到的每个工具从哪里来（2026.10.1.3+）：工具链、每个载荷， +以及每个插件运行的工具，连同为它查过什么。`mcpp why tool ` 与 +`mcpp why payload ` 收窄到其中一个： + +``` +$ mcpp why payload cmake +sources: + payload:xim:cmake /usr/bin/cmake + custom · mcpp.toml:22 for mcpp:plugins + considered: payload xim:cmake@>=3.31 (not installed: overridden) ``` `mcpp why deps` 在 `mcpp.lock` 各行之前列出解析出的依赖图（2026.9.14.2+）: @@ -535,8 +548,8 @@ side_effect = false ## 当前边界 -- `mcpp why --format json` 只对 `toolchain` 话题有定义。其余话题报 - `'' has no machine-readable shape yet` 并以非零退出。 +- `mcpp why --format json` 对 `toolchain`、`sources`、`tool` 与 `payload` 话题有定义。 + 其余话题报 `'' has no machine-readable shape yet` 并以非零退出。 - `mcpp search` 按子串匹配；没有字段选择器，也没有把搜索限定到单个命名 空间的办法。 - `mcpp clean --stale` 读 `target/.build_cache`，它保存的近期条目数量 diff --git a/docs/zh/20-toolchains.md b/docs/zh/20-toolchains.md index a77e652cc..863953cd6 100644 --- a/docs/zh/20-toolchains.md +++ b/docs/zh/20-toolchains.md @@ -348,6 +348,93 @@ mcpp 对宿主依赖的规则，并不是各条轴统一的，这个分叉是刻 闭包会被拒绝，因为产物根本起不来。见 [二进制分发](12-binary-distribution.md)。 +### `{ path = … }` —— 本机已有的工具链（2026.10.1.3+） + +一个不是 mcpp 安装的工具链 —— 自己构建的 LLVM、厂商的交叉工具链、手工解包的发行包 —— +由路径命名： + +```toml +[toolchain] +default = { path = "/opt/llvm-trunk" } + +# 驱动与工具带前缀的交叉工具链，自带 sysroot，并点名一个不在布局位置上的工具： +[toolchain.linux] +path = "/opt/acme-gcc" +prefix = "aarch64-none-linux-gnu-" +sysroot = "/opt/acme-sysroot" +family = "gcc" # 与驱动核对 +launcher = "ccache" # 前置于每次编译 +tools = { ld = "/opt/lld-dev/bin/ld.lld" } # cc、cxx、ld、ar、ranlib、nm、objcopy、strip、as +``` + +或者只对一次构建生效，不改清单： + +```bash +MCPP_TOOLCHAIN=path:/opt/llvm-trunk mcpp build +``` + +**布局。** 驱动在 `/bin`：`clang++`（llvm）或 `g++`（gcc），有 `prefix` 时带上它。 +族由其中哪一个存在决定，版本、目标三元组、标准库以及是否支持 `import std` 都由探测该驱动 +得到。旁边的工具按 ``、`llvm-`、`` 查找；`tools` 点名树里没有的 +那些。 + +**mcpp 如何驱动它。** 与它自己安装的载荷相同：自己的链接行、自己的 hermetic 检查、自己 +对 `import std` 的判定。它不向该目录树写入任何东西 —— 托管载荷里生成的 `clang++.cfg` 是 +mcpp 自己的文件，而一棵不属于 mcpp 的树不会得到一份。 + +**身份。** 驱动与 `tools` 点名的每个程序以路径、大小与修改时间进入构建指纹，因此原地重建 +工具链会重建它产出的东西。用过它的构建在 `mcpp.lock` 中记为 `local`：没有这个工具链的 +机器会被点名告知，而不是改用另一个构建。 + +这不是上一节的 `system` 编译器。`system` 是「`PATH` 上碰巧有什么」，而这里是工程点名的 +一棵树，由 mcpp 识别并记录，与 `msvc@system` 一直以来的形状相同。 + +### `bootstrap` —— 构建构建程序的工具链（2026.10.1.3+） + +```toml +[toolchain] +default = { path = "/opt/llvm-trunk" } +bootstrap = "llvm@22.1.8" +``` + +构建程序（`build.mcpp`）、宿主工具与宿主模块在执行构建的那台机器上编译并运行。 +`bootstrap` 命名做这件事的工具链 —— 当它不应当是构建工程的那一个时，例如一个正在开发的 +工具链，它必须能在不妨碍选择它的那个程序运行的前提下失败。不写 `bootstrap` 时行为不变： +原生构建用自己的工具链，交叉构建解析一个宿主的。 + +### `{ configure = "build.mcpp" }` —— 由构建程序陈述工具链（2026.10.1.3+） + +清单表达不了的工具链 —— 来自多处的部件、厂商 SDK 的环境脚本、看过机器之后才定下的版本 +—— 由根构建程序陈述： + +```toml +[toolchain] +default = { configure = "build.mcpp" } +``` + +```cpp +import mcpp; +import mcpp.plugins.toolchain; // mcpp:plugins，feature plugins-toolchain +namespace tc = mcpp::plugins::toolchain; + +int main() { + if (tc::configure([] { + auto d = tc::layout(tc::env("ACME_LLVM", "/opt/acme-llvm")); + tc::use(tc::with_launcher(d, "ccache")); + })) + return 0; + // 普通的构建阶段 +} +``` + +该程序运行两次。它的**工具链阶段**先运行，由 bootstrap 工具链编译，在目标依赖图解析之前； +那里 `mcpp::phase()` 为 `"toolchain"`，而它唯一可以陈述的就是工具链 —— 该阶段中的一条 +flag、一个源文件或一个 action 会被拒绝并点名该指令。随后它的**构建阶段**像任何构建程序 +一样运行，用它刚陈述的工具链。 + +键就是 `[toolchain]` 表接受的那些（托管工具链用 `spec`，或 `path`、`prefix`、`sysroot`、 +`family`、`launcher`、`tool.`），因此清单能说的与程序能说的是同一套词汇。 + ### `msvc@system` —— 机器自己的 Visual Studio mcpp 只负责定位并识别已安装的 Visual Studio / Build Tools，**从不**安装、 @@ -665,6 +752,49 @@ mcpp build --target aarch64-ios # resolves llvm@22.1.8 + the iPhoneOS SDK mcpp build --target aarch64-ios-sim # resolves llvm@22.1.8 + the Simulator SDK ``` +## 每个工具的来源（2026.10.1.3+） + +一次构建用到一个工具链、它的插件声明的载荷，以及那些插件运行的工具。每一项都有一个来源， +而全部来源都是生态缺省的构建，输出与没有本机制时逐字相同。 + +| 类 | 含义 | +|---|---| +| `managed` | 生态缺省：什么都没写 | +| `pinned` | 清单或机器默认选定了版本的托管载荷或工具链 | +| `custom` | 写在 `mcpp.toml`、环境变量或 `config.toml` 里的路径 | +| `program` | 由构建程序决定 | +| `host` | 在 `PATH` 上找到，版本无人陈述 | + +除前两类之外，每一项各占一行，写出它是什么、从哪里来、以及陈述它的那一处： + +``` + Resolving toolchain + Bootstrap llvm@22.1.8 → @mcpp/registry/data/xpkgs/xim-x-llvm/22.1.8/bin/clang++ + Using toolchain clang 23.0.0git ← /opt/acme-llvm [program · build.mcpp:9] + Target x86_64-unknown-linux-gnu + Using xim:cmake ← /usr/bin/cmake [custom · mcpp.toml:22] + Compiling app v0.1.0 (.) + Finished dev [unoptimized + debuginfo] in 2.51s · custom: xim:cmake; program: toolchain +``` + +标签承载全部陈述，因此没有颜色时这一行读起来一样。插件从一个被覆盖的载荷取得的工具，与 +那条覆盖是同一次陈述，只报告一次。 + +`mcpp why sources` 列出每一个来源以及为它查过什么；`mcpp why tool ` 与 +`mcpp why payload ` 收窄到其中一个。`--format json` 以 `mcpp.why.sources` 回答 +同一件事，每次构建也把该记录写入 +`target///resolution.json`。 + +### `--managed-only` —— 拒绝生态之外的来源 + +```bash +mcpp build --managed-only # 或 MCPP_MANAGED_ONLY=1 +``` + +工具链、载荷或插件工具来自别处的构建会被拒绝，并逐条点名它是在哪里被陈述的。这是一次 +发布构建或一次可复现性审计所要的；在它之下快速路径会让行，因为来源由它跳过的那次解析 +决定。 + ## mcpp 保留的宿主面，以及每一项的理由 这个引擎围绕的那条规矩是：**一次构建可复现，当且仅当造出它的工具来自下一台 @@ -918,6 +1048,9 @@ mcpp 的运行行为可以通过下列环境变量调整： | `MCPP_OFFLINE=1` | 完全不访问网络，等价于全局 `--offline` | | `MCPP_NO_COLOR=1` / `NO_COLOR=1` | 禁用彩色输出 | | `MCPP_LOG_LEVEL=debug\|info\|warn\|error\|off` | 日志级别 | +| `MCPP_TOOLCHAIN=path:<目录>` | 本次构建使用该目录中的工具链 | +| `MCPP_XLINGS_OVERRIDE__` | 该载荷从哪里来（`path:<名字>` 在 PATH 上查找） | +| `MCPP_MANAGED_ONLY=1` | 拒绝来源不是生态缺省的构建 | 未显式设置 `MCPP_HOME` 时，mcpp 会基于二进制所在目录的上一级路径自动定位 沙盒（一份 release tarball 解压到 `~/.mcpp/` 之后，`~/.mcpp/` 就是 home）， diff --git a/docs/zh/23-the-project-environment.md b/docs/zh/23-the-project-environment.md index 89334f5a0..c1a5944c2 100644 --- a/docs/zh/23-the-project-environment.md +++ b/docs/zh/23-the-project-environment.md @@ -356,6 +356,59 @@ hardware = {} 它不为任何人激活，也不安装任何东西，而一个工具的缺席若只表现为 「设备永远不可达」，那是最难诊断的一类问题。 +### `provision = "on-request"` —— 构建程序请求时才安装（2026.10.1.3+） + +一个条目可以在版本与层之外说明它何时被安装： + +```toml +[feature-xlings.deps-cmake] +"xim:cmake" = { version = ">=3.31", provision = "on-request" } +``` + +缺省的 `eager` 在任何构建程序运行前安装它。`on-request` 则在构建程序以 +`mcpp::xpkg_request` 请求它时安装 —— 因此一次自带 cmake 的构建，或一次计划根本没走到 +该工具的构建，什么都不下载。一次调用中的全部请求合为一次安装，只有请求过的程序会再 +运行一次。 + +只有当声明它的每一份清单都这样写时，这个包才被推迟：有一份清单需要它立即到位，它就立即 +被安装。 + +`mcpp emit build-database` 不因请求安装任何东西。它记一条 note +（`MCPP_BUILD_DATABASE_PAYLOAD_DEFERRED`）点名程序与包，并在没有它们的情况下描述计划。 + +### `[xlings.overrides]` —— 载荷的来源（2026.10.1.3+） + +一个已声明的载荷可以来自本机的别处： + +```toml +[xlings.overrides] +"xim:cmake" = "/usr/bin/cmake" # 一个文件：程序本身 +"xim:vcpkg" = "/opt/vcpkg" # 一个目录：与载荷同布局的 root +"xim:slang" = { program = "slangc" } # 一个名字：在 PATH 上查找 +"xim:glslang" = { program = "/usr/bin/glslangValidator", version = "15.1.0" } +``` + +被覆盖的包**不会被安装**：它离开供给列表，`--offline` 也因此不会因它而拒绝。 +`mcpp::xpkg_dir` 回答该条目所指的 root，`mcpp::xpkg_program` 回答它点名的程序， +`mcpp::xpkg_source` 回答 `override`。 + +三个位置可以写覆盖，由高到低： + +| | 位置 | 面向 | +|---|---|---| +| 1 | `MCPP_XLINGS_OVERRIDE__` | CI 与发行版打包，无需改清单。`path:cmake` 在 PATH 上查找该名字 | +| 2 | 根清单的 `[xlings.overrides]`，也可写在 `[target.'cfg(..)']` 下 | 工程自己的陈述 | +| 3 | `~/.mcpp/config.toml` 的 `[xlings.overrides]` | 关于这台机器的事实 | + +条目里的 `version` 会与依赖图中每一条要求比较，低于其中任一条则拒绝并点出两侧。没写 +`version` 时无从比较，构建会说明哪一条要求未被校验。 + +只有正在被构建的工程可以写覆盖。依赖写了这张表会被拒绝并点名该包：一个包需要哪些载荷 +是它自己的陈述，它们从哪里来是工程的陈述。 + +构建会报告它用到的每一条覆盖，`--managed-only` 则拒绝用到任何一条的构建：见 +[20 —— 工具链管理](20-toolchains.md)。 + ### 规则包自带它的环境（2026.9.6.6+） 上面这张表，是工程有主张时要写的内容。大多数工程没有主张， @@ -469,6 +522,7 @@ error: `xim:cuda-nvcc` is pinned to 12.0.0 by this project, and mcpp:plugins ## 当前边界 +- 覆盖陈述的是载荷从哪里来，不是要哪一个：键不带版本，包的身份仍由各处声明决定。 - **工具不能以加速器为条件。** 加速器在依赖图之后才解析，因此这样的工具 会被声明、却永远不会被安装——一次成功的构建里那个工具干脆缺席。 写了这样一条的 manifest 会被拒绝。 diff --git a/docs/zh/30-build-mcpp.md b/docs/zh/30-build-mcpp.md index a2347f4c9..8751c6f58 100644 --- a/docs/zh/30-build-mcpp.md +++ b/docs/zh/30-build-mcpp.md @@ -102,6 +102,9 @@ mcpp build # compiles + runs build.mcpp, then builds the project | `mcpp:warning=` *(2026.8.21.2+)* | 对用户说一句话并**继续**。唯一一条不改变编译行、链接行与源码集的指令。它**穿过构建缓存** —— 见下 | | `mcpp:fact==` *(2026.9.5.2+)* | 陈述程序**测得的机器事实**(`cuda.driver=12.4`)。在编译任何东西之前与 floor 比较；见下 | | `mcpp:floor= >= ` *(2026.9.5.2+)* | 陈述本包对该量的**下界**。不满足 ⇒ 构建被拒并给出两侧取值（`version-floor-unmet`）；没有人陈述事实的下界保持沉默 | +| `mcpp:decision=<主体>\t<来源>\t<值>\t<文件>\t<行>\t<载荷>` *(协议 15)* | 陈述这个插件运行哪个工具、它从哪里来；它进入构建的来源记录，不改变构建做的任何事 | +| `mcpp:xpkg-request=:` *(协议 15)* | 请求一个声明了 `provision = "on-request"` 的载荷。一次调用中的全部请求合为一次安装，该程序随后再运行一次；发出请求的那次运行被丢弃，因此它不得配置别的东西 | +| `mcpp:toolchain=<键>=<值>` *(协议 15)* | 陈述构建工具链，只在清单写了 `configure = "build.mcpp"` 的根程序的工具链阶段中。键就是 `[toolchain]` 表的那些（`spec`、`path`、`prefix`、`sysroot`、`family`、`launcher`、`tool.`、`origin`） | | `mcpp:rerun-if-changed=` | 该文件变化时重跑 `build.mcpp` | | `mcpp:rerun-if-env-changed=` | 该环境变量变化时重跑 `build.mcpp` | @@ -162,6 +165,11 @@ int main() { | `mcpp::link_script(p)` *(2026.8.19+)* | `mcpp:link-script=` | | `mcpp::runner(tok)` *(2026.8.19.2+)* | `mcpp:runner=` —— 见下 | | `mcpp::xpkg_dir(ns, name)` / `mcpp::xpkg_dir(name)` *(2026.8.19+)* | `[xlings.workspace]` 里声明的包的载荷目录 —— 本 manifest 声明的，或编进本构建程序的某个依赖声明的（2026.9.6.6+）；没声明或没安装时返回 `""`（见下） | +| `mcpp::xpkg_source(ns, name)` / `mcpp::xpkg_program(ns, name)` *(协议 15)* | 该载荷从哪里来 —— `payload`、`override`、`pending` 或 `""` —— 以及一条覆盖点名的程序 | +| `mcpp::xpkg_request(ns, name)` / `mcpp::xpkg_pending()` *(协议 15)* | 一个声明了 `provision = "on-request"` 的载荷的目录；尚未安装时请求它，此时 `xpkg_pending()` 为真，程序应当返回（见下） | +| `mcpp::phase()` *(协议 15)* | 根程序陈述构建工具链时为 `"toolchain"`，其余为 `"build"` | +| `mcpp::decision(subject, from, value, file, line, payload)` *(协议 15)* | `mcpp:decision=` —— 记录一个工具从哪里来。`mcpp.plugins.tool` 为每个成员陈述它 | +| `mcpp::toolchain(key, value)` *(协议 15)* | `mcpp:toolchain=` —— 在工具链阶段陈述构建工具链。`mcpp.plugins.toolchain` 构造该陈述 | | `mcpp::warning(text)` *(2026.8.21.2+)* | `mcpp:warning=` —— 见下 | | `mcpp::action{…}.submit()` *(2026.8.5.1+)* | `mcpp:action=` —— **声明一个构建图节点**，而不是在这里把活干了（见下） | @@ -384,6 +392,44 @@ store 内部结构 —— 与 `dep_dir` 存在的理由相同。 `[xlings.workspace]` 填充，于是载荷明明在盘上，`xpkg_dir` 却返回 `""`。这时构建程序 唯一说得出口的话是「请声明这个包」，而它指的那条声明作者早已写下。 +### 构建程序请求时才安装的载荷：`xpkg_request`（协议 15） + +一个插件只在部分构建中需要的载荷，声明为 `provision = "on-request"`（docs/23），并在这里 +请求： + +```cpp +const char* dir = mcpp::xpkg_request("xim", "cmake"); +if (mcpp::xpkg_pending()) return 0; // 引擎安装它，然后再次运行本程序 +``` + +载荷已安装或已被覆盖时，`xpkg_request` 的回答与 `xpkg_dir` 相同。否则它向引擎请求该载荷 +并回答 `""`：该次调用中的全部请求合为一次安装，发出过请求的程序带着目录再运行一次。发出 +请求的那次运行会被**丢弃** —— 因此程序必须在不配置任何依赖该工具的东西的情况下返回，它 +打印的其余内容不会被应用。 + +自带工具的程序从不调用它，这正是「工程自己点名 cmake 就不下载」的实现方式。 + +在 `mcpp emit build-database` 下不安装任何东西：计划记一条 +`MCPP_BUILD_DATABASE_PAYLOAD_DEFERRED` note，并描述其余部分。 + +### 陈述构建工具链：工具链阶段（协议 15） + +清单写了 `[toolchain] default = { configure = "build.mcpp" }` 的根程序运行两次。第一次 +`mcpp::phase()` 为 `"toolchain"`，程序在那里陈述构建这个工程所用的工具链： + +```cpp +if (std::string_view(mcpp::phase()) == "toolchain") { + mcpp::toolchain("path", "/opt/acme-llvm"); + mcpp::toolchain("origin", "build.mcpp:7"); // 构建报告的那一行 + return 0; +} +``` + +该阶段在目标依赖图解析之前运行，因此工具链是它唯一可以陈述的东西：那里的一条 flag、一个 +源文件或一个 action 会被按名拒绝。`mcpp.plugins.toolchain`（mcpp:plugins，feature +`plugins-toolchain`）构造该陈述，包括从厂商 SDK 的环境脚本构造。见 +[20 —— 工具链管理](20-toolchains.md)。 + ### 依赖产出的 host 工具（2026.8.5.1+） 在 `mcpp.toml` 里声明需求，然后调用它： diff --git a/docs/zh/31-authoring-a-rule-package.md b/docs/zh/31-authoring-a-rule-package.md index 0efc25328..f7efc7364 100644 --- a/docs/zh/31-authoring-a-rule-package.md +++ b/docs/zh/31-authoring-a-rule-package.md @@ -340,6 +340,20 @@ host module 是这条规则的例外，它的声明在它被编入的**每一个 "xim:android-build-tools" = "" ``` +**一个工具从哪里来，只有一个顺序**（2026.10.1.3+）。一个规则的工具按每个成员共用的顺序 +解析：构建程序自己的选项、该成员历来读取的变量、引擎的覆盖（`[xlings.overrides]`）、然后 +是声明的载荷。构建程序点名了工具时，该载荷**不得**因此被安装，所以规则只在最后一步请求 +载荷 —— 而只有部分构建需要的载荷声明为 `provision = "on-request"`，并以 +`mcpp::xpkg_request` 请求（docs/23、docs/30）。 + +`mcpp.plugins.tool`（mcpp:plugins，在 `plugins-core` 中）实现这个顺序，包括那一份列出所有 +点名方式的拒绝文案，并以 `mcpp::decision` 记录答案，因此构建会报告来源， +`mcpp why tool ` 也能回答。自己解析工具的规则，欠的是同一个顺序与同一份记录。 + +规则**禁止**在无人指定时搜索 `PATH`：在那里找到的工具可以经由构建程序自己的选择 +（`mcpp::plugins::tool::on_path()`）或一条覆盖到达，而自行回落到它的规则必须说出它用了 +哪一个程序、以及如何陈述它（SPEC-007 R6.2）。 + **构建不许碰网络，而被包起来的工具可能会碰。** 在 `appimagetool` 1.9.1 上 实测：除非用 `--runtime-file` 指定一份本地副本，它每次被调用都会从一个 GitHub release 下载它的 type-2 runtime 存根。包装这类工具的成员必须从 diff --git a/docs/zh/32-authoring-a-payload.md b/docs/zh/32-authoring-a-payload.md index 6cd788fd6..d441f98f1 100644 --- a/docs/zh/32-authoring-a-payload.md +++ b/docs/zh/32-authoring-a-payload.md @@ -20,7 +20,9 @@ 一切由 mcpp 安装而不编译的东西：编译器、着色器编译器、设备工具包、 模拟器、探针驱动、预编译的 C 库。工程在 `[xlings.workspace]` 中点名它， 或者规则包在 `[feature-xlings.]` 中点名它，mcpp 会在构建运行之前把它 -供给到位。 +供给到位 —— 或者，写了 `provision = "on-request"` 时，在构建程序请求它时 +供给（docs/23）。工程也可以陈述某个载荷来自本机的别处 +（`[xlings.overrides]`），那时它根本不会被安装；无论哪一种，配方都不受影响。 一个载荷在 `xim-pkgindex` 中是一个 Lua 文件：一个描述它的 `package` 表，加两个把它放好并登记的函数。 diff --git a/docs/zh/50-machine-output.md b/docs/zh/50-machine-output.md index 80279f329..9a8a4c2fb 100644 --- a/docs/zh/50-machine-output.md +++ b/docs/zh/50-machine-output.md @@ -311,6 +311,45 @@ mcpp cache list --format json `data` 是 `{root, entries[]}`，与 `--json` 裸打印出来的一致。 +### `mcpp.why.sources` —— 每个工具的来源 *(mcpp 2026.10.1.3+)* + +``` +mcpp why sources|tool |payload --format json +``` + +它解析并报告，不构建。`data` 为： + +| 字段 | | +|---|---| +| `topic` | `sources`、`tool` 或 `payload` | +| `subject` | 问题收窄到的名字，`sources` 下为空 | +| `status` | `ok` 或 `refused` | +| `reason` | 一个拒绝 token，或 `none` | +| `sources[]` | 每个主体一条 | + +每条为 `{subject, value, class, origin, decidedFor, considered[]}`： + +- `subject` 是 `toolchain.build`、`toolchain.bootstrap`、`payload::` + 或 `tool::`； +- `class` 是 `managed`、`pinned`、`custom`、`program` 或 `host` —— 前两者属于生态， + 其余属于某个人或某台机器； +- `origin` 是 `{kind, file, line, key}`，其中 `kind` 为 `default`、`manifest`、`env`、 + `config`、`build-program` 或 `graph`； +- `decidedFor` 点名它所回应的那个包的声明； +- `considered` 列出查过什么以及各自答了什么。 + +```jsonc +"sources": [ { "subject": "payload:xim:cmake", "value": "/usr/bin/cmake", + "class": "custom", + "origin": { "kind": "manifest", "file": "mcpp.toml", "line": 22, + "key": "[xlings.overrides]" }, + "decidedFor": "mcpp:plugins", + "considered": ["payload xim:cmake@>=3.31 (not installed: overridden)"] } ] +``` + +每次构建把同一份记录写入 +`target///resolution.json` 的 `sources`。 + ### `mcpp.toolchain.list` —— 已安装的工具链，以及这台宿主能服务的目标 ``` @@ -398,6 +437,10 @@ replaced}` —— `origin` 与构建的状态行使用的是同一句话 | `tier-planned` | 词表里存在这一行，但还没有任何东西接线 | | `host-cannot-serve` | 本机没有载荷，也没有依赖供给这个系统 | | `capability-pin` | 这一行的工具链是一项能力陈述，不是一个偏好 | +| `managed-only` | `--managed-only` 遇到了不属于生态的来源：消息逐条点名它与陈述它的位置 *(2026.10.1.3+)* | +| `payload-override` | 一条 `[xlings.overrides]` 点名的路径不存在、版本被某条要求拒绝，或它由依赖写出 *(2026.10.1.3+)* | +| `payload-request` | 构建程序请求了本次构建中没有任何清单声明 `provision = "on-request"` 的载荷，或连续三次运行都在请求 *(2026.10.1.3+)* | +| `local-toolchain` | 由路径命名的工具链，或构建程序工具链阶段陈述的工具链无法使用：没有驱动、族与驱动矛盾、缺少某个工具或 sysroot *(2026.10.1.3+)* | | `convention-unreplaced` | 约定被推翻了，而没有任何东西接替它 | | `os-mismatch` | 请求的三元组与解析出的三元组命名不同的系统 | | `layer-requirement` | 某个包要求的层，解析结果没有提供 | @@ -501,6 +544,7 @@ mcpp emit build-database [--spec s1|compile-commands] --format json | `MCPP_GENERATED_FILE_NOT_MATERIALIZED` | 警告 | 根包 `[build] generated_files` 中的某个文件缺失或内容已过期，命令不写这个文件 | | `MCPP_BUILD_DATABASE_STD_UNIT_UNDESCRIBED` | 警告 | 没有任何标准库构建命令点名它的模块源文件，该单元因此不被列出 | | `MCPP_BUILD_DATABASE_HOST_TOOL_DEFERRED` | 说明 | 被请求的宿主工具不在工具库中，命令不构建它；计划给出它将被发布到的路径（2026.9.27.1+；取代 2026.9.26.2 的警告 `MCPP_BUILD_DATABASE_HOST_TOOL_UNBUILT`） | +| `MCPP_BUILD_DATABASE_PAYLOAD_DEFERRED` | 说明 | 构建程序请求了一个声明 `provision = "on-request"` 的载荷；规划不安装任何东西，点名该程序与这些包，并在没有它们的情况下描述计划（2026.10.1.3+） | | `MCPP_BUILD_DATABASE_PROGRAM_FAILED` | 错误 | 构建程序失败；它所属的包被描述为不含它产生的指令 | | `MCPP_INDEX_REQUIRES_NEWER_MCPP` | 说明 | 本次运行刷新的某个索引要求更新的 mcpp；先前的副本被保留或恢复，或者没有可用的副本（2026.9.28.1+；终端运行以结尾的 `tip:` 行打印同一条说明） | diff --git a/mcpp.toml b/mcpp.toml index b4e40fc06..7bd71af1a 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -1,6 +1,6 @@ [package] name = "mcpp" -version = "2026.10.1.2" +version = "2026.10.1.3" description = "Modern C++ build & package management tool" license = "Apache-2.0" authors = ["mcpp-community"] diff --git a/modules/buildmcpp/src/directives.cppm b/modules/buildmcpp/src/directives.cppm index cfa60adf3..279dc3f1e 100644 --- a/modules/buildmcpp/src/directives.cppm +++ b/modules/buildmcpp/src/directives.cppm @@ -187,6 +187,15 @@ enum class Slot : std::size_t { // `--strict` for a degradation) and is replayed on a cache hit as a // warning is. Diagnostics, + // mcpp#755, protocol 15: SOURCES. A plugin's tool decision + // (`\t\t\t\t`), a payload a build + // program asks to have installed (`:`), and the root build + // program's statement of the build toolchain (`=`, one line + // per key, read only in the toolchain phase). None reaches a compile or + // link line; prepare reads each after the run. + ToolDecisions, + XpkgRequests, + ToolchainStatement, Count }; inline constexpr std::size_t kSlotCount = static_cast(Slot::Count); @@ -285,7 +294,7 @@ struct Def { int sinceProtocol; }; -inline constexpr std::array kTable{{ +inline constexpr std::array kTable{{ // wire tag slot scope transform must missingPrefix missingSuffix since {"cxxflag", "cxxflag", Slot::CxxFlags, Scope::PackagePrivate, Transform::Verbatim, false, "", "", 1}, {"cflag", "cflag", Slot::CFlags, Scope::PackagePrivate, Transform::Verbatim, false, "", "", 1}, @@ -463,6 +472,17 @@ inline constexpr std::array kTable{{ // the entry is still correct. An older engine reading a newer entry // already discards the whole record through the unknown-tag path. {"runtime-search-dir", "runtime-search-dir", Slot::RuntimeSearchDir, Scope::LinkGlobal, Transform::AbsPath, false, "", "", 12}, + // v15 (mcpp#755): sources. `decision` is ADVISORY -- it changes no build + // input and reaches the user through the decision record -- and is + // persisted so a cached run reports the same sources a fresh one did. + // `xpkg-request` and `toolchain` are CLAIMS prepare acts on after the run. + // A request is never replayed in practice: a run that asked for a payload + // returns before its result is cached, and the run after the installation + // receives the directory instead of asking. kCacheEpoch is not bumped, for + // the reason `runtime-search-dir` states above. + {"decision", "decision", Slot::ToolDecisions, Scope::Advisory, Transform::Verbatim, false, "", "", 15}, + {"xpkg-request", "xpkg-request", Slot::XpkgRequests, Scope::Claim, Transform::Verbatim, false, "", "", 15}, + {"toolchain", "toolchain", Slot::ToolchainStatement, Scope::Claim, Transform::Verbatim, false, "", "", 15}, }}; // ── Collected output of one run ──────────────────────────────────────────── @@ -1061,6 +1081,13 @@ void apply(mcpp::manifest::Manifest& m, const Directives& d) { for (auto const& f : d.at(Slot::PackFormats)) bc.packFormats.push_back(f); + // Sources (mcpp#755). Carried, not interpreted: prepare turns decisions + // into the decision record, requests into one installation, and the + // toolchain statement into the build toolchain. + for (auto const& v : d.at(Slot::ToolDecisions)) bc.toolDecisions.push_back(v); + for (auto const& v : d.at(Slot::XpkgRequests)) bc.xpkgRequests.push_back(v); + for (auto const& v : d.at(Slot::ToolchainStatement)) bc.toolchainStatement.push_back(v); + // A named executable's subsystem and entry. `target_directive_error` has // refused every value that names no executable, so the conditions below // only keep this function total. #622 A3: `is_program()`, not `Binary` diff --git a/modules/buildmcpp/src/program_protocol.cppm b/modules/buildmcpp/src/program_protocol.cppm index 92bdd5fe9..0a792074b 100644 --- a/modules/buildmcpp/src/program_protocol.cppm +++ b/modules/buildmcpp/src/program_protocol.cppm @@ -111,7 +111,14 @@ export namespace mcpp::build::program_protocol { // together with the batched placement and structured diagnostics of the same // release. No directive of v13 changes spelling, so a program that uses none of // these serialises to the bytes it did under v13. -inline constexpr int kProtocolVersion = 14; +// v15 (mcpp#755): sources. `xpkg_program`, `xpkg_source` and `xpkg_request` +// answer where a declared payload comes from (an `[xlings.overrides]` entry, +// the payload, or "pending" for one declared `provision = "on-request"` that +// is not installed yet); `mcpp:xpkg-request=` asks for such a payload, +// `mcpp:decision=` records which tool a plugin runs, `mcpp:toolchain=` states +// the build toolchain from the root's toolchain phase, and `mcpp::phase()` +// says which phase is running. No directive of v14 changes spelling. +inline constexpr int kProtocolVersion = 15; // ── Cache-format epoch ───────────────────────────────────────────────────── // diff --git a/modules/manifest/src/toml.cppm b/modules/manifest/src/toml.cppm index 6ca25db54..9c8bfbb3e 100644 --- a/modules/manifest/src/toml.cppm +++ b/modules/manifest/src/toml.cppm @@ -338,6 +338,128 @@ inline XlingsEntry parse_address(std::string_view address) { return XlingsEntry{ ns, target, std::string(version) }; } +// The identity an `[xlings.overrides]` key names: `:`, the namespace +// defaulted to `xim` -- the identity mcpp.xlings.address_set gives a package, +// restated here because the manifest reader cannot import it. +inline std::string override_package_key(std::string_view key) { + const auto e = parse_address(key); + return (e.ns.empty() ? std::string("xim") : e.ns) + ":" + e.target; +} + +// One `[xlings.overrides]` entry (mcpp#755). The value is a path, or a table +// naming exactly one of `program` and `root`, with an optional `version`. +// +// A VERSION IN THE KEY IS REFUSED. The key names the package whose source is +// being stated; a version there would read as "override only this version", +// which is not a statement the override can keep -- one version of a package +// is used per build, and the entry replaces where it comes from. +inline std::expected, std::string> +read_xlings_override(std::string_view key, const mcpp::libs::toml::Value& val) { + const auto entry = parse_address(key); + if (entry.target.empty()) return std::unexpected(std::string("names no package")); + if (!entry.version.empty()) + return std::unexpected(std::format( + "the key names a version ('{}'); an override states where the " + "package comes from, so write the package alone and put the version " + "the program has in `version`", entry.version)); + XlingsOverride o; + o.key = std::string(key); + o.line = static_cast(val.position.line); + if (val.is_string()) { + o.value = val.as_string(); + } else if (val.is_table()) { + const auto& t = val.as_table(); + for (auto const& [k, v] : t) { + if (k != "program" && k != "root" && k != "version") + return std::unexpected(std::format( + "unknown key '{}'; an override is a path, or a table of " + "`program` or `root` and an optional `version`", k)); + if (!v.is_string()) + return std::unexpected(std::format("'{}' must be a string", k)); + } + auto prog = t.find("program"); + auto root = t.find("root"); + if ((prog == t.end()) == (root == t.end())) + return std::unexpected(std::string( + "an override names exactly one of `program` (a file, or a " + "program name looked up on PATH) and `root` (a directory laid " + "out like the payload)")); + o.kind = prog != t.end() ? XlingsOverride::Kind::Program : XlingsOverride::Kind::Root; + o.value = (prog != t.end() ? prog : root)->second.as_string(); + if (auto v = t.find("version"); v != t.end()) o.version = v->second.as_string(); + } else { + return std::unexpected(std::string( + "expected a path, or a table of `program` or `root` and `version`")); + } + if (o.value.empty()) return std::unexpected(std::string("the path is empty")); + return std::pair{ override_package_key(key), std::move(o) }; +} + +// One `[toolchain]` table (mcpp#755): a toolchain named by path, or left to +// the root build program. Every key is optional except `path`, and `configure` +// excludes the rest -- a table that both names a toolchain and hands the +// choice to the build program states two answers to one question. +inline std::expected +read_local_toolchain(const mcpp::libs::toml::Value& val) { + LocalToolchain lt; + lt.line = static_cast(val.position.line); + const auto& t = val.as_table(); + for (auto const& [k, v] : t) { + if (k == "tools") { + if (!v.is_table()) + return std::unexpected(std::string( + "`tools` must be a table of role = program, e.g. " + "`tools = { ld = \"/opt/lld/bin/ld.lld\" }`")); + for (auto const& [role, prog] : v.as_table()) { + if (!prog.is_string()) + return std::unexpected(std::format("tools.{} must be a string", role)); + static constexpr std::string_view kRoles[] = { + "cc", "cxx", "ld", "ar", "ranlib", "nm", "objcopy", "strip", "as", + }; + if (std::ranges::find(kRoles, std::string_view(role)) == std::end(kRoles)) + return std::unexpected(std::format( + "tools.{} is not a tool role; the roles are cc, cxx, ld, " + "ar, ranlib, nm, objcopy, strip and as", role)); + lt.tools.emplace_back(role, prog.as_string()); + } + continue; + } + if (!v.is_string()) + return std::unexpected(std::format("'{}' must be a string", k)); + const auto& sv = v.as_string(); + if (k == "path") lt.path = sv; + else if (k == "prefix") lt.prefix = sv; + else if (k == "sysroot") lt.sysroot = sv; + else if (k == "launcher") lt.launcher = sv; + else if (k == "family") { + if (sv != "gcc" && sv != "llvm") + return std::unexpected(std::format( + "family = '{}': a toolchain named by path is \"gcc\" or \"llvm\"", sv)); + lt.family = sv; + } else if (k == "configure") { + if (sv != "build.mcpp") + return std::unexpected(std::format( + "configure = '{}': the one value is \"build.mcpp\", which " + "hands the choice to the root build program's toolchain phase", sv)); + lt.configure = true; + } else { + return std::unexpected(std::format( + "unknown key '{}'; a toolchain table has `path`, `prefix`, " + "`sysroot`, `family`, `launcher` and `tools`, or `configure`", k)); + } + } + if (lt.configure && t.size() != 1) + return std::unexpected(std::string( + "`configure = \"build.mcpp\"` hands the choice to the build program, " + "so the table names nothing else; the build program states the " + "toolchain with the same keys")); + if (!lt.configure && lt.path.empty()) + return std::unexpected(std::string( + "a toolchain table names its root with `path`, or hands the choice " + "to the build program with `configure = \"build.mcpp\"`")); + return lt; +} + // An entry's value, split into the part that names a version and the tier that // part belongs to. // @@ -356,6 +478,9 @@ inline XlingsEntry parse_address(std::string_view address) { struct WhenSplit { const mcpp::libs::toml::Value* value = nullptr; // string, or platform table ToolWhen when = ToolWhen::Always; + // `provision = "on-request"` (mcpp#755): installed when a build program + // asks for it, not before the program runs. `"eager"` is the default. + bool onRequest = false; }; inline std::expected parse_when(std::string_view w) { @@ -374,7 +499,8 @@ split_when(const mcpp::libs::toml::Value& v) { const auto& t = v.as_table(); auto itVer = t.find("version"); auto itWhen = t.find("when"); - if (itVer == t.end() && itWhen == t.end()) + auto itProv = t.find("provision"); + if (itVer == t.end() && itWhen == t.end() && itProv == t.end()) return WhenSplit{ &v, ToolWhen::Always }; // a platform table // A SCOPED ENTRY MUST NAME ITS VERSION KEY EVEN TO LEAVE IT EMPTY. // `{ when = "run" }` alone reads as "present, unconstrained, run tier", @@ -382,10 +508,10 @@ split_when(const mcpp::libs::toml::Value& v) { // the two would be indistinguishable. The key is required, `""` says // unconstrained, and a table carrying anything else is refused by name. for (auto const& [k, _] : t) - if (k != "version" && k != "when") + if (k != "version" && k != "when" && k != "provision") return std::unexpected(std::format( - "unknown key '{}' in a scoped entry; expected 'version' and " - "'when'", k)); + "unknown key '{}' in a scoped entry; expected 'version', " + "'when' and 'provision'", k)); if (itVer == t.end()) return std::unexpected(std::string( "a scoped entry needs 'version' (write version = \"\" for " @@ -398,6 +524,17 @@ split_when(const mcpp::libs::toml::Value& v) { if (!w) return std::unexpected(w.error()); out.when = *w; } + if (itProv != t.end()) { + if (!itProv->second.is_string()) + return std::unexpected(std::string("'provision' must be a string")); + const auto& p = itProv->second.as_string(); + if (p == "on-request") out.onRequest = true; + else if (p != "eager") + return std::unexpected(std::format( + "provision = '{}' is not a provisioning mode; expected " + "'on-request' (installed when a build program asks for it) or " + "'eager' (the default: installed before build programs run)", p)); + } return out; } @@ -2384,13 +2521,39 @@ std::expected parse_string(std::string_view content, } // [toolchain] — platform → "pkg@version" map (docs/21) + // + // An entry is a managed spec (`"gcc@16.1.0"`) or a table (mcpp#755): a + // toolchain named by path, or `configure = "build.mcpp"`. The table is + // recorded in the string map as `path:` / `configure:build.mcpp`, so a + // reader that only needs the spelling reads one string; the rest of the + // table rides `localByPlatform` under the same key. `bootstrap` is not a + // platform: it names the toolchain that builds the build programs. if (auto* tt = doc->get_table("toolchain")) { for (auto& [platform, val] : *tt) { + if (platform == "bootstrap") { + if (!val.is_string()) + return std::unexpected(error(origin, + "[toolchain].bootstrap must be a managed spec like \"llvm@22.1.8\"")); + m.toolchain.bootstrap = val.as_string(); + m.toolchain.bootstrapLine = static_cast(val.position.line); + continue; + } + if (val.is_table()) { + auto lt = read_local_toolchain(val); + if (!lt) return std::unexpected(error(origin, + std::format("[toolchain].{}: {}", platform, lt.error()))); + m.toolchain.byPlatform[platform] = lt->configure + ? std::string("configure:build.mcpp") : "path:" + lt->path; + m.toolchain.localByPlatform[platform] = std::move(*lt); + continue; + } if (!val.is_string()) { - return std::unexpected(error(origin, - std::format("[toolchain].{} must be a string like \"gcc@15.1.0\"", platform))); + return std::unexpected(error(origin, std::format( + "[toolchain].{} must be a string like \"gcc@15.1.0\", or a " + "table `{{ path = \"\" }}` naming a toolchain by path", platform))); } m.toolchain.byPlatform[platform] = val.as_string(); + m.toolchain.lineByPlatform[platform] = static_cast(val.position.line); } } @@ -2587,6 +2750,7 @@ std::expected parse_string(std::string_view content, m.xlings.deps.push_back(entry->address()); if (scoped->when != ToolWhen::Always) m.xlings.depWhen[entry->address()] = scoped->when; + if (scoped->onRequest) m.xlings.onRequest.insert(entry->address()); } } // `[feature-xlings.]` — the same table, gated on a feature of the @@ -2643,9 +2807,27 @@ std::expected parse_string(std::string_view content, m.xlings.featurePins[entry->address()] = entry->pin(); if (scoped->when != ToolWhen::Always) m.xlings.depWhen[entry->address()] = scoped->when; + if (scoped->onRequest) m.xlings.onRequest.insert(entry->address()); } } } + // `[xlings.overrides]` (mcpp#755): where a declared payload comes from on + // this machine instead of the registry. Read for every manifest and + // honoured only for the root of a build; a dependency's is refused where + // the root is known. + if (auto* ot = doc->get_table("xlings.overrides")) { + for (auto& [k, val] : *ot) { + auto o = read_xlings_override(k, val); + if (!o) return std::unexpected(error(origin, + std::format("[xlings.overrides] {}: {}", k, o.error()))); + if (auto prev = m.xlings.overrides.find(o->first); + prev != m.xlings.overrides.end()) + return std::unexpected(error(origin, std::format( + "[xlings.overrides] names '{}' twice, as '{}' and as '{}'; " + "write it once", o->first, prev->second.key, k))); + m.xlings.overrides.emplace(std::move(o->first), std::move(o->second)); + } + } if (doc->get("xlings.subos")) { m.xlings.subosDeclared = true; if (auto v = doc->get_string("xlings.subos")) m.xlings.subos = *v; @@ -3943,6 +4125,7 @@ std::expected parse_string(std::string_view content, writtenAs.emplace(entry->target, k); if (scoped->when != ToolWhen::Always) cc.xlings.depWhen[entry->address()] = scoped->when; + if (scoped->onRequest) cc.xlings.onRequest.insert(entry->address()); out.push_back(*entry); } return {}; @@ -3971,6 +4154,20 @@ std::expected parse_string(std::string_view content, } continue; } + if (k == "overrides") { + if (!v.is_table()) + return std::unexpected(error(origin, std::format( + "[target.{}.xlings.overrides] must be a table of " + "package = path", triple))); + for (auto& [ok, ov] : v.as_table()) { + auto o = read_xlings_override(ok, ov); + if (!o) return std::unexpected(error(origin, std::format( + "[target.{}.xlings.overrides] {}: {}", triple, ok, o.error()))); + cc.xlings.overrides.insert_or_assign(std::move(o->first), + std::move(o->second)); + } + continue; + } // `subos` names the project's environment, of which there // is one per project rather than one per target. Refused // rather than ignored: a silently dropped environment is @@ -3989,8 +4186,10 @@ std::expected parse_string(std::string_view content, "target needs is declared under " "[target.{}.xlings.workspace] as `\"\" = " "\"\"`, and is installed only when this target " - "is built. `subos` names the project's environment and " - "belongs in the top-level [xlings].", triple, k, triple))); + "is built; where it comes from instead is " + "[target.{}.xlings.overrides]. `subos` names the " + "project's environment and belongs in the top-level " + "[xlings].", triple, k, triple, triple))); } } if (auto fit = body.find("feature-xlings"); diff --git a/modules/manifest/src/types.cppm b/modules/manifest/src/types.cppm index b89a8b94a..133408303 100644 --- a/modules/manifest/src/types.cppm +++ b/modules/manifest/src/types.cppm @@ -286,8 +286,53 @@ inline std::string library_linkage_problem(std::string_view statementHead, // macos = "llvm@20" // windows = "msvc@system" // default = "gcc@15.1.0" (used when current platform isn't listed) +// A toolchain the project names by PATH, or leaves to its build program +// (`[toolchain] = { ... }`, mcpp#755). The managed spelling is a string; +// this is the table form, kept beside the string map so every reader of a spec +// still reads one string: the table is recorded there as `path:` or +// `configure:build.mcpp`, and this record carries the rest. +// +// THE SAME FIELDS A BUILD PROGRAM STATES in its toolchain phase +// (`mcpp:toolchain==`), so the two entry points share one +// description and the engine reads it in one place. +struct LocalToolchain { + std::string path; // the toolchain's root: `/bin/` + std::string prefix; // a driver and tool prefix (`aarch64-none-linux-gnu-`) + std::string sysroot; // `--sysroot` for the compiler and the linker + std::string family; // "gcc" | "llvm"; empty = decided by the drivers present + std::string launcher; // a compile-command prefix (`ccache`) + std::vector> tools; // role -> program + bool configure = false; // `configure = "build.mcpp"` + int line = 0; // where the table was written, 0 = unknown +}; + struct Toolchain { std::map byPlatform; // platform -> "pkg@ver" + // The table form of an entry, by the same key (mcpp#755). + std::map localByPlatform; + // `[toolchain] bootstrap`: the toolchain that compiles and runs build + // programs when it should differ from the one that builds the project. + std::string bootstrap; + int bootstrapLine = 0; + // Where each string entry was written, for the source a build reports. + std::map lineByPlatform; + + // The table behind the entry `for_platform` answers with, by the same key. + const LocalToolchain* local_for(std::string_view platform) const { + std::string key(platform); + if (!byPlatform.contains(key)) key = "default"; + auto it = localByPlatform.find(key); + return it == localByPlatform.end() ? nullptr : &it->second; + } + int line_for(std::string_view platform) const { + std::string key(platform); + if (!byPlatform.contains(key)) key = "default"; + if (auto it = localByPlatform.find(key); it != localByPlatform.end()) + return it->second.line; + if (auto it = lineByPlatform.find(key); it != lineByPlatform.end()) + return it->second; + return 0; + } // Returns the toolchain spec for a platform, falling back to "default". std::optional for_platform(std::string_view platform) const { @@ -893,6 +938,19 @@ struct BuildConfig : BuildInputs { // notarisation each couple a release to a release mcpp does not control, // and a name here is the whole of what the engine learns. std::vector packFormats; + // mcpp#755, protocol 15. What a build program stated about SOURCES, carried + // to prepare the way every other directive is: folded in here, read back + // after the program ran, replayed with it on a cache hit. + // + // toolDecisions `mcpp:decision=`: which tool a plugin runs and where + // it came from, `\t\t\t\t`. + // xpkgRequests `mcpp:xpkg-request=`: payloads declared + // `provision = "on-request"` that the program asked for. + // toolchainStatement `mcpp:toolchain=`: the build toolchain, `=` + // per line, read only from the root's toolchain phase. + std::vector toolDecisions; + std::vector xpkgRequests; + std::vector toolchainStatement; bool staticStdlib = true; // #336 — the C++ runtime DISTRIBUTION contract: what the artifact promises // about the machine that runs it ("self-contained" | "toolchain-coupled" | @@ -1250,6 +1308,23 @@ inline std::string_view to_string(ToolWhen w) { } } +// `[xlings.overrides]` (mcpp#755): where a declared payload comes from instead +// of the registry. An overridden payload is not provisioned, and `xpkg_dir` +// answers with the root this entry names. +// +// THE VALUE IS RECORDED, NOT RESOLVED. Whether a string names a file or a +// directory, and where a bare program name is on PATH, are facts about the +// machine the build runs on, and the manifest reader does not consult the +// filesystem; prepare resolves them where the answer is reported. +struct XlingsOverride { + enum class Kind { Path, Program, Root }; + Kind kind = Kind::Path; // `Path`: a string, a file or a directory + std::string value; // as written + std::string version; // stated version, empty = not stated + std::string key; // the entry's key as written + int line = 0; // where it was written, 0 = unknown +}; + struct XlingsConfig { // The install addresses `[xlings.workspace]` asks for, resolved for THIS // host: `[:][@]`. Derived rather than authored — the @@ -1296,11 +1371,22 @@ struct XlingsConfig { // explicitly written `subos = "default"` selects NamedSubos("default"). // A string alone cannot distinguish absence from an invalid empty value. bool subosDeclared = false; + // The addresses declared `provision = "on-request"` (mcpp#755): installed + // when a build program asks for them with `xpkg_request`, not before it + // runs. Beside `deps` for the reason `depWhen` is: the materialised + // `.xlings.json` has no such axis. + std::set onRequest; + // `[xlings.overrides]`, keyed by package identity (`:`, the + // namespace defaulted to `xim`). Read only from the ROOT of a build. + std::map overrides; ToolWhen when_of(std::string_view address) const { auto it = depWhen.find(std::string(address)); return it == depWhen.end() ? ToolWhen::Always : it->second; } + bool on_request(std::string_view address) const { + return onRequest.contains(std::string(address)); + } bool empty() const { return deps.empty() && workspace.empty() && featureDeps.empty() @@ -1567,7 +1653,8 @@ inline bool is_empty(const ConditionalConfig& c) { && c.dependencies.empty() && c.devDependencies.empty() && c.buildDependencies.empty() && c.featureDeps.empty() && c.targetKinds.empty() - && c.xlings.empty() && !c.abiThreadsDeclared && !c.abiExceptionsDeclared + && c.xlings.empty() && c.xlings.overrides.empty() + && !c.abiThreadsDeclared && !c.abiExceptionsDeclared && !c.requiresAbiThreads && !c.requiresAbiExceptions && c.featureRequiresAbiThreads.empty() && c.featureRequiresAbiExceptions.empty() // #717: `dialect_cxxflags` is a member of ConditionalConfig, not of diff --git a/modules/toolchain-model/src/linkmodel.cppm b/modules/toolchain-model/src/linkmodel.cppm index fc07195ed..b8d26b1a0 100644 --- a/modules/toolchain-model/src/linkmodel.cppm +++ b/modules/toolchain-model/src/linkmodel.cppm @@ -343,6 +343,17 @@ ClangDriverModel resolve_clang_driver(const Toolchain& tc) { dm.cfgPath = tc.binaryPath.parent_path() / (tc.binaryPath.stem().string() + ".cfg"); dm.hasCfg = std::filesystem::exists(dm.cfgPath); + // A TOOLCHAIN NAMED BY PATH HAS NO GENERATED CFG (mcpp#755). The cfg of a + // managed payload is an OUTPUT of this machinery (docs/91 §5.1), written + // so a direct invocation of the payload's clang works; mcpp's own + // invocations bypass it and state every path themselves. A tree mcpp does + // not own is not written into, so the model is opened by what the cfg + // would have pointed at instead: libc++ beside the driver. + if (!dm.hasCfg && !tc.localRoot.empty()) { + std::error_code ec; + const auto root = tc.binaryPath.parent_path().parent_path(); + dm.hasCfg = std::filesystem::exists(root / "include" / "c++" / "v1", ec); + } if (!dm.hasCfg) return dm; dm.llvmRoot = tc.binaryPath.parent_path().parent_path(); auto libcxxInclude = dm.llvmRoot / "include" / "c++" / "v1"; diff --git a/modules/toolchain-model/src/model.cppm b/modules/toolchain-model/src/model.cppm index 286ca4017..3c02aef82 100644 --- a/modules/toolchain-model/src/model.cppm +++ b/modules/toolchain-model/src/model.cppm @@ -310,6 +310,25 @@ struct Toolchain { // kept finding: "it did not happen" and "it succeeded" producing identical // output. std::string resolutionNote; + // A TOOLCHAIN NAMED BY PATH (mcpp#755): its root, empty for a managed one. + // Set, mcpp drives the compiler as it drives a managed payload (its own + // link line, `--no-default-config` for clang) without writing into the + // tree, which is not mcpp's. + std::filesystem::path localRoot; + // A compile-command prefix (`ccache`), stated with the toolchain. It runs + // the compiler and changes no output, so it is not part of the identity. + std::string launcher; + // Tools stated by role where the layout does not have them + // (`tools = { ld = ... }`): `ld`, `ar`, `ranlib`, `nm`, `objcopy`, + // `strip`, `as`. Read before any derivation from the driver's directory. + std::vector> toolOverrides; + // The driver-and-tool prefix of a cross toolchain (`aarch64-none-linux-gnu-`). + std::string toolPrefix; + + const std::filesystem::path* tool_override(std::string_view role) const { + for (auto const& [r, p] : toolOverrides) if (r == role) return &p; + return nullptr; + } std::string label() const { return std::format("{} {} ({})", compiler_name(), version, targetTriple); diff --git a/modules/versioning/src/version.cppm b/modules/versioning/src/version.cppm index 56694e638..0c24be549 100644 --- a/modules/versioning/src/version.cppm +++ b/modules/versioning/src/version.cppm @@ -31,6 +31,6 @@ import std; export namespace mcpp { -inline constexpr std::string_view MCPP_VERSION = "2026.10.1.2"; +inline constexpr std::string_view MCPP_VERSION = "2026.10.1.3"; } // namespace mcpp diff --git a/src/build/build_program.cppm b/src/build/build_program.cppm index 6336b349a..7f99051a5 100644 --- a/src/build/build_program.cppm +++ b/src/build/build_program.cppm @@ -307,6 +307,19 @@ struct BuildProgramEnv { // instead of reconstructing `/data/xpkgs/-x-/` // — the same reason depDirs exists for mcpp dependencies. std::vector> xpkgDirs; + // THE PHASE THIS RUN BELONGS TO (mcpp#755). Empty for every ordinary run, + // and then `MCPP_PHASE` is not set, so the contract of a program that + // never asks is unchanged. "toolchain" for the root program's toolchain + // phase, whose only accepted statements are the build toolchain, re-run + // keys and messages. + std::string phase; + // A run that asked for a payload (`mcpp:xpkg-request=`) is normally + // DISCARDED: the requests are handed back in `xpkgRequests`, nothing else + // is applied and nothing is cached, because the engine installs the + // payloads and runs the program again. Under `plan_only` no payload is + // installed, so the run is kept -- applied, reported, and still not cached, + // so the next build asks again. + bool keepRequestingRun = false; // #355: HOST tools this package asked its dependencies for, as // (env var name → absolute path to the executable) pairs. The caller has // already resolved them (built, taken from the store, or an override), so @@ -408,7 +421,12 @@ struct BuildProgramEnv { // The env-var name `hostprogram::xpkg_dir` reads back. One spelling of the // sanitizer, shared by both sides — the two drifting apart would make the // interface answer "" for a package that is right there. -inline std::string xpkg_env_var(std::string_view ns, std::string_view name) { +// +// The suffix selects the fact: `DIR` (the payload directory), `SOURCE` and +// `PROGRAM` (where it came from, mcpp#755), read back by `xpkg_source` and +// `xpkg_program`. +inline std::string xpkg_env_var(std::string_view ns, std::string_view name, + std::string_view suffix = "DIR") { std::string out = "MCPP_XPKG_"; auto put = [&](std::string_view s) { for (char c : s) @@ -417,7 +435,8 @@ inline std::string xpkg_env_var(std::string_view ns, std::string_view name) { }; if (!ns.empty()) { put(ns); out += '_'; } put(name); - out += "_DIR"; + out += '_'; + out += suffix; return out; } @@ -847,6 +866,7 @@ contract_env(const fs::path& root, const fs::path& outDir, const BuildProgramEnv auto [it, inserted] = depVarValue.try_emplace(var, dir); if (inserted) e.emplace_back(var, dir); } + if (!env.phase.empty()) e.emplace_back("MCPP_PHASE", env.phase); // #355: MCPP_DEP__BIN_ — absolute path to a host tool the // consumer declared via `tools = [...]`. A PATH rather than a directory: // the store keys an entry per (package, target), the typed reader can @@ -1968,6 +1988,50 @@ std::expected run_build_program_impl( mcpp::ui::warning(std::format( "build.mcpp: ignoring unknown directive 'mcpp:{}'", k)); } + // THE TOOLCHAIN PHASE STATES THE TOOLCHAIN AND NOTHING ELSE (mcpp#755). + // It runs before the dependency graph is resolved, so a flag, a source or + // an action stated here would describe a build that does not exist yet; a + // payload request cannot be answered either, because the payloads are + // declared by a graph that has not been read. Refused rather than + // dropped, naming the first directive that does not belong. + if (env.phase == "toolchain") { + for (auto const& def : dirs::kTable) { + switch (def.slot) { + case Slot::ToolchainStatement: case Slot::RerunFiles: + case Slot::RerunEnv: case Slot::RerunGlobs: + case Slot::Warnings: case Slot::Diagnostics: + continue; + default: break; + } + if (d.at(def.slot).empty()) continue; + return std::unexpected(std::format( + "build.mcpp stated `mcpp:{}=` in its toolchain phase.\n" + " That phase runs before the dependency graph is resolved, and\n" + " it states the build toolchain only (`mcpp::toolchain`,\n" + " `mcpp::plugins::toolchain::use`). Return from main() after\n" + " stating it: `mcpp::phase()` is \"toolchain\" there, and\n" + " `mcpp::plugins::toolchain::configure` returns true.", + def.wire)); + } + } else if (!d.at(Slot::ToolchainStatement).empty()) { + return std::unexpected(std::string( + "build.mcpp stated `mcpp:toolchain=` outside the toolchain phase.\n" + " The build toolchain is stated by the root build program of a\n" + " project whose manifest says\n" + " [toolchain]\n" + " default = { configure = \"build.mcpp\" }\n" + " and only while `mcpp::phase()` is \"toolchain\".")); + } + // A PAYLOAD REQUEST (mcpp#755). The payloads are installed by the caller, + // in one batch with every other program's, and this program runs again + // with their directories -- so this run is discarded: nothing applied, + // nothing cached, and no line of its own (the run that answers reports). + if (!d.at(Slot::XpkgRequests).empty() && !env.keepRequestingRun) { + m.buildConfig.xpkgRequests = d.at(Slot::XpkgRequests); + programReport.reported = true; + return {}; + } + const bool cacheThisRun = d.at(Slot::XpkgRequests).empty(); // Dependency mode (genBase set): relative `generated=` paths resolve // against OUT_DIR-style genBase, not the (possibly read-only, shared) @@ -2004,7 +2068,9 @@ std::expected run_build_program_impl( for (auto const& a : dirs::advisories(m.package.name, d)) mcpp::ui::warning(a); report_stated_diagnostics(m.package.name, d); - write_cache(bdir, root, programHash, compilerHash, ctxHash, d); + // A run kept although it asked for a payload (`plan_only`) is never + // cached: replaying it would keep the build from ever asking again. + if (cacheThisRun) write_cache(bdir, root, programHash, compilerHash, ctxHash, d); return {}; } diff --git a/src/build/execute.cppm b/src/build/execute.cppm index 6f57e40d1..dc1954281 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -1057,6 +1057,7 @@ export int run_build_plan(BuildContext& ctx, bool verbose, bool no_cache, // manifest, which `compute_flags` reads. It once read the declared level // while the compile used another, and said `[optimized]` over `-Og`. The // step record's header carries the same value for the fast path. + mcpp::build::progress::note_sources(ctx.plan.outputDir); mcpp::build::progress::finished( ctx.profile, mcpp::build::profile_descriptor(ctx.plan.manifest.buildConfig)); report_freestanding_size(ctx); @@ -1553,6 +1554,31 @@ bool xlings_payloads_present(const BuildCacheEntry& e) { }); } +// A toolchain named by path (mcpp#755) is unchanged since the entry's build: +// each of its programs has the size and modification time prepare recorded +// beside the build. A managed toolchain never changes in place, and a build +// without the record used none. +bool local_toolchain_unchanged(const std::filesystem::path& outputDir) { + std::ifstream in(outputDir / "local-toolchain.stamp", std::ios::binary); + if (!in) return true; + std::string line; + while (std::getline(in, line)) { + auto t1 = line.find('\t'); + auto t2 = t1 == std::string::npos ? t1 : line.find('\t', t1 + 1); + if (t2 == std::string::npos) return false; + const std::filesystem::path p(line.substr(t2 + 1)); + std::error_code ec; + const auto size = std::filesystem::file_size(p, ec); + if (ec) return false; + const auto time = std::filesystem::last_write_time(p, ec); + if (ec) return false; + if (std::to_string(size) != line.substr(0, t1) + || std::to_string(time.time_since_epoch().count()) != line.substr(t1 + 1, t2 - t1 - 1)) + return false; + } + return true; +} + // Why the project fast path declined, under `-v` (#734 E5). Each refusal names // its condition, so a platform on which the fast path never serves shows which // precondition it fails instead of only the slower build. @@ -1579,6 +1605,10 @@ export std::optional try_fast_build(const std::filesystem::path& projectRoo // builds, audits and CI, none of which are the case the fast path serves. if (mcpp::platform::env::get("MCPP_LOCKED").value_or("") == "1") return fast_path_declined("build", "MCPP_LOCKED=1 asks for a locked resolution"); + // `--managed-only` is checked against the sources prepare decides, which + // a fast path does not run (mcpp#755). + if (auto v = mcpp::platform::env::get("MCPP_MANAGED_ONLY"); v && *v != "0") + return fast_path_declined("build", "MCPP_MANAGED_ONLY asks for the sources to be checked"); auto want = fast_path_identity(projectRoot); if (!want) return fast_path_declined("build", "the manifest or the request could not be read"); @@ -1674,6 +1704,8 @@ export std::optional try_fast_build(const std::filesystem::path& projectRoo if (dep_sources_newer_than(match->depSourceRoots, ninjaTime, want->extTable)) return fast_path_declined("build", "a path dependency's manifest or source is newer than build.ninja"); if (!xlings_payloads_present(*match)) return fast_path_declined("build", "a recorded xlings payload is missing"); + if (!local_toolchain_unchanged(match->outputDir)) + return fast_path_declined("build", "a program of the toolchain named by path changed"); auto validatedBefore = mcpp::build::runtime_validation::validated_artifact_snapshot( @@ -1697,6 +1729,7 @@ export std::optional try_fast_build(const std::filesystem::path& projectRoo // The descriptor the plan recorded (revision 3, §7.3); empty for a record // written before it was carried. + mcpp::build::progress::note_sources(outputDir); mcpp::build::progress::finished(want->profile, mcpp::build::progress::read_descriptor(outputDir)); return 0; @@ -1716,6 +1749,10 @@ export std::optional try_fast_workspace_build( if (no_cache) return fast_path_declined("workspace", "the build cache is off (--no-cache)"); if (mcpp::platform::env::get("MCPP_LOCKED").value_or("") == "1") return fast_path_declined("workspace", "MCPP_LOCKED=1 asks for a locked resolution"); + // `--managed-only` is checked against the sources prepare decides, which + // a fast path does not run (mcpp#755). + if (auto v = mcpp::platform::env::get("MCPP_MANAGED_ONLY"); v && *v != "0") + return fast_path_declined("workspace", "MCPP_MANAGED_ONLY asks for the sources to be checked"); auto join = [](const std::vector& v) { std::string out; for (auto const& x : v) { if (!out.empty()) out += '\x1e'; out += x; } @@ -1778,6 +1815,8 @@ export std::optional try_fast_workspace_build( return fast_path_declined("workspace", "a member's or a path dependency's manifest or source is newer than build.ninja"); if (!xlings_payloads_present(*match)) return fast_path_declined("workspace", "a recorded xlings payload is missing"); + if (!local_toolchain_unchanged(match->outputDir)) + return fast_path_declined("workspace", "a program of the toolchain named by path changed"); auto validated = mcpp::build::runtime_validation::validated_artifact_snapshot( outputDir, *match->runtimeBinding); if (!validated) @@ -1807,6 +1846,7 @@ export std::optional try_fast_workspace_build( if (*rc != 0) return rc; if (!mcpp::build::runtime_validation::artifact_snapshot_unchanged(r.validated)) return fast_path_declined("workspace", "ninja relinked an artifact, whose closure the full path validates"); + mcpp::build::progress::note_sources(r.outputDir); } // The groups share the profile, and so its descriptor. mcpp::build::progress::finished( @@ -1844,6 +1884,10 @@ std::optional try_fast_run(const std::filesystem::path& projectRoot, // `--locked` is an assertion about resolution. if (mcpp::platform::env::get("MCPP_LOCKED").value_or("") == "1") return fast_path_declined("run", "MCPP_LOCKED=1 asks for a locked resolution"); + // `--managed-only` is checked against the sources prepare decides, which + // a fast path does not run (mcpp#755). + if (auto v = mcpp::platform::env::get("MCPP_MANAGED_ONLY"); v && *v != "0") + return fast_path_declined("run", "MCPP_MANAGED_ONLY asks for the sources to be checked"); auto want = fast_path_identity(projectRoot); if (!want) return fast_path_declined("run", "the manifest or the request could not be read"); @@ -1963,6 +2007,8 @@ std::optional try_fast_run(const std::filesystem::path& projectRoot, if (dep_sources_newer_than(match->depSourceRoots, ninjaTime, want->extTable)) return fast_path_declined("run", "a path dependency's manifest or source is newer than build.ninja"); if (!xlings_payloads_present(*match)) return fast_path_declined("run", "a recorded xlings payload is missing"); + if (!local_toolchain_unchanged(match->outputDir)) + return fast_path_declined("run", "a program of the toolchain named by path changed"); auto validatedBefore = mcpp::build::runtime_validation::validated_artifact_snapshot( diff --git a/src/build/flags.cppm b/src/build/flags.cppm index 13a88646e..7c20e245b 100644 --- a/src/build/flags.cppm +++ b/src/build/flags.cppm @@ -65,6 +65,11 @@ struct CompileFlags { // artifact loaded a different build of a library than it linked against. // It reaches the line through `link_line::UnitTail::runtimeFallback`. std::string ldRuntimeFallback; + // A compile-command prefix stated with a toolchain named by path + // (`launcher = "ccache"`, mcpp#755): written in front of the compiler in + // the build graph, and not into the compile database, whose consumers + // want the driver. + std::string launcher; std::filesystem::path cxxBinary; // g++ / clang++ / cl.exe std::filesystem::path ccBinary; // gcc / clang (derived; cl.exe = same) std::filesystem::path arBinary; // ar / llvm-ar / lib.exe (empty → PATH) @@ -693,6 +698,7 @@ CompileFlags compute_flags(const BuildPlan& plan) { targetIsMacos, plan.manifest.buildConfig.macosDeploymentTarget); f.cxxBinary = plan.toolchain.binaryPath; + f.launcher = plan.toolchain.launcher; f.ccBinary = mcpp::toolchain::derive_c_compiler(plan.toolchain); const bool isMsvcDialect = (d.id == "msvc"); @@ -969,6 +975,14 @@ CompileFlags compute_flags(const BuildPlan& plan) { link_toolchain_flags += mcpp::toolchain::ClangDriverModel::kLinkDriverFlags; } + // A LINKER STATED BY ROLE (mcpp#755, `tools = { ld = ... }`): clang's + // `--ld-path` names the program, after `-fuse-ld` has chosen the + // flavour, so the flags that follow still speak to the same linker. + if (auto* ld = plan.toolchain.tool_override("ld")) { + const auto opt = " --ld-path=" + escape_path(*ld); + link_toolchain_flags += opt; + link_toolchain_flags_c += opt; + } f.sysroot = link_toolchain_flags; } else if (lm.mode != mcpp::toolchain::CLibMode::None) { // GCC (or Clang without cfg): --sysroot from probe, or the payload diff --git a/src/build/hostprogram.cppm b/src/build/hostprogram.cppm index 0f773fdfb..fc2aa0da7 100644 --- a/src/build/hostprogram.cppm +++ b/src/build/hostprogram.cppm @@ -799,23 +799,99 @@ inline const char* dep_linkage(const char* name) { // declared. Returns "" when the package was not declared or is not installed — // a caller that needs it should say so itself, because only it knows whether // the absence is fatal. -inline const char* xpkg_dir(const char* ns, const char* name) { +// `MCPP_XPKG___`, spelled as `xpkg_dir` always spelled it. +inline const char* xpkg_value_(const char* ns, const char* name, const char* suffix) { char buf[256] = "MCPP_XPKG_"; unsigned long o = 10; auto put = [&](const char* s) { - for (const char* p = s; *p && o + 6 < sizeof buf; ++p, ++o) { + for (const char* p = s; *p && o + 12 < sizeof buf; ++p, ++o) { char c = *p; buf[o] = (c >= 'a' && c <= 'z') ? char(c - 'a' + 'A') : ((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) ? c : '_'; } }; - if (ns && *ns) { put(ns); if (o + 6 < sizeof buf) buf[o++] = '_'; } + if (ns && *ns) { put(ns); if (o + 12 < sizeof buf) buf[o++] = '_'; } put(name); - buf[o++] = '_'; buf[o++] = 'D'; buf[o++] = 'I'; buf[o++] = 'R'; buf[o] = 0; + buf[o++] = '_'; + for (const char* p = suffix; *p && o + 1 < sizeof buf; ++p) buf[o++] = *p; + buf[o] = 0; return env_or(buf); } +inline const char* xpkg_dir(const char* ns, const char* name) { return xpkg_value_(ns, name, "DIR"); } inline const char* xpkg_dir(const char* name) { return xpkg_dir("", name); } +// WHERE A DECLARED PAYLOAD COMES FROM (protocol 15, mcpp#755). +// +// "payload" installed from the registry; `xpkg_dir` names it +// "override" stated by `[xlings.overrides]`, `MCPP_XLINGS_OVERRIDE__` +// or the global configuration; nothing was installed, and +// `xpkg_dir` names the root the override implies +// "pending" declared `provision = "on-request"` and not installed yet; +// `xpkg_request` asks for it +// "" not declared for this build, or not installed +inline const char* xpkg_source(const char* ns, const char* name) { return xpkg_value_(ns, name, "SOURCE"); } +inline const char* xpkg_source(const char* name) { return xpkg_source("", name); } +// The program an override named, when it named one (`program = "..."`, a +// path that is a file, or a name found on PATH); "" otherwise. A payload, and +// an override that named a root, leave the program to the plugin's layout. +inline const char* xpkg_program(const char* ns, const char* name) { return xpkg_value_(ns, name, "PROGRAM"); } +inline const char* xpkg_program(const char* name) { return xpkg_program("", name); } + +inline bool& xpkg_pending_flag_() { static bool pending = false; return pending; } +// The directory of a payload this program needs, installing it on request. +// +// A payload declared `provision = "on-request"` is not installed before build +// programs run. This answers like `xpkg_dir` when the payload is installed or +// overridden. When it is pending, it asks the engine for it and answers ""; +// `xpkg_pending()` is then true, and the program should return without +// configuring what needs the payload: the engine installs every payload asked +// for in one batch, discards this run, and runs the program again, which then +// receives the directory. A plugin that names its own tool never calls this, +// so its payload is never installed. +inline const char* xpkg_request(const char* ns, const char* name) { + const char* dir = xpkg_dir(ns, name); + if (*dir) return dir; + const char* src = xpkg_source(ns, name); + if (src[0] == 'p' && src[1] == 'e') { // "pending" + std::printf("mcpp:xpkg-request=%s:%s\n", (ns && *ns) ? ns : "xim", name); + xpkg_pending_flag_() = true; + } + return ""; +} +inline bool xpkg_pending() { return xpkg_pending_flag_(); } + +// Which phase is running: "toolchain" while the root build program states the +// build toolchain (`[toolchain] = { configure = "build.mcpp" }`), "build" +// otherwise. A program in the toolchain phase states only the toolchain. +inline const char* phase() { + const char* p = env_or("MCPP_PHASE"); + return *p ? p : "build"; +} + +// THE SOURCE OF A TOOL A PLUGIN RUNS, recorded with the build's other sources. +// +// `subject` names the tool (`tool::`), `from` how it was found +// (`choice` -- named by the build program; `env` -- by an environment variable +// the plugin reads; `override` -- by an engine override; `payload` -- the +// declared payload; `path` -- found on PATH), `value` the program, and +// `file`/`line` the statement that chose it when the build program did, and +// `payload` the declared payload (`:`) the tool stands for, so an +// override or a payload is reported with that payload's own source. Fields +// must not contain a tab or a newline. +inline void decision(const char* subject, const char* from, const char* value, + const char* file = "", unsigned line = 0, const char* payload = "") { + std::printf("mcpp:decision=%s\t%s\t%s\t%s\t%u\t%s\n", subject, from, value, file, + line, payload); +} + +// One key of the build toolchain, stated in the toolchain phase: `spec` (a +// managed spec such as "llvm@23.1.3"), or `path`, `prefix`, `sysroot`, +// `family`, `launcher`, `tool.` -- the keys of a `[toolchain]` table -- +// and `origin` (`:` of the statement). +inline void toolchain(const char* key, const char* value) { + std::printf("mcpp:toolchain=%s=%s\n", key, value); +} + // mcpp#355: absolute path to a HOST tool built by a dependency — the binary // behind one of its `kind = "bin"` targets. Returns "" unless the consumer // declared it: = { version = "…", tools = ["protoc"] } diff --git a/src/build/ninja_backend.cppm b/src/build/ninja_backend.cppm index 3e9dfdf26..fbf3822e1 100644 --- a/src/build/ninja_backend.cppm +++ b/src/build/ninja_backend.cppm @@ -1424,7 +1424,15 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // The macOS initializer-ordering shim (#336) is a C translation unit, so // it needs the C driver bindings even in a project with no .c sources. const bool need_ios_init_shim = flags.needsStreamInitShim; - append(std::format("cxx = {}\n", escape_ninja_path(flags.cxxBinary))); + // The launcher (mcpp#755) prefixes the compiler in every rule that runs it; + // a link through ccache is passed to the compiler unchanged. + const std::string launch = flags.launcher.empty() ? std::string{} + : escape_ninja_path(std::filesystem::path(flags.launcher)) + " "; + append(std::format("cxx = {}{}\n", launch, escape_ninja_path(flags.cxxBinary))); + // The driver alone, for the dependency scan: `clang-scan-deps` reads its + // argv[0] as the compiler (and the resource directory beside it), and a + // launcher there is not one. + append(std::format("cxx_driver = {}\n", escape_ninja_path(flags.cxxBinary))); append(std::format("cxxflags = {}\n", flags.cxx)); // ALWAYS emitted, for the reason `c_ldflags` is, 26 lines below — and the // two are referenced by the SAME rules. mcpp#426 recorded that reasoning @@ -1447,7 +1455,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // static case, which goes to `ar` and produces an empty archive with exit // 0, it does not touch at all. `check_undefined_ninja_variables` below is // what stops the class. - append(std::format("cc = {}\n", escape_ninja_path(flags.ccBinary))); + append(std::format("cc = {}{}\n", launch, escape_ninja_path(flags.ccBinary))); if (need_c_rule || need_ios_init_shim) { append(std::format("cflags = {}\n", flags.cc)); } @@ -2217,7 +2225,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, if (msvcDeps) { // MSVC: compiler-integrated P1689 via /scanDependencies (scan // only — no codegen); /TP because our module units are .cppm. - append(std::format(" command = $cxx{} $cxxflags $unit_cxxflags " + append(std::format(" command = $cxx_driver{} $cxxflags $unit_cxxflags " "/scanDependencies $out /TP /c $in /Fo:$compile_target\n", rsp_ref(scanPayload))); // No $unit_lang here: /TP above already applies to every input, @@ -2226,7 +2234,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // the scan does not do. } else if (plan.scanDepsPath.empty()) { // GCC path: compiler-integrated P1689 scanning. - append(std::format(" command = $cxx{} $cxxflags $unit_cxxflags -fmodules " + append(std::format(" command = $cxx_driver{} $cxxflags $unit_cxxflags -fmodules " "-fdeps-format=p1689r5 " "-fdeps-file=$out -fdeps-target=$deps_target " "-M -MM -MF $out.dep $unit_lang -E $in -o $compile_target\n", @@ -2241,7 +2249,7 @@ std::string emit_ninja_string(const BuildPlan& plan, std::string* placements, // overruns (#261: 48 -I entries at a deep consumer path). append(std::format( " command = $scan_deps -format=p1689 -o $out -- " - "$cxx{} $cxxflags $unit_cxxflags $unit_lang -c $in " + "$cxx_driver{} $cxxflags $unit_cxxflags $unit_lang -c $in " "-o $compile_target\n", rsp_ref(scanPayload))); } diff --git a/src/build/prepare.cppm b/src/build/prepare.cppm index b1dd679ed..4342a38df 100644 --- a/src/build/prepare.cppm +++ b/src/build/prepare.cppm @@ -246,6 +246,7 @@ export enum class TcOrigin { TargetPin, // triple.cppm vocabulary convention GraphRequirement, // `requires = ["mcpp:compiler=…"]` in the graph FirstRun, // chosen and persisted by this very invocation + BuildProgram, // the root build program's toolchain phase (mcpp#755) — user explicit }; // `GlobalDefault` IS DELIBERATELY NOT LISTED, AND THE REASON IS A MEASURED @@ -270,7 +271,8 @@ export enum class TcOrigin { // pin the way the target side itself was deferred — resolve it after the graph, // where the question it answers has an answer. export inline bool tc_origin_is_user_explicit(TcOrigin o) { - return o == TcOrigin::ManifestToolchain || o == TcOrigin::TargetSection; + return o == TcOrigin::ManifestToolchain || o == TcOrigin::TargetSection + || o == TcOrigin::BuildProgram; } // MAY A BUILD THAT RESOLVED THIS WAY WRITE THE MACHINE'S DEFAULT? @@ -302,6 +304,7 @@ export constexpr std::string_view tc_origin_name(TcOrigin o) { case TcOrigin::TargetPin: return "target default"; case TcOrigin::GraphRequirement: return "required by the dependency graph"; case TcOrigin::FirstRun: return "first-run default"; + case TcOrigin::BuildProgram: return "the build program's toolchain phase"; case TcOrigin::None: break; } return {}; @@ -324,6 +327,73 @@ export std::optional parse_cache_mode(std::string_view v); export std::string_view cache_mode_name(CacheMode m); +// ── Sources (mcpp#755) ───────────────────────────────────────────────────── +// +// WHERE EACH THING THIS BUILD USES CAME FROM: the build toolchain, the +// toolchain that runs build programs, every xlings payload, and every tool a +// plugin runs. One record per subject, decided once and read by everything +// that reports it -- the status lines, the `Finished` summary, `mcpp why`, +// `resolution.json` and `--managed-only` -- so no two of them can disagree. +// +// THE CLASS IS A CLOSED VOCABULARY, and the line it draws is the one a reader +// needs: whether the ecosystem chose (managed, pinned) or a person or a +// machine did (custom, program, host). A build whose every source is managed +// or pinned prints exactly what it printed before this record existed. +export enum class SourceClass { + Managed, // the ecosystem's default: nothing was written + Pinned, // a managed payload or toolchain at a version a manifest chose + Custom, // a path stated in mcpp.toml, an environment variable or config.toml + Program, // decided by a build program + Host, // found on PATH or on the host, its version stated by nobody +}; + +export constexpr std::string_view source_class_name(SourceClass c) { + switch (c) { + case SourceClass::Managed: return "managed"; + case SourceClass::Pinned: return "pinned"; + case SourceClass::Custom: return "custom"; + case SourceClass::Program: return "program"; + case SourceClass::Host: return "host"; + } + return "managed"; +} + +export constexpr bool source_class_is_default(SourceClass c) { + return c == SourceClass::Managed || c == SourceClass::Pinned; +} + +export struct SourceDecision { + // `toolchain.build`, `toolchain.bootstrap`, `payload::`, + // `tool::`. + std::string subject; + // What was used: a spec, a directory, a program. + std::string value; + // What it is, when the value is only where it is: `clang 22.1.8` for a + // toolchain named by path. Shown before the value on a `Using` line. + std::string detail; + SourceClass cls = SourceClass::Managed; + // Who said so: `default`, `manifest`, `env`, `config`, `build-program`, + // `graph`; with the file and line, or the variable, that said it. + std::string originKind = "default"; + std::string originFile; + int originLine = 0; + std::string originKey; + // The package whose declaration this answers, when it answers one. + std::string decidedFor; + // The candidates in priority order and what became of each, for `mcpp why`. + std::vector considered; + // A tool that stands for a payload (`:`): the statement that + // chose it is the payload's own when it took an override or the payload, + // and then the payload's line is the one reported. + std::string payload; + bool announce = true; +}; + +// `custom · mcpp.toml:22`, `program · build.mcpp:9`, `host · PATH`: the tag a +// status line and `mcpp why` print after a non-default source. +export std::string source_tag(const SourceDecision& d, + const std::filesystem::path& relativeTo = {}); + // A condition a planning pass reports instead of acting on (plan_only): the // code is stable and the message is for people. Emitted as diagnostics by the // command that asked for the plan, at the severity carried here — most notes @@ -446,6 +516,9 @@ export struct BuildContext { std::string replaced; // the spec displaced, when one was }; CompilerChoice compilerChoice; + // THE SOURCES THIS BUILD USED (mcpp#755), one entry per subject. See + // SourceDecision. + std::vector sources; // Resolved global-cache mode. Read side is honored in prepare_build; write // side in run_build_plan. CacheMode cacheMode = CacheMode::Global; @@ -788,6 +861,17 @@ export struct BuildOverrides { // none of them, and receives no stage at all, which is what lets one run of // its program serve every member. std::map pack_stages; + // ── Sources (mcpp#755) ────────────────────────────────────────────────── + // `--managed-only` / `MCPP_MANAGED_ONLY=1`: refuse a build any of whose + // sources is not the ecosystem's (custom, program or host), naming each. + bool managed_only = false; + // THE TOOLCHAIN THE ROOT BUILD PROGRAM STATED, when this prepare is the + // second pass of a project whose `[toolchain]` says `configure = + // "build.mcpp"`: the `=` lines of its toolchain phase, and the + // spec of the toolchain that ran it. Empty on every other prepare. Set by + // `prepare_build` itself, never by a caller. + std::vector toolchain_statement; + std::string bootstrap_spec; }; // ── git dependency helpers ────────────────────────────────────────────────── diff --git a/src/build/prepare/config.cpp b/src/build/prepare/config.cpp index b7e0ff416..a318c2df7 100644 --- a/src/build/prepare/config.cpp +++ b/src/build/prepare/config.cpp @@ -196,6 +196,12 @@ void merge_conditional_xlings(mcpp::manifest::Manifest& m, // same reason: the address that survived above is the conditional one. for (auto const& [addr, w] : cc.xlings.depWhen) m.xlings.depWhen.insert_or_assign(addr, w); + m.xlings.onRequest.insert(cc.xlings.onRequest.begin(), cc.xlings.onRequest.end()); + // An override written under a selector is the more specific statement of + // where this package comes from for that target, so it replaces the + // top-level one for the same package (mcpp#755). + for (auto const& [pkg, o] : cc.xlings.overrides) + m.xlings.overrides.insert_or_assign(pkg, o); for (auto const& [f, addrs] : cc.xlings.featureDeps) { auto& dst = m.xlings.featureDeps[f]; for (auto const& a : addrs) { diff --git a/src/build/prepare/driver.cpp b/src/build/prepare/driver.cpp index 20b56e054..d4871e4a5 100644 --- a/src/build/prepare/driver.cpp +++ b/src/build/prepare/driver.cpp @@ -22,6 +22,7 @@ import mcpp.build.backend; // BuildOptions for the tool sub-build import mcpp.build.ninja; // make_ninja_backend — driving that sub-build import mcpp.platform; import mcpp.log; +import mcpp.ui; namespace mcpp::build { @@ -41,11 +42,49 @@ std::vector take_notes_on_failure() { return notes; } +namespace { +std::expected +prepare_build_pass(bool print_fingerprint, + bool includeDevDeps, + std::vector extraTargets, + BuildOverrides overrides); +} // namespace + +// TWO PASSES WHEN THE BUILD PROGRAM STATES THE TOOLCHAIN (mcpp#755). +// +// `[toolchain] = { configure = "build.mcpp" }` hands the choice of the +// build toolchain to the root build program. The program needs its host +// modules, and they come from the dependency graph; the graph's resolution +// needs the toolchain (`cfg(compiler = ...)`, `requires`). So the first pass +// prepares with the bootstrap toolchain as far as the host modules, runs the +// program's toolchain phase, and stops; the second prepares with the stated +// toolchain from the start, the bootstrap compiling and running the build +// programs. The first pass narrates nothing: everything it would say, the +// second says about the build that happens. std::expected prepare_build(bool print_fingerprint, bool includeDevDeps, std::vector extraTargets, BuildOverrides overrides) { + if (!overrides.toolchain_statement.empty()) + return prepare_build_pass(print_fingerprint, includeDevDeps, + std::move(extraTargets), std::move(overrides)); + auto first = prepare_build_pass(print_fingerprint, includeDevDeps, extraTargets, overrides); + if (first) { (void)take_toolchain_restart(); return first; } + auto restart = take_toolchain_restart(); + if (!restart) return first; + overrides.toolchain_statement = std::move(restart->first); + overrides.bootstrap_spec = std::move(restart->second); + return prepare_build_pass(print_fingerprint, includeDevDeps, + std::move(extraTargets), std::move(overrides)); +} + +namespace { +std::expected +prepare_build_pass(bool print_fingerprint, + bool includeDevDeps, + std::vector extraTargets, + BuildOverrides overrides) { PrepareState state(print_fingerprint, includeDevDeps, std::move(extraTargets), std::move(overrides)); pending_flag_words_notes().clear(); @@ -84,6 +123,16 @@ prepare_build(bool print_fingerprint, if (auto r = check_engine_floors(state, /*rootOnly=*/true); !r) return fail(r.error()); if (auto r = timed("toolchain request", [&] { return phase1_toolchain_spec_and_axes(state); }); !r) return fail(r.error()); + // The first pass of a toolchain phase says nothing (see prepare_build). + struct QuietPass { + bool active = false, prev = false; + ~QuietPass() { if (active) mcpp::ui::set_quiet(prev); } + } quietPass; + if (state.toolchainConfigure && state.overrides.toolchain_statement.empty()) { + quietPass.active = true; + quietPass.prev = mcpp::ui::is_quiet(); + mcpp::ui::set_quiet(true); + } if (auto r = timed("toolchain resolver", [&] { return phase2_define_toolchain_resolver(state); }); !r) return fail(r.error()); if (auto r = timed("xlings", [&] { return phase3_xlings_before_graph(state); }); !r) @@ -105,6 +154,7 @@ prepare_build(bool print_fingerprint, g_notesOnFailure.clear(); return timed("finish", [&] { return phase13_finish(state); }); } +} // namespace } // namespace mcpp::build diff --git a/src/build/prepare/features.cpp b/src/build/prepare/features.cpp index 762aaf145..74b147d3d 100644 --- a/src/build/prepare/features.cpp +++ b/src/build/prepare/features.cpp @@ -875,6 +875,8 @@ static std::expected step6_xlings_workspace_from_graph(Prepar for (auto const& pkg : state.packages) if (auto why = layer_predicated_xlings_refusal(pkg.manifest)) return std::unexpected(*why); + if (auto why = dependency_override_refusal(state)) + return std::unexpected(*why); auto split = state.graph_xlings_split(); if (!split) { refusal::record(refusal::Code::ToolVersionConflict); @@ -1956,9 +1958,12 @@ static std::expected step6_dependency_build_programs(PrepareS const auto runnerN = bcDep.runner.size(); auto namedBefore = bcDep.namedRunners; // by value: the delta below const bool exclusiveBefore = bcDep.runExclusive; - if (auto r = mcpp::build::run_build_program( - pkg.manifest, pkg.root, host->first, host->second, - pkg.manifest.cppStandard, bpEnv); + if (auto r = run_answering_requests(state, pkg.manifest, bpEnv, i, + pkg.manifest.package.name, [&] { + return mcpp::build::run_build_program( + pkg.manifest, pkg.root, host->first, host->second, + pkg.manifest.cppStandard, bpEnv); + }); !r) { // #699 item 2 (E3): under `emit build-database` (`plan_only`), // a failing build program describes its package without that @@ -2239,6 +2244,10 @@ std::expected phase6_features_and_host_tools(PrepareState& st auto toolRequests = step6_host_module_registration(state); if (!toolRequests) return std::unexpected(toolRequests.error()); + // The root build program's toolchain phase (mcpp#755): its host modules + // are registered, and nothing that depends on the build toolchain has + // been built. A statement ends this pass; prepare starts again with it. + if (auto r = step6_toolchain_phase(state); !r) return std::unexpected(r.error()); if (auto r = step6_provision_host_tools(state, *toolRequests); !r) return std::unexpected(r.error()); diff --git a/src/build/prepare/graph.cpp b/src/build/prepare/graph.cpp index 61d1e70b8..2f9553647 100644 --- a/src/build/prepare/graph.cpp +++ b/src/build/prepare/graph.cpp @@ -2109,77 +2109,13 @@ step4b_define_provisioning_closures(PrepareState& state) { // a NAMED runner inherits it per name, because a board may legitimately // supply `flash` while a different package supplies `monitor`. + // The payload facts a build program receives: where each declared payload + // is, and where it came from (mcpp#755). One definition, in sources.cpp, + // beside the override and on-request rules it reads. state.fillXpkgDirs = [&](mcpp::build::BuildProgramEnv& e, const mcpp::manifest::Manifest& owner, std::size_t consumer) { - // `[feature-xlings.]` is provisioned when `` is active, so it has - // to be answerable here too. Before this, a tool a feature declared was - // downloaded and installed and then `mcpp::xpkg_dir` returned "" for it - // — the build program was told to declare a package it had already - // declared, which is a diagnostic pointing at the wrong file. - // - // The set is taken from the SAME env the caller already computed, so - // "which features are on" is answered once. Installation stays the - // filter below: a declared address whose payload is absent answers "", - // which is what a `when = "dev"` entry looks like to a consumer. - std::vector declared = owner.xlings.deps; - for (auto const& f : e.features) - if (auto it = owner.xlings.featureDeps.find(f); - it != owner.xlings.featureDeps.end()) - for (auto const& address : it->second) - if (std::ranges::find(declared, address) == declared.end()) - declared.push_back(address); - // …and what the rule packages compiled INTO this build program - // declared. Their own active features, not the consumer's: the - // consumer asked for `features = ["rules-cuda"]` on the edge, and that - // is what decides which of the rule's `[feature-xlings]` tables apply. - if (auto pit = state.hostModuleProvidersByConsumer.find(consumer); - pit != state.hostModuleProvidersByConsumer.end()) { - for (auto q : pit->second) { - if (q >= state.packages.size()) continue; - auto const& pm = state.packages[q].manifest; - auto want = [&](const std::string& address) { - if (std::ranges::find(declared, address) == declared.end()) - declared.push_back(address); - }; - for (auto const& address : pm.xlings.deps) want(address); - const auto& pf = q < state.activeFeaturesByPackage.size() - ? state.activeFeaturesByPackage[q] : std::vector{}; - for (auto const& f : pf) - if (auto it = pm.xlings.featureDeps.find(f); - it != pm.xlings.featureDeps.end()) - for (auto const& address : it->second) want(address); - } - } - if (declared.empty()) return; - auto cfg = state.get_cfg(true); - if (!cfg) return; - auto xlEnv = mcpp::config::make_xlings_env(**cfg); - std::set answered; - for (auto const& raw : declared) { - // THE VERSION THIS BUILD INSTALLED, NOT THE ONE THIS MANIFEST - // WROTE. Both statements are about one package, and only one - // version of it exists on disk; answering from the local spelling - // is how a rule package could declare `>=8.5.0`, have the project's - // exact pin installed instead, and then be told nothing is there. - // `xlingsWinner` is empty only before the split has run, and every - // caller of this lambda runs after it — the fallback keeps that a - // fact about ordering rather than a crash. - const auto key = mcpp::xlings::addrset::package_key(raw); - if (!answered.insert(key).second) continue; - auto wit = state.xlingsWinner.find(key); - const std::string spec = wit == state.xlingsWinner.end() ? raw : wit->second; - auto ref = mcpp::xlings::paths::parse_xpkg_ref(spec); - auto dir = mcpp::xlings::paths::xpkg_payload(xlEnv, ref); - if (!dir) continue; // declared but not installed: "" is the answer - // Namespaced first — it is the exact spelling, and the bare form - // below must not shadow it (the receiver keeps the first value it - // is given for a name). - e.xpkgDirs.emplace_back( - mcpp::build::xpkg_env_var(ref.ns, ref.name), dir->string()); - e.xpkgDirs.emplace_back( - mcpp::build::xpkg_env_var("", ref.name), dir->string()); - } + fill_xpkg_env(state, e, owner, consumer); }; // `linkForms` (#642 E2): when given, each dependency that has a resolved diff --git a/src/build/prepare/graph_load.cpp b/src/build/prepare/graph_load.cpp index abd077205..57bb276bc 100644 --- a/src/build/prepare/graph_load.cpp +++ b/src/build/prepare/graph_load.cpp @@ -93,11 +93,16 @@ static void step4a_define_split_and_identity_closures(PrepareState& state) { : pkg.namespace_ + ":" + pkg.name; }; std::vector claims; + // Whether each claim was declared `provision = "on-request"`, by the + // manifest that made it (mcpp#755). + std::vector onRequest; for (auto const& spec : applicable_xlings_addresses( *state.runtimeOwnerManifest, state.activeFeaturesByPackage.empty() ? std::vector{} : state.activeFeaturesByPackage[0], - state.toolPurpose, /*isRoot=*/true)) + state.toolPurpose, /*isRoot=*/true)) { claims.push_back({spec, "this project", 0}); + onRequest.push_back(state.runtimeOwnerManifest->xlings.on_request(spec)); + } // THE BUCKET IS DECIDED BY WHERE THE WINNING CLAIM SITS IN THIS LIST, // not by its distance. The root's own pass provisions exactly the // addresses collected above; anything else has to reach the graph pass @@ -110,15 +115,25 @@ static void step4a_define_split_and_identity_closures(PrepareState& state) { ? state.activeFeaturesByPackage[i] : std::vector{}; for (auto const& spec : applicable_xlings_addresses( man, feats, state.toolPurpose, /*isRoot=*/i == 0 - || state.packages[i].selectedMember)) + || state.packages[i].selectedMember)) { claims.push_back({spec, describe(i), i == 0 ? 0 : 1}); + onRequest.push_back(man.xlings.on_request(spec)); + } } auto unified = addrset::unify(claims); if (!unified) return std::unexpected(unified.error()); + for (auto const& w : unified->winners) + state.xlingsWinner[addrset::package_key(w.address)] = w.address; + // WHAT THE REGISTRY INSTALLS, which is not every winner (mcpp#755): an + // overridden package comes from where its override says, and one every + // declaration states `provision = "on-request"` waits for a build + // program to ask. Both are recorded as decisions there. + auto install = payloads_to_provision(state, *unified, claims, onRequest); + if (!install) return std::unexpected(install.error()); std::vector rootSpecs, fromGraph; for (auto const& w : unified->winners) { + if (std::ranges::find(*install, w.address) == install->end()) continue; (w.claim < rootClaims ? rootSpecs : fromGraph).push_back(w.address); - state.xlingsWinner[addrset::package_key(w.address)] = w.address; } // REPORTED, NOT INFERRED. An override that is only visible as "two // versions were declared and one directory exists" is a fact the reader diff --git a/src/build/prepare/local_toolchain.cpp b/src/build/prepare/local_toolchain.cpp new file mode 100644 index 000000000..fa1045a4a --- /dev/null +++ b/src/build/prepare/local_toolchain.cpp @@ -0,0 +1,359 @@ +// local_toolchain.cpp -- a toolchain named by path, the toolchain that runs +// build programs when it differs, and the root build program's toolchain +// phase (mcpp#755). Declared in `:state`. +// +// A TOOLCHAIN NAMED BY PATH IS NOT A PATH COMPILER. `[toolchain] = "system"` +// stays refused: it is whatever PATH happens to hold. A `[toolchain]` table +// names a tree, and mcpp probes the drivers in it, identifies them (version, +// triple, standard library, `import std`), drives them with its own link +// model, and records the source on every line that reports it -- the shape +// `msvc@system` already has, generalised to gcc and llvm. + +module; +#include + +module mcpp.build.prepare; +import :state; + +import std; +import mcpp.build.refusal; +import mcpp.build.build_program; +import mcpp.config; +import mcpp.home; +import mcpp.manifest; +import mcpp.platform; +import mcpp.toolchain.registry; +import mcpp.toolchain.post_install; +import mcpp.ui; + +namespace mcpp::build { + +namespace fs = std::filesystem; + +namespace { + +// The statement a toolchain phase made, carried to the second pass. +struct ToolchainRestart { + std::vector statement; + std::string bootstrap; +}; +thread_local std::optional g_restart; + +fs::path absolute_against(const fs::path& base, const std::string& p) { + if (p.empty()) return {}; + fs::path v(p); + if (v.is_relative()) v = base / v; + return v.lexically_normal(); +} + +// A description read from `key=value` lines: the build program's statement. +std::expected +read_statement(const std::vector& lines, std::string& spec, + std::string& originFile, int& originLine) { + mcpp::manifest::LocalToolchain lt; + for (auto const& line : lines) { + const auto eq = line.find('='); + if (eq == std::string::npos || eq == 0) + return std::unexpected(std::format("`mcpp:toolchain={}` is not `=`", line)); + const auto key = line.substr(0, eq); + const auto val = line.substr(eq + 1); + if (key == "spec") spec = val; + else if (key == "path") lt.path = val; + else if (key == "prefix") lt.prefix = val; + else if (key == "sysroot") lt.sysroot = val; + else if (key == "launcher") lt.launcher = val; + else if (key == "family") { + if (val != "gcc" && val != "llvm") + return std::unexpected(std::format("family = '{}': a toolchain is \"gcc\" or \"llvm\"", val)); + lt.family = val; + } else if (key.starts_with("tool.")) { + lt.tools.emplace_back(key.substr(5), val); + } else if (key == "origin") { + const auto colon = val.rfind(':'); + originFile = colon == std::string::npos ? val : val.substr(0, colon); + if (colon != std::string::npos) originLine = std::atoi(val.c_str() + colon + 1); + } else { + return std::unexpected(std::format( + "`mcpp:toolchain={}`: '{}' is not a key; the keys are spec, path, prefix, " + "sysroot, family, launcher, tool. and origin", line, key)); + } + } + if (spec.empty() && lt.path.empty()) + return std::unexpected(std::string( + "the toolchain phase stated neither `spec` (a managed toolchain) nor `path`")); + if (!spec.empty() && !lt.path.empty()) + return std::unexpected(std::string( + "the toolchain phase stated both `spec` and `path`; a build toolchain is one of them")); + return lt; +} + +// What makes a toolchain named by path a different toolchain without the +// version changing: a rebuilt trunk driver, a replaced linker. The path, size +// and modification time of each program, not their bytes -- a driver is +// hundreds of megabytes and this is read on every prepare. +std::string local_identity(const mcpp::toolchain::Toolchain& tc) { + std::uint64_t h = 1469598103934665603ull; + auto mix = [&](std::string_view s) { + for (unsigned char c : s) { h ^= c; h *= 1099511628211ull; } + h ^= 0x1f; h *= 1099511628211ull; + }; + auto stamp = [&](const fs::path& p) { + std::error_code ec; + mix(p.generic_string()); + mix(std::to_string(fs::file_size(p, ec))); + auto t = fs::last_write_time(p, ec); + mix(std::to_string(t.time_since_epoch().count())); + }; + stamp(tc.binaryPath); + for (auto const& [role, p] : tc.toolOverrides) { mix(role); stamp(p); } + mix(tc.toolPrefix); + return std::format("{:016x}", h); +} + +} // namespace + +std::optional, std::string>> take_toolchain_restart() { + if (!g_restart) return std::nullopt; + auto r = std::move(*g_restart); + g_restart.reset(); + return std::pair{std::move(r.statement), std::move(r.bootstrap)}; +} + +std::expected step1_local_toolchain(PrepareState& state) { + // THE SECOND PASS OF A TOOLCHAIN PHASE: the build program stated it. + if (!state.overrides.toolchain_statement.empty()) { + std::string spec, file; + int line = 0; + auto lt = read_statement(state.overrides.toolchain_statement, spec, file, line); + if (!lt) { + refusal::record(refusal::Code::LocalToolchain); + return std::unexpected(std::format("the root build program's toolchain phase: {}", lt.error())); + } + state.tcOrigin = TcOrigin::BuildProgram; + state.bootstrapSpec = state.overrides.bootstrap_spec; + SourceDecision d; + d.subject = "toolchain.build"; + d.originKind = "build-program"; + d.originFile = file.empty() ? (*state.root / "build.mcpp").string() : file; + d.originLine = line; + if (!spec.empty()) { + state.tcSpec = spec; + d.cls = SourceClass::Pinned; + d.value = spec; + state.localToolchainOrigin = std::move(d); + return {}; + } + const auto base = *state.root; + lt->path = absolute_against(base, lt->path).generic_string(); + lt->sysroot = absolute_against(base, lt->sysroot).generic_string(); + for (auto& [role, p] : lt->tools) p = absolute_against(base, p).generic_string(); + state.tcSpec = "path:" + lt->path; + state.localToolchain = std::move(*lt); + d.cls = SourceClass::Program; + state.localToolchainOrigin = std::move(d); + return {}; + } + if (!state.tcSpec) { + if (!state.m->toolchain.bootstrap.empty()) state.bootstrapSpec = state.m->toolchain.bootstrap; + return {}; + } + // THE FIRST PASS OF A TOOLCHAIN PHASE: the bootstrap toolchain builds + // until the root build program has stated the build toolchain. + if (state.tcSpec->starts_with("configure:")) { + state.toolchainConfigure = true; + if (!state.m->toolchain.bootstrap.empty()) { + state.tcSpec = state.m->toolchain.bootstrap; + state.tcOrigin = TcOrigin::ManifestToolchain; + } else if (auto cfg = state.get_cfg(true); cfg && !(*cfg)->defaultToolchain.empty()) { + state.tcSpec = (*cfg)->defaultToolchain; + state.tcOrigin = TcOrigin::GlobalDefault; + } else { + state.tcSpec.reset(); + state.tcOrigin = TcOrigin::None; + } + return {}; + } + if (!state.m->toolchain.bootstrap.empty()) state.bootstrapSpec = state.m->toolchain.bootstrap; + if (!state.tcSpec->starts_with("path:")) return {}; + + // A toolchain named by path: by the manifest's table, by + // `MCPP_TOOLCHAIN=path:`, or handed to a host-tool sub-build. + const mcpp::manifest::LocalToolchain* table = + (state.tcFromCommandLine || state.tcFromConsumer) ? nullptr + : state.m->toolchain.local_for(kCurrentPlatform); + mcpp::manifest::LocalToolchain lt = table ? *table : mcpp::manifest::LocalToolchain{}; + if (!table) lt.path = state.tcSpec->substr(5); + const auto base = (state.tcFromCommandLine || state.tcFromConsumer) + ? fs::current_path() : state.m->sourcePath.parent_path(); + lt.path = absolute_against(base, lt.path).generic_string(); + lt.sysroot = absolute_against(base, lt.sysroot).generic_string(); + for (auto& [role, p] : lt.tools) p = absolute_against(base, p).generic_string(); + state.tcSpec = "path:" + lt.path; + SourceDecision d; + d.subject = "toolchain.build"; + d.cls = SourceClass::Custom; + if (state.tcFromCommandLine) { + d.originKind = "env"; + d.originKey = "MCPP_TOOLCHAIN"; + } else if (state.tcFromConsumer) { + d.originKind = "graph"; + d.originKey = state.overrides.tool_chain; + } else { + d.originKind = "manifest"; + d.originFile = state.m->sourcePath.string(); + d.originLine = state.m->toolchain.line_for(kCurrentPlatform); + d.originKey = "[toolchain]"; + } + state.localToolchain = std::move(lt); + state.localToolchainOrigin = std::move(d); + return {}; +} + +std::expected +step2_use_local_toolchain(PrepareState& state, const mcpp::toolchain::ToolchainSpec& spec) { + if (!state.localToolchain) { + state.localToolchain = mcpp::manifest::LocalToolchain{}; + state.localToolchain->path = spec.localRoot.generic_string(); + } + auto& lt = *state.localToolchain; + const fs::path root(lt.path); + std::error_code ec; + auto refuse = [&](std::string why) -> std::unexpected { + refusal::record(refusal::Code::LocalToolchain); + return std::unexpected(std::format( + "the toolchain named by path '{}' cannot be used: {}", root.generic_string(), why)); + }; + if (!fs::is_directory(root, ec)) return refuse("the directory does not exist"); + std::string family = lt.family; + if (family.empty()) + family = spec.family == mcpp::toolchain::Family::Llvm ? "llvm" : "gcc"; + fs::path driver; + for (auto const& [role, p] : lt.tools) if (role == "cxx") driver = p; + if (driver.empty()) + driver = root / "bin" / (lt.prefix + (family == "llvm" ? "clang++" : "g++") + + std::string(mcpp::platform::exe_suffix)); + if (!fs::exists(driver, ec)) + return refuse(std::format("no C++ driver at '{}'; a {} toolchain keeps it in " + "`/bin/{}{}`, or names it with `tools = {{ cxx = ... }}`", + driver.generic_string(), family, lt.prefix, + family == "llvm" ? "clang++" : "g++")); + for (auto const& [role, p] : lt.tools) + if (!fs::exists(p, ec)) + return refuse(std::format("tools.{} names '{}', which does not exist", role, p)); + if (!lt.sysroot.empty() && !fs::is_directory(lt.sysroot, ec)) + return refuse(std::format("sysroot '{}' is not a directory", lt.sysroot)); + state.explicit_compiler = driver; + // THE C LIBRARY A NATIVE BUILD LINKS IS THE ECOSYSTEM'S, AS FOR A MANAGED + // TOOLCHAIN: the binding's payload, linked by mcpp's own link model. A + // toolchain that states its sysroot, or a prefixed (cross) toolchain that + // carries one, links what it carries instead. + if (lt.sysroot.empty() && lt.prefix.empty()) { + mcpp::toolchain::XimToolchainPackage pkg; + pkg.ximName = family; + pkg.family = family == "llvm" ? mcpp::toolchain::Family::Llvm : mcpp::toolchain::Family::Gcc; + pkg.needsGccPostInstallFixup = family == "gcc"; + state.provide_runtime_payload(pkg); + } + return {}; +} + +std::expected step2_apply_local_toolchain(PrepareState& state) { + if (!state.localToolchain || !state.tc) return {}; + auto& lt = *state.localToolchain; + auto& tc = *state.tc; + if (!lt.family.empty()) { + const bool isLlvm = tc.compiler == mcpp::toolchain::CompilerId::Clang; + if ((lt.family == "llvm") != isLlvm) { + refusal::record(refusal::Code::LocalToolchain); + return std::unexpected(std::format( + "the toolchain at '{}' is stated as family \"{}\", and its driver '{}' is {} {}", + lt.path, lt.family, tc.binaryPath.generic_string(), tc.compiler_name(), tc.version)); + } + } + tc.localRoot = lt.path; + tc.toolPrefix = lt.prefix; + for (auto const& [role, p] : lt.tools) + if (role != "cxx") tc.toolOverrides.emplace_back(role, p); + if (!lt.launcher.empty()) { + fs::path l(lt.launcher); + if (!l.has_parent_path()) { + auto found = mcpp::platform::fs::which(lt.launcher); + if (!found) { + refusal::record(refusal::Code::LocalToolchain); + return std::unexpected(std::format( + "launcher '{}' is not found on PATH", lt.launcher)); + } + l = *found; + } + tc.launcher = l.generic_string(); + } + if (!lt.sysroot.empty()) { + // A STATED SYSROOT IS THE C LIBRARY: the link model's sysroot mode, + // not the binding's payload. + tc.sysroot = lt.sysroot; + tc.payloadPaths.reset(); + } + tc.driverIdent += "\nmcpp-local-toolchain " + local_identity(tc); + // Announced beside the resolution, with what the probe found. + auto d = state.localToolchainOrigin; + d.subject = "toolchain.build"; + d.value = lt.path; + d.detail = std::format("{} {}", tc.compiler_name(), tc.version); + d.considered = {tc.binaryPath.generic_string()}; + if (!tc.launcher.empty()) d.considered.push_back("launcher " + tc.launcher); + for (auto const& [role, p] : tc.toolOverrides) + d.considered.push_back(std::format("{} {}", role, p.generic_string())); + record_source(state, std::move(d)); + return {}; +} + +std::expected step6_toolchain_phase(PrepareState& state) { + if (!state.toolchainConfigure || !state.overrides.toolchain_statement.empty()) return {}; + if (!std::filesystem::exists(*state.root / "build.mcpp")) { + refusal::record(refusal::Code::LocalToolchain); + return std::unexpected(std::string( + "[toolchain] says `configure = \"build.mcpp\"`, and this project has no build.mcpp")); + } + auto host = state.host_tc_for_build_program(); + if (!host) return std::unexpected(host.error()); + mcpp::build::BuildProgramEnv bpEnv; + bpEnv.targetTriple = state.resolvedTargetCanonical; + fill_target_build_env(bpEnv, *state.m, state.tc ? &*state.tc : nullptr, + state.cfg_opt ? &*state.cfg_opt : nullptr); + bpEnv.toolsBin = state.projectSubosBin; + bpEnv.profile = state.effectiveProfile; + fill_package_build_env(bpEnv, *state.m); + bpEnv.languageModules = state.m->language.modules; + // ITS OWN RECORD: the toolchain phase and the build phase are two runs of + // one program with two contracts, and one record would make each + // invalidate the other on every build. + bpEnv.artifactsDir = state.workRoot / "target" / ".build-mcpp" / "toolchain-phase"; + bpEnv.moduleStore = state.workRoot / "target" / ".build-mcpp" / "host-modules"; + if (state.cacheMode == CacheMode::Global) + bpEnv.moduleCacheRoot = mcpp::home::cache_root(); + bpEnv.features = feature_closure(*state.m, parse_feature_request(state.overrides.features)); + state.fillDepDirs(bpEnv, 0, nullptr); + bpEnv.hostModules = state.hostModulesByConsumer.count(0u) + ? state.hostModulesByConsumer.at(0u) : decltype(bpEnv.hostModules){}; + bpEnv.dormantFeatures = state.dormantFeaturesByConsumer.count(0u) + ? state.dormantFeaturesByConsumer.at(0u) : decltype(bpEnv.dormantFeatures){}; + bpEnv.phase = "toolchain"; + // A copy: the first pass is discarded, and what the phase states is the + // whole of what is carried to the second. + auto manifest = *state.m; + auto r = mcpp::build::run_build_program(manifest, *state.root, host->first, host->second, + manifest.cppStandard, bpEnv); + if (!r) return std::unexpected(std::format("the root build program's toolchain phase: {}", r.error())); + if (manifest.buildConfig.toolchainStatement.empty()) { + refusal::record(refusal::Code::LocalToolchain); + return std::unexpected(std::string( + "[toolchain] says `configure = \"build.mcpp\"`, and the root build program stated no\n" + " toolchain in its toolchain phase. While `mcpp::phase()` is \"toolchain\", it\n" + " states one with `mcpp::plugins::toolchain::use(...)` (or `mcpp::toolchain(k, v)`).")); + } + g_restart = ToolchainRestart{manifest.buildConfig.toolchainStatement, + state.tcSpec.value_or(std::string{})}; + return std::unexpected(std::string("mcpp:toolchain-phase-restart")); +} + +} // namespace mcpp::build diff --git a/src/build/prepare/plan.cpp b/src/build/prepare/plan.cpp index ed1482afc..523068507 100644 --- a/src/build/prepare/plan.cpp +++ b/src/build/prepare/plan.cpp @@ -278,10 +278,17 @@ static std::expected step13_runner_and_xlings(PrepareState& s for (auto const& spec : applicable_xlings_addresses( man, feats, ToolPurpose::Run, /*isRoot=*/i == 0 || state.packages[i].selectedMember)) - if (std::ranges::find(xlingsSpecs, spec) == xlingsSpecs.end()) + if (std::ranges::find(xlingsSpecs, spec) == xlingsSpecs.end() + && !state.xlingsSkipped.contains(spec)) { ctx.runTierPending = true; break; } } } + // An overridden payload's programs are looked up where the override + // put them (mcpp#755), first: the project said that is where they are. + for (auto const& key : state.xlingsOverridden) + if (auto ov = payload_override(state, key); ov && *ov) + for (auto& d : mcpp::build::runner_lookup::payload_search_dirs((*ov)->root)) + ctx.xlingsDepBinDirs.push_back(std::move(d)); if (!xlingsSpecs.empty()) { if (auto cfg = state.get_cfg(true)) { auto xlEnv = mcpp::config::make_xlings_env(**cfg); @@ -2296,6 +2303,8 @@ std::expected phase13_finish(PrepareState& state) { return r; }; + if (auto r = timed("sources", [&] { return step13_sources(state, ctx); }); !r) + return std::unexpected(r.error()); if (auto r = timed("source packages", [&] { return step13_source_packages(state, ctx); }); !r) return std::unexpected(r.error()); if (auto r = timed("runner and xlings", [&] { return step13_runner_and_xlings(state, ctx); }); !r) diff --git a/src/build/prepare/records.cpp b/src/build/prepare/records.cpp index d15f36ccd..431cb9e12 100644 --- a/src/build/prepare/records.cpp +++ b/src/build/prepare/records.cpp @@ -558,8 +558,54 @@ void step13_resolution_json(PrepareState& state, BuildContext& ctx) { {"root", tcr.windowsSdkRoot.generic_string()}, }; } + // WHERE EACH SOURCE CAME FROM (mcpp#755): the decision record, one + // entry per subject, as `mcpp why` reads it. Added beside the other + // keys; no reader keys on `schema_version`. + { + nlohmann::json sources = nlohmann::json::array(); + for (auto const& d : ctx.sources) { + nlohmann::json e = { + {"subject", d.subject}, {"value", d.value}, + {"class", std::string(source_class_name(d.cls))}, + {"origin", {{"kind", d.originKind}, {"file", d.originFile}, + {"line", d.originLine}, {"key", d.originKey}}}, + {"decidedFor", d.decidedFor}, + {"considered", d.considered}, + }; + sources.push_back(std::move(e)); + } + j["sources"] = std::move(sources); + } std::error_code ec; std::filesystem::create_directories(ctx.plan.outputDir, ec); + // The `Finished` line of a build that skips prepare reads the summary + // from here, beside the steps record the fast path already reads. + { + auto summary = sources_summary(ctx.sources); + auto sp = ctx.plan.outputDir / "sources.summary"; + if (summary.empty()) std::filesystem::remove(sp, ec); + else if (std::ofstream so(sp); so) so << summary << "\n"; + ec.clear(); + } + // A toolchain named by path can change in place; the fast paths + // compare what is recorded here (mcpp#755). + { + auto stampPath = ctx.plan.outputDir / "local-toolchain.stamp"; + if (ctx.tc.localRoot.empty()) { + std::filesystem::remove(stampPath, ec); + } else if (std::ofstream st(stampPath, std::ios::binary); st) { + auto put = [&](const std::filesystem::path& p) { + std::error_code fe; + const auto size = std::filesystem::file_size(p, fe); + const auto time = std::filesystem::last_write_time(p, fe); + st << size << '\t' << time.time_since_epoch().count() << '\t' + << p.string() << '\n'; + }; + put(ctx.tc.binaryPath); + for (auto const& [role, p] : ctx.tc.toolOverrides) put(p); + } + ec.clear(); + } auto path = ctx.plan.outputDir / "resolution.json"; auto tmp = path; tmp += ".tmp"; diff --git a/src/build/prepare/sources.cpp b/src/build/prepare/sources.cpp new file mode 100644 index 000000000..46d96aba3 --- /dev/null +++ b/src/build/prepare/sources.cpp @@ -0,0 +1,644 @@ +// sources.cpp -- where each thing a build uses comes from (mcpp#755): the +// decision record, payload overrides, on-request provisioning, the payload +// facts a build program receives, and `--managed-only`. Declared in `:state`. +// +// ONE RECORD, MANY READERS. A source is decided once, here, and the status +// line, the `Finished` summary, `mcpp why`, `resolution.json` and the +// `--managed-only` refusal all read the same entry. Each of them used to be a +// place where the answer could be re-derived and drift, which is the shape +// this file exists to remove. + +module; +#include + +module mcpp.build.prepare; +import :state; + +import std; +import mcpp.build.refusal; +import mcpp.build.build_program; +import mcpp.config; +import mcpp.diag; +import mcpp.wire; +import mcpp.manifest; +import mcpp.platform; +import mcpp.ui; +import mcpp.xlings; +import mcpp.xlings.address_set; + +namespace mcpp::build { + +namespace fs = std::filesystem; +namespace addrset = mcpp::xlings::addrset; + +namespace { + +// `mcpp.toml:22`, relative to the project when the file is inside it. +std::string origin_place(const SourceDecision& d, const fs::path& relativeTo) { + if (d.originFile.empty()) return d.originKey; + fs::path f(d.originFile); + std::string shown = f.generic_string(); + if (!relativeTo.empty()) { + std::error_code ec; + auto rel = fs::relative(f, relativeTo, ec); + if (!ec && !rel.empty() && !rel.generic_string().starts_with("..")) + shown = rel.generic_string(); + } + const char* home = std::getenv(mcpp::platform::is_windows ? "USERPROFILE" : "HOME"); + if (home && *home) { + const auto h = fs::path(home).generic_string(); + if (shown.starts_with(h + "/")) shown = "~" + shown.substr(h.size()); + } + return d.originLine > 0 ? std::format("{}:{}", shown, d.originLine) : shown; +} + +// The thing a `Using` line names, by the subject's kind. +std::string subject_shown(const SourceDecision& d) { + if (d.subject.starts_with("payload:")) return d.subject.substr(8); + if (d.subject.starts_with("tool:")) { + // `tool::` -> ` ()` + const auto rest = std::string_view(d.subject).substr(5); + const auto colon = rest.rfind(':'); + if (colon == std::string_view::npos) return std::string(rest); + return std::format("{} ({})", rest.substr(colon + 1), rest.substr(0, colon)); + } + if (d.subject == "toolchain.build") return "toolchain"; + if (d.subject == "toolchain.bootstrap") return "bootstrap toolchain"; + return d.subject; +} + +// `MCPP_XLINGS_OVERRIDE__`, sanitised as `xpkg_dir`'s variables are. +std::string override_env_var(std::string_view key) { + std::string out = "MCPP_XLINGS_OVERRIDE_"; + for (char c : key) + out += (c >= 'a' && c <= 'z') ? char(c - 'a' + 'A') + : ((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) ? c : '_'; + return out; +} + +bool names_a_file_path(std::string_view v) { + return v.contains('/') || v.contains('\\') || v.starts_with(".") + || (v.size() > 1 && v[1] == ':'); +} + +// The root a program implies: its directory, or that directory's parent when +// it is a `bin/` -- so `/bin/`, the layout every payload +// plugin already reads, holds for `/usr/bin/cmake` as it does for a payload. +fs::path root_of_program(const fs::path& program) { + auto dir = program.parent_path(); + if (dir.filename() == "bin") return dir.parent_path(); + return dir; +} + +} // namespace + +std::string source_tag(const SourceDecision& d, const fs::path& relativeTo) { + const auto cls = std::string(source_class_name(d.cls)); + if (d.originKind == "env") + return std::format("{} · env {}", cls, d.originKey); + const auto place = origin_place(d, relativeTo); + if (place.empty()) return cls; + return std::format("{} · {}", cls, place); +} + +void record_source(PrepareState& state, SourceDecision d) { + const bool announce = d.announce && !source_class_is_default(d.cls) + && state.announcedSources.insert(d.subject + "\x1f" + d.value).second; + if (announce) { + const auto root = state.root ? *state.root : fs::path{}; + const auto what = d.detail.empty() ? subject_shown(d) + : subject_shown(d) + " " + d.detail; + mcpp::ui::source("Using", std::format("{} ← {}", what, d.value), + source_tag(d, root), d.cls == SourceClass::Host); + } + auto it = std::ranges::find(state.sources, d.subject, &SourceDecision::subject); + if (it == state.sources.end()) state.sources.push_back(std::move(d)); + else *it = std::move(d); +} + +std::expected +payload_override(PrepareState& state, std::string_view key) { + if (auto it = state.payloadOverrideCache.find(std::string(key)); + it != state.payloadOverrideCache.end()) + return it->second ? &*it->second : nullptr; + + // Three places may state it, nearest first. The environment outranks the + // manifest so that CI and packaging can override without editing it, and + // says so on the `Using` line; the manifest outranks the machine's + // configuration because a project's statement is the more specific one. + struct Stated { + std::string kind = "path", value, version, originKind, originFile, originKey; + int line = 0; + fs::path base; // what a relative path is relative to + }; + std::optional stated; + const auto var = override_env_var(key); + if (const char* v = std::getenv(var.c_str()); v && *v) { + std::string value(v); + Stated s{.originKind = "env", .originKey = var, .base = fs::current_path()}; + if (value.starts_with("path:")) { s.kind = "program"; value = value.substr(5); } + s.value = std::move(value); + stated = std::move(s); + } + auto from_manifest = [&](const mcpp::manifest::Manifest* man) { + if (stated || !man) return; + auto it = man->xlings.overrides.find(std::string(key)); + if (it == man->xlings.overrides.end()) return; + const auto& o = it->second; + Stated s; + s.kind = o.kind == mcpp::manifest::XlingsOverride::Kind::Program ? "program" + : o.kind == mcpp::manifest::XlingsOverride::Kind::Root ? "root" : "path"; + s.value = o.value; s.version = o.version; s.line = o.line; + s.originKind = "manifest"; + s.originFile = man->sourcePath.string(); + s.originKey = "[xlings.overrides]"; + s.base = man->sourcePath.parent_path(); + stated = std::move(s); + }; + from_manifest(state.m ? &*state.m : nullptr); + from_manifest(state.runtimeOwnerManifest); + if (!stated) { + if (auto cfg = state.get_cfg(true)) { + if (auto it = (*cfg)->payloadOverrides.find(std::string(key)); + it != (*cfg)->payloadOverrides.end()) { + Stated s; + s.kind = it->second.kind; s.value = it->second.value; + s.version = it->second.version; s.line = it->second.line; + s.originKind = "config"; + s.originFile = (*cfg)->configFile.string(); + s.originKey = "[xlings.overrides] in config.toml"; + s.base = (*cfg)->configFile.parent_path(); + stated = std::move(s); + } + } + } + if (!stated) { + state.payloadOverrideCache.emplace(std::string(key), std::nullopt); + return nullptr; + } + + const auto where = stated->originKind == "env" ? stated->originKey + : stated->line > 0 ? std::format("{} ({}:{})", stated->originKey, + stated->originFile, stated->line) + : std::format("{} ({})", stated->originKey, stated->originFile); + auto refuse = [&](std::string why) -> std::unexpected { + refusal::record(refusal::Code::PayloadOverride); + return std::unexpected(std::format( + "`{}` is overridden by {}, and {}.\n" + " An override names a program (a file, or a name found on PATH)\n" + " or a root laid out like the payload; remove the entry to\n" + " install the payload instead.", key, where, why)); + }; + + PrepareState::PayloadOverride out; + out.version = stated->version; + out.originKind = stated->originKind; + out.originFile = stated->originFile; + out.originKey = stated->originKey; + out.originLine = stated->line; + out.cls = SourceClass::Custom; + std::error_code ec; + if (stated->kind == "program" && !names_a_file_path(stated->value)) { + // A NAME, LOOKED UP ON PATH ONCE, HERE. The answer is the program a + // later step runs and the one the `Using` line names, so a reader can + // see which PATH entry served it. Without a stated version it is the + // one class of source mcpp cannot describe beyond its location. + auto found = mcpp::platform::fs::which(stated->value); + if (!found) return refuse(std::format("'{}' is not found on PATH", stated->value)); + out.program = found->generic_string(); + out.root = root_of_program(*found).generic_string(); + if (out.version.empty()) out.cls = SourceClass::Host; + } else { + fs::path p(stated->value); + if (p.is_relative()) p = stated->base / p; + p = p.lexically_normal(); + const bool isDir = fs::is_directory(p, ec); + const bool isFile = !isDir && fs::exists(p, ec); + if (stated->kind == "root" || (stated->kind == "path" && isDir)) { + if (!isDir) return refuse(std::format("'{}' is not a directory", p.generic_string())); + out.root = p.generic_string(); + } else { + if (!isFile) return refuse(std::format("'{}' does not exist", p.generic_string())); + out.program = p.generic_string(); + out.root = root_of_program(p).generic_string(); + } + } + auto [it, _] = state.payloadOverrideCache.emplace(std::string(key), std::move(out)); + return &*it->second; +} + +std::expected, std::string> +payloads_to_provision(PrepareState& state, + const addrset::Resolution& unified, + std::span claims, + const std::vector& onRequest) { + std::vector install; + for (auto const& w : unified.winners) { + const auto key = addrset::package_key(w.address); + auto ov = payload_override(state, key); + if (!ov) return std::unexpected(ov.error()); + if (*ov) { + const auto& o = **ov; + const auto statedBy = o.originKind == "env" ? o.originKey : o.originKey; + if (auto why = addrset::override_violation(claims, key, o.version, statedBy)) { + refusal::record(refusal::Code::PayloadOverride); + return std::unexpected(*why); + } + if (o.version.empty()) + if (auto reqs = addrset::requirements_for(claims, key); !reqs.empty()) { + std::string list; + for (auto const& r : reqs) list += (list.empty() ? "" : ", ") + r; + mcpp::diag::note("xlings/override-unversioned", std::format( + "`{}` is overridden without a stated version, so the requirement " + "({}) is not checked; state `version` in the override to have it " + "checked", key, list)); + } + state.xlingsOverridden.insert(key); + state.xlingsSkipped.insert(w.address); + SourceDecision d; + d.subject = "payload:" + key; + d.value = o.program.empty() ? o.root : o.program; + d.cls = o.cls; + d.originKind = o.originKind; + d.originFile = o.originFile; + d.originLine = o.originLine; + d.originKey = o.originKey; + d.decidedFor = w.claim < claims.size() ? claims[w.claim].declaredBy : std::string{}; + d.considered.push_back(std::format("payload {} (not installed: overridden)", w.address)); + record_source(state, std::move(d)); + continue; + } + // DEFERRED ONLY WHEN EVERY DECLARATION SAYS SO. A package one manifest + // needs eagerly is installed eagerly, whoever else declared it on + // request; and one a build program already asked for is installed. + bool allOnRequest = false; + for (std::size_t i = 0; i < claims.size(); ++i) { + if (addrset::package_key(claims[i].address) != key) continue; + if (i >= onRequest.size() || !onRequest[i]) { allOnRequest = false; break; } + allOnRequest = true; + } + SourceDecision d; + d.subject = "payload:" + key; + d.value = w.address; + d.originKind = "graph"; + d.decidedFor = w.claim < claims.size() ? claims[w.claim].declaredBy : std::string{}; + d.cls = (w.claim < claims.size() && claims[w.claim].distance == 0 + && !addrset::version_of(w.address).empty()) + ? SourceClass::Pinned : SourceClass::Managed; + if (d.cls == SourceClass::Pinned) d.originKind = "manifest"; + if (allOnRequest && !state.requestedPayloads.contains(key)) { + state.xlingsDeferred.insert(key); + state.xlingsSkipped.insert(w.address); + d.considered.push_back("installed on request; not requested by this build"); + record_source(state, std::move(d)); + continue; + } + state.xlingsDeferred.erase(key); + if (state.requestedPayloads.contains(key)) + d.considered.push_back("installed on request of a build program"); + record_source(state, std::move(d)); + install.push_back(w.address); + } + return install; +} + +std::optional dependency_override_refusal(const PrepareState& state) { + for (std::size_t i = 1; i < state.packages.size(); ++i) { + const auto& pkg = state.packages[i]; + if (pkg.selectedMember) continue; + if (pkg.manifest.xlings.overrides.empty()) continue; + std::string keys; + for (auto const& [k, o] : pkg.manifest.xlings.overrides) + keys += (keys.empty() ? "" : ", ") + o.key; + refusal::record(refusal::Code::PayloadOverride); + return std::format( + "`{}` states [xlings.overrides] ({}), and it is a dependency of this build.\n" + " Where a payload comes from is stated by the project being built, its\n" + " environment (MCPP_XLINGS_OVERRIDE__) or config.toml; a\n" + " package states which payloads it needs, in [xlings.workspace] or\n" + " [feature-xlings.].", + pkg.manifest.package.name, keys); + } + return std::nullopt; +} + +void fill_xpkg_env(PrepareState& state, mcpp::build::BuildProgramEnv& e, + const mcpp::manifest::Manifest& owner, std::size_t consumer) { + // `[feature-xlings.]` is provisioned when `` is active, so it has + // to be answerable here too. The set is taken from the SAME env the caller + // already computed, so "which features are on" is answered once. + // Installation stays the filter below: a declared address whose payload is + // absent answers "", which is what a `when = "dev"` entry looks like to a + // consumer. + std::vector declared = owner.xlings.deps; + for (auto const& f : e.features) + if (auto it = owner.xlings.featureDeps.find(f); it != owner.xlings.featureDeps.end()) + for (auto const& address : it->second) + if (std::ranges::find(declared, address) == declared.end()) + declared.push_back(address); + // …and what the rule packages compiled INTO this build program declared. + // Their own active features, not the consumer's: the consumer asked for + // `features = ["rules-cuda"]` on the edge, and that is what decides which + // of the rule's `[feature-xlings]` tables apply. + if (auto pit = state.hostModuleProvidersByConsumer.find(consumer); + pit != state.hostModuleProvidersByConsumer.end()) { + for (auto q : pit->second) { + if (q >= state.packages.size()) continue; + auto const& pm = state.packages[q].manifest; + auto want = [&](const std::string& address) { + if (std::ranges::find(declared, address) == declared.end()) + declared.push_back(address); + }; + for (auto const& address : pm.xlings.deps) want(address); + const auto& pf = q < state.activeFeaturesByPackage.size() + ? state.activeFeaturesByPackage[q] : std::vector{}; + for (auto const& f : pf) + if (auto it = pm.xlings.featureDeps.find(f); it != pm.xlings.featureDeps.end()) + for (auto const& address : it->second) want(address); + } + } + if (declared.empty()) return; + auto cfg = state.get_cfg(true); + if (!cfg) return; + auto xlEnv = mcpp::config::make_xlings_env(**cfg); + std::set answered; + // Namespaced first -- it is the exact spelling, and the bare form must not + // shadow it (the receiver keeps the first value it is given for a name). + auto put = [&](const mcpp::xlings::paths::XpkgRef& ref, std::string_view suffix, + const std::string& value) { + e.xpkgDirs.emplace_back(mcpp::build::xpkg_env_var(ref.ns, ref.name, suffix), value); + e.xpkgDirs.emplace_back(mcpp::build::xpkg_env_var("", ref.name, suffix), value); + }; + for (auto const& raw : declared) { + // THE VERSION THIS BUILD INSTALLED, NOT THE ONE THIS MANIFEST WROTE. + // Both statements are about one package, and only one version of it + // exists on disk; answering from the local spelling is how a rule + // package could declare `>=8.5.0`, have the project's exact pin + // installed instead, and then be told nothing is there. + const auto key = addrset::package_key(raw); + if (!answered.insert(key).second) continue; + auto wit = state.xlingsWinner.find(key); + const std::string spec = wit == state.xlingsWinner.end() ? raw : wit->second; + auto ref = mcpp::xlings::paths::parse_xpkg_ref(spec); + // AN OVERRIDE ANSWERS INSTEAD OF THE REGISTRY (mcpp#755): its root as + // the directory, so a plugin that reads `/bin/` needs no + // change, its program where it named one, and the source. + if (state.xlingsOverridden.contains(key)) { + if (auto ov = payload_override(state, key); ov && *ov) { + put(ref, "DIR", (*ov)->root); + if (!(*ov)->program.empty()) put(ref, "PROGRAM", (*ov)->program); + put(ref, "SOURCE", "override"); + } + continue; + } + auto dir = mcpp::xlings::paths::xpkg_payload(xlEnv, ref); + if (dir) { + put(ref, "DIR", dir->string()); + put(ref, "SOURCE", "payload"); + continue; + } + // Declared on request and not installed: the program may ask. + if (state.xlingsDeferred.contains(key)) put(ref, "SOURCE", "pending"); + } +} + +std::expected +answer_payload_requests(PrepareState& state, mcpp::manifest::Manifest& m, + mcpp::build::BuildProgramEnv& e, std::size_t consumer, + std::string_view who) { + auto requests = std::move(m.buildConfig.xpkgRequests); + m.buildConfig.xpkgRequests.clear(); + if (requests.empty()) return false; + std::vector addresses, keys; + for (auto const& r : requests) { + const auto key = addrset::package_key(r); + if (std::ranges::find(keys, key) != keys.end()) continue; + if (!state.xlingsDeferred.contains(key) && !state.requestedPayloads.contains(key)) { + refusal::record(refusal::Code::PayloadRequest); + return std::unexpected(std::format( + "the build program of `{}` asked for `{}`, which no manifest of this build " + "declares `provision = \"on-request\"` for this build.\n" + " A build program asks only for a payload its package (or a host\n" + " module compiled into it) declares on request; `xpkg_dir` answers\n" + " for one declared without it.", who, key)); + } + keys.push_back(key); + auto wit = state.xlingsWinner.find(key); + addresses.push_back(wit == state.xlingsWinner.end() ? key : wit->second); + } + if (state.overrides.plan_only) { + std::string list; + for (auto const& a : addresses) list += (list.empty() ? "" : ", ") + a; + state.planNotes.push_back({"MCPP_BUILD_DATABASE_PAYLOAD_DEFERRED", + std::format("the build program of `{}` asked for {}, declared `provision = " + "\"on-request\"`; planning installs nothing, so the program is " + "described without them, and `mcpp build` installs them", who, list), + mcpp::wire::Severity::Note, {}}); + return false; + } + auto cfg = state.get_cfg(true); + if (!cfg) return std::unexpected(cfg.error()); + if (auto pv = provision_xlings_addresses( + **cfg, addresses, *state.root, + std::format("payloads requested by the build program of `{}`", who)); + !pv) return std::unexpected(pv.error()); + for (std::size_t i = 0; i < keys.size(); ++i) { + state.requestedPayloads.insert(keys[i]); + state.xlingsDeferred.erase(keys[i]); + state.xlingsSkipped.erase(addresses[i]); + auto it = std::ranges::find(state.sources, "payload:" + keys[i], &SourceDecision::subject); + if (it != state.sources.end()) { + it->considered.clear(); + it->considered.push_back(std::format("installed on request of `{}`", who)); + } + } + // The program runs again with the directories it asked for. + std::erase_if(e.xpkgDirs, [](const auto& kv) { return kv.first.starts_with("MCPP_XPKG_"); }); + fill_xpkg_env(state, e, m, consumer); + return true; +} + +std::expected +run_answering_requests(PrepareState& state, mcpp::manifest::Manifest& m, + mcpp::build::BuildProgramEnv& e, std::size_t consumer, + std::string_view who, + const std::function()>& run) { + // At most three rounds: a program may learn what it needs only from what + // it received, but a program that asks for something new every time is a + // loop, and a named refusal beats an unbounded one. + e.keepRequestingRun = state.overrides.plan_only; + for (int round = 0;; ++round) { + auto r = run(); + if (!r) return r; + auto again = answer_payload_requests(state, m, e, consumer, who); + if (!again) return std::unexpected(again.error()); + if (!*again) return {}; + if (round == 2) { + refusal::record(refusal::Code::PayloadRequest); + return std::unexpected(std::format( + "the build program of `{}` asked for payloads in three consecutive runs; " + "a program asks for every payload it needs in one run, and runs again " + "with all of them installed", who)); + } + } +} + +void record_tool_decisions(PrepareState& state) { + auto take = [&](const mcpp::manifest::Manifest& man) { + for (auto const& line : man.buildConfig.toolDecisions) { + // `\t\t\t\t[\t]` + std::vector f; + std::size_t at = 0; + while (true) { + auto tab = line.find('\t', at); + f.push_back(line.substr(at, tab == std::string::npos ? std::string::npos : tab - at)); + if (tab == std::string::npos) break; + at = tab + 1; + } + if (f.size() < 3 || f[0].empty()) continue; + SourceDecision d; + d.subject = f[0]; + d.value = f[2]; + d.decidedFor = man.package.name; + const auto& from = f[1]; + if (from == "choice") { + d.cls = SourceClass::Program; + d.originKind = "build-program"; + d.originFile = f.size() > 3 ? f[3] : std::string{}; + d.originLine = f.size() > 4 ? std::atoi(f[4].c_str()) : 0; + if (d.originFile.empty()) d.originKey = "build.mcpp"; + } else if (from == "env") { + d.cls = SourceClass::Custom; + d.originKind = "env"; + d.originKey = f.size() > 3 && !f[3].empty() ? f[3] : std::string("environment"); + } else if (from == "override") { + // THE PAYLOAD'S OWN STATEMENT, AND ITS OWN LINE. The override + // was already reported where the download was skipped, so the + // tool takes that source and is recorded without a second line. + const auto payload = f.size() > 5 ? f[5] : std::string{}; + d.payload = payload; + d.announce = false; + auto it = std::ranges::find(state.sources, "payload:" + payload, + &SourceDecision::subject); + if (it != state.sources.end()) { + d.cls = it->cls; d.originKind = it->originKind; + d.originFile = it->originFile; d.originLine = it->originLine; + d.originKey = it->originKey; + } else { + d.cls = SourceClass::Custom; d.originKind = "env"; + } + } else if (from == "path") { + d.cls = SourceClass::Host; + d.originKind = "host"; + d.originKey = "PATH"; + } else { + // The payload: its own entry carries the source, so this one + // records which member runs it and states no line of its own. + d.cls = SourceClass::Managed; + d.originKind = "graph"; + d.announce = false; + if (f.size() > 5) { + d.payload = f[5]; + auto it = std::ranges::find(state.sources, "payload:" + f[5], + &SourceDecision::subject); + if (it != state.sources.end()) d.cls = it->cls; + } + } + d.considered.push_back(std::format("stated by the build program of `{}` ({})", + man.package.name, from)); + record_source(state, std::move(d)); + } + }; + if (state.m) take(*state.m); + for (std::size_t i = 1; i < state.packages.size(); ++i) take(state.packages[i].manifest); +} + +std::expected step13_sources(PrepareState& state, BuildContext& ctx) { + // THE BUILD TOOLCHAIN, when nothing stated it as a source of its own: a + // managed payload, and how it was chosen. A toolchain named by path or by + // the build program was recorded where it was resolved, and announced + // there, next to the `Resolving toolchain` line. + if (std::ranges::none_of(state.sources, [](auto const& d) { + return d.subject == "toolchain.build"; })) { + SourceDecision d; + d.subject = "toolchain.build"; + d.value = state.tcSpec.value_or(ctx.tc.label()); + d.detail = ctx.tc.label(); + switch (state.tcOrigin) { + case TcOrigin::ManifestToolchain: + case TcOrigin::TargetSection: + d.cls = SourceClass::Pinned; + if (state.tcFromCommandLine) { + d.originKind = "env"; d.originKey = "MCPP_TOOLCHAIN"; + } else if (state.tcFromConsumer) { + d.originKind = "graph"; d.originKey = state.overrides.tool_chain; + } else { + d.originKind = "manifest"; + d.originFile = state.m->sourcePath.string(); + d.originLine = state.m->toolchain.line_for(kCurrentPlatform); + d.originKey = state.tcOrigin == TcOrigin::TargetSection + ? std::format("[target.{}] toolchain", state.overrides.target_triple) + : std::string("[toolchain]"); + } + break; + case TcOrigin::GlobalDefault: + d.cls = SourceClass::Pinned; + d.originKind = "config"; + d.originKey = "`mcpp toolchain default`"; + break; + default: + d.cls = SourceClass::Managed; + d.originKind = state.tcOrigin == TcOrigin::GraphRequirement ? "graph" : "default"; + d.originKey = std::string(tc_origin_name(state.tcOrigin)); + break; + } + d.considered.push_back(ctx.tc.binaryPath.generic_string()); + record_source(state, std::move(d)); + } + record_tool_decisions(state); + if (auto why = managed_only_refusal(state)) return std::unexpected(*why); + ctx.sources = state.sources; + return {}; +} + +std::optional managed_only_refusal(const PrepareState& state) { + const char* env = std::getenv("MCPP_MANAGED_ONLY"); + const bool on = state.overrides.managed_only || (env && *env && std::string_view(env) != "0"); + if (!on) return std::nullopt; + std::string list; + const auto root = state.root ? *state.root : fs::path{}; + for (auto const& d : state.sources) { + if (source_class_is_default(d.cls)) continue; + list += std::format("\n {} ← {} [{}]", subject_shown(d), d.value, + source_tag(d, root)); + } + if (list.empty()) return std::nullopt; + refusal::record(refusal::Code::ManagedOnly); + return std::format( + "--managed-only: this build uses sources that are not the ecosystem's:{}\n" + " Remove the statements above, or build without --managed-only\n" + " (MCPP_MANAGED_ONLY).", list); +} + +std::string sources_summary(const std::vector& sources) { + std::map> byClass; + for (auto const& d : sources) { + if (source_class_is_default(d.cls)) continue; + // A tool whose source is its payload's is already summarised by that + // payload: one statement, one entry. + if (!d.payload.empty() && std::ranges::any_of(sources, [&](auto const& o) { + return o.subject == "payload:" + d.payload; })) + continue; + byClass[d.cls].push_back(subject_shown(d)); + } + std::string out; + for (auto const& [cls, names] : byClass) { + if (!out.empty()) out += "; "; + out += std::format("{}: ", source_class_name(cls)); + for (std::size_t i = 0; i < names.size(); ++i) + out += (i ? ", " : "") + names[i]; + } + return out; +} + +} // namespace mcpp::build diff --git a/src/build/prepare/state.cppm b/src/build/prepare/state.cppm index 147c70377..010cc036a 100644 --- a/src/build/prepare/state.cppm +++ b/src/build/prepare/state.cppm @@ -52,6 +52,7 @@ import mcpp.build.backend; // BuildOptions for the tool sub-build import mcpp.build.ninja; // make_ninja_backend — driving that sub-build import mcpp.config; import mcpp.xlings; +import mcpp.xlings.address_set; // mcpp#755: override checks reuse the unification's claims import mcpp.xlings.runtime_selection; import mcpp.runtime.binding; import mcpp.toolchain.post_install; @@ -564,6 +565,44 @@ struct PrepareState { std::map> dormantFeaturesByConsumer; std::map xlingsWinner; + + // ── Sources (mcpp#755): see sources.cpp ───────────────────────────────── + // The decision record, one entry per subject, and the subjects whose + // `Using` line has been printed (each is printed once per prepare). + std::vector sources; + std::set announcedSources; + // Where an overridden payload comes from, resolved once per package key: + // the environment variable, the project's `[xlings.overrides]`, then + // config.toml. A key with no override maps to nullopt. + struct PayloadOverride { + std::string root; // the directory `xpkg_dir` answers + std::string program; // the program, when the override named one + std::string version; // the version it states, empty when none + SourceClass cls = SourceClass::Custom; + std::string originKind, originFile, originKey; + int originLine = 0; + }; + std::map> payloadOverrideCache; + // Package keys whose payload this build does not install: overridden, or + // declared `provision = "on-request"` and not asked for (yet). And the + // addresses so skipped, for the run-tier record in plan.cpp. + std::set xlingsOverridden; + std::set xlingsDeferred; + std::set xlingsSkipped; + // Package keys a build program asked for, installed on request. + std::set requestedPayloads; + // A toolchain named by path, or stated by the build program (mcpp#755): + // the description the resolver uses, and the decision it is reported as. + std::optional localToolchain; + SourceDecision localToolchainOrigin; + // `[toolchain] = { configure = "build.mcpp" }` and this is the first + // pass: the toolchain resolved is the bootstrap, and the root build + // program runs its toolchain phase once host modules are registered. + bool toolchainConfigure = false; + // The toolchain that compiles and runs build programs, when it is not the + // build toolchain on a native build: `[toolchain] bootstrap`, or the + // toolchain that ran the toolchain phase. Empty otherwise. + std::string bootstrapSpec; std::vector> dep_manifests; std::vector dep_cache_identities; std::map root_git_lock_identities; @@ -815,4 +854,61 @@ provision_xlings_addresses(const mcpp::config::GlobalConfig& cfg, std::string_view label); std::string with_index_cause(std::string msg); +// ── Sources (mcpp#755), defined in sources.cpp ───────────────────────────── +// Record (or replace) the decision for `d.subject`; a non-default source is +// announced once with its `Using` line. +void record_source(PrepareState& state, SourceDecision d); +// The override for a package key, resolved and cached; nullptr when none. +std::expected +payload_override(PrepareState& state, std::string_view key); +// The winners of one unification that this build installs from the registry +// now: overridden and deferred (`on-request`, not asked for) packages are left +// out, recorded, and announced. `onRequest[i]` says whether claim `i` was +// declared `provision = "on-request"`. +std::expected, std::string> +payloads_to_provision(PrepareState& state, + const mcpp::xlings::addrset::Resolution& unified, + std::span claims, + const std::vector& onRequest); +// A dependency that states `[xlings.overrides]` is refused: where a payload +// comes from is the root's statement (mcpp#755). +std::optional dependency_override_refusal(const PrepareState& state); +// `MCPP_XPKG___{DIR,SOURCE,PROGRAM}` for what `owner` and the host +// modules compiled into consumer `consumer`'s program declared. +void fill_xpkg_env(PrepareState& state, mcpp::build::BuildProgramEnv& e, + const mcpp::manifest::Manifest& owner, std::size_t consumer); +// After a run of `who`'s build program into `m`: when it asked for payloads, +// install them in one batch and refill `e`; true when the program must run +// again. Under `plan_only` nothing is installed: a note is recorded and false +// is returned. +std::expected +answer_payload_requests(PrepareState& state, mcpp::manifest::Manifest& m, + mcpp::build::BuildProgramEnv& e, std::size_t consumer, + std::string_view who); +// Run a build program through `run`, answering its payload requests: when a +// run asked for payloads, install them and run it again (at most three runs). +std::expected +run_answering_requests(PrepareState& state, mcpp::manifest::Manifest& m, + mcpp::build::BuildProgramEnv& e, std::size_t consumer, + std::string_view who, + const std::function()>& run); +// The `mcpp:decision=` lines every program of this build stated, as decisions. +void record_tool_decisions(PrepareState& state); +// The toolchain's own decision, the plugins' tool decisions, `--managed-only`, +// and `ctx.sources`: the last step of the record, before `resolution.json`. +std::expected step13_sources(PrepareState& state, BuildContext& ctx); +// A toolchain named by path, and the toolchain phase (local_toolchain.cpp). +std::expected step1_local_toolchain(PrepareState& state); +std::expected +step2_use_local_toolchain(PrepareState& state, const mcpp::toolchain::ToolchainSpec& spec); +std::expected step2_apply_local_toolchain(PrepareState& state); +std::expected step6_toolchain_phase(PrepareState& state); +// The statement and bootstrap spec of a toolchain phase that asked prepare to +// start again, taken once. +std::optional, std::string>> take_toolchain_restart(); +// `--managed-only`: a refusal naming every source that is not the ecosystem's. +std::optional managed_only_refusal(const PrepareState& state); +// The `Finished`-line summary of the non-default sources, empty when none. +std::string sources_summary(const std::vector& sources); + } // namespace mcpp::build diff --git a/src/build/prepare/target_side.cpp b/src/build/prepare/target_side.cpp index 922ca6a94..cc6f9bcaa 100644 --- a/src/build/prepare/target_side.cpp +++ b/src/build/prepare/target_side.cpp @@ -1723,9 +1723,12 @@ static std::expected step9_root_build_program(PrepareState& s const auto namedBeforeRoot = bcRoot.namedRunners; // The root's program is the requested package's. bpEnv.requested = !state.m->package.virtualRoot; - auto bp = mcpp::build::run_build_program( - *state.m, *state.root, host->first, host->second, - state.m->cppStandard, bpEnv); + auto bp = run_answering_requests(state, *state.m, bpEnv, 0, + state.m->package.name, [&] { + return mcpp::build::run_build_program( + *state.m, *state.root, host->first, host->second, + state.m->cppStandard, bpEnv); + }); if (!bp && !state.overrides.plan_only) { return std::unexpected(bp.error()); } @@ -2132,9 +2135,12 @@ static std::expected step9_member_build_programs(PrepareState const auto runnerN = bc.runner.size(); auto namedBefore = bc.namedRunners; const bool exclusiveBefore = bc.runExclusive; - auto bp = mcpp::build::run_build_program( - pkg.manifest, pkg.root, host->first, host->second, - pkg.manifest.cppStandard, bpEnv, made.compiled ? &made : nullptr); + auto bp = run_answering_requests(state, pkg.manifest, bpEnv, i, + pkg.manifest.package.name, [&] { + return mcpp::build::run_build_program( + pkg.manifest, pkg.root, host->first, host->second, + pkg.manifest.cppStandard, bpEnv, made.compiled ? &made : nullptr); + }); if (!bp) { if (!state.overrides.plan_only) return std::unexpected(std::format( diff --git a/src/build/prepare/toolchain.cpp b/src/build/prepare/toolchain.cpp index 8f587677b..f0af74ec2 100644 --- a/src/build/prepare/toolchain.cpp +++ b/src/build/prepare/toolchain.cpp @@ -351,6 +351,10 @@ static std::expected step1_define_early_toolchain_closures(Pr state.tcOrigin = TcOrigin::GlobalDefault; } } + // A toolchain named by path, the toolchain phase's statement, and the + // bootstrap toolchain (mcpp#755). Rewrites `tcSpec` into the one spelling + // every later reader parses: a managed spec, or `path:`. + if (auto r = step1_local_toolchain(state); !r) return std::unexpected(r.error()); // ─── Windows first run without Visual Studio ──────────────────────── // The host triple on Windows is MSVC-ABI, so the historical default @@ -2205,6 +2209,9 @@ std::expected phase2_define_toolchain_resolver(PrepareState& step2_use_installed_pin(state, ctx); } else if (state.tcSpecIsMsvc) { if (auto r = step2_use_system_msvc(state); !r) return std::unexpected(r.error()); + } else if (ctx.parsedSpec && !ctx.parsedSpec->localRoot.empty()) { + if (auto r = step2_use_local_toolchain(state, *ctx.parsedSpec); !r) + return std::unexpected(r.error()); } else if (ctx.parsedSpec) { if (auto r = step2_resolve_explicit_spec(state, ctx); !r) return std::unexpected(r.error()); } else if (state.tcSpec.has_value() && *state.tcSpec == "system") { @@ -2261,6 +2268,7 @@ std::expected phase2_define_toolchain_resolver(PrepareState& } if (auto r = step2_detect_toolchain(state); !r) return std::unexpected(r.error()); + if (auto r = step2_apply_local_toolchain(state); !r) return std::unexpected(r.error()); if (auto r = step2_retarget_for_retargetable_driver(state); !r) return std::unexpected(r.error()); if (auto r = step2_bind_msvc_toolset(state); !r) return std::unexpected(r.error()); step2_windows_runtime_identity(state); diff --git a/src/build/prepare/xlings.cpp b/src/build/prepare/xlings.cpp index 3229308af..7a2a806c2 100644 --- a/src/build/prepare/xlings.cpp +++ b/src/build/prepare/xlings.cpp @@ -115,6 +115,8 @@ step3_define_host_tc_closures_and_refresh_index(PrepareState& state) { // sub-build is handed when its package names no toolchain (#710), so the // tool is built by the compiler the store key records. state.host_spec_for_build_program = [&]() -> std::string { + // The bootstrap toolchain, when one was stated (mcpp#755). + if (!state.bootstrapSpec.empty()) return state.bootstrapSpec; if (!state.tcSpec) return {}; if (state.overrides.target_triple.empty()) return *state.tcSpec; return (state.tcOrigin == TcOrigin::TargetPin && state.hostSpecBeforeRowPin.has_value() @@ -180,7 +182,10 @@ step3_define_host_tc_closures_and_refresh_index(PrepareState& state) { // The CROSS branch below is a different question and deliberately // unchanged: there `explicit_compiler` is empty because NO host // toolchain was resolved at all, and its classified refusal is correct. - if (state.overrides.target_triple.empty()) + // A native build compiles its build programs with its own toolchain, + // unless a bootstrap toolchain was stated (mcpp#755): then the + // resolution below serves it, exactly as it serves a cross build. + if (state.overrides.target_triple.empty() && state.bootstrapSpec.empty()) return std::pair{ state.explicit_compiler.empty() ? state.tc->binaryPath : state.explicit_compiler, as_host(*state.tc)}; @@ -294,8 +299,25 @@ step3_define_host_tc_closures_and_refresh_index(PrepareState& state) { auto htc = mcpp::toolchain::detect( frontend, state.runtimePayload, state.runtimeBindingSnapshot.contractHash); if (!htc) return std::unexpected(htc.error().message); - mcpp::ui::info("Resolved", std::format( - "host toolchain for build.mcpp: {}", htc->label())); + if (state.overrides.target_triple.empty() && !state.bootstrapSpec.empty()) { + // The bootstrap toolchain of a native build (mcpp#755): said with + // its own verb, because the line after it names another toolchain. + mcpp::ui::info("Bootstrap", std::format( + "{} → {}", hostSpecText, + mcpp::ui::shorten_path(frontend, mcpp::fetcher::make_path_ctx(*cfgH, *state.root)))); + SourceDecision d; + d.subject = "toolchain.bootstrap"; + d.value = hostSpecText; + d.cls = SourceClass::Pinned; + d.originKind = state.overrides.bootstrap_spec.empty() ? "manifest" : "build-program"; + d.originKey = state.overrides.bootstrap_spec.empty() ? "[toolchain] bootstrap" + : "the toolchain that ran the toolchain phase"; + d.considered.push_back(frontend.generic_string()); + record_source(state, std::move(d)); + } else { + mcpp::ui::info("Resolved", std::format( + "host toolchain for build.mcpp: {}", htc->label())); + } state.hostTcCache = std::pair{frontend, *htc}; return std::pair{state.hostTcCache->first, as_host(state.hostTcCache->second)}; }; @@ -461,6 +483,45 @@ std::expected phase3_xlings_before_graph(PrepareState& state) penv.workspace.emplace_back(entry.target, entry.pin()); } } + // ONE PACKAGE, ONE VERSION, INSIDE ONE MANIFEST TOO. The + // conditional merge already unified the two tool AXES by package; + // what it cannot see is `[xlings.workspace]` and + // `[feature-xlings.]` naming one package at two versions, which + // reaches here as two addresses and used to install both. + std::vector rootClaims; + std::vector rootOnRequest; + for (auto const& spec : applicable_xlings_addresses( + *state.runtimeOwnerManifest, + feature_closure(*state.runtimeOwnerManifest, + parse_feature_request(state.overrides.features)), + state.toolPurpose, /*isRoot=*/true)) { + rootClaims.push_back({spec, "this project", 0}); + rootOnRequest.push_back(state.runtimeOwnerManifest->xlings.on_request(spec)); + } + auto rootUnified = mcpp::xlings::addrset::unify(rootClaims); + if (!rootUnified) { + refusal::record(refusal::Code::ToolVersionConflict); + return std::unexpected(rootUnified.error()); + } + for (auto const& note : rootUnified->overrides) + mcpp::diag::warning("xlings/version-override", note); + // What the registry installs for the project itself: an + // overridden package and one declared on request are left out and + // recorded (mcpp#755). Decided BEFORE the environment file is + // written, because an overridden package is not part of it. + auto rootInstall = payloads_to_provision(state, *rootUnified, rootClaims, + rootOnRequest); + if (!rootInstall) return std::unexpected(rootInstall.error()); + std::vector declaredDeps = std::move(*rootInstall); + if (materializeRootRuntime && !state.xlingsOverridden.empty()) { + namespace addrset = mcpp::xlings::addrset; + std::erase_if(penv.deps, [&](const std::string& a) { + return state.xlingsOverridden.contains(addrset::package_key(a)); + }); + std::erase_if(penv.workspace, [&](const auto& kv) { + return state.xlingsOverridden.contains(addrset::package_key(kv.first)); + }); + } // Two halves, two roots. The custom-indices half belongs to // `state.workRoot`, where this invocation writes. The runtime- // environment half (`penv`: deps/subos/workspace) belongs to the @@ -524,28 +585,6 @@ std::expected phase3_xlings_before_graph(PrepareState& state) // declared. The graph's own declarations are provisioned after // resolution, which is the first moment they are known — see the // second pass near `xlingsDepBinDirs`. - // ONE PACKAGE, ONE VERSION, INSIDE ONE MANIFEST TOO. The - // conditional merge already unified the two tool AXES by package; - // what it cannot see is `[xlings.workspace]` and - // `[feature-xlings.]` naming one package at two versions, which - // reaches here as two addresses and used to install both. - std::vector rootClaims; - for (auto const& spec : applicable_xlings_addresses( - *state.runtimeOwnerManifest, - feature_closure(*state.runtimeOwnerManifest, - parse_feature_request(state.overrides.features)), - state.toolPurpose, /*isRoot=*/true)) - rootClaims.push_back({spec, "this project", 0}); - auto rootUnified = mcpp::xlings::addrset::unify(rootClaims); - if (!rootUnified) { - refusal::record(refusal::Code::ToolVersionConflict); - return std::unexpected(rootUnified.error()); - } - for (auto const& note : rootUnified->overrides) - mcpp::diag::warning("xlings/version-override", note); - std::vector declaredDeps; - for (auto const& w : rootUnified->winners) - declaredDeps.push_back(w.address); if (materializeRootRuntime && !declaredDeps.empty()) { if (auto pv = provision_xlings_addresses( **cfg2, declaredDeps, state.runtimeSelection.ownerRoot, diff --git a/src/build/progress.cppm b/src/build/progress.cppm index cde5b0043..206079643 100644 --- a/src/build/progress.cppm +++ b/src/build/progress.cppm @@ -222,6 +222,12 @@ void checking(); // `Finished` (design §4.5): the whole command's time, and how it was spent. // Written once per command; a later call does nothing. void finished(std::string_view profile, std::string_view descriptor); +// The sources of a build directory that are not the ecosystem's default +// (mcpp#755), read from `/sources.summary`, which prepare writes. +// Merged across the configurations of a command and appended to `Finished`; +// a directory without the file adds nothing, so a default build's line is +// unchanged. +void note_sources(const std::filesystem::path& buildDir); // A command that builds several configurations writes one `Finished`, after // all of them: `finished` then only records what it was given, and // `finish_deferred` writes it. @@ -742,6 +748,8 @@ struct Report { std::optional> deferredFinish; // `Finished` was written: a command states it once. bool finishedWritten = false; + // Non-default sources by class, in the order first noted (mcpp#755). + std::vector>> sources; // The status row's screen (revision 3, §5.9 to §5.13): an animation fed // by the build, or none. std::unique_ptr animation; @@ -1355,6 +1363,33 @@ void finish_deferred() { if (f) finished(f->first, f->second); } +void note_sources(const std::filesystem::path& buildDir) { + std::ifstream f(buildDir / "sources.summary", std::ios::binary); + std::string line; + if (!f || !std::getline(f, line) || line.empty()) return; + auto& r = report(); + std::lock_guard lock(r.m); + // `custom: a, b; host: c` + std::size_t at = 0; + while (at < line.size()) { + auto end = line.find("; ", at); + auto group = line.substr(at, end == std::string::npos ? std::string::npos : end - at); + at = end == std::string::npos ? line.size() : end + 2; + auto colon = group.find(": "); + if (colon == std::string::npos) continue; + auto cls = group.substr(0, colon); + auto it = std::ranges::find(r.sources, cls, &decltype(r.sources)::value_type::first); + if (it == r.sources.end()) { r.sources.emplace_back(cls, std::vector{}); it = r.sources.end() - 1; } + std::size_t n = colon + 2; + while (n < group.size()) { + auto comma = group.find(", ", n); + auto name = group.substr(n, comma == std::string::npos ? std::string::npos : comma - n); + n = comma == std::string::npos ? group.size() : comma + 2; + if (std::ranges::find(it->second, name) == it->second.end()) it->second.push_back(name); + } + } +} + void finished(std::string_view profile, std::string_view descriptor) { auto& r = report(); std::string detail; @@ -1398,6 +1433,20 @@ void finished(std::string_view profile, std::string_view descriptor) { mcpp::ui::format_duration(ms(longest))); } } + // THE NON-DEFAULT SOURCES, LAST (mcpp#755): the one place a reader who + // missed the `Using` lines learns that this build was not the ecosystem's + // default. + { + std::lock_guard lock(r.m); + std::string src; + for (auto const& [cls, names] : r.sources) { + if (names.empty()) continue; + if (!src.empty()) src += "; "; + src += cls + ": "; + for (std::size_t i = 0; i < names.size(); ++i) src += (i ? ", " : "") + names[i]; + } + if (!src.empty()) detail += (detail.empty() ? "" : " · ") + src; + } // `Finished` ends the report: the region is erased before it, and not // drawn again below it. std::string played; diff --git a/src/build/refusal.cppm b/src/build/refusal.cppm index 299314181..eb8762855 100644 --- a/src/build/refusal.cppm +++ b/src/build/refusal.cppm @@ -162,6 +162,15 @@ enum class Code { // about a toolset that cannot deliver a copy: here a copy is asked for // where the contract says there is none. CrtDeclaredUnderHostCoupled, + // Sources (mcpp#755). `--managed-only` met a source that is not the + // ecosystem's; an override names a path that does not exist, or is stated + // by a dependency; a build program asked for a payload no manifest + // declared on request; a toolchain named by path, or stated by the + // toolchain phase, cannot be used. + ManagedOnly, + PayloadOverride, + PayloadRequest, + LocalToolchain, Other, // a refusal that has not been given a code yet }; @@ -210,6 +219,10 @@ constexpr std::string_view name(Code c) { return "msvc-redist-unavailable"; case Code::CrtDeclaredUnderHostCoupled: return "crt-declared-under-host-coupled"; + case Code::ManagedOnly: return "managed-only"; + case Code::PayloadOverride: return "payload-override"; + case Code::PayloadRequest: return "payload-request"; + case Code::LocalToolchain: return "local-toolchain"; case Code::Other: return "other"; } return "other"; diff --git a/src/cli.cppm b/src/cli.cppm index 790d17dc8..cf80a8361 100644 --- a/src/cli.cppm +++ b/src/cli.cppm @@ -186,6 +186,11 @@ int run(int argc, char** argv) { // half that reproducibility needs first. else if (a == "--locked" || a == "--frozen") mcpp::platform::env::set("MCPP_LOCKED", "1"); + // `--managed-only` (mcpp#755) rides the same channel: its consumer is + // the decision record at the end of prepare, for every command that + // prepares, and the fast paths that would skip that record. + else if (a == "--managed-only") + mcpp::platform::env::set("MCPP_MANAGED_ONLY", "1"); // --jobs rides the same side channel as --offline, for the same reason // recorded there: its consumer is deep in mcpp.build.execute and // threading a parameter down would touch every caller in between. @@ -347,6 +352,9 @@ int run(int argc, char** argv) { .option(cl::Option("frozen") .help("Alias for --locked") .global()) + .option(cl::Option("managed-only") + .help("Fail if a toolchain, payload or plugin tool does not come from the ecosystem") + .global()) // Answers "what do you speak" without spawning a command that might // fail. An optimisation, NOT the client's detection rule: on any mcpp // predating it this is itself an unknown option, so a client must @@ -563,8 +571,9 @@ int run(int argc, char** argv) { .help("Keep unrecorded directories written more recently than this, e.g. 12h, 3d (default 1d; 0 keeps none; implies --stale)")) .action(wrap_rc(cmd_clean))) .subcommand(cl::App("why") - .description("Explain how the toolchain / runtime / deps / runners were resolved") - .arg(cl::Arg("topic").help("toolchain | runtime | deps | runners (default: all)")) + .description("Explain how the toolchain / runtime / deps / runners / sources were resolved") + .arg(cl::Arg("topic").help("toolchain | runtime | deps | runners | sources | tool | payload (default: all)")) + .arg(cl::Arg("subject").help("for `tool` and `payload`: the name to look up")) // `--target` / `--toolchain` make this a QUERY rather than a // report on the current directory's default: "what would a build // for THIS pair resolve to" is the question the target matrix asks @@ -1367,6 +1376,10 @@ int run(int argc, char** argv) { {"why toolchain", {Effect::InitMcppHome, Effect::ReadProject, Effect::Network, Effect::WriteGlobalCache, Effect::ExecBuildScript}}, + // The same prepare, read for its decision record (mcpp#755). + {"why sources", {Effect::InitMcppHome, Effect::ReadProject, + Effect::Network, Effect::WriteGlobalCache, + Effect::ExecBuildScript}}, // The same resolution as `why toolchain` and as `build // --configure-only`, and therefore the same declaration, with one // difference that is the command's reason to exist: no diff --git a/src/cli/cmd_self.cppm b/src/cli/cmd_self.cppm index 7a7b2d54a..8ba9b3078 100644 --- a/src/cli/cmd_self.cppm +++ b/src/cli/cmd_self.cppm @@ -104,10 +104,14 @@ export int cmd_why(const mcpplibs::cmdline::ParsedArgs& parsed) { return 2; } const std::string topic = parsed.positional(0); + if (topic == "sources" || topic == "tool" || topic == "payload") + return mcpp::doctor::why_sources_json(topic, parsed.positional(1), + parsed.option_or_empty("features").value()); if (!topic.empty() && topic != "toolchain") { std::println(stderr, - "error: --format json is defined for `mcpp why toolchain`; " - "'{}' has no machine-readable shape yet", topic); + "error: --format json is defined for `mcpp why toolchain` and " + "`mcpp why sources|tool|payload`; '{}' has no machine-readable " + "shape yet", topic); return 2; } return mcpp::doctor::why_toolchain_json( @@ -115,7 +119,8 @@ export int cmd_why(const mcpplibs::cmdline::ParsedArgs& parsed) { parsed.option_or_empty("toolchain").value()); } return mcpp::doctor::why_report(parsed.positional(0), - parsed.option_or_empty("features").value()); + parsed.option_or_empty("features").value(), + parsed.positional(1)); } // Also called directly by the dispatcher for the legacy `--explain CODE` form. diff --git a/src/config.cppm b/src/config.cppm index ed8c05412..6f05f7c46 100644 --- a/src/config.cppm +++ b/src/config.cppm @@ -129,6 +129,19 @@ struct GlobalConfig { // target = triple are orthogonal axes). Empty means host. std::string defaultTarget; + // From config.toml [xlings.overrides] (mcpp#755): where a declared xlings + // payload comes from on THIS MACHINE. The same entries a root manifest's + // `[xlings.overrides]` writes, below it in precedence: the environment + // variable, then the project, then this file. Keyed by package identity, + // `:` with the namespace defaulted to `xim`. + struct PayloadOverride { + std::string kind = "path"; // "path" | "program" | "root" + std::string value; + std::string version; + int line = 0; + }; + std::map payloadOverrides; + // Resolved xlings home (registryDir unless overridden) std::filesystem::path xlingsHome() const { return xlingsHomeOverride.empty() ? registryDir : xlingsHomeOverride; @@ -718,6 +731,36 @@ std::expected load_or_init( cfg.defaultJobs = doc->get_int("build.default_jobs").value_or(0); cfg.defaultToolchain = doc->get_string("toolchain.default").value_or(""); cfg.defaultTarget = doc->get_string("toolchain.default_target").value_or(""); + // [xlings.overrides] (mcpp#755). Read in the manifest's two shapes -- a + // path, or a table of `program` or `root` and `version` -- and refused by + // name when it is neither, as a malformed config.toml key always is. + if (auto* ot = doc->get_table("xlings.overrides")) { + for (auto const& [key, val] : *ot) { + GlobalConfig::PayloadOverride o; + o.line = static_cast(val.position.line); + if (val.is_string()) { + o.value = val.as_string(); + } else if (val.is_table()) { + for (auto const& [k, v] : val.as_table()) { + if ((k != "program" && k != "root" && k != "version") || !v.is_string()) + return std::unexpected(ConfigError{std::format( + "config.toml [xlings.overrides] {}: '{}' is not a key of an " + "override; an override is a path, or a table of `program` " + "or `root` and an optional `version`", key, k)}); + if (k == "version") o.version = v.as_string(); + else { o.kind = k; o.value = v.as_string(); } + } + } + if (o.value.empty()) + return std::unexpected(ConfigError{std::format( + "config.toml [xlings.overrides] {}: expected a path, or a table " + "naming `program` or `root`", key)}); + auto colon = key.find(':'); + std::string ident = colon == std::string::npos ? "xim:" + key : key; + if (auto at = ident.find('@'); at != std::string::npos) ident.resize(at); + cfg.payloadOverrides.insert_or_assign(std::move(ident), std::move(o)); + } + } // [log] section — re-initialize logger with config values { diff --git a/src/doctor.cppm b/src/doctor.cppm index 2e9629b07..5080636a0 100644 --- a/src/doctor.cppm +++ b/src/doctor.cppm @@ -1201,8 +1201,100 @@ export int why_toolchain_json(std::string_view target, std::string_view tcSpec) return 0; } -export int why_report(const std::string& topic, const std::string& features) { +// WHERE EACH SOURCE CAME FROM (mcpp#755): the decision record of a prepare, +// filtered to a topic. `sources` lists every subject; `tool ` the plugin +// tools whose name or module contains ``; `payload ` the payloads +// whose identity contains ``. Read from the record itself, so it is what +// the build reported and `resolution.json` holds. +namespace { +bool source_matches(const mcpp::build::SourceDecision& d, std::string_view topic, + std::string_view subject) { + if (topic == "tool" && !d.subject.starts_with("tool:")) return false; + if (topic == "payload" && !d.subject.starts_with("payload:")) return false; + return subject.empty() || d.subject.contains(subject); +} + +nlohmann::json source_json(const mcpp::build::SourceDecision& d) { + return { + {"subject", d.subject}, {"value", d.value}, + {"class", std::string(mcpp::build::source_class_name(d.cls))}, + {"origin", {{"kind", d.originKind}, {"file", d.originFile}, + {"line", d.originLine}, {"key", d.originKey}}}, + {"decidedFor", d.decidedFor}, {"considered", d.considered}, + }; +} + +void print_sources(const std::vector& sources, + std::string_view topic, std::string_view subject, + const std::filesystem::path& root) { + std::println("sources:"); + bool any = false; + for (auto const& d : sources) { + if (!source_matches(d, topic, subject)) continue; + any = true; + std::println(" {} {}", d.subject, d.value); + std::println(" {}{}", mcpp::build::source_tag(d, root), + d.decidedFor.empty() ? std::string{} : " for " + d.decidedFor); + for (auto const& c : d.considered) std::println(" considered: {}", c); + } + if (!any) std::println(" (none{})", subject.empty() ? "" : std::format(" matching '{}'", subject)); +} +} // namespace + +export int why_sources_json(std::string_view topic, std::string_view subject, + const std::string& features) { + const bool wasQuiet = mcpp::ui::is_quiet(); + mcpp::ui::set_quiet(true); + struct QuietGuard { + bool prev; + ~QuietGuard() { mcpp::ui::set_quiet(prev); } + } quietGuard{wasQuiet}; + (void)mcpp::build::refusal::take(); + mcpp::build::BuildOverrides ov; + ov.features = features; + auto ctx = mcpp::build::prepare_build(/*print_fingerprint=*/false, + /*includeDevDeps=*/false, {}, ov); + nlohmann::json data; + std::vector diags; + data["topic"] = std::string(topic); + data["subject"] = std::string(subject); + if (!ctx) { + const auto code = mcpp::build::refusal::take(); + data["status"] = "refused"; + data["reason"] = std::string(mcpp::build::refusal::name(code)); + data["sources"] = nlohmann::json::array(); + diags.push_back({.code = std::format("prepare.{}", mcpp::build::refusal::name(code)), + .severity = mcpp::wire::Severity::Error, + .message = ctx.error()}); + } else { + data["status"] = "ok"; + data["reason"] = "none"; + nlohmann::json list = nlohmann::json::array(); + for (auto const& d : ctx->sources) + if (source_matches(d, topic, subject)) list.push_back(source_json(d)); + data["sources"] = std::move(list); + } + mcpp::wire::emit({ + .kind = "mcpp.why.sources", + .effects = { mcpp::wire::Effect::ReadProject }, + .data = data, + .diagnostics = diags, + }); + return 0; +} + +export int why_report(const std::string& topic, const std::string& features, + const std::string& subject = {}) { const bool all = topic.empty() || topic == "all"; + static constexpr std::string_view kTopics[] = { + "all", "toolchain", "runtime", "deps", "runners", "sources", "tool", "payload", + }; + if (!topic.empty() && std::ranges::find(kTopics, std::string_view(topic)) == std::end(kTopics)) { + std::println(stderr, "error: '{}' is not a topic of `mcpp why`; the topics are " + "toolchain, runtime, deps, runners, sources, tool and " + "payload ", topic); + return 2; + } // The dedicated runtime view is a pure interpreter of the build's stored // facts: no dependency resolution, xlings invocation, hardware query, or @@ -1224,7 +1316,16 @@ export int why_report(const std::string& topic, const std::string& features) { std::println("toolchain: {}", tc.label()); std::println(" abi(libc)={} cxxstdlib={} arch={} os={} triple={}", prof.libc, prof.cxxStdlib, prof.arch, prof.os, tc.targetTriple); - std::println(" reason: [toolchain] in mcpp.toml if set, else platform-native default"); + // The reason the decision record states (mcpp#755), not a sentence + // describing every way a toolchain can be chosen. + if (auto it = std::ranges::find(ctx->sources, std::string("toolchain.build"), + &mcpp::build::SourceDecision::subject); + it != ctx->sources.end()) + std::println(" source: {}", mcpp::build::source_tag(*it, ctx->projectRoot)); + if (!ctx->compilerChoice.origin.empty()) + std::println(" reason: {}{}", ctx->compilerChoice.origin, + ctx->compilerChoice.requiredBy.empty() ? std::string{} + : " (" + ctx->compilerChoice.requiredBy + ")"); if (!ctx->manifest.package.platforms.empty()) { std::string ps; for (auto& p : ctx->manifest.package.platforms) { @@ -1245,6 +1346,9 @@ export int why_report(const std::string& topic, const std::string& features) { std::println(" declared accelerators: {} (CI matrix hint)", as); } } + if (all || topic == "sources" || topic == "tool" || topic == "payload") + print_sources(ctx->sources, all ? std::string_view("sources") : std::string_view(topic), + subject, ctx->projectRoot); if (all) (void)print_stored_runtime_resolution(); // WHERE A NAMED RUNNER BECOMES DISCOVERABLE. // diff --git a/src/project.cppm b/src/project.cppm index 643fbdbac..7fd73474e 100644 --- a/src/project.cppm +++ b/src/project.cppm @@ -247,18 +247,24 @@ export void inherit_workspace_xlings(mcpp::manifest::Manifest& member, to.workspace.try_emplace(pin->first, pin->second); if (auto w = from.depWhen.find(a); w != from.depWhen.end()) to.depWhen.try_emplace(a, w->second); + if (from.onRequest.contains(a)) to.onRequest.insert(a); } to.deps.insert(to.deps.begin(), taken.begin(), taken.end()); + // `[xlings.overrides]` states where a payload comes from on THIS + // machine, so a member built as the root of its own build takes the + // workspace's statement unless it makes its own (mcpp#755). + for (auto const& [pkg, o] : from.overrides) to.overrides.try_emplace(pkg, o); }; take(workspace.xlings, member.xlings); std::vector rows; for (auto const& cc : workspace.conditionalConfigs) { - if (cc.xlings.deps.empty()) continue; + if (cc.xlings.deps.empty() && cc.xlings.overrides.empty()) continue; mcpp::manifest::ConditionalConfig row; row.predicate = cc.predicate; take(cc.xlings, row.xlings); - if (!row.xlings.deps.empty()) rows.push_back(std::move(row)); + if (!row.xlings.deps.empty() || !row.xlings.overrides.empty()) + rows.push_back(std::move(row)); } member.conditionalConfigs.insert(member.conditionalConfigs.begin(), std::make_move_iterator(rows.begin()), diff --git a/src/toolchain/gcc.cppm b/src/toolchain/gcc.cppm index 5ea3fbb73..79e38c5fb 100644 --- a/src/toolchain/gcc.cppm +++ b/src/toolchain/gcc.cppm @@ -168,6 +168,14 @@ std::filesystem::path binutils_prefix_dir(const Toolchain& tc) { // at all, so answering with one would describe a flag nobody passes. if (tc.compiler != CompilerId::GCC) return {}; if (is_musl_target(tc) || is_mingw_target(tc)) return {}; + // A GCC named by path (mcpp#755) finds its own assembler and linker, as + // every self-contained GCC does; it is pointed elsewhere only when the + // project stated where `as` (or `ld`) is. + if (!tc.localRoot.empty()) { + if (auto* as = tc.tool_override("as")) return as->parent_path(); + if (auto* ld = tc.tool_override("ld")) return ld->parent_path(); + return {}; + } if (auto bin = find_binutils_bin(tc.binaryPath)) return *bin; return {}; } diff --git a/src/toolchain/registry.cppm b/src/toolchain/registry.cppm index 0beb0aa65..7b17f718a 100644 --- a/src/toolchain/registry.cppm +++ b/src/toolchain/registry.cppm @@ -103,7 +103,15 @@ struct ToolchainSpec { // `msvc@` takes an installed toolset of that version first. bool ecosystemOnly = false; + // A TOOLCHAIN NAMED BY PATH (mcpp#755): `path:`, the spelling a + // `[toolchain]` table and `MCPP_TOOLCHAIN=path:` reach the resolver + // as. Empty for every managed spec. The family is the drivers' (a + // `clang++` is llvm, a `g++` gcc) and the version is unknown until the + // driver is probed, so nothing here is a payload address. + std::filesystem::path localRoot; + std::string spec_str() const { + if (!localRoot.empty()) return "path:" + localRoot.generic_string(); auto base = std::format("{}@{}", payloadName.empty() ? family_name(family) : std::string_view(payloadName), @@ -113,6 +121,8 @@ struct ToolchainSpec { // "gcc@16.1.0" or "gcc@16.1.0 → x86_64-windows-gnu" — user-facing. std::string display() const { + if (!localRoot.empty()) + return std::format("{} at {}", family_name(family), localRoot.generic_string()); if (target.empty()) return spec_str(); return std::format("{} → {}", spec_str(), target.str()); } @@ -524,10 +534,48 @@ std::filesystem::path derive_c_compiler_path(const std::filesystem::path& cxxPat return parent / (cc_stem + ext.string()); } +// The family of a toolchain named by path, from the drivers in `/bin`: +// any `*clang++` is llvm, any `*g++` gcc, llvm first when both are present +// (an LLVM tree may carry a `g++` compatibility link; a GCC tree carries no +// clang). Nothing is run. +std::optional family_of_local_toolchain(const std::filesystem::path& root) { + std::error_code ec; + bool clang = false, gnu = false; + for (auto const& e : std::filesystem::directory_iterator(root / "bin", ec)) { + auto name = e.path().filename().string(); + if (name.ends_with(".exe")) name.resize(name.size() - 4); + if (name.ends_with("clang++")) clang = true; + else if (name.ends_with("g++")) gnu = true; + } + if (clang) return Family::Llvm; + if (gnu) return Family::Gcc; + return std::nullopt; +} + std::expected parse_toolchain_spec(std::string compilerArg, std::string versionArg, bool requireCompiler) { + // `path:` (mcpp#755): the family from the drivers present; the + // version, the triple and the rest from probing them, later. + if (compilerArg.starts_with("path:")) { + ToolchainSpec spec; + spec.localRoot = std::filesystem::path(compilerArg.substr(5)).lexically_normal(); + if (spec.localRoot.empty()) + return std::unexpected(std::string("'path:' names no directory")); + auto fam = family_of_local_toolchain(spec.localRoot); + if (!fam) + return std::unexpected(std::format( + "'{}' has no C++ driver in bin/ (a `clang++` or a `g++`, optionally with a " + "prefix); a toolchain named by path keeps its drivers in `/bin`", + spec.localRoot.generic_string())); + spec.family = *fam; + return spec; + } + if (compilerArg.starts_with("configure:")) + return std::unexpected(std::string( + "`configure = \"build.mcpp\"` is answered by the root build program's " + "toolchain phase, which has not stated a toolchain here")); if (auto at = compilerArg.find('@'); at != std::string::npos) { if (versionArg.empty()) versionArg = compilerArg.substr(at + 1); compilerArg = compilerArg.substr(0, at); @@ -1454,6 +1502,21 @@ std::filesystem::path binutils_tool(const Toolchain& tc, std::string_view name) std::error_code ec; auto dir = tc.binaryPath.parent_path(); + // A TOOL STATED BY ROLE WINS (mcpp#755): `tools = { ar = ... }` is the + // project saying where this one is, because the layout does not have it. + if (auto* o = tc.tool_override(name)) return *o; + // A toolchain named by path keeps its tools beside the driver, under its + // prefix if it has one: `ar`, then the LLVM spelling `llvm-ar`. + if (!tc.localRoot.empty()) { + for (auto const& cand : {tc.toolPrefix + std::string(name), + std::string("llvm-") + std::string(name), + std::string(name)}) { + auto p = dir / (cand + std::string(mcpp::platform::exe_suffix)); + if (std::filesystem::exists(p, ec)) return p; + } + return {}; + } + // Clang ships the whole family as `llvm-` beside the frontend. if (is_clang(tc)) { auto llvmTool = dir / (std::string("llvm-") + std::string(name) diff --git a/src/ui.cppm b/src/ui.cppm index 0feb5c489..374446de9 100644 --- a/src/ui.cppm +++ b/src/ui.cppm @@ -64,6 +64,14 @@ void result(std::string_view verb, std::string_view message); // Cyan verb (Updating, Downloading, Cleaned), on the narration stream. void info(std::string_view verb, std::string_view message); +// A SOURCE THAT IS NOT THE ECOSYSTEM'S DEFAULT (mcpp#755): `Using`, the thing +// and where it came from, then its tag (`[custom · mcpp.toml:22]`) dimmed. A +// cyan verb for a source a project or machine stated; yellow for one found on +// the host with a version nobody stated. The tag carries the whole meaning, +// so the line reads the same without colour. +void source(std::string_view verb, std::string_view message, std::string_view tag, + bool host); + // Bold green Finished line, on the narration stream, preceded by a blank line // when the command narrated a line before it (design §4.5). // `descriptor` annotates the profile's actual effect (e.g. "optimized", @@ -751,6 +759,15 @@ void info(std::string_view verb, std::string_view message) { narrate(verb_line(kBrightCyan, verb, message) + "\n"); } +void source(std::string_view verb, std::string_view message, std::string_view tag, + bool host) { + if (g_quiet) return; + init(); + const auto bracket = std::format("[{}]", tag); + narrate(verb_line(host ? kYellow : kBrightCyan, verb, + std::format("{} {}", message, with_color(kDim, bracket))) + "\n"); +} + void finished(std::string_view profile, std::chrono::milliseconds elapsed, std::string_view descriptor, std::string_view detail) { if (g_quiet) return; diff --git a/src/wire.cppm b/src/wire.cppm index 63fce2479..9b384ab7e 100644 --- a/src/wire.cppm +++ b/src/wire.cppm @@ -68,7 +68,7 @@ inline constexpr int kEnvelopeVersion = 1; // from what comes back. struct KindVersion { std::string_view kind; int version; }; -inline constexpr std::array kKinds{{ +inline constexpr std::array kKinds{{ {"mcpp.env", 1}, {"mcpp.xpkg", 1}, {"mcpp.cache", 1}, @@ -77,6 +77,10 @@ inline constexpr std::array kKinds{{ // the driver, the triple, the C-library model, and either `ok` or a refusal // whose `reason` is a token from mcpp.build.refusal. {"mcpp.why.toolchain", 1}, + // `mcpp why sources|tool|payload --format json` (mcpp#755): the decision + // record of a prepare -- where the toolchain, each payload and each plugin + // tool came from, its class and origin -- filtered to the topic. + {"mcpp.why.sources", 1}, // `mcpp toolchain list --format json`: which toolchains are installed and // which target rows this host serves, with their status. {"mcpp.toolchain.list", 1}, diff --git a/src/xlings/address_set.cppm b/src/xlings/address_set.cppm index b2fb05bef..2c2570624 100644 --- a/src/xlings/address_set.cppm +++ b/src/xlings/address_set.cppm @@ -84,6 +84,19 @@ std::string version_of(std::string_view address); // duplicate install it would be preventing. std::expected unify(std::span claims); +// AN OVERRIDE IS CHECKED THE WAY A WINNER IS (mcpp#755). `[xlings.overrides]` +// replaces where a package comes from, not what the declarations require of +// it: a version the override states is compared with every requirement a +// claim on `key` made, and a version that fails one is refused naming both +// sides. An override that states no version cannot be compared, which is not +// a refusal -- `requirements_for` lists what went unchecked, for the note. +std::optional override_violation(std::span claims, + std::string_view key, + std::string_view version, + std::string_view statedBy); +std::vector requirements_for(std::span claims, + std::string_view key); + } // namespace mcpp::xlings::addrset // ── implementation ────────────────────────────────────────────────────────── @@ -122,6 +135,38 @@ Verdict check(std::string_view chosen, std::string_view stated) { } // namespace +std::optional override_violation(std::span claims, + std::string_view key, + std::string_view version, + std::string_view statedBy) { + if (version.empty()) return std::nullopt; + for (auto const& c : claims) { + if (package_key(c.address) != key) continue; + const auto stated = version_of(c.address); + if (check(version, stated) != Verdict::Violated) continue; + return std::format( + "`{}` is overridden by {} with version {}, and {} requires {}.\n" + " An override replaces where the package comes from, not what the\n" + " declarations require of it.\n" + " fix: name a program satisfying {}, or state the version it has.", + key, statedBy, version, c.declaredBy, stated, stated); + } + return std::nullopt; +} + +std::vector requirements_for(std::span claims, + std::string_view key) { + std::vector out; + for (auto const& c : claims) { + if (package_key(c.address) != key) continue; + const auto stated = version_of(c.address); + if (stated.empty() || !mcpp::version_req::is_constraint(stated)) continue; + auto line = std::format("{} by {}", stated, c.declaredBy); + if (std::ranges::find(out, line) == out.end()) out.push_back(std::move(line)); + } + return out; +} + std::expected unify(std::span claims) { Resolution out; std::vector order; diff --git a/tests/e2e/873_payload_overrides.sh b/tests/e2e/873_payload_overrides.sh new file mode 100755 index 000000000..6067c9291 --- /dev/null +++ b/tests/e2e/873_payload_overrides.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# requires: gcc unix-shell +# `[xlings.overrides]`, `MCPP_XLINGS_OVERRIDE__` and config.toml +# state where a declared payload comes from (mcpp#755): an overridden payload +# is not provisioned, the build program receives the override, and the build +# says so on a `Using` line and in its `Finished` line. +# +# THE ASSERTION IS ON WHAT mcpp ASKS FOR. The payload is a name no index +# carries, and every build runs with `MCPP_NO_AUTO_INSTALL=1`: a build that +# still wanted it is refused naming it, so "built" means "not asked for". The +# baseline is the control: the same project without an override is refused. +set -e + +MCPP="${MCPP:-mcpp}" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +export NO_COLOR=1 + +# A plugin that declares the payload behind a feature, the way a rule package +# declares its tool, and a consumer whose build program reads it. +mkdir -p "$work/plug/src" "$work/plug/members" "$work/app/src" +cat > "$work/plug/mcpp.toml" <<'TOML' +[package] +name = "plug" +namespace = "e2e" +version = "0.1.0" + +[build] +sources = ["src/plug.cppm"] + +[features.tools-x] +sources = ["members/x.cppm"] + +[feature-xlings.tools-x] +"xim:mcpp-e2e-absent-tool" = ">=2.0" + +[targets.plug] +kind = "lib" +TOML +echo 'export module e2e.plug;' > "$work/plug/src/plug.cppm" +cat > "$work/plug/members/x.cppm" <<'CPP' +export module e2e.tools.x; +export namespace e2e::x { inline int v = 1; } +CPP +cat > "$work/app/mcpp.toml" < "$work/app/build.mcpp" <<'CPP' +import std; +import mcpp; +import e2e.tools.x; +int main() { + std::string s = std::string("dir=") + mcpp::xpkg_dir("xim", "mcpp-e2e-absent-tool") + + " source=" + mcpp::xpkg_source("xim", "mcpp-e2e-absent-tool") + + " program=" + mcpp::xpkg_program("xim", "mcpp-e2e-absent-tool"); + mcpp::warning(s.c_str()); + return 0; +} +CPP +echo 'int main() { return 0; }' > "$work/app/src/main.cpp" +mkdir -p "$work/opt/bin" +printf '#!/bin/sh\necho tool\n' > "$work/opt/bin/absent-tool"; chmod +x "$work/opt/bin/absent-tool" + +cd "$work/app" +probe() { rm -rf target; MCPP_NO_AUTO_INSTALL=1 "$MCPP" "$@" 2>&1 || true; } +fail() { echo "FAIL: $*"; echo "----"; echo "$out"; exit 1; } + +# Control: declared and not overridden, so it is asked for. +out="$(probe build)" +grep -q "not provisioned" <<<"$out" || fail "the baseline did not ask for the payload" +grep -q "mcpp-e2e-absent-tool" <<<"$out" || fail "the refusal does not name the payload" + +# The environment variable. +out="$(MCPP_XLINGS_OVERRIDE_XIM_MCPP_E2E_ABSENT_TOOL="$work/opt/bin/absent-tool" probe build)" +grep -q "Finished" <<<"$out" || fail "an env override still needed the payload" +grep -q "Using xim:mcpp-e2e-absent-tool ← $work/opt/bin/absent-tool \[custom · env MCPP_XLINGS_OVERRIDE_XIM_MCPP_E2E_ABSENT_TOOL\]" <<<"$out" \ + || fail "no Using line naming the env override" +grep -q "dir=$work/opt source=override program=$work/opt/bin/absent-tool" <<<"$out" \ + || fail "the build program did not receive the override (root, source, program)" +grep -q "Finished .* · custom: xim:mcpp-e2e-absent-tool" <<<"$out" || fail "Finished does not summarise the source" + +# The manifest, with a version that satisfies the plugin's requirement. +cp mcpp.toml.base mcpp.toml +printf '\n[xlings.overrides]\n"xim:mcpp-e2e-absent-tool" = { program = "%s", version = "2.1" }\n' \ + "$work/opt/bin/absent-tool" >> mcpp.toml +out="$(probe build)" +grep -q "\[custom · mcpp.toml:[0-9]*\]" <<<"$out" || fail "no Using line naming mcpp.toml" +# --managed-only refuses it, naming it. +out="$(probe --managed-only build)" +grep -q "managed-only" <<<"$out" && grep -q "mcpp-e2e-absent-tool" <<<"$out" \ + || fail "--managed-only did not refuse the override" + +# A stated version below the requirement is refused, naming both sides. +cp mcpp.toml.base mcpp.toml +printf '\n[xlings.overrides]\n"xim:mcpp-e2e-absent-tool" = { program = "%s", version = "1.0" }\n' \ + "$work/opt/bin/absent-tool" >> mcpp.toml +out="$(probe build)" +grep -q ">=2.0" <<<"$out" && grep -q "e2e:plug" <<<"$out" || fail "a version below the requirement was accepted" + +# A path that does not exist is refused, naming it. +cp mcpp.toml.base mcpp.toml +printf '\n[xlings.overrides]\n"xim:mcpp-e2e-absent-tool" = "%s"\n' "$work/nope" >> mcpp.toml +out="$(probe build)" +grep -q "does not exist" <<<"$out" || fail "a missing override path was accepted" + +# A dependency may not state where a payload comes from. +cp mcpp.toml.base mcpp.toml +printf '\n[xlings.overrides]\n"xim:mcpp-e2e-absent-tool" = "/bin/sh"\n' >> "$work/plug/mcpp.toml" +out="$(probe build)" +grep -q "is a dependency of this build" <<<"$out" || fail "a dependency's override was accepted" + +echo "PASS: overrides skip provisioning, reach the build program, and are reported" diff --git a/tests/e2e/874_payload_on_request.sh b/tests/e2e/874_payload_on_request.sh new file mode 100755 index 000000000..6f8263642 --- /dev/null +++ b/tests/e2e/874_payload_on_request.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# requires: gcc unix-shell +# `provision = "on-request"` (mcpp#755): a payload installed when a build +# program asks for it with `xpkg_request`, not before the program runs. +# +# Every build runs with `MCPP_NO_AUTO_INSTALL=1`, so "built" means "nothing was +# asked for", and a refusal names the requester. The pair is the criterion: the +# same project builds while the program does not ask, and is refused naming +# the program when it does. +set -e + +MCPP="${MCPP:-mcpp}" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +export NO_COLOR=1 + +mkdir -p "$work/plug/src" "$work/plug/members" "$work/app/src" +cat > "$work/plug/mcpp.toml" <<'TOML' +[package] +name = "plug" +namespace = "e2e" +version = "0.1.0" + +[build] +sources = ["src/plug.cppm"] + +[features.tools-x] +sources = ["members/x.cppm"] + +[feature-xlings.tools-x] +"xim:mcpp-e2e-absent-tool" = { version = "1.0.0", provision = "on-request" } + +[targets.plug] +kind = "lib" +TOML +echo 'export module e2e.plug;' > "$work/plug/src/plug.cppm" +cat > "$work/plug/members/x.cppm" <<'CPP' +export module e2e.tools.x; +export namespace e2e::x { inline int v = 1; } +CPP +cat > "$work/app/mcpp.toml" < "$work/app/build.mcpp" <<'CPP' +import std; +import mcpp; +import e2e.tools.x; +int main() { + std::string s = std::string("source=") + mcpp::xpkg_source("xim", "mcpp-e2e-absent-tool"); + if (const char* ask = std::getenv("E2E_ASK"); ask && *ask) { + if (std::string_view(ask) == "undeclared") { + std::printf("mcpp:xpkg-request=xim:mcpp-e2e-undeclared\n"); + return 0; + } + mcpp::xpkg_request("xim", "mcpp-e2e-absent-tool"); + if (mcpp::xpkg_pending()) return 0; + } + mcpp::warning(s.c_str()); + return 0; +} +CPP +echo 'int main() { return 0; }' > "$work/app/src/main.cpp" + +cd "$work/app" +probe() { rm -rf target; MCPP_NO_AUTO_INSTALL=1 "$MCPP" "$@" 2>&1 || true; } +fail() { echo "FAIL: $*"; echo "----"; echo "$out"; exit 1; } + +out="$(probe build)" +grep -q "Finished" <<<"$out" || fail "a payload nobody asked for was required" +grep -q "source=pending" <<<"$out" || fail "the build program was not told the payload is pending" + +out="$(E2E_ASK=1 probe build)" +grep -q "payloads requested by the build program of \`app\`" <<<"$out" \ + || fail "a request did not reach the installer naming the requester" +grep -q "mcpp-e2e-absent-tool@1.0.0" <<<"$out" || fail "the refusal does not name the payload" + +out="$(E2E_ASK=undeclared probe build)" +grep -q "no manifest of this build declares" <<<"$out" || fail "a request for an undeclared payload was accepted" + +out="$(E2E_ASK=1 MCPP_NO_AUTO_INSTALL=1 "$MCPP" emit build-database --format json 2>/dev/null || true)" +grep -q "MCPP_BUILD_DATABASE_PAYLOAD_DEFERRED" <<<"$out" || fail "planning did not defer the request" + +echo "PASS: on-request payloads wait for a request, and a request names its requester" diff --git a/tests/e2e/875_toolchain_by_path.sh b/tests/e2e/875_toolchain_by_path.sh new file mode 100755 index 000000000..213c0f24c --- /dev/null +++ b/tests/e2e/875_toolchain_by_path.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# requires: llvm unix-shell symlink +# A toolchain named by path (mcpp#755): `[toolchain] default = { path = ... }` +# and `MCPP_TOOLCHAIN=path:`. mcpp probes the drivers in the tree, drives +# them with its own link model, writes nothing into the tree, and reports the +# source. +# +# THE TREE HAS NO CFG. It is the installed LLVM payload seen through symlinks, +# without the `.cfg` files mcpp generates for a payload it manages -- the shape +# of an LLVM release package extracted by hand. +set -e + +MCPP="${MCPP:-mcpp}" +source "$(dirname "$0")/_llvm_env.sh" +if [[ ! -x "$LLVM_ROOT/bin/clang++" ]]; then + echo "SKIP: no llvm payload installed ($LLVM_ROOT)"; exit 0 +fi +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +export NO_COLOR=1 + +T="$work/llvm" +mkdir -p "$T/bin" +for f in "$LLVM_ROOT"/bin/*; do + case "$f" in *.cfg) ;; *) ln -s "$f" "$T/bin/" ;; esac +done +for d in include lib share libexec; do [ -e "$LLVM_ROOT/$d" ] && ln -s "$LLVM_ROOT/$d" "$T/$d"; done +# A linker stated by role, as a wrapper the test can change in place. +printf '#!/bin/sh\nexec "%s/bin/ld.lld" "$@"\n' "$LLVM_ROOT" > "$work/ld-wrapper" +chmod +x "$work/ld-wrapper" + +mkdir -p "$work/app/src" +cat > "$work/app/src/main.cpp" <<'CPP' +#include +int main() { std::puts("built by a toolchain named by path"); return 0; } +CPP +write_manifest() { + cat > "$work/app/mcpp.toml" <&1)" || fail "the build failed" +grep -q "Using toolchain clang .* ← $T \[custom · mcpp.toml:[0-9]*\]" <<<"$out" || fail "no Using line for the toolchain" +grep -q "Finished .* · custom: toolchain" <<<"$out" || fail "Finished does not summarise the source" +run="$(./target/*/*/bin/app)" +[[ "$run" == "built by a toolchain named by path" ]] || fail "the program did not run: $run" +ninja="$(cat target/*/*/build.ninja)" +grep -q "^cxx *= /usr/bin/env $T/bin/clang++" <<<"$ninja" || fail "the launcher does not prefix the compiler" +grep -q -- "--ld-path=$work/ld-wrapper" <<<"$ninja" || fail "the stated linker is not used" +[[ -z "$(find "$T/" -maxdepth 2 -name '*.cfg' -print -quit)" ]] || fail "mcpp wrote a cfg into the tree" + +# The fast path serves an unchanged tree, and declines once a program of it changed. +out="$("$MCPP" build -v 2>&1)" +grep -q "fast-path: build declined" <<<"$out" && fail "an unchanged build declined the fast path" +touch -d '+1 minute' "$work/ld-wrapper" 2>/dev/null || { sleep 1; touch "$work/ld-wrapper"; } +out="$("$MCPP" build -v 2>&1)" +grep -q "a program of the toolchain named by path changed" <<<"$out" || fail "a changed linker was not noticed" + +# MCPP_TOOLCHAIN=path:, with no table. +cat > "$work/app/mcpp.toml" <<'TOML' +[package] +name = "app" +version = "0.1.0" +TOML +rm -rf target +out="$(MCPP_TOOLCHAIN="path:$T" "$MCPP" build 2>&1)" || fail "MCPP_TOOLCHAIN=path: failed" +grep -q "\[custom · env MCPP_TOOLCHAIN\]" <<<"$out" || fail "the env-named toolchain is not reported" + +# A stated family the drivers contradict is refused. +write_manifest "$T" ', family = "gcc", tools = { cxx = "'"$T/bin/clang++"'" }' +rm -rf target +out="$("$MCPP" build 2>&1 || true)" +grep -q 'stated as family "gcc"' <<<"$out" || fail "a contradicting family was accepted" + +# A tree without a driver is refused at the declaration. +mkdir -p "$work/empty/bin" +write_manifest "$work/empty" '' +out="$("$MCPP" build 2>&1 || true)" +grep -q "no C++ driver in bin/" <<<"$out" || fail "a tree without a driver was accepted" + +echo "PASS: a toolchain named by path builds, is reported, and is identified" diff --git a/tests/e2e/876_toolchain_phase.sh b/tests/e2e/876_toolchain_phase.sh new file mode 100755 index 000000000..30ddf4b5a --- /dev/null +++ b/tests/e2e/876_toolchain_phase.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# requires: llvm unix-shell symlink +# `[toolchain] default = { configure = "build.mcpp" }` (mcpp#755): the root +# build program states the build toolchain in its toolchain phase, the +# bootstrap toolchain compiles and runs the build programs, and the phase +# states nothing but the toolchain. +set -e + +MCPP="${MCPP:-mcpp}" +source "$(dirname "$0")/_llvm_env.sh" +if [[ ! -x "$LLVM_ROOT/bin/clang++" ]]; then + echo "SKIP: no llvm payload installed ($LLVM_ROOT)"; exit 0 +fi +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +export NO_COLOR=1 + +T="$work/llvm" +mkdir -p "$T/bin" +for f in "$LLVM_ROOT"/bin/*; do + case "$f" in *.cfg) ;; *) ln -s "$f" "$T/bin/" ;; esac +done +for d in include lib share libexec; do [ -e "$LLVM_ROOT/$d" ] && ln -s "$LLVM_ROOT/$d" "$T/$d"; done + +mkdir -p "$work/app/src" +cat > "$work/app/mcpp.toml" < "$work/app/src/main.cpp" <<'CPP' +int main() { return 0; } +CPP +write_program() { # $1: what the toolchain phase states + cat > "$work/app/build.mcpp" <&1)" || fail "the build failed" +grep -q "Using toolchain clang .* ← $T \[program · build.mcpp:6\]" <<<"$out" || fail "no Using line naming the build program" +grep -q "Bootstrap llvm@$LLVM_VERSION" <<<"$out" || fail "no Bootstrap line" +grep -q "build phase, compiler clang" <<<"$out" || fail "the build phase did not run" +grep -q "Finished .* · program: toolchain" <<<"$out" || fail "Finished does not summarise the source" +[[ "$(grep -c 'Resolving toolchain' <<<"$out")" -le 1 ]] || fail "the first pass narrated" + +# A managed spec stated by the phase is a pinned source, and is not announced. +write_program "mcpp::toolchain(\"spec\", \"llvm@$LLVM_VERSION\");" +rm -rf target +out="$("$MCPP" build 2>&1)" || fail "a managed spec from the phase failed" +grep -q "Using toolchain" <<<"$out" && fail "a managed spec was announced as a custom source" + +# A phase that states nothing, and a phase that states a flag, are refused. +write_program "" +rm -rf target +out="$("$MCPP" build 2>&1 || true)" +grep -q "stated no" <<<"$out" || fail "a phase without a statement was accepted" +write_program "mcpp::cxxflag(\"-DX\"); mcpp::toolchain(\"path\", \"$T\");" +rm -rf target +out="$("$MCPP" build 2>&1 || true)" +grep -q "stated \`mcpp:cxxflag=\` in its toolchain phase" <<<"$out" || fail "a flag in the toolchain phase was accepted" + +echo "PASS: the build program states the toolchain, and the phase states nothing else" diff --git a/tests/e2e/877_why_sources.sh b/tests/e2e/877_why_sources.sh new file mode 100755 index 000000000..09a1c0bdd --- /dev/null +++ b/tests/e2e/877_why_sources.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# requires: gcc unix-shell python3 +# `mcpp why sources|tool|payload` and `--format json` (mcpp#755): the decision +# record of a prepare, as the build reported it. +set -e + +MCPP="${MCPP:-mcpp}" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +export NO_COLOR=1 +mkdir -p "$work/app/src" "$work/opt/bin" +printf '#!/bin/sh\n' > "$work/opt/bin/tool"; chmod +x "$work/opt/bin/tool" +cat > "$work/app/mcpp.toml" < "$work/app/src/main.cpp" +cd "$work/app" +fail() { echo "FAIL: $*"; echo "----"; echo "$out"; exit 1; } + +out="$(MCPP_NO_AUTO_INSTALL=1 "$MCPP" why payload mcpp-e2e 2>&1)" || fail "why payload failed" +grep -q "payload:xim:mcpp-e2e-absent-tool $work/opt/bin/tool" <<<"$out" || fail "the payload is not listed" +grep -q "custom · mcpp.toml:[0-9]*" <<<"$out" || fail "the origin is not listed" + +out="$(MCPP_NO_AUTO_INSTALL=1 "$MCPP" why sources --format json 2>/dev/null)" || fail "why sources --format json failed" +python3 - "$out" <<'PY' || exit 1 +import json, sys +env = json.loads(sys.argv[1]) +assert env["kind"] == "mcpp.why.sources", env["kind"] +subjects = {s["subject"]: s for s in env["data"]["sources"]} +p = subjects["payload:xim:mcpp-e2e-absent-tool"] +assert p["class"] == "custom" and p["origin"]["kind"] == "manifest", p +assert "toolchain.build" in subjects, list(subjects) +PY + +out="$("$MCPP" why nonsense 2>&1 || true)" +grep -q "is not a topic of \`mcpp why\`" <<<"$out" || fail "an unknown topic was accepted" + +echo "PASS: mcpp why reports the decision record" diff --git a/tests/unit/test_build_directives.cpp b/tests/unit/test_build_directives.cpp index 33941d70f..eb3c353fc 100644 --- a/tests/unit/test_build_directives.cpp +++ b/tests/unit/test_build_directives.cpp @@ -1062,13 +1062,13 @@ TEST(BuildDirectives, DeployRowIsProtocolElevenWithLinkGlobalScopeAndATag) { EXPECT_EQ(def->scope, dirs::Scope::LinkGlobal); EXPECT_EQ(def->sinceProtocol, 11); EXPECT_FALSE(def->tag.empty()); - EXPECT_EQ(dirs::kProtocolVersion, 14); + EXPECT_EQ(dirs::kProtocolVersion, 15); } -TEST(BuildDirectives, ProtocolFourteenIsAcceptedAndFifteenIsNot) { - auto ok = parse("mcpp:protocol=14\n"); +TEST(BuildDirectives, ProtocolFifteenIsAcceptedAndSixteenIsNot) { + auto ok = parse("mcpp:protocol=15\n"); EXPECT_FALSE(dirs::protocol_error(ok).has_value()); - auto no = parse("mcpp:protocol=15\n"); + auto no = parse("mcpp:protocol=16\n"); EXPECT_TRUE(dirs::protocol_error(no).has_value()); } diff --git a/tests/unit/test_sources.cpp b/tests/unit/test_sources.cpp new file mode 100644 index 000000000..79798abdb --- /dev/null +++ b/tests/unit/test_sources.cpp @@ -0,0 +1,252 @@ +// Sources (mcpp#755): the manifest keys that state where a payload or a +// toolchain comes from, the protocol-15 directives a build program states +// sources with, and the override checks of the payload unification. + +#include + +import std; +import mcpp.manifest; +import mcpp.build.directives; +import mcpp.toolchain.dialect; +import mcpp.toolchain.registry; +import mcpp.xlings.address_set; + +namespace dirs = mcpp::build::directives; +namespace addrset = mcpp::xlings::addrset; + +namespace { + +dirs::Directives parse_directives(std::string_view text) { + dirs::Directives d; + const auto root = (std::filesystem::current_path() / "pkg").lexically_normal(); + dirs::accept_output(d, mcpp::toolchain::gnu_dialect(), root, text); + return d; +} + +std::expected +manifest(std::string_view body) { + return mcpp::manifest::parse_string(std::format(R"( +[package] +name = "app" +version = "0.1.0" +{})", body)); +} + +} // namespace + +// ── [xlings.overrides] ────────────────────────────────────────────────────── + +TEST(Sources, OverridePathAndTableFormsParse) { + auto m = manifest(R"( +[xlings.overrides] +"xim:cmake" = "/usr/bin/cmake" +vcpkg = { root = "/opt/vcpkg" } +"xim:slang" = { program = "slangc", version = "2026.14.1" } +)"); + ASSERT_TRUE(m.has_value()) << m.error().format(); + using K = mcpp::manifest::XlingsOverride::Kind; + const auto& o = m->xlings.overrides; + ASSERT_EQ(o.size(), 3u); + EXPECT_EQ(o.at("xim:cmake").kind, K::Path); + EXPECT_EQ(o.at("xim:cmake").value, "/usr/bin/cmake"); + // A key without a namespace names the `xim` package, as everywhere else. + EXPECT_EQ(o.at("xim:vcpkg").kind, K::Root); + EXPECT_EQ(o.at("xim:slang").kind, K::Program); + EXPECT_EQ(o.at("xim:slang").version, "2026.14.1"); + EXPECT_GT(o.at("xim:cmake").line, 0); +} + +TEST(Sources, OverrideKeyWithAVersionIsRefused) { + auto m = manifest(R"( +[xlings.overrides] +"xim:cmake@3.31" = "/usr/bin/cmake" +)"); + ASSERT_FALSE(m.has_value()); + EXPECT_NE(m.error().message.find("names a version"), std::string::npos) << m.error().message; +} + +TEST(Sources, OverrideNamingBothProgramAndRootIsRefused) { + auto m = manifest(R"( +[xlings.overrides] +cmake = { program = "/usr/bin/cmake", root = "/usr" } +)"); + ASSERT_FALSE(m.has_value()); + EXPECT_NE(m.error().message.find("exactly one of"), std::string::npos) << m.error().message; +} + +TEST(Sources, OverrideUnknownKeyIsRefused) { + auto m = manifest(R"( +[xlings.overrides] +cmake = { programme = "/usr/bin/cmake" } +)"); + ASSERT_FALSE(m.has_value()); + EXPECT_NE(m.error().message.find("programme"), std::string::npos) << m.error().message; +} + +TEST(Sources, OverrideUnderATargetSelectorParses) { + auto m = manifest(R"( +[target.'cfg(os = "linux")'.xlings.overrides] +"xim:cmake" = "/usr/bin/cmake" +)"); + ASSERT_TRUE(m.has_value()) << m.error().format(); + ASSERT_FALSE(m->conditionalConfigs.empty()); + bool found = false; + for (auto const& cc : m->conditionalConfigs) + if (cc.xlings.overrides.contains("xim:cmake")) found = true; + EXPECT_TRUE(found); +} + +// ── provision = "on-request" ──────────────────────────────────────────────── + +TEST(Sources, ProvisionOnRequestIsRecordedOnBothTables) { + auto m = manifest(R"( +[features] +tools = {} +[xlings.workspace] +"xim:cmake" = { version = ">=3.31", provision = "on-request" } +"xim:ninja" = { version = "1.12", provision = "eager" } +[feature-xlings.tools] +"xim:slang" = { version = "", provision = "on-request", when = "build" } +)"); + ASSERT_TRUE(m.has_value()) << m.error().format(); + EXPECT_TRUE(m->xlings.on_request("xim:cmake@>=3.31")); + EXPECT_FALSE(m->xlings.on_request("xim:ninja@1.12")); + EXPECT_TRUE(m->xlings.on_request("xim:slang")); + EXPECT_EQ(m->xlings.when_of("xim:slang"), mcpp::manifest::ToolWhen::Build); +} + +TEST(Sources, ProvisionUnknownModeIsRefused) { + auto m = manifest(R"( +[xlings.workspace] +"xim:cmake" = { version = "", provision = "lazy" } +)"); + ASSERT_FALSE(m.has_value()); + EXPECT_NE(m.error().message.find("on-request"), std::string::npos) << m.error().message; +} + +// ── [toolchain] tables ────────────────────────────────────────────────────── + +TEST(Sources, ToolchainTableNamesAPath) { + auto m = manifest(R"( +[toolchain] +default = { path = "/opt/llvm", launcher = "ccache", tools = { ld = "/opt/lld/bin/ld.lld" } } +bootstrap = "llvm@22.1.8" +)"); + ASSERT_TRUE(m.has_value()) << m.error().format(); + EXPECT_EQ(m->toolchain.for_platform("linux").value_or(""), "path:/opt/llvm"); + auto* lt = m->toolchain.local_for("linux"); + ASSERT_NE(lt, nullptr); + EXPECT_EQ(lt->launcher, "ccache"); + ASSERT_EQ(lt->tools.size(), 1u); + EXPECT_EQ(lt->tools[0].first, "ld"); + EXPECT_EQ(m->toolchain.bootstrap, "llvm@22.1.8"); + // `bootstrap` is not a platform. + EXPECT_FALSE(m->toolchain.byPlatform.contains("bootstrap")); +} + +TEST(Sources, ToolchainConfigureIsAloneInItsTable) { + auto ok = manifest(R"( +[toolchain] +default = { configure = "build.mcpp" } +)"); + ASSERT_TRUE(ok.has_value()) << ok.error().format(); + EXPECT_EQ(ok->toolchain.for_platform("macos").value_or(""), "configure:build.mcpp"); + auto both = manifest(R"( +[toolchain] +default = { configure = "build.mcpp", path = "/opt/llvm" } +)"); + ASSERT_FALSE(both.has_value()); + auto other = manifest(R"( +[toolchain] +default = { configure = "cmake" } +)"); + ASSERT_FALSE(other.has_value()); + EXPECT_NE(other.error().message.find("build.mcpp"), std::string::npos) << other.error().message; +} + +TEST(Sources, ToolchainTableWithoutPathIsRefused) { + auto m = manifest(R"( +[toolchain] +linux = { prefix = "aarch64-none-linux-gnu-" } +)"); + ASSERT_FALSE(m.has_value()); + EXPECT_NE(m.error().message.find("path"), std::string::npos) << m.error().message; +} + +TEST(Sources, ToolchainUnknownToolRoleIsRefused) { + auto m = manifest(R"( +[toolchain] +default = { path = "/opt/llvm", tools = { linker = "/x" } } +)"); + ASSERT_FALSE(m.has_value()); + EXPECT_NE(m.error().message.find("linker"), std::string::npos) << m.error().message; +} + +// ── Protocol-15 directives ────────────────────────────────────────────────── + +TEST(Sources, SourceDirectivesFoldIntoTheBuildConfig) { + auto d = parse_directives( + "mcpp:protocol=15\n" + "mcpp:decision=tool:mcpp.deps.cmake:cmake\tchoice\t/usr/bin/cmake\t/p/build.mcpp\t9\txim:cmake\n" + "mcpp:xpkg-request=xim:cmake\n" + "mcpp:toolchain=path=/opt/llvm\n" + "mcpp:toolchain=origin=/p/build.mcpp:4\n"); + EXPECT_FALSE(dirs::protocol_error(d).has_value()); + mcpp::manifest::Manifest m; + dirs::apply(m, d); + ASSERT_EQ(m.buildConfig.toolDecisions.size(), 1u); + EXPECT_TRUE(m.buildConfig.toolDecisions[0].starts_with("tool:mcpp.deps.cmake:cmake\tchoice")); + ASSERT_EQ(m.buildConfig.xpkgRequests.size(), 1u); + EXPECT_EQ(m.buildConfig.xpkgRequests[0], "xim:cmake"); + ASSERT_EQ(m.buildConfig.toolchainStatement.size(), 2u); + EXPECT_EQ(m.buildConfig.toolchainStatement[0], "path=/opt/llvm"); +} + +TEST(Sources, SourceDirectivesAreProtocolFifteen) { + for (auto wire : {"decision", "xpkg-request", "toolchain"}) { + auto def = dirs::find_by_wire(wire); + ASSERT_NE(def, nullptr) << wire; + EXPECT_EQ(def->sinceProtocol, 15) << wire; + EXPECT_FALSE(def->tag.empty()) << wire; + } +} + +// ── Override checks in the unification ───────────────────────────────────── + +TEST(Sources, OverrideVersionIsCheckedAgainstRequirements) { + std::vector claims{ + {"xim:cmake@>=3.31", "mcpp:plugins", 1}, + {"xim:cmake", "this project", 0}, + }; + EXPECT_FALSE(addrset::override_violation(claims, "xim:cmake", "3.31.6", "[xlings.overrides]")); + auto bad = addrset::override_violation(claims, "xim:cmake", "3.28.3", "[xlings.overrides]"); + ASSERT_TRUE(bad.has_value()); + EXPECT_NE(bad->find(">=3.31"), std::string::npos) << *bad; + EXPECT_NE(bad->find("mcpp:plugins"), std::string::npos) << *bad; + // No stated version: nothing to compare, and the requirement is listed. + EXPECT_FALSE(addrset::override_violation(claims, "xim:cmake", "", "[xlings.overrides]")); + auto reqs = addrset::requirements_for(claims, "xim:cmake"); + ASSERT_EQ(reqs.size(), 1u); + EXPECT_EQ(reqs[0], ">=3.31 by mcpp:plugins"); +} + +// ── A toolchain named by path ─────────────────────────────────────────────── + +TEST(Sources, PathSpecTakesTheFamilyFromTheDrivers) { + const auto base = std::filesystem::temp_directory_path() / "mcpp-test-sources-tc"; + std::filesystem::remove_all(base); + std::filesystem::create_directories(base / "llvm" / "bin"); + std::filesystem::create_directories(base / "gcc" / "bin"); + std::ofstream(base / "llvm" / "bin" / "clang++") << ""; + std::ofstream(base / "gcc" / "bin" / "aarch64-none-linux-gnu-g++") << ""; + auto llvm = mcpp::toolchain::parse_toolchain_spec("path:" + (base / "llvm").string()); + ASSERT_TRUE(llvm.has_value()) << llvm.error(); + EXPECT_EQ(llvm->family, mcpp::toolchain::Family::Llvm); + EXPECT_EQ(llvm->spec_str(), "path:" + (base / "llvm").lexically_normal().generic_string()); + auto gcc = mcpp::toolchain::parse_toolchain_spec("path:" + (base / "gcc").string()); + ASSERT_TRUE(gcc.has_value()) << gcc.error(); + EXPECT_EQ(gcc->family, mcpp::toolchain::Family::Gcc); + auto none = mcpp::toolchain::parse_toolchain_spec("path:" + (base / "missing").string()); + EXPECT_FALSE(none.has_value()); + std::filesystem::remove_all(base); +} From d7da9cc43cf30ef99edf1e809232bb36cf18079a Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 22:25:30 +0800 Subject: [PATCH 02/20] fix: the toolchain stamp formats its numbers explicitly, and the design record carries front matter `std::to_string` over a filesystem clock's rep and over `uintmax_t` is ambiguous on libc++, so the three sites that write or compare the stamp of a toolchain named by path format the two values through `std::format` with an explicit type. The design record gains the `subject`/`status` front matter every record dated 2026-09-08 or later carries. --- .../docs/2026-10-01-tool-and-toolchain-sources-design.md | 5 +++++ .agents/docs/README.md | 3 ++- src/build/execute.cppm | 8 ++++++-- src/build/prepare/local_toolchain.cpp | 4 ++-- src/build/prepare/records.cpp | 5 ++++- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md b/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md index 6d61cf601..f1bac702a 100644 --- a/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md +++ b/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md @@ -1,3 +1,8 @@ +--- +subject: design +status: landed +--- + # 工具与工具链的来源：声明、编程决定、可观察 日期：2026-10-01。状态：已落地（mcpp 2026.10.1.3,mcpp#755)。 diff --git a/.agents/docs/README.md b/.agents/docs/README.md index 4263aa9d2..db02537d4 100644 --- a/.agents/docs/README.md +++ b/.agents/docs/README.md @@ -30,6 +30,7 @@ Records that declare one. Everything else is listed by date below. ### design +- [工具与工具链的来源：声明、编程决定、可观察](2026-10-01-tool-and-toolchain-sources-design.md) — landed - [A pack's build reported as a build, and a unit's compile independent of the member selection: triage and design (#753, #751)](2026-10-01-pack-drive-and-selection-independent-compile-design.md) — landed - [Member selection, build programs prepared once, a pack over several members, and the output streams of `mcpp run`: the plan for the release after 2026.9.30.2 (#748, #749, #750)](2026-09-30-member-selection-and-build-program-cost-plan.md) — landed - [The build's wall time, its progress count, a hang after the build, and #732 and #744: measurements and a remediation plan](2026-09-30-build-wall-time-progress-count-and-hang-plan.md) — landed @@ -113,7 +114,7 @@ Records that declare one. Everything else is listed by date below. ### 2026-10 -- [工具与工具链的来源：声明、编程决定、可观察](2026-10-01-tool-and-toolchain-sources-design.md) +- [工具与工具链的来源：声明、编程决定、可观察](2026-10-01-tool-and-toolchain-sources-design.md) — landed - [A pack's build reported as a build, and a unit's compile independent of the member selection: triage and design (#753, #751)](2026-10-01-pack-drive-and-selection-independent-compile-design.md) — landed ### 2026-09 diff --git a/src/build/execute.cppm b/src/build/execute.cppm index dc1954281..ea1e65ef0 100644 --- a/src/build/execute.cppm +++ b/src/build/execute.cppm @@ -1572,8 +1572,12 @@ bool local_toolchain_unchanged(const std::filesystem::path& outputDir) { if (ec) return false; const auto time = std::filesystem::last_write_time(p, ec); if (ec) return false; - if (std::to_string(size) != line.substr(0, t1) - || std::to_string(time.time_since_epoch().count()) != line.substr(t1 + 1, t2 - t1 - 1)) + // `std::format`, not `std::to_string`: the clock's rep and + // `uintmax_t` both convert to two integer overloads of the latter, and + // libc++ calls that ambiguous. + if (std::format("{}", static_cast(size)) != line.substr(0, t1) + || std::format("{}", static_cast(time.time_since_epoch().count())) + != line.substr(t1 + 1, t2 - t1 - 1)) return false; } return true; diff --git a/src/build/prepare/local_toolchain.cpp b/src/build/prepare/local_toolchain.cpp index fa1045a4a..db8c8801e 100644 --- a/src/build/prepare/local_toolchain.cpp +++ b/src/build/prepare/local_toolchain.cpp @@ -100,9 +100,9 @@ std::string local_identity(const mcpp::toolchain::Toolchain& tc) { auto stamp = [&](const fs::path& p) { std::error_code ec; mix(p.generic_string()); - mix(std::to_string(fs::file_size(p, ec))); + mix(std::format("{}", static_cast(fs::file_size(p, ec)))); auto t = fs::last_write_time(p, ec); - mix(std::to_string(t.time_since_epoch().count())); + mix(std::format("{}", static_cast(t.time_since_epoch().count()))); }; stamp(tc.binaryPath); for (auto const& [role, p] : tc.toolOverrides) { mix(role); stamp(p); } diff --git a/src/build/prepare/records.cpp b/src/build/prepare/records.cpp index 431cb9e12..ca68e2736 100644 --- a/src/build/prepare/records.cpp +++ b/src/build/prepare/records.cpp @@ -598,7 +598,10 @@ void step13_resolution_json(PrepareState& state, BuildContext& ctx) { std::error_code fe; const auto size = std::filesystem::file_size(p, fe); const auto time = std::filesystem::last_write_time(p, fe); - st << size << '\t' << time.time_since_epoch().count() << '\t' + // The same two spellings the fast path compares against. + st << std::format("{}", static_cast(size)) << '\t' + << std::format("{}", static_cast( + time.time_since_epoch().count())) << '\t' << p.string() << '\n'; }; put(ctx.tc.binaryPath); From e28609f5b31fa82e256035e9fb5f7894ba072964 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 22:35:07 +0800 Subject: [PATCH 03/20] fix: the stated tools of a toolchain are a named struct, not a std::pair specialization Exporting `std::vector>` from mcpp.toolchain.model made clang 20.1.7 on Windows crash while generating code for `mcpp::pack::interface_set_digest`, a function of another module that instantiates the same specialization and sorts by a pointer to its `first`. The report named a file this branch never touched, which is how this hazard always reads; `modules/manifest/src/types.cppm` records a GCC 16 case of the same shape. The field is now a `ToolOverride` of two named members, and every reader uses the same structured binding it used before. --- modules/toolchain-model/src/model.cppm | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/toolchain-model/src/model.cppm b/modules/toolchain-model/src/model.cppm index 3c02aef82..f7cad2332 100644 --- a/modules/toolchain-model/src/model.cppm +++ b/modules/toolchain-model/src/model.cppm @@ -321,7 +321,21 @@ struct Toolchain { // Tools stated by role where the layout does not have them // (`tools = { ld = ... }`): `ld`, `ar`, `ranlib`, `nm`, `objcopy`, // `strip`, `as`. Read before any derivation from the driver's directory. - std::vector> toolOverrides; + // + // A NAMED STRUCT, NOT `std::pair`, AND + // THAT IS NOT A STYLE CHOICE. Exporting that specialization from this + // module made clang 20.1.7 crash while generating code for + // `mcpp::pack::interface_set_digest` -- a function in another module that + // instantiates the same specialization and sorts by a pointer to its + // `first`. The failure named a file this change never touched, which is + // the signature of this hazard (`modules/manifest/src/types.cppm` records + // a GCC 16 case of the same shape). Measured on windows-2022, clang + // 20.1.7, 2026-10-01. + struct ToolOverride { + std::string role; + std::filesystem::path program; + }; + std::vector toolOverrides; // The driver-and-tool prefix of a cross toolchain (`aarch64-none-linux-gnu-`). std::string toolPrefix; From 6193d62e536f35e1a3d178ba24ab173bd8d73539 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 22:43:14 +0800 Subject: [PATCH 04/20] fix: the new lookups use a predicate, not a projection by pointer-to-member A `std::ranges` projection spelled as a pointer-to-member into a type the module imports makes clang 20.1.7 crash in code generation, and the report names an unrelated function: first `mcpp::pack::interface_set_digest`, then `mcpp::doctor::why_report`, both on windows-2022. Every lookup this branch added over the decision record now passes a predicate, which reads the same and compiles on every host. --- src/build/prepare/sources.cpp | 26 +++++++++++++++++++------- src/build/progress.cppm | 4 +++- src/doctor.cppm | 5 +++-- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/build/prepare/sources.cpp b/src/build/prepare/sources.cpp index 46d96aba3..70d15ed28 100644 --- a/src/build/prepare/sources.cpp +++ b/src/build/prepare/sources.cpp @@ -111,7 +111,15 @@ void record_source(PrepareState& state, SourceDecision d) { mcpp::ui::source("Using", std::format("{} ← {}", what, d.value), source_tag(d, root), d.cls == SourceClass::Host); } - auto it = std::ranges::find(state.sources, d.subject, &SourceDecision::subject); + // A PREDICATE, NOT A PROJECTION BY POINTER-TO-MEMBER. The latter into a + // type this module imports makes clang 20.1.7 crash while generating code, + // and the report names an unrelated function (measured on windows-2022, + // 2026-10-01; `mcpp.toolchain.model`'s `ToolOverride` records the same + // hazard for an exported `std::pair`). + auto by_subject = [](std::string_view subject) { + return [subject](const SourceDecision& o) { return o.subject == subject; }; + }; + auto it = std::ranges::find_if(state.sources, by_subject(d.subject)); if (it == state.sources.end()) state.sources.push_back(std::move(d)); else *it = std::move(d); } @@ -446,7 +454,9 @@ answer_payload_requests(PrepareState& state, mcpp::manifest::Manifest& m, state.requestedPayloads.insert(keys[i]); state.xlingsDeferred.erase(keys[i]); state.xlingsSkipped.erase(addresses[i]); - auto it = std::ranges::find(state.sources, "payload:" + keys[i], &SourceDecision::subject); + const auto subject = "payload:" + keys[i]; + auto it = std::ranges::find_if(state.sources, [&](const SourceDecision& o) { + return o.subject == subject; }); if (it != state.sources.end()) { it->considered.clear(); it->considered.push_back(std::format("installed on request of `{}`", who)); @@ -518,8 +528,9 @@ void record_tool_decisions(PrepareState& state) { const auto payload = f.size() > 5 ? f[5] : std::string{}; d.payload = payload; d.announce = false; - auto it = std::ranges::find(state.sources, "payload:" + payload, - &SourceDecision::subject); + const auto psubject = "payload:" + payload; + auto it = std::ranges::find_if(state.sources, [&](const SourceDecision& o) { + return o.subject == psubject; }); if (it != state.sources.end()) { d.cls = it->cls; d.originKind = it->originKind; d.originFile = it->originFile; d.originLine = it->originLine; @@ -539,8 +550,9 @@ void record_tool_decisions(PrepareState& state) { d.announce = false; if (f.size() > 5) { d.payload = f[5]; - auto it = std::ranges::find(state.sources, "payload:" + f[5], - &SourceDecision::subject); + const auto psubject = "payload:" + f[5]; + auto it = std::ranges::find_if(state.sources, [&](const SourceDecision& o) { + return o.subject == psubject; }); if (it != state.sources.end()) d.cls = it->cls; } } @@ -558,7 +570,7 @@ std::expected step13_sources(PrepareState& state, BuildContex // managed payload, and how it was chosen. A toolchain named by path or by // the build program was recorded where it was resolved, and announced // there, next to the `Resolving toolchain` line. - if (std::ranges::none_of(state.sources, [](auto const& d) { + if (std::ranges::none_of(state.sources, [](const SourceDecision& d) { return d.subject == "toolchain.build"; })) { SourceDecision d; d.subject = "toolchain.build"; diff --git a/src/build/progress.cppm b/src/build/progress.cppm index 206079643..57004992b 100644 --- a/src/build/progress.cppm +++ b/src/build/progress.cppm @@ -1378,7 +1378,9 @@ void note_sources(const std::filesystem::path& buildDir) { auto colon = group.find(": "); if (colon == std::string::npos) continue; auto cls = group.substr(0, colon); - auto it = std::ranges::find(r.sources, cls, &decltype(r.sources)::value_type::first); + auto it = std::ranges::find_if(r.sources, + [&](const std::pair>& e) { + return e.first == cls; }); if (it == r.sources.end()) { r.sources.emplace_back(cls, std::vector{}); it = r.sources.end() - 1; } std::size_t n = colon + 2; while (n < group.size()) { diff --git a/src/doctor.cppm b/src/doctor.cppm index 5080636a0..c633f9751 100644 --- a/src/doctor.cppm +++ b/src/doctor.cppm @@ -1318,8 +1318,9 @@ export int why_report(const std::string& topic, const std::string& features, prof.libc, prof.cxxStdlib, prof.arch, prof.os, tc.targetTriple); // The reason the decision record states (mcpp#755), not a sentence // describing every way a toolchain can be chosen. - if (auto it = std::ranges::find(ctx->sources, std::string("toolchain.build"), - &mcpp::build::SourceDecision::subject); + if (auto it = std::ranges::find_if(ctx->sources, + [](const mcpp::build::SourceDecision& d) { + return d.subject == "toolchain.build"; }); it != ctx->sources.end()) std::println(" source: {}", mcpp::build::source_tag(*it, ctx->projectRoot)); if (!ctx->compilerChoice.origin.empty()) From 11431544ebd1cb7724ce6348c14090142b41a900 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 22:46:23 +0800 Subject: [PATCH 05/20] docs: a toolchain named by path is recorded beside the build, and mcpp.lock is not claimed to hold it The chapter, its mirror, SPEC-006 and the release notes said a build records such a toolchain in mcpp.lock as `local`. It does not: the lock holds the result of dependency resolution, a toolchain is not a resolved dependency, and nothing in this branch writes one there. What is true is that the driver and each stated tool enter the fingerprint and are recorded beside the build, so the fast paths decline once one of them changed, and a machine without the tree is refused where the declaration is read. The design record states the correction. --- .../docs/2026-10-01-tool-and-toolchain-sources-design.md | 4 ++++ CHANGELOG.md | 4 ++-- docs/20-toolchains.md | 7 ++++--- docs/specs/toolchain-management.md | 2 +- docs/zh/20-toolchains.md | 4 ++-- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md b/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md index f1bac702a..6afe7eb70 100644 --- a/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md +++ b/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md @@ -105,6 +105,10 @@ cfg 文件是否存在。 `local-toolchain.stamp` 供三条快速路径比较。不读字节:一个驱动几百 MB,而每次 prepare 都要 读它。 +**`mcpp.lock` 不记工具链**,本次也没有加。lock 的内容是依赖解析的结果,而工具链不是被解析 +的依赖;一台没有这棵树的机器在读到声明处就被拒绝,这已经是「不可移植」要的那句话。设计稿 +曾写成「lock 中记为 `local`」,那是没有实现的断言,文档与规范按实际行为更正。 + ### 2.6 工具链阶段:两遍 prepare,第一遍不说话 构建程序需要它的宿主模块,宿主模块来自依赖图;而依赖图的解析需要工具链 diff --git a/CHANGELOG.md b/CHANGELOG.md index f0d55e130..8de48997f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,8 +33,8 @@ unchanged. sysroot, family, launcher, tools }`, or `MCPP_TOOLCHAIN=path:`. mcpp probes the drivers in the tree, identifies them, drives them with its own link model, and writes nothing into the tree. The driver and each stated tool - enter the fingerprint by content, the fast paths decline when one changed, - and `mcpp.lock` records the toolchain as `local`. + enter the fingerprint by content, and a build records them beside its output, + so the fast paths decline once one of them changed. - **`[toolchain] bootstrap`** names the toolchain that compiles and runs build programs when it should not be the one building the project. - **`[toolchain] = { configure = "build.mcpp" }`** hands the build diff --git a/docs/20-toolchains.md b/docs/20-toolchains.md index 3a22432de..dcd9447c2 100644 --- a/docs/20-toolchains.md +++ b/docs/20-toolchains.md @@ -409,9 +409,10 @@ mcpp's own file, and a tree mcpp does not own does not get one. **Identity.** The driver and each tool named by `tools` enter the build's fingerprint by path, size and modification time, so rebuilding the toolchain in -place rebuilds what it produced. A build that used one records it in -`mcpp.lock` as `local`: a machine without that toolchain is told so by name -rather than building with another one. +place rebuilds what it produced, and a build records them beside its output so +the fast paths decline once one of them changed. A machine that does not have +the tree is told so by name — the declaration is refused where it is read — +rather than building with another toolchain. This is not the `system` compiler of the section above. `system` is whatever `PATH` happens to hold; this is a tree the project names, that mcpp identifies diff --git a/docs/specs/toolchain-management.md b/docs/specs/toolchain-management.md index d90e3bf2a..6c90275a4 100644 --- a/docs/specs/toolchain-management.md +++ b/docs/specs/toolchain-management.md @@ -56,7 +56,7 @@ - **禁止**写入该目录树(它不是 mcpp 管理的); - 把驱动与 `tools` 所列每个程序的身份(路径、大小、修改时间)纳入指纹与快速路径的判定, 因为这样的工具链可以原地改变; -- 在 `mcpp.lock` 中记为 `local`,即不可移植; +- 把它们记在该次构建的产物旁,使快速路径在其中之一改变后让行; - 在构建输出、`resolution.json` 与 `mcpp why toolchain` 中陈述其来源(§3.3)。 `[toolchain] bootstrap = "<族>@<版本>"` 命名编译并运行构建程序的工具链;未写时按 §3 的 diff --git a/docs/zh/20-toolchains.md b/docs/zh/20-toolchains.md index 863953cd6..4267eeed5 100644 --- a/docs/zh/20-toolchains.md +++ b/docs/zh/20-toolchains.md @@ -383,8 +383,8 @@ MCPP_TOOLCHAIN=path:/opt/llvm-trunk mcpp build mcpp 自己的文件，而一棵不属于 mcpp 的树不会得到一份。 **身份。** 驱动与 `tools` 点名的每个程序以路径、大小与修改时间进入构建指纹，因此原地重建 -工具链会重建它产出的东西。用过它的构建在 `mcpp.lock` 中记为 `local`：没有这个工具链的 -机器会被点名告知，而不是改用另一个构建。 +工具链会重建它产出的东西；构建还把它们记在产物旁边，于是其中之一改变后快速路径会让行。 +没有这棵树的机器会被点名告知——声明在被读取的地方就被拒绝——而不是改用另一个工具链。 这不是上一节的 `system` 编译器。`system` 是「`PATH` 上碰巧有什么」，而这里是工程点名的 一棵树，由 mcpp 识别并记录，与 `msvc@system` 一直以来的形状相同。 From 23c5843fe4366b638ec7708f94a475c256e4777f Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 22:54:26 +0800 Subject: [PATCH 06/20] test: config.toml's overrides are read by a function a test can call The third place an override may be stated was parsed inside `load_or_init`, which bootstraps a home, so no unit test could reach it. `parse_payload_overrides` is now a pure function of the parsed document, and three cases cover the two shapes it accepts, the key it refuses, and a config with no such table. --- src/config.cppm | 79 +++++++++++++++++++++++-------------- tests/unit/test_sources.cpp | 43 ++++++++++++++++++++ 2 files changed, 92 insertions(+), 30 deletions(-) diff --git a/src/config.cppm b/src/config.cppm index 6f05f7c46..f3f6dc395 100644 --- a/src/config.cppm +++ b/src/config.cppm @@ -315,6 +315,13 @@ std::expected load_or_init( // Pretty-print resolved config for `mcpp env` command. void print_env(const GlobalConfig& cfg); +// `[xlings.overrides]` of a parsed config.toml, keyed by package identity +// (`:`, the namespace defaulted to `xim`): where a declared xlings +// payload comes from on this machine (mcpp#755). A pure function of the +// document, so a test reads it without a home to bootstrap. +std::expected, std::string> +parse_payload_overrides(const mcpp::libs::toml::Document& doc); + // Normalize legacy index naming in a loaded config (exported for tests): // org migration mcpp-community/mcpp-index -> mcpplibs/mcpp-index, index name // mcpp-index -> mcpplibs, then name+url dedup. Order matters: URL first, so @@ -652,6 +659,43 @@ void canonicalize_legacy_index_names(GlobalConfig& cfg) { cfg.indexRepos = std::move(normalized); } +std::expected, std::string> +parse_payload_overrides(const mcpp::libs::toml::Document& doc) { + std::map out; + auto* ot = doc.get_table("xlings.overrides"); + if (!ot) return out; + for (auto const& [key, val] : *ot) { + GlobalConfig::PayloadOverride o; + o.line = static_cast(val.position.line); + if (val.is_string()) { + o.value = val.as_string(); + } else if (val.is_table()) { + for (auto const& [k, v] : val.as_table()) { + if ((k != "program" && k != "root" && k != "version") || !v.is_string()) + return std::unexpected(std::format( + "config.toml [xlings.overrides] {}: '{}' is not a key of an " + "override; an override is a path, or a table of `program` " + "or `root` and an optional `version`", key, k)); + if (k == "version") o.version = v.as_string(); + else { o.kind = k; o.value = v.as_string(); } + } + } + if (o.value.empty()) + return std::unexpected(std::format( + "config.toml [xlings.overrides] {}: expected a path, or a table " + "naming `program` or `root`", key)); + // The identity, as the manifest and the address set spell it: a key + // without a namespace names the `xim` package, and a version in the key + // is dropped -- an override states where a package comes from, not + // which version of it is wanted. + auto colon = key.find(':'); + std::string ident = colon == std::string::npos ? "xim:" + key : key; + if (auto at = ident.find('@'); at != std::string::npos) ident.resize(at); + out.insert_or_assign(std::move(ident), std::move(o)); + } + return out; +} + std::expected load_or_init( bool quiet, BootstrapProgressCallback onBootstrapProgress, @@ -731,36 +775,11 @@ std::expected load_or_init( cfg.defaultJobs = doc->get_int("build.default_jobs").value_or(0); cfg.defaultToolchain = doc->get_string("toolchain.default").value_or(""); cfg.defaultTarget = doc->get_string("toolchain.default_target").value_or(""); - // [xlings.overrides] (mcpp#755). Read in the manifest's two shapes -- a - // path, or a table of `program` or `root` and `version` -- and refused by - // name when it is neither, as a malformed config.toml key always is. - if (auto* ot = doc->get_table("xlings.overrides")) { - for (auto const& [key, val] : *ot) { - GlobalConfig::PayloadOverride o; - o.line = static_cast(val.position.line); - if (val.is_string()) { - o.value = val.as_string(); - } else if (val.is_table()) { - for (auto const& [k, v] : val.as_table()) { - if ((k != "program" && k != "root" && k != "version") || !v.is_string()) - return std::unexpected(ConfigError{std::format( - "config.toml [xlings.overrides] {}: '{}' is not a key of an " - "override; an override is a path, or a table of `program` " - "or `root` and an optional `version`", key, k)}); - if (k == "version") o.version = v.as_string(); - else { o.kind = k; o.value = v.as_string(); } - } - } - if (o.value.empty()) - return std::unexpected(ConfigError{std::format( - "config.toml [xlings.overrides] {}: expected a path, or a table " - "naming `program` or `root`", key)}); - auto colon = key.find(':'); - std::string ident = colon == std::string::npos ? "xim:" + key : key; - if (auto at = ident.find('@'); at != std::string::npos) ident.resize(at); - cfg.payloadOverrides.insert_or_assign(std::move(ident), std::move(o)); - } - } + // [xlings.overrides] (mcpp#755), read by the same function the tests read. + if (auto overrides = parse_payload_overrides(*doc); overrides) + cfg.payloadOverrides = std::move(*overrides); + else + return std::unexpected(ConfigError{overrides.error()}); // [log] section — re-initialize logger with config values { diff --git a/tests/unit/test_sources.cpp b/tests/unit/test_sources.cpp index 79798abdb..e7b634a9d 100644 --- a/tests/unit/test_sources.cpp +++ b/tests/unit/test_sources.cpp @@ -6,6 +6,8 @@ import std; import mcpp.manifest; +import mcpp.config; +import mcpp.libs.toml; import mcpp.build.directives; import mcpp.toolchain.dialect; import mcpp.toolchain.registry; @@ -230,6 +232,47 @@ TEST(Sources, OverrideVersionIsCheckedAgainstRequirements) { EXPECT_EQ(reqs[0], ">=3.31 by mcpp:plugins"); } +// ── [xlings.overrides] in config.toml ─────────────────────────────────────── + +TEST(Sources, ConfigOverridesParseWithTheManifestsTwoShapes) { + auto doc = mcpp::libs::toml::parse(R"( +[xlings.overrides] +"xim:cmake" = "/usr/bin/cmake" +vcpkg = { root = "/opt/vcpkg" } +"xim:slang" = { program = "slangc", version = "2026.14.1" } +)"); + ASSERT_TRUE(doc.has_value()); + auto o = mcpp::config::parse_payload_overrides(*doc); + ASSERT_TRUE(o.has_value()) << o.error(); + ASSERT_EQ(o->size(), 3u); + EXPECT_EQ(o->at("xim:cmake").kind, "path"); + EXPECT_EQ(o->at("xim:cmake").value, "/usr/bin/cmake"); + // A key without a namespace names the `xim` package, as everywhere else. + EXPECT_EQ(o->at("xim:vcpkg").kind, "root"); + EXPECT_EQ(o->at("xim:slang").kind, "program"); + EXPECT_EQ(o->at("xim:slang").version, "2026.14.1"); + EXPECT_GT(o->at("xim:cmake").line, 0); +} + +TEST(Sources, ConfigOverrideWithAnUnknownKeyIsRefused) { + auto doc = mcpp::libs::toml::parse(R"( +[xlings.overrides] +cmake = { programme = "/usr/bin/cmake" } +)"); + ASSERT_TRUE(doc.has_value()); + auto o = mcpp::config::parse_payload_overrides(*doc); + ASSERT_FALSE(o.has_value()); + EXPECT_NE(o.error().find("programme"), std::string::npos) << o.error(); +} + +TEST(Sources, ConfigWithoutTheTableHasNoOverrides) { + auto doc = mcpp::libs::toml::parse("[toolchain]\ndefault = \"gcc@16.1.0\"\n"); + ASSERT_TRUE(doc.has_value()); + auto o = mcpp::config::parse_payload_overrides(*doc); + ASSERT_TRUE(o.has_value()) << o.error(); + EXPECT_TRUE(o->empty()); +} + // ── A toolchain named by path ─────────────────────────────────────────────── TEST(Sources, PathSpecTakesTheFamilyFromTheDrivers) { From d2ba022683c48effaa710d6788385076d42b075f Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 23:09:06 +0800 Subject: [PATCH 07/20] build: a build program's compile command goes through a response file when it outgrows the channel A build program that imports many host modules carries one `-fmodule-file==` per module, with absolute paths, and on Windows `capture_exec` reaches a shell that tolerates 8191 bytes. mcpp-plugins' all-rules-compile fixture imports fifteen and crossed that line the moment the collection gained one more module, reporting only The command line is too long. build.mcpp failed to compile (exit 1) which names neither the length nor the cause -- the family mcpp.build.cmdlimits exists to make legible. The command now goes through `@file` when it is over the budget that module states, which every driver mcpp supports reads, and the file stays beside the program for a failed compile to show. Its quoting is `response_file_body`, exported and covered by a unit case, because the command it serves cannot be run on a host whose limit it does not cross. --- src/build/build_program.cppm | 72 ++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/src/build/build_program.cppm b/src/build/build_program.cppm index 7f99051a5..941d5d353 100644 --- a/src/build/build_program.cppm +++ b/src/build/build_program.cppm @@ -23,6 +23,7 @@ import mcpp.platform.process; import mcpp.toolchain.cppfly; // std_flag (dialect- and c++fly-aware -std= spelling) import mcpp.toolchain.dialect; // CommandDialect — gnu vs cl.exe spellings import mcpp.toolchain.fingerprint; // hash_file / hash_string (FNV-1a, 16 hex) +import mcpp.build.cmdlimits; import mcpp.build.directives; // the directive definition table (own module: see its header) import mcpp.build.progress; // the program's line (build progress design 2026-09-29) import mcpp.build.refusal; // the machine-readable identity of a refusal @@ -425,6 +426,30 @@ struct BuildProgramEnv { // The suffix selects the fact: `DIR` (the payload directory), `SOURCE` and // `PROGRAM` (where it came from, mcpp#755), read back by `xpkg_source` and // `xpkg_program`. +std::string response_file_body(std::span args) { + std::string body; + for (auto const& a : args) { + if (a.find_first_of(" \t\"") == std::string::npos) { + body += a; + body.push_back('\n'); + continue; + } + body.push_back('"'); + std::size_t slashes = 0; + for (char c : a) { + if (c == '\\') { ++slashes; body.push_back(c); continue; } + if (c == '"') { body.append(slashes, '\\'); body += "\\\""; } + else body.push_back(c); + slashes = 0; + } + // A run of backslashes that ends the argument would escape the closing + // quote, so it is doubled. + body.append(slashes, '\\'); + body += "\"\n"; + } + return body; +} + inline std::string xpkg_env_var(std::string_view ns, std::string_view name, std::string_view suffix = "DIR") { std::string out = "MCPP_XPKG_"; @@ -440,6 +465,17 @@ inline std::string xpkg_env_var(std::string_view ns, std::string_view name, return out; } +// THE ARGV THAT PASSES `args` THROUGH A RESPONSE FILE, and the file's content. +// +// Every compiler driver mcpp supports reads `@file` with one argument per line. +// An argument that carries whitespace or a quote is quoted, and the backslashes +// before a quote -- including a run that would otherwise escape the closing one +// -- are escaped, which is what makes a Windows path safe to write here. +// +// Exported because its quoting is the part worth testing, and the command it +// serves cannot be run on a host whose limit it does not cross. +std::string response_file_body(std::span args); + // Does a compiler's output say the program asked for something the bundled // `mcpp` module does not have? // @@ -1854,6 +1890,42 @@ std::expected run_build_program_impl( // module", e2e 807 under GCC). Otherwise the project root is fine. const bool needsBmiCwd = usesModule || stdStagedInBdir || !env.hostModules.empty(); std::string compileCwd = needsBmiCwd ? bdir.string() : root.string(); + // A COMMAND THAT OUTGREW ITS CHANNEL GOES THROUGH A RESPONSE FILE. + // + // The engine's rule for an unbounded payload is a response file + // (mcpp.build.cmdlimits, and the architecture record it names), and this + // command carries one: a `-fmodule-file==` for every host + // module the program imports, with absolute paths. A collection that + // offers many rules through features is the case -- mcpp-plugins' + // `all-rules-compile` imports fifteen -- and on Windows + // `capture_exec` reaches the shell, whose 8191 bytes this crossed the + // moment that collection gained one more module: + // + // The command line is too long. + // build.mcpp failed to compile (exit 1) + // + // Every driver mcpp supports reads `@file`, one argument per line, and + // the file is written beside the program it compiles, so a failed + // compile leaves it to read. + { + std::string flat; + for (auto const& a : compileArgv) { flat += a; flat.push_back(' '); } + if (mcpp::build::cmdlimits::check_inline( + flat, mcpp::platform::is_windows, /*needsShell=*/true)) { + const auto rsp = bdir / "build.mcpp.compile.rsp"; + const auto body = response_file_body( + std::span(compileArgv).subspan(1)); + std::ofstream out(rsp, std::ios::binary | std::ios::trunc); + out << body; + out.close(); + if (out) { + mcpp::log::verbose("buildmcpp-host", std::format( + "build.mcpp {}: the compile command is {} bytes and goes through {}", + who, flat.size(), rsp.string())); + compileArgv = { compileArgv.front(), "@" + rsp.string() }; + } + } + } auto cres = mcpp::platform::process::capture_exec(compileArgv, compileEnv, compileCwd); mcpp::log::verbose("buildmcpp-host", std::format("build.mcpp {}: compile end", who)); From eb386f65a3e192b5e6816cf77ae5ec5bc6a9faf7 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 23:11:57 +0800 Subject: [PATCH 08/20] test: the response file's quoting, the part of it a host cannot demonstrate --- tests/unit/test_sources.cpp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/unit/test_sources.cpp b/tests/unit/test_sources.cpp index e7b634a9d..8a8eae9fb 100644 --- a/tests/unit/test_sources.cpp +++ b/tests/unit/test_sources.cpp @@ -7,6 +7,7 @@ import std; import mcpp.manifest; import mcpp.config; +import mcpp.build.build_program; import mcpp.libs.toml; import mcpp.build.directives; import mcpp.toolchain.dialect; @@ -232,6 +233,29 @@ TEST(Sources, OverrideVersionIsCheckedAgainstRequirements) { EXPECT_EQ(reqs[0], ">=3.31 by mcpp:plugins"); } +// ── The response file a long compile command goes through ─────────────────── + +TEST(Sources, ResponseFileWritesOneArgumentPerLineAndQuotesWhatNeedsIt) { + const std::vector args{ + "-std=c++23", + "-fmodule-file=mcpp=/home/u/p/mcpp.pcm", + "/Tp C:/Program Files/x/build.mcpp", + R"(-DNAME="v")", + "C:\\with space\\dir\\", + }; + const auto body = mcpp::build::response_file_body(args); + EXPECT_EQ(std::ranges::count(body, '\n'), 5); + EXPECT_NE(body.find("-std=c++23\n"), std::string::npos) << body; + // Quoted only where it has to be. + EXPECT_NE(body.find("-fmodule-file=mcpp=/home/u/p/mcpp.pcm\n"), std::string::npos) << body; + EXPECT_NE(body.find("\"/Tp C:/Program Files/x/build.mcpp\"\n"), std::string::npos) << body; + // A quote inside the argument survives as one. + EXPECT_NE(body.find("\"-DNAME=\\\"v\\\"\"\n"), std::string::npos) << body; + // The run of backslashes that ends the argument is doubled, so it does not + // escape the closing quote. + EXPECT_NE(body.find("\"C:\\with space\\dir\\\\\"\n"), std::string::npos) << body; +} + // ── [xlings.overrides] in config.toml ─────────────────────────────────────── TEST(Sources, ConfigOverridesParseWithTheManifestsTwoShapes) { From 89ed2cd1f45544f6c6298db9c6535606a10c2465 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 23:18:23 +0800 Subject: [PATCH 09/20] platform: which() resolves a name the shell answers for itself `command -v true` prints `true`, not a path: a shell answers with what it would run, and for a builtin that is the word. which() then found no file and reported the name missing, so a bare-name payload override of such a name was refused with `'true' is not found on PATH` on a machine carrying /usr/bin/true -- an answer that sends the reader to the wrong place. Found while verifying the host class of mcpp#755. PATH is now walked for a bare word the shell returned, and two cases cover it: the builtin name resolves to its program, and a name no host has still answers nothing. --- modules/platform/src/fs.cppm | 33 +++++++++++++++++++++++++++++++++ tests/unit/test_platform_fs.cpp | 24 ++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/modules/platform/src/fs.cppm b/modules/platform/src/fs.cppm index 75d7c2339..abcee2e1f 100644 --- a/modules/platform/src/fs.cppm +++ b/modules/platform/src/fs.cppm @@ -50,6 +50,8 @@ std::filesystem::path self_exe_path(); // Find an executable by name in PATH. // Windows: `where ` // POSIX: `command -v ` +// A name that the shell answers with a bare word -- a builtin such as `true` -- +// is then looked for in PATH directly, so the program is found where one exists. std::optional which(std::string_view binary_name); // ── extended_length ─────────────────────────────────────────────────────── @@ -166,6 +168,37 @@ std::optional which(std::string_view binary_name) { auto nl = out.find('\n'); if (nl != std::string::npos) out.resize(nl); + // A NAME THAT IS ALSO A SHELL BUILTIN comes back as the bare word: `command + // -v true` prints `true`, not `/usr/bin/true`, because the shell answers + // with what it would run. The answer is still a name on PATH -- the program + // exists -- so PATH is walked here rather than the name reported as missing. + // Found while verifying a bare-name payload override (mcpp#755): the + // refusal said `'true' is not found on PATH` on a machine carrying + // /usr/bin/true, which is the kind of answer that sends a reader to the + // wrong place. + if (rc == 0 && !out.empty() && !std::filesystem::exists(out) + && std::filesystem::path(out).filename() == out) { + if (const char* path = std::getenv("PATH")) { +#if defined(_WIN32) + constexpr char sep = ';'; +#else + constexpr char sep = ':'; +#endif + std::string_view rest(path); + while (!rest.empty()) { + const auto at = rest.find(sep); + const auto dir = rest.substr(0, at); + if (!dir.empty()) { + std::error_code ec; + const auto cand = std::filesystem::path(dir) / out; + if (std::filesystem::is_regular_file(cand, ec)) return cand; + } + if (at == std::string_view::npos) break; + rest.remove_prefix(at + 1); + } + } + } + if (rc != 0 || out.empty()) return std::nullopt; if (!std::filesystem::exists(out)) return std::nullopt; return std::filesystem::path(out); diff --git a/tests/unit/test_platform_fs.cpp b/tests/unit/test_platform_fs.cpp index 68887b275..e11b4417f 100644 --- a/tests/unit/test_platform_fs.cpp +++ b/tests/unit/test_platform_fs.cpp @@ -103,3 +103,27 @@ TEST(PlatformFs, WindowsExtendedLengthLeavesPrefixedAndRelativePathsAlone) { EXPECT_EQ(windows_extended_length_spelling("obj/a.ddi"), "obj/a.ddi"); EXPECT_EQ(windows_extended_length_spelling(""), ""); } + +// A NAME THE SHELL ANSWERS FOR ITSELF still resolves to the program on PATH. +// `command -v true` prints `true`, because a shell runs its own builtin, and +// the path is then found by walking PATH (mcpp#755: a bare-name payload +// override of such a name was refused as "not found on PATH" on a machine +// carrying /usr/bin/true). +TEST(PlatformFs, WhichResolvesANameThatIsAlsoAShellBuiltin) { +#if defined(_WIN32) + GTEST_SKIP() << "`where` reports programs only; the shell answers nothing"; +#else + std::error_code ec; + if (!std::filesystem::is_regular_file("/usr/bin/true", ec) + && !std::filesystem::is_regular_file("/bin/true", ec)) + GTEST_SKIP() << "this host has no `true` program to find"; + const auto found = mcpp::platform::fs::which("true"); + ASSERT_TRUE(found.has_value()); + EXPECT_EQ(found->filename(), "true"); + EXPECT_TRUE(std::filesystem::is_regular_file(*found, ec)) << found->string(); +#endif +} + +TEST(PlatformFs, WhichStillAnswersNothingForANameNoHostHas) { + EXPECT_FALSE(mcpp::platform::fs::which("mcpp-no-such-program-7c25d9f6").has_value()); +} From 02df8c30742cceac7c8ef1c16373f7aa527e6099 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 23:24:22 +0800 Subject: [PATCH 10/20] build: the response file is written in the grammar its driver reads A response file is not one format. clang and GCC tokenize it the GNU way, where a backslash escapes the next character, so the Windows paths written plainly came back with their separators eaten: clang++: error: no such file or directory: 'D:amcpp-pluginsmcpp-pluginstestsall-rules-compiletarget.build-mcpp...' Each argument is therefore wrapped in single quotes for those drivers, inside which nothing is special, and an embedded single quote is closed, escaped and reopened; cl and clang-cl keep Windows quoting, where a backslash is literal. One case per grammar. --- src/build/build_program.cppm | 35 +++++++++++++++++++++++++++-------- tests/unit/test_sources.cpp | 27 ++++++++++++++++++++------- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/src/build/build_program.cppm b/src/build/build_program.cppm index 941d5d353..9bae7f323 100644 --- a/src/build/build_program.cppm +++ b/src/build/build_program.cppm @@ -426,9 +426,27 @@ struct BuildProgramEnv { // The suffix selects the fact: `DIR` (the payload directory), `SOURCE` and // `PROGRAM` (where it came from, mcpp#755), read back by `xpkg_source` and // `xpkg_program`. -std::string response_file_body(std::span args) { +std::string response_file_body(std::span args, bool gnuQuoting) { std::string body; for (auto const& a : args) { + if (gnuQuoting) { + // GNU TOKENIZATION TREATS A BACKSLASH AS AN ESCAPE, everywhere but + // inside single quotes. A Windows path written plainly therefore + // arrives with its separators eaten -- clang read + // `D:\a\mcpp-plugins\...` back as `D:amcpp-plugins...` and + // reported `no such file or directory`. Inside single quotes every + // character is literal, so each argument is wrapped, and an embedded + // single quote is closed, escaped and reopened. + body.push_back('\''); + for (char c : a) { + if (c == '\'') body += "'\\''"; + else body.push_back(c); + } + body += "'\n"; + continue; + } + // Windows tokenization (cl, clang-cl): a backslash is literal except + // before a quote, so only whitespace and quotes need handling. if (a.find_first_of(" \t\"") == std::string::npos) { body += a; body.push_back('\n'); @@ -465,16 +483,17 @@ inline std::string xpkg_env_var(std::string_view ns, std::string_view name, return out; } -// THE ARGV THAT PASSES `args` THROUGH A RESPONSE FILE, and the file's content. +// THE CONTENT OF A RESPONSE FILE CARRYING `args`, one argument per line. // -// Every compiler driver mcpp supports reads `@file` with one argument per line. -// An argument that carries whitespace or a quote is quoted, and the backslashes -// before a quote -- including a run that would otherwise escape the closing one -// -- are escaped, which is what makes a Windows path safe to write here. +// Every compiler driver mcpp supports reads `@file`, but not with one grammar: +// clang and GCC tokenize it the GNU way, where a backslash escapes the next +// character, and cl and clang-cl tokenize it the Windows way, where a backslash +// is literal. `gnuQuoting` picks between them -- single quotes, inside which +// nothing is special, or Windows quoting of whitespace and quotes. // // Exported because its quoting is the part worth testing, and the command it // serves cannot be run on a host whose limit it does not cross. -std::string response_file_body(std::span args); +std::string response_file_body(std::span args, bool gnuQuoting); // Does a compiler's output say the program asked for something the bundled // `mcpp` module does not have? @@ -1914,7 +1933,7 @@ std::expected run_build_program_impl( flat, mcpp::platform::is_windows, /*needsShell=*/true)) { const auto rsp = bdir / "build.mcpp.compile.rsp"; const auto body = response_file_body( - std::span(compileArgv).subspan(1)); + std::span(compileArgv).subspan(1), !msvcHost); std::ofstream out(rsp, std::ios::binary | std::ios::trunc); out << body; out.close(); diff --git a/tests/unit/test_sources.cpp b/tests/unit/test_sources.cpp index 8a8eae9fb..e7d5667e3 100644 --- a/tests/unit/test_sources.cpp +++ b/tests/unit/test_sources.cpp @@ -235,27 +235,40 @@ TEST(Sources, OverrideVersionIsCheckedAgainstRequirements) { // ── The response file a long compile command goes through ─────────────────── -TEST(Sources, ResponseFileWritesOneArgumentPerLineAndQuotesWhatNeedsIt) { +TEST(Sources, ResponseFileQuotesForTheWindowsTokenizer) { const std::vector args{ "-std=c++23", - "-fmodule-file=mcpp=/home/u/p/mcpp.pcm", "/Tp C:/Program Files/x/build.mcpp", R"(-DNAME="v")", "C:\\with space\\dir\\", }; - const auto body = mcpp::build::response_file_body(args); - EXPECT_EQ(std::ranges::count(body, '\n'), 5); + const auto body = mcpp::build::response_file_body(args, /*gnuQuoting=*/false); + EXPECT_EQ(std::ranges::count(body, '\n'), 4); EXPECT_NE(body.find("-std=c++23\n"), std::string::npos) << body; - // Quoted only where it has to be. - EXPECT_NE(body.find("-fmodule-file=mcpp=/home/u/p/mcpp.pcm\n"), std::string::npos) << body; EXPECT_NE(body.find("\"/Tp C:/Program Files/x/build.mcpp\"\n"), std::string::npos) << body; - // A quote inside the argument survives as one. EXPECT_NE(body.find("\"-DNAME=\\\"v\\\"\"\n"), std::string::npos) << body; // The run of backslashes that ends the argument is doubled, so it does not // escape the closing quote. EXPECT_NE(body.find("\"C:\\with space\\dir\\\\\"\n"), std::string::npos) << body; } +TEST(Sources, ResponseFileKeepsBackslashesLiteralForTheGnuTokenizer) { + // clang and GCC read a backslash as an escape, so a Windows path written + // plainly comes back with its separators eaten; inside single quotes + // nothing is special. + const std::vector args{ + "-fmodule-file=mcpp=D:\\a\\p\\mcpp.pcm", + "D:\\a\\obj\\x.o", + "-DNAME=it's", + }; + const auto body = mcpp::build::response_file_body(args, /*gnuQuoting=*/true); + EXPECT_EQ(std::ranges::count(body, '\n'), 3); + EXPECT_NE(body.find("'-fmodule-file=mcpp=D:\\a\\p\\mcpp.pcm'\n"), std::string::npos) << body; + EXPECT_NE(body.find("'D:\\a\\obj\\x.o'\n"), std::string::npos) << body; + // A single quote in the argument closes, escapes and reopens. + EXPECT_NE(body.find("'-DNAME=it'\\''s'\n"), std::string::npos) << body; +} + // ── [xlings.overrides] in config.toml ─────────────────────────────────────── TEST(Sources, ConfigOverridesParseWithTheManifestsTwoShapes) { From f1cf90db2d6caffb1cb1b46892689ef741c32534 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 23:25:18 +0800 Subject: [PATCH 11/20] docs: the design record states where the response-file path is covered --- .agents/docs/2026-10-01-tool-and-toolchain-sources-design.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md b/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md index 6afe7eb70..ca163979a 100644 --- a/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md +++ b/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md @@ -146,5 +146,9 @@ action 描述的是一次还不存在的构建;一个载荷请求也无法回答 - 覆盖与按需供给都只作用于 xlings 载荷。依赖包 `kind = "bin"` 的宿主工具仍走 `[tools.overrides]`:两者的键空间与语义不同(一个是 `:` 的程序,一个是 `ns:name` 的目录),合并会让一张表有两种键。 +- 构建程序的编译命令在超过预算时走响应文件。这条路径在本仓库的 CI 里**到不了**:Linux 与 macOS + 的预算是 128 KiB,而只有 Windows 上 `capture_exec` 所经的 shell 是 8191 字节。覆盖它的是 + mcpp-plugins 的 `all-rules-compile`(导入十五个宿主模块)与两个单测——每种 tokenize 语法 + 一个,因为 clang 与 GCC 把反斜杠当转义,cl 与 clang-cl 不当。 - 工具链描述数据化(核心读描述文件、不再硬编码 `to_xim_package`)不在本次范围; `[toolchain] { path }` 的字段已经与那份描述同形,所以它是后续的第三种载体,而不是改写。 From 7819a28caf11983ece1257815222b022d883e404 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 23:27:52 +0800 Subject: [PATCH 12/20] docs: the project's own statement is the root manifest, or the workspace manifest where a member is built Both are what the code reads (prepare's runtime owner is the workspace manifest when there is one); the table named only the root. --- docs/23-the-project-environment.md | 2 +- docs/zh/23-the-project-environment.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/23-the-project-environment.md b/docs/23-the-project-environment.md index bdd74ca01..c553ee4e0 100644 --- a/docs/23-the-project-environment.md +++ b/docs/23-the-project-environment.md @@ -450,7 +450,7 @@ Three places may state an override, highest first: | | place | purpose | |---|---|---| | 1 | `MCPP_XLINGS_OVERRIDE__` | CI and distribution packaging, without editing the manifest. `path:cmake` looks the name up on PATH | -| 2 | `[xlings.overrides]` in the root manifest, also under `[target.'cfg(..)']` | the project's own statement | +| 2 | `[xlings.overrides]` in the root manifest -- or the workspace manifest, where a member is built -- also under `[target.'cfg(..)']` | the project's own statement | | 3 | `[xlings.overrides]` in `~/.mcpp/config.toml` | a fact about this machine | A `version` in the entry is checked against every requirement a package of the diff --git a/docs/zh/23-the-project-environment.md b/docs/zh/23-the-project-environment.md index c1a5944c2..b5c010305 100644 --- a/docs/zh/23-the-project-environment.md +++ b/docs/zh/23-the-project-environment.md @@ -397,7 +397,7 @@ hardware = {} | | 位置 | 面向 | |---|---|---| | 1 | `MCPP_XLINGS_OVERRIDE__` | CI 与发行版打包，无需改清单。`path:cmake` 在 PATH 上查找该名字 | -| 2 | 根清单的 `[xlings.overrides]`，也可写在 `[target.'cfg(..)']` 下 | 工程自己的陈述 | +| 2 | 根清单（构建工作区成员时为工作区清单）的 `[xlings.overrides]`，也可写在 `[target.'cfg(..)']` 下 | 工程自己的陈述 | | 3 | `~/.mcpp/config.toml` 的 `[xlings.overrides]` | 关于这台机器的事实 | 条目里的 `version` 会与依赖图中每一条要求比较，低于其中任一条则拒绝并点出两侧。没写 From a5728dd3ebe4b5a37ab5ffabf1d40f53f1004ab3 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 23:37:23 +0800 Subject: [PATCH 13/20] manifest: a key this engine does not know says which engine the package needs Measured with mcpp-plugins 0.19.0 on the released 2026.10.1.2: the reader is told error: mcpp.toml: error: [feature-xlings.deps-archive] xim:cmake: unknown key 'provision' in a scoped entry; expected 'version' and 'when' and nothing about the version, because the floor check needs the document that this very parse failed to produce. Every release of a plugin collection raises its floor, so this is the first thing a user on an older engine meets. The floor is therefore read from the file's text in the parse-failure path -- `mcpp` inside `[package]`, nothing else -- and when this engine is below it, the refusal says so and names the upgrade, in the words the floor check already uses. `stated_mcpp_floor` is exported and seven cases pin the shapes it reads. --- modules/manifest/src/toml.cppm | 73 +++++++++++++++++++++++++++++++++- tests/unit/test_sources.cpp | 19 +++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/modules/manifest/src/toml.cppm b/modules/manifest/src/toml.cppm index 9c8bfbb3e..9af085299 100644 --- a/modules/manifest/src/toml.cppm +++ b/modules/manifest/src/toml.cppm @@ -15,6 +15,7 @@ import mcpp.pm.index_spec; import mcpp.platform; import mcpp.platform.axis; // the one macos/macosx spelling rule import mcpp.xpkg_version; // the release grammar of `mcpp = ">=V"` +import mcpp.version; // MCPP_VERSION -- a refused key may be a newer one // ANONYMOUS NAMESPACE, AND THIS COST TWO WINDOWS JOBS TO LEARN. // @@ -557,6 +558,14 @@ make_xlings_entry(std::string_view key, std::string_view value) { std::expected parse_string(std::string_view content, const std::filesystem::path& origin = "mcpp.toml", LoadContext ctx = {}); +// THE ENGINE FLOOR A MANIFEST STATES, read from its text rather than from the +// parsed document: a manifest that fails to parse still says which engine it was +// written for, and that is exactly the case where the answer matters. Only +// `mcpp` inside `[package]` counts. Empty when the file states none. +// +// Exported for the test that pins the shapes it reads. +std::string stated_mcpp_floor(std::string_view text); + std::expected load(const std::filesystem::path& path, LoadContext ctx = {}); @@ -4600,6 +4609,45 @@ void apply_defaults_and_infer(Manifest& m, const std::filesystem::path& root) { } // namespace +std::string stated_mcpp_floor(std::string_view text) { + bool inPackage = false; + std::size_t at = 0; + while (at <= text.size()) { + const auto nl = text.find('\n', at); + std::string_view line = text.substr(at, nl == std::string_view::npos + ? std::string_view::npos : nl - at); + at = nl == std::string_view::npos ? text.size() + 1 : nl + 1; + while (!line.empty() && (line.front() == ' ' || line.front() == '\t')) + line.remove_prefix(1); + if (line.starts_with('#')) continue; + if (line.starts_with('[')) { + // `[package]` only. A table that merely begins with it, such as + // `[package.metadata]`, states no floor. + inPackage = line.starts_with("[package]"); + continue; + } + if (!inPackage || !line.starts_with("mcpp")) continue; + auto rest = line.substr(4); + while (!rest.empty() && (rest.front() == ' ' || rest.front() == '\t')) + rest.remove_prefix(1); + if (!rest.starts_with('=')) continue; // `mcpp_something = ...` + rest.remove_prefix(1); + while (!rest.empty() && (rest.front() == ' ' || rest.front() == '\t')) + rest.remove_prefix(1); + if (rest.empty() || rest.front() != '"') continue; + rest.remove_prefix(1); + const auto end = rest.find('"'); + if (end == std::string_view::npos) continue; + std::string_view value = rest.substr(0, end); + // The grammar of the key is `">=V"`, and a bare `"V"` is read the same + // way the parsed form reads it. + if (value.starts_with(">=")) value.remove_prefix(2); + while (!value.empty() && value.front() == ' ') value.remove_prefix(1); + return std::string(value); + } + return {}; +} + std::expected load(const std::filesystem::path& path, LoadContext ctx) { std::ifstream is(path); @@ -4610,8 +4658,29 @@ std::expected load(const std::filesystem::path& path, } std::stringstream ss; ss << is.rdbuf(); - auto m = parse_string(ss.str(), path, ctx); - if (!m) return m; + const std::string text = ss.str(); + auto m = parse_string(text, path, ctx); + if (!m) { + // A KEY THIS ENGINE DOES NOT KNOW MAY BE A KEY OF A NEWER ONE, and the + // package says which engine it was written for. Without this the reader + // of a new plugin collection on an old engine is told + // `unknown key 'provision' in a scoped entry` and nothing about the + // version -- measured with mcpp-plugins 0.19.0 on mcpp 2026.10.1.2, + // where the floor check never runs because it needs the document this + // very parse failed to produce. + const auto floor = stated_mcpp_floor(text); + const auto need = floor.empty() ? std::nullopt : mcpp::xpkg_version::parse(floor); + const auto have = mcpp::xpkg_version::parse(mcpp::MCPP_VERSION); + if (need && have && mcpp::xpkg_version::compare(*have, *need) < 0) + m.error().message += std::format( + "\n This package requires mcpp >= {}, and this is mcpp {}, so the " + "key may be\n" + " one a newer engine reads.\n" + " hint: pin \"mcpp\": \"{}\" (or newer) in .xlings.json and run " + "`xlings install`, or\n run `xlings install mcpp@{}`", + floor, mcpp::MCPP_VERSION, floor, floor); + return m; + } // M5.0: defaults + target inference (uses filesystem context relative to mcpp.toml). apply_defaults_and_infer(*m, path.parent_path()); diff --git a/tests/unit/test_sources.cpp b/tests/unit/test_sources.cpp index e7d5667e3..1363b8c6e 100644 --- a/tests/unit/test_sources.cpp +++ b/tests/unit/test_sources.cpp @@ -330,3 +330,22 @@ TEST(Sources, PathSpecTakesTheFamilyFromTheDrivers) { EXPECT_FALSE(none.has_value()); std::filesystem::remove_all(base); } + +// ── The engine floor read from a manifest that does not parse ─────────────── + +TEST(Sources, StatedMcppFloorIsReadFromPackageOnly) { + using mcpp::manifest::stated_mcpp_floor; + EXPECT_EQ(stated_mcpp_floor("[package]\nname = \"p\"\nmcpp = \">=2026.10.1.3\"\n"), + "2026.10.1.3"); + // A bare version is the same statement. + EXPECT_EQ(stated_mcpp_floor("[package]\nmcpp = \"2026.9.28.3\"\n"), "2026.9.28.3"); + // Only `[package]`: a dependency named mcpp states a dependency, not a floor. + EXPECT_EQ(stated_mcpp_floor("[package]\nname = \"p\"\n\n[dependencies]\nmcpp = \"1.0\"\n"), + ""); + // `[package.metadata]` is another table. + EXPECT_EQ(stated_mcpp_floor("[package.metadata]\nmcpp = \"9.9.9.9\"\n"), ""); + // A key that merely starts with the name is not the key. + EXPECT_EQ(stated_mcpp_floor("[package]\nmcpp_home = \"/x\"\n"), ""); + EXPECT_EQ(stated_mcpp_floor("[package]\n# mcpp = \"9.9.9.9\"\n"), ""); + EXPECT_EQ(stated_mcpp_floor(""), ""); +} From 77b632fd0839991fb2b4130783797fafa666341e Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 23:40:48 +0800 Subject: [PATCH 14/20] docs: the release notes carry the three fixes the other platforms asked for --- CHANGELOG.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8de48997f..75f2b13ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,27 @@ unchanged. resolution recorded, in place of a sentence listing every way one can be chosen. +### Fixed + +- **A build program's compile command goes through a response file when it + outgrows the channel it travels.** The command carries one + `-fmodule-file==` per host module the program imports, with + absolute paths, and on Windows it reaches a shell that tolerates 8191 bytes: a + program importing fifteen modules reported only `The command line is too + long.`, naming neither the length nor the cause. The file is written in the + grammar its driver reads -- single quotes for clang and GCC, which treat a + backslash as an escape, Windows quoting for cl and clang-cl -- and stays beside + the program for a failed compile to show. +- **A manifest key this engine does not know says which engine the package + needs.** A package written for a newer mcpp was refused with `unknown key + ''` and nothing about the version, because the engine floor is checked on + the document that very parse failed to produce. The refusal now names the + floor, this engine and the upgrade, which is what a reader meets first after a + plugin collection raises it. +- **`which()` resolves a name that is also a shell builtin.** `command -v true` + prints `true`, not a path, so a bare-name payload override of such a name was + refused as not found on a machine carrying `/usr/bin/true`. + ## [2026.10.1.2] - 2026-10-01 This release implements the design for a pack's build and a compile that does From 49abb10a25808bee5e9bd78b9dccd7213173a345 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Thu, 1 Oct 2026 23:56:35 +0800 Subject: [PATCH 15/20] build: the GNU response file doubles backslashes, because the tokenizer escapes them inside quotes too The first form single-quoted each argument, which a POSIX shell would take literally and this tokenizer does not: LLVM's GNU tokenizer escapes a backslash inside quotes as well as outside, so the Windows paths still arrived with their separators eaten. Measured with clang 22.1.8 -- a response file holding `'-DX=a\b'` yields `X=ab`, one holding `-DX=a\\b` yields `X=a\b` -- so every backslash is doubled, every quote escaped, and whitespace handled by quoting the whole argument. The case states the measurement. --- src/build/build_program.cppm | 25 +++++++++++++++---------- tests/unit/test_sources.cpp | 23 ++++++++++++++--------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/build/build_program.cppm b/src/build/build_program.cppm index 9bae7f323..abb2a21ea 100644 --- a/src/build/build_program.cppm +++ b/src/build/build_program.cppm @@ -430,19 +430,24 @@ std::string response_file_body(std::span args, bool gnuQuotin std::string body; for (auto const& a : args) { if (gnuQuoting) { - // GNU TOKENIZATION TREATS A BACKSLASH AS AN ESCAPE, everywhere but - // inside single quotes. A Windows path written plainly therefore - // arrives with its separators eaten -- clang read + // GNU TOKENIZATION TREATS A BACKSLASH AS AN ESCAPE, INCLUDING INSIDE + // QUOTES -- which is where this differs from a POSIX shell, and where + // the first attempt at this function was wrong. A Windows path + // written plainly arrives with its separators eaten: clang read // `D:\a\mcpp-plugins\...` back as `D:amcpp-plugins...` and - // reported `no such file or directory`. Inside single quotes every - // character is literal, so each argument is wrapped, and an embedded - // single quote is closed, escaped and reopened. - body.push_back('\''); + // reported `no such file or directory`, and single-quoting it changed + // nothing. Measured with clang 22.1.8: a response file holding + // `'-DX=a\b'` yields `X=ab`, and one holding `-DX=a\\b` yields + // `X=a\b`. So every backslash is doubled and every quote escaped, + // and whitespace is handled by quoting the whole argument. + const bool quote = a.find_first_of(" \t") != std::string::npos; + if (quote) body.push_back('"'); for (char c : a) { - if (c == '\'') body += "'\\''"; - else body.push_back(c); + if (c == '\\' || c == '"') body.push_back('\\'); + body.push_back(c); } - body += "'\n"; + if (quote) body.push_back('"'); + body.push_back('\n'); continue; } // Windows tokenization (cl, clang-cl): a backslash is literal except diff --git a/tests/unit/test_sources.cpp b/tests/unit/test_sources.cpp index 1363b8c6e..1fdfccb8c 100644 --- a/tests/unit/test_sources.cpp +++ b/tests/unit/test_sources.cpp @@ -253,20 +253,25 @@ TEST(Sources, ResponseFileQuotesForTheWindowsTokenizer) { } TEST(Sources, ResponseFileKeepsBackslashesLiteralForTheGnuTokenizer) { - // clang and GCC read a backslash as an escape, so a Windows path written - // plainly comes back with its separators eaten; inside single quotes - // nothing is special. + // clang and GCC read a backslash as an escape, inside quotes as well as + // outside, so a Windows path survives only when every backslash is doubled. + // Measured with clang 22.1.8: `'-DX=a\b'` in a response file yields `X=ab`, + // and `-DX=a\\b` yields `X=a\b`. const std::vector args{ "-fmodule-file=mcpp=D:\\a\\p\\mcpp.pcm", "D:\\a\\obj\\x.o", - "-DNAME=it's", + "/Tp D:\\a\\b c\\build.mcpp", + R"(-DNAME="v")", }; const auto body = mcpp::build::response_file_body(args, /*gnuQuoting=*/true); - EXPECT_EQ(std::ranges::count(body, '\n'), 3); - EXPECT_NE(body.find("'-fmodule-file=mcpp=D:\\a\\p\\mcpp.pcm'\n"), std::string::npos) << body; - EXPECT_NE(body.find("'D:\\a\\obj\\x.o'\n"), std::string::npos) << body; - // A single quote in the argument closes, escapes and reopens. - EXPECT_NE(body.find("'-DNAME=it'\\''s'\n"), std::string::npos) << body; + EXPECT_EQ(std::ranges::count(body, '\n'), 4); + EXPECT_NE(body.find("-fmodule-file=mcpp=D:\\\\a\\\\p\\\\mcpp.pcm\n"), std::string::npos) << body; + EXPECT_NE(body.find("D:\\\\a\\\\obj\\\\x.o\n"), std::string::npos) << body; + // Whitespace is handled by quoting the whole argument; the doubling holds + // inside the quotes too. + EXPECT_NE(body.find("\"/Tp D:\\\\a\\\\b c\\\\build.mcpp\"\n"), std::string::npos) << body; + // A quote of the argument's own is escaped. + EXPECT_NE(body.find("-DNAME=\\\"v\\\"\n"), std::string::npos) << body; } // ── [xlings.overrides] in config.toml ─────────────────────────────────────── From cb918615e81bd297b1997f0f8f29e32c39db8846 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Fri, 2 Oct 2026 00:03:52 +0800 Subject: [PATCH 16/20] toolchain: a stated linker reaches the link on every shape, and gcc says it cannot `--ld-path` was appended inside the Linux clang branch, the only one that consumes `link_toolchain_flags`. On macOS the stated linker therefore entered the fingerprint -- touching the wrapper declined the fast path -- and took no part in the link, with nothing said; the toolchain lab measured it on macos-15, where build.ninja held no `--ld-path`. e2e 875 asserts that flag but skips on a host without the llvm payload, which macOS CI is. The flag belongs to the driver, not to a platform, so it is added once after every shape has built its line. For a gcc toolchain the declaration is now refused where it is read: gcc selects a linker by the name `ld` inside a `-B` directory, so a program named anything else could not be chosen, and a silent `-B` would be the same defect in the other direction. --- ...10-01-tool-and-toolchain-sources-design.md | 6 ++++ src/build/flags.cppm | 29 ++++++++++++++----- src/build/prepare/local_toolchain.cpp | 14 +++++++++ 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md b/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md index ca163979a..ff497d42f 100644 --- a/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md +++ b/.agents/docs/2026-10-01-tool-and-toolchain-sources-design.md @@ -150,5 +150,11 @@ action 描述的是一次还不存在的构建;一个载荷请求也无法回答 的预算是 128 KiB,而只有 Windows 上 `capture_exec` 所经的 shell 是 8191 字节。覆盖它的是 mcpp-plugins 的 `all-rules-compile`(导入十五个宿主模块)与两个单测——每种 tokenize 语法 一个,因为 clang 与 GCC 把反斜杠当转义,cl 与 clang-cl 不当。 +- `tools = { ld = ... }` 只对 clang 驱动成立(`--ld-path`);gcc 按 `-B` 目录里的名字 `ld` + 选链接器,所以一个别名程序无法那样被选中,声明处直接拒绝而不是在链接时忽略。这条起初写成了 + Linux clang 分支里的一行,于是 macOS 的 Apple 链接形状拿不到它——被陈述的链接器进了指纹 + (改动 wrapper 会让快速路径失效),却不参与链接,而且什么都不说。现在它在所有形状之后追加 + 一次。覆盖它的是 toolchain-lab:e2e 875 在没有装 llvm 载荷的宿主上 SKIP,而 macOS CI 正是 + 这样的宿主。 - 工具链描述数据化(核心读描述文件、不再硬编码 `to_xim_package`)不在本次范围; `[toolchain] { path }` 的字段已经与那份描述同形,所以它是后续的第三种载体,而不是改写。 diff --git a/src/build/flags.cppm b/src/build/flags.cppm index 7c20e245b..defa273d4 100644 --- a/src/build/flags.cppm +++ b/src/build/flags.cppm @@ -975,14 +975,6 @@ CompileFlags compute_flags(const BuildPlan& plan) { link_toolchain_flags += mcpp::toolchain::ClangDriverModel::kLinkDriverFlags; } - // A LINKER STATED BY ROLE (mcpp#755, `tools = { ld = ... }`): clang's - // `--ld-path` names the program, after `-fuse-ld` has chosen the - // flavour, so the flags that follow still speak to the same linker. - if (auto* ld = plan.toolchain.tool_override("ld")) { - const auto opt = " --ld-path=" + escape_path(*ld); - link_toolchain_flags += opt; - link_toolchain_flags_c += opt; - } f.sysroot = link_toolchain_flags; } else if (lm.mode != mcpp::toolchain::CLibMode::None) { // GCC (or Clang without cfg): --sysroot from probe, or the payload @@ -2401,6 +2393,27 @@ CompileFlags compute_flags(const BuildPlan& plan) { } } + // A LINKER STATED BY ROLE (mcpp#755, `tools = { ld = ... }`), AFTER EVERY + // SHAPE HAS BUILT ITS LINE. + // + // It was appended inside the Linux clang branch, the only one that consumes + // `link_toolchain_flags`: on macOS the stated linker entered the fingerprint + // -- the fast path declined when the wrapper changed -- and took no part in + // the link, with nothing said (measured in the toolchain lab on macos-15, + // where `build.ninja` held no `--ld-path`). The flag belongs to the driver, + // not to a platform, so it is added once here, for every shape. + // + // Clang only: `--ld-path` is clang's. GCC chooses its linker by name inside a + // `-B` directory, so a program named anything else could not be selected that + // way, and a silent `-B` would be the same defect in the other direction. A + // gcc toolchain that states `ld` is refused where the declaration is read. + if (plan.toolchain.compiler == mcpp::toolchain::CompilerId::Clang) + if (auto* ld = plan.toolchain.tool_override("ld")) { + const auto opt = " --ld-path=" + escape_path(*ld); + if (f.ld.find("--ld-path=") == std::string::npos) f.ld += opt; + if (f.ldC.find("--ld-path=") == std::string::npos) f.ldC += opt; + } + return f; } diff --git a/src/build/prepare/local_toolchain.cpp b/src/build/prepare/local_toolchain.cpp index db8c8801e..9088b9b9a 100644 --- a/src/build/prepare/local_toolchain.cpp +++ b/src/build/prepare/local_toolchain.cpp @@ -240,6 +240,20 @@ step2_use_local_toolchain(PrepareState& state, const mcpp::toolchain::ToolchainS for (auto const& [role, p] : lt.tools) if (!fs::exists(p, ec)) return refuse(std::format("tools.{} names '{}', which does not exist", role, p)); + // A STATED LINKER REACHES THE LINK THROUGH CLANG'S `--ld-path`, which gcc + // does not have: gcc selects a linker by the name `ld` inside a `-B` + // directory, so a program named anything else cannot be chosen that way. + // Refused here rather than ignored in the link: the alternative is a stated + // tool that enters the fingerprint and takes no part in the build, which is + // what this mechanism exists to prevent. + if (family == "gcc") + for (auto const& [role, p] : lt.tools) + if (role == "ld") + return refuse(std::format( + "tools.ld names '{}', and this is a gcc toolchain: a linker is stated " + "through clang's `--ld-path`, which gcc has no counterpart for. Either " + "name an llvm toolchain, or place a program called `ld` in the tree " + "gcc searches", p)); if (!lt.sysroot.empty() && !fs::is_directory(lt.sysroot, ec)) return refuse(std::format("sysroot '{}' is not a directory", lt.sysroot)); state.explicit_compiler = driver; From 3fa5533517e32d0aff54dea2965ac13143401c81 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Fri, 2 Oct 2026 00:08:50 +0800 Subject: [PATCH 17/20] docs: the toolchain page states that a stated linker is clang's, and its gcc example no longer states one The cross example named `tools = { ld = ... }` on a gcc tree, which the engine now refuses, and nothing said which trees read that role. --- docs/20-toolchains.md | 8 +++++++- docs/zh/20-toolchains.md | 6 +++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/20-toolchains.md b/docs/20-toolchains.md index dcd9447c2..74a7b861a 100644 --- a/docs/20-toolchains.md +++ b/docs/20-toolchains.md @@ -386,7 +386,7 @@ prefix = "aarch64-none-linux-gnu-" sysroot = "/opt/acme-sysroot" family = "gcc" # checked against the drivers launcher = "ccache" # prefixes every compile -tools = { ld = "/opt/lld-dev/bin/ld.lld" } # cc, cxx, ld, ar, ranlib, nm, objcopy, strip, as +tools = { ar = "/opt/acme-gcc/bin/gcc-ar" } # cc, cxx, ar, ranlib, nm, objcopy, strip, as ``` Or for one build, without editing the manifest: @@ -402,6 +402,12 @@ whether `import std` is available are read from the driver itself. The tools beside them are found as ``, then `llvm-`, then ``; `tools` names any the tree does not have. +`tools = { ld = … }` is read by a clang tree, where the linker reaches the link +as `--ld-path`. A gcc tree that states `ld` is refused: gcc chooses its linker by +the name `ld` inside a directory it is given with `-B`, so a program under any +other name could not be selected, and a stated tool that took no part in the +build is what this mechanism exists to prevent. + **What mcpp does with it.** The same as with a payload it installed: its own link line, its own hermetic check, its own `import std` decision. It writes nothing into the tree — the generated `clang++.cfg` of a managed payload is diff --git a/docs/zh/20-toolchains.md b/docs/zh/20-toolchains.md index 4267eeed5..72819a37d 100644 --- a/docs/zh/20-toolchains.md +++ b/docs/zh/20-toolchains.md @@ -364,7 +364,7 @@ prefix = "aarch64-none-linux-gnu-" sysroot = "/opt/acme-sysroot" family = "gcc" # 与驱动核对 launcher = "ccache" # 前置于每次编译 -tools = { ld = "/opt/lld-dev/bin/ld.lld" } # cc、cxx、ld、ar、ranlib、nm、objcopy、strip、as +tools = { ar = "/opt/acme-gcc/bin/gcc-ar" } # cc、cxx、ar、ranlib、nm、objcopy、strip、as ``` 或者只对一次构建生效，不改清单： @@ -378,6 +378,10 @@ MCPP_TOOLCHAIN=path:/opt/llvm-trunk mcpp build 得到。旁边的工具按 ``、`llvm-`、`` 查找；`tools` 点名树里没有的 那些。 +`tools = { ld = … }` 只对 clang 的树成立——被陈述的链接器以 `--ld-path` 进入链接。gcc 的树 +陈述 `ld` 会被拒绝：gcc 按 `-B` 给出的目录里的名字 `ld` 选链接器，叫别的名字的程序无法那样 +被选中，而一个进了声明却不参与构建的工具，正是这套机制要防的事。 + **mcpp 如何驱动它。** 与它自己安装的载荷相同：自己的链接行、自己的 hermetic 检查、自己 对 `import std` 的判定。它不向该目录树写入任何东西 —— 托管载荷里生成的 `clang++.cfg` 是 mcpp 自己的文件，而一棵不属于 mcpp 的树不会得到一份。 From 2939d654300392107800e71c0fe269da1b3f287d Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Fri, 2 Oct 2026 00:09:49 +0800 Subject: [PATCH 18/20] tests: the counterexample for a gcc tree that states a linker Every refusal this feature adds has a case; this one did not. It runs against an installed gcc payload and says so when none is present, rather than passing silently on a host without one. --- tests/e2e/875_toolchain_by_path.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/e2e/875_toolchain_by_path.sh b/tests/e2e/875_toolchain_by_path.sh index 213c0f24c..dd51eee26 100755 --- a/tests/e2e/875_toolchain_by_path.sh +++ b/tests/e2e/875_toolchain_by_path.sh @@ -88,4 +88,21 @@ write_manifest "$work/empty" '' out="$("$MCPP" build 2>&1 || true)" grep -q "no C++ driver in bin/" <<<"$out" || fail "a tree without a driver was accepted" +# A gcc tree that states `ld` is refused: the role reaches a link through clang's +# `--ld-path`, and gcc selects a linker by the name `ld` in a `-B` directory, so a +# program under another name could not be chosen. Refused at the declaration +# rather than ignored in the link. +gcc_base="$HOME/.mcpp/registry/data/xpkgs/xim-x-gcc" +[[ -d "$gcc_base" && -n "${USERPROFILE:-}" ]] || true +gcc_ver="$(ls -1 "$gcc_base" 2>/dev/null | grep -E '^[0-9]+(\.[0-9]+)*$' | sort -V | tail -1)" +if [[ -n "$gcc_ver" && -x "$gcc_base/$gcc_ver/bin/g++" ]]; then + write_manifest "$gcc_base/$gcc_ver" ', tools = { ld = "'"$work/ld-wrapper"'" }' + rm -rf target + out="$("$MCPP" build 2>&1 || true)" + grep -q "this is a gcc toolchain" <<<"$out" \ + || fail "a gcc toolchain that states ld was accepted: $out" +else + echo "note: no gcc payload installed, so the gcc refusal is not exercised here" +fi + echo "PASS: a toolchain named by path builds, is reported, and is identified" From 136eb2a7dbedb3de35e09c81f714c0ca6c6947f9 Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Fri, 2 Oct 2026 00:16:32 +0800 Subject: [PATCH 19/20] sources: an override's path resolves the executable suffix the host appends itself A shell on Windows answers `C:/Program Files/CMake/bin/cmake` for a `cmake.exe`, and process creation there appends `.exe`, so a path stated without it names a program the machine would run; refusing it answers about spelling rather than about the machine. The plugin-side resolver gained the same rule in mcpp-plugins 0.19.0, where CI measured the refusal. Verified against a declared payload overridden by `{ program = "bin/faketool" }` with only `bin/faketool.exe` present: the build reports `Using xim:prec-absent <- .../bin/faketool.exe [custom . mcpp.toml:13]`. --- src/build/prepare/sources.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/build/prepare/sources.cpp b/src/build/prepare/sources.cpp index 70d15ed28..db33906ec 100644 --- a/src/build/prepare/sources.cpp +++ b/src/build/prepare/sources.cpp @@ -226,7 +226,18 @@ payload_override(PrepareState& state, std::string_view key) { if (!isDir) return refuse(std::format("'{}' is not a directory", p.generic_string())); out.root = p.generic_string(); } else { - if (!isFile) return refuse(std::format("'{}' does not exist", p.generic_string())); + // THE EXECUTABLE SUFFIX A HOST APPENDS ITSELF. A shell on Windows + // answers `C:/Program Files/CMake/bin/cmake` for a `cmake.exe`, and + // process creation there appends `.exe`, so a path stated without it + // names a program the machine would run. Refusing it would be an + // answer about spelling, not about the machine (the same rule the + // plugin-side resolver applies to a stated path). + if (!isFile) { + auto withExe = p; + withExe += ".exe"; + if (fs::is_regular_file(withExe, ec)) p = withExe; + else return refuse(std::format("'{}' does not exist", p.generic_string())); + } out.program = p.generic_string(); out.root = root_of_program(p).generic_string(); } From 51b66cec17046da0dedc0b65e1d273ff653a26ce Mon Sep 17 00:00:00 2001 From: sunrisepeak Date: Fri, 2 Oct 2026 00:54:24 +0800 Subject: [PATCH 20/20] tests: the new fixtures write host spellings into the manifests they generate `00_fixture_path_hygiene.sh` caught three of them: 873 and 874 interpolated `$work` into a `[build-dependencies]` path, and 875 wrote the toolchain root, the linker wrapper and a driver straight from shell variables. MSYS rewrites paths in argv and the environment but not in file content, so on Windows a native mcpp would read `/d/a/...` and resolve it against the current drive -- the failure the helper's own header records costing a day. Each path now goes through `host_path`, and the assertions that compare mcpp's output compare host spellings too. --- tests/e2e/873_payload_overrides.sh | 9 ++++++--- tests/e2e/874_payload_on_request.sh | 5 ++++- tests/e2e/875_toolchain_by_path.sh | 20 +++++++++++++------- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/tests/e2e/873_payload_overrides.sh b/tests/e2e/873_payload_overrides.sh index 6067c9291..17940c5a8 100755 --- a/tests/e2e/873_payload_overrides.sh +++ b/tests/e2e/873_payload_overrides.sh @@ -13,6 +13,9 @@ set -e MCPP="${MCPP:-mcpp}" work="$(mktemp -d)" +# A path written INTO a manifest goes through host_path (see _host_path.sh). +source "$(dirname "$0")/_host_path.sh" +work_HOST="$(host_path "$work")" trap 'rm -rf "$work"' EXIT export NO_COLOR=1 @@ -48,7 +51,7 @@ name = "app" version = "0.1.0" [build-dependencies.e2e] -plug = { path = "$work/plug", features = ["tools-x"], host-module = true } +plug = { path = "$work_HOST/plug", features = ["tools-x"], host-module = true } [targets.app] kind = "bin" @@ -83,9 +86,9 @@ grep -q "mcpp-e2e-absent-tool" <<<"$out" || fail "the refusal does not name the # The environment variable. out="$(MCPP_XLINGS_OVERRIDE_XIM_MCPP_E2E_ABSENT_TOOL="$work/opt/bin/absent-tool" probe build)" grep -q "Finished" <<<"$out" || fail "an env override still needed the payload" -grep -q "Using xim:mcpp-e2e-absent-tool ← $work/opt/bin/absent-tool \[custom · env MCPP_XLINGS_OVERRIDE_XIM_MCPP_E2E_ABSENT_TOOL\]" <<<"$out" \ +grep -q "Using xim:mcpp-e2e-absent-tool ← $work_HOST/opt/bin/absent-tool \[custom · env MCPP_XLINGS_OVERRIDE_XIM_MCPP_E2E_ABSENT_TOOL\]" <<<"$out" \ || fail "no Using line naming the env override" -grep -q "dir=$work/opt source=override program=$work/opt/bin/absent-tool" <<<"$out" \ +grep -q "dir=$work_HOST/opt source=override program=$work_HOST/opt/bin/absent-tool" <<<"$out" \ || fail "the build program did not receive the override (root, source, program)" grep -q "Finished .* · custom: xim:mcpp-e2e-absent-tool" <<<"$out" || fail "Finished does not summarise the source" diff --git a/tests/e2e/874_payload_on_request.sh b/tests/e2e/874_payload_on_request.sh index 6f8263642..1f4472641 100755 --- a/tests/e2e/874_payload_on_request.sh +++ b/tests/e2e/874_payload_on_request.sh @@ -11,6 +11,9 @@ set -e MCPP="${MCPP:-mcpp}" work="$(mktemp -d)" +# A path written INTO a manifest goes through host_path (see _host_path.sh). +source "$(dirname "$0")/_host_path.sh" +work_HOST="$(host_path "$work")" trap 'rm -rf "$work"' EXIT export NO_COLOR=1 @@ -44,7 +47,7 @@ name = "app" version = "0.1.0" [build-dependencies.e2e] -plug = { path = "$work/plug", features = ["tools-x"], host-module = true } +plug = { path = "$work_HOST/plug", features = ["tools-x"], host-module = true } [targets.app] kind = "bin" diff --git a/tests/e2e/875_toolchain_by_path.sh b/tests/e2e/875_toolchain_by_path.sh index dd51eee26..36a68fd6e 100755 --- a/tests/e2e/875_toolchain_by_path.sh +++ b/tests/e2e/875_toolchain_by_path.sh @@ -16,10 +16,13 @@ if [[ ! -x "$LLVM_ROOT/bin/clang++" ]]; then echo "SKIP: no llvm payload installed ($LLVM_ROOT)"; exit 0 fi work="$(mktemp -d)" +# Paths written INTO a manifest go through host_path (see _host_path.sh). +source "$(dirname "$0")/_host_path.sh" trap 'rm -rf "$work"' EXIT export NO_COLOR=1 T="$work/llvm" +T_HOST="$(host_path "$T")" mkdir -p "$T/bin" for f in "$LLVM_ROOT"/bin/*; do case "$f" in *.cfg) ;; *) ln -s "$f" "$T/bin/" ;; esac @@ -28,6 +31,7 @@ for d in include lib share libexec; do [ -e "$LLVM_ROOT/$d" ] && ln -s "$LLVM_RO # A linker stated by role, as a wrapper the test can change in place. printf '#!/bin/sh\nexec "%s/bin/ld.lld" "$@"\n' "$LLVM_ROOT" > "$work/ld-wrapper" chmod +x "$work/ld-wrapper" +wrapper_HOST="$(host_path "$work/ld-wrapper")" mkdir -p "$work/app/src" cat > "$work/app/src/main.cpp" <<'CPP' @@ -35,28 +39,29 @@ cat > "$work/app/src/main.cpp" <<'CPP' int main() { std::puts("built by a toolchain named by path"); return 0; } CPP write_manifest() { + local root_HOST; root_HOST="$(host_path "$1")" cat > "$work/app/mcpp.toml" <&1)" || fail "the build failed" -grep -q "Using toolchain clang .* ← $T \[custom · mcpp.toml:[0-9]*\]" <<<"$out" || fail "no Using line for the toolchain" +grep -q "Using toolchain clang .* ← $T_HOST \[custom · mcpp.toml:[0-9]*\]" <<<"$out" || fail "no Using line for the toolchain" grep -q "Finished .* · custom: toolchain" <<<"$out" || fail "Finished does not summarise the source" run="$(./target/*/*/bin/app)" [[ "$run" == "built by a toolchain named by path" ]] || fail "the program did not run: $run" ninja="$(cat target/*/*/build.ninja)" -grep -q "^cxx *= /usr/bin/env $T/bin/clang++" <<<"$ninja" || fail "the launcher does not prefix the compiler" -grep -q -- "--ld-path=$work/ld-wrapper" <<<"$ninja" || fail "the stated linker is not used" +grep -q "^cxx *= /usr/bin/env $T_HOST/bin/clang++" <<<"$ninja" || fail "the launcher does not prefix the compiler" +grep -q -- "--ld-path=$wrapper_HOST" <<<"$ninja" || fail "the stated linker is not used" [[ -z "$(find "$T/" -maxdepth 2 -name '*.cfg' -print -quit)" ]] || fail "mcpp wrote a cfg into the tree" # The fast path serves an unchanged tree, and declines once a program of it changed. @@ -77,7 +82,8 @@ out="$(MCPP_TOOLCHAIN="path:$T" "$MCPP" build 2>&1)" || fail "MCPP_TOOLCHAIN=pat grep -q "\[custom · env MCPP_TOOLCHAIN\]" <<<"$out" || fail "the env-named toolchain is not reported" # A stated family the drivers contradict is refused. -write_manifest "$T" ', family = "gcc", tools = { cxx = "'"$T/bin/clang++"'" }' +driver_HOST="$(host_path "$T/bin/clang++")" +write_manifest "$T" ', family = "gcc", tools = { cxx = "'"$driver_HOST"'" }' rm -rf target out="$("$MCPP" build 2>&1 || true)" grep -q 'stated as family "gcc"' <<<"$out" || fail "a contradicting family was accepted" @@ -96,7 +102,7 @@ gcc_base="$HOME/.mcpp/registry/data/xpkgs/xim-x-gcc" [[ -d "$gcc_base" && -n "${USERPROFILE:-}" ]] || true gcc_ver="$(ls -1 "$gcc_base" 2>/dev/null | grep -E '^[0-9]+(\.[0-9]+)*$' | sort -V | tail -1)" if [[ -n "$gcc_ver" && -x "$gcc_base/$gcc_ver/bin/g++" ]]; then - write_manifest "$gcc_base/$gcc_ver" ', tools = { ld = "'"$work/ld-wrapper"'" }' + write_manifest "$gcc_base/$gcc_ver" ', tools = { ld = "'"$wrapper_HOST"'" }' rm -rf target out="$("$MCPP" build 2>&1 || true)" grep -q "this is a gcc toolchain" <<<"$out" \