diff --git a/AGENTS.md b/AGENTS.md index a272051..32dad65 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,11 @@ its committed dependency closure, tracked by `native/provenance.json`. Preserve callback-local pointer lifetimes, one outstanding request, scoped generations, acknowledged counts, provider errors and cancellation quarantine. Keep the native-executor turn budget so ready-only callbacks cannot starve it. +- `crates/host` owns only the host-side facade over the exported `kore-vol` + procedure, never DagFS semantics: no second locate/recover/commit path, no + host-side format writer, no parsing of volume bytes outside the engine. Volumes + are offline; never open one a running machine holds. Behavior changes go to + Kore's `tools/kore-vol` and arrive through `tools/sync-native.py`. - C headers come from cbindgen 0.29.4; Zig aliases come from those declarations and target layouts from Zig 0.16.0 translation. Regenerate before checking. `test-foreign` is a host-test feature; never add a runtime C/Zig build dependency. diff --git a/Cargo.lock b/Cargo.lock index 51e9b3d..df51275 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -207,27 +207,27 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.135.6" +version = "0.135.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9195e7cefae4f6c586ef4abace39e08a4cfbbde0839479f7075f0739db5ec108" +checksum = "10c7459262c93a3c408bda96acaaa3d92868f166b33d445b20921f8378325038" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.135.6" +version = "0.135.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f47195db7c0d11bb54fc4a03f94136344059121baad9e59dcf537cf007ed45f9" +checksum = "97a25bfba0da265ef1f98728358e9bb3ab7489913e7bd759c743108892288934" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.135.6" +version = "0.135.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e38972e913f89b465f99d3a1add645d507ad8d2cb32b05161769330b72648c" +checksum = "32a9ad33269057337a70db66950f426fbdce436a66fe8071f55851a13a2e40cd" dependencies = [ "cranelift-entity", "wasmtime-internal-core", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "cranelift-bitset" -version = "0.135.6" +version = "0.135.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3352069928676bf94f502e352f4d879ba16dcd2fc413fddb9617d72eaf42127" +checksum = "d322e6584efe3b229b1164b49edf55aec561553426f7fe3e228a45e80a15a190" dependencies = [ "serde", "serde_derive", @@ -246,9 +246,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.135.6" +version = "0.135.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7332e689fc423e86b03617b071889a0e67089ed529b0cba1e22e2d630e120987" +checksum = "6a7b9f73f97da69d6edc9adbfeec068cdb2649f0221be41abd3909267cb1cd53" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -277,9 +277,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.135.6" +version = "0.135.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dc7d4ec2d00466628fa9cb8197b7850771a2db0c16a902065c8513706a7ac18" +checksum = "e7015ef10993d7129e0bce933874b60dab8187239aca4ceca352987d9907d528" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -290,24 +290,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.135.6" +version = "0.135.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08c970e27b123ff6764a525aa50dfd4a6e350022e8c27027d10b3dde5cc49fc" +checksum = "37916b08a7b17e063bae743cb746da542df7df283ced961aae213620dc11e27a" [[package]] name = "cranelift-control" -version = "0.135.6" +version = "0.135.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985a7c32c3a72fd8f44a8ead3211352c364ce509bf8b0b177184d7574f4e9397" +checksum = "8154d189f2cc2a84a95a8129773799ddcf2f4609f667047db30715db4be6aa0a" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.135.6" +version = "0.135.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce7ae3678365a2fced842f320dd90760816d57e62dc41e65804c7a86ce16dff2" +checksum = "ccbd0c43e96db753f44a84e44f680f9423ed0a475b81f301ba7c44b5db52c37b" dependencies = [ "cranelift-bitset", "serde", @@ -317,9 +317,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.135.6" +version = "0.135.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21e84fb8be0b38d9d1c52831050957586e9b1140565f47dcbcf9a08d2d124940" +checksum = "5b28ae2961cdda7a6814431b40e87bee46e57b7f92294467b32f447108e14674" dependencies = [ "cranelift-codegen", "hashbrown 0.17.1", @@ -330,15 +330,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.135.6" +version = "0.135.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f295be6751bdae90f2ecdda3ecc5ef60192ac204ec2345ec1b924b9a6554ed4" +checksum = "41eef45b50970e024d673e3fc4a0703e4fee75ca31d63b26566eb0f142300179" [[package]] name = "cranelift-native" -version = "0.135.6" +version = "0.135.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b00de3571d1817908e3a6d0e04349400a360d336be8ad07ca0f7cb0beeb3c353" +checksum = "649c7b9ad93d5d92287fee74608cacfb139602116293a46e2f25e6ea5402f05c" dependencies = [ "cranelift-codegen", "libc", @@ -347,9 +347,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.135.6" +version = "0.135.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1932d832b1d6c0fcb852d4b1fd1fe9930fc2e0cfa7d125a90ee3f86978310d" +checksum = "da04416d8e400ad7ae6d9722db72bfbddfbf36d0ff528ec422c4cc336621a172" [[package]] name = "crc" @@ -832,6 +832,7 @@ dependencies = [ "kore-ring", "kore-rt", "kore-scrub", + "rustix", "zerocopy", ] @@ -858,6 +859,13 @@ dependencies = [ "raw-cpuid", ] +[[package]] +name = "kore-host" +version = "0.1.0-alpha.1" +dependencies = [ + "kore-vol", +] + [[package]] name = "kore-late" version = "0.1.0" @@ -889,6 +897,14 @@ dependencies = [ "libm", ] +[[package]] +name = "kore-nvme" +version = "0.0.0" +dependencies = [ + "kore-abi", + "kore-driver", +] + [[package]] name = "kore-ring" version = "0.0.0" @@ -946,6 +962,46 @@ dependencies = [ "zeroize", ] +[[package]] +name = "kore-svc-block" +version = "0.0.1" +dependencies = [ + "kore-abi", + "kore-driver", + "kore-nvme", + "kore-rt", + "kore-svc-device", + "zerocopy", +] + +[[package]] +name = "kore-svc-dagfs" +version = "0.0.1" +dependencies = [ + "blake3", + "crc", + "heapless", + "kore-abi", + "kore-collections", + "kore-dagfs", + "kore-driver", + "kore-nvme", + "kore-ring", + "kore-rt", + "kore-svc-block", + "kore-svc-extract", + "zerocopy", +] + +[[package]] +name = "kore-svc-device" +version = "0.0.0" +dependencies = [ + "heapless", + "kore-abi", + "kore-rt", +] + [[package]] name = "kore-svc-extract" version = "0.0.1" @@ -955,6 +1011,16 @@ dependencies = [ "kore-rt", ] +[[package]] +name = "kore-vol" +version = "0.0.0" +dependencies = [ + "blake3", + "kore-driver", + "kore-svc-dagfs", + "rustix", +] + [[package]] name = "kore-wasi" version = "0.1.0-alpha.1" @@ -1137,9 +1203,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da9e35352d5ed6636a3b18e2099f96cf00e9dcc707f7d9d599a64ec5fabcef7" +checksum = "4e6d73adcf2f76b98d3de7a31831b6d6436fd2b6518e0ad4f7cc722a12ad4133" dependencies = [ "cranelift-bitset", "log", @@ -1149,9 +1215,9 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf76d5588cc67bac2ba972c751d4f1b4545eebf2dea8117410dc7f450bae6c97" +checksum = "5bde547f939ab9aa321a54c6694aef2e1e27741c2523c0627ff306677b3cbcff" dependencies = [ "proc-macro2", "quote", @@ -1732,9 +1798,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a26206e1b2f1403d4d17c8fa7e736b693d7e9b1a7b9a5bf860e1c580472704d" +checksum = "a2afcadb2e7d0417cb8f3f322c66eff49338b288f22290c0ca0bd3e9570732aa" dependencies = [ "addr2line", "async-trait", @@ -1784,9 +1850,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca61f6e56877ffa9633e0d1ee3ee01c4b5dd08d8e1609417a245b496163526af" +checksum = "2de252f4aedfff6c2054ea5bea2454878a931c56e2553c6356b581e5e30a7e4e" dependencies = [ "anyhow", "cpp_demangle", @@ -1815,9 +1881,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-cache" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cccad5a336fe29654eb40d1d9f515fd2354f48f262eb8608235c0ccb8cd46b5" +checksum = "bc15ec65327e0dc659da1c15658e94a3d4ad3a2ed75a35a320c84af5984b13c2" dependencies = [ "base64", "directories-next", @@ -1835,9 +1901,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d217ee3ec268aaa301fdf63af2b560c1a00ea774c5789fc8537364c2726be4ab" +checksum = "14127b0c5d6e543fa0b90747df60b3f96af499df8570f829acb7b8e2a98faa85" dependencies = [ "anyhow", "proc-macro2", @@ -1850,15 +1916,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d4bb47a14f6458a45125d6fedaac3ad95769f946755ae282763fc0343c8ccdc" +checksum = "e5ad52499e6246d30bf8bf509506bd637248eee17c8dd623e0c7dbe43e7a1036" [[package]] name = "wasmtime-internal-core" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84493645ddd62c4615bbe0bd1a6bf9740e69c11b70404f2db8726ee793442b88" +checksum = "d8838d7debfc98a69342d260a990ea6f64adf498184915fccfdbd6718bb40a9b" dependencies = [ "anyhow", "hashbrown 0.17.1", @@ -1868,9 +1934,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-cranelift" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de4b3fb57ab495f3a08032859e1b940fa63e63cb755804fb032a06bf530aeb33" +checksum = "478c09af53c545d96227b349d35c70fc78e99816923f41428ffff00974bcd145" dependencies = [ "cranelift-codegen", "cranelift-control", @@ -1894,9 +1960,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "984950b72b8660737ec8deb590c95e4103877c23c83bd0ad23b9990a789fcfe2" +checksum = "96dce7a8ea302c0f1eb4154eb1756525c90c595bbabbc590abad64d4c5ff8bbb" dependencies = [ "cc", "libc", @@ -1908,9 +1974,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "679f24b3251a3c6235bc0170efa0d503ac570678308d60bd2e7f325d16f51536" +checksum = "895f3231b24ce0c12c8a7bb129a9f6ddc6772ec78e59748e53af743dcdd733d1" dependencies = [ "cc", "object", @@ -1920,9 +1986,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2627fc8f496dcb22fad20cb97e8b77b494c225d84522642d23ec425bf17713b" +checksum = "28b60a0c1f735da6b862679b763f8bab653caa477ccc603a25752488b795fec5" dependencies = [ "libc", "wasmtime-internal-core", @@ -1931,9 +1997,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-unwinder" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64752c0f1351970e20ed8fd7704220d7871b8824bdeb252258738b58264ef191" +checksum = "b97b2c29c8f458bdf1b948058bfbb8141aad0f7abcf0f7cb3479866f4316bf3c" dependencies = [ "cranelift-codegen", "log", @@ -1943,9 +2009,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1b5eb4b2838bbbeaadd02c61e307a1fdbd47efca9d96684c04de5721157dca" +checksum = "32766b38d56927eca9a65858ea8c7b1803424db82562aafd69727aa970057561" dependencies = [ "proc-macro2", "quote", @@ -1954,9 +2020,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf92ecab96882cd1a5d9ee41c457053bccf8f1c57cfe617d38549b929ab82e91" +checksum = "407a8c055545e5c4c6c942237d934c9906cd995639a17af89f3368f0a05534eb" dependencies = [ "anyhow", "bitflags", @@ -1968,9 +2034,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi-io" -version = "48.0.6" +version = "48.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10795358c41e607f47f89847373bcbb6b5539404fda59048c457a8e56791b3e1" +checksum = "0879bd441bddd61e1c15cc5f39ec297de8ee34ffc12f5459aad3b7ab6afcf510" dependencies = [ "async-trait", "bytes", @@ -2134,27 +2200,27 @@ checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zstd" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +checksum = "cdb02c3132d251ab1606e9fc913a7da6c6ed08cc263df15edf082e1dc497d157" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.3.0" +version = "7.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +checksum = "1c8ea04668f5de8d8d51051714ba560c9536519cadcdb8829bba4fe2ad159b03" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.1.2+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 201698d..b69f68b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ repository = "https://github.com/MicroPerceptron/libkore" [workspace] resolver = "3" -members = ["tests/consumer", "crates/ffi", "crates/entry", "crates/wasi", "crates/lua"] +members = ["tests/consumer", "crates/ffi", "crates/entry", "crates/wasi", "crates/lua", "crates/host"] exclude = ["native"] [features] @@ -30,5 +30,9 @@ kore-dagfs = { path = "native/crates/dagfs", optional = true } kore-driver = { path = "native/crates/driver", optional = true } [package.metadata.kore] -# Used by tools/local-kore.py to construct one coherent local source override. -owner-paths = { kore-start = "crates/start", kore-abi = "crates/abi", kore-app = "crates/app", kore-rt = "crates/rt", kore-dagfs = "crates/dagfs", kore-driver = "crates/driver", kore-late = "crates/late" } +# Used by tools/local-kore.py to construct one coherent local source override, +# and by tools/sync-native.py as the export's roots. `kore-vol` is the host +# root: the offline volume procedure (and the filesystem service closure it +# drives) that host programs such as vKore consume; it never enters a Kore +# executable's graph. +owner-paths = { kore-start = "crates/start", kore-abi = "crates/abi", kore-app = "crates/app", kore-rt = "crates/rt", kore-dagfs = "crates/dagfs", kore-driver = "crates/driver", kore-late = "crates/late", kore-vol = "tools/kore-vol" } diff --git a/Makefile b/Makefile index 5b653e1..ff12454 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,12 @@ export ZIG .PHONY: test check fmt docs owners cross miri native-check +# `kore-vol` is the exported host root (kore-host's procedure), tested against +# this distribution's own lockfile rather than Kore's workspace. test: cargo test --locked --workspace --all-features cargo test --locked --manifest-path native/Cargo.toml -p kore-rt -p kore-dagfs --features kore-dagfs/runtime + cargo test --locked --manifest-path native/Cargo.toml -p kore-vol python3 -m unittest discover -s tools -p 'test_*.py' check: @@ -14,7 +17,7 @@ check: python3 tools/check-native.py fmt: - cargo fmt -p kore-std -p libkore-consumer -p kore-ffi -p kore-entry -p kore-wasi -p kore-lua --check + cargo fmt -p kore-std -p libkore-consumer -p kore-ffi -p kore-entry -p kore-wasi -p kore-lua -p kore-host --check $(ZIG) fmt --check build.zig build.zig.zon crates/ffi/tests/consumer.zig extern/zig/src/root.zig extern/zig/src/allocator.zig docs: @@ -45,8 +48,10 @@ cross: python3 tools/check-alloc-symbols.py python3 tools/check-entry.py +# kore-host drives real image files under POSIX record locks, which Miri does +# not model; `make test` exercises its procedure. miri: - cargo miri test --locked --workspace --all-features --exclude kore-wasi + cargo miri test --locked --workspace --all-features --exclude kore-wasi --exclude kore-host cargo miri test --locked --manifest-path native/Cargo.toml -p kore-rt erased_poll_and_drop native-check: diff --git a/README.md b/README.md index f5f9e7f..caae6f0 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,9 @@ It is an application facade, not a kernel package or a hosted operating-system runtime. Filesystem, clocks, process operations and devices use capabilities actually endowed to the process; selecting a Cargo feature grants none of them. The libraries do not install a global allocator or start a second executor. +`kore-host` is the facade's counterpart on the other side of the machine +boundary: host programs (vKore, volume browsers, packagers) reach a Kore +machine's volume through the same exported owners a booted Kore runs. | Package | Use | | --- | --- | @@ -14,8 +17,9 @@ The libraries do not install a global allocator or start a second executor. | `kore-ffi` | C headers and Zig module over a Rust-founded host, including filesystem, time, random, allocation and worker APIs; no standalone C/Zig CRT | | `kore-lua` | Embeddable Lua 5.5 engine and bridge; interactive and opt-in restricted worker policies | | `kore-wasi` | WASI 0.2 component host composed by an embedder with its Wasmtime engine and native owners | +| `kore-host` | Host-side (Linux/macOS) tooling: a Kore machine's DagFS volume, offline, through the filesystem row's own engine | -All five package manifests set `publish = false`; consume a reviewed Git +All six package manifests set `publish = false`; consume a reviewed Git revision or a deliberately composed local checkout, not a registry version. The WASI host supports the `wasi:cli/command` world and native-backed I/O, filesystem, clocks and random imports; unsupported imports such as sockets @@ -124,10 +128,31 @@ The host tests exercise Rust logic and the linked foreign/Lua harnesses; they do not prove founded service behavior or physical-device operation. Ordinary Rust dependency builds do not need the C/Zig generation tools. -The roughly 5 MiB `native/` export contains 14 required packages, including optional -and test dependencies, with unchanged source paths and manifests. It contains no -kernel or boot images. `make native-check` detects source drift. For work beside -a local Kore tree, follow [local integration](docs/integration.md). +The roughly 11 MiB `native/` export contains 26 required packages, including optional +and test dependencies, with unchanged source paths and manifests: the application +owners above, and the host root `tools/kore-vol` with the filesystem service closure +it drives (DagFS engine, block, device, memory and swap services, the NVMe driver). +It contains no kernel or boot images. `make native-check` detects source drift. For +work beside a local Kore tree, follow [local integration](docs/integration.md). + +## Host programs + +A program on a Linux or macOS host that keeps Kore machines depends on `kore-host`: + +```toml +[target.'cfg(unix)'.dependencies] +kore-host = { git = "https://github.com/MicroPerceptron/libkore", rev = "" } +``` + +`kore_host::volume` formats a blank disk image as a first boot would, installs a +host tree under a terminal's home, removes one, and mounts a volume read-only to +list, walk and read what a machine left — screenshots, logs, a project's files — +through the exported `kore-vol` procedure over the DagFS engine, so what it reads +is what Kore reads and what it writes is what the next boot mounts. Volumes are +offline only: the image device refuses an image a running QEMU holds. A commit +writes `volume::FORMAT_VERSION`; a volume a newer Kore wrote is refused, so pick the +libkore revision whose export matches the machines' Kore build. See the crate +documentation for bounds and the Linux whole-disk mode. ## Alpha boundary diff --git a/crates/host/Cargo.toml b/crates/host/Cargo.toml new file mode 100644 index 0000000..ac47f59 --- /dev/null +++ b/crates/host/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "kore-host" +version = "0.1.0-alpha.1" +edition = "2024" +publish = false +description = "Host-side Kore tooling: a Kore machine's volume, offline, through the native filesystem owner" +repository = "https://github.com/MicroPerceptron/libkore" + +# The exported host root (native/provenance.json): the offline volume +# procedure over the filesystem row's own engine. A host program depends on +# this facade, never on a second copy of that procedure. Unix hosts only: +# the image device takes POSIX record locks, as QEMU does on the images it +# runs. +[dependencies] +kore-vol = { path = "../../native/tools/kore-vol" } diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs new file mode 100644 index 0000000..e184dd8 --- /dev/null +++ b/crates/host/src/lib.rs @@ -0,0 +1,76 @@ +//! Host-side Kore tooling: what a program on a Linux or macOS host needs to +//! work with a Kore machine it keeps. +//! +//! [`kore-std`](https://github.com/MicroPerceptron/libkore) is the facade for +//! programs running *under* Kore. This crate is its counterpart on the other +//! side of the machine boundary — a VM supervisor such as vKore, a volume +//! browser, a packager — and it is built from the same exported native +//! owners (`native/provenance.json`), never from a second copy of them. +//! +//! # A machine's volume, offline +//! +//! A Kore machine keeps its files in DagFS, which no host kernel mounts. +//! [`volume`] drives the filesystem row's own engine — the exported +//! `kore-vol` procedure over `kore-svc-dagfs` and `kore-driver`'s image +//! device — so what it reads is what a booted Kore reads, and what it writes +//! is what the next boot mounts: +//! +//! - **Found as the boot finds it:** the whole disk when it is Kore's, else +//! the GPT partition designated `KORE_FS`. A raw disk image, or on Linux a +//! machine's disk itself (its block device, with read and, to write, write +//! permission on its node). +//! - **Offline only.** A running machine keeps its free extents and its next +//! root slot in memory, so a second writer would corrupt the volume +//! silently. The image device takes a whole-file record lock and refuses an +//! image another process holds — QEMU holds the images it runs — so a +//! machine must be shut down before its volume is opened here. +//! - **All or nothing.** [`volume::put`] and [`volume::rm`] are one root +//! publication each, read back through a fresh recovery before they +//! report success; a failure part way leaves the volume as it was. +//! - **One format version.** A commit writes [`volume::FORMAT_VERSION`]. A +//! volume a newer Kore wrote is refused as `Incompatible`: use the libkore +//! revision whose export matches the Kore build the machine boots. +//! - **Kore's bounds,** checked before anything is written: +//! [`volume::MAX_OBJECTS`] objects on a volume, [`volume::MAX_DIR_ENTRIES`] +//! entries in a directory, [`volume::MAX_PATH_DEPTH`] levels, names of at +//! most 32 bytes ([`volume::Name`]). +//! +//! Paths are a terminal's: relative to terminal `face`'s home (`work/face-N` +//! on the volume), or from the volume's root when they start with `/` +//! (reads only). Every call runs the engine on a thread of its own with the +//! stack recovery needs, so callers need none. +//! +//! ```no_run +//! use kore_host::volume::{self, Kind}; +//! use std::path::Path; +//! +//! # fn main() -> Result<(), volume::VolError> { +//! let disk = Path::new("machine.raw"); +//! // What terminal 0 saved with PrintScreen, as the machine left it. +//! let shots = volume::read(disk, None, |mount| { +//! let mut shots = Vec::new(); +//! mount.walk(0, "pictures/screenshots", &mut |entry| { +//! if entry.kind == Kind::File { +//! shots.push((entry.path.to_owned(), entry.len)); +//! } +//! Ok(()) +//! })?; +//! Ok(shots) +//! })?; +//! for (name, len) in shots { +//! println!("{name}: {len} bytes"); +//! } +//! # Ok(()) +//! # } +//! ``` +#![forbid(unsafe_code)] + +/// A Kore machine's volume, offline, through the filesystem row's own engine +/// (the crate docs). +pub mod volume { + pub use kore_vol::{ + DEFAULT_BLOCK_SIZE, FORMAT_VERSION, Formatted, Installed, Kind, Listed, Listing, + MAX_DIR_ENTRIES, MAX_OBJECTS, MAX_PATH_DEPTH, Mount, MountStatus, Name, Removed, + VOLUME_STACK_BYTES, VolError, Walked, format, is_face, put, read, rm, + }; +} diff --git a/crates/host/tests/volume.rs b/crates/host/tests/volume.rs new file mode 100644 index 0000000..c19f5b5 --- /dev/null +++ b/crates/host/tests/volume.rs @@ -0,0 +1,109 @@ +//! The exported volume procedure as a host program consumes it from libkore: +//! the distribution's own closure and lockfile, not Kore's workspace. + +use std::{ + fs, + path::{Path, PathBuf}, + sync::atomic::{AtomicU32, Ordering}, +}; + +use kore_host::volume::{self, Kind, Listing, VolError}; + +/// A fresh scratch directory under Cargo's per-target temporary area. +fn scratch(test: &str) -> PathBuf { + static SEQUENCE: AtomicU32 = AtomicU32::new(0); + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!( + "kore-host-{test}-{}-{}", + std::process::id(), + SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir +} + +/// A zeroed (sparse) raw image: blank roots, as `qemu-img create` makes one. +fn blank_image(dir: &Path, bytes: u64) -> PathBuf { + let path = dir.join("disk.raw"); + fs::File::create(&path).unwrap().set_len(bytes).unwrap(); + path +} + +#[test] +fn a_host_tree_installs_and_reads_back_through_the_distribution() { + let dir = scratch("roundtrip"); + let image = blank_image(&dir, 32 << 20); + let source = dir.join("shots"); + fs::create_dir_all(source.join("2026-11")).unwrap(); + fs::write(source.join("a.png"), [0x89, b'P', b'N', b'G']).unwrap(); + fs::write(source.join("2026-11/b.log"), b"boot: ok\n".repeat(1000)).unwrap(); + + // Blank roots are the first boot's to format, unless asked. + assert!(matches!( + volume::read(&image, None, |_| Ok(())), + Err(VolError::Blank) + )); + let formatted = volume::format(&image, None).unwrap(); + let installed = volume::put(&image, None, 0, &source, "pictures/screenshots", false).unwrap(); + assert_eq!((installed.files, installed.directories), (2, 2)); + assert!(installed.after > formatted.generation); + + let (listing, mut walked, log) = volume::read(&image, None, |mount| { + let listing = mount.list(0, Some("pictures/screenshots"))?; + let mut walked = Vec::new(); + mount.walk(0, "pictures", &mut |entry| { + walked.push((entry.path.to_owned(), entry.kind, entry.len)); + Ok(()) + })?; + let mut log = Vec::new(); + mount.read_to(0, "pictures/screenshots/2026-11/b.log", &mut log)?; + Ok((listing, walked, log)) + }) + .unwrap(); + let Listing::Directory(mut entries) = listing else { + panic!("the screenshots directory lists as a directory"); + }; + entries.sort_by(|a, b| a.name.cmp(&b.name)); + assert_eq!( + entries + .iter() + .map(|entry| (entry.name.as_str(), entry.kind, entry.len)) + .collect::>(), + [("2026-11", Kind::Directory, 0), ("a.png", Kind::File, 4)] + ); + walked.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!( + walked, + [ + ("screenshots".into(), Kind::Directory, 0), + ("screenshots/2026-11".into(), Kind::Directory, 0), + ("screenshots/2026-11/b.log".into(), Kind::File, 9000), + ("screenshots/a.png".into(), Kind::File, 4), + ] + ); + assert_eq!(log, b"boot: ok\n".repeat(1000)); + + let removed = volume::rm(&image, None, 0, "pictures/screenshots/a.png").unwrap(); + assert!(removed.after > removed.before); + let gone = volume::read(&image, None, |mount| { + mount.list(0, Some("pictures/screenshots/a.png")) + }); + assert!(matches!(gone, Err(VolError::NotFound(_)))); +} + +#[test] +fn what_kore_cannot_name_is_refused_before_the_image_is_written() { + let dir = scratch("names"); + let image = blank_image(&dir, 16 << 20); + volume::format(&image, None).unwrap(); + let before = fs::read(&image).unwrap(); + let source = dir.join("too-long"); + fs::create_dir_all(&source).unwrap(); + fs::write(source.join("a-name-of-thirty-three-bytes.txt0"), b"x").unwrap(); + assert!(volume::Name::new("a-name-of-thirty-three-bytes.txt0").is_err()); + let refused = volume::put(&image, None, 0, &source, "x", false); + assert!(matches!(refused, Err(VolError::Host(message)) if message.contains("not a kore name"))); + assert!(fs::read(&image).unwrap() == before); + assert!(volume::is_face(0)); + const { assert!(volume::FORMAT_VERSION >= 5) }; +} diff --git a/crates/wasi/Cargo.toml b/crates/wasi/Cargo.toml index 62bf120..f2698fe 100644 --- a/crates/wasi/Cargo.toml +++ b/crates/wasi/Cargo.toml @@ -15,16 +15,16 @@ kore = { package = "kore-std", path = "../..", features = ["heap"] } # Components, and async host imports: a guest's call into WASI suspends its # fiber and the Kore executor runs until the import's future is ready. The # engine's platform (memory, fibers, ISA) is the embedding program's choice. -wasmtime = { version = "48.0.6", default-features = false, features = [ +wasmtime = { version = "48.0.8", default-features = false, features = [ "runtime", "component-model", "async", ] } # WASI 0.2's `wasi:io` (pollables, input and output streams) as traits the # native process stream implements; `no_std` by design. -wasmtime-wasi-io = { version = "48.0.6", default-features = false } +wasmtime-wasi-io = { version = "48.0.8", default-features = false } # Host tests resolve the guest fixture against the linker with a std # Wasmtime: the Kore-side engine has no host to run on. [dev-dependencies] -wasmtime = { version = "48.0.6", features = ["std", "cranelift", "component-model"] } +wasmtime = { version = "48.0.8", features = ["std", "cranelift", "component-model"] } diff --git a/docs/integration.md b/docs/integration.md index 25a1fb5..40d3437 100644 --- a/docs/integration.md +++ b/docs/integration.md @@ -27,6 +27,22 @@ snapshot must be resolved deliberately upstream; regenerating may not silently erase the edit. The snapshot is one source per owner, not a parallel allocator, runtime or handle implementation to evolve independently. +## Host programs and the exported host root + +`tools/kore-vol` is the export's host root: the offline volume procedure a host +program drives through `kore-host` (`crates/host`). The exporter follows its path +dependencies like any other root, so the filesystem service closure it composes — +`kore-svc-dagfs`, `kore-svc-block`, `kore-svc-device`, `kore-svc-mm`, `kore-svc-swap` +and `kore-nvme` — is exported byte-exact beside the application owners. Those +packages exist here only as that procedure's dependencies: a Kore executable's graph +never reaches them through `kore-std`, and `kore-host` exposes only the volume +procedure, never the service internals. + +A host consumer pins one libkore revision; its `native/provenance.json` names the +Kore revision whose DagFS format the procedure writes (`kore_host::volume::FORMAT_VERSION`). +Choose the libkore revision whose export matches the Kore build the host's machines +boot: the engine refuses a volume a newer Kore wrote, rather than guessing. + ## Composing with a local Kore checkout Cargo treats a path dependency and a Git dependency as different packages even diff --git a/docs/provenance.md b/docs/provenance.md index 138fc13..6554b1d 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -212,3 +212,109 @@ assertions on all three Kore architectures. The complete local-owner override passed its type-identity checks and duplicate-owner negative control. Both workspace dependency locks remain unchanged and pass locked resolution; the exporter verifies byte equality against the recorded Kore commit. + +## Host volume access: `kore-host` over the exported `kore-vol` + +The export now records Kore `ab6e7b157659cc4158ca30bb70a14bb64b0c6820`: Kore main +`e5e0b96a` (Cargo version 0.40.0, 2026-11-01) plus four commits on Kore's +`ft/amazing-cray-qf1tpm`. Three make `tools/kore-vol` a library with its binary +printing over it (`beda0947`, `6f56689d`, `1b6cb322`); the fourth (`ab6e7b15`) +makes a mount release every engine handle it opens on every path, so a +long-lived reader (a volume browser) cannot exhaust its session's 32 handles +on missing files, directories read as files, or reads it stops early. The library +returns `format`/`put`/`rm` results as data, mounts a volume read-only for +`list`/`read_to`/`walk`, names the DagFS `FORMAT_VERSION`, reports +`VolError::is_in_use`, and documents that its image lock is a per-process +POSIX record lock. The binary's output, refusals and exit statuses are +unchanged. + +`tools/kore-vol` is a new owner root. Following its path dependencies adds the +filesystem service closure it drives — `kore-svc-dagfs`, `kore-svc-block`, +`kore-svc-device`, `kore-svc-mm`, `kore-svc-swap`, `kore-nvme` and +`kore-svc-extract-worker-c` — for 26 packages and 381 exact files. Publishing +that closure was an explicit decision: host programs (vKore first) must read and +write Kore volumes with the filesystem row's own engine rather than a second +implementation of the DagFS format. `crates/host` (`kore-host`) is the facade +they depend on; it adds no behavior of its own. + +Locally, with the pinned nightly: `cargo test` for the workspace (every feature +except `test-foreign`), the native `kore-rt`/`kore-dagfs` suites and the +exported `kore-vol` suites (9 CLI, 5 library, 1 unit, at `1b6cb322`) against +this distribution's lock, the tool tests, the no-default-feature check, Clippy with +warnings denied, documentation with warnings denied, all three Kore user-target +checks for `kore-std`/`kore-ffi`/`kore-entry`, `check-entry`, Miri, the owner +graph, the native drift check and the local-composition fixture all passed. +`kore-host`'s own tests format, install, walk, read and remove through the +exported closure. Not run locally: everything needing Zig 0.16.0 (`zig fmt`, +`test-foreign`, `bindings-check`, `kore-lua`'s Kore-target build), because this +environment could not execute a Zig toolchain; CI's host job runs them. The +upstream device suite (loop-device GPT disks, as root) passed in Kore's tree. + +`ab6e7b15` changes two files, both in `tools/kore-vol`, and nothing the +workspace builds besides. For it, the native drift check, the tool tests, the +exported `kore-vol` suites (9 CLI, 6 library, 1 unit; the sixth library test +drives a hundred of each refusal through one mount and then lists and reads +whole) and `kore-host`'s tests were rerun and passed; the rest of the gate set +above was not, since no other exported byte or workspace source changed. + +A production x86 ISO built from Kore `1b6cb322` (`ab6e7b15` changes only the +host-side tool, which the ISO does not carry; `make iso-sealed-boot-x86 +BOOT_FEATURES=boot PAGING_PROFILE=4k48 MODE=release`, SHA-256 +`029ddd7e1e1cc1819c7e3ecf7ec22ffc3f0f6e01105972cd4d968313465aab23`) booted under +QEMU 8.2 TCG on the gate's topology: 19/19 services and beacons, `first-shell: +ready`, `app-store: seeded 44/44`, `home-skeleton: homes: 1/1`. After the machine +was stopped, the host-side `kore-vol` read the volume that guest had formatted +and populated. These are emulator results, not physical-hardware acceptance. + +## Kore main, with the host volume procedure merged + +The export now records Kore `7074bda28c416ffe34562857ee43cbc883482782`, main +once MicroPerceptron/kore#1475 landed. That pull request was merged by rebase, +so the four `tools/kore-vol` commits the previous export named are on main as +`e39d7308`, `99ef4b86`, `3445d543` and `9d616ce0`, and the exported +`tools/kore-vol` is byte-identical to `ab6e7b15`'s. `7074bda2` itself makes the +kernel access the ACPI PM1 registers at their width; it changes nothing in the +closure. + +Main took 52 commits between `e5e0b96a` and the merge. Eleven reach the +closure, which keeps its 26 packages and now holds 382 files (`kore-window` +gains `desktop.rs`): + +- DagFS. A cwd anchor, and `FsRequest::MatchView` to confirm that a remembered + route still names the launcher's view (`45e7d02b`, across `kore-dagfs`, + `kore-svc-dagfs` and `kore-abi`); `.` adds no path step, and `path::resolve` + folds `..` and rooted paths (`87fa74e5`); the service reaps a dead view by + probing its lineage in the kernel instead of trusting its record + (`77f00903`, with `kore-rt`'s `probe_chan_send_lineage`). +- `kore-abi`: a launch context may fill the payload it borrows; + `PROCESS_CONTEXT_BYTES` is `PROCESS_OUTPUT_BYTES`, and `PROCESS_IO_ABI` + stays 13 (`44abeeca`). +- `kore-rt`: a mapped view's registry entry is retired exactly once + (`602ab362`). +- `kore-driver`: `IntakeError::failed_before_device_publish` tells a DMA + import refused at its CPU view from one refused at device publication + (`8eacc00e`). +- `kore-window` and `kore-app`: System Settings (`047f82f2`, `751fa23c`, + `387b9a08`), and the dock's capacity without the window runtime feature + (`afa21902`). +- The services' `boot.toml` digests, resealed twice: over Wasmtime 48.0.8 + (`89a85e45`) and over the POSIX shims (`0ed97d12`). + +No exported manifest changed, so `native/Cargo.lock` did not. The DagFS +on-disk format stays at version 5 (`DIGEST_FORMAT_VERSION`). + +Locally, with the pinned nightly, the gate set the previous section lists passed +again on this export: `cargo test` for the workspace (every feature except +`test-foreign`), the native `kore-rt`/`kore-dagfs` suites, the exported +`kore-vol` suites (1 unit, 9 CLI, 6 library) and, as root over loop devices, +its device suite (4), `kore-host`'s volume tests (2), the tool tests, the +no-default-feature check, Clippy and documentation with warnings denied, the +three Kore user-target checks for `kore-std`/`kore-ffi`/`kore-entry`, +`check-alloc-symbols`, `check-entry`, Miri, the owner graph, the native drift +check and the local-composition fixture. The declaration half of +`bindings-check` ran without Zig: cbindgen 0.29.4 regenerates +`extern/c/include/kore.h` byte-identical, and the C consumers compile for all +three ISAs. The Zig namespace and the `stdlib.h`/`pthread.h` aliases derive from +that header's function list alone, so they are current as well. Not run locally: +`zig fmt`, `test-foreign`, the rest of `bindings-check` and `kore-lua`'s +Kore-target build; CI's host job runs them. diff --git a/native/Cargo.lock b/native/Cargo.lock index f135334..8eef346 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -158,6 +158,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "find-msvc-tools" version = "0.1.10" @@ -269,6 +279,7 @@ dependencies = [ "kore-ring", "kore-rt", "kore-scrub", + "rustix", "virtio-accel-core", "zerocopy", ] @@ -303,6 +314,21 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "kore-nvme" +version = "0.0.0" +dependencies = [ + "bytemuck", + "crc", + "gpt_disk_types", + "kore-abi", + "kore-driver", + "kore-nvme", + "kore-ring", + "kore-rt", + "kore-svc-swap", +] + [[package]] name = "kore-ring" version = "0.0.0" @@ -347,6 +373,48 @@ dependencies = [ "kore-rt", ] +[[package]] +name = "kore-svc-block" +version = "0.0.1" +dependencies = [ + "kore-abi", + "kore-driver", + "kore-nvme", + "kore-rt", + "kore-svc-device", + "zerocopy", +] + +[[package]] +name = "kore-svc-dagfs" +version = "0.0.1" +dependencies = [ + "blake3", + "crc", + "heapless", + "kore-abi", + "kore-collections", + "kore-dagfs", + "kore-driver", + "kore-nvme", + "kore-ring", + "kore-rt", + "kore-svc-block", + "kore-svc-extract", + "kore-svc-extract-worker-c", + "kore-svc-extract-worker-rust", + "zerocopy", +] + +[[package]] +name = "kore-svc-device" +version = "0.0.0" +dependencies = [ + "heapless", + "kore-abi", + "kore-rt", +] + [[package]] name = "kore-svc-extract" version = "0.0.1" @@ -356,6 +424,17 @@ dependencies = [ "kore-rt", ] +[[package]] +name = "kore-svc-extract-worker-c" +version = "0.0.1" +dependencies = [ + "blake3", + "cc", + "heapless", + "kore-svc-extract", + "kore-svc-extract-worker-rust", +] + [[package]] name = "kore-svc-extract-worker-rust" version = "0.0.1" @@ -370,6 +449,14 @@ dependencies = [ "xmlparser", ] +[[package]] +name = "kore-svc-mm" +version = "0.0.0" +dependencies = [ + "kore-abi", + "kore-rt", +] + [[package]] name = "kore-svc-output" version = "0.0.1" @@ -380,10 +467,30 @@ dependencies = [ "kore-uart", ] +[[package]] +name = "kore-svc-swap" +version = "0.0.1" +dependencies = [ + "kore-abi", + "kore-rt", + "kore-svc-mm", + "lz4_flex", +] + [[package]] name = "kore-uart" version = "0.0.0" +[[package]] +name = "kore-vol" +version = "0.0.0" +dependencies = [ + "blake3", + "kore-driver", + "kore-svc-dagfs", + "rustix", +] + [[package]] name = "kore-window" version = "0.0.1" @@ -401,12 +508,24 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "log" version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" + [[package]] name = "memchr" version = "2.8.3" @@ -464,6 +583,19 @@ dependencies = [ "bitflags", ] +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -614,6 +746,21 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7747200a37db419a82c884730d5c31d1d80a6b4d14cc3de312064e4793dc289" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "winnow" version = "1.0.4" diff --git a/native/Cargo.toml b/native/Cargo.toml index 308239f..20a77a3 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -16,7 +16,15 @@ members = [ "crates/start", "crates/uart", "crates/window", + "drivers/nvme", + "services/block", + "services/dagfs", + "services/device", "services/extract", + "services/extract-worker-c", "services/extract-worker-rust", + "services/mm", "services/output", + "services/swap", + "tools/kore-vol", ] diff --git a/native/crates/abi/src/cap.rs b/native/crates/abi/src/cap.rs index 2b767f1..f4d138f 100644 --- a/native/crates/abi/src/cap.rs +++ b/native/crates/abi/src/cap.rs @@ -191,6 +191,9 @@ pub enum ResourceKind { SystemObserve, /// Inert, one-shot proof that one executable endpoint authorized one /// exact invocation supply. It exposes no endpoint or memory authority. + /// Its resource id is that supply's revocation [`Tag`] index, so a + /// receiver holding both can read which landed supply it vouches for + /// ([`CapRecord::scope_id`] against the supply's [`Cid::tag`]). InvocationWitness, /// A task's **SPMD batch-reservation claim** on a CPU (G6c — /// `res/SPMD.md` §7): the `CpuShare`/`Fpu` sibling marking a core @@ -303,6 +306,58 @@ impl CapRecord { mac: [0; 16], } } + + /// The per-kind id of the resource this record authorizes: the id half + /// of [`scope_rid`](Self::scope_rid). Meaningful together with + /// [`kind`](Self::kind); ids are unique only within a kind. + #[inline] + pub const fn scope_id(&self) -> u32 { + rid_id(self.scope_rid) + } +} + +/// A resource's global identity: its kind interleaved bit by bit with its +/// per-kind id, the kind on the even bits and the id on the odd bits. Ids are +/// unique within a kind (the catalog discipline), so a rid is unique across +/// kinds, and the kernel binds it into every stamped record as +/// [`CapRecord::scope_rid`]. This is the one encoding the kernel's +/// `Resource::rid` and every userland reader of a record share. +#[inline] +pub const fn rid(kind: u32, id: u32) -> u64 { + spread_even(kind) | (spread_even(id) << 1) +} + +/// The kind half of a [`rid`]. +#[inline] +pub const fn rid_kind(rid: u64) -> u32 { + gather_even(rid) +} + +/// The per-kind id half of a [`rid`]. +#[inline] +pub const fn rid_id(rid: u64) -> u32 { + gather_even(rid >> 1) +} + +/// Move bit `i` of `x` to bit `2i` (the classic Morton spread). +const fn spread_even(x: u32) -> u64 { + let mut x = x as u64; + x = (x | (x << 16)) & 0x0000_FFFF_0000_FFFF; + x = (x | (x << 8)) & 0x00FF_00FF_00FF_00FF; + x = (x | (x << 4)) & 0x0F0F_0F0F_0F0F_0F0F; + x = (x | (x << 2)) & 0x3333_3333_3333_3333; + (x | (x << 1)) & 0x5555_5555_5555_5555 +} + +/// The inverse of [`spread_even`]: bit `2i` of `x` back to bit `i`, ignoring +/// the odd bits. +const fn gather_even(x: u64) -> u32 { + let mut x = x & 0x5555_5555_5555_5555; + x = (x | (x >> 1)) & 0x3333_3333_3333_3333; + x = (x | (x >> 2)) & 0x0F0F_0F0F_0F0F_0F0F; + x = (x | (x >> 4)) & 0x00FF_00FF_00FF_00FF; + x = (x | (x >> 8)) & 0x0000_FFFF_0000_FFFF; + ((x | (x >> 16)) & 0x0000_0000_FFFF_FFFF) as u32 } // Frozen wire pins (#81; the crate-header wire-freeze discipline) — the stamped arena-slot record @@ -314,4 +369,55 @@ const _: () = { assert!(size_of::() == 8); assert!(size_of::() == 4); assert!(size_of::() == 1); + // The rid layout is wire: every stamped record carries one. Pin the bit + // assignment (kind even, id odd) so a change cannot pass as a refactor. + assert!(rid(1, 0) == 0b01); + assert!(rid(0, 1) == 0b10); + assert!(rid(u32::MAX, 0) == 0x5555_5555_5555_5555); + assert!(rid(0, u32::MAX) == 0xAAAA_AAAA_AAAA_AAAA); }; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_rid_splits_back_into_its_kind_and_id() { + let samples = [ + 0, + 1, + 2, + 0x5555_5555, + 0xAAAA_AAAA, + 0x8000_0001, + 0x1234_5678, + u32::MAX - 1, + u32::MAX, + ]; + for kind in samples { + for id in samples { + let rid = rid(kind, id); + assert_eq!( + (rid_kind(rid), rid_id(rid)), + (kind, id), + "{kind:#x}/{id:#x}" + ); + } + } + } + + #[test] + fn an_invocation_witness_record_names_its_supply_tag() { + let supply = Cid::new(7, Tag(0x0123_4567)); + let witness = CapRecord { + scope_rid: rid(ResourceKind::InvocationWitness as u32, supply.tag().0), + kind: ResourceKind::InvocationWitness as u16, + ..CapRecord::vacant() + }; + assert_eq!(witness.scope_id(), supply.tag().0); + assert_eq!( + rid_kind(witness.scope_rid), + ResourceKind::InvocationWitness as u32 + ); + } +} diff --git a/native/crates/abi/src/command.rs b/native/crates/abi/src/command.rs index 9cb510e..d62b18c 100644 --- a/native/crates/abi/src/command.rs +++ b/native/crates/abi/src/command.rs @@ -11,9 +11,10 @@ use crate::image::{ImageName, NAME_BYTES}; pub const COMMAND_SNAPSHOT_ABI: u8 = 1; /// The largest task-local command set accepted by the decoder. Raised /// from 32 when the desktop's `editor` and `power-ui` rows joined the two -/// native filesystem verbs; the byte budget (`PROCESS_CONTEXT_BYTES`) is -/// the other bound, proven by exec's host test over the real catalogue. -pub const MAX_COMMAND_SNAPSHOT_ENTRIES: usize = 40; +/// native filesystem verbs, and from 40 when Files and the documents editor +/// became two rows; the byte budget (`PROCESS_CONTEXT_BYTES`) is the other +/// bound, proven by exec's host test over the real catalogue. +pub const MAX_COMMAND_SNAPSHOT_ENTRIES: usize = 48; /// The launcher holds a power-control face and therefore handles these two /// privileged controls in-process. pub const COMMAND_SNAPSHOT_SYSTEM_POWER: u8 = 1 << 0; diff --git a/native/crates/abi/src/crash.rs b/native/crates/abi/src/crash.rs index dc02995..642099a 100644 --- a/native/crates/abi/src/crash.rs +++ b/native/crates/abi/src/crash.rs @@ -7,7 +7,7 @@ use zerocopy::byteorder::little_endian::{U16, U32, U64}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned}; -pub const VERSION: u16 = 2; +pub const VERSION: u16 = 3; pub const MIN_BUFFER_BYTES: usize = 1024; pub const MAX_FRAMES: usize = 16; pub const MESSAGE_BYTES: usize = 256; @@ -71,6 +71,13 @@ bitflags::bitflags! { const REGISTERS_PARTIAL = 64; /// The parent-registered RAM buffer received the record before exit. const BUFFER_CAPTURED = 128; + /// The faulting thread's fault upcall was given this fault and + /// declined it (`session::DERIVE_KIND_FAULT_UPCALL`): the record is + /// the fault as it was raised, not the upcall's decline. + const UPCALL_DECLINED = 256; + /// Raised inside the thread's fault upcall, while it handled the + /// fault at [`CrashRecord::upcall_pc`]. + const IN_UPCALL = 512; } } @@ -104,7 +111,10 @@ pub struct CrashRecord { task_name: [u8; 63], pub task: crate::task::TaskId, pub task_id_quality: u8, - reserved: [u8; 223], + /// With [`CaptureFlags::IN_UPCALL`]: the `pc` of the fault the upcall + /// was handling. Zero otherwise. + pub upcall_pc: U64, + reserved: [u8; 215], } impl CrashRecord { pub fn set_task_name(&mut self, name: &str) { @@ -142,7 +152,8 @@ impl CrashRecord { task_name: [0; 63], task: crate::task::TaskId::from_bytes([0; 16]), task_id_quality: 0, - reserved: [0; 223], + upcall_pc: U64::new(0), + reserved: [0; 215], } } pub fn set_register(&mut self, index: usize, value: u64) -> bool { @@ -160,6 +171,22 @@ impl CrashRecord { self.address = U64::new(address); self.add_flags(CaptureFlags::ADDRESS_VALID); } + /// Mark this fault as raised inside the thread's fault upcall, which + /// was handling the fault at `pc`. + pub fn set_in_upcall(&mut self, pc: u64) { + self.upcall_pc = U64::new(pc); + self.add_flags(CaptureFlags::IN_UPCALL); + } + /// A fault is either one the upcall declined or one raised inside an + /// upcall, never both, and only the latter carries an upcall `pc`. + fn upcall_evidence_is_consistent(&self) -> bool { + let flags = CaptureFlags::from_bits_retain(self.flags.get()); + if flags.contains(CaptureFlags::IN_UPCALL) { + !flags.contains(CaptureFlags::UPCALL_DECLINED) + } else { + self.upcall_pc.get() == 0 + } + } pub fn set_message(&mut self, message: &str) { let mut len = message.len().min(MESSAGE_BYTES); while !message.is_char_boundary(len) { @@ -183,7 +210,8 @@ impl CrashRecord { && self.registers_valid.get() >> 32 == 0 && self.frame_count.get() as usize <= MAX_FRAMES && self.message_len.get() as usize <= MESSAGE_BYTES - && self.reserved == [0; 223] + && self.reserved == [0; 215] + && self.upcall_evidence_is_consistent() && self.reserved_identity.get() == 0 && crate::task::TaskIdQuality::decode(self.task_id_quality).is_some() && self.task_name_len <= 63 @@ -238,13 +266,44 @@ mod tests { assert!(r.encode_into(&mut bytes)); assert_eq!(CrashRecord::decode(&bytes), Some(r)); assert!(CrashRecord::decode(&bytes[..1023]).is_none()); - bytes[8] = 3; - assert!(CrashRecord::decode(&bytes).is_none()); + for version in [VERSION - 1, VERSION + 1] { + bytes[8..10].copy_from_slice(&version.to_le_bytes()); + assert!(CrashRecord::decode(&bytes).is_none(), "version {version}"); + } assert!(r.encode_into(&mut bytes)); bytes[1023] = 1; assert!(CrashRecord::decode(&bytes).is_none()); } #[test] + fn upcall_evidence_round_trips_and_stays_consistent() { + let mut nested = + CrashRecord::new(Architecture::X86_64, FaultKind::Memory, 0x5000, 0x9000, 6); + nested.set_in_upcall(0x4321); + let mut bytes = [0; MIN_BUFFER_BYTES]; + assert!(nested.encode_into(&mut bytes)); + let back = CrashRecord::decode(&bytes).expect("a nested fault decodes"); + assert_eq!(back.upcall_pc.get(), 0x4321); + assert!(CaptureFlags::from_bits_retain(back.flags.get()).contains(CaptureFlags::IN_UPCALL)); + + let mut declined = CrashRecord::new( + Architecture::Aarch64, + FaultKind::InvalidInstruction, + 0x10, + 0x20, + 0, + ); + declined.add_flags(CaptureFlags::UPCALL_DECLINED); + assert!(declined.validate()); + // Both at once, or an upcall pc without the flag, is not a record. + let mut both = declined; + both.set_in_upcall(0x30); + assert!(!both.validate()); + let mut stray = CrashRecord::new(Architecture::Riscv64, FaultKind::Memory, 0, 0, 0); + stray.upcall_pc = U64::new(1); + assert!(!stray.validate()); + assert!(!stray.encode_into(&mut bytes)); + } + #[test] fn validity_distinguishes_zero_from_missing_and_text_is_utf8_bounded() { let mut r = CrashRecord::new(Architecture::Aarch64, FaultKind::Panic, 0, 0, 0); assert!(r.set_register(0, 0)); diff --git a/native/crates/abi/src/identity.rs b/native/crates/abi/src/identity.rs new file mode 100644 index 0000000..0be8d8f --- /dev/null +++ b/native/crates/abi/src/identity.rs @@ -0,0 +1,135 @@ +//! **The build-identity block** — the seventh frozen boot-page block, after +//! the machine block (`kore-rt::entry::IDENTITY_BLOCK_AT`). +//! +//! Which build the running kernel is: the Makefile's `KORE_BUILD_STAMP`, a +//! twelve-digit commit plus `-dirty` for an uncommitted tree. The kernel +//! paints it at boot, but no console pixel survives the display handoff +//! (`res/NETWORK_SERVICE.md`). Operators' screenshots were rejected again +//! and again for lacking it. `status sys` and the screenshot writer (#1414) +//! read it here. +//! +//! The stamp names the whole image, not only the kernel: the tree at that +//! commit pins the sealed service ELFs and archives by digest +//! (`res/boot.toml`), and `-dirty` says the image may not be exactly a +//! commit. +//! +//! **Kernel testimony, like the founding and machine blocks.** The kernel +//! writes it at the founding site: a userland relayer could claim any build. +//! It is a compile-time constant, so a plain copy with no seqlock. A zeroed +//! region reads `abi == 0` and exposes nothing, so a page from a founder +//! that predates the block fails closed. + +use zerocopy::byteorder::little_endian::U16; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +/// Layout version. A zeroed block reads `abi == 0` and exposes nothing. +pub const IDENTITY_ABI: u16 = 1; + +/// The longest stamp the block carries. The Makefile's is at most 18 bytes +/// (`git rev-parse --short=12` and `-dirty`); a kernel stamped with a longer +/// one refuses to build rather than testify to a truncated name. +pub const STAMP_BYTES: usize = 28; + +/// The build-identity block — kernel-stamped per spawn. 32 bytes, frozen +/// ABI: evolution is additive into a new layout version, never a reorder. +#[repr(C)] +#[derive(Clone, Copy, Debug, FromBytes, IntoBytes, KnownLayout, Immutable)] +pub struct IdentityBlock { + /// Layout version ([`IDENTITY_ABI`]); `0` = no record (fail closed). + pub abi: U16, + /// How many bytes of [`stamp`](Self::stamp) are the stamp. + pub len: U16, + /// The build stamp, printable ASCII, zero-padded. + pub stamp: [u8; STAMP_BYTES], +} + +impl IdentityBlock { + /// The block for `stamp`, or `None` if it is empty, longer than + /// [`STAMP_BYTES`] or not printable ASCII. `const`, so a kernel builds + /// its own block at compile time and refuses a stamp it cannot carry. + pub const fn new(stamp: &str) -> Option { + let bytes = stamp.as_bytes(); + if bytes.is_empty() || bytes.len() > STAMP_BYTES { + return None; + } + let mut out = [0u8; STAMP_BYTES]; + let mut i = 0; + while i < bytes.len() { + if !printable(bytes[i]) { + return None; + } + out[i] = bytes[i]; + i += 1; + } + Some(Self { + abi: U16::from_bytes(IDENTITY_ABI.to_le_bytes()), + len: U16::from_bytes((bytes.len() as u16).to_le_bytes()), + stamp: out, + }) + } + + /// The build stamp, or `None` on an unstamped or malformed block. + pub fn stamp(&self) -> Option<&str> { + if self.abi.get() != IDENTITY_ABI { + return None; + } + let stamp = self.stamp.get(..usize::from(self.len.get()))?; + if stamp.is_empty() || !stamp.iter().all(|&b| printable(b)) { + return None; + } + core::str::from_utf8(stamp).ok() + } +} + +/// Printable ASCII: what a build stamp is made of. +const fn printable(byte: u8) -> bool { + byte >= 0x20 && byte <= 0x7e +} + +const _: () = assert!( + core::mem::size_of::() == 32, + "the identity block is frozen ABI" +); + +#[cfg(test)] +mod tests { + use super::*; + use zerocopy::FromZeros; + + #[test] + fn a_stamp_reads_back_as_written() { + for stamp in ["304bfef9ca08", "304bfef9ca08-dirty", "nostamp", &"x".repeat(STAMP_BYTES)] { + let block = IdentityBlock::new(stamp).unwrap(); + assert_eq!(block.stamp(), Some(stamp)); + let (read, _) = IdentityBlock::read_from_prefix(block.as_bytes()).unwrap(); + assert_eq!(read.stamp(), Some(stamp), "through the wire bytes"); + } + } + + #[test] + fn a_stamp_it_cannot_carry_is_refused() { + let long = "x".repeat(STAMP_BYTES + 1); + for stamp in ["", long.as_str(), "tab\there", "café"] { + assert!(IdentityBlock::new(stamp).is_none(), "{stamp:?}"); + } + } + + /// A zeroed region, an unknown layout, an overlong length or a stray + /// byte each read as no record: every one fails closed. + #[test] + fn a_damaged_or_absent_block_exposes_nothing() { + assert_eq!(IdentityBlock::new_zeroed().stamp(), None); + let good = IdentityBlock::new("304bfef9ca08").unwrap(); + let mut future = good; + future.abi = U16::new(IDENTITY_ABI + 1); + let mut overlong = good; + overlong.len = U16::new(STAMP_BYTES as u16 + 1); + let mut stray = good; + stray.stamp[3] = 0x07; + let mut empty = good; + empty.len = U16::new(0); + for block in [future, overlong, stray, empty] { + assert_eq!(block.stamp(), None); + } + } +} diff --git a/native/crates/abi/src/image.rs b/native/crates/abi/src/image.rs index 0a4e11e..38cbe11 100644 --- a/native/crates/abi/src/image.rs +++ b/native/crates/abi/src/image.rs @@ -166,6 +166,20 @@ pub const PROFILE_GPU: u32 = 1 << 12; /// receives no archive export authority. Missing data refuses the launch. pub const PROFILE_MODEL_DATA: u32 = 1 << 13; +/// Launch profile: the launch composes a **window**. Which window — its +/// face coordinates and its wake topology — is the exact preset the whole +/// profile matches (`kore_app::WindowBundle`); a window profile that matches +/// no preset launches nowhere. +/// +/// Declared, never inferred, because a window's resources are not its own: +/// every other bit names a facility a headless program may need as much as a +/// window does. `lua-tool-worker` needs exactly the filesystem, heap, engine +/// plan and channel space a JavaScript application's window does, so no +/// combination of those bits could say which of the two a launch is. +/// Meaningful only beside [`PROFILE_HEAP`] and [`PROFILE_CHANNELS`], the +/// facilities every window preset is built on. +pub const PROFILE_WINDOW: u32 = 1 << 14; + pub const PROFILE_KNOWN: u32 = PROFILE_TELEMETRY | PROFILE_FILESYSTEM | PROFILE_FS_VIEW_BINDER @@ -179,15 +193,20 @@ pub const PROFILE_KNOWN: u32 = PROFILE_TELEMETRY | PROFILE_ACCELERATOR | PROFILE_ENGINE | PROFILE_GPU - | PROFILE_MODEL_DATA; + | PROFILE_MODEL_DATA + | PROFILE_WINDOW; /// Validate the profile vocabulary and its dependencies: both filesystem -/// auxiliary endpoint kinds are meaningful only beside an actual FS face. +/// auxiliary endpoint kinds are meaningful only beside an actual FS face, an +/// engine only beside a heap, and a window only beside the heap and channel +/// space every window preset carries. pub const fn profile_valid(profile: u32) -> bool { + const WINDOW_BASE: u32 = PROFILE_HEAP | PROFILE_CHANNELS; profile & !PROFILE_KNOWN == 0 && (profile & PROFILE_FS_VIEW_BINDER == 0 || profile & PROFILE_FILESYSTEM != 0) && (profile & PROFILE_FS_GRANT_TARGET == 0 || profile & PROFILE_FILESYSTEM != 0) && (profile & PROFILE_ENGINE == 0 || profile & PROFILE_HEAP != 0) + && (profile & PROFILE_WINDOW == 0 || profile & WINDOW_BASE == WINDOW_BASE) } /// Segment ceiling per image — up to eight `PT_LOAD`s plus one `PT_TLS` @@ -1258,6 +1277,15 @@ mod tests { assert!(!profile_valid(PROFILE_FS_VIEW_BINDER)); assert!(!profile_valid(PROFILE_FS_GRANT_TARGET)); assert!(!profile_valid(1 << 31)); + + // A window stands on the heap and channel space of every window + // preset; either missing, the declaration is malformed. + assert!(profile_valid( + PROFILE_HEAP | PROFILE_CHANNELS | PROFILE_WINDOW + )); + assert!(!profile_valid(PROFILE_WINDOW)); + assert!(!profile_valid(PROFILE_HEAP | PROFILE_WINDOW)); + assert!(!profile_valid(PROFILE_CHANNELS | PROFILE_WINDOW)); } #[test] diff --git a/native/crates/abi/src/keyboard.rs b/native/crates/abi/src/keyboard.rs new file mode 100644 index 0000000..33bccd1 --- /dev/null +++ b/native/crates/abi/src/keyboard.rs @@ -0,0 +1,165 @@ +//! # `keyboard` — what a keyboard [`InputReport`](crate::svc::InputReport) says +//! +//! A keyboard snapshot is HID boot-protocol **state**: a modifier bitmap, up +//! to six held usages of the Keyboard/Keypad page, and — stamped by the +//! keymap owner, never by a device — the seat's [`Locks`]. This module is +//! the vocabulary for those three fields, shared by every producer (the +//! PS/2, USB and I2C drivers) and every consumer (the terminal discipline, +//! the compositor's window clients), so no layer keeps a private copy of a +//! spec table that could drift from the others. +//! +//! What a key *means* in a given layout is policy, not ABI, and lives with +//! the keymap; this module only names the keys, the modifier bits and the +//! lock bits — facts of the HID specification and of the report layout. + +#[rustfmt::skip] +pub mod usage; + +/// The modifier bitmap — boot report byte 0, a keyboard report's `state`. +/// +/// Bit *n* is usage `0xE0 + n` (HID 1.11 Appendix B.1): the eight modifier +/// keys never occupy the six-key array, they ride this byte instead. +pub mod modifier { + use super::usage; + + pub const LEFT_CTRL: u8 = 1 << 0; + pub const LEFT_SHIFT: u8 = 1 << 1; + pub const LEFT_ALT: u8 = 1 << 2; + pub const LEFT_GUI: u8 = 1 << 3; + pub const RIGHT_CTRL: u8 = 1 << 4; + pub const RIGHT_SHIFT: u8 = 1 << 5; + pub const RIGHT_ALT: u8 = 1 << 6; + pub const RIGHT_GUI: u8 = 1 << 7; + + /// Either Control. + pub const CTRL: u8 = LEFT_CTRL | RIGHT_CTRL; + /// Either Shift. + pub const SHIFT: u8 = LEFT_SHIFT | RIGHT_SHIFT; + /// Either Alt. + pub const ALT: u8 = LEFT_ALT | RIGHT_ALT; + /// Either GUI (Super / Windows / Command). + pub const GUI: u8 = LEFT_GUI | RIGHT_GUI; + + /// The bitmap bit a modifier usage occupies; `None` for every other + /// usage. + pub const fn bit(key: u8) -> Option { + if key >= usage::LEFT_CTRL && key <= usage::RIGHT_GUI { + Some(1 << (key - usage::LEFT_CTRL)) + } else { + None + } + } +} + +/// The lock state a keyboard report carries: which of Num Lock, Caps Lock +/// and Scroll Lock are engaged. +/// +/// The bit order is the HID LED page's (usages 1–3, bit = usage − 1), +/// which is also the boot keyboard's LED output report (HID 1.11 Appendix +/// B.1) — so the byte a report carries is, unchanged, the byte a USB +/// keyboard's indicators are set with. Only the three toggling locks are +/// representable: a byte with any other bit is not a lock state. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct Locks(u8); + +impl Locks { + /// No lock engaged. + pub const NONE: Self = Self(0); + /// Num Lock (LED usage 0x01). + pub const NUM: Self = Self(1 << 0); + /// Caps Lock (LED usage 0x02). + pub const CAPS: Self = Self(1 << 1); + /// Scroll Lock (LED usage 0x03). + pub const SCROLL: Self = Self(1 << 2); + /// Every representable lock bit. + pub const MASK: u8 = Self::NUM.0 | Self::CAPS.0 | Self::SCROLL.0; + + /// A lock state from its wire byte; `None` when any bit outside + /// [`MASK`](Self::MASK) is set. + pub const fn from_bits(bits: u8) -> Option { + if bits & !Self::MASK == 0 { + Some(Self(bits)) + } else { + None + } + } + + /// The wire byte (also the boot keyboard LED output report byte). + pub const fn bits(self) -> u8 { + self.0 + } + + /// Every lock in `other` is engaged here. + pub const fn contains(self, other: Self) -> bool { + self.0 & other.0 == other.0 + } + + /// This state with the locks in `other` flipped. + #[must_use] + pub const fn toggled(self, other: Self) -> Self { + Self(self.0 ^ other.0) + } + + /// The lock a momentary lock key toggles on its press edge, `None` for + /// every other key. + pub const fn toggled_by(key: u8) -> Option { + match key { + usage::CAPS_LOCK => Some(Self::CAPS), + usage::NUM_LOCK => Some(Self::NUM), + usage::SCROLL_LOCK => Some(Self::SCROLL), + _ => None, + } + } + + /// The lock a **latching** lock key holds, `None` for every other key. + /// + /// HUT 1.5 §10 defines Locking Caps/Num/Scroll Lock (0x82–0x84) as keys + /// "implemented as a locking key; sent as a toggle button": the key is + /// reported held for as long as it is mechanically latched down. Each of + /// its edges — latching and unlatching — is therefore one toggle, where + /// a momentary key's release is none. + pub const fn latched_by(key: u8) -> Option { + match key { + usage::LOCKING_CAPS_LOCK => Some(Self::CAPS), + usage::LOCKING_NUM_LOCK => Some(Self::NUM), + usage::LOCKING_SCROLL_LOCK => Some(Self::SCROLL), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn modifier_bits_follow_the_usage_order() { + assert_eq!(modifier::bit(usage::LEFT_CTRL), Some(modifier::LEFT_CTRL)); + assert_eq!(modifier::bit(usage::LEFT_GUI), Some(modifier::LEFT_GUI)); + assert_eq!(modifier::bit(usage::RIGHT_GUI), Some(modifier::RIGHT_GUI)); + assert_eq!(modifier::bit(usage::A), None); + assert_eq!(modifier::bit(0xE8), None); + } + + #[test] + fn locks_reject_unrepresentable_bits_and_toggle_per_key() { + assert_eq!(Locks::from_bits(0b111), Some(Locks(0b111))); + assert_eq!(Locks::from_bits(1 << 3), None, "Compose is an LED, not a lock"); + let caps = Locks::NONE.toggled(Locks::toggled_by(usage::CAPS_LOCK).unwrap()); + assert!(caps.contains(Locks::CAPS) && !caps.contains(Locks::NUM)); + assert_eq!(caps.toggled(Locks::CAPS), Locks::NONE); + assert_eq!(Locks::toggled_by(usage::LOCKING_NUM_LOCK), None); + assert_eq!(Locks::latched_by(usage::LOCKING_NUM_LOCK), Some(Locks::NUM)); + assert_eq!(Locks::latched_by(usage::NUM_LOCK), None); + assert_eq!(Locks::toggled_by(usage::KP_1), None); + } + + /// The bit order is the boot keyboard's LED output report — the byte a + /// report carries goes to a USB keyboard unchanged. + #[test] + fn lock_bits_are_the_boot_led_report() { + assert_eq!(Locks::NUM.bits(), 0x01); + assert_eq!(Locks::CAPS.bits(), 0x02); + assert_eq!(Locks::SCROLL.bits(), 0x04); + } +} diff --git a/native/crates/abi/src/keyboard/usage.rs b/native/crates/abi/src/keyboard/usage.rs new file mode 100644 index 0000000..749ed57 --- /dev/null +++ b/native/crates/abi/src/keyboard/usage.rs @@ -0,0 +1,446 @@ +// @generated by tools/keyboard_tables.py — do not edit. +// Regenerate with `python3 tools/keyboard_tables.py emit`; provenance and +// the upstream cross-check (`verify`) are documented in the tool. + +//! Every defined usage of the HID Keyboard/Keypad page (0x07), named +//! after USB HID Usage Tables 1.5 §10. These are the key identities an +//! [`InputReport`](crate::svc::InputReport) keyboard snapshot carries. + +/// Reserved (no event indicated). +pub const NONE: u8 = 0x00; +/// Keyboard ErrorRollOver. +pub const ERROR_ROLL_OVER: u8 = 0x01; +/// Keyboard POSTFail. +pub const POST_FAIL: u8 = 0x02; +/// Keyboard ErrorUndefined. +pub const ERROR_UNDEFINED: u8 = 0x03; +/// Keyboard a and A. +pub const A: u8 = 0x04; +/// Keyboard b and B. +pub const B: u8 = 0x05; +/// Keyboard c and C. +pub const C: u8 = 0x06; +/// Keyboard d and D. +pub const D: u8 = 0x07; +/// Keyboard e and E. +pub const E: u8 = 0x08; +/// Keyboard f and F. +pub const F: u8 = 0x09; +/// Keyboard g and G. +pub const G: u8 = 0x0a; +/// Keyboard h and H. +pub const H: u8 = 0x0b; +/// Keyboard i and I. +pub const I: u8 = 0x0c; +/// Keyboard j and J. +pub const J: u8 = 0x0d; +/// Keyboard k and K. +pub const K: u8 = 0x0e; +/// Keyboard l and L. +pub const L: u8 = 0x0f; +/// Keyboard m and M. +pub const M: u8 = 0x10; +/// Keyboard n and N. +pub const N: u8 = 0x11; +/// Keyboard o and O. +pub const O: u8 = 0x12; +/// Keyboard p and P. +pub const P: u8 = 0x13; +/// Keyboard q and Q. +pub const Q: u8 = 0x14; +/// Keyboard r and R. +pub const R: u8 = 0x15; +/// Keyboard s and S. +pub const S: u8 = 0x16; +/// Keyboard t and T. +pub const T: u8 = 0x17; +/// Keyboard u and U. +pub const U: u8 = 0x18; +/// Keyboard v and V. +pub const V: u8 = 0x19; +/// Keyboard w and W. +pub const W: u8 = 0x1a; +/// Keyboard x and X. +pub const X: u8 = 0x1b; +/// Keyboard y and Y. +pub const Y: u8 = 0x1c; +/// Keyboard z and Z. +pub const Z: u8 = 0x1d; +/// Keyboard 1 and !. +pub const DIGIT_1: u8 = 0x1e; +/// Keyboard 2 and @. +pub const DIGIT_2: u8 = 0x1f; +/// Keyboard 3 and #. +pub const DIGIT_3: u8 = 0x20; +/// Keyboard 4 and $. +pub const DIGIT_4: u8 = 0x21; +/// Keyboard 5 and %. +pub const DIGIT_5: u8 = 0x22; +/// Keyboard 6 and ^. +pub const DIGIT_6: u8 = 0x23; +/// Keyboard 7 and &. +pub const DIGIT_7: u8 = 0x24; +/// Keyboard 8 and *. +pub const DIGIT_8: u8 = 0x25; +/// Keyboard 9 and (. +pub const DIGIT_9: u8 = 0x26; +/// Keyboard 0 and ). +pub const DIGIT_0: u8 = 0x27; +/// Keyboard Return (ENTER). +pub const ENTER: u8 = 0x28; +/// Keyboard ESCAPE. +pub const ESCAPE: u8 = 0x29; +/// Keyboard DELETE (Backspace). +pub const BACKSPACE: u8 = 0x2a; +/// Keyboard Tab. +pub const TAB: u8 = 0x2b; +/// Keyboard Spacebar. +pub const SPACE: u8 = 0x2c; +/// Keyboard - and (underscore). +pub const MINUS: u8 = 0x2d; +/// Keyboard = and +. +pub const EQUAL: u8 = 0x2e; +/// Keyboard [ and {. +pub const LEFT_BRACKET: u8 = 0x2f; +/// Keyboard ] and }. +pub const RIGHT_BRACKET: u8 = 0x30; +/// Keyboard \ and |. +pub const BACKSLASH: u8 = 0x31; +/// Keyboard Non-US # and ~. +pub const NON_US_HASH: u8 = 0x32; +/// Keyboard ; and :. +pub const SEMICOLON: u8 = 0x33; +/// Keyboard ' and ". +pub const APOSTROPHE: u8 = 0x34; +/// Keyboard Grave Accent and Tilde. +pub const GRAVE: u8 = 0x35; +/// Keyboard , and <. +pub const COMMA: u8 = 0x36; +/// Keyboard . and >. +pub const PERIOD: u8 = 0x37; +/// Keyboard / and ?. +pub const SLASH: u8 = 0x38; +/// Keyboard Caps Lock. +pub const CAPS_LOCK: u8 = 0x39; +/// Keyboard F1. +pub const F1: u8 = 0x3a; +/// Keyboard F2. +pub const F2: u8 = 0x3b; +/// Keyboard F3. +pub const F3: u8 = 0x3c; +/// Keyboard F4. +pub const F4: u8 = 0x3d; +/// Keyboard F5. +pub const F5: u8 = 0x3e; +/// Keyboard F6. +pub const F6: u8 = 0x3f; +/// Keyboard F7. +pub const F7: u8 = 0x40; +/// Keyboard F8. +pub const F8: u8 = 0x41; +/// Keyboard F9. +pub const F9: u8 = 0x42; +/// Keyboard F10. +pub const F10: u8 = 0x43; +/// Keyboard F11. +pub const F11: u8 = 0x44; +/// Keyboard F12. +pub const F12: u8 = 0x45; +/// Keyboard PrintScreen. +pub const PRINT_SCREEN: u8 = 0x46; +/// Keyboard Scroll Lock. +pub const SCROLL_LOCK: u8 = 0x47; +/// Keyboard Pause. +pub const PAUSE: u8 = 0x48; +/// Keyboard Insert. +pub const INSERT: u8 = 0x49; +/// Keyboard Home. +pub const HOME: u8 = 0x4a; +/// Keyboard PageUp. +pub const PAGE_UP: u8 = 0x4b; +/// Keyboard Delete Forward. +pub const DELETE: u8 = 0x4c; +/// Keyboard End. +pub const END: u8 = 0x4d; +/// Keyboard PageDown. +pub const PAGE_DOWN: u8 = 0x4e; +/// Keyboard RightArrow. +pub const RIGHT: u8 = 0x4f; +/// Keyboard LeftArrow. +pub const LEFT: u8 = 0x50; +/// Keyboard DownArrow. +pub const DOWN: u8 = 0x51; +/// Keyboard UpArrow. +pub const UP: u8 = 0x52; +/// Keypad Num Lock and Clear. +pub const NUM_LOCK: u8 = 0x53; +/// Keypad /. +pub const KP_DIVIDE: u8 = 0x54; +/// Keypad *. +pub const KP_MULTIPLY: u8 = 0x55; +/// Keypad -. +pub const KP_SUBTRACT: u8 = 0x56; +/// Keypad +. +pub const KP_ADD: u8 = 0x57; +/// Keypad ENTER. +pub const KP_ENTER: u8 = 0x58; +/// Keypad 1 and End. +pub const KP_1: u8 = 0x59; +/// Keypad 2 and Down Arrow. +pub const KP_2: u8 = 0x5a; +/// Keypad 3 and PageDn. +pub const KP_3: u8 = 0x5b; +/// Keypad 4 and Left Arrow. +pub const KP_4: u8 = 0x5c; +/// Keypad 5. +pub const KP_5: u8 = 0x5d; +/// Keypad 6 and Right Arrow. +pub const KP_6: u8 = 0x5e; +/// Keypad 7 and Home. +pub const KP_7: u8 = 0x5f; +/// Keypad 8 and Up Arrow. +pub const KP_8: u8 = 0x60; +/// Keypad 9 and PageUp. +pub const KP_9: u8 = 0x61; +/// Keypad 0 and Insert. +pub const KP_0: u8 = 0x62; +/// Keypad . and Delete. +pub const KP_PERIOD: u8 = 0x63; +/// Keyboard Non-US \ and |. +pub const NON_US_BACKSLASH: u8 = 0x64; +/// Keyboard Application. +pub const APPLICATION: u8 = 0x65; +/// Keyboard Power. +pub const POWER: u8 = 0x66; +/// Keypad =. +pub const KP_EQUAL: u8 = 0x67; +/// Keyboard F13. +pub const F13: u8 = 0x68; +/// Keyboard F14. +pub const F14: u8 = 0x69; +/// Keyboard F15. +pub const F15: u8 = 0x6a; +/// Keyboard F16. +pub const F16: u8 = 0x6b; +/// Keyboard F17. +pub const F17: u8 = 0x6c; +/// Keyboard F18. +pub const F18: u8 = 0x6d; +/// Keyboard F19. +pub const F19: u8 = 0x6e; +/// Keyboard F20. +pub const F20: u8 = 0x6f; +/// Keyboard F21. +pub const F21: u8 = 0x70; +/// Keyboard F22. +pub const F22: u8 = 0x71; +/// Keyboard F23. +pub const F23: u8 = 0x72; +/// Keyboard F24. +pub const F24: u8 = 0x73; +/// Keyboard Execute. +pub const EXECUTE: u8 = 0x74; +/// Keyboard Help. +pub const HELP: u8 = 0x75; +/// Keyboard Menu. +pub const MENU: u8 = 0x76; +/// Keyboard Select. +pub const SELECT: u8 = 0x77; +/// Keyboard Stop. +pub const STOP: u8 = 0x78; +/// Keyboard Again. +pub const AGAIN: u8 = 0x79; +/// Keyboard Undo. +pub const UNDO: u8 = 0x7a; +/// Keyboard Cut. +pub const CUT: u8 = 0x7b; +/// Keyboard Copy. +pub const COPY: u8 = 0x7c; +/// Keyboard Paste. +pub const PASTE: u8 = 0x7d; +/// Keyboard Find. +pub const FIND: u8 = 0x7e; +/// Keyboard Mute. +pub const MUTE: u8 = 0x7f; +/// Keyboard Volume Up. +pub const VOLUME_UP: u8 = 0x80; +/// Keyboard Volume Down. +pub const VOLUME_DOWN: u8 = 0x81; +/// Keyboard Locking Caps Lock. +pub const LOCKING_CAPS_LOCK: u8 = 0x82; +/// Keyboard Locking Num Lock. +pub const LOCKING_NUM_LOCK: u8 = 0x83; +/// Keyboard Locking Scroll Lock. +pub const LOCKING_SCROLL_LOCK: u8 = 0x84; +/// Keypad Comma. +pub const KP_COMMA: u8 = 0x85; +/// Keypad Equal Sign. +pub const KP_EQUAL_SIGN: u8 = 0x86; +/// Keyboard International1 (Ro). +pub const INTERNATIONAL_1: u8 = 0x87; +/// Keyboard International2 (Katakana/Hiragana). +pub const INTERNATIONAL_2: u8 = 0x88; +/// Keyboard International3 (Yen). +pub const INTERNATIONAL_3: u8 = 0x89; +/// Keyboard International4 (Henkan). +pub const INTERNATIONAL_4: u8 = 0x8a; +/// Keyboard International5 (Muhenkan). +pub const INTERNATIONAL_5: u8 = 0x8b; +/// Keyboard International6 (Keypad JP comma). +pub const INTERNATIONAL_6: u8 = 0x8c; +/// Keyboard International7. +pub const INTERNATIONAL_7: u8 = 0x8d; +/// Keyboard International8. +pub const INTERNATIONAL_8: u8 = 0x8e; +/// Keyboard International9. +pub const INTERNATIONAL_9: u8 = 0x8f; +/// Keyboard LANG1 (Hangul/English). +pub const LANG_1: u8 = 0x90; +/// Keyboard LANG2 (Hanja). +pub const LANG_2: u8 = 0x91; +/// Keyboard LANG3 (Katakana). +pub const LANG_3: u8 = 0x92; +/// Keyboard LANG4 (Hiragana). +pub const LANG_4: u8 = 0x93; +/// Keyboard LANG5 (Zenkaku/Hankaku). +pub const LANG_5: u8 = 0x94; +/// Keyboard LANG6. +pub const LANG_6: u8 = 0x95; +/// Keyboard LANG7. +pub const LANG_7: u8 = 0x96; +/// Keyboard LANG8. +pub const LANG_8: u8 = 0x97; +/// Keyboard LANG9. +pub const LANG_9: u8 = 0x98; +/// Keyboard Alternate Erase. +pub const ALTERNATE_ERASE: u8 = 0x99; +/// Keyboard SysReq/Attention. +pub const SYS_REQ: u8 = 0x9a; +/// Keyboard Cancel. +pub const CANCEL: u8 = 0x9b; +/// Keyboard Clear. +pub const CLEAR: u8 = 0x9c; +/// Keyboard Prior. +pub const PRIOR: u8 = 0x9d; +/// Keyboard Return. +pub const RETURN: u8 = 0x9e; +/// Keyboard Separator. +pub const SEPARATOR: u8 = 0x9f; +/// Keyboard Out. +pub const OUT: u8 = 0xa0; +/// Keyboard Oper. +pub const OPER: u8 = 0xa1; +/// Keyboard Clear/Again. +pub const CLEAR_AGAIN: u8 = 0xa2; +/// Keyboard CrSel/Props. +pub const CR_SEL: u8 = 0xa3; +/// Keyboard ExSel. +pub const EX_SEL: u8 = 0xa4; +/// Keypad 00. +pub const KP_00: u8 = 0xb0; +/// Keypad 000. +pub const KP_000: u8 = 0xb1; +/// Thousands Separator. +pub const THOUSANDS_SEPARATOR: u8 = 0xb2; +/// Decimal Separator. +pub const DECIMAL_SEPARATOR: u8 = 0xb3; +/// Currency Unit. +pub const CURRENCY_UNIT: u8 = 0xb4; +/// Currency Sub-unit. +pub const CURRENCY_SUB_UNIT: u8 = 0xb5; +/// Keypad (. +pub const KP_LEFT_PAREN: u8 = 0xb6; +/// Keypad ). +pub const KP_RIGHT_PAREN: u8 = 0xb7; +/// Keypad {. +pub const KP_LEFT_BRACE: u8 = 0xb8; +/// Keypad }. +pub const KP_RIGHT_BRACE: u8 = 0xb9; +/// Keypad Tab. +pub const KP_TAB: u8 = 0xba; +/// Keypad Backspace. +pub const KP_BACKSPACE: u8 = 0xbb; +/// Keypad A. +pub const KP_A: u8 = 0xbc; +/// Keypad B. +pub const KP_B: u8 = 0xbd; +/// Keypad C. +pub const KP_C: u8 = 0xbe; +/// Keypad D. +pub const KP_D: u8 = 0xbf; +/// Keypad E. +pub const KP_E: u8 = 0xc0; +/// Keypad F. +pub const KP_F: u8 = 0xc1; +/// Keypad XOR. +pub const KP_XOR: u8 = 0xc2; +/// Keypad ^. +pub const KP_CARET: u8 = 0xc3; +/// Keypad %. +pub const KP_PERCENT: u8 = 0xc4; +/// Keypad <. +pub const KP_LESS: u8 = 0xc5; +/// Keypad >. +pub const KP_GREATER: u8 = 0xc6; +/// Keypad &. +pub const KP_AMPERSAND: u8 = 0xc7; +/// Keypad &&. +pub const KP_DOUBLE_AMPERSAND: u8 = 0xc8; +/// Keypad |. +pub const KP_BAR: u8 = 0xc9; +/// Keypad ||. +pub const KP_DOUBLE_BAR: u8 = 0xca; +/// Keypad :. +pub const KP_COLON: u8 = 0xcb; +/// Keypad #. +pub const KP_HASH: u8 = 0xcc; +/// Keypad Space. +pub const KP_SPACE: u8 = 0xcd; +/// Keypad @. +pub const KP_AT: u8 = 0xce; +/// Keypad !. +pub const KP_EXCLAMATION: u8 = 0xcf; +/// Keypad Memory Store. +pub const KP_MEMORY_STORE: u8 = 0xd0; +/// Keypad Memory Recall. +pub const KP_MEMORY_RECALL: u8 = 0xd1; +/// Keypad Memory Clear. +pub const KP_MEMORY_CLEAR: u8 = 0xd2; +/// Keypad Memory Add. +pub const KP_MEMORY_ADD: u8 = 0xd3; +/// Keypad Memory Subtract. +pub const KP_MEMORY_SUBTRACT: u8 = 0xd4; +/// Keypad Memory Multiply. +pub const KP_MEMORY_MULTIPLY: u8 = 0xd5; +/// Keypad Memory Divide. +pub const KP_MEMORY_DIVIDE: u8 = 0xd6; +/// Keypad +/-. +pub const KP_PLUS_MINUS: u8 = 0xd7; +/// Keypad Clear. +pub const KP_CLEAR: u8 = 0xd8; +/// Keypad Clear Entry. +pub const KP_CLEAR_ENTRY: u8 = 0xd9; +/// Keypad Binary. +pub const KP_BINARY: u8 = 0xda; +/// Keypad Octal. +pub const KP_OCTAL: u8 = 0xdb; +/// Keypad Decimal. +pub const KP_DECIMAL: u8 = 0xdc; +/// Keypad Hexadecimal. +pub const KP_HEXADECIMAL: u8 = 0xdd; +/// Keyboard LeftControl. +pub const LEFT_CTRL: u8 = 0xe0; +/// Keyboard LeftShift. +pub const LEFT_SHIFT: u8 = 0xe1; +/// Keyboard LeftAlt. +pub const LEFT_ALT: u8 = 0xe2; +/// Keyboard Left GUI. +pub const LEFT_GUI: u8 = 0xe3; +/// Keyboard RightControl. +pub const RIGHT_CTRL: u8 = 0xe4; +/// Keyboard RightShift. +pub const RIGHT_SHIFT: u8 = 0xe5; +/// Keyboard RightAlt. +pub const RIGHT_ALT: u8 = 0xe6; +/// Keyboard Right GUI. +pub const RIGHT_GUI: u8 = 0xe7; diff --git a/native/crates/abi/src/lib.rs b/native/crates/abi/src/lib.rs index 75d503c..2f64a33 100644 --- a/native/crates/abi/src/lib.rs +++ b/native/crates/abi/src/lib.rs @@ -57,9 +57,11 @@ pub mod device; pub mod execution; pub mod founding; pub mod guest; +pub mod identity; pub mod image; pub mod input_status; pub mod iommu_status; +pub mod keyboard; pub mod machine; pub mod mem; pub mod monitor; diff --git a/native/crates/abi/src/machine.rs b/native/crates/abi/src/machine.rs index b65ad15..f6456d1 100644 --- a/native/crates/abi/src/machine.rs +++ b/native/crates/abi/src/machine.rs @@ -212,10 +212,98 @@ pub const fn pack_spmd_fit_width(width: u32) -> u16 { (if width > max { max } else { width } as u16) << MACHINE_SPMD_FIT_SHIFT } +/// [`MachineBlock::flags`] bits 2..5: the **SPMD gang placement** the +/// kernel granted the founding this block was stamped for — the G6c batch +/// decision (`res/SPMD.md` §G6c), reported to the worker it placed so a +/// gang can learn which class it landed on instead of assuming the best +/// one. Bit 2 marks a stamped placement (a founding outside a gang leaves +/// it clear, and every earlier founder wrote zero, which reads as "no +/// gang"), bits 3..4 the core class in the kernel's rank order +/// ([`GangClass`]), bit 5 whether the pin is time-shared with an incumbent +/// reservation rather than exclusively reserved. Additive on the frozen +/// block like the fit width. +const MACHINE_GANG_PRESENT: u16 = 1 << 2; +const MACHINE_GANG_CLASS_SHIFT: u16 = 3; +const MACHINE_GANG_CLASS_MASK: u16 = 0b11 << MACHINE_GANG_CLASS_SHIFT; +const MACHINE_GANG_SHARED: u16 = 1 << 5; + +/// The core class a gang was placed on, in the kernel's class-rank order +/// (`Performance` outranks `Standard` outranks `Efficient` outranks +/// `LowPowerEfficient` — the homogeneity ruling, `res/SPMD.md`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum GangClass { + Performance = 0, + Standard = 1, + Efficient = 2, + LowPowerEfficient = 3, +} + +impl GangClass { + /// The witness letter the kernel's `spmd-reserve` / `spmd-pin` lines use. + pub const fn letter(self) -> &'static str { + match self { + Self::Performance => "P", + Self::Standard => "std", + Self::Efficient => "E", + Self::LowPowerEfficient => "LP-E", + } + } + + const fn from_code(code: u16) -> Self { + match code & 0b11 { + 0 => Self::Performance, + 1 => Self::Standard, + 2 => Self::Efficient, + _ => Self::LowPowerEfficient, + } + } +} + +/// What the kernel's batch reservation granted this worker's gang. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct GangPlacement { + /// The homogeneous class every member of the gang runs on. + pub class: GangClass, + /// `true` when the gang's cores are time-shared with an incumbent + /// reservation (the best fitting class was held); `false` when they + /// are exclusively reserved for this gang. + pub shared: bool, +} + +/// Encode a granted gang placement for the machine flags. +pub const fn pack_gang_placement(class: GangClass, shared: bool) -> u16 { + MACHINE_GANG_PRESENT + | ((class as u16) << MACHINE_GANG_CLASS_SHIFT) + | if shared { MACHINE_GANG_SHARED } else { 0 } +} + +impl MachineBlock { + /// The gang placement the kernel granted this founding, or `None` for + /// a founding outside a gang (and for every block from a founder that + /// predates the field — the fail-closed direction: a reader then sizes + /// as it always did). + pub fn gang_placement(&self) -> Option { + let flags = self.flags.get(); + (self.validate() && flags & MACHINE_GANG_PRESENT != 0).then(|| GangPlacement { + class: GangClass::from_code( + (flags & MACHINE_GANG_CLASS_MASK) >> MACHINE_GANG_CLASS_SHIFT, + ), + shared: flags & MACHINE_GANG_SHARED != 0, + }) + } +} + const _: () = assert!( core::mem::size_of::() == 20, "the machine block is frozen ABI" ); +const _: () = assert!( + (MACHINE_GANG_PRESENT | MACHINE_GANG_CLASS_MASK | MACHINE_GANG_SHARED) + & (MACHINE_FLAG_CPU_ENTROPY | MACHINE_FLAG_HW_FEEDBACK | MACHINE_SPMD_FIT_MASK) + == 0, + "the gang placement bits must not overlap the other flag claims" +); #[cfg(test)] mod tests { @@ -242,6 +330,51 @@ mod tests { assert_eq!(absent.spmd_fit_width(), None); } + #[test] + fn a_gang_placement_rides_the_flags_beside_the_fit_and_reads_absent_from_older_founders() { + let older = MachineBlock::new( + 16, + 4, + 0, + MACHINE_FLAG_HW_FEEDBACK | pack_spmd_fit_width(8), + 0, + ); + assert_eq!( + older.gang_placement(), + None, + "a pre-placement founder: no gang" + ); + assert_eq!(older.spmd_fit_width(), Some(8)); + for (class, shared) in [ + (GangClass::Performance, false), + (GangClass::Performance, true), + (GangClass::Standard, false), + (GangClass::Efficient, true), + (GangClass::LowPowerEfficient, false), + ] { + let flags = MACHINE_FLAG_HW_FEEDBACK + | pack_spmd_fit_width(8) + | pack_gang_placement(class, shared); + let b = MachineBlock::new(16, 4, 0, flags, 0); + assert_eq!(b.gang_placement(), Some(GangPlacement { class, shared })); + assert_eq!( + b.spmd_fit_width(), + Some(8), + "the fit survives beside the placement" + ); + assert!(b.hardware_feedback()); + } + let zeroed = + MachineBlock::read_from_bytes(&[0u8; core::mem::size_of::()]).unwrap(); + assert_eq!( + zeroed.gang_placement(), + None, + "an unstamped block fails closed" + ); + assert_eq!(GangClass::Efficient.letter(), "E"); + assert_eq!(GangClass::LowPowerEfficient.letter(), "LP-E"); + } + #[test] fn an_empty_block_validates_but_exposes_zero_facts() { let b = MachineBlock::EMPTY; diff --git a/native/crates/abi/src/monitor.rs b/native/crates/abi/src/monitor.rs index c28e721..961a3d8 100644 --- a/native/crates/abi/src/monitor.rs +++ b/native/crates/abi/src/monitor.rs @@ -421,7 +421,9 @@ pub struct MemorySnapshot { flags: U32, extents: U32, supplies: U32, - reserved: U32, + /// Former reserved word: whole 2 MiB supplies still allowed on the + /// requesting face, only when SUPPLY_BUDGET is present. + huge_supply_units: U32, granted: U64, released: U64, reused: U64, @@ -431,11 +433,13 @@ pub struct MemorySnapshot { } impl MemorySnapshot { + /// The provider sampled the requesting face's huge-supply meter. + const SUPPLY_BUDGET: u32 = 1 << 1; pub const EMPTY: Self = Self { flags: U32::new(0), extents: U32::new(0), supplies: U32::new(0), - reserved: U32::new(0), + huge_supply_units: U32::new(0), granted: U64::new(0), released: U64::new(0), reused: U64::new(0), @@ -459,7 +463,7 @@ impl MemorySnapshot { flags: U32::new(AVAILABLE), extents: U32::new(extents), supplies: U32::new(supplies), - reserved: U32::new(0), + huge_supply_units: U32::new(0), granted: U64::new(granted), released: U64::new(released), reused: U64::new(reused), @@ -470,10 +474,29 @@ impl MemorySnapshot { } pub fn available(self) -> bool { - self.flags.get() == AVAILABLE + self.flags.get() & AVAILABLE != 0 } pub fn validate(self) -> bool { - self.reserved.get() == 0 && (self.available() || self == Self::EMPTY) + self.flags.get() & !(AVAILABLE | Self::SUPPLY_BUDGET) == 0 + && (self.available() || self == Self::EMPTY) + && (self.flags.get() & Self::SUPPLY_BUDGET != 0 || self.huge_supply_units.get() == 0) + } + /// Attach this arrival's allowance in whole minimum huge-supply units. + /// Rounding down never promises a request that the byte quota refuses. + /// The former reserved word keeps the wire and launch-page sizes intact. + /// None denotes inconsistent accounting or an unrepresentable allowance. + pub fn with_supply_budget(mut self, limit: u64, used: u64) -> Option { + if !self.validate() || !self.available() { return None; } + let units = limit.checked_sub(used)? >> crate::session::LEASE_ALIGN_2M; + self.huge_supply_units = U32::new(u32::try_from(units).ok()?); + self.flags = U32::new(self.flags.get() | Self::SUPPLY_BUDGET); + Some(self) + } + /// Remaining 2 MiB / 1 GiB aligned supply allowance on this face. + /// None means unavailable or malformed, never an unlimited allowance. + pub fn huge_supply_remaining(self) -> Option { + (self.validate() && self.flags.get() & Self::SUPPLY_BUDGET != 0) + .then(|| u64::from(self.huge_supply_units.get()) << crate::session::LEASE_ALIGN_2M) } pub fn extents(self) -> u32 { self.extents.get() @@ -1909,6 +1932,23 @@ mod tests { assert!(FilesystemSnapshot::EMPTY.validate()); } + #[test] + fn supply_budget_distinguishes_absent_exhausted_and_invalid() { + let sample = MemorySnapshot::new(0, 0, 0, 0, 0, 0, 0, 0); + assert_eq!(sample.huge_supply_remaining(), None); + let budget = sample.with_supply_budget(26 << 30, 24 << 30).unwrap(); + assert!(budget.validate()); + assert_eq!(budget.huge_supply_remaining(), Some(2 << 30)); + assert_eq!(sample.with_supply_budget(0, 0).unwrap().huge_supply_remaining(), Some(0)); + assert!(sample.with_supply_budget(1, 2).is_none()); + assert!(sample.with_supply_budget(u64::MAX, 0).is_none()); + assert!(MemorySnapshot::EMPTY.with_supply_budget(1, 0).is_none()); + assert_eq!(sample.with_supply_budget((2 << 20) - 1, 0).unwrap().huge_supply_remaining(), Some(0)); + let mut untagged = budget; + untagged.flags = U32::new(AVAILABLE); + assert!(!untagged.validate()); + } + #[test] fn provider_snapshots_validate_only_defined_postures() { let system = SystemSnapshot::new( @@ -1948,7 +1988,7 @@ mod tests { ); let mut corrupt_memory = memory; - corrupt_memory.reserved = U32::new(1); + corrupt_memory.huge_supply_units = U32::new(1); assert!( !TelemetrySnapshot::new(system, corrupt_memory, exec, ManifoldSnapshot::EMPTY) .validate() diff --git a/native/crates/abi/src/process.rs b/native/crates/abi/src/process.rs index b4694ba..48b0841 100644 --- a/native/crates/abi/src/process.rs +++ b/native/crates/abi/src/process.rs @@ -23,7 +23,7 @@ //! bounded grace before it revokes the task; a child that never reads its //! input simply meets the revoke (`res/LIFECYCLE_HARDENING.md` §3). -use zerocopy::byteorder::little_endian::{U16, U32}; +use zerocopy::byteorder::little_endian::{U16, U32, U64}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned}; /// Boot-role used by the generic process-I/O view. @@ -130,12 +130,60 @@ pub const PROCESS_STDIO_TO_PARENT_SEQUENCE_OFFSET: usize = /// [`PROCESS_STDIO_TO_PARENT_SEQUENCE_OFFSET`]. The child sends on the /// output-wake channel only after atomically observing this as set. pub const PROCESS_STDIO_PARENT_WAITING_OFFSET: usize = PROCESS_STDIO_TO_PARENT_SEQUENCE_OFFSET + 4; +/// The stream's **profile**: one little-endian `u32` of launcher-declared +/// facts about the stream's far end, on a line of its own. The launcher writes +/// it while it formats the page, before the child's view is conveyed, and +/// neither side writes it again, so a child reads it once at attach. Bits +/// this ABI does not define are written zero and read as absent. +pub const PROCESS_STDIO_PROFILE_OFFSET: usize = PROCESS_STDIO_TO_PARENT_SEQUENCE_OFFSET + 64; +/// Profile bit: the far end is a **terminal**. The launcher answers each +/// `InputReady` with a line its terminal edited and echoed, and renders +/// output through a terminal that decodes ANSI escape sequences (kore's +/// shell profile, `crates/terminal`). A launcher that captures or replays +/// the stream — a tool worker, a boot witness — leaves it clear, so a +/// program that asks whether its standard streams are a terminal (`isatty`) +/// is told the truth either way. +pub const PROCESS_STDIO_PROFILE_TERMINAL: u32 = 1 << 0; +/// Every profile bit this ABI defines. +pub const PROCESS_STDIO_PROFILE_DEFINED: u32 = PROCESS_STDIO_PROFILE_TERMINAL; /// End of the live process-stream layout. Each directional coordination tuple /// occupies a complete 64-byte line, so the two hot publication sequences do /// not share a line. The opposite-writer waiting flag intentionally joins its -/// sequence and is touched only across park transitions. The rest of the page -/// is reserved and zeroed when the launcher formats it. -pub const PROCESS_STDIO_LAYOUT_BYTES: usize = PROCESS_STDIO_TO_PARENT_SEQUENCE_OFFSET + 64; +/// sequence and is touched only across park transitions; the profile follows +/// on its own line. +/// Binary output uses the same shared page and wake protocol as text. Two +/// 512-byte records plus the SPSC header fit in the previously reserved tail. +pub const PROCESS_BULK_OFFSET: usize = PROCESS_STDIO_PROFILE_OFFSET + 64; +pub const PROCESS_BULK_CAPACITY: u32 = 2; +pub const PROCESS_BULK_PAYLOAD: usize = 512 - 16; +pub const PROCESS_BULK_RING_BYTES: usize = 192 + PROCESS_BULK_CAPACITY as usize * 512; +pub const PROCESS_STDIO_LAYOUT_BYTES: usize = PROCESS_BULK_OFFSET + PROCESS_BULK_RING_BYTES; + +/// Bounded binary fragment, interpreted only by a consumer that requested it. +/// Offsets and tags correlate fragments; no pointers or authority cross here. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] +pub struct ProcessBulkChunk { + pub tag: U64, + pub offset: U32, + len: U16, + reserved: U16, + payload: [u8; PROCESS_BULK_PAYLOAD], +} +impl ProcessBulkChunk { + pub fn new(tag: u64, offset: u32, bytes: &[u8]) -> Result { + if bytes.is_empty() || bytes.len() > PROCESS_BULK_PAYLOAD { return Err(ProcessStreamError::Invalid); } + let mut record = Self { tag: U64::new(tag), offset: U32::new(offset), len: U16::new(bytes.len() as u16), + reserved: U16::new(0), payload: [0; PROCESS_BULK_PAYLOAD] }; + record.payload[..bytes.len()].copy_from_slice(bytes); + Ok(record) + } + pub fn bytes(&self) -> Result<&[u8], ProcessStreamError> { + let len = usize::from(self.len.get()); + if self.reserved.get() != 0 || len == 0 || len > self.payload.len() { return Err(ProcessStreamError::Invalid); } + Ok(&self.payload[..len]) + } +} /// Maximum bytes carried by one input record. pub const PROCESS_STDIN_CHUNK_BYTES: usize = 24; @@ -424,6 +472,9 @@ const _: () = { assert!(core::mem::align_of::() == 1); assert!(core::mem::size_of::() == 32); assert!(core::mem::align_of::() == 1); + assert!(core::mem::size_of::() == 512); + assert!(core::mem::align_of::() == 1); + assert!(PROCESS_BULK_OFFSET % 64 == 0); assert!(PROCESS_STDIN_RING_OFFSET % 64 == 0); assert!(PROCESS_OUTPUT_RING_OFFSET % 64 == 0); assert!(PROCESS_STDIO_TO_CHILD_SEQUENCE_OFFSET % 64 == 0); @@ -456,21 +507,17 @@ pub const PROCESS_ARG_BYTES: usize = 272; /// the payload at opposite sides of the task-lifecycle handoff; they are never /// simultaneously live. /// -/// So this bound is policy, not geometry: the shared payload is -/// [`PROCESS_OUTPUT_BYTES`] wide, and a context may use any of it without -/// growing the page or touching the wire. What sets the number is the -/// largest context anyone builds, which is the bootstrap command catalogue -/// `help` reads — [`MAX_COMMAND_SNAPSHOT_ENTRIES`](crate::command::MAX_COMMAND_SNAPSHOT_ENTRIES) -/// rows of name, usage and summary plus four framing bytes each. The -/// catalogue has 40 rows, including the delegated Lua worker. Its complete -/// snapshot no longer fits 2560 bytes. 3072 carries the current catalogue -/// and remains within the existing 3208-byte alternating payload; no page -/// layout or command-output capacity changes. -pub const PROCESS_CONTEXT_BYTES: usize = 3072; -const _: () = assert!( - PROCESS_CONTEXT_BYTES <= PROCESS_OUTPUT_BYTES, - "context borrows the output payload and cannot outgrow it" -); +/// So the bound is that payload, [`PROCESS_OUTPUT_BYTES`] wide: a context may +/// use all of it without growing the page or touching the wire. The largest +/// context anyone builds is the bootstrap command catalogue `help` reads — +/// [`MAX_COMMAND_SNAPSHOT_ENTRIES`](crate::command::MAX_COMMAND_SNAPSHOT_ENTRIES) +/// rows of name, usage and summary plus four framing bytes each. Its +/// complete snapshot outgrew 2560 bytes at 40 rows, and with the DagFS +/// verbs' usage lines naming their options it needs 3087 at 45, past the +/// 3072 that followed. A bound below the payload only leaves a round number +/// to outgrow. Widening it moves no offset and gives no byte a new meaning, +/// so the process ABI keeps its version. +pub const PROCESS_CONTEXT_BYTES: usize = PROCESS_OUTPUT_BYTES; /// Header bytes before the argument and alternating payload arrays: 8 fixed /// (`abi`/`flags`/`arg_len`/`output_len`) + the embedded [`TelemetrySnapshot`] /// (240 — the IOMMU and timer censuses plus the manifold's own state) + the launcher's @@ -883,7 +930,11 @@ mod tests { assert_eq!(PROCESS_STDIO_CHILD_WAITING_OFFSET, 2436); assert_eq!(PROCESS_STDIO_TO_PARENT_SEQUENCE_OFFSET, 2496); assert_eq!(PROCESS_STDIO_PARENT_WAITING_OFFSET, 2500); - assert_eq!(PROCESS_STDIO_LAYOUT_BYTES, 2560); + assert_eq!(PROCESS_STDIO_PROFILE_OFFSET, 2560); + assert_eq!(PROCESS_STDIO_PROFILE_TERMINAL, 1); + assert_eq!(PROCESS_STDIO_PROFILE_DEFINED, PROCESS_STDIO_PROFILE_TERMINAL); + assert_eq!(PROCESS_STDIO_LAYOUT_BYTES, 3840); + assert!(PROCESS_STDIO_LAYOUT_BYTES <= PROCESS_STDIO_BYTES); assert_eq!(core::mem::size_of::(), 32); assert_eq!(core::mem::size_of::(), 32); diff --git a/native/crates/abi/src/session.rs b/native/crates/abi/src/session.rs index 70d3933..ac3aa27 100644 --- a/native/crates/abi/src/session.rs +++ b/native/crates/abi/src/session.rs @@ -233,7 +233,8 @@ pub const DERIVE_KIND_I2C_HID_RECORD: u8 = 0xf3; /// `Derive × PhysRegion` lifecycle observation for MM supply accounting. /// The operation writes one bind-inert [`SupplyReleaseReport`] for a returned -/// tracked supply, or refuses when no completed return is pending. +/// tracked supply, or refuses with [`Status::WouldBlock`] when no completed +/// return is pending on the named lane — a poll's ordinary miss, not a fault. /// /// `0xf2` is the next unused high flavor below /// [`DERIVE_KIND_I2C_HID_RECORD`]. @@ -858,6 +859,131 @@ impl ProtectReq { pub const WIRE: usize = 24; } +/// `Derive × AddrSpace` (the caller's **own** space, verified by +/// page-table root exactly as [`DERIVE_KIND_SET_TP`] is) that **sets the +/// calling thread's fault upcall** to [`FaultUpcallReq`] — the delivery a +/// runtime with its own trap handling (a wasm engine's guard regions, +/// `res/WASM_PLATFORM.md` §12) needs, the way a POSIX runtime needs +/// `sigaction` and `sigaltstack`. An *effect*, not a derivation: it mints +/// no child and the result slot stays vacant. `MODIFY` on the space +/// authorizes it, the write projection `SET_TP` takes. +/// +/// **Delivery.** A synchronous user fault the kernel would otherwise end +/// the process for — after the pager has refused it — is delivered to the +/// thread that raised it, unless that thread is already on its upcall +/// stack. The kernel keeps the fault's record and returns to user mode at +/// `entry`, with the stack pointer at `stack_hi` and the C ABI's first +/// five argument registers holding [`UPCALL_ARGS`]: the faulting `pc`, the +/// frame pointer at the fault, whether the fault names an address, the +/// address, and its [`FaultKind`](crate::crash::FaultKind). Every other +/// register holds what it held at the fault. An extended-state refusal is +/// the kernel's own policy and is never delivered. +/// +/// **In an upcall exactly while on its stack.** The kernel keeps no upcall +/// state: the handler leaves `[stack_lo, stack_hi]` when it resumes the +/// faulting code's recovery path, and stays on it when it gives up. The +/// top is on it: the delivery leaves the stack pointer there, and a +/// trampoline whose handler returned reaches its decline with it there +/// again. So a fault raised with the stack pointer in that range ends the +/// process: +/// +/// - at `decline`, the handler's trap instruction for "not mine", the +/// record reported is the fault the upcall was given, flagged +/// [`UPCALL_DECLINED`](crate::crash::CaptureFlags::UPCALL_DECLINED); +/// - anywhere else, the nested fault itself, flagged +/// [`IN_UPCALL`](crate::crash::CaptureFlags::IN_UPCALL) with the +/// upcall's fault's `pc` beside it. +/// +/// A zero `entry` clears the thread's upcall. The kernel reads and writes +/// no user memory to deliver one. +/// +/// `0xdb` is the next unused selector below [`DERIVE_KIND_WAKE_WORD`]; +/// distinct from every other `AddrSpace`-subject flavor (`SET_TP` = +/// `0xfd`, `PROTECT` = `0xfa`, `ARENA_EXTEND` = `0xf7`). +pub const DERIVE_KIND_FAULT_UPCALL: u8 = 0xdb; + +/// How many argument registers an upcall's entry receives +/// ([`DERIVE_KIND_FAULT_UPCALL`]): `pc`, frame pointer, has-address, +/// address, kind — `rdi`..`r8` on x86-64, `x0`..`x4` on AArch64, `a0`..`a4` +/// on RISC-V. +pub const UPCALL_ARGS: usize = 5; + +/// Argument payload for the [`DERIVE_KIND_FAULT_UPCALL`] flavor. LE, +/// unaligned-safe, 32 bytes. +/// +/// All four addresses are user-half — below the kernel half's first +/// address, so canonical in the active paging model — `stack_hi` too, +/// since a delivery returns to user mode with the stack pointer there. +/// `stack_lo < stack_hi`, both 16-byte aligned; the range includes any +/// guard pages below the usable stack, so a handler that overflows its +/// stack is still inside it. `entry == 0` clears the upcall, and then the +/// other three fields must be zero too. +#[repr(C)] +#[derive(Clone, Copy, Debug, TryFromBytes, KnownLayout, Immutable, Unaligned)] +pub struct FaultUpcallReq { + /// Where a delivered fault resumes the thread. + pub entry: U64, + /// The handler's "not mine" trap instruction. + pub decline: U64, + /// The lowest byte of the upcall stack, guard pages included. + pub stack_lo: U64, + /// One past the highest byte: the stack pointer a delivery sets. + pub stack_hi: U64, +} + +impl FaultUpcallReq { + /// Wire size. + pub const WIRE: usize = 32; +} + +/// `Derive × MemLease` **reservation release**: remove from the caller's own +/// `RESERVE` window (`geom::RESERVE`) every page this lessor committed there +/// — the views its carves were mapped as with +/// [`ViewMapWindow::Reserve`] — inside the page range a +/// [`ReserveReleaseReq`] names. The teardown half of a runtime's own +/// reservations: a wasm memory dropped, its committed pages unmapped in one +/// session. +/// +/// `MODIFY` on the subject: the release mutates the lease's record of where +/// its pages are mapped, as the grant return does. Only the subject's own +/// pages (and those of lessors carved beneath it) in the caller's own space +/// are touched — another supply's, a region's, and every other space's stay +/// where they are — and only whole commits: a commit that straddles the +/// range's edge refuses the release ([`Status::Malformed`]) before any page +/// is touched. Only bind-inert pages, too: a page in the range that a view +/// capability names is that capability's to unmap, and refuses the release +/// ([`Status::Denied`]) as whole. A supply that is closing refuses +/// ([`Status::Revoked`]): its teardown is removing the pages. The pages +/// stay spent, as every view's are under the supply's monotone cursors. +/// +/// A post-commit effect, and the session's only invocation: the unmap cannot +/// be undone, so it runs once nothing else can refuse. The result slot is +/// untouched. +/// +/// `0xda` is the next unused selector below [`DERIVE_KIND_FAULT_UPCALL`]; +/// distinct from every other flavor a `MemLease` subject takes +/// ([`DERIVE_KIND_WAKE_WORD`], [`DERIVE_KIND_OWNED_MAP`], +/// [`DERIVE_KIND_IOVA_ANON`], [`DERIVE_KIND_EXEC_SEAL`]). +pub const DERIVE_KIND_RESERVE_RELEASE: u8 = 0xda; + +/// Argument payload for [`DERIVE_KIND_RESERVE_RELEASE`]: the page range of +/// the caller's `RESERVE` window, `[page, page + pages)` in +/// [`RESERVE_PAGE_BYTES`] units. LE, unaligned-safe, 8 bytes. `pages` is +/// non-zero and the range lies inside the window. +#[repr(C)] +#[derive(Clone, Copy, Debug, TryFromBytes, KnownLayout, Immutable, Unaligned)] +pub struct ReserveReleaseReq { + /// The first page of the range, counted from the window's base. + pub page: U32, + /// The range's length in pages. + pub pages: U32, +} + +impl ReserveReleaseReq { + /// Wire size. + pub const WIRE: usize = 8; +} + /// Retired selector for the former derive-an-executable-alias operation. /// /// Frozen ABI values are never reused. New code uses @@ -981,6 +1107,16 @@ pub enum Status { FabricRouteCapacity = 16, /// The registration capability could not authorize a route child. FabricRouteCapability = 17, + /// A park carrying `WAIT_WORD` resolved by its **word arm** (EVENT.md + /// §6): the session committed, the caller parked on a memory word, + /// and either a `wake_word` on the same memory or an idle CPU's + /// hardware monitor observing the store woke it — not a delivery, + /// reap, deadline, or IRQ. The standing post is left standing + /// (exactly [`Deadline`](Self::Deadline)'s rule). The return is + /// never a promise about the word's value: the caller re-reads the + /// word, the contract `futex` teaches. Additive on the frozen + /// vocabulary. + Word = 18, } /// Hard cap on a session's invocation count — heapless, fixed-capacity @@ -1098,6 +1234,31 @@ bitflags! { /// delivery and sleep payloads in the park block. The kernel binds /// the arena record and resolves the endpoint from the held capability. const WAIT_IRQ_SELECTED = 1 << 5; + /// Park the caller after commit until a **store to a watched + /// memory word** is observed (EVENT.md §6 — the kernel + /// address-wait): the [`WordWaitReq`] names a held memory lease or + /// view (the *subject*, by arena slot — authority is the mapping, + /// never an address), a byte offset within it, and the `u32` value + /// the caller last read. The kernel keys its waiter on the word's + /// **physical** identity, so every mapping of the same memory + /// names the same wait; a `wake_word` on any of them (or, on an + /// idle CPU, the hardware monitor seeing the store) resolves + /// [`Status::Word`]. A word that already differs from `expected` + /// at park time resolves `Word` without parking — the `futex` + /// contract. + /// + /// Always bounded: requires [`WAIT_SLEEP`](Self::WAIT_SLEEP) (a + /// deadline win resolves [`Status::Deadline`]). Composes with a + /// **standing** [`WAIT_DELIVERY`](Self::WAIT_DELIVERY) post (± + /// [`WAIT_IRQ`](Self::WAIT_IRQ)) under the first-wins rule — + /// delivery is checked first as the one destructive edge, then the + /// word, then revocation/deadline/IRQ. Malformed with an + /// in-session post, [`WAIT_DELIVERY_ALT`](Self::WAIT_DELIVERY_ALT), + /// or [`WAIT_TASK`](Self::WAIT_TASK), and without `WAIT_SLEEP`. + /// Revocation of the subject's lease unparks its waiters + /// [`Status::Revoked`]. The request is the **last** park-block + /// member. + const WAIT_WORD = 1 << 6; } } @@ -1122,6 +1283,7 @@ bitflags! { /// | any of the above + `WAIT_IRQ` | unchanged — legacy default route | /// | above + `WAIT_IRQ_SELECTED` | append [`IrqParkReq`] | /// | any of the above + `WAIT_TASK` | unchanged — the staged join derive carries its own subject | +/// | any of the above + `WAIT_WORD` | append [`WordWaitReq`] (always last; `WAIT_SLEEP` mandatory) | /// /// A flagged session whose argument region cannot supply the block /// refuses [`Status::Malformed`] before anything commits. Ordinary @@ -1160,6 +1322,34 @@ impl IrqParkReq { pub const WIRE: usize = 8; } +/// Argument payload for a **`WAIT_WORD`** park (EVENT.md §6): the memory +/// word the caller waits on, named by *authority* — an arena slot holding +/// a memory lease or a mapped view — plus a byte offset within that +/// subject's span and the value the caller last observed. The kernel +/// resolves the slot through the caller's holdings to the word's +/// physical identity; a raw address never crosses. Rides the park block +/// as its last member (see [`SleepReq`]'s convention table). LE +/// byteorder (`align(1)`). +#[repr(C)] +#[derive(Clone, Copy, Debug, FromBytes, KnownLayout, Immutable, Unaligned)] +pub struct WordWaitReq { + /// Arena slot of the held `MemLease` (or mapped view) the word lives in. + pub subject: U32, + /// Byte offset of the `u32` word from the subject's base; 4-aligned and + /// `offset + 4 <= span`. + pub offset: U32, + /// The value the caller last read; the park refuses to sleep (resolves + /// [`Status::Word`]) if the word already differs. + pub expected: U32, + /// Reserved; must be zero. + pub reserved: U32, +} + +impl WordWaitReq { + /// Wire size (repr(C), 4+4+4+4). + pub const WIRE: usize = 16; +} + /// Argument payload for a **park-on-standing-post** `WAIT_DELIVERY` /// (USERLAND.md §3.4.3; SYSCALL.md D2 tier 1): names the standing recv /// post — committed by a *previous* session — the caller parks on. The @@ -1205,7 +1395,7 @@ pub struct SessionHeader { /// ABI version of this descriptor layout ([`DESCRIPTOR_ABI`]). pub abi: U16, /// Session flags ([`SessionFlags`]): `WAIT_DELIVERY` / `WAIT_IRQ` / - /// `WAIT_SLEEP` / `WAIT_TASK` / `WAIT_DELIVERY_ALT`, all honored by the engine; unknown bits refuse + /// `WAIT_SLEEP` / `WAIT_TASK` / `WAIT_DELIVERY_ALT` / `WAIT_WORD`, all honored by the engine; unknown bits refuse /// [`Status::Malformed`]. pub flags: U16, /// Reserved; must be zero. Frozen additive headroom. @@ -1238,6 +1428,17 @@ const _: () = { assert!(size_of::() == FoundReq::WIRE); assert!(size_of::() == SleepReq::WIRE); assert!(size_of::() == IrqParkReq::WIRE); + assert!(size_of::() == WordWaitReq::WIRE); + assert!(size_of::() == WakeWordReq::WIRE); + // WAKE_WORD takes a MemLease / view subject: it shares that subject with + // the lease self-verbs, so it must be distinct from every flavor that + // can name one, and it owns the next free high-band byte. + assert!(DERIVE_KIND_WAKE_WORD != DERIVE_KIND_DEVICE_CLAIM_DMA_RETIRE); + assert!(DERIVE_KIND_WAKE_WORD != DERIVE_KIND_DEVICE_CLAIM_QUIESCE); + assert!(DERIVE_KIND_WAKE_WORD != DERIVE_KIND_DEVICE_CLAIM_CLOSE); + assert!(DERIVE_KIND_WAKE_WORD != DERIVE_KIND_EXEC_SEAL); + assert!(DERIVE_KIND_WAKE_WORD != DERIVE_KIND_DEVICE_ACTIVATE); + assert!((crate::cap::ResourceKind::DeviceMmu as u16) < DERIVE_KIND_WAKE_WORD as u16); assert!(size_of::() == ExitReq::WIRE); assert!(size_of::() == DeviceActivateReq::WIRE); assert!(size_of::() == GuestFoundReq::WIRE); @@ -1246,6 +1447,8 @@ const _: () = { assert!(size_of::() == NoticeReq::WIRE); assert!(size_of::() == TpReq::WIRE); assert!(size_of::() == ProtectReq::WIRE); + assert!(size_of::() == FaultUpcallReq::WIRE); + assert!(size_of::() == ReserveReleaseReq::WIRE); assert!(size_of::() == PortReq::WIRE); // The self-verb discriminants share one space above `ResourceKind` // and are told apart by SUBJECT kind, not by value — `SET_TP` and @@ -1278,6 +1481,25 @@ const _: () = { assert!(DERIVE_KIND_ARENA_EXTEND != DERIVE_KIND_IOVA_ANON); assert!(DERIVE_KIND_ARENA_EXTEND != DERIVE_KIND_EXEC_SEAL); assert!((crate::cap::ResourceKind::MemoryController as u16) < DERIVE_KIND_ARENA_EXTEND as u16); + // FAULT_UPCALL is the fourth AddrSpace self-verb: pairwise distinct + // from the other three for the same reason they are from each other. + assert!(DERIVE_KIND_FAULT_UPCALL != DERIVE_KIND_SET_TP); + assert!(DERIVE_KIND_FAULT_UPCALL != DERIVE_KIND_PROTECT); + assert!(DERIVE_KIND_FAULT_UPCALL != DERIVE_KIND_ARENA_EXTEND); + assert!((crate::cap::ResourceKind::MemoryController as u16) < DERIVE_KIND_FAULT_UPCALL as u16); + // RESERVE_RELEASE takes a MemLease subject, so it must differ from every + // flavor that can name one — WAKE_WORD's list, plus WAKE_WORD itself. + assert!(DERIVE_KIND_RESERVE_RELEASE != DERIVE_KIND_WAKE_WORD); + assert!(DERIVE_KIND_RESERVE_RELEASE != DERIVE_KIND_OWNED_MAP); + assert!(DERIVE_KIND_RESERVE_RELEASE != DERIVE_KIND_IOVA_ANON); + assert!(DERIVE_KIND_RESERVE_RELEASE != DERIVE_KIND_EXEC_SEAL); + assert!(DERIVE_KIND_RESERVE_RELEASE != DERIVE_KIND_DEVICE_CLAIM_DMA_RETIRE); + assert!(DERIVE_KIND_RESERVE_RELEASE != DERIVE_KIND_DEVICE_CLAIM_QUIESCE); + assert!(DERIVE_KIND_RESERVE_RELEASE != DERIVE_KIND_DEVICE_CLAIM_CLOSE); + assert!(DERIVE_KIND_RESERVE_RELEASE != DERIVE_KIND_DEVICE_ACTIVATE); + assert!( + (crate::cap::ResourceKind::MemoryController as u16) < DERIVE_KIND_RESERVE_RELEASE as u16 + ); // DEVICE_ACTIVATE shares the PciDevice subject with resource derives but // owns the next high-band byte; keep it distinct from every effect flavor. assert!(DERIVE_KIND_DEVICE_ACTIVATE != DERIVE_KIND_ARENA_EXTEND); @@ -1336,7 +1558,7 @@ const _: () = { /// Their authenticated subject kinds disambiguate both pairs. This table is a /// vocabulary, not a uniqueness proof; the pairwise same-subject assertions /// above carry the distinctness that matters. -const DERIVE_FLAVORS: [u8; 35] = [ +const DERIVE_FLAVORS: [u8; 38] = [ DERIVE_KIND_TASK_IDENTITY, DERIVE_KIND_TASK_NAME, DERIVE_KIND_PROCESS_DRAIN, @@ -1372,6 +1594,9 @@ const DERIVE_FLAVORS: [u8; 35] = [ DERIVE_KIND_DEVICE_CLAIM_CLOSE, DERIVE_KIND_DEVICE_CLAIM_QUIESCE, DERIVE_KIND_DEVICE_CLAIM_DMA_RETIRE, + DERIVE_KIND_WAKE_WORD, + DERIVE_KIND_FAULT_UPCALL, + DERIVE_KIND_RESERVE_RELEASE, ]; /// Is `kind` a byte this ABI's `Derive` speaks? — the low, append-only @@ -1550,6 +1775,7 @@ impl Committed { 15 => Status::FabricRouteInfeasible, 16 => Status::FabricRouteCapacity, 17 => Status::FabricRouteCapability, + 18 => Status::Word, _ => Status::Malformed, } } @@ -1776,6 +2002,34 @@ pub const DERIVE_KIND_DEVICE_CLAIM_QUIESCE: u8 = 0xe4; /// Uses DeviceClaimCloseReq and returns DeviceClaimDmaReport, never a regrant. pub const DERIVE_KIND_DEVICE_CLAIM_DMA_RETIRE: u8 = 0xe3; +/// **Wake the waiters of a memory word** (EVENT.md §6 — the producer half +/// of the kernel address-wait). A post-commit **effect** on a held +/// `MemLease` / mapped-view subject with `MODIFY` ("you may wake what you +/// can write"): the [`WakeWordReq`] names the byte offset and how many +/// parked waiters of that word's *physical* identity to release. Mints +/// nothing; the result slot is the subject's own (inert). Replaces the +/// notice-mode send for a producer that shares the memory but holds no +/// send end — the `futex` provisioning gap and the SPMD hub relay. +/// The store itself is the caller's; this only publishes it to parked +/// waiters, so a caller stores first and wakes second. +pub const DERIVE_KIND_WAKE_WORD: u8 = 0xdc; + +/// Argument payload for [`DERIVE_KIND_WAKE_WORD`]. +#[repr(C)] +#[derive(Clone, Copy, Debug, FromBytes, KnownLayout, Immutable, Unaligned)] +pub struct WakeWordReq { + /// Byte offset of the `u32` word from the subject's base; 4-aligned and + /// `offset + 4 <= span`. + pub offset: U32, + /// Maximum waiters to release (`u32::MAX` = all); `0` is malformed. + pub count: U32, +} + +impl WakeWordReq { + /// Wire size (repr(C), 4+4). + pub const WIRE: usize = 8; +} + #[repr(C)] #[derive(Clone, Copy, Debug, TryFromBytes, KnownLayout, Immutable, Unaligned)] pub struct DeviceClaimCloseReq { @@ -1919,6 +2173,12 @@ pub enum ViewMapWindow { /// like [`Map`](Self::Map)'s but task-level and off the per-thread stride /// every view shares. Stamped for founded tasks only. Heap, + /// The task's reservation window (`geom::RESERVE`, GEOM_ABI v8): the view + /// lands at the exact page [`ViewMapReq::page`] names, where a runtime + /// commits the next pages of an address range it reserved for itself (a + /// guarded wasm memory growing). No cursor: an occupied page refuses, as + /// a [`Surfaces`](Self::Surfaces) lane does. + Reserve, } /// One of the two fixed, reusable halves of a task's `SURFACES` window. @@ -1952,13 +2212,15 @@ impl ViewMapWindow { Self::Map => crate::task::geom::MAP, Self::Surfaces => crate::task::geom::SURFACES, Self::Heap => crate::task::geom::HEAP, + Self::Reserve => crate::task::geom::RESERVE, } } } /// The whole-view map request for [`DERIVE_KIND_VIEW_MAP`]. No caller-supplied /// VA, length, destination space or access mask: each is derived from the -/// receiver's stamped geometry and authenticated view. Reserved bytes are zero. +/// receiver's stamped geometry and authenticated view — the one placement a +/// caller picks is a page of its own `RESERVE` window, never an address. #[repr(C)] #[derive(Clone, Copy, Debug, TryFromBytes, KnownLayout, Immutable, Unaligned)] pub struct ViewMapReq { @@ -1969,9 +2231,19 @@ pub struct ViewMapReq { /// [`VIEW_MAP_SINK`], or zero. (Formerly the first reserved byte, so a /// zeroed request keeps its meaning.) pub flags: u8, - pub reserved: [u8; 4], + /// For `RESERVE`, the page of the window the view's first page lands on, + /// in [`RESERVE_PAGE_BYTES`] units; zero for every other window. + /// (Formerly the four reserved bytes, so a zeroed request keeps its + /// meaning.) + pub page: U32, } +/// The unit of a `RESERVE` page index ([`ViewMapReq::page`], +/// [`ReserveReleaseReq`]): 4 KiB on every target, the page kore's user +/// targets link for (`max-page-size=4096`, `res/IMAGE.md` §7). A `u32` of +/// them spans 16 TiB, past any window the layout can stamp. +pub const RESERVE_PAGE_BYTES: u64 = 0x1000; + /// [`ViewMapReq::flags`]: when the lessor's lease dies under this mapping, /// repoint its pages at the receiver space's sink page instead of unmapping /// them (`res/STORAGE_TRANSPORT.md` §12, option B). For a row that reads a @@ -1984,18 +2256,22 @@ impl ViewMapReq { pub const WIRE: usize = 8; pub fn placement(&self) -> Option { - if self.reserved != [0; 4] || self.flags & !VIEW_MAP_SINK != 0 { + if self.flags & !VIEW_MAP_SINK != 0 { return None; } if self.sink() && self.window.get() != crate::task::geom::MAP { return None; } + if self.page.get() != 0 && self.window.get() != crate::task::geom::RESERVE { + return None; + } match self.window.get() { crate::task::geom::MAP if self.lane == 0 => Some(ViewMapWindow::Map), crate::task::geom::HEAP if self.lane == 0 => Some(ViewMapWindow::Heap), crate::task::geom::SURFACES if SurfaceMapLane::from_wire(self.lane).is_some() => { Some(ViewMapWindow::Surfaces) } + crate::task::geom::RESERVE if self.lane == 0 => Some(ViewMapWindow::Reserve), _ => None, } } @@ -2011,6 +2287,11 @@ impl ViewMapReq { .then(|| SurfaceMapLane::from_wire(self.lane)) .flatten() } + + /// The `RESERVE` page the view lands on, for a valid `RESERVE` request. + pub fn reserve_page(&self) -> Option { + (self.placement() == Some(ViewMapWindow::Reserve)).then(|| self.page.get()) + } } /// Argument payload for a **detached window carve** (`Derive × MemLease → @@ -4976,6 +5257,65 @@ mod tests { } } + /// The fault upcall's request is a frozen 32-byte block of four LE + /// addresses, and its selector is registered, distinct from the other + /// `AddrSpace` self-verbs, with the byte below it still failing closed. + #[test] + fn fault_upcall_request_layout_is_frozen() { + use zerocopy::TryFromBytes; + + let mut bytes = [0u8; FaultUpcallReq::WIRE]; + for (i, word) in [0x1000u64, 0x1010, 0x7000_0000, 0x7001_0000] + .iter() + .enumerate() + { + bytes[i * 8..i * 8 + 8].copy_from_slice(&word.to_le_bytes()); + } + let (req, rest) = FaultUpcallReq::try_ref_from_prefix(&bytes[..]).expect("decodes"); + assert!(rest.is_empty()); + assert_eq!( + ( + req.entry.get(), + req.decline.get(), + req.stack_lo.get(), + req.stack_hi.get() + ), + (0x1000, 0x1010, 0x7000_0000, 0x7001_0000) + ); + assert!(FaultUpcallReq::try_ref_from_prefix(&bytes[..FaultUpcallReq::WIRE - 1]).is_err()); + + assert!(derive_kind_known(DERIVE_KIND_FAULT_UPCALL)); + for other in [ + DERIVE_KIND_SET_TP, + DERIVE_KIND_PROTECT, + DERIVE_KIND_ARENA_EXTEND, + ] { + assert_ne!(DERIVE_KIND_FAULT_UPCALL, other); + } + assert_eq!(DERIVE_KIND_WAKE_WORD - 1, DERIVE_KIND_FAULT_UPCALL); + } + + #[test] + fn reserve_release_request_layout_is_frozen() { + use zerocopy::TryFromBytes; + + let mut bytes = [0u8; ReserveReleaseReq::WIRE]; + bytes[0..4].copy_from_slice(&0x10_0000u32.to_le_bytes()); + bytes[4..8].copy_from_slice(&0x10_4000u32.to_le_bytes()); + let (req, rest) = ReserveReleaseReq::try_ref_from_prefix(&bytes[..]).expect("decodes"); + assert!(rest.is_empty()); + assert_eq!((req.page.get(), req.pages.get()), (0x10_0000, 0x10_4000)); + assert!( + ReserveReleaseReq::try_ref_from_prefix(&bytes[..ReserveReleaseReq::WIRE - 1]).is_err() + ); + + assert!(derive_kind_known(DERIVE_KIND_RESERVE_RELEASE)); + assert_eq!(DERIVE_KIND_FAULT_UPCALL - 1, DERIVE_KIND_RESERVE_RELEASE); + assert!( + !derive_kind_known(DERIVE_KIND_RESERVE_RELEASE - 1), + "the next future flavor fails closed" + ); + } #[test] fn line_and_i2c_gpio_flavors_are_known_without_aliasing() { assert!(derive_kind_known(DERIVE_KIND_IRQ_ROUTE_LINE)); @@ -5320,7 +5660,8 @@ mod tests { (window == ViewMapWindow::Surfaces).then_some(SurfaceMapLane::First) ); // Byte 3 is the flags byte: the sink bit is admitted on MAP only, - // an unknown bit never. Bytes 4.. stay reserved. + // an unknown bit never. Bytes 4.. are the RESERVE page, zero + // for every other window. let mut sink = bytes; sink[3] = VIEW_MAP_SINK; let (req, _) = ViewMapReq::try_ref_from_prefix(&sink).unwrap(); @@ -5352,7 +5693,7 @@ mod tests { window: ViewMapWindow::Map.role().into(), lane: SurfaceMapLane::Second as u8, flags: 0, - reserved: [0; 4], + page: 0.into(), }; assert_eq!(invalid_map_lane.placement(), None); for role in [ @@ -5365,16 +5706,36 @@ mod tests { window: role.into(), lane: 0, flags: 0, - reserved: [0; 4], + page: 0.into(), }; assert_eq!(req.placement(), None); } + // RESERVE takes its page from bytes 4..8 — any page, the window's + // bounds are the kernel's to check — and neither a lane nor a sink. + for page in [0u32, 1, 0x10_0400, u32::MAX] { + let mut bytes = [0u8; ViewMapReq::WIRE]; + bytes[..2].copy_from_slice(&ViewMapWindow::Reserve.role().to_le_bytes()); + bytes[4..8].copy_from_slice(&page.to_le_bytes()); + let (req, _) = ViewMapReq::try_ref_from_prefix(&bytes).unwrap(); + assert_eq!(req.placement(), Some(ViewMapWindow::Reserve)); + assert_eq!(req.reserve_page(), Some(page)); + assert_eq!(req.surface_lane(), None); + let mut lane = bytes; + lane[2] = 1; + let (req, _) = ViewMapReq::try_ref_from_prefix(&lane).unwrap(); + assert_eq!((req.placement(), req.reserve_page()), (None, None)); + let mut sink = bytes; + sink[3] = VIEW_MAP_SINK; + let (req, _) = ViewMapReq::try_ref_from_prefix(&sink).unwrap(); + assert_eq!((req.placement(), req.reserve_page()), (None, None)); + } + assert_eq!(second.reserve_page(), None); // The sink flag is admitted on MAP alone; an unknown flag refuses. let sink = ViewMapReq { window: ViewMapWindow::Map.role().into(), lane: 0, flags: VIEW_MAP_SINK, - reserved: [0; 4], + page: 0.into(), }; assert_eq!(sink.placement(), Some(ViewMapWindow::Map)); assert!(sink.sink()); diff --git a/native/crates/abi/src/svc.rs b/native/crates/abi/src/svc.rs index dc2f661..703543e 100644 --- a/native/crates/abi/src/svc.rs +++ b/native/crates/abi/src/svc.rs @@ -174,6 +174,10 @@ impl MmRepWire { /// (an admission in flight, or a retired face whose supplies have not /// all returned); ask again. pub const REFUSE_PENDING: u8 = 6; + /// `MmRefusal::Full` — no per-launch face fits: every serving slot is + /// attached or still crediting returns, or mm has no room to grow one. + /// A capacity refusal, the face admission's `TableFull`. + pub const REFUSE_FULL: u8 = 7; } // ─── monitor — federated telemetry assembly ─────────────────────── @@ -1353,6 +1357,11 @@ pub struct TaskReqWire { /// Why a library did not seed (`TaskRepWire::LIBRARY_FAILURE_*`): the /// stage `SeedLibrary` stopped at, so a `libs: L/M` verdict short of M names /// its cause on the kernel's store line instead of swallowing it. +/// +/// The same vocabulary names where `ProvisionHome` stopped: the home +/// skeleton is the same archive-export copy into a different directory, so +/// its stages are these, plus [`HOME`](Self::HOME) and +/// [`RECORD`](Self::RECORD), which only a home has. pub struct LibraryFailure; impl LibraryFailure { pub const NONE: u8 = 0; @@ -1410,6 +1419,13 @@ impl LibraryFailure { pub const NO_IMAGE_CLIENT: u8 = 26; /// Write stage: the copy answered false without naming a stage (a new exit lacks its stamp). pub const UNCLASSIFIED: u8 = 27; + /// Home skeleton: the face is not live, or its durable home + /// (`work/`) is absent or not a directory. + pub const HOME: u8 = 28; + /// Home skeleton: the home's provisioning record (`home-skeleton/`, + /// beside `work/`) could not be read, does not parse, or could not be + /// written. + pub const RECORD: u8 = 29; } impl TaskReqWire { @@ -1479,6 +1495,13 @@ impl TaskReqWire { /// gains no peer, no projection, and no launch authority; a task links /// it into itself at runtime. Seal-exempt, via==0 only. pub const OP_SEED_LIBRARY: u8 = 22; + /// Provision one minted face's durable home with the home skeleton: copy + /// each archive DATA member the home's record says it has not had yet + /// into `//`, never replacing a file already there, then + /// record the skeleton version (`services/exec` `HOME_SKELETON`). + /// `slot` names the face; no plan, no target, no grants. Seal-exempt, + /// via==0 only: provisioning is the administrator's, not a task's. + pub const OP_PROVISION_HOME: u8 = 25; /// Every retired `TaskReqWire` discriminant, in one definition the /// decoder cites (the [`derive_kind_known`](crate::session::derive_kind_known) @@ -1719,6 +1742,19 @@ impl TaskRepWire { /// Complete live bit vocabulary for `TAG_LIBRARY_SEEDED.index`. pub const LIBRARY_SEED_MASK: u8 = Self::LIBRARY_SEEDED | Self::LIBRARY_MANIFEST | Self::LIBRARY_FAILURE_MASK; + /// Reply to [`TaskReqWire::OP_PROVISION_HOME`]: `index` is + /// [`HOME_PROVISIONED`](Self::HOME_PROVISIONED) plus the + /// [`LibraryFailure`](crate::svc::LibraryFailure) stage in the + /// `LIBRARY_FAILURE_*` bits; `status` is how many files this call copied + /// in (zero for a home already at the current skeleton version). An + /// unprovisioned home is a posture (no store, or a stage that failed and + /// is retried next boot), never a refusal. + pub const TAG_HOME_PROVISIONED: u8 = 25; + /// `TAG_HOME_PROVISIONED.index`: the home is recorded at the current + /// skeleton version. + pub const HOME_PROVISIONED: u8 = 1 << 0; + /// Complete live bit vocabulary for `TAG_HOME_PROVISIONED.index`. + pub const HOME_PROVISION_MASK: u8 = Self::HOME_PROVISIONED | Self::LIBRARY_FAILURE_MASK; /// A task-owned exec client face was attached; `index` is its bounded id. pub const TAG_CLIENT_MINTED: u8 = 11; /// A minted face received its complete reply-grant receive bank. @@ -1757,10 +1793,14 @@ impl TaskRepWire { /// `TaskRefusal::Retired` — the request named a frozen tombstone /// discriminant ([`TaskReqWire::op_retired`]); nothing was attempted. pub const REFUSE_RETIRED: u8 = 10; - /// `TaskRefusal::Contended` — concurrent invocations landed admissions - /// for more than one generation, so no landing could be attributed to - /// this request; every in-flight admission was retired and the caller - /// re-derives and retries. + /// `TaskRefusal::Contended` — a launch-time face was still settling + /// earlier work: its service refused while an earlier admission was + /// unclaimed (accelerator or memory), or its administration pair still + /// carried a request the launch could not resume and nothing was + /// allocated. Every moved input was retired and the caller re-derives + /// and retries. (Concurrent + /// launches of different generations are attributed by each + /// invocation's possession proof and never meet here.) pub const REFUSE_CONTENDED: u8 = 11; /// `TaskRefusal::PeerAdmissionPending` — the face's service accepted the /// child's peer bundle but had not yet published the admission verdict @@ -2072,6 +2112,10 @@ pub struct InputReport { /// Zero for legacy records. Contacts may carry version-1 axis metadata: /// `[1, class, max_x_le[2], max_y_le[2], 0]`. Coordinates are normalized /// to a zero minimum by the descriptor decoder, not scaled to a display. + /// Keyboards carry `[locks, 0, 0, 0, 0, 0, 0]`: the seat's + /// [`Locks`](crate::keyboard::Locks) as of this snapshot, stamped by the + /// keymap owner (`kore-svc-input-stream`) — a device writes zero, since a + /// lock is a property of the seat, not of the key matrix. pub reserved: [u8; 7], } @@ -2162,6 +2206,27 @@ impl InputReport { } } + /// This keyboard snapshot with the seat's lock state stamped in. + /// Any other kind is returned unchanged: locks mean nothing there. + pub const fn with_locks(mut self, locks: crate::keyboard::Locks) -> Self { + if self.kind == Self::KIND_KEYBOARD { + self.reserved[0] = locks.bits(); + } + self + } + + /// The lock state of a well-formed keyboard record: `None` for any other + /// kind, and for a keyboard record whose lock byte names a bit no lock + /// owns or whose remaining reserved bytes are not zero — a record a + /// consumer must refuse rather than interpret. + pub const fn keyboard_locks(&self) -> Option { + let [locks, a, b, c, d, e, f] = self.reserved; + if self.kind != Self::KIND_KEYBOARD || (a | b | c | d | e | f) != 0 { + return None; + } + crate::keyboard::Locks::from_bits(locks) + } + /// A mouse report: button bitmap + signed relative deltas. pub const fn mouse(source: u8, buttons: u8, dx: i8, dy: i8, wheel: i8) -> Self { Self { @@ -2273,6 +2338,10 @@ impl InputReqWire { /// `InputRequest::Status` — read the service's own books. Carries no /// operand: it steers nothing and names no subscription. pub const OP_STATUS: u8 = 4; + /// `InputRequest::SetIndicators` — `slot` carries the keyboard's wire + /// source identity (below 256), `subscription` the lock state's + /// [`Locks`](crate::keyboard::Locks) bits; `focus` and `reserved` zero. + pub const OP_SET_INDICATORS: u8 = 6; /// `SetFocus`: steer subsequent reports to nobody. pub const FOCUS_NONE: u8 = 0; @@ -2352,6 +2421,8 @@ impl InputRepWire { pub const TAG_REFUSED: u8 = 4; /// `InputResponse::Status` — see `rung` and the four counters. pub const TAG_STATUS: u8 = 5; + /// `InputResponse::Indicated` — the indicator state was recorded. + pub const TAG_INDICATED: u8 = 7; /// `InputRefusal::BadQueue` — the delivered slot was absent, /// non-writable, or not a well-formed report ring. @@ -2449,6 +2520,10 @@ impl HypervisorReqWire { pub const OP_SPAWN_MMIO: u8 = 12; /// Two-vCPU SPMD guest witness. pub const OP_SPAWN_SMP: u8 = 13; + /// Modern virtio-MMIO console guest. + pub const OP_SPAWN_VIRTIO: u8 = 14; + /// Explicit port-UART compatibility witness; normal guests use virtio. + pub const OP_SPAWN_UART: u8 = 15; } /// Capacity of the hypervisor pair window's **guest-output** console @@ -2608,6 +2683,8 @@ impl HypervisorStatusWire { /// The echo guest on the memory-mapped console (#765). pub const PAYLOAD_MMIO: u8 = 6; pub const PAYLOAD_SMP: u8 = 7; + pub const PAYLOAD_VIRTIO: u8 = 8; + pub const PAYLOAD_UART: u8 = 9; /// A live guest task currently exists. pub const STATUS_GUEST_EXISTS: u16 = 1 << 0; @@ -3548,6 +3625,11 @@ const _: () = { // histogram. assert!(size_of::() == 24 + 80 + 29 * 8 + 66); // 402 assert!(size_of::() == FsSystemStatusWire::BYTES); + // 16-byte head ‖ 2×8 sizes ‖ 2×4 ‖ 8×2 books ‖ 4×4 latencies ‖ 8 bytes ‖ + // 5×4 refresh counters. + assert!(size_of::() == 16 + 16 + 8 + 16 + 16 + 8 + 20); // 100 + assert!(size_of::() == PackMaintenanceWire::BYTES); + assert!(size_of::() == PackBucketWire::BYTES); // Both 8-deep rings must fit ONE founded pair window, and the filesystem // now asks the founding for `SvcProtocol::SERVE_PAIRS` of them — so this // is a per-pair budget the founder pays four times over, not once. @@ -3716,7 +3798,8 @@ impl OutChunk { )] pub struct InReqWire { /// Verb discriminant: [`OP_ATTACH_RX`](Self::OP_ATTACH_RX) / - /// [`OP_DETACH_RX`](Self::OP_DETACH_RX) / [`OP_STATUS`](Self::OP_STATUS); + /// [`OP_DETACH_RX`](Self::OP_DETACH_RX) / [`OP_STATUS`](Self::OP_STATUS) + /// / [`OP_HOLD`](Self::OP_HOLD) / [`OP_REPRINT`](Self::OP_REPRINT); /// zero (unseeded) is invalid. pub op: u8, /// Reserved; must be zero. @@ -3744,6 +3827,13 @@ impl InReqWire { pub const OP_DETACH_RX: u8 = 2; /// `InputStreamRequest::Status` — snapshot the counters. pub const OP_STATUS: u8 = 3; + /// `InputStreamRequest::Hold` — the line editor stops editing and + /// echoing until `Reprint`: keys queue, so the client can print an + /// asynchronous notice over an idle prompt. + pub const OP_HOLD: u8 = 5; + /// `InputStreamRequest::Reprint` — echo the line being edited, cursor + /// in place, then apply every key queued since `Hold`. + pub const OP_REPRINT: u8 = 6; } /// The stream-input-service reply wire form. The legacy stream counters are @@ -3755,7 +3845,8 @@ impl InReqWire { pub struct InRepWire { /// Outcome discriminant: [`TAG_ATTACHED`](Self::TAG_ATTACHED) / /// [`TAG_DETACHED`](Self::TAG_DETACHED) / [`TAG_STATUS`](Self::TAG_STATUS) - /// / [`TAG_REFUSED`](Self::TAG_REFUSED). + /// / [`TAG_REFUSED`](Self::TAG_REFUSED) / [`TAG_HELD`](Self::TAG_HELD) / + /// [`TAG_REPRINTED`](Self::TAG_REPRINTED). pub tag: u8, /// `Refused`: the `InputStreamRefusal` discriminant /// ([`REFUSE_BUSY`](Self::REFUSE_BUSY)…); zero otherwise. @@ -3830,6 +3921,11 @@ impl InRepWire { pub const TAG_STATUS: u8 = 3; /// `InputStreamResponse::Refused` — see `refusal`. pub const TAG_REFUSED: u8 = 4; + /// `InputStreamResponse::Held` — the line editor holds. + pub const TAG_HELD: u8 = 6; + /// `InputStreamResponse::Reprinted` — the line is echoed again and the + /// held keys applied. + pub const TAG_REPRINTED: u8 = 7; /// `InputStreamRefusal::Busy` — a queue is already attached (the v1 /// single-peer bound, `res/USERLAND.md` §5.9). @@ -3927,7 +4023,26 @@ mod tests { // shared report window's peer is untrusted) and round-trips. let kbd = InputReport::keyboard(3, 0x02, [0x04, 0x05, 0, 0, 0, 0]); assert_eq!(InputReport::read_from_bytes(kbd.as_bytes()).unwrap(), kbd); + // A device writes no locks; the keymap owner stamps them into the + // first reserved byte and nowhere else. + use crate::keyboard::Locks; + assert_eq!(kbd.keyboard_locks(), Some(Locks::NONE)); + let caps = kbd.with_locks(Locks::CAPS.toggled(Locks::NUM)); + assert_eq!(caps.reserved, [0x03, 0, 0, 0, 0, 0, 0]); + assert_eq!( + InputReport::read_from_bytes(caps.as_bytes()) + .unwrap() + .keyboard_locks(), + Some(Locks::CAPS.toggled(Locks::NUM)) + ); + for (index, value) in [(0, 0x08), (1, 1), (6, 0x80)] { + let mut invalid = caps; + invalid.reserved[index] = value; + assert_eq!(invalid.keyboard_locks(), None, "byte {index} = {value:#x}"); + } let mouse = InputReport::mouse(4, 0x01, 5, -3, 0); + assert_eq!(mouse.with_locks(Locks::CAPS), mouse, "only keyboards carry locks"); + assert_eq!(mouse.keyboard_locks(), None); assert_eq!(mouse.data, [5, (-3i8) as u8, 0, 0, 0, 0]); let raw = [0xC3u8; 16]; assert_eq!(InputReport::read_from_bytes(&raw).unwrap().kind, 0xC3); @@ -4076,9 +4191,11 @@ mod tests { SurReqWire::OP_QUERY, SurReqWire::OP_CREATE_SURFACE, SurReqWire::OP_SIGNAL, + SurReqWire::OP_SHOW_MENU, SurRepWire::TAG_INFO, SurRepWire::TAG_SURFACE_MADE, SurRepWire::TAG_REFUSED, + SurRepWire::TAG_MENU_SHOWN, ] { assert_ne!(d, 0); } @@ -4164,6 +4281,83 @@ impl SurReqWire { /// leaves only on the echo. No words; `subject` is zero /// ([`SurRepWire::TAG_CLOSE_REQUESTED`]). pub const OP_REQUEST_CLOSE: u8 = 6; + /// `SurfaceRequest::SetCursor` — the cursor this window shows while the + /// pointer is over its content: `a` = one `CURSOR_*` code below, any + /// other value refused. A request, not a grant: the compositor's own + /// pixels (chrome, the dock) choose for themselves, and the window's + /// choice holds until it asks again. No subject + /// ([`SurRepWire::TAG_CURSOR_SET`]). + pub const OP_SET_CURSOR: u8 = 7; + /// `SurfaceRequest::OpenObject` — ask this window's controller to open + /// a filesystem object in its kind's default handler: `a` = the + /// object's stable identity (never zero), `b` = its document-kind code + /// (`kore_window::document::DocumentKind`; zero for unknown, which + /// opens in the catalogue's document app). Queued as an inert + /// `ShellAction::Open` for the controller that launched this window, + /// which decides; the identity is data the opened window locates from + /// its own root, never authority. No subject + /// ([`SurRepWire::TAG_OPEN_QUEUED`]). + pub const OP_OPEN_OBJECT: u8 = 8; + /// `SurfaceRequest::ShowMenu` — the compositor draws the context menu + /// this window encoded in its pair lease's menu plane + /// (`kore_window::menu`): `a` = the anchor's content-relative + /// `x as u32 | (y as u32) << 32`, `b` = the encoded length (1 to the + /// plane's size) `| serial << 32` (a nonzero `u32` the window picks). + /// The compositor snapshots and validates the plane before replying, + /// so the window may reuse it once answered; the choice returns as an + /// `Event::Menu` carrying the serial. No subject + /// ([`SurRepWire::TAG_MENU_SHOWN`]). + pub const OP_SHOW_MENU: u8 = 9; + /// `SurfaceRequest::TakeCapture` — a capture writer's one verb: lend + /// the frozen frame its pair was admitted for, read-only + /// ([`SurRepWire::TAG_CAPTURE`]). Only a capture peer may ask; no + /// words, no subject. + pub const OP_TAKE_CAPTURE: u8 = 10; + /// `SurfaceRequest::PickObject` — ask the user, through this window's + /// controller, to choose a file (`res/FILE_PICKER.md`): `a` = the file + /// types accepted, a nonzero mask over `kore_kit::kinds` rows; `b` = a + /// nonzero `u32` serial the window picks. Queued as an inert + /// `ShellAction::Pick` carrying a fresh token; the choice returns as + /// `Event::Picked` carrying the serial. No subject + /// ([`SurRepWire::TAG_PICK_QUEUED`]). + pub const OP_PICK_OBJECT: u8 = 11; + /// `SurfaceRequest::ClaimPick` — the picker the controller launched + /// for a pick binds itself to it: `a` = the pick's token (never zero), + /// handed to it at launch. Only a window of the asking window's own + /// controller, and only the first, may claim. No subject + /// ([`SurRepWire::TAG_PICK_CLAIMED`]). + pub const OP_CLAIM_PICK: u8 = 12; + /// `SurfaceRequest::Picked` — the claimed picker answers its pick: `a` + /// = the token, `b` = the chosen object's stable identity (zero: the + /// user cancelled), `c` = the length of the chosen entry's name, which + /// the picker wrote at the head of its menu plane (zero when + /// cancelled, else 1 to `kore_window::pick::PICK_NAME_BYTES`). The + /// compositor snapshots the name and ends the pick. No subject + /// ([`SurRepWire::TAG_PICK_ANSWERED`]). + pub const OP_PICKED: u8 = 13; + + /// [`OP_SET_CURSOR`](Self::OP_SET_CURSOR) `a`: the standard arrow. + pub const CURSOR_DEFAULT: u64 = 0; + /// The I-beam: text a press would place a caret in. + pub const CURSOR_TEXT: u64 = 1; + /// The pointing hand: a control a press would activate. + pub const CURSOR_POINTER: u64 = 2; + /// The open hand: something a press would pick up. + pub const CURSOR_GRAB: u64 = 3; + /// The closed hand: something being carried. + pub const CURSOR_GRABBING: u64 = 4; + /// The vertical resize arrow: an edge that moves up and down. + pub const CURSOR_RESIZE_NS: u64 = 5; + /// The horizontal resize arrow: an edge that moves left and right. + pub const CURSOR_RESIZE_EW: u64 = 6; + /// The diagonal resize arrow from top-left to bottom-right. + pub const CURSOR_RESIZE_NWSE: u64 = 7; + /// The diagonal resize arrow from top-right to bottom-left. + pub const CURSOR_RESIZE_NESW: u64 = 8; + /// The hand under a no-entry badge: what a press here would not do. + pub const CURSOR_NOT_ALLOWED: u64 = 9; + /// The spinner: work under way while the window still answers. + pub const CURSOR_PROGRESS: u64 = 10; /// [`OP_CREATE_SURFACE`](Self::OP_CREATE_SURFACE) `b`: CPU-reachable /// backing — the one v1 reach. @@ -4239,6 +4433,29 @@ impl SurRepWire { /// close was raised under (never zero: an unconfigured window cannot /// be closing). The echo itself rides the event channel. pub const TAG_CLOSE_REQUESTED: u8 = 7; + /// `SurfaceResponse::CursorSet` — `a` echoes the + /// [`SurReqWire`] `CURSOR_*` code now held for the window. + pub const TAG_CURSOR_SET: u8 = 8; + /// `SurfaceResponse::OpenQueued` — `a` echoes the identity queued. + pub const TAG_OPEN_QUEUED: u8 = 9; + /// `SurfaceResponse::MenuShown` — `a` echoes the menu's serial; the + /// menu is open and the plane may be reused. + pub const TAG_MENU_SHOWN: u8 = 10; + /// `SurfaceResponse::Capture` — `id` = what the freeze's copy cost, in + /// nanoseconds, `a` = `width ‖ height << 16 ‖ mode << 32 ‖ screen + /// << 40 ‖ density << 48` (`kore_window::capture`), `b` = nanoseconds + /// since the UNIX epoch at the freeze (zero without a wall anchor), + /// `c` = nanoseconds since boot at the freeze. The reply's one grant + /// is the frozen frame's view, ACCESS without MODIFY, the pixels in + /// whole pages: the writer can read it and never write it. + pub const TAG_CAPTURE: u8 = 11; + /// `SurfaceResponse::PickQueued` — `a` echoes the pick's serial. + pub const TAG_PICK_QUEUED: u8 = 12; + /// `SurfaceResponse::PickClaimed` — `a` echoes the token claimed. + pub const TAG_PICK_CLAIMED: u8 = 13; + /// `SurfaceResponse::PickAnswered` — `a` echoes the token answered; the + /// pick has ended and the plane may be reused. + pub const TAG_PICK_ANSWERED: u8 = 14; /// A v1 vocabulary gate: an unknown op, a non-XRGB8888 format, a /// non-CPU reach, a non-LINEAR modifier, a non-empty flag set. @@ -4563,7 +4780,53 @@ impl FsReqWire { /// stale completions (low half) and refused timeline completions (high /// half) in `revision`. No operand. pub const OP_STOR_INSPECT: u8 = 61; - pub const OP_MAX: u8 = Self::OP_STOR_INSPECT; + /// One page of an open object's visible parents into the data window: + /// the directories reachable from the handle's authority root that hold + /// an edge to it, each as an [`FsSearchHitWire`] whose `path` is the + /// shortest route from that root (empty for the root itself). The cursor + /// rides `length` and the page lands at `window_offset`, as `List`'s do; + /// the reply is `List`'s (`count`, `cursor`, [`FsRepWire::FLAG_MORE`]). + /// Observation data only: a caller opens a parent by its route. + pub const OP_PARENTS: u8 = 62; + /// Locate an object by its stable identity from the handle's authority + /// root: one [`FsSearchHitWire`] at `window_offset` whose `path` is the + /// shortest route from that root to the object, its own name last. The + /// identity rides `handle2`; the reply is `List`'s with a `count` of one. + /// An object the root does not reach is `NOT_FOUND`, exactly as one that + /// does not exist: the verb names nothing outside the caller's view. + /// Observation data only: a caller opens the object by its route. + pub const OP_LOCATE: u8 = 63; + + /// Maintain the designated pack region (`res/PACK_STREAMING.md` §7): + /// `kind` names the action (`MAINTAIN_*`); a forced refresh carries its + /// first bucket in `handle` and its bucket count in `handle2`. Every + /// action answers with the region's [`PackMaintenanceWire`] report and + /// up to its `entries` [`PackBucketWire`] records, landed in the data + /// window at `window_offset`; the reply carries the bytes landed in + /// `count`. The row runs the job itself, between requests: a scan or a + /// refresh replies at once, and the client asks + /// [`MAINTAIN_STATUS`](Self::MAINTAIN_STATUS) until the phase is idle. + pub const OP_STOR_MAINTAIN: u8 = 64; + /// Whether `handle` routes to the very object one returned view endpoint + /// is bound to: a launcher's cwd anchor, carrying only REVOKE, so it + /// identifies the view without conveying it. Answered `Done` when it + /// does; `CONFLICT` when the view is bound to another object, as an + /// expectation that no longer holds; `ACCESS_LOST` when the view's own + /// route no longer validates. The view must belong to the caller's task. + /// `window_offset` is return-post correlation. + pub const OP_MATCH_VIEW: u8 = 65; + pub const OP_MAX: u8 = Self::OP_MATCH_VIEW; + + /// [`OP_STOR_MAINTAIN`](Self::OP_STOR_MAINTAIN): report only. + pub const MAINTAIN_STATUS: u8 = 0; + /// Probe every bucket's read latency and judge which are stale. + pub const MAINTAIN_SCAN: u8 = 1; + /// Rewrite, content unchanged, every bucket the last scan judged stale. + pub const MAINTAIN_REFRESH_STALE: u8 = 2; + /// Rewrite buckets `handle..handle + handle2`, judged or not. + pub const MAINTAIN_REFRESH_RANGE: u8 = 3; + /// Stop the running job once the device returns what it holds. + pub const MAINTAIN_CANCEL: u8 = 4; /// [`kind`](Self::kind): a file. pub const KIND_FILE: u8 = 1; @@ -5003,6 +5266,133 @@ impl FsSystemStatusWire { pub const LIFECYCLE_CURRENT: u16 = 7; } +/// The pack region's maintenance report (`res/PACK_STREAMING.md` §7): +/// what its scan measured, what its refresh rewrote, and where the job is. +/// Lands in the data window ahead of `entries` [`PackBucketWire`] records, +/// one per bucket in region order. +/// +/// **Aggregates of one region.** The pack's buckets are block ranges; no +/// name, file or session rides here, so any attached session may ask. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct PackMaintenanceWire { + /// `PHASE_*`. + pub phase: u8, + /// How the last job ended (`OUTCOME_*`). + pub outcome: u8, + /// With [`OUTCOME_FAULTED`](Self::OUTCOME_FAULTED): the device error, + /// as `kore_driver::block::BlockError`'s discriminant plus one. + pub fault: u8, + /// `FLAG_*`. + pub flags: u8, + /// Buckets the region is divided into. + pub buckets: U16, + /// [`PackBucketWire`] records following this one. + pub entries: U16, + /// Probes a scan times per bucket. + pub probes: U16, + /// A bucket is stale at this multiple of the 25th-percentile median. + pub stale_ratio: U16, + /// Commands of the job's on the device now. + pub in_flight: U16, + /// Passes a scan makes over the same probe pages; a bucket's estimate + /// is its fastest pass. + pub passes: U16, + /// Bytes in one bucket (the last may be short). + pub bucket_bytes: U64, + /// Bytes in the region. + pub region_bytes: U64, + /// Bytes one refresh command moves. + pub chunk_bytes: U32, + /// Scans completed since boot. + pub scans: U32, + /// Buckets probed in the running pass, or by the last scan once + /// complete. + pub scanned: U16, + /// Of those, buckets whose probes read mapped (non-zero) data. + pub mapped: U16, + /// Buckets judged stale. + pub stale: U16, + /// Buckets the last (or running) refresh set out to rewrite. + pub planned: U16, + /// Of those, rewritten and flushed. + pub done: U16, + /// Asked for but skipped: rewritten earlier this boot. + pub skipped: U16, + /// Buckets rewritten since boot. + pub lifetime: U16, + /// The running scan pass, from one; `passes` once the scan is complete. + pub pass: U16, + /// Probes the device failed. + pub failed_probes: U32, + /// Percentiles of the mapped buckets' median probe latency, ns. + pub p25_ns: U32, + pub p50_ns: U32, + pub p90_ns: U32, + /// Bytes rewritten and verified since boot. + pub bytes: U64, + /// Chunks left alone because they read as zeros. + pub zero_chunks: U32, + /// Reads the device failed; their chunks were left alone. + pub read_errors: U32, + /// Chunks whose two reads never agreed; left alone. + pub unstable_chunks: U32, + /// Writes repeated after a refusal or a disagreeing check. + pub write_retries: U32, + /// Checks that disagreed with the bytes just written. + pub mismatches: U32, +} + +impl PackMaintenanceWire { + /// Bytes the record occupies in the data window. + pub const BYTES: usize = 100; + pub const PHASE_IDLE: u8 = 0; + pub const PHASE_SCANNING: u8 = 1; + pub const PHASE_REFRESHING: u8 = 2; + /// Cancelled or faulted, waiting for the device to return its commands. + pub const PHASE_DRAINING: u8 = 3; + pub const OUTCOME_NONE: u8 = 0; + pub const OUTCOME_SCANNED: u8 = 1; + pub const OUTCOME_REFRESHED: u8 = 2; + pub const OUTCOME_CANCELLED: u8 = 3; + /// A write could not be made to stick; see [`fault`](Self::fault). + pub const OUTCOME_FAULTED: u8 = 4; + /// The last scan ran to its end. + pub const FLAG_SCAN_COMPLETE: u8 = 1 << 0; +} + +/// One bucket of the pack region, after a [`PackMaintenanceWire`]. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct PackBucketWire { + /// The last scan's estimate, ns: the fastest of its passes' medians of + /// mapped probes; zero unscanned. + pub median_ns: U32, + /// Probes of the last scan that read non-zero data. + pub mapped_probes: u8, + /// Probes of the last scan the device failed. + pub failed_probes: u8, + /// `FLAG_*`. + pub flags: u8, + pub reserved: u8, +} + +impl PackBucketWire { + pub const BYTES: usize = 8; + /// The last scan judged the bucket stale, and it is not rewritten yet. + pub const FLAG_STALE: u8 = 1 << 0; + /// Rewritten this boot: no refresh touches it again before the next. + pub const FLAG_REFRESHED: u8 = 1 << 1; + /// A chunk could not be read cleanly and was left as it is. + pub const FLAG_INCOMPLETE: u8 = 1 << 2; + /// Most probes read non-zero data. + pub const FLAG_MAPPED: u8 = 1 << 3; +} + /// Startup-only display owner evidence. Data only; reporting never admits sleep. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] diff --git a/native/crates/abi/src/task.rs b/native/crates/abi/src/task.rs index 83eca93..baed7a9 100644 --- a/native/crates/abi/src/task.rs +++ b/native/crates/abi/src/task.rs @@ -393,6 +393,22 @@ pub mod role { /// launcher keeps that arena mapped. Exec-synthesized, like /// [`STOR_RING`]. pub const STOR_TARGET: u16 = 46; + /// Init's one-shot **boot-settled** lane to the compositor (#1332): a + /// channel the founder mints for this lane alone. The compositor's block + /// names its receive end, and init's names the only send leaf. Init + /// sends once, when its boot work is done, and the compositor's welcome + /// holds its first frame until then, leaving the cores to boot. No other + /// task receives either end, so no one else can release the welcome + /// early. Both ends are founder-stamped chain wiring, outside every + /// manifest profile. + pub const BOOT_SETTLE: u16 = 47; + /// A non-invocable selector for the launcher's current filesystem view — + /// its cwd — moved in beside an ordinary launch whose paths the launcher + /// spelled from that cwd's route. It carries REVOKE only, like + /// [`FS_GRANT_TARGET`]: the child presents it once, so the filesystem + /// confirms the route still names the very directory the view is bound + /// to, and can neither invoke nor delegate it. + pub const FS_VIEW_ANCHOR: u16 = 48; /// First caller-assigned role — roles `>=` this are convention-defined. pub const USER_BASE: u16 = 0x8000; } @@ -446,17 +462,19 @@ pub const INTERACTIVE_CHILD_LOADER_FACES: usize = DESKTOP_BOOTSTRAP_WINDOW_SLOTS + 2 * DESKTOP_CONTROLLER_CAPACITY; /// Live immutable image-generation peers the boot catalogue can hold at once: -/// the 40-row bootstrap catalogue — the desktop's `editor` and `power-ui` -/// beside the native filesystem verbs `append`, `sync` and `pack`, the two -/// WebAssembly rows, `wasm-witness` and `wasm`, `inference` / -/// `model-data-witness`, and `shell`, the guest shell `term` opens — plus the -/// founded shell, the graphical terminal, and three extraction parsers. +/// the 45-row bootstrap catalogue — the desktop's `files`, its documents +/// `editor`, `photos`, `settings` and `power-ui`, the screenshot writer `capture`, beside +/// the native filesystem verbs `append`, `sync` and `pack`, the two +/// WebAssembly rows, `wasm-witness` and `wasm`, `jsapp`, the JS app host, +/// `inference` / `model-data-witness`, and `shell`, the guest shell `term` +/// opens — plus the founded shell, the graphical terminal, and three +/// extraction parsers. /// Exec asserts this covers its catalogue (`kore-svc-exec`), the one place /// that sees both. Named /// separately from the interactive reservation below because the two grow for /// unrelated reasons: this one when a row joins the catalogue, that one when /// the desktop admits another window or controller. -pub const BOOTSTRAP_IMAGE_PEERS: usize = 45; +pub const BOOTSTRAP_IMAGE_PEERS: usize = 50; /// Bootstrap face-supply sizing. Image and runtime grow their face tables and /// rings on demand; this reserves enough initial memory for the boot catalogue @@ -574,6 +592,22 @@ pub mod geom { /// tasks, absent for spawned threads, whose growth stays on their stride /// because their holdings die with them. pub const HEAP: u16 = 10; + /// The task's **reservation window** (GEOM_ABI v8) — address space a + /// runtime reserves for itself and commits page by page: Wasmtime's + /// guarded linear memories, each a 4 GiB range with a guard on either + /// side of which only what the guest has grown to is mapped + /// (`res/WASM_PLATFORM.md` §12). Stamping it maps nothing. The task + /// commits a carve of a supply it holds at an exact page of the window + /// ([`ViewMapWindow::Reserve`](crate::session::ViewMapWindow::Reserve)) + /// and releases a page range of it + /// ([`DERIVE_KIND_RESERVE_RELEASE`](crate::session::DERIVE_KIND_RESERVE_RELEASE)). + /// + /// The one task-level window with no cursor: placement is the caller's + /// and the page table is its only state — a commit over a mapped page + /// refuses — so there is nothing per thread to keep apart, and a spawned + /// thread is stamped the task's own window where [`HEAP`](Self::HEAP) + /// and its siblings are stamped none. + pub const RESERVE: u16 = 11; } /// One startup handle: a delegated capability tagged by [`role`] (+ an instance @@ -788,20 +822,22 @@ impl Default for BootBlock { /// task-level [`geom::DEVICE`] BAR window — stamped only for a row that claims /// a PCI function with a large BAR (the NPU's 16 MiB BAR0). v7 widens the /// table (10 → 12) and adds the task-level [`geom::HEAP`] window, leaving one -/// entry spare as v4 did. -pub const GEOM_ABI: u16 = 7; +/// entry spare as v4 did. v8 names that twelfth entry: the +/// [`geom::RESERVE`] window, the first stamped for spawned threads as well. +pub const GEOM_ABI: u16 = 8; /// Window capacity of a [`GeomBlock`] — design-bounded like the handle table. /// A task learns at most a handful of founder-mapped windows (scratch, slots, /// map cursor, heap, ring); the v4 widening added a tenth entry, now named by -/// [`geom::DEVICE`] (v5), and v7 widened to twelve for [`geom::HEAP`]. +/// [`geom::DEVICE`] (v5), v7 widened to twelve for [`geom::HEAP`], and v8 +/// spent the twelfth on [`geom::RESERVE`]. pub const MAX_GEOM_WINDOWS: usize = 12; -// Eleven founder entries are now stampable (SCRATCH, SLOTS, MAP, LINK, -// THREAD_BLOCK, APERTURE, TRANSIENT, SLOTS_EXT, SURFACES — display row only — -// DEVICE — device-claiming rows only — and HEAP); one entry is spare. A +// Twelve founder entries are now stampable (SCRATCH, SLOTS, MAP, LINK, +// THREAD_BLOCK, APERTURE, TRANSIENT, SLOTS_EXT, SURFACES, DEVICE, HEAP and +// RESERVE), and the founder stamps every one: the table is full. A // thirteenth window needs a `MAX_GEOM_WINDOWS` bump (a wire change, the // GEOM_ABI v4/v7 precedent). -const _: () = assert!(11 <= MAX_GEOM_WINDOWS); +const _: () = assert!(12 <= MAX_GEOM_WINDOWS); /// One geometry window: a `(va, len)` the founder mapped, tagged by [`geom`] /// role (+ an `index` so a task could hold several of one kind). 24 bytes, diff --git a/native/crates/app/src/lib.rs b/native/crates/app/src/lib.rs index 01a4975..eb6ead4 100644 --- a/native/crates/app/src/lib.rs +++ b/native/crates/app/src/lib.rs @@ -6,15 +6,21 @@ //! presets exactly before exec and the window controller transfer affine //! capabilities through their existing transactions. //! -//! The exact `Window`, `FilesWindow`, and `PowerWindow` bit combinations are reserved launch -//! presets. A legal superset is a different profile, not an automatically -//! compatible variant: callers must never infer window topology by testing a -//! subset of profile bits. +//! A window is declared, never inferred: every window preset carries +//! `PROFILE_WINDOW`, and a profile without it is never a window, whatever +//! facilities it names (a headless engine needs the filesystem, heap, engine +//! plan and channel space a JavaScript application's window does). Which +//! window — its face and wake topology — is the exact `Window`, +//! `FilesWindow`, `PowerWindow`, `AcceleratedWindow`, `EngineWindow` or `SettingsWindow` +//! preset the whole profile matches. A legal superset is a different +//! profile, not an automatically compatible variant: callers must never infer +//! window topology by testing a subset of profile bits, and a window profile +//! that matches no preset launches nowhere. #![no_std] use kore_abi::image::{ - PROFILE_ACCELERATOR, PROFILE_CHANNELS, PROFILE_FILESYSTEM, PROFILE_HEAP, PROFILE_SYSTEM_POWER, - profile_valid, + PROFILE_ACCELERATOR, PROFILE_CHANNELS, PROFILE_ENGINE, PROFILE_FILESYSTEM, PROFILE_HEAP, + PROFILE_SYSTEM_POWER, PROFILE_WINDOW, profile_valid, }; pub use kore_dagfs::FilesystemFace; pub use kore_window::WindowFace; @@ -93,11 +99,11 @@ impl AppBundle for FsCommand { } /// A standalone window application with heap and bounded channel-space -/// facilities. +/// facilities. Every window preset is this profile plus its own faces. pub enum Window {} impl Window { - pub const PROFILE: u32 = PROFILE_HEAP | PROFILE_CHANNELS; + pub const PROFILE: u32 = PROFILE_HEAP | PROFILE_CHANNELS | PROFILE_WINDOW; pub const WINDOW_FACE: Option = Some(WindowFace::STANDALONE); pub const FILESYSTEM_FACE: Option = None; pub const WAKE: WakeTopology = WakeTopology::Window; @@ -116,7 +122,7 @@ impl AppBundle for Window { pub enum FilesWindow {} impl FilesWindow { - pub const PROFILE: u32 = PROFILE_HEAP | PROFILE_CHANNELS | PROFILE_FILESYSTEM; + pub const PROFILE: u32 = Window::PROFILE | PROFILE_FILESYSTEM; pub const WINDOW_FACE: Option = Some(WindowFace::WITH_FILESYSTEM); pub const FILESYSTEM_FACE: Option = Some(FilesystemFace::CANONICAL); pub const WAKE: WakeTopology = WakeTopology::SharedWindowService; @@ -151,6 +157,22 @@ impl AppBundle for PowerWindow { const WAKE: WakeTopology = PowerWindow::WAKE; } +/// Settings composes storage, metered image decoding, power and a window. +pub enum SettingsWindow {} +impl SettingsWindow { + pub const PROFILE: u32 = EngineWindow::PROFILE | PROFILE_SYSTEM_POWER; + pub const WINDOW_FACE: Option = FilesWindow::WINDOW_FACE; + pub const FILESYSTEM_FACE: Option = FilesWindow::FILESYSTEM_FACE; + pub const WAKE: WakeTopology = WakeTopology::SharedWindowService; +} +impl sealed::Sealed for SettingsWindow {} +impl AppBundle for SettingsWindow { + const PROFILE: u32 = Self::PROFILE; + const WINDOW_FACE: Option = Self::WINDOW_FACE; + const FILESYSTEM_FACE: Option = Self::FILESYSTEM_FACE; + const WAKE: WakeTopology = Self::WAKE; +} + /// A window application composed with one private recyclable accelerator /// face. The service and compositor share the process executor's wake root; /// accelerator completion channels remain explicit per-client grants. @@ -172,6 +194,29 @@ impl AppBundle for AcceleratedWindow { const WAKE: WakeTopology = AcceleratedWindow::WAKE; } +/// A window application composed with the canonical filesystem face whose +/// launch hosts an execution engine: [`FilesWindow`] plus `PROFILE_ENGINE`, +/// so exec also provisions the per-launch mm face its guests' memory is +/// reserved on. The filesystem and mm faces and the compositor share the +/// process executor's wake root, as `FilesWindow`'s do. +pub enum EngineWindow {} + +impl EngineWindow { + pub const PROFILE: u32 = FilesWindow::PROFILE | PROFILE_ENGINE; + pub const WINDOW_FACE: Option = Some(WindowFace::WITH_FILESYSTEM); + pub const FILESYSTEM_FACE: Option = Some(FilesystemFace::CANONICAL); + pub const WAKE: WakeTopology = WakeTopology::SharedWindowService; +} + +impl sealed::Sealed for EngineWindow {} + +impl AppBundle for EngineWindow { + const PROFILE: u32 = EngineWindow::PROFILE; + const WINDOW_FACE: Option = EngineWindow::WINDOW_FACE; + const FILESYSTEM_FACE: Option = EngineWindow::FILESYSTEM_FACE; + const WAKE: WakeTopology = EngineWindow::WAKE; +} + /// The window-capable preset selected by an authenticated image profile. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct WindowBundle { @@ -194,7 +239,7 @@ impl WindowBundle { face: WindowFace::STANDALONE, wake: WakeTopology::SharedWindowService, }) - } else if profile == FilesWindow::PROFILE { + } else if profile == FilesWindow::PROFILE || profile == SettingsWindow::PROFILE { Some(Self { profile, face: WindowFace::WITH_FILESYSTEM, @@ -206,6 +251,12 @@ impl WindowBundle { face: WindowFace::STANDALONE, wake: WakeTopology::SharedWindowService, }) + } else if profile == EngineWindow::PROFILE { + Some(Self { + profile, + face: WindowFace::WITH_FILESYSTEM, + wake: WakeTopology::SharedWindowService, + }) } else { None } @@ -224,10 +275,12 @@ impl WindowBundle { } } -/// Small import surface for application source. It contains only the five +/// Small import surface for application source. It contains only the /// closed inert presets; implementation coordinates remain explicitly named. pub mod prelude { - pub use crate::{AcceleratedWindow, Command, FilesWindow, FsCommand, PowerWindow, Window}; + pub use crate::{ + AcceleratedWindow, Command, EngineWindow, FilesWindow, FsCommand, PowerWindow, SettingsWindow, Window, + }; } const _: () = { @@ -237,6 +290,17 @@ const _: () = { assert!(profile_valid(FilesWindow::PROFILE)); assert!(profile_valid(PowerWindow::PROFILE)); assert!(profile_valid(AcceleratedWindow::PROFILE)); + assert!(profile_valid(EngineWindow::PROFILE)); + assert!(profile_valid(SettingsWindow::PROFILE)); + // Every window preset declares the window; no command preset does. + assert!(Command::PROFILE & PROFILE_WINDOW == 0); + assert!(FsCommand::PROFILE & PROFILE_WINDOW == 0); + assert!(Window::PROFILE & PROFILE_WINDOW != 0); + assert!(FilesWindow::PROFILE & PROFILE_WINDOW != 0); + assert!(PowerWindow::PROFILE & PROFILE_WINDOW != 0); + assert!(AcceleratedWindow::PROFILE & PROFILE_WINDOW != 0); + assert!(EngineWindow::PROFILE & PROFILE_WINDOW != 0); + assert!(SettingsWindow::PROFILE & PROFILE_WINDOW != 0); assert!(WindowFace::WITH_FILESYSTEM.ring() != FilesystemFace::CANONICAL.control_ring()); assert!(WindowFace::WITH_FILESYSTEM.ring() != FilesystemFace::CANONICAL.data_window()); assert!(WindowFace::WITH_FILESYSTEM.request_kick() != FilesystemFace::CANONICAL.request_kick()); @@ -250,10 +314,13 @@ mod tests { fn presets_are_exact_and_faces_do_not_collide() { assert_eq!(Command::PROFILE, 0); assert_eq!(FsCommand::PROFILE, PROFILE_FILESYSTEM); - assert_eq!(Window::PROFILE, PROFILE_HEAP | PROFILE_CHANNELS); + assert_eq!( + Window::PROFILE, + PROFILE_HEAP | PROFILE_CHANNELS | PROFILE_WINDOW + ); assert_eq!( FilesWindow::PROFILE, - PROFILE_HEAP | PROFILE_CHANNELS | PROFILE_FILESYSTEM + PROFILE_HEAP | PROFILE_CHANNELS | PROFILE_WINDOW | PROFILE_FILESYSTEM ); assert_eq!(Window::WINDOW_FACE, Some(WindowFace::STANDALONE)); assert_eq!(FsCommand::FILESYSTEM_FACE, Some(FilesystemFace::CANONICAL)); @@ -264,9 +331,15 @@ mod tests { assert_eq!(FilesWindow::FILESYSTEM_FACE, FsCommand::FILESYSTEM_FACE); assert_eq!( AcceleratedWindow::PROFILE, - PROFILE_HEAP | PROFILE_CHANNELS | PROFILE_ACCELERATOR + PROFILE_HEAP | PROFILE_CHANNELS | PROFILE_WINDOW | PROFILE_ACCELERATOR ); assert_eq!(AcceleratedWindow::WINDOW_FACE, Some(WindowFace::STANDALONE)); + assert_eq!( + EngineWindow::PROFILE, + PROFILE_HEAP | PROFILE_CHANNELS | PROFILE_WINDOW | PROFILE_FILESYSTEM | PROFILE_ENGINE + ); + assert_eq!(EngineWindow::WINDOW_FACE, FilesWindow::WINDOW_FACE); + assert_eq!(EngineWindow::FILESYSTEM_FACE, FilesWindow::FILESYSTEM_FACE); } #[test] @@ -280,7 +353,9 @@ mod tests { assert_eq!(classified.unwrap().wake(), WakeTopology::Window); } else if profile == FilesWindow::PROFILE || profile == PowerWindow::PROFILE + || profile == SettingsWindow::PROFILE || profile == AcceleratedWindow::PROFILE + || profile == EngineWindow::PROFILE { assert_eq!( classified.unwrap().wake(), @@ -294,4 +369,34 @@ mod tests { } } } + + /// The facilities a window is built from are not a window: each preset + /// without its declaration is a headless profile, and launches as one. + /// `lua-tool-worker` is exactly the engine window's facilities — the + /// filesystem, heap, engine plan and channel space — and runs headless + /// for inference and init. + #[test] + fn a_window_preset_without_its_declaration_is_headless() { + for preset in [ + Window::PROFILE, + FilesWindow::PROFILE, + PowerWindow::PROFILE, + SettingsWindow::PROFILE, + AcceleratedWindow::PROFILE, + EngineWindow::PROFILE, + ] { + assert_eq!(WindowBundle::from_profile(preset & !PROFILE_WINDOW), None); + } + assert_eq!( + WindowBundle::from_profile( + PROFILE_FILESYSTEM | PROFILE_HEAP | PROFILE_ENGINE | PROFILE_CHANNELS + ), + None + ); + for profile in 0..=kore_abi::image::PROFILE_KNOWN { + if profile & PROFILE_WINDOW == 0 { + assert_eq!(WindowBundle::from_profile(profile), None, "{profile:#x}"); + } + } + } } diff --git a/native/crates/collections/src/slotvec.rs b/native/crates/collections/src/slotvec.rs index 290bcdb..4b0ac7c 100644 --- a/native/crates/collections/src/slotvec.rs +++ b/native/crates/collections/src/slotvec.rs @@ -55,6 +55,14 @@ unsafe impl Send for AtomicSlot unsafe impl Sync for AtomicSlotVec {} impl AtomicSlotVec { + /// Heap bytes one slot costs: its value cell, its state word and its + /// reader count. A caller budgeting an arena of `n` slots — a service + /// holding its heap to what its arenas can reach — multiplies by this + /// rather than respelling the layout. + pub const SLOT_BYTES: usize = core::mem::size_of::>>() + + core::mem::size_of::() + + core::mem::size_of::(); + /// Build an empty arena of exactly `capacity` slots, backed by `alloc`. /// /// Fallible by construction: a founded service never aborts on allocator diff --git a/native/crates/dagfs/src/bounds.rs b/native/crates/dagfs/src/bounds.rs index e375f7c..8e18f3d 100644 --- a/native/crates/dagfs/src/bounds.rs +++ b/native/crates/dagfs/src/bounds.rs @@ -14,19 +14,20 @@ //! | Constant | Value | Dominant cost | //! |---|---|---| //! | [`MAX_OBJECTS`] | 512 | 512 × ~48 B ≈ 24 KiB | -//! | [`MAX_DIRECTORIES`] × [`MAX_DIR_ENTRIES`] | 64 × 128 | ≈ 328 KiB | +//! | live directories × [`MAX_DIR_ENTRIES`] | ≤ 512 × 128 | ≈ 7 KiB per live directory, allocated when it is created | //! | [`MAX_NAME_LEN`] | 32 | (per entry, inline) | //! | [`RAM_CHUNKS`] × [`RAM_CHUNK_BYTES`] | 64 × 256 | 16 KiB pool | -//! | `MAX_SESSIONS` (service-side) × [`MAX_HANDLES_PER_SESSION`] | 12 × 32 | ≈ 212 KiB | +//! | [`MAX_HANDLES_PER_SESSION`] + [`MAX_EDGES_PER_SESSION`] | 32 + 32 | ≈ 5.7 KiB per attached session slot, allocated when the slot first attaches | //! | [`MAX_PLACEMENTS`] | 72 | ≈ 5 KiB RAM placement table | //! | [`MAX_BLOCK_PLACEMENTS`] | 1024 | file (≤512) + durable-derived (≤512); block provider only | //! | [`MAX_BLOCK_EXTENT_NODES`] | 6147 | ≈ 192 KiB heap-backed 32-byte nodes + free heap; block provider only | -//! | [`MAX_AUTHORITY_EDGES`] | 64 | ≈ 4 KiB | //! //! Directory child lists live in their own arena rather than inside //! [`MAX_OBJECTS`] precisely so a *file* object does not reserve directory //! space: an inline `Body::Directory` would have made every object as large -//! as the largest directory. +//! as the largest directory. The arena reserves only a slot per possible +//! directory; a directory's child list is allocated when the directory is +//! created and freed when it is reclaimed, so the cost follows the tree. //! //! The volume-scaled tables are **heap reservations, not inline fields**: //! since `res/FILESYSTEM_SCALE.md` stage B the object, directory, placement, @@ -43,16 +44,15 @@ //! ## Deliberately compact presentation bounds //! //! [`MAX_NAME_LEN`] is **32**, not POSIX's common 255-byte ceiling. Names are -//! stored inline in a directory's entry slots, so the -//! component maximum multiplies by [`MAX_DIR_ENTRIES`] × [`MAX_DIRECTORIES`]: -//! at 255 bytes the directory arena alone would be ~140 KiB, more than -//! everything else combined. Thirty-two bytes covers every current native +//! stored inline in a directory's entry slots, so the component maximum +//! multiplies by [`MAX_DIR_ENTRIES`] in every live directory's table: at 255 +//! bytes one directory would be ~33 KiB. Thirty-two bytes covers every current native //! service, shell, and gate name. //! -//! [`MAX_PATH_DEPTH`] is **32**, raised from 8 on measurement. It is the -//! length of the edge proof stored in *every* open handle, so it multiplies -//! by the handle arena — session-scaled and heap-backed now — and it fixes -//! the wire spelling of a search hit: `kore_abi::svc::FS_SEARCH_PATH_BYTES` +//! [`MAX_PATH_DEPTH`] is **32**, raised from 8 on measurement. It bounds the +//! edge proof an open handle carries, which the service stores at the depth +//! actually walked, so it costs a handle only as deep as its path; and it +//! fixes the wire spelling of a search hit: `kore_abi::svc::FS_SEARCH_PATH_BYTES` //! is pinned to `MAX_PATH_DEPTH × MAX_NAME_LEN` plus separators, and the //! assertion beside the bound refuses to compile a depth the wire cannot //! spell. @@ -76,19 +76,15 @@ /// constant. pub const MAX_OBJECTS: usize = 512; -/// Raised 16 → 64. It no longer enters the durable reservation at all, since -/// `MAX_DURABLE_EDGES` is decoupled from the directory × entry product, so -/// this is now purely a service-heap cost. It must also stay comfortably -/// above [`MAX_PATH_DEPTH`]: a chain as deep as the proof allows needs one -/// directory per level, and when the two are equal the arena refuses before -/// the depth bound can. -/// -/// Maximum live directory *child lists*. Every directory object owns one, -/// so this is the ceiling on live directories — necessarily -/// `<= MAX_OBJECTS`. Exhaustion refuses with -/// [`FsError::OutOfObjects`](crate::FsError::OutOfObjects), since from -/// a client's view the thing that could not be created is the directory. -pub const MAX_DIRECTORIES: usize = 64; +/// Maximum live directory *child lists* — derived, not chosen. Every +/// directory is an object, so the object arena already bounds how many can +/// exist; a separate knob could only refuse a directory the object arena had +/// room for (it was raised 16 → 64 by hand before it was derived). Each +/// child list is allocated when its directory is created, so a high ceiling +/// costs a slot, not a table. Exhaustion refuses with +/// [`FsError::OutOfObjects`](crate::FsError::OutOfObjects), since from a +/// client's view the thing that could not be created is the directory. +pub const MAX_DIRECTORIES: usize = MAX_OBJECTS; /// Maximum containment edges beneath one directory. Exhaustion refuses with /// [`FsError::OutOfEdges`](crate::FsError::OutOfEdges). @@ -108,8 +104,9 @@ pub const MAX_NAME_LEN: usize = 32; /// so more than half of ordinary content was literally unaddressable, while /// the deepest observed path was 28 (`res/FILESYSTEM_SCALE.md`). Costs /// nothing on disk — depth is client-side traversal, never a stored fact — -/// and multiplies only the handle arena, which is session-scaled and now -/// heap-backed. It does reach the wire: a search hit spells its path into +/// and costs a handle only as deep as its own path: the service stores each +/// proof at the depth walked, so the bound is a ceiling, not a reservation. +/// It does reach the wire: a search hit spells its path into /// `kore_abi::svc::FS_SEARCH_PATH_BYTES`, pinned below. pub const MAX_PATH_DEPTH: usize = 32; @@ -139,10 +136,31 @@ pub const MAX_TASK_VIEWS: usize = 4; /// halfway down. The two bounds are related and must move together. pub const MAX_HANDLES_PER_SESSION: usize = 32; -/// Maximum live authority edges — private roots plus direct grants — across -/// all sessions. Exhaustion refuses with +/// Maximum authority edges one session holds — its root, the store views a +/// task view carries, the working directories pinned beneath it, and the +/// grants others issue to it. Exhaustion refuses with /// [`FsError::OutOfEdges`](crate::FsError::OutOfEdges). -pub const MAX_AUTHORITY_EDGES: usize = 64; +/// +/// Per session, like [`MAX_HANDLES_PER_SESSION`], and for the same reason: a +/// session's edges live beside its handles in storage of its own, so no +/// session can spend another's, and the service's footprint follows the +/// sessions actually attached rather than a machine-wide table. The edge a +/// grant creates is stored with its *holder*. The service pins the bound +/// against the most its own views can make one session hold (a task view's +/// root, three store views and a pinned directory per invocation view). +pub const MAX_EDGES_PER_SESSION: usize = 32; + +/// Maximum depth of a delegation lineage: an authority edge and the parents +/// it was attenuated from, up to the root they all descend from. A grant +/// that would sit deeper refuses with [`FsError::Range`](crate::FsError::Range). +/// +/// Every operation revalidates its edge's whole lineage, so this bounds that +/// walk, and the read guards it holds, independently of how many edges +/// exist. The deepest chain the system builds itself — session zero's root, +/// the system view, the administration root, the workspace, a home anchor, a +/// task view, a pinned directory and a command's root — is eight edges; the +/// rest is room for clients' own delegation. +pub const MAX_LINEAGE_DEPTH: usize = 32; /// Maximum directory entries returned in one `list` /// page. A larger directory is read across several calls via the returned @@ -260,10 +278,13 @@ mod tests { use super::*; /// A directory object needs a child list, so the child-list arena must - /// not claim to hold more directories than there are objects. + /// not claim to hold more directories than there are objects — nor + /// fewer, which would refuse a directory the object arena has room for. #[test] - fn directory_arena_fits_inside_object_arena() { - assert!(MAX_DIRECTORIES <= MAX_OBJECTS); + fn directory_arena_is_the_object_arena() { + assert_eq!(MAX_DIRECTORIES, MAX_OBJECTS); + // A chain as deep as the proof allows needs one directory per level. + assert!(MAX_DIRECTORIES > MAX_PATH_DEPTH); } /// The edge proof is a `heapless::Vec<_, MAX_PATH_DEPTH>`; a traversal diff --git a/native/crates/dagfs/src/capability.rs b/native/crates/dagfs/src/capability.rs index 370091b..9845bd0 100644 --- a/native/crates/dagfs/src/capability.rs +++ b/native/crates/dagfs/src/capability.rs @@ -46,6 +46,14 @@ pub struct FsViewCapability { pub const FS_GRANT_TARGET_RIGHTS: CapRights = CapRights::REVOKE; pub const FS_GRANT_TARGET_DELEGATE_RIGHTS: CapRights = CapRights::DERIVE.union(CapRights::REVOKE); +/// A cwd anchor carries, like a grant target, no invocation or delegation +/// authority. It identifies one of the launcher's own views to the service +/// ([`crate::FsRequest::MatchView`]) so a command can confirm that the route +/// it was told names the directory that view is bound to, and nothing more: +/// it cannot be invoked as a cwd, and the service matches it only against a +/// view of the presenting task's own. +pub const FS_VIEW_ANCHOR_RIGHTS: CapRights = CapRights::REVOKE; + impl FsViewCapability { pub const DELEGABLE_RIGHTS: CapRights = CapRights::ACCESS .union(CapRights::DERIVE) @@ -96,6 +104,12 @@ impl FsViewCapability { ) } + /// A REVOKE-only selector of this view ([`FS_VIEW_ANCHOR_RIGHTS`]), for a + /// command that must confirm a route still names it. + pub fn derive_anchor(&self) -> Option { + verbs::derive_owned_chan_send_with_rights(self.endpoint.slot(), FS_VIEW_ANCHOR_RIGHTS) + } + pub fn derive_binding(&self) -> Option { verbs::derive_owned_chan_send_with_rights( self.endpoint.slot(), diff --git a/native/crates/dagfs/src/client.rs b/native/crates/dagfs/src/client.rs index a493d73..151994e 100644 --- a/native/crates/dagfs/src/client.rs +++ b/native/crates/dagfs/src/client.rs @@ -30,8 +30,8 @@ use kore_abi::{ }; use kore_rt::{ svc::{ - CallError, ReturnCallError, ReturnCapability, ReturnDeriveError, ReturnDeriver, - ReturnLeaseSource, StagedReturn, StagedReturnError, SvcClient, + CallError, Observation, ReturnCallError, ReturnCapability, ReturnDeriveError, + ReturnDeriver, ReturnLeaseSource, StagedReturn, StagedReturnError, SvcClient, }, transport as rt_transport, }; @@ -44,8 +44,8 @@ use crate::{ }; pub use crate::{ - DirEntry, FS_GRANT_TARGET_DELEGATE_RIGHTS, FS_GRANT_TARGET_RIGHTS, FilesystemFace, - FsViewCapability, + DirEntry, FS_GRANT_TARGET_DELEGATE_RIGHTS, FS_GRANT_TARGET_RIGHTS, FS_VIEW_ANCHOR_RIGHTS, + FilesystemFace, FsViewCapability, }; /// Bytes written by [`FsRequest::LendVersion`] or @@ -189,6 +189,9 @@ pub struct RingFs<'w, 't, T: rt_transport::Transport> { /// The per-request transfer limit the service advertised at attach; /// the protocol floor until then. transfer_limit: u32, + /// Opt-in bounded control observations. Expiry keeps the exact request + /// outstanding; the owner must resume it or contain the entire face. + observation: Observation, } /// Failure of the production ring client, preserving the distinction between @@ -299,6 +302,7 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { return_tx, return_land, transfer_limit: crate::bounds::MAX_TRANSFER as u32, + observation: Observation::Settle, } } @@ -318,10 +322,21 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { /// Issue one ordinary data-only filesystem operation. pub async fn call(&mut self, request: &FsRequest) -> Result { - let reply = self.pair.call(request).await?; + let reply = match self.observation { + Observation::Settle => self.pair.call(request).await?, + observation => self.pair.call_exact(request, observation).await?, + }; self.finish(reply) } + /// Change ordinary control-call observation only at an idle boundary. + /// A timeout never clears correlation or permits a different request. + pub fn set_observation(&mut self, observation: Observation) -> bool { + if !self.pair.is_idle() { return false; } + self.observation = observation; + true + } + /// Whether the exclusive administration pair can publish a new request. pub fn is_idle(&self) -> bool { self.pair.is_idle() @@ -355,25 +370,18 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { &mut self, request: &FsRequest, ) -> Result { - let reply = if self.pair.is_idle() { - self.pair.call(request).await? - } else { - self.pair.resume_exact(request).await? - }; - self.finish(reply) + self.call_exact_observed(request, self.observation).await } - /// Give one exact administration request a bounded observation slice. - /// Expiry leaves its request correlated on this pair for a later pass. - async fn call_exact_bounded( + /// [`call_exact`](Self::call_exact) under an explicit observation. An + /// expired [`Observation::Slice`] leaves its request correlated on this + /// pair for a later call with the same request. + async fn call_exact_observed( &mut self, request: &FsRequest, - budget_ns: u64, + observation: Observation, ) -> Result { - let reply = match self.pair.resume_exact_bounded(request, budget_ns).await { - Err(CallError::NoOutstanding) => self.pair.call_bounded(request, budget_ns).await, - result => result, - }?; + let reply = self.pair.call_exact(request, observation).await?; self.finish(reply) } @@ -641,7 +649,10 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { budget_ns: u64, ) -> Result { match self - .call_exact_bounded(&FsRequest::SystemStatus { window_offset }, budget_ns) + .call_exact_observed( + &FsRequest::SystemStatus { window_offset }, + Observation::Slice(budget_ns), + ) .await .map_err(RingFsError::Infrastructure)? { @@ -1107,6 +1118,25 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { self.finish(reply).map_err(ReturnCallError::AfterTransfer) } + /// Whether `handle` routes to the object the returned `anchor` view is + /// bound to ([`FsRequest::MatchView`]): `Done`, or `Refused(Conflict)` + /// when that view is bound elsewhere. The anchor is consumed either way. + pub async fn match_view( + &mut self, + handle: Handle, + anchor: C, + ) -> Result> { + let request = FsRequest::MatchView { + handle, + announced: self.return_land, + }; + let reply = self + .pair + .call_with_exact_return(&request, self.return_tx, anchor) + .await?; + self.finish(reply).map_err(ReturnCallError::AfterTransfer) + } + pub async fn grant_view( &mut self, handle: Handle, @@ -1193,11 +1223,12 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { self.call(&FsRequest::TakeAdmission).await } - pub async fn take_admission_bounded( + /// Claim the published verdict as one exact lifecycle request. + pub async fn take_admission_observed( &mut self, - budget_ns: u64, + observation: Observation, ) -> Result { - self.call_exact_bounded(&FsRequest::TakeAdmission, budget_ns) + self.call_exact_observed(&FsRequest::TakeAdmission, observation) .await } @@ -1209,12 +1240,12 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { self.call_exact(&FsRequest::RetirePeer { peer }).await } - pub async fn retire_peer_bounded( + pub async fn retire_peer_observed( &mut self, peer: kore_rt::svc::PeerId, - budget_ns: u64, + observation: Observation, ) -> Result { - self.call_exact_bounded(&FsRequest::RetirePeer { peer }, budget_ns) + self.call_exact_observed(&FsRequest::RetirePeer { peer }, observation) .await } @@ -1228,12 +1259,12 @@ impl<'w, 't, T: rt_transport::Transport> RingFs<'w, 't, T> { self.call_exact(&FsRequest::ConfirmRetired { peer }).await } - pub async fn confirm_retired_bounded( + pub async fn confirm_retired_observed( &mut self, peer: kore_rt::svc::PeerId, - budget_ns: u64, + observation: Observation, ) -> Result { - self.call_exact_bounded(&FsRequest::ConfirmRetired { peer }, budget_ns) + self.call_exact_observed(&FsRequest::ConfirmRetired { peer }, observation) .await } } @@ -1419,6 +1450,40 @@ impl FsClient { } } + /// One page of an object's visible parents, landed in the data window at + /// `window_offset` as search-hit records ([`read_parent`]): the + /// directories reachable from the handle's authority root that hold an + /// edge to it, each with the shortest route from that root. + pub fn parents(&mut self, handle: Handle, cursor: u32, window_offset: u32) -> FsResult { + match self.checked(&FsRequest::Parents { + handle, + cursor, + window_offset, + })? { + FsResponse::Listed { entries, next } => Ok(ListPage { entries, next }), + _ => Err(FsError::ProviderFailure(crate::error::ProviderError::Media)), + } + } + + /// Locate the object with identity `stable` from the handle's authority + /// root, its route landed at `window_offset` as one search-hit record + /// ([`read_located`] reads it). + pub fn locate( + &mut self, + handle: Handle, + stable: crate::StableObjectId, + window_offset: u32, + ) -> FsResult { + match self.checked(&FsRequest::Locate { + handle, + stable, + window_offset, + })? { + FsResponse::Listed { entries, next } => Ok(ListPage { entries, next }), + _ => Err(FsError::ProviderFailure(crate::error::ProviderError::Media)), + } + } + /// Search only the namespace reachable through this client's native view. /// Results are written into the attached data window at `window_offset`. pub fn search( @@ -1800,6 +1865,46 @@ impl FsClient { /// Lift one authority-filtered search hit from service-written window bytes. pub fn read_search_hit(window: &[u8], window_offset: u32, index: usize) -> FsResult { + read_hit_record(window, window_offset, index, false) +} + +/// Lift the one `Locate` record from service-written window bytes: the +/// located object itself, file or directory, whose `path` is its route from +/// the handle's authority root (empty when it is that root). Structural like +/// a parent — no score, no snippet — but it names the object, so its kind +/// and length are whatever the object's are. +pub fn read_located(window: &[u8], window_offset: u32) -> FsResult { + let located = read_hit_record(window, window_offset, 0, true)?; + if located.score.get() != 0 || !located.snippet.is_empty() { + return Err(FsError::Range); + } + Ok(located) +} + +/// Lift one `Parents` record from service-written window bytes: a search-hit +/// record naming a directory, whose `path` is its route from the handle's +/// authority root — empty when the parent is that root. Stricter than a +/// hit: no score, no snippet, no length. +pub fn read_parent(window: &[u8], window_offset: u32, index: usize) -> FsResult { + let parent = read_hit_record(window, window_offset, index, true)?; + if parent.kind != ObjectKind::Directory + || parent.score.get() != 0 + || !parent.snippet.is_empty() + || parent.len != 0 + { + return Err(FsError::Range); + } + Ok(parent) +} + +/// The record both readers share; `anchor` admits the empty path (the +/// authority root itself), which a search hit never is. +fn read_hit_record( + window: &[u8], + window_offset: u32, + index: usize, + anchor: bool, +) -> FsResult { use zerocopy::FromBytes; let at = usize::try_from(window_offset) .map_err(|_| FsError::Range)? @@ -1818,12 +1923,15 @@ pub fn read_search_hit(window: &[u8], window_offset: u32, index: usize) -> FsRes return Err(FsError::Range); } let len = usize::from(wire.path_len.get()); - if len == 0 || len > wire.path.len() || wire.path[len..].iter().any(|byte| *byte != 0) { + if (len == 0 && !anchor) + || len > wire.path.len() + || wire.path[len..].iter().any(|byte| *byte != 0) + { return Err(FsError::InvalidName); } let path = core::str::from_utf8(&wire.path[..len]).map_err(|_| FsError::InvalidName)?; let mut components = heapless::Vec::new(); - for component in path.split('/') { + for component in path.split('/').filter(|_| len != 0) { components .push(Name::new(component)?) .map_err(|_| FsError::Range)?; @@ -2147,6 +2255,29 @@ mod exact_attach_tests { ); } + #[test] + fn observation_policy_changes_only_at_idle_boundaries() { + let transport = RecordingTransport::refusing(); + let (mut fs, mut serve) = ring_fs(&transport, SlotRef::from_wire(0x41)); + assert!(fs.set_observation(Observation::Slice(1_000_000))); + assert_eq!(fs.observation, Observation::Slice(1_000_000)); + // A bare host block_on has no executor clock. Exercise the policy + // boundary with a retained transport failure; runtime's exact-pair + // tests separately cover an actual expired timed observation. + assert!(fs.set_observation(Observation::Settle)); + let request = FsRequest::Close { handle: crate::Handle::from_bits(7) }; + assert_eq!(crate::block_on(fs.call_exact(&request)), Err(CallError::Transport)); + assert_eq!(take_request(&mut serve), request); + assert!(!fs.set_observation(Observation::Slice(1_000_000))); + publish_done(&mut serve); + let other = FsRequest::Close { handle: crate::Handle::from_bits(8) }; + assert_eq!(crate::block_on(fs.call_exact(&other)), Err(CallError::Outstanding)); + transport.succeed.set(true); + assert_eq!(crate::block_on(fs.call_exact(&request)), Ok(FsResponse::Done)); + assert!(fs.set_observation(Observation::Settle)); + assert!(serve.req.pop().is_none()); + } + #[test] fn status_request_identity_rejects_a_late_reply_for_another_window() { let transport = RecordingTransport::refusing(); diff --git a/native/crates/dagfs/src/error.rs b/native/crates/dagfs/src/error.rs index 041003f..e34289b 100644 --- a/native/crates/dagfs/src/error.rs +++ b/native/crates/dagfs/src/error.rs @@ -28,7 +28,9 @@ pub enum FsError { AccessDenied, /// An edge the handle's proof depends on is gone or retired. AccessLost, - /// The expected revision did not match the current file revision. + /// An expectation the caller stated no longer holds: the expected + /// revision did not match the current file revision, or a route no + /// longer names the directory a view is bound to. Conflict, /// The operation would have made the containment graph cyclic. Cycle, diff --git a/native/crates/dagfs/src/fs.rs b/native/crates/dagfs/src/fs.rs index 87b9ac4..55b4b8d 100644 --- a/native/crates/dagfs/src/fs.rs +++ b/native/crates/dagfs/src/fs.rs @@ -38,7 +38,7 @@ use kore_rt::{ }; use crate::{ - DagfsProtocol, FS_GRANT_TARGET_RIGHTS, FsRequest, FsResponse, FsViewCapability, Handle, + DagfsProtocol, FS_GRANT_TARGET_RIGHTS, FS_VIEW_ANCHOR_RIGHTS, FsRequest, FsResponse, FsViewCapability, Handle, Refusal, RingFs, bounds::MAX_HANDLES_PER_SESSION, process_face, }; @@ -66,6 +66,15 @@ impl Fs<'_, '_> { pub fn set_direct_polling(&mut self, direct: bool) { self.client.set_direct_polling(direct); } + + /// Opt into the existing exact-pair deadline policy for data-only + /// control calls. On expiry this face must not be reused for unrelated + /// work; `status_pair_idle` remains false. Authority-return operations + /// keep their own custody policy. Change only while the pair is idle. + pub fn set_observation(&mut self, observation: kore_rt::svc::Observation) -> bool { + self.client.set_observation(observation) + } + } pub struct Fs<'w, 't> { @@ -85,6 +94,10 @@ pub struct Fs<'w, 't> { /// endpoint carries REVOKE only, so it cannot be used as a cwd or further /// delegated by the command. pub(crate) grant_target: Option, + /// Optional one-shot selector for the launcher's own cwd view, delivered + /// with a line whose paths the launcher spelled from that cwd's route + /// ([`Fs::confirm_position`]). REVOKE only, like the grant target. + pub(crate) view_anchor: Option, /// This launch's storage ring, when its founding delivered one /// ([`role::STOR_RING`] and [`role::STOR_TARGET`]): set up on the first /// read that streams through it ([`crate::stor::FacadeRing`]). @@ -148,6 +161,10 @@ impl<'w, 't> Fs<'w, 't> { .bootstrap() .handle(role::FS_GRANT_TARGET, 0) .and_then(|slot| verbs::take_chan_send(slot, FS_GRANT_TARGET_RIGHTS)); + let view_anchor = rt + .bootstrap() + .handle(role::FS_VIEW_ANCHOR, 0) + .and_then(|slot| verbs::take_chan_send(slot, FS_VIEW_ANCHOR_RIGHTS)); let pair = SvcClient::new( transport, rings, @@ -202,6 +219,7 @@ impl<'w, 't> Fs<'w, 't> { _data_lease: data_lease, view_binding, grant_target, + view_anchor, ring, ring_files: 0, ring_note: "unused", diff --git a/native/crates/dagfs/src/graph.rs b/native/crates/dagfs/src/graph.rs index de24336..13f20e3 100644 --- a/native/crates/dagfs/src/graph.rs +++ b/native/crates/dagfs/src/graph.rs @@ -11,7 +11,7 @@ use kore_svc_extract::ParserId; use crate::{ DirEntry, Fs, FsError, FsRequest, FsResponse, Name, ObjectKind, ObjectSubtype, Refusal, Revision, SearchCursor, SearchHit, SearchQuery, View, WALK_MUTATE, WALK_READ, path, - read_dir_entry, read_search_hit, + read_dir_entry, read_located, read_parent, read_search_hit, }; impl<'w, 't> Fs<'w, 't> { @@ -174,6 +174,75 @@ impl<'w, 't> Fs<'w, 't> { } } + /// One page of `view`'s visible parents into the window, from `cursor`: + /// the directories reachable from its authority root that hold an edge + /// to it. Yields the record count and the cursor to resume at; read + /// each with [`Fs::parent`] before the next request. + pub async fn parents_page( + &mut self, + view: &View, + cursor: u32, + ) -> Result<(u32, Option), Refusal> { + match self + .checked(&FsRequest::Parents { + handle: view.handle(), + cursor, + window_offset: 0, + }) + .await? + { + // A resume cursor that does not advance would page forever. + FsResponse::Listed { + next: Some(next), .. + } if next <= cursor => Err(Refusal::Pair), + FsResponse::Listed { entries, next } => Ok((entries, next)), + _ => Err(Refusal::Pair), + } + } + + /// The route from this face's root to the object with identity + /// `stable`, spelled as a root-relative path into `out`: how a window + /// handed an object's identity opens it. The identity is data — the + /// route is opened through this face's own root like any other, and an + /// object that root does not reach is `NotFound`. + pub async fn locate( + &mut self, + stable: crate::StableObjectId, + out: &mut heapless::Vec, + ) -> Result<(), Refusal> { + let root = self.open_root(crate::Rights::READ).await?; + let located = self + .checked(&FsRequest::Locate { + handle: root.handle(), + stable, + window_offset: 0, + }) + .await; + self.close(root).await; + match located? { + FsResponse::Listed { entries: 1, next: None } => {} + _ => return Err(Refusal::Pair), + } + // The record names the object itself — a file has a length and is + // no directory, so this is never the parents reader. + let hit = read_located(&self.window[..], 0).map_err(Refusal::Fs)?; + out.clear(); + for (index, name) in hit.path.iter().enumerate() { + if index > 0 { + out.push(b'/').map_err(|_| Refusal::Fs(FsError::Range))?; + } + out.extend_from_slice(name.as_bytes()).map_err(|_| Refusal::Fs(FsError::Range))?; + } + Ok(()) + } + + /// The `index`th parent of the page the last [`Fs::parents_page`] + /// landed: its route from the authority root (`path`, empty for the root + /// itself) and its identity — observation data, never authority. + pub fn parent(&self, index: usize) -> Result { + read_parent(&self.window[..], 0, index).map_err(Refusal::Fs) + } + /// Visit every entry of the directory at `path`, page by page, copying /// each entry out of the window before the next request may reuse it. pub async fn list_directory(&mut self, path: &[u8], mut consume: F) -> Result<(), Refusal> diff --git a/native/crates/dagfs/src/index.rs b/native/crates/dagfs/src/index.rs index 8e1f33b..bae5404 100644 --- a/native/crates/dagfs/src/index.rs +++ b/native/crates/dagfs/src/index.rs @@ -10,7 +10,7 @@ use kore_svc_extract::{ Confidence, Fact, FactKind, FactValue, InputIdentity, NormalizedResult, ParserProvenance, - SourceRange, encode_result, + ResultEncodeError, SourceRange, WORKER_REPORT_BYTES, encode_result, registry::{PLAIN_TEXT, Registry}, }; use kore_svc_extract_worker_rust::parse_plain_text; @@ -111,13 +111,9 @@ pub async fn publish( } else { None }; - normalized_index_result(identity, bytes) + encode_batch(identity, bytes, &mut fs.window[..]) .map_err(Refusal::Fs) - .and_then(|result| { - let length = encode_result(&result, &mut fs.window[..]) - .map_err(|_| Refusal::Fs(FsError::Unsupported))?; - u32::try_from(length).map_err(|_| Refusal::Fs(FsError::Range)) - }) + .and_then(|length| u32::try_from(length).map_err(|_| Refusal::Fs(FsError::Range))) } Err(error) => Err(error), }; @@ -144,6 +140,52 @@ pub async fn publish( } } +/// The batch [`publish`] commits for the version `identity` names, encoded +/// into `window`: its length, or the typed reason the version stays +/// unindexed. Free of the pair, so the producer half of a commit can be +/// driven against the real store on a host. +/// +/// The store keeps one #476 wire batch per version, at most +/// [`WORKER_REPORT_BYTES`] — durable recovery refuses a larger extent — so +/// that is the bound the batch is encoded against, not the data window. +/// Encoded into the whole window, half a page of ordinary notes made a batch +/// the store could not keep, and `CommitDerived` refused it `Range` after +/// the write it describes had committed: the editor's save failed, and +/// `find reindex` with it. +/// +/// A batch that does not fit sheds this producer's own tokens, last first, +/// and nothing else. The text spans are the file's text, which full-text and +/// semantic search read whole, so a shed token costs those planes no hit; it +/// leaves the fact-key plane exactly as partial as the `MAX_FACTS` cap in +/// [`token_result`] already makes it. Text that does not fit on its own is +/// never cut — a partial batch would answer `CurrentOnly` with false +/// negatives — so that version stays unindexed. +pub fn encode_batch( + identity: InputIdentity, + bytes: Option<&[u8]>, + window: &mut [u8], +) -> Result { + let mut result = normalized_index_result(identity, bytes)?; + let bound = window.len().min(WORKER_REPORT_BYTES); + let batch = &mut window[..bound]; + // Each refused pass sheds one token or ends, so the passes are bounded + // by the facts the result holds. + loop { + match encode_result(&result, batch) { + Ok(length) => return Ok(length), + Err(ResultEncodeError::Short) + if result + .facts + .last() + .is_some_and(|fact| fact.kind == FactKind::Custom) => + { + result.facts.pop(); + } + Err(_) => return Err(FsError::Unsupported), + } + } +} + async fn release(fs: &mut Fs<'_, '_>, lease: VersionLease) -> Result<(), Refusal> { match fs.checked(&FsRequest::ReleaseVersion { lease }).await? { FsResponse::Done => Ok(()), @@ -355,4 +397,98 @@ mod tests { ); } } + + /// Twenty short lines, each word new: more tokens than one batch keeps. + fn notes() -> std::string::String { + (0..20) + .map(|line| { + std::format!( + "minute{line} reviewed storage{line} batches{line} answer{line} quickly{line}\n" + ) + }) + .collect() + } + + fn custom_keys(result: &NormalizedResult) -> std::vec::Vec<&str> { + result + .facts + .iter() + .filter(|fact| fact.kind == FactKind::Custom) + .map(|fact| fact.key.as_str()) + .collect() + } + + /// The batch the store refused `Range` once it reached `CommitDerived`: + /// encoded whole it is larger than the one batch the store keeps, so it + /// sheds tokens, last first, until it fits. Every line of text and both + /// parser facts survive, so full-text search still reads the whole file. + #[test] + fn an_ordinary_page_sheds_trailing_tokens_to_fit_the_stores_batch() { + let text = notes(); + let identity = InputIdentity::of(7, 1, text.as_bytes()); + let whole = token_result(identity, text.as_bytes()).unwrap(); + let mut unbounded = [0u8; 4 * WORKER_REPORT_BYTES]; + assert!( + encode_result(&whole, &mut unbounded).unwrap() > WORKER_REPORT_BYTES, + "the fixture must not fit whole, or it proves nothing" + ); + + let mut window = [0u8; 4 * WORKER_REPORT_BYTES]; + let length = encode_batch(identity, Some(text.as_bytes()), &mut window).unwrap(); + assert!(length <= WORKER_REPORT_BYTES); + let batch = decode_result(&window[..length]).unwrap(); + Registry::validate(&batch, identity, 0).unwrap(); + assert_eq!(batch.text, whole.text, "no text is shed"); + assert_eq!( + batch + .facts + .iter() + .filter(|fact| fact.kind != FactKind::Custom) + .count(), + whole + .facts + .iter() + .filter(|fact| fact.kind != FactKind::Custom) + .count(), + "the parser's facts are never shed" + ); + let (kept, all) = (custom_keys(&batch), custom_keys(&whole)); + assert!(!kept.is_empty() && kept.len() < all.len()); + assert_eq!(kept, all[..kept.len()], "the earliest tokens are kept"); + } + + /// A batch that fits is the producer's whole result — nothing is shed. + #[test] + fn a_batch_that_fits_is_published_whole() { + let bytes = b"Test test another"; + let identity = InputIdentity::of(7, 1, bytes); + let mut window = [0u8; WORKER_REPORT_BYTES]; + let length = encode_batch(identity, Some(bytes), &mut window).unwrap(); + assert_eq!( + decode_result(&window[..length]).unwrap(), + token_result(identity, bytes).unwrap() + ); + } + + /// Text alone past the batch is not cut down to fit: a partial batch + /// would answer `CurrentOnly` with false negatives, so the version stays + /// unindexed and search says so. + #[test] + fn text_too_long_for_one_batch_stays_unindexed() { + // Thirty-one 128-byte lines: under one transfer and the parser's span + // and line limits, so only the batch bound can refuse it. + let line = [b'a'; 127]; + let mut text = std::vec::Vec::new(); + for _ in 0..31 { + text.extend_from_slice(&line); + text.push(b'\n'); + } + assert!(text.len() <= MAX_TRANSFER); + let identity = InputIdentity::of(7, 1, &text); + let mut window = [0u8; 4 * WORKER_REPORT_BYTES]; + assert_eq!( + encode_batch(identity, Some(&text), &mut window), + Err(FsError::Unsupported) + ); + } } diff --git a/native/crates/dagfs/src/io.rs b/native/crates/dagfs/src/io.rs index e9db6b3..875445b 100644 --- a/native/crates/dagfs/src/io.rs +++ b/native/crates/dagfs/src/io.rs @@ -348,6 +348,32 @@ impl<'w, 't> Fs<'w, 't> { path: &[u8], bytes: &[u8], ) -> Result { + let indexed = self.replace(path, bytes, true).await?; + Ok(match indexed { + Some(crate::index::Indexed::Published) | None => SaveOutcome::Indexed, + Some(crate::index::Indexed::Incomplete) => SaveOutcome::IndexIncomplete, + }) + } + + /// [`Self::replace_file`] for bytes no index parser reads — a photo, an + /// archive: the same one-transaction replacement and the same single + /// commit, without the plain-text index batch. That batch would only be + /// refused as `Unsupported` after reading the whole file, and report a + /// save that landed as an incomplete one. + #[cfg(feature = "index")] + pub async fn replace_binary(&mut self, path: &[u8], bytes: &[u8]) -> Result<(), Refusal> { + self.replace(path, bytes, false).await.map(|_| ()) + } + + /// The replacement both saves share; `index` publishes the plain-text + /// batch inside the transaction and answers its verdict. + #[cfg(feature = "index")] + async fn replace( + &mut self, + path: &[u8], + bytes: &[u8], + index: bool, + ) -> Result, Refusal> { let file = match self.open(path, WALK_WRITE).await { Ok(file) => file, Err(Refusal::Fs(FsError::NotFound)) => { @@ -368,7 +394,8 @@ impl<'w, 't> Fs<'w, 't> { let held = self.hold_commits(true).await.is_ok(); let written = self.write_bytes(&file, bytes).await; let outcome = match written { - Ok(()) => crate::index::classify(crate::index::publish(self, &file, Some(bytes)).await), + Ok(()) if index => crate::index::classify(crate::index::publish(self, &file, Some(bytes)).await).map(Some), + Ok(()) => Ok(None), Err(error) => Err(error), }; let released = if held { @@ -382,10 +409,7 @@ impl<'w, 't> Fs<'w, 't> { // commit was refused is not saved; say so rather than report the // index verdict of an unpublished tree. released?; - Ok(match indexed { - crate::index::Indexed::Published => SaveOutcome::Indexed, - crate::index::Indexed::Incomplete => SaveOutcome::IndexIncomplete, - }) + Ok(indexed) } /// Ask the provider to make `file`'s committed state durable. diff --git a/native/crates/dagfs/src/lib.rs b/native/crates/dagfs/src/lib.rs index bc60e7b..4800948 100644 --- a/native/crates/dagfs/src/lib.rs +++ b/native/crates/dagfs/src/lib.rs @@ -70,14 +70,17 @@ mod walk; pub mod wire; #[cfg(feature = "runtime")] -pub use capability::{FS_GRANT_TARGET_DELEGATE_RIGHTS, FS_GRANT_TARGET_RIGHTS, FsViewCapability}; +pub use capability::{ + FS_GRANT_TARGET_DELEGATE_RIGHTS, FS_GRANT_TARGET_RIGHTS, FS_VIEW_ANCHOR_RIGHTS, FsViewCapability, +}; #[cfg(feature = "runtime")] pub use client::{ FilesystemFace as ClientFace, FsClient, GrantEntry, IdentifyVersionFailure, LentVersion, Opened, RING_TARGET_RIGHTS, PendingVersion, ReleasePendingVersionFailure, ReleaseVersionFailure, RingFs, RingFsError, RingReleasePendingVersionFailure, RingReleaseVersionFailure, SearchResultPage, Stat, Transport, VERSION_IDENTITY_BYTES, - process_face, read_dir_entry, read_grant_entry, read_search_hit, read_system_status, + process_face, read_dir_entry, read_grant_entry, read_located, read_parent, read_search_hit, + read_system_status, read_version_identity, }; pub use entry::DirEntry; @@ -91,8 +94,8 @@ pub use io::SaveOutcome; pub use name::Name; pub use object::{ANY_REVISION, ObjectId, ObjectKind, ObjectSubtype, Revision, StableObjectId}; #[cfg(feature = "runtime")] -pub use protocol::{DagfsProtocol, FsRequest, FsResponse}; -pub use refusal::{Refusal, Stop, describe}; +pub use protocol::{DagfsProtocol, FsRequest, FsResponse, MaintainAction}; +pub use refusal::{Refusal, Stop, Unconfirmed, describe}; #[cfg(feature = "runtime")] pub use search::{ FreshnessPolicy, MAX_SEARCH_PAGE, MAX_SEARCH_SCAN, MAX_SEARCH_SNIPPET, MAX_SEARCH_WORK, diff --git a/native/crates/dagfs/src/path.rs b/native/crates/dagfs/src/path.rs index dce3b92..35236dc 100644 --- a/native/crates/dagfs/src/path.rs +++ b/native/crates/dagfs/src/path.rs @@ -3,8 +3,15 @@ //! The service resolves one validated [`crate::Name`] at a time. A path is //! therefore presentation-side iteration, never ambient authority and never //! a string interpreted by the filesystem service. +//! +//! Two dot spellings, two treatments. `.` names the directory it stands in, +//! so it adds no step, and no edge can carry it ([`crate::Name`] refuses +//! it): [`components`] skips it, which can only turn a refused path into the +//! one it plainly meant. `..` is kept: the containment DAG stores no parent +//! edges, so a walk cannot honour it and name validation refuses it. Only +//! [`resolve`] folds `..`, against a position the caller tracks. -/// The components of a path, empty segments skipped. +/// The components of a path, empty and `.` segments skipped. #[derive(Clone, Copy, Debug)] pub struct Components<'a> { rest: &'a [u8], @@ -20,20 +27,24 @@ impl<'a> Iterator for Components<'a> { type Item = &'a [u8]; fn next(&mut self) -> Option<&'a [u8]> { - while let [b'/', tail @ ..] = self.rest { - self.rest = tail; - } - if self.rest.is_empty() { - return None; + loop { + while let [b'/', tail @ ..] = self.rest { + self.rest = tail; + } + if self.rest.is_empty() { + return None; + } + let end = self + .rest + .iter() + .position(|byte| *byte == b'/') + .unwrap_or(self.rest.len()); + let component = &self.rest[..end]; + self.rest = &self.rest[end..]; + if component != b"." { + return Some(component); + } } - let end = self - .rest - .iter() - .position(|byte| *byte == b'/') - .unwrap_or(self.rest.len()); - let component = &self.rest[..end]; - self.rest = &self.rest[end..]; - Some(component) } } @@ -71,6 +82,71 @@ pub fn depth(path: &[u8]) -> usize { components(path).count() } +/// Why a path has no canonical root-relative form. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Unresolved { + /// A `..` would climb past the view root, which has no parent. + AboveRoot, + /// The canonical form does not fit the caller's buffer. + TooLong, +} + +/// Whether `target` reaches outside the subtree of the position it is +/// resolved against: it is rooted (`/…`) or climbs (a `..` component). +/// Only such a target needs [`resolve`] and the view root to be named. +pub fn leaves_position(target: &[u8]) -> bool { + target.first() == Some(&b'/') || target.split(|byte| *byte == b'/').any(|part| part == b"..") +} + +/// Fold `target` onto `position` — both paths within one view — into the +/// canonical path from that view's root, written to `out`; returns its +/// length. +/// +/// A rooted `target` (`/…`) starts at the view root rather than at +/// `position`. `.` and empty components are inert, `..` pops one component, +/// and climbing past the root is refused rather than clamped: the root has +/// no parent, and pretending otherwise would silently name a different +/// object. The result has no empty, `.` or `..` component and no leading or +/// trailing `/`; the root itself is the empty path. +pub fn resolve(position: &[u8], target: &[u8], out: &mut [u8]) -> Result { + let mut len = 0; + if target.first() != Some(&b'/') { + len = fold(out, len, position)?; + } + fold(out, len, target) +} + +/// Append `path`'s components to the canonical prefix `out[..len]`. +fn fold(out: &mut [u8], mut len: usize, path: &[u8]) -> Result { + for component in path.split(|byte| *byte == b'/') { + match component { + b"" | b"." => {} + b".." => { + if len == 0 { + return Err(Unresolved::AboveRoot); + } + len = out[..len] + .iter() + .rposition(|byte| *byte == b'/') + .unwrap_or(0); + } + name => { + let sep = usize::from(len > 0); + let end = len + sep + name.len(); + if end > out.len() { + return Err(Unresolved::TooLong); + } + if sep == 1 { + out[len] = b'/'; + } + out[len + sep..end].copy_from_slice(name); + len = end; + } + } + } + Ok(len) +} + #[cfg(test)] mod tests { use super::*; @@ -93,10 +169,76 @@ mod tests { } #[test] - fn dot_components_remain_for_name_validation() { + fn a_dot_adds_no_step_and_a_parent_remains_for_name_validation() { + assert_eq!(split("./a/./b/."), ["a", "b"]); + assert!(split(".").is_empty()); + assert!(split("./").is_empty()); + // `.` names where it stands; no edge can carry it, so skipping it + // never changes which object a path names. + assert!(crate::Name::from_bytes(b".").is_err()); + // `..` stays for the walk to refuse: the DAG stores no parent. assert_eq!(split("a/../b"), ["a", "..", "b"]); assert!(crate::Name::from_bytes(b"..").is_err()); - assert!(crate::Name::from_bytes(b".").is_err()); + // A name that merely starts with a dot is an ordinary name. + assert_eq!(split("./.config/..x"), [".config", "..x"]); + } + + #[test] + fn a_dotted_prefix_is_the_same_directory() { + assert_eq!( + split_parent(b"./file"), + Some((b"./".as_slice(), b"file".as_slice())) + ); + assert!(same_directory(b"./", b"")); + assert!(same_directory(b"a/./b", b"a/b")); + assert_eq!(depth(b"./a/."), 1); + } + + fn resolved(position: &str, target: &str) -> Result { + let mut out = [0u8; 64]; + let len = resolve(position.as_bytes(), target.as_bytes(), &mut out)?; + Ok(std::string::String::from_utf8(out[..len].to_vec()).unwrap()) + } + + #[test] + fn resolution_folds_onto_the_position_or_the_root() { + assert_eq!(resolved("", "a/b").unwrap(), "a/b"); + assert_eq!(resolved("docs", "a").unwrap(), "docs/a"); + assert_eq!(resolved("docs/a", "..").unwrap(), "docs"); + assert_eq!(resolved("docs/a", "../../x").unwrap(), "x"); + assert_eq!(resolved("docs/a", "./b/../c/").unwrap(), "docs/a/c"); + // A rooted target ignores the position. + assert_eq!(resolved("docs/a", "/x/y").unwrap(), "x/y"); + assert_eq!(resolved("docs/a", "/").unwrap(), ""); + assert_eq!(resolved("docs", "").unwrap(), "docs"); + // `..` at the root is refused, never clamped. + assert_eq!(resolved("", ".."), Err(Unresolved::AboveRoot)); + assert_eq!(resolved("docs", "/.."), Err(Unresolved::AboveRoot)); + assert_eq!(resolved("docs", "../.."), Err(Unresolved::AboveRoot)); + // Names that only look like dots are names. + assert_eq!(resolved("", ".config/..x").unwrap(), ".config/..x"); + } + + #[test] + fn resolution_is_bounded_by_the_buffer() { + let mut out = [0u8; 5]; + assert_eq!(resolve(b"", b"abcde", &mut out), Ok(5)); + assert_eq!(resolve(b"", b"abc/de", &mut out), Err(Unresolved::TooLong)); + // Popping can bring a long walk back within bounds. + assert_eq!(resolve(b"", b"ab/../cde", &mut out), Ok(3)); + assert_eq!(&out[..3], b"cde"); + } + + #[test] + fn only_a_rooted_or_climbing_target_leaves_its_position() { + assert!(leaves_position(b"/")); + assert!(leaves_position(b"/a")); + assert!(leaves_position(b"..")); + assert!(leaves_position(b"a/../b")); + assert!(!leaves_position(b"")); + assert!(!leaves_position(b"a/b")); + assert!(!leaves_position(b"./a")); + assert!(!leaves_position(b"..x/a")); } #[test] diff --git a/native/crates/dagfs/src/protocol.rs b/native/crates/dagfs/src/protocol.rs index 1138465..abfbe60 100644 --- a/native/crates/dagfs/src/protocol.rs +++ b/native/crates/dagfs/src/protocol.rs @@ -63,6 +63,24 @@ pub enum FsRequest { cursor: u32, window_offset: u32, }, + /// One page of an object's visible parents — the directories reachable + /// from the handle's authority root that hold an edge to it, each with + /// the shortest route from that root — landed in the data window as + /// search-hit records. Answered as [`FsResponse::Listed`]. + Parents { + handle: Handle, + cursor: u32, + window_offset: u32, + }, + /// The route from the handle's authority root to the object with this + /// stable identity, landed in the data window as one search-hit record. + /// Answered as [`FsResponse::Listed`] with one entry; an object outside + /// the root's reach is `NotFound`, as one that does not exist. + Locate { + handle: Handle, + stable: crate::StableObjectId, + window_offset: u32, + }, Create { parent: Handle, name: Name, @@ -283,6 +301,15 @@ pub enum FsRequest { rights: Rights, announced: kore_abi::session::SlotRef, }, + /// Whether `handle` routes to the object the returned view endpoint is + /// bound to — a launcher's cwd anchor ([`crate::FS_VIEW_ANCHOR_RIGHTS`]), + /// which identifies one of this task's views without conveying it. + /// `Done` when it does; `Conflict` when that view is bound elsewhere. + /// `announced` is return-post correlation only. + MatchView { + handle: Handle, + announced: kore_abi::session::SlotRef, + }, /// Land one aggregate diagnostic snapshot ([`FsSystemStatusWire`]) in /// the caller's data window: the serving frontend's load beside the /// storage backend's posture. Any attached session may ask — the record @@ -360,6 +387,32 @@ pub enum FsRequest { /// Refused `Busy` while commands are still in flight — the ring stops /// taking submissions at the first ask; ask again. StorDetach, + /// Ask the pack region's maintenance job for `action` + /// (`res/PACK_STREAMING.md` §7), and land its report — a + /// `PackMaintenanceWire` and the region's `PackBucketWire` records — in + /// the data window at `window_offset`. A scan or refresh starts and + /// replies at once; the row runs it between requests. + StorMaintain { + action: MaintainAction, + window_offset: u32, + }, +} + +/// What a [`FsRequest::StorMaintain`] asks of the pack region's +/// maintenance job. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MaintainAction { + /// Report only. + Status, + /// Time every bucket's reads and judge which are stale. + Scan, + /// Rewrite, content unchanged, every bucket the last scan judged stale. + RefreshStale, + /// Rewrite buckets `first..first + count`, judged or not — still at + /// most once per bucket per boot. + RefreshRange { first: u32, count: u32 }, + /// Stop the running job once the device returns what it holds. + Cancel, } impl FsRequest { @@ -373,6 +426,8 @@ impl FsRequest { | FsRequest::WriteReplacement { .. } | FsRequest::Append { .. } | FsRequest::List { .. } + | FsRequest::Parents { .. } + | FsRequest::Locate { .. } | FsRequest::ListGrants { .. } | FsRequest::LendVersion { .. } | FsRequest::LendExecutable { .. } @@ -380,6 +435,7 @@ impl FsRequest { | FsRequest::CommitDerived { .. } | FsRequest::Search { .. } | FsRequest::SystemStatus { .. } + | FsRequest::StorMaintain { .. } ) } } @@ -414,8 +470,9 @@ pub enum FsResponse { links: u32, grants: u32, }, - /// `entries` directory entries were written to the data window; `next` is - /// the cursor to resume at, or `None` when the listing is complete. + /// `entries` records were written to the data window — directory entries + /// for `List`, parent records for `Parents`; `next` is the cursor to + /// resume at, or `None` when the listing is complete. Listed { entries: u32, next: Option, @@ -536,6 +593,11 @@ pub enum FsResponse { /// discriminant plus one; zero for none. last_stage_error: u8, }, + /// The pack region's maintenance report landed in the data window: + /// `len` bytes, a `PackMaintenanceWire` and its bucket records. + StorMaintained { + len: u32, + }, /// The operation was refused. The engine's vocabulary crosses intact. Refused(FsError), } diff --git a/native/crates/dagfs/src/refusal.rs b/native/crates/dagfs/src/refusal.rs index 6e43ff5..534851d 100644 --- a/native/crates/dagfs/src/refusal.rs +++ b/native/crates/dagfs/src/refusal.rs @@ -38,6 +38,22 @@ pub enum Refusal { Usage(&'static [u8]), } +/// Why a launch's position was not confirmed ([`crate::Fs::confirm_position`]). +/// +/// Its own vocabulary rather than a [`Refusal`] arm: confirming the route a +/// launcher spelled a line through is a launch-protocol step, taken once +/// before any verb runs, and no filesystem caller that never confirms one +/// (a WASI host, a script's file library) has a case to handle for it. +#[derive(Debug)] +pub enum Unconfirmed { + /// The route no longer names the launcher's current directory: that + /// directory was renamed or moved after it was entered. Nothing ran. + Moved, + /// The service refused, the pair failed, or a position arrived without + /// the anchor that alone gives it meaning. + Refused(Refusal), +} + impl From for Stop { fn from(refusal: Refusal) -> Self { Self::Refused(refusal) diff --git a/native/crates/dagfs/src/stor.rs b/native/crates/dagfs/src/stor.rs index e35b191..ed21f53 100644 --- a/native/crates/dagfs/src/stor.rs +++ b/native/crates/dagfs/src/stor.rs @@ -31,7 +31,8 @@ use kore_rt::svc::{ReturnCapability, ReturnLeaseSource}; use kore_rt::transport as rt_transport; use crate::client::{LentVersion, RingFs, RingFsError}; -use crate::{Fs, FsError, FsRequest, FsResponse, ProviderError}; +use crate::{Fs, FsError, FsRequest, FsResponse, MaintainAction, ProviderError}; +use kore_abi::svc::{PackBucketWire, PackMaintenanceWire}; pub use kore_abi::stor::{Class, GrantId, StorStatus}; /// What every ring call answers when it does not succeed. @@ -920,6 +921,26 @@ impl RingFs<'_, '_, T> { } } + /// Ask the pack region's maintenance job for `action`; its report lands + /// in the data window at `window_offset`. Answers the bytes landed — + /// lift them with [`read_pack_report`]. + pub async fn stor_maintain( + &mut self, + action: MaintainAction, + window_offset: u32, + ) -> Result { + match self + .stor_call(&FsRequest::StorMaintain { + action, + window_offset, + }) + .await? + { + FsResponse::StorMaintained { len } => Ok(len), + _ => Err(Refusal::Pair), + } + } + /// Send the notice a parked row is owed, unconditionally — a lost-wake /// probe: work that completes only after this was never seen. pub async fn stor_kick(&mut self) -> Result<(), Refusal> { @@ -1056,6 +1077,8 @@ impl Fs<'_, '_> { /// ring grant ([`stor_grant_version`](Self::stor_grant_version)) and /// [`read_version`](Self::read_version) read, unchanged by later writes, /// until [`release_version`](Self::release_version). + /// Keep `file` open until the lease is released: pinning the bytes does + /// not replace the view's authority, which grant/read revalidate. pub async fn lend_version(&mut self, file: &crate::View) -> Result { let pending = match self.client.lend_version(file.handle(), 0).await { Ok(pending) => pending, @@ -1175,6 +1198,46 @@ impl Fs<'_, '_> { pub async fn stor_inspect(&mut self) -> Result { self.client.stor_inspect().await } + + /// See [`RingFs::stor_maintain`]: the report, lifted from the start of + /// the data window. Valid until the next request that uses the window. + pub async fn stor_maintain(&mut self, action: MaintainAction) -> Result, Refusal> { + let len = self.client.stor_maintain(action, 0).await?; + read_pack_report(self.window(), 0, len).map_err(Refusal::Fs) + } +} + +/// The pack region's maintenance report as a client reads it: the summary +/// and one record per bucket, in region order. +pub struct PackReport<'w> { + pub summary: PackMaintenanceWire, + pub buckets: &'w [PackBucketWire], +} + +/// Lift the `len` bytes a [`FsRequest::StorMaintain`] landed at +/// `window_offset`: a [`PackMaintenanceWire`] and exactly its `entries` +/// bucket records, each field inside its vocabulary. +pub fn read_pack_report(window: &[u8], window_offset: u32, len: u32) -> Result, FsError> { + use zerocopy::FromBytes; + let at = usize::try_from(window_offset).map_err(|_| FsError::Range)?; + let end = at + .checked_add(usize::try_from(len).map_err(|_| FsError::Range)?) + .ok_or(FsError::Range)?; + let bytes = window.get(at..end).ok_or(FsError::Range)?; + let (head, rest) = bytes + .split_at_checked(PackMaintenanceWire::BYTES) + .ok_or(FsError::Range)?; + let summary = PackMaintenanceWire::read_from_bytes(head).map_err(|_| FsError::Range)?; + let entries = usize::from(summary.entries.get()); + if rest.len() != entries * PackBucketWire::BYTES + || entries > usize::from(summary.buckets.get()) + || summary.phase > PackMaintenanceWire::PHASE_DRAINING + || summary.outcome > PackMaintenanceWire::OUTCOME_FAULTED + { + return Err(FsError::Range); + } + let buckets = <[PackBucketWire]>::ref_from_bytes(rest).map_err(|_| FsError::Range)?; + Ok(PackReport { summary, buckets }) } #[cfg(test)] diff --git a/native/crates/dagfs/src/text.rs b/native/crates/dagfs/src/text.rs index 40b2e3f..49ac01e 100644 --- a/native/crates/dagfs/src/text.rs +++ b/native/crates/dagfs/src/text.rs @@ -5,7 +5,10 @@ /// A whitespace splitter over one completed command line. /// /// Borrowed, never copied: a command line is a `&[u8]` the caller already -/// holds bounded, so there is nothing here to size. +/// holds bounded, so there is nothing here to size. Copying the splitter +/// copies only its position, so a verb can look ahead (count its operands) +/// without consuming them. +#[derive(Clone, Copy, Debug)] pub struct Args<'a> { rest: &'a [u8], } diff --git a/native/crates/dagfs/src/view.rs b/native/crates/dagfs/src/view.rs index 8833dd4..70fb60b 100644 --- a/native/crates/dagfs/src/view.rs +++ b/native/crates/dagfs/src/view.rs @@ -10,7 +10,8 @@ use kore_rt::svc::ReturnCallError; use crate::{ - Fs, FsError, FsRequest, FsResponse, GrantEntry, Name, Refusal, Rights, read_grant_entry, + Fs, FsError, FsRequest, FsResponse, GrantEntry, Name, Refusal, Rights, Unconfirmed, + read_grant_entry, }; impl<'w, 't> Fs<'w, 't> { @@ -43,6 +44,57 @@ impl<'w, 't> Fs<'w, 't> { } } + /// Confirm that `position` — the route from this task view's root that + /// the launcher spelled this line's paths from — still names the + /// directory its cwd view is bound to, before the verb acts on any of + /// them. The DAG keeps no parent edges, so a launcher folds `..` and + /// rooted paths onto its remembered route; a rename of that directory or + /// of one above it keeps the cwd view working but leaves the route + /// naming whatever took the name since, and acting on it would act + /// there. The anchor makes the service the judge ([`FsRequest::MatchView`]). + /// + /// A launch with no anchor has nothing to confirm, and a position must + /// never arrive without one: unconfirmed, it would mean nothing. + pub async fn confirm_position(&mut self, position: &[u8]) -> Result<(), Unconfirmed> { + let refused = |refusal| Err(Unconfirmed::Refused(refusal)); + if self.view_anchor.is_none() { + return if position.is_empty() { + Ok(()) + } else { + refused(Refusal::Fs(FsError::AccessDenied)) + }; + } + let view = match self.open(position, Rights::READ).await { + Ok(view) => view, + // The remembered route now reaches nothing, or a file: the cwd + // left it. + Err(Refusal::Fs(FsError::NotFound | FsError::NotDirectory)) => { + return Err(Unconfirmed::Moved); + } + Err(error) => return refused(error), + }; + let Some(anchor) = self.view_anchor.take() else { + self.close(view).await; + return refused(Refusal::Pair); + }; + let matched = self.client.match_view(view.handle(), anchor).await; + self.close(view).await; + match matched { + Ok(FsResponse::Done) => Ok(()), + // Bound elsewhere, or the view's own route was retired by a move. + Ok(FsResponse::Refused(FsError::Conflict | FsError::AccessLost)) => { + Err(Unconfirmed::Moved) + } + Ok(FsResponse::Refused(error)) => refused(Refusal::Fs(error)), + Ok(_) => refused(Refusal::Pair), + Err(ReturnCallError::NotMoved { capability, .. }) => { + self.view_anchor = Some(capability); + refused(Refusal::Pair) + } + Err(ReturnCallError::AfterTransfer(_)) => refused(Refusal::Pair), + } + } + /// Share the object at `path` into the recipient task view the /// launcher selected, displayed there as `display_name` with `rights`. /// The route is opened with `SHARE` in addition to `rights`, so a route diff --git a/native/crates/dagfs/src/walk.rs b/native/crates/dagfs/src/walk.rs index 6660d60..94ea5c4 100644 --- a/native/crates/dagfs/src/walk.rs +++ b/native/crates/dagfs/src/walk.rs @@ -250,6 +250,27 @@ impl<'w, 't> Fs<'w, 't> { Ok((parent, name)) } + /// How many folders hold each step of a route from the root, written + /// into each step's count: a step held by several is a fork a path bar + /// marks. A step that does not open ends the walk, the rest keeping + /// what they had. + pub async fn links_along(&mut self, steps: &mut [(Name, u32)]) { + let Ok(mut current) = self.open_root(Rights::READ).await else { + return; + }; + for step in steps.iter_mut() { + let Ok(next) = self.open_at(¤t, step.0, Rights::READ).await else { + break; + }; + self.close(current).await; + current = next; + if let Ok(stat) = self.stat(¤t).await { + step.1 = stat.links; + } + } + self.close(current).await; + } + /// Metadata for an open view, revalidated by the service on this call. pub async fn stat(&mut self, view: &View) -> Result { match self diff --git a/native/crates/dagfs/src/wire.rs b/native/crates/dagfs/src/wire.rs index 2ba46f8..638461e 100644 --- a/native/crates/dagfs/src/wire.rs +++ b/native/crates/dagfs/src/wire.rs @@ -24,6 +24,8 @@ //! | `OpenGrant` | — | grant id | — | — | — | //! | `Close`/`Stat`/`Sync` | object | — | — | — | — | //! | `List` | directory | — | — | cursor | — | +//! | `Parents` | object | — | — | cursor | — | +//! | `Locate` | anchor | stable identity | — | — | — | //! | `Create` | parent | — | — | — | `name` | //! | `Read` | file | — | file offset | length | — | //! | `WriteAt` | file | — | file offset | length | — | @@ -53,6 +55,7 @@ use zerocopy::byteorder::little_endian::{U16, U32, U64}; use crate::{ AuthorityQuota, FsError, FsRequest, FsResponse, GrantId, Handle, Name, ObjectId, ObjectKind, ProviderError, Rights, SearchCursor, SearchQuery, SessionId, bounds::MAX_NAME_LEN, + protocol::MaintainAction, }; /// The engine's inline name bound and the wire's name field must agree, or a @@ -228,6 +231,26 @@ pub fn encode_req(req: &FsRequest) -> FsReqWire { w.length = U32::new(cursor); w.window_offset = U32::new(window_offset); } + FsRequest::Parents { + handle, + cursor, + window_offset, + } => { + w.op = FsReqWire::OP_PARENTS; + w.handle = U64::new(handle.to_bits()); + w.length = U32::new(cursor); + w.window_offset = U32::new(window_offset); + } + FsRequest::Locate { + handle, + stable, + window_offset, + } => { + w.op = FsReqWire::OP_LOCATE; + w.handle = U64::new(handle.to_bits()); + w.handle2 = U64::new(stable.get()); + w.window_offset = U32::new(window_offset); + } FsRequest::ListGrants { window_offset } => { w.op = FsReqWire::OP_LIST_GRANTS; w.window_offset = U32::new(window_offset); @@ -552,6 +575,11 @@ pub fn encode_req(req: &FsRequest) -> FsReqWire { w.rights = rights.bits(); w.window_offset = U32::new(announced.index()); } + FsRequest::MatchView { handle, announced } => { + w.op = FsReqWire::OP_MATCH_VIEW; + w.handle = U64::new(handle.to_bits()); + w.window_offset = U32::new(announced.index()); + } FsRequest::AdmitPeer { part, announced } => { w.op = FsReqWire::OP_ADMIT_PEER; w.kind = peer_part_code(part); @@ -599,6 +627,24 @@ pub fn encode_req(req: &FsRequest) -> FsReqWire { FsRequest::StorInspect => { w.op = FsReqWire::OP_STOR_INSPECT; } + FsRequest::StorMaintain { + action, + window_offset, + } => { + w.op = FsReqWire::OP_STOR_MAINTAIN; + w.window_offset = U32::new(window_offset); + w.kind = match action { + MaintainAction::Status => FsReqWire::MAINTAIN_STATUS, + MaintainAction::Scan => FsReqWire::MAINTAIN_SCAN, + MaintainAction::RefreshStale => FsReqWire::MAINTAIN_REFRESH_STALE, + MaintainAction::RefreshRange { first, count } => { + w.handle = U64::new(u64::from(first)); + w.handle2 = U64::new(u64::from(count)); + FsReqWire::MAINTAIN_REFRESH_RANGE + } + MaintainAction::Cancel => FsReqWire::MAINTAIN_CANCEL, + }; + } } w } @@ -666,6 +712,7 @@ pub fn decode_req(w: &FsReqWire) -> Option { | FsReqWire::OP_SEARCH | FsReqWire::OP_DEFER_COMMITS | FsReqWire::OP_SET_SYSTEM_SEED + | FsReqWire::OP_STOR_MAINTAIN ) && w.kind != 0 { return None; @@ -689,6 +736,40 @@ pub fn decode_req(w: &FsReqWire) -> Option { cursor: w.length.get(), window_offset: w.window_offset.get(), }, + // Exactly `List`'s operands — handle, cursor, window — and nothing + // else: an operand this verb does not read is refused, not ignored. + FsReqWire::OP_PARENTS => { + if w.handle2.get() != 0 + || w.offset.get() != 0 + || w.expected_revision.get() != 0 + || w.session.get() != 0 + || !names_are_empty(w) + { + return None; + } + FsRequest::Parents { + handle, + cursor: w.length.get(), + window_offset: w.window_offset.get(), + } + } + // The anchor, the identity and the window: nothing else. A zero + // identity is no object, refused rather than searched for. + FsReqWire::OP_LOCATE => { + if w.length.get() != 0 + || w.offset.get() != 0 + || w.expected_revision.get() != 0 + || w.session.get() != 0 + || !names_are_empty(w) + { + return None; + } + FsRequest::Locate { + handle, + stable: crate::StableObjectId::new(w.handle2.get())?, + window_offset: w.window_offset.get(), + } + } FsReqWire::OP_LIST_GRANTS => FsRequest::ListGrants { window_offset: w.window_offset.get(), }, @@ -967,6 +1048,16 @@ pub fn decode_req(w: &FsReqWire) -> Option { announced: SlotRef::from_wire(w.window_offset.get()), } } + FsReqWire::OP_MATCH_VIEW => { + // Not a rights verb: a set mask already refused above. + if w.kind != 0 || !control_payload_except_handle_is_empty(w) { + return None; + } + FsRequest::MatchView { + handle, + announced: SlotRef::from_wire(w.window_offset.get()), + } + } FsReqWire::OP_ADMIT_PEER => { if !control_payload_is_empty(w) { return None; @@ -1013,6 +1104,34 @@ pub fn decode_req(w: &FsReqWire) -> Option { _ => FsRequest::StorDetach, } } + FsReqWire::OP_STOR_MAINTAIN => { + // Only a forced refresh carries operands: its first bucket and + // count. Every other action rides `kind` alone. + if w.offset.get() != 0 + || w.expected_revision.get() != 0 + || w.length.get() != 0 + || !names_are_empty(w) + { + return None; + } + let (first, count) = (w.handle.get(), w.handle2.get()); + let action = match w.kind { + FsReqWire::MAINTAIN_REFRESH_RANGE => MaintainAction::RefreshRange { + first: u32::try_from(first).ok()?, + count: u32::try_from(count).ok()?, + }, + _ if first != 0 || count != 0 => return None, + FsReqWire::MAINTAIN_STATUS => MaintainAction::Status, + FsReqWire::MAINTAIN_SCAN => MaintainAction::Scan, + FsReqWire::MAINTAIN_REFRESH_STALE => MaintainAction::RefreshStale, + FsReqWire::MAINTAIN_CANCEL => MaintainAction::Cancel, + _ => return None, + }; + FsRequest::StorMaintain { + action, + window_offset: w.window_offset.get(), + } + } FsReqWire::OP_STOR_GRANT_VERSION => { if w.window_offset.get() != 0 || !control_payload_except_handle_is_empty(w) { return None; @@ -1066,6 +1185,12 @@ fn blank_rep() -> FsRepWire { } } +/// The wire status for a refusal. Public as the frozen numeric vocabulary a +/// startup witness also prints (`kore-svc-dagfs`'s recovery refusal). +pub fn fs_status_code(err: FsError) -> u8 { + status_of(err) +} + /// The wire status for a refusal. fn status_of(err: FsError) -> u8 { match err { @@ -1100,7 +1225,7 @@ fn status_of(err: FsError) -> u8 { /// A provider failure's cause, as the refusal's `kind` byte carries it. /// Frozen: append, never renumber. Zero is no cause, from an encoder that /// predates the byte, and decodes as [`ProviderError::Media`]. -fn provider_code(error: ProviderError) -> u8 { +pub fn provider_code(error: ProviderError) -> u8 { match error { ProviderError::Exhausted => 1, ProviderError::PlacementsFull => 2, @@ -1263,7 +1388,7 @@ pub fn encode_rep(rep: &FsResponse, grants: u8) -> FsRepWire { w.status = FsRepWire::STATUS_OK; w.count = U32::new(entries); } - FsResponse::SystemStatusReported { len } => { + FsResponse::SystemStatusReported { len } | FsResponse::StorMaintained { len } => { w.status = FsRepWire::STATUS_OK; w.count = U32::new(len); } @@ -1448,7 +1573,13 @@ pub fn decode_rep_for(request: &FsRequest, w: &FsRepWire) -> Option<(FsResponse, } // Only a listing may set flags, and `FLAG_MORE` is the only defined bit. - let listing = matches!(request, FsRequest::List { .. } | FsRequest::Search { .. }); + let listing = matches!( + request, + FsRequest::List { .. } + | FsRequest::Parents { .. } + | FsRequest::Locate { .. } + | FsRequest::Search { .. } + ); if w.flags & !FsRepWire::FLAG_MORE != 0 { return None; } @@ -1478,7 +1609,7 @@ pub fn decode_rep_for(request: &FsRequest, w: &FsRepWire) -> Option<(FsResponse, links: w.count.get(), grants: w.cursor.get(), }, - FsRequest::List { .. } => FsResponse::Listed { + FsRequest::List { .. } | FsRequest::Parents { .. } | FsRequest::Locate { .. } => FsResponse::Listed { entries: w.count.get(), next: if w.flags & FsRepWire::FLAG_MORE != 0 { Some(w.cursor.get()) @@ -1495,6 +1626,7 @@ pub fn decode_rep_for(request: &FsRequest, w: &FsRepWire) -> Option<(FsResponse, entries: w.count.get(), }, FsRequest::SystemStatus { .. } => FsResponse::SystemStatusReported { len: w.count.get() }, + FsRequest::StorMaintain { .. } => FsResponse::StorMaintained { len: w.count.get() }, FsRequest::Read { .. } | FsRequest::ReadVersion { .. } => FsResponse::Transferred { count: w.count.get(), }, @@ -1596,6 +1728,7 @@ pub fn decode_rep_for(request: &FsRequest, w: &FsRepWire) -> Option<(FsResponse, | FsRequest::RegisterPrivateView { .. } | FsRequest::ApplyView { .. } | FsRequest::BindView { .. } + | FsRequest::MatchView { .. } | FsRequest::Close { .. } | FsRequest::Rename { .. } | FsRequest::Relocate { .. } @@ -1698,7 +1831,9 @@ pub fn reply_matches(request: &FsRequest, response: &FsResponse) -> bool { | FsRequest::OpenGrant { .. } | FsRequest::Create { .. } => matches!(response, FsResponse::Opened { .. }), FsRequest::Stat { .. } => matches!(response, FsResponse::Stat { .. }), - FsRequest::List { .. } => matches!(response, FsResponse::Listed { .. }), + FsRequest::List { .. } | FsRequest::Parents { .. } | FsRequest::Locate { .. } => { + matches!(response, FsResponse::Listed { .. }) + } FsRequest::ListGrants { .. } => matches!(response, FsResponse::GrantsListed { .. }), FsRequest::SystemStatus { .. } => { matches!(response, FsResponse::SystemStatusReported { .. }) @@ -1727,10 +1862,12 @@ pub fn reply_matches(request: &FsRequest, response: &FsResponse) -> bool { } FsRequest::StorTarget { .. } => matches!(response, FsResponse::StorTargetAdopted { .. }), FsRequest::StorInspect => matches!(response, FsResponse::StorInspected { .. }), + FsRequest::StorMaintain { .. } => matches!(response, FsResponse::StorMaintained { .. }), FsRequest::RegisterSystemView { .. } | FsRequest::RegisterPrivateView { .. } | FsRequest::ApplyView { .. } | FsRequest::BindView { .. } + | FsRequest::MatchView { .. } | FsRequest::Close { .. } | FsRequest::Rename { .. } | FsRequest::Relocate { .. } @@ -1855,6 +1992,82 @@ mod tests { assert_eq!(decode_rep_for(&FsRequest::StorGrantPack, &wide), None); } + /// `Locate` carries its anchor, the identity and the window, and + /// nothing else; a zero identity is no object and is refused. + #[test] + fn locate_round_trips_and_refuses_stray_operands() { + let request = FsRequest::Locate { + handle: Handle::from_bits(7), + stable: crate::StableObjectId::new(0x2a).unwrap(), + window_offset: 64, + }; + let wire = encode_req(&request); + assert_eq!(decode_req(&wire), Some(request)); + let reply = FsResponse::Listed { + entries: 1, + next: None, + }; + assert!(reply_matches(&request, &reply)); + assert_eq!(decode_rep_for(&request, &encode_rep(&reply, 0)), Some((reply, 0))); + let mut zero = wire; + zero.handle2 = U64::new(0); + assert_eq!(decode_req(&zero), None); + let mut stray = wire; + stray.length = U32::new(1); + assert_eq!(decode_req(&stray), None); + let mut stray = wire; + stray.offset = U64::new(1); + assert_eq!(decode_req(&stray), None); + } + + /// Every maintenance action survives the wire; only a forced refresh + /// carries operands, and an operand on any other action — or a range + /// past `u32` — is refused. + #[test] + fn stor_maintain_round_trips_and_refuses_stray_operands() { + for action in [ + MaintainAction::Status, + MaintainAction::Scan, + MaintainAction::RefreshStale, + MaintainAction::RefreshRange { first: 7, count: u32::MAX }, + MaintainAction::Cancel, + ] { + let req = FsRequest::StorMaintain { + action, + window_offset: 512, + }; + assert!(req.needs_window()); + assert_eq!(decode_req(&encode_req(&req)), Some(req)); + let reply = FsResponse::StorMaintained { len: 100 + 8 * 1024 }; + assert!(reply_matches(&req, &reply)); + assert_eq!(decode_rep_for(&req, &encode_rep(&reply, 0)), Some((reply, 0))); + } + let mut stray = encode_req(&FsRequest::StorMaintain { + action: MaintainAction::Scan, + window_offset: 0, + }); + stray.handle = U64::new(1); + assert_eq!(decode_req(&stray), None); + let mut wide = encode_req(&FsRequest::StorMaintain { + action: MaintainAction::RefreshRange { first: 0, count: 1 }, + window_offset: 0, + }); + wide.handle2 = U64::new(1 << 32); + assert_eq!(decode_req(&wide), None); + let mut unknown = encode_req(&FsRequest::StorMaintain { + action: MaintainAction::Status, + window_offset: 0, + }); + unknown.kind = 5; + assert_eq!(decode_req(&unknown), None); + let mut length = encode_req(&FsRequest::StorMaintain { + action: MaintainAction::Status, + window_offset: 0, + }); + length.length = U32::new(1); + assert_eq!(decode_req(&length), None); + } + /// The pack lane's retired op codes decode to nothing: a peer still /// speaking them is refused, never served. #[test] diff --git a/native/crates/driver/Cargo.toml b/native/crates/driver/Cargo.toml index 4935d12..deee477 100644 --- a/native/crates/driver/Cargo.toml +++ b/native/crates/driver/Cargo.toml @@ -35,6 +35,10 @@ zerocopy = { version = "0.8", default-features = false, features = ["derive"] } gpt_disk_types = { version = "0.16.1", default-features = false, features = ["bytemuck"] } crc = { version = "3.4.0", default-features = false } bytemuck = { version = "1.25.2", default-features = false } +# The image-file device's whole-file record lock (`image` feature only): the +# safe `fcntl` the vendored wasmtime already resolves, so the lock costs no +# new crate. A host tool's dependency, never the kernel's or a service's. +rustix = { version = "1.1.4", default-features = false, features = ["std", "fs"], optional = true } [features] # Native launch and health hooks over the allocation-free upstream lifecycle. @@ -51,6 +55,10 @@ host-double = ["alloc"] # caps (`kore_driver::host`). Pulls kore-rt; kept off the kernel's # default-feature build. service = ["dep:kore-rt", "dep:kore-collections"] +# A regular image file as a block device (`kore_driver::block::image`), for +# host tools that read or write a kore volume offline. Needs `std`: host +# builds only. +image = ["dep:rustix"] # The self-dev-dependency turns `service` on for this crate's own test # builds, so the service face stays test-compiled. diff --git a/native/crates/driver/src/block/image.rs b/native/crates/driver/src/block/image.rs new file mode 100644 index 0000000..d73bc4d --- /dev/null +++ b/native/crates/driver/src/block/image.rs @@ -0,0 +1,633 @@ +//! A disk's blocks, from the host, as a [`DirectBlockDevice`]: the medium a +//! host tool reads or writes a kore volume on while no machine is running +//! it. Either a regular file holding a disk's image (the disk a QEMU run +//! boots from, as raw bytes, or one copied off a machine) or, on Linux, a +//! machine's disk itself through a block device node. +//! +//! - **A file or a disk's node, presented and nothing more.** +//! [`ImageFile::open`] takes a regular file, at the block size its caller +//! names, since an image carries none of its own. +//! [`ImageFile::open_device`] takes a block device node, a whole disk or +//! one partition of it, at the logical block size the kernel reports for +//! it. Which blocks of a disk may be written is its caller's to decide +//! (`tools/kore-vol` asks the boot's own `locate`), and partitioning one is +//! provisioning (`tools/provision_fs.py`), which nothing here does. +//! - **Never beside a running machine.** A mounted volume keeps its free +//! extents and its next root slot in memory, so a second writer would +//! allocate over blocks the first believes free, and payload blocks carry +//! no checksum that would catch it. Both opens take a whole-file record +//! lock, exclusive to write and shared to read, and refuse a file or node +//! another process holds: QEMU locks the images and devices it runs, so +//! the one lock answers both "is a machine running on this?" and "is +//! another tool writing it?". The lock is a POSIX record lock, which +//! belongs to the process and is dropped when *any* descriptor of the file +//! closes: a process holds one `ImageFile` per file or node and closes no +//! other descriptor of it while that one lives. A device opened to write +//! is also claimed exclusively (`O_EXCL`), which Linux refuses while a +//! filesystem on it, or on a partition of the same disk, is mounted, or +//! another program has claimed one of them. +//! - **No cache between a device and its medium.** A device is read and +//! written with `O_DIRECT`. Linux caches each node's blocks separately, so +//! a disk and each of its partitions hold their own copies of the same +//! blocks, and a buffered read through one node can answer, stale, what +//! was written through another. Direct transfers take memory aligned to +//! the logical block size, so every buffer begins on a [`BUFFER_ALIGN`] +//! boundary (or on the block size, where that is larger) and a transfer +//! begins a whole number of blocks into it, as the block channel already +//! requires of its callers. +//! - **Synchronous.** A command runs to completion inside `submit` and its +//! completion waits in a queue for `poll`: the asynchronous contract kept, +//! with nothing ever in flight. A flush is `fdatasync`, since an image's +//! length never changes and its metadata has nothing to make durable; on a +//! device it also flushes the device's volatile write cache. + +extern crate std; + +use std::{ + collections::VecDeque, + fs::{File, OpenOptions}, + io, + os::unix::fs::FileExt, + path::Path, + vec, + vec::Vec, +}; + +use super::{ + BlockBuffer, BlockError, BlockGeometry, BlockIoHints, DeviceCommand, DeviceCompletion, + DirectBlockDevice, +}; + +/// Completions queued and not yet polled. A channel keeps no more commands +/// outstanding than its own bound (32 in the providers built on this), so +/// the queue is never the limit in practice; past it `submit` answers +/// [`BlockError::Backpressure`], as a full hardware queue would. +const QUEUE_DEPTH: usize = 64; + +/// The largest single transfer, and so the largest buffer: 256 KiB. The +/// filesystem provider's own transfers are at most 64 KiB of payload and +/// its 128 KiB metadata images; anything larger is refused rather than +/// allocated. +const MAX_TRANSFER_BYTES: u32 = 256 * 1024; + +/// Where every buffer begins: on a 4 KiB boundary, the largest logical +/// block kore formats a volume with (an NVMe namespace's 4 KiB format), and +/// so at least the alignment a direct transfer to such a device takes. A +/// device whose logical blocks are larger aligns its buffers to its block +/// size instead. +pub const BUFFER_ALIGN: usize = 4096; + +/// How an image is opened: read-only images are opened, locked and +/// presented read-only all the way down. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Access { + ReadOnly, + ReadWrite, +} + +/// Why an image could not be opened. +#[derive(Debug)] +pub enum ImageError { + /// The operating system's answer, for a path that could not be opened, + /// read or locked. + Io(io::Error), + /// The path names something other than a regular file + /// ([`ImageFile::open`]). + NotRegular, + /// The path names something other than a block device + /// ([`ImageFile::open_device`]). + NotDevice, + /// The kernel refused the exclusive claim a device is written under: a + /// filesystem on it, or on a partition of the same disk, is mounted, or + /// another program has claimed one of them. + Claimed, + /// The file's length is not a whole number of blocks. + Unaligned { len: u64, block_size: u32 }, + /// The geometry the file would present is not a valid one. + Geometry(BlockError), + /// Another process holds the image: a machine running on it, or another + /// tool. + InUse, +} + +impl From for ImageError { + fn from(error: io::Error) -> Self { + Self::Io(error) + } +} + +/// A registered buffer: heap memory, since nothing here is DMA, whose first +/// byte sits on the alignment a direct transfer takes (the module docs). +/// The vector is never resized, so `start` stays where it was measured. +pub struct ImageBuffer { + bytes: Vec, + start: usize, + len: usize, +} + +impl ImageBuffer { + /// `len` zeroed bytes beginning on an `align` boundary, a power of two. + fn new(len: usize, align: usize) -> Self { + let bytes = vec![0; len + align - 1]; + let at = bytes.as_ptr().addr(); + let start = at.next_multiple_of(align) - at; + Self { bytes, start, len } + } +} + +impl BlockBuffer for ImageBuffer { + fn len(&self) -> usize { + self.len + } + fn bytes(&self) -> &[u8] { + &self.bytes[self.start..self.start + self.len] + } + fn bytes_mut(&mut self) -> &mut [u8] { + &mut self.bytes[self.start..self.start + self.len] + } +} + +pub struct ImageFile { + file: File, + geometry: BlockGeometry, + ready: VecDeque, +} + +impl ImageFile { + /// Open, lock and size `path` as a device of `block_size`-byte logical + /// blocks. The block size is the caller's: an image carries none of its + /// own, and a kore volume refuses to recover under any but the one it + /// was formatted with. + pub fn open(path: &Path, access: Access, block_size: u32) -> Result { + let writable = access == Access::ReadWrite; + let file = OpenOptions::new().read(true).write(writable).open(path)?; + if !file.metadata()?.file_type().is_file() { + return Err(ImageError::NotRegular); + } + lock(&file, writable)?; + // Measured under the lock: nothing that honours it resizes the file + // from here on. + let len = file.metadata()?.len(); + Self::over(file, len, block_size, writable) + } + + /// Open the block device node at `path` (a whole disk, or a partition + /// of one), unbuffered, claimed exclusively when writing, locked as + /// [`open`](Self::open) locks a file, and sized as the kernel reports + /// it: its capacity and its logical block size, the size a kore volume + /// on it was formatted with by the machine that first booted it. + #[cfg(target_os = "linux")] + pub fn open_device(path: &Path, access: Access) -> Result { + use rustix::fs::{Mode, OFlags}; + use std::{ + io::{Seek, SeekFrom}, + os::unix::fs::FileTypeExt, + }; + // Refused before it is opened: what is not a device is neither + // claimed nor opened unbuffered. Checked again on what was opened. + if !std::fs::metadata(path)?.file_type().is_block_device() { + return Err(ImageError::NotDevice); + } + let writable = access == Access::ReadWrite; + let mode = if writable { + OFlags::RDWR | OFlags::EXCL + } else { + OFlags::RDONLY + }; + let file = + match rustix::fs::open(path, mode | OFlags::DIRECT | OFlags::CLOEXEC, Mode::empty()) { + Ok(fd) => File::from(fd), + Err(rustix::io::Errno::BUSY) => return Err(ImageError::Claimed), + Err(error) => return Err(ImageError::Io(error.into())), + }; + if !file.metadata()?.file_type().is_block_device() { + return Err(ImageError::NotDevice); + } + lock(&file, writable)?; + let block_size = + rustix::fs::ioctl_blksszget(&file).map_err(|error| ImageError::Io(error.into()))?; + // A device's end is its capacity. + let len = (&file).seek(SeekFrom::End(0))?; + Self::over(file, len, block_size, writable) + } + + /// The device over an opened, locked file of `len` bytes. + fn over(file: File, len: u64, block_size: u32, writable: bool) -> Result { + if block_size == 0 || !block_size.is_power_of_two() || len % u64::from(block_size) != 0 { + return Err(ImageError::Unaligned { len, block_size }); + } + let geometry = BlockGeometry { + block_size, + block_count: len / u64::from(block_size), + max_transfer_blocks: MAX_TRANSFER_BYTES / block_size, + atomic_write_blocks: 1, + flush: true, + discard: false, + read_only: !writable, + io_hints: BlockIoHints::NONE, + } + .validate() + .map_err(ImageError::Geometry)?; + Ok(Self { + file, + geometry, + ready: VecDeque::with_capacity(QUEUE_DEPTH), + }) + } + + /// Run one command against the file, synchronously. + fn execute( + &self, + command: DeviceCommand, + buffer: Option<&mut ImageBuffer>, + ) -> Result<(), BlockError> { + let (lba, blocks, offset, write) = match command { + DeviceCommand::Flush => return self.file.sync_data().map_err(|_| BlockError::Media), + DeviceCommand::Discard { .. } => return Err(BlockError::Unsupported), + DeviceCommand::Read { + lba, + blocks, + buffer_offset, + } => (lba, blocks, buffer_offset, false), + DeviceCommand::Write { + lba, + blocks, + buffer_offset, + } => (lba, blocks, buffer_offset, true), + }; + if write && self.geometry.read_only { + return Err(BlockError::ReadOnly); + } + if blocks > self.geometry.max_transfer_blocks { + return Err(BlockError::TooLarge); + } + if lba + .checked_add(u64::from(blocks)) + .is_none_or(|end| end > self.geometry.block_count) + { + return Err(BlockError::Range); + } + // A transfer begins a whole number of blocks into its buffer, so on + // a block boundary in memory, which a direct transfer needs. + if !offset.is_multiple_of(self.geometry.block_size as usize) { + return Err(BlockError::Alignment); + } + // Bounded by the transfer ceiling above, so neither product overflows. + let len = blocks as usize * self.geometry.block_size as usize; + let at = lba * u64::from(self.geometry.block_size); + let buffer = buffer.ok_or(BlockError::UnknownBuffer)?; + let bytes = offset + .checked_add(len) + .and_then(|end| buffer.bytes_mut().get_mut(offset..end)) + .ok_or(BlockError::Range)?; + if write { + self.file.write_all_at(bytes, at) + } else { + self.file.read_exact_at(bytes, at) + } + .map_err(|_| BlockError::Media) + } +} + +/// Take the whole-file record lock, without waiting for it. +fn lock(file: &File, exclusive: bool) -> Result<(), ImageError> { + use rustix::fs::{FlockOperation, fcntl_lock}; + let operation = if exclusive { + FlockOperation::NonBlockingLockExclusive + } else { + FlockOperation::NonBlockingLockShared + }; + match fcntl_lock(file, operation) { + Ok(()) => Ok(()), + // POSIX lets a conflicting `F_SETLK` answer either. + Err(rustix::io::Errno::AGAIN | rustix::io::Errno::ACCESS) => Err(ImageError::InUse), + Err(error) => Err(ImageError::Io(error.into())), + } +} + +impl DirectBlockDevice for ImageFile { + type Buffer = ImageBuffer; + const INDEPENDENT_PROGRESS: bool = true; + + fn geometry(&self) -> BlockGeometry { + self.geometry + } + fn allocate_buffer(&mut self, bytes: usize) -> Result { + if bytes > self.max_buffer_bytes() { + return Err(BlockError::TooLarge); + } + let align = BUFFER_ALIGN.max(self.geometry.block_size as usize); + Ok(ImageBuffer::new(bytes, align)) + } + fn submit( + &mut self, + command: DeviceCommand, + buffer: Option<&mut ImageBuffer>, + cookie: u64, + ) -> Result<(), BlockError> { + if self.ready.len() >= QUEUE_DEPTH { + return Err(BlockError::Backpressure); + } + let result = self.execute(command, buffer); + self.ready.push_back(DeviceCompletion { cookie, result }); + Ok(()) + } + fn poll(&mut self) -> Option { + self.ready.pop_front() + } + fn quiesce(&mut self) -> Result<(), BlockError> { + if self.geometry.read_only { + return Ok(()); + } + self.file.sync_data().map_err(|_| BlockError::Media) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + fs, + path::PathBuf, + string::{String, ToString}, + }; + + /// A scratch path of this test's own beside the test binary's. + fn scratch(name: &str, bytes: Option) -> PathBuf { + let path = std::env::temp_dir().join(std::format!( + "kore-driver-image-{}-{name}", + std::process::id() + )); + let _ = fs::remove_file(&path); + if let Some(bytes) = bytes { + File::create(&path).unwrap().set_len(bytes).unwrap(); + } + path + } + + fn run( + image: &mut ImageFile, + command: DeviceCommand, + buffer: Option<&mut ImageBuffer>, + ) -> Result<(), BlockError> { + image.submit(command, buffer, 7)?; + let completion = image.poll().expect("a submitted command completes at once"); + assert_eq!(completion.cookie, 7); + assert!(image.poll().is_none()); + completion.result + } + + #[test] + fn a_written_block_reads_back_at_its_offset() { + let path = scratch("round-trip", Some(64 * 512)); + let mut image = ImageFile::open(&path, Access::ReadWrite, 512).unwrap(); + assert_eq!(image.geometry().block_count, 64); + let mut buffer = image.allocate_buffer(3 * 512).unwrap(); + for (at, byte) in buffer.bytes_mut().iter_mut().enumerate() { + *byte = (at % 251) as u8; + } + let write = DeviceCommand::Write { + lba: 5, + blocks: 2, + buffer_offset: 512, + }; + run(&mut image, write, Some(&mut buffer)).unwrap(); + run(&mut image, DeviceCommand::Flush, None).unwrap(); + let on_disk = fs::read(&path).unwrap(); + assert_eq!(&on_disk[5 * 512..7 * 512], &buffer.bytes()[512..]); + assert!(on_disk[..5 * 512].iter().all(|&b| b == 0)); + let mut back = image.allocate_buffer(2 * 512).unwrap(); + let read = DeviceCommand::Read { + lba: 5, + blocks: 2, + buffer_offset: 0, + }; + run(&mut image, read, Some(&mut back)).unwrap(); + assert_eq!(back.bytes(), &buffer.bytes()[512..]); + fs::remove_file(path).unwrap(); + } + + #[test] + fn what_leaves_the_image_or_the_buffer_is_refused() { + let path = scratch("bounds", Some(8 * 512)); + let mut image = ImageFile::open(&path, Access::ReadWrite, 512).unwrap(); + let mut buffer = image.allocate_buffer(512).unwrap(); + let past_end = DeviceCommand::Read { + lba: 8, + blocks: 1, + buffer_offset: 0, + }; + assert_eq!( + run(&mut image, past_end, Some(&mut buffer)), + Err(BlockError::Range) + ); + let past_buffer = DeviceCommand::Read { + lba: 0, + blocks: 2, + buffer_offset: 0, + }; + assert_eq!( + run(&mut image, past_buffer, Some(&mut buffer)), + Err(BlockError::Range) + ); + let no_buffer = DeviceCommand::Write { + lba: 0, + blocks: 1, + buffer_offset: 0, + }; + assert_eq!( + run(&mut image, no_buffer, None), + Err(BlockError::UnknownBuffer) + ); + let discard = DeviceCommand::Discard { lba: 0, blocks: 1 }; + assert_eq!(run(&mut image, discard, None), Err(BlockError::Unsupported)); + assert_eq!( + image.allocate_buffer(MAX_TRANSFER_BYTES as usize + 1).err(), + Some(BlockError::TooLarge) + ); + drop(image); + + let mut read_only = ImageFile::open(&path, Access::ReadOnly, 512).unwrap(); + assert!(read_only.geometry().read_only); + let mut buffer = read_only.allocate_buffer(512).unwrap(); + let write = DeviceCommand::Write { + lba: 0, + blocks: 1, + buffer_offset: 0, + }; + assert_eq!( + run(&mut read_only, write, Some(&mut buffer)), + Err(BlockError::ReadOnly) + ); + fs::remove_file(path).unwrap(); + } + + #[test] + fn only_a_regular_whole_number_of_blocks_opens() { + let directory = std::env::temp_dir(); + assert!(matches!( + ImageFile::open(&directory, Access::ReadOnly, 512), + Err(ImageError::NotRegular) + )); + let ragged = scratch("ragged", Some(4096 + 100)); + assert!(matches!( + ImageFile::open(&ragged, Access::ReadOnly, 512), + Err(ImageError::Unaligned { + len: 4196, + block_size: 512 + }) + )); + // 4 KiB logical blocks: a whole number of them, or not. + let sectors = scratch("sectors", Some(3 * 512)); + assert!(ImageFile::open(§ors, Access::ReadOnly, 512).is_ok()); + assert!(matches!( + ImageFile::open(§ors, Access::ReadOnly, 4096), + Err(ImageError::Unaligned { .. }) + )); + let missing = scratch("missing", None); + assert!(matches!( + ImageFile::open(&missing, Access::ReadOnly, 512), + Err(ImageError::Io(_)) + )); + let _ = fs::remove_file(ragged); + let _ = fs::remove_file(sectors); + } + + #[test] + fn every_buffer_begins_where_a_direct_transfer_can_reach_it() { + for block_size in [512, 4096] { + let path = scratch("aligned", Some(64 * 4096)); + let mut image = ImageFile::open(&path, Access::ReadOnly, block_size).unwrap(); + for bytes in [0, 1, 511, 512, 4096, 65_536, MAX_TRANSFER_BYTES as usize] { + let buffer = image.allocate_buffer(bytes).unwrap(); + assert_eq!(buffer.len(), bytes); + assert_eq!(buffer.bytes().len(), bytes); + assert_eq!(buffer.bytes().as_ptr().addr() % BUFFER_ALIGN, 0); + assert!(buffer.bytes().iter().all(|&byte| byte == 0)); + } + fs::remove_file(path).unwrap(); + } + } + + #[test] + fn a_transfer_begins_on_a_block_boundary_in_its_buffer() { + let path = scratch("boundary", Some(8 * 512)); + let mut image = ImageFile::open(&path, Access::ReadWrite, 512).unwrap(); + let mut buffer = image.allocate_buffer(3 * 512).unwrap(); + let read = DeviceCommand::Read { + lba: 0, + blocks: 1, + buffer_offset: 100, + }; + assert_eq!( + run(&mut image, read, Some(&mut buffer)), + Err(BlockError::Alignment) + ); + fs::remove_file(path).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn only_a_block_device_opens_as_one() { + let file = scratch("not-a-device", Some(8 * 512)); + for access in [Access::ReadOnly, Access::ReadWrite] { + assert!(matches!( + ImageFile::open_device(&file, access), + Err(ImageError::NotDevice) + )); + assert!(matches!( + ImageFile::open_device(&std::env::temp_dir(), access), + Err(ImageError::NotDevice) + )); + } + let missing = scratch("missing-device", None); + assert!(matches!( + ImageFile::open_device(&missing, Access::ReadOnly), + Err(ImageError::Io(_)) + )); + fs::remove_file(file).unwrap(); + } + + /// A loop device over a scratch file, detached when dropped. + #[cfg(target_os = "linux")] + struct Loop(PathBuf); + + #[cfg(target_os = "linux")] + impl Loop { + fn attach(file: &Path, sector_size: u32) -> Self { + let output = std::process::Command::new("losetup") + .args(["--find", "--show", "--sector-size"]) + .arg(sector_size.to_string()) + .arg(file) + .output() + .expect("losetup runs"); + assert!( + output.status.success(), + "losetup: {}", + String::from_utf8_lossy(&output.stderr) + ); + let node = String::from_utf8(output.stdout).unwrap(); + Self(PathBuf::from(node.trim())) + } + } + + #[cfg(target_os = "linux")] + impl Drop for Loop { + fn drop(&mut self) { + let _ = std::process::Command::new("losetup") + .arg("--detach") + .arg(&self.0) + .status(); + } + } + + #[cfg(target_os = "linux")] + #[test] + #[ignore = "needs root and loop devices: sudo -E cargo test -p kore-driver --features image -- --ignored"] + fn a_device_is_read_and_written_unbuffered_at_its_own_block_size() { + for sector_size in [512, 4096] { + let file = scratch("loop", Some(64 * 4096)); + let device = Loop::attach(&file, sector_size); + let mut disk = ImageFile::open_device(&device.0, Access::ReadWrite).unwrap(); + let geometry = disk.geometry(); + assert_eq!(geometry.block_size, sector_size); + assert_eq!(geometry.block_count, 64 * 4096 / u64::from(sector_size)); + let block = sector_size as usize; + let mut buffer = disk.allocate_buffer(3 * block).unwrap(); + for (at, byte) in buffer.bytes_mut().iter_mut().enumerate() { + *byte = (at % 251) as u8 ^ 0x5a; + } + let write = DeviceCommand::Write { + lba: 5, + blocks: 2, + buffer_offset: block, + }; + run(&mut disk, write, Some(&mut buffer)).unwrap(); + run(&mut disk, DeviceCommand::Flush, None).unwrap(); + // Through the medium, not a cache: the file behind it holds it. + let behind = fs::read(&file).unwrap(); + assert_eq!(&behind[5 * block..7 * block], &buffer.bytes()[block..]); + let mut back = disk.allocate_buffer(2 * block).unwrap(); + let read = DeviceCommand::Read { + lba: 5, + blocks: 2, + buffer_offset: 0, + }; + run(&mut disk, read, Some(&mut back)).unwrap(); + assert_eq!(back.bytes(), &buffer.bytes()[block..]); + + // Claimed while it is written: a second writer is refused by the + // kernel; a reader is not. + assert!(matches!( + ImageFile::open_device(&device.0, Access::ReadWrite), + Err(ImageError::Claimed) + )); + drop(disk); + let reader = ImageFile::open_device(&device.0, Access::ReadOnly).unwrap(); + assert!(reader.geometry().read_only); + drop(reader); + drop(device); + fs::remove_file(file).unwrap(); + } + } +} diff --git a/native/crates/driver/src/block/mod.rs b/native/crates/driver/src/block/mod.rs index 4f0a133..7475d3c 100644 --- a/native/crates/driver/src/block/mod.rs +++ b/native/crates/driver/src/block/mod.rs @@ -20,7 +20,10 @@ use core::{cell::Cell, marker::PhantomData}; use kore_abi::monitor::FilesystemLatencySnapshot; +#[cfg(feature = "image")] +pub mod image; pub mod memory; +pub mod refresh; pub mod region; pub use region::RegionDevice; diff --git a/native/crates/driver/src/block/refresh.rs b/native/crates/driver/src/block/refresh.rs new file mode 100644 index 0000000..e69b6f2 --- /dev/null +++ b/native/crates/driver/src/block/refresh.rs @@ -0,0 +1,1700 @@ +//! **Selective refresh** of a read-mostly region (`res/PACK_STREAMING.md` §7). +//! +//! Flash read speed has two slow states, and only one of them is curable. +//! +//! 1. **Folded.** A client SSD writes new data into a pseudo-SLC cache and, +//! when idle, folds it into native TLC/QLC. Native reads are slower: on +//! Biggertron's Phison E29T a 4 KiB read takes 47 µs from the cache and +//! 113 µs folded, and folded data streams at 1.6–2.8 GB/s against 5 GB/s +//! cached (`res/PACK_STREAMING.md` §7). Rewriting folded data only puts it +//! back in the cache until the drive folds it again, about an hour of +//! idle later. Nothing a host does changes this state. +//! 2. **Decayed.** Charge leaks out of cells over months. The controller +//! then needs read-retry and soft-decode to recover the data, and every +//! retry costs another native read time, so reads slow by whole +//! multiples and keep slowing. Only a rewrite cures it, and most client +//! firmware never rewrites on its own. +//! +//! [`RegionRefresh`] tells the two apart and rewrites only the second. +//! +//! - **Measure.** A scan splits the region into buckets and times +//! [`PROBES`] single 4 KiB reads per bucket, one at a time, issued only +//! while the device is otherwise idle, so a sample is the read's own +//! latency rather than a queue's. It makes [`PASSES`] passes over the same +//! pages and keeps each bucket's fastest pass: slowness that comes and goes +//! (a cold mapping cache, the drive's own background work) does not +//! survive the minimum, and decay, a property of the data, shows in every +//! pass. +//! - **Classify.** A bucket is stale when its median is at least +//! [`STALE_RATIO`] times the region's 25th percentile. The quartile, not +//! the median, anchors the comparison so a region that has mostly decayed +//! is still judged against its healthy part; the ratio sits above the +//! folded-versus-cached spread so a folded bucket never qualifies. +//! - **Refresh.** Each chunk of a stale bucket is read twice into the job's +//! own buffers; only bytes the two reads agree on are written back to the +//! same blocks, and a third read must match them before the chunk counts. +//! A bucket is flushed when its last chunk lands. +//! +//! **The content invariant.** A refresh never changes a byte. The write's +//! source is the job's own buffer, filled by the device from exactly the +//! blocks it is written back to (a chunk's extent is fixed for its whole +//! read → confirm → write → check cycle; [`Chunk::command`] derives every +//! command from it), and no other party can reach that buffer. A reader +//! racing the rewrite sees, block by block, the old copy or the new one — +//! the same bytes either way — so a refresh needs no exclusion against the +//! region's readers and changes nothing they can observe. A chunk that +//! reads as zeros is skipped: zeros are most likely never-written blocks, +//! and writing them would allocate flash for nothing. +//! +//! **The cost bound.** Each bucket is rewritten at most once per job +//! lifetime (per boot, for a row that keeps one job), forced or not, so the +//! endurance a refresh can spend is bounded by the region's size per boot. +//! +//! **Driving.** The job never waits. [`RegionRefresh::advance`] reaps what +//! completed, stages what is next, publishes once and returns; the owner +//! calls it on every wake, and the job's own completions are wakes. + +use super::{BlockBuffer, BlockError, BlockGeometry, DeviceCommand, DeviceCompletion, DirectBlockDevice}; + +/// Probes per bucket. A median of sixteen single reads is stable against a +/// stray slow sample (Biggertron's idle 4 KiB QD1 p99 is 1.3× its median) +/// while a whole scan of a 40 GiB pack in 1024 buckets stays near two +/// seconds of idle device time (16 384 reads of about 120 µs). +pub const PROBES: usize = 16; + +/// Passes a scan makes over the same probe pages; a bucket's estimate is its +/// fastest pass. On Biggertron (2026-10-16) a scan straight after boot read +/// p25 264 µs, p90 922 µs and judged 65 of 1024 buckets stale, while a scan +/// minutes later read p25 138 µs, p90 153 µs and none — and a bucket the +/// first scan put at 1617 µs, never rewritten, read 137 µs. That slowness +/// came and went; decay does not. Under the minimum, a false verdict needs +/// the same bucket slowed in every pass, and the first pass also warms the +/// drive's mapping cache for the pages the later ones read. +pub const PASSES: u32 = 3; + +/// A bucket is stale at this multiple of the region's 25th-percentile +/// median. Healthy buckets differ by where the drive keeps them: Biggertron +/// read 47 µs from its cache and 113 µs folded, a 2.4× spread, so the ratio +/// must clear 2.4 with margin or a cached quartile would condemn every +/// folded bucket. Decay costs whole extra read times per read (a read-retry +/// is another full native read), so a decayed bucket crosses 4× of a folded +/// quartile after a few retries — and of a cached one after one. +pub const STALE_RATIO: u64 = 4; + +/// Mapped buckets a scan needs before it judges any of them: below four a +/// quartile is one bucket, and a region that small is not worth judging. +pub const MIN_BASELINE: u32 = 4; + +/// Chunks a refresh keeps in flight. One chunk's cycle (read, confirm read, +/// write, check read) is serial; Biggertron's folded-data rate stops rising +/// past eight 128 KiB commands in flight, and four overlapping chunks keep +/// at least that many on the device once the engine splits each at the +/// transfer limit. Each slot holds two buffers of up to [`CHUNK_CEILING`]. +pub const SLOTS: usize = 4; + +/// The largest chunk, so the job's DMA memory stays bounded at +/// `SLOTS × 2 × 1 MiB` on a device whose transfer limit is larger. A +/// smaller limit (Biggertron's MDTS is 256 KiB) sizes the chunk instead. +pub const CHUNK_CEILING: usize = 1 << 20; + +/// One probe reads this much: one 4 KiB flash page, the unit a read-retry +/// is paid in. A device whose blocks are larger probes one block. +const PROBE_BYTES: usize = 4096; + +/// Attempts at one step before the chunk is given up: one retry. A second +/// failure of the same step on the same blocks is the device's answer. +const ATTEMPTS: u8 = 2; + +/// A cookie is the job's generation in its high half and a tag in its low +/// half: a slot index, or one of the two tags beside them. +const TAG_BITS: u32 = 32; +const TAG_MASK: u64 = (1 << TAG_BITS) - 1; +const PROBE_TAG: u64 = SLOTS as u64; +const FLUSH_TAG: u64 = SLOTS as u64 + 1; + +/// One bucket's books. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Bucket { + /// The last scan's estimate: the fastest of its passes' medians of + /// non-zero probes, in nanoseconds (saturating); zero before the bucket + /// was scanned. + pub median_ns: u32, + /// Probes of the last scan that read non-zero data. + pub mapped_probes: u8, + /// Probes of the last scan the device failed. + pub failed_probes: u8, + /// The last scan judged this bucket stale and it has not been + /// refreshed since. + pub stale: bool, + /// Rewritten during this job's lifetime: never again before the next. + pub refreshed: bool, + /// A chunk of this bucket could not be refreshed (a failed or unstable + /// read): the rest of the bucket was, and the chunk is untouched. + pub incomplete: bool, +} + +impl Bucket { + /// Whether the last scan found mapped data here: most probes read + /// something other than zeros. + pub const fn mapped(&self) -> bool { + 2 * self.mapped_probes as usize >= PROBES + } +} + +/// What the job is doing. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Phase { + Idle, + Scanning, + Refreshing, + /// Cancelled or faulted: no new commands, waiting for the device to + /// return the ones it holds. + Draining, +} + +/// How the last job ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Outcome { + /// No job has run. + None, + Scanned, + Refreshed, + Cancelled, + /// A write could not be made to stick: the device refused it, or a + /// read after it disagreed with what was written, twice. The bucket was + /// left as the device has it, flushed or not; the data buffer still + /// held the agreed bytes when the job stopped. + Faulted(BlockError), +} + +/// Why a request to start was refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Refusal { + /// A job is running. + Busy, + /// No scan has completed: nothing is classified yet. + NotScanned, + /// The bucket range is empty or outside the region. + Range, + /// The device reads no clock, so nothing can be timed. + NoClock, +} + +/// What [`RegionRefresh::advance`] left behind. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Progress { + /// No job, nothing in flight. + Idle, + /// Commands are in flight: their completion is the next wake. + InFlight, + /// Work is waiting with nothing of the job's in flight — the device was + /// busy for a probe, or refused a command. Another party's completion + /// is the wake. + Waiting, +} + +/// A scan's verdict. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ScanSummary { + /// The last scan ran to its end. + pub complete: bool, + /// Buckets probed in the running pass, or by the last scan once it is + /// complete. + pub scanned: u32, + /// The running pass, from one; [`PASSES`] once the scan is complete. + pub pass: u32, + /// Of those, buckets whose probes read mapped data. + pub mapped: u32, + /// Buckets judged stale. + pub stale: u32, + /// Percentiles of the mapped buckets' medians, in nanoseconds. + pub p25_ns: u32, + pub p50_ns: u32, + pub p90_ns: u32, + /// Probes the device failed. + pub failed_probes: u32, + /// Scans completed by this job. + pub scans: u32, +} + +/// A refresh's books. `buckets_*` describe the last (or running) refresh; +/// the byte and error counters accumulate over the job's lifetime. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct RefreshSummary { + /// Buckets the last refresh set out to rewrite. + pub buckets_planned: u32, + /// Of those, rewritten and flushed. + pub buckets_done: u32, + /// Asked for but skipped: already rewritten this boot. + pub buckets_skipped: u32, + /// Buckets rewritten over the job's lifetime. + pub buckets_lifetime: u32, + /// Bytes rewritten and verified. + pub bytes: u64, + /// Chunks left alone because they read as zeros. + pub zero_chunks: u32, + /// Reads the device failed (a failed chunk read is left untouched). + pub read_errors: u32, + /// Chunks whose two reads disagreed twice: left untouched. + pub unstable_chunks: u32, + /// Writes retried after a failure or a disagreeing check. + pub write_retries: u32, + /// Checks that disagreed with the bytes just written. + pub mismatches: u32, +} + +/// The region's shape as the job divides it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Shape { + pub block_bytes: u32, + pub region_blocks: u64, + pub bucket_blocks: u64, + pub buckets: u32, + pub chunk_blocks: u32, + pub probe_blocks: u32, +} + +impl Shape { + fn new(geometry: BlockGeometry, max_buffer_bytes: usize, max_buckets: usize) -> Result { + let block = geometry.block_size as usize; + if block == 0 || max_buckets == 0 || geometry.block_count == 0 { + return Err(BlockError::Geometry); + } + let probe_blocks = if block >= PROBE_BYTES { + 1 + } else if PROBE_BYTES % block == 0 { + PROBE_BYTES / block + } else { + return Err(BlockError::Geometry); + }; + let probe_bytes = probe_blocks * block; + let chunk_bytes = max_buffer_bytes.min(CHUNK_CEILING) / probe_bytes * probe_bytes; + if chunk_bytes == 0 { + return Err(BlockError::TooLarge); + } + let chunk_blocks = (chunk_bytes / block) as u64; + let floor = (PROBES * probe_blocks) as u64; + let wanted = geometry + .block_count + .div_ceil(max_buckets as u64) + .max(chunk_blocks) + .max(floor); + let bucket_blocks = wanted.div_ceil(chunk_blocks) * chunk_blocks; + let buckets = geometry.block_count.div_ceil(bucket_blocks); + Ok(Self { + block_bytes: geometry.block_size, + region_blocks: geometry.block_count, + bucket_blocks, + buckets: u32::try_from(buckets).map_err(|_| BlockError::Geometry)?, + chunk_blocks: chunk_blocks as u32, + probe_blocks: probe_blocks as u32, + }) + } + + /// Bucket `index` as `(first block, blocks)`; the last may be short. + pub fn bucket(&self, index: u32) -> (u64, u64) { + let start = u64::from(index) * self.bucket_blocks; + (start, self.bucket_blocks.min(self.region_blocks - start)) + } + + /// Bytes in one full bucket. + pub fn bucket_bytes(&self) -> u64 { + self.bucket_blocks * u64::from(self.block_bytes) + } + + /// Bytes in one full chunk. + pub fn chunk_bytes(&self) -> u32 { + self.chunk_blocks * self.block_bytes + } + + /// Where probe `probe` of bucket `index` reads during scan `scan`: an + /// even stride with a jitter inside it, so successive scans sample + /// different pages and no two probes of one scan share one. + fn probe_lba(&self, index: u32, probe: usize, scan: u32) -> u64 { + let (start, blocks) = self.bucket(index); + let pages = blocks / u64::from(self.probe_blocks); + let stride = (pages / PROBES as u64).max(1); + let slot = (probe as u64 * pages / PROBES as u64).min(pages.saturating_sub(1)); + let room = stride.min(pages - slot); + let jitter = mix(u64::from(scan) << 32 | u64::from(index) << 8 | probe as u64) % room; + start + (slot + jitter) * u64::from(self.probe_blocks) + } +} + +/// SplitMix64's finalizer: a deterministic spread, no state. +const fn mix(mut x: u64) -> u64 { + x = (x ^ (x >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + x = (x ^ (x >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + x ^ (x >> 31) +} + +/// Which step of its cycle a chunk is at. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Step { + /// Read the blocks into the data buffer. + Read, + /// Read them again into the check buffer; the two must agree. + Confirm, + /// Write the data buffer back to the same blocks. + Write, + /// Read them into the check buffer; it must match the data buffer. + Check, +} + +/// One chunk's cycle: a fixed extent and where it is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Chunk { + bucket: u32, + lba: u64, + blocks: u32, + step: Step, + attempts: u8, + in_flight: bool, +} + +impl Chunk { + /// The command this step issues, and whether it lands in the check + /// buffer. Every step names the same blocks: the extent is fixed when + /// the chunk is taken, so a write can only return bytes to the blocks + /// they were read from. + const fn command(&self) -> (DeviceCommand, bool) { + let (lba, blocks, buffer_offset) = (self.lba, self.blocks, 0); + match self.step { + Step::Read => (DeviceCommand::Read { lba, blocks, buffer_offset }, false), + Step::Confirm | Step::Check => (DeviceCommand::Read { lba, blocks, buffer_offset }, true), + Step::Write => (DeviceCommand::Write { lba, blocks, buffer_offset }, false), + } + } +} + +struct Slot { + data: B, + check: B, + chunk: Option, +} + +/// What a refresh rewrites. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Plan { + /// Buckets the last scan judged stale. + Stale, + /// Buckets `first..end`, judged or not. + Range { first: u32, end: u32 }, +} + +#[derive(Clone, Copy, Debug)] +struct ScanCursor { + pass: u32, + bucket: u32, + probe: usize, + staged_at: Option, + samples: [u32; PROBES], + mapped: u8, + failed: u8, +} + +#[derive(Clone, Copy, Debug)] +struct RefreshCursor { + plan: Plan, + /// Next bucket to consider. + next: u32, + /// The bucket being rewritten, its next chunk's block and its end. + current: Option<(u32, u64, u64)>, + /// The bucket's flush is in flight. + flushing: bool, +} + +#[derive(Clone, Copy, Debug)] +enum Work { + None, + Scan(ScanCursor), + Refresh(RefreshCursor), +} + +/// The scan-and-refresh job over one region. See the module docs. +pub struct RegionRefresh<'b, D: DirectBlockDevice> { + device: D, + shape: Shape, + slots: [Slot; SLOTS], + buckets: &'b mut [Bucket], + work: Work, + /// No new commands: the job is draining toward `outcome`. + stopping: Option, + in_flight: u32, + /// Bumped per job: a completion carrying another generation is not + /// this job's, and is counted rather than believed. + generation: u32, + stale_completions: u32, + outcome: Outcome, + scan: ScanSummary, + refresh: RefreshSummary, +} + +impl<'b, D: DirectBlockDevice> RegionRefresh<'b, D> { + /// A job over `device`'s whole extent, keeping one [`Bucket`] per entry + /// of `buckets` at most. Allocates its `2 × SLOTS` chunk buffers now, + /// once: a refusal comes back with the device. + pub fn new(mut device: D, buckets: &'b mut [Bucket]) -> Result { + let shape = match Shape::new(device.geometry(), device.max_buffer_bytes(), buckets.len()) { + Ok(shape) => shape, + Err(error) => return Err((device, error)), + }; + let bytes = shape.chunk_bytes() as usize; + let mut built: [Option>; SLOTS] = core::array::from_fn(|_| None); + for slot in &mut built { + let data = device.allocate_buffer(bytes); + let check = device.allocate_buffer(bytes); + match (data, check) { + (Ok(data), Ok(check)) => { + *slot = Some(Slot { + data, + check, + chunk: None, + }) + } + (Err(error), _) | (_, Err(error)) => return Err((device, error)), + } + } + buckets.fill(Bucket::default()); + Ok(Self { + device, + shape, + // Every slot was built above or the constructor returned. + slots: built.map(|slot| slot.expect("every slot was built")), + buckets, + work: Work::None, + stopping: None, + in_flight: 0, + generation: 0, + stale_completions: 0, + outcome: Outcome::None, + scan: ScanSummary::default(), + refresh: RefreshSummary::default(), + }) + } + + pub fn shape(&self) -> Shape { + self.shape + } + + pub fn phase(&self) -> Phase { + match (self.stopping, self.work) { + (Some(_), _) => Phase::Draining, + (None, Work::None) => Phase::Idle, + (None, Work::Scan(_)) => Phase::Scanning, + (None, Work::Refresh(_)) => Phase::Refreshing, + } + } + + pub fn outcome(&self) -> Outcome { + self.outcome + } + + pub fn scan_summary(&self) -> ScanSummary { + self.scan + } + + pub fn refresh_summary(&self) -> RefreshSummary { + self.refresh + } + + /// The region's buckets, in order. + pub fn buckets(&self) -> &[Bucket] { + &self.buckets[..self.shape.buckets as usize] + } + + /// Commands of the job's on the device. + pub fn in_flight(&self) -> u32 { + self.in_flight + } + + /// Completions that named no command of the running job. + pub fn stale_completions(&self) -> u32 { + self.stale_completions + } + + /// The device, for the owner's control plane. + pub fn device(&self) -> &D { + &self.device + } + + /// Start a scan. Refused while a job runs or without a clock. + pub fn scan(&mut self) -> Result<(), Refusal> { + self.idle_or_busy()?; + if self.device.clock_ns().is_none() { + return Err(Refusal::NoClock); + } + self.begin(); + self.scan = ScanSummary { + scans: self.scan.scans, + pass: 1, + ..ScanSummary::default() + }; + for bucket in self.buckets.iter_mut() { + bucket.stale = false; + } + self.work = Work::Scan(ScanCursor { + pass: 0, + bucket: 0, + probe: 0, + staged_at: None, + samples: [0; PROBES], + mapped: 0, + failed: 0, + }); + Ok(()) + } + + /// Rewrite every bucket the last complete scan judged stale. + pub fn refresh_stale(&mut self) -> Result<(), Refusal> { + self.idle_or_busy()?; + if !self.scan.complete { + return Err(Refusal::NotScanned); + } + self.start_refresh(Plan::Stale); + Ok(()) + } + + /// Rewrite buckets `first..first + count`, judged or not — the forced + /// form, for an operator who knows better than the scan and for the + /// gate that proves the rewrite. Still at most once per bucket. + pub fn refresh_range(&mut self, first: u32, count: u32) -> Result<(), Refusal> { + self.idle_or_busy()?; + let end = first.checked_add(count).ok_or(Refusal::Range)?; + if count == 0 || end > self.shape.buckets { + return Err(Refusal::Range); + } + self.start_refresh(Plan::Range { first, end }); + Ok(()) + } + + /// Stop the running job: nothing new is issued, and it ends + /// [`Outcome::Cancelled`] once the device returned what it holds. + pub fn cancel(&mut self) { + if !matches!(self.work, Work::None) && self.stopping.is_none() { + self.stopping = Some(Outcome::Cancelled); + } + } + + fn idle_or_busy(&self) -> Result<(), Refusal> { + match self.phase() { + Phase::Idle => Ok(()), + _ => Err(Refusal::Busy), + } + } + + fn begin(&mut self) { + self.generation = self.generation.wrapping_add(1); + self.stopping = None; + } + + fn start_refresh(&mut self, plan: Plan) { + self.begin(); + let planned = (0..self.shape.buckets).filter(|&index| self.wanted(plan, index)).count() as u32; + let asked = match plan { + Plan::Stale => planned, + Plan::Range { first, end } => end - first, + }; + self.refresh.buckets_planned = planned; + self.refresh.buckets_done = 0; + self.refresh.buckets_skipped = asked - planned; + let next = match plan { + Plan::Stale => 0, + Plan::Range { first, .. } => first, + }; + self.work = Work::Refresh(RefreshCursor { + plan, + next, + current: None, + flushing: false, + }); + } + + fn wanted(&self, plan: Plan, index: u32) -> bool { + let bucket = &self.buckets[index as usize]; + if bucket.refreshed { + return false; + } + match plan { + Plan::Stale => bucket.stale, + Plan::Range { first, end } => (first..end).contains(&index), + } + } + + fn cookie(&self, tag: u64) -> u64 { + u64::from(self.generation) << TAG_BITS | tag + } + + /// Reap what completed, stage what is next, publish. Never waits. + /// `idle`: nothing but this job is on the device — the owner's word, + /// since only it sees the other parties. Probes are issued only then. + pub fn advance(&mut self, idle: bool) -> Progress { + while self.in_flight != 0 { + let Some(completion) = self.device.poll() else { + break; + }; + self.complete(completion); + } + let staged = if self.stopping.is_some() { + false + } else { + match self.work { + Work::None => false, + Work::Scan(_) => self.stage_probe(idle), + Work::Refresh(_) => self.stage_refresh(), + } + }; + if staged { + self.device.publish(); + } + if self.in_flight == 0 { + self.settle(); + } + match (self.in_flight, self.work) { + (0, Work::None) => Progress::Idle, + (0, _) => Progress::Waiting, + _ => Progress::InFlight, + } + } + + /// With nothing in flight: end a stopped job, or a finished one. + fn settle(&mut self) { + if let Some(outcome) = self.stopping.take() { + self.finish(outcome); + return; + } + match self.work { + Work::Scan(cursor) if cursor.bucket >= self.shape.buckets => { + self.judge(); + self.finish(Outcome::Scanned); + } + Work::Refresh(cursor) + if cursor.current.is_none() && !cursor.flushing && cursor.next >= self.shape.buckets => + { + self.finish(Outcome::Refreshed); + } + _ => {} + } + } + + fn finish(&mut self, outcome: Outcome) { + for slot in &mut self.slots { + slot.chunk = None; + } + self.work = Work::None; + self.outcome = outcome; + } + + fn fault(&mut self, error: BlockError) { + if !matches!(self.stopping, Some(Outcome::Faulted(_))) { + self.stopping = Some(Outcome::Faulted(error)); + } + } + + fn stage(&mut self, command: DeviceCommand, slot: Option<(usize, bool)>, tag: u64) -> bool { + let cookie = self.cookie(tag); + let buffer = slot.map(|(index, check)| { + let slot = &mut self.slots[index]; + if check { &mut slot.check } else { &mut slot.data } + }); + match self.device.stage(command, buffer, cookie) { + Ok(()) => { + self.in_flight += 1; + true + } + // Refused before it reached the device: try again on a later + // wake, when what fills the queue has drained. + Err(_) => false, + } + } + + // ── scan ──────────────────────────────────────────────────────────── + + fn stage_probe(&mut self, idle: bool) -> bool { + let Work::Scan(mut cursor) = self.work else { + return false; + }; + if cursor.staged_at.is_some() || cursor.bucket >= self.shape.buckets || !idle { + return false; + } + let Some(now) = self.device.clock_ns() else { + self.fault(BlockError::Unsupported); + return false; + }; + let lba = self.shape.probe_lba(cursor.bucket, cursor.probe, self.scan.scans); + let command = DeviceCommand::Read { + lba, + blocks: self.shape.probe_blocks, + buffer_offset: 0, + }; + if !self.stage(command, Some((0, false)), PROBE_TAG) { + return false; + } + cursor.staged_at = Some(now); + self.work = Work::Scan(cursor); + true + } + + fn probed(&mut self, result: Result<(), BlockError>) { + let Work::Scan(mut cursor) = self.work else { + return; + }; + let Some(staged_at) = cursor.staged_at.take() else { + return; + }; + match result { + Ok(()) => { + let bytes = (self.shape.probe_blocks * self.shape.block_bytes) as usize; + if !is_zero(&self.slots[0].data.bytes()[..bytes]) { + let elapsed = self.device.clock_ns().unwrap_or(staged_at).saturating_sub(staged_at); + cursor.samples[cursor.mapped as usize] = u32::try_from(elapsed).unwrap_or(u32::MAX); + cursor.mapped += 1; + } + } + Err(_) => { + cursor.failed += 1; + self.scan.failed_probes += 1; + } + } + cursor.probe += 1; + if cursor.probe == PROBES { + let samples = &mut cursor.samples[..cursor.mapped as usize]; + samples.sort_unstable(); + let median = samples.get(samples.len() / 2).copied(); + let bucket = &mut self.buckets[cursor.bucket as usize]; + // The first pass sets the books; a later pass can only lower the + // estimate. A pass whose probes all read zeros or failed has no + // median and leaves it where it was. + if cursor.pass == 0 { + bucket.median_ns = median.unwrap_or(0); + bucket.mapped_probes = cursor.mapped; + bucket.failed_probes = cursor.failed; + } else { + if let Some(median) = median { + bucket.median_ns = if bucket.median_ns == 0 { median } else { bucket.median_ns.min(median) }; + } + bucket.mapped_probes = bucket.mapped_probes.max(cursor.mapped); + bucket.failed_probes = bucket.failed_probes.max(cursor.failed); + } + self.scan.scanned += 1; + let (pass, bucket) = if cursor.bucket + 1 < self.shape.buckets || cursor.pass + 1 == PASSES { + (cursor.pass, cursor.bucket + 1) + } else { + self.scan.pass += 1; + self.scan.scanned = 0; + (cursor.pass + 1, 0) + }; + cursor = ScanCursor { + pass, + bucket, + probe: 0, + staged_at: None, + samples: [0; PROBES], + mapped: 0, + failed: 0, + }; + } + self.work = Work::Scan(cursor); + } + + /// The verdict on a finished scan: percentiles of the mapped buckets + /// and the stale marks. + fn judge(&mut self) { + let count = self.shape.buckets as usize; + let mapped = self.buckets[..count].iter().filter(|bucket| bucket.mapped()).count() as u32; + self.scan.mapped = mapped; + self.scan.complete = true; + self.scan.scans += 1; + if mapped == 0 { + return; + } + let p25 = self.percentile(25); + self.scan.p25_ns = p25; + self.scan.p50_ns = self.percentile(50); + self.scan.p90_ns = self.percentile(90); + if mapped < MIN_BASELINE { + return; + } + let threshold = u64::from(p25) * STALE_RATIO; + let mut stale = 0; + for bucket in self.buckets[..count].iter_mut() { + bucket.stale = bucket.mapped() && !bucket.refreshed && u64::from(bucket.median_ns) >= threshold; + stale += u32::from(bucket.stale); + } + self.scan.stale = stale; + } + + /// The smallest mapped median with at least `percent` of the mapped + /// medians at or below it. A binary search over the value range rather + /// than a sort: the books stay in place and no bucket-sized scratch + /// array lands on a stack the owner may have little of. + fn percentile(&self, percent: u64) -> u32 { + let medians = || { + self.buckets[..self.shape.buckets as usize] + .iter() + .filter(|bucket| bucket.mapped()) + .map(|bucket| bucket.median_ns) + }; + let total = medians().count() as u64; + let need = (total * percent).div_ceil(100).max(1); + let (mut low, mut high) = (0u32, u32::MAX); + while low < high { + let mid = low + (high - low) / 2; + if medians().filter(|&median| median <= mid).count() as u64 >= need { + high = mid; + } else { + low = mid + 1; + } + } + low + } + + // ── refresh ───────────────────────────────────────────────────────── + + fn stage_refresh(&mut self) -> bool { + let Work::Refresh(mut cursor) = self.work else { + return false; + }; + let mut staged = false; + // Restage chunks whose last step completed (or was refused). + for index in 0..SLOTS { + if let Some(chunk) = self.slots[index].chunk + && !chunk.in_flight + { + staged |= self.stage_chunk(index, chunk); + } + } + if cursor.flushing { + return staged; + } + // Take new chunks: the current bucket's next, else the next bucket. + loop { + if cursor.current.is_none() { + while cursor.next < self.shape.buckets && !self.wanted(cursor.plan, cursor.next) { + cursor.next += 1; + } + if cursor.next >= self.shape.buckets { + break; + } + let (start, blocks) = self.shape.bucket(cursor.next); + cursor.current = Some((cursor.next, start, start + blocks)); + cursor.next += 1; + } + let Some((bucket, lba, end)) = cursor.current else { + break; + }; + if lba >= end { + // Every chunk is taken. Once the last one lands, the flush. + if self.slots.iter().any(|slot| slot.chunk.is_some()) { + break; + } + if self.stage(DeviceCommand::Flush, None, FLUSH_TAG) { + cursor.flushing = true; + staged = true; + } + break; + } + let Some(free) = self.slots.iter().position(|slot| slot.chunk.is_none()) else { + break; + }; + let blocks = (end - lba).min(u64::from(self.shape.chunk_blocks)) as u32; + let chunk = Chunk { + bucket, + lba, + blocks, + step: Step::Read, + attempts: 0, + in_flight: false, + }; + self.slots[free].chunk = Some(chunk); + cursor.current = Some((bucket, lba + u64::from(blocks), end)); + staged |= self.stage_chunk(free, chunk); + } + self.work = Work::Refresh(cursor); + staged + } + + fn stage_chunk(&mut self, index: usize, mut chunk: Chunk) -> bool { + let (command, into_check) = chunk.command(); + if !self.stage(command, Some((index, into_check)), index as u64) { + return false; + } + chunk.in_flight = true; + self.slots[index].chunk = Some(chunk); + true + } + + fn flushed(&mut self, result: Result<(), BlockError>) { + let Work::Refresh(mut cursor) = self.work else { + return; + }; + cursor.flushing = false; + if let Err(error) = result { + self.work = Work::Refresh(cursor); + self.fault(error); + return; + } + if let Some((bucket, _, _)) = cursor.current.take() { + let bucket = &mut self.buckets[bucket as usize]; + bucket.refreshed = true; + bucket.stale = false; + self.refresh.buckets_done += 1; + self.refresh.buckets_lifetime += 1; + } + self.work = Work::Refresh(cursor); + } + + fn chunk_done(&mut self, index: usize, result: Result<(), BlockError>) { + let Some(mut chunk) = self.slots[index].chunk else { + return; + }; + chunk.in_flight = false; + let slot = &self.slots[index]; + let bytes = (chunk.blocks * self.shape.block_bytes) as usize; + let next = match (chunk.step, result) { + (Step::Read, Ok(())) if is_zero(&slot.data.bytes()[..bytes]) => { + self.refresh.zero_chunks += 1; + None + } + (Step::Read, Ok(())) => Some((Step::Confirm, chunk.attempts)), + (Step::Read | Step::Confirm, Err(_)) => { + self.refresh.read_errors += 1; + self.buckets[chunk.bucket as usize].incomplete = true; + None + } + (Step::Confirm, Ok(())) if slot.data.bytes()[..bytes] == slot.check.bytes()[..bytes] => { + Some((Step::Write, 0)) + } + // Two reads of the same blocks disagreed: the device is not + // giving one answer, so there is nothing safe to write. Read + // both again once; then leave the chunk exactly as it is. + (Step::Confirm, Ok(())) if chunk.attempts + 1 < ATTEMPTS => Some((Step::Read, chunk.attempts + 1)), + (Step::Confirm, Ok(())) => { + self.refresh.unstable_chunks += 1; + self.buckets[chunk.bucket as usize].incomplete = true; + None + } + (Step::Write, Ok(())) => Some((Step::Check, chunk.attempts)), + (Step::Check, Ok(())) if slot.data.bytes()[..bytes] == slot.check.bytes()[..bytes] => { + self.refresh.bytes += bytes as u64; + None + } + // The write did not stick, or a check after it disagreed. The + // data buffer still holds the agreed bytes: write them again, + // once. + (Step::Write | Step::Check, outcome) if chunk.attempts + 1 < ATTEMPTS => { + if outcome.is_ok() { + self.refresh.mismatches += 1; + } + self.refresh.write_retries += 1; + Some((Step::Write, chunk.attempts + 1)) + } + (Step::Write | Step::Check, outcome) => { + if outcome.is_ok() { + self.refresh.mismatches += 1; + } + self.slots[index].chunk = None; + self.fault(outcome.err().unwrap_or(BlockError::Integrity)); + return; + } + }; + self.slots[index].chunk = next.map(|(step, attempts)| Chunk { step, attempts, ..chunk }); + } + + fn complete(&mut self, completion: DeviceCompletion) { + if completion.cookie >> TAG_BITS != u64::from(self.generation) { + self.stale_completions += 1; + return; + } + self.in_flight = self.in_flight.saturating_sub(1); + match completion.cookie & TAG_MASK { + PROBE_TAG => self.probed(completion.result), + FLUSH_TAG => self.flushed(completion.result), + tag if (tag as usize) < SLOTS => self.chunk_done(tag as usize, completion.result), + _ => self.stale_completions += 1, + } + } + + // ── retirement ────────────────────────────────────────────────────── + + /// Cancel and wait out what the device holds, through the device's own + /// bounded [`wait`](DirectBlockDevice::wait). `true` once nothing of + /// the job's is in flight. + pub fn drain(&mut self) -> bool { + self.cancel(); + while self.in_flight != 0 { + let Some(completion) = self.device.wait() else { + break; + }; + self.complete(completion); + } + if self.in_flight == 0 { + self.settle(); + } + self.in_flight == 0 + } + + /// The device back, once [`drain`](Self::drain) has emptied the job. + /// Refused, the job handed back, while a command is in flight. + pub fn into_device(self) -> Result { + if self.in_flight != 0 { + return Err(self); + } + Ok(self.device) + } +} + +/// Whether every byte is zero: an OR-fold, which vectorizes, rather than an +/// early-exit scan, which does not. +fn is_zero(bytes: &[u8]) -> bool { + // SAFETY: every bit pattern is a valid `u64`; `align_to` hands back + // only whole, aligned words between the unaligned head and tail. + let (head, words, tail) = unsafe { bytes.align_to::() }; + let fold = words.iter().fold(0u64, |acc, word| acc | word); + fold == 0 && head.iter().chain(tail).all(|byte| *byte == 0) +} + +#[cfg(test)] +mod tests { + extern crate std; + + use std::boxed::Box; + use std::collections::VecDeque; + use std::vec::Vec; + + use super::*; + use crate::block::memory::{MemoryBlockDevice, MemoryBuffer}; + + const BLOCK: u32 = 512; + /// 512 KiB region of 512-byte blocks; 8 KiB buffers, so 8 KiB chunks. + const DISK: usize = 512 << 10; + const BUF: usize = 8 << 10; + const BLOCKS: u64 = (DISK / BLOCK as usize) as u64; + /// The shape these sizes give: eight buckets of 64 KiB (the floor of + /// sixteen 4 KiB probe pages), eight chunks each. + const BUCKETS: u32 = 8; + const BUCKET_BLOCKS: u64 = 128; + const CHUNK_BLOCKS: u64 = 16; + const CACHED_NS: u64 = 47_000; + const FOLDED_NS: u64 = 113_000; + const DECAYED_NS: u64 = 900_000; + /// What a block inside `cold` costs on its first read only: the cold + /// mapping cache Biggertron's first scan after boot paid, and anything + /// else that slows a read once and not again. + const COLD_NS: u64 = 1_500_000; + + /// A drive model over the memory device: every block starts folded; + /// blocks inside `decayed` read slower; a block written since the model + /// was built reads from the cache. The clock advances by each + /// completion's latency as it is polled. + struct Drive { + inner: Box>, + now: u64, + pending: VecDeque, + decayed: core::ops::Range, + cold: core::ops::Range, + touched: Vec, + written: Vec, + /// Reads starting at this block come back with their first byte + /// changed — differently each time — this many more times. + flaky: Option<(u64, u32)>, + flips: u8, + /// Writes starting at this block land with a byte flipped on the + /// medium, this many more times. + corrupt: Option<(u64, u32)>, + /// Writes starting at this block fail, this many more times. + refuse: Option<(u64, u32)>, + flushes: u64, + } + + fn take(target: &mut Option<(u64, u32)>, lba: u64) -> bool { + match target { + Some((at, left)) if *at == lba && *left > 0 => { + *left -= 1; + true + } + _ => false, + } + } + + impl Drive { + fn new() -> Self { + Self { + inner: Box::new(MemoryBlockDevice::new(BLOCK, 1024).unwrap()), + now: 1_000_000, + pending: VecDeque::new(), + decayed: 0..0, + cold: 0..0, + touched: std::vec![false; BLOCKS as usize], + written: std::vec![false; BLOCKS as usize], + flaky: None, + flips: 0, + corrupt: None, + refuse: None, + flushes: 0, + } + } + + /// Fill blocks `[0, end)` with a pattern of non-zero bytes; the + /// rest stays zero, the never-written posture. + fn fill(&mut self, end: u64) { + let mut buffer = self.inner.allocate_buffer(BLOCK as usize).unwrap(); + for lba in 0..end.min(BLOCKS) { + for (i, byte) in buffer.bytes_mut().iter_mut().enumerate() { + *byte = (mix(lba * 512 + i as u64) as u8) | 1; + } + let write = DeviceCommand::Write { lba, blocks: 1, buffer_offset: 0 }; + self.inner.submit(write, Some(&mut buffer), 0).unwrap(); + self.inner.poll().unwrap().result.unwrap(); + } + } + + /// The medium's bytes, read around the model. + fn image(&mut self) -> Vec { + let mut out = Vec::with_capacity(DISK); + let mut buffer = self.inner.allocate_buffer(BUF).unwrap(); + let per = (BUF / BLOCK as usize) as u32; + for lba in (0..BLOCKS).step_by(per as usize) { + let read = DeviceCommand::Read { lba, blocks: per, buffer_offset: 0 }; + self.inner.submit(read, Some(&mut buffer), 0).unwrap(); + self.inner.poll().unwrap().result.unwrap(); + out.extend_from_slice(buffer.bytes()); + } + out + } + + fn latency(&mut self, lba: u64, blocks: u32) -> u64 { + let end = lba + u64::from(blocks); + let first = !self.touched[lba as usize]; + self.touched[lba as usize..end as usize].fill(true); + let cold = if first && self.cold.contains(&lba) { COLD_NS } else { 0 }; + cold + if self.written[lba as usize..end as usize].iter().all(|written| *written) { + CACHED_NS + } else if self.decayed.contains(&lba) { + DECAYED_NS + } else { + FOLDED_NS + } + } + + fn written_in(&self, range: core::ops::Range) -> usize { + self.written[range.start as usize..range.end as usize].iter().filter(|w| **w).count() + } + } + + impl DirectBlockDevice for Drive { + type Buffer = MemoryBuffer; + const INDEPENDENT_PROGRESS: bool = true; + + fn geometry(&self) -> BlockGeometry { + self.inner.geometry() + } + fn max_buffer_bytes(&self) -> usize { + BUF + } + fn clock_ns(&self) -> Option { + Some(self.now) + } + fn allocate_buffer(&mut self, bytes: usize) -> Result { + self.inner.allocate_buffer(bytes) + } + fn submit( + &mut self, + command: DeviceCommand, + mut buffer: Option<&mut Self::Buffer>, + cookie: u64, + ) -> Result<(), BlockError> { + let latency = match command { + DeviceCommand::Read { lba, blocks, .. } => self.latency(lba, blocks), + _ => CACHED_NS, + }; + match command { + DeviceCommand::Write { lba, .. } if take(&mut self.refuse, lba) => { + self.inner.inject_completion(cookie, Err(BlockError::Media))?; + } + _ => self.inner.submit(command, buffer.as_deref_mut(), cookie)?, + } + match command { + DeviceCommand::Read { lba, buffer_offset, .. } if take(&mut self.flaky, lba) => { + self.flips = self.flips.wrapping_add(1).max(1); + buffer.unwrap().bytes_mut()[buffer_offset] ^= self.flips; + } + DeviceCommand::Write { lba, blocks, .. } => { + for block in lba..lba + u64::from(blocks) { + self.written[block as usize] = true; + } + if take(&mut self.corrupt, lba) { + self.inner.corrupt_byte((lba * u64::from(BLOCK)) as usize, 0x5a).unwrap(); + } + } + DeviceCommand::Flush => self.flushes += 1, + _ => {} + } + self.pending.push_back(latency); + Ok(()) + } + fn poll(&mut self) -> Option { + let completion = self.inner.poll()?; + self.now += self.pending.pop_front().unwrap_or(0); + Some(completion) + } + fn quiesce(&mut self) -> Result<(), BlockError> { + self.inner.quiesce() + } + } + + /// The drive model's medium and the job's chunk buffers are inline + /// arrays; a debug build moves them by value more than once, past a + /// test thread's default stack. + fn on_big_stack(body: fn()) { + std::thread::Builder::new() + .stack_size(64 << 20) + .spawn(body) + .expect("spawn") + .join() + .expect("the test body passed"); + } + + fn run(job: &mut RegionRefresh<'_, Drive>) { + for _ in 0..100_000 { + if job.advance(true) == Progress::Idle { + return; + } + } + panic!("the job never went idle"); + } + + fn table() -> Vec { + std::vec![Bucket::default(); 16] + } + + fn build(drive: Drive, books: &mut [Bucket]) -> RegionRefresh<'_, Drive> { + RegionRefresh::new(drive, books).ok().expect("the job builds") + } + + fn marked(job: &RegionRefresh<'_, Drive>, pick: impl Fn(&Bucket) -> bool) -> Vec { + job.buckets().iter().enumerate().filter(|(_, b)| pick(b)).map(|(i, _)| i).collect() + } + + fn bucket(index: u64) -> core::ops::Range { + index * BUCKET_BLOCKS..(index + 1) * BUCKET_BLOCKS + } + + #[test] + fn shape_buckets_the_region_in_whole_chunks() { + on_big_stack(shape_buckets_the_region_in_whole_chunks_body); + } + + fn shape_buckets_the_region_in_whole_chunks_body() { + let mut books = table(); + let job = build(Drive::new(), &mut books); + let shape = job.shape(); + assert_eq!(shape.probe_blocks, 8); + assert_eq!(u64::from(shape.chunk_blocks), CHUNK_BLOCKS); + assert_eq!(shape.bucket_blocks, BUCKET_BLOCKS); + assert_eq!(shape.buckets, BUCKETS); + // Every probe lands inside its own bucket, on a page boundary, and + // no two probes of one scan share a page. + for index in 0..shape.buckets { + let (start, blocks) = shape.bucket(index); + let mut seen = Vec::new(); + for probe in 0..PROBES { + let lba = shape.probe_lba(index, probe, 3); + assert!(lba >= start && lba + u64::from(shape.probe_blocks) <= start + blocks); + assert_eq!(lba % u64::from(shape.probe_blocks), 0); + assert!(!seen.contains(&lba)); + seen.push(lba); + } + } + } + + #[test] + fn a_short_region_keeps_a_short_last_bucket() { + on_big_stack(a_short_region_keeps_a_short_last_bucket_body); + } + + fn a_short_region_keeps_a_short_last_bucket_body() { + let geometry = BlockGeometry { + block_count: 1000, + ..Drive::new().geometry() + }; + let shape = Shape::new(geometry, BUF, 4).unwrap(); + assert_eq!(shape.bucket_blocks, 256); + assert_eq!(shape.buckets, 4); + let (start, blocks) = shape.bucket(3); + assert_eq!((start, blocks), (768, 232)); + // A bucket table smaller than the region asks for wider buckets. + let wide = Shape::new(geometry, BUF, 2).unwrap(); + assert_eq!((wide.bucket_blocks, wide.buckets), (512, 2)); + // A probe page larger than the buffer cannot be chunked. + assert_eq!(Shape::new(geometry, 2048, 4), Err(BlockError::TooLarge)); + } + + #[test] + fn folded_buckets_are_healthy_and_decayed_ones_are_stale() { + on_big_stack(folded_buckets_are_healthy_and_decayed_ones_are_stale_body); + } + + fn folded_buckets_are_healthy_and_decayed_ones_are_stale_body() { + let mut drive = Drive::new(); + drive.fill(BLOCKS); + drive.decayed = bucket(5).start..bucket(6).end; + let mut books = table(); + let mut job = build(drive, &mut books); + job.scan().unwrap(); + run(&mut job); + assert_eq!(job.outcome(), Outcome::Scanned); + let scan = job.scan_summary(); + assert_eq!((scan.scanned, scan.mapped, scan.stale, scan.scans), (8, 8, 2, 1)); + assert_eq!(u64::from(scan.p25_ns), FOLDED_NS); + assert_eq!(u64::from(scan.p90_ns), DECAYED_NS); + assert_eq!(marked(&job, |b| b.stale), [5, 6]); + } + + #[test] + fn slowness_that_does_not_repeat_is_not_decay() { + on_big_stack(slowness_that_does_not_repeat_is_not_decay_body); + } + + fn slowness_that_does_not_repeat_is_not_decay_body() { + let mut drive = Drive::new(); + drive.fill(BLOCKS); + // Buckets 1 and 4 read slowly once, as Biggertron's did right after + // boot; bucket 6 has decayed and reads slowly every time. + drive.cold = bucket(1).start..bucket(5).end; + drive.decayed = bucket(6); + let mut books = table(); + let mut job = build(drive, &mut books); + job.scan().unwrap(); + run(&mut job); + let scan = job.scan_summary(); + assert_eq!((scan.pass, scan.scanned), (PASSES, BUCKETS)); + assert_eq!(marked(&job, |b| b.stale), [6]); + // Every cold bucket is judged by its warm passes. + assert!(job.buckets()[1..5].iter().all(|b| u64::from(b.median_ns) == FOLDED_NS)); + assert_eq!(u64::from(scan.p25_ns), FOLDED_NS); + } + + #[test] + fn a_cached_quartile_does_not_condemn_folded_buckets() { + on_big_stack(a_cached_quartile_does_not_condemn_folded_buckets_body); + } + + fn a_cached_quartile_does_not_condemn_folded_buckets_body() { + let mut drive = Drive::new(); + drive.fill(BLOCKS); + // Three buckets sit in the cache: the quartile is cached. + drive.written[..3 * BUCKET_BLOCKS as usize].fill(true); + let mut books = table(); + let mut job = build(drive, &mut books); + job.scan().unwrap(); + run(&mut job); + let scan = job.scan_summary(); + assert_eq!(u64::from(scan.p25_ns), CACHED_NS); + assert_eq!(scan.stale, 0, "folded is 2.4x cached, under the 4x ratio"); + } + + #[test] + fn never_written_buckets_are_unmapped_and_never_judged() { + on_big_stack(never_written_buckets_are_unmapped_and_never_judged_body); + } + + fn never_written_buckets_are_unmapped_and_never_judged_body() { + let mut drive = Drive::new(); + drive.fill(6 * BUCKET_BLOCKS); + drive.decayed = bucket(2); + let mut books = table(); + let mut job = build(drive, &mut books); + job.scan().unwrap(); + run(&mut job); + let scan = job.scan_summary(); + assert_eq!(scan.mapped, 6); + assert!(job.buckets()[6..].iter().all(|b| !b.mapped() && !b.stale)); + assert_eq!(marked(&job, |b| b.stale), [2]); + } + + #[test] + fn too_few_mapped_buckets_judge_nothing() { + on_big_stack(too_few_mapped_buckets_judge_nothing_body); + } + + fn too_few_mapped_buckets_judge_nothing_body() { + let mut drive = Drive::new(); + drive.fill(3 * BUCKET_BLOCKS); + drive.decayed = bucket(0); + let mut books = table(); + let mut job = build(drive, &mut books); + job.scan().unwrap(); + run(&mut job); + assert_eq!(job.scan_summary().mapped, 3); + assert_eq!(job.scan_summary().stale, 0); + } + + #[test] + fn refresh_rewrites_only_stale_buckets_and_preserves_every_byte() { + on_big_stack(refresh_rewrites_only_stale_buckets_and_preserves_every_byte_body); + } + + fn refresh_rewrites_only_stale_buckets_and_preserves_every_byte_body() { + let mut drive = Drive::new(); + drive.fill(7 * BUCKET_BLOCKS); + drive.decayed = bucket(3).start..bucket(4).end; + let before = drive.image(); + let mut books = table(); + let mut job = build(drive, &mut books); + assert_eq!(job.refresh_stale(), Err(Refusal::NotScanned)); + job.scan().unwrap(); + run(&mut job); + job.refresh_stale().unwrap(); + run(&mut job); + assert_eq!(job.outcome(), Outcome::Refreshed); + let refresh = job.refresh_summary(); + assert_eq!((refresh.buckets_planned, refresh.buckets_done), (2, 2)); + assert_eq!(refresh.bytes, 2 * BUCKET_BLOCKS * u64::from(BLOCK)); + assert_eq!(refresh.mismatches + refresh.unstable_chunks + refresh.read_errors, 0); + assert_eq!(marked(&job, |b| b.refreshed), [3, 4]); + assert!(marked(&job, |b| b.stale).is_empty()); + let mut drive = job.into_device().ok().unwrap(); + // One flush per bucket; nothing outside the two buckets written. + assert_eq!(drive.flushes, 2); + assert_eq!(drive.written_in(0..bucket(3).start) + drive.written_in(bucket(5).start..BLOCKS), 0); + assert_eq!(drive.written_in(bucket(3).start..bucket(5).start), 2 * BUCKET_BLOCKS as usize); + assert!(drive.image() == before, "a refresh changed a byte"); + } + + #[test] + fn a_rescan_after_refresh_finds_the_bucket_cached() { + on_big_stack(a_rescan_after_refresh_finds_the_bucket_cached_body); + } + + fn a_rescan_after_refresh_finds_the_bucket_cached_body() { + let mut drive = Drive::new(); + drive.fill(BLOCKS); + drive.decayed = bucket(0); + let mut books = table(); + let mut job = build(drive, &mut books); + job.scan().unwrap(); + run(&mut job); + job.refresh_stale().unwrap(); + run(&mut job); + job.scan().unwrap(); + run(&mut job); + assert_eq!(u64::from(job.buckets()[0].median_ns), CACHED_NS); + assert_eq!(job.scan_summary().stale, 0); + assert_eq!(job.scan_summary().scans, 2); + } + + #[test] + fn a_bucket_is_rewritten_at_most_once_per_job() { + on_big_stack(a_bucket_is_rewritten_at_most_once_per_job_body); + } + + fn a_bucket_is_rewritten_at_most_once_per_job_body() { + let mut drive = Drive::new(); + drive.fill(BLOCKS); + let mut books = table(); + let mut job = build(drive, &mut books); + job.refresh_range(2, 3).unwrap(); + run(&mut job); + assert_eq!(job.refresh_summary().buckets_done, 3); + job.refresh_range(0, 6).unwrap(); + run(&mut job); + let refresh = job.refresh_summary(); + assert_eq!((refresh.buckets_planned, refresh.buckets_done, refresh.buckets_skipped), (3, 3, 3)); + assert_eq!(refresh.buckets_lifetime, 6); + assert_eq!(job.refresh_range(7, 2), Err(Refusal::Range)); + assert_eq!(job.refresh_range(0, 0), Err(Refusal::Range)); + assert_eq!(job.refresh_range(u32::MAX, 2), Err(Refusal::Range)); + } + + #[test] + fn zero_chunks_are_never_written() { + on_big_stack(zero_chunks_are_never_written_body); + } + + fn zero_chunks_are_never_written_body() { + let mut drive = Drive::new(); + // Bucket 3 is half data, half zeros. + drive.fill(bucket(3).start + BUCKET_BLOCKS / 2); + let mut books = table(); + let mut job = build(drive, &mut books); + job.refresh_range(3, 1).unwrap(); + run(&mut job); + let refresh = job.refresh_summary(); + assert_eq!(refresh.zero_chunks, 4); + assert_eq!(refresh.bytes, BUCKET_BLOCKS / 2 * u64::from(BLOCK)); + let drive = job.into_device().ok().unwrap(); + assert_eq!(drive.written_in(bucket(3).start + BUCKET_BLOCKS / 2..bucket(3).end), 0); + } + + #[test] + fn a_read_that_disagrees_once_is_read_again() { + on_big_stack(a_read_that_disagrees_once_is_read_again_body); + } + + fn a_read_that_disagrees_once_is_read_again_body() { + let mut drive = Drive::new(); + drive.fill(BLOCKS); + drive.flaky = Some((0, 1)); + let before = drive.image(); + let mut books = table(); + let mut job = build(drive, &mut books); + job.refresh_range(0, 1).unwrap(); + run(&mut job); + assert_eq!(job.refresh_summary().unstable_chunks, 0); + assert_eq!(job.refresh_summary().bytes, BUCKET_BLOCKS * u64::from(BLOCK)); + let mut drive = job.into_device().ok().unwrap(); + assert!(drive.image() == before); + } + + #[test] + fn reads_that_never_agree_leave_the_chunk_untouched() { + on_big_stack(reads_that_never_agree_leave_the_chunk_untouched_body); + } + + fn reads_that_never_agree_leave_the_chunk_untouched_body() { + let mut drive = Drive::new(); + drive.fill(BLOCKS); + let first = bucket(1).start; + drive.flaky = Some((first, u32::MAX)); + let before = drive.image(); + let mut books = table(); + let mut job = build(drive, &mut books); + job.refresh_range(1, 1).unwrap(); + run(&mut job); + let refresh = job.refresh_summary(); + assert_eq!(refresh.unstable_chunks, 1); + assert!(job.buckets()[1].incomplete && job.buckets()[1].refreshed); + assert_eq!(refresh.bytes, (BUCKET_BLOCKS - CHUNK_BLOCKS) * u64::from(BLOCK)); + let mut drive = job.into_device().ok().unwrap(); + assert_eq!(drive.written_in(first..first + CHUNK_BLOCKS), 0, "an unstable chunk was written"); + assert!(drive.image() == before); + } + + #[test] + fn a_write_that_does_not_stick_is_written_again() { + on_big_stack(a_write_that_does_not_stick_is_written_again_body); + } + + fn a_write_that_does_not_stick_is_written_again_body() { + let mut drive = Drive::new(); + drive.fill(BLOCKS); + drive.corrupt = Some((0, 1)); + let before = drive.image(); + let mut books = table(); + let mut job = build(drive, &mut books); + job.refresh_range(0, 1).unwrap(); + run(&mut job); + let refresh = job.refresh_summary(); + assert_eq!(job.outcome(), Outcome::Refreshed); + assert_eq!((refresh.mismatches, refresh.write_retries), (1, 1)); + let mut drive = job.into_device().ok().unwrap(); + assert!(drive.image() == before, "the second write restored the agreed bytes"); + } + + #[test] + fn a_write_that_never_sticks_faults_the_job() { + on_big_stack(a_write_that_never_sticks_faults_the_job_body); + } + + fn a_write_that_never_sticks_faults_the_job_body() { + let mut drive = Drive::new(); + drive.fill(BLOCKS); + drive.corrupt = Some((bucket(1).start, 2)); + let mut books = table(); + let mut job = build(drive, &mut books); + job.refresh_range(1, 1).unwrap(); + run(&mut job); + assert_eq!(job.outcome(), Outcome::Faulted(BlockError::Integrity)); + assert!(!job.buckets()[1].refreshed); + assert_eq!(job.refresh_summary().mismatches, 2); + } + + #[test] + fn a_refused_write_is_retried_then_faults() { + on_big_stack(a_refused_write_is_retried_then_faults_body); + } + + fn a_refused_write_is_retried_then_faults_body() { + let mut drive = Drive::new(); + drive.fill(BLOCKS); + drive.refuse = Some((0, 1)); + let mut books = table(); + let mut job = build(drive, &mut books); + job.refresh_range(0, 1).unwrap(); + run(&mut job); + assert_eq!(job.outcome(), Outcome::Refreshed); + assert_eq!(job.refresh_summary().write_retries, 1); + + let mut drive = job.into_device().ok().unwrap(); + drive.refuse = Some((bucket(2).start, 2)); + let mut books = table(); + let mut job = build(drive, &mut books); + job.refresh_range(2, 1).unwrap(); + run(&mut job); + assert_eq!(job.outcome(), Outcome::Faulted(BlockError::Media)); + assert!(!job.buckets()[2].refreshed); + } + + #[test] + fn a_failed_read_leaves_its_chunk_and_the_bucket_completes() { + on_big_stack(a_failed_read_leaves_its_chunk_and_the_bucket_completes_body); + } + + fn a_failed_read_leaves_its_chunk_and_the_bucket_completes_body() { + let mut drive = Drive::new(); + drive.fill(BLOCKS); + // The job's first command is chunk zero's read. + drive.inner.inject(BlockError::Media); + let mut books = table(); + let mut job = build(drive, &mut books); + job.refresh_range(0, 1).unwrap(); + run(&mut job); + assert_eq!(job.outcome(), Outcome::Refreshed); + assert_eq!(job.refresh_summary().read_errors, 1); + assert!(job.buckets()[0].incomplete && job.buckets()[0].refreshed); + let drive = job.into_device().ok().unwrap(); + assert_eq!(drive.written_in(0..CHUNK_BLOCKS), 0); + } + + #[test] + fn probes_wait_for_an_idle_device_and_cancel_drains() { + on_big_stack(probes_wait_for_an_idle_device_and_cancel_drains_body); + } + + fn probes_wait_for_an_idle_device_and_cancel_drains_body() { + let mut drive = Drive::new(); + drive.fill(BLOCKS); + let mut books = table(); + let mut job = build(drive, &mut books); + job.scan().unwrap(); + assert_eq!(job.advance(false), Progress::Waiting); + assert_eq!(job.in_flight(), 0); + assert_eq!(job.advance(true), Progress::InFlight); + assert_eq!(job.scan(), Err(Refusal::Busy)); + job.cancel(); + assert_eq!(job.phase(), Phase::Draining); + assert_eq!(job.advance(true), Progress::Idle); + assert_eq!(job.outcome(), Outcome::Cancelled); + assert!(!job.scan_summary().complete); + // A cancelled job drains the device and hands it back. + job.refresh_range(0, 2).unwrap(); + assert_eq!(job.advance(true), Progress::InFlight); + assert!(job.drain()); + assert_eq!(job.outcome(), Outcome::Cancelled); + assert!(job.into_device().is_ok()); + } + + #[test] + fn is_zero_folds_every_alignment() { + let mut bytes = [0u8; 67]; + for start in 0..8 { + assert!(is_zero(&bytes[start..])); + } + for at in [0, 3, 9, 33, 66] { + bytes[at] = 1; + assert!(!is_zero(&bytes)); + bytes[at] = 0; + } + } +} diff --git a/native/crates/driver/src/host.rs b/native/crates/driver/src/host.rs index 8ab8359..5c88836 100644 --- a/native/crates/driver/src/host.rs +++ b/native/crates/driver/src/host.rs @@ -301,6 +301,15 @@ pub enum ClassStage { } impl IntakeError { + /// A committed CPU-view refusal has not reached the device-publish step + /// of a DMA import. Earlier CPU/sublease owners retain their normal + /// retirement path, with no GPU translation to contain. A code-zero map + /// fault can mean uncertain transport; all other intake failures remain + /// conservatively uncertain here. + pub const fn failed_before_device_publish(self) -> bool { + matches!(self, Self::DmaMap { code } if code < 0x80 && code & 7 == 4 && code >> 3 >= 1 && code >> 3 <= 6) + } + /// Stable, allocation-free hardware diagnostic. Low byte is the intake /// class; upper bytes retain the class-specific stage/status detail. pub const fn diagnostic_code(self) -> u64 { @@ -3432,6 +3441,22 @@ fn validate_declaration() -> Result<(u8, Option::new(); diff --git a/native/crates/driver/src/ring.rs b/native/crates/driver/src/ring.rs index bba83db..07232db 100644 --- a/native/crates/driver/src/ring.rs +++ b/native/crates/driver/src/ring.rs @@ -8,8 +8,9 @@ //! Ordering (easy for a unit double to forgive, load-bearing on hardware): a staged //! entry is published to the device with a **release** fence before the //! doorbell ([`SubRing::ring`]); a completion read takes an **acquire** -//! fence after a matching validity bit ([`CompRing::poll`]), so the rest of -//! the entry is observed coherently. +//! fence after a matching validity bit ([`CompRing::poll`]), then reads the +//! published entry again. Payload copied before the validity observation may +//! belong to the previous generation and is never returned. use core::marker::PhantomData; use core::sync::atomic::{Ordering, fence}; @@ -32,6 +33,8 @@ pub trait SqEntry: Copy { /// A **completion-ring entry** — the device is the single writer and flips a /// per-entry validity bit on each wrap; the reader polls it. +/// Once published, the device must leave the entry stable until the consumer +/// acknowledges it or returns ownership through the device's ring protocol. pub trait CqEntry: Copy { /// Whether this entry has been published for the expected `phase` /// (NVMe: the CQE status **Phase** bit; xHCI: the event TRB **Cycle** @@ -263,25 +266,30 @@ impl CompRing { /// **Poll one completion** (non-blocking): read the head slot; if it /// carries the expected phase the device has published it — take an - /// acquire fence, advance the head, flip the expected phase on wrap, and - /// return it. `None` when the head slot is still stale. + /// acquire fence and re-read the entry before advancing the head and + /// flipping the expected phase on wrap. `None` when the head is stale. /// /// The caller acknowledges (NVMe: ring the CQ head doorbell; xHCI: write /// ERDP) — per-completion or once-per-drain, its choice. pub fn poll(&mut self) -> Option { // SAFETY: `buf` is `capacity * size_of::()` device-visible bytes; // `head < capacity`; the entry is naturally aligned. - let entry = unsafe { - (self.buf.cpu() as *const E) - .add(self.head as usize) - .read_volatile() - }; - if !entry.valid(self.phase) { + let slot = unsafe { (self.buf.cpu() as *const E).add(self.head as usize) }; + // SAFETY: the slot is within the live DMA buffer as established above. + let observed = unsafe { slot.read_volatile() }; + if !observed.valid(self.phase) { return None; } - // The validity bit matched: the device published this entry — observe - // the rest of it coherently. + // A wide volatile read can lower to several loads: the old payload + // may have been copied before a device write, with the new validity + // bit read afterward. An acquire fence cannot repair that snapshot. + // Re-read only after publication; the device cannot reuse this slot + // until the caller acknowledges the advanced head. fence(Ordering::Acquire); + // SAFETY: same live slot; matching validity grants a stable entry until + // the caller's later device acknowledgment. Volatile prevents reuse of + // the pre-acquire snapshot even on strongly ordered architectures. + let entry = unsafe { slot.read_volatile() }; self.head = (self.head + 1) % self.capacity; if self.head == 0 { self.phase = !self.phase; @@ -449,6 +457,56 @@ mod tests { ); } + #[test] + fn completion_body_is_observed_after_publication() { + // Model a split DMA read: the consumer copied the old body, then + // the device published the new body before the validity observation. + // The validity hook schedules that interleaving deterministically; + // no host thread or probabilistic timing is needed. + #[derive(Clone, Copy)] + #[repr(C)] + struct RacingCompletion { + payload: u64, + live_payload: *mut u64, + } + impl CqEntry for RacingCompletion { + fn valid(&self, phase: bool) -> bool { + if !phase { + return false; + } + // SAFETY: the test points this at the live ring slot's first + // field, and the buffer outlives this synchronous callback. + unsafe { self.live_payload.write_volatile(42) }; + true + } + } + // Typed, caller-owned storage preserves alignment without an allocator. + struct DeviceBuffer<'a>(&'a core::cell::UnsafeCell); + impl Dma for DeviceBuffer<'_> { + fn cpu(&self) -> *mut u8 { + self.0.get().cast() + } + fn iova(&self) -> IovaAddr { + IovaAddr(0x4000) + } + fn len(&self) -> usize { + core::mem::size_of::() + } + } + let memory = core::cell::UnsafeCell::new(RacingCompletion { + payload: 7, + live_payload: core::ptr::null_mut(), + }); + // SAFETY: initialize the self-pointer before any ring read; memory + // stays at this address until the borrowing ring has been dropped. + unsafe { (*memory.get()).live_payload = memory.get().cast::() }; + let mut ring = CompRing::::new(DeviceBuffer(&memory), 1); + assert_eq!(ring.poll().expect("published completion").payload, 42); + assert_eq!(ring.head(), 0); + assert!(!ring.phase()); + assert!(ring.poll().is_none(), "one publication is consumed once"); + } + #[test] fn ring_doorbell_passes_the_tail() { let buf = HostBuf::new(4 * 16, 0x5000); diff --git a/native/crates/driver/src/stor/lanes.rs b/native/crates/driver/src/stor/lanes.rs index 9e8ee6d..0e78d99 100644 --- a/native/crates/driver/src/stor/lanes.rs +++ b/native/crates/driver/src/stor/lanes.rs @@ -829,6 +829,82 @@ mod tests { )) } + /// Pack maintenance's composition (`res/PACK_STREAMING.md` §7): a + /// region refresh drives a Stream lane through its region clamp, times + /// its probes off the lane's clock, and rewrites the clamped extent + /// without changing a byte anywhere on the disk. + #[test] + fn a_region_refresh_rides_a_stream_lane_inside_its_clamp() { + use crate::block::refresh::{Bucket, Outcome, Progress, RegionRefresh}; + use crate::block::{BlockBuffer, DeviceCommand}; + // Eight buffers for the job's four chunk slots. + type Backend8 = EngineBackend<'static, Single, OneQueue, 2, 3, 2, 8, 32, 32>; + type Engine8 = StorEngine<'static, Single, OneQueue, 3, 2, 8, 32, 32>; + let mut dev = Dev::new(512, 8).unwrap(); + let mut buffer = dev.allocate_buffer(512).unwrap(); + for lba in 0..64u64 { + buffer.bytes_mut().fill(lba as u8 | 0x80); + dev.submit(DeviceCommand::Write { lba, blocks: 1, buffer_offset: 0 }, Some(&mut buffer), 0).unwrap(); + dev.poll().unwrap().result.unwrap(); + } + dev.set_clock(1_000, 50_000); + let image = |dev: &mut Dev| { + let mut out = [0u8; 32 * 1024]; + let mut buffer = dev.allocate_buffer(4096).unwrap(); + for (index, chunk) in out.chunks_exact_mut(4096).enumerate() { + let read = DeviceCommand::Read { lba: index as u64 * 8, blocks: 8, buffer_offset: 0 }; + dev.submit(read, Some(&mut buffer), 0).unwrap(); + dev.poll().unwrap().result.unwrap(); + chunk.copy_from_slice(buffer.bytes()); + } + out + }; + let before = image(&mut dev); + let place = Box::leak(Box::new(core::mem::MaybeUninit::::uninit())); + let engine = Engine8::init_in(place, Single::new(dev)).ok().expect("policy fits"); + let backend: &'static Backend8 = Box::leak(Box::new( + Backend8::new(engine, [window(), window()], [LaneClasses::FILESYSTEM, LaneClasses::STREAM]) + .unwrap(), + )); + let lane = backend.claim(1).unwrap(); + let region = RegionDevice::whole(lane).unwrap().narrow(16, 32).ok().expect("clamp"); + let mut books = [Bucket::default(); 4]; + let mut job = RegionRefresh::new(region, &mut books).ok().expect("the job builds"); + let run = |job: &mut RegionRefresh<'_, _>| { + for _ in 0..10_000 { + if job.advance(true) == Progress::Idle { + return; + } + } + panic!("the job never went idle"); + }; + job.scan().unwrap(); + run(&mut job); + assert_eq!(job.outcome(), Outcome::Scanned); + assert_eq!((job.scan_summary().scanned, job.scan_summary().mapped), (1, 1)); + assert!(job.buckets()[0].median_ns > 0, "probes were timed off the lane's clock"); + job.refresh_range(0, 1).unwrap(); + run(&mut job); + assert_eq!(job.outcome(), Outcome::Refreshed); + assert_eq!(job.refresh_summary().bytes, 32 * 512); + assert_eq!(job.stale_completions(), 0); + assert!(job.into_device().is_ok()); + assert_eq!(backend.outstanding(1), Some(0)); + let (stats, expired) = backend.stats().unwrap(); + assert_eq!((stats.stale_completions, expired), (0, 0)); + // Read the whole disk back through the other lane. + let whole = RegionDevice::whole(backend.claim(0).unwrap()).unwrap(); + let mut channel: BlockChannel<_, 2, 4> = BlockChannel::new(whole, CompletionMode::Poll).unwrap(); + let buffer = channel.register_buffer(4096).unwrap(); + let mut after = [0u8; 32 * 1024]; + for (index, chunk) in after.chunks_exact_mut(4096).enumerate() { + let read = channel.submit_read(index as u64 * 8, 8, buffer, 0).unwrap(); + assert_eq!(channel.wait().unwrap().request, read); + channel.read_buffer(buffer, 0, chunk).unwrap(); + } + assert!(after == before, "a refresh changed a byte"); + } + /// The ordinary composition runs unchanged on an engine lane: a block /// channel over a region clamp writes and reads back, and the two lanes' /// regions never see each other's bytes. diff --git a/native/crates/ld/tests/built_images.rs b/native/crates/ld/tests/built_images.rs index b3277fb..f196515 100644 --- a/native/crates/ld/tests/built_images.rs +++ b/native/crates/ld/tests/built_images.rs @@ -2,7 +2,7 @@ //! epic #291, P4 #295): the closed loop behind `make user-verify`. //! //! `KORE_USER_IMAGES` names the `user/` cargo target dir; for each -//! `/release/{libdemo.so,libpci_names.so,hello,time,sleep,status,lspci,help,window,clear,lua,liblua_demo.so,kore-editor}` present this asserts, +//! `/release/{libdemo.so,libpci_names.so,libkore_grammars.so,hello,time,sleep,status,lspci,help,window,clear,lua,liblua_demo.so,kore-terminal,kore-files,kore-editor}` present this asserts, //! by raw ELF //! inspection (the `elf` crate — the same substrate `kore_ld::parse` trusts): //! @@ -16,7 +16,11 @@ //! - `libdemo.so` carries `PT_TLS` and ≥1 TPREL-class reloc; //! - `hello` names `libdemo.so`, while `status` and `lspci` name //! `libpci_names.so` through `DT_NEEDED`, proving both shared-library -//! boundaries survive the build. +//! boundaries survive the build; +//! - `libkore_grammars.so` imports exactly the C runtime symbols its +//! consumers export ([`GRAMMAR_IMPORTS`]), and the editor and the terminal +//! name it through `DT_NEEDED` and export each of them — the launch-time +//! library that binds into its executable. //! //! `KORE_USER_IMAGE_ARCH`, when set, closes the walk to exactly one of //! `x86_64`, `aarch64`, or `riscv64`. Production packaging uses that @@ -40,6 +44,11 @@ use kore_ld::reloc::{AArch64, RelocArch, RelocKind, Riscv64, X86_64}; const PAGE: u64 = 4096; +/// What `libkore_grammars.so` imports from the executable it links into: +/// its scanners' allocator, and the `abort` its panic ends in. The Makefile's +/// `GRAMMAR_IMPORTS` exports the same set from each consumer. +const GRAMMAR_IMPORTS: &[&str] = &["abort", "calloc", "free", "malloc", "realloc"]; + /// Per-image census of eager relocation semantics — the printed /// evidence that the §7 set (and nothing else) is what the toolchain /// emitted. @@ -205,7 +214,39 @@ fn verify_image(path: &Path, kind: ImageKind) { .unwrap_or_else(|e| panic!("{tag}: late protection layout: {e:?}")); } } - if let ImageKind::NeedsLibrary(required) = kind { + if let ImageKind::HostedLibrary { imports } | ImageKind::HostsLibrary { exports: imports, .. } = kind { + let (symbols, strings) = elf + .dynamic_symbol_table() + .unwrap_or_else(|e| panic!("{tag}: dynsym parse: {e}")) + .unwrap_or_else(|| panic!("{tag}: no .dynsym")); + let mut undefined: Vec<&str> = Vec::new(); + let mut defined: Vec<&str> = Vec::new(); + for sym in symbols.iter() { + let name = strings + .get(sym.st_name as usize) + .unwrap_or_else(|e| panic!("{tag}: dynsym strtab: {e}")); + if name.is_empty() { + continue; + } + if sym.st_shndx == abi::SHN_UNDEF { + undefined.push(name); + } else { + defined.push(name); + } + } + if matches!(kind, ImageKind::HostedLibrary { .. }) { + undefined.sort_unstable(); + assert_eq!( + undefined, imports, + "{tag}: a hosted library imports exactly what its executables export" + ); + } else { + for symbol in imports { + assert!(defined.contains(symbol), "{tag}: .dynsym does not export {symbol}"); + } + } + } + if let ImageKind::NeedsLibrary(required) | ImageKind::HostsLibrary { library: required, .. } = kind { let dynstr_hdr = elf .section_header_by_name(".dynstr") .unwrap_or_else(|e| panic!("{tag}: shdr walk: {e}")) @@ -260,6 +301,16 @@ enum ImageKind { opener: &'static str, host_prefix: &'static str, }, + /// A launch-time library that binds into its executable: its undefined + /// dynamic symbols are exactly `imports`, which the stub resolves against + /// the executable first (`crates/ld/src/link.rs`). + HostedLibrary { imports: &'static [&'static str] }, + /// An executable that names `library` through `DT_NEEDED` and defines + /// each of `exports` in `.dynsym`, for that library to bind. + HostsLibrary { + library: &'static str, + exports: &'static [&'static str], + }, } /// Verify one exact profile directory. Aggregate callers may treat absence as @@ -277,6 +328,12 @@ fn check_variant(root: &Path, variant: &str, required: bool) -> us for (name, kind) in [ ("libdemo.so", ImageKind::DemoLibrary), ("libpci_names.so", ImageKind::SharedLibrary), + ( + "libkore_grammars.so", + ImageKind::HostedLibrary { + imports: GRAMMAR_IMPORTS, + }, + ), ("hello", ImageKind::NeedsLibrary("libdemo.so")), ("time", ImageKind::Executable), ("sleep", ImageKind::Executable), @@ -294,8 +351,21 @@ fn check_variant(root: &Path, variant: &str, required: bool) -> us host_prefix: "lua_", }, ), - ("kore-terminal", ImageKind::Executable), - ("kore-editor", ImageKind::Executable), + ( + "kore-terminal", + ImageKind::HostsLibrary { + library: "libkore_grammars.so", + exports: GRAMMAR_IMPORTS, + }, + ), + ("kore-files", ImageKind::Executable), + ( + "kore-editor", + ImageKind::HostsLibrary { + library: "libkore_grammars.so", + exports: GRAMMAR_IMPORTS, + }, + ), ("kore-aftershock", ImageKind::Executable), ("kore-skylilt", ImageKind::Executable), ] { diff --git a/native/crates/ld/tests/link_engine.rs b/native/crates/ld/tests/link_engine.rs index aa9c9df..debf616 100644 --- a/native/crates/ld/tests/link_engine.rs +++ b/native/crates/ld/tests/link_engine.rs @@ -1279,10 +1279,12 @@ fn stub_arena() -> &'static kore_ld::store::LinkArena<'static> { } /// The caps `user/runtime/ld-stub` passes: a program is itself plus at most -/// one library, with a single init array. +/// one library, with the engine's full init capacity (`main.rs`'s +/// `max_inits: MAX_INIT_FNS` — `hello`'s `libdemo.so` initializers need more +/// than one). const STUB_CAPS: LinkCaps = LinkCaps { max_modules: 2, - max_inits: 1, + max_inits: kore_ld::link::MAX_INIT_FNS, }; /// **Every built executable must link under the stub's own caps.** diff --git a/native/crates/ring/src/event.rs b/native/crates/ring/src/event.rs index a6bc839..91834ab 100644 --- a/native/crates/ring/src/event.rs +++ b/native/crates/ring/src/event.rs @@ -56,9 +56,20 @@ pub const EVENT_ABI: u32 = 1; const ALIGN: usize = 64; /// Byte offset of the wake word (the producer's cache line). -const SEQ_AT: usize = 64; +pub const SEQ_AT: usize = 64; /// Byte offset of the park intent (the consumer's cache line). -const PARKED_AT: usize = 128; +pub const PARKED_AT: usize = 128; + +/// The park intent's **doorbell** value: the consumer committed to waiting +/// on its executor's standing post, and a batch owes it a notice. +pub const PARK_DOORBELL: u32 = 1; +/// The park intent's **word** value: the consumer committed to waiting on +/// the wake word itself through the kernel address-wait (`res/EVENT.md` +/// §6), and a batch owes it a `wake_word` on that memory. A producer that +/// cannot name the word (no mapping of its own noted) sends the notice +/// instead — the consumer's word park carries its doorbell post too, so +/// either wake reaches it. +pub const PARK_WORD: u32 = 2; /// Byte offset of the state block. const STATE_AT: usize = 192; @@ -355,11 +366,28 @@ impl EventSource<'_, S, R> { /// instead of waiting. #[must_use = "true means the consumer parked: a notice is owed"] pub fn commit(&mut self) -> bool { + self.commit_intent() != 0 + } + + /// [`commit`](Self::commit), answering the collected intent itself: + /// `0` = nobody parked, [`PARK_DOORBELL`] = a notice is owed, + /// [`PARK_WORD`] = the consumer is parked on the wake word (a + /// `wake_word` reaches it; so does a notice). The runtime's + /// `commit_and_kick` chooses its kick by this word. + #[must_use = "a non-zero intent means the consumer parked: a wake is owed"] + pub fn commit_intent(&mut self) -> u32 { let seq = self.raw.seq(); // `state` may already have advanced `seq`; a records-only batch // advances it here. Always move it so one word means "new". seq.fetch_add(2, Ordering::SeqCst); - self.raw.parked().swap(0, Ordering::SeqCst) != 0 + self.raw.parked().swap(0, Ordering::SeqCst) + } + + /// The wake word itself — what the producer names to the kernel + /// address-wait's `wake_word` (by the mapping it holds, through + /// `kore_rt::cap::mapped::locate`). + pub fn seq_word(&self) -> &AtomicU32 { + self.raw.seq() } } @@ -442,7 +470,7 @@ impl EventSink<'_, S, R> { /// The intent may go stale (a later wake for an earlier park); a /// spurious notice is the harmless direction, exactly as in `futex`. pub fn park(&self, seen: u32) -> bool { - self.raw.parked().store(1, Ordering::SeqCst); + self.raw.parked().store(PARK_DOORBELL, Ordering::SeqCst); if self.raw.seq().load(Ordering::SeqCst) != seen { self.raw.parked().store(0, Ordering::SeqCst); return false; @@ -450,6 +478,18 @@ impl EventSink<'_, S, R> { true } + /// Restate a committed intent as a **word** park ([`PARK_WORD`]): the + /// consumer is about to sleep on the wake word itself (the kernel + /// address-wait), so the producer may answer with a `wake_word` + /// instead of a notice. Called after [`park`](Self::park) said the + /// wait stands, and again before every re-park; the value is a hint + /// about *how* to wake, and the Dekker proof is unchanged — the kernel + /// makes the final `seq` check against the physical word at park time, + /// so a batch landing after this store resolves the park at once. + pub fn park_on_word(&self) { + self.raw.parked().store(PARK_WORD, Ordering::SeqCst); + } + /// Withdraw a published park intent (a wait that ends for another /// reason — deadline, shutdown — so a later batch is not charged a /// notice for a waiter that left). diff --git a/native/crates/rt/src/barrier.rs b/native/crates/rt/src/barrier.rs index ddbb409..09a05b4 100644 --- a/native/crates/rt/src/barrier.rs +++ b/native/crates/rt/src/barrier.rs @@ -276,6 +276,70 @@ pub(crate) fn wait_while(word: &AtomicU32, expected: u32) { } } +/// Wait for the **next job** rather than for a sibling: [`wait_while`]'s +/// fast path first, backing off while nothing arrives. +/// +/// [`wait_while`]'s budget is a scheduling-jitter window — up to 64 µs of +/// spin before each park, and a [`PARK_SLICE`] re-check — because a barrier +/// sibling is microseconds away. An idle resident worker waits on a +/// submitter that may be seconds away, and paying that budget on every +/// slice is a duty cycle, not a jitter window: ~64 µs spent spinning per +/// ~314 µs, forever, per idle worker. Sampling a two-core guest showed the +/// idle SPMD gangs of the terminal and compositor holding ~30% of all CPU +/// in exactly that spin while a game rendered. +/// +/// So only the first wait after a job keeps the full two-tier budget (a +/// back-to-back submit, the common in-frame case, still resolves in the +/// spin). Each slice that rides out with the word unmoved drops the spin and +/// doubles the slice, up to [`IDLE_SLICE_MAX`]. A submit's kick ends the +/// park at any slice length; the slice only bounds a *missed* kick, now at +/// [`IDLE_SLICE_MAX`] rather than [`PARK_SLICE`] once idle. +pub(crate) fn wait_while_idle(word: &AtomicU32, expected: u32) { + wait_while_idle_with( + word, + expected, + |spin, slice| { + if spin { + futex::wait_barrier(word, expected, Some(slice)) + } else { + futex::wait_idle(word, expected, Some(slice)) + } + }, + || { + crate::hint::wait_on(word, expected, SPIN_SLICE); + }, + ); +} + +/// One wait policy for the live futex transport and scripted wake interleavings. +/// The callbacks return after each bounded wait; the shared job word alone +/// decides whether a notification actually published work. +fn wait_while_idle_with( + word: &AtomicU32, + expected: u32, + mut wait: impl FnMut(bool, Duration) -> futex::WaitOutcome, + mut unsupported: impl FnMut(), +) { + use core::sync::atomic::Ordering; + let mut slice = PARK_SLICE; + let mut spin = true; + while word.load(Ordering::SeqCst) == expected { + let outcome = wait(spin, slice); + match outcome { + futex::WaitOutcome::Unsupported => { + unsupported(); + } + // A wake without a job is noise, not a new burst. + futex::WaitOutcome::TimedOut | futex::WaitOutcome::Woken => { + spin = false; + slice = next_idle_slice(slice); + } + // Retry: the word is the truth; the loop head re-reads. + _ => {} + } + } +} + /// The monitored-turn cap for a thread that cannot park. /// /// The futex's own spin tier uses 1 µs turns for the same reason (its @@ -283,12 +347,151 @@ pub(crate) fn wait_while(word: &AtomicU32, expected: u32) { /// bound on how long a monitor that never fires can hold the core. const SPIN_SLICE: Duration = Duration::from_micros(1); +/// Where the idle ladder stops: the longest a resident worker **between +/// jobs** sleeps before re-reading the job word on its own. +/// +/// [`PARK_SLICE`] prices the residual of a wait a kick is coming for. A +/// worker idling between jobs has no kick coming: nothing it rides the +/// slice for ever arrives, so at a quarter-millisecond it re-enters the +/// futex four thousand times a second — a monitored spin and a kernel +/// park each time — for as long as its region lives, per worker, per +/// gang. Two terminal seats' render gangs so held three of a 4-CPU +/// acceptance machine's cores at idle, and a program compiling on the +/// fourth was sampled running one time in six. The ladder doubles the +/// slice from [`PARK_SLICE`] while the word stays put and a job resets +/// it, so an idle gang costs the machine what an idle thread should. +/// The cap is one display frame: the submit's kick still ends a park +/// the instant the job is stored, so the slice bounds only the kick +/// the mesh misses — a first frame after a long idle, at worst, lands a +/// frame late; the next job starts the ladder over. +pub(crate) const IDLE_SLICE_MAX: Duration = Duration::from_millis(16); + +/// The next rung of the idle ladder: double, to [`IDLE_SLICE_MAX`]. +pub(crate) fn next_idle_slice(slice: Duration) -> Duration { + slice.saturating_mul(2).min(IDLE_SLICE_MAX) +} + #[cfg(test)] mod tests { use super::*; use core::sync::atomic::{AtomicU32, Ordering}; use std::thread; + #[test] + fn idle_timeouts_drop_spin_but_notifications_still_require_a_new_job() { + let job = AtomicU32::new(1); + let mut calls = 0; + let mut previous = PARK_SLICE; + let mut reached_ceiling = false; + wait_while_idle_with( + &job, + 1, + |spin, slice| { + assert!(calls < 16, "a published job must end the wait"); + assert_eq!(spin, calls == 0, "idle retries must not spin again"); + assert!(slice >= previous && slice <= IDLE_SLICE_MAX); + if calls == 0 { + assert_eq!(slice, PARK_SLICE); + } + previous = slice; + reached_ceiling |= slice == IDLE_SLICE_MAX; + calls += 1; + match calls { + // A spurious notice must not manufacture a job or reset + // the wait to its busy first-turn posture. + 5 => futex::WaitOutcome::Woken, + 8 => futex::WaitOutcome::Retry, + 12 => { + job.store(2, Ordering::SeqCst); + futex::WaitOutcome::Woken + } + _ => futex::WaitOutcome::TimedOut, + } + }, + || panic!("supported transport must not take the fallback"), + ); + assert_eq!(calls, 12); + assert!( + reached_ceiling, + "idle waits should reach their bounded backoff" + ); + + // After completing this job, a back-to-back submission gets the fast + // first turn again; an old idle history does not slow a fresh burst. + wait_while_idle_with( + &job, + 2, + |spin, slice| { + assert!(spin); + assert_eq!(slice, PARK_SLICE); + job.store(3, Ordering::SeqCst); + futex::WaitOutcome::Retry + }, + || panic!("supported transport must not take the fallback"), + ); + } + + #[test] + fn idle_wait_checks_a_published_job_before_parking_and_after_fallback() { + let job = AtomicU32::new(2); + wait_while_idle_with( + &job, + 1, + |_, _| panic!("already-published work must not park"), + || panic!("already-published work must not wait"), + ); + let mut waits = 0; + wait_while_idle_with( + &job, + 2, + |_, _| { + waits += 1; + futex::WaitOutcome::Unsupported + }, + || job.store(3, Ordering::SeqCst), + ); + assert_eq!(waits, 1, "a fallback-observed submission must end the wait"); + } + + /// The idle ladder climbs by doubling and stops at its cap, so an + /// idle worker's re-check rate falls geometrically and never past + /// one frame. + #[test] + fn the_idle_ladder_doubles_to_its_cap() { + let mut slice = PARK_SLICE; + let mut rungs = 0; + while slice < IDLE_SLICE_MAX { + let next = next_idle_slice(slice); + assert_eq!(next, (slice * 2).min(IDLE_SLICE_MAX)); + slice = next; + rungs += 1; + } + assert_eq!(slice, IDLE_SLICE_MAX); + assert_eq!(next_idle_slice(IDLE_SLICE_MAX), IDLE_SLICE_MAX); + // 250 µs doubles six times to 16 ms. + assert_eq!(rungs, 6); + } + + /// An idle worker still returns the moment the word moves, however + /// far its slice has climbed: a store ends the wait, and one that + /// precedes the wait is seen before any park. + #[test] + fn an_idle_wait_ends_when_the_word_moves() { + static JOB: AtomicU32 = AtomicU32::new(0); + // Already moved: no park at all. + JOB.store(1, Ordering::SeqCst); + wait_while_idle(&JOB, 0); + // Moved while waiting, after enough rungs for the slice to climb. + let waiter = thread::spawn(|| wait_while_idle(&JOB, 1)); + thread::sleep(Duration::from_millis(5)); + JOB.store(2, Ordering::SeqCst); + let _ = futex::wake(&JOB); + waiter + .join() + .expect("the idle waiter returns once the word moved"); + assert_eq!(JOB.load(Ordering::SeqCst), 2); + } + /// The property a barrier exists for: nothing published before an /// arrival is unobserved after the release. Each worker counts itself /// in before arriving; after returning, it must see everyone. diff --git a/native/crates/rt/src/cap/mapped.rs b/native/crates/rt/src/cap/mapped.rs new file mode 100644 index 0000000..f834396 --- /dev/null +++ b/native/crates/rt/src/cap/mapped.rs @@ -0,0 +1,460 @@ +//! `mapped` — **which held slot a mapped window came from**, so a memory +//! word can be named to the kernel by *authority* (`res/EVENT.md` §6). +//! +//! The kernel address-wait parks on a `u32` named as `(subject slot, byte +//! offset)` — a held lease or mapped view plus an offset within it — and +//! never as an address: authority is the mapping, and a raw address +//! crosses no boundary. Userland code, though, holds the word as a +//! `&AtomicU32` inside a window some map verb returned. This registry is +//! the bridge: every verb that installs a held slot into this task's +//! address space [`note`]s `(base, len, slot)` here, every unmap +//! [`forget_exact`]s it, and [`locate`] turns a word's address back into the +//! `(slot, offset)` pair the kernel wants. That is what makes the rung +//! **automatic** for existing callers — `event::wait_past` and the futex +//! consult it and fall back to the doorbell park when the word is not in +//! a noted window (an ordinary heap or static word, or a window mapped by +//! a path that does not note itself). +//! +//! Heapless and lock-free: a static bounded table of atomics claimed by +//! CAS on the base word (`0` = free, the null page is never mapped). The +//! table is a **hint**, never an authority: the kernel re-validates the +//! slot it is handed against the caller's holdings, so a stale entry costs +//! a refused park (and the doorbell fallback), never a wrong wait. A full +//! table drops the note — the window is still mapped, only the word rung +//! is declined for it. + +use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; + +use kore_abi::session::SlotRef; + +/// Concurrently noted windows per task — a design bound sized to the +/// windows a service holds at once (rings, event channels, surfaces, +/// stashed views), with room to spare. Refusal on overflow is a declined +/// rung, not a lost wake. +#[cfg(feature = "heap")] +const MAX_MAPPED: usize = 64; + +/// The vacant entry. Zero, and safely so: no mapped window starts at the +/// null page in any kore task. +#[cfg(feature = "heap")] +const FREE: usize = 0; + +/// An entry mid-claim: base reserved, `len`/`slot` not yet published. A +/// scan skips it exactly as it skips [`FREE`]. Not a mappable base either +/// (the top of the address space is never a window). +#[cfg(feature = "heap")] +const CLAIMING: usize = usize::MAX; + +#[cfg(feature = "heap")] +struct Entry { + base: AtomicUsize, + len: AtomicUsize, + slot: AtomicU32, + /// The thread whose arena holds `slot` — capabilities are per-thread + /// arenas, so a slot is meaningful only to the thread that mapped the + /// window; a sibling handing that slot to the kernel is `Forged`/`Denied` + /// (the storm the first `USER_SIMD=v3` boot hit in the terminal's parallel + /// renderer). `locate` answers only the noting thread; every other + /// thread keeps the doorbell rung for that window. + owner: AtomicUsize, +} + +#[cfg(feature = "heap")] +impl Entry { + const fn new() -> Self { + Self { + base: AtomicUsize::new(FREE), + len: AtomicUsize::new(0), + slot: AtomicU32::new(0), + owner: AtomicUsize::new(0), + } + } +} + +/// The bounded registry — generic over its capacity so the host suite can +/// exercise a private instance (including exhaustion) without touching +/// the task-global one. +#[cfg(feature = "heap")] +struct Registry { + entries: [Entry; N], +} + +#[cfg(feature = "heap")] +impl Registry { + const fn new() -> Self { + Self { + entries: [const { Entry::new() }; N], + } + } + + /// Claim a free entry for `(base, len, slot)`. `false` when the table + /// is full or the window is empty / degenerate (nothing to name). + fn note(&self, base: usize, len: usize, slot: SlotRef, owner: usize) -> bool { + if base == FREE || base == CLAIMING || len == 0 || base.checked_add(len).is_none() { + return false; + } + // A remap supersedes: every entry of this thread overlapping the new + // window is stale (its view died kernel-side — a lease retired by + // its owner unmaps without a verb here — or it was replaced at the + // same address). Left in place it would be located first, and once + // its slot index is reissued to another capability the wait keys + // the wrong memory with no error at either end. + self.forget_overlapping(base, len, owner); + for entry in &self.entries { + if entry + .base + .compare_exchange(FREE, CLAIMING, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + entry.len.store(len, Ordering::Relaxed); + entry.slot.store(slot.index(), Ordering::Relaxed); + entry.owner.store(owner, Ordering::Relaxed); + // Publish: a reader that sees `base` sees `len`/`slot`/`owner`. + entry.base.store(base, Ordering::Release); + return true; + } + } + false + } + + /// Release the entries noted at `base`, if any. Test-only broad cleanup. + #[cfg(test)] + fn forget(&self, base: usize) { + if base == FREE || base == CLAIMING { + return; + } + for entry in &self.entries { + let _ = entry + .base + .compare_exchange(base, FREE, Ordering::AcqRel, Ordering::Relaxed); + } + } + + /// Forget only the caller's exact view. A retired owner may drop after a + /// replacement has been noted at the same VA or slot index. + fn forget_exact(&self, base: usize, slot: SlotRef, owner: usize) { + if base == FREE || base == CLAIMING { + return; + } + for entry in &self.entries { + if entry.base.load(Ordering::Acquire) == base + && entry.slot.load(Ordering::Relaxed) == slot.index() + && entry.owner.load(Ordering::Relaxed) == owner + { + let _ = entry + .base + .compare_exchange(base, FREE, Ordering::AcqRel, Ordering::Relaxed); + } + } + } + + /// Release every entry of `owner` whose window overlaps `[base, base+len)`. + fn forget_overlapping(&self, base: usize, len: usize, owner: usize) { + let end = base.saturating_add(len); + for entry in &self.entries { + let old = entry.base.load(Ordering::Acquire); + if old == FREE || old == CLAIMING || entry.owner.load(Ordering::Relaxed) != owner { + continue; + } + let old_end = old.saturating_add(entry.len.load(Ordering::Relaxed)); + if old < end && base < old_end { + let _ = entry + .base + .compare_exchange(old, FREE, Ordering::AcqRel, Ordering::Relaxed); + } + } + } + + /// Release every entry noted under `slot` — the unmap face that knows + /// the capability name but not the window it installed. + fn forget_owner(&self, owner: usize) { + for entry in &self.entries { + let base = entry.base.load(Ordering::Acquire); + if base != FREE && base != CLAIMING && entry.owner.load(Ordering::Relaxed) == owner { + let _ = entry + .base + .compare_exchange(base, FREE, Ordering::AcqRel, Ordering::Relaxed); + } + } + } + + fn forget_slot(&self, slot: SlotRef) { + for entry in &self.entries { + let base = entry.base.load(Ordering::Acquire); + if base != FREE && base != CLAIMING && entry.slot.load(Ordering::Relaxed) == slot.index() + { + let _ = entry + .base + .compare_exchange(base, FREE, Ordering::AcqRel, Ordering::Relaxed); + } + } + } + + /// The `(slot, offset)` naming `addr`, when it falls inside a noted + /// window. Bounded scan; `None` when no window holds it or the offset + /// does not fit the wire's `u32`. + fn locate(&self, addr: usize, owner: usize) -> Option<(SlotRef, u32)> { + for entry in &self.entries { + let base = entry.base.load(Ordering::Acquire); + if base == FREE || base == CLAIMING || addr < base { + continue; + } + if entry.owner.load(Ordering::Relaxed) != owner { + continue; + } + let len = entry.len.load(Ordering::Relaxed); + if addr - base < len { + let offset = u32::try_from(addr - base).ok()?; + return Some((SlotRef::from_wire(entry.slot.load(Ordering::Relaxed)), offset)); + } + } + None + } +} + +/// The task-global registry — one per address space, shared by every +/// thread exactly as the futex registry is. +/// +/// **Program builds only** (the `heap` feature — "everything a userland +/// program wants"). The one consumer built without it is the linker stub, +/// whose image the founder **shares read-only into every child** +/// (`SharedSegment` carries `execute`, never write): a static the stub +/// writes is a `#PF` (error 7) on its first store. The stub runs the map +/// verbs that note windows here, so in that build the registry must not +/// merely be unused — it must not exist, or the stub's `.bss` grows by a +/// page that nothing may touch. Without it the stub's own mappings are +/// simply not located, and every wait it could have taken keeps the +/// doorbell rung, which is what a task with no runtime wants anyway. +/// The calling thread's index — the arena the noted slot lives in. The +/// futex registry's numbering (`entry::geometry::shard_index`); `None` when +/// the thread cannot name itself, which notes nothing and locates nothing. +#[cfg(feature = "heap")] +fn current_thread() -> Option { + #[cfg(target_os = "none")] + { + crate::entry::geometry::shard_index() + } + #[cfg(not(target_os = "none"))] + { + Some(0) + } +} + +#[cfg(feature = "heap")] +static MAPPED: Registry = Registry::new(); + +/// Note that `slot` (a held lease or mapped view) is installed at +/// `base..base+len` in this address space. Called by every map verb that +/// returns a window; a full table declines quietly (the word rung falls +/// back to the doorbell for that window). A no-op in the stub build (see +/// [`MAPPED`]). +#[cfg_attr(all(not(target_os = "none"), not(test)), expect(dead_code, reason = "noted only by the metal map verbs"))] +pub(crate) fn note(base: *const u8, len: usize, slot: SlotRef) { + #[cfg(feature = "heap")] + { + if let Some(owner) = current_thread() { + let _ = MAPPED.note(base as usize, len, slot, owner); + } + } + #[cfg(not(feature = "heap"))] + { + let _ = (base, len, slot); + } +} + +/// Broad cleanup by VA, retained for registry tests only. +#[cfg(test)] +pub(crate) fn forget(base: *const u8) { + #[cfg(feature = "heap")] + MAPPED.forget(base as usize); + #[cfg(not(feature = "heap"))] + let _ = base; +} + +/// Forget one retired view without touching a replacement at its former VA. +pub(crate) fn forget_exact(base: *const u8, slot: SlotRef) { + #[cfg(feature = "heap")] + if let Some(owner) = current_thread() { + MAPPED.forget_exact(base as usize, slot, owner); + } + #[cfg(not(feature = "heap"))] + let _ = (base, slot); +} + +/// Forget every window installed under `slot` — the unmap face that names +/// the capability rather than the address (`retire_mapped_view`). +pub(crate) fn forget_slot(slot: SlotRef) { + #[cfg(feature = "heap")] + MAPPED.forget_slot(slot); + #[cfg(not(feature = "heap"))] + let _ = slot; +} + +/// Forget every window noted by thread `owner` — the thread-index reuse +/// face. A thread's entries outlive it (its windows die with it, but the +/// registry cannot see that), and `claim_thread_index` hands the same index +/// to a later thread, which would then locate a dead sibling's slots as its +/// own — the confused deputy the per-thread key exists to prevent. Called +/// at spawn, before the new thread runs. +pub(crate) fn forget_thread(owner: usize) { + #[cfg(feature = "heap")] + MAPPED.forget_owner(owner); + #[cfg(not(feature = "heap"))] + let _ = owner; +} + +/// Name the word at `addr` by authority: the held slot whose window holds +/// it and the byte offset within that window — exactly what a +/// `WordWaitReq` / `WakeWordReq` carries. `None` when the word is not in +/// a noted window (always, in the stub build); the caller keeps its +/// doorbell rung. +pub fn locate(addr: *const u8) -> Option<(SlotRef, u32)> { + #[cfg(feature = "heap")] + { + MAPPED.locate(addr as usize, current_thread()?) + } + #[cfg(not(feature = "heap"))] + { + let _ = addr; + None + } +} + +#[cfg(test)] +mod tests { + #[test] + fn a_reused_thread_index_inherits_nothing() { + let table = super::Registry::<2>::new(); + let slot = super::SlotRef::from_wire(3); + assert!(table.note(0x1000, 0x100, slot, 7)); + table.forget_owner(7); + assert_eq!(table.locate(0x1010, 7), None, "the dead thread's window is gone"); + assert!(table.note(0x2000, 0x100, slot, 7), "the entry is free again"); + } + + #[test] + fn a_sibling_thread_does_not_locate_a_window_another_thread_noted() { + let table = super::Registry::<2>::new(); + let slot = super::SlotRef::from_wire(3); + assert!(table.note(0x1000, 0x100, slot, 7)); + assert_eq!(table.locate(0x1010, 7), Some((slot, 0x10))); + assert_eq!(table.locate(0x1010, 8), None, "the slot lives in thread 7's arena"); + } + + use super::*; + + #[test] + fn a_noted_window_locates_its_words_by_slot_and_offset() { + let table = Registry::<4>::new(); + let slot = SlotRef::from_wire(17); + assert!(table.note(0x1000, 0x200, slot, 0)); + assert_eq!(table.locate(0x1000, 0), Some((slot, 0))); + assert_eq!(table.locate(0x1000 + 0x1fc, 0), Some((slot, 0x1fc))); + assert_eq!(table.locate(0x1000 + 0x200, 0), None, "one past the end"); + assert_eq!(table.locate(0xfff, 0), None, "one before the base"); + } + + #[test] + fn forgetting_by_base_or_slot_releases_the_entry() { + let table = Registry::<4>::new(); + let a = SlotRef::from_wire(3); + let b = SlotRef::from_wire(5); + assert!(table.note(0x1000, 0x100, a, 0)); + assert!(table.note(0x2000, 0x100, b, 0)); + table.forget(0x1000); + assert_eq!(table.locate(0x1010, 0), None); + assert_eq!(table.locate(0x2010, 0), Some((b, 0x10)), "siblings survive"); + table.forget(0x1000); + table.forget_slot(b); + assert_eq!(table.locate(0x2010, 0), None); + assert!(table.note(0x3000, 0x100, a, 0), "released entries are reusable"); + } + + /// The window lifecycle keytest's "editor did not receive keyboard + /// input after re-layout": the client's view died kernel-side (the + /// window's lease retired by its owner — no unmap verb ran here), the + /// re-layout mapped the replacement at the same address, and the stale + /// entry stood in front of the new one. Once its slot index was + /// reissued, every word wait keyed another capability's memory. + #[test] + fn a_remap_over_a_noted_window_supersedes_the_stale_entry() { + let table = Registry::<4>::new(); + let stale = SlotRef::from_wire(3); + let fresh = SlotRef::from_wire(11); + assert!(table.note(0x1000, 0x2000, stale, 0)); + // The same address, a shorter window: the stale entry overlaps. + assert!(table.note(0x1000, 0x1000, fresh, 0)); + assert_eq!(table.locate(0x1010, 0), Some((fresh, 0x10)), "the remap wins"); + assert_eq!(table.locate(0x1800, 0), Some((fresh, 0x800))); + assert_eq!(table.locate(0x1fff, 0), Some((fresh, 0xfff))); + assert_eq!(table.locate(0x2800, 0), None, "the stale tail is gone too"); + // A partial overlap from below evicts as well; a sibling thread's + // entry at the same address is not this thread's to evict. + assert!(table.note(0x1800, 0x100, stale, 5)); + assert!(table.note(0x0800, 0x1000, fresh, 0)); + assert_eq!(table.locate(0x1010, 0), Some((fresh, 0x810)), "the overlapping remap owns the address now"); + assert_eq!(table.locate(0x1900, 0), None, "the evicted tail past the remap is gone"); + assert_eq!(table.locate(0x1810, 5), Some((stale, 0x10)), "thread 5 keeps its own"); + } + + #[test] + fn forgetting_by_base_releases_every_entry_at_that_base() { + let table = Registry::<4>::new(); + let a = SlotRef::from_wire(3); + let b = SlotRef::from_wire(5); + assert!(table.note(0x1000, 0x100, a, 1)); + assert!(table.note(0x1000, 0x100, b, 2)); + table.forget(0x1000); + assert_eq!(table.locate(0x1010, 1), None); + assert_eq!(table.locate(0x1010, 2), None, "the second entry at the base is released too"); + } + + #[test] + fn exact_retirement_preserves_replacement_and_sibling_thread() { + let table = Registry::<4>::new(); + let old = SlotRef::from_wire(3); + let replacement = SlotRef::from_wire(5); + assert!(table.note(0x1000, 0x100, old, 1)); + assert!(table.note(0x1000, 0x100, old, 2)); + table.forget_exact(0x1000, old, 1); + assert_eq!(table.locate(0x1010, 1), None); + assert_eq!(table.locate(0x1010, 2), Some((old, 0x10))); + assert!(table.note(0x1000, 0x100, replacement, 1)); + table.forget_exact(0x1000, old, 1); + assert_eq!(table.locate(0x1010, 1), Some((replacement, 0x10))); + } + + #[test] + fn a_full_table_declines_the_note_and_degenerate_windows_are_refused() { + let table = Registry::<2>::new(); + let slot = SlotRef::from_wire(1); + assert!(table.note(0x1000, 0x100, slot, 0)); + assert!(table.note(0x2000, 0x100, slot, 0)); + assert!(!table.note(0x3000, 0x100, slot, 0), "bounded: the third is declined"); + assert_eq!(table.locate(0x3000, 0), None); + assert!(!table.note(FREE, 0x100, slot, 0)); + assert!(!table.note(0x4000, 0, slot, 0)); + assert!(!table.note(usize::MAX - 1, 0x100, slot, 0), "wrapping span"); + } + + #[test] + fn an_offset_past_u32_is_not_nameable() { + let table = Registry::<1>::new(); + let slot = SlotRef::from_wire(9); + let len = 1usize << 33; + assert!(table.note(0x1000, len, slot, 0)); + assert_eq!(table.locate(0x1000 + 4, 0), Some((slot, 4))); + assert_eq!(table.locate(0x1000 + (1usize << 32), 0), None); + } + + #[test] + fn the_task_global_face_notes_and_forgets_through_pointers() { + // Distinct fake bases so parallel tests never alias one another. + let base = 0x7f00_0000_1000usize as *const u8; + let slot = SlotRef::from_wire(42); + note(base, 64, slot); + assert_eq!(locate(base.wrapping_add(8)), Some((slot, 8))); + forget(base); + assert_eq!(locate(base.wrapping_add(8)), None); + } +} diff --git a/native/crates/rt/src/cap/mod.rs b/native/crates/rt/src/cap/mod.rs index b76a17d..2266846 100644 --- a/native/crates/rt/src/cap/mod.rs +++ b/native/crates/rt/src/cap/mod.rs @@ -3025,6 +3025,9 @@ impl MappedLeaseView { len: usize, ) -> Self { owned.activate(); + // The view's own slot names this window to the kernel address-wait + // (`cap::mapped`): a word inside it is `(view slot, offset)`. + mapped::note(base.as_ptr(), len, owned.slot()); Self { owned: core::mem::ManuallyDrop::new(owned), lease, @@ -3128,8 +3131,18 @@ impl MappedLeaseView { /// precise share-list edge and PTE before reporting success. pub fn try_retire_outcome(&mut self) -> RetireOutcome { let lease = self.lease; - self.owned - .retire_via(|view| crate::cap::verbs::retire_owned_view(view, lease)) + let was_live = self.owned.is_live(); + let slot = self.owned.slot(); + let outcome = self + .owned + .retire_via(|view| crate::cap::verbs::retire_owned_view(view, lease)); + if was_live && outcome.is_closed() { + // The PTE is gone (or the lineage is dead): a word in this window + // can no longer be named through it. A later drop must not forget + // a replacement that reused this VA. + mapped::forget_exact(self.base.as_ptr(), slot); + } + outcome } /// Whether the exact mapping still owns a live local capability name. @@ -3984,7 +3997,12 @@ impl DmaWindow { }; }; let parent = view.lease; + let was_live = view.owned.is_live(); + let slot = view.owned.slot(); let outcome = view.owned.retire_via(|slot| retire(slot, parent)); + if was_live && outcome.is_closed() { + mapped::forget_exact(view.base.as_ptr(), slot); + } if outcome.is_closed() { self.cpu = core::ptr::null_mut(); } @@ -4220,6 +4238,7 @@ pub enum RawDelivery { Cap, } +pub mod mapped; mod record; pub mod verbs; @@ -4273,6 +4292,17 @@ mod retirement_tests { } } + #[test] + fn dropping_retired_view_preserves_replacement_at_same_va() { + let mut bytes = [0u8; 8]; + let retired = mapped_test_view(&mut bytes); + let replacement = SlotRef::from_wire(46); + mapped::note(bytes.as_ptr(), bytes.len(), replacement); + drop(retired); + assert_eq!(mapped::locate(bytes.as_ptr()), Some((replacement, 0))); + mapped::forget_exact(bytes.as_ptr(), replacement); + } + fn writable_test_code(bytes: &mut [u8]) -> WritableCode { WritableCode { view: mapped_test_view(bytes), @@ -4404,6 +4434,7 @@ mod retirement_tests { let mut bytes = [0u8; 8]; let mut view = mapped_test_view(&mut bytes); view.owned = core::mem::ManuallyDrop::new(live_slot(42)); + mapped::note(bytes.as_ptr(), bytes.len(), SlotRef::from_wire(42)); let mut window = DmaWindow::new(bytes.as_mut_ptr(), 0x4000, 8, live_slot(43)) .with_device_leaf(live_slot(45)) .with_cpu_view(view); @@ -4430,6 +4461,7 @@ mod retirement_tests { RetireOutcome::Retired ); assert!(window.cpu().is_null()); + assert_eq!(mapped::locate(bytes.as_ptr()), None); assert!( window._owned.slot().is_some(), "allocation remains available for hand-back" @@ -4438,9 +4470,17 @@ mod retirement_tests { window.retire_cpu_via(|_, _| panic!("already retired")), RetireOutcome::AlreadyDead ); + let replacement = SlotRef::from_wire(46); + mapped::note(bytes.as_ptr(), bytes.len(), replacement); let mut lease = window .try_into_lease() .expect("both named children are closed"); + assert_eq!( + mapped::locate(bytes.as_ptr()), + Some((replacement, 0)), + "the old view's drop must preserve a new view at its VA" + ); + mapped::forget_exact(bytes.as_ptr(), replacement); assert_eq!(lease.slot(), Some(SlotRef::from_wire(43))); assert!(lease.is_live()); lease.owned.release_reservation(); diff --git a/native/crates/rt/src/cap/verbs.rs b/native/crates/rt/src/cap/verbs.rs index 9500670..d6603ef 100644 --- a/native/crates/rt/src/cap/verbs.rs +++ b/native/crates/rt/src/cap/verbs.rs @@ -44,19 +44,31 @@ mod metal { Indeterminate, } - /// Result of a non-consuming lineage probe on a receive endpoint. + /// Result of a non-consuming lineage probe on a channel endpoint + /// ([`probe_chan_recv_lineage`], [`probe_chan_send_lineage`]). /// /// `Dead` is positive kernel evidence that the endpoint or an ancestor was - /// revoked. `Live` means a same-kind descendant was derived and then - /// retired without changing the probed parent. Everything else preserves - /// the caller's reconciliation state for a later retry. + /// revoked. `Live` is positive evidence the kernel bound it live: a + /// same-kind descendant was derived and then retired without changing the + /// probed parent, or (a send endpoint only) the derive was refused for a + /// right the parent lacks, which the kernel checks only after liveness. + /// Everything else preserves the caller's reconciliation state for a + /// later retry. #[derive(Clone, Copy, Debug, PartialEq, Eq)] - pub enum ChanRecvLineage { + pub enum ChanLineage { Live, Dead, Retryable, } + /// The face of a channel a lineage probe derives from. + #[cfg(any(test, target_os = "none"))] + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum ChanFace { + Send, + Recv, + } + /// Why a direct Found result could not become the caller's exact task /// authority. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -208,11 +220,19 @@ mod metal { #[cfg(any(test, target_os = "none"))] #[inline] - fn classify_chan_recv_probe(committed: kore_abi::session::Committed) -> ChanRecvLineage { + fn classify_chan_probe(committed: kore_abi::session::Committed, face: ChanFace) -> ChanLineage { + use kore_abi::session::Status; + match (committed.status(), committed.failing_index()) { - (kore_abi::session::Status::Ok, 0) => ChanRecvLineage::Live, - (kore_abi::session::Status::Revoked, 0) => ChanRecvLineage::Dead, - _ => ChanRecvLineage::Retryable, + (Status::Ok, 0) => ChanLineage::Live, + (Status::Revoked, 0) => ChanLineage::Dead, + // A send endpoint may carry no DERIVE (a bound filesystem view's + // identity canary does not). Binding the parent checks + // authenticity and liveness first, and a bind fault is `Forged` + // or `Revoked`, never `Denied`: a derive refused for the missing + // right proves the lineage live, and created nothing. + (Status::Denied, 0) if face == ChanFace::Send => ChanLineage::Live, + _ => ChanLineage::Retryable, } } @@ -943,6 +963,10 @@ mod metal { /// may use the number only to correlate an already-presented capability /// with an immutable service-owned binding. Every effect still names the /// original slot and is re-authenticated by the kernel. + /// + /// Nor is it a liveness test: an ancestor's revocation leaves this record + /// unchanged, so a dead lineage keeps answering its scope id. Ask the + /// kernel instead ([`probe_chan_send_lineage`], [`probe_chan_recv_lineage`]). #[cfg(target_os = "none")] pub fn capability_scope_rid( slot: SlotRef, @@ -1505,52 +1529,96 @@ mod metal { /// caller-owned `TaskToken` reaps a child, without retaining a second task /// handle or consuming a still-running child's canary. #[cfg(target_os = "none")] - pub fn probe_chan_recv_lineage(parent: SlotRef) -> ChanRecvLineage { + pub fn probe_chan_recv_lineage(parent: SlotRef) -> ChanLineage { + probe_chan_lineage(parent, ChanFace::Recv) + } + + /// Probe whether a held send endpoint's complete lineage is still live, + /// without revoking or using it. + /// + /// The send-side twin of [`probe_chan_recv_lineage`], and the only sound + /// liveness test for an endpoint an ancestor may have revoked: revocation + /// retires the ancestor's tag, and every descendant dies through its + /// stamped chain *without* its record being rewritten, so a held record + /// keeps reading as present (and its scope id as current) after its + /// lineage is dead. Only a kernel bind tells. An endpoint without DERIVE + /// (an identity canary) is answered by the refusal of the derive, which + /// the kernel reaches only once the bind has proven it live. + #[cfg(target_os = "none")] + pub fn probe_chan_send_lineage(parent: SlotRef) -> ChanLineage { + probe_chan_lineage(parent, ChanFace::Send) + } + + /// One derive-and-retire transaction against `parent`'s `face`. + #[cfg(target_os = "none")] + fn probe_chan_lineage(parent: SlotRef, face: ChanFace) -> ChanLineage { use crate::session::SessionBuilder; use crate::transport::{Transport, TrapTransport}; let Some(mut child) = crate::cap::OwnedSlot::reserve() else { - return ChanRecvLineage::Retryable; + return ChanLineage::Retryable; }; let mut probe = SessionBuilder::new(); - let Ok(staged) = - probe.derive_chan_recv(parent, crate::cap::CHANNEL_ENDPOINT_RIGHTS, child.slot()) - else { - return ChanRecvLineage::Retryable; + let staged = match face { + ChanFace::Recv => { + probe.derive_chan_recv(parent, crate::cap::CHANNEL_ENDPOINT_RIGHTS, child.slot()) + } + // Any subset of the parent's rights names a valid descendant; a + // dead parent refuses `Revoked` whatever is asked. + ChanFace::Send => match authority_record(parent, kore_abi::cap::ResourceKind::ChanSend) { + Some(rights) => probe.derive_chan_send(parent, rights, child.slot()), + None => return ChanLineage::Retryable, + }, + }; + let Ok(staged) = staged else { + return ChanLineage::Retryable; }; // The probe result is a transactional witness only. Kill it in the // same session so a live-parent probe cannot accumulate descendants; // the local result name is then settled as AlreadyDead below. if probe.revoke(staged.into()).is_err() { - return ChanRecvLineage::Retryable; + return ChanLineage::Retryable; } let Ok(committed) = TrapTransport::new().commit_now(&probe) else { child.quarantine_unknown_publish(); - return ChanRecvLineage::Retryable; + return ChanLineage::Retryable; }; - match classify_chan_recv_probe(committed) { - ChanRecvLineage::Live => { - let owner = ChanRecvToken::new(child); - match owner.try_retire() { - Ok(_) => ChanRecvLineage::Live, - Err(_) => ChanRecvLineage::Retryable, + match classify_chan_probe(committed, face) { + // A refused derive published nothing: the reservation is free. + ChanLineage::Live if committed.status() != kore_abi::session::Status::Ok => { + ChanLineage::Live + } + ChanLineage::Live => { + let retired = match face { + ChanFace::Recv => crate::cap::ChanRecvToken::new(child).try_retire().is_ok(), + ChanFace::Send => crate::cap::ChanSendToken::new(child).try_retire().is_ok(), + }; + if retired { + ChanLineage::Live + } else { + ChanLineage::Retryable } } - ChanRecvLineage::Dead => ChanRecvLineage::Dead, - ChanRecvLineage::Retryable => { + ChanLineage::Dead => ChanLineage::Dead, + ChanLineage::Retryable => { if committed.failing_index() != 0 || committed.raw() as u16 > kore_abi::session::Status::AltDelivery as u16 { child.quarantine_unknown_publish(); } - ChanRecvLineage::Retryable + ChanLineage::Retryable } } } #[cfg(not(target_os = "none"))] - pub fn probe_chan_recv_lineage(_parent: SlotRef) -> ChanRecvLineage { - ChanRecvLineage::Retryable + pub fn probe_chan_recv_lineage(_parent: SlotRef) -> ChanLineage { + ChanLineage::Retryable + } + + #[cfg(not(target_os = "none"))] + pub fn probe_chan_send_lineage(_parent: SlotRef) -> ChanLineage { + ChanLineage::Retryable } /// Derive one locally retained endpoint below `source`. @@ -2969,6 +3037,99 @@ mod metal { } } + /// Outcome of one blocking wait on a memory word — the kernel + /// address-wait (`res/EVENT.md` §6). Every answer is a hint about the + /// *wait*, never a promise about the word: the caller re-reads it, the + /// `futex` contract. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum WordWait { + /// The word was seen to differ from `expected` (at park time, or a + /// `wake_word` on the same memory fired). + Changed, + /// The deadline expired first; the word may or may not have moved. + Deadline, + /// The subject's lease died while this task was parked. + Revoked, + /// The kernel's bounded waiter board was full — nothing parked; the + /// caller re-checks and parks again. + WouldBlock, + } + + /// Why [`wait_word`] could not complete one word-backed wait. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum WordWaitError { + /// The target has no live parking trap or descriptor scratch. + TransportUnavailable, + /// The kernel refused the wait session with this status — a + /// malformed request (offset outside the subject, misaligned), a + /// subject that is not held memory, or a forged slot. + Refused(kore_abi::session::Status), + } + + /// Park this thread until the `u32` at byte `offset` of the held memory + /// `subject` (a lease or mapped view, by arena slot — see + /// [`mapped::locate`](crate::cap::mapped::locate)) differs from + /// `expected`, or `deadline_ns` (absolute monotonic nanoseconds) passes + /// — the **sleep-or-word park** (`WAIT_WORD | WAIT_SLEEP`, no doorbell + /// arm). The kernel keys the wait on the word's physical identity, so + /// a [`wake_word`] on any mapping of the same memory resolves it. + /// + /// The bare-thread twin of the executor's word arm + /// (`executor::next_wake_on_word`): a thread with no doorbell — a + /// worker sharing a window and nothing else — can wait here, which is + /// the provisioning gap the `futex` module documents closing. Deadline + /// discipline is [`wait_interrupt_until`]'s: name one absolute instant + /// and park against it for the whole retry loop. + pub fn wait_word( + subject: SlotRef, + offset: u32, + expected: u32, + deadline_ns: u64, + ) -> Result { + use crate::session::SessionBuilder; + use crate::transport::{ParkTransport, TrapTransport}; + use kore_abi::session::Status; + + let mut wait = SessionBuilder::new(); + wait.wait_word(subject, offset, expected).wait_sleep(deadline_ns); + let committed = TrapTransport::new() + .commit_word_park(&wait) + .map_err(|_| WordWaitError::TransportUnavailable)?; + match committed.status() { + Status::Word => Ok(WordWait::Changed), + Status::Deadline => Ok(WordWait::Deadline), + Status::Revoked => Ok(WordWait::Revoked), + Status::WouldBlock => Ok(WordWait::WouldBlock), + status => Err(WordWaitError::Refused(status)), + } + } + + /// One **synchronous** wake of up to `count` waiters parked on the + /// `u32` at byte `offset` of the held memory `subject` (`MODIFY`) — + /// the producer half of the kernel address-wait, and [`post_notice`]'s + /// twin for a producer that shares the memory but holds no send end. + /// The caller stores first and wakes second. `true` on `Status::Ok`; + /// `false` on any refusal — a droppable kick, exactly as for a notice: + /// a waiter that was not yet parked re-reads the word before it parks. + #[cfg(target_os = "none")] + pub fn wake_word(subject: SlotRef, offset: u32, count: u32) -> bool { + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + + let mut b = SessionBuilder::new(); + if b.wake_word(subject, offset, count).is_err() { + return false; + } + TrapTransport::new() + .commit_now(&b) + .is_ok_and(|c| c.status() == kore_abi::session::Status::Ok) + } + + #[cfg(not(target_os = "none"))] + pub fn wake_word(_subject: SlotRef, _offset: u32, _count: u32) -> bool { + false + } + /// **Park until `deadline_ns`** (absolute monotonic nanoseconds) — the /// pure `WAIT_SLEEP` sliver (USERLAND.md §3.4.2): no doorbell, no /// route, just the per-CPU timer wheel. The thread shape (#407): a @@ -3127,7 +3288,11 @@ mod metal { /// [`MappedLeaseView`]: crate::cap::MappedLeaseView #[inline] pub fn retire_mapped_view(view: SlotRef, lease: SlotRef) -> RetireOutcome { - retire_owned_view(view, lease) + let outcome = retire_owned_view(view, lease); + if outcome.is_closed() { + crate::cap::mapped::forget_slot(view); + } + outcome } /// Carve one affine DMA sub-lease, map its CPU face, zero it, and only then @@ -4096,11 +4261,15 @@ mod metal { use crate::transport::{Transport, TrapTransport}; let report_slot = crate::entry::geometry::report_slot()?; + // The view's arena slot stays the subject after the map (the record + // is not consumed) — captured before the token is spent so the + // window can be named to the kernel address-wait by that slot. + let subject = view.slot(); let mut b = SessionBuilder::new(); if sink { - b.map_view_sink(view.slot(), report_slot).ok()?; + b.map_view_sink(subject, report_slot).ok()?; } else { - b.derive_raw(view.slot(), Rights::empty(), 0, report_slot, None) + b.derive_raw(subject, Rights::empty(), 0, report_slot, None) .ok()?; } let committed = TrapTransport::new().commit_now(&b).ok()?; @@ -4117,10 +4286,9 @@ mod metal { if g.len == 0 { return None; } - Some(NonNull::slice_from_raw_parts( - NonNull::new(g.base as *mut u8)?, - g.len as usize, - )) + let base = NonNull::new(g.base as *mut u8)?; + crate::cap::mapped::note(base.as_ptr(), g.len as usize, subject); + Some(NonNull::slice_from_raw_parts(base, g.len as usize)) } /// Map a runtime-owned detached view while retaining the affine arena @@ -4150,8 +4318,12 @@ mod metal { map_stashed_view_request(view, Some(kore_abi::session::SurfaceMapLane::First)) } // The heap window places the task's own heap-supply carves - // (`heap::source`), never a peer's view. - kore_abi::session::ViewMapWindow::Heap => None, + // (`heap::source`), never a peer's view; the reservation window + // holds only commits a `reserve::Reservation` made and will + // release, never a view something else still names. + kore_abi::session::ViewMapWindow::Heap | kore_abi::session::ViewMapWindow::Reserve => { + None + } } } @@ -4178,14 +4350,15 @@ mod metal { use crate::transport::{Transport, TrapTransport}; let report_slot = crate::entry::geometry::report_slot()?; + let subject = view.slot()?; let mut b = SessionBuilder::new(); match surface_lane { Some(lane) => b - .map_view_surface_lane(view.slot()?, lane, report_slot) + .map_view_surface_lane(subject, lane, report_slot) .ok()?, None => b .map_view_in( - view.slot()?, + subject, kore_abi::session::ViewMapWindow::Map, report_slot, ) @@ -4204,10 +4377,9 @@ mod metal { if geom.len == 0 { return None; } - Some(NonNull::slice_from_raw_parts( - NonNull::new(geom.base as *mut u8)?, - geom.len as usize, - )) + let base = NonNull::new(geom.base as *mut u8)?; + crate::cap::mapped::note(base.as_ptr(), geom.len as usize, subject); + Some(NonNull::slice_from_raw_parts(base, geom.len as usize)) } #[cfg(not(target_os = "none"))] @@ -4270,10 +4442,10 @@ mod metal { if g.len == 0 { return None; } - Some(NonNull::slice_from_raw_parts( - NonNull::new(g.base as *mut u8)?, - g.len as usize, - )) + // A whole-lease map: the lease itself names the window. + let base = NonNull::new(g.base as *mut u8)?; + crate::cap::mapped::note(base.as_ptr(), g.len as usize, lease); + Some(NonNull::slice_from_raw_parts(base, g.len as usize)) } /// Off the metal userland targets — the honest typed refusal (there is no @@ -4314,10 +4486,10 @@ mod metal { if g.len == 0 { return None; } - Some(NonNull::slice_from_raw_parts( - NonNull::new(g.base as *mut u8)?, - g.len as usize, - )) + // A whole-lease map: the lease itself names the window. + let base = NonNull::new(g.base as *mut u8)?; + crate::cap::mapped::note(base.as_ptr(), g.len as usize, lease); + Some(NonNull::slice_from_raw_parts(base, g.len as usize)) } /// Off the metal userland targets — the honest typed refusal (there is no @@ -4445,9 +4617,13 @@ mod metal { { return false; } - TrapTransport::new() + let mapped = TrapTransport::new() .commit_now(&b) - .is_ok_and(|committed| committed.status() == kore_abi::session::Status::Ok) + .is_ok_and(|committed| committed.status() == kore_abi::session::Status::Ok); + if mapped && let Ok(len) = usize::try_from(len) { + crate::cap::mapped::note(va as usize as *const u8, len, result); + } + mapped } #[cfg(not(target_os = "none"))] @@ -5199,6 +5375,10 @@ mod metal { let Ok(committed) = TrapTransport::new().commit_now(&b) else { return false; }; + // A revoked slot names no window any more, whatever it was: drop any + // registry entry that still points at it, or a later wait would hand + // the kernel a dead slot (or, once reused, someone else's). + crate::cap::mapped::forget_slot(slot); committed.status() == kore_abi::session::Status::Ok } @@ -5378,6 +5558,59 @@ mod metal { false } + /// The supply tag an [`InvocationWitness`] delivered into `slot` vouches + /// for, or `None` when the slot holds anything else. + /// + /// The kernel mints the witness from one executable endpoint for one + /// exact held supply, moves it before that supply through the same + /// endpoint in one session, and refuses a later `Found` that delivers + /// the witness without that exact supply. Its resource id is the + /// supply's tag index, and tags are never reissued, so a receiver that + /// holds both can tell which landed supply belongs to which invocation. + /// This reads the authenticated record as metadata; the founding commit + /// still performs the authoritative check. + /// + /// [`InvocationWitness`]: kore_abi::cap::ResourceKind::InvocationWitness + #[cfg(target_os = "none")] + pub fn delivered_invocation_supply(slot: SlotRef) -> Option { + use kore_abi::cap::{CapRecord, ResourceKind, Tag}; + + let ptr = crate::entry::geometry::slot_record_va(slot)?; + // SAFETY: the R/O `SLOTS_VA` alias over this task's own slot records + // (the `take_delivery` argument above); `slot` was range-checked by + // `slot_record_va`, and the kernel is the sole writer. + let record = unsafe { core::ptr::read_volatile(ptr as *const CapRecord) }; + (!record.cid.is_null() && record.kind == ResourceKind::InvocationWitness as u16) + .then(|| Tag(record.scope_id())) + } + + /// Off the metal userland targets — no live record to inspect. + #[cfg(not(target_os = "none"))] + pub fn delivered_invocation_supply(_slot: SlotRef) -> Option { + None + } + + /// The revocation tag of a memory lease delivered into `slot`, or `None` + /// when the slot holds anything else. A rendezvous move keeps the moved + /// lease's identity, so this is the tag an + /// [`delivered_invocation_supply`] witness names for it. + #[cfg(target_os = "none")] + pub fn delivered_lease_tag(slot: SlotRef) -> Option { + use kore_abi::cap::{CapRecord, ResourceKind}; + + let ptr = crate::entry::geometry::slot_record_va(slot)?; + // SAFETY: as in `delivered_invocation_supply`. + let record = unsafe { core::ptr::read_volatile(ptr as *const CapRecord) }; + (!record.cid.is_null() && record.kind == ResourceKind::MemLease as u16) + .then(|| record.cid.tag()) + } + + /// Off the metal userland targets — no live record to inspect. + #[cfg(not(target_os = "none"))] + pub fn delivered_lease_tag(_slot: SlotRef) -> Option { + None + } + /// **The metal take-handle** (Phase S; Phase-A checkpoint C) — read a /// delivered task handle off the R/O cap-slot alias at /// [`SLOTS`](kore_abi::task::geom::SLOTS) alias and return a [`TaskToken`] @@ -5943,22 +6176,41 @@ mod metal { } #[test] - fn receive_lineage_probe_requires_positive_live_or_dead_evidence() { - assert_eq!( - classify_chan_recv_probe(committed(Status::Ok, 0)), - ChanRecvLineage::Live - ); + fn lineage_probes_require_positive_live_or_dead_evidence() { + for face in [ChanFace::Recv, ChanFace::Send] { + assert_eq!(classify_chan_probe(committed(Status::Ok, 0), face), ChanLineage::Live); + assert_eq!( + classify_chan_probe(committed(Status::Revoked, 0), face), + ChanLineage::Dead + ); + assert_eq!( + classify_chan_probe(committed(Status::WouldBlock, 0), face), + ChanLineage::Retryable + ); + assert_eq!( + classify_chan_probe(committed(Status::Ok, 1), face), + ChanLineage::Retryable + ); + // A forged or wrong-holder record proves nothing either way. + assert_eq!( + classify_chan_probe(committed(Status::Forged, 0), face), + ChanLineage::Retryable + ); + } + // A send endpoint without DERIVE is answered by the derive's + // refusal, which the kernel reaches only after binding it live; a + // receive parent always carries DERIVE, so there it means nothing. assert_eq!( - classify_chan_recv_probe(committed(Status::Revoked, 0)), - ChanRecvLineage::Dead + classify_chan_probe(committed(Status::Denied, 0), ChanFace::Send), + ChanLineage::Live ); assert_eq!( - classify_chan_recv_probe(committed(Status::WouldBlock, 0)), - ChanRecvLineage::Retryable + classify_chan_probe(committed(Status::Denied, 0), ChanFace::Recv), + ChanLineage::Retryable ); assert_eq!( - classify_chan_recv_probe(committed(Status::Ok, 1)), - ChanRecvLineage::Retryable + classify_chan_probe(committed(Status::Denied, 1), ChanFace::Send), + ChanLineage::Retryable ); } diff --git a/native/crates/rt/src/crash.rs b/native/crates/rt/src/crash.rs index 16ce04e..b06b58e 100644 --- a/native/crates/rt/src/crash.rs +++ b/native/crates/rt/src/crash.rs @@ -310,6 +310,16 @@ pub fn render(record: &CrashRecord, out: &mut impl Write, color: bool) -> fmt::R if flags.contains(CaptureFlags::BUFFER_CAPTURED) { out.write_str("note: parent RAM destination captured\n")?; } + if flags.contains(CaptureFlags::UPCALL_DECLINED) { + out.write_str("note: the thread's fault upcall declined this fault\n")?; + } + if flags.contains(CaptureFlags::IN_UPCALL) { + writeln!( + out, + "note: raised inside the thread's fault upcall, handling the fault at {:#018x}", + record.upcall_pc.get() + )?; + } writeln!( out, "note: {title}; snapshot captured (durability reported separately)" @@ -343,6 +353,35 @@ mod tests { assert!(out.contains("invalid or unsupported")); } #[test] + fn upcall_evidence_is_reported() { + let mut declined = + CrashRecord::new(Architecture::X86_64, FaultKind::Memory, 0x1000, 0x2000, 6); + declined.add_flags(CaptureFlags::UPCALL_DECLINED); + let mut out = std::string::String::new(); + render(&declined, &mut out, false).unwrap(); + assert!( + out.contains("note: the thread's fault upcall declined this fault"), + "{out}" + ); + assert!(!out.contains("inside the thread's fault upcall"), "{out}"); + + let mut nested = CrashRecord::new( + Architecture::X86_64, + FaultKind::InvalidInstruction, + 0x3000, + 0x4000, + 0, + ); + nested.set_in_upcall(0x1000); + out.clear(); + render(&nested, &mut out, false).unwrap(); + assert!( + out.contains("note: raised inside the thread's fault upcall, handling the fault at 0x0000000000001000"), + "{out}" + ); + assert!(!out.contains("declined"), "{out}"); + } + #[test] fn preparing_a_buffer_does_not_publish_old_bytes() { let mut small = [0; MIN_BUFFER_BYTES - 1]; assert!(CrashBuffer::new(&mut small).is_none()); diff --git a/native/crates/rt/src/entry.rs b/native/crates/rt/src/entry.rs index c20fc64..aff740d 100644 --- a/native/crates/rt/src/entry.rs +++ b/native/crates/rt/src/entry.rs @@ -44,6 +44,7 @@ use kore_abi::clock::ClockPage; use kore_abi::founding::FoundingBlock; use kore_abi::image::{IMAGE_ABI, LdBlock}; +use kore_abi::identity::IdentityBlock; use kore_abi::machine::MachineBlock; use kore_abi::session::SlotRef; use kore_abi::task::{BootBlock, Bootstrap, BootstrapError, GeomBlock, role}; @@ -123,6 +124,22 @@ const _: () = assert!( "the machine block must preserve its natural alignment" ); +/// Byte offset of the **build-identity block** +/// ([`IdentityBlock`](kore_abi::identity::IdentityBlock)) — the seventh +/// block: the running kernel's build stamp, kernel-stamped at spawn so +/// `status sys` and a screenshot can name the image they ran on +/// ([`kernel_build`]). A zeroed region reads `abi == 0` and exposes nothing. +pub const IDENTITY_BLOCK_AT: usize = MACHINE_BLOCK_AT + core::mem::size_of::(); + +const _: () = assert!( + IDENTITY_BLOCK_AT + core::mem::size_of::() <= BOOT_PAGE_BYTES, + "the frozen boot-page blocks must fit one page" +); +const _: () = assert!( + IDENTITY_BLOCK_AT.is_multiple_of(core::mem::align_of::()), + "the identity block must preserve its natural alignment" +); + /// Why the entry refused to come up (fail-closed: a task with an /// invalid boot page never reaches `main`). #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -148,6 +165,7 @@ pub struct Rt { clock: Option<&'static ClockPage>, founding: Option<&'static FoundingBlock>, machine: Option<&'static MachineBlock>, + identity: Option<&'static IdentityBlock>, page: &'static [u8], } @@ -271,6 +289,14 @@ impl Rt { self.machine } + /// The running kernel's build stamp (`KORE_BUILD_STAMP`), kernel + /// testimony on this task's boot page. Ambient data like the machine + /// block: reading it grants nothing. `None` on a pre-record founder or + /// a zeroed region. + pub fn kernel_build(&self) -> Option<&'static str> { + self.identity?.stamp() + } + /// The immutable parent-written boot page. /// /// Generic runtime code consumes the standard blocks through the typed @@ -393,6 +419,11 @@ pub fn boot_context(page: &'static [u8]) -> Result { .and_then(|b| LdBlock::ref_from_prefix(b).ok()) .map(|(ld, _)| ld) .filter(|ld| ld.abi.get() == IMAGE_ABI); + let identity = page + .get(IDENTITY_BLOCK_AT..) + .and_then(|b| IdentityBlock::ref_from_prefix(b).ok()) + .map(|(identity, _)| identity) + .filter(|identity| identity.stamp().is_some()); // **Install the geometry statics** (metal only) before any verb or the trap // transport runs — they replace the hand-copied `M2B_SCRATCH_VA`/`SLOTS_VA` // constants with the founder's actual mapping, learned from the boot page. @@ -429,6 +460,7 @@ pub fn boot_context(page: &'static [u8]) -> Result { clock, founding, machine, + identity, page, }) } @@ -1741,28 +1773,14 @@ fn worker_caps_with_heap( declared: &crate::service::WorkerEndowment, out: &mut [crate::thread::WorkerCap; crate::thread::MAX_WORKER_CAPS], ) -> Option { - use kore_abi::cap::{ResourceKind, Rights}; use kore_abi::task::role; - // The global allocator is process-wide memory but its grow operation is - // capability-gated per task. A worker therefore needs its own explicit - // lessor position over the founder-seeded supply; sharing the parent's VAS - // does not make the parent's holding reachable. The no-arg self-kind - // MemLease derive is the session engine's founding/endowment attenuation: - // the parent keeps the supply and the child receives a whole-span sublease. + // A worker allocates, so it is founded holding the process heap + // (`WorkerCap::heap` says why its own position is needed). if declared.len() >= out.len() { return None; } - out[0] = crate::thread::WorkerCap { - role: role::HEAP, - index: 0, - subject: heap, - kind: ResourceKind::MemLease as u16 as u8, - // Heap growth needs only carve + map authority. The parent retains - // REVOKE over this sub-lessor; the worker does not receive teardown - // authority it never exercises. - rights: Rights::ACCESS | Rights::MODIFY | Rights::DERIVE, - }; + out[0] = crate::thread::WorkerCap::heap(heap); for (i, cap) in declared.caps().enumerate() { // HEAP@0 is runtime infrastructure, not a service-selected device // authority. Reject a duplicate instead of letting boot stamping fail @@ -2211,6 +2229,37 @@ fn runtime_slots_after(top: u32) -> Option { }) } +#[cfg(test)] +mod identity_tests { + use super::*; + use zerocopy::IntoBytes; + + /// A boot page with a valid boot block and, optionally, an identity + /// block written as the kernel writes it. + fn page(identity: Option<&[u8]>) -> &'static [u8] { + let page = std::vec![0u8; BOOT_PAGE_BYTES].leak(); + let boot = kore_abi::task::BootBlock::new(); + page[BOOT_BLOCK_AT..][..core::mem::size_of_val(&boot)].copy_from_slice(boot.as_bytes()); + if let Some(bytes) = identity { + page[IDENTITY_BLOCK_AT..][..bytes.len()].copy_from_slice(bytes); + } + page + } + + #[test] + fn the_kernel_build_reads_off_the_boot_page_and_fails_closed() { + let stamped = IdentityBlock::new("304bfef9ca08-dirty").unwrap(); + let rt = boot_context(page(Some(stamped.as_bytes()))).unwrap(); + assert_eq!(rt.kernel_build(), Some("304bfef9ca08-dirty")); + // A founder that predates the block leaves the region zero. + assert_eq!(boot_context(page(None)).unwrap().kernel_build(), None); + // A damaged block names no build, rather than a wrong one. + let mut damaged = stamped; + damaged.stamp[0] = 0; + assert_eq!(boot_context(page(Some(damaged.as_bytes()))).unwrap().kernel_build(), None); + } +} + #[cfg(test)] mod runtime_slot_tests { use super::*; @@ -2441,6 +2490,24 @@ pub fn run_service(rt: Rt) -> ! { run_main(rt, serve::) } +/// **How many fixed serving pairs this row was founded with** — the +/// protocol's [`SERVE_PAIRS`](crate::svc::SvcProtocol::SERVE_PAIRS) plus one +/// per manifest row that declares this service among its `clients` (the +/// founder stamps the total as [`SERVICE_PAIR_COUNT_DATUM`] whenever the two +/// differ). A service that numbers its own pairs — a per-pair meter, or a +/// recyclable peer's slot that must never land on a fixed pair — counts this, +/// not the protocol constant: the declared clients' pairs follow the +/// protocol's and are just as fixed. `None` for a count no host can attach. +/// +/// [`SERVICE_PAIR_COUNT_DATUM`]: kore_abi::task::SERVICE_PAIR_COUNT_DATUM +pub fn service_pair_count(rt: &Rt) -> Option { + let count = rt + .bootstrap() + .datum(role::SVC_RING, kore_abi::task::SERVICE_PAIR_COUNT_DATUM) + .map_or(S::SERVE_PAIRS, |value| value as usize); + (count >= S::SERVE_PAIRS && count != 0 && count <= crate::svc::MAX_SERVE_PAIRS).then_some(count) +} + /// The service ladder **as a root future** — what `#[kore_rt::main]` /// drives for the canned service shape: /// @@ -2457,14 +2524,6 @@ pub fn run_service(rt: Rt) -> ! { /// implementation rather than two that drift — which is what happened /// while services bypassed it: this ladder grew endowment resolution and /// the readiness beacon, and `run_main` did not. -fn service_pair_count(rt: &Rt) -> Option { - let count = rt - .bootstrap() - .datum(role::SVC_RING, kore_abi::task::SERVICE_PAIR_COUNT_DATUM) - .map_or(S::SERVE_PAIRS, |value| value as usize); - (count >= S::SERVE_PAIRS && count != 0 && count <= crate::svc::MAX_SERVE_PAIRS).then_some(count) -} - pub async fn serve(rt: Rt) -> ! { use crate::service::Bundle; if !service_ready::(&rt) { @@ -2698,6 +2757,12 @@ async fn serve_forever_with_return_targets( return; } } + #[cfg(feature = "heap")] + if let Some(arena) = host.service().peer_arena() { + if !host.enable_peer_growth(arena) { + return; + } + } // The extra pairs of a multi-client row — same ladder, that pair's // role indices: ring `SVC_RING@k`, kick `SVC_SEND@k`, grant channels // `SVC_GRANT@(k·grants ..)`, return legs `SVC_RETURN@2k/2k+1`. The diff --git a/native/crates/rt/src/event.rs b/native/crates/rt/src/event.rs index df3bbb7..cdae36d 100644 --- a/native/crates/rt/src/event.rs +++ b/native/crates/rt/src/event.rs @@ -10,14 +10,23 @@ //! nobody's configuration: **check** (the word already moved — return), //! **monitor** (sleep the core on the word's cache line for a few //! microseconds; the producer's ordinary store is the wake, no syscall -//! on either side), **park** (the executor's standing doorbell post -//! with the deadline folded in; the producer's conditional notice -//! resolves it). A kernel address-wait slots between the last two -//! when it lands (`res/EVENT.md` §6) without changing a caller. +//! on either side), **park on the word** (rung 3′ — the kernel +//! address-wait, `res/EVENT.md` §6: the executor's idle park carries +//! the wake word by the *authority* that mapped it, and the producer's +//! `wake_word` — or the word being seen moved at park time — resolves +//! it), else **park on the doorbell** (rung 3 — the executor's standing +//! post with the deadline folded in; the producer's conditional notice +//! resolves it). Rung 3′ is taken exactly when the channel's window is +//! one this task mapped through a noting verb +//! ([`cap::mapped::locate`](crate::cap::mapped::locate)) and the +//! executor's one word arm is free; otherwise rung 3, unchanged. No +//! caller changed for it to land. //! - [`commit_and_kick`] — the producer's one batch end. Advances the -//! wake word, and sends the notice **only if the consumer had -//! committed to waiting** — a running consumer costs the producer -//! zero syscalls, which is the whole point at load. +//! wake word, and wakes the consumer **only if it had committed to +//! waiting** — by `wake_word` on the memory when the consumer parked on +//! the word and this side can name it, by the notice otherwise — so a +//! running consumer costs the producer zero syscalls, which is the +//! whole point at load. //! //! Every return is a hint; the memory is the truth. Callers re-check in //! a loop, exactly as [`futex`](crate::futex) and the ring teach. What @@ -30,9 +39,11 @@ use core::sync::atomic::AtomicU32; use core::time::Duration; use kore_abi::session::Notice; -use kore_ring::event::{EventSink, EventSource, EventState}; +use kore_ring::event::{EventSink, EventSource, EventState, PARK_WORD}; use kore_ring::spsc::RingRecord; +use crate::cap::mapped; +use crate::cap::verbs; use crate::chan::ChanTx; use crate::transport::Transport; @@ -64,6 +75,14 @@ pub enum EventWait { const MONITOR_TURNS: usize = 2; const MONITOR_CAP: Duration = Duration::from_micros(2); +/// Bound of one word park when the caller asked for no deadline: the +/// kernel address-wait is never unbounded, so an open-ended wait is a +/// loop of bounded parks, each re-checking the word — a slice long enough +/// that an idle consumer is not paced by it, short enough that a lost +/// wake (a producer that could name neither word nor doorbell) costs one +/// slice, not forever. +const WORD_PARK_SLICE_NS: u64 = 250_000_000; + /// Wait until the channel's wake word moves past `seen` (a value the /// caller read with [`EventSink::seq`] *before* its last check), or /// until `timeout` elapses; `None` waits until woken. @@ -87,6 +106,11 @@ const MONITOR_CAP: Duration = Duration::from_micros(2); /// A cancelled wait (the future dropped mid-park) withdraws its park /// intent on drop, so a departed waiter is never charged a notice. /// +/// The word rung (3′) never lengthens a wait: it re-checks `seq` on every +/// wake exactly as the doorbell rung does, keeps the same deadline, and +/// keeps the doorbell post standing beneath it — a producer that cannot +/// name the word still reaches this waiter with its notice. +/// /// # Panics /// Polled outside the kore executor, as /// [`next_wake`](crate::executor::next_wake): the park rung is the @@ -158,10 +182,56 @@ async fn wait_with( ), None => None, }; + // Rung 3′ — park on the word itself, when this task can name it by + // the authority that mapped it. The intent is restated as a word park + // before every re-park (the value tells the producer which wake + // reaches us; the Dekker pair above is untouched — the kernel makes + // the final `seq` check against the physical word at park time). The + // arm can be refused (the executor's one word arm is owned) or evicted + // (an alternate delivery arm arrived); either way the wait falls back + // to rung 3 by re-publishing the doorbell intent through `park`, whose + // own re-check covers the gap. + let mut word = mapped::locate(sink.seq_word() as *const AtomicU32 as *const u8); loop { if sink.seq() != seen { return EventWait::Changed; } + if let Some((subject, offset)) = word { + // An unbounded wait still bounds each word park (the kernel + // requires it); the loop's re-check makes the bound invisible. + let bound = match deadline { + Some(at) => { + if crate::executor::clock_ns().await >= at { + return EventWait::TimedOut; + } + at + } + None => crate::executor::clock_ns() + .await + .saturating_add(WORD_PARK_SLICE_NS), + }; + sink.park_on_word(); + match crate::executor::next_wake_on_word(subject, offset, seen, bound).await { + Some(_) if once => { + // A word park also carries the executor's doorbell. A + // composed waiter must drain its other inputs on that + // wake, even when this channel's word stayed unchanged. + return if sink.seq() != seen { + EventWait::Changed + } else { + EventWait::Woken + }; + } + Some(_) => continue, + None => { + word = None; + if !sink.park(seen) { + return EventWait::Changed; + } + continue; + } + } + } match deadline { None => { let _ = crate::executor::next_wake().await; @@ -189,10 +259,12 @@ async fn wait_with( } } -/// End a producer batch: advance the wake word, and kick the consumer's -/// doorbell **only if it had committed to waiting**. +/// End a producer batch: advance the wake word, and wake the consumer +/// **only if it had committed to waiting** — a `wake_word` on the memory +/// when it parked on the word and this side can name it (`res/EVENT.md` +/// §6), the doorbell notice otherwise. /// -/// Returns whether a notice went out. `Err(WouldBlock)` from the send +/// Returns whether a wake went out. `Err(WouldBlock)` from the send /// is treated as delivered-enough and dropped: a full doorbell means the /// consumer has an unconsumed wake standing, and the §3.1 order (the /// channel writes happen-before this kick) guarantees its next drain @@ -203,9 +275,21 @@ pub async fn commit_and_kick( source: &mut EventSource<'_, S, R>, kick: &ChanTx<'_, T>, ) -> bool { - if !source.commit() { + let intent = source.commit_intent(); + if intent == 0 { return false; } + // The consumer parked on the word (rung 3′): wake it on the memory, + // by this side's own mapping of it — no send end needed, one trap. + // A word this side cannot name, or a refused wake, falls back to the + // notice: the consumer's word park carries its doorbell post too. + if intent == PARK_WORD + && let Some((subject, offset)) = + mapped::locate(source.seq_word() as *const AtomicU32 as *const u8) + && verbs::wake_word(subject, offset, u32::MAX) + { + return true; + } kick.send(Notice { cookie: COOKIE_EVENT, payload: 0, @@ -217,6 +301,63 @@ pub async fn commit_and_kick( #[cfg(test)] mod tests { use super::*; + use kore_abi::session::SlotRef; + use kore_ring::event::{PARK_DOORBELL, SEQ_AT, split}; + use std::vec; + + #[repr(C)] + #[derive(Clone, Copy, Debug, PartialEq, Eq, zerocopy::FromBytes, zerocopy::IntoBytes, zerocopy::Immutable)] + struct State { + serial: u64, + } + + #[repr(C)] + #[derive(Clone, Copy, Debug, PartialEq, Eq, zerocopy::FromBytes, zerocopy::IntoBytes, zerocopy::Immutable)] + struct Record { + value: u32, + _pad: u32, + } + + /// Rung selection is a fact about the window, not the caller: a channel + /// in memory nobody noted keeps the doorbell rung, and once its window + /// is noted the wake word names itself as `(slot, SEQ_AT)` — the exact + /// pair the `WordWaitReq` / `WakeWordReq` carry — without the consumer + /// or producer changing a line. + #[test] + fn the_word_rung_is_taken_exactly_when_the_window_is_noted() { + let bytes = kore_ring::event::region_bytes::(4) + 64; + let mut backing = vec![0u8; bytes]; + let off = backing.as_ptr().align_offset(64); + let window = &mut backing[off..]; + let (base, len) = (window.as_ptr(), window.len()); + let (mut source, sink) = split::(window, 4).unwrap(); + + let word = sink.seq_word() as *const AtomicU32 as *const u8; + assert_eq!(mapped::locate(word), None, "an unnoted window: rung 3"); + + let slot = SlotRef::from_wire(21); + mapped::note(base, len, slot); + assert_eq!(mapped::locate(word), Some((slot, SEQ_AT as u32))); + assert_eq!( + mapped::locate(source.seq_word() as *const AtomicU32 as *const u8), + Some((slot, SEQ_AT as u32)), + "both halves name the same word" + ); + + // The intent word tells the producer which wake reaches the + // consumer: a plain park is the doorbell, restated as the word park. + let seen = sink.seq(); + assert!(sink.park(seen)); + assert_eq!(sink.parked_word().load(core::sync::atomic::Ordering::SeqCst), PARK_DOORBELL); + sink.park_on_word(); + assert_eq!(sink.parked_word().load(core::sync::atomic::Ordering::SeqCst), PARK_WORD); + assert_eq!(source.commit_intent(), PARK_WORD, "collected as a word park"); + assert_eq!(source.commit_intent(), 0, "collected once"); + + mapped::forget(base); + assert_eq!(mapped::locate(word), None, "an unmapped window: rung 3 again"); + } + use core::future::Future; use core::sync::atomic::Ordering; use core::task::{Context, Poll, Waker}; diff --git a/native/crates/rt/src/executor/mod.rs b/native/crates/rt/src/executor/mod.rs index 1075f0a..1124641 100644 --- a/native/crates/rt/src/executor/mod.rs +++ b/native/crates/rt/src/executor/mod.rs @@ -20,7 +20,8 @@ //! coalescing: the state that matters is in the rings, the notice only //! says "look". The idle cycle runs the **pinned order**: //! -//! 1. **consume** the delivered notice (the cookie is a hint); +//! 1. **consume** the delivered notice (the cookie is a hint; a doorbell +//! owner that named a [`NoticeSink`] is offered it); //! 2. **re-post** the doorbell recv *as its own commit* — the post now //! stands, so any kick during the drain is accepted, not refused //! (which is what makes a `WouldBlock`ed kick droppable: the @@ -68,13 +69,109 @@ use crate::wake; pub use task::{SpawnRefused, SpawnToken, TaskStorage}; pub use time::{ WakeReason, clock_ns, enable_irq_parks, next_wake, next_wake_for, next_wake_on, - next_wake_on_for, next_wake_until, quiesce_asked, quiesce_deadline, retire_irq_parks, - sleep_for, sleep_until, take_quiesce_ask, + next_wake_on_for, next_wake_on_word, next_wake_until, quiesce_asked, quiesce_deadline, + retire_irq_parks, sleep_for, sleep_until, take_quiesce_ask, }; use task::{TaskHeader, poll_task, waker_for}; pub(crate) use time::ExecCtx; +/// Where an executor hands the doorbell notices it consumes. +/// +/// For the runtime's own kicks, a service's reply or report, a doorbell +/// notice is only a wake: the executor consumes it, re-stands the post and +/// re-polls every source, and the cookie is a hint. A doorbell can also carry +/// notices whose payload is data, such as a child's testimony on its parent's +/// doorbell, which the parent relays or acts on. An executor lent such a +/// doorbell ([`Executor::from_standing`]) would discard them, so the doorbell's +/// owner names a sink ([`Executor::with_notice_sink`]): every notice the +/// executor consumes, in its idle park or inline in a blocking call, is offered +/// to it in arrival order. The kernel's quiesce ask is the executor's own +/// ([`take_quiesce_ask`]) and is never offered. +pub trait NoticeSink { + /// One consumed notice. Called on the executor's thread, possibly from + /// within a poll; it must not re-enter the executor. + fn notice(&self, cookie: u64, payload: u64); +} + +/// Whether `cookie` names one of the runtime's bare wake kicks: a service's +/// request or reply, a data-plane report, an event. Each says only "look": +/// its payload is empty and the consumer re-checks its own sources on every +/// wake, so a doorbell owner holding notices for their data has nothing to +/// keep from one ([`NoticeHold`]). +pub const fn is_wake_hint(cookie: u64) -> bool { + matches!( + cookie, + crate::svc::COOKIE_REQUEST + | crate::svc::COOKIE_REPLY + | crate::svc::COOKIE_REPORT + | crate::event::COOKIE_EVENT + ) +} + +/// A [`NoticeSink`] that holds, in arrival order, the notices an executor +/// consumed on a lent doorbell that carry data, every one but the runtime's +/// wake hints ([`is_wake_hint`]), for the doorbell's owner to take and act on +/// as its own park would have. It holds at most `N`; a notice that meets it +/// full is counted ([`take_lost`](Self::take_lost)), never dropped silently. +pub struct NoticeHold { + ring: [Cell<(u64, u64)>; N], + head: Cell, + len: Cell, + lost: Cell, +} + +impl NoticeHold { + /// An empty hold. + pub const fn new() -> Self { + const { assert!(N > 0, "a notice hold holds at least one notice") }; + Self { + ring: [const { Cell::new((0, 0)) }; N], + head: Cell::new(0), + len: Cell::new(0), + lost: Cell::new(0), + } + } + + /// The oldest held notice, as `(cookie, payload)`. + pub fn take(&self) -> Option<(u64, u64)> { + let len = self.len.get(); + if len == 0 { + return None; + } + let head = self.head.get(); + self.head.set((head + 1) % N); + self.len.set(len - 1); + Some(self.ring[head].get()) + } + + /// How many notices met the hold full since the last call. + pub fn take_lost(&self) -> u32 { + self.lost.replace(0) + } +} + +impl Default for NoticeHold { + fn default() -> Self { + Self::new() + } +} + +impl NoticeSink for NoticeHold { + fn notice(&self, cookie: u64, payload: u64) { + if is_wake_hint(cookie) { + return; + } + let len = self.len.get(); + if len == N { + self.lost.set(self.lost.get().saturating_add(1)); + return; + } + self.ring[(self.head.get() + len) % N].set((cookie, payload)); + self.len.set(len + 1); + } +} + /// Concurrently admitted tasks per executor (excluding the root) — the /// §3.3/§8 design bound: spawn capacity is the declared task set, /// overflow refuses. @@ -152,6 +249,11 @@ pub(crate) enum ExecFault { /// compose, and this is executor-internal state rather than anything the /// kernel reported: two futures installed contradictory wake sources. AltDeliveryWithIrqPark, + /// A word arm and an alternate-delivery post stood at once — the wire + /// refuses `WAIT_WORD | WAIT_DELIVERY_ALT`, and the registry evicts the + /// word when the post arrives, so this is executor-internal state + /// contradicting itself. + WordArmWithAltPost, /// The kernel reported alternate delivery into a landing slot this /// thread's read-only slot alias cannot read at all — distinct from a /// readable slot holding no record, which is tolerated (retired rung @@ -177,6 +279,7 @@ impl ExecFault { Self::ParkTransport => (0x67, 0), Self::AltDeliveryWithIrqPark => (0x68, 0), Self::AltDeliveryUnreadable(slot) => (0x69, u64::from(slot)), + Self::WordArmWithAltPost => (0x6a, 0), }; rung | (detail << 8) } @@ -338,6 +441,13 @@ impl<'t, T: ParkTransport> Executor<'t, T> { Self::with_post_state(transport, doorbell, true) } + /// Offer every doorbell notice this executor consumes to `sink` + /// ([`NoticeSink`]). + pub fn with_notice_sink(self, sink: &'t dyn NoticeSink) -> Self { + self.ctx.set_notice_sink(sink); + self + } + fn with_post_state(transport: &'t T, doorbell: Doorbell, posted: bool) -> Self { Self { transport, @@ -525,11 +635,22 @@ impl<'t, T: ParkTransport> Executor<'t, T> { if !self.posted.get() { self.repost(); } - let deadline = self.ctx.timers_min(); let alternate = self.ctx.delivery_post(); if alternate.is_some() && self.ctx.irq() { executor_fault(ExecFault::AltDeliveryWithIrqPark); } + // The one word arm (`res/EVENT.md` §6). Its registrant also holds + // a timer entry for its deadline, so the folded minimum already + // carries the bound; the fold below is the belt to that brace — a + // word park is never unbounded. + let word = self.ctx.word_arm(); + if alternate.is_some() && word.is_some() { + executor_fault(ExecFault::WordArmWithAltPost); + } + let deadline = match word { + Some((_, bound)) => Some(self.ctx.timers_min().map_or(bound, |d| d.min(bound))), + None => self.ctx.timers_min(), + }; // ── the bell tier: monitor, then arm, then park ────────────── // // Two steps, and only the first is optional. The monitored window @@ -573,6 +694,9 @@ impl<'t, T: ParkTransport> Executor<'t, T> { if self.ctx.irq() { park.wait_irq(); } + if let Some((arm, _)) = word { + park.wait_word(arm.subject, arm.offset, arm.expected); + } let spec = ParkSpec { chan: self.doorbell.chan, land: self.doorbell.land, @@ -580,7 +704,15 @@ impl<'t, T: ParkTransport> Executor<'t, T> { deadline_ns: deadline, irq: self.ctx.irq(), }; - let Ok(committed) = self.transport.commit_and_park(&park, &spec) else { + // A word arm names a held capability the wire already carries in + // full, so it commits through the held-authority entry (the join + // park's precedent); the doorbell post and deadline ride alongside. + let committed = if word.is_some() { + self.transport.commit_word_park(&park) + } else { + self.transport.commit_and_park(&park, &spec) + }; + let Ok(committed) = committed else { executor_fault(ExecFault::ParkTransport) }; // Running again, whatever the outcome was: republish `AWAKE` before @@ -600,18 +732,14 @@ impl<'t, T: ParkTransport> Executor<'t, T> { // re-post as its own commit, then the TOTAL drain — the §3.1 // pinned order; the caller's poll loop is the drain. Status::Ok => { - match verbs::take_notice(self.doorbell.land) { - None => executor_fault(ExecFault::NoDoorbellNotice), - // The kernel's terminal quiesce ask rides the ordinary - // doorbell (#809 §L); its reserved cookie is refused to - // userland senders, so seeing it here is the kernel. - Some((cookie, budget_ns)) - if cookie == kore_abi::session::NOTICE_COOKIE_POWER_QUIESCE => - { - self.ctx.note_quiesce_ask(budget_ns); - } - Some(_) => {} - } + let Some((cookie, payload)) = verbs::take_notice(self.doorbell.land) else { + executor_fault(ExecFault::NoDoorbellNotice) + }; + // The kernel's terminal quiesce ask rides the ordinary + // doorbell (#809 §L); its reserved cookie is refused to + // userland senders, so seeing it here is the kernel. Every + // other notice goes to the owner's sink, if any. + self.ctx.note_notice(cookie, payload); self.cycles.set(self.cycles.get() + 1); self.ctx.note_wake(time::WakeReason::Doorbell); self.posted.set(false); @@ -647,10 +775,8 @@ impl<'t, T: ParkTransport> Executor<'t, T> { // Both posts may have completed before resolution. Consume // and re-stand the level-triggered doorbell when present, but // preserve Delivery as the edge-visible reason. - if let Some((cookie, budget_ns)) = verbs::take_notice(self.doorbell.land) { - if cookie == kore_abi::session::NOTICE_COOKIE_POWER_QUIESCE { - self.ctx.note_quiesce_ask(budget_ns); - } + if let Some((cookie, payload)) = verbs::take_notice(self.doorbell.land) { + self.ctx.note_notice(cookie, payload); self.cycles.set(self.cycles.get() + 1); self.posted.set(false); self.repost(); @@ -675,12 +801,39 @@ impl<'t, T: ParkTransport> Executor<'t, T> { self.ctx.note_wake(time::WakeReason::Irq); self.wake_all(root_hdr); } + // The word moved (or a `wake_word` on its memory fired): the + // post stands (the deadline win's rule again), and exactly the + // arm's owner wakes — its ready leg clears the registration and + // re-reads the word. + Status::Word => { + self.ctx.note_wake(time::WakeReason::Word); + self.ctx.wake_word_owner(); + } // A scheduler notification resumed this task while the exact // doorbell post still stood. There is no notice to consume and // no post to replace; loop back to the same standing park. This // outcome is deliberately quiet and does not wake every future: // no executor-visible source became ready. - Status::WouldBlock => {} + Status::WouldBlock => { + // A word park the kernel could not arm (its bounded waiter + // board was full) returns without having parked at all; + // withdraw the arm so its owner falls back to the doorbell + // rung instead of this cycle repeating forever. + if word.is_some() { + self.ctx.refuse_word(); + } + } + // The word's memory was torn down under the standing arm (the + // lease behind the window revoked — a window re-founded or + // retired while its client still waited). The wait is not + // wrong, its word is gone: withdraw the arm and let the owner + // re-check and take the doorbell, where the window's own + // replacement or refusal reaches it. Without a word arm a + // `Revoked` park is the unknown it always was. + Status::Revoked if word.is_some() => { + self.ctx.note_wake(time::WakeReason::Word); + self.ctx.refuse_word(); + } // The doorbell's recv end was reaped — the runtime's wake edge is // gone; nothing correct remains for THIS executor. // @@ -906,9 +1059,9 @@ pub(crate) async fn settle_inline_doorbell( doorbell: Doorbell, stamp_ms: u32, ) -> bool { - if verbs::take_notice(doorbell.land).is_none() { + let Some((cookie, payload)) = verbs::take_notice(doorbell.land) else { return false; - } + }; let Some(reposted_ms) = stand_doorbell(transport, doorbell).await else { return false; }; @@ -925,7 +1078,11 @@ pub(crate) async fn settle_inline_doorbell( ); // SAFETY: the assertion above and executor poll discipline prove this // pointer remains live through this immediate Ready result. - unsafe { &*ctx }.note_inline_doorbell(settled_ms); + let ctx = unsafe { &*ctx }; + // The call consumed the notice, not the executor's idle park; it is + // offered, and a quiesce ask latched, exactly as there. + ctx.note_notice(cookie, payload); + ctx.note_inline_doorbell(settled_ms); Poll::Ready(()) }) .await; @@ -1103,4 +1260,69 @@ mod tests { "a zero budget would make the prelude a no-op that still reads as live" ); } + + /// A hold keeps the notices that carry data in arrival order, keeps none + /// of the runtime's wake hints, counts what meets it full, and keeps + /// accepting once a taker makes room, across the ring's wrap. + #[test] + fn a_notice_hold_keeps_data_in_order_and_counts_what_it_cannot_hold() { + use super::{NoticeHold, NoticeSink, is_wake_hint}; + use crate::svc::{COOKIE_REPLY, COOKIE_REPORT, COOKIE_REQUEST, COOKIE_SHELL}; + + for hint in [ + COOKIE_REQUEST, + COOKIE_REPLY, + COOKIE_REPORT, + crate::event::COOKIE_EVENT, + ] { + assert!(is_wake_hint(hint)); + } + assert!(!is_wake_hint(COOKIE_SHELL)); + + let hold = NoticeHold::<3>::new(); + hold.notice(COOKIE_SHELL, 1); + hold.notice(COOKIE_REPLY, 0); + hold.notice(COOKIE_SHELL, 2); + hold.notice(COOKIE_REPORT, 0); + hold.notice(COOKIE_SHELL, 3); + hold.notice(COOKIE_SHELL, 4); + hold.notice(COOKIE_SHELL, 5); + assert_eq!(hold.take_lost(), 2); + assert_eq!(hold.take_lost(), 0); + assert_eq!(hold.take(), Some((COOKIE_SHELL, 1))); + hold.notice(COOKIE_SHELL, 6); + hold.notice(COOKIE_SHELL, 7); + assert_eq!(hold.take_lost(), 1); + for payload in [2, 3, 6] { + assert_eq!(hold.take(), Some((COOKIE_SHELL, payload))); + } + assert_eq!(hold.take(), None); + } + + /// Every notice the executor consumes reaches one funnel: the kernel's + /// quiesce ask is latched for the service loop and never offered, every + /// other notice goes to the owner's sink, and without one nothing does. + #[test] + fn consumed_notices_reach_the_owners_sink_and_the_quiesce_ask_stays_the_executors() { + use super::{ExecCtx, NoticeHold}; + use crate::svc::{COOKIE_REPLY, COOKIE_SHELL}; + use kore_abi::session::NOTICE_COOKIE_POWER_QUIESCE; + + let unowned = ExecCtx::new(); + unowned.note_notice(COOKIE_SHELL, 9); + unowned.note_notice(NOTICE_COOKIE_POWER_QUIESCE, 40); + assert_eq!(unowned.take_quiesce_ask(), Some(40)); + + let hold = NoticeHold::<4>::new(); + let ctx = ExecCtx::new(); + ctx.set_notice_sink(&hold); + ctx.note_notice(COOKIE_SHELL, 1); + ctx.note_notice(NOTICE_COOKIE_POWER_QUIESCE, 50); + ctx.note_notice(COOKIE_REPLY, 0); + ctx.note_notice(COOKIE_SHELL, 2); + assert_eq!(ctx.take_quiesce_ask(), Some(50)); + assert_eq!(hold.take(), Some((COOKIE_SHELL, 1))); + assert_eq!(hold.take(), Some((COOKIE_SHELL, 2))); + assert_eq!(hold.take(), None); + } } diff --git a/native/crates/rt/src/executor/time.rs b/native/crates/rt/src/executor/time.rs index 79ebaad..e9ce532 100644 --- a/native/crates/rt/src/executor/time.rs +++ b/native/crates/rt/src/executor/time.rs @@ -19,8 +19,9 @@ use core::ptr::NonNull; use core::sync::atomic::Ordering; use core::task::{Context, Poll}; +use super::NoticeSink; use super::task::{TaskHeader, header_of}; -use crate::transport::ParkPost; +use crate::transport::{ParkPost, WordArm}; /// Concurrent pending deadlines per executor — a design bound (§8's /// posture: refusal on overflow, sized for the declared task set). @@ -60,6 +61,11 @@ pub enum WakeReason { Irq, /// The folded sleep deadline won (`Status::Deadline`). Deadline, + /// The armed memory word was seen to move, or a `wake_word` on its + /// memory fired (`Status::Word` — the kernel address-wait, + /// `res/EVENT.md` §6). Like every other reason, a hint: the waiter + /// re-reads the word. + Word, } /// One-way lifecycle of the executor's optional hardware-wake posture. @@ -93,12 +99,20 @@ pub(crate) struct ExecCtx { ParkPost, NonNull, u64, - NonNull>, + NonNull>, )>, >, - /// Monotone identity for the one alternate registration. Matching the - /// generation as well as the task prevents one cancelled future from - /// clearing a successor owned by the same executor task. + /// At most one task may add a **word arm** (the kernel address-wait, + /// `res/EVENT.md` §6) to the executor's next idle park — the park + /// carries ONE `WordWaitReq`. A second registrant is *refused*, never + /// silently dropped (its future resolves `None` and the caller falls + /// back to the doorbell rung); a delivery arm registering while a word + /// arm stands **evicts** the word (the two do not compose on the wire), + /// waking its owner to fall back the same way. + word: Cell>, + /// Monotone identity shared by the alternate and word registrations. + /// Matching the generation as well as the task prevents one cancelled + /// future from clearing a successor owned by the same executor task. next_delivery: Cell, /// A protocol call consumed and re-posted the executor doorbell inside a /// blocking send-and-wait trap. The outer executor spends this edge as a @@ -114,6 +128,21 @@ pub(crate) struct ExecCtx { /// row's remaining budget in nanoseconds. Latched by the outcome arm, /// taken exactly once by whichever loop owns the service. quiesce_ask: Cell>, + /// The doorbell owner's [`NoticeSink`], if it named one, its lifetime + /// erased ([`set_notice_sink`](Self::set_notice_sink)). + notice_sink: Cell>>, +} + +/// The one word arm registered on an executor (see [`ExecCtx::word`]). +#[derive(Clone, Copy)] +struct WordRegistration { + arm: WordArm, + /// The arm's mandatory bound — also registered on the timer table, so + /// the idle park's folded deadline never lacks it. + deadline_ns: u64, + task: NonNull, + generation: u64, + owner: NonNull>, } impl ExecCtx { @@ -123,10 +152,46 @@ impl ExecCtx { last_wake: Cell::new(WakeReason::Doorbell), timers: [const { Cell::new(None) }; MAX_TIMERS], delivery: Cell::new(None), + word: Cell::new(None), next_delivery: Cell::new(0), inline_doorbell: Cell::new(false), irq: Cell::new(IrqParkPolicy::Unarmed), quiesce_ask: Cell::new(None), + notice_sink: Cell::new(None), + } + } + + /// Offer every notice this executor consumes to `sink`. + /// + /// Only [`Executor::with_notice_sink`](super::Executor::with_notice_sink) + /// calls this, on the context the `Executor<'t, _>` owns, with a sink + /// borrowed for that same `'t`: the context cannot outlive the borrow, + /// and [`note_notice`](Self::note_notice) is its only reader. + pub(crate) fn set_notice_sink<'t>(&self, sink: &'t dyn NoticeSink) { + let sink: NonNull = NonNull::from(sink); + // SAFETY: only the trait object's lifetime bound changes; the pointer + // and its vtable are untouched. The erased pointer is dereferenced + // only by `note_notice`, on this context, which the owning executor + // holds no longer than `'t` (see above). + let sink = unsafe { + core::mem::transmute::, NonNull>(sink) + }; + self.notice_sink.set(Some(sink)); + } + + /// One doorbell notice this executor consumed, in its idle park or inline + /// in a blocking call: the kernel's quiesce ask is latched here, and + /// every other notice goes to the owner's sink, if one was named. + pub(crate) fn note_notice(&self, cookie: u64, payload: u64) { + if cookie == kore_abi::session::NOTICE_COOKIE_POWER_QUIESCE { + self.note_quiesce_ask(payload); + return; + } + if let Some(sink) = self.notice_sink.get() { + // SAFETY: `set_notice_sink` stored a sink borrowed for the life of + // the executor that owns this context; this call runs on that + // executor's thread while it is alive. + unsafe { sink.as_ref() }.notice(cookie, payload); } } @@ -261,26 +326,111 @@ impl ExecCtx { .min() } + fn next_generation(&self) -> u64 { + let mut generation = self.next_delivery.get().wrapping_add(1); + if generation == 0 { + generation = 1; + } + self.next_delivery.set(generation); + generation + } + fn register_delivery( &self, post: ParkPost, task: NonNull, - future_owner: NonNull>, + future_owner: NonNull>, ) -> u64 { assert!( self.delivery.get().is_none(), "executor alternate delivery wait already owned" ); - let mut generation = self.next_delivery.get().wrapping_add(1); - if generation == 0 { - generation = 1; + // A word arm and an alternate post do not compose in one park + // (`WAIT_WORD | WAIT_DELIVERY_ALT` is malformed). The affine + // delivery wins: the word waiter is evicted and woken to re-check + // and fall back to the doorbell rung — a spurious wake, the + // harmless direction. + if let Some(mut word) = self.word.replace(None) { + // SAFETY: a registered word owner pointer names the pinned + // `owner` field of a live, still-pending `ArmedWake` (its Drop + // clears the registry first). + if let Some(owner) = unsafe { word.owner.as_mut() } { + owner.evicted = true; + } + // SAFETY: registered headers are live for the run. + unsafe { word.task.as_ref() }.mark_ready(); } - self.next_delivery.set(generation); + let generation = self.next_generation(); self.delivery .set(Some((post, task, generation, future_owner))); generation } + /// Claim the park's one word arm for `task`, or `None` when it (or an + /// alternate post, which the arm cannot ride beside) is already owned. + fn register_word( + &self, + arm: WordArm, + deadline_ns: u64, + task: NonNull, + future_owner: NonNull>, + ) -> Option { + if self.word.get().is_some() || self.delivery.get().is_some() { + return None; + } + let generation = self.next_generation(); + self.word.set(Some(WordRegistration { + arm, + deadline_ns, + task, + generation, + owner: future_owner, + })); + Some(generation) + } + + fn clear_word(&self, task: NonNull, generation: u64) { + if self + .word + .get() + .is_some_and(|w| w.task == task && w.generation == generation) + { + self.word.set(None); + } + } + + /// The word arm the next idle park carries, with its mandatory deadline. + pub(crate) fn word_arm(&self) -> Option<(WordArm, u64)> { + self.word.get().map(|w| (w.arm, w.deadline_ns)) + } + + /// The park resolved `Status::Word`: wake exactly the arm's owner (the + /// future's ready leg clears the registration). + pub(crate) fn wake_word_owner(&self) { + if let Some(word) = self.word.get() { + // SAFETY: registered headers are live for the run. + unsafe { word.task.as_ref() }.mark_ready(); + } + } + + /// The kernel refused the word arm (`WouldBlock` — its bounded waiter + /// board was full): the arm is withdrawn for this wait exactly as an + /// alternate-delivery eviction withdraws it, so the owner's + /// `next_wake_on_word` answers `None` and the wait re-publishes its + /// doorbell intent and takes rung 3. Re-parking with the arm standing + /// would ask the same full board again at once — a livelock across every + /// waiting thread on a machine whose gangs out-number the board. + pub(crate) fn refuse_word(&self) { + if let Some(mut word) = self.word.replace(None) { + // SAFETY: the owner slot and header are live for the run, as in + // `register_delivery`'s eviction of the same arm. + if let Some(owner) = unsafe { word.owner.as_mut() } { + owner.evicted = true; + } + unsafe { word.task.as_ref() }.mark_ready(); + } + } + fn clear_delivery(&self, task: NonNull, generation: u64) { if self .delivery @@ -341,12 +491,16 @@ impl ExecCtx { } if let Some((_, _, _, mut future_owner)) = self.delivery.replace(None) { // SAFETY: alternate registration accepts this pointer only from a - // pinned `AlternateWake`. Its Drop clears the registry before the + // pinned `ArmedWake`. Its Drop clears the registry before the // field dies; therefore a still-registered pointer names a live, // immovable owner field. Executor teardown invalidates the future's // raw context backlink before this context can itself die. unsafe { *future_owner.as_mut() = None }; } + if let Some(mut word) = self.word.replace(None) { + // SAFETY: as for the alternate registration above. + unsafe { *word.owner.as_mut() = None }; + } self.inline_doorbell.set(false); } } @@ -657,40 +811,91 @@ pub fn quiesce_deadline(budget_ns: u64) -> u64 { /// standing delivery arm beside the ordinary doorbell. The empty descriptor /// stays inside x86-64's caller-saved-register carrier. pub async fn next_wake_on(post: ParkPost) -> WakeReason { - AlternateWake::new(post, None).await + ArmedWake::alternate(post, None) + .await + .expect("an alternate delivery arm is never refused or evicted") } /// Deadline-bounded sibling of [`next_wake_on`]. Either standing post or the /// relative deadline wakes the root, with both registrations settled on the /// ready leg or cancellation drop. pub async fn next_wake_on_for(post: ParkPost, delta_ns: u64) -> WakeReason { - AlternateWake::new(post, Some(delta_ns)).await + ArmedWake::alternate(post, Some(delta_ns)) + .await + .expect("an alternate delivery arm is never refused or evicted") +} + +/// Complete on the next executor wake while arming the idle park's **word +/// arm** on the `u32` at byte `offset` of the held memory `subject` (a +/// lease or mapped view, by arena slot — `cap::mapped::locate` names one +/// from a pointer), last read as `expected`, bounded by the **absolute** +/// deadline `deadline_ns` (mandatory: the kernel address-wait is never +/// unbounded). The first of {doorbell, word, deadline, IRQ} wakes the +/// root; a [`WakeReason::Word`] means the word moved or a `wake_word` on +/// its memory fired, and — like every reason — is a hint to re-read. +/// +/// `None` when the arm was **refused**: the executor's park carries one +/// word arm, and another future owns it (or an alternate delivery arm +/// stands, which the word cannot ride beside). Nothing was registered; +/// the caller re-checks its word and falls back to the doorbell park +/// ([`next_wake_for`]) for this wait. A pending wait that a later +/// alternate arm evicts resolves `None` the same way. +pub async fn next_wake_on_word( + subject: kore_abi::session::SlotRef, + offset: u32, + expected: u32, + deadline_ns: u64, +) -> Option { + ArmedWake::word( + WordArm { + subject, + offset, + expected, + }, + deadline_ns, + ) + .await +} + +/// Which executor arm an [`ArmedWake`] installs. +#[derive(Clone, Copy)] +enum Arm { + /// The bounded alternate standing post. + Post(ParkPost), + /// The kernel address-wait's word (its deadline rides `deadline_ns`). + Word(WordArm), } -/// One alternate executor registration, owned by the future that installed -/// it. The ready leg and `Drop` share the same idempotent teardown, so a -/// `select`-style caller cannot leave a stale post (or folded deadline) behind -/// when another branch cancels this wait while it is pending. -struct AlternateWake { - post: ParkPost, +/// One executor arm registration, owned by the future that installed it. +/// The ready leg and `Drop` share the same idempotent teardown, so a +/// `select`-style caller cannot leave a stale arm (or folded deadline) +/// behind when another branch cancels this wait while it is pending. +struct ArmedWake { + arm: Arm, + /// A relative bound, resolved to `deadline_ns` on first poll. delta_ns: Option, + /// The absolute bound (a word arm names it directly). deadline_ns: Option, - owner: Option, + owner: Option, timer: TimerRegistration, _pin: PhantomPinned, } #[derive(Clone, Copy)] -struct AlternateOwner { +struct ArmedOwner { task: NonNull, ctx: NonNull, - delivery_generation: Option, + /// The registry generation this future owns (delivery or word). + generation: Option, + /// Set by the executor through the registry's backlink when a later + /// alternate arm displaced this word arm: the ready leg answers `None`. + evicted: bool, } -impl AlternateWake { - const fn new(post: ParkPost, delta_ns: Option) -> Self { +impl ArmedWake { + const fn alternate(post: ParkPost, delta_ns: Option) -> Self { Self { - post, + arm: Arm::Post(post), delta_ns, deadline_ns: None, owner: None, @@ -699,6 +904,17 @@ impl AlternateWake { } } + const fn word(arm: WordArm, deadline_ns: u64) -> Self { + Self { + arm: Arm::Word(arm), + delta_ns: None, + deadline_ns: Some(deadline_ns), + owner: None, + timer: TimerRegistration::new(), + _pin: PhantomPinned, + } + } + fn clear_registration(&mut self) { self.timer.clear_registration(); let Some(owner) = self.owner.take() else { @@ -710,61 +926,83 @@ impl AlternateWake { // leave `run_pinned`; target panics abort rather than unwind. Normal // executor exit also clears the complete registry as a backstop. let ctx = unsafe { owner.ctx.as_ref() }; - if let Some(generation) = owner.delivery_generation { - ctx.clear_delivery(owner.task, generation); + if let Some(generation) = owner.generation { + match self.arm { + Arm::Post(_) => ctx.clear_delivery(owner.task, generation), + Arm::Word(_) => ctx.clear_word(owner.task, generation), + } } } } -impl Future for AlternateWake { - type Output = WakeReason; +impl Future for ArmedWake { + type Output = Option; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { // SAFETY: no field is moved. Pinning makes `owner`'s address stable // while the executor registry holds its teardown backlink. let this = unsafe { self.get_unchecked_mut() }; let hdr = header_of(cx.waker()) - .expect("alternate wake requires the kore executor (foreign waker)"); + .expect("armed wake requires the kore executor (foreign waker)"); // SAFETY: the kore waker vouches for a live executor-owned header. let ctx = unsafe { hdr.as_ref() }.exec.load(Ordering::Relaxed); assert!( !ctx.is_null(), - "alternate wake polled outside an executor pass" + "armed wake polled outside an executor pass" ); // SAFETY: the executor installs this pointer for the duration of poll. let ctx = unsafe { &*ctx }; if let Some(owner) = this.owner { - assert_eq!(owner.task, hdr, "alternate wake changed owning task"); + assert_eq!(owner.task, hdr, "armed wake changed owning task"); assert_eq!(owner.ctx.as_ptr(), ctx as *const ExecCtx as *mut ExecCtx); let reason = ctx.last_wake(); + let evicted = owner.evicted; this.clear_registration(); - return Poll::Ready(reason); + return Poll::Ready((!evicted).then_some(reason)); } if let Some(delta_ns) = this.delta_ns { - let deadline = *this - .deadline_ns + this.deadline_ns .get_or_insert_with(|| ctx.now().saturating_add(delta_ns)); - if ctx.now() >= deadline { - return Poll::Ready(WakeReason::Deadline); - } + } + if let Some(deadline) = this.deadline_ns + && ctx.now() >= deadline + { + return Poll::Ready(Some(WakeReason::Deadline)); } // Publish the owner first so either registration panicking still // unwinds through the same idempotent cleanup. - this.owner = Some(AlternateOwner { + this.owner = Some(ArmedOwner { task: hdr, ctx: NonNull::from(ctx), - delivery_generation: None, + generation: None, + evicted: false, }); let future_owner = NonNull::from(&mut this.owner); - let generation = ctx.register_delivery(this.post, hdr, future_owner); + let generation = match this.arm { + Arm::Post(post) => ctx.register_delivery(post, hdr, future_owner), + Arm::Word(arm) => { + let deadline = this + .deadline_ns + .expect("a word arm always carries its deadline"); + match ctx.register_word(arm, deadline, hdr, future_owner) { + Some(generation) => generation, + // Refused: the park's one word arm is owned. Nothing + // was registered, so the owner is withdrawn whole. + None => { + this.owner = None; + return Poll::Ready(None); + } + } + } + }; let owner = this .owner .as_mut() - .expect("alternate owner disappeared during registration"); - owner.delivery_generation = Some(generation); + .expect("armed owner disappeared during registration"); + owner.generation = Some(generation); if let Some(deadline) = this.deadline_ns { // SAFETY: this field stays pinned with AlternateWake and is // cleared before that parent's Drop can release its storage. @@ -774,7 +1012,7 @@ impl Future for AlternateWake { } } -impl Drop for AlternateWake { +impl Drop for ArmedWake { fn drop(&mut self) { self.clear_registration(); } @@ -988,7 +1226,7 @@ mod tests { }; let mut ordinary = core::pin::pin!(next_wake_until(41)); { - let mut alternate = core::pin::pin!(AlternateWake::new(post, Some(29))); + let mut alternate = core::pin::pin!(ArmedWake::alternate(post, Some(29))); if reverse { assert_eq!(ordinary.as_mut().poll(&mut cx), Poll::Pending); assert_eq!(alternate.as_mut().poll(&mut cx), Poll::Pending); @@ -1003,7 +1241,7 @@ mod tests { ctx.note_wake(WakeReason::Delivery); assert_eq!( alternate.as_mut().poll(&mut cx), - Poll::Ready(WakeReason::Delivery) + Poll::Ready(Some(WakeReason::Delivery)) ); } assert_eq!(ctx.delivery_post(), None); @@ -1017,7 +1255,7 @@ mod tests { // Conversely, a cancelled ordinary wait cannot remove the // alternate post's later deadline. - let mut alternate = core::pin::pin!(AlternateWake::new(post, Some(59))); + let mut alternate = core::pin::pin!(ArmedWake::alternate(post, Some(59))); assert_eq!(alternate.as_mut().poll(&mut cx), Poll::Pending); { let mut cancelled = core::pin::pin!(next_wake_for(17)); @@ -1063,7 +1301,7 @@ mod tests { }; { - let wait = AlternateWake::new(post, Some(29)); + let wait = ArmedWake::alternate(post, Some(29)); let mut wait = core::pin::pin!(wait); assert_eq!(wait.as_mut().poll(&mut cx), Poll::Pending); assert_eq!(ctx.delivery_post(), Some(post)); @@ -1077,7 +1315,7 @@ mod tests { // The released bounded slot can be claimed immediately by another // wait instead of poisoning the executor's next idle park. { - let replacement = AlternateWake::new(post, None); + let replacement = ArmedWake::alternate(post, None); let mut replacement = core::pin::pin!(replacement); assert_eq!(replacement.as_mut().poll(&mut cx), Poll::Pending); } @@ -1122,7 +1360,7 @@ mod tests { land: SlotRef::from_wire(47), recv_tag: 53, }; - let wait = AlternateWake::new(post, Some(59)); + let wait = ArmedWake::alternate(post, Some(59)); let mut wait = core::pin::pin!(wait); assert_eq!(wait.as_mut().poll(&mut cx), Poll::Pending); @@ -1155,6 +1393,64 @@ mod tests { assert_eq!(ctx.timers_min(), None); } + /// A console waiting for guest output must return to its input drain on a + /// keyboard doorbell even when the output word has not changed. Exercise + /// both wait rungs with the real executor registrations, then check that + /// the predicate-only wait still ignores an unrelated wake. + #[test] + fn composed_event_wait_returns_on_other_input_with_either_park_rung() { + use crate::event::{EventWait, wait_once, wait_past}; + use kore_ring::event::{SEQ_AT, region_bytes, split}; + + #[repr(align(64))] + struct Window([u8; region_bytes::(1)]); + + for noted in [false, true] { + let ctx = ExecCtx::new(); + let task = TaskHeader::new(); + let waker = task_waker(&ctx, &task); + let mut cx = Context::from_waker(&waker); + let mut window = Window([0; region_bytes::(1)]); + let base = window.0.as_ptr(); + let slot = SlotRef::from_wire(21); + if noted { + crate::cap::mapped::note(base, window.0.len(), slot); + } + let (mut source, sink) = + split::(&mut window.0, 1).unwrap(); + let seen = sink.seq(); + { + let mut wait = core::pin::pin!(wait_once(&sink, seen, None)); + assert_eq!(wait.as_mut().poll(&mut cx), Poll::Pending); + assert_eq!( + ctx.word_arm().map(|(arm, _)| (arm.subject, arm.offset)), + noted.then_some((slot, SEQ_AT as u32)) + ); + ctx.note_wake(WakeReason::Doorbell); + assert_eq!(wait.as_mut().poll(&mut cx), Poll::Ready(EventWait::Woken)); + } + assert_eq!(ctx.word_arm(), None); + assert_eq!(ctx.timers_min(), None); + assert_eq!(sink.parked_word().load(Ordering::SeqCst), 0); + { + let mut wait = core::pin::pin!(wait_past(&sink, seen, None)); + assert_eq!(wait.as_mut().poll(&mut cx), Poll::Pending); + ctx.note_wake(WakeReason::Doorbell); + assert_eq!(wait.as_mut().poll(&mut cx), Poll::Pending); + source.commit(); + ctx.note_wake(WakeReason::Word); + assert_eq!( + wait.as_mut().poll(&mut cx), + Poll::Ready(EventWait::Changed) + ); + } + assert_eq!(ctx.word_arm(), None); + assert_eq!(ctx.timers_min(), None); + assert_eq!(sink.parked_word().load(Ordering::SeqCst), 0); + crate::cap::mapped::forget(base); + } + } + #[test] fn inline_doorbell_is_a_one_shot_total_drain_edge() { let ctx = ExecCtx::new(); diff --git a/native/crates/rt/src/fiber_stack.rs b/native/crates/rt/src/fiber_stack.rs index 6ae187a..949f7da 100644 --- a/native/crates/rt/src/fiber_stack.rs +++ b/native/crates/rt/src/fiber_stack.rs @@ -28,7 +28,9 @@ pub(crate) const WINDOW_BASE: u64 = 0x7fff_e200_0000; /// The window's granule capacity: 32 MiB of reservation, no byte mapped by /// declaring it. Wasmtime's default async stack is 2 MiB, so fifteen live -/// fibers fit with their guards — each call parks one. +/// fibers fit with their guards — each call parks one. The `wasm` command's +/// are 5.5 MiB (`user/apps/utils/wasm`, a 4 MiB wasm stack), so five fit, +/// against the one a launch makes. pub(crate) const WINDOW_GRANULES: u32 = 8192; /// **The process's guarded-stack plan.** Carved by [`GuardedStack::new`], diff --git a/native/crates/rt/src/futex.rs b/native/crates/rt/src/futex.rs index 842911a..04fe2d6 100644 --- a/native/crates/rt/src/futex.rs +++ b/native/crates/rt/src/futex.rs @@ -80,6 +80,21 @@ //! each child with the ends, which needs a `role::CHANNEL_SPACE` grant in the //! task that spawns. Nothing in the API changes when it lands. //! +//! **For a word in a shared window the gap is closed** by the kernel +//! address-wait (`res/EVENT.md` §6). A word that +//! [`cap::mapped::locate`](crate::cap::mapped::locate) can name by the held +//! slot that mapped it — a ring, an event channel, a barrier word placed in +//! a shared lease — parks through [`verbs::wait_word`] (no doorbell needed: +//! the mapping is the wait authority) and is woken through +//! [`verbs::wake_word`] (no send end needed: "you may wake what you can +//! write"). The registry records *which* wake reaches each waiter, so a +//! [`wake`] spends a `wake_word` on word-parked waiters and a doorbell kick +//! on the rest; a word the registry cannot name keeps today's doorbell path +//! unchanged. +//! +//! [`verbs::wait_word`]: crate::cap::verbs::wait_word +//! [`verbs::wake_word`]: crate::cap::verbs::wake_word +//! //! # Two forms of the wait, one registry //! //! [`wait`] blocks the thread in the kernel; [`wait_async`] yields to the @@ -127,6 +142,15 @@ use crate::thread::MAX_THREADS; /// a lost wakeup with no diagnostic anywhere. const WAITS_PER_THREAD: usize = crate::executor::MAX_TASKS + 1; +/// Bound of one **word** park when the caller asked for no timeout: the +/// kernel address-wait is never unbounded, so an open-ended wait is a loop +/// of bounded parks, each returning a spurious [`Woken`](WaitOutcome::Woken) +/// the caller's re-check absorbs. Long enough that a blocked lock waiter +/// is not paced by it; short enough that the one wake a stale window hint +/// could lose costs a slice, not forever. +#[cfg_attr(not(target_os = "none"), expect(dead_code, reason = "the word park is a metal-only trap"))] +const WORD_SLICE_NS: u64 = 250_000_000; + /// The registry: **which addresses each thread has waiters on**, or /// [`NOT_WAITING`]. /// @@ -154,9 +178,29 @@ static WAITING_ON: [[AtomicUsize; WAITS_PER_THREAD]; MAX_THREADS as usize] = /// asserted, because the cost of being wrong about that is a fail-closed /// refusal here and a lost wakeup if it were ignored. fn claim_wait(thread: usize, addr: usize) -> Option { + claim_wait_entry(thread, addr) +} + +/// [`claim_wait`] for a waiter that will park **on the word** through the +/// kernel address-wait rather than on its doorbell: the entry carries +/// [`WORD_MODE`], so a waker knows a `wake_word` — not a kick — reaches it. +fn claim_wait_on_word(thread: usize, addr: usize) -> Option { + claim_wait_entry(thread, addr | WORD_MODE) +} + +/// Flip a word-mode registry entry back to doorbell mode in place — the +/// futex-side half of a word park the kernel refused. `true` iff the entry +/// still stood in word mode. +fn demote_to_bell(thread: usize, slot: usize, addr: usize) -> bool { + WAITING_ON[thread][slot] + .compare_exchange(addr | WORD_MODE, addr, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() +} + +fn claim_wait_entry(thread: usize, entry: usize) -> Option { for (slot, cell) in WAITING_ON[thread].iter().enumerate() { if cell - .compare_exchange(NOT_WAITING, addr, Ordering::SeqCst, Ordering::SeqCst) + .compare_exchange(NOT_WAITING, entry, Ordering::SeqCst, Ordering::SeqCst) .is_ok() { return Some(slot); @@ -165,6 +209,32 @@ fn claim_wait(thread: usize, addr: usize) -> Option { None } +/// The registry entry's **mode bit**: set when the waiter parks on the word +/// itself (the kernel address-wait) and a `wake_word` is what reaches it; +/// clear when it parks on its doorbell and a kick does. Bit 0 is free +/// because a `u32` word is 4-aligned — the address and the mode ride one +/// atomic store, so a waker never sees one without the other. +const WORD_MODE: usize = 1; + +/// How the waiters `thread` has on `addr` can be reached: `(by kick, by +/// wake_word)` — both may hold at once (a thread's suspended futures keep +/// doorbell entries while the thread itself blocks on a word). +fn thread_wait_modes(thread: usize, addr: usize) -> (bool, bool) { + let mut bell = false; + let mut word = false; + for cell in &WAITING_ON[thread] { + let entry = cell.load(Ordering::SeqCst); + if entry & !WORD_MODE == addr { + if entry & WORD_MODE != 0 { + word = true; + } else { + bell = true; + } + } + } + (bell, word) +} + /// A claimed wait slot, released when the waiter stops waiting **however it /// stops**. /// @@ -198,14 +268,13 @@ fn release_wait(thread: usize, slot: usize) { WAITING_ON[thread][slot].store(NOT_WAITING, Ordering::SeqCst); } -/// Whether `thread` has any waiter registered on `addr`. -/// -/// One kick per thread is enough however many of its futures are waiting: the -/// executor's drain is total, so a single notice re-polls all of them. +/// Whether `thread` has any waiter registered on `addr`, in either mode — +/// the registry's own fact, which [`thread_wait_modes`] refines for wakers. +#[cfg(test)] fn thread_waits_on(thread: usize, addr: usize) -> bool { WAITING_ON[thread] .iter() - .any(|cell| cell.load(Ordering::SeqCst) == addr) + .any(|cell| cell.load(Ordering::SeqCst) & !WORD_MODE == addr) } /// The vacant registry entry. @@ -336,6 +405,16 @@ pub(crate) fn wait_barrier( wait_inner(word, expected, timeout, true, BARRIER_SPIN) } +/// [`wait_barrier`] with **no** jitter-window spin: straight to the park. +/// +/// For a waiter the jitter window has already failed — an idle resident +/// worker whose last slice rode out with no job (`barrier::wait_while_idle`). +/// Nothing co-resident is about to store the word, so the spin would only +/// occupy the core; counted like the barrier wait it stands in for. +pub(crate) fn wait_idle(word: &AtomicU32, expected: u32, timeout: Option) -> WaitOutcome { + wait_inner(word, expected, timeout, true, NO_SPIN) +} + /// [`wait`] without the [`stats`] accounting — for the star's hub, /// whose housekeeping slices would otherwise pollute the very numbers /// a terminal bench reads to judge the WORKERS' wait behavior. @@ -359,12 +438,22 @@ fn wait_inner( }; let addr = address_of(word); + // ── 0 · Route ─────────────────────────────────────────────────────── + // A word inside a window this task mapped through a noting verb is + // named to the kernel by that authority and parks on the word itself + // (the kernel address-wait); every other word keeps the doorbell park. + // Decided once, before the registry entry, because the entry records + // the decision for wakers. + let route = crate::cap::mapped::locate(word as *const AtomicU32 as *const u8); + // ── 1 · Arm ───────────────────────────────────────────────────────── // The standing post must exist before this thread is discoverable as a // waiter, or a notice sent in the window between registering and parking // would find no post and be dropped (`WouldBlock`, on the floor). Arming - // first makes "too early" indistinguishable from "on time". - if !imp::arm() { + // first makes "too early" indistinguishable from "on time". A word park + // needs no post: the mapping is its wait authority, which is what lets a + // doorbell-less worker wait here at all. + if route.is_none() && !imp::arm() { return WaitOutcome::Unsupported; } @@ -372,7 +461,11 @@ fn wait_inner( // This store must be visible before the load below, and `SeqCst` on both // is what forbids the store-load reordering that would let this thread // and a waker miss each other (module docs, mechanism 2). - let Some(slot) = claim_wait(me, addr) else { + let slot = match route { + Some(_) => claim_wait_on_word(me, addr), + None => claim_wait(me, addr), + }; + let Some(slot) = slot else { return WaitOutcome::Unsupported; }; let _slot = WaitSlot { thread: me, slot }; @@ -392,7 +485,23 @@ fn wait_inner( if counted { STAT_PARKS.fetch_add(1, Ordering::Relaxed); } - let outcome = imp::park(timeout); + let outcome = match route { + Some((subject, offset)) => { + imp::park_word(subject, offset, expected, timeout, || { + // Fall back to the doorbell for this call: demote the + // registry entry FIRST (a waker that reads it from here on + // kicks instead of `wake_word`), then re-check the word (a + // waker that read word mode before the demotion has already + // stored — SeqCst on both sides, one of us sees the other), + // then arm the doorbell. Without the demotion a word-mode + // waiter parked on its doorbell is unreachable by any waker. + demote_to_bell(me, slot, addr) + && word.load(Ordering::SeqCst) == expected + && imp::arm() + }) + } + None => imp::park(timeout), + }; if counted { match outcome { WaitOutcome::Woken => STAT_WOKEN.fetch_add(1, Ordering::Relaxed), @@ -589,6 +698,12 @@ const BARRIER_SPIN: SpinBudget = SpinBudget { cap: Duration::from_micros(8), }; +/// No spin at all — [`wait_idle`]'s budget. +const NO_SPIN: SpinBudget = SpinBudget { + turns: 0, + cap: Duration::ZERO, +}; + /// Sleep on the line for a few turns, hoping to skip the park entirely. /// /// `Some` when the word moved — the caller is done and never blocks. `None` @@ -711,9 +826,24 @@ pub(crate) fn notify_resident_completion() -> bool { /// resolves the park it is about to enter. pub fn wake(word: &AtomicU32) -> bool { let addr = address_of(word); + let route = crate::cap::mapped::locate(word as *const AtomicU32 as *const u8); let mut unreachable_waiter = false; for thread in 0..MAX_THREADS as usize { - if thread_waits_on(thread, addr) { + let (bell, on_word) = thread_wait_modes(thread, addr); + // A word-parked waiter is reached on the memory, by this side's own + // mapping of it: one waiter released, no send end spent. The + // thread's block is what a kick could not end, so the word comes + // first when both modes stand. + if on_word { + if let Some((subject, offset)) = route + && imp::wake_word(subject, offset, 1) + { + STAT_KICKS.fetch_add(1, Ordering::Relaxed); + return true; + } + unreachable_waiter = true; + } + if bell { if imp::kick(thread) { STAT_KICKS.fetch_add(1, Ordering::Relaxed); return true; @@ -743,10 +873,16 @@ pub fn wake(word: &AtomicU32) -> bool { /// synchronization fact. pub fn wake_all(word: &AtomicU32) -> usize { let addr = address_of(word); + let route = crate::cap::mapped::locate(word as *const AtomicU32 as *const u8); let mut woken = 0; let mut missed = false; + let mut word_waiters = 0; for thread in 0..MAX_THREADS as usize { - if thread_waits_on(thread, addr) { + let (bell, on_word) = thread_wait_modes(thread, addr); + if on_word { + word_waiters += 1; + } + if bell { if imp::kick(thread) { STAT_KICKS.fetch_add(1, Ordering::Relaxed); woken += 1; @@ -756,6 +892,18 @@ pub fn wake_all(word: &AtomicU32) -> usize { } } } + // Every word-parked waiter is released by ONE `wake_word` on the memory + // (the kernel keys them by the word's physical identity), counted per + // thread it reached — the same unit the kicks above are counted in. + if word_waiters > 0 { + if route.is_some_and(|(subject, offset)| imp::wake_word(subject, offset, u32::MAX)) { + STAT_KICKS.fetch_add(word_waiters, Ordering::Relaxed); + woken += word_waiters as usize; + } else { + STAT_MISSED.fetch_add(word_waiters, Ordering::Relaxed); + missed = true; + } + } // The relay hop (the star, `res/SPMD.md` §3): every waiter this // caller could not kick is the hub's job — one notice covers them // all, counted once. With the intra-chunk kick mesh this is the @@ -924,6 +1072,67 @@ mod imp { } } + /// Park on the word itself through the kernel address-wait, `timeout` + /// folded in as the mandatory deadline. + /// + /// An open-ended wait is a bounded park the caller's re-check loop + /// renews: the slice expiring reports [`Woken`](WaitOutcome::Woken) (a + /// spurious wake, which the contract already requires callers to + /// tolerate), never a fabricated timeout. A kernel refusal of the + /// *shape* — a stale window hint the kernel no longer honours — falls + /// back to the doorbell park for this call rather than refusing the + /// wait; the registry entry's mode is then optimistic for one park, + /// which costs at most a slice, never a lost wake on a live window. + pub(super) fn park_word( + subject: kore_abi::session::SlotRef, + offset: u32, + expected: u32, + timeout: Option, + mut fall_back: impl FnMut() -> bool, + ) -> WaitOutcome { + use crate::cap::verbs::{WordWait, WordWaitError}; + + let Ok(now) = crate::clock::uptime() else { + return WaitOutcome::Unsupported; + }; + let deadline = match timeout { + Some(d) => (now + d).as_nanos(), + None => now.as_nanos().saturating_add(super::WORD_SLICE_NS), + }; + match verbs::wait_word(subject, offset, expected, deadline) { + Ok(WordWait::Changed) => WaitOutcome::Woken, + Ok(WordWait::Deadline) => { + if timeout.is_some() { + WaitOutcome::TimedOut + } else { + WaitOutcome::Woken + } + } + // The window died under the waiter: no later wait on this word + // can succeed either. + Ok(WordWait::Revoked) | Err(WordWaitError::TransportUnavailable) => { + WaitOutcome::Unsupported + } + // The kernel did not park us (a full waiter board, or a refused + // subject): this call rides the doorbell instead. `fall_back` + // demotes the registry entry, re-checks the word and arms the + // bell; `false` means the word already moved (Woken) or the bell + // cannot be armed — either way, do not sleep. + Ok(WordWait::WouldBlock) | Err(WordWaitError::Refused(_)) => { + if !fall_back() { + return WaitOutcome::Woken; + } + park(timeout) + } + } + } + + /// Release up to `count` waiters parked on the word at `offset` of the + /// held `subject` — the wake a shared window needs no send end for. + pub(super) fn wake_word(subject: kore_abi::session::SlotRef, offset: u32, count: u32) -> bool { + verbs::wake_word(subject, offset, count) + } + /// Shared parking transport for the fixed thread doorbell and an affine /// alternate receiver. Only the caller owning the landing may consume it. fn park_status(post: crate::transport::ParkPost, timeout: Option) -> Option { @@ -1027,6 +1236,16 @@ mod imp { super::WaitOutcome::Unsupported } + pub(super) fn park_word( + _subject: kore_abi::session::SlotRef, + _offset: u32, + _expected: u32, + _timeout: Option, + _fall_back: impl FnMut() -> bool, + ) -> super::WaitOutcome { + super::WaitOutcome::Unsupported + } + pub(super) fn park_post( _post: crate::transport::ParkPost, _timeout: Option, @@ -1034,6 +1253,10 @@ mod imp { super::WaitOutcome::Unsupported } + pub(super) fn wake_word(_subject: kore_abi::session::SlotRef, _offset: u32, _count: u32) -> bool { + false + } + pub(super) fn kick(_index: usize) -> bool { false } @@ -1054,6 +1277,34 @@ mod imp { #[cfg(test)] mod tests { + #[test] + fn idle_budget_never_spins_with_or_without_a_clock() { + let word = AtomicU32::new(1); + for clock in [None, Some(0)] { + assert_eq!( + spin_briefly_with( + &word, + 1, + NO_SPIN, + || clock, + |_| panic!("idle wait must park without a monitored spin"), + ), + None, + ); + } + word.store(2, Ordering::SeqCst); + assert_eq!( + spin_briefly_with( + &word, + 1, + NO_SPIN, + || None, + |_| panic!("changed word must not spin"), + ), + Some(WaitOutcome::Retry), + ); + } + #[test] fn jitter_window_is_time_bounded_without_a_monitor() { use core::cell::Cell; diff --git a/native/crates/rt/src/heap/mod.rs b/native/crates/rt/src/heap/mod.rs index e58dd39..b2f36d9 100644 --- a/native/crates/rt/src/heap/mod.rs +++ b/native/crates/rt/src/heap/mod.rs @@ -33,16 +33,37 @@ //! //! ## The shape //! -//! - **Small/medium** (`<= 8 KiB` after folding alignment): per-shard, per-size +//! - **Small/medium** (`<= 64 KiB` after folding alignment): per-shard, per-size //! free-lists. A task pops from *its own* shard — zero atomics on the hot //! path. A free block holds its successor pointer intrusively, so the //! allocator never allocates to allocate. //! - **Cross-thread free**: a block owned by another shard is pushed onto that //! shard's lock-free MPSC remote list (the owner drains it on its own miss). -//! - **Huge** (`> 8 KiB`): a dedicated whole-extent allocation, aligned to the -//! request, recycled through a generation-tagged free stack. +//! - **Huge** (`> 64 KiB`): a dedicated whole-extent allocation, aligned to +//! the request, recycled whole through generation-tagged free stacks kept +//! by size ([`HUGE_BUCKETS`]). //! - **Grow on miss**: map a fresh extent through the [`MemSource`] seam and //! carve it — synchronously, since userland has no ISR/FP/lock-held hazard. +//! - **Routing is O(1)**, never a directory walk. Every extent starts on a +//! page and no two overlap, so each page of the address space belongs to +//! at most one extent, and a four-level **page map** (a radix over the +//! user half of the address space, keyed by page number) names that +//! extent: a free is four dependent loads. The map asks nothing of how +//! or where a source places an extent, so a 2 MiB service supply pays no +//! alignment skip and a worker's per-extent carve costs what it asks. +//! - **The extent directory grows.** Live extents are registered in pages of +//! [`PAGE_SLOTS`] pointers: the first page is inline in the [`Heap`] (so a +//! `static` heap needs no init pass), later pages are added as grows need +//! them. Compiling a 7.8 MB WebAssembly component on kore needs some +//! 3,900 extents; a fixed 1,024-slot directory failed that compile at +//! 90 MiB mapped. +//! - **A grow is one mapping.** The metadata a grow needs — a directory +//! page, the page-map nodes its span lacks — is carved from the tail of +//! the mapping it asked for, or taken from a reserve of pages earlier +//! grows left over (metadata self-hosted in source memory, never +//! allocated from the heap itself). A source that answers a miss with +//! one extent of what was asked, once (`kore-svc-mm`'s refill protocol), +//! is therefore always enough. //! //! See [`source`] for the [`MemSource`] seam — `kore-svc-mm`'s default //! backing routes growth through the memory-manager service, so a heap @@ -54,10 +75,14 @@ mod shard; mod source; pub use crate::mem_source::MemSource; +pub use source::MAX_ADOPTED; #[cfg(target_os = "none")] pub use source::carve_and_map; +pub(crate) use source::{CommitSupply, commit_reserved, release_reserved}; pub use source::{HeapSupplySource, StaticMemSource}; -pub use source::{adopt_supply, reserve_from_top}; +pub use source::{ + adopt_supply, adopt_window_supply, heap_room, reserve_from_top, window_supply_room, +}; use core::alloc::{AllocError, Allocator, GlobalAlloc, Layout}; use core::ptr::NonNull; @@ -65,7 +90,7 @@ use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, AtomicUsize, Ordering} use kore_collections::AtomicBitSet; -use pool::{MIN_BLOCK, align_up, checked_align_up, class_index, class_size, is_huge}; +use pool::{MIN_BLOCK, align_up, carve_lane, checked_align_up, class_index, class_size, is_huge}; use shard::{NUM_SHARDS, Shard, current_shard}; /// Alignment of every mapped extent (and its self-hosted header) — a page. The @@ -78,17 +103,129 @@ pub const EXTENT_ALIGN: usize = crate::mem_source::ALIGN; /// over many small allocations, small enough that growth tracks live demand. pub const POOL_EXTENT_BYTES: usize = 64 * 1024; -/// Chunk-directory width — the structural ceiling on live extents. The -/// directory is `DIR_N` pointers in the `Heap`; the extents it points at are -/// mapped on demand, so memory tracks demand, not this ceiling (the #205 shape). -const DIR_N: usize = 1024; -const DIR_WORDS: usize = DIR_N.div_ceil(u64::BITS as usize); +/// The most blocks a large class's extent holds. A large class's first +/// extent in a shard holds one block and each later one twice what the +/// last held, up to this ([`Heap::pool_extent_bytes`]): what a lane maps +/// ahead of its demand is never more than it has carved already, so a +/// heap serving a few large requests — a bounded arena of a megabyte — +/// pays about a block and a page for each large class it uses, while one +/// serving thousands (a Cranelift compile) maps them eight to an extent +/// after its first fifteen. Eight keeps the mapping a few times the +/// block: a miss costs no more than the shared lane's, relative to what +/// it serves. +const LARGE_CLASS_MAX_BLOCKS: usize = 8; + +/// The alignment `class`'s blocks are carved at. In the shared lane, the +/// class size: that is how a small class honours any `align` up to its +/// size. In a large class's lane, a page: its blocks follow the header +/// page back to back, so an extent of one block wastes nothing to +/// alignment, and a large request aligned past a page is served on the +/// huge path instead ([`Heap::try_alloc`]). +#[inline] +const fn block_align(class: usize) -> usize { + if carve_lane(class) == 0 { + class_size(class) + } else { + EXTENT_ALIGN + } +} + +// A large class's header fits the page its blocks follow, and its blocks +// are whole pages, so every block in its lane is page-aligned. +const _: () = { + assert!(extent_header_bytes() <= EXTENT_ALIGN); + assert!(class_size(pool::SHARED_LANE_TOP_CLASS + 1) % EXTENT_ALIGN == 0); +}; +// The shared lane carves a grow's metadata off its own tail, down to half. +const _: () = assert!(meta_pages_bound(POOL_EXTENT_BYTES) * EXTENT_ALIGN <= POOL_EXTENT_BYTES / 2); + +/// **The page map's shape.** Keyed by page number ([`EXTENT_ALIGN`] is a +/// page), four levels of [`PAGE_MAP_FANOUT`]-way nodes cover +/// `PAGE_MAP_BITS` bits of page number — every user address below +/// 2^47, which is where kore places a task's windows on all three ISAs. +/// The first level is inline in the [`Heap`] (`PAGE_MAP_FANOUT` pointers, +/// 2 KiB — the inline directory page shrank by the same amount so a +/// `static` heap's footprint did not grow); every other node is one 4 KiB +/// source extent, mapped when the first extent in its span is registered: +/// a heap's first extent costs three nodes, 12 KiB, and a 16 MiB span of +/// extents costs one leaf node after that. +/// +/// **What breaks if it moves.** A leaf covering less than a pool extent +/// would cost a node per extent; a wider inline root is `.bss` every image +/// pays from its link supply (64 KiB on the ordinary launch plan). An +/// address past the covered bits cannot be registered, so an extent there +/// refuses at the grow rather than routing wrong. +const PAGE_MAP_FANOUT: usize = 512; +const PAGE_MAP_LEVELS: usize = 4; +const PAGE_MAP_ROOT_FANOUT: usize = 256; +const PAGE_MAP_LEVEL_BITS: usize = PAGE_MAP_FANOUT.trailing_zeros() as usize; +const PAGE_MAP_ROOT_BITS: usize = PAGE_MAP_ROOT_FANOUT.trailing_zeros() as usize; +const PAGE_BITS: usize = EXTENT_ALIGN.trailing_zeros() as usize; +/// Bits of page number the map covers: `8 + 9 + 9 + 9 = 35`, so addresses +/// below `2^47`. +const PAGE_MAP_BITS: usize = PAGE_MAP_ROOT_BITS + (PAGE_MAP_LEVELS - 1) * PAGE_MAP_LEVEL_BITS; +const _: () = assert!(PAGE_MAP_FANOUT.is_power_of_two() && PAGE_MAP_ROOT_FANOUT.is_power_of_two()); +const _: () = assert!(EXTENT_ALIGN.is_power_of_two()); +/// A page-map node is exactly one page; a directory page fits one. +const _: () = assert!(core::mem::size_of::() == EXTENT_ALIGN); +const DIR_PAGE_BYTES: usize = align_up(core::mem::size_of::(), EXTENT_ALIGN); +const _: () = assert!(DIR_PAGE_BYTES == EXTENT_ALIGN); + +/// Extent-directory slots per page: what fits one page with the page's +/// own ledger, so a directory page is one unit of grow metadata like a +/// map node. The first page is inline in the [`Heap`] (its only up-front +/// cost: `PAGE_SLOTS` pointers and a bitset in the `static`); every later +/// page is carved from a grow's own mapping. +pub const PAGE_SLOTS: usize = 256; +const PAGE_WORDS: usize = PAGE_SLOTS.div_ceil(u64::BITS as usize); +/// How many directory pages a heap may hold: an architectural ceiling on +/// live extents of `DIR_PAGES × PAGE_SLOTS` (131,072 — 8 GiB of pool +/// extents; the CPython compile of `res/WASM_PLATFORM.md` §11 holds some +/// 3,900), with `DIR_PAGES - 1` page pointers inline (4 KiB of `.bss`). +/// Memory tracks demand, not this ceiling (the #205 shape). +pub const DIR_PAGES: usize = 512; +const _: () = assert!(DIR_PAGES * PAGE_SLOTS < FREE_EMPTY as usize); /// How many free huge extents one cold allocation examines before mapping. /// A bounded scan defeats head-of-line blocking (one undersized head hiding a /// suitable extent) without turning a huge allocation into an unbounded walk. const HUGE_RECYCLE_SCAN: usize = 16; +/// **Recycled huge extents are kept by size.** Bucket `b` holds the free +/// huge extents whose length is in `[EXTENT_ALIGN << b, EXTENT_ALIGN << (b + 1))`, +/// each bucket a generation-tagged Treiber stack. A request looks in the +/// bucket its length falls in — where a fit is likely but not certain, so +/// it scans [`HUGE_RECYCLE_SCAN`] entries — and then takes the head of the +/// first non-empty larger bucket, which fits by construction, so a free +/// extent of any size is found in O(buckets) with no scan. Thirty-two +/// buckets reach lengths past the page map's 2^47 bytes of address space. +/// +/// A recycled extent is handed out **whole**, however long: splitting a +/// longer one down to the request was measured and rejected. Without +/// coalescing on free, split tails fragment the long extents a compiler's +/// code buffers double into, and every doubling then maps afresh — the +/// large requests alone (above 64 KiB, 88 MiB live at their peak) mapped +/// 378 MiB split against 122 MiB whole, compiling CPython on the host +/// harness. Whole, the buckets' best fit keeps that population within +/// 1.4× of its peak; the small-but-huge population (8 KiB to 64 KiB) is +/// the pool's ([`pool::LARGE_THRESHOLD`]). Coalescing adjacent free +/// extents, which would let a split pay, needs the recycle links and a +/// per-extent state out of the extent's own memory and is the next step. +const HUGE_BUCKETS: usize = 32; +const EXTENT_ALIGN_BITS: usize = EXTENT_ALIGN.trailing_zeros() as usize; + +/// The recycle bucket a huge extent of `len` bytes (`>= EXTENT_ALIGN`) goes in. +#[inline] +const fn huge_bucket(len: usize) -> usize { + let bits = if len == 0 { 0 } else { len.ilog2() as usize }; + let bucket = bits.saturating_sub(EXTENT_ALIGN_BITS); + if bucket >= HUGE_BUCKETS { + HUGE_BUCKETS - 1 + } else { + bucket + } +} + // ── Huge-free stack packing (the board.rs generation-tagged Treiber scheme) ─── const FREE_EMPTY: u32 = u32::MAX; @@ -114,6 +251,20 @@ enum ExtentKind { Huge, } +/// What a fresh extent's payload is once its metadata is carved +/// ([`Heap::install_extent`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Payload { + /// All the mapping the metadata leaves, which may carve it down to + /// half the request: the shared lane, whose small blocks use any tail. + Rest, + /// The request, page-rounded. The metadata comes from the reserve or + /// the over-ask past the request, and what it leaves goes to the + /// reserve: a huge extent, and a large class's, whose whole blocks a + /// stray page could not hold. + Exact, +} + /// One mapped extent, its bookkeeping **self-hosted in its own head** (the /// #205/§9.4 rule: the allocator cannot allocate its own metadata *from /// itself*). Its address is stable for the heap's life — extents are held, @@ -127,8 +278,12 @@ struct Extent { /// Start of the mapped region — the header lives here, and `base` is the /// deallocate-routing key with `len`. base: usize, - /// Total mapped bytes. + /// Payload bytes: the header, then what is carved or served. len: usize, + /// Bytes of the mapping this extent came in — `len`, the metadata + /// carved from its tail and what it left to the reserve — for the + /// occupancy snapshot. + mapped: usize, /// Owning shard index (Pool), or `usize::MAX` (Huge — owned by no shard). owner: usize, /// Which path this extent backs. @@ -143,13 +298,17 @@ struct Extent { } impl Extent { + #[inline] + fn len(&self) -> usize { + self.len + } #[inline] fn end(&self) -> usize { - self.base + self.len + self.base + self.len() } #[inline] fn contains(&self, addr: usize) -> bool { - addr >= self.base && addr - self.base < self.len + addr >= self.base && addr - self.base < self.len() } } @@ -160,6 +319,45 @@ const fn extent_header_bytes() -> usize { align_up(core::mem::size_of::(), MIN_BLOCK) } +/// One interior or leaf node of the page map: [`PAGE_MAP_FANOUT`] pointers, +/// `null` until published — to a child node (interior levels) or to the +/// [`Extent`] owning that page (the leaf level). One 4 KiB source extent +/// each, written in place and published by a compare-and-swap on the parent +/// slot, so two threads registering into one empty span race on the slot +/// and the loser keeps its node for the next empty slot it meets. +struct PageMapNode { + slots: [AtomicPtr; PAGE_MAP_FANOUT], +} + +impl PageMapNode { + const fn new() -> Self { + Self { + slots: [const { AtomicPtr::new(core::ptr::null_mut()) }; PAGE_MAP_FANOUT], + } + } +} + +/// The slot index at each level of the page map for the page holding +/// `addr`: the root's first, then the three node levels, the leaf last. +/// `None` for an address past the covered bits. +#[inline] +fn page_map_path(addr: usize) -> Option<[usize; PAGE_MAP_LEVELS]> { + let page = addr >> PAGE_BITS; + if page >> PAGE_MAP_BITS != 0 { + return None; + } + let mut path = [0; PAGE_MAP_LEVELS]; + let mut rest = page; + let mut level = PAGE_MAP_LEVELS - 1; + while level > 0 { + path[level] = rest & (PAGE_MAP_FANOUT - 1); + rest >>= PAGE_MAP_LEVEL_BITS; + level -= 1; + } + path[0] = rest & (PAGE_MAP_ROOT_FANOUT - 1); + Some(path) +} + /// Occupancy snapshot (diagnostics / tests). Live-byte accounting is /// deliberately *not* maintained — it would put an atomic on the hot path — so /// this reports the mapped footprint, computed by a cold directory scan. @@ -180,6 +378,32 @@ enum GrowOutcome { Exhausted, } +/// One page of the extent directory: [`PAGE_SLOTS`] slots published +/// `Release` by [`Heap::install_extent`], read `Acquire`, `null` until +/// published (#205), with the page's own free-slot ledger and cold-scan +/// bound. Page 0 lives inline in the [`Heap`]; every later page is one +/// source extent, written in place and published by a compare-and-swap on +/// its [`Heap::pages`] pointer. +struct DirPage { + slots: [AtomicPtr; PAGE_SLOTS], + /// Free slots. Claims happen before mapping; rollback on source refusal + /// makes the capacity failure-atomic. + free: AtomicBitSet, + /// One past the highest slot ever published in this page — the cold scan + /// bound. + high_water: AtomicUsize, +} + +impl DirPage { + const fn new() -> Self { + Self { + slots: [const { AtomicPtr::new(core::ptr::null_mut()) }; PAGE_SLOTS], + free: AtomicBitSet::with_first_set(PAGE_SLOTS), + high_water: AtomicUsize::new(0), + } + } +} + /// Affine reservation of one extent-directory slot. /// /// A miss claims before it calls the source, so directory exhaustion cannot @@ -187,15 +411,19 @@ enum GrowOutcome { /// the claim and restores the bit; publication consumes it permanently because /// live extents remain mapped for the process lifetime. struct DirectoryClaim<'a> { - free: &'a AtomicBitSet, - index: usize, + page: &'a DirPage, + /// The slot within `page`. + slot: usize, + /// The directory-wide id (`page index × PAGE_SLOTS + slot`): what the + /// extent records and the huge-free stack links by. + id: u32, published: bool, } impl DirectoryClaim<'_> { #[inline] - const fn index(&self) -> usize { - self.index + const fn id(&self) -> u32 { + self.id } #[inline] @@ -207,8 +435,258 @@ impl DirectoryClaim<'_> { impl Drop for DirectoryClaim<'_> { fn drop(&mut self) { if !self.published { - self.free.release_run(self.index, 1); + self.page.free.release_run(self.slot, 1); + } + } +} + +/// **What a grow may need beyond its payload.** Every grow completes with +/// the one mapping it asked for: its directory page (when every installed +/// page is full) and the page-map nodes its span lacks are carved from +/// that mapping's tail, or taken from the metadata reserve — never mapped +/// on their own. A source that answers one miss with one extent and no +/// more (`kore-svc-mm`'s `MmSource`: a miss records the size, the service +/// refills once, the allocation retries once) is therefore served exactly. +/// +/// The bound for a span of `len` bytes: one directory page, and at each +/// node level the spans the extent can touch — one more than its length +/// divides into, since it need not start on one. 64 KiB: 1 + 2 + 2 + 2 = +/// 7 pages; 16 MiB: 1 + 2 + 2 + 9 = 14. What the grow does not use goes +/// to the reserve (a huge grow) or stays carve space (a pool grow). +const fn meta_pages_bound(len: usize) -> usize { + let mut pages = DIR_PAGE_BYTES / EXTENT_ALIGN; + let mut level = 1; + while level < PAGE_MAP_LEVELS { + pages += (len >> level_shift(level)) + 2; + level += 1; + } + pages +} + +/// The address bits below one node `level` steps under the root: the span +/// one of its slots covers is `1 << level_shift(level)` bytes (a leaf's, +/// at `PAGE_MAP_LEVELS - 1`, is a page). +const fn level_shift(level: usize) -> usize { + PAGE_BITS + (PAGE_MAP_LEVELS - level) * PAGE_MAP_LEVEL_BITS +} + +/// How many runs the metadata reserve holds. A run is one over-ask's +/// leftover; a grow over-asks only when the reserve is short of its bound, +/// which drains the reserve first, so runs seldom accumulate — sixteen +/// is room for concurrent growers. A seventeenth is that race's lost +/// pages. +const META_RESERVE_RUNS: usize = 16; +const META_RUN_COUNT_BITS: u32 = 16; +const _: () = assert!(PAGE_MAP_BITS + META_RUN_COUNT_BITS as usize <= 64); + +/// A run of unregistered, page-aligned pages held for metadata. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct MetaRun { + base: usize, + pages: usize, +} + +impl MetaRun { + const fn pack(self) -> u64 { + (((self.base >> PAGE_BITS) as u64) << META_RUN_COUNT_BITS) | self.pages as u64 + } + const fn unpack(word: u64) -> Self { + Self { + base: ((word >> META_RUN_COUNT_BITS) as usize) << PAGE_BITS, + pages: (word & ((1 << META_RUN_COUNT_BITS) - 1)) as usize, + } + } +} + +/// **The metadata reserve**: runs of pages a huge grow over-asked and did +/// not need, held for the nodes and directory pages of later grows so +/// that those ask their source for their payload alone. Lock-free: a run +/// is claimed from, or offered into, its slot by compare-and-swap. +struct MetaReserve { + runs: [AtomicU64; META_RESERVE_RUNS], +} + +impl MetaReserve { + const fn new() -> Self { + Self { + runs: [const { AtomicU64::new(0) }; META_RESERVE_RUNS], + } + } + + /// Move up to `pages` pages into `budget`, draining the reserve's runs + /// in slot order; what it holds short of that, the grow over-asks. + fn claim(&self, mut pages: usize, budget: &mut MetaBudget) { + for slot in &self.runs { + let mut word = slot.load(Ordering::Acquire); + while pages > 0 { + let run = MetaRun::unpack(word); + if run.pages == 0 { + break; + } + let take = run.pages.min(pages); + let rest = MetaRun { + base: run.base + (take << PAGE_BITS), + pages: run.pages - take, + }; + let next = if rest.pages == 0 { 0 } else { rest.pack() }; + match slot.compare_exchange_weak(word, next, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => { + if budget.add(MetaRun { + base: run.base, + pages: take, + }) { + pages -= take; + } else { + // No room in the budget: hand the run back. + let _ = self.offer(MetaRun { + base: run.base, + pages: take, + }); + return; + } + break; + } + Err(actual) => word = actual, + } + } + if pages == 0 { + return; + } + } + } + + /// Hold `run` for later grows; `false` when every slot is taken, and + /// the run is this heap's lost pages. + fn offer(&self, run: MetaRun) -> bool { + if run.pages == 0 { + return true; + } + if run.pages >= 1 << META_RUN_COUNT_BITS { + return false; } + self.runs.iter().any(|slot| { + slot.compare_exchange(0, run.pack(), Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + }) + } + + /// Pages held: what the suite checks a grow left behind. + #[cfg(test)] + fn pages(&self) -> usize { + self.runs + .iter() + .map(|slot| MetaRun::unpack(slot.load(Ordering::Acquire)).pages) + .sum() + } +} + +/// The pages one grow may spend on metadata: runs claimed from the +/// reserve, the pages a lost publication race hands back, and — last — +/// the tail of the grow's own mapping down to a floor that keeps its +/// payload whole. Local to the grow, so no atomics; what is left when +/// the grow is done goes back to the reserve. +struct MetaBudget { + runs: [MetaRun; META_BUDGET_RUNS], + len: usize, + /// The grow's own mapping: pages are carved downward from `end`, + /// never below `floor`. + tail_end: usize, + tail_floor: usize, +} + +/// Reserve runs, the tail, and a page per lost race at each level. +const META_BUDGET_RUNS: usize = META_RESERVE_RUNS + PAGE_MAP_LEVELS + 1; + +impl MetaBudget { + const fn new() -> Self { + Self { + runs: [MetaRun { base: 0, pages: 0 }; META_BUDGET_RUNS], + len: 0, + tail_end: 0, + tail_floor: 0, + } + } + + /// Let the budget carve `[floor, end)` of the grow's own mapping. + fn with_tail(&mut self, floor: usize, end: usize) { + self.tail_floor = floor; + self.tail_end = end; + } + + /// Pages in hand, the tail excluded. + fn pages(&self) -> usize { + self.runs[..self.len].iter().map(|run| run.pages).sum() + } + + fn add(&mut self, run: MetaRun) -> bool { + if run.pages == 0 { + return true; + } + if self.len == META_BUDGET_RUNS { + return false; + } + self.runs[self.len] = run; + self.len += 1; + true + } + + /// One page: from a run in hand first, else off the tail. + fn take(&mut self) -> Option { + while self.len > 0 { + let run = &mut self.runs[self.len - 1]; + if run.pages == 0 { + self.len -= 1; + continue; + } + let page = run.base; + run.base += EXTENT_ALIGN; + run.pages -= 1; + if run.pages == 0 { + self.len -= 1; + } + return Some(page); + } + let end = self.tail_end.checked_sub(EXTENT_ALIGN)?; + if end < self.tail_floor { + return None; + } + self.tail_end = end; + Some(end) + } + + /// A page back — from a lost publication race, or carved for a node + /// another grower published first — merged into the run it adjoins + /// when it adjoins one. + fn put(&mut self, page: usize) { + for run in &mut self.runs[..self.len] { + if page == run.base + (run.pages << PAGE_BITS) { + run.pages += 1; + return; + } + if page + EXTENT_ALIGN == run.base { + run.base = page; + run.pages += 1; + return; + } + } + let _ = self.add(MetaRun { + base: page, + pages: 1, + }); + } + + /// Where the grow's mapping now ends: its payload and whatever tail + /// the metadata did not take. + fn tail_end(&self) -> usize { + self.tail_end + } + + /// Hand every run still in hand to the reserve. + fn release(&mut self, reserve: &MetaReserve) { + for run in &self.runs[..self.len] { + let _ = reserve.offer(*run); + } + self.len = 0; } } @@ -218,26 +696,32 @@ impl Drop for DirectoryClaim<'_> { pub struct Heap { /// Per-task allocator lanes. shards: [Shard; NUM_SHARDS], - /// Lock-free extent directory: slots published `Release` by - /// [`install_extent`](Self::install_extent), read `Acquire`, `null` until - /// published (#205). - directory: [AtomicPtr; DIR_N], - /// Free directory slots. Claims happen before mapping; rollback on source - /// refusal makes the capacity failure-atomic. - directory_free: AtomicBitSet, - /// One past the highest slot ever published — the cold scan bound. - high_water: AtomicUsize, - /// Generation-tagged Treiber stack of free huge extents (by directory id), - /// for recycling (the board.rs scheme — the generation defeats ABA on the - /// multi-consumer pop). - huge_free: AtomicU64, + /// The extent directory's first page, inline: a `static` heap registers + /// its first [`PAGE_SLOTS`] extents without mapping anything. + page0: DirPage, + /// The directory's later pages, each mapped from the source when every + /// earlier page is full; `null` until published (`Release` by the + /// installing compare-and-swap, read `Acquire`). Pages are installed in + /// order, so the live pages are always a prefix. + pages: [AtomicPtr; DIR_PAGES - 1], + /// The page map's root: the deallocate route. Each slot covers + /// 2^39 bytes of address space through three node levels below it. + page_map: [AtomicPtr; PAGE_MAP_ROOT_FANOUT], + /// Pages held for later grows' metadata ([`MetaReserve`]). + reserve: MetaReserve, + /// Free huge extents for recycling, by size bucket ([`HUGE_BUCKETS`]): + /// each a generation-tagged Treiber stack of directory ids (the + /// board.rs scheme — the generation defeats ABA on the multi-consumer + /// pop). + huge_free: [AtomicU64; HUGE_BUCKETS], /// Where fresh extents come from. source: S, } -// SAFETY: cross-thread state is all atomic — the directory and slot ledger, -// `high_water`, `huge_free`, each shard's remote free-lists, and the pool carve -// cursors. The only non-atomic state — each shard's local free-lists and its +// SAFETY: cross-thread state is all atomic — the directory pages (their slot +// pointers, free ledgers and high-water marks, and the page pointers that +// publish them), the `huge_free` buckets, each shard's remote free-lists, and the pool +// carve cursors. The only non-atomic state — each shard's local free-lists and its // current-extent pointer — is touched solely by that shard's owning task, per // `current_shard()`'s injective task→shard map (see `shard.rs`). No global lock // sits on the hot path, so a `#[global_allocator] static` is sound. In @@ -252,10 +736,11 @@ impl Heap { pub const fn new_in(source: S) -> Self { Self { shards: [const { Shard::new() }; NUM_SHARDS], - directory: [const { AtomicPtr::new(core::ptr::null_mut()) }; DIR_N], - directory_free: AtomicBitSet::with_first_set(DIR_N), - high_water: AtomicUsize::new(0), - huge_free: AtomicU64::new(free_pack(0, FREE_EMPTY)), + page0: DirPage::new(), + pages: [const { AtomicPtr::new(core::ptr::null_mut()) }; DIR_PAGES - 1], + page_map: [const { AtomicPtr::new(core::ptr::null_mut()) }; PAGE_MAP_ROOT_FANOUT], + reserve: MetaReserve::new(), + huge_free: [const { AtomicU64::new(free_pack(0, FREE_EMPTY)) }; HUGE_BUCKETS], source, } } @@ -266,21 +751,53 @@ impl Heap { &self.source } - /// Occupancy snapshot (cold directory scan). + /// Directory page `index`, or `None` past the published prefix. + #[inline] + fn page(&self, index: usize) -> Option<&DirPage> { + if index == 0 { + return Some(&self.page0); + } + let p = self.pages.get(index - 1)?.load(Ordering::Acquire); + // SAFETY: a published page pointer names a `DirPage` written in place + // before the `Release` publication, held for the heap's life. + (!p.is_null()).then(|| unsafe { &*p }) + } + + /// The directory slot of extent id `id` (a published id — every page an + /// id names is installed before any extent in it is). + #[inline] + fn slot(&self, id: u32) -> &AtomicPtr { + let id = id as usize; + let page = self + .page(id / PAGE_SLOTS) + .expect("a published extent id names an installed directory page"); + &page.slots[id % PAGE_SLOTS] + } + + /// Occupancy snapshot: a cold scan of the directory, page by page. + /// `mapped_bytes` is every byte taken from the source — each extent's + /// whole mapping, the metadata carved from its tail and the pages it + /// left to the reserve included — so it bounds what the source gave + /// out; `extent_count` is the live extents. pub fn stats(&self) -> HeapStats { - let n = self.high_water.load(Ordering::Acquire).min(DIR_N); let mut mapped_bytes = 0; let mut extent_count = 0; - for slot in &self.directory[..n] { - let p = slot.load(Ordering::Acquire); - if p.is_null() { - continue; + for index in 0..DIR_PAGES { + let Some(page) = self.page(index) else { + break; + }; + let n = page.high_water.load(Ordering::Acquire).min(PAGE_SLOTS); + for slot in &page.slots[..n] { + let p = slot.load(Ordering::Acquire); + if p.is_null() { + continue; + } + // SAFETY: a published pointer names a live `Extent` held for + // the heap's life. + let ext = unsafe { &*p }; + mapped_bytes += ext.mapped; + extent_count += 1; } - // SAFETY: a published pointer names a live `Extent` held for the - // heap's life. - let ext = unsafe { &*p }; - mapped_bytes += ext.len; - extent_count += 1; } HeapStats { mapped_bytes, @@ -311,6 +828,11 @@ impl Heap { return self.huge_alloc(size, align); } let class = class_index(need); + if align > block_align(class) { + // A large class's blocks are page-aligned; the huge path aligns + // any payload. The free routes by the extent, not the layout. + return self.huge_alloc(size, align); + } let shard = &self.shards[current_shard()]; // Hold the shard's owner guard across the whole owner-only region (local // pop, then carve/grow in `alloc_slow`) — acquired once here, not per @@ -347,7 +869,7 @@ impl Heap { } // 3. Grow through this thread's independent miss lane and retry, // or surface exhaustion. - match self.grow_pool(shard) { + match self.grow_pool(shard, class) { GrowOutcome::Grew => continue, GrowOutcome::Exhausted => return Err(AllocError), } @@ -355,11 +877,13 @@ impl Heap { } /// Bump one `class`-sized block off the shard's current pool extent, aligning - /// the block's **absolute address** to the class size (a power of two), which - /// is `>= align` — so the returned pointer honors the request's alignment. + /// the block's **absolute address** to [`block_align`] — the class size in + /// the shared lane, a page in a large class's — which `try_alloc` has + /// checked is `>= align`, so the returned pointer honors the request's + /// alignment. fn carve(&self, shard: &Shard, class: usize) -> Option<*mut u8> { // SAFETY: owner reads its own current-extent pointer. - let ext_ptr = unsafe { shard.current() }; + let ext_ptr = unsafe { shard.current(carve_lane(class)) }; if ext_ptr.is_null() { return None; } @@ -368,7 +892,7 @@ impl Heap { let sz = class_size(class); // Owner-only cursor: a plain load/store pair, no CAS (no rival writer). let cursor = ext.cursor.load(Ordering::Relaxed); - let block = checked_align_up(cursor, sz)?; + let block = checked_align_up(cursor, block_align(class))?; let end = block.checked_add(sz)?; if end > ext.end() { return None; // this extent is spent for `class` @@ -377,55 +901,108 @@ impl Heap { Some(block as *mut u8) } - /// Map and install one fresh pool extent for `shard`. + // ── The grow path ───────────────────────────────────────────────────── + + /// The extent a grow maps for `class` in `shard`: the shared lane's + /// [`POOL_EXTENT_BYTES`]; for a large class, its header page and its + /// blocks — one when the lane has no extent yet, else twice what the + /// lane's current extent holds, up to [`LARGE_CLASS_MAX_BLOCKS`]. + /// Derived from the lane's own extent, so a refused grow asks the same + /// size again on its retry. + fn pool_extent_bytes(shard: &Shard, class: usize) -> usize { + let lane = carve_lane(class); + if lane == 0 { + return POOL_EXTENT_BYTES; + } + let size = class_size(class); + // SAFETY: the owner reads its own current-extent pointer. + let current = unsafe { shard.current(lane) }; + let blocks = if current.is_null() { + 1 + } else { + // SAFETY: a published extent pointer names a live `Extent`, + // whose geometry is immutable. + let held = unsafe { &*current }.len().saturating_sub(EXTENT_ALIGN) / size; + held.saturating_mul(2).clamp(1, LARGE_CLASS_MAX_BLOCKS) + }; + EXTENT_ALIGN + blocks * size + } + + /// Map and install one fresh pool extent for `shard`'s lane of `class`: + /// one mapping of [`pool_extent_bytes`](Self::pool_extent_bytes), so a + /// source that answers a miss with one extent of what was asked serves + /// the grow exactly. The shared lane carves the grow's metadata off + /// its own tail (down to half the extent, which the bound never + /// reaches) and keeps whatever tail is left as carve space. A large + /// class's extent is its header page and whole blocks: a page carved + /// off its tail would cost it a block, and pages left over would never + /// hold one, so it over-asks by what the reserve is short, as a huge + /// grow does, keeps exactly its blocks, and leaves the reserve the rest. /// /// Every live thread owns a distinct shard and distinct session scratch, so /// concurrent misses may map concurrently. Directory/source reservation is /// already atomic; a process-wide grow flag would only create preemption /// inversion around a synchronous kernel session. #[cold] - fn grow_pool(&self, shard: &Shard) -> GrowOutcome { - let Some(claim) = self.claim_directory() else { - return GrowOutcome::Exhausted; + fn grow_pool(&self, shard: &Shard, class: usize) -> GrowOutcome { + let bytes = Self::pool_extent_bytes(shard, class); + let mut meta = MetaBudget::new(); + let bound = meta_pages_bound(bytes); + self.reserve.claim(bound, &mut meta); + let (short, payload) = if carve_lane(class) == 0 { + (0, Payload::Rest) + } else { + (bound.saturating_sub(meta.pages()), Payload::Exact) }; - match self.source.map_extent(POOL_EXTENT_BYTES) { + let want = match short + .checked_mul(EXTENT_ALIGN) + .and_then(|extra| bytes.checked_add(extra)) + { + Some(want) => want, + None => return GrowOutcome::Exhausted, + }; + let outcome = match self.source.map_extent(want) { None => GrowOutcome::Exhausted, Some(mem) => { let base = mem.cast::().as_ptr() as usize; - let len = mem.len(); + let total = mem.len(); match self.install_extent( - claim, base, - len, - POOL_EXTENT_BYTES, + total, + bytes, + payload, ExtentKind::Pool, current_shard(), + &mut meta, ) { Ok(ext_ptr) => { // SAFETY: owner sets its own current-extent pointer. - unsafe { shard.set_current(ext_ptr) }; + unsafe { shard.set_current(carve_lane(class), ext_ptr) }; GrowOutcome::Grew } - // An unsafe source violated its geometry contract. Refuse - // before writing the self-hosted header; the mapping cannot - // be returned through this one-way source seam, but no - // allocator state or directory capacity is published. + // An unsafe source violated its geometry contract, or the + // directory is at its ceiling. Nothing is published; the + // mapping cannot be returned through this one-way seam. Err(()) => GrowOutcome::Exhausted, } } - } + }; + meta.release(&self.reserve); + outcome } - /// Claim one directory slot without mapping. The bitset's failed-claim - /// rollback makes concurrent scans lock-free; a bounded retry gives a later - /// allocation another chance if this snapshot lost every observed bit. - fn claim_directory(&self) -> Option> { - for _ in 0..DIR_N { - let index = self.directory_free.find_first_set_from(0)?; - if self.directory_free.try_claim_run(index, 1).is_ok() { + /// Claim one slot in `page` (directory page `index`) without mapping. The + /// bitset's failed-claim rollback makes concurrent scans lock-free; a + /// bounded retry gives a later allocation another chance if this snapshot + /// lost every observed bit. `None`: the page is full. + fn claim_in<'a>(page: &'a DirPage, index: usize) -> Option> { + for _ in 0..PAGE_SLOTS { + let slot = page.free.find_first_set_from(0)?; + if page.free.try_claim_run(slot, 1).is_ok() { return Some(DirectoryClaim { - free: &self.directory_free, - index, + page, + slot, + id: (index * PAGE_SLOTS + slot) as u32, published: false, }); } @@ -433,70 +1010,274 @@ impl Heap { None } - /// Consume `[base, base+len)` and its affine slot claim into a fresh - /// [`Extent`]: validate geometry, self-host the header, then - /// `Release`-publish the pointer. A concurrent reader sees the slot as - /// `null` until the complete header is visible. - fn install_extent( + /// Claim one directory slot: the first free slot in the first page + /// that has one, installing the next page — written in place on a + /// page from `meta` (#205/§9.4: metadata self-hosted in source + /// memory, never allocated from the heap) and published by + /// compare-and-swap — when every installed page is full. Two threads + /// growing at once race on the page pointer; the loser's page goes + /// back into its budget. `None`: the [`DIR_PAGES`] ceiling, or no + /// page to install one with. + fn claim_directory(&self, meta: &mut MetaBudget) -> Option> { + for index in 0..DIR_PAGES { + let page = match self.page(index) { + Some(page) => page, + None => { + let fresh = meta.take()? as *mut DirPage; + // SAFETY: a fresh, exclusively owned, page-aligned page + // (`DirPage` fits one); written before any publication. + unsafe { fresh.write(DirPage::new()) }; + match self.pages[index - 1].compare_exchange( + core::ptr::null_mut(), + fresh, + Ordering::AcqRel, + Ordering::Acquire, + ) { + // SAFETY: just published, held for the heap's life. + Ok(_) => unsafe { &*fresh }, + Err(installed) => { + meta.put(fresh as usize); + // SAFETY: the winner's page, published `Release`. + unsafe { &*installed } + } + } + } + }; + if let Some(claim) = Self::claim_in(page, index) { + return Some(claim); + } + } + None + } + + /// The published node `level` steps below the root on `addr`'s path + /// (`1`: a root slot's node; `PAGE_MAP_LEVELS - 1`: the leaf). + fn node_at(&self, addr: usize, level: usize) -> Option<&PageMapNode> { + let path = page_map_path(addr)?; + let mut node = self.page_map[path[0]].load(Ordering::Acquire); + for &index in &path[1..level] { + if node.is_null() { + return None; + } + // SAFETY: a published node pointer names a live `PageMapNode`. + node = unsafe { &*node }.slots[index].load(Ordering::Acquire); + } + // SAFETY: as above. + (!node.is_null()).then(|| unsafe { &*node }) + } + + /// The page-map nodes `[base, base + len)` lacks: at each level, those + /// of its spans whose node is not published. A concurrent grower can + /// only publish more of them, so the count is an upper bound by the + /// time the pages are carved. + fn missing_nodes(&self, base: usize, len: usize) -> usize { + let last = base + len - 1; + let mut missing = 0; + for level in 1..PAGE_MAP_LEVELS { + let shift = level_shift(level); + for span in (base >> shift)..=(last >> shift) { + if self.node_at(span << shift, level).is_none() { + missing += 1; + } + } + } + missing + } + + /// The node below `slot`, installing one there if it is still empty on + /// a page from `meta`, written in place and published by + /// compare-and-swap; a lost race puts the page back. `None`: the slot + /// is empty and the budget is spent, which the bound rules out. + fn descend<'a>( + &self, + slot: &'a AtomicPtr, + meta: &mut MetaBudget, + ) -> Option<&'a PageMapNode> { + let mut node = slot.load(Ordering::Acquire); + if node.is_null() { + let fresh = meta.take()? as *mut PageMapNode; + // SAFETY: a fresh, exclusively owned, page-aligned page (a node + // fits one); written before any publication. + unsafe { fresh.write(PageMapNode::new()) }; + node = match slot.compare_exchange( + core::ptr::null_mut(), + fresh, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => fresh, + Err(installed) => { + meta.put(fresh as usize); + installed + } + }; + } + // SAFETY: a published node pointer names a `PageMapNode` written in + // place before its `Release` publication, held for the heap's life. + Some(unsafe { &*node }) + } + + /// Enter every page of `[base, base + len)` into the page map as owned + /// by `ext`, installing the nodes the span lacks from `meta`. Pages of + /// one extent are claimed by nobody else (the source's disjointness), + /// so the leaf stores race with nothing; a reader that sees the + /// extent's slot `null` is routing a pointer that was never handed + /// out. The caller has checked the span lies within the map's reach + /// and budgeted its nodes; `Err`: the budget ran out regardless. + fn register_pages( &self, - claim: DirectoryClaim<'_>, base: usize, len: usize, + ext: *mut Extent, + meta: &mut MetaBudget, + ) -> Result<(), ()> { + let mut page = base; + let end = align_up(base + len, EXTENT_ALIGN); + while page < end { + let path = page_map_path(page).ok_or(())?; + let mut slot = &self.page_map[path[0]]; + for &index in &path[1..PAGE_MAP_LEVELS - 1] { + let node = self.descend(slot, meta).ok_or(())?; + slot = &node.slots[index]; + } + let leaf = self.descend(slot, meta).ok_or(())?; + // Consecutive pages share a leaf until its span ends: fill the + // leaf's run in one pass rather than walking per page. + let first = path[PAGE_MAP_LEVELS - 1]; + let run = ((end - page) >> PAGE_BITS).min(PAGE_MAP_FANOUT - first); + for index in first..first + run { + leaf.slots[index].store(ext.cast::(), Ordering::Release); + } + page += run << PAGE_BITS; + } + Ok(()) + } + + /// The extent owning the page `addr` lies in — the deallocate route, four + /// loads — or `None` for an address no registered extent covers. + fn route(&self, addr: usize) -> Option<&Extent> { + let path = page_map_path(addr)?; + let mut node = self.page_map[path[0]].load(Ordering::Acquire); + for &index in &path[1..] { + if node.is_null() { + return None; + } + // SAFETY: a published node pointer names a live `PageMapNode`. + node = unsafe { &*node }.slots[index].load(Ordering::Acquire); + } + // SAFETY: a leaf slot holds a published `Extent` pointer or null. + (!node.is_null()).then(|| unsafe { &*node.cast::() }) + } + + /// Consume the fresh mapping `[base, base + total)` into a published + /// [`Extent`] of at least `min_len` payload bytes: validate the + /// geometry, claim a directory slot, carve the metadata the span needs + /// (from `meta`, then the mapping's tail, down to where `payload` + /// says), self-host the header, enter the pages into the page map, and + /// `Release`-publish. A concurrent reader sees the slot `null` until + /// the complete header is visible. What the metadata leaves of the + /// tail is carve space or goes to the reserve, as `payload` says. + /// `Err`: malformed geometry (refused before any write), an address + /// past the map's reach, the directory ceiling, or a budget the bound + /// did not cover — the mapping is lost, nothing reachable names it. + #[allow(clippy::too_many_arguments)] + fn install_extent( + &self, + base: usize, + total: usize, min_len: usize, + payload: Payload, kind: ExtentKind, owner: usize, + meta: &mut MetaBudget, ) -> Result<*mut Extent, ()> { + let floor = match payload { + Payload::Rest => min_len / 2, + Payload::Exact => min_len, + }; if base % EXTENT_ALIGN != 0 - || len < min_len.max(extent_header_bytes()) - || len > isize::MAX as usize - || base.checked_add(len).is_none() + || total % EXTENT_ALIGN != 0 + || total < min_len.max(extent_header_bytes()) + || total > isize::MAX as usize + || base + .checked_add(total - 1) + .and_then(page_map_path) + .is_none() { return Err(()); } - let idx = claim.index(); + meta.with_tail(base + floor, base + total); + let claim = self.claim_directory(meta).ok_or(())?; + // What the extent will cover — all the metadata leaves it, or its + // page-rounded payload — and the nodes that span lacks, counted on + // its widest form: carving them off the tail only shrinks it, and + // the bound covers the payload's page rounding. + let span = match payload { + Payload::Rest => meta.tail_end() - base, + Payload::Exact => align_up(min_len, EXTENT_ALIGN), + }; + let nodes = self.missing_nodes(base, span); + let mut stock = MetaBudget::new(); + for _ in 0..nodes { + let page = meta.take().ok_or(())?; + stock.put(page); + } + let end = meta.tail_end(); + let len = match payload { + Payload::Rest => end - base, + Payload::Exact => { + // The over-ask the metadata did not take goes to the + // reserve. The span was budgeted for the payload alone, so + // when the reserve has no room (sixteen runs held already) + // those pages are this heap's lost ones, never its extent's. + let payload_end = base + span; + if end > payload_end { + let _ = self.reserve.offer(MetaRun { + base: payload_end, + pages: (end - payload_end) >> PAGE_BITS, + }); + } + span + } + }; + let id = claim.id(); let ext_ptr = base as *mut Extent; - // SAFETY: `base` is a fresh page-aligned mapping of `len >= header` bytes, - // owned exclusively here (not yet published), so the header write is sound - // and lands before the `Release` store below publishes the pointer. + // SAFETY: `base` is a fresh page-aligned mapping of `len >= header` + // bytes, owned exclusively here (not yet published), so the header + // write is sound and lands before the `Release` store below publishes + // the pointer. unsafe { ext_ptr.write(Extent { base, len, + mapped: total, owner, kind, - dir_idx: idx as u32, + dir_idx: id, cursor: AtomicUsize::new(base + extent_header_bytes()), next_free: AtomicU32::new(0), }); } - self.directory[idx].store(ext_ptr, Ordering::Release); - self.high_water.fetch_max(idx + 1, Ordering::Release); + let registered = self.register_pages(base, len, ext_ptr, &mut stock); + // Node pages a concurrent grower made unnecessary go back to the + // grow's budget, and from there to the reserve. + while let Some(page) = stock.take() { + meta.put(page); + } + registered?; + claim.page.slots[claim.slot].store(ext_ptr, Ordering::Release); + claim + .page + .high_water + .fetch_max(claim.slot + 1, Ordering::Release); claim.publish(); Ok(ext_ptr) } - /// The extent whose `[base, base+len)` contains `addr` (the deallocate route), - /// or `None` for a pointer this heap never handed out. - fn route(&self, addr: usize) -> Option<&Extent> { - let n = self.high_water.load(Ordering::Acquire).min(DIR_N); - for slot in &self.directory[..n] { - let p = slot.load(Ordering::Acquire); - if p.is_null() { - continue; - } - // SAFETY: published pointer → live `Extent`. - let ext = unsafe { &*p }; - if ext.contains(addr) { - return Some(ext); - } - } - None - } - /// Return `layout`'s block to its owner — **never leaks**. Routes the - /// pointer to its extent, then: a huge extent goes onto the recycle stack; a - /// pool block goes to the owning shard's local list (if we are the owner) or - /// its MPSC remote list (if not). + /// pointer to its extent through the page map (O(1)), then: a huge extent + /// goes onto the recycle stack; a pool block goes to the owning shard's + /// local list (if we are the owner) or its MPSC remote list (if not). /// /// # Safety /// `ptr`/`layout` must name a live allocation this heap produced. @@ -512,8 +1293,9 @@ impl Heap { ); return; }; + debug_assert!(ext.contains(addr)); match ext.kind { - ExtentKind::Huge => self.huge_push(ext.dir_idx), + ExtentKind::Huge => self.huge_push(huge_bucket(ext.len()), ext.dir_idx), ExtentKind::Pool => { let class = class_index(layout.size().max(layout.align())); let owner = ext.owner; @@ -533,93 +1315,123 @@ impl Heap { // ── The huge path (whole-extent allocations, recycled) ──────────────────── - /// Serve a `> LARGE_THRESHOLD` request: search a bounded prefix of recycled - /// extents for a fit, else map fresh `header + align + size` backing. + /// Serve a `> LARGE_THRESHOLD` request: the best recycled extent that + /// fits, whole ([`HUGE_BUCKETS`]), else map fresh `header + align + size` + /// backing. #[cold] fn huge_alloc(&self, size: usize, align: usize) -> Result, AllocError> { - let mut deferred = [FREE_EMPTY; HUGE_RECYCLE_SCAN]; - let mut deferred_len = 0; + let bytes = extent_header_bytes() + .checked_add(align) + .and_then(|head| head.checked_add(size)) + .ok_or(AllocError)?; + let needed = checked_align_up(bytes, EXTENT_ALIGN).ok_or(AllocError)?; + let first = huge_bucket(needed); let mut recycled = None; - while deferred_len < HUGE_RECYCLE_SCAN { - let Some(id) = self.huge_pop() else { - break; - }; - // SAFETY: an id off the huge-free stack names a published Huge extent. - let ext = unsafe { &*self.directory[id as usize].load(Ordering::Acquire) }; - if let Some(slice) = huge_slice(ext, size, align) { - recycled = Some(slice); + for bucket in first..HUGE_BUCKETS { + if bucket == first { + // The request's own bucket: a fit is likely, not certain. + // Search a bounded prefix, restoring the skipped entries in + // reverse pop order so the stack keeps its relative order + // and every off-stack extent stays exclusively held. + let mut deferred = [FREE_EMPTY; HUGE_RECYCLE_SCAN]; + let mut deferred_len = 0; + while deferred_len < HUGE_RECYCLE_SCAN { + let Some(id) = self.huge_pop(bucket) else { + break; + }; + // SAFETY: an id off a huge-free stack names a published Huge extent. + let ext = unsafe { &*self.slot(id).load(Ordering::Acquire) }; + if ext.len() >= needed { + recycled = Some(id); + break; + } + deferred[deferred_len] = id; + deferred_len += 1; + } + for &id in deferred[..deferred_len].iter().rev() { + self.huge_push(bucket, id); + } + } else if let Some(id) = self.huge_pop(bucket) { + // Every extent of a larger bucket is at least twice the + // request's bucket floor, so it fits. + recycled = Some(id); + } + if recycled.is_some() { break; } - deferred[deferred_len] = id; - deferred_len += 1; } - // Restore the skipped nodes in reverse pop order, preserving the stack's - // relative order and keeping each off-stack extent exclusively held. - for &id in deferred[..deferred_len].iter().rev() { - self.huge_push(id); - } - if let Some(slice) = recycled { - return Ok(slice); + if let Some(id) = recycled { + // SAFETY: a popped id names a published Huge extent, held + // exclusively by this thread until it is handed out. + let ext = unsafe { &*self.slot(id).load(Ordering::Acquire) }; + return huge_slice(ext, size, align).ok_or(AllocError); } - let bytes = extent_header_bytes() - .checked_add(align) - .and_then(|head| head.checked_add(size)) - .ok_or(AllocError)?; - let claim = self.claim_directory().ok_or(AllocError)?; - let mem = self.source.map_extent(bytes).ok_or(AllocError)?; - let base = mem.cast::().as_ptr() as usize; - let len = mem.len(); - let ext_ptr = self - .install_extent(claim, base, len, bytes, ExtentKind::Huge, usize::MAX) - .map_err(|()| AllocError)?; + // The metadata the span may need comes from the reserve, else is + // over-asked alongside the payload: the grow completes with the one + // mapping it asks for, and leaves the reserve what it did not use. + let bound = meta_pages_bound(bytes); + let mut meta = MetaBudget::new(); + self.reserve.claim(bound, &mut meta); + let short = bound.saturating_sub(meta.pages()); + let served = short + .checked_mul(EXTENT_ALIGN) + .and_then(|extra| checked_align_up(bytes, EXTENT_ALIGN)?.checked_add(extra)) + .and_then(|want| self.source.map_extent(want)) + .ok_or(()) + .and_then(|mem| { + let base = mem.cast::().as_ptr() as usize; + self.install_extent( + base, + mem.len(), + bytes, + Payload::Exact, + ExtentKind::Huge, + usize::MAX, + &mut meta, + ) + }); + meta.release(&self.reserve); + let ext_ptr = served.map_err(|()| AllocError)?; // SAFETY: just published. let ext = unsafe { &*ext_ptr }; huge_slice(ext, size, align).ok_or(AllocError) } - /// Push a huge extent (by directory id) onto the recycle stack. The + /// Push a huge extent (by directory id) onto recycle `bucket`. The /// generation bump defeats the ABA a racing [`huge_pop`](Self::huge_pop) /// would otherwise admit. - fn huge_push(&self, id: u32) { + fn huge_push(&self, bucket: usize, id: u32) { // SAFETY: `id` is a published extent's directory slot. - let ext = unsafe { &*self.directory[id as usize].load(Ordering::Acquire) }; - let mut head = self.huge_free.load(Ordering::Acquire); + let ext = unsafe { &*self.slot(id).load(Ordering::Acquire) }; + let stack = &self.huge_free[bucket]; + let mut head = stack.load(Ordering::Acquire); loop { ext.next_free.store(free_id(head), Ordering::Relaxed); let new = free_pack(free_gen(head).wrapping_add(1), id); - match self.huge_free.compare_exchange_weak( - head, - new, - Ordering::Release, - Ordering::Acquire, - ) { + match stack.compare_exchange_weak(head, new, Ordering::Release, Ordering::Acquire) { Ok(_) => return, Err(actual) => head = actual, } } } - /// Pop a recycled huge extent (by directory id), or `None` when the stack is - /// empty. Multi-consumer safe: the generation in the CAS'd word rejects a - /// stale head. - fn huge_pop(&self) -> Option { - let mut head = self.huge_free.load(Ordering::Acquire); + /// Pop a recycled huge extent (by directory id) off `bucket`, or `None` + /// when it is empty. Multi-consumer safe: the generation in the CAS'd + /// word rejects a stale head. + fn huge_pop(&self, bucket: usize) -> Option { + let stack = &self.huge_free[bucket]; + let mut head = stack.load(Ordering::Acquire); loop { let id = free_id(head); if id == FREE_EMPTY { return None; } // SAFETY: an id on the stack was pushed for a published extent. - let ext = unsafe { &*self.directory[id as usize].load(Ordering::Acquire) }; + let ext = unsafe { &*self.slot(id).load(Ordering::Acquire) }; let next = ext.next_free.load(Ordering::Relaxed); let new = free_pack(free_gen(head).wrapping_add(1), next); - match self.huge_free.compare_exchange_weak( - head, - new, - Ordering::AcqRel, - Ordering::Acquire, - ) { + match stack.compare_exchange_weak(head, new, Ordering::AcqRel, Ordering::Acquire) { Ok(_) => return Some(id), Err(actual) => head = actual, } @@ -655,7 +1467,8 @@ fn huge_slice(ext: &Extent, size: usize, align: usize) -> Option> // ── The two trait faces ────────────────────────────────────────────────────── // SAFETY: `try_alloc` returns blocks of `>= layout.size()` bytes aligned to -// `layout.align()` (the class-address alignment / huge-payload alignment), and +// `layout.align()` (a block's [`block_align`], which `try_alloc` checks covers +// it, else the huge payload's alignment), and // `dealloc_ptr` returns exactly such blocks to their owner; a null return is the // sole failure signal, which the caller's `handle_alloc_error` turns into an // abort — the intended `GlobalAlloc` semantics. @@ -750,8 +1563,9 @@ mod tests { #[test] fn alignment_is_honored_across_classes() { let heap = test_heap(EXTENT_ALIGN * 64); - // Spans small classes, a large-but-pool alignment (4096 <= threshold), - // and a huge alignment (16384 > threshold → huge path aligns the extent). + // Spans small classes, a page alignment a small class honours, and an + // alignment past a page in a large class (16384 → the huge path aligns + // the extent). for &(size, align) in &[ (1usize, 1usize), (8, 8), @@ -782,7 +1596,7 @@ mod tests { #[test] fn huge_path_allocates_and_recycles() { let heap = test_heap(EXTENT_ALIGN * 64); - let l = layout(20_000, 64); // > 8 KiB → huge + let l = layout(100_000, 64); // > 64 KiB → huge assert_eq!(heap.stats().extent_count, 0); let a = heap.allocate(l).expect("huge alloc"); let a_ptr = a.as_ptr() as *mut u8; @@ -800,9 +1614,9 @@ mod tests { #[test] fn huge_recycle_search_looks_past_an_undersized_head() { - let heap = test_heap(EXTENT_ALIGN * 128); - let large = layout(48_000, 64); - let small = layout(12_000, 64); + let heap = test_heap(EXTENT_ALIGN * 1024); + let large = layout(480_000, 64); + let small = layout(120_000, 64); let large_block = heap.allocate(large).expect("large huge extent"); let large_ptr = large_block.as_ptr() as *mut u8; let small_block = heap.allocate(small).expect("small huge extent"); @@ -836,46 +1650,663 @@ mod tests { assert_eq!(heap.stats().extent_count, 0); } + /// A source that admits a chosen number of maps and refuses the rest, so + /// the heap's growth can be driven into a refusal at a chosen moment. + struct GatedSource { + inner: StaticMemSource, + /// Maps still admitted; `usize::MAX` admits every one. + allow: AtomicUsize, + } + + impl GatedSource { + fn admit(&self, maps: usize) { + self.allow.store(maps, Ordering::Release); + } + } + + // SAFETY: `inner` is an atomic bump over one leaked exclusive arena; the + // gate only refuses, never changing the geometry it returns. + unsafe impl MemSource for GatedSource { + fn map_extent(&self, min_bytes: usize) -> Option> { + self.allow + .try_update(Ordering::AcqRel, Ordering::Acquire, |n| { + (n == usize::MAX).then_some(n).or_else(|| n.checked_sub(1)) + }) + .ok()?; + self.inner.map_extent(min_bytes) + } + } + + fn gated_heap(arena_bytes: usize) -> Heap { + let buf: &'static mut [u8] = alloc::vec![0u8; arena_bytes].leak(); + // SAFETY: `buf` is leaked (process-lived) and owned solely by the source. + let inner = unsafe { StaticMemSource::over(buf.as_mut_ptr(), arena_bytes) }; + Heap::new_in(GatedSource { + inner, + allow: AtomicUsize::new(usize::MAX), + }) + } + + /// `kore-svc-mm`'s `MmSource` protocol: a miss records the size asked, + /// the owner refills with exactly that, and the allocation retries + /// once. This source refuses every map but one it was told to admit, + /// of exactly the size last refused. + struct OneShotSource { + inner: StaticMemSource, + /// The size of the last refused map. + asked: AtomicUsize, + /// The one size admitted next; zero admits nothing. + admit: AtomicUsize, + } + + impl OneShotSource { + fn asked(&self) -> usize { + self.asked.load(Ordering::Acquire) + } + /// Admit one map of the size last refused. + fn refill(&self) { + self.admit.store(self.asked(), Ordering::Release); + } + } + + // SAFETY: `inner` is an atomic bump over one leaked exclusive arena; the + // gate only refuses, never changing the geometry it returns. + unsafe impl MemSource for OneShotSource { + fn map_extent(&self, min_bytes: usize) -> Option> { + if self + .admit + .compare_exchange(min_bytes, 0, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + self.asked.store(min_bytes, Ordering::Release); + return None; + } + self.inner.map_extent(min_bytes) + } + } + + /// The arena starts on a leaf's span (`1 << level_shift(PAGE_MAP_LEVELS - 1)`, + /// 2 MiB), so the extents this test sizes in nodes straddle no leaf + /// boundary and the node counts it asserts do not depend on where the + /// allocator put the arena. + fn one_shot_heap(arena_bytes: usize) -> Heap { + let span = 1usize << level_shift(PAGE_MAP_LEVELS - 1); + let buf: &'static mut [u8] = alloc::vec![0u8; arena_bytes + span].leak(); + let raw = buf.as_mut_ptr() as usize; + let base = align_up(raw, span); + // SAFETY: `buf` is leaked (process-lived), owned solely by the source, + // and `[base, base + arena_bytes)` lies inside it. + let inner = unsafe { StaticMemSource::over(base as *mut u8, arena_bytes) }; + Heap::new_in(OneShotSource { + inner, + asked: AtomicUsize::new(0), + admit: AtomicUsize::new(0), + }) + } + + /// Every pool block routes through the page map to the pool extent + /// holding it — one whose tail may have carried the map's own nodes. #[test] - fn a_rejected_extent_restores_its_directory_claim() { - let heap = test_heap(EXTENT_ALIGN * 4); - let first = heap.claim_directory().expect("slot zero"); - assert_eq!(first.index(), 0); + fn pool_blocks_route_through_the_page_map() { + let heap = test_heap(EXTENT_ALIGN * 64); + let l = layout(48, 8); + let mut held = Vec::new(); + for _ in 0..2000 { + let p = heap.allocate(l).expect("alloc"); + let addr = p.as_ptr() as *mut u8 as usize; + let ext = heap.route(addr).expect("a live block routes"); + assert_eq!(ext.kind, ExtentKind::Pool); + assert!(ext.contains(addr)); + assert_eq!(ext.mapped, POOL_EXTENT_BYTES); + assert!(ext.len() <= POOL_EXTENT_BYTES && ext.len() >= POOL_EXTENT_BYTES / 2); + held.push(p); + } assert!( - heap.install_extent(first, 1, EXTENT_ALIGN, EXTENT_ALIGN, ExtentKind::Pool, 0,) - .is_err(), - "misaligned geometry refuses before the header write" + heap.stats().extent_count >= 2, + "the working set spans extents" ); - let again = heap.claim_directory().expect("rolled-back slot zero"); - assert_eq!(again.index(), 0, "affine claim rolled back on refusal"); + for p in held { + unsafe { heap.deallocate(p.cast::(), l) }; + } } + /// Every huge payload routes through the page map to its own extent — + /// from its first page and its last, however far the alignment pushed + /// it — and back onto the recycle stack when freed. #[test] - fn a_short_aligned_extent_refuses_before_the_header_write() { - let heap = test_heap(EXTENT_ALIGN * 4); - let buf: &'static mut [u8] = alloc::vec![0u8; EXTENT_ALIGN * 2].leak(); - let raw = buf.as_mut_ptr() as usize; - let aligned = checked_align_up(raw, EXTENT_ALIGN).expect("host address aligns"); - let first = heap.claim_directory().expect("slot zero"); + fn huge_payloads_route_through_the_page_map() { + let heap = test_heap(EXTENT_ALIGN * 1024); + for &(size, align) in &[(200_000usize, 16usize), (200_000, 4096), (90_000, 65_536)] { + let l = layout(size, align); + let p = heap.allocate(l).expect("huge alloc"); + let addr = p.as_ptr() as *mut u8 as usize; + let ext = heap.route(addr).expect("a live payload routes"); + assert_eq!(ext.kind, ExtentKind::Huge, "size {size} align {align}"); + assert!(ext.contains(addr)); + let tail = heap.route(addr + size - 1).expect("its last page routes"); + assert!(core::ptr::eq(tail, ext), "one extent, first page to last"); + if align <= MIN_BLOCK { + assert_eq!(addr, ext.base + extent_header_bytes(), "unpadded payload"); + } + unsafe { heap.deallocate(p.cast::(), l) }; + // Routed to the recycle stack: the next fit comes back from it. + let again = heap.allocate(l).expect("recycled"); + assert_eq!(again.as_ptr() as *mut u8 as usize, addr); + unsafe { heap.deallocate(again.cast::(), l) }; + } + } + /// The map answers only for pages a live extent covers, and an extent + /// past its reach refuses before the header write — a write there would + /// fault, which is the test — leaving the directory untouched. + #[test] + fn the_page_map_ends_where_user_addresses_do() { + let heap = test_heap(EXTENT_ALIGN * 32); + let beyond = 1usize << (PAGE_BITS + PAGE_MAP_BITS); + assert_eq!(beyond, 1 << 47); + assert!(page_map_path(beyond).is_none()); + assert!(page_map_path(beyond - 1).is_some()); + assert!( + heap.route(0).is_none(), + "the null page is never an extent's" + ); + assert!(heap.route(beyond).is_none()); + + let mut meta = MetaBudget::new(); assert!( heap.install_extent( - first, - aligned, - extent_header_bytes(), - POOL_EXTENT_BYTES, + beyond, + EXTENT_ALIGN, + EXTENT_ALIGN, + Payload::Exact, ExtentKind::Pool, 0, + &mut meta ) .is_err(), - "source geometry shorter than the request must be rejected" + "an extent starting past the map refuses" + ); + assert!( + heap.install_extent( + beyond - EXTENT_ALIGN, + 2 * EXTENT_ALIGN, + EXTENT_ALIGN, + Payload::Exact, + ExtentKind::Pool, + 0, + &mut meta + ) + .is_err(), + "an extent whose last page lies past the map refuses" + ); + assert_eq!(heap.stats().extent_count, 0); + let l = layout(48, 8); + let p = heap.allocate(l).expect("alloc"); + let ext = heap.route(p.as_ptr() as *mut u8 as usize).expect("routes"); + assert_eq!(ext.dir_idx, 0, "no claim was spent on the refusals"); + unsafe { heap.deallocate(p.cast::(), l) }; + } + + /// A grow completes with the one mapping it asked for: a source that + /// answers a miss with exactly that extent, once, serves a pool grow + /// (its nodes carved from the extent's own tail) and a huge grow (over- + /// asked by the metadata bound while the reserve is short, the unused + /// part left to the reserve, so the next huge grow asks its payload + /// alone). + #[test] + fn a_grow_completes_with_the_one_extent_it_asked_for() { + let heap = one_shot_heap(EXTENT_ALIGN * 1024); + let small = layout(48, 8); + assert_eq!(heap.allocate(small), Err(AllocError), "the first miss"); + assert_eq!(heap.source().asked(), POOL_EXTENT_BYTES); + heap.source().refill(); + let p = heap.allocate(small).expect("the refill serves the retry"); + let ext = heap.route(p.as_ptr() as *mut u8 as usize).expect("routes"); + assert_eq!(ext.mapped, POOL_EXTENT_BYTES); + assert_eq!( + ext.len(), + POOL_EXTENT_BYTES - (PAGE_MAP_LEVELS - 1) * EXTENT_ALIGN, + "the map's three nodes came off the pool extent's tail" + ); + assert_eq!(heap.reserve.pages(), 0, "a pool grow leaves nothing over"); + unsafe { heap.deallocate(p.cast::(), small) }; + + let big = layout(200_000, 64); + let bytes = extent_header_bytes() + 64 + 200_000; + assert_eq!(heap.allocate(big), Err(AllocError), "a huge miss"); + let asked = heap.source().asked(); + assert_eq!( + asked, + align_up(bytes, EXTENT_ALIGN) + meta_pages_bound(bytes) * EXTENT_ALIGN, + "over-asked by the bound with the reserve empty" + ); + heap.source().refill(); + let p = heap.allocate(big).expect("the refill serves the retry"); + let ext = heap.route(p.as_ptr() as *mut u8 as usize).expect("routes"); + assert_eq!(ext.kind, ExtentKind::Huge); + assert_eq!(ext.mapped, asked); + assert_eq!(ext.len(), align_up(bytes, EXTENT_ALIGN), "payload alone"); + let reserved = heap.reserve.pages(); + assert_eq!( + reserved, + meta_pages_bound(bytes), + "the span shared the pool extent's nodes: the whole over-ask is reserved" + ); + + let more = layout(200_000, 64); + assert_eq!(heap.allocate(more), Err(AllocError), "another huge miss"); + assert_eq!( + heap.source().asked(), + align_up(bytes, EXTENT_ALIGN), + "the reserve covers the bound: the payload alone is asked" + ); + heap.source().refill(); + let q = heap.allocate(more).expect("served"); + assert_eq!( + heap.reserve.pages(), + reserved, + "nothing taken, nothing lost" + ); + unsafe { + heap.deallocate(p.cast::(), big); + heap.deallocate(q.cast::(), more); + } + } + + /// The reserve hands runs to a budget and takes them back, page by + /// page, merging what comes back adjacent. + #[test] + fn the_reserve_claims_and_offers_runs() { + let reserve = MetaReserve::new(); + let base = 0x7000_0000usize; + assert!(reserve.offer(MetaRun { base, pages: 3 })); + assert!(reserve.offer(MetaRun { + base: base + 16 * EXTENT_ALIGN, + pages: 2 + })); + assert_eq!(reserve.pages(), 5); + + let mut budget = MetaBudget::new(); + reserve.claim(4, &mut budget); + assert_eq!(budget.pages(), 4); + assert_eq!(reserve.pages(), 1, "the second run kept its last page"); + assert_eq!( + budget.take(), + Some(base + 16 * EXTENT_ALIGN), + "last run first" + ); + assert_eq!( + budget.take(), + Some(base), + "then the first run, from its base" + ); + assert_eq!(budget.take(), Some(base + EXTENT_ALIGN)); + budget.put(base + EXTENT_ALIGN); + assert_eq!(budget.pages(), 2, "the page back merged with its run"); + assert_eq!(budget.len, 1); + budget.release(&reserve); + assert_eq!(reserve.pages(), 3); + + // The tail serves after the runs, down to its floor. + let mut budget = MetaBudget::new(); + budget.with_tail(base + 2 * EXTENT_ALIGN, base + 4 * EXTENT_ALIGN); + assert_eq!(budget.take(), Some(base + 3 * EXTENT_ALIGN)); + assert_eq!(budget.take(), Some(base + 2 * EXTENT_ALIGN)); + assert_eq!(budget.take(), None, "the floor holds"); + assert_eq!(budget.tail_end(), base + 2 * EXTENT_ALIGN); + } + + /// Past the inline page, the directory installs further pages on + /// metadata the grows carry: more live extents than one page holds, + /// every one of them routable, and recycled rather than remapped once + /// freed. + #[test] + fn the_directory_grows_past_its_inline_page() { + // 320 huge extents of 17 pages (header + align + 65,537 bytes, page + // rounded) plus the metadata bound a few of them over-ask. + let count = PAGE_SLOTS + PAGE_SLOTS / 4; + let heap = test_heap(count * 17 * EXTENT_ALIGN + 128 * EXTENT_ALIGN); + let l = layout(pool::LARGE_THRESHOLD + 1, 16); + let mut held: Vec> = Vec::with_capacity(count); + for i in 0..count { + let p = heap + .allocate(l) + .unwrap_or_else(|_| panic!("extent {i} of {count}")); + held.push(p); + } + let grown = heap.stats(); + assert_eq!( + grown.extent_count, count, + "every allocation is its own extent" + ); + assert!(count > PAGE_SLOTS, "the working set needs a second page"); + assert!( + heap.page(1).is_some() && heap.page(2).is_none(), + "one mapped page" + ); + // Each routes to its extent, the last with a directory id past page 0. + let last = held[count - 1].as_ptr() as *mut u8 as usize; + let ext = heap.route(last).expect("a live payload routes"); + assert!( + ext.dir_idx as usize >= PAGE_SLOTS, + "registered in a mapped page" + ); + assert!(ext.contains(last)); + assert!(core::ptr::eq( + heap.slot(ext.dir_idx).load(Ordering::Acquire), + ext + )); + for p in held.drain(..) { + unsafe { heap.deallocate(p.cast::(), l) }; + } + // Freed extents recycle through the stack; nothing new is mapped. + for _ in 0..count { + held.push(heap.allocate(l).expect("recycled extent")); + } + assert_eq!(heap.stats(), grown, "all recycled"); + for p in held { + unsafe { heap.deallocate(p.cast::(), l) }; + } + } + + /// A large class (16, 32, 64 KiB) carves from extents of its own: a + /// header page, then whole blocks back to back, page-aligned. A + /// shard's first holds one block and each later one twice the last's, + /// up to eight, so twenty-three requests take extents of 1, 2, 4, 8 + /// and 8 blocks; the class recycles exactly — a free block serves the + /// next request without a mapping — and a small class keeps its own + /// lane. + #[test] + fn a_large_class_ramps_its_extents_from_one_block() { + let heap = test_heap(16 << 20); + let size = 64 * 1024; + let l = layout(size, 16); + assert!(!is_huge(size)); + let mut held: Vec> = Vec::new(); + let mut bases: Vec = Vec::new(); + for _ in 0..23 { + let block = heap.allocate(l).expect("a 64 KiB block"); + let addr = block.cast::().as_ptr() as usize; + assert_eq!(addr % EXTENT_ALIGN, 0, "page-aligned"); + let ext = heap.route(addr).expect("routes"); + assert_eq!(ext.kind, ExtentKind::Pool); + if bases.last() != Some(&ext.base) { + assert_eq!( + addr, + ext.base + EXTENT_ALIGN, + "the first block follows the header page" + ); + bases.push(ext.base); + } else { + let last = held.last().expect("an earlier block").cast::().as_ptr() as usize; + assert_eq!(addr, last + size, "blocks lie back to back"); + } + held.push(block); + } + let blocks: Vec = bases + .iter() + .map(|&base| { + let len = heap.route(base).expect("routes").len(); + assert_eq!( + (len - EXTENT_ALIGN) % size, + 0, + "whole blocks, nothing left over" + ); + (len - EXTENT_ALIGN) / size + }) + .collect(); + assert_eq!(blocks, [1, 2, 4, 8, 8]); + let grown = heap.stats(); + // SAFETY: a live allocation of `l`. + unsafe { heap.deallocate(held.pop().expect("held").cast(), l) }; + let again = heap.allocate(l).expect("a recycled 64 KiB block"); + assert_eq!( + heap.stats().mapped_bytes, + grown.mapped_bytes, + "recycled, not mapped" + ); + let small = heap.allocate(layout(32, 16)).expect("a small block"); + assert_eq!( + heap.stats().extent_count, + grown.extent_count + 1, + "the shared lane grows" ); + // SAFETY: live allocations of their layouts. + unsafe { + heap.deallocate(small.cast(), layout(32, 16)); + heap.deallocate(again.cast(), l); + for block in held { + heap.deallocate(block.cast(), l); + } + } + } + + /// A bounded arena pays about a block and a page for each large class + /// it uses: one request in each of the 16, 32 and 64 KiB classes fits + /// a 256 KiB arena, where extents of eight blocks would have asked for + /// 896 KiB — what refused a megabyte arena's first few large requests. + /// What the grows mapped is their blocks, header pages and the metadata + /// they over-asked for, the unused part of which waits in the reserve. + #[test] + fn a_bounded_arena_pays_a_block_and_a_page_per_large_class() { + let heap = test_heap(256 * 1024 + EXTENT_ALIGN); + let sizes = [16 * 1024, 32 * 1024, 64 * 1024]; + let held: Vec<(NonNull<[u8]>, Layout)> = sizes + .iter() + .map(|&size| { + let l = layout(size, 16); + ( + heap.allocate(l) + .unwrap_or_else(|_| panic!("a {size}-byte block")), + l, + ) + }) + .collect(); + let stats = heap.stats(); + assert_eq!(stats.extent_count, sizes.len()); + let payload: usize = sizes.iter().map(|size| EXTENT_ALIGN + size).sum(); + let over_ask: usize = sizes + .iter() + .map(|size| meta_pages_bound(EXTENT_ALIGN + size) * EXTENT_ALIGN) + .sum(); + assert!( + stats.mapped_bytes >= payload && stats.mapped_bytes <= payload + over_ask, + "a block and a page each, and their metadata: {stats:?}" + ); + for (p, l) in held { + // SAFETY: live allocations of their layouts. + unsafe { heap.deallocate(p.cast(), l) }; + } + } + + /// A large class's blocks are page-aligned, so a large request aligned + /// past a page is served on the huge path, which aligns any payload; + /// its free routes by the extent, back to the huge buckets. + #[test] + fn a_large_request_aligned_past_a_page_is_served_whole() { + let heap = test_heap(4 << 20); + let paged = layout(20_000, EXTENT_ALIGN); + let p = heap.allocate(paged).expect("page-aligned"); + let ext = heap + .route(p.cast::().as_ptr() as usize) + .expect("routes"); + assert_eq!( + ext.kind, + ExtentKind::Pool, + "a page-aligned large request is a class block" + ); + let wide = layout(20_000, 4 * EXTENT_ALIGN); + let q = heap.allocate(wide).expect("aligned past a page"); + let addr = q.cast::().as_ptr() as usize; + assert_eq!(addr % (4 * EXTENT_ALIGN), 0, "aligned as asked"); + assert_eq!(heap.route(addr).expect("routes").kind, ExtentKind::Huge); + let before = heap.stats(); + // SAFETY: a live allocation of `wide`. + unsafe { heap.deallocate(q.cast(), wide) }; + let again = heap.allocate(wide).expect("recycled"); + assert_eq!( + again.cast::(), + q.cast::(), + "the huge extent came back" + ); + assert_eq!( + heap.stats().mapped_bytes, + before.mapped_bytes, + "recycled, not mapped" + ); + // SAFETY: live allocations of their layouts. + unsafe { + heap.deallocate(again.cast(), wide); + heap.deallocate(p.cast(), paged); + } + } + + /// On `kore-svc-mm`'s protocol a large class's grow is one miss, one + /// refill of exactly what was asked, one retry: its size is derived + /// from the lane's own extent, which a refused grow leaves as it was, + /// so the retry asks the same again — one block, then two. + #[test] + fn a_large_class_grow_is_served_by_one_refill() { + let heap = one_shot_heap(EXTENT_ALIGN * 1024); + let size = 64 * 1024; + let l = layout(size, 16); + let mut held = Vec::new(); + for blocks in [1, 2] { + assert_eq!(heap.allocate(l), Err(AllocError), "a miss"); + heap.source().refill(); + let p = heap.allocate(l).expect("the refill serves the retry"); + let ext = heap + .route(p.cast::().as_ptr() as usize) + .expect("routes"); + assert_eq!(ext.len(), EXTENT_ALIGN + blocks * size); + held.push(p); + } + for p in held { + // SAFETY: live allocations of `l`. + unsafe { heap.deallocate(p.cast(), l) }; + } + } + + /// Free huge extents are found by size: a request takes the smallest + /// free extent that fits, not the most recently freed one. + #[test] + fn a_huge_request_takes_the_smallest_free_extent_that_fits() { + let heap = test_heap(64 << 20); + let big = layout(1 << 20, 16); + let fit = layout(128 << 10, 16); + let p_fit = heap.allocate(fit).expect("128 KiB"); + let p_big = heap.allocate(big).expect("1 MiB"); + let mapped = heap.stats(); + // Free the fit first, then the big one: a size-blind stack would hand + // the big one (most recent) to the next 128 KiB request. + // SAFETY: live allocations of their layouts. + unsafe { + heap.deallocate(p_fit.cast(), fit); + heap.deallocate(p_big.cast(), big); + } + let again = heap.allocate(fit).expect("128 KiB again"); + let after = heap.stats(); + assert_eq!( + again.cast::(), + p_fit.cast::(), + "the 128 KiB extent, not the 1 MiB one" + ); + assert_eq!( + after.mapped_bytes, mapped.mapped_bytes, + "recycled, not mapped" + ); + assert_eq!(after.extent_count, mapped.extent_count, "and not split"); + // A request past every free extent maps fresh. + let bigger = layout(2 << 20, 16); + let fresh = heap.allocate(bigger).expect("2 MiB"); + assert!(heap.stats().mapped_bytes > mapped.mapped_bytes); + // SAFETY: live allocations of their layouts. + unsafe { + heap.deallocate(again.cast(), fit); + heap.deallocate(fresh.cast(), bigger); + } + } + + /// The bucket ladder: an extent's bucket is the power of two of its + /// length above the alignment, and the last bucket holds the rest. + #[test] + fn huge_buckets_follow_the_length() { + assert_eq!(huge_bucket(EXTENT_ALIGN), 0); + assert_eq!(huge_bucket(2 * EXTENT_ALIGN - 1), 0); + assert_eq!(huge_bucket(2 * EXTENT_ALIGN), 1); + assert_eq!(huge_bucket(1 << 20), 20 - EXTENT_ALIGN_BITS); + assert_eq!(huge_bucket(usize::MAX), HUGE_BUCKETS - 1); + } + + /// With the inline page full and the source spent, the next grow + /// refuses with no page published, and the first grow after the source + /// returns brings the page with it and lands in its first slot. + #[test] + fn a_spent_source_refuses_a_directory_page() { + // One 17-page huge extent each (header + align + 65,537 bytes, page + // rounded), and the metadata bound a few of them over-ask. + let heap = gated_heap(PAGE_SLOTS * 17 * EXTENT_ALIGN + 128 * EXTENT_ALIGN); + let l = layout(pool::LARGE_THRESHOLD + 1, 16); + let mut held: Vec> = Vec::with_capacity(PAGE_SLOTS + 1); + for i in 0..PAGE_SLOTS { + held.push( + heap.allocate(l) + .unwrap_or_else(|_| panic!("extent {i} fills the inline page")), + ); + } + let full = heap.stats(); + assert_eq!(full.extent_count, PAGE_SLOTS); + assert!(heap.page(1).is_none()); + + heap.source().admit(0); + assert_eq!(heap.allocate(l), Err(AllocError), "no extent, no page"); + assert!(heap.page(1).is_none(), "nothing published"); + assert_eq!(heap.stats(), full); + + heap.source().admit(usize::MAX); + let p = heap.allocate(l).expect("the source is back"); + let ext = heap.route(p.as_ptr() as *mut u8 as usize).expect("routes"); + assert_eq!( + ext.dir_idx as usize, PAGE_SLOTS, + "first slot of the mapped page" + ); + assert!(heap.page(1).is_some()); + held.push(p); + for p in held { + unsafe { heap.deallocate(p.cast::(), l) }; + } + } + + #[test] + fn a_short_aligned_extent_refuses_before_the_header_write() { + let heap = test_heap(EXTENT_ALIGN * 4); + let buf: &'static mut [u8] = alloc::vec![0u8; EXTENT_ALIGN * 2].leak(); + let raw = buf.as_mut_ptr() as usize; + let aligned = checked_align_up(raw, EXTENT_ALIGN).expect("host address aligns"); + let mut meta = MetaBudget::new(); + for (total, note) in [ + (extent_header_bytes(), "shorter than the request"), + (EXTENT_ALIGN + 1, "not a whole number of pages"), + ] { + assert!( + heap.install_extent( + aligned, + total, + EXTENT_ALIGN, + Payload::Exact, + ExtentKind::Pool, + 0, + &mut meta, + ) + .is_err(), + "source geometry {note} must be rejected" + ); + } assert!( buf.iter().all(|byte| *byte == 0), "validation must happen before writing self-hosted metadata" ); - let again = heap.claim_directory().expect("rolled-back slot zero"); - assert_eq!(again.index(), 0, "short geometry restores its claim"); + assert_eq!(heap.stats().extent_count, 0); } #[test] @@ -900,9 +2331,10 @@ mod tests { #[test] fn allocator_face_errs_when_source_exhausted() { - // Room for ~one pool extent; the second grow must fail with AllocError, - // NOT abort (the Allocator face never aborts). - let heap = test_heap(EXTENT_ALIGN * 2 + POOL_EXTENT_BYTES); + // Room for one pool extent on its own boundary (whatever the arena's + // front alignment costs) and no second; the second grow must fail + // with AllocError, NOT abort (the Allocator face never aborts). + let heap = test_heap(EXTENT_ALIGN + 2 * POOL_EXTENT_BYTES); let l = layout(8192, 8); let mut ok = 0; let err = loop { @@ -1030,7 +2462,9 @@ mod tests { } } - let buf: &'static mut [u8] = alloc::vec![0u8; 4 * POOL_EXTENT_BYTES].leak(); + // Room for two pool extents after the arena's front alignment; the + // map's nodes come off the extents themselves. + let buf: &'static mut [u8] = alloc::vec![0u8; 3 * POOL_EXTENT_BYTES].leak(); let source = ConcurrentSource { // SAFETY: leaked and handed exclusively to this source. inner: unsafe { StaticMemSource::over(buf.as_mut_ptr(), buf.len()) }, @@ -1044,6 +2478,8 @@ mod tests { let b = scope.spawn(|| heap.allocate(l).is_ok()); assert!(a.join().unwrap() && b.join().unwrap()); }); + // One map per miss: each pool extent carries the metadata its span + // needs, so the source saw exactly the two. assert_eq!(*heap.source.entered.lock().unwrap(), 2); assert_eq!(heap.stats().extent_count, 2); } diff --git a/native/crates/rt/src/heap/pool.rs b/native/crates/rt/src/heap/pool.rs index 2fd4d3d..a0366df 100644 --- a/native/crates/rt/src/heap/pool.rs +++ b/native/crates/rt/src/heap/pool.rs @@ -9,13 +9,16 @@ //! # Why power-of-two classes //! //! Classes are the powers of two from [`MIN_BLOCK`] (16 B) up to -//! [`LARGE_THRESHOLD`] (8 KiB). The choice is deliberate: it makes the +//! [`LARGE_THRESHOLD`] (64 KiB). The choice is deliberate: it makes the //! **alignment contract fall out for free**. A request folds its size and //! its alignment into one figure — `need = max(size, align)` — and the class -//! is `ceil_pow2(need)`. Blocks of a class are laid out contiguously from a -//! class-aligned offset, so every block is `class_size`-aligned; since -//! `class_size >= need >= align` and both are powers of two, the block is -//! `align`-aligned by construction (see the `Heap` carve path). Coarser than +//! is `ceil_pow2(need)`. In the shared lane ([`carve_lane`]) blocks of a +//! class are laid out contiguously from a class-aligned offset, so every block +//! is `class_size`-aligned; since `class_size >= need >= align` and both are +//! powers of two, the block is `align`-aligned by construction (see the `Heap` +//! carve path). A large class's blocks are page-aligned instead (whole pages +//! after a header page), and the `Heap` sends a large request aligned past a +//! page to the huge route, so the contract holds there too. Coarser than //! mimalloc's ~3-bit-mantissa classes — up to 2x internal waste — but the //! correctness argument is a one-liner, which is the right trade for a //! from-scratch allocator whose first duty is *never hand back a bad pointer*. @@ -33,11 +36,48 @@ pub(crate) const MIN_BLOCK: usize = 16; /// The largest segregated class. A `need` above this is a "huge" request the /// `Heap` serves from a dedicated whole-extent allocation, not a free-list. -pub(crate) const LARGE_THRESHOLD: usize = 8192; +/// +/// 64 KiB, up from 8 KiB: a huge extent is a mapping's grain (64 KiB) at +/// the least, so every request between the two cost a whole one — and a +/// compiler's working set lives there. Compiling CPython on the host +/// harness asked 59,024 times for 8–16 KiB, 37,116 for 16–32 KiB and +/// 16,035 for 32–64 KiB (137 MiB live at their peak), and mapped 364 MiB +/// for them as huge extents. As classes they recycle exactly and share +/// class-sized extents ([`carve_lane`]). +pub(crate) const LARGE_THRESHOLD: usize = 64 * 1024; /// Number of segregated size classes — the powers of two in -/// `[MIN_BLOCK, LARGE_THRESHOLD]`, i.e. `16, 32, …, 8192` (indices `0..=9`). -pub(crate) const NUM_CLASSES: usize = 10; +/// `[MIN_BLOCK, LARGE_THRESHOLD]`, i.e. `16, 32, …, 65536` (indices `0..=12`). +pub(crate) const NUM_CLASSES: usize = 13; + +/// The last class that carves from the shard's shared bump extent: 8 KiB. +/// Classes above it carve from extents of their own ([`carve_lane`]). +pub(crate) const SHARED_LANE_TOP_CLASS: usize = 9; + +/// Carve lanes per shard: the shared one, then one per class above +/// [`SHARED_LANE_TOP_CLASS`] (16, 32 and 64 KiB). +pub(crate) const NUM_LANES: usize = NUM_CLASSES - SHARED_LANE_TOP_CLASS; + +/// Which of a shard's current extents class `class` carves fresh blocks +/// from. +/// +/// Small classes share one bump extent: a 16 B block after a 4 KiB one +/// costs at most that much alignment padding, and a 64 KiB extent holds +/// many of either. A block of 16 KiB and up carved from a shared extent +/// would pad the cursor to its own size on every alternation with a small +/// class — up to the block's size lost per carve — and a 64 KiB block does +/// not fit a 64 KiB extent at all once the header is at its base. So each +/// large class carves from extents of its own (`Heap::pool_extent_bytes`): +/// a header page, then its blocks back to back — one in a shard's first, +/// twice the last's in each later one, up to eight. +#[inline] +pub(crate) const fn carve_lane(class: usize) -> usize { + if class <= SHARED_LANE_TOP_CLASS { + 0 + } else { + class - SHARED_LANE_TOP_CLASS + } +} // The class table and the huge threshold are one definition, pinned so a later // edit to one constant that desyncs the other fails the build, not a test. @@ -45,6 +85,8 @@ const _: () = { assert!(MIN_BLOCK << (NUM_CLASSES - 1) == LARGE_THRESHOLD); assert!(MIN_BLOCK.is_power_of_two()); assert!(LARGE_THRESHOLD.is_power_of_two()); + assert!(MIN_BLOCK << SHARED_LANE_TOP_CLASS == 8192); + assert!(NUM_LANES == 4); }; /// Round `x` up to a multiple of `align` (a power of two). The workhorse for @@ -81,12 +123,12 @@ pub(crate) const fn is_huge(need: usize) -> bool { /// /// Caller guarantees `!is_huge(need)`; the result is in `0..NUM_CLASSES` and /// [`class_size`] of it is `>= need`. `need = 1..16 → 0`, `17..32 → 1`, …, -/// `4097..8192 → 9`. +/// `32769..65536 → 12`. #[inline] pub(crate) fn class_index(need: usize) -> usize { debug_assert!(!is_huge(need)); let block = need.max(MIN_BLOCK).next_power_of_two(); - // `block` is a power of two in `[16, 8192]`, so `trailing_zeros` is its + // `block` is a power of two in `[16, 65536]`, so `trailing_zeros` is its // log2; subtract log2(MIN_BLOCK)=4 to index from the first class. (block.trailing_zeros() - MIN_BLOCK.trailing_zeros()) as usize } @@ -188,7 +230,9 @@ mod tests { #[test] fn class_size_always_covers_the_request() { // The core allocator guarantee: the served block is ≥ the ask. - for need in [1usize, 8, 15, 16, 31, 33, 100, 1000, 4096, 8191, 8192] { + for need in [ + 1usize, 8, 15, 16, 31, 33, 100, 1000, 4096, 8191, 8192, 8193, 16384, 40_000, 65536, + ] { let sz = class_size(class_index(need)); assert!(sz >= need, "class {sz} must cover need {need}"); assert!(sz.is_power_of_two()); diff --git a/native/crates/rt/src/heap/shard.rs b/native/crates/rt/src/heap/shard.rs index 23c608d..e930569 100644 --- a/native/crates/rt/src/heap/shard.rs +++ b/native/crates/rt/src/heap/shard.rs @@ -24,7 +24,7 @@ use core::cell::UnsafeCell; use core::sync::atomic::{AtomicBool, AtomicPtr}; use super::Extent; -use super::pool::{self, NUM_CLASSES}; +use super::pool::{self, NUM_CLASSES, NUM_LANES}; /// The number of shards. A design constant, not a discovery: the ceiling on /// concurrently-allocating tasks that each get a lock-free lane before two must @@ -47,9 +47,11 @@ pub(crate) struct Shard { /// drains. One per class so a drain is targeted and the class of a remote /// block is implicit in which stack holds it (no per-block class tag). remote: [AtomicPtr; NUM_CLASSES], - /// The pool [`Extent`](super::Extent) this shard bump-carves fresh blocks - /// from, or null before its first grow. Owner-only. - current: UnsafeCell<*mut Extent>, + /// The pool [`Extent`](super::Extent)s this shard bump-carves fresh + /// blocks from, one per carve lane ([`pool::carve_lane`]: the small + /// classes' shared extent, then one per large class), each null before + /// its first grow. Owner-only. + current: [UnsafeCell<*mut Extent>; NUM_LANES], /// Debug-only single-owner detector: set while a thread is inside this /// shard's owner-only region ([`Shard::owner_guard`]). A second concurrent /// entry — the exact precondition for the `UnsafeCell` data race a @@ -77,7 +79,7 @@ impl Shard { Self { local: [const { UnsafeCell::new(core::ptr::null_mut()) }; NUM_CLASSES], remote: [const { AtomicPtr::new(core::ptr::null_mut()) }; NUM_CLASSES], - current: UnsafeCell::new(core::ptr::null_mut()), + current: [const { UnsafeCell::new(core::ptr::null_mut()) }; NUM_LANES], owner_active: AtomicBool::new(false), } } @@ -159,24 +161,27 @@ impl Shard { reclaimed } - /// The pool extent this shard is carving from (null before the first grow). + /// The pool extent this shard is carving `lane` from (null before the + /// lane's first grow). /// /// # Safety /// Owner-only read. #[inline] - pub(crate) unsafe fn current(&self) -> *mut Extent { + pub(crate) unsafe fn current(&self, lane: usize) -> *mut Extent { + debug_assert!(lane < NUM_LANES); // SAFETY: owner-only access (caller contract). - unsafe { *self.current.get() } + unsafe { *self.current[lane].get() } } - /// Set the pool extent this shard carves from (after a grow). + /// Set the pool extent this shard carves `lane` from (after a grow). /// /// # Safety /// Owner-only write. #[inline] - pub(crate) unsafe fn set_current(&self, extent: *mut Extent) { + pub(crate) unsafe fn set_current(&self, lane: usize, extent: *mut Extent) { + debug_assert!(lane < NUM_LANES); // SAFETY: owner-only access (caller contract). - unsafe { *self.current.get() = extent } + unsafe { *self.current[lane].get() = extent } } /// Enter this shard's owner-only region, returning a guard that marks it busy diff --git a/native/crates/rt/src/heap/source.rs b/native/crates/rt/src/heap/source.rs index c3564b9..5a480d2 100644 --- a/native/crates/rt/src/heap/source.rs +++ b/native/crates/rt/src/heap/source.rs @@ -147,99 +147,300 @@ unsafe impl MemSource for HeapSupplySource { } } -/// This task's monotone offset into its heap supply — the client half of the -/// grow (which window of the supply the next carve names). A refused carve -/// leaves the over-reserved span dead (terminal exhaustion — every later -/// grow refuses too). Checked `try_update` preserves that monotone rule -/// without admitting arithmetic wrap back into a live offset. -#[cfg(target_os = "none")] -static HEAP_GROWN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); - -/// Bytes reserved off the **top** of the heap supply for code and data -/// windows a task carves itself (`reserve_from_top`) — the late loader's -/// arenas. One supply, two monotone cursors: the allocator grows upward -/// from 0 behind [`HEAP_GROWN`], reservations grow downward from the -/// supply's end behind this one, and each side refuses the moment it -/// would cross the other's current mark. A refused reservation, like a -/// refused grow, leaves its interval spent: the two cursors only ever -/// advance, so "spent" is what keeps them apart without a lock. -#[cfg(target_os = "none")] -static HEAP_RESERVED: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +/// **One supply, two monotone cursors.** The allocator claims extents and +/// granules upward from offset 0 behind `grown`; [`reserve_from_top`] claims +/// code and data windows a task maps itself (the late loader's arenas, a +/// Wasmtime engine's compiled images and guest memories) downward from the +/// supply's end behind `reserved`. Every supply a task draws on — the launch +/// supply and each adopted one — is held apart by one of these, so the rule is +/// written once. +/// +/// Each claim is made first and checked second: it advances its own cursor, +/// then refuses if its interval reaches past the other cursor's mark. A claim +/// that loses that race stays spent — the cursors only ever advance, so +/// "spent" is what keeps them apart without a lock — and a refused carve after +/// a successful claim is terminal the same way. A claim the unclaimed middle +/// visibly cannot hold is refused before it is made, and spends nothing: a +/// task tries a window against each of its supplies in turn (a guest memory +/// relocating to a larger one, `reserve_from_top`), and a spent misfit would +/// take the rest of that supply from both claimants. +/// +/// **Why `SeqCst`.** Claim-then-check across two words is the +/// store-buffering shape: each side writes its own cursor, then reads the +/// other's. Acquire and release alone let both reads miss both writes, and +/// then both claims pass into the same bytes. Every cursor access here is +/// `SeqCst`, whose single total order puts one claim first, so the later +/// claim's check sees it. On x86-64 and AArch64 that costs nothing over +/// acquire-release (a locked RMW and a plain load; `LDAXR`/`STLXR` and +/// `LDAR`), and a claim is made once per granule, extent or window, never per +/// allocation. +#[cfg(any(target_os = "none", test))] +struct Cursors { + /// Bytes claimed off the bottom so far: the allocator's. + grown: core::sync::atomic::AtomicU64, + /// Bytes claimed off the top so far: [`reserve_from_top`]'s. + reserved: core::sync::atomic::AtomicU64, +} -/// The supply's total length, learned once from the kernel -/// (`lease_geometry`) the first time either cursor needs the far end. -/// Zero = not yet known. -#[cfg(target_os = "none")] -static HEAP_SUPPLY_LEN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +#[cfg(any(target_os = "none", test))] +impl Cursors { + const fn new() -> Self { + Self { + grown: core::sync::atomic::AtomicU64::new(0), + reserved: core::sync::atomic::AtomicU64::new(0), + } + } -/// Reserve one disjoint supply interval without admitting wraparound. -/// -/// A refused carve deliberately leaves its interval consumed (terminal supply -/// exhaustion), but arithmetic overflow is a refusal with no cursor mutation: -/// wrapping to an old offset would alias a live heap extent. + /// What neither cursor has claimed yet of a `total`-byte supply — a + /// snapshot for sizing a claim, which the claim itself then checks. + fn room(&self, total: u64) -> Option { + total + .checked_sub(self.reserved.load(Ordering::SeqCst))? + .checked_sub(self.grown.load(Ordering::SeqCst)) + } + + /// Claim `want` bytes off the bottom of a `total`-byte supply: the offset + /// of `[offset, offset + want)`, refused when the interval reaches into + /// the reserved top. + fn claim_bottom(&self, total: u64, want: u64) -> Option { + if self.room(total)? < want { + return None; + } + let offset = claim(&self.grown, want)?; + let limit = total.checked_sub(self.reserved.load(Ordering::SeqCst))?; + (offset.checked_add(want)? <= limit).then_some(offset) + } + + /// Claim `want` bytes off the top of a `total`-byte supply: the offset of + /// the window, refused when it reaches below the grow mark. + fn claim_top(&self, total: u64, want: u64) -> Option { + if self.room(total)? < want { + return None; + } + let end = claim(&self.reserved, want)?.checked_add(want)?; + let offset = total.checked_sub(end)?; + (offset >= self.grown.load(Ordering::SeqCst)).then_some(offset) + } +} + +/// Advance a monotone cursor by `want`: where the claim starts. A refused +/// carve deliberately leaves its interval consumed (terminal supply +/// exhaustion), but arithmetic overflow is a refusal with no cursor +/// mutation: wrapping to an old offset would alias a live heap extent. #[cfg(any(target_os = "none", test))] -fn reserve_supply(cursor: &core::sync::atomic::AtomicU64, want: u64) -> Option { +fn claim(cursor: &core::sync::atomic::AtomicU64, want: u64) -> Option { cursor - .try_update(Ordering::AcqRel, Ordering::Acquire, |offset| { + .try_update(Ordering::SeqCst, Ordering::SeqCst, |offset| { offset.checked_add(want) }) .ok() } -/// The bottom-up grow's check against the top-down reservations: the -/// interval `[offset, offset + want)` may not reach into the reserved -/// tail. `total == 0` means the far end is unknown, which can only be -/// the case while nothing has been reserved (a reservation needs the -/// total first), so an unknown total refuses nothing here — the kernel's -/// own supply bound still applies. -#[cfg(any(target_os = "none", test))] -fn grow_clears_reserved(offset: u64, want: u64, total: u64, reserved: u64) -> bool { - if total == 0 { - return true; +/// The launch supply's cursors (`role::HEAP@0`): the per-extent grow and the +/// heap window's launch granules claim its bottom, [`reserve_from_top`] its +/// top once every adopted supply has refused. +#[cfg(target_os = "none")] +static LAUNCH: Cursors = Cursors::new(); + +/// The launch supply's total length, learned once from the kernel +/// (`lease_geometry`) the first time a claim needs the far end. Zero = not +/// yet known. +#[cfg(target_os = "none")] +static HEAP_SUPPLY_LEN: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + +/// Carve a page-granular sub-lease of `len` bytes off the **top** of a +/// supply this task holds — memory the task owns for a window it maps +/// itself (a late-linked module's code arena and data span, `res/IMAGE.md` +/// §8; a Wasmtime engine's every compiled image and guest memory) — kept +/// apart from the allocator's extents by the two-cursor rule ([`Cursors`]). +/// Supplies adopted for windows alone ([`adopt_window_supply`]) are tried +/// first, then those adopted for the heap ([`adopt_supply`]), each in +/// adoption order; the launch supply `role::HEAP@0` last. A window thus +/// lands where its program reserved room for windows, and only then in a +/// heap reservation — which a program sized for its job's windows too, so +/// they land there rather than in the launch supply the job's heap growth +/// would otherwise have left alone. A supply too short for the window is +/// passed over intact ([`Cursors`]). The lease carries the ordinary carve rights, so it can +/// become a `JitArena` or an exact writable mapping. `None`: no supply, +/// no geometry, or every supply is spent or would meet its allocator +/// cursor. +#[cfg(target_os = "none")] +pub fn reserve_from_top(len: u64) -> Option { + let align = EXTENT_ALIGN as u64; + let want = len.max(1).checked_add(align - 1)? & !(align - 1); + window_supplies().find_map(|(_, supply, total, cursors)| { + let offset = cursors.claim_top(total, want)?; + crate::cap::verbs::carve_owned_lease_window(supply, offset, want) + }) +} + +/// The supplies a window this task maps itself draws on, in +/// [`reserve_from_top`]'s order — those adopted for windows alone, then those +/// adopted for the heap, each in adoption order, then the launch supply — +/// each with its slot, its length and its cursors. Lazy: the launch +/// supply's length is learned only once every adopted supply is passed over. +#[cfg(target_os = "none")] +fn window_supplies() -> impl Iterator< + Item = ( + CommitSupply, + kore_abi::session::SlotRef, + u64, + &'static Cursors, + ), +> { + let adopted = |heap: bool| { + ADOPTED.iter().enumerate().filter_map(move |(i, entry)| { + let slot = entry.slot.load(Ordering::Acquire).checked_sub(1)?; + (entry.heap.load(Ordering::Relaxed) == heap).then(|| { + ( + CommitSupply(i as u8), + kore_abi::session::SlotRef::from_wire(slot), + entry.len.load(Ordering::Relaxed), + &entry.cursors, + ) + }) + }) + }; + let launch = core::iter::once_with(|| { + let supply = crate::entry::geometry::heap_supply_slot()?; + Some((CommitSupply::LAUNCH, supply, supply_len(supply)?, &LAUNCH)) + }) + .flatten(); + adopted(false).chain(adopted(true)).chain(launch) +} + +/// **Where a reservation commit drew its pages** ([`commit_reserved`]): one +/// of this task's supplies — an adoption entry, or the launch supply — and +/// what [`release_reserved`] names to unmap them, since a release removes +/// only the named lessor's commits. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct CommitSupply(u8); + +impl CommitSupply { + /// How many supplies a commit may draw on: every adoption entry and the + /// launch supply — the width of a set of them, which [`index`](Self::index) + /// numbers densely. + pub(crate) const COUNT: usize = MAX_ADOPTED + 1; + + /// The launch supply (`role::HEAP@0`), numbered after every adoption + /// entry. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + const LAUNCH: Self = Self(MAX_ADOPTED as u8); + + /// This supply's number, `< COUNT`. + pub(crate) const fn index(self) -> usize { + self.0 as usize } - match (offset.checked_add(want), total.checked_sub(reserved)) { - (Some(end), Some(limit)) => end <= limit, - _ => false, + + /// The supply numbered `index`, for `index < COUNT`. + pub(crate) const fn from_index(index: usize) -> Option { + if index < Self::COUNT { + Some(Self(index as u8)) + } else { + None + } } } -/// The top-down reservation: claim `want` more bytes off the end of a -/// `total`-byte supply, refusing when the claim would cross the grow -/// cursor's current mark (`grown`). The claim is made first and checked -/// second — a refused claim stays spent, exactly like a refused grow — so -/// the two cursors can never both succeed into the same bytes. Returns -/// the offset of the reserved window. -#[cfg(any(target_os = "none", test))] -fn reserve_top( - reserved: &core::sync::atomic::AtomicU64, - grown: &core::sync::atomic::AtomicU64, - total: u64, - want: u64, -) -> Option { - let end = reserve_supply(reserved, want)?.checked_add(want)?; - if end > total { - return None; +// Every supply's number fits the byte it is carried in. +const _: () = assert!(CommitSupply::COUNT <= u8::MAX as usize + 1); + +/// **Commit `len` bytes at page `page` of this task's reservation window** +/// (`geom::RESERVE`, `res/WASM_PLATFORM.md` §12) — the body of +/// [`Reservation::commit`](crate::reserve::Reservation::commit), which owns +/// the range and has proven the span uncommitted. The bytes are carved off +/// the top of a supply this task holds, [`reserve_from_top`]'s supplies in +/// its order under the same two-cursor rule, and the carve is mapped at +/// exactly that page ([`map_view_reserved`]) in one session that holds +/// nothing afterwards: [`carve_and_map_in`]'s shape, so a memory grown in +/// hundreds of steps costs hundreds of sessions and no capability. `page` +/// counts [`RESERVE_PAGE_BYTES`] from the window's base; `len` is a whole +/// number of them. +/// +/// The answer names the supply the pages came from: what +/// [`release_reserved`] must name to unmap them. A map refused partway +/// leaves the prefix it published, so a refused commit releases its own +/// range from the supply it tried before it answers — exact, because the +/// caller proved no earlier commit lies there — and leaves nothing mapped; +/// the carved bytes stay spent, as every refused carve's do. +/// +/// **The task's own thread only**, the heap window's refill rule: the +/// supplies are named by slots of its arena, which on a spawned thread name +/// something else or nothing. Elsewhere this refuses before any claim. +/// +/// [`map_view_reserved`]: crate::session::SessionBuilder::map_view_reserved +/// [`RESERVE_PAGE_BYTES`]: kore_abi::session::RESERVE_PAGE_BYTES +#[cfg(target_os = "none")] +pub(crate) fn commit_reserved( + page: u32, + len: u64, +) -> Result { + use crate::reserve::CommitError; + use kore_abi::session::RESERVE_PAGE_BYTES; + + if len == 0 || len % RESERVE_PAGE_BYTES != 0 { + return Err(CommitError::OutOfRange); + } + if crate::entry::geometry::shard_index() != Some(0) { + return Err(CommitError::Refused); } - let offset = total - end; - (offset >= grown.load(Ordering::Acquire)).then_some(offset) + let pages = u32::try_from(len / RESERVE_PAGE_BYTES).map_err(|_| CommitError::OutOfRange)?; + let (id, supply, offset) = window_supplies() + .find_map(|(id, supply, total, cursors)| Some((id, supply, cursors.claim_top(total, len)?))) + .ok_or(CommitError::NoRoom)?; + if carve_and_map_at(supply, offset, len, Placement::ReservePage(page)).is_some() { + return Ok(id); + } + let _ = release_reserved(id, page, pages); + Err(CommitError::Refused) +} + +/// Off metal there is no window to commit into. +#[cfg(not(target_os = "none"))] +pub(crate) fn commit_reserved( + _page: u32, + _len: u64, +) -> Result { + Err(crate::reserve::CommitError::Refused) } -/// Carve a page-granular sub-lease of `len` bytes off the **top** of this -/// task's `role::HEAP@0` supply — memory the task owns for a window it -/// maps itself (a late-linked module's code arena and data span, -/// `res/IMAGE.md` §8), kept apart from the allocator's extents by the -/// two-cursor rule on [`HEAP_RESERVED`]. The lease carries the ordinary -/// carve rights, so it can become a `JitArena` or an exact writable -/// mapping. `None`: no supply, no geometry, the supply is spent, or the -/// reservation would meet the allocator's cursor. +/// **Release `[page, page + pages)` of this task's reservation window**: every +/// commit [`commit_reserved`] drew from `supply` inside the range is unmapped +/// (`DERIVE_KIND_RESERVE_RELEASE`), in one session. Commits from another +/// supply stay — a reservation grown from several releases once per supply +/// it drew on. `false` off the task's own thread ([`commit_reserved`]'s +/// rule), when the supply is gone, the range leaves the window, or a commit +/// straddles the range's edge (and then nothing was unmapped). #[cfg(target_os = "none")] -pub fn reserve_from_top(len: u64) -> Option { - let align = EXTENT_ALIGN as u64; - let want = len.max(1).checked_add(align - 1)? & !(align - 1); - let supply = crate::entry::geometry::heap_supply_slot()?; - let total = supply_len(supply)?; - let offset = reserve_top(&HEAP_RESERVED, &HEAP_GROWN, total, want)?; - crate::cap::verbs::carve_owned_lease_window(supply, offset, want) +pub(crate) fn release_reserved(supply: CommitSupply, page: u32, pages: u32) -> bool { + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + + if crate::entry::geometry::shard_index() != Some(0) { + return false; + } + let slot = if supply == CommitSupply::LAUNCH { + crate::entry::geometry::heap_supply_slot() + } else { + ADOPTED + .get(supply.index()) + .and_then(|entry| entry.slot.load(Ordering::Acquire).checked_sub(1)) + .map(kore_abi::session::SlotRef::from_wire) + }; + let Some(slot) = slot else { + return false; + }; + let mut b = SessionBuilder::new(); + b.release_reserved(slot, page, pages).is_ok() + && TrapTransport::new() + .commit_now(&b) + .is_ok_and(|done| done.status() == kore_abi::session::Status::Ok) +} + +/// Off metal there is nothing to release. +#[cfg(not(target_os = "none"))] +pub(crate) fn release_reserved(_supply: CommitSupply, _page: u32, _pages: u32) -> bool { + false } /// Off metal there is no supply to carve. @@ -248,31 +449,103 @@ pub fn reserve_from_top(_len: u64) -> Option { None } -/// A supply a program **adopted** into its heap ([`adopt_supply`]): a root -/// lease it obtained itself — an mm supply minted on its own per-launch face -/// — drawn on before the launch supply. +/// **The longest window a supply held for windows could take now**: the +/// most room any one supply adopted for windows alone +/// ([`adopt_window_supply`]) has left between its two cursors, in whole +/// extents. A window is one carve, so it must fit in one supply — the sum +/// of the rooms is no answer. [`reserve_from_top`] and a reservation's +/// commit try those supplies before any other, so a window no longer than +/// this lands in one, and the supplies the heap draws on keep the room +/// their program reserved for work that cannot wait on mm (a frame's +/// document mutations). A snapshot for deciding whether to reserve a +/// window supply before a window is needed (a guest memory about to grow); +/// the carve itself still checks. +#[cfg(target_os = "none")] +pub fn window_supply_room() -> u64 { + let rooms = ADOPTED.iter().filter_map(|entry| { + entry.slot.load(Ordering::Acquire).checked_sub(1)?; + // Published with the slot, as the length is. + if entry.heap.load(Ordering::Relaxed) { + return None; + } + entry.cursors.room(entry.len.load(Ordering::Relaxed)) + }); + whole_extents(rooms.max().unwrap_or(0)) +} + +/// Off metal there is no supply. +#[cfg(not(target_os = "none"))] +pub fn window_supply_room() -> u64 { + 0 +} + +/// **The room heap granules can still draw on**: what the heap can still +/// grow by off the supplies this task adopted for it ([`adopt_supply`]), +/// without a new one — their room in whole 2 MiB granules, and the heap +/// window's free bytes a granule placed next extends. Any allocation, or run +/// of allocations, the heap must map fresh up to this size finds room: one +/// longer than any one supply's room is assembled from several in the heap +/// window. A snapshot for deciding whether to reserve more before a burst of +/// synchronous allocation needs it (a frame's document mutations, a photo's +/// decode — `kore-svc-mm`'s `ProcessMm::reserve_heap`), which cannot wait on +/// mm; the carve itself still checks. A lower bound, never an over-promise: +/// the launch supply is not counted — [`reserve_from_top`]'s windows share +/// it, and the heap never assembles a run from it — nor are supplies held +/// for one window ([`adopt_window_supply`]), nor free blocks the heap +/// already holds, which the burst may reuse. +#[cfg(target_os = "none")] +pub fn heap_room() -> u64 { + window::room() +} + +/// Off metal there is no supply. +#[cfg(not(target_os = "none"))] +pub fn heap_room() -> u64 { + 0 +} + +/// `bytes` down to whole extents: room a granule can use. +#[cfg(target_os = "none")] +const fn whole_extents(bytes: u64) -> u64 { + bytes & !(EXTENT_ALIGN as u64 - 1) +} + +/// A supply a program **adopted** ([`adopt_supply`], [`adopt_window_supply`]): +/// a root lease it obtained itself — an mm supply minted on its own +/// per-launch face — drawn on before the launch supply: heap granules from +/// its bottom (unless it was adopted for windows alone), [`reserve_from_top`]'s +/// windows from its top. #[cfg(target_os = "none")] struct Adopted { /// The lease's slot index + 1; zero marks a vacant entry. slot: core::sync::atomic::AtomicU32, /// The lease's length. len: core::sync::atomic::AtomicU64, - /// Bytes carved off it so far (a monotone cursor, like [`HEAP_GROWN`]). - grown: core::sync::atomic::AtomicU64, + /// Whether heap granules may draw on it: false for a supply obtained for + /// one window, which a granule mapped first would leave too short. + heap: core::sync::atomic::AtomicBool, + /// Its two-cursor rule, the launch supply's own ([`Cursors`]). + cursors: Cursors, } -/// Adopted supplies one task may hold — design-bounded: a program adopts one -/// reservation per large synchronous job it prepares for, not per -/// allocation. -#[cfg(target_os = "none")] -const MAX_ADOPTED: usize = 8; +/// Adopted supplies one task may hold — design-bounded: a reservation is +/// adopted as leases of `kore-svc-mm`'s `HEAP_RESERVATION_LEASE_BYTES` +/// (128 MiB) down to its `HEAP_RESERVATION_LEASE_MIN_BYTES` (16 MiB) when +/// the class is fragmented, so sixty-four hold the `wasm` command's 1 GiB +/// ceiling at the shortest lease; a program adopts one reservation per +/// large synchronous job it prepares for, and one window per guest memory +/// relocation that finds no supply with room (CPython's 104 MiB workload +/// relocates five times), never one per allocation. Two KiB of +/// every image's `.bss`. +pub const MAX_ADOPTED: usize = 64; #[cfg(target_os = "none")] static ADOPTED: [Adopted; MAX_ADOPTED] = [const { Adopted { slot: core::sync::atomic::AtomicU32::new(0), len: core::sync::atomic::AtomicU64::new(0), - grown: core::sync::atomic::AtomicU64::new(0), + heap: core::sync::atomic::AtomicBool::new(false), + cursors: Cursors::new(), } }; MAX_ADOPTED]; @@ -292,6 +565,26 @@ static ADOPTED: [Adopted; MAX_ADOPTED] = [const { /// is unreadable, or when every adoption entry is taken. #[cfg(target_os = "none")] pub fn adopt_supply(lease: crate::cap::StashedLease) -> Result<(), crate::cap::StashedLease> { + adopt(lease, true) +} + +/// **Hold `lease` for one window the task will map itself** — a code image, +/// a guest memory ([`reserve_from_top`]) — that no supply it holds has room +/// for. Adopted as [`adopt_supply`] adopts, refused for the same reasons, +/// but heap granules never draw on it: a granule mapped into its bottom +/// before the window is carved would leave it too short for the window it +/// was obtained for. What the window leaves stays for later windows. +#[cfg(target_os = "none")] +pub fn adopt_window_supply( + lease: crate::cap::StashedLease, +) -> Result<(), crate::cap::StashedLease> { + adopt(lease, false) +} + +/// [`adopt_supply`] and [`adopt_window_supply`]: `heap` says whether heap +/// granules may draw on the lease. +#[cfg(target_os = "none")] +fn adopt(lease: crate::cap::StashedLease, heap: bool) -> Result<(), crate::cap::StashedLease> { let has_window = crate::entry::thread_context() .and_then(|rt| rt.geom()) .and_then(|g| g.window(kore_abi::task::geom::HEAP)) @@ -315,8 +608,11 @@ pub fn adopt_supply(lease: crate::cap::StashedLease) -> Result<(), crate::cap::S else { return Err(lease); }; + // Its cursors are still at zero: an entry is never vacated, and both + // claimants pass over a vacant one. The slot's release publishes the + // length and the heap flag with it. entry.len.store(len, Ordering::Relaxed); - entry.grown.store(0, Ordering::Relaxed); + entry.heap.store(heap, Ordering::Relaxed); entry.slot.store(slot.index() + 1, Ordering::Release); // Held for the task's life: its granules back live heap memory. core::mem::forget(lease); @@ -329,6 +625,14 @@ pub fn adopt_supply(lease: crate::cap::StashedLease) -> Result<(), crate::cap::S Err(lease) } +/// Off metal there is no window to map. +#[cfg(not(target_os = "none"))] +pub fn adopt_window_supply( + lease: crate::cap::StashedLease, +) -> Result<(), crate::cap::StashedLease> { + Err(lease) +} + /// The supply's length, cached after one kernel query. #[cfg(target_os = "none")] fn supply_len(supply: kore_abi::session::SlotRef) -> Option { @@ -360,17 +664,13 @@ fn grow_from_supply(min_bytes: usize) -> Option> { window::Grow::NoWindow => {} } let supply = crate::entry::geometry::heap_supply_slot()?; - let offset = reserve_supply(&HEAP_GROWN, want)?; - // The reserved tail is the other cursor's; a grow that would enter it + // The far end is learned before the claim, not read from the cache: a + // grow that took an unlearned length as "nothing reserved" could pass + // its check against a reservation another thread had just claimed. + let total = supply_len(supply)?; + // The reserved top is the other cursor's; a grow that would enter it // refuses with its interval spent (the allocator surfaces AllocError). - if !grow_clears_reserved( - offset, - want, - HEAP_SUPPLY_LEN.load(Ordering::Acquire), - HEAP_RESERVED.load(Ordering::Acquire), - ) { - return None; - } + let offset = LAUNCH.claim_bottom(total, want)?; carve_and_map(supply, offset, want) } @@ -407,30 +707,45 @@ fn grow_from_supply(min_bytes: usize) -> Option> { /// supply's other consumers, and a large heap still costs a handful of /// sessions. /// +/// **One extent, several supplies.** A reservation arrives as leases of at +/// most 128 MiB (`kore-svc-mm`'s `HEAP_RESERVATION_LEASE_BYTES`), each +/// claimed from the bottom as the heap grows, so its room ends up spread +/// over the tails of several. An extent no one supply holds — a 100 MP +/// photo's 400 MB of pixels, or a 65 MB one once the leases are part spent +/// — is assembled in the window instead: each **part** is the whole room of +/// the adopted supply that has the most, in whole granules, which the +/// kernel places end to end ([`Window::room`] says when), until one supply +/// holds what is still missing. A run the free interval and the adopted +/// supplies cannot hold together is refused before any part is mapped. +/// +/// The policy is [`Window`]'s, written once over the [`Granules`] seam: the +/// task's supplies and the kernel on metal, a simulated kernel in the host +/// tests. +/// /// [`ViewMapWindow::Heap`]: kore_abi::session::ViewMapWindow::Heap -#[cfg(target_os = "none")] +#[cfg(any(target_os = "none", test))] mod window { use core::ptr::NonNull; use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; + use super::Cursors; + /// The doubling's first step: 2 MiB — the frame class the physical - /// allocator reserves for huge leaves (`res/ACCEL_MEMORY_SCALE.md` §1). + /// allocator reserves for huge leaves (`res/ACCEL_MEMORY_SCALE.md` §1), + /// and the boundary the kernel places a view of at least that on. const GRANULE_MIN: u64 = 2 << 20; - /// Where doubling stops: five doublings reach it. + /// Where doubling stops: four doublings reach it. const GRANULE_MAX: u64 = 32 << 20; + /// Doublings from [`GRANULE_MIN`] to [`GRANULE_MAX`]. + const DOUBLINGS: u32 = GRANULE_MAX.ilog2() - GRANULE_MIN.ilog2(); /// A refill may take at most this fraction of the supply that remains. const REFILL_SHARE: u64 = 4; /// Interval offsets count pages of this size. const PAGE: u64 = 4096; - - /// The window base, published before any interval is non-empty (zero = - /// not learned yet). - static BASE: AtomicU64 = AtomicU64::new(0); - /// The free interval: bits 0..32 its start, bits 32..64 its end, both in - /// pages from [`BASE`]. - static FREE: AtomicU64 = AtomicU64::new(0); - /// Granules mapped so far: the doubling's exponent. - static GRANULES: AtomicU32 = AtomicU32::new(0); + /// The most refills one grow makes: a part off each adopted supply, the + /// granule that completes the run, and one more for a worker's bump that + /// raced it. + const REFILLS: usize = super::MAX_ADOPTED + 2; const fn pack(at: u64, end: u64) -> u64 { (end << 32) | at @@ -439,71 +754,195 @@ mod window { (word & 0xffff_ffff, word >> 32) } - pub(super) enum Grow { - Extent(NonNull<[u8]>), - /// The window exists and could not supply `want`: the heap surfaces - /// `AllocError`, exactly as the per-extent path would. - Refused, - /// Not from the window: this thread cannot refill it, or the task has - /// none. Use the per-extent path. - NoWindow, + /// What the window takes granules from, and how one is placed: the + /// task's supplies and the kernel on metal ([`Supplies`]), a simulated + /// kernel in the host tests. Which supply, how much and when the free + /// interval extends are [`Window`]'s alone. + pub(super) trait Granules { + /// A supply, as [`map`](Self::map) names it. + type Supply: Copy; + /// The adopted supplies heap granules may draw on, in adoption + /// order: each one, its length and its cursors. + fn adopted(&self) -> impl Iterator; + /// The launch supply (`role::HEAP@0`), when the task has one. + fn launch(&self) -> Option<(Self::Supply, u64, &Cursors)>; + /// Map the claimed `[offset, offset + size)` of `supply` in the + /// window: the address it landed at. + fn map(&self, supply: Self::Supply, offset: u64, size: u64) -> Option; } - pub(super) fn grow(want: u64) -> Grow { - if let Some(extent) = bump(want) { - return Grow::Extent(extent); + /// The window's state: its base and the one free interval. + pub(super) struct Window { + /// The window base, published before any interval is non-empty (zero + /// = not learned yet). + base: AtomicU64, + /// The free interval: bits 0..32 its start, bits 32..64 its end, both + /// in pages from `base`. + free: AtomicU64, + /// Granules mapped so far: the doubling's exponent. + granules: AtomicU32, + } + + impl Window { + pub(super) const fn new() -> Self { + Self { + base: AtomicU64::new(0), + free: AtomicU64::new(0), + granules: AtomicU32::new(0), + } } - if crate::entry::geometry::shard_index() != Some(0) { - return Grow::NoWindow; + + /// The window base, zero until learned. + pub(super) fn base(&self) -> u64 { + self.base.load(Ordering::Acquire) } - if BASE.load(Ordering::Relaxed) == 0 { - let Some((base, _)) = crate::entry::thread_context() - .and_then(|rt| rt.geom()) - .and_then(|g| g.window(kore_abi::task::geom::HEAP)) - else { - return Grow::NoWindow; - }; - BASE.store(base, Ordering::Release); - } - // A worker may bump the fresh bytes away between the refill and this - // thread's own bump; one more refill answers that race, and a second - // loss is reported rather than chased. - for _ in 0..2 { - if refill(want).is_none() { - return Grow::Refused; + + /// Publish the window base, before the first refill. + pub(super) fn learn_base(&self, base: u64) { + self.base.store(base, Ordering::Release); + } + + /// Take `want` bytes off the front of the free interval, lock-free: + /// the compare-and-swap retries only when another thread changed the + /// interval first. + pub(super) fn bump(&self, want: u64) -> Option> { + let pages = want / PAGE; + let mut word = self.free.load(Ordering::Acquire); + loop { + let (at, end) = unpack(word); + let next = at.checked_add(pages).filter(|&next| next <= end)?; + match self.free.compare_exchange_weak( + word, + pack(next, end), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + let va = self.base().checked_add(at * PAGE)?; + return Some(NonNull::slice_from_raw_parts( + NonNull::new(va as *mut u8)?, + usize::try_from(want).ok()?, + )); + } + Err(actual) => word = actual, + } } - if let Some(extent) = bump(want) { - return Grow::Extent(extent); + } + + /// Take `want` bytes, refilling off `supplies` until the free + /// interval holds them. `None` when the supplies cannot: a run is + /// assembled from parts only when they hold it together, and a + /// refill the kernel refuses ends the grow. + pub(super) fn fill(&self, supplies: &G, want: u64) -> Option> { + // A worker may bump the fresh bytes away between a refill and + // this thread's own bump; one more refill answers that race, and + // a second loss is reported rather than chased. + for _ in 0..REFILLS { + if let Some(extent) = self.bump(want) { + return Some(extent); + } + self.refill(supplies, want)?; } + self.bump(want) } - Grow::Refused - } - - /// Take `want` bytes off the front of the free interval, lock-free: the - /// compare-and-swap retries only when another thread changed the - /// interval first. - fn bump(want: u64) -> Option> { - let pages = want / PAGE; - let mut word = FREE.load(Ordering::Acquire); - loop { - let (at, end) = unpack(word); - let next = at.checked_add(pages).filter(|&next| next <= end)?; - match FREE.compare_exchange_weak( - word, - pack(next, end), - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => { - let va = BASE.load(Ordering::Acquire).checked_add(at * PAGE)?; - return Some(NonNull::slice_from_raw_parts( - NonNull::new(va as *mut u8)?, - usize::try_from(want).ok()?, - )); + + /// **What the heap can still map** off the adopted supplies without + /// a new one: every adopted supply's room in whole granules, and the + /// free interval when a granule placed next extends it. Any extent, + /// or run of extents, up to this maps — one longer than any one + /// supply's room in parts ([`fill`](Self::fill)). + /// + /// The kernel places a granule of [`GRANULE_MIN`] or more on a + /// [`GRANULE_MIN`] boundary, so one lands at the interval's end only + /// when that end is on one; a free interval ending elsewhere (a + /// page-grain granule took a supply's last pages) serves the extents + /// it holds alone, and a longer one replaces it. The launch supply is + /// left out: [`reserve_from_top`](super::reserve_from_top)'s windows + /// share it, and a refill takes at most its share. + pub(super) fn room(&self, supplies: &G) -> u64 { + supplies + .adopted() + .filter_map(|(_, total, cursors)| cursors.room(total)) + .fold(self.extendable(), |room, part| { + room.saturating_add(part & !(GRANULE_MIN - 1)) + }) + } + + /// The free interval's bytes a granule placed next extends: all of + /// them when its end lies on a [`GRANULE_MIN`] boundary, else none. + fn extendable(&self) -> u64 { + let (at, end) = unpack(self.free.load(Ordering::Acquire)); + let on_boundary = self + .base() + .checked_add(end * PAGE) + .is_some_and(|va| va % GRANULE_MIN == 0); + if on_boundary { (end - at) * PAGE } else { 0 } + } + + /// Carve the next granule off a supply and have it mapped in the + /// window, then publish it as free: the whole of what is missing from + /// the first supply that holds it, else a part of the run. + fn refill(&self, supplies: &G, want: u64) -> Option<()> { + // What the granule must bring: what the free interval lacks. The + // kernel places a granule under `GRANULE_MIN` at page grain, at + // the interval's very end, so the whole free tail counts; one it + // places on a boundary past a gap replaces the interval, and the + // next refill counts from it. + let (at, end) = unpack(self.free.load(Ordering::Acquire)); + let missing = want.saturating_sub((end - at) * PAGE); + let doubling = GRANULE_MIN << self.granules.load(Ordering::Relaxed).min(DOUBLINGS); + // Adopted supplies first: they were obtained for the heap alone, + // while the launch supply is shared with `reserve_from_top`'s code + // windows (a Wasmtime engine maps every compiled module there). + let (supply, offset, size) = adopted_granule(supplies, missing, doubling) + .or_else(|| launch_granule(supplies, missing, doubling)) + .or_else(|| self.part(supplies, want))?; + let va = supplies.map(supply, offset, size)?; + self.granules.fetch_add(1, Ordering::Relaxed); + let start = va.checked_sub(self.base())? / PAGE; + let stop = start + .checked_add(size / PAGE) + .filter(|&stop| stop <= u32::MAX as u64)?; + // Extend the interval when the granule lands at its end; otherwise + // the new granule replaces it and the old tail is left unused. + let mut word = self.free.load(Ordering::Acquire); + loop { + let (at, end) = unpack(word); + let next = if start == end { + pack(at, stop) + } else { + pack(start, stop) + }; + match self + .free + .compare_exchange_weak(word, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return Some(()), + Err(actual) => word = actual, } - Err(actual) => word = actual, } } + + /// One part of a run no one supply holds whole: all the room of the + /// adopted supply that has the most (the first, of equals), in whole + /// granules — reserved, and only when the free interval and the + /// adopted supplies hold `want` together ([`room`](Self::room)), so a + /// run that cannot be assembled spends nothing. The launch supply is + /// never taken whole for a part. + fn part(&self, supplies: &G, want: u64) -> Option<(G::Supply, u64, u64)> { + if self.room(supplies) < want { + return None; + } + let (supply, total, cursors, size) = supplies + .adopted() + .filter_map(|(supply, total, cursors)| { + let size = cursors.room(total)? & !(GRANULE_MIN - 1); + (size != 0).then_some((supply, total, cursors, size)) + }) + .reduce(|best, next| if next.3 > best.3 { next } else { best })?; + let offset = cursors.claim_bottom(total, size)?; + Some((supply, offset, size)) + } } /// How big the next granule is: what is missing, rounded up to @@ -519,15 +958,13 @@ mod window { /// The next granule off the launch supply (`role::HEAP@0`), reserved: /// `(supply, offset, size)`. - fn launch_granule( + fn launch_granule( + supplies: &G, missing: u64, doubling: u64, - ) -> Option<(kore_abi::session::SlotRef, u64, u64)> { - let supply = crate::entry::geometry::heap_supply_slot()?; - let total = super::supply_len(supply)?; - let room = total - .checked_sub(super::HEAP_RESERVED.load(Ordering::Acquire))? - .checked_sub(super::HEAP_GROWN.load(Ordering::Acquire))?; + ) -> Option<(G::Supply, u64, u64)> { + let (supply, total, cursors) = supplies.launch()?; + let room = cursors.room(total)?; // The kernel places a view of at least 2 MiB on a 2 MiB boundary (the // huge-leaf alignment), so whole multiples of it land end to end and // the interval extends. That rounding is taken only inside the share: @@ -535,75 +972,328 @@ mod window { // supply on the first refill and starve `reserve_from_top` (Lua's // native `require` then has no JIT arena). Page grain is the rest. let size = granule_size(missing, doubling, room, room / REFILL_SHARE)?; - let offset = super::reserve_supply(&super::HEAP_GROWN, size)?; - super::grow_clears_reserved( - offset, - size, - total, - super::HEAP_RESERVED.load(Ordering::Acquire), - ) - .then_some((supply, offset, size)) + let offset = cursors.claim_bottom(total, size)?; + Some((supply, offset, size)) } /// The next granule off an adopted supply ([`super::adopt_supply`]), - /// reserved. Nothing else draws on an adopted lease, so its share is all - /// that remains of it. - fn adopted_granule( + /// reserved, from the first whose unclaimed middle holds it — a supply + /// adopted for one window ([`super::adopt_window_supply`]) is passed + /// over. Its top serves [`super::reserve_from_top`]'s windows, but both + /// claimants were sized into the lease by the program that obtained it + /// for this job, so a granule may take what neither has claimed — the + /// doubling still bounds it — rather than the launch supply's quarter. + fn adopted_granule( + supplies: &G, missing: u64, doubling: u64, - ) -> Option<(kore_abi::session::SlotRef, u64, u64)> { - super::ADOPTED.iter().find_map(|entry| { - let slot = entry.slot.load(Ordering::Acquire).checked_sub(1)?; - let room = entry - .len - .load(Ordering::Relaxed) - .checked_sub(entry.grown.load(Ordering::Relaxed))?; + ) -> Option<(G::Supply, u64, u64)> { + supplies.adopted().find_map(|(supply, total, cursors)| { + let room = cursors.room(total)?; let size = granule_size(missing, doubling, room, room)?; - let offset = super::reserve_supply(&entry.grown, size)?; - Some((kore_abi::session::SlotRef::from_wire(slot), offset, size)) + let offset = cursors.claim_bottom(total, size)?; + Some((supply, offset, size)) }) } - /// Carve the next granule off the supply and have the kernel map it in - /// the heap window, then publish it as free. - fn refill(want: u64) -> Option<()> { - let (at, end) = unpack(FREE.load(Ordering::Acquire)); - let missing = want.saturating_sub((end - at) * PAGE); - let doubling = GRANULE_MIN - .checked_shl(GRANULES.load(Ordering::Relaxed)) - .unwrap_or(GRANULE_MAX) - .min(GRANULE_MAX); - // Adopted supplies first: they were obtained for the heap alone, - // while the launch supply is shared with `reserve_from_top`'s code - // windows (a Wasmtime engine maps every compiled module there). - let (supply, offset, size) = - adopted_granule(missing, doubling).or_else(|| launch_granule(missing, doubling))?; - let granule = super::carve_and_map_in( - supply, - offset, - size, - Some(kore_abi::session::ViewMapWindow::Heap), - )?; - GRANULES.fetch_add(1, Ordering::Relaxed); - let start = - (granule.as_ptr() as *mut u8 as u64).checked_sub(BASE.load(Ordering::Relaxed))? / PAGE; - let stop = start - .checked_add(size / PAGE) - .filter(|&stop| stop <= u32::MAX as u64)?; - // Extend the interval when the granule lands at its end; otherwise - // the new granule replaces it and the old tail is left unused. - let mut word = FREE.load(Ordering::Acquire); - loop { - let (at, end) = unpack(word); - let next = if start == end { - pack(at, stop) - } else { - pack(start, stop) + /// The task's own window. + #[cfg(target_os = "none")] + static WINDOW: Window = Window::new(); + + /// The task's supplies and the kernel's heap window: the [`Granules`] + /// [`WINDOW`] grows over on metal. + #[cfg(target_os = "none")] + struct Supplies; + + #[cfg(target_os = "none")] + impl Granules for Supplies { + type Supply = kore_abi::session::SlotRef; + + fn adopted(&self) -> impl Iterator { + super::ADOPTED.iter().filter_map(|entry| { + let slot = entry.slot.load(Ordering::Acquire).checked_sub(1)?; + entry.heap.load(Ordering::Relaxed).then(|| { + ( + kore_abi::session::SlotRef::from_wire(slot), + entry.len.load(Ordering::Relaxed), + &entry.cursors, + ) + }) + }) + } + + fn launch(&self) -> Option<(Self::Supply, u64, &Cursors)> { + let supply = crate::entry::geometry::heap_supply_slot()?; + Some((supply, super::supply_len(supply)?, &super::LAUNCH)) + } + + fn map(&self, supply: Self::Supply, offset: u64, size: u64) -> Option { + let granule = super::carve_and_map_in( + supply, + offset, + size, + Some(kore_abi::session::ViewMapWindow::Heap), + )?; + Some(granule.cast::().as_ptr() as u64) + } + } + + #[cfg(target_os = "none")] + pub(super) enum Grow { + Extent(NonNull<[u8]>), + /// The window exists and could not supply `want`: the heap surfaces + /// `AllocError`, exactly as the per-extent path would. + Refused, + /// Not from the window: this thread cannot refill it, or the task has + /// none. Use the per-extent path. + NoWindow, + } + + #[cfg(target_os = "none")] + pub(super) fn grow(want: u64) -> Grow { + if let Some(extent) = WINDOW.bump(want) { + return Grow::Extent(extent); + } + if crate::entry::geometry::shard_index() != Some(0) { + return Grow::NoWindow; + } + if WINDOW.base() == 0 { + let Some((base, _)) = crate::entry::thread_context() + .and_then(|rt| rt.geom()) + .and_then(|g| g.window(kore_abi::task::geom::HEAP)) + else { + return Grow::NoWindow; }; - match FREE.compare_exchange_weak(word, next, Ordering::AcqRel, Ordering::Acquire) { - Ok(_) => return Some(()), - Err(actual) => word = actual, + WINDOW.learn_base(base); + } + match WINDOW.fill(&Supplies, want) { + Some(extent) => Grow::Extent(extent), + None => Grow::Refused, + } + } + + /// [`Window::room`] of the task's own window. + #[cfg(target_os = "none")] + pub(super) fn room() -> u64 { + WINDOW.room(&Supplies) + } + + #[cfg(test)] + mod tests { + use super::*; + use core::cell::RefCell; + use std::vec::Vec; + + const KIB: u64 = 1 << 10; + const MIB: u64 = 1 << 20; + /// Where the simulated window starts: on a 2 MiB boundary, as the + /// founder stamps it. + const BASE: u64 = 1 << 40; + /// The launch supply's handle in the simulation. + const LAUNCH: usize = usize::MAX; + + /// Leases and a kernel heap cursor, placing as the session engine + /// does: a view of at least 2 MiB over a 2 MiB-aligned backing on a + /// 2 MiB boundary, anything else at the next page. A lease's backing + /// starts on a 2 MiB frame (its class), so the backing is aligned + /// when the carve's offset is. + struct Sim { + adopted: Vec<(u64, Cursors)>, + launch: Option<(u64, Cursors)>, + cursor: RefCell, + /// Every granule mapped: `(supply, offset, size, va)`. + maps: RefCell>, + } + + impl Sim { + fn leases(lengths: &[u64]) -> Self { + Self { + adopted: lengths.iter().map(|&len| (len, Cursors::new())).collect(), + launch: None, + cursor: RefCell::new(BASE), + maps: RefCell::new(Vec::new()), + } + } + + fn rooms(&self) -> Vec { + self.adopted + .iter() + .map(|(total, cursors)| cursors.room(*total).unwrap()) + .collect() + } + + fn sizes(&self) -> Vec { + self.maps.borrow().iter().map(|&(_, _, size, _)| size).collect() + } + } + + impl Granules for Sim { + type Supply = usize; + + fn adopted(&self) -> impl Iterator { + self.adopted + .iter() + .enumerate() + .map(|(index, (total, cursors))| (index, *total, cursors)) + } + + fn launch(&self) -> Option<(usize, u64, &Cursors)> { + self.launch.as_ref().map(|(total, cursors)| (LAUNCH, *total, cursors)) + } + + fn map(&self, supply: usize, offset: u64, size: u64) -> Option { + let align = if offset % GRANULE_MIN == 0 && size >= GRANULE_MIN { + GRANULE_MIN + } else { + PAGE + }; + let mut cursor = self.cursor.borrow_mut(); + let va = cursor.next_multiple_of(align); + *cursor = va + size; + self.maps.borrow_mut().push((supply, offset, size, va)); + Some(va) + } + } + + fn window() -> Window { + let window = Window::new(); + window.learn_base(BASE); + window + } + + fn at(extent: NonNull<[u8]>) -> (u64, u64) { + (extent.cast::().as_ptr() as u64, extent.len() as u64) + } + + /// The photo that could not be saved: 64 MiB of pixels against two + /// leases with 28 and 56 MiB left. Neither holds it, together they + /// do, and the run is placed whole: the longer lease's room, then + /// what is still missing off the first lease that holds it, end to + /// end. + #[test] + fn a_run_longer_than_any_one_supply_is_assembled_from_several() { + let sim = Sim::leases(&[128 * MIB, 56 * MIB]); + assert_eq!(sim.adopted[0].1.claim_bottom(128 * MIB, 100 * MIB), Some(0)); + let window = window(); + assert_eq!(window.room(&sim), 84 * MIB); + + let extent = window.fill(&sim, 64 * MIB).expect("assembled"); + assert_eq!(at(extent), (BASE, 64 * MIB)); + assert_eq!( + *sim.maps.borrow(), + [(1, 0, 56 * MIB, BASE), (0, 100 * MIB, 8 * MIB, BASE + 56 * MIB)], + "the longest room whole, then the rest from the first lease holding it" + ); + // Eight of the first lease's 28 MiB are the run's tail; the doubling + // took nothing more, and none of it went astray. + assert_eq!(sim.rooms(), [20 * MIB, 0]); + assert_eq!(window.room(&sim), 20 * MIB); + } + + /// A run the supplies cannot hold together is refused before a + /// granule is mapped: their room stays whole for what does fit. + #[test] + fn a_run_the_supplies_cannot_hold_spends_nothing() { + let sim = Sim::leases(&[128 * MIB, 56 * MIB]); + assert_eq!(sim.adopted[0].1.claim_bottom(128 * MIB, 100 * MIB), Some(0)); + let window = window(); + assert!(window.fill(&sim, 84 * MIB + PAGE).is_none()); + assert!(sim.maps.borrow().is_empty()); + assert_eq!(sim.rooms(), [28 * MIB, 56 * MIB]); + // What the room says fits does. + assert_eq!(at(window.fill(&sim, 84 * MIB).unwrap()), (BASE, 84 * MIB)); + assert_eq!(window.room(&sim), 0); + } + + /// A free interval that ends off a 2 MiB boundary is not counted: + /// the next granule of 2 MiB or more lands past a gap and replaces + /// it. What the room says still maps, the stranded tail aside. + #[test] + fn the_room_counts_the_interval_only_where_a_granule_extends_it() { + // Three of four MiB heap's, one taken off its top for a window. + let sim = Sim::leases(&[4 * MIB, 8 * MIB]); + assert_eq!(sim.adopted[0].1.claim_top(4 * MIB, MIB), Some(3 * MIB)); + // A busy second lease, so the first is drawn on alone. + assert_eq!(sim.adopted[1].1.claim_bottom(8 * MIB, 8 * MIB), Some(0)); + let window = window(); + assert_eq!(at(window.fill(&sim, MIB).unwrap()), (BASE, MIB)); + // Its last MiB takes page grain: the interval ends at 3 MiB. + let tail = 3 * MIB / 2; + assert_eq!(at(window.fill(&sim, tail).unwrap()), (BASE + MIB, tail)); + assert_eq!(sim.sizes(), [2 * MIB, MIB]); + assert_eq!(window.room(&sim), 0, "half a MiB free, but off the boundary"); + + // A fresh lease of 8 MiB: the room is its 8, not 8½. + let mut sim = sim; + sim.adopted.push((8 * MIB, Cursors::new())); + assert_eq!(window.room(&sim), 8 * MIB); + assert!(window.fill(&sim, 8 * MIB + PAGE).is_none()); + let extent = window.fill(&sim, 8 * MIB).expect("the room maps"); + assert_eq!(at(extent), (BASE + 4 * MIB, 8 * MIB), "past the gap"); + } + + /// A founded service's heap: a 2 MiB launch supply alone. Its quarter + /// is under a granule, so every granule is of page grain, which the + /// kernel places at the interval's very end, and the free interval + /// ends off a 2 MiB boundary from the first refill on. The free tail + /// still counts toward what the next granule must bring, and the + /// supply is spent to its last page. Counting only what a 2 MiB + /// granule would extend asked each refill for the whole extent: the + /// hypervisor's MicroPython package, some 1.8 MiB of extents, no + /// longer fit once anything was allocated before it, and every + /// boot-userland-gate lane with hardware virtualization went red on + /// 2570cd4. + #[test] + fn a_service_supply_is_spent_to_its_last_page() { + let mut sim = Sim::leases(&[]); + sim.launch = Some((2 * MIB, Cursors::new())); + let window = window(); + assert_eq!(at(window.fill(&sim, 64 * KIB).unwrap()), (BASE, 64 * KIB)); + assert_eq!(sim.sizes(), [512 * KIB], "the supply's quarter, in pages"); + // 448 KiB free, ending off the boundary; 1½ MiB left to claim. + let rest = 2 * MIB - 64 * KIB; + let extent = window.fill(&sim, rest).expect("the tail and the supply's rest hold it"); + assert_eq!(at(extent), (BASE + 64 * KIB, rest), "one run, the tail first"); + assert_eq!(sim.sizes(), [512 * KIB, 3 * MIB / 2], "only what was missing"); + assert!(window.fill(&sim, PAGE).is_none(), "spent"); + } + + /// The doubling still sizes whole granules — 2, 4, 8, 16, then 32 MiB + /// — and stays at 32 MiB however many granules a long-lived heap has + /// mapped (a shift once wrapped it to zero after forty-three). + #[test] + fn whole_granules_double_to_their_ceiling_and_stay_there() { + let sim = Sim::leases(&[512 * MIB]); + let window = window(); + for _ in 0..31 { + window.fill(&sim, 2 * MIB).unwrap(); } + assert_eq!( + sim.sizes(), + [2 * MIB, 4 * MIB, 8 * MIB, 16 * MIB, 32 * MIB], + "62 MiB in five sessions" + ); + window.granules.store(60, Ordering::Relaxed); + window.fill(&sim, 2 * MIB).unwrap(); + assert_eq!(sim.sizes().last(), Some(&(32 * MIB))); + } + + /// The launch supply serves whole granules as it did, and is never + /// taken whole for a part: what the adopted supplies cannot hold is + /// refused, and the launch supply is left to its windows. + #[test] + fn the_launch_supply_is_never_split_into_a_run() { + let mut sim = Sim::leases(&[10 * MIB]); + sim.launch = Some((16 * MIB, Cursors::new())); + let window = window(); + assert_eq!(window.room(&sim), 10 * MIB); + assert!(window.fill(&sim, 20 * MIB).is_none()); + assert!(sim.maps.borrow().is_empty()); + // A granule the launch supply holds whole still comes from it once + // the adopted supplies cannot. + assert_eq!(sim.adopted[0].1.claim_bottom(10 * MIB, 10 * MIB), Some(0)); + window.fill(&sim, 3 * MIB).unwrap(); + assert_eq!(sim.maps.borrow()[0].0, LAUNCH); } } } @@ -639,6 +1329,34 @@ pub fn carve_and_map_in( offset: u64, len: u64, window: Option, +) -> Option> { + carve_and_map_at( + supply, + offset, + len, + window.map_or(Placement::Cursor, Placement::Window), + ) +} + +/// Where [`carve_and_map_at`] maps its carve. +#[cfg(target_os = "none")] +#[derive(Clone, Copy)] +enum Placement { + /// The thread's receiver-map cursor. + Cursor, + /// A named receiver window, at its own placement. + Window(kore_abi::session::ViewMapWindow), + /// An exact page of the `RESERVE` window ([`commit_reserved`]). + ReservePage(u32), +} + +/// [`carve_and_map_in`]'s session, with the carve's [`Placement`]. +#[cfg(target_os = "none")] +fn carve_and_map_at( + supply: kore_abi::session::SlotRef, + offset: u64, + len: u64, + placement: Placement, ) -> Option> { use kore_abi::cap::{CapRecord, ResourceKind, Rights}; use kore_abi::session::{GeomReport, Status}; @@ -671,9 +1389,10 @@ pub fn carve_and_map_in( ) .ok()?; // ι1: the staged map — subject is ι0's result slot (Move D dataflow). - match window { - None => b.derive_raw(carve, Rights::empty(), 0, report, None).ok()?, - Some(window) => b.map_view_in(carve, window, report).ok()?, + match placement { + Placement::Cursor => b.derive_raw(carve, Rights::empty(), 0, report, None).ok()?, + Placement::Window(window) => b.map_view_in(carve, window, report).ok()?, + Placement::ReservePage(page) => b.map_view_reserved(carve, page, report).ok()?, }; let committed = TrapTransport::new().commit_now(&b).ok()?; if committed.status() != Status::Ok { @@ -705,29 +1424,144 @@ mod tests { use super::*; #[test] - fn the_two_cursors_refuse_each_other_and_stay_spent() { - use core::sync::atomic::AtomicU64; - let reserved = AtomicU64::new(0); - let grown = AtomicU64::new(0); + fn the_two_cursors_refuse_each_other() { let total = 0x10_0000; // Top-down: two pages off the end. - assert_eq!( - reserve_top(&reserved, &grown, total, 0x2000), - Some(total - 0x2000) - ); - // Bottom-up growth may use everything below the tail... - assert!(grow_clears_reserved(0, total - 0x2000, total, 0x2000)); - // ...but not one byte into it. - assert!(!grow_clears_reserved(0, total - 0x1fff, total, 0x2000)); - // An unknown total cannot have a tail, so it refuses nothing. - assert!(grow_clears_reserved(0, total, 0, 0)); - // A reservation that would cross the grow mark refuses, and the - // claim stays spent: the next reservation lands below it. - grown.store(total - 0x3000, core::sync::atomic::Ordering::Release); - assert_eq!(reserve_top(&reserved, &grown, total, 0x2000), None); - assert_eq!(reserved.load(core::sync::atomic::Ordering::Acquire), 0x4000); - // Overflow past the supply refuses without wrapping. - assert_eq!(reserve_top(&reserved, &grown, total, u64::MAX), None); + let cursors = Cursors::new(); + assert_eq!(cursors.claim_top(total, 0x2000), Some(total - 0x2000)); + assert_eq!(cursors.room(total), Some(total - 0x2000)); + // Bottom-up growth may use everything below the top... + assert_eq!(cursors.claim_bottom(total, total - 0x2000), Some(0)); + assert_eq!(cursors.room(total), Some(0)); + // ...but not one page into it. The misfit is seen before it is + // claimed, so it spends nothing. + let cursors = Cursors::new(); + assert_eq!(cursors.claim_top(total, 0x2000), Some(total - 0x2000)); + assert_eq!(cursors.claim_bottom(total, total - 0x1000), None); + assert_eq!(cursors.grown.load(Ordering::SeqCst), 0); + assert_eq!(cursors.room(total), Some(total - 0x2000)); + // A reservation that would cross the grow mark refuses the same way. + let cursors = Cursors::new(); + assert_eq!(cursors.claim_bottom(total, total - 0x3000), Some(0)); + assert_eq!(cursors.claim_top(total, 0x2000), Some(total - 0x2000)); + assert_eq!(cursors.claim_top(total, 0x2000), None); + assert_eq!(cursors.reserved.load(Ordering::SeqCst), 0x2000); + // Overflow past the supply refuses, on either side, without moving + // the cursor it would have wrapped. + assert_eq!(cursors.claim_top(total, u64::MAX), None); + assert_eq!(cursors.reserved.load(Ordering::SeqCst), 0x2000); + assert_eq!(cursors.claim_bottom(total, u64::MAX), None); + assert_eq!(cursors.grown.load(Ordering::SeqCst), total - 0x3000); + // The page left between them is still there for either side. + assert_eq!(cursors.room(total), Some(0x1000)); + assert_eq!(cursors.claim_top(total, 0x1000), Some(total - 0x3000)); + assert_eq!(cursors.room(total), Some(0)); + } + + /// A claim that loses the race to the other cursor is refused and stays + /// spent: the pre-check reads a snapshot, and only claim-then-check keeps + /// the two apart. Replayed by hand: the bottom's snapshot saw the whole + /// supply free, the top then claimed two pages, and the bottom's claim + /// landed after it — the half of `claim_bottom` its check then refuses. + #[test] + fn a_claim_that_loses_the_race_stays_spent() { + let total = 0x10_0000; + let cursors = Cursors::new(); + assert_eq!(cursors.room(total), Some(total)); + assert_eq!(cursors.claim_top(total, 0x2000), Some(total - 0x2000)); + assert_eq!(claim(&cursors.grown, total), Some(0)); + // Spent: the supply refuses both sides rather than let them overlap. + assert_eq!(cursors.room(total), None); + assert_eq!(cursors.claim_top(total, 0x1000), None); + assert_eq!(cursors.claim_bottom(total, 0x1000), None); + } + + /// An adopted supply serves both claimants, so a heap granule sized off + /// it must leave its reserved top alone: with 96 of 128 MiB grown and a + /// 16 MiB window reserved, 16 MiB remain, and a 32 MiB granule refuses + /// rather than mapping the heap over the live window. + #[test] + fn a_granule_never_reaches_a_reserved_window() { + const MIB: u64 = 1 << 20; + let total = 128 * MIB; + let cursors = Cursors::new(); + assert_eq!(cursors.claim_bottom(total, 96 * MIB), Some(0)); + assert_eq!(cursors.claim_top(total, 16 * MIB), Some(112 * MIB)); + assert_eq!(cursors.room(total), Some(16 * MIB)); + assert_eq!(cursors.claim_bottom(total, 32 * MIB), None); + // The refused granule spent nothing: the 16 MiB between them is + // whole, and a granule it holds is placed below the window, ending + // exactly at it. + assert_eq!(cursors.room(total), Some(16 * MIB)); + assert_eq!(cursors.claim_bottom(total, 16 * MIB), Some(96 * MIB)); + assert_eq!(cursors.room(total), Some(0)); + assert_eq!(cursors.claim_top(total, 4096), None); + } + + /// A guest memory relocating tries its window against each supply in + /// turn: the ones too short for it are passed over intact, and the + /// first that holds it takes it. + #[test] + fn a_window_too_long_for_a_supply_leaves_it_whole() { + const MIB: u64 = 1 << 20; + let short = Cursors::new(); + let long = Cursors::new(); + assert_eq!(short.claim_bottom(128 * MIB, 100 * MIB), Some(0)); + let window = 60 * MIB; + let placed = [(&short, 128 * MIB), (&long, 64 * MIB)] + .iter() + .position(|(cursors, total)| cursors.claim_top(*total, window).is_some()); + assert_eq!(placed, Some(1)); + // The short supply's 28 MiB are still the heap's and later windows'. + assert_eq!(short.room(128 * MIB), Some(28 * MIB)); + assert_eq!(short.claim_top(128 * MIB, 16 * MIB), Some(112 * MIB)); + assert_eq!(long.room(64 * MIB), Some(4 * MIB)); + } + + /// Claims racing from both ends never hand out one byte twice: every + /// interval either claimant was granted is disjoint from every other. + #[test] + fn racing_claims_from_both_ends_are_disjoint() { + use std::sync::Barrier; + use std::vec::Vec; + + const PAGE: u64 = 4096; + const CLAIMANTS: usize = 4; + let total = 512 * PAGE; + for round in 0..64u64 { + let cursors = Cursors::new(); + let start = Barrier::new(CLAIMANTS); + let mut granted: Vec<(u64, u64)> = std::thread::scope(|scope| { + let handles: Vec<_> = (0..CLAIMANTS) + .map(|claimant| { + let (cursors, start) = (&cursors, &start); + scope.spawn(move || { + let mut mine = Vec::new(); + start.wait(); + for step in 0..64u64 { + let want = PAGE * (1 + (round + step + claimant as u64) % 3); + let offset = if claimant % 2 == 0 { + cursors.claim_bottom(total, want) + } else { + cursors.claim_top(total, want) + }; + mine.extend(offset.map(|offset| (offset, offset + want))); + } + mine + }) + }) + .collect(); + handles + .into_iter() + .flat_map(|handle| handle.join().expect("a claimant panicked")) + .collect() + }); + granted.sort_unstable(); + for pair in granted.windows(2) { + assert!(pair[0].1 <= pair[1].0, "round {round}: {pair:?} overlap"); + } + assert!(granted.last().is_none_or(|&(_, end)| end <= total)); + } } /// A leaked host arena for a `StaticMemSource` — bytes valid for the whole @@ -760,7 +1594,7 @@ mod tests { #[test] fn supply_reservation_refuses_wrap_without_moving_the_cursor() { let cursor = core::sync::atomic::AtomicU64::new(u64::MAX - 1023); - assert_eq!(reserve_supply(&cursor, 4096), None); + assert_eq!(claim(&cursor, 4096), None); assert_eq!(cursor.load(Ordering::Acquire), u64::MAX - 1023); } diff --git a/native/crates/rt/src/lib.rs b/native/crates/rt/src/lib.rs index 0283d5d..3c5bb09 100644 --- a/native/crates/rt/src/lib.rs +++ b/native/crates/rt/src/lib.rs @@ -104,6 +104,8 @@ pub mod pace; pub mod pinned; pub mod process; pub mod random; +#[cfg(feature = "heap")] +pub mod reserve; pub mod room; pub mod service; pub mod session; @@ -118,6 +120,7 @@ pub mod termination; pub mod thread; pub mod thread_controller; pub mod transport; +pub mod upcall; #[cfg(feature = "validate")] pub mod validate; pub mod wake; @@ -128,8 +131,8 @@ pub use block::{block_on, block_on_pinned, try_block_on}; pub use chan::{CapRx, ChanRx, ChanTx, Delivery, NoticeRx}; pub use entry::run_service; pub use executor::{ - Executor, TaskStorage, enable_irq_parks, next_wake_until, retire_irq_parks, sleep_for, - sleep_until, + Executor, NoticeHold, NoticeSink, TaskStorage, enable_irq_parks, next_wake_until, + retire_irq_parks, sleep_for, sleep_until, }; pub use exit::ExitCode; #[cfg(feature = "heap")] diff --git a/native/crates/rt/src/process.rs b/native/crates/rt/src/process.rs index 5c07158..4417d58 100644 --- a/native/crates/rt/src/process.rs +++ b/native/crates/rt/src/process.rs @@ -1,6 +1,7 @@ //! Generic foreground-process launch/result and streaming-I/O pages. pub mod interrupt; +pub mod words; use core::marker::PhantomData; use core::sync::atomic::{AtomicU32, Ordering}; @@ -17,9 +18,11 @@ use kore_abi::process::{ use kore_abi::process::{ PROCESS_OUTPUT_RING_OFFSET, PROCESS_OUTPUT_WAKE_COOKIE, PROCESS_STDIN_WAKE_COOKIE, PROCESS_STDIO_CHILD_WAITING_OFFSET, PROCESS_STDIO_PARENT_WAITING_OFFSET, + PROCESS_STDIO_PROFILE_DEFINED, PROCESS_STDIO_PROFILE_OFFSET, PROCESS_STDIO_PROFILE_TERMINAL, PROCESS_STDIO_RING_BYTES, PROCESS_STDIO_RING_CAPACITY, PROCESS_STDIO_RINGS_BYTES, PROCESS_STDIO_TO_CHILD_SEQUENCE_OFFSET, PROCESS_STDIO_TO_PARENT_SEQUENCE_OFFSET, - ProcessInputChunk, ProcessIo, ProcessIoError, ProcessOutputEvent, + ProcessInputChunk, ProcessIo, ProcessIoError, ProcessOutputEvent, ProcessBulkChunk, + PROCESS_BULK_OFFSET, PROCESS_BULK_RING_BYTES, PROCESS_BULK_CAPACITY, }; use kore_abi::session::{Notice, SlotRef}; use kore_ring::spsc::{self, Consumer, Full, Producer, RingError}; @@ -517,6 +520,7 @@ impl ProcessStdioParkTicket<'_> { pub struct ProcessStdioParent<'w, 't, T: Transport> { stdin: Producer<'w, ProcessInputChunk>, output: Consumer<'w, ProcessOutputEvent>, + bulk: Consumer<'w, ProcessBulkChunk>, peer: ProcessStdioPeer<'w, 't, T>, } @@ -536,6 +540,7 @@ impl ProcessStdioParent<'_, '_, T> { let Self { stdin: _, output: _, + bulk: _, peer, } = self; peer.close_after_recv_root_revoked(); @@ -554,7 +559,7 @@ impl ProcessStdioParent<'_, '_, T> { assert!(core::ptr::eq(drain.source, self.peer.observed)); self.peer.waiting.store(1, Ordering::SeqCst); self.peer.observed.load(Ordering::SeqCst) == drain.sequence - && self.output.outstanding() == 0 + && self.output.outstanding() == 0 && self.bulk.outstanding() == 0 } /// Enqueue one canonical-input record and notify the child. [`Full`] @@ -574,6 +579,12 @@ impl ProcessStdioParent<'_, '_, T> { Some((record, self.peer.signal().await)) } + /// Consume binary output through the same capacity-notification seam. + pub async fn try_recv_bulk(&mut self) -> Option<(ProcessBulkChunk, ProcessStdioWake)> { + let record = self.bulk.pop()?; + Some((record, self.peer.signal().await)) + } + /// Advisory free slots in the input producer. pub fn stdin_capacity_left(&self) -> u32 { self.stdin.capacity_left() @@ -585,9 +596,9 @@ impl ProcessStdioParent<'_, '_, T> { self.stdin.capacity().saturating_sub(self.stdin.capacity_left()) } - /// Advisory published output events awaiting this parent. + /// Advisory published text/control and binary records awaiting this parent. pub fn output_outstanding(&self) -> u32 { - self.output.outstanding() + self.output.outstanding() + self.bulk.outstanding() } /// Recheck the rings/peer epoch, take one hardware-monitor turn when @@ -600,10 +611,11 @@ impl ProcessStdioParent<'_, '_, T> { ) -> Result<(), ProcessStdioWaitError> { let stdin = &self.stdin; let output = &self.output; + let bulk = &self.bulk; self.peer .park_after_drain_inner( drain, - || wait_for.satisfied(output.outstanding() != 0, stdin.capacity_left() != 0), + || wait_for.satisfied(output.outstanding() != 0 || bulk.outstanding() != 0, stdin.capacity_left() != 0), None, ) .await @@ -626,10 +638,11 @@ impl ProcessStdioParent<'_, '_, T> { ) -> Result { let stdin = &self.stdin; let output = &self.output; + let bulk = &self.bulk; self.peer .park_after_drain_inner( drain, - || wait_for.satisfied(output.outstanding() != 0, stdin.capacity_left() != 0), + || wait_for.satisfied(output.outstanding() != 0 || bulk.outstanding() != 0, stdin.capacity_left() != 0), Some(delta_ns), ) .await @@ -641,7 +654,10 @@ impl ProcessStdioParent<'_, '_, T> { pub struct ProcessStdioChild<'w, 't, T: Transport> { stdin: Consumer<'w, ProcessInputChunk>, output: Producer<'w, ProcessOutputEvent>, + bulk: Producer<'w, ProcessBulkChunk>, peer: ProcessStdioPeer<'w, 't, T>, + /// What the launcher declared about the far end, read at attach. + profile: StdioProfile, } impl<'w, T: Transport> ProcessStdioChild<'w, '_, T> { @@ -652,11 +668,19 @@ impl<'w, T: Transport> ProcessStdioChild<'w, '_, T> { let Self { stdin: _, output: _, + bulk: _, peer, + profile: _, } = self; peer.close_after_recv_root_revoked(); } + /// What the launcher declared about this stream's far end — whether it + /// is a terminal. + pub fn profile(&self) -> StdioProfile { + self.profile + } + /// Snapshot the parent-to-child sequence before a total drain. pub fn begin_drain(&self) -> ProcessStdioDrain { self.peer.begin_drain() @@ -698,6 +722,15 @@ impl<'w, T: Transport> ProcessStdioChild<'w, '_, T> { Ok(self.peer.signal_now()) } + /// Publish binary output; Full leaves ownership with the caller. + pub fn send_bulk_now(&mut self, record: ProcessBulkChunk) -> Result { + self.bulk.push(record)?; + Ok(self.peer.signal_now()) + } + pub fn bulk_unconsumed(&self) -> u32 { + self.bulk.capacity().saturating_sub(self.bulk.capacity_left()) + } + /// Advisory published input records awaiting this child. pub fn stdin_outstanding(&self) -> u32 { self.stdin.outstanding() @@ -838,11 +871,13 @@ const _: () = { spsc::region_bytes::(PROCESS_STDIO_RING_CAPACITY) == PROCESS_STDIO_RING_BYTES ); + assert!(spsc::region_bytes::(PROCESS_BULK_CAPACITY) == PROCESS_BULK_RING_BYTES); }; struct StdioRegions<'w> { stdin: &'w mut [u8], output: &'w mut [u8], + bulk: &'w mut [u8], to_child: &'w AtomicU32, child_waiting: &'w AtomicU32, to_parent: &'w AtomicU32, @@ -857,6 +892,8 @@ fn stdio_regions(window: &mut [u8]) -> Result, RingError> { return Err(RingError::Misaligned); } let page = &mut window[..kore_abi::process::PROCESS_STDIO_BYTES]; + let (page, bulk) = page.split_at_mut(PROCESS_BULK_OFFSET); + let bulk = &mut bulk[..PROCESS_BULK_RING_BYTES]; let (rings, signals) = page.split_at_mut(PROCESS_STDIO_RINGS_BYTES); let (stdin, output) = rings.split_at_mut(PROCESS_OUTPUT_RING_OFFSET); let output = &mut output[..PROCESS_STDIO_RING_BYTES]; @@ -884,6 +921,7 @@ fn stdio_regions(window: &mut [u8]) -> Result, RingError> { Ok(StdioRegions { stdin, output, + bulk, to_child, child_waiting, to_parent, @@ -891,13 +929,47 @@ fn stdio_regions(window: &mut [u8]) -> Result, RingError> { }) } +/// What a launcher declares about its stream's far end +/// ([`PROCESS_STDIO_PROFILE_OFFSET`]): written once when it formats the page, +/// read once when the child attaches. Only bits the ABI defines are ever +/// represented, so a page written by a newer launcher reads here as what +/// this ABI can name. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct StdioProfile(u32); + +impl StdioProfile { + /// Nothing declared: a stream a launcher captures or replays. + pub const PLAIN: Self = Self(0); + /// The far end is a terminal ([`PROCESS_STDIO_PROFILE_TERMINAL`]): lines + /// edited and echoed by it, output rendered by its ANSI decoder. + pub const TERMINAL: Self = Self(PROCESS_STDIO_PROFILE_TERMINAL); + + /// Whether the far end is a terminal — what a program asks with `isatty`. + pub const fn is_terminal(self) -> bool { + self.0 & PROCESS_STDIO_PROFILE_TERMINAL != 0 + } + + /// The profile word as the page carries it. + pub const fn bits(self) -> u32 { + self.0 + } + + /// The profile a page's word declares: the bits this ABI defines. + pub const fn from_bits(word: u32) -> Self { + Self(word & PROCESS_STDIO_PROFILE_DEFINED) + } +} + /// Format a fresh stream page before the launcher transfers the child's -/// sibling view. +/// sibling view, declaring its far end as `profile`. /// /// The entire page is cleared, including the ABI-reserved suffix, before the -/// two fixed-depth rings and the directional sequence lines are published. -/// The returned affine token is the only safe parent attachment. -pub fn init_stdio(window: &mut [u8]) -> Result, RingError> { +/// profile, the two fixed-depth rings and the directional sequence lines are +/// published. The returned affine token is the only safe parent attachment. +pub fn init_stdio( + window: &mut [u8], + profile: StdioProfile, +) -> Result, RingError> { if window.len() < kore_abi::process::PROCESS_STDIO_BYTES { return Err(RingError::TooSmall); } @@ -905,9 +977,12 @@ pub fn init_stdio(window: &mut [u8]) -> Result, RingErr return Err(RingError::Misaligned); } window[..kore_abi::process::PROCESS_STDIO_BYTES].fill(0); + window[PROCESS_STDIO_PROFILE_OFFSET..PROCESS_STDIO_PROFILE_OFFSET + 4] + .copy_from_slice(&profile.bits().to_le_bytes()); let regions = stdio_regions(window)?; spsc::init::(regions.stdin, PROCESS_STDIO_RING_CAPACITY)?; spsc::init::(regions.output, PROCESS_STDIO_RING_CAPACITY)?; + spsc::init::(regions.bulk, PROCESS_BULK_CAPACITY)?; Ok(PreparedProcessStdio { base: window.as_mut_ptr(), len: kore_abi::process::PROCESS_STDIO_BYTES, @@ -915,6 +990,15 @@ pub fn init_stdio(window: &mut [u8]) -> Result, RingErr }) } +/// The profile a formatted page declares. The word is written before the +/// child's view exists and never again, so a plain read is the whole +/// protocol. +fn page_profile(window: &[u8]) -> StdioProfile { + let mut word = [0u8; 4]; + word.copy_from_slice(&window[PROCESS_STDIO_PROFILE_OFFSET..PROCESS_STDIO_PROFILE_OFFSET + 4]); + StdioProfile::from_bits(u32::from_le_bytes(word)) +} + /// Bind the parent end. `output_recv` is `None` only for a stream host, /// whose child output wake rings the host's own executor doorbell. unsafe fn bind_parent<'w, 't, T: Transport>( @@ -930,6 +1014,7 @@ unsafe fn bind_parent<'w, 't, T: Transport>( Ok(ProcessStdioParent { stdin: spsc::producer::(regions.stdin)?, output: spsc::consumer::(regions.output)?, + bulk: spsc::consumer::(regions.bulk)?, peer: ProcessStdioPeer { transport, send: stdin_send, @@ -957,10 +1042,16 @@ unsafe fn bind_child<'w, 't, T: Transport>( ) -> Result, RingError> { // SAFETY: the caller owns this local mapping attachment for `'w`. let window = unsafe { core::slice::from_raw_parts_mut(base, len) }; + if window.len() < kore_abi::process::PROCESS_STDIO_BYTES { + return Err(RingError::TooSmall); + } + let profile = page_profile(window); let regions = stdio_regions(window)?; Ok(ProcessStdioChild { + profile, stdin: spsc::consumer::(regions.stdin)?, output: spsc::producer::(regions.output)?, + bulk: spsc::producer::(regions.bulk)?, peer: ProcessStdioPeer { transport, send: output_send, @@ -984,7 +1075,23 @@ fn split_stdio<'w, 'p, 'c, P: Transport, C: Transport>( parent_transport: &'p P, child_transport: &'c C, ) -> Result<(ProcessStdioParent<'w, 'p, P>, ProcessStdioChild<'w, 'c, C>), RingError> { - let prepared = init_stdio(window)?; + split_stdio_as( + window, + StdioProfile::PLAIN, + parent_transport, + child_transport, + ) +} + +/// [`split_stdio`] over a page whose launcher declared `profile`. +#[cfg(test)] +fn split_stdio_as<'w, 'p, 'c, P: Transport, C: Transport>( + window: &'w mut [u8], + profile: StdioProfile, + parent_transport: &'p P, + child_transport: &'c C, +) -> Result<(ProcessStdioParent<'w, 'p, P>, ProcessStdioChild<'w, 'c, C>), RingError> { + let prepared = init_stdio(window, profile)?; let (base, len) = prepared.into_raw(); // SAFETY: this is the same-address-space test form of the cross-mapping // protocol. Exactly one direction-specific producer and consumer are @@ -1355,12 +1462,13 @@ mod tests { assert_eq!(PROCESS_STDIO_CHILD_WAITING_OFFSET, 2436); assert_eq!(PROCESS_STDIO_TO_PARENT_SEQUENCE_OFFSET, 2496); assert_eq!(PROCESS_STDIO_PARENT_WAITING_OFFSET, 2500); - assert_eq!(PROCESS_STDIO_LAYOUT_BYTES, 2560); + assert_eq!(PROCESS_STDIO_PROFILE_OFFSET, 2560); + assert_eq!(PROCESS_STDIO_LAYOUT_BYTES, 3840); assert!(PROCESS_STDIO_LAYOUT_BYTES <= kore_abi::process::PROCESS_STDIO_BYTES); let mut too_small = [0u8; kore_abi::process::PROCESS_STDIO_BYTES - 1]; assert!(matches!( - init_stdio(&mut too_small), + init_stdio(&mut too_small, StdioProfile::PLAIN), Err(RingError::TooSmall) )); @@ -1368,11 +1476,48 @@ mod tests { struct Oversized([u8; kore_abi::process::PROCESS_STDIO_BYTES + 1]); let mut oversized = Oversized([0; kore_abi::process::PROCESS_STDIO_BYTES + 1]); assert!(matches!( - init_stdio(&mut oversized.0[1..]), + init_stdio(&mut oversized.0[1..], StdioProfile::PLAIN), Err(RingError::Misaligned) )); } + /// The child reads the far end its launcher declared: a terminal is + /// told so, a plain stream is not, and a page carrying bits this ABI + /// does not define reads as only the bits it does. + #[test] + fn the_child_reads_the_profile_its_launcher_declared() { + let transport = OkTransport::new(); + for profile in [StdioProfile::PLAIN, StdioProfile::TERMINAL] { + let mut page = Page::zeroed(); + let (_parent, child) = + split_stdio_as(&mut page.0, profile, &transport, &transport).unwrap(); + assert_eq!(child.profile(), profile); + assert_eq!( + child.profile().is_terminal(), + profile == StdioProfile::TERMINAL + ); + } + let mut page = Page::zeroed(); + let prepared = init_stdio(&mut page.0, StdioProfile::TERMINAL).unwrap(); + let (base, len) = prepared.into_raw(); + page.0[PROCESS_STDIO_PROFILE_OFFSET..PROCESS_STDIO_PROFILE_OFFSET + 4] + .copy_from_slice(&u32::MAX.to_le_bytes()); + // SAFETY: the test owns the page for the child's whole life, and only + // this one child end is bound. + let child = unsafe { + bind_child( + base, + len, + &transport, + ProcessStdioSend::Test(SlotRef::from_wire(43)), + ProcessStdioRecv::Test(SlotRef::from_wire(44)), + ) + } + .unwrap(); + assert_eq!(child.profile(), StdioProfile::TERMINAL); + assert!(!StdioProfile::from_bits(0).is_terminal()); + } + #[test] fn shared_doorbell_covers_both_sides_of_registration_and_coalesces() { let transport = OkTransport::new(); @@ -1404,9 +1549,31 @@ mod tests { assert!(parent.arm_shared_doorbell(parent.begin_drain())); } - /// A stream host binds the parent end with no dedicated receive: it parks - /// only through the shared-doorbell fence, never through a notice post it - /// does not hold, and closes without disarming a landing it never armed. + /// Binary output shares the parent's lost-wakeup fence and capacity wakes. + #[test] + fn binary_output_is_bounded_and_participates_in_the_shared_doorbell_fence() { + let transport = OkTransport::new(); + let mut page = Page::zeroed(); + let (mut parent, mut child) = split_stdio(&mut page.0, &transport, &transport).unwrap(); + assert_eq!(spsc::region_bytes::(PROCESS_BULK_CAPACITY), PROCESS_BULK_RING_BYTES); + assert!(parent.arm_shared_doorbell(parent.begin_drain())); + let first = ProcessBulkChunk::new(7, 0, &[11; 496]).unwrap(); + let second = ProcessBulkChunk::new(7, 496, &[22; 4]).unwrap(); + child.send_bulk_now(first).unwrap(); + child.send_bulk_now(second).unwrap(); + assert_eq!(parent.output_outstanding(), PROCESS_BULK_CAPACITY); + assert!(child.send_bulk_now(first).is_err()); + assert!(!parent.arm_shared_doorbell(parent.begin_drain())); + assert_eq!(drive(parent.try_recv_bulk()).unwrap().0, first); + assert_eq!(child.bulk_unconsumed(), 1); + assert_eq!(drive(parent.try_recv_bulk()).unwrap().0, second); + assert_eq!(child.bulk_unconsumed(), 0); + assert_eq!(parent.output_outstanding(), 0); + assert!(parent.arm_shared_doorbell(parent.begin_drain())); + child.send_bulk_now(first).unwrap(); + assert!(!parent.arm_shared_doorbell(parent.begin_drain())); + } + #[test] fn a_capacity_wait_ignores_unread_input() { use ProcessStdioWaitFor::{Capacity, Incoming, IncomingOrCapacity}; @@ -1422,7 +1589,7 @@ mod tests { fn a_stream_host_parks_only_through_its_shared_doorbell() { let transport = OkTransport::new(); let mut page = Page::zeroed(); - let prepared = init_stdio(&mut page.0).unwrap(); + let prepared = init_stdio(&mut page.0, StdioProfile::PLAIN).unwrap(); let (base, len) = prepared.into_raw(); let (mut host, mut child) = unsafe { ( @@ -1627,7 +1794,7 @@ mod tests { }; assert!(matches!(result, Err(RingError::BadHeader))); - let prepared = init_stdio(&mut page.0).unwrap(); + let prepared = init_stdio(&mut page.0, StdioProfile::PLAIN).unwrap(); let (base, len) = prepared.into_raw(); // Attaching either side validates both ring headers and their fixed // geometry at the ABI-assigned semantic offsets. diff --git a/native/crates/rt/src/process/words.rs b/native/crates/rt/src/process/words.rs new file mode 100644 index 0000000..e6db3c9 --- /dev/null +++ b/native/crates/rt/src/process/words.rs @@ -0,0 +1,287 @@ +//! **The words of an argument tail.** A launcher hands a program its +//! arguments as one opaque byte string ([`ProcessIo::args`]); a program that +//! wants an argument vector splits it here, the way a POSIX shell splits a +//! command line into words — with quoting, without expansion: +//! +//! - unquoted whitespace separates words; +//! - `'…'` is literal to the next `'`; +//! - `"…"` is literal to the next unescaped `"`, where `\"` and `\\` stand for +//! `"` and `\` and every other backslash is itself; +//! - an unquoted `\` takes the next byte literally (`\ ` is a space inside a +//! word); +//! - quoted and unquoted pieces that touch form one word (`a"b c"d` is +//! `ab cd`), and `""` or `''` alone is an empty word. +//! +//! So `-c 'print("hello world")'` passes `print("hello world")` whole. Bytes +//! at or above 0x80 never equal a quote, a backslash or ASCII whitespace, so +//! UTF-8 passes through unchanged. Splitting allocates nothing: [`words`] +//! validates each word as it finds it, and [`Word::bytes`] decodes it lazily. +//! +//! [`ProcessIo::args`]: kore_abi::process::ProcessIo::args + +use core::fmt; + +/// Why an argument tail does not split into words. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WordsError { + /// The quote (`'` or `"`) opened at byte `at` is never closed. + Unterminated { quote: u8, at: usize }, + /// The tail ends in a backslash with nothing left to escape. + TrailingBackslash, +} + +impl fmt::Display for WordsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Self::Unterminated { quote, at } => { + write!( + f, + "the {} opened at byte {at} is never closed", + char::from(quote) + ) + } + Self::TrailingBackslash => f.write_str("the arguments end in a lone backslash"), + } + } +} + +/// The words of `line`, in order ([`Words`]). +pub fn words(line: &[u8]) -> Words<'_> { + Words { line, at: 0 } +} + +/// An iterator over the words of an argument tail. The first malformed word +/// is reported once and ends the iteration. +#[derive(Clone, Debug)] +pub struct Words<'a> { + line: &'a [u8], + at: usize, +} + +impl<'a> Iterator for Words<'a> { + type Item = Result, WordsError>; + + fn next(&mut self) -> Option { + let rest = self.line.get(self.at..)?; + let start = self.at + rest.iter().take_while(|b| b.is_ascii_whitespace()).count(); + if start == self.line.len() { + self.at = start; + return None; + } + match word_end(self.line, start) { + Ok(end) => { + self.at = end; + Some(Ok(Word { + raw: &self.line[start..end], + })) + } + Err(error) => { + self.at = self.line.len(); + Some(Err(error)) + } + } + } +} + +impl core::iter::FusedIterator for Words<'_> {} + +/// Where the word that starts at `start` ends: the first unquoted whitespace, +/// or the end of the line. +fn word_end(line: &[u8], start: usize) -> Result { + let mut i = start; + while let Some(&byte) = line.get(i) { + match byte { + b if b.is_ascii_whitespace() => return Ok(i), + b'\\' => { + if i + 1 == line.len() { + return Err(WordsError::TrailingBackslash); + } + i += 2; + } + b'\'' => { + let close = line[i + 1..].iter().position(|&b| b == b'\'').ok_or( + WordsError::Unterminated { + quote: b'\'', + at: i, + }, + )?; + i += close + 2; + } + b'"' => { + let mut j = i + 1; + loop { + match line.get(j) { + None => return Err(WordsError::Unterminated { quote: b'"', at: i }), + Some(b'"') => break, + Some(b'\\') if matches!(line.get(j + 1), Some(b'"' | b'\\')) => j += 2, + Some(_) => j += 1, + } + } + i = j + 1; + } + _ => i += 1, + } + } + Ok(i) +} + +/// One word of an argument tail, as written (quotes and escapes included); +/// [`bytes`](Self::bytes) is what it stands for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Word<'a> { + raw: &'a [u8], +} + +impl<'a> Word<'a> { + /// The word as written on the line. + pub fn raw(&self) -> &'a [u8] { + self.raw + } + + /// The word's bytes with its quoting removed. + pub fn bytes(&self) -> Unquoted<'a> { + Unquoted { + raw: self.raw, + at: 0, + quote: None, + } + } +} + +/// The bytes a [`Word`] stands for, decoded as they are read. +#[derive(Clone, Debug)] +pub struct Unquoted<'a> { + raw: &'a [u8], + at: usize, + /// The quote currently open, if any. + quote: Option, +} + +impl Iterator for Unquoted<'_> { + type Item = u8; + + fn next(&mut self) -> Option { + loop { + let byte = *self.raw.get(self.at)?; + self.at += 1; + match (self.quote, byte) { + (None, b'\'' | b'"') => self.quote = Some(byte), + (Some(open), b) if b == open => self.quote = None, + (None, b'\\') => { + let escaped = *self.raw.get(self.at)?; + self.at += 1; + return Some(escaped); + } + (Some(b'"'), b'\\') if matches!(self.raw.get(self.at), Some(b'"' | b'\\')) => { + let escaped = self.raw[self.at]; + self.at += 1; + return Some(escaped); + } + (_, b) => return Some(b), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::vec::Vec; + + /// Every word's decoded bytes, or the first error. + fn split(line: &str) -> Result>, WordsError> { + words(line.as_bytes()) + .map(|word| word.map(|word| word.bytes().collect())) + .collect() + } + + fn ok(line: &str) -> Vec> { + split(line).unwrap_or_else(|error| panic!("{line:?}: {error}")) + } + + fn words_of(items: &[&str]) -> Vec> { + items.iter().map(|item| item.as_bytes().to_vec()).collect() + } + + #[test] + fn unquoted_whitespace_separates_words() { + assert_eq!(ok("a bc d\te\n"), words_of(&["a", "bc", "d", "e"])); + assert_eq!( + ok(" lead and trail "), + words_of(&["lead", "and", "trail"]) + ); + assert!(ok("").is_empty()); + assert!(ok(" \t \n").is_empty()); + } + + #[test] + fn single_quotes_are_literal() { + assert_eq!( + ok(r#"-c 'print("hello world")'"#), + words_of(&["-c", r#"print("hello world")"#]) + ); + assert_eq!(ok(r"'a\b \' x"), words_of(&[r"a\b \", "x"])); + } + + #[test] + fn double_quotes_escape_only_quote_and_backslash() { + assert_eq!(ok(r#""a b""#), words_of(&["a b"])); + assert_eq!(ok(r#""say \"hi\"""#), words_of(&[r#"say "hi""#])); + assert_eq!(ok(r#""c:\\dir""#), words_of(&[r"c:\dir"])); + assert_eq!( + ok(r#""keep \n and \' as written""#), + words_of(&[r"keep \n and \' as written"]) + ); + assert_eq!(ok(r#""it's""#), words_of(&["it's"])); + } + + #[test] + fn an_unquoted_backslash_takes_the_next_byte() { + assert_eq!(ok(r"my\ file.txt next"), words_of(&["my file.txt", "next"])); + assert_eq!(ok(r#"\"quoted\" \\"#), words_of(&[r#""quoted""#, r"\"])); + } + + #[test] + fn touching_pieces_form_one_word() { + assert_eq!(ok(r#"a"b c"'d e'f g"#), words_of(&["ab cd ef", "g"])); + assert_eq!(ok(r#"--name="two words""#), words_of(&["--name=two words"])); + } + + #[test] + fn empty_quotes_are_empty_words() { + assert_eq!(ok(r#"a "" b ''"#), words_of(&["a", "", "b", ""])); + } + + #[test] + fn utf8_passes_through() { + assert_eq!(ok("\"héllo wörld\" ü"), words_of(&["héllo wörld", "ü"])); + } + + #[test] + fn a_malformed_tail_is_refused_and_ends_the_words() { + assert_eq!( + split("ok 'open"), + Err(WordsError::Unterminated { + quote: b'\'', + at: 3 + }) + ); + assert_eq!( + split(r#"ok "open \""#), + Err(WordsError::Unterminated { quote: b'"', at: 3 }) + ); + assert_eq!(split(r"ok \"), Err(WordsError::TrailingBackslash)); + let mut iter = words(b"a 'b c"); + assert!(matches!(iter.next(), Some(Ok(_)))); + assert!(matches!(iter.next(), Some(Err(_)))); + assert_eq!(iter.next(), None); + assert_eq!(iter.next(), None); + } + + #[test] + fn a_word_keeps_its_raw_spelling() { + let mut iter = words(br#"x "a b"c"#); + assert_eq!(iter.next().unwrap().unwrap().raw(), b"x"); + assert_eq!(iter.next().unwrap().unwrap().raw(), br#""a b"c"#); + } +} diff --git a/native/crates/rt/src/reserve.rs b/native/crates/rt/src/reserve.rs new file mode 100644 index 0000000..18ef38b --- /dev/null +++ b/native/crates/rt/src/reserve.rs @@ -0,0 +1,607 @@ +//! **Reservations** — address ranges a task reserves in its reservation +//! window (`geom::RESERVE`) and commits page range by page range: a guarded +//! wasm linear memory is one, 4 GiB with a 32 MiB guard on either side, of +//! which only what the guest has grown to is mapped (`res/WASM_PLATFORM.md` +//! §12). +//! +//! A [`Reservation`] is affine and owns its range. It places the range +//! ([`Reservation::new`]), commits supply carves into it +//! ([`Reservation::commit`]: one session per commit, nothing held after), and +//! on drop releases every commit — one session per supply it drew on — and +//! hands the range back for the next reservation. The kernel keeps no +//! reservation object: the window's page table is its only state, and a +//! commit over a mapped page refuses there. +//! +//! **One placer per window.** The placer is this module's, process-wide, so +//! every consumer of the window shares it: two cursors over one window hand +//! out each other's addresses (`kore_ld::place::SharedPlacer`'s warning). +//! +//! **The task's own thread only.** Commits draw on the supplies the heap +//! holds, which are slots of the task thread's arena (the heap window's +//! refill rule), so a reservation is made, committed and released there and +//! nowhere else; the type is `!Send`. + +use core::marker::PhantomData; +#[cfg(target_os = "none")] +use core::sync::atomic::{AtomicPtr, AtomicUsize}; +use core::sync::atomic::{AtomicU64, Ordering}; + +use kore_abi::session::RESERVE_PAGE_BYTES; + +use crate::heap::CommitSupply; + +/// The placement grain: a reservation's base is aligned to it and its length +/// rounded up to it. 64 MiB divides a guarded wasm memory exactly — two +/// 32 MiB guards and a 4 GiB reservation are 65 of them — and puts each +/// memory's base on a 32 MiB boundary, past any large leaf's alignment. The +/// 1 TiB window is 16384 granules: a 2 KiB bitmap, allocated by the first +/// reservation, so a program that never reserves pays nothing. +pub const GRANULE: u64 = 64 << 20; + +/// Committed ranges one reservation tracks: Wasmtime grows a memory +/// contiguously from its first page, so one range is the steady state, and a +/// commit that would need more than this many refuses rather than allocate. +const COMMITTED_RANGES: usize = 8; + +/// Why a reservation could not be made. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ReserveError { + /// Off the kore target, or off the task's own thread. + Unsupported, + /// The task was stamped no reservation window. + NoWindow, + /// No run of free granules is long enough, or the length overflows. + Full, +} + +/// Why a commit did not land. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CommitError { + /// The span is empty, not page-granular, or leaves the reservation. + OutOfRange, + /// The committed span would need more ranges than a reservation tracks. + Fragmented, + /// No supply this task holds has room for the span. + NoRoom, + /// The kernel refused the map; nothing was left mapped. + Refused, +} + +/// **An address range this task reserved**, committed as it grows and +/// released whole on drop. Thread-bound: see the module docs. +#[must_use = "a dropped reservation releases its commits at once"] +pub struct Reservation { + /// First byte of the range. + base: u64, + /// The range's length, a whole number of [`GRANULE`]s. + len: u64, + /// Disjoint committed spans, as offsets from `base`, unordered. + committed: [Option<(u64, u64)>; COMMITTED_RANGES], + /// The supplies commits drew on, by [`CommitSupply::index`]. + supplies: [u64; SUPPLY_WORDS], + _thread_bound: PhantomData<*mut ()>, +} + +/// Words of a supply set: one bit per supply a commit may draw on. +const SUPPLY_WORDS: usize = CommitSupply::COUNT.div_ceil(64); + +/// Bytes whose release refused: a range still holding a commit is never +/// placed again, so it is counted rather than lost silently. +static LEAKED: AtomicU64 = AtomicU64::new(0); + +/// Bytes of the window whose release refused. Zero on a healthy task. +pub fn leaked_bytes() -> u64 { + LEAKED.load(Ordering::Relaxed) +} + +impl Reservation { + /// Reserve `len` bytes of this task's reservation window, rounded up to + /// [`GRANULE`]s and aligned to one. Nothing is mapped. + pub fn new(len: u64) -> Result { + let (base, window) = placer()?; + let granules = len.max(1).div_ceil(GRANULE); + let len = granules.checked_mul(GRANULE).ok_or(ReserveError::Full)?; + let first = window.claim(usize::try_from(granules).map_err(|_| ReserveError::Full)?); + let first = first.ok_or(ReserveError::Full)?; + Ok(Self { + base: base + first as u64 * GRANULE, + len, + committed: [None; COMMITTED_RANGES], + supplies: [0; SUPPLY_WORDS], + _thread_bound: PhantomData, + }) + } + + /// The range's first byte. + pub fn base(&self) -> u64 { + self.base + } + + /// The range's length. + pub fn len(&self) -> u64 { + self.len + } + + /// **Make `[offset, offset + len)` of the range committed**: carve what + /// of it is not yet off a supply this task holds and map it there, + /// writable — one [`heap::commit_reserved`](crate::heap::commit_reserved) + /// per gap, so a span already committed costs nothing. The span is + /// page-granular and inside the range. + /// + /// All or nothing. A gap's commit refused partway leaves the prefix the + /// kernel published, and is released by its own range — which holds no + /// earlier commit, being a gap — and the gaps this call already filled + /// are released with it, so a refusal leaves the range as it found it. + /// A release that itself refuses keeps what it could not remove tracked, + /// committed and drawn from its supply, so the tracker never claims a + /// page is free that is mapped, and the drop still releases it. + pub fn commit(&mut self, offset: u64, len: u64) -> Result<(), CommitError> { + let end = offset + .checked_add(len) + .filter(|&end| len != 0 && end <= self.len) + .ok_or(CommitError::OutOfRange)?; + if (offset | len) % RESERVE_PAGE_BYTES != 0 { + return Err(CommitError::OutOfRange); + } + let gaps = gaps(&self.committed, offset, end); + // Every span the commit leaves must fit the tracker before anything + // is mapped. + let mut merged = self.committed; + for &(at, until) in gaps.iter().flatten() { + merged = merge(&merged, at, until).ok_or(CommitError::Fragmented)?; + } + let mut filled = [None; COMMITTED_RANGES + 1]; + for (i, &(at, until)) in gaps.iter().flatten().enumerate() { + let outcome = self + .page(at) + .zip(u32::try_from((until - at) / RESERVE_PAGE_BYTES).ok()) + .ok_or(CommitError::OutOfRange) + .and_then(|(page, pages)| { + crate::heap::commit_reserved(page, until - at) + .map(|supply| (supply, page, pages)) + }); + match outcome { + Ok((supply, page, pages)) => filled[i] = Some((supply, page, pages, at, until)), + Err(refusal) => { + self.roll_back(&filled); + return Err(refusal); + } + } + } + self.committed = merged; + for &(supply, ..) in filled.iter().flatten() { + self.note(supply); + } + Ok(()) + } + + /// Release the gaps a refused [`commit`](Self::commit) already filled. + /// One that will not release stays tracked: mapped, so committed, and + /// drawn from its supply, which the drop then releases. + fn roll_back(&mut self, filled: &[Option<(CommitSupply, u32, u32, u64, u64)>]) { + for &(supply, page, pages, at, until) in filled.iter().flatten() { + if !crate::heap::release_reserved(supply, page, pages) { + if let Some(kept) = merge(&self.committed, at, until) { + self.committed = kept; + } + self.note(supply); + } + } + } + + /// Record that a commit drew on `supply`. + fn note(&mut self, supply: CommitSupply) { + self.supplies[supply.index() / 64] |= 1 << (supply.index() % 64); + } + + /// Whether `[offset, offset + len)` lies wholly inside one committed span. + pub fn is_committed(&self, offset: u64, len: u64) -> bool { + offset.checked_add(len).is_some_and(|end| { + self.committed + .iter() + .flatten() + .any(|&(at, until)| at <= offset && end <= until) + }) + } + + /// Whether any of `[offset, offset + len)` is committed. + pub fn meets_committed(&self, offset: u64, len: u64) -> bool { + let end = offset.saturating_add(len); + self.committed + .iter() + .flatten() + .any(|&(at, until)| offset < until && at < end) + } + + /// The window page `offset` falls on. + fn page(&self, offset: u64) -> Option { + let (window, _) = window_span()?; + u32::try_from((self.base + offset - window) / RESERVE_PAGE_BYTES).ok() + } +} + +impl Drop for Reservation { + /// Release every commit, once per supply drawn on, then hand the range + /// back. A refused release leaves a commit mapped, and the range is + /// then never placed again (counted in [`leaked_bytes`]): a reservation + /// placed over a live commit would refuse every commit of its own. + fn drop(&mut self) { + let (Some(page), Ok(pages)) = (self.page(0), u32::try_from(self.len / RESERVE_PAGE_BYTES)) + else { + LEAKED.fetch_add(self.len, Ordering::Relaxed); + return; + }; + // Every supply is released even past a refusal: what can go, goes. + let mut released = true; + for supply in (0..CommitSupply::COUNT).filter_map(CommitSupply::from_index) { + let index = supply.index(); + if self.supplies[index / 64] & (1 << (index % 64)) != 0 { + released &= crate::heap::release_reserved(supply, page, pages); + } + } + if !released { + LEAKED.fetch_add(self.len, Ordering::Relaxed); + return; + } + if let Ok((base, window)) = placer() { + window.free( + ((self.base - base) / GRANULE) as usize, + (self.len / GRANULE) as usize, + ); + } + } +} + +/// The parts of `[at, until)` no span of `spans` covers, in address order — +/// at most one more than there are spans. +fn gaps( + spans: &[Option<(u64, u64)>; COMMITTED_RANGES], + at: u64, + until: u64, +) -> [Option<(u64, u64)>; COMMITTED_RANGES + 1] { + // The spans that meet the range, by start: a handful, sorted in place. + let mut meeting = [(0u64, 0u64); COMMITTED_RANGES]; + let mut n = 0; + for &(start, end) in spans.iter().flatten() { + if start < until && at < end { + meeting[n] = (start, end); + n += 1; + } + } + meeting[..n].sort_unstable(); + let mut out = [None; COMMITTED_RANGES + 1]; + let (mut cursor, mut found) = (at, 0); + for &(start, end) in &meeting[..n] { + if cursor < start { + out[found] = Some((cursor, start)); + found += 1; + } + cursor = cursor.max(end); + } + if cursor < until { + out[found] = Some((cursor, until)); + } + out +} + +/// `spans` with `[at, until)` merged in, joining the spans it abuts; `None` +/// when no slot is left for it. +fn merge( + spans: &[Option<(u64, u64)>; COMMITTED_RANGES], + mut at: u64, + mut until: u64, +) -> Option<[Option<(u64, u64)>; COMMITTED_RANGES]> { + let mut out = [None; COMMITTED_RANGES]; + let mut kept = 0; + for &(start, end) in spans.iter().flatten() { + if end == at || start == until { + at = at.min(start); + until = until.max(end); + } else { + out[kept] = Some((start, end)); + kept += 1; + } + } + *out.get_mut(kept)? = Some((at, until)); + Some(out) +} + +/// The window's `(base, len)`, from this thread's geometry. +#[cfg(target_os = "none")] +fn window_span() -> Option<(u64, u64)> { + crate::entry::thread_context() + .and_then(|rt| rt.geom()) + .and_then(|g| g.window(kore_abi::task::geom::RESERVE)) +} + +/// Off metal there is no geometry. +#[cfg(not(target_os = "none"))] +fn window_span() -> Option<(u64, u64)> { + None +} + +/// The process's granule bitmap over the window, allocated by the first +/// reservation; null until then. +#[cfg(target_os = "none")] +static BITMAP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); +/// The bitmap's length in granules (zero until allocated). +#[cfg(target_os = "none")] +static BITMAP_GRANULES: AtomicUsize = AtomicUsize::new(0); + +/// The window's base and its granule bitmap — this thread must be the +/// task's own, in a task stamped a window. +#[cfg(target_os = "none")] +fn placer() -> Result<(u64, Granules<'static>), ReserveError> { + if crate::entry::geometry::shard_index() != Some(0) { + return Err(ReserveError::Unsupported); + } + let (base, len) = window_span().ok_or(ReserveError::NoWindow)?; + let granules = usize::try_from(len / GRANULE).map_err(|_| ReserveError::NoWindow)?; + if base % GRANULE != 0 || granules == 0 { + return Err(ReserveError::NoWindow); + } + let mut words = BITMAP.load(Ordering::Acquire); + if words.is_null() { + let fresh: alloc::boxed::Box<[AtomicU64]> = (0..granules.div_ceil(64)) + .map(|_| AtomicU64::new(0)) + .collect(); + let fresh = alloc::boxed::Box::into_raw(fresh).cast::(); + // The task's own thread is the only placer, so the exchange cannot + // lose; it is one anyway, so a lost race would only free its own. + match BITMAP.compare_exchange( + core::ptr::null_mut(), + fresh, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + BITMAP_GRANULES.store(granules, Ordering::Release); + words = fresh; + } + Err(winner) => { + // SAFETY: `fresh` came from `Box::into_raw` just above and + // was never published. + drop(unsafe { + alloc::boxed::Box::from_raw(core::ptr::slice_from_raw_parts_mut( + fresh, + granules.div_ceil(64), + )) + }); + words = winner; + } + } + } + let granules = BITMAP_GRANULES.load(Ordering::Acquire); + // SAFETY: the bitmap is published once, never freed, and spans + // `granules.div_ceil(64)` words (allocated for this same window). + let words = unsafe { core::slice::from_raw_parts(words, granules.div_ceil(64)) }; + Ok((base, Granules { words, granules })) +} + +/// Off metal there is no window to place in. +#[cfg(not(target_os = "none"))] +fn placer() -> Result<(u64, Granules<'static>), ReserveError> { + Err(ReserveError::Unsupported) +} + +/// **A bitmap of granules, claimed in runs** — lock-free: a run is claimed +/// word by word, each with one compare-and-swap, and a run that meets a bit +/// claimed meanwhile hands back what it took and looks further. The task's +/// own thread is the only claimant today, so the hand-back never fires; it +/// is what keeps the bitmap sound if it ever is not. +#[derive(Clone, Copy)] +struct Granules<'a> { + words: &'a [AtomicU64], + granules: usize, +} + +impl Granules<'_> { + /// Claim `run` free granules in a row, first fit: the first's index. + fn claim(&self, run: usize) -> Option { + if run == 0 || run > self.granules { + return None; + } + let mut start = 0; + while start + run <= self.granules { + match self.first_set(start, start + run) { + // Something in the way: look past it. + Some(taken) => start = taken + 1, + None => match self.try_claim(start, run) { + Ok(()) => return Some(start), + Err(taken) => start = taken + 1, + }, + } + } + None + } + + /// Release the run `[first, first + run)`. + fn free(&self, first: usize, run: usize) { + for (word, mask) in masks(first, run) { + if let Some(word) = self.words.get(word) { + word.fetch_and(!mask, Ordering::AcqRel); + } + } + } + + /// The first set granule in `[from, to)`, if any. + fn first_set(&self, from: usize, to: usize) -> Option { + masks(from, to - from).find_map(|(word, mask)| { + let hit = self.words[word].load(Ordering::Acquire) & mask; + (hit != 0).then(|| word * 64 + hit.trailing_zeros() as usize) + }) + } + + /// Claim `[first, first + run)` word by word; on meeting a set granule, + /// hand back every word already claimed and answer where it was. + fn try_claim(&self, first: usize, run: usize) -> Result<(), usize> { + for (claimed, (word, mask)) in masks(first, run).enumerate() { + let taken = self.words[word].fetch_or(mask, Ordering::AcqRel) & mask; + if taken != 0 { + // Keep the bits another claimant holds; hand back ours, in + // this word and in every word before it. + self.words[word].fetch_and(!(mask & !taken), Ordering::AcqRel); + for (word, mask) in masks(first, run).take(claimed) { + self.words[word].fetch_and(!mask, Ordering::AcqRel); + } + return Err(word * 64 + taken.trailing_zeros() as usize); + } + } + Ok(()) + } +} + +/// The `(word, mask)` pairs covering granules `[first, first + run)`. +fn masks(first: usize, run: usize) -> impl Iterator { + let end = first + run; + (first / 64..end.div_ceil(64)).map(move |word| { + let lo = first.max(word * 64) - word * 64; + let hi = end.min(word * 64 + 64) - word * 64; + let mask = if hi - lo == 64 { + u64::MAX + } else { + ((1u64 << (hi - lo)) - 1) << lo + }; + (word, mask) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::vec::Vec; + + fn bitmap(granules: usize) -> Vec { + (0..granules.div_ceil(64)) + .map(|_| AtomicU64::new(0)) + .collect() + } + + /// Runs land first fit, span words, refuse when nothing fits, and come + /// back — coalesced with their neighbours — when freed. + #[test] + fn runs_claim_first_fit_and_coalesce_when_freed() { + let words = bitmap(16384); + let g = Granules { + words: &words, + granules: 16384, + }; + // A guarded wasm memory: 65 granules, word-straddling. + let memories: Vec = (0..252).map(|_| g.claim(65).expect("fits")).collect(); + assert_eq!(memories[0], 0); + assert_eq!(memories[1], 65); + assert_eq!(memories[251], 251 * 65); + assert_eq!(g.claim(65), None, "252 memories fill a 1 TiB window"); + assert_eq!( + g.claim(4), + Some(252 * 65), + "the tail still holds a small run" + ); + // Two freed neighbours coalesce into a run neither held alone. + g.free(memories[10], 65); + g.free(memories[11], 65); + assert_eq!(g.claim(130), Some(memories[10])); + assert_eq!(g.claim(1), None); + assert_eq!(g.claim(0), None); + assert_eq!(g.claim(16385), None); + } + + /// A claim that meets granules taken under it hands back what it took. + #[test] + fn a_contended_claim_hands_back_its_partial_run() { + let words = bitmap(256); + let g = Granules { + words: &words, + granules: 256, + }; + // Granule 100 is someone else's: a run over it refuses, leaves 100 + // set, and leaves nothing else. + words[1].fetch_or(1 << (100 - 64), Ordering::Relaxed); + assert_eq!(g.try_claim(60, 65), Err(100)); + let set: u32 = words + .iter() + .map(|w| w.load(Ordering::Relaxed).count_ones()) + .sum(); + assert_eq!(set, 1); + // First fit: the 100 granules below it hold one run; the next + // starts past it. + assert_eq!(g.claim(65), Some(0)); + assert_eq!(g.claim(65), Some(101)); + } + + /// Concurrent claimants never share a granule. + #[test] + fn concurrent_claims_are_disjoint() { + let words = bitmap(4096); + let g = Granules { + words: &words, + granules: 4096, + }; + let runs: Vec> = std::thread::scope(|s| { + let workers: Vec<_> = (0..8) + .map(|_| { + s.spawn(move || { + let mut mine = Vec::new(); + while let Some(first) = g.claim(7) { + mine.push(first); + } + mine + }) + }) + .collect(); + workers.into_iter().map(|w| w.join().unwrap()).collect() + }); + let mut all: Vec = runs.into_iter().flatten().collect(); + all.sort_unstable(); + for pair in all.windows(2) { + assert!(pair[0] + 7 <= pair[1], "two runs overlap"); + } + assert_eq!(all.len(), 4096 / 7); + } + + /// Committed spans merge where they abut and refuse past capacity. + #[test] + fn committed_spans_merge_where_they_abut() { + let none = [None; COMMITTED_RANGES]; + let one = merge(&none, 0x200_0000, 0x201_0000).unwrap(); + let grown = merge(&one, 0x201_0000, 0x240_0000).unwrap(); + assert_eq!(grown.iter().flatten().count(), 1); + assert!(grown.contains(&Some((0x200_0000, 0x240_0000)))); + let below = merge(&grown, 0x100_0000, 0x200_0000).unwrap(); + assert!(below.contains(&Some((0x100_0000, 0x240_0000)))); + let mut spans = none; + for i in 0..COMMITTED_RANGES as u64 { + spans = merge(&spans, i * 0x2000, i * 0x2000 + 0x1000).unwrap(); + } + assert_eq!(merge(&spans, 0x10_0000, 0x10_1000), None, "full"); + assert!( + merge(&spans, 0x1000, 0x2000).is_some(), + "a join needs no slot" + ); + } + + /// The gaps of a span are what no committed span covers, in order. + #[test] + fn gaps_are_the_uncovered_parts_in_order() { + let mut spans = [None; COMMITTED_RANGES]; + spans[3] = Some((0x4000, 0x6000)); + spans[0] = Some((0x1000, 0x2000)); + let found: Vec<(u64, u64)> = gaps(&spans, 0, 0x8000).into_iter().flatten().collect(); + assert_eq!(found, [(0, 0x1000), (0x2000, 0x4000), (0x6000, 0x8000)]); + assert_eq!(gaps(&spans, 0x4000, 0x6000).iter().flatten().count(), 0); + let inside: Vec<(u64, u64)> = gaps(&spans, 0x1800, 0x5000).into_iter().flatten().collect(); + assert_eq!(inside, [(0x2000, 0x4000)]); + let none = [None; COMMITTED_RANGES]; + let whole: Vec<(u64, u64)> = gaps(&none, 0x3000, 0x9000).into_iter().flatten().collect(); + assert_eq!(whole, [(0x3000, 0x9000)]); + } + + /// Off the kore target there is no window: reserving refuses. + #[test] + fn off_target_nothing_reserves() { + assert_eq!( + Reservation::new(GRANULE).err(), + Some(ReserveError::Unsupported) + ); + } +} diff --git a/native/crates/rt/src/service/mod.rs b/native/crates/rt/src/service/mod.rs index 17ee8f4..bf11006 100644 --- a/native/crates/rt/src/service/mod.rs +++ b/native/crates/rt/src/service/mod.rs @@ -292,6 +292,21 @@ pub trait SystemService: crate::svc::SvcProtocol + Sized { None } + /// Named metadata arena for a service whose **recyclable peers** — the + /// per-launch faces exec admits — must outnumber the bootstrap table. + /// + /// [`MAX_SERVE_PAIRS`](crate::svc::MAX_SERVE_PAIRS) bounds the manifest's + /// fixed roster, not a population: without an arena the host keeps its + /// allocation-free inline table, and every fixed pair it serves is a face + /// slot fewer. A service that hands one face to each program of a kind + /// that can run many at once returns its arena here, and the serving + /// ladder grows the host's peer table from it on demand, up to the + /// [`PeerId`](crate::svc::PeerId) slot space. + #[cfg(feature = "heap")] + fn peer_arena(&self) -> Option<&'static (dyn core::alloc::Allocator + Sync)> { + None + } + /// **What this row's `role::DEVICE` endowments select at founding** — /// entry `n` is the selector for the row's `role::DEVICE` index `n`, so /// a driver that owns two controllers (the input service: an xHCI diff --git a/native/crates/rt/src/session/builder.rs b/native/crates/rt/src/session/builder.rs index f4da552..fa60e93 100644 --- a/native/crates/rt/src/session/builder.rs +++ b/native/crates/rt/src/session/builder.rs @@ -8,7 +8,7 @@ use kore_abi::session::{ DERIVE_KIND_IOVA_ANON, DERIVE_KIND_IRQ_ROUTE_MSIX, DERIVE_KIND_PEER_IOVA, DESCRIPTOR_ABI, FoundReq, GuestFoundReq, I2C_HID_QUERY_COUNT, I2cHidReq, INVENTORY_QUERY_COUNT, IRQ_AFFINITY_ANY, InventoryReq, Invocation, IrqReq, MAX_FOUND_DELIVERIES, MAX_INVOCATIONS, - OpCode, SessionFlags, SlotRef, + OpCode, SessionFlags, SlotRef, WakeWordReq, WordWaitReq, }; use kore_abi::task::{SpawnFlags, SpawnReq}; @@ -28,11 +28,12 @@ pub use kore_abi::session::MAX_SESSION_FOUNDS; const ARGS_MAX: usize = MAX_INVOCATIONS * 24 + MAX_SESSION_FOUNDS * (FoundReq::WIRE - 24); /// Descriptor bytes a full session needs — the buffer floor for -/// [`SessionBuilder::finish`]. The `+ 32` is the park block ceiling that -/// can accompany invocations (`ParkReq` 16 + `SleepReq` 8 + `IrqParkReq` 8). The larger -/// two-post block is restricted to an empty session and is only 48 bytes -/// including the header, so it does not raise this full-descriptor bound. -pub const DESCRIPTOR_MAX: usize = 8 + MAX_INVOCATIONS * 16 + 32 + ARGS_MAX; +/// [`SessionBuilder::finish`]. The `+ 48` is the park block ceiling that +/// can accompany invocations (`ParkReq` 16 + `SleepReq` 8 + `IrqParkReq` 8 +/// + `WordWaitReq` 16). The larger two-post block is restricted to an empty +/// session and is only 48 bytes including the header, so it does not raise +/// this full-descriptor bound. +pub const DESCRIPTOR_MAX: usize = 8 + MAX_INVOCATIONS * 16 + 48 + ARGS_MAX; // A full descriptor must fit the two-page `[layout]` scratch window // (`scratch = { bytes = 0x2000 }` in res/boot.toml), priced with @@ -78,6 +79,12 @@ pub enum BuildError { /// The bounded standing-post wait is malformed: an alternate post lacks /// its primary, aliases the same channel, or accompanies invocations. BadParkShape, + /// A [`wait_word`](SessionBuilder::wait_word) arm without a + /// [`wait_sleep`](SessionBuilder::wait_sleep) deadline: the kernel + /// address-wait is always bounded (`SessionFlags::WAIT_WORD` requires + /// `WAIT_SLEEP`), so the shape is refused here, before anything is + /// staged against the wire. + UnboundedWordWait, /// TASK_EXIT requires the typed [`SessionBuilder::exit`] entry point. UntypedExit, } @@ -224,6 +231,10 @@ pub struct SessionBuilder { sleep: Option, /// Exact held route, emitted after the other park payloads. irq: Option, + /// The watched memory word ([`wait_word`](Self::wait_word)) — `(subject, + /// offset, expected)`, emitted as the park block's **last** member, the + /// `WordWaitReq`. + word: Option<(SlotRef, u32, u32)>, /// `Found`s staged so far — refused past [`MAX_SESSION_FOUNDS`], the /// argument region's exact founding capacity. founds: usize, @@ -243,6 +254,7 @@ impl SessionBuilder { park_alt: None, sleep: None, irq: None, + word: None, founds: 0, } } @@ -301,6 +313,23 @@ impl SessionBuilder { self } + /// Park after commit until the `u32` at byte `offset` inside the held + /// memory `subject` (a lease or mapped view, by arena slot) is seen to + /// differ from `expected` — [`SessionFlags::WAIT_WORD`], the kernel + /// address-wait (`res/EVENT.md` §6). The kernel keys the wait on the + /// word's *physical* identity, so a [`wake_word`](Self::wake_word) on + /// any mapping of the same memory resolves it. Always bounded: + /// [`encoded_len`](Self::encoded_len) refuses the shape without a + /// [`wait_sleep`](Self::wait_sleep) deadline. Composes with a standing + /// post ([`park_standing`](Self::park_standing)) and a route + /// ([`wait_irq`](Self::wait_irq)); the `WordWaitReq` is the park + /// block's last member. + pub fn wait_word(&mut self, subject: SlotRef, offset: u32, expected: u32) -> &mut Self { + self.flags |= SessionFlags::WAIT_WORD; + self.word = Some((subject, offset, expected)); + self + } + /// Park after commit on the session's staged **join** until the joined /// task exits or is reaped ([`SessionFlags::WAIT_TASK`]) — the /// blocking join. Without it a staged [`join`](Self::join) is a @@ -554,6 +583,37 @@ impl SessionBuilder { .map(|_| ()) } + /// `Derive × MemLease × WAKE_WORD` — **release up to `count` waiters** + /// parked on the `u32` at byte `offset` of the held memory `subject` + /// (`MODIFY`: you may wake what you can write), on every mapping of + /// that memory (`res/EVENT.md` §6, the producer half of the kernel + /// address-wait). The store itself is the caller's and must precede + /// this effect. Mints nothing — the result slot is the subject's own, + /// inert, exactly [`activate_device`](Self::activate_device)'s + /// convention — and must stand alone as a one-invocation session. + pub fn wake_word( + &mut self, + subject: SlotRef, + offset: u32, + count: u32, + ) -> Result<(), BuildError> { + if self.count != 0 || !self.flags.is_empty() { + return Err(BuildError::EffectMustStandAlone); + } + let mut req = [0u8; WakeWordReq::WIRE]; + req[0..4].copy_from_slice(&offset.to_le_bytes()); + req[4..8].copy_from_slice(&count.to_le_bytes()); + self.derive_raw( + subject, + Rights::empty(), + kore_abi::session::DERIVE_KIND_WAKE_WORD, + // No result is assigned; the authenticated subject slot is inert. + subject, + Some(&req), + ) + .map(|_| ()) + } + /// Park after commit on the timer wheel until `deadline_ns` (absolute /// monotonic nanoseconds) — [`SessionFlags::WAIT_SLEEP`] /// (`res/USERLAND.md` §3.4.2). [`finish`](Self::finish) emits the @@ -612,7 +672,8 @@ impl SessionBuilder { || inv.kind == kore_abi::session::DERIVE_KIND_TASK_START || inv.kind == kore_abi::session::DERIVE_KIND_ARENA_EXTEND || inv.kind == kore_abi::session::DERIVE_KIND_DEVICE_ACTIVATE - || inv.kind == kore_abi::session::DERIVE_KIND_DEVICE_QUIESCE); + || inv.kind == kore_abi::session::DERIVE_KIND_DEVICE_QUIESCE + || inv.kind == kore_abi::session::DERIVE_KIND_WAKE_WORD); self.invs[self.count] = inv; self.count += 1; self.has_effect = is_effect; @@ -1029,13 +1090,65 @@ impl SessionBuilder { /// Exercise a detached view in a receiver-owned, founder-stamped window. /// The held or same-session view needs ACCESS, not delegation authority. /// No rights, length or destination address space are caller-selectable. + /// `SURFACES` takes its first lane and `RESERVE` its first page; the + /// other placements have their own methods. pub fn map_view_in( &mut self, view: impl Into, window: kore_abi::session::ViewMapWindow, result: SlotRef, ) -> Result { - self.map_view_request(view, window, 0, result) + self.map_view_request(view, window, 0, 0, result) + } + + /// Commit a view at an exact page of the caller's `RESERVE` window + /// ([`ViewMapWindow::Reserve`](kore_abi::session::ViewMapWindow::Reserve)): + /// the first page of `[page, page + len)`, in + /// [`RESERVE_PAGE_BYTES`](kore_abi::session::RESERVE_PAGE_BYTES) units. + /// The kernel refuses a page outside the window or one already mapped; + /// authority and length still come from the view. + pub fn map_view_reserved( + &mut self, + view: impl Into, + page: u32, + result: SlotRef, + ) -> Result { + self.map_view_request( + view, + kore_abi::session::ViewMapWindow::Reserve, + 0, + page, + result, + ) + } + + /// Release `[page, page + pages)` of the caller's `RESERVE` window: + /// every view `lessor` (a lease held with `MODIFY`, or a lessor beneath + /// it) was committed as there is unmapped + /// ([`DERIVE_KIND_RESERVE_RELEASE`](kore_abi::session::DERIVE_KIND_RESERVE_RELEASE)). + /// A post-commit effect, so it stands alone, as + /// [`wake_word`](Self::wake_word) does. + pub fn release_reserved( + &mut self, + lessor: SlotRef, + page: u32, + pages: u32, + ) -> Result<(), BuildError> { + if self.count != 0 || !self.flags.is_empty() { + return Err(BuildError::EffectMustStandAlone); + } + let mut req = [0u8; kore_abi::session::ReserveReleaseReq::WIRE]; + req[..4].copy_from_slice(&page.to_le_bytes()); + req[4..].copy_from_slice(&pages.to_le_bytes()); + self.derive_raw( + lessor, + Rights::empty(), + kore_abi::session::DERIVE_KIND_RESERVE_RELEASE, + // No result is assigned; the authenticated subject slot is inert. + lessor, + Some(&req), + ) + .map(|_| ()) } /// Map a held view into the `MAP` window as a **sink** mapping @@ -1071,6 +1184,7 @@ impl SessionBuilder { view, kore_abi::session::ViewMapWindow::Surfaces, lane as u8, + 0, result, ) } @@ -1080,11 +1194,13 @@ impl SessionBuilder { view: impl Into, window: kore_abi::session::ViewMapWindow, lane: u8, + page: u32, result: SlotRef, ) -> Result { let mut req = [0u8; kore_abi::session::ViewMapReq::WIRE]; req[..2].copy_from_slice(&window.role().to_le_bytes()); req[2] = lane; + req[4..8].copy_from_slice(&page.to_le_bytes()); self.derive_raw( view.into(), Rights::empty(), @@ -2530,6 +2646,16 @@ impl SessionBuilder { Ok(Staged(result)) } + /// Bytes of the park block the flags describe, in the kore-abi pinned + /// order: `ParkReq ‖ ParkReq(alt) ‖ SleepReq ‖ IrqParkReq ‖ WordWaitReq`. + fn park_block_len(&self) -> usize { + self.park.map_or(0, |_| 16) + + self.park_alt.map_or(0, |_| 16) + + self.sleep.map_or(0, |_| 8) + + self.irq.map_or(0, |_| kore_abi::session::IrqParkReq::WIRE) + + self.word.map_or(0, |_| WordWaitReq::WIRE) + } + /// Exact byte length of the frozen descriptor this builder will emit. /// /// **The park block** (the kore-abi flag-gated convention): a @@ -2550,10 +2676,10 @@ impl SessionBuilder { return Err(BuildError::BadParkShape); } } - let block = self.park.map_or(0, |_| 16) - + self.park_alt.map_or(0, |_| 16) - + self.sleep.map_or(0, |_| 8) - + self.irq.map_or(0, |_| kore_abi::session::IrqParkReq::WIRE); + if self.word.is_some() && self.sleep.is_none() { + return Err(BuildError::UnboundedWordWait); + } + let block = self.park_block_len(); Ok(8 + self.count * 16 + block + self.args_len) } @@ -2566,10 +2692,7 @@ impl SessionBuilder { /// carrier before writing a byte; both paths then use this one encoder. pub fn finish(&self, out: &mut [u8]) -> Result { let need = self.encoded_len()?; - let block = self.park.map_or(0, |_| 16) - + self.park_alt.map_or(0, |_| 16) - + self.sleep.map_or(0, |_| 8) - + self.irq.map_or(0, |_| kore_abi::session::IrqParkReq::WIRE); + let block = self.park_block_len(); if out.len() < need { return Err(BuildError::OutTooSmall); } @@ -2617,6 +2740,14 @@ impl SessionBuilder { out[at + 4..at + 8].fill(0); at += kore_abi::session::IrqParkReq::WIRE; } + if let Some((subject, offset, expected)) = self.word { + // WordWaitReq: subject@0 ‖ offset@4 ‖ expected@8 ‖ reserved@12. + out[at..at + 4].copy_from_slice(&subject.index().to_le_bytes()); + out[at + 4..at + 8].copy_from_slice(&offset.to_le_bytes()); + out[at + 8..at + 12].copy_from_slice(&expected.to_le_bytes()); + out[at + 12..at + 16].fill(0); + at += WordWaitReq::WIRE; + } out[at..at + self.args_len].copy_from_slice(&self.args[..self.args_len]); Ok(need) } diff --git a/native/crates/rt/src/session/tests.rs b/native/crates/rt/src/session/tests.rs index 409c73a..8446618 100644 --- a/native/crates/rt/src/session/tests.rs +++ b/native/crates/rt/src/session/tests.rs @@ -27,6 +27,7 @@ fn view_placement_requests_only_a_window_not_more_authority() { ViewMapWindow::Map, ViewMapWindow::Surfaces, ViewMapWindow::Heap, + ViewMapWindow::Reserve, ] { let mut builder = SessionBuilder::new(); builder @@ -47,7 +48,7 @@ fn view_placement_requests_only_a_window_not_more_authority() { assert_eq!(req.placement(), Some(window)); assert_eq!(req.lane, 0); assert_eq!(req.flags, 0); - assert_eq!(req.reserved, [0; 4]); + assert_eq!(req.page.get(), 0); assert!(!req.sink()); } @@ -65,7 +66,7 @@ fn view_placement_requests_only_a_window_not_more_authority() { assert_eq!(req.placement(), Some(ViewMapWindow::Map)); assert!(req.sink()); assert_eq!(req.flags, kore_abi::session::VIEW_MAP_SINK); - assert_eq!(req.reserved, [0; 4]); + assert_eq!(req.page.get(), 0); let mut builder = SessionBuilder::new(); builder @@ -84,6 +85,54 @@ fn view_placement_requests_only_a_window_not_more_authority() { req.surface_lane(), Some(kore_abi::session::SurfaceMapLane::Second) ); + + // A reservation commit names the window and its page, nothing more. + let mut builder = SessionBuilder::new(); + builder + .map_view_reserved(SlotRef::from_wire(2), 0x10_0400, SlotRef::from_wire(3)) + .unwrap(); + let mut out = [0u8; DESCRIPTOR_MAX]; + let len = builder.finish(&mut out).unwrap(); + let (_, invs, args) = parse(&out[..len]); + assert_eq!(invs[0].kind, DERIVE_KIND_VIEW_MAP); + assert_eq!(invs[0].requested, 0); + let (req, _) = ViewMapReq::try_ref_from_prefix(args).unwrap(); + assert_eq!(req.placement(), Some(ViewMapWindow::Reserve)); + assert_eq!(req.reserve_page(), Some(0x10_0400)); + assert!(!req.sink()); +} + +#[test] +fn a_reservation_release_names_its_lessor_and_range_only() { + use kore_abi::session::{DERIVE_KIND_RESERVE_RELEASE, ReserveReleaseReq}; + use zerocopy::TryFromBytes; + + let mut builder = SessionBuilder::new(); + builder + .release_reserved(SlotRef::from_wire(4), 0x10_0000, 0x10_4000) + .unwrap(); + let mut out = [0u8; DESCRIPTOR_MAX]; + let len = builder.finish(&mut out).unwrap(); + let (_, invs, args) = parse(&out[..len]); + assert_eq!(invs.len(), 1); + assert_eq!(invs[0].op.get(), OpCode::Derive as u16); + assert_eq!(invs[0].kind, DERIVE_KIND_RESERVE_RELEASE); + assert_eq!(invs[0].requested, 0); + assert_eq!(invs[0].subject.get(), 4); + assert_eq!(invs[0].result.get(), 4, "no result is assigned"); + let (req, rest) = ReserveReleaseReq::try_ref_from_prefix(args).unwrap(); + assert!(rest.is_empty()); + assert_eq!((req.page.get(), req.pages.get()), (0x10_0000, 0x10_4000)); + + // A post-commit effect stands alone. + let mut builder = SessionBuilder::new(); + builder + .map_view_reserved(SlotRef::from_wire(2), 0, SlotRef::from_wire(3)) + .unwrap(); + assert_eq!( + builder.release_reserved(SlotRef::from_wire(4), 0, 1), + Err(BuildError::EffectMustStandAlone) + ); } #[test] @@ -2111,6 +2160,171 @@ fn selected_irq_is_encoded_after_deadline_and_preserves_argument_offsets() { } } +/// The word arm (`res/EVENT.md` §6) rides the park block as its **last** +/// member — after `ParkReq ‖ SleepReq` (and `IrqParkReq` when a route is +/// selected) — whatever order the methods were called in, and every +/// staged argument is relocated past the whole block. +#[test] +fn word_wait_is_encoded_last_in_the_park_block_and_relocates_args() { + use kore_abi::session::{IrqParkReq, NoticeReq, ParkReq, SleepReq, WordWaitReq}; + use zerocopy::TryFromBytes; + for route in [None, Some(19)] { + let mut b = SessionBuilder::new(); + b.notice(SlotRef::from_wire(2), 0xab, 0xcd).unwrap(); + // Word first, deliberately: method order must not move it. + b.wait_word(SlotRef::from_wire(9), 64, 0x1122_3344); + if let Some(route) = route { + b.wait_irq_on(SlotRef::from_wire(route)); + } + b.park_standing(11, 4, 23).wait_sleep(9_000); + let mut bytes = [0u8; DESCRIPTOR_MAX]; + let n = b.finish(&mut bytes).unwrap(); + assert_eq!(n, b.encoded_len().unwrap()); + let (header, invs, args) = parse(&bytes[..n]); + let mut flags = SessionFlags::WAIT_DELIVERY | SessionFlags::WAIT_SLEEP | SessionFlags::WAIT_WORD; + if route.is_some() { + flags |= SessionFlags::WAIT_IRQ | SessionFlags::WAIT_IRQ_SELECTED; + } + assert_eq!(header.flags.get(), flags.bits()); + let (park, tail) = ParkReq::try_ref_from_prefix(args).unwrap(); + assert_eq!((park.chan.get(), park.slot.get(), park.recv_tag.get()), (11, 4, 23)); + let (sleep, mut tail) = SleepReq::try_ref_from_prefix(tail).unwrap(); + assert_eq!(sleep.deadline_ns.get(), 9_000); + if let Some(route) = route { + let (irq, rest) = IrqParkReq::try_ref_from_prefix(tail).unwrap(); + assert_eq!(irq.slot.get(), route); + tail = rest; + } + let (word, _) = WordWaitReq::try_ref_from_prefix(tail).unwrap(); + assert_eq!(word.subject.get(), 9); + assert_eq!(word.offset.get(), 64); + assert_eq!(word.expected.get(), 0x1122_3344); + assert_eq!(word.reserved.get(), 0); + let block = 16 + 8 + route.map_or(0, |_| IrqParkReq::WIRE) + WordWaitReq::WIRE; + assert_eq!(invs[0].arg.get() as usize, block, "arg relocated past the block"); + let (notice, _) = NoticeReq::try_ref_from_prefix(&args[block..]).unwrap(); + assert_eq!((notice.cookie.get(), notice.payload.get()), (0xab, 0xcd)); + } +} + +/// The executor's word park — header 8 + `ParkReq` 16 + `SleepReq` 8 + +/// `WordWaitReq` 16 — is exactly 48 bytes: it fits the x86-64 caller-saved +/// GPR carrier with nothing to spare, so the rung costs no descriptor +/// scratch (the same budget the two-post park was priced against). The +/// bare word park (no doorbell) is 32. +#[test] +fn word_park_shapes_fit_the_inline_carrier_exactly() { + for doorbell in [false, true] { + let mut b = SessionBuilder::new(); + if doorbell { + b.park_standing(7, 5, 0x1234); + } + b.wait_word(SlotRef::from_wire(9), 4, 1).wait_sleep(42); + let expect = if doorbell { 48 } else { 32 }; + assert_eq!(b.encoded_len().unwrap(), expect); + let mut bytes = [0u8; 48]; + let n = b.finish(&mut bytes).unwrap(); + assert_eq!(n, expect); + let (_, invs, args) = parse(&bytes[..n]); + assert!(invs.is_empty()); + let off = if doorbell { 16 + 8 } else { 8 }; + assert_eq!(u32::from_le_bytes(args[off..off + 4].try_into().unwrap()), 9); + assert_eq!(u32::from_le_bytes(args[off + 4..off + 8].try_into().unwrap()), 4); + assert_eq!(u32::from_le_bytes(args[off + 8..off + 12].try_into().unwrap()), 1); + } +} + +/// `WAIT_WORD` requires `WAIT_SLEEP`: the shape is refused before a byte +/// is written, at `encoded_len` (the transport's carrier choice) and so at +/// `finish`. +#[test] +fn a_word_wait_without_a_deadline_is_refused_before_encoding() { + let mut b = SessionBuilder::new(); + b.wait_word(SlotRef::from_wire(9), 4, 1); + assert_eq!(b.encoded_len(), Err(BuildError::UnboundedWordWait)); + let mut bytes = [0u8; DESCRIPTOR_MAX]; + assert_eq!(b.finish(&mut bytes), Err(BuildError::UnboundedWordWait)); + // A standing post does not bound it either. + b.park_standing(7, 5, 0x1234); + assert_eq!(b.encoded_len(), Err(BuildError::UnboundedWordWait)); + // The deadline does. + b.wait_sleep(1); + assert!(b.encoded_len().is_ok()); +} + +/// `wake_word` lowers onto the effect derive exactly as `activate_device` +/// does: the subject's own slot as the inert result, no rights, the +/// `WakeWordReq { offset, count }` bytes as its argument — and it stands +/// alone. +#[test] +fn wake_word_stages_the_standalone_effect_with_its_payload() { + use kore_abi::session::{DERIVE_KIND_WAKE_WORD, WakeWordReq}; + use zerocopy::TryFromBytes; + let mut b = SessionBuilder::new(); + b.wake_word(SlotRef::from_wire(13), 64, u32::MAX).unwrap(); + let mut bytes = [0u8; DESCRIPTOR_MAX]; + let n = b.finish(&mut bytes).unwrap(); + assert_eq!(n, 8 + 16 + WakeWordReq::WIRE); + let (header, invs, args) = parse(&bytes[..n]); + assert_eq!(header.flags.get(), 0); + assert_eq!(invs.len(), 1); + assert_eq!(invs[0].op.get(), OpCode::Derive as u16); + assert_eq!(invs[0].kind, DERIVE_KIND_WAKE_WORD); + assert_eq!(invs[0].requested, 0); + assert_eq!(invs[0].subject.get(), 13); + assert_eq!(invs[0].result.get(), 13, "inert: the subject's own slot"); + let (req, _) = WakeWordReq::try_ref_from_prefix(&args[invs[0].arg.get() as usize..]).unwrap(); + assert_eq!((req.offset.get(), req.count.get()), (64, u32::MAX)); + + // An effect: nothing may follow it, and it may not follow anything. + assert_eq!( + b.notice(SlotRef::from_wire(2), 1, 2), + Err(BuildError::EffectMustStandAlone) + ); + let mut after = SessionBuilder::new(); + after.notice(SlotRef::from_wire(2), 1, 2).unwrap(); + assert_eq!( + after.wake_word(SlotRef::from_wire(13), 0, 1), + Err(BuildError::EffectMustStandAlone) + ); + let mut flagged = SessionBuilder::new(); + flagged.wait_sleep(1); + assert_eq!( + flagged.wake_word(SlotRef::from_wire(13), 0, 1), + Err(BuildError::EffectMustStandAlone) + ); +} + +/// `encoded_len` and `finish` agree on every word shape the runtime emits, +/// so the transport's carrier choice is never wrong by the block. +#[test] +fn encoded_len_matches_finish_for_every_word_shape() { + let shapes: [fn(&mut SessionBuilder); 4] = [ + |b| { + b.wait_word(SlotRef::from_wire(9), 4, 1).wait_sleep(1); + }, + |b| { + b.park_standing(7, 5, 3).wait_word(SlotRef::from_wire(9), 4, 1).wait_sleep(1); + }, + |b| { + b.park_standing(7, 5, 3) + .wait_irq_on(SlotRef::from_wire(19)) + .wait_word(SlotRef::from_wire(9), 4, 1) + .wait_sleep(1); + }, + |b| { + b.notice(SlotRef::from_wire(2), 1, 2).unwrap(); + b.park_standing(7, 5, 3).wait_word(SlotRef::from_wire(9), 4, 1).wait_sleep(1); + }, + ]; + for shape in shapes { + let mut b = SessionBuilder::new(); + shape(&mut b); + let mut bytes = [0u8; DESCRIPTOR_MAX]; + assert_eq!(b.finish(&mut bytes).unwrap(), b.encoded_len().unwrap()); + } +} + #[test] fn selected_irq_only_and_deadline_waits_fit_the_inline_carrier() { for deadline in [None, Some(42)] { diff --git a/native/crates/rt/src/spmd.rs b/native/crates/rt/src/spmd.rs index 4efe076..135b2c9 100644 --- a/native/crates/rt/src/spmd.rs +++ b/native/crates/rt/src/spmd.rs @@ -246,6 +246,10 @@ pub struct SpmdCtx { /// or when this same static context later founds another resident epoch. /// It is deliberately not reset by [`restage`](Self::restage). resident_sequence: core::sync::atomic::AtomicU64, + /// The kernel's placement of this region's gang, as reported by the + /// first worker through the gate — `0` unknown, else `1 | class << 1 | + /// shared << 3` ([`SpmdCtx::placement`]). + placement: core::sync::atomic::AtomicU32, } impl SpmdCtx { @@ -290,9 +294,28 @@ impl SpmdCtx { done: Line::new(0), complete: Line::new(0), resident_sequence: core::sync::atomic::AtomicU64::new(0), + placement: core::sync::atomic::AtomicU32::new(0), } } + /// Where the kernel placed this region's gang — the class and whether + /// it is time-shared — as the first worker through the gate reported + /// it from its own machine block. `None` before any worker entered, + /// off metal, and under a kernel that predates the report; a caller + /// then sizes and labels as it always did. + pub fn placement(&self) -> Option { + let word = self.placement.load(Ordering::Acquire); + (word & 1 != 0).then(|| kore_abi::machine::GangPlacement { + class: match (word >> 1) & 0b11 { + 0 => kore_abi::machine::GangClass::Performance, + 1 => kore_abi::machine::GangClass::Standard, + 2 => kore_abi::machine::GangClass::Efficient, + _ => kore_abi::machine::GangClass::LowPowerEfficient, + }, + shared: word & (1 << 3) != 0, + }) + } + /// The launched worker count — what a worker strides the tile grid /// by. The value a launch armed (== the constructed count for a /// const-width region); before any launch, the constructed capacity. @@ -435,6 +458,18 @@ impl SpmdCtx { /// wait is the barrier's own two-tier slice ([`wait_while`]), so a /// staged worker costs a parked thread, not a hot core. pub(crate) fn enter(ctx: &'static SpmdCtx) -> bool { + // Report the placement the kernel stamped into THIS worker's machine + // block (the whole gang shares one batch decision, so any member's + // answer is the region's). One relaxed store per worker, before the + // gate: the founder reads it after the region is up. + #[cfg(target_os = "none")] + if let Some(placed) = crate::entry::thread_context() + .and_then(|rt| rt.machine()) + .and_then(|block| block.gang_placement()) + { + let word = 1 | ((placed.class as u32) << 1) | (u32::from(placed.shared) << 3); + ctx.placement.store(word, Ordering::Release); + } wait_while(&ctx.gate.0, STAGED); ctx.gate.0.load(Ordering::SeqCst) == RUN } @@ -651,6 +686,21 @@ pub fn spawn_spmd( launch::(ctx, entry, N, 0, core::ptr::null_mut(), true) } +/// The heap a region's compute workers are founded holding: the +/// process's own, when this binary links the allocator, so code a job +/// runs may allocate on a worker exactly as on thread 0. A heapless +/// binary's workers take none — there is no allocator for them to grow. +#[cfg(all(target_os = "none", feature = "heap"))] +fn worker_heap() -> Option { + crate::thread::heap_endowment() +} + +/// Heapless: nothing on a worker can allocate, so nothing is endowed. +#[cfg(all(target_os = "none", not(feature = "heap")))] +fn worker_heap() -> Option { + None +} + /// The one metal launch, over a runtime `active ≤ N` width. /// /// `N` is the storage capacity ([`Spmd`]'s array size); `active` is the @@ -710,6 +760,10 @@ fn launch( } else { crate::session::MAX_SESSION_FOUNDS }; + // Every compute worker runs caller code, so each is founded holding + // the process heap (`thread::spawn_gang_workers`): asked once here, + // derived per worker in its founding. + let heap = worker_heap(); let mut ok = true; for (at, end) in founding_sessions(active as u32, ceiling as u32) { let founded = crate::thread::spawn_gang_workers( @@ -719,6 +773,7 @@ fn launch( true, &[], star_plan.as_ref(), + heap, &mut workers[at as usize..end as usize], ); if !founded { @@ -761,6 +816,7 @@ fn launch( false, &[&hub_caps[..filled]], None, + None, &mut hub_out, ) }) @@ -856,8 +912,12 @@ pub fn resident_run(ctx: &'static SpmdCtx, worker: // is kicked by the submit over the founder's send. A long // monitored spin here is NOT an option: a spinning worker holds its // core against the kernel until the tick, and the launcher may be - // sharing it (`res/SPMD.md` §14.2). The word is the truth. - crate::barrier::wait_while(ctx.job_word(), ran); + // sharing it (`res/SPMD.md` §14.2). Nor is riding the phase + // slice: between jobs no kick is coming, and a quarter-millisecond + // re-check held three cores of a four-CPU machine at idle + // (`barrier::IDLE_SLICE_MAX`). The slice climbs while the word + // stays put and starts over with each job. The word is the truth. + crate::barrier::wait_while_idle(ctx.job_word(), ran); // A retire flips the gate off RUN, THEN steps `job` to wake us; the // SeqCst total order makes the DONE visible once the step that woke // us is. @@ -2230,8 +2290,13 @@ mod tests { let meshed = crate::star::MESH_CEILING as u32; for ceiling in [meshed, crate::session::MAX_SESSION_FOUNDS as u32] { for width in 1..crate::thread::MAX_THREADS { - let sessions: std::vec::Vec<(u32, u32)> = founding_sessions(width, ceiling).collect(); - assert_eq!(sessions.len() as u32, width.div_ceil(ceiling), "width {width}"); + let sessions: std::vec::Vec<(u32, u32)> = + founding_sessions(width, ceiling).collect(); + assert_eq!( + sessions.len() as u32, + width.div_ceil(ceiling), + "width {width}" + ); let mut next = 0; for &(start, end) in &sessions { assert_eq!(start, next, "width {width}: sessions are contiguous"); diff --git a/native/crates/rt/src/star.rs b/native/crates/rt/src/star.rs index 56c060a..7be7730 100644 --- a/native/crates/rt/src/star.rs +++ b/native/crates/rt/src/star.rs @@ -188,15 +188,16 @@ pub(crate) fn provision(workers: usize) -> Option { /// Descriptor invocations one meshed worker costs its gang session in a /// chunk of `chunk`: two views, the boot delegate and the Found, plus its -/// endowment of `chunk + 2` caps (own recv, hub send, the founder's send, -/// and a peer send per chunk sibling). +/// endowment of `chunk + 3` caps (own recv, hub send, the founder's send, +/// a peer send per chunk sibling, and the process heap every compute +/// worker holds — `thread::spawn_gang_workers`). const fn meshed_worker_invocations(chunk: usize) -> usize { - 4 + chunk + 2 + 4 + chunk + 3 } /// The widest chunk one FULLY MESHED gang session can carry — derived /// from the descriptor's bounds, never chosen: a chunk of `n` costs -/// `n·(n + 6)` invocations against [`MAX_INVOCATIONS`], and no session +/// `n·(n + 7)` invocations against [`MAX_INVOCATIONS`], and no session /// founds more than [`MAX_SESSION_FOUNDS`] (seven, at today's 107 and 12). /// A wider region founds in several sessions (see [`spmd::launch`]): the /// mesh spans a session's chunk, and a cross-chunk waiter is the hub's to diff --git a/native/crates/rt/src/svc/client.rs b/native/crates/rt/src/svc/client.rs index c230d29..8d187dc 100644 --- a/native/crates/rt/src/svc/client.rs +++ b/native/crates/rt/src/svc/client.rs @@ -328,6 +328,32 @@ pub enum CallError { Timeout, } +/// How long a lifecycle owner watches one exact control request's reply +/// ([`SvcClient::call_exact`]). +/// +/// A pair carries one request at a time and its replies name no request, so +/// a request whose observation expires stays published, owned by whoever +/// published it, and every other request on the pair answers +/// [`CallError::Outstanding`] until that owner carries it to its reply. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Observation { + /// Until the reply lands: the pair is idle again on return, unless the + /// transport itself failed. + Settle, + /// At most this many nanoseconds. Expiry is [`CallError::Timeout`] and + /// leaves the exact request published for its owner's next call. + Slice(u64), +} + +impl Observation { + const fn budget_ns(self) -> Option { + match self { + Self::Settle => None, + Self::Slice(budget_ns) => Some(budget_ns), + } + } +} + /// Failure of an authority-bearing call, split by proven source custody. /// /// A non-`Ok` commit is not generally proof that the source survived: an OCC @@ -938,7 +964,7 @@ fn classify_request_park_commit(committed: Committed, deadline_armed: bool) -> R let raw_status = committed.raw() as u16; let status = committed.status(); let failing = committed.failing_index(); - if raw_status > Status::FabricRouteCapability as u16 || failing != 0 { + if raw_status > Status::Word as u16 || failing != 0 { return RequestParkCommit::Terminal(CallError::Refused(Status::Malformed)); } match status { @@ -960,7 +986,8 @@ fn classify_request_park_commit(committed: Committed, deadline_armed: bool) -> R | Status::FabricRouteWrongAccess | Status::FabricRouteInfeasible | Status::FabricRouteCapacity - | Status::FabricRouteCapability) => RequestParkCommit::Terminal(CallError::Refused(status)), + | Status::FabricRouteCapability + | Status::Word) => RequestParkCommit::Terminal(CallError::Refused(status)), } } @@ -992,7 +1019,7 @@ fn classify_return_commit( } else { failing >= invocations }; - if invocations == 0 || raw_status > Status::FabricRouteCapability as u16 || impossible_index { + if invocations == 0 || raw_status > Status::Word as u16 || impossible_index { // Unknown status vocabulary, an empty transfer, or an impossible // invocation index cannot prove that the affine move did not run. return ReturnCommit::AfterTransfer(CallError::Refused(Status::Malformed)); @@ -1029,7 +1056,8 @@ fn classify_return_commit( | Status::FabricRouteWrongAccess | Status::FabricRouteInfeasible | Status::FabricRouteCapacity - | Status::FabricRouteCapability) => ReturnCommit::AfterTransfer(CallError::Refused(status)), + | Status::FabricRouteCapability + | Status::Word) => ReturnCommit::AfterTransfer(CallError::Refused(status)), } } @@ -1069,7 +1097,8 @@ fn classify_parked_return_commit(committed: Committed, invocations: usize) -> Re | Status::FabricRouteWrongAccess | Status::FabricRouteInfeasible | Status::FabricRouteCapacity - | Status::FabricRouteCapability) => ReturnCommit::AfterTransfer(CallError::Refused(status)), + | Status::FabricRouteCapability + | Status::Word) => ReturnCommit::AfterTransfer(CallError::Refused(status)), } } @@ -1548,6 +1577,35 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { self.resume_bounded(budget_ns).await } + /// Carry one lifecycle request forward: continue `req` when it is the + /// exact request this pair already carries, otherwise publish it, then + /// watch its reply per `observation`. + /// + /// An owner whose earlier slice expired calls this again with the same + /// request; it never publishes a replacement. Any other request's + /// occupancy answers [`CallError::Outstanding`] before a kick or a reply + /// is consumed — that reply belongs to its own owner. + pub async fn call_exact( + &mut self, + req: &S::Request, + observation: Observation, + ) -> Result, CallError> + where + S::ReqWire: PartialEq, + { + let budget_ns = observation.budget_ns(); + match exact_outstanding_request(&self.state, &S::encode_req(req)) { + Ok(()) => self.resume_with_budget(budget_ns).await, + Err(CallError::NoOutstanding) => self.call_with_budget(req, budget_ns).await, + Err(error) => { + if error == CallError::Poisoned { + self.retry_poison_cleanup(); + } + Err(error) + } + } + } + /// Continue the outstanding call and return the exact encoded request /// beside its completed reply. /// @@ -3053,7 +3111,7 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { }; let committed = commit(self.transport, &session, &spec).map_err(|_| CallError::Transport)?; - if committed.raw() as u16 > Status::FabricRouteCapability as u16 + if committed.raw() as u16 > Status::Word as u16 || committed.failing_index() != 0 { return Err(CallError::Refused(Status::Malformed)); @@ -3100,7 +3158,8 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { | Status::FabricRouteWrongAccess | Status::FabricRouteInfeasible | Status::FabricRouteCapacity - | Status::FabricRouteCapability) => return Err(CallError::Refused(status)), + | Status::FabricRouteCapability + | Status::Word) => return Err(CallError::Refused(status)), } } Err(CallError::Backpressure) @@ -3514,6 +3573,94 @@ mod tests { ); } + /// Accepts every commit and counts them: a kick is a commit. + struct CountingTransport { + commits: core::cell::Cell, + } + + impl Transport for CountingTransport { + type Error = core::convert::Infallible; + + async fn commit(&self, session: &SessionBuilder) -> Result { + self.commit_now(session) + } + + fn commit_now(&self, _: &SessionBuilder) -> Result { + self.commits.set(self.commits.get() + 1); + Ok(Committed::from_raw(Status::Ok.encode(0))) + } + } + + fn ready(future: F) -> F::Output { + use core::task::{Context, Poll, Waker}; + let mut future = core::pin::pin!(future); + match future + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + { + Poll::Ready(output) => output, + Poll::Pending => panic!("the call waited although its outcome was decided"), + } + } + + /// The state an expired slice leaves (see the bounded-park test above): + /// the request stays published and owns the pair. A different request is + /// refused before any kick; settling the same request carries it to its + /// reply without publishing it again. + #[test] + fn an_expired_slice_is_settled_only_by_its_own_request() { + let transport = CountingTransport { + commits: core::cell::Cell::new(0), + }; + let mut window = RetainedTestWindow([0; RETAINED_TEST_WINDOW_BYTES]); + let (rings, mut serve) = + crate::svc::split_window::(&mut window.0, REQ_CAP, REP_CAP) + .unwrap(); + let mut client = SvcClient::new( + &transport, + rings, + ClientEndpoints { + kick: SlotRef::from_wire(100), + grants: [None; MAX_GRANTS], + }, + ); + client.rings.req.push(17).unwrap(); + client.state = ClientState::AwaitingReply { request: 17 }; + + let other = ready(client.call_exact(&18, Observation::Settle)); + assert_eq!(other.err(), Some(CallError::Outstanding)); + assert!(matches!( + &client.state, + ClientState::AwaitingReply { request } if *request == 17 + )); + assert_eq!(transport.commits.get(), 0, "the refusal kicked nothing"); + + // The service serves the one published request and replies. + assert_eq!(serve.req.pop(), Some(17)); + assert_eq!(serve.req.pop(), None, "18 was never published"); + serve + .rep + .push(RetainedTestProtocol::encode_rep(&71, 0)) + .unwrap(); + + let settled = ready(client.call_exact(&17, Observation::Settle)) + .expect("settling the owner's request collects its reply"); + assert_eq!(settled.response, 71); + assert!(client.is_idle()); + assert_eq!(serve.req.pop(), None, "17 was not published twice"); + + // From idle, the same call publishes. + serve + .rep + .push(RetainedTestProtocol::encode_rep(&72, 0)) + .unwrap(); + let published = ready(client.call_exact(&19, Observation::Settle)) + .expect("an idle pair publishes the request"); + assert_eq!(published.response, 72); + assert_eq!(serve.req.pop(), Some(19)); + assert!(client.is_idle()); + } + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum LifecycleRequest { Retire(u32), diff --git a/native/crates/rt/src/svc/mod.rs b/native/crates/rt/src/svc/mod.rs index 8dda50d..86aa8fb 100644 --- a/native/crates/rt/src/svc/mod.rs +++ b/native/crates/rt/src/svc/mod.rs @@ -49,7 +49,8 @@ mod serve; pub use client::{ AffineBundleCallError, CallError, CallReply, ClientEndpoints, GrantPin, InvocationAuxiliary, - InvocationAuxiliaryCapability, MAX_INVOCATION_AUXILIARIES, ReplyWaitStats, RetainedReturnCallError, + InvocationAuxiliaryCapability, MAX_INVOCATION_AUXILIARIES, Observation, ReplyWaitStats, + RetainedReturnCallError, ReturnCallError, ReturnCapability, ReturnDeriveError, ReturnDeriver, ReturnLeaseSource, StagedReturn, StagedReturnError, SvcClient, }; @@ -58,6 +59,7 @@ pub use serve::{ MAX_MINTED_PAIRS, MAX_SERVE_PAIRS, MintedPair, PeerAdmissionError, PeerAdmissionRefusal, PeerAssemblyError, PeerAssemblyRefusal, PeerBundleAssembler, PeerBundlePart, PeerId, PeerMapPlan, RecyclablePeerBundle, ServeEndpoints, ServeHost, ServePair, ServeStats, + peer_slot_bytes, }; pub(crate) use serve::{ServiceOwner, own_service}; @@ -407,8 +409,9 @@ pub const COOKIE_SHELL_CONFIG_POSTED: u64 = 0x4b4f_5245_5343_4601; // "KORESCF\x pub const INIT_SHELL_CONFIG_PRIMARY_RESOURCES: u64 = INIT_SHELL_CONFIG_RESOURCES + 2; /// Number of affine configuration members in a sibling seat's package. -/// Sibling seats omit the primary seat's hypervisor console. -pub const INIT_SHELL_CONFIG_RESOURCES: u64 = 14; +/// Sibling seats omit the primary seat's hypervisor console. The final member +/// is an ACCESS-only sender to the founder's isolated diagnostic receiver. +pub const INIT_SHELL_CONFIG_RESOURCES: u64 = 15; /// A shell observed its separately founded Axiom presenter through compositor /// status. Low 32 bits: presented epoch; bits 32..47: terminal; bit 63: the @@ -491,6 +494,8 @@ const _: () = { assert!(!kernel_reserved(COOKIE_REFUSED)); assert!(!kernel_reserved(COOKIE_SHELL)); assert!(!kernel_reserved(COOKIE_STORE)); + assert!(!kernel_reserved(COOKIE_HOMES)); + assert!(COOKIE_HOMES != COOKIE_STORE); assert!(!kernel_reserved(COOKIE_SPMD)); assert!(!kernel_reserved(COOKIE_LUA)); assert!(!kernel_reserved(COOKIE_LUA_REPL)); @@ -500,6 +505,7 @@ const _: () = { assert!(COOKIE_TERMINAL_APP != COOKIE_SHELL); assert!(COOKIE_MPY != COOKIE_LUA); assert!(COOKIE_MPY != COOKIE_LUA_REPL); + assert!(!kernel_reserved(COOKIE_BOOT_SETTLED)); }; /// Shell witness payload: wired, attached, and serving the prompt. @@ -931,6 +937,17 @@ mod shell_failure_tests { /// testimony), and a seed verdict is not shell testimony. pub const COOKIE_STORE: u64 = 0x6B72_5F73_746F_7265; // "kr_store" +/// Completion cookie for the home skeleton (`"kr_homes"`), posted by INIT +/// last of its post-ready work, once it has asked exec to provision each +/// terminal's durable home (`res/PHOTOS.md` § Sample pictures). Bits 0..7: +/// the homes asked; 8..15: how many are recorded at the current skeleton +/// version; 16..23: files copied in this boot (saturating — the skeleton's +/// whole size on a first boot, zero after); 24..29: the first unprovisioned +/// home's stage (`kore_abi::svc::LibraryFailure`), 0 when every home is. +/// Non-fatal like [`COOKIE_STORE`], and for the same reason the ONLY place a +/// dead skeleton and a working one differ — the gates key on it. +pub const COOKIE_HOMES: u64 = 0x6B72_5F68_6F6D_6573; // "kr_homes" + /// One init phase boundary (`"kr_initP"`): `[7:0]` the phase ordinal, /// `[63:8]` init's committed session clock in milliseconds. Posted at /// every step of the bring-up ladder so a boot log attributes the @@ -952,6 +969,13 @@ pub const COOKIE_INIT_PHASE: u64 = 0x6B72_5F69_6E69_7450; // "kr_initP" /// or the step, so a stall is one call rather than forty. pub const COOKIE_INIT_STEP: u64 = 0x6B72_5F69_6E69_7453; // "kr_initS" +/// Doorbell notices init could not hold (`"kr_iRlst"`): the payload is how +/// many. Init holds what an executor handoff consumes from its doorbell and +/// then acts on it, or relays it, in arrival order; a notice that met the hold +/// full is counted and named here instead of leaving a silent gap in the +/// testimony it relays. +pub const COOKIE_INIT_RELAY_LOST: u64 = 0x6B72_5F69_526C_7374; // "kr_iRlst" + /// Three ordered, failure-only words of an exec cleanup-row snapshot. Init /// sends A, B, C on its retained founder channel; C completes one bounded /// serial record. This is testimony, never a launch or scheduling command. @@ -1005,6 +1029,39 @@ pub const COOKIE_LUA_REPL: u64 = 0x6B72_5F6C_7561_5250; // "kr_luaRP" /// readiness (and therefore prevent that row from being founded). pub const COOKIE_MPY: u64 = 0x6B72_5F6D_7079_4F4B; // "kr_mpyOK" +/// Diagnostic fields packed above [`COOKIE_MPY`]'s verdict byte. The shell +/// produces these state-nibble values and the kernel's founding logger decodes +/// them; one vocabulary prevents the two images from assigning different +/// meanings to a failure. Values 0..=4 remain the observed guest states. +pub mod mpy_verdict { + /// The bounded Status request got no reply within its observation slice. + pub const STATUS_TIMEOUT: u64 = 0x5; + /// The service replied but could not publish the status-plane record. + pub const STATUS_UNAVAILABLE: u64 = 0x6; + /// StatusReady arrived without the promised status-plane record. + pub const STATUS_MISSING_RECORD: u64 = 0x7; + /// The published status-plane record failed wire decoding. + pub const STATUS_MALFORMED_RECORD: u64 = 0x8; + /// The reply was not one of the Status verb's two legal responses. + pub const STATUS_UNEXPECTED_REPLY: u64 = 0x9; + /// A non-timeout request/reply infrastructure error occurred. + pub const STATUS_CALL_ERROR: u64 = 0xa; + /// Older shell images used this undifferentiated no-snapshot code. + pub const STATUS_UNSPECIFIED: u64 = 0xe; + /// The verdict's upper byte counts 80 ms units. + pub const ELAPSED_UNIT_MS: u64 = 80; + /// The one-byte elapsed field saturates at this many units. + pub const ELAPSED_MAX_UNITS: u64 = u8::MAX as u64; +} + +/// Init's boot is done (`"kr_bootS"`), sent once on the boot-settle lane +/// ([`role::BOOT_SETTLE`](kore_abi::task::role::BOOT_SETTLE)) after init's +/// last boot step, the SPMD witness. The compositor holds the welcome's +/// animation until it arrives (#1332). Payload 0. Init sends it only after a +/// boot that ran to its end: on a failure, init stops and the welcome stays +/// still. +pub const COOKIE_BOOT_SETTLED: u64 = 0x6B72_5F62_6F6F_7453; // "kr_bootS" + /// Request-ring capacity of a founded pair's shared window — the /// **cross-boundary geometry contract** (Phase-A checkpoint G2a-3a): the /// founder formats the window with [`init_window`]`(REQ_CAP, REP_CAP)` before diff --git a/native/crates/rt/src/svc/serve.rs b/native/crates/rt/src/svc/serve.rs index d97ec06..1b79832 100644 --- a/native/crates/rt/src/svc/serve.rs +++ b/native/crates/rt/src/svc/serve.rs @@ -151,6 +151,19 @@ fn grant_commit_action(committed: Committed, invocations: usize) -> GrantCommitA pub const MAX_SERVE_PAIRS: usize = kore_abi::task::DESKTOP_BOOTSTRAP_WINDOW_SLOTS + kore_abi::task::DESKTOP_CONTROLLER_CAPACITY; +/// **What one recyclable serving slot costs a host whose peer table grows** +/// ([`ServeHost::enable_peer_growth`]): the pair, its generation, its affine +/// admission resources and its closing note — one element of each table the +/// growth moves into the named arena. A service sizes the arena it names in +/// [`SystemService::peer_arena`](crate::service::SystemService::peer_arena) +/// by this, rather than by a respelled guess at the host's layout. +pub const fn peer_slot_bytes() -> usize { + core::mem::size_of::>>() + + core::mem::size_of::() + + core::mem::size_of::>() + + core::mem::size_of::>() +} + /// A recyclable serving-peer identity. /// /// The slot selects one bounded host cell and the generation distinguishes diff --git a/native/crates/rt/src/thread.rs b/native/crates/rt/src/thread.rs index 537d574..bbbe78a 100644 --- a/native/crates/rt/src/thread.rs +++ b/native/crates/rt/src/thread.rs @@ -595,6 +595,46 @@ pub struct WorkerCap { pub rights: kore_abi::cap::Rights, } +impl WorkerCap { + /// **The process heap, endowed** — `role::HEAP@0`: the position a + /// thread must hold for the global allocator to grow on it. + /// + /// The allocator is process-wide memory, but its grow is + /// capability-gated per task: a thread that finds the shared heap + /// window short carves its own extent off the supply it holds + /// (`heap::source`), and sharing the parent's address space does not + /// make the parent's holding reachable. Every thread that runs code + /// which may allocate is founded holding this, whatever else it is + /// endowed with. The parent keeps the supply and the child receives a + /// whole-span sublease with carve and map authority only; REVOKE stays + /// with the parent, which tears the sublease down. + pub fn heap(supply: SlotRef) -> Self { + use kore_abi::cap::{ResourceKind, Rights}; + Self { + role: role::HEAP, + index: 0, + subject: supply, + kind: ResourceKind::MemLease as u16 as u8, + rights: Rights::ACCESS | Rights::MODIFY | Rights::DERIVE, + } + } +} + +/// [`WorkerCap::heap`] over this thread's own `role::HEAP@0` — `None` when +/// the task was founded without a heap supply (its allocator cannot grow +/// on any thread), or before the thread is anchored. +#[cfg(target_os = "none")] +pub(crate) fn heap_endowment() -> Option { + let rt = crate::entry::thread_context()?; + Some(WorkerCap::heap(rt.bootstrap().handle(role::HEAP, 0)?)) +} + +/// Off metal no thread founds another; there is no heap to endow. +#[cfg(not(target_os = "none"))] +pub(crate) fn heap_endowment() -> Option { + None +} + /// Build exactly the capability namespace a spawned worker receives. /// /// This stays target-independent so host tests can pin a subtle shared-space @@ -798,7 +838,7 @@ pub fn spawn_with_doorbell(entry: fn(), extra: &[WorkerCap]) -> Option Option Option { // here, seven times, against nothing. let owner = crate::entry::geometry::thread_owner_identity()?; let index = claim_thread_index()?; + // The index may have belonged to a dead sibling: its noted windows must + // not become this thread's (`cap::mapped::forget_thread`). + crate::cap::mapped::forget_thread(index as usize); let mut w = BANDS.windows(index)?; let tls = match template { Some(t) => Some(t.place(w.map)?), @@ -1132,6 +1169,20 @@ fn stamp_stage( return None; } } + // `geom::RESERVE` is the one task-level window a thread IS stamped: its + // placement is exact and keeps no cursor, so there is no second cursor + // to put on one span (#428), and the stamp is what makes the kernel keep + // this thread's exact-VA shares out of the task's reservations too, as + // it does the task's own (`refuse_reserve_window`). Copied from the + // spawning thread's geometry, which carries it in turn. + if let Some(reserve) = crate::entry::thread_context() + .and_then(|rt| rt.geom()) + .and_then(|g| g.window_of(geom::RESERVE)) + { + if !gblock.push_window(geom::RESERVE, 0, reserve) { + return None; + } + } let stage_va = w.stage.base(); let stack_top = stack_base + stack_len; // SAFETY: the provisioning session just mapped the whole `w.stage` @@ -1170,6 +1221,34 @@ fn stamp_stage( Some(()) } +/// The widest planned gang endowment: the star's mesh at the widest chunk +/// (own recv, hub send, the founder's send, and a send per chunk sibling — +/// `MESH_CEILING + 2`), closed with the heap. +#[cfg(any(target_os = "none", test))] +const PLANNED_CAPS: usize = crate::star::MESH_CEILING + 3; + +/// The fill pattern of an endowment buffer, never read past its length. +#[cfg(any(target_os = "none", test))] +const VACANT_CAP: WorkerCap = WorkerCap { + role: 0, + index: 0, + subject: SlotRef::from_wire(0), + kind: 0, + rights: kore_abi::cap::Rights::empty(), +}; + +/// Close the `filled` capabilities planned in `out` with `heap`, the +/// process heap when the task holds one: the endowment's new length, or +/// `None` when the heap does not fit. +#[cfg(any(target_os = "none", test))] +fn close_with_heap(out: &mut [WorkerCap], filled: usize, heap: Option) -> Option { + let Some(heap) = heap else { + return Some(filled); + }; + *out.get_mut(filled)? = heap; + filled.checked_add(1) +} + /// **The gang founding** (G5 — `res/SPMD.md` §7): found `out.len()` SPMD /// workers of one region in TWO sessions total — one provisioning /// session mapping every worker's stack, runtime-block, and staging @@ -1182,6 +1261,15 @@ fn stamp_stage( /// occupy a class-best pin nor count in a reservation batch /// (`spawn_spmd` chunks a wide launch across calls). /// +/// `heap` closes every **planned** endowment — none, or the star's mesh — +/// with the process heap ([`WorkerCap::heap`]): a compute worker runs +/// caller code, and caller code allocates. Without it a worker can only +/// bump what thread 0 already mapped into the shared heap window, and its +/// first allocation that finds the window short is refused (#1316). An +/// explicit `endows` list is the caller's whole endowment and takes no +/// heap: the star's hub, its one caller, runs only `star::hub_main`, +/// which posts notices and parks and never allocates. +/// /// Returns whether EVERY worker founded (`Status::Ok`). Once the /// founding session has committed, `out` is populated with a join /// ticket per worker in either case — on a partial founding @@ -1199,6 +1287,7 @@ pub(crate) fn spawn_gang_workers( pin: bool, endows: &[&[WorkerCap]], star: Option<&crate::star::StarPlan>, + heap: Option, out: &mut [Option], ) -> bool { use crate::session::MAX_SESSION_FOUNDS; @@ -1217,9 +1306,12 @@ pub(crate) fn spawn_gang_workers( // list per worker (the hub founding's shape), or a star plan — // whose per-worker kick-mesh endows name PEER BANDS and therefore // cannot exist yet: they are built below, after the claims loop - // fixes every member's band. A meshed chunk is invocation-bounded - // at [`MESH_CEILING`](crate::star::MESH_CEILING) (n·(n+6) invocations). - if !endows.is_empty() && (endows.len() != n || star.is_some()) { + // fixes every member's band. A planned endowment (absent or the + // mesh) is closed with `heap`; an explicit one is complete as given. + // A meshed chunk is invocation-bounded at + // [`MESH_CEILING`](crate::star::MESH_CEILING) (n·(n+7) invocations, + // the heap included). + if !endows.is_empty() && (endows.len() != n || star.is_some() || heap.is_some()) { return false; } if star.is_some() && n > crate::star::MESH_CEILING { @@ -1374,40 +1466,40 @@ pub(crate) fn spawn_gang_workers( if committed.status() != kore_abi::session::Status::Ok { return false; } - // ── The star's kick mesh, buildable only now: every member's band - // is claimed, so each worker's peer sends can be NAMED - // (`DOORBELL@PEER_BASE+band` — the futex lookup's arithmetic). - // One vacant WorkerCap is the fill pattern; the lens bound what - // is read. ── - let vacant_cap = WorkerCap { - role: 0, - index: 0, - subject: SlotRef::from_wire(0), - kind: 0, - rights: kore_abi::cap::Rights::empty(), - }; - let mut mesh_store = [[vacant_cap; { crate::star::MESH_CEILING + 2 }]; MAX_SESSION_FOUNDS]; - let mut mesh_lens = [0usize; MAX_SESSION_FOUNDS]; - if let Some(plan) = star { + // ── The planned endowments, buildable only now: every member's band + // is claimed, so the star's peer sends can be NAMED + // (`DOORBELL@PEER_BASE+band` — the futex lookup's arithmetic), + // and each plan is closed with the heap. One vacant WorkerCap is + // the fill pattern; the lens bound what is read. ── + let mut planned = [[VACANT_CAP; PLANNED_CAPS]; MAX_SESSION_FOUNDS]; + let mut planned_lens = [0usize; MAX_SESSION_FOUNDS]; + if endows.is_empty() { let mut bands = [(0usize, 0u32); MAX_SESSION_FOUNDS]; for (k, claim) in claims.iter().take(n).enumerate() { let Some(c) = claim else { return false }; bands[k] = (first_worker as usize + k, c.index); } for k in 0..n { - let Some(filled) = - plan.worker_endow_mesh(first_worker as usize + k, &bands[..n], &mut mesh_store[k]) - else { + let meshed = match star { + Some(plan) => match plan.worker_endow_mesh( + first_worker as usize + k, + &bands[..n], + &mut planned[k], + ) { + Some(filled) => filled, + None => return false, + }, + None => 0, + }; + let Some(filled) = close_with_heap(&mut planned[k], meshed, heap) else { return false; }; - mesh_lens[k] = filled; + planned_lens[k] = filled; } } let endow_for = |k: usize| -> &[WorkerCap] { - if star.is_some() { - &mesh_store[k][..mesh_lens[k]] - } else if endows.is_empty() { - &[] + if endows.is_empty() { + &planned[k][..planned_lens[k]] } else { endows[k] } @@ -2313,19 +2405,77 @@ mod tests { } } + /// The heap endowment is a carve-and-map sublease of the parent's + /// supply, under the name the allocator's grow looks up: never the + /// teardown right, which stays with the parent. #[test] - fn worker_bootstrap_installs_an_explicit_heap_supply() { + fn the_heap_endowment_is_a_carve_and_map_sublease() { use kore_abi::cap::{ResourceKind, Rights}; + let heap = WorkerCap::heap(SlotRef::from_wire(71)); + assert_eq!((heap.role, heap.index), (role::HEAP, 0)); + assert_eq!(heap.subject, SlotRef::from_wire(71)); + assert_eq!(heap.kind, ResourceKind::MemLease as u16 as u8); + assert_eq!(heap.rights, Rights::ACCESS | Rights::MODIFY | Rights::DERIVE); + assert!(!heap.rights.contains(Rights::REVOKE)); + } + + /// A planned gang endowment — none, or the star's mesh — is closed + /// with the heap, so every compute worker can grow the allocator; a + /// task without a heap adds nothing, and a heap that does not fit is + /// refused rather than dropped (#1316). + #[test] + fn a_planned_gang_endowment_closes_with_the_heap() { + let heap = WorkerCap::heap(SlotRef::from_wire(71)); + let doorbell = WorkerCap { + role: role::DOORBELL, + ..VACANT_CAP + }; + + let mut starless = [VACANT_CAP; PLANNED_CAPS]; + assert_eq!(close_with_heap(&mut starless, 0, Some(heap)), Some(1)); + assert_eq!((starless[0].role, starless[0].index), (role::HEAP, 0)); + + // The widest mesh still leaves the heap its place. + let widest = 2 + crate::star::MESH_CEILING; + let mut meshed = [doorbell; PLANNED_CAPS]; + assert_eq!(close_with_heap(&mut meshed, widest, Some(heap)), Some(widest + 1)); + assert_eq!(meshed[widest].role, role::HEAP); + assert!(meshed[..widest].iter().all(|cap| cap.role == role::DOORBELL)); + + let mut unendowed = [VACANT_CAP; PLANNED_CAPS]; + assert_eq!(close_with_heap(&mut unendowed, 3, None), Some(3)); + assert_eq!(unendowed[3].role, 0); + + let mut full = [doorbell; 2]; + assert_eq!(close_with_heap(&mut full, 2, Some(heap)), None); + } + + /// A meshed worker's planned endowment, the heap included, is a lawful + /// founding endowment at the widest chunk. + #[test] + fn the_widest_planned_endowment_fits_a_founding() { + assert!(PLANNED_CAPS <= MAX_WORKER_CAPS); + let mut plan = [VACANT_CAP; PLANNED_CAPS]; + for (i, cap) in plan.iter_mut().enumerate() { + *cap = WorkerCap { + role: role::DOORBELL, + index: crate::futex::PEER_BASE + i as u16, + ..VACANT_CAP + }; + } + let heap = WorkerCap::heap(SlotRef::from_wire(71)); + assert_eq!(close_with_heap(&mut plan, PLANNED_CAPS - 1, Some(heap)), Some(PLANNED_CAPS)); + let block = worker_boot_block(&plan).expect("the widest planned endowment fits a boot block"); + let bootstrap = kore_abi::task::Bootstrap::of(&block).expect("worker block validates"); + assert!(bootstrap.handle(role::HEAP, 0).is_some()); + } + + #[test] + fn worker_bootstrap_installs_an_explicit_heap_supply() { let parent_heap = SlotRef::from_wire(71); - let block = worker_boot_block(&[WorkerCap { - role: role::HEAP, - index: 0, - subject: parent_heap, - kind: ResourceKind::MemLease as u16 as u8, - rights: Rights::ACCESS | Rights::MODIFY | Rights::DERIVE, - }]) - .expect("one runtime heap endowment fits"); + let block = worker_boot_block(&[WorkerCap::heap(parent_heap)]) + .expect("one runtime heap endowment fits"); let bootstrap = kore_abi::task::Bootstrap::of(&block).expect("worker block validates"); assert_eq!( diff --git a/native/crates/rt/src/thread_controller.rs b/native/crates/rt/src/thread_controller.rs index ed7a3bb..65c6289 100644 --- a/native/crates/rt/src/thread_controller.rs +++ b/native/crates/rt/src/thread_controller.rs @@ -8,6 +8,10 @@ use core::cell::UnsafeCell; use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +// Each owner derives the process heap into its children using its own local +// slot. No creator's slot number is ever forwarded across the controller seam. +use super::thread::heap_endowment; + const FREE: u64 = 0; const BUILD: u64 = 1; const REQUEST: u64 = 2; @@ -403,28 +407,6 @@ fn controller_entry(argument: usize) { } } -// Each owner derives the process heap into its children using its own local -// slot. No creator's slot number is ever forwarded across the controller seam. -fn heap_endowment() -> Option { - #[cfg(target_os = "none")] - { - use kore_abi::{ - cap::{ResourceKind, Rights}, - task::role, - }; - let rt = crate::entry::thread_context()?; - Some(super::thread::WorkerCap { - role: role::HEAP, - index: 0, - subject: rt.bootstrap().handle(role::HEAP, 0)?, - kind: ResourceKind::MemLease as u16 as u8, - rights: Rights::ACCESS | Rights::MODIFY | Rights::DERIVE, - }) - } - #[cfg(not(target_os = "none"))] - None -} - #[cfg(test)] mod tests { extern crate std; diff --git a/native/crates/rt/src/transport/mod.rs b/native/crates/rt/src/transport/mod.rs index d6570ba..c4b052c 100644 --- a/native/crates/rt/src/transport/mod.rs +++ b/native/crates/rt/src/transport/mod.rs @@ -42,6 +42,21 @@ pub struct ParkPost { pub recv_tag: u32, } +/// One watched memory word that may be folded into a parking trap — the +/// executor's transport-level description of a `WAIT_WORD` arm +/// (`res/EVENT.md` §6). Named by *authority*: the held lease / mapped +/// view's arena slot and a byte offset within it, never an address. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WordArm { + /// The held `MemLease` / mapped-view slot the word lives in. + pub subject: SlotRef, + /// Byte offset of the `u32` word from the subject's base. + pub offset: u32, + /// The value the waiter last read; the kernel refuses to sleep on a + /// word that already differs. + pub expected: u32, +} + use crate::service::{Grant, GrantArg}; use crate::session::{BuildError, SessionBuilder, Staged}; @@ -86,9 +101,9 @@ pub trait Transport { /// it but must preserve the non-parking contract. fn commit_affine(&self, session: &SessionBuilder) -> Result { debug_assert!( - !session - .flags() - .intersects(SessionFlags::WAIT_DELIVERY | SessionFlags::WAIT_IRQ), + !session.flags().intersects( + SessionFlags::WAIT_DELIVERY | SessionFlags::WAIT_IRQ | SessionFlags::WAIT_WORD + ), "affine commit cannot carry WAIT bits" ); self.commit_now(session) @@ -184,6 +199,21 @@ pub trait ParkTransport: Transport + sealed::Sealed { /// doorbell, deadline, or route may ride alongside; the composite is /// resolved by the first-wins rule the flags document. fn commit_join_park(&self, session: &SessionBuilder) -> Result; + + /// Commit a [`SessionFlags::WAIT_WORD`] session and park on the memory + /// word it names — the **kernel address-wait** (`res/EVENT.md` §6). + /// + /// Its own entry for the reason [`commit_join_park`](Self::commit_join_park) + /// has one: the wait authority is a *held capability* (the lease or + /// mapped view the `WordWaitReq` names by slot), which the wire already + /// carries in full — a spec would restate it. The deadline is + /// mandatory (the builder refuses an unbounded word wait); a standing + /// doorbell post or a route may ride alongside, resolved by the + /// first-wins rule the flags document. `Word` = the word moved (or a + /// `wake_word` fired), `Deadline` = the bound won, `Revoked` = the + /// subject's lease died, `WouldBlock` = the kernel's bounded waiter + /// board was full (re-park). + fn commit_word_park(&self, session: &SessionBuilder) -> Result; } impl ParkTransport for TrapTransport { @@ -254,6 +284,21 @@ impl ParkTransport for TrapTransport { self.trap(session) } + #[cfg(target_os = "none")] + fn commit_word_park(&self, session: &SessionBuilder) -> Result { + debug_assert!( + session + .flags() + .contains(SessionFlags::WAIT_WORD | SessionFlags::WAIT_SLEEP), + "a word park must carry WAIT_WORD with its WAIT_SLEEP bound" + ); + // The bell guard rides exactly as on `commit_and_park`: a word park + // may carry the executor's standing doorbell post, so a peer's + // notice is owed the same `AWAKE` republish on return. + let _parked = crate::wake::Parked::enter(); + self.trap(session) + } + /// Off the metal userland targets (host tests): the honest typed refusal — /// same posture as the plain verbs, minus their WAIT-bits-clear assertion /// (this IS the flagged form). @@ -275,6 +320,11 @@ impl ParkTransport for TrapTransport { fn commit_join_park(&self, _session: &SessionBuilder) -> Result { Err(TrapUnavailable) } + + #[cfg(not(target_os = "none"))] + fn commit_word_park(&self, _session: &SessionBuilder) -> Result { + Err(TrapUnavailable) + } } // The task's descriptor-scratch and cap-slot-alias VAs were fixed constants @@ -309,9 +359,9 @@ impl TrapTransport { /// plain verb debug-asserts the bits clear (`res/USERLAND.md` §4). fn issue(&self, session: &SessionBuilder) -> Result { debug_assert!( - !session - .flags() - .intersects(SessionFlags::WAIT_DELIVERY | SessionFlags::WAIT_IRQ), + !session.flags().intersects( + SessionFlags::WAIT_DELIVERY | SessionFlags::WAIT_IRQ | SessionFlags::WAIT_WORD + ), "plain commit with WAIT bits set — parking commits are commit_and_park's (Phase A)" ); self.trap(session) diff --git a/native/crates/rt/src/upcall.rs b/native/crates/rt/src/upcall.rs new file mode 100644 index 0000000..e3db73c --- /dev/null +++ b/native/crates/rt/src/upcall.rs @@ -0,0 +1,292 @@ +//! **Fault upcalls** — the program's half of +//! `kore_abi::session::DERIVE_KIND_FAULT_UPCALL` (the kernel's is +//! `runtime::sched::upcall`): a synchronous fault on a thread that armed +//! itself is delivered to the process's handler, on the thread's own upcall +//! stack, instead of ending the process. This is what a runtime with its +//! own trap handling needs — a wasm engine whose guard regions make an +//! out-of-bounds access a page fault (`res/WASM_PLATFORM.md` §12). +//! +//! One [`Handler`] per process ([`install`]); one guarded stack and one +//! registration per thread that may fault into it ([`arm_thread`]). +//! +//! # The handler's contract +//! +//! The kernel enters this module's trampoline with the stack pointer at the +//! top of the thread's upcall stack and every register but five as the +//! fault left them; the trampoline calls the handler with those five: the +//! faulting `pc`, the frame pointer at the fault, whether the fault names +//! an address (`0` or `1`), the address, and the fault's +//! [`FaultKind`](kore_abi::crash::FaultKind) as its `u16`. +//! +//! - **To recover**, the handler transfers control to a recovery point on +//! the thread's own stack, as Wasmtime's trap handler does with the state +//! its entry trampoline saved. It never returns then, and the thread is +//! off the upcall stack from that jump on. +//! - **To decline**, it returns. The trampoline then traps at its decline +//! instruction, and the kernel ends the process with the fault as it was +//! raised, not with that trap (`CaptureFlags::UPCALL_DECLINED`). +//! +//! A fault raised while on the upcall stack — the handler's own — is never +//! delivered: it ends the process, flagged as raised inside the upcall. The +//! upcall stack's guard page counts as on it, so a handler that overflows +//! its stack ends the process rather than re-entering itself. + +use core::sync::atomic::{AtomicUsize, Ordering}; + +use crate::fiber_stack::GuardedStack; + +/// The handler the trampoline calls: the faulting `pc`, the frame pointer +/// at the fault, whether the fault names an address (`0` or `1`), the +/// address, and the fault's kind (`FaultKind as u16`). +/// +/// # Safety +/// +/// It runs on the upcall stack at an arbitrary point of the interrupted +/// code: it may touch only state that code cannot have left inconsistent, +/// as a POSIX signal handler may. +pub type Handler = + unsafe extern "C" fn(pc: usize, fp: usize, has_address: usize, address: usize, kind: usize); + +/// Each armed thread's upcall stack: enough for a handler that walks the +/// faulting code's frames and records a trap (Wasmtime's own sigaltstack is +/// 64 KiB), with the guarded stack's unmapped page below it. +pub const STACK_BYTES: usize = 64 << 10; + +/// The process's handler, as an address; `0` until [`install`]. Read by +/// the trampolines, which call through it. +static HANDLER: AtomicUsize = AtomicUsize::new(0); + +/// Why the upcall machinery refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Refusal { + /// A different handler is already installed. + HandlerTaken, + /// [`arm_thread`] before [`install`]: a delivery would have nothing to + /// call. + NoHandler, + /// No guarded stack: its window is full, the thread has no slot to hold + /// its mapping, or the kernel refused the map. + NoStack, + /// The kernel refused the registration, or the thread holds no view of + /// its own space to register through. + Refused, + /// Off the kore target: there is no kernel to deliver anything. + Unsupported, +} + +/// Install the process's fault handler. Idempotent for the same handler; +/// a different one refuses rather than silently redirecting the threads +/// already armed. +pub fn install(handler: Handler) -> Result<(), Refusal> { + let handler = handler as usize; + match HANDLER.compare_exchange(0, handler, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => Ok(()), + Err(installed) if installed == handler => Ok(()), + Err(_) => Err(Refusal::HandlerTaken), + } +} + +/// The calling thread's registration. Dropping it — on the thread that +/// armed (the type is `!Send`) — clears the registration first, then +/// unmaps the stack, so no fault is ever delivered onto a stack that is +/// gone. +#[must_use = "dropping the registration disarms the thread at once"] +pub struct ArmedThread { + stack: Option, +} + +impl ArmedThread { + /// The upcall stack's top: where a delivery sets the stack pointer. + pub fn stack_top(&self) -> u64 { + self.stack.as_ref().map_or(0, GuardedStack::top) + } +} + +impl Drop for ArmedThread { + fn drop(&mut self) { + // Clear first. A refused clear keeps the stack mapped — leaked with + // the registration that still names it — rather than unmapping + // pages a delivery would land on. + if register(0, 0, 0, 0) { + drop(self.stack.take()); + } else { + core::mem::forget(self.stack.take()); + } + } +} + +/// Arm the calling thread: map its upcall stack and register it, with this +/// module's trampoline as the entry. [`install`] first. +pub fn arm_thread() -> Result { + if !cfg!(target_os = "none") { + return Err(Refusal::Unsupported); + } + if HANDLER.load(Ordering::Acquire) == 0 { + return Err(Refusal::NoHandler); + } + let stack = GuardedStack::new(STACK_BYTES).ok_or(Refusal::NoStack)?; + // The registered range takes in the guard page: a handler overflowing + // its stack faults there, and is still on it. + let (lo, hi) = (stack.guard().start, stack.top()); + if !register(trampoline::entry(), trampoline::decline(), lo, hi) { + return Err(Refusal::Refused); + } + Ok(ArmedThread { stack: Some(stack) }) +} + +/// Issue the registration: `entry == 0` (with everything else zero) clears +/// it. The result slot is a borrowed scratch slot, left vacant by the +/// verb and handed straight back. +#[cfg(target_os = "none")] +fn register(entry: u64, decline: u64, stack_lo: u64, stack_hi: u64) -> bool { + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + use kore_abi::cap::Rights; + use kore_abi::session::{DERIVE_KIND_FAULT_UPCALL, FaultUpcallReq, Status}; + + let Some(space) = crate::entry::geometry::self_space_slot() else { + return false; + }; + let Some(scratch) = crate::entry::geometry::acquire_cap_slot() else { + return false; + }; + let mut req = [0u8; FaultUpcallReq::WIRE]; + for (i, word) in [entry, decline, stack_lo, stack_hi].into_iter().enumerate() { + req[i * 8..i * 8 + 8].copy_from_slice(&word.to_le_bytes()); + } + let mut b = SessionBuilder::new(); + let committed = b + .derive_raw( + space, + Rights::empty(), + DERIVE_KIND_FAULT_UPCALL, + scratch, + Some(&req), + ) + .is_ok() + && TrapTransport::new() + .commit_now(&b) + .is_ok_and(|done| done.status() == Status::Ok); + crate::entry::geometry::release_cap_slot(scratch); + committed +} + +#[cfg(not(target_os = "none"))] +fn register(_entry: u64, _decline: u64, _stack_lo: u64, _stack_hi: u64) -> bool { + false +} + +/// The trampolines the kernel enters. Each calls the installed handler +/// with the five argument registers exactly as delivered, and traps at its +/// decline instruction if the handler returns — or if none is installed, +/// which [`arm_thread`]'s order makes unreachable. +#[cfg(target_os = "none")] +mod trampoline { + use super::HANDLER; + + pub(super) fn entry() -> u64 { + upcall_entry as *const () as usize as u64 + } + + pub(super) fn decline() -> u64 { + upcall_decline as *const () as usize as u64 + } + + /// x86-64: `rdi`..`r8` hold the arguments and `rsp` the stack's + /// 16-aligned top, so the `call` leaves the handler the SysV entry + /// alignment. + #[cfg(target_arch = "x86_64")] + #[unsafe(naked)] + unsafe extern "C" fn upcall_entry() -> ! { + core::arch::naked_asm!( + "mov rax, qword ptr [rip + {handler}]", + "test rax, rax", + "jz 2f", + "call rax", + "2:", + "jmp {decline}", + handler = sym HANDLER, + decline = sym upcall_decline, + ) + } + + #[cfg(target_arch = "x86_64")] + #[unsafe(naked)] + unsafe extern "C" fn upcall_decline() -> ! { + core::arch::naked_asm!("ud2") + } + + /// AArch64: `x0`..`x4` hold the arguments; `x16` (IP0) carries the + /// handler, which no argument occupies. + #[cfg(target_arch = "aarch64")] + #[unsafe(naked)] + unsafe extern "C" fn upcall_entry() -> ! { + core::arch::naked_asm!( + "adrp x16, {handler}", + "ldr x16, [x16, :lo12:{handler}]", + "cbz x16, 2f", + "blr x16", + "2:", + "b {decline}", + handler = sym HANDLER, + decline = sym upcall_decline, + ) + } + + #[cfg(target_arch = "aarch64")] + #[unsafe(naked)] + unsafe extern "C" fn upcall_decline() -> ! { + core::arch::naked_asm!("udf #0") + } + + /// RISC-V: `a0`..`a4` hold the arguments; `t0` carries the handler. + #[cfg(target_arch = "riscv64")] + #[unsafe(naked)] + unsafe extern "C" fn upcall_entry() -> ! { + core::arch::naked_asm!( + "lla t0, {handler}", + "ld t0, 0(t0)", + "beqz t0, 2f", + "jalr t0", + "2:", + "tail {decline}", + handler = sym HANDLER, + decline = sym upcall_decline, + ) + } + + #[cfg(target_arch = "riscv64")] + #[unsafe(naked)] + unsafe extern "C" fn upcall_decline() -> ! { + core::arch::naked_asm!("unimp") + } +} + +#[cfg(not(target_os = "none"))] +mod trampoline { + pub(super) fn entry() -> u64 { + 0 + } + + pub(super) fn decline() -> u64 { + 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + unsafe extern "C" fn first(_: usize, _: usize, _: usize, _: usize, _: usize) {} + unsafe extern "C" fn second(_: usize, _: usize, _: usize, _: usize, _: usize) {} + + /// One handler per process: installing it again is a no-op, another + /// one refuses, and off the kore target no thread arms. + #[test] + fn one_handler_per_process_and_no_arming_off_target() { + assert_eq!(install(first), Ok(())); + assert_eq!(install(first), Ok(())); + assert_eq!(install(second), Err(Refusal::HandlerTaken)); + assert_eq!(arm_thread().err(), Some(Refusal::Unsupported)); + } +} diff --git a/native/crates/rt/src/validate.rs b/native/crates/rt/src/validate.rs index f4b2a37..cbd39b1 100644 --- a/native/crates/rt/src/validate.rs +++ b/native/crates/rt/src/validate.rs @@ -629,6 +629,433 @@ pub fn spmd_resident() -> Result { }) } +// ── The fault-upcall exercise (`res/WASM_PLATFORM.md` §12) ────────────── + +/// What the fault-upcall exercise proved. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct UpcallWitness { + /// Faults delivered to the handler and recovered from: a load from an + /// unmapped page, an invalid instruction, and on x86-64 a division by + /// zero (AArch64 and RISC-V do not trap on one). + pub delivered: u32, +} + +/// Where the fault-upcall exercise failed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum UpcallFault { + /// Installing the handler or arming the thread refused — always, off + /// the kore target. + Arm(crate::upcall::Refusal), + /// No unmapped page to fault on: the probe's guarded stack refused. + Target, + /// A probe ran to completion: its fault was never delivered. + NotDelivered { probe: u32 }, + /// The handler was given the wrong `pc`, address or kind. + Evidence { probe: u32 }, +} + +/// **The fault-upcall exercise**: install a handler, arm this thread, and +/// raise each fault a guarded guest raises — a load from an unmapped page, +/// an invalid instruction, a division by zero — checking that each reaches +/// the handler on the upcall stack with its exact evidence (the faulting +/// instruction, the unmapped address, the kind), and that the handler's +/// jump back resumes this thread on its own stack. A wrong delivery is a +/// wrong answer, never a missing line; a delivery the kernel refused ends +/// the process, which the boot gate reads as the line never printed. +/// +/// What a run cannot show in-process is a fault the upcall itself raises, or +/// a declined one: either ends the process. The kernel's policy holds both +/// to account on the host (`runtime::sched::upcall`). +#[cfg(target_os = "none")] +pub fn upcall() -> Result { + use kore_abi::crash::FaultKind; + + crate::upcall::install(upcall_probe::handler).map_err(UpcallFault::Arm)?; + let armed = crate::upcall::arm_thread().map_err(UpcallFault::Arm)?; + // A page no mapping backs: a guarded stack's guard page. + let target = crate::fiber_stack::GuardedStack::new(1).ok_or(UpcallFault::Target)?; + let unmapped = target.guard().start; + upcall_probe::TARGET.store(unmapped as usize, Ordering::Relaxed); + + let probes = upcall_probe::probes(); + let mut delivered = 0; + for (index, probe) in probes.iter().enumerate() { + let index = index as u32; + let seen = + upcall_probe::run(probe.code).ok_or(UpcallFault::NotDelivered { probe: index })?; + let start = probe.code as usize; + let expected_address = match probe.kind { + FaultKind::Memory => Some(unmapped as usize), + _ => None, + }; + let right = (start..start + upcall_probe::PROBE_BYTES).contains(&seen.pc) + && seen.kind == probe.kind as usize + && seen.address == expected_address; + if !right { + return Err(UpcallFault::Evidence { probe: index }); + } + delivered += 1; + } + drop(target); + drop(armed); + Ok(UpcallWitness { delivered }) +} + +/// Off the kore target there is no kernel to deliver a fault. +#[cfg(not(target_os = "none"))] +pub fn upcall() -> Result { + Err(UpcallFault::Arm(crate::upcall::Refusal::Unsupported)) +} + +/// The exercise's machinery: the probes that fault, a minimal catch and +/// resume (the callee-saved registers, stack and return address, as a C +/// `setjmp`/`longjmp` pair keeps them), and the handler that records what +/// it was given and resumes the catch. +#[cfg(target_os = "none")] +mod upcall_probe { + use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; + + use kore_abi::crash::FaultKind; + + /// Every probe faults within its first this-many bytes. + pub(super) const PROBE_BYTES: usize = 16; + + /// The unmapped address the memory probe loads from. + pub(super) static TARGET: AtomicUsize = AtomicUsize::new(0); + + /// The catch a fault resumes; null while no probe runs, so a fault + /// outside one is declined and ends the process as it would have. + static ACTIVE: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); + + static SEEN_PC: AtomicUsize = AtomicUsize::new(0); + static SEEN_HAS_ADDRESS: AtomicUsize = AtomicUsize::new(0); + static SEEN_ADDRESS: AtomicUsize = AtomicUsize::new(0); + static SEEN_KIND: AtomicUsize = AtomicUsize::new(0); + + /// A catch's saved state: room for the largest callee-saved set (RISC-V's + /// twelve integer and twelve FP registers, `ra` and `sp`). + type Saved = [u64; 32]; + + /// One probe: its code, and the fault it raises. + pub(super) struct Probe { + pub code: unsafe extern "C" fn(), + pub kind: FaultKind, + } + + /// What the handler was given. + pub(super) struct Seen { + pub pc: usize, + pub address: Option, + pub kind: usize, + } + + const LOAD: Probe = Probe { + code: probe_load, + kind: FaultKind::Memory, + }; + const INVALID: Probe = Probe { + code: probe_invalid, + kind: FaultKind::InvalidInstruction, + }; + + /// The probes this ISA can raise: a division by zero traps on x86-64 + /// only (AArch64 and RISC-V define its result). + #[cfg(target_arch = "x86_64")] + const PROBES: &[Probe] = &[ + LOAD, + INVALID, + Probe { + code: probe_divide, + kind: FaultKind::Arithmetic, + }, + ]; + #[cfg(not(target_arch = "x86_64"))] + const PROBES: &[Probe] = &[LOAD, INVALID]; + + pub(super) fn probes() -> &'static [Probe] { + PROBES + } + + /// Run `probe` under a catch: what the handler was given if it faulted + /// and was resumed, `None` if it ran to completion. + #[inline(never)] + pub(super) fn run(probe: unsafe extern "C" fn()) -> Option { + let mut saved: Saved = [0; 32]; + ACTIVE.store(&mut saved, Ordering::Release); + // SAFETY: `catch` saves this frame's callee-saved state into + // `saved`, which outlives every resume of it: the handler resumes + // it only while ACTIVE names it, and ACTIVE is cleared below before + // `saved` goes out of scope. Nothing live across the call is held + // outside the state `catch` restores. + let caught = unsafe { catch(probe, &mut saved) }; + ACTIVE.store(core::ptr::null_mut(), Ordering::Release); + (caught == 1).then(|| Seen { + pc: SEEN_PC.load(Ordering::Relaxed), + address: (SEEN_HAS_ADDRESS.load(Ordering::Relaxed) != 0) + .then(|| SEEN_ADDRESS.load(Ordering::Relaxed)), + kind: SEEN_KIND.load(Ordering::Relaxed), + }) + } + + /// The exercise's handler: record, then resume the catch. With no + /// catch armed (a fault outside a probe, or a second one), it declines. + pub(super) unsafe extern "C" fn handler( + pc: usize, + _fp: usize, + has_address: usize, + address: usize, + kind: usize, + ) { + let saved = ACTIVE.swap(core::ptr::null_mut(), Ordering::AcqRel); + if saved.is_null() { + return; + } + SEEN_PC.store(pc, Ordering::Relaxed); + SEEN_HAS_ADDRESS.store(has_address, Ordering::Relaxed); + SEEN_ADDRESS.store(address, Ordering::Relaxed); + SEEN_KIND.store(kind, Ordering::Relaxed); + // SAFETY: `saved` is the live catch `run` armed, still on its frame. + unsafe { resume(saved) } + } + + // ── x86-64 ────────────────────────────────────────────────────────── + + /// `catch(probe, saved) -> 0` when the probe returns, `1` when resumed: + /// saves `rbx`, `rbp`, `r12`–`r15`, the caller's stack pointer and the + /// return address. + #[cfg(target_arch = "x86_64")] + #[unsafe(naked)] + unsafe extern "C" fn catch(_probe: unsafe extern "C" fn(), _saved: *mut Saved) -> u64 { + core::arch::naked_asm!( + "mov [rsi + 0], rbx", + "mov [rsi + 8], rbp", + "mov [rsi + 16], r12", + "mov [rsi + 24], r13", + "mov [rsi + 32], r14", + "mov [rsi + 40], r15", + "lea rax, [rsp + 8]", + "mov [rsi + 48], rax", + "mov rax, [rsp]", + "mov [rsi + 56], rax", + "sub rsp, 8", + "call rdi", + "add rsp, 8", + "xor eax, eax", + "ret", + ) + } + + #[cfg(target_arch = "x86_64")] + #[unsafe(naked)] + unsafe extern "C" fn resume(_saved: *const Saved) -> ! { + core::arch::naked_asm!( + "mov rbx, [rdi + 0]", + "mov rbp, [rdi + 8]", + "mov r12, [rdi + 16]", + "mov r13, [rdi + 24]", + "mov r14, [rdi + 32]", + "mov r15, [rdi + 40]", + "mov rsp, [rdi + 48]", + "mov eax, 1", + "jmp qword ptr [rdi + 56]", + ) + } + + #[cfg(target_arch = "x86_64")] + #[unsafe(naked)] + unsafe extern "C" fn probe_load() { + core::arch::naked_asm!( + "mov rax, qword ptr [rip + {target}]", + "mov rax, qword ptr [rax]", + "ret", + target = sym TARGET, + ) + } + + #[cfg(target_arch = "x86_64")] + #[unsafe(naked)] + unsafe extern "C" fn probe_invalid() { + core::arch::naked_asm!("ud2", "ret") + } + + #[cfg(target_arch = "x86_64")] + #[unsafe(naked)] + unsafe extern "C" fn probe_divide() { + core::arch::naked_asm!( + "xor ecx, ecx", + "xor edx, edx", + "mov eax, 1", + "div ecx", + "ret" + ) + } + + // ── AArch64 ───────────────────────────────────────────────────────── + + /// Saves `x19`–`x30`, `sp` and `d8`–`d15`; resumed, it returns `1` + /// through the saved link register. + #[cfg(target_arch = "aarch64")] + #[unsafe(naked)] + unsafe extern "C" fn catch(_probe: unsafe extern "C" fn(), _saved: *mut Saved) -> u64 { + core::arch::naked_asm!( + "stp x19, x20, [x1, #0]", + "stp x21, x22, [x1, #16]", + "stp x23, x24, [x1, #32]", + "stp x25, x26, [x1, #48]", + "stp x27, x28, [x1, #64]", + "stp x29, x30, [x1, #80]", + "mov x9, sp", + "str x9, [x1, #96]", + "stp d8, d9, [x1, #104]", + "stp d10, d11, [x1, #120]", + "stp d12, d13, [x1, #136]", + "stp d14, d15, [x1, #152]", + "stp x29, x30, [sp, #-16]!", + "mov x29, sp", + "blr x0", + "ldp x29, x30, [sp], #16", + "mov x0, #0", + "ret", + ) + } + + #[cfg(target_arch = "aarch64")] + #[unsafe(naked)] + unsafe extern "C" fn resume(_saved: *const Saved) -> ! { + core::arch::naked_asm!( + "ldp x19, x20, [x0, #0]", + "ldp x21, x22, [x0, #16]", + "ldp x23, x24, [x0, #32]", + "ldp x25, x26, [x0, #48]", + "ldp x27, x28, [x0, #64]", + "ldp x29, x30, [x0, #80]", + "ldr x9, [x0, #96]", + "mov sp, x9", + "ldp d8, d9, [x0, #104]", + "ldp d10, d11, [x0, #120]", + "ldp d12, d13, [x0, #136]", + "ldp d14, d15, [x0, #152]", + "mov x0, #1", + "ret", + ) + } + + #[cfg(target_arch = "aarch64")] + #[unsafe(naked)] + unsafe extern "C" fn probe_load() { + core::arch::naked_asm!( + "adrp x9, {target}", + "ldr x9, [x9, :lo12:{target}]", + "ldr x9, [x9]", + "ret", + target = sym TARGET, + ) + } + + #[cfg(target_arch = "aarch64")] + #[unsafe(naked)] + unsafe extern "C" fn probe_invalid() { + core::arch::naked_asm!("udf #0", "ret") + } + + // ── RISC-V ────────────────────────────────────────────────────────── + + /// Saves `s0`–`s11`, `ra`, `sp` and `fs0`–`fs11`; resumed, it returns + /// `1` through the saved return address. + #[cfg(target_arch = "riscv64")] + #[unsafe(naked)] + unsafe extern "C" fn catch(_probe: unsafe extern "C" fn(), _saved: *mut Saved) -> u64 { + core::arch::naked_asm!( + "sd s0, 0(a1)", + "sd s1, 8(a1)", + "sd s2, 16(a1)", + "sd s3, 24(a1)", + "sd s4, 32(a1)", + "sd s5, 40(a1)", + "sd s6, 48(a1)", + "sd s7, 56(a1)", + "sd s8, 64(a1)", + "sd s9, 72(a1)", + "sd s10, 80(a1)", + "sd s11, 88(a1)", + "sd ra, 96(a1)", + "sd sp, 104(a1)", + "fsd fs0, 112(a1)", + "fsd fs1, 120(a1)", + "fsd fs2, 128(a1)", + "fsd fs3, 136(a1)", + "fsd fs4, 144(a1)", + "fsd fs5, 152(a1)", + "fsd fs6, 160(a1)", + "fsd fs7, 168(a1)", + "fsd fs8, 176(a1)", + "fsd fs9, 184(a1)", + "fsd fs10, 192(a1)", + "fsd fs11, 200(a1)", + "addi sp, sp, -16", + "sd ra, 8(sp)", + "jalr a0", + "ld ra, 8(sp)", + "addi sp, sp, 16", + "li a0, 0", + "ret", + ) + } + + #[cfg(target_arch = "riscv64")] + #[unsafe(naked)] + unsafe extern "C" fn resume(_saved: *const Saved) -> ! { + core::arch::naked_asm!( + "ld s0, 0(a0)", + "ld s1, 8(a0)", + "ld s2, 16(a0)", + "ld s3, 24(a0)", + "ld s4, 32(a0)", + "ld s5, 40(a0)", + "ld s6, 48(a0)", + "ld s7, 56(a0)", + "ld s8, 64(a0)", + "ld s9, 72(a0)", + "ld s10, 80(a0)", + "ld s11, 88(a0)", + "ld ra, 96(a0)", + "ld sp, 104(a0)", + "fld fs0, 112(a0)", + "fld fs1, 120(a0)", + "fld fs2, 128(a0)", + "fld fs3, 136(a0)", + "fld fs4, 144(a0)", + "fld fs5, 152(a0)", + "fld fs6, 160(a0)", + "fld fs7, 168(a0)", + "fld fs8, 176(a0)", + "fld fs9, 184(a0)", + "fld fs10, 192(a0)", + "fld fs11, 200(a0)", + "li a0, 1", + "ret", + ) + } + + #[cfg(target_arch = "riscv64")] + #[unsafe(naked)] + unsafe extern "C" fn probe_load() { + core::arch::naked_asm!( + "lla t0, {target}", + "ld t0, 0(t0)", + "ld t0, 0(t0)", + "ret", + target = sym TARGET, + ) + } + + #[cfg(target_arch = "riscv64")] + #[unsafe(naked)] + unsafe extern "C" fn probe_invalid() { + core::arch::naked_asm!("unimp", "ret") + } +} + #[cfg(test)] mod lane_tests { use super::*; @@ -659,4 +1086,14 @@ mod lane_tests { fn the_dispatch_exercise_refuses_off_metal() { assert_eq!(dispatch().unwrap_err(), DispatchFault::Launch); } + + /// Off metal there is no kernel to deliver a fault: the upcall exercise + /// refuses rather than reporting deliveries it never saw. + #[test] + fn the_upcall_exercise_refuses_off_metal() { + assert_eq!( + upcall().unwrap_err(), + UpcallFault::Arm(crate::upcall::Refusal::Unsupported) + ); + } } diff --git a/native/crates/uart/src/lib.rs b/native/crates/uart/src/lib.rs index 6a292ae..de60f6d 100644 --- a/native/crates/uart/src/lib.rs +++ b/native/crates/uart/src/lib.rs @@ -103,6 +103,22 @@ pub trait Uart { fn init(&self, regs: &dyn RegisterAccess, cfg: LineConfig); /// `true` when the transmit holding register can accept a byte. fn tx_ready(&self, regs: &dyn RegisterAccess) -> bool; + /// Enable the transmit FIFO where the layout has one, and report how many + /// bytes one observed [`tx_ready`](Self::tx_ready) admits before it must + /// be polled again. + /// + /// For a queued console that never waits on the wire: it pushes this many + /// bytes each time it finds the transmitter ready, then leaves. Framing + /// and line rate are untouched, so the trust-what-you-find rule of + /// [`init`](Self::init) still holds. + /// + /// The default is the conservative contract: a ready transmitter + /// guarantees room for one byte. A layout whose ready bit already means + /// "room for one more" (PL011's `TXFF`) needs nothing more, because the + /// caller polls again before every byte. + fn enable_tx_fifo(&self, _regs: &dyn RegisterAccess) -> usize { + 1 + } /// Push one byte into the transmit holding register. The caller has /// confirmed [`tx_ready`](Self::tx_ready). fn tx_byte(&self, regs: &dyn RegisterAccess, byte: u8); @@ -178,6 +194,15 @@ impl Uart for SerialBackend { } } + #[inline] + fn enable_tx_fifo(&self, regs: &dyn RegisterAccess) -> usize { + match self { + Self::Uart16550(u) => u.enable_tx_fifo(regs), + Self::Pl011(p) => p.enable_tx_fifo(regs), + Self::Geni(g) => g.enable_tx_fifo(regs), + } + } + #[inline] fn tx_byte(&self, regs: &dyn RegisterAccess, byte: u8) { match self { @@ -621,6 +646,63 @@ mod tests { assert!(!uart.rx_ready(®s)); } + /// A 16550 whose IIR[7:6] answers `fifo_bits` once FCR0 is set: `0b11` + /// models a 16550A, `0b10` the original 16550, `0b00` a 16450. + struct FifoProbe { + fcr: core::cell::Cell, + fifo_bits: u8, + } + + impl RegisterAccess for FifoProbe { + fn read8(&self, offset: usize) -> u8 { + match offset { + // IIR: no interrupt pending (bit 0), FIFO state in [7:6]. + FCR if self.fcr.get() & 1 != 0 => (self.fifo_bits << 6) | 1, + FCR => 1, + _ => 0, + } + } + fn write8(&self, offset: usize, value: u8) { + if offset == FCR { + self.fcr.set(value); + } + } + fn read32(&self, _: usize) -> u32 { + 0 + } + fn write32(&self, _: usize, _: u32) {} + } + + #[test] + fn a_16550a_enables_its_fifo_and_admits_sixteen_bytes_per_ready() { + let regs = FifoProbe { + fcr: core::cell::Cell::new(0), + fifo_bits: 0b11, + }; + let uart = SerialBackend::new(BackendKind::Uart16550, 1); + assert_eq!(uart.enable_tx_fifo(®s), 16); + // Enabled without the clear bits: whatever is queued is kept. + assert_eq!(regs.fcr.get(), 0x01); + } + + #[test] + fn a_part_without_a_usable_fifo_stays_at_one_byte_per_ready() { + for fifo_bits in [0b00, 0b10] { + let regs = FifoProbe { + fcr: core::cell::Cell::new(0), + fifo_bits, + }; + assert_eq!(Uart16550::new(1).enable_tx_fifo(®s), 1); + } + } + + #[test] + fn layouts_without_a_fifo_burst_keep_the_one_byte_contract() { + let regs = MockRegisters::new(); + assert_eq!(SerialBackend::new(BackendKind::Pl011, 4).enable_tx_fifo(®s), 1); + assert_eq!(SerialBackend::new(BackendKind::Geni, 4).enable_tx_fifo(®s), 1); + } + #[test] fn geni_rx_reads_a_zero_valid_count_as_a_full_word() { let regs = MockRegisters::new(); diff --git a/native/crates/uart/src/uart16550.rs b/native/crates/uart/src/uart16550.rs index e20638f..eb5a3cc 100644 --- a/native/crates/uart/src/uart16550.rs +++ b/native/crates/uart/src/uart16550.rs @@ -14,6 +14,7 @@ use super::{LineConfig, RegisterAccess, Uart}; const REG_RBR: usize = 0; // receive buffer (read; DLL when DLAB) const REG_THR: usize = 0; // transmit holding (write; DLL when DLAB) const REG_IER: usize = 1; // interrupt enable (DLM when DLAB) +const REG_IIR: usize = 2; // interrupt identification (read) const REG_FCR: usize = 2; // FIFO control (write) const REG_LCR: usize = 3; // line control const REG_LSR: usize = 5; // line status @@ -22,6 +23,11 @@ const REG_SCR: usize = 7; // scratch (16550A and later; absent on a 16450) const LCR_8N1: u8 = 0x03; // 8 data bits, no parity, 1 stop; clears DLAB const LCR_DLAB: u8 = 0x80; // divisor-latch access const FCR_ENABLE_CLEAR: u8 = 0x07; // enable FIFOs + clear RX/TX +const FCR_ENABLE: u8 = 0x01; // enable FIFOs, keep what is queued +const IIR_FIFOS: u8 = 0xC0; // IIR[7:6]: `11` once FCR0 has enabled usable FIFOs +/// The 16550A transmit FIFO. Deeper descendants (a 64-byte 16750, a +/// DesignWare part) still hold at least this many, so it is a safe floor. +const TX_FIFO_DEPTH: usize = 16; const LSR_DR: u8 = 0x01; // data ready (receive holding register non-empty) const LSR_THRE: u8 = 0x20; // transmit holding register empty @@ -120,6 +126,24 @@ impl Uart for Uart16550 { self.read_reg(regs, REG_LSR) & LSR_THRE != 0 } + /// PC16550D: FCR0 enables both FIFOs, and in FIFO mode LSR.THRE means + /// the whole transmit FIFO is empty, so one ready observation admits + /// [`TX_FIFO_DEPTH`] bytes. IIR[7:6] reads `11` only on a part whose + /// FIFOs then work; `00` (a 16450) or `10` (the original 16550, whose + /// FIFO is unusable) stays one byte per observation. Linux's + /// `serial8250` autoconfig makes the same IIR distinction. + /// + /// Call this before the first transmitted byte: entering FIFO mode + /// clears the FIFOs, and only an idle transmitter has nothing to lose. + fn enable_tx_fifo(&self, regs: &dyn RegisterAccess) -> usize { + self.write_reg(regs, REG_FCR, FCR_ENABLE); + if self.read_reg(regs, REG_IIR) & IIR_FIFOS == IIR_FIFOS { + TX_FIFO_DEPTH + } else { + 1 + } + } + fn tx_byte(&self, regs: &dyn RegisterAccess, byte: u8) { self.write_reg(regs, REG_THR, byte); } diff --git a/native/crates/window/src/capture.rs b/native/crates/window/src/capture.rs new file mode 100644 index 0000000..f7bf98c --- /dev/null +++ b/native/crates/window/src/capture.rs @@ -0,0 +1,206 @@ +//! The screenshot vocabulary (`res/SCREENSHOTS.md`): what a capture asks +//! for, what a frozen frame is, and why one was refused. Data only; the +//! authority is the read-only lend on the capture writer's own pair. + +use crate::appearance::Density; + +/// Where screenshots are written, from the home's root. The writer +/// requires it to exist; nothing creates it for the user. +pub const FOLDER: &str = "pictures/screenshots"; + +/// The writer's one launch argument, `--app `: the program a window +/// capture's window runs, as its controller named it, for the file's text +/// and its line. Inert words; the frame is the lend's. +pub const APP_FLAG: &str = "--app"; + +/// What a capture asks for: a PrintScreen chord (`res/KEYBOARD.md` §5) or +/// the `screenshot` command. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum CaptureMode { + /// The whole screen the seat is on. + Screen = 1, + /// One window's own content, covered or minimized: the seat's focused + /// window, or the one the request names. + Window = 2, + /// A region the user selects over the frozen screen. + Region = 3, +} + +impl CaptureMode { + pub const fn code(self) -> u8 { + self as u8 + } + + pub const fn from_code(code: u8) -> Option { + match code { + 1 => Some(Self::Screen), + 2 => Some(Self::Window), + 3 => Some(Self::Region), + _ => None, + } + } + + /// The word a file name and its metadata use. + pub const fn name(self) -> &'static str { + match self { + Self::Screen => "screen", + Self::Window => "window", + Self::Region => "region", + } + } +} + +/// One frozen frame, as the compositor lends it to the capture writer. +/// +/// The lent view holds `height` rows of `width` XRGB8888 pixels, packed +/// (`width * 4` bytes a row), at the head of `bytes`. The high byte of each +/// pixel is not part of the image: the compositor's painters leave it +/// `0xff` in places and zero in others. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CaptureFrame { + pub width: u16, + pub height: u16, + pub mode: CaptureMode, + /// The screen it was frozen from (`0` is the primary). + pub screen: u8, + /// The UI density the screen was drawn at. + pub density: Density, + /// Nanoseconds since the UNIX epoch at the freeze, or `None` while the + /// clock has no wall anchor. + pub wall_ns: Option, + /// Nanoseconds since boot at the freeze. + pub uptime_ns: u64, + /// What the freeze cost the compositor's pass: the frame's allocation + /// and copy, in nanoseconds. Every shot carries the measurement of its + /// own (`res/SCREENSHOTS.md`, Budgets). + pub freeze_ns: u64, + /// Where the image's top-left pixel sits on its screen: the selected + /// region's origin, or the window's client aperture's (its last + /// placement while minimized). `(0, 0)` for a whole screen. + pub x: u16, + pub y: u16, + /// Whether the pixels are a frame presented for this capture. A window + /// that presents nothing within the bound is copied as its backing + /// stood, which may be mid-paint: `false`, and the file says so. A + /// screen or region is the compositor's own complete frame: `true`. + pub synced: bool, +} + +impl CaptureFrame { + /// The bytes the pixels occupy: `width * height * 4`. + pub const fn pixel_bytes(&self) -> u64 { + self.width as u64 * self.height as u64 * 4 + } + + /// The lend's length: the pixels in whole pages, so the frozen + /// allocation it maps is page-exclusive. + pub const fn bytes(&self) -> u64 { + self.pixel_bytes().next_multiple_of(4096) + } + + /// A frame with pixels to lend. + pub const fn is_valid(&self) -> bool { + self.width != 0 && self.height != 0 + } +} + +/// Why a capture did not happen. The controller says so on the seat's +/// terminal; a capture is never dropped silently. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum CaptureRefusal { + /// This seat's previous capture is still being written. + Busy = 1, + /// The compositor had no room for the frozen frame. + OutOfMemory = 2, + /// The seat has no screen to capture. + Unavailable = 3, + // 4 was `Unsupported`, a mode not yet built, until window and region + // capture landed; it decodes to nothing. + /// A window capture found no window of this seat to take: nothing is + /// focused, the named window is gone or another seat's, or it has never + /// presented a frame. + NoWindow = 5, + /// The user cancelled the selection. + Cancelled = 6, +} + +impl CaptureRefusal { + pub const fn code(self) -> u8 { + self as u8 + } + + pub const fn from_code(code: u8) -> Option { + match code { + 1 => Some(Self::Busy), + 2 => Some(Self::OutOfMemory), + 3 => Some(Self::Unavailable), + 5 => Some(Self::NoWindow), + 6 => Some(Self::Cancelled), + _ => None, + } + } + + /// The reason, as the seat's terminal shows it. + pub const fn describe(self) -> &'static str { + match self { + Self::Busy => "the previous screenshot is still being saved", + Self::OutOfMemory => "no memory for the frozen screen", + Self::Unavailable => "this seat has no screen", + Self::NoWindow => "there is no window to capture", + Self::Cancelled => "cancelled", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn codes_round_trip_and_zero_is_none() { + for mode in [CaptureMode::Screen, CaptureMode::Window, CaptureMode::Region] { + assert_eq!(CaptureMode::from_code(mode.code()), Some(mode)); + } + for refusal in [ + CaptureRefusal::Busy, + CaptureRefusal::OutOfMemory, + CaptureRefusal::Unavailable, + CaptureRefusal::NoWindow, + CaptureRefusal::Cancelled, + ] { + assert_eq!(CaptureRefusal::from_code(refusal.code()), Some(refusal)); + assert!(!refusal.describe().is_empty()); + } + for code in [0, 4, 0xff] { + assert_eq!(CaptureMode::from_code(code), None); + } + for code in [0, 4, 7, 0xff] { + assert_eq!(CaptureRefusal::from_code(code), None); + } + } + + #[test] + fn a_frame_lends_its_pixels_in_whole_pages() { + let frame = CaptureFrame { + width: 1280, + height: 800, + mode: CaptureMode::Screen, + screen: 0, + density: Density::Native, + wall_ns: None, + uptime_ns: 1, + freeze_ns: 0, + x: 0, + y: 0, + synced: true, + }; + assert!(frame.is_valid()); + assert_eq!(frame.bytes(), 1280 * 800 * 4, "already whole pages"); + let odd = CaptureFrame { width: 33, height: 7, ..frame }; + assert_eq!((odd.pixel_bytes(), odd.bytes()), (33 * 7 * 4, 4096)); + assert!(!CaptureFrame { width: 0, ..frame }.is_valid()); + assert!(!CaptureFrame { height: 0, ..frame }.is_valid()); + } +} diff --git a/native/crates/window/src/channel.rs b/native/crates/window/src/channel.rs index be53182..f5c7004 100644 --- a/native/crates/window/src/channel.rs +++ b/native/crates/window/src/channel.rs @@ -85,6 +85,15 @@ impl WindowState { /// Legacy two-pixels-per-unit flag, used when no quarter-step code is present. pub const DOUBLE_DENSITY: u32 = 1 << 3; + /// A screenshot waits on this window (`res/SCREENSHOTS.md`, window + /// capture): present one frame even while not [`VISIBLE`](Self::VISIBLE). + /// The compositor copies the first frame committed after it set this, at + /// commit, while the client is still blocked in `present`; it clears the + /// flag once it has a copy. The client library folds it into + /// visibility, and presents its current backing itself when the app has + /// nothing to repaint, so no app needs code of its own. + pub const CAPTURE: u32 = 1 << 8; + // Four unused flag bits carry exact quarter steps without changing the // mapped state layout. Old peers retain the native/double fallback. const DENSITY_SHIFT: u32 = 4; @@ -221,8 +230,31 @@ mod tests { assert_eq!(state.density(), Density::Native); } + /// The capture request is its own bit: density updates never touch it, + /// and it never reads as a density. + #[test] + fn the_capture_flag_is_disjoint_from_every_other_bit() { + use crate::appearance::Density; + let named = WindowState::FOCUSED + | WindowState::VISIBLE + | WindowState::LIGHT + | WindowState::DOUBLE_DENSITY + | WindowState::DENSITY_MASK; + assert_eq!(WindowState::CAPTURE & named, 0); + let mut state = WindowState { + flags: WindowState::CAPTURE, + ..WindowState::default() + }; + for density in Density::ALL { + state.set_density(density); + assert_ne!(state.flags & WindowState::CAPTURE, 0); + assert_eq!(state.density(), density); + } + } + /// The whole pair — request/reply rings, commit plane, terminal - /// output, and this channel — fits the frozen pair lease. + /// output, this channel and the menu plane — fits the frozen pair + /// lease. #[test] fn the_pair_window_still_fits_its_lease() { let pair = kore_rt::svc::window_bytes::( @@ -233,7 +265,7 @@ mod tests { let output = kore_ring::spsc::region_bytes::( crate::wire::TERMINAL_OUTPUT_CAP, ); - let total = pair + commits + output + region_bytes(); + let total = pair + commits + output + region_bytes() + crate::menu::MENU_PLANE_BYTES; assert!( total <= crate::wire::SURFACE_PAIR_BYTES, "pair layout overflows the lease: {total:#x} > {:#x}", diff --git a/native/crates/window/src/client.rs b/native/crates/window/src/client.rs index fcebcd4..bd6892b 100644 --- a/native/crates/window/src/client.rs +++ b/native/crates/window/src/client.rs @@ -9,7 +9,7 @@ use core::{marker::PhantomData, ptr::NonNull}; use kore_abi::{ cap::Rights, session::{DISPLAY_FORMAT_XRGB8888, Notice, SlotRef, SurfaceMapLane}, - svc::{CommitWire, OutChunk, SurReqWire}, + svc::{CommitWire, MAX_COMMIT_RECTS, OutChunk, SurReqWire}, task::role, }; use kore_ring::spsc::{Consumer, Producer}; @@ -28,7 +28,7 @@ use crate::{ configure::Configure, control::{ControlRefusal, TabModel}, event::Event, - wire::{self, Commit, Damage, SurfaceRefusal, SurfaceRequest, SurfaceResponse}, + wire::{self, Commit, Damage, SurRect, SurfaceRefusal, SurfaceRequest, SurfaceResponse}, }; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -51,6 +51,8 @@ pub enum ClientError { CommitRejected, EpochExhausted, EventMalformed, + /// A menu the encoder refused before anything reached the compositor. + Menu(crate::menu::MenuError), } impl ClientError { @@ -159,6 +161,30 @@ impl PairMapping { } } +/// A frozen frame, mapped read-only for its capture writer. The view's +/// owner is held for as long as the pixels are borrowed. +pub struct CaptureView { + frame: crate::capture::CaptureFrame, + pixels: NonNull, + _view: StashedView, +} + +impl CaptureView { + pub const fn frame(&self) -> crate::capture::CaptureFrame { + self.frame + } + + /// Row `y`: `width` XRGB pixels, packed. + pub fn row(&self, y: u32) -> &[u32] { + let (width, height) = (usize::from(self.frame.width), u32::from(self.frame.height)); + assert!(y < height, "row {y} of a {height}-row frame"); + // SAFETY: `take_capture` checked the mapping holds `width * height` + // aligned pixels; the lend outlives `self`, and nothing writes the + // frozen frame once it is lent. + unsafe { core::slice::from_raw_parts(self.pixels.as_ptr().add(y as usize * width), width) } + } +} + /// Checked linear XRGB geometry; raw pointer access borrows the backing owner. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SurfaceInfo { @@ -311,6 +337,9 @@ pub struct WindowClient<'w, 't, T: Transport> { commits: Producer<'w, CommitWire>, output: Consumer<'w, OutChunk>, channel: crate::channel::WindowChannelSink<'w>, + /// The menu plane ([`crate::menu`]): written here, snapshotted by the + /// compositor on `ShowMenu` before it answers. + menu: &'w mut [u8], current: Option, previous: Option, tombstone: Option, @@ -319,6 +348,12 @@ pub struct WindowClient<'w, 't, T: Transport> { needs_configuration: bool, focused: bool, visible: bool, + /// The compositor's capture ask ([`WindowState::CAPTURE`]) as last + /// synced, and whether a frame was presented since it was raised. + /// + /// [`WindowState::CAPTURE`]: crate::channel::WindowState::CAPTURE + capture_asked: bool, + capture_answered: bool, observed_scheme: Option, observed_density: Option, queried: bool, @@ -344,12 +379,13 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { let (pair, aux) = memory.split_at_mut(pair_bytes); let rings = kore_rt::svc::client_rings::(pair, REQ_CAP, REP_CAP) .map_err(|_| ClientError::Ring)?; - let (commits, output, channel) = wire::split_window_aux(aux).ok_or(ClientError::Ring)?; + let aux = wire::split_window_aux(aux).ok_or(ClientError::Ring)?; Ok(Self { control: SvcClient::new(transport, rings, endpoints), - commits: kore_ring::spsc::producer(commits).map_err(|_| ClientError::Ring)?, - output: kore_ring::spsc::consumer(output).map_err(|_| ClientError::Ring)?, - channel: crate::channel::sink(channel).map_err(|_| ClientError::Ring)?, + commits: kore_ring::spsc::producer(aux.commits).map_err(|_| ClientError::Ring)?, + output: kore_ring::spsc::consumer(aux.output).map_err(|_| ClientError::Ring)?, + channel: crate::channel::sink(aux.events).map_err(|_| ClientError::Ring)?, + menu: aux.menu, current: None, previous: None, tombstone: None, @@ -358,6 +394,8 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { needs_configuration: true, focused: false, visible: true, + capture_asked: false, + capture_answered: false, observed_scheme: None, observed_density: None, queried: false, @@ -377,6 +415,10 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { /// window is minimized, closing, or fully occluded under a higher /// window's opaque core. While `false`, a client may stop producing /// frames and park on [`wait_change`](Self::wait_change). + /// + /// A screenshot of this window (`res/SCREENSHOTS.md`, window capture) + /// reads as visible until a frame is presented for it, so an app that + /// pauses while hidden draws its current state for the shot. pub const fn visible(&self) -> bool { self.visible } @@ -402,13 +444,32 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { pub async fn wait_change(&mut self, bound: core::time::Duration) { let seen = self.channel.seq(); - if self.change_pending() { + if self.change_pending() || self.answer_capture().await { return; } kore_rt::event::wait_past(&self.channel, seen, Some(bound)).await; let _ = self.sync_status(); } + /// A screenshot waits on this window and no frame has been presented + /// since it asked: present the current backing, unchanged, so the shot + /// is a whole frame of this window's own rather than the compositor's + /// late copy of a backing that may be mid-paint. Called only where the + /// app is about to park, so every writer has finished. Once per ask: + /// a refused or overtaken frame leaves the ask to the compositor's + /// bound. Whether a frame was offered. + async fn answer_capture(&mut self) -> bool { + // A capture must not publish an unpainted replacement backing. + if !self.capture_asked || self.capture_answered || self.needs_configuration || self.needs_initial_frame() { + return false; + } + self.capture_answered = true; + let mut rects = [SurRect::default(); MAX_COMMIT_RECTS]; + rects[0] = SurRect { x: 0, y: 0, w: 1, h: 1 }; + let _ = self.present(Damage::Rects { rects, count: 1 }).await; + true + } + /// [`wait_change`](Self::wait_change) for a window whose task also drains /// other planes woken through the same executor doorbell — a stream host /// whose child rings it (`res/SHELL_TERMINAL.md` §6). It publishes this @@ -419,7 +480,7 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { /// its own deadlines. pub async fn wait_any(&mut self, bound: core::time::Duration) { let seen = self.channel.seq(); - if self.change_pending() { + if self.change_pending() || self.answer_capture().await { return; } kore_rt::event::wait_once(&self.channel, seen, Some(bound)).await; @@ -436,6 +497,13 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { self.focused } + /// A new backing has no retained pixels, even if an overtaken resize + /// returned to the application's previous dimensions. Retained renderers + /// must paint it completely before publishing its first frame. + pub fn needs_initial_frame(&self) -> bool { + self.current.as_ref().is_some_and(|current| !current.destroyed && current.presented == 0) + } + pub fn surface(&self) -> Result, ClientError> { if self.needs_configuration { return Err(ClientError::Reconfigure); @@ -479,9 +547,11 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { self.observed_scheme.unwrap_or_default() } - /// Consume all event types. Old-coordinate input is discarded; newer input - /// first requests a configure and resets held keys/buttons. Close and reset - /// remain actionable regardless of a superseded geometry serial. + /// Consume all event types. Old-coordinate pointer input is discarded; + /// newer pointer input first requests a configure and resets held + /// buttons. Keyboard snapshots, focus, menu, close and reset carry no geometry + /// and remain actionable regardless of a superseded configure serial; a + /// newer serial on any of them still requests the configure. pub fn pop_event(&mut self) -> Result { let mut advanced = false; for _ in 0..crate::event::EVENT_CAP { @@ -662,6 +732,9 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { .push(wire) .map_err(|_| ClientError::CommitQueue)?; self.current.as_mut().unwrap().epoch = epoch; + // The first frame consumed after a capture ask is the one the + // compositor copies. + self.capture_answered |= self.capture_asked; // The record alone wakes nobody: without this kick the compositor // learns of the frame only when something else wakes it, and this // peer's status re-read below (bounded at `PRESENT_WAIT_BOUND_NS`) @@ -766,6 +839,113 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { } } + /// Show `cursor` while the pointer is over this window's content + /// ([`crate::cursor`]): the shape that says what a press there would + /// do. It holds until the window asks again, so ask on a change, not + /// per frame. A `Busy` refusal is retryable. + pub async fn set_cursor(&mut self, cursor: crate::cursor::Cursor) -> Result<(), ClientError> { + match self + .surface_request(SurfaceRequest::SetCursor { cursor }) + .await? + { + SurfaceResponse::CursorSet { cursor: held } if held == cursor => Ok(()), + _ => Err(ClientError::Protocol), + } + } + + /// Ask this window's controller to open the filesystem object with + /// stable identity `object` in the default handler for its `kind`. + /// Inert: the controller decides. A `Busy` refusal (its queue is full) + /// is retryable. + pub async fn open_object( + &mut self, + object: u64, + kind: crate::document::DocumentKind, + ) -> Result<(), ClientError> { + match self + .surface_request(SurfaceRequest::OpenObject { object, kind }) + .await? + { + SurfaceResponse::OpenQueued { object: queued } if queued == object => Ok(()), + _ => Err(ClientError::Protocol), + } + } + + /// Show a context menu at the content-relative anchor `(x, y)`: `build` + /// encodes it into the menu plane, and the compositor draws it once it + /// answers. The menu ends with an [`Event::Menu`] naming `serial` + /// (nonzero): the chosen entry's id, or zero if it was dismissed. A + /// refusal leaves no menu open; `Busy` is retryable. + pub async fn show_menu( + &mut self, + x: i32, + y: i32, + serial: u32, + build: impl FnOnce(&mut crate::menu::MenuBuilder<'_>), + ) -> Result<(), ClientError> { + if serial == 0 { + return Err(ClientError::Protocol); + } + let mut menu = crate::menu::MenuBuilder::new(self.menu); + build(&mut menu); + let len = menu.finish().map_err(ClientError::Menu)?; + let len = u16::try_from(len).map_err(|_| ClientError::Protocol)?; + match self + .surface_request(SurfaceRequest::ShowMenu { x, y, len, serial }) + .await? + { + SurfaceResponse::MenuShown { serial: shown } if shown == serial => Ok(()), + _ => Err(ClientError::Protocol), + } + } + + /// Ask the user, through this window's controller, to choose a file of + /// the types `types` accepts (a nonzero mask over `kore_kit::kinds` + /// rows). The answer arrives as [`Event::Picked`] naming `serial` + /// (nonzero), a chosen file's name just before it in `PickedName` + /// records ([`crate::pick::NameParts`]). `Busy` (the controller's queue + /// is full) is retryable. + pub async fn pick_object(&mut self, types: u64, serial: u32) -> Result<(), ClientError> { + if types == 0 || serial == 0 { + return Err(ClientError::Protocol); + } + match self + .surface_request(SurfaceRequest::PickObject { types, serial }) + .await? + { + SurfaceResponse::PickQueued { serial: queued } if queued == serial => Ok(()), + _ => Err(ClientError::Protocol), + } + } + + /// Bind this window to the pick it was launched for, by its `token`. + pub async fn claim_pick(&mut self, token: u64) -> Result<(), ClientError> { + match self.surface_request(SurfaceRequest::ClaimPick { token }).await? { + SurfaceResponse::PickClaimed { token: claimed } if claimed == token => Ok(()), + _ => Err(ClientError::Protocol), + } + } + + /// Answer the pick this window claimed: `object` and its `name` chosen, + /// or `object` zero and no name when the user cancelled. The name is + /// written at the head of the menu plane, which the compositor + /// snapshots before it answers. + pub async fn picked(&mut self, token: u64, object: u64, name: &[u8]) -> Result<(), ClientError> { + let chosen = object != 0; + if chosen != crate::pick::valid_name(name) || (!chosen && !name.is_empty()) { + return Err(ClientError::Protocol); + } + self.menu[..name.len()].copy_from_slice(name); + let len = name.len() as u8; + match self + .surface_request(SurfaceRequest::Picked { token, object, len }) + .await? + { + SurfaceResponse::PickAnswered { token: answered } if answered == token => Ok(()), + _ => Err(ClientError::Protocol), + } + } + async fn read_configuration(&mut self) -> Result { let reply = self .control @@ -812,7 +992,11 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { let Some(state) = self.channel.state() else { return Ok(()); }; - self.visible = state.flags & crate::channel::WindowState::VISIBLE != 0; + use crate::channel::WindowState; + self.capture_asked = state.flags & WindowState::CAPTURE != 0; + // A new ask re-arms only after one was seen settled. + self.capture_answered &= self.capture_asked; + self.visible = state.flags & (WindowState::VISIBLE | WindowState::CAPTURE) != 0; let Some(current) = self.current.as_mut() else { return Ok(()); }; @@ -892,6 +1076,150 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { Ok(()) } + /// The capture writer's one verb (`res/SCREENSHOTS.md`): borrow the + /// frozen frame this pair was admitted for. The lend carries ACCESS + /// without MODIFY, so the kernel maps it with no write permission; the + /// frame is checked against the mapping before any pixel is read. + pub async fn take_capture(&mut self) -> Result { + let mut reply = self + .control + .call(&Request::Surface(SurfaceRequest::TakeCapture)) + .await + .map_err(|_| ClientError::Transport)?; + let frame = match reply.response { + Response::Surface(SurfaceResponse::Capture(frame)) => frame, + Response::Surface(SurfaceResponse::Refused { refusal, .. }) => { + return Err(ClientError::SurfaceRefused(refusal)); + } + _ => return Err(ClientError::Protocol), + }; + let returned = reply.retained[0].take().ok_or(ClientError::Protocol)?; + let view = returned + .try_into_stashed_view(Rights::ACCESS) + .map_err(|_| ClientError::Protocol)?; + let lane = self.surface_lanes.claim().ok_or(ClientError::Busy)?; + let mapped = + verbs::map_stashed_view_surface_lane(&view, lane).ok_or(ClientError::Mapping)?; + let pixels = usize::try_from(frame.pixel_bytes()).map_err(|_| ClientError::Geometry)?; + if !frame.is_valid() + || mapped.len() < pixels + || mapped.cast::().as_ptr().addr() % 4 != 0 + { + return Err(ClientError::Geometry); + } + Ok(CaptureView { + frame, + pixels: mapped.cast::(), + _view: view, + }) + } + + /// Read or change preferences on a Settings-authorized pair. + pub async fn desktop( + &mut self, + request: crate::desktop::DesktopRequest, + ) -> Result { + // Shared uploads must retain their returned owner in `wallpaper`. + if matches!(request, crate::desktop::DesktopRequest::CreateWallpaper { .. }) { + return Err(ClientError::Protocol); + } + let reply = self + .control + .call(&Request::Desktop(request)) + .await + .map_err(|_| ClientError::Transport)?; + match reply.response { + Response::ControlRefused(reason) => Err(ClientError::Refused(reason)), + Response::Surface(SurfaceResponse::Refused { refusal, .. }) => { + Err(ClientError::SurfaceRefused(refusal)) + } + response => Ok(response), + } + } + + /// Upload an output-sized wallpaper through one temporary shared surface. + /// Cleanup keeps its exact view and lane in `tombstone` until revocation + /// settles; no mapping is reused after an ambiguous response. + pub async fn wallpaper( + &mut self, + width: u16, + height: u16, + mut pixel: impl FnMut(u32, u32) -> u32, + ) -> Result<(), ClientError> { + self.retire_tombstone().await?; + let mut reply = self + .control + .call(&Request::Desktop( + crate::desktop::DesktopRequest::CreateWallpaper { width, height }, + )) + .await + .map_err(|_| ClientError::Transport)?; + let (surface, stride, bytes) = match reply.response { + Response::Surface(SurfaceResponse::SurfaceMade { + surface, + stride, + bytes, + }) => (surface, stride, bytes), + Response::ControlRefused(reason) => return Err(ClientError::Refused(reason)), + Response::Surface(SurfaceResponse::Refused { refusal, .. }) => { + return Err(ClientError::SurfaceRefused(refusal)); + } + _ => return Err(ClientError::Protocol), + }; + self.tombstone = Some(BackingTombstone { + surface, + owner: reply.retained[0].take().map(CleanupOwner::Returned), + destroyed: false, + lane: None, + }); + let Some(CleanupOwner::Returned(returned)) = + self.tombstone.as_mut().and_then(|t| t.owner.take()) + else { + return Err(ClientError::Protocol); + }; + let view = match returned.try_into_stashed_view(Rights::ACCESS | Rights::MODIFY) { + Ok(view) => view, + Err(returned) => { + self.tombstone.as_mut().unwrap().owner = Some(CleanupOwner::Returned(returned)); + return Err(ClientError::Protocol); + } + }; + self.tombstone.as_mut().unwrap().owner = Some(CleanupOwner::View(view)); + let lane = self.surface_lanes.claim().ok_or(ClientError::Busy)?; + self.tombstone.as_mut().unwrap().lane = Some(lane); + let Some(CleanupOwner::View(view)) = self.tombstone.as_ref().and_then(|t| t.owner.as_ref()) + else { + return Err(ClientError::Protocol); + }; + let mapped = + verbs::map_stashed_view_surface_lane(view, lane).ok_or(ClientError::Mapping)?; + let info = SurfaceInfo::checked(surface, width, height, stride, bytes, mapped.len())?; + if mapped.cast::().as_ptr().addr() % 4 != 0 { + return Err(ClientError::Geometry); + } + for y in 0..u32::from(height) { + for x in 0..u32::from(width) { + // SAFETY: checked stride and span; this process is the upload's sole + // writer, and the compositor reads only after the following call. + unsafe { + mapped + .cast::() + .as_ptr() + .add(y as usize * info.stride / 4 + x as usize) + .write(pixel(x, y)); + } + } + } + let applied = self + .desktop(crate::desktop::DesktopRequest::ApplyWallpaper { surface }) + .await; + let cleanup = self.retire_tombstone().await; + match applied? { + Response::Accepted => cleanup, + _ => Err(ClientError::Protocol), + } + } + async fn create_backing(&mut self, config: Configure) -> Result<(), ClientError> { debug_assert!(self.current.is_none()); debug_assert!(self.tombstone.is_none()); @@ -1080,7 +1408,31 @@ fn valid_configuration(previous: Option, next: Configure) -> bool { } fn scoped_event(current: u64, serial: u64, event: Event) -> Option { - if matches!(event, Event::Reset | Event::CloseRequested) { + // A configure serial scopes geometry. A pointer report is geometry — its + // coordinates mean nothing under a superseded aperture — but a keyboard + // snapshot carries none, and the app derives presses by diffing it + // against its own previous snapshot, so a held key across a re-layout + // cannot become a phantom press. Dropping or resetting it instead loses + // the character outright: the compositor stamps keys with the layout + // still on screen while this client may already hold a newer serial it + // has not presented, and two quick re-layouts leave that window open + // exactly when a user types into the window that just grew (#1054). + // + // Focus carries no geometry either, and the compositor stamps it with + // the live configure serial, so during a re-layout it is always ahead + // of this client. Rewriting it to a Reset left the app unaware it was + // focused until the app layer's corrective Focus after the drain — too + // late for a key drained in the same pass, which the app's own focus + // guard then discarded. A menu choice names its own menu serial, not + // geometry, so a reconfigure while it was open cannot invalidate it. + if matches!( + event, + Event::Reset + | Event::CloseRequested + | Event::Keyboard { .. } + | Event::Focus(_) + | Event::Menu { .. } + ) { return Some(event); } if serial < current { @@ -1185,6 +1537,72 @@ mod tests { ); } + /// A capture ask reads as visibility and is answered once before a park: + /// the same ask never draws a second frame, however the first fared, + /// and a later ask re-arms only after the first was seen settled. + #[test] + fn a_capture_ask_is_visible_and_answered_once_per_ask() { + use crate::channel::WindowState; + use kore_rt::svc::SvcProtocol; + #[repr(align(64))] + struct Buffer([u8; N]); + let mut pair = Buffer([0; wire::SURFACE_PAIR_BYTES]); + let mut event_memory = Buffer([0; 4096]); + WindowWire::init_pair_window(&mut pair.0, REQ_CAP, REP_CAP).unwrap(); + let transport = CountingTransport::new(); + let mut client = WindowClient::from_memory( + &transport, + &mut pair.0, + ClientEndpoints { + kick: SlotRef::from_wire(1), + grants: [const { None }; MAX_GRANTS], + }, + ) + .unwrap(); + let (mut producer, channel) = kore_ring::event::split::( + &mut event_memory.0, + crate::event::EVENT_CAP, + ) + .unwrap(); + client.channel = channel; + + // Minimized, and the app has not configured yet: the ask reads as + // visibility, but there is no frame to offer. + let mut state = WindowState::default(); + producer.replace_state(state); + client.sync_status().unwrap(); + assert!(!client.visible()); + state.flags = WindowState::CAPTURE; + producer.replace_state(state); + client.sync_status().unwrap(); + assert!(client.visible(), "an ask reads as visible"); + assert!(!drive(client.answer_capture()), "no configuration: nothing to present"); + assert!(!client.capture_answered); + + // Configured, the backing not yet live: the one offer is spent even + // though the frame failed, so a wait returns once and then parks. + client.needs_configuration = false; + assert!(drive(client.answer_capture()), "the ask is answered before parking"); + assert!(client.capture_answered); + assert!(!drive(client.answer_capture()), "the same ask never draws twice"); + client.sync_status().unwrap(); + assert!(client.capture_answered, "a resync of the same ask keeps it spent"); + assert_eq!(transport.commits.get(), 0, "a frame that failed kicks nobody"); + + // The compositor settles the ask: the window is hidden again, and a + // fresh ask is answered anew. + state.flags = 0; + producer.replace_state(state); + client.sync_status().unwrap(); + assert!(!client.visible() && !client.capture_answered); + assert!(!drive(client.answer_capture())); + state.flags = WindowState::CAPTURE | WindowState::VISIBLE; + producer.replace_state(state); + client.sync_status().unwrap(); + assert!(drive(client.answer_capture()), "a new ask is answered"); + assert!(client.visible()); + } + #[test] fn actual_event_queue_filters_old_geometry_and_updates_focus() { use kore_rt::svc::SvcProtocol; @@ -1223,9 +1641,9 @@ mod tests { focused: false, }); client.needs_configuration = false; - producer - .push(EventWire::encode(1, Event::Focus(true))) - .unwrap(); + // Focus is not geometry and is never filtered by serial (see + // `a_focus_change_reaches_the_app_before_the_key_typed_under_it`); + // the old-geometry filtering under test here is the pointer's. producer .push(EventWire::encode(2, Event::Focus(true))) .unwrap(); @@ -1246,7 +1664,7 @@ mod tests { x: 10, y: 20, buttons: 1, - wheel: 0, + scroll: crate::event::Scroll::NONE, }, )) .unwrap(); @@ -1328,16 +1746,19 @@ mod tests { }); client.needs_configuration = false; - // A resize can overtake records already in the bounded ring. They are - // intentionally hidden from the application, but consuming them still - // returns producer capacity and must wake a pending Reset retry. + // A resize can overtake coordinate records already in the bounded + // ring. They are intentionally hidden from the application, but + // consuming them still returns producer capacity and must wake a + // pending Reset retry. (Keyboard snapshots are not hidden this way; + // see `a_keystroke_crosses_a_configure_boundary_in_either_direction`.) producer .push(EventWire::encode( 1, - Event::Keyboard { - source: 0, - modifiers: 0, - keys: [4, 0, 0, 0, 0, 0], + Event::Pointer { + x: 23, + y: 12, + buttons: 1, + scroll: crate::event::Scroll::NONE, }, )) .unwrap(); @@ -1525,24 +1946,77 @@ mod tests { assert_eq!(lanes.live(), 0); } + /// #1054: a keyboard snapshot has no geometry, so a configure serial must + /// never decide whether it reaches the app. Across two quick re-layouts + /// the compositor keeps stamping keys with the layout still on screen + /// while this client has already acknowledged a newer configure it has + /// not presented yet; a key typed then was silently dropped as stale, and + /// one typed under an even newer stamp was replaced by a Reset. Either + /// way the character was gone and the window looked focused and alive. + #[test] + fn a_keystroke_crosses_a_configure_boundary_in_either_direction() { + let keyboard = Event::Keyboard { + source: 1, + modifiers: 0x02, + keys: [0x1e, 0, 0, 0, 0, 0], + locks: kore_abi::keyboard::Locks::NONE, + }; + assert_eq!( + scoped_event(2, 1, keyboard), + Some(keyboard), + "a key stamped with the still-presented layout is not stale input" + ); + assert_eq!( + scoped_event(2, 3, keyboard), + Some(keyboard), + "a key stamped ahead of this client's configure is still the key" + ); + } + + /// #1054, the half that decides whether the delivered key is *used*: the + /// compositor stamps `Focus` with the live configure serial, so during a + /// re-layout it is always ahead of this client and was rewritten to a + /// Reset. The app never learned it was focused, and a key drained in the + /// same pass — before the app layer's corrective Focus — was discarded by + /// the app's own `if focused` guard. Focus carries no geometry either. + #[test] + fn a_focus_change_reaches_the_app_before_the_key_typed_under_it() { + let keyboard = Event::Keyboard { + source: 1, + modifiers: 0x02, + keys: [0x1e, 0, 0, 0, 0, 0], + locks: kore_abi::keyboard::Locks::NONE, + }; + assert_eq!( + scoped_event(2, 3, Event::Focus(true)), + Some(Event::Focus(true)) + ); + assert_eq!( + scoped_event(2, 1, Event::Focus(false)), + Some(Event::Focus(false)) + ); + assert_eq!(scoped_event(2, 3, keyboard), Some(keyboard)); + } + #[test] fn input_never_uses_an_old_coordinate_configuration_and_reset_survives() { let pointer = Event::Pointer { x: 23, y: 12, buttons: 1, - wheel: 0, + scroll: crate::event::Scroll::NONE, }; let keyboard = Event::Keyboard { source: 2, modifiers: 0, keys: [4, 0, 0, 0, 0, 0], + locks: kore_abi::keyboard::Locks::CAPS, }; assert_eq!(scoped_event(2, 1, pointer), None); - assert_eq!(scoped_event(2, 1, keyboard), None); assert_eq!(scoped_event(2, 3, pointer), Some(Event::Reset)); - assert_eq!(scoped_event(2, 3, keyboard), Some(Event::Reset)); assert_eq!(scoped_event(2, 2, pointer), Some(pointer)); + // Keyboard snapshots are not coordinate input: every serial delivers + // the key itself (see `a_keystroke_crosses_a_configure_boundary_in_either_direction`). assert_eq!(scoped_event(2, 2, keyboard), Some(keyboard)); assert_eq!(scoped_event(2, 1, Event::Reset), Some(Event::Reset)); assert_eq!( @@ -1550,7 +2024,6 @@ mod tests { Some(Event::CloseRequested) ); assert_eq!(scoped_event(2, 3, Event::Configure), Some(Event::Configure)); - assert_eq!(scoped_event(2, 1, Event::Focus(false)), None); } #[test] diff --git a/native/crates/window/src/control.rs b/native/crates/window/src/control.rs index a28d50a..8aec8e2 100644 --- a/native/crates/window/src/control.rs +++ b/native/crates/window/src/control.rs @@ -5,7 +5,9 @@ //! arrival identity, not an opcode or a terminal number supplied by a client, //! decides which vocabulary the service permits. +use crate::capture::{CaptureMode, CaptureRefusal}; use crate::configure::{Configure, Content, Preference}; +use crate::document::{DocumentKind, KindMask}; use crate::wire::{self as surface, SurfaceRequest, SurfaceResponse}; use kore_abi::{ session::SlotRef, @@ -14,6 +16,11 @@ use kore_abi::{ use kore_rt::svc::{PeerBundlePart, PeerId}; use kore_svc_output::service::{OutputRequest, OutputResponse}; +use crate::appearance::Density; +use crate::desktop::{DesktopRequest, DesktopState, Mode}; +const PREPARE_SETTINGS: u8 = 0x52; +const DESKTOP: u8 = 0x53; + const OUTPUT: u8 = 0x80; const ADMIT: u8 = 0x40; const TAKE: u8 = 0x41; @@ -30,6 +37,9 @@ const SET_TABS: u8 = 0x4b; const PUBLISH_DOCK_ITEM: u8 = 0x4c; const TAKE_SHELL_ACTION: u8 = 0x4d; const SET_LAYOUT_HINTS: u8 = 0x4e; +const PREPARE_CAPTURE: u8 = 0x4f; +const REQUEST_CAPTURE: u8 = 0x50; +const RELEASE_CAPTURE: u8 = 0x51; const ACCEPTED: u8 = 0x40; const ADMITTED: u8 = 0x41; const REFUSED: u8 = 0x42; @@ -38,7 +48,7 @@ const SHELL_ACTION: u8 = 0x4d; /// Bound on one window's published tab strip. pub const MAX_TABS: usize = 8; /// Bound on the compositor-drawn dock catalog. -pub const MAX_DOCK_ITEMS: usize = 8; +pub const MAX_DOCK_ITEMS: usize = crate::MAX_DOCK_ITEMS; /// NUL-padded printable-ASCII bytes carried by one dock label. pub const DOCK_LABEL_BYTES: usize = 24; @@ -150,6 +160,10 @@ pub struct DockItem { icon: u64, kind: u8, flags: u8, + /// The document classes this row's app opens — its handler contract, + /// so "Open With" lists it for those kinds. Data, never authority: the + /// controller still decides what an activation launches. + opens: KindMask, label: [u8; DOCK_LABEL_BYTES], } @@ -173,6 +187,7 @@ impl DockItem { kind, flags: 0, icon: 0, + opens: KindMask::NONE, label, }) } @@ -209,6 +224,15 @@ impl DockItem { self.icon } + /// The same row declaring the document classes its app opens. + pub const fn with_opens(self, opens: KindMask) -> Self { + Self { opens, ..self } + } + + pub const fn opens(&self) -> KindMask { + self.opens + } + pub const fn kind(&self) -> u8 { self.kind } @@ -253,33 +277,126 @@ impl DockItem { pub enum ShellAction { /// Dock row `index` of catalog `version` was activated. Launch { version: u16, index: u8 }, + /// A window this controller launched asked to open the filesystem + /// object with stable identity `object` (never zero) in the default + /// handler for its `kind`. Data only: the opened window locates it from + /// its own root. + Open { object: u64, kind: DocumentKind }, + /// A window's "Open With" menu chose dock row `index` of catalog + /// `version` to open the object with stable identity `object`. + OpenWith { version: u16, index: u8, object: u64 }, + /// The seat's screen is frozen (`res/SCREENSHOTS.md`): launch the + /// capture writer on a pair prepared with [`DisplayRequest:: + /// PrepareCapture`] for `ticket`. The geometry is data for the + /// controller's own books; the writer reads the frame's own. + /// `window` is the window a [`CaptureMode::Window`] frame was copied + /// from, so the controller can name its app; `None` for every other + /// mode. Inert identity: the controller matches it against the peers it + /// launched. + Captured { + ticket: u32, + mode: CaptureMode, + width: u16, + height: u16, + window: Option, + }, + /// A capture of `mode` did not happen, and why. The controller says so. + CaptureRefused { + mode: CaptureMode, + refusal: CaptureRefusal, + }, + /// A window this controller launched asked the user to choose a file of + /// the types `types` accepts (`res/FILE_PICKER.md`): launch the picker + /// with `token` (never zero), by which it claims the pick. Data only; + /// the compositor brokers the answer. + Pick { token: u64, types: u64 }, } impl ShellAction { - const fn from_wire(id: u64, a: u64) -> Option> { + /// The dock row word `Launch` and `OpenWith` share: `version | index << 16`. + fn row(word: u64) -> Option<(u16, u8)> { + let (version, index) = (word as u16, (word >> 16) as u8); + (word >> 24 == 0 && version != 0 && (index as usize) < MAX_DOCK_ITEMS).then_some((version, index)) + } + + fn from_wire(id: u64, a: u64, b: u64) -> Option> { match id { - 0 => { - if a != 0 { - return None; - } - Some(None) - } - 1 => { - let version = a as u16; - let index = (a >> 16) as u8; - if a >> 24 != 0 || version == 0 || index as usize >= MAX_DOCK_ITEMS { - return None; - } + 0 if a == 0 && b == 0 => Some(None), + 1 if b == 0 => { + let (version, index) = Self::row(a)?; Some(Some(Self::Launch { version, index })) } + 2 if a != 0 => Some(Some(Self::Open { + object: a, + kind: DocumentKind::from_code(b)?, + })), + 3 if a != 0 => { + let (version, index) = Self::row(b)?; + Some(Some(Self::OpenWith { + version, + index, + object: a, + })) + } + 4 if a as u32 != 0 && a >> 40 == 0 && b >> 48 == 0 => { + let (width, height) = (b as u16, (b >> 16) as u16); + (width != 0 && height != 0).then_some(())?; + let mode = CaptureMode::from_code((a >> 32) as u8)?; + let window = match (b >> 32) as u16 { + 0 => None, + arrival => Some(PeerId::from_arrival(arrival)?), + }; + // Only a window capture names a window, and it always does. + (window.is_some() == (mode == CaptureMode::Window)).then_some(())?; + Some(Some(Self::Captured { + ticket: a as u32, + mode, + width, + height, + window, + })) + } + 5 if a >> 8 == 0 && b >> 8 == 0 => Some(Some(Self::CaptureRefused { + mode: CaptureMode::from_code(a as u8)?, + refusal: CaptureRefusal::from_code(b as u8)?, + })), + 6 if a != 0 && b != 0 => Some(Some(Self::Pick { token: a, types: b })), _ => None, } } - const fn words(action: Option) -> (u64, u64) { + const fn words(action: Option) -> (u64, u64, u64) { match action { - None => (0, 0), - Some(Self::Launch { version, index }) => (1, version as u64 | ((index as u64) << 16)), + None => (0, 0, 0), + Some(Self::Launch { version, index }) => { + (1, version as u64 | ((index as u64) << 16), 0) + } + Some(Self::Open { object, kind }) => (2, object, kind.code() as u64), + Some(Self::OpenWith { + version, + index, + object, + }) => (3, object, version as u64 | ((index as u64) << 16)), + Some(Self::Captured { + ticket, + mode, + width, + height, + window, + }) => ( + 4, + ticket as u64 | (mode.code() as u64) << 32, + width as u64 + | (height as u64) << 16 + | match window { + Some(peer) => (peer.arrival() as u64) << 32, + None => 0, + }, + ), + Some(Self::CaptureRefused { mode, refusal }) => { + (5, mode.code() as u64, refusal.code() as u64) + } + Some(Self::Pick { token, types }) => (6, token, types), } } } @@ -338,12 +455,17 @@ impl kore_rt::svc::SvcProtocol for WindowWire { index == 0 } fn request_grants(request: &SurReqWire) -> usize { - usize::from(matches!( - decode_req(request), - Some(DisplayRequest::Surface( - SurfaceRequest::CreateSurface { .. } - )) - )) + usize::from( + matches!( + decode_req(request), + Some(DisplayRequest::Desktop( + DesktopRequest::CreateWallpaper { .. } + )) + ) || matches!( + decode_req(request), + Some(DisplayRequest::Surface(request)) if surface::request_carries_grant(&request) + ), + ) } fn init_pair_window( window: &mut [u8], @@ -469,6 +591,12 @@ impl LayoutHints { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DisplayRequest { + /// Controller-only: authorize this admission for desktop preferences. + PrepareSettings { + preference: Preference, + icon: u64, + }, + Desktop(DesktopRequest), Output(OutputRequest), Surface(SurfaceRequest), AdmitPeer { @@ -492,6 +620,10 @@ pub enum DisplayRequest { PrepareWindow { content: Content, preference: Preference, + /// The app icon the window presents under (a `kore_app_icons` id, the + /// one its dock row publishes; `0` = none). Inert identity, never + /// authority: it is how a running dock row finds its own windows. + icon: u64, }, Configure, Acknowledge { @@ -515,10 +647,32 @@ pub enum DisplayRequest { }, /// Controller verb: dequeue the oldest pending dock action, if any. TakeShellAction, + /// Controller verb: the next peer this controller admits is the + /// capture writer for `ticket` (from [`ShellAction::Captured`]). It + /// gets no window; its one verb is + /// [`SurfaceRequest::TakeCapture`], and its teardown retires the + /// frozen frame. + PrepareCapture { ticket: u32 }, + /// Controller verb: capture this controller's seat, as its + /// PrintScreen chord would (the `screenshot` command). A + /// [`CaptureMode::Window`] capture takes `window`, a peer this + /// controller launched, or the seat's focused window when `None`; every + /// other mode names none. + RequestCapture { + mode: CaptureMode, + window: Option, + }, + /// Controller verb: give back the frozen frame for `ticket` without + /// writing it (the writer could not be launched). A frame a writer + /// was admitted for is retired by that writer's teardown instead. + ReleaseCapture { + ticket: u32, + }, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DisplayResponse { + Desktop(DesktopState), Output(OutputResponse), Surface(SurfaceResponse), Accepted, @@ -552,6 +706,34 @@ pub enum ControlRefusal { pub fn encode_req(request: &DisplayRequest) -> SurReqWire { let mut w = surface::encode_req(&SurfaceRequest::Query); match *request { + DisplayRequest::PrepareSettings { preference, icon } => { + w.op = PREPARE_SETTINGS; + let (kind, value) = preference.words(); + w.a = kind.into(); + w.b = value.into(); + w.c = icon.into(); + } + DisplayRequest::Desktop(request) => { + w.op = DESKTOP; + match request { + DesktopRequest::Query => {} + DesktopRequest::RestoreComplete => w.subject = 4.into(), + DesktopRequest::Appearance { mode, density } => { + w.subject = 1.into(); + w.a = (mode as u64).into(); + w.b = (density.quarters() as u64).into(); + } + DesktopRequest::CreateWallpaper { width, height } => { + w.subject = 2.into(); + w.a = u64::from(width).into(); + w.b = u64::from(height).into(); + } + DesktopRequest::ApplyWallpaper { surface } => { + w.subject = 3.into(); + w.a = surface.into(); + } + } + } DisplayRequest::Surface(request) => return surface::encode_req(&request), DisplayRequest::Output(request) => { let out = kore_svc_output::wire::encode_req(&request); @@ -578,12 +760,14 @@ pub fn encode_req(request: &DisplayRequest) -> SurReqWire { DisplayRequest::PrepareWindow { content, preference, + icon, } => { w.op = PREPARE; w.subject = u64::from(content == Content::Application).into(); let (kind, value) = preference.words(); w.a = kind.into(); w.b = value.into(); + w.c = icon.into(); } DisplayRequest::Configure => w.op = CONFIGURE, DisplayRequest::Acknowledge { serial, surface } => { @@ -642,7 +826,8 @@ pub fn encode_req(request: &DisplayRequest) -> SurReqWire { | (u64::from(index) << 16) | (u64::from(count) << 24) | (u64::from(item.kind) << 32) - | (u64::from(item.flags) << 40)) + | (u64::from(item.flags) << 40) + | (u64::from(item.opens.bits()) << 48)) .into(); let [b, c, d] = label_words(&item.label); w.b = b.into(); @@ -650,12 +835,43 @@ pub fn encode_req(request: &DisplayRequest) -> SurReqWire { w.d = d.into(); } DisplayRequest::TakeShellAction => w.op = TAKE_SHELL_ACTION, + DisplayRequest::PrepareCapture { ticket } => { + w.op = PREPARE_CAPTURE; + w.subject = u64::from(ticket).into(); + } + DisplayRequest::RequestCapture { mode, window } => { + w.op = REQUEST_CAPTURE; + w.subject = window.map_or(0, |peer| u64::from(peer.arrival())).into(); + w.a = u64::from(mode.code()).into(); + } + DisplayRequest::ReleaseCapture { ticket } => { + w.op = RELEASE_CAPTURE; + w.subject = u64::from(ticket).into(); + } } w } pub fn decode_req(w: &SurReqWire) -> Option { let request = match w.op { + PREPARE_SETTINGS => DisplayRequest::PrepareSettings { + preference: Preference::from_words(w.a.get(), w.b.get())?, + icon: w.c.get(), + }, + DESKTOP => DisplayRequest::Desktop(match w.subject.get() { + 0 => DesktopRequest::Query, + 4 => DesktopRequest::RestoreComplete, + 1 => DesktopRequest::Appearance { + mode: Mode::from_code(u8::try_from(w.a.get()).ok()?)?, + density: Density::from_quarters(u8::try_from(w.b.get()).ok()?)?, + }, + 2 if w.a.get() != 0 && w.b.get() != 0 => DesktopRequest::CreateWallpaper { + width: u16::try_from(w.a.get()).ok()?, + height: u16::try_from(w.b.get()).ok()?, + }, + 3 if w.a.get() != 0 => DesktopRequest::ApplyWallpaper { surface: w.a.get() }, + _ => return None, + }), ADMIT => DisplayRequest::AdmitPeer { part: match w.a.get() { 0 => PeerBundlePart::Ring, @@ -677,6 +893,7 @@ pub fn decode_req(w: &SurReqWire) -> Option { _ => return None, }, preference: Preference::from_words(w.a.get(), w.b.get())?, + icon: w.c.get(), }, CONFIGURE => DisplayRequest::Configure, ACK_CONFIGURE if w.a.get() != 0 && w.subject.get() != 0 => DisplayRequest::Acknowledge { @@ -722,8 +939,7 @@ pub fn decode_req(w: &SurReqWire) -> Option { PUBLISH_DOCK_ITEM => { let a = w.a.get(); let (version, index, count) = (a as u16, (a >> 16) as u8, (a >> 24) as u8); - if a >> 48 != 0 - || version == 0 + if version == 0 || count == 0 || count as usize > MAX_DOCK_ITEMS || index >= count @@ -739,10 +955,32 @@ pub fn decode_req(w: &SurReqWire) -> Option { (a >> 40) as u8, label_bytes([w.b.get(), w.c.get(), w.d.get()]), )? - .with_icon(w.subject.get()), + .with_icon(w.subject.get()) + .with_opens(KindMask::from_bits((a >> 48) as u16)?), } } TAKE_SHELL_ACTION => DisplayRequest::TakeShellAction, + PREPARE_CAPTURE => DisplayRequest::PrepareCapture { + ticket: u32::try_from(w.subject.get()) + .ok() + .filter(|ticket| *ticket != 0)?, + }, + REQUEST_CAPTURE => { + let mode = CaptureMode::from_code(u8::try_from(w.a.get()).ok()?)?; + let window = match u16::try_from(w.subject.get()).ok()? { + 0 => None, + arrival => Some(PeerId::from_arrival(arrival)?), + }; + if window.is_some() && mode != CaptureMode::Window { + return None; + } + DisplayRequest::RequestCapture { mode, window } + } + RELEASE_CAPTURE => DisplayRequest::ReleaseCapture { + ticket: u32::try_from(w.subject.get()) + .ok() + .filter(|ticket| *ticket != 0)?, + }, op if op & OUTPUT != 0 => { DisplayRequest::Output(kore_svc_output::wire::decode_req(&OutReqWire { op: op & !OUTPUT, @@ -764,6 +1002,12 @@ pub fn encode_rep(response: &DisplayResponse, grants: u8) -> SurRepWire { // overwrites the tag and its own fields. let mut w = surface::encode_rep(&SurfaceResponse::Destroyed { surface: 0 }, grants); match *response { + DisplayResponse::Desktop(state) => { + w.tag = DESKTOP; + w.id = (state.mode as u64).into(); + w.a = u64::from(state.density.quarters()).into(); + w.b = (u64::from(state.width) | u64::from(state.height) << 16).into(); + } DisplayResponse::Surface(response) => return surface::encode_rep(&response, grants), DisplayResponse::Output(response) => { let out = kore_svc_output::wire::encode_rep(&response, grants); @@ -804,9 +1048,10 @@ pub fn encode_rep(response: &DisplayResponse, grants: u8) -> SurRepWire { } DisplayResponse::ShellAction(action) => { w.tag = SHELL_ACTION; - let (id, a) = ShellAction::words(action); + let (id, a, b) = ShellAction::words(action); w.id = id.into(); w.a = a.into(); + w.b = b.into(); } } w @@ -814,6 +1059,16 @@ pub fn encode_rep(response: &DisplayResponse, grants: u8) -> SurRepWire { pub fn decode_rep(w: &SurRepWire) -> Option<(DisplayResponse, u8)> { let response = match w.tag { + DESKTOP + if w.b.get() >> 32 == 0 && w.b.get() as u16 != 0 && (w.b.get() >> 16) as u16 != 0 => + { + DisplayResponse::Desktop(DesktopState { + mode: Mode::from_code(u8::try_from(w.id.get()).ok()?)?, + density: Density::from_quarters(u8::try_from(w.a.get()).ok()?)?, + width: w.b.get() as u16, + height: (w.b.get() >> 16) as u16, + }) + } ACCEPTED => DisplayResponse::Accepted, ADMITTED => DisplayResponse::Admitted { peer: PeerId::from_arrival(u16::try_from(w.id.get()).ok()?)?, @@ -862,7 +1117,7 @@ pub fn decode_rep(w: &SurRepWire) -> Option<(DisplayResponse, u8)> { }) } SHELL_ACTION => { - DisplayResponse::ShellAction(ShellAction::from_wire(w.id.get(), w.a.get())?) + DisplayResponse::ShellAction(ShellAction::from_wire(w.id.get(), w.a.get(), w.b.get())?) } tag if tag & OUTPUT != 0 => { DisplayResponse::Output(kore_svc_output::wire::decode_rep(&OutRepWire { @@ -889,6 +1144,26 @@ pub fn reply_matches(w: &SurReqWire, response: &DisplayResponse, grants: u8) -> return false; }; match (request, *response) { + ( + DisplayRequest::Desktop(DesktopRequest::CreateWallpaper { .. }), + DisplayResponse::Surface(SurfaceResponse::SurfaceMade { .. }), + ) => grants == 1, + (DisplayRequest::Desktop(_), DisplayResponse::Surface(SurfaceResponse::Refused { .. })) => { + grants == 0 + } + ( + DisplayRequest::Desktop(DesktopRequest::Query | DesktopRequest::Appearance { .. }), + DisplayResponse::Desktop(_), + ) => grants == 0, + ( + DisplayRequest::Desktop(DesktopRequest::ApplyWallpaper { .. } | DesktopRequest::RestoreComplete) + | DisplayRequest::PrepareSettings { .. }, + DisplayResponse::Accepted, + ) => grants == 0, + ( + DisplayRequest::Desktop(_) | DisplayRequest::PrepareSettings { .. }, + DisplayResponse::ControlRefused(_), + ) => grants == 0, (DisplayRequest::Surface(_), DisplayResponse::Surface(response)) => { surface::reply_matches(w, &response, grants) } @@ -911,7 +1186,10 @@ pub fn reply_matches(w: &SurReqWire, response: &DisplayResponse, grants: u8) -> | DisplayRequest::Focus { .. } | DisplayRequest::SetTabs(_) | DisplayRequest::SetLayoutHints(_) - | DisplayRequest::PublishDockItem { .. }, + | DisplayRequest::PublishDockItem { .. } + | DisplayRequest::PrepareCapture { .. } + | DisplayRequest::RequestCapture { .. } + | DisplayRequest::ReleaseCapture { .. }, DisplayResponse::Accepted, ) => true, (DisplayRequest::TakeShellAction, DisplayResponse::ShellAction(_)) => true, @@ -939,7 +1217,10 @@ pub fn reply_matches(w: &SurReqWire, response: &DisplayResponse, grants: u8) -> | DisplayRequest::SetTabs(_) | DisplayRequest::SetLayoutHints(_) | DisplayRequest::PublishDockItem { .. } - | DisplayRequest::TakeShellAction, + | DisplayRequest::TakeShellAction + | DisplayRequest::PrepareCapture { .. } + | DisplayRequest::RequestCapture { .. } + | DisplayRequest::ReleaseCapture { .. }, DisplayResponse::ControlRefused(_), ) => true, (DisplayRequest::CancelAdmission, DisplayResponse::ControlRefused(_)) => true, @@ -972,6 +1253,12 @@ mod tests { width: 2, height: 3, }, + icon: 0x9e37_79b9_7f4a_7c15, + }, + DisplayRequest::PrepareWindow { + content: Content::Terminal, + preference: Preference::Tile, + icon: 0, }, DisplayRequest::Configure, DisplayRequest::Acknowledge { @@ -1042,6 +1329,148 @@ mod tests { } } + #[test] + fn desktop_requests_and_state_are_strict_and_grants_are_correlated() { + use kore_rt::svc::SvcProtocol; + let state = DesktopState { mode: Mode::System, density: Density::Double, width: 1920, height: 1200 }; + let requests = [ + DisplayRequest::PrepareSettings { preference: Preference::Tile, icon: 42 }, + DisplayRequest::Desktop(DesktopRequest::Query), + DisplayRequest::Desktop(DesktopRequest::RestoreComplete), + DisplayRequest::Desktop(DesktopRequest::Appearance { mode: state.mode, density: state.density }), + DisplayRequest::Desktop(DesktopRequest::CreateWallpaper { width: state.width, height: state.height }), + DisplayRequest::Desktop(DesktopRequest::ApplyWallpaper { surface: 42 }), + ]; + for request in requests { + let wire = encode_req(&request); + assert_eq!(decode_req(&wire), Some(request)); + assert_eq!(WindowWire::request_grants(&wire), usize::from(matches!(request, DisplayRequest::Desktop(DesktopRequest::CreateWallpaper { .. })))); + let mut invalid = wire; invalid.d = 1.into(); + assert_eq!(decode_req(&invalid), None); + assert!(reply_matches(&wire, &DisplayResponse::ControlRefused(ControlRefusal::AccessDenied), 0)); + assert!(!reply_matches(&wire, &DisplayResponse::Accepted, 1)); + } + let reply = DisplayResponse::Desktop(state); + assert!(reply_matches(&encode_req(&requests[2]), &DisplayResponse::Accepted, 0)); + assert!(!reply_matches(&encode_req(&requests[2]), &reply, 0)); + assert_eq!(decode_rep(&encode_rep(&reply, 0)), Some((reply, 0))); + assert_eq!(decode_rep(&encode_rep(&reply, 1)), None); + for invalid in [ + DesktopRequest::CreateWallpaper { width: 0, height: 1 }, + DesktopRequest::ApplyWallpaper { surface: 0 }, + ] { assert_eq!(decode_req(&encode_req(&DisplayRequest::Desktop(invalid))), None); } + let mut bad = encode_req(&requests[3]); bad.a = 3.into(); + assert_eq!(decode_req(&bad), None); + let mut bad = encode_req(&requests[3]); bad.b = 11.into(); + assert_eq!(decode_req(&bad), None); + assert!(!reply_matches(&encode_req(&requests[4]), &reply, 0)); + } + + /// The capture vocabulary rides the controller face byte-exactly: a + /// zero ticket, an unknown mode or refusal, and stray bits refuse. + #[test] + fn capture_verbs_and_actions_round_trip_and_fail_closed() { + for request in [ + DisplayRequest::PrepareCapture { ticket: 1 }, + DisplayRequest::PrepareCapture { ticket: u32::MAX }, + DisplayRequest::RequestCapture { + mode: CaptureMode::Screen, + window: None, + }, + DisplayRequest::RequestCapture { + mode: CaptureMode::Region, + window: None, + }, + DisplayRequest::RequestCapture { + mode: CaptureMode::Window, + window: None, + }, + DisplayRequest::RequestCapture { + mode: CaptureMode::Window, + window: PeerId::from_arrival(0x1_05), + }, + DisplayRequest::ReleaseCapture { ticket: 3 }, + ] { + let wire = encode_req(&request); + assert_eq!(decode_req(&wire), Some(request)); + assert!(reply_matches(&wire, &DisplayResponse::Accepted, 0)); + assert!(reply_matches( + &wire, + &DisplayResponse::ControlRefused(ControlRefusal::Busy), + 0 + )); + assert!(!reply_matches(&wire, &DisplayResponse::Accepted, 1)); + let mut stray = wire; + stray.d = 1.into(); + assert_eq!(decode_req(&stray), None); + } + let mut zero = encode_req(&DisplayRequest::PrepareCapture { ticket: 1 }); + zero.subject = 0.into(); + assert_eq!(decode_req(&zero), None, "a zero ticket names nothing"); + let mut wide = zero; + wide.subject = (u64::from(u32::MAX) + 1).into(); + assert_eq!(decode_req(&wide), None); + let mut unknown = encode_req(&DisplayRequest::RequestCapture { + mode: CaptureMode::Screen, + window: None, + }); + unknown.a = 9.into(); + assert_eq!(decode_req(&unknown), None); + let mut named = encode_req(&DisplayRequest::RequestCapture { + mode: CaptureMode::Screen, + window: None, + }); + named.subject = 0x1_05.into(); + assert_eq!(decode_req(&named), None, "only a window capture names a window"); + named.a = u64::from(CaptureMode::Window.code()).into(); + named.subject = 0x05.into(); + assert_eq!(decode_req(&named), None, "a static pair is no window"); + named.subject = 0x1_0000.into(); + assert_eq!(decode_req(&named), None, "an arrival is 16 bits"); + + for action in [ + ShellAction::Captured { + ticket: 7, + mode: CaptureMode::Screen, + width: 2560, + height: 1600, + window: None, + }, + ShellAction::Captured { + ticket: u32::MAX, + mode: CaptureMode::Window, + width: 640, + height: 400, + window: PeerId::from_arrival(0xffff), + }, + ShellAction::CaptureRefused { + mode: CaptureMode::Window, + refusal: CaptureRefusal::NoWindow, + }, + ] { + let response = DisplayResponse::ShellAction(Some(action)); + let wire = encode_rep(&response, 0); + assert_eq!(decode_rep(&wire), Some((response, 0))); + let mut stray = wire; + stray.c = 1.into(); + assert_eq!(decode_rep(&stray), None); + } + for (a, b) in [ + (7 | 1 << 32, 0), // zero geometry + (1 << 32, 2560 | 1600 << 16), // zero ticket + (7 | 9 << 32, 2560 | 1600 << 16), // unknown mode + (7 | 1 << 40, 2560 | 1600 << 16), // a bit past the mode + (7 | 1 << 32, 2560 | 1600 << 16 | 0x1_05 << 32), // a screen names no window + (7 | 2 << 32, 2560 | 1600 << 16), // a window capture names its window + (7 | 2 << 32, 2560 | 1600 << 16 | 0x05 << 32), // a static pair is no window + (7 | 2 << 32, 2560 | 1600 << 16 | 1 << 48), // a bit past the window + ] { + assert_eq!(ShellAction::from_wire(4, a, b), None, "{a:#x} {b:#x}"); + } + assert_eq!(ShellAction::from_wire(5, 1, 0), None, "no refusal code"); + assert_eq!(ShellAction::from_wire(5, 1, 9), None, "unknown refusal"); + } + #[test] fn surface_clients_share_wire_but_cannot_accept_control_replies() { let request = surface::encode_req(&SurfaceRequest::Status { surface: 9 }); @@ -1273,14 +1702,26 @@ mod tests { let mut wire = publish; wire.a = (1 | (1 << 16) | (1 << 24) | (2 << 32)).into(); assert_eq!(decode_req(&wire), None); - wire.a = (1 | (9 << 24) | (2 << 32)).into(); + wire.a = (1 | (((MAX_DOCK_ITEMS + 1) as u64) << 24) | (2 << 32)).into(); assert_eq!(decode_req(&wire), None); - // Undefined flag bits and bits above the flag byte fail closed; - // the defined running bit decodes to the flagged row. + // Undefined flag bits and handler bits no document class has fail + // closed; the defined running bit decodes to the flagged row, and + // the handler mask to the row that opens those classes. wire.a = (1 | (1 << 24) | (2 << 32) | (2 << 40)).into(); assert_eq!(decode_req(&wire), None); - wire.a = (1 | (1 << 24) | (2 << 32) | (1 << 48)).into(); + wire.a = (1 | (1 << 24) | (2 << 32) | (1 << (48 + DocumentKind::ALL.len()))).into(); assert_eq!(decode_req(&wire), None); + let editor = KindMask::of(&[DocumentKind::Text, DocumentKind::Code]); + wire.a = (1 | (1 << 24) | (2 << 32) | (u64::from(editor.bits()) << 48)).into(); + assert_eq!( + decode_req(&wire), + Some(DisplayRequest::PublishDockItem { + version: 1, + index: 0, + count: 1, + item: item.with_opens(editor), + }) + ); wire.a = (1 | (1 << 24) | (2 << 32) | (1 << 40)).into(); assert_eq!( decode_req(&wire), @@ -1308,6 +1749,62 @@ mod tests { wire.b = 1.into(); assert_eq!(decode_rep(&wire), None); } + // Opens carry their object's class in `b`; an "Open With" its dock + // row there, under the same row law as a launch. + for response in [ + DisplayResponse::ShellAction(Some(ShellAction::Open { + object: 0x5eed, + kind: DocumentKind::Unknown, + })), + DisplayResponse::ShellAction(Some(ShellAction::Open { + object: u64::MAX, + kind: DocumentKind::Binary, + })), + DisplayResponse::ShellAction(Some(ShellAction::OpenWith { + version: 4, + index: 7, + object: 9, + })), + DisplayResponse::ShellAction(Some(ShellAction::Pick { + token: u64::MAX, + types: 0b100, + })), + ] { + let wire = encode_rep(&response, 0); + assert_eq!(decode_rep(&wire), Some((response, 0))); + } + // A pick names its token and at least one type. + let pick = encode_rep(&DisplayResponse::ShellAction(Some(ShellAction::Pick { token: 5, types: 1 })), 0); + for (a, b) in [(0, 1), (5, 0)] { + let mut bad = pick; + bad.a = a.into(); + bad.b = b.into(); + assert_eq!(decode_rep(&bad), None, "token {a} types {b}"); + } + let mut open = encode_rep( + &DisplayResponse::ShellAction(Some(ShellAction::Open { + object: 1, + kind: DocumentKind::Text, + })), + 0, + ); + open.b = (DocumentKind::ALL.len() as u64).into(); + assert_eq!(decode_rep(&open), None); + let mut with = encode_rep( + &DisplayResponse::ShellAction(Some(ShellAction::OpenWith { + version: 4, + index: 0, + object: 9, + })), + 0, + ); + with.b = (4 | ((MAX_DOCK_ITEMS as u64) << 16)).into(); + assert_eq!(decode_rep(&with), None, "a row past the dock"); + with.b = 0.into(); + assert_eq!(decode_rep(&with), None, "no catalog version"); + with.b = 4.into(); + with.a = 0.into(); + assert_eq!(decode_rep(&with), None, "no object"); let mut wire = encode_rep(&DisplayResponse::ShellAction(None), 0); wire.a = 1.into(); assert_eq!(decode_rep(&wire), None); @@ -1316,12 +1813,28 @@ mod tests { assert_eq!(decode_rep(&wire), None); wire.id = 1.into(); assert_eq!(decode_rep(&wire), None); - wire.a = (1 | (8 << 16)).into(); + wire.a = (1 | ((MAX_DOCK_ITEMS as u64) << 16)).into(); assert_eq!(decode_rep(&wire), None); wire.a = (1 | (1 << 24)).into(); assert_eq!(decode_rep(&wire), None); } + #[test] + fn the_ninth_dock_row_round_trips_publication_and_launch() { + let request = DisplayRequest::PublishDockItem { + version: 1, + index: 8, + count: 9, + item: DockItem::from_label(DOCK_KIND_APPLICATION, "Skylilt").unwrap(), + }; + assert_eq!(decode_req(&encode_req(&request)), Some(request)); + let response = DisplayResponse::ShellAction(Some(ShellAction::Launch { + version: 1, + index: 8, + })); + assert_eq!(decode_rep(&encode_rep(&response, 0)), Some((response, 0))); + } + #[test] fn configure_refuses_empty_geometry_or_serial_and_ack_refuses_zero_subjects() { let configured = DisplayResponse::Configured(Configure { diff --git a/native/crates/window/src/cursor.rs b/native/crates/window/src/cursor.rs new file mode 100644 index 0000000..7805564 --- /dev/null +++ b/native/crates/window/src/cursor.rs @@ -0,0 +1,107 @@ +//! The cursor a window asks for over its own content +//! ([`SurReqWire::OP_SET_CURSOR`]). +//! +//! What the pointer shows says what a press would do there, and only the app +//! knows that about its own pixels: text takes a caret, a control activates. +//! The compositor still chooses for the pixels it owns — chrome, the dock, +//! the desktop — and draws every shape from one family, so a window names a +//! shape and never supplies an image. + +use kore_abi::svc::SurReqWire; + +/// One cursor a window may ask for. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Cursor { + /// The standard arrow. + #[default] + Default, + /// The I-beam: text a press would place a caret in. + Text, + /// The pointing hand: a control a press would activate. + Pointer, + /// The open hand: something a press would pick up. + Grab, + /// The closed hand: something being carried. + Grabbing, + /// The vertical resize arrow. + ResizeNs, + /// The horizontal resize arrow. + ResizeEw, + /// The diagonal resize arrow from top-left to bottom-right. + ResizeNwse, + /// The diagonal resize arrow from top-right to bottom-left. + ResizeNesw, + /// The hand under a no-entry badge: what a press here would not do. + NotAllowed, + /// The spinner: work under way while the window still answers. + Progress, +} + +impl Cursor { + /// Every cursor, in wire-code order. + pub const ALL: [Self; 11] = [ + Self::Default, + Self::Text, + Self::Pointer, + Self::Grab, + Self::Grabbing, + Self::ResizeNs, + Self::ResizeEw, + Self::ResizeNwse, + Self::ResizeNesw, + Self::NotAllowed, + Self::Progress, + ]; + + /// The [`SurReqWire`] `CURSOR_*` code this cursor rides as. + pub const fn code(self) -> u64 { + match self { + Self::Default => SurReqWire::CURSOR_DEFAULT, + Self::Text => SurReqWire::CURSOR_TEXT, + Self::Pointer => SurReqWire::CURSOR_POINTER, + Self::Grab => SurReqWire::CURSOR_GRAB, + Self::Grabbing => SurReqWire::CURSOR_GRABBING, + Self::ResizeNs => SurReqWire::CURSOR_RESIZE_NS, + Self::ResizeEw => SurReqWire::CURSOR_RESIZE_EW, + Self::ResizeNwse => SurReqWire::CURSOR_RESIZE_NWSE, + Self::ResizeNesw => SurReqWire::CURSOR_RESIZE_NESW, + Self::NotAllowed => SurReqWire::CURSOR_NOT_ALLOWED, + Self::Progress => SurReqWire::CURSOR_PROGRESS, + } + } + + /// The cursor a wire code names; `None` for a code no cursor has, which + /// the codec refuses rather than guessing. + pub const fn from_code(code: u64) -> Option { + Some(match code { + SurReqWire::CURSOR_DEFAULT => Self::Default, + SurReqWire::CURSOR_TEXT => Self::Text, + SurReqWire::CURSOR_POINTER => Self::Pointer, + SurReqWire::CURSOR_GRAB => Self::Grab, + SurReqWire::CURSOR_GRABBING => Self::Grabbing, + SurReqWire::CURSOR_RESIZE_NS => Self::ResizeNs, + SurReqWire::CURSOR_RESIZE_EW => Self::ResizeEw, + SurReqWire::CURSOR_RESIZE_NWSE => Self::ResizeNwse, + SurReqWire::CURSOR_RESIZE_NESW => Self::ResizeNesw, + SurReqWire::CURSOR_NOT_ALLOWED => Self::NotAllowed, + SurReqWire::CURSOR_PROGRESS => Self::Progress, + _ => return None, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_cursor_round_trips_its_code_and_no_other_code_names_one() { + for (index, cursor) in Cursor::ALL.into_iter().enumerate() { + assert_eq!(cursor.code(), index as u64, "{cursor:?}"); + assert_eq!(Cursor::from_code(cursor.code()), Some(cursor)); + } + for code in [Cursor::ALL.len() as u64, 255, u64::MAX] { + assert_eq!(Cursor::from_code(code), None); + } + } +} diff --git a/native/crates/window/src/desktop.rs b/native/crates/window/src/desktop.rs new file mode 100644 index 0000000..2a0a55f --- /dev/null +++ b/native/crates/window/src/desktop.rs @@ -0,0 +1,46 @@ +//! Settings-only desktop controls. File paths remain in the Settings process. +use crate::appearance::Density; +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(u8)] +pub enum Mode { + Light = 0, + #[default] + Dark = 1, + System = 2, +} +impl Mode { + pub const fn from_code(v: u8) -> Option { + match v { + 0 => Some(Self::Light), + 1 => Some(Self::Dark), + 2 => Some(Self::System), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DesktopRequest { + Query, + /// Startup restoration finished, including fallback after unreadable preferences. + RestoreComplete, + Appearance { + mode: Mode, + density: Density, + }, + /// One temporary shared upload, independent of the app's window size. + CreateWallpaper { + width: u16, + height: u16, + }, + /// Snapshot the owned upload into the compositor's retained background. + ApplyWallpaper { + surface: u64, + }, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DesktopState { + pub mode: Mode, + pub density: Density, + pub width: u16, + pub height: u16, +} diff --git a/native/crates/window/src/document.rs b/native/crates/window/src/document.rs new file mode 100644 index 0000000..154f79a --- /dev/null +++ b/native/crates/window/src/document.rs @@ -0,0 +1,106 @@ +//! What kind of document an object is, as apps hand documents to each other. +//! +//! A dagfs object carries no content type; its kind is read from its name +//! (`kore_kit::kinds`). Handlers declare the kinds they open by class, not by +//! extension — the editor opens text and code, an image viewer images — so a +//! class is what crosses the wire: in `OpenObject`'s second word, a "Open +//! With" menu entry, and a dock row's handler mask. Codes are stable: a +//! class is added at the end, never renumbered. + +#[repr(u8)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum DocumentKind { + /// No class known: the opener falls back to the catalogue's document app. + #[default] + Unknown = 0, + /// Prose and notes: plain text, Markdown. + Text = 1, + /// Source and settings: Lua, Python, C, Rust, Zig, TOML, JSON, YAML. + Code = 2, + Image = 3, + /// Page documents: PDF, word processor files. + Document = 4, + Spreadsheet = 5, + /// Executable or binary formats (WebAssembly). + Binary = 6, +} + +impl DocumentKind { + pub const ALL: [DocumentKind; 7] = [ + Self::Unknown, + Self::Text, + Self::Code, + Self::Image, + Self::Document, + Self::Spreadsheet, + Self::Binary, + ]; + + pub const fn code(self) -> u8 { + self as u8 + } + + /// The class `code` names; `None` for a code no class has. + pub fn from_code(code: u64) -> Option { + Self::ALL.into_iter().find(|kind| u64::from(kind.code()) == code) + } + + /// This class's bit in a [`KindMask`]. + pub const fn bit(self) -> u16 { + 1 << self as u8 + } +} + +/// A set of [`DocumentKind`]s — the classes one handler opens. Bit `n` is +/// the class with code `n`; bits no class has are refused on the wire. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct KindMask(u16); + +impl KindMask { + pub const NONE: Self = Self(0); + const VALID: u16 = (1 << DocumentKind::ALL.len()) - 1; + + pub const fn of(kinds: &[DocumentKind]) -> Self { + let mut bits = 0; + let mut i = 0; + while i < kinds.len() { + bits |= kinds[i].bit(); + i += 1; + } + Self(bits) + } + + pub const fn bits(self) -> u16 { + self.0 + } + + pub fn from_bits(bits: u16) -> Option { + (bits & !Self::VALID == 0).then_some(Self(bits)) + } + + pub const fn contains(self, kind: DocumentKind) -> bool { + self.0 & kind.bit() != 0 + } + + pub const fn is_empty(self) -> bool { + self.0 == 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn codes_are_stable_and_masks_refuse_unknown_bits() { + for (code, kind) in DocumentKind::ALL.into_iter().enumerate() { + assert_eq!(usize::from(kind.code()), code); + assert_eq!(DocumentKind::from_code(code as u64), Some(kind)); + } + assert_eq!(DocumentKind::from_code(DocumentKind::ALL.len() as u64), None); + let editor = KindMask::of(&[DocumentKind::Text, DocumentKind::Code]); + assert!(editor.contains(DocumentKind::Code) && !editor.contains(DocumentKind::Image)); + assert_eq!(KindMask::from_bits(editor.bits()), Some(editor)); + assert_eq!(KindMask::from_bits(1 << DocumentKind::ALL.len()), None); + } +} diff --git a/native/crates/window/src/event.rs b/native/crates/window/src/event.rs index c518a5b..fd2d84d 100644 --- a/native/crates/window/src/event.rs +++ b/native/crates/window/src/event.rs @@ -1,6 +1,9 @@ //! Ordered, bounded compositor-to-client events in the window's pair lease. //! Keyboard reports are complete HID state snapshots, not lossy key edges. +/// The vocabulary an [`Event::Keyboard`] is read in — its usages, modifier +/// bits and lock state — so a client names keys without a second source. +pub use kore_abi::keyboard::{Locks, modifier, usage}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned, little_endian::U64}; pub const EVENT_CAP: u32 = 64; @@ -23,17 +26,21 @@ pub enum Event { /// Re-read Configure. Geometry is never inferred from a pointer event. Configure, Focus(bool), + /// One complete keyboard snapshot: the HID modifier bitmap, up to six + /// held Keyboard/Keypad-page usages, and the seat's lock state, so a + /// client resolves keys exactly as the terminal discipline does. Keyboard { source: u8, modifiers: u8, keys: [u8; 6], + locks: Locks, }, /// Coordinates are relative to the configured content bounds. Pointer { x: i32, y: i32, buttons: u8, - wheel: i16, + scroll: Scroll, }, CloseRequested, /// Queue overflow or focus loss: release all client-held keys/buttons. @@ -43,6 +50,151 @@ pub enum Event { TabActivated { index: u8, }, + /// The context menu this window showed as `serial` has ended: `chosen` + /// is the id of the entry chosen, or zero if it was dismissed + /// (`crate::menu`). An "Open With" entry reports its own id, if it has + /// one, once the compositor has queued the open. + Menu { + serial: u32, + chosen: u16, + }, + /// The pick this window asked for as `serial` has ended (`crate::pick`): + /// `object` is the chosen file's stable identity, data the window opens + /// from its own root, or zero if the user cancelled. A chosen file's + /// name arrived just before, in `PickedName` records. + Picked { + serial: u32, + object: u64, + }, + /// Part `part` of the name of the file chosen for pick `serial`: the + /// name's bytes from `part * NAME_PART_BYTES`, zero-padded + /// ([`crate::pick::NameParts`] puts them back together). + PickedName { + serial: u32, + part: u8, + bytes: [u8; crate::pick::NAME_PART_BYTES], + }, +} + +/// How far one pointer event scrolls, in [`ScrollUnit`]s. Positive `y` +/// scrolls toward the top (a wheel turned away from the user: the view moves +/// back toward its start), positive `x` toward the right. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Scroll { + pub x: i16, + pub y: i16, + /// What the steps measure. A scroll of zero has no unit: it encodes as + /// [`ScrollUnit::Detent`]. + pub unit: ScrollUnit, +} + +/// What a [`Scroll`]'s steps measure, numbered as the pointer transport's +/// scroll record numbers them (`kore_abi::pointer::Record::Scroll`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ScrollUnit { + /// Wheel detents: the view moves by its own wheel step per detent. + #[default] + Detent = 0, + /// Pixels of the window's content at its scale: a precision touchpad's + /// fingers, which the content should follow. + Pixel = 1, + /// The view's own lines or rows. + Line = 2, +} + +impl Scroll { + /// No scrolling: a pointer event that only moves or clicks. + pub const NONE: Self = Self { + x: 0, + y: 0, + unit: ScrollUnit::Detent, + }; + + pub const fn is_none(self) -> bool { + self.x == 0 && self.y == 0 + } + + /// Bits 8.. of a pointer event's second word: vertical, horizontal, + /// then the unit. The vertical sits where the wheel always did, so a + /// wheel event's encoding is unchanged. + const fn bits(self) -> u64 { + if self.is_none() { + return 0; + } + (self.y as u16 as u64) << 8 | (self.x as u16 as u64) << 24 | (self.unit as u64) << 40 + } + + const fn from_bits(b: u64) -> Option { + let unit = match (b >> 40) & 0b11 { + 0 => ScrollUnit::Detent, + 1 => ScrollUnit::Pixel, + 2 => ScrollUnit::Line, + _ => return None, + }; + Some(Self { + x: (b >> 24) as u16 as i16, + y: (b >> 8) as u16 as i16, + unit, + }) + } +} + +/// Pointer scrolling as whole steps (rows, lines or list items down the +/// view; columns across) for a view that scrolls by them, positive toward +/// the top and the right as in [`Scroll`]. +/// +/// A detent is the view's wheel step, a line one step, and pixels fill +/// steps of the view's step size: what does not fill a step waits for the +/// next event, so slow fingers still scroll and fast ones scroll exactly as +/// far as they travel. A reversal starts from zero rather than first +/// unwinding the fraction left from the other way. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ScrollSteps { + /// Pixels toward the next step on each axis. + pending: [i64; 2], +} + +impl ScrollSteps { + /// The whole steps `scroll` moves on each axis (`[x, y]`). `step` is + /// the view's step on each axis in its content pixels (zero is taken as + /// one), and `per_detent` how many steps a wheel detent moves. + pub fn steps(&mut self, scroll: Scroll, step: [u32; 2], per_detent: i32) -> [i32; 2] { + let deltas = [i64::from(scroll.x), i64::from(scroll.y)]; + let mut out = [0i32; 2]; + for axis in 0..2 { + let delta = deltas[axis]; + out[axis] = match scroll.unit { + ScrollUnit::Detent | ScrollUnit::Line => { + if delta != 0 { + // A wheel between two strokes owes them nothing. + self.pending[axis] = 0; + } + let per = if scroll.unit == ScrollUnit::Line { + 1 + } else { + i64::from(per_detent) + }; + clamp(delta.saturating_mul(per)) + } + ScrollUnit::Pixel => { + let pending = &mut self.pending[axis]; + if delta.signum() * pending.signum() < 0 { + *pending = 0; + } + *pending += delta; + let size = i64::from(step[axis].max(1)); + let whole = *pending / size; + *pending -= whole * size; + clamp(whole) + } + }; + } + out + } +} + +fn clamp(steps: i64) -> i32 { + steps.clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32 } impl EventWire { @@ -65,22 +217,24 @@ impl EventWire { source, modifiers, keys, + locks, } => { wire.kind = 3; wire.a = u64::from_le_bytes([ source, modifiers, keys[0], keys[1], keys[2], keys[3], keys[4], keys[5], ]) .into(); + wire.b = u64::from(locks.bits()).into(); } Event::Pointer { x, y, buttons, - wheel, + scroll, } => { wire.kind = 4; wire.a = (u64::from(x as u32) | (u64::from(y as u32) << 32)).into(); - wire.b = (u64::from(buttons) | (u64::from(wheel as u16) << 8)).into(); + wire.b = (u64::from(buttons) | scroll.bits()).into(); } Event::CloseRequested => wire.kind = 5, Event::Reset => wire.kind = 6, @@ -88,6 +242,26 @@ impl EventWire { wire.kind = 7; wire.a = u64::from(index).into(); } + Event::Menu { serial, chosen } => { + wire.kind = 8; + wire.a = u64::from(serial).into(); + wire.b = u64::from(chosen).into(); + } + Event::Picked { serial, object } => { + wire.kind = 9; + wire.a = u64::from(serial).into(); + wire.b = object.into(); + } + // The name fills both words, so the pick's serial rides the + // reserved bytes and the part the flags: this kind alone. + Event::PickedName { serial, part, bytes } => { + wire.kind = 10; + wire.flags = part; + wire.reserved[..4].copy_from_slice(&serial.to_le_bytes()); + let (low, high) = bytes.split_at(8); + wire.a = u64::from_le_bytes(low.try_into().unwrap()).into(); + wire.b = u64::from_le_bytes(high.try_into().unwrap()).into(); + } } wire } @@ -98,23 +272,47 @@ impl EventWire { 2 if self.flags <= 1 => Event::Focus(self.flags != 0), 3 => { let [source, modifiers, a, b, c, d, e, f] = self.a.get().to_le_bytes(); + // A lock byte outside the representable locks is not a + // keyboard snapshot; the re-encode below refuses the rest. + let locks = u8::try_from(self.b.get()) + .ok() + .and_then(Locks::from_bits)?; Event::Keyboard { source, modifiers, keys: [a, b, c, d, e, f], + locks, } } 4 => Event::Pointer { x: self.a.get() as u32 as i32, y: (self.a.get() >> 32) as u32 as i32, buttons: self.b.get() as u8, - wheel: (self.b.get() >> 8) as u16 as i16, + scroll: Scroll::from_bits(self.b.get())?, }, 5 => Event::CloseRequested, 6 => Event::Reset, 7 if self.a.get() < crate::control::MAX_TABS as u64 => Event::TabActivated { index: self.a.get() as u8, }, + 8 if self.a.get() != 0 => Event::Menu { + serial: u32::try_from(self.a.get()).ok()?, + chosen: u16::try_from(self.b.get()).ok()?, + }, + 9 if self.a.get() != 0 => Event::Picked { + serial: u32::try_from(self.a.get()).ok()?, + object: self.b.get(), + }, + 10 if usize::from(self.flags) < crate::pick::NAME_PARTS => { + let serial = u32::from_le_bytes(self.reserved[..4].try_into().unwrap()); + if serial == 0 { + return None; + } + let mut bytes = [0; crate::pick::NAME_PART_BYTES]; + bytes[..8].copy_from_slice(&self.a.get().to_le_bytes()); + bytes[8..].copy_from_slice(&self.b.get().to_le_bytes()); + Event::PickedName { serial, part: self.flags, bytes } + } _ => return None, }; let serial = self.serial.get(); @@ -128,6 +326,15 @@ const _: () = assert!(core::mem::size_of::() == 32); mod tests { use super::*; + /// A wheel turned `y` detents (positive away from the user). + const fn detents(y: i16) -> Scroll { + Scroll { + x: 0, + y, + unit: ScrollUnit::Detent, + } + } + #[test] fn events_roundtrip_exactly_and_reject_unused_data() { for event in [ @@ -138,17 +345,67 @@ mod tests { source: 2, modifiers: 8, keys: [4, 0, 0, 0, 0, 0], + locks: Locks::NONE, + }, + Event::Keyboard { + source: 2, + modifiers: 0, + keys: [0x39, 0, 0, 0, 0, 0], + locks: Locks::CAPS.toggled(Locks::NUM), }, Event::Pointer { x: -1, y: 1200, buttons: 3, - wheel: -2, + scroll: detents(-2), + }, + Event::Pointer { + x: 4, + y: 5, + buttons: 0, + scroll: Scroll { + x: -300, + y: i16::MAX, + unit: ScrollUnit::Pixel, + }, + }, + Event::Pointer { + x: 4, + y: 5, + buttons: 1, + scroll: Scroll { + x: 2, + y: 0, + unit: ScrollUnit::Line, + }, }, Event::CloseRequested, Event::Reset, Event::TabActivated { index: 0 }, Event::TabActivated { index: 7 }, + Event::Menu { + serial: 1, + chosen: 0, + }, + Event::Menu { + serial: u32::MAX, + chosen: u16::MAX, + }, + Event::Picked { serial: 1, object: 0 }, + Event::Picked { + serial: u32::MAX, + object: u64::MAX, + }, + Event::PickedName { + serial: 1, + part: 0, + bytes: *b"beach.png\0\0\0\0\0\0\0", + }, + Event::PickedName { + serial: u32::MAX, + part: (crate::pick::NAME_PARTS - 1) as u8, + bytes: [0xff; crate::pick::NAME_PART_BYTES], + }, ] { let mut wire = EventWire::encode(17, event); assert_eq!(wire.decode(), Some((17, event))); @@ -162,16 +419,96 @@ mod tests { x: 0, y: 0, buttons: 0, - wheel: 0, + scroll: Scroll::NONE, + }, + ); + // A wheel event keeps the encoding it always had. + assert_eq!( + EventWire::encode( + 1, + Event::Pointer { + x: 0, + y: 0, + buttons: 2, + scroll: detents(-1), + }, + ) + .b + .get(), + 2 | 0xffff << 8 + ); + // An unknown unit, bits past the unit, and a unit on no scroll at all + // are refused. + for bad in [3u64 << 40, 1 << 42, 1 << 63, 1 << 40] { + pointer.b = bad.into(); + assert_eq!(pointer.decode(), None, "{bad:#x}"); + } + let mut keyboard = EventWire::encode( + 1, + Event::Keyboard { + source: 1, + modifiers: 0, + keys: [0; 6], + locks: Locks::SCROLL, }, ); - pointer.b = (1u64 << 24).into(); - assert_eq!(pointer.decode(), None); + for bad in [0x08, 0x100] { + keyboard.b = bad.into(); + assert_eq!(keyboard.decode(), None, "lock word {bad:#x}"); + } let mut tab = EventWire::encode(1, Event::TabActivated { index: 7 }); tab.a = 8.into(); assert_eq!(tab.decode(), None); tab.a = 0.into(); tab.flags = 1; assert_eq!(tab.decode(), None); + // A name's record names its pick and a part the name has. + let mut name = EventWire::encode(1, Event::PickedName { serial: 3, part: 1, bytes: [7; 16] }); + name.flags = crate::pick::NAME_PARTS as u8; + assert_eq!(name.decode(), None, "a part past the last"); + name.flags = 1; + name.reserved[..4].fill(0); + assert_eq!(name.decode(), None, "no pick's serial"); + let mut picked = EventWire::encode(1, Event::Picked { serial: 2, object: 5 }); + picked.reserved[0] = 2; + assert_eq!(picked.decode(), None, "only a name's record carries a serial there"); + } + + #[test] + fn scroll_steps_carry_pixels_into_whole_steps() { + let pixels = |x, y| Scroll { + x, + y, + unit: ScrollUnit::Pixel, + }; + let mut steps = ScrollSteps::default(); + // 20-pixel rows: 12 pixels is nothing yet, 12 more is one row. + assert_eq!(steps.steps(pixels(0, 12), [8, 20], 3), [0, 0]); + assert_eq!(steps.steps(pixels(0, 12), [8, 20], 3), [0, 1]); + assert_eq!(steps.steps(pixels(0, 36), [8, 20], 3), [0, 2], "4 + 36 = 40"); + // A reversal starts from zero instead of unwinding the 0 left over. + assert_eq!(steps.steps(pixels(0, 10), [8, 20], 3), [0, 0]); + assert_eq!(steps.steps(pixels(0, -19), [8, 20], 3), [0, 0]); + assert_eq!(steps.steps(pixels(0, -1), [8, 20], 3), [0, -1]); + // Across, the column width counts. + assert_eq!(steps.steps(pixels(-17, 0), [8, 20], 3), [-2, 0]); + // A detent is the view's wheel step and drops the pixel fraction; a + // line is one step. + assert_eq!(steps.steps(pixels(0, 15), [8, 20], 3), [0, 0]); + assert_eq!(steps.steps(detents(-2), [8, 20], 3), [0, -6]); + assert_eq!(steps.steps(pixels(0, 15), [8, 20], 3), [0, 0], "the 15 before the wheel is gone"); + let line = Scroll { + x: 1, + y: 2, + unit: ScrollUnit::Line, + }; + assert_eq!(steps.steps(line, [8, 20], 3), [1, 2]); + // A zero step size is one pixel, never a division by zero. + assert_eq!(steps.steps(pixels(0, 5), [0, 0], 3), [0, 5]); + assert_eq!( + steps.steps(detents(i16::MAX), [1, 1], i32::MAX), + [0, i32::MAX], + "saturates" + ); } } diff --git a/native/crates/window/src/lib.rs b/native/crates/window/src/lib.rs index f4518e0..b0e622d 100644 --- a/native/crates/window/src/lib.rs +++ b/native/crates/window/src/lib.rs @@ -3,11 +3,18 @@ #![no_std] pub mod appearance; +pub mod capture; #[cfg(feature = "runtime")] pub mod channel; #[cfg(feature = "runtime")] pub mod client; pub mod command; +pub mod cursor; +pub mod desktop; +pub mod document; +pub mod menu; +pub mod notify; +pub mod pick; #[cfg(feature = "runtime")] pub mod configure; #[cfg(feature = "runtime")] @@ -24,6 +31,8 @@ pub mod wire; /// Six windows fit the retained chrome scene and two buffers each fit the /// sixteen-slot surface engine. Controllers reserve separate serving peers. pub const BOOTSTRAP_WINDOW_SLOTS: usize = kore_abi::task::DESKTOP_BOOTSTRAP_WINDOW_SLOTS; +/// Dock entries in one published desktop catalogue, including the terminal. +pub const MAX_DOCK_ITEMS: usize = 9; /// Bounded number of in-flight surface commits. #[cfg(feature = "runtime")] diff --git a/native/crates/window/src/menu.rs b/native/crates/window/src/menu.rs new file mode 100644 index 0000000..fd84ec1 --- /dev/null +++ b/native/crates/window/src/menu.rs @@ -0,0 +1,635 @@ +//! Context menus the compositor draws for a window (`ShowMenu`). +//! +//! A menu is data, not authority: the app describes it — labels, shortcuts, +//! states, one level of submenus — in its pair lease's menu plane, and the +//! compositor draws it above every window, runs hover, keys and dismissal, +//! and answers with the chosen item's id ([`crate::event::Event::Menu`]). +//! One system entry, [`flag::OPEN_WITH`], carries an object and its +//! [`DocumentKind`] +//! instead of children: the compositor fills it from the shell's handler +//! table, so an app never learns which other apps exist. +//! +//! The encoding is little-endian and closed: a header, then entries in +//! display order, each a fixed head followed by its text. A submenu's +//! children follow it directly at depth 1. [`Menu::parse`] refuses anything +//! the [`MenuBuilder`] cannot write — unknown flags, nonzero reserved bytes, +//! text that is not printable UTF-8, a second submenu level, duplicate ids, +//! trailing bytes — so the compositor validates a snapshot it copied out of +//! the plane, never the live plane the app can still write. + +/// The menu plane: a page of the window pair lease after the event channel. +/// It bounds a whole encoded menu; the per-entry limits below bound each part. +pub const MENU_PLANE_BYTES: usize = 0x1000; +/// Top-level entries the compositor draws (separators included): the kit's +/// longest menu, the explorer's file menu, has fourteen. +pub const MAX_TOP_ENTRIES: usize = 16; +/// Children of one submenu. +pub const MAX_SUBMENU_ENTRIES: usize = 8; +/// Entries in one encoded menu, children included. +pub const MAX_ENTRIES: usize = 48; +/// UTF-8 bytes of one label ("Delete from All Folders…" is 26). +pub const LABEL_BYTES: usize = 48; +/// UTF-8 bytes of an entry's second line (the drop menu's "Stays in {folder} +/// as well"). +pub const DETAIL_BYTES: usize = 64; +/// UTF-8 bytes of a shortcut hint ("Ctrl+Shift+L"). +pub const SHORTCUT_BYTES: usize = 16; + +use crate::document::DocumentKind; + +const VERSION: u8 = 1; +const HEADER_BYTES: usize = 8; +const HEAD_BYTES: usize = 24; + +/// Entry flags. +pub mod flag { + /// Drawn, never chosen. + pub const DISABLED: u8 = 1 << 0; + /// A destructive command, drawn in the danger colour. + pub const DANGER: u8 = 1 << 1; + /// A check mark beside the label. + pub const CHECKED: u8 = 1 << 2; + /// A divider: no id, no text, no other flag. + pub const SEPARATOR: u8 = 1 << 3; + /// Opens the entries after it at depth 1; it is never chosen itself. + pub const SUBMENU: u8 = 1 << 4; + /// "Open With ▸": the compositor fills its submenu with the apps that + /// handle the entry's object kind and opens the object in the one chosen. + pub const OPEN_WITH: u8 = 1 << 5; + pub(super) const ALL: u8 = DISABLED | DANGER | CHECKED | SEPARATOR | SUBMENU | OPEN_WITH; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MenuError { + /// The menu does not fit the plane, or a text exceeds its limit. + TooLarge, + /// More entries than [`MAX_TOP_ENTRIES`], [`MAX_SUBMENU_ENTRIES`] or + /// [`MAX_ENTRIES`]. + TooManyEntries, + /// No top-level entry, or a submenu with no children. + Empty, + /// A version, flag, reserved byte or length the encoding does not have. + Malformed, + /// Text that is not UTF-8, or holds a control character. + BadText, + /// A child outside a submenu, a submenu inside a submenu, or a submenu + /// left open. + BadNesting, + /// A choosable entry with id zero, or two entries with one id. + BadId, +} + +/// Where one text lives in the encoded bytes. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Span { + at: u16, + len: u8, +} + +impl Span { + pub fn is_empty(self) -> bool { + self.len == 0 + } +} + +/// One parsed entry. Texts are [`Span`]s into the bytes it was parsed from; +/// read them with [`text`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Entry { + /// What [`crate::event::Event::Menu`] reports when this entry is chosen; + /// zero for separators and submenus. + pub id: u16, + pub flags: u8, + /// 0 at the top level, 1 inside a submenu. + pub depth: u8, + pub label: Span, + pub detail: Span, + pub shortcut: Span, + /// [`flag::OPEN_WITH`]'s object and its [`DocumentKind`] code; zero + /// otherwise. + pub object: u64, + pub kind: u32, + /// A submenu's child count; zero otherwise. + pub children: u8, +} + +impl Entry { + pub fn has(&self, flag: u8) -> bool { + self.flags & flag != 0 + } + /// "Open With"'s object class (parse admitted only known codes). + pub fn document_kind(&self) -> DocumentKind { + DocumentKind::from_code(u64::from(self.kind)).unwrap_or_default() + } + /// Whether choosing it ends the menu with its id: not a separator, a + /// submenu or "Open With", and not disabled. + pub fn choosable(&self) -> bool { + self.flags & (flag::SEPARATOR | flag::SUBMENU | flag::OPEN_WITH | flag::DISABLED) == 0 + } +} + +/// The text `span` names in `bytes`, the bytes [`Menu::parse`] accepted. +/// Bytes that changed since are read as no text, never as a panic. +pub fn text(bytes: &[u8], span: Span) -> &str { + let at = usize::from(span.at); + bytes + .get(at..at + usize::from(span.len)) + .and_then(|b| core::str::from_utf8(b).ok()) + .unwrap_or("") +} + +/// A validated menu: entries in display order, children after their parent. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Menu { + entries: [Entry; MAX_ENTRIES], + len: u8, +} + +fn printable(bytes: &[u8]) -> Result<(), MenuError> { + let text = core::str::from_utf8(bytes).map_err(|_| MenuError::BadText)?; + if text.chars().any(char::is_control) { + return Err(MenuError::BadText); + } + Ok(()) +} + +impl Menu { + /// Validate `bytes`, an encoded menu (exactly its length, no more). + pub fn parse(bytes: &[u8]) -> Result { + if bytes.len() > MENU_PLANE_BYTES { + return Err(MenuError::TooLarge); + } + let header = bytes.get(..HEADER_BYTES).ok_or(MenuError::Malformed)?; + if header[0] != VERSION || header[2..].iter().any(|&b| b != 0) { + return Err(MenuError::Malformed); + } + let count = usize::from(header[1]); + if count > MAX_ENTRIES { + return Err(MenuError::TooManyEntries); + } + let mut menu = Self { + entries: [Entry::default(); MAX_ENTRIES], + len: 0, + }; + let mut at = HEADER_BYTES; + let (mut top, mut parent) = (0, None::); + for index in 0..count { + let head = bytes.get(at..at + HEAD_BYTES).ok_or(MenuError::Malformed)?; + let word = |i: usize, n: usize| head[i..i + n].iter().rev().fold(0_u64, |w, &b| w << 8 | u64::from(b)); + let (id, flags, depth) = (word(0, 2) as u16, head[2], head[3]); + let lengths = [head[4], head[5], head[6]]; + let (object, kind) = (word(8, 8), word(16, 4) as u32); + if head[7] != 0 || word(20, 4) != 0 || flags & !flag::ALL != 0 || depth > 1 { + return Err(MenuError::Malformed); + } + let limits = [LABEL_BYTES, DETAIL_BYTES, SHORTCUT_BYTES]; + if lengths.iter().zip(limits).any(|(&n, limit)| usize::from(n) > limit) { + return Err(MenuError::TooLarge); + } + at += HEAD_BYTES; + let mut spans = [Span::default(); 3]; + for (span, &len) in spans.iter_mut().zip(&lengths) { + let text = bytes.get(at..at + usize::from(len)).ok_or(MenuError::Malformed)?; + printable(text)?; + *span = Span { at: at as u16, len }; + at += usize::from(len); + } + let entry = Entry { + id, + flags, + depth, + label: spans[0], + detail: spans[1], + shortcut: spans[2], + object, + kind, + children: 0, + }; + // The shape of each entry kind, exactly as the builder writes it. + let separator = entry.has(flag::SEPARATOR); + let submenu = entry.has(flag::SUBMENU); + let open_with = entry.has(flag::OPEN_WITH); + let shaped = if separator { + flags == flag::SEPARATOR && id == 0 && lengths == [0; 3] && object == 0 && kind == 0 + } else if submenu { + flags & !(flag::SUBMENU | flag::DISABLED) == 0 + && id == 0 + && depth == 0 + && lengths[0] > 0 + && lengths[2] == 0 + && object == 0 + && kind == 0 + } else if open_with { + flags & !(flag::OPEN_WITH | flag::DISABLED) == 0 + && depth == 0 + && lengths[0] > 0 + && lengths[2] == 0 + && object != 0 + && DocumentKind::from_code(u64::from(kind)).is_some() + } else { + id != 0 && lengths[0] > 0 && object == 0 && kind == 0 + }; + if !shaped { + return Err(if id == 0 && !separator && !submenu && !open_with { + MenuError::BadId + } else { + MenuError::Malformed + }); + } + if depth == 0 { + if let Some(p) = parent.take() { + if menu.entries[p].children == 0 { + return Err(MenuError::Empty); + } + } + top += 1; + if top > MAX_TOP_ENTRIES { + return Err(MenuError::TooManyEntries); + } + if submenu { + parent = Some(index); + } + } else { + let Some(p) = parent else { + return Err(MenuError::BadNesting); + }; + if submenu || open_with { + return Err(MenuError::BadNesting); + } + if usize::from(menu.entries[p].children) == MAX_SUBMENU_ENTRIES { + return Err(MenuError::TooManyEntries); + } + menu.entries[p].children += 1; + } + // Ids name one entry. Bounded by MAX_ENTRIES, a design constant. + if id != 0 && menu.entries[..index].iter().any(|e| e.id == id) { + return Err(MenuError::BadId); + } + menu.entries[index] = entry; + menu.len += 1; + } + if parent.is_some_and(|p| menu.entries[p].children == 0) || top == 0 { + return Err(MenuError::Empty); + } + if at != bytes.len() { + return Err(MenuError::Malformed); + } + Ok(menu) + } + + pub fn entries(&self) -> &[Entry] { + &self.entries[..usize::from(self.len)] + } + + /// The indices of the top-level entries, in order. + pub fn top(&self) -> impl Iterator + '_ { + (0..usize::from(self.len)).filter(|&i| self.entries[i].depth == 0) + } + + /// The children of the submenu at `index` (empty for any other entry). + pub fn children(&self, index: usize) -> &[Entry] { + let Some(entry) = self.entries().get(index) else { + return &[]; + }; + let first = index + 1; + &self.entries[first..first + usize::from(entry.children)] + } + + /// The entry with `id`, if a choosable one has it. + pub fn find(&self, id: u16) -> Option<&Entry> { + self.entries().iter().find(|e| e.id == id && id != 0) + } +} + +/// One choosable entry for [`MenuBuilder::item`]. +#[derive(Clone, Copy, Debug)] +pub struct Item<'s> { + id: u16, + label: &'s str, + detail: &'s str, + shortcut: &'s str, + flags: u8, +} + +impl<'s> Item<'s> { + /// An entry reported as `id` (nonzero, unique in its menu). + pub const fn new(id: u16, label: &'s str) -> Self { + Self { + id, + label, + detail: "", + shortcut: "", + flags: 0, + } + } + /// A right-aligned shortcut hint, drawn faint. + pub const fn shortcut(mut self, shortcut: &'s str) -> Self { + self.shortcut = shortcut; + self + } + /// A second line under the label. + pub const fn detail(mut self, detail: &'s str) -> Self { + self.detail = detail; + self + } + pub const fn enabled(mut self, enabled: bool) -> Self { + if enabled { + self.flags &= !flag::DISABLED; + } else { + self.flags |= flag::DISABLED; + } + self + } + pub const fn checked(mut self, checked: bool) -> Self { + if checked { + self.flags |= flag::CHECKED; + } else { + self.flags &= !flag::CHECKED; + } + self + } + pub const fn danger(mut self) -> Self { + self.flags |= flag::DANGER; + self + } +} + +/// Encodes a menu into a plane, in display order. The first error sticks and +/// [`finish`](Self::finish) reports it; nothing reaches the compositor that +/// [`Menu::parse`] would refuse. +pub struct MenuBuilder<'b> { + buf: &'b mut [u8], + at: usize, + count: usize, + top: usize, + /// The open submenu's head offset and child count. + submenu: Option<(usize, usize)>, + ids: [u16; MAX_ENTRIES], + error: Option, +} + +impl<'b> MenuBuilder<'b> { + pub fn new(buf: &'b mut [u8]) -> Self { + let error = (buf.len() < HEADER_BYTES).then_some(MenuError::TooLarge); + Self { + buf, + at: HEADER_BYTES, + count: 0, + top: 0, + submenu: None, + ids: [0; MAX_ENTRIES], + error, + } + } + + pub fn item(&mut self, item: Item<'_>) -> &mut Self { + if item.id == 0 { + return self.fail(MenuError::BadId); + } + if self.ids[..self.count].contains(&item.id) { + return self.fail(MenuError::BadId); + } + let depth = u8::from(self.submenu.is_some()); + self.entry(item.id, item.flags, depth, [item.label, item.detail, item.shortcut], 0, 0) + } + + pub fn separator(&mut self) -> &mut Self { + let depth = u8::from(self.submenu.is_some()); + self.entry(0, flag::SEPARATOR, depth, ["", "", ""], 0, 0) + } + + /// Start a submenu: the next entries, until + /// [`end_submenu`](Self::end_submenu), are its children. + pub fn submenu(&mut self, label: &str) -> &mut Self { + if self.submenu.is_some() { + return self.fail(MenuError::BadNesting); + } + let head = self.at; + self.entry(0, flag::SUBMENU, 0, [label, "", ""], 0, 0); + if self.error.is_none() { + self.submenu = Some((head, 0)); + } + self + } + + pub fn end_submenu(&mut self) -> &mut Self { + match self.submenu.take() { + Some((_, 0)) => self.fail(MenuError::Empty), + Some(_) => self, + None => self.fail(MenuError::BadNesting), + } + } + + /// "Open With ▸" for `object` of `kind`, labelled by the app. `id`, if + /// nonzero, is reported when the object is opened through it. + pub fn open_with(&mut self, id: u16, label: &str, object: u64, kind: DocumentKind) -> &mut Self { + if self.submenu.is_some() { + return self.fail(MenuError::BadNesting); + } + if object == 0 || (id != 0 && self.ids[..self.count].contains(&id)) { + return self.fail(if object == 0 { MenuError::Malformed } else { MenuError::BadId }); + } + self.entry(id, flag::OPEN_WITH, 0, [label, "", ""], object, u32::from(kind.code())) + } + + /// Seal the menu: its length in the plane, or the first error. + pub fn finish(&mut self) -> Result { + if self.submenu.is_some() { + self.fail(MenuError::BadNesting); + } + if self.top == 0 { + self.fail(MenuError::Empty); + } + if let Some(error) = self.error { + return Err(error); + } + self.buf[..HEADER_BYTES].copy_from_slice(&[VERSION, self.count as u8, 0, 0, 0, 0, 0, 0]); + Ok(self.at) + } + + fn fail(&mut self, error: MenuError) -> &mut Self { + self.error.get_or_insert(error); + self + } + + fn entry(&mut self, id: u16, flags: u8, depth: u8, texts: [&str; 3], object: u64, kind: u32) -> &mut Self { + if self.error.is_some() { + return self; + } + let limits = [LABEL_BYTES, DETAIL_BYTES, SHORTCUT_BYTES]; + if texts.iter().zip(limits).any(|(t, limit)| t.len() > limit) { + return self.fail(MenuError::TooLarge); + } + if texts.iter().any(|t| t.chars().any(char::is_control)) { + return self.fail(MenuError::BadText); + } + if flags & flag::SEPARATOR == 0 && texts[0].is_empty() { + return self.fail(MenuError::Malformed); + } + if self.count == MAX_ENTRIES { + return self.fail(MenuError::TooManyEntries); + } + if depth == 0 { + if self.top == MAX_TOP_ENTRIES { + return self.fail(MenuError::TooManyEntries); + } + } else if let Some((_, children)) = self.submenu { + if children == MAX_SUBMENU_ENTRIES { + return self.fail(MenuError::TooManyEntries); + } + } + let size = HEAD_BYTES + texts.iter().map(|t| t.len()).sum::(); + let Some(out) = self.buf.get_mut(self.at..self.at + size) else { + return self.fail(MenuError::TooLarge); + }; + let (head, mut tail) = out.split_at_mut(HEAD_BYTES); + head.fill(0); + head[0..2].copy_from_slice(&id.to_le_bytes()); + head[2] = flags; + head[3] = depth; + for (i, t) in texts.iter().enumerate() { + head[4 + i] = t.len() as u8; + } + head[8..16].copy_from_slice(&object.to_le_bytes()); + head[16..20].copy_from_slice(&kind.to_le_bytes()); + for t in texts { + let (dst, rest) = tail.split_at_mut(t.len()); + dst.copy_from_slice(t.as_bytes()); + tail = rest; + } + self.ids[self.count] = id; + self.count += 1; + self.at += size; + if depth == 0 { + self.top += 1; + } else if let Some((head, children)) = self.submenu { + self.submenu = Some((head, children + 1)); + } + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file_menu(plane: &mut [u8]) -> usize { + let mut m = MenuBuilder::new(plane); + m.item(Item::new(1, "Open").shortcut("Enter")); + m.open_with(2, "Open With", 0xfeed, DocumentKind::Code); + m.separator(); + m.submenu("Sort By"); + m.item(Item::new(3, "Name").checked(true)); + m.item(Item::new(4, "Kind")); + m.end_submenu(); + m.item(Item::new(5, "Delete from All Folders…").danger().detail("Every folder it is in")); + m.item(Item::new(6, "Get Info").shortcut("Ctrl+I").enabled(false)); + m.finish().unwrap() + } + + #[test] + fn a_built_menu_parses_back_entry_for_entry() { + let mut plane = [0_u8; MENU_PLANE_BYTES]; + let len = file_menu(&mut plane); + let bytes = &plane[..len]; + let menu = Menu::parse(bytes).unwrap(); + let top: [usize; 6] = core::array::from_fn(|i| menu.top().nth(i).unwrap()); + assert_eq!(menu.top().count(), 6); + assert_eq!(text(bytes, menu.entries()[top[0]].label), "Open"); + assert_eq!(text(bytes, menu.entries()[top[0]].shortcut), "Enter"); + let open_with = menu.entries()[top[1]]; + assert!(open_with.has(flag::OPEN_WITH) && !open_with.choosable()); + assert_eq!((open_with.id, open_with.object, open_with.kind), (2, 0xfeed, 2)); + assert!(menu.entries()[top[2]].has(flag::SEPARATOR)); + let sort = top[3]; + assert_eq!(menu.entries()[sort].children, 2); + let children = menu.children(sort); + assert_eq!(text(bytes, children[0].label), "Name"); + assert!(children[0].has(flag::CHECKED) && children[0].depth == 1); + let delete = menu.find(5).unwrap(); + assert!(delete.has(flag::DANGER)); + assert_eq!(text(bytes, delete.detail), "Every folder it is in"); + assert!(!menu.find(6).unwrap().choosable()); + assert!(menu.find(0).is_none()); + } + + #[test] + fn the_builder_refuses_what_the_parser_would() { + let mut plane = [0_u8; MENU_PLANE_BYTES]; + let mut m = MenuBuilder::new(&mut plane); + assert_eq!(m.finish(), Err(MenuError::Empty)); + let mut m = MenuBuilder::new(&mut plane); + m.item(Item::new(0, "Zero")); + assert_eq!(m.finish(), Err(MenuError::BadId)); + let mut m = MenuBuilder::new(&mut plane); + m.item(Item::new(1, "A")).item(Item::new(1, "B")); + assert_eq!(m.finish(), Err(MenuError::BadId)); + let mut m = MenuBuilder::new(&mut plane); + m.submenu("Empty").end_submenu(); + assert_eq!(m.finish(), Err(MenuError::Empty)); + let mut m = MenuBuilder::new(&mut plane); + m.submenu("A").submenu("B"); + assert_eq!(m.finish(), Err(MenuError::BadNesting)); + let mut m = MenuBuilder::new(&mut plane); + m.item(Item::new(1, "Line\nbreak")); + assert_eq!(m.finish(), Err(MenuError::BadText)); + let long = core::str::from_utf8(&[b'x'; LABEL_BYTES + 1]).unwrap(); + let mut m = MenuBuilder::new(&mut plane); + m.item(Item::new(1, long)); + assert_eq!(m.finish(), Err(MenuError::TooLarge)); + let mut m = MenuBuilder::new(&mut plane); + for id in 1..=MAX_TOP_ENTRIES as u16 + 1 { + m.item(Item::new(id, "Item")); + } + assert_eq!(m.finish(), Err(MenuError::TooManyEntries)); + // The header and one entry's head and text: 8 + 24 + 4 bytes. + let mut exact = [0_u8; 36]; + let mut m = MenuBuilder::new(&mut exact); + m.item(Item::new(1, "Open")); + assert_eq!(m.finish(), Ok(36)); + let mut small = [0_u8; 35]; + let mut m = MenuBuilder::new(&mut small); + m.item(Item::new(1, "Open")); + assert_eq!(m.finish(), Err(MenuError::TooLarge)); + } + + #[test] + fn every_corrupted_byte_is_refused_or_still_a_valid_menu() { + let mut plane = [0_u8; MENU_PLANE_BYTES]; + let len = file_menu(&mut plane); + let good = plane; + assert!(Menu::parse(&good[..len - 1]).is_err(), "a truncated menu"); + let mut longer = good; + longer[len] = 0; + assert_eq!(Menu::parse(&longer[..len + 1]), Err(MenuError::Malformed), "a trailing byte"); + // Any single flipped byte either refuses or yields a menu that still + // keeps every invariant: no panic, spans in bounds, ids unique. + for at in 0..len { + for bit in 0..8 { + let mut bytes = good; + bytes[at] ^= 1 << bit; + if let Ok(menu) = Menu::parse(&bytes[..len]) { + for (i, e) in menu.entries().iter().enumerate() { + for span in [e.label, e.detail, e.shortcut] { + assert!(usize::from(span.at) + usize::from(span.len) <= len); + } + assert!(e.id == 0 || menu.entries()[..i].iter().all(|o| o.id != e.id)); + } + } + } + } + } + + #[test] + fn a_child_must_follow_a_submenu() { + let mut plane = [0_u8; MENU_PLANE_BYTES]; + let len = { + let mut m = MenuBuilder::new(&mut plane); + m.item(Item::new(1, "Open")); + m.finish().unwrap() + }; + // Make the only entry a depth-1 child with no parent. + plane[HEADER_BYTES + 3] = 1; + assert_eq!(Menu::parse(&plane[..len]), Err(MenuError::BadNesting)); + } +} diff --git a/native/crates/window/src/notify.rs b/native/crates/window/src/notify.rs new file mode 100644 index 0000000..496a72b --- /dev/null +++ b/native/crates/window/src/notify.rs @@ -0,0 +1,189 @@ +//! The notification record (`res/SCREENSHOTS.md`, Feedback). +//! +//! Data only and bounded, copied like a [`ShellAction`]: a title, a body and +//! an optional subject the notification is about. The record says nothing of +//! how it is shown. Its first sink is a line on the seat's terminal, which +//! the controller prints; the compositor's popup is a later sink behind the +//! same record. +//! +//! A source posts by handing its controller the encoded record. The capture +//! writer leaves it in its process output, which its controller reads when +//! it reaps the writer. +//! +//! [`ShellAction`]: crate::control::ShellAction + +use crate::document::DocumentKind; + +/// The longest title, in bytes of UTF-8. +pub const TITLE_BYTES: usize = 32; +/// The longest body, in bytes of UTF-8. +pub const BODY_BYTES: usize = 192; + +const MAGIC: [u8; 4] = *b"KNT1"; +/// Magic, title and body lengths, flags, subject kind, subject object. +const HEADER_BYTES: usize = 4 + 1 + 1 + 1 + 1 + 8; +/// The longest encoded record. +pub const RECORD_BYTES: usize = HEADER_BYTES + TITLE_BYTES + BODY_BYTES; +const FLAG_SUBJECT: u8 = 1; + +/// What a notification is about: a filesystem object a click would open. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Subject { + /// Its stable identity (never zero), as `ShellAction::Open` carries it. + pub object: u64, + pub kind: DocumentKind, +} + +/// One notification. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Notification { + title: [u8; TITLE_BYTES], + title_len: u8, + body: [u8; BODY_BYTES], + body_len: u8, + subject: Option, +} + +impl Notification { + /// A notification, or `None` if `title` or `body` is too long or holds + /// a control character, or the subject names no object. The text is + /// display text only: a sink shows it, and nothing in it may steer the + /// sink (a terminal line must not carry an escape sequence). + pub fn new(title: &str, body: &str, subject: Option) -> Option { + if title.len() > TITLE_BYTES + || body.len() > BODY_BYTES + || title.chars().chain(body.chars()).any(char::is_control) + || subject.is_some_and(|subject| subject.object == 0) + { + return None; + } + let mut record = Self { + title: [0; TITLE_BYTES], + title_len: title.len() as u8, + body: [0; BODY_BYTES], + body_len: body.len() as u8, + subject, + }; + record.title[..title.len()].copy_from_slice(title.as_bytes()); + record.body[..body.len()].copy_from_slice(body.as_bytes()); + Some(record) + } + + pub fn title(&self) -> &str { + // Validated UTF-8 at construction and at decode. + core::str::from_utf8(&self.title[..usize::from(self.title_len)]).unwrap_or("") + } + + pub fn body(&self) -> &str { + core::str::from_utf8(&self.body[..usize::from(self.body_len)]).unwrap_or("") + } + + pub const fn subject(&self) -> Option { + self.subject + } + + /// Encode into `out`, answering the bytes written; `None` if `out` is + /// shorter than the record. + pub fn encode(&self, out: &mut [u8]) -> Option { + let (title, body) = (usize::from(self.title_len), usize::from(self.body_len)); + let len = HEADER_BYTES + title + body; + let out = out.get_mut(..len)?; + out[..4].copy_from_slice(&MAGIC); + out[4] = self.title_len; + out[5] = self.body_len; + out[6] = if self.subject.is_some() { FLAG_SUBJECT } else { 0 }; + let (kind, object) = self + .subject + .map_or((0, 0), |subject| (subject.kind.code(), subject.object)); + out[7] = kind; + out[8..16].copy_from_slice(&object.to_le_bytes()); + out[HEADER_BYTES..HEADER_BYTES + title].copy_from_slice(&self.title[..title]); + out[HEADER_BYTES + title..].copy_from_slice(&self.body[..body]); + Some(len) + } + + /// The record `bytes` holds exactly, or `None`: a wrong magic or length, + /// text that is not UTF-8 or holds a control character, a subject flag + /// without an object, or any stray byte. + pub fn decode(bytes: &[u8]) -> Option { + if bytes.len() < HEADER_BYTES || bytes[..4] != MAGIC { + return None; + } + let (title, body) = (usize::from(bytes[4]), usize::from(bytes[5])); + if bytes.len() != HEADER_BYTES + title + body { + return None; + } + let object = u64::from_le_bytes(bytes[8..16].try_into().ok()?); + let subject = match bytes[6] { + 0 if bytes[7] == 0 && object == 0 => None, + FLAG_SUBJECT => Some(Subject { + object, + kind: DocumentKind::from_code(u64::from(bytes[7]))?, + }), + _ => return None, + }; + Self::new( + core::str::from_utf8(&bytes[HEADER_BYTES..HEADER_BYTES + title]).ok()?, + core::str::from_utf8(&bytes[HEADER_BYTES + title..]).ok()?, + subject, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_record_round_trips_exactly() { + let subject = Subject { + object: 0xfeed, + kind: DocumentKind::Image, + }; + for record in [ + Notification::new("Screenshot", "pictures/screenshots/a.png (1280×800, 1.2 MB)", Some(subject)), + Notification::new("", "", None), + Notification::new(&"t".repeat(TITLE_BYTES), &"b".repeat(BODY_BYTES), None), + ] { + let record = record.unwrap(); + let mut out = [0u8; RECORD_BYTES]; + let len = record.encode(&mut out).unwrap(); + assert_eq!(Notification::decode(&out[..len]), Some(record)); + assert_eq!(record.encode(&mut out[..len - 1]), None, "a short buffer"); + } + } + + #[test] + fn a_bad_record_is_refused() { + assert_eq!(Notification::new(&"t".repeat(TITLE_BYTES + 1), "", None), None); + assert_eq!(Notification::new("", &"b".repeat(BODY_BYTES + 1), None), None); + let zero = Subject { + object: 0, + kind: DocumentKind::Image, + }; + assert_eq!(Notification::new("", "", Some(zero)), None); + for control in ["\x1b[2J", "a\nb", "\u{7f}", "\u{9b}31m"] { + assert_eq!(Notification::new(control, "", None), None, "{control:?} as a title"); + assert_eq!(Notification::new("", control, None), None, "{control:?} as a body"); + } + + let record = Notification::new("Screenshot", "saved", None).unwrap(); + let mut out = [0u8; RECORD_BYTES]; + let len = record.encode(&mut out).unwrap(); + let good = &out[..len]; + let mutate = |at: usize, byte: u8| { + let mut bad = [0u8; RECORD_BYTES]; + bad[..len].copy_from_slice(good); + bad[at] = byte; + Notification::decode(&bad[..len]) + }; + assert_eq!(mutate(0, b'X'), None, "magic"); + assert_eq!(mutate(4, 11), None, "title length"); + assert_eq!(mutate(6, 2), None, "unknown flag"); + assert_eq!(mutate(7, 3), None, "a kind without a subject"); + assert_eq!(mutate(9, 1), None, "an object without a subject"); + assert_eq!(mutate(HEADER_BYTES, 0xff), None, "not UTF-8"); + assert_eq!(mutate(HEADER_BYTES, 0x1b), None, "an escape"); + assert_eq!(Notification::decode(&good[..len - 1]), None, "truncated"); + } +} diff --git a/native/crates/window/src/pick.rs b/native/crates/window/src/pick.rs new file mode 100644 index 0000000..d4c4171 --- /dev/null +++ b/native/crates/window/src/pick.rs @@ -0,0 +1,153 @@ +//! A file the user chooses for a window (`res/FILE_PICKER.md`): the bounds +//! of a pick's answer, and its name carried back in event records. +//! +//! A window asks with `SurfaceRequest::PickObject` and the file types it +//! accepts. Its controller launches the picker, which claims the pick by +//! its token and answers with the chosen object's stable identity and its +//! name. The window hears the answer as [`NAME_PARTS`] `Event::PickedName` +//! records, then the `Event::Picked` that ends it. The identity is data the +//! window opens from its own root, never authority, and the name is only +//! what to call it. + +/// The longest picked name: a filesystem name's width (a DAGFS name's 32 +/// bytes, `kore_abi::svc::FS_NAME_BYTES`). +pub const PICK_NAME_BYTES: usize = 32; + +/// The bytes one `Event::PickedName` record carries: its two event words. +pub const NAME_PART_BYTES: usize = 16; + +/// The records a name takes. +pub const NAME_PARTS: usize = PICK_NAME_BYTES / NAME_PART_BYTES; + +const _: () = assert!(NAME_PARTS * NAME_PART_BYTES == PICK_NAME_BYTES); + +/// Whether `name` can answer a pick: one filesystem name a window can show +/// — UTF-8, 1 to [`PICK_NAME_BYTES`] long, without a `/`, a NUL or any +/// control character. A NUL would end it early in its zero-padded records. +pub fn valid_name(name: &[u8]) -> bool { + core::str::from_utf8(name).is_ok_and(|name| { + !name.is_empty() + && name.len() <= PICK_NAME_BYTES + && !name.contains('/') + && !name.chars().any(char::is_control) + }) +} + +/// Part `part` of `name`, zero-padded: what that `Event::PickedName` +/// record carries. +pub fn name_part(name: &[u8], part: usize) -> [u8; NAME_PART_BYTES] { + let mut bytes = [0; NAME_PART_BYTES]; + let start = (part * NAME_PART_BYTES).min(name.len()); + let end = (start + NAME_PART_BYTES).min(name.len()); + bytes[..end - start].copy_from_slice(&name[start..end]); + bytes +} + +/// A picked file's name. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PickName { + bytes: [u8; PICK_NAME_BYTES], + len: u8, +} + +impl PickName { + pub fn as_bytes(&self) -> &[u8] { + &self.bytes[..usize::from(self.len)] + } + + /// The name; empty if its bytes did not form one (a compositor that + /// broke the protocol). + pub fn as_str(&self) -> &str { + core::str::from_utf8(self.as_bytes()).unwrap_or("") + } +} + +/// A picked name, reassembled from its `Event::PickedName` records as they +/// arrive: each record lands in its part, and the `Event::Picked` that +/// follows takes the whole. A record for another pick starts over, so a +/// pick that ended unheard leaves nothing behind. +#[derive(Clone, Copy, Debug, Default)] +pub struct NameParts { + serial: u32, + bytes: [u8; PICK_NAME_BYTES], + /// One bit for each part heard. + heard: u8, +} + +impl NameParts { + /// One `Event::PickedName` record. A part past the last is ignored. + pub fn part(&mut self, serial: u32, part: u8, bytes: [u8; NAME_PART_BYTES]) { + let part = usize::from(part); + if part >= NAME_PARTS { + return; + } + if serial != self.serial { + *self = Self { serial, ..Self::default() }; + } + self.bytes[part * NAME_PART_BYTES..(part + 1) * NAME_PART_BYTES].copy_from_slice(&bytes); + self.heard |= 1 << part; + } + + /// The name for the `Event::Picked` naming `serial`, once every part of + /// that pick arrived, and a valid name; then nothing is held. + pub fn take(&mut self, serial: u32) -> Option { + let whole = serial == self.serial && self.heard == (1 << NAME_PARTS) - 1; + let bytes = self.bytes; + *self = Self::default(); + let len = bytes.iter().position(|&byte| byte == 0).unwrap_or(PICK_NAME_BYTES); + (whole && valid_name(&bytes[..len])).then_some(PickName { bytes, len: len as u8 }) + } + + /// Drop what is held: the event queue overflowed (`Event::Reset`). + pub fn clear(&mut self) { + *self = Self::default(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_name_splits_into_its_records_and_reassembles_whole() { + let name = b"iss-night-aurora-over-the-sea.jpg"; + assert!(name.len() > PICK_NAME_BYTES, "longer than a pick carries"); + assert!(!valid_name(name)); + let name = &name[..PICK_NAME_BYTES]; + assert!(valid_name(name)); + let mut parts = NameParts::default(); + for part in 0..NAME_PARTS { + parts.part(7, part as u8, name_part(name, part)); + } + assert_eq!(parts.take(7).unwrap().as_bytes(), name); + assert_eq!(parts.take(7), None, "taken once"); + + // A short name pads its last record and its second with zeros. + let mut parts = NameParts::default(); + parts.part(8, 0, name_part(b"beach.png", 0)); + assert_eq!(parts.take(8), None, "a part is missing"); + for part in 0..NAME_PARTS { + parts.part(8, part as u8, name_part(b"beach.png", part)); + } + assert_eq!(parts.take(8).unwrap().as_str(), "beach.png"); + } + + #[test] + fn records_of_another_pick_start_over_and_bad_names_are_none() { + let mut parts = NameParts::default(); + parts.part(1, 0, name_part(b"old.png", 0)); + parts.part(2, 1, name_part(b"new.png", 1)); + parts.part(2, 0, name_part(b"new.png", 0)); + assert_eq!(parts.take(1), None, "another pick's name"); + for (serial, name) in [(3, &b"a/b.png"[..]), (4, b"tab\t.png"), (5, b"\xff.png")] { + for part in 0..NAME_PARTS { + parts.part(serial, part as u8, name_part(name, part)); + } + assert_eq!(parts.take(serial), None, "{name:?}"); + } + parts.part(6, 0, name_part(b"x.png", 0)); + parts.clear(); + parts.part(6, 1, name_part(b"x.png", 1)); + assert_eq!(parts.take(6), None, "a reset dropped part 0"); + } +} diff --git a/native/crates/window/src/relay.rs b/native/crates/window/src/relay.rs index c4ff123..6ecf091 100644 --- a/native/crates/window/src/relay.rs +++ b/native/crates/window/src/relay.rs @@ -4,6 +4,12 @@ //! one atomic route mailbox and a bounded raw-snapshot ring. The founder alone //! initializes them; input-stream owns echo/raw production and the compositor //! owns focus publication and consumption. IDs and epochs convey no authority. +//! +//! The mailbox also carries the seat's **capture intent** (#1414): the +//! PrintScreen chord is a trusted key that input-stream takes out of every +//! route and turns into a request here, so no application can see, swallow +//! or forge it. Like the route, it is a level word, not a queued event: a +//! dropped advisory kick cannot lose it. use core::sync::atomic::{AtomicU64, Ordering}; use kore_abi::svc::{InputReport, OutChunk}; @@ -18,6 +24,8 @@ pub const RELAY_BYTES: usize = spsc::region_bytes::(RELAY_CAP); pub const WINDOW_BYTES: usize = ECHO_BYTES + FOCUS_BYTES + RELAY_BYTES; const MAGIC: u64 = u64::from_le_bytes(*b"koreFCS1"); const MAX_EPOCH: u64 = u64::MAX >> 1; +/// The capture word is `seq << 8 | mode`: a 56-bit press sequence. +const MAX_CAPTURE_SEQ: u64 = u64::MAX >> 8; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FocusTarget { @@ -51,6 +59,16 @@ impl Focus { } } +pub use crate::capture::CaptureMode; + +/// One capture intent. `seq` counts this seat's requests from 1 and is never +/// reused, so a consumer can tell a new press from one it already took. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CaptureRequest { + pub seq: u64, + pub mode: CaptureMode, +} + #[repr(C, align(64))] struct FocusCell { magic: u64, @@ -59,7 +77,14 @@ struct FocusCell { /// epoch's Reset and acknowledges it. Newer snapshots cannot overtake it. reset_epoch: AtomicU64, dropped: AtomicU64, - reserved: [u8; 32], + /// The newest capture request, `seq << 8 | mode` (`0` = none yet). + /// Only input-stream writes it. + capture: AtomicU64, + /// The newest request sequence the compositor took. Only the + /// compositor writes it. Claimed from the reserved tail, which a + /// founder zeroes, so an older founder's cell reads as no request. + capture_taken: AtomicU64, + reserved: [u8; 16], } pub struct FocusReader<'a> { @@ -146,10 +171,26 @@ impl RelayProducer<'_> { } /// Level state, not a one-shot flag: a failed/dropped advisory kick cannot - /// clear queued data or an unacknowledged Reset. + /// clear queued data, an unacknowledged Reset or an untaken capture. pub fn needs_wake(&self) -> bool { self.cell.reset_epoch.load(Ordering::Acquire) != 0 || self.ring.capacity_left() != self.ring.capacity() + || capture_untaken(self.cell) + } + + /// Publish a capture request for this seat. The newest request replaces + /// one the compositor has not taken yet: presses closer together than a + /// compositor pass would capture the same frame. Exhaustion refuses; a + /// sequence is never reused. + pub fn request_capture(&mut self, mode: CaptureMode) -> Option { + // This side is the only writer of the word. + let seq = (self.cell.capture.load(Ordering::Relaxed) >> 8) + .checked_add(1) + .filter(|&seq| seq <= MAX_CAPTURE_SEQ)?; + self.cell + .capture + .store((seq << 8) | mode as u64, Ordering::Release); + Some(CaptureRequest { seq, mode }) } pub fn dropped(&self) -> u64 { self.cell.dropped.load(Ordering::Relaxed) @@ -195,17 +236,41 @@ impl RelayConsumer<'_> { None } pub fn pending(&self) -> bool { - self.cell.reset_epoch.load(Ordering::Acquire) != 0 || self.ring.outstanding() != 0 + self.cell.reset_epoch.load(Ordering::Acquire) != 0 + || self.ring.outstanding() != 0 + || capture_untaken(self.cell) } pub fn dropped(&self) -> u64 { self.cell.dropped.load(Ordering::Relaxed) } + + /// Take the seat's newest capture request, once. Taking acknowledges it, + /// which settles the producer's wake. A malformed mode is acknowledged + /// too and captures nothing, so it cannot hold that wake high. + pub fn take_capture(&mut self) -> Option { + let word = self.cell.capture.load(Ordering::Acquire); + let seq = word >> 8; + // This side is the only writer of `capture_taken`. + if seq == self.cell.capture_taken.load(Ordering::Relaxed) { + return None; + } + self.cell.capture_taken.store(seq, Ordering::Release); + Some(CaptureRequest { + seq, + mode: CaptureMode::from_code(word as u8)?, + }) + } +} + +/// A request the compositor has not taken: level state for both peers. +fn capture_untaken(cell: &FocusCell) -> bool { + cell.capture.load(Ordering::Acquire) >> 8 != cell.capture_taken.load(Ordering::Acquire) } +/// A relayable keyboard snapshot: a keyboard record whose reserved bytes +/// carry nothing but a valid lock state, and which is not a rollover report. pub fn keyboard(report: InputReport) -> bool { - report.kind == InputReport::KIND_KEYBOARD - && report.reserved == [0; 7] - && !report.data.contains(&1) + report.keyboard_locks().is_some() && !report.data.contains(&1) } /// The returned byte regions are disjoint and exact; neither ring's mutable @@ -247,7 +312,9 @@ pub fn init(window: &mut [u8]) -> Result<(), RingError> { route: AtomicU64::new(Focus::INITIAL.encode().unwrap()), reset_epoch: AtomicU64::new(0), dropped: AtomicU64::new(0), - reserved: [0; 32], + capture: AtomicU64::new(0), + capture_taken: AtomicU64::new(0), + reserved: [0; 16], }); } Ok(()) @@ -258,7 +325,7 @@ fn focus_cell(window: &mut [u8]) -> Result<&FocusCell, RingError> { // immutable after founder init; all shared mutation uses atomic fields. let cell = unsafe { &*window.as_ptr().cast::() }; if cell.magic != MAGIC - || cell.reserved != [0; 32] + || cell.reserved != [0; 16] || Focus::decode(cell.route.load(Ordering::Acquire)).is_none() { return Err(RingError::BadHeader); @@ -468,9 +535,13 @@ mod tests { let mut malformed = report; malformed.kind = 0; assert!(!input.reports.push(focus, malformed)); - malformed = report; - malformed.reserved[0] = 1; - assert!(!input.reports.push(focus, malformed)); + // The lock byte is the only meaningful reserved byte, and only its + // three lock bits: anything else is not a keyboard snapshot. + for (index, value) in [(0, 0x08), (1, 1), (6, 1)] { + malformed = report; + malformed.reserved[index] = value; + assert!(!input.reports.push(focus, malformed)); + } malformed = report; malformed.data[0] = 1; assert!(!input.reports.push(focus, malformed)); @@ -482,6 +553,23 @@ mod tests { assert_eq!(compositor.echo.pop(), None); } + /// The seat's lock state rides the snapshot to the window unchanged — a + /// client resolves Caps Lock exactly as the terminal discipline does. + #[test] + fn a_locked_snapshot_reaches_the_window_with_its_locks() { + use kore_abi::keyboard::Locks; + let mut page = Page([0; 4096]); + let (mut input, mut compositor) = split(&mut page.0).unwrap(); + let focus = compositor.focus.advance(FocusTarget::Window).unwrap(); + let report = InputReport::keyboard(1, 0, [4, 0, 0, 0, 0, 0]) + .with_locks(Locks::CAPS.toggled(Locks::SCROLL)); + assert!(input.reports.push(focus, report)); + let Some(RelayEvent::Snapshot { report: got, .. }) = compositor.reports.poll() else { + panic!("the locked snapshot must be relayed"); + }; + assert_eq!(got.keyboard_locks(), Some(Locks::CAPS.toggled(Locks::SCROLL))); + } + #[test] fn invalid_wire_epochs_do_not_mask_valid_queued_input() { let mut page = Page([0; 4096]); @@ -507,7 +595,7 @@ mod tests { }) ); let mut malformed = report; - malformed.reserved[0] = 1; + malformed.reserved[1] = 1; input .reports .ring @@ -580,6 +668,119 @@ mod tests { assert_eq!(input.reports.dropped(), 2); } + #[test] + fn a_capture_request_is_taken_once_and_settles_the_wake() { + let mut page = Page([0; 4096]); + let (mut input, mut compositor) = split(&mut page.0).unwrap(); + // A founder's zeroed words read as no request. + assert_eq!(compositor.reports.take_capture(), None); + assert!(!input.reports.needs_wake()); + assert!(!compositor.reports.pending()); + + let request = input.reports.request_capture(CaptureMode::Screen).unwrap(); + assert_eq!( + request, + CaptureRequest { + seq: 1, + mode: CaptureMode::Screen + } + ); + assert!( + input.reports.needs_wake(), + "an untaken capture holds the wake" + ); + assert!(compositor.reports.pending()); + // The request is not a snapshot: the ring stays empty. + assert_eq!(compositor.reports.poll(), None); + assert_eq!(compositor.reports.take_capture(), Some(request)); + assert_eq!(compositor.reports.take_capture(), None, "taken once"); + assert!(!input.reports.needs_wake()); + assert!(!compositor.reports.pending()); + + let next = input.reports.request_capture(CaptureMode::Region).unwrap(); + assert_eq!(next.seq, 2, "a sequence is never reused"); + assert_eq!(compositor.reports.take_capture(), Some(next)); + } + + #[test] + fn a_newer_capture_replaces_an_untaken_one() { + let mut page = Page([0; 4096]); + let (mut input, mut compositor) = split(&mut page.0).unwrap(); + input.reports.request_capture(CaptureMode::Screen).unwrap(); + let newest = input.reports.request_capture(CaptureMode::Window).unwrap(); + assert_eq!(compositor.reports.take_capture(), Some(newest)); + assert_eq!(compositor.reports.take_capture(), None); + assert!(!input.reports.needs_wake()); + } + + /// A capture rides beside the snapshot ring, not in it: a pending Reset + /// neither blocks nor drops it, and draining the Reset does not take it. + #[test] + fn a_capture_is_independent_of_snapshot_loss() { + let mut page = Page([0; 4096]); + let (mut input, mut compositor) = split(&mut page.0).unwrap(); + let focus = compositor.focus.advance(FocusTarget::Window).unwrap(); + let held = InputReport::keyboard(1, 0, [4, 0, 0, 0, 0, 0]); + for _ in 0..=RELAY_CAP { + let _ = input.reports.push(focus, held); + } + let request = input.reports.request_capture(CaptureMode::Window).unwrap(); + assert_eq!( + compositor.reports.poll(), + Some(RelayEvent::Reset { epoch: focus.epoch }) + ); + assert_eq!(compositor.reports.poll(), None); + assert!(input.reports.needs_wake(), "the capture is still owed"); + assert_eq!(compositor.reports.take_capture(), Some(request)); + assert!(!input.reports.needs_wake()); + } + + #[test] + fn capture_exhaustion_refuses_and_a_malformed_mode_settles() { + let mut page = Page([0; 4096]); + let (mut input, mut compositor) = split(&mut page.0).unwrap(); + let last = (MAX_CAPTURE_SEQ << 8) | CaptureMode::Screen as u64; + input.reports.cell.capture.store(last, Ordering::Release); + assert_eq!(input.reports.request_capture(CaptureMode::Window), None); + assert_eq!(input.reports.cell.capture.load(Ordering::Acquire), last); + assert_eq!( + compositor.reports.take_capture(), + Some(CaptureRequest { + seq: MAX_CAPTURE_SEQ, + mode: CaptureMode::Screen + }) + ); + + // A mode no chord produces captures nothing, and is still taken. + for mode in [0u8, 4, 0xff] { + let seq = 7 + u64::from(mode); + input + .reports + .cell + .capture + .store((seq << 8) | u64::from(mode), Ordering::Release); + assert!(input.reports.needs_wake()); + assert_eq!(compositor.reports.take_capture(), None); + assert!(!input.reports.needs_wake(), "mode {mode:#x} settled"); + } + } + + /// The reserved tail still fails closed after the capture claim. + #[test] + fn a_dirty_reserved_tail_refuses_attachment() { + let mut page = Page([0; 4096]); + init(&mut page.0).unwrap(); + let tail = ECHO_BYTES + FOCUS_BYTES - 1; + page.0[tail] = 1; + assert!(matches!(input_side(&mut page.0), Err(RingError::BadHeader))); + assert!(matches!( + compositor_side(&mut page.0), + Err(RingError::BadHeader) + )); + page.0[tail] = 0; + assert!(input_side(&mut page.0).is_ok()); + } + #[test] fn reset_acknowledgment_allows_next_snapshot_across_threads() { // Named stack backing and scoped ownership exercise the same SPSC diff --git a/native/crates/window/src/wire.rs b/native/crates/window/src/wire.rs index 37067fa..3581b10 100644 --- a/native/crates/window/src/wire.rs +++ b/native/crates/window/src/wire.rs @@ -14,6 +14,12 @@ use kore_abi::session::DISPLAY_FORMAT_XRGB8888; use kore_abi::svc::{CommitWire, MAX_COMMIT_RECTS, SurRectWire, SurRepWire, SurReqWire}; + +use crate::capture::{CaptureFrame, CaptureMode}; +use crate::cursor::Cursor; +use crate::document::DocumentKind; +use crate::menu::MENU_PLANE_BYTES; +use crate::pick::PICK_NAME_BYTES; use zerocopy::little_endian::{U16, U32, U64}; /// Grant channels a surface client pre-pins — exactly one: it carries @@ -36,20 +42,39 @@ pub fn split_surface_aux(aux: &mut [u8]) -> Option<(&mut [u8], &mut [u8])> { Some((commits, &mut tail[..output])) } +/// The planes of a window pair's auxiliary tail, in lease order. +pub struct WindowAux<'a> { + pub commits: &'a mut [u8], + pub output: &'a mut [u8], + /// The event channel (`crate::channel`, `res/EVENT.md`): the + /// compositor's level state and event records, with one conditional + /// wake. + pub events: &'a mut [u8], + /// The window's context menu as it asks for it ([`crate::menu`]): the + /// app writes, the compositor snapshots on `ShowMenu`. + pub menu: &'a mut [u8], +} + /// Window clients share the same four capability auxiliaries as surface -/// clients. The event channel (`crate::channel`, `res/EVENT.md`) occupies -/// the pair's remaining bytes, not a fifth endpoint: the compositor's -/// level state and event records, with one conditional wake. -pub fn split_window_aux(aux: &mut [u8]) -> Option<(&mut [u8], &mut [u8], &mut [u8])> { +/// clients. The event channel and the menu plane occupy the pair's +/// remaining bytes, not a fifth endpoint. +pub fn split_window_aux(aux: &mut [u8]) -> Option> { let commits = kore_ring::spsc::region_bytes::(crate::COMMIT_CAP); let output = kore_ring::spsc::region_bytes::(TERMINAL_OUTPUT_CAP); let channel = crate::channel::region_bytes(); - if aux.len() < commits.checked_add(output)?.checked_add(channel)? { + let total = commits.checked_add(output)?.checked_add(channel)?.checked_add(MENU_PLANE_BYTES)?; + if aux.len() < total { return None; } let (commits, tail) = aux.split_at_mut(commits); let (output, tail) = tail.split_at_mut(output); - Some((commits, output, &mut tail[..channel])) + let (events, tail) = tail.split_at_mut(channel); + Some(WindowAux { + commits, + output, + events, + menu: &mut tail[..MENU_PLANE_BYTES], + }) } /// The backend-free wire marker — what a *client* names to speak the @@ -80,10 +105,7 @@ impl kore_rt::svc::SvcProtocol for SurfaceWire { } fn request_grants(request: &Self::ReqWire) -> usize { - usize::from(matches!( - decode_req(request), - Some(SurfaceRequest::CreateSurface { .. }) - )) + usize::from(decode_req(request).is_some_and(|request| request_carries_grant(&request))) } fn init_pair_window( @@ -96,11 +118,11 @@ impl kore_rt::svc::SvcProtocol for SurfaceWire { let Some(plane) = window.get_mut(at..) else { return Err(kore_ring::spsc::RingError::TooSmall); }; - let (commits, output, events) = - split_window_aux(plane).ok_or(kore_ring::spsc::RingError::TooSmall)?; - kore_ring::spsc::init::(commits, crate::COMMIT_CAP)?; - kore_ring::spsc::init::(output, TERMINAL_OUTPUT_CAP)?; - crate::channel::init(events) + let aux = split_window_aux(plane).ok_or(kore_ring::spsc::RingError::TooSmall)?; + kore_ring::spsc::init::(aux.commits, crate::COMMIT_CAP)?; + kore_ring::spsc::init::(aux.output, TERMINAL_OUTPUT_CAP)?; + aux.menu.fill(0); + crate::channel::init(aux.events) } fn encode_req(req: &Self::Request) -> Self::ReqWire { @@ -156,6 +178,38 @@ pub enum SurfaceRequest { /// controller, the compositor and the app tearing one window down in /// three orders at once. RequestClose, + /// The cursor this window shows while the pointer is over its content + /// ([`crate::cursor`]). It holds until the window asks again; the + /// compositor's own pixels choose for themselves. + SetCursor { cursor: Cursor }, + /// Ask this window's controller to open the filesystem object with + /// stable identity `object` (never zero) in the default handler for + /// its `kind` ([`DocumentKind::Unknown`]: the document app). Inert: the + /// controller decides, and the identity is data the opened window + /// locates from its own root. + OpenObject { object: u64, kind: DocumentKind }, + /// Draw the context menu this window encoded in its menu plane + /// (`len` bytes, [`crate::menu`]) at the content-relative anchor + /// `(x, y)`. `serial` (nonzero) names it in the + /// [`Event::Menu`](crate::event::Event::Menu) that ends it. + ShowMenu { x: i32, y: i32, len: u16, serial: u32 }, + /// A capture writer's one verb (`res/SCREENSHOTS.md`): lend the frozen + /// frame this pair was admitted for. The reply's one grant reads it + /// and cannot write it. Any other peer is refused. + TakeCapture, + /// Ask the user, through this window's controller, to choose a file of + /// the types `types` accepts: a nonzero mask over `kore_kit::kinds` + /// rows. `serial` (nonzero) names the pick in the + /// [`Event::Picked`](crate::event::Event::Picked) that ends it + /// ([`crate::pick`]). Inert: the controller launches the picker. + PickObject { types: u64, serial: u32 }, + /// Bind this window, launched for a pick, to that pick by the token it + /// was launched with. + ClaimPick { token: u64 }, + /// Answer the pick this window claimed: `object` is the chosen file's + /// stable identity, or zero if the user cancelled; its name is the + /// first `len` bytes of this window's menu plane (none when cancelled). + Picked { token: u64, object: u64, len: u8 }, } /// The surface-class reply vocabulary. @@ -194,6 +248,21 @@ pub enum SurfaceResponse { /// raised under (never zero). Idempotent — a window already asked to /// close answers this again rather than refusing. CloseRequested { serial: u64 }, + /// The window's cursor is now `cursor`, echoed. + CursorSet { cursor: Cursor }, + /// The open is queued for this window's controller, echoed. + OpenQueued { object: u64 }, + /// The menu is open; its plane may be reused. The serial, echoed. + MenuShown { serial: u32 }, + /// The frozen frame; its read-only view rides the reply's one grant. + Capture(CaptureFrame), + /// The pick is queued for this window's controller. The serial, echoed. + PickQueued { serial: u32 }, + /// This window is the pick's picker now. The token, echoed. + PickClaimed { token: u64 }, + /// The answer is delivered and the pick has ended; the plane may be + /// reused. The token, echoed. + PickAnswered { token: u64 }, /// The refusal, echoing the subject the verb named (zero if none). Refused { refusal: SurfaceRefusal, @@ -356,6 +425,38 @@ pub fn encode_req(req: &SurfaceRequest) -> SurReqWire { SurfaceRequest::RequestClose => { w.op = SurReqWire::OP_REQUEST_CLOSE; } + SurfaceRequest::SetCursor { cursor } => { + w.op = SurReqWire::OP_SET_CURSOR; + w.a = U64::new(cursor.code()); + } + SurfaceRequest::OpenObject { object, kind } => { + w.op = SurReqWire::OP_OPEN_OBJECT; + w.a = U64::new(object); + w.b = U64::new(u64::from(kind.code())); + } + SurfaceRequest::ShowMenu { x, y, len, serial } => { + w.op = SurReqWire::OP_SHOW_MENU; + w.a = U64::new(u64::from(x as u32) | u64::from(y as u32) << 32); + w.b = U64::new(u64::from(len) | u64::from(serial) << 32); + } + SurfaceRequest::TakeCapture => { + w.op = SurReqWire::OP_TAKE_CAPTURE; + } + SurfaceRequest::PickObject { types, serial } => { + w.op = SurReqWire::OP_PICK_OBJECT; + w.a = U64::new(types); + w.b = U64::new(u64::from(serial)); + } + SurfaceRequest::ClaimPick { token } => { + w.op = SurReqWire::OP_CLAIM_PICK; + w.a = U64::new(token); + } + SurfaceRequest::Picked { token, object, len } => { + w.op = SurReqWire::OP_PICKED; + w.a = U64::new(token); + w.b = U64::new(object); + w.c = U64::new(u64::from(len)); + } } w } @@ -381,6 +482,56 @@ fn raw_decode_req(w: &SurReqWire) -> Option { surface: w.subject.get(), }, SurReqWire::OP_REQUEST_CLOSE => SurfaceRequest::RequestClose, + SurReqWire::OP_SET_CURSOR => SurfaceRequest::SetCursor { + cursor: Cursor::from_code(w.a.get())?, + }, + SurReqWire::OP_OPEN_OBJECT => { + let object = w.a.get(); + if object == 0 { + return None; + } + SurfaceRequest::OpenObject { + object, + kind: DocumentKind::from_code(w.b.get())?, + } + } + SurReqWire::OP_SHOW_MENU => { + let (a, b) = (w.a.get(), w.b.get()); + let (len, serial) = (b as u16, (b >> 32) as u32); + if len == 0 || usize::from(len) > MENU_PLANE_BYTES || serial == 0 { + return None; + } + SurfaceRequest::ShowMenu { + x: a as u32 as i32, + y: (a >> 32) as u32 as i32, + len, + serial, + } + } + SurReqWire::OP_TAKE_CAPTURE => SurfaceRequest::TakeCapture, + SurReqWire::OP_PICK_OBJECT => { + let (types, serial) = (w.a.get(), u32::try_from(w.b.get()).ok()?); + if types == 0 || serial == 0 { + return None; + } + SurfaceRequest::PickObject { types, serial } + } + SurReqWire::OP_CLAIM_PICK => { + let token = w.a.get(); + if token == 0 { + return None; + } + SurfaceRequest::ClaimPick { token } + } + SurReqWire::OP_PICKED => { + let (token, object) = (w.a.get(), w.b.get()); + let len = u8::try_from(w.c.get()).ok()?; + // A choice carries its name and a cancel none. + if token == 0 || usize::from(len) > PICK_NAME_BYTES || (object == 0) != (len == 0) { + return None; + } + SurfaceRequest::Picked { token, object, len } + } _ => return None, }) } @@ -394,12 +545,42 @@ pub fn decode_req(w: &SurReqWire) -> Option { /// The grant-count law: which tags carry authority beside their bytes. pub const fn reply_grants(rep: &SurfaceResponse) -> u8 { match rep { - // The backing's detached window view. - SurfaceResponse::SurfaceMade { .. } => 1, + // The backing's detached window view; the frozen frame's + // read-only view. + SurfaceResponse::SurfaceMade { .. } | SurfaceResponse::Capture(_) => 1, _ => 0, } } +/// The verbs whose answer carries a grant: what a client reserves a landing +/// for before it asks. +pub const fn request_carries_grant(request: &SurfaceRequest) -> bool { + matches!( + request, + SurfaceRequest::CreateSurface { .. } | SurfaceRequest::TakeCapture + ) +} + +/// The capture word `Capture.a` carries: `width ‖ height << 16 ‖ mode << 32 +/// ‖ screen << 40 ‖ density << 48`. +const fn capture_word(frame: &CaptureFrame) -> u64 { + frame.width as u64 + | (frame.height as u64) << 16 + | (frame.mode.code() as u64) << 32 + | (frame.screen as u64) << 40 + | (frame.density.quarters() as u64) << 48 +} + +/// `Capture.d`'s bit 32: the window gave no frame in time, and its backing +/// was copied as it stood ([`CaptureFrame::synced`] is `false`). +const CAPTURE_UNSYNCED: u64 = 1 << 32; + +/// The placement word `Capture.d` carries: `x ‖ y << 16`, and +/// [`CAPTURE_UNSYNCED`]. +const fn placement_word(frame: &CaptureFrame) -> u64 { + frame.x as u64 | (frame.y as u64) << 16 | if frame.synced { 0 } else { CAPTURE_UNSYNCED } +} + /// Lower a response to its wire form (total). `grants` is the count /// actually riding the channel — the law says what it must be /// ([`reply_grants`]), the serving layer says what it was. @@ -461,6 +642,38 @@ pub fn encode_rep(rep: &SurfaceResponse, grants: u8) -> SurRepWire { w.tag = SurRepWire::TAG_CLOSE_REQUESTED; w.a = U64::new(serial); } + SurfaceResponse::CursorSet { cursor } => { + w.tag = SurRepWire::TAG_CURSOR_SET; + w.a = U64::new(cursor.code()); + } + SurfaceResponse::OpenQueued { object } => { + w.tag = SurRepWire::TAG_OPEN_QUEUED; + w.a = U64::new(object); + } + SurfaceResponse::MenuShown { serial } => { + w.tag = SurRepWire::TAG_MENU_SHOWN; + w.a = U64::new(u64::from(serial)); + } + SurfaceResponse::Capture(frame) => { + w.tag = SurRepWire::TAG_CAPTURE; + w.id = U64::new(frame.freeze_ns); + w.a = U64::new(capture_word(&frame)); + w.b = U64::new(frame.wall_ns.unwrap_or(0)); + w.c = U64::new(frame.uptime_ns); + w.d = U64::new(placement_word(&frame)); + } + SurfaceResponse::PickQueued { serial } => { + w.tag = SurRepWire::TAG_PICK_QUEUED; + w.a = U64::new(u64::from(serial)); + } + SurfaceResponse::PickClaimed { token } => { + w.tag = SurRepWire::TAG_PICK_CLAIMED; + w.a = U64::new(token); + } + SurfaceResponse::PickAnswered { token } => { + w.tag = SurRepWire::TAG_PICK_ANSWERED; + w.a = U64::new(token); + } SurfaceResponse::Refused { refusal, subject } => { w.tag = SurRepWire::TAG_REFUSED; w.refusal = refusal.code(); @@ -500,6 +713,44 @@ fn raw_decode_rep(w: &SurRepWire) -> Option { presented_at: w.d.get(), }, SurRepWire::TAG_CLOSE_REQUESTED => SurfaceResponse::CloseRequested { serial: w.a.get() }, + SurRepWire::TAG_CURSOR_SET => SurfaceResponse::CursorSet { + cursor: Cursor::from_code(w.a.get())?, + }, + SurRepWire::TAG_OPEN_QUEUED => SurfaceResponse::OpenQueued { object: w.a.get() }, + SurRepWire::TAG_MENU_SHOWN => SurfaceResponse::MenuShown { + serial: u32::try_from(w.a.get()).ok()?, + }, + SurRepWire::TAG_CAPTURE => { + let (a, d) = (w.a.get(), w.d.get()); + if a >> 56 != 0 || d >> 33 != 0 { + return None; + } + let mode = CaptureMode::from_code((a >> 32) as u8)?; + let synced = d & CAPTURE_UNSYNCED == 0; + // Only a window can be copied unsynchronized: a screen or + // region is the compositor's own complete frame. + if !synced && mode != CaptureMode::Window { + return None; + } + SurfaceResponse::Capture(CaptureFrame { + width: a as u16, + height: (a >> 16) as u16, + mode, + screen: (a >> 40) as u8, + density: crate::appearance::Density::from_quarters((a >> 48) as u8)?, + wall_ns: (w.b.get() != 0).then_some(w.b.get()), + uptime_ns: w.c.get(), + freeze_ns: w.id.get(), + x: d as u16, + y: (d >> 16) as u16, + synced, + }) + } + SurRepWire::TAG_PICK_QUEUED => SurfaceResponse::PickQueued { + serial: u32::try_from(w.a.get()).ok()?, + }, + SurRepWire::TAG_PICK_CLAIMED => SurfaceResponse::PickClaimed { token: w.a.get() }, + SurRepWire::TAG_PICK_ANSWERED => SurfaceResponse::PickAnswered { token: w.a.get() }, SurRepWire::TAG_REFUSED => SurfaceResponse::Refused { refusal: SurfaceRefusal::from_code(w.refusal)?, subject: w.id.get(), @@ -598,6 +849,24 @@ pub fn reply_matches(request: &SurReqWire, response: &SurfaceResponse, grants: u // A close raised under no configure serial names a window that was // never presented; the compositor refuses that instead. (SurfaceRequest::RequestClose, SurfaceResponse::CloseRequested { serial }) => serial != 0, + // The echo names the cursor asked for, never another. + (SurfaceRequest::SetCursor { cursor }, SurfaceResponse::CursorSet { cursor: got }) => { + cursor == got + } + (SurfaceRequest::OpenObject { object, .. }, SurfaceResponse::OpenQueued { object: got }) => { + object == got + } + (SurfaceRequest::ShowMenu { serial, .. }, SurfaceResponse::MenuShown { serial: got }) => { + serial == got + } + (SurfaceRequest::TakeCapture, SurfaceResponse::Capture(frame)) => frame.is_valid(), + (SurfaceRequest::PickObject { serial, .. }, SurfaceResponse::PickQueued { serial: got }) => { + serial == got + } + (SurfaceRequest::ClaimPick { token }, SurfaceResponse::PickClaimed { token: got }) + | (SurfaceRequest::Picked { token, .. }, SurfaceResponse::PickAnswered { token: got }) => { + token == got + } _ => false, } } @@ -668,6 +937,104 @@ pub const SURFACE_FORMAT_XRGB8888: u8 = DISPLAY_FORMAT_XRGB8888; #[cfg(test)] mod tests { + use crate::capture::{CaptureFrame, CaptureMode}; + + fn frame() -> CaptureFrame { + CaptureFrame { + width: 1280, + height: 800, + mode: CaptureMode::Screen, + screen: 0, + density: crate::appearance::Density::OneHalf, + wall_ns: Some(1_800_000_000_000_000_000), + uptime_ns: 42_000_000_000, + freeze_ns: 1_234_567, + x: 0, + y: 0, + synced: true, + } + } + + /// A window's placement and its sync travel with the lend; only a window + /// may arrive unsynchronized, and stray placement bits refuse it. + #[test] + fn a_window_capture_carries_its_placement_and_sync() { + let window = CaptureFrame { + mode: CaptureMode::Window, + width: 640, + height: 400, + x: 312, + y: 0xfffe, + synced: false, + ..frame() + }; + let region = CaptureFrame { + mode: CaptureMode::Region, + x: 7, + y: 9, + ..frame() + }; + for frame in [window, CaptureFrame { synced: true, ..window }, region] { + let response = SurfaceResponse::Capture(frame); + assert_eq!(decode_rep(&encode_rep(&response, 1)), Some((response, 1))); + } + let unsynced_screen = SurfaceResponse::Capture(CaptureFrame { + synced: false, + ..frame() + }); + assert_eq!( + decode_rep(&encode_rep(&unsynced_screen, 1)), + None, + "a screen is always the compositor's own complete frame" + ); + let mut stray = encode_rep(&SurfaceResponse::Capture(window), 1); + stray.d = (stray.d.get() | 1 << 40).into(); + assert_eq!(decode_rep(&stray), None); + } + + /// A capture lend is one grant, byte-exact, and pairs only with the + /// verb that asked for it. + #[test] + fn a_capture_lend_round_trips_with_its_one_grant() { + let request = encode_req(&SurfaceRequest::TakeCapture); + assert_eq!(decode_req(&request), Some(SurfaceRequest::TakeCapture)); + assert!(request_carries_grant(&SurfaceRequest::TakeCapture)); + assert_eq!( + ::request_grants(&request), + 1 + ); + for frame in [frame(), CaptureFrame { wall_ns: None, ..frame() }] { + let response = SurfaceResponse::Capture(frame); + assert_eq!(reply_grants(&response), 1); + let wire = encode_rep(&response, 1); + assert_eq!(decode_rep(&wire), Some((response, 1))); + assert!(reply_matches(&request, &response, 1)); + assert!(!reply_matches(&request, &response, 0), "the lend must ride it"); + assert_eq!(decode_rep(&encode_rep(&response, 0)), None); + let status = encode_req(&SurfaceRequest::Status { surface: 1 }); + assert!(!reply_matches(&status, &response, 1)); + // The picker and screenshot protocols share this wire, but a + // pick must never consume the capture writer's read-only grant. + let pick = encode_req(&SurfaceRequest::PickObject { types: 1, serial: 1 }); + assert_ne!(pick.op, request.op); + assert!(!reply_matches(&pick, &response, 1)); + let queued = SurfaceResponse::PickQueued { serial: 1 }; + assert_ne!(encode_rep(&queued, 0).tag, wire.tag); + assert!(!reply_matches(&request, &queued, 0)); + } + let empty = SurfaceResponse::Capture(CaptureFrame { + height: 0, + ..frame() + }); + assert!(!reply_matches(&request, &empty, 1), "a lend with no pixels"); + let mut stray = encode_rep(&SurfaceResponse::Capture(frame()), 1); + stray.a = (stray.a.get() | 1 << 56).into(); + assert_eq!(decode_rep(&stray), None); + let mut density = encode_rep(&SurfaceResponse::Capture(frame()), 1); + density.a = (density.a.get() & !(0xff << 48) | 1 << 48).into(); + assert_eq!(decode_rep(&density), None, "a density no screen draws at"); + } + use super::*; use kore_rt::svc::SvcProtocol; use zerocopy::{FromBytes, IntoBytes}; @@ -800,7 +1167,7 @@ mod tests { } } - fn every_request() -> [SurfaceRequest; 6] { + fn every_request() -> [SurfaceRequest; 14] { [ SurfaceRequest::Query, SurfaceRequest::CreateSurface { @@ -820,10 +1187,40 @@ mod tests { SurfaceRequest::DestroySurface { surface: 0x11 }, SurfaceRequest::Status { surface: 0x2002 }, SurfaceRequest::RequestClose, + SurfaceRequest::SetCursor { + cursor: Cursor::Default, + }, + SurfaceRequest::SetCursor { + cursor: Cursor::Progress, + }, + SurfaceRequest::OpenObject { + object: 0x5eed, + kind: DocumentKind::Unknown, + }, + SurfaceRequest::OpenObject { + object: u64::MAX, + kind: DocumentKind::Image, + }, + SurfaceRequest::ShowMenu { + x: -12, + y: 4000, + len: MENU_PLANE_BYTES as u16, + serial: u32::MAX, + }, + SurfaceRequest::PickObject { + types: u64::MAX, + serial: u32::MAX, + }, + SurfaceRequest::ClaimPick { token: u64::MAX }, + SurfaceRequest::Picked { + token: 0x7071_ce4e, + object: u64::MAX, + len: PICK_NAME_BYTES as u8, + }, ] } - fn every_response() -> [SurfaceResponse; 7] { + fn every_response() -> [SurfaceResponse; 13] { [ SurfaceResponse::Info { width: 1920, @@ -855,9 +1252,121 @@ mod tests { subject: 0, }, SurfaceResponse::CloseRequested { serial: 0x2A }, + SurfaceResponse::CursorSet { + cursor: Cursor::Text, + }, + SurfaceResponse::OpenQueued { object: 0x5eed }, + SurfaceResponse::MenuShown { serial: 7 }, + SurfaceResponse::PickQueued { serial: 3 }, + SurfaceResponse::PickClaimed { token: 0x7071_ce4e }, + SurfaceResponse::PickAnswered { token: 0x7071_ce4e }, ] } + /// A pick is answered only by its own serial's echo, a claim and an + /// answer only by their token's, none carries a grant, and a pick of no + /// types, a zero serial or token, or an answer whose name contradicts + /// its object refuses. + #[test] + fn picks_are_answered_only_by_their_own_serial_or_token() { + let pick = SurfaceRequest::PickObject { types: 0b1100, serial: 4 }; + let wire = encode_req(&pick); + assert_eq!(SurfaceWire::request_grants(&wire), 0); + assert_eq!(decode_req(&wire), Some(pick)); + assert!(reply_matches(&wire, &SurfaceResponse::PickQueued { serial: 4 }, 0)); + assert!(!reply_matches(&wire, &SurfaceResponse::PickQueued { serial: 5 }, 0)); + assert!(!reply_matches(&wire, &SurfaceResponse::MenuShown { serial: 4 }, 0)); + let claim = encode_req(&SurfaceRequest::ClaimPick { token: 9 }); + assert!(reply_matches(&claim, &SurfaceResponse::PickClaimed { token: 9 }, 0)); + assert!(!reply_matches(&claim, &SurfaceResponse::PickAnswered { token: 9 }, 0)); + let answer = encode_req(&SurfaceRequest::Picked { token: 9, object: 0x77, len: 9 }); + assert!(reply_matches(&answer, &SurfaceResponse::PickAnswered { token: 9 }, 0)); + assert!(!reply_matches(&answer, &SurfaceResponse::PickAnswered { token: 8 }, 0)); + let cancel = SurfaceRequest::Picked { token: 9, object: 0, len: 0 }; + assert_eq!(decode_req(&encode_req(&cancel)), Some(cancel)); + for bad in [ + SurfaceRequest::PickObject { types: 0, serial: 4 }, + SurfaceRequest::PickObject { types: 1, serial: 0 }, + SurfaceRequest::ClaimPick { token: 0 }, + SurfaceRequest::Picked { token: 0, object: 1, len: 1 }, + SurfaceRequest::Picked { token: 9, object: 1, len: 0 }, + SurfaceRequest::Picked { token: 9, object: 0, len: 1 }, + SurfaceRequest::Picked { token: 9, object: 1, len: PICK_NAME_BYTES as u8 + 1 }, + ] { + assert_eq!(decode_req(&encode_req(&bad)), None, "{bad:?}"); + } + // The serial's word holds a `u32`; its high bits are not spare. + let mut stray = wire; + stray.b = U64::new(stray.b.get() | 1 << 40); + assert_eq!(decode_req(&stray), None); + } + + /// A menu request is answered only by its own serial's echo, carries + /// no grant, and an empty or oversized plane or a zero serial refuses. + #[test] + fn a_menu_request_is_answered_only_by_its_own_serial() { + let show = SurfaceRequest::ShowMenu { + x: 40, + y: -3, + len: 96, + serial: 9, + }; + let wire = encode_req(&show); + assert_eq!(SurfaceWire::request_grants(&wire), 0); + assert_eq!(decode_req(&wire), Some(show)); + assert!(reply_matches(&wire, &SurfaceResponse::MenuShown { serial: 9 }, 0)); + assert!(!reply_matches(&wire, &SurfaceResponse::MenuShown { serial: 8 }, 0)); + assert!(!reply_matches(&wire, &SurfaceResponse::OpenQueued { object: 9 }, 0)); + for (len, serial) in [(0, 9), (MENU_PLANE_BYTES as u16 + 1, 9), (96, 0)] { + let bad = encode_req(&SurfaceRequest::ShowMenu { + x: 0, + y: 0, + len, + serial, + }); + assert_eq!(decode_req(&bad), None, "len {len} serial {serial}"); + } + // The length's spare high bits are not the serial's to borrow. + let mut stray = wire; + stray.b = U64::new(stray.b.get() | 1 << 20); + assert_eq!(decode_req(&stray), None); + // An open names a kind a class has, or refuses. + let mut open = encode_req(&SurfaceRequest::OpenObject { + object: 1, + kind: DocumentKind::Text, + }); + open.b = U64::new(DocumentKind::ALL.len() as u64); + assert_eq!(decode_req(&open), None); + } + + /// A cursor request is answered only by the echo of the cursor it + /// named, carries no grant, and a code no cursor has refuses at decode. + #[test] + fn a_cursor_request_is_answered_only_by_its_own_echo() { + for cursor in Cursor::ALL { + let wire = encode_req(&SurfaceRequest::SetCursor { cursor }); + assert_eq!(SurfaceWire::request_grants(&wire), 0); + assert_eq!(decode_req(&wire), Some(SurfaceRequest::SetCursor { cursor })); + for echoed in Cursor::ALL { + let held = SurfaceResponse::CursorSet { cursor: echoed }; + assert_eq!(reply_matches(&wire, &held, 0), echoed == cursor); + } + } + let mut stray = encode_req(&SurfaceRequest::SetCursor { + cursor: Cursor::Text, + }); + stray.a = U64::new(Cursor::ALL.len() as u64); + assert_eq!(decode_req(&stray), None); + let mut reply = encode_rep( + &SurfaceResponse::CursorSet { + cursor: Cursor::Text, + }, + 0, + ); + reply.a = U64::new(u64::MAX); + assert_eq!(decode_rep(&reply), None); + } + /// A close request answers only under a live configure serial, and a /// refusal of it names no subject. #[test] diff --git a/native/drivers/nvme/Cargo.toml b/native/drivers/nvme/Cargo.toml new file mode 100644 index 0000000..59e7610 --- /dev/null +++ b/native/drivers/nvme/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "kore-nvme" +version = "0.0.0" +edition = "2024" +description = "kore NVMe driver core: SPDK-shaped per-core qpairs over the shared register/DMA seam; the production service composition lives in kore-svc-swap-nvme (services/swap-nvme), which claims the controller from the device broker and serves the swap protocol over this core." +publish = false + +# The driver core is pure logic over the `seam` traits + the kore-abi device +# ABI's typed addresses — [`IovaAddr`]: "no phys in userland" is a *type*, +# and the driver speaks it end to end (raw `u64` never names an address). +# The kernel first-light impl and production userland host each provide the +# seam. The in-crate hardware double is `#[cfg(test)]` only and proves core +# behavior, not production composition. +[dependencies] +kore-abi = { path = "../../crates/abi" } +# The shared driver substrate: the platform seam (re-exported as +# `kore_nvme::seam`) + the async ring mechanics kore-nvme's `Qpair` composes +# from (`SubRing` / `CompRing` / `CookieTable` / `RingStats`). kore-nvme is +# its first customer. +kore-driver = { path = "../../crates/driver" } +# The swap-service face only (`service` feature): the SystemService trait +# + framework and the real cap verbs the serve step uses to bind delivered +# pages (kore-rt), and the SwapRequest vocabulary (kore-svc-swap — the +# swap service's own crate, res/USERLAND.md §1.1). The kernel dep uses +# default features, so neither ever enters the kernel binary. +kore-rt = { path = "../../crates/rt", optional = true } +kore-svc-swap = { path = "../../services/swap", optional = true } + +[features] +# Userland posture: size runtime tables (the qpair cookie table) exactly to +# the controller's negotiated depth through the global allocator — which in +# kore userland IS the mm-backed heap (kore-svc-mm). Off (the kernel +# boot-test): fixed inline ceilings, heapless. +alloc = ["kore-driver/alloc"] +# The disk-swap service face: NvmeSwap = SystemService +# over the driver core — the lessor loop's disk backend. +service = ["alloc", "dep:kore-rt", "dep:kore-svc-swap"] + +# The self-dev-dependency turns `service` on for this crate's own test +# builds, so the service face stays test-compiled. +[dev-dependencies] +kore-nvme = { path = ".", features = ["service"] } +# The storage transport window, for the engine-over-a-queue-group tests. +kore-ring = { path = "../../crates/ring" } +# GPT fixture building for the discovery tests: the same crates (and +# versions) `kore-driver`'s parse side uses, here writing the tables the +# mock disk serves back through the real chunked-read path. +gpt_disk_types = { version = "0.16.1", default-features = false, features = ["bytemuck"] } +crc = { version = "3.4.0", default-features = false } +bytemuck = { version = "1.25.2", default-features = false } diff --git a/native/drivers/nvme/src/block.rs b/native/drivers/nvme/src/block.rs new file mode 100644 index 0000000..6176881 --- /dev/null +++ b/native/drivers/nvme/src/block.rs @@ -0,0 +1,1640 @@ +//! Hardware-neutral direct block-channel adapter over one NVMe qpair. +//! +//! This is the only layer where [`kore_driver::block`] vocabulary meets NVMe +//! namespace commands, PRPs, IOVAs, and completion status. Consumers above +//! it own a `BlockChannel>` and cannot name those device +//! details. + +use core::mem::size_of; + +use kore_driver::block::{ + BlockBuffer, BlockError, BlockGeometry, BlockHealth, BlockIoHints, CompletionPacer, + CountedPacer, DeviceCommand, DeviceCompletion, DirectBlockDevice, +}; +use kore_driver::seam::{Dma, DmaPool, Regs}; + +use crate::{ + Controller, Error, Namespace, Qpair, + commands::{DsmRange, HmbDescriptor}, + controller::{DeviceInfo, HMB_MAX_CHUNKS}, + namespace::MAX_NLB_BLOCKS, + prp::{ListPage, MemPage, max_single_list_transfer}, +}; + +/// Identify Controller ONCS bit 2 (NVM Command Set: Dataset Management). +const ONCS_DATASET_MANAGEMENT: u16 = 1 << 2; + +/// PRP-list pages shared by commands into **adopted** buffers — the +/// caller-funded streaming targets ([`NvmeBlockDevice::adopt_buffer`]). +/// +/// A registered buffer keeps a private list page because it has at most one +/// command in flight; an adopted target is multi-GiB and takes many commands +/// at once, each needing its own list for as long as the device may read it. +/// So adopted transfers borrow a page from this pool at submission and return +/// it at their completion: the pool is sized by *queue occupancy*, never by +/// target size, which is what keeps a 16 GiB target from costing 32 MiB of +/// this row's DMA pool in list pages. One hundred and twenty-eight covers a +/// ring streaming adopted commands at full depth on one queue (the +/// transport engine's class budgets bound the depth), and exhaustion is +/// ordinary backpressure the engine retries. +pub const ADOPTED_LIST_PAGES: usize = 128; + +/// Polls an Abort's admin completion may take before the abort counts as +/// refused — bounded, because it runs on the engine's deadline scan. +const ABORT_POLLS: u32 = 1 << 20; + +/// One registered DMA buffer plus an optional private PRP-list page. Buffers +/// spanning at most two controller pages fit directly in PRP1/PRP2; larger +/// buffers keep a private list so a later submission cannot rewrite entries +/// the device may still be consuming. +pub struct NvmeBuffer { + data: B, + list: Option, + len: usize, +} + +impl NvmeBuffer { + /// Wrap an allocation that was never adopted, so a refused adoption can + /// still travel the one release path. It carries no list page and must + /// not be submitted against. + pub fn from_data(data: B) -> Self { + let len = data.len(); + Self { + data, + list: None, + len, + } + } + + /// Recover the DMA allocation from an idle buffer — how an adopted + /// target's device mapping returns to the owner that must release it + /// quiescently. A private list page, if any, is dropped with the + /// wrapper: it is this row's own pool allocation. + pub fn into_data(self) -> B { + self.data + } +} + +impl BlockBuffer for NvmeBuffer { + fn len(&self) -> usize { + self.len + } + + fn bytes(&self) -> &[u8] { + // SAFETY: `len` was checked against the owned DMA allocation. + unsafe { core::slice::from_raw_parts(self.data.cpu(), self.len) } + } + + fn bytes_mut(&mut self) -> &mut [u8] { + // SAFETY: as above; the channel prevents CPU access while busy. + unsafe { core::slice::from_raw_parts_mut(self.data.cpu(), self.len) } + } +} + +/// Each submission queue's arbitration priority, by index: urgent, then +/// high, then medium (with the three-class policy: Demand, Commit, Stream). +/// Honoured only under weighted round robin +/// ([`Controller::weighted_arbitration`]); plain round robin ignores it, and +/// a pair's one queue is urgent exactly as every queue was before +/// priorities existed. +const PRIORITIES: [crate::commands::SqPriority; 4] = [ + crate::commands::SqPriority::Urgent, + crate::commands::SqPriority::High, + crate::commands::SqPriority::Medium, + crate::commands::SqPriority::Low, +]; + +/// One passive NVMe data queue. It owns the controller parent so the qpair's +/// BAR/DMA lifetime cannot outlive the authority it derives from. +pub struct NvmeBlockDevice { + controller: Controller, + namespace: Namespace, + qpair: Qpair, + geometry: BlockGeometry, + dsm: P::Buf, + /// Reused SMART/Health DMA buffer; status probes never grow the DMA pool. + health: P::Buf, + dsm_cookie: Option, + /// The shared list-page pool for adopted targets, allocated on the first + /// adoption (a row that never adopts pays nothing), and the cookie each + /// page is lent to while its command is in flight. + adopted_lists: [Option; ADOPTED_LIST_PAGES], + adopted_owners: [Option; ADOPTED_LIST_PAGES], + shutdown_polls: u32, + /// Caller-selected relaxation for SMART and lifecycle command waits. + /// The driver core cannot choose a runtime address-monitor implementation. + lifecycle_pause: fn(), + quiesced: bool, + /// What a controller reset must replay to bring the I/O queues back: + /// the `(submission, completion)` queue counts asked for (Number of + /// Queues does not survive a reset) and the queue group's vector. + queue_request: (u16, u16), + vector: Option, + /// Controller resets this device has been through. + resets: u32, +} + +/// How much host memory to offer the controller for its mapping tables: the +/// least of what it asks for (Identify HMPRE) and the consumer's allowance, +/// rounded down to whole controller pages — and nothing at all when the +/// controller uses none, or when what could be offered is below the floors +/// it advertises (a buffer it may refuse is not worth the round trip). +pub fn host_memory_buffer_bytes(info: &DeviceInfo, allowance: u64, page: MemPage) -> u64 { + if info.hmb_preferred_bytes == 0 || allowance == 0 { + return 0; + } + let page = page.bytes() as u64; + let bytes = info.hmb_preferred_bytes.min(allowance) / page * page; + if bytes == 0 || bytes < info.hmb_min_bytes || bytes < info.hmb_min_descriptor_bytes { + return 0; + } + bytes +} + +impl NvmeBlockDevice { + /// Finish identify/queue provisioning and bind one direct channel. + /// `hmb_max_bytes` is the most host memory the controller may be given + /// for its mapping tables (0 offers none). + pub fn provision( + controller: Controller, + qid: u16, + depth: u16, + vector: Option, + hmb_max_bytes: u64, + max_polls: u32, + pause: fn(), + ) -> Result { + let mut pacer = CountedPacer::new(max_polls, pause); + Self::provision_paced( + controller, + qid, + depth, + vector, + hmb_max_bytes, + &mut pacer, + pause, + max_polls, + ) + } + + /// Finish controller provisioning under one caller-owned pacing budget. + /// This is the production boot path: all admin commands share the same + /// absolute deadline instead of each receiving a fresh poll count. + #[allow(clippy::too_many_arguments)] + pub fn provision_paced( + controller: Controller, + qid: u16, + depth: u16, + vector: Option, + hmb_max_bytes: u64, + pacer: &mut dyn CompletionPacer, + lifecycle_pause: fn(), + shutdown_polls: u32, + ) -> Result { + Self::provision_queues_paced( + controller, + qid, + depth, + vector, + hmb_max_bytes, + pacer, + lifecycle_pause, + shutdown_polls, + ) + } +} + +impl NvmeBlockDevice { + /// [`provision_paced`](NvmeBlockDevice::provision_paced) for a device of + /// `SQS` submission queues over one completion queue (ids from `qid`), + /// each `depth` deep — one per service class. + #[allow(clippy::too_many_arguments)] + pub fn provision_queues_paced( + mut controller: Controller, + qid: u16, + depth: u16, + vector: Option, + hmb_max_bytes: u64, + pacer: &mut dyn CompletionPacer, + lifecycle_pause: fn(), + shutdown_polls: u32, + ) -> Result { + let info = controller.identify_paced(pacer).map_err(map_submit_error)?; + let namespace = controller.namespace(); + let wanted = qid.max(1); + // A pair asks for `qid` queues of each kind, as it always did. A + // group asks for one submission queue per class beside its one + // completion queue, and runs with however many it is granted (at + // least one) — fewer queues fold classes together, never refuse. + let mut queue_request = (wanted, wanted); + let live = if SQS == 1 { + let (sq, cq) = controller + .set_num_queues_paced(wanted, pacer) + .map_err(map_submit_error)?; + if sq < wanted || cq < wanted { + return Err(BlockError::Backpressure); + } + 1 + } else { + let want_sqs = u16::try_from(SQS) + .ok() + .and_then(|n| n.checked_add(wanted - 1)) + .ok_or(BlockError::Geometry)?; + queue_request = (want_sqs, wanted); + let (sq, cq) = controller + .set_queue_counts_paced(want_sqs, wanted, pacer) + .map_err(map_submit_error)?; + if cq < wanted || sq < wanted { + return Err(BlockError::Backpressure); + } + usize::from(sq - (wanted - 1)).min(SQS) + }; + // The host memory buffer, before the I/O queues exist (the order + // Linux keeps). One page-aligned allocation is one descriptor: the + // pool's IOVA window is contiguous by construction, whatever the + // frames beneath it. A controller that uses none, an allowance + // below its minimum, a pool that cannot carry it, or a refusal all + // leave the device running WITHOUT one — a slower filesystem, never + // a missing one — and the enable takes its own counted budget so a + // controller that is slow to take 64 MiB cannot spend the shared + // bring-up deadline the queue creation below still needs. The + // buffer is disabled again by `Controller::shutdown`, which the + // quiesce hook reaches while bus mastering is still open; the + // revoke path that skips quiesce is contained by the host closing + // bus mastering (or binding the requester to Abort) before the + // frames are reaped, so the controller can never reach memory it + // no longer owns. + let hmb_bytes = host_memory_buffer_bytes(&info, hmb_max_bytes, controller.page()); + if hmb_bytes != 0 { + let page = controller.page().bytes() as usize; + let list = controller + .pool() + .alloc(size_of::() * HMB_MAX_CHUNKS); + let chunk = controller.pool().alloc_aligned(hmb_bytes as usize, page); + if let (Some(list), Some(chunk)) = (list, chunk) { + let mut chunks = [const { None }; HMB_MAX_CHUNKS]; + chunks[0] = Some(chunk); + let mut hmb_pacer = CountedPacer::new(shutdown_polls, lifecycle_pause); + let _ = controller.enable_host_memory_buffer(list, chunks, &mut hmb_pacer); + } + } + let qpair = controller + .create_io_queue_group_paced::( + qid, + qid, + live, + depth, + core::array::from_fn(|index| PRIORITIES[index.min(PRIORITIES.len() - 1)]), + vector, + pacer, + ) + .map_err(map_submit_error)?; + // An arbitration burst of one command per queue per round keeps a + // deep bulk queue from taking a whole burst ahead of a demand read. + // Under weighted round robin the urgent queue is served first and + // the weights split the rest: high four turns to medium's one, so + // durability writes outpace bulk streams. Best effort: a controller + // that refuses the feature keeps its default burst and weights. + if SQS > 1 { + let _ = controller.set_arbitration_paced( + crate::commands::Arbitration { + burst: crate::commands::ArbitrationBurst::Pow2(0), + high: 3, + medium: 0, + low: 0, + }, + pacer, + ); + } + let dsm = controller + .pool() + .alloc(size_of::()) + .ok_or(BlockError::Backpressure)?; + let health = controller + .pool() + .alloc(size_of::()) + .ok_or(BlockError::Backpressure)?; + + let prp_ceiling = max_single_list_transfer(controller.page()); + if info.block_size == 0 || info.block_count == 0 { + return Err(BlockError::Geometry); + } + let mdts = if info.mdts_bytes == 0 { + prp_ceiling + } else { + prp_ceiling.min(info.mdts_bytes) + }; + let max_transfer_blocks = + (mdts / info.block_size as u64).min(u64::from(MAX_NLB_BLOCKS)) as u32; + let geometry = BlockGeometry { + block_size: info.block_size, + block_count: info.block_count, + max_transfer_blocks, + // The core does not yet resolve namespace NAWUN's controller + // fallback and boundary rules; one logical block is therefore + // the only honest guaranteed atomic unit. + atomic_write_blocks: 1, + flush: true, + discard: info.oncs & ONCS_DATASET_MANAGEMENT != 0, + read_only: false, + io_hints: BlockIoHints { + preferred_write_blocks: info.preferred_write_blocks, + write_alignment_blocks: info.write_alignment_blocks, + optimal_write_blocks: info.optimal_write_blocks, + optimal_boundary_blocks: info.optimal_boundary_blocks, + supported_block_sizes: info.supported_block_sizes, + }, + } + .validate()?; + + Ok(Self { + controller, + namespace, + qpair, + geometry, + dsm, + health, + dsm_cookie: None, + adopted_lists: [const { None }; ADOPTED_LIST_PAGES], + adopted_owners: [None; ADOPTED_LIST_PAGES], + shutdown_polls, + lifecycle_pause, + quiesced: false, + queue_request, + vector, + resets: 0, + }) + } + + /// **Reset the controller** and bring the I/O queues back over the same + /// memory (`res/STORAGE_TRANSPORT.md` §7): the escalation for a command + /// the device will not give up — an Abort it refused. Every command in + /// flight is lost with the reset; nothing the controller held survives. + /// Namespace, geometry, buffers and adopted targets do. Answers how many + /// commands were lost; an `Err` leaves the device unusable, as any + /// failed bring-up does. + pub fn reset_paced(&mut self, pacer: &mut dyn CompletionPacer) -> Result { + self.controller.reset_paced(pacer).map_err(map_submit_error)?; + let lost = self.qpair.reset(); + // Number of Queues is volatile across a reset: ask again, and settle + // for no fewer than the group already runs. + let (sqs, cqs) = self.queue_request; + let (granted_sq, granted_cq) = if SQS == 1 { + self.controller.set_num_queues_paced(cqs, pacer) + } else { + self.controller.set_queue_counts_paced(sqs, cqs, pacer) + } + .map_err(map_submit_error)?; + if granted_cq < cqs || usize::from(granted_sq) < self.qpair.live_sqs() { + return Err(BlockError::Backpressure); + } + self.controller + .recreate_io_queue_group_paced( + &self.qpair, + core::array::from_fn(|index| PRIORITIES[index.min(PRIORITIES.len() - 1)]), + self.vector, + pacer, + ) + .map_err(map_submit_error)?; + if SQS > 1 { + let _ = self.controller.set_arbitration_paced( + crate::commands::Arbitration { + burst: crate::commands::ArbitrationBurst::Pow2(0), + high: 3, + medium: 0, + low: 0, + }, + pacer, + ); + } + // Every command that borrowed a list page is gone with the reset. + self.adopted_owners = [None; ADOPTED_LIST_PAGES]; + self.dsm_cookie = None; + self.resets += 1; + Ok(lost) + } + + /// Controller resets this device has been through. + pub fn resets(&self) -> u32 { + self.resets + } + + /// Wrap a DMA allocation the **caller** funded — a device mapping of a + /// client's lease, not this row's pool — as a buffer a channel can adopt + /// as a streaming target (`BlockChannel::adopt_target`). + /// + /// It carries no private list page: its commands borrow one from the + /// shared [`ADOPTED_LIST_PAGES`] pool for exactly their flight, so any + /// number of commands may target one adopted buffer at once. The pool is + /// allocated here on first use; a refusal returns the allocation, whose + /// owner must still release it. + /// + /// The allocation must be page-aligned (every DMA window this tree hands + /// out is) and a whole number of logical blocks long. + pub fn adopt_buffer( + &mut self, + data: P::Buf, + ) -> Result, (P::Buf, BlockError)> { + let page = self.controller.page().bytes() as usize; + let len = data.len(); + if len == 0 + || len % self.geometry.block_size as usize != 0 + || data.iova().0 % page as u64 != 0 + { + return Err((data, BlockError::Alignment)); + } + if self.adopted_lists[0].is_none() { + for slot in &mut self.adopted_lists { + match self.controller.pool().alloc_aligned(page, page) { + Some(list) => *slot = Some(list), + None => return Err((data, BlockError::Backpressure)), + } + } + } + Ok(NvmeBuffer { + data, + list: None, + len, + }) + } + + /// Return an adopted buffer's allocation to the pool it came from for + /// quiescent release — for an import, the step that lets the owner's + /// DMA host retire its device mapping. + /// + /// # Safety + /// No command may be in flight against `buffer` and the device must hold + /// no other reference to it — the [`DmaPool::release_quiesced`] contract. + /// A channel hands an adopted target back only when nothing is in flight + /// against it, which is the first half; the caller proves the rest. + pub unsafe fn release_adopted( + &self, + buffer: NvmeBuffer, + ) -> Result<(), kore_driver::seam::DmaReleaseFailure> { + // SAFETY: forwarded to the caller above. + unsafe { self.controller.pool().release_quiesced(buffer.into_data()) } + } + + /// Lend a free shared list page to `cookie`'s command. + fn lend_adopted_list(&mut self, cookie: u64) -> Option { + let index = self + .adopted_owners + .iter() + .zip(self.adopted_lists.iter()) + .position(|(owner, list)| owner.is_none() && list.is_some())?; + self.adopted_owners[index] = Some(cookie); + Some(index) + } + + /// Return the page lent to `cookie`, if any — at its completion, or when + /// its submission was refused before the device saw it. + fn return_adopted_list(&mut self, cookie: u64) { + if let Some(owner) = self + .adopted_owners + .iter_mut() + .find(|owner| **owner == Some(cookie)) + { + *owner = None; + } + } + + /// Shared list pages currently lent to in-flight commands. + pub fn adopted_lists_in_flight(&self) -> usize { + self.adopted_owners.iter().flatten().count() + } + + pub fn qpair_stats(&self) -> crate::QpairStats { + self.qpair.stats() + } + + /// Whether the controller arbitrates its queues by weighted round robin + /// with urgent priority (`CC.AMS`), rather than plain round robin. + pub fn weighted_arbitration(&self) -> bool { + self.controller.weighted_arbitration() + } + + /// The controller's raw `CSTS`, read from its BAR without a command — + /// the same register [`liveness`](DirectBlockDevice::liveness) judges. + pub fn controller_status(&self) -> u32 { + self.controller.regs().read32(crate::regs::REG_CSTS) + } + + /// The data queue's submission side: `(software tail, the head the + /// controller last reported, doorbell writes)`. Tail ahead of head with + /// nothing completing is the shape of commands the controller was never + /// told about, or never fetched. + pub fn submission_probe(&self) -> (u16, u16, u64) { + ( + self.qpair.sq_tail(), + self.qpair.last_sq_head(), + self.qpair.doorbells(), + ) + } + + /// The raw submission entry `back` slots behind the tail (0 = the last + /// staged) — see [`Qpair::sq_entry_raw`](crate::Qpair::sq_entry_raw). + pub fn submission_entry(&self, back: u16) -> Option<[u8; 64]> { + self.qpair.sq_entry_raw(back) + } + + /// Host memory the controller holds for its mapping tables, in bytes + /// (0 = none granted, or already disabled). + pub fn host_memory_buffer_bytes(&self) -> u64 { + self.controller.info().hmb_bytes + } + + /// **The word a completion's arrival stores to** — this channel's data + /// queue, forwarded from [`Qpair::completion_word`]. + /// + /// For a caller composing an address-monitor tier under its completion + /// wait: snapshot the word, sleep while it is unchanged, then poll. The + /// wait itself is not offered here, because the mechanism lives in + /// `kore-rt` and this crate's core must not depend on it — see + /// [`Qpair::completion_word`] for why the split falls exactly there. + /// + /// `None` on a ring base this crate cannot prove `u32`-aligned; the caller + /// keeps its ordinary pacing. + pub fn completion_word(&self) -> Option<&core::sync::atomic::AtomicU32> { + self.qpair.completion_word() + } + + /// Return the capability-backed register and DMA seams after the qpair + /// has been quiesced. The common driver host uses this to retire the + /// ordinary BAR/IOMMU/region descendants in dependency order. + pub fn into_seam(self) -> (R, P) { + let Self { + controller, + qpair, + dsm, + health, + adopted_lists, + .. + } = self; + drop(qpair); + drop(dsm); + drop(health); + drop(adopted_lists); + controller.into_seam() + } + + fn list_page<'a>(buffer: &'a mut NvmeBuffer) -> Option> { + buffer.list.as_mut().map(|list| { + let count = list.len() / size_of::(); + // SAFETY: Dma allocations are page-aligned and `count` covers + // only the owned list allocation. + let entries = + unsafe { core::slice::from_raw_parts_mut(list.cpu().cast::(), count) }; + ListPage { + iova: list.iova(), + entries, + } + }) + } + + /// Build one data transfer. A registered buffer uses its private list + /// page; an adopted buffer (no private page) first tries PRP1/PRP2 only + /// and borrows a shared page when the span needs a list. A borrowed + /// page stays lent to `cookie` until [`return_adopted_list`] — the + /// caller returns it on a refused submission, completion returns it + /// otherwise. + fn build_transfer( + &mut self, + write: bool, + lba: u64, + blocks: u32, + buffer: &mut NvmeBuffer, + buffer_offset: usize, + cookie: u64, + ) -> Result { + let end = buffer_offset + .checked_add(self.namespace.bytes(blocks) as usize) + .ok_or(BlockError::Range)?; + if end > buffer.len { + return Err(BlockError::Range); + } + let iova = kore_abi::device::IovaAddr( + buffer + .data + .iova() + .0 + .checked_add(buffer_offset as u64) + .ok_or(BlockError::Range)?, + ); + let build = |namespace: &Namespace, list: Option>| { + if write { + namespace.write(lba, blocks, iova, list) + } else { + namespace.read(lba, blocks, iova, list) + } + }; + if buffer.list.is_some() { + let list = Self::list_page(buffer); + return build(&self.namespace, list).map_err(map_submit_error); + } + match build(&self.namespace, None) { + Err(Error::Prp(crate::prp::PrpError::ListNeeded)) => {} + other => return other.map_err(map_submit_error), + } + let index = self + .lend_adopted_list(cookie) + .ok_or(BlockError::Backpressure)?; + let page = self.adopted_lists[index] + .as_mut() + .expect("lent page exists"); + let count = page.len() / size_of::(); + // SAFETY: the page is a page-aligned pool allocation lent to this + // command alone; no other command can hold it until it is returned. + let entries = unsafe { core::slice::from_raw_parts_mut(page.cpu().cast::(), count) }; + let list = ListPage { + iova: page.iova(), + entries, + }; + let built = build(&self.namespace, Some(list)).map_err(map_submit_error); + if built.is_err() { + self.adopted_owners[index] = None; + } + built + } + + fn build_sqe( + &mut self, + command: DeviceCommand, + buffer: Option<&mut NvmeBuffer>, + cookie: u64, + ) -> Result { + Ok(match command { + DeviceCommand::Read { + lba, + blocks, + buffer_offset, + } => { + let buffer = buffer.ok_or(BlockError::UnknownBuffer)?; + self.build_transfer(false, lba, blocks, buffer, buffer_offset, cookie)? + } + DeviceCommand::Write { + lba, + blocks, + buffer_offset, + } => { + let buffer = buffer.ok_or(BlockError::UnknownBuffer)?; + self.build_transfer(true, lba, blocks, buffer, buffer_offset, cookie)? + } + DeviceCommand::Flush => self.namespace.flush(), + DeviceCommand::Discard { lba, blocks } => { + if self.dsm_cookie.is_some() { + return Err(BlockError::Backpressure); + } + // SAFETY: `dsm` owns at least one complete DsmRange. + unsafe { + self.dsm + .cpu() + .cast::() + .write(DsmRange::new(lba, blocks)) + }; + self.dsm_cookie = Some(cookie); + self.namespace + .deallocate(self.dsm.iova(), 1) + .map_err(map_submit_error)? + } + }) + } + + fn device_completion(&mut self, completion: crate::Completion) -> DeviceCompletion { + if self.dsm_cookie == Some(completion.cookie) { + self.dsm_cookie = None; + } + // NVMe completes a command only after its data transfer, so the + // device has finished reading the list page by now. + self.return_adopted_list(completion.cookie); + DeviceCompletion { + cookie: completion.cookie, + result: if completion.is_success() { + Ok(()) + } else { + Err(map_status(completion.cqe.status_code_type())) + }, + } + } +} + +impl NvmeBlockDevice { + /// Submission queues this device runs (`1..=SQS`; fewer than `SQS` when + /// the controller granted fewer). + pub fn live_queues(&self) -> usize { + self.qpair.live_sqs() + } + + /// Commands submission queue `sq` can still take. + pub fn room_on(&self, sq: usize) -> usize { + usize::from(self.qpair.room_on(sq)) + } + + /// Stage one command on submission queue `sq` (not yet visible to the + /// device — [`publish_on`](Self::publish_on)). The class-aware twin of + /// [`DirectBlockDevice::stage`], which uses queue 0. + pub fn stage_on( + &mut self, + sq: usize, + command: DeviceCommand, + buffer: Option<&mut NvmeBuffer>, + cookie: u64, + ) -> Result<(), BlockError> { + let sqe = self.build_sqe(command, buffer, cookie)?; + if let Err(error) = self.qpair.submit_on(sq, sqe, cookie) { + if self.dsm_cookie == Some(cookie) { + self.dsm_cookie = None; + } + self.return_adopted_list(cookie); + return Err(map_qpair_error(error)); + } + Ok(()) + } + + /// Abort the command `cookie` staged on queue `sq` (NVMe Abort, 08h). + /// `true` when the controller agreed: the command's own completion then + /// arrives as *Command Abort Requested*. The admin wait is bounded. + pub fn abort_on(&mut self, sq: usize, cookie: u64) -> bool { + let Some((qid, cid)) = self.qpair.locate(sq, cookie) else { + return false; + }; + let mut pacer = kore_driver::block::CountedPacer::new(ABORT_POLLS, core::hint::spin_loop); + matches!( + self.controller.abort_paced(qid, cid, &mut pacer), + Ok(crate::controller::AbortOutcome::Aborted) + ) + } + + /// Ring submission queue `sq`'s doorbell for everything staged on it. + pub fn publish_on(&mut self, sq: usize) { + self.qpair.ring_on(sq, self.controller.regs()); + } +} + +/// The device as the storage transport engine drives it: `SQS` submission +/// queues over one completion queue, a class per queue +/// ([`kore_driver::stor::engine::ThreeClass`]). Named by path, never +/// imported, so the device's [`DirectBlockDevice`] methods keep their +/// unambiguous names everywhere else. +impl kore_driver::stor::QueueSet + for NvmeBlockDevice +{ + type Target = NvmeBuffer; + const SQS: usize = SQS; + + fn geometry(&self) -> BlockGeometry { + self.geometry + } + + fn target_bytes(target: &Self::Target) -> u64 { + target.len() as u64 + } + + fn allocate(&mut self, bytes: usize) -> Result { + DirectBlockDevice::allocate_buffer(self, bytes) + } + + fn max_target_bytes(&self) -> usize { + DirectBlockDevice::max_buffer_bytes(self) + } + + fn room(&self, sq: usize) -> usize { + self.room_on(sq) + } + + fn stage( + &mut self, + sq: usize, + command: DeviceCommand, + target: Option<&mut Self::Target>, + tag: kore_driver::stor::Tag, + ) -> Result<(), BlockError> { + self.stage_on(sq, command, target, tag.cookie()) + } + + fn abort(&mut self, sq: usize, tag: kore_driver::stor::Tag) -> bool { + self.abort_on(sq, tag.cookie()) + } + + fn reset(&mut self) -> bool { + kore_driver::block::DirectBlockDevice::reset(self) + } + + fn publish(&mut self, sq: usize) { + self.publish_on(sq); + } + + fn reap(&mut self, out: &mut [DeviceCompletion]) -> usize { + let mut count = 0; + while count < out.len() { + let Some(completion) = DirectBlockDevice::poll(self) else { + break; + }; + out[count] = completion; + count += 1; + } + count + } + + fn liveness(&mut self) -> kore_driver::block::Liveness { + DirectBlockDevice::liveness(self) + } + + fn health(&mut self) -> Result { + DirectBlockDevice::health(self) + } + + fn quiesce(&mut self) -> Result<(), BlockError> { + DirectBlockDevice::quiesce(self) + } + + fn completion_word(&self) -> Option<&core::sync::atomic::AtomicU32> { + self.qpair.completion_word() + } +} + +impl DirectBlockDevice for NvmeBlockDevice { + type Buffer = NvmeBuffer; + + /// A submitted command advances in the controller's own queues; there is + /// no correlated reply for this side to await, which is why `progress` + /// stays the ready default. Sharing one controller between several + /// designated regions relies on exactly that. + const INDEPENDENT_PROGRESS: bool = true; + + fn abort(&mut self, cookie: u64) -> bool { + self.abort_on(0, cookie) + } + + fn reset(&mut self) -> bool { + let mut pacer = kore_driver::block::CountedPacer::new(self.shutdown_polls, self.lifecycle_pause); + self.reset_paced(&mut pacer).is_ok() + } + + /// One `CSTS` read — the controller's own verdict on itself. + /// + /// This is why a late completion need not be treated as a dead device: + /// NVMe publishes `CSTS.CFS` (Controller Fatal Status) and `CSTS.RDY` in + /// a BAR register, so liveness is answerable *without* the command path + /// that is in doubt. `health()` cannot serve here — fetching a SMART log + /// is itself a command, and would be exactly as stuck. + /// + /// The driver's bring-up waits already use `CSTS.CFS` as their fault + /// predicate; this is the same predicate on the data path. + fn liveness(&mut self) -> kore_driver::block::Liveness { + use kore_driver::block::Liveness; + let csts = self.controller.regs().read32(crate::regs::REG_CSTS); + // An all-ones read is a vanished device (a PCIe link down reads as + // 0xFFFF_FFFF), not a controller reporting every bit set. + if csts == u32::MAX { + return Liveness::Failed; + } + if csts & crate::regs::CSTS_CFS != 0 { + return Liveness::Failed; + } + if csts & crate::regs::CSTS_RDY == 0 { + // Ready dropped out from under a provisioned queue: the + // controller reset or is resetting, and nothing outstanding will + // complete. + return Liveness::Failed; + } + Liveness::Healthy + } + + fn geometry(&self) -> BlockGeometry { + self.geometry + } + + fn max_buffer_bytes(&self) -> usize { + usize::try_from( + u64::from(self.geometry.max_transfer_blocks) + .saturating_mul(u64::from(self.geometry.block_size)), + ) + .unwrap_or(usize::MAX) + } + + fn health(&mut self) -> Result { + let log = self + .controller + .smart_into(&mut self.health, self.shutdown_polls, self.lifecycle_pause) + .map_err(map_submit_error)?; + Ok(BlockHealth { + critical_warning: log.critical_warning, + temperature_celsius: i16::try_from(log.temp_celsius()).unwrap_or(i16::MAX), + available_spare_percent: log.avail_spare, + spare_threshold_percent: log.spare_threshold, + percentage_used: log.percentage_used, + power_cycles: crate::commands::SmartLog::counter(&log.power_cycles), + power_on_hours: crate::commands::SmartLog::counter(&log.power_on_hours), + unsafe_shutdowns: crate::commands::SmartLog::counter(&log.unsafe_shutdowns), + media_errors: crate::commands::SmartLog::counter(&log.media_errors), + }) + } + + fn allocate_buffer(&mut self, bytes: usize) -> Result { + if bytes == 0 + || bytes as u64 + > self.geometry.max_transfer_blocks as u64 * self.geometry.block_size as u64 + { + return Err(BlockError::TooLarge); + } + let data = self + .controller + .pool() + .alloc_aligned(bytes, self.controller.page().bytes() as usize) + .ok_or(BlockError::Backpressure)?; + if data.len() < bytes { + return Err(BlockError::Integrity); + } + let page_bytes = self.controller.page().bytes() as usize; + let list = if bytes > page_bytes.saturating_mul(2) { + Some( + self.controller + .pool() + .alloc_aligned(page_bytes, page_bytes) + .ok_or(BlockError::Backpressure)?, + ) + } else { + None + }; + Ok(NvmeBuffer { + data, + list, + len: bytes, + }) + } + + fn submit( + &mut self, + command: DeviceCommand, + buffer: Option<&mut Self::Buffer>, + cookie: u64, + ) -> Result<(), BlockError> { + let sqe = self.build_sqe(command, buffer, cookie)?; + if let Err(error) = self.qpair.submit_one(sqe, cookie, self.controller.regs()) { + if self.dsm_cookie == Some(cookie) { + self.dsm_cookie = None; + } + self.return_adopted_list(cookie); + return Err(match error { + crate::QpairError::Full => BlockError::Backpressure, + crate::QpairError::BadDepth => BlockError::Geometry, + }); + } + Ok(()) + } + + fn stage( + &mut self, + command: DeviceCommand, + buffer: Option<&mut Self::Buffer>, + cookie: u64, + ) -> Result<(), BlockError> { + let sqe = self.build_sqe(command, buffer, cookie)?; + if let Err(error) = self.qpair.submit(sqe, cookie) { + if self.dsm_cookie == Some(cookie) { + self.dsm_cookie = None; + } + self.return_adopted_list(cookie); + return Err(map_qpair_error(error)); + } + Ok(()) + } + + fn publish(&mut self) { + self.qpair.ring(self.controller.regs()); + } + + fn poll(&mut self) -> Option { + let completion = self.qpair.poll(self.controller.regs())?; + Some(self.device_completion(completion)) + } + + fn poll_batch(&mut self, out: &mut [Option]) -> usize { + const MAX_BATCH: usize = kore_driver::block::DEFAULT_OUTSTANDING; + let limit = out.len().min(MAX_BATCH); + let mut raw = [None; MAX_BATCH]; + let count = self + .qpair + .poll_batch(self.controller.regs(), &mut raw[..limit]); + for index in 0..count { + out[index] = raw[index] + .take() + .map(|completion| self.device_completion(completion)); + } + count + } + + fn quiesce(&mut self) -> Result<(), BlockError> { + if self.quiesced { + return Ok(()); + } + let stats = self.qpair.stats(); + if stats.submitted != stats.completed { + return Err(BlockError::Draining); + } + self.controller + .shutdown(self.shutdown_polls, self.lifecycle_pause) + .map_err(map_submit_error)?; + self.quiesced = true; + Ok(()) + } +} + +fn map_qpair_error(error: crate::QpairError) -> BlockError { + match error { + crate::QpairError::Full => BlockError::Backpressure, + crate::QpairError::BadDepth => BlockError::Geometry, + } +} + +fn map_submit_error(error: Error) -> BlockError { + match error { + Error::DmaAlloc | Error::Qpair(crate::QpairError::Full) => BlockError::Backpressure, + Error::NotReady | Error::Timeout => BlockError::Transport, + Error::Status { sct, .. } => map_status(sct), + Error::Qpair(crate::QpairError::BadDepth) | Error::Geometry => BlockError::Geometry, + Error::Prp(crate::prp::PrpError::ListNeeded | crate::prp::PrpError::TooLarge) + | Error::ExceedsMdts => BlockError::TooLarge, + Error::Prp(crate::prp::PrpError::NotEnoughPages) => BlockError::Integrity, + } +} + +fn map_status(status_code_type: u8) -> BlockError { + match status_code_type { + 0 => BlockError::Transport, + 2 => BlockError::Media, + 3 => BlockError::Integrity, + _ => BlockError::Transport, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::rc::Rc; + use core::cell::Cell; + + use crate::mock::{MOCK_BLOCK_SIZE, MockBuf, MockNvmeDevice, MockPool}; + use kore_driver::block::{BlockChannel, CompletionMode}; + + /// Production [`CapPool`](kore_driver::host::CapPool) geometry over the + /// mock's host-pointer DMA buffers: one 4 KiB-rounded bump cursor with a + /// fixed limit and stronger alignment charged as real skipped capacity. + struct BudgetPool { + inner: MockPool, + cursor: Rc>, + limit: usize, + } + + impl BudgetPool { + fn new(limit: usize) -> (Self, Rc>) { + let cursor = Rc::new(Cell::new(0)); + ( + Self { + inner: MockPool, + cursor: Rc::clone(&cursor), + limit, + }, + cursor, + ) + } + + fn allocate(&self, bytes: usize, alignment: usize) -> Option { + let alignment = alignment.max(4096); + if !alignment.is_power_of_two() { + return None; + } + let size = bytes.max(1).checked_add(4095)? & !4095; + let mask = alignment - 1; + let start = self.cursor.get().checked_add(mask)? & !mask; + let end = start.checked_add(size)?; + if end > self.limit { + return None; + } + let buffer = self.inner.alloc_aligned(bytes, alignment)?; + self.cursor.set(end); + Some(buffer) + } + } + + impl DmaPool for BudgetPool { + type Buf = MockBuf; + + fn alloc(&self, bytes: usize) -> Option { + self.allocate(bytes, 4096) + } + + fn alloc_aligned(&self, bytes: usize, alignment: usize) -> Option { + self.allocate(bytes, alignment) + } + } + + /// The storage transport engine over a three-queue group: a Demand + /// read, a Stream read and a Commit flush each ride their own hardware + /// submission queue (qids 1, 2, 3) and complete through the one shared + /// completion queue, the read bytes landing in the ring's target. + #[test] + fn the_engine_puts_each_class_on_its_own_hardware_queue() { + extern crate std; + use kore_abi::stor::Class; + use kore_driver::stor::engine::{StorEngine, StorHost, TargetAccess, ThreeClass}; + use kore_driver::stor::grant::{ClassSet, ExtentRun, Grant, Rights}; + use kore_ring::stor::{Op, Request, StorClient, WINDOW_BYTES, split}; + + struct Quiet; + impl StorHost for Quiet { + fn kick(&mut self, _: kore_driver::stor::engine::RingId) {} + fn drained(&mut self, _: kore_driver::stor::engine::RingId, _: kore_abi::stor::GrantId) {} + fn departed(&mut self, _: kore_driver::stor::engine::RingId) {} + fn expired(&mut self, _: kore_driver::stor::engine::Expiry) {} + } + + let dev = MockNvmeDevice::new(); + let ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + let mut pacer = kore_driver::block::CountedPacer::new(16, || {}); + let backend = NvmeBlockDevice::<_, _, 3>::provision_queues_paced( + ctrl, 1, 8, None, 0, &mut pacer, || {}, 16, + ) + .unwrap(); + let layout = std::alloc::Layout::from_size_align(WINDOW_BYTES, 4096).unwrap(); + // SAFETY: non-zero size; leaked for a 'static window. + let window: &'static mut [u8] = unsafe { + core::slice::from_raw_parts_mut(std::alloc::alloc_zeroed(layout), WINDOW_BYTES) + }; + let (client, service) = split(window).unwrap(); + let mut engine: std::boxed::Box> = + std::boxed::Box::new(StorEngine::new(backend).ok().expect("three queues")); + let ring = engine.attach(service, ClassSet::ALL).ok().unwrap(); + let grant = engine + .install( + ring, + Grant::new( + &[ExtentRun { lba: 0, blocks: 16 }], + Rights::READ | Rights::FLUSH, + ClassSet::ALL, + ) + .unwrap(), + ) + .unwrap(); + let target = { + let queues = engine.queues_mut(); + kore_driver::stor::QueueSet::allocate(queues, 2 * MOCK_BLOCK_SIZE as usize).unwrap() + }; + let t = engine.adopt_target(ring, target, TargetAccess::ReadInto).ok().unwrap(); + let mut client = StorClient::new(client); + let request = |op, offset, target_offset, user_data| Request { + op, + grant, + offset, + len: if op == Op::Flush { 0 } else { MOCK_BLOCK_SIZE }, + target: t, + target_offset, + user_data, + timeline: 0, + deadline_ns: 0, + }; + client + .submit(Class::Demand, &request(Op::Read, 0, 0, 1)) + .unwrap(); + client + .submit( + Class::Stream, + &request(Op::Read, u64::from(MOCK_BLOCK_SIZE), u64::from(MOCK_BLOCK_SIZE), 2), + ) + .unwrap(); + client.submit(Class::Commit, &request(Op::Flush, 0, 0, 3)).unwrap(); + client.submit(Class::Commit, &request(Op::Flush, 0, 0, 4)).unwrap(); + for class in Class::ALL { + client.publish(class); + } + engine.advance(&mut Quiet); + engine.advance(&mut Quiet); + let mut done = alloc::vec::Vec::new(); + while let Some(completion) = client.reap() { + done.push((completion.user_data, completion.result)); + } + done.sort_unstable_by_key(|(user, _)| *user); + assert_eq!( + done, + [ + (1, Ok(MOCK_BLOCK_SIZE)), + (2, Ok(MOCK_BLOCK_SIZE)), + (3, Ok(0)), + (4, Ok(0)) + ] + ); + // Each class on its own hardware queue, in priority order: Demand + // (urgent), Commit (high), Stream (medium). + assert_eq!(dev.sq_consumed(1), Some(1), "Demand on SQ 1"); + assert_eq!(dev.sq_consumed(2), Some(2), "both Commit flushes on SQ 2"); + assert_eq!(dev.sq_consumed(3), Some(1), "Stream on SQ 3"); + let target = engine.release_target(ring, t).unwrap().into_inner(); + assert_eq!(&target.bytes()[..MOCK_BLOCK_SIZE as usize], &dev.disk_snapshot(0, 1)[..]); + assert_eq!(&target.bytes()[MOCK_BLOCK_SIZE as usize..], &dev.disk_snapshot(1, 1)[..]); + } + + /// A three-queue group: one submission queue per class, all bound to + /// one completion queue; commands staged on each queue complete through + /// that one queue, each with its own cookie and its own bytes. + /// A command the device holds on one queue of a group is aborted by + /// its cookie: the Abort names that queue and the command's id, and the + /// command's own completion comes back failed. A cookie nothing holds + /// is not aborted. + #[test] + fn a_held_command_on_a_group_queue_is_aborted_by_its_cookie() { + let dev = MockNvmeDevice::new(); + let ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + let mut pacer = kore_driver::block::CountedPacer::new(16, || {}); + let mut backend = NvmeBlockDevice::<_, _, 3>::provision_queues_paced( + ctrl, 1, 8, None, 0, &mut pacer, || {}, 16, + ) + .unwrap(); + let mut buffer = backend.allocate_buffer(MOCK_BLOCK_SIZE as usize).unwrap(); + dev.swallow_completions(1); + backend + .stage_on( + 1, + DeviceCommand::Read { + lba: 2, + blocks: 1, + buffer_offset: 0, + }, + Some(&mut buffer), + 42, + ) + .unwrap(); + backend.publish_on(1); + assert!(backend.poll().is_none(), "the device holds it"); + + assert!(!backend.abort_on(1, 43), "nothing carries that cookie"); + assert!(backend.abort_on(1, 42)); + let aborts = dev.aborts(); + assert_eq!(aborts.len(), 1); + assert_eq!((aborts[0].0, aborts[0].2), (2, true), "queue 1 of the group is qid 2"); + let completion = backend.poll().expect("the victim posts its own completion"); + assert_eq!(completion.cookie, 42); + assert!(completion.result.is_err(), "failed, not delivered"); + assert!(backend.poll().is_none()); + } + + /// A reset drops the command the device held, and brings the whole + /// queue group back over the same memory: every submission queue bound + /// to its completion queue again, and the next read served. + #[test] + fn a_reset_loses_what_the_device_held_and_brings_every_queue_back() { + let dev = MockNvmeDevice::new(); + let ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + let mut pacer = kore_driver::block::CountedPacer::new(16, || {}); + let mut backend = NvmeBlockDevice::<_, _, 3>::provision_queues_paced( + ctrl, 1, 8, None, 0, &mut pacer, || {}, 16, + ) + .unwrap(); + let mut buffer = backend.allocate_buffer(MOCK_BLOCK_SIZE as usize).unwrap(); + dev.swallow_completions(1); + backend + .stage_on( + 1, + DeviceCommand::Read { + lba: 2, + blocks: 1, + buffer_offset: 0, + }, + Some(&mut buffer), + 42, + ) + .unwrap(); + backend.publish_on(1); + assert!(backend.poll().is_none(), "the device holds it"); + + assert_eq!(backend.reset_paced(&mut pacer), Ok(1), "one command lost"); + assert_eq!(dev.resets(), 1); + assert_eq!(backend.resets(), 1); + assert!(backend.poll().is_none(), "the held completion never posts"); + for qid in 1..=3 { + assert_eq!(dev.sq_binding(qid), Some(1), "SQ {qid} is bound to CQ 1 again"); + } + backend + .stage_on( + 2, + DeviceCommand::Read { + lba: 3, + blocks: 1, + buffer_offset: 0, + }, + Some(&mut buffer), + 43, + ) + .unwrap(); + backend.publish_on(2); + let completion = backend.poll().expect("the recreated queue serves"); + assert_eq!(completion.cookie, 43); + assert!(completion.result.is_ok()); + assert_eq!( + &buffer.bytes()[..MOCK_BLOCK_SIZE as usize], + &dev.disk_snapshot(3, 1)[..] + ); + } + + #[test] + fn a_three_queue_group_shares_one_completion_queue() { + let dev = MockNvmeDevice::new(); + let ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + let mut pacer = kore_driver::block::CountedPacer::new(16, || {}); + let mut backend = NvmeBlockDevice::<_, _, 3>::provision_queues_paced( + ctrl, 1, 8, None, 0, &mut pacer, || {}, 16, + ) + .unwrap(); + assert_eq!(backend.live_queues(), 3); + for qid in 1..=3 { + assert_eq!(dev.sq_binding(qid), Some(1), "SQ {qid} posts to CQ 1"); + } + let mut buffers: alloc::vec::Vec<_> = (0..3) + .map(|_| backend.allocate_buffer(MOCK_BLOCK_SIZE as usize).unwrap()) + .collect(); + for (sq, buffer) in buffers.iter_mut().enumerate() { + backend + .stage_on( + sq, + DeviceCommand::Read { + lba: sq as u64, + blocks: 1, + buffer_offset: 0, + }, + Some(buffer), + 100 + sq as u64, + ) + .unwrap(); + backend.publish_on(sq); + } + let mut cookies = alloc::vec::Vec::new(); + while let Some(completion) = backend.poll() { + assert_eq!(completion.result, Ok(())); + cookies.push(completion.cookie); + } + cookies.sort_unstable(); + assert_eq!(cookies, [100, 101, 102]); + for (sq, buffer) in buffers.iter().enumerate() { + assert_eq!(buffer.bytes(), &dev.disk_snapshot(sq as u64, 1)[..]); + } + assert_eq!(backend.qpair_stats().stray, 0); + } + + #[test] + fn hardware_neutral_channel_drives_the_real_nvme_core() { + let dev = MockNvmeDevice::new(); + let ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + let mut backend = NvmeBlockDevice::provision(ctrl, 1, 8, Some(0), 0, 16, || {}).unwrap(); + assert!(backend.geometry().discard); + let health = backend.health().expect("typed SMART health sample"); + assert_eq!(health.temperature_celsius, 36); + assert_eq!(health.percentage_used, 3); + assert_eq!(health.power_cycles, 7); + assert_eq!(health.media_errors, 0); + // A second sample reuses the backend's dedicated DMA buffer. + assert_eq!(backend.health().unwrap(), health); + let mut channel: BlockChannel<_, 3, 8> = + BlockChannel::new(backend, CompletionMode::Interrupt).unwrap(); + assert_eq!(channel.mode(), CompletionMode::Interrupt); + let a = channel.register_buffer(MOCK_BLOCK_SIZE as usize).unwrap(); + let b = channel.register_buffer(MOCK_BLOCK_SIZE as usize).unwrap(); + + // A failed later stage cannot leave the accepted prefix invisible: + // dropping the affine batch publishes exactly that prefix. + let before = dev.io_sq_doorbells(); + let staged = { + let mut batch = channel.batch(); + let staged = batch.stage_read(0, 1, a, 0).unwrap(); + assert_eq!(batch.stage_read(1, 1, a, 0), Err(BlockError::BufferBusy)); + staged + }; + assert_eq!(dev.io_sq_doorbells() - before, 1); + assert_eq!(channel.poll().unwrap().request, staged); + + channel + .write_buffer(a, 0, &[0x5a; MOCK_BLOCK_SIZE as usize]) + .unwrap(); + + // Two distinct buffers permit two outstanding commands on one owned + // qpair; cookies return to the right request. + let before = dev.io_sq_doorbells(); + let (write, read) = { + let mut batch = channel.batch(); + let write = batch.stage_write(20, 1, a, 0).unwrap(); + let read = batch.stage_read(3, 1, b, 0).unwrap(); + batch.publish(); + (write, read) + }; + assert_eq!(dev.io_sq_doorbells() - before, 1, "one SQ doorbell"); + assert_eq!(channel.outstanding(), 2); + let cq_before = dev.io_cq_doorbells(); + let [Some(c1), Some(c2)] = channel.poll_batch::<2>() else { + panic!("both mock completions are ready") + }; + assert_eq!(dev.io_cq_doorbells() - cq_before, 1, "one CQ doorbell"); + assert_eq!([c1.request, c2.request], [write, read]); + assert!(c1.result.is_ok() && c2.result.is_ok()); + assert!(dev.disk_snapshot(20, 1).iter().all(|b| *b == 0x5a)); + + let flush = channel.submit_flush().unwrap(); + assert_eq!(channel.poll().unwrap().request, flush); + let discard = channel.submit_discard(20, 1).unwrap(); + assert_eq!(channel.poll().unwrap().request, discard); + assert!(dev.disk_snapshot(20, 1).iter().all(|b| *b == 0)); + channel.quiesce().unwrap(); + } + + #[test] + fn larger_controller_page_aligns_data_and_prp_list_buffers() { + let dev = MockNvmeDevice::new().with_mps(1).unwrap(); + let ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + let mut backend = NvmeBlockDevice::provision(ctrl, 1, 8, None, 0, 16, || {}).unwrap(); + let page_bytes = 2 * MOCK_BLOCK_SIZE as usize; + let mut buffer = backend.allocate_buffer(2 * page_bytes).unwrap(); + assert_eq!(buffer.data.iova().0 & 0x1fff, 0); + assert!(buffer.list.is_none(), "two pages fit directly in PRP1/PRP2"); + let listed = backend.allocate_buffer(2 * page_bytes + 1).unwrap(); + assert_eq!( + listed.list.as_ref().unwrap().iova().0 & 0x1fff, + 0, + "larger buffers own an aligned private PRP list" + ); + + backend + .submit( + DeviceCommand::Read { + lba: 3, + blocks: 4, + buffer_offset: 0, + }, + Some(&mut buffer), + 7, + ) + .unwrap(); + let completion = backend.poll().unwrap(); + assert_eq!(completion.cookie, 7); + assert_eq!(completion.result, Ok(())); + assert_eq!(buffer.bytes(), dev.disk_snapshot(3, 4)); + } + + /// A caller-funded buffer adopted as a streaming target takes several + /// list-sized reads in flight at once: each borrows a shared list page for + /// exactly its flight, every byte lands where the disk says, and the pool + /// is whole again once the queue drains. + #[test] + fn an_adopted_target_streams_list_sized_reads_concurrently() { + let dev = MockNvmeDevice::new(); + let ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + let mut backend = NvmeBlockDevice::provision(ctrl, 1, 8, None, 0, 16, || {}).unwrap(); + // 64 blocks of "client" memory: a mapping this row did not allocate + // from its channel budget. + let funded = dev + .pool() + .alloc_aligned(64 * MOCK_BLOCK_SIZE as usize, 4096) + .unwrap(); + let target = backend + .adopt_buffer(funded) + .map_err(|(_, error)| error) + .unwrap(); + assert!(target.list.is_none(), "no private list page"); + let mut channel: BlockChannel<_, 2, 8> = + BlockChannel::new(backend, CompletionMode::Poll).unwrap(); + let id = channel + .adopt_target(target) + .map_err(|(_, error)| error) + .unwrap(); + + // Three 8-block (eight-page) reads need a PRP list each. + let reads = [(40u64, 0usize), (8, 16), (20, 48)]; + let mut ids = [kore_driver::block::RequestId(0); 3]; + { + let mut batch = channel.batch(); + for ((lba, block), slot) in reads.iter().zip(ids.iter_mut()) { + *slot = batch + .stage_read(*lba, 8, id, block * MOCK_BLOCK_SIZE as usize) + .unwrap(); + } + batch.publish(); + } + assert_eq!(channel.buffer_inflight(id), Some(3)); + assert_eq!( + channel.control(|d| d.adopted_lists_in_flight()), + Err(BlockError::BufferBusy), + "control is refused while data is in flight" + ); + let mut done = 0; + while let Some(completion) = channel.poll() { + assert!(ids.contains(&completion.request)); + completion.result.unwrap(); + done += 1; + } + assert_eq!(done, 3); + assert_eq!(channel.control(|d| d.adopted_lists_in_flight()), Ok(0)); + // The channel never reads a target through the CPU; its owner does, + // once the mapping has come back. + let buffer = channel.release_buffer(id).unwrap(); + for (lba, block) in reads { + let at = block * MOCK_BLOCK_SIZE as usize; + let landed = &kore_driver::block::BlockBuffer::bytes(&buffer) + [at..at + 8 * MOCK_BLOCK_SIZE as usize]; + assert_eq!(landed, &dev.disk_snapshot(lba, 8)[..], "lba {lba}"); + } + // The mapping returns to its owner for quiescent release. + let data = buffer.into_data(); + assert_eq!(data.len(), 64 * MOCK_BLOCK_SIZE as usize); + } + + /// A DRAM-less controller is handed the least of its appetite and the + /// consumer's allowance, as one descriptor; a controller that uses none, + /// or an allowance under its floor, gets nothing and the mount proceeds. + #[test] + fn a_dram_less_controller_is_given_a_host_memory_buffer_of_one_descriptor() { + // 64 KiB preferred, 16 KiB minimum, four descriptors at most. + let dev = MockNvmeDevice::new(); + dev.set_hmb(16, 4, 4); + let ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + let backend = NvmeBlockDevice::provision(ctrl, 1, 8, None, 1 << 20, 16, || {}).unwrap(); + assert_eq!( + dev.hmb(), + Some((16, 1)), + "64 KiB = 16 pages, one descriptor" + ); + assert_eq!(backend.host_memory_buffer_bytes(), 64 * 1024); + + // The allowance caps it, in whole pages. + let dev = MockNvmeDevice::new(); + dev.set_hmb(16, 4, 4); + let ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + let backend = + NvmeBlockDevice::provision(ctrl, 1, 8, None, 32 * 1024 + 100, 16, || {}).unwrap(); + assert_eq!(dev.hmb(), Some((8, 1))); + assert_eq!(backend.host_memory_buffer_bytes(), 32 * 1024); + + // Below the controller's minimum: not offered. + let dev = MockNvmeDevice::new(); + dev.set_hmb(16, 4, 4); + let ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + let backend = NvmeBlockDevice::provision(ctrl, 1, 8, None, 8 * 1024, 16, || {}).unwrap(); + assert_eq!(dev.hmb(), None); + assert_eq!(backend.host_memory_buffer_bytes(), 0); + + // A controller with no appetite gets none whatever the allowance. + let dev = MockNvmeDevice::new(); + let ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + let backend = NvmeBlockDevice::provision(ctrl, 1, 8, None, 1 << 30, 16, || {}).unwrap(); + assert_eq!(dev.hmb(), None); + assert_eq!(backend.host_memory_buffer_bytes(), 0); + } + + /// The buffer is disabled at the controller before the orderly + /// shutdown, through the same quiesce the filesystem takes. + #[test] + fn quiesce_disables_the_host_memory_buffer_before_shutdown() { + let dev = MockNvmeDevice::new(); + dev.set_hmb(16, 4, 4); + let ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + let mut backend = NvmeBlockDevice::provision(ctrl, 1, 8, None, 1 << 20, 16, || {}).unwrap(); + assert!(dev.hmb().is_some()); + backend.quiesce().unwrap(); + assert_eq!(dev.hmb(), None); + assert_eq!(dev.hmb_disables(), 1); + assert_eq!(backend.host_memory_buffer_bytes(), 0); + } + + /// A pool that cannot carry the buffer leaves the controller without + /// one and the filesystem's own budget untouched: provisioning still + /// succeeds. + #[test] + fn a_pool_too_small_for_the_buffer_still_provisions_the_device() { + let dev = MockNvmeDevice::new(); + dev.set_hmb(16_384, 1_024, 8); // 64 MiB preferred, 4 MiB minimum + let (pool, _cursor) = BudgetPool::new(2 << 20); + let ctrl = Controller::bring_up(dev.clone(), pool, 16, || {}).unwrap(); + let backend = NvmeBlockDevice::provision(ctrl, 1, 8, None, 64 << 20, 16, || {}).unwrap(); + assert_eq!( + dev.hmb(), + None, + "2 MiB cannot carry 4 MiB, so none was offered" + ); + assert_eq!(backend.host_memory_buffer_bytes(), 0); + } + + #[test] + fn mps_256k_and_four_fs_buffers_fit_the_exact_2mib_budget() { + const DMA_BUDGET: usize = 2 * 1024 * 1024; + const FS_BUFFER: usize = 256 * 1024; + + let dev = MockNvmeDevice::new().with_mps(6).unwrap(); + let (pool, cursor) = BudgetPool::new(DMA_BUDGET); + let ctrl = Controller::bring_up(dev, pool, 16, || {}).unwrap(); + let mut backend = NvmeBlockDevice::provision(ctrl, 1, 32, None, 0, 16, || {}).unwrap(); + let buffers = [ + backend.allocate_buffer(FS_BUFFER).unwrap(), + backend.allocate_buffer(FS_BUFFER).unwrap(), + backend.allocate_buffer(FS_BUFFER).unwrap(), + backend.allocate_buffer(FS_BUFFER).unwrap(), + ]; + + assert!(buffers.iter().all(|buffer| buffer.list.is_none())); + assert_eq!(cursor.get(), DMA_BUDGET, "the fourth buffer ends at 2 MiB"); + } +} diff --git a/native/drivers/nvme/src/commands.rs b/native/drivers/nvme/src/commands.rs new file mode 100644 index 0000000..16cc42c --- /dev/null +++ b/native/drivers/nvme/src/commands.rs @@ -0,0 +1,946 @@ +//! NVMe command + completion + identify structures — **spec facts** (NVMe +//! base spec §4, §5), ported from `os0/user/dev/block/nvme/commands.rs`. +//! The wire layouts are fixed by the spec; the `size_of` asserts guard +//! them. + +use core::mem; + +use kore_abi::device::IovaAddr; + +use crate::prp::Prp; + +/// Admin command opcodes (spec §5). +#[repr(u8)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AdminOpcode { + /// Delete I/O Submission Queue. + DeleteIoSq = 0x00, + /// Create I/O Submission Queue. + CreateIoSq = 0x01, + /// Get Log Page. + GetLogPage = 0x02, + /// Delete I/O Completion Queue. + DeleteIoCq = 0x04, + /// Create I/O Completion Queue. + CreateIoCq = 0x05, + /// Identify. + Identify = 0x06, + /// Abort. + Abort = 0x08, + /// Set Features. + SetFeatures = 0x09, +} + +/// A submission queue's arbitration priority (Create I/O Submission Queue +/// CDW11 `QPRIO`, bits 2:1; NVMe base spec, Figure "Create I/O Submission +/// Queue — Command Dword 11"). +/// +/// The controller honours it only under weighted round robin with urgent +/// priority class (`CC.AMS = 001b`, offered when `CAP.AMS` bit 17 is set); +/// under plain round robin the field is ignored, which is why every queue +/// the core created before this type existed could leave it zero (urgent). +#[repr(u8)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SqPriority { + /// Urgent — served ahead of every weighted class. + Urgent = 0b00, + /// High weight class. + High = 0b01, + /// Medium weight class. + Medium = 0b10, + /// Low weight class. + Low = 0b11, +} + +/// Arbitration Burst (Set Features FID 01h, CDW11 bits 2:0): the most +/// commands the controller may fetch from one submission queue per +/// arbitration round, as a power of two, or no limit. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ArbitrationBurst { + /// `2^n` commands per round; `n` must be at most 6 (`111b` is reserved + /// for "no limit"). + Pow2(u8), + /// No limit (`111b`). + Unlimited, +} + +impl ArbitrationBurst { + /// The field's encoding, or `None` for an exponent the field cannot + /// carry (7 and above would alias "no limit"). + pub const fn encode(self) -> Option { + match self { + Self::Pow2(n) if n <= 6 => Some(n as u32), + Self::Pow2(_) => None, + Self::Unlimited => Some(0b111), + } + } +} + +/// The Arbitration feature (FID 01h): the burst, and the three weighted +/// classes' weights. Weights are **0-based** on the wire (a weight of 0 +/// means one command), as the spec defines them; they matter only under +/// weighted round robin. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Arbitration { + /// Commands fetched from one queue per round. + pub burst: ArbitrationBurst, + /// High Priority Weight (0-based). + pub high: u8, + /// Medium Priority Weight (0-based). + pub medium: u8, + /// Low Priority Weight (0-based). + pub low: u8, +} + +/// NVM I/O command opcodes (spec §6). +#[repr(u8)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum IoOpcode { + /// Flush. + Flush = 0x00, + /// Write. + Write = 0x01, + /// Read. + Read = 0x02, + /// Dataset Management (deallocate/TRIM — the swap `Discard` backend). + DatasetManagement = 0x09, +} + +/// A 64-byte NVMe submission queue entry (spec Figure — SQE). +#[repr(C)] +#[derive(Clone, Copy)] +pub struct NvmeSqe { + /// CDW0: opcode / fused / PSDT / CID. + pub cdw0: u32, + /// Namespace ID. + pub nsid: u32, + pub cdw2: u32, + pub cdw3: u32, + /// Metadata pointer. + pub mptr: u64, + /// PRP entry 1. + pub prp1: u64, + /// PRP entry 2 (or PRP-list pointer). + pub prp2: u64, + pub cdw10: u32, + pub cdw11: u32, + pub cdw12: u32, + pub cdw13: u32, + pub cdw14: u32, + pub cdw15: u32, +} + +const _: () = assert!(mem::size_of::() == 64); + +impl NvmeSqe { + /// A zeroed command. + pub const fn zeroed() -> Self { + Self { + cdw0: 0, + nsid: 0, + cdw2: 0, + cdw3: 0, + mptr: 0, + prp1: 0, + prp2: 0, + cdw10: 0, + cdw11: 0, + cdw12: 0, + cdw13: 0, + cdw14: 0, + cdw15: 0, + } + } + + /// Stamp the command id into CDW0 (bits 31:16). + pub fn set_cid(&mut self, cid: u16) { + self.cdw0 = (self.cdw0 & 0x0000_FFFF) | ((cid as u32) << 16); + } + + /// The command id. + pub fn cid(&self) -> u16 { + (self.cdw0 >> 16) as u16 + } + + /// The opcode (CDW0 bits 7:0). + pub fn opcode(&self) -> u8 { + (self.cdw0 & 0xFF) as u8 + } + + // ── Admin commands ────────────────────────────────────────────────── + // Builders speak typed [`IovaAddr`]; the raw `u64` exists only in the + // wire fields they fill (the one place the spec fixes the layout). + + /// Identify Controller (CNS = 1). + pub fn identify_controller(dma: IovaAddr) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = AdminOpcode::Identify as u32; + c.prp1 = dma.0; + c.cdw10 = 1; + c + } + + /// Identify Namespace (CNS = 0). + pub fn identify_namespace(nsid: u32, dma: IovaAddr) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = AdminOpcode::Identify as u32; + c.nsid = nsid; + c.prp1 = dma.0; + c.cdw10 = 0; + c + } + + /// Set Features — Number of Queues (FID 0x07). `nsq`/`ncq` are + /// 0-based (pass 3 to request 4). + pub fn set_num_queues(nsq: u16, ncq: u16) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = AdminOpcode::SetFeatures as u32; + c.cdw10 = 0x07; + c.cdw11 = ((ncq as u32) << 16) | (nsq as u32); + c + } + + /// Set Features — Host Memory Buffer (FID 0x0D), enabling it. + /// + /// `hsize_pages` is the buffer's total size and `list` the IOVA of the + /// [`HmbDescriptor`] table with `entries` rows; both sizes are in + /// **`CC.MPS` units**, not the 4 KiB units Identify reports in. The + /// table's address must be 16-byte aligned (CDW13 bits 3:0 are + /// reserved). Memory Return (CDW11 bit 1) is never set: this core + /// allocates a fresh buffer every bring-up and hands nothing back. + pub fn enable_host_memory_buffer(hsize_pages: u32, list: IovaAddr, entries: u32) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = AdminOpcode::SetFeatures as u32; + c.cdw10 = 0x0D; + c.cdw11 = 1; // EHM + c.cdw12 = hsize_pages; + c.cdw13 = list.0 as u32; + c.cdw14 = (list.0 >> 32) as u32; + c.cdw15 = entries; + c + } + + /// Set Features — Host Memory Buffer (FID 0x0D), disabling it: the + /// controller stops using the buffer, after which its memory may be + /// released. Every other field is zero, as the spec asks for a disable. + pub fn disable_host_memory_buffer() -> Self { + let mut c = Self::zeroed(); + c.cdw0 = AdminOpcode::SetFeatures as u32; + c.cdw10 = 0x0D; + c + } + + /// Create I/O Completion Queue (physically contiguous). + pub fn create_io_cq(qid: u16, size: u16, dma: IovaAddr, vector: u16, ien: bool) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = AdminOpcode::CreateIoCq as u32; + c.prp1 = dma.0; + c.cdw10 = (((size - 1) as u32) << 16) | (qid as u32); + c.cdw11 = ((vector as u32) << 16) | (if ien { 2 } else { 0 }) | 1; + c + } + + /// Create I/O Submission Queue (physically contiguous, bound to `cqid`), + /// at [`SqPriority::Urgent`] — the encoding every queue carried before + /// priorities existed, and the one plain round robin ignores. + pub fn create_io_sq(qid: u16, size: u16, dma: IovaAddr, cqid: u16) -> Self { + Self::create_io_sq_prio(qid, size, dma, cqid, SqPriority::Urgent) + } + + /// Create I/O Submission Queue (physically contiguous) bound to `cqid` + /// at arbitration priority `prio` (CDW11 bits 2:1). Several submission + /// queues may name one `cqid`; the completion queue must exist first + /// and be deep enough for every queue bound to it. + pub fn create_io_sq_prio( + qid: u16, + size: u16, + dma: IovaAddr, + cqid: u16, + prio: SqPriority, + ) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = AdminOpcode::CreateIoSq as u32; + c.prp1 = dma.0; + c.cdw10 = (((size - 1) as u32) << 16) | (qid as u32); + c.cdw11 = ((cqid as u32) << 16) | ((prio as u32) << 1) | 1; + c + } + + /// Abort (admin opcode 08h): ask the controller to abort command `cid` + /// on submission queue `sqid` (CDW10: CID in bits 31:16, SQID in 15:0). + /// + /// Best effort by specification: completion DW0 bit 0 is **clear** when + /// the command was aborted and **set** when it was not, and the aborted + /// command still posts its own completion (status *Command Abort + /// Requested*). A caller may reuse the command's buffer only after that + /// completion, never after the Abort's. + pub fn abort(sqid: u16, cid: u16) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = AdminOpcode::Abort as u32; + c.cdw10 = ((cid as u32) << 16) | (sqid as u32); + c + } + + /// Set Features — Arbitration (FID 01h), or `None` when the burst + /// exponent does not fit its three-bit field. + /// + /// CDW11: High Priority Weight 31:24, Medium 23:16, Low 15:8, Arbitration + /// Burst 2:0. + pub fn set_arbitration(arbitration: Arbitration) -> Option { + let burst = arbitration.burst.encode()?; + let mut c = Self::zeroed(); + c.cdw0 = AdminOpcode::SetFeatures as u32; + c.cdw10 = 0x01; + c.cdw11 = ((arbitration.high as u32) << 24) + | ((arbitration.medium as u32) << 16) + | ((arbitration.low as u32) << 8) + | burst; + Some(c) + } + + /// Delete I/O Submission Queue. + pub fn delete_io_sq(qid: u16) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = AdminOpcode::DeleteIoSq as u32; + c.cdw10 = qid as u32; + c + } + + /// Delete I/O Completion Queue. + pub fn delete_io_cq(qid: u16) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = AdminOpcode::DeleteIoCq as u32; + c.cdw10 = qid as u32; + c + } + + // ── I/O commands ──────────────────────────────────────────────────── + + /// Read `nlb + 1` logical blocks from `lba` (NLB is 0-based). + pub fn read(nsid: u32, lba: u64, nlb: u16, prp: Prp) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = IoOpcode::Read as u32; + c.nsid = nsid; + c.prp1 = prp.prp1.0; + c.prp2 = prp.prp2.0; + c.cdw10 = lba as u32; + c.cdw11 = (lba >> 32) as u32; + c.cdw12 = nlb as u32; + c + } + + /// Write `nlb + 1` logical blocks to `lba` (NLB is 0-based). + pub fn write(nsid: u32, lba: u64, nlb: u16, prp: Prp) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = IoOpcode::Write as u32; + c.nsid = nsid; + c.prp1 = prp.prp1.0; + c.prp2 = prp.prp2.0; + c.cdw10 = lba as u32; + c.cdw11 = (lba >> 32) as u32; + c.cdw12 = nlb as u32; + c + } + + /// Flush the namespace. + pub fn flush(nsid: u32) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = IoOpcode::Flush as u32; + c.nsid = nsid; + c + } + + /// Dataset Management — **deallocate** (`AD` set) the `nr + 1` ranges + /// in the [`DsmRange`] table at `ranges` (0-based count, ≤ 255). The + /// discard path a swap backend rides. + pub fn dsm_deallocate(nsid: u32, nr: u8, ranges: IovaAddr) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = IoOpcode::DatasetManagement as u32; + c.nsid = nsid; + c.prp1 = ranges.0; + c.cdw10 = nr as u32; + c.cdw11 = 1 << 2; // AD — Attribute: Deallocate + c + } + + /// Get Log Page — SMART / Health Information (LID 0x02), the full + /// 512-byte page into `dma`. `NUMD` is in dwords, 0-based. + pub fn get_log_smart(dma: IovaAddr) -> Self { + let mut c = Self::zeroed(); + c.cdw0 = AdminOpcode::GetLogPage as u32; + c.nsid = 0xFFFF_FFFF; // controller-scoped + c.prp1 = dma.0; + let numd = (512 / 4) - 1u32; // 0-based dword count + c.cdw10 = (numd << 16) | 0x02; + c + } +} + +/// One Dataset Management range descriptor (16 bytes, spec Figure). +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +pub struct DsmRange { + /// Context attributes (0 for a plain deallocate). + pub cattr: u32, + /// Length in logical blocks (1-based). + pub nlb: u32, + /// Starting LBA. + pub slba: u64, +} + +const _: () = assert!(mem::size_of::() == 16); + +/// One Host Memory Buffer descriptor (16 bytes, spec Figure): a memory +/// page aligned chunk the controller may read and write for as long as the +/// buffer is enabled. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +pub struct HmbDescriptor { + /// Buffer address — device-visible, `CC.MPS`-aligned. + pub badd: u64, + /// Buffer size in `CC.MPS` units. + pub bsize: u32, + _reserved: u32, +} + +const _: () = assert!(mem::size_of::() == 16); + +impl HmbDescriptor { + /// A chunk of `pages` memory pages at `badd`. + pub const fn new(badd: IovaAddr, pages: u32) -> Self { + Self { + badd: badd.0, + bsize: pages, + _reserved: 0, + } + } +} + +impl DsmRange { + /// A deallocate range: `count` blocks from `lba`. + pub const fn new(lba: u64, count: u32) -> Self { + Self { + cattr: 0, + nlb: count, + slba: lba, + } + } +} + +/// SMART / Health Information log (LID 0x02, 512 bytes, spec Figure). +/// Only the fields the driver reads are named; offsets are asserted. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct SmartLog { + /// Critical warning bits. + pub critical_warning: u8, + /// Composite temperature, Kelvin — **unaligned** at offset 1 (the + /// spec's layout), so byte-array + [`temp_kelvin`](Self::temp_kelvin). + pub composite_temp: [u8; 2], + /// Available spare (%). + pub avail_spare: u8, + /// Available spare threshold (%). + pub spare_threshold: u8, + /// Percentage of rated device life used. + pub percentage_used: u8, + _pad6: [u8; 26], + /// Data units read (thousands of 512-byte units, 128-bit LE). + pub data_units_read: [u8; 16], + /// Data units written. + pub data_units_written: [u8; 16], + /// Host read/write commands + controller busy time (unread here). + _pad64: [u8; 48], + /// Power cycles (128-bit LE). + pub power_cycles: [u8; 16], + /// Power-on hours (128-bit LE). + pub power_on_hours: [u8; 16], + /// Unsafe shutdowns (128-bit LE). + pub unsafe_shutdowns: [u8; 16], + /// Media and data-integrity errors (128-bit LE). + pub media_errors: [u8; 16], + _tail: [u8; 336], +} + +const _: () = assert!(mem::size_of::() == 512); +const _: () = assert!(mem::offset_of!(SmartLog, composite_temp) == 1); +const _: () = assert!(mem::offset_of!(SmartLog, percentage_used) == 5); +const _: () = assert!(mem::offset_of!(SmartLog, data_units_read) == 32); +const _: () = assert!(mem::offset_of!(SmartLog, power_cycles) == 112); + +impl SmartLog { + /// Composite temperature, Kelvin. + pub fn temp_kelvin(&self) -> u16 { + u16::from_le_bytes(self.composite_temp) + } + + /// Composite temperature in °C (the log stores Kelvin). + pub fn temp_celsius(&self) -> i32 { + self.temp_kelvin() as i32 - 273 + } + + /// The low 64 bits of a 128-bit LE counter field. + pub fn counter(field: &[u8; 16]) -> u64 { + u64::from_le_bytes(field[..8].try_into().unwrap()) + } +} + +/// A 16-byte NVMe completion queue entry. +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct NvmeCqe { + /// Command-specific result (DW0). + pub result: u32, + pub rsvd: u32, + /// SQ head pointer. + pub sq_head: u16, + /// SQ identifier. + pub sq_id: u16, + /// Command identifier. + pub cid: u16, + /// Status field (P, SC, SCT, …). + pub status: u16, +} + +const _: () = assert!(mem::size_of::() == 16); + +impl NvmeCqe { + /// The phase (P) bit — toggles each time the CQ wraps. + #[inline] + pub fn phase(&self) -> bool { + self.status & 1 != 0 + } + + /// Whether the command succeeded (SC = SCT = 0). + #[inline] + pub fn is_success(&self) -> bool { + (self.status >> 1) & 0x3FFF == 0 + } + + /// Status Code. + #[inline] + pub fn status_code(&self) -> u8 { + ((self.status >> 1) & 0xFF) as u8 + } + + /// Status Code Type. + #[inline] + pub fn status_code_type(&self) -> u8 { + ((self.status >> 9) & 0x7) as u8 + } +} + +/// Identify Controller data (4096 B, spec Figure). Only the fields the +/// core reads are named; the rest is a single reserved tail, keeping the +/// struct exact (the size assert guards the layout). +#[repr(C)] +#[derive(Clone, Copy)] +pub struct IdentifyController { + /// PCI vendor id. + pub vid: u16, + /// PCI subsystem vendor id. + pub ssvid: u16, + /// Serial number. + pub sn: [u8; 20], + /// Model number. + pub mn: [u8; 40], + /// Firmware revision. + pub fr: [u8; 8], + /// rab + ieee[3] + cmic — 5 bytes to `mdts` at offset 77. + _pad_rab_to_mdts: [u8; 5], + /// Maximum Data Transfer Size (`2^mdts` pages; 0 = unlimited). + pub mdts: u8, + /// cntlid..hmpre — to `hmpre` at offset 272. + _pad_mdts_to_hmpre: [u8; 194], + /// Host Memory Buffer Preferred Size, in **4 KiB units** (0 = the + /// controller does not use a host memory buffer). A DRAM-less + /// controller keeps its mapping tables here; without one it pages them + /// from NAND on every access. + pub hmpre: u32, + /// Host Memory Buffer Minimum Size, in 4 KiB units: a smaller buffer + /// may be refused. + pub hmmin: u32, + /// tnvmcap..nn — to `nn` at offset 516. + _pad_hmmin_to_nn: [u8; 236], + /// Number of Namespaces. + pub nn: u32, + /// Optional NVM Command Support. + pub oncs: u16, + /// fuses..hmminds — to `hmminds` at offset 528. + _pad_oncs_to_hmminds: [u8; 6], + /// Host Memory Buffer Minimum Descriptor Entry Size, in 4 KiB units + /// (0 = no constraint): every descriptor must cover at least this. + pub hmminds: u32, + /// Host Memory Maximum Descriptor Entries (0 = no limit). + pub hmmaxd: u16, + _tail: [u8; 3562], +} + +const _: () = assert!(mem::size_of::() == 4096); +// Pin the offsets the padding-collapsed fields depend on (spec Figure). +const _: () = assert!(mem::offset_of!(IdentifyController, mdts) == 77); +const _: () = assert!(mem::offset_of!(IdentifyController, hmpre) == 272); +const _: () = assert!(mem::offset_of!(IdentifyController, hmmin) == 276); +const _: () = assert!(mem::offset_of!(IdentifyController, nn) == 516); +const _: () = assert!(mem::offset_of!(IdentifyController, oncs) == 520); +const _: () = assert!(mem::offset_of!(IdentifyController, hmminds) == 528); +const _: () = assert!(mem::offset_of!(IdentifyController, hmmaxd) == 532); + +/// One LBA format descriptor. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct LbaFormat { + /// Metadata size. + pub ms: u16, + /// LBA data size as a power of two (`2^lbads` bytes). + pub lbads: u8, + /// Relative performance. + pub rp: u8, +} + +const _: () = assert!(mem::size_of::() == 4); + +/// Identify Namespace data (4096 B, spec Figure). As above, only the +/// read fields are named. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct IdentifyNamespace { + /// Namespace size, in logical blocks. + pub nsze: u64, + /// Namespace capacity, in logical blocks. + pub ncap: u64, + /// Namespace utilization. + pub nuse: u64, + /// Namespace features. + pub nsfeat: u8, + /// Number of LBA formats (0-based). + pub nlbaf: u8, + /// Formatted LBA Size. Bits 3:0 are the low format-index bits and + /// bits 6:5 are the high format-index bits; bit 4 selects separate vs. + /// extended metadata and is not part of the index. + pub flbas: u8, + /// Metadata Capabilities. + pub mc: u8, + /// End-to-end Data Protection Capabilities. + pub dpc: u8, + /// End-to-end Data Protection Type Settings. + pub dps: u8, + /// Namespace Multi-path I/O and Namespace Sharing Capabilities. + pub nmic: u8, + /// Reservation Capabilities. + pub rescap: u8, + /// Format Progress Indicator. + pub fpi: u8, + /// Deallocate Logical Block Features. + pub dlfeat: u8, + /// Namespace Atomic Write Unit Normal (0-based). + pub nawun: u16, + /// Namespace Atomic Write Unit Power Fail (0-based). + pub nawupf: u16, + /// Namespace Atomic Compare & Write Unit (0-based). + pub nacwu: u16, + /// Namespace Atomic Boundary Size Normal (0-based). + pub nabsn: u16, + /// Namespace Atomic Boundary Offset. + pub nabo: u16, + /// Namespace Atomic Boundary Size Power Fail (0-based). + pub nabspf: u16, + /// Namespace Optimal I/O Boundary, in logical blocks (0 = none). + pub noiob: u16, + /// Namespace NVM Capacity. + pub nvmcap: [u8; 16], + /// Namespace Preferred Write Granularity (0-based when supported). + pub npwg: u16, + /// Namespace Preferred Write Alignment (0-based when supported). + pub npwa: u16, + /// Namespace Preferred Deallocate Granularity (0-based when supported). + pub npdg: u16, + /// Namespace Preferred Deallocate Alignment (0-based when supported). + pub npda: u16, + /// Namespace Optimal Write Size (0-based when supported). + pub nows: u16, + /// mssrl..lbaf — to the first LBA format descriptor at offset 128. + _pad_mssrl_to_lbaf: [u8; 54], + /// LBA format descriptors. + pub lbaf: [LbaFormat; 64], + _tail: [u8; 3712], +} + +const _: () = assert!(mem::size_of::() == 4096); +const _: () = assert!(mem::offset_of!(IdentifyNamespace, flbas) == 26); +const _: () = assert!(mem::offset_of!(IdentifyNamespace, nawun) == 34); +const _: () = assert!(mem::offset_of!(IdentifyNamespace, noiob) == 46); +const _: () = assert!(mem::offset_of!(IdentifyNamespace, npwg) == 64); +const _: () = assert!(mem::offset_of!(IdentifyNamespace, nows) == 72); +const _: () = assert!(mem::offset_of!(IdentifyNamespace, lbaf) == 128); + +impl IdentifyNamespace { + /// The active LBA format selected by the split six-bit FLBAS index. + /// Device-controlled indices outside both `NLBAF` and the descriptor + /// table refuse instead of indexing an unrelated descriptor. + pub fn active_lba_format(&self) -> Option<&LbaFormat> { + let high = if self.nlbaf > 16 { + (self.flbas >> 1) & 0x30 + } else { + 0 + }; + let index = ((self.flbas & 0x0f) | high) as usize; + (index <= self.nlbaf as usize) + .then(|| self.lbaf.get(index)) + .flatten() + } + + /// Metadata bytes attached to each active logical block. The current + /// data path carries no metadata buffer, so callers must refuse a + /// non-zero result rather than issue a short transfer. + pub fn metadata_size(&self) -> u16 { + self.active_lba_format().map_or(0, |format| format.ms) + } + + /// Logical block size in bytes — `0` when the descriptor's LBADS + /// cannot encode a `u32` size (a shift past the width). Identify + /// data is device-controlled input: a hostile or broken descriptor + /// must decode to a sentinel the caller validates, never to a panic + /// (`kore_nvme::swap::blocks_per_page` is that validation for the + /// swap backend; `0` fails it like every other unusable size). + pub fn block_size(&self) -> u32 { + self.active_lba_format() + .filter(|format| format.lbads >= 9) + .and_then(|format| 1u32.checked_shl(format.lbads as u32)) + .unwrap_or(0) + } + + /// Performance fields the generic block seam can normalize. NVMe + /// `NPW*`/`NOWS` fields are 0-based and defined only when NSFEAT.OPTPERF + /// is set. `NOIOB` is an independent 1-based block count, with zero + /// meaning that the namespace reports no optimal boundary. + /// Every LBA data size this namespace can be formatted to, as a mask: + /// bit `k` set means `512 << k` bytes is offered by a format with no + /// metadata (the only kind this core can drive). Reads the whole + /// descriptor table, not just the active entry, so an operator can see + /// that a 512-byte namespace *could* be a 4 KiB one before deciding on + /// the destructive Format that would make it so. + pub fn supported_block_sizes(&self) -> u32 { + let count = (usize::from(self.nlbaf) + 1).min(self.lbaf.len()); + self.lbaf[..count] + .iter() + .filter(|format| format.ms == 0 && (9..=9 + 31).contains(&format.lbads)) + .fold(0u32, |mask, format| mask | (1u32 << (format.lbads - 9))) + } + + pub fn io_performance(&self) -> (u32, u32, u32, u32) { + let (granularity, alignment, optimal) = if self.nsfeat & (1 << 4) != 0 { + ( + u32::from(self.npwg) + 1, + u32::from(self.npwa) + 1, + u32::from(self.nows) + 1, + ) + } else { + (0, 0, 0) + }; + (granularity, alignment, optimal, u32::from(self.noiob)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// QPRIO sits in CDW11 bits 2:1 beside PC (bit 0) and CQID (31:16); + /// the priority-less builder must stay byte-identical to what every + /// queue carried before priorities existed. + #[test] + fn create_io_sq_carries_its_priority_in_cdw11() { + let dma = IovaAddr(0x8000); + let legacy = NvmeSqe::create_io_sq(3, 64, dma, 2); + assert_eq!(legacy.cdw11, (2 << 16) | 1); + assert_eq!(legacy.cdw10, (63 << 16) | 3); + + for (prio, bits) in [ + (SqPriority::Urgent, 0b00), + (SqPriority::High, 0b01), + (SqPriority::Medium, 0b10), + (SqPriority::Low, 0b11), + ] { + let c = NvmeSqe::create_io_sq_prio(3, 64, dma, 2, prio); + assert_eq!(c.opcode(), AdminOpcode::CreateIoSq as u8); + assert_eq!(c.cdw11, (2 << 16) | (bits << 1) | 1, "{prio:?}"); + assert_eq!(c.prp1, dma.0); + } + } + + /// Abort names the victim by (CID, SQID) in CDW10, CID high. + #[test] + fn abort_packs_cid_high_and_sqid_low() { + let c = NvmeSqe::abort(4, 0x1234); + assert_eq!(c.opcode(), AdminOpcode::Abort as u8); + assert_eq!(c.opcode(), 0x08); + assert_eq!(c.cdw10, 0x1234_0004); + } + + /// Arbitration: HPW 31:24, MPW 23:16, LPW 15:8, AB 2:0; an exponent + /// that would alias "no limit" is refused rather than truncated. + #[test] + fn arbitration_feature_packs_weights_and_burst() { + let c = NvmeSqe::set_arbitration(Arbitration { + burst: ArbitrationBurst::Pow2(0), + high: 0x30, + medium: 0x20, + low: 0x10, + }) + .unwrap(); + assert_eq!(c.opcode(), AdminOpcode::SetFeatures as u8); + assert_eq!(c.cdw10, 0x01); + assert_eq!(c.cdw11, 0x3020_1000); + + let unlimited = NvmeSqe::set_arbitration(Arbitration { + burst: ArbitrationBurst::Unlimited, + high: 0, + medium: 0, + low: 0, + }) + .unwrap(); + assert_eq!(unlimited.cdw11, 0b111); + assert_eq!(ArbitrationBurst::Pow2(6).encode(), Some(6)); + assert_eq!(ArbitrationBurst::Pow2(7).encode(), None); + assert!( + NvmeSqe::set_arbitration(Arbitration { + burst: ArbitrationBurst::Pow2(7), + high: 0, + medium: 0, + low: 0, + }) + .is_none() + ); + } + + #[test] + fn sqe_cid_and_opcode_pack_into_cdw0() { + let prp = Prp { + prp1: IovaAddr(0xdead_0000), + prp2: IovaAddr(0), + }; + let mut c = NvmeSqe::read(1, 0x1000, 7, prp); + assert_eq!(c.opcode(), IoOpcode::Read as u8); + c.set_cid(0xABCD); + assert_eq!(c.cid(), 0xABCD); + assert_eq!( + c.opcode(), + IoOpcode::Read as u8, + "cid does not clobber opcode" + ); + assert_eq!(c.cdw12, 7, "NLB is 0-based (7 ⇒ 8 blocks)"); + assert_eq!(c.cdw10, 0x1000, "LBA low"); + } + + #[test] + fn cqe_status_decoding() { + // Success, phase set. + let ok = NvmeCqe { + result: 0, + rsvd: 0, + sq_head: 1, + sq_id: 0, + cid: 5, + status: 0x0001, + }; + assert!(ok.phase()); + assert!(ok.is_success()); + + // Status Code 0x02 (SCT 0), phase clear ⇒ failure. + let err = NvmeCqe { + status: 0x0004, + ..ok + }; + assert!(!err.phase()); + assert!(!err.is_success()); + assert_eq!(err.status_code(), 0x02); + } + + #[test] + fn create_cq_encodes_qid_size_vector_and_ien() { + let c = NvmeSqe::create_io_cq(1, 64, IovaAddr(0xcafe_0000), 3, true); + assert_eq!(c.opcode(), AdminOpcode::CreateIoCq as u8); + assert_eq!(c.prp1, 0xcafe_0000); + assert_eq!(c.cdw10, (63 << 16) | 1, "QSIZE 0-based, QID 1"); + assert_eq!(c.cdw11, (3 << 16) | 2 | 1, "IV=3, IEN, PC"); + } + + #[test] + fn namespace_block_size_reads_active_format() { + let mut ns: IdentifyNamespace = unsafe { core::mem::zeroed() }; + ns.nsze = 0x10_0000; + ns.nlbaf = 1; + ns.flbas = 1; // select lbaf[1] + ns.lbaf[1].lbads = 12; // 4096-byte blocks + assert_eq!(ns.block_size(), 4096); + assert_eq!(ns.nsze, 0x10_0000); + } + + #[test] + fn namespace_format_index_uses_both_flbas_fragments() { + let mut ns: IdentifyNamespace = unsafe { core::mem::zeroed() }; + ns.nlbaf = 63; + // Index 0b10_0011 (35): high bits in FLBAS 6:5, low bits in 3:0. + ns.flbas = (0b10 << 5) | 0b0011; + ns.lbaf[35].lbads = 13; + assert_eq!(ns.block_size(), 8192); + } + + #[test] + fn legacy_format_count_ignores_reserved_flbas_high_bits() { + let mut ns: IdentifyNamespace = unsafe { core::mem::zeroed() }; + ns.nlbaf = 1; + ns.flbas = (0b11 << 5) | 1; + ns.lbaf[1].lbads = 12; + assert_eq!(ns.block_size(), 4096); + } + + #[test] + fn malformed_namespace_geometry_refuses_without_panicking() { + let mut ns: IdentifyNamespace = unsafe { core::mem::zeroed() }; + ns.nlbaf = 0; + ns.flbas = 1; + ns.lbaf[1].lbads = 12; + assert_eq!(ns.block_size(), 0, "FLBAS may not exceed NLBAF"); + + ns.flbas = 0; + ns.lbaf[0].lbads = 8; + assert_eq!(ns.block_size(), 0, "NVMe LBA data is at least 512 bytes"); + ns.lbaf[0].lbads = u8::MAX; + assert_eq!(ns.block_size(), 0, "hostile shifts remain checked"); + } + + #[test] + fn supported_block_sizes_read_the_whole_descriptor_table() { + let mut ns: IdentifyNamespace = unsafe { core::mem::zeroed() }; + ns.nlbaf = 2; // three formats + ns.lbaf[0].lbads = 9; // 512 B + ns.lbaf[1].lbads = 12; // 4 KiB + ns.lbaf[2].lbads = 12; // 4 KiB + metadata: not drivable here + ns.lbaf[2].ms = 8; + ns.lbaf[3].lbads = 13; // past NLBAF: not offered + assert_eq!(ns.supported_block_sizes(), (1 << 0) | (1 << 3)); + ns.lbaf[0].lbads = 8; // sub-512 is not an NVMe LBA + assert_eq!(ns.supported_block_sizes(), 1 << 3); + // A hostile count clamps to the table rather than indexing past it, + // so the fourth descriptor now legitimately counts. + ns.nlbaf = u8::MAX; + assert_eq!(ns.supported_block_sizes(), (1 << 3) | (1 << 4)); + } + + #[test] + fn io_performance_normalizes_zero_based_fields() { + let mut ns: IdentifyNamespace = unsafe { core::mem::zeroed() }; + ns.nsfeat = 1 << 4; + ns.npwg = 7; + ns.npwa = 3; + ns.nows = 31; + ns.noiob = 256; + assert_eq!(ns.io_performance(), (8, 4, 32, 256)); + + ns.nsfeat = 0; + assert_eq!(ns.io_performance(), (0, 0, 0, 256)); + ns.nsfeat = 1 << 5; // reserved, not OPTPERF + assert_eq!(ns.io_performance(), (0, 0, 0, 256)); + } +} diff --git a/native/drivers/nvme/src/controller.rs b/native/drivers/nvme/src/controller.rs new file mode 100644 index 0000000..6d195b2 --- /dev/null +++ b/native/drivers/nvme/src/controller.rs @@ -0,0 +1,1159 @@ +//! The **controller** — `spdk_nvme_ctrlr`: owns the admin qpair and the +//! bring-up sequence, and answers Identify. Generic over the platform +//! [`seam`](crate::seam); the boot-test provides a kernel seam, the +//! [`mock`](crate::mock) a host one. +//! +//! Bring-up (NVMe base spec §3.1.1 — the controller-initialization flow): +//! read `CAP`, disable and wait for `!CSTS.RDY`, size + allocate the admin +//! SQ/CQ, program `AQA`/`ASQ`/`ACQ`, then enable and wait for `CSTS.RDY`. +//! Preparation and activation are separate type states so a capability host +//! can publish the complete DMA graph before the device is allowed to master +//! it. Then Identify Controller + Namespace fill [`DeviceInfo`]. + +use crate::commands::{HmbDescriptor, IdentifyController, IdentifyNamespace, NvmeSqe}; +use crate::prp::MemPage; +use crate::qpair::{Completion, Qpair}; +use crate::regs::{ + CC_EN, CC_SHN_MASK, CC_SHN_NORMAL, CSTS_CFS, CSTS_RDY, CSTS_SHST_COMPLETE, CSTS_SHST_MASK, + Capabilities, REG_ACQ, REG_AQA, REG_ASQ, REG_CAP, REG_CC, REG_CSTS, REG_VS, + cc_enable_arbitrated, +}; +use crate::seam::{Dma, DmaPool, Regs}; +use crate::{Error, Result}; +use kore_driver::block::{CompletionPacer, CountedPacer}; + +use core::fmt; +use core::mem::size_of; +use core::sync::atomic::{Ordering, fence}; + +/// Admin queue depth (spec allows up to 4096; 32 is the conventional +/// bring-up size and covers every setup command outstanding at once). +pub const ADMIN_DEPTH: u16 = 32; + +/// What the controller answered an Abort with (completion DW0 bit 0). +/// +/// Advisory in both arms: the victim command still posts its own +/// completion, and only that completion releases its buffer. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AbortOutcome { + /// DW0 bit 0 clear: the victim will complete as *Command Abort Requested*. + Aborted, + /// DW0 bit 0 set: the controller did not abort it; it may still complete + /// normally, late. + NotAborted, +} + +/// Identify-derived controller + namespace facts the driver acts on. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DeviceInfo { + /// Controller version (`VS`). + pub version: u32, + /// Maximum Data Transfer Size in bytes (0 = unlimited). + pub mdts_bytes: u64, + /// Number of namespaces. + pub num_namespaces: u32, + /// Optional NVM Command Support (Identify Controller ONCS). + pub oncs: u16, + /// Namespace 1's logical block size in bytes. + pub block_size: u32, + /// Namespace 1's size in logical blocks. + pub block_count: u64, + /// Preferred namespace write granularity, in logical blocks. + pub preferred_write_blocks: u32, + /// Preferred namespace write alignment, in logical blocks. + pub write_alignment_blocks: u32, + /// Optimal namespace write command size, in logical blocks. + pub optimal_write_blocks: u32, + /// Optimal I/O boundary, in logical blocks (0 = unspecified). + pub optimal_boundary_blocks: u32, + /// Every metadata-free LBA data size the namespace offers (bit `k` = + /// `512 << k` bytes) — the active one is `block_size`. + pub supported_block_sizes: u32, + /// Host Memory Buffer the controller would like, in bytes (Identify + /// HMPRE; 0 = it uses none). A DRAM-less controller keeps its mapping + /// tables here and pages them from NAND without one. + pub hmb_preferred_bytes: u64, + /// The smallest Host Memory Buffer the controller accepts (HMMIN). + pub hmb_min_bytes: u64, + /// The smallest chunk one descriptor may name (HMMINDS; 0 = no floor). + pub hmb_min_descriptor_bytes: u64, + /// Descriptors the controller accepts at most (HMMAXD; 0 = no limit). + pub hmb_max_descriptors: u16, + /// Host Memory Buffer currently enabled, in bytes (0 = none). + pub hmb_bytes: u64, +} + +/// Identify reports the Host Memory Buffer sizes in **fixed 4 KiB units** +/// — not `CC.MPS`, which is the unit Set Features takes them in. The +/// conversion to pages happens once, at the command builder. +const fn hmb_units_to_bytes(units: u32) -> u64 { + (units as u64) << 12 +} + +/// Chunks a host memory buffer may be built from — 64 MiB in the 4 MiB +/// pieces a capability host carves from a grown DMA window, which is more +/// than any consumer-class HMMAXD this driver has met. +pub const HMB_MAX_CHUNKS: usize = 16; + +/// A host memory buffer the controller is using: the descriptor table and +/// the chunks it names. Retained by the controller for exactly as long as +/// the feature is enabled — the device reads and writes these frames at +/// any moment, so they outlive every command and are released only after +/// the controller acknowledged the disable. +pub struct HostMemoryBuffer { + list: B, + chunks: [Option; HMB_MAX_CHUNKS], + bytes: u64, +} + +impl HostMemoryBuffer { + /// Bytes the controller was given. + pub const fn bytes(&self) -> u64 { + self.bytes + } + + /// Chunks the buffer is made of. + pub fn chunks(&self) -> usize { + self.chunks.iter().flatten().count() + } + + /// The descriptor table's storage. + pub fn list(&self) -> &B { + &self.list + } +} + +/// Decode Identify Controller MDTS in the units fixed by the base spec: +/// `2^MDTS` times the controller's minimum memory-page size, not a fixed +/// 4 KiB and not the currently configured page. Zero means no limit. +fn mdts_bytes(mdts: u8, mpsmin: u8) -> Result { + if mdts == 0 { + return Ok(0); + } + let exponent = 12u32 + .checked_add(u32::from(mpsmin)) + .and_then(|value| value.checked_add(u32::from(mdts))) + .ok_or(Error::Geometry)?; + 1u64.checked_shl(exponent).ok_or(Error::Geometry) +} + +/// A bound NVMe controller. +pub struct Controller { + regs: R, + pool: D, + caps: Capabilities, + /// The **configured** memory page (`CC.MPS = MPSMIN` — the smallest + /// page the controller accepts; never a hardcoded 4 KiB). All PRP + /// math derives from this. + page: MemPage, + admin: Qpair, + info: DeviceInfo, + /// The host memory buffer in use, if one was enabled. + hmb: Option>, + /// Enabled under weighted round robin with urgent priority (`CC.AMS`). + weighted: bool, +} + +/// A disabled NVMe controller whose admin queues have been allocated and +/// programmed, but which has not been allowed to fetch from them. +/// +/// This affine type is the boundary a capability-backed host uses to publish +/// the pool's IOMMU window: [`Controller::prepare`] establishes every device +/// pointer while `CC.EN == 0`, and [`start`](Self::start) is the only path that +/// consumes that prepared graph and enables the controller. +pub struct PreparedController { + regs: R, + pool: D, + caps: Capabilities, + page: MemPage, + admin: Qpair, + info: DeviceInfo, + /// Asked for weighted round robin, honoured only if `CAP.AMS` offers it. + weighted: bool, +} + +/// A refused transition from prepared to running. +/// +/// The controller seams and every admin-queue handle ride back with the error. +/// This is load-bearing for capability hosts: they can restore the seams to +/// their common resource owner, and only then disable bus mastering and retire +/// the mappings. The value deliberately cannot become a `PreparedController` +/// again: the best-effort rollback may not have observed `!RDY`, so that type +/// state would claim more than the hardware proved. Dropping this value is the +/// legacy behavior for hosts whose pool was already device-visible. +#[must_use = "a refused start still owns the prepared controller seams"] +pub struct StartError { + error: Error, + // Driver-internal DMA children drop before either outer platform seam. + admin: Qpair, + regs: R, + pool: D, +} + +impl fmt::Debug for StartError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StartError") + .field("error", &self.error) + .finish_non_exhaustive() + } +} + +impl StartError { + /// The hardware refusal, without consuming the retained seams. + pub fn error(&self) -> Error { + self.error + } + + /// Recover both platform seams beside the hardware refusal. + pub fn into_seam(self) -> (Error, R, D) { + let Self { + error, + admin, + regs, + pool, + } = self; + drop(admin); + (error, regs, pool) + } + + /// Discard the prepared seams and retain only the old scalar error. + /// Convenience bring-up paths use this when their DMA was visible before + /// preparation and no outer capability owner needs reconstruction. + pub fn into_error(self) -> Error { + self.error + } +} + +impl PreparedController { + /// The controller capabilities decoded during preparation. + pub fn caps(&self) -> &Capabilities { + &self.caps + } + + /// The configured memory page that will be written to `CC.MPS` at start. + pub fn page(&self) -> MemPage { + self.page + } + + /// Release the platform seams without ever enabling the controller. + pub fn into_seam(self) -> (R, D) { + let Self { regs, pool, .. } = self; + (regs, pool) + } + + /// Enable the prepared controller after its DMA graph has become visible + /// to the device, then wait for `CSTS.RDY`. + /// + /// If readiness is refused, this method clears `CC.EN` again and makes a + /// bounded best-effort wait for `!RDY`, then returns the still-owned + /// prepared state in [`StartError`]. The capability host still owns the + /// outer containment policy (notably disabling PCI bus mastering) when a + /// device ignores that stop. + pub fn start( + self, + max_polls: u32, + pause: impl FnMut(), + ) -> core::result::Result, StartError> { + let mut pacer = CountedPacer::new(max_polls, pause); + self.start_paced(&mut pacer) + } + + /// Start under **weighted round robin with urgent priority** when the + /// controller offers it (`CAP.AMS` bit 17): each I/O submission queue's + /// `QPRIO` then decides its share, an urgent queue served ahead of every + /// other. A controller without it starts under plain round robin, as + /// before; [`Controller::weighted_arbitration`] reports which. + pub fn with_weighted_arbitration(mut self) -> Self { + self.weighted = true; + self + } + + /// Elapsed-time/composed-pacer sibling of [`start`](Self::start). + pub fn start_paced( + self, + pacer: &mut dyn CompletionPacer, + ) -> core::result::Result, StartError> { + let Self { + regs, + pool, + caps, + page, + admin, + info, + weighted, + } = self; + + // `CC.AMS` naming a mechanism `CAP.AMS` lacks is invalid; ask only + // for what the controller offers. + let weighted = weighted && caps.ams_wrr; + regs.write32(REG_CC, cc_enable_arbitrated(caps.mpsmin, weighted)); + if let Err(error) = + Controller::::wait_csts_paced(®s, |csts| csts & CSTS_RDY != 0, pacer) + { + let cc = regs.read32(REG_CC); + regs.write32(REG_CC, cc & !(CC_EN | CC_SHN_MASK)); + let _ = Controller::::wait_csts_paced(®s, |csts| csts & CSTS_RDY == 0, pacer); + return Err(StartError { + error, + admin, + regs, + pool, + }); + } + + Ok(Controller { + regs, + pool, + caps, + page, + admin, + info, + hmb: None, + weighted, + }) + } +} + +impl Controller { + /// Whether the controller runs weighted round robin with urgent priority + /// ([`PreparedController::with_weighted_arbitration`]) — only then do + /// the queues' `QPRIO` and the Arbitration weights mean anything. + pub const fn weighted_arbitration(&self) -> bool { + self.weighted + } + + /// **Bring the controller up** over the seam. Convenience composition of + /// [`prepare`](Self::prepare) and [`PreparedController::start`] for hosts + /// whose DMA pool is already device-visible (the mock and legacy paths). + /// + /// `max_polls`/`pause` bound the two `CSTS` waits (disable-ack and + /// ready): on hardware `pause` is a real inter-poll delay; the host + /// mock completes synchronously, so a no-op `pause` and a small budget + /// suffice. + pub fn bring_up(regs: R, pool: D, max_polls: u32, mut pause: impl FnMut()) -> Result { + Self::prepare(regs, pool, max_polls, &mut pause)? + .start(max_polls, pause) + .map_err(StartError::into_error) + } + + /// Disable the controller and wait for `!CSTS.RDY`, then allocate and + /// program its complete admin-queue DMA graph without setting `CC.EN`. + /// + /// A capability host can prepare against a CPU-visible pool, publish that + /// pool into the device's IOMMU domain, and only then consume the returned + /// value with [`PreparedController::start`]. + pub fn prepare( + regs: R, + pool: D, + max_polls: u32, + pause: impl FnMut(), + ) -> Result> { + let mut pacer = CountedPacer::new(max_polls, pause); + Self::prepare_paced(regs, pool, &mut pacer) + } + + /// Prepare the controller using a host-composed completion pacer. + pub fn prepare_paced( + regs: R, + pool: D, + pacer: &mut dyn CompletionPacer, + ) -> Result> { + let caps = Capabilities::from_cap(regs.read64(REG_CAP)); + let version = regs.read32(REG_VS); + // Configure the smallest page the controller accepts — correct + // even when MPSMIN > 0 (a hardcoded 4 KiB would be illegal there). + let page = MemPage::from_mps(caps.mpsmin); + + // Disable first: no queue buffer is allocated or programmed until the + // controller has acknowledged that it can no longer fetch DMA. + let cc = regs.read32(REG_CC); + regs.write32(REG_CC, cc & !(CC_EN | CC_SHN_MASK)); + Self::wait_csts_paced(®s, |csts| csts & CSTS_RDY == 0, pacer)?; + + // Size the admin queue within the controller's MQES. `max_queue_depth` + // saturates: `max_queue_entries() as u16` wraps a maximal MQES to zero + // and refuses the most capable controllers. + let depth = ADMIN_DEPTH.min(caps.max_queue_depth()); + let sq = pool + .alloc_aligned(depth as usize * size_of::(), page.bytes() as usize) + .ok_or(Error::DmaAlloc)?; + let cq = pool + .alloc_aligned( + depth as usize * size_of::(), + page.bytes() as usize, + ) + .ok_or(Error::DmaAlloc)?; + let admin = Qpair::new(0, sq, cq, depth, &caps)?; + + // Program the admin queue: attributes (0-based sizes) + base IOVAs + // (typed to the wire edge — the raw u64 exists only here). + let aqa = ((admin.depth() as u32 - 1) << 16) | (admin.depth() as u32 - 1); + regs.write32(REG_AQA, aqa); + regs.write64(REG_ASQ, admin.sq_iova().0); + regs.write64(REG_ACQ, admin.cq_iova().0); + + Ok(PreparedController { + regs, + pool, + caps, + page, + admin, + info: DeviceInfo { + version, + ..DeviceInfo::default() + }, + weighted: false, + }) + } + + /// Poll `CSTS` until `done` holds, failing on `CSTS.CFS` (fatal) or + /// timeout. + /// + /// The loop is [`kore_driver::wait::handshake`], shared with every other + /// driver's bring-up waits; `CSTS.CFS` is its fault predicate, which is + /// where the abort-early behaviour this controller has always had now + /// lives for everyone. + fn wait_csts( + regs: &R, + done: impl Fn(u32) -> bool, + max_polls: u32, + pause: &mut impl FnMut(), + ) -> Result<()> { + kore_driver::wait::handshake( + regs, + // Still a probe count, and still named as one: this controller's + // bring-up budget is caller-supplied all the way up its public + // API, so clocking it is an API change to `bring_up`/`identify`/ + // `smart` rather than a local one. `Budget::spins` keeps today's + // behaviour exactly and makes the remaining count greppable. + kore_driver::Budget::spins(max_polls), + pause, + |r: &R| done(r.read32(REG_CSTS)), + |r: &R| r.read32(REG_CSTS) & CSTS_CFS != 0, + ) + // Both failures are `NotReady` to this caller, as before — the + // distinction is preserved in the `Ack` for a future caller that wants + // to tell "dead controller" from "slow controller". + .ok_or(Error::NotReady, Error::NotReady) + } + + fn wait_csts_paced( + regs: &R, + done: impl Fn(u32) -> bool, + pacer: &mut dyn CompletionPacer, + ) -> Result<()> { + kore_driver::wait::handshake_paced( + regs, + pacer, + |r: &R| done(r.read32(REG_CSTS)), + |r: &R| r.read32(REG_CSTS) & CSTS_CFS != 0, + ) + .ok_or(Error::NotReady, Error::NotReady) + } + + /// The controller capabilities. + pub fn caps(&self) -> &Capabilities { + &self.caps + } + + /// The configured memory page (`CC.MPS`) all PRP math derives from. + pub fn page(&self) -> MemPage { + self.page + } + + /// The identify-derived device info (populated by [`identify`](Self::identify)). + pub fn info(&self) -> &DeviceInfo { + &self.info + } + + /// Borrow the register seam (doorbells, MSI-X table — the boot-test / + /// I/O paths). + pub fn regs(&self) -> &R { + &self.regs + } + + /// Borrow the DMA pool (I/O buffers, PRP pages). + pub fn pool(&self) -> &D { + &self.pool + } + + /// Run one admin command to completion, mapping a bad status to + /// [`Error::Status`]. + fn admin_wait( + &mut self, + cmd: NvmeSqe, + max_polls: u32, + pause: impl FnMut(), + ) -> Result { + let mut pacer = CountedPacer::new(max_polls, pause); + self.admin_wait_paced(cmd, &mut pacer) + } + + fn admin_wait_paced( + &mut self, + cmd: NvmeSqe, + pacer: &mut dyn CompletionPacer, + ) -> Result { + let c = self + .admin + .submit_and_wait_paced(cmd, &self.regs, pacer) + .ok_or(Error::Timeout)?; + if !c.is_success() { + return Err(Error::Status { + sct: c.cqe.status_code_type(), + sc: c.cqe.status_code(), + }); + } + Ok(c) + } + + /// **Identify** the controller + namespace 1, filling [`DeviceInfo`]. + pub fn identify(&mut self, max_polls: u32, mut pause: impl FnMut()) -> Result { + let mut pacer = CountedPacer::new(max_polls, &mut pause); + self.identify_paced(&mut pacer) + } + + /// Identify through a caller-composed elapsed-time/counting pacer. + pub fn identify_paced(&mut self, pacer: &mut dyn CompletionPacer) -> Result { + // Identify Controller into a 4 KiB DMA buffer. + let buf = self.pool.alloc(4096).ok_or(Error::DmaAlloc)?; + self.admin_wait_paced(NvmeSqe::identify_controller(buf.iova()), pacer)?; + // SAFETY: the device wrote 4096 valid bytes of IdentifyController + // into the buffer we own for this read. + let idc: IdentifyController = + unsafe { (buf.cpu() as *const IdentifyController).read_volatile() }; + self.info.mdts_bytes = mdts_bytes(idc.mdts, self.caps.mpsmin)?; + self.info.num_namespaces = idc.nn; + self.info.oncs = idc.oncs; + self.info.hmb_preferred_bytes = hmb_units_to_bytes(idc.hmpre); + self.info.hmb_min_bytes = hmb_units_to_bytes(idc.hmmin); + self.info.hmb_min_descriptor_bytes = hmb_units_to_bytes(idc.hmminds); + self.info.hmb_max_descriptors = idc.hmmaxd; + + // Identify Namespace 1 into the same buffer (reused). + self.admin_wait_paced(NvmeSqe::identify_namespace(1, buf.iova()), pacer)?; + // SAFETY: as above, IdentifyNamespace. + let idn: IdentifyNamespace = + unsafe { (buf.cpu() as *const IdentifyNamespace).read_volatile() }; + self.info.block_size = idn.block_size(); + self.info.block_count = idn.nsze; + if idn.metadata_size() != 0 { + // This core has no metadata pointer/data-protection transport. + // Accepting an LBA format with metadata would make every PRP + // transfer shorter than the namespace command describes. + return Err(Error::Geometry); + } + let (granularity, alignment, optimal, boundary) = idn.io_performance(); + self.info.preferred_write_blocks = granularity; + self.info.write_alignment_blocks = alignment; + self.info.optimal_write_blocks = optimal; + self.info.optimal_boundary_blocks = boundary; + self.info.supported_block_sizes = idn.supported_block_sizes(); + + Ok(self.info) + } + + /// Namespace 1 as an addressable block store (valid after + /// [`identify`](Self::identify)) — carries the configured page (PRP + /// math matches `CC.MPS`) and the controller's MDTS (oversized + /// transfers refuse before reaching the device). + pub fn namespace(&self) -> crate::Namespace { + crate::Namespace::new( + 1, + self.info.block_size, + self.info.block_count, + self.page, + self.info.mdts_bytes, + ) + } + + /// Fetch the **SMART / Health** log (temperature, wear, counters) — + /// the driver's health row. + pub fn smart( + &mut self, + max_polls: u32, + pause: impl FnMut(), + ) -> Result { + let mut buf = self.pool.alloc(512).ok_or(Error::DmaAlloc)?; + self.smart_into(&mut buf, max_polls, pause) + } + + /// Fetch SMART / Health into caller-owned DMA storage. Long-lived block + /// devices use this form so repeated status probes reuse one allocation + /// instead of consuming their capability-backed bump pool. + pub fn smart_into( + &mut self, + buf: &mut D::Buf, + max_polls: u32, + mut pause: impl FnMut(), + ) -> Result { + if buf.len() < 512 { + return Err(Error::Geometry); + } + self.admin_wait(NvmeSqe::get_log_smart(buf.iova()), max_polls, &mut pause)?; + // SAFETY: the device wrote 512 valid bytes of SmartLog. + Ok(unsafe { (buf.cpu() as *const crate::commands::SmartLog).read_volatile() }) + } + + /// **Create an I/O queue pair** `qid` of `depth` entries, optionally + /// interrupt-driven (`msix_vector`). Allocates the SQ/CQ, then issues + /// Create-CQ **then** Create-SQ on the admin queue (order matters — the + /// SQ binds to an existing CQ). Returns the ready qpair. + pub fn create_io_qpair( + &mut self, + qid: u16, + depth: u16, + msix_vector: Option, + max_polls: u32, + pause: impl FnMut(), + ) -> Result> { + let mut pacer = CountedPacer::new(max_polls, pause); + self.create_io_qpair_paced(qid, depth, msix_vector, &mut pacer) + } + + /// Create one I/O queue pair through a composed pacer. + pub fn create_io_qpair_paced( + &mut self, + qid: u16, + depth: u16, + msix_vector: Option, + pacer: &mut dyn CompletionPacer, + ) -> Result> { + // Saturating, for the reason given at the admin queue above: the I/O + // queue carries the identical wrap. + let depth = depth.min(self.caps.max_queue_depth()); + let sq = self + .pool + .alloc_aligned( + depth as usize * size_of::(), + self.page.bytes() as usize, + ) + .ok_or(Error::DmaAlloc)?; + let cq = self + .pool + .alloc_aligned( + depth as usize * size_of::(), + self.page.bytes() as usize, + ) + .ok_or(Error::DmaAlloc)?; + let qp = Qpair::new(qid, sq, cq, depth, &self.caps)?; + + let (iv, ien) = msix_vector.map_or((0, false), |v| (v, true)); + self.admin_wait_paced( + NvmeSqe::create_io_cq(qid, depth, qp.cq_iova(), iv, ien), + pacer, + )?; + self.admin_wait_paced(NvmeSqe::create_io_sq(qid, depth, qp.sq_iova(), qid), pacer)?; + Ok(qp) + } + + /// Create an I/O **queue group**: one completion queue `cqid` on + /// `msix_vector`, then `live` submission queues (ids `first_sqid..`) bound + /// to it at `prios[i]`. `live` is what the caller negotiated (at most + /// `SQS`). Each SQ is `depth` deep (clamped to MQES), and the CQ holds + /// all of them — `live × depth`, the per-SQ depth shrunk until that fits + /// MQES — so a full set of completions can never overrun it. + /// + /// With `SQS = 1` and `first_sqid == cqid` this issues exactly what + /// [`create_io_qpair_paced`](Self::create_io_qpair_paced) always did: + /// Create I/O CQ, then Create I/O SQ bound to it at urgent priority. + #[allow(clippy::too_many_arguments)] + pub fn create_io_queue_group_paced( + &mut self, + cqid: u16, + first_sqid: u16, + live: usize, + depth: u16, + prios: [crate::commands::SqPriority; SQS], + msix_vector: Option, + pacer: &mut dyn CompletionPacer, + ) -> Result> { + if live == 0 || live > SQS { + return Err(Error::Geometry); + } + let mqes = self.caps.max_queue_depth(); + let mut depth = depth.min(mqes); + while depth > 2 && live as u32 * u32::from(depth) > u32::from(mqes) { + depth -= 1; + } + let cq_depth = u16::try_from(live * depth as usize).map_err(|_| Error::Geometry)?; + if cq_depth > mqes { + return Err(Error::Geometry); + } + let page = self.page.bytes() as usize; + let mut sq_bufs: [Option; SQS] = core::array::from_fn(|_| None); + let mut sqids = [0u16; SQS]; + for index in 0..live { + sq_bufs[index] = Some( + self.pool + .alloc_aligned(depth as usize * size_of::(), page) + .ok_or(Error::DmaAlloc)?, + ); + sqids[index] = first_sqid + index as u16; + } + let cq = self + .pool + .alloc_aligned(cq_depth as usize * size_of::(), page) + .ok_or(Error::DmaAlloc)?; + let group = Qpair::new_group(cqid, &sqids[..live], sq_bufs, cq, depth, cq_depth, &self.caps)?; + + let (iv, ien) = msix_vector.map_or((0, false), |v| (v, true)); + self.admin_wait_paced( + NvmeSqe::create_io_cq(cqid, cq_depth, group.cq_iova(), iv, ien), + pacer, + )?; + for index in 0..live { + let (sqid, iova) = group.sq_of(index).ok_or(Error::Geometry)?; + self.admin_wait_paced( + NvmeSqe::create_io_sq_prio(sqid, depth, iova, cqid, prios[index]), + pacer, + )?; + } + Ok(group) + } + + /// Create an I/O queue group **again**, over the memory `group` already + /// owns — after a [`reset_paced`](Self::reset_paced) dropped every queue. + /// The group must itself have been [reset](Qpair::reset). Same ids, + /// depths, vector and priorities as its first creation. + pub fn recreate_io_queue_group_paced( + &mut self, + group: &Qpair, + prios: [crate::commands::SqPriority; SQS], + msix_vector: Option, + pacer: &mut dyn CompletionPacer, + ) -> Result<()> { + let cqid = group.qid(); + let (iv, ien) = msix_vector.map_or((0, false), |v| (v, true)); + self.admin_wait_paced( + NvmeSqe::create_io_cq(cqid, group.cq_depth(), group.cq_iova(), iv, ien), + pacer, + )?; + for index in 0..group.live_sqs() { + let (sqid, iova) = group.sq_of(index).ok_or(Error::Geometry)?; + self.admin_wait_paced( + NvmeSqe::create_io_sq_prio(sqid, group.depth(), iova, cqid, prios[index]), + pacer, + )?; + } + Ok(()) + } + + /// **Reset the controller** (base spec §3.7.2) and bring it back up + /// over the memory it already has: clear `CC.EN` and wait for `!RDY` — + /// the controller drops every queue and every command in flight — then + /// start the admin queue over, enable under the same arbitration, and + /// hand the host memory buffer back. The I/O queues are gone; the caller + /// resets and [recreates](Self::recreate_io_queue_group_paced) its own. + /// + /// This is the escalation an Abort the device refuses leaves: the only + /// way to get a command the controller will not give up out of its + /// hands. Nothing the controller held survives it, which is the point. + pub fn reset_paced(&mut self, pacer: &mut dyn CompletionPacer) -> Result<()> { + let cc = self.regs.read32(REG_CC); + self.regs.write32(REG_CC, cc & !(CC_EN | CC_SHN_MASK)); + Self::wait_csts_paced(&self.regs, |csts| csts & CSTS_RDY == 0, pacer)?; + let _ = self.admin.reset(); + let aqa = ((self.admin.depth() as u32 - 1) << 16) | (self.admin.depth() as u32 - 1); + self.regs.write32(REG_AQA, aqa); + self.regs.write64(REG_ASQ, self.admin.sq_iova().0); + self.regs.write64(REG_ACQ, self.admin.cq_iova().0); + self.regs + .write32(REG_CC, cc_enable_arbitrated(self.caps.mpsmin, self.weighted)); + Self::wait_csts_paced(&self.regs, |csts| csts & CSTS_RDY != 0, pacer)?; + // The controller forgot the host memory buffer with everything else; + // its descriptors still describe the same frames, which this task + // never released. + if let Some(hmb) = self.hmb.as_ref() { + let page = self.page.bytes() as u64; + let hsize = u32::try_from(hmb.bytes / page).map_err(|_| Error::Geometry)?; + let entries = hmb.chunks() as u32; + let list = hmb.list.iova(); + if self + .admin_wait_paced(NvmeSqe::enable_host_memory_buffer(hsize, list, entries), pacer) + .is_err() + { + // Refused: the controller runs without it, and the frames + // stay retained until teardown, as after any refusal. + self.info.hmb_bytes = 0; + } + } + Ok(()) + } + + /// Tell the controller how many I/O queues to allocate (Set Features + /// 0x07) — called once before creating any. Returns the granted + /// `(sq, cq)` counts. + pub fn set_num_queues( + &mut self, + want: u16, + max_polls: u32, + pause: impl FnMut(), + ) -> Result<(u16, u16)> { + let mut pacer = CountedPacer::new(max_polls, pause); + self.set_num_queues_paced(want, &mut pacer) + } + + /// **Enable a Host Memory Buffer** made of `chunks` (device-visible, + /// `CC.MPS`-aligned, each a whole number of pages) described in `list` + /// (16 bytes per chunk, 16-byte aligned). The controller retains both + /// until [`disable_host_memory_buffer`](Self::disable_host_memory_buffer) + /// succeeds — which [`shutdown`](Self::shutdown) attempts first — because + /// the device may read or write these frames at any moment after this + /// returns `Ok`. Returns the bytes the controller was given. + /// + /// The sizing decision is the caller's: it has Identify's + /// [`hmb_preferred_bytes`](DeviceInfo::hmb_preferred_bytes) and the + /// floors, and it owns the memory. What this checks is the spec's + /// shape — at least one chunk, no more than HMMAXD, each chunk + /// page-aligned and page-sized, none below HMMINDS, the whole not below + /// HMMIN — and refuses `Geometry` rather than hand the device a table it + /// would reject or, worse, misread. + pub fn enable_host_memory_buffer( + &mut self, + list: D::Buf, + chunks: [Option; HMB_MAX_CHUNKS], + pacer: &mut dyn CompletionPacer, + ) -> Result { + if self.hmb.is_some() { + return Err(Error::Geometry); + } + let page = self.page.bytes() as u64; + let count = chunks.iter().flatten().count(); + let max = usize::from(self.info.hmb_max_descriptors); + if count == 0 || (max != 0 && count > max) { + return Err(Error::Geometry); + } + if list.len() < count * size_of::() || list.iova().0 & 0xF != 0 { + return Err(Error::Geometry); + } + let mut total = 0u64; + for (index, chunk) in chunks.iter().flatten().enumerate() { + let bytes = chunk.len() as u64; + let aligned = chunk.iova().0 % page == 0 && bytes % page == 0; + if bytes == 0 || !aligned || bytes < self.info.hmb_min_descriptor_bytes { + return Err(Error::Geometry); + } + let pages = u32::try_from(bytes / page).map_err(|_| Error::Geometry)?; + total = total.checked_add(bytes).ok_or(Error::Geometry)?; + let descriptor = HmbDescriptor::new(chunk.iova(), pages); + // SAFETY: `list` is a DMA buffer this task owns for its whole + // life, and the bound above proved room for `count` entries; + // volatile because the device reads the table after submit. + unsafe { + (list.cpu() as *mut HmbDescriptor) + .add(index) + .write_volatile(descriptor); + } + } + if total < self.info.hmb_min_bytes { + return Err(Error::Geometry); + } + let hsize = u32::try_from(total / page).map_err(|_| Error::Geometry)?; + // The table is complete before the doorbell publishes it. + fence(Ordering::Release); + let entries = count as u32; + self.admin_wait_paced( + NvmeSqe::enable_host_memory_buffer(hsize, list.iova(), entries), + pacer, + )?; + self.info.hmb_bytes = total; + self.hmb = Some(HostMemoryBuffer { + list, + chunks, + bytes: total, + }); + Ok(total) + } + + /// The host memory buffer in use, if any. + pub fn host_memory_buffer(&self) -> Option<&HostMemoryBuffer> { + self.hmb.as_ref() + } + + /// **Disable the Host Memory Buffer**, releasing its memory only once + /// the controller acknowledged: a refused or timed-out disable keeps the + /// buffer retained, because the device may still be writing it. A + /// no-op when none is enabled. + pub fn disable_host_memory_buffer( + &mut self, + max_polls: u32, + mut pause: impl FnMut(), + ) -> Result<()> { + if self.hmb.is_none() { + return Ok(()); + } + self.admin_wait(NvmeSqe::disable_host_memory_buffer(), max_polls, &mut pause)?; + self.info.hmb_bytes = 0; + self.hmb = None; + Ok(()) + } + + /// Negotiate queue counts through a composed pacer. + pub fn set_num_queues_paced( + &mut self, + want: u16, + pacer: &mut dyn CompletionPacer, + ) -> Result<(u16, u16)> { + let c = self.admin_wait_paced( + NvmeSqe::set_num_queues(want.saturating_sub(1), want.saturating_sub(1)), + pacer, + )?; + let nsq = (c.cqe.result & 0xFFFF) as u16 + 1; + let ncq = ((c.cqe.result >> 16) & 0xFFFF) as u16 + 1; + Ok((nsq, ncq)) + } + + /// Negotiate **distinct** submission and completion queue counts (Set + /// Features 0x07) — the shape several submission queues sharing one + /// completion queue needs, where [`set_num_queues_paced`] + /// (Self::set_num_queues_paced) asks for equal counts. Returns the + /// granted `(sq, cq)`; the controller may grant fewer, never more, and a + /// caller plans against the grant, not the request. + pub fn set_queue_counts_paced( + &mut self, + sqs: u16, + cqs: u16, + pacer: &mut dyn CompletionPacer, + ) -> Result<(u16, u16)> { + if sqs == 0 || cqs == 0 { + return Err(Error::Geometry); + } + let c = self.admin_wait_paced(NvmeSqe::set_num_queues(sqs - 1, cqs - 1), pacer)?; + let nsq = (c.cqe.result & 0xFFFF) as u16 + 1; + let ncq = ((c.cqe.result >> 16) & 0xFFFF) as u16 + 1; + Ok((nsq, ncq)) + } + + /// Program the **Arbitration** feature (Set Features 01h). Refuses + /// `Geometry`, sending nothing, for a burst exponent the field cannot + /// carry. The weights matter only once the controller runs weighted + /// round robin ([`cc_enable_arbitrated`](crate::regs::cc_enable_arbitrated)); + /// the burst applies under either mechanism. + pub fn set_arbitration_paced( + &mut self, + arbitration: crate::commands::Arbitration, + pacer: &mut dyn CompletionPacer, + ) -> Result<()> { + let cmd = NvmeSqe::set_arbitration(arbitration).ok_or(Error::Geometry)?; + self.admin_wait_paced(cmd, pacer)?; + Ok(()) + } + + /// Ask the controller to **abort** command `cid` on submission queue + /// `sqid` (admin opcode 08h). The answer is advisory by specification — + /// [`AbortOutcome::Aborted`] means the controller will complete the + /// victim with *Command Abort Requested*, `NotAborted` that it may still + /// complete it normally — and in both cases the victim posts its own + /// completion. Its buffer is reusable only after that completion + /// arrives, never on this reply. + pub fn abort_paced( + &mut self, + sqid: u16, + cid: u16, + pacer: &mut dyn CompletionPacer, + ) -> Result { + let c = self.admin_wait_paced(NvmeSqe::abort(sqid, cid), pacer)?; + Ok(if c.cqe.result & 1 == 0 { + AbortOutcome::Aborted + } else { + AbortOutcome::NotAborted + }) + } + + /// **Orderly shutdown** (base spec §3.1.5): notify normal shutdown, wait + /// for the controller to report shutdown complete, then disable and wait + /// for `!RDY` so the next owner's bring-up starts from a clean state. + /// + /// This is a driver's quiesce hook: after it returns `Ok`, the + /// controller has flushed and stopped all DMA, so the derived BAR/IOVA + /// authority above it can be retired without racing an in-flight + /// transfer. A failure means the device did not acknowledge within the + /// poll budget (or faulted); the caller's containment still runs — a + /// half-quiesced controller is exactly why capability retirement is + /// unconditional. + pub fn shutdown(&mut self, max_polls: u32, mut pause: impl FnMut()) -> Result<()> { + // The host memory buffer goes first, while the controller can still + // take an admin command. A refused disable is not fatal to the + // shutdown: once `CSTS.SHST` reports complete the controller masters + // nothing, so the frames are safe — and they stay retained in + // `self.hmb` until `into_seam` regardless, which is what makes a + // silent device harmless rather than a use-after-release. + let _ = self.disable_host_memory_buffer(max_polls, &mut pause); + let cc = self.regs.read32(REG_CC); + self.regs + .write32(REG_CC, (cc & !CC_SHN_MASK) | CC_SHN_NORMAL); + Self::wait_csts( + &self.regs, + |csts| csts & CSTS_SHST_MASK == CSTS_SHST_COMPLETE, + max_polls, + &mut pause, + )?; + let cc = self.regs.read32(REG_CC); + self.regs.write32(REG_CC, cc & !(CC_EN | CC_SHN_MASK)); + Self::wait_csts( + &self.regs, + |csts| csts & CSTS_RDY == 0, + max_polls, + &mut pause, + ) + } + + /// Release the platform seam — the unbind half of the driver-host + /// containment contract ([`kore_driver::host::DeviceBinding::unbind`] + /// must reconstruct the resource set it was bound from). Drops the + /// admin qpair (its ring buffers are pool slices with no lifecycle of + /// their own) and returns the register window and DMA pool. + /// + /// Call after [`shutdown`](Self::shutdown): a live controller still owns + /// the admin queues this drops. + pub fn into_seam(self) -> (R, D) { + let Self { regs, pool, .. } = self; + (regs, pool) + } +} + +#[cfg(test)] +mod tests { + use super::mdts_bytes; + use crate::Error; + + #[test] + fn mdts_uses_cap_mpsmin_units() { + assert_eq!(mdts_bytes(0, 9), Ok(0)); + assert_eq!(mdts_bytes(5, 0), Ok(128 * 1024)); + assert_eq!(mdts_bytes(5, 1), Ok(256 * 1024)); + } + + #[test] + fn hostile_mdts_shift_refuses() { + assert_eq!(mdts_bytes(u8::MAX, 0), Err(Error::Geometry)); + } + + use super::{Controller, HMB_MAX_CHUNKS}; + use crate::mock::{MockBuf, MockNvmeDevice}; + use crate::seam::DmaPool; + use kore_driver::block::CountedPacer; + + /// A live mock controller, identified, advertising a DRAM-less + /// appetite: `preferred`/`min` in 4 KiB units, at most `max` chunks. + fn dram_less( + preferred: u32, + min: u32, + max: u16, + ) -> ( + MockNvmeDevice, + Controller, + ) { + let dev = MockNvmeDevice::new(); + dev.set_hmb(preferred, min, max); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + ctrl.identify(16, || {}).unwrap(); + (dev, ctrl) + } + + fn chunks( + ctrl: &Controller, + sizes: &[usize], + ) -> [Option; HMB_MAX_CHUNKS] { + let mut out = [const { None }; HMB_MAX_CHUNKS]; + for (slot, bytes) in sizes.iter().enumerate() { + out[slot] = Some(ctrl.pool().alloc_aligned(*bytes, 4096).unwrap()); + } + out + } + + #[test] + fn identify_decodes_the_host_memory_buffer_appetite_in_bytes() { + let (_dev, ctrl) = dram_less(16_384, 1_024, 8); + let info = ctrl.info(); + assert_eq!( + info.hmb_preferred_bytes, + 64 << 20, + "HMPRE is in 4 KiB units" + ); + assert_eq!(info.hmb_min_bytes, 4 << 20); + assert_eq!(info.hmb_max_descriptors, 8); + assert_eq!(info.hmb_bytes, 0, "nothing enabled yet"); + } + + #[test] + fn a_host_memory_buffer_is_enabled_from_page_chunks_and_disabled_first_at_shutdown() { + // 16 KiB preferred, 8 KiB minimum, four descriptors at most. + let (dev, mut ctrl) = dram_less(4, 2, 4); + let table = chunks(&ctrl, &[8192, 8192]); + let list = ctrl.pool().alloc(4096).unwrap(); + let mut pacer = CountedPacer::new(16, || {}); + assert_eq!( + ctrl.enable_host_memory_buffer(list, table, &mut pacer), + Ok(16_384) + ); + // HSIZE is in the configured page (4 KiB here): 4 pages, 2 entries. + assert_eq!(dev.hmb(), Some((4, 2))); + let hmb = ctrl.host_memory_buffer().expect("retained while enabled"); + assert_eq!(hmb.bytes(), 16_384); + assert_eq!(hmb.chunks(), 2); + assert_eq!(ctrl.info().hmb_bytes, 16_384); + // A second enable while one stands is refused, not stacked. + let again = chunks(&ctrl, &[8192, 8192]); + let list2 = ctrl.pool().alloc(4096).unwrap(); + assert_eq!( + ctrl.enable_host_memory_buffer(list2, again, &mut pacer), + Err(Error::Geometry) + ); + // Shutdown disables the buffer before CC.SHN, and releases it. + ctrl.shutdown(16, || {}).unwrap(); + assert_eq!(dev.hmb(), None); + assert_eq!(dev.hmb_disables(), 1); + assert!(ctrl.host_memory_buffer().is_none()); + assert_eq!(ctrl.info().hmb_bytes, 0); + } + + #[test] + fn a_table_the_controller_would_reject_is_refused_before_it_is_sent() { + // Too many descriptors for HMMAXD. + let (dev, mut ctrl) = dram_less(4, 1, 1); + let table = chunks(&ctrl, &[8192, 8192]); + let list = ctrl.pool().alloc(4096).unwrap(); + let mut pacer = CountedPacer::new(16, || {}); + assert_eq!( + ctrl.enable_host_memory_buffer(list, table, &mut pacer), + Err(Error::Geometry) + ); + assert_eq!(dev.hmb(), None, "nothing reached the device"); + + // Below the controller's minimum. + let (dev, mut ctrl) = dram_less(8, 4, 0); + let table = chunks(&ctrl, &[4096]); + let list = ctrl.pool().alloc(4096).unwrap(); + assert_eq!( + ctrl.enable_host_memory_buffer(list, table, &mut pacer), + Err(Error::Geometry) + ); + assert_eq!(dev.hmb(), None); + + // A chunk that is not a whole number of the CONFIGURED page: an + // 8 KiB-page controller handed a 4 KiB chunk (the pool cannot + // produce a non-4 KiB length, so the page is what moves). + let dev = MockNvmeDevice::new().with_mps(1).unwrap(); + dev.set_hmb(4, 0, 0); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + ctrl.identify(16, || {}).unwrap(); + assert_eq!(ctrl.page().bytes(), 8192); + let mut table = [const { None }; HMB_MAX_CHUNKS]; + table[0] = Some(ctrl.pool().alloc_aligned(4096, 8192).unwrap()); + let list = ctrl.pool().alloc(4096).unwrap(); + assert_eq!( + ctrl.enable_host_memory_buffer(list, table, &mut pacer), + Err(Error::Geometry) + ); + assert_eq!(dev.hmb(), None); + } + + #[test] + fn a_controller_without_a_buffer_shuts_down_without_a_disable() { + let (dev, mut ctrl) = dram_less(0, 0, 0); + ctrl.shutdown(16, || {}).unwrap(); + assert_eq!(dev.hmb_disables(), 0); + } +} diff --git a/native/drivers/nvme/src/lib.rs b/native/drivers/nvme/src/lib.rs new file mode 100644 index 0000000..01123bf --- /dev/null +++ b/native/drivers/nvme/src/lib.rs @@ -0,0 +1,112 @@ +//! # kore-nvme — the kore NVMe driver core (SPDK-shaped) +//! +//! The first native kore hardware driver, and the **template every future +//! driver follows**: a pure driver *core* generic over a small platform +//! [`seam`] ([`Regs`](seam::Regs) + [`DmaPool`](seam::DmaPool)), so the +//! same code can run in a capability-backed userland owner, a kernel-internal +//! first-light test, and `cfg(test)` hardware doubles. Those contexts do not +//! carry equal acceptance weight: only the capability-backed task is a +//! production driver service, and this crate does not yet assemble one. +//! +//! **SPDK-shaped, not os0-shaped** (architect direction): where the sibling +//! `os0` driver is a central block *server* (nameserver + per-queue worker +//! threads mediating every I/O — a scheduling hop SPDK exists to reject), +//! kore-nvme is **per-core lockless queue pairs** ([`Qpair`]), kernel off the +//! data path, polled completions, zero-copy DMA into caller buffers. A +//! qpair *is* a capability; the concepts mirror `spdk_nvme_*` so code +//! written against SPDK ports by renaming. The spec-fact layers +//! ([`regs`], [`commands`], [`prp`]) are ported from os0 (a working +//! driver); the *architecture* is rewritten. +//! +//! **Current scope**: the raw controller/qpair/namespace API + submit/poll, +//! plus a feature-gated `NvmeSwap` face. The production service task is +//! `kore-svc-swap-nvme` (`services/swap-nvme`): it claims the controller +//! from the device broker (`res/DEVICE_SERVICE.md` §10), brings this core +//! up over the capability seam, and serves the swap protocol. The +//! page datapath moves real bytes: they arrive through the client's +//! staging window and transit a driver-owned bounce page on their way to +//! and from the device (§8.1). + +#![no_std] +// Typed portable SIMD for the PRP list fill (the doctrine's "typed over +// hand-rolled" for the one hot scan in the core). +#![feature(portable_simd)] + +#[cfg(any(test, feature = "alloc"))] +extern crate alloc; + +pub mod block; +pub mod commands; +pub mod prp; +pub mod qpair; +pub mod regs; + +/// The platform seam, re-exported from [`kore_driver`] — it never contained +/// anything NVMe-specific, so it lives in the shared substrate now and +/// `kore_nvme::seam` is source-compatible with where it used to be defined. +pub use kore_driver::seam; + +mod controller; +mod namespace; + +#[cfg(feature = "service")] +pub mod swap; + +pub use controller::{AbortOutcome, Controller, DeviceInfo, PreparedController, StartError}; +pub use namespace::Namespace; +pub use qpair::{Completion, MAX_QDEPTH, Qpair, QpairError}; +pub use regs::Capabilities; +pub use seam::{Dma, DmaPool, Regs}; + +/// The qpair's visible books — the [`kore_driver::RingStats`] type under its +/// NVMe-facing name (the field vocabulary is shared: `submitted` / +/// `completed` / `errors`). +pub use kore_driver::RingStats as QpairStats; + +#[cfg(test)] +mod mock; + +/// A driver-core failure. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Error { + /// A DMA buffer could not be allocated from the pool. + DmaAlloc, + /// The controller did not reach the expected `CSTS.RDY` state in time + /// (or reported `CSTS.CFS`, a fatal fault). + NotReady, + /// An admin command timed out with no completion. + Timeout, + /// An admin command completed with a non-success status. + Status { + /// Status Code Type. + sct: u8, + /// Status Code. + sc: u8, + }, + /// A qpair operation refused. + Qpair(qpair::QpairError), + /// A PRP build refused. + Prp(prp::PrpError), + /// A transfer exceeds the controller's MDTS (split it). + ExceedsMdts, + /// The namespace's identify-reported geometry is one this consumer + /// cannot lawfully do arithmetic on (e.g. an LBA format that cannot + /// carry the swap page — see `kore_nvme::swap::blocks_per_page`). + /// Identify data is device input; refusing it is bring-up's job. + Geometry, +} + +impl From for Error { + fn from(e: qpair::QpairError) -> Self { + Error::Qpair(e) + } +} + +impl From for Error { + fn from(e: prp::PrpError) -> Self { + Error::Prp(e) + } +} + +/// The driver-core result type. +pub type Result = core::result::Result; diff --git a/native/drivers/nvme/src/mock.rs b/native/drivers/nvme/src/mock.rs new file mode 100644 index 0000000..1567c65 --- /dev/null +++ b/native/drivers/nvme/src/mock.rs @@ -0,0 +1,1403 @@ +//! `MockNvmeDevice` — a host-memory NVMe controller, the driver core's +//! test backend (the D2 `virtio-accel-mock` lesson: prove the semantics on +//! the host, prove the *seam* on hardware). It implements [`Regs`] and +//! hands out a [`DmaPool`]; a doorbell write drives its command engine +//! (read the new SQEs, process, publish CQEs with the phase bit), so the +//! **entire core** — bring-up, identify, qpair create, read — runs with no +//! hardware. +//! +//! The trick that makes it simple: a mock IOVA **is** the host pointer +//! (there is no IOMMU to translate), so the device reads SQEs / writes +//! CQEs / fills identify buffers by casting the IOVAs the driver +//! programmed straight back to pointers — the same frames the driver's CPU +//! window sees. + +use alloc::alloc::{Layout, alloc_zeroed}; +use alloc::boxed::Box; +use alloc::rc::Rc; +use alloc::vec::Vec; +use core::cell::{Cell, RefCell}; + +use kore_abi::device::IovaAddr; + +use crate::commands::{ + AdminOpcode, HmbDescriptor, IdentifyController, IdentifyNamespace, NvmeCqe, NvmeSqe, +}; +use crate::regs::{ + CC_EN, CC_SHN_MASK, CSTS_RDY, CSTS_SHST_COMPLETE, CSTS_SHST_MASK, REG_ACQ, REG_AQA, REG_ASQ, + REG_CAP, REG_CC, REG_CSTS, REG_DOORBELL_BASE, REG_VS, +}; +use crate::seam::{Dma, DmaPool}; + +/// The mock's advertised `CAP`: MQES 0x3F (64 entries), CQR (bit 16), +/// DSTRD 0 — enough for a 32-deep admin queue and 4-byte doorbell stride. +const MOCK_CAP: u64 = 0x0000_0000_0001_003F; +/// The mock's advertised version (NVMe 1.4.0). +const MOCK_VS: u32 = 0x0001_0400; +/// The mock namespace's logical block size. +pub const MOCK_BLOCK_SIZE: u32 = 4096; +/// The mock namespace's block count. +pub const MOCK_BLOCK_COUNT: u64 = 0x10_0000; // 4 GiB at 4 KiB blocks +/// Blocks the backing disk actually holds (I/O past it fails with LBA +/// Out of Range — deliberately smaller than the advertised namespace). +pub const MOCK_DISK_BLOCKS: u32 = 64; + +// ── The DMA pool: page-aligned host memory, iova == cpu pointer ────────── + +/// A mock DMA buffer over leaked, page-aligned host memory. +pub struct MockBuf { + ptr: *mut u8, + len: usize, +} + +impl Dma for MockBuf { + fn cpu(&self) -> *mut u8 { + self.ptr + } + fn iova(&self) -> IovaAddr { + // The mock's device "domain" is host memory: iova == pointer. + IovaAddr(self.ptr as u64) + } + fn len(&self) -> usize { + self.len + } +} + +/// Hands out page-aligned zeroed host buffers (leaked — valid for the test +/// process, so a returned `MockBuf` never dangles). +#[derive(Default)] +pub struct MockPool; + +impl DmaPool for MockPool { + type Buf = MockBuf; + fn alloc(&self, bytes: usize) -> Option { + self.alloc_aligned(bytes, 4096) + } + + fn alloc_aligned(&self, bytes: usize, alignment: usize) -> Option { + let alignment = alignment.max(4096); + if !alignment.is_power_of_two() { + return None; + } + let len = bytes.next_multiple_of(4096).max(4096); + let layout = Layout::from_size_align(len, alignment).ok()?; + // SAFETY: non-zero, well-formed layout; leaked for the process. + let ptr = unsafe { alloc_zeroed(layout) }; + if ptr.is_null() { + return None; + } + Some(MockBuf { ptr, len }) + } +} + +// ── The device: register file + command engine ────────────────────────── + +/// A created I/O queue the device knows about — its ring IOVAs, depth, +/// and the device-side head/tail/phase the I/O completion path advances. +#[derive(Clone, Copy, Default)] +pub(crate) struct IoQueueState { + pub(crate) sq_iova: u64, + pub(crate) cq_iova: u64, + /// The completion queue's depth (this qid as a CQ). + pub(crate) depth: u16, + /// The submission queue's own depth (this qid as an SQ); 0 until its + /// Create I/O SQ, when a queue created the pre-group way falls back to + /// its CQ depth. + pub(crate) sq_depth: u16, + /// The CQ this qid's SQ posts to (Create I/O SQ CDW11 bits 31:16); 0 + /// until created. + pub(crate) cqid: u16, + pub(crate) sq_head: u16, + pub(crate) cq_tail: u16, + pub(crate) cq_phase: bool, +} + +/// The mock controller's mutable state. +pub(crate) struct DeviceState { + cap: u64, + cc: u32, + csts: u32, + admin_depth: u16, + asq_iova: u64, + acq_iova: u64, + admin_sq_head: u16, + admin_cq_tail: u16, + admin_cq_phase: bool, + pub(crate) io_queues: Vec, + /// I/O SQ-tail MMIO writes (admin and CQ doorbells excluded). + io_sq_doorbells: u64, + /// I/O CQ-head MMIO writes. + io_cq_doorbells: u64, + /// A backing "disk" the Read/Write path moves bytes to/from. + pub(crate) disk: Box<[u8]>, + /// Deallocated ranges (test probe for the DSM path): `(slba, nlb)`. + pub(crate) deallocated: Vec<(u64, u32)>, + /// The LBA format Identify Namespace advertises (`2^lbads`-byte + /// blocks) — a test knob; the I/O engine itself always moves + /// [`MOCK_BLOCK_SIZE`] blocks. + lbads: u8, + /// I/O completions to withhold (test knob): the command executes, its + /// CQE waits for [`MockNvmeDevice::release_swallowed`] — a device + /// whose completion arrives after the driver stopped waiting. + swallow: u32, + /// The withheld completions: `(queue index, cid, status code)`. + held: Vec<(usize, u16, u8)>, + /// Host Memory Buffer facts Identify advertises (test knob), in the + /// spec's 4 KiB units: preferred, minimum, and the descriptor ceiling. + hmpre: u32, + hmmin: u32, + hmmaxd: u16, + /// The buffer the driver enabled: `(HSIZE in pages, descriptor count)`. + hmb: Option<(u32, u32)>, + /// Set Features 0x0D disables accepted. + hmb_disables: u32, + /// The last Arbitration feature (Set Features 0x01) CDW11 accepted. + arbitration: Option, + /// Aborts answered, as `(sqid, cid, aborted)`. + aborts: Vec<(u16, u16, bool)>, + /// Controller resets: `CC.EN` cleared on an enabled controller. + resets: u32, +} + +impl DeviceState { + fn new() -> Self { + // A small backing disk, LBA-addressed, seeded so reads are checkable. + let mut disk = alloc::vec![0u8; MOCK_DISK_BLOCKS as usize * MOCK_BLOCK_SIZE as usize] + .into_boxed_slice(); + for (i, b) in disk.iter_mut().enumerate() { + *b = (i as u32).wrapping_mul(2654435761).to_le_bytes()[0]; + } + Self { + cap: MOCK_CAP, + cc: 0, + csts: 0, + admin_depth: 0, + asq_iova: 0, + acq_iova: 0, + admin_sq_head: 0, + admin_cq_tail: 0, + admin_cq_phase: true, + io_queues: Vec::new(), + io_sq_doorbells: 0, + io_cq_doorbells: 0, + disk, + deallocated: Vec::new(), + lbads: MOCK_BLOCK_SIZE.trailing_zeros() as u8, + swallow: 0, + held: Vec::new(), + hmpre: 0, + hmmin: 0, + hmmaxd: 0, + hmb: None, + hmb_disables: 0, + arbitration: None, + aborts: Vec::new(), + resets: 0, + } + } + + /// Publish one admin completion (status is the raw SC<<1; the phase + /// bit is OR'd in). + fn complete_admin(&mut self, cid: u16, sc: u8, result: u32) { + let cqe = NvmeCqe { + result, + rsvd: 0, + sq_head: self.admin_sq_head, + sq_id: 0, + cid, + status: ((sc as u16) << 1) | (self.admin_cq_phase as u16), + }; + // SAFETY: acq_iova is a host pointer to `admin_depth * 16` bytes; + // cq_tail < depth. + unsafe { + let slot = (self.acq_iova as *mut NvmeCqe).add(self.admin_cq_tail as usize); + slot.write_volatile(cqe); + } + self.admin_cq_tail = (self.admin_cq_tail + 1) % self.admin_depth; + if self.admin_cq_tail == 0 { + self.admin_cq_phase = !self.admin_cq_phase; + } + } + + /// Process the admin SQ up to `new_tail` (called on the admin SQ + /// doorbell write). + fn process_admin(&mut self, new_tail: u16) { + while self.admin_sq_head != new_tail { + // SAFETY: asq_iova is a host pointer to `admin_depth * 64` + // bytes; sq_head < depth. + let sqe: NvmeSqe = unsafe { + (self.asq_iova as *const NvmeSqe) + .add(self.admin_sq_head as usize) + .read_volatile() + }; + let (sc, result) = self.dispatch_admin(&sqe); + self.complete_admin(sqe.cid(), sc, result); + self.admin_sq_head = (self.admin_sq_head + 1) % self.admin_depth; + } + } + + /// Execute one admin command, returning `(status_code, result_dw0)`. + fn dispatch_admin(&mut self, sqe: &NvmeSqe) -> (u8, u32) { + match sqe.opcode() { + x if x == AdminOpcode::Identify as u8 => { + match sqe.cdw10 & 0xFF { + 1 => self.write_identify_controller(sqe.prp1), + 0 => self.write_identify_namespace(sqe.prp1), + _ => return (0x02, 0), // Invalid Field + } + (0, 0) + } + x if x == AdminOpcode::GetLogPage as u8 => { + // SMART/Health (LID 0x02): fixed, checkable values. + if sqe.cdw10 & 0xFF == 0x02 { + // SAFETY: prp1 is a host pointer to a ≥512-byte buffer. + let log = unsafe { &mut *(sqe.prp1 as *mut crate::commands::SmartLog) }; + *log = unsafe { core::mem::zeroed() }; + log.composite_temp = (273u16 + 36).to_le_bytes(); // 36 °C + log.percentage_used = 3; + log.data_units_read[..8].copy_from_slice(&1234u64.to_le_bytes()); + log.power_cycles[..8].copy_from_slice(&7u64.to_le_bytes()); + (0, 0) + } else { + (0x02, 0) // Invalid Field + } + } + x if x == AdminOpcode::SetFeatures as u8 => { + match sqe.cdw10 & 0xFF { + // Arbitration: record the burst and weights as given. + 0x01 => { + self.arbitration = Some(sqe.cdw11); + (0, 0) + } + // Number of Queues: echo the request back (allocated == asked). + 0x07 => (0, sqe.cdw11), + // Host Memory Buffer: check the shape the spec fixes, then + // record what the driver handed over. + 0x0D => { + if sqe.cdw11 & 1 == 0 { + self.hmb = None; + self.hmb_disables += 1; + (0, 0) + } else { + let list = u64::from(sqe.cdw13) | (u64::from(sqe.cdw14) << 32); + let entries = sqe.cdw15; + let too_many = self.hmmaxd != 0 && entries > u32::from(self.hmmaxd); + if list & 0xF != 0 || entries == 0 || too_many { + (0x02, 0) // Invalid Field + } else { + // Sum the table the way a controller would: MPS + // pages per descriptor, at the host pointer the + // mock's IOVAs are. + let mps = 1u64 << (12 + ((self.cc >> 7) & 0xF)); + let mut pages = 0u64; + for i in 0..entries as usize { + // SAFETY: the driver handed a host pointer to + // `entries` descriptors it owns. + let d = unsafe { &*((list as *const HmbDescriptor).add(i)) }; + if d.badd % mps != 0 || d.bsize == 0 { + pages = u64::MAX; + break; + } + pages += u64::from(d.bsize); + } + let hmmin_pages = (u64::from(self.hmmin) << 12) / mps; + if pages == u64::MAX + || pages != u64::from(sqe.cdw12) + || pages < hmmin_pages + { + (0x02, 0) + } else { + self.hmb = Some((sqe.cdw12, entries)); + (0, 0) + } + } + } + } + _ => (0, 0), + } + } + x if x == AdminOpcode::CreateIoCq as u8 => { + let qid = (sqe.cdw10 & 0xFFFF) as u16; + let depth = ((sqe.cdw10 >> 16) & 0xFFFF) as u16 + 1; + self.ensure_io_queue(qid).cq_iova = sqe.prp1; + self.ensure_io_queue(qid).depth = depth; + (0, 0) + } + x if x == AdminOpcode::CreateIoSq as u8 => { + let qid = (sqe.cdw10 & 0xFFFF) as u16; + let size = ((sqe.cdw10 >> 16) & 0xFFFF) as u16 + 1; + let cqid = (sqe.cdw11 >> 16) as u16; + // The bound CQ must already exist, as a controller demands. + let cq_exists = cqid != 0 + && self + .io_queues + .get(cqid as usize - 1) + .is_some_and(|q| q.cq_iova != 0); + if !cq_exists { + return (0x01, 0); + } + let q = self.ensure_io_queue(qid); + q.sq_iova = sqe.prp1; + q.sq_depth = size; + q.cqid = cqid; + (0, 0) + } + x if x == AdminOpcode::DeleteIoSq as u8 || x == AdminOpcode::DeleteIoCq as u8 => (0, 0), + // Abort: a command this device is still holding (a withheld + // completion) is aborted — its own completion posts now as + // *Command Abort Requested* (SCT 0, SC 07h) and DW0 bit 0 is + // clear. Anything else already completed or never existed: DW0 + // bit 0 set, "not aborted", as a real controller answers. + x if x == AdminOpcode::Abort as u8 => { + let sqid = (sqe.cdw10 & 0xFFFF) as u16; + let cid = (sqe.cdw10 >> 16) as u16; + let victim = (sqid as usize) + .checked_sub(1) + .and_then(|idx| { + self.held + .iter() + .position(|&(q, c, _)| q == idx && c == cid) + .map(|at| (idx, at)) + }); + let aborted = if let Some((idx, at)) = victim { + self.held.remove(at); + self.publish_io_cqe(idx, cid, 0x07); + true + } else { + false + }; + self.aborts.push((sqid, cid, aborted)); + (0, u32::from(!aborted)) + } + _ => (0x01, 0), // Invalid Opcode + } + } + + /// Process the I/O SQ of queue `qid` up to `new_tail` (called on its SQ + /// doorbell write): read each command, move disk bytes per its PRP, + /// and publish a completion on its CQ (or hold it back while the + /// swallow knob is armed — the work is done either way). + fn process_io(&mut self, qid: u16, new_tail: u16) { + let idx = qid as usize - 1; + if idx >= self.io_queues.len() { + return; + } + loop { + let q = self.io_queues[idx]; + let sq_depth = if q.sq_depth != 0 { q.sq_depth } else { q.depth }; + if sq_depth == 0 || q.sq_head == new_tail { + break; + } + // SAFETY: sq_iova is a host pointer to `depth * 64` bytes; + // sq_head < depth. + let sqe: NvmeSqe = unsafe { + (q.sq_iova as *const NvmeSqe) + .add(q.sq_head as usize) + .read_volatile() + }; + let sc = self.dispatch_io(&sqe); + + // The command is consumed whether or not its CQE is withheld. + let e = &mut self.io_queues[idx]; + e.sq_head = (e.sq_head + 1) % sq_depth; + + if self.swallow > 0 { + self.swallow -= 1; + self.held.push((idx, sqe.cid(), sc)); + } else { + self.publish_io_cqe(idx, sqe.cid(), sc); + } + } + } + + /// Publish one I/O completion on queue `idx`'s CQ. + /// Post a completion for SQ index `idx` into the CQ that SQ is bound to + /// — its own, for a queue pair; a shared one, for a queue group — with + /// the SQ's id and head, as a controller reports them. + fn publish_io_cqe(&mut self, idx: usize, cid: u16, sc: u8) { + let sq = self.io_queues[idx]; + let cq_idx = if sq.cqid != 0 { sq.cqid as usize - 1 } else { idx }; + let q = self.io_queues[cq_idx]; + let cqe = NvmeCqe { + result: 0, + rsvd: 0, + sq_head: sq.sq_head, + sq_id: idx as u16 + 1, + cid, + status: ((sc as u16) << 1) | (q.cq_phase as u16), + }; + // SAFETY: cq_iova is a host pointer to `depth * 16` bytes; + // cq_tail < depth. + unsafe { + (q.cq_iova as *mut NvmeCqe) + .add(q.cq_tail as usize) + .write_volatile(cqe); + } + let e = &mut self.io_queues[cq_idx]; + e.cq_tail = (e.cq_tail + 1) % e.depth; + if e.cq_tail == 0 { + e.cq_phase = !e.cq_phase; + } + } + + /// Execute one I/O command, returning its status code. Read/Write + /// honor the command's **PRP** exactly (the driver's `prp::build` + /// output is validated here), moving `blocks` from `disk[lba..]`. + /// Read/Write geometry from CDW10..12 — **valid only for those + /// opcodes** (CDW meanings are per-command: DSM's CDW10 is a range + /// count and CDW11 its attributes — decoding those as an LBA once + /// produced a phantom out-of-range refusal here). + fn rw_geometry(&self, sqe: &NvmeSqe) -> Option<(usize, usize)> { + let lba = sqe.cdw10 as u64 | ((sqe.cdw11 as u64) << 32); + let blocks = (sqe.cdw12 & 0xFFFF) + 1; + let size = blocks as usize * MOCK_BLOCK_SIZE as usize; + let disk_off = (lba as usize).checked_mul(MOCK_BLOCK_SIZE as usize)?; + (disk_off + size <= self.disk.len()).then_some((disk_off, size)) + } + + fn dispatch_io(&mut self, sqe: &NvmeSqe) -> u8 { + use crate::commands::IoOpcode; + match sqe.opcode() { + x if x == IoOpcode::Read as u8 => { + let Some((disk_off, size)) = self.rw_geometry(sqe) else { + return 0x80; // LBA Out of Range (NVM command set) + }; + self.prp_walk(sqe.prp1, sqe.prp2, size, |page, poff, n| { + // disk → device page. + let src = &self.disk[disk_off + poff..disk_off + poff + n]; + // SAFETY: `page` is a device (host) buffer span inside + // the configured NVMe memory page. + unsafe { core::ptr::copy_nonoverlapping(src.as_ptr(), page, n) }; + }); + 0 + } + x if x == IoOpcode::Write as u8 => { + let Some((disk_off, size)) = self.rw_geometry(sqe) else { + return 0x80; + }; + // Snapshot the pages first (borrow discipline), then apply. + let mut staged = alloc::vec![0u8; size]; + self.prp_walk(sqe.prp1, sqe.prp2, size, |page, poff, n| { + // SAFETY: as above. + unsafe { core::ptr::copy_nonoverlapping(page, staged[poff..].as_mut_ptr(), n) }; + }); + self.disk[disk_off..disk_off + size].copy_from_slice(&staged); + 0 + } + x if x == IoOpcode::Flush as u8 => 0, + x if x == IoOpcode::DatasetManagement as u8 => { + // Deallocate: read the (nr + 1)-entry DsmRange table via + // PRP1 and record each range (zeroing the blocks — the + // deterministic deallocate the spec's DLFEAT=001 offers). + let nr = (sqe.cdw10 & 0xFF) as usize + 1; + for i in 0..nr { + // SAFETY: prp1 is a host pointer to the range table. + let r: crate::commands::DsmRange = unsafe { + (sqe.prp1 as *const crate::commands::DsmRange) + .add(i) + .read_volatile() + }; + let off = r.slba as usize * MOCK_BLOCK_SIZE as usize; + let len = r.nlb as usize * MOCK_BLOCK_SIZE as usize; + if off + len <= self.disk.len() { + self.disk[off..off + len].fill(0); + } + self.deallocated.push((r.slba, r.nlb)); + } + 0 + } + _ => 0x01, + } + } + + /// Walk the PRP-described device pages of a `size`-byte transfer, + /// calling `f(page_ptr, transfer_offset, page_bytes)` per page — the + /// mock's reading of `prp1` / `prp2` (direct or list). The driver's + /// PRP1 may start at an offset within the configured memory page. + fn prp_walk( + &self, + prp1: u64, + prp2: u64, + size: usize, + mut f: impl FnMut(*mut u8, usize, usize), + ) { + let page = 1usize << (12 + ((self.cc >> 7) & 0x0f)); + let mut off = 0; + // Page 0 via PRP1. + let first_remainder = page - (prp1 as usize & (page - 1)); + let n0 = size.min(first_remainder); + f(prp1 as *mut u8, 0, n0); + off += n0; + if off >= size { + return; + } + let remaining_pages = (size - off).div_ceil(page); + if remaining_pages == 1 { + // PRP2 is the second page directly. + f(prp2 as *mut u8, off, size - off); + return; + } + // PRP2 points at a list of page addresses. + for i in 0..remaining_pages { + // SAFETY: prp2 is a host pointer to a list page of `>= remaining_pages` u64s. + let address = unsafe { *(prp2 as *const u64).add(i) }; + let n = (size - off).min(page); + f(address as *mut u8, off, n); + off += n; + } + } + + fn ensure_io_queue(&mut self, qid: u16) -> &mut IoQueueState { + let idx = qid as usize - 1; + if idx >= self.io_queues.len() { + self.io_queues.resize( + idx + 1, + IoQueueState { + cq_phase: true, + ..Default::default() + }, + ); + } + &mut self.io_queues[idx] + } + + fn write_identify_controller(&self, prp1: u64) { + // SAFETY: prp1 is a host pointer to the 4 KiB identify buffer. + let idc = unsafe { &mut *(prp1 as *mut IdentifyController) }; + *idc = unsafe { core::mem::zeroed() }; + idc.vid = 0x1B36; // QEMU + idc.mdts = 5; // 4096 << 5 = 128 KiB + idc.nn = 1; + // Identify Controller ONCS bit 2: Dataset Management supported. + idc.oncs = 1 << 2; + idc.hmpre = self.hmpre; + idc.hmmin = self.hmmin; + idc.hmmaxd = self.hmmaxd; + idc.mn[..5].copy_from_slice(b"kmock"); + } + + fn write_identify_namespace(&self, prp1: u64) { + // SAFETY: as above. + let idn = unsafe { &mut *(prp1 as *mut IdentifyNamespace) }; + *idn = unsafe { core::mem::zeroed() }; + idn.nsze = MOCK_BLOCK_COUNT; + idn.ncap = MOCK_BLOCK_COUNT; + idn.flbas = 0; // lbaf[0] + idn.lbaf[0].lbads = self.lbads; // 12 ⇒ 4096, unless the knob turned + } +} + +/// The mock controller — clone it to hand `Controller::bring_up` the +/// register seam while keeping a handle to inspect device state. +#[derive(Clone)] +pub struct MockNvmeDevice { + state: Rc>, +} + +impl MockNvmeDevice { + /// A fresh mock controller (disabled, admin queue unprogrammed). + pub fn new() -> Self { + Self { + state: Rc::new(RefCell::new(DeviceState::new())), + } + } + + /// Advertise one exact minimum/maximum NVMe memory-page encoding. + pub fn with_mps(self, mps: u8) -> Option { + if mps > 15 { + return None; + } + let mut state = self.state.borrow_mut(); + state.cap &= !(0xff << 48); + state.cap |= u64::from(mps) << 48; + state.cap |= u64::from(mps) << 52; + drop(state); + Some(self) + } + + /// A fresh DMA pool (independent of the device — buffers are plain + /// host memory the device reads by IOVA). + pub fn pool(&self) -> MockPool { + MockPool + } + + /// How many I/O queues the driver has created (test probe). + pub fn io_queue_count(&self) -> usize { + self.state.borrow().io_queues.len() + } + + /// Commands submission queue `qid` has consumed — its device-side head + /// (test probe; wraps at the queue's depth). + pub fn sq_consumed(&self, qid: u16) -> Option { + let state = self.state.borrow(); + state + .io_queues + .get(usize::from(qid).checked_sub(1)?) + .map(|queue| queue.sq_head) + } + + /// The completion queue submission queue `qid` was created bound to + /// (test probe); `None` for a qid never created as an SQ. + pub fn sq_binding(&self, qid: u16) -> Option { + let state = self.state.borrow(); + let queue = state.io_queues.get(usize::from(qid).checked_sub(1)?)?; + (queue.cqid != 0).then_some(queue.cqid) + } + + /// I/O submission doorbells observed by the mock controller. + pub fn io_sq_doorbells(&self) -> u64 { + self.state.borrow().io_sq_doorbells + } + + /// I/O completion-head doorbells observed by the mock controller. + pub fn io_cq_doorbells(&self) -> u64 { + self.state.borrow().io_cq_doorbells + } + + /// A copy of the backing disk at `lba` for `blocks` — the expected + /// bytes a read returns (test oracle). + pub fn disk_snapshot(&self, lba: u64, blocks: u32) -> Vec { + let off = lba as usize * MOCK_BLOCK_SIZE as usize; + let len = blocks as usize * MOCK_BLOCK_SIZE as usize; + self.state.borrow().disk[off..off + len].to_vec() + } + + /// Ranges the driver deallocated via DSM (test probe). + pub fn deallocated(&self) -> Vec<(u64, u32)> { + self.state.borrow().deallocated.clone() + } + + /// Overwrite the backing disk at `lba` (test fixture writer — how a + /// GPT lands on the mock disk for discovery tests). + pub fn disk_write(&self, lba: u64, bytes: &[u8]) { + let off = lba as usize * MOCK_BLOCK_SIZE as usize; + self.state.borrow_mut().disk[off..off + bytes.len()].copy_from_slice(bytes); + } + + /// Advertise `2^lbads`-byte blocks from the next Identify Namespace + /// on (test knob — geometry-validation coverage; the I/O engine is + /// untouched). + pub fn set_lbads(&self, lbads: u8) { + self.state.borrow_mut().lbads = lbads; + } + + /// Advertise a Host Memory Buffer appetite from the next Identify + /// Controller on, in the spec's 4 KiB units (test knob — a DRAM-less + /// controller). + pub fn set_hmb(&self, preferred_units: u32, min_units: u32, max_descriptors: u16) { + let mut state = self.state.borrow_mut(); + state.hmpre = preferred_units; + state.hmmin = min_units; + state.hmmaxd = max_descriptors; + } + + /// The host memory buffer the driver enabled — `(HSIZE in memory + /// pages, descriptor count)` — or `None` (test probe). + pub fn hmb(&self) -> Option<(u32, u32)> { + self.state.borrow().hmb + } + + /// Set Features 0x0D disables the controller accepted (test probe). + pub fn hmb_disables(&self) -> u32 { + self.state.borrow().hmb_disables + } + + /// Advertise weighted round robin with urgent priority (`CAP.AMS` bit + /// 17), as a controller that implements it does. Call before bring-up. + pub fn advertise_weighted_arbitration(&self) { + self.state.borrow_mut().cap |= 1 << 17; + } + + /// The controller configuration word last written (test probe). + pub fn cc(&self) -> u32 { + self.state.borrow().cc + } + + /// The last Arbitration CDW11 the device accepted (test probe). + pub fn arbitration(&self) -> Option { + self.state.borrow().arbitration + } + + /// Aborts the device answered, as `(sqid, cid, aborted)` (test probe). + /// Controller resets taken (`CC.EN` cleared while enabled). + pub fn resets(&self) -> u32 { + self.state.borrow().resets + } + + pub fn aborts(&self) -> Vec<(u16, u16, bool)> { + self.state.borrow().aborts.clone() + } + + /// Withhold the next `n` I/O completions: each command executes, but + /// its CQE arrives only at [`release_swallowed`](Self::release_swallowed) + /// — the late-completing device the driver's timeout discipline must + /// survive (test knob). + pub fn swallow_completions(&self, n: u32) { + self.state.borrow_mut().swallow += n; + } + + /// Publish every withheld completion — the late arrivals land now. + pub fn release_swallowed(&self) { + let mut s = self.state.borrow_mut(); + let held: Vec<(usize, u16, u8)> = core::mem::take(&mut s.held); + for (idx, cid, sc) in held { + s.publish_io_cqe(idx, cid, sc); + } + } +} + +impl Default for MockNvmeDevice { + fn default() -> Self { + Self::new() + } +} + +impl crate::seam::Regs for MockNvmeDevice { + fn read32(&self, off: usize) -> u32 { + let s = self.state.borrow(); + match off { + REG_CAP => s.cap as u32, + x if x == REG_CAP + 4 => (s.cap >> 32) as u32, + REG_VS => MOCK_VS, + REG_CC => s.cc, + REG_CSTS => s.csts, + _ => 0, + } + } + + fn write32(&self, off: usize, val: u32) { + let mut s = self.state.borrow_mut(); + match off { + REG_CC => { + // A controller reset (`CC.EN` cleared while enabled, base spec + // §3.7.2): every queue and every command in flight is gone — + // held completions never post — and the admin queue, the + // I/O queues and the host memory buffer must be set up anew. + if s.cc & CC_EN != 0 && val & CC_EN == 0 { + s.admin_sq_head = 0; + s.admin_cq_tail = 0; + s.admin_cq_phase = true; + s.io_queues.clear(); + s.held.clear(); + s.hmb = None; + s.resets += 1; + } + s.cc = val; + if val & CC_EN != 0 { + s.csts |= CSTS_RDY; + } else { + s.csts &= !CSTS_RDY; + } + // Shutdown notification: the mock has nothing to flush, so + // any SHN value reports shutdown complete immediately (and + // clearing SHN clears the report). + if val & CC_SHN_MASK != 0 { + s.csts = (s.csts & !CSTS_SHST_MASK) | CSTS_SHST_COMPLETE; + } else { + s.csts &= !CSTS_SHST_MASK; + } + } + REG_AQA => { + // Admin CQ size (0-based) in 27:16; SQ size in 15:0. + s.admin_depth = ((val >> 16) & 0xFFF) as u16 + 1; + } + REG_ASQ => s.asq_iova = (s.asq_iova & !0xFFFF_FFFF) | val as u64, + x if x == REG_ASQ + 4 => s.asq_iova = (s.asq_iova & 0xFFFF_FFFF) | ((val as u64) << 32), + REG_ACQ => s.acq_iova = (s.acq_iova & !0xFFFF_FFFF) | val as u64, + x if x == REG_ACQ + 4 => s.acq_iova = (s.acq_iova & 0xFFFF_FFFF) | ((val as u64) << 32), + // Doorbells (0x1000+): decode (qid, SQ-vs-CQ) from the offset + // at the mock's 4-byte stride (DSTRD 0). An SQ-tail write + // drives that queue's engine; a CQ-head write is informational + // (the driver's bookkeeping — the mock needs nothing). + off if off >= REG_DOORBELL_BASE => { + let dbl = (off - REG_DOORBELL_BASE) / 4; + let qid = (dbl / 2) as u16; + let is_sq = dbl % 2 == 0; + if is_sq { + if qid == 0 { + s.process_admin(val as u16); + } else { + s.io_sq_doorbells = s.io_sq_doorbells.saturating_add(1); + s.process_io(qid, val as u16); + } + } else if qid != 0 { + s.io_cq_doorbells = s.io_cq_doorbells.saturating_add(1); + } + } + _ => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Controller; + + /// Abort is advisory and the victim keeps its own completion: aborting a + /// command the device still holds makes that command complete as + /// *Command Abort Requested* (SC 07h) and the Abort reply says so, while + /// aborting one that already completed answers "not aborted" and posts + /// nothing. The queue sees exactly one completion per command either way + /// — the invariant a deadline path relies on to know when a buffer is + /// free. + #[test] + fn abort_completes_a_held_command_and_refuses_a_finished_one() { + use crate::controller::AbortOutcome; + use kore_driver::block::CountedPacer; + + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + ctrl.identify(16, || {}).unwrap(); + let ns = ctrl.namespace(); + ctrl.set_num_queues(1, 16, || {}).unwrap(); + let mut qp = ctrl.create_io_qpair(1, 8, None, 16, || {}).unwrap(); + + let buf = ctrl.pool().alloc(ns.block_size as usize).unwrap(); + dev.swallow_completions(1); + let held = qp + .submit_one(ns.read(3, 1, buf.iova(), None).unwrap(), 7, ctrl.regs()) + .unwrap(); + assert!(qp.poll(ctrl.regs()).is_none(), "the device holds it"); + + let mut pacer = CountedPacer::new(16, || {}); + assert_eq!(ctrl.abort_paced(1, held, &mut pacer), Ok(AbortOutcome::Aborted)); + let c = qp.poll(ctrl.regs()).expect("the victim posts its own completion"); + assert_eq!((c.cid, c.cookie), (held, 7)); + assert!(!c.is_success()); + assert_eq!(c.cqe.status_code(), 0x07, "Command Abort Requested"); + assert!(qp.poll(ctrl.regs()).is_none(), "exactly one completion"); + + // Already complete: nothing to abort, nothing posted. + let done = qp + .submit_one(ns.read(4, 1, buf.iova(), None).unwrap(), 8, ctrl.regs()) + .unwrap(); + assert!(qp.poll(ctrl.regs()).expect("completes").is_success()); + assert_eq!( + ctrl.abort_paced(1, done, &mut pacer), + Ok(AbortOutcome::NotAborted) + ); + assert!(qp.poll(ctrl.regs()).is_none()); + assert_eq!(dev.aborts(), [(1, held, true), (1, done, false)]); + } + + /// The Arbitration feature reaches the device with burst and weights + /// in their fields; an exponent the field cannot carry never leaves the + /// driver. + #[test] + fn arbitration_is_programmed_and_an_unencodable_burst_refused() { + use crate::commands::{Arbitration, ArbitrationBurst}; + use kore_driver::block::CountedPacer; + + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + let mut pacer = CountedPacer::new(16, || {}); + let arbitration = Arbitration { + burst: ArbitrationBurst::Pow2(0), + high: 3, + medium: 2, + low: 1, + }; + assert_eq!(ctrl.set_arbitration_paced(arbitration, &mut pacer), Ok(())); + assert_eq!(dev.arbitration(), Some(0x0302_0100)); + + let refused = Arbitration { + burst: ArbitrationBurst::Pow2(9), + ..arbitration + }; + assert_eq!( + ctrl.set_arbitration_paced(refused, &mut pacer), + Err(crate::Error::Geometry) + ); + assert_eq!(dev.arbitration(), Some(0x0302_0100), "nothing was sent"); + + // Distinct SQ/CQ counts travel 0-based and come back as granted. + assert_eq!(ctrl.set_queue_counts_paced(3, 1, &mut pacer), Ok((3, 1))); + assert_eq!( + ctrl.set_queue_counts_paced(0, 1, &mut pacer), + Err(crate::Error::Geometry) + ); + } + + /// The N1a thesis: the whole bring-up + identify sequence runs over a + /// host mock — no hardware — and reads back the device's advertised + /// geometry. + #[test] + fn bring_up_and_identify_over_the_mock() { + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}) + .expect("controller comes up over the mock"); + + assert_eq!(ctrl.caps().max_queue_entries(), 64); + assert_eq!(ctrl.info().version, MOCK_VS); + + let info = ctrl.identify(16, || {}).expect("identify"); + assert_eq!(info.num_namespaces, 1); + assert_eq!(info.mdts_bytes, 4096 << 5, "MDTS = 128 KiB"); + assert_eq!(info.block_size, MOCK_BLOCK_SIZE); + assert_eq!(info.block_count, MOCK_BLOCK_COUNT); + } + + #[test] + fn larger_memory_page_aligns_every_queue_and_scales_mdts() { + let dev = MockNvmeDevice::new().with_mps(1).unwrap(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}) + .expect("8 KiB controller comes up"); + assert_eq!(ctrl.page().bytes(), 8192); + { + let state = dev.state.borrow(); + assert_eq!(state.asq_iova & 0x1fff, 0); + assert_eq!(state.acq_iova & 0x1fff, 0); + } + + let info = ctrl.identify(16, || {}).expect("identify at 8 KiB MPS"); + assert_eq!(info.mdts_bytes, 8192 << 5); + assert_eq!(ctrl.set_num_queues(1, 16, || {}), Ok((1, 1))); + let qpair = ctrl + .create_io_qpair(1, 8, None, 16, || {}) + .expect("I/O queues align to CC.MPS"); + assert_eq!(qpair.sq_iova().0 & 0x1fff, 0); + assert_eq!(qpair.cq_iova().0 & 0x1fff, 0); + } + + /// Preparation is the DMA-publication cut: it must stop a previously + /// enabled controller before allocating either admin queue, program both + /// queue pointers while halted, and leave `CC.EN` clear until `start`. + /// Weighted round robin is asked for, and written into `CC.AMS`, only + /// when `CAP.AMS` offers it; otherwise the controller starts under plain + /// round robin and says so. + #[test] + fn weighted_arbitration_starts_only_where_the_controller_offers_it() { + use crate::regs::CC_AMS_WRR; + use crate::seam::Regs as _; + + let plain = MockNvmeDevice::new(); + let ctrl = Controller::prepare(plain.clone(), plain.pool(), 16, || {}) + .expect("prepare") + .with_weighted_arbitration() + .start(16, || {}) + .expect("start"); + assert!(!ctrl.weighted_arbitration(), "no CAP.AMS: plain round robin"); + assert_eq!(plain.read32(REG_CC) & CC_AMS_WRR, 0); + + let weighted = MockNvmeDevice::new(); + weighted.advertise_weighted_arbitration(); + let ctrl = Controller::prepare(weighted.clone(), weighted.pool(), 16, || {}) + .expect("prepare") + .with_weighted_arbitration() + .start(16, || {}) + .expect("start"); + assert!(ctrl.weighted_arbitration()); + assert_ne!(weighted.cc() & CC_AMS_WRR, 0, "CC.AMS names weighted round robin"); + + let unasked = MockNvmeDevice::new(); + unasked.advertise_weighted_arbitration(); + let ctrl = Controller::prepare(unasked.clone(), unasked.pool(), 16, || {}) + .expect("prepare") + .start(16, || {}) + .expect("start"); + assert!(!ctrl.weighted_arbitration(), "offered but not asked for"); + assert_eq!(unasked.cc() & CC_AMS_WRR, 0); + } + + #[test] + fn prepare_programs_admin_queues_while_halted_then_start_enables() { + struct HaltCheckedPool(MockNvmeDevice); + + impl DmaPool for HaltCheckedPool { + type Buf = MockBuf; + + fn alloc(&self, bytes: usize) -> Option { + use crate::seam::Regs as _; + assert_eq!( + self.0.read32(REG_CC) & CC_EN, + 0, + "CC.EN must be clear before allocating controller DMA" + ); + assert_eq!( + self.0.read32(REG_CSTS) & CSTS_RDY, + 0, + "the controller must acknowledge !RDY before allocation" + ); + MockPool.alloc(bytes) + } + } + + use crate::seam::Regs as _; + let dev = MockNvmeDevice::new(); + dev.write32(REG_CC, CC_EN); + assert_ne!(dev.read32(REG_CSTS) & CSTS_RDY, 0, "fixture starts live"); + + let prepared = Controller::prepare(dev.clone(), HaltCheckedPool(dev.clone()), 16, || {}) + .expect("prepare"); + + { + let state = dev.state.borrow(); + assert_eq!(state.cc & CC_EN, 0, "prepare does not enable"); + assert_eq!(state.csts & CSTS_RDY, 0, "prepare remains halted"); + assert_eq!(state.admin_depth, crate::controller::ADMIN_DEPTH); + assert_ne!(state.asq_iova, 0, "admin SQ programmed while halted"); + assert_ne!(state.acq_iova, 0, "admin CQ programmed while halted"); + } + + let ctrl = prepared.start(16, || {}).expect("start"); + assert_ne!(dev.read32(REG_CC) & CC_EN, 0, "start sets CC.EN"); + assert_ne!(dev.read32(REG_CSTS) & CSTS_RDY, 0, "start observes RDY"); + assert_eq!(ctrl.info().version, MOCK_VS); + } + + /// The quiesce path: an orderly shutdown notifies `SHN`, observes + /// shutdown complete, disables, and observes `!RDY` — then the seam + /// comes back out whole ([`Controller::into_seam`]), which is what the + /// class binding's `unbind` restores into the common resource set. + #[test] + fn shutdown_completes_and_releases_the_seam() { + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}) + .expect("controller comes up over the mock"); + ctrl.shutdown(16, || {}).expect("orderly shutdown"); + use crate::seam::Regs as _; + let csts = dev.read32(crate::regs::REG_CSTS); + assert_eq!(csts & CSTS_RDY, 0, "disabled after shutdown"); + let (regs, _pool) = ctrl.into_seam(); + assert_eq!( + regs.read32(crate::regs::REG_VS), + MOCK_VS, + "the released register seam still decodes" + ); + } + + /// The N1b thesis: create an I/O qpair and **read a real LBA** over + /// the mock, zero-copy — the bytes land in the caller's DMA buffer and + /// match the device's backing disk, with the PRP the driver built + /// honored by the device. + #[test] + fn create_io_qpair_and_read_an_lba() { + let dev = MockNvmeDevice::new(); + let pool = dev.pool(); + let mut ctrl = Controller::bring_up(dev.clone(), pool, 16, || {}).unwrap(); + ctrl.identify(16, || {}).unwrap(); + let ns = ctrl.namespace(); + assert_eq!(ns.block_size, MOCK_BLOCK_SIZE); + + ctrl.set_num_queues(4, 16, || {}).unwrap(); + let mut qp = ctrl + .create_io_qpair(1, 64, Some(0), 16, || {}) + .expect("I/O qpair"); + assert_eq!(dev.io_queue_count(), 1, "the device registered the queue"); + + // Read one block (LBA 5) into a caller DMA buffer — single PRP. + let buf = ctrl.pool().alloc(ns.block_size as usize).unwrap(); + let cmd = ns.read(5, 1, buf.iova(), None).unwrap(); + let cid = qp.submit_one(cmd, 0xF00D, ctrl.regs()).unwrap(); + + // Poll to completion (the mock completed synchronously on the ring). + let mut done = None; + for _ in 0..8 { + if let Some(c) = qp.poll(ctrl.regs()) { + done = Some(c); + break; + } + } + let c = done.expect("completion arrives"); + assert_eq!(c.cid, cid); + assert_eq!(c.cookie, 0xF00D, "cookie correlation"); + assert!(c.is_success()); + + // The bytes are the device's disk at LBA 5 — zero-copy into `buf`. + let disk = dev.disk_snapshot(5, 1); + // SAFETY: `buf` is `block_size` bytes the device just wrote. + let got = unsafe { core::slice::from_raw_parts(buf.cpu(), ns.block_size as usize) }; + assert_eq!(got, &disk[..], "read returned the backing bytes"); + } + + /// A two-block read spans two pages (PRP1 + PRP2 direct); a write + /// round-trips through the same PRP path — proving `prp::build` against + /// a real device consumer. + #[test] + fn two_page_read_and_write_round_trip_via_prp() { + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + ctrl.identify(16, || {}).unwrap(); + let ns = ctrl.namespace(); + let mut qp = ctrl.create_io_qpair(1, 64, Some(0), 16, || {}).unwrap(); + + // Read 2 blocks (8 KiB ⇒ PRP1 + PRP2-direct) at LBA 10. + let buf = ctrl.pool().alloc(ns.bytes(2) as usize).unwrap(); + let cmd = ns.read(10, 2, buf.iova(), None).unwrap(); + assert_ne!(cmd.prp2, 0, "two pages use PRP2 directly"); + qp.submit_one(cmd, 0, ctrl.regs()).unwrap(); + while qp.poll(ctrl.regs()).is_none() {} + let got = unsafe { core::slice::from_raw_parts(buf.cpu(), ns.bytes(2) as usize) }; + assert_eq!(got, &dev.disk_snapshot(10, 2)[..]); + + // Write a pattern to LBA 20, then read it back. + let wbuf = ctrl.pool().alloc(ns.block_size as usize).unwrap(); + unsafe { core::ptr::write_bytes(wbuf.cpu(), 0x5A, ns.block_size as usize) }; + let wcmd = ns.write(20, 1, wbuf.iova(), None).unwrap(); + qp.submit_one(wcmd, 0, ctrl.regs()).unwrap(); + while qp.poll(ctrl.regs()).is_none() {} + assert!( + dev.disk_snapshot(20, 1).iter().all(|&b| b == 0x5A), + "write reached the disk" + ); + } + + /// **Breadth on the wire**: a PRP-list read (8 pages through the SIMD + /// iota + the device's list walk), batched submissions with ONE + /// doorbell ring, a second qpair interleaved (the per-core shape), + /// a device-surfaced error (LBA beyond the disk), and the honest + /// counters recording all of it. + #[test] + fn prp_list_io_batched_doorbell_two_qpairs_and_counters() { + use crate::prp::ListPage; + + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + ctrl.identify(16, || {}).unwrap(); + let ns = ctrl.namespace(); + let mut qp1 = ctrl.create_io_qpair(1, 64, Some(0), 16, || {}).unwrap(); + let mut qp2 = ctrl.create_io_qpair(2, 64, Some(1), 16, || {}).unwrap(); + assert_eq!(dev.io_queue_count(), 2, "two hardware queues live"); + + // An 8-page (32 KiB) read via a PRP list on qp1. + let data = ctrl.pool().alloc(8 * 4096).unwrap(); + let listpage = ctrl.pool().alloc(4096).unwrap(); + let mut entries = [0u64; 512]; + let cmd = ns + .read( + 8, + 8, + data.iova(), + Some(ListPage { + iova: listpage.iova(), + entries: &mut entries, + }), + ) + .unwrap(); + // The driver-built list is in driver memory; the DEVICE reads it + // from the list page — copy the entries into the DMA page (the + // driver-task flow writes them there directly). + unsafe { + core::ptr::copy_nonoverlapping(entries.as_ptr(), listpage.cpu() as *mut u64, 512) + }; + + // Batch on qp1 (no doorbell yet), plus a 1-block read on qp2. + qp1.submit(cmd, 0xAA).unwrap(); + let small = ctrl.pool().alloc(4096).unwrap(); + qp2.submit_one( + ns.read(3, 1, small.iova(), None).unwrap(), + 0xBB, + ctrl.regs(), + ) + .unwrap(); + qp1.ring(ctrl.regs()); // ONE ring for the batch + + while qp1.poll(ctrl.regs()).is_none() {} + while qp2.poll(ctrl.regs()).is_none() {} + + let got = unsafe { core::slice::from_raw_parts(data.cpu(), 8 * 4096) }; + assert_eq!( + got, + &dev.disk_snapshot(8, 8)[..], + "8 pages via the PRP list" + ); + let got2 = unsafe { core::slice::from_raw_parts(small.cpu(), 4096) }; + assert_eq!(got2, &dev.disk_snapshot(3, 1)[..], "qp2 interleaved"); + + // A read past the disk end surfaces the device's error status. + let bad = ns.read(ns.block_count - 1, 1, small.iova(), None).unwrap(); + qp1.submit_one(bad, 0, ctrl.regs()).unwrap(); + let c = loop { + if let Some(c) = qp1.poll(ctrl.regs()) { + break c; + } + }; + assert!(!c.is_success(), "mock disk is smaller than the namespace"); + + let (s1, s2) = (qp1.stats(), qp2.stats()); + assert_eq!((s1.submitted, s1.completed, s1.errors), (2, 2, 1)); + assert_eq!((s2.submitted, s2.completed, s2.errors), (1, 1, 0)); + } + + /// **DSM deallocate + SMART**: the discard path a swap backend rides + /// (ranges reach the device; blocks read back zeroed), and the health + /// log decodes. + #[test] + fn deallocate_and_smart_log() { + use crate::commands::DsmRange; + + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + ctrl.identify(16, || {}).unwrap(); + let ns = ctrl.namespace(); + let mut qp = ctrl.create_io_qpair(1, 64, Some(0), 16, || {}).unwrap(); + + // Deallocate LBAs 4..6 (two ranges). + let table = ctrl.pool().alloc(4096).unwrap(); + let ranges = [DsmRange::new(4, 1), DsmRange::new(5, 2)]; + unsafe { + core::ptr::copy_nonoverlapping( + ranges.as_ptr(), + table.cpu() as *mut DsmRange, + ranges.len(), + ) + }; + let cmd = ns.deallocate(table.iova(), 2).unwrap(); + qp.submit_one(cmd, 0, ctrl.regs()).unwrap(); + while qp.poll(ctrl.regs()).is_none() {} + assert_eq!(dev.deallocated(), alloc::vec![(4, 1), (5, 2)]); + assert!( + dev.disk_snapshot(5, 2).iter().all(|&b| b == 0), + "deallocated blocks read back zeroed" + ); + + // SMART decodes through the typed log. + let log = ctrl.smart(16, || {}).unwrap(); + assert_eq!(log.temp_celsius(), 36); + assert_eq!(log.percentage_used, 3); + assert_eq!( + crate::commands::SmartLog::counter(&log.data_units_read), + 1234 + ); + assert_eq!(crate::commands::SmartLog::counter(&log.power_cycles), 7); + } + + /// The MDTS guard refuses an oversized transfer before it reaches the + /// device (the mock advertises MDTS = 128 KiB). + #[test] + fn mdts_guard_refuses_oversized_transfers() { + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + ctrl.identify(16, || {}).unwrap(); + let ns = ctrl.namespace(); + assert_eq!(ns.mdts_bytes, 4096 << 5, "128 KiB advertised"); + // 33 blocks × 4 KiB = 132 KiB > 128 KiB. + assert!(matches!( + ns.read(0, 33, IovaAddr(0x1000), None), + Err(crate::Error::ExceedsMdts) + )); + } + + /// Bring-up actually drives the register protocol: a controller that + /// never asserts RDY (a mock that ignores CC.EN) times out rather than + /// hanging or falsely succeeding. + #[test] + fn a_controller_that_never_readies_times_out() { + struct DeadRegs { + cc: Rc>, + } + impl crate::seam::Regs for DeadRegs { + fn read32(&self, off: usize) -> u32 { + match off { + REG_CAP => MOCK_CAP as u32, + x if x == REG_CAP + 4 => (MOCK_CAP >> 32) as u32, + REG_VS => MOCK_VS, + REG_CC => self.cc.get(), + _ => 0, // CSTS always 0 ⇒ never ready + } + } + fn write32(&self, off: usize, value: u32) { + if off == REG_CC { + self.cc.set(value); + } + } + } + let cc = Rc::new(Cell::new(0)); + let r = Controller::bring_up(DeadRegs { cc: cc.clone() }, MockPool, 8, || {}); + assert!(matches!(r, Err(crate::Error::NotReady))); + assert_eq!( + cc.get() & CC_EN, + 0, + "a failed start best-effort disables before releasing its queues" + ); + } + + /// A capability host must retain both seams across a refused start so it + /// can restore BAR/pool custody before disabling bus mastering and + /// retiring mappings. The typed error makes dropping either seam before + /// that ownership decision impossible inside `PreparedController::start`. + #[test] + fn a_refused_start_returns_the_prepared_seams() { + struct HeldRegs { + cc: Rc>, + dropped: Rc>, + } + + impl Drop for HeldRegs { + fn drop(&mut self) { + self.dropped.set(true); + } + } + + impl crate::seam::Regs for HeldRegs { + fn read32(&self, off: usize) -> u32 { + match off { + REG_CAP => MOCK_CAP as u32, + x if x == REG_CAP + 4 => (MOCK_CAP >> 32) as u32, + REG_VS => MOCK_VS, + REG_CC => self.cc.get(), + _ => 0, + } + } + + fn write32(&self, off: usize, value: u32) { + if off == REG_CC { + self.cc.set(value); + } + } + } + + struct HeldPool { + dropped: Rc>, + } + + impl Drop for HeldPool { + fn drop(&mut self) { + self.dropped.set(true); + } + } + + impl DmaPool for HeldPool { + type Buf = MockBuf; + + fn alloc(&self, bytes: usize) -> Option { + MockPool.alloc(bytes) + } + } + + let cc = Rc::new(Cell::new(0)); + let regs_dropped = Rc::new(Cell::new(false)); + let pool_dropped = Rc::new(Cell::new(false)); + let prepared = Controller::prepare( + HeldRegs { + cc: cc.clone(), + dropped: regs_dropped.clone(), + }, + HeldPool { + dropped: pool_dropped.clone(), + }, + 8, + || {}, + ) + .expect("preparation succeeds while CSTS is halted"); + + let failure = match prepared.start(8, || {}) { + Ok(_) => panic!("a controller that never sets RDY cannot start"), + Err(failure) => failure, + }; + assert_eq!(failure.error(), crate::Error::NotReady); + assert_eq!(cc.get() & CC_EN, 0, "refused start rolls CC.EN back"); + assert!(!regs_dropped.get(), "the register seam rides back"); + assert!(!pool_dropped.get(), "the pool seam rides back"); + + let (error, regs, pool) = failure.into_seam(); + assert_eq!(error, crate::Error::NotReady); + assert!(!regs_dropped.get(), "recovery transfers register custody"); + assert!(!pool_dropped.get(), "recovery transfers pool custody"); + drop(regs); + drop(pool); + assert!(regs_dropped.get()); + assert!(pool_dropped.get()); + } +} diff --git a/native/drivers/nvme/src/namespace.rs b/native/drivers/nvme/src/namespace.rs new file mode 100644 index 0000000..c68ed2b --- /dev/null +++ b/native/drivers/nvme/src/namespace.rs @@ -0,0 +1,200 @@ +//! The **namespace** — `spdk_nvme_ns`: an addressable block store on the +//! controller (nsid + geometry), and the builder for its I/O commands. +//! +//! Kept a **pure command builder** (SPDK-shaped): a namespace holds +//! metadata and constructs [`NvmeSqe`]s; the [`Qpair`](crate::Qpair) is the +//! I/O engine that submits and polls them. Data rides **caller DMA +//! buffers** (zero-copy — the device reads/writes the frames the caller +//! owns), addressed by PRP at the controller's **configured page size** +//! ([`MemPage`], carried here so every transfer's math matches `CC.MPS`). + +use kore_abi::device::IovaAddr; + +use crate::commands::{DsmRange, NvmeSqe}; +use crate::prp::{self, ListPage, MemPage}; +use crate::{Error, Result}; + +// Re-assert the DSM table shape where the builder is (16 B × count). +const _: () = assert!(core::mem::size_of::() == 16); + +/// NVM Read/Write CDW12.NLB is a 16-bit zero-based field: one command can +/// encode at most 65,536 logical blocks, independent of MDTS and PRP reach. +pub const MAX_NLB_BLOCKS: u32 = u16::MAX as u32 + 1; + +/// One namespace on a controller. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Namespace { + /// Namespace id (1-based). + pub nsid: u32, + /// Logical block size in bytes. + pub block_size: u32, + /// Size in logical blocks. + pub block_count: u64, + /// The controller's configured memory page (PRP math). + pub page: MemPage, + /// The controller's MDTS in bytes (0 = unlimited) — transfers beyond + /// it refuse with [`Error::ExceedsMdts`] instead of reaching the + /// device as an invalid command. + pub mdts_bytes: u64, +} + +impl Namespace { + /// A namespace from its identify-derived geometry + the controller's + /// configured page + MDTS. + pub const fn new( + nsid: u32, + block_size: u32, + block_count: u64, + page: MemPage, + mdts_bytes: u64, + ) -> Self { + Self { + nsid, + block_size, + block_count, + page, + mdts_bytes, + } + } + + /// Byte length of `blocks` logical blocks. + pub const fn bytes(&self, blocks: u32) -> u64 { + blocks as u64 * self.block_size as u64 + } + + /// Build a **Read** command: `blocks` logical blocks from `lba` into + /// the caller's DMA buffer at `data_iova` (one contiguous device + /// window — the [`Dma`](crate::seam::Dma) contract), addressed by PRP. + /// A transfer spanning more than two pages needs `list` (a PRP-list + /// page); the range is bounds-checked against the namespace. + pub fn read( + &self, + lba: u64, + blocks: u32, + data_iova: IovaAddr, + list: Option>, + ) -> Result { + self.check_range(lba, blocks)?; + let p = prp::build(data_iova, self.bytes(blocks), self.page, list)?; + Ok(NvmeSqe::read(self.nsid, lba, (blocks - 1) as u16, p)) + } + + /// Build a **Write** command (mirror of [`read`](Self::read)). + pub fn write( + &self, + lba: u64, + blocks: u32, + data_iova: IovaAddr, + list: Option>, + ) -> Result { + self.check_range(lba, blocks)?; + let p = prp::build(data_iova, self.bytes(blocks), self.page, list)?; + Ok(NvmeSqe::write(self.nsid, lba, (blocks - 1) as u16, p)) + } + + /// Build a **Flush** command. + pub fn flush(&self) -> NvmeSqe { + NvmeSqe::flush(self.nsid) + } + + /// Build a **Deallocate** (Dataset Management, the discard/TRIM path + /// a swap backend rides): the `count`-entry [`DsmRange`] table sits in + /// the caller's DMA buffer at `ranges_iova` (`count` is 1-based here, + /// ≤ 256). + pub fn deallocate(&self, ranges_iova: IovaAddr, count: u16) -> Result { + if count == 0 || count > 256 { + return Err(Error::Qpair(crate::qpair::QpairError::BadDepth)); + } + Ok(NvmeSqe::dsm_deallocate( + self.nsid, + (count - 1) as u8, + ranges_iova, + )) + } + + /// `blocks ≥ 1`, `lba + blocks ≤ block_count`, and within MDTS. + fn check_range(&self, lba: u64, blocks: u32) -> Result<()> { + let ok = blocks >= 1 + && lba + .checked_add(blocks as u64) + .is_some_and(|end| end <= self.block_count); + if !ok { + return Err(Error::Qpair(crate::qpair::QpairError::BadDepth)); + } + if blocks > MAX_NLB_BLOCKS { + return Err(Error::Geometry); + } + if self.mdts_bytes != 0 && self.bytes(blocks) > self.mdts_bytes { + return Err(Error::ExceedsMdts); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::prp::PrpError; + + const PG: MemPage = MemPage::MIN; + + #[test] + fn read_builds_a_single_prp_command_within_range() { + let ns = Namespace::new(1, 4096, 0x1000, PG, 0); + let cmd = ns.read(0, 1, IovaAddr(0xdead_0000), None).unwrap(); + assert_eq!(cmd.opcode(), crate::commands::IoOpcode::Read as u8); + assert_eq!(cmd.nsid, 1); + assert_eq!(cmd.prp1, 0xdead_0000); + assert_eq!(cmd.prp2, 0, "one 4 KiB block is a single PRP"); + assert_eq!(cmd.cdw12, 0, "NLB is 0-based (1 block ⇒ 0)"); + } + + #[test] + fn out_of_range_and_zero_length_refuse() { + // 4 KiB blocks so a 1-block read stays a single PRP (isolates the + // range check from PRP sizing). + let ns = Namespace::new(1, 4096, 100, PG, 0); + assert!( + ns.read(100, 1, IovaAddr(0x1000), None).is_err(), + "first LBA past the end" + ); + assert!( + ns.read(99, 2, IovaAddr(0x1000), None).is_err(), + "spills past the end" + ); + assert!( + ns.read(0, 0, IovaAddr(0x1000), None).is_err(), + "zero blocks" + ); + assert!( + ns.read(99, 1, IovaAddr(0x1000), None).is_ok(), + "the last block fits" + ); + } + + #[test] + fn a_large_transfer_needs_a_list() { + // 4 KiB blocks, 4 blocks = 4 pages ⇒ needs a PRP list. + let ns = Namespace::new(1, 4096, 0x1000, PG, 0); + assert!(matches!( + ns.read(0, 4, IovaAddr(0x1000), None), + Err(Error::Prp(PrpError::ListNeeded)) + )); + } + + #[test] + fn a_larger_configured_page_relaxes_the_list_requirement() { + // The same 4-block (16 KiB) read fits two 8 KiB pages ⇒ no list — + // the namespace's page (from CC.MPS) governs, not a constant. + let ns = Namespace::new(1, 4096, 0x1000, MemPage::from_mps(1), 0); + let cmd = ns.read(0, 4, IovaAddr(0x4000), None).unwrap(); + assert_eq!(cmd.prp2, 0x6000, "PRP2-direct at 8 KiB pages"); + } + + #[test] + fn nlb_encoding_limit_is_independent_of_namespace_and_mdts() { + let ns = Namespace::new(1, 512, u64::MAX, PG, 0); + assert!(ns.check_range(0, MAX_NLB_BLOCKS).is_ok()); + assert_eq!(ns.check_range(0, MAX_NLB_BLOCKS + 1), Err(Error::Geometry)); + } +} diff --git a/native/drivers/nvme/src/prp.rs b/native/drivers/nvme/src/prp.rs new file mode 100644 index 0000000..3d95b59 --- /dev/null +++ b/native/drivers/nvme/src/prp.rs @@ -0,0 +1,325 @@ +//! PRP (Physical Region Page) scatter-gather — **spec facts** (NVMe base +//! spec §4.3), the device-address math a data transfer needs. +//! +//! Three generalities, all deliberate: +//! - **Typed addresses**: every device address is a +//! [`kore_abi::device::IovaAddr`] — "no phys in userland" is a *type*; raw +//! `u64` appears only inside the wire SQE. +//! - **Page-size parameterized**: the memory page is the controller's +//! *configured* `CC.MPS` (`2^(12+mps)`, [`MemPage`]), not a hardcoded +//! 4096 — a controller with `MPSMIN > 0` is handled correctly. +//! - **Contiguity is a contract, not an assumption**: [`build`] serves the +//! seam's contiguous-window buffers ([`Dma`](crate::seam::Dma)'s stated +//! contract) and computes trailing pages arithmetically; +//! [`build_scattered`] takes the page list explicitly for any backing +//! that cannot promise one window. The two entry points make the +//! assumption visible at every call site. +//! +//! The list fill is the core's one hot linear pass — vectorized with +//! `core::simd` (8 lanes of page-address iota per step). + +use core::simd::Simd; + +use kore_abi::device::IovaAddr; + +/// A validated NVMe memory page size — `2^(12 + mps)` bytes, the +/// controller's configured `CC.MPS`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MemPage { + bytes: u64, +} + +impl MemPage { + /// The spec minimum (4 KiB, `mps = 0`) — every controller's floor. + pub const MIN: Self = Self { bytes: 4096 }; + + /// From the `CC.MPS` field value (`0..=15`). + pub const fn from_mps(mps: u8) -> Self { + Self { + bytes: 1 << (12 + mps as u32), + } + } + + /// Page size in bytes. + pub const fn bytes(self) -> u64 { + self.bytes + } + + /// PRP entries one list page of this size holds (8 bytes each). + pub const fn prps_per_page(self) -> u64 { + self.bytes / 8 + } +} + +/// The two PRP fields a command carries (still typed; the conversion to +/// the wire's raw `u64` happens inside the SQE builders). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Prp { + /// PRP1 — the first (possibly page-offset) address. + pub prp1: IovaAddr, + /// PRP2 — the second page, a PRP-list pointer, or null. + pub prp2: IovaAddr, +} + +impl Prp { + const NULL: Self = Self { + prp1: IovaAddr(0), + prp2: IovaAddr(0), + }; +} + +/// Why a PRP build refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PrpError { + /// The transfer needs a PRP list but no list page was supplied. + ListNeeded, + /// The transfer needs more entries than one list page holds + /// (multi-list chaining is a later tier). + TooLarge, + /// A scattered build's page list ran short of the transfer size. + NotEnoughPages, +} + +/// A caller-supplied PRP-**list page** the builder fills when a transfer +/// spans more than two pages — a `DmaPool` buffer the driver keeps for +/// reuse (device-visible: `iova` is what the command's PRP2 points at, +/// `entries` is the CPU-writable view of the same page). +pub struct ListPage<'a> { + /// The list page's device address (becomes PRP2). + pub iova: IovaAddr, + /// CPU-writable entries of the same page. + pub entries: &'a mut [u64], +} + +/// Build the PRP fields for a `size`-byte transfer starting at +/// `buffer_iova`, within **one contiguous device-visible window** (the +/// [`Dma`](crate::seam::Dma) contract — trailing pages are computed +/// arithmetically from the base). +/// +/// - `size ≤ first-page-remainder`: single page, PRP2 = 0. +/// - two pages: PRP2 = the second page's address (no list). +/// - more: PRP2 points at `list`, filled with the trailing page addresses +/// (requires `list`; the tail must fit one list page). +pub fn build( + buffer_iova: IovaAddr, + size: u64, + page: MemPage, + list: Option>, +) -> Result { + if size == 0 { + return Ok(Prp::NULL); + } + let pg = page.bytes(); + let base = buffer_iova.0; + let offset = base & (pg - 1); + let first_page_bytes = pg - offset; + let prp1 = buffer_iova; + + if size <= first_page_bytes { + return Ok(Prp { + prp1, + prp2: IovaAddr(0), + }); + } + + let remaining = size - first_page_bytes; + let additional = remaining.div_ceil(pg); + let second_page = (base & !(pg - 1)) + pg; + + if additional == 1 { + return Ok(Prp { + prp1, + prp2: IovaAddr(second_page), + }); + } + + // A PRP list: the trailing pages, contiguous from the second page — + // a page-address iota, filled 8 lanes per step (the core's one hot + // linear pass, vectorized). + let list = list.ok_or(PrpError::ListNeeded)?; + let n = additional as usize; + if n > list.entries.len() || additional > page.prps_per_page() { + return Err(PrpError::TooLarge); + } + fill_page_iota(&mut list.entries[..n], second_page, pg); + Ok(Prp { + prp1, + prp2: list.iova, + }) +} + +/// Build the PRP fields for a transfer over an **explicit page list** — +/// the scatter-aware twin of [`build`], for any backing that is not one +/// contiguous device window. `pages` yields each page's device address +/// in transfer order; the first may be page-offset (its offset rides +/// PRP1, per spec), the rest must be page-aligned. +pub fn build_scattered( + mut pages: impl Iterator, + size: u64, + page: MemPage, + list: Option>, +) -> Result { + if size == 0 { + return Ok(Prp::NULL); + } + let pg = page.bytes(); + let first = pages.next().ok_or(PrpError::NotEnoughPages)?; + let first_page_bytes = pg - (first.0 & (pg - 1)); + if size <= first_page_bytes { + return Ok(Prp { + prp1: first, + prp2: IovaAddr(0), + }); + } + + let additional = (size - first_page_bytes).div_ceil(pg); + if additional == 1 { + let second = pages.next().ok_or(PrpError::NotEnoughPages)?; + return Ok(Prp { + prp1: first, + prp2: second, + }); + } + + let list = list.ok_or(PrpError::ListNeeded)?; + let n = additional as usize; + if n > list.entries.len() || additional > page.prps_per_page() { + return Err(PrpError::TooLarge); + } + for slot in list.entries.iter_mut().take(n) { + *slot = pages.next().ok_or(PrpError::NotEnoughPages)?.0; + } + Ok(Prp { + prp1: first, + prp2: list.iova, + }) +} + +/// Fill `out` with the page-address iota `start, start+pg, …` — 8 lanes +/// per step, scalar tail. +fn fill_page_iota(out: &mut [u64], start: u64, pg: u64) { + const LANES: usize = 8; + // [0,1,…,7]·pg once; each step splats the running base over it. + let steps = Simd::::from_array([0, 1, 2, 3, 4, 5, 6, 7]) * Simd::splat(pg); + let mut base = start; + let mut chunks = out.chunks_exact_mut(LANES); + for chunk in &mut chunks { + (Simd::splat(base) + steps).copy_to_slice(chunk); + base += pg * LANES as u64; + } + for slot in chunks.into_remainder() { + *slot = base; + base += pg; + } +} + +/// The largest single transfer one PRP list page describes at `page` +/// size (the first page plus a full list of trailing pages) — the MDTS +/// the driver caps I/O at before multi-list chaining. +pub const fn max_single_list_transfer(page: MemPage) -> u64 { + page.bytes() + page.prps_per_page() * page.bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + + const PG: MemPage = MemPage::MIN; + + #[test] + fn mem_page_decodes_mps() { + assert_eq!(MemPage::from_mps(0).bytes(), 4096); + assert_eq!(MemPage::from_mps(1).bytes(), 8192); + assert_eq!(MemPage::from_mps(0).prps_per_page(), 512); + } + + #[test] + fn single_page_needs_no_prp2() { + let p = build(IovaAddr(0x1000), 4096, PG, None).unwrap(); + assert_eq!(p.prp1, IovaAddr(0x1000)); + assert_eq!(p.prp2, IovaAddr(0)); + } + + #[test] + fn two_pages_use_prp2_directly_no_list() { + let p = build(IovaAddr(0x1000), 8192, PG, None).unwrap(); + assert_eq!( + p, + Prp { + prp1: IovaAddr(0x1000), + prp2: IovaAddr(0x2000) + } + ); + + // Offset into the first page still spills into the second. + let p = build(IovaAddr(0x1F00), 0x200, PG, None).unwrap(); + assert_eq!(p.prp1, IovaAddr(0x1F00)); + assert_eq!(p.prp2, IovaAddr(0x2000), "spilled 0x100 bytes"); + } + + #[test] + fn many_pages_fill_the_list_simd_and_point_prp2_at_it() { + let mut entries = [0u64; 512]; + let list = ListPage { + iova: IovaAddr(0x9000), + entries: &mut entries, + }; + // 21 pages: prp1 + a 20-entry list — crosses the 8-lane SIMD + // boundary twice with a scalar tail of 4. + let p = build(IovaAddr(0x1000), 21 * 4096, PG, Some(list)).unwrap(); + assert_eq!(p.prp2, IovaAddr(0x9000)); + for (i, &e) in entries[..20].iter().enumerate() { + assert_eq!(e, 0x2000 + (i as u64) * 4096, "iota entry {i}"); + } + assert_eq!(entries[20], 0, "unused entries untouched"); + } + + #[test] + fn larger_mps_changes_the_geometry() { + // 8 KiB pages: a 16 KiB transfer is exactly two pages ⇒ PRP2 + // direct, no list — where 4 KiB pages would need a list. + let pg8 = MemPage::from_mps(1); + let p = build(IovaAddr(0x4000), 16384, pg8, None).unwrap(); + assert_eq!(p.prp2, IovaAddr(0x6000), "second 8 KiB page"); + assert!(matches!( + build(IovaAddr(0x4000), 16384, PG, None), + Err(PrpError::ListNeeded), + )); + } + + #[test] + fn scattered_build_takes_explicit_pages() { + // Three non-contiguous pages — inexpressible by `build`. + let mut entries = [0u64; 512]; + let list = ListPage { + iova: IovaAddr(0xA000), + entries: &mut entries, + }; + let pages = [IovaAddr(0x1000), IovaAddr(0x7000), IovaAddr(0x3000)]; + let p = build_scattered(pages.into_iter(), 3 * 4096, PG, Some(list)).unwrap(); + assert_eq!(p.prp1, IovaAddr(0x1000)); + assert_eq!(p.prp2, IovaAddr(0xA000)); + assert_eq!(&entries[..2], &[0x7000, 0x3000], "the scattered tail"); + } + + #[test] + fn scattered_refuses_a_short_page_list() { + assert!(matches!( + build_scattered([IovaAddr(0x1000)].into_iter(), 2 * 4096, PG, None), + Err(PrpError::NotEnoughPages), + )); + } + + #[test] + fn a_list_is_required_when_the_transfer_spills_past_two_pages() { + assert_eq!( + build(IovaAddr(0x1000), 4 * 4096, PG, None), + Err(PrpError::ListNeeded) + ); + } + + #[test] + fn zero_size_is_a_null_prp() { + assert_eq!(build(IovaAddr(0xdead), 0, PG, None), Ok(Prp::NULL)); + } +} diff --git a/native/drivers/nvme/src/qpair.rs b/native/drivers/nvme/src/qpair.rs new file mode 100644 index 0000000..3cb67c4 --- /dev/null +++ b/native/drivers/nvme/src/qpair.rs @@ -0,0 +1,838 @@ +//! The **queue pair** — an NVMe SQ/CQ pair, the SPDK `spdk_nvme_qpair` +//! shape: per-qpair, **lockless** (no shared state, no server mediating — +//! the kore no-global-locks doctrine, which SPDK arrived at independently), +//! polled completions, batched doorbell. The admin queue is just qpair 0. +//! +//! Since X0 the ring *mechanics* live in [`kore_driver`]: an NVMe qpair is a +//! [`SubRing`]`` (Modulo wrap — the SQ tail doorbell is the publish +//! signal, SQEs carry no validity bit) + a [`CompRing`]`` (the CQ +//! Phase bit is the validity bit) + a [`CookieTable`]. This file is what's +//! left once the shared parts are factored out: the **NVMe-specific** cid +//! stamping, cookie correlation, doorbell-offset math, and the books — plus +//! the [`SqEntry`]/[`CqEntry`] impls that teach the substrate what an NVMe +//! entry's validity bit is (SQE: none; CQE: Phase). +//! +//! **The hot path is here and it is short**: [`submit`](Qpair::submit) +//! stages one SQE and records its cookie (no MMIO); [`ring`](Qpair::ring) +//! writes the doorbell **once** for a whole batch; [`poll`](Qpair::poll) +//! reads the CQ phase bit and correlates the completion to its cookie. No +//! allocation, no lock, no kernel — the device and this qpair share only the +//! ring frames. + +use core::sync::atomic::AtomicU32; + +use kore_abi::device::IovaAddr; + +use kore_driver::ring::{CompRing, CqEntry, SqEntry, SubRing, Wrap}; + +use crate::commands::{NvmeCqe, NvmeSqe}; +use crate::regs::Capabilities; +use crate::seam::{Dma, Regs}; + +/// The **heapless** per-qpair depth ceiling — the fixed inline cookie +/// table's size when the `alloc` feature is off (the kernel boot-test +/// posture). With `alloc` (userland — the global allocator is the +/// mm-backed heap), the table is sized **exactly** to the negotiated +/// depth instead and this ceiling does not apply. (The substrate's +/// [`kore_driver::MAX_SLOTS`] under its NVMe-facing name.) +pub use kore_driver::MAX_SLOTS as MAX_QDEPTH; + +/// The qpair's visible books — [`kore_driver::RingStats`] under its NVMe-facing +/// name. +pub use kore_driver::RingStats as QpairStats; + +/// An NVMe SQE carries **no in-entry validity bit** — the SQ tail doorbell +/// is the sole publish signal — so cycle stamping is the no-op default. +impl SqEntry for NvmeSqe {} + +/// An NVMe CQE's validity bit is its status **Phase** bit. +impl CqEntry for NvmeCqe { + fn valid(&self, phase: bool) -> bool { + self.phase() == phase + } +} + +/// Why a submit/poll refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum QpairError { + /// The submission ring is full — reap completions, then retry. + Full, + /// A requested depth exceeds [`MAX_QDEPTH`] or the controller's MQES. + BadDepth, +} + +/// A reaped completion, correlated to the cookie its submission carried. +#[derive(Clone, Copy, Debug)] +pub struct Completion { + /// The command id the device echoed. + pub cid: u16, + /// The cookie the submitter attached (kore-ring-shaped correlation). + pub cookie: u64, + /// The full completion entry (status, result, …). + pub cqe: NvmeCqe, +} + +impl Completion { + /// Whether the command succeeded. + pub fn is_success(&self) -> bool { + self.cqe.is_success() + } +} + +/// One submission queue of a [`Qpair`]: its ring, doorbell, command-id +/// table and books. CIDs are unique per submission queue, as the spec +/// requires; a completion names its queue (`sq_id`) and its CID. +struct SubQueue { + qid: u16, + ring: SubRing, + doorbell_off: usize, + /// cid → cookie; an id is claimed on submit and released only by its + /// own completion. + cookies: kore_driver::CookieTable, + /// Submitted, not yet completed. + inflight: u16, + /// Tail doorbell writes since construction. + doorbells: u64, + /// The head the controller reported in this queue's last completion. + last_head: u16, +} + +/// NVMe submission queues over one completion queue — a **queue pair** +/// (`SQS = 1`, the admin queue and every queue before classes existed) or a +/// **queue group**: several submission queues bound to one CQ and one +/// interrupt vector, so each service class has its own hardware queue +/// (`res/STORAGE_TRANSPORT.md` §3). +/// +/// Every single-queue method (`submit`, `ring`, `sq_tail`, …) acts on +/// submission queue 0; the `_on` forms take a queue index. A group the +/// controller granted fewer queues than `SQS` is built with the queues it +/// has, and an index past them folds onto the last live queue — degraded +/// isolation, never a refusal. +pub struct Qpair { + /// The completion queue's id (a pair's SQ shares it). + qid: u16, + sqs: [Option>; SQS], + /// Submission queues actually created (`1..=SQS`). + live: usize, + cq: CompRing, + /// Each submission queue's depth. + depth: u16, + cq_doorbell_off: usize, + /// The books, across every submission queue. + stats: QpairStats, +} + +impl Qpair { + /// Build a qpair over pre-allocated `sq`/`cq` buffers. Doorbell + /// offsets come from [`Capabilities`]. Heapless posture: `depth ≤ + /// `[`MAX_QDEPTH`]; with `alloc`, the cookie table is sized exactly + /// to `depth` (any depth the controller's MQES admits). + pub fn new( + qid: u16, + sq: S, + cq: C, + depth: u16, + caps: &Capabilities, + ) -> Result { + Self::new_group(qid, &[qid], [Some(sq)], cq, depth, depth, caps) + } +} + +impl Qpair { + /// Build a queue group: submission queues `sqids[i]` over `sq_bufs[i]` + /// (the first `sqids.len()` present), all bound to completion queue + /// `cqid` over `cq`. The CQ must hold every command the queues can have + /// in flight — `cq_depth ≥ live × depth` — or a full set of + /// completions could overrun it. + pub fn new_group( + cqid: u16, + sqids: &[u16], + sq_bufs: [Option; SQS], + cq: C, + depth: u16, + cq_depth: u16, + caps: &Capabilities, + ) -> Result { + let live = sqids.len(); + if depth == 0 || live == 0 || live > SQS { + return Err(QpairError::BadDepth); + } + #[cfg(not(feature = "alloc"))] + if depth as usize > MAX_QDEPTH { + return Err(QpairError::BadDepth); + } + if (cq_depth as usize) < live * depth as usize { + return Err(QpairError::BadDepth); + } + let mut sq_bufs = sq_bufs; + let mut sqs: [Option>; SQS] = core::array::from_fn(|_| None); + for (index, &qid) in sqids.iter().enumerate() { + let ring = sq_bufs[index].take().ok_or(QpairError::BadDepth)?; + sqs[index] = Some(SubQueue { + qid, + ring: SubRing::new(ring, depth, Wrap::Modulo), + doorbell_off: caps.sq_doorbell(qid), + cookies: kore_driver::CookieTable::new(depth), + inflight: 0, + doorbells: 0, + last_head: 0, + }); + } + Ok(Self { + qid: cqid, + sqs, + live, + cq: CompRing::new(cq, cq_depth), + depth, + cq_doorbell_off: caps.cq_doorbell(cqid), + stats: QpairStats::default(), + }) + } + + /// Submission queues actually created. + pub fn live_sqs(&self) -> usize { + self.live + } + + /// The live queue index `sq` folds onto. + fn index(&self, sq: usize) -> usize { + sq.min(self.live - 1) + } + + fn queue(&self, sq: usize) -> &SubQueue { + self.sqs[self.index(sq)] + .as_ref() + .expect("every index below live is built") + } + + fn queue_mut(&mut self, sq: usize) -> &mut SubQueue { + let index = self.index(sq); + self.sqs[index] + .as_mut() + .expect("every index below live is built") + } + + /// SQ tail doorbell writes on queue 0 since construction. + pub fn doorbells(&self) -> u64 { + self.queue(0).doorbells + } + + /// The software submission tail of queue 0 — the value the last + /// doorbell carried, or would carry. + pub fn sq_tail(&self) -> u16 { + self.queue(0).ring.tail() + } + + /// The submission head the controller reported in queue 0's last + /// completion. Against [`sq_tail`](Self::sq_tail) it says whether the + /// controller has consumed what was rung in. + pub fn last_sq_head(&self) -> u16 { + self.queue(0).last_head + } + + /// The raw bytes of queue 0's entry `back` slots behind the tail (0 = + /// the last staged), as the ring holds them — what the controller + /// fetches, for a command it will not fetch. `None` past the ring. + pub fn sq_entry_raw(&self, back: u16) -> Option<[u8; 64]> { + const _: () = assert!(size_of::() == 64); + if back >= self.depth { + return None; + } + let queue = self.queue(0); + let slot = (queue.ring.tail() + self.depth - 1 - back) % self.depth; + // SAFETY: the ring is `depth` entries of 64 device-visible bytes and + // `slot < depth`; a volatile read, since the device may be reading + // the same bytes. + Some(unsafe { + queue + .ring + .cpu() + .add(usize::from(slot) * 64) + .cast::<[u8; 64]>() + .read_volatile() + }) + } + + /// The books, across every submission queue. + pub fn stats(&self) -> QpairStats { + self.stats + } + + /// The completion queue's id. + pub fn qid(&self) -> u16 { + self.qid + } + + /// Queue 0's SQ base IOVA (what `ASQ` / Create-SQ point at). + pub fn sq_iova(&self) -> IovaAddr { + self.queue(0).ring.iova() + } + + /// `(qid, cid)` of the command in flight on queue `sq` (folded as + /// submission folds it) carrying `cookie` — what an Abort names. + pub fn locate(&self, sq: usize, cookie: u64) -> Option<(u16, u16)> { + let queue = self.sqs.get(self.index(sq))?.as_ref()?; + Some((queue.qid, queue.cookies.find(cookie)?)) + } + + /// Submission queue `sq`'s id and base IOVA — what its Create I/O SQ + /// names. `None` past the live queues. + pub fn sq_of(&self, sq: usize) -> Option<(u16, IovaAddr)> { + self.sqs + .get(sq) + .and_then(Option::as_ref) + .map(|queue| (queue.qid, queue.ring.iova())) + } + + /// CQ base IOVA (what `ACQ` / Create-CQ point at). + pub fn cq_iova(&self) -> IovaAddr { + self.cq.iova() + } + + /// Each submission queue's depth. + pub fn depth(&self) -> u16 { + self.depth + } + + /// The completion queue's depth. + pub fn cq_depth(&self) -> u16 { + self.cq.capacity() + } + + /// Forget every queue's position and every command in flight, as the + /// controller did when it reset: rings back to slot 0, the completion + /// ring zeroed, command ids all free. The memory stays; the queues are + /// created again over it. Answers how many commands were lost. + pub fn reset(&mut self) -> u32 { + let mut lost = 0u32; + for queue in self.sqs.iter_mut().flatten() { + queue.ring.reset(); + lost += u32::from(queue.cookies.clear()); + queue.inflight = 0; + queue.last_head = 0; + } + self.cq.reset(); + lost + } + + /// Whether `n` more submissions fit on queue 0. + pub fn has_space(&self, n: u16) -> bool { + self.has_space_on(0, n) + } + + /// Whether `n` more submissions fit on queue `sq`. + pub fn has_space_on(&self, sq: usize, n: u16) -> bool { + self.queue(sq).inflight + n < self.depth + } + + /// Commands queue `sq` can still take. + pub fn room_on(&self, sq: usize) -> u16 { + (self.depth - 1).saturating_sub(self.queue(sq).inflight) + } + + /// **Stage** one command on queue 0 — see [`submit_on`](Self::submit_on). + pub fn submit(&mut self, cmd: NvmeSqe, cookie: u64) -> Result { + self.submit_on(0, cmd, cookie) + } + + /// **Stage** one command on queue `sq` with a correlation `cookie` — + /// writes the SQE, stamps a fresh cid, advances the tail. Does **not** + /// ring the doorbell (batch first, [`ring_on`](Self::ring_on) once). + /// Returns the cid. + pub fn submit_on(&mut self, sq: usize, mut cmd: NvmeSqe, cookie: u64) -> Result { + if !self.has_space_on(sq, 1) { + return Err(QpairError::Full); + } + let queue = self.queue_mut(sq); + // The cid is claimed, not counted: a command completes out of order + // on a drive serving random reads, so an id is free again only when + // its own completion was reaped, never because the ring wrapped. + let Some(cid) = queue.cookies.claim(cookie) else { + return Err(QpairError::Full); + }; + cmd.set_cid(cid); + queue.ring.stage(cmd); + queue.inflight += 1; + self.stats.submitted += 1; + Ok(cid) + } + + /// **Ring queue 0's SQ doorbell** — see [`ring_on`](Self::ring_on). + pub fn ring(&mut self, regs: &impl Regs) { + self.ring_on(0, regs); + } + + /// **Ring queue `sq`'s SQ doorbell** — publish its staged batch (release + /// fence, then one MMIO write of the tail). + pub fn ring_on(&mut self, sq: usize, regs: &impl Regs) { + let queue = self.queue_mut(sq); + let off = queue.doorbell_off; + queue.ring.ring(|tail| regs.write32(off, tail as u32)); + queue.doorbells += 1; + } + + /// Submit + ring in one call — the convenience for a lone command. + pub fn submit_one( + &mut self, + cmd: NvmeSqe, + cookie: u64, + regs: &impl Regs, + ) -> Result { + let cid = self.submit(cmd, cookie)?; + self.ring(regs); + Ok(cid) + } + + /// **Poll one completion** (non-blocking): if the CQ head holds an + /// entry with the expected phase, consume it, advance the head, ring + /// the CQ doorbell, and return it correlated to its cookie. + pub fn poll(&mut self, regs: &impl Regs) -> Option { + let completion = self.reap_one()?; + regs.write32(self.cq_doorbell_off, self.cq.head() as u32); + Some(completion) + } + + /// **The CQ head slot's status dword** — the word a completion's arrival + /// stores to, for a caller that wants to sleep its core on that store. + /// + /// The controller publishes a CQE by DMA-writing all sixteen of its bytes, + /// the phase bit among them. So the dword carrying that bit is exactly the + /// line an address monitor + /// ([`kore_rt::hint::wait_on`](../../kore_rt/hint/fn.wait_on.html)) should + /// watch: the wake is the completion itself, needing no interrupt, no + /// route, and no waker — which is what makes it usable on the route-less + /// posture where nothing exists that could ever deliver a wake. + /// + /// **This crate deliberately does not do the sleeping.** `kore-rt` is an + /// optional `service`-feature dependency precisely so the driver core stays + /// out of the kernel binary, and the monitored wait is a runtime mechanism. + /// So the split is by what each layer knows: this crate knows *which* word + /// carries the validity bit (it defines [`NvmeCqe`]), and its caller knows + /// how long a core may be held. Publishing the word rather than the wait + /// keeps [`CompRing`] hardware-neutral — no NVMe layout crosses into + /// `kore_driver` — while costing that crate nothing. + /// + /// The caller snapshots this word, then sleeps while it still reads that + /// snapshot. It must **not** treat a wake as a completion: a neighbour in + /// the same line, an interrupt, or the cap all end the sleep, so the poll + /// after it is the only truth. That is the same discipline + /// [`CompRing::poll`]'s phase check already is. + /// + /// `None` when the ring base is not `u32`-aligned, which no DMA pool in + /// this tree produces (a CQ is page-aligned) but which cannot be *proved* + /// here — and an unaligned atomic is undefined behaviour, not a wrong + /// answer. Refusing costs the caller its fast tier and nothing else. + pub fn completion_word(&self) -> Option<&AtomicU32> { + /// Byte offset of the dword holding the CQE's phase bit. + /// + /// **Derived, not written down.** `status` is a `u16` at offset 14 of + /// a `repr(C)` [`NvmeCqe`], so the enclosing aligned dword starts at + /// 12 — where `cid` also lives. Masking the field offset down to its + /// dword means a layout change moves this with it instead of leaving a + /// stale constant pointing at `sq_head`. + const PHASE_DWORD: usize = core::mem::offset_of!(NvmeCqe, status) & !3; + const _: () = assert!(PHASE_DWORD == core::mem::offset_of!(NvmeCqe, cid)); + const _: () = assert!(PHASE_DWORD + 4 <= core::mem::size_of::()); + + let base = self.cq.cpu(); + if !base.cast::().is_aligned() { + return None; + } + let offset = self.cq.head() as usize * core::mem::size_of::() + PHASE_DWORD; + // SAFETY: `cq` is `capacity * size_of::()` device-visible + // bytes and `head < capacity`, so `offset .. offset + 4` is inside the + // ring. The base is `u32`-aligned (checked above) and both addends are + // multiples of 4, so the result is too. The device writes this word + // concurrently, which is why it is handed out as an `AtomicU32` rather + // than a `&u32`: every CPU-side access is then a single atomic load, + // the same posture `CompRing::poll`'s `read_volatile` of the whole + // entry already takes against the same writer. + Some(unsafe { &*base.add(offset).cast::() }) + } + + /// Reap up to `out.len()` ready completions and acknowledge the final CQ + /// head with one MMIO write. Output slots are overwritten from the + /// beginning; the returned count names the initialized prefix. + pub fn poll_batch(&mut self, regs: &impl Regs, out: &mut [Option]) -> usize { + let mut count = 0; + for slot in out { + let Some(completion) = self.reap_one() else { + break; + }; + *slot = Some(completion); + count += 1; + } + if count != 0 { + regs.write32(self.cq_doorbell_off, self.cq.head() as u32); + } + count + } + + /// Consume one ready CQE without ringing the head doorbell. Callers must + /// acknowledge the final head after their bounded drain. + fn reap_one(&mut self) -> Option { + loop { + let cqe = self.cq.poll()?; + // The completion names its submission queue; a CID is unique + // only within one. A queue id this group does not hold, or a cid + // naming no command in flight on it, is a completion the device + // owes nobody: consumed and counted, never delivered, so it can + // neither free an id twice nor answer a request with another + // command's data. + // + // A lone queue takes every completion, exactly as a queue pair + // always did — its behaviour never depends on the `sq_id` field. + // Only a group routes by it, as it must. + let live = self.live; + let found = if live == 1 { + self.sqs[0].as_mut() + } else { + self.sqs[..live] + .iter_mut() + .flatten() + .find(|queue| queue.qid == cqe.sq_id) + }; + let Some(queue) = found else { + self.stats.stray += 1; + continue; + }; + queue.last_head = cqe.sq_head; + let Some(cookie) = queue.cookies.release(cqe.cid) else { + self.stats.stray += 1; + continue; + }; + queue.inflight = queue.inflight.saturating_sub(1); + self.stats.completed += 1; + if !cqe.is_success() { + self.stats.errors += 1; + } + return Some(Completion { + cid: cqe.cid, + cookie, + cqe, + }); + } + } + + /// Drain every ready completion, calling `f` per entry; returns the + /// count (`spdk_nvme_qpair_process_completions`). + pub fn process_completions(&mut self, regs: &impl Regs, mut f: impl FnMut(Completion)) -> u32 { + let mut n = 0; + while let Some(c) = self.reap_one() { + f(c); + n += 1; + } + if n != 0 { + regs.write32(self.cq_doorbell_off, self.cq.head() as u32); + } + n + } + + /// Submit one command and **poll to completion** — the admin-queue + /// convenience (a blocking op on a synchronous queue). Spins up to + /// `max_polls` times; between polls the caller-supplied `pause` runs + /// (a `spin_loop`, or a real inter-poll delay on hardware). `None` + /// on timeout. + pub fn submit_and_wait( + &mut self, + cmd: NvmeSqe, + regs: &impl Regs, + max_polls: u32, + pause: impl FnMut(), + ) -> Option { + let mut pacer = kore_driver::block::CountedPacer::new(max_polls, pause); + self.submit_and_wait_paced(cmd, regs, &mut pacer) + } + + /// Submit one admin command and wait through the shared completion-pacing + /// seam. Production uses an elapsed-time pacer; counted test callers keep + /// [`submit_and_wait`](Self::submit_and_wait). + pub fn submit_and_wait_paced( + &mut self, + cmd: NvmeSqe, + regs: &impl Regs, + pacer: &mut dyn kore_driver::block::CompletionPacer, + ) -> Option { + let cid = self.submit_one(cmd, 0, regs).ok()?; + loop { + if let Some(c) = self.poll(regs) + && c.cid == cid + { + return Some(c); + } + if !pacer.pace() { + return self.poll(regs).filter(|completion| completion.cid == cid); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use core::sync::atomic::Ordering; + + /// A CQ backing store the qpair really owns. + /// + /// [`StubBuf`] hands out a dangling pointer, which is enough for the + /// constructor's geometry bookkeeping and useless here: `completion_word` + /// returns a pointer *into* the ring, and these tests store through it. + /// + /// `skew` shifts the base off its natural alignment. It is the only way to + /// reach the refusal path from a test, since every DMA pool in this tree + /// page-aligns a completion queue. + struct RingBuf { + slots: [NvmeCqe; N], + skew: usize, + } + + impl RingBuf { + const fn new(skew: usize) -> Self { + Self { + slots: [NvmeCqe { + result: 0, + rsvd: 0, + sq_head: 0, + sq_id: 0, + cid: 0, + status: 0, + }; N], + skew, + } + } + } + + impl Dma for RingBuf { + fn cpu(&self) -> *mut u8 { + // SAFETY: `skew` is 0 or 1 and the array is `N * 16` bytes with + // `N > 0`, so the result stays inside the allocation. Handing out + // `*mut` from `&self` is `Dma::cpu`'s contract — the posture + // `NvmeBuffer` already takes against the same concurrent writer. + unsafe { self.slots.as_ptr().cast::().cast_mut().add(self.skew) } + } + fn iova(&self) -> IovaAddr { + IovaAddr(0x1000) + } + fn len(&self) -> usize { + N * core::mem::size_of::() - self.skew + } + } + + /// Doorbell writes go nowhere: these tests are about the ring, not MMIO. + struct NullRegs; + impl Regs for NullRegs { + fn read32(&self, _off: usize) -> u32 { + 0 + } + fn write32(&self, _off: usize, _val: u32) {} + } + + /// The published dword, as the controller's DMA would leave it. + /// + /// `cid` is the low half and `status` the high half of one little-endian + /// dword (every target in this tree is little-endian), and the phase bit + /// is `status` bit 0. `CompRing` expects phase `1` on its first pass. + const fn published(cid: u16) -> u32 { + const _: () = assert!(cfg!(target_endian = "little")); + (1u32 << 16) | cid as u32 + } + + /// **The word `completion_word` hands out is the one a completion lands + /// in** — proven end to end through the real reap path rather than by + /// comparing addresses. + /// + /// The failure this guards is quiet and expensive. Aliasing the wrong + /// dword — `sq_head`, say, which is what a hardcoded offset invites when + /// the struct changes — leaves a monitored wait sleeping on a word the + /// controller never stores to. Every turn would then run to its cap and + /// wake having learned nothing, and the tier would read as present and + /// working while delivering strictly worse latency than the spin it + /// replaced. + #[test] + fn the_monitored_word_is_where_a_completion_lands() { + let mut qp = + Qpair::new(1, SqBuf::<4>::new(), RingBuf::<4>::new(0), 4, &caps()).expect("depth fits"); + // A completion answers a command in flight; one is submitted so the + // published cid names it (an unowned cid is dropped as stray). + let cid = qp.submit(NvmeSqe::zeroed(), 1).expect("one fits"); + + let word = qp + .completion_word() + .expect("an aligned ring can be monitored"); + assert_eq!(word.load(Ordering::Relaxed), 0, "a fresh slot reads stale"); + word.store(published(cid), Ordering::Relaxed); + + let completion = qp.poll(&NullRegs).expect("the published entry reaps"); + assert_eq!(completion.cid, cid, "the store landed in the head CQE"); + assert!(completion.cqe.phase(), "the phase bit shares that dword"); + } + + /// The watched word tracks the head, so the turn after a completion + /// watches the NEXT slot and not the one already consumed. + /// + /// A word cached across completions would watch a slot the controller has + /// finished with — the same silent never-fires failure as above, reached + /// by a different route. + #[test] + fn the_monitored_word_follows_the_head() { + let mut qp = + Qpair::new(1, SqBuf::<4>::new(), RingBuf::<4>::new(0), 4, &caps()).expect("depth fits"); + let cid = qp.submit(NvmeSqe::zeroed(), 1).expect("one fits"); + + let first = qp.completion_word().expect("aligned").as_ptr(); + qp.completion_word() + .expect("aligned") + .store(published(cid), Ordering::Relaxed); + qp.poll(&NullRegs).expect("the published entry reaps"); + let second = qp.completion_word().expect("aligned").as_ptr(); + + assert_eq!( + second as usize - first as usize, + core::mem::size_of::(), + "the watch must advance exactly one slot" + ); + } + + /// An unaligned ring base declines the monitor rather than handing out a + /// misaligned atomic — which is undefined behaviour, not a wrong answer. + /// The caller keeps its ordinary pacing and loses only the fast tier. + #[test] + fn an_unaligned_ring_declines_the_monitor() { + let qp = Qpair::new(1, StubBuf, RingBuf::<4>::new(1), 4, &caps()).expect("depth fits"); + assert!(qp.completion_word().is_none()); + } + + /// A submission ring the qpair really stages into. + struct SqBuf { + slots: [NvmeSqe; N], + } + + impl SqBuf { + const fn new() -> Self { + Self { + slots: [NvmeSqe::zeroed(); N], + } + } + } + + impl Dma for SqBuf { + fn cpu(&self) -> *mut u8 { + // SAFETY: as `RingBuf::cpu` — the staged entries land inside + // the array, which is `N * 64` bytes. + self.slots.as_ptr().cast::().cast_mut() + } + fn iova(&self) -> IovaAddr { + IovaAddr(0x2000) + } + fn len(&self) -> usize { + N * core::mem::size_of::() + } + } + + /// Publish a completion for `cid` at the head and reap it. + fn complete(qp: &mut Qpair, RingBuf<8>>, cid: u16) -> Option { + qp.completion_word() + .expect("aligned") + .store(published(cid), Ordering::Relaxed); + qp.poll(&NullRegs) + } + + /// **An id is never reissued while its command is in flight.** With + /// every id but one claimed, the one completion that arrives — out of + /// order, from the middle of the batch — frees exactly that id, and the + /// next submit takes it; nothing else is free until its own completion. + /// Round-robin issue would have handed out the oldest command's id here + /// while that command was still in flight. + #[test] + fn an_id_is_reissued_only_after_its_own_completion() { + let mut qp = + Qpair::new(1, SqBuf::<8>::new(), RingBuf::<8>::new(0), 8, &caps()).expect("depth fits"); + let mut cids = [0u16; 7]; + for (n, cid) in cids.iter_mut().enumerate() { + *cid = qp + .submit(NvmeSqe::zeroed(), 100 + n as u64) + .expect("seven fit in a ring of eight"); + } + assert!(matches!( + qp.submit(NvmeSqe::zeroed(), 0), + Err(QpairError::Full) + )); + let middle = cids[3]; + let done = complete(&mut qp, middle).expect("a live completion reaps"); + assert_eq!(done.cookie, 103, "the completion answers its own submit"); + let again = qp.submit(NvmeSqe::zeroed(), 200).expect("one id is free"); + assert_eq!(again, middle, "the freed id, not the oldest command's"); + assert!( + matches!(qp.submit(NvmeSqe::zeroed(), 0), Err(QpairError::Full)), + "the six others still hold their ids" + ); + let oldest = complete(&mut qp, cids[0]).expect("live"); + assert_eq!(oldest.cookie, 100); + assert_eq!(qp.submit(NvmeSqe::zeroed(), 201), Ok(cids[0])); + } + + /// A completion naming an id no command holds is consumed and counted, + /// never delivered, and frees nothing. + #[test] + fn a_completion_for_no_command_is_dropped_and_counted() { + let mut qp = + Qpair::new(1, SqBuf::<8>::new(), RingBuf::<8>::new(0), 8, &caps()).expect("depth fits"); + let a = qp.submit(NvmeSqe::zeroed(), 7).expect("submit"); + assert!( + complete(&mut qp, 5).is_none(), + "nothing in flight has cid 5" + ); + assert_eq!(qp.stats().stray, 1); + assert_eq!(qp.stats().completed, 0); + let done = complete(&mut qp, a).expect("the real one still reaps"); + assert_eq!(done.cookie, 7); + assert!(complete(&mut qp, a).is_none(), "a second answer is stray"); + assert_eq!(qp.stats().stray, 2); + assert_eq!(qp.stats().completed, 1); + } + + /// A stub DMA buffer — the qpair constructor only records geometry. + struct StubBuf; + impl Dma for StubBuf { + fn cpu(&self) -> *mut u8 { + core::ptr::NonNull::dangling().as_ptr() + } + fn iova(&self) -> IovaAddr { + IovaAddr(0x1000) + } + fn len(&self) -> usize { + 0x1000 + } + } + + fn caps() -> Capabilities { + Capabilities::from_cap(0x0000_0000_0001_003F) + } + + /// The `alloc` posture sizes the cookie table to the *negotiated* + /// depth through the global allocator (in kore-rt, the mm-backed heap) — + /// a depth beyond the heapless inline ceiling is fine. Without + /// `alloc`, the same depth is a `BadDepth` refusal (the fixed table + /// is the ceiling). + #[test] + fn depth_beyond_the_inline_ceiling_needs_alloc() { + let deep = (MAX_QDEPTH + 256) as u16; + let r = Qpair::new(1, StubBuf, StubBuf, deep, &caps()); + #[cfg(feature = "alloc")] + assert!(r.is_ok(), "alloc posture sizes exactly to depth"); + #[cfg(not(feature = "alloc"))] + assert!( + matches!(r, Err(QpairError::BadDepth)), + "heapless posture refuses past the inline ceiling" + ); + } + + /// Zero depth refuses in every posture. + #[test] + fn zero_depth_refuses() { + assert!(matches!( + Qpair::new(1, StubBuf, StubBuf, 0, &caps()), + Err(QpairError::BadDepth) + )); + } +} diff --git a/native/drivers/nvme/src/regs.rs b/native/drivers/nvme/src/regs.rs new file mode 100644 index 0000000..f00ea90 --- /dev/null +++ b/native/drivers/nvme/src/regs.rs @@ -0,0 +1,368 @@ +//! NVMe controller register layout + `CAP` parse — **spec facts** (NVMe +//! base spec §3.1), the stable core every NVMe driver shares. Ported from +//! `os0/user/dev/block/nvme/controller.rs` (the same constants; the +//! reference is a working driver). +//! +//! BAR0 layout: +//! +//! | Offset | Reg | Description | +//! |---------|-------|--------------------------------------| +//! | 0x00 | CAP | Controller Capabilities (64-bit) | +//! | 0x08 | VS | Version | +//! | 0x14 | CC | Controller Configuration | +//! | 0x1C | CSTS | Controller Status | +//! | 0x24 | AQA | Admin Queue Attributes | +//! | 0x28 | ASQ | Admin SQ Base Address (64-bit) | +//! | 0x30 | ACQ | Admin CQ Base Address (64-bit) | +//! | 0x1000+ | DBs | SQ/CQ doorbells (stride 4 << DSTRD) | + +/// Controller Capabilities (64-bit). +pub const REG_CAP: usize = 0x00; +/// Version. +pub const REG_VS: usize = 0x08; +/// Controller Configuration. +pub const REG_CC: usize = 0x14; +/// Controller Status. +pub const REG_CSTS: usize = 0x1C; +/// Admin Queue Attributes. +pub const REG_AQA: usize = 0x24; +/// Admin SQ Base Address (64-bit). +pub const REG_ASQ: usize = 0x28; +/// Admin CQ Base Address (64-bit). +pub const REG_ACQ: usize = 0x30; +/// Base of the doorbell region. +pub const REG_DOORBELL_BASE: usize = 0x1000; + +// ── Controller Configuration (CC) fields ──────────────────────────────── + +/// Enable. +pub const CC_EN: u32 = 1 << 0; +/// I/O Command Set: NVM. +pub const CC_CSS_NVM: u32 = 0 << 4; +/// Memory Page Size 4 KiB (`2^(12+MPS)`, MPS = 0). +pub const CC_MPS_4K: u32 = 0 << 7; +/// Arbitration: round-robin. +pub const CC_AMS_RR: u32 = 0 << 11; +/// Arbitration: weighted round robin with urgent priority class (`001b`). +/// Legal only when `CAP.AMS` bit 17 is set ([`Capabilities::ams_wrr`]). +pub const CC_AMS_WRR: u32 = 1 << 11; +/// I/O SQ entry size 64 B (`2^6`). +pub const CC_IOSQES_64: u32 = 6 << 16; +/// I/O CQ entry size 16 B (`2^4`). +pub const CC_IOCQES_16: u32 = 4 << 20; + +/// The enable word the core writes to `CC` for a configured `mps` +/// (`MPS` field, bits 10:7): NVM command set, round-robin, the spec +/// entry sizes, enabled. `mps` must lie within the controller's +/// advertised `[MPSMIN, MPSMAX]` — the core passes `MPSMIN` (the +/// smallest page the controller accepts; a `MPSMIN > 0` controller is +/// thereby handled correctly, where a hardcoded 4 KiB would be an +/// illegal configuration). +pub const fn cc_enable(mps: u8) -> u32 { + cc_enable_arbitrated(mps, false) +} + +/// [`cc_enable`] with the arbitration mechanism chosen: weighted round +/// robin with urgent priority class when `wrr`, plain round robin +/// otherwise. The caller must pass `wrr` only for a controller whose +/// [`Capabilities::ams_wrr`] is set — `CC.AMS` naming an unsupported +/// mechanism is an invalid configuration the controller may refuse to +/// enable under. +pub const fn cc_enable_arbitrated(mps: u8, wrr: bool) -> u32 { + let ams = if wrr { CC_AMS_WRR } else { CC_AMS_RR }; + CC_EN | CC_CSS_NVM | ((mps as u32) << 7) | ams | CC_IOSQES_64 | CC_IOCQES_16 +} + +/// The `mps = 0` (4 KiB) enable word — the spec-floor configuration. +pub const CC_ENABLE: u32 = cc_enable(0); + +// ── Shutdown Notification (CC bits 15:14) ─────────────────────────────── + +/// Shutdown Notification field mask. +pub const CC_SHN_MASK: u32 = 0b11 << 14; +/// Normal shutdown notification (flush, then report SHST complete). +pub const CC_SHN_NORMAL: u32 = 0b01 << 14; + +// ── Controller Status (CSTS) fields ───────────────────────────────────── + +/// Ready. +pub const CSTS_RDY: u32 = 1 << 0; +/// Controller Fatal Status. +pub const CSTS_CFS: u32 = 1 << 1; +/// Shutdown Status field mask (bits 3:2). +pub const CSTS_SHST_MASK: u32 = 0b11 << 2; +/// Shutdown processing complete. +pub const CSTS_SHST_COMPLETE: u32 = 0b10 << 2; + +/// Parsed `CAP` register — the controller's advertised limits. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Capabilities { + /// Maximum Queue Entries Supported (0-based; real max is `+1`). + pub mqes: u16, + /// Contiguous Queues Required. + pub cqr: bool, + /// `CAP.AMS` bit 17: weighted round robin with urgent priority class is + /// supported, so submission-queue priorities (`QPRIO`) can be honoured. + pub ams_wrr: bool, + /// `CAP.AMS` bit 18: a vendor-specific arbitration mechanism exists. + /// Recorded, never selected. + pub ams_vendor: bool, + /// Controller ready timeout, in the register's 500 ms units. + pub timeout_500ms: u8, + /// Doorbell Stride (`4 << dstrd` bytes). + pub dstrd: u8, + /// NVM Subsystem Reset Supported. + pub nssrs: bool, + /// Memory Page Size Minimum (`2^(12+mpsmin)`). + pub mpsmin: u8, + /// Memory Page Size Maximum (`2^(12+mpsmax)`). + pub mpsmax: u8, +} + +impl Capabilities { + /// Parse from a raw `CAP` value. + pub const fn from_cap(cap: u64) -> Self { + Self { + mqes: (cap & 0xFFFF) as u16, + cqr: (cap >> 16) & 1 != 0, + ams_wrr: (cap >> 17) & 1 != 0, + ams_vendor: (cap >> 18) & 1 != 0, + timeout_500ms: ((cap >> 24) & 0xFF) as u8, + dstrd: ((cap >> 32) & 0xF) as u8, + nssrs: (cap >> 36) & 1 != 0, + mpsmin: ((cap >> 48) & 0xF) as u8, + mpsmax: ((cap >> 52) & 0xF) as u8, + } + } + + /// Doorbell stride in bytes. + pub const fn doorbell_stride(&self) -> usize { + 4 << self.dstrd + } + + /// `CAP.TO` as nanoseconds. The NVMe base specification defines the raw + /// field directly in 500 ms units; this is the worst-case `CSTS.RDY` + /// transition time after changing `CC.EN`. + pub const fn ready_timeout_ns(&self) -> u64 { + self.timeout_500ms as u64 * 500_000_000 + } + + /// Byte offset of the SQ tail doorbell for queue `qid`. + pub const fn sq_doorbell(&self, qid: u16) -> usize { + REG_DOORBELL_BASE + (2 * qid as usize) * self.doorbell_stride() + } + + /// Byte offset of the CQ head doorbell for queue `qid`. + pub const fn cq_doorbell(&self, qid: u16) -> usize { + REG_DOORBELL_BASE + (2 * qid as usize + 1) * self.doorbell_stride() + } + + /// The real maximum queue depth (`mqes + 1`). + /// + /// Returns `u32` because the answer does not fit in `u16`: `CAP.MQES` is + /// 0-based, so its maximum legal value `0xFFFF` means **65536** entries. + /// Use [`Self::max_queue_depth`] to clamp it against a queue depth — + /// casting this to `u16` wraps the largest legal controller to zero. + pub const fn max_queue_entries(&self) -> u32 { + self.mqes as u32 + 1 + } + + /// The controller's queue-entry ceiling, saturated into the `u16` a queue + /// depth is expressed in. + /// + /// `mqes + 1` is exactly one larger than `u16::MAX` on a controller + /// advertising the maximal legal `CAP.MQES`, so `max_queue_entries() as + /// u16` wraps it to `0` — and every queue constructor then refuses that + /// depth as [`QpairError::BadDepth`](crate::QpairError::BadDepth). A real + /// disk shipped that value (Micron 2500, `1344:5425`), which no emulated + /// controller in the test matrix advertises, so the wrap only ever + /// appeared on metal: the drive was rejected for being *too capable*. + /// + /// Saturating is not a compromise here — a depth is always clamped + /// against a driver maximum orders of magnitude below 65535, so the + /// distinction between 65535 and 65536 available entries cannot change + /// any caller's result. + pub const fn max_queue_depth(&self) -> u16 { + // Deliberately not `max_queue_entries() as u16`: that is the bug. + if self.mqes == u16::MAX { + u16::MAX + } else { + self.mqes + 1 + } + } +} + +/// Does this BAR hold a live, decodable NVMe controller? The pool-free +/// **candidate probe** (`res/DEVICE_SERVICE.md` §10) — the disk sibling of +/// `kore_xhci::regs::connected_ports`: a claiming driver runs it over a +/// [`DeviceProbe`](../seam) BAR before spending its one DMA supply on +/// bring-up, so a dead function is released back to the broker's ledger +/// instead of failing the whole founding. +/// +/// A power-gated (D3) or unrouted function reads all-ones; an absent decode +/// reads zeros. Both are rejected, as is a BAR too short to reach the +/// doorbell region every operable controller needs, and a `CAP` whose +/// queue-depth floor (`MQES`) the spec forbids. +pub fn controller_present(regs: &impl crate::seam::Regs, bar_len: usize) -> bool { + // The register file through ACQ plus the first doorbell pair is the + // minimum any operable controller decodes. + if bar_len < REG_DOORBELL_BASE + 8 { + return false; + } + let vs = regs.read32(REG_VS); + if vs == 0 || vs == u32::MAX { + return false; + } + let cap = regs.read64(REG_CAP); + if cap == 0 || cap == u64::MAX { + return false; + } + // MQES is 0-based with a spec minimum of 1 (two entries); a zero here + // is a decode artifact, not a controller. + Capabilities::from_cap(cap).mqes >= 1 +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A controller advertising the maximal legal `CAP.MQES` must not be + /// rejected for it. `mqes + 1` is 65536 there — one past `u16` — so the + /// old `max_queue_entries() as u16` wrapped to a depth of `0`, which + /// `Qpair::new` refuses as `BadDepth`. That is the whole of the Micron + /// 2500 (`1344:5425`) metal bring-up failure: the drive was refused for + /// advertising the largest queue the spec allows, and no emulated + /// controller in the matrix advertises it, so only metal ever saw it. + #[test] + fn a_maximal_mqes_does_not_wrap_to_a_zero_queue_depth() { + let c = Capabilities::from_cap(0x0000_0000_0000_FFFF); + assert_eq!(c.mqes, 0xFFFF); + // The exact count still does not fit a u16, and still must not be cast. + assert_eq!(c.max_queue_entries(), 65_536); + assert_eq!(c.max_queue_depth(), u16::MAX); + // What the admin queue actually computes. + assert_eq!(32u16.min(c.max_queue_depth()), 32); + // The bug, pinned: this is what the two callsites used to do. + assert_eq!(c.max_queue_entries() as u16, 0); + } + + /// Saturation must not disturb ordinary controllers, including the + /// boundary one entry below the wrap. + #[test] + fn max_queue_depth_matches_the_entry_count_below_the_wrap() { + for mqes in [0u16, 1, 0x3F, 0x7FF, 0xFFFE] { + let c = Capabilities::from_cap(u64::from(mqes)); + assert_eq!(u32::from(c.max_queue_depth()), c.max_queue_entries()); + } + } + + /// `CAP.AMS` is bits 18:17 — WRR at 17, vendor-specific at 18 — and + /// the enable word's `CC.AMS` (bits 13:11) follows the choice, leaving + /// every other field exactly as the round-robin word has it. + #[test] + fn arbitration_support_parses_and_selects_cc_ams() { + let rr_only = Capabilities::from_cap(0x0000_0000_0001_003F); + assert!(!rr_only.ams_wrr && !rr_only.ams_vendor); + let wrr = Capabilities::from_cap(1 << 17); + assert!(wrr.ams_wrr && !wrr.ams_vendor); + let vendor = Capabilities::from_cap(1 << 18); + assert!(!vendor.ams_wrr && vendor.ams_vendor); + + assert_eq!(cc_enable(0), cc_enable_arbitrated(0, false)); + assert_eq!(cc_enable_arbitrated(0, true) & (0b111 << 11), 1 << 11); + assert_eq!( + cc_enable_arbitrated(2, true) & !(0b111 << 11), + cc_enable(2) & !(0b111 << 11) + ); + // The round-robin word is unchanged by the refactor. + assert_eq!(CC_ENABLE, 0x0046_0001); + } + + #[test] + fn cap_parses_and_doorbells_stride_correctly() { + // MQES=0x3F (64 entries), CQR=1 (bit 16), DSTRD=0. + let cap = 0x0000_0000_0001_003F; + let c = Capabilities::from_cap(cap); + assert_eq!(c.mqes, 0x3F); + assert!(c.cqr); + assert_eq!(c.timeout_500ms, 0); + assert_eq!(c.max_queue_entries(), 64); + assert_eq!(c.doorbell_stride(), 4); + // qid 0 (admin): SQ at 0x1000, CQ at 0x1004. + assert_eq!(c.sq_doorbell(0), 0x1000); + assert_eq!(c.cq_doorbell(0), 0x1004); + // qid 1 (first I/O): SQ at 0x1008, CQ at 0x100C. + assert_eq!(c.sq_doorbell(1), 0x1008); + assert_eq!(c.cq_doorbell(1), 0x100C); + + // A wider stride (DSTRD=2 ⇒ 16-byte stride) spaces doorbells out. + let wide = Capabilities::from_cap(0x0000_0002_0000_0000); + assert_eq!(wide.doorbell_stride(), 16); + assert_eq!(wide.cq_doorbell(0), 0x1000 + 16); + + let timed = Capabilities::from_cap(30u64 << 24); + assert_eq!(timed.timeout_500ms, 30); + assert_eq!(timed.ready_timeout_ns(), 15_000_000_000); + } + + #[test] + fn cc_enable_places_mps_in_bits_10_7() { + assert_eq!(cc_enable(0), CC_ENABLE); + assert_eq!(cc_enable(1) & (0xF << 7), 1 << 7, "MPS=1 (8 KiB)"); + assert_eq!(cc_enable(1) & CC_EN, CC_EN, "still enabled"); + } + + /// The candidate probe's three rejection tiers: a D3/unrouted function + /// (all-ones), an undecoded BAR (zeros), and a window too short for the + /// doorbell region — against the one accepting shape. + #[test] + fn controller_present_rejects_dead_and_short_candidates() { + struct Fixed { + vs: u32, + cap: u64, + } + impl crate::seam::Regs for Fixed { + fn read32(&self, off: usize) -> u32 { + match off { + REG_VS => self.vs, + REG_CAP => self.cap as u32, + x if x == REG_CAP + 4 => (self.cap >> 32) as u32, + _ => 0, + } + } + fn write32(&self, _: usize, _: u32) {} + } + + // MQES=0x3F, a real version — the live shape. + let live = Fixed { + vs: 0x0001_0400, + cap: 0x0000_0000_0001_003F, + }; + assert!(controller_present(&live, 0x2000)); + assert!( + !controller_present(&live, REG_DOORBELL_BASE), + "a BAR without the first doorbell pair refuses" + ); + + let dead = Fixed { + vs: u32::MAX, + cap: u64::MAX, + }; + assert!( + !controller_present(&dead, 0x2000), + "all-ones is D3, not a device" + ); + + let vacant = Fixed { vs: 0, cap: 0 }; + assert!(!controller_present(&vacant, 0x2000), "zeros decode nothing"); + + let zero_mqes = Fixed { + vs: 0x0001_0400, + cap: 0x0000_0000_0001_0000, + }; + assert!( + !controller_present(&zero_mqes, 0x2000), + "a spec-illegal MQES floor is a decode artifact" + ); + } +} diff --git a/native/drivers/nvme/src/swap.rs b/native/drivers/nvme/src/swap.rs new file mode 100644 index 0000000..51bcad0 --- /dev/null +++ b/native/drivers/nvme/src/swap.rs @@ -0,0 +1,1969 @@ +//! **NvmeSwap** — the disk swap backend: `SystemService` over the kore-nvme core, the lessor loop's disk tier (the +//! claim made when the vocabulary was designed, now cashed: the loop +//! takes this backend **without changing a line**). +//! +//! Target performance is structural, not bolted on—but the target is not yet +//! a production claim: +//! +//! - **Page bytes transit a bounce page this driver owns.** They reach +//! the service through the client's staging window (a plain CPU +//! mapping), and there is no edge from a mapped view to a +//! device-visible address in this driver's domain, so `Out` copies +//! staging → [`bounce`](NvmeSwap::bounce) → DMA and `In` reverses it. +//! One 4 KiB memcpy per page, and every byte the controller touches +//! lies in frames this driver published to it at intake — the honest +//! alternative to casting a CPU pointer into an IOVA. +//! - **Bounded immutable extents.** A logical extent is up to 512 contiguous +//! page slots behind one generation-checked identity. Next-fit admission +//! scans only the candidate run in the steady streaming case, allocates +//! nothing, and release becomes one DSM range. +//! - **Batched, deferred discards.** `Discard` retires the slot +//! immediately but queues the DSM range; one Dataset Management command +//! deallocates up to [`DSM_BATCH`] ranges when the table fills (or on +//! [`flush_discards`](NvmeSwap::flush_discards)). The table page is +//! allocated once at bind and reused. +//! - **No flush on the swap path.** Swap pages need no crash durability +//! (a lost swap page is lost RAM either way), so `Out` never pays for +//! `Flush`; write-back caching in the device stays fully effective. +//! - **Single-PRP transfers.** A 4 KiB page at the controller's 4 KiB+ +//! MPS is always PRP1-only — no list allocation on the hot path. +//! +//! **The completion wait is a seam** ([`CompletionPacer`]): the ops poll +//! the completion queue and pace between empty polls however the caller +//! chose — the bounded spin ([`SpinPacer`]: host tests, and the metal +//! fallback when no interrupt route was realized) or a real park on the +//! task's held MSI-X route (`kore-svc-swap-nvme`'s pacer, the production +//! posture). Unit tests complete against the `cfg(test)` controller +//! double at the spin posture; test completion is evidence of the core's +//! logic, never of interrupt delivery — that proof lives in the service's +//! readiness probe and the boot gate's route witness. + +use alloc::vec::Vec; + +use kore_abi::device::IovaAddr; +use kore_abi::svc::{SwapRepWire, SwapReqWire}; +use kore_abi::task::{Bootstrap, Profile, RoleSlot, role}; +use kore_rt::service::{BindError, Bundle, Serve, SystemService}; +use kore_rt::svc::SvcProtocol; +use kore_svc_swap::{ + ExtentTable, SwapExtent, SwapRefusal, SwapRequest, SwapResponse, staging_extent, + staging_extent_mut, +}; + +use crate::commands::DsmRange; +use crate::seam::{Dma, DmaPool, Regs}; +use crate::{Controller, Namespace, Qpair}; + +/// The swap page (the lessor's unit — matches `kore-svc-mm`'s `PAGE_BYTES`). +pub const PAGE_BYTES: u64 = 4096; + +/// LBAs per swap page for a namespace of `block_size`-byte blocks — or +/// `None` when the format cannot carry whole pages at all: a zero size +/// (identify's own overflow sentinel), one past [`PAGE_BYTES`], or one +/// that is not a power of two. +/// +/// Identify data is **device-controlled input**, so the shape is +/// validated, never assumed: dividing by an unvalidated block size is how +/// a hostile or broken descriptor (LBADS 0, or 13+) becomes a panic in +/// the swap row. `None` here means this backend must do no LBA +/// arithmetic on the namespace — the class binding refuses bring-up on +/// it, and a generic bind over it yields a zero-capacity store. +pub fn blocks_per_page(block_size: u32) -> Option { + (block_size.is_power_of_two() && (block_size as u64) <= PAGE_BYTES) + .then(|| (PAGE_BYTES / block_size as u64) as u32) +} + +/// Word `i` of the self-test's probe pattern for `seed` — the splitmix64 +/// finalizer over `seed + i`, so every word differs and the whole page is +/// a function of the boot-unique seed (see [`NvmeSwap::self_test`] for +/// why fixed bytes would be a false-pass hazard). +fn probe_word(seed: u64, i: usize) -> u64 { + let mut z = seed.wrapping_add(0x9E37_79B9_7F4A_7C15u64.wrapping_mul(i as u64 + 1)); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// Discard ranges accumulated before one DSM command deallocates them +/// all (the table page holds up to 256; 64 amortizes well while keeping +/// reclaim latency bounded). +pub const DSM_BATCH: usize = 64; + +/// The completion-wait seam and its polled posture, hoisted to the +/// hardware-neutral block crate so every channel consumer — this swap face, +/// the filesystem's direct provider — waits through one contract. The +/// re-export preserves this module's original paths; the semantics +/// (including the pace-`false` → one final poll discipline) are unchanged. +pub use kore_driver::block::{CompletionPacer, MAX_POLLS, SpinPacer}; + +/// The backend's visible books. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct NvmeSwapStats { + /// Pages currently stored on disk. + pub stored_pages: u32, + /// Immutable extents currently stored. + pub stored_extents: u32, + /// Slot capacity (the swap partition size, in pages). + pub capacity_pages: u32, + /// Pages written out / restored in / discarded, lifetime. + pub outs: u64, + /// Pages restored. + pub ins: u64, + /// Pages discarded. + pub discards: u64, + /// DSM commands issued (each covers up to [`DSM_BATCH`] ranges). + pub dsm_commands: u64, + /// Ops refused [`SwapRefusal::Backend`] at the device layer — a failed + /// status, a completion that never came ([`timeouts`](Self::timeouts) + /// counts those root events), or a submit the wedge blocked. + pub io_errors: u64, + /// Commands that never completed inside their wait budget. Each one + /// wedges the datapath until its late completion is reaped — see + /// [`NvmeSwap::io`]. + pub timeouts: u64, +} + +/// The endowment: a brought-up controller + one I/O qpair + the slot budget. +/// +/// This constructor-shaped generic bundle is host-testable but cannot resolve +/// itself from a production bootstrap. The concrete class adapter is +/// `kore_svc_swap_nvme::NvmeSwapBinding` (`services/swap-nvme`): it consumes +/// the realized device resources, brings up the controller, creates the +/// qpair, and retains the resulting affine authority tree. +/// **The designated swap region** — the block range this backend owns. +/// +/// It exists because the alternative destroyed a disk. This store used to +/// address the namespace **raw**: slot `n` meant LBA `n × blocks_per_page`, +/// so slot 0 was LBA 0 — the protective MBR and primary GPT header — and +/// the whole store was "every block on the device". On QEMU that was an +/// empty scratch image and nothing complained; on a real machine the first +/// write took the partition table with it. +/// +/// A region must therefore be **given**, never assumed, and +/// [`base_lba`](Self::base_lba) is validated non-zero: a swap store that +/// can address LBA 0 is not a bug to be careful around, it is a shape that +/// must not be representable. The production row binds with `None` — a +/// zero-capacity store that refuses every write — and earns a region only +/// by reading the machine's partition table +/// ([`NvmeSwap::discover_region`]): the blocks a human or an installer +/// designated with the kore type GUID, and no others. An unprovisioned +/// disk leaves the store empty for the row's whole life, which is the +/// only honest posture for a driver that cannot prove which blocks are +/// its. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SwapRegion { + /// First LBA of the region — **never zero**. + base_lba: u64, + /// Region length in blocks. + blocks: u64, +} + +impl SwapRegion { + /// A region at `base_lba` spanning `blocks` — `None` for a zero base + /// (LBA 0 is never swap), an empty span, or arithmetic that would wrap + /// past the end of the device. + pub fn new(base_lba: u64, blocks: u64) -> Option { + if base_lba == 0 || blocks == 0 || base_lba.checked_add(blocks).is_none() { + return None; + } + Some(Self { base_lba, blocks }) + } + + /// First LBA of the region. + pub fn base_lba(&self) -> u64 { + self.base_lba + } + + /// Region length in blocks. + pub fn blocks(&self) -> u64 { + self.blocks + } +} + +pub struct NvmeSwapEndowment { + /// The controller (owns the seam + admin qpair). + pub ctrl: Controller, + /// The I/O qpair swap traffic rides. + pub qp: Qpair, + /// **The block range this store owns**, or `None` when nothing has + /// designated one — then the store binds at zero capacity and every + /// `Out` refuses [`SwapRefusal::StoreFull`], which is exactly true: a + /// store with no region has no room. See [`SwapRegion`]. + pub region: Option, + /// Swap capacity ceiling in pages (clamped to the region). + pub capacity_pages: u32, +} + +impl Bundle for NvmeSwapEndowment { + /// The driver profile: **the DMA pool and nothing else** + /// (`res/DEVICE_SERVICE.md` §10). No `role::DEVICE`: a driver row + /// *claims* its controller from the device broker at startup, so the + /// founder never chooses an NVMe function — exactly the input row's + /// posture. (Swap adds no extra role — the store capacity is a + /// bind-time parameter, not a founded cap.) + const PROFILE: Profile = Profile::new("nvme-swap", &[RoleSlot::at(role::DMA_POOL)]); + + /// Fail-closed for the **generic** endowment — deliberately, and not for + /// lack of the intake. The device-cap seam now exists + /// (`kore_driver::host::DeviceEndowment`, `res/DEVICE_SERVICE.md` §8): a userland + /// swap task resolves it to `CapRegs`/`CapPool`, then `Controller::bring_up` + /// over that seam. But `NvmeSwapEndowment` is *generic* over the + /// seam (the mock uses `MockNvmeDevice`/`MockPool`; the metal task uses + /// `CapRegs`/`CapPool`), so this impl cannot pick a concrete `R`/`D` — the + /// concrete `NvmeSwap` task does the resolve + bring-up, + /// and that bring-up is an **on-metal path** (real MMIO behind the BAR; a + /// byte-backed mock BAR does not drive an NVMe controller). Host tests + /// build the endowment and [`SystemService::bind`] it directly. + fn resolve(_boot: &Bootstrap<'_>) -> Result { + Err(BindError::MissingRole { + role: role::DEVICE, + index: 0, + }) + } +} + +/// The disk swap service. See the module docs for the performance +/// contract. +pub struct NvmeSwap { + ctrl: Controller, + qp: Qpair, + ns: Namespace, + /// LBAs per page-slot. + blocks_per_page: u32, + /// First LBA of the designated region — every slot is relative to it, + /// so no slot can ever address LBA 0 (see [`SwapRegion`]). Zero only + /// when `capacity == 0`, where no I/O is reachable at all. + base_lba: u64, + /// Slot capacity — zero until a region is discovered. + capacity: u32, + /// The bind-time capacity ceiling a discovered region is clamped to + /// (bounding this store's own bookkeeping, not the disk). + capacity_ceiling: u32, + /// Shared generation-checked extent ownership. The bitmap and descriptor + /// storage are fixed at provision time; request handling allocates nothing. + extents: ExtentTable, + /// Deferred deallocate ranges + the reused DMA table page. + pending_discards: Vec, + dsm_table: D::Buf, + /// The reused transfer page — see [`bounce`](Self::bounce). + bounce: D::Buf, + /// The cid of a command that timed out and is still owed a completion + /// — **the wedge**. While set, no new command may be submitted: the + /// device may yet DMA the bounce page the stale command's PRP names, + /// so reusing it would hand a client bytes the device is still + /// writing (or write a torn page to disk). Cleared when the late + /// completion is finally reaped ([`io`](Self::io)'s entry drain). + wedged: Option, + stats: NvmeSwapStats, +} + +impl NvmeSwap { + #[inline] + /// The device LBA of slot `slot` — **relative to the designated + /// region**, so slot 0 is the region's first block and never LBA 0. + fn lba(&self, slot: u32) -> u64 { + self.base_lba + slot as u64 * self.blocks_per_page as u64 + } + + /// Submit one command and drive its completion: poll the queue, and + /// between empty polls wait however the [`CompletionPacer`] says — + /// a spin hint (the polled posture) or a real park on the held MSI-X + /// route. On a spent pacer, one final drain before refusing: a + /// completion that landed without a delivered interrupt is still a + /// completion. + /// + /// **A completion answers only its own command.** Every wait matches + /// on the cid the submit stamped (the admin path's `submit_and_wait` + /// guard, which this path used to lack): after a timeout, the timed-out + /// command's completion still arrives *eventually*, and without the + /// match the next op would claim it — returning before its own + /// transfer finished, which for a restore means handing the client + /// whatever the bounce page held before the device wrote it. + /// + /// **A timeout wedges the datapath.** The unfinished command's PRP + /// names the bounce page (or the DSM table), and the device may DMA it + /// at any moment until the completion is reaped — so while one is + /// outstanding, no new command is submitted and every op refuses + /// [`SwapRefusal::Backend`]. The wedge clears itself: each call first + /// drains the queue, and reaping the late completion (the device is + /// done with its DMA by then — NVMe completes *after* the transfer) + /// reopens the path. A device that never completes leaves the row + /// serving honest refusals, which is what a hung disk is. + fn io( + &mut self, + cmd: crate::commands::NvmeSqe, + cookie: u64, + pacer: &mut dyn CompletionPacer, + ) -> Result<(), SwapRefusal> { + self.io_paced(cmd, cookie, pacer, false) + } + + /// [`io`](Self::io) with the completion wait's ORDER chosen by the + /// caller. `park_first = false` is the hot path: poll before pacing, + /// because a served op must never yield the CPU when its answer is + /// already in the queue. `park_first = true` is the **delivery + /// witness's** order — one pace *before* the first poll. + /// + /// It exists because the witness raced its own evidence: under a + /// synchronous device model (QEMU TCG runs the NVMe bottom-half on + /// the vCPU's own thread) the completion can be posted before the + /// submitting instruction stream reaches its first poll, so the + /// poll-first loop returns without ever parking and the route's fire + /// count stays zero — [`note::NO_DELIVERY`] reported against a + /// machine whose delivery works. KVM's separate iothread usually + /// hides the window; any code-size change re-rolls it (CI caught it, + /// local runs did not). The kernel LATCHES a fire that arrives + /// before its park (`runtime/irq`: `fire` on an empty waiter slot + /// records `pending`; `register` consumes it and completes + /// immediately), so parking first is correct in both orders: the + /// park resolves `Fired` whether the interrupt beat it or not, and + /// `fired == 0` after a park-first probe genuinely means + /// undelivered. + fn io_paced( + &mut self, + cmd: crate::commands::NvmeSqe, + cookie: u64, + pacer: &mut dyn CompletionPacer, + park_first: bool, + ) -> Result<(), SwapRefusal> { + self.ensure_bounce_idle()?; + let cid = self + .qp + .submit_one(cmd, cookie, self.ctrl.regs()) + .map_err(|_| SwapRefusal::Backend)?; + if park_first { + // The witness order (see above). A spent budget here is not + // a refusal yet — the loop's final polls still run. + let _ = pacer.pace(); + } + loop { + match self.qp.poll(self.ctrl.regs()) { + // A predecessor's stray completion is never this op's + // answer — drain it and poll again at once. + Some(c) if c.cid != cid => continue, + Some(c) if c.is_success() => return Ok(()), + Some(_) => { + self.stats.io_errors += 1; + return Err(SwapRefusal::Backend); + } + None => { + if !pacer.pace() { + break; + } + } + } + } + while let Some(c) = self.qp.poll(self.ctrl.regs()) { + if c.cid != cid { + continue; + } + if c.is_success() { + return Ok(()); + } + self.stats.io_errors += 1; + return Err(SwapRefusal::Backend); + } + // Never completed: the command — and the bounce page its PRP + // names — stays the device's until the late completion is reaped. + self.wedged = Some(cid); + self.stats.timeouts += 1; + self.stats.io_errors += 1; + Err(SwapRefusal::Backend) + } + + /// Prove the shared bounce page is no longer named by a timed-out + /// command before CPU code reads or writes it. This guard must run + /// before staging an `Out`, not merely before the next submission. + fn ensure_bounce_idle(&mut self) -> Result<(), SwapRefusal> { + // Reap late completions first — the only lawful wedge release. + while let Some(c) = self.qp.poll(self.ctrl.regs()) { + if self.wedged == Some(c.cid) { + self.wedged = None; + } + } + if self.wedged.is_some() { + // The timed-out command is still the device's: its DMA may yet + // touch the bounce page, so nothing new may be staged there. + self.stats.io_errors += 1; + return Err(SwapRefusal::Backend); + } + Ok(()) + } + + /// The **bounce buffer** — one page of this driver's own DMA pool, + /// allocated at bind and reused for every transfer. + /// + /// Why a copy and not DMA straight from the tenant's frames: a page's + /// bytes reach this service through the client's staging window (a + /// plain CPU mapping), and there is no edge from a mapped view to a + /// device-visible address in this driver's IOMMU domain — the + /// `Derive × kind = IovaRegion` leaf takes a `MemLease` lessor + /// position, which a receiver of a view does not hold, and minting a + /// sub-lease to fake one is barred on purpose (ckG2b-1: a sub is + /// never registered for reachability, so a revoke could not clear the + /// device mapping). + /// + /// So the page transits memory this driver *does* own end to end: the + /// pool window was carved, mapped, and published to the device at + /// intake, so `iova()` names the very frames `cpu()` writes. One + /// 4 KiB memcpy per page — the same copy the compressing sibling pays + /// anyway, and the honest alternative to a cast that would have the + /// controller DMA to whatever physical address a CPU pointer happens + /// to equal. + fn bounce(&self) -> (*mut u8, IovaAddr) { + (self.bounce.cpu(), self.bounce.iova()) + } + + fn store_extent_from( + &mut self, + bytes: &[u8], + pages: u32, + pacer: &mut dyn CompletionPacer, + ) -> Result { + if pages == 0 || bytes.len() != pages as usize * PAGE_BYTES as usize { + return Err(SwapRefusal::BadPage); + } + // A timed-out predecessor may still have device ownership of the + // shared bounce page. Refuse before even copying client bytes into it. + self.ensure_bounce_idle()?; + let extent = self.extents.claim(pages).ok_or(SwapRefusal::StoreFull)?; + let range = self + .extents + .range(extent, pages) + .expect("fresh extent claim resolves"); + // A released extent is immediately eligible for logical reuse while + // its advisory DSM may still be batched. Never let that stale range + // survive across a new publication: a later flush could otherwise + // deallocate freshly written blocks. Dropping an overlapping range + // is safe (DSM is optional) and keeps this hot path allocation-free. + self.cancel_overlapping_discards(range.start as u32, pages); + for (page, slot) in bytes.chunks_exact(PAGE_BYTES as usize).zip(range.clone()) { + let (cpu, iova) = self.bounce(); + // SAFETY: `bounce` is a whole pool-allocated page this driver + // owns exclusively, and `page` is a distinct staging borrow. + unsafe { core::ptr::copy_nonoverlapping(page.as_ptr(), cpu, PAGE_BYTES as usize) }; + let cmd = self + .ns + .write(self.lba(slot as u32), self.blocks_per_page, iova, None) + .map_err(|_| SwapRefusal::Backend); + let result = cmd.and_then(|cmd| self.io(cmd, slot as u64, pacer)); + if let Err(refusal) = result { + // Some disk blocks may have been written, but no handle was + // returned and no logical page remains live. A later claim + // overwrites them before publication. + let _ = self.extents.release(extent, pages); + return Err(refusal); + } + } + self.stats.stored_pages += pages; + self.stats.stored_extents += 1; + self.stats.outs += 1; + Ok(extent) + } + + fn cancel_overlapping_discards(&mut self, start_slot: u32, pages: u32) { + let first = self.lba(start_slot); + let blocks = u64::from(self.blocks_per_page) * u64::from(pages); + let end = first.saturating_add(blocks); + self.pending_discards.retain(|range| { + let range_end = range.slba.saturating_add(u64::from(range.nlb)); + range_end <= first || range.slba >= end + }); + } + + fn load_extent_into( + &mut self, + extent: SwapExtent, + pages: u32, + bytes: &mut [u8], + pacer: &mut dyn CompletionPacer, + ) -> Result<(), SwapRefusal> { + let range = self + .extents + .range(extent, pages) + .ok_or(SwapRefusal::BadToken)?; + if bytes.len() != pages as usize * PAGE_BYTES as usize { + return Err(SwapRefusal::BadDst); + } + for (page, slot) in bytes.chunks_exact_mut(PAGE_BYTES as usize).zip(range) { + let (cpu, iova) = self.bounce(); + let cmd = self + .ns + .read(self.lba(slot as u32), self.blocks_per_page, iova, None) + .map_err(|_| SwapRefusal::Backend)?; + self.io(cmd, slot as u64, pacer)?; + // SAFETY: completion proves the device is done writing the + // driver-owned bounce page; `page` is a distinct staging borrow. + unsafe { core::ptr::copy_nonoverlapping(cpu, page.as_mut_ptr(), PAGE_BYTES as usize) }; + } + self.stats.ins += 1; + Ok(()) + } + + fn release_extent( + &mut self, + extent: SwapExtent, + pages: u32, + pacer: &mut dyn CompletionPacer, + ) -> Result<(), SwapRefusal> { + let range = self + .extents + .range(extent, pages) + .ok_or(SwapRefusal::BadToken)?; + let start = range.start as u32; + let blocks = self + .blocks_per_page + .checked_mul(pages) + .ok_or(SwapRefusal::Backend)?; + self.extents + .release(extent, pages) + .expect("validated extent releases"); + self.stats.stored_pages -= pages; + self.stats.stored_extents -= 1; + // One contiguous extent becomes one DSM range, independent of its + // page count. + self.pending_discards + .push(DsmRange::new(self.lba(start), blocks)); + if self.pending_discards.len() >= DSM_BATCH { + self.flush_discards_paced(pacer); + } + Ok(()) + } + + fn page_out_from( + &mut self, + page: &[u8], + pacer: &mut dyn CompletionPacer, + ) -> Result { + self.store_extent_from(page, 1, pacer) + } + + fn page_in_into( + &mut self, + extent: SwapExtent, + page: &mut [u8], + pacer: &mut dyn CompletionPacer, + ) -> Result<(), SwapRefusal> { + self.load_extent_into(extent, 1, page, pacer)?; + self.release_extent(extent, 1, pacer) + } + + fn discard( + &mut self, + extent: SwapExtent, + pacer: &mut dyn CompletionPacer, + ) -> Result<(), SwapRefusal> { + self.release_extent(extent, 1, pacer)?; + self.stats.discards += 1; + Ok(()) + } + + /// Issue the batched Dataset Management deallocate for every pending + /// discard (public: the host loop flushes on idle) — the spin-paced + /// form; the serve loop's paced sibling is + /// [`flush_discards_paced`](Self::flush_discards_paced). + pub fn flush_discards(&mut self) { + self.flush_discards_paced(&mut SpinPacer::default()); + } + + /// [`flush_discards`](Self::flush_discards) over a caller-chosen + /// pacer. A DSM failure is + /// deliberately non-fatal — deallocate is advisory (the blocks are + /// already logically free); the ranges are dropped either way and the + /// error stays visible in the books. + pub fn flush_discards_paced(&mut self, pacer: &mut dyn CompletionPacer) { + let n = self.pending_discards.len(); + if n == 0 { + return; + } + debug_assert!(n <= 256, "a DSM table page holds 256 ranges"); + // Write the range table into the reused DMA page. + // SAFETY: dsm_table is a CPU-visible page ≥ n × 16 bytes. + unsafe { + core::ptr::copy_nonoverlapping( + self.pending_discards.as_ptr(), + self.dsm_table.cpu() as *mut DsmRange, + n, + ); + } + if let Ok(cmd) = self.ns.deallocate(self.dsm_table.iova(), n as u16) { + if self.io(cmd, u64::MAX, pacer).is_ok() { + self.stats.dsm_commands += 1; + } + } + self.pending_discards.clear(); + } + + /// Serve one **byte-moving** swap op over the client's staging window + /// and a caller-chosen pacer — the form the production serve loop + /// drives: the completion wait inside is a park on the held MSI-X + /// route when the caller's pacer carries one. + /// + /// `staging` is the window the row adopted at + /// [`SwapRequest::Attach`]; the row owns that mapping (it is + /// capability-shaped and this core is not), so it arrives here as + /// plain bytes. `None` — or a page index outside the window — + /// refuses [`SwapRefusal::NoStaging`]: the index is client data, and + /// the window's length is the only authority on it. + /// + /// `Attach` itself is **not** handled here. Adopting a transferred + /// capability is the row's business (`kore-svc-swap-nvme`), and this + /// core holds no session authority at all. + pub fn handle( + &mut self, + req: SwapRequest, + staging: Option<&mut [u8]>, + pacer: &mut dyn CompletionPacer, + ) -> SwapResponse { + match req { + SwapRequest::Attach { .. } => SwapResponse::Refused(SwapRefusal::NoStaging), + SwapRequest::Out { page } => { + let Some(bytes) = staging + .as_deref() + .and_then(|s| kore_svc_swap::staging_page(s, page)) + else { + return SwapResponse::Refused(SwapRefusal::NoStaging); + }; + // The staging borrow ends with this copy (page_out stages + // into the bounce buffer immediately), so the driver's own + // `&mut self` and the window never alias. + match self.page_out_from(bytes, pacer) { + Ok(extent) => SwapResponse::Stored { extent }, + Err(r) => SwapResponse::Refused(r), + } + } + SwapRequest::In { extent, page } => { + let mut staging = staging; + let Some(bytes) = staging + .as_deref_mut() + .and_then(|s| kore_svc_swap::staging_page_mut(s, page)) + else { + return SwapResponse::Refused(SwapRefusal::NoStaging); + }; + match self.page_in_into(extent, bytes, pacer) { + Ok(()) => SwapResponse::Restored, + Err(r) => SwapResponse::Refused(r), + } + } + SwapRequest::Discard { extent } => match self.discard(extent, pacer) { + Ok(()) => SwapResponse::Discarded, + Err(r) => SwapResponse::Refused(r), + }, + SwapRequest::Store { page, pages } => { + let Some(bytes) = staging + .as_deref() + .and_then(|window| staging_extent(window, page, pages)) + else { + return SwapResponse::Refused(SwapRefusal::NoStaging); + }; + match self.store_extent_from(bytes, pages, pacer) { + Ok(extent) => SwapResponse::Stored { extent }, + Err(r) => SwapResponse::Refused(r), + } + } + SwapRequest::Load { + extent, + page, + pages, + } => { + let mut staging = staging; + let Some(bytes) = staging + .as_deref_mut() + .and_then(|window| staging_extent_mut(window, page, pages)) + else { + return SwapResponse::Refused(SwapRefusal::NoStaging); + }; + match self.load_extent_into(extent, pages, bytes, pacer) { + Ok(()) => SwapResponse::Loaded, + Err(r) => SwapResponse::Refused(r), + } + } + SwapRequest::Release { extent, pages } => { + match self.release_extent(extent, pages, pacer) { + Ok(()) => { + self.stats.discards += 1; + SwapResponse::Released + } + Err(r) => SwapResponse::Refused(r), + } + } + // The books are the ROW's business, like `Attach`: the core + // knows its own stats but not the route posture or the + // staging window, and half an answer would read as whole. + SwapRequest::Status => SwapResponse::Refused(SwapRefusal::Backend), + } + } + + /// **The I/O-path self-test**, driven through `pacer` — a + /// write/read/verify round-trip when the store owns blocks to write, + /// a bounded read when it owns none. + /// + /// The write leg once ran unconditionally, and with no region concept + /// "slot 0" was LBA 0 — every boot overwrote the machine's protective + /// MBR and primary GPT header, and QEMU's scratch image made it + /// invisible. The line that survives: **a driver may write only + /// inside a region something designated as its** ([`SwapRegion`]). + /// So: + /// + /// - **No region** (or slot 0 holds a stored page): one read of the + /// first addressable block, bytes discarded unread — proves the + /// queue and, route-parked, the MSI-X delivery, and changes + /// nothing. + /// - **A designated region with slot 0 free**: write a `seed`-derived + /// pattern to the region's first slot, clobber the bounce page (a + /// read that silently no-ops must not pass), read the slot back, + /// and compare — the complete datapath, bounce page → device write + /// → device read → bytes intact, before the row ever beacons. + /// + /// `seed` must be **boot-unique** (the row passes its kernel-anchored + /// probe deadline): the pattern lands at the same LBA every boot, so + /// a fixed pattern would let a device whose writes silently fail + /// round-trip *last* boot's bytes and pass. Slot 0's blocks stay + /// scratch to the store afterwards — nothing marks them stored, and + /// the first real `Out` to the slot overwrites them. + pub fn self_test( + &mut self, + pacer: &mut dyn CompletionPacer, + seed: u64, + ) -> Result<(), SwapRefusal> { + let (cpu, iova) = self.bounce(); + if self.capacity == 0 || self.extents.is_claimed(0) { + let read = self + .ns + .read(self.lba(0), self.blocks_per_page, iova, None) + .map_err(|_| SwapRefusal::Backend)?; + // Park-first: this read IS the delivery witness (see + // `io_paced`) — a completion that beats the first poll must + // not zero the fire count. + return self.io_paced(read, u64::MAX - 1, pacer, true); + } + // The write leg — lawful exactly here: slot 0 lies inside the + // designated region and holds no stored page. + let words = PAGE_BYTES as usize / 8; + // SAFETY: the bounce page is a whole pool-allocated, page-aligned + // (hence u64-aligned) page this driver owns exclusively. + let bounce = unsafe { core::slice::from_raw_parts_mut(cpu.cast::(), words) }; + for (i, w) in bounce.iter_mut().enumerate() { + *w = probe_word(seed, i); + } + let write = self + .ns + .write(self.lba(0), self.blocks_per_page, iova, None) + .map_err(|_| SwapRefusal::Backend)?; + // Park-first on the sequence's FIRST command — the delivery + // witness (see `io_paced`); the read-back keeps the hot order. + self.io_paced(write, u64::MAX - 1, pacer, true)?; + bounce.fill(!seed); + let read = self + .ns + .read(self.lba(0), self.blocks_per_page, iova, None) + .map_err(|_| SwapRefusal::Backend)?; + self.io(read, u64::MAX - 1, pacer)?; + // SAFETY: as above — the read's completion was reaped, so the + // device is done writing the page. + let bounce = unsafe { core::slice::from_raw_parts(cpu.cast::(), words) }; + for (i, w) in bounce.iter().enumerate() { + if *w != probe_word(seed, i) { + self.stats.io_errors += 1; + return Err(SwapRefusal::Backend); + } + } + Ok(()) + } + + /// **Discover this store's region from the machine's partition + /// table** — read-only, and the only thing that ever gives this store + /// blocks to write. + /// + /// Reads the primary GPT header and its entry array through the + /// driver's own bounce page, then looks for the partition whose type + /// is [`KORE_SWAP_TYPE`](kore_driver::gpt::KORE_SWAP_TYPE). On a + /// match the store is provisioned in place: base LBA, capacity, and + /// the stored-ness bitmap all come from what the table says, so the + /// blocks this store can address are exactly the blocks a human (or an + /// installer) designated. + /// + /// Every failure — no table, a damaged one, no kore partition — leaves + /// the store at **zero capacity**, which refuses every write. The + /// error is returned for the row's readiness note; nothing about a + /// failed discovery is fatal, and nothing about it writes. + pub fn discover_region( + &mut self, + pacer: &mut dyn CompletionPacer, + ) -> Result<(), kore_driver::gpt::GptError> { + use kore_driver::gpt; + + // A namespace whose LBA format cannot carry pages admits no LBA + // arithmetic at all (`blocks_per_page` refused it at bind) — + // refuse before the first division by its block size. + if self.blocks_per_page == 0 { + return Err(gpt::GptError::BadGeometry); + } + // Provisioned already: the extent books are live, and + // re-provisioning would reset them under + // stored pages while `stats.stored_pages` still counts them. One + // designation per binding; a second read of the table is a no-op. + if self.capacity != 0 { + return Ok(()); + } + let bs = self.ns.block_size as usize; + let per_read = (self.bounce.len() / bs).max(1) as u32; + + // 1 · the primary header, at LBA 1. + self.read_into_bounce(gpt::PRIMARY_HEADER_LBA, 1, pacer) + .map_err(|_| gpt::GptError::NotGpt)?; + // SAFETY: the bounce page is driver-owned and at least one block; + // the device just wrote `bs` bytes of it. + let block = unsafe { core::slice::from_raw_parts(self.bounce.cpu(), bs) }; + let geom = gpt::parse_header(block)?; + + // 2 · the entry array, in bounce-sized chunks. The whole array + // must be checksummed together, so it is staged into a fixed + // buffer sized by the module's own ceiling — no allocation on + // a path that runs before the store exists. + let bytes = geom.entry_array_bytes(); + if bytes > gpt::MAX_ENTRY_ARRAY_BYTES { + return Err(gpt::GptError::BadGeometry); + } + let mut entries = [0u8; gpt::MAX_ENTRY_ARRAY_BYTES]; + let mut done = 0usize; + let mut lba = geom.entry_lba; + while done < bytes { + let want = ((bytes - done).div_ceil(bs) as u32).min(per_read); + self.read_into_bounce(lba, want, pacer) + .map_err(|_| gpt::GptError::BadGeometry)?; + let got = (want as usize * bs).min(bytes - done); + // SAFETY: as above — `got` bytes the device just wrote into + // the driver-owned bounce page. + let chunk = unsafe { core::slice::from_raw_parts(self.bounce.cpu(), got) }; + entries[done..done + got].copy_from_slice(chunk); + done += got; + lba += want as u64; + } + + let extent = gpt::find_partition(&entries[..bytes], &geom, gpt::KORE_SWAP_TYPE)?; + // The table is the authority, but the device is the last word: a + // partition running past the namespace is a designation bug. + if extent.end_lba >= self.ns.block_count { + return Err(gpt::GptError::BadGeometry); + } + let Some(region) = SwapRegion::new(extent.start_lba, extent.blocks()) else { + return Err(gpt::GptError::BadGeometry); + }; + self.provision(region); + Ok(()) + } + + /// One bounded read into the bounce page. + fn read_into_bounce( + &mut self, + lba: u64, + blocks: u32, + pacer: &mut dyn CompletionPacer, + ) -> Result<(), SwapRefusal> { + let (_, iova) = self.bounce(); + let cmd = self + .ns + .read(lba, blocks, iova, None) + .map_err(|_| SwapRefusal::Backend)?; + self.io(cmd, u64::MAX - 2, pacer) + } + + /// Install a discovered region: base LBA, capacity, and the + /// bookkeeping sized to it. Private because a region may only come + /// from [`discover_region`](Self::discover_region) — the store never + /// designates its own blocks, provisions twice, or provisions over an + /// unusable LBA format (both guarded at the discovery door). + fn provision(&mut self, region: SwapRegion) { + debug_assert!(self.blocks_per_page > 0, "guarded at discover_region"); + debug_assert_eq!(self.capacity, 0, "one designation per binding"); + let region_pages = (region.blocks() / self.blocks_per_page.max(1) as u64) as u32; + self.base_lba = region.base_lba(); + self.capacity = self.capacity_ceiling.min(region_pages); + self.extents = ExtentTable::new(self.capacity); + } + + /// Whether deferred discard ranges await their DSM — the serve loop's + /// idle-flush predicate (see [`flush_discards`](Self::flush_discards)). + pub fn has_pending_discards(&self) -> bool { + !self.pending_discards.is_empty() + } + + /// The books. + pub fn stats(&self) -> NvmeSwapStats { + NvmeSwapStats { + capacity_pages: self.capacity, + stored_extents: self.extents.live_extents(), + ..self.stats + } + } + + /// The qpair's books (submitted / completed / errors), used by the + /// service row to compose its typed [`SwapStatus`](kore_svc_swap::SwapStatus). + pub fn qp_stats(&self) -> crate::QpairStats { + self.qp.stats() + } + + /// **Quiesce the device** — flush the deferred discards, then run the + /// controller's orderly shutdown. The class binding's quiesce hook + /// ([`kore_driver::host::DeviceBinding::quiesce`]): after `Ok`, the + /// controller has stopped all DMA, so the authority tree above the seam + /// can be retired without racing an in-flight transfer. + pub fn quiesce(&mut self, max_polls: u32, mut pause: impl FnMut()) -> crate::Result<()> { + self.flush_discards(); + self.ctrl.shutdown(max_polls, &mut pause) + } + + /// Release the platform seam — the unbind half of the containment + /// contract. Drops the I/O qpair and the DSM table page (pool slices + /// with no lifecycle of their own) and passes through the controller's + /// own [`into_seam`](crate::Controller::into_seam). + pub fn into_seam(self) -> (R, D) { + let Self { ctrl, qp, .. } = self; + drop(qp); + ctrl.into_seam() + } +} + +/// The **wire** half — the swap vocabulary this backend speaks, cited +/// from the vocabulary's own crate exactly as `kore-svc-swap-nvme`'s +/// production composition does: *one wire form, N backend services*. +/// Every codec delegates rather than re-encoding, so the zram tier, the +/// production disk row, and this driver-core face answer the same rings +/// byte-for-byte — which is what lets a pager client take any of them +/// without changing a line. +/// +/// A swap backend can mint no authority +/// ([`REPLY_GRANTS`](SvcProtocol::REPLY_GRANTS) `= 0`): it holds pages. +impl SvcProtocol for NvmeSwap { + type Request = SwapRequest; + type Response = SwapResponse; + type ReqWire = SwapReqWire; + type RepWire = SwapRepWire; + + const REPLY_GRANTS: usize = 0; + + fn encode_req(req: &SwapRequest) -> SwapReqWire { + kore_svc_swap::wire::encode_req(req) + } + fn decode_req(wire: &SwapReqWire) -> Option { + kore_svc_swap::wire::decode_req(wire) + } + fn encode_rep(rep: &SwapResponse, grants: u8) -> SwapRepWire { + kore_svc_swap::wire::encode_rep(rep, grants) + } + fn decode_rep(wire: &SwapRepWire) -> Option<(SwapResponse, u8)> { + kore_svc_swap::wire::decode_rep(wire) + } +} + +impl SystemService for NvmeSwap { + const NAME: &'static str = "nvme-swap"; + type Endowment = NvmeSwapEndowment; + + fn bind(e: NvmeSwapEndowment) -> Self { + let ns = e.ctrl.namespace(); + // Identify data is device input — an LBA format that cannot carry + // whole pages binds a store that must do no LBA arithmetic at all + // (`blocks_per_page` 0 ⇒ capacity 0 below, and discovery refuses + // before its first division). The production class binding + // refuses such a device outright; this guard is for the generic + // path, where `bind` cannot fail. + let blocks_per_page = blocks_per_page(ns.block_size).unwrap_or(0); + // Capacity comes from the DESIGNATED REGION, never from the + // namespace: "every block on the device" is what cost a real + // machine its partition table. No region ⇒ no capacity ⇒ every + // write refuses, and `base_lba` stays 0 only because no I/O can + // reach it. + let (base_lba, capacity) = match e.region { + Some(region) if blocks_per_page > 0 => { + // Whole pages inside the region, clamped to the ceiling and + // to what the namespace actually holds. + let region_pages = (region.blocks() / blocks_per_page as u64) as u32; + let fits = region + .base_lba() + .checked_add(region.blocks()) + .is_some_and(|end| end <= ns.block_count); + if fits { + (region.base_lba(), e.capacity_pages.min(region_pages)) + } else { + // A region past the end of the device is a designation + // bug; bind with no store rather than a truncated one. + (0, 0) + } + } + _ => (0, 0), + }; + let dsm_table = e + .ctrl + .pool() + .alloc(4096) + .expect("bind-time DSM table page (the zram bind-allocates posture)"); + let bounce = e + .ctrl + .pool() + .alloc(PAGE_BYTES as usize) + .expect("bind-time bounce page (the zram bind-allocates posture)"); + Self { + ctrl: e.ctrl, + qp: e.qp, + ns, + blocks_per_page, + base_lba, + capacity, + capacity_ceiling: e.capacity_pages, + extents: ExtentTable::new(capacity), + pending_discards: Vec::with_capacity(DSM_BATCH), + dsm_table, + bounce, + wedged: None, + stats: NvmeSwapStats::default(), + } + } + + /// Serve one swap op — [`handle`](Self::handle) at the spin-paced + /// posture (the host tests' path; production drives `handle` with the + /// route-park pacer). Never calls [`Serve::grant`] — a swap backend + /// holds pages, it mints nothing (the zram contract, verbatim). + fn serve(&mut self, req: SwapRequest, sv: &mut Serve<'_, SwapResponse>) { + // No staging at this altitude: the window is a capability the ROW + // adopts (`kore-svc-swap-nvme`'s `DiskSwap`), and this core holds + // no session authority. Every byte-moving verb therefore refuses + // here — the production entry drives `handle` with the row's + // adopted window instead. Kept rather than dropped so the type + // still carries its name, endowment, and books. + let resp = self.handle(req, None, &mut SpinPacer::default()); + sv.reply(resp); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::MockNvmeDevice; + use kore_svc_swap::{staging_page, staging_page_mut}; + + /// The tests' designated region — non-zero on purpose (the whole + /// point), and inside the mock's 64-block backing disk. + const TEST_REGION_LBA: u64 = 8; + const TEST_REGION_BLOCKS: u64 = 48; + + /// A bound swap face over the host mock, plus a staging window. + fn bound(pages: usize) -> (NvmeSwap, Vec) { + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).expect("bring-up"); + ctrl.identify(16, || {}).expect("identify"); + ctrl.set_num_queues(1, 16, || {}).expect("set queues"); + let qp = ctrl.create_io_qpair(1, 32, None, 16, || {}).expect("qpair"); + let svc = NvmeSwap::bind(NvmeSwapEndowment { + ctrl, + qp, + // A region well clear of LBA 0 — the conventional first-partition + // start, so the tests exercise the offset rather than the + // shape that destroyed a disk. + region: SwapRegion::new(TEST_REGION_LBA, TEST_REGION_BLOCKS), + capacity_pages: 64, + }); + (svc, alloc::vec![0u8; pages * PAGE_BYTES as usize]) + } + + /// **The datapath, end to end**: a page staged by the client goes out + /// to the device through the bounce buffer and comes back byte-identical. + /// This is what `view_iova`'s typed refusal could never do — the store + /// and restore now move real bytes over real (mock) DMA. + #[test] + fn a_page_round_trips_through_the_bounce_buffer() { + let (mut svc, mut staging) = bound(2); + let page: Vec = (0..PAGE_BYTES as usize).map(|i| (i * 7) as u8).collect(); + staging_page_mut(&mut staging, 1) + .unwrap() + .copy_from_slice(&page); + + let stored = svc.handle( + SwapRequest::Out { page: 1 }, + Some(&mut staging), + &mut SpinPacer::default(), + ); + let SwapResponse::Stored { extent } = stored else { + panic!("page-out stores: {stored:?}"); + }; + assert_eq!(svc.stats().outs, 1); + + // Clobber staging: a restore that silently no-ops must not pass. + staging_page_mut(&mut staging, 1).unwrap().fill(0xAA); + assert_eq!( + svc.handle( + SwapRequest::In { extent, page: 1 }, + Some(&mut staging), + &mut SpinPacer::default() + ), + SwapResponse::Restored + ); + assert_eq!( + kore_svc_swap::staging_page(&staging, 1).unwrap(), + &page[..], + "the restored bytes are the stored bytes, through the device" + ); + assert_eq!(svc.stats().ins, 1); + assert_eq!(svc.stats().stored_pages, 0, "restore consumed the copy"); + assert_eq!(svc.stats().io_errors, 0); + } + + /// One control operation moves a non-power-of-two residency segment; + /// repeated loads preserve the disk copy and release queues one + /// contiguous DSM range for the whole extent. + #[test] + fn an_extent_load_is_repeatable_and_release_is_one_range() { + let (mut svc, mut staging) = bound(6); + for page in 0..3 { + staging_page_mut(&mut staging, page) + .unwrap() + .fill(0x61 + page as u8); + } + let SwapResponse::Stored { extent } = svc.handle( + SwapRequest::Store { page: 0, pages: 3 }, + Some(&mut staging), + &mut SpinPacer::default(), + ) else { + panic!("extent stores"); + }; + assert_eq!(svc.stats().stored_pages, 3); + assert_eq!(svc.stats().stored_extents, 1); + + staging[3 * PAGE_BYTES as usize..].fill(0); + assert_eq!( + svc.handle( + SwapRequest::Load { + extent, + page: 3, + pages: 3, + }, + Some(&mut staging), + &mut SpinPacer::default(), + ), + SwapResponse::Loaded + ); + for page in 0..3 { + assert!( + kore_svc_swap::staging_page(&staging, 3 + page) + .unwrap() + .iter() + .all(|&byte| byte == 0x61 + page as u8) + ); + } + assert_eq!(svc.stats().stored_pages, 3, "load is non-destructive"); + + assert_eq!( + svc.handle( + SwapRequest::Release { extent, pages: 3 }, + Some(&mut staging), + &mut SpinPacer::default(), + ), + SwapResponse::Released + ); + assert_eq!(svc.stats().stored_pages, 0); + assert_eq!(svc.stats().stored_extents, 0); + assert_eq!(svc.pending_discards.len(), 1); + assert_eq!(svc.pending_discards[0].slba, TEST_REGION_LBA); + assert_eq!(svc.pending_discards[0].nlb, 3); + assert_eq!( + svc.handle( + SwapRequest::Load { + extent, + page: 0, + pages: 3, + }, + Some(&mut staging), + &mut SpinPacer::default(), + ), + SwapResponse::Refused(SwapRefusal::BadToken) + ); + } + + /// A deferred DSM belongs to the retired generation, not to the physical + /// blocks forever. Reusing any part of its range cancels that advisory + /// command before new data is published. + #[test] + fn extent_reuse_cancels_a_stale_deferred_dsm() { + let (mut svc, mut staging) = bound(1); + // Constrain the logical store to one slot so the next generation must + // reuse the same physical LBAs instead of advancing next-fit. + svc.capacity = 1; + svc.extents = ExtentTable::new(1); + staging_page_mut(&mut staging, 0).unwrap().fill(0x17); + let SwapResponse::Stored { extent: first } = svc.handle( + SwapRequest::Out { page: 0 }, + Some(&mut staging), + &mut SpinPacer::default(), + ) else { + panic!("first generation stores"); + }; + assert_eq!( + svc.handle( + SwapRequest::Discard { extent: first }, + Some(&mut staging), + &mut SpinPacer::default(), + ), + SwapResponse::Discarded + ); + assert_eq!(svc.pending_discards.len(), 1); + + staging_page_mut(&mut staging, 0).unwrap().fill(0x29); + let SwapResponse::Stored { extent: second } = svc.handle( + SwapRequest::Out { page: 0 }, + Some(&mut staging), + &mut SpinPacer::default(), + ) else { + panic!("second generation stores"); + }; + assert_ne!(first, second); + assert!( + svc.pending_discards.is_empty(), + "the old generation's DSM must not outlive physical reuse" + ); + + staging_page_mut(&mut staging, 0).unwrap().fill(0); + assert_eq!( + svc.handle( + SwapRequest::In { + extent: second, + page: 0, + }, + Some(&mut staging), + &mut SpinPacer::default(), + ), + SwapResponse::Restored + ); + assert!( + staging_page(&staging, 0) + .unwrap() + .iter() + .all(|&byte| byte == 0x29) + ); + } + + /// Two pages stored back to back keep their own bytes — the bounce + /// buffer is reused, so a second transfer must not serve the first + /// one's contents. + #[test] + fn the_reused_bounce_page_does_not_leak_between_transfers() { + let (mut svc, mut staging) = bound(2); + staging_page_mut(&mut staging, 0).unwrap().fill(0x11); + staging_page_mut(&mut staging, 1).unwrap().fill(0x22); + + let SwapResponse::Stored { extent: a } = svc.handle( + SwapRequest::Out { page: 0 }, + Some(&mut staging), + &mut SpinPacer::default(), + ) else { + panic!("first stores"); + }; + let SwapResponse::Stored { extent: b } = svc.handle( + SwapRequest::Out { page: 1 }, + Some(&mut staging), + &mut SpinPacer::default(), + ) else { + panic!("second stores"); + }; + assert_ne!(a, b, "distinct slots"); + + staging.fill(0); + svc.handle( + SwapRequest::In { extent: a, page: 0 }, + Some(&mut staging), + &mut SpinPacer::default(), + ); + svc.handle( + SwapRequest::In { extent: b, page: 1 }, + Some(&mut staging), + &mut SpinPacer::default(), + ); + assert!( + kore_svc_swap::staging_page(&staging, 0) + .unwrap() + .iter() + .all(|&x| x == 0x11) + ); + assert!( + kore_svc_swap::staging_page(&staging, 1) + .unwrap() + .iter() + .all(|&x| x == 0x22) + ); + } + + /// Every byte-moving verb refuses without staging, and refuses an + /// index outside the window — the client's page index is data. + #[test] + fn missing_or_out_of_bounds_staging_is_a_refusal() { + let (mut svc, mut staging) = bound(1); + assert_eq!( + svc.handle( + SwapRequest::Out { page: 0 }, + None, + &mut SpinPacer::default() + ), + SwapResponse::Refused(SwapRefusal::NoStaging), + "no window" + ); + assert_eq!( + svc.handle( + SwapRequest::Out { page: 1 }, + Some(&mut staging), + &mut SpinPacer::default() + ), + SwapResponse::Refused(SwapRefusal::NoStaging), + "one past the last page" + ); + // Attach is the row's business, never the core's. + assert_eq!( + svc.handle( + SwapRequest::Attach { + announced: kore_abi::session::SlotRef::from_wire(3) + }, + Some(&mut staging), + &mut SpinPacer::default() + ), + SwapResponse::Refused(SwapRefusal::NoStaging), + ); + } + + /// A discarded page is unrestorable, and the DSM batch is deferred — + /// the store's accounting stays honest across the staging rewrite. + #[test] + fn discard_retires_the_token_and_defers_the_dsm() { + let (mut svc, mut staging) = bound(1); + staging_page_mut(&mut staging, 0).unwrap().fill(0x5A); + let SwapResponse::Stored { extent } = svc.handle( + SwapRequest::Out { page: 0 }, + Some(&mut staging), + &mut SpinPacer::default(), + ) else { + panic!("stores"); + }; + assert_eq!( + svc.handle( + SwapRequest::Discard { extent }, + Some(&mut staging), + &mut SpinPacer::default() + ), + SwapResponse::Discarded + ); + assert_eq!(svc.stats().discards, 1); + assert_eq!(svc.stats().dsm_commands, 0, "one discard defers its DSM"); + assert!( + svc.has_pending_discards(), + "the deferred range is visible to the idle-flush predicate" + ); + assert_eq!( + svc.handle( + SwapRequest::In { extent, page: 0 }, + Some(&mut staging), + &mut SpinPacer::default() + ), + SwapResponse::Refused(SwapRefusal::BadToken), + "a discarded token restores nothing" + ); + svc.flush_discards(); + assert_eq!(svc.stats().dsm_commands, 1, "the flush issues the batch"); + assert!(!svc.has_pending_discards(), "the flush drained the table"); + } +} + +/// The completion-attribution + wedge discipline: a timed-out command's +/// late completion must never be claimed by a successor, and the bounce +/// page stays quarantined until the device is provably done with it. +#[cfg(test)] +mod completion_tests { + use super::*; + use crate::mock::MockNvmeDevice; + use kore_svc_swap::{staging_page, staging_page_mut}; + + fn rig() -> ( + NvmeSwap, + MockNvmeDevice, + ) { + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).expect("bring-up"); + ctrl.identify(16, || {}).expect("identify"); + ctrl.set_num_queues(1, 16, || {}).expect("set queues"); + let qp = ctrl.create_io_qpair(1, 32, None, 16, || {}).expect("qpair"); + let svc = NvmeSwap::bind(NvmeSwapEndowment { + ctrl, + qp, + region: SwapRegion::new(8, 48), + capacity_pages: 64, + }); + (svc, dev) + } + + /// **The regression: a successor claiming its predecessor's + /// completion.** A command whose completion the device withholds + /// refuses at the pacer's edge and wedges the datapath; the next op + /// refuses WITHOUT submitting (the bounce page is still the + /// device's); reaping the late completion reopens the path, and the + /// bytes that then round-trip are the successor's own. + #[test] + fn a_timed_out_command_wedges_until_its_completion_is_reaped() { + let (mut svc, dev) = rig(); + let mut staging = alloc::vec![0u8; 2 * PAGE_BYTES as usize]; + staging_page_mut(&mut staging, 0).unwrap().fill(0x11); + staging_page_mut(&mut staging, 1).unwrap().fill(0x22); + + dev.swallow_completions(1); + assert_eq!( + svc.handle( + SwapRequest::Out { page: 0 }, + Some(&mut staging), + &mut SpinPacer::new(4), + ), + SwapResponse::Refused(SwapRefusal::Backend), + "no completion inside the budget" + ); + assert_eq!(svc.stats().timeouts, 1); + assert_eq!(svc.qp_stats().submitted, 1); + + // Wedged: the withheld command's PRP still names the bounce page, + // so the next op must refuse BEFORE anything reaches the device — + // without the cid match it would have claimed the stale + // completion as its own instead. + assert_eq!( + svc.handle( + SwapRequest::Out { page: 1 }, + Some(&mut staging), + &mut SpinPacer::default(), + ), + SwapResponse::Refused(SwapRefusal::Backend), + "wedged ops refuse" + ); + assert_eq!( + svc.qp_stats().submitted, + 1, + "nothing new reached the device while wedged" + ); + + // The late completion lands; the next op reaps it, unwedges, and + // serves correctly. + dev.release_swallowed(); + let SwapResponse::Stored { extent } = svc.handle( + SwapRequest::Out { page: 1 }, + Some(&mut staging), + &mut SpinPacer::default(), + ) else { + panic!("the post-release op stores"); + }; + assert_eq!(svc.qp_stats().submitted, 2); + staging_page_mut(&mut staging, 1).unwrap().fill(0); + assert_eq!( + svc.handle( + SwapRequest::In { extent, page: 1 }, + Some(&mut staging), + &mut SpinPacer::default(), + ), + SwapResponse::Restored + ); + assert!( + staging_page(&staging, 1) + .unwrap() + .iter() + .all(|&b| b == 0x22), + "the restored bytes are the successor's, not the timed-out op's" + ); + } + + /// The books tell the story: a timeout counts once under `timeouts` + /// (and as an io_error), the backend remains wedged, and the late + /// completion reopens it. + #[test] + fn timeouts_are_counted_and_the_wedge_is_visible() { + let (mut svc, dev) = rig(); + let mut staging = alloc::vec![0u8; PAGE_BYTES as usize]; + + dev.swallow_completions(1); + svc.handle( + SwapRequest::Out { page: 0 }, + Some(&mut staging), + &mut SpinPacer::new(4), + ); + assert_eq!(svc.stats().timeouts, 1); + assert_eq!(svc.stats().io_errors, 1); + + assert!(svc.wedged.is_some(), "the timed-out command remains owed"); + + dev.release_swallowed(); + svc.handle( + SwapRequest::Out { page: 0 }, + Some(&mut staging), + &mut SpinPacer::default(), + ); + assert!(svc.wedged.is_none(), "reaping the completion unwedges"); + } +} + +/// Namespace geometry is device-controlled input: every shape identify +/// can report must decode and bind without panicking, and only formats +/// that carry whole pages may reach LBA arithmetic. +#[cfg(test)] +mod geometry_tests { + use super::*; + use crate::mock::MockNvmeDevice; + + #[test] + fn blocks_per_page_validates_device_input() { + assert_eq!(blocks_per_page(4096), Some(1)); + assert_eq!(blocks_per_page(512), Some(8)); + assert_eq!( + blocks_per_page(1), + Some(4096), + "absurd but arithmetically safe" + ); + assert_eq!(blocks_per_page(0), None, "identify's own overflow sentinel"); + assert_eq!(blocks_per_page(8192), None, "cannot carry a 4 KiB page"); + assert_eq!(blocks_per_page(3072), None, "not a power of two"); + } + + /// **The regression: `bind` divided by an unvalidated block size.** + /// An 8 KiB LBA format binds a store with no capacity — even over a + /// designated region — and discovery refuses before its first + /// division; nothing panics anywhere. + #[test] + fn an_unusable_lba_format_binds_empty_and_never_panics() { + let dev = MockNvmeDevice::new(); + dev.set_lbads(13); // 8 KiB blocks + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + ctrl.identify(16, || {}).unwrap(); + assert_eq!(ctrl.info().block_size, 8192); + ctrl.set_num_queues(1, 16, || {}).unwrap(); + let qp = ctrl.create_io_qpair(1, 32, None, 16, || {}).unwrap(); + let mut svc = NvmeSwap::bind(NvmeSwapEndowment { + ctrl, + qp, + region: SwapRegion::new(2048, 4096), + capacity_pages: 64, + }); + assert_eq!(svc.stats().capacity_pages, 0, "no page math ⇒ no store"); + + let mut staging = alloc::vec![0u8; PAGE_BYTES as usize]; + assert_eq!( + svc.handle( + SwapRequest::Out { page: 0 }, + Some(&mut staging), + &mut SpinPacer::default(), + ), + SwapResponse::Refused(SwapRefusal::StoreFull) + ); + assert_eq!( + svc.discover_region(&mut SpinPacer::default()), + Err(kore_driver::gpt::GptError::BadGeometry), + "discovery refuses before dividing by the block size" + ); + } + + /// An LBADS past the u32 width decodes to the 0 sentinel — never a + /// shift panic — and 0 fails the page-math validation like every + /// other unusable size. + #[test] + fn an_absurd_lbads_decodes_to_the_zero_sentinel() { + let dev = MockNvmeDevice::new(); + dev.set_lbads(40); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + ctrl.identify(16, || {}).unwrap(); + assert_eq!(ctrl.info().block_size, 0, "the sentinel, not a panic"); + ctrl.set_num_queues(1, 16, || {}).unwrap(); + let qp = ctrl.create_io_qpair(1, 32, None, 16, || {}).unwrap(); + let svc = NvmeSwap::bind(NvmeSwapEndowment { + ctrl, + qp, + region: SwapRegion::new(2048, 4096), + capacity_pages: 64, + }); + assert_eq!(svc.stats().capacity_pages, 0); + } +} + +/// GPT discovery driven over the device: the mock disk carries a real +/// table (built with the same crates the parser trusts), and the store +/// reads it through its own bounce page — the chunked entry-array path +/// the QEMU gate exercises, now host-covered. +#[cfg(test)] +mod discovery_tests { + use super::*; + use crate::mock::{MOCK_BLOCK_SIZE, MOCK_DISK_BLOCKS, MockNvmeDevice}; + use gpt_disk_types::{ + Crc32, GptHeader, GptHeaderRevision, GptHeaderSignature, GptPartitionAttributes, + GptPartitionEntry, GptPartitionName, GptPartitionType, LbaLe, U32Le, guid, + }; + use kore_driver::gpt::{GptError, KORE_SWAP_TYPE}; + use kore_svc_swap::{staging_page, staging_page_mut}; + + const ENTRY_COUNT: u32 = 128; + const ENTRY_SIZE: u32 = 128; + + fn crc32(bytes: &[u8]) -> u32 { + const ALGO: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); + ALGO.checksum(bytes) + } + + /// Write a valid one-partition table into the mock disk: header at + /// LBA 1, the 16 KiB entry array at LBA 2 — four of the mock's 4 KiB + /// blocks, which is exactly what makes discovery's chunked + /// entry-array read real (the bounce page holds one block per read). + fn provision_mock_disk(dev: &MockNvmeDevice, start: u64, end: u64) { + let mut entries = [0u8; (ENTRY_COUNT * ENTRY_SIZE) as usize]; + let entry = GptPartitionEntry { + partition_type_guid: GptPartitionType(KORE_SWAP_TYPE), + unique_partition_guid: guid!("11111111-2222-3333-4444-555555555555"), + starting_lba: LbaLe::from_u64(start), + ending_lba: LbaLe::from_u64(end), + attributes: GptPartitionAttributes::default(), + name: GptPartitionName::default(), + }; + entries[..core::mem::size_of::()] + .copy_from_slice(bytemuck::bytes_of(&entry)); + + let mut header = GptHeader { + signature: GptHeaderSignature::EFI_COMPATIBLE_PARTITION_TABLE_HEADER, + revision: GptHeaderRevision::VERSION_1_0, + header_size: U32Le::from_u32(92), + header_crc32: Crc32(U32Le::from_u32(0)), + reserved: U32Le::from_u32(0), + my_lba: LbaLe::from_u64(1), + alternate_lba: LbaLe::from_u64(MOCK_DISK_BLOCKS as u64 - 1), + first_usable_lba: LbaLe::from_u64(6), + last_usable_lba: LbaLe::from_u64(MOCK_DISK_BLOCKS as u64 - 2), + disk_guid: guid!("99999999-8888-7777-6666-555555555555"), + partition_entry_lba: LbaLe::from_u64(2), + number_of_partition_entries: U32Le::from_u32(ENTRY_COUNT), + size_of_partition_entry: U32Le::from_u32(ENTRY_SIZE), + partition_entry_array_crc32: Crc32(U32Le::from_u32(crc32(&entries))), + }; + let crc = crc32(&bytemuck::bytes_of(&header)[..92]); + header.header_crc32 = Crc32(U32Le::from_u32(crc)); + + let mut block = [0u8; MOCK_BLOCK_SIZE as usize]; + block[..core::mem::size_of::()].copy_from_slice(bytemuck::bytes_of(&header)); + dev.disk_write(1, &block); + dev.disk_write(2, &entries); + } + + fn rig() -> ( + NvmeSwap, + MockNvmeDevice, + ) { + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).expect("bring-up"); + ctrl.identify(16, || {}).expect("identify"); + ctrl.set_num_queues(1, 16, || {}).expect("set queues"); + let qp = ctrl.create_io_qpair(1, 32, None, 16, || {}).expect("qpair"); + let svc = NvmeSwap::bind(NvmeSwapEndowment { + ctrl, + qp, + region: None, + capacity_pages: 64, + }); + (svc, dev) + } + + /// **Discovery end to end over the device**: the store provisions + /// itself to exactly the designated extent, and a page round-trips + /// inside it. + #[test] + fn discovery_provisions_the_store_from_the_disk_table() { + let (mut svc, dev) = rig(); + provision_mock_disk(&dev, 8, 55); + assert_eq!(svc.stats().capacity_pages, 0, "binds empty"); + + svc.discover_region(&mut SpinPacer::default()) + .expect("the table designates a region"); + assert_eq!( + svc.stats().capacity_pages, + 48, + "LBAs 8..=55 = 48 blocks = 48 pages at 4 KiB blocks" + ); + + let mut staging = alloc::vec![0u8; PAGE_BYTES as usize]; + staging_page_mut(&mut staging, 0).unwrap().fill(0x77); + let SwapResponse::Stored { extent } = svc.handle( + SwapRequest::Out { page: 0 }, + Some(&mut staging), + &mut SpinPacer::default(), + ) else { + panic!("a page stores inside the discovered region"); + }; + assert!( + dev.disk_snapshot(8, 1).iter().all(|&b| b == 0x77), + "slot 0 is the REGION's first block" + ); + staging_page_mut(&mut staging, 0).unwrap().fill(0); + assert_eq!( + svc.handle( + SwapRequest::In { extent, page: 0 }, + Some(&mut staging), + &mut SpinPacer::default(), + ), + SwapResponse::Restored + ); + assert!( + staging_page(&staging, 0) + .unwrap() + .iter() + .all(|&b| b == 0x77) + ); + } + + /// A second discovery is a no-op once provisioned: the extent books and + /// counters survive it untouched. + #[test] + fn rediscovery_never_resets_a_provisioned_store() { + let (mut svc, dev) = rig(); + provision_mock_disk(&dev, 8, 55); + svc.discover_region(&mut SpinPacer::default()).unwrap(); + + let mut staging = alloc::vec![0u8; PAGE_BYTES as usize]; + staging_page_mut(&mut staging, 0).unwrap().fill(0x3C); + let SwapResponse::Stored { extent } = svc.handle( + SwapRequest::Out { page: 0 }, + Some(&mut staging), + &mut SpinPacer::default(), + ) else { + panic!("stores"); + }; + assert_eq!(svc.stats().stored_pages, 1); + + svc.discover_region(&mut SpinPacer::default()) + .expect("rediscovery is an idempotent Ok"); + assert_eq!(svc.stats().stored_pages, 1, "the books survived"); + staging_page_mut(&mut staging, 0).unwrap().fill(0); + assert_eq!( + svc.handle( + SwapRequest::In { extent, page: 0 }, + Some(&mut staging), + &mut SpinPacer::default(), + ), + SwapResponse::Restored, + "the stored page is still restorable" + ); + } + + /// The mock's seeded (garbage) disk carries no signature: discovery + /// says `NotGpt` and the store stays empty — the QEMU gate's rung 9. + #[test] + fn an_unpartitioned_disk_designates_nothing() { + let (mut svc, _dev) = rig(); + assert_eq!( + svc.discover_region(&mut SpinPacer::default()), + Err(GptError::NotGpt) + ); + assert_eq!(svc.stats().capacity_pages, 0); + } +} + +/// The readiness probe: write/read/verify inside a designated region, +/// read-only everywhere else. +#[cfg(test)] +mod probe_tests { + use super::*; + use crate::mock::MockNvmeDevice; + use kore_svc_swap::{staging_page, staging_page_mut}; + + const REGION_LBA: u64 = 8; + + fn rig() -> ( + NvmeSwap, + MockNvmeDevice, + ) { + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).expect("bring-up"); + ctrl.identify(16, || {}).expect("identify"); + ctrl.set_num_queues(1, 16, || {}).expect("set queues"); + let qp = ctrl.create_io_qpair(1, 32, None, 16, || {}).expect("qpair"); + let svc = NvmeSwap::bind(NvmeSwapEndowment { + ctrl, + qp, + region: SwapRegion::new(REGION_LBA, 48), + capacity_pages: 64, + }); + (svc, dev) + } + + /// With a region the probe WRITES — and only inside the region: the + /// pattern lands at the region's first block, round-trips, and every + /// block outside stays byte-identical. + #[test] + fn the_probe_writes_only_inside_the_region_and_round_trips() { + let (mut svc, dev) = rig(); + let outside_before = dev.disk_snapshot(0, REGION_LBA as u32); + let inside_before = dev.disk_snapshot(REGION_LBA, 1); + + svc.self_test(&mut SpinPacer::default(), 0xC0FF_EE00) + .expect("the write/read/verify probe passes"); + + assert_eq!( + dev.disk_snapshot(0, REGION_LBA as u32), + outside_before, + "nothing outside the region changed — LBA 0 included" + ); + assert_ne!( + dev.disk_snapshot(REGION_LBA, 1), + inside_before, + "the pattern reached the region's first block" + ); + } + + /// The pattern is a function of the seed: two probes leave different + /// bytes at the same LBA, which is what keeps a boot from passing on + /// the PREVIOUS boot's pattern when the device silently drops writes. + #[test] + fn the_probe_pattern_is_seed_unique() { + let (mut svc, dev) = rig(); + svc.self_test(&mut SpinPacer::default(), 1).unwrap(); + let first_boot = dev.disk_snapshot(REGION_LBA, 1); + svc.self_test(&mut SpinPacer::default(), 2).unwrap(); + assert_ne!( + dev.disk_snapshot(REGION_LBA, 1), + first_boot, + "a different seed writes different bytes" + ); + } + + /// A stored page in slot 0 downgrades the probe to the read leg — a + /// probe must never clobber a live page, whenever it runs. + #[test] + fn the_probe_never_overwrites_a_stored_page() { + let (mut svc, dev) = rig(); + let mut staging = alloc::vec![0u8; PAGE_BYTES as usize]; + staging_page_mut(&mut staging, 0).unwrap().fill(0x99); + let SwapResponse::Stored { extent } = svc.handle( + SwapRequest::Out { page: 0 }, + Some(&mut staging), + &mut SpinPacer::default(), + ) else { + panic!("stores into slot 0"); + }; + + let before = dev.disk_snapshot(REGION_LBA, 1); + svc.self_test(&mut SpinPacer::default(), 0xABCD) + .expect("the read-leg probe still passes"); + assert_eq!( + dev.disk_snapshot(REGION_LBA, 1), + before, + "the stored page's blocks are untouched" + ); + + staging_page_mut(&mut staging, 0).unwrap().fill(0); + assert_eq!( + svc.handle( + SwapRequest::In { extent, page: 0 }, + Some(&mut staging), + &mut SpinPacer::default(), + ), + SwapResponse::Restored + ); + assert!( + staging_page(&staging, 0) + .unwrap() + .iter() + .all(|&b| b == 0x99) + ); + } +} + +#[cfg(test)] +mod region_tests { + use super::*; + use crate::mock::MockNvmeDevice; + + /// **The regression that cost a real disk its partition table.** + /// + /// A store must never address LBA 0. With a region designated, slot 0 + /// is the region's first block; with none, the store has no capacity + /// and no slot exists to address anything at all. + #[test] + fn no_slot_ever_addresses_lba_zero() { + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + ctrl.identify(16, || {}).unwrap(); + ctrl.set_num_queues(1, 16, || {}).unwrap(); + let qp = ctrl.create_io_qpair(1, 32, None, 16, || {}).unwrap(); + let svc = NvmeSwap::bind(NvmeSwapEndowment { + ctrl, + qp, + region: SwapRegion::new(2048, 4096), + capacity_pages: 64, + }); + assert_eq!(svc.lba(0), 2048, "slot 0 is the REGION's first block"); + assert!(svc.stats().capacity_pages > 0); + for slot in 0..svc.stats().capacity_pages { + assert!(svc.lba(slot) >= 2048, "every slot sits inside the region"); + } + } + + /// With no region designated the store binds EMPTY: zero capacity, so + /// no write is reachable and the self-test stays a **read** — the + /// posture of every machine whose disk designates nothing for kore. + #[test] + fn an_undesignated_store_has_no_capacity_and_writes_nothing() { + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + ctrl.identify(16, || {}).unwrap(); + ctrl.set_num_queues(1, 16, || {}).unwrap(); + let qp = ctrl.create_io_qpair(1, 32, None, 16, || {}).unwrap(); + let mut svc = NvmeSwap::bind(NvmeSwapEndowment { + ctrl, + qp, + region: None, + capacity_pages: 64, + }); + assert_eq!(svc.stats().capacity_pages, 0, "no region ⇒ no store"); + + // The WHOLE disk, not just LBA 0: an undesignated store owns no + // block anywhere, so the self-test may change none of them. + let before = dev.disk_snapshot(0, crate::mock::MOCK_DISK_BLOCKS); + let mut staging = alloc::vec![0u8; PAGE_BYTES as usize]; + assert_eq!( + svc.handle( + SwapRequest::Out { page: 0 }, + Some(&mut staging), + &mut SpinPacer::default() + ), + SwapResponse::Refused(SwapRefusal::StoreFull), + "an undesignated store refuses every write" + ); + svc.self_test(&mut SpinPacer::default(), 0xB00B_5EED) + .expect("the read-only self-test still proves the queue"); + assert_eq!( + dev.disk_snapshot(0, crate::mock::MOCK_DISK_BLOCKS), + before, + "no block changed — the whole point" + ); + } + + /// A region may not start at LBA 0, be empty, or wrap — the shape is + /// refused at construction, not guarded against at every use. + #[test] + fn a_region_at_lba_zero_is_unrepresentable() { + assert!(SwapRegion::new(0, 4096).is_none(), "LBA 0 is never swap"); + assert!(SwapRegion::new(2048, 0).is_none(), "an empty region"); + assert!(SwapRegion::new(u64::MAX, 2).is_none(), "a wrapping region"); + assert!(SwapRegion::new(2048, 4096).is_some()); + } + + /// A region declared past the end of the namespace binds empty rather + /// than silently truncating into blocks it was never given. + #[test] + fn a_region_past_the_device_end_binds_empty() { + let dev = MockNvmeDevice::new(); + let mut ctrl = Controller::bring_up(dev.clone(), dev.pool(), 16, || {}).unwrap(); + ctrl.identify(16, || {}).unwrap(); + ctrl.set_num_queues(1, 16, || {}).unwrap(); + let qp = ctrl.create_io_qpair(1, 32, None, 16, || {}).unwrap(); + let svc = NvmeSwap::bind(NvmeSwapEndowment { + ctrl, + qp, + region: SwapRegion::new(crate::mock::MOCK_BLOCK_COUNT, 4096), + capacity_pages: 64, + }); + assert_eq!(svc.stats().capacity_pages, 0); + } +} diff --git a/native/provenance.json b/native/provenance.json index c3a8223..6f475c1 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,6 +1,6 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "d929d9a3990a9e3f05d38d37b15904ab9551b0a0", + "revision": "7074bda28c416ffe34562857ee43cbc883482782", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", @@ -17,18 +17,26 @@ "crates/start": "kore-start", "crates/uart": "kore-uart", "crates/window": "kore-window", + "drivers/nvme": "kore-nvme", + "services/block": "kore-svc-block", + "services/dagfs": "kore-svc-dagfs", + "services/device": "kore-svc-device", "services/extract": "kore-svc-extract", + "services/extract-worker-c": "kore-svc-extract-worker-c", "services/extract-worker-rust": "kore-svc-extract-worker-rust", - "services/output": "kore-svc-output" + "services/mm": "kore-svc-mm", + "services/output": "kore-svc-output", + "services/swap": "kore-svc-swap", + "tools/kore-vol": "kore-vol" }, - "workspace_sha256": "f66a2093652e6dfdcd4d04c3558d8447385d3608730af28fb642931a4ff4a395", + "workspace_sha256": "17edafdee9c32e592cda34f7db82a7075ab803793d04119a25b42b11a89784e3", "files": { "LICENSE": "c9b2b12d3bad9080090798a442b4b2a77c48312ae75089f041281b7c8ad21738", "crates/abi/Cargo.toml": "f9a4811b4b6d06947a47a763354cad3bc739a5ea387194a4dcf22409351f3c7b", - "crates/abi/src/cap.rs": "72c350a93a23e6c985a806696f27630f0dbbf0132553161ebeb62c55f6ae122d", + "crates/abi/src/cap.rs": "41ba3318d372ae447a47f1ef05932204057de17c7df3dd6b9d9a3dff4a935b3e", "crates/abi/src/clock.rs": "cce1c2839cba2377d3cc8696d60812f9195f071f422bd97bb4530ea0368972fc", - "crates/abi/src/command.rs": "0670b8b281ac04192cb3fd650013243f2ae038f7c8efa4983302e1472221fed7", - "crates/abi/src/crash.rs": "9d2e756b06195d0597250cbc915156c9b97aea3ea722b935db2ebeccc62a4d9c", + "crates/abi/src/command.rs": "206b8b150688cff68345c9237b78d1170e92f01c8323951e2a3e54b5fb808b97", + "crates/abi/src/crash.rs": "f3c9e5a954e06743f6993b0a5eeb6937d2199c50af5fe95b6be446393ccf999b", "crates/abi/src/device.rs": "ad4578975e8056f375b0338124f497c9b4b5caa6272ea8a9930e2877d44efac6", "crates/abi/src/device/census.rs": "6fb54ecaade94b116f41e045b098dc5926e8e1c75eeea33870dd4be01bd1ce23", "crates/abi/src/device/i2c_admission.rs": "1ea2fc04cdb0b15186d2f1d80bde53187860e2197263e225f330965d96e40876", @@ -39,24 +47,27 @@ "crates/abi/src/guest/riscv64/mod.rs": "c3ea09ccbc04959a527e1c75e4122dbf3bb7f30e2ae4f495fd59b6d980aca1a3", "crates/abi/src/guest/wire.rs": "bec45a73cee701b07d40b37f7c296cb5e96f96d2994cd512fd1d9aad7bc7432f", "crates/abi/src/guest/x86_64/mod.rs": "58fe4ec322a03c83e30faf3fef4c7ed6ec06b64d42a33a921e977306148be342", - "crates/abi/src/image.rs": "5014cf054a1f602af9876e5efd34708625abaf45c38e78d02e1192e31e4a6010", + "crates/abi/src/identity.rs": "cf69309ca5e0d53b752f427b8d82eddab874973cfa0a8e120633c326f723f264", + "crates/abi/src/image.rs": "9cc689a9f73015f8e2d3429e57fcc3243dc94a6c929ede623c50515f060065ed", "crates/abi/src/input_status.rs": "5e48ead8883878c52b7ccdb22ff56d7bf04e4d74d7c689573278e7acc5e5b79c", "crates/abi/src/iommu_status.rs": "2a2ef44ec698136ebe213db38b66b08b090b8d2c74a91346a38368ba43c20b02", - "crates/abi/src/lib.rs": "2d0dc5ca2e5ecd87b2b596ceb1c9bf6c6ddd180fd620232dd58918367b8c4940", - "crates/abi/src/machine.rs": "4c9f97ed3e002063430f9bee894676d7d085505984fcf3476a4ebf73498d8f07", + "crates/abi/src/keyboard.rs": "2a4e763ed9bd76151b1f7450394f3a9a4986a4b4e1a1326124db1e38eb11b8a7", + "crates/abi/src/keyboard/usage.rs": "67dfa4ce0f5c4d7062d47e8e3c84846399a16bc1280fe79e82ee4db56bdc6d52", + "crates/abi/src/lib.rs": "845de17bf06d1bcbd3df69da7b8ef251290459418b1798f65d9a30768d006b0b", + "crates/abi/src/machine.rs": "7427c50ec38f71dbc83818a5075d03096cd34f5509e22646b04839eeb93cc783", "crates/abi/src/mem.rs": "ae502311778be8ef73f7b9edb1c47f157564f96a58db9add4ad74a313eb3a147", - "crates/abi/src/monitor.rs": "a875eae2da322aa14c52e00d20f5b0aee5748342f5bd28db16cd2cb12608c045", + "crates/abi/src/monitor.rs": "8f01b9165d493f78127c3e2d37da401e3c969777b69535612ab2bcbd27eb7871", "crates/abi/src/pkg.rs": "2893ad39761611ddee84d314bebbed45a9c9823ff145c6a8f28d3844cb40155d", "crates/abi/src/pointer.rs": "9971a292c1d00b275d8030e4f11fdd71206eaf12671cda8775d220b106ebcbf8", - "crates/abi/src/process.rs": "9f08d5cf6b9a213c63580798bd2e2c307070754724577d1c04da9502323c7f66", + "crates/abi/src/process.rs": "1dc29781e8d5f500e1e44e8ac36ca466aacf7f14a74b0f730af84336a4319cce", "crates/abi/src/processor_status.rs": "c963e48f9b07046ab6c82d4cc2735ee2cc6c94561509a4e6e15927c837c0b0c6", - "crates/abi/src/session.rs": "1bc516ad898112465b68cf171281b4cf1cd6abcf8be9474c4baee89cfa075c99", + "crates/abi/src/session.rs": "030b2bee9b46426656db7ebde001738a2cfbdbe775a3be1ae892a14330f2d66f", "crates/abi/src/stor.rs": "eabad4e5588fd343472660e15cccf32cfcc8d8f4c25eb18dffbd1598b6007db2", - "crates/abi/src/svc.rs": "0706435929cc42e337de6d0aa93fd4ee289f4844da041cc2f2e0984c7c3c088b", - "crates/abi/src/task.rs": "ac51cd35ec0c767b525c47306735811f4550427bd1719b9c1846fb79f905f2c1", + "crates/abi/src/svc.rs": "d857cc812bdbc228c5f852a02ed02a349951cdc268291e51abe4628b894610da", + "crates/abi/src/task.rs": "57cb01a29bd2dbf870a1c49e35424f091fad597396ed5d080fc4f4cdd79c5f2a", "crates/abi/src/uring.rs": "e98d9db77fbe00d5547b582700a1104d2c5b2546bbe57742dfff8362c32dcc2e", "crates/app/Cargo.toml": "fb793b25eec583da7b5e404b19bd656a6b8bcee2a86a20b0da399be2210a9398", - "crates/app/src/lib.rs": "330cea793beb81af147e4f50b1e6fe4ad13a88842937351d493bbc0b5c44e997", + "crates/app/src/lib.rs": "5e7e8d791236f278ee85a02a3c37f41412d879cc41d67f6f6d3d701811973d7d", "crates/collections/Cargo.toml": "0b570a215ecd3d18b86b0f1b554e95de6636520c513eefd6f02d16cc2f37ebd6", "crates/collections/src/bitmat.rs": "35c1b7e572de0fa4e3dfc8c1c2cb8b7c5a031a812fb9ed7d4c86466b2a8399ec", "crates/collections/src/bitset.rs": "941e147ef204a6f3100c7d06652c6b9c3efb796ca7bdb738660bf6975a7ed815", @@ -68,36 +79,38 @@ "crates/collections/src/priority.rs": "c6cf3ff16f53688f97671166428ecfa50270f0b858d5725febb9d701810ff089", "crates/collections/src/slotarena.rs": "846ff776196e68552d9b32e5ab55b5874071ff222f1a07397154c34d1920759a", "crates/collections/src/slotmap.rs": "891ed1b871293800f377ac7ed81919f564578563390e91f070f8316b829596c5", - "crates/collections/src/slotvec.rs": "3b08b9fb5778cd6d8a4b4fa388f008a53f2916244da31eb4ebec2f2462c03188", + "crates/collections/src/slotvec.rs": "353534ef72311a1ff5c9723876c26bceb4e3d280494f7740502c9120b0caf2b0", "crates/collections/src/testing.rs": "ca9f73b2954ee46e002542b464640cb67d8b0e97309b397185dc8001c5e05be3", "crates/dagfs/Cargo.toml": "434ebe9eb980b8b9aa674b3920c8d034606413143a2f2c6f7f0b8db193e72b35", - "crates/dagfs/src/bounds.rs": "6a9691cdba604ab12f184379513a4d081117e3e1be36188c40c50c51ca169bd7", - "crates/dagfs/src/capability.rs": "820b60b9794947aa8a4afdd5d42cc805e78ce56de7104da333e9a95dbcd32c96", - "crates/dagfs/src/client.rs": "745987f54b5c7296d0b09a3d72a8e95f253777b50575dd32f0985c0c2f8fd0c9", + "crates/dagfs/src/bounds.rs": "be6512cfdec3fbf303943c07ecaa5bc3636830b065ff6c2c0e75adb27d38b991", + "crates/dagfs/src/capability.rs": "bbeb3d203da762b1a2fde7cb236dc1215d800a6f90f0a50fa7d1740281ede75c", + "crates/dagfs/src/client.rs": "06f56497f1c033d0d16a3ea1253b70d778bc1df788f9fd3457274c3600f33959", "crates/dagfs/src/entry.rs": "ae077f13df5f0949fbc7c867b5f026aa494843ae605dde4fda845c2a531c33a3", - "crates/dagfs/src/error.rs": "bda2cab29a5b65519888c8be8e571b96b6c291b02b6aa45716e979af78d21c76", + "crates/dagfs/src/error.rs": "f8b93e779e3cc7966db4eaee3098fac4d3615d89441a6fc7ed34061eb6cbaf88", "crates/dagfs/src/face.rs": "b22d6f071397395a8ffda7487f5b876a03fc39590e4a1adefea622d5ae737bc9", - "crates/dagfs/src/fs.rs": "5814ced189369baa1eafdef83f954e7a45616a02bc800708c3dae89e1dc1fdb3", - "crates/dagfs/src/graph.rs": "ee43a3f6cfda159311b4594ecb0988698388ac3c531316f2e07f690f254d500d", + "crates/dagfs/src/fs.rs": "0e0f98c347c6e71363e7f022445e1dc3ccfe6921d2e9ad94f092fcc0296ce1b3", + "crates/dagfs/src/graph.rs": "833a50a1e6a722ac0d5cefb34b101b9553a9ddd59a5c3d7af0fb723141a063c3", "crates/dagfs/src/ids.rs": "a6cd3127adbbfb3e02c7467f5736e2cabe6b53b9047d98f1cee2ba5de2c78227", - "crates/dagfs/src/index.rs": "6c64df4092fd97974db52880460ada1e68390aa5ab993bff1e7355748e0c482e", - "crates/dagfs/src/io.rs": "e03bdd30b4ce3fedb8f5a0486d8fefede714f1e7dcdbb92d2231e8f179712bce", - "crates/dagfs/src/lib.rs": "1e86372ce6c6e89fddb3404ef57bf19a32d1b3272ae5c8d2a6e50c6666d5ef29", + "crates/dagfs/src/index.rs": "cd20892b18859bcdf19f63152f994003556bc048b0b0de3816bce250e99d9b81", + "crates/dagfs/src/io.rs": "a91034153e6f82d32c9e5b5c8ce21f8b53f35c96adaeb2d507b56b07c75ea577", + "crates/dagfs/src/lib.rs": "aee6c0dd3bec2d44667691a110623a4526a50ff1af3133869b2c7b110e578132", "crates/dagfs/src/name.rs": "a59b8464b96e3a4bb65bd1056e3db5ea80bc3a99da528624ee77757cff4efaf9", "crates/dagfs/src/object.rs": "7c6821771008a5f83b45d8fa53fd9aa00bba405f7a26abec2274962adc1cc00c", - "crates/dagfs/src/path.rs": "e462a2b3a25c8414f6d42315214a89a8435eb53ff7e4bcb0d1ee7d1d25d9cd61", - "crates/dagfs/src/protocol.rs": "5c3216ce0c3bb6b4319a27921ba5561fe2b0dc112de937c165f10871571edc2e", - "crates/dagfs/src/refusal.rs": "b50bf0b5e52a76be8e3af25261fe459f2d06f7aecea2ea5cdf756452621ad6c0", + "crates/dagfs/src/path.rs": "db0bef7330be961a483e7582df14647d043f2a8fe722eb6320ece488d2fc243d", + "crates/dagfs/src/protocol.rs": "e9a35a3a2d28b1b09e44dc45200323fa258c3ef240d5494afd0b07ea19e17ece", + "crates/dagfs/src/refusal.rs": "71d612406e9a86509056e6a755e3b3530396232007790588868e8756a5797fe8", "crates/dagfs/src/search.rs": "cfc79463a6367d2fc62a9598ca3fe20d6771c3d9aad07709192da93338a9e4ab", - "crates/dagfs/src/stor.rs": "04f8f1832df2282016a52ee839f5ab83e60d58916b95fc8b76c87c37d0964483", - "crates/dagfs/src/text.rs": "2fa8ebcf08781c2cce07a5e3af0d27106522f53d249ed4f3f573154f23921512", - "crates/dagfs/src/view.rs": "75bb79d8c6258eb28cd67550494cea0334e6738213649bd631e3f5b310cdb106", - "crates/dagfs/src/walk.rs": "81e165d867a901ea50c13abd94816719d56efdc217d79943545f3a83c35dcaa0", - "crates/dagfs/src/wire.rs": "b4523d118cdeef548ff839a4329ee1ac7f2c66dc0d3e5b3dab8ff985d67471eb", - "crates/driver/Cargo.toml": "bb386b97586426ae14b1c9abf6290cab7187ab9997ca67b9c8e6808dc09133f8", + "crates/dagfs/src/stor.rs": "06a270d01e2f3a2dcdb41f803926d863d5fca6229aa14e0cc84337c651acdddb", + "crates/dagfs/src/text.rs": "878ee0abc41bfd30ece0f0a7c6be5935487fb945248773d3eebda8066f8f6032", + "crates/dagfs/src/view.rs": "24d661589ce1bb05e79112cda5ce1027166c7794adc3bcbcfdb3d63e663dfd69", + "crates/dagfs/src/walk.rs": "3903476532c8f0933a0db3bf7cae9075c6db8dc06953b479709497329ed7e6a4", + "crates/dagfs/src/wire.rs": "3b5da5293e34e8a6c8b6133228e32b5477c6fef8550c58755ec174b7734feaee", + "crates/driver/Cargo.toml": "15154ecb6517360f34580dba34799de5a38d546d8be8a64a626562c59e25e158", "crates/driver/src/accel.rs": "c2355916f1f6647d4bcf8414806c3bb9da45799e458133da70e31190316febfb", + "crates/driver/src/block/image.rs": "5bb433b99cafc1a51812469ef8ea637dcc84ad2d2b8a79a3385fb8972884ffb8", "crates/driver/src/block/memory.rs": "634c09591f0ffbb091fd45fd791cbd306c16755df0f4c92e34868e362a758478", - "crates/driver/src/block/mod.rs": "292dd64d5220adb1bc3161ac405c62ff392cf64e5f7276863a59a7a1fa05eaff", + "crates/driver/src/block/mod.rs": "0028d0c8d9c4e477e6d32fbd1954fda114095f58e396cabc3277a209538792a4", + "crates/driver/src/block/refresh.rs": "3553e395b7c2685d75b494fb11f169959fecdc98c63be677195d3de7fd616e73", "crates/driver/src/block/region.rs": "d6099887fc1777f4bdb33c0b44427849a99aff7a18013bae32f9a29c2c0e59c5", "crates/driver/src/common/aperture.rs": "ef7a2ba28666c2381aa6df625c86384cf0769e1afc695ed4ee92c47479df24e7", "crates/driver/src/common/flags.rs": "f7d1c95eba00cdf3aac352b2950485d2f3b51ef6044a258d008f79719af1e5cd", @@ -113,7 +126,7 @@ "crates/driver/src/common/smmuv3/stage1.rs": "95b69d9e17c001087e70c8003b1c36bfd702853098247c3ce0a8eb906888e161", "crates/driver/src/cookie.rs": "9c60639fd1646b293eb6c7846c5ce67205e9d2e35e55cc0ad683cfcb329467f8", "crates/driver/src/gpt.rs": "71f6b9fd9b4f4d78622a48b5a706276fbc1f6aea1ec1005aed6fc8396635f59e", - "crates/driver/src/host.rs": "97cd5a7c4694d36564ec0a59b3c9734de177b1c0853dc47560f61b27c95e69c9", + "crates/driver/src/host.rs": "ef27b227b4b605497eae9ebee7bc7c47755d218874f5c3d4b9ebdd97ec83e834", "crates/driver/src/host_double.rs": "e5850fc7d7d39a090a8d30ad5d589e68294669b14edb96d15ff4794a0282a6a4", "crates/driver/src/lib.rs": "8808eb41d003e94815dbfa56b7875d3866bdbada0b968056ccd06e3936318867", "crates/driver/src/net/addr.rs": "c14556ffc4ea3885e2cb94c27994d3cb7deb91991e01a9f3d704243bceebb080", @@ -124,12 +137,12 @@ "crates/driver/src/net/policy.rs": "87a287ff727c9acd87e9ff6f726b5ec0c5228b1013e3f92fd5c7c158730f1647", "crates/driver/src/net/vif.rs": "3390ef56c41062e24946a8a35c4584b910fce0543a17e0a5c478ed4f7d6efe5f", "crates/driver/src/net/window.rs": "d2f0e2e774ce28a65be10afad5883f1298b292d2a9fe7a0314ae427ec0de6d1d", - "crates/driver/src/ring.rs": "fe02aeb350c852ab28095516e0f5efc31bd787377cb91c95b24523b3bf999817", + "crates/driver/src/ring.rs": "7821bc27ad0baaf893ebc421c35431944ed6c98d0c114bcc10b2a2e562739a8b", "crates/driver/src/seam.rs": "391db6184832923c085395d35a7001d759082b9d557aec2a9837f29ad6bfc5e2", "crates/driver/src/stats.rs": "ae7a7785429481a709e11407738d92dd8db0e18d423025afbc3e153ee454c07d", "crates/driver/src/stor/engine.rs": "d4c661785bebd7708a56c46a4225c64342544fe9cfb2bafd17f65a9e0410d950", "crates/driver/src/stor/grant.rs": "6cb866bd924914f3169c212ef950b4698ec9ae6b174ab4a51c101fd7957babf1", - "crates/driver/src/stor/lanes.rs": "b1a4c5da0cce3d4210c3b161c12220503f50f63d191b71eaf843f69cea116a38", + "crates/driver/src/stor/lanes.rs": "9c962a9d6cf06d2049248a7b44497fcf742bcdbfab173c66ff5f89cdc3b23f27", "crates/driver/src/stor/mod.rs": "faa061c11e3001f83f35c3bb8aaa9ce2c9378854184b862b8a7fd39afff55cbe", "crates/driver/src/stor/timeline.rs": "5adb419bfc501ca4b679c1c02c705eb28b848eb3ec8055870a79bc39130d1aa9", "crates/driver/src/wait.rs": "f73935286f7a5e6644fbe528f36cb6058529499880b6cd52e996691bfdda4483", @@ -155,15 +168,15 @@ "crates/ld/src/store.rs": "ba5a432a2a3e34e64cdd8f02dccc2c69bab5719b6a4223393c72beca0c7a2b18", "crates/ld/src/stub.rs": "b670715026a889f5baaf233a35d3d24c1891223a0e57665782ee326d90e49f5e", "crates/ld/tests/apply_engine.rs": "e01f5f805f34d614717ceb297275d65936c1b48989df90079fcd2f8ec83b6b1c", - "crates/ld/tests/built_images.rs": "cb635657e4ae99b00b1b504b7949c67dd16c86be81a843d50f098338f1482300", + "crates/ld/tests/built_images.rs": "7e6716758f34fab028ac67ce2f016b7e8d7ea93966dd2d300ed4ca254872c055", "crates/ld/tests/check_gate.rs": "0ead133777f18f929d5e9b0353a289a5b4fa9e2af929155f525524096ee5578c", "crates/ld/tests/common/mod.rs": "1e4bdfa37511848d9e9458d9a550a1ff2410154411c6d026e9106035044ea438", "crates/ld/tests/late_layout.rs": "9bfc460d8036d69c1acdc52c430ee475e25062519cdc7dd30f235b882414a339", - "crates/ld/tests/link_engine.rs": "a6959aa6f07b4776dadb80d192fa0064fee2823017c9555108fcbfa81a922e6d", + "crates/ld/tests/link_engine.rs": "a9daca8bbb29b510da1acaccc9bbb24466f9c693bd114b7a9ccb8188f31f2a59", "crates/ld/tests/shipping_list.rs": "113bf634f5a8f83103e266149c2c8daf834f3c99b86d7e3e0693f520283a1469", "crates/ld/tests/static_boot_images.rs": "25fb47a0a8e5b11dca5b8c31f64b90e8d6a6edc3c4d81b0fb5e1733afea3ebe3", "crates/ring/Cargo.toml": "b0a6e8d0f919494ef9e2f8c229ff64e24709347ca76c3d7f5a6e413387c20bcb", - "crates/ring/src/event.rs": "72dd838dfe2f163fc240c46ec8160051ba57f91a98dcdbaf35624b7227d2033b", + "crates/ring/src/event.rs": "a6db5f78e8472f09da1312456f488cf7e6f9a244b1a2cc63b5ea107eb7cd9020", "crates/ring/src/lib.rs": "847200aad2d16c084b1430a847fc6768e6d9765d0e22af79537c9a81896aa9bb", "crates/ring/src/spsc.rs": "136730d1c645044e0ddce3fc67e9b32410935a522249af5d69039f91e9b5a636", "crates/ring/src/stor.rs": "e8b08ac46a9e72550d052e88ed82a332c220b6f3a2d5d00aa5a88ac241c4a65e", @@ -174,65 +187,69 @@ "crates/rt-macros/Cargo.toml": "c84f262e96a3515dcabb61c96c34d9aec4b5f5a377af0f6da54e2e71b2c7807f", "crates/rt-macros/src/lib.rs": "ae051ab730bd408b6e0449a8871910257b6ccdac6f3324372dff8cec62ede24a", "crates/rt/Cargo.toml": "0b472a92c92d74738202204ccafe53088ba9d303c3c8d4ac09a9dc29204a4d23", - "crates/rt/src/barrier.rs": "5397d4de097e454bd67f60bdaac0a94198c51e42ca1526eafd86269b85370027", + "crates/rt/src/barrier.rs": "c5bc3bef52f2bcc24edb1016fde955bd01e73074d4ce357d79cae9d3b63e6f3b", "crates/rt/src/block.rs": "70c2aef8bf149da2f32c7c995cd0f4c63cb492647896b107fdd5dbecd3d3042a", - "crates/rt/src/cap/mod.rs": "9e1d428a1f020f8abc2a896ec94e0aa95dc84033852f3b359aaf2f1a8a22b91c", + "crates/rt/src/cap/mapped.rs": "b29ce431638a29b01edfd72500348f5ae201afe97ec4684ea4449d01e75ec1e7", + "crates/rt/src/cap/mod.rs": "959a357dedc329cb84563f2a8677b462e89da379373022d644ca08ccdb157bc5", "crates/rt/src/cap/record.rs": "640b0ff60052d4775c259489af5fbff5cd03164370aceb4775186be956817bcd", - "crates/rt/src/cap/verbs.rs": "6c14391e990375c51dbf7c706d4f8df8e2536671a6bd8bb647c739da47b9bb3c", + "crates/rt/src/cap/verbs.rs": "0de798ceebe67399a5994980cce2a5a49b8152639af2e3758cd84673701d7cee", "crates/rt/src/chan.rs": "fd50f962b23c2bfa15e72a1800790475d5cf6fcc55503ed39269852f025397e5", "crates/rt/src/clock/aarch64.rs": "47828a573f5fc30a988f012a648a8cb55596b45331cdcdfe1b390cd1d8543339", "crates/rt/src/clock/mod.rs": "d402d793448359338a361563a0fea91ea43d0264a5eb38c75b9abbb87df89af9", "crates/rt/src/clock/riscv64.rs": "17d4ecdb2b4f8152c0cf51f71ce28fe331d929a62405810a1b7eb332bb275c3d", "crates/rt/src/clock/x86_64.rs": "ca499d4dc51b7ed0eb196b831295b5fdecb09a1bd2df628a82e749e27011994a", "crates/rt/src/core_id.rs": "cadde39801a1d0ed05d70d0634a9e52d58043878737d790c4130183751b622cf", - "crates/rt/src/crash.rs": "c0c59f8c96c91bdadb4a469f90a4874323f6fd4da72661ad03e1da8d19072531", - "crates/rt/src/entry.rs": "cc534f3474ea2547ada3c7d3008aded316a6b17749bd8a61d621cf579935631f", - "crates/rt/src/event.rs": "6d523a6bd1c8a7f956359dab99980731196cb9bc9de7ce076a47a986e758e0af", - "crates/rt/src/executor/mod.rs": "4f5ba3aa87dd248aa44f21573b45f05f6226135ea05c271a30331d49fa63cf32", + "crates/rt/src/crash.rs": "c750a44492cd981bef6a55c6e0891ef1039e1663f3959e2a5f5289fb354ffe86", + "crates/rt/src/entry.rs": "c08657dd5fa6d88774173730db2501c73a4186aff2689ff866125c2e2189260f", + "crates/rt/src/event.rs": "709a5500231735ad5bdb9ab9ab2fcee7b8f72f93e1b4741afba72fdc8b7c54f3", + "crates/rt/src/executor/mod.rs": "071e3dc66bca448e6dabf96976702600bc77fdc98933d58847662fb2de073cb8", "crates/rt/src/executor/task.rs": "b3017cb97f287b8796c61d245c49e5c1b0ebfabe8c535cc2b3842cdbc854dc29", - "crates/rt/src/executor/time.rs": "0ba0ca633be749f7e51426de9e1f4478c864a5575b68c34b24e732dfabc1ca12", + "crates/rt/src/executor/time.rs": "62f9ca9e5e17b395ec2b19a52fc6ed1bb53fd4acc3554cfca81eaf20e48fc272", "crates/rt/src/exit.rs": "3bf7e8ce6dac63b4ce365bdf2e743a9fd171851f1c5e40f8afbb4548877cef1b", - "crates/rt/src/fiber_stack.rs": "0d7f634321d3b5d04cf9ce1b0a7dda03e22743083c021ffe81363ffda6754cce", - "crates/rt/src/futex.rs": "5161418bf642366b701719dd70838edc00ac1e046dec5a7ef257faeee60ba8b6", - "crates/rt/src/heap/mod.rs": "d53eb6ba61d18369a537a4cc148800a03ba7ad04c6fba1728c8c7228f5018b34", - "crates/rt/src/heap/pool.rs": "789b0e08f01b9d720fa2f7ef933f91aa5843d9f8571b2e12f781ff1715008c3e", - "crates/rt/src/heap/shard.rs": "e23e8583fa768c269dc614579956bb95f026e7504ac163988fb0a3a81ae87de8", - "crates/rt/src/heap/source.rs": "40f1e7aea42c0174a64f6a21790ca4a3883092b3a2791c7904a9c078cf74a7a1", + "crates/rt/src/fiber_stack.rs": "48bf4fedba07ea2826615368e22882301ae4e0bbce0d893d0d7754d86bd02ce6", + "crates/rt/src/futex.rs": "9ec11227ff7260d141589fa63fdbb238eed3f52e9731ed4d9c7a14c9523a6709", + "crates/rt/src/heap/mod.rs": "a9c39d3de56187a14d4633a229cbbb6e7ddb2c438d6f9f154b3b2551031b9f81", + "crates/rt/src/heap/pool.rs": "67d42c6ffc7c7e176d21e97dd3179a3e4ff331f46dd457ed4d29914b1a404de8", + "crates/rt/src/heap/shard.rs": "d97f91b3ecc71e3ec56fde0017f9d98cb99c5fbeeffe6ff79ddd153f5c6690fa", + "crates/rt/src/heap/source.rs": "e26374a3fd8a52ee80532dc63129bb3f9fa64cb5ecff322635c76144b23c64b2", "crates/rt/src/hint.rs": "dec1a305bcda6df6569fc72bf534531a411b2af61b121d90c6bbebeb8e2b1098", "crates/rt/src/lane/fiber.rs": "3755850096e916544cae4b269f301d035848bbad69d284a3b75f049d82a384f9", "crates/rt/src/lane/mod.rs": "6912c495dfbb3bec8e9ddbb6cf0e5458ca5318b6e90882ae3433c2f0649d2313", "crates/rt/src/lane/switch/mod.rs": "932f1e204f8abeae5614a9caea091e982875c7eea2db87a2a7a643f4a1ff577d", "crates/rt/src/lane/switch/x86_64.rs": "97c83af0e63523a220924f50fb2b058fcda650cc41445f0d010fa128ecc41366", - "crates/rt/src/lib.rs": "344dd15bb2aa428f751664c41f4db22d2c05eeb006af1c350b3637d37ee60da3", + "crates/rt/src/lib.rs": "205cfacaf11ab1a1f883b24caf2abc2d221d8a0b97ccf501939a1d985e69952c", "crates/rt/src/mem_source.rs": "41cbfb3200c222c2d2d0094b19445a9cf7a016e80f20241f8f5cf22872c5ce56", "crates/rt/src/pace.rs": "81217909cee01fa148df9e953e6281d7abf328eed7c6257aac8fbcebc824710c", "crates/rt/src/pinned.rs": "9f86a46572178965c3a111363eb20431fc19916eb9e06c2da50f8f918f3eccf8", - "crates/rt/src/process.rs": "d551775314b6eff02843a039bd82f7c66e2867cb14c34fa45531e0efeee33a62", + "crates/rt/src/process.rs": "b9286ee4cda4417019e607f258aced1e71fa85d677690099b9602267d27f86a6", "crates/rt/src/process/interrupt.rs": "553bb4aa5678b4efa8d7537c96df0369d8fb810d0b3abb398b71dc7edb3db174", + "crates/rt/src/process/words.rs": "cccc85ee65665f1a313c79be8aced6e408899a73986e0d8341ae3734ca79ce31", "crates/rt/src/random.rs": "b6d2738f97a9717d3d9481c49003e6c00df545faf52564189ad1cec508d16af6", + "crates/rt/src/reserve.rs": "b2a9cb91873cc404555818d66740e54d4517d7676ad1c1efb620cfd3e43fcdc0", "crates/rt/src/room/mod.rs": "c6f56dda5064ebe75ee85ef122d82c81a507f3fe498b6f9a8463a977a0566c04", "crates/rt/src/room/x86_64.rs": "122e283074fa754a49c73beb733a92660a5d8c53701b5223297ded797c61fc70", - "crates/rt/src/service/mod.rs": "e64f73db200c075fe3a567e325cb913ea27d1cafaaea0cc974a32a640cbb2879", - "crates/rt/src/session/builder.rs": "b78a7a96e842b6598bf97f89b8fdbe96d47329ed56d608c03c7c47b3e051bb7d", + "crates/rt/src/service/mod.rs": "b59017ca341af1f2267dfc1b06d1f11a6e505255b8c327bccee1dfe5d04356fe", + "crates/rt/src/session/builder.rs": "3c08d7a5d4c9c085f11163b4fbf715f31fd1895e0ba67869d06ce73a65401255", "crates/rt/src/session/flows.rs": "7205afbbe1d859868b313a24dccff8902e6ba3c1c8c45031e9531f6286259cae", "crates/rt/src/session/mod.rs": "466c2fbd8af228b3277075ad90a6d81031c1cb0a4c877f75669fff80e15338ea", "crates/rt/src/session/slots.rs": "82594125aa491429ed8c10b1316c19a82a162dc99f6dc97cdaa76398b92558e9", - "crates/rt/src/session/tests.rs": "27e044f75be3585d21e80beba0e667a56ab666e85064b50e178f1641d65959a9", - "crates/rt/src/spmd.rs": "8310d720182b16e5b98681758d4d2483b6905201431568c599247f65e2cd6b9f", + "crates/rt/src/session/tests.rs": "f0f0ccce35ec87e133e3dcd2d29c7f32790e4be094d31aa113efe4010caf03f2", + "crates/rt/src/spmd.rs": "ebaaa608c8177ed80091215b2b00240ae0d689e620f5fd418c860e78034b7e4b", "crates/rt/src/stack_arena.rs": "b252d45bfe845038ca9c8dc32941c634e448a2150dd49cf130e537048602f77b", - "crates/rt/src/star.rs": "59d03d7d7b55469a0de19f537e6871b1e91114dc09fec0edbdb6a34655d0a151", + "crates/rt/src/star.rs": "a6cf5ff1cbf01f18bb3605e3597da6921ca611da087a3a30ecd61cd026ce6d30", "crates/rt/src/startup.rs": "b1106a10e82659b8526ba7c94b7e40849edfdcbc30b077a77242712da8ebbd93", - "crates/rt/src/svc/client.rs": "b3488fea0c25cb5ca307e716ab94c58793f73276fddb397d6f178bbfe5266e0f", - "crates/rt/src/svc/mod.rs": "f518d462a19cc46257c1b2e1ffd0d9e266d90926a8641df7d01d3e2b5d10262f", + "crates/rt/src/svc/client.rs": "72fb7d66de1981ae253512a1f2026922ce618c9c34eb46ccb47bde465a7fecea", + "crates/rt/src/svc/mod.rs": "a0f3497792d67233da0c96a31a33fe9d291f3ae870fc5570a3940e559795406d", "crates/rt/src/svc/recyclable.rs": "9e5af8ad316bdd89e001fef7fbb6914b5a936dcdfbab2122740e03c1cf0ee7ec", - "crates/rt/src/svc/serve.rs": "184636ade6f76e803591a2c6553608c9001b5817fb11edef19fe4142cd453d17", + "crates/rt/src/svc/serve.rs": "a22172638a94e3ab52b88095dff9d78c88a1d693c81c6242af4932881d268706", "crates/rt/src/task_watch.rs": "96b646f8785e427640519191cbf1f294c6908e5062a8082db271364048aa5b9a", "crates/rt/src/termination.rs": "920685ebb2071db58ae667e57bd02a6127166e8224ed13ec81216465ca315a55", - "crates/rt/src/thread.rs": "fd1e690e296483577e0f41eb39b0bb88997b03b319493191a1143087e02ca030", - "crates/rt/src/thread_controller.rs": "9bd123d95349780caa4e2d50f281326eae37c207c41fd4171014c5ddbcebae52", + "crates/rt/src/thread.rs": "33c50deece11724565c0d5ca0f8843d41156e6b03123f1f565585c1be1ddc8b7", + "crates/rt/src/thread_controller.rs": "d64362e7e42f310b5fb7ad61c166938b05ee9edf2cedf7742ca6eba59a222137", "crates/rt/src/tls.rs": "85bd95bafb3de73d249d7a2eb8779b5afc929077e27b4baa5f7c7428063063ab", - "crates/rt/src/transport/mod.rs": "9fb930d714458db515d6963f5229ced4e211fa2ae772aa5afe18feea81735e01", - "crates/rt/src/validate.rs": "b7a4d3741dffd4a560bdae252128030cc4e6c4117e7274d121c60d8877aef121", + "crates/rt/src/transport/mod.rs": "d5af890c800dd53abdbea8329383068cf43f0bf294ccb6b46dd7d8592ae5f321", + "crates/rt/src/upcall.rs": "c0cbd2a59479dfba006850bb1bc5dd4f20f59317d178af776ba678f0e6404f02", + "crates/rt/src/validate.rs": "6eb49c9b9ec4f9ad7307fb350765475b2fce1bfef96ec8a821839b9a38ae6ea6", "crates/rt/src/wake.rs": "ba92b949d9ba3102b2ebff03f9f31c45c85a31c8ceffd0d823976489947fde87", "crates/rt/src/worker.rs": "97d0e1a6bde5800e4409c1cd7a7b688bc8f06d604a6c86338e1fdc78bed15d50", "crates/rt/tests/bundle.rs": "464ddd671c3f6fbbd376be1b90a64f8bd9534c21027cd79c43c0faf4d3fdcdbd", @@ -243,22 +260,97 @@ "crates/start/src/lib.rs": "c41bea895b5399b13073bccdc9ee32eab249be55e5999916d0fd0af3a43891b9", "crates/uart/Cargo.toml": "c3812a9dc9c33bb49ce4ec27d8440b9937b9895f5f15d429e325965a3d6b2e4f", "crates/uart/src/geni.rs": "b22c387c400edb0792a801705627d2a58c30b543ad458a230dfd13ccaad3ac13", - "crates/uart/src/lib.rs": "6e7b17f2628dad807905f1e82fdb2c3d3f90fe66b810a507977d0565edef8960", + "crates/uart/src/lib.rs": "b40998aac81f7c296cc26c105762bb5fabab0ceed1078a82ee5fc4d47545e0e6", "crates/uart/src/pl011.rs": "920c1ad2b67197ea6508de8355d966384a72b9f2ffb72cdfc1c2cbc6b056b261", "crates/uart/src/tx.rs": "532e0fef75563bf5294211502fc0da6534e51e96b4da0a71896fe629c3e5d7be", - "crates/uart/src/uart16550.rs": "9612883882ce72f0628709099944df5598f492d9bcacd0df79736df3b075b145", + "crates/uart/src/uart16550.rs": "689427e689025653727db7bbc30a9eb4d0dbf12cc8202d4ae1aef1978f11555c", "crates/window/Cargo.toml": "96c2d4c89e179c3718d4b4e897bb0a44c55ac7ad66341529da4bfd35e203592e", "crates/window/src/appearance.rs": "b22e65286b94215bc4265a09ef280a5118f368ac45120311b870a56f7cc40a81", - "crates/window/src/channel.rs": "256a40e48eaf0a5c96abe4d88edb49767b8bd3da3c74ef647ca5f0f56a94ff46", - "crates/window/src/client.rs": "ca9b7dfb83e31868d062a3de2a1307181d91fc8a5da8a55b5d54cea87438de7d", + "crates/window/src/capture.rs": "d793f70fff35dfb9b13547f016b5abf8d851a23e53cfe28c1c24ad9949e6ef44", + "crates/window/src/channel.rs": "6c96578badb21ddf6f3eb587e65b78af275bd8bcc0e03fd359c88eba6d91138f", + "crates/window/src/client.rs": "2e82a529684f4413cbb1d1475f5d6f108a8eb64f961aa0caec441045c4862d35", "crates/window/src/command.rs": "3ad4867ce62f3a430a68e4d105991229938e131e8cfde8afbbb0f522f99355d3", "crates/window/src/configure.rs": "5731662e919827aa6c9b189dce1d4ae06ebc7f7940c57fafd8662fc33bc46154", - "crates/window/src/control.rs": "bf482439516c5d0334acfe764cc3ae4c838743cb2082da5e6476286acdf5d922", - "crates/window/src/event.rs": "bbe36a67815ab5190308a11299c078bb232f44ca332f9173a591a81e0a6df683", + "crates/window/src/control.rs": "0ccad3310bc1abc5efa44ff26d929ee7f7e54c6b83d1a3467c8dca5b3b3f8ca3", + "crates/window/src/cursor.rs": "2c320f4f19af4a5f9cd0d4059d1452661ac8cea23f0f6a44e0833cccd4d5f4bd", + "crates/window/src/desktop.rs": "2aca94383249fac9f770c4edae35f634ad89388c463b17e4484125190757532b", + "crates/window/src/document.rs": "1c4fb5a4bd3ce170a8a79ea97e7efd9aa013b1d2678b5ca1dde06ea1508ddaca", + "crates/window/src/event.rs": "cf6565ea882e6c77672f31f0b354b03c57f25591a558893bb9466070b3f503a4", "crates/window/src/face.rs": "1078d8664e6da529c9d4fd43a7e3df77f287d0061f418ed21859d289e86765a0", - "crates/window/src/lib.rs": "2045cc4a46d934ae6eb65cada65ef7518c118ad47a7e593bda28ad22de63c04c", - "crates/window/src/relay.rs": "c58b879c2f8fc93da59e456f0f49a260caed85fbe223930a81898137ea442c95", - "crates/window/src/wire.rs": "efa0657132be5814102d1f688f6220e9772c0e151371ea104a18a6a43a6d50fe", + "crates/window/src/lib.rs": "310eeb5e3fdafe4e881008d311d0de4346bc1f7a78d417cba2c1e59e354670d6", + "crates/window/src/menu.rs": "8baf547a2dd58f83eef635e16e8658a011c917b60125b5d041c885b9eb69db1e", + "crates/window/src/notify.rs": "8215605e26509fd615a983eab746ddcec18a156543349f52ddff6f301bdfe569", + "crates/window/src/pick.rs": "d861daa5137f7513bc142737da8bd104f84c423e9c26af4f48ef19a3afe1b5aa", + "crates/window/src/relay.rs": "8f56568bcf643825906c74180ee13a14acb8299f6fb08f968919370e82669348", + "crates/window/src/wire.rs": "572daa34bc6233540779c542cd47533ec9c294123562de88ea6a5c2fb54572bf", + "drivers/nvme/Cargo.toml": "21acf81448d0cc403eb10ada1c407baf7ff8cea4778ccdeb6c4f0d3508a66fbd", + "drivers/nvme/src/block.rs": "4df5122b9c8f28ae878bffb6eeacfdba94569f73182c147a555f345a4d877723", + "drivers/nvme/src/commands.rs": "a4cdbfb2b7822143beb7f351f4bba7b4643da1bffa80ffe443f065e47545c097", + "drivers/nvme/src/controller.rs": "b5b281c386a2387fae7757809effeff628d2bb6c2e1ad6c9f7ef0c796fd986d9", + "drivers/nvme/src/lib.rs": "7799c678f21e617c074776d9129e2186b3dbba30d3f40c2ecbcc0427f4439297", + "drivers/nvme/src/mock.rs": "1bc0d8440a15ec68c4f2ff1a10df72530047c0353b6aac648af4154fb5d3a3dd", + "drivers/nvme/src/namespace.rs": "d3b43787bed1b39d6f88226d96ac7fa845d379cced09e3cbd4c1b016838b3b70", + "drivers/nvme/src/prp.rs": "3828e171833d724a42e0de01b48626cdfe7a37d7e661dab15b816b6e046b3feb", + "drivers/nvme/src/qpair.rs": "45931cd91894c7e0758947e5dd9180ce12796e9698d3c70067b7af08eb53f2d8", + "drivers/nvme/src/regs.rs": "d9410b5c64bb70f99e042f7209119c0f0fdb57f859c6a1c07eba90199f1e98c1", + "drivers/nvme/src/swap.rs": "f5e1099b83b2cdeb9d288f4d337f1a234d9dfd16c31e9d9303c10f6f14972141", + "services/block/Cargo.toml": "2729b36369486a6f5c1c29200eaa4603b25c26352f06f52301b70d9d57260e43", + "services/block/boot.toml": "a6d8defc624013f6edd2f489cda144375b9e5d94bbaf25f11efd24ecb0ebc68f", + "services/block/src/client.rs": "6aed79a404620f89ffa13078f615e846fdf6f3db5629c5cee5fbccbeaa7636cf", + "services/block/src/lib.rs": "cf45960460cf6c9dba72f7a6727b51497275c686166a90be531d675097c275f1", + "services/block/src/service.rs": "191e9d887d3f3267d6e2aa96e81996f01321bb2febb85e68581159f6e098b4b5", + "services/block/src/wire.rs": "a9d64aef9ec3d58d37ca55b841933fc51b76ab94e18b245278b4519e96c65939", + "services/dagfs/Cargo.toml": "d229a126616b1d7638f0f5a6edb4ac37d9d116f32afaa1897ffe484fbb286e5e", + "services/dagfs/boot.toml": "60d7176256fbdf96b7d8f6025c8346db9f847f806842bc58b5cfb2432a3051aa", + "services/dagfs/examples/fsdemo.rs": "1666a14ef62fd4fc46f7f9efa2eaa5021ed79823f273a77f53758307f884e7d0", + "services/dagfs/examples/fswire.rs": "994b97d699796b317741e2c87cbe812cc1776ddcd01173323e72a5f74c8ac267", + "services/dagfs/examples/model_volume.rs": "0e3623e962da02b2126062a19d198b9e0a8f6e91896fd9a6f7d13f6f30ddc5f1", + "services/dagfs/examples/volume_read.rs": "bd653b086745febd2b19ade1748207138f70d3165aebad0bc8ca66c1f5d8b9f3", + "services/dagfs/src/authority.rs": "3f924460db27d9d3632705261bd473a37b2e1ab3050c3d88c66d335897bd1e9f", + "services/dagfs/src/bounds.rs": "c825fcab0f30777e772c21fcf68cd092b10cf57a741eac22747de87ec3742e87", + "services/dagfs/src/client.rs": "3532931af9484f548f7ff60bfd58f56fe392d8a770779f766f4e0edb0580d7d4", + "services/dagfs/src/digests.rs": "88a256596ab314714ef523a2bea44dd8d5265436b284534d25d4538535454f3a", + "services/dagfs/src/durable.rs": "93df70108d12fd28134dbaf2434a8e067df2a415bbcd0cdb969a84b86403495d", + "services/dagfs/src/engine.rs": "3c52e78d1acf1bdd89d4bc9a77d95e97c139cebdb2009c2cd25921caa634365e", + "services/dagfs/src/error.rs": "0a0cac522b38baa34221adda4ccca010fd8119c5aaf1f7b6015ab7a5f602ac33", + "services/dagfs/src/extraction.rs": "502c7ba6cc8a37016a6c4a43f46664c3f27fe4c01d2cac21f8eb2cc2591a531d", + "services/dagfs/src/lib.rs": "9e4c0e957c81a9504e4217e0c16a83a0dad48fe35ead5a5785e7d139105d6cc8", + "services/dagfs/src/namespace.rs": "638a66619320f580de82b082f5520f35d47d8ab9b39b21ce413365dbf89dacaa", + "services/dagfs/src/object.rs": "ce3468a2a536069eddd3598e8593c7eddc8e5107bd96f6e939af0133eb02fb91", + "services/dagfs/src/parent_index.rs": "a46346a378ff2e9dd2ffe2d8a9514d8c777a7d0170ff3b90a9345def3b452a02", + "services/dagfs/src/path.rs": "98ddd0026e43f6a4957b896bfa0c33ec217b016229662ebc0402283d1e8b0518", + "services/dagfs/src/persistence.rs": "1bf80a2654317434ce174fe950ce47f080083ddacc8ebc0ba10bdc5337ea7b12", + "services/dagfs/src/provider/block.rs": "89180a583d3e651c8edebfa851b155a991da33dda4df66c3fe1e259f67245bde", + "services/dagfs/src/provider/extent.rs": "94656ebe2ddaab5b257c89c2af8a1ab56e745eddc301fd02c99a28fabff9d6bb", + "services/dagfs/src/provider/mod.rs": "1e40047a77fc1f5140626bc25c9d57e681a38302d8b7a30c9e1c01c140cb3db8", + "services/dagfs/src/provider/ram.rs": "fa3175f2d98854f2dc99f41cec1eb47ae7e4a560f9307de5f2c3778f95b7f7f1", + "services/dagfs/src/reach.rs": "0b72f192ef37320d0a9a6883b7bdba2cd1a291f76dc343f24c52059d588e94ac", + "services/dagfs/src/router.rs": "07e9ceb72b8af7ab3ec944212df2009fe89d276b9179013d32baaa1e70f7f8d4", + "services/dagfs/src/search.rs": "98e3807b60e039082e2f16cd975d4637520642b380ea5ee3b7a25a4b843de448", + "services/dagfs/src/service/contract.rs": "32219b508a02f9136416618d2e2ab2db1f3bda46f18f8654683caf43e7eb4a66", + "services/dagfs/src/service/mod.rs": "c815acf69647b61a3b907b19b46e1456894d19237d7f19206892f336c9b1b3e0", + "services/dagfs/src/slot.rs": "e8f8751c09a4e6006bcaccd2d189cbb6762da1be6c4ea56bd331fbdc7f5cf827", + "services/dagfs/src/window.rs": "8be4bd5c8f1e51cfa5744801e12a9406e7dafb351d0340a625ca8b8bddeb3364", + "services/dagfs/src/wire.rs": "3c85fd1cdf0e35bb3a85cef10093f1b0d08626a9198fe219ff68fe09ba64595b", + "services/dagfs/tests/authority.rs": "1432f45aef2bdffd63b83a2e93481673ba3412b14d4dc2abc1ef91f105172641", + "services/dagfs/tests/common/mod.rs": "e9d14fc2afc827dcd1ad8ed17177130f546ec090cd45a9d313e6237f5661a189", + "services/dagfs/tests/concurrency.rs": "ad7a8e54da2f123ae861f09449f5fc0865db327136f7ba845a4fc7df66b04d94", + "services/dagfs/tests/fs_search_witness.rs": "7bac1416fd7c0537211691a6ec1b42bff916a601b91b4dea55e87d4866c91b50", + "services/dagfs/tests/loopback.rs": "8d15e099fec648c680dc6bbf26c2828fb5e5d2c65ffe2a23d27ba4b566280fc6", + "services/dagfs/tests/namespace.rs": "16e80c2689cef21d52adc1d103776657ee1dfed40909a51a5552dfbdeb96b3e1", + "services/dagfs/tests/pair.rs": "10265d0de5e965878d7ad12ad33982c83abf4ddec3f9f4698e75701220086e99", + "services/dagfs/tests/protocol.rs": "cc79eda8f6cd7d47378bcdcbe2cad15c826c3ea77776320880e480225b77086a", + "services/dagfs/tests/provider.rs": "ba4070af34ae034c109e391e97bcc4755c6bd70b8e2fb229aaf9d747fef71e2d", + "services/dagfs/tests/query_experiment.rs": "6fe147810474b6754ef31ad0fb95493e0b3c0886fbbafd78deb3729d410e8155", + "services/device/Cargo.toml": "8f35c9ddd701b58a538f151e95e93ce618f0b346960491adad6d843287165d1c", + "services/device/boot.toml": "18b640a44bda45b29f3fb381477500a989c9f94b77dcb69eac0ec79dddbeed36", + "services/device/src/client.rs": "71660deb4a9f51edb7a72fbb3edac94170bf4cccf9ce182cbd478ee3daf406eb", + "services/device/src/lib.rs": "215c7c7e81d5d51664ac77b72b2edb0bab739d5a54b30bc9ad08f9dd3a3cfea4", + "services/device/src/service/mod.rs": "817a718222ab737712ba8223bdd5392d1ca59b02d1edcf119a5812d2e8f96aa4", + "services/device/src/wire.rs": "aaec2baa85f4d5ceab233302bd1ba8e54c158f56dcbdef3bbd59e0d99f449d18", + "services/extract-worker-c/Cargo.toml": "0a84769f520d5f2735711a21efebfc93380d91724ba2ed690cfe0fe1529e1513", + "services/extract-worker-c/build.rs": "66b66d671df56f116176896f7f6c9aaba008ada2d51d0b3dc1f6ca4db08e74bd", + "services/extract-worker-c/src/lib.rs": "1f630d54e75b0c4a3828e5386ac10cc978eb4e216bc3dda85ee57438315da1f7", "services/extract-worker-rust/Cargo.toml": "dcd585ea908827f474ad0e2131db6adffebff3fef2ec2ce1945bcb48b44211cd", "services/extract-worker-rust/LICENSE-TTF-MIT": "bb7d16c928ad39fd073ae51d636ef8b62ec893ccea4058a0b5829bec76ad1fca", "services/extract-worker-rust/TTF_PROVENANCE.md": "310c740d7a97dd70ee3df6b2be85047a2867d6b4319c0ce2b51629101e7767b8", @@ -292,10 +384,34 @@ "services/extract/vendor/jsmn/LICENSE": "466d7a688dfcab0e1a274de43c7c679b775c5fd00f607037dcc4fe789bbe9549", "services/extract/vendor/jsmn/jsmn.h": "babfcb7413354a43dcd539c600a1e00bd2d1579ad0a6f6c7d8b81dccedb16cd0", "services/extract/vendor/jsmn/kore_jsmn.c": "a666060a7af03c819d1faaccc0d6faa19b76bcfe78e4ef61aeab273ad052c95c", + "services/mm/Cargo.toml": "25809b6a03916d96b8de64a5988d005c71c49314fafbe0d920c3ec06f2dc5951", + "services/mm/boot.toml": "d4794c49f41698733a0569a0dd0354d17238321b0048ad3f3a348612784f0cfc", + "services/mm/src/client.rs": "c6dcc0dc3ed59e0a4cf7532fd306af4c1dc4d4e5ad1edd1d86887def791c5aa6", + "services/mm/src/heap.rs": "e55210f3609f2e3e90f30d5497355ddab5801acc60ccbe3f8acbbffff71dbfde", + "services/mm/src/lib.rs": "d3820740666ed993e547ee1b3b579231bf439a813d9766860e3b1b57fe815bd6", + "services/mm/src/service/mod.rs": "886cbc83e6e3be5091d819737c638e70ef724f4a5cfa7674d01ac5db1096fda0", + "services/mm/src/wire.rs": "6c77f4c07f248d55a28f668216eff758e7222d55e9652a6726be77e6bb4d2fef", "services/output/Cargo.toml": "0c65c4a959f372d5063f505d2a5a904f525c0bbba48ac44c96df031126100dca", "services/output/src/client.rs": "31e1ce0dd65a600e168bf5f3ebaf216d15256e0871d9a781edf016abb9590b99", "services/output/src/lib.rs": "29b99983a081a6c0c4a84adcd319a750e09d7ff3df0765aeb7b2545769d76915", "services/output/src/service/mod.rs": "d80c3beb50979b3e1e500abdfa3c0b9d38e354c24e2e34aea443732abbead6f1", - "services/output/src/wire.rs": "3752e994eb18714518facae1b78a9ee2b34f99aad8687653f0b89dc8fdeca621" + "services/output/src/wire.rs": "3752e994eb18714518facae1b78a9ee2b34f99aad8687653f0b89dc8fdeca621", + "services/swap/Cargo.toml": "9c0af01403a6ad5a7f4ff69a02b18c8b1b2624a684814d6dc1ac07968c5bbe53", + "services/swap/boot.toml": "046fbdab1a4d21d6a2e623b15a52d38b841fbb44b0e4f70148c2b16356af93d3", + "services/swap/src/client.rs": "04f735d059b491765d5ba63a4f5f640332735bf336e62dcbae86c17647583e2c", + "services/swap/src/extent.rs": "d6705f77951df1c42cda51fc4526ba6262cab6af4a0545405ce06d2d2e1db7b4", + "services/swap/src/lib.rs": "883684df54a87ae66aa7895ea19a872cabc8ab7fd4aded4ea65b7e3dd51fe0cf", + "services/swap/src/service/mod.rs": "34597aeb2af68085cf129ba6f6641fac52fd9d8fd9eca26dd264c9feadc77dd0", + "services/swap/src/wire.rs": "ff1ab3e6b42214d0c009b8cb1bd07ffa5238348b63ca273337b427dd4637ca14", + "tools/kore-vol/Cargo.toml": "d7f3f5c13b1fddd010b5d70d5e16500cec66e8a48221667f5aec84c3ce7af4d7", + "tools/kore-vol/src/device.rs": "21011b8ca119bcf987486f0d79bafdd86e7219183577b27b5dd3145f8acb33f2", + "tools/kore-vol/src/lib.rs": "41f7fc123e770cc1f34e97de0c2fc6a0ee9211cc29172e23cb42f01e84d2b034", + "tools/kore-vol/src/main.rs": "badf82b2ca2c83e5c29af760f4b50021d679ad75e65a94259b01efc082019a25", + "tools/kore-vol/src/source.rs": "eaa9da2c97057be86b4dc8543213728ee22ecbf1be7484f3899affd03c09c68b", + "tools/kore-vol/src/volume.rs": "67bb73f4a5ae5cc90b2511ecd1e74c91d573ca593f2fff986d6cbdad478f76e6", + "tools/kore-vol/tests/cli.rs": "66983b612af2ec94f30a4dcecdd73a38e52e97523f2efecd678d450e7e984bd4", + "tools/kore-vol/tests/common/mod.rs": "b561540f1cca00f440adf90123eafad96076ed5c8291af634528b4ce2edbb159", + "tools/kore-vol/tests/device.rs": "4082b44d7c9dc05b23b459eaed612a6160cd964da40496295d0136d4b546074b", + "tools/kore-vol/tests/library.rs": "a2e943e86b00edf9a3fffbd3894fa5bca581e1495d30e72f0dc661bc7b90fc0e" } } diff --git a/native/services/block/Cargo.toml b/native/services/block/Cargo.toml new file mode 100644 index 0000000..5a14560 --- /dev/null +++ b/native/services/block/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "kore-svc-block" +version = "0.0.1" +edition = "2024" +description = "Capability-native block-channel provisioner for kore" +publish = false + +[dependencies] +kore-abi = { path = "../../crates/abi" } +kore-rt = { path = "../../crates/rt" } +kore-driver = { path = "../../crates/driver", features = ["service"] } +kore-svc-device = { path = "../device" } +kore-nvme = { path = "../../drivers/nvme", features = ["alloc"] } +zerocopy = { version = "0.8", default-features = false } diff --git a/native/services/block/boot.toml b/native/services/block/boot.toml new file mode 100644 index 0000000..dc9f49e --- /dev/null +++ b/native/services/block/boot.toml @@ -0,0 +1,16 @@ +# services/block/boot.toml — capability-bounded block-owner row. +# Founding order lives in res/boot.toml; this fragment declares only the +# service-local image, bundle, and protocol citations. + +# The block provisioner claims an NVMe controller through its own device- +# broker client face and retains the ordinary device and DMA-supply ancestors. +# It exposes only a bounded generic block-service endpoint to the filesystem. +# Device children, DMA supply, and the direct qpair remain inside this task. +[service] +image = "kore-svc-block" +chain = true +endowment = "kore_svc_block::BlockEndowment" +protocol = "kore_svc_block::BlockService" +endows = ["DMA_POOL@1"] +digest_x86_64 = "blake3:c852d96c707ed0e95aad161be935fcc8bee5310598ec94f0f3bc75849cf586b0" +digest_aarch64 = "blake3:98023e83300526b637f67f62bafb1b523cd8657dabe4c980f5133f282d6f3ed0" diff --git a/native/services/block/src/client.rs b/native/services/block/src/client.rs new file mode 100644 index 0000000..55ab4f1 --- /dev/null +++ b/native/services/block/src/client.rs @@ -0,0 +1,85 @@ +//! Typed client for the block control plane. +#![allow(async_fn_in_trait)] + +use kore_rt::cap::{DeviceToken, SupplyToken, verbs}; +use kore_rt::svc::SvcClient; +use kore_rt::transport::Transport; + +use crate::{BlockRefusal, BlockRequest, BlockResponse, BlockService, ProvisionRequest}; + +pub struct OpenedChannel { + pub device: DeviceToken, + pub pool: SupplyToken, +} + +pub trait BlockClient { + async fn open(&mut self, request: ProvisionRequest) -> Result; + async fn status(&mut self) -> Result<(bool, u16, u16), BlockRefusal>; + async fn close(&mut self) -> Result<(), BlockRefusal>; +} + +pub struct RingBlock<'w, 't, T: Transport>(pub SvcClient<'w, 't, T, BlockService>); + +impl BlockClient for RingBlock<'_, '_, T> { + async fn open(&mut self, request: ProvisionRequest) -> Result { + let mut reply = self + .0 + .call(&BlockRequest::Open(request)) + .await + .map_err(|_| BlockRefusal::Unavailable)?; + match reply.response { + BlockResponse::Opened { .. } => {} + BlockResponse::Refused(refusal) => return Err(refusal), + _ => return Err(BlockRefusal::Unavailable), + } + let device = reply.retained[0].take().ok_or(BlockRefusal::Unavailable)?; + let pool = reply.retained[1].take().ok_or(BlockRefusal::Unavailable)?; + let device = match verbs::claim_returned_device(device) { + Ok(device) => device, + Err(returned) => { + reply.retained[0] = Some(returned); + self.0.poison_retained(reply.retained); + return Err(BlockRefusal::Unavailable); + } + }; + let pool = match verbs::claim_returned_supply(pool) { + Ok(pool) => pool, + Err(returned) => { + reply.retained[1] = Some(returned); + self.0.poison_retained(reply.retained); + return Err(BlockRefusal::Unavailable); + } + }; + Ok(OpenedChannel { device, pool }) + } + + async fn status(&mut self) -> Result<(bool, u16, u16), BlockRefusal> { + let reply = self + .0 + .call(&BlockRequest::Status) + .await + .map_err(|_| BlockRefusal::Unavailable)?; + match reply.response { + BlockResponse::Status { + available, + channels, + limit, + } => Ok((available, channels, limit)), + BlockResponse::Refused(refusal) => Err(refusal), + _ => Err(BlockRefusal::Unavailable), + } + } + + async fn close(&mut self) -> Result<(), BlockRefusal> { + let reply = self + .0 + .call(&BlockRequest::Close) + .await + .map_err(|_| BlockRefusal::Unavailable)?; + match reply.response { + BlockResponse::Closed => Ok(()), + BlockResponse::Refused(refusal) => Err(refusal), + _ => Err(BlockRefusal::Unavailable), + } + } +} diff --git a/native/services/block/src/lib.rs b/native/services/block/src/lib.rs new file mode 100644 index 0000000..3da4b87 --- /dev/null +++ b/native/services/block/src/lib.rs @@ -0,0 +1,343 @@ +//! Block-channel control plane. +//! +//! This crate deliberately contains no per-I/O request protocol. A +//! [`BlockProvisioner`] authenticates policy, constructs bounded affine +//! [`BlockChannel`](kore_driver::block::BlockChannel)s, observes their lifecycle, and +//! revokes them. Reads, writes, flushes, discards, and completions stay on the +//! granted channel and never call back through this control object. + +#![no_std] + +#[cfg(test)] +extern crate std; + +pub mod client; +pub mod service; +pub mod wire; + +pub use client::{BlockClient, OpenedChannel, RingBlock}; +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +pub use service::run; +pub use service::{BlockEndowment, BlockRefusal, BlockRequest, BlockResponse, BlockService}; + +/// Architecture-neutral service entry selected at this crate boundary. +/// +/// x86 currently composes the direct NVMe implementation; other targets keep +/// the same founded control plane and report its honest unavailable posture. +/// Clients and crt0 do not branch on the provider implementation. +#[cfg(not(all(target_arch = "x86_64", target_os = "none")))] +pub async fn run(rt: kore_rt::entry::Rt) -> ! { + kore_rt::entry::serve::(rt).await +} + +use core::ops::{Deref, DerefMut}; + +pub use kore_driver::block::*; + +/// Provisioning policy for one consumer/worker. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProvisionRequest { + pub writable: bool, + pub mode: CompletionMode, + pub requested_queue_depth: u16, + pub requested_buffer_bytes: usize, +} + +/// Control-plane failures, distinct from data-plane failures. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ProvisionError { + Unavailable, + Policy, + Exhausted, + InvalidGeometry, + Busy, + Lost, +} + +/// Observable provisioner state. No queue or device implementation detail is +/// exposed to callers. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProvisionStatus { + pub available: bool, + pub channels: u16, + pub limit: u16, + pub geometry: Option, +} + +/// Maximum independently revocable channel lineages tracked by one control +/// service instance. Policy may choose a smaller runtime limit. +pub const MAX_CHANNELS: usize = 16; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct ProvisionId { + slot: u16, + generation: u32, +} + +#[derive(Clone, Copy)] +struct Lineage { + generation: u32, + live: bool, +} + +/// One affine provisioned channel plus its control-plane lineage. +/// +/// This wrapper is deliberately neither `Clone` nor `Copy`. Dereferencing it +/// drives the channel directly; no method calls back into the provisioner. +pub struct Provisioned { + id: ProvisionId, + channel: C, +} + +impl Deref for Provisioned { + type Target = C; + + fn deref(&self) -> &Self::Target { + &self.channel + } +} + +impl DerefMut for Provisioned { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.channel + } +} + +/// A bounded control-plane allocator. Device-specific construction is a +/// closure so the service can compose the existing driver-host intake without +/// making the generic provisioner name PCI or NVMe. +pub struct BlockProvisioner { + limit: u16, + live: u16, + available: bool, + geometry: Option, + lineages: [Lineage; MAX_CHANNELS], +} + +impl BlockProvisioner { + pub const fn unavailable(limit: u16) -> Self { + Self { + limit, + live: 0, + available: false, + geometry: None, + lineages: [Lineage { + generation: 1, + live: false, + }; MAX_CHANNELS], + } + } + + pub fn online(limit: u16, geometry: BlockGeometry) -> Result { + if limit == 0 || limit as usize > MAX_CHANNELS { + return Err(ProvisionError::Exhausted); + } + let geometry = geometry + .validate() + .map_err(|_| ProvisionError::InvalidGeometry)?; + Ok(Self { + limit, + live: 0, + available: true, + geometry: Some(geometry), + lineages: [Lineage { + generation: 1, + live: false, + }; MAX_CHANNELS], + }) + } + + pub const fn status(&self) -> ProvisionStatus { + ProvisionStatus { + available: self.available, + channels: self.live, + limit: self.limit, + geometry: self.geometry, + } + } + + /// Construct and account one affine channel. The returned value has one + /// owner; dropping/moving it is Rust ownership, not a copyable token. + pub fn provision( + &mut self, + request: ProvisionRequest, + build: impl FnOnce(ProvisionRequest) -> Result, + ) -> Result, ProvisionError> { + if !self.available { + return Err(ProvisionError::Unavailable); + } + if self.live >= self.limit { + return Err(ProvisionError::Exhausted); + } + if request.requested_queue_depth == 0 || request.requested_buffer_bytes == 0 { + return Err(ProvisionError::Policy); + } + if request.writable && self.geometry.is_some_and(|g| g.read_only) { + return Err(ProvisionError::Policy); + } + let slot = self.lineages[..self.limit as usize] + .iter() + .position(|lineage| !lineage.live) + .ok_or(ProvisionError::Exhausted)?; + let channel = build(request)?; + let lineage = &mut self.lineages[slot]; + lineage.live = true; + self.live += 1; + Ok(Provisioned { + id: ProvisionId { + slot: slot as u16, + generation: lineage.generation, + }, + channel, + }) + } + + /// Drain a returned channel child-before-parent and close its exact + /// lineage. A failed drain returns the affine channel to its owner. + pub fn close( + &mut self, + mut provisioned: Provisioned, + drain: impl FnOnce(&mut C) -> Result<(), ProvisionError>, + ) -> Result<(), (ProvisionError, Provisioned)> { + if let Err(error) = drain(&mut provisioned.channel) { + return Err((error, provisioned)); + } + if !self.retire_id(provisioned.id) { + return Err((ProvisionError::Lost, provisioned)); + } + drop(provisioned); + Ok(()) + } + + /// Force a returned channel through a service-specific revoke action, + /// then retire its lineage. Used for reset/failure teardown rather than + /// graceful close. + pub fn revoke( + &mut self, + mut provisioned: Provisioned, + revoke: impl FnOnce(&mut C), + ) -> Result<(), ProvisionError> { + revoke(&mut provisioned.channel); + if !self.retire_id(provisioned.id) { + return Err(ProvisionError::Lost); + } + drop(provisioned); + Ok(()) + } + + fn retire_id(&mut self, id: ProvisionId) -> bool { + let Some(lineage) = self.lineages.get_mut(id.slot as usize) else { + return false; + }; + if !lineage.live || lineage.generation != id.generation { + return false; + } + lineage.live = false; + lineage.generation = lineage.generation.wrapping_add(1).max(1); + self.live -= 1; + true + } + + pub fn mark_lost(&mut self) { + self.available = false; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use kore_driver::block::memory::MemoryBlockDevice; + + #[test] + fn provision_is_bounded_non_amplifying_and_data_plane_bypasses_control() { + type Dev = MemoryBlockDevice<4096, 1024>; + type Chan = BlockChannel; + let geometry = Dev::new(512, 2).unwrap().geometry(); + let mut provisioner = BlockProvisioner::online(1, geometry).unwrap(); + let request = ProvisionRequest { + writable: true, + mode: CompletionMode::Poll, + requested_queue_depth: 4, + requested_buffer_bytes: 1024, + }; + let mut channel: Provisioned = provisioner + .provision(request, |_| { + BlockChannel::new(Dev::new(512, 2).unwrap(), CompletionMode::Poll) + .map_err(|_| ProvisionError::InvalidGeometry) + }) + .unwrap(); + assert_eq!(provisioner.status().channels, 1); + assert!(matches!( + provisioner.provision(request, |_| -> Result<(), ProvisionError> { Ok(()) }), + Err(ProvisionError::Exhausted) + )); + + // Direct I/O mutates only the channel; the provisioner's books do not + // participate in or count the request path. + let buffer = channel.register_buffer(512).unwrap(); + channel.write_buffer(buffer, 0, &[7; 512]).unwrap(); + let id = channel.submit_write(0, 1, buffer, 0).unwrap(); + assert_eq!(channel.poll().unwrap().request, id); + assert_eq!(provisioner.status().channels, 1); + assert!( + provisioner + .close(channel, |channel| { + channel.quiesce().map_err(|_| ProvisionError::Busy) + }) + .is_ok() + ); + assert_eq!(provisioner.status().channels, 0); + } + + #[test] + fn diskless_posture_is_honest_and_nonfatal() { + let mut provisioner = BlockProvisioner::unavailable(4); + let request = ProvisionRequest { + writable: false, + mode: CompletionMode::Poll, + requested_queue_depth: 1, + requested_buffer_bytes: 512, + }; + assert!(matches!( + provisioner.provision(request, |_| -> Result<(), ProvisionError> { Ok(()) }), + Err(ProvisionError::Unavailable) + )); + assert!(!provisioner.status().available); + } + + #[test] + fn failed_close_returns_the_affine_channel_and_keeps_its_lineage_live() { + type Dev = MemoryBlockDevice<4096, 1024>; + type Chan = BlockChannel; + let geometry = Dev::new(512, 2).unwrap().geometry(); + let mut provisioner = BlockProvisioner::online(1, geometry).unwrap(); + let request = ProvisionRequest { + writable: true, + mode: CompletionMode::Poll, + requested_queue_depth: 2, + requested_buffer_bytes: 512, + }; + let provisioned: Provisioned = provisioner + .provision(request, |_| { + BlockChannel::new(Dev::new(512, 2).unwrap(), CompletionMode::Poll) + .map_err(|_| ProvisionError::InvalidGeometry) + }) + .unwrap(); + + let (_, provisioned) = provisioner + .close(provisioned, |_| Err(ProvisionError::Busy)) + .err() + .expect("close refuses with ownership preserved"); + assert_eq!(provisioner.status().channels, 1); + assert!(matches!( + provisioner.provision(request, |_| -> Result<(), ProvisionError> { Ok(()) }), + Err(ProvisionError::Exhausted) + )); + assert!( + provisioner + .revoke(provisioned, |channel| channel.mark_lost()) + .is_ok() + ); + assert_eq!(provisioner.status().channels, 0); + } +} diff --git a/native/services/block/src/service.rs b/native/services/block/src/service.rs new file mode 100644 index 0000000..41382a5 --- /dev/null +++ b/native/services/block/src/service.rs @@ -0,0 +1,394 @@ +//! Founded block-channel control plane. + +use kore_abi::task::role; +use kore_driver::host::{ + InterruptBlock, InterruptPolicy, PciBindingContract, PciClaimPosture, pci_selector_for_contract, +}; +use kore_rt::cap::{DeviceToken, SupplyToken, verbs}; +use kore_rt::service::{DeviceSelector, GrantArg, Serve, SystemService}; + +use crate::ProvisionRequest; + +/// The one NVMe PCI contract shared by the block provisioner's claim ladder +/// and the filesystem binding that eventually realizes the controller. +pub struct NvmeIoPci; + +impl PciBindingContract for NvmeIoPci { + const BARS: kore_rt::service::PciBarMask = kore_rt::service::PciBarMask::BAR0; + const INTERRUPTS: InterruptPolicy = InterruptPolicy::Optional(InterruptBlock::SINGLE); +} + +/// The founder gives the control plane one DMA-capable physical supply of +/// the LARGE class (`DMA_POOL@1`, 128 MiB — the size the accelerator rows +/// take). The NVMe controller itself is claimed from the device broker. +/// +/// The whole pool is subleased to the filesystem, whose own queue and +/// buffer graph fits the ordinary 2 MiB class exactly; the remainder exists +/// for the controller's **host memory buffer**: a DRAM-less controller keeps +/// its mapping tables in host memory and, without one, pages them from NAND +/// on every access. +#[derive(kore_rt::Bundle)] +#[bundle(name = "block")] +pub struct BlockEndowment { + #[bundle(role = role::DMA_POOL, index = 1)] + pub pool: SupplyToken, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BlockRequest { + Open(ProvisionRequest), + Status, + /// Revoke the sole live channel lineage after the client has quiesced it. + Close, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BlockRefusal { + Unavailable, + Policy, + Exhausted, + Lost, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BlockResponse { + /// Exactly two grants accompany this reply: device then DMA supply. + Opened { + channels: u16, + limit: u16, + }, + Status { + available: bool, + channels: u16, + limit: u16, + }, + Closed, + Refused(BlockRefusal), +} + +/// A single-controller provisioner. It retains the ordinary device and +/// physical-region ancestors and grants attenuated children; filesystem I/O +/// never re-enters this object. +pub struct BlockService { + device: Option, + pool: Option, + live: bool, + granted: u64, + revoked: u64, + note: u64, +} + +impl BlockService { + pub const LIMIT: u16 = 1; + + pub fn note(&self) -> u64 { + self.note + } + + fn available(&self) -> bool { + self.device.is_some() && self.pool.is_some() + } + + pub fn adopt_device(&mut self, device: DeviceToken) { + self.device = Some(device); + } +} + +impl SystemService for BlockService { + const NAME: &'static str = "block"; + /// An I/O worker: parked on completions for most of its life, its work + /// done in microseconds between parks. Keep it off the performance + /// cores; a completion wait cannot tell the difference. + const CORE_PREFERENCE: kore_abi::task::CorePreference = + kore_abi::task::CorePreference::Efficient; + type Endowment = BlockEndowment; + const DEVICES: &'static [kore_rt::service::DeviceRequest] = + &[kore_rt::service::DeviceRequest { + selector: DeviceSelector::Pci(pci_selector_for_contract::( + 0x01, + 0x08, + 0x02, + kore_rt::service::PciIdentitySelector::ANY, + // The declared FLOOR of the claim ladder, deliberately not a + // match constraint: the channel's owner realizes its own + // MSI-family route from the granted device + // (`InterruptPolicy::Optional`), so a controller on a + // machine without remapped-MSI delivery still matches and + // serves at the polled posture. The entry's claim asks for + // the interrupt-capable carrier FIRST + // ([`preferred_selector`]) and only then falls back to this + // row, so a mixed machine never pairs the filesystem with a + // polled controller while a remapped one sits idle. + PciClaimPosture::PollingFallback, + )), + required: false, + }]; + + /// The NVMe function this row claims is provisioned to the filesystem, + /// which binds and — on the way down — stops it (#809 §L.5). + const DEVICE_CLASSES: &'static [kore_rt::service::DeviceClass] = + &[kore_rt::service::DeviceClass::Delegated { + to: "kore-svc-dagfs", + }]; + + fn bind(endowment: BlockEndowment) -> Self { + Self { + device: None, + pool: Some(endowment.pool), + live: false, + granted: 0, + revoked: 0, + note: 0, + } + } + + fn serve(&mut self, request: BlockRequest, sv: &mut Serve<'_, BlockResponse>) { + match request { + BlockRequest::Status => sv.reply(BlockResponse::Status { + available: self.available(), + channels: u16::from(self.live), + limit: Self::LIMIT, + }), + BlockRequest::Open(request) => { + if !self.available() { + sv.reply(BlockResponse::Refused(BlockRefusal::Unavailable)); + return; + } + if self.live { + sv.reply(BlockResponse::Refused(BlockRefusal::Exhausted)); + return; + } + // Both completion postures are lawful policy: the channel's + // owner builds the data plane over the granted device, and + // whether its binding realizes an interrupt route is that + // owner's declaration, not this control plane's. + if request.requested_queue_depth == 0 || request.requested_buffer_bytes == 0 { + sv.reply(BlockResponse::Refused(BlockRefusal::Policy)); + return; + } + let device = self.device.as_ref().expect("availability checked"); + let pool = self.pool.as_ref().expect("availability checked"); + sv.grant( + device.slot(), + device.rights(), + GrantArg::Attenuate { + kind: kore_abi::cap::ResourceKind::PciDevice, + }, + ); + sv.grant( + pool.slot(), + pool.rights(), + GrantArg::Sublease { + offset: 0, + len: pool.len(), + }, + ); + self.live = true; + self.granted = self.granted.saturating_add(1); + sv.reply(BlockResponse::Opened { + channels: 1, + limit: Self::LIMIT, + }); + } + BlockRequest::Close => { + if !self.live { + sv.reply(BlockResponse::Refused(BlockRefusal::Lost)); + return; + } + // One root per kind and one channel maximum: retiring both + // ancestors cuts the exact provisioned bundle and all BAR, + // VM and IOMMU descendants beneath it. Reopening requires a + // fresh broker claim and supply, so this instance becomes + // honestly unavailable. + let device = self.device.take(); + let pool = self.pool.take(); + if let Some(device) = device { + let _ = verbs::revoke_slot(device.slot()); + } + if let Some(pool) = pool { + let _ = verbs::revoke_slot(pool.slot()); + } + self.live = false; + self.revoked = self.revoked.saturating_add(1); + sv.reply(BlockResponse::Closed); + } + } + } + + fn grants_dropped(&mut self, response: &mut BlockResponse, emitted: usize) { + if matches!(response, BlockResponse::Opened { .. }) { + debug_assert_eq!(emitted, 2); + self.live = false; + *response = BlockResponse::Refused(BlockRefusal::Unavailable); + } + } +} + +/// Run the founded provisioner's broker claim and serving loop on the +/// executor owned by `#[kore_rt::main]`. Startup is ordinary async work: this +/// helper must never construct another executor over the task's sole +/// doorbell. +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +pub async fn run(rt: kore_rt::entry::Rt) -> ! { + use kore_rt::entry::{ + RUNG_SERVICE_ENDOWMENT, RUNG_SERVICE_SERVE_SETUP, RUNG_SERVICE_START, refuse_with, + }; + use kore_rt::service::{Bundle, SystemService}; + + let Ok(endowment) = BlockEndowment::resolve(rt.bootstrap()) else { + refuse_with(&rt, RUNG_SERVICE_ENDOWMENT); + }; + let mut service = BlockService::bind(endowment); + let transport = kore_rt::entry::transport(); + let note = match claim_controller(transport, rt.bootstrap()).await { + Ok(device) => { + service.adopt_device(device); + 0 + } + // A diskless machine still founds and serves the honest unavailable + // posture; the readiness note preserves the broker's reason. + Err(ClaimFailure::Unavailable(note)) => note, + Err(ClaimFailure::Wiring) => refuse_with(&rt, RUNG_SERVICE_START), + }; + service.note = note; + + // Pair 0 serves BlockService; pair 1 is this task's client face onto the + // device broker. Observability is supplied through the ordinary monitor + // provider seam, not a second service-specific status ring. + kore_rt::entry::serve_forever(&rt, service, None, note).await; + refuse_with(&rt, RUNG_SERVICE_SERVE_SETUP) +} + +/// The claim ladder's first rung: the declared class row with the +/// remapped-MSI match constraint added. +/// +/// The filesystem paces its completions by the route it realizes from the +/// granted device, so when a machine offers several matching controllers +/// the one whose MSI path the platform already remapped is strictly +/// preferable — the swap row's constraint, tried first. The declared +/// selector remains the fallback floor, so an unremapped machine still +/// gets storage at the polled posture rather than none at all. +#[cfg_attr(not(all(target_arch = "x86_64", target_os = "none")), allow(dead_code))] +fn preferred_selector(selector: DeviceSelector) -> DeviceSelector { + match selector { + DeviceSelector::Pci(pci) => DeviceSelector::Pci(pci_selector_for_contract::( + pci.class, + pci.subclass, + pci.progif, + pci.identity, + PciClaimPosture::RouteCapable, + )), + other => other, + } +} + +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ClaimFailure { + Wiring, + Unavailable(u64), +} + +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +async fn claim_controller( + transport: &kore_rt::transport::TrapTransport, + boot: &kore_abi::task::Bootstrap<'_>, +) -> Result { + use kore_svc_device::DeviceRefusal; + use kore_svc_device::client::{Claimed, ClaimedDevice, FaceLayout, MultiBroker}; + + // Every broker face the founder stamped — one per live root complex + // (#572). @0/@1 are this service's own two outgoing grant channels; + // the broker-client receive endpoints stride from @2, deliberately + // disjoint — derived from the protocol's own pair and grant counts, + // exactly as the founder stamps them. + let mut broker = MultiBroker::open( + transport, + boot, + FaceLayout::for_row( + ::SERVE_PAIRS, + ::BOOT_REPLY_GRANTS, + ), + ) + .ok_or(ClaimFailure::Wiring)?; + let selector = BlockService::DEVICES[0].selector; + // Rung one: the interrupt-capable carrier, sought across EVERY live + // root before descending. `NoMatch` and `Taken` are both honest + // reasons to fall to the declared floor — either no remapped + // controller exists anywhere, or another claimant already carries it + // and a polled sibling is still better than no storage. + match broker.claim_first(preferred_selector(selector)).await { + Ok(Claimed { + device: ClaimedDevice::Pci(device), + .. + }) => return Ok(device), + // A non-PCI shape from a PCI ask is the same broker-contract + // anomaly the floor claim refuses; never abandon a claimed bundle + // to double-claim below it. + Ok(_) => return Err(ClaimFailure::Unavailable(3)), + Err(DeviceRefusal::NoMatch) | Err(DeviceRefusal::Taken) => {} + Err(_) => return Err(ClaimFailure::Wiring), + } + match broker.claim_first(selector).await { + Ok(Claimed { + device: ClaimedDevice::Pci(device), + .. + }) => Ok(device), + Ok(_) => Err(ClaimFailure::Unavailable(3)), + Err(DeviceRefusal::NoMatch) => Err(ClaimFailure::Unavailable(1)), + Err(DeviceRefusal::Taken) => Err(ClaimFailure::Unavailable(2)), + Err(_) => Err(ClaimFailure::Wiring), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use kore_rt::service::PciInterruptRequirement; + + #[test] + fn nvme_selector_demands_dma_but_not_an_interrupt_route() { + let DeviceSelector::Pci(selector) = BlockService::DEVICES[0].selector else { + panic!("block controller must be PCI") + }; + assert_eq!( + (selector.class, selector.subclass, selector.progif), + (1, 8, 2) + ); + assert!(selector.dma); + // The declared floor: matchable on machines without remapped MSI. + assert_eq!(selector.interrupts, PciInterruptRequirement::None); + } + + #[test] + fn the_claim_ladder_prefers_the_remapped_carrier_over_the_same_floor() { + let floor = BlockService::DEVICES[0].selector; + let DeviceSelector::Pci(preferred) = preferred_selector(floor) else { + panic!("a PCI floor must prefer a PCI carrier") + }; + let DeviceSelector::Pci(declared) = floor else { + panic!("block controller must be PCI") + }; + // Rung one adds exactly the remapped-MSI constraint and nothing else, + // so descending to the floor can only ever widen the match. + assert_eq!(preferred.interrupts, PciInterruptRequirement::MsiFamily); + assert_eq!( + ( + preferred.class, + preferred.subclass, + preferred.progif, + preferred.dma, + preferred.bars, + preferred.identity, + ), + ( + declared.class, + declared.subclass, + declared.progif, + declared.dma, + declared.bars, + declared.identity, + ) + ); + } +} diff --git a/native/services/block/src/wire.rs b/native/services/block/src/wire.rs new file mode 100644 index 0000000..6c544a6 --- /dev/null +++ b/native/services/block/src/wire.rs @@ -0,0 +1,210 @@ +//! Fail-closed block-provisioner codecs. + +use kore_abi::svc::{BlockRepWire, BlockReqWire}; +use kore_rt::svc::SvcProtocol; + +use crate::{ + BlockRefusal, BlockRequest, BlockResponse, BlockService, CompletionMode, ProvisionRequest, +}; + +pub fn encode_req(request: &BlockRequest) -> BlockReqWire { + let mut wire = BlockReqWire { + op: 0, + mode: 0, + writable: 0, + reserved: 0, + depth: 0.into(), + reserved2: 0.into(), + buffer_bytes: 0.into(), + reserved3: 0.into(), + }; + match request { + BlockRequest::Open(request) => { + wire.op = BlockReqWire::OP_OPEN; + wire.mode = match request.mode { + CompletionMode::Poll => 0, + CompletionMode::Interrupt => 1, + }; + wire.writable = request.writable as u8; + wire.depth = request.requested_queue_depth.into(); + wire.buffer_bytes = u32::try_from(request.requested_buffer_bytes) + .unwrap_or(u32::MAX) + .into(); + } + BlockRequest::Status => wire.op = BlockReqWire::OP_STATUS, + BlockRequest::Close => wire.op = 3, + } + wire +} + +pub fn decode_req(wire: &BlockReqWire) -> Option { + if wire.reserved != 0 + || wire.reserved2.get() != 0 + || wire.reserved3.get() != 0 + || wire.writable > 1 + || wire.mode > 1 + { + return None; + } + Some(match wire.op { + BlockReqWire::OP_OPEN => BlockRequest::Open(ProvisionRequest { + writable: wire.writable != 0, + mode: if wire.mode == 0 { + CompletionMode::Poll + } else { + CompletionMode::Interrupt + }, + requested_queue_depth: wire.depth.get(), + requested_buffer_bytes: wire.buffer_bytes.get() as usize, + }), + BlockReqWire::OP_STATUS + if wire.mode == 0 + && wire.writable == 0 + && wire.depth.get() == 0 + && wire.buffer_bytes.get() == 0 => + { + BlockRequest::Status + } + 3 if wire.mode == 0 + && wire.writable == 0 + && wire.depth.get() == 0 + && wire.buffer_bytes.get() == 0 => + { + BlockRequest::Close + } + _ => return None, + }) +} + +pub fn encode_rep(response: &BlockResponse, grants: u8) -> BlockRepWire { + let (tag, refusal, available, channels, limit) = match *response { + BlockResponse::Opened { channels, limit } => { + (BlockRepWire::TAG_OPENED, 0, true, channels, limit) + } + BlockResponse::Status { + available, + channels, + limit, + } => (BlockRepWire::TAG_STATUS, 0, available, channels, limit), + BlockResponse::Closed => (4, 0, false, 0, BlockService::LIMIT), + BlockResponse::Refused(refusal) => ( + BlockRepWire::TAG_REFUSED, + match refusal { + BlockRefusal::Unavailable => BlockRepWire::REFUSE_UNAVAILABLE, + BlockRefusal::Policy => BlockRepWire::REFUSE_POLICY, + BlockRefusal::Exhausted => BlockRepWire::REFUSE_EXHAUSTED, + BlockRefusal::Lost => BlockRepWire::REFUSE_LOST, + }, + false, + 0, + BlockService::LIMIT, + ), + }; + BlockRepWire { + tag, + refusal, + grants, + flags: available as u8, + block_size: 0.into(), + max_transfer_blocks: 0.into(), + atomic_write_blocks: 0.into(), + block_count: 0.into(), + channels: channels.into(), + limit: limit.into(), + reserved: 0.into(), + } +} + +pub fn decode_rep(wire: &BlockRepWire) -> Option<(BlockResponse, u8)> { + if wire.reserved.get() != 0 + || wire.block_size.get() != 0 + || wire.max_transfer_blocks.get() != 0 + || wire.atomic_write_blocks.get() != 0 + || wire.block_count.get() != 0 + || wire.flags > 1 + { + return None; + } + let response = match (wire.tag, wire.refusal, wire.grants) { + (BlockRepWire::TAG_OPENED, 0, 2) if wire.flags == 1 => BlockResponse::Opened { + channels: wire.channels.get(), + limit: wire.limit.get(), + }, + (BlockRepWire::TAG_STATUS, 0, 0) => BlockResponse::Status { + available: wire.flags == 1, + channels: wire.channels.get(), + limit: wire.limit.get(), + }, + (4, 0, 0) if wire.flags == 0 && wire.channels.get() == 0 => BlockResponse::Closed, + (BlockRepWire::TAG_REFUSED, refusal, 0) if wire.flags == 0 && wire.channels.get() == 0 => { + BlockResponse::Refused(match refusal { + BlockRepWire::REFUSE_UNAVAILABLE => BlockRefusal::Unavailable, + BlockRepWire::REFUSE_POLICY => BlockRefusal::Policy, + BlockRepWire::REFUSE_EXHAUSTED => BlockRefusal::Exhausted, + BlockRepWire::REFUSE_LOST => BlockRefusal::Lost, + _ => return None, + }) + } + _ => return None, + }; + Some((response, wire.grants)) +} + +impl SvcProtocol for BlockService { + type Request = BlockRequest; + type Response = BlockResponse; + type ReqWire = BlockReqWire; + type RepWire = BlockRepWire; + + const REPLY_GRANTS: usize = 2; + + fn retain_grant(_index: usize) -> bool { + true + } + + fn reply_matches(request: &BlockReqWire, response: &BlockResponse, grants: u8) -> bool { + matches!( + (decode_req(request), response, grants), + (Some(BlockRequest::Open(_)), BlockResponse::Opened { .. }, 2) + | (Some(BlockRequest::Open(_)), BlockResponse::Refused(_), 0) + | (Some(BlockRequest::Status), BlockResponse::Status { .. }, 0) + | (Some(BlockRequest::Close), BlockResponse::Closed, 0) + | (Some(BlockRequest::Close), BlockResponse::Refused(_), 0) + ) + } + + fn encode_req(request: &BlockRequest) -> BlockReqWire { + encode_req(request) + } + fn decode_req(wire: &BlockReqWire) -> Option { + decode_req(wire) + } + fn encode_rep(response: &BlockResponse, grants: u8) -> BlockRepWire { + encode_rep(response, grants) + } + fn decode_rep(wire: &BlockRepWire) -> Option<(BlockResponse, u8)> { + decode_rep(wire) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn open_round_trip_requires_exactly_two_grants() { + let request = BlockRequest::Open(ProvisionRequest { + writable: true, + mode: CompletionMode::Poll, + requested_queue_depth: 32, + requested_buffer_bytes: 0x20_0000, + }); + assert_eq!(decode_req(&encode_req(&request)), Some(request)); + let response = BlockResponse::Opened { + channels: 1, + limit: 1, + }; + assert_eq!(decode_rep(&encode_rep(&response, 2)), Some((response, 2))); + assert!(decode_rep(&encode_rep(&response, 1)).is_none()); + } +} diff --git a/native/services/dagfs/Cargo.toml b/native/services/dagfs/Cargo.toml new file mode 100644 index 0000000..6dbc27a --- /dev/null +++ b/native/services/dagfs/Cargo.toml @@ -0,0 +1,65 @@ +[package] +name = "kore-svc-dagfs" +version = "0.0.1" +edition = "2024" +description = "kore filesystem service: a capability-confined userland object graph with private task views, bounded generational route proofs, and a replaceable async StorageProvider seam." +publish = false + +# Phase 1 is the *pure engine*: namespace, authority, revisions, and the +# provider seam, proven by host tests without any service IPC (plan §9 +# Phase 1). It therefore carries no transport dependency at all — +# `kore-abi`/`kore-ring`/`kore-rt` join in Phase 2 with `wire.rs`, +# `service/`, and `client.rs`, which is when a frozen ABI record and a real +# founded pair first exist. Keeping them out until then is what makes the +# engine's test suite independent of the serving framework. +[dependencies] +# Design-bounded inline storage (kore-design-doctrine §4 tier (b)): every +# arena here has a capacity that is an architectural constant in `bounds`, +# and exhaustion is a typed refusal — never hidden growth. Same version +# the device broker already carries. +heapless = { version = "0.9.3", default-features = false } +# The filesystem's bounded object, directory, authority, and handle arenas use +# independently claimable generational slots. This is the same shared +# lock-free substrate available to every userland service, not an FS-local +# duplicate of the kernel slab. +kore-collections = { path = "../../crates/collections", default-features = false, features = ["alloc"] } +# Process-visible naming/refusal/capability leaves live outside the storage +# service so ordinary applications do not compile the NVMe/block/provider +# stack merely to hold a directory entry or projected-view capability. +kore-dagfs = { path = "../../crates/dagfs", features = ["runtime"] } +# Phase 2: the frozen filesystem request/reply records live beside every other +# service protocol in `kore-abi::svc`. Pure layouts — the codec that gives them +# meaning is `wire.rs`, and the engine below it still knows nothing of the wire. +kore-abi = { path = "../../crates/abi" } +# The little-endian field wrappers the frozen records are built from. No +# derive feature: this crate defines no wire layout of its own, it only reads +# and writes kore-abi's. +zerocopy = { version = "0.8", default-features = false } +# The `SystemService`/`SvcProtocol` contract and the pair's ring helpers. +kore-rt = { path = "../../crates/rt" } +kore-ring = { path = "../../crates/ring" } +# Immutable extraction-version identities use the same allocator-free digest +# as the parser-neutral contract, and results cross that contract without an +# FS-specific duplicate representation. +blake3 = { version = "=1.8.7", default-features = false, features = ["pure"] } +kore-svc-extract = { path = "../extract" } +kore-driver = { path = "../../crates/driver", features = ["service"] } +kore-nvme = { path = "../../drivers/nvme", features = ["alloc"] } +kore-svc-block = { path = "../block" } +crc = { version = "3.4.0", default-features = false } + +[dev-dependencies] +# The protocol suite reads records from adversary-shaped bytes, which needs the +# `FromBytes`/`IntoBytes` faces the library itself has no reason to name. +zerocopy = { version = "0.8", default-features = false, features = ["derive"] } +# The pair suite formats a real ring window with the tree's own helpers. +kore-ring = { path = "../../crates/ring" } +kore-rt = { path = "../../crates/rt", features = ["std"] } +kore-svc-extract-worker-rust = { path = "../extract-worker-rust" } +kore-svc-extract-worker-c = { path = "../extract-worker-c" } +# The witness suite commits the producer's own plain-text batch — the bytes a +# founded `find`, `write` or editor save hands this store — not a test copy. +kore-dagfs = { path = "../../crates/dagfs", features = ["index"] } +# The host volume tools in `examples/` read and write disk images through the +# driver's image-file device. +kore-driver = { path = "../../crates/driver", features = ["service", "image"] } diff --git a/native/services/dagfs/boot.toml b/native/services/dagfs/boot.toml new file mode 100644 index 0000000..fe9b8b5 --- /dev/null +++ b/native/services/dagfs/boot.toml @@ -0,0 +1,51 @@ +# services/dagfs/boot.toml — the filesystem service's boot-manifest row. +# Founding order lives in res/boot.toml; this fragment declares only the +# service-local image, bundle, and protocol citations. + +# The filesystem follows the terminal in the retained chain: it consumes a +# client face onto block, then leaves its own service pair resident before the +# exec and init rounds. Client multiplication belongs to the recyclable-session +# seam, not to a fixed bank of shell-specific boot pairs. +[service] +image = "kore-svc-dagfs" +chain = true +endowment = "kore_svc_dagfs::FsEndowment" +protocol = "kore_svc_dagfs::FsProtocol" +endows = [] +# The measured archive digest is immutable startup data, not a capability. +# The generated founder row stamps it as ARCHIVE_ERA@0..=7; FsEndowment +# resolves all eight words before the durable policy can run. +boot_data = "archive_digest" +# Pair 1 is exec's authority-bearing administration face. Command processes +# themselves arrive through the recyclable-peer seam; the fixed face count +# therefore stays independent of the installed command set. +exec_client = true +# The filesystem's volume-scaled metadata is heap-backed rather than inline: +# object and directory arenas, placement and extent indexes, the derived +# index, and the handle arena all live here now, so raising a bound is a heap +# number instead of a stack one. At MAX_OBJECTS = 512 the fixed startup +# reservations come to roughly 1.75 MiB PER MOUNTED ENGINE, of which the +# derived wire-chunk arena alone is 1 MiB — and a provisioned disk (the +# `kore-fs` + `kore-system` GPT pair every metal machine carries, and the +# `run-boot-gate-x86-fs-dual` fixture) mounts TWO engines, user and system, +# each with its own arenas, plus the recovery transients of both. 4 MiB fit +# the single engine a blank disk mounts and ran out on the second one: the +# `service heap undersized` expect in `ExtractionState::new` NACKed the row +# (rung 127, extraction.rs) on every provisioned boot — Gigatron and +# Biggertron, 2026-09-15, #1003 — while every blank-disk QEMU boot passed. +# 8 MiB is two engines' fixed reservations with better than 2× headroom for +# recovery transients and the heap's growth granularity; the fs-dual gate is +# the witness that keeps it honest (`kore_svc_dagfs::VOLUME_HEAP_BYTES`). +# +# Sessions come out of the same heap now. Each program that reads files holds +# a face, and its session's handles and authority edges are allocated per +# slot when it attaches, in both engines — about 5.7 KiB a slot — so the +# faces grow into every slot the serve host can name instead of stopping at +# six. `kore_svc_dagfs::HEAP_DEMAND_BYTES` derives the floor from the volume +# arenas, every session slot attached in both engines, and the host's peer +# table grown to every slot: 12.3 MiB today. exec's founder test +# `dagfs_heap_supply_holds_every_session` holds this row to it; the rest is +# room for the proofs open handles carry, allocated per open at their depth. +heap_supply_bytes = 0x00D0_0000 +digest_x86_64 = "blake3:2f00411e60e4c43e73c1290e225c8d1908f27e3ff36bf1d3be7709200bb5c454" +digest_aarch64 = "blake3:d8ce59acf3bdf5bb66912e3be2477c3a1de50c97614f66c83138f3fce1995680" diff --git a/native/services/dagfs/examples/fsdemo.rs b/native/services/dagfs/examples/fsdemo.rs new file mode 100644 index 0000000..67c9526 --- /dev/null +++ b/native/services/dagfs/examples/fsdemo.rs @@ -0,0 +1,862 @@ +//! A host-side interactive driver for the filesystem engine. +//! +//! Phase 1 is the engine only — there is no service pair, no founded client, +//! and no shell yet — so this is how a human explores the same behaviour the +//! test suites assert. It is a **development tool**, not part of the service: +//! it lives in `examples/`, uses `std`, and nothing in the crate depends on +//! it. The real thing arrives in Phase 4 as shell built-ins over the same +//! public API this file calls. +//! +//! Run it: +//! +//! ```text +//! cargo run -p kore-svc-dagfs --example fsdemo # interactive +//! cargo run -p kore-svc-dagfs --example fsdemo -- tour # scripted walkthrough +//! ``` +//! +//! Three sessions are attached at startup, each with its own private root, so +//! isolation and sharing are both demonstrable. `use 1` switches which client +//! you are speaking as — that is the whole point: the same path text means +//! different things to different sessions. +//! +//! Note what this file has to do that the engine deliberately does not: +//! **it walks paths itself**. The engine's vocabulary is single-component +//! (`open_at(parent, name)`), because a path is client-side input and not +//! authority. Every command here re-walks its path from a root each time, +//! which is exactly how a real client behaves. + +use std::io::{self, BufRead, Write}; + +use kore_svc_dagfs::{ + ANY_REVISION, AuthorityKind, FsError, GrantId, Handle, Name, ObjectKind, RamEngine, Rights, + SessionId, StorageProvider, block_on, +}; + +/// How many private-root clients the demo attaches. +const CLIENTS: u16 = 3; + +/// One client's view: which session it is, and the handle to its own root. +struct Client { + session: SessionId, + root: Handle, +} + +/// A grant this demo watched being issued. +/// +/// The service deliberately does not keep this list. `Engine::grants` reports +/// what a session **holds**, and a grant is held by its *recipient* — so the +/// issuer's copy of the `GrantId` is the issuer's own business, exactly as +/// plan §5's `Revoke(grant_handle)` implies. A real client would store it the +/// same way this demo does. +struct Issued { + by: u16, + to: u16, + grant: GrantId, + label: String, +} + +/// Where a resolution starts, and with what authority. +struct Anchor { + handle: Handle, + /// `true` if the caller must close it (an attachment root we opened). + owned: bool, + rights: Rights, +} + +fn main() { + let mut engine = Box::new(RamEngine::default()); + let mut clients = Vec::new(); + for index in 0..CLIENTS { + let session = SessionId::new(index); + block_on(engine.attach_session(session)).expect("attach session"); + let root = engine + .open_root(session, Rights::ALL) + .expect("open private root") + .handle; + clients.push(Client { session, root }); + } + + let mut issued: Vec = Vec::new(); + + if std::env::args().any(|arg| arg == "tour") { + tour(&mut engine, &clients, &mut issued); + return; + } + + println!("kore filesystem engine — interactive demo (RAM provider)"); + println!( + "{} clients attached, each with its own empty private root.", + CLIENTS + ); + println!("`help` for commands, `tour` for a guided walkthrough, `quit` to exit.\n"); + + let mut current = 0usize; + let stdin = io::stdin(); + loop { + print!("[session {current}] $ "); + let _ = io::stdout().flush(); + let mut line = String::new(); + match stdin.lock().read_line(&mut line) { + Ok(0) => break, + Ok(_) => {} + Err(err) => { + eprintln!("input error: {err}"); + break; + } + } + // Strip a UTF-8 BOM before trimming: PowerShell prepends one when a + // here-string is piped into a process, and it would otherwise glue + // itself to the first command word. + let line = line.trim_start_matches('\u{feff}').trim(); + if line.is_empty() { + continue; + } + if matches!(line, "quit" | "exit") { + break; + } + if let Some(rest) = line.strip_prefix("use") { + match rest.trim().parse::() { + Ok(index) if index < clients.len() => { + current = index; + println!("now speaking as session {index}"); + } + _ => println!("usage: use <0..{}>", clients.len() - 1), + } + continue; + } + if line == "tour" { + tour(&mut engine, &clients, &mut issued); + continue; + } + if let Err(err) = run(&mut engine, &clients, current, line, &mut issued) { + println!("refused: {err:?}"); + } + } +} + +/// Dispatch one command line. +fn run( + engine: &mut RamEngine, + clients: &[Client], + current: usize, + line: &str, + issued: &mut Vec, +) -> Result<(), FsError> { + let client = &clients[current]; + let mut parts = line.split_whitespace(); + let command = parts.next().unwrap_or(""); + // Everything after the command word, for commands taking free text. + let tail = line[command.len()..].trim(); + + match command { + "help" => { + print_help(); + Ok(()) + } + "ls" => { + let path = first_arg(tail); + let (handle, owned) = resolve(engine, client, path)?; + let result = list(engine, client, handle); + release(engine, client, handle, owned); + result + } + "stat" => { + let (handle, owned) = resolve(engine, client, first_arg(tail))?; + let result = engine.stat(client.session, handle); + release(engine, client, handle, owned); + let info = result?; + println!( + " {:?} len={} revision={} names={} grants={} object={:#x}", + info.kind, + info.len, + info.revision, + info.links, + info.grants, + info.object.to_bits() + ); + Ok(()) + } + "mkdir" => create_at(engine, client, tail, ObjectKind::Directory), + "touch" => create_at(engine, client, tail, ObjectKind::File), + "cat" => { + let (handle, owned) = resolve(engine, client, first_arg(tail))?; + let result = read_all(engine, client, handle); + release(engine, client, handle, owned); + match result { + Ok(bytes) => { + println!("{}", String::from_utf8_lossy(&bytes)); + Ok(()) + } + Err(err) => Err(err), + } + } + "write" | "append" => { + let (path, text) = split_two(tail); + if path.is_empty() { + println!("usage: {command} "); + return Ok(()); + } + // `write` behaves like a shell redirect and creates the file if it + // is not there yet — which is what plan §10.3's interactive + // sequence assumes. `append` requires an existing file. + let (handle, owned) = match resolve(engine, client, path) { + Ok(found) => found, + Err(FsError::NotFound) if command == "write" => { + let (parent, parent_owned, name) = resolve_parent(engine, client, path)?; + let made = + block_on(engine.create(client.session, parent, &name, ObjectKind::File)); + release(engine, client, parent, parent_owned); + (made?.handle, true) + } + Err(err) => return Err(err), + }; + let result = if command == "write" { + block_on(engine.write_at(client.session, handle, 0, text.as_bytes(), ANY_REVISION)) + .map(|revision| (0u64, revision)) + } else { + block_on(engine.append(client.session, handle, text.as_bytes(), ANY_REVISION)) + }; + release(engine, client, handle, owned); + let (offset, revision) = result?; + println!( + " {} bytes at offset {offset}; revision {revision}", + text.len() + ); + Ok(()) + } + "rm" => { + let path = first_arg(tail); + let (parent, owned, name) = resolve_parent(engine, client, path)?; + let result = block_on(engine.unlink(client.session, parent, &name)); + release(engine, client, parent, owned); + result + } + "ln" => { + let (src, dst) = split_two(tail); + if src.is_empty() || dst.is_empty() { + println!("usage: ln "); + return Ok(()); + } + let (file, file_owned) = resolve(engine, client, src)?; + let parent = resolve_parent(engine, client, dst); + let result = match parent { + Ok((parent, parent_owned, name)) => { + let outcome = engine.link(client.session, parent, &name, file); + release(engine, client, parent, parent_owned); + outcome + } + Err(err) => Err(err), + }; + release(engine, client, file, file_owned); + result + } + "mv" => { + let (src, dst) = split_two(tail); + if src.is_empty() || dst.is_empty() { + println!("usage: mv "); + return Ok(()); + } + let (src_parent, src_owned, src_name) = resolve_parent(engine, client, src)?; + let result = match resolve_parent(engine, client, dst) { + Ok((dst_parent, dst_owned, dst_name)) => { + let outcome = engine.relocate( + client.session, + src_parent, + &src_name, + dst_parent, + &dst_name, + ); + release(engine, client, dst_parent, dst_owned); + outcome + } + Err(err) => Err(err), + }; + release(engine, client, src_parent, src_owned); + result + } + "share" => { + // share [name] + let args: Vec<&str> = tail.split_whitespace().collect(); + if args.len() < 3 { + println!("usage: share [attachment-name]"); + return Ok(()); + } + let Ok(destination) = args[1].parse::() else { + println!("session must be a number"); + return Ok(()); + }; + let rights = match args[2] { + "ro" => Rights::READ, + "rw" => Rights::READ + .union(Rights::WRITE) + .union(Rights::MUTATE_NAMESPACE), + other => { + println!("unknown rights {other:?} — use `ro` or `rw`"); + return Ok(()); + } + }; + let display = match args.get(3) { + Some(text) => Some(Name::new(text)?), + None => None, + }; + let (handle, owned) = resolve(engine, client, args[0])?; + let result = engine.grant( + client.session, + handle, + SessionId::new(destination), + rights, + display, + ); + release(engine, client, handle, owned); + let grant = result?; + let label = display + .map(|n| format!("@{}", n.as_str())) + .unwrap_or_else(|| "".into()); + println!( + " granted to session {destination} as {label} ({})", + describe_rights(rights) + ); + // Keep the issuer's copy of the id — this is the only thing that + // lets it revoke later. + issued.push(Issued { + by: client.session.index(), + to: destination, + grant, + label, + }); + Ok(()) + } + "grants" => { + for (index, (_, description)) in routes(engine, client, issued).iter().enumerate() { + println!(" [{index}] {description}"); + } + println!(" reach an attachment with a path like `@name/file`"); + Ok(()) + } + "revoke" => { + let Ok(index) = first_arg(tail).parse::() else { + println!("usage: revoke "); + return Ok(()); + }; + let routes = routes(engine, client, issued); + let Some((grant, description)) = routes.get(index) else { + println!("no such index — run `grants`"); + return Ok(()); + }; + let grant = *grant; + println!(" retiring {description}"); + let outcome = block_on(engine.revoke(client.session, grant)); + if outcome.is_ok() { + issued.retain(|entry| entry.grant != grant); + } + outcome + } + "stats" => { + let limits = engine.provider().limits(); + println!(" objects {}", engine.object_count()); + println!(" directories {}", engine.directory_count()); + println!(" authority edges{:>3}", engine.authority_edge_count()); + println!( + " storage {} / {} bytes in {} placements", + engine.provider().bytes_in_use(), + limits.total_bytes, + engine.provider().placements() + ); + println!(" leaked {}", engine.leaked_placements()); + for client in 0..CLIENTS { + println!( + " session {client} handles {}", + engine.live_handles(SessionId::new(client)) + ); + } + Ok(()) + } + other => { + println!("unknown command {other:?} — try `help`"); + Ok(()) + } + } +} + +fn print_help() { + println!( + "\ + use speak as session n (each has its own private root) + ls [path] list a directory + stat object metadata, including its identity + mkdir create a directory + touch create an empty file + write overwrite a file from offset 0 + append append (offset chosen atomically) + cat read a whole file + ln a second name for the same file object + mv move: retires the source edge + rm remove one name + share [as] grant a file or subtree to session n + grants list this session's authority edges + revoke retire a grant + stats arena and storage accounting + tour scripted walkthrough of the guarantees + quit + + paths are client-side: `a/b/c` walks from your own root, `@name/x` walks + from an attachment someone granted you." + ); +} + +/// Every authority edge this session can act on, as `(id, description)`. +/// +/// Two sources, deliberately: the ones it **holds** come from the service +/// (`Engine::grants`), and the ones it **issued** come from the demo's own +/// registry, because the service does not track an issuer's copy. `revoke` +/// indexes this combined list, so either side of a grant can end it — which is +/// exactly the rule the engine enforces. +fn routes(engine: &RamEngine, client: &Client, issued: &[Issued]) -> Vec<(GrantId, String)> { + let mut out = Vec::new(); + for info in engine + .grants(client.session) + .expect("attached session grant snapshot") + .iter() + { + let label = match info.display_name { + Some(name) => format!("@{}", name.as_str()), + None => "".to_string(), + }; + let kind = match info.kind { + AuthorityKind::PrivateRoot => "hold: private root", + AuthorityKind::Grant => "hold: grant", + }; + out.push(( + info.grant, + format!("{kind:<19} {label:<14} {}", describe_rights(info.rights)), + )); + } + for entry in issued.iter().filter(|e| e.by == client.session.index()) { + out.push(( + entry.grant, + format!( + "{:<19} {:<14} to session {}", + "issued: grant", entry.label, entry.to + ), + )); + } + out +} + +// ── path resolution (client-side, by design) ──────────────────────────────── + +/// Where to start resolving `path`, and the components still to walk. +fn anchor<'a>( + engine: &mut RamEngine, + client: &Client, + path: &'a str, +) -> Result<(Anchor, Vec<&'a str>), FsError> { + if let Some(stripped) = path.strip_prefix('@') { + let mut split = stripped.splitn(2, '/'); + let wanted = split.next().unwrap_or(""); + let rest = split.next().unwrap_or(""); + let listed = engine.grants(client.session)?; + let found = listed.iter().find(|info| { + info.kind == AuthorityKind::Grant + && info + .display_name + .map(|n| n.as_str() == wanted) + .unwrap_or(false) + }); + let Some(info) = found else { + return Err(FsError::NotFound); + }; + let opened = engine.open_grant(client.session, info.grant, info.rights)?; + Ok(( + Anchor { + handle: opened.handle, + owned: true, + rights: info.rights, + }, + components(rest), + )) + } else { + Ok(( + Anchor { + handle: client.root, + owned: false, + rights: Rights::ALL, + }, + components(path), + )) + } +} + +fn components(path: &str) -> Vec<&str> { + path.split('/').filter(|part| !part.is_empty()).collect() +} + +/// Resolve `path` to a handle. The `bool` says whether the caller must close it. +fn resolve(engine: &mut RamEngine, client: &Client, path: &str) -> Result<(Handle, bool), FsError> { + let (mut at, parts) = anchor(engine, client, path)?; + for part in parts { + let name = match Name::new(part) { + Ok(name) => name, + Err(err) => { + release(engine, client, at.handle, at.owned); + return Err(err); + } + }; + match engine.open_at(client.session, at.handle, &name, at.rights) { + Ok(next) => { + release(engine, client, at.handle, at.owned); + at.handle = next.handle; + at.owned = true; + } + Err(err) => { + release(engine, client, at.handle, at.owned); + return Err(err); + } + } + } + Ok((at.handle, at.owned)) +} + +/// Resolve `path`'s parent directory plus its final component. +fn resolve_parent( + engine: &mut RamEngine, + client: &Client, + path: &str, +) -> Result<(Handle, bool, Name), FsError> { + let (at, mut parts) = anchor(engine, client, path)?; + let Some(last) = parts.pop() else { + release(engine, client, at.handle, at.owned); + return Err(FsError::InvalidName); + }; + let name = match Name::new(last) { + Ok(name) => name, + Err(err) => { + release(engine, client, at.handle, at.owned); + return Err(err); + } + }; + // Walk whatever is left of the path to reach the parent. + let mut current = at; + for part in parts { + let component = match Name::new(part) { + Ok(component) => component, + Err(err) => { + release(engine, client, current.handle, current.owned); + return Err(err); + } + }; + match engine.open_at(client.session, current.handle, &component, current.rights) { + Ok(next) => { + release(engine, client, current.handle, current.owned); + current.handle = next.handle; + current.owned = true; + } + Err(err) => { + release(engine, client, current.handle, current.owned); + return Err(err); + } + } + } + Ok((current.handle, current.owned, name)) +} + +/// Close a handle if we opened it. A session's own root handle is long-lived +/// and must survive; everything we walked through is ours to release. +fn release(engine: &mut RamEngine, client: &Client, handle: Handle, owned: bool) { + if owned { + let _ = engine.close(client.session, handle); + } +} + +// ── command bodies ────────────────────────────────────────────────────────── + +fn create_at( + engine: &mut RamEngine, + client: &Client, + path: &str, + kind: ObjectKind, +) -> Result<(), FsError> { + let path = first_arg(path); + let (parent, owned, name) = resolve_parent(engine, client, path)?; + let result = block_on(engine.create(client.session, parent, &name, kind)); + release(engine, client, parent, owned); + let made = result?; + // The creator is handed working authority immediately; this demo has no + // further use for it, so it closes it rather than leaking a handle. + let _ = engine.close(client.session, made.handle); + Ok(()) +} + +fn list(engine: &mut RamEngine, client: &Client, handle: Handle) -> Result<(), FsError> { + let mut cursor = 0u32; + let mut shown = 0usize; + loop { + let page = engine.list(client.session, handle, cursor)?; + for entry in page.entries.iter() { + let marker = if entry.kind == ObjectKind::Directory { + "/" + } else { + "" + }; + println!(" {}{}", entry.name.as_str(), marker); + shown += 1; + } + match page.next { + Some(next) => cursor = next, + None => break, + } + } + if shown == 0 { + println!(" (empty)"); + } + Ok(()) +} + +fn read_all(engine: &mut RamEngine, client: &Client, handle: Handle) -> Result, FsError> { + let len = engine.stat(client.session, handle)?.len as usize; + let mut buf = vec![0u8; len]; + let read = block_on(engine.read(client.session, handle, 0, &mut buf))?; + buf.truncate(read); + Ok(buf) +} + +fn describe_rights(rights: Rights) -> String { + let mut parts = Vec::new(); + for (bit, label) in [ + (Rights::READ, "read"), + (Rights::WRITE, "write"), + (Rights::MUTATE_NAMESPACE, "namespace"), + (Rights::SHARE, "share"), + ] { + if rights.contains(bit) { + parts.push(label); + } + } + if parts.is_empty() { + "none".to_string() + } else { + parts.join("+") + } +} + +fn first_arg(tail: &str) -> &str { + tail.split_whitespace().next().unwrap_or("") +} + +/// Split into the first whitespace-delimited word and the rest, verbatim. +fn split_two(tail: &str) -> (&str, &str) { + match tail.split_once(char::is_whitespace) { + Some((head, rest)) => (head, rest.trim_start()), + None => (tail, ""), + } +} + +// ── the guided walkthrough ────────────────────────────────────────────────── + +/// Run through the properties the MVP exists to demonstrate, printing what +/// happened at each step. +/// +/// This is the same ground the test suites cover, arranged to be *read* rather +/// than asserted — each step says what it did and what the engine answered. +fn tour(engine: &mut RamEngine, clients: &[Client], issued: &mut Vec) { + let a = &clients[0]; + let b = &clients[1]; + + heading("1. every session starts with its own private root"); + for client in clients.iter() { + let info = engine.stat(client.session, client.root).unwrap(); + println!( + " session {} root: object {:#x}, {} entries", + client.session.index(), + info.object.to_bits(), + info.links + ); + } + println!(" → no machine-wide `/`; three distinct root objects"); + + heading("2. the same path text means different objects to different clients"); + run_quiet(engine, clients, issued, 0, "touch notes"); + run_quiet(engine, clients, issued, 1, "touch notes"); + show(engine, clients, issued, 0, "write notes belongs to A"); + show(engine, clients, issued, 1, "write notes belongs to B"); + let (handle_a, owned_a) = resolve(engine, a, "notes").unwrap(); + let (handle_b, owned_b) = resolve(engine, b, "notes").unwrap(); + let id_a = engine.stat(a.session, handle_a).unwrap().object; + let id_b = engine.stat(b.session, handle_b).unwrap().object; + println!(" A's `notes` = object {:#x}", id_a.to_bits()); + println!(" B's `notes` = object {:#x}", id_b.to_bits()); + println!( + " A reads {:?}", + String::from_utf8_lossy(&read_all(engine, a, handle_a).unwrap()) + ); + println!( + " B reads {:?}", + String::from_utf8_lossy(&read_all(engine, b, handle_b).unwrap()) + ); + println!(" → same name, different objects: {}", id_a != id_b); + release(engine, a, handle_a, owned_a); + release(engine, b, handle_b, owned_b); + + heading("3. one file can carry two names, without being copied"); + run_quiet(engine, clients, issued, 0, "mkdir projects"); + run_quiet(engine, clients, issued, 0, "touch projects/main.rs"); + show( + engine, + clients, + issued, + 0, + "write projects/main.rs fn main() {}", + ); + show( + engine, + clients, + issued, + 0, + "ln projects/main.rs second-name", + ); + let (via_first, o1) = resolve(engine, a, "projects/main.rs").unwrap(); + let (via_second, o2) = resolve(engine, a, "second-name").unwrap(); + let first = engine.stat(a.session, via_first).unwrap(); + let second = engine.stat(a.session, via_second).unwrap(); + println!( + " projects/main.rs -> object {:#x} ({} names)", + first.object.to_bits(), + first.links + ); + println!( + " second-name -> object {:#x} ({} names)", + second.object.to_bits(), + second.links + ); + println!(" → one object, two containment edges"); + release(engine, a, via_first, o1); + release(engine, a, via_second, o2); + + heading("4. removing one name leaves the other, and the bytes"); + show(engine, clients, issued, 0, "rm projects/main.rs"); + show(engine, clients, issued, 0, "cat second-name"); + println!(" → unlink removed one edge, not the object"); + + heading("5. sharing is explicit, and read-only means read-only"); + show(engine, clients, issued, 0, "share second-name 1 ro gift"); + show(engine, clients, issued, 1, "grants"); + show(engine, clients, issued, 1, "cat @gift"); + print!(" B tries to write through the read-only grant: "); + report(engine, clients, issued, 1, "write @gift overwritten"); + print!(" B tries to re-share it: "); + report(engine, clients, issued, 1, "share @gift 2 ro passed-on"); + print!(" B tries to link it into its own root: "); + report(engine, clients, issued, 1, "ln @gift stolen"); + println!(" → the last one is refused because a containment edge carries no"); + println!(" rights: naming it under B's own root would relabel it with B's"); + println!(" full authority, laundering read-only into read-write."); + + heading("6. losing the route ends access immediately"); + let (b_view, b_owned) = resolve(engine, b, "@gift").unwrap(); + println!( + " B holds a live handle; reading now gives {:?}", + String::from_utf8_lossy(&read_all(engine, b, b_view).unwrap()) + ); + show(engine, clients, issued, 0, "revoke 1"); + print!(" after A revokes, B's *existing* handle reads: "); + match read_all(engine, b, b_view) { + Ok(bytes) => println!("{:?} (unexpected!)", String::from_utf8_lossy(&bytes)), + Err(err) => println!("{err:?}"), + } + println!(" → no cache to expire: the handle's proof is re-walked every call"); + release(engine, b, b_view, b_owned); + + heading("7. a second live route still works"); + show(engine, clients, issued, 0, "cat second-name"); + println!(" → A never lost anything; only B's route was retired"); + + heading("8. writes are revision-checked"); + let (file, owned) = resolve(engine, a, "second-name").unwrap(); + let revision = engine.stat(a.session, file).unwrap().revision; + println!(" current revision is {revision}"); + let stale = revision; + let fresh = block_on(engine.write_at(a.session, file, 0, b"first change", ANY_REVISION)) + .expect("unconditional write"); + println!(" an unconditional write committed, revision now {fresh}"); + match block_on(engine.write_at(a.session, file, 0, b"second change", stale)) { + Ok(_) => println!(" a write against the stale revision succeeded (unexpected!)"), + Err(err) => println!(" a write against revision {stale} was refused: {err:?}"), + } + println!(" → explicit conflict detection, not last-writer-wins by accident"); + release(engine, a, file, owned); + + heading("9. append chooses its offset atomically"); + run_quiet(engine, clients, issued, 0, "touch log"); + for _ in 0..3 { + let (log, owned) = resolve(engine, a, "log").unwrap(); + let (offset, _) = block_on(engine.append(a.session, log, b"entry ", ANY_REVISION)).unwrap(); + println!(" append landed at offset {offset}"); + release(engine, a, log, owned); + } + show(engine, clients, issued, 0, "cat log"); + println!(" → each append got its own space; none overwrote another"); + + heading("10. reclamation returns storage when the last route goes"); + let before = engine.provider().bytes_in_use(); + println!(" storage in use: {before} bytes"); + show(engine, clients, issued, 0, "rm second-name"); + show(engine, clients, issued, 0, "rm log"); + show(engine, clients, issued, 0, "rm notes"); + show(engine, clients, issued, 0, "rm projects"); + println!( + " after removing A's files: {} bytes in {} placements, {} leaked", + engine.provider().bytes_in_use(), + engine.provider().placements(), + engine.leaked_placements() + ); + println!(" the remainder is session 1's own `notes`, which nobody removed —"); + println!(" reclamation is per-object and never reached into B's tree."); + println!(" → last edge and last grant gone ⇒ object and its bytes released"); + + println!("\nTour complete. `help` to explore by hand.\n"); +} + +fn heading(text: &str) { + println!("\n\x1b[1m── {text}\x1b[0m"); +} + +/// Run a command, printing the command but discarding ordinary output. +fn run_quiet( + engine: &mut RamEngine, + clients: &[Client], + issued: &mut Vec, + session: usize, + line: &str, +) { + if let Err(err) = run(engine, clients, session, line, issued) { + println!(" [{session}] {line} -> refused: {err:?}"); + } +} + +/// Run a command, echoing it first. +fn show( + engine: &mut RamEngine, + clients: &[Client], + issued: &mut Vec, + session: usize, + line: &str, +) { + println!(" [session {session}] $ {line}"); + if let Err(err) = run(engine, clients, session, line, issued) { + println!(" refused: {err:?}"); + } +} + +/// Run a command expecting it to be refused, printing only the outcome. +fn report( + engine: &mut RamEngine, + clients: &[Client], + issued: &mut Vec, + session: usize, + line: &str, +) { + match run(engine, clients, session, line, issued) { + Ok(()) => println!("allowed (unexpected!)"), + Err(err) => println!("{err:?}"), + } +} diff --git a/native/services/dagfs/examples/fswire.rs b/native/services/dagfs/examples/fswire.rs new file mode 100644 index 0000000..930d583 --- /dev/null +++ b/native/services/dagfs/examples/fswire.rs @@ -0,0 +1,702 @@ +//! A host-side driver for the filesystem **protocol** — everything crosses the +//! frozen wire records as bytes. +//! +//! Where `fsdemo` calls the engine directly, this one goes +//! client → `FsReqWire` → codec → dispatch → `FsRepWire` → client for every +//! command, over a real data window. It exists so the protocol path can be +//! driven by hand and inspected, rather than only asserted in tests. +//! +//! ```text +//! cargo run -p kore-svc-dagfs --example fswire # interactive +//! cargo run -p kore-svc-dagfs --example fswire -- demo # scripted +//! ``` +//! +//! `bytes ` shows the 120-byte request record a command encodes to and +//! proves it decodes back unchanged — the quickest way to see that no address +//! and no capability rides the wire. + +use std::{ + cell::RefCell, + io::{self, BufRead, Write}, + rc::Rc, +}; + +use kore_abi::svc::{FsRepWire, FsReqWire}; +use kore_svc_dagfs::{ + ANY_REVISION, DataWindow, FsClient, FsError, FsRequest, FsResponse, FsResult, Handle, Name, + ObjectKind, RamService, Rights, SessionId, Transport, block_on, + wire::{decode_req, encode_rep, encode_req}, +}; + +const CLIENTS: u16 = 3; +const WINDOW_BYTES: usize = 4096; + +type Shared = Rc>; + +/// The client's end of a pair. `window` is the client's own memory; the service +/// sees it only through the view handed to `dispatch`. +struct Wire { + service: Shared, + session: SessionId, + window: Vec, + /// Round trips this client has made — visible in `stats`. + calls: usize, +} + +impl Transport for Wire { + fn exchange(&mut self, request: &FsReqWire) -> FsResult { + self.calls += 1; + let Some(decoded) = decode_req(request) else { + return Ok(encode_rep(&FsResponse::Refused(FsError::Unsupported), 0)); + }; + let mut window = DataWindow::new(&mut self.window); + let response = + self.service + .borrow_mut() + .dispatch(self.session, &decoded, Some(&mut window)); + Ok(encode_rep(&response, 0)) + } +} + +/// A grant this driver watched being issued. +/// +/// The service tracks only what a session *holds*, and a grant is held by its +/// recipient — so the issuer's copy of the id is the issuer's own business, +/// exactly as `Revoke(grant_handle)` implies. A real client stores it the same +/// way. +struct Issued { + by: usize, + to: u16, + grant: kore_svc_dagfs::GrantId, + label: String, +} + +fn name(text: &str) -> FsResult { + Name::new(text) +} + +fn main() { + let service: Shared = Rc::new(RefCell::new(RamService::default())); + let mut clients = Vec::new(); + for index in 0..CLIENTS { + let session = SessionId::new(index); + block_on(service.borrow_mut().engine().attach_session(session)).expect("attach"); + let mut client = FsClient::new(Wire { + service: Rc::clone(&service), + session, + window: vec![0u8; WINDOW_BYTES], + calls: 0, + }); + client + .attach_window(index as u32 + 1, WINDOW_BYTES as u32) + .expect("attach window"); + clients.push(client); + } + + let mut issued: Vec = Vec::new(); + + if std::env::args().any(|a| a == "demo") { + demo(&mut clients, &mut issued); + return; + } + + println!("kore filesystem — protocol driver (every command crosses the wire)"); + println!("{CLIENTS} clients, each with a {WINDOW_BYTES}-byte data window."); + println!("`help` for commands, `quit` to exit.\n"); + + let mut current = 0usize; + let stdin = io::stdin(); + loop { + print!("[wire {current}] $ "); + let _ = io::stdout().flush(); + let mut line = String::new(); + match stdin.lock().read_line(&mut line) { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + let line = line.trim_start_matches('\u{feff}').trim(); + if line.is_empty() { + continue; + } + if matches!(line, "quit" | "exit") { + break; + } + if let Some(rest) = line.strip_prefix("use ") { + match rest.trim().parse::() { + Ok(i) if i < clients.len() => { + current = i; + println!("now session {i}"); + } + _ => println!("usage: use <0..{}>", clients.len() - 1), + } + continue; + } + if let Err(err) = run(&mut clients, current, line, &mut issued) { + println!("refused: {err:?}"); + } + } +} + +fn run( + clients: &mut [FsClient], + current: usize, + line: &str, + issued: &mut Vec, +) -> FsResult<()> { + let (command, tail) = split(line); + match command { + "help" => { + println!( + "\ + use speak as session n + ls [path] list a directory (entries cross in the data window) + stat metadata, including the object id + mkdir create a directory + touch create an empty file + write overwrite from offset 0 + append append (offset chosen by the service) + cat read the whole file back through the window + ln a second name for one object + mv move: retires the source edge + rm remove one name + share [as] grant to session n + grants list this session's authority edges + revoke retire a grant (issuer or holder) + bytes show the encoded request record for a command + stats round trips, storage, arenas + quit" + ); + Ok(()) + } + "bytes" => show_bytes(clients, current, tail), + "ls" => with_resolved(clients, current, first(tail), |clients, dir| { + let client = &mut clients[current]; + let mut cursor = 0u32; + let mut shown = 0; + loop { + let page = client.list(dir, cursor, 0)?; + let window = client.transport().window.clone(); + for i in 0..page.entries as usize { + let e = FsClient::::read_entry(&window, 0, i)?; + let mark = if e.kind == ObjectKind::Directory { + "/" + } else { + "" + }; + println!(" {}{}", e.name.as_str(), mark); + shown += 1; + } + match page.next { + Some(next) => cursor = next, + None => break, + } + } + if shown == 0 { + println!(" (empty)"); + } + Ok(()) + }), + "stat" => with_resolved(clients, current, first(tail), |clients, h| { + let s = clients[current].stat(h)?; + println!( + " {:?} len={} revision={} names={} grants={} object={:#x}", + s.kind, + s.len, + s.revision, + s.links, + s.grants, + s.object.to_bits() + ); + Ok(()) + }), + "mkdir" => create(clients, current, first(tail), ObjectKind::Directory), + "touch" => create(clients, current, first(tail), ObjectKind::File), + "cat" => with_resolved(clients, current, first(tail), |clients, h| { + let client = &mut clients[current]; + let len = client.stat(h)?.len as u32; + let count = client.read(h, 0, 0, len)?; + let bytes = client.transport().window[..count as usize].to_vec(); + println!("{}", String::from_utf8_lossy(&bytes)); + Ok(()) + }), + "write" | "append" => { + let (path, text) = split(tail); + if path.is_empty() { + println!("usage: {command} "); + return Ok(()); + } + let bytes = text.as_bytes(); + if bytes.len() > WINDOW_BYTES { + return Err(FsError::Range); + } + let append = command == "append"; + // `write` on a name that does not exist creates it, then retries. + // Retry rather than probe-then-write: `put` has already returned + // its handle by the time this matches, so the fallback costs no + // handles and the common case stays one walk. + match put(clients, current, path, bytes, append) { + Err(FsError::NotFound) if !append => { + create(clients, current, path, ObjectKind::File)?; + put(clients, current, path, bytes, append) + } + other => other, + } + } + "rm" => with_parent(clients, current, first(tail), |clients, parent, leaf| { + clients[current].unlink(parent, &leaf) + }), + "ln" | "mv" => { + let (src, dst) = split(tail); + if src.is_empty() || dst.is_empty() { + println!("usage: {command} "); + return Ok(()); + } + // Nested guards: `mv` holds three handles at once (both parents + // plus the walk's), and every one is released on the way out. + with_parent(clients, current, dst, |clients, dst_parent, dst_name| { + if command == "ln" { + with_resolved(clients, current, src, |clients, file| { + clients[current].link(dst_parent, &dst_name, file) + }) + } else { + with_parent(clients, current, src, |clients, src_parent, src_name| { + clients[current].relocate(src_parent, &src_name, dst_parent, &dst_name) + }) + } + }) + } + "share" => { + let args: Vec<&str> = tail.split_whitespace().collect(); + if args.len() < 3 { + println!("usage: share [name]"); + return Ok(()); + } + let Ok(dest) = args[1].parse::() else { + println!("session must be a number"); + return Ok(()); + }; + let rights = match args[2] { + "ro" => Rights::READ, + "rw" => Rights::READ + .union(Rights::WRITE) + .union(Rights::MUTATE_NAMESPACE), + other => { + println!("unknown rights {other:?}"); + return Ok(()); + } + }; + let display = match args.get(3) { + Some(t) => Some(name(t)?), + None => None, + }; + with_resolved(clients, current, args[0], |clients, h| { + let grant = clients[current].grant(h, SessionId::new(dest), rights, display)?; + let label = display + .map(|n| format!("@{}", n.as_str())) + .unwrap_or_else(|| "".into()); + println!( + " granted to session {dest} as {label} ({})", + describe(rights) + ); + issued.push(Issued { + by: current, + to: dest, + grant, + label, + }); + Ok(()) + }) + } + "grants" => { + let client = &mut clients[current]; + let count = client.list_grants(0)?; + let window = client.transport().window.clone(); + for i in 0..count as usize { + let e = FsClient::::read_grant_entry(&window, 0, i)?; + let label = e + .display_name + .map(|n| format!("@{}", n.as_str())) + .unwrap_or_else(|| "".into()); + let kind = if e.is_private_root { + "private root" + } else { + "grant" + }; + println!(" [{i}] {kind:<13} {label:<14} {}", describe(e.rights)); + } + let held = count as usize; + for (offset, e) in issued.iter().filter(|e| e.by == current).enumerate() { + println!( + " [{}] {:<13} {:<14} issued to session {}", + held + offset, + "grant", + e.label, + e.to + ); + } + println!(" reach an attachment with `@name/file`"); + Ok(()) + } + "revoke" => { + let Ok(index) = first(tail).parse::() else { + println!("usage: revoke "); + return Ok(()); + }; + let client = &mut clients[current]; + let count = client.list_grants(0)?; + let window = client.transport().window.clone(); + let grant = if index < count as usize { + FsClient::::read_grant_entry(&window, 0, index)?.grant + } else { + // Above the held range, it names one this session issued. + let mine: Vec<&Issued> = issued.iter().filter(|e| e.by == current).collect(); + match mine.get(index - count as usize) { + Some(e) => e.grant, + None => { + println!("no such index"); + return Ok(()); + } + } + }; + client.revoke(grant)?; + issued.retain(|e| e.grant != grant); + Ok(()) + } + "stats" => { + for (i, client) in clients.iter_mut().enumerate() { + println!(" session {i}: {} round trips", client.transport().calls); + } + println!(" request record {} bytes", size_of::()); + println!(" reply record {} bytes", size_of::()); + Ok(()) + } + other => { + println!("unknown command {other:?} — try `help`"); + Ok(()) + } + } +} + +/// Encode a command's request and show the bytes, then decode it back. +fn show_bytes(clients: &mut [FsClient], current: usize, tail: &str) -> FsResult<()> { + use zerocopy::IntoBytes; + let (command, rest) = split(tail); + // A representative request per verb, built without touching the service. + // The handles this representative request borrowed, returned once the + // encoding has been shown. `bytes` only INSPECTS a record, so it must cost + // the session nothing — before this it silently spent a handle per call. + let (request, held): (FsRequest, Vec) = match command { + "ls" => { + let handle = resolve(clients, current, first(rest))?; + ( + FsRequest::List { + handle, + cursor: 0, + window_offset: 0, + }, + vec![handle], + ) + } + "stat" => { + let handle = resolve(clients, current, first(rest))?; + (FsRequest::Stat { handle }, vec![handle]) + } + "cat" => { + let handle = resolve(clients, current, first(rest))?; + ( + FsRequest::Read { + handle, + offset: 0, + window_offset: 0, + length: 64, + }, + vec![handle], + ) + } + "touch" => { + let (parent, leaf) = resolve_parent(clients, current, first(rest))?; + ( + FsRequest::Create { + parent, + name: leaf, + kind: ObjectKind::File, + subtype: kore_svc_dagfs::ObjectSubtype::PLAIN, + }, + vec![parent], + ) + } + _ => { + println!("usage: bytes "); + return Ok(()); + } + }; + let wire = encode_req(&request); + let bytes = wire.as_bytes(); + println!(" {request:?}"); + for (row, chunk) in bytes.chunks(16).enumerate() { + let hex: Vec = chunk.iter().map(|b| format!("{b:02x}")).collect(); + println!(" {:04x} {}", row * 16, hex.join(" ")); + } + match decode_req(&wire) { + Some(back) if back == request => println!(" decodes back unchanged ✓"), + Some(_) => println!(" DECODED TO SOMETHING ELSE ✗"), + None => println!(" REFUSED ITS OWN ENCODING ✗"), + } + for handle in held { + shut(clients, current, handle); + } + Ok(()) +} + +// ── client-side path walking (the engine takes one component at a time) ───── + +/// Where a resolution starts, and **with what authority** — the second half is +/// what this driver used to throw away. +/// +/// Every component was opened with `Rights::ALL`, which made any path inside a +/// read-only grant unreachable: the grant root opens at the grant's own rights, +/// so `stat @g` and `ls @g` worked, but `cat @g/f` demanded full rights one +/// step in and the engine — correctly — refused. Attenuation was +/// indistinguishable from a bug. +/// +/// The fix is `fsdemo`'s: carry the anchor's rights and walk with them +/// unchanged. A private root yields `Rights::ALL`; a grant yields exactly what +/// was delegated, so the walk can never out-run the authority it started from, +/// and a refusal is the engine's real answer rather than this driver over-asking. +struct Anchor { + handle: Handle, + rights: Rights, +} + +fn anchor(clients: &mut [FsClient], current: usize, path: &str) -> FsResult { + if let Some(rest) = path.strip_prefix('@') { + let wanted = rest.split('/').next().unwrap_or(""); + let client = &mut clients[current]; + let count = client.list_grants(0)?; + let window = client.transport().window.clone(); + for i in 0..count as usize { + let e = FsClient::::read_grant_entry(&window, 0, i)?; + if !e.is_private_root + && e.display_name + .map(|n| n.as_str() == wanted) + .unwrap_or(false) + { + // The grant's OWN rights, which then bound the whole walk. + return Ok(Anchor { + handle: client.open_grant(e.grant, e.rights)?.handle, + rights: e.rights, + }); + } + } + return Err(FsError::NotFound); + } + Ok(Anchor { + handle: clients[current].open_root(Rights::ALL)?.handle, + rights: Rights::ALL, + }) +} + +fn components(path: &str) -> Vec<&str> { + let path = path.strip_prefix('@').map_or(path, |r| { + r.split_once('/').map(|(_, rest)| rest).unwrap_or("") + }); + path.split('/').filter(|p| !p.is_empty()).collect() +} + +/// Hand a handle back, swallowing the outcome. +/// +/// Every handle this driver opens must be returned, including on the error +/// paths — a leak is permanent, and `MAX_HANDLES_PER_SESSION` is 16, so a +/// session that leaks a few per command wedges on `OutOfHandles` after a +/// handful of lines. This driver used to do exactly that: a `Close` request +/// was never sent on any path, so `mkdir d` + `touch d/f` + three appends +/// exhausted session 0 and every later command refused. +/// +/// The result is dropped deliberately: a failed close tells a development +/// tool nothing it can act on, and the caller's original error is the one +/// worth reporting. +fn shut(clients: &mut [FsClient], current: usize, handle: Handle) { + let _ = clients[current].close(handle); +} + +/// Resolve `path`, lend the handle to `f`, and close it either way. +/// +/// A scope guard rather than a close at the end of each arm, because the +/// interesting outcomes here leave early with `?`: `AccessDenied`, +/// `NotFound`, `Conflict`. Those are precisely the paths a protocol driver is +/// used to explore, so they are the paths that must not leak. +fn with_resolved( + clients: &mut [FsClient], + current: usize, + path: &str, + f: impl FnOnce(&mut [FsClient], Handle) -> FsResult, +) -> FsResult { + let handle = resolve(clients, current, path)?; + let out = f(clients, handle); + shut(clients, current, handle); + out +} + +/// [`with_resolved`]'s parent form, for the verbs that name a leaf. +fn with_parent( + clients: &mut [FsClient], + current: usize, + path: &str, + f: impl FnOnce(&mut [FsClient], Handle, Name) -> FsResult, +) -> FsResult { + let (parent, leaf) = resolve_parent(clients, current, path)?; + let out = f(clients, parent, leaf); + shut(clients, current, parent); + out +} + +/// Walk `parts` from `at`, closing each handle stepped OFF. +/// +/// Dropping the parent is safe: a handle is authority over one object, and the +/// engine re-walks a handle's proof on every call, so a child handle does not +/// depend on its parent's staying open. Both exits close the handle in hand, +/// so no path through here leaks. +/// Every component opens at the ANCHOR's rights, unchanged — `fsdemo`'s rule. +fn walk( + clients: &mut [FsClient], + current: usize, + at: Anchor, + parts: &[&str], +) -> FsResult { + let Anchor { handle, rights } = at; + let mut at = handle; + for &part in parts { + let n = match name(part) { + Ok(n) => n, + Err(e) => { + shut(clients, current, at); + return Err(e); + } + }; + match clients[current].open_at(at, &n, rights) { + Ok(opened) => { + shut(clients, current, at); + at = opened.handle; + } + Err(e) => { + shut(clients, current, at); + return Err(e); + } + } + } + Ok(at) +} + +fn resolve(clients: &mut [FsClient], current: usize, path: &str) -> FsResult { + let at = anchor(clients, current, path)?; + walk(clients, current, at, &components(path)) +} + +fn resolve_parent( + clients: &mut [FsClient], + current: usize, + path: &str, +) -> FsResult<(Handle, Name)> { + let mut parts = components(path); + let last = parts.pop().ok_or(FsError::InvalidName)?; + let leaf = name(last)?; + let at = anchor(clients, current, path)?; + let parent = walk(clients, current, at, &parts)?; + Ok((parent, leaf)) +} + +fn create( + clients: &mut [FsClient], + current: usize, + path: &str, + kind: ObjectKind, +) -> FsResult<()> { + with_parent(clients, current, path, |clients, parent, leaf| { + // `create` HANDS BACK a handle to the new object. Dropping it leaked + // one per `mkdir`/`touch` on top of the walk's own leaks. + let opened = clients[current].create(parent, &leaf, kind)?; + shut(clients, current, opened.handle); + Ok(()) + }) +} + +/// `write`/`append`'s shared body — named so the create-then-retry path below +/// reuses it instead of duplicating the request. +fn put( + clients: &mut [FsClient], + current: usize, + path: &str, + bytes: &[u8], + append: bool, +) -> FsResult<()> { + with_resolved(clients, current, path, |clients, h| { + let client = &mut clients[current]; + client.transport().window[..bytes.len()].copy_from_slice(bytes); + if append { + let (off, rev) = client.append(h, 0, bytes.len() as u32, ANY_REVISION)?; + println!(" {} bytes at {off}; revision {rev}", bytes.len()); + } else { + let rev = client.write_at(h, 0, 0, bytes.len() as u32, ANY_REVISION)?; + println!(" {} bytes at 0; revision {rev}", bytes.len()); + } + Ok(()) + }) +} + +fn describe(rights: Rights) -> String { + let mut parts = Vec::new(); + for (bit, label) in [ + (Rights::READ, "read"), + (Rights::WRITE, "write"), + (Rights::MUTATE_NAMESPACE, "namespace"), + (Rights::SHARE, "share"), + ] { + if rights.contains(bit) { + parts.push(label); + } + } + if parts.is_empty() { + "none".into() + } else { + parts.join("+") + } +} + +fn first(tail: &str) -> &str { + tail.split_whitespace().next().unwrap_or("") +} + +fn split(line: &str) -> (&str, &str) { + match line.split_once(char::is_whitespace) { + Some((head, rest)) => (head, rest.trim_start()), + None => (line, ""), + } +} + +/// A scripted run showing the protocol path end to end. +fn demo(clients: &mut [FsClient], issued: &mut Vec) { + let script = [ + (0usize, "touch hello"), + (0, "write hello kore over the wire"), + (0, "cat hello"), + (0, "stat hello"), + (0, "bytes stat hello"), + (0, "ln hello second-name"), + (0, "ls"), + (0, "share hello 1 ro inbox"), + (0, "grants"), + (1, "grants"), + (1, "cat @inbox"), + (1, "write @inbox nope"), + (0, "revoke 1"), + (1, "cat @inbox"), + (0, "stats"), + ]; + for (session, line) in script { + println!("\n[session {session}] $ {line}"); + if let Err(err) = run(clients, session, line, issued) { + println!("refused: {err:?}"); + } + } + println!("\nEvery line above crossed the frozen 120-byte request record.\n"); +} diff --git a/native/services/dagfs/examples/model_volume.rs b/native/services/dagfs/examples/model_volume.rs new file mode 100644 index 0000000..f88ed5b --- /dev/null +++ b/native/services/dagfs/examples/model_volume.rs @@ -0,0 +1,205 @@ +//! Host-only builder/verifier. Uses the real DAGFS provider and durable codec +//! over the image-file device (`kore_driver::block::image`), which never +//! opens a block device; destination creation is exclusive. +use kore_driver::block::{ + BlockChannel, CompletionMode, + image::{Access, ImageFile}, +}; +use kore_svc_dagfs::{ + ANY_REVISION, Name, ObjectKind, Rights, SYSTEM_WORKSPACE_DIR, SessionId, block_on, + face_home_label, persistence::PersistentFs, provider::BlockProvider, +}; +use std::{ + fs::{File, OpenOptions}, + io::Read, + path::Path, +}; + +/// The logical block size of the disks this example's volumes are spliced +/// into: QEMU's. +const BLOCK_SIZE: u32 = 512; + +fn err(e: impl std::fmt::Debug) -> String { + format!("{e:?}") +} +type Volume = PersistentFs; +fn provider(path: &Path, writable: bool) -> Result, String> { + let access = if writable { + Access::ReadWrite + } else { + Access::ReadOnly + }; + let image = ImageFile::open(path, access, BLOCK_SIZE).map_err(err)?; + BlockProvider::new(BlockChannel::new(image, CompletionMode::Poll).map_err(err)?).map_err(err) +} +/// `work/face-0/models/`: terminal 0's home, where the model a +/// keytest's terminal opens lives. +fn components() -> [String; 4] { + let member = match std::env::var("MODEL_VOLUME_MEMBER").as_deref() { + Ok("witness.model") => "witness.model", + Ok("lfm25.model") | Err(_) => "lfm25.model", + _ => panic!("unsupported model member"), + }; + let face = face_home_label(0).expect("terminal 0 has a home"); + [ + SYSTEM_WORKSPACE_DIR.into(), + String::from_utf8_lossy(&face).into_owned(), + "models".into(), + member.into(), + ] +} +fn build(image: &Path, source: &Path, bytes: u64) -> Result<(), String> { + let file = OpenOptions::new() + .write(true) + .create_new(true) + .open(image) + .map_err(err)?; + if bytes < 1024 * 1024 || !bytes.is_multiple_of(u64::from(BLOCK_SIZE)) { + return Err("invalid volume size".into()); + } + file.set_len(bytes).map_err(err)?; + drop(file); + let session = SessionId::new(0); + let mut volume = block_on(Volume::format(provider(image, true)?, session)).map_err(err)?; + let engine = volume.engine(); + let mut at = engine.open_root(session, Rights::ALL).map_err(err)?.handle; + for component in &components()[..3] { + let next = block_on(engine.create( + session, + at, + &Name::new(component).map_err(err)?, + ObjectKind::Directory, + )) + .map_err(err)? + .handle; + engine.close(session, at).map_err(err)?; + at = next; + } + let handle = block_on(engine.create( + session, + at, + &Name::new(&components()[3]).map_err(err)?, + ObjectKind::File, + )) + .map_err(err)? + .handle; + let mut source = File::open(source).map_err(err)?; + let len = source.metadata().map_err(err)?.len(); + block_on(engine.resize(session, handle, len, ANY_REVISION)).map_err(err)?; + let mut buffer = vec![0; 64 * 1024]; + let mut offset = 0; + while offset < len { + let want = buffer.len().min((len - offset) as usize); + source.read_exact(&mut buffer[..want]).map_err(err)?; + block_on(engine.write_at(session, handle, offset, &buffer[..want], ANY_REVISION)) + .map_err(err)?; + offset += want as u64; + if offset % (256 << 20) == 0 { + eprintln!("imported {offset}/{len} bytes"); + } + } + engine.close(session, handle).map_err(err)?; + engine.close(session, at).map_err(err)?; + // Both roots must name the imported file, rather than a blank fallback. + block_on(volume.commit(session)).map_err(err)?; + block_on(volume.commit(session)).map_err(err)?; + Ok(()) +} +fn verify(image: &Path, source: &Path) -> Result<(), String> { + let session = SessionId::new(0); + let volume = block_on(Volume::recover(provider(image, false)?, session)).map_err(err)?; + if volume.status().used_fallback_root { + return Err("unexpected fallback root".into()); + } + let engine = volume.engine(); + let mut at = engine.open_root(session, Rights::READ).map_err(err)?.handle; + for component in components() { + let next = engine + .open_at( + session, + at, + &Name::new(&component).map_err(err)?, + Rights::READ, + ) + .map_err(err)? + .handle; + engine.close(session, at).map_err(err)?; + at = next; + } + let mut source = File::open(source).map_err(err)?; + let len = source.metadata().map_err(err)?.len(); + if engine.stat(session, at).map_err(err)?.len != len { + return Err("length mismatch".into()); + } + let mut expected = vec![0; 64 * 1024]; + let mut actual = vec![0; 64 * 1024]; + let mut offset = 0; + let mut digest = blake3::Hasher::new(); + while offset < len { + let n = block_on(engine.read(session, at, offset, &mut actual)).map_err(err)?; + if n == 0 { + return Err("short model read".into()); + } + source.read_exact(&mut expected[..n]).map_err(err)?; + if actual[..n] != expected[..n] { + return Err(format!("model mismatch at {offset}")); + } + digest.update(&actual[..n]); + offset += n as u64; + } + println!( + "verified: path={} bytes={len} generation={} blake3={}", + components().join("/"), + volume.status().generation, + digest.finalize() + ); + Ok(()) +} +fn probe_write(image: &Path) -> Result<(), String> { + let session = SessionId::new(0); + let mut volume = block_on(Volume::recover(provider(image, true)?, session)).map_err(err)?; + eprintln!("recovered: {:?}", volume.status()); + let engine = volume.engine(); + let root = engine.open_root(session, Rights::ALL).map_err(err)?.handle; + let file = block_on(engine.create( + session, + root, + &Name::new("write-probe").map_err(err)?, + ObjectKind::File, + )) + .map_err(err)?; + engine.close(session, file.handle).map_err(err)?; + engine.close(session, root).map_err(err)?; + block_on(volume.commit(session)).map_err(err)?; + println!("cold recovery write: PASS"); + Ok(()) +} +fn run() -> Result<(), String> { + let args: Vec<_> = std::env::args().collect(); + if args.len() < 4 { + return Err("model_volume build|verify|probe-write IMAGE SOURCE [VOLUME_BYTES]".into()); + } + let image = Path::new(&args[2]); + let source = Path::new(&args[3]); + match args[1].as_str() { + "build" if args.len() == 5 => { + build(image, source, args[4].parse().map_err(err)?)?; + verify(image, source) + } + "verify" if args.len() == 4 => verify(image, source), + "probe-write" if args.len() == 4 => probe_write(image), + _ => Err("invalid arguments".into()), + } +} +fn main() { + let result = std::thread::Builder::new() + .stack_size(16 << 20) + .spawn(run) + .unwrap() + .join() + .unwrap(); + if let Err(e) = result { + eprintln!("model_volume: {e}"); + std::process::exit(1); + } +} diff --git a/native/services/dagfs/examples/volume_read.rs b/native/services/dagfs/examples/volume_read.rs new file mode 100644 index 0000000..7980a2f --- /dev/null +++ b/native/services/dagfs/examples/volume_read.rs @@ -0,0 +1,211 @@ +//! Host-only reader: one directory or one file of an offline kore volume +//! image, read through the real DAGFS provider and durable codec. +//! +//! ```text +//! volume_read IMAGE ls PATH # the names in a directory, one per line +//! volume_read IMAGE cat PATH # a file's bytes, to stdout +//! ``` +//! +//! `PATH` is from the volume's root (terminal N's home is `work/face-N`). +//! The image is a raw copy (`qemu-img convert -O raw`) of a disk no machine +//! is running: it is opened read-only, recovered from its newest root, and +//! never written. A volume that mounts only from its fallback root is +//! refused, as is anything that is not a kore volume. The screenshot +//! keytest reads the PNGs a guest saved with it (`make +//! keytest-screenshot-x86`); `kore-vol` (MicroPerceptron/kore#1391) is the +//! general tool, reading and writing a terminal's home. +use kore_driver::block::{ + BlockBuffer, BlockChannel, BlockError, BlockGeometry, BlockIoHints, CompletionMode, + DeviceCommand, DeviceCompletion, DirectBlockDevice, +}; +use kore_svc_dagfs::{ + Name, Rights, SessionId, block_on, persistence::PersistentFs, provider::BlockProvider, +}; +use std::{ + collections::VecDeque, + fs::File, + io::Write, + os::unix::fs::FileExt, + path::Path, +}; + +const SECTOR: u64 = 512; + +struct Buffer(Vec); +impl BlockBuffer for Buffer { + fn len(&self) -> usize { + self.0.len() + } + fn bytes(&self) -> &[u8] { + &self.0 + } + fn bytes_mut(&mut self) -> &mut [u8] { + &mut self.0 + } +} + +/// A raw image as a read-only block device: writes, flushes and discards +/// are refused, so nothing here can change the volume. +struct Image { + file: File, + geometry: BlockGeometry, + ready: VecDeque, +} + +impl Image { + fn open(path: &Path) -> Result { + let file = File::open(path).map_err(err)?; + let metadata = file.metadata().map_err(err)?; + if !metadata.is_file() || metadata.len() % SECTOR != 0 { + return Err("a regular, sector-aligned raw image is required".into()); + } + let geometry = BlockGeometry { + block_size: SECTOR as u32, + block_count: metadata.len() / SECTOR, + max_transfer_blocks: 512, + atomic_write_blocks: 1, + flush: false, + discard: false, + read_only: true, + io_hints: BlockIoHints::NONE, + }; + geometry.validate().map_err(err)?; + Ok(Self { + file, + geometry, + ready: VecDeque::new(), + }) + } +} + +impl DirectBlockDevice for Image { + type Buffer = Buffer; + const INDEPENDENT_PROGRESS: bool = true; + fn geometry(&self) -> BlockGeometry { + self.geometry + } + fn allocate_buffer(&mut self, n: usize) -> Result { + if n > 256 * 1024 { + return Err(BlockError::TooLarge); + } + Ok(Buffer(vec![0; n])) + } + fn submit( + &mut self, + command: DeviceCommand, + buffer: Option<&mut Buffer>, + cookie: u64, + ) -> Result<(), BlockError> { + if self.ready.len() >= 32 { + return Err(BlockError::Backpressure); + } + let result = (|| { + let DeviceCommand::Read { + lba, + blocks, + buffer_offset, + } = command + else { + return Err(BlockError::ReadOnly); + }; + if lba + .checked_add(u64::from(blocks)) + .is_none_or(|end| end > self.geometry.block_count) + { + return Err(BlockError::Range); + } + let len = blocks as usize * SECTOR as usize; + let buffer = buffer.ok_or(BlockError::UnknownBuffer)?; + let bytes = buffer + .0 + .get_mut(buffer_offset..buffer_offset.checked_add(len).ok_or(BlockError::Range)?) + .ok_or(BlockError::Range)?; + self.file + .read_exact_at(bytes, lba * SECTOR) + .map_err(|_| BlockError::Media) + })(); + self.ready.push_back(DeviceCompletion { cookie, result }); + Ok(()) + } + fn poll(&mut self) -> Option { + self.ready.pop_front() + } + fn quiesce(&mut self) -> Result<(), BlockError> { + Ok(()) + } +} + +fn err(e: impl std::fmt::Debug) -> String { + format!("{e:?}") +} + +type Volume = PersistentFs; + +fn run() -> Result<(), String> { + let args: Vec = std::env::args().collect(); + let [_, image, verb, path] = args.as_slice() else { + return Err("usage: volume_read IMAGE ls|cat PATH".into()); + }; + let components: Vec<&str> = path.split('/').filter(|c| !c.is_empty()).collect(); + let channel = BlockChannel::new(Image::open(Path::new(image))?, CompletionMode::Poll).map_err(err)?; + let provider = BlockProvider::new(channel).map_err(err)?; + let session = SessionId::new(0); + let volume = block_on(Volume::recover(provider, session)).map_err(err)?; + if volume.status().used_fallback_root { + return Err("the volume mounts only from its fallback root".into()); + } + let engine = volume.engine(); + let mut at = engine.open_root(session, Rights::READ).map_err(err)?.handle; + for component in &components { + let next = engine + .open_at(session, at, &Name::new(component).map_err(err)?, Rights::READ) + .map_err(|e| format!("{path}: {component}: {e:?}"))? + .handle; + engine.close(session, at).map_err(err)?; + at = next; + } + let mut out = std::io::stdout().lock(); + match verb.as_str() { + "ls" => { + let mut cursor = Some(0); + while let Some(from) = cursor { + let page = engine.list(session, at, from).map_err(err)?; + for entry in &page.entries { + writeln!(out, "{}", entry.name.as_str()).map_err(err)?; + } + cursor = page.next; + } + } + "cat" => { + let len = engine.stat(session, at).map_err(err)?.len; + let mut buffer = vec![0; 64 * 1024]; + let mut offset = 0; + while offset < len { + let n = block_on(engine.read(session, at, offset, &mut buffer)).map_err(err)?; + if n == 0 { + return Err(format!("{path}: short read at {offset} of {len}")); + } + out.write_all(&buffer[..n]).map_err(err)?; + offset += n as u64; + } + } + other => return Err(format!("unknown verb {other:?}: ls or cat")), + } + out.flush().map_err(err)?; + engine.close(session, at).map_err(err) +} + +fn main() { + // Recovery's image scratch and the engine's tables are deep; the main + // thread's stack is not. + let result = std::thread::Builder::new() + .stack_size(16 << 20) + .spawn(run) + .unwrap() + .join() + .unwrap(); + if let Err(e) = result { + eprintln!("volume_read: {e}"); + std::process::exit(1); + } +} diff --git a/native/services/dagfs/src/authority.rs b/native/services/dagfs/src/authority.rs new file mode 100644 index 0000000..82b38a5 --- /dev/null +++ b/native/services/dagfs/src/authority.rs @@ -0,0 +1,2721 @@ +//! Views, grants, rights attenuation, and the bounded handle proof. +//! +//! This module is where "a handle is not authority" is made true. +//! +//! # The two authority edges +//! +//! A session reaches objects through two **authority edge** kinds +//! ([`AuthorityKind`]): +//! +//! - its **private root** — every session that receives filesystem access +//! starts with its own, so there is no machine-wide `/` and a displayed +//! `/` means "this client's root"; and +//! - zero or more **direct grants** to a file or a directory subtree, each +//! bound to the *destination* session. +//! +//! A [`GrantId`] is therefore not bearer authority: the edge records both the +//! session allowed to open it and the session that issued it, and both sides +//! are checked. Handing the bits to a third session buys nothing. +//! +//! # The bounded proof +//! +//! An [`OpenHandle`] records the session that opened it, the authority edge +//! it entered through, the containment chain it followed ([`EdgeProof`]), the +//! rights attenuated along that route, and — through the edges' own slot +//! generations — the generation of every edge involved. +//! [`Authority::revalidate`] re-walks that proof on **every** operation: +//! +//! ```text +//! grant.root ──edge₀──▶ o₁ ──edge₁──▶ o₂ … ──edgeₙ──▶ handle.object +//! ``` +//! +//! Each step must still exist, still hang beneath the previous object, and +//! still carry its recorded generation. If any link is gone the operation +//! returns [`FsError::AccessLost`] even though the client still holds the old +//! numeric handle. There is no cached decision to bypass — the walk is the +//! check, and it is O(depth) with depth bounded by +//! [`MAX_PATH_DEPTH`](crate::bounds::MAX_PATH_DEPTH). This gives immediate +//! loss of access without a global reachability scan. +//! +//! # Rights only narrow +//! +//! [`Rights`] is a small bit set with no "administrator" bypass. Every +//! transition — delegating a grant, opening a child, opening at reduced +//! rights — is an intersection with what the caller already holds, so +//! authority can only shrink. In particular [`Rights::SHARE`] is never +//! inherited implicitly: a grant carries it only if the issuer both held it +//! and asked for it. + +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, AtomicU64, Ordering}; + +use alloc::{boxed::Box, vec::Vec}; + +use crate::{ + ServiceAlloc, + bounds::{ + MAX_EDGES_PER_SESSION, MAX_HANDLES_PER_SESSION, MAX_LINEAGE_DEPTH, MAX_PATH_DEPTH, + MAX_SESSIONS, + }, + error::{FsError, FsResult}, + namespace::{EdgeId, Name, Namespace}, + object::ObjectId, + service_alloc, + slot::{AtomicSlotVec, Keyed, SlotAccess, SlotKey, SlotVecRead, SlotVecWrite, fs_access}, +}; + +/// The widest handle index: every session's handle slots, end to end. +/// +/// A session's handles live in an arena of its own (`SessionArena`), and a +/// handle's index names the session slot first and the arena slot second +/// (`slot × MAX_HANDLES_PER_SESSION + local`). That keeps one fact exact +/// which per-session tables would otherwise lose: session B presenting +/// session A's handle bits names A's slot, not a live slot of B's, so the +/// refusal is exact rather than a lookup that happens to miss. The record +/// still carries its owner, and the check reads both. +pub const MAX_HANDLE_INDEX: usize = MAX_SESSIONS * MAX_HANDLES_PER_SESSION; + +/// The widest grant index, laid out like [`MAX_HANDLE_INDEX`]: an edge is +/// stored with the session that holds it. +pub const MAX_GRANT_INDEX: usize = MAX_SESSIONS * MAX_EDGES_PER_SESSION; + +/// Ancestors a validated edge can have: a lineage is the edge itself and at +/// most this many parents ([`MAX_LINEAGE_DEPTH`]). +const MAX_ANCESTORS: usize = MAX_LINEAGE_DEPTH - 1; + +/// The session slot an arena-wide id names, and the id within that slot's +/// arena. `None` for an index past every session (`per_session` slots each). +const fn split(id: SlotKey, per_session: usize) -> Option<(usize, SlotKey)> { + let index = id.index() as usize; + let slot = index / per_session; + if slot >= MAX_SESSIONS { + return None; + } + Some(( + slot, + SlotKey::new((index % per_session) as u32, id.generation()), + )) +} + +/// The arena-wide id of `local` in session slot `slot` ([`split`]'s inverse). +const fn join(slot: usize, local: SlotKey, per_session: usize) -> SlotKey { + SlotKey::new( + (slot * per_session + local.index() as usize) as u32, + local.generation(), + ) +} + +pub use kore_dagfs::{AuthorityQuota, GrantId, Handle, Rights, SessionId}; + +/// Why an authority edge exists. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum AuthorityKind { + /// The session's own starting view. + PrivateRoot, + /// A direct grant from another session. + Grant, +} + +/// One authority edge. +#[derive(Debug)] +pub(crate) struct AuthorityEdge { + /// The session allowed to enter through this edge. + holder: SessionId, + /// The session that created it. A private root's issuer is its holder. + issuer: SessionId, + /// The authority edge this edge was attenuated from. A private root has + /// no parent; every delegated edge has exactly one. The generational key + /// is the revocation lineage: removing an ancestor can never be confused + /// with a later occupant of the same arena slot. + parent: Option, + /// Where entering lands. + root: ObjectId, + rights: Rights, + quota: AuthorityQuota, + kind: AuthorityKind, + /// The name the holder sees this attachment under, if it chose one. + display_name: Option, + /// Handles rooted at this exact authority generation. The atomic claim is + /// what makes a per-grant ceiling exact under concurrent opens; an arena + /// scan alone would let two callers observe the same free budget. + open_handles: AtomicU32, + /// In-flight quota reservations. They bridge the interval between a live + /// graph snapshot and publication without maintaining a duplicate usage + /// ledger: completed usage remains derived from the graph. + pending_objects: AtomicU32, + pending_edges: AtomicU32, + pending_bytes: AtomicU64, + /// Advances before one reservation leaves the pending set. A claimant + /// whose graph snapshot straddled another completed mutation retries + /// instead of admitting growth against stale usage. + mutation_epoch: AtomicU64, +} + +impl AuthorityEdge { + #[inline] + pub(crate) const fn holder(&self) -> SessionId { + self.holder + } + + #[inline] + pub(crate) const fn issuer(&self) -> SessionId { + self.issuer + } + + #[inline] + pub(crate) const fn parent(&self) -> Option { + self.parent + } + + #[inline] + pub(crate) const fn root(&self) -> ObjectId { + self.root + } + + #[inline] + pub(crate) const fn rights(&self) -> Rights { + self.rights + } + + #[inline] + pub(crate) const fn quota(&self) -> AuthorityQuota { + self.quota + } + + #[inline] + pub(crate) const fn kind(&self) -> AuthorityKind { + self.kind + } + + #[inline] + pub(crate) const fn display_name(&self) -> Option { + self.display_name + } +} + +/// The containment chain an open handle followed, from its authority edge's +/// root to the object itself. +/// +/// Empty when the handle *is* the authority edge's root — opening a private +/// root or a granted object directly. Bounded by +/// [`MAX_PATH_DEPTH`](crate::bounds::MAX_PATH_DEPTH), so a traversal that +/// would outrun the proof is refused rather than recorded incompletely: a +/// truncated proof would silently stop revalidating its upper edges. +/// +/// Stored at its own length on the service heap, not at the bound: a handle +/// three directories down holds three edges. Reserving the deepest legal +/// path in every handle made the proof nearly all of a session's storage +/// (`SessionArena`) — 512 of a handle record's 544 bytes — while almost +/// every path a client walks is shallow (`res/FILESYSTEM_SCALE.md`). +#[derive(Debug, Default)] +pub struct EdgeProof(Box<[EdgeId], ServiceAlloc>); + +impl EdgeProof { + /// The empty proof — the authority edge's own root. Allocates nothing. + pub fn root() -> Self { + Self::default() + } + + /// This proof extended by one edge: [`FsError::Range`] past the depth + /// bound, [`FsError::OutOfHandles`] when the heap cannot hold it. The + /// extended proof is a new allocation, so a refusal leaves this one + /// whole. + pub fn extend(&self, edge: EdgeId) -> FsResult { + let depth = self.0.len(); + if depth >= MAX_PATH_DEPTH { + return Err(FsError::Range); + } + let mut next = Box::try_new_uninit_slice_in(depth + 1, *service_alloc()) + .map_err(|_| FsError::OutOfHandles)?; + let (prefix, last) = next.split_at_mut(depth); + prefix.write_copy_of_slice(&self.0); + last[0].write(edge); + // SAFETY: the prefix and the one slot after it are every element. + Ok(Self(unsafe { next.assume_init() })) + } + + #[inline] + pub fn depth(&self) -> usize { + self.0.len() + } + + #[inline] + pub fn edges(&self) -> &[EdgeId] { + &self.0 + } +} + +/// One open handle. +/// +/// Deliberately does **not** cache the revision observed at open time. The +/// plan's data model sketches one, but the engine already returns the +/// revision from every open and every write, so the client holds it, and a +/// conditional write names the revision it expects in the request itself. A +/// second copy inside the service would be state that could only go stale +/// (`kore-design-doctrine` §4 — derive, don't store). +#[derive(Debug)] +pub(crate) struct OpenHandle { + session: SessionId, + object: ObjectId, + grant: GrantId, + proof: EdgeProof, + rights: Rights, +} + +impl OpenHandle { + #[inline] + pub(crate) const fn grant(&self) -> GrantId { + self.grant + } + + #[inline] + pub(crate) const fn proof(&self) -> &EdgeProof { + &self.proof + } +} + +/// Per-session bookkeeping. +struct Session { + /// Zero is vacant, `SESSION_TRANSITION` is unpublished attach ownership, + /// and every other value is the exact private-root grant generation. + root: AtomicU64, + /// Peer/session retirement leaves the exact root key published while + /// refusing new work. That lets a contended retirement drop its claims + /// and retry without losing the identity it still owns. + closing: AtomicBool, + /// Serializes only this session's task-visible top-level name set: direct + /// children of the private root plus named authority attachments. It is + /// deliberately per-session rather than a namespace lock; mutations in + /// unrelated views remain independent. + view_claimed: AtomicBool, + /// This slot's storage ([`SessionArena`]): null until the slot first + /// attaches, then the same arena for every later occupant. + arena: AtomicPtr, +} + +const SESSION_VACANT: u64 = 0; +const SESSION_TRANSITION: u64 = u64::MAX; + +impl Session { + const fn vacant() -> Self { + Self { + root: AtomicU64::new(SESSION_VACANT), + closing: AtomicBool::new(false), + view_claimed: AtomicBool::new(false), + arena: AtomicPtr::new(core::ptr::null_mut()), + } + } +} + +/// One session slot's authority storage: the handles its occupant opens and +/// the authority edges it holds, each in an arena of its own. +/// +/// Allocated from the service heap the first time the slot attaches, and kept +/// until the [`Authority`] drops: a read or write guard borrows a slot inside +/// it and may cross an `await`, so the storage never moves and is never freed +/// under one. The slot's next occupant reuses it, and each arena slot's +/// generation keeps counting, so a handle or grant left over from an earlier +/// occupant fails closed. The serve host reuses the lowest free slot, so the +/// arenas follow the most sessions ever attached at once, not +/// [`MAX_SESSIONS`]. +/// +/// The arenas are the per-session ceilings themselves: a session that has +/// filled its [`MAX_HANDLES_PER_SESSION`] handles or its +/// [`MAX_EDGES_PER_SESSION`] edges is refused, and no other session's +/// storage is touched. +struct SessionArena { + handles: AtomicSlotVec, + edges: AtomicSlotVec, +} + +impl SessionArena { + /// The heap bytes one attached slot costs: both arenas' slots and the + /// arena record. What a service's heap must hold for every session it + /// can serve at once. The proofs its handles carry are allocated per + /// open, at their own depth: at most [`MAX_HANDLES_PER_SESSION`] × + /// [`MAX_PATH_DEPTH`] edges a session, apart from this. + pub(crate) const BYTES: usize = MAX_HANDLES_PER_SESSION + * AtomicSlotVec::::SLOT_BYTES + + MAX_EDGES_PER_SESSION * AtomicSlotVec::::SLOT_BYTES + + core::mem::size_of::(); + + fn try_new() -> Option> { + let handles = AtomicSlotVec::try_new_in(MAX_HANDLES_PER_SESSION, service_alloc()).ok()?; + let edges = AtomicSlotVec::try_new_in(MAX_EDGES_PER_SESSION, service_alloc()).ok()?; + Box::try_new_in(Self { handles, edges }, service_alloc()).ok() + } +} + +/// The heap bytes one engine's authority can come to with every session +/// attached at once (`SessionArena::BYTES`, about 5.7 KiB, per slot). +pub const SESSION_HEAP_BYTES: usize = MAX_SESSIONS * SessionArena::BYTES; + +/// One affine claim over a session's combined native-view root names. +pub(crate) struct ViewClaim<'a> { + session: &'a Session, +} + +impl Drop for ViewClaim<'_> { + fn drop(&mut self) { + self.session.view_claimed.store(false, Ordering::Release); + } +} + +/// A validated route: what an operation may do, and to what. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Route { + pub(crate) object: ObjectId, + pub(crate) rights: Rights, + pub(crate) grant: GrantId, + pub(crate) quota: AuthorityQuota, +} + +/// One validated edge plus every exact ancestor generation that authorizes it. +/// +/// Retaining the whole bounded lineage closes the validate→publish gap: an +/// ancestor retirement cannot claim its root while a descendant grant, handle, +/// quota reservation, or request is still being established. +pub(crate) struct ValidatedEdge<'a> { + leaf: SlotVecRead<'a, AuthorityEdge, ServiceAlloc>, + ancestors: heapless::Vec, MAX_ANCESTORS>, +} + +impl ValidatedEdge<'_> { + /// Edges in this lineage: the edge itself and every parent above it. + fn lineage_len(&self) -> usize { + self.ancestors.len() + 1 + } +} + +impl core::ops::Deref for ValidatedEdge<'_> { + type Target = AuthorityEdge; + + fn deref(&self) -> &Self::Target { + &self.leaf + } +} + +/// The authority arena: one record per session, each owning the storage of +/// the handles it opens and the edges it holds ([`SessionArena`]). +/// +/// There is no machine-wide handle or edge table, so nothing here is sized +/// by a guess at how many sessions run at once: [`MAX_SESSIONS`] is the +/// serve host's slot space, and storage is allocated per slot as slots are +/// first used. The session records themselves are a few words each, inline. +pub(crate) struct Authority { + sessions: [Session; MAX_SESSIONS], +} + +/// The object roots a retirement released, one per retired edge, for the +/// engine's grant-count and reclamation fold. Heap-backed: one retirement can +/// reach every edge descending from the retired one, across sessions. +pub(crate) type Retired = Vec; + +/// A heap list could not be grown for a closure being retired. +/// +/// The closure is the retiring session's own edges and every edge derived +/// from them, wherever it is held, so it has no per-session bound; staging it +/// on the heap is what lets any number of sessions descend from one task +/// view. The refusal is retryable: nothing has been removed yet, and every +/// claim taken so far is republished on the way out. +fn unstaged(_: E) -> FsError { + FsError::OutOfEdges +} + +struct ClaimedEdge<'a> { + grant: GrantId, + edge: SlotVecWrite<'a, AuthorityEdge, ServiceAlloc>, + /// Claimed ancestors above this edge within the closure, set when the + /// closure is ordered for retirement. + depth: usize, +} + +/// A claimed derivation closure, kept sorted by grant id so membership is a +/// binary search: the closure can span every session descending from a task +/// view, so the rescans that collect and order it must not be quadratic. +struct EdgeClaims<'a>(Vec, &'static ServiceAlloc>); + +impl<'a> EdgeClaims<'a> { + fn new() -> Self { + Self(Vec::new_in(service_alloc())) + } + + fn len(&self) -> usize { + self.0.len() + } + + fn contains(&self, grant: GrantId) -> bool { + self.position_within(grant, self.0.len()).is_some() + } + + /// Where `grant` sits among the first `sorted` claims, which must be in + /// grant id order. + fn position_within(&self, grant: GrantId, sorted: usize) -> Option { + self.0[..sorted] + .binary_search_by_key(&grant.key(), |claim| claim.grant.key()) + .ok() + } + + fn sort(&mut self) { + self.0.sort_unstable_by_key(|claim| claim.grant.key()); + } + + /// Order the closure for retirement, children first, before anything is + /// removed. + /// + /// An edge's depth in the closure is the number of claimed edges above + /// it on its parent chain; a child is one deeper than its parent, so + /// removing the deepest first never leaves a live edge whose parent is + /// gone. One sort, rather than a rescan for a leaf before every removal. + /// A chain longer than any lineage ([`MAX_LINEAGE_DEPTH`]) could only be + /// a cycle, and fails closed here, while every claim is still held and + /// nothing has been retired. + fn into_retirement(mut self) -> FsResult> { + for index in 0..self.0.len() { + let mut depth = 0; + let mut cursor = self.0[index].edge.parent(); + while let Some(parent) = cursor + && let Some(at) = self.position_within(parent, self.0.len()) + { + depth += 1; + if depth > MAX_ANCESTORS { + return Err(FsError::AccessLost); + } + cursor = self.0[at].edge.parent(); + } + self.0[index].depth = depth; + } + self.0 + .sort_unstable_by(|left, right| right.depth.cmp(&left.depth)); + Ok(Retirement(self.0)) + } +} + +/// A claimed closure in retirement order: deepest first. +struct Retirement<'a>(Vec, &'static ServiceAlloc>); + +impl Retirement<'_> { + fn len(&self) -> usize { + self.0.len() + } + + /// Remove every claimed edge in order, pushing each one's root. + /// `retired` must already have room for [`len`](Self::len) more roots, + /// so a heap that cannot grow cannot stop a retirement halfway. + fn retire_into(self, retired: &mut Retired) -> FsResult<()> { + if retired.capacity() - retired.len() < self.0.len() { + return Err(FsError::OutOfEdges); + } + for claim in self.0 { + let edge = claim.edge.remove().map_err(|_| FsError::AccessLost)?; + retired.push(edge.root()); + } + Ok(()) + } +} + +/// One affine reservation against an authority edge's growth budget. +/// +/// The edge read guard pins this exact grant generation (and, through +/// child-first retirement, its ancestry) for the operation. Dropping the +/// claim advances the mutation epoch before releasing pending usage, so a +/// concurrent claimant can never combine an old graph snapshot with a newly +/// freed reservation and over-admit. +pub(crate) struct QuotaClaim<'a> { + edge: ValidatedEdge<'a>, + objects: u32, + edges: u32, + bytes: u64, +} + +impl Drop for QuotaClaim<'_> { + fn drop(&mut self) { + self.edge.mutation_epoch.fetch_add(1, Ordering::AcqRel); + if self.objects != 0 { + self.edge + .pending_objects + .fetch_sub(self.objects, Ordering::AcqRel); + } + if self.edges != 0 { + self.edge + .pending_edges + .fetch_sub(self.edges, Ordering::AcqRel); + } + if self.bytes != 0 { + self.edge + .pending_bytes + .fetch_sub(self.bytes, Ordering::AcqRel); + } + } +} + +fn reserve_u32(counter: &AtomicU32, live: u32, delta: u32, limit: u32) -> bool { + counter + .try_update(Ordering::AcqRel, Ordering::Acquire, |pending| { + live.checked_add(pending)? + .checked_add(delta) + .filter(|total| *total <= limit) + .map(|_| pending + delta) + }) + .is_ok() +} + +fn reserve_u64(counter: &AtomicU64, live: u64, delta: u64, limit: u64) -> bool { + counter + .try_update(Ordering::AcqRel, Ordering::Acquire, |pending| { + live.checked_add(pending)? + .checked_add(delta) + .filter(|total| *total <= limit) + .map(|_| pending + delta) + }) + .is_ok() +} + +impl Authority { + pub(crate) fn new() -> Self { + Self { + sessions: [const { Session::vacant() }; MAX_SESSIONS], + } + } + + // ── session storage ───────────────────────────────────────────────────── + + /// A session slot's storage, once the slot has attached. + fn arena(&self, slot: usize) -> Option<&SessionArena> { + let arena = self.sessions.get(slot)?.arena.load(Ordering::Acquire); + // SAFETY: a non-null pointer was published once, with release + // ordering, from a live allocation (`attach_session`), and is freed + // only by `Drop`, whose `&mut self` excludes every borrow of `self`. + unsafe { arena.as_ref() } + } + + /// The arena holding `grant`'s edge, and the edge's id inside it. A + /// grant naming no session, or a slot that never attached, is stale. + fn edge_slot( + &self, + grant: GrantId, + ) -> Result<(&AtomicSlotVec, SlotKey), SlotAccess> { + let (slot, local) = split(grant.key(), MAX_EDGES_PER_SESSION).ok_or(SlotAccess::Stale)?; + let arena = self.arena(slot).ok_or(SlotAccess::Stale)?; + Ok((&arena.edges, local)) + } + + /// Every attached slot's edge arena with its slot: the closure walks. + fn edge_arenas( + &self, + ) -> impl Iterator)> { + (0..MAX_SESSIONS).filter_map(|slot| Some((slot, &self.arena(slot)?.edges))) + } + + /// Store an edge with its holder, which must hold its slot's attach + /// ownership or its published root. A holder whose edges are all live is + /// refused [`FsError::OutOfEdges`]. + fn insert_edge(&self, edge: AuthorityEdge) -> FsResult { + let slot = self.session_slot(edge.holder)?; + let arena = self.arena(slot).ok_or(FsError::InvalidHandle)?; + let local = arena.edges.insert(edge).map_err(|_| FsError::OutOfEdges)?; + Ok(GrantId::from_key(join(slot, local, MAX_EDGES_PER_SESSION))) + } + + // ── sessions ──────────────────────────────────────────────────────────── + + /// Attach a session, giving it a private root at `root` with full rights + /// over it. + /// + /// Refuses [`FsError::AlreadyExists`] for a session that is already + /// attached: silently replacing its root would strand every handle and + /// grant it holds. The slot's first attach allocates its storage + /// ([`SessionArena`]); a heap that cannot hold it refuses + /// [`FsError::OutOfHandles`] and leaves the slot vacant, so the launch + /// that asked reads as a full table. + pub(crate) fn attach_session(&self, session: SessionId) -> FsResult { + let slot = self.session_slot(session)?; + if self.sessions[slot].closing.load(Ordering::Acquire) { + return Err(FsError::Busy); + } + match self.sessions[slot].root.compare_exchange( + SESSION_VACANT, + SESSION_TRANSITION, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => {} + Err(SESSION_TRANSITION) => return Err(FsError::Busy), + Err(_) => return Err(FsError::AlreadyExists), + } + // The transition is this caller's alone, so it is the only one that + // can find the slot without storage and publish it. + if self.sessions[slot].arena.load(Ordering::Acquire).is_null() { + let Some(arena) = SessionArena::try_new() else { + self.sessions[slot] + .root + .store(SESSION_VACANT, Ordering::Release); + return Err(FsError::OutOfHandles); + }; + let (arena, _) = Box::into_raw_with_allocator(arena); + self.sessions[slot].arena.store(arena, Ordering::Release); + } + Ok(slot) + } + + /// Record a freshly attached session's private-root edge. + pub(crate) fn install_root(&self, session: SessionId, root: ObjectId) -> FsResult { + let slot = self.session_slot(session)?; + if self.sessions[slot].root.load(Ordering::Acquire) != SESSION_TRANSITION { + return Err(FsError::InvalidHandle); + } + let grant = self.insert_edge(AuthorityEdge { + holder: session, + issuer: session, + parent: None, + root, + rights: Rights::ALL, + quota: AuthorityQuota::MAX, + kind: AuthorityKind::PrivateRoot, + display_name: None, + open_handles: AtomicU32::new(0), + pending_objects: AtomicU32::new(0), + pending_edges: AtomicU32::new(0), + pending_bytes: AtomicU64::new(0), + mutation_epoch: AtomicU64::new(0), + })?; + self.sessions[slot].closing.store(false, Ordering::Release); + self.sessions[slot] + .root + .store(grant.to_bits(), Ordering::Release); + Ok(grant) + } + + /// Install an attenuated descendant of `source` as an already-claimed + /// session's structural root — the invocation-view posture: the session + /// enters the filesystem *through a delegation* of an existing view + /// instead of receiving a fresh ambient private namespace. The source + /// generation remains the revocation ancestor, rights can only narrow, + /// and no object is created. + /// + /// Like [`install_root`](Self::install_root), this publishes the root of + /// a session parked in its attach transition; the two are the only exits + /// from that state besides [`abort_attach`](Self::abort_attach). + pub(crate) fn install_delegated_root( + &self, + session: SessionId, + source: GrantId, + rights: Rights, + ) -> FsResult { + let slot = self.session_slot(session)?; + if self.sessions[slot].root.load(Ordering::Acquire) != SESSION_TRANSITION { + return Err(FsError::AlreadyExists); + } + let (delegated, quota, root, issuer) = { + let source_edge = self.validate_lineage(source)?; + if source_edge.lineage_len() >= MAX_LINEAGE_DEPTH { + return Err(FsError::Range); + } + ( + source_edge.rights().attenuate(rights)?, + source_edge.quota(), + source_edge.root(), + source_edge.holder(), + ) + }; + // A closing issuer cannot extend a lineage while its peer teardown + // is claiming that same bounded derivation closure. + self.root_grant(issuer)?; + let grant = self.insert_edge(AuthorityEdge { + holder: session, + issuer, + parent: Some(source), + root, + rights: delegated, + quota, + kind: AuthorityKind::PrivateRoot, + display_name: None, + open_handles: AtomicU32::new(0), + pending_objects: AtomicU32::new(0), + pending_edges: AtomicU32::new(0), + pending_bytes: AtomicU64::new(0), + mutation_epoch: AtomicU64::new(0), + })?; + self.sessions[slot].closing.store(false, Ordering::Release); + self.sessions[slot] + .root + .store(grant.to_bits(), Ordering::Release); + Ok(grant) + } + + /// Give back an unpublished attach claim. No handle or root edge exists + /// while a session is in this state. + pub(crate) fn abort_attach(&self, session: SessionId) { + let Ok(slot) = self.session_slot(session) else { + return; + }; + let _ = self.sessions[slot].root.compare_exchange( + SESSION_TRANSITION, + SESSION_VACANT, + Ordering::AcqRel, + Ordering::Acquire, + ); + self.sessions[slot].closing.store(false, Ordering::Release); + } + + fn session_slot(&self, session: SessionId) -> FsResult { + let slot = session.index() as usize; + if slot >= MAX_SESSIONS { + return Err(FsError::Range); + } + Ok(slot) + } + + pub(crate) fn is_attached(&self, session: SessionId) -> bool { + self.root_grant(session).is_ok() + } + + /// Sessions whose root slot is occupied (rooted or mid-transition) — the + /// diagnostic aggregate, deliberately a bare count. + pub(crate) fn live_sessions(&self) -> u16 { + self.sessions + .iter() + .filter(|session| session.root.load(Ordering::Acquire) != SESSION_VACANT) + .count() as u16 + } + + /// The session's private-root authority edge. + pub(crate) fn root_grant(&self, session: SessionId) -> FsResult { + let slot = self.session_slot(session)?; + if self.sessions[slot].closing.load(Ordering::Acquire) { + return Err(FsError::Busy); + } + let bits = self.sessions[slot].root.load(Ordering::Acquire); + match bits { + SESSION_VACANT => Err(FsError::InvalidHandle), + SESSION_TRANSITION => Err(FsError::Busy), + _ => Ok(GrantId::from_bits(bits)), + } + } + + pub(crate) fn root_object(&self, session: SessionId) -> FsResult { + let grant = self.root_grant(session)?; + Ok(self.validated_edge(grant)?.root()) + } + + /// Claim one task's combined private-root/attachment name set. + pub(crate) fn claim_view(&self, session: SessionId) -> FsResult> { + self.root_grant(session)?; + let slot = self.session_slot(session)?; + self.claim_view_slot(slot) + } + + fn claim_view_slot(&self, slot: usize) -> FsResult> { + self.sessions[slot] + .view_claimed + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| FsError::Busy)?; + Ok(ViewClaim { + session: &self.sessions[slot], + }) + } + + /// Snapshot the per-grant mutation generation before graph usage is + /// derived. [`Self::claim_growth`] closes the observation by requiring + /// this exact value after reserving against all in-flight growth. + pub(crate) fn quota_epoch(&self, grant: GrantId) -> FsResult { + Ok(self + .validated_edge(grant)? + .mutation_epoch + .load(Ordering::Acquire)) + } + + /// Reserve one operation's growth against live graph usage plus every + /// other in-flight reservation on this exact authority edge. + pub(crate) fn claim_growth( + &self, + grant: GrantId, + expected_epoch: u64, + live_objects: u32, + live_edges: u32, + live_bytes: u64, + objects: u32, + edges: u32, + bytes: u64, + ) -> FsResult> { + let edge = self.validated_edge(grant)?; + if edge.mutation_epoch.load(Ordering::Acquire) != expected_epoch { + return Err(FsError::Busy); + } + if !reserve_u32( + &edge.pending_objects, + live_objects, + objects, + edge.quota.max_objects(), + ) { + return Err(FsError::OutOfObjects); + } + if !reserve_u32( + &edge.pending_edges, + live_edges, + edges, + edge.quota.max_edges(), + ) { + edge.pending_objects.fetch_sub(objects, Ordering::AcqRel); + return Err(FsError::OutOfEdges); + } + if !reserve_u64( + &edge.pending_bytes, + live_bytes, + bytes, + edge.quota.max_bytes(), + ) { + edge.pending_edges.fetch_sub(edges, Ordering::AcqRel); + edge.pending_objects.fetch_sub(objects, Ordering::AcqRel); + return Err(FsError::OutOfStorage); + } + if edge.mutation_epoch.load(Ordering::Acquire) != expected_epoch { + edge.pending_bytes.fetch_sub(bytes, Ordering::AcqRel); + edge.pending_edges.fetch_sub(edges, Ordering::AcqRel); + edge.pending_objects.fetch_sub(objects, Ordering::AcqRel); + return Err(FsError::Busy); + } + Ok(QuotaClaim { + edge, + objects, + edges, + bytes, + }) + } + + /// A session's live handles: its own arena's, which no other session's + /// handles can occupy. + fn session_handle_ids( + &self, + session: SessionId, + ) -> FsResult> { + let slot = self.session_slot(session)?; + let mut ids = heapless::Vec::new(); + let Some(arena) = self.arena(slot) else { + return Ok(ids); + }; + for index in 0..arena.handles.capacity() { + let Some(id) = arena.handles.live_id_at(index) else { + continue; + }; + let record = arena.handles.try_read(id).map_err(|error| match error { + SlotAccess::Busy => FsError::Busy, + SlotAccess::Stale => FsError::Busy, + })?; + if record.session == session { + ids.push(Handle::from_key(join(slot, id, MAX_HANDLES_PER_SESSION))) + .map_err(|_| FsError::OutOfHandles)?; + } + } + Ok(ids) + } + + fn claim_handles<'a>( + &'a self, + session: SessionId, + ids: &[Handle], + ) -> FsResult< + heapless::Vec< + (Handle, SlotVecWrite<'a, OpenHandle, ServiceAlloc>), + MAX_HANDLES_PER_SESSION, + >, + > { + let mut claims = heapless::Vec::new(); + for handle in ids { + let (handles, local) = self.handle_slot(session, *handle)?; + let record = handles.try_write(local).map_err(|_| FsError::Busy)?; + if record.session != session { + return Err(FsError::Busy); + } + claims + .push((*handle, record)) + .map_err(|_| FsError::OutOfHandles)?; + } + Ok(claims) + } + + /// The edges a session holds: its own arena's. + fn session_edge_roots( + &self, + session: SessionId, + ) -> FsResult> { + let slot = self.session_slot(session)?; + let mut roots = heapless::Vec::new(); + let Some(arena) = self.arena(slot) else { + return Ok(roots); + }; + for index in 0..arena.edges.capacity() { + let Some(id) = arena.edges.live_id_at(index) else { + continue; + }; + let edge = arena.edges.try_read(id).map_err(|_| FsError::Busy)?; + if edge.holder == session { + roots + .push(GrantId::from_key(join(slot, id, MAX_EDGES_PER_SESSION))) + .map_err(|_| FsError::OutOfEdges)?; + } + } + Ok(roots) + } + + /// Claim a complete derivation closure before removing its first edge. + /// + /// Claimed roots make any new descendant's lineage validation return + /// `Busy`. Existing descendants are discovered from immutable parent ids, + /// one generation per pass over every attached session's edges; a + /// concurrently claimed edge causes the whole attempt to back off with + /// every already-acquired guard republished. The closure is staged on the + /// heap ([`unstaged`]): a task view's reaches every command descending + /// from it, wherever their edges are held. + fn claim_edge_closure<'a>(&'a self, roots: &[GrantId]) -> FsResult> { + // Claims are taken in canonical (grant id) order, here and below, so + // two overlapping retirements back off rather than chase each other. + let mut unique = heapless::Vec::::new(); + for root in roots { + if !unique.contains(root) { + unique.push(*root).map_err(|_| FsError::OutOfEdges)?; + } + } + unique + .as_mut_slice() + .sort_unstable_by_key(|grant| grant.key()); + + let mut claimed = EdgeClaims::new(); + claimed.0.try_reserve(unique.len()).map_err(unstaged)?; + for grant in unique { + let (edges, local) = self.edge_slot(grant).map_err(|_| FsError::Busy)?; + let edge = edges.try_write(local).map_err(|_| FsError::Busy)?; + claimed.0.push(ClaimedEdge { + grant, + edge, + depth: 0, + }); + } + + let mut children = Vec::new_in(service_alloc()); + loop { + children.clear(); + for (slot, edges) in self.edge_arenas() { + for index in 0..edges.capacity() { + let Some(id) = edges.live_id_at(index) else { + continue; + }; + let grant = GrantId::from_key(join(slot, id, MAX_EDGES_PER_SESSION)); + if claimed.contains(grant) { + continue; + } + let edge = edges.try_read(id).map_err(|_| FsError::Busy)?; + if edge.parent().is_some_and(|parent| claimed.contains(parent)) { + children.try_reserve(1).map_err(unstaged)?; + children.push(grant); + } + } + } + if children.is_empty() { + break; + } + // Collected slot by slot and index by index, so already in + // ascending grant id order: the canonical claim order. Every + // child's parent was claimed in an earlier generation — one + // claimed only now was not yet a parent when this one was + // collected — so the parent check searches that sorted prefix, + // not this generation's unsorted tail. + claimed.0.try_reserve(children.len()).map_err(unstaged)?; + let parents = claimed.len(); + for grant in children.iter().copied() { + let (edges, local) = self.edge_slot(grant).map_err(|_| FsError::Busy)?; + let edge = edges.try_write(local).map_err(|_| FsError::Busy)?; + if !edge + .parent() + .is_some_and(|parent| claimed.position_within(parent, parents).is_some()) + { + return Err(FsError::Busy); + } + claimed.0.push(ClaimedEdge { + grant, + edge, + depth: 0, + }); + } + // One sort per generation keeps the membership tests above + // binary searches; a generation never claims an edge twice, + // because every candidate was unclaimed when it was collected. + claimed.sort(); + } + Ok(claimed) + } + + /// Detach a session: retire every handle it holds and every authority + /// edge it holds, and report the roots whose last route just went away so + /// the caller can reclaim them. + pub(crate) fn detach_session(&self, session: SessionId) -> FsResult { + let slot = self.session_slot(session)?; + self.sessions[slot] + .closing + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| FsError::Busy)?; + + let attempt = (|| { + let root_bits = self.sessions[slot].root.load(Ordering::Acquire); + match root_bits { + SESSION_VACANT => return Err(FsError::InvalidHandle), + // Admission parks a peer's session here until a view installs + // its root, and a peer can close before it is ever viewed (a + // launch that failed after admission). Such a session holds no + // handle — opening one needs a published root — and no edge, + // because a root's edge is published with the root; giving the + // slot back is the whole of its retirement. Refusing it instead + // pinned the slot for good, its close retried on every wake. + // An edge already held is a root install in flight, which + // publishes or aborts on its own: retry after it. + SESSION_TRANSITION => { + return if self.session_edge_roots(session)?.is_empty() { + Ok(Retired::new_in(service_alloc())) + } else { + Err(FsError::Busy) + }; + } + _ => {} + } + let private_root = GrantId::from_bits(root_bits); + let _view = self.claim_view_slot(slot)?; + + // Closing blocks every new handle/grant insertion. Claim all + // existing records before removing the first one, so an in-flight + // request makes peer cleanup retry rather than disappear from a + // best-effort iterator. + let handle_ids = self.session_handle_ids(session)?; + let handles = self.claim_handles(session, &handle_ids)?; + let roots = self.session_edge_roots(session)?; + if !roots.contains(&private_root) { + return Err(FsError::AccessLost); + } + let edges = self.claim_edge_closure(&roots)?; + if handles + .iter() + .any(|(_, handle)| !edges.contains(handle.grant())) + { + return Err(FsError::AccessLost); + } + // Everything that can refuse — the order and the room for its + // roots — is settled before the first record goes. + let edges = edges.into_retirement()?; + let mut stranded = Retired::new_in(service_alloc()); + stranded.try_reserve_exact(edges.len()).map_err(unstaged)?; + + for (_, handle) in handles { + handle.remove().map_err(|_| FsError::AccessLost)?; + } + edges.retire_into(&mut stranded)?; + Ok(stranded) + })(); + + match attempt { + Ok(stranded) => { + self.sessions[slot] + .root + .store(SESSION_VACANT, Ordering::Release); + self.sessions[slot].closing.store(false, Ordering::Release); + Ok(stranded) + } + Err(error) => { + self.sessions[slot].closing.store(false, Ordering::Release); + Err(error) + } + } + } + + // ── authority edges ───────────────────────────────────────────────────── + + pub(crate) fn edge( + &self, + grant: GrantId, + ) -> FsResult> { + self.edge_slot(grant) + .and_then(|(edges, local)| edges.try_read(local)) + .map_err(|error| fs_access(error, FsError::AccessLost)) + } + + /// One edge after validating its complete delegation ancestry. + pub(crate) fn validated_edge(&self, grant: GrantId) -> FsResult> { + self.validate_lineage(grant) + } + + /// Create a grant for `holder` rooted at `root` with `rights`. + pub(crate) fn insert_grant( + &self, + issuer: SessionId, + holder: SessionId, + parent: GrantId, + root: ObjectId, + rights: Rights, + quota: AuthorityQuota, + display_name: Option, + ) -> FsResult { + // A closing issuer cannot extend a lineage while its peer teardown is + // claiming that same bounded derivation closure. + self.root_grant(issuer)?; + // A grant to a session that has never attached could never be opened + // and would keep its root alive forever. + if !self.is_attached(holder) { + return Err(FsError::InvalidHandle); + } + let source = self.validate_lineage(parent)?; + if source.holder() != issuer { + return Err(FsError::AccessDenied); + } + if source.lineage_len() >= MAX_LINEAGE_DEPTH { + return Err(FsError::Range); + } + source.rights().attenuate(rights)?; + source.quota().attenuate(quota)?; + self.insert_edge(AuthorityEdge { + holder, + issuer, + parent: Some(parent), + root, + rights, + quota, + kind: AuthorityKind::Grant, + display_name, + open_handles: AtomicU32::new(0), + pending_objects: AtomicU32::new(0), + pending_edges: AtomicU32::new(0), + pending_bytes: AtomicU64::new(0), + mutation_epoch: AtomicU64::new(0), + }) + } + + /// Pin an authorized route beneath its nearest non-process ancestor. + /// + /// A recyclable process root is a short-lived descendant installed from + /// an invocation view. Pinning a `cd` result beneath that process root + /// would make peer teardown revoke the new cwd even though the caller + /// retained its independent channel-root revocation pivot. Reparent the + /// narrowed object beneath the exact stable invocation ancestor already + /// present in the validated route; no unrelated root or pathname is + /// consulted. The current route still needs SHARE and bounds both rights + /// and quota, so this cannot amplify authority. + pub(crate) fn pin_route( + &self, + session: SessionId, + route: Route, + rights: Rights, + ) -> FsResult<(GrantId, SessionId)> { + route.rights.require(Rights::SHARE)?; + let delegated = route.rights.attenuate(rights)?; + let (grant, holder) = self.stable_ancestor(session, route.grant)?; + let edge = self.validated_edge(grant)?; + edge.rights().attenuate(delegated)?; + edge.quota().attenuate(route.quota)?; + drop(edge); + let pinned = self.insert_grant( + holder, + holder, + grant, + route.object, + delegated, + route.quota, + None, + )?; + Ok((pinned, holder)) + } + + /// The nearest edge on `grant`'s validated lineage held by a session + /// other than `session`, and its holder: the stable invocation view a + /// recyclable process descends from. [`Self::pin_route`] pins beneath + /// it, and a cwd anchor is matched against its task by it. + pub(crate) fn stable_ancestor( + &self, + session: SessionId, + grant: GrantId, + ) -> FsResult<(GrantId, SessionId)> { + let mut cursor = Some(grant); + for _ in 0..MAX_LINEAGE_DEPTH { + let grant = cursor.ok_or(FsError::AccessDenied)?; + let edge = self.validated_edge(grant)?; + let holder = edge.holder(); + if holder != session { + return Ok((grant, holder)); + } + cursor = edge.parent(); + } + Err(FsError::AccessLost) + } + + /// Grant a transient process route from its exact stable task ancestor. + /// This is the sharing analogue of [`Self::pin_route`]: it walks the + /// already-validated generational lineage and refuses unless + /// `source_view` is on that lineage. The new edge is issued by the stable + /// task view, not by the process that happens to perform the request. + pub(crate) fn grant_route_from_ancestor( + &self, + session: SessionId, + source_view: SessionId, + destination_view: SessionId, + route: Route, + rights: Rights, + display_name: Option, + ) -> FsResult { + route.rights.require(Rights::SHARE)?; + route.rights.attenuate(rights)?; + let mut cursor = Some(route.grant); + for _ in 0..MAX_LINEAGE_DEPTH { + let grant = cursor.ok_or(FsError::AccessDenied)?; + let edge = self.validated_edge(grant)?; + let holder = edge.holder(); + cursor = edge.parent(); + if holder == session { + continue; + } + if holder != source_view { + return Err(FsError::AccessDenied); + } + edge.rights().attenuate(rights)?; + edge.quota().attenuate(route.quota)?; + drop(edge); + return self.insert_grant( + source_view, + destination_view, + grant, + route.object, + rights, + route.quota, + display_name, + ); + } + Err(FsError::AccessLost) + } + + /// Retire an authority edge, returning the object it rooted. + /// + /// Retiring bumps the edge slot's generation, so every handle that + /// entered through it fails its next revalidation — which is the whole of + /// "losing a direct-grant route makes existing reads fail immediately". + /// + /// A private root may not be revoked this way: a session's starting view + /// goes away with the session ([`Self::detach_session`]), and letting a + /// caller retire it would leave an attached session with no route at all. + pub(crate) fn retire_edge(&self, caller: SessionId, grant: GrantId) -> FsResult { + let edge = self.edge(grant)?; + if edge.kind() == AuthorityKind::PrivateRoot { + return Err(FsError::Unsupported); + } + // Either side of the grant may end it; nobody else may. + if edge.holder() != caller && edge.issuer() != caller { + return Err(FsError::AccessDenied); + } + drop(edge); + let mut retired = Retired::new_in(service_alloc()); + self.collect_and_retire(grant, &mut retired)?; + Ok(retired) + } + + /// Validate one authority edge and every generational parent it derives + /// from. Every edge is created within [`MAX_LINEAGE_DEPTH`] of its root, + /// so a longer walk would be a cycle or corrupt state and fails closed. + fn validate_lineage(&self, grant: GrantId) -> FsResult> { + let leaf = self.edge(grant)?; + let mut cursor = leaf.parent(); + let mut ancestors = heapless::Vec::new(); + while let Some(parent) = cursor { + let edge = self.edge(parent)?; + cursor = edge.parent(); + ancestors.push(edge).map_err(|_| FsError::AccessLost)?; + } + Ok(ValidatedEdge { leaf, ancestors }) + } + + /// Retire `root` and the bounded derivation closure below it. Each edge is + /// removed once and contributes one object root for the engine's exact + /// grant-count/reclamation fold. + fn collect_and_retire(&self, root: GrantId, retired: &mut Retired) -> FsResult<()> { + let closure = self.claim_edge_closure(&[root])?.into_retirement()?; + retired.try_reserve_exact(closure.len()).map_err(unstaged)?; + closure.retire_into(retired) + } + + /// The grants a session holds, as `(id, kind, rights, display name)`. + pub(crate) fn grants_of( + &self, + session: SessionId, + ) -> FsResult< + heapless::Vec< + (GrantId, AuthorityKind, Rights, AuthorityQuota, Option), + MAX_EDGES_PER_SESSION, + >, + > { + let _view = self.claim_view(session)?; + let slot = self.session_slot(session)?; + let mut grants = heapless::Vec::new(); + let Some(arena) = self.arena(slot) else { + return Ok(grants); + }; + for index in 0..arena.edges.capacity() { + let Some(id) = arena.edges.live_id_at(index) else { + continue; + }; + let edge = arena.edges.try_read(id).map_err(|_| FsError::Busy)?; + if edge.holder() == session { + grants + .push(( + GrantId::from_key(join(slot, id, MAX_EDGES_PER_SESSION)), + edge.kind(), + edge.rights(), + edge.quota(), + edge.display_name(), + )) + .map_err(|_| FsError::OutOfEdges)?; + } + } + Ok(grants) + } + + /// `true` if `session` already sees an attachment named `name`. + /// + /// Used to keep a sender from overwriting a name in the recipient's view + /// because the recipient's namespace of attachment names is its own. + pub(crate) fn display_name_taken(&self, session: SessionId, name: &Name) -> FsResult { + let slot = self.session_slot(session)?; + let Some(arena) = self.arena(slot) else { + return Ok(false); + }; + for index in 0..arena.edges.capacity() { + let Some(id) = arena.edges.live_id_at(index) else { + continue; + }; + let edge = arena.edges.try_read(id).map_err(|_| FsError::Busy)?; + if edge.holder() == session && edge.display_name().as_ref() == Some(name) { + return Ok(true); + } + } + Ok(false) + } + + // ── handles ───────────────────────────────────────────────────────────── + + /// Mint a handle in `session`'s own arena. + /// + /// The arena's capacity is the per-session ceiling + /// ([`MAX_HANDLES_PER_SESSION`]): a session that holds that many is + /// refused, and no other session's handles share its storage, so one + /// client cannot starve another. + pub(crate) fn insert_handle( + &self, + session: SessionId, + object: ObjectId, + grant: GrantId, + proof: EdgeProof, + rights: Rights, + ) -> FsResult { + let slot = self.session_slot(session)?; + // A transition-owned session accepts no new work. This check also + // rejects a forged/unattached session before consuming either quota. + self.root_grant(session)?; + let arena = self.arena(slot).ok_or(FsError::InvalidHandle)?; + if arena.handles.is_full() { + return Err(FsError::OutOfHandles); + } + let edge = self.validate_lineage(grant)?; + if edge + .open_handles + .try_update(Ordering::AcqRel, Ordering::Acquire, |live| { + (live < edge.quota().max_handles()).then_some(live + 1) + }) + .is_err() + { + return Err(FsError::OutOfHandles); + } + let inserted = arena.handles.insert(OpenHandle { + session, + object, + grant, + proof, + rights, + }); + match inserted { + Ok(local) => Ok(Handle::from_key(join(slot, local, MAX_HANDLES_PER_SESSION))), + Err(_) => { + edge.open_handles.fetch_sub(1, Ordering::AcqRel); + Err(FsError::OutOfHandles) + } + } + } + + /// The arena holding `session`'s handle `handle`, and the handle's id + /// inside it. A handle whose index names another session's slot is not + /// this session's at all: [`FsError::InvalidHandle`], exactly. + fn handle_slot( + &self, + session: SessionId, + handle: Handle, + ) -> FsResult<(&AtomicSlotVec, SlotKey)> { + let (slot, local) = + split(handle.key(), MAX_HANDLES_PER_SESSION).ok_or(FsError::InvalidHandle)?; + if slot != self.session_slot(session)? { + return Err(FsError::InvalidHandle); + } + let arena = self.arena(slot).ok_or(FsError::InvalidHandle)?; + Ok((&arena.handles, local)) + } + + /// The record behind a handle, if `session` owns it. + /// + /// A handle presented by the wrong session is [`FsError::InvalidHandle`], + /// not `AccessDenied`: from the presenting session's point of view the + /// handle does not exist at all. + pub(crate) fn handle( + &self, + session: SessionId, + handle: Handle, + ) -> FsResult> { + self.root_grant(session)?; + let (handles, local) = self.handle_slot(session, handle)?; + let record = handles + .try_read(local) + .map_err(|error| fs_access(error, FsError::InvalidHandle))?; + if record.session != session { + return Err(FsError::InvalidHandle); + } + Ok(record) + } + + /// Close a handle. + pub(crate) fn remove_handle(&self, session: SessionId, handle: Handle) -> FsResult { + // Check ownership before removing, so one session cannot close + // another's handle. + self.handle(session, handle)?; + let (handles, local) = self.handle_slot(session, handle)?; + let record = handles + .try_remove(local) + .map_err(|error| fs_access(error, FsError::InvalidHandle))?; + if let Ok(edge) = self.edge(record.grant) { + let _ = edge + .open_handles + .try_update(Ordering::AcqRel, Ordering::Acquire, |live| { + Some(live.saturating_sub(1)) + }); + } + Ok(record) + } + + /// Live handles held by a session. + pub(crate) fn live_handles(&self, session: SessionId) -> usize { + self.root_grant(session) + .ok() + .and_then(|_| self.session_slot(session).ok()) + .and_then(|slot| self.arena(slot)) + .map_or(0, |arena| arena.handles.len()) + } + + /// Live authority edges, every session's together. + pub(crate) fn edge_count(&self) -> usize { + self.edge_arenas().map(|(_, edges)| edges.len()).sum() + } + + /// Session slots whose storage has been allocated: the most sessions + /// ever attached at once, which is what the authority's heap footprint + /// follows ([`SessionArena::BYTES`] each). + #[cfg(test)] + fn session_arenas(&self) -> usize { + (0..MAX_SESSIONS) + .filter(|&slot| self.arena(slot).is_some()) + .count() + } + + // ── the proof walk ────────────────────────────────────────────────────── + + /// Re-establish that a handle still names a reachable object, and report + /// what the route permits. + /// + /// Walked in full on every operation. The steps, in order, are the + /// design: the authority edge must still be live; its root must still + /// exist; every containment edge in the proof must still hang beneath the + /// previous object at its recorded generation; and the chain must still + /// arrive at the handle's object. + pub(crate) fn revalidate( + &self, + session: SessionId, + handle: Handle, + ns: &Namespace, + ) -> FsResult { + let record = self.handle(session, handle)?; + + // 1. The authority edge itself — retired grants land here. + let edge = self.validate_lineage(record.grant)?; + // A grant that changed hands cannot be ridden by the old holder. + if edge.holder != session { + return Err(FsError::AccessLost); + } + + // 2. Its root must still be a live object. + let mut current = edge.root(); + if !ns.exists(current) { + return Err(FsError::AccessLost); + } + + // 3. Every containment edge, in order. + for step in record.proof.edges() { + // The chain must be connected: this edge has to hang beneath the + // object the previous step produced, or the proof is describing + // some other route. + if step.parent() != current { + return Err(FsError::AccessLost); + } + // `edge_child` fails closed on a removed edge or a retired entry + // generation — the per-slot generation doing its job. + current = ns.edge_child(*step)?; + } + + // 4. And it must arrive where the handle says. + if current != record.object { + return Err(FsError::AccessLost); + } + + Ok(Route { + object: record.object, + rights: record.rights, + grant: record.grant, + quota: edge.quota(), + }) + } +} + +impl Drop for Authority { + fn drop(&mut self) { + for session in &mut self.sessions { + let arena = *session.arena.get_mut(); + if !arena.is_null() { + // SAFETY: published once by `attach_session` from + // `Box::into_raw_with_allocator` over `service_alloc()`, and + // `&mut self` excludes every guard that could still borrow it. + drop(unsafe { Box::from_raw_in(arena, service_alloc()) }); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ── rights ────────────────────────────────────────────────────────────── + + #[test] + fn rights_bits_are_distinct() { + let all = [ + Rights::READ, + Rights::WRITE, + Rights::MUTATE_NAMESPACE, + Rights::SHARE, + Rights::BIND_EXECUTABLE, + ]; + for (i, a) in all.iter().enumerate() { + for (j, b) in all.iter().enumerate() { + if i != j { + assert!(a.intersection(*b).is_empty(), "rights must not share a bit"); + } + } + } + assert!(Rights::ALL.contains(Rights::READ)); + assert!(Rights::ALL.contains(Rights::SHARE)); + assert!(Rights::ALL.contains(Rights::BIND_EXECUTABLE)); + assert!(Rights::NONE.is_empty()); + } + + #[test] + fn contains_and_subset_agree() { + let rw = Rights::READ.union(Rights::WRITE); + assert!(rw.contains(Rights::READ)); + assert!(rw.contains(Rights::WRITE)); + assert!(!rw.contains(Rights::SHARE)); + assert!(Rights::READ.is_subset_of(rw)); + assert!(!rw.is_subset_of(Rights::READ)); + assert!(rw.is_subset_of(Rights::ALL)); + } + + #[test] + fn attenuation_cannot_add_a_right() { + let read_only = Rights::READ; + // Asking for more than you hold is refused, not clamped. + assert_eq!( + read_only.attenuate(Rights::WRITE).unwrap_err(), + FsError::AccessDenied + ); + assert_eq!( + read_only.attenuate(Rights::ALL).unwrap_err(), + FsError::AccessDenied + ); + // Asking for a subset yields exactly that subset. + assert_eq!(Rights::ALL.attenuate(Rights::READ).unwrap(), Rights::READ); + assert_eq!(read_only.attenuate(Rights::READ).unwrap(), Rights::READ); + } + + #[test] + fn share_is_never_implied() { + // Holding everything else must not confer SHARE. + let no_share = Rights::READ + .union(Rights::WRITE) + .union(Rights::MUTATE_NAMESPACE) + .union(Rights::BIND_EXECUTABLE); + assert!(!no_share.contains(Rights::SHARE)); + assert_eq!( + no_share.attenuate(Rights::SHARE).unwrap_err(), + FsError::AccessDenied + ); + } + + #[test] + fn unknown_right_bits_are_discarded() { + // An unrecognized wire flag must not become authority. + let forged = Rights::from_bits_truncating(0xff); + assert_eq!(forged, Rights::ALL); + assert_eq!(forged.bits(), Rights::ALL.bits()); + assert_eq!(Rights::from_bits_truncating(1 << 7), Rights::NONE); + } + + #[test] + fn require_reports_the_missing_right() { + assert!(Rights::ALL.require(Rights::WRITE).is_ok()); + assert_eq!( + Rights::READ.require(Rights::WRITE).unwrap_err(), + FsError::AccessDenied + ); + assert!(Rights::NONE.require(Rights::NONE).is_ok()); + } + + // ── proofs ────────────────────────────────────────────────────────────── + + #[test] + fn a_root_proof_is_empty() { + let proof = EdgeProof::root(); + assert_eq!(proof.depth(), 0); + assert!(proof.edges().is_empty()); + } + + #[test] + fn proofs_extend_up_to_the_depth_bound_then_refuse() { + let mut ns = Namespace::::new(); + let root = ns.create_dir_object().unwrap(); + let file = ns.create_file_object().unwrap(); + let edge = ns.link(root, &Name::new("f").unwrap(), file).unwrap(); + + let mut proof = EdgeProof::root(); + for expected in 1..=MAX_PATH_DEPTH { + proof = proof.extend(edge).unwrap(); + assert_eq!(proof.depth(), expected); + } + // Refuses rather than truncating: a truncated proof would stop + // revalidating its upper edges. + assert_eq!(proof.extend(edge).unwrap_err(), FsError::Range); + assert_eq!(proof.depth(), MAX_PATH_DEPTH); + } + + #[test] + fn extend_does_not_mutate_the_original() { + let mut ns = Namespace::::new(); + let root = ns.create_dir_object().unwrap(); + let file = ns.create_file_object().unwrap(); + let edge = ns.link(root, &Name::new("f").unwrap(), file).unwrap(); + + let base = EdgeProof::root(); + let extended = base.extend(edge).unwrap(); + assert_eq!(base.depth(), 0); + assert_eq!(extended.depth(), 1); + } + + // ── sessions, grants, handles ─────────────────────────────────────────── + + /// An authority arena with one attached session over a small namespace. + fn fixture() -> (Authority, Namespace, SessionId, ObjectId, GrantId) { + let mut ns = Namespace::::new(); + let mut auth = Authority::new(); + let session = SessionId::new(0); + let root = ns.create_dir_object().unwrap(); + auth.attach_session(session).unwrap(); + let grant = auth.install_root(session, root).unwrap(); + ns.object_mut(root).unwrap().grants_inc(); + (auth, ns, session, root, grant) + } + + #[test] + fn a_private_root_carries_full_rights_and_is_its_own_issuer() { + let (auth, _ns, session, root, grant) = fixture(); + let edge = auth.edge(grant).unwrap(); + assert_eq!(edge.root(), root); + assert_eq!(edge.rights(), Rights::ALL); + assert_eq!(edge.kind(), AuthorityKind::PrivateRoot); + assert_eq!(edge.holder(), session); + assert_eq!(edge.issuer(), session); + assert_eq!(auth.root_grant(session).unwrap(), grant); + assert!(auth.is_attached(session)); + } + + #[test] + fn attaching_a_session_twice_is_refused() { + let (mut auth, _ns, session, _root, _grant) = fixture(); + assert_eq!( + auth.attach_session(session).unwrap_err(), + FsError::AlreadyExists + ); + } + + #[test] + fn an_out_of_range_session_is_refused() { + let mut auth = Authority::new(); + let bogus = SessionId::new(MAX_SESSIONS as u16); + assert_eq!(auth.attach_session(bogus).unwrap_err(), FsError::Range); + assert!(!auth.is_attached(bogus)); + } + + #[test] + fn an_unattached_session_has_no_root() { + let auth = Authority::new(); + assert_eq!( + auth.root_grant(SessionId::new(1)).unwrap_err(), + FsError::InvalidHandle + ); + } + + #[test] + fn a_handle_is_invalid_when_presented_by_another_session() { + let (mut auth, mut ns, session_a, root, grant_a) = fixture(); + let session_b = SessionId::new(1); + let root_b = ns.create_dir_object().unwrap(); + auth.attach_session(session_b).unwrap(); + auth.install_root(session_b, root_b).unwrap(); + + let handle_a = auth + .insert_handle(session_a, root, grant_a, EdgeProof::root(), Rights::ALL) + .unwrap(); + // Session B presenting A's handle bits must be refused outright. + assert_eq!( + auth.handle(session_b, handle_a).unwrap_err(), + FsError::InvalidHandle + ); + assert!(auth.handle(session_a, handle_a).is_ok()); + } + + #[test] + fn forged_handles_fail_closed() { + let (auth, _ns, session, _root, _grant) = fixture(); + assert_eq!( + auth.handle(session, Handle::from_bits(0)).unwrap_err(), + FsError::InvalidHandle + ); + assert_eq!( + auth.handle(session, Handle::from_bits(u64::MAX)) + .unwrap_err(), + FsError::InvalidHandle + ); + } + + #[test] + fn one_session_cannot_close_anothers_handle() { + let (mut auth, mut ns, session_a, root, grant_a) = fixture(); + let session_b = SessionId::new(1); + let root_b = ns.create_dir_object().unwrap(); + auth.attach_session(session_b).unwrap(); + auth.install_root(session_b, root_b).unwrap(); + + let handle_a = auth + .insert_handle(session_a, root, grant_a, EdgeProof::root(), Rights::ALL) + .unwrap(); + assert_eq!( + auth.remove_handle(session_b, handle_a).unwrap_err(), + FsError::InvalidHandle + ); + // A's handle survived the attempt. + assert!(auth.handle(session_a, handle_a).is_ok()); + assert_eq!(auth.live_handles(session_a), 1); + } + + #[test] + fn the_per_session_handle_ceiling_is_enforced_per_session() { + let (mut auth, mut ns, session_a, root, grant_a) = fixture(); + let session_b = SessionId::new(1); + let root_b = ns.create_dir_object().unwrap(); + auth.attach_session(session_b).unwrap(); + let grant_b = auth.install_root(session_b, root_b).unwrap(); + + for _ in 0..MAX_HANDLES_PER_SESSION { + auth.insert_handle(session_a, root, grant_a, EdgeProof::root(), Rights::ALL) + .unwrap(); + } + assert_eq!( + auth.insert_handle(session_a, root, grant_a, EdgeProof::root(), Rights::ALL) + .unwrap_err(), + FsError::OutOfHandles + ); + // One client exhausting its own ceiling must not starve another. + assert!( + auth.insert_handle(session_b, root_b, grant_b, EdgeProof::root(), Rights::ALL) + .is_ok() + ); + assert_eq!(auth.live_handles(session_a), MAX_HANDLES_PER_SESSION); + assert_eq!(auth.live_handles(session_b), 1); + } + + #[test] + fn closing_a_handle_frees_its_quota() { + let (mut auth, _ns, session, root, grant) = fixture(); + let handle = auth + .insert_handle(session, root, grant, EdgeProof::root(), Rights::ALL) + .unwrap(); + assert_eq!(auth.live_handles(session), 1); + auth.remove_handle(session, handle).unwrap(); + assert_eq!(auth.live_handles(session), 0); + // And the closed handle is gone for good. + assert_eq!( + auth.remove_handle(session, handle).unwrap_err(), + FsError::InvalidHandle + ); + } + + // ── revalidation ──────────────────────────────────────────────────────── + + #[test] + fn a_root_handle_revalidates() { + let (mut auth, ns, session, root, grant) = fixture(); + let handle = auth + .insert_handle(session, root, grant, EdgeProof::root(), Rights::ALL) + .unwrap(); + let route = auth.revalidate(session, handle, &ns).unwrap(); + assert_eq!(route.object, root); + assert_eq!(route.rights, Rights::ALL); + } + + #[test] + fn a_handle_through_one_edge_revalidates_then_fails_when_the_edge_goes() { + let (mut auth, mut ns, session, root, grant) = fixture(); + let file = ns.create_file_object().unwrap(); + let name = Name::new("f").unwrap(); + let edge = ns.link(root, &name, file).unwrap(); + + let handle = auth + .insert_handle( + session, + file, + grant, + EdgeProof::root().extend(edge).unwrap(), + Rights::ALL, + ) + .unwrap(); + assert_eq!(auth.revalidate(session, handle, &ns).unwrap().object, file); + + // Remove the one edge the proof depends on. + ns.detach(root, &name).unwrap(); + assert_eq!( + auth.revalidate(session, handle, &ns).unwrap_err(), + FsError::AccessLost + ); + } + + #[test] + fn removing_an_intermediate_edge_invalidates_a_deeper_handle() { + let (mut auth, mut ns, session, root, grant) = fixture(); + let mid = ns.create_dir_object().unwrap(); + let leaf = ns.create_file_object().unwrap(); + let mid_name = Name::new("mid").unwrap(); + let mid_edge = ns.link(root, &mid_name, mid).unwrap(); + let leaf_edge = ns.link(mid, &Name::new("leaf").unwrap(), leaf).unwrap(); + + let proof = EdgeProof::root() + .extend(mid_edge) + .unwrap() + .extend(leaf_edge) + .unwrap(); + let handle = auth + .insert_handle(session, leaf, grant, proof, Rights::ALL) + .unwrap(); + assert!(auth.revalidate(session, handle, &ns).is_ok()); + + // Emptying and removing the intermediate directory cuts the chain at + // its first link even though the leaf edge itself is what vanishes. + ns.detach(mid, &Name::new("leaf").unwrap()).unwrap(); + ns.detach(root, &mid_name).unwrap(); + assert_eq!( + auth.revalidate(session, handle, &ns).unwrap_err(), + FsError::AccessLost + ); + } + + #[test] + fn retiring_a_grant_invalidates_handles_that_entered_through_it() { + let (mut auth, mut ns, session_a, root_a, grant_a) = fixture(); + let session_b = SessionId::new(1); + let root_b = ns.create_dir_object().unwrap(); + auth.attach_session(session_b).unwrap(); + auth.install_root(session_b, root_b).unwrap(); + + // A shares one of its files with B, read-only. + let file = ns.create_file_object().unwrap(); + ns.link(root_a, &Name::new("shared").unwrap(), file) + .unwrap(); + let grant = auth + .insert_grant( + session_a, + session_b, + grant_a, + file, + Rights::READ_ONLY, + AuthorityQuota::MAX, + None, + ) + .unwrap(); + ns.object_mut(file).unwrap().grants_inc(); + + let handle = auth + .insert_handle(session_b, file, grant, EdgeProof::root(), Rights::READ) + .unwrap(); + let route = auth.revalidate(session_b, handle, &ns).unwrap(); + assert_eq!(route.rights, Rights::READ); + assert!(route.rights.require(Rights::WRITE).is_err()); + + // A revokes. B's existing handle must fail on its very next use. + assert_eq!( + auth.retire_edge(session_a, grant).unwrap().as_slice(), + &[file] + ); + assert_eq!( + auth.revalidate(session_b, handle, &ns).unwrap_err(), + FsError::AccessLost + ); + // The file object itself is untouched — A still has its own name for it. + assert!(ns.exists(file)); + } + + #[test] + fn a_third_session_cannot_revoke_someone_elses_grant() { + let (mut auth, mut ns, session_a, root_a, grant_a) = fixture(); + let session_b = SessionId::new(1); + let session_c = SessionId::new(2); + for session in [session_b, session_c] { + let r = ns.create_dir_object().unwrap(); + auth.attach_session(session).unwrap(); + auth.install_root(session, r).unwrap(); + } + let file = ns.create_file_object().unwrap(); + ns.link(root_a, &Name::new("f").unwrap(), file).unwrap(); + let grant = auth + .insert_grant( + session_a, + session_b, + grant_a, + file, + Rights::READ, + AuthorityQuota::MAX, + None, + ) + .unwrap(); + + assert_eq!( + auth.retire_edge(session_c, grant).unwrap_err(), + FsError::AccessDenied + ); + // Still live for its holder. + assert!(auth.edge(grant).is_ok()); + // Both the issuer and the holder may end it. + assert!(auth.retire_edge(session_b, grant).is_ok()); + } + + #[test] + fn validated_descendant_pins_its_whole_lineage_during_retirement() { + let (auth, ns, session_a, root_a, grant_a) = fixture(); + let session_b = SessionId::new(1); + let session_c = SessionId::new(2); + for session in [session_b, session_c] { + let root = ns.create_dir_object().unwrap(); + auth.attach_session(session).unwrap(); + auth.install_root(session, root).unwrap(); + } + let file = ns.create_file_object().unwrap(); + ns.link(root_a, &Name::new("f").unwrap(), file).unwrap(); + let parent = auth + .insert_grant( + session_a, + session_b, + grant_a, + file, + Rights::ALL, + AuthorityQuota::MAX, + None, + ) + .unwrap(); + let child = auth + .insert_grant( + session_b, + session_c, + parent, + file, + Rights::READ, + AuthorityQuota::MAX, + None, + ) + .unwrap(); + + let in_flight = auth.validated_edge(child).unwrap(); + assert_eq!(auth.retire_edge(session_a, parent), Err(FsError::Busy)); + assert!(auth.edge(parent).is_ok() && auth.edge(child).is_ok()); + drop(in_flight); + + assert_eq!(auth.retire_edge(session_a, parent).unwrap().len(), 2); + assert_eq!(auth.edge(parent).unwrap_err(), FsError::AccessLost); + assert_eq!(auth.edge(child).unwrap_err(), FsError::AccessLost); + } + + #[test] + fn a_private_root_cannot_be_revoked() { + let (mut auth, _ns, session, _root, grant) = fixture(); + assert_eq!( + auth.retire_edge(session, grant).unwrap_err(), + FsError::Unsupported + ); + assert!(auth.edge(grant).is_ok()); + } + + #[test] + fn a_grant_to_an_unattached_session_is_refused() { + let (mut auth, mut ns, session_a, root_a, grant_a) = fixture(); + let file = ns.create_file_object().unwrap(); + ns.link(root_a, &Name::new("f").unwrap(), file).unwrap(); + // Nobody is attached at slot 3; a grant there could never be opened + // and would pin the file forever. + assert_eq!( + auth.insert_grant( + session_a, + SessionId::new(3), + grant_a, + file, + Rights::READ, + AuthorityQuota::MAX, + None, + ) + .unwrap_err(), + FsError::InvalidHandle + ); + } + + #[test] + fn a_proof_naming_a_disconnected_edge_is_refused() { + // A handcrafted proof whose edge does not hang beneath the grant root + // must not validate just because the edge itself is live. + let (mut auth, mut ns, session, root, grant) = fixture(); + let elsewhere = ns.create_dir_object().unwrap(); + ns.link(root, &Name::new("elsewhere").unwrap(), elsewhere) + .unwrap(); + let stray = ns.create_file_object().unwrap(); + let stray_edge = ns + .link(elsewhere, &Name::new("stray").unwrap(), stray) + .unwrap(); + + // Claim a route from the root straight to `stray` via an edge whose + // parent is `elsewhere`, not `root`. + let handle = auth + .insert_handle( + session, + stray, + grant, + EdgeProof::root().extend(stray_edge).unwrap(), + Rights::ALL, + ) + .unwrap(); + assert_eq!( + auth.revalidate(session, handle, &ns).unwrap_err(), + FsError::AccessLost + ); + } + + #[test] + fn a_proof_arriving_at_the_wrong_object_is_refused() { + let (mut auth, mut ns, session, root, grant) = fixture(); + let real = ns.create_file_object().unwrap(); + let other = ns.create_file_object().unwrap(); + let edge = ns.link(root, &Name::new("real").unwrap(), real).unwrap(); + ns.link(root, &Name::new("other").unwrap(), other).unwrap(); + + // The chain is intact but lands on `real`, not the claimed `other`. + let handle = auth + .insert_handle( + session, + other, + grant, + EdgeProof::root().extend(edge).unwrap(), + Rights::ALL, + ) + .unwrap(); + assert_eq!( + auth.revalidate(session, handle, &ns).unwrap_err(), + FsError::AccessLost + ); + } + + #[test] + fn a_handle_survives_a_rename_of_the_edge_it_proves_through() { + let (mut auth, mut ns, session, root, grant) = fixture(); + let file = ns.create_file_object().unwrap(); + let edge = ns.link(root, &Name::new("before").unwrap(), file).unwrap(); + let handle = auth + .insert_handle( + session, + file, + grant, + EdgeProof::root().extend(edge).unwrap(), + Rights::ALL, + ) + .unwrap(); + + ns.rename( + root, + &Name::new("before").unwrap(), + &Name::new("after").unwrap(), + ) + .unwrap(); + // Rename keeps the edge, so the route — and the handle — survive. + assert_eq!(auth.revalidate(session, handle, &ns).unwrap().object, file); + } + + #[test] + fn a_handle_through_a_moved_edge_fails_while_a_parallel_link_survives() { + let (mut auth, mut ns, session, root, grant) = fixture(); + let dst = ns.create_dir_object().unwrap(); + ns.link(root, &Name::new("dst").unwrap(), dst).unwrap(); + let file = ns.create_file_object().unwrap(); + let src_edge = ns.link(root, &Name::new("f").unwrap(), file).unwrap(); + let parallel = ns.link(root, &Name::new("also").unwrap(), file).unwrap(); + + let via_src = auth + .insert_handle( + session, + file, + grant, + EdgeProof::root().extend(src_edge).unwrap(), + Rights::ALL, + ) + .unwrap(); + let via_parallel = auth + .insert_handle( + session, + file, + grant, + EdgeProof::root().extend(parallel).unwrap(), + Rights::ALL, + ) + .unwrap(); + + ns.relocate( + root, + &Name::new("f").unwrap(), + dst, + &Name::new("moved").unwrap(), + ) + .unwrap(); + + assert_eq!( + auth.revalidate(session, via_src, &ns).unwrap_err(), + FsError::AccessLost + ); + assert_eq!( + auth.revalidate(session, via_parallel, &ns).unwrap().object, + file + ); + } + + // ── attachment names and teardown ─────────────────────────────────────── + + #[test] + fn attachment_names_are_per_recipient() { + let (mut auth, mut ns, session_a, root_a, grant_a) = fixture(); + let session_b = SessionId::new(1); + let root_b = ns.create_dir_object().unwrap(); + auth.attach_session(session_b).unwrap(); + auth.install_root(session_b, root_b).unwrap(); + + let file = ns.create_file_object().unwrap(); + ns.link(root_a, &Name::new("f").unwrap(), file).unwrap(); + let shared = Name::new("shared").unwrap(); + auth.insert_grant( + session_a, + session_b, + grant_a, + file, + Rights::READ, + AuthorityQuota::MAX, + Some(shared), + ) + .unwrap(); + + assert!(auth.display_name_taken(session_b, &shared).unwrap()); + // A's own view is unaffected by what B calls it. + assert!(!auth.display_name_taken(session_a, &shared).unwrap()); + } + + #[test] + fn grants_of_lists_a_sessions_routes() { + let (mut auth, mut ns, session_a, root_a, grant_a) = fixture(); + let session_b = SessionId::new(1); + let root_b = ns.create_dir_object().unwrap(); + auth.attach_session(session_b).unwrap(); + auth.install_root(session_b, root_b).unwrap(); + + let file = ns.create_file_object().unwrap(); + ns.link(root_a, &Name::new("f").unwrap(), file).unwrap(); + auth.insert_grant( + session_a, + session_b, + grant_a, + file, + Rights::READ, + AuthorityQuota::MAX, + Some(Name::new("gift").unwrap()), + ) + .unwrap(); + + let listed = auth.grants_of(session_b).unwrap(); + assert_eq!(listed.len(), 2, "its private root plus one grant"); + assert!( + listed + .iter() + .any(|(_, kind, _, _, _)| *kind == AuthorityKind::PrivateRoot) + ); + let gift = listed + .iter() + .find(|(_, kind, _, _, _)| *kind == AuthorityKind::Grant) + .unwrap(); + assert_eq!(gift.2, Rights::READ); + assert_eq!(gift.3, AuthorityQuota::MAX); + assert_eq!(gift.4.unwrap().as_str(), "gift"); + } + + #[test] + fn grant_snapshots_refuse_contention_instead_of_hiding_a_live_edge() { + let (auth, ns, session_a, root_a, grant_a) = fixture(); + let session_b = SessionId::new(1); + let root_b = ns.create_dir_object().unwrap(); + auth.attach_session(session_b).unwrap(); + auth.install_root(session_b, root_b).unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(root_a, &Name::new("f").unwrap(), file).unwrap(); + let shared = Name::new("shared").unwrap(); + let gift = auth + .insert_grant( + session_a, + session_b, + grant_a, + file, + Rights::READ, + AuthorityQuota::MAX, + Some(shared), + ) + .unwrap(); + let (edges, local) = auth.edge_slot(gift).unwrap(); + let _writer = edges.try_write(local).unwrap(); + + assert_eq!(auth.grants_of(session_b).unwrap_err(), FsError::Busy); + assert_eq!( + auth.display_name_taken(session_b, &shared), + Err(FsError::Busy) + ); + } + + #[test] + fn task_view_name_claims_are_scoped_per_session() { + let (auth, ns, session_a, _root_a, _grant_a) = fixture(); + let session_b = SessionId::new(1); + let root_b = ns.create_dir_object().unwrap(); + auth.attach_session(session_b).unwrap(); + auth.install_root(session_b, root_b).unwrap(); + + let _a = auth.claim_view(session_a).unwrap(); + let b = auth.claim_view(session_b).unwrap(); + assert!(matches!(auth.claim_view(session_b), Err(FsError::Busy))); + drop(b); + assert!(auth.claim_view(session_b).is_ok()); + } + + #[test] + fn detaching_a_session_drops_its_handles_and_edges() { + let (mut auth, mut ns, session_a, root_a, grant_a) = fixture(); + let session_b = SessionId::new(1); + let root_b = ns.create_dir_object().unwrap(); + auth.attach_session(session_b).unwrap(); + auth.install_root(session_b, root_b).unwrap(); + + let file = ns.create_file_object().unwrap(); + ns.link(root_a, &Name::new("f").unwrap(), file).unwrap(); + let to_b = auth + .insert_grant( + session_a, + session_b, + grant_a, + file, + Rights::READ, + AuthorityQuota::MAX, + None, + ) + .unwrap(); + let a_handle = auth + .insert_handle(session_a, root_a, grant_a, EdgeProof::root(), Rights::ALL) + .unwrap(); + let b_handle = auth + .insert_handle(session_b, file, to_b, EdgeProof::root(), Rights::READ) + .unwrap(); + + let stranded = auth.detach_session(session_b).unwrap(); + // B's own root and its grant were the edges it held. + assert_eq!(stranded.len(), 2); + assert!(stranded.contains(&root_b) && stranded.contains(&file)); + assert!(!auth.is_attached(session_b)); + assert_eq!( + auth.handle(session_b, b_handle).unwrap_err(), + FsError::InvalidHandle + ); + // A is entirely unaffected. + assert!(auth.handle(session_a, a_handle).is_ok()); + assert!(auth.revalidate(session_a, a_handle, &ns).is_ok()); + } + + #[test] + fn contended_session_detach_republishes_every_exact_record_for_retry() { + let (auth, _ns, session, root, grant) = fixture(); + let handle = auth + .insert_handle(session, root, grant, EdgeProof::root(), Rights::ALL) + .unwrap(); + let in_flight = auth.handle(session, handle).unwrap(); + + assert_eq!(auth.detach_session(session), Err(FsError::Busy)); + assert!( + auth.handle(session, handle).is_ok(), + "a refused teardown reopens the same session generation" + ); + drop(in_flight); + + let stranded = auth.detach_session(session).unwrap(); + assert_eq!(stranded.as_slice(), &[root]); + assert_eq!( + auth.handle(session, handle).unwrap_err(), + FsError::InvalidHandle + ); + } + + #[test] + fn detaching_invalidates_grants_derived_from_the_sessions_authority() { + let (mut auth, mut ns, session_a, root_a, grant_a) = fixture(); + let session_b = SessionId::new(1); + let root_b = ns.create_dir_object().unwrap(); + auth.attach_session(session_b).unwrap(); + auth.install_root(session_b, root_b).unwrap(); + + let file = ns.create_file_object().unwrap(); + ns.link(root_a, &Name::new("f").unwrap(), file).unwrap(); + let to_b = auth + .insert_grant( + session_a, + session_b, + grant_a, + file, + Rights::READ, + AuthorityQuota::MAX, + None, + ) + .unwrap(); + + // A's private root was the grant's parent. Once A goes away the + // derived route cannot survive as ambient authority in B. + auth.detach_session(session_a).unwrap(); + assert_eq!(auth.edge(to_b).unwrap_err(), FsError::AccessLost); + } + + #[test] + fn detaching_an_unattached_session_is_refused() { + let mut auth = Authority::new(); + assert_eq!( + auth.detach_session(SessionId::new(0)).unwrap_err(), + FsError::InvalidHandle + ); + } + + #[test] + fn a_full_session_is_refused_without_touching_another() { + let (auth, ns, session_a, root, root_grant) = fixture(); + let session_b = SessionId::new(1); + let root_b = ns.create_dir_object().unwrap(); + auth.attach_session(session_b).unwrap(); + auth.install_root(session_b, root_b).unwrap(); + + let file = ns.create_file_object().unwrap(); + ns.link(root, &Name::new("f").unwrap(), file).unwrap(); + let self_grant = || { + auth.insert_grant( + session_a, + session_a, + root_grant, + file, + Rights::READ, + AuthorityQuota::MAX, + None, + ) + }; + // The root is the first of the session's edges. + for _ in 1..MAX_EDGES_PER_SESSION { + self_grant().unwrap(); + } + assert_eq!(self_grant().unwrap_err(), FsError::OutOfEdges); + assert_eq!(auth.edge_count(), MAX_EDGES_PER_SESSION + 1); + + // The other session's storage is its own: it still takes a grant, + // even one issued by the full session. + auth.insert_grant( + session_a, + session_b, + root_grant, + file, + Rights::READ, + AuthorityQuota::MAX, + None, + ) + .unwrap(); + assert_eq!(auth.edge_count(), MAX_EDGES_PER_SESSION + 2); + } + + /// Attach `session` with a fresh private root. + fn attach(auth: &Authority, ns: &Namespace, session: SessionId) -> (ObjectId, GrantId) { + let root = ns.create_dir_object().unwrap(); + auth.attach_session(session).unwrap(); + let grant = auth.install_root(session, root).unwrap(); + (root, grant) + } + + #[test] + fn every_slot_the_host_can_name_attaches_a_session() { + let ns = Namespace::::new(); + let auth = Authority::new(); + let mut opened = std::vec::Vec::new(); + for slot in 0..MAX_SESSIONS { + let session = SessionId::new(slot as u16); + let (root, grant) = attach(&auth, &ns, session); + let handle = auth + .insert_handle(session, root, grant, EdgeProof::root(), Rights::READ) + .unwrap(); + opened.push((session, handle)); + } + assert_eq!(auth.live_sessions() as usize, MAX_SESSIONS); + assert_eq!(auth.session_arenas(), MAX_SESSIONS); + assert_eq!(auth.edge_count(), MAX_SESSIONS); + for (session, handle) in &opened { + assert_eq!(auth.live_handles(*session), 1); + assert_eq!( + auth.revalidate(*session, *handle, &ns).unwrap().rights, + Rights::READ + ); + } + // Past the slot space there is nothing to attach. + assert_eq!( + auth.attach_session(SessionId::new(MAX_SESSIONS as u16)) + .unwrap_err(), + FsError::Range + ); + } + + #[test] + fn storage_follows_the_sessions_attached_not_the_slot_space() { + let (auth, ns, _session, _root, _grant) = fixture(); + assert_eq!(auth.session_arenas(), 1); + // A slot far up the space costs one arena, not every slot below it. + let far = SessionId::new(crate::bounds::FIRST_TASK_VIEW_SESSION as u16 + 1); + attach(&auth, &ns, far); + assert_eq!(auth.session_arenas(), 2); + // Detaching keeps the slot's arena for its next occupant. + auth.detach_session(far).unwrap(); + assert_eq!(auth.session_arenas(), 2); + attach(&auth, &ns, far); + assert_eq!(auth.session_arenas(), 2); + } + + #[test] + fn a_handle_names_its_sessions_slot_and_no_other() { + let (auth, ns, session_a, root_a, grant_a) = fixture(); + let session_b = SessionId::new(9); + let (root_b, grant_b) = attach(&auth, &ns, session_b); + let handle_a = auth + .insert_handle(session_a, root_a, grant_a, EdgeProof::root(), Rights::READ) + .unwrap(); + let handle_b = auth + .insert_handle(session_b, root_b, grant_b, EdgeProof::root(), Rights::READ) + .unwrap(); + // The same arena position in two sessions is two different handles. + assert_ne!(handle_a, handle_b); + assert_eq!( + auth.handle(session_b, handle_a).unwrap_err(), + FsError::InvalidHandle + ); + assert_eq!( + auth.handle(session_a, handle_b).unwrap_err(), + FsError::InvalidHandle + ); + // An index past every session's slots names nothing. + let forged = Handle::from_key(SlotKey::new(MAX_HANDLE_INDEX as u32, 1)); + assert_eq!( + auth.handle(session_a, forged).unwrap_err(), + FsError::InvalidHandle + ); + assert!(auth.handle(session_a, handle_a).is_ok()); + assert!(auth.handle(session_b, handle_b).is_ok()); + } + + #[test] + fn a_session_parked_in_its_attach_transition_detaches_empty() { + let (auth, ns, _session, _root, _grant) = fixture(); + let parked = SessionId::new(7); + auth.attach_session(parked).unwrap(); + assert!(!auth.is_attached(parked)); + let retired = auth.detach_session(parked).unwrap(); + assert!(retired.is_empty()); + assert_eq!(auth.live_sessions(), 1); + // The slot takes its next occupant, rooted as usual. + let (root, grant) = attach(&auth, &ns, parked); + assert_eq!(auth.root_grant(parked).unwrap(), grant); + assert_eq!(auth.root_object(parked).unwrap(), root); + } + + #[test] + fn a_reused_slot_refuses_its_previous_occupants_ids() { + let (auth, ns, _session, _root, _grant) = fixture(); + let slot = SessionId::new(5); + let (root, grant) = attach(&auth, &ns, slot); + let handle = auth + .insert_handle(slot, root, grant, EdgeProof::root(), Rights::READ) + .unwrap(); + auth.detach_session(slot).unwrap(); + + let (next_root, next_grant) = attach(&auth, &ns, slot); + // The arena slots are reused; their generations moved on. + assert_ne!(next_grant, grant); + assert_eq!( + auth.handle(slot, handle).unwrap_err(), + FsError::InvalidHandle + ); + assert_eq!(auth.edge(grant).unwrap_err(), FsError::AccessLost); + let next = auth + .insert_handle(slot, next_root, next_grant, EdgeProof::root(), Rights::READ) + .unwrap(); + assert_ne!(next, handle); + assert!(auth.revalidate(slot, next, &ns).is_ok()); + } + + #[test] + fn a_lineage_deeper_than_the_bound_is_refused() { + let (auth, ns, session_a, root, root_grant) = fixture(); + let session_b = SessionId::new(1); + attach(&auth, &ns, session_b); + let file = ns.create_file_object().unwrap(); + ns.link(root, &Name::new("f").unwrap(), file).unwrap(); + + // Alternate holders so the chain, not a session's edge count, is + // what runs out: each grant is issued by its parent's holder. + let mut parent = root_grant; + let mut issuer = session_a; + for depth in 2..=MAX_LINEAGE_DEPTH { + let holder = if issuer == session_a { + session_b + } else { + session_a + }; + parent = auth + .insert_grant( + issuer, + holder, + parent, + file, + Rights::READ, + AuthorityQuota::MAX, + None, + ) + .unwrap(); + assert_eq!(auth.validated_edge(parent).unwrap().lineage_len(), depth); + issuer = holder; + } + let holder = if issuer == session_a { + session_b + } else { + session_a + }; + assert_eq!( + auth.insert_grant( + issuer, + holder, + parent, + file, + Rights::READ, + AuthorityQuota::MAX, + None, + ) + .unwrap_err(), + FsError::Range + ); + // The deepest edge still validates its whole lineage. + assert!(auth.validated_edge(parent).is_ok()); + } + + #[test] + fn detaching_retires_a_closure_spread_over_many_sessions_children_first() { + let (auth, ns, session_a, root, root_grant) = fixture(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &Name::new("f").unwrap(), file).unwrap(); + + // A fan of holders, each re-granting to the next: every edge below + // the root descends from session A's authority. + let holders = 24; + let mut parent = root_grant; + let mut issuer = session_a; + let mut descendants = 0; + for index in 1..=holders { + let holder = SessionId::new(index as u16); + attach(&auth, &ns, holder); + let grant = |parent| { + auth.insert_grant( + issuer, + holder, + parent, + file, + Rights::READ, + AuthorityQuota::MAX, + None, + ) + .unwrap() + }; + // A sibling beside the chain, from the same parent, then the + // chain's next link. + grant(parent); + parent = grant(parent); + descendants += 2; + issuer = holder; + } + let deepest = parent; + // Every session's root, plus the chain and its siblings. + assert_eq!(auth.edge_count(), 1 + holders + descendants); + + let retired = auth.detach_session(session_a).unwrap(); + // A's root and every edge descending from it, once each. + assert_eq!(retired.len(), 1 + descendants); + assert_eq!(retired.iter().filter(|object| **object == root).count(), 1); + assert_eq!(auth.edge_count(), holders); + assert_eq!(auth.edge(deepest).unwrap_err(), FsError::AccessLost); + for index in 1..=holders { + assert!(auth.is_attached(SessionId::new(index as u16))); + } + } + + #[test] + fn a_handle_record_is_small_and_its_proof_holds_its_own_depth() { + let (_auth, ns, _session, root, _grant) = fixture(); + let dir = ns.create_dir_object().unwrap(); + let edge = ns.link(root, &Name::new("d").unwrap(), dir).unwrap(); + // The proof is a pointer and a length; the edges live beside it at + // the depth actually walked. + assert!(core::mem::size_of::() <= 64); + let proof = EdgeProof::root().extend(edge).unwrap(); + assert_eq!(proof.edges(), &[edge]); + assert!(SessionArena::BYTES < 8 * 1024); + assert_eq!(SESSION_HEAP_BYTES, MAX_SESSIONS * SessionArena::BYTES); + } +} diff --git a/native/services/dagfs/src/bounds.rs b/native/services/dagfs/src/bounds.rs new file mode 100644 index 0000000..78af75f --- /dev/null +++ b/native/services/dagfs/src/bounds.rs @@ -0,0 +1,70 @@ +//! Capacity and shape constants — the engine's memory budget. +//! +//! The bounds themselves live in the process-side leaf +//! ([`kore_dagfs::bounds`]): a client sizes its window, its page, and its +//! walk from the same constants the engine sizes its arenas from, so the two +//! cannot disagree about what fits. The bounds that stay here are the +//! session ones, because they are derived from the runtime's serving-slot +//! space, which the leaf deliberately does not link. + +pub use kore_dagfs::bounds::*; + +/// Sessions keyed by a serving slot: every slot the serve host can name. A +/// pair's session is its slot (pair *n* → session *n*) — the founded pairs +/// first, then the per-launch faces the host grows past its bootstrap table +/// — so this is the host's whole slot space, [`PeerId::SLOT_CAPACITY`]. +/// Sizing it by the bootstrap table (`MAX_SERVE_PAIRS`) once left six faces +/// beside the two founded pairs: a seventh program that reads files was +/// refused however much memory was free. +/// +/// [`PeerId::SLOT_CAPACITY`]: kore_rt::svc::PeerId::SLOT_CAPACITY +pub const PAIR_SESSIONS: usize = kore_rt::svc::PeerId::SLOT_CAPACITY; + +/// The first stable task-view session: the band directly above every +/// pair's, so no slot the host grows into can name a task view. +pub const FIRST_TASK_VIEW_SESSION: usize = PAIR_SESSIONS; + +/// Maximum authority sessions: one per serving slot, then the stable task +/// views. A command pair is its own session, but its root is a descendant +/// of exactly one task view and disappears when the command exits. +/// +/// A ceiling, not a reservation: a session's storage — its +/// [`MAX_HANDLES_PER_SESSION`] handles and [`MAX_EDGES_PER_SESSION`] edges — +/// is allocated when its slot first attaches, and the host reuses the lowest +/// free slot, so the footprint follows the most sessions ever attached at +/// once. +pub const MAX_SESSIONS: usize = PAIR_SESSIONS + MAX_TASK_VIEWS; + +const _: () = assert!( + MAX_SESSIONS <= u16::MAX as usize + 1, + "every session must be nameable by a SessionId" +); + +#[cfg(test)] +mod tests { + use super::*; + + /// Every session holds at least its own root, and every chain the + /// system builds fits the lineage bound with room to delegate. + #[test] + fn a_session_holds_its_root_and_the_system_chain_fits() { + assert!(MAX_EDGES_PER_SESSION >= 1); + assert!(MAX_LINEAGE_DEPTH > 8); + } + + /// The task views sit above every slot a pair can occupy. + #[test] + fn task_views_sit_above_every_pair_slot() { + assert_eq!(FIRST_TASK_VIEW_SESSION, kore_rt::svc::PeerId::SLOT_CAPACITY); + assert!(FIRST_TASK_VIEW_SESSION >= kore_rt::svc::MAX_SERVE_PAIRS); + assert_eq!(MAX_SESSIONS - FIRST_TASK_VIEW_SESSION, MAX_TASK_VIEWS); + } + + /// The reachability walk's visited row is keyed by directory slot, so it + /// must cover the arena exactly and stay a stack-sized row. + #[test] + fn reach_set_covers_the_directory_arena() { + assert_eq!(crate::reach::ReachSet::CAPACITY, MAX_DIRECTORIES); + assert!(crate::reach::ReachSet::BYTES <= 512); + } +} diff --git a/native/services/dagfs/src/client.rs b/native/services/dagfs/src/client.rs new file mode 100644 index 0000000..a5966aa --- /dev/null +++ b/native/services/dagfs/src/client.rs @@ -0,0 +1,9 @@ +//! The typed filesystem clients. +//! +//! [`RingFs`] (the pair's SPSC rings, on metal) and [`FsClient`] (the +//! synchronous host form over a [`Transport`]) live in the process-side leaf +//! ([`kore_dagfs::client`]): an application holds a client without linking +//! this service. The service re-exports them so its host suites and the +//! serving contract keep one spelling. + +pub use kore_dagfs::client::*; diff --git a/native/services/dagfs/src/digests.rs b/native/services/dagfs/src/digests.rs new file mode 100644 index 0000000..9524243 --- /dev/null +++ b/native/services/dagfs/src/digests.rs @@ -0,0 +1,286 @@ +//! **Content digests, remembered** — the durable format 5 sidecar +//! (`res/STORAGE_TRANSPORT.md` §6). +//! +//! Lending a file version needs the BLAKE3 of exactly the bytes the lease +//! pins, and mount revalidation needs the same digest for every file behind +//! a stored search result. Both used to reread and rehash the whole file — +//! at every boot, for every catalogue member exec seeds, and for every +//! indexed file at mount. A revision's bytes never change, so its digest is +//! a fact worth keeping. +//! +//! [`DigestMemo`] keeps one digest per object, keyed by the revision it was +//! computed over, so a later revision can never be answered with an earlier +//! one's digest. Commits persist the memo into a sidecar extent the root +//! names, and mount loads it back. +//! +//! **The sidecar is derived data, never authority.** It is covered by its +//! own CRC and keyed by revision; a sidecar that fails its CRC, names a +//! revision the image no longer records, or is simply absent is ignored, and +//! the digest is computed again from the bytes. Nothing a sidecar says can +//! make a lease or a search result claim bytes the file does not hold — +//! the worst a stale or damaged sidecar can do is cost one rehash. + +use alloc::vec::Vec; + +use crc::{CRC_32_ISCSI, Crc}; + +use crate::StableObjectId; + +const CRC: Crc = Crc::::new(&CRC_32_ISCSI); + +/// Sidecar magic — "KOREDIG1". +pub const MAGIC: &[u8; 8] = b"KOREDIG1"; +/// Header: magic, record count, CRC of the records. +pub const HEADER_BYTES: usize = 16; +/// One record: stable id, revision, digest. +pub const RECORD_BYTES: usize = 8 + 8 + 32; + +/// One remembered digest. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Entry { + object: u64, + revision: u64, + digest: [u8; 32], +} + +/// Why a sidecar was not loaded. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SidecarError { + /// Not a sidecar, or a count past the capacity. + Shape, + /// The records fail their CRC. + Crc, +} + +/// The remembered digests, one per object, at most `capacity`. +pub struct DigestMemo { + entries: Vec, + capacity: usize, + dirty: bool, +} + +impl DigestMemo { + /// An empty memo for up to `capacity` objects. + pub fn new(capacity: usize) -> Self { + Self { + entries: Vec::new(), + capacity, + dirty: false, + } + } + + /// The digest of `object` at `revision`, if remembered. + pub fn get(&self, object: StableObjectId, revision: u64) -> Option<[u8; 32]> { + self.entries + .iter() + .find(|entry| entry.object == object.get() && entry.revision == revision) + .map(|entry| entry.digest) + } + + /// Remember `digest` for `object` at `revision`, replacing whatever the + /// object had. A full memo keeps what it has: forgetting costs a rehash, + /// never a wrong answer. + pub fn put(&mut self, object: StableObjectId, revision: u64, digest: [u8; 32]) { + let key = object.get(); + if let Some(entry) = self.entries.iter_mut().find(|entry| entry.object == key) { + if entry.revision != revision || entry.digest != digest { + *entry = Entry { + object: key, + revision, + digest, + }; + self.dirty = true; + } + return; + } + if self.entries.len() >= self.capacity || self.entries.try_reserve(1).is_err() { + return; + } + self.entries.push(Entry { + object: key, + revision, + digest, + }); + self.dirty = true; + } + + /// Keep only entries whose object still exists at the remembered + /// revision (`current` answers an object's revision now, `None` for + /// gone). Called before a commit encodes the memo. + pub fn prune(&mut self, mut current: impl FnMut(StableObjectId) -> Option) { + let before = self.entries.len(); + self.entries.retain(|entry| { + StableObjectId::new(entry.object) + .and_then(&mut current) + .is_some_and(|revision| revision == entry.revision) + }); + if self.entries.len() != before { + self.dirty = true; + } + } + + /// Whether the memo changed since it was last encoded or loaded. + pub fn dirty(&self) -> bool { + self.dirty + } + + /// Mark the memo as not yet durable — a commit that encoded it failed, + /// or a loaded sidecar was damaged and must be replaced. + pub fn mark_dirty(&mut self) { + self.dirty = true; + } + + /// Mark the memo durable without encoding it — its emptiness is what + /// the next root records. + pub fn mark_clean(&mut self) { + self.dirty = false; + } + + /// Entries remembered. + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Whether nothing is remembered. + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Bytes the encoded sidecar occupies. + pub fn encoded_bytes(&self) -> usize { + HEADER_BYTES + self.entries.len() * RECORD_BYTES + } + + /// Encode into `dst` (at least [`encoded_bytes`](Self::encoded_bytes) + /// long), sorted by object so the bytes are canonical, and mark the memo + /// clean. Returns the encoded length. + pub fn encode(&mut self, dst: &mut [u8]) -> Option { + let len = self.encoded_bytes(); + let out = dst.get_mut(..len)?; + self.entries.sort_unstable_by_key(|entry| entry.object); + out[..8].copy_from_slice(MAGIC); + out[8..12].copy_from_slice(&(self.entries.len() as u32).to_le_bytes()); + for (index, entry) in self.entries.iter().enumerate() { + let at = HEADER_BYTES + index * RECORD_BYTES; + out[at..at + 8].copy_from_slice(&entry.object.to_le_bytes()); + out[at + 8..at + 16].copy_from_slice(&entry.revision.to_le_bytes()); + out[at + 16..at + 48].copy_from_slice(&entry.digest); + } + let crc = CRC.checksum(&out[HEADER_BYTES..len]); + out[12..16].copy_from_slice(&crc.to_le_bytes()); + self.dirty = false; + Some(len) + } + + /// Load a sidecar's records, replacing the memo's contents and leaving + /// it clean. On any error the memo is left empty — and the caller + /// rehashes on demand, as it would with no sidecar at all. + pub fn load(&mut self, src: &[u8]) -> Result<(), SidecarError> { + self.entries.clear(); + self.dirty = false; + let header = src.get(..HEADER_BYTES).ok_or(SidecarError::Shape)?; + if &header[..8] != MAGIC { + return Err(SidecarError::Shape); + } + let count = u32::from_le_bytes(header[8..12].try_into().expect("four bytes")) as usize; + let crc = u32::from_le_bytes(header[12..16].try_into().expect("four bytes")); + if count > self.capacity { + return Err(SidecarError::Shape); + } + let records = src + .get(HEADER_BYTES..HEADER_BYTES + count * RECORD_BYTES) + .ok_or(SidecarError::Shape)?; + if CRC.checksum(records) != crc { + return Err(SidecarError::Crc); + } + self.entries + .try_reserve(count) + .map_err(|_| SidecarError::Shape)?; + for record in records.chunks_exact(RECORD_BYTES) { + let mut digest = [0u8; 32]; + digest.copy_from_slice(&record[16..48]); + self.entries.push(Entry { + object: u64::from_le_bytes(record[..8].try_into().expect("eight bytes")), + revision: u64::from_le_bytes(record[8..16].try_into().expect("eight bytes")), + digest, + }); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn id(n: u64) -> StableObjectId { + StableObjectId::new(n).unwrap() + } + + #[test] + fn a_digest_answers_only_its_own_revision() { + let mut memo = DigestMemo::new(4); + assert!(!memo.dirty()); + memo.put(id(7), 3, [1; 32]); + assert!(memo.dirty()); + assert_eq!(memo.get(id(7), 3), Some([1; 32])); + assert_eq!(memo.get(id(7), 4), None, "a later revision is never answered"); + memo.put(id(7), 4, [2; 32]); + assert_eq!(memo.get(id(7), 3), None, "one entry per object"); + assert_eq!(memo.get(id(7), 4), Some([2; 32])); + assert_eq!(memo.len(), 1); + } + + #[test] + fn a_full_memo_forgets_rather_than_answers_wrongly() { + let mut memo = DigestMemo::new(2); + memo.put(id(1), 1, [1; 32]); + memo.put(id(2), 1, [2; 32]); + memo.put(id(3), 1, [3; 32]); + assert_eq!(memo.len(), 2); + assert_eq!(memo.get(id(3), 1), None); + } + + #[test] + fn prune_keeps_only_live_objects_at_their_remembered_revision() { + let mut memo = DigestMemo::new(4); + memo.put(id(1), 1, [1; 32]); + memo.put(id(2), 5, [2; 32]); + memo.put(id(3), 9, [3; 32]); + let mut encoded = [0u8; 256]; + memo.encode(&mut encoded).unwrap(); + assert!(!memo.dirty()); + memo.prune(|object| match object.get() { + 1 => Some(1), + 2 => Some(6), + _ => None, + }); + assert!(memo.dirty()); + assert_eq!(memo.len(), 1); + assert_eq!(memo.get(id(1), 1), Some([1; 32])); + } + + #[test] + fn a_sidecar_round_trips_and_a_damaged_one_loads_nothing() { + let mut memo = DigestMemo::new(8); + memo.put(id(9), 2, [9; 32]); + memo.put(id(4), 7, [4; 32]); + let mut bytes = [0u8; 512]; + let len = memo.encode(&mut bytes).unwrap(); + assert_eq!(len, HEADER_BYTES + 2 * RECORD_BYTES); + + let mut back = DigestMemo::new(8); + back.load(&bytes[..len]).unwrap(); + assert_eq!(back.get(id(9), 2), Some([9; 32])); + assert_eq!(back.get(id(4), 7), Some([4; 32])); + assert!(!back.dirty()); + + let mut flipped = bytes; + flipped[HEADER_BYTES + 20] ^= 1; + assert_eq!(back.load(&flipped[..len]), Err(SidecarError::Crc)); + assert!(back.is_empty(), "a damaged sidecar leaves nothing to trust"); + assert_eq!(back.load(&[0u8; 32]), Err(SidecarError::Shape)); + let mut small = DigestMemo::new(1); + assert_eq!(small.load(&bytes[..len]), Err(SidecarError::Shape), "past capacity"); + } +} diff --git a/native/services/dagfs/src/durable.rs b/native/services/dagfs/src/durable.rs new file mode 100644 index 0000000..adf8b89 --- /dev/null +++ b/native/services/dagfs/src/durable.rs @@ -0,0 +1,4475 @@ +//! Production filesystem composition over a provisioned direct NVMe channel. +//! +//! Two production postures land at this one entry seam: +//! +//! - **Completion pacing.** The class declaration asks for one optional +//! MSI-family message; when the platform realizes it, the I/O completion +//! queue signals that message and every completion wait parks the task on the +//! held route ([`kore_driver::host::RoutePacer`]) with a deadline read +//! from the founder-published clock page. A machine that realizes no +//! route still bounds every wait by that same clock deadline +//! ([`kore_driver::host::DeadlinePacer`]) — it polls instead of parking, +//! but what it is bounded BY is elapsed time either way. Only a machine +//! whose clock page cannot be read at all falls back to the spin budget, +//! because there is then no time source to bound against. +//! - **The designated regions.** A whole disk is only written when it is +//! provably kore's (recognized roots) or provably nobody's (all-blank +//! roots). A disk carrying foreign metadata is consulted read-only for a +//! conservative user partition ([`kore_driver::gpt::KORE_FS_TYPE`]) and a +//! rebuildable system partition ([`kore_driver::gpt::KORE_SYSTEM_TYPE`]). +//! Foreign or stale-era bytes remain evidence on the user side; only the +//! measured archive era authorizes replacement inside the system clamp. +//! Two affine [`EngineLane`]s over one [`EngineBackend`] — each lane a ring +//! on the storage transport engine (`res/STORAGE_TRANSPORT.md`) — feed +//! independent engines, each mounted behind an exact [`RegionDevice`] +//! clamp. The frontend tags +//! engine-local authority words and exposes one unchanged filesystem row. +//! No user designation still means the bounded RAM posture; no system +//! designation keeps the compatibility store on that user engine. +//! - **Client rings.** A session may attach its own ring on the same engine +//! (`res/STORAGE_TRANSPORT.md` §4, slice S4): a window it funds and lends, +//! into which the row installs a read grant over the +//! [`kore_driver::gpt::KORE_PACK_TYPE`] partition, so that immutable image +//! streams into client-funded memory at queue depth with no call per read. +//! Absent a designation the row mounts exactly as before and a pack grant +//! answers `Unsupported`. + +use alloc::boxed::Box; +use alloc::vec::Vec; + +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +use core::sync::atomic::Ordering; +use kore_driver::block::{ + BlockChannel, BlockError, CompletionMode, DeviceCommand, DeviceCompletion, DirectBlockDevice, + RegionDevice, +}; +use kore_driver::block::refresh::{self, Bucket, RegionRefresh}; +use kore_driver::stor::engine::ThreeClass; +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +use kore_driver::stor::engine::TargetAccess; +use kore_driver::stor::engine::RingId; +use kore_abi::stor::GrantId; +use kore_driver::stor::grant::{ClassSet, ExtentRun, Grant, Rights}; +use kore_driver::stor::lanes::{EngineBackend, EngineLane, LaneClasses}; +use kore_driver::block::{CompletionPacer, SpinPacer}; +use kore_driver::gpt; +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +use kore_driver::host::RoutePacer; +use kore_driver::host::{ + BoundDevice, CapBuf, CapPool, CapRegs, ClassQuiesce, DeadlinePacer, DeviceBinding, + DeviceResources, DmaActive, DmaPrepared, +}; +use kore_driver::host::QuietRetire; +use kore_driver::seam::{Quiet, Regs}; +use kore_nvme::{ + PreparedController, + block::{NvmeBlockDevice, NvmeBuffer}, + prp::MemPage, +}; +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +use kore_rt::hint; +use kore_rt::service::{PeerClose, ReturnMode, Serve, StartFailure, SystemService}; +use kore_rt::svc::SvcProtocol; +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +use kore_rt::svc::{ClientEndpoints, GrantPin, SvcClient, client_rings}; +use kore_rt::svc::{PeerAdmissionRefusal, PeerId, PeerMapPlan}; +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +use kore_svc_block::{BlockClient, BlockRefusal, BlockService, ProvisionRequest, RingBlock}; + + +use crate::persistence::{PersistentFs, VolumeProbe, bootstrap_data_start}; +use crate::{ + FsEndowment, FsError, FsRequest, FsResponse, FsService, ProviderError, RamService, SessionId, + persistence::{CommitStage, PersistenceError, PersistenceState, commit_engine}, + provider::BlockProvider, + router::{self, RoutedRequest, Store}, + window::DataWindow, +}; + +/// Boot-critical durable-storage budget. `CAP.TO` is the controller's +/// worst-case legal transition time, not permission for an optional service +/// to hold the first shell for up to 127.5 seconds. We honor the advertised +/// value up to this policy ceiling, then degrade to the volatile filesystem. +const BRING_UP_BUDGET_NS: u64 = 1_000_000_000; +const BRING_UP_FALLBACK_POLLS: u32 = kore_driver::block::MAX_POLLS; + +enum BringUpPacer { + Deadline(DeadlinePacer), + Spin(SpinPacer), +} + +impl CompletionPacer for BringUpPacer { + fn pace(&mut self) -> bool { + match self { + Self::Deadline(pacer) => pacer.pace(), + Self::Spin(pacer) => pacer.pace(), + } + } +} + +fn bring_up_deadline(caps: kore_nvme::Capabilities) -> Option { + let now = kore_rt::clock::uptime().ok()?.as_nanos(); + Some(now.saturating_add(bring_up_budget_ns(caps))) +} + +const fn bring_up_budget_ns(caps: kore_nvme::Capabilities) -> u64 { + let advertised = caps.ready_timeout_ns(); + if advertised == 0 { + BRING_UP_BUDGET_NS + } else if advertised < BRING_UP_BUDGET_NS { + advertised + } else { + BRING_UP_BUDGET_NS + } +} + +fn bring_up_pacer(deadline_ns: Option) -> BringUpPacer { + match deadline_ns { + Some(deadline_ns) => BringUpPacer::Deadline(DeadlinePacer::new(deadline_ns)), + None => BringUpPacer::Spin(SpinPacer::new(BRING_UP_FALLBACK_POLLS)), + } +} +/// The one I/O queue's depth, shared by every ring. The engine's +/// [`ENGINE_COMMANDS`] fit with room (client rings stream at full depth while +/// the filesystem lanes keep theirs), and the controller clamps it to its own +/// `CAP.MQES`. The ring costs 40 KiB of the pool at this depth. +const IO_QDEPTH: u16 = 512; +/// Submission queues the filesystem's controller runs: one per service class +/// (Demand, Stream, Commit — `res/STORAGE_TRANSPORT.md` §3), all bound to one +/// completion queue on the one interrupt vector the binding realizes. +const FS_QUEUES: usize = 3; +/// Each class queue's depth: past the largest class budget (Stream's 128 +/// commands) with room for the ring's one reserved slot. The completion +/// queue holds all three (480 entries), below the single queue's 512. +const FS_SQ_DEPTH: u16 = 160; +const _: () = assert!(FS_QUEUES * FS_SQ_DEPTH as usize <= IO_QDEPTH as usize); +/// The filesystem's controller: a three-queue group. +type FsNvme = NvmeBlockDevice; +/// Capability-backed DMA bytes requested from the block provisioner — a +/// mirror of the block row's `DMA_POOL@1` (the founder's LARGE class), which +/// is subleased whole; the request is an intent, not a size. +const FS_DMA_POOL_BYTES: usize = 0x800_0000; +/// The filesystem's OWN share of that pool: the queue/control graph plus the +/// four cumulatively allocated buffers of a whole-disk -> region rebuild. +/// Held at the ordinary 2 MiB class the row used to be founded with, so the +/// page ceiling below does not move with the pool; everything above it is +/// the controller's host memory buffer. +const FS_IO_DMA_BYTES: usize = 0x20_0000; +/// Largest controller page whose complete queue/control graph plus the four +/// cumulatively allocated buffers of a whole-disk -> region rebuild fits the +/// filesystem's 2 MiB share. At 512 KiB, the four page-aligned queue starts +/// consume so much padding that even the first filesystem buffer cannot be +/// admitted. +const MAX_FS_NVME_PAGE_BYTES: u64 = (FS_IO_DMA_BYTES / 8) as u64; +/// Most host memory the controller is offered for its mapping tables +/// (Identify HMPRE decides how much of it is taken; 0 there takes none). +/// 64 MiB is what a DRAM-less PCIe 4 consumer part asks for and what Linux +/// grants it by default; the pool leaves room for it beside the +/// filesystem's own share. +const FS_HMB_MAX_BYTES: u64 = 0x400_0000; +const _: () = assert!( + FS_IO_DMA_BYTES as u64 + FS_HMB_MAX_BYTES <= FS_DMA_POOL_BYTES as u64, + "the filesystem's share and the host memory buffer must fit the founded pool" +); + +const fn fs_nvme_page_supported(page: MemPage) -> bool { + page.bytes() <= MAX_FS_NVME_PAGE_BYTES +} + +/// Completion-wait budget for one route-parked block operation, in +/// nanoseconds — one **round** of the wait, not the device timeout. It is +/// the interrupt-loss recovery cadence: on expiry the wait's final poll +/// catches a completion whose interrupt never arrived, at a cost of one +/// second rather than the whole stall ceiling. A healthy controller that +/// has simply not finished is granted further rounds by the provider +/// (`provider::block::STALL_BUDGETS`, 30 s in all — Linux's I/O timeout), +/// because a DRAM-less consumer drive under garbage collection is slow for +/// seconds and was being refused, and the filesystem operation on top of it +/// failed, on a budget written for a device that answers in microseconds. +/// Read fresh per wait from the founder-published clock page, so it needs +/// no executor anchor. +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +const OP_BUDGET_NS: u64 = 1_000_000_000; + +/// **How long one monitored turn may hold this core** — the nested fast tier +/// under the route-less wait, sized to nothing like its container. +/// +/// The route-less posture has no interrupt and therefore nothing that could +/// ever deliver a wake, so its wait must find the completion itself. It used +/// to do that by spinning until [`OP_BUDGET_NS`]. An address monitor +/// ([`kore_rt::hint::wait_on`] over the CQ dword the controller's DMA stores +/// to) finds it strictly better on both axes that matter: the core idles in +/// `C0.1` instead of retiring `PAUSE`, and the wake is the completion's own +/// store rather than the next poll. +/// +/// **The sizing is the whole design, and it is a ratio, not a duration.** +/// `UMWAIT` parks a core *without yielding it to the scheduler*, so a +/// monitored wait sized like the device budget would hold a core for a full +/// second and starve every runnable task on it — a nested bound sized like the +/// thing it nests inside, which is a defect this tree has already measured at +/// a 2.3x gate regression elsewhere. It is exactly +/// [`kore_driver::host::PROBE_PAUSE`], the cadence at which the outer elapsed +/// deadline pacer rechecks its clock. A shorter turn would add wake/re-arm +/// work without tightening that outer observation; a longer turn would make +/// the deadline coarser and hold this core past the shared probe policy. +/// +/// A healthy NVMe command completes in tens of microseconds, so a healthy wait +/// spends a handful of these turns rather than one. That is the better shape, +/// not a compromise: every turn is the same `C0.1` idle, so the power posture +/// is unchanged, and each one re-reads the clock and so honours the outer +/// deadline more tightly than a single long sleep could. +/// +/// [`kore_rt::hint::wait_on`]: ../../kore_rt/hint/fn.wait_on.html +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +const MONITOR_CAP: core::time::Duration = core::time::Duration::from_micros(4); +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +const BLOCK_CLIENT_INDEX: u16 = ::SERVE_PAIRS as u16; + +pub struct NvmeFsBinding; + +pub struct NvmeFsPrepared { + controller: PreparedController, + vector: Option, + pacer: BringUpPacer, +} + +pub struct NvmeFsBound { + backend: Option, + resources: DeviceResources, +} + +impl DeviceBinding for NvmeFsBinding { + type Pci = kore_svc_block::service::NvmeIoPci; + type Config = (); + type Prepared = NvmeFsPrepared; + type Bound = NvmeFsBound; + type Error = BlockError; + + fn prepare( + resources: &mut DeviceResources, + (): Self::Config, + ) -> Result { + // Read before the BAR moves into the controller: whether the intake + // realized the declared vector decides the CQ's interrupt posture. + let vector = resources.route().is_some().then_some(0); + let regs = resources.take_bar(0).ok_or(BlockError::Transport)?; + let pool = resources.take_pool().ok_or(BlockError::Backpressure)?; + let caps = kore_nvme::Capabilities::from_cap(regs.read64(kore_nvme::regs::REG_CAP)); + let mut pacer = bring_up_pacer(bring_up_deadline(caps)); + let controller = kore_nvme::Controller::prepare_paced(regs, pool, &mut pacer) + .map_err(|_| BlockError::Transport)?; + if !fs_nvme_page_supported(controller.page()) { + let (regs, pool) = controller.into_seam(); + resources.restore_bar(0, regs); + resources.restore_pool(pool); + return Err(BlockError::Geometry); + } + Ok(NvmeFsPrepared { + controller, + vector, + pacer, + }) + } + + fn start( + prepared: Self::Prepared, + mut resources: DeviceResources, + ) -> Result { + let NvmeFsPrepared { + controller, + vector, + mut pacer, + } = prepared; + // Weighted round robin where the controller offers it: the class + // queues' priorities (Demand urgent, Commit high, Stream medium) + // then govern what the controller fetches next. + let controller = match controller.with_weighted_arbitration().start_paced(&mut pacer) { + Ok(controller) => controller, + Err(failure) => { + let (_error, regs, pool) = failure.into_seam(); + resources.restore_bar(0, regs); + resources.restore_pool(pool); + return Err(BlockError::Transport); + } + }; + // The CQ signals MSI-X table entry 0 — the same entry the realized + // route remapped — iff the route exists; a route-less bind creates + // a silent CQ and the completion wait stays the bounded spin. + let backend = FsNvme::provision_queues_paced( + controller, + 1, + FS_SQ_DEPTH, + vector, + FS_HMB_MAX_BYTES, + &mut pacer, + kore_driver::host::relax, + BRING_UP_FALLBACK_POLLS, + )?; + Ok(NvmeFsBound { + backend: Some(backend), + resources, + }) + } + + fn quiesce(bound: &mut Self::Bound) -> Result { + bound.backend.as_mut().ok_or(BlockError::Lost)?.quiesce()?; + Ok(ClassQuiesce::Stopped) + } + + fn unbind(mut bound: Self::Bound) -> DeviceResources { + let backend = bound + .backend + .take() + .expect("NVMe filesystem binding lost backend"); + let (regs, pool) = backend.into_seam(); + bound.resources.restore_bar(0, regs); + bound.resources.restore_pool(pool); + bound.resources + } +} + +/// The direct backend value owned by one filesystem worker. The common host +/// retains all ordinary capability children and performs ordered containment. +pub struct DirectNvme { + /// `None` only after the terminal teardown (#809 §L) consumed it; every + /// lane has retired by then, so no I/O path can observe the absence. + bound: Option>, + /// Route parks issued by the completion wait (the wait's denominator). + route_parks: u64, + /// Waits that took the bounded spin because the clock could supply no + /// deadline at all — the only posture left with no time source. Counted + /// because it is otherwise invisible: such a machine bounds its disk + /// waits by retired instructions and still reports `route=held`. + spin_waits: u64, + /// Parks resolved as real route fires — the MSI-X-landed witness + /// (`0` with growing parks = a delivery-dead machine running on the + /// final-poll fallback). + route_fires: u64, + /// Waits bounded by a clock deadline with no route to park on — the + /// route-less posture on a machine whose clock *is* readable. Counted + /// apart from `spin_waits` because the two are no longer the same + /// fallback: this one is bounded by elapsed time, that one only by + /// retired instructions. + deadline_waits: u64, + /// Route-less turns that actually slept the core on the CQ line — the + /// monitored fast tier's numerator. + /// + /// The one fact that cannot be inferred from anything else on the box. No + /// emulator implements a user-mode monitor (QEMU's TCG decodes neither + /// `UMWAIT` nor `MWAITX`), so under the boot gates this is *always* zero + /// while `monitor_turns` climbs — and a metal run that also reported zero + /// would look identical to a green gate. Without this counter, "the fast + /// path is live on this machine" and "the fast path silently degraded to a + /// spin" are the same observation. + monitor_sleeps: u64, + /// Route-less turns that reached the monitored tier at all — its + /// denominator, counted whether or not the machine could sleep. + /// + /// `sleeps / turns` is the ratio a metal A/B is read by; `turns` alone is + /// how many times a route-less wait went round, which is the latency + /// signal the old spin loop never exposed. + monitor_turns: u64, + /// Targets whose stage-1 release was refused. They stay owned and + /// unreachable instead of poisoning unrelated targets or the filesystem + /// lanes. Terminal containment retires their import owners. + quarantined_targets: Vec, + /// Commands staged through the queue-set face and not yet reaped — so + /// a park never sleeps on a device holding nothing. + in_flight: usize, + /// The completion a park took from the route wait, reaped first. + parked: Option, + /// What imported-window retirements still owe. + retirements: RetirementBooks, +} + +/// What this controller's imported-window retirements still owe +/// (`res/LIFECYCLE_HARDENING.md` §6). +/// +/// A released window that has not retired yet holds its own IOVA range and +/// owner, and no command names it: nothing a new import uses. So only a +/// failed requester-wide retirement, which leaves the latch unproven, refuses +/// new client windows. A deferred window is retried: live (its engine's +/// `Quiet` proof still stands) before the next adoption, and whole at every +/// idle pass. One stuck window used to refuse every later client until a +/// reboot. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct RetirementBooks { + /// Released windows the host still owns. + deferred: usize, + /// The idle-boundary retirement failed: the requester's latch is + /// unproven until a later idle pass succeeds. + failed: bool, +} + +impl RetirementBooks { + /// Whether an idle pass has anything to do. + fn owed(&self) -> bool { + self.failed || self.deferred != 0 + } + + /// An idle-boundary pass: it retires every released window and replays + /// the latch, so its outcome replaces the failure. + fn settle_idle(&mut self, healthy: bool, deferred: usize) { + self.failed = !healthy; + self.deferred = deferred; + } + + /// A live pass retires only proven windows and never touches the latch: + /// it cannot clear a failure, only add one. + fn settle_live(&mut self, healthy: bool, deferred: usize) { + self.failed |= !healthy; + self.deferred = deferred; + } + + /// The refusal a new client window meets, if any. + fn admission(&self) -> Result<(), ProviderError> { + if self.failed { + Err(ProviderError::Retiring) + } else { + Ok(()) + } + } +} + +impl DirectNvme { + #[cfg(all(target_arch = "x86_64", target_os = "none"))] + fn claim(opened: kore_svc_block::OpenedChannel) -> Result { + let bound = BoundDevice::::claim(opened.device, opened.pool) + .map_err(|_| BlockError::Transport)?; + Ok(Self { + bound: Some(bound), + route_parks: 0, + spin_waits: 0, + route_fires: 0, + deadline_waits: 0, + monitor_sleeps: 0, + monitor_turns: 0, + quarantined_targets: Vec::new(), + in_flight: 0, + parked: None, + retirements: RetirementBooks::default(), + }) + } + + /// Whether the intake realized the optional MSI-X route — the + /// completion-pacing and `enable_irq_parks` decider. + pub fn has_route(&self) -> bool { + self.bound + .as_ref() + .is_some_and(|bound| bound.parts().resources.route().is_some()) + } + + /// The live binding. Unreachable after the terminal teardown: it runs + /// only once every lane has retired, and a lane is the only I/O door. + fn bound(&self) -> &BoundDevice { + self.bound + .as_ref() + .expect("terminal teardown consumed the binding under a live lane") + } + + fn bound_mut(&mut self) -> &mut BoundDevice { + self.bound + .as_mut() + .expect("terminal teardown consumed the binding under a live lane") + } + + /// The terminal teardown (#809 §L): class stop (NVMe `CC.SHN` already + /// completed by the last lane's quiesce), the kernel's fabric proof, then + /// containment. `None` if already torn down. + fn teardown(&mut self) -> Option> { + self.bound.take().map(BoundDevice::teardown) + } + + /// Host memory the controller holds for its mapping tables, in bytes + /// (0 = none granted). + pub fn host_memory_buffer_bytes(&self) -> u64 { + self.backend().host_memory_buffer_bytes() + } + + /// The completion wait's books: `(parks, fires, deadline waits, spin + /// waits, monitored sleeps, monitored turns)`. + pub fn route_stats(&self) -> (u64, u64, u64, u64, u64, u64) { + ( + self.route_parks, + self.route_fires, + self.deadline_waits, + self.spin_waits, + self.monitor_sleeps, + self.monitor_turns, + ) + } + + fn backend(&self) -> &FsNvme { + self.bound() + .parts() + .backend + .as_ref() + .expect("live binding has backend") + } + + /// Map one client-lent fabric route into this controller's DMA domain + /// and wrap it as an adoptable ring target (`import_dma`, the path the + /// accelerator rows already take for client leases). + /// + /// Reached through [`EngineBackend::control`]: an import maps new memory + /// into the domain and touches nothing a command in flight uses. The + /// controller only ever *writes* a target, but `import_dma`'s + /// `device_write` is exactly that admission. + #[cfg(all(target_arch = "x86_64", target_os = "none"))] + /// A refusal carries a diagnostic code beside its error: an + /// [`IntakeError`] code for a refused import, [`ADOPT_REFUSED`] with the + /// block error for a refused adoption. + /// + /// [`IntakeError`]: kore_driver::host::IntakeError + fn adopt_target( + &mut self, + owner: kore_rt::cap::ReturnedCapability, + ) -> Result, (FsError, u32)> { + // Only a failed requester-wide retirement refuses the import; the + // retry that clears it needs the controller idle + // ([`Self::retry_retirement`]). + if let Err(cause) = self.retirements.admission() { + return Err(( + FsError::ProviderFailure(cause), + ADOPT_REFUSED | (BlockError::Backpressure as u32) << 8, + )); + } + // A route that is not a read-write device route is refused here and + // dropped: the filesystem mints no reply grants, and the client still + // holds the parent it scoped this child from. + let import = kore_driver::host::DmaImport::claim( + owner, + kore_abi::session::FabricAccess::READ | kore_abi::session::FabricAccess::WRITE, + ) + .map_err(|_| (FsError::AccessDenied, CLAIM_REFUSED))?; + let parts = self.bound_mut().parts_mut(); + let data = match import { + // A lent lease takes the accelerators' import: mapped here too, + // so it costs this service window, but it is the path the Xe and + // NPU clients prove on metal — the control for a route that + // never completes. + lease @ kore_driver::host::DmaImport::Lease(_) => { + parts.resources.import_dma(lease, true) + } + // The lane only ever points the controller at a target, so + // nothing of it is mapped into this service: a multi-GiB model + // arena must not cost the filesystem row address space or page + // tables. + route => parts.resources.import_dma_device(route, true), + } + .map_err(|error| { + ( + FsError::ProviderFailure(ProviderError::Transport), + error.diagnostic_code() as u32, + ) + })?; + let backend = parts.backend.as_mut().expect("live binding has backend"); + match backend.adopt_buffer(data) { + Ok(buffer) => Ok(buffer), + Err((data, error)) => { + // Nothing was submitted against it; hand it straight back. + let buffer = NvmeBuffer::from_data(data); + self.release_target(buffer); + Err(( + FsError::ProviderFailure(crate::provider::map_block(error)), + ADOPT_REFUSED | (error as u32) << 8, + )) + } + } + } + + /// Return a target the controller refused before any command could name + /// it: the allocation goes back to the pool, and its device mapping + /// retires at the next idle pass. Reached with lanes possibly busy, so it + /// never claims the requester idle itself. + fn release_target(&mut self, buffer: NvmeBuffer) { + let parts = self.bound_mut().parts_mut(); + let backend = parts.backend.as_ref().expect("live binding has backend"); + // SAFETY: the channel handed this target back only with nothing in + // flight against it, and `control_idle` admitted this call only with + // no command in flight on any lane, so the controller holds no + // reference. + let released = unsafe { backend.release_adopted(buffer) }; + if let Err(kore_driver::seam::DmaReleaseFailure::Refused(data)) = released { + // This target failed its own stage-1 retirement invariant. Keep + // its exact owner unreachable; other imported mappings and the + // filesystem's established windows remain valid. + if self.quarantined_targets.try_reserve(1).is_ok() { + self.quarantined_targets.push(data); + } else { + core::mem::forget(data); + } + return; + } + self.retirements.deferred = parts.resources.deferred_retirements(); + } + + /// Retire one target the engine handed back with its proof, while the + /// controller runs on: only the imported window's device mapping goes, + /// the latch stays open for every other window + /// (`res/STORAGE_TRANSPORT.md` §11). A live retirement left unfinished + /// falls to the idle boundary ([`retry_retirement`](Self::retry_retirement)). + /// `true` once the device mapping is gone. + fn retire_quiet(&mut self, target: Quiet>) -> bool { + let parts = self.bound_mut().parts_mut(); + match parts.resources.retire_quiet(target.map(NvmeBuffer::into_data)) { + QuietRetire::Retired => true, + QuietRetire::Deferred => { + self.retirements.deferred = parts.resources.deferred_retirements(); + false + } + QuietRetire::Refused(data) => { + // Not a live import of this binding: keep its owner + // unreachable, as a refused stage-1 release does. + if self.quarantined_targets.try_reserve(1).is_ok() { + self.quarantined_targets.push(data); + } else { + core::mem::forget(data); + } + false + } + } + } + + /// Retry every incomplete retirement. Reached only through + /// [`EngineBackend::control_idle`] — an ordinary release needs the whole + /// requester quiescent. + #[cfg_attr(not(all(target_arch = "x86_64", target_os = "none")), allow(dead_code))] + fn retry_retirement(&mut self) { + if !self.retirements.owed() { + return; + } + let resources = &self.bound().parts().resources; + let healthy = resources.resume_retired_dma(true); + let deferred = resources.deferred_retirements(); + self.retirements.settle_idle(healthy, deferred); + } + + /// Retry the live retirements alone, with lanes running: each window + /// still carries its engine's proof. + #[cfg_attr(not(all(target_arch = "x86_64", target_os = "none")), allow(dead_code))] + fn retry_live_retirement(&mut self) { + if self.retirements.deferred == 0 { + return; + } + let resources = &self.bound().parts().resources; + let healthy = resources.resume_retired_dma(false); + let deferred = resources.deferred_retirements(); + self.retirements.settle_live(healthy, deferred); + } + + fn retirement_owed(&self) -> bool { + self.retirements.owed() + } + + fn backend_mut(&mut self) -> &mut FsNvme { + self.bound_mut() + .parts_mut() + .backend + .as_mut() + .expect("live binding has backend") + } +} + +/// The controller as the transport engine drives it: the group's class +/// queues, with this wrapper's route-aware wait as the park. Named by path, +/// never imported, so `DirectNvme`'s device methods keep their names. +impl kore_driver::stor::QueueSet for DirectNvme { + type Target = NvmeBuffer; + const SQS: usize = FS_QUEUES; + + fn geometry(&self) -> kore_driver::block::BlockGeometry { + DirectBlockDevice::geometry(self) + } + + fn target_bytes(target: &Self::Target) -> u64 { + kore_driver::block::BlockBuffer::len(target) as u64 + } + + fn allocate(&mut self, bytes: usize) -> Result { + DirectBlockDevice::allocate_buffer(self, bytes) + } + + fn max_target_bytes(&self) -> usize { + DirectBlockDevice::max_buffer_bytes(self) + } + + fn park(&mut self) -> bool { + if self.parked.is_some() { + return true; + } + if self.in_flight == 0 { + return false; + } + self.parked = DirectBlockDevice::wait(self); + self.parked.is_some() + } + + fn room(&self, sq: usize) -> usize { + self.backend().room_on(sq) + } + + fn stage( + &mut self, + sq: usize, + command: DeviceCommand, + target: Option<&mut Self::Target>, + tag: kore_driver::stor::Tag, + ) -> Result<(), BlockError> { + self.backend_mut().stage_on(sq, command, target, tag.cookie())?; + self.in_flight += 1; + Ok(()) + } + + fn abort(&mut self, sq: usize, tag: kore_driver::stor::Tag) -> bool { + self.backend_mut().abort_on(sq, tag.cookie()) + } + + fn reset(&mut self) -> bool { + if !kore_driver::block::DirectBlockDevice::reset(self.backend_mut()) { + return false; + } + // Nothing staged survives: no completion is owed, none is parked. + self.in_flight = 0; + self.parked = None; + true + } + + fn publish(&mut self, sq: usize) { + self.backend_mut().publish_on(sq); + } + + fn reap(&mut self, out: &mut [DeviceCompletion]) -> usize { + let mut count = 0; + if let Some(first) = out.first_mut() + && let Some(parked) = self.parked.take() + { + *first = parked; + count = 1; + } + while count < out.len() { + let Some(completion) = DirectBlockDevice::poll(self) else { + break; + }; + out[count] = completion; + count += 1; + } + self.in_flight = self.in_flight.saturating_sub(count); + count + } + + fn liveness(&mut self) -> kore_driver::block::Liveness { + DirectBlockDevice::liveness(self) + } + + fn health(&mut self) -> Result { + DirectBlockDevice::health(self) + } + + fn quiesce(&mut self) -> Result<(), BlockError> { + if self.in_flight != 0 || self.parked.is_some() { + return Err(BlockError::Draining); + } + DirectBlockDevice::quiesce(self) + } + + fn clock_ns(&self) -> Option { + DirectBlockDevice::clock_ns(self) + } +} + +impl DirectBlockDevice for DirectNvme { + type Buffer = NvmeBuffer; + + /// The controller's queues advance on their own — inherited from the + /// backend this wrapper only adds route-aware completion pacing to. It + /// is what lets one binding be shared by the two designated regions. + const INDEPENDENT_PROGRESS: bool = + ::INDEPENDENT_PROGRESS; + + fn geometry(&self) -> kore_driver::block::BlockGeometry { + self.backend().geometry() + } + + fn allocate_buffer(&mut self, bytes: usize) -> Result { + self.backend_mut().allocate_buffer(bytes) + } + + fn max_buffer_bytes(&self) -> usize { + self.backend().max_buffer_bytes() + } + + fn health(&mut self) -> Result { + self.backend_mut().health() + } + + fn liveness(&mut self) -> kore_driver::block::Liveness { + self.backend_mut().liveness() + } + + /// The founder-published clock page, so the channel above books every + /// completion's latency at the depth it was driven — `res/RESIDENCY.md` + /// §8's "NVMe latency at queue depth", measured on this box. An + /// uncalibrated page reads as no clock and measures nothing. + #[cfg(all(target_arch = "x86_64", target_os = "none"))] + fn clock_ns(&self) -> Option { + kore_rt::clock::uptime().ok().map(|now| now.as_nanos()) + } + + fn submit( + &mut self, + command: DeviceCommand, + buffer: Option<&mut Self::Buffer>, + cookie: u64, + ) -> Result<(), BlockError> { + self.backend_mut().submit(command, buffer, cookie) + } + + fn stage( + &mut self, + command: DeviceCommand, + buffer: Option<&mut Self::Buffer>, + cookie: u64, + ) -> Result<(), BlockError> { + self.backend_mut().stage(command, buffer, cookie) + } + + fn publish(&mut self) { + self.backend_mut().publish(); + } + + fn poll(&mut self) -> Option { + self.backend_mut().poll() + } + + fn poll_batch(&mut self, out: &mut [Option]) -> usize { + self.backend_mut().poll_batch(out) + } + + fn quiesce(&mut self) -> Result<(), BlockError> { + self.backend_mut().quiesce() + } + + /// The production completion wait: park on the held MSI-X route with a + /// deadline read fresh from the founder-published clock page, so the + /// CPU is yielded while the command is in flight. Route-less bindings + /// — and an uncalibrated clock page — keep the trait's bounded-spin + /// posture; either way the final poll catches a completion that landed + /// without a delivered interrupt. + #[cfg(all(target_arch = "x86_64", target_os = "none"))] + fn wait(&mut self) -> Option { + // A field borrow, not `bound_mut()`: the wait books below write other + // fields of `self` while `parts` is live. + let parts = self + .bound + .as_mut() + .expect("terminal teardown consumed the binding under a live lane") + .parts_mut(); + let backend = parts.backend.as_mut().expect("live binding has backend"); + let route = parts.resources.route(); + let deadline_ns = kore_rt::clock::uptime() + .ok() + .map(|now| now.as_nanos().saturating_add(OP_BUDGET_NS)); + let route = match (route, deadline_ns) { + (Some(route), Some(deadline_ns)) => Some((route, deadline_ns)), + // No route, but the clock answers: this wait is still bounded by + // ELAPSED TIME. It used to take the spin budget, which bounds + // retired instructions instead — so a slow device could exhaust + // it in a duration nobody chose, and the provider treats an + // expired wait as evidence to declare the channel lost. A + // route-less machine is the degraded posture; it is the worst + // place to answer a slow disk by declaring it dead. + (None, Some(deadline_ns)) => { + self.deadline_waits = self.deadline_waits.saturating_add(1); + let mut pacer = DeadlinePacer::new(deadline_ns); + let mut sleeps = 0u64; + let mut turns = 0u64; + // **The monitored fast tier, nested inside the clocked bound.** + // This posture has no route, so nothing exists that could ever + // deliver a wake — but a completion IS a store: the controller + // DMAs the CQE and flips its phase bit. So this core can sleep + // on that very word and be woken by the silicon, needing no + // interrupt and no waker. Per turn, capped at `MONITOR_CAP` + // (microseconds) and NOT at `deadline_ns` (one second): a + // monitored sleep does not yield the core, so a turn sized like + // the container would starve every runnable task on it. + let completion = loop { + if let Some(completion) = backend.poll() { + break Some(completion); + } + turns += 1; + let slept = match backend.completion_word() { + // `Relaxed` is sufficient and is not a shortcut: this + // value is a comparison token for `wait_on`, never a + // synchronization edge. Reading it stale costs one + // turn slept against an event that already landed; + // reading it fresh costs one turn not slept. The edge + // that publishes the CQE's other fifteen bytes is the + // acquire fence inside `CompRing::poll`, which the + // poll at the top of this loop still takes. + Some(word) => { + let stale = word.load(Ordering::Relaxed); + hint::wait_on(word, stale, MONITOR_CAP) == hint::WaitEnd::Monitored + } + // No monitor expressible on this ring; the outer tier + // is unchanged and still correct. + None => false, + }; + // A turn that genuinely slept has already bought its wait, + // and bought a better one — a store ends it immediately + // where a spin batch only notices at its end. Spinning + // again would pay twice and hand back the power the sleep + // just saved. Either way the bound is checked on the same + // counter, so an expired wait still means elapsed time. + let within = if slept { + sleeps += 1; + pacer.pace_after_wait() + } else { + pacer.pace() + }; + if !within { + break backend.poll(); + } + }; + self.monitor_sleeps = self.monitor_sleeps.saturating_add(sleeps); + self.monitor_turns = self.monitor_turns.saturating_add(turns); + return completion; + } + // No clock at all — no time source exists to bound against, so + // the spin budget is the honest last resort rather than a choice. + // Counted, because a machine waiting on an instruction count is a + // fact that appears nowhere else. + (_, None) => { + self.spin_waits = self.spin_waits.saturating_add(1); + let mut pacer = SpinPacer::default(); + loop { + if let Some(completion) = backend.poll() { + return Some(completion); + } + if !pacer.pace() { + return backend.poll(); + } + } + } + }; + let (route, deadline_ns) = route.expect("both arms above return"); + let mut pacer = RoutePacer::new(route, deadline_ns); + let completion = loop { + if let Some(completion) = backend.poll() { + break Some(completion); + } + if !pacer.pace() { + break backend.poll(); + } + }; + self.route_parks = self.route_parks.saturating_add(pacer.parks()); + self.route_fires = self.route_fires.saturating_add(pacer.fired()); + completion + } +} + +/// The user and system lanes over the one controller. Every other reader +/// of the disk — the pack region's streams among them — is a client ring. +const FS_LANES: usize = 2; +/// The pack region's maintenance lane (`res/PACK_STREAMING.md` §7), after +/// the filesystem lanes: the row's own scan and content-preserving refresh +/// of the pack, clamped to its extent. Claimed only when a session first +/// asks for maintenance, so a row that never maintains its pack is the row +/// it was before — and an unclaimed lane never holds the controller up. +const PACK_LANE: usize = FS_LANES; +/// Every lane the row drives itself. +const ROW_LANES: usize = FS_LANES + 1; +/// Commands one lane keeps in flight, and the depth one class of client +/// rings is budgeted. A Nemotron token's routed rows are thousands of 4 KiB +/// reads, and a DRAM-less NVMe reaches its random-read rate only past a +/// hundred or so in flight: at 32 the first chat on the Lunar Lake box +/// streamed them at 280 MB/s against a region that does 5 GB/s +/// sequentially. The driver's adopted list-page pool is sized to match, so +/// full-depth streaming of large records is not throttled by list pages. +const LANE_INFLIGHT: usize = 128; +/// Commands the transport engine keeps on the controller at once: both +/// filesystem lanes' full depth and as much again for the client rings, +/// below the queue's own so the controller never refuses a staged command. +/// The classes' budgets, not this table, share it out; the pack +/// maintenance lane rides the Stream budget beside the client rings, and +/// holds at most a handful of commands (`kore_driver::block::refresh::SLOTS` +/// chunks and a flush). +const ENGINE_COMMANDS: usize = (FS_LANES + 1) * LANE_INFLIGHT; +const _: () = assert!(ENGINE_COMMANDS < IO_QDEPTH as usize); +/// Submissions the engine holds open across every ring. +const ENGINE_OPS: usize = ENGINE_COMMANDS; +/// Targets one ring can hold: a lane's block-channel buffers, or a +/// client's destinations (a model arena and its expert-cache chunks). +const RING_TARGETS: usize = 64; +/// The controller as the transport engine drives it (`res/STORAGE_TRANSPORT.md` +/// §3, slice S2): one submission queue per class over one completion queue +/// and one interrupt vector, every lane a ring on the engine. +type NvmeQueues = DirectNvme; +/// Client rings the engine serves beside the lanes (`res/STORAGE_TRANSPORT.md` +/// §9, slice S4): one per filesystem session that attaches one. +const STOR_CLIENTS: usize = 4; +/// Ring slots: the lanes first, then the client rings. +const ENGINE_RINGS: usize = ROW_LANES + STOR_CLIENTS; +/// Grants one ring holds: a lane's whole-namespace grant, or a client's +/// pack grant — with room for the file grants reads move onto (S5). +const ENGINE_GRANTS: usize = 4; +/// The engine the backend borrows, placed on the heap in the start path. +#[cfg_attr(not(all(target_arch = "x86_64", target_os = "none")), allow(dead_code))] +type NvmeEngine = kore_driver::stor::engine::StorEngine< + 'static, + NvmeQueues, + ThreeClass, + ENGINE_RINGS, + ENGINE_GRANTS, + RING_TARGETS, + ENGINE_OPS, + ENGINE_COMMANDS, +>; +type SharedNvme = EngineBackend< + 'static, + NvmeQueues, + ThreeClass, + ROW_LANES, + ENGINE_RINGS, + ENGINE_GRANTS, + RING_TARGETS, + ENGINE_OPS, + ENGINE_COMMANDS, +>; +// The backend is built in the start path, inside the serve future on a +// main stack the row nearly fills; its engine tables (over 100 KiB) live on +// the heap, placed by `StorEngine::init_in`, so the backend itself must stay +// small. Growing it past this is a stack-budget change, not a refactor. +const _: () = assert!(core::mem::size_of::() <= 4096); + +/// Each lane's class map: the filesystem lanes read on Demand and write on +/// Commit, so their metadata and reads fill ahead of client rings' bulk +/// Stream reads, which stay inside the Stream budget. Pack maintenance is +/// bulk background work: everything it issues rides Stream, behind every +/// read a thread is blocked on. +#[cfg_attr(not(all(target_arch = "x86_64", target_os = "none")), allow(dead_code))] +const LANE_CLASSES: [LaneClasses; ROW_LANES] = [ + LaneClasses::FILESYSTEM, + LaneClasses::FILESYSTEM, + LaneClasses::STREAM, +]; + +/// One lane's transport window: page-aligned, owned for the row's lifetime +/// (the engine and the lane's client both borrow it `'static`, as the lanes +/// borrow the backend). +#[cfg_attr(not(all(target_arch = "x86_64", target_os = "none")), allow(dead_code))] +fn lane_window() -> Option<&'static mut [u8]> { + let bytes = kore_ring::stor::WINDOW_BYTES; + let layout = core::alloc::Layout::from_size_align(bytes, 4096).ok()?; + // SAFETY: the layout is non-zero; the allocation is leaked on purpose — + // it lives as long as the row, like the backend that borrows it. + let base = unsafe { alloc::alloc::alloc_zeroed(layout) }; + let base = core::ptr::NonNull::new(base)?; + // SAFETY: freshly allocated, `bytes` long, exclusively this caller's. + Some(unsafe { core::slice::from_raw_parts_mut(base.as_ptr(), bytes) }) +} + +/// Diagnostic classes a refused target import carries beside its error, +/// above the [`IntakeError`](kore_driver::host::IntakeError) range (1..=11), +/// with the `BlockError` in the next byte where one applies: the lent +/// capability was not a memory import, or the backend refused to adopt the +/// imported window. +const CLAIM_REFUSED: u32 = 0xc1; +const ADOPT_REFUSED: u32 = 0xc2; + +type SharedLane = EngineLane< + 'static, + 'static, + NvmeQueues, + ThreeClass, + ROW_LANES, + ENGINE_RINGS, + ENGINE_GRANTS, + RING_TARGETS, + ENGINE_OPS, + ENGINE_COMMANDS, +>; +type Provider = BlockProvider, 8, 32>; + +/// Buckets the pack region is judged and refreshed in. A 40 GiB pack makes +/// 40 MiB buckets and a scan of 16 384 probes, about two seconds of idle +/// device time; the report is one 8-byte record per bucket, 8 KiB beside +/// a 64 KiB data window. A finer table costs scan time linearly. +const PACK_BUCKETS: usize = 1024; + +/// The pack region's scan-and-refresh job, on [`PACK_LANE`] clamped to the +/// pack's extent. +type PackJob = RegionRefresh<'static, RegionDevice>; + +/// Where the pack maintenance lane stands. +#[cfg_attr(not(all(target_arch = "x86_64", target_os = "none")), allow(dead_code))] +enum PackMaintenance { + /// No session has asked yet: the lane is unclaimed. + Unclaimed, + Live(Box), + /// The lane was claimed and could not carry a job (an extent the clamp + /// refused, no buffer memory), or it was retired at quiesce: every ask + /// answers `Unsupported` for the rest of this boot. + Unavailable, +} + +/// One engine and its independent persistence generation. The user side may +/// deliberately be volatile; a system region is installed only after its +/// exactly-clamped volume has recovered at, or been rebuilt for, the measured +/// archive era. +#[cfg_attr(not(all(target_arch = "x86_64", target_os = "none")), allow(dead_code))] +enum MountedEngine { + Durable { + fs: Box>, + mount: PersistenceState, + commits: u64, + }, + Volatile(Box), +} + +#[cfg_attr(not(all(target_arch = "x86_64", target_os = "none")), allow(dead_code))] +impl MountedEngine { + fn configure_peer_map(&mut self, map: PeerMapPlan) { + match self { + Self::Durable { fs, .. } => fs.configure_peer_map(map), + Self::Volatile(fs) => fs.configure_peer_map(map), + } + } + + fn establish_admin_namespace(&mut self) -> bool { + match self { + Self::Durable { fs, .. } => fs.establish_admin_namespace(), + Self::Volatile(fs) => fs.establish_admin_namespace(), + } + } + + fn establish_admin_source(&mut self) -> bool { + match self { + Self::Durable { fs, .. } => fs.establish_admin_source(), + Self::Volatile(fs) => fs.establish_admin_source(), + } + } + + fn establish_admin_apps(&mut self) -> bool { + match self { + Self::Durable { fs, .. } => fs.establish_admin_apps(), + Self::Volatile(fs) => fs.establish_admin_apps(), + } + } + + fn establish_admin_libs(&mut self) -> bool { + match self { + Self::Durable { fs, .. } => fs.establish_admin_libs(), + Self::Volatile(fs) => fs.establish_admin_libs(), + } + } + + fn establish_admin_share(&mut self) -> bool { + match self { + Self::Durable { fs, .. } => fs.establish_admin_share(), + Self::Volatile(fs) => fs.establish_admin_share(), + } + } + + fn establish_admin_pkgs(&mut self) -> bool { + match self { + Self::Durable { fs, .. } => fs.establish_admin_pkgs(), + Self::Volatile(fs) => fs.establish_admin_pkgs(), + } + } + + fn establish_admin_workspace(&mut self) -> bool { + match self { + Self::Durable { fs, .. } => fs.establish_admin_workspace(), + Self::Volatile(fs) => fs.establish_admin_workspace(), + } + } + + fn peer_admitted(&mut self, result: Result) -> bool { + match self { + Self::Durable { fs, .. } => fs.peer_admitted(result), + Self::Volatile(fs) => fs.peer_admitted(result), + } + } + + fn mirror_peer_admitted(&mut self, peer: PeerId) -> Result<(), PeerAdmissionRefusal> { + match self { + Self::Durable { fs, .. } => fs.mirror_peer_admitted(peer), + Self::Volatile(fs) => fs.mirror_peer_admitted(peer), + } + } + + fn peer_closed(&mut self, via: u16, reason: PeerClose) -> bool { + match self { + Self::Durable { fs, .. } => fs.peer_closed(via, reason), + Self::Volatile(fs) => fs.peer_closed(via, reason), + } + } + + fn session_for_via(&self, via: u16) -> Option { + match self { + Self::Durable { fs, .. } => fs.session_for_via(via), + Self::Volatile(fs) => fs.session_for_via(via), + } + } + + fn ensure_dynamic_session(&mut self, session: SessionId) -> bool { + match self { + Self::Durable { fs, .. } => fs.ensure_dynamic_session(session), + Self::Volatile(fs) => fs.ensure_dynamic_session(session), + } + } + + fn install_apps_view(&mut self, session: SessionId) -> bool { + match self { + Self::Durable { fs, .. } => fs.install_apps_view(session), + Self::Volatile(fs) => fs.install_apps_view(session), + } + } + + fn install_libs_view(&mut self, session: SessionId) -> bool { + match self { + Self::Durable { fs, .. } => fs.install_libs_view(session), + Self::Volatile(fs) => fs.install_libs_view(session), + } + } + + fn install_share_view(&mut self, session: SessionId) -> bool { + match self { + Self::Durable { fs, .. } => fs.install_share_view(session), + Self::Volatile(fs) => fs.install_share_view(session), + } + } + + fn mapped_window_span(&self, session: SessionId) -> Option { + match self { + Self::Durable { fs, .. } => fs.mapped_window_span(session), + Self::Volatile(fs) => fs.mapped_window_span(session), + } + } + + async fn serve_request_async( + &mut self, + request: FsRequest, + sv: &mut Serve<'_, FsResponse>, + ) -> FsResponse { + let session = self.session_for_via(sv.via()); + let response = match self { + Self::Durable { fs, .. } => fs.serve_request_async(request, sv).await, + Self::Volatile(fs) => fs.serve_request_async(request, sv).await, + }; + self.commit_after(session, &request, response).await + } + + async fn dispatch_async( + &mut self, + session: SessionId, + request: &FsRequest, + window: Option<&mut DataWindow<'_>>, + ) -> FsResponse { + let response = match self { + Self::Durable { fs, .. } => fs.dispatch_async(session, request, window).await, + Self::Volatile(fs) => fs.dispatch_async(session, request, window).await, + }; + self.commit_after(Some(session), request, response).await + } + + async fn commit_after( + &mut self, + session: Option, + request: &FsRequest, + mut response: FsResponse, + ) -> FsResponse { + let Self::Durable { fs, mount, commits } = self else { + return response; + }; + let deferring = session.is_some_and(|session| fs.is_deferring(session)); + let closing = matches!(request, FsRequest::DeferCommits { hold: false }) + && !matches!(response, FsResponse::Refused(_)); + if (DurableService::needs_commit(request, &response) && !deferring) || closing { + if let Err(error) = + commit_engine(fs.engine(), mount, SessionId::new(0), None::).await + { + response = FsResponse::Refused(DurableService::persistence_error(error)); + } else { + *commits = commits.saturating_add(1); + } + } + response + } + + fn degraded(&self) -> bool { + matches!( + self, + Self::Durable { mount, .. } + if mount.status().mode == crate::persistence::MountMode::DegradedReadOnly + ) + } + + fn durable(&self) -> bool { + matches!(self, Self::Durable { .. }) + } +} + +/// One client-visible filesystem row over a conservative user engine and an +/// optional rebuildable system engine. The router is the only layer that sees +/// both; each engine retains its original session and authority model. +#[cfg_attr(not(all(target_arch = "x86_64", target_os = "none")), allow(dead_code))] +struct MountedFs { + user: MountedEngine, + system: Option, + /// A GPT system designation suppresses the legacy user-volume fallback + /// even if its engine could not be made current. Otherwise a failed era + /// repair would redirect seeding into user-owned bytes. + system_designated: bool, + /// Independent logical-region lifecycle facts. They deliberately do not + /// live on the shared block backend: one controller can host a refused + /// user region beside a rebuilt system region (or the reverse). + user_lifecycle: RegionLifecycle, + system_lifecycle: RegionLifecycle, + /// Stable post-seed posture restored after exec closes its catalogue + /// verification/copy window. + system_seed_resume: RegionLifecycle, +} + +impl MountedFs { + fn set_system_seed_active(&mut self, via: u16, active: bool) -> Result<(), FsError> { + // Authorize the fixed pair itself, not a derived session number: pair + // one is exec's founder-installed administration capability, while + // pair zero is the legacy boot client and recyclable pairs are mapped + // dynamically. + if via != crate::service::contract::ADMIN_VIA { + return Err(FsError::AccessDenied); + } + if !self.system_designated || self.system.is_none() { + return Ok(()); + } + if active { + if self.system_lifecycle != RegionLifecycle::Reseeding { + self.system_seed_resume = self.system_lifecycle; + self.system_lifecycle = RegionLifecycle::Reseeding; + } + } else if self.system_lifecycle == RegionLifecycle::Reseeding { + self.system_lifecycle = self.system_seed_resume; + } + Ok(()) + } + + fn configure_peer_map(&mut self, map: PeerMapPlan) { + self.user.configure_peer_map(map); + if let Some(system) = self.system.as_mut() { + system.configure_peer_map(map); + } + } + + fn establish_boot_policy(&mut self) -> Result<(), u64> { + if !self.user.establish_admin_namespace() { + return Err(19); + } + if !self.user.establish_admin_workspace() { + return Err(24); + } + match self.system.as_mut() { + Some(system) => { + if !system.establish_admin_namespace() || !system.establish_admin_source() { + return Err(19); + } + if !system.establish_admin_apps() { + return Err(25); + } + if !system.establish_admin_libs() { + return Err(26); + } + if !system.establish_admin_pkgs() { + return Err(27); + } + if !system.establish_admin_share() { + return Err(31); + } + } + None if !self.system_designated => { + if !self.user.establish_admin_source() { + return Err(19); + } + if !self.user.establish_admin_apps() { + return Err(25); + } + if !self.user.establish_admin_libs() { + return Err(26); + } + if !self.user.establish_admin_pkgs() { + return Err(27); + } + if !self.user.establish_admin_share() { + return Err(31); + } + } + // A designated system extent exists but is unavailable. Publish + // neither system grant: exec observes an unavailable store and a + // later boot can retry the bounded repair. The user workspace is + // already live and remains the only mutable namespace. + None => {} + } + Ok(()) + } + + fn has_system(&self) -> bool { + self.system.is_some() + } + + fn durable(&self) -> bool { + self.user.durable() || self.system.as_ref().is_some_and(MountedEngine::durable) + } + + fn degraded(&self) -> bool { + self.user.degraded() || self.system.as_ref().is_some_and(MountedEngine::degraded) + } + + fn peer_admitted(&mut self, result: Result) -> bool { + let Ok(peer) = result else { + return self.user.peer_admitted(result); + }; + if let Some(system) = self.system.as_mut() + && let Err(refusal) = system.mirror_peer_admitted(peer) + { + // Publish the secondary's refusal through the primary engine's + // one admission completion slot — the secondary owns no + // control-plane reply — so a system store whose heap is full + // reads as the full table it is. + let _ = self.user.peer_admitted(Err(refusal)); + return false; + } + let accepted = self.user.peer_admitted(Ok(peer)); + if !accepted && let Some(system) = self.system.as_mut() { + let _ = system.peer_closed(peer.arrival(), PeerClose::Died); + } + accepted + } + + fn peer_closed(&mut self, via: u16, reason: PeerClose) -> bool { + let user = self.user.peer_closed(via, reason); + let system = self + .system + .as_mut() + .is_none_or(|system| system.peer_closed(via, reason)); + user && system + } + + fn window_for<'a>(span: crate::service::MappedWindowSpan) -> DataWindow<'a> { + // SAFETY: the primary frontend retains the affine returned-capability + // owner for `span`; `ServeHost` serializes this row, and every window + // produced here is consumed before another is made. The fabricated + // lifetime cannot escape this private function's immediate caller. + DataWindow::new(unsafe { core::slice::from_raw_parts_mut(span.base.as_ptr(), span.len) }) + } + + fn session(&self, via: u16) -> Result { + self.user + .session_for_via(via) + .ok_or(FsResponse::Refused(FsError::AccessDenied)) + } + + async fn serve_request_async( + &mut self, + request: FsRequest, + sv: &mut Serve<'_, FsResponse>, + ) -> FsResponse { + let routed = match router::route(request, self.has_system()) { + Ok(routed) => routed, + Err(error) => return FsResponse::Refused(error), + }; + match routed { + RoutedRequest::One(Store::User, request) => { + let applied_peer = match request { + FsRequest::ApplyView { peer, .. } => Some(peer), + _ => None, + }; + let response = self.user.serve_request_async(request, sv).await; + // The user view is the terminal's structural root. The + // secondary engine mirrors only the explicit read-only apps + // projection; failure is the established best-effort posture + // (the terminal still boots without a store view). + if !matches!(response, FsResponse::Refused(_)) + && let (Some(peer), Some(system)) = (applied_peer, self.system.as_mut()) + { + let _ = system.install_apps_view(SessionId::new(peer.slot() as u16)); + let _ = system.install_libs_view(SessionId::new(peer.slot() as u16)); + let _ = system.install_share_view(SessionId::new(peer.slot() as u16)); + } + router::tag_response(Store::User, response) + } + RoutedRequest::One(Store::System, request) => { + let session = match self + .system + .as_ref() + .and_then(|system| system.session_for_via(sv.via())) + { + Some(session) => session, + None => return FsResponse::Refused(FsError::AccessDenied), + }; + let host_control = matches!( + request, + FsRequest::BindView { .. } + | FsRequest::GrantView { .. } + | FsRequest::MatchView { .. } + ); + let span = request + .needs_window() + .then(|| self.user.mapped_window_span(session)) + .flatten(); + let Some(system) = self.system.as_mut() else { + return FsResponse::Refused(FsError::InvalidHandle); + }; + let response = if host_control { + system.serve_request_async(request, sv).await + } else { + if !system.ensure_dynamic_session(session) { + return FsResponse::Refused(FsError::OutOfObjects); + } + let mut window = span.map(Self::window_for); + system + .dispatch_async(session, &request, window.as_mut()) + .await + }; + router::tag_response(Store::System, response) + } + RoutedRequest::Both(request @ FsRequest::AttachWindow { .. }) => { + let session = match self.session(sv.via()) { + Ok(session) => session, + Err(response) => return response, + }; + let response = self.user.serve_request_async(request, sv).await; + if matches!(response, FsResponse::Refused(_)) { + return response; + } + if let Some(system) = self.system.as_mut() { + if !system.ensure_dynamic_session(session) { + return FsResponse::Refused(FsError::OutOfObjects); + } + let mirrored = system.dispatch_async(session, &request, None).await; + if matches!(mirrored, FsResponse::Refused(_)) { + return mirrored; + } + // One window serves both regions, so one limit must hold + // for both: the smaller. + if let ( + FsResponse::WindowAttached { max_transfer: user }, + FsResponse::WindowAttached { + max_transfer: system, + }, + ) = (response, mirrored) + { + return FsResponse::WindowAttached { + max_transfer: user.min(system), + }; + } + } + response + } + RoutedRequest::Both(request @ FsRequest::DeferCommits { .. }) => { + let session = match self.session(sv.via()) { + Ok(session) => session, + Err(response) => return response, + }; + if !self.user.ensure_dynamic_session(session) { + return FsResponse::Refused(FsError::OutOfObjects); + } + let user = self.user.dispatch_async(session, &request, None).await; + if matches!(user, FsResponse::Refused(_)) { + return user; + } + if let Some(system) = self.system.as_mut() { + if !system.ensure_dynamic_session(session) { + return FsResponse::Refused(FsError::OutOfObjects); + } + let system = system.dispatch_async(session, &request, None).await; + if matches!(system, FsResponse::Refused(_)) { + return system; + } + } + FsResponse::Done + } + RoutedRequest::ListGrants { window_offset } => { + let session = match self.session(sv.via()) { + Ok(session) => session, + Err(response) => return response, + }; + if !self.user.ensure_dynamic_session(session) { + return FsResponse::Refused(FsError::OutOfObjects); + } + let Some(span) = self.user.mapped_window_span(session) else { + return FsResponse::Refused(FsError::NoWindow); + }; + let request = FsRequest::ListGrants { window_offset }; + let mut window = Self::window_for(span); + let user = self + .user + .dispatch_async(session, &request, Some(&mut window)) + .await; + let FsResponse::GrantsListed { + entries: user_entries, + } = user + else { + return user; + }; + if let Err(error) = router::tag_grant_entries( + &mut window, + window_offset, + 0, + user_entries as usize, + Store::User, + ) { + return FsResponse::Refused(error); + } + let Some(system) = self.system.as_mut() else { + return FsResponse::GrantsListed { + entries: user_entries, + }; + }; + if !system.ensure_dynamic_session(session) { + return FsResponse::Refused(FsError::OutOfObjects); + } + let system_offset = match router::grant_append_offset(window_offset, user_entries) { + Ok(offset) => offset, + Err(error) => return FsResponse::Refused(error), + }; + let request = FsRequest::ListGrants { + window_offset: system_offset, + }; + let response = system + .dispatch_async(session, &request, Some(&mut window)) + .await; + let FsResponse::GrantsListed { + entries: system_entries, + } = response + else { + return response; + }; + if let Err(error) = router::tag_grant_entries( + &mut window, + system_offset, + 0, + system_entries as usize, + Store::System, + ) { + return FsResponse::Refused(error); + } + match user_entries.checked_add(system_entries) { + Some(entries) => FsResponse::GrantsListed { entries }, + None => FsResponse::Refused(FsError::Range), + } + } + RoutedRequest::SystemStatus { window_offset } => { + let session = match self.session(sv.via()) { + Ok(session) => session, + Err(response) => return response, + }; + if !self.user.ensure_dynamic_session(session) { + return FsResponse::Refused(FsError::OutOfObjects); + } + let Some(span) = self.user.mapped_window_span(session) else { + return FsResponse::Refused(FsError::NoWindow); + }; + let request = FsRequest::SystemStatus { window_offset }; + let mut window = Self::window_for(span); + let response = self + .user + .dispatch_async(session, &request, Some(&mut window)) + .await; + if !matches!(response, FsResponse::SystemStatusReported { .. }) { + return response; + } + let user = match router::read_status(&window, window_offset) { + Ok(status) => status, + Err(error) => return FsResponse::Refused(error), + }; + let Some(system) = self.system.as_mut() else { + return response; + }; + if !system.ensure_dynamic_session(session) { + return FsResponse::Refused(FsError::OutOfObjects); + } + let response = system + .dispatch_async(session, &request, Some(&mut window)) + .await; + if !matches!(response, FsResponse::SystemStatusReported { .. }) { + return response; + } + let system = match router::read_status(&window, window_offset) { + Ok(status) => status, + Err(error) => return FsResponse::Refused(error), + }; + let folded = router::fold_status(user, system); + match window.put_system_status(window_offset, &folded) { + Ok(()) => response, + Err(error) => FsResponse::Refused(error), + } + } + RoutedRequest::SystemSeed { active } => { + if let Err(error) = self.set_system_seed_active(sv.via(), active) { + return FsResponse::Refused(error); + } + FsResponse::Done + } + RoutedRequest::Both(_) => FsResponse::Refused(FsError::Unsupported), + // `DurableService::serve_async` answers the ring verbs before the + // engines are consulted; reaching here means this row has none. + RoutedRequest::Stor(_) => FsResponse::Refused(FsError::Unsupported), + } + } +} + +/// The production filesystem row. `None` exists only during the bounded +/// pre-readiness acquisition of its generic block endpoint. +pub struct DurableService { + fs: Option, + /// The one physical backend both lanes borrow — remembered at mount so + /// the terminal path (#809 §L) can reach the binding after both lanes + /// have retired. `None` in the volatile posture. + shared: Option<&'static SharedNvme>, + peer_map: Option, + /// The measured image-archive digest founder stamped as immutable boot + /// data. Only the already-clamped system locator may interpret it as + /// destructive rebuild policy. + archive_era: [u8; crate::persistence::ERA_BYTES], + route_held: bool, + /// The readiness note ([`note`]) — `0` until initialize decides. + note: u64, + /// The pack region, for the read grants client rings are given. + pack_region: Option, + /// The pack region's maintenance (`res/PACK_STREAMING.md` §7). + pack: PackMaintenance, + /// Client rings, at most one per session. + rings: [Option; STOR_CLIENTS], +} + +/// The durable row's readiness-note map — the swap row's discipline: a +/// note-less ready line is the fully healthy posture (durable mount, MSI-X +/// route held), and every degradation names itself on the beacon, because +/// on a serial-less machine that line is the only place the fact appears. +pub mod note { + /// Volatile: the whole disk is foreign and its GPT designates no kore + /// filesystem partition — the undesignated-machine posture. + pub const VOLATILE_UNDESIGNATED: u64 = 1; + /// Volatile: a designated partition exists but was refused — foreign + /// bytes inside it, an impossible extent, or too small to mount. + pub const VOLATILE_REFUSED_REGION: u64 = 2; + /// Durable, but no MSI-X route was realized: the polled posture. + pub const DURABLE_POLLED: u64 = 3; + /// Durable, but recovery used the fallback root: degraded read-only. + pub const DURABLE_DEGRADED: u64 = 4; + /// The designated system store could not be made current. The user engine + /// remains mounted; system grants are not published. + pub const SYSTEM_UNAVAILABLE: u64 = 5; + /// The designated system store was rebuilt for the measured archive era. + /// Seeding remains an explicit exec transaction after publication. + pub const SYSTEM_REBUILT: u64 = 6; +} + +/// One logical region's durable lifecycle, separate from the shared physical +/// controller counters and from the row's scalar readiness note. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RegionLifecycle { + Unknown, + Undesignated, + Refused, + Reformatting, + Reseeding, + Unavailable, + Rebuilt, + Current, +} + +impl RegionLifecycle { + const fn wire(self) -> u16 { + use kore_abi::svc::FsSystemStatusWire; + match self { + Self::Unknown => FsSystemStatusWire::LIFECYCLE_UNKNOWN, + Self::Undesignated => FsSystemStatusWire::LIFECYCLE_UNDESIGNATED, + Self::Refused => FsSystemStatusWire::LIFECYCLE_REFUSED, + Self::Reformatting => FsSystemStatusWire::LIFECYCLE_REFORMATTING, + Self::Reseeding => FsSystemStatusWire::LIFECYCLE_RESEEDING, + Self::Unavailable => FsSystemStatusWire::LIFECYCLE_UNAVAILABLE, + Self::Rebuilt => FsSystemStatusWire::LIFECYCLE_REBUILT, + Self::Current => FsSystemStatusWire::LIFECYCLE_CURRENT, + } + } +} + +/// Where the media classification landed: a provider to mount at the +/// stated posture, or the deliberate RAM fallback after exact containment +/// (carrying the readiness note that names why). +/// +/// Public, with [`DurableService::locate`] and +/// [`DurableService::whole_medium_provider`], so that a host tool working on +/// an offline disk image (`tools/kore-vol`) reaches exactly the region, and +/// takes exactly the posture, the next boot will: one decision procedure, +/// not a second copy of it. +pub enum Located { + /// A kore volume: recover it. + Recover(BlockProvider, B, O>), + /// Blank roots: format the region, stamped with `era`. + Format { + provider: BlockProvider, B, O>, + era: [u8; crate::persistence::ERA_BYTES], + }, + /// Reachable only from [`DurableService::locate_system`], after the + /// `KORE_SYSTEM_TYPE` extent is beneath the provider's clamp. + Reformat { + provider: BlockProvider, B, O>, + era: [u8; crate::persistence::ERA_BYTES], + }, + /// No durable region: the RAM posture, with the [`note`] that says why. + Volatile(u64), +} + +impl DurableService { + /// Consult a foreign disk's GPT — read-only — for the partition whose + /// type GUID designates the kore filesystem. Every failure (no table, + /// damaged table, no match, an entry array outside the provider's + /// guarded metadata window) answers `None`: without an exact + /// designation there is no write authority anywhere on this disk. + async fn discover_region( + provider: &BlockProvider, B, O>, + type_guid: gpt::Guid, + ) -> Option { + let geometry = provider.geometry(); + let block = geometry.block_size as usize; + if block > 4096 { + return None; + } + let mut header = [0u8; 4096]; + provider + .read_metadata(gpt::PRIMARY_HEADER_LBA, &mut header[..block]) + .await + .ok()?; + let table = gpt::parse_header(&header[..block]).ok()?; + let array_bytes = table.entry_array_bytes(); + let blocks = array_bytes.div_ceil(block); + let end = table.entry_lba.checked_add(blocks as u64)?; + if end > crate::provider::GPT_DISCOVERY_END_LBA { + return None; + } + let len = blocks * block; + let mut entries = [0u8; gpt::MAX_ENTRY_ARRAY_BYTES]; + provider + .read_metadata(table.entry_lba, &mut entries[..len]) + .await + .ok()?; + // The type GUID is a parameter, not a constant, because issue #626 + // discovers TWO kore regions on one disk: the user region + // ([`gpt::KORE_FS_TYPE`]) and the self-healing system store + // ([`gpt::KORE_SYSTEM_TYPE`]). GPT partitions never overlap, so the two + // extents this returns for the two type GUIDs are disjoint by + // construction — the foundation of the destructive-ops containment the + // system formatter relies on. + gpt::find_partition(&entries[..len], &table, type_guid).ok() + } + + /// The provider over a whole medium, as the row composes it before it + /// classifies anything: the device behind an identity [`RegionDevice`] + /// (which [`Self::locate`] may narrow onto a designated partition), its + /// channel, and the current profile's bootstrap data start. Detail 14 + /// is a device or channel the composition refuses, 15 a geometry with no + /// room for the metadata reservation. + pub fn whole_medium_provider( + device: D, + mode: CompletionMode, + ) -> Result, B, O>, StartFailure> { + let device = RegionDevice::whole(device).map_err(|_| StartFailure::new(14))?; + let channel = BlockChannel::new(device, mode).map_err(|_| StartFailure::new(14))?; + let bootstrap = bootstrap_data_start(channel.geometry().block_size) + .map_err(|_| StartFailure::new(15))?; + BlockProvider::new_with_data_start(channel, bootstrap).map_err(|_| StartFailure::new(15)) + } + + /// Classify the provisioned medium and land on exactly one mount + /// posture. A whole disk that is kore's or blank keeps the identity + /// view; a foreign disk is consulted for its designated partition and, + /// when one exists, the composition narrows onto it behind the + /// [`RegionDevice`] clamp and re-probes inside. Designation admits + /// the region; only recognized or blank roots inside it authorize user + /// writes — stale foreign bytes inside a user partition still land in + /// the RAM posture rather than being overwritten. The separate system + /// locator owns its explicit archive-era rebuild policy, so this never + /// answers [`Located::Reformat`]. + pub async fn locate( + provider: BlockProvider, B, O>, + mode: CompletionMode, + ) -> Result, StartFailure> { + let probe = PersistentFs::probe(&provider) + .await + .map_err(|_| StartFailure::new(16))?; + let provider = match probe { + VolumeProbe::Kore => return Ok(Located::Recover(provider)), + VolumeProbe::Blank => { + return Ok(Located::Format { + provider, + era: [0; crate::persistence::ERA_BYTES], + }); + } + VolumeProbe::Foreign => provider, + }; + let Some(extent) = Self::discover_region(&provider, gpt::KORE_FS_TYPE).await else { + // A normal metal boot disk begins with GPT/MBR metadata and + // designates nothing for kore. Close the direct DMA channel + // child-before-parent, then keep the service available through + // the identical bounded RAM composition until an installer + // provisions a designated partition. + provider.retire().await.map_err(|_| StartFailure::new(20))?; + return Ok(Located::Volatile(note::VOLATILE_UNDESIGNATED)); + }; + // Rebuild the composition narrowed onto the designation. The + // channel and the bound backend ride through intact; only the + // boot-local buffers and the whole-disk view retire. + let channel = provider.into_channel().map_err(|_| StartFailure::new(22))?; + let device = match channel.into_device() { + Ok(device) => device, + Err(_) => return Err(StartFailure::new(22)), + }; + let device = match device.narrow(extent.start_lba, extent.blocks()) { + Ok(device) => device, + // A designated partition the clamp refuses (impossible extent) + // conveys no authority; the drop path contains the binding. + Err(_) => return Ok(Located::Volatile(note::VOLATILE_REFUSED_REGION)), + }; + let channel = BlockChannel::new(device, mode).map_err(|_| StartFailure::new(22))?; + let bootstrap = match bootstrap_data_start(channel.geometry().block_size) { + Ok(bootstrap) => bootstrap, + Err(_) => return Ok(Located::Volatile(note::VOLATILE_REFUSED_REGION)), + }; + let provider = match BlockProvider::new_with_data_start(channel, bootstrap) { + Ok(provider) => provider, + // Too small for the metadata reservation: a provisioning fault + // that conveys no formatting authority. + Err(_) => return Ok(Located::Volatile(note::VOLATILE_REFUSED_REGION)), + }; + let probe = PersistentFs::probe(&provider) + .await + .map_err(|_| StartFailure::new(23))?; + match probe { + VolumeProbe::Kore => Ok(Located::Recover(provider)), + VolumeProbe::Blank => Ok(Located::Format { + provider, + era: [0; crate::persistence::ERA_BYTES], + }), + VolumeProbe::Foreign => { + provider.retire().await.map_err(|_| StartFailure::new(20))?; + Ok(Located::Volatile(note::VOLATILE_REFUSED_REGION)) + } + } + } + + #[cfg(all(target_arch = "x86_64", target_os = "none"))] + async fn mount_located( + located: Located, + session: SessionId, + ) -> Result<(MountedEngine, Option, RegionLifecycle), StartFailure> { + match located { + Located::Recover(provider) => { + // Detail 17 keeps its low byte; the witness above it names + // the recovery stage and its cause, so a photographed + // refusal says why the workspace did not mount. + let volume = PersistentFs::recover_witnessed(provider, session) + .await + .map_err(|failure| StartFailure::new(17 | (u64::from(failure.witness()) << 8)))?; + let (engine, mount) = volume.into_parts(); + let mut fs = Box::new(FsService::new(engine)); + fs.bind_session(session); + Ok(( + MountedEngine::Durable { + fs, + mount, + commits: 0, + }, + None, + RegionLifecycle::Current, + )) + } + Located::Format { provider, era } => { + let lifecycle = if era.iter().all(|byte| *byte == 0) { + RegionLifecycle::Current + } else { + RegionLifecycle::Rebuilt + }; + let volume = PersistentFs::format_with_era(provider, session, era) + .await + .map_err(|_| StartFailure::new(18))?; + let (engine, mount) = volume.into_parts(); + let mut fs = Box::new(FsService::new(engine)); + fs.bind_session(session); + Ok(( + MountedEngine::Durable { + fs, + mount, + commits: 0, + }, + None, + lifecycle, + )) + } + Located::Reformat { provider, era } => { + let volume = PersistentFs::reformat_with_era(provider, session, era) + .await + .map_err(|_| StartFailure::new(30))?; + let (engine, mount) = volume.into_parts(); + let mut fs = Box::new(FsService::new(engine)); + fs.bind_session(session); + Ok(( + MountedEngine::Durable { + fs, + mount, + commits: 0, + }, + Some(note::SYSTEM_REBUILT), + RegionLifecycle::Rebuilt, + )) + } + Located::Volatile(note) => { + let lifecycle = match note { + note::VOLATILE_UNDESIGNATED => RegionLifecycle::Undesignated, + note::VOLATILE_REFUSED_REGION => RegionLifecycle::Refused, + _ => RegionLifecycle::Unknown, + }; + Ok(( + MountedEngine::Volatile(Box::new(::bind( + FsEndowment::UNSTAMPED, + ))), + Some(note), + lifecycle, + )) + } + } + } + + /// Build lane one directly inside the authenticated system extent and + /// apply the archive-era policy there, never at the whole-disk or user + /// provider layers. + /// + /// All usable roots must be current before recovery is admitted. That is + /// deliberately stronger than checking only the newest root: ordinary + /// recovery may select the fallback if the newest image is corrupt. + async fn locate_system( + lane: D, + extent: gpt::Extent, + mode: CompletionMode, + archive_era: [u8; crate::persistence::ERA_BYTES], + ) -> Result>, StartFailure> { + let whole = RegionDevice::whole(lane).map_err(|_| StartFailure::new(26))?; + let device = match whole.narrow(extent.start_lba, extent.blocks()) { + Ok(device) => device, + Err((whole, _)) => { + // A rejected GPT extent must not strand a live shared lane. + // Retiring this one lane leaves the user mount untouched and + // quiesces the controller only if it was the last owner. + let mut lane = whole.into_inner(); + let _ = lane.quiesce(); + return Err(StartFailure::new(26)); + } + }; + let channel = BlockChannel::new(device, mode).map_err(|_| StartFailure::new(26))?; + let bootstrap = bootstrap_data_start(channel.geometry().block_size) + .map_err(|_| StartFailure::new(26))?; + let provider = BlockProvider::new_with_data_start(channel, bootstrap) + .map_err(|_| StartFailure::new(26))?; + // A TBD manifest digest is absence of policy, not an era. Never turn + // it into authority to destroy a designated region. + if archive_era.iter().all(|byte| *byte == 0) { + provider.retire().await.map_err(|_| StartFailure::new(28))?; + return Ok(None); + } + let probe = match PersistentFs::probe(&provider).await { + Ok(probe) => probe, + Err(_) => { + // Probe still borrows the provider, so contain the sibling + // lane explicitly before degrading to the user-only mount. + provider.retire().await.map_err(|_| StartFailure::new(28))?; + return Err(StartFailure::new(27)); + } + }; + match probe { + VolumeProbe::Kore => { + let current = match PersistentFs::roots_match_era(&provider, archive_era).await { + Ok(current) => current, + Err(_) => { + provider.retire().await.map_err(|_| StartFailure::new(28))?; + return Err(StartFailure::new(29)); + } + }; + Ok(Some(if current { + Located::Recover(provider) + } else { + Located::Reformat { + provider, + era: archive_era, + } + })) + } + VolumeProbe::Blank => Ok(Some(Located::Format { + provider, + era: archive_era, + })), + VolumeProbe::Foreign => Ok(Some(Located::Reformat { + provider, + era: archive_era, + })), + } + } + + #[cfg(all(target_arch = "x86_64", target_os = "none"))] + async fn initialize( + &mut self, + opened: kore_svc_block::OpenedChannel, + ) -> Result<(), StartFailure> { + let backend = DirectNvme::claim(opened).map_err(|_| StartFailure::new(13))?; + let realized_route = backend.has_route(); + let mode = if realized_route { + CompletionMode::Interrupt + } else { + CompletionMode::Poll + }; + // The service heap owns the one physical backend for the row's whole + // lifetime. Both affine lanes borrow it, so their provider types prove + // the controller outlives either mount without `Rc` or a global. + let windows = [lane_window(), lane_window(), lane_window()]; + let [Some(user_window), Some(system_window), Some(pack_window)] = windows else { + return Err(StartFailure::new(14)); + }; + // The engine's tables (over 100 KiB) are written straight into + // their heap slot: this runs inside the serve future on a main + // stack the row already nearly fills, so an engine-sized temporary + // here is a stack overflow waiting for the next table to grow. + let place: &'static mut core::mem::MaybeUninit = + Box::leak(Box::::new_uninit()); + let engine = NvmeEngine::init_in(place, backend).map_err(|_| StartFailure::new(14))?; + let shared: &'static SharedNvme = Box::leak(Box::new( + SharedNvme::new(engine, [user_window, system_window, pack_window], LANE_CLASSES) + .map_err(|_| StartFailure::new(14))?, + )); + self.shared = Some(shared); + let user_lane = shared.claim(0).map_err(|_| StartFailure::new(14))?; + let provider = Self::whole_medium_provider(user_lane, mode)?; + let session = SessionId::new(0); + // Read every designation while lane zero still has the whole-disk + // view. Claim the other lanes before `locate` may retire lane zero: + // the last live lane is what quiesces the physical controller. + let foreign = PersistentFs::probe(&provider).await == Ok(VolumeProbe::Foreign); + let system_extent = if foreign { + Self::discover_region(&provider, gpt::KORE_SYSTEM_TYPE).await + } else { + None + }; + let system_lane = match system_extent { + Some(_) => Some(shared.claim(1).map_err(|_| StartFailure::new(26))?), + None => None, + }; + // The pack region is optional in every posture: without one the + // filesystem mounts exactly as before and a pack grant answers + // `Unsupported`. + let pack_extent = if foreign { + Self::discover_region(&provider, gpt::KORE_PACK_TYPE).await + } else { + None + }; + let system_designated = system_extent.is_some(); + let user_located = Self::locate(provider, mode).await?; + let (user, user_note, user_lifecycle) = Self::mount_located(user_located, session).await?; + if let Some(note) = user_note { + self.note = note; + } + let mut system_lifecycle = if system_designated { + RegionLifecycle::Unavailable + } else { + RegionLifecycle::Undesignated + }; + let mut system_seed_resume = system_lifecycle; + let system = match (system_lane, system_extent) { + (Some(lane), Some(extent)) => { + match Self::locate_system(lane, extent, mode, self.archive_era).await { + Ok(Some(located)) => match Self::mount_located(located, session).await { + Ok((mounted, mount_note, lifecycle)) => { + system_seed_resume = lifecycle; + // A rebuild has made the volume transactionally + // sound, but its measured catalogue is not current + // until exec verifies/copies every member. Enter + // that phase before status can observe the mount. + system_lifecycle = if lifecycle == RegionLifecycle::Rebuilt { + RegionLifecycle::Reseeding + } else { + lifecycle + }; + if let Some(note) = mount_note { + self.note = note; + } + Some(mounted) + } + Err(_) => { + // A failed rebuild/mount never retracts the user + // engine already mounted above and never publishes + // system grants. The next boot retries from the + // flushed dual-root transaction boundary. + self.note = note::SYSTEM_UNAVAILABLE; + system_lifecycle = RegionLifecycle::Unavailable; + None + } + }, + Ok(None) => { + self.note = note::SYSTEM_UNAVAILABLE; + system_lifecycle = RegionLifecycle::Unavailable; + None + } + Err(_) => { + self.note = note::SYSTEM_UNAVAILABLE; + system_lifecycle = RegionLifecycle::Refused; + None + } + } + } + _ => None, + }; + self.pack_region = pack_extent; + let mut mounted = MountedFs { + user, + system, + system_designated, + user_lifecycle, + system_lifecycle, + system_seed_resume, + }; + mounted.establish_boot_policy().map_err(StartFailure::new)?; + if let Some(map) = self.peer_map { + mounted.configure_peer_map(map); + } + self.route_held = realized_route && mounted.durable(); + self.note = if matches!(self.note, note::SYSTEM_UNAVAILABLE | note::SYSTEM_REBUILT) { + // System-era posture is the one fact the scalar readiness note + // cannot reconstruct from the mounted engines. Route and degraded + // state remain present in the structured status record. + self.note + } else if mounted.degraded() { + note::DURABLE_DEGRADED + } else if mounted.durable() && !self.route_held { + note::DURABLE_POLLED + } else { + self.note + }; + self.fs = Some(mounted); + Ok(()) + } + + /// The readiness note computed by [`initialize`](Self::initialize) — + /// `0` is the fully healthy durable posture; see [`note`]. + #[cfg(all(target_arch = "x86_64", target_os = "none"))] + fn readiness_note(&self) -> u64 { + self.note + } + + /// Whether the mounted composition holds its realized MSI-X route — + /// the entry's `enable_irq_parks` decider. The RAM posture (and the + /// pre-initialize window) holds none: its route died with the retired + /// channel, and arming the executor's idle parks on a dead route would + /// wedge every later park. + #[cfg(all(target_arch = "x86_64", target_os = "none"))] + fn holds_route(&self) -> bool { + self.route_held + } + + fn refresh_engine_status( + mounted: &mut MountedEngine, + note: u64, + route_held: bool, + include_shared_waits: bool, + user_lifecycle: RegionLifecycle, + system_lifecycle: RegionLifecycle, + ) { + match mounted { + MountedEngine::Durable { fs, mount, commits } => { + let provider = fs.engine_ref().provider(); + let volume = provider.geometry(); + let occupied_blocks = provider.occupied_blocks(); + let channel_lost = + provider.channel_state() == kore_driver::block::ChannelState::Lost; + let device = provider + .control(|region| { + let shared = region.inner().backend(); + let physical = shared.geometry(); + shared.control(|queues| { + let device = queues; + let route = device.route_stats(); + let health = device.health().ok(); + let hmb = device.host_memory_buffer_bytes(); + (route, physical, health, hmb) + }) + }) + .ok() + .and_then(Result::ok); + let ( + mut route_parks, + mut route_fires, + mut deadline_waits, + mut spin_waits, + mut monitor_sleeps, + mut monitor_turns, + ) = device.map(|(route, _, _, _)| route).unwrap_or_default(); + let host_memory_buffer = device.is_some_and(|(_, _, _, hmb)| hmb != 0); + // These counters live on the one physical backend, not on a + // lane. Report them through exactly one engine before the two + // status records are folded, or every wait would double. + if !include_shared_waits { + route_parks = 0; + route_fires = 0; + deadline_waits = 0; + spin_waits = 0; + monitor_sleeps = 0; + monitor_turns = 0; + } + let storage = match (device, occupied_blocks) { + (Some((_, physical, health, _)), Some(occupied_blocks)) + if physical.block_size == volume.block_size => + { + let block_bytes = u64::from(volume.block_size); + let storage = kore_abi::monitor::FilesystemStorageSnapshot::new( + volume.block_size, + physical.bytes(), + volume.bytes(), + occupied_blocks.saturating_mul(block_bytes), + u64::from(volume.max_transfer_blocks).saturating_mul(block_bytes), + volume.read_only + || mount.status().mode + == crate::persistence::MountMode::DegradedReadOnly, + volume.flush, + volume.discard, + ) + .with_layout( + volume.io_hints.supported_block_sizes, + u64::from(volume.io_hints.preferred_write_blocks) + .saturating_mul(block_bytes), + u64::from(volume.io_hints.optimal_write_blocks) + .saturating_mul(block_bytes), + ); + health.map_or(storage, |health| { + storage.with_health(kore_abi::monitor::FilesystemHealthSnapshot::new( + health.critical_warning, + health.temperature_celsius, + health.available_spare_percent, + health.spare_threshold_percent, + health.percentage_used, + health.power_cycles, + health.power_on_hours, + health.unsafe_shutdowns, + health.media_errors, + )) + }) + } + _ => kore_abi::monitor::FilesystemStorageSnapshot::EMPTY, + }; + let backend = crate::service::FsBackendStatus { + route_parks, + route_fires, + deadline_waits, + spin_waits, + monitor_sleeps, + monitor_turns, + durable: true, + degraded_read_only: mount.status().mode + == crate::persistence::MountMode::DegradedReadOnly, + route_held, + channel_lost, + host_memory_buffer, + note, + user_lifecycle: user_lifecycle.wire(), + system_lifecycle: system_lifecycle.wire(), + commits: *commits, + provider: fs.engine_ref().provider().stats(), + channel: fs.engine_ref().provider().channel_stats(), + storage, + }; + fs.set_backend_status(backend); + } + MountedEngine::Volatile(fs) => { + fs.set_backend_status(crate::service::FsBackendStatus { + note, + user_lifecycle: user_lifecycle.wire(), + system_lifecycle: system_lifecycle.wire(), + ..Default::default() + }); + } + } + } + + /// Freeze each backend's independent counters immediately before the + /// router asks the engines for their status records. + fn refresh_backend_status(&mut self) { + let note = self.note; + let route_held = self.route_held; + let Some(mounted) = self.fs.as_mut() else { + return; + }; + let user_owns_shared_waits = mounted.user.durable(); + Self::refresh_engine_status( + &mut mounted.user, + note, + route_held, + user_owns_shared_waits, + mounted.user_lifecycle, + mounted.system_lifecycle, + ); + if let Some(system) = mounted.system.as_mut() { + Self::refresh_engine_status( + system, + 0, + route_held, + !user_owns_shared_waits, + RegionLifecycle::Unknown, + RegionLifecycle::Unknown, + ); + } + } + + fn needs_commit(request: &FsRequest, response: &FsResponse) -> bool { + if matches!(response, FsResponse::Refused(_)) { + return false; + } + matches!( + request, + FsRequest::Create { .. } + | FsRequest::WriteAt { .. } + | FsRequest::CommitReplace { .. } + | FsRequest::Append { .. } + | FsRequest::Resize { .. } + | FsRequest::Rename { .. } + | FsRequest::Relocate { .. } + | FsRequest::Link { .. } + | FsRequest::Unlink { .. } + | FsRequest::DetachEdge { .. } + | FsRequest::Sync { .. } + | FsRequest::CommitDerived { .. } + ) + } + + fn persistence_error(error: PersistenceError) -> FsError { + match error { + PersistenceError::Provider(error) => error.into(), + PersistenceError::Filesystem(error) => error, + PersistenceError::Unformatted + | PersistenceError::Incompatible + | PersistenceError::Corrupt + | PersistenceError::Bounds + | PersistenceError::Interrupted(_) => { + FsError::ProviderFailure(ProviderError::Integrity) + } + } + } +} + +/// Retire a target the engine gave back with its proof, at once and with +/// the controller running ([`DirectNvme::retire_quiet`]). `true` once its +/// device mapping is gone. +fn retire_target(shared: &SharedNvme, target: Quiet>) -> bool { + shared.control(|queues| queues.retire_quiet(target)).unwrap_or(false) +} + + +/// One session's storage ring (`res/STORAGE_TRANSPORT.md` §9, slice S4): a +/// window the client funded and lent, attached to the engine beside the +/// lanes. The client submits by writing its own window; the pair carries +/// only control. +struct ClientRing { + session: SessionId, + ring: RingId, + /// The lent view's owner. The engine borrows the mapped window for + /// `'static`, so this is dropped only after the engine retired the ring + /// and handed the window back ([`EngineBackend::retire_client`]). + _window: kore_rt::cap::ReturnedCapability, + /// Target slots adopted and not yet released. + targets: u64, + /// Version grants: the lease (store-tagged wire bits) each pins, and its + /// grant. A lease with a grant here is released only once the grant + /// has drained ([`DurableService::settle_version_grant`]). + versions: [Option<(u64, GrantId)>; VERSION_GRANTS], +} + +/// Version grants one ring may hold at once: every grant slot but the one a +/// pack grant takes. +const VERSION_GRANTS: usize = ENGINE_GRANTS - 1; + +/// A version's whole blocks as one read grant (the extent is contiguous; +/// runs split only at the run length's width). +fn version_grant(blocks: crate::provider::DeviceBlocks) -> Result { + let mut runs = [ExtentRun::default(); kore_driver::stor::grant::MAX_RUNS]; + let mut count = 0; + let mut lba = blocks.lba; + let end = blocks.lba.checked_add(blocks.blocks).ok_or(FsError::Range)?; + while lba < end { + let run = runs.get_mut(count).ok_or(FsError::Range)?; + let span = (end - lba).min(u64::from(u32::MAX)) as u32; + *run = ExtentRun { lba, blocks: span }; + count += 1; + lba += u64::from(span); + } + Grant::new(&runs[..count], Rights::READ, ClassSet::DATA).map_err(|_| FsError::Range) +} + +const _: () = assert!(RING_TARGETS <= 64, "a ring's targets are a u64 mask"); + +/// The pack region as one read grant: every block, in as many runs as the +/// run length's width needs. Demand for reads a thread waits on, Stream for +/// read-ahead — never Commit. +fn pack_grant(extent: gpt::Extent) -> Result { + let mut runs = [ExtentRun::default(); kore_driver::stor::grant::MAX_RUNS]; + let mut count = 0; + let mut lba = extent.start_lba; + let end = extent.start_lba.saturating_add(extent.blocks()); + while lba < end { + let run = runs.get_mut(count).ok_or(FsError::Range)?; + let blocks = (end - lba).min(u64::from(u32::MAX)) as u32; + *run = ExtentRun { lba, blocks }; + count += 1; + lba += u64::from(blocks); + } + Grant::new(&runs[..count], Rights::READ, ClassSet::DATA).map_err(|_| FsError::Range) +} + +/// Take down one client ring: stop its popping, hand back every target +/// nothing uses, retire it once drained, then drop the window's owner. +/// `false` while commands are still in flight: ask again after a later wake. +fn retire_ring(shared: &SharedNvme, slot: &mut Option) -> bool { + let Some(entry) = slot.as_mut() else { + return true; + }; + if shared.detach_client(entry.ring).is_err() { + return false; + } + let mut adopted = entry.targets; + while adopted != 0 { + let index = adopted.trailing_zeros() as u16; + adopted &= adopted - 1; + if let Ok(target) = shared.release_client(entry.ring, index) { + entry.targets &= !(1u64 << index); + let _ = retire_target(shared, target); + } + } + if entry.targets != 0 || shared.retire_client(entry.ring).is_err() { + return false; + } + // The engine no longer references the window: the mapping may go. + *slot = None; + true +} + +/// Map a lent ring window and attach it. The window is the client's shared +/// memory; the engine reads it only through the channel protocols. +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +fn attach_ring( + shared: &SharedNvme, + owner: kore_rt::cap::ReturnedCapability, +) -> Result<(RingId, kore_rt::cap::ReturnedCapability), FsError> { + use kore_rt::cap::verbs; + let slot = owner.slot().ok_or(FsError::NoWindow)?; + let view = verbs::take_view(slot).ok_or(FsError::NoWindow)?; + // A sink mapping (`res/STORAGE_TRANSPORT.md` §12, option B): should the + // client die with its ring attached, the engine's next pass reads a sink + // page — bytes the channel protocols already decode safely — instead of + // faulting on a hole, until `peer_closed` retires the ring. + let mapped = verbs::map_view_sink(view).ok_or(FsError::NoWindow)?; + if mapped.len() < kore_ring::stor::WINDOW_BYTES { + return Err(FsError::Range); + } + // SAFETY: `map_view` installed `[base, +len)` in this task's space over + // the lent view, page-aligned; `owner` keeps it mapped and is dropped + // only after the engine retires the ring (`retire_ring`). The client + // writes the same bytes concurrently: the engine touches them only + // through the channel protocols (atomics, masked indices, records that + // decode from any bytes), never through this slice directly. + let window: &'static mut [u8] = unsafe { + core::slice::from_raw_parts_mut(mapped.cast::().as_ptr(), kore_ring::stor::WINDOW_BYTES) + }; + let ring = shared.attach_client(window, ClassSet::DATA).map_err(|error| match error { + // `service` refuses a window the client never formatted. + BlockError::Geometry => FsError::Range, + BlockError::Backpressure => FsError::OutOfHandles, + other => FsError::ProviderFailure(crate::provider::map_block(other)), + })?; + Ok((ring, owner)) +} + +/// Host builds have no capability arena to map from. +#[cfg(not(all(target_arch = "x86_64", target_os = "none")))] +fn attach_ring( + _shared: &SharedNvme, + _owner: kore_rt::cap::ReturnedCapability, +) -> Result<(RingId, kore_rt::cap::ReturnedCapability), FsError> { + Err(FsError::Unsupported) +} + +/// Import a lent device route and adopt it as a target of `ring`, reads +/// landing in it. +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +fn attach_ring_target( + shared: &SharedNvme, + ring: RingId, + owner: kore_rt::cap::ReturnedCapability, +) -> Result<(u16, u64), FsError> { + // Settle what earlier clients' windows owe before admitting this one: + // the whole pass if the controller is idle, else the live retries. + if shared.control_idle(|queues| queues.retry_retirement()).is_err() { + let _ = shared.control(|queues| queues.retry_live_retirement()); + } + let target = shared + .control(|queues| queues.adopt_target(owner)) + .map_err(|error| FsError::ProviderFailure(crate::provider::map_block(error)))? + .map_err(|(error, _)| error)?; + let bytes = kore_driver::block::BlockBuffer::len(&target) as u64; + match shared.adopt_client(ring, target, TargetAccess::ReadInto) { + Ok(slot) => Ok((slot, bytes)), + // The mapping retires before the refusal answers, so the lender may + // retire its route; one that outlived it answers `Busy`, as a + // release does. + Err((target, error)) => Err(if !retire_target(shared, target) { + FsError::Busy + } else { + match error { + BlockError::Backpressure => FsError::OutOfHandles, + other => FsError::ProviderFailure(crate::provider::map_block(other)), + } + }), + } +} + +/// Host builds have no capability arena to import from. +#[cfg(not(all(target_arch = "x86_64", target_os = "none")))] +fn attach_ring_target( + _shared: &SharedNvme, + _ring: RingId, + _owner: kore_rt::cap::ReturnedCapability, +) -> Result<(u16, u64), FsError> { + Err(FsError::Unsupported) +} + +/// Claim [`PACK_LANE`] and build the pack's maintenance job over it, +/// clamped to `extent`. Every refusal retires the claimed lane, so a failed +/// claim never holds the controller up at quiesce. +#[cfg_attr(not(all(target_arch = "x86_64", target_os = "none")), allow(dead_code))] +fn claim_pack(shared: &'static SharedNvme, extent: gpt::Extent) -> PackMaintenance { + let Ok(lane) = shared.claim(PACK_LANE) else { + return PackMaintenance::Unavailable; + }; + // The lane's geometry is the one the backend validated when it was + // built, so the identity view cannot be refused. + let Ok(whole) = RegionDevice::whole(lane) else { + return PackMaintenance::Unavailable; + }; + let device = match whole.narrow(extent.start_lba, extent.blocks()) { + Ok(device) => device, + Err((whole, _)) => { + let _ = whole.into_inner().quiesce(); + return PackMaintenance::Unavailable; + } + }; + // The bucket books live as long as the row, as the lanes' windows do. + let buckets: &'static mut [Bucket] = + Box::leak(alloc::vec![Bucket::default(); PACK_BUCKETS].into_boxed_slice()); + match RegionRefresh::new(device, buckets) { + Ok(job) => PackMaintenance::Live(Box::new(job)), + Err((device, _)) => { + let _ = device.into_inner().quiesce(); + PackMaintenance::Unavailable + } + } +} + +/// Stop `pack`'s job, wait out what the device holds, and retire its lane. +/// `false` when a command would not come back: the lane stays claimed and +/// the sweep is partial. +#[cfg_attr(not(all(target_arch = "x86_64", target_os = "none")), allow(dead_code))] +fn retire_pack(pack: PackMaintenance) -> bool { + let PackMaintenance::Live(mut job) = pack else { + return true; + }; + if !job.drain() { + // Leaked with its lane: the device may still write into its + // buffers, which must therefore never be reused. + Box::leak(job); + return false; + } + match (*job).into_device() { + Ok(device) => device.into_inner().quiesce().is_ok(), + Err(job) => { + Box::leak(Box::new(job)); + false + } + } +} + +/// Land `job`'s report in `window` at `window_offset`: the summary, then +/// as many bucket records as fit. Answers the bytes landed. +#[cfg_attr(not(all(target_arch = "x86_64", target_os = "none")), allow(dead_code))] +fn write_pack_report(job: &PackJob, window: &mut DataWindow<'_>, window_offset: u32) -> Result { + use kore_abi::svc::{PackBucketWire, PackMaintenanceWire as Wire}; + use zerocopy::IntoBytes; + use zerocopy::byteorder::little_endian::{U16, U32, U64}; + let short = |n: u32| U16::new(u16::try_from(n).unwrap_or(u16::MAX)); + let room = window + .len() + .saturating_sub(window_offset as usize) + .saturating_sub(Wire::BYTES) + / PackBucketWire::BYTES; + let buckets = job.buckets(); + let entries = buckets.len().min(room); + let shape = job.shape(); + let scan = job.scan_summary(); + let books = job.refresh_summary(); + let (outcome, fault) = match job.outcome() { + refresh::Outcome::None => (Wire::OUTCOME_NONE, 0), + refresh::Outcome::Scanned => (Wire::OUTCOME_SCANNED, 0), + refresh::Outcome::Refreshed => (Wire::OUTCOME_REFRESHED, 0), + refresh::Outcome::Cancelled => (Wire::OUTCOME_CANCELLED, 0), + refresh::Outcome::Faulted(error) => (Wire::OUTCOME_FAULTED, (error as u8).saturating_add(1)), + }; + let summary = Wire { + phase: match job.phase() { + refresh::Phase::Idle => Wire::PHASE_IDLE, + refresh::Phase::Scanning => Wire::PHASE_SCANNING, + refresh::Phase::Refreshing => Wire::PHASE_REFRESHING, + refresh::Phase::Draining => Wire::PHASE_DRAINING, + }, + outcome, + fault, + flags: if scan.complete { Wire::FLAG_SCAN_COMPLETE } else { 0 }, + buckets: short(shape.buckets), + entries: short(entries as u32), + probes: short(refresh::PROBES as u32), + stale_ratio: short(refresh::STALE_RATIO as u32), + in_flight: short(job.in_flight()), + passes: short(refresh::PASSES), + bucket_bytes: U64::new(shape.bucket_bytes()), + region_bytes: U64::new(shape.region_blocks * u64::from(shape.block_bytes)), + chunk_bytes: U32::new(shape.chunk_bytes()), + scans: U32::new(scan.scans), + scanned: short(scan.scanned), + mapped: short(scan.mapped), + stale: short(scan.stale), + planned: short(books.buckets_planned), + done: short(books.buckets_done), + skipped: short(books.buckets_skipped), + lifetime: short(books.buckets_lifetime), + pass: short(scan.pass), + failed_probes: U32::new(scan.failed_probes), + p25_ns: U32::new(scan.p25_ns), + p50_ns: U32::new(scan.p50_ns), + p90_ns: U32::new(scan.p90_ns), + bytes: U64::new(books.bytes), + zero_chunks: U32::new(books.zero_chunks), + read_errors: U32::new(books.read_errors), + unstable_chunks: U32::new(books.unstable_chunks), + write_retries: U32::new(books.write_retries), + mismatches: U32::new(books.mismatches), + }; + let len = Wire::BYTES + entries * PackBucketWire::BYTES; + let len = u32::try_from(len).map_err(|_| FsError::Range)?; + let out = window.destination(window_offset, len)?; + let (head, records) = out.split_at_mut(Wire::BYTES); + head.copy_from_slice(summary.as_bytes()); + for (bucket, record) in buckets.iter().zip(records.chunks_exact_mut(PackBucketWire::BYTES)) { + let flag = |set: bool, bit: u8| if set { bit } else { 0 }; + let wire = PackBucketWire { + median_ns: U32::new(bucket.median_ns), + mapped_probes: bucket.mapped_probes, + failed_probes: bucket.failed_probes, + flags: flag(bucket.stale, PackBucketWire::FLAG_STALE) + | flag(bucket.refreshed, PackBucketWire::FLAG_REFRESHED) + | flag(bucket.incomplete, PackBucketWire::FLAG_INCOMPLETE) + | flag(bucket.mapped(), PackBucketWire::FLAG_MAPPED), + reserved: 0, + }; + record.copy_from_slice(wire.as_bytes()); + } + Ok(len) +} + +impl DurableService { + /// The pack's maintenance job, claimed on the first ask. `Unsupported` + /// without a pack region, without a running controller (a volatile + /// posture retired every filesystem lane), or after a failed claim. + fn pack_job(&mut self) -> Result<&mut PackJob, FsError> { + if matches!(self.pack, PackMaintenance::Unclaimed) { + let (Some(shared), Some(extent)) = (self.shared, self.pack_region) else { + return Err(FsError::Unsupported); + }; + if !(0..FS_LANES).any(|lane| shared.is_live(lane)) { + return Err(FsError::Unsupported); + } + self.pack = claim_pack(shared, extent); + } + match &mut self.pack { + PackMaintenance::Live(job) => Ok(job), + _ => Err(FsError::Unsupported), + } + } + + /// Serve one pack maintenance ask (`res/PACK_STREAMING.md` §7): start + /// or stop the job, give it its first pass, and land the report in the + /// asking session's window. + /// + /// No authority beyond an attached session: the job only rewrites the + /// pack's own bytes back unchanged, and at most once per bucket per + /// boot — less endurance than any session with a writable home already + /// spends. + fn serve_maintain( + &mut self, + action: kore_dagfs::MaintainAction, + window_offset: u32, + via: u16, + ) -> FsResponse { + use kore_dagfs::MaintainAction; + let Some(fs) = self.fs.as_ref() else { + return FsResponse::Refused(FsError::Unsupported); + }; + let Some(session) = fs.user.session_for_via(via) else { + return FsResponse::Refused(FsError::AccessDenied); + }; + let Some(span) = fs.user.mapped_window_span(session) else { + return FsResponse::Refused(FsError::NoWindow); + }; + let job = match self.pack_job() { + Ok(job) => job, + Err(error) => return FsResponse::Refused(error), + }; + let started = match action { + MaintainAction::Status => Ok(()), + MaintainAction::Scan => job.scan(), + MaintainAction::RefreshStale => job.refresh_stale(), + MaintainAction::RefreshRange { first, count } => job.refresh_range(first, count), + MaintainAction::Cancel => { + job.cancel(); + Ok(()) + } + }; + if let Err(refusal) = started { + return FsResponse::Refused(match refusal { + refresh::Refusal::Busy => FsError::Busy, + refresh::Refusal::Range => FsError::Range, + refresh::Refusal::NoClock => FsError::Unsupported, + // Nothing to refresh by until a scan has judged the + // buckets: the health index is not built yet. + refresh::Refusal::NotScanned => FsError::IndexIncomplete, + }); + } + self.advance_stor(); + let PackMaintenance::Live(job) = &self.pack else { + return FsResponse::Refused(FsError::Unsupported); + }; + let mut window = MountedFs::window_for(span); + match write_pack_report(job, &mut window, window_offset) { + Ok(len) => FsResponse::StorMaintained { len }, + Err(error) => FsResponse::Refused(error), + } + } + + /// One engine pass for the client rings on every wake of the row — each + /// request, and the controller's interrupt through + /// [`SystemService::poll_background`]. When the engine goes idle it + /// parks every ring, so a client that publishes afterwards owes a + /// `StorKick`; while commands are in flight their completion is the + /// wake, and a client's publish costs nothing. + fn advance_stor(&mut self) { + let Some(shared) = self.shared else { + return; + }; + let maintaining = matches!(&self.pack, PackMaintenance::Live(job) if job.phase() != refresh::Phase::Idle); + if maintaining || self.rings.iter().any(Option::is_some) { + let _ = shared.pump(); + } + if maintaining && let PackMaintenance::Live(job) = &mut self.pack { + // Probes time one read alone on the device: only when nothing + // else is on it. Every other step rides the Stream budget. + let idle = shared.commands_in_flight() == Some(0); + job.advance(idle); + } + // What earlier windows' retirements owe settles the whole-latch way + // on the first pass that finds the controller idle. + if shared + .inspect(|queues| queues.retirement_owed()) + .unwrap_or(false) + { + let _ = shared.control_idle(|queues| queues.retry_retirement()); + } + } + + /// Before a version lease is released, revoke the ring grant it pins. + /// `false` while a read under the grant is still in flight — the + /// release is refused `Busy` and the client asks again; the lease (and + /// with it the blocks) outlives every command that reads them. + fn settle_version_grant(&mut self, via: u16, lease: kore_dagfs::VersionLease) -> bool { + let (Some(shared), Some(fs)) = (self.shared, self.fs.as_ref()) else { + return true; + }; + let Some(session) = fs.user.session_for_via(via) else { + return true; + }; + let wire = lease.to_bits(); + for entry in self.rings.iter_mut().flatten() { + if entry.session != session { + continue; + } + let ring = entry.ring; + for slot in entry.versions.iter_mut() { + let Some((bits, grant)) = *slot else { continue }; + if bits != wire { + continue; + } + match shared.revoke(ring, grant) { + Ok(kore_driver::stor::engine::Settle::Later) => return false, + // Revoked now, or already drained and freed (a stale id). + Ok(_) | Err(_) => *slot = None, + } + } + } + true + } + + /// Serve one storage-ring verb (`res/STORAGE_TRANSPORT.md` §9, S4). + fn serve_stor(&mut self, request: FsRequest, sv: &mut Serve<'_, FsResponse>) -> FsResponse { + // A capability lent to a refused request is dropped — revoked back + // to the client's parent, never kept. + let lent = match request { + FsRequest::StorAttach { announced } | FsRequest::StorTarget { announced } => sv + .returned_input(announced) + .and_then(|mut input| input.take()), + _ => None, + }; + let (Some(shared), Some(fs)) = (self.shared, self.fs.as_ref()) else { + return FsResponse::Refused(FsError::Unsupported); + }; + let Some(session) = fs.user.session_for_via(sv.via()) else { + return FsResponse::Refused(FsError::AccessDenied); + }; + let index = self + .rings + .iter() + .position(|ring| ring.as_ref().is_some_and(|ring| ring.session == session)); + let ring = index.and_then(|index| self.rings[index].as_ref().map(|ring| ring.ring)); + match request { + FsRequest::StorAttach { .. } => { + let Some(owner) = lent else { + return FsResponse::Refused(FsError::NoWindow); + }; + if index.is_some() { + return FsResponse::Refused(FsError::AlreadyExists); + } + let Some(free) = self.rings.iter().position(Option::is_none) else { + return FsResponse::Refused(FsError::OutOfHandles); + }; + match attach_ring(shared, owner) { + Ok((ring, owner)) => { + self.rings[free] = Some(ClientRing { + session, + ring, + _window: owner, + targets: 0, + versions: [None; VERSION_GRANTS], + }); + let geometry = shared.geometry(); + let max_transfer = u64::from(geometry.max_transfer_blocks) + * u64::from(geometry.block_size); + FsResponse::StorAttached { + block_size: geometry.block_size, + max_transfer: u32::try_from(max_transfer).unwrap_or(u32::MAX), + } + } + Err(error) => FsResponse::Refused(error), + } + } + FsRequest::StorGrantPack => { + let (Some(ring), Some(extent)) = (ring, self.pack_region) else { + return FsResponse::Refused(if ring.is_none() { + FsError::NoWindow + } else { + FsError::Unsupported + }); + }; + let bytes = extent.blocks() * u64::from(shared.geometry().block_size); + match pack_grant(extent).and_then(|grant| { + shared.install(ring, grant).map_err(|_| FsError::OutOfHandles) + }) { + Ok(grant) => FsResponse::StorGranted { + grant: grant.fd() as u32, + bytes, + }, + Err(error) => FsResponse::Refused(error), + } + } + FsRequest::StorGrantVersion { lease } => { + let (Some(index), Some(ring)) = (index, ring) else { + return FsResponse::Refused(FsError::NoWindow); + }; + let Some(free) = self.rings[index] + .as_ref() + .and_then(|entry| entry.versions.iter().position(Option::is_none)) + else { + return FsResponse::Refused(FsError::OutOfHandles); + }; + let wire = lease.to_bits(); + if self.rings[index] + .as_ref() + .is_some_and(|entry| entry.versions.iter().flatten().any(|(bits, _)| *bits == wire)) + { + return FsResponse::Refused(FsError::AlreadyExists); + } + let (store, lease) = crate::router::split_lease(lease); + let engine = match store { + crate::router::Store::User => Some(&fs.user), + crate::router::Store::System => fs.system.as_ref(), + }; + let blocks = match engine { + Some(MountedEngine::Durable { fs: engine, .. }) => engine.lent_blocks(session, lease), + _ => Err(FsError::Unsupported), + }; + let blocks = match blocks { + Ok(Some(blocks)) if blocks.blocks != 0 => blocks, + // Nothing whole to grant: the version reads through its + // lease alone. + Ok(_) => return FsResponse::Refused(FsError::Range), + Err(error) => return FsResponse::Refused(error), + }; + let bytes = blocks.blocks * u64::from(blocks.block_size); + match version_grant(blocks).and_then(|grant| { + shared.install(ring, grant).map_err(|_| FsError::OutOfHandles) + }) { + Ok(grant) => { + if let Some(entry) = self.rings[index].as_mut() { + entry.versions[free] = Some((wire, grant)); + } + FsResponse::StorGranted { + grant: grant.fd() as u32, + bytes, + } + } + Err(error) => FsResponse::Refused(error), + } + } + FsRequest::StorTarget { .. } => { + let (Some(index), Some(ring), Some(owner)) = (index, ring, lent) else { + return FsResponse::Refused(FsError::NoWindow); + }; + match attach_ring_target(shared, ring, owner) { + Ok((slot, bytes)) => { + if let Some(entry) = self.rings[index].as_mut() { + entry.targets |= 1u64 << slot; + } + FsResponse::StorTargetAdopted { slot, bytes } + } + Err(error) => FsResponse::Refused(error), + } + } + FsRequest::StorRelease { slot } => { + let (Some(index), Some(ring)) = (index, ring) else { + return FsResponse::Refused(FsError::NoWindow); + }; + match shared.release_client(ring, slot) { + Ok(target) => { + if let Some(entry) = self.rings[index].as_mut() { + entry.targets &= !(1u64 << slot); + } + // `Done` promises the lender its route has no device + // child left here, so it may retire it. A mapping + // that outlives the release answers `Busy`: the + // slot is gone, and the lender must hold the route. + if retire_target(shared, target) { + FsResponse::Done + } else { + FsResponse::Refused(FsError::Busy) + } + } + Err(BlockError::BufferBusy) => FsResponse::Refused(FsError::Busy), + Err(_) => FsResponse::Refused(FsError::InvalidHandle), + } + } + FsRequest::StorInspect => { + let narrow = |n: u64| u32::try_from(n).unwrap_or(u32::MAX); + match shared.stats() { + Some((stats, _)) => FsResponse::StorInspected { + submissions: stats.submissions, + completions: stats.completions, + in_flight: narrow(shared.commands_in_flight().unwrap_or(0) as u64), + expired: narrow(stats.expired), + stale: narrow(stats.stale_completions), + timeline_refusals: narrow(stats.timeline_refusals), + stage_refusals: narrow(stats.stage_refusals), + last_stage_error: stats + .last_stage_error + .map_or(0, |error| (error as u8).saturating_add(1)), + aborts: narrow(stats.aborts), + aborts_refused: u8::try_from(stats.aborts_refused).unwrap_or(u8::MAX), + weighted: shared + .inspect(|queues| queues.backend().weighted_arbitration()) + .unwrap_or(false), + rings: u8::try_from( + FS_LANES + + usize::from(matches!(self.pack, PackMaintenance::Live(_))) + + self.rings.iter().flatten().count(), + ) + .unwrap_or(u8::MAX), + }, + None => FsResponse::Refused(FsError::Busy), + } + } + FsRequest::StorKick => { + // The wake itself: `serve_async` pumped before dispatching, + // and the reply leaves the ring advanced. + FsResponse::Done + } + FsRequest::StorDetach => { + let Some(index) = index else { + return FsResponse::Done; + }; + if retire_ring(shared, &mut self.rings[index]) { + FsResponse::Done + } else { + FsResponse::Refused(FsError::Busy) + } + } + _ => FsResponse::Refused(FsError::Unsupported), + } + } +} + +impl SystemService for DurableService { + const NAME: &'static str = "fs"; + /// An I/O worker: parked on completions for most of its life, its work + /// done in microseconds between parks. Keep it off the performance + /// cores; a completion wait cannot tell the difference. + const CORE_PREFERENCE: kore_abi::task::CorePreference = + kore_abi::task::CorePreference::Efficient; + type Endowment = FsEndowment; + const RETURN_MODE: ReturnMode = ReturnMode::RetainedAny; + + /// See [`crate::FS_DEVICE_CLASSES`] — declared once, cited by both impls. + const DEVICE_CLASSES: &'static [kore_rt::service::DeviceClass] = crate::FS_DEVICE_CLASSES; + + /// The terminal sequence (#809 §L, L.8 item 2): the row holds two lanes + /// and the client rings over one controller, and the controller's own + /// quiesce (`CC.SHN`) runs only when the **last** live lane retires — so + /// this is an ordered sequence, not one call. The client rings go first, + /// releasing the client memory mapped into the controller while the + /// filesystem lanes still keep it running. Retire the user and system engines' providers + /// (unregister their DMA buffers, quiesce their channels; the last one + /// takes the controller through orderly shutdown), then reach the binding + /// through the shared backend for the host's class → fabric → + /// containment teardown. A lane that could not drain leaves `partial` + /// set: that is a different state from skipped, and the containment + /// sweep after the drain knows which. + async fn quiesce_devices(&mut self, _deadline_ns: u64) -> kore_rt::service::QuiesceSweep { + let mut sweep = kore_rt::service::QuiesceSweep::NONE; + let Some(shared) = self.shared.take() else { + // Volatile posture: nothing was ever bound. + return sweep; + }; + // Client rings first: their targets are client memory mapped into + // this controller. A ring still draining leaves the sweep partial. + for slot in self.rings.iter_mut() { + if !retire_ring(shared, slot) { + sweep.partial = true; + } + } + // Then the pack maintenance lane, while the filesystem lanes still + // keep the controller running: its job stops issuing, the device + // returns what it holds, and the lane retires without being the + // last one. + if !retire_pack(core::mem::replace(&mut self.pack, PackMaintenance::Unavailable)) { + sweep.partial = true; + } + if let Some(mounted) = self.fs.take() { + let MountedFs { user, system, .. } = mounted; + for engine in [Some(user), system].into_iter().flatten() { + if let MountedEngine::Durable { fs, .. } = engine + && (*fs).into_engine().into_provider().retire().await.is_err() + { + sweep.partial = true; + } + } + } + match shared.control_idle(|queues| { + let direct = queues; + direct.retry_retirement(); + direct.teardown() + }) { + Ok(Some(outcome)) => outcome.record(&mut sweep), + // Already consumed — a second ask after the first completed. + Ok(None) => {} + // A lane still has work outstanding: the controller cannot be + // stopped under it. Partial, and the kernel-side containment + // after the drain owns the function. + Err(_) => sweep.partial = true, + } + sweep + } + + /// The faces grow with the programs that read files, as the RAM + /// composition's do ([`RamService`]): both engines' + /// sessions are allocated per slot from the same heap. + fn peer_arena(&self) -> Option<&'static (dyn core::alloc::Allocator + Sync)> { + Some(crate::service_alloc()) + } + + fn bind(endowment: FsEndowment) -> Self { + Self { + fs: None, + shared: None, + peer_map: None, + archive_era: endowment.archive_era(), + route_held: false, + note: 0, + pack_region: None, + pack: PackMaintenance::Unclaimed, + rings: core::array::from_fn(|_| None), + } + } + + fn start_on_metal(&mut self, rt: &kore_rt::entry::Rt) -> Result<(), StartFailure> { + let map = crate::service::contract::peer_map(rt).map_err(|_| StartFailure::new(21))?; + self.peer_map = Some(map); + if let Some(fs) = self.fs.as_mut() { + fs.configure_peer_map(map); + } + Ok(()) + } + + fn serve(&mut self, request: FsRequest, sv: &mut Serve<'_, FsResponse>) { + let Some(mounted) = self.fs.as_mut() else { + sv.reply(FsResponse::Refused(FsError::ProviderFailure( + ProviderError::Lost, + ))); + return; + }; + let response = if mounted.durable() { + // A durable request must retain the executor waker across its + // dependent block-service transaction. The metal host calls + // `serve_async`; fail closed if a synchronous harness attempts to + // bypass that progress seam. + FsResponse::Refused(FsError::ProviderFailure(ProviderError::Lost)) + } else { + crate::block_on(mounted.serve_request_async(request, sv)) + }; + sv.reply(response); + } + + async fn serve_async(&mut self, request: FsRequest, sv: &mut Serve<'_, FsResponse>) { + // Advance the client rings without waiting on the controller; + // ordinary filesystem traffic remains dispatchable while they run. + self.advance_stor(); + if let FsRequest::StorMaintain { + action, + window_offset, + } = request + { + let response = self.serve_maintain(action, window_offset, sv.via()); + sv.reply(response); + return; + } + if matches!( + request, + FsRequest::StorAttach { .. } + | FsRequest::StorGrantPack + | FsRequest::StorGrantVersion { .. } + | FsRequest::StorTarget { .. } + | FsRequest::StorRelease { .. } + | FsRequest::StorKick + | FsRequest::StorInspect + | FsRequest::StorDetach + ) { + let response = self.serve_stor(request, sv); + // A detach or release may have freed work the next pass parks. + self.advance_stor(); + sv.reply(response); + return; + } + // A version whose blocks a ring grant names outlives every read + // under that grant: the release waits for the grant to drain. + if let FsRequest::ReleaseVersion { lease } = request + && !self.settle_version_grant(sv.via(), lease) + { + sv.reply(FsResponse::Refused(FsError::Busy)); + return; + } + // A status answer must read as one moment: fold the mount's current + // posture and counters into the frontend immediately before the + // request is served, and only then — every other verb pays nothing. + if matches!(request, FsRequest::SystemStatus { .. }) { + self.refresh_backend_status(); + } + let Some(mounted) = self.fs.as_mut() else { + sv.reply(FsResponse::Refused(FsError::ProviderFailure( + ProviderError::Lost, + ))); + return; + }; + let response = mounted.serve_request_async(request, sv).await; + sv.reply(response); + } + + fn peer_admitted(&mut self, result: Result) -> bool { + match self.fs.as_mut() { + Some(fs) => fs.peer_admitted(result), + None => result.is_err(), + } + } + + fn poll_background(&mut self) { + self.advance_stor(); + } + + fn peer_closed(&mut self, via: u16, reason: PeerClose) -> bool { + // A departed session's ring goes first: its targets are client + // memory mapped into the controller, and a command still in flight + // owns its destination, so the peer stays closing until the ring + // drained. The host retries this hook after later wakes, each of + // which pumps the engine. + if let (Some(fs), Some(shared)) = (self.fs.as_ref(), self.shared) + && let Some(session) = fs.user.session_for_via(via) + && let Some(slot) = self + .rings + .iter_mut() + .find(|ring| ring.as_ref().is_some_and(|ring| ring.session == session)) + && !retire_ring(shared, slot) + { + return false; + } + if let Some(fs) = self.fs.as_mut() { + fs.peer_closed(via, reason) + } else { + true + } + } +} + +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +async fn open_block_channel( + transport: &kore_rt::transport::TrapTransport, + boot: &kore_abi::task::Bootstrap<'_>, +) -> Result { + use kore_abi::task::role; + use kore_rt::service::MAX_GRANTS; + + let kick = boot + .handle(role::SVC_SEND, BLOCK_CLIENT_INDEX) + .ok_or(StartFailure::new(1))?; + let ring = boot + .handle(role::SVC_RING, BLOCK_CLIENT_INDEX) + .ok_or(StartFailure::new(2))?; + let leased = kore_rt::cap::verbs::take_view(ring).ok_or(StartFailure::new(3))?; + let window = kore_rt::cap::verbs::map_view(leased).ok_or(StartFailure::new(4))?; + // SAFETY: this task is the sole client-side attacher; block owns the + // serving half and both use only the typed SPSC contract. + let window: &'static mut [u8] = + unsafe { core::slice::from_raw_parts_mut(window.cast::().as_ptr(), window.len()) }; + let rings = client_rings::( + window, + BlockService::REQUEST_CAPACITY, + BlockService::RESPONSE_CAPACITY, + ) + .map_err(|_| StartFailure::new(5))?; + let mut grants = [const { None }; MAX_GRANTS]; + grants[0] = Some(GrantPin::retained( + boot.handle(role::SVC_GRANT, 0) + .ok_or(StartFailure::new(6))?, + )); + grants[1] = Some(GrantPin::retained( + boot.handle(role::SVC_GRANT, 1) + .ok_or(StartFailure::new(7))?, + )); + let mut client = RingBlock(SvcClient::new( + transport, + rings, + ClientEndpoints { kick, grants }, + )); + client + .open(ProvisionRequest { + writable: true, + // The declared intent; whether the platform realizes the + // optional route is discovered at claim time, and a route-less + // machine serves the identical channel at the polled posture. + mode: CompletionMode::Interrupt, + requested_queue_depth: IO_QDEPTH, + requested_buffer_bytes: FS_DMA_POOL_BYTES, + }) + .await + .map_err(|refusal| { + StartFailure::new(match refusal { + BlockRefusal::Unavailable => 9, + BlockRefusal::Policy => 10, + BlockRefusal::Exhausted => 11, + BlockRefusal::Lost => 12, + }) + }) +} + +/// Run block acquisition and both filesystem serving pairs on the executor +/// owned by `#[kore_rt::main]`. The task has one doorbell; startup and serving +/// are phases of one root future, not independently driven runtimes. +/// +/// # Bring-up detail map +/// +/// A founder report of `rung 4` is the runtime's service-start rung; its detail +/// decodes here. Details 1–7 cover the block client endpoints/rings/grants, 8 +/// is retired, 9–12 are block provisioning refusals (unavailable, policy, +/// exhausted, lost), 13 is direct NVMe claim, 14 whole-view/channel +/// construction, 15 provider construction, 16 whole-disk media probe, 17 +/// kore-volume recovery (its higher bytes, when present, are the +/// recovery witness: stage, error kind, cause — `RecoverFailure::witness`, +/// so `0x__1104` reads stage, kind and cause from the digits above `11`), +/// 18 blank formatting, 19 administration-root +/// publication, 20 foreign-media channel containment, 21 recyclable-peer +/// mapping geometry, 22 user-region channel rebuild, 23 the probe inside the +/// user region, 24 durable-workspace publication, 25 application-store +/// publication, 26 system-region channel construction, 27 the probe inside +/// the system region, 28 system-lane containment or missing-era retirement, +/// 29 archive-era root admission, 30 bounded system reformat, and 31 +/// shared-data (`share/`) publication. (`establish_boot_policy` also reports +/// 26 and 27 for library and package publication, colliding with the +/// system-region rungs above; 31 is appended fresh so the data directory +/// does not add a third collision.) +/// Renumber nothing; append new details so a photographed metal failure +/// remains decodable. +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +pub async fn run(rt: kore_rt::entry::Rt) -> ! { + use kore_rt::entry::{ + RUNG_SERVICE_ENDOWMENT, RUNG_SERVICE_SERVE_SETUP, RUNG_SERVICE_START, refuse_with, + }; + use kore_rt::service::{Bundle, SystemService}; + + let Ok(endowment) = FsEndowment::resolve(rt.bootstrap()) else { + refuse_with(&rt, RUNG_SERVICE_ENDOWMENT); + }; + let mut service = DurableService::bind(endowment); + if let Err(failure) = service.start_on_metal(&rt) { + refuse_with(&rt, failure.payload(RUNG_SERVICE_START)); + } + let transport = kore_rt::entry::transport(); + let opened = match open_block_channel(transport, rt.bootstrap()).await { + Ok(opened) => opened, + Err(failure) => refuse_with(&rt, failure.payload(RUNG_SERVICE_START)), + }; + if let Err(failure) = service.initialize(opened).await { + refuse_with(&rt, failure.payload(RUNG_SERVICE_START)); + } + // The steady park arms the held MSI-X route beside the doorbell (the + // kernel's combination is lawful for a route-holding task and refused + // otherwise) — which is also what earns the boot gate's + // `-remapped-msi-irq` witness at the beacon. Enabled only once the + // mounted composition provably retains the route: the RAM fallback + // retired its channel, and its dead route must never join the parks. + if service.holds_route() { + kore_rt::executor::enable_irq_parks().await; + } + // `serve_forever` owns the complete bounded filesystem engine. Pin its + // long-lived future in this task's named heap supply so the outer root + // future carries only the box pointer after startup; inlining the full + // serve state beside initialization exceeds the fixed service stack. + let note = service.readiness_note(); + let serving = Box::pin(kore_rt::entry::serve_forever(&rt, service, None, note)); + serving.await; + refuse_with(&rt, RUNG_SERVICE_SERVE_SETUP) +} + +impl SvcProtocol for DurableService { + type Request = ::Request; + type Response = ::Response; + type ReqWire = ::ReqWire; + type RepWire = ::RepWire; + + const REPLY_GRANTS: usize = ::REPLY_GRANTS; + const SERVE_PAIRS: usize = ::SERVE_PAIRS; + + fn reply_matches(request: &Self::ReqWire, response: &FsResponse, grants: u8) -> bool { + ::reply_matches(request, response, grants) + } + + fn encode_req(request: &FsRequest) -> Self::ReqWire { + ::encode_req(request) + } + fn decode_req(wire: &Self::ReqWire) -> Option { + ::decode_req(wire) + } + fn encode_rep(response: &FsResponse, grants: u8) -> Self::RepWire { + ::encode_rep(response, grants) + } + fn decode_rep(wire: &Self::RepWire) -> Option<(FsResponse, u8)> { + ::decode_rep(wire) + } + fn decode_rep_for(request: &Self::ReqWire, reply: &Self::RepWire) -> Option<(FsResponse, u8)> { + ::decode_rep_for(request, reply) + } +} + +// The boot manifest cites `FsProtocol`; the durable entry must remain the same +// service face even though its provider startup is different. +const _: () = { + // `run` constructs this value on the founded service stack before moving + // it into the named heap. Keep the provider-neutral fallback bounded well + // below the 4 MiB founder stack rather than letting a future engine-growth + // change turn foreign-media containment into a stack fault. + assert!(core::mem::size_of::() < 512 * 1024); + assert!( + ::REPLY_GRANTS + == ::REPLY_GRANTS + ); + assert!( + ::SERVE_PAIRS + == ::SERVE_PAIRS + ); +}; + +#[cfg(test)] +mod retirement_tests { + use super::{ProviderError, RetirementBooks}; + + /// A window still waiting to retire refuses no one; only a failed + /// requester-wide retirement does, and only until an idle pass succeeds. + #[test] + fn only_a_failed_requester_retirement_refuses_new_windows() { + let mut books = RetirementBooks::default(); + assert!(!books.owed()); + // A live retirement deferred: owed, but admission stays open. + books.deferred = 1; + assert!(books.owed()); + assert_eq!(books.admission(), Ok(())); + // The live retry still refused: nothing failed. + books.settle_live(true, 1); + assert_eq!(books.admission(), Ok(())); + // The idle pass fails: the latch is unproven, windows are refused. + books.settle_idle(false, 1); + assert_eq!(books.admission(), Err(ProviderError::Retiring)); + // A live pass cannot prove the latch again, even when it retires. + books.settle_live(true, 0); + assert_eq!(books.admission(), Err(ProviderError::Retiring)); + assert!(books.owed(), "the failure still needs an idle pass"); + // The next successful idle pass clears it. + books.settle_idle(true, 0); + assert_eq!(books.admission(), Ok(())); + assert!(!books.owed()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::persistence::VolumeProbe; + use kore_driver::stor::Single; + use kore_driver::stor::engine::{OneQueue, StorEngine}; + use crate::{Name, ObjectKind, persistence::PersistentFs}; + use kore_driver::block::memory::MemoryBlockDevice; + use kore_driver::gpt::{KORE_FS_TYPE, KORE_SYSTEM_TYPE, synthetic}; + + // A 640-block (512-byte) disk; the GPT designation is large enough for + // the current dual 128 KiB metadata reservation plus a payload arena, + // without making the inline cold-boot fixture dominate the test stack. + type Device = MemoryBlockDevice<327680, 8192>; + type TestProvider = BlockProvider, 3, 8>; + type Volume = PersistentFs, 3, 8>; + + // Slice 4's two-region witness needs a larger disk, but keeping that + // by-value backend out of every pre-existing fixture is load-bearing in + // unoptimized tests: async state retains several device-shaped variants. + type DualDevice = MemoryBlockDevice<786432, 8192>; + + /// The transport engine over one memory disk, `L` lanes — production's + /// composition (`initialize`) with the device a test can take back. + type TestQueues = Single; + type TestEngine = + StorEngine<'static, TestQueues, OneQueue, L, 1, 16, 64, 32>; + type TestBackend = + EngineBackend<'static, TestQueues, OneQueue, L, L, 1, 16, 64, 32>; + + /// `device` behind a fresh engine and `L` filesystem lanes; the engine + /// is placed in its heap slot as `initialize` places it. + fn engine_backend(device: D) -> TestBackend + where + D::Buffer: kore_driver::block::BlockBuffer, + { + let place = Box::leak(Box::>::new_uninit()); + let engine = TestEngine::::init_in(place, Single::new(device)) + .ok() + .expect("the one-queue policy fits"); + let windows = core::array::from_fn(|_| lane_window().expect("a lane window")); + EngineBackend::new(engine, windows, [LaneClasses::FILESYSTEM; L]) + .ok() + .expect("a fresh engine takes its lanes") + } + + /// Take the disk back from a backend whose lanes are all idle — the + /// "power off" between two boots of one disk. + fn engine_device(backend: TestBackend) -> D + where + D::Buffer: kore_driver::block::BlockBuffer, + { + let engine = backend.into_engine().ok().expect("every lane idle"); + // SAFETY: the engine lives in a leaked slot nothing else reaches; + // reading it out moves it, and the slot is never read or dropped + // again. + let engine = unsafe { core::ptr::read(engine) }; + engine + .into_queues() + .ok() + .expect("no ring left attached") + .into_device() + .ok() + .expect("nothing outstanding") + } + type DualProvider = BlockProvider, 3, 8>; + type SystemProvider = BlockProvider, 8, 32>; + type SystemVolume = PersistentFs, 8, 32>; + + const PART_START: u64 = 80; + const PART_END: u64 = 630; + const SYSTEM_PART_START: u64 = 700; + const SYSTEM_PART_END: u64 = 1500; + const TEST_ERA: [u8; crate::persistence::ERA_BYTES] = [0x6a; crate::persistence::ERA_BYTES]; + + #[test] + fn fs_nvme_page_ceiling_matches_the_bounded_dma_budget() { + assert!(fs_nvme_page_supported(MemPage::from_mps(6))); + assert!(!fs_nvme_page_supported(MemPage::from_mps(7))); + } + + // DetachEdge mutates the persisted namespace exactly like Unlink (it can + // even reclaim a populated subtree Unlink refuses), so a bare detach must + // reach commit_after or a crash resurrects the detached edge on reboot. + #[test] + fn detach_edge_is_a_committing_verb() { + let request = FsRequest::DetachEdge { + parent: crate::authority::Handle::from_bits(0), + name: Name::new("tool").unwrap(), + }; + assert!(DurableService::needs_commit(&request, &FsResponse::Done)); + assert!(!DurableService::needs_commit( + &request, + &FsResponse::Refused(FsError::NotFound) + )); + } + + #[test] + fn nvme_bring_up_budget_honors_cap_to_but_cannot_hold_the_shell() { + let missing = kore_nvme::Capabilities::from_cap(0); + let half_second = kore_nvme::Capabilities::from_cap(1u64 << 24); + let fifteen_seconds = kore_nvme::Capabilities::from_cap(30u64 << 24); + + assert_eq!(bring_up_budget_ns(missing), BRING_UP_BUDGET_NS); + assert_eq!(bring_up_budget_ns(half_second), 500_000_000); + assert_eq!(bring_up_budget_ns(fifteen_seconds), BRING_UP_BUDGET_NS); + } + + fn whole_provider(device: Device) -> TestProvider { + DurableService::whole_medium_provider(device, CompletionMode::Poll).unwrap() + } + + /// Take the raw backend out of a provider for a cold-boot simulation. + fn eject(provider: TestProvider) -> Device { + provider + .into_channel() + .unwrap() + .into_device() + .map_err(|(_, error)| error) + .unwrap() + .into_inner() + } + + /// A disk that is foreign as a whole — protective MBR at LBA 0, a GPT + /// at LBA 1 — and designates `[PART_START, PART_END]` for kore. + fn designated_disk() -> Device { + let mut mbr = [0u8; 512]; + mbr[510] = 0x55; + mbr[511] = 0xaa; + let (header, entries) = synthetic::table(KORE_FS_TYPE, PART_START, PART_END, PART_END); + let provider = whole_provider(Device::new(512, 16).unwrap()); + crate::block_on(provider.write_metadata(0, &mbr)).unwrap(); + crate::block_on(provider.write_metadata(1, &header)).unwrap(); + crate::block_on(provider.write_metadata(2, &entries)).unwrap(); + eject(provider) + } + + fn dual_whole_provider(device: DualDevice) -> DualProvider { + DurableService::whole_medium_provider(device, CompletionMode::Poll).unwrap() + } + + fn dual_eject(provider: DualProvider) -> DualDevice { + provider + .into_channel() + .unwrap() + .into_device() + .map_err(|(_, error)| error) + .unwrap() + .into_inner() + } + + fn system_eject(provider: SystemProvider) -> DualDevice { + provider + .into_channel() + .unwrap() + .into_device() + .map_err(|(_, error)| error) + .unwrap() + .into_inner() + } + + fn format_system_era( + device: DualDevice, + extent: gpt::Extent, + era: [u8; crate::persistence::ERA_BYTES], + ) -> DualDevice { + let session = SessionId::new(0); + let Some(Located::Format { + provider, + era: selected, + }) = crate::block_on(DurableService::locate_system( + device, + extent, + CompletionMode::Poll, + era, + )) + .unwrap() + else { + panic!("a blank system extent must format"); + }; + assert_eq!(selected, era); + let mut volume = + crate::block_on(SystemVolume::format_with_era(provider, session, selected)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("stale-system-name"), + ObjectKind::File, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + let (engine, _) = volume.into_parts(); + system_eject(engine.into_provider()) + } + + fn write_system_sentinels(device: DualDevice) -> DualDevice { + let mut channel: BlockChannel = + BlockChannel::new(device, CompletionMode::Poll).unwrap(); + let buffer = channel.register_buffer(512).unwrap(); + for (lba, byte) in [(SYSTEM_PART_START - 1, 0xa5), (SYSTEM_PART_END + 1, 0x5a)] { + channel.fill_buffer(buffer, byte).unwrap(); + let request = channel.submit_write(lba, 1, buffer, 0).unwrap(); + assert_eq!(channel.wait().unwrap().request, request); + } + channel.unregister_buffer(buffer).unwrap(); + channel.into_device().map_err(|(_, error)| error).unwrap() + } + + fn admit_matching_system_era( + device: DualDevice, + extent: gpt::Extent, + era: [u8; crate::persistence::ERA_BYTES], + ) -> DualDevice { + let Some(Located::Recover(provider)) = crate::block_on(DurableService::locate_system( + device, + extent, + CompletionMode::Poll, + era, + )) + .unwrap() else { + panic!("matching v3 roots must recover without reseeding policy"); + }; + system_eject(provider) + } + + fn reformat_system_era( + device: DualDevice, + extent: gpt::Extent, + era: [u8; crate::persistence::ERA_BYTES], + ) -> DualDevice { + let Some(Located::Reformat { + provider, + era: selected, + }) = crate::block_on(DurableService::locate_system( + device, + extent, + CompletionMode::Poll, + era, + )) + .unwrap() + else { + panic!("an old system era must request a bounded reformat"); + }; + assert_eq!(selected, era); + let volume = crate::block_on(SystemVolume::reformat_with_era( + provider, + SessionId::new(0), + selected, + )) + .unwrap(); + assert_eq!(volume.status().era, era); + let (engine, _) = volume.into_parts(); + system_eject(engine.into_provider()) + } + + fn assert_system_sentinels(device: DualDevice) { + let mut channel: BlockChannel = + BlockChannel::new(device, CompletionMode::Poll).unwrap(); + let buffer = channel.register_buffer(512).unwrap(); + let mut observed = [0u8; 512]; + for (lba, byte) in [(SYSTEM_PART_START - 1, 0xa5), (SYSTEM_PART_END + 1, 0x5a)] { + let request = channel.submit_read(lba, 1, buffer, 0).unwrap(); + assert_eq!(channel.wait().unwrap().request, request); + channel.read_buffer(buffer, 0, &mut observed).unwrap(); + assert!(observed.iter().all(|seen| *seen == byte)); + } + } + + fn format_old_system_device( + extent: gpt::Extent, + era: [u8; crate::persistence::ERA_BYTES], + ) -> Device { + let shared = engine_backend::(Device::new(512, 16).unwrap()); + let lane = shared.claim(0).unwrap(); + let Some(Located::Format { + provider, + era: selected, + }) = crate::block_on(DurableService::locate_system( + lane, + extent, + CompletionMode::Poll, + era, + )) + .unwrap() + else { + panic!("blank system media must request format"); + }; + let volume = crate::block_on(PersistentFs::format_with_era( + provider, + SessionId::new(0), + selected, + )) + .unwrap(); + drop(volume); + engine_device(shared) + } + + fn reformat_system_device( + device: Device, + extent: gpt::Extent, + era: [u8; crate::persistence::ERA_BYTES], + cut_after: Option, + ) -> (Device, bool, usize) { + let shared = engine_backend::(device); + let lane = shared.claim(0).unwrap(); + let Some(Located::Reformat { + provider, + era: selected, + }) = crate::block_on(DurableService::locate_system( + lane, + extent, + CompletionMode::Poll, + era, + )) + .unwrap() + else { + panic!("stale system media must request reformat"); + }; + let before = shared.control(|queues| queues.device().submissions()).unwrap(); + if let Some(commands) = cut_after { + shared + .control(|queues| queues.device_mut().inject_after(commands, BlockError::Media)) + .unwrap(); + } + let rebuilt = crate::block_on(PersistentFs::reformat_with_era( + provider, + SessionId::new(0), + selected, + )); + let succeeded = rebuilt.is_ok(); + drop(rebuilt); + let commands = usize::try_from( + shared.control(|queues| queues.device().submissions()).unwrap() - before, + ) + .unwrap(); + (engine_device(shared), succeeded, commands) + } + + fn assert_reboot_never_admits_a_stale_era( + device: Device, + extent: gpt::Extent, + era: [u8; crate::persistence::ERA_BYTES], + ) { + let shared = engine_backend::(device); + let lane = shared.claim(0).unwrap(); + let located = crate::block_on(DurableService::locate_system( + lane, + extent, + CompletionMode::Poll, + era, + )) + .expect("a consumed one-shot cut must leave readable media") + .expect("the nonzero archive era keeps system policy active"); + match &located { + Located::Recover(provider) => assert!( + crate::block_on(PersistentFs::roots_match_era(provider, era)).unwrap(), + "ordinary recovery may only admit the requested era" + ), + Located::Format { .. } | Located::Reformat { .. } => {} + Located::Volatile(_) => panic!("system policy never fabricates a RAM recovery"), + } + drop(located); + drop(shared); + } + + fn dual_designated_disk() -> DualDevice { + let mut mbr = [0u8; 512]; + mbr[510] = 0x55; + mbr[511] = 0xaa; + let (header, entries) = synthetic::table_pair( + (KORE_FS_TYPE, PART_START, PART_END), + Some((KORE_SYSTEM_TYPE, SYSTEM_PART_START, SYSTEM_PART_END)), + SYSTEM_PART_END, + ); + let provider = dual_whole_provider(DualDevice::new(512, 16).unwrap()); + crate::block_on(provider.write_metadata(0, &mbr)).unwrap(); + crate::block_on(provider.write_metadata(1, &header)).unwrap(); + crate::block_on(provider.write_metadata(2, &entries)).unwrap(); + dual_eject(provider) + } + + fn name(value: &str) -> Name { + Name::new(value).unwrap() + } + + fn on_large_stack(f: impl FnOnce() + Send + 'static) { + // The region-discovery cold-boot arc holds the whole-disk and narrowed + // provider/engine shapes across a locate boundary. The raised inline + // directory arena makes this the larger of the two durability + // fixtures even though publication images themselves are heap-backed. + std::thread::Builder::new() + .name("durable-region-test".into()) + .stack_size(64 * 1024 * 1024) + .spawn(f) + .unwrap() + .join() + .unwrap() + } + + fn on_system_rebuild_stack(f: impl FnOnce() + Send + 'static) { + // This witness deliberately carries the 768 KiB whole-disk double by + // value through four independent cold-boot futures. Production uses a + // small shared-lane reference, but the test must retain the backing to + // inspect bytes outside the clamp after every provider is consumed. + // Keep that debug-only ownership shape off the runner's normal stack. + std::thread::Builder::new() + .name("system-era-rebuild-test".into()) + .stack_size(256 * 1024 * 1024) + .spawn(f) + .unwrap() + .join() + .unwrap() + } + + #[test] + fn a_disk_designating_nothing_lands_volatile() { + on_large_stack(|| { + // Foreign roots, no GPT: the exact metal boot-disk shape the + // RAM fallback exists for. + let provider = whole_provider(Device::new(512, 16).unwrap()); + let mut junk = [0u8; 512]; + junk[..8].copy_from_slice(b"NTFS "); + crate::block_on(provider.write_metadata(0, &junk)).unwrap(); + let located = + crate::block_on(DurableService::locate(provider, CompletionMode::Poll)).unwrap(); + assert!(matches!( + located, + Located::Volatile(note::VOLATILE_UNDESIGNATED) + )); + }); + } + + #[test] + fn an_unavailable_designated_system_store_never_falls_back_into_user_bytes() { + let user = ::bind(FsEndowment::UNSTAMPED); + let mut mounted = MountedFs { + user: MountedEngine::Volatile(Box::new(user)), + system: None, + system_designated: true, + user_lifecycle: RegionLifecycle::Current, + system_lifecycle: RegionLifecycle::Unavailable, + system_seed_resume: RegionLifecycle::Unavailable, + }; + assert!(mounted.establish_boot_policy().is_ok()); + let MountedEngine::Volatile(user) = &mounted.user else { + unreachable!() + }; + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let grants = user.engine_ref().grants(admin).unwrap(); + assert!(grants.iter().all(|grant| { + grant.display_name != Some(name(crate::SYSTEM_SOURCE_GRANT)) + && grant.display_name != Some(name(crate::SYSTEM_APPS_GRANT)) + })); + } + + #[test] + fn system_seed_lifecycle_is_idempotent_and_restores_its_mount_outcome() { + let user = ::bind(FsEndowment::UNSTAMPED); + let system = ::bind(FsEndowment::UNSTAMPED); + let mut mounted = MountedFs { + user: MountedEngine::Volatile(Box::new(user)), + system: Some(MountedEngine::Volatile(Box::new(system))), + system_designated: true, + user_lifecycle: RegionLifecycle::Current, + // A rebuilt mount enters this posture before status becomes + // reachable; exec's first `active` publication is idempotent. + system_lifecycle: RegionLifecycle::Reseeding, + system_seed_resume: RegionLifecycle::Rebuilt, + }; + + assert_eq!( + mounted.set_system_seed_active(0, true), + Err(FsError::AccessDenied) + ); + assert_eq!(mounted.system_lifecycle, RegionLifecycle::Reseeding); + mounted + .set_system_seed_active(crate::service::contract::ADMIN_VIA, true) + .unwrap(); + assert_eq!(mounted.system_lifecycle, RegionLifecycle::Reseeding); + mounted + .set_system_seed_active(crate::service::contract::ADMIN_VIA, true) + .unwrap(); + assert_eq!(mounted.system_seed_resume, RegionLifecycle::Rebuilt); + mounted + .set_system_seed_active(crate::service::contract::ADMIN_VIA, false) + .unwrap(); + assert_eq!(mounted.system_lifecycle, RegionLifecycle::Rebuilt); + + // The reformatting vocabulary is a real bounded phase even though a + // service cannot answer clients until initialization has completed. + assert_eq!( + RegionLifecycle::Reformatting.wire(), + kore_abi::svc::FsSystemStatusWire::LIFECYCLE_REFORMATTING + ); + } + + #[test] + fn an_app_only_secondary_peer_completes_and_releases_its_session() { + use kore_rt::svc::PeerId; + + let mut user = ::bind(FsEndowment::UNSTAMPED); + let mut system = ::bind(FsEndowment::UNSTAMPED); + assert!(user.establish_admin_source()); + assert!(system.establish_admin_source()); + assert!(system.establish_admin_apps()); + user.register_system_invocation_view(0xD1, crate::Rights::ALL) + .unwrap(); + + let mut mounted = MountedFs { + user: MountedEngine::Volatile(Box::new(user)), + system: Some(MountedEngine::Volatile(Box::new(system))), + system_designated: true, + user_lifecycle: RegionLifecycle::Current, + system_lifecycle: RegionLifecycle::Rebuilt, + system_seed_resume: RegionLifecycle::Rebuilt, + }; + let first = PeerId::from_arrival(0x102).unwrap(); + assert!(mounted.peer_admitted(Ok(first))); + + let MountedEngine::Volatile(user) = &mut mounted.user else { + unreachable!() + }; + user.apply_invocation_view(0xD1, first, crate::Rights::ALL) + .unwrap(); + let MountedEngine::Volatile(system) = mounted.system.as_mut().unwrap() else { + unreachable!() + }; + let session = SessionId::new(first.slot() as u16); + assert!(system.install_apps_view(session)); + let grants = system.engine_ref().grants(session).unwrap(); + assert!( + grants + .iter() + .all(|grant| grant.rights == crate::Rights::READ_ONLY), + "the secondary session exposes no authority wider than the app view" + ); + let apps = grants + .into_iter() + .find(|grant| grant.display_name == Some(name(crate::SYSTEM_APPS_VIEW))) + .expect("the split system engine publishes the canonical apps view"); + assert_eq!(apps.rights, crate::Rights::READ_ONLY); + + assert!( + mounted.peer_closed(first.arrival(), PeerClose::Orderly), + "both engines must leave the peer cell reusable" + ); + let replacement = PeerId::from_arrival(0x202).unwrap(); + assert_eq!(replacement.slot(), first.slot()); + assert!( + mounted.peer_admitted(Ok(replacement)), + "the exact bounded cell is available to its next generation" + ); + } + + #[test] + fn dual_designations_mount_two_disjoint_engines_over_one_backend() { + on_large_stack(|| { + let shared = engine_backend::(dual_designated_disk()); + let user_lane = shared.claim(0).unwrap(); + let user_device = RegionDevice::whole(user_lane).unwrap(); + let user_channel = BlockChannel::new(user_device, CompletionMode::Poll).unwrap(); + let bootstrap = bootstrap_data_start(user_channel.geometry().block_size).unwrap(); + let user_provider = + BlockProvider::<_, 8, 32>::new_with_data_start(user_channel, bootstrap).unwrap(); + + let user_extent = crate::block_on(DurableService::discover_region( + &user_provider, + KORE_FS_TYPE, + )) + .unwrap(); + let system_extent = crate::block_on(DurableService::discover_region( + &user_provider, + KORE_SYSTEM_TYPE, + )) + .unwrap(); + assert!(user_extent.end_lba < system_extent.start_lba); + + // Claim the sibling before the user locator may retire its lane; + // this is the production lifecycle ordering, not test ceremony. + let system_lane = shared.claim(1).unwrap(); + let Located::Format { + provider: user_provider, + .. + } = crate::block_on(DurableService::locate(user_provider, CompletionMode::Poll)) + .unwrap() + else { + panic!("blank user designation formats"); + }; + let Some(Located::Format { + provider: system_provider, + era, + }) = crate::block_on(DurableService::locate_system( + system_lane, + system_extent, + CompletionMode::Poll, + TEST_ERA, + )) + .unwrap() + else { + panic!("blank system designation formats"); + }; + assert_eq!(era, TEST_ERA); + assert_eq!(user_provider.geometry().block_count, user_extent.blocks()); + assert_eq!( + system_provider.geometry().block_count, + system_extent.blocks() + ); + + let session = SessionId::new(0); + let user = crate::block_on(PersistentFs::format(user_provider, session)).unwrap(); + let system = crate::block_on(PersistentFs::format(system_provider, session)).unwrap(); + let (user_engine, _) = user.into_parts(); + let (system_engine, _) = system.into_parts(); + let mut user = FsService::new(user_engine); + let mut system = FsService::new(system_engine); + user.bind_session(session); + system.bind_session(session); + assert!(user.establish_admin_namespace()); + assert!(user.establish_admin_workspace()); + assert!(system.establish_admin_namespace()); + assert!(system.establish_admin_source()); + assert!(system.establish_admin_apps()); + + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + assert!( + user.engine_ref() + .grants(admin) + .unwrap() + .iter() + .all(|grant| grant.display_name != Some(name(crate::SYSTEM_APPS_GRANT))) + ); + assert!( + system + .engine_ref() + .grants(admin) + .unwrap() + .iter() + .any(|grant| grant.display_name == Some(name(crate::SYSTEM_APPS_GRANT))) + ); + assert_eq!(shared.stats().map(|(stats, _)| stats.stale_completions), Some(0)); + }); + } + + #[test] + fn changed_system_era_reformats_inside_both_adjacent_sentinels() { + on_system_rebuild_stack(|| { + const OLD_ERA: [u8; crate::persistence::ERA_BYTES] = + [0x71; crate::persistence::ERA_BYTES]; + const NEW_ERA: [u8; crate::persistence::ERA_BYTES] = + [0x72; crate::persistence::ERA_BYTES]; + let extent = gpt::Extent { + start_lba: SYSTEM_PART_START, + end_lba: SYSTEM_PART_END, + }; + // Write one entire sentinel block on each side through the raw + // whole-disk channel. The narrowed system provider cannot name + // either LBA, so these are an adversarial witness of the clamp, + // not merely an assertion about offset arithmetic. + let device = format_system_era(dual_designated_disk(), extent, OLD_ERA); + let device = write_system_sentinels(device); + let device = admit_matching_system_era(device, extent, OLD_ERA); + let device = reformat_system_era(device, extent, NEW_ERA); + assert_system_sentinels(device); + }); + } + + #[test] + fn every_reformat_command_cut_reboots_without_admitting_the_stale_era() { + on_large_stack(|| { + const OLD_ERA: [u8; crate::persistence::ERA_BYTES] = + [0x81; crate::persistence::ERA_BYTES]; + const NEW_ERA: [u8; crate::persistence::ERA_BYTES] = + [0x82; crate::persistence::ERA_BYTES]; + let extent = gpt::Extent { + start_lba: PART_START, + end_lba: PART_END, + }; + + // Measure the exact command count rather than hand-maintaining a + // guessed cut matrix beside provider batching. Then replay every + // boundary against a fresh old-era medium. + let old = format_old_system_device(extent, OLD_ERA); + let (current, succeeded, command_count) = + reformat_system_device(old, extent, NEW_ERA, None); + assert!(succeeded); + assert!( + command_count > 2, + "the witness includes clear, flush, and publish" + ); + drop(current); + + for cut_after in 0..command_count { + let old = format_old_system_device(extent, OLD_ERA); + let (interrupted, succeeded, _) = + reformat_system_device(old, extent, NEW_ERA, Some(cut_after)); + assert!(!succeeded, "cut {cut_after} must interrupt its command"); + assert_reboot_never_admits_a_stale_era(interrupted, extent, NEW_ERA); + } + }); + } + + #[test] + fn a_designated_but_foreign_region_is_never_overwritten() { + on_large_stack(|| { + // Stale bytes at the partition's first block: designation + // admits the region, but old data is evidence, not consent. + // The fixture writes through a raw channel — the provider's + // own metadata window deliberately cannot reach the partition. + let mut channel: BlockChannel = + BlockChannel::new(designated_disk(), CompletionMode::Poll).unwrap(); + let buffer = channel.register_buffer(512).unwrap(); + let mut stale = [0u8; 512]; + stale[..4].copy_from_slice(b"ext4"); + channel.write_buffer(buffer, 0, &stale).unwrap(); + let request = channel.submit_write(PART_START, 1, buffer, 0).unwrap(); + assert_eq!(channel.wait().unwrap().request, request); + channel.unregister_buffer(buffer).unwrap(); + let device = channel.into_device().map_err(|(_, error)| error).unwrap(); + + let located = crate::block_on(DurableService::locate( + whole_provider(device), + CompletionMode::Poll, + )) + .unwrap(); + assert!(matches!( + located, + Located::Volatile(note::VOLATILE_REFUSED_REGION) + )); + }); + } + + #[test] + fn a_dynamic_peers_writes_land_in_the_persisted_tree_and_survive_recovery() { + use kore_rt::svc::PeerId; + + on_large_stack(|| { + let session = SessionId::new(0); + let device = designated_disk(); + let Located::Format { provider, .. } = crate::block_on(DurableService::locate( + whole_provider(device), + CompletionMode::Poll, + )) + .unwrap() else { + panic!("a blank designated region must land on Format"); + }; + let volume = crate::block_on(Volume::format(provider, session)).unwrap(); + let (engine, mut mount) = volume.into_parts(); + let mut fs = FsService::new(engine); + fs.bind_session(session); + assert!(fs.establish_admin_source()); + + // Admit a dynamic peer exactly the way the serving host does. + // Admission parks it in the attach transition; its structural + // root arrives only through the endpoint-authenticated view + // application — a delegation of the persisted system root, so + // what it creates is a fact of the persisted tree. + let peer = PeerId::from_arrival(0x106).unwrap(); + assert!(fs.peer_admitted(Ok(peer))); + fs.register_system_invocation_view(0xD1, crate::Rights::ALL) + .unwrap(); + fs.apply_invocation_view(0xD1, peer, crate::Rights::ALL) + .unwrap(); + let peer_session = SessionId::new(peer.slot() as u16); + let root = fs + .engine() + .open_root(peer_session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(fs.engine().create( + peer_session, + root.handle, + &name("data"), + ObjectKind::File, + )) + .unwrap(); + crate::block_on(fs.engine().write_at( + peer_session, + file.handle, + 0, + b"peer-durable", + crate::ANY_REVISION, + )) + .unwrap(); + // The durable entry's own commit: session zero's namespace. + crate::block_on(commit_engine(fs.engine(), &mut mount, session, None)).unwrap(); + + // Peer teardown must retire only the delegation, never the tree + // it was a view of. + assert!(fs.peer_closed(peer.arrival(), PeerClose::Orderly)); + + // Cold boot: rediscover, recover, and read the peer's bytes back + // through a fresh peer generation. + let device = eject(fs.into_engine().into_provider()); + let Located::Recover(provider) = crate::block_on(DurableService::locate( + whole_provider(device), + CompletionMode::Poll, + )) + .unwrap() else { + panic!("a committed designated region must land on Recover"); + }; + let volume = crate::block_on(Volume::recover(provider, session)).unwrap(); + let (engine, _) = volume.into_parts(); + let mut fs = FsService::new(engine); + fs.bind_session(session); + assert!(fs.establish_admin_source()); + let reborn = PeerId::from_arrival(0x206).unwrap(); + assert!(fs.peer_admitted(Ok(reborn))); + fs.register_system_invocation_view(0xD2, crate::Rights::ALL) + .unwrap(); + fs.apply_invocation_view(0xD2, reborn, crate::Rights::ALL) + .unwrap(); + let peer_session = SessionId::new(reborn.slot() as u16); + let root = fs + .engine() + .open_root(peer_session, crate::Rights::ALL) + .unwrap(); + let opened = fs + .engine() + .open_at(peer_session, root.handle, &name("data"), crate::Rights::ALL) + .unwrap(); + let mut back = [0u8; 12]; + let read = crate::block_on(fs.engine().read(peer_session, opened.handle, 0, &mut back)) + .unwrap(); + assert_eq!(&back[..read], b"peer-durable"); + }); + } + + #[test] + fn a_designated_region_formats_persists_and_recovers_across_cold_boots() { + on_large_stack(|| { + let session = SessionId::new(0); + let device = designated_disk(); + + // First boot: whole disk is foreign, the designation admits + // the blank region, and the mount formats inside it. + let located = crate::block_on(DurableService::locate( + whole_provider(device), + CompletionMode::Poll, + )) + .unwrap(); + let Located::Format { provider, .. } = located else { + panic!("a blank designated region must land on Format"); + }; + assert_eq!(provider.geometry().block_count, PART_END - PART_START + 1); + + let mut volume = crate::block_on(Volume::format(provider, session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("data"), + ObjectKind::File, + )) + .unwrap(); + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"region-durable", + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + + // Cold boot: rediscover from the whole disk, land on Recover, + // and read the exact bytes back from inside the region. + let (engine, _) = volume.into_parts(); + let device = eject(engine.into_provider()); + let located = crate::block_on(DurableService::locate( + whole_provider(device), + CompletionMode::Poll, + )) + .unwrap(); + let Located::Recover(provider) = located else { + panic!("a formatted designated region must land on Recover"); + }; + let recovered = SessionId::new(1); + let mut volume = crate::block_on(Volume::recover(provider, recovered)).unwrap(); + let root = volume + .engine_mut() + .open_root(recovered, crate::Rights::ALL) + .unwrap(); + let opened = volume + .engine_mut() + .open_at(recovered, root.handle, &name("data"), crate::Rights::ALL) + .unwrap(); + let mut back = [0u8; 14]; + let read = crate::block_on(volume.engine_mut().read( + recovered, + opened.handle, + 0, + &mut back, + )) + .unwrap(); + assert_eq!(&back[..read], b"region-durable"); + + // The partition table itself was never rewritten: the whole + // disk still probes foreign with an intact designation. + let (engine, _) = volume.into_parts(); + let device = eject(engine.into_provider()); + let provider = whole_provider(device); + assert_eq!( + crate::block_on(PersistentFs::probe(&provider)), + Ok(VolumeProbe::Foreign), + "the region mount must never touch the whole-disk roots" + ); + assert!( + crate::block_on(DurableService::discover_region( + &provider, + gpt::KORE_FS_TYPE + )) + .is_some() + ); + }); + } +} diff --git a/native/services/dagfs/src/engine.rs b/native/services/dagfs/src/engine.rs new file mode 100644 index 0000000..dd2adb0 --- /dev/null +++ b/native/services/dagfs/src/engine.rs @@ -0,0 +1,4139 @@ +//! The request engine — generic over its [`StorageProvider`]. +//! +//! This is the whole of the filesystem's behaviour, and it is deliberately +//! the only place the three lower layers meet: it revalidates a route through +//! [`Authority`], mutates the DAG through [`Namespace`], and moves bytes +//! through the provider. Nothing below it knows about the others. +//! +//! # Every operation revalidates +//! +//! There is no fast path. Each method begins by re-walking the handle's +//! bounded edge proof ([`Authority::revalidate`]), and only then checks +//! rights. A client that holds a numeric handle whose route has been +//! dismantled gets [`FsError::AccessLost`] on its next call, not on some +//! later cache expiry — and because the engine keeps no decision from a +//! previous request, there is nothing that could bypass the check. +//! +//! # What makes a request atomic +//! +//! A mutation either commits completely or has no visible effect. +//! Two properties deliver that: +//! +//! 1. **Capacity is reserved before any byte lands.** The engine calls +//! [`StorageProvider::resize`] to cover the whole target range first, so +//! the failure that can run out of room happens while nothing is visible. +//! The subsequent write goes into already-reserved space. +//! 2. **Length and revision commit last, together.** A file's visible length +//! and its revision advance in one step *after* the bytes are in place +//! ([`Object::commit_len`](crate::object)). A refused or failed write +//! leaves both untouched, so a reader can never observe a half-applied +//! request — it sees the previous complete revision. +//! +//! `append` gets its atomicity from the same structure: it claims the exact +//! file generation before choosing the current end and retains that affine +//! claim through provider I/O and the final metadata commit. A conflicting +//! operation receives bounded [`FsError::Busy`] backpressure while unrelated +//! objects and directories continue independently; there is no global +//! namespace mutation gate. + +use crate::{ + ServiceAlloc, + authority::{ + Authority, AuthorityKind, AuthorityQuota, EdgeProof, GrantId, Handle, QuotaClaim, Retired, + Rights, Route, SessionId, + }, + bounds::{ + MAX_DERIVED_RESULTS, MAX_EDGES_PER_SESSION, MAX_LIST_PAGE, MAX_OBJECTS, MAX_PATH_DEPTH, + MAX_TRANSFER, RECLAIM_CONTENTION_TURNS, + }, + error::{FsError, FsResult, ProviderError}, + extraction::{ + DerivedResultInfo, ExtractionError, ExtractionOwner, ExtractionState, Snapshot, + VersionLease, + }, + namespace::{ListEntry, Name, Namespace}, + object::{Object, ObjectId, ObjectKind, ObjectSubtype, Revision, StableObjectId}, + provider::{RamProvider, StorageProvider}, + search::{ + FreshnessPolicy, MAX_SEARCH_PAGE, MAX_SEARCH_SCAN, QueryIndex, SearchCursor, SearchHit, + SearchPage, SearchQuery, SearchScore, + }, + slot::BoundedVec, +}; +use kore_svc_extract::{ + registry::ParserId, + schema::{InputIdentity, NormalizedResult}, +}; + +/// Require the authority `subtype`'s stamp demands +/// ([`ObjectSubtype::stamp_rights`]) on top of an operation's own right. +/// +/// Asked twice for one reason. At creation, so a writer cannot forge an +/// attesting stamp; and wherever a file's bytes or length change, so a +/// writer cannot keep the installer's stamp and swap what it vouches for — +/// an attestation stays true only while its bytes change through the +/// authority that may make it. Callers ask while they hold the object, so +/// the check and the mutation see one object (a subtype never changes). +fn require_stamp_authority(rights: Rights, subtype: ObjectSubtype) -> FsResult<()> { + rights.require(subtype.stamp_rights()) +} + +struct SearchWalk { + hash: u64, + snapshot: u64, + scanned: u32, + work: u32, + work_limit: u32, + index_generation: u64, + start: u32, + next: Option, + hits: heapless::Vec, +} + +impl SearchWalk { + const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; + const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; + + fn new(start: u32, work_limit: u32) -> Self { + Self { + hash: Self::FNV_OFFSET, + snapshot: Self::FNV_OFFSET, + scanned: 0, + work: 0, + work_limit, + index_generation: Self::FNV_OFFSET, + start, + next: None, + hits: heapless::Vec::new(), + } + } + + fn hash_bytes(&mut self, bytes: &[u8]) { + for byte in bytes { + self.hash ^= u64::from(*byte); + self.hash = self.hash.wrapping_mul(Self::FNV_PRIME); + } + } + + fn hash_u64(&mut self, value: u64) { + self.hash_bytes(&value.to_le_bytes()); + } + + fn hash_structural_bytes(&mut self, bytes: &[u8]) { + self.hash_bytes(bytes); + for byte in bytes { + self.snapshot ^= u64::from(*byte); + self.snapshot = self.snapshot.wrapping_mul(Self::FNV_PRIME); + } + } + + fn hash_structural_u64(&mut self, value: u64) { + self.hash_structural_bytes(&value.to_le_bytes()); + } + + fn hash_index_generation(&mut self, value: u64) { + for byte in value.to_le_bytes() { + self.index_generation ^= u64::from(byte); + self.index_generation = self.index_generation.wrapping_mul(Self::FNV_PRIME); + } + } + + fn charge(&mut self, units: u32) -> FsResult<()> { + self.work = self + .work + .checked_add(units) + .ok_or(FsError::BudgetExceeded)?; + if self.work > self.work_limit { + return Err(FsError::BudgetExceeded); + } + Ok(()) + } +} + +/// Yield one executor turn without a timer or allocation. +async fn yield_once() { + let mut yielded = false; + core::future::poll_fn(|cx| { + if core::mem::replace(&mut yielded, true) { + core::task::Poll::Ready(()) + } else { + cx.waker().wake_by_ref(); + core::task::Poll::Pending + } + }) + .await +} + +/// The bounded in-memory composition of the provider-neutral engine. +/// +/// `Engine>` uses this same engine with no provider branch +/// in namespace or authority code. +pub type RamEngine = Engine; + +/// What a client learns when it opens or creates an object. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct OpenedObject { + pub handle: Handle, + /// Durable identity for cache correlation. It grants no authority; the + /// live session-bound `handle` remains required for every operation. + pub stable_object: StableObjectId, + pub kind: ObjectKind, + /// Data, never authority ([`ObjectSubtype`]). + pub subtype: ObjectSubtype, + pub len: u64, + pub revision: Revision, +} + +/// An object's metadata. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct StatInfo { + /// Which object this is. + /// + /// Safe to hand out: an object id conveys no authority, and a + /// client that has one still needs a live route to do anything with it. + /// Exposing it is what lets a caller tell whether two names it holds are + /// two names for the *same* object rather than for copies. + pub object: ObjectId, + /// Stable naming identity stored in durable metadata. It grants no + /// authority; handles and [`ObjectId`] remain boot-local. + pub stable_object: StableObjectId, + pub kind: ObjectKind, + /// Data, never authority ([`ObjectSubtype`]). + pub subtype: ObjectSubtype, + pub len: u64, + pub revision: Revision, + /// Containment edges naming this object — how many names it has. + pub links: u32, + /// Live authority edges rooted at it. + pub grants: u32, +} + +/// One page of a directory listing. +#[derive(Clone, Debug)] +pub struct ListPage { + pub entries: heapless::Vec, + /// Cursor to resume at, or `None` when the listing is complete. + pub next: Option, +} + +/// One visible parent of an object, as [`Engine::parents`] reports it. +#[derive(Clone, Debug)] +pub struct ParentInfo { + /// The parent directory — observation data, never a handle. + pub object: ObjectId, + pub stable_object: StableObjectId, + pub revision: Revision, + /// Names from the handle's authority root to the parent; empty when the + /// parent is that root. + pub path: heapless::Vec, +} + +/// An object found by its identity, as [`Engine::locate`] reports it. +#[derive(Clone, Debug)] +pub struct LocateInfo { + /// The object — observation data, never a handle. + pub object: ObjectId, + pub stable_object: StableObjectId, + pub kind: ObjectKind, + pub len: u64, + pub revision: Revision, + /// Names from the handle's authority root to the object, its own last; + /// empty when the object is that root. + pub path: heapless::Vec, +} + +/// One page of an object's visible parents. +#[derive(Clone, Debug)] +pub struct ParentsPage { + pub parents: heapless::Vec, + /// Cursor to resume at, or `None` when every parent was reported. + pub next: Option, +} + +/// One of a session's authority edges, as the session sees it. +#[derive(Clone, Copy, Debug)] +pub struct GrantInfo { + pub grant: GrantId, + pub kind: AuthorityKind, + pub rights: Rights, + pub quota: AuthorityQuota, + pub display_name: Option, +} + +/// Private placement owned by one service session, never by a namespace +/// route or durable root. Not Copy: cleanup retains custody on refusal. +pub(crate) struct Replacement { + pub(crate) handle: Handle, + object: ObjectId, + revision: Revision, + key: K, + length: u64, + written: u64, +} + +/// The filesystem engine. +pub struct Engine { + ns: Namespace, + auth: Authority, + provider: P, + extraction: ExtractionState, + /// Whether the volume behind this engine can record an object subtype. + /// A fresh or RAM engine always can; a recovered volume can only from + /// the durable format that added the byte, and it keeps committing at + /// its own version — so on an older volume a non-plain create refuses + /// rather than holding a fact the disk would silently drop. + durable_subtypes: bool, + /// Every file version's content digest this engine has computed, keyed + /// by revision ([`crate::digests`]). A lend answers from it instead of + /// rereading the file; a durable engine persists it in its root's + /// sidecar and loads it back at mount. + digests: core::cell::RefCell, +} + +impl Engine

{ + pub fn new(provider: P) -> Self { + Self { + ns: Namespace::new(), + auth: Authority::new(), + provider, + extraction: ExtractionState::new(), + durable_subtypes: true, + digests: core::cell::RefCell::new(crate::digests::DigestMemo::new( + crate::bounds::MAX_OBJECTS, + )), + } + } + + /// The remembered content digests. + pub(crate) fn digests(&self) -> &core::cell::RefCell { + &self.digests + } + + pub(crate) fn namespace(&self) -> &Namespace { + &self.ns + } + + /// Live sessions (attached or mid-transition), for the diagnostic + /// snapshot. A count, never an enumeration. + pub(crate) fn live_sessions(&self) -> u16 { + self.auth.live_sessions() + } + + /// The persisted tree's aggregate usage — objects, containment edges, + /// file bytes reachable from session zero's root. + pub(crate) fn system_tree_usage(&self) -> FsResult<(u32, u32, u64)> { + let root = self.auth.root_object(SessionId::new(0))?; + let usage = self.ns.graph_usage(root)?; + Ok((usage.objects, usage.edges, usage.bytes)) + } + + pub(crate) fn root_object(&self, session: SessionId) -> FsResult { + self.auth.root_object(session) + } + + pub(crate) fn from_recovered( + provider: P, + ns: Namespace, + session: SessionId, + root: ObjectId, + durable_subtypes: bool, + ) -> FsResult { + if !ns.kind(root)?.is_dir() { + return Err(FsError::NotDirectory); + } + let auth = Authority::new(); + auth.attach_session(session)?; + auth.install_root(session, root)?; + ns.object_mut(root)?.grants_inc(); + Ok(Self { + ns, + auth, + provider, + extraction: ExtractionState::new(), + durable_subtypes, + digests: core::cell::RefCell::new(crate::digests::DigestMemo::new( + crate::bounds::MAX_OBJECTS, + )), + }) + } + + /// The provider, for limits and (in tests) fault injection. + pub fn provider(&self) -> &P { + &self.provider + } + + pub(crate) fn into_provider(self) -> P { + self.provider + } + + /// Live objects across every session's namespace. + pub fn object_count(&self) -> usize { + self.ns.object_count() + } + + /// Live directory child lists — the arena bounded separately from + /// objects, so it is worth being able to see it drain. + pub fn directory_count(&self) -> usize { + self.ns.directory_count() + } + + /// Live authority edges. + pub fn authority_edge_count(&self) -> usize { + self.auth.edge_count() + } + + /// Placements the provider refused to release. See + /// [`Namespace::reclaim`] on why a release failure is counted rather than + /// reported as a failure of the request that caused it. + pub fn leaked_placements(&self) -> u32 { + self.ns.leaked_placements() + } + + /// Reclamations that stopped before reaching the whole unreachable + /// subtree — a structural refusal or exhausted contention budget. Non-zero + /// means storage is occupied by objects nothing can reach. + pub fn abandoned_reclaims(&self) -> u32 { + self.ns.abandoned_reclaims() + } + + /// Open handles a session holds. + pub fn live_handles(&self, session: SessionId) -> usize { + self.auth.live_handles(session) + } + + // ── sessions ──────────────────────────────────────────────────────────── + + /// Attach a session and give it a fresh, empty private root. + /// + /// This is the only way a session acquires filesystem access: there is no + /// machine-wide root to inherit, so a task that is never attached has no + /// route to anything: filesystem access is never ambient. + pub async fn attach_session(&self, session: SessionId) -> FsResult<()> { + // Validate the session slot before allocating anything, so a bad + // session id cannot leave a stray root behind. + self.auth.attach_session(session)?; + let root = match self.ns.create_dir_object() { + Ok(root) => root, + Err(error) => { + self.auth.abort_attach(session); + return Err(error); + } + }; + match self.auth.install_root(session, root) { + Ok(_) => { + self.ns.object_mut(root)?.grants_inc(); + Ok(()) + } + Err(err) => { + self.auth.abort_attach(session); + // Undo the root: nothing names it and nothing grants it, so + // reclamation removes it cleanly. + self.reclaim(root).await; + Err(err) + } + } + } + + /// Claim a task session without manufacturing an ambient root. A later + /// endpoint-authenticated invocation view completes the root transition. + pub(crate) fn attach_empty_session(&self, session: SessionId) -> FsResult<()> { + self.auth.attach_session(session).map(|_| ()) + } + + /// Pin one caller-authorized route as a service-private delegation + /// ancestor. The returned id never crosses the public filesystem wire. + pub(crate) fn pin_view( + &self, + session: SessionId, + handle: Handle, + rights: Rights, + ) -> FsResult<(GrantId, SessionId)> { + let route = self.route(session, handle)?; + // Session zero is the persistent bootstrap owner and therefore has + // no earlier invocation ancestor. Every recyclable process must pin + // beneath the stable ancestor from which its root was installed. + if session == SessionId::new(0) { + return self + .grant(session, handle, session, rights, None) + .map(|grant| (grant, session)); + } + self.auth.pin_route(session, route, rights) + } + + /// Whether `handle` routes to the object the view rooted at `view` is + /// bound to: a cwd anchor's check that the route a launcher spelled from + /// its cwd still names that cwd. The view must be `task`'s own — the + /// stable ancestor of the handle's route — so a selector of another + /// task's view answers nothing about it. A view whose route no longer + /// validates is `AccessLost`, as any handle through a retired edge. + pub(crate) fn matches_view( + &self, + session: SessionId, + handle: Handle, + view: GrantId, + task: SessionId, + ) -> FsResult { + let route = self.route(session, handle)?; + let (_, ancestor) = self.auth.stable_ancestor(session, route.grant)?; + if ancestor != task { + return Err(FsError::AccessDenied); + } + Ok(self.auth.validated_edge(view)?.root() == route.object) + } + + /// Complete an empty dynamic session with an attenuated descendant of a + /// pinned structural view. + pub(crate) fn install_view_root( + &self, + session: SessionId, + view: GrantId, + rights: Rights, + ) -> FsResult<()> { + let grant = self.auth.install_delegated_root(session, view, rights)?; + let object = self.auth.validated_edge(grant)?.root(); + self.ns.object_mut(object)?.grants_inc(); + Ok(()) + } + + /// The private-root edge owned by a stable task-view session. The id is + /// service-private and is used only as a derivation ancestor for transient + /// command sessions. + pub(crate) fn task_view_root(&self, session: SessionId) -> FsResult { + self.auth.root_grant(session) + } + + /// Attach a session whose structural root is an attenuated delegation of + /// `source` — the invocation-view posture ported from the + /// capability-selected execution branch: the session enters the + /// filesystem through an existing view (typically the persisted system + /// root pinned at bootstrap) instead of receiving a fresh ambient + /// private namespace. Everything it creates therefore lands inside the + /// source's tree — which is what lets ordinary task writes become + /// durable facts — while rights can only narrow and the source edge + /// remains the revocation ancestor. + /// + /// The delegated root edge participates in the ordinary lifecycle: it + /// pins the source's root object like any grant, and + /// [`detach_session`](Self::detach_session) retires it through the same + /// stranded-root accounting, where reachability keeps the shared tree + /// alive. + pub fn attach_delegated_session( + &self, + session: SessionId, + source: GrantId, + rights: Rights, + ) -> FsResult<()> { + self.auth.attach_session(session)?; + match self.auth.install_delegated_root(session, source, rights) { + Ok(_) => { + let root = self.auth.root_object(session)?; + self.ns.object_mut(root)?.grants_inc(); + Ok(()) + } + Err(error) => { + self.auth.abort_attach(session); + Err(error) + } + } + } + + /// Detach a session: drop its handles, retire the authority edges it + /// holds, and reclaim whatever that left unreachable. + pub async fn detach_session(&self, session: SessionId) -> FsResult<()> { + let stranded = self.detach_session_authority(session)?; + self.reclaim_stranded(&stranded).await; + Ok(()) + } + + /// The synchronous half of [`Self::detach_session`]: revoke the session's + /// authority and drop its grants, returning the private roots left with + /// no owner. Nothing here awaits, so a service may run it from a + /// synchronous hook — the peer-death hook in particular — and reclaim + /// the roots later with [`Self::reclaim_stranded`] on its own async path. + /// + /// Why the split exists: `peer_closed` used to `block_on` the whole + /// detach inside the serve loop. Reclamation is bounded by executor + /// turns of *contention*, and a nested executor run cannot let the outer + /// loop's in-flight requests release the claims it is waiting on, so + /// under load a dead client's subtree was abandoned (counted, not freed) + /// exactly when the filesystem was busiest. Detached authority is + /// immediate either way: after this returns the session cannot open, + /// and its roots are unreachable by every other session. + pub(crate) fn detach_session_authority(&self, session: SessionId) -> FsResult { + let stranded = self.auth.detach_session(session)?; + for root in stranded.iter() { + if let Ok(mut object) = self.ns.object_mut(*root) { + object.grants_dec(); + } + } + Ok(stranded) + } + + /// Reclaim the roots [`Self::detach_session_authority`] handed back — on + /// the caller's executor, where a `Busy` claim can actually be released + /// by the request holding it. + pub(crate) async fn reclaim_stranded(&self, roots: &[ObjectId]) { + for root in roots { + self.reclaim(*root).await; + } + } + + /// Open the session's private root. + pub fn open_root(&self, session: SessionId, requested: Rights) -> FsResult { + let grant = self.auth.root_grant(session)?; + self.open_authority_edge(session, grant, requested) + } + + /// Open a granted file or subtree root. + /// + /// Only the grant's holder may do this — a [`GrantId`] handed to a third + /// session is inert. + pub fn open_grant( + &self, + session: SessionId, + grant: GrantId, + requested: Rights, + ) -> FsResult { + self.open_authority_edge(session, grant, requested) + } + + /// Open an attachment held by `view` into one transient command session. + /// The command receives only a handle; the stable grant remains owned by + /// the task view and therefore survives process teardown. + pub(crate) fn open_task_grant( + &self, + session: SessionId, + view: SessionId, + grant: GrantId, + requested: Rights, + ) -> FsResult { + let edge = self.auth.validated_edge(grant)?; + if edge.holder() != view || edge.kind() == AuthorityKind::PrivateRoot { + return Err(FsError::InvalidHandle); + } + let root = edge.root(); + let rights = edge.rights().attenuate(requested)?; + let quota = edge.quota(); + drop(edge); + let projected = self + .auth + .insert_grant(view, session, grant, root, rights, quota, None)?; + self.ns.object_mut(root)?.grants_inc(); + match self.open_authority_edge(session, projected, rights) { + Ok(opened) => Ok(opened), + Err(error) => { + if let Ok(roots) = self.auth.retire_edge(session, projected) { + for object in roots { + if let Ok(mut object) = self.ns.object_mut(object) { + object.grants_dec(); + } + } + } + Err(error) + } + } + } + + fn open_authority_edge( + &self, + session: SessionId, + grant: GrantId, + requested: Rights, + ) -> FsResult { + let edge = self.auth.validated_edge(grant)?; + if edge.holder() != session { + // Not this session's route; from here it does not exist. + return Err(FsError::InvalidHandle); + } + let root = edge.root(); + let rights = edge.rights().attenuate(requested)?; + let (stable_object, kind, subtype, len, revision) = { + let object = self.ns.object(root)?; + ( + object.stable(), + object.kind(), + object.subtype(), + object.len(), + object.revision(), + ) + }; + let handle = self + .auth + .insert_handle(session, root, grant, EdgeProof::root(), rights)?; + Ok(OpenedObject { + handle, + stable_object, + kind, + subtype, + len, + revision, + }) + } + + /// The authority edges this session holds — its private root plus every + /// grant made to it. This is how a client discovers its task-local + /// attachments. + pub fn grants( + &self, + session: SessionId, + ) -> FsResult> { + Ok(self + .auth + .grants_of(session)? + .iter() + .map(|(grant, kind, rights, quota, display_name)| GrantInfo { + grant: *grant, + kind: *kind, + rights: *rights, + quota: *quota, + display_name: *display_name, + }) + .collect()) + } + + // ── traversal ─────────────────────────────────────────────────────────── + + /// Resolve one name component beneath an open directory handle. + /// + /// Single-component by design: paths are client-side traversal input, and + /// each step extends the new handle's proof by exactly one edge. + pub fn open_at( + &self, + session: SessionId, + parent: Handle, + name: &Name, + requested: Rights, + ) -> FsResult { + let route = self.route(session, parent)?; + // Resolving a name is reading the directory. + route.rights.require(Rights::READ)?; + if !self.ns.kind(route.object)?.is_dir() { + return Err(FsError::NotDirectory); + } + let rights = route.rights.attenuate(requested)?; + let (edge, child) = self.ns.lookup(route.object, name)?; + + let (grant, proof) = { + let record = self.auth.handle(session, parent)?; + (record.grant(), record.proof().extend(edge)?) + }; + let (stable_object, kind, subtype, len, revision) = { + let object = self.ns.object(child)?; + ( + object.stable(), + object.kind(), + object.subtype(), + object.len(), + object.revision(), + ) + }; + let handle = self + .auth + .insert_handle(session, child, grant, proof, rights)?; + Ok(OpenedObject { + handle, + stable_object, + kind, + subtype, + len, + revision, + }) + } + + /// Close a handle. Does not affect the object or any other route to it. + pub fn close(&self, session: SessionId, handle: Handle) -> FsResult<()> { + self.auth.remove_handle(session, handle).map(|_| ()) + } + + /// An object's metadata. + pub fn stat(&self, session: SessionId, handle: Handle) -> FsResult { + let route = self.route(session, handle)?; + route.rights.require(Rights::READ)?; + let object = self.ns.object(route.object)?; + Ok(StatInfo { + object: route.object, + stable_object: object.stable(), + kind: object.kind(), + subtype: object.subtype(), + len: object.len(), + revision: object.revision(), + links: object.incoming(), + grants: object.grants(), + }) + } + + /// One page of a directory's entries. + pub fn list(&self, session: SessionId, handle: Handle, cursor: u32) -> FsResult { + let route = self.route(session, handle)?; + route.rights.require(Rights::READ)?; + let (entries, next) = self.ns.list(route.object, cursor)?; + Ok(ListPage { entries, next }) + } + + /// One page of an open object's visible parents: every directory + /// reachable from the root of the authority edge the handle entered + /// through that holds an edge to it, each with the shortest route from + /// that root. The scope is that one edge's subgraph (ruling 10: never + /// the union of the session's grants), so a parent outside it is never + /// named. Data only — a caller that wants one opens its route through + /// the same live anchor. A page holds at most `limit` parents. + pub fn parents( + &self, + session: SessionId, + handle: Handle, + cursor: u32, + limit: usize, + ) -> FsResult { + let route = self.route(session, handle)?; + route.rights.require(Rights::READ)?; + let root = self.auth.validated_edge(route.grant)?.root(); + let (found, next) = self.ns.parents(root, route.object, cursor, limit)?; + let mut parents = heapless::Vec::new(); + for entry in found { + let (stable_object, revision) = { + let object = self.ns.object(entry.object)?; + (object.stable(), object.revision()) + }; + let pushed = parents.push(ParentInfo { + object: entry.object, + stable_object, + revision, + path: entry.path, + }); + debug_assert!(pushed.is_ok(), "the namespace page is the same bound"); + } + Ok(ParentsPage { parents, next }) + } + + /// The object with identity `stable`, as seen from the root of the + /// authority edge the handle entered through: its shortest route from + /// that root, its own name last. An object that root does not reach is + /// `NotFound`, exactly as one that does not exist — the identity names + /// nothing outside the edge's subgraph (ruling 10). Data only: the + /// caller opens the route through the same live anchor. + pub fn locate(&self, session: SessionId, handle: Handle, stable: StableObjectId) -> FsResult { + let route = self.route(session, handle)?; + route.rights.require(Rights::READ)?; + let root = self.auth.validated_edge(route.grant)?.root(); + let object = self.ns.find_stable(stable).ok_or(FsError::NotFound)?; + let path = self.ns.route_up(root, object)?.ok_or(FsError::NotFound)?; + let found = self.ns.object(object)?; + Ok(LocateInfo { + object, + stable_object: found.stable(), + kind: found.kind(), + len: found.len(), + revision: found.revision(), + path, + }) + } + + /// Search begins at the caller's revalidated route. Candidate generation + /// is authority-filtered before predicates run; object ids in results are + /// observation data and never handles. + pub fn search( + &self, + session: SessionId, + handle: Handle, + query: SearchQuery, + cursor: Option, + ) -> FsResult { + let route = self.route(session, handle)?; + route.rights.require(Rights::READ)?; + if !self.ns.kind(route.object)?.is_dir() || query.min_len > query.max_len { + return Err(if query.min_len > query.max_len { + FsError::Range + } else { + FsError::NotDirectory + }); + } + match query.freshness { + FreshnessPolicy::StructuralOnly + if query.content.is_some() || query.max_index_lag != 0 => + { + return Err(FsError::Range); + } + FreshnessPolicy::CurrentOnly if query.max_index_lag != 0 => { + return Err(FsError::Range); + } + _ => {} + } + + let start = cursor.map_or(0, |cursor| cursor.position); + let mut walk = SearchWalk::new(start, query.work_limit()); + walk.hash_u64(handle.to_bits()); + walk.hash_u64(route.grant.to_bits()); + walk.hash_u64(route.object.to_bits()); + walk.hash_bytes(&[route.rights.bits()]); + walk.hash_u64(query.min_len); + walk.hash_u64(query.max_len); + walk.hash_bytes(&[query.freshness as u8]); + walk.hash_u64(u64::from(query.max_index_lag)); + walk.hash_u64(u64::from(query.max_work)); + walk.hash_bytes(&[match query.kind { + None => 0, + Some(ObjectKind::File) => 1, + Some(ObjectKind::Directory) => 2, + }]); + if let Some(name) = query.name_contains { + walk.hash_bytes(&[name.len() as u8]); + walk.hash_bytes(name.as_bytes()); + } else { + walk.hash_bytes(&[0]); + } + if let Some(content) = query.content { + walk.hash_bytes(&[content.plane as u8, content.term.len() as u8]); + walk.hash_bytes(content.term.as_bytes()); + } else { + walk.hash_bytes(&[0]); + } + + let mut path = heapless::Vec::new(); + self.search_dir(route.object, &query, &mut path, &mut walk)?; + if walk.hash == 0 { + walk.hash = SearchWalk::FNV_OFFSET; + } + if walk.snapshot == 0 { + walk.snapshot = SearchWalk::FNV_OFFSET; + } + let index_generation = if query.freshness == FreshnessPolicy::StructuralOnly { + 0 + } else { + walk.index_generation + }; + walk.hash_u64(index_generation); + if usize::try_from(start).map_or(true, |start| start > walk.scanned as usize) { + return Err(FsError::Range); + } + if cursor.is_some_and(|cursor| cursor.observation != walk.hash) { + return Err(FsError::Conflict); + } + // Ranking is scoped to this authorized result page. Keep traversal as + // the deterministic tie-breaker while presenting higher-confidence + // content matches first; hidden candidates never enter this vector. + for index in 1..walk.hits.len() { + let mut at = index; + while at > 0 && walk.hits[at].score > walk.hits[at - 1].score { + walk.hits.swap(at, at - 1); + at -= 1; + } + } + Ok(SearchPage { + hits: walk.hits, + observation: walk.hash, + snapshot: walk.snapshot, + index_generation, + freshness: query.freshness, + next: walk.next.map(|position| SearchCursor { + observation: walk.hash, + position, + }), + scanned: walk.scanned, + work: walk.work, + }) + } + + fn search_dir( + &self, + directory: ObjectId, + query: &SearchQuery, + path: &mut heapless::Vec, + walk: &mut SearchWalk, + ) -> FsResult<()> { + let mut cursor = 0; + loop { + let (entries, next) = self.ns.list(directory, cursor)?; + for entry in entries { + if walk.scanned as usize >= MAX_SEARCH_SCAN { + return Err(FsError::BudgetExceeded); + } + walk.charge(1)?; + let position = walk.scanned; + walk.scanned += 1; + let object = self.ns.object(entry.object)?; + let stable = object.stable(); + let len = object.len(); + let revision = object.revision(); + + walk.hash_structural_bytes(&[path.len() as u8]); + for component in path.iter() { + walk.hash_structural_bytes(&[component.len() as u8]); + walk.hash_structural_bytes(component.as_bytes()); + } + walk.hash_structural_bytes(&[entry.name.len() as u8]); + walk.hash_structural_bytes(entry.name.as_bytes()); + walk.hash_structural_u64(entry.object.to_bits()); + walk.hash_structural_u64(stable.get()); + walk.hash_structural_u64(len); + walk.hash_structural_u64(revision); + walk.hash_structural_bytes(&[if entry.kind.is_dir() { 2 } else { 1 }]); + + let structural_matches = query.matches_structural(&entry.name, entry.kind, len); + let indexed = if structural_matches && entry.kind.is_file() { + if let Some(content) = query.content { + let indexed = self.extraction.evaluate(entry.object, revision, content)?; + walk.charge(indexed.work)?; + indexed + } else { + crate::search::IndexMatch { + matched: true, + score: SearchScore::NONE, + snippet: heapless::String::new(), + work: 0, + generation: 0, + } + } + } else if structural_matches && query.content.is_none() { + crate::search::IndexMatch { + matched: true, + score: SearchScore::NONE, + snippet: heapless::String::new(), + work: 0, + generation: 0, + } + } else { + crate::search::IndexMatch::miss(0) + }; + if query.content.is_some() && structural_matches { + walk.hash_index_generation(indexed.generation); + } + + path.push(entry.name).map_err(|_| FsError::Range)?; + if structural_matches && indexed.matched && position >= walk.start { + if walk.hits.is_full() { + walk.next.get_or_insert(position); + } else { + walk.hits + .push(SearchHit { + path: path.clone(), + object: entry.object, + stable_object: stable, + kind: entry.kind, + len, + revision, + score: indexed.score, + snippet: indexed.snippet, + }) + .map_err(|_| FsError::Range)?; + } + } + if entry.kind.is_dir() { + self.search_dir(entry.object, query, path, walk)?; + } + let _ = path.pop(); + } + let Some(resume) = next else { + break; + }; + cursor = resume; + } + Ok(()) + } + + // ── namespace mutation ────────────────────────────────────────────────── + + /// Create one object and one containment edge naming it, and hand the + /// creator a handle to it. + /// + /// The creator immediately receives the rights its parent authority + /// allowed, so a task can use what it just made without a + /// second lookup. All-or-nothing: if the edge or the handle cannot be + /// made, the object is reclaimed and nothing is left behind. + pub async fn create( + &self, + session: SessionId, + parent: Handle, + name: &Name, + kind: ObjectKind, + ) -> FsResult { + self.create_typed(session, parent, name, kind, ObjectSubtype::PLAIN) + .await + } + + /// [`Self::create`], stamping `subtype` — data the object carries for its + /// life, never authority over it. An attesting stamp is not free: the + /// parent route must also carry [`ObjectSubtype::stamp_rights`], so an + /// `EXECUTABLE_IMAGE` stamp means the install authority made it. + pub async fn create_typed( + &self, + session: SessionId, + parent: Handle, + name: &Name, + kind: ObjectKind, + subtype: ObjectSubtype, + ) -> FsResult { + let route = self.route(session, parent)?; + route.rights.require(Rights::MUTATE_NAMESPACE)?; + require_stamp_authority(route.rights, subtype)?; + // A subtype is data on a file. A directory carries none, and a volume + // whose durable format predates subtypes cannot record one — refusing + // here keeps "what the engine holds" equal to "what the disk can say". + if !subtype.is_plain() && (kind.is_dir() || !self.durable_subtypes) { + return Err(FsError::Unsupported); + } + if !self.ns.kind(route.object)?.is_dir() { + return Err(FsError::NotDirectory); + } + let view_claim = self.claim_view_if_root(session, parent)?; + self.require_view_name_free(session, parent, name)?; + let (grant, parent_depth) = { + let record = self.auth.handle(session, parent)?; + (record.grant(), record.proof().depth()) + }; + // Check the proof has room before creating anything: a created object + // we then could not hand back would have to be unwound. + if parent_depth >= MAX_PATH_DEPTH { + return Err(FsError::Range); + } + let _quota = self.require_growth(route, 1, 1, 0)?; + + let child = match kind { + ObjectKind::File => self.ns.create_file_object_with(subtype)?, + ObjectKind::Directory => self.ns.create_dir_object()?, + }; + let edge = match self.ns.link(route.object, name, child) { + Ok(edge) => edge, + Err(err) => { + // Nothing names the fresh object, so reclamation drops it. + drop(view_claim); + self.reclaim(child).await; + return Err(err); + } + }; + let proof = { + let record = self.auth.handle(session, parent)?; + record.proof().extend(edge)? + }; + let (stable_object, revision) = { + let object = self.ns.object(child)?; + (object.stable(), object.revision()) + }; + match self + .auth + .insert_handle(session, child, grant, proof, route.rights) + { + Ok(handle) => Ok(OpenedObject { + handle, + stable_object, + kind, + subtype, + len: 0, + revision, + }), + Err(err) => { + // Unwind the edge too, so a refused create is invisible. + let _ = self.ns.detach(route.object, name); + drop(view_claim); + self.reclaim(child).await; + Err(err) + } + } + } + + /// Add another containment route to an existing file or directory. + /// + /// Directory links run a bounded graph reachability proof before the edge + /// is published, so diamonds are accepted and cycles are refused. + /// + /// Both handles must have been entered through the *same* authority edge + /// — see [`Self::require_same_authority`], which explains the escalation + /// that rule prevents. + pub fn link( + &self, + session: SessionId, + parent: Handle, + name: &Name, + child: Handle, + ) -> FsResult<()> { + let parent_route = self.route(session, parent)?; + parent_route.rights.require(Rights::MUTATE_NAMESPACE)?; + // You must be able to see the object to give it a name. + let child_route = self.route(session, child)?; + child_route.rights.require(Rights::READ)?; + self.require_same_authority(session, parent, child)?; + let _view_claim = self.claim_view_if_root(session, parent)?; + self.require_view_name_free(session, parent, name)?; + let _quota = self.require_growth(parent_route, 0, 1, 0)?; + self.ns + .link(parent_route.object, name, child_route.object) + .map(|_| ()) + } + + /// Change one edge's label. The edge — and therefore every handle + /// proving through it — survives. + pub fn rename( + &self, + session: SessionId, + parent: Handle, + old: &Name, + new: &Name, + ) -> FsResult<()> { + let route = self.route(session, parent)?; + route.rights.require(Rights::MUTATE_NAMESPACE)?; + let _view_claim = self.claim_view_if_root(session, parent)?; + self.require_view_name_free(session, parent, new)?; + self.ns.rename(route.object, old, new).map(|_| ()) + } + + /// Atomically remove one edge and create one edge. + /// + /// Requires namespace-mutation rights at *both* parents. A handle opened + /// through the source edge becomes invalid; one opened through a separate + /// link to the same object stays valid. + /// + /// Both parents must be reached through the same authority edge — a move + /// across two attachments is refused explicitly so a weaker grant cannot + /// be laundered into the destination root's stronger authority. See + /// [`Self::require_same_authority`]. + pub fn relocate( + &self, + session: SessionId, + src_parent: Handle, + name: &Name, + dst_parent: Handle, + new_name: &Name, + ) -> FsResult<()> { + let src = self.route(session, src_parent)?; + src.rights.require(Rights::MUTATE_NAMESPACE)?; + let dst = self.route(session, dst_parent)?; + dst.rights.require(Rights::MUTATE_NAMESPACE)?; + self.require_same_authority(session, src_parent, dst_parent)?; + let _view_claim = self.claim_view_if_root(session, dst_parent)?; + self.require_view_name_free(session, dst_parent, new_name)?; + self.ns + .relocate(src.object, name, dst.object, new_name) + .map(|_| ()) + } + + /// Remove exactly one native graph edge, including an edge to a populated + /// directory. Other containment routes and grants are untouched; graph + /// reclamation runs only if this was the final route. + pub async fn detach_edge( + &self, + session: SessionId, + parent: Handle, + name: &Name, + ) -> FsResult<()> { + let route = self.route(session, parent)?; + route.rights.require(Rights::MUTATE_NAMESPACE)?; + let (_, child) = self.ns.detach(route.object, name)?; + self.reclaim(child).await; + Ok(()) + } + + /// Compatibility unlink/rmdir. Files and empty directories are removed; + /// a populated directory is refused. Use [`Self::detach_edge`] for the + /// native edge-removal operation. + pub async fn unlink(&self, session: SessionId, parent: Handle, name: &Name) -> FsResult<()> { + let route = self.route(session, parent)?; + route.rights.require(Rights::MUTATE_NAMESPACE)?; + let (_, child) = self.ns.unlink_empty(route.object, name)?; + self.reclaim(child).await; + Ok(()) + } + + // ── sharing ───────────────────────────────────────────────────────────── + + /// Give another session an attenuated authority edge to this object. + /// + /// Requires [`Rights::SHARE`] on the route, and the delegated rights are + /// intersected with what the caller holds — sharing can only narrow. The + /// recipient's `display_name`, if given, must be free in *its* view, so a + /// sender cannot overwrite a name in someone else's namespace of + /// attachments. + pub fn grant( + &self, + session: SessionId, + handle: Handle, + destination: SessionId, + rights: Rights, + display_name: Option, + ) -> FsResult { + let quota = self.route(session, handle)?.quota; + self.grant_scoped(session, handle, destination, rights, quota, display_name) + } + + /// Give another session an attenuated route with an explicit budget. + /// + /// Quota attenuation follows rights attenuation: a child edge may ask for + /// less, never more. No capacity is reserved eagerly. Every later growth + /// derives exact usage from this grant's live graph before mutation. + pub fn grant_scoped( + &self, + session: SessionId, + handle: Handle, + destination: SessionId, + rights: Rights, + quota: AuthorityQuota, + display_name: Option, + ) -> FsResult { + let route = self.route(session, handle)?; + route.rights.require(Rights::SHARE)?; + let delegated = route.rights.attenuate(rights)?; + let delegated_quota = route.quota.attenuate(quota)?; + let _destination_view = self.auth.claim_view(destination)?; + if let Some(name) = display_name.as_ref() { + if self.view_name_taken(destination, name)? { + return Err(FsError::AlreadyExists); + } + } + let grant = self.auth.insert_grant( + session, + destination, + route.grant, + route.object, + delegated, + delegated_quota, + display_name, + )?; + // The grant is now a live route to the object, so it keeps it alive. + self.ns.object_mut(route.object)?.grants_inc(); + Ok(grant) + } + + /// Publish a route resolved by a transient command into another stable + /// task view. `source_view` must be an actual ancestor of the command's + /// route; the authority layer proves that relation before reparenting the + /// grant, so command teardown cannot revoke a successful share. + pub(crate) fn grant_between_task_views( + &self, + session: SessionId, + handle: Handle, + source_view: SessionId, + destination_view: SessionId, + rights: Rights, + display_name: Option, + ) -> FsResult { + let route = self.route(session, handle)?; + route.rights.require(Rights::SHARE)?; + let delegated = route.rights.attenuate(rights)?; + let _destination = self.auth.claim_view(destination_view)?; + if let Some(name) = display_name.as_ref() + && self.view_name_taken(destination_view, name)? + { + return Err(FsError::AlreadyExists); + } + let grant = self.auth.grant_route_from_ancestor( + session, + source_view, + destination_view, + route, + delegated, + display_name, + )?; + self.ns.object_mut(route.object)?.grants_inc(); + Ok(grant) + } + + /// Retire a grant. Either side of it may do so; nobody else may. + /// + /// Every handle that entered through the grant fails its next + /// revalidation with [`FsError::AccessLost`]. + pub async fn revoke(&self, session: SessionId, grant: GrantId) -> FsResult<()> { + let roots = self.auth.retire_edge(session, grant)?; + for root in roots.iter() { + if let Ok(mut object) = self.ns.object_mut(*root) { + object.grants_dec(); + } + } + for root in roots { + self.reclaim(root).await; + } + Ok(()) + } + + // ── bytes ─────────────────────────────────────────────────────────────── + + /// Read from a file into `dst`, returning the bytes transferred. + /// + /// Short at end-of-file rather than an error; a read starting at or past + /// the end transfers zero. + pub async fn read( + &self, + session: SessionId, + handle: Handle, + offset: u64, + dst: &mut [u8], + ) -> FsResult { + let route = self.route(session, handle)?; + route.rights.require(Rights::READ)?; + let (key, len) = { + let object = self.ns.object(route.object)?; + if !object.kind().is_file() { + return Err(FsError::IsDirectory); + } + (object.storage(), object.len()) + }; + // Overflow is a refusal, never a wrap into a valid-looking range. + offset.checked_add(dst.len() as u64).ok_or(FsError::Range)?; + if offset >= len { + return Ok(0); + } + let want = (len - offset).min(dst.len() as u64); + let want = usize::try_from(want).map_err(|_| FsError::Range)?; + let want = want.min(self.provider.limits().max_transfer); + let Some(key) = key else { + // No placement: the file has never held bytes, so its length is + // zero and the early return above already covered it. + return Ok(0); + }; + Ok(self.provider.read(key, offset, &mut dst[..want]).await?) + } + + // ── immutable extraction versions ──────────────────────────────────────── + + /// Freeze the exact file revision visible through `handle` and lend only + /// that byte range to an extraction connection. + /// + /// The snapshot is copied before the lease becomes visible. Subsequent + /// writes continue against the file's ordinary placement, so a parser + /// can never observe a mixture of two revisions. + pub(crate) async fn lend_version( + &self, + owner: ExtractionOwner, + session: SessionId, + handle: Handle, + ) -> Result<(VersionLease, InputIdentity), ExtractionError> { + self.lend_version_with_rights(owner, session, handle, Rights::READ) + .await + } + + /// Freeze one executable source revision. + /// + /// Reading a file is intentionally insufficient: the route must also + /// carry the separately attenuable [`Rights::BIND_EXECUTABLE`] edge. The + /// returned version is still only immutable bytes; ImageService performs + /// independent ELF validation and is the only component that can mint X. + pub(crate) async fn lend_executable_version( + &self, + owner: ExtractionOwner, + session: SessionId, + handle: Handle, + ) -> Result<(VersionLease, InputIdentity), ExtractionError> { + self.lend_version_with_rights( + owner, + session, + handle, + Rights::READ.union(Rights::BIND_EXECUTABLE), + ) + .await + } + + async fn lend_version_with_rights( + &self, + owner: ExtractionOwner, + session: SessionId, + handle: Handle, + required: Rights, + ) -> Result<(VersionLease, InputIdentity), ExtractionError> { + let route = self.route(session, handle)?; + route.rights.require(required)?; + let (source, len, revision, stable) = { + let object = self.ns.object(route.object)?; + if !object.kind().is_file() { + return Err(FsError::IsDirectory.into()); + } + (object.storage(), object.len(), object.revision(), object.stable()) + }; + if !self.extraction.can_lend() { + return Err(ExtractionError::NoSnapshotSlots); + } + + // The lease borrows the object's live placement: no copy. It stays + // frozen because every mutation path runs through `ensure_capacity`, + // which moves the object to a fresh placement first while a lease + // borrows this one, and reclaim hands a borrowed placement to its + // borrowers instead of releasing it. + let snapshot_key = if len == 0 { + None + } else { + Some(source.ok_or(ExtractionError::Provider(ProviderError::Media))?) + }; + + // The digest of exactly the bytes the lease will read: remembered + // when this revision was hashed before (a revision's bytes never + // change), otherwise computed now — a short read is the medium's + // refusal, not a copy's — and remembered. + let remembered = self.digests.borrow().get(stable, revision); + let digest = match remembered { + Some(digest) => digest, + None => { + let mut hasher = blake3::Hasher::new(); + if let Some(key) = snapshot_key { + let mut buffer = [0u8; MAX_TRANSFER]; + let chunk = self.provider.limits().max_transfer.min(buffer.len()); + let mut offset = 0u64; + while offset < len { + let want = usize::try_from((len - offset).min(chunk as u64)) + .map_err(|_| ExtractionError::Range)?; + let read = self.provider.read(key, offset, &mut buffer[..want]).await?; + if read != want { + return Err(ProviderError::Media.into()); + } + hasher.update(&buffer[..read]); + offset += read as u64; + } + } + let digest = *hasher.finalize().as_bytes(); + self.digests.borrow_mut().put(stable, revision, digest); + digest + } + }; + let identity = InputIdentity { + source: route.object.to_bits(), + version: revision, + len, + digest, + }; + let snapshot = Snapshot { + owner, + session, + handle, + object: route.object, + identity, + storage: snapshot_key, + }; + match self.extraction.install_snapshot(snapshot) { + Ok(lease) => Ok((lease, identity)), + // The placement is the object's own; nothing to release. + Err(_) => Err(ExtractionError::NoSnapshotSlots), + } + } + + /// Read through a frozen lease. No namespace handle is accepted here. + pub(crate) async fn read_lent_version( + &self, + owner: ExtractionOwner, + lease: VersionLease, + offset: u64, + dst: &mut [u8], + ) -> Result { + let snapshot = self.extraction.snapshot(owner, lease)?; + let route = self.route(snapshot.session, snapshot.handle)?; + route.rights.require(Rights::READ)?; + if route.object != snapshot.object { + return Err(FsError::AccessLost.into()); + } + offset + .checked_add(dst.len() as u64) + .ok_or(ExtractionError::Range)?; + if dst.len() > self.provider.limits().max_transfer { + return Err(ExtractionError::Range); + } + if offset >= snapshot.identity.len { + return Ok(0); + } + let want = usize::try_from((snapshot.identity.len - offset).min(dst.len() as u64)) + .map_err(|_| ExtractionError::Range)?; + let Some(key) = snapshot.storage else { + return Ok(0); + }; + let read = self.provider.read(key, offset, &mut dst[..want]).await; + drop(snapshot); + Ok(read?) + } + + /// The device blocks of a lent version, for a read grant: the same + /// authority a read through the lease needs (its route still readable + /// and still naming the object), then the whole blocks inside the + /// version's length. The lease pins the placement — a write moves the + /// object to fresh blocks and reclaim hands a borrowed placement to its + /// lease — so the blocks stay the version's until the lease is + /// released. `Ok(None)` for an empty version or a provider with no + /// device namespace. + pub(crate) fn lent_blocks( + &self, + owner: ExtractionOwner, + lease: VersionLease, + ) -> Result, ExtractionError> { + let snapshot = self.extraction.snapshot(owner, lease)?; + let route = self.route(snapshot.session, snapshot.handle)?; + route.rights.require(Rights::READ)?; + if route.object != snapshot.object { + return Err(FsError::AccessLost.into()); + } + let len = snapshot.identity.len; + let blocks = snapshot.storage.and_then(|key| self.provider.device_blocks(key)); + drop(snapshot); + Ok(blocks.map(|mut blocks| { + // The snapshot's length is the version's: never grant past it. + blocks.blocks = blocks.blocks.min(len / u64::from(blocks.block_size)); + blocks + })) + } + + /// Publish a validated parser-neutral result only if the lent revision is + /// still the object's selected version. + pub(crate) fn commit_derived_result( + &self, + owner: ExtractionOwner, + lease: VersionLease, + result: NormalizedResult, + output_window_bytes: u64, + ) -> Result { + let snapshot = self.extraction.snapshot(owner, lease)?; + let route = self.route(snapshot.session, snapshot.handle)?; + route.rights.require(Rights::READ)?; + if route.object != snapshot.object { + return Err(FsError::AccessLost.into()); + } + let object = self + .ns + .object(snapshot.object) + .map_err(ExtractionError::from)?; + let current_revision = Some(object.revision()); + let committed = self.extraction.commit_result( + owner, + lease, + current_revision, + result, + output_window_bytes, + ); + drop(object); + drop(snapshot); + committed + } + + /// Validate and publish one normalized batch together with its durable + /// provider-backed wire representation. The new placement is complete and + /// synced before it becomes reachable; the replaced placement is released + /// only after the in-memory generation points at the successor. + pub(crate) async fn commit_encoded_derived_result( + &self, + owner: ExtractionOwner, + lease: VersionLease, + result: NormalizedResult, + encoded: &[u8], + output_window_bytes: u64, + ) -> Result { + let snapshot = self.extraction.snapshot(owner, lease)?; + let route = self.route(snapshot.session, snapshot.handle)?; + route.rights.require(Rights::READ)?; + if route.object != snapshot.object { + return Err(FsError::AccessLost.into()); + } + let object = snapshot.object; + let identity = snapshot.identity; + drop(snapshot); + kore_svc_extract::registry::Registry::validate(&result, identity, output_window_bytes) + .map_err(ExtractionError::InvalidResult)?; + // The store keeps one #476 batch per version, and durable recovery + // refuses a larger extent. A batch past that bound is one this index + // cannot hold — refused like one it cannot decode or validate, and + // before a placement is spent writing bytes nothing will keep. + if encoded.len() > kore_svc_extract::WORKER_REPORT_BYTES { + return Err(ExtractionError::Oversize); + } + let encoded_len = u32::try_from(encoded.len()).map_err(|_| ExtractionError::Range)?; + if encoded.is_empty() + || encoded.len() > self.provider.limits().max_transfer + || encoded.len() as u64 > self.provider.limits().max_object_bytes + { + return Err(ExtractionError::Range); + } + + let storage = if P::PERSISTS_ACROSS_RESTART { + let key = self + .provider + .allocate(ObjectKind::File, encoded.len() as u64) + .await?; + if let Err(error) = self.provider.write(key, 0, encoded).await { + let _ = self.provider.release(key).await; + return Err(error.into()); + } + if let Err(error) = self.provider.sync(key).await { + let _ = self.provider.release(key).await; + return Err(error.into()); + } + Some(key) + } else { + None + }; + + let info = match self.commit_derived_result(owner, lease, result, output_window_bytes) { + Ok(info) => info, + Err(error) => { + if let Some(key) = storage { + let _ = self.provider.release(key).await; + } + return Err(error); + } + }; + if let Some(key) = storage { + let old = match self.extraction.attach_storage( + object, + info.parser, + key, + encoded_len, + *blake3::hash(encoded).as_bytes(), + ) { + Ok(old) => old, + Err(error) => { + let _ = self.provider.release(key).await; + return Err(error.into()); + } + }; + if let Some(old) = old { + if self.provider.release(old).await.is_err() { + self.ns.note_leaked_placement(); + } + } + } + Ok(info) + } + + pub(crate) fn durable_derived_results( + &self, + ) -> FsResult, N>> { + let records = self.extraction.durable_results::()?; + let mut current = heapless::Vec::new(); + for record in records { + // A content mutation advances the authoritative object revision + // before its application-owned extractor can publish the + // successor batch. That interval is IndexIncomplete, not durable + // corruption: omit the obsolete derived extent from the new root. + // CommitDerived will replace the same parser row and reclaim its + // old placement after the successor is fully synced. + let Ok(object) = self.ns.object(record.object) else { + continue; + }; + let selected = object.kind().is_file() + && object.revision() == record.input.version + && object.len() == record.input.len; + drop(object); + if selected { + current.push(record).map_err(|_| FsError::OutOfStorage)?; + } + } + Ok(current) + } + + pub(crate) fn restore_derived_result( + &self, + object: ObjectId, + result: NormalizedResult, + storage: P::Key, + len: u32, + digest: [u8; 32], + ) -> FsResult<()> { + self.extraction + .restore_result(object, result, storage, len, digest) + } + + /// Release an immutable lease. A failed provider release retains the + /// lease so teardown can be retried without losing ownership. + pub(crate) async fn release_lent_version( + &self, + owner: ExtractionOwner, + lease: VersionLease, + ) -> Result<(), ExtractionError> { + // The write claim is the storage-lifetime owner: reads retain a + // shared snapshot guard across provider I/O, so release receives + // `Busy` instead of freeing a placement beneath an in-flight read. + let snapshot = self.extraction.claim_snapshot(owner, lease)?; + if let Some(key) = snapshot.storage { + // Release only a placement nobody else holds: not the object + // (still on it, never mutated in place) and no other lease. + let owned = self + .ns + .object(snapshot.object) + .map(|object| object.storage() == Some(key)) + .unwrap_or(false); + if !owned && self.extraction.storage_borrowers(key, Some(lease))? == 0 { + self.provider.release(key).await?; + } + } + let _ = self.extraction.remove_claimed_snapshot(snapshot)?; + Ok(()) + } + + /// Selected derived data for an authorized live file route. + pub fn derived_result( + &self, + session: SessionId, + handle: Handle, + parser: ParserId, + ) -> FsResult> { + let route = self.route(session, handle)?; + route.rights.require(Rights::READ)?; + let object = self.ns.object(route.object)?; + let result = self + .extraction + .result(route.object, object.revision(), parser); + drop(object); + result + } + + pub(crate) async fn begin_replace( + &self, + session: SessionId, + handle: Handle, + length: u64, + expected: Revision, + ) -> FsResult> { + let route = self.route(session, handle)?; + route.rights.require(Rights::WRITE)?; + let object = self.ns.object_mut(route.object)?; + if !object.kind().is_file() { + return Err(FsError::IsDirectory); + } + // Refused before a placement is spent; `commit_replace` asks again + // at publication. The staged bytes in between are private. + require_stamp_authority(route.rights, object.subtype())?; + if !object.revision_admits(expected) { + return Err(FsError::Conflict); + } + if length > self.provider.limits().max_object_bytes { + return Err(FsError::OutOfStorage); + } + let _quota = self.require_growth_with_claimed( + route, + &object, + 0, + 0, + length.saturating_sub(object.len()), + )?; + let key = self.provider.allocate(ObjectKind::File, length).await?; + Ok(Replacement { + handle, + object: route.object, + revision: object.revision(), + key, + length, + written: 0, + }) + } + + pub(crate) async fn write_replacement( + &self, + session: SessionId, + stage: &mut Replacement, + offset: u64, + bytes: &[u8], + ) -> FsResult<()> { + let route = self.route(session, stage.handle)?; + route.rights.require(Rights::WRITE)?; + if route.object != stage.object { + return Err(FsError::AccessLost); + } + let end = offset + .checked_add(bytes.len() as u64) + .ok_or(FsError::Range)?; + if offset != stage.written || end > stage.length { + return Err(FsError::Range); + } + self.provider.write(stage.key, offset, bytes).await?; + stage.written = end; + Ok(()) + } + + /// Failure retains the private placement; no fallible operation follows + /// retirement of the old placement. The object claim excludes readers + /// through that synchronous publication, and immutable leases keep their + /// own placement alive through the existing borrower accounting. + pub(crate) async fn commit_replace( + &self, + session: SessionId, + stage: &Replacement, + ) -> FsResult { + let route = self.route(session, stage.handle)?; + route.rights.require(Rights::WRITE)?; + if route.object != stage.object { + return Err(FsError::AccessLost); + } + if stage.written != stage.length { + return Err(FsError::Range); + } + let mut object = self.ns.object_mut(route.object)?; + require_stamp_authority(route.rights, object.subtype())?; + if object.revision() != stage.revision { + return Err(FsError::Conflict); + } + let _quota = self.require_growth_with_claimed( + route, + &object, + 0, + 0, + stage.length.saturating_sub(object.len()), + )?; + if let Some(old) = object.storage() { + if self.extraction.storage_borrowers(old, None)? == 0 { + self.provider.release(old).await?; + } + } + object.set_storage(stage.key); + Ok(self + .ns + .commit_file_len(route.object, &mut object, stage.length)) + } + + pub(crate) async fn abort_replace(&self, stage: &Replacement) -> FsResult<()> { + self.provider.release(stage.key).await?; + Ok(()) + } + + /// Write `src` at `offset`, returning the new revision. + /// + /// `expected` is either [`ANY_REVISION`](crate::object::ANY_REVISION) — + /// unconditional — or an exact revision, which refuses with + /// [`FsError::Conflict`] if stale. + pub async fn write_at( + &self, + session: SessionId, + handle: Handle, + offset: u64, + src: &[u8], + expected: Revision, + ) -> FsResult { + let route = self.route(session, handle)?; + route.rights.require(Rights::WRITE)?; + self.commit_write(route, Some(offset), src, expected) + .await + .map(|(_, revision)| revision) + } + + /// Append `src` to a file, returning the offset it landed at and the new + /// revision. + /// + /// Choosing the end and writing are one indivisible step: both happen + /// under the file's exact write claim, so a concurrent append receives + /// [`FsError::Busy`] rather than landing on the same offset. + pub async fn append( + &self, + session: SessionId, + handle: Handle, + src: &[u8], + expected: Revision, + ) -> FsResult<(u64, Revision)> { + let route = self.route(session, handle)?; + route.rights.require(Rights::WRITE)?; + self.commit_write(route, None, src, expected).await + } + + /// Set a file's length, returning the new revision. Growth reads as + /// zeros; shrinking discards the tail. + pub async fn resize( + &self, + session: SessionId, + handle: Handle, + len: u64, + expected: Revision, + ) -> FsResult { + let route = self.route(session, handle)?; + route.rights.require(Rights::WRITE)?; + if len > self.provider.limits().max_object_bytes { + return Err(FsError::OutOfStorage); + } + let mut object = self.ns.object_mut(route.object)?; + if !object.kind().is_file() { + return Err(FsError::IsDirectory); + } + require_stamp_authority(route.rights, object.subtype())?; + if !object.revision_admits(expected) { + return Err(FsError::Conflict); + } + let _quota = self.require_growth_with_claimed( + route, + &object, + 0, + 0, + len.saturating_sub(object.len()), + )?; + let key = self.ensure_capacity(&mut object, len).await?; + // Capacity is in place; commit the visible length and revision. + let _ = key; + Ok(self.ns.commit_file_len(route.object, &mut object, len)) + } + + /// Cross the provider's durability contract for this file. + /// + /// For the RAM provider this reports success without doing anything, + /// because "visible in this boot" is all RAM promises and a successful + /// write already achieved it. It is never reported as more than that. + pub async fn sync(&self, session: SessionId, handle: Handle) -> FsResult<()> { + let route = self.route(session, handle)?; + route.rights.require(Rights::WRITE)?; + let key = { + let object = self.ns.object(route.object)?; + if !object.kind().is_file() { + return Err(FsError::IsDirectory); + } + object.storage() + }; + match key { + Some(key) => Ok(self.provider.sync(key).await?), + // Nothing has ever been placed, so there is nothing to flush. + None => Ok(()), + } + } + + // ── internals ─────────────────────────────────────────────────────────── + + /// Revalidate a handle and report what its route permits. The first act + /// of every operation. + fn route(&self, session: SessionId, handle: Handle) -> FsResult { + self.auth.revalidate(session, handle, &self.ns) + } + + /// Refuse graph growth that would outrun this authority edge's budget. + /// + /// Usage starts at the edge root, not at the object a child handle names, + /// so walking deeper cannot reset quota. Diamonds count each object once + /// and each independently named containment edge once. + fn require_growth( + &self, + route: Route, + objects: u32, + edges: u32, + bytes: u64, + ) -> FsResult> { + let epoch = self.auth.quota_epoch(route.grant)?; + let root = self.auth.validated_edge(route.grant)?.root(); + let usage = self.ns.graph_usage(root)?; + self.auth.claim_growth( + route.grant, + epoch, + usage.objects, + usage.edges, + usage.bytes, + objects, + edges, + bytes, + ) + } + + /// Reserve growth while one exact object is already write-claimed. + /// + /// This is load-bearing for write/append/resize: calculating growth before + /// the claim lets a concurrent shrink make the later transaction consume + /// more quota than it reserved. The namespace substitutes `claimed` into + /// its graph walk, so the exact current length and the reservation remain + /// one transaction. + fn require_growth_with_claimed( + &self, + route: Route, + claimed: &Object, + objects: u32, + edges: u32, + bytes: u64, + ) -> FsResult> { + let epoch = self.auth.quota_epoch(route.grant)?; + let root = self.auth.validated_edge(route.grant)?.root(); + let usage = self + .ns + .graph_usage_with_claimed(root, route.object, claimed)?; + self.auth.claim_growth( + route.grant, + epoch, + usage.objects, + usage.edges, + usage.bytes, + objects, + edges, + bytes, + ) + } + + /// Refuse an operation that would span two authority edges. + /// + /// This closes a real privilege escalation. A containment edge carries no + /// rights of its own — the rights an object is used with come from the + /// authority edge the *caller entered through*. So if a session holding a + /// read-only grant to one of A's files could `link` that object into its + /// own private root, it would afterwards reach the same object through its + /// own root's full-rights authority edge: read-only access laundered into + /// read-write, and access that survives A revoking the grant. `relocate` + /// offers the same trick across an attachment boundary. + /// + /// Requiring one authority edge for every handle in a + /// multiple-handle namespace operation removes the laundering step: the + /// route in and the route out are the same edge, so the rights cannot + /// change. Cross-attachment moves are therefore refused explicitly rather + /// than pretending two authority lineages form one commit domain. + /// + /// Note this is *not* a restriction on sharing: [`Self::grant`] is how an + /// object crosses between sessions, and it attenuates instead of + /// laundering. + fn require_same_authority( + &self, + session: SessionId, + first: Handle, + second: Handle, + ) -> FsResult<()> { + let a = self.auth.handle(session, first)?.grant(); + let b = self.auth.handle(session, second)?.grant(); + if a != b { + return Err(FsError::AccessDenied); + } + Ok(()) + } + + /// Keep the task-visible root unambiguous. + /// + /// A session's view root is the union of its private root's containment + /// entries and the named authority edges it holds. The wire protocol + /// keeps those two edge kinds separate, but ordinary pathname resolution + /// must not have a precedence rule that silently hides one behind the + /// other. Mutations at the private root and incoming grants therefore + /// share one name set. + fn require_view_name_free( + &self, + session: SessionId, + parent: Handle, + name: &Name, + ) -> FsResult<()> { + let record = self.auth.handle(session, parent)?; + if record.grant() == self.auth.root_grant(session)? + && record.proof().depth() == 0 + && self.auth.display_name_taken(session, name)? + { + return Err(FsError::AlreadyExists); + } + Ok(()) + } + + /// Whether `name` is already occupied in a session's task-visible root. + fn view_name_taken(&self, session: SessionId, name: &Name) -> FsResult { + if self.auth.display_name_taken(session, name)? { + return Ok(true); + } + let root = self.auth.edge(self.auth.root_grant(session)?)?.root(); + match self.ns.lookup(root, name) { + Ok(_) => Ok(true), + Err(FsError::NotFound) => Ok(false), + Err(err) => Err(err), + } + } + + /// Claim the combined top-level name set only when `parent` is the + /// session's private-root route. Nested directories are already isolated + /// by their exact directory claims and do not contend on this cache line. + fn claim_view_if_root( + &self, + session: SessionId, + parent: Handle, + ) -> FsResult>> { + let record = self.auth.handle(session, parent)?; + if record.grant() == self.auth.root_grant(session)? && record.proof().depth() == 0 { + Ok(Some(self.auth.claim_view(session)?)) + } else { + Ok(None) + } + } + + /// The write commit path shared by `write_at` and `append`. + /// + /// Ordering is the atomicity argument: validate, reserve capacity, write + /// bytes, then commit length and revision together. A failure at any + /// step before the last leaves nothing visible. + async fn commit_write( + &self, + route: Route, + requested_offset: Option, + src: &[u8], + expected: Revision, + ) -> FsResult<(u64, Revision)> { + if src.len() > self.provider.limits().max_transfer { + return Err(FsError::Range); + } + let mut object = self.ns.object_mut(route.object)?; + if !object.kind().is_file() { + return Err(FsError::IsDirectory); + } + require_stamp_authority(route.rights, object.subtype())?; + if !object.revision_admits(expected) { + return Err(FsError::Conflict); + } + let current_len = object.len(); + let offset = requested_offset.unwrap_or(current_len); + let end = offset.checked_add(src.len() as u64).ok_or(FsError::Range)?; + if end > self.provider.limits().max_object_bytes { + return Err(FsError::OutOfStorage); + } + let new_len = current_len.max(end); + let _quota = self.require_growth_with_claimed( + route, + &object, + 0, + 0, + new_len.saturating_sub(current_len), + )?; + + // Reserve first: this is the step that can run out of room, and it + // happens while nothing is visible. + let key = self.ensure_capacity(&mut object, new_len).await?; + // Then the bytes, into space that is already reserved. + self.provider.write(key, offset, src).await?; + // Then, and only then, the visible length and revision. + let revision = self.ns.commit_file_len(route.object, &mut object, new_len); + Ok((offset, revision)) + } + + /// Make the object's placement hold at least `len` bytes, allocating one + /// if it has never had bytes. + async fn ensure_capacity( + &self, + object: &mut crate::object::Object, + len: u64, + ) -> FsResult { + let existing = object.storage(); + match existing { + Some(key) => { + let key = self.detach_borrowed(object, key).await?; + self.provider.resize(key, len).await?; + Ok(key) + } + None => { + let key = self.provider.allocate(ObjectKind::File, len).await?; + object.set_storage(key); + Ok(key) + } + } + } + + /// Before mutating `key` in place, move the object off it if a lease + /// borrows it: the object continues on a fresh placement holding the + /// same bytes (the provider's own copy), and the lease keeps the old + /// one frozen. The one moment a lend costs a copy — and only when a + /// write races the lease, which the loader's short lends never do. + async fn detach_borrowed( + &self, + object: &mut crate::object::Object, + key: P::Key, + ) -> FsResult { + if self.extraction.storage_borrowers(key, None)? == 0 { + return Ok(key); + } + let len = object.len(); + let fresh = self.provider.allocate(ObjectKind::File, len).await?; + if let Err(error) = self.provider.copy_object(key, fresh, len).await { + if self.provider.release(fresh).await.is_err() { + self.ns.note_leaked_placement(); + } + return Err(error.into()); + } + object.set_storage(fresh); + Ok(fresh) + } + + /// Release `start` if nothing reaches it any more, along with anything it + /// made unreachable, and return their placements to the provider. + /// + /// **Every failure here is counted, never raised and never fatal.** + /// + /// A provider that refuses a release is counted through + /// [`Self::leaked_placements`]; a structural refusal or an exhausted + /// contention budget is counted through [`Self::abandoned_reclaims`] and + /// abandons only the rest of the walk. Neither is reported to a client, + /// because the namespace effect that caused the reclamation has already + /// committed — failing the request would describe an operation that did in + /// fact happen. + /// + /// Neither may panic or spin, either. This runs inside the founded + /// filesystem service, which has no `role::SELF` to exit through and whose + /// `COOKIE_REFUSED` reaches no drained channel after readiness: a panic + /// here parks the whole filesystem permanently with no witness, and every + /// client then waits on it forever (`SvcClient::call` has no reply + /// deadline). A resident unreachable subtree is a bounded, reportable + /// loss; a wedged filesystem is neither. + async fn reclaim(&self, start: ObjectId) { + // Arena-backed because all three are sized by the object bound, and + // this runs inside the service's `serve` future on the founder's + // shared stack (see `PLACEMENT_BYTES` in the block provider). One + // reclamation is one subtree unlink, not a per-object event, so three + // allocations here are not on a path where that matters. + // + // An allocator that cannot satisfy them takes the **same** route as a + // full worklist below: a counted, reportable abandonment. Neither a + // panic nor a spin is available to this function — see the contract + // above — so the failure must be a loss, not a refusal. + let alloc = crate::service_alloc(); + let (Ok(mut placements), Ok(mut objects), Ok(mut worklist)) = ( + BoundedVec::::try_new_in(MAX_OBJECTS, alloc), + BoundedVec::::try_new_in(MAX_OBJECTS, alloc), + BoundedVec::::try_new_in(MAX_OBJECTS, alloc), + ) else { + self.ns.note_abandoned_reclaim(); + return; + }; + if worklist.push(start).is_err() { + self.ns.note_abandoned_reclaim(); + return; + } + // One budget for the whole reclamation, spanning both phases. + let mut turns: u32 = 0; + while let Some(object) = worklist.pop() { + match self.ns.reclaim_one(object, &mut placements, &mut objects) { + Ok(children) => { + for child in children { + // The worklist holds one entry per object that crossed + // to zero reachability, and an object crosses once, so + // MAX_OBJECTS is sufficient by construction. A full + // queue would mean that invariant broke: lose the + // child, keep draining what is already queued. + if worklist.push(child).is_err() { + self.ns.note_abandoned_reclaim(); + break; + } + } + } + // A claim held by another in-flight operation. Keep the exact + // generation in the bounded worklist and let that operation + // make progress — for a bounded number of turns. + Err(FsError::Busy) if turns < RECLAIM_CONTENTION_TURNS => { + turns += 1; + if worklist.insert(0, object).is_err() { + self.ns.note_abandoned_reclaim(); + break; + } + yield_once().await; + } + // Contention that outlasted the budget, or a structural + // refusal — a concurrently stale slot (`AccessLost`) or an + // exhausted arena (`OutOfObjects`), both reachable. The + // objects below this point stay resident and unreachable. + Err(_) => { + self.ns.note_abandoned_reclaim(); + break; + } + } + } + let Ok(mut derived_placements) = + BoundedVec::::try_new_in(MAX_DERIVED_RESULTS, alloc) + else { + self.ns.note_abandoned_reclaim(); + return; + }; + for object in objects { + loop { + match self.extraction.purge_object(object) { + Ok(keys) => { + for key in keys { + if derived_placements.push(key).is_err() { + self.ns.note_abandoned_reclaim(); + break; + } + } + break; + } + Err(FsError::Busy) if turns < RECLAIM_CONTENTION_TURNS => { + turns += 1; + yield_once().await; + } + Err(_) => { + self.ns.note_abandoned_reclaim(); + break; + } + } + } + } + for key in placements.iter() { + // A placement a lease still borrows passes to that lease: the + // last borrower releases it. + match self.extraction.storage_borrowers(*key, None) { + Ok(0) => { + if self.provider.release(*key).await.is_err() { + self.ns.note_leaked_placement(); + } + } + Ok(_) => {} + Err(_) => self.ns.note_leaked_placement(), + } + } + for key in derived_placements { + if self.provider.release(key).await.is_err() { + self.ns.note_leaked_placement(); + } + } + } +} + +impl Default for RamEngine { + fn default() -> Self { + Self::new(RamProvider::new()) + } +} + +#[cfg(test)] +mod tests { + //! Engine-level unit tests. The plan's full §10.1 matrix lives in the + //! integration suites under `tests/`; these cover the wiring each method + //! does — rights gates, kind gates, and the commit ordering — so a + //! regression in one method is localized here rather than only showing up + //! as a failed end-to-end scenario. + + use super::*; + use crate::{block_on, error::ProviderError, object::ANY_REVISION, provider::Fault}; + use kore_svc_extract::{ + registry::{C_JSON, RUST_JSON}, + schema::{FactValue, Reference, SourceRange}, + }; + use kore_svc_extract_worker_c::parse_json_c; + use kore_svc_extract_worker_rust::{parse_json_rust, parse_plain_text}; + + fn name(s: &str) -> Name { + Name::new(s).unwrap() + } + + /// An engine with one attached session and an open root handle. + fn engine() -> (RamEngine, SessionId, Handle) { + let mut engine = RamEngine::default(); + let session = SessionId::new(0); + block_on(engine.attach_session(session)).unwrap(); + let root = engine.open_root(session, Rights::ALL).unwrap(); + (engine, session, root.handle) + } + + #[test] + fn replacement_is_private_complete_and_revision_checked() { + let (engine, session, root) = engine(); + let file = block_on(engine.create(session, root, &name("save"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, file.handle, 0, b"old tail", ANY_REVISION)).unwrap(); + let before = engine.stat(session, file.handle).unwrap(); + let lease_owner = ExtractionOwner::new(1); + let (lease, _) = block_on(engine.lend_version(lease_owner, session, file.handle)).unwrap(); + let mut stage = + block_on(engine.begin_replace(session, file.handle, 3, ANY_REVISION)).unwrap(); + block_on(engine.write_replacement(session, &mut stage, 0, b"n")).unwrap(); + let mut bytes = [0; 8]; + assert_eq!( + block_on(engine.read(session, file.handle, 0, &mut bytes)).unwrap(), + 8 + ); + assert_eq!(&bytes, b"old tail"); + assert_eq!( + block_on(engine.commit_replace(session, &stage)), + Err(FsError::Range) + ); + assert_eq!( + block_on(engine.write_replacement(session, &mut stage, 2, b"w")), + Err(FsError::Range) + ); + block_on(engine.write_replacement(session, &mut stage, 1, b"ew")).unwrap(); + block_on(engine.commit_replace(session, &stage)).unwrap(); + assert_eq!( + block_on(engine.read(session, file.handle, 0, &mut bytes)).unwrap(), + 3 + ); + assert_eq!(&bytes[..3], b"new"); + let after = engine.stat(session, file.handle).unwrap(); + assert_eq!(before.object, after.object); + assert_eq!(before.stable_object, after.stable_object); + assert_eq!(after.revision, before.revision + 1); + block_on(engine.read_lent_version(lease_owner, lease, 0, &mut bytes)).unwrap(); + assert_eq!(&bytes, b"old tail"); + block_on(engine.release_lent_version(lease_owner, lease)).unwrap(); + + let stage = block_on(engine.begin_replace(session, file.handle, 0, ANY_REVISION)).unwrap(); + block_on(engine.write_at(session, file.handle, 0, b"NEW", ANY_REVISION)).unwrap(); + assert_eq!( + block_on(engine.commit_replace(session, &stage)), + Err(FsError::Conflict) + ); + block_on(engine.abort_replace(&stage)).unwrap(); + let empty = block_on(engine.begin_replace(session, file.handle, 0, ANY_REVISION)).unwrap(); + block_on(engine.commit_replace(session, &empty)).unwrap(); + assert_eq!(engine.stat(session, file.handle).unwrap().len, 0); + } + + /// What a window handed an identity does: the identity a stat reported, + /// located from the root handle, is the route to the same object — + /// written, and replaced, first. + #[test] + fn locate_routes_a_stated_identity_from_the_root_handle() { + let (engine, session, root) = engine(); + let at_root = block_on(engine.create(session, root, &name("editor-test"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, at_root.handle, 0, b"before", ANY_REVISION)).unwrap(); + let dir = block_on(engine.create(session, root, &name("short"), ObjectKind::Directory)).unwrap(); + let nested = block_on(engine.create(session, dir.handle, &name("f"), ObjectKind::File)).unwrap(); + let stage = block_on(engine.begin_replace(session, nested.handle, 3, ANY_REVISION)).unwrap(); + let mut stage = stage; + block_on(engine.write_replacement(session, &mut stage, 0, b"new")).unwrap(); + block_on(engine.commit_replace(session, &stage)).unwrap(); + + for (handle, route) in [(at_root.handle, &["editor-test"][..]), (nested.handle, &["short", "f"][..])] { + let stat = engine.stat(session, handle).unwrap(); + let found = engine.locate(session, root, stat.stable_object).unwrap(); + assert_eq!(found.object, stat.object); + assert_eq!(found.stable_object, stat.stable_object); + let spelled: heapless::Vec<&[u8], 4> = found.path.iter().map(Name::as_bytes).collect(); + let expected: heapless::Vec<&[u8], 4> = route.iter().map(|part| part.as_bytes()).collect(); + assert_eq!(spelled, expected); + } + } + + #[test] + fn replacement_rechecks_revocation_before_publication() { + let (mut engine, owner, root) = engine(); + let writer = SessionId::new(1); + block_on(engine.attach_session(writer)).unwrap(); + let file = block_on(engine.create(owner, root, &name("saved"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(owner, file.handle, 0, b"old", ANY_REVISION)).unwrap(); + let grant = engine + .grant(owner, file.handle, writer, Rights::ALL, None) + .unwrap(); + let writable = engine.open_grant(writer, grant, Rights::ALL).unwrap(); + let mut stage = + block_on(engine.begin_replace(writer, writable.handle, 3, ANY_REVISION)).unwrap(); + block_on(engine.write_replacement(writer, &mut stage, 0, b"new")).unwrap(); + block_on(engine.revoke(owner, grant)).unwrap(); + assert!(block_on(engine.commit_replace(writer, &stage)).is_err()); + block_on(engine.abort_replace(&stage)).unwrap(); + let mut bytes = [0; 3]; + block_on(engine.read(owner, file.handle, 0, &mut bytes)).unwrap(); + assert_eq!(&bytes, b"old"); + } + + #[test] + fn replacement_provider_failures_preserve_old_contents_and_retry_custody() { + let (engine, session, root) = engine(); + let file = block_on(engine.create(session, root, &name("save"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, file.handle, 0, b"old", ANY_REVISION)).unwrap(); + let mut stage = + block_on(engine.begin_replace(session, file.handle, 3, ANY_REVISION)).unwrap(); + engine.provider().inject(Fault::Write(ProviderError::Media)); + assert!(block_on(engine.write_replacement(session, &mut stage, 0, b"new")).is_err()); + assert_eq!(stage.written, 0); + block_on(engine.write_replacement(session, &mut stage, 0, b"new")).unwrap(); + engine + .provider() + .inject(Fault::Release(ProviderError::Media)); + assert!(block_on(engine.commit_replace(session, &stage)).is_err()); + let mut bytes = [0; 3]; + block_on(engine.read(session, file.handle, 0, &mut bytes)).unwrap(); + assert_eq!(&bytes, b"old"); + engine + .provider() + .inject(Fault::Release(ProviderError::Media)); + assert!(block_on(engine.abort_replace(&stage)).is_err()); + block_on(engine.abort_replace(&stage)).unwrap(); + } + + #[test] + fn a_fresh_session_sees_an_empty_root() { + let (engine, session, root) = engine(); + let info = engine.stat(session, root).unwrap(); + assert_eq!(info.kind, ObjectKind::Directory); + assert_eq!(info.len, 0); + let page = engine.list(session, root, 0).unwrap(); + assert!(page.entries.is_empty()); + assert_eq!(page.next, None); + assert_eq!(engine.live_handles(session), 1); + } + + #[test] + fn engine_inline_footprint_stays_below_the_founded_stack() { + let bytes = core::mem::size_of::(); + std::println!("RamEngine bytes: {bytes}"); + assert!(bytes < 512 * 1024, "inline engine grew to {bytes} bytes"); + } + + #[test] + fn attaching_twice_is_refused() { + let (mut engine, session, _root) = engine(); + assert_eq!( + block_on(engine.attach_session(session)).unwrap_err(), + FsError::AlreadyExists + ); + } + + #[test] + fn an_unattached_session_has_no_root_to_open() { + let (mut engine, _session, _root) = engine(); + assert_eq!( + engine + .open_root(SessionId::new(1), Rights::ALL) + .unwrap_err(), + FsError::InvalidHandle + ); + } + + #[test] + fn create_hands_the_creator_a_usable_handle() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + assert_eq!(made.kind, ObjectKind::File); + assert_eq!(made.len, 0); + // Immediately usable, with no second lookup. + let revision = + block_on(engine.write_at(session, made.handle, 0, b"kore", ANY_REVISION)).unwrap(); + assert!(revision > made.revision); + let mut buf = [0u8; 4]; + assert_eq!( + block_on(engine.read(session, made.handle, 0, &mut buf)), + Ok(4) + ); + assert_eq!(&buf, b"kore"); + } + + #[test] + fn create_needs_namespace_rights() { + let (mut engine, session, _root) = engine(); + let read_only = engine.open_root(session, Rights::READ).unwrap(); + assert_eq!( + block_on(engine.create(session, read_only.handle, &name("f"), ObjectKind::File)) + .unwrap_err(), + FsError::AccessDenied + ); + // And nothing was created on the way to the refusal. + let page = engine.list(session, read_only.handle, 0).unwrap(); + assert!(page.entries.is_empty()); + } + + #[test] + fn create_beneath_a_file_is_refused() { + let (mut engine, session, root) = engine(); + let file = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + assert_eq!( + block_on(engine.create(session, file.handle, &name("x"), ObjectKind::File)) + .unwrap_err(), + FsError::NotDirectory + ); + } + + #[test] + fn create_with_a_duplicate_name_leaves_nothing_behind() { + let (mut engine, session, root) = engine(); + block_on(engine.create(session, root, &name("dup"), ObjectKind::File)).unwrap(); + let before = engine.object_count(); + assert_eq!( + block_on(engine.create(session, root, &name("dup"), ObjectKind::File)).unwrap_err(), + FsError::AlreadyExists + ); + assert_eq!( + engine.object_count(), + before, + "the refused create must not leak an object" + ); + } + + #[test] + fn writes_are_gated_on_write_rights() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + let reader = engine + .open_at(session, root, &name("f"), Rights::READ) + .unwrap(); + assert_eq!( + block_on(engine.write_at(session, reader.handle, 0, b"no", ANY_REVISION)).unwrap_err(), + FsError::AccessDenied + ); + // The writable handle still works. + assert!(block_on(engine.write_at(session, made.handle, 0, b"ok", ANY_REVISION)).is_ok()); + } + + #[test] + fn reads_and_writes_refuse_a_directory() { + let (mut engine, session, root) = engine(); + let mut buf = [0u8; 4]; + assert_eq!( + block_on(engine.read(session, root, 0, &mut buf)).unwrap_err(), + FsError::IsDirectory + ); + assert_eq!( + block_on(engine.write_at(session, root, 0, b"x", ANY_REVISION)).unwrap_err(), + FsError::IsDirectory + ); + assert_eq!( + block_on(engine.resize(session, root, 4, ANY_REVISION)).unwrap_err(), + FsError::IsDirectory + ); + assert_eq!( + block_on(engine.append(session, root, b"x", ANY_REVISION)).unwrap_err(), + FsError::IsDirectory + ); + } + + #[test] + fn listing_refuses_a_file_and_reading_refuses_a_directory() { + let (mut engine, session, root) = engine(); + let file = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + assert_eq!( + engine.list(session, file.handle, 0).unwrap_err(), + FsError::NotDirectory + ); + } + + #[test] + fn a_hole_left_by_a_sparse_write_reads_as_zeros() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, made.handle, 100, b"tail", ANY_REVISION)).unwrap(); + + let mut buf = [0xffu8; 104]; + assert_eq!( + block_on(engine.read(session, made.handle, 0, &mut buf)), + Ok(104) + ); + assert!( + buf[..100].iter().all(|b| *b == 0), + "the hole reads as zeros" + ); + assert_eq!(&buf[100..], b"tail"); + } + + #[test] + fn reads_past_the_end_are_short_then_empty() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, made.handle, 0, b"abcd", ANY_REVISION)).unwrap(); + + let mut buf = [0u8; 8]; + assert_eq!( + block_on(engine.read(session, made.handle, 2, &mut buf)), + Ok(2) + ); + assert_eq!(&buf[..2], b"cd"); + assert_eq!( + block_on(engine.read(session, made.handle, 4, &mut buf)), + Ok(0) + ); + assert_eq!( + block_on(engine.read(session, made.handle, 99, &mut buf)), + Ok(0) + ); + } + + #[test] + fn a_never_written_file_reads_empty() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + let mut buf = [0u8; 4]; + assert_eq!( + block_on(engine.read(session, made.handle, 0, &mut buf)), + Ok(0) + ); + // And it cost the provider nothing. + assert_eq!(engine.provider().chunks_in_use(), 0); + } + + #[test] + fn offset_plus_length_overflow_is_refused() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + let mut buf = [0u8; 8]; + assert_eq!( + block_on(engine.read(session, made.handle, u64::MAX, &mut buf)).unwrap_err(), + FsError::Range + ); + assert_eq!( + block_on(engine.write_at(session, made.handle, u64::MAX, b"x", ANY_REVISION)) + .unwrap_err(), + FsError::Range + ); + } + + #[test] + fn an_oversized_transfer_is_refused() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + let big = [0u8; crate::bounds::MAX_TRANSFER + 1]; + assert_eq!( + block_on(engine.write_at(session, made.handle, 0, &big, ANY_REVISION)).unwrap_err(), + FsError::Range + ); + } + + #[test] + fn a_file_larger_than_the_provider_allows_is_refused_as_storage() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + let ceiling = engine.provider().limits().max_object_bytes; + assert_eq!( + block_on(engine.resize(session, made.handle, ceiling + 1, ANY_REVISION)).unwrap_err(), + FsError::OutOfStorage + ); + // The refusal left the file as it was. + assert_eq!(engine.stat(session, made.handle).unwrap().len, 0); + } + + #[test] + fn resize_grows_with_zeros_and_shrinks_by_discarding() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, made.handle, 0, b"abcdefgh", ANY_REVISION)).unwrap(); + + block_on(engine.resize(session, made.handle, 4, ANY_REVISION)).unwrap(); + assert_eq!(engine.stat(session, made.handle).unwrap().len, 4); + let mut buf = [0u8; 8]; + assert_eq!( + block_on(engine.read(session, made.handle, 0, &mut buf)), + Ok(4) + ); + assert_eq!(&buf[..4], b"abcd"); + + block_on(engine.resize(session, made.handle, 8, ANY_REVISION)).unwrap(); + let mut buf = [0xffu8; 8]; + assert_eq!( + block_on(engine.read(session, made.handle, 0, &mut buf)), + Ok(8) + ); + assert_eq!(&buf[..4], b"abcd"); + assert!(buf[4..].iter().all(|b| *b == 0), "regrowth reads as zeros"); + } + + #[test] + fn sync_reports_success_without_promising_more() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + // Before any bytes there is no placement, and that is still a success. + assert!(block_on(engine.sync(session, made.handle)).is_ok()); + block_on(engine.write_at(session, made.handle, 0, b"x", ANY_REVISION)).unwrap(); + assert!(block_on(engine.sync(session, made.handle)).is_ok()); + } + + #[test] + fn sync_needs_write_rights() { + let (mut engine, session, root) = engine(); + block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + let reader = engine + .open_at(session, root, &name("f"), Rights::READ) + .unwrap(); + assert_eq!( + block_on(engine.sync(session, reader.handle)).unwrap_err(), + FsError::AccessDenied + ); + } + + #[test] + fn a_failed_provider_write_leaves_no_visible_effect() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + let committed = + block_on(engine.write_at(session, made.handle, 0, b"original", ANY_REVISION)).unwrap(); + + engine.provider().inject(Fault::Write(ProviderError::Media)); + assert_eq!( + block_on(engine.write_at(session, made.handle, 0, b"replaced", ANY_REVISION)) + .unwrap_err(), + FsError::ProviderFailure(ProviderError::Media) + ); + + // Length, revision, and bytes are all as they were. + let info = engine.stat(session, made.handle).unwrap(); + assert_eq!(info.len, 8); + assert_eq!(info.revision, committed); + let mut buf = [0u8; 8]; + assert_eq!( + block_on(engine.read(session, made.handle, 0, &mut buf)), + Ok(8) + ); + assert_eq!(&buf, b"original"); + } + + #[test] + fn close_drops_the_handle_but_not_the_object() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, made.handle, 0, b"keep", ANY_REVISION)).unwrap(); + engine.close(session, made.handle).unwrap(); + + assert_eq!( + engine.stat(session, made.handle).unwrap_err(), + FsError::InvalidHandle + ); + // Reopening through the still-live name finds the same bytes. + let again = engine + .open_at(session, root, &name("f"), Rights::ALL) + .unwrap(); + assert_eq!(again.len, 4); + let mut buf = [0u8; 4]; + assert_eq!( + block_on(engine.read(session, again.handle, 0, &mut buf)), + Ok(4) + ); + assert_eq!(&buf, b"keep"); + } + + #[test] + fn a_closed_handle_cannot_be_closed_twice() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + engine.close(session, made.handle).unwrap(); + assert_eq!( + engine.close(session, made.handle).unwrap_err(), + FsError::InvalidHandle + ); + } + + #[test] + fn opening_more_rights_than_the_route_holds_is_refused() { + let (mut engine, session, root) = engine(); + block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + let read_only_root = engine.open_root(session, Rights::READ).unwrap(); + assert_eq!( + engine + .open_at(session, read_only_root.handle, &name("f"), Rights::WRITE) + .unwrap_err(), + FsError::AccessDenied + ); + } + + #[test] + fn a_traversal_deeper_than_the_proof_bound_is_refused() { + let (mut engine, session, root) = engine(); + // Build a chain as deep as the proof allows, then one more. + let mut parent = root; + for depth in 0..MAX_PATH_DEPTH { + let made = block_on(engine.create(session, parent, &name("d"), ObjectKind::Directory)); + match made { + Ok(made) => parent = made.handle, + Err(err) => { + // The directory arena may bound this before the proof + // does; either way it must be a typed refusal. + assert!( + matches!( + err, + FsError::Range | FsError::OutOfObjects | FsError::OutOfHandles + ), + "unexpected refusal at depth {depth}: {err:?}" + ); + return; + } + } + } + // At the bound, a further create must refuse rather than record a + // truncated proof. + assert_eq!( + block_on(engine.create(session, parent, &name("d"), ObjectKind::Directory)) + .unwrap_err(), + FsError::Range + ); + } + + #[test] + fn detaching_a_session_reclaims_its_whole_root() { + let (mut engine, session, root) = engine(); + let dir = + block_on(engine.create(session, root, &name("d"), ObjectKind::Directory)).unwrap(); + let file = + block_on(engine.create(session, dir.handle, &name("f"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, file.handle, 0, b"bytes", ANY_REVISION)).unwrap(); + assert!(engine.provider().chunks_in_use() > 0); + + block_on(engine.detach_session(session)).unwrap(); + assert_eq!(engine.object_count(), 0, "root, dir, and file all released"); + assert_eq!(engine.provider().chunks_in_use(), 0); + assert_eq!(engine.provider().placements(), 0); + assert_eq!(engine.leaked_placements(), 0); + // Every handle it held is gone. + assert_eq!( + engine.stat(session, root).unwrap_err(), + FsError::InvalidHandle + ); + } + + /// Contention must not be able to hang the filesystem. + /// + /// Before this was bounded, a claim that outlived the turn it was taken in + /// made `reclaim` retry forever: the founded service livelocked, and + /// because it can neither exit nor report after readiness, the only + /// symptom was a machine that stopped answering. The harness reproduces + /// exactly that — `block_on` panics after 1024 polls — so an unbounded + /// retry fails this test by hanging rather than by asserting. + #[test] + fn contention_that_outlives_its_budget_is_abandoned_not_awaited_forever() { + let (engine, session, _root) = engine(); + let root_id = engine.root_object(session).unwrap(); + + // Exactly the contention a concurrent operation creates: every + // `reclaim_one` on this object now answers `Busy`. + let held = engine + .ns + .claim_object(root_id) + .expect("a fresh root object is claimable"); + + block_on(engine.reclaim(root_id)); + + // It gave up rather than spinning, and said so. + assert_eq!(engine.abandoned_reclaims(), 1); + // The object is still there — abandoning leaves it resident and + // unreachable, which is the whole bounded loss being traded for. + drop(held); + assert!(engine.object_count() > 0); + // And the engine still serves: the wedge is what we were avoiding. + assert!(engine.stat(session, _root).is_ok()); + } + + /// The budget is spent, not per-object: a reclaim that already burned it + /// on one contended object does not then spend it again on the next. + #[test] + fn an_uncontended_reclaim_still_completes_and_counts_nothing() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, made.handle, 0, b"bytes", ANY_REVISION)).unwrap(); + + assert!(block_on(engine.unlink(session, root, &name("f"))).is_ok()); + assert_eq!(engine.abandoned_reclaims(), 0); + assert_eq!(engine.leaked_placements(), 0); + assert_eq!(engine.provider().chunks_in_use(), 0); + } + + #[test] + fn a_refused_release_is_counted_not_reported() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create(session, root, &name("f"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, made.handle, 0, b"bytes", ANY_REVISION)).unwrap(); + + engine + .provider() + .inject(Fault::Release(ProviderError::Media)); + // The unlink genuinely happened, so it reports success… + assert!(block_on(engine.unlink(session, root, &name("f"))).is_ok()); + // …and the placement the provider would not take back is accounted + // for rather than silently forgotten. + assert_eq!(engine.leaked_placements(), 1); + assert_eq!( + engine + .open_at(session, root, &name("f"), Rights::ALL) + .unwrap_err(), + FsError::NotFound + ); + } + + #[test] + fn a_grant_cannot_hide_a_private_root_name() { + let (mut engine, source, source_root) = engine(); + let destination = SessionId::new(1); + block_on(engine.attach_session(destination)).unwrap(); + let destination_root = engine.open_root(destination, Rights::ALL).unwrap().handle; + block_on(engine.create( + destination, + destination_root, + &name("shared"), + ObjectKind::File, + )) + .unwrap(); + let source_file = + block_on(engine.create(source, source_root, &name("source"), ObjectKind::File)) + .unwrap(); + + assert_eq!( + engine + .grant( + source, + source_file.handle, + destination, + Rights::READ, + Some(name("shared")), + ) + .unwrap_err(), + FsError::AlreadyExists + ); + } + + #[test] + fn scoped_grant_enforces_graph_byte_and_handle_budgets() { + let (mut engine, source, source_root) = engine(); + let destination = SessionId::new(1); + block_on(engine.attach_session(destination)).unwrap(); + let quota = AuthorityQuota::new(2, 1, 3, 2); + let grant = engine + .grant_scoped( + source, + source_root, + destination, + Rights::ALL, + quota, + Some(name("data")), + ) + .unwrap(); + let root = engine.open_grant(destination, grant, Rights::ALL).unwrap(); + let file = + block_on(engine.create(destination, root.handle, &name("one"), ObjectKind::File)) + .unwrap(); + block_on(engine.write_at(destination, file.handle, 0, b"abc", ANY_REVISION)).unwrap(); + assert_eq!( + block_on(engine.write_at(destination, file.handle, 3, b"d", ANY_REVISION)).unwrap_err(), + FsError::OutOfStorage + ); + assert_eq!( + block_on(engine.create(destination, root.handle, &name("two"), ObjectKind::File,)) + .unwrap_err(), + FsError::OutOfObjects + ); + assert_eq!( + engine + .list(destination, root.handle, 0) + .unwrap() + .entries + .len(), + 1 + ); + } + + #[test] + fn a_file_claim_scopes_contention_and_quota_to_that_exact_object() { + let (engine, source, source_root) = engine(); + let destination = SessionId::new(1); + block_on(engine.attach_session(destination)).unwrap(); + let grant = engine + .grant_scoped( + source, + source_root, + destination, + Rights::ALL, + AuthorityQuota::new(3, 2, 3, 3), + Some(name("data")), + ) + .unwrap(); + let root = engine.open_grant(destination, grant, Rights::ALL).unwrap(); + let first = + block_on(engine.create(destination, root.handle, &name("first"), ObjectKind::File)) + .unwrap(); + let second = + block_on(engine.create(destination, root.handle, &name("second"), ObjectKind::File)) + .unwrap(); + + let route = engine.route(destination, first.handle).unwrap(); + let claimed = engine.ns.object_mut(route.object).unwrap(); + let quota = engine + .require_growth_with_claimed(route, &claimed, 0, 0, 3) + .unwrap(); + + assert_eq!( + block_on(engine.append(destination, first.handle, b"x", ANY_REVISION)).unwrap_err(), + FsError::Busy, + "the same file reports typed retryable contention" + ); + assert!( + block_on(engine.append(destination, second.handle, b"", ANY_REVISION)).is_ok(), + "an unrelated file remains independently usable" + ); + assert!(matches!( + engine.require_growth_with_claimed(route, &claimed, 0, 0, 1), + Err(FsError::OutOfStorage) + )); + + drop(quota); + drop(claimed); + assert!(block_on(engine.append(destination, first.handle, b"abc", ANY_REVISION)).is_ok()); + } + + #[test] + fn scoped_quota_cannot_widen_while_delegating() { + let (mut engine, source, source_root) = engine(); + let middle = SessionId::new(1); + let destination = SessionId::new(2); + block_on(engine.attach_session(middle)).unwrap(); + block_on(engine.attach_session(destination)).unwrap(); + let quota = AuthorityQuota::new(4, 4, 64, 4); + let first = engine + .grant_scoped(source, source_root, middle, Rights::ALL, quota, None) + .unwrap(); + let middle_root = engine.open_grant(middle, first, Rights::ALL).unwrap(); + assert_eq!( + engine + .grant_scoped( + middle, + middle_root.handle, + destination, + Rights::READ, + AuthorityQuota::new(5, 4, 64, 4), + None, + ) + .unwrap_err(), + FsError::AccessDenied + ); + } + + #[test] + fn detached_temporary_root_lives_exactly_as_long_as_its_grant() { + let (mut engine, launcher, launcher_root) = engine(); + let task = SessionId::new(1); + block_on(engine.attach_session(task)).unwrap(); + let tmp = block_on(engine.create( + launcher, + launcher_root, + &name("staging"), + ObjectKind::Directory, + )) + .unwrap(); + let grant = engine + .grant_scoped( + launcher, + tmp.handle, + task, + Rights::ALL, + AuthorityQuota::new(4, 4, 64, 4), + Some(name("tmp")), + ) + .unwrap(); + + block_on(engine.detach_edge(launcher, launcher_root, &name("staging"))).unwrap(); + engine.close(launcher, tmp.handle).unwrap(); + assert_eq!( + engine + .open_at(launcher, launcher_root, &name("staging"), Rights::READ) + .unwrap_err(), + FsError::NotFound, + "the launcher-only containment label is gone" + ); + + let task_tmp = engine.open_grant(task, grant, Rights::ALL).unwrap(); + block_on(engine.create(task, task_tmp.handle, &name("scratch"), ObjectKind::File)).unwrap(); + assert_eq!(engine.object_count(), 4, "two private roots plus tmp graph"); + + block_on(engine.detach_session(task)).unwrap(); + assert_eq!( + engine.object_count(), + 1, + "retiring the task grant reclaims the detached temporary graph" + ); + } + + #[test] + fn a_private_root_mutation_cannot_hide_a_grant_name() { + let (mut engine, source, source_root) = engine(); + let destination = SessionId::new(1); + block_on(engine.attach_session(destination)).unwrap(); + let destination_root = engine.open_root(destination, Rights::ALL).unwrap().handle; + let source_file = + block_on(engine.create(source, source_root, &name("source"), ObjectKind::File)) + .unwrap(); + engine + .grant( + source, + source_file.handle, + destination, + Rights::READ, + Some(name("shared")), + ) + .unwrap(); + + assert_eq!( + block_on(engine.create( + destination, + destination_root, + &name("shared"), + ObjectKind::File, + )) + .unwrap_err(), + FsError::AlreadyExists + ); + + let local = block_on(engine.create( + destination, + destination_root, + &name("local"), + ObjectKind::File, + )) + .unwrap(); + assert_eq!( + engine + .link(destination, destination_root, &name("shared"), local.handle) + .unwrap_err(), + FsError::AlreadyExists + ); + assert_eq!( + engine + .rename( + destination, + destination_root, + &name("local"), + &name("shared"), + ) + .unwrap_err(), + FsError::AlreadyExists + ); + } + + /// A lend borrows the object's live placement — no copy — and stays + /// frozen: a write during the lease moves the object to a fresh + /// placement, an unlink hands the placement to the lease, and the last + /// release returns it. Placement counts prove which path ran. + #[test] + fn a_lend_borrows_the_placement_and_only_a_racing_write_copies() { + let (mut engine, session, root) = engine(); + let file = block_on(engine.create(session, root, &name("img"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, file.handle, 0, b"abcdef", ANY_REVISION)).unwrap(); + let owner = ExtractionOwner::new(3); + let baseline = engine.provider().placements(); + + // Lend and release without a write: no placement ever appears. + let (lease, _) = block_on(engine.lend_version(owner, session, file.handle)).unwrap(); + assert_eq!( + engine.provider().placements(), + baseline, + "a lend allocates nothing" + ); + let mut back = [0u8; 6]; + assert_eq!( + block_on(engine.read_lent_version(owner, lease, 0, &mut back)), + Ok(6) + ); + assert_eq!(&back, b"abcdef"); + block_on(engine.release_lent_version(owner, lease)).unwrap(); + assert_eq!( + engine.provider().placements(), + baseline, + "the object keeps its placement" + ); + let mut live = [0u8; 6]; + assert_eq!( + block_on(engine.read(session, file.handle, 0, &mut live)), + Ok(6) + ); + assert_eq!(&live, b"abcdef"); + + // A write during the lease detaches: one fresh placement for the + // object, the lease frozen on the old one, released with the lease. + let (lease, _) = block_on(engine.lend_version(owner, session, file.handle)).unwrap(); + block_on(engine.write_at(session, file.handle, 0, b"ABCDEF", ANY_REVISION)).unwrap(); + assert_eq!(engine.provider().placements(), baseline + 1); + assert_eq!( + block_on(engine.read_lent_version(owner, lease, 0, &mut back)), + Ok(6) + ); + assert_eq!(&back, b"abcdef"); + assert_eq!( + block_on(engine.read(session, file.handle, 0, &mut live)), + Ok(6) + ); + assert_eq!(&live, b"ABCDEF"); + // A second write during the same lease costs nothing more. + block_on(engine.write_at(session, file.handle, 0, b"abCDEF", ANY_REVISION)).unwrap(); + assert_eq!(engine.provider().placements(), baseline + 1); + block_on(engine.release_lent_version(owner, lease)).unwrap(); + assert_eq!(engine.provider().placements(), baseline); + assert_eq!(engine.leaked_placements(), 0); + + // Two leases on one placement: a write detaches once; the first + // release keeps the placement for the second, the last frees it. + let (first, _) = block_on(engine.lend_version(owner, session, file.handle)).unwrap(); + let (second, _) = block_on(engine.lend_version(owner, session, file.handle)).unwrap(); + block_on(engine.write_at(session, file.handle, 0, b"xxxxxx", ANY_REVISION)).unwrap(); + assert_eq!(engine.provider().placements(), baseline + 1); + block_on(engine.release_lent_version(owner, first)).unwrap(); + assert_eq!(engine.provider().placements(), baseline + 1); + assert_eq!( + block_on(engine.read_lent_version(owner, second, 0, &mut back)), + Ok(6) + ); + assert_eq!(&back, b"abCDEF"); + block_on(engine.release_lent_version(owner, second)).unwrap(); + assert_eq!(engine.provider().placements(), baseline); + + // Reclaimed during the lease: unlinked and closed, the object is + // gone, but its placement passes to the lease rather than being + // released beneath it, and the lease's release returns it. + let (lease, _) = block_on(engine.lend_version(owner, session, file.handle)).unwrap(); + assert!(block_on(engine.unlink(session, root, &name("img"))).is_ok()); + engine.close(session, file.handle).unwrap(); + assert_eq!( + engine.provider().placements(), + baseline, + "not released beneath the lease" + ); + block_on(engine.release_lent_version(owner, lease)).unwrap(); + assert_eq!(engine.provider().placements(), baseline - 1); + assert_eq!(engine.leaked_placements(), 0); + } + + #[test] + fn lent_version_is_frozen_across_writes_and_bound_to_its_owner() { + let (mut engine, session, root) = engine(); + let file = block_on(engine.create(session, root, &name("doc"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, file.handle, 0, br#"{"old":1}"#, ANY_REVISION)).unwrap(); + let owner = ExtractionOwner::new(7); + let stranger = ExtractionOwner::new(8); + let (lease, identity) = block_on(engine.lend_version(owner, session, file.handle)).unwrap(); + + block_on(engine.write_at(session, file.handle, 0, br#"{"new":2}"#, ANY_REVISION)).unwrap(); + let mut frozen = [0u8; 9]; + assert_eq!( + block_on(engine.read_lent_version(owner, lease, 0, &mut frozen)), + Ok(9) + ); + assert_eq!(&frozen, br#"{"old":1}"#); + assert_eq!(identity.digest, *blake3::hash(&frozen).as_bytes()); + assert_eq!( + block_on(engine.read_lent_version(stranger, lease, 0, &mut frozen)), + Err(ExtractionError::InvalidLease) + ); + + block_on(engine.release_lent_version(owner, lease)).unwrap(); + assert_eq!( + block_on(engine.read_lent_version(owner, lease, 0, &mut frozen)), + Err(ExtractionError::InvalidLease) + ); + // Cancelling extraction changed neither the current bytes nor its + // namespace identity. + let mut current = [0u8; 9]; + block_on(engine.read(session, file.handle, 0, &mut current)).unwrap(); + assert_eq!(¤t, br#"{"new":2}"#); + } + + #[test] + fn create_stamps_a_subtype_that_open_stat_and_list_report() { + let (mut engine, session, root) = engine(); + let made = block_on(engine.create_typed( + session, + root, + &name("tool"), + ObjectKind::File, + ObjectSubtype::EXECUTABLE_IMAGE, + )) + .unwrap(); + assert_eq!(made.subtype, ObjectSubtype::EXECUTABLE_IMAGE); + let plain = + block_on(engine.create(session, root, &name("note"), ObjectKind::File)).unwrap(); + assert_eq!(plain.subtype, ObjectSubtype::PLAIN); + + let reopened = engine + .open_at(session, root, &name("tool"), Rights::READ) + .unwrap(); + assert_eq!(reopened.subtype, ObjectSubtype::EXECUTABLE_IMAGE); + assert_eq!( + engine.stat(session, reopened.handle).unwrap().subtype, + ObjectSubtype::EXECUTABLE_IMAGE + ); + let page = engine.list(session, root, 0).unwrap(); + let tool = page + .entries + .iter() + .find(|entry| entry.name == name("tool")) + .unwrap(); + assert_eq!(tool.subtype, ObjectSubtype::EXECUTABLE_IMAGE); + let note = page + .entries + .iter() + .find(|entry| entry.name == name("note")) + .unwrap(); + assert_eq!(note.subtype, ObjectSubtype::PLAIN); + } + + /// A directory carries no subtype, and the refusal leaves nothing behind. + #[test] + fn a_directory_refuses_a_subtype() { + let (mut engine, session, root) = engine(); + assert_eq!( + block_on(engine.create_typed( + session, + root, + &name("dir"), + ObjectKind::Directory, + ObjectSubtype::EXECUTABLE_IMAGE, + )) + .unwrap_err(), + FsError::Unsupported + ); + assert!(engine.list(session, root, 0).unwrap().entries.is_empty()); + assert_eq!(engine.live_handles(session), 1); + } + + /// The stamp is data: an executable-image file is opened, read and + /// written like any other, and it is *not* executable-source authority — + /// that stays with `BIND_EXECUTABLE` on the route. + #[test] + fn a_subtype_grants_no_rights() { + let (mut engine, session, root) = engine(); + let file = block_on(engine.create_typed( + session, + root, + &name("tool"), + ObjectKind::File, + ObjectSubtype::EXECUTABLE_IMAGE, + )) + .unwrap(); + block_on(engine.write_at(session, file.handle, 0, b"ELF", ANY_REVISION)).unwrap(); + let readable = engine + .open_at(session, root, &name("tool"), Rights::READ) + .unwrap(); + assert_eq!(readable.subtype, ObjectSubtype::EXECUTABLE_IMAGE); + assert_eq!( + block_on(engine.lend_executable_version( + ExtractionOwner::new(21), + session, + readable.handle + )), + Err(ExtractionError::Filesystem(FsError::AccessDenied)), + "a subtype is data; it never stands in for BIND_EXECUTABLE" + ); + } + + /// An engine whose volume predates subtypes refuses to hold a fact the + /// disk would drop; plain creates are unaffected. + #[test] + fn a_legacy_version_engine_refuses_subtyped_creates() { + let (mut engine, session, root) = engine(); + engine.durable_subtypes = false; + assert_eq!( + block_on(engine.create_typed( + session, + root, + &name("tool"), + ObjectKind::File, + ObjectSubtype::EXECUTABLE_IMAGE, + )) + .unwrap_err(), + FsError::Unsupported + ); + let plain = + block_on(engine.create(session, root, &name("note"), ObjectKind::File)).unwrap(); + assert_eq!(plain.subtype, ObjectSubtype::PLAIN); + } + + #[test] + fn executable_lending_requires_its_separate_route_authority() { + let (mut engine, session, root) = engine(); + let file = block_on(engine.create(session, root, &name("tool"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, file.handle, 0, b"ELF", ANY_REVISION)).unwrap(); + + let visible = engine + .open_at(session, root, &name("tool"), Rights::READ) + .expect("ordinary readable view"); + let owner = ExtractionOwner::new(17); + assert_eq!( + block_on(engine.lend_executable_version(owner, session, visible.handle)), + Err(ExtractionError::Filesystem(FsError::AccessDenied)), + "readable bytes alone are not executable-source authority" + ); + + let bindable = engine + .open_at( + session, + root, + &name("tool"), + Rights::READ.union(Rights::BIND_EXECUTABLE), + ) + .expect("explicit executable source view"); + let (lease, identity) = + block_on(engine.lend_executable_version(owner, session, bindable.handle)) + .expect("binding authority freezes the exact revision"); + assert_eq!(identity.len, 3); + block_on(engine.release_lent_version(owner, lease)).unwrap(); + } + + #[test] + fn a_home_face_cannot_nominate_its_own_bytes_as_executable_source() { + // The self-bless hole, closed. A task's writable home is `HOME`, + // which withholds `BIND_EXECUTABLE`. It may create and write files — + // but it can never open ANY of them with the executable-source right, + // so it cannot freeze its own output as launch authority; that stays + // the install authority's (`ALL`) privilege. + let mut engine = RamEngine::default(); + let session = SessionId::new(0); + block_on(engine.attach_session(session)).unwrap(); + let home = engine.open_root(session, Rights::HOME).unwrap().handle; + + // Writing works — HOME carries WRITE and MUTATE_NAMESPACE. + let file = block_on(engine.create(session, home, &name("tool"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, file.handle, 0, b"ELF", ANY_REVISION)).unwrap(); + + // Nominating does not — the route lacks BIND_EXECUTABLE, so the + // executable-source open is refused before any binding can begin. + assert!( + engine + .open_at( + session, + home, + &name("tool"), + Rights::READ.union(Rights::BIND_EXECUTABLE), + ) + .is_err(), + "a writable home must not open its own bytes as executable source" + ); + } + + /// The same hole through the stamp. A late loader links only what the + /// store says the installer wrote (`EXECUTABLE_IMAGE`), so a home face — + /// WRITE and MUTATE_NAMESPACE, no BIND_EXECUTABLE — must not be able to + /// say it: not that stamp, not the package manifest's, not one this + /// engine cannot interpret. A refused stamp leaves nothing behind; + /// `PLAIN` stays the home's to make, and the install authority stamps. + #[test] + fn a_home_face_cannot_stamp_an_attesting_subtype() { + let mut engine = RamEngine::default(); + let session = SessionId::new(0); + block_on(engine.attach_session(session)).unwrap(); + let home = engine.open_root(session, Rights::HOME).unwrap().handle; + let objects = engine.object_count(); + for (leaf, subtype) in [ + ("lua_demo.so", ObjectSubtype::EXECUTABLE_IMAGE), + ("manifest", ObjectSubtype::PACKAGE_MANIFEST), + ("future", ObjectSubtype::from_bits(0x7f)), + ] { + assert_eq!( + block_on(engine.create_typed( + session, + home, + &name(leaf), + ObjectKind::File, + subtype + )) + .unwrap_err(), + FsError::AccessDenied, + "a writer forged the {subtype:?} stamp" + ); + } + assert_eq!(engine.object_count(), objects); + assert!(engine.list(session, home, 0).unwrap().entries.is_empty()); + + let plain = block_on(engine.create_typed( + session, + home, + &name("notes"), + ObjectKind::File, + ObjectSubtype::PLAIN, + )) + .unwrap(); + assert_eq!(plain.subtype, ObjectSubtype::PLAIN); + + let install = engine.open_root(session, Rights::ALL).unwrap().handle; + let stamped = block_on(engine.create_typed( + session, + install, + &name("lua_demo.so"), + ObjectKind::File, + ObjectSubtype::EXECUTABLE_IMAGE, + )) + .unwrap(); + assert_eq!(stamped.subtype, ObjectSubtype::EXECUTABLE_IMAGE); + } + + /// A stamp vouches for bytes, so it holds only while they change through + /// the authority that may stamp: a WRITE route without BIND_EXECUTABLE + /// can read an installed image but cannot write, append, resize, or + /// stage a replacement of it — the swap that would keep the installer's + /// stamp over a writer's code. The same route still writes its plain + /// files, and the install authority still rewrites its own (exec's + /// re-seed of a different archive member). + #[test] + fn an_attested_objects_bytes_change_only_through_the_install_authority() { + let (engine, session, root) = engine(); + let image = block_on(engine.create_typed( + session, + root, + &name("lua_demo.so"), + ObjectKind::File, + ObjectSubtype::EXECUTABLE_IMAGE, + )) + .unwrap(); + block_on(engine.write_at(session, image.handle, 0, b"ELF", ANY_REVISION)).unwrap(); + let note = block_on(engine.create(session, root, &name("note"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, note.handle, 0, b"hi", ANY_REVISION)).unwrap(); + + let home = engine.open_root(session, Rights::HOME).unwrap().handle; + let writable = Rights::READ.union(Rights::WRITE); + let forged = engine + .open_at(session, home, &name("lua_demo.so"), writable) + .expect("WRITE on the route is not the question; the stamp is"); + assert_eq!(forged.subtype, ObjectSubtype::EXECUTABLE_IMAGE); + assert_eq!( + block_on(engine.write_at(session, forged.handle, 0, b"EVIL", ANY_REVISION)), + Err(FsError::AccessDenied) + ); + assert_eq!( + block_on(engine.append(session, forged.handle, b"!", ANY_REVISION)), + Err(FsError::AccessDenied) + ); + assert_eq!( + block_on(engine.resize(session, forged.handle, 0, ANY_REVISION)), + Err(FsError::AccessDenied) + ); + assert!(matches!( + block_on(engine.begin_replace(session, forged.handle, 4, ANY_REVISION)), + Err(FsError::AccessDenied) + )); + let mut bytes = [0u8; 3]; + assert_eq!( + block_on(engine.read(session, forged.handle, 0, &mut bytes)), + Ok(3) + ); + assert_eq!(&bytes, b"ELF", "the installer's bytes are untouched"); + + let plain = engine + .open_at(session, home, &name("note"), writable) + .unwrap(); + block_on(engine.write_at(session, plain.handle, 0, b"ok", ANY_REVISION)).unwrap(); + + block_on(engine.write_at(session, image.handle, 0, b"ELF2", ANY_REVISION)).unwrap(); + let mut stage = block_on(engine.begin_replace(session, image.handle, 4, ANY_REVISION)) + .expect("the install authority replaces its own image"); + block_on(engine.write_replacement(session, &mut stage, 0, b"ELF3")).unwrap(); + block_on(engine.commit_replace(session, &stage)).unwrap(); + let mut replaced = [0u8; 4]; + block_on(engine.read(session, image.handle, 0, &mut replaced)).unwrap(); + assert_eq!(&replaced, b"ELF3"); + } + + #[test] + fn task_local_executable_routes_freeze_distinct_revisions() { + let (mut engine, first, first_root) = engine(); + let second = SessionId::new(1); + block_on(engine.attach_session(second)).unwrap(); + let second_root = engine.open_root(second, Rights::ALL).unwrap().handle; + + // Local display names are not executable authority. Each task opens + // the object through its own native App grant and the exact immutable + // revision is what later becomes launch authority. + let tool = name("tool"); + + let first_file = + block_on(engine.create(first, first_root, &tool, ObjectKind::File)).unwrap(); + let second_file = + block_on(engine.create(second, second_root, &tool, ObjectKind::File)).unwrap(); + block_on(engine.write_at(first, first_file.handle, 0, b"task-a", ANY_REVISION)).unwrap(); + block_on(engine.write_at(second, second_file.handle, 0, b"task-b", ANY_REVISION)).unwrap(); + + let source_rights = Rights::READ.union(Rights::BIND_EXECUTABLE); + let first_source = engine + .open_at(first, first_root, &tool, source_rights) + .unwrap(); + let second_source = engine + .open_at(second, second_root, &tool, source_rights) + .unwrap(); + let first_owner = ExtractionOwner::new(21); + let second_owner = ExtractionOwner::new(22); + let (first_lease, first_identity) = + block_on(engine.lend_executable_version(first_owner, first, first_source.handle)) + .unwrap(); + let (second_lease, second_identity) = + block_on(engine.lend_executable_version(second_owner, second, second_source.handle)) + .unwrap(); + + assert_ne!(first_identity.source, second_identity.source); + assert_ne!(first_identity.digest, second_identity.digest); + + // Replacing the first task's selected file creates a new source + // identity, but the old lease remains an immutable launch source. + block_on(engine.write_at(first, first_file.handle, 0, b"task-c", ANY_REVISION)).unwrap(); + let (replacement_lease, replacement_identity) = + block_on(engine.lend_executable_version(first_owner, first, first_source.handle)) + .unwrap(); + assert_eq!(replacement_identity.source, first_identity.source); + assert!(replacement_identity.version > first_identity.version); + assert_ne!(replacement_identity.digest, first_identity.digest); + + let mut old = [0u8; 6]; + let mut replacement = [0u8; 6]; + let mut other = [0u8; 6]; + assert_eq!( + block_on(engine.read_lent_version(first_owner, first_lease, 0, &mut old)), + Ok(old.len()) + ); + assert_eq!( + block_on( + engine.read_lent_version(first_owner, replacement_lease, 0, &mut replacement,) + ), + Ok(replacement.len()) + ); + assert_eq!( + block_on(engine.read_lent_version(second_owner, second_lease, 0, &mut other)), + Ok(other.len()) + ); + assert_eq!(&old, b"task-a"); + assert_eq!(&replacement, b"task-c"); + assert_eq!(&other, b"task-b"); + + block_on(engine.release_lent_version(first_owner, first_lease)).unwrap(); + block_on(engine.release_lent_version(first_owner, replacement_lease)).unwrap(); + block_on(engine.release_lent_version(second_owner, second_lease)).unwrap(); + } + + #[test] + fn revoked_source_authority_blocks_lent_reads_and_result_commit_but_not_release() { + let (mut engine, source, source_root) = engine(); + let destination = SessionId::new(1); + block_on(engine.attach_session(destination)).unwrap(); + let file = + block_on(engine.create(source, source_root, &name("shared.json"), ObjectKind::File)) + .unwrap(); + let bytes = br#"{"shared":true}"#; + block_on(engine.write_at(source, file.handle, 0, bytes, ANY_REVISION)).unwrap(); + let grant = engine + .grant(source, file.handle, destination, Rights::READ, None) + .unwrap(); + let shared = engine.open_grant(destination, grant, Rights::READ).unwrap(); + let owner = ExtractionOwner::new(u64::from(destination.index())); + let (lease, input) = + block_on(engine.lend_version(owner, destination, shared.handle)).unwrap(); + let result = parse_json_rust(input, bytes).unwrap(); + + block_on(engine.revoke(source, grant)).unwrap(); + let mut frozen = [0u8; 15]; + assert_eq!( + block_on(engine.read_lent_version(owner, lease, 0, &mut frozen)), + Err(ExtractionError::Filesystem(FsError::AccessLost)) + ); + assert_eq!( + engine.commit_derived_result(owner, lease, result, 0), + Err(ExtractionError::Filesystem(FsError::AccessLost)) + ); + + // Revocation cancels authority to use the frozen version, but never + // strands its storage: the stamped owner can still release it. + block_on(engine.release_lent_version(owner, lease)).unwrap(); + let mut current = [0u8; 15]; + assert_eq!( + block_on(engine.read(source, file.handle, 0, &mut current)), + Ok(bytes.len()) + ); + assert_eq!(¤t, bytes); + assert_eq!(engine.extraction.result_count(), 0); + } + + #[test] + fn result_publication_is_exact_validated_and_obsolete_versions_fail_closed() { + let (mut engine, session, root) = engine(); + let file = block_on(engine.create(session, root, &name("doc"), ObjectKind::File)).unwrap(); + let bytes = br#"{"answer":42}"#; + block_on(engine.write_at(session, file.handle, 0, bytes, ANY_REVISION)).unwrap(); + let owner = ExtractionOwner::new(11); + let (lease, input) = block_on(engine.lend_version(owner, session, file.handle)).unwrap(); + + let valid = parse_json_rust(input, bytes).unwrap(); + let info = engine + .commit_derived_result(owner, lease, valid, 0) + .unwrap(); + assert_eq!(info.input, input); + assert_eq!(info.parser, RUST_JSON); + assert!( + engine + .derived_result(session, file.handle, RUST_JSON) + .unwrap() + .is_some() + ); + + let mut forged = parse_json_rust(input, bytes).unwrap(); + forged.provenance.input.digest[0] ^= 1; + assert!(matches!( + engine.commit_derived_result(owner, lease, forged, 0), + Err(ExtractionError::InvalidResult(_)) + )); + assert_eq!(engine.extraction.result_count(), 1); + + block_on(engine.append(session, file.handle, b" ", ANY_REVISION)).unwrap(); + assert!( + engine + .derived_result(session, file.handle, RUST_JSON) + .unwrap() + .is_none() + ); + let stale = parse_json_rust(input, bytes).unwrap(); + assert_eq!( + engine.commit_derived_result(owner, lease, stale, 0), + Err(ExtractionError::ObsoleteVersion) + ); + assert_eq!(engine.extraction.result_count(), 1); + block_on(engine.release_lent_version(owner, lease)).unwrap(); + } + + #[test] + fn reprocessing_replaces_only_the_selected_parser_lineage() { + let (mut engine, session, root) = engine(); + let file = block_on(engine.create(session, root, &name("doc"), ObjectKind::File)).unwrap(); + let bytes = br#"{"a":1,"b":2}"#; + block_on(engine.write_at(session, file.handle, 0, bytes, ANY_REVISION)).unwrap(); + let owner = ExtractionOwner::new(12); + let (lease, input) = block_on(engine.lend_version(owner, session, file.handle)).unwrap(); + + engine + .commit_derived_result(owner, lease, parse_json_rust(input, bytes).unwrap(), 0) + .unwrap(); + engine + .commit_derived_result(owner, lease, parse_json_c(input, bytes).unwrap(), 0) + .unwrap(); + assert_eq!(engine.extraction.result_count(), 2); + + let mut rerun = parse_json_rust(input, bytes).unwrap(); + rerun.facts[1].value = FactValue::Unsigned(99); + engine + .commit_derived_result(owner, lease, rerun, 0) + .unwrap(); + assert_eq!(engine.extraction.result_count(), 2); + let selected = engine + .derived_result(session, file.handle, RUST_JSON) + .unwrap() + .unwrap(); + assert_eq!(selected.facts[1].value, FactValue::Unsigned(99)); + assert!( + engine + .derived_result(session, file.handle, C_JSON) + .unwrap() + .is_some() + ); + + block_on(engine.release_lent_version(owner, lease)).unwrap(); + block_on(engine.unlink(session, root, &name("doc"))).unwrap(); + assert_eq!(engine.extraction.result_count(), 0); + } + + fn query(plane: crate::search::SearchPlane, term: &str) -> SearchQuery { + SearchQuery { + content: Some(crate::search::SearchContent { + plane, + term: crate::search::SearchTerm::new(term).unwrap(), + }), + freshness: FreshnessPolicy::CurrentOnly, + ..SearchQuery::ALL + } + } + + #[test] + fn unlinking_an_unindexed_witness_restores_search_completeness() { + let (mut engine, session, root) = engine(); + let witness = name("kore-fs-search-witness"); + let file = block_on(engine.create(session, root, &witness, ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, file.handle, 0, &[0xff], ANY_REVISION)).unwrap(); + + let full_text = query(crate::search::SearchPlane::FullText, "answer"); + assert_eq!( + engine.search(session, root, full_text, None), + Err(FsError::IndexIncomplete) + ); + + block_on(engine.unlink(session, root, &witness)).unwrap(); + let complete = engine.search(session, root, full_text, None).unwrap(); + assert!(complete.hits.is_empty()); + } + + #[test] + fn all_content_planes_use_current_parser_results_and_typed_budgets() { + let (mut engine, session, root) = engine(); + let file = + block_on(engine.create(session, root, &name("manual"), ObjectKind::File)).unwrap(); + let bytes = b"capability graph search"; + block_on(engine.write_at(session, file.handle, 0, bytes, ANY_REVISION)).unwrap(); + let owner = ExtractionOwner::new(31); + let (lease, input) = block_on(engine.lend_version(owner, session, file.handle)).unwrap(); + let mut result = parse_plain_text(input, bytes).unwrap(); + result + .references + .push(Reference { + source: SourceRange::new(0, 10), + target: heapless::String::try_from("object://manual").unwrap(), + }) + .unwrap(); + engine + .commit_derived_result(owner, lease, result, 0) + .unwrap(); + + for (plane, term) in [ + (crate::search::SearchPlane::FactKey, "lines"), + (crate::search::SearchPlane::FullText, "graph"), + (crate::search::SearchPlane::Relationship, "object://manual"), + (crate::search::SearchPlane::Semantic, "capability search"), + ] { + let page = engine + .search(session, root, query(plane, term), None) + .unwrap(); + assert_eq!(page.hits.len(), 1, "{plane:?}"); + assert!(page.hits[0].score.get() > 0, "{plane:?}"); + assert!(!page.hits[0].snippet.is_empty(), "{plane:?}"); + assert_ne!(page.index_generation, 0); + } + + let mut bounded = query(crate::search::SearchPlane::FullText, "graph"); + bounded.max_work = 1; + assert_eq!( + engine.search(session, root, bounded, None), + Err(FsError::BudgetExceeded) + ); + block_on(engine.release_lent_version(owner, lease)).unwrap(); + } + + #[test] + fn semantic_page_orders_higher_token_coverage_before_traversal_order() { + let (mut engine, session, root) = engine(); + for (ordinal, file_name, bytes) in [ + (0, "partial", &b"capability only"[..]), + (1, "complete", &b"capability graph"[..]), + ] { + let file = + block_on(engine.create(session, root, &name(file_name), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, file.handle, 0, bytes, ANY_REVISION)).unwrap(); + let owner = ExtractionOwner::new(80 + ordinal); + let (lease, input) = + block_on(engine.lend_version(owner, session, file.handle)).unwrap(); + engine + .commit_derived_result(owner, lease, parse_plain_text(input, bytes).unwrap(), 0) + .unwrap(); + block_on(engine.release_lent_version(owner, lease)).unwrap(); + } + + let page = engine + .search( + session, + root, + query(crate::search::SearchPlane::Semantic, "capability graph"), + None, + ) + .unwrap(); + assert_eq!(page.hits.len(), 2); + assert_eq!(page.hits[0].path[0], name("complete")); + assert_eq!(page.hits[0].score.get(), 10_000); + assert_eq!(page.hits[1].score.get(), 5_000); + } + + #[test] + fn current_only_search_never_presents_an_unindexed_file_as_no_match() { + let (engine, session, root) = engine(); + let file = + block_on(engine.create(session, root, &name("legacy"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(session, file.handle, 0, b"needle", ANY_REVISION)).unwrap(); + assert_eq!( + engine.search( + session, + root, + query(crate::search::SearchPlane::FullText, "needle"), + None, + ), + Err(FsError::IndexIncomplete) + ); + let structural = engine + .search( + session, + root, + SearchQuery { + name_contains: Some(name("legacy")), + ..SearchQuery::ALL + }, + None, + ) + .unwrap(); + assert_eq!(structural.hits.len(), 1); + } + + #[test] + fn relationship_rebuild_preserves_durable_identity() { + let (mut engine, session, root) = engine(); + let file = + block_on(engine.create(session, root, &name("source"), ObjectKind::File)).unwrap(); + let bytes = b"source text"; + block_on(engine.write_at(session, file.handle, 0, bytes, ANY_REVISION)).unwrap(); + let identity = engine.stat(session, file.handle).unwrap(); + let owner = ExtractionOwner::new(32); + let (lease, input) = block_on(engine.lend_version(owner, session, file.handle)).unwrap(); + + let mut first = parse_plain_text(input, bytes).unwrap(); + first + .references + .push(Reference { + source: SourceRange::new(0, 6), + target: heapless::String::try_from("object://old").unwrap(), + }) + .unwrap(); + engine + .commit_derived_result(owner, lease, first, 0) + .unwrap(); + + let mut rebuilt = parse_plain_text(input, bytes).unwrap(); + rebuilt + .references + .push(Reference { + source: SourceRange::new(0, 6), + target: heapless::String::try_from("object://new").unwrap(), + }) + .unwrap(); + engine + .commit_derived_result(owner, lease, rebuilt, 0) + .unwrap(); + + assert!( + engine + .search( + session, + root, + query(crate::search::SearchPlane::Relationship, "object://old"), + None, + ) + .unwrap() + .hits + .is_empty() + ); + assert_eq!( + engine + .search( + session, + root, + query(crate::search::SearchPlane::Relationship, "object://new"), + None, + ) + .unwrap() + .hits + .len(), + 1 + ); + let after = engine.stat(session, file.handle).unwrap(); + assert_eq!(after.stable_object, identity.stable_object); + assert_eq!(after.revision, identity.revision); + assert_eq!(after.len, identity.len); + block_on(engine.release_lent_version(owner, lease)).unwrap(); + } + + #[test] + fn hidden_index_publication_does_not_change_another_views_reply() { + let (mut engine, visible_session, visible_root) = engine(); + let visible = block_on(engine.create( + visible_session, + visible_root, + &name("visible"), + ObjectKind::File, + )) + .unwrap(); + let visible_bytes = b"shared graph text"; + block_on(engine.write_at( + visible_session, + visible.handle, + 0, + visible_bytes, + ANY_REVISION, + )) + .unwrap(); + let visible_owner = ExtractionOwner::new(33); + let (visible_lease, visible_input) = + block_on(engine.lend_version(visible_owner, visible_session, visible.handle)).unwrap(); + engine + .commit_derived_result( + visible_owner, + visible_lease, + parse_plain_text(visible_input, visible_bytes).unwrap(), + 0, + ) + .unwrap(); + let wanted = query(crate::search::SearchPlane::FullText, "graph"); + let before = engine + .search(visible_session, visible_root, wanted, None) + .unwrap(); + + let hidden_session = SessionId::new(1); + block_on(engine.attach_session(hidden_session)).unwrap(); + let hidden_root = engine + .open_root(hidden_session, Rights::ALL) + .unwrap() + .handle; + let hidden = block_on(engine.create( + hidden_session, + hidden_root, + &name("hidden"), + ObjectKind::File, + )) + .unwrap(); + let hidden_bytes = b"hidden graph match"; + block_on(engine.write_at(hidden_session, hidden.handle, 0, hidden_bytes, ANY_REVISION)) + .unwrap(); + let hidden_owner = ExtractionOwner::new(34); + let (hidden_lease, hidden_input) = + block_on(engine.lend_version(hidden_owner, hidden_session, hidden.handle)).unwrap(); + engine + .commit_derived_result( + hidden_owner, + hidden_lease, + parse_plain_text(hidden_input, hidden_bytes).unwrap(), + 0, + ) + .unwrap(); + + let after = engine + .search(visible_session, visible_root, wanted, None) + .unwrap(); + assert_eq!( + after, before, + "hidden facts must not perturb any reply field" + ); + block_on(engine.release_lent_version(visible_owner, visible_lease)).unwrap(); + block_on(engine.release_lent_version(hidden_owner, hidden_lease)).unwrap(); + } +} diff --git a/native/services/dagfs/src/error.rs b/native/services/dagfs/src/error.rs new file mode 100644 index 0000000..4eb0e46 --- /dev/null +++ b/native/services/dagfs/src/error.rs @@ -0,0 +1,6 @@ +//! Compatibility home for the shared filesystem refusal contract. +//! +//! The values live in `kore-dagfs` so a process can classify a refusal +//! without linking this service's provider and device dependencies. + +pub use kore_dagfs::{FsError, FsResult, ProviderError}; diff --git a/native/services/dagfs/src/extraction.rs b/native/services/dagfs/src/extraction.rs new file mode 100644 index 0000000..6602e2c --- /dev/null +++ b/native/services/dagfs/src/extraction.rs @@ -0,0 +1,1053 @@ +//! Immutable file-version lending and parser-result association. +//! +//! This is the filesystem-specific half of extraction. The generic broker +//! and workers never receive namespace handles, paths, or containment-graph +//! authority. They receive one owner-bound [`VersionLease`] whose only +//! operation is reading a frozen byte range. A committed write therefore +//! cannot change the bytes beneath an in-flight parser. + +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +use heapless::Vec; +use kore_svc_extract::{ + WORKER_REPORT_BYTES, decode_result, encode_result, + registry::{ParserId, Registry, ValidationError}, + schema::{FactValue, InputIdentity, NormalizedResult}, +}; + +use crate::{ + ServiceAlloc, + authority::{Handle, SessionId}, + bounds::{ + DERIVED_WIRE_CHUNK_BYTES, MAX_DERIVED_RESULTS, MAX_DERIVED_WIRE_CHUNKS, MAX_LENT_VERSIONS, + MAX_OBJECTS, + }, + error::{FsError, FsResult, ProviderError}, + object::{ObjectId, ObjectIdExt, Revision}, + search::{IndexMatch, MAX_SEARCH_SNIPPET, QueryIndex, SearchContent, SearchPlane, SearchScore}, + slot::{ + AtomicBitVec, AtomicSlotVec, Keyed, SlotAccess, SlotKey, SlotRead, SlotTable, SlotVecWrite, + SlotWrite, + }, +}; +use alloc::boxed::Box; + +/// Identity stamped by the FS serving host for one extraction connection. +/// +/// It is deliberately crate-private to construct: an untrusted request can +/// carry a lease number, but cannot choose the owner against which the +/// service validates it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ExtractionOwner(u64); + +impl ExtractionOwner { + pub(crate) const fn new(stamped_peer: u64) -> Self { + Self(stamped_peer) + } +} +pub use kore_dagfs::VersionLease; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ExtractionError { + Filesystem(FsError), + Provider(ProviderError), + NoSnapshotSlots, + NoResultSlots, + InvalidLease, + ObsoleteVersion, + InvalidResult(ValidationError), + /// A valid result whose wire batch is larger than the one + /// [`WORKER_REPORT_BYTES`] batch the store keeps per version. + Oversize, + Range, +} + +impl From for ExtractionError { + fn from(value: FsError) -> Self { + Self::Filesystem(value) + } +} + +impl From for ExtractionError { + fn from(value: ProviderError) -> Self { + Self::Provider(value) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DerivedResultInfo { + pub input: InputIdentity, + pub parser: ParserId, + pub parser_version: u32, +} + +#[derive(Clone, Copy)] +pub(crate) struct Snapshot { + pub(crate) owner: ExtractionOwner, + pub(crate) session: SessionId, + pub(crate) handle: Handle, + pub(crate) object: ObjectId, + pub(crate) identity: InputIdentity, + /// The placement the lease reads: the object's own live placement, + /// **borrowed**, never a copy. The engine keeps it frozen by moving + /// the object to a fresh placement before any mutation while a lease + /// borrows it ([`Engine::ensure_capacity`](crate::engine::Engine)), + /// and a lease that outlives its object becomes the placement's + /// owner; the last borrower releases it. + pub(crate) storage: Option, +} + +struct DerivedResult { + object: ObjectId, + input: InputIdentity, + parser: ParserId, + parser_version: u32, + /// Exact bounded #476 wire batch. Keeping the normalized object decoded in + /// every arena cell multiplies its largest enum/collection variants and + /// exhausts the founded stack after only a handful of files. + encoded: Vec, + hot_len: u16, + /// Provider-backed copy of the validated wire batch. Present only for a + /// provider whose placements survive restart; the in-memory result is the + /// rebuildable hot representation used by synchronous queries. + storage: Option, + encoded_len: u32, + encoded_digest: [u8; 32], +} + +struct WireChunk { + len: u16, + bytes: [u8; DERIVED_WIRE_CHUNK_BYTES], +} + +/// Durable placement metadata exported to the persistence layer. `K` remains +/// the provider's non-authority placement identifier. +#[derive(Clone, Copy)] +pub(crate) struct DurableDerived { + pub(crate) object: ObjectId, + pub(crate) input: InputIdentity, + pub(crate) key: K, + pub(crate) len: u32, + pub(crate) digest: [u8; 32], +} + +pub(crate) struct ExtractionState { + snapshots: SlotTable, MAX_LENT_VERSIONS>, + /// Derived-index results, sized at construction to the namespace bound + /// ([`MAX_DERIVED_RESULTS`] = [`MAX_OBJECTS`]) and drawn from the service + /// heap, so a fully-indexed namespace fits — the fixed 64-slot arena sat + /// below the object ceiling and refused mid-reindex on a full volume. + results: AtomicSlotVec, ServiceAlloc>, + wire_chunks: AtomicSlotVec, + /// Result-arena cells owned by each object slot, as a + /// `MAX_OBJECTS` × [`MAX_DERIVED_RESULTS`] occupancy bitmap (row = object + /// slot). Query lookup touches only the visible candidate's row; hidden + /// result occupancy and contention cannot change another view's work or + /// refusal behavior. Replaces a per-object `AtomicU64` whose 64-bit width + /// capped the result arena below the object count. + result_slots: AtomicBitVec, + /// Rebuildable per-object index generations. A hidden object's publication + /// cannot perturb another authority partition's observation identity. + result_generations: Box<[AtomicU64]>, + /// Result mutation is scoped to one object-arena cell. A replacement + /// generation may briefly contend with reclamation of its predecessor, + /// but every record comparison still uses the full generational + /// [`ObjectId`], so neither generation can select or purge the other. + /// Heap-backed with the arenas above: both are one entry per object, so + /// they scale with the volume and must not sit in the `serve` future's + /// stack frame. + result_claims: Box<[AtomicBool]>, +} + +struct ResultClaim<'a>(&'a AtomicBool); + +impl Drop for ResultClaim<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } +} + +impl ExtractionState { + pub(crate) fn new() -> Self { + // The derived index scales with the namespace, so its arenas are + // heap-backed rather than inline `.bss` — this both lifts the old + // 64-slot ceiling to the object count and keeps the by-value engine + // small enough to move on the founding stack. The allocations are + // bootstrap-scale and fixed (not volume-scaled), so an allocator that + // cannot satisfy them at mount is an undersized service heap — a build + // configuration error, the same class as the compile-time size guard — + // not a runtime volume condition. The genuinely volume-scaled durable + // *data* path remains fallible via the provider. + let alloc = crate::service_alloc(); + Self { + snapshots: SlotTable::new(), + results: AtomicSlotVec::try_new_in(MAX_DERIVED_RESULTS, alloc).expect( + "fs derived-result index arena (bootstrap-scale) — service heap undersized", + ), + wire_chunks: AtomicSlotVec::try_new_in(MAX_DERIVED_WIRE_CHUNKS, alloc) + .expect("fs derived wire-chunk arena (bootstrap-scale) — service heap undersized"), + result_slots: AtomicBitVec::try_new_in(MAX_OBJECTS * MAX_DERIVED_RESULTS, alloc) + .expect("fs result-occupancy bitmap (bootstrap-scale) — service heap undersized"), + result_generations: { + let mut v = alloc::vec::Vec::new(); + v.try_reserve_exact(MAX_OBJECTS).expect( + "fs derived-result generations (bootstrap-scale) — service heap undersized", + ); + for _ in 0..MAX_OBJECTS { + v.push(AtomicU64::new(1)); + } + v.into_boxed_slice() + }, + result_claims: { + let mut v = alloc::vec::Vec::new(); + v.try_reserve_exact(MAX_OBJECTS) + .expect("fs derived-result claims (bootstrap-scale) — service heap undersized"); + for _ in 0..MAX_OBJECTS { + v.push(AtomicBool::new(false)); + } + v.into_boxed_slice() + }, + } + } + + pub(crate) fn can_lend(&self) -> bool { + !self.snapshots.is_full() + } + + /// How many live leases borrow placement `key`, not counting `except`. + /// A key with a borrower must not be mutated in place or released. + pub(crate) fn storage_borrowers( + &self, + key: K, + except: Option, + ) -> Result + where + K: PartialEq + Copy, + { + let mut count = 0; + for index in 0..self.snapshots.capacity() { + let Some(id) = self.snapshots.live_id_at(index) else { + continue; + }; + if except == Some(VersionLease::from_key(id)) { + continue; + } + let snapshot = self.snapshots.try_read(id).map_err(|_| FsError::Busy)?; + if snapshot.storage == Some(key) { + count += 1; + } + } + Ok(count) + } + + pub(crate) fn install_snapshot( + &self, + snapshot: Snapshot, + ) -> Result> { + self.snapshots.insert(snapshot).map(VersionLease::from_key) + } + + pub(crate) fn snapshot( + &self, + owner: ExtractionOwner, + lease: VersionLease, + ) -> Result, MAX_LENT_VERSIONS>, ExtractionError> { + let snapshot = self + .snapshots + .try_read(lease.key()) + .map_err(|error| match error { + SlotAccess::Stale => ExtractionError::InvalidLease, + SlotAccess::Busy => ExtractionError::Filesystem(FsError::Busy), + })?; + if snapshot.owner != owner { + return Err(ExtractionError::InvalidLease); + } + Ok(snapshot) + } + + pub(crate) fn claim_snapshot( + &self, + owner: ExtractionOwner, + lease: VersionLease, + ) -> Result, MAX_LENT_VERSIONS>, ExtractionError> { + let snapshot = self + .snapshots + .try_write(lease.key()) + .map_err(|error| match error { + SlotAccess::Stale => ExtractionError::InvalidLease, + SlotAccess::Busy => ExtractionError::Filesystem(FsError::Busy), + })?; + if snapshot.owner != owner { + return Err(ExtractionError::InvalidLease); + } + Ok(snapshot) + } + + pub(crate) fn remove_claimed_snapshot( + &self, + snapshot: SlotWrite<'_, Snapshot, MAX_LENT_VERSIONS>, + ) -> Result, ExtractionError> { + snapshot.remove().map_err(|error| match error { + SlotAccess::Stale => ExtractionError::InvalidLease, + SlotAccess::Busy => ExtractionError::Filesystem(FsError::Busy), + }) + } + + fn claim_results(&self, object: ObjectId) -> Result, FsError> { + let index = object.key().index() as usize; + let claim = self.result_claims.get(index).ok_or(FsError::AccessLost)?; + claim + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| FsError::Busy)?; + Ok(ResultClaim(claim)) + } + + /// Base bit index of one object's occupancy row in `result_slots`. The map + /// is a `MAX_OBJECTS` × [`MAX_DERIVED_RESULTS`] bit matrix; a result at + /// arena slot `s` owned by this object sets bit `base + s`. Replaces the + /// per-object `AtomicU64` word — the width is now the result arena, not 64. + fn result_row_base(&self, object: ObjectId) -> Result { + let object = object.key().index() as usize; + if object >= MAX_OBJECTS { + return Err(FsError::AccessLost); + } + Ok(object * MAX_DERIVED_RESULTS) + } + + fn result_generation(&self, object: ObjectId) -> Result<&AtomicU64, FsError> { + self.result_generations + .get(object.key().index() as usize) + .ok_or(FsError::AccessLost) + } + + fn object_result_keys( + &self, + object: ObjectId, + ) -> Result, FsError> { + let base = self.result_row_base(object)?; + let mut keys = Vec::new(); + for slot in 0..self.results.capacity() { + if !self.result_slots.contains(base + slot) { + continue; + } + keys.push(self.results.live_id_at(slot).ok_or(FsError::Busy)?) + .map_err(|_| FsError::OutOfEdges)?; + } + Ok(keys) + } + + fn matching_result( + &self, + object: ObjectId, + parser: ParserId, + ) -> Result, FsError> { + let mut found = None; + for key in self.object_result_keys(object)? { + let stored = self.results.try_read(key).map_err(|error| match error { + SlotAccess::Stale | SlotAccess::Busy => FsError::Busy, + })?; + if stored.object == object && stored.parser == parser { + if found.replace(key).is_some() { + // The per-object claim makes a duplicate unreachable in + // the current implementation. Treat a legacy/corrupt + // duplicate as contention rather than selecting one. + return Err(FsError::Busy); + } + } + } + Ok(found) + } + + fn pack_result( + &self, + result: &NormalizedResult, + ) -> Result<(Vec, u16), ExtractionError> { + let mut wire = [0u8; WORKER_REPORT_BYTES]; + let used = encode_result(result, &mut wire).map_err(|_| ExtractionError::Oversize)?; + let hot_len = u16::try_from(used).map_err(|_| ExtractionError::Range)?; + let mut encoded = Vec::new(); + for part in wire[..used].chunks(DERIVED_WIRE_CHUNK_BYTES) { + let mut bytes = [0u8; DERIVED_WIRE_CHUNK_BYTES]; + bytes[..part.len()].copy_from_slice(part); + let chunk = WireChunk { + len: part.len() as u16, + bytes, + }; + let key = match self.wire_chunks.insert(chunk) { + Ok(key) => key, + Err(_) => { + let _ = self.release_packed(&encoded); + return Err(ExtractionError::NoResultSlots); + } + }; + if encoded.push(key).is_err() { + let _ = self.wire_chunks.remove(key); + let _ = self.release_packed(&encoded); + return Err(ExtractionError::Range); + } + } + Ok((encoded, hot_len)) + } + + fn release_packed(&self, encoded: &[SlotKey]) -> FsResult<()> { + for key in encoded { + self.wire_chunks.remove(*key).ok_or(FsError::Busy)?; + } + Ok(()) + } + + fn decode_packed(&self, encoded: &[SlotKey], hot_len: u16) -> FsResult { + let wanted = usize::from(hot_len); + if wanted == 0 || wanted > WORKER_REPORT_BYTES { + return Err(FsError::Conflict); + } + let mut wire = [0u8; WORKER_REPORT_BYTES]; + let mut at = 0usize; + for key in encoded { + let chunk = self.wire_chunks.try_read(*key).map_err(|_| FsError::Busy)?; + let len = usize::from(chunk.len); + let end = at.checked_add(len).ok_or(FsError::Conflict)?; + if len == 0 || len > DERIVED_WIRE_CHUNK_BYTES || end > wanted { + return Err(FsError::Conflict); + } + wire[at..end].copy_from_slice(&chunk.bytes[..len]); + at = end; + } + if at != wanted { + return Err(FsError::Conflict); + } + decode_result(&wire[..wanted]).map_err(|_| FsError::Conflict) + } + + pub(crate) fn commit_result( + &self, + owner: ExtractionOwner, + lease: VersionLease, + current_revision: Option, + result: NormalizedResult, + output_window_bytes: u64, + ) -> Result { + let snapshot = self.snapshot(owner, lease)?; + if current_revision != Some(snapshot.identity.version) { + return Err(ExtractionError::ObsoleteVersion); + } + Registry::validate(&result, snapshot.identity, output_window_bytes) + .map_err(ExtractionError::InvalidResult)?; + let (encoded, hot_len) = self.pack_result(&result)?; + + let info = DerivedResultInfo { + input: snapshot.identity, + parser: result.provenance.parser, + parser_version: result.provenance.parser_version, + }; + + let _claim = match self.claim_results(snapshot.object) { + Ok(claim) => claim, + Err(error) => { + let _ = self.release_packed(&encoded); + return Err(ExtractionError::Filesystem(error)); + } + }; + + // One selected result per object and parser lineage. A new parser + // version replaces that lineage, while results from other parsers + // remain independently selectable. + let mut record = DerivedResult { + object: snapshot.object, + input: result.provenance.input, + parser: result.provenance.parser, + parser_version: result.provenance.parser_version, + encoded, + hot_len, + storage: None, + encoded_len: 0, + encoded_digest: [0; 32], + }; + let replaced = match self.matching_result(snapshot.object, record.parser) { + Ok(replaced) => replaced, + Err(error) => { + let _ = self.release_packed(&record.encoded); + return Err(ExtractionError::Filesystem(error)); + } + }; + if let Some(key) = replaced { + let mut stored = match self.results.try_write(key) { + Ok(stored) => stored, + Err(SlotAccess::Stale | SlotAccess::Busy) => { + let _ = self.release_packed(&record.encoded); + return Err(ExtractionError::Filesystem(FsError::Busy)); + } + }; + // Keep custody of the previous durable placement until the new + // encoded batch is attached. The engine releases it only after + // that publication succeeds. + record.storage = stored.storage; + record.encoded_len = stored.encoded_len; + let predecessor = core::mem::replace(&mut *stored, record); + drop(stored); + self.release_packed(&predecessor.encoded) + .map_err(ExtractionError::Filesystem)?; + } else { + let key = self.results.insert(record).map_err(|record| { + let _ = self.release_packed(&record.encoded); + ExtractionError::NoResultSlots + })?; + let base = self + .result_row_base(snapshot.object) + .map_err(ExtractionError::Filesystem)?; + self.result_slots.set(base + key.index() as usize); + } + self.result_generation(snapshot.object) + .map_err(ExtractionError::Filesystem)? + .fetch_add(1, Ordering::AcqRel); + Ok(info) + } + + /// Attach the durable wire placement after the ordinary validated commit. + /// This is a separate, object-scoped publication step so RAM users retain + /// the exact same path and a failed durable write cannot publish metadata + /// that names missing bytes. + pub(crate) fn attach_storage( + &self, + object: ObjectId, + parser: ParserId, + storage: K, + len: u32, + digest: [u8; 32], + ) -> Result, FsError> { + let _claim = self.claim_results(object)?; + let key = self + .matching_result(object, parser)? + .ok_or(FsError::IndexIncomplete)?; + let mut stored = self.results.try_write(key).map_err(|_| FsError::Busy)?; + let previous = stored.storage.replace(storage); + stored.encoded_len = len; + stored.encoded_digest = digest; + Ok(previous) + } + + /// Restore a checksummed durable batch after its provider extent has been + /// adopted and decoded. Recovery never fabricates a lease; it revalidates + /// provenance before calling this method and installs only derived state. + pub(crate) fn restore_result( + &self, + object: ObjectId, + result: NormalizedResult, + storage: K, + len: u32, + digest: [u8; 32], + ) -> FsResult<()> { + let _claim = self.claim_results(object)?; + if self + .matching_result(object, result.provenance.parser)? + .is_some() + { + return Err(FsError::Conflict); + } + let (encoded, hot_len) = self + .pack_result(&result) + .map_err(|_| FsError::OutOfStorage)?; + let key = self + .results + .insert(DerivedResult { + object, + input: result.provenance.input, + parser: result.provenance.parser, + parser_version: result.provenance.parser_version, + encoded, + hot_len, + storage: Some(storage), + encoded_len: len, + encoded_digest: digest, + }) + .map_err(|record| { + let _ = self.release_packed(&record.encoded); + FsError::OutOfStorage + })?; + let base = self.result_row_base(object)?; + self.result_slots.set(base + key.index() as usize); + self.result_generation(object)? + .fetch_add(1, Ordering::AcqRel); + Ok(()) + } + + pub(crate) fn durable_results(&self) -> FsResult, N>> { + let mut out = Vec::new(); + for index in 0..self.results.capacity() { + let Some(key) = self.results.live_id_at(index) else { + continue; + }; + let stored = self.results.try_read(key).map_err(|_| FsError::Busy)?; + let Some(storage) = stored.storage else { + continue; + }; + out.push(DurableDerived { + object: stored.object, + input: stored.input, + key: storage, + len: stored.encoded_len, + digest: stored.encoded_digest, + }) + .map_err(|_| FsError::OutOfStorage)?; + } + Ok(out) + } + + pub(crate) fn result( + &self, + object: ObjectId, + current_revision: Revision, + parser: ParserId, + ) -> Result, FsError> { + let Some(key) = self.matching_result(object, parser)? else { + return Ok(None); + }; + let stored = self.results.try_read(key).map_err(|error| match error { + SlotAccess::Stale | SlotAccess::Busy => FsError::Busy, + })?; + if stored.input.version != current_revision { + return Ok(None); + } + self.decode_packed(&stored.encoded, stored.hot_len) + .map(Some) + } + + /// Visit parser-neutral results bound to exactly the candidate's current + /// revision. Concurrent replacement is reported as bounded backpressure; + /// search never observes a torn derived record. + pub(crate) fn visit_current( + &self, + object: ObjectId, + current_revision: Revision, + mut visit: impl FnMut(&NormalizedResult), + ) -> Result<(), FsError> { + for key in self.object_result_keys(object)? { + let stored = self.results.try_read(key).map_err(|_| FsError::Busy)?; + if stored.object != object { + return Err(FsError::Busy); + } + if stored.input.version == current_revision { + let result = self.decode_packed(&stored.encoded, stored.hot_len)?; + if result.provenance.input != stored.input + || result.provenance.parser != stored.parser + || result.provenance.parser_version != stored.parser_version + { + return Err(FsError::Conflict); + } + visit(&result); + } + } + Ok(()) + } + + pub(crate) fn purge_object( + &self, + object: ObjectId, + ) -> Result, FsError> { + let _claim = self.claim_results(object)?; + let matches = self.object_result_keys(object)?; + for key in &matches { + let stored = self.results.try_read(*key).map_err(|error| match error { + SlotAccess::Stale | SlotAccess::Busy => FsError::Busy, + })?; + if stored.object != object { + return Err(FsError::Busy); + } + } + + // Claim the complete footprint before removing the first record. + let mut claimed = + Vec::, ServiceAlloc>, MAX_DERIVED_RESULTS>::new(); + for key in matches { + claimed + .push(self.results.try_write(key).map_err(|error| match error { + SlotAccess::Stale | SlotAccess::Busy => FsError::Busy, + })?) + .map_err(|_| FsError::OutOfEdges)?; + } + let mut storage = Vec::new(); + for result in claimed { + if let Some(placement) = result.storage { + storage.push(placement).map_err(|_| FsError::OutOfStorage)?; + } + let removed = result.remove().map_err(|_| FsError::Busy)?; + self.release_packed(&removed.encoded)?; + } + let base = self.result_row_base(object)?; + for slot in 0..self.results.capacity() { + self.result_slots.clear(base + slot); + } + // Reclamation is the only caller. Reset rather than carry a retired + // object's update history into a future occupant of the same slot. + self.result_generation(object)?.store(1, Ordering::Release); + Ok(storage) + } + + #[cfg(test)] + pub(crate) fn result_count(&self) -> usize { + self.results.len() + } +} + +impl Default for ExtractionState { + fn default() -> Self { + Self::new() + } +} + +fn contains_folded(haystack: &[u8], needle: &[u8]) -> bool { + needle.is_empty() + || haystack.windows(needle.len()).any(|candidate| { + candidate + .iter() + .zip(needle) + .all(|(left, right)| left.eq_ignore_ascii_case(right)) + }) +} + +fn snippet(bytes: &[u8]) -> heapless::String { + let mut out = heapless::String::new(); + let Ok(text) = core::str::from_utf8(bytes) else { + return out; + }; + for character in text.chars() { + if out.push(character).is_err() { + break; + } + } + out +} + +const fn token_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-' || !byte.is_ascii() +} + +fn for_each_token(bytes: &[u8], mut visit: impl FnMut(&[u8])) { + let mut at = 0; + while at < bytes.len() { + while at < bytes.len() && !token_byte(bytes[at]) { + at += 1; + } + let start = at; + while at < bytes.len() && token_byte(bytes[at]) { + at += 1; + } + if start != at { + visit(&bytes[start..at]); + } + } +} + +fn contains_folded_token(bytes: &[u8], wanted: &[u8]) -> bool { + let mut found = false; + for_each_token(bytes, |candidate| { + if candidate.len() == wanted.len() + && candidate + .iter() + .zip(wanted) + .all(|(left, right)| left.eq_ignore_ascii_case(right)) + { + found = true; + } + }); + found +} + +impl QueryIndex for ExtractionState { + fn evaluate( + &self, + object: ObjectId, + revision: Revision, + query: SearchContent, + ) -> Result { + let generation = self.result_generation(object)?.load(Ordering::Acquire); + let needle = query.term.as_bytes(); + let mut matched = false; + let mut score = SearchScore::NONE; + let mut visible_snippet = heapless::String::new(); + let mut work = 0u32; + let mut current_batches = 0u32; + let mut semantic_terms = 0u32; + let mut semantic_found = 0u32; + + self.visit_current(object, revision, |result| { + current_batches = current_batches.saturating_add(1); + match query.plane { + SearchPlane::FactKey => { + for fact in &result.facts { + work = work.saturating_add(1); + if contains_folded(fact.key.as_bytes(), needle) { + matched = true; + score = SearchScore::EXACT; + if visible_snippet.is_empty() { + visible_snippet = snippet(fact.key.as_bytes()); + } + } + } + } + SearchPlane::FullText => { + for span in &result.text { + work = work.saturating_add(1); + if contains_folded(span.text.as_bytes(), needle) { + matched = true; + score = SearchScore::EXACT; + if visible_snippet.is_empty() { + visible_snippet = snippet(span.text.as_bytes()); + } + } + } + for fact in &result.facts { + let FactValue::Utf8(value) = &fact.value else { + continue; + }; + work = work.saturating_add(1); + if contains_folded(value.as_bytes(), needle) { + matched = true; + score = SearchScore::EXACT; + if visible_snippet.is_empty() { + visible_snippet = snippet(value.as_bytes()); + } + } + } + for symbol in &result.symbols { + work = work.saturating_add(1); + if contains_folded(symbol.name.as_bytes(), needle) { + matched = true; + score = SearchScore::EXACT; + if visible_snippet.is_empty() { + visible_snippet = snippet(symbol.name.as_bytes()); + } + } + } + } + SearchPlane::Relationship => { + for reference in &result.references { + work = work.saturating_add(1); + if contains_folded(reference.target.as_bytes(), needle) { + matched = true; + score = SearchScore::EXACT; + if visible_snippet.is_empty() { + visible_snippet = snippet(reference.target.as_bytes()); + } + } + } + } + SearchPlane::Semantic => { + // This plane is deliberately lexical-semantic until a + // model-backed QueryIndex implementation is installed. + // Compare tokens exactly instead of a 64-bit bloom sketch: + // collisions must never manufacture a filesystem hit. + for_each_token(needle, |wanted| { + semantic_terms = semantic_terms.saturating_add(1); + let mut term_found = false; + for span in &result.text { + work = work.saturating_add(1); + term_found |= contains_folded_token(span.text.as_bytes(), wanted); + if visible_snippet.is_empty() { + visible_snippet = snippet(span.text.as_bytes()); + } + } + for fact in &result.facts { + let FactValue::Utf8(value) = &fact.value else { + continue; + }; + work = work.saturating_add(1); + term_found |= contains_folded_token(value.as_bytes(), wanted); + if visible_snippet.is_empty() { + visible_snippet = snippet(value.as_bytes()); + } + } + for reference in &result.references { + work = work.saturating_add(1); + term_found |= + contains_folded_token(reference.target.as_bytes(), wanted); + } + for symbol in &result.symbols { + work = work.saturating_add(1); + term_found |= contains_folded_token(symbol.name.as_bytes(), wanted); + } + if term_found { + semantic_found = semantic_found.saturating_add(1); + } + }); + } + } + })?; + + if current_batches == 0 { + return Err(FsError::IndexIncomplete); + } + + if query.plane == SearchPlane::Semantic { + matched = semantic_found != 0; + score = if semantic_terms == 0 { + SearchScore::NONE + } else { + SearchScore::new(((semantic_found * 10_000) / semantic_terms) as u16) + }; + if !matched { + visible_snippet.clear(); + } + } + + if self.result_generation(object)?.load(Ordering::Acquire) != generation { + return Err(FsError::Conflict); + } + Ok(IndexMatch { + matched, + score, + snippet: visible_snippet, + work, + generation, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use kore_svc_extract::schema::{NormalizedResult, ParserProvenance}; + + fn object(index: u32, generation: u32) -> ObjectId { + ObjectId::from_bits(SlotKey::new(index, generation).to_bits()) + } + + fn snapshot(owner: ExtractionOwner, object: ObjectId) -> Snapshot { + Snapshot { + owner, + session: SessionId::new(0), + handle: Handle::from_bits(SlotKey::new(0, 1).to_bits()), + object, + identity: InputIdentity::of(object.to_bits(), 1, b"immutable"), + storage: Some(7), + } + } + + fn derived( + state: &ExtractionState, + object: ObjectId, + parser: ParserId, + bytes: &[u8], + ) -> DerivedResult { + let input = InputIdentity::of(object.to_bits(), 1, bytes); + let result = NormalizedResult::empty(ParserProvenance::deterministic(input, parser, 1)); + let (encoded, hot_len) = state.pack_result(&result).unwrap(); + DerivedResult { + object, + input, + parser, + parser_version: 1, + encoded, + hot_len, + storage: None, + encoded_len: 0, + encoded_digest: [0; 32], + } + } + + #[test] + fn snapshot_release_waits_for_every_guarded_reader() { + let state = ExtractionState::::new(); + let owner = ExtractionOwner::new(9); + let lease = state + .install_snapshot(snapshot(owner, object(1, 1))) + .ok() + .expect("snapshot slot"); + + let reader = state.snapshot(owner, lease).unwrap(); + assert!(matches!( + state.claim_snapshot(owner, lease), + Err(ExtractionError::Filesystem(FsError::Busy)) + )); + drop(reader); + + let writer = state.claim_snapshot(owner, lease).unwrap(); + assert!(matches!( + state.snapshot(owner, lease), + Err(ExtractionError::Filesystem(FsError::Busy)) + )); + let removed = state.remove_claimed_snapshot(writer).unwrap(); + assert_eq!(removed.storage, Some(7)); + assert!(matches!( + state.snapshot(owner, lease), + Err(ExtractionError::InvalidLease) + )); + } + + #[test] + fn contended_result_purge_is_atomic_and_retryable() { + let state = ExtractionState::::new(); + let object = object(2, 1); + let parser = ParserId(4); + let key = state + .results + .insert(derived(&state, object, parser, b"source")) + .ok() + .expect("result slot"); + state + .result_slots + .set(object.key().index() as usize * MAX_DERIVED_RESULTS + key.index() as usize); + + let reader = state.results.try_read(key).unwrap(); + assert_eq!(state.purge_object(object), Err(FsError::Busy)); + assert_eq!(state.result_count(), 1); + drop(reader); + + assert_eq!(state.purge_object(object), Ok(Vec::new())); + assert_eq!(state.result_count(), 0); + } + + #[test] + fn hidden_result_contention_cannot_change_a_visible_query() { + let state = ExtractionState::::new(); + let visible = object(3, 1); + let hidden = object(4, 1); + let parser = ParserId(7); + + let visible_key = state + .results + .insert(derived(&state, visible, parser, b"visible")) + .ok() + .expect("visible result slot"); + state.result_slots.set( + visible.key().index() as usize * MAX_DERIVED_RESULTS + visible_key.index() as usize, + ); + + let hidden_key = state + .results + .insert(derived(&state, hidden, parser, b"hidden")) + .ok() + .expect("hidden result slot"); + state + .result_slots + .set(hidden.key().index() as usize * MAX_DERIVED_RESULTS + hidden_key.index() as usize); + + let hidden_writer = state.results.try_write(hidden_key).unwrap(); + let mut visited = 0; + assert_eq!( + state.visit_current(visible, 1, |_| visited += 1), + Ok(()), + "a hidden shard's writer must not backpressure visible search" + ); + assert_eq!(visited, 1); + drop(hidden_writer); + } + + #[test] + fn hot_arena_covers_one_result_per_namespace_object() { + let state = ExtractionState::::new(); + for index in 0..MAX_DERIVED_RESULTS as u32 { + let object = object(index, 1); + let key = state + .results + .insert(derived(&state, object, ParserId(7), b"small text")) + .ok() + .expect("one result slot per writable file"); + state + .result_slots + .set(index as usize * MAX_DERIVED_RESULTS + key.index() as usize); + } + assert_eq!(state.result_count(), MAX_DERIVED_RESULTS); + assert!(state.results.is_full()); + } +} diff --git a/native/services/dagfs/src/lib.rs b/native/services/dagfs/src/lib.rs new file mode 100644 index 0000000..0373d1c --- /dev/null +++ b/native/services/dagfs/src/lib.rs @@ -0,0 +1,237 @@ +//! # kore-svc-dagfs — the filesystem service +//! +//! A **capability-confined userland filesystem over a named containment +//! graph**. The kernel learns +//! no filenames, directories, paths, or filesystem policy; it supplies +//! isolation, capability transfer, shared memory, and scheduling, and this +//! service owns everything else. +//! +//! ## The two edge kinds +//! +//! The graph has exactly two kinds of edge, and keeping them distinct is the +//! design: +//! +//! - a **containment edge** places a child object beneath a parent directory +//! under a name ([`namespace`]) — these and only these form the namespace +//! DAG; and +//! - an **authority edge** lets a session enter a file or subtree with +//! bounded rights ([`authority`]) — a private root, or a direct grant. +//! +//! More than one containment edge may name the same file or directory object. +//! Diamonds are lawful; removing one route leaves every other route alone, +//! while bounded reachability checks refuse self-links and back-edges. +//! +//! ## Handles are not authority +//! +//! An object ID or open handle conveys nothing by itself. Every open handle +//! is bound to the session that opened it, the authority edge it entered +//! through, the containment chain it followed, the rights attenuated along +//! that route, and the *generations* of every edge involved — an +//! [`authority::EdgeProof`]. Each operation revalidates that proof, so +//! retiring any edge on the route makes the next operation fail with +//! [`FsError::AccessLost`] even though the client still holds the old +//! numeric handle. A client holding a second live route may reopen the +//! object through it and proceed. There is deliberately no cache anywhere +//! that could bypass this check. +//! +//! ## Layering +//! +//! ```text +//! engine the request engine — generic over the provider +//! ├ authority sessions, private roots, grants, rights, handle proofs +//! ├ namespace containment edges, names, traversal, link/move/unlink +//! ├ object stable object identity, kinds, revisions, reclamation +//! └ provider the async storage seam; `provider::ram` is the first impl +//! ``` +//! +//! [`slot`] supplies the one arena shape all of the above use, and +//! [`bounds`] is the entire memory budget as compile-time constants. +//! +//! Above the service, [`client`] is the typed caller and [`path`] is the +//! `/`-separated path syntax clients walk with — neither is reachable from +//! the engine, and the service never sees a path. +//! +//! ## Phase +//! +//! This crate now includes the Phase-1 engine, compact IPC protocol, founded +//! multi-client service, typed async client, and capability-backed bulk data +//! windows used by ordinary founded processes. The RAM and direct-block +//! providers implement one conformance contract; [`persistence`] adds +//! copy-on-write payloads, atomic root publication, and recovery without +//! serializing runtime authority. Exec provisions each filesystem-capable +//! image a private recyclable pair rather than routing commands through a +//! shell-specific protocol. + +#![no_std] +// The volume-scaled derived and block-extent indexes are heap-backed via the +// `Allocator` seam (`service_alloc` → the service's own global heap), which is +// the unstable `allocator_api`. Their capacities remain fixed after startup. +#![feature(allocator_api)] + +extern crate alloc; + +/// The one place filesystem-owned, volume-scaled metadata names its allocator. +/// The service binary installs a private global heap; host tests receive the +/// host allocator through the same seam. +/// +/// Named concretely rather than as `dyn Allocator` because an arena's +/// `Sync`ness is inherited from its allocator, and `Namespace` **must** stay +/// `Sync`: the per-object and per-directory claim maps are shared across +/// threads by the tests that witness "no filesystem-global lock" +/// (`DAGFS.md` §2), and `dyn Allocator` is not `Sync`. The seam is still one +/// name in one place; it is simply a type alias instead of an erased trait +/// object, which is strictly more information and costs no generality — both +/// builds resolve to the same allocator, differing only in which global heap +/// `#[global_allocator]` installs behind it. +pub(crate) type ServiceAlloc = alloc::alloc::Global; + +/// The allocator every volume-scaled filesystem arena is built in. +pub(crate) fn service_alloc() -> &'static ServiceAlloc { + static GLOBAL: ServiceAlloc = alloc::alloc::Global; + &GLOBAL +} + +/// Engines one filesystem row mounts at most: a provisioned disk's user and +/// system stores, each with its own authority and service frontend. +pub const MOUNTED_ENGINES: usize = 2; + +/// What the volume-scaled arenas need of the service heap: two mounted +/// engines' fixed reservations — about 1.75 MiB each at +/// [`bounds::MAX_OBJECTS`] = 512, the derived wire-chunk arena alone 1 MiB — +/// with better than 2× headroom for recovery transients and the heap's +/// growth granularity. Measured rather than derived: 4 MiB fit the one engine +/// a blank disk mounts and ran out on the second (#1003), and the fs-dual +/// gate is the witness that keeps it honest. It was the row's whole supply +/// until sessions were allocated per slot. +pub const VOLUME_HEAP_BYTES: usize = 8 << 20; + +/// **What the filesystem's heap must hold** — the floor its manifest row's +/// `heap_supply_bytes` is held to (exec's founder test +/// `dagfs_heap_supply_holds_every_session`): the volume arenas +/// ([`VOLUME_HEAP_BYTES`]); every session slot attached at once in both +/// engines ([`authority::SESSION_HEAP_BYTES`]), with each frontend's slot +/// table grown to all of them; and the serve host's peer table grown to +/// every slot a face can occupy. The two tables count twice: a doubling table +/// holds its next block before it frees the one it outgrew, and every block +/// it ever held sums to under twice its last. Derived from the per-slot costs +/// themselves, so a heavier session or serving pair raises the floor instead +/// of overrunning it. +/// +/// It does not reserve the proofs open handles carry — 16 KiB at most per +/// session and engine, every handle at the deepest path +/// ([`bounds::MAX_HANDLES_PER_SESSION`] × [`bounds::MAX_PATH_DEPTH`] edges): +/// those are allocated per open at their own depth from what the heap has +/// left, and a heap that cannot hold one refuses that open +/// ([`FsError::OutOfHandles`]) without touching any session's admission. +pub const HEAP_DEMAND_BYTES: u64 = { + let serve_slot = { + let durable = kore_rt::svc::peer_slot_bytes::(); + let ram = kore_rt::svc::peer_slot_bytes::(); + if durable > ram { durable } else { ram } + }; + (VOLUME_HEAP_BYTES + + MOUNTED_ENGINES + * (authority::SESSION_HEAP_BYTES + + 2 * bounds::MAX_SESSIONS * service::SESSION_SLOT_BYTES) + + 2 * kore_rt::svc::PeerId::SLOT_CAPACITY * serve_slot) as u64 +}; + +#[cfg(test)] +extern crate std; + +pub mod authority; +pub mod bounds; +pub mod client; +pub mod digests; +pub mod durable; +pub mod engine; +pub mod error; +pub mod extraction; +pub mod namespace; +pub mod object; +mod parent_index; +pub mod path; +pub mod persistence; +pub mod provider; +mod reach; +mod router; +pub mod search; +pub mod service; +pub mod slot; +pub mod window; +pub mod wire; + +pub use authority::{AuthorityKind, AuthorityQuota, EdgeProof, GrantId, Handle, Rights, SessionId}; +pub use client::process_face; +pub use client::{ + DirEntry, FS_GRANT_TARGET_DELEGATE_RIGHTS, FS_GRANT_TARGET_RIGHTS, FS_VIEW_ANCHOR_RIGHTS, + FilesystemFace, FsClient, + FsViewCapability, GrantEntry, IdentifyVersionFailure, LentVersion, Opened, PendingVersion, + ReleasePendingVersionFailure, ReleaseVersionFailure, RingFs, RingFsError, + RingReleasePendingVersionFailure, RingReleaseVersionFailure, SearchResultPage, Stat, Transport, + VERSION_IDENTITY_BYTES, read_dir_entry, read_grant_entry, read_parent, read_search_hit, + read_system_status, + read_version_identity, +}; +pub use durable::DurableService; +#[cfg(all(target_arch = "x86_64", target_os = "none"))] +pub use durable::run; +pub use engine::{ + Engine, GrantInfo, ListPage, OpenedObject, ParentInfo, ParentsPage, RamEngine, StatInfo, +}; +pub use error::{FsError, FsResult, ProviderError}; +pub use extraction::{DerivedResultInfo, ExtractionError, VersionLease}; +pub use kore_dagfs::DagfsProtocol; +/// The storage-ring client (`res/STORAGE_TRANSPORT.md` §4), for rows that +/// read through the filesystem's transport — exec's image loads. +pub use kore_dagfs::stor; +pub use namespace::{EdgeId, ListEntry, Name}; +pub use object::{ANY_REVISION, ObjectId, ObjectKind, ObjectSubtype, Revision, StableObjectId}; +pub use provider::{ + BlockKey, BlockProvider, BlockProviderStats, DurableExtent, RamProvider, StorageLimits, + StorageProvider, +}; +pub use search::{ + FreshnessPolicy, IndexMatch, QueryIndex, SearchContent, SearchCursor, SearchHit, SearchPage, + SearchPlane, SearchQuery, SearchScore, SearchTerm, +}; +pub use service::{ + FACE_HOME_LABEL_BYTES, FsEndowment, FsRequest, FsResponse, FsService, NativeView, RamService, + SYSTEM_APPS_DIR, SYSTEM_APPS_GRANT, SYSTEM_APPS_VIEW, SYSTEM_LIBS_DIR, SYSTEM_LIBS_GRANT, + SYSTEM_LIBS_VIEW, SYSTEM_PKGS_DIR, SYSTEM_PKGS_GRANT, SYSTEM_SHARE_DIR, SYSTEM_SHARE_GRANT, + SYSTEM_SHARE_VIEW, SYSTEM_SOURCE_GRANT, SYSTEM_WORKSPACE_DIR, WindowAttachment, + face_home_label, +}; + +/// Architecture-neutral service entry selected at this crate boundary. +/// +/// x86 composes the persistent block-backed provider. Targets without that +/// binding serve the identical filesystem authority and wire contract from +/// the bounded RAM provider; callers and crt0 never branch on storage. +#[cfg(not(all(target_arch = "x86_64", target_os = "none")))] +pub async fn run(rt: kore_rt::entry::Rt) -> ! { + kore_rt::entry::serve::(rt).await +} + +/// The provider-neutral filesystem wire contract cited by the boot manifest. +/// +/// RAM and durable entries implement this exact protocol; naming the alias in +/// the manifest keeps provider selection at the entry seam rather than making +/// the founder pretend that x86 always runs the RAM composition. +pub type FsProtocol = RamService; + +/// **Who stops the filesystem's device** (#809 §L.5). The row selects no +/// device at founding (`DEVICES` is empty): its NVMe function arrives at +/// runtime from the block row, which `Delegated` it here. So the binding is +/// cited as a runtime-claimed class — the one citation that puts this row in +/// the terminal sweep. Declared once and cited by both `SystemService` +/// impls: [`FsProtocol`] is what the manifest reads, `DurableService` is +/// what holds the binding on metal. +pub const FS_DEVICE_CLASSES: &[kore_rt::service::DeviceClass] = + &[kore_driver::host::device_class::()]; +pub use slot::{SlotKey, SlotTable}; +pub use window::DataWindow; + +/// The client-side future driver, shared with every host suite (moved to the +/// leaf with the client it drives; see [`kore_dagfs::block_on`]). +pub use kore_dagfs::block_on; diff --git a/native/services/dagfs/src/namespace.rs b/native/services/dagfs/src/namespace.rs new file mode 100644 index 0000000..b5ac08d --- /dev/null +++ b/native/services/dagfs/src/namespace.rs @@ -0,0 +1,2766 @@ +//! The containment DAG: names, edges, traversal, and the link/move/unlink +//! rules. +//! +//! # Where an edge lives +//! +//! A containment edge is a **slot in its parent directory's entry table**, +//! not a row in a global edge arena. That is a deliberate choice with two +//! consequences the authority model depends on: +//! +//! - **One source of truth.** An edge's name, its child, and its generation +//! are one record in one place. A global edge table plus per-directory +//! child indexes would be two structures that could disagree. +//! - **Per-edge staleness for free.** [`SlotTable`] already bumps a slot's +//! generation on release, so [`EdgeId`] — parent plus entry key — goes +//! stale exactly when that one edge is removed, while every sibling and +//! every other route stays live. That is the bounded edge proof's central +//! route-identity guarantee. +//! +//! # No paths here +//! +//! The engine's vocabulary is single-component: `lookup(parent, name)`. +//! Paths are client-side traversal input, so this module never +//! parses a `/`-joined string, has no notion of a current directory, and has +//! no ambient root. A client walks components itself, and each step extends +//! its handle's proof by one edge. +//! +//! # Directory shape +//! +//! Files and directories may both have multiple incoming containment edges. +//! Every directory-edge insertion performs a bounded reachability proof before +//! publication, so diamonds are lawful while self-links and back-edges fail +//! closed. No parent pointer is stored: incoming counts govern reclamation, +//! while exact edge generations govern route validity. + +use alloc::{boxed::Box, vec::Vec}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; + +use crate::{ + ServiceAlloc, + bounds::{MAX_DIR_ENTRIES, MAX_DIRECTORIES, MAX_LIST_PAGE, MAX_OBJECTS, MAX_PATH_DEPTH}, + error::{FsError, FsResult, ProviderError}, + object::{ + DirId, Object, ObjectId, ObjectIdExt, ObjectKind, ObjectSubtype, Revision, StableObjectId, + }, + parent_index::{ParentClaim, ParentIndex}, + reach::ReachSet, + slot::{AtomicSlotVec, BoundedVec, SlotKey, SlotTable, SlotVecRead, SlotVecWrite, fs_access}, +}; + +/// One object copied from a fully pinned durable graph snapshot. +pub(crate) struct DurableGraphObject { + pub(crate) stable: StableObjectId, + pub(crate) kind: ObjectKind, + pub(crate) subtype: ObjectSubtype, + pub(crate) len: u64, + pub(crate) revision: Revision, + pub(crate) storage: Option, +} + +/// One containment edge copied from a fully pinned durable graph snapshot. +pub(crate) struct DurableGraphEdge { + pub(crate) parent: StableObjectId, + pub(crate) name: Name, + pub(crate) child: StableObjectId, +} + +/// A complete, internally consistent durable namespace image. +/// +/// Both tables are arena-backed. Inline this type was **119,832 bytes**, its +/// edge table alone 114,688, and it is built on **every commit** rather than +/// only at mount — so it was the largest single consumer of the `serve` +/// future's stack, on the hottest path that has one. The edge bound is a +/// runtime capacity now rather than a const generic, because the caller +/// already knows it from the volume's profile. +pub(crate) struct DurableGraph { + pub(crate) root: StableObjectId, + pub(crate) objects: BoundedVec, ServiceAlloc>, + pub(crate) edges: BoundedVec, +} + +pub use kore_dagfs::Name; + +/// A containment edge's identity: which directory holds it, and which entry +/// slot (with generation) it is. +/// +/// Goes stale the moment that one edge is removed — including when a +/// [`Namespace::relocate`] removes the source edge, which is what makes a +/// handle opened through the source fail while a handle opened through a +/// parallel link keeps working. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct EdgeId { + parent: ObjectId, + entry: SlotKey, +} + +impl EdgeId { + /// The directory this edge hangs beneath. + #[inline] + pub const fn parent(self) -> ObjectId { + self.parent + } +} + +/// One directory entry — the edge's payload. +#[derive(Clone, Copy, Debug)] +struct DirEntry { + name: Name, + child: ObjectId, +} + +/// A directory's child list. +type DirEntries = SlotTable; + +// The object arena and the directory arena are the namespace's two +// **volume-scaled** stores, so they are heap-backed rather than inline. A +// service's whole state is polled inside one `serve` future on the founder's +// shared 4 MiB stack, which made the object bound and the stack budget the +// same number: PR #958 overflowed that stack by 744 bytes by growing one +// placement record 8 bytes. Inline arenas are therefore a capacity ceiling +// disguised as a footprint, and these aliases are where that coupling is cut. +// Matches the reasoning already recorded on `ExtractionState`'s arenas. +// Session-scaled state (handles, authority edges) stays inline on purpose. + +/// Heap-backed object arena. +type ObjectArena = AtomicSlotVec, ServiceAlloc>; +/// A read claim on one object slot. +type ObjectRead<'a, K> = SlotVecRead<'a, Object, ServiceAlloc>; +/// An exclusive claim on one object slot. +type ObjectWrite<'a, K> = SlotVecWrite<'a, Object, ServiceAlloc>; +/// Heap-backed arena of directory child lists. The arena holds a slot per +/// possible directory ([`MAX_DIRECTORIES`], derived from the object bound); +/// each child list is boxed when its directory is created and freed with +/// its slot, so a directory costs its table only while it lives. The box +/// keeps the stable address the slot guards rely on. +type DirTable = Box; +type DirArena = AtomicSlotVec; +/// A read claim on one directory's child list. +type DirRead<'a> = SlotVecRead<'a, DirTable, ServiceAlloc>; +/// An exclusive claim on one directory's child list. +type DirWrite<'a> = SlotVecWrite<'a, DirTable, ServiceAlloc>; + +/// A canonically ordered set of exact directory mutation claims. +/// +/// Multi-directory commits acquire every member before final validation and +/// publication, so refusal drops the whole bounded footprint without a +/// partial move, stale emptiness decision, or graph cycle. +struct DirectoryClaims<'a> { + /// Heap-backed and sized to the claim: a subtree claim may name every + /// directory, and an inline list would put that worst case on the + /// serve stack. + entries: BoundedVec, ServiceAlloc>, + /// Directory-arena slot index → position in `entries`, or [`UNCLAIMED`]. + /// The walks look a claim up once per popped node and once per edge; + /// an index makes each of those O(1) instead of a scan of the claim + /// list. It is filled in claim order, after every generation is pinned. + by_dir: [ClaimAt; MAX_DIRECTORIES], +} + +/// A claim's position in `entries`: wide enough for every directory slot. +type ClaimAt = u16; +/// The `by_dir` sentinel: no claim stands for that directory slot. +const UNCLAIMED: ClaimAt = ClaimAt::MAX; +const _: () = assert!(MAX_DIRECTORIES < UNCLAIMED as usize); + +struct DirectoryClaim<'a> { + object: ObjectId, + dir: DirId, + entries: DirWrite<'a>, +} + +impl DirectoryClaims<'_> { + /// The claim standing for `dir`, when it is the claim of exactly `object` + /// — the object↔directory correspondence `claim_directories` pinned is + /// re-checked, so a reclaimed slot can never answer for its successor. + fn position_of_dir(&self, dir: DirId, object: ObjectId) -> Option { + let at = *self.by_dir.get(dir.0.index() as usize)?; + if at == UNCLAIMED { + return None; + } + let claim = self.entries.get(at as usize)?; + (claim.dir == dir && claim.object == object).then_some(at as usize) + } + + fn entries_at(&self, at: usize) -> &DirEntries { + &self.entries[at].entries + } + + fn contains(&self, object: ObjectId) -> bool { + self.entries.iter().any(|claim| claim.object == object) + } + + fn get(&self, object: ObjectId) -> FsResult<&DirEntries> { + self.entries + .iter() + .find(|claim| claim.object == object) + .map(|claim| &**claim.entries) + .ok_or(FsError::Busy) + } + + fn get_mut(&mut self, object: ObjectId) -> FsResult<&mut DirEntries> { + self.entries + .iter_mut() + .find(|claim| claim.object == object) + .map(|claim| &mut **claim.entries) + .ok_or(FsError::Busy) + } +} + +/// A visible parent [`Namespace::parents`] found: the directory, and the +/// names walked to it from the scan's root (empty when it is the root). +#[derive(Clone, Debug)] +pub struct ParentEntry { + pub object: ObjectId, + pub path: heapless::Vec, +} + +/// One directory as the upward parents walk discovered it. +#[derive(Clone, Copy)] +struct Discovered { + object: ObjectId, + /// The discovery index it was reached from — the child it holds an + /// edge to; `None` for the walk's start. + from: Option, + /// The name it gives that child. + name: Option, +} + +/// The names from the walk's end (the root) back down to its start, +/// following each discovery the way the upward walk came, or `None` when +/// the route is longer than a path can name. +fn route_down(found: &[Discovered], index: usize) -> Option> { + let mut path: heapless::Vec = heapless::Vec::new(); + let mut at = index; + while let Some(Discovered { + from: Some(from), + name: Some(name), + .. + }) = found.get(at) + { + path.push(*name).ok()?; + at = *from; + } + Some(path) +} + +/// One entry as reported by [`Namespace::list`]. +#[derive(Clone, Copy, Debug)] +pub struct ListEntry { + pub name: Name, + pub object: ObjectId, + pub kind: ObjectKind, + /// Data, never authority ([`ObjectSubtype`]). + pub subtype: ObjectSubtype, +} + +/// The object arena plus the containment DAG over it. +pub(crate) struct Namespace { + objects: ObjectArena, + /// Lock-free quota/traversal projection of each object slot. + /// + /// File provider transactions retain the canonical object write claim + /// across suspension. Readers that only need immutable graph shape and + /// the last committed length use this generation-validated atomic + /// snapshot instead of contending on the file payload. + snapshots: Box<[ObjectSnapshot]>, + dirs: DirArena, + /// Every object's incoming containment edges, written in the same + /// commit as the forward edge (`crate::parent_index`). + parents_of: ParentIndex, + /// Placements the provider refused to release. A reclamation's + /// namespace effect has already committed by the time the provider is + /// asked, so a failed release cannot un-remove the edge; it is counted + /// here and surfaced in engine statistics rather than reported as a + /// failure of a request that genuinely succeeded. + leaked_placements: AtomicU32, + /// Reclamations that abandoned part of a subtree — a structural refusal + /// (a concurrently stale slot, an exhausted arena) or contention that + /// outlasted [`RECLAIM_CONTENTION_TURNS`](crate::bounds::RECLAIM_CONTENTION_TURNS). + /// + /// Distinct from [`leaked_placements`](Self::leaked_placements), which is + /// the *provider* refusing to release bytes the namespace already + /// disowned. This one says the namespace walk itself stopped early, so the + /// objects below the stopping point stay resident and unreachable. Counted + /// rather than raised for the same reason: the effect that triggered the + /// reclamation has already committed. + abandoned_reclaims: AtomicU32, + next_stable: AtomicU64, +} + +/// Exact live graph usage below one authority root. +/// +/// Objects are counted once even when a directory diamond reaches them by +/// several routes. Containment edges are counted individually because each is +/// independently named, retired, and consumes one bounded entry slot. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct GraphUsage { + pub(crate) objects: u32, + pub(crate) edges: u32, + pub(crate) bytes: u64, +} + +/// One exact object's atomically published graph/accounting projection. +#[derive(Clone, Copy, Debug)] +struct ObjectSnapshotView { + len: u64, + dir: Option, +} + +/// Keep independently hot object snapshots on distinct cache lines. +/// +/// `sequence` is even while stable and odd only for the handful of stores in +/// a metadata commit. A reader never spins behind a preempted writer: it +/// returns typed [`FsError::Busy`] and lets the bounded request retry policy +/// decide when to revisit the snapshot. +#[repr(align(64))] +struct ObjectSnapshot { + sequence: AtomicU64, + generation: AtomicU32, + _pad: AtomicU32, + len: AtomicU64, + dir: AtomicU64, +} + +impl ObjectSnapshot { + const fn vacant() -> Self { + Self { + sequence: AtomicU64::new(0), + generation: AtomicU32::new(0), + _pad: AtomicU32::new(0), + len: AtomicU64::new(0), + dir: AtomicU64::new(0), + } + } + + fn publish(&self, id: ObjectId, len: u64, dir: Option) { + let stable = self.sequence.load(Ordering::Acquire); + assert_eq!(stable & 1, 0, "one object snapshot writer per slot"); + self.sequence + .store(stable.wrapping_add(1), Ordering::Release); + self.generation + .store(id.key().generation(), Ordering::Relaxed); + self.len.store(len, Ordering::Relaxed); + self.dir + .store(dir.map_or(0, |dir| dir.0.to_bits()), Ordering::Relaxed); + self.sequence + .store(stable.wrapping_add(2), Ordering::Release); + } + + fn clear(&self, id: ObjectId) { + let stable = self.sequence.load(Ordering::Acquire); + assert_eq!(stable & 1, 0, "one object snapshot writer per slot"); + self.sequence + .store(stable.wrapping_add(1), Ordering::Release); + if self.generation.load(Ordering::Relaxed) == id.key().generation() { + self.generation.store(0, Ordering::Relaxed); + self.len.store(0, Ordering::Relaxed); + self.dir.store(0, Ordering::Relaxed); + } + self.sequence + .store(stable.wrapping_add(2), Ordering::Release); + } + + fn read(&self, id: ObjectId) -> FsResult { + let before = self.sequence.load(Ordering::Acquire); + if before & 1 != 0 { + return Err(FsError::Busy); + } + let generation = self.generation.load(Ordering::Relaxed); + let len = self.len.load(Ordering::Relaxed); + let dir = self.dir.load(Ordering::Relaxed); + let after = self.sequence.load(Ordering::Acquire); + if before != after { + return Err(FsError::Busy); + } + if generation != id.key().generation() || generation == 0 { + return Err(FsError::AccessLost); + } + Ok(ObjectSnapshotView { + len, + dir: (dir != 0).then(|| DirId(SlotKey::from_bits(dir))), + }) + } +} + +/// One vacant snapshot per object slot, heap-backed for the reason recorded on +/// [`ObjectArena`]. `ObjectSnapshot` holds atomics and is deliberately not +/// `Clone`, so the slice is filled by construction rather than by `resize`. +fn vacant_snapshots() -> Box<[ObjectSnapshot]> { + let mut snapshots = Vec::new(); + snapshots + .try_reserve_exact(MAX_OBJECTS) + .expect("fs object-snapshot projection (bootstrap-scale) — service heap undersized"); + for _ in 0..MAX_OBJECTS { + snapshots.push(ObjectSnapshot::vacant()); + } + snapshots.into_boxed_slice() +} + +impl Namespace { + pub(crate) fn new() -> Self { + // Bootstrap-scale and fixed after startup, exactly like + // `ExtractionState`'s arenas: an allocator that cannot satisfy these + // at mount is an undersized service heap, which is a build + // configuration error of the same class as the compile-time size + // guard — not a runtime volume condition. Keeping construction + // infallible is therefore deliberate, and it keeps every caller + // (including `Engine::new`) unchanged. + let alloc = crate::service_alloc(); + Self { + objects: AtomicSlotVec::try_new_in(MAX_OBJECTS, alloc) + .expect("fs object arena (bootstrap-scale) — service heap undersized"), + snapshots: vacant_snapshots(), + dirs: AtomicSlotVec::try_new_in(MAX_DIRECTORIES, alloc) + .expect("fs directory arena (bootstrap-scale) — service heap undersized"), + parents_of: ParentIndex::try_new() + .expect("fs parent index (bootstrap-scale) — service heap undersized"), + leaked_placements: AtomicU32::new(0), + abandoned_reclaims: AtomicU32::new(0), + next_stable: AtomicU64::new(1), + } + } + + // ── object access ─────────────────────────────────────────────────────── + + pub(crate) fn object(&self, id: ObjectId) -> FsResult> { + self.objects + .try_read(id.key()) + .map_err(|error| fs_access(error, FsError::AccessLost)) + } + + pub(crate) fn object_mut(&self, id: ObjectId) -> FsResult> { + self.objects + .try_write(id.key()) + .map_err(|error| fs_access(error, FsError::AccessLost)) + } + + fn snapshot(&self, id: ObjectId) -> FsResult { + let index = usize::try_from(id.key().index()).map_err(|_| FsError::AccessLost)?; + self.snapshots + .get(index) + .ok_or(FsError::AccessLost)? + .read(id) + } + + /// Publish one file's visible length and revision as a single file + /// transaction. The quota snapshot is updated while the caller still + /// holds the exact object claim and its pending growth reservation. + pub(crate) fn commit_file_len( + &self, + id: ObjectId, + object: &mut ObjectWrite<'_, K>, + len: u64, + ) -> Revision { + debug_assert_eq!(object.id(), id.key()); + object.commit_len(len); + self.snapshots[id.key().index() as usize].publish(id, len, object.dir()); + object.revision() + } + + pub(crate) fn exists(&self, id: ObjectId) -> bool { + self.objects.contains(id.key()) + } + + pub(crate) fn kind(&self, id: ObjectId) -> FsResult { + Ok(self.object(id)?.kind()) + } + + pub(crate) fn object_count(&self) -> usize { + self.objects.len() + } + + pub(crate) fn directory_count(&self) -> usize { + self.dirs.len() + } + + pub(crate) fn leaked_placements(&self) -> u32 { + self.leaked_placements.load(Ordering::Relaxed) + } + + pub(crate) fn abandoned_reclaims(&self) -> u32 { + self.abandoned_reclaims.load(Ordering::Relaxed) + } + + pub(crate) fn objects(&self) -> impl Iterator)> { + self.objects + .iter() + .map(|(key, object)| (ObjectId::from_bits(key.to_bits()), object)) + } + + pub(crate) fn find_stable(&self, stable: StableObjectId) -> Option { + self.objects() + .find_map(|(id, object)| (object.stable() == stable).then_some(id)) + } + + #[cfg(test)] + pub(crate) fn collect_reachable( + &self, + root: ObjectId, + ) -> FsResult> { + let mut found = heapless::Vec::new(); + let mut pending = heapless::Vec::::new(); + pending.push(root).map_err(|_| FsError::OutOfObjects)?; + while let Some(id) = pending.pop() { + if found.contains(&id) { + continue; + } + let snapshot = self.snapshot(id)?; + found.push(id).map_err(|_| FsError::OutOfObjects)?; + if let Some(dir) = snapshot.dir { + let entries = self + .dirs + .try_read(dir.0) + .map_err(|error| fs_access(error, FsError::AccessLost))?; + for (_, entry) in entries.iter() { + if !found.contains(&entry.child) && !pending.contains(&entry.child) { + pending + .push(entry.child) + .map_err(|_| FsError::OutOfObjects)?; + } + } + } + } + Ok(found) + } + + /// Copy one complete reachable graph while retaining exact per-slot read + /// guards over every object and directory in the footprint. + /// + /// Writers never wait behind this snapshot: a conflicting request gets + /// typed [`FsError::Busy`]. The guards are released before the caller + /// performs provider or device I/O, so an fsync cannot pin hot namespace + /// cache lines for the duration of an NVMe flush. + /// `max_edges` is the volume profile's containment-edge ceiling, which + /// the caller already holds; it bounds the two edge buffers exactly as the + /// old const generic did. + /// + /// An allocation that cannot be satisfied reports the same error a full + /// buffer reports, because it is the same condition from the caller's + /// side: the buffer this snapshot needs cannot be provided. Commit is a + /// live path in a founded service, so it may not panic — see the contract + /// on `Engine::reclaim`. + pub(crate) fn durable_graph( + &self, + root: ObjectId, + max_edges: usize, + ) -> FsResult> { + let alloc = crate::service_alloc(); + let mut pending = BoundedVec::::try_new_in(MAX_OBJECTS, alloc) + .map_err(|_| FsError::OutOfObjects)?; + let mut objects = BoundedVec::<(ObjectId, ObjectRead<'_, K>), ServiceAlloc>::try_new_in( + MAX_OBJECTS, + alloc, + ) + .map_err(|_| FsError::OutOfObjects)?; + let mut dirs = + BoundedVec::<(DirId, DirRead<'_>), ServiceAlloc>::try_new_in(MAX_DIRECTORIES, alloc) + .map_err(|_| FsError::OutOfObjects)?; + let mut raw_edges = + BoundedVec::<(ObjectId, Name, ObjectId), ServiceAlloc>::try_new_in(max_edges, alloc) + .map_err(|_| FsError::OutOfEdges)?; + pending.push(root).map_err(|_| FsError::OutOfObjects)?; + + while let Some(id) = pending.pop() { + if objects.iter().any(|(seen, _)| *seen == id) { + continue; + } + let object = self.objects.try_read(id.key()).map_err(|_| FsError::Busy)?; + let dir = object.dir(); + objects + .push((id, object)) + .map_err(|_| FsError::OutOfObjects)?; + let Some(dir) = dir else { continue }; + if dirs.iter().any(|(seen, _)| *seen == dir) { + return Err(FsError::AccessLost); + } + let entries = self.dirs.try_read(dir.0).map_err(|_| FsError::Busy)?; + for (_, entry) in entries.iter() { + raw_edges + .push((id, entry.name, entry.child)) + .map_err(|_| FsError::OutOfEdges)?; + if !objects.iter().any(|(seen, _)| *seen == entry.child) + && !pending.contains(&entry.child) + { + pending + .push(entry.child) + .map_err(|_| FsError::OutOfObjects)?; + } + } + dirs.push((dir, entries)) + .map_err(|_| FsError::OutOfObjects)?; + } + + let stable = |wanted: ObjectId| { + objects + .iter() + .find_map(|(id, object)| (*id == wanted).then(|| object.stable())) + .ok_or(FsError::AccessLost) + }; + let root_stable = stable(root)?; + let mut copied_objects = + BoundedVec::, ServiceAlloc>::try_new_in(MAX_OBJECTS, alloc) + .map_err(|_| FsError::OutOfObjects)?; + for (id, object) in objects.iter() { + copied_objects + .push(DurableGraphObject { + stable: object.stable(), + kind: object.kind(), + subtype: object.subtype(), + len: object.len(), + revision: object.revision(), + storage: object.storage(), + }) + .map_err(|_| FsError::OutOfObjects)?; + } + let mut copied_edges = + BoundedVec::::try_new_in(max_edges, alloc) + .map_err(|_| FsError::OutOfEdges)?; + for (parent, name, child) in raw_edges { + copied_edges + .push(DurableGraphEdge { + parent: stable(parent)?, + name, + child: stable(child)?, + }) + .map_err(|_| FsError::OutOfEdges)?; + } + Ok(DurableGraph { + root: root_stable, + objects: copied_objects, + edges: copied_edges, + }) + } + + /// Derive quota usage from the live namespace rather than maintaining a + /// second counter ledger beside it. + pub(crate) fn graph_usage(&self, root: ObjectId) -> FsResult { + self.graph_usage_inner(root) + } + + /// Derive graph usage while the caller owns one exact object write claim. + /// + /// The commit snapshot deliberately remains readable while that canonical + /// record is claimed, so quota observes the last visible length plus all + /// pending reservations. The arguments document and type-check the + /// claimed object/route correspondence at the engine boundary; no payload + /// alias is created here. + pub(crate) fn graph_usage_with_claimed( + &self, + root: ObjectId, + _claimed_id: ObjectId, + _claimed: &Object, + ) -> FsResult { + self.graph_usage_inner(root) + } + + fn graph_usage_inner(&self, root: ObjectId) -> FsResult { + let alloc = crate::service_alloc(); + let mut found = BoundedVec::::try_new_in(MAX_OBJECTS, alloc) + .map_err(|_| FsError::OutOfObjects)?; + let mut pending = BoundedVec::::try_new_in(MAX_OBJECTS, alloc) + .map_err(|_| FsError::OutOfObjects)?; + pending.push(root).map_err(|_| FsError::OutOfObjects)?; + let mut usage = GraphUsage::default(); + while let Some(id) = pending.pop() { + if found.contains(&id) { + continue; + } + let snapshot = self.snapshot(id)?; + found.push(id).map_err(|_| FsError::OutOfObjects)?; + usage.objects = usage.objects.checked_add(1).ok_or(FsError::OutOfObjects)?; + usage.bytes = usage + .bytes + .checked_add(snapshot.len) + .ok_or(FsError::OutOfStorage)?; + if let Some(dir) = snapshot.dir { + let entries = self + .dirs + .try_read(dir.0) + .map_err(|error| fs_access(error, FsError::AccessLost))?; + usage.edges = usage + .edges + .checked_add(u32::try_from(entries.len()).map_err(|_| FsError::OutOfEdges)?) + .ok_or(FsError::OutOfEdges)?; + for (_, entry) in entries.iter() { + if !found.contains(&entry.child) && !pending.contains(&entry.child) { + pending + .push(entry.child) + .map_err(|_| FsError::OutOfObjects)?; + } + } + } + } + Ok(usage) + } + + pub(crate) fn restore_object( + &self, + stable: StableObjectId, + kind: ObjectKind, + subtype: ObjectSubtype, + len: u64, + revision: u64, + storage: Option, + ) -> FsResult { + if stable == StableObjectId::INVALID || self.find_stable(stable).is_some() { + return Err(FsError::ProviderFailure(ProviderError::Integrity)); + } + // A directory carries no subtype; the durable decoder refuses one, so + // reaching here with one is the caller disagreeing with itself. + if kind.is_dir() && !subtype.is_plain() { + return Err(FsError::ProviderFailure(ProviderError::Integrity)); + } + let id = match kind { + ObjectKind::File => self + .objects + .insert(Object::new_file_with(subtype).with_stable(stable)) + .map(|key| ObjectId::from_bits(key.to_bits())) + .map_err(|_| FsError::OutOfObjects)?, + ObjectKind::Directory => { + let dir = self.new_dir_table()?; + match self + .objects + .insert(Object::new_dir(dir).with_stable(stable)) + { + Ok(key) => ObjectId::from_bits(key.to_bits()), + Err(_) => { + self.dirs.remove(dir.0); + return Err(FsError::OutOfObjects); + } + } + } + }; + let mut object = self.object_mut(id)?; + if let Some(key) = storage { + object.set_storage(key); + } + object.restore_len_revision(len, revision); + let dir = object.dir(); + drop(object); + self.snapshots[id.key().index() as usize].publish(id, len, dir); + self.next_stable + .fetch_max(stable.get().saturating_add(1), Ordering::Relaxed); + Ok(id) + } + + pub(crate) fn restore_edge( + &self, + parent: StableObjectId, + name: Name, + child: StableObjectId, + ) -> FsResult<()> { + let parent = self + .find_stable(parent) + .ok_or(FsError::ProviderFailure(ProviderError::Integrity))?; + let child = self + .find_stable(child) + .ok_or(FsError::ProviderFailure(ProviderError::Integrity))?; + self.link(parent, &name, child)?; + Ok(()) + } + + fn allocate_stable(&self) -> FsResult { + let mut current = self.next_stable.load(Ordering::Relaxed); + loop { + let id = StableObjectId::new(current).ok_or(FsError::OutOfObjects)?; + let next = current.checked_add(1).ok_or(FsError::OutOfObjects)?; + match self.next_stable.compare_exchange_weak( + current, + next, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return Ok(id), + Err(observed) => current = observed, + } + } + } + + // ── object creation ───────────────────────────────────────────────────── + + /// Create an unlinked file object. The caller links it in the same + /// request; until then it is reclaimable, which is what makes a failed + /// `create` leave nothing behind. + pub(crate) fn create_file_object(&self) -> FsResult { + self.create_file_object_with(ObjectSubtype::PLAIN) + } + + /// [`Self::create_file_object`], stamped `subtype`. + pub(crate) fn create_file_object_with(&self, subtype: ObjectSubtype) -> FsResult { + let stable = self.allocate_stable()?; + let id = self + .objects + .insert(Object::new_file_with(subtype).with_stable(stable)) + .map(|key| ObjectId::from_bits(key.to_bits())) + .map_err(|_| FsError::OutOfObjects)?; + self.snapshots[id.key().index() as usize].publish(id, 0, None); + Ok(id) + } + + /// A new directory's child list: boxed on the service heap and placed + /// in the arena, both refusals reported as the directory that could not + /// be created. + fn new_dir_table(&self) -> FsResult { + let table = Box::try_new_in(DirEntries::new(), *crate::service_alloc()) + .map_err(|_| FsError::OutOfObjects)?; + self.dirs + .insert(table) + .map(DirId) + .map_err(|_| FsError::OutOfObjects) + } + + /// Create an unlinked directory object, allocating its child list. + pub(crate) fn create_dir_object(&self) -> FsResult { + let stable = self.allocate_stable()?; + // Reserve the child list first: if the object arena is full we must + // not leak a directory slot, and if the directory arena is full we + // must not leak an object slot. + let dir = self.new_dir_table()?; + match self + .objects + .insert(Object::new_dir(dir).with_stable(stable)) + { + Ok(key) => { + let id = ObjectId::from_bits(key.to_bits()); + self.snapshots[id.key().index() as usize].publish(id, 0, Some(dir)); + Ok(id) + } + Err(_) => { + self.dirs.remove(dir.0); + Err(FsError::OutOfObjects) + } + } + } + + // ── edges ─────────────────────────────────────────────────────────────── + + fn dir_entries(&self, parent: ObjectId) -> FsResult> { + let dir = self.object(parent)?.dir().ok_or(FsError::NotDirectory)?; + // A live directory object always owns a live child list; a missing + // one would mean the two arenas had desynchronized. + self.dirs + .try_read(dir.0) + .map_err(|error| fs_access(error, FsError::AccessLost)) + } + + fn dir_entries_mut(&self, parent: ObjectId) -> FsResult> { + let dir = self.object(parent)?.dir().ok_or(FsError::NotDirectory)?; + self.dirs + .try_write(dir.0) + .map_err(|error| fs_access(error, FsError::AccessLost)) + } + + /// Claim an explicit directory set in stable slot order. + fn claim_directories<'a>(&'a self, objects: &[ObjectId]) -> FsResult> { + let mut wanted = BoundedVec::<(ObjectId, DirId), ServiceAlloc>::try_new_in( + objects.len(), + crate::service_alloc(), + ) + .map_err(|_| FsError::OutOfObjects)?; + for object in objects { + let dir = self.snapshot(*object)?.dir.ok_or(FsError::NotDirectory)?; + if let Some((_, existing)) = wanted.iter().find(|(id, _)| id == object) { + if *existing != dir { + return Err(FsError::AccessLost); + } + continue; + } + if wanted.iter().any(|(_, existing)| *existing == dir) { + return Err(FsError::AccessLost); + } + wanted + .push((*object, dir)) + .map_err(|_| FsError::OutOfObjects)?; + } + wanted.as_mut_slice().sort_unstable_by_key(|(_, dir)| dir.0); + + let mut claimed = DirectoryClaims { + entries: BoundedVec::try_new_in(wanted.len(), crate::service_alloc()) + .map_err(|_| FsError::OutOfObjects)?, + by_dir: [UNCLAIMED; MAX_DIRECTORIES], + }; + for (object, dir) in wanted { + let entries = self + .dirs + .try_write(dir.0) + .map_err(|error| fs_access(error, FsError::AccessLost))?; + claimed + .entries + .push(DirectoryClaim { + object, + dir, + entries, + }) + .map_err(|_| FsError::OutOfObjects)?; + } + + // Close the optimistic object→directory lookup after every exact + // directory generation is pinned. A reclaimed/reused object cannot + // make this set accidentally name its successor. + for (at, claim) in claimed.entries.iter().enumerate() { + if self.snapshot(claim.object)?.dir != Some(claim.dir) { + return Err(FsError::AccessLost); + } + let slot = claimed + .by_dir + .get_mut(claim.dir.0.index() as usize) + .ok_or(FsError::Range)?; + *slot = ClaimAt::try_from(at).map_err(|_| FsError::Range)?; + } + Ok(claimed) + } + + /// Snapshot the directory subgraph below `root` for later canonical + /// claiming. The claimed traversal revalidates this set before mutation; + /// an edge added in the observation→claim interval yields `Busy` rather + /// than escaping the cycle footprint. + fn directory_subtree(&self, root: ObjectId) -> FsResult> { + let alloc = crate::service_alloc(); + let mut found = BoundedVec::::try_new_in(MAX_DIRECTORIES, alloc) + .map_err(|_| FsError::OutOfObjects)?; + let mut pending = BoundedVec::::try_new_in(MAX_DIRECTORIES, alloc) + .map_err(|_| FsError::OutOfObjects)?; + // Marked on enqueue, keyed by directory slot: each directory is + // queued exactly once, so the worklist is bounded by the arena and + // membership is O(1) rather than two list scans per edge. + let mut seen = ReachSet::new(); + let Some(dir) = self.snapshot(root)?.dir else { + return Ok(found); + }; + seen.insert(dir.0.index() as usize)?; + pending.push(root).map_err(|_| FsError::OutOfObjects)?; + while let Some(object) = pending.pop() { + let Some(dir) = self.snapshot(object)?.dir else { + continue; + }; + found.push(object).map_err(|_| FsError::OutOfObjects)?; + let entries = self + .dirs + .try_read(dir.0) + .map_err(|error| fs_access(error, FsError::AccessLost))?; + for (_, entry) in entries.iter() { + let Some(child) = self.snapshot(entry.child)?.dir else { + continue; + }; + if seen.insert(child.0.index() as usize)? { + pending + .push(entry.child) + .map_err(|_| FsError::OutOfObjects)?; + } + } + } + Ok(found) + } + + fn claim_subtree_with<'a>( + &'a self, + root: ObjectId, + extras: &[ObjectId], + ) -> FsResult> { + let mut objects = self.directory_subtree(root)?; + for extra in extras { + if !objects.contains(extra) { + objects.push(*extra).map_err(|_| FsError::OutOfObjects)?; + } + } + self.claim_directories(&objects) + } + + fn is_reachable_claimed( + &self, + claims: &DirectoryClaims<'_>, + ancestor: ObjectId, + candidate: ObjectId, + ) -> FsResult { + if ancestor == candidate { + return Ok(true); + } + // The walk speaks claim positions, keyed by directory slot: one bit + // row for "seen", marked on enqueue so every directory is queued at + // most once, and the claim index answers each edge in O(1). The + // protocol is unchanged — a child directory outside the pinned + // footprint is `Busy`, and the caller decides `Cycle`. + let mut seen = ReachSet::new(); + let mut pending = + BoundedVec::::try_new_in(MAX_DIRECTORIES, crate::service_alloc()) + .map_err(|_| FsError::OutOfObjects)?; + let start_dir = self.snapshot(ancestor)?.dir.ok_or(FsError::NotDirectory)?; + let start = claims + .position_of_dir(start_dir, ancestor) + .ok_or(FsError::Busy)?; + seen.insert(start_dir.0.index() as usize)?; + pending.push(start as ClaimAt).map_err(|_| FsError::Range)?; + while let Some(at) = pending.pop() { + for (_, entry) in claims.entries_at(at as usize).iter() { + if entry.child == candidate { + return Ok(true); + } + let Some(dir) = self.snapshot(entry.child)?.dir else { + continue; + }; + // The optimistic footprint changed before it was pinned. + // Retrying takes a complete new snapshot. + let Some(next) = claims.position_of_dir(dir, entry.child) else { + return Err(FsError::Busy); + }; + if seen.insert(dir.0.index() as usize)? { + pending.push(next as ClaimAt).map_err(|_| FsError::Range)?; + } + } + } + Ok(false) + } + + /// Resolve one name component beneath `parent`. + pub(crate) fn lookup(&self, parent: ObjectId, name: &Name) -> FsResult<(EdgeId, ObjectId)> { + let entries = self.dir_entries(parent)?; + for (key, entry) in entries.iter() { + if entry.name == *name { + return Ok((EdgeId { parent, entry: key }, entry.child)); + } + } + Err(FsError::NotFound) + } + + /// The child an edge currently names — `AccessLost` if the edge has been + /// removed or its generation retired. This is the primitive the bounded + /// edge proof revalidates against. + pub(crate) fn edge_child(&self, edge: EdgeId) -> FsResult { + let entries = self.dir_entries(edge.parent)?; + entries + .try_read(edge.entry) + .map(|entry| entry.child) + .map_err(|error| fs_access(error, FsError::AccessLost)) + } + + /// Add a containment edge `parent/name -> child`, incrementing the + /// child's incoming count. + /// + /// Directory children are legal only when adding this edge cannot make + /// `parent` reachable from `child`. The bounded visited set distinguishes + /// a lawful diamond from a back-edge; all checks complete before the slot + /// and incoming count are mutated. + pub(crate) fn link(&self, parent: ObjectId, name: &Name, child: ObjectId) -> FsResult { + if !self.kind(parent)?.is_dir() { + return Err(FsError::NotDirectory); + } + if !self.exists(child) { + return Err(FsError::AccessLost); + } + let child_is_dir = self.kind(child)?.is_dir(); + let mut claims = if child_is_dir { + self.claim_subtree_with(child, &[parent])? + } else { + self.claim_directories(&[parent])? + }; + if child_is_dir && self.is_reachable_claimed(&claims, child, parent)? { + return Err(FsError::Cycle); + } + // The final name check happens under the exact destination claim. Two + // concurrent creators cannot both publish the same label. + let entries = claims.get_mut(parent)?; + if entries.iter().any(|(_, entry)| entry.name == *name) { + return Err(FsError::AlreadyExists); + } + // Claim the child's incoming counter and its parent row, with room + // for the edge, before publishing it, so contention cannot leave a + // visible name without its matching count or reverse entry. + let mut child_object = self.object_mut(child)?; + let mut parents = self.parents_of.claim(child)?; + parents.reserve()?; + let entry = entries + .insert(DirEntry { name: *name, child }) + .map_err(|_| FsError::OutOfEdges)?; + // The edge exists now, so the child has one more name. + child_object.incoming_inc(); + let edge = EdgeId { parent, entry }; + parents.add(edge); + Ok(edge) + } + + /// Remove exactly one containment edge and decrement the child's incoming + /// count, regardless of whether a directory child is empty. + /// + /// This is the native graph operation. The caller invokes reclamation only + /// if that exact edge made the child unreachable; another containment or + /// authority edge keeps the full subgraph live. + pub(crate) fn detach(&self, parent: ObjectId, name: &Name) -> FsResult<(EdgeId, ObjectId)> { + let entries = self.dir_entries_mut(parent)?; + let (entry, child) = entries + .iter() + .find(|(_, candidate)| candidate.name == *name) + .map(|(key, candidate)| (key, candidate.child)) + .ok_or(FsError::NotFound)?; + let edge = EdgeId { parent, entry }; + let mut child_object = self.object_mut(child)?; + let mut parents = self.parents_of.claim(child)?; + entries + .try_remove(edge.entry) + .map_err(|error| fs_access(error, FsError::AccessLost))?; + child_object.incoming_dec(); + parents.remove(edge); + Ok((edge, child)) + } + + /// POSIX-style `unlink`/`rmdir` compatibility: directory children must be + /// empty. Native callers that mean “remove this graph edge” use + /// [`Self::detach`]. + pub(crate) fn unlink_empty( + &self, + parent: ObjectId, + name: &Name, + ) -> FsResult<(EdgeId, ObjectId)> { + let (_, observed_child) = self.lookup(parent, name)?; + let child_is_dir = self.kind(observed_child)?.is_dir(); + let mut claims = if child_is_dir { + self.claim_directories(&[parent, observed_child])? + } else { + self.claim_directories(&[parent])? + }; + let (entry, child) = claims + .get(parent)? + .iter() + .find(|(_, candidate)| candidate.name == *name) + .map(|(key, candidate)| (key, candidate.child)) + .ok_or(FsError::NotFound)?; + if child != observed_child { + return Err(FsError::Busy); + } + if child_is_dir && !claims.get(child)?.is_empty() { + return Err(FsError::NotEmpty); + } + let mut child_object = self.object_mut(child)?; + let mut parents = self.parents_of.claim(child)?; + claims + .get_mut(parent)? + .try_remove(entry) + .map_err(|error| fs_access(error, FsError::AccessLost))?; + child_object.incoming_dec(); + let edge = EdgeId { parent, entry }; + parents.remove(edge); + Ok((edge, child)) + } + + /// Change one edge's label, keeping the edge — and therefore every + /// handle whose proof runs through it. + /// + /// Rename changes the edge label without changing the child object or + /// retiring the route, so the entry slot and its generation are preserved. + /// That is the visible difference + /// from [`Self::relocate`], which does retire the source edge. + /// + /// Refuses when the destination name exists — replacement + /// semantics wait until atomic reclamation is proven. + pub(crate) fn rename(&self, parent: ObjectId, old: &Name, new: &Name) -> FsResult { + let entries = self.dir_entries_mut(parent)?; + let entry = entries + .iter() + .find(|(_, candidate)| candidate.name == *old) + .map(|(key, _)| key) + .ok_or(FsError::NotFound)?; + let edge = EdgeId { parent, entry }; + if old != new && entries.iter().any(|(_, candidate)| candidate.name == *new) { + return Err(FsError::AlreadyExists); + } + let mut entry = entries + .try_write(edge.entry) + .map_err(|error| fs_access(error, FsError::AccessLost))?; + entry.name = *new; + Ok(edge) + } + + /// Atomically remove `src/name` and create `dst/new_name` for the same + /// object. + /// + /// The source edge is retired, so a handle opened through it fails closed + /// on its next operation; a handle opened through a *different* link to + /// the same file keeps working. Everything that can refuse + /// is checked before anything is mutated, so a refused move leaves both + /// directories exactly as they were. + pub(crate) fn relocate( + &self, + src: ObjectId, + name: &Name, + dst: ObjectId, + new_name: &Name, + ) -> FsResult<(EdgeId, EdgeId)> { + if !self.kind(dst)?.is_dir() { + return Err(FsError::NotDirectory); + } + let (observed_src_edge, observed_child) = self.lookup(src, name)?; + // The kind comes from the lock-free snapshot, not the object: a file + // transaction holds the object's claim across provider I/O, and a + // move never needs it (an object's kind never changes; the claimed + // commit below re-checks the edge). + let child_is_dir = self.snapshot(observed_child)?.dir.is_some(); + let mut claims = if child_is_dir { + self.claim_subtree_with(observed_child, &[src, dst])? + } else { + self.claim_directories(&[src, dst])? + }; + let (src_entry, child) = claims + .get(src)? + .iter() + .find(|(_, candidate)| candidate.name == *name) + .map(|(key, candidate)| (key, candidate.child)) + .ok_or(FsError::NotFound)?; + let src_edge = EdgeId { + parent: src, + entry: src_entry, + }; + if src_edge != observed_src_edge || child != observed_child { + return Err(FsError::Busy); + } + + // Same directory, same name: nothing would change, and doing the + // remove/insert anyway would needlessly retire live handles. + if src == dst && name == new_name { + return Ok((src_edge, src_edge)); + } + + // The destination name must be free. When moving within one + // directory the source edge is about to vacate, so only a *different* + // name can collide. + if claims + .get(dst)? + .iter() + .any(|(_, candidate)| candidate.name == *new_name) + { + return Err(FsError::AlreadyExists); + } + + // Moving a directory into its own subtree would orphan a cycle. + if child_is_dir && self.is_reachable_claimed(&claims, child, dst)? { + return Err(FsError::Cycle); + } + + // Reserve destination room before removing the source edge, so the + // insert below cannot fail and leave the object nameless. Moving + // within one directory reuses the vacated slot, so no room is needed. + if src != dst && claims.get(dst)?.is_full() { + return Err(FsError::OutOfEdges); + } + + // The child's parent row moves with the edge; it is claimed on its + // own, so a file mid-transaction can still be moved. + let mut parents = self.parents_of.claim(child)?; + + // Commit: retire the source edge, then create the destination edge. + claims + .get_mut(src)? + .try_remove(src_edge.entry) + .map_err(|error| fs_access(error, FsError::AccessLost))?; + let entry = claims + .get_mut(dst)? + .insert(DirEntry { + name: *new_name, + child, + }) + .map_err(|_| FsError::OutOfEdges)?; + // One edge out, one edge in: the incoming count is unchanged, so it + // is deliberately not touched here; the parent row swaps in place. + let dst_edge = EdgeId { parent: dst, entry }; + parents.replace(src_edge, dst_edge); + Ok((src_edge, dst_edge)) + } + + /// One page of `parent`'s entries starting at slot `cursor`, plus the + /// cursor to resume at (or `None` when the listing is complete). + pub(crate) fn list( + &self, + parent: ObjectId, + cursor: u32, + ) -> FsResult<(heapless::Vec, Option)> { + let entries = self.dir_entries(parent)?; + let mut page: heapless::Vec = heapless::Vec::new(); + let mut next = None; + for (key, entry) in entries.iter_from(cursor) { + if page.is_full() { + // Resume *at* the entry that did not fit — never past it. + next = Some(key.index()); + break; + } + let (kind, subtype) = { + let object = self.object(entry.child)?; + (object.kind(), object.subtype()) + }; + let pushed = page.push(ListEntry { + name: entry.name, + object: entry.child, + kind, + subtype, + }); + debug_assert!(pushed.is_ok(), "checked is_full above"); + if pushed.is_err() { + next = Some(key.index()); + break; + } + } + Ok((page, next)) + } + + /// One page of `target`'s visible parents — the directories holding an + /// edge to it, in the order the edges were made — skipping the first + /// `cursor`, each with the shortest route to it from `root` as names. A + /// directory holding `target` under two names is one parent. A parent + /// is visible only when `root` is among its ancestors, found by walking + /// upward through the parent index, so a caller anchored at `root` + /// learns nothing about edges outside its view; a parent deeper than a + /// route can name ([`MAX_PATH_DEPTH`]) is one the caller could not open + /// either, and is left out. + /// + /// A page holds at most `limit` parents (and never more than + /// [`MAX_LIST_PAGE`]): the caller's window decides, so a small window + /// pages rather than refuses. The cost is `target`'s edges plus, per + /// parent, the ancestors walked up to `root` — not the size of the tree. + pub(crate) fn parents( + &self, + root: ObjectId, + target: ObjectId, + cursor: u32, + limit: usize, + ) -> FsResult<(heapless::Vec, Option)> { + let limit = limit.min(MAX_LIST_PAGE); + if limit == 0 { + return Err(FsError::Range); + } + // The distinct parents in edge order; a directory holding `target` + // twice is marked once (by slot), so the dedupe stays linear in the + // fan-in. + let edges = self.incoming_edges(target)?; + let mut direct = + BoundedVec::::try_new_in(edges.len(), crate::service_alloc()) + .map_err(|_| FsError::OutOfObjects)?; + let mut marked = ReachSet::new(); + for edge in edges.iter() { + if marked.insert(edge.parent.key().index() as usize)? { + direct + .push(edge.parent) + .map_err(|_| FsError::OutOfObjects)?; + } + } + let mut page: heapless::Vec = heapless::Vec::new(); + let mut rank = 0u32; + for &object in direct.iter() { + let Some(path) = self.route_up(root, object)? else { + continue; + }; + if rank >= cursor { + if page.len() == limit { + return Ok((page, Some(rank))); + } + let pushed = page.push(ParentEntry { object, path }); + debug_assert!(pushed.is_ok(), "limit is within the page"); + } + rank += 1; + } + Ok((page, None)) + } + + /// `object`'s incoming edges, copied out of its parent row so no claim + /// is held while they are walked. + fn incoming_edges(&self, object: ObjectId) -> FsResult> { + let row = self.parents_of.claim(object)?; + let mut edges = BoundedVec::try_new_in(row.len(), crate::service_alloc()) + .map_err(|_| FsError::OutOfObjects)?; + for edge in row.iter() { + edges.push(edge).map_err(|_| FsError::OutOfObjects)?; + } + Ok(edges) + } + + /// The name an edge's parent gives it — `AccessLost` if the edge has + /// been retired since it was read. + fn edge_name(&self, edge: EdgeId) -> FsResult { + let entries = self.dir_entries(edge.parent)?; + entries + .try_read(edge.entry) + .map(|entry| entry.name) + .map_err(|error| fs_access(error, FsError::AccessLost)) + } + + /// The shortest route from `root` down to the directory `to`, as names: + /// a breadth-first walk upward through the parent index, every directory + /// once ([`ReachSet`]). `None` when `root` is not an ancestor of `to` + /// or the route is longer than a path can name. An edge retired while + /// the walk reads it is skipped, as a parent that just left. + pub(crate) fn route_up( + &self, + root: ObjectId, + to: ObjectId, + ) -> FsResult>> { + if to == root { + return Ok(Some(heapless::Vec::new())); + } + let alloc = crate::service_alloc(); + let mut found = BoundedVec::::try_new_in(MAX_DIRECTORIES, alloc) + .map_err(|_| FsError::OutOfObjects)?; + let mut seen = ReachSet::new(); + seen.insert(to.key().index() as usize)?; + found + .push(Discovered { + object: to, + from: None, + name: None, + }) + .map_err(|_| FsError::OutOfObjects)?; + let mut head = 0usize; + while let Some(&Discovered { object, .. }) = found.get(head) { + for edge in self.incoming_edges(object)?.iter() { + let parent = edge.parent; + let Ok(name) = self.edge_name(*edge) else { + continue; + }; + if !seen.insert(parent.key().index() as usize)? { + continue; + } + found + .push(Discovered { + object: parent, + from: Some(head), + name: Some(name), + }) + .map_err(|_| FsError::OutOfObjects)?; + if parent == root { + return Ok(route_down(&found, found.len() - 1)); + } + } + head += 1; + } + Ok(None) + } + + // ── reclamation ───────────────────────────────────────────────────────── + + /// Try to release one exact object, returning children that became + /// reclaimable when its containment edges disappeared. + /// + /// Called after an operation drops the last route to an object. A + /// directory becoming unreachable takes its whole subtree with it — + /// `unlink` refuses non-empty directories, but retiring a *grant* or a + /// private root can legitimately strand a populated subtree, and leaving + /// it in the arenas would leak every object under it. + /// + /// Every payload needed for this one commit is claimed before the first + /// incoming count changes. Contention therefore returns [`FsError::Busy`] + /// with the object intact; the engine retains the work item and retries it + /// cooperatively instead of silently leaking a skipped slot. + pub(crate) fn reclaim_one( + &self, + id: ObjectId, + placements: &mut BoundedVec, + objects: &mut BoundedVec, + ) -> FsResult> { + let object = match self.objects.try_write(id.key()) { + Ok(object) => object, + Err(crate::slot::SlotAccess::Stale) => return Ok(heapless::Vec::new()), + Err(crate::slot::SlotAccess::Busy) => return Err(FsError::Busy), + }; + if !object.is_reclaimable() { + return Ok(heapless::Vec::new()); + } + if objects.is_full() || (object.storage().is_some() && placements.is_full()) { + return Err(FsError::OutOfObjects); + } + + let storage = object.storage(); + let mut directory = match object.dir() { + Some(dir) => Some( + self.dirs + .try_write(dir.0) + .map_err(|error| fs_access(error, FsError::AccessLost))?, + ), + None => None, + }; + + let mut child_ids = heapless::Vec::::new(); + if let Some(entries) = directory.as_ref() { + for (_, entry) in entries.iter() { + if !child_ids.contains(&entry.child) { + child_ids + .push(entry.child) + .map_err(|_| FsError::OutOfObjects)?; + } + } + } + child_ids.as_mut_slice().sort_unstable_by_key(|id| id.key()); + + let mut children = heapless::Vec::< + (ObjectId, ObjectWrite<'_, K>, ParentClaim<'_>), + MAX_DIR_ENTRIES, + >::new(); + for child in child_ids { + let claimed = self + .objects + .try_write(child.key()) + .map_err(|error| fs_access(error, FsError::AccessLost))?; + let parents = self.parents_of.claim(child)?; + children + .push((child, claimed, parents)) + .map_err(|_| FsError::OutOfObjects)?; + } + // The reclaimed object names nothing once it is gone; its own row + // is emptied with it (it is already empty: nothing names it). + let mut own_parents = self.parents_of.claim(id)?; + + // All-or-nothing commit: decrement once per removed edge, and drop + // its reverse entry, then record only the child generations that + // crossed to zero reachability. + if let Some(entries) = directory.as_ref() { + for (key, entry) in entries.iter() { + let (_, child, parents) = children + .iter_mut() + .find(|(id, _, _)| *id == entry.child) + .ok_or(FsError::AccessLost)?; + child.incoming_dec(); + parents.remove(EdgeId { + parent: id, + entry: key, + }); + } + } + own_parents.clear(); + let mut became_reclaimable = heapless::Vec::new(); + for (id, child, _) in children.iter() { + if child.is_reclaimable() { + became_reclaimable + .push(*id) + .map_err(|_| FsError::OutOfObjects)?; + } + } + + if let Some(entries) = directory.take() { + entries.remove().map_err(|_| FsError::AccessLost)?; + } + if let Some(key) = storage { + placements.push(key).map_err(|_| FsError::OutOfObjects)?; + } + object.remove().map_err(|_| FsError::AccessLost)?; + self.snapshots[id.key().index() as usize].clear(id); + objects.push(id).map_err(|_| FsError::OutOfObjects)?; + Ok(became_reclaimable) + } + + #[cfg(test)] + pub(crate) fn reclaim( + &self, + start: ObjectId, + placements: &mut BoundedVec, + objects: &mut BoundedVec, + ) -> usize { + let before = objects.len(); + // The caller's batches are already arena-backed; so is this, for the + // same reason. An allocation failure here takes the route a full + // worklist takes below: a counted, reportable abandonment, never a + // panic (see the contract on `Engine::reclaim`). + let Ok(mut worklist) = + BoundedVec::::try_new_in(MAX_OBJECTS, crate::service_alloc()) + else { + self.note_abandoned_reclaim(); + return 0; + }; + worklist + .push(start) + .expect("one root fits the object bound"); + while let Some(id) = worklist.pop() { + let children = self + .reclaim_one(id, placements, objects) + .expect("single-threaded reclaim fixture cannot contend"); + for child in children { + worklist + .push(child) + .expect("one queue entry per reclaimed object"); + } + } + objects.len() - before + } + + /// Record that the provider refused to release a placement. + pub(crate) fn note_leaked_placement(&self) { + let _ = + self.leaked_placements + .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + Some(current.saturating_add(1)) + }); + } + + /// Hold a write claim on one object, so a test can create the exact + /// contention a concurrent operation creates. + /// + /// Ordinary `pub(crate)` API for the same reason + /// [`RamProvider::inject`](crate::provider::ram::RamProvider::inject) is + /// ordinary API: the alternative is a racy test that only sometimes + /// exercises the path it names, and contention is precisely the axis this + /// service was nondeterministic along. + pub(crate) fn claim_object(&self, id: ObjectId) -> Option> { + self.objects.try_write(id.key()).ok() + } + + /// Record that a reclamation walk stopped before it reached the whole + /// unreachable subtree. + pub(crate) fn note_abandoned_reclaim(&self) { + let _ = + self.abandoned_reclaims + .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + Some(current.saturating_add(1)) + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The namespace under test uses a `u32` placement key: these tests are + /// about the DAG, and the key's only job here is to be carried. + type Ns = Namespace; + + fn name(s: &str) -> Name { + Name::new(s).expect("test names are valid") + } + + /// The namespace must not spend the `serve` future's stack on state that + /// scales with the volume. + /// + /// Before the arenas moved to the heap this type was **130,944 bytes**, + /// which is why the object bound and the 4 MiB shared stack were one + /// number and why an 8-byte growth in an unrelated record overflowed that + /// stack on metal (PR #958). The ceiling here is deliberately loose: it + /// pins the *property* (handles and counters only, no arena) rather than + /// an exact figure, so adding a counter is free and re-inlining an arena + /// fails. + #[test] + fn the_namespace_holds_no_volume_scaled_arena_inline() { + let bytes = core::mem::size_of::(); + assert!( + bytes <= 1024, + "namespace grew to {bytes} B: a volume-scaled arena is inline again" + ); + } + + /// The commit-path snapshot must not put the volume on the stack either. + /// + /// Inline this was **119,832 bytes**, its edge table alone 114,688, and + /// unlike the decoded image it is built on *every commit* rather than at + /// mount. Pins the property, not a figure. + #[test] + fn the_durable_snapshot_holds_no_table_inline() { + let bytes = core::mem::size_of::>(); + assert!( + bytes <= 512, + "DurableGraph grew to {bytes} B: a volume-scaled table is inline again" + ); + } + + /// A root directory plus the namespace holding it. + fn ns_with_root() -> (Ns, ObjectId) { + let mut ns = Ns::new(); + let root = ns.create_dir_object().unwrap(); + // A root is reached by an authority edge, not a name; the engine + // marks that with a grant, which these tests stand in for. + ns.object_mut(root).unwrap().grants_inc(); + (ns, root) + } + + // ── names ─────────────────────────────────────────────────────────────── + + #[test] + fn valid_names_round_trip() { + for s in ["a", "main.rs", "hello-again", "Ünïcødé"] { + let n = name(s); + assert_eq!(n.as_str(), s); + assert_eq!(n.as_bytes(), s.as_bytes()); + assert_eq!(n.len(), s.len()); + assert!(!n.is_empty()); + } + } + + #[test] + fn empty_and_oversized_names_are_refused() { + assert_eq!(Name::new("").unwrap_err(), FsError::InvalidName); + let too_long = "x".repeat(crate::bounds::MAX_NAME_LEN + 1); + assert_eq!(Name::new(&too_long).unwrap_err(), FsError::InvalidName); + // Exactly at the bound is legal — the refusal must be off-by-none. + assert!(Name::new(&"x".repeat(crate::bounds::MAX_NAME_LEN)).is_ok()); + } + + #[test] + fn separator_nul_and_dot_names_are_refused() { + for bad in ["a/b", "/", "a\0b", ".", ".."] { + assert_eq!( + Name::new(bad).unwrap_err(), + FsError::InvalidName, + "{bad:?} must be refused" + ); + } + // A name merely *containing* dots is fine; only the two reserved + // components are refused. + assert!(Name::new("...").is_ok()); + assert!(Name::new(".hidden").is_ok()); + } + + #[test] + fn invalid_utf8_names_are_refused() { + assert_eq!( + Name::from_bytes(&[0xff, 0xfe]).unwrap_err(), + FsError::InvalidName + ); + } + + #[test] + fn names_compare_by_content_not_by_padding() { + // Two names built from different-length inputs must not compare + // equal just because their tails are both zero-padded. + assert_eq!(name("ab"), name("ab")); + assert_ne!(name("ab"), name("abc")); + } + + // ── creation and lookup ───────────────────────────────────────────────── + + #[test] + fn create_and_look_up_a_file() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + let edge = ns.link(root, &name("hello"), file).unwrap(); + let (found_edge, found) = ns.lookup(root, &name("hello")).unwrap(); + assert_eq!(found, file); + assert_eq!(found_edge, edge); + assert_eq!(ns.edge_child(edge).unwrap(), file); + assert_eq!(ns.object(file).unwrap().incoming(), 1); + } + + #[test] + fn lookup_of_an_absent_name_is_not_found() { + let (ns, root) = ns_with_root(); + assert_eq!( + ns.lookup(root, &name("nope")).unwrap_err(), + FsError::NotFound + ); + } + + #[test] + fn lookup_beneath_a_file_is_not_a_directory() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("f"), file).unwrap(); + assert_eq!( + ns.lookup(file, &name("x")).unwrap_err(), + FsError::NotDirectory + ); + } + + #[test] + fn duplicate_names_are_refused() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_file_object().unwrap(); + let b = ns.create_file_object().unwrap(); + ns.link(root, &name("dup"), a).unwrap(); + assert_eq!( + ns.link(root, &name("dup"), b).unwrap_err(), + FsError::AlreadyExists + ); + // The refused insert left nothing behind. + assert_eq!(ns.object(b).unwrap().incoming(), 0); + } + + // ── links ─────────────────────────────────────────────────────────────── + + #[test] + fn one_file_can_carry_two_names() { + let (mut ns, root) = ns_with_root(); + let projects = ns.create_dir_object().unwrap(); + ns.link(root, &name("projects"), projects).unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(projects, &name("main.rs"), file).unwrap(); + ns.link(root, &name("notes.rs"), file).unwrap(); + + assert_eq!(ns.object(file).unwrap().incoming(), 2); + assert_eq!(ns.lookup(projects, &name("main.rs")).unwrap().1, file); + assert_eq!(ns.lookup(root, &name("notes.rs")).unwrap().1, file); + } + + #[test] + fn unlinking_one_name_leaves_the_other() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("one"), file).unwrap(); + ns.link(root, &name("two"), file).unwrap(); + + let (_, child) = ns.detach(root, &name("one")).unwrap(); + assert_eq!(child, file); + assert_eq!(ns.object(file).unwrap().incoming(), 1); + assert_eq!( + ns.lookup(root, &name("one")).unwrap_err(), + FsError::NotFound + ); + assert_eq!(ns.lookup(root, &name("two")).unwrap().1, file); + } + + #[test] + fn directory_diamonds_are_lawful_and_count_each_edge() { + let (mut ns, root) = ns_with_root(); + let left = ns.create_dir_object().unwrap(); + let right = ns.create_dir_object().unwrap(); + let dir = ns.create_dir_object().unwrap(); + ns.link(root, &name("left"), left).unwrap(); + ns.link(root, &name("right"), right).unwrap(); + let first = ns.link(left, &name("shared"), dir).unwrap(); + let second = ns.link(right, &name("shared"), dir).unwrap(); + + assert_eq!(ns.object(dir).unwrap().incoming(), 2); + assert_eq!(ns.edge_child(first).unwrap(), dir); + assert_eq!(ns.edge_child(second).unwrap(), dir); + let reachable = ns.collect_reachable(root).unwrap(); + assert_eq!(reachable.iter().filter(|id| **id == dir).count(), 1); + } + + /// Every directory but the root, each linked to the [`FAN`] directories + /// below it and the topmost linked from `root`: acyclic, as convergent + /// as the entry bound allows (the routes to the bottom grow + /// combinatorially), and it fills the object arena exactly — the root + /// plus `MAX_DIRECTORIES - 1` directories is every object slot. + const FAN: usize = 3; + + fn dense_layers(ns: &mut Ns, root: ObjectId) -> std::vec::Vec { + let mut dirs = std::vec::Vec::with_capacity(MAX_DIRECTORIES - 1); + for index in 0..MAX_DIRECTORIES - 1 { + let dir = ns.create_dir_object().unwrap(); + for below in index.saturating_sub(FAN)..index { + ns.link(dir, &name(&std::format!("to-{below}")), dirs[below]) + .unwrap(); + } + dirs.push(dir); + } + ns.link(root, &name("top"), dirs[dirs.len() - 1]).unwrap(); + dirs + } + + #[test] + fn dense_convergence_spends_worklist_capacity_once_per_object() { + let (mut ns, root) = ns_with_root(); + let dirs = dense_layers(&mut ns, root); + // An implementation that budgets queue entries rather than unique + // objects exhausts here despite the graph fitting the arena. + let reachable = ns.collect_reachable(root).unwrap(); + assert_eq!(reachable.len(), MAX_DIRECTORIES); + for dir in dirs { + assert_eq!(reachable.iter().filter(|id| **id == dir).count(), 1); + } + } + + /// A diamond is lawful, a back edge across it is not, and the proof + /// visits the shared directory once whichever route reaches it first. + #[test] + fn back_edge_across_a_diamond_is_refused_and_a_forward_link_is_lawful() { + let (mut ns, root) = ns_with_root(); + let left = ns.create_dir_object().unwrap(); + let right = ns.create_dir_object().unwrap(); + let shared = ns.create_dir_object().unwrap(); + let deeper = ns.create_dir_object().unwrap(); + ns.link(root, &name("left"), left).unwrap(); + ns.link(root, &name("right"), right).unwrap(); + ns.link(left, &name("shared"), shared).unwrap(); + ns.link(right, &name("shared"), shared).unwrap(); + ns.link(shared, &name("deeper"), deeper).unwrap(); + + // `left` is reachable from `deeper` only through the diamond's + // second route — the proof must follow it, not stop at the first + // visit of `shared`. + assert_eq!( + ns.link(deeper, &name("up-left"), left).unwrap_err(), + FsError::Cycle + ); + assert_eq!( + ns.link(deeper, &name("up-root"), root).unwrap_err(), + FsError::Cycle + ); + // A second name for the shared base from the root is a lawful + // forward projection. + ns.link(root, &name("shared2"), shared).unwrap(); + assert_eq!(ns.object(shared).unwrap().incoming(), 3); + assert_eq!(ns.object(left).unwrap().incoming(), 1); + assert_eq!(ns.lookup(deeper, &name("up-left")), Err(FsError::NotFound)); + } + + /// Moving a directory beneath its own descendant is a cycle even when + /// the descendant is reached through two routes. + #[test] + fn subtree_move_into_its_own_descendant_through_a_diamond_is_refused() { + let (mut ns, root) = ns_with_root(); + let outer = ns.create_dir_object().unwrap(); + let a = ns.create_dir_object().unwrap(); + let b = ns.create_dir_object().unwrap(); + let deeper = ns.create_dir_object().unwrap(); + ns.link(root, &name("outer"), outer).unwrap(); + ns.link(outer, &name("a"), a).unwrap(); + ns.link(outer, &name("b"), b).unwrap(); + ns.link(a, &name("deeper"), deeper).unwrap(); + ns.link(b, &name("deeper"), deeper).unwrap(); + + assert_eq!( + ns.relocate(root, &name("outer"), deeper, &name("moved")) + .unwrap_err(), + FsError::Cycle + ); + assert_eq!(ns.lookup(root, &name("outer")).unwrap().1, outer); + assert_eq!(ns.lookup(deeper, &name("moved")), Err(FsError::NotFound)); + assert_eq!(ns.object(outer).unwrap().incoming(), 1); + } + + /// The full arena is a lawful footprint: a walk keyed by directory + /// slot never runs out of room before the arena does, and a claim over + /// every directory — more than a byte can index — keeps its positions. + #[test] + fn dense_convergent_footprint_fits_the_directory_bound_exactly() { + let (mut ns, root) = ns_with_root(); + let dirs = dense_layers(&mut ns, root); + // One more forward edge across the whole footprint is lawful; the + // matching back edge is a cycle. Neither may report `Range`. + let last = dirs[dirs.len() - 1]; + let first = dirs[0]; + ns.link(last, &name("forward"), first).unwrap(); + assert_eq!( + ns.link(first, &name("back"), last).unwrap_err(), + FsError::Cycle + ); + } + + /// A child directory that entered the footprint after it was observed + /// and before it was claimed is reported as contention, never walked. + #[test] + fn reachability_reports_busy_when_a_child_directory_escapes_the_claimed_footprint() { + let (ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + let stray = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + ns.link(a, &name("stray"), stray).unwrap(); + + // Claim only `a`: `stray` is a directory child outside the claim. + let claims = ns.claim_directories(&[a]).unwrap(); + assert_eq!( + ns.is_reachable_claimed(&claims, a, root), + Err(FsError::Busy) + ); + } + + #[test] + fn directory_self_links_and_back_edges_are_refused_without_mutation() { + let (mut ns, root) = ns_with_root(); + let outer = ns.create_dir_object().unwrap(); + let inner = ns.create_dir_object().unwrap(); + ns.link(root, &name("outer"), outer).unwrap(); + ns.link(outer, &name("inner"), inner).unwrap(); + + assert_eq!( + ns.link(inner, &name("back"), outer).unwrap_err(), + FsError::Cycle + ); + assert_eq!( + ns.link(inner, &name("self"), inner).unwrap_err(), + FsError::Cycle + ); + assert_eq!(ns.object(outer).unwrap().incoming(), 1); + assert_eq!(ns.object(inner).unwrap().incoming(), 1); + assert_eq!(ns.lookup(inner, &name("back")), Err(FsError::NotFound)); + } + + #[test] + fn removing_an_edge_retires_only_that_edge() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + let first = ns.link(root, &name("one"), file).unwrap(); + let second = ns.link(root, &name("two"), file).unwrap(); + + ns.detach(root, &name("one")).unwrap(); + // The removed edge fails closed; its sibling is untouched. + assert_eq!(ns.edge_child(first).unwrap_err(), FsError::AccessLost); + assert_eq!(ns.edge_child(second).unwrap(), file); + } + + // ── rename ────────────────────────────────────────────────────────────── + + #[test] + fn rename_changes_the_label_and_keeps_the_edge() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + let edge = ns.link(root, &name("before"), file).unwrap(); + + let renamed = ns.rename(root, &name("before"), &name("after")).unwrap(); + // Same edge identity: a handle proving through it stays valid. + assert_eq!(renamed, edge); + assert_eq!(ns.edge_child(edge).unwrap(), file); + assert_eq!( + ns.lookup(root, &name("before")).unwrap_err(), + FsError::NotFound + ); + assert_eq!(ns.lookup(root, &name("after")).unwrap().1, file); + assert_eq!(ns.object(file).unwrap().incoming(), 1); + } + + /// One claimed directory is not a namespace-wide mutex. A mutation in a + /// disjoint directory completes while the exact conflicting directory + /// reports retryable backpressure, with neither operation changing the + /// other's entries. + #[test] + fn disjoint_directory_mutations_do_not_serialize_globally() { + let (mut ns, root) = ns_with_root(); + let left = ns.create_dir_object().unwrap(); + let right = ns.create_dir_object().unwrap(); + ns.link(root, &name("left"), left).unwrap(); + ns.link(root, &name("right"), right).unwrap(); + let left_file = ns.create_file_object().unwrap(); + let right_file = ns.create_file_object().unwrap(); + ns.link(left, &name("old"), left_file).unwrap(); + ns.link(right, &name("old"), right_file).unwrap(); + + let left_claim = ns.dir_entries_mut(left).unwrap(); + std::thread::scope(|scope| { + let worker = scope.spawn(|| { + assert_eq!( + ns.rename(left, &name("old"), &name("blocked")), + Err(FsError::Busy) + ); + ns.rename(right, &name("old"), &name("new")).unwrap(); + }); + worker.join().unwrap(); + }); + drop(left_claim); + + assert_eq!(ns.lookup(left, &name("old")).unwrap().1, left_file); + assert_eq!(ns.lookup(left, &name("blocked")), Err(FsError::NotFound)); + assert_eq!(ns.lookup(right, &name("new")).unwrap().1, right_file); + } + + #[test] + fn durable_graph_refuses_contention_then_copies_one_complete_footprint() { + let (ns, root) = ns_with_root(); + let dir = ns.create_dir_object().unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("left"), dir).unwrap(); + ns.link(root, &name("right"), dir).unwrap(); + ns.link(dir, &name("file"), file).unwrap(); + + let held = ns.object_mut(file).unwrap(); + assert!(matches!(ns.durable_graph(root, 16), Err(FsError::Busy))); + drop(held); + + let graph = ns.durable_graph(root, 16).unwrap(); + assert_eq!( + graph.objects.len(), + 3, + "the diamond copies each object once" + ); + assert_eq!(graph.edges.len(), 3, "each named edge remains distinct"); + assert_eq!(graph.root, ns.object(root).unwrap().stable()); + } + + #[test] + fn rename_onto_an_existing_name_is_refused() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_file_object().unwrap(); + let b = ns.create_file_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + ns.link(root, &name("b"), b).unwrap(); + assert_eq!( + ns.rename(root, &name("a"), &name("b")).unwrap_err(), + FsError::AlreadyExists + ); + // Neither name moved. + assert_eq!(ns.lookup(root, &name("a")).unwrap().1, a); + assert_eq!(ns.lookup(root, &name("b")).unwrap().1, b); + } + + #[test] + fn rename_to_the_same_name_is_permitted() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_file_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + // Self-rename must not trip the AlreadyExists check against itself. + assert!(ns.rename(root, &name("a"), &name("a")).is_ok()); + assert_eq!(ns.lookup(root, &name("a")).unwrap().1, a); + } + + // ── move ──────────────────────────────────────────────────────────────── + + #[test] + fn move_retires_the_source_edge_and_keeps_the_object() { + let (mut ns, root) = ns_with_root(); + let dst = ns.create_dir_object().unwrap(); + ns.link(root, &name("dst"), dst).unwrap(); + let file = ns.create_file_object().unwrap(); + let src_edge = ns.link(root, &name("f"), file).unwrap(); + + let (retired, created) = ns.relocate(root, &name("f"), dst, &name("moved")).unwrap(); + assert_eq!(retired, src_edge); + // Source edge fails closed; the object itself is untouched. + assert_eq!(ns.edge_child(src_edge).unwrap_err(), FsError::AccessLost); + assert_eq!(ns.edge_child(created).unwrap(), file); + assert_eq!(ns.object(file).unwrap().incoming(), 1); + assert_eq!(ns.lookup(root, &name("f")).unwrap_err(), FsError::NotFound); + assert_eq!(ns.lookup(dst, &name("moved")).unwrap().1, file); + } + + #[test] + fn contended_destination_cannot_partially_remove_a_move_source() { + let (ns, root) = ns_with_root(); + let dst = ns.create_dir_object().unwrap(); + ns.link(root, &name("dst"), dst).unwrap(); + let file = ns.create_file_object().unwrap(); + let source = ns.link(root, &name("f"), file).unwrap(); + + let held_destination = ns.dir_entries_mut(dst).unwrap(); + assert_eq!( + ns.relocate(root, &name("f"), dst, &name("moved")), + Err(FsError::Busy) + ); + assert_eq!(ns.edge_child(source).unwrap(), file); + assert_eq!(ns.lookup(root, &name("f")).unwrap().1, file); + drop(held_destination); + assert_eq!(ns.lookup(dst, &name("moved")), Err(FsError::NotFound)); + } + + #[test] + fn opposing_directory_links_cannot_commit_a_cycle() { + let (ns, _root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + let b = ns.create_dir_object().unwrap(); + let start = std::sync::Barrier::new(3); + let (left, right) = std::thread::scope(|scope| { + let left = scope.spawn(|| { + start.wait(); + ns.link(a, &name("b"), b) + }); + let right = scope.spawn(|| { + start.wait(); + ns.link(b, &name("a"), a) + }); + start.wait(); + (left.join().unwrap(), right.join().unwrap()) + }); + + // What must never happen is both directions committing. Both racers + // *losing* is lawful: a claim the other side holds answers `Busy` + // (bounded backpressure, the engine's contract), and a reader in the + // footprint snapshot can refuse a writer just as a writer refuses a + // reader, so the race may leave neither edge published. + assert!( + !(left.is_ok() && right.is_ok()), + "both directions committed" + ); + for outcome in [&left, &right] { + if let Err(error) = outcome { + assert!( + matches!(error, FsError::Busy | FsError::AccessLost | FsError::Cycle), + "a racer may only lose to contention or to the other's edge: {error:?}" + ); + } + } + // Driven to a fixed point, exactly one direction stands and the other + // is a cycle. + let mut forward = left.map(|_| ()); + if forward.is_err() && right.is_err() { + forward = ns.link(a, &name("b"), b).map(|_| ()); + } + let refused = if forward.is_ok() { + ns.link(b, &name("a"), a) + } else { + ns.link(a, &name("b"), b) + }; + assert_eq!(refused, Err(FsError::Cycle)); + } + + #[test] + fn move_leaves_a_parallel_link_working() { + let (mut ns, root) = ns_with_root(); + let dst = ns.create_dir_object().unwrap(); + ns.link(root, &name("dst"), dst).unwrap(); + let file = ns.create_file_object().unwrap(); + let src_edge = ns.link(root, &name("f"), file).unwrap(); + let parallel = ns.link(root, &name("also-f"), file).unwrap(); + + ns.relocate(root, &name("f"), dst, &name("moved")).unwrap(); + assert_eq!(ns.edge_child(src_edge).unwrap_err(), FsError::AccessLost); + // The other route to the same object is unaffected. + assert_eq!(ns.edge_child(parallel).unwrap(), file); + assert_eq!(ns.object(file).unwrap().incoming(), 2); + } + + #[test] + fn move_within_one_directory_renames_by_retiring_the_edge() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + let edge = ns.link(root, &name("a"), file).unwrap(); + + let (retired, created) = ns.relocate(root, &name("a"), root, &name("b")).unwrap(); + assert_eq!(retired, edge); + assert_ne!(created, edge); + assert_eq!(ns.edge_child(edge).unwrap_err(), FsError::AccessLost); + assert_eq!(ns.lookup(root, &name("b")).unwrap().1, file); + assert_eq!(ns.object(file).unwrap().incoming(), 1); + } + + #[test] + fn move_to_the_identical_place_is_a_no_op() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + let edge = ns.link(root, &name("a"), file).unwrap(); + let (retired, created) = ns.relocate(root, &name("a"), root, &name("a")).unwrap(); + // Nothing changed, so nothing was retired. + assert_eq!(retired, edge); + assert_eq!(created, edge); + assert_eq!(ns.edge_child(edge).unwrap(), file); + } + + #[test] + fn move_onto_an_existing_name_is_refused_and_changes_nothing() { + let (mut ns, root) = ns_with_root(); + let dst = ns.create_dir_object().unwrap(); + ns.link(root, &name("dst"), dst).unwrap(); + let a = ns.create_file_object().unwrap(); + let b = ns.create_file_object().unwrap(); + let a_edge = ns.link(root, &name("a"), a).unwrap(); + ns.link(dst, &name("taken"), b).unwrap(); + + assert_eq!( + ns.relocate(root, &name("a"), dst, &name("taken")) + .unwrap_err(), + FsError::AlreadyExists + ); + // The source edge survived the refusal. + assert_eq!(ns.edge_child(a_edge).unwrap(), a); + assert_eq!(ns.lookup(root, &name("a")).unwrap().1, a); + assert_eq!(ns.lookup(dst, &name("taken")).unwrap().1, b); + } + + #[test] + fn moving_a_directory_into_itself_is_a_cycle() { + let (mut ns, root) = ns_with_root(); + let dir = ns.create_dir_object().unwrap(); + ns.link(root, &name("d"), dir).unwrap(); + assert_eq!( + ns.relocate(root, &name("d"), dir, &name("self")) + .unwrap_err(), + FsError::Cycle + ); + } + + #[test] + fn moving_a_directory_beneath_its_own_descendant_is_a_cycle() { + let (mut ns, root) = ns_with_root(); + let outer = ns.create_dir_object().unwrap(); + let inner = ns.create_dir_object().unwrap(); + let deeper = ns.create_dir_object().unwrap(); + ns.link(root, &name("outer"), outer).unwrap(); + ns.link(outer, &name("inner"), inner).unwrap(); + ns.link(inner, &name("deeper"), deeper).unwrap(); + + assert_eq!( + ns.relocate(root, &name("outer"), deeper, &name("loop")) + .unwrap_err(), + FsError::Cycle + ); + // The tree is intact after the refusal. + assert_eq!(ns.lookup(root, &name("outer")).unwrap().1, outer); + assert_eq!(ns.lookup(inner, &name("deeper")).unwrap().1, deeper); + } + + #[test] + fn moving_a_directory_to_an_unrelated_place_is_permitted() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + let b = ns.create_dir_object().unwrap(); + let child = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + ns.link(root, &name("b"), b).unwrap(); + ns.link(a, &name("child"), child).unwrap(); + + // `b` is not under `child`, so this is a legal move. + assert!(ns.relocate(a, &name("child"), b, &name("child")).is_ok()); + assert_eq!(ns.lookup(b, &name("child")).unwrap().1, child); + assert_eq!(ns.object(child).unwrap().incoming(), 1); + } + + #[test] + fn move_into_a_file_is_refused() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + let victim = ns.create_file_object().unwrap(); + ns.link(root, &name("f"), file).unwrap(); + ns.link(root, &name("v"), victim).unwrap(); + assert_eq!( + ns.relocate(root, &name("v"), file, &name("x")).unwrap_err(), + FsError::NotDirectory + ); + } + + // ── unlink and directories ────────────────────────────────────────────── + + #[test] + fn unlinking_a_non_empty_directory_is_refused() { + let (mut ns, root) = ns_with_root(); + let dir = ns.create_dir_object().unwrap(); + ns.link(root, &name("d"), dir).unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(dir, &name("f"), file).unwrap(); + + assert_eq!( + ns.unlink_empty(root, &name("d")).unwrap_err(), + FsError::NotEmpty + ); + assert_eq!(ns.lookup(root, &name("d")).unwrap().1, dir); + + // Emptying it first makes the unlink succeed. + ns.detach(dir, &name("f")).unwrap(); + assert!(ns.unlink_empty(root, &name("d")).is_ok()); + } + + #[test] + fn unlink_rechecks_directory_emptiness_under_the_child_claim() { + let (ns, root) = ns_with_root(); + let dir = ns.create_dir_object().unwrap(); + let edge = ns.link(root, &name("d"), dir).unwrap(); + + let held_child = ns.dir_entries_mut(dir).unwrap(); + assert_eq!(ns.unlink_empty(root, &name("d")), Err(FsError::Busy)); + assert_eq!(ns.edge_child(edge).unwrap(), dir); + drop(held_child); + assert!(ns.unlink_empty(root, &name("d")).is_ok()); + } + + // ── listing ───────────────────────────────────────────────────────────── + + #[test] + fn listing_pages_without_losing_or_repeating_entries() { + let (mut ns, root) = ns_with_root(); + let total = MAX_LIST_PAGE + 3; + let mut created = std::vec::Vec::new(); + for i in 0..total { + let file = ns.create_file_object().unwrap(); + let mut buf = [0u8; 8]; + let n = write_index(&mut buf, i); + ns.link(root, &Name::from_bytes(&buf[..n]).unwrap(), file) + .unwrap(); + created.push(file); + } + + let mut seen = std::vec::Vec::new(); + let mut cursor = 0u32; + loop { + let (page, next) = ns.list(root, cursor).unwrap(); + assert!(page.len() <= MAX_LIST_PAGE); + for entry in page.iter() { + seen.push(entry.object); + } + match next { + Some(c) => { + // The cursor must advance, or paging would not terminate. + assert!(c > cursor || cursor == 0); + cursor = c; + } + None => break, + } + } + assert_eq!(seen.len(), total); + for object in &created { + assert!(seen.contains(object)); + } + } + + #[test] + fn listing_an_empty_directory_yields_one_empty_page() { + let (ns, root) = ns_with_root(); + let (page, next) = ns.list(root, 0).unwrap(); + assert!(page.is_empty()); + assert_eq!(next, None); + } + + #[test] + fn listing_a_file_is_refused() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("f"), file).unwrap(); + assert_eq!(ns.list(file, 0).unwrap_err(), FsError::NotDirectory); + } + + #[test] + fn listing_reports_each_entry_kind() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + let dir = ns.create_dir_object().unwrap(); + ns.link(root, &name("f"), file).unwrap(); + ns.link(root, &name("d"), dir).unwrap(); + let (page, _) = ns.list(root, 0).unwrap(); + let kinds: std::vec::Vec<_> = page.iter().map(|e| (e.name.as_str(), e.kind)).collect(); + assert!(kinds.contains(&("f", ObjectKind::File))); + assert!(kinds.contains(&("d", ObjectKind::Directory))); + } + + // ── exhaustion ────────────────────────────────────────────────────────── + + #[test] + fn a_full_directory_refuses_more_edges() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + for i in 0..MAX_DIR_ENTRIES { + let mut buf = [0u8; 8]; + let n = write_index(&mut buf, i); + ns.link(root, &Name::from_bytes(&buf[..n]).unwrap(), file) + .unwrap(); + } + let extra = ns.create_file_object().unwrap(); + assert_eq!( + ns.link(root, &name("overflow"), extra).unwrap_err(), + FsError::OutOfEdges + ); + } + + /// Directories are bounded by the object arena alone: every slot can be + /// a directory, and the refusal after the last is the object arena's, + /// files included. + #[test] + fn directories_are_bounded_by_the_object_arena_alone() { + let mut ns = Ns::new(); + let mut made = 0usize; + while ns.create_dir_object().is_ok() { + made += 1; + assert!(made <= MAX_OBJECTS, "the arena must stop at its bound"); + } + assert_eq!(made, MAX_OBJECTS); + assert_eq!(ns.create_file_object().unwrap_err(), FsError::OutOfObjects); + } + + // ── the parent index ──────────────────────────────────────────────────── + + /// Every live object's parent row names exactly the forward edges that + /// point at it, in some order: the index is the forward graph read + /// backwards, nothing more and nothing less. + fn assert_index_matches_forward_edges(ns: &Ns) { + let objects: std::vec::Vec = ns.objects().map(|(id, _)| id).collect(); + for &object in &objects { + let mut forward = std::vec::Vec::new(); + for &parent in &objects { + let Ok(entries) = ns.dir_entries(parent) else { + continue; + }; + for (key, entry) in entries.iter() { + if entry.child == object { + forward.push(EdgeId { parent, entry: key }); + } + } + } + let mut reverse: std::vec::Vec = + ns.incoming_edges(object).unwrap().iter().copied().collect(); + let order = |e: &EdgeId| (e.parent.to_bits(), e.entry.to_bits()); + forward.sort_by_key(order); + reverse.sort_by_key(order); + assert_eq!(forward, reverse, "object {object:?}"); + assert_eq!(forward.len() as u32, ns.object(object).unwrap().incoming()); + } + } + + #[test] + fn the_parent_index_follows_every_edge_change() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + let b = ns.create_dir_object().unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + ns.link(root, &name("b"), b).unwrap(); + ns.link(a, &name("f"), file).unwrap(); + ns.link(b, &name("also-f"), file).unwrap(); + ns.link(b, &name("twice"), file).unwrap(); + assert_index_matches_forward_edges(&ns); + assert_eq!(ns.incoming_edges(file).unwrap().len(), 3); + + // A rename keeps the edge, so the row stays right untouched and the + // new name is what an upward walk reads. + ns.rename(b, &name("twice"), &name("renamed")).unwrap(); + assert_index_matches_forward_edges(&ns); + + // A move swaps one edge for another in place. + ns.relocate(b, &name("renamed"), a, &name("moved")).unwrap(); + assert_index_matches_forward_edges(&ns); + + // Removing names drops their reverse entries, spill and all. + ns.detach(a, &name("moved")).unwrap(); + ns.unlink_empty(b, &name("also-f")).unwrap(); + assert_index_matches_forward_edges(&ns); + assert_eq!(ns.incoming_edges(file).unwrap().len(), 1); + + // Reclaiming a directory drops the edges it held from its children. + let (_, dropped) = ns.detach(root, &name("a")).unwrap(); + let mut placements: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + let mut reclaimed: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + ns.reclaim(dropped, &mut placements, &mut reclaimed); + assert!(!ns.exists(a)); + assert!(!ns.exists(file), "its last name went with `a`"); + assert_index_matches_forward_edges(&ns); + } + + /// The parent row is claimed on its own: a file transaction holds the + /// object's claim across provider I/O, and moving or renaming that file + /// meanwhile must not turn `Busy` because of the index. + #[test] + fn a_file_mid_transaction_can_still_be_moved() { + let (mut ns, root) = ns_with_root(); + let dir = ns.create_dir_object().unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("dir"), dir).unwrap(); + ns.link(root, &name("f"), file).unwrap(); + let transaction = ns.object_mut(file).unwrap(); + ns.relocate(root, &name("f"), dir, &name("f")).unwrap(); + ns.rename(dir, &name("f"), &name("g")).unwrap(); + drop(transaction); + assert_index_matches_forward_edges(&ns); + } + + /// Parents are found upward, scoped to the anchor, by shortest route. + #[test] + fn parents_walk_upward_to_the_anchor_by_the_shortest_route() { + let (mut ns, root) = ns_with_root(); + let deep = ns.create_dir_object().unwrap(); + let deeper = ns.create_dir_object().unwrap(); + let near = ns.create_dir_object().unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("deep"), deep).unwrap(); + ns.link(deep, &name("deeper"), deeper).unwrap(); + ns.link(root, &name("near"), near).unwrap(); + // `deeper` is also reachable straight from the root. + ns.link(root, &name("short"), deeper).unwrap(); + ns.link(deeper, &name("f"), file).unwrap(); + ns.link(near, &name("f"), file).unwrap(); + let routes = |anchor: ObjectId| { + let (page, next) = ns.parents(anchor, file, 0, MAX_LIST_PAGE).unwrap(); + assert_eq!(next, None); + page.iter() + .map(|parent| { + parent.path.iter().map(|n| n.as_str()).collect::>().join("/") + }) + .collect::>() + }; + assert_eq!(routes(root), ["short", "near"]); + // Anchored at `deep`, only the parent below it is visible. + assert_eq!(routes(deep), ["deeper"]); + // A one-record page pages. + let (page, next) = ns.parents(root, file, 0, 1).unwrap(); + assert_eq!((page.len(), next), (1, Some(1))); + let (page, next) = ns.parents(root, file, 1, 1).unwrap(); + assert_eq!((page.len(), next), (1, None)); + // An object's own route ends in its own name, by the shortest way; + // between two of one length, the parent linked first. + let spelled = |anchor: ObjectId| { + ns.route_up(anchor, file) + .unwrap() + .map(|path| path.iter().map(|n| n.as_str()).collect::>().join("/")) + }; + assert_eq!(spelled(root).as_deref(), Some("short/f")); + assert_eq!(spelled(deep).as_deref(), Some("deeper/f")); + assert_eq!(spelled(near).as_deref(), Some("f")); + assert_eq!(ns.find_stable(ns.object(file).unwrap().stable()), Some(file)); + } + + // ── reclamation ───────────────────────────────────────────────────────── + + #[test] + fn reclaim_releases_an_unreachable_file_and_its_placement() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("f"), file).unwrap(); + ns.object_mut(file).unwrap().set_storage(7); + + let (_, child) = ns.detach(root, &name("f")).unwrap(); + let mut placements: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + let mut objects: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + assert_eq!(ns.reclaim(child, &mut placements, &mut objects), 1); + assert_eq!(placements.as_slice(), &[7]); + assert_eq!(objects.as_slice(), &[file]); + assert!(!ns.exists(file)); + } + + #[test] + fn contended_reclaim_keeps_exact_custody_for_retry() { + let (ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("f"), file).unwrap(); + ns.object_mut(file).unwrap().set_storage(7); + let (_, child) = ns.detach(root, &name("f")).unwrap(); + let held = ns.object_mut(file).unwrap(); + let mut placements: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + let mut objects: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + + assert!(matches!( + ns.reclaim_one(child, &mut placements, &mut objects), + Err(FsError::Busy) + )); + assert!(placements.is_empty() && objects.is_empty()); + assert!(ns.exists(file), "a busy object remains in its exact slot"); + + drop(held); + assert_eq!(ns.reclaim(child, &mut placements, &mut objects), 1); + assert_eq!(placements.as_slice(), &[7]); + assert!(!ns.exists(file)); + } + + #[test] + fn reclaim_spares_an_object_that_still_has_a_name() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("one"), file).unwrap(); + ns.link(root, &name("two"), file).unwrap(); + ns.object_mut(file).unwrap().set_storage(9); + + let (_, child) = ns.detach(root, &name("one")).unwrap(); + let mut placements: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + let mut objects: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + assert_eq!(ns.reclaim(child, &mut placements, &mut objects), 0); + assert!(placements.is_empty()); + assert!(ns.exists(file)); + } + + #[test] + fn reclaim_spares_an_object_that_still_has_a_grant() { + let (mut ns, root) = ns_with_root(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("f"), file).unwrap(); + ns.object_mut(file).unwrap().grants_inc(); + + let (_, child) = ns.detach(root, &name("f")).unwrap(); + let mut placements: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + let mut objects: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + assert_eq!(ns.reclaim(child, &mut placements, &mut objects), 0); + assert!(ns.exists(file)); + + // Dropping the grant too makes it collectable. + ns.object_mut(file).unwrap().grants_dec(); + assert_eq!(ns.reclaim(child, &mut placements, &mut objects), 1); + assert!(!ns.exists(file)); + } + + #[test] + fn reclaiming_a_stranded_root_takes_its_whole_subtree() { + // Retiring a private root can strand a populated subtree; leaving it + // in the arenas would leak every object beneath it. + let (mut ns, root) = ns_with_root(); + let dir = ns.create_dir_object().unwrap(); + ns.link(root, &name("d"), dir).unwrap(); + let a = ns.create_file_object().unwrap(); + let b = ns.create_file_object().unwrap(); + ns.link(dir, &name("a"), a).unwrap(); + ns.link(dir, &name("b"), b).unwrap(); + ns.object_mut(a).unwrap().set_storage(1); + ns.object_mut(b).unwrap().set_storage(2); + + let objects_before = ns.object_count(); + let dirs_before = ns.directory_count(); + + // Retire the root's only route. + ns.object_mut(root).unwrap().grants_dec(); + let mut placements: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + let mut reclaimed: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + let released = ns.reclaim(root, &mut placements, &mut reclaimed); + + assert_eq!(released, 4, "root, dir, and both files"); + assert_eq!(objects_before - ns.object_count(), 4); + assert_eq!(dirs_before - ns.directory_count(), 2); + assert!(placements.contains(&1) && placements.contains(&2)); + assert_eq!(placements.len(), 2, "each placement released exactly once"); + assert!(!ns.exists(root) && !ns.exists(dir) && !ns.exists(a) && !ns.exists(b)); + } + + #[test] + fn reclaiming_a_subtree_spares_a_file_linked_from_outside() { + let (mut ns, root) = ns_with_root(); + let doomed = ns.create_dir_object().unwrap(); + ns.link(root, &name("doomed"), doomed).unwrap(); + let shared = ns.create_file_object().unwrap(); + ns.link(doomed, &name("shared"), shared).unwrap(); + // A second name from outside the doomed subtree. + ns.link(root, &name("kept"), shared).unwrap(); + + let (_, child) = ns.detach(root, &name("doomed")).unwrap(); + let mut placements: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + let mut objects: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + ns.reclaim(child, &mut placements, &mut objects); + + assert!(!ns.exists(doomed)); + // The outside name kept the file alive. + assert!(ns.exists(shared)); + assert_eq!(ns.object(shared).unwrap().incoming(), 1); + assert!(placements.is_empty()); + assert_eq!(ns.lookup(root, &name("kept")).unwrap().1, shared); + } + + #[test] + fn detaching_one_directory_route_preserves_a_diamond_until_the_last_route() { + let (mut ns, root) = ns_with_root(); + let left = ns.create_dir_object().unwrap(); + let right = ns.create_dir_object().unwrap(); + let shared = ns.create_dir_object().unwrap(); + let leaf = ns.create_file_object().unwrap(); + ns.link(root, &name("left"), left).unwrap(); + ns.link(root, &name("right"), right).unwrap(); + let left_edge = ns.link(left, &name("shared"), shared).unwrap(); + let right_edge = ns.link(right, &name("shared"), shared).unwrap(); + ns.link(shared, &name("leaf"), leaf).unwrap(); + + let (_, child) = ns.detach(left, &name("shared")).unwrap(); + let mut placements: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + let mut objects: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + assert_eq!(ns.reclaim(child, &mut placements, &mut objects), 0); + assert_eq!(ns.edge_child(left_edge), Err(FsError::AccessLost)); + assert_eq!(ns.edge_child(right_edge), Ok(shared)); + assert_eq!(ns.lookup(shared, &name("leaf")).unwrap().1, leaf); + + let (_, child) = ns.detach(right, &name("shared")).unwrap(); + assert_eq!(ns.reclaim(child, &mut placements, &mut objects), 2); + assert!(!ns.exists(shared)); + assert!(!ns.exists(leaf)); + assert!(ns.exists(left) && ns.exists(right)); + } + + #[test] + fn leaked_placement_counter_records_provider_refusals() { + let (mut ns, _root) = ns_with_root(); + assert_eq!(ns.leaked_placements(), 0); + ns.note_leaked_placement(); + ns.note_leaked_placement(); + assert_eq!(ns.leaked_placements(), 2); + } + + /// Write `value` as decimal ASCII into `buf`, returning the byte count. + /// Test-local so name generation needs no formatting machinery. + fn write_index(buf: &mut [u8; 8], value: usize) -> usize { + if value == 0 { + buf[0] = b'0'; + return 1; + } + let mut digits = [0u8; 8]; + let mut n = 0; + let mut v = value; + while v > 0 && n < digits.len() { + digits[n] = b'0' + (v % 10) as u8; + v /= 10; + n += 1; + } + for i in 0..n { + buf[i] = digits[n - 1 - i]; + } + n + } +} diff --git a/native/services/dagfs/src/object.rs b/native/services/dagfs/src/object.rs new file mode 100644 index 0000000..9db88c7 --- /dev/null +++ b/native/services/dagfs/src/object.rs @@ -0,0 +1,398 @@ +//! Stable object identity, kinds, revisions, and reclamation accounting. +//! +//! An object's identity is a [`SlotKey`] in the engine's object arena — not a +//! name and not a storage placement. That is what lets `rename` change a +//! label, `move` swap one containment edge for another, and `link` add a +//! second name, all without the object becoming a different object. Paths +//! are client-side traversal input; they are neither identity nor authority. +//! +//! # Reclamation +//! +//! Every object counts the two things that can keep it reachable: incoming +//! containment edges and live authority edges (grants). When both reach +//! zero the object is unreachable by construction — no route exists to open +//! it — and [`Object::is_reclaimable`] says so, at which point the engine +//! releases its storage and its arena slot. +//! +//! Outstanding *handles* deliberately do not keep an object alive. "No live +//! route means no read" is immediate: a +//! handle is a bounded proof over edges, so once the edges are gone the +//! handle fails closed on its next use. This is an intentional difference +//! from Unix, where an open file descriptor commonly keeps an unlinked +//! object alive. + +use crate::slot::SlotKey; + +pub use kore_dagfs::{ANY_REVISION, ObjectId, ObjectKind, ObjectSubtype, Revision, StableObjectId}; + +/// Service-private access to the generational naming key. +/// +/// The public contract exposes only stable wire bits. Reconstructing the slot +/// key here keeps the arena implementation out of the application API while +/// preserving the exact stale-generation checks used throughout the engine. +pub(crate) trait ObjectIdExt { + fn key(self) -> SlotKey; +} + +impl ObjectIdExt for ObjectId { + #[inline] + fn key(self) -> SlotKey { + SlotKey::from_bits(self.to_bits()) + } +} + +/// A handle into the directory child-list arena. +/// +/// Directory child lists live in their own arena rather than inline in +/// [`Object`], so a file object does not reserve directory-sized space (see +/// [`crate::bounds`]). Purely internal: a client never sees one. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct DirId(pub(crate) SlotKey); + +/// The revision a freshly created object carries. Above [`ANY_REVISION`] so +/// the sentinel is unambiguous. +pub const FIRST_REVISION: Revision = 1; + +/// What an object's kind-specific state is. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum Body { + /// A file's placement in the provider. `None` until the file first + /// needs bytes, so an empty file costs no storage. + File { storage: Option }, + /// A directory's child list. + Directory { dir: DirId }, +} + +/// One object record. +/// +/// Generic over the provider's placement key `K` +/// ([`StorageProvider::Key`](crate::provider::StorageProvider::Key)) so the +/// engine stays generic over its provider rather than erasing the key — +/// the seam's associated type reaches all the way into the object model, and +/// a key that grants no authority stays that way. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct Object { + stable: StableObjectId, + kind: ObjectKind, + /// What the object contains — data set at creation, never authority + /// ([`ObjectSubtype`]). Immutable for the object's life: `commit_len` + /// and every other mutation leave it alone. + subtype: ObjectSubtype, + body: Body, + len: u64, + revision: Revision, + /// Incoming containment edges — how many names place this object + /// somewhere. + incoming: u32, + /// Live authority edges rooted at this object — private roots and + /// direct grants. + grants: u32, +} + +impl Object { + /// A new plain file with no placement and no bytes. + pub(crate) const fn new_file() -> Self { + Self::new_file_with(ObjectSubtype::PLAIN) + } + + /// A new file stamped `subtype`, with no placement and no bytes. + pub(crate) const fn new_file_with(subtype: ObjectSubtype) -> Self { + Self { + stable: StableObjectId::INVALID, + kind: ObjectKind::File, + subtype, + body: Body::File { storage: None }, + len: 0, + revision: FIRST_REVISION, + incoming: 0, + grants: 0, + } + } + + /// A new directory owning the child list `dir`. + pub(crate) const fn new_dir(dir: DirId) -> Self { + Self { + stable: StableObjectId::INVALID, + kind: ObjectKind::Directory, + subtype: ObjectSubtype::PLAIN, + body: Body::Directory { dir }, + len: 0, + revision: FIRST_REVISION, + incoming: 0, + grants: 0, + } + } + + pub(crate) const fn with_stable(mut self, stable: StableObjectId) -> Self { + self.stable = stable; + self + } + + pub(crate) const fn stable(&self) -> StableObjectId { + self.stable + } + + pub(crate) fn restore_len_revision(&mut self, len: u64, revision: Revision) { + self.len = len; + self.revision = revision; + } + + #[inline] + pub(crate) const fn kind(&self) -> ObjectKind { + self.kind + } + + #[inline] + pub(crate) const fn subtype(&self) -> ObjectSubtype { + self.subtype + } + + #[inline] + pub(crate) const fn len(&self) -> u64 { + self.len + } + + #[inline] + pub(crate) const fn revision(&self) -> Revision { + self.revision + } + + #[inline] + pub(crate) const fn incoming(&self) -> u32 { + self.incoming + } + + #[inline] + pub(crate) const fn grants(&self) -> u32 { + self.grants + } + + /// This object's child-list handle, or `None` if it is a file. + #[inline] + pub(crate) const fn dir(&self) -> Option { + match self.body { + Body::Directory { dir } => Some(dir), + Body::File { .. } => None, + } + } + + /// This object's provider placement, or `None` if it is a directory or a + /// file that has never needed bytes. + #[inline] + pub(crate) const fn storage(&self) -> Option { + match self.body { + Body::File { storage } => storage, + Body::Directory { .. } => None, + } + } + + /// Record the placement the provider handed back. A file only. + pub(crate) fn set_storage(&mut self, key: K) { + if let Body::File { storage } = &mut self.body { + *storage = Some(key); + } + } + + /// Commit a new length and advance the revision. Called once, at the + /// end of a mutation that has already succeeded, so a refused or failed + /// request leaves the revision untouched and is invisible to readers. + pub(crate) fn commit_len(&mut self, len: u64) { + self.len = len; + self.revision = self.revision.saturating_add(1); + } + + /// `true` if `expected` permits a write against the current revision: + /// either it is [`ANY_REVISION`] (unconditional) or it matches exactly. + #[inline] + pub(crate) const fn revision_admits(&self, expected: Revision) -> bool { + expected == ANY_REVISION || expected == self.revision + } + + pub(crate) fn incoming_inc(&mut self) { + self.incoming = self.incoming.saturating_add(1); + } + + pub(crate) fn incoming_dec(&mut self) { + self.incoming = self.incoming.saturating_sub(1); + } + + pub(crate) fn grants_inc(&mut self) { + self.grants = self.grants.saturating_add(1); + } + + pub(crate) fn grants_dec(&mut self) { + self.grants = self.grants.saturating_sub(1); + } + + /// No name places this object and no grant reaches it — so no route to + /// it can exist, and its storage and slot may be released. + /// + /// Outstanding handles are intentionally not consulted; see the module + /// docs. + #[inline] + pub(crate) const fn is_reclaimable(&self) -> bool { + self.incoming == 0 && self.grants == 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + type Obj = Object; + + #[test] + fn a_fresh_file_has_no_storage_and_no_bytes() { + let file = Obj::new_file(); + assert_eq!(file.kind(), ObjectKind::File); + assert!(file.kind().is_file() && !file.kind().is_dir()); + assert_eq!(file.len(), 0); + assert_eq!(file.storage(), None); + assert_eq!(file.dir(), None); + } + + #[test] + fn a_fresh_directory_owns_a_child_list_and_no_storage() { + let dir = Obj::new_dir(DirId(SlotKey::new(3, 1))); + assert_eq!(dir.kind(), ObjectKind::Directory); + assert!(dir.kind().is_dir()); + assert_eq!(dir.dir(), Some(DirId(SlotKey::new(3, 1)))); + assert_eq!(dir.storage(), None); + } + + #[test] + fn revisions_start_above_the_any_sentinel() { + // The sentinel shares the wire field with real revisions, so it must + // never collide with one. + assert!(FIRST_REVISION > ANY_REVISION); + assert_eq!(Obj::new_file().revision(), FIRST_REVISION); + } + + #[test] + fn commit_len_advances_the_revision_monotonically() { + let mut file = Obj::new_file(); + let before = file.revision(); + file.commit_len(10); + assert_eq!(file.len(), 10); + assert!(file.revision() > before); + let mid = file.revision(); + file.commit_len(4); + assert_eq!(file.len(), 4); + assert!(file.revision() > mid); + } + + #[test] + fn any_revision_is_unconditional_and_exact_revisions_are_not() { + let mut file = Obj::new_file(); + file.commit_len(1); + let live = file.revision(); + assert!(file.revision_admits(ANY_REVISION)); + assert!(file.revision_admits(live)); + assert!(!file.revision_admits(live - 1)); + assert!(!file.revision_admits(live + 1)); + } + + #[test] + fn storage_placement_is_recorded_once_set() { + let mut file = Obj::new_file(); + assert_eq!(file.storage(), None); + file.set_storage(42); + assert_eq!(file.storage(), Some(42)); + } + + #[test] + fn a_directory_never_takes_a_storage_placement() { + let mut dir = Obj::new_dir(DirId(SlotKey::new(0, 1))); + dir.set_storage(7); + assert_eq!(dir.storage(), None); + } + + #[test] + fn reclaimable_only_when_no_name_and_no_grant_remain() { + let mut file = Obj::new_file(); + // A brand-new object is unreachable until an edge names it — the + // engine links it in the same request that creates it. + assert!(file.is_reclaimable()); + + file.incoming_inc(); + assert!(!file.is_reclaimable()); + file.grants_inc(); + assert!(!file.is_reclaimable()); + + // Dropping the only name still leaves the grant as a live route. + file.incoming_dec(); + assert!(!file.is_reclaimable()); + assert_eq!(file.incoming(), 0); + assert_eq!(file.grants(), 1); + + file.grants_dec(); + assert!(file.is_reclaimable()); + } + + #[test] + fn two_names_survive_losing_one() { + let mut file = Obj::new_file(); + file.incoming_inc(); + file.incoming_inc(); + assert_eq!(file.incoming(), 2); + file.incoming_dec(); + assert_eq!(file.incoming(), 1); + assert!(!file.is_reclaimable()); + } + + #[test] + fn counts_saturate_rather_than_wrap() { + let mut file = Obj::new_file(); + // An extra decrement must not wrap to u32::MAX and resurrect the + // object forever. + file.incoming_dec(); + file.grants_dec(); + assert_eq!(file.incoming(), 0); + assert_eq!(file.grants(), 0); + assert!(file.is_reclaimable()); + } + + #[test] + fn object_id_bits_round_trip() { + let id = ObjectId::from_bits(SlotKey::new(5, 9).to_bits()); + assert_eq!(ObjectId::from_bits(id.to_bits()), id); + assert_eq!(ObjectId::INVALID.to_bits(), 0); + } +} + +#[cfg(test)] +mod subtype_tests { + use super::*; + + #[test] + fn a_fresh_file_is_plain_and_a_stamped_one_keeps_its_stamp() { + let plain = Object::::new_file(); + assert_eq!(plain.subtype(), ObjectSubtype::PLAIN); + let stamped = Object::::new_file_with(ObjectSubtype::EXECUTABLE_IMAGE); + assert_eq!(stamped.subtype(), ObjectSubtype::EXECUTABLE_IMAGE); + assert_eq!( + Object::::new_dir(DirId(crate::slot::SlotKey::INVALID)).subtype(), + ObjectSubtype::PLAIN + ); + } + + /// There is no setter: every mutation the object admits leaves the + /// stamp alone. + #[test] + fn a_subtype_is_fixed_at_creation() { + let mut stamped = Object::::new_file_with(ObjectSubtype::EXECUTABLE_IMAGE); + stamped.set_storage(9); + stamped.commit_len(77); + stamped.restore_len_revision(3, 5); + stamped.incoming_inc(); + stamped.grants_inc(); + assert_eq!(stamped.subtype(), ObjectSubtype::EXECUTABLE_IMAGE); + } + + /// The byte lands in the record's existing padding after `kind`, so the + /// arena rows the bounds table prices do not grow. + #[test] + fn object_size_is_unchanged_by_the_subtype() { + assert!(core::mem::size_of::>() <= 48); + } +} diff --git a/native/services/dagfs/src/parent_index.rs b/native/services/dagfs/src/parent_index.rs new file mode 100644 index 0000000..410f457 --- /dev/null +++ b/native/services/dagfs/src/parent_index.rs @@ -0,0 +1,174 @@ +//! The reverse-edge index: for every object, the containment edges that +//! name it. +//! +//! A forward edge lives in its parent directory's child list; this is the +//! same edge seen from the child, so `Parents` reads upward in time +//! proportional to what it finds rather than walking every directory. It +//! is derived state: every row is written in the same commit as the forward +//! edge it mirrors, rebuilt at mount by the edge restore that re-links the +//! tree, and never persisted — the durable format is unchanged. +//! +//! Most objects have one parent (`res/FILESYSTEM_SCALE.md` measured 559,796 +//! edges for 559,799 objects), so a row keeps its first edge inline and +//! spills only the extra links to the service heap. A row is claimed on its +//! own, not through the object: a file transaction holds the object's claim +//! across provider I/O, and moving or renaming that file must not turn +//! `Busy` for it. Claims never wait, like every other engine claim; a +//! contended row is `Busy` and the request retries. +//! +//! An edge is named by its [`EdgeId`] — parent directory and entry slot, +//! both generational — so a rename, which keeps the entry, never touches +//! the index, and a stale entry left by a bug would fail its revalidation +//! rather than name a successor. + +use alloc::boxed::Box; +use alloc::vec::Vec; +use core::cell::UnsafeCell; +use core::sync::atomic::{AtomicBool, Ordering}; + +use crate::bounds::MAX_OBJECTS; +use crate::error::{FsError, FsResult}; +use crate::namespace::EdgeId; +use crate::object::{ObjectId, ObjectIdExt}; + +/// One row per object slot. +pub(crate) struct ParentIndex { + rows: Box<[ParentRow]>, +} + +struct ParentRow { + claimed: AtomicBool, + edges: UnsafeCell, +} + +/// A row's edges, in the order they were made. +#[derive(Default)] +struct Edges { + first: Option, + rest: Vec, +} + +// SAFETY: a row's edges are read and written only through a `ParentClaim`, +// which holds the row's `claimed` flag exclusively for its lifetime. +unsafe impl Sync for ParentRow {} + +impl ParentIndex { + /// An empty index with a row for every object slot. + pub(crate) fn try_new() -> Option { + let mut rows = Vec::new(); + rows.try_reserve_exact(MAX_OBJECTS).ok()?; + rows.resize_with(MAX_OBJECTS, || ParentRow { + claimed: AtomicBool::new(false), + edges: UnsafeCell::new(Edges::default()), + }); + Some(Self { + rows: rows.into_boxed_slice(), + }) + } + + /// Claim `object`'s row, or `Busy` while another commit holds it. + pub(crate) fn claim(&self, object: ObjectId) -> FsResult> { + let row = usize::try_from(object.key().index()) + .ok() + .and_then(|index| self.rows.get(index)) + .ok_or(FsError::AccessLost)?; + row.claimed + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .map_err(|_| FsError::Busy)?; + Ok(ParentClaim { row }) + } +} + +/// Exclusive access to one object's row until dropped. +pub(crate) struct ParentClaim<'a> { + row: &'a ParentRow, +} + +impl Drop for ParentClaim<'_> { + fn drop(&mut self) { + self.row.claimed.store(false, Ordering::Release); + } +} + +impl ParentClaim<'_> { + fn edges(&self) -> &Edges { + // SAFETY: this claim holds the row exclusively (`ParentIndex::claim`). + unsafe { &*self.row.edges.get() } + } + + fn edges_mut(&mut self) -> &mut Edges { + // SAFETY: as `edges`, and `&mut self` makes this the only borrow. + unsafe { &mut *self.row.edges.get() } + } + + /// Edges naming the object. + pub(crate) fn len(&self) -> usize { + let edges = self.edges(); + usize::from(edges.first.is_some()) + edges.rest.len() + } + + /// The edges naming the object, in the order they were made. + pub(crate) fn iter(&self) -> impl Iterator + '_ { + let edges = self.edges(); + edges.first.into_iter().chain(edges.rest.iter().copied()) + } + + /// Make room for one more edge, so a commit that then publishes its + /// forward edge never has to refuse the reverse one. + pub(crate) fn reserve(&mut self) -> FsResult<()> { + let edges = self.edges_mut(); + if edges.first.is_some() { + edges.rest.try_reserve(1).map_err(|_| FsError::OutOfEdges)?; + } + Ok(()) + } + + /// Record `edge`, after [`Self::reserve`] made room for it. + pub(crate) fn add(&mut self, edge: EdgeId) { + let edges = self.edges_mut(); + match edges.first { + None => edges.first = Some(edge), + Some(_) => { + debug_assert!(edges.rest.len() < edges.rest.capacity(), "reserve first"); + edges.rest.push(edge); + } + } + } + + /// Forget `edge`; an edge the row does not hold is ignored. A spill + /// that empties is returned to the heap. + pub(crate) fn remove(&mut self, edge: EdgeId) { + let edges = self.edges_mut(); + if edges.first == Some(edge) { + edges.first = if edges.rest.is_empty() { + None + } else { + Some(edges.rest.remove(0)) + }; + } else if let Some(at) = edges.rest.iter().position(|held| *held == edge) { + edges.rest.remove(at); + } + if edges.rest.is_empty() && edges.rest.capacity() != 0 { + edges.rest = Vec::new(); + } + } + + /// The edge `old` became `new` (a move): same object, same place in + /// the row, no allocation. + pub(crate) fn replace(&mut self, old: EdgeId, new: EdgeId) { + let edges = self.edges_mut(); + if let Some(held) = edges + .first + .iter_mut() + .chain(edges.rest.iter_mut()) + .find(|held| **held == old) + { + *held = new; + } + } + + /// Forget every edge (the object's slot is being freed). + pub(crate) fn clear(&mut self) { + *self.edges_mut() = Edges::default(); + } +} diff --git a/native/services/dagfs/src/path.rs b/native/services/dagfs/src/path.rs new file mode 100644 index 0000000..2734b0c --- /dev/null +++ b/native/services/dagfs/src/path.rs @@ -0,0 +1,3 @@ +//! Compatibility re-export of the process-side filesystem path syntax. + +pub use kore_dagfs::path::*; diff --git a/native/services/dagfs/src/persistence.rs b/native/services/dagfs/src/persistence.rs new file mode 100644 index 0000000..0e8db89 --- /dev/null +++ b/native/services/dagfs/src/persistence.rs @@ -0,0 +1,5378 @@ +//! Versioned, checksummed durable namespace publication and recovery. +//! +//! Two one-block root records select two copy-on-write metadata arenas. A +//! commit writes the inactive arena, flushes, publishes its root record, and +//! flushes again. Recovery validates both roots and every referenced record, +//! then chooses the highest completely valid generation. Disk records carry +//! stable object IDs and extents only: session IDs, handles, grants, +//! capabilities, channels, mappings, and request state are never encoded. + +use alloc::{boxed::Box, vec::Vec}; + +use crc::{CRC_32_ISCSI, Crc}; +use kore_driver::block::DirectBlockDevice; + +use crate::{ + Engine, FsError, Name, ObjectKind, ObjectSubtype, ProviderError, SessionId, StableObjectId, + bounds::{ + MAX_BLOCK_PLACEMENTS, MAX_DERIVED_RESULTS, MAX_DIR_ENTRIES, MAX_DIRECTORIES, MAX_NAME_LEN, + MAX_OBJECTS, + }, + namespace::Namespace, + object::FIRST_REVISION, + provider::{BlockKey, BlockProvider, DurableExtent, LEGACY_DATA_START_LBA, StorageProvider}, +}; + +const CRC: Crc = Crc::::new(&CRC_32_ISCSI); +const ROOT_MAGIC: &[u8; 8] = b"KOREROOT"; +const IMAGE_MAGIC: &[u8; 8] = b"KOREFSM1"; +const LEGACY_FORMAT_VERSION: u32 = 1; +/// The version that added the derived-result table to the image. Named +/// because the table's presence is decided by "version 2 or later", never by +/// "equals the current version" — the latter silently drops a v2 volume's +/// index the moment the current version moves on. +const DERIVED_FORMAT_VERSION: u32 = 2; +/// The version that added the three facts the mount previously had to +/// assume: where the data arena starts, how many blocks it spans, and the +/// 32-byte era stamp identifying the build whose store this is. +const ARENA_FORMAT_VERSION: u32 = 3; +/// The version that gave the object record its subtype byte (offset 10, +/// reserved-must-be-zero before it). Named because "can this volume record +/// a subtype" is decided by "version 4 or later", never by "equals the +/// current version". +pub const SUBTYPE_FORMAT_VERSION: u32 = 4; +/// The version whose root names a content-digest sidecar ([`crate::digests`]): +/// derived data, CRC'd and keyed by revision, so mount and every lend stop +/// rehashing files whose bytes have not changed. A version-4 volume mounts +/// as it is and is upgraded by its next commit (`res/STORAGE_TRANSPORT.md` +/// §1 ruling 3); older versions keep committing at their own version. +pub const DIGEST_FORMAT_VERSION: u32 = 5; +/// The current version. +pub const FORMAT_VERSION: u32 = DIGEST_FORMAT_VERSION; +/// Metadata reservation used by versions 1 and 2. +const LEGACY_IMAGE_BYTES: usize = 16 * 1024; +/// Maximum encoded namespace image in the current format. +/// +/// Two fixed reservations of this size live below the volume's recorded +/// `data_start`; commits transfer only the encoded, block-rounded prefix. +pub const MAX_IMAGE_BYTES: usize = 128 * 1024; +/// Root-record header through version 2. +const ROOT_HEADER_V2: usize = 60; +/// Root-record header at version 3: `data_start` (8) + `arena_blocks` (8) + +/// `era` (32) after v2's fields, with the trailing checksum after them. +const ROOT_HEADER_V3: usize = 108; +/// Root-record header at version 5: the version-3 fields, then the digest +/// sidecar's extent (`lba` 8, `blocks` 4, `len` 4) and its CRC (4), with the +/// trailing checksum after them. Zero `blocks` means no sidecar. +const ROOT_HEADER_V5: usize = 128; +/// Bytes of era stamp carried in a version-3 root record. +pub const ERA_BYTES: usize = 32; + +/// The header length one format version declares. +/// +/// The length is validated *exactly*, so it cannot be a single constant +/// across versions: a v2 volume records 60 and a v3 volume records 108, and +/// each must still be recognized by a reader that knows both. +const fn root_header_bytes(version: u32) -> Option { + match version { + LEGACY_FORMAT_VERSION | DERIVED_FORMAT_VERSION => Some(ROOT_HEADER_V2), + ARENA_FORMAT_VERSION | SUBTYPE_FORMAT_VERSION => Some(ROOT_HEADER_V3), + DIGEST_FORMAT_VERSION => Some(ROOT_HEADER_V5), + _ => None, + } +} +const IMAGE_HEADER: usize = 64; +const OBJECT_BYTES: usize = 48; +const EDGE_BYTES: usize = 56; +const DERIVED_BYTES: usize = 72; +const LEGACY_MAX_DURABLE_OBJECTS: usize = 64; +const LEGACY_MAX_DURABLE_EDGES: usize = 128; +const LEGACY_MAX_DIR_ENTRIES: usize = 32; +pub const MAX_DURABLE_OBJECTS: usize = MAX_OBJECTS; + +/// Containment edges one image may record. +/// +/// **Deliberately not `MAX_DIRECTORIES * MAX_DIR_ENTRIES`.** That product is +/// the shape a namespace *could* take if every directory filled; it is not +/// the shape one takes. A desktop tree measured for +/// `res/FILESYSTEM_SCALE.md` carries about **1.05 edges per object**, because +/// a namespace is a tree with a few extra links, while the product reserved +/// sixteen per object and so spent 87% of the image on adjacency that does +/// not exist. Paying that ratio is what held the object bound at 136. +/// +/// The engine's *structural* maximum is still the product, which is larger +/// than this. A tree that exceeds this cap is refused at commit with +/// [`PersistenceError::Bounds`] rather than at `link`, because the engine +/// keeps no global edge total and adding one would put a counter on every +/// mutation. That is a real gap, and it is bounded on both sides: reaching it +/// needs twice as many edges as the object bound allows objects, against a +/// measured ratio near one. Closing it properly wants either a global +/// admission check or variable-length edge records, which is the same change +/// that lets `MAX_NAME_LEN` leave 32. +/// +/// Chosen so an existing volume keeps mounting: `MAX_DIR_ENTRIES` and the +/// slot size are untouched, and the **reader** admits everything a current +/// image could record before this decoupling +/// ([`PRE_DECOUPLING_MAX_DURABLE_EDGES`]), so nothing that was recordable +/// becomes unreadable and no reformat is implied. This constant is the +/// **writer's** ceiling only. A pre-decoupling volume carrying more edges +/// than this — reachable only through hard links, at more than seven per +/// object — mounts and reads whole; its next commit refuses +/// [`PersistenceError::Bounds`] until links drop below the ceiling, which is +/// the same typed refusal any over-bound tree receives. +pub const MAX_DURABLE_EDGES: usize = 1024; + +/// Containment edges a current-profile image written **before** +/// [`MAX_DURABLE_EDGES`] was decoupled may record: the `16 × 128` +/// directory × entry product the profile reserved when it was introduced. +/// Reading validates against this rather than the writer's ceiling, so a +/// volume that mounted under the old bound still mounts. A count in +/// `MAX_DURABLE_EDGES + 1 ..= PRE_DECOUPLING_MAX_DURABLE_EDGES` therefore +/// identifies an old writer, and a count above it is corruption. +/// +/// A literal because neither factor is a live constant any more: +/// `MAX_DIRECTORIES` has since been derived from `MAX_OBJECTS`, and the +/// product was never a bound this format should have carried +/// (`res/FILESYSTEM_SCALE.md` §4.1). +const PRE_DECOUPLING_MAX_DURABLE_EDGES: usize = 2048; + +// A reader must admit at least what the writer emits, or a freshly committed +// image would refuse to recover. +const _: () = assert!(PRE_DECOUPLING_MAX_DURABLE_EDGES >= MAX_DURABLE_EDGES); + +// The serialized maxima are one contract: raising any constituent without +// raising the reservation must fail at compile time, not as a late commit. +const _: () = assert!( + IMAGE_HEADER + + MAX_DURABLE_OBJECTS * OBJECT_BYTES + + MAX_DURABLE_EDGES * EDGE_BYTES + + MAX_DERIVED_RESULTS * DERIVED_BYTES + <= MAX_IMAGE_BYTES, + "the current durable graph must fit its recorded image reservation" +); + +#[derive(Clone, Copy)] +struct ImageLayout { + bytes: usize, + slot_blocks: u32, + data_start: u64, +} + +/// The wire version and capacity profile are deliberately separate. Slice 5 +/// produced v3 roots before slice 7b enlarged the slots; those roots declare +/// the legacy floor and must keep using the legacy slot addresses. Fresh v3 +/// formats declare the current floor and receive the raised bounds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ImageProfile { + Legacy, + Current, +} + +fn image_layout(profile: ImageProfile, block_size: u32) -> Result { + let bytes = match profile { + ImageProfile::Legacy => LEGACY_IMAGE_BYTES, + ImageProfile::Current => MAX_IMAGE_BYTES, + }; + let block = block_size as usize; + if block == 0 || !bytes.is_multiple_of(block) { + return Err(PersistenceError::Incompatible); + } + let slot_blocks = u32::try_from(bytes / block).map_err(|_| PersistenceError::Bounds)?; + let metadata_end = 2u64 + .checked_add( + u64::from(slot_blocks) + .checked_mul(2) + .ok_or(PersistenceError::Bounds)?, + ) + .ok_or(PersistenceError::Bounds)?; + Ok(ImageLayout { + bytes, + slot_blocks, + // Versions 1 and 2 reserved 72 blocks even when their two 16 KiB + // slots ended earlier. Current volumes extend that separation where + // the larger slots require. + // Current profiles always use a distinguishable floor. On 4 KiB + // media both slot layouts otherwise end below legacy block 72, and a + // pre-7b v3 root would not say which slot-1 address it meant. + data_start: match profile { + ImageProfile::Legacy => metadata_end.max(LEGACY_DATA_START_LBA), + ImageProfile::Current => metadata_end.max(LEGACY_DATA_START_LBA + 1), + }, + }) +} + +fn root_profile( + version: u32, + data_start: u64, + block_size: u32, +) -> Result { + match version { + LEGACY_FORMAT_VERSION | DERIVED_FORMAT_VERSION => Ok(ImageProfile::Legacy), + // A pre-7b version-3 volume kept the legacy slot profile; the profile + // is decided by where the data arena starts, never by the version. + ARENA_FORMAT_VERSION | SUBTYPE_FORMAT_VERSION | DIGEST_FORMAT_VERSION + if data_start == LEGACY_DATA_START_LBA => + { + Ok(ImageProfile::Legacy) + } + ARENA_FORMAT_VERSION | SUBTYPE_FORMAT_VERSION | DIGEST_FORMAT_VERSION + if data_start >= image_layout(ImageProfile::Current, block_size)?.data_start => + { + Ok(ImageProfile::Current) + } + ARENA_FORMAT_VERSION | SUBTYPE_FORMAT_VERSION | DIGEST_FORMAT_VERSION => { + Err(PersistenceError::Corrupt) + } + _ => Err(PersistenceError::Incompatible), + } +} + +/// The writer's `(objects, edges)` ceilings for `profile`. +fn durable_bounds(profile: ImageProfile) -> (usize, usize) { + match profile { + ImageProfile::Legacy => (LEGACY_MAX_DURABLE_OBJECTS, LEGACY_MAX_DURABLE_EDGES), + ImageProfile::Current => (MAX_DURABLE_OBJECTS, MAX_DURABLE_EDGES), + } +} + +/// Containment edges a reader admits from an image of `profile`: the +/// writer's ceiling plus everything an older writer of the same profile could +/// lawfully have recorded. The legacy profile never moved, so its two +/// figures coincide. +const fn readable_durable_edges(profile: ImageProfile) -> usize { + match profile { + ImageProfile::Legacy => LEGACY_MAX_DURABLE_EDGES, + ImageProfile::Current => PRE_DECOUPLING_MAX_DURABLE_EDGES, + } +} + +const fn durable_dir_entries(profile: ImageProfile) -> usize { + match profile { + ImageProfile::Legacy => LEGACY_MAX_DIR_ENTRIES, + ImageProfile::Current => MAX_DIR_ENTRIES, + } +} + +/// Conservative provider floor used before either root has authenticated its +/// exact per-volume arena. +pub(crate) fn bootstrap_data_start(block_size: u32) -> Result { + Ok(image_layout(ImageProfile::Current, block_size)?.data_start) +} + +/// One fallibly allocated metadata image retained by a mounted volume and +/// reused for every recovery read and commit. +struct ImageScratch(Box<[u8]>); + +impl ImageScratch { + fn new() -> Result { + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(MAX_IMAGE_BYTES) + .map_err(|_| PersistenceError::Bounds)?; + bytes.resize(MAX_IMAGE_BYTES, 0); + Ok(Self(bytes.into_boxed_slice())) + } + + fn as_mut(&mut self) -> &mut [u8] { + &mut self.0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PersistenceError { + Unformatted, + Incompatible, + Corrupt, + Bounds, + Interrupted(CommitStage), + Provider(ProviderError), + Filesystem(FsError), +} + +/// What the two reserved root blocks say about the underlying medium. +/// +/// This classification is deliberately weaker than recovery: recognizing the +/// kore magic does not bless a root record, it only says that recovery owns +/// the right to diagnose it. Conversely, arbitrary nonzero bytes are +/// [`Foreign`](Self::Foreign), not a damaged kore volume. Real boot disks +/// normally carry GPT/MBR data in these blocks and must never be formatted by +/// an automatic service bring-up path. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VolumeProbe { + /// Both reserved root blocks are entirely zero. + Blank, + /// At least one root block begins with the kore filesystem magic. + Kore, + /// Nonzero metadata exists, but neither root names the kore format. + Foreign, +} + +impl From for PersistenceError { + fn from(value: ProviderError) -> Self { + Self::Provider(value) + } +} + +impl From for PersistenceError { + fn from(value: FsError) -> Self { + Self::Filesystem(value) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CommitStage { + BeforeMetadata, + AfterMetadata, + AfterMetadataFlush, + AfterRootWrite, + AfterRootFlush, +} + +/// The stage of [`PersistentFs::recover_witnessed`] that refused a volume, in +/// the order recovery runs. Frozen: append, never renumber — the number is +/// printed inside the startup refusal of a boot that cannot mount its +/// workspace (`durable.rs`, detail 17), and a photograph of that screen has +/// to stay decodable. +/// +/// The image stages (`RootDecode` through `ImageDecode`) are per root: a +/// root that fails one falls back to the other, so one of them names the +/// newer root's failure only when the older root failed too. Every later +/// stage refuses the mount outright from whichever root reached it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum RecoverStage { + /// The medium's block is larger than a root record's scratch. + Geometry = 1, + /// Reading the two reserved root blocks. + RootBlocks = 2, + /// Neither root block names the format. + Unformatted = 3, + /// Both roots claim one generation. + SameGeneration = 4, + /// The roots disagree on the allocation arena. + ArenaDisagrees = 5, + /// The image scratch could not be allocated. + Scratch = 6, + /// A root record did not decode. + RootDecode = 7, + /// The root's arena bounds were refused by the provider. + Arena = 8, + /// The root's version names no image profile this reader has. + Profile = 9, + /// The image length does not fit its region. + ImageSpan = 10, + /// Reading the image. + ImageRead = 11, + /// The image failed its checksum. + ImageCrc = 12, + /// The image did not decode. + ImageDecode = 13, + /// The live or fallback extent tables overflowed. + ExtentTable = 14, + /// The provider refused the recovered extents. + Extents = 15, + /// Restoring an object into the namespace. + Objects = 16, + /// The derived-result table overflowed. + DerivedTable = 17, + /// The recovered extents and the image's objects disagree in count. + ExtentWalk = 18, + /// Restoring a containment edge. + Edges = 19, + /// The image's root object is not among its objects. + Root = 20, + /// The engine refused the recovered namespace. + Engine = 21, +} + +/// Why and where a recovery refused a volume. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RecoverFailure { + pub stage: RecoverStage, + pub error: PersistenceError, +} + +impl RecoverFailure { + pub const fn at(stage: RecoverStage, error: PersistenceError) -> Self { + Self { stage, error } + } + + /// The failure as four bytes, low to high: the stage, the error's kind + /// (1 unformatted, 2 incompatible, 3 corrupt, 4 bounds, 5 interrupted, + /// 6 provider, 7 filesystem), then the cause — the commit stage for an + /// interrupted commit, the provider's wire code, or the filesystem's + /// wire status with its provider code beside it. Every code is a frozen + /// wire vocabulary ([`kore_dagfs::wire`]), never a new one. + pub fn witness(self) -> u32 { + use kore_dagfs::wire::{fs_status_code, provider_code}; + let (kind, cause, beside) = match self.error { + PersistenceError::Unformatted => (1, 0, 0), + PersistenceError::Incompatible => (2, 0, 0), + PersistenceError::Corrupt => (3, 0, 0), + PersistenceError::Bounds => (4, 0, 0), + PersistenceError::Interrupted(stage) => (5, stage as u8 + 1, 0), + PersistenceError::Provider(error) => (6, provider_code(error), 0), + PersistenceError::Filesystem(error) => { + let beside = match error { + FsError::ProviderFailure(provider) => provider_code(provider), + _ => 0, + }; + (7, fs_status_code(error), beside) + } + }; + u32::from_le_bytes([self.stage as u8, kind, cause, beside]) + } +} + +/// Tag a fallible step of recovery with its stage. +fn at>(stage: RecoverStage) -> impl FnOnce(E) -> RecoverFailure { + move |error| RecoverFailure::at(stage, error.into()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MountStatus { + pub generation: u64, + pub used_fallback_root: bool, + pub mode: MountMode, + /// The era this volume carries, round-tripped through mount and commit. + /// + /// Held here rather than defaulted at each commit because a field that is + /// re-zeroed on every write is not persisted at all: the stamp has to + /// survive the commits between the mount that read it and the policy that + /// will judge it. Nothing reads it for a decision yet. + pub era: [u8; ERA_BYTES], +} + +/// Whether recovered state may be published again without an explicit +/// repair/revalidation step. Falling back around corruption is readable but +/// deliberately not writable: silently overwriting the evidence would turn +/// a deterministic degraded mount into data loss. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MountMode { + ReadWrite, + DegradedReadOnly, +} + +#[derive(Clone, Copy)] +struct DiskObject { + stable: StableObjectId, + kind: ObjectKind, + subtype: ObjectSubtype, + len: u64, + revision: u64, + extent: Option, +} + +#[derive(Clone, Copy)] +struct DiskEdge { + parent: StableObjectId, + child: StableObjectId, + name: Name, +} + +#[derive(Clone, Copy)] +struct DiskDerived { + object: StableObjectId, + input_version: u64, + extent: DurableExtent, + digest: [u8; 32], +} + +/// One root's decoded metadata image. +/// +/// The three tables are heap-backed and reserved to the **counts this image +/// actually records**, not to the profile's worst case. Inline they came to +/// 132,168 bytes — the edge table alone was 114,696 — and `Decoded` is built +/// on the mount path, which runs inside the service's `serve` future on the +/// founder's shared 4 MiB stack. That is the same coupling the namespace +/// arenas had, and recovery decodes each of the two roots in turn. +/// +/// Reserving the recorded counts rather than the bounds also means a small +/// volume pays for a small volume: the profile maxima are already validated +/// against [`durable_bounds`] before a single record is read, so the +/// reservation cannot be driven past them by disk bytes. +struct Decoded { + generation: u64, + era: [u8; ERA_BYTES], + root: StableObjectId, + objects: Vec, + edges: Vec, + derived: Vec, +} + +/// The one canonical order of a decoded root's payload extents: each object +/// that carries one, then every derived record. `recover_extents` returns +/// the live root's keys following exactly this sequence, and the mount path +/// consumes them through one iterator — the pairing between extents and +/// their objects has this single definition, never twin hand-maintained +/// walks that must agree by luck. +fn root_payload_extents(decoded: &Decoded) -> impl Iterator + '_ { + decoded + .objects + .iter() + .filter_map(|object| object.extent) + .chain(decoded.derived.iter().map(|record| record.extent)) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct RootRecord { + version: u32, + generation: u64, + block_size: u32, + block_count: u64, + slot: u8, + root: StableObjectId, + image_len: u32, + image_crc: u32, + /// First block of the data arena. Recorded from version 3 so the floor + /// is a property of the volume rather than a compile-time constant every + /// reader must already agree on; versions 1 and 2 decode to + /// [`LEGACY_DATA_START_LBA`], which is what they meant. + data_start: u64, + /// Blocks the data arena spans. Versions 1 and 2 decode to everything + /// above the floor, which is what they meant. + arena_blocks: u64, + /// Which build's store this is — blake3 of the image archive. All-zero + /// means unstamped, which is what versions 1 and 2 decode to. + era: [u8; ERA_BYTES], + /// The content-digest sidecar this root names (version 5), if any. + sidecar: Option, +} + +/// Where a root's content-digest sidecar lives, and the CRC its bytes carry. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct SidecarRef { + extent: DurableExtent, + crc: u32, +} + +/// An engine whose durable root is published through its direct block +/// provider. Runtime authority is recreated for `session` on recovery. +pub struct PersistentFs< + D: DirectBlockDevice, + const BUFFERS: usize = 8, + const OUTSTANDING: usize = 32, +> { + engine: Engine>, + state: PersistenceState, +} + +/// The durability cursor kept beside a mounted request engine. +/// +/// `version` pins legacy volumes to their legacy reservation until deliberate +/// reprovisioning; `image` is the single reusable heap allocation that keeps +/// the large current reservation off every async commit stack. +pub struct PersistenceState { + status: MountStatus, + version: u32, + profile: ImageProfile, + image: ImageScratch, + /// The digest sidecar the live root names, and its placement — replaced, + /// never rewritten in place, when a commit finds the memo changed. + sidecar: Option<(BlockKey, SidecarRef)>, +} + +impl PersistenceState { + pub const fn status(&self) -> MountStatus { + self.status + } +} + +impl PersistentFs { + /// Classify the reserved roots without modifying the medium. + /// + /// The observation borrows the provider so its affine channel can then be + /// moved into recovery, explicit blank-media formatting, or containment. + /// Only an all-zero pair is blank. A GPT header, partition table, another + /// filesystem, or any other non-kore data is foreign and is never an + /// implicit invitation to overwrite the namespace. + pub async fn probe(provider: &BlockProvider) -> Result { + let geometry = provider.geometry(); + let block = geometry.block_size as usize; + if block == 0 || block > 4096 { + return Err(PersistenceError::Incompatible); + } + let mut roots = [[0u8; 4096]; 2]; + provider.read_metadata(0, &mut roots[0][..block]).await?; + provider.read_metadata(1, &mut roots[1][..block]).await?; + let classify = |root: &[u8]| { + if root.iter().all(|byte| *byte == 0) { + VolumeProbe::Blank + } else if root.get(..ROOT_MAGIC.len()) == Some(ROOT_MAGIC.as_slice()) { + VolumeProbe::Kore + } else { + VolumeProbe::Foreign + } + }; + let roots = [classify(&roots[0][..block]), classify(&roots[1][..block])]; + Ok(if roots.contains(&VolumeProbe::Kore) { + VolumeProbe::Kore + } else if roots.iter().all(|root| *root == VolumeProbe::Blank) { + VolumeProbe::Blank + } else { + VolumeProbe::Foreign + }) + } + + /// Whether every root that recovery could select is a valid version-3 + /// root stamped with `expected`, with at least one such root present. + /// + /// This is the system-store admission check. Requiring agreement across + /// both nonblank slots is stronger than comparing only the newest header: + /// if its image later proves corrupt, ordinary recovery may fall back to + /// the other slot. That fallback must never publish an older archive era. + /// User volumes do not call this function and remain era-agnostic. + pub async fn roots_match_era( + provider: &BlockProvider, + expected: [u8; ERA_BYTES], + ) -> Result { + let geometry = provider.geometry(); + let block = geometry.block_size as usize; + if block == 0 || block > 4096 { + return Err(PersistenceError::Incompatible); + } + let mut roots = [[0u8; 4096]; 2]; + provider.read_metadata(0, &mut roots[0][..block]).await?; + provider.read_metadata(1, &mut roots[1][..block]).await?; + let mut matched = false; + for bytes in roots { + let bytes = &bytes[..block]; + if bytes.iter().all(|byte| *byte == 0) { + continue; + } + if bytes.get(..ROOT_MAGIC.len()) != Some(ROOT_MAGIC.as_slice()) { + return Ok(false); + } + let Ok(root) = decode_root(bytes, geometry.block_size, geometry.block_count) else { + return Ok(false); + }; + if root.version != FORMAT_VERSION || root.era != expected { + return Ok(false); + } + matched = true; + } + Ok(matched) + } + + pub async fn format( + provider: BlockProvider, + session: SessionId, + ) -> Result { + Self::format_with_era(provider, session, [0u8; ERA_BYTES]).await + } + + /// Format a fresh namespace and stamp its first durable generation with + /// `era`. The caller owns policy: user formatting passes zero; only the + /// already-clamped system path passes the measured archive digest. + pub async fn format_with_era( + provider: BlockProvider, + session: SessionId, + era: [u8; ERA_BYTES], + ) -> Result { + let geometry = provider.geometry(); + let layout = image_layout(ImageProfile::Current, geometry.block_size)?; + provider.configure_arena( + layout.data_start, + geometry.block_count.saturating_sub(layout.data_start), + )?; + let engine = Engine::new(provider); + engine.attach_session(session).await?; + let mut volume = Self { + engine, + state: PersistenceState { + status: MountStatus { + generation: 0, + used_fallback_root: false, + mode: MountMode::ReadWrite, + era, + }, + version: FORMAT_VERSION, + profile: ImageProfile::Current, + image: ImageScratch::new()?, + sidecar: None, + }, + }; + volume.commit(session).await?; + Ok(volume) + } + + /// Destructively replace a rebuildable system namespace after its provider + /// has been narrowed to the designated system extent. + /// + /// Both root slots are zeroed and flushed before the new generation is + /// formatted. A cut before that flush leaves an old mismatched root and is + /// retried by the era policy; a cut after it leaves blank media; a cut + /// during the format leaves either blank media or one complete current-era + /// root. No state can make recovery select a stale-era fallback. + pub async fn reformat_with_era( + provider: BlockProvider, + session: SessionId, + era: [u8; ERA_BYTES], + ) -> Result { + let block = provider.geometry().block_size as usize; + if block == 0 || block > 4096 { + return Err(PersistenceError::Incompatible); + } + let roots = [0u8; 8192]; + let bytes = block + .checked_mul(2) + .filter(|bytes| *bytes <= roots.len()) + .ok_or(PersistenceError::Bounds)?; + provider.write_metadata(0, &roots[..bytes]).await?; + provider.flush_device().await?; + // Keep the fresh-format state machine inline. Awaiting + // `format_with_era` here would nest two futures that each own the + // affine provider/engine return shape; in an unoptimized build that + // needlessly more than doubles the bounded startup frame (and does the + // same to the founded service's root future). + let geometry = provider.geometry(); + let layout = image_layout(ImageProfile::Current, geometry.block_size)?; + provider.configure_arena( + layout.data_start, + geometry.block_count.saturating_sub(layout.data_start), + )?; + let engine = Engine::new(provider); + engine.attach_session(session).await?; + let mut volume = Self { + engine, + state: PersistenceState { + status: MountStatus { + generation: 0, + used_fallback_root: false, + mode: MountMode::ReadWrite, + era, + }, + version: FORMAT_VERSION, + profile: ImageProfile::Current, + image: ImageScratch::new()?, + sidecar: None, + }, + }; + volume.commit(session).await?; + Ok(volume) + } + + pub fn engine(&self) -> &Engine> { + &self.engine + } + pub fn engine_mut(&mut self) -> &mut Engine> { + &mut self.engine + } + pub const fn status(&self) -> MountStatus { + self.state.status() + } + + pub async fn commit(&mut self, session: SessionId) -> Result { + self.commit_with_cut(session, None).await + } + + /// Deterministic power-cut injection. A cut means no durability + /// acknowledgement; subsequent recovery may select the complete old or + /// complete new root depending on whether publication became durable. + pub async fn commit_with_cut( + &mut self, + session: SessionId, + cut: Option, + ) -> Result { + commit_engine(&mut self.engine, &mut self.state, session, cut).await + } + + /// Clean shutdown crosses the same durability boundary as an ordinary + /// commit. No boot-local handle or capability is serialized; a later + /// process still performs full root validation and recreates authority. + pub async fn clean_shutdown(&mut self, session: SessionId) -> Result { + self.commit(session).await + } + + pub async fn recover( + provider: BlockProvider, + session: SessionId, + ) -> Result { + Self::recover_witnessed(provider, session) + .await + .map_err(|failure| failure.error) + } + + /// [`Self::recover`], naming the stage that refused the volume: what a + /// boot that cannot mount its workspace reports, so a photographed + /// failure says where recovery stopped and why + /// ([`RecoverFailure::witness`]). + pub async fn recover_witnessed( + provider: BlockProvider, + session: SessionId, + ) -> Result { + let geometry = provider.geometry(); + let block = geometry.block_size as usize; + if block > 4096 { + return Err(RecoverFailure::at(RecoverStage::Geometry, PersistenceError::Incompatible)); + } + let mut blocks = [[0u8; 4096]; 2]; + provider.read_metadata(0, &mut blocks[0][..block]).await.map_err(at(RecoverStage::RootBlocks))?; + provider.read_metadata(1, &mut blocks[1][..block]).await.map_err(at(RecoverStage::RootBlocks))?; + let parsed = [ + decode_root( + &blocks[0][..block], + geometry.block_size, + geometry.block_count, + ), + decode_root( + &blocks[1][..block], + geometry.block_size, + geometry.block_count, + ), + ]; + if parsed + .iter() + .all(|entry| matches!(entry, Err(PersistenceError::Unformatted))) + { + return Err(RecoverFailure::at(RecoverStage::Unformatted, PersistenceError::Unformatted)); + } + if let (Ok(a), Ok(b)) = (parsed[0], parsed[1]) + && a.generation == b.generation + { + // Two independently published slots cannot lawfully claim the + // same generation. Choosing by slot would silently bless an + // ambiguous/tampered history. + return Err(RecoverFailure::at(RecoverStage::SameGeneration, PersistenceError::Corrupt)); + } + if let (Ok(a), Ok(b)) = (parsed[0], parsed[1]) + && (a.data_start != b.data_start || a.arena_blocks != b.arena_blocks) + { + // Both slots are generations of one volume and therefore one + // allocation arena. Selecting either of two contradictory bounds + // would make the other root's extents mean something different. + return Err(RecoverFailure::at(RecoverStage::ArenaDisagrees, PersistenceError::Corrupt)); + } + let mut order = [0usize, 1usize]; + let generation = |i: usize| parsed[i].as_ref().map_or(0, |root| root.generation); + if generation(order[1]) > generation(order[0]) { + order.swap(0, 1); + } + let newest_generation = generation(order[0]); + let mut last = RecoverFailure::at(RecoverStage::RootDecode, PersistenceError::Corrupt); + let mut image = ImageScratch::new().map_err(at(RecoverStage::Scratch))?; + for (attempt, index) in order.into_iter().enumerate() { + let root = match parsed[index] { + Ok(root) => root, + Err(error) => { + last = RecoverFailure::at(RecoverStage::RootDecode, error); + continue; + } + }; + if let Err(error) = provider.configure_arena(root.data_start, root.arena_blocks) { + last = RecoverFailure::at(RecoverStage::Arena, error.into()); + continue; + } + let profile = root_profile(root.version, root.data_start, geometry.block_size).map_err(at(RecoverStage::Profile))?; + let layout = image_layout(profile, geometry.block_size).map_err(at(RecoverStage::Profile))?; + let arena_lba = image_lba(profile, root.slot, geometry.block_size).map_err(at(RecoverStage::Profile))?; + let len = root.image_len as usize; + let span = match image_transfer_bytes(len, geometry.block_size, layout.bytes) { + Ok(span) => span, + Err(error) => { + last = RecoverFailure::at(RecoverStage::ImageSpan, error); + continue; + } + }; + if let Err(error) = provider + .read_metadata(arena_lba, &mut image.as_mut()[..span]) + .await + { + last = RecoverFailure::at(RecoverStage::ImageRead, error.into()); + continue; + } + if CRC.checksum(&image.as_mut()[..len]) != root.image_crc { + last = RecoverFailure::at(RecoverStage::ImageCrc, PersistenceError::Corrupt); + continue; + } + let decoded = match decode_image( + &image.as_mut()[..len], + root, + geometry.block_size, + geometry.block_count, + ) { + Ok(decoded) => decoded, + Err(error) => { + last = RecoverFailure::at(RecoverStage::ImageDecode, error); + continue; + } + }; + // Read both recovery-selectable roots before installing any + // allocation state, then rebuild the free index as one sorted + // transaction. Exact current/fallback matches need one + // reservation; fallback-only extents remain protected. + let other = 1usize - index; + let older = if let Ok(other_root) = parsed[other] + && other_root.generation < decoded.generation + { + read_decoded(&provider, other_root, image.as_mut()) + .await + .ok() + } else { + None + }; + // Arena-backed like the decoded image above: these are sized by + // the placement and derived bounds, and mount runs inside the + // service's `serve` future on the founder's shared stack. + let alloc = crate::service_alloc(); + let mut live_extents = + crate::slot::BoundedVec::::try_new_in( + MAX_BLOCK_PLACEMENTS, + alloc, + ) + .map_err(|_| RecoverFailure::at(RecoverStage::ExtentTable, PersistenceError::Bounds))?; + for extent in root_payload_extents(&decoded) { + live_extents + .push(extent) + .map_err(|_| RecoverFailure::at(RecoverStage::ExtentTable, PersistenceError::Bounds))?; + } + // The digest sidecar last, after every payload extent — the + // order the recovered keys are consumed in below. A sidecar whose + // extent does not lie inside the arena is derived data gone + // wrong: it is dropped, never allowed to fail the mount. + let sidecar = root + .sidecar + .filter(|sidecar| sidecar_in_arena(sidecar.extent, root, geometry.block_size)); + if let Some(sidecar) = sidecar { + live_extents + .push(sidecar.extent) + .map_err(|_| RecoverFailure::at(RecoverStage::ExtentTable, PersistenceError::Bounds))?; + } + let mut fallback_extents = + crate::slot::BoundedVec::::try_new_in( + MAX_BLOCK_PLACEMENTS, + alloc, + ) + .map_err(|_| RecoverFailure::at(RecoverStage::ExtentTable, PersistenceError::Bounds))?; + if let Some(older) = older.as_ref() { + for extent in root_payload_extents(older) { + fallback_extents + .push(extent) + .map_err(|_| RecoverFailure::at(RecoverStage::ExtentTable, PersistenceError::Bounds))?; + } + // The older root's sidecar stays protected while that root + // is still a recovery choice. + if let Ok(other_root) = parsed[other] + && let Some(sidecar) = other_root.sidecar.filter(|sidecar| { + sidecar_in_arena(sidecar.extent, other_root, geometry.block_size) + }) + { + fallback_extents + .push(sidecar.extent) + .map_err(|_| RecoverFailure::at(RecoverStage::ExtentTable, PersistenceError::Bounds))?; + } + } + let recovered_extents = provider.recover_extents(&live_extents, &fallback_extents) + .map_err(at(RecoverStage::Extents))?; + // Consume the keys through one iterator in the same canonical + // order [`root_payload_extents`] defines; the exhaustion check + // at the end proves the walks agreed. + let mut recovered = recovered_extents.iter().copied(); + let ns = Namespace::::new(); + for object in decoded.objects.iter() { + let storage = match object.extent { + Some(_) => Some(recovered.next().ok_or(RecoverFailure::at(RecoverStage::ExtentWalk, PersistenceError::Corrupt))?), + None => None, + }; + ns.restore_object( + object.stable, + object.kind, + object.subtype, + object.len, + object.revision, + storage, + ) + .map_err(at(RecoverStage::Objects))?; + } + let mut recovered_derived = crate::slot::BoundedVec::< + (DiskDerived, BlockKey), + crate::ServiceAlloc, + >::try_new_in(MAX_DERIVED_RESULTS, alloc) + .map_err(|_| RecoverFailure::at(RecoverStage::DerivedTable, PersistenceError::Bounds))?; + for record in decoded.derived.iter().copied() { + let key = recovered.next().ok_or(RecoverFailure::at(RecoverStage::ExtentWalk, PersistenceError::Corrupt))?; + recovered_derived + .push((record, key)) + .map_err(|_| RecoverFailure::at(RecoverStage::DerivedTable, PersistenceError::Bounds))?; + } + let sidecar_key = match sidecar { + Some(sidecar) => Some((recovered.next().ok_or(RecoverFailure::at(RecoverStage::ExtentWalk, PersistenceError::Corrupt))?, sidecar)), + None => None, + }; + if recovered.next().is_some() { + return Err(RecoverFailure::at(RecoverStage::ExtentWalk, PersistenceError::Corrupt)); + } + for edge in decoded.edges.iter() { + ns.restore_edge(edge.parent, edge.name, edge.child) + .map_err(at(RecoverStage::Edges))?; + } + let root_id = ns + .find_stable(decoded.root) + .ok_or(RecoverFailure::at(RecoverStage::Root, PersistenceError::Corrupt))?; + let engine = Engine::from_recovered( + provider, + ns, + session, + root_id, + root.version >= SUBTYPE_FORMAT_VERSION, + ) + .map_err(at(RecoverStage::Engine))?; + // Load the remembered digests before revalidation needs them. A + // sidecar that fails its CRC or shape loads nothing and is + // marked for replacement; mount goes on either way. + if let Some((key, sidecar)) = sidecar_key { + let loaded = load_sidecar(&engine, key, sidecar).await; + if !loaded { + engine.digests().borrow_mut().mark_dirty(); + } + } + for (record, storage) in recovered_derived { + let recovered: Result<(), PersistenceError> = async { + let object = engine + .namespace() + .find_stable(record.object) + .ok_or(PersistenceError::Corrupt)?; + let current = engine.namespace().object(object)?; + let file_storage = current.storage(); + let file_len = current.len(); + let current_revision = current.revision(); + drop(current); + if current_revision != record.input_version { + return Err(PersistenceError::Corrupt); + } + + let mut wire = [0u8; kore_svc_extract::WORKER_REPORT_BYTES]; + let wire_len = + usize::try_from(record.extent.len).map_err(|_| PersistenceError::Bounds)?; + let read = engine + .provider() + .read(storage, 0, &mut wire[..wire_len]) + .await?; + if read != wire_len + || blake3::hash(&wire[..wire_len]).as_bytes() != &record.digest + { + return Err(PersistenceError::Corrupt); + } + let mut result = kore_svc_extract::decode_result(&wire[..wire_len]) + .map_err(|_| PersistenceError::Corrupt)?; + + // The source word is a boot-local object correlation id, not + // durable authority. Rebind it to the recovered generation, + // while proving the durable digest against the selected file + // payload before publishing the hot index. + let remembered = engine.digests().borrow().get(record.object, current_revision); + let mut hasher = blake3::Hasher::new(); + let mut offset = if remembered.is_some() { file_len } else { 0u64 }; + let mut chunk = [0u8; crate::bounds::MAX_TRANSFER]; + while offset < file_len { + let want = usize::try_from((file_len - offset).min(chunk.len() as u64)) + .map_err(|_| PersistenceError::Bounds)?; + let Some(file_storage) = file_storage else { + return Err(PersistenceError::Corrupt); + }; + let count = engine + .provider() + .read(file_storage, offset, &mut chunk[..want]) + .await?; + if count != want { + return Err(PersistenceError::Corrupt); + } + hasher.update(&chunk[..count]); + offset = offset + .checked_add(count as u64) + .ok_or(PersistenceError::Bounds)?; + } + let digest = match remembered { + Some(digest) => digest, + None => { + let digest = *hasher.finalize().as_bytes(); + engine + .digests() + .borrow_mut() + .put(record.object, current_revision, digest); + digest + } + }; + if result.provenance.input.len != file_len + || result.provenance.input.version != current_revision + || result.provenance.input.digest != digest + { + return Err(PersistenceError::Corrupt); + } + result.provenance.input.source = object.to_bits(); + let parser = kore_svc_extract::registry::Registry::get_version( + result.provenance.parser, + result.provenance.parser_version, + ) + .ok_or(PersistenceError::Incompatible)?; + kore_svc_extract::registry::Registry::validate( + &result, + result.provenance.input, + parser.limits.max_output, + ) + .map_err(|_| PersistenceError::Corrupt)?; + engine.restore_derived_result( + object, + result, + storage, + u32::try_from(wire_len).map_err(|_| PersistenceError::Bounds)?, + record.digest, + )?; + Ok(()) + } + .await; + // Derived data is rebuildable, never authority. A corrupt, + // obsolete, or no-longer-supported batch is omitted from the + // hot index and becomes a typed IndexIncomplete observation; + // it cannot make the authoritative namespace unmountable. + if recovered.is_err() { + continue; + } + } + let used_fallback_root = attempt != 0 + || decoded.generation != newest_generation + || matches!( + parsed[other], + Err(PersistenceError::Corrupt | PersistenceError::Incompatible) + ); + return Ok(Self { + engine, + state: PersistenceState { + status: MountStatus { + generation: decoded.generation, + used_fallback_root, + mode: if geometry.read_only || used_fallback_root { + MountMode::DegradedReadOnly + } else { + MountMode::ReadWrite + }, + era: decoded.era, + }, + version: root.version, + profile, + image, + sidecar: sidecar_key, + }, + }); + } + Err(last) + } + + pub async fn restart(self, session: SessionId) -> Result { + let provider = self.engine.into_provider(); + let bootstrap = bootstrap_data_start(provider.geometry().block_size)?; + let provider = provider.restart_with_data_start(bootstrap)?; + Self::recover(provider, session).await + } + + /// Split the mounted volume into the request engine and the durability + /// cursor a founded service keeps beside it. + pub fn into_parts(self) -> (Engine>, PersistenceState) { + (self.engine, self.state) + } +} + +/// Publish one engine snapshot through the same dual-root transaction used by +/// [`PersistentFs`]. This is the founded service's commit hook: request +/// dispatch owns the engine, while this helper owns durable publication. +pub async fn commit_engine( + engine: &mut Engine>, + state: &mut PersistenceState, + session: SessionId, + cut: Option, +) -> Result { + if state.status.mode == MountMode::DegradedReadOnly { + return Err(PersistenceError::Provider(ProviderError::ReadOnly)); + } + // A version-4 volume is upgraded by this commit; older ones keep their + // version (their reservation and records stay what they were). + let version = if state.version == SUBTYPE_FORMAT_VERSION { + DIGEST_FORMAT_VERSION + } else { + state.version + }; + // The digest sidecar is staged into a fresh extent before the image, so + // the root below can name it. It is derived data: a sidecar that cannot + // be written leaves the root naming the previous one (or none), never a + // failed commit. + let staged = if version >= DIGEST_FORMAT_VERSION { + stage_sidecar(engine, state).await + } else { + SidecarStage::Keep + }; + let sidecar = match staged { + SidecarStage::Keep => state.sidecar.map(|(_, sidecar)| sidecar), + SidecarStage::Fresh { sidecar, .. } => Some(sidecar), + SidecarStage::Drop => None, + }; + let mut submitted = false; + match publish_root(engine, state, session, cut, version, sidecar, &mut submitted).await { + Ok(generation) => { + state.version = version; + let retired = match staged { + SidecarStage::Keep => None, + SidecarStage::Fresh { key, sidecar } => state.sidecar.replace((key, sidecar)), + SidecarStage::Drop => state.sidecar.take(), + }; + if let Some((old, _)) = retired + && engine.provider().release(old).await.is_err() + { + engine.namespace().note_leaked_placement(); + } + Ok(generation) + } + Err(error) => { + // The memo's change is not durable: the next commit writes it. + if !matches!(staged, SidecarStage::Keep) { + engine.digests().borrow_mut().mark_dirty(); + } + // Before the root write was submitted nothing durable can name + // the fresh sidecar; after it, the new root may have landed, so + // the extent stays put until a mount rebuilds the allocator. + if let SidecarStage::Fresh { key, .. } = staged + && !submitted + && engine.provider().release(key).await.is_err() + { + engine.namespace().note_leaked_placement(); + } + Err(error) + } + } +} + +/// What a commit does with the digest sidecar. +#[derive(Clone, Copy)] +enum SidecarStage { + /// The memo is unchanged (or could not be written): the root names the + /// sidecar the previous one did. + Keep, + /// A fresh sidecar was written; the root names it. + Fresh { key: BlockKey, sidecar: SidecarRef }, + /// The memo is empty: the root names none. + Drop, +} + +/// Prune the memo to live revisions and, if it changed, write it into a +/// fresh extent. Never fails the commit: every refusal is [`SidecarStage::Keep`] +/// with the memo left dirty for the next commit. +async fn stage_sidecar( + engine: &Engine>, + state: &PersistenceState, +) -> SidecarStage { + let encoded = { + let ns = engine.namespace(); + let mut memo = engine.digests().borrow_mut(); + memo.prune(|stable| { + let id = ns.find_stable(stable)?; + ns.object(id).ok().map(|object| object.revision()) + }); + if !memo.dirty() && (state.sidecar.is_some() || memo.is_empty()) { + return SidecarStage::Keep; + } + if memo.is_empty() { + memo.mark_clean(); + return SidecarStage::Drop; + } + let mut encoded = Vec::new(); + if encoded.try_reserve_exact(memo.encoded_bytes()).is_err() { + return SidecarStage::Keep; + } + encoded.resize(memo.encoded_bytes(), 0); + if memo.encode(&mut encoded).is_none() { + memo.mark_dirty(); + return SidecarStage::Keep; + } + encoded + }; + let provider = engine.provider(); + let written: Result = async { + let key = provider + .allocate(ObjectKind::File, encoded.len() as u64) + .await?; + if let Err(error) = async { + provider.write(key, 0, &encoded).await?; + provider.sync(key).await + } + .await + { + let _ = provider.release(key).await; + return Err(error); + } + Ok(key) + } + .await; + let Ok(key) = written else { + engine.digests().borrow_mut().mark_dirty(); + return SidecarStage::Keep; + }; + let Ok(extent) = provider.extent(key) else { + let _ = provider.release(key).await; + engine.digests().borrow_mut().mark_dirty(); + return SidecarStage::Keep; + }; + SidecarStage::Fresh { + key, + sidecar: SidecarRef { + extent, + crc: CRC.checksum(&encoded), + }, + } +} + +/// The root publication itself: image, flush, root, flush, checkpoint. +/// `submitted` turns true the moment the root write may have landed. +async fn publish_root( + engine: &mut Engine>, + state: &mut PersistenceState, + session: SessionId, + cut: Option, + version: u32, + sidecar: Option, + submitted: &mut bool, +) -> Result { + let generation = state + .status + .generation + .checked_add(1) + .ok_or(PersistenceError::Bounds)?; + let slot = (generation & 1) as u8; + let root_id = engine.root_object(session)?; + let image = state.image.as_mut(); + let (image_len, root) = encode_image( + engine, + root_id, + generation, + version, + state.profile, + image, + )?; + let provider = engine.provider(); + let geometry = provider.geometry(); + let layout = image_layout(state.profile, geometry.block_size)?; + let arena_lba = image_lba(state.profile, slot, geometry.block_size)?; + if cut == Some(CommitStage::BeforeMetadata) { + return Err(PersistenceError::Interrupted(CommitStage::BeforeMetadata)); + } + // Zero the tail of the written span rather than trusting the buffer to + // have arrived clean: the padding lands on disk, and a future scratch + // buffer reused across commits would otherwise publish stale bytes. + let span = image_transfer_bytes(image_len, geometry.block_size, layout.bytes)?; + image + .get_mut(image_len..span) + .ok_or(PersistenceError::Bounds)? + .fill(0); + provider.write_metadata(arena_lba, &image[..span]).await?; + if cut == Some(CommitStage::AfterMetadata) { + return Err(PersistenceError::Interrupted(CommitStage::AfterMetadata)); + } + provider.flush_device().await?; + if cut == Some(CommitStage::AfterMetadataFlush) { + return Err(PersistenceError::Interrupted( + CommitStage::AfterMetadataFlush, + )); + } + let mut root_block = [0u8; 4096]; + let block = geometry.block_size as usize; + encode_root( + RootRecord { + version, + generation, + block_size: geometry.block_size, + block_count: geometry.block_count, + slot, + root, + image_len: image_len as u32, + image_crc: CRC.checksum(&image[..image_len]), + data_start: provider.arena().0, + arena_blocks: provider.arena().1, + // Preserved, not re-stamped: this commit is not the authority on + // which build's store this is. + era: state.status.era, + sidecar, + }, + &mut root_block[..block], + )?; + // Once the root write is submitted, a failed completion or flush can + // still leave this root durable. Freeze every extent it may name before + // that cut so a later request cannot overwrite a recoverable generation + // in place. + provider.prepare_publication(); + *submitted = true; + provider + .write_metadata(slot as u64, &root_block[..block]) + .await?; + if cut == Some(CommitStage::AfterRootWrite) { + return Err(PersistenceError::Interrupted(CommitStage::AfterRootWrite)); + } + provider.flush_device().await?; + if cut == Some(CommitStage::AfterRootFlush) { + return Err(PersistenceError::Interrupted(CommitStage::AfterRootFlush)); + } + state.status = MountStatus { + generation, + used_fallback_root: false, + mode: MountMode::ReadWrite, + era: state.status.era, + }; + provider.checkpoint().await; + Ok(generation) +} + +/// Whether a root's sidecar extent lies inside that root's data arena and +/// is long enough to hold what it claims. +fn sidecar_in_arena(extent: DurableExtent, root: RootRecord, block_size: u32) -> bool { + let arena_end = root.data_start.saturating_add(root.arena_blocks); + let end = extent.lba.checked_add(u64::from(extent.blocks)); + extent.blocks != 0 + && extent.lba >= root.data_start + && end.is_some_and(|end| end <= arena_end) + && extent.len <= u64::from(extent.blocks) * u64::from(block_size) + && extent.len as usize >= crate::digests::HEADER_BYTES +} + +/// Read and load the sidecar the root names; `false` when it could not be +/// read, fails its CRC, or does not decode. +async fn load_sidecar( + engine: &Engine>, + key: BlockKey, + sidecar: SidecarRef, +) -> bool { + let Ok(len) = usize::try_from(sidecar.extent.len) else { + return false; + }; + let mut bytes = Vec::new(); + if bytes.try_reserve_exact(len).is_err() { + return false; + } + bytes.resize(len, 0); + let mut read = 0; + while read < len { + let want = (len - read).min(engine.provider().limits().max_transfer); + match engine + .provider() + .read(key, read as u64, &mut bytes[read..read + want]) + .await + { + Ok(n) if n == want => read += n, + _ => return false, + } + } + if CRC.checksum(&bytes) != sidecar.crc { + return false; + } + engine.digests().borrow_mut().load(&bytes).is_ok() +} + +async fn read_decoded( + provider: &BlockProvider, + root: RootRecord, + image: &mut [u8], +) -> Result { + let geometry = provider.geometry(); + let profile = root_profile(root.version, root.data_start, geometry.block_size)?; + let layout = image_layout(profile, geometry.block_size)?; + let arena_lba = image_lba(profile, root.slot, geometry.block_size)?; + let len = root.image_len as usize; + let span = image_transfer_bytes(len, geometry.block_size, layout.bytes)?; + provider + .read_metadata(arena_lba, &mut image[..span]) + .await?; + if CRC.checksum(&image[..len]) != root.image_crc { + return Err(PersistenceError::Corrupt); + } + decode_image( + &image[..len], + root, + geometry.block_size, + geometry.block_count, + ) +} + +/// Bytes of an image slot a generation actually has to move. +/// +/// The slot is a fixed *reservation*; a commit only has to transfer the bytes +/// it wrote. `image_len` is already recorded in the root record and already +/// bounds the CRC, so both halves agree without inventing a second source of +/// truth. A tree that fits in one block costs one block rather than the whole +/// reservation — every `touch` and `write` on a small volume was moving 16 KiB +/// of mostly-zero metadata, and it is what makes raising the reservation +/// affordable instead of an 8x write amplification on every file operation. +fn image_transfer_bytes( + image_len: usize, + block_size: u32, + image_bytes: usize, +) -> Result { + let block = block_size as usize; + if block == 0 || !image_bytes.is_multiple_of(block) { + return Err(PersistenceError::Incompatible); + } + if image_len == 0 || image_len > image_bytes { + return Err(PersistenceError::Bounds); + } + // `block` divides `image_bytes` and `image_len <= image_bytes`, so this + // cannot exceed the reservation. + Ok(image_len.div_ceil(block) * block) +} + +fn image_lba(profile: ImageProfile, slot: u8, block_size: u32) -> Result { + if slot > 1 { + return Err(PersistenceError::Corrupt); + } + let layout = image_layout(profile, block_size)?; + let start = 2u64 + .checked_add(u64::from(slot) * u64::from(layout.slot_blocks)) + .ok_or(PersistenceError::Bounds)?; + let end = start + .checked_add(u64::from(layout.slot_blocks)) + .ok_or(PersistenceError::Bounds)?; + if end > layout.data_start { + return Err(PersistenceError::Incompatible); + } + Ok(start) +} + +fn encode_image( + engine: &Engine>, + root_id: crate::ObjectId, + generation: u64, + version: u32, + profile: ImageProfile, + image: &mut [u8], +) -> Result<(usize, StableObjectId), PersistenceError> { + let (max_objects, max_edges) = durable_bounds(profile); + let graph = engine + .namespace() + .durable_graph(root_id, MAX_DURABLE_EDGES)?; + if graph.objects.len() > max_objects || graph.edges.len() > max_edges { + return Err(PersistenceError::Bounds); + } + let max_dir_entries = durable_dir_entries(profile); + if graph.objects.iter().any(|object| { + object.kind.is_dir() + && graph + .edges + .iter() + .filter(|edge| edge.parent == object.stable) + .count() + > max_dir_entries + }) { + return Err(PersistenceError::Bounds); + } + let root = graph.root; + let derived = if version >= DERIVED_FORMAT_VERSION { + engine.durable_derived_results::()? + } else { + heapless::Vec::new() + }; + image + .get_mut(..IMAGE_HEADER) + .ok_or(PersistenceError::Bounds)? + .fill(0); + put(image, 0, IMAGE_MAGIC)?; + put_u32(image, 8, version)?; + put_u32(image, 12, IMAGE_HEADER as u32)?; + put_u64(image, 16, generation)?; + put_u64(image, 24, root.get())?; + put_u32(image, 32, graph.objects.len() as u32)?; + put_u32(image, 36, graph.edges.len() as u32)?; + put_u32(image, 40, OBJECT_BYTES as u32)?; + put_u32(image, 44, EDGE_BYTES as u32)?; + if version >= DERIVED_FORMAT_VERSION { + put_u32(image, 48, derived.len() as u32)?; + put_u32(image, 52, DERIVED_BYTES as u32)?; + } + let mut at = IMAGE_HEADER; + for object in graph.objects { + image + .get_mut(at..at + OBJECT_BYTES) + .ok_or(PersistenceError::Bounds)? + .fill(0); + let extent = match object.storage { + Some(key) => Some(engine.provider().extent(key)?), + None => None, + }; + put_u64(image, at, object.stable.get())?; + image[at + 8] = if object.kind.is_file() { 1 } else { 2 }; + image[at + 9] = u8::from(extent.is_some()); + // Byte 10 is the subtype from version 4; before it the byte is + // reserved-must-be-zero, and a fact the version cannot record is + // refused rather than dropped. + if version >= SUBTYPE_FORMAT_VERSION { + image[at + 10] = object.subtype.to_bits(); + } else if !object.subtype.is_plain() { + return Err(PersistenceError::Incompatible); + } + put_u64(image, at + 16, object.len)?; + put_u64(image, at + 24, object.revision)?; + if let Some(extent) = extent { + put_u64(image, at + 32, extent.lba)?; + put_u32(image, at + 40, extent.blocks)?; + } + let checksum = CRC.checksum(&image[at..at + 44]); + put_u32(image, at + 44, checksum)?; + at += OBJECT_BYTES; + } + for edge in graph.edges { + image + .get_mut(at..at + EDGE_BYTES) + .ok_or(PersistenceError::Bounds)? + .fill(0); + put_u64(image, at, edge.parent.get())?; + put_u64(image, at + 8, edge.child.get())?; + image[at + 16] = edge.name.len() as u8; + put(image, at + 20, edge.name.as_bytes())?; + let checksum = CRC.checksum(&image[at..at + 52]); + put_u32(image, at + 52, checksum)?; + at += EDGE_BYTES; + } + for record in derived { + image + .get_mut(at..at + DERIVED_BYTES) + .ok_or(PersistenceError::Bounds)? + .fill(0); + let object = engine.namespace().object(record.object)?; + if !object.kind().is_file() + || object.revision() != record.input.version + || object.len() != record.input.len + { + return Err(PersistenceError::Corrupt); + } + let extent = engine.provider().extent(record.key)?; + if extent.len != u64::from(record.len) { + return Err(PersistenceError::Corrupt); + } + put_u64(image, at, object.stable().get())?; + put_u64(image, at + 8, record.input.version)?; + put_u64(image, at + 16, extent.lba)?; + put_u32(image, at + 24, extent.blocks)?; + put_u32(image, at + 28, record.len)?; + put(image, at + 32, &record.digest)?; + let checksum = CRC.checksum(&image[at..at + 64]); + put_u32(image, at + 64, checksum)?; + at += DERIVED_BYTES; + } + Ok((at, root)) +} + +fn encode_root(root: RootRecord, block: &mut [u8]) -> Result<(), PersistenceError> { + let header = root_header_bytes(root.version).ok_or(PersistenceError::Incompatible)?; + if block.len() < header { + return Err(PersistenceError::Incompatible); + } + block.fill(0); + put(block, 0, ROOT_MAGIC)?; + put_u32(block, 8, root.version)?; + put_u32(block, 12, header as u32)?; + put_u64(block, 16, root.generation)?; + put_u32(block, 24, root.block_size)?; + block[28] = root.slot; + block[29] = 1; // complete root record + put_u64(block, 32, root.block_count)?; + put_u64(block, 40, root.root.get())?; + put_u32(block, 48, root.image_len)?; + put_u32(block, 52, root.image_crc)?; + if header >= ROOT_HEADER_V3 { + put_u64(block, 56, root.data_start)?; + put_u64(block, 64, root.arena_blocks)?; + put(block, 72, &root.era)?; + } + if header >= ROOT_HEADER_V5 + && let Some(sidecar) = root.sidecar + { + put_u64(block, 104, sidecar.extent.lba)?; + put_u32(block, 112, sidecar.extent.blocks)?; + put_u32( + block, + 116, + u32::try_from(sidecar.extent.len).map_err(|_| PersistenceError::Bounds)?, + )?; + put_u32(block, 120, sidecar.crc)?; + } + // The checksum is the header's last four bytes and covers everything + // before it — one rule at every version, so a new field is added by + // moving the boundary rather than by inventing a second layout. + let covered = header - 4; + let checksum = CRC.checksum(&block[..covered]); + put_u32(block, covered, checksum) +} + +fn decode_root( + block: &[u8], + block_size: u32, + block_count: u64, +) -> Result { + if block.iter().all(|byte| *byte == 0) { + return Err(PersistenceError::Unformatted); + } + if block.get(..8) != Some(ROOT_MAGIC.as_slice()) { + return Err(PersistenceError::Corrupt); + } + let version = get_u32(block, 8)?; + // A version this build does not know, or a header length that disagrees + // with the one that version defines, is incompatible — never guessed at. + let Some(header) = root_header_bytes(version) else { + return Err(PersistenceError::Incompatible); + }; + if get_u32(block, 12)? as usize != header { + return Err(PersistenceError::Incompatible); + } + if get_u32(block, 24)? != block_size || get_u64(block, 32)? != block_count { + return Err(PersistenceError::Incompatible); + } + let covered = header - 4; + if block.get(29) != Some(&1) + || block + .get(30..32) + .is_none_or(|reserved| reserved.iter().any(|byte| *byte != 0)) + || block + .get(header..) + .is_none_or(|reserved| reserved.iter().any(|byte| *byte != 0)) + || CRC.checksum(block.get(..covered).ok_or(PersistenceError::Corrupt)?) + != get_u32(block, covered)? + { + return Err(PersistenceError::Corrupt); + } + // Versions before 3 recorded neither the arena nor the era. They decode + // to what they meant: the compile-time floor, everything above it, and an + // unstamped era — not to zero, which would describe an empty volume. + // + // The fields are present exactly when the header is the version-3 one — + // the rule `encode_root` writes by. Keying this on "equals the current + // version" instead read every version-3 root as legacy once version 4 + // landed: a raised-profile v3 volume then located its image at the + // legacy slot and its era read as unstamped. + let (data_start, arena_blocks, era) = if header >= ROOT_HEADER_V3 { + let mut era = [0u8; ERA_BYTES]; + era.copy_from_slice( + block + .get(72..72 + ERA_BYTES) + .ok_or(PersistenceError::Corrupt)?, + ); + (get_u64(block, 56)?, get_u64(block, 64)?, era) + } else { + ( + LEGACY_DATA_START_LBA, + block_count.saturating_sub(LEGACY_DATA_START_LBA), + [0u8; ERA_BYTES], + ) + }; + let sidecar = if header >= ROOT_HEADER_V5 { + let blocks = get_u32(block, 112)?; + (blocks != 0) + .then(|| -> Result { + Ok(SidecarRef { + extent: DurableExtent { + lba: get_u64(block, 104)?, + blocks, + len: u64::from(get_u32(block, 116)?), + }, + crc: get_u32(block, 120)?, + }) + }) + .transpose()? + } else { + None + }; + let root = RootRecord { + version, + generation: get_u64(block, 16)?, + block_size, + block_count, + slot: *block.get(28).ok_or(PersistenceError::Corrupt)?, + root: StableObjectId::new(get_u64(block, 40)?).ok_or(PersistenceError::Corrupt)?, + image_len: get_u32(block, 48)?, + image_crc: get_u32(block, 52)?, + data_start, + arena_blocks, + era, + sidecar, + }; + let profile = root_profile(version, root.data_start, block_size)?; + let layout = image_layout(profile, block_size)?; + if root.generation == 0 || root.slot > 1 || root.image_len as usize > layout.bytes { + return Err(PersistenceError::Corrupt); + } + // An arena that starts below the metadata reservation, or runs past the + // medium, describes a volume that cannot exist. + let arena_end = root + .data_start + .checked_add(root.arena_blocks) + .ok_or(PersistenceError::Corrupt)?; + if root.data_start < layout.data_start || root.arena_blocks == 0 || arena_end > block_count { + return Err(PersistenceError::Corrupt); + } + Ok(root) +} + +fn decode_image( + image: &[u8], + root: RootRecord, + block_size: u32, + _block_count: u64, +) -> Result { + if image.get(..8) != Some(IMAGE_MAGIC.as_slice()) + || get_u32(image, 8)? != root.version + || get_u32(image, 12)? as usize != IMAGE_HEADER + || get_u64(image, 16)? != root.generation + || get_u64(image, 24)? != root.root.get() + || get_u32(image, 40)? as usize != OBJECT_BYTES + || get_u32(image, 44)? as usize != EDGE_BYTES + { + return Err(PersistenceError::Corrupt); + } + let object_count = get_u32(image, 32)? as usize; + let edge_count = get_u32(image, 36)? as usize; + let derived_count = if root.version >= DERIVED_FORMAT_VERSION { + if get_u32(image, 52)? as usize != DERIVED_BYTES + || image[56..IMAGE_HEADER].iter().any(|byte| *byte != 0) + { + return Err(PersistenceError::Corrupt); + } + get_u32(image, 48)? as usize + } else { + if image[48..IMAGE_HEADER].iter().any(|byte| *byte != 0) { + return Err(PersistenceError::Corrupt); + } + 0 + }; + let profile = root_profile(root.version, root.data_start, block_size)?; + let (max_objects, _) = durable_bounds(profile); + // Edges are read against what any writer of this profile could have + // recorded, not the current writer's ceiling: see + // `PRE_DECOUPLING_MAX_DURABLE_EDGES`. The byte total is still pinned by + // the exact-length check below, and the image itself by its slot. + let max_edges = readable_durable_edges(profile); + if object_count == 0 + || object_count > max_objects + || edge_count > max_edges + || derived_count > MAX_DERIVED_RESULTS + { + return Err(PersistenceError::Bounds); + } + let required = IMAGE_HEADER + .checked_add( + object_count + .checked_mul(OBJECT_BYTES) + .ok_or(PersistenceError::Bounds)?, + ) + .and_then(|n| n.checked_add(edge_count.checked_mul(EDGE_BYTES)?)) + .and_then(|n| n.checked_add(derived_count.checked_mul(DERIVED_BYTES)?)) + .ok_or(PersistenceError::Bounds)?; + if required != image.len() { + return Err(PersistenceError::Corrupt); + } + // Counts are bounds-checked above, so these reservations are bounded by + // the profile and fail closed rather than growing during the fill. + let mut objects = Vec::new(); + objects + .try_reserve_exact(object_count) + .map_err(|_| PersistenceError::Bounds)?; + let mut at = IMAGE_HEADER; + for _ in 0..object_count { + if CRC.checksum(&image[at..at + 44]) != get_u32(image, at + 44)? { + return Err(PersistenceError::Corrupt); + } + let stable = StableObjectId::new(get_u64(image, at)?).ok_or(PersistenceError::Corrupt)?; + if objects.iter().any(|old: &DiskObject| old.stable == stable) { + return Err(PersistenceError::Corrupt); + } + let kind = match image[at + 8] { + 1 => ObjectKind::File, + 2 => ObjectKind::Directory, + _ => return Err(PersistenceError::Corrupt), + }; + let has_extent = match image[at + 9] { + 0 => false, + 1 => true, + _ => return Err(PersistenceError::Corrupt), + }; + let subtype = if root.version >= SUBTYPE_FORMAT_VERSION { + ObjectSubtype::from_bits(image[at + 10]) + } else { + if image[at + 10] != 0 { + return Err(PersistenceError::Corrupt); + } + ObjectSubtype::PLAIN + }; + if kind.is_dir() && !subtype.is_plain() { + return Err(PersistenceError::Corrupt); + } + if image[at + 11..at + 16].iter().any(|byte| *byte != 0) { + return Err(PersistenceError::Corrupt); + } + let len = get_u64(image, at + 16)?; + let revision = get_u64(image, at + 24)?; + if revision < FIRST_REVISION { + return Err(PersistenceError::Corrupt); + } + let extent = has_extent.then(|| DurableExtent { + lba: get_u64(image, at + 32).unwrap_or(0), + blocks: get_u32(image, at + 40).unwrap_or(0), + len, + }); + if !has_extent && (get_u64(image, at + 32)? != 0 || get_u32(image, at + 40)? != 0) { + return Err(PersistenceError::Corrupt); + } + if kind.is_dir() && (len != 0 || extent.is_some()) { + return Err(PersistenceError::Corrupt); + } + if kind.is_file() && len != 0 && extent.is_none() { + return Err(PersistenceError::Corrupt); + } + if let Some(extent) = extent { + let expected = len + .checked_add(block_size as u64 - 1) + .ok_or(PersistenceError::Bounds)? + / block_size as u64; + let end = extent + .lba + .checked_add(extent.blocks as u64) + .ok_or(PersistenceError::Bounds)?; + let arena_end = root + .data_start + .checked_add(root.arena_blocks) + .ok_or(PersistenceError::Bounds)?; + // An empty file's placement holds no blocks and so no address: + // the provider records it as `lba 0, blocks 0` + // (`BlockProvider::allocate`), and `recover_extents` reserves + // nothing for it. That one form is lawful outside the arena; any + // other address on a zero-block extent is not something a writer + // produces. + let empty = extent.blocks == 0; + if expected != extent.blocks as u64 + || (empty && extent.lba != 0) + || (!empty && (extent.lba < root.data_start || end > arena_end)) + { + return Err(PersistenceError::Corrupt); + } + if objects + .iter() + .filter_map(|old| old.extent) + .any(|old| extent.lba < old.lba + old.blocks as u64 && old.lba < end) + { + return Err(PersistenceError::Corrupt); + } + } + objects.push(DiskObject { + stable, + kind, + subtype, + len, + revision, + extent, + }); + at += OBJECT_BYTES; + } + if objects.iter().filter(|object| object.kind.is_dir()).count() > MAX_DIRECTORIES { + return Err(PersistenceError::Bounds); + } + let mut edges = Vec::new(); + edges + .try_reserve_exact(edge_count) + .map_err(|_| PersistenceError::Bounds)?; + for _ in 0..edge_count { + if CRC.checksum(&image[at..at + 52]) != get_u32(image, at + 52)? { + return Err(PersistenceError::Corrupt); + } + let parent = StableObjectId::new(get_u64(image, at)?).ok_or(PersistenceError::Corrupt)?; + let child = + StableObjectId::new(get_u64(image, at + 8)?).ok_or(PersistenceError::Corrupt)?; + let len = image[at + 16] as usize; + if len == 0 + || len > MAX_NAME_LEN + || image[at + 17..at + 20].iter().any(|byte| *byte != 0) + || image[at + 20 + len..at + 52].iter().any(|byte| *byte != 0) + { + return Err(PersistenceError::Corrupt); + } + let name = Name::from_bytes(&image[at + 20..at + 20 + len]) + .map_err(|_| PersistenceError::Corrupt)?; + if !objects + .iter() + .any(|o| o.stable == parent && o.kind.is_dir()) + || !objects.iter().any(|o| o.stable == child) + || edges + .iter() + .any(|old: &DiskEdge| old.parent == parent && old.name == name) + { + return Err(PersistenceError::Corrupt); + } + edges.push(DiskEdge { + parent, + child, + name, + }); + at += EDGE_BYTES; + } + let max_dir_entries = durable_dir_entries(profile); + if objects.iter().any(|object| { + object.kind.is_dir() + && edges + .iter() + .filter(|edge| edge.parent == object.stable) + .count() + > max_dir_entries + }) { + return Err(PersistenceError::Bounds); + } + let mut derived = Vec::new(); + derived + .try_reserve_exact(derived_count) + .map_err(|_| PersistenceError::Bounds)?; + for _ in 0..derived_count { + if CRC.checksum(&image[at..at + 64]) != get_u32(image, at + 64)? + || image[at + 68..at + DERIVED_BYTES] + .iter() + .any(|byte| *byte != 0) + { + return Err(PersistenceError::Corrupt); + } + let object = StableObjectId::new(get_u64(image, at)?).ok_or(PersistenceError::Corrupt)?; + let input_version = get_u64(image, at + 8)?; + let extent = DurableExtent { + lba: get_u64(image, at + 16)?, + blocks: get_u32(image, at + 24)?, + len: u64::from(get_u32(image, at + 28)?), + }; + let digest = image[at + 32..at + 64] + .try_into() + .map_err(|_| PersistenceError::Corrupt)?; + let owner = objects + .iter() + .find(|candidate| candidate.stable == object) + .ok_or(PersistenceError::Corrupt)?; + let expected_blocks = extent + .len + .checked_add(block_size as u64 - 1) + .ok_or(PersistenceError::Bounds)? + / block_size as u64; + let end = extent + .lba + .checked_add(u64::from(extent.blocks)) + .ok_or(PersistenceError::Bounds)?; + let arena_end = root + .data_start + .checked_add(root.arena_blocks) + .ok_or(PersistenceError::Bounds)?; + if !owner.kind.is_file() + || owner.revision != input_version + || extent.len == 0 + || extent.len > kore_svc_extract::WORKER_REPORT_BYTES as u64 + || expected_blocks != u64::from(extent.blocks) + || extent.lba < root.data_start + || end > arena_end + || objects + .iter() + .filter_map(|old| old.extent) + .any(|old| extent.lba < old.lba + u64::from(old.blocks) && old.lba < end) + || derived.iter().any(|old: &DiskDerived| { + extent.lba < old.extent.lba + u64::from(old.extent.blocks) && old.extent.lba < end + }) + || derived.iter().any(|old: &DiskDerived| old.object == object) + { + return Err(PersistenceError::Corrupt); + } + derived.push(DiskDerived { + object, + input_version, + extent, + digest, + }); + at += DERIVED_BYTES; + } + validate_graph(root.root, &objects, &edges)?; + Ok(Decoded { + generation: root.generation, + era: root.era, + root: root.root, + objects, + edges, + derived, + }) +} + +/// A decoded image must not put the volume on the mount path's stack. +/// +/// Inline, `Decoded` was **132,168 bytes** — the edge table alone 114,696 — +/// and recovery decodes each of the two roots in turn inside the service's +/// `serve` future on the founder's shared 4 MiB stack. The ceiling here is +/// loose on purpose: it pins the *property* (three handles plus scalars, no +/// table) rather than a figure, so adding a field is free and re-inlining a +/// table fails. +#[cfg(test)] +#[test] +fn a_decoded_image_holds_no_table_inline() { + let bytes = core::mem::size_of::(); + assert!( + bytes <= 512, + "Decoded grew to {bytes} B: a volume-scaled table is inline again" + ); +} + +fn validate_graph( + root: StableObjectId, + objects: &[DiskObject], + edges: &[DiskEdge], +) -> Result<(), PersistenceError> { + let Some(root_index) = objects + .iter() + .position(|object| object.stable == root && object.kind.is_dir()) + else { + return Err(PersistenceError::Corrupt); + }; + + // Kahn's algorithm is both the cycle proof and the reachability proof. + // Crucially, a directory reached from two parents has indegree two and is + // queued only after both lawful routes are processed; it is not mistaken + // for a cycle merely because it was seen before. + let mut incoming = [0usize; MAX_DURABLE_OBJECTS]; + for edge in edges { + let child = objects + .iter() + .position(|object| object.stable == edge.child) + .ok_or(PersistenceError::Corrupt)?; + incoming[child] = incoming[child] + .checked_add(1) + .ok_or(PersistenceError::Bounds)?; + } + if incoming[root_index] != 0 { + return Err(PersistenceError::Corrupt); + } + + let mut pending = heapless::Vec::::new(); + pending + .push(root_index) + .map_err(|_| PersistenceError::Bounds)?; + let mut processed = [false; MAX_DURABLE_OBJECTS]; + let mut count = 0usize; + while let Some(parent_index) = pending.pop() { + if processed[parent_index] { + return Err(PersistenceError::Corrupt); + } + processed[parent_index] = true; + count += 1; + let parent = objects[parent_index].stable; + for edge in edges.iter().filter(|edge| edge.parent == parent) { + let child = objects + .iter() + .position(|object| object.stable == edge.child) + .ok_or(PersistenceError::Corrupt)?; + incoming[child] = incoming[child] + .checked_sub(1) + .ok_or(PersistenceError::Corrupt)?; + if incoming[child] == 0 { + pending.push(child).map_err(|_| PersistenceError::Bounds)?; + } + } + } + // A disconnected component never enters the root-seeded queue; a cycle + // retains positive indegree. Both therefore fail the same closed check. + if count != objects.len() { + return Err(PersistenceError::Corrupt); + } + Ok(()) +} + +fn put(dst: &mut [u8], at: usize, src: &[u8]) -> Result<(), PersistenceError> { + let end = at.checked_add(src.len()).ok_or(PersistenceError::Bounds)?; + dst.get_mut(at..end) + .ok_or(PersistenceError::Bounds)? + .copy_from_slice(src); + Ok(()) +} +fn put_u32(dst: &mut [u8], at: usize, value: u32) -> Result<(), PersistenceError> { + put(dst, at, &value.to_le_bytes()) +} +fn put_u64(dst: &mut [u8], at: usize, value: u64) -> Result<(), PersistenceError> { + put(dst, at, &value.to_le_bytes()) +} +fn get_u32(src: &[u8], at: usize) -> Result { + let bytes: [u8; 4] = src + .get(at..at + 4) + .ok_or(PersistenceError::Corrupt)? + .try_into() + .map_err(|_| PersistenceError::Corrupt)?; + Ok(u32::from_le_bytes(bytes)) +} +fn get_u64(src: &[u8], at: usize) -> Result { + let bytes: [u8; 8] = src + .get(at..at + 8) + .ok_or(PersistenceError::Corrupt)? + .try_into() + .map_err(|_| PersistenceError::Corrupt)?; + Ok(u64::from_le_bytes(bytes)) +} + +#[cfg(test)] +mod tests { + use super::*; + use kore_driver::block::{BlockChannel, BlockError, CompletionMode, memory::MemoryBlockDevice}; + use kore_svc_extract::{ + NormalizedResult, ParserProvenance, WORKER_REPORT_BYTES, encode_result, + registry::{PLAIN_TEXT, Registry}, + }; + use kore_svc_extract_worker_rust::parse_plain_text; + + type Device = MemoryBlockDevice<327680, 8192>; + type Volume = PersistentFs; + + fn provider() -> BlockProvider { + let channel = + BlockChannel::new(Device::new(512, 16).unwrap(), CompletionMode::Poll).unwrap(); + let bootstrap = bootstrap_data_start(channel.geometry().block_size).unwrap(); + BlockProvider::new_with_data_start(channel, bootstrap).unwrap() + } + + fn name(value: &str) -> Name { + Name::new(value).unwrap() + } + + fn on_large_stack(f: impl FnOnce() + Send + 'static) { + std::thread::Builder::new() + .name("persistent-fs-test".into()) + // Async durability moves the by-value provider, decoded graph, + // and engine through one recovery future while I/O is pending. + // Exercise that real ownership shape without relying on the host + // test runner's much smaller default stack; the 128 KiB image + // itself remains heap-backed and is tested separately below. + .stack_size(32 * 1024 * 1024) + .spawn(f) + .unwrap() + .join() + .unwrap(); + } + + fn fixture(objects: &[DiskObject], edges: &[DiskEdge]) -> (Box<[u8]>, usize, RootRecord) { + // Mirror the mounted volume's retained publication scratch. Returning + // the full current reservation inline would make adversarial parser + // tests retain several 128 KiB arrays in one stack frame. + let mut image = ImageScratch::new().unwrap().0; + put(&mut image, 0, IMAGE_MAGIC).unwrap(); + put_u32(&mut image, 8, FORMAT_VERSION).unwrap(); + put_u32(&mut image, 12, IMAGE_HEADER as u32).unwrap(); + put_u64(&mut image, 16, 7).unwrap(); + put_u64(&mut image, 24, 1).unwrap(); + put_u32(&mut image, 32, objects.len() as u32).unwrap(); + put_u32(&mut image, 36, edges.len() as u32).unwrap(); + put_u32(&mut image, 40, OBJECT_BYTES as u32).unwrap(); + put_u32(&mut image, 44, EDGE_BYTES as u32).unwrap(); + put_u32(&mut image, 48, 0).unwrap(); + put_u32(&mut image, 52, DERIVED_BYTES as u32).unwrap(); + let mut at = IMAGE_HEADER; + for object in objects { + put_u64(&mut image, at, object.stable.get()).unwrap(); + image[at + 8] = if object.kind.is_file() { 1 } else { 2 }; + image[at + 9] = u8::from(object.extent.is_some()); + image[at + 10] = object.subtype.to_bits(); + put_u64(&mut image, at + 16, object.len).unwrap(); + put_u64(&mut image, at + 24, object.revision).unwrap(); + if let Some(extent) = object.extent { + put_u64(&mut image, at + 32, extent.lba).unwrap(); + put_u32(&mut image, at + 40, extent.blocks).unwrap(); + } + let checksum = CRC.checksum(&image[at..at + 44]); + put_u32(&mut image, at + 44, checksum).unwrap(); + at += OBJECT_BYTES; + } + for edge in edges { + put_u64(&mut image, at, edge.parent.get()).unwrap(); + put_u64(&mut image, at + 8, edge.child.get()).unwrap(); + image[at + 16] = edge.name.len() as u8; + put(&mut image, at + 20, edge.name.as_bytes()).unwrap(); + let checksum = CRC.checksum(&image[at..at + 52]); + put_u32(&mut image, at + 52, checksum).unwrap(); + at += EDGE_BYTES; + } + let root = RootRecord { + version: FORMAT_VERSION, + generation: 7, + block_size: 512, + block_count: 256, + slot: 1, + root: StableObjectId::new(1).unwrap(), + image_len: at as u32, + image_crc: CRC.checksum(&image[..at]), + data_start: LEGACY_DATA_START_LBA, + arena_blocks: 256 - LEGACY_DATA_START_LBA, + era: [0u8; ERA_BYTES], + sidecar: None, + }; + (image, at, root) + } + + /// Rewrite the active fresh-v3 generation into the legacy slot profile. + /// This models both a real v2 disk and a v3 disk produced after slice 5 + /// but before slice 7b enlarged the metadata reservation. + fn rewrite_active_as_legacy(volume: &Volume, version: u32) { + let provider = volume.engine().provider(); + let block_size = provider.geometry().block_size; + let block = block_size as usize; + let slot = (volume.status().generation & 1) as u8; + let current_lba = image_lba(ImageProfile::Current, slot, block_size).unwrap(); + let legacy_lba = image_lba(ImageProfile::Legacy, slot, block_size).unwrap(); + + let mut image = [0u8; LEGACY_IMAGE_BYTES]; + let mut head = [0u8; 4096]; + crate::block_on(provider.read_metadata(slot as u64, &mut head[..block])).unwrap(); + let current = + decode_root(&head[..block], block_size, provider.geometry().block_count).unwrap(); + assert_eq!( + root_profile(current.version, current.data_start, block_size), + Ok(ImageProfile::Current), + "the fixture must start at the raised v3 profile" + ); + let span = image_transfer_bytes(current.image_len as usize, block_size, LEGACY_IMAGE_BYTES) + .unwrap(); + crate::block_on(provider.read_metadata(current_lba, &mut image[..span])).unwrap(); + + put_u32(&mut image, 8, version).unwrap(); + let image_len = current.image_len as usize; + let legacy = RootRecord { + version, + image_crc: CRC.checksum(&image[..image_len]), + data_start: LEGACY_DATA_START_LBA, + arena_blocks: provider + .geometry() + .block_count + .saturating_sub(LEGACY_DATA_START_LBA), + ..current + }; + encode_root(legacy, &mut head[..block]).unwrap(); + crate::block_on(provider.write_metadata(legacy_lba, &image[..span])).unwrap(); + crate::block_on(provider.write_metadata(slot as u64, &head[..block])).unwrap(); + + // The older fallback still names the raised synthetic fixture. A real + // legacy disk never has that mixed history, so leave one valid root + // instead of manufacturing contradictory arena bounds. + head[..block].fill(0); + crate::block_on(provider.write_metadata(u64::from(slot ^ 1), &head[..block])).unwrap(); + crate::block_on(provider.flush_device()).unwrap(); + } + + #[test] + fn probe_distinguishes_blank_kore_and_foreign_media_without_writing() { + on_large_stack(|| { + let blank = provider(); + assert_eq!( + crate::block_on(Volume::probe(&blank)), + Ok(VolumeProbe::Blank) + ); + + let foreign = provider(); + let mut gpt = [0u8; 512]; + gpt[..8].copy_from_slice(b"EFI PART"); + crate::block_on(foreign.write_metadata(0, &gpt)).unwrap(); + assert_eq!( + crate::block_on(Volume::probe(&foreign)), + Ok(VolumeProbe::Foreign) + ); + let mut observed = [0u8; 512]; + crate::block_on(foreign.read_metadata(0, &mut observed)).unwrap(); + assert_eq!(observed, gpt, "the probe never rewrites foreign media"); + + let volume = crate::block_on(Volume::format(provider(), SessionId::new(0))).unwrap(); + assert_eq!( + crate::block_on(Volume::probe(volume.engine().provider())), + Ok(VolumeProbe::Kore) + ); + }); + } + + #[test] + fn v3_era_survives_commits_and_cold_recovery() { + on_large_stack(|| { + let session = SessionId::new(0); + let era = [0x35; ERA_BYTES]; + let mut volume = + crate::block_on(Volume::format_with_era(provider(), session, era)).unwrap(); + assert_eq!(volume.status().era, era); + assert!( + crate::block_on(Volume::roots_match_era(volume.engine().provider(), era)).unwrap() + ); + + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("era"), + ObjectKind::File, + )) + .unwrap(); + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"current", + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + assert!( + crate::block_on(Volume::roots_match_era(volume.engine().provider(), era)).unwrap() + ); + + let recovered = crate::block_on(volume.restart(SessionId::new(1))).unwrap(); + assert_eq!(recovered.status().era, era); + assert!( + crate::block_on(Volume::roots_match_era(recovered.engine().provider(), era,)) + .unwrap() + ); + }); + } + + #[test] + fn system_admission_rejects_a_stale_fallback_era() { + on_large_stack(|| { + let session = SessionId::new(0); + let era = [0x46; ERA_BYTES]; + let mut volume = + crate::block_on(Volume::format_with_era(provider(), session, era)).unwrap(); + // Publish the other slot so recovery has two lawful candidates. + crate::block_on(volume.commit(session)).unwrap(); + let provider = volume.engine().provider(); + let block = provider.geometry().block_size as usize; + let mut root = [0u8; 4096]; + crate::block_on(provider.read_metadata(0, &mut root[..block])).unwrap(); + let mut decoded = decode_root( + &root[..block], + provider.geometry().block_size, + provider.geometry().block_count, + ) + .unwrap(); + decoded.era = [0x47; ERA_BYTES]; + encode_root(decoded, &mut root[..block]).unwrap(); + crate::block_on(provider.write_metadata(0, &root[..block])).unwrap(); + crate::block_on(provider.flush_device()).unwrap(); + + assert!( + !crate::block_on(Volume::roots_match_era(provider, era)).unwrap(), + "a recovery-selectable stale era must forbid ordinary recovery" + ); + }); + } + + #[test] + fn reformat_replaces_only_the_namespace_and_stamps_the_new_era() { + on_large_stack(|| { + let session = SessionId::new(0); + let old_era = [0x58; ERA_BYTES]; + let new_era = [0x59; ERA_BYTES]; + let mut old = + crate::block_on(Volume::format_with_era(provider(), session, old_era)).unwrap(); + let root = old + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + crate::block_on(old.engine_mut().create( + session, + root.handle, + &name("old-era-only"), + ObjectKind::File, + )) + .unwrap(); + crate::block_on(old.commit(session)).unwrap(); + + let (engine, _) = old.into_parts(); + let provider = engine.into_provider(); + assert!(!crate::block_on(Volume::roots_match_era(&provider, new_era)).unwrap()); + let mut current = + crate::block_on(Volume::reformat_with_era(provider, session, new_era)).unwrap(); + assert_eq!(current.status().era, new_era); + assert!( + crate::block_on(Volume::roots_match_era( + current.engine().provider(), + new_era, + )) + .unwrap() + ); + let root = current + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + assert_eq!( + current.engine().open_at( + session, + root.handle, + &name("old-era-only"), + crate::Rights::READ + ), + Err(crate::FsError::NotFound), + "old system names must not survive an era rebuild" + ); + }); + } + + /// Byte 10 of the object record is the subtype from version 4: a stamped + /// file round-trips, a directory may not carry one, and the bytes after + /// it stay reserved. + #[test] + fn a_version_4_object_record_round_trips_its_subtype() { + let root_object = DiskObject { + stable: StableObjectId::new(1).unwrap(), + kind: ObjectKind::Directory, + subtype: ObjectSubtype::PLAIN, + len: 0, + revision: FIRST_REVISION, + extent: None, + }; + let tool = DiskObject { + stable: StableObjectId::new(2).unwrap(), + kind: ObjectKind::File, + subtype: ObjectSubtype::EXECUTABLE_IMAGE, + len: 0, + revision: FIRST_REVISION, + extent: None, + }; + let edge = DiskEdge { + parent: StableObjectId::new(1).unwrap(), + name: Name::new("tool").unwrap(), + child: StableObjectId::new(2).unwrap(), + }; + let (image, len, root) = fixture(&[root_object, tool], &[edge]); + // The fixture is at the current version, which carries subtypes. + assert!(root.version >= SUBTYPE_FORMAT_VERSION); + let decoded = decode_image(&image[..len], root, 512, 256).unwrap(); + assert_eq!(decoded.objects[1].subtype, ObjectSubtype::EXECUTABLE_IMAGE); + assert_eq!(decoded.objects[0].subtype, ObjectSubtype::PLAIN); + + // An unknown subtype is data: carried, not refused. + let (mut image, len, mut root) = fixture(&[root_object, tool], &[edge]); + let at = IMAGE_HEADER + OBJECT_BYTES; + image[at + 10] = 0x7f; + let checksum = CRC.checksum(&image[at..at + 44]); + put_u32(&mut image, at + 44, checksum).unwrap(); + root.image_crc = CRC.checksum(&image[..len]); + let decoded = decode_image(&image[..len], root, 512, 256).unwrap(); + assert_eq!(decoded.objects[1].subtype, ObjectSubtype::from_bits(0x7f)); + } + + #[test] + fn a_directory_with_a_subtype_is_corrupt() { + let root_object = DiskObject { + stable: StableObjectId::new(1).unwrap(), + kind: ObjectKind::Directory, + subtype: ObjectSubtype::EXECUTABLE_IMAGE, + len: 0, + revision: FIRST_REVISION, + extent: None, + }; + let (image, len, root) = fixture(&[root_object], &[]); + assert!(matches!( + decode_image(&image[..len], root, 512, 256), + Err(PersistenceError::Corrupt) + )); + } + + /// Before version 4 the byte was reserved-must-be-zero, and a reader of + /// those versions keeps saying so; an encoder asked to record a subtype + /// at such a version refuses rather than drops it. + #[test] + fn a_version_3_reader_keeps_byte_ten_reserved() { + let root_object = DiskObject { + stable: StableObjectId::new(1).unwrap(), + kind: ObjectKind::Directory, + subtype: ObjectSubtype::PLAIN, + len: 0, + revision: FIRST_REVISION, + extent: None, + }; + let tool = DiskObject { + stable: StableObjectId::new(2).unwrap(), + kind: ObjectKind::File, + subtype: ObjectSubtype::EXECUTABLE_IMAGE, + len: 0, + revision: FIRST_REVISION, + extent: None, + }; + let edge = DiskEdge { + parent: StableObjectId::new(1).unwrap(), + name: Name::new("tool").unwrap(), + child: StableObjectId::new(2).unwrap(), + }; + let (mut image, len, mut root) = fixture(&[root_object, tool], &[edge]); + put_u32(&mut image, 8, ARENA_FORMAT_VERSION).unwrap(); + root.version = ARENA_FORMAT_VERSION; + root.image_crc = CRC.checksum(&image[..len]); + assert!( + matches!( + decode_image(&image[..len], root, 512, 256), + Err(PersistenceError::Corrupt) + ), + "a version-3 record with byte ten set is corrupt, not subtyped" + ); + + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + crate::block_on(volume.engine_mut().create_typed( + session, + root.handle, + &name("tool"), + ObjectKind::File, + ObjectSubtype::EXECUTABLE_IMAGE, + )) + .unwrap(); + let root_id = volume.engine().root_object(session).unwrap(); + let mut scratch = ImageScratch::new().unwrap(); + assert_eq!( + encode_image( + volume.engine(), + root_id, + 11, + ARENA_FORMAT_VERSION, + ImageProfile::Current, + scratch.as_mut(), + ) + .unwrap_err(), + PersistenceError::Incompatible, + "a version that cannot record the stamp refuses rather than drops it" + ); + }); + } + + #[test] + fn subtype_survives_commit_and_cold_recovery() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let made = crate::block_on(volume.engine_mut().create_typed( + session, + root.handle, + &name("tool"), + ObjectKind::File, + ObjectSubtype::EXECUTABLE_IMAGE, + )) + .unwrap(); + assert_eq!(made.subtype, ObjectSubtype::EXECUTABLE_IMAGE); + crate::block_on(volume.engine_mut().write_at( + session, + made.handle, + 0, + b"ELF", + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + + let mut recovered = crate::block_on(volume.restart(SessionId::new(1))).unwrap(); + let session = SessionId::new(1); + let root = recovered + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let tool = recovered + .engine_mut() + .open_at(session, root.handle, &name("tool"), crate::Rights::READ) + .unwrap(); + assert_eq!(tool.subtype, ObjectSubtype::EXECUTABLE_IMAGE); + assert_eq!( + recovered + .engine() + .stat(session, tool.handle) + .unwrap() + .subtype, + ObjectSubtype::EXECUTABLE_IMAGE + ); + // A recovered version-4 volume keeps recording subtypes. + crate::block_on(recovered.engine_mut().create_typed( + session, + root.handle, + &name("another"), + ObjectKind::File, + ObjectSubtype::EXECUTABLE_IMAGE, + )) + .unwrap(); + }); + } + + /// A version-3 volume mounts, reads every object as plain, keeps + /// committing at version 3, and refuses to record a subtype it could not + /// keep. + #[test] + fn a_version_3_volume_mounts_plain_and_refuses_subtyped_creates() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("kept"), + ObjectKind::File, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + rewrite_active_as_legacy(&volume, ARENA_FORMAT_VERSION); + + let mut recovered = crate::block_on(volume.restart(session)).unwrap(); + assert_eq!(recovered.status().mode, MountMode::ReadWrite); + let root = recovered + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let kept = recovered + .engine_mut() + .open_at(session, root.handle, &name("kept"), crate::Rights::READ) + .unwrap(); + assert_eq!(kept.subtype, ObjectSubtype::PLAIN); + assert_eq!( + crate::block_on(recovered.engine_mut().create_typed( + session, + root.handle, + &name("tool"), + ObjectKind::File, + ObjectSubtype::EXECUTABLE_IMAGE, + )) + .unwrap_err(), + crate::FsError::Unsupported + ); + crate::block_on(recovered.engine_mut().create( + session, + root.handle, + &name("plain"), + ObjectKind::File, + )) + .unwrap(); + crate::block_on(recovered.commit(session)).unwrap(); + + // Preserved, not re-stamped: the volume still says version 3. + let provider = recovered.engine().provider(); + let block = provider.geometry().block_size as usize; + let mut head = [0u8; 4096]; + let slot = (recovered.status().generation & 1) as u64; + crate::block_on(provider.read_metadata(slot, &mut head[..block])).unwrap(); + let current = decode_root( + &head[..block], + provider.geometry().block_size, + provider.geometry().block_count, + ) + .unwrap(); + assert_eq!(current.version, ARENA_FORMAT_VERSION); + }); + } + + #[test] + fn legacy_v1_image_decodes_without_fabricating_derived_state() { + let root_object = DiskObject { + stable: StableObjectId::new(1).unwrap(), + kind: ObjectKind::Directory, + subtype: ObjectSubtype::PLAIN, + len: 0, + revision: FIRST_REVISION, + extent: None, + }; + let (mut image, len, mut root) = fixture(&[root_object], &[]); + put_u32(&mut image, 8, LEGACY_FORMAT_VERSION).unwrap(); + image[48..IMAGE_HEADER].fill(0); + root.version = LEGACY_FORMAT_VERSION; + root.image_crc = CRC.checksum(&image[..len]); + + let decoded = decode_image(&image[..len], root, 512, 256).unwrap(); + assert_eq!(decoded.objects.len(), 1); + assert!(decoded.derived.is_empty()); + } + + #[test] + fn retained_scratch_reencodes_v1_without_stale_reserved_bytes() { + on_large_stack(|| { + let session = SessionId::new(0); + let volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root_id = volume.engine().root_object(session).unwrap(); + let mut scratch = ImageScratch::new().unwrap(); + scratch.as_mut().fill(0xa5); + let (len, root) = encode_image( + volume.engine(), + root_id, + 11, + LEGACY_FORMAT_VERSION, + ImageProfile::Legacy, + scratch.as_mut(), + ) + .unwrap(); + assert!( + scratch.as_mut()[48..IMAGE_HEADER] + .iter() + .all(|byte| *byte == 0) + ); + let record = RootRecord { + version: LEGACY_FORMAT_VERSION, + generation: 11, + block_size: 512, + block_count: 640, + slot: 1, + root, + image_len: len as u32, + image_crc: CRC.checksum(&scratch.as_mut()[..len]), + data_start: LEGACY_DATA_START_LBA, + arena_blocks: 640 - LEGACY_DATA_START_LBA, + era: [0u8; ERA_BYTES], + sidecar: None, + }; + assert!( + decode_image(&scratch.as_mut()[..len], record, 512, 640) + .unwrap() + .derived + .is_empty() + ); + }); + } + + #[test] + fn legacy_profile_refuses_a_directory_beyond_its_32_entry_contract() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("base"), + ObjectKind::File, + )) + .unwrap(); + for index in 0..32 { + volume + .engine_mut() + .link( + session, + root.handle, + &name(&std::format!("l{index}")), + file.handle, + ) + .unwrap(); + } + let root_id = volume.engine().root_object(session).unwrap(); + let mut scratch = ImageScratch::new().unwrap(); + assert_eq!( + encode_image( + volume.engine(), + root_id, + 12, + DERIVED_FORMAT_VERSION, + ImageProfile::Legacy, + scratch.as_mut(), + ), + Err(PersistenceError::Bounds) + ); + }); + } + + #[test] + fn decoder_refuses_graphs_that_exceed_live_directory_bounds() { + let dir = |stable| DiskObject { + stable: StableObjectId::new(stable).unwrap(), + kind: ObjectKind::Directory, + subtype: ObjectSubtype::PLAIN, + len: 0, + revision: FIRST_REVISION, + extent: None, + }; + let file = DiskObject { + stable: StableObjectId::new(2).unwrap(), + kind: ObjectKind::File, + subtype: ObjectSubtype::PLAIN, + len: 0, + revision: FIRST_REVISION, + extent: None, + }; + let current = image_layout(ImageProfile::Current, 512).unwrap(); + let make_current = |record: &mut RootRecord| { + record.block_count = 640; + record.data_start = current.data_start; + record.arena_blocks = 640 - current.data_start; + }; + + let mut edges = std::vec::Vec::new(); + for index in 0..=MAX_DIR_ENTRIES { + edges.push(DiskEdge { + parent: StableObjectId::new(1).unwrap(), + child: file.stable, + name: name(&std::format!("e{index}")), + }); + } + let (image, len, mut record) = fixture(&[dir(1), file], &edges); + make_current(&mut record); + assert!(matches!( + decode_image(&image[..len], record, 512, 640), + Err(PersistenceError::Bounds) + )); + + let mut directories = std::vec::Vec::new(); + let mut edges = std::vec::Vec::new(); + for stable in 1..=MAX_DIRECTORIES as u64 + 1 { + directories.push(dir(stable)); + if stable != 1 { + edges.push(DiskEdge { + parent: StableObjectId::new(1).unwrap(), + child: StableObjectId::new(stable).unwrap(), + name: name(&std::format!("d{stable}")), + }); + } + } + let (image, len, mut record) = fixture(&directories, &edges); + make_current(&mut record); + assert!(matches!( + decode_image(&image[..len], record, 512, 640), + Err(PersistenceError::Bounds) + )); + } + + /// A current-profile image written before the edge budget was decoupled + /// could record up to the old `16 × 128` product. Such a volume must keep + /// mounting: the reader admits the old product, and only a count past it + /// is refused. The writer's ceiling is a separate, lower figure. + #[test] + fn decoder_admits_pre_decoupling_edge_counts_but_not_more() { + let dir = |stable: u64| DiskObject { + stable: StableObjectId::new(stable).unwrap(), + kind: ObjectKind::Directory, + subtype: ObjectSubtype::PLAIN, + len: 0, + revision: FIRST_REVISION, + extent: None, + }; + let file = DiskObject { + stable: StableObjectId::new(10).unwrap(), + kind: ObjectKind::File, + subtype: ObjectSubtype::PLAIN, + len: 0, + revision: FIRST_REVISION, + extent: None, + }; + let current = image_layout(ImageProfile::Current, 512).unwrap(); + let make_current = |record: &mut RootRecord| { + record.block_count = 640; + record.data_start = current.data_start; + record.arena_blocks = 640 - current.data_start; + }; + let edge = |parent: u64, child: u64, name_text: &str| DiskEdge { + parent: StableObjectId::new(parent).unwrap(), + child: StableObjectId::new(child).unwrap(), + name: name(name_text), + }; + + // Root 1 holds eight directories and one link to the file; each + // directory holds 127 more links. 8 + 1 + 8 × 127 = 1025 edges: one + // past the writer's ceiling, well inside the old product, and every + // directory inside `MAX_DIR_ENTRIES`. + let mut objects = std::vec![dir(1)]; + let mut edges = std::vec::Vec::new(); + for sub in 2..=9u64 { + objects.push(dir(sub)); + edges.push(edge(1, sub, &std::format!("d{sub}"))); + for link in 0..127 { + edges.push(edge(sub, 10, &std::format!("l{link}"))); + } + } + objects.push(file); + edges.push(edge(1, 10, "f")); + assert_eq!(edges.len(), MAX_DURABLE_EDGES + 1); + let (image, len, mut record) = fixture(&objects, &edges); + make_current(&mut record); + let decoded = decode_image(&image[..len], record, 512, 640).unwrap(); + assert_eq!(decoded.edges.len(), MAX_DURABLE_EDGES + 1); + assert_eq!(decoded.objects.len(), objects.len()); + + // One past the old product is not an old writer; it is corruption. + let mut edges = std::vec::Vec::new(); + for index in 0..=PRE_DECOUPLING_MAX_DURABLE_EDGES { + edges.push(edge(1, 10, &std::format!("e{index}"))); + } + let (image, len, mut record) = fixture(&[dir(1), file], &edges); + make_current(&mut record); + assert!(matches!( + decode_image(&image[..len], record, 512, 640), + Err(PersistenceError::Bounds) + )); + } + + /// A version-2 image still yields its derived table. + /// + /// The gate used to read `root.version == FORMAT_VERSION`, so the moment + /// the current version moved past 2 every v2 volume would have mounted + /// with an EMPTY index while its batches sat on disk — a silent + /// false-negative for every `CurrentOnly` search on that volume. The gate + /// is "2 or later" for exactly this reason. + #[test] + fn a_version_2_image_keeps_its_derived_table() { + let root_object = DiskObject { + stable: StableObjectId::new(1).unwrap(), + kind: ObjectKind::Directory, + subtype: ObjectSubtype::PLAIN, + len: 0, + revision: FIRST_REVISION, + extent: None, + }; + let (mut image, len, mut root) = fixture(&[root_object], &[]); + put_u32(&mut image, 8, DERIVED_FORMAT_VERSION).unwrap(); + root.version = DERIVED_FORMAT_VERSION; + root.image_crc = CRC.checksum(&image[..len]); + + let decoded = decode_image(&image[..len], root, 512, 256).unwrap(); + assert_eq!(decoded.objects.len(), 1); + // The fixture publishes no batches, but the COUNT FIELD was read + // rather than forced to zero: a v2 image whose header says otherwise + // is no longer silently treated as unindexed. + assert!(decoded.derived.is_empty()); + } + + /// Everything version 3 added has to survive a round trip, or the fields + /// exist on disk without meaning anything — at EVERY version that carries + /// the version-3 header, not only the current one. + /// + /// This test once encoded only `FORMAT_VERSION`, so when version 4 landed + /// it kept passing while every real version-3 root decoded as legacy: a + /// raised-profile v3 volume located its image at the legacy slot and lost + /// its era. Version 3 is enumerated explicitly so the next bump cannot + /// repeat that. + #[test] + fn a_version_3_root_round_trips_its_arena_and_era() { + let era = core::array::from_fn(|index| index as u8 ^ 0xa5); + let data_start = image_layout(ImageProfile::Current, 512).unwrap().data_start + 8; + for version in [ARENA_FORMAT_VERSION, SUBTYPE_FORMAT_VERSION, FORMAT_VERSION] { + let record = RootRecord { + version, + generation: 9, + block_size: 512, + block_count: 4096, + slot: 1, + root: StableObjectId::new(3).unwrap(), + image_len: 64, + image_crc: 0x1234_5678, + data_start, + arena_blocks: 1000, + era, + sidecar: None, + }; + let mut block = [0u8; 512]; + encode_root(record, &mut block).unwrap(); + assert_eq!( + Some(get_u32(&block, 12).unwrap() as usize), + root_header_bytes(version) + ); + + let back = decode_root(&block, 512, 4096).unwrap(); + assert_eq!(back.version, version); + assert_eq!(back.generation, 9); + assert_eq!(back.data_start, data_start, "v{version} arena start"); + assert_eq!(back.arena_blocks, 1000, "v{version} arena span"); + assert_eq!(back.era, era, "v{version} era"); + assert_eq!(back.root, record.root); + assert_eq!(back.image_crc, record.image_crc); + // The raised profile is what the recorded arena start selects; + // decoding v3 as legacy would have answered `Legacy` here. + assert_eq!( + root_profile(back.version, back.data_start, 512), + Ok(ImageProfile::Current), + "v{version} slot profile" + ); + } + } + + /// A pre-v3 root records neither arena nor era, so it must decode to what + /// it MEANT — the compile-time floor and everything above it — not to + /// zero, which would describe a volume with no data region at all. + #[test] + fn a_pre_v3_root_decodes_to_the_arena_it_meant() { + for version in [LEGACY_FORMAT_VERSION, DERIVED_FORMAT_VERSION] { + let record = RootRecord { + version, + generation: 4, + block_size: 512, + block_count: 4096, + slot: 0, + root: StableObjectId::new(1).unwrap(), + image_len: 64, + image_crc: 7, + // Ignored at this version: the fields do not exist on disk. + data_start: 999, + arena_blocks: 999, + era: [0xff; ERA_BYTES], + sidecar: None, + }; + let mut block = [0u8; 512]; + encode_root(record, &mut block).unwrap(); + assert_eq!( + get_u32(&block, 12).unwrap() as usize, + ROOT_HEADER_V2, + "v{version} must keep its own header length" + ); + // Nothing of the v3 fields was written into a v2 record. + assert!(block[ROOT_HEADER_V2..].iter().all(|byte| *byte == 0)); + + let back = decode_root(&block, 512, 4096).unwrap(); + assert_eq!(back.version, version); + assert_eq!(back.data_start, LEGACY_DATA_START_LBA); + assert_eq!(back.arena_blocks, 4096 - LEGACY_DATA_START_LBA); + assert_eq!(back.era, [0u8; ERA_BYTES]); + } + } + + /// The header length is validated against the one its version defines, so + /// a record claiming a version it does not carry the fields for is + /// refused rather than read past its own data. + #[test] + fn a_version_and_header_length_that_disagree_are_incompatible() { + let record = RootRecord { + version: DERIVED_FORMAT_VERSION, + generation: 4, + block_size: 512, + block_count: 4096, + slot: 0, + root: StableObjectId::new(1).unwrap(), + image_len: 64, + image_crc: 7, + data_start: LEGACY_DATA_START_LBA, + arena_blocks: 4096 - LEGACY_DATA_START_LBA, + era: [0u8; ERA_BYTES], + sidecar: None, + }; + let mut block = [0u8; 512]; + encode_root(record, &mut block).unwrap(); + // v2 bytes relabelled as v3: the v3 reader would look for an era at + // offset 72 that was never written. + put_u32(&mut block, 8, FORMAT_VERSION).unwrap(); + assert_eq!( + decode_root(&block, 512, 4096), + Err(PersistenceError::Incompatible) + ); + + // And a version this build has never heard of is refused, not guessed. + let mut block = [0u8; 512]; + encode_root(record, &mut block).unwrap(); + put_u32(&mut block, 8, 99).unwrap(); + assert_eq!( + decode_root(&block, 512, 4096), + Err(PersistenceError::Incompatible) + ); + } + + /// An arena below the metadata reservation or past the end of the medium + /// describes a volume that cannot exist; recording it does not make it + /// real. + #[test] + fn an_impossible_v3_arena_is_refused() { + let base = RootRecord { + version: FORMAT_VERSION, + generation: 4, + block_size: 512, + block_count: 4096, + slot: 0, + root: StableObjectId::new(1).unwrap(), + image_len: 64, + image_crc: 7, + data_start: LEGACY_DATA_START_LBA, + arena_blocks: 4096 - LEGACY_DATA_START_LBA, + era: [0u8; ERA_BYTES], + sidecar: None, + }; + for (data_start, arena_blocks) in [ + (0u64, 16u64), // under the reservation + (LEGACY_DATA_START_LBA, 4096), // past the medium + (LEGACY_DATA_START_LBA, u64::MAX), // overflowing + ] { + let mut block = [0u8; 512]; + encode_root( + RootRecord { + data_start, + arena_blocks, + ..base + }, + &mut block, + ) + .unwrap(); + assert_eq!( + decode_root(&block, 512, 4096), + Err(PersistenceError::Corrupt), + "arena [{data_start}, +{arena_blocks}) must be refused" + ); + } + } + + /// A commit moves only the bytes it wrote. + /// + /// Proven by poisoning the tail of the slot the next generation publishes + /// into and showing the poison SURVIVES: if the commit still transferred + /// the whole reservation, it would have overwritten it. This is what makes + /// raising the reservation affordable — otherwise every `touch` on a small + /// volume pays the full metadata arena. + #[test] + fn a_commit_transfers_only_the_image_it_wrote() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let block_size = volume.engine().provider().geometry().block_size; + let block = block_size as usize; + let slot_blocks = image_layout(ImageProfile::Current, block_size) + .unwrap() + .slot_blocks; + + // The generation about to be published, and therefore its slot. + let next_slot = ((volume.status().generation + 1) & 1) as u8; + let arena_lba = image_lba(ImageProfile::Current, next_slot, block_size).unwrap(); + + // Poison every block of that slot except the first. + let poison = [0xc7u8; 512]; + for offset in 1..slot_blocks as u64 { + crate::block_on( + volume + .engine() + .provider() + .write_metadata(arena_lba + offset, &poison[..block]), + ) + .unwrap(); + } + + crate::block_on(volume.commit(session)).unwrap(); + assert_eq!( + (volume.status().generation & 1) as u8, + next_slot, + "the commit must have published into the poisoned slot" + ); + + // A fresh tree's image is one block, so every poisoned block is + // still poisoned — the commit did not touch the reservation's tail. + let mut back = [0u8; 512]; + for offset in 1..slot_blocks as u64 { + crate::block_on( + volume + .engine() + .provider() + .read_metadata(arena_lba + offset, &mut back[..block]), + ) + .unwrap(); + assert_eq!( + &back[..block], + &poison[..block], + "block {offset} of the slot was rewritten despite carrying no image" + ); + } + + // And the stale tail is never read: recovery reads only the length + // the root records, so the volume still mounts. + let recovered = crate::block_on(volume.restart(session)).unwrap(); + assert_eq!(recovered.status().mode, MountMode::ReadWrite); + }); + } + + /// The transfer span is a whole number of blocks, never more than the + /// reservation, and an empty or oversized image is refused rather than + /// silently clamped. + #[test] + fn the_image_transfer_span_is_block_aligned_and_bounded() { + assert_eq!(image_transfer_bytes(1, 512, MAX_IMAGE_BYTES), Ok(512)); + assert_eq!(image_transfer_bytes(512, 512, MAX_IMAGE_BYTES), Ok(512)); + assert_eq!(image_transfer_bytes(513, 512, MAX_IMAGE_BYTES), Ok(1024)); + assert_eq!( + image_transfer_bytes(MAX_IMAGE_BYTES, 512, MAX_IMAGE_BYTES), + Ok(MAX_IMAGE_BYTES) + ); + assert_eq!( + image_transfer_bytes(MAX_IMAGE_BYTES + 1, 512, MAX_IMAGE_BYTES), + Err(PersistenceError::Bounds) + ); + assert_eq!( + image_transfer_bytes(0, 512, MAX_IMAGE_BYTES), + Err(PersistenceError::Bounds) + ); + // A block size the reservation is not a multiple of cannot address the + // slot at all. + assert_eq!( + image_transfer_bytes(64, 768, MAX_IMAGE_BYTES), + Err(PersistenceError::Incompatible) + ); + } + + #[test] + fn current_and_legacy_profiles_have_unambiguous_slot_boundaries() { + let legacy_512 = image_layout(ImageProfile::Legacy, 512).unwrap(); + let current_512 = image_layout(ImageProfile::Current, 512).unwrap(); + assert_eq!(legacy_512.data_start, LEGACY_DATA_START_LBA); + assert_eq!(current_512.data_start, 514); + assert_eq!(bootstrap_data_start(512), Ok(current_512.data_start)); + + // Both natural metadata ends fit below block 72 on 4 KiB media, so + // current uses block 73 as an explicit profile discriminator. + let legacy_4k = image_layout(ImageProfile::Legacy, 4096).unwrap(); + let current_4k = image_layout(ImageProfile::Current, 4096).unwrap(); + assert_eq!(legacy_4k.data_start, LEGACY_DATA_START_LBA); + assert_eq!(current_4k.data_start, LEGACY_DATA_START_LBA + 1); + assert_ne!(legacy_4k.slot_blocks, current_4k.slot_blocks); + } + + /// **A version-2 volume written by an older build still mounts, and its + /// data comes back.** + /// + /// The two decode halves are unit-tested above, but the guarantee that + /// matters is end to end: a real disk carrying a v2 root and a v2 image — + /// which is what every volume in the field carries — recovers its whole + /// namespace and its bytes under a build whose current version is 3. This + /// downgrades a freshly formatted volume's on-disk records to v2 and + /// recovers through the ordinary path, no test seam in production code. + #[test] + fn a_version_2_volume_on_disk_still_mounts_and_returns_its_data() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("kept"), + ObjectKind::File, + )) + .unwrap(); + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"v2-bytes", + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + + // Rewrite the published generation as version 2, exactly as a + // build that predates v3 would have left it. + rewrite_active_as_legacy(&volume, DERIVED_FORMAT_VERSION); + + // The ordinary recovery path, over v2 bytes. + let mut recovered = crate::block_on(volume.restart(session)).unwrap(); + assert_eq!( + recovered.status().mode, + MountMode::ReadWrite, + "a v2 volume must mount writable, not land in a degraded posture" + ); + assert_eq!( + recovered.status().era, + [0u8; ERA_BYTES], + "an unstamped volume reads as unstamped, never as some other era" + ); + + let root = recovered + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let kept = recovered + .engine_mut() + .open_at(session, root.handle, &name("kept"), crate::Rights::ALL) + .unwrap(); + let mut back = [0u8; 8]; + assert_eq!( + crate::block_on( + recovered + .engine_mut() + .read(session, kept.handle, 0, &mut back) + ), + Ok(8) + ); + assert_eq!(&back, b"v2-bytes"); + + // And it can be written again without silently raising its + // reservation. Capacity grows only through deliberate reprovision. + crate::block_on(recovered.commit(session)).unwrap(); + }); + } + + #[test] + fn a_pre_7b_version_3_volume_keeps_its_legacy_slot_profile() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("old-v3"), + ObjectKind::File, + )) + .unwrap(); + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"legacy-v3", + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + rewrite_active_as_legacy(&volume, FORMAT_VERSION); + + let mut recovered = crate::block_on(volume.restart(session)).unwrap(); + assert_eq!(recovered.state.profile, ImageProfile::Legacy); + assert_eq!( + recovered.engine().provider().arena().0, + LEGACY_DATA_START_LBA + ); + let root = recovered + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = recovered + .engine_mut() + .open_at(session, root.handle, &name("old-v3"), crate::Rights::ALL) + .unwrap(); + let mut back = [0u8; 9]; + assert_eq!( + crate::block_on( + recovered + .engine_mut() + .read(session, file.handle, 0, &mut back,) + ), + Ok(9) + ); + assert_eq!(&back, b"legacy-v3"); + + // A commit preserves the old slot addresses and bounds. It does + // not silently reinterpret a pre-7b v3 volume as a raised v3 one. + crate::block_on(recovered.commit(session)).unwrap(); + let provider = recovered.engine().provider(); + let block_size = provider.geometry().block_size; + let block = block_size as usize; + let slot = (recovered.status().generation & 1) as u64; + let mut head = [0u8; 4096]; + assert_eq!( + crate::block_on(provider.read_metadata(slot, &mut head[..block])), + Ok(()) + ); + let root = + decode_root(&head[..block], block_size, provider.geometry().block_count).unwrap(); + assert_eq!(root.version, FORMAT_VERSION); + assert_eq!(root.data_start, LEGACY_DATA_START_LBA); + assert_eq!( + root_profile(root.version, root.data_start, block_size), + Ok(ImageProfile::Legacy) + ); + }); + } + + #[test] + fn current_profile_commits_a_graph_beyond_both_legacy_totals() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let directory = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("links"), + ObjectKind::Directory, + )) + .unwrap(); + for index in 0..64 { + let entry = name(&std::format!("f{index}")); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &entry, + ObjectKind::File, + )) + .unwrap(); + volume + .engine_mut() + .link(session, directory.handle, &entry, file.handle) + .unwrap(); + volume.engine().close(session, file.handle).unwrap(); + } + let root_id = volume.engine().root_object(session).unwrap(); + let graph = volume + .engine() + .namespace() + .durable_graph(root_id, MAX_DURABLE_EDGES) + .unwrap(); + assert!(graph.objects.len() > LEGACY_MAX_DURABLE_OBJECTS); + assert!(graph.edges.len() > LEGACY_MAX_DURABLE_EDGES); + crate::block_on(volume.commit(session)).unwrap(); + + let mut recovered = crate::block_on(volume.restart(session)).unwrap(); + let root = recovered + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let directory = recovered + .engine_mut() + .open_at(session, root.handle, &name("links"), crate::Rights::ALL) + .unwrap(); + assert!( + recovered + .engine_mut() + .open_at(session, directory.handle, &name("f63"), crate::Rights::ALL,) + .is_ok() + ); + }); + } + + #[test] + fn complete_namespace_and_bytes_survive_with_fresh_authority() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let left = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("left"), + ObjectKind::Directory, + )) + .unwrap(); + let right = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("right"), + ObjectKind::Directory, + )) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + left.handle, + &name("data"), + ObjectKind::File, + )) + .unwrap(); + let revision = crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"durable-bytes", + crate::ANY_REVISION, + )) + .unwrap(); + volume + .engine_mut() + .link(session, root.handle, &name("alias"), file.handle) + .unwrap(); + volume + .engine_mut() + .rename(session, root.handle, &name("alias"), &name("renamed")) + .unwrap(); + volume + .engine_mut() + .relocate( + session, + left.handle, + &name("data"), + right.handle, + &name("moved"), + ) + .unwrap(); + crate::block_on( + volume + .engine_mut() + .unlink(session, root.handle, &name("renamed")), + ) + .unwrap(); + let moved = volume + .engine_mut() + .open_at(session, right.handle, &name("moved"), crate::Rights::ALL) + .unwrap(); + let revision = crate::block_on(volume.engine_mut().resize( + session, + moved.handle, + 7, + revision, + )) + .unwrap(); + crate::block_on(volume.engine_mut().sync(session, moved.handle)).unwrap(); + let stable = volume + .engine() + .stat(session, moved.handle) + .unwrap() + .stable_object; + let stale_handle = moved.handle; + crate::block_on(volume.commit(session)).unwrap(); + + let recovered_session = SessionId::new(2); + let mut volume = crate::block_on(volume.restart(recovered_session)).unwrap(); + assert_eq!( + volume.engine().authority_edge_count(), + 1, + "disk minted no old grants or sessions" + ); + assert_eq!( + volume.engine().stat(recovered_session, stale_handle), + Err(FsError::InvalidHandle) + ); + let root = volume + .engine_mut() + .open_root(recovered_session, crate::Rights::ALL) + .unwrap(); + assert!( + volume + .engine_mut() + .open_at( + recovered_session, + root.handle, + &name("left"), + crate::Rights::ALL + ) + .is_ok() + ); + assert_eq!( + volume + .engine_mut() + .open_at( + recovered_session, + root.handle, + &name("renamed"), + crate::Rights::ALL + ) + .unwrap_err(), + FsError::NotFound + ); + let right = volume + .engine_mut() + .open_at( + recovered_session, + root.handle, + &name("right"), + crate::Rights::ALL, + ) + .unwrap(); + let moved = volume + .engine_mut() + .open_at( + recovered_session, + right.handle, + &name("moved"), + crate::Rights::ALL, + ) + .unwrap(); + let stat = volume + .engine() + .stat(recovered_session, moved.handle) + .unwrap(); + assert_eq!(stat.stable_object, stable); + assert_eq!(stat.revision, revision); + assert_eq!(stat.links, 1); + let mut bytes = [0u8; 16]; + assert_eq!( + crate::block_on(volume.engine_mut().read( + recovered_session, + moved.handle, + 0, + &mut bytes + )) + .unwrap(), + 7 + ); + assert_eq!(&bytes[..7], b"durable"); + }); + } + + #[test] + fn current_content_index_survives_a_cold_provider_restart() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("note"), + ObjectKind::File, + )) + .unwrap(); + // A newly created empty file already has a complete application + // batch. Mutating its authoritative revision must publish a root + // without that now-stale derived extent, not reject the namespace + // commit as corrupt. + let empty_owner = crate::extraction::ExtractionOwner::new(40); + let (empty_lease, empty_input) = crate::block_on(volume.engine_mut().lend_version( + empty_owner, + session, + file.handle, + )) + .unwrap(); + let parser = Registry::get(PLAIN_TEXT).unwrap(); + let empty = NormalizedResult::empty(ParserProvenance::deterministic( + empty_input, + parser.id, + parser.version, + )); + let mut empty_wire = [0u8; WORKER_REPORT_BYTES]; + let empty_used = encode_result(&empty, &mut empty_wire).unwrap(); + crate::block_on(volume.engine_mut().commit_encoded_derived_result( + empty_owner, + empty_lease, + empty, + &empty_wire[..empty_used], + 0, + )) + .unwrap(); + crate::block_on( + volume + .engine_mut() + .release_lent_version(empty_owner, empty_lease), + ) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + + let bytes = b"durable capability search"; + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + bytes, + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + assert_eq!( + volume.engine().search( + session, + root.handle, + crate::SearchQuery { + content: Some(crate::SearchContent { + plane: crate::SearchPlane::FullText, + term: crate::SearchTerm::new("capability").unwrap(), + }), + freshness: crate::FreshnessPolicy::CurrentOnly, + ..crate::SearchQuery::ALL + }, + None, + ), + Err(FsError::IndexIncomplete) + ); + let owner = crate::extraction::ExtractionOwner::new(41); + let (lease, input) = crate::block_on(volume.engine_mut().lend_version( + owner, + session, + file.handle, + )) + .unwrap(); + let result = parse_plain_text(input, bytes).unwrap(); + let mut wire = [0u8; WORKER_REPORT_BYTES]; + let used = encode_result(&result, &mut wire).unwrap(); + crate::block_on(volume.engine_mut().commit_encoded_derived_result( + owner, + lease, + result, + &wire[..used], + 0, + )) + .unwrap(); + crate::block_on(volume.engine_mut().release_lent_version(owner, lease)).unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + + let recovered_session = SessionId::new(2); + let mut volume = crate::block_on(volume.restart(recovered_session)).unwrap(); + let root = volume + .engine_mut() + .open_root(recovered_session, crate::Rights::ALL) + .unwrap(); + let page = volume + .engine() + .search( + recovered_session, + root.handle, + crate::SearchQuery { + content: Some(crate::SearchContent { + plane: crate::SearchPlane::FullText, + term: crate::SearchTerm::new("capability").unwrap(), + }), + freshness: crate::FreshnessPolicy::CurrentOnly, + ..crate::SearchQuery::ALL + }, + None, + ) + .unwrap(); + assert_eq!(page.hits.len(), 1); + assert_eq!(page.hits[0].path[0], name("note")); + }); + } + + #[test] + fn block_backed_file_larger_than_one_transfer_stays_index_incomplete_without_a_batch() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("large-note"), + ObjectKind::File, + )) + .unwrap(); + let first = [b'x'; crate::bounds::MAX_TRANSFER]; + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + &first, + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + crate::bounds::MAX_TRANSFER as u64, + b"n", + crate::ANY_REVISION, + )) + .unwrap(); + assert_eq!( + volume.engine().stat(session, file.handle).unwrap().len, + crate::bounds::MAX_TRANSFER as u64 + 1 + ); + assert_eq!( + volume.engine().search( + session, + root.handle, + crate::SearchQuery { + content: Some(crate::SearchContent { + plane: crate::SearchPlane::FullText, + term: crate::SearchTerm::new("needle").unwrap(), + }), + freshness: crate::FreshnessPolicy::CurrentOnly, + ..crate::SearchQuery::ALL + }, + None, + ), + Err(FsError::IndexIncomplete) + ); + }); + } + + /// A valid batch larger than the one the store keeps per version, on the + /// durable provider a machine mounts: before, it passed the provider's + /// transfer bound, was written and synced to a fresh placement, and was + /// then refused `Range` when the hot store could not pack it — failing + /// the write it describes, and `find reindex`. Now it is refused as a + /// batch this index cannot hold before a block is spent, and the version + /// stays unindexed. + #[test] + fn an_oversized_batch_is_refused_before_a_placement_is_spent() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("notes"), + ObjectKind::File, + )) + .unwrap(); + let text: std::string::String = (0..20) + .map(|line| { + std::format!( + "minute{line} reviewed storage{line} batches{line} answer{line} quickly{line}\n" + ) + }) + .collect(); + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + text.as_bytes(), + crate::ANY_REVISION, + )) + .unwrap(); + let owner = crate::extraction::ExtractionOwner::new(43); + let (lease, input) = crate::block_on(volume.engine_mut().lend_version( + owner, + session, + file.handle, + )) + .unwrap(); + // The parser's spans plus a token fact per distinct word: the + // batch the plain-text producer published before it bounded it. + let mut result = parse_plain_text(input, text.as_bytes()).unwrap(); + for word in text.split(|byte: char| !byte.is_ascii_alphanumeric()) { + if word.is_empty() || result.facts.iter().any(|fact| fact.key.as_str() == word) { + continue; + } + let fact = kore_svc_extract::Fact { + kind: kore_svc_extract::FactKind::Custom, + key: word.try_into().unwrap(), + value: kore_svc_extract::FactValue::Utf8(word.try_into().unwrap()), + confidence: kore_svc_extract::Confidence::CERTAIN, + source: None, + }; + if result.facts.push(fact).is_err() { + break; + } + } + let mut wire = [0u8; 2 * WORKER_REPORT_BYTES]; + let used = encode_result(&result, &mut wire).unwrap(); + assert!( + used > WORKER_REPORT_BYTES + && used <= volume.engine().provider().limits().max_transfer, + "the batch must clear the provider's bound and miss the store's" + ); + let (placements, writes) = ( + volume.engine().provider().placements(), + volume.engine().provider().stats().writes, + ); + + assert_eq!( + crate::block_on(volume.engine_mut().commit_encoded_derived_result( + owner, + lease, + result, + &wire[..used], + 0, + )), + Err(crate::ExtractionError::Oversize) + ); + assert_eq!(volume.engine().provider().placements(), placements); + assert_eq!(volume.engine().provider().stats().writes, writes); + crate::block_on(volume.engine_mut().release_lent_version(owner, lease)).unwrap(); + assert_eq!( + volume + .engine() + .derived_result(session, file.handle, PLAIN_TEXT), + Ok(None) + ); + }); + } + + #[test] + fn corrupt_derived_batch_does_not_make_the_namespace_unmountable() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("note"), + ObjectKind::File, + )) + .unwrap(); + let bytes = b"rebuildable derived search"; + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + bytes, + crate::ANY_REVISION, + )) + .unwrap(); + let owner = crate::extraction::ExtractionOwner::new(42); + let (lease, input) = crate::block_on(volume.engine_mut().lend_version( + owner, + session, + file.handle, + )) + .unwrap(); + let result = parse_plain_text(input, bytes).unwrap(); + let mut wire = [0u8; WORKER_REPORT_BYTES]; + let used = encode_result(&result, &mut wire).unwrap(); + crate::block_on(volume.engine_mut().commit_encoded_derived_result( + owner, + lease, + result, + &wire[..used], + 0, + )) + .unwrap(); + crate::block_on(volume.engine_mut().release_lent_version(owner, lease)).unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + + let durable = volume.engine().durable_derived_results::<1>().unwrap(); + let extent = volume.engine().provider().extent(durable[0].key).unwrap(); + volume + .engine() + .provider() + .control(|device| device.corrupt_byte(extent.lba as usize * 512 + 3, 0x80)) + .unwrap() + .unwrap(); + + let recovered_session = SessionId::new(2); + let mut volume = crate::block_on(volume.restart(recovered_session)).unwrap(); + let root = volume + .engine_mut() + .open_root(recovered_session, crate::Rights::ALL) + .unwrap(); + assert_eq!( + volume.engine().search( + recovered_session, + root.handle, + crate::SearchQuery { + content: Some(crate::SearchContent { + plane: crate::SearchPlane::FullText, + term: crate::SearchTerm::new("derived").unwrap(), + }), + freshness: crate::FreshnessPolicy::CurrentOnly, + ..crate::SearchQuery::ALL + }, + None, + ), + Err(FsError::IndexIncomplete) + ); + assert!( + volume + .engine_mut() + .open_at( + recovered_session, + root.handle, + &name("note"), + crate::Rights::READ, + ) + .is_ok(), + "derived corruption cannot hide or invalidate authoritative files" + ); + }); + } + + #[test] + fn directory_diamond_survives_commit_and_detaches_one_route_exactly() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let left = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("left"), + ObjectKind::Directory, + )) + .unwrap(); + let right = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("right"), + ObjectKind::Directory, + )) + .unwrap(); + let shared = crate::block_on(volume.engine_mut().create( + session, + left.handle, + &name("shared"), + ObjectKind::Directory, + )) + .unwrap(); + volume + .engine_mut() + .link(session, right.handle, &name("shared"), shared.handle) + .unwrap(); + crate::block_on(volume.engine_mut().create( + session, + shared.handle, + &name("leaf"), + ObjectKind::File, + )) + .unwrap(); + let stable = volume + .engine() + .stat(session, shared.handle) + .unwrap() + .stable_object; + crate::block_on(volume.commit(session)).unwrap(); + + let mut volume = crate::block_on(volume.restart(session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let left = volume + .engine_mut() + .open_at(session, root.handle, &name("left"), crate::Rights::ALL) + .unwrap(); + let right = volume + .engine_mut() + .open_at(session, root.handle, &name("right"), crate::Rights::ALL) + .unwrap(); + let through_left = volume + .engine_mut() + .open_at(session, left.handle, &name("shared"), crate::Rights::ALL) + .unwrap(); + let through_right = volume + .engine_mut() + .open_at(session, right.handle, &name("shared"), crate::Rights::ALL) + .unwrap(); + assert_eq!( + volume + .engine() + .stat(session, through_left.handle) + .unwrap() + .stable_object, + stable + ); + assert_eq!( + volume + .engine() + .stat(session, through_right.handle) + .unwrap() + .stable_object, + stable + ); + + crate::block_on( + volume + .engine_mut() + .detach_edge(session, left.handle, &name("shared")), + ) + .unwrap(); + assert_eq!( + volume.engine().stat(session, through_left.handle), + Err(FsError::AccessLost) + ); + assert!(volume.engine().stat(session, through_right.handle).is_ok()); + assert!( + volume + .engine_mut() + .open_at( + session, + through_right.handle, + &name("leaf"), + crate::Rights::ALL, + ) + .is_ok() + ); + }); + } + + #[test] + fn atomic_replacement_cuts_recover_old_or_complete_new() { + on_large_stack(|| { + for stage in [ + CommitStage::BeforeMetadata, + CommitStage::AfterMetadata, + CommitStage::AfterMetadataFlush, + CommitStage::AfterRootWrite, + CommitStage::AfterRootFlush, + ] { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("state"), + ObjectKind::File, + )) + .unwrap(); + let old_revision = crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"old", + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + let mut replacement = crate::block_on(volume.engine().begin_replace( + session, + file.handle, + 3, + old_revision, + )) + .unwrap(); + crate::block_on(volume.engine().write_replacement( + session, + &mut replacement, + 0, + b"n", + )) + .unwrap(); + // Another session's commit while staging must still encode old bytes. + crate::block_on(volume.commit(session)).unwrap(); + crate::block_on(volume.engine().write_replacement( + session, + &mut replacement, + 1, + b"ew", + )) + .unwrap(); + let new_revision = + crate::block_on(volume.engine().commit_replace(session, &replacement)).unwrap(); + assert_eq!( + crate::block_on(volume.commit_with_cut(session, Some(stage))), + Err(PersistenceError::Interrupted(stage)) + ); + let mut recovered = crate::block_on(volume.restart(session)).unwrap(); + let root = recovered + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = recovered + .engine_mut() + .open_at(session, root.handle, &name("state"), crate::Rights::ALL) + .unwrap(); + let mut bytes = [0; 3]; + crate::block_on( + recovered + .engine_mut() + .read(session, file.handle, 0, &mut bytes), + ) + .unwrap(); + let stat = recovered.engine().stat(session, file.handle).unwrap(); + let old = bytes == *b"old" && stat.revision == old_revision; + let new = bytes == *b"new" && stat.revision == new_revision; + assert!(old || new, "{stage:?} recovered a hybrid"); + } + }); + } + + #[test] + fn every_publication_cut_recovers_one_complete_root() { + on_large_stack(|| { + for stage in [ + CommitStage::BeforeMetadata, + CommitStage::AfterMetadata, + CommitStage::AfterMetadataFlush, + CommitStage::AfterRootWrite, + CommitStage::AfterRootFlush, + ] { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("state"), + ObjectKind::File, + )) + .unwrap(); + let old_revision = crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"old", + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + let new_revision = crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"new", + old_revision, + )) + .unwrap(); + assert_eq!( + crate::block_on(volume.commit_with_cut(session, Some(stage))), + Err(PersistenceError::Interrupted(stage)) + ); + let mut recovered = crate::block_on(volume.restart(session)).unwrap(); + let root = recovered + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = recovered + .engine_mut() + .open_at(session, root.handle, &name("state"), crate::Rights::ALL) + .unwrap(); + let mut bytes = [0; 3]; + crate::block_on( + recovered + .engine_mut() + .read(session, file.handle, 0, &mut bytes), + ) + .unwrap(); + let stat = recovered.engine().stat(session, file.handle).unwrap(); + let old = bytes == *b"old" && stat.revision == old_revision; + let new = bytes == *b"new" && stat.revision == new_revision; + assert!(old || new, "{stage:?} recovered a hybrid"); + } + }); + } + + /// Model exec's exact per-member seed window over the real persistent + /// engine: create, chunked writes with no intervening publication, detach + /// on failure, then one closing commit. Retryable backpressure and a + /// terminal channel loss are different backend facts, but neither may leave the + /// already-written prefix discoverable after reboot. + #[test] + fn seed_member_failures_recover_without_a_partial_name() { + on_large_stack(|| { + for (failure, expected) in [ + (BlockError::Backpressure, ProviderError::Backpressure), + (BlockError::Lost, ProviderError::Lost), + ] { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format_with_era( + provider(), + session, + [0x71; ERA_BYTES], + )) + .unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let app = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("app"), + ObjectKind::Directory, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + + let member = crate::block_on(volume.engine_mut().create( + session, + app.handle, + &name("tool"), + ObjectKind::File, + )) + .unwrap(); + let revision = crate::block_on(volume.engine_mut().resize( + session, + member.handle, + 8192, + member.revision, + )) + .unwrap(); + let revision = crate::block_on(volume.engine_mut().write_at( + session, + member.handle, + 0, + &[0x11; 4096], + revision, + )) + .unwrap(); + volume + .engine() + .provider() + .control(|device| device.inject(failure)) + .unwrap(); + assert_eq!( + crate::block_on(volume.engine_mut().write_at( + session, + member.handle, + 4096, + &[0x22; 4096], + revision, + )), + Err(FsError::ProviderFailure(expected)) + ); + + // This is write_member's failure branch before it closes the + // deferral window. Commit the cleaned namespace, then model a + // cold reboot rather than trusting the live engine's tables. + crate::block_on(volume.engine_mut().detach_edge( + session, + app.handle, + &name("tool"), + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + + let mut recovered = crate::block_on(volume.restart(session)).unwrap(); + let root = recovered + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let app = recovered + .engine_mut() + .open_at(session, root.handle, &name("app"), crate::Rights::ALL) + .unwrap(); + assert_eq!( + recovered.engine_mut().open_at( + session, + app.handle, + &name("tool"), + crate::Rights::READ, + ), + Err(FsError::NotFound), + "{failure:?} exposed a partially copied seed after reboot" + ); + } + }); + } + + /// Extend the generic root-publication cut matrix through exec's seed + /// shape. The closing commit may recover as the old root (no name) or the + /// new root (the complete member), but no cut may publish one written + /// prefix as a seeded executable. + #[test] + fn every_seed_publication_cut_recovers_absent_or_complete() { + on_large_stack(|| { + const FIRST: [u8; 4096] = [0x31; 4096]; + const SECOND: [u8; 4096] = [0x32; 4096]; + for stage in [ + CommitStage::BeforeMetadata, + CommitStage::AfterMetadata, + CommitStage::AfterMetadataFlush, + CommitStage::AfterRootWrite, + CommitStage::AfterRootFlush, + ] { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format_with_era( + provider(), + session, + [0x72; ERA_BYTES], + )) + .unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let app = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("app"), + ObjectKind::Directory, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + + let member = crate::block_on(volume.engine_mut().create( + session, + app.handle, + &name("tool"), + ObjectKind::File, + )) + .unwrap(); + let revision = crate::block_on(volume.engine_mut().resize( + session, + member.handle, + (FIRST.len() + SECOND.len()) as u64, + member.revision, + )) + .unwrap(); + let revision = crate::block_on(volume.engine_mut().write_at( + session, + member.handle, + 0, + &FIRST, + revision, + )) + .unwrap(); + crate::block_on(volume.engine_mut().write_at( + session, + member.handle, + FIRST.len() as u64, + &SECOND, + revision, + )) + .unwrap(); + assert_eq!( + crate::block_on(volume.commit_with_cut(session, Some(stage))), + Err(PersistenceError::Interrupted(stage)) + ); + + let mut recovered = crate::block_on(volume.restart(session)).unwrap(); + let root = recovered + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let app = recovered + .engine_mut() + .open_at(session, root.handle, &name("app"), crate::Rights::ALL) + .unwrap(); + let member = recovered.engine_mut().open_at( + session, + app.handle, + &name("tool"), + crate::Rights::READ, + ); + let Ok(member) = member else { + assert_eq!(member, Err(FsError::NotFound)); + continue; + }; + assert_eq!(member.len, (FIRST.len() + SECOND.len()) as u64); + let mut bytes = [0u8; FIRST.len() + SECOND.len()]; + assert_eq!( + crate::block_on(recovered.engine_mut().read( + session, + member.handle, + 0, + &mut bytes[..FIRST.len()], + )), + Ok(FIRST.len()) + ); + assert_eq!( + crate::block_on(recovered.engine_mut().read( + session, + member.handle, + FIRST.len() as u64, + &mut bytes[FIRST.len()..], + )), + Ok(SECOND.len()) + ); + assert_eq!( + &bytes[..FIRST.len()], + &FIRST, + "{stage:?} published a bad prefix" + ); + assert_eq!( + &bytes[FIRST.len()..], + &SECOND, + "{stage:?} published a bad suffix" + ); + } + }); + } + + #[test] + fn a_root_write_cut_freezes_every_extent_it_may_have_published() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("state"), + ObjectKind::File, + )) + .unwrap(); + let old_revision = crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"old", + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + let new_revision = crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"new", + old_revision, + )) + .unwrap(); + assert_eq!( + crate::block_on(volume.commit_with_cut(session, Some(CommitStage::AfterRootWrite))), + Err(PersistenceError::Interrupted(CommitStage::AfterRootWrite)) + ); + + // The interrupted publisher cannot know whether the new root is + // durable. This third, unpublished revision must therefore COW + // rather than overwrite bytes either recoverable root may name. + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"bad", + new_revision, + )) + .unwrap(); + + let mut recovered = crate::block_on(volume.restart(session)).unwrap(); + let root = recovered + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = recovered + .engine_mut() + .open_at(session, root.handle, &name("state"), crate::Rights::ALL) + .unwrap(); + let mut bytes = [0; 3]; + crate::block_on( + recovered + .engine_mut() + .read(session, file.handle, 0, &mut bytes), + ) + .unwrap(); + assert!(bytes == *b"old" || bytes == *b"new"); + }); + } + + #[test] + fn a_file_saved_empty_recovers() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let engine = volume.engine_mut(); + let root = engine.open_root(session, crate::Rights::ALL).unwrap(); + let file = crate::block_on(engine.create(session, root.handle, &name("note"), ObjectKind::File)).unwrap(); + crate::block_on(engine.write_at(session, file.handle, 0, b"text", crate::ANY_REVISION)).unwrap(); + // Saved empty, as the editor saves: a replacement of no bytes, + // whose placement holds no blocks and so no address. + let stage = crate::block_on(engine.begin_replace(session, file.handle, 0, crate::ANY_REVISION)).unwrap(); + crate::block_on(engine.commit_replace(session, &stage)).unwrap(); + engine.close(session, file.handle).unwrap(); + // A second empty file: two addressless extents side by side are + // not an overlap. + let other = crate::block_on(engine.create(session, root.handle, &name("other"), ObjectKind::File)).unwrap(); + let stage = crate::block_on(engine.begin_replace(session, other.handle, 0, crate::ANY_REVISION)).unwrap(); + crate::block_on(engine.commit_replace(session, &stage)).unwrap(); + engine.close(session, other.handle).unwrap(); + // Both roots carry the empty files, so neither may refuse them. + crate::block_on(volume.commit(session)).unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + let provider = volume.engine.into_provider(); + let bootstrap = bootstrap_data_start(provider.geometry().block_size).unwrap(); + let provider = provider.restart_with_data_start(bootstrap).unwrap(); + let mut recovered = match crate::block_on(Volume::recover_witnessed(provider, session)) { + Ok(recovered) => recovered, + Err(failure) => panic!("recovery refused: {failure:?} (witness {:#x})", failure.witness()), + }; + assert_eq!(recovered.status().mode, MountMode::ReadWrite); + // The file is there, empty, and takes bytes again. + let engine = recovered.engine_mut(); + let root = engine.open_root(session, crate::Rights::ALL).unwrap(); + let file = engine.open_at(session, root.handle, &name("note"), crate::Rights::ALL).unwrap(); + assert_eq!(engine.stat(session, file.handle).unwrap().len, 0); + crate::block_on(engine.write_at(session, file.handle, 0, b"again", crate::ANY_REVISION)).unwrap(); + crate::block_on(recovered.commit(session)).unwrap(); + }); + } + + #[test] + fn a_wide_and_deep_directory_tree_recovers() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let engine = volume.engine_mut(); + let root = engine.open_root(session, crate::Rights::ALL).unwrap(); + // Wide: one directory holding many directories, each with a + // child directory, so restore links populated subtrees. + let top = crate::block_on(engine.create(session, root.handle, &name("top"), ObjectKind::Directory)).unwrap(); + for i in 0..120 { + let dir = crate::block_on(engine.create( + session, + top.handle, + &name(&std::format!("d{i}")), + ObjectKind::Directory, + )) + .unwrap(); + let inner = crate::block_on(engine.create(session, dir.handle, &name("inner"), ObjectKind::Directory)).unwrap(); + engine.close(session, inner.handle).unwrap(); + engine.close(session, dir.handle).unwrap(); + } + engine.close(session, top.handle).unwrap(); + // Deep: a chain of directories. + let mut at = crate::block_on(engine.create(session, root.handle, &name("deep"), ObjectKind::Directory)).unwrap(); + for i in 0..crate::bounds::MAX_PATH_DEPTH - 2 { + let next = crate::block_on(engine.create( + session, + at.handle, + &name(&std::format!("c{i}")), + ObjectKind::Directory, + )) + .unwrap(); + engine.close(session, at.handle).unwrap(); + at = next; + } + engine.close(session, at.handle).unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + let provider = volume.engine.into_provider(); + let bootstrap = bootstrap_data_start(provider.geometry().block_size).unwrap(); + let provider = provider.restart_with_data_start(bootstrap).unwrap(); + if let Err(failure) = crate::block_on(Volume::recover_witnessed(provider, session)) { + panic!("recovery refused: {failure:?} (witness {:#x})", failure.witness()); + } + }); + } + + #[test] + fn a_refused_recovery_names_its_stage_and_cause() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + // Both root records damaged: each root falls back to the other, + // and the last refusal is the one reported. + for root in [16, 512 + 16] { + volume + .engine() + .provider() + .control(|device| device.corrupt_byte(root, 0x80)) + .unwrap() + .unwrap(); + } + let provider = volume.engine.into_provider(); + let bootstrap = bootstrap_data_start(provider.geometry().block_size).unwrap(); + let provider = provider.restart_with_data_start(bootstrap).unwrap(); + let failure = match crate::block_on(Volume::recover_witnessed(provider, session)) { + Err(failure) => failure, + Ok(_) => panic!("two damaged roots must refuse the mount"), + }; + assert_eq!(failure.stage, RecoverStage::RootDecode); + let [stage, kind, _, _] = failure.witness().to_le_bytes(); + assert_eq!(stage, RecoverStage::RootDecode as u8); + assert_eq!(kind, match failure.error { + PersistenceError::Corrupt => 3, + PersistenceError::Incompatible => 2, + PersistenceError::Unformatted => 1, + other => panic!("a damaged root record is not {other:?}"), + }); + }); + // A filesystem refusal carries its wire status, and a provider + // failure inside it its provider code beside it. + let failure = RecoverFailure::at( + RecoverStage::Edges, + PersistenceError::Filesystem(FsError::ProviderFailure(ProviderError::Integrity)), + ); + assert_eq!( + failure.witness().to_le_bytes(), + [ + RecoverStage::Edges as u8, + 7, + kore_dagfs::wire::fs_status_code(FsError::ProviderFailure(ProviderError::Integrity)), + kore_dagfs::wire::provider_code(ProviderError::Integrity), + ] + ); + let interrupted = RecoverFailure::at(RecoverStage::Extents, PersistenceError::Interrupted(CommitStage::BeforeMetadata)); + assert_eq!(interrupted.witness().to_le_bytes(), [RecoverStage::Extents as u8, 5, 1, 0]); + } + + #[test] + fn corrupt_newest_root_falls_back_without_losing_old_payload() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("state"), + ObjectKind::File, + )) + .unwrap(); + let old_revision = crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"old", + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); // generation 2, slot 0 + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"new", + old_revision, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); // generation 3, slot 1 + volume + .engine() + .provider() + .control(|device| device.corrupt_byte(512 + 16, 0x80)) + .unwrap() + .unwrap(); + let mut recovered = crate::block_on(volume.restart(session)).unwrap(); + assert_eq!( + recovered.status(), + MountStatus { + generation: 2, + used_fallback_root: true, + mode: MountMode::DegradedReadOnly, + era: [0u8; ERA_BYTES], + } + ); + assert_eq!( + crate::block_on(recovered.commit(session)), + Err(PersistenceError::Provider(ProviderError::ReadOnly)) + ); + let root = recovered + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = recovered + .engine_mut() + .open_at(session, root.handle, &name("state"), crate::Rights::ALL) + .unwrap(); + let mut bytes = [0; 3]; + crate::block_on( + recovered + .engine_mut() + .read(session, file.handle, 0, &mut bytes), + ) + .unwrap(); + assert_eq!(bytes, *b"old"); + assert_eq!( + recovered + .engine() + .stat(session, file.handle) + .unwrap() + .revision, + old_revision + ); + }); + } + + #[test] + fn parser_rejects_stale_duplicate_cyclic_overlapping_and_out_of_range_records() { + let sid = |value| StableObjectId::new(value).unwrap(); + let root = DiskObject { + stable: sid(1), + kind: ObjectKind::Directory, + subtype: ObjectSubtype::PLAIN, + len: 0, + revision: 1, + extent: None, + }; + let file = |stable, lba| DiskObject { + stable: sid(stable), + kind: ObjectKind::File, + subtype: ObjectSubtype::PLAIN, + len: 512, + revision: 2, + extent: Some(DurableExtent { + lba, + blocks: 1, + len: 512, + }), + }; + let edge = |parent, child, label| DiskEdge { + parent: sid(parent), + child: sid(child), + name: name(label), + }; + + let (image, len, mut record) = fixture(&[root], &[]); + record.generation += 1; + assert!(matches!( + decode_image(&image[..len], record, 512, 256), + Err(PersistenceError::Corrupt) + )); + + let (image, len, record) = fixture(&[root, root], &[]); + assert!(matches!( + decode_image(&image[..len], record, 512, 256), + Err(PersistenceError::Corrupt) + )); + + let dir = DiskObject { + stable: sid(2), + kind: ObjectKind::Directory, + subtype: ObjectSubtype::PLAIN, + len: 0, + revision: 1, + extent: None, + }; + let (image, len, record) = fixture(&[root, dir], &[edge(1, 2, "down"), edge(2, 1, "up")]); + assert!(matches!( + decode_image(&image[..len], record, 512, 256), + Err(PersistenceError::Corrupt) + )); + + let (image, len, record) = fixture( + &[ + root, + file(2, LEGACY_DATA_START_LBA), + file(3, LEGACY_DATA_START_LBA), + ], + &[edge(1, 2, "a"), edge(1, 3, "b")], + ); + assert!(matches!( + decode_image(&image[..len], record, 512, 256), + Err(PersistenceError::Corrupt) + )); + + let (image, len, record) = fixture(&[root, file(2, 256)], &[edge(1, 2, "past-end")]); + assert!(matches!( + decode_image(&image[..len], record, 512, 256), + Err(PersistenceError::Corrupt) + )); + + let (mut image, len, record) = fixture(&[root], &[]); + image[IMAGE_HEADER + 3] ^= 0x40; + assert!(matches!( + decode_image(&image[..len], record, 512, 256), + Err(PersistenceError::Corrupt) + )); + + let (image, len, record) = fixture( + &[root, file(2, LEGACY_DATA_START_LBA)], + &[edge(2, 1, "file-is-parent")], + ); + assert!(matches!( + decode_image(&image[..len], record, 512, 256), + Err(PersistenceError::Corrupt) + )); + } + + #[test] + fn parser_accepts_directory_diamonds_but_rejects_a_back_edge() { + let sid = |value| StableObjectId::new(value).unwrap(); + let dir = |stable| DiskObject { + stable: sid(stable), + kind: ObjectKind::Directory, + subtype: ObjectSubtype::PLAIN, + len: 0, + revision: 1, + extent: None, + }; + let edge = |parent, child, label| DiskEdge { + parent: sid(parent), + child: sid(child), + name: name(label), + }; + let objects = [dir(1), dir(2), dir(3), dir(4)]; + let diamond = [ + edge(1, 2, "left"), + edge(1, 3, "right"), + edge(2, 4, "shared"), + edge(3, 4, "shared"), + ]; + let (image, len, record) = fixture(&objects, &diamond); + assert!(decode_image(&image[..len], record, 512, 256).is_ok()); + + let cyclic = [ + edge(1, 2, "left"), + edge(1, 3, "right"), + edge(2, 4, "shared"), + edge(3, 4, "shared"), + edge(4, 2, "back"), + ]; + let (image, len, record) = fixture(&objects, &cyclic); + assert!(matches!( + decode_image(&image[..len], record, 512, 256), + Err(PersistenceError::Corrupt) + )); + } + + #[test] + fn provider_loss_never_acknowledges_and_old_root_recovers() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("state"), + ObjectKind::File, + )) + .unwrap(); + let old_revision = crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"old", + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"new", + old_revision, + )) + .unwrap(); + volume + .engine() + .provider() + .control(|device| device.inject(kore_driver::block::BlockError::Lost)) + .unwrap(); + assert!(matches!( + crate::block_on(volume.commit(session)), + Err(PersistenceError::Provider(ProviderError::Lost)) + )); + let mut recovered = crate::block_on(volume.restart(session)).unwrap(); + let root = recovered + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = recovered + .engine_mut() + .open_at(session, root.handle, &name("state"), crate::Rights::ALL) + .unwrap(); + let mut bytes = [0; 3]; + crate::block_on( + recovered + .engine_mut() + .read(session, file.handle, 0, &mut bytes), + ) + .unwrap(); + assert_eq!(bytes, *b"old"); + }); + } + + #[test] + fn replacing_the_fallback_root_reclaims_only_then_unreachable_extents() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("state"), + ObjectKind::File, + )) + .unwrap(); + let mut revision = crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"one", + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + revision = crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"two", + revision, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + assert_eq!(volume.engine().provider().stats().orphan_reclaims, 0); + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + b"tri", + revision, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + assert!(volume.engine().provider().stats().orphan_reclaims > 0); + }); + } + + /// The digest a lend computed is persisted in the root's sidecar and + /// answers the same lend after a cold restart without a single read of + /// the file — the boot-time rehash the sidecar exists to remove. + /// A lent version's grant names its whole blocks in namespace + /// coordinates, never the partial last block, and stays on the same + /// blocks after the file is rewritten: the lease pins the placement, + /// and the rewrite lands in fresh blocks a new lend names instead. + #[test] + fn a_lent_version_grants_its_whole_blocks_and_keeps_them_across_a_rewrite() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("blob"), + ObjectKind::File, + )) + .unwrap(); + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + &[0x11u8; 3000], + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + let owner = crate::extraction::ExtractionOwner::new(7); + let (lease, _) = + crate::block_on(volume.engine_mut().lend_version(owner, session, file.handle)) + .unwrap(); + let pinned = volume.engine().lent_blocks(owner, lease).unwrap().unwrap(); + let block = u64::from(pinned.block_size); + assert_eq!(pinned.blocks, 3000 / block, "whole blocks only, never the tail"); + assert!(pinned.blocks * block <= 3000); + + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + &[0x22u8; 3000], + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + assert_eq!( + volume.engine().lent_blocks(owner, lease).unwrap(), + Some(pinned), + "the lease keeps the version's blocks" + ); + let other = crate::extraction::ExtractionOwner::new(8); + let (fresh, _) = + crate::block_on(volume.engine_mut().lend_version(other, session, file.handle)) + .unwrap(); + let rewritten = volume.engine().lent_blocks(other, fresh).unwrap().unwrap(); + assert_ne!(rewritten.lba, pinned.lba, "the rewrite landed in fresh blocks"); + assert!( + volume.engine().lent_blocks(other, lease).is_err(), + "a lease answers only its owner" + ); + crate::block_on(volume.engine_mut().release_lent_version(owner, lease)).unwrap(); + crate::block_on(volume.engine_mut().release_lent_version(other, fresh)).unwrap(); + assert!(volume.engine().lent_blocks(owner, lease).is_err(), "released"); + + // A lease pins bytes, not independent read authority. Closing the + // view before grant/read was the inference attachment regression. + let (closed, identity) = + crate::block_on(volume.engine().lend_version(owner, session, file.handle)).unwrap(); + volume.engine().close(session, file.handle).unwrap(); + assert!(matches!(volume.engine().lent_blocks(owner, closed), + Err(crate::extraction::ExtractionError::Filesystem(FsError::InvalidHandle)))); + let mut tail = [0u8; 1]; + assert!(crate::block_on(volume.engine().read_lent_version(owner, closed, identity.len - 1, &mut tail)).is_err()); + crate::block_on(volume.engine().release_lent_version(owner, closed)).unwrap(); + + // A refused snapshot must not poison the session: reopen, keep + // its view through the body grant and tail, release, then close. + let reopened = volume.engine().open_at(session, root.handle, &name("blob"), crate::Rights::READ).unwrap(); + let (retry, _) = + crate::block_on(volume.engine().lend_version(owner, session, reopened.handle)).unwrap(); + assert!(volume.engine().lent_blocks(owner, retry).unwrap().is_some()); + assert_eq!(crate::block_on(volume.engine().read_lent_version(owner, retry, identity.len - 1, &mut tail)).unwrap(), 1); + assert_eq!(tail, [0x22]); + crate::block_on(volume.engine().release_lent_version(owner, retry)).unwrap(); + volume.engine().close(session, reopened.handle).unwrap(); + }); + } + + #[test] + fn a_lend_after_restart_answers_its_digest_from_the_sidecar() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("blob"), + ObjectKind::File, + )) + .unwrap(); + let bytes = [0x5au8; 3000]; + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + &bytes, + crate::ANY_REVISION, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + let owner = crate::extraction::ExtractionOwner::new(7); + let (lease, first) = + crate::block_on(volume.engine_mut().lend_version(owner, session, file.handle)) + .unwrap(); + assert_eq!(first.digest, *blake3::hash(&bytes).as_bytes()); + crate::block_on(volume.engine_mut().release_lent_version(owner, lease)).unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + assert!(volume.state.sidecar.is_some(), "the commit wrote the sidecar"); + + let recovered = SessionId::new(2); + let mut volume = crate::block_on(volume.restart(recovered)).unwrap(); + let root = volume + .engine_mut() + .open_root(recovered, crate::Rights::ALL) + .unwrap(); + let file = volume + .engine_mut() + .open_at(recovered, root.handle, &name("blob"), crate::Rights::READ) + .unwrap(); + let reads = volume.engine().provider().stats().reads; + let (lease, again) = + crate::block_on(volume.engine_mut().lend_version(owner, recovered, file.handle)) + .unwrap(); + assert_eq!(again.digest, first.digest); + assert_eq!( + volume.engine().provider().stats().reads, + reads, + "answered from the sidecar, not by rereading the file" + ); + crate::block_on(volume.engine_mut().release_lent_version(owner, lease)).unwrap(); + }); + } + + /// Ruling 3: a version-4 volume mounts as it is, keeps its contents, and + /// its next commit publishes a version-5 root; the upgraded volume + /// mounts again with everything in place. + #[test] + fn a_version_4_volume_is_upgraded_to_5_by_its_next_commit() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("kept"), + ObjectKind::File, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + rewrite_active_as_legacy(&volume, SUBTYPE_FORMAT_VERSION); + + let active_version = |volume: &Volume| { + let provider = volume.engine().provider(); + let block = provider.geometry().block_size as usize; + let mut head = [0u8; 4096]; + let slot = (volume.status().generation & 1) as u64; + crate::block_on(provider.read_metadata(slot, &mut head[..block])).unwrap(); + decode_root( + &head[..block], + provider.geometry().block_size, + provider.geometry().block_count, + ) + .unwrap() + .version + }; + + let mut volume = crate::block_on(volume.restart(session)).unwrap(); + assert_eq!(volume.status().mode, MountMode::ReadWrite); + assert_eq!(active_version(&volume), SUBTYPE_FORMAT_VERSION); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("added"), + ObjectKind::File, + )) + .unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + assert_eq!(active_version(&volume), DIGEST_FORMAT_VERSION); + + let mut volume = crate::block_on(volume.restart(session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + for kept in ["kept", "added"] { + volume + .engine_mut() + .open_at(session, root.handle, &name(kept), crate::Rights::READ) + .unwrap(); + } + }); + } + + /// A sidecar whose bytes are damaged is derived data gone wrong: the + /// volume mounts, the lend computes the digest from the file again (and + /// gets it right), and the next commit replaces the sidecar. + #[test] + fn a_damaged_sidecar_is_ignored_and_replaced() { + on_large_stack(|| { + let session = SessionId::new(0); + let mut volume = crate::block_on(Volume::format(provider(), session)).unwrap(); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = crate::block_on(volume.engine_mut().create( + session, + root.handle, + &name("blob"), + ObjectKind::File, + )) + .unwrap(); + let bytes = [0x33u8; 1500]; + crate::block_on(volume.engine_mut().write_at( + session, + file.handle, + 0, + &bytes, + crate::ANY_REVISION, + )) + .unwrap(); + let owner = crate::extraction::ExtractionOwner::new(9); + let (lease, _) = + crate::block_on(volume.engine_mut().lend_version(owner, session, file.handle)) + .unwrap(); + crate::block_on(volume.engine_mut().release_lent_version(owner, lease)).unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + let (_, sidecar) = volume.state.sidecar.expect("written"); + // One flipped bit inside the first record: the CRC must catch it. + volume + .engine() + .provider() + .control(|device| { + device.corrupt_byte( + sidecar.extent.lba as usize * 512 + crate::digests::HEADER_BYTES + 3, + 0x80, + ) + }) + .unwrap() + .unwrap(); + + let mut volume = crate::block_on(volume.restart(session)).unwrap(); + assert_eq!(volume.status().mode, MountMode::ReadWrite); + assert!(volume.engine().digests().borrow().is_empty()); + assert!(volume.engine().digests().borrow().dirty(), "marked for replacement"); + let root = volume + .engine_mut() + .open_root(session, crate::Rights::ALL) + .unwrap(); + let file = volume + .engine_mut() + .open_at(session, root.handle, &name("blob"), crate::Rights::READ) + .unwrap(); + let reads = volume.engine().provider().stats().reads; + let (lease, input) = + crate::block_on(volume.engine_mut().lend_version(owner, session, file.handle)) + .unwrap(); + assert_eq!(input.digest, *blake3::hash(&bytes).as_bytes()); + assert!(volume.engine().provider().stats().reads > reads, "recomputed"); + crate::block_on(volume.engine_mut().release_lent_version(owner, lease)).unwrap(); + crate::block_on(volume.commit(session)).unwrap(); + let (_, fresh) = volume.state.sidecar.expect("replaced"); + assert_ne!(fresh.extent.lba, sidecar.extent.lba, "a fresh extent, not the damaged one"); + }); + } +} diff --git a/native/services/dagfs/src/provider/block.rs b/native/services/dagfs/src/provider/block.rs new file mode 100644 index 0000000..c7f1413 --- /dev/null +++ b/native/services/dagfs/src/provider/block.rs @@ -0,0 +1,2769 @@ +//! Filesystem placement over one affine direct block channel. +//! +//! This adapter is the sole translation point between filesystem placement +//! semantics and block geometry. The engine above remains generic over +//! [`StorageProvider`]; the channel below remains unaware of files, names, +//! revisions, and authority. + +use core::cell::{Cell, RefCell}; + +use kore_driver::block::{BlockChannel, BlockError, BufferId, Completion, DirectBlockDevice}; + +use crate::{ + ServiceAlloc, + bounds::MAX_BLOCK_PLACEMENTS, + error::ProviderError, + object::ObjectKind, + provider::{ + StorageLimits, StorageProvider, + extent::{ExtentAllocator, ExtentId}, + }, + slot::{AtomicSlotVec, BoundedVec, SlotKey}, +}; + +/// Payload floor encoded by versions 1 and 2, and by pre-7b version 3 roots. +pub const LEGACY_DATA_START_LBA: u64 = 72; +/// Exclusive end of the whole-disk GPT discovery window. +/// +/// This remains fixed when a mounted volume selects a different payload +/// floor; broadening the filesystem reservation must not broaden which +/// foreign-disk blocks GPT parsing is willing to treat as metadata. +pub const GPT_DISCOVERY_END_LBA: u64 = 72; +/// Per-buffer bulk-I/O ceiling. Four 256 KiB buffers across the whole-disk -> +/// designated-region rebuild, plus any needed private PRP lists and controller +/// queues, fit inside the founded 2 MiB bump DMA pool without relying on +/// reclamation; the pool itself is already mapped by a large IOMMU leaf. +const IO_BUFFER_BYTES: usize = 256 * 1024; +/// Largest byte count one payload read or write may transfer on the block +/// provider — 128 logical blocks at 512 B, one command. +/// +/// The engine's shared [`MAX_TRANSFER`] (4 KiB) bounds the RAM provider's +/// chunk model and the engine's own stack buffers; it is not a property of +/// a device that accepts 256 KiB per command. Measured on metal +/// (`status fs`, 2026-09-09): 17400 commands moved 9 MiB, about 1 KiB +/// each, at 49 µs per command — the per-command cost, not the bytes, was +/// the throughput. 64 KiB keeps a whole read inside the DMA buffer with +/// room for the block-alignment head, needs no stack scratch (bytes move +/// between the registered buffer and the caller directly), and sits at the +/// point past which a single NVMe command gains little. +const BLOCK_MAX_TRANSFER: usize = 64 * 1024; +/// How far a payload read runs ahead of what was asked, when the placement +/// has that much more to give: the next sequential read is then served from +/// the buffer the provider already owns, with no command at all. Half the +/// DMA buffer, so a 64 KiB request never has to evict what it just fetched; +/// 128 KiB is also where an NVMe read stops getting cheaper per byte. +const READ_AHEAD_BYTES: usize = 128 * 1024; +/// Stack scratch for the one copy that must go through the CPU: mirroring +/// a partial edge block between the two DMA buffers on a write. One block +/// at most — the provider refuses devices with larger ones at construction. +const COPY_CHUNK_BYTES: usize = 4096; +const BULK_DEPTH: usize = 2; + +/// What the `io` buffer currently holds from the device — the read-ahead +/// window. Cleared by every path that writes, reuses `io`, or changes the +/// bytes it mirrors, so a stale hit is structurally impossible: the only +/// reader is `read_bytes`, and every other operation forgets it first. +#[derive(Clone, Copy, Debug)] +struct HeldRead { + lba: u64, + blocks: u32, +} + +/// A generational placement key. It is naming data, never authority. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BlockKey(SlotKey); + +/// One tracked payload placement. `lba`/`blocks` are hot-path caches of the +/// geometry the extent index also holds for `extent` — the IO paths read +/// them without touching the index, so every site that installs or replaces +/// `extent` must refresh them from the same allocation, and `extent` is +/// `None` exactly when `blocks == 0` (nothing allocated). `retire` and the +/// checkpoint drains police that pairing with `Integrity` refusals. +#[derive(Clone, Copy, Debug)] +struct Placement { + lba: u64, + blocks: u32, + len: u64, + committed: bool, + extent: Option, + /// High-water mark of meaningful bytes, in blocks: every block at or + /// past it holds the zeros `allocate`/`resize` put there and nothing + /// has overwritten. A write that begins at or after it has no old + /// bytes to fetch or keep — the rollback image is zeros by + /// construction. Recovered and copied placements start at `blocks`: + /// nothing is known about their bytes except that they are all + /// meaningful. A `u32` in blocks, not a `u64` in bytes: it rides in + /// the record's padding, see [`PLACEMENT_BYTES`]. + written_blocks: u32, +} + +/// The size of one [`Placement`], pinned. +/// +/// Three placement-bound tables of these (`placements`, `retired`, +/// `fallback`) live inside the provider state, which lives inside the +/// service's `serve` future, which is polled on the founder's 4 MiB main +/// stack. Growing this record by one word grew that future by tens of +/// KiB and overflowed the stack by 744 bytes on metal (2026-09-09, two +/// boots of PR #958): the record's size is part of the stack budget, so a +/// change to it must be deliberate. +/// +/// All three tables are arena-backed now, so this record's size is a heap cost +/// rather than a stack one. The figure stays pinned because it is the unit +/// those tables multiply, so it remains the thing to re-derive when +/// `MAX_BLOCK_PLACEMENTS` rises. +/// +/// `retired` and `fallback` needed [`BoundedVec`] rather than a bare +/// `alloc::vec::Vec`, because their **bound is load-bearing**: `is_full` gates +/// an admission decision and `push` carries the typed `PlacementsFull` +/// refusal. A growing vector would turn both of those into silent successes. +const PLACEMENT_BYTES: usize = 40; +const _: () = assert!(core::mem::size_of::() == PLACEMENT_BYTES); + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct BlockProviderStats { + pub reads: u64, + pub writes: u64, + pub flushes: u64, + pub discards: u64, + pub copied_bytes: u64, + pub rollbacks: u64, + pub late_completions: u64, + pub orphan_reclaims: u64, + pub reclaim_failures: u64, + pub bulk_batches: u64, + /// Payload reads answered from the read-ahead window with no command. + pub readahead_hits: u64, + /// Device reads issued on behalf of the read-ahead window. + pub readahead_reads: u64, + /// Payload writes that began and ended on block boundaries, so no old + /// byte had to be fetched into the write image first. + pub aligned_writes: u64, + /// Single-block reads of a partial head or tail block, fetched so an + /// unaligned write into a fresh extent overwrites only its own bytes. + pub edge_reads: u64, + /// Writes into an uncommitted extent past its high-water mark: the old + /// bytes were the allocation's zeros, so no rollback image was read. + pub pristine_writes: u64, + /// Whole-object copies moved DMA buffer to DMA buffer for a version + /// lend, never through the CPU. + pub snapshot_copies: u64, + /// Completion waits that ran one budget out without the awaited + /// completion arriving. Every round of a slow command counts, so this + /// reads as "how often a command ran past one budget", not "how many + /// commands failed". + pub expired_waits: u64, + /// Of those, the rounds where the backend reported itself HEALTHY: the + /// device was slow, not dead. Up to [`STALL_BUDGETS`] such rounds are + /// waited out and the command still completes; past the ceiling the + /// command is refused `Timeout` and the channel is kept. `expired - + /// slow` is therefore exactly the count of condemned waits. + pub slow_waits: u64, +} + +/// Durable coordinates of one file payload. This is allocation metadata, +/// not authority and not the process-local [`BlockKey`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DurableExtent { + pub lba: u64, + pub blocks: u32, + pub len: u64, +} + +#[derive(Clone, Copy)] +enum RecoveryOwner { + Live(usize), + Fallback, +} + +#[derive(Clone, Copy)] +struct RecoveryRecord { + extent: DurableExtent, + owner: RecoveryOwner, +} + +/// Empty completion budgets a HEALTHY controller may run through before a +/// command is refused as timed out. +/// +/// One budget is the backend's own bound — the 1 s route park on NVMe +/// (`durable.rs` `OP_BUDGET_NS`), a spin batch on the polled posture — and +/// it is the interrupt-loss recovery cadence, not a device timeout: a +/// healthy device that has not answered inside one budget is slow, and a +/// DRAM-less consumer drive under garbage collection is slow for whole +/// seconds. Thirty budgets on NVMe is 30 s, the I/O timeout Linux ships, +/// so a stall Linux would ride out is ridden out here too instead of +/// failing the filesystem operation on top of it. A controller that +/// reports a fault, or one with no liveness signal at all, gets no +/// extension: for those the first empty budget is still the verdict. +const STALL_BUDGETS: u32 = 30; + +struct State { + channel: BlockChannel, + io: BufferId, + rollback: BufferId, + buffer_bytes: usize, + /// The read-ahead window `io` holds, if any. + held: Option, + placements: AtomicSlotVec, + retired: BoundedVec, + fallback: BoundedVec, + extents: ExtentAllocator, + /// First block the payload allocator may name. Metadata I/O is confined + /// below this boundary; payload I/O is confined at or above it. + data_start: u64, + /// Exclusive end of the volume-owned payload arena. + arena_end: u64, + stats: BlockProviderStats, +} + +/// One filesystem-worker provider over one exclusively-owned direct channel. +/// `RefCell` is task-local and deliberately `!Sync`; parallel workers receive +/// distinct provider/channel values rather than locking one qpair. +pub struct BlockProvider { + state: RefCell>, + limits: Cell, +} + +impl BlockProvider { + pub fn new(channel: BlockChannel) -> Result { + Self::new_with_data_start(channel, LEGACY_DATA_START_LBA) + } + + /// Construct behind a conservative pre-root metadata boundary. + /// + /// Durable mounting passes the largest supported reservation here, reads + /// only roots/GPT below it, then calls [`configure_arena`](Self::configure_arena) + /// with the authenticated per-volume bounds before constructing an engine. + pub(crate) fn new_with_data_start( + mut channel: BlockChannel, + data_start: u64, + ) -> Result { + let geometry = channel.geometry(); + let block = geometry.block_size as usize; + if block > 4096 || data_start < LEGACY_DATA_START_LBA || data_start >= geometry.block_count + { + return Err(ProviderError::Integrity); + } + let command_bytes = usize::try_from( + u64::from(geometry.max_transfer_blocks) * u64::from(geometry.block_size), + ) + .unwrap_or(usize::MAX); + let io_bytes = IO_BUFFER_BYTES + .min(command_bytes) + .min(channel.max_buffer_bytes()); + if io_bytes < block || B < 2 { + return Err(ProviderError::Backpressure); + } + let io = channel.register_buffer(io_bytes).map_err(map_block)?; + let rollback = channel.register_buffer(io_bytes).map_err(map_block)?; + let total = (geometry.block_count - data_start) + .checked_mul(geometry.block_size as u64) + .ok_or(ProviderError::Integrity)?; + let max_object = total.min(u32::MAX as u64 * geometry.block_size as u64); + // A byte transfer beginning at the last byte of a block can cover + // one more block than its payload. Keep that head/tail allowance in + // the declared limit while still admitting a one-byte transfer on a + // device whose maximum is exactly one block. + let payload_limit = io_bytes.saturating_sub(block.saturating_sub(1)); + if payload_limit == 0 { + return Err(ProviderError::Integrity); + } + let limits = StorageLimits { + total_bytes: total, + max_object_bytes: max_object, + max_transfer: BLOCK_MAX_TRANSFER.min(payload_limit), + granularity: block, + }; + Ok(Self { + state: RefCell::new(State { + channel, + io, + rollback, + buffer_bytes: io_bytes, + held: None, + placements: AtomicSlotVec::try_new_in(MAX_BLOCK_PLACEMENTS, crate::service_alloc()) + .expect("fs block-placement arena (bootstrap-scale) — service heap undersized"), + retired: BoundedVec::try_new_in(MAX_BLOCK_PLACEMENTS, crate::service_alloc()) + .expect("fs retired-extent stack (bootstrap-scale) — service heap undersized"), + fallback: BoundedVec::try_new_in(MAX_BLOCK_PLACEMENTS, crate::service_alloc()) + .expect("fs fallback-extent stack (bootstrap-scale) — service heap undersized"), + extents: ExtentAllocator::new(data_start, geometry.block_count)?, + data_start, + arena_end: geometry.block_count, + stats: BlockProviderStats::default(), + }), + limits: Cell::new(limits), + }) + } + + /// Configure the exact payload arena declared by the selected durable + /// root before any extent is adopted or allocated. + /// + /// Recovery begins behind the largest supported reservation so it can + /// safely inspect roots from every supported profile. Once a root has + /// authenticated its `data_start` and `arena_blocks`, this call makes that + /// on-disk fact the provider's one addressability boundary. Refusing a + /// populated provider prevents a live allocation from changing meaning + /// underneath its key. + pub fn configure_arena(&self, data_start: u64, arena_blocks: u64) -> Result<(), ProviderError> { + let mut state = self.state.borrow_mut(); + if !state.placements.is_empty() || !state.retired.is_empty() || !state.fallback.is_empty() { + return Err(ProviderError::Integrity); + } + let geometry = state.channel.geometry(); + let arena_end = data_start + .checked_add(arena_blocks) + .ok_or(ProviderError::Integrity)?; + if data_start < LEGACY_DATA_START_LBA + || arena_blocks == 0 + || arena_end > geometry.block_count + { + return Err(ProviderError::Integrity); + } + let total = arena_blocks + .checked_mul(u64::from(geometry.block_size)) + .ok_or(ProviderError::Integrity)?; + let max_object = total.min(u64::from(u32::MAX) * u64::from(geometry.block_size)); + state.extents.reset(data_start, arena_end)?; + state.data_start = data_start; + state.arena_end = arena_end; + self.limits.set(StorageLimits { + total_bytes: total, + max_object_bytes: max_object, + max_transfer: self.limits.get().max_transfer, + granularity: geometry.block_size as usize, + }); + Ok(()) + } + + /// The exact payload arena currently enforced by the provider. + pub fn arena(&self) -> (u64, u64) { + let state = self.state.borrow(); + (state.data_start, state.arena_end - state.data_start) + } + + pub fn stats(&self) -> BlockProviderStats { + let state = self.state.borrow(); + BlockProviderStats { + late_completions: state.channel.late_completions(), + ..state.stats + } + } + + pub fn placements(&self) -> usize { + self.state.borrow().placements.len() + } + + pub fn geometry(&self) -> kore_driver::block::BlockGeometry { + self.state.borrow().channel.geometry() + } + + /// Blocks the allocator cannot currently reuse, including the fixed + /// metadata prefix and every live, retired, or fallback-root extent. + /// + /// Retired and fallback extents are intentionally counted even though + /// they are absent from `placements`: a currently or previously durable + /// root can still name them, so presenting them as free capacity would be + /// a false filesystem-pressure metric. + pub fn occupied_blocks(&self) -> Option { + let state = self.state.borrow(); + state.arena_end.checked_sub(state.extents.free_blocks()) + } + + pub fn channel_stats(&self) -> kore_driver::block::ChannelStats { + self.state.borrow().channel.stats() + } + + /// Current lifecycle of the mounted block channel. + /// + /// Unlike operation counters this is a one-way posture edge: once `Lost`, + /// status must continue to report the condemned channel even when no + /// later request is attempted. + pub fn channel_state(&self) -> kore_driver::block::ChannelState { + self.state.borrow().channel.state() + } + + pub fn mark_lost(&self) { + self.state.borrow_mut().channel.mark_lost(); + } + + pub fn extent(&self, key: BlockKey) -> Result { + let state = self.state.borrow(); + let placement = *state + .placements + .get(key.0) + .ok_or(ProviderError::UnknownKey)?; + Ok(DurableExtent { + lba: placement.lba, + blocks: placement.blocks, + len: placement.len, + }) + } + + /// Rebuild the boot-local extent index from both recovery-selectable + /// roots. `live` keys are returned in input order; extents unique to + /// `fallback` remain protected until its root slot is replaced. + /// + /// The records are sorted once and then admitted in ascending LBA order, + /// so mount costs O(N log N) rather than performing one growing occupied + /// scan per record. The rebuild is prepared off to the side and only + /// installed after all bounds, overlap, and table-capacity checks pass. + pub(crate) fn recover_extents( + &self, + live: &[DurableExtent], + fallback: &[DurableExtent], + ) -> Result, ProviderError> { + if live.len() > MAX_BLOCK_PLACEMENTS || fallback.len() > MAX_BLOCK_PLACEMENTS { + return Err(ProviderError::PlacementsFull); + } + + let mut state = self.state.borrow_mut(); + if !state.placements.is_empty() || !state.retired.is_empty() || !state.fallback.is_empty() { + return Err(ProviderError::Integrity); + } + for extent in live.iter().chain(fallback) { + state.validate_extent(*extent)?; + } + + // Design-bounded like every sibling buffer here: the entry guard + // caps each input at MAX_BLOCK_PLACEMENTS, so a refused push below + // is index corruption, never volume pressure. + // + // Arena-backed rather than inline for the reason recorded on + // `PLACEMENT_BYTES`: all four of these are sized by the placement + // bound, and mount runs inside the service's `serve` future on the + // founder's shared stack. One allocation each per mount, which is not + // a path where that matters, and the rebuild still stays prepared off + // to the side and is installed only after every check passes. + let alloc = crate::service_alloc(); + let mut records = + BoundedVec::::try_new_in(MAX_BLOCK_PLACEMENTS * 2, alloc) + .map_err(|_| ProviderError::Integrity)?; + for (index, extent) in live.iter().copied().enumerate() { + if extent.blocks != 0 { + records + .push(RecoveryRecord { + extent, + owner: RecoveryOwner::Live(index), + }) + .map_err(|_| ProviderError::Integrity)?; + } + } + for extent in fallback.iter().copied() { + if extent.blocks != 0 { + records + .push(RecoveryRecord { + extent, + owner: RecoveryOwner::Fallback, + }) + .map_err(|_| ProviderError::Integrity)?; + } + } + // Intra-group order is immaterial — the group walk below gathers + // live/fallback from the whole (lba, blocks)-equal group — so the + // sort key is exactly the group identity. + records.sort_unstable_by(|left, right| { + left.extent + .lba + .cmp(&right.extent.lba) + .then(left.extent.blocks.cmp(&right.extent.blocks)) + }); + + let mut allocations = + BoundedVec::, ServiceAlloc>::try_new_in(MAX_BLOCK_PLACEMENTS, alloc) + .map_err(|_| ProviderError::Integrity)?; + for _ in live { + allocations + .push(None) + .map_err(|_| ProviderError::Integrity)?; + } + let mut protected = + BoundedVec::::try_new_in(MAX_BLOCK_PLACEMENTS, alloc) + .map_err(|_| ProviderError::Integrity)?; + let mut rebuilt = ExtentAllocator::new(state.data_start, state.arena_end)?; + let mut previous_end = state.data_start; + for group in records.chunk_by(|left, right| { + left.extent.lba == right.extent.lba && left.extent.blocks == right.extent.blocks + }) { + let extent = group[0].extent; + let end = extent + .lba + .checked_add(u64::from(extent.blocks)) + .ok_or(ProviderError::Integrity)?; + if extent.lba < previous_end { + return Err(ProviderError::Integrity); + } + + let mut live_index = None; + let mut fallback_record = None; + for record in group { + match record.owner { + RecoveryOwner::Live(index) => { + if live_index.replace(index).is_some() { + return Err(ProviderError::Integrity); + } + } + RecoveryOwner::Fallback => { + if fallback_record.replace(record.extent).is_some() { + return Err(ProviderError::Integrity); + } + } + } + } + + let allocation = rebuilt.reserve_sorted(extent.lba, extent.blocks)?; + if let Some(index) = live_index { + allocations[index] = Some(allocation); + } else if let Some(extent) = fallback_record { + protected + .push(Placement { + lba: extent.lba, + blocks: extent.blocks, + len: extent.len, + committed: true, + extent: Some(allocation), + written_blocks: extent.blocks, + }) + .map_err(|_| ProviderError::PlacementsFull)?; + } + previous_end = end; + } + + let mut keys = + BoundedVec::::try_new_in(MAX_BLOCK_PLACEMENTS, alloc) + .map_err(|_| ProviderError::Integrity)?; + for (extent, allocation) in live.iter().copied().zip(allocations) { + // The entry guard bounds `live` by the verified-empty table's own + // capacity, so neither insert can refuse; a refusal here is + // index corruption, not a partial mount to roll back. + let key = state + .placements + .insert(Placement { + lba: extent.lba, + blocks: extent.blocks, + len: extent.len, + committed: true, + extent: allocation, + written_blocks: extent.blocks, + }) + .map(BlockKey) + .map_err(|_| ProviderError::Integrity)?; + keys.push(key).map_err(|_| ProviderError::Integrity)?; + } + state.extents = rebuilt; + state.fallback = protected; + Ok(keys) + } + + /// Read an aligned span from the metadata reservation. + pub async fn read_metadata(&self, lba: u64, dst: &mut [u8]) -> Result<(), ProviderError> { + self.state.borrow_mut().metadata_read(lba, dst).await + } + + /// Write an aligned span in the inactive metadata arena. + pub async fn write_metadata(&self, lba: u64, src: &[u8]) -> Result<(), ProviderError> { + self.state.borrow_mut().metadata_write(lba, src).await + } + + pub async fn flush_device(&self) -> Result<(), ProviderError> { + self.state.borrow_mut().flush().await + } + + /// Retire extents made obsolete by the root that has just become durable. + /// Reclamation is best-effort and never weakens the durability ack: the + /// published root no longer references these blocks, so a discard failure + /// is a leak to report, not a failed commit. + pub async fn checkpoint(&self) { + let mut state = self.state.borrow_mut(); + while let Some(placement) = state.fallback.pop() { + if placement.blocks > 0 && state.channel.geometry().discard { + match state.discard(placement.lba, placement.blocks).await { + Ok(()) => { + state.stats.orphan_reclaims = state.stats.orphan_reclaims.saturating_add(1) + } + Err(_) => { + state.stats.reclaim_failures = + state.stats.reclaim_failures.saturating_add(1) + } + } + } + if let Some(extent) = placement.extent + && state.extents.release(extent).is_err() + { + state.stats.reclaim_failures = state.stats.reclaim_failures.saturating_add(1); + } + } + while let Some(placement) = state.retired.pop() { + // Capacity is identical and fallback was drained above. + let _ = state.fallback.push(placement); + } + for (_, mut placement) in state.placements.iter_mut() { + placement.committed = true; + } + } + + /// Freeze every live placement before a root record may name it. + /// + /// Root publication is a cut: after the root write is submitted, a + /// failed completion or flush may still leave either the old or the new + /// root durable. Mark the new root's extents conservatively before that + /// submission so later mutations use COW even when the publisher cannot + /// tell which root won. [`checkpoint`](Self::checkpoint) performs the + /// post-flush reclamation only once the new root is known durable. + pub(crate) fn prepare_publication(&self) { + let state = self.state.borrow_mut(); + for (_, mut placement) in state.placements.iter_mut() { + placement.committed = true; + } + } + + /// Simulate a service restart over the same already-provisioned medium. + /// Production obtains a fresh channel from the provisioner; this helper + /// retires only boot-local buffers and placement keys. + pub fn restart(self) -> Result { + Self::new(self.into_channel()?) + } + + /// Simulate a cold durable mount, restoring the conservative boundary + /// that exists before either root authenticates its exact arena. + pub(crate) fn restart_with_data_start(self, data_start: u64) -> Result { + Self::new_with_data_start(self.into_channel()?, data_start) + } + + /// Recover the affine channel for re-provisioning — the probing path: + /// a provider that classified a whole device as foreign hands its + /// channel back so the composition can narrow onto a discovered + /// designated region and rebuild. Boot-local buffers and placement + /// keys retire; the channel and the backend beneath it ride out intact. + pub fn into_channel(self) -> Result, ProviderError> { + let mut state = self.state.into_inner(); + state + .channel + .unregister_buffer(state.io) + .map_err(map_block)?; + state + .channel + .unregister_buffer(state.rollback) + .map_err(map_block)?; + Ok(state.channel) + } + + /// Close this provider before abandoning its affine direct channel. + /// + /// Registered DMA buffers are children of the device/pool authority held + /// by the channel backend, so they must leave before the backend is + /// quiesced and dropped. This explicit consuming path is used when a + /// provisioned namespace proves to contain foreign data: the filesystem + /// must neither keep DMA enabled against an unused disk nor let ordinary + /// struct field drop order retire the device ahead of its buffers. + pub async fn retire(self) -> Result<(), ProviderError> { + let mut state = self.state.into_inner(); + // Evaluate every containment step even if an earlier one refuses; the + // returned error reports the first broken invariant, while later + // steps still make the best bounded attempt to close authority. + let io = state.channel.unregister_buffer(state.io); + let rollback = state.channel.unregister_buffer(state.rollback); + let quiesce = state.channel.quiesce_async().await; + io.and(rollback).and(quiesce).map_err(map_block) + } + + /// Infrequent test/lifecycle access; refused while the qpair is busy. + pub fn control(&self, f: impl FnOnce(&mut D) -> T) -> Result { + self.state + .borrow_mut() + .channel + .control(f) + .map_err(map_block) + } +} + +impl State { + fn block_size(&self) -> usize { + self.channel.geometry().block_size as usize + } + + /// Maximum blocks that fit both the device command and our registered + /// I/O window. Device MDTS can be much larger than the filesystem's + /// bounded scratch window; every buffered loop must honor the smaller. + fn buffered_transfer_blocks(&self) -> u32 { + self.channel + .geometry() + .max_transfer_blocks + .min((self.buffer_bytes / self.block_size()) as u32) + } + + /// Clamp one streaming command to the registered-buffer/device ceiling + /// and the namespace's optional optimal-I/O boundary. The boundary is a + /// performance hint rather than an addressability rule, but honoring it + /// here costs one remainder on a slow setup loop and avoids controller + /// read-modify-split work on the hot transfer path. + fn streaming_blocks(&self, lba: u64, remaining: u32) -> u32 { + let mut count = remaining.min(self.buffered_transfer_blocks()); + let boundary = self.channel.geometry().io_hints.optimal_boundary_blocks; + if boundary != 0 { + let within = (lba % u64::from(boundary)) as u32; + count = count.min(boundary - within); + } + count + } + + /// Shape a write so its aligned interior uses the namespace's preferred + /// granularity and multi-block optimal command size. A one-block optimum + /// does not cap bulk writes: it is a preference, not a transfer limit, + /// and splitting at every block amplifies command/completion overhead. + /// Unaligned head/tail fragments remain legal; the registered-buffer, + /// device-transfer, and optimal-boundary limits still apply. + fn streaming_write_blocks(&self, lba: u64, remaining: u32) -> u32 { + let mut count = self.streaming_blocks(lba, remaining); + let hints = self.channel.geometry().io_hints; + if hints.write_alignment_blocks > 1 { + let within = (lba % u64::from(hints.write_alignment_blocks)) as u32; + if within != 0 { + return count.min(hints.write_alignment_blocks - within); + } + } + if hints.optimal_write_blocks > 1 { + count = count.min(hints.optimal_write_blocks); + } + if hints.preferred_write_blocks > 1 && count > hints.preferred_write_blocks { + let rounded = count - count % hints.preferred_write_blocks; + if rounded != 0 { + count = rounded; + } + } + count + } + + fn blocks_for(&self, len: u64) -> Result { + let block = self.block_size() as u64; + let blocks = len.checked_add(block - 1).ok_or(ProviderError::Range)? / block; + u32::try_from(blocks).map_err(|_| ProviderError::TooLarge) + } + + fn validate_extent(&self, extent: DurableExtent) -> Result<(), ProviderError> { + if extent.blocks != self.blocks_for(extent.len)? { + return Err(ProviderError::Integrity); + } + if extent.blocks == 0 { + return if extent.lba == 0 { + Ok(()) + } else { + Err(ProviderError::Integrity) + }; + } + let end = extent + .lba + .checked_add(extent.blocks as u64) + .ok_or(ProviderError::Integrity)?; + if extent.lba < self.data_start || end > self.arena_end { + return Err(ProviderError::Integrity); + } + Ok(()) + } + + fn metadata_shape(&self, lba: u64, len: usize) -> Result<(usize, u32), ProviderError> { + let block = self.block_size(); + if len == 0 || !len.is_multiple_of(block) { + return Err(ProviderError::Range); + } + let blocks = u32::try_from(len / block).map_err(|_| ProviderError::TooLarge)?; + let end = lba.checked_add(blocks as u64).ok_or(ProviderError::Range)?; + if end > self.data_start { + return Err(ProviderError::Range); + } + Ok((block, blocks)) + } + + /// Drop the read-ahead window. Every path that writes, reuses `io`, or + /// changes bytes the window may mirror calls this first. + fn forget_held(&mut self) { + self.held = None; + } + + async fn metadata_read(&mut self, lba: u64, bytes: &mut [u8]) -> Result<(), ProviderError> { + self.forget_held(); + let (block, blocks) = self.metadata_shape(lba, bytes.len())?; + let mut done = 0u32; + while done < blocks { + let count = self.streaming_blocks(lba + u64::from(done), blocks - done); + let at = done as usize * block; + let len = count as usize * block; + self.read_blocks(lba + done as u64, count, self.io).await?; + self.channel + .read_buffer(self.io, 0, &mut bytes[at..at + len]) + .map_err(map_block)?; + done += count; + } + Ok(()) + } + + async fn metadata_write(&mut self, lba: u64, bytes: &[u8]) -> Result<(), ProviderError> { + self.forget_held(); + let (block, blocks) = self.metadata_shape(lba, bytes.len())?; + let mut done = 0u32; + while done < blocks { + let count = self.streaming_write_blocks(lba + u64::from(done), blocks - done); + let at = done as usize * block; + let len = count as usize * block; + self.channel + .write_buffer(self.io, 0, &bytes[at..at + len]) + .map_err(map_block)?; + self.write_blocks(lba + done as u64, count, self.io).await?; + done += count; + } + Ok(()) + } + + /// Decide what an exhausted wait means, and retire the channel only on + /// evidence. + /// + /// A wait that expired is **not** evidence a device is dead. A completion + /// can be late because the controller failed, but equally because its + /// queue is deep, because it is garbage-collecting or thermally + /// throttled, or because the host descheduled this task. Only the first + /// justifies `mark_lost`, which is a one-way door: the channel is retired + /// and the mounted volume goes with it. + /// + /// So ask the backend out of band ([`DirectBlockDevice::liveness`], one + /// `CSTS` read for NVMe). A controller that reports itself healthy makes + /// this a slow operation, and the request fails without condemning the + /// channel — the next one may well succeed. `Failed` is the controller's + /// own verdict and retires it. `Unknown` keeps the old conservative + /// posture, because a backend with no out-of-band signal has told us + /// nothing and a timeout is all the evidence that exists. + fn expired(&mut self) -> ProviderError { + let liveness = self.channel.liveness(); + self.stats.expired_waits = self.stats.expired_waits.saturating_add(1); + if liveness == kore_driver::block::Liveness::Healthy { + self.stats.slow_waits = self.stats.slow_waits.saturating_add(1); + return ProviderError::Timeout; + } + self.channel.mark_lost(); + ProviderError::Lost + } + + /// One empty budget on a controller that reports itself healthy is a + /// slow command, not a failed one: grant it another budget, up to + /// [`STALL_BUDGETS`]. Each grant is booked as an expired-but-healthy + /// wait so `status fs` shows how often a command ran past one budget. + /// `false` means the caller's next step is [`expired`](Self::expired): + /// the ceiling is spent, or the controller did not vouch for itself. + fn extend_wait(&mut self, extensions: &mut u32) -> bool { + if *extensions >= STALL_BUDGETS + || self.channel.liveness() != kore_driver::block::Liveness::Healthy + { + return false; + } + *extensions += 1; + self.stats.expired_waits = self.stats.expired_waits.saturating_add(1); + self.stats.slow_waits = self.stats.slow_waits.saturating_add(1); + true + } + + async fn wait(&mut self, request: kore_driver::block::RequestId) -> Result<(), ProviderError> { + // Each empty wait spends the backend's whole bounded budget — the + // spin default, or a real park on the task's held MSI-X route + // ([`DirectBlockDevice::wait`]). This provider submits + // synchronously, so a mismatched id is never the answer; the + // anomalous cross-request drain is bounded by the queue depth + // rather than a spin count. An empty budget is not yet the end: a + // controller that vouches for itself is granted further budgets + // (`extend_wait`), so a slow device is waited out and only a spent + // ceiling or a silent controller reaches `expired`. + let mut extensions = 0u32; + loop { + for _ in 0..=O { + // An executor-aware transport makes its correlated progress + // here; the direct device's ready default paces in `wait` below. + self.channel.progress().await.map_err(map_block)?; + match self.channel.wait() { + Some(Completion { + request: completed, + result, + }) if completed == request => return result.map_err(map_block), + Some(_) => { + self.stats.late_completions = self.stats.late_completions.saturating_add(1); + } + None => break, + } + } + if !self.extend_wait(&mut extensions) { + break; + } + } + Err(self.expired()) + } + + /// Drain an exact bounded request set without assuming completion order. + /// Every accepted request is consumed even after the first device error, + /// so its DMA buffer is idle before the error escapes. + async fn wait_many( + &mut self, + requests: &mut [Option; N], + ) -> Result<(), ProviderError> { + let mut first_error = None; + let mut extensions = 0u32; + 'budget: loop { + for _ in 0..=O { + self.channel.progress().await.map_err(map_block)?; + for completion in self.channel.poll_batch::().into_iter().flatten() { + self.settle_many(requests, completion, &mut first_error); + } + if requests.iter().all(Option::is_none) { + return first_error.map_or(Ok(()), Err); + } + if let Some(completion) = self.channel.wait() { + self.settle_many(requests, completion, &mut first_error); + if requests.iter().all(Option::is_none) { + return first_error.map_or(Ok(()), Err); + } + } else { + break; + } + } + if !self.extend_wait(&mut extensions) { + break 'budget; + } + } + Err(self.expired()) + } + + fn settle_many( + &mut self, + requests: &mut [Option; N], + completion: Completion, + first_error: &mut Option, + ) { + let Some(index) = requests + .iter() + .position(|request| *request == Some(completion.request)) + else { + self.stats.late_completions = self.stats.late_completions.saturating_add(1); + return; + }; + requests[index] = None; + if let Err(error) = completion.result + && first_error.is_none() + { + *first_error = Some(map_block(error)); + } + } + + async fn read_blocks( + &mut self, + lba: u64, + blocks: u32, + buffer: BufferId, + ) -> Result<(), ProviderError> { + self.read_blocks_into(lba, blocks, buffer, 0).await + } + + async fn read_blocks_into( + &mut self, + lba: u64, + blocks: u32, + buffer: BufferId, + buffer_offset: usize, + ) -> Result<(), ProviderError> { + let request = self + .channel + .submit_read(lba, blocks, buffer, buffer_offset) + .map_err(map_block)?; + self.wait(request).await?; + self.stats.reads = self.stats.reads.saturating_add(1); + Ok(()) + } + + /// Fetch into `io`, where they will be written back from, the old bytes + /// of the partial edge blocks of a `covered`-block write that begins + /// `within` bytes into its first block and carries `len` bytes. The + /// interior is overwritten whole and is never read; a write that begins + /// and ends on block boundaries fetches nothing. Two distinct edges go + /// out in one batch — `io` carries one command at a time, so the tail + /// rides `rollback` and is mirrored across (one block) once both land. + async fn read_edges( + &mut self, + lba: u64, + within: usize, + len: usize, + covered: usize, + ) -> Result<(), ProviderError> { + let block = self.block_size(); + let head = within != 0; + let tail = (within + len) % block != 0; + let last = covered - 1; + if !head && !tail { + self.stats.aligned_writes = self.stats.aligned_writes.saturating_add(1); + return Ok(()); + } + if last == 0 || head != tail { + // One edge — or one block that is both — straight into place. + let at = if head || last == 0 { 0 } else { last }; + self.read_blocks_into(lba + at as u64, 1, self.io, at * block) + .await?; + self.stats.edge_reads = self.stats.edge_reads.saturating_add(1); + return Ok(()); + } + // Both edges, distinct blocks. A queue with one usable slot refuses + // the second stage with Backpressure; that edge then follows as its + // own command once `io` is free, straight into place. + let staged = [(lba, self.io), (lba + last as u64, self.rollback)]; + let mut requests = [None; 2]; + let mut stage_error = None; + { + let mut batch = self.channel.batch(); + for (index, (at, buffer)) in staged.into_iter().enumerate() { + match batch.stage_read(at, 1, buffer, 0) { + Ok(request) => requests[index] = Some(request), + Err(error) => { + stage_error = Some(error); + break; + } + } + } + batch.publish(); + } + let accepted = requests.iter().filter(|r| r.is_some()).count(); + let tail_rode_rollback = requests[1].is_some(); + if accepted != 0 { + self.wait_many(&mut requests).await?; + } + self.stats.reads = self.stats.reads.saturating_add(accepted as u64); + self.stats.edge_reads = self.stats.edge_reads.saturating_add(accepted as u64); + match stage_error { + None => {} + Some(BlockError::Backpressure) if accepted == 1 => {} + Some(error) => return Err(map_block(error)), + } + if tail_rode_rollback { + let mut chunk = [0u8; COPY_CHUNK_BYTES]; + self.channel + .read_buffer(self.rollback, 0, &mut chunk[..block]) + .map_err(map_block)?; + self.channel + .write_buffer(self.io, last * block, &chunk[..block]) + .map_err(map_block)?; + } else { + self.read_blocks_into(lba + last as u64, 1, self.io, last * block) + .await?; + self.stats.edge_reads = self.stats.edge_reads.saturating_add(1); + } + Ok(()) + } + + async fn write_blocks( + &mut self, + lba: u64, + blocks: u32, + buffer: BufferId, + ) -> Result<(), ProviderError> { + let request = self + .channel + .submit_write(lba, blocks, buffer, 0) + .map_err(map_block)?; + self.wait(request).await?; + self.stats.writes = self.stats.writes.saturating_add(1); + Ok(()) + } + + async fn flush(&mut self) -> Result<(), ProviderError> { + let request = self.channel.submit_flush().map_err(map_block)?; + self.wait(request).await?; + self.stats.flushes = self.stats.flushes.saturating_add(1); + Ok(()) + } + + async fn discard(&mut self, lba: u64, blocks: u32) -> Result<(), ProviderError> { + self.forget_held(); + let mut done = 0u32; + while done < blocks { + let count = self.streaming_blocks(lba + u64::from(done), blocks - done); + let request = self + .channel + .submit_discard(lba + done as u64, count) + .map_err(map_block)?; + self.wait(request).await?; + self.stats.discards = self.stats.discards.saturating_add(1); + done += count; + } + Ok(()) + } + + async fn zero(&mut self, lba: u64, blocks: u32) -> Result<(), ProviderError> { + self.forget_held(); + let buffers = [self.io, self.rollback]; + for buffer in buffers { + self.channel.fill_buffer(buffer, 0).map_err(map_block)?; + } + let mut done = 0u32; + while done < blocks { + let wave_start = done; + let mut counts = [0u32; BULK_DEPTH]; + let mut planned = 0usize; + let mut planned_done = done; + for count in &mut counts { + *count = self + .streaming_write_blocks(lba + u64::from(planned_done), blocks - planned_done); + if *count == 0 { + break; + } + planned += 1; + planned_done += *count; + } + let mut requests = [None; BULK_DEPTH]; + let mut accepted = 0usize; + let mut stage_error = None; + let mut at = wave_start; + { + let mut batch = self.channel.batch(); + for index in 0..planned { + match batch.stage_write(lba + u64::from(at), counts[index], buffers[index], 0) { + Ok(request) => { + requests[index] = Some(request); + accepted += 1; + } + Err(error) => { + stage_error = Some(error); + break; + } + } + at += counts[index]; + } + batch.publish(); + } + if accepted == 0 { + return Err(map_block(stage_error.unwrap_or(BlockError::Backpressure))); + } + self.wait_many(&mut requests).await?; + if let Some(error) = stage_error + && error != BlockError::Backpressure + { + return Err(map_block(error)); + } + done += counts[..accepted].iter().copied().sum::(); + self.stats.writes = self.stats.writes.saturating_add(accepted as u64); + self.stats.bulk_batches = self.stats.bulk_batches.saturating_add(1); + } + Ok(()) + } + + async fn copy_extent(&mut self, from: u64, to: u64, blocks: u32) -> Result<(), ProviderError> { + self.forget_held(); + let buffers = [self.io, self.rollback]; + let mut done = 0u32; + while done < blocks { + let wave_start = done; + let mut counts = [0u32; BULK_DEPTH]; + let mut planned = 0usize; + let mut planned_done = done; + for count in &mut counts { + *count = self + .streaming_blocks(from + u64::from(planned_done), blocks - planned_done) + .min(self.streaming_blocks(to + u64::from(planned_done), blocks - planned_done)) + .min(self.streaming_write_blocks( + to + u64::from(planned_done), + blocks - planned_done, + )); + if *count == 0 { + break; + } + planned += 1; + planned_done += *count; + } + let mut reads = [None; BULK_DEPTH]; + let mut issued = 0usize; + let mut read_stage_error = None; + let mut at = wave_start; + { + let mut batch = self.channel.batch(); + for index in 0..planned { + match batch.stage_read(from + u64::from(at), counts[index], buffers[index], 0) { + Ok(request) => { + reads[index] = Some(request); + issued += 1; + } + Err(error) => { + read_stage_error = Some(error); + break; + } + } + at += counts[index]; + } + batch.publish(); + } + if issued == 0 { + return Err(map_block( + read_stage_error.unwrap_or(BlockError::Backpressure), + )); + } + self.wait_many(&mut reads).await?; + if let Some(error) = read_stage_error + && error != BlockError::Backpressure + { + return Err(map_block(error)); + } + self.stats.reads = self.stats.reads.saturating_add(issued as u64); + self.stats.bulk_batches = self.stats.bulk_batches.saturating_add(1); + + let mut written = 0usize; + while written < issued { + let mut writes = [None; BULK_DEPTH]; + let mut accepted = 0usize; + let mut write_stage_error = None; + let mut at = wave_start + counts[..written].iter().copied().sum::(); + { + let mut batch = self.channel.batch(); + for index in written..issued { + let count = counts[index]; + match batch.stage_write(to + u64::from(at), count, buffers[index], 0) { + Ok(request) => { + writes[index] = Some(request); + accepted += 1; + } + Err(error) => { + write_stage_error = Some(error); + break; + } + } + at += count; + } + batch.publish(); + } + if accepted == 0 { + return Err(map_block( + write_stage_error.unwrap_or(BlockError::Backpressure), + )); + } + self.wait_many(&mut writes).await?; + if let Some(error) = write_stage_error + && error != BlockError::Backpressure + { + return Err(map_block(error)); + } + written += accepted; + self.stats.writes = self.stats.writes.saturating_add(accepted as u64); + self.stats.bulk_batches = self.stats.bulk_batches.saturating_add(1); + } + done += counts[..issued].iter().copied().sum::(); + } + Ok(()) + } + + async fn retire(&mut self, placement: Placement) -> Result<(), ProviderError> { + self.forget_held(); + if placement.blocks == 0 { + if placement.extent.is_some() { + return Err(ProviderError::Integrity); + } + return Ok(()); + } + if !placement.committed { + if self.channel.geometry().discard { + let _ = self.discard(placement.lba, placement.blocks).await; + } + self.extents + .release(placement.extent.ok_or(ProviderError::Integrity)?)?; + return Ok(()); + } + self.retired + .push(placement) + // A full retired-extent table is a slot ceiling, not a broken + // provider: `Backpressure` here reached the client as + // `ProviderFailure`, telling it the storage backend had failed + // when in fact a bound was reached. + .map_err(|_| ProviderError::PlacementsFull) + } + + async fn resize_placement(&mut self, key: BlockKey, len: u64) -> Result<(), ProviderError> { + self.forget_held(); + let old = *self + .placements + .get(key.0) + .ok_or(ProviderError::UnknownKey)?; + let wanted = self.blocks_for(len)?; + if len == old.len { + return Ok(()); + } + if wanted == old.blocks && len < old.len { + self.placements + .get_mut(key.0) + .ok_or(ProviderError::UnknownKey)? + .len = len; + return Ok(()); + } + if wanted == 0 { + self.retire(old).await?; + *self + .placements + .get_mut(key.0) + .ok_or(ProviderError::UnknownKey)? = Placement { + lba: 0, + blocks: 0, + len, + committed: false, + extent: None, + written_blocks: 0, + }; + return Ok(()); + } + if old.blocks > 0 && self.retired.is_full() { + return Err(ProviderError::Backpressure); + } + let (new_extent, new_lba) = self.extents.allocate(wanted)?; + let replacement = Placement { + lba: new_lba, + blocks: wanted, + len, + committed: false, + extent: Some(new_extent), + // The copy below carries the old meaningful blocks; the rest of + // the new extent is the zeros written first. + written_blocks: old.written_blocks.min(wanted), + }; + let prepared = async { + self.zero(new_lba, wanted).await?; + if old.blocks > 0 { + self.copy_extent(old.lba, new_lba, old.blocks.min(wanted)) + .await?; + } + if len > old.len { + let block = self.block_size() as u64; + let tail = ((old.len.div_ceil(block) * block).min(len) - old.len) as usize; + if tail > 0 { + let zeros = [0u8; 4096]; + self.write_new_bytes(replacement, old.len, &zeros[..tail]) + .await?; + } + } + Ok::<(), ProviderError>(()) + } + .await; + if let Err(error) = prepared { + self.extents.release(new_extent)?; + return Err(error); + } + if let Err(error) = self.retire(old).await { + self.extents.release(new_extent)?; + return Err(error); + } + let mut placement = self + .placements + .get_mut(key.0) + .ok_or(ProviderError::UnknownKey)?; + placement.lba = new_lba; + placement.blocks = wanted; + placement.len = len; + placement.committed = false; + placement.extent = Some(new_extent); + Ok(()) + } + + async fn read_bytes( + &mut self, + placement: Placement, + offset: u64, + dst: &mut [u8], + ) -> Result { + offset + .checked_add(dst.len() as u64) + .ok_or(ProviderError::Range)?; + if offset > placement.len { + return Err(ProviderError::Range); + } + let want = dst.len().min((placement.len - offset) as usize); + if want == 0 { + return Ok(0); + } + let block = self.block_size(); + let first = offset / block as u64; + let within = offset as usize % block; + let covered = (within + want).div_ceil(block) as u32; + let lba = placement.lba + first; + // Served from the window `io` already holds, or fetched with room + // to spare so the next sequential read is. The window is bounded + // by the placement (never a neighbour's blocks), by the buffer, and + // by the device's streaming limits; a boundary that would split the + // request itself falls back to exactly what was asked, as before. + let at = match self.held { + Some(held) + if held.lba <= lba + && lba + u64::from(covered) <= held.lba + u64::from(held.blocks) => + { + self.stats.readahead_hits = self.stats.readahead_hits.saturating_add(1); + (lba - held.lba) as usize * block + } + _ => { + self.held = None; + let remaining = placement + .blocks + .saturating_sub(u32::try_from(first).unwrap_or(u32::MAX)) + .max(covered); + let window = u32::try_from(READ_AHEAD_BYTES / block) + .unwrap_or(u32::MAX) + .max(covered) + .min(remaining); + let mut count = self.streaming_blocks(lba, window); + if count < covered { + count = covered; + } + self.read_blocks(lba, count, self.io).await?; + self.held = Some(HeldRead { lba, blocks: count }); + self.stats.readahead_reads = self.stats.readahead_reads.saturating_add(1); + 0 + } + }; + self.channel + .read_buffer(self.io, at + within, &mut dst[..want]) + .map_err(map_block)?; + self.stats.copied_bytes = self.stats.copied_bytes.saturating_add(want as u64); + Ok(want) + } + + async fn write_new_bytes( + &mut self, + placement: Placement, + offset: u64, + src: &[u8], + ) -> Result<(), ProviderError> { + let end = offset + .checked_add(src.len() as u64) + .ok_or(ProviderError::Range)?; + if end > placement.len { + return Err(ProviderError::Range); + } + if src.is_empty() { + return Ok(()); + } + let block = self.block_size(); + let first = offset / block as u64; + let within = offset as usize % block; + let covered = (within + src.len()).div_ceil(block); + + // Only a partial edge block has old bytes to keep; the interior is + // overwritten whole. Edges land in `io` where they will be written + // from, the payload patches over them in place, `io` goes back: no + // stack image of the transfer, and an aligned write costs no read. + self.forget_held(); + self.read_edges(placement.lba + first, within, src.len(), covered) + .await?; + self.channel + .write_buffer(self.io, within, src) + .map_err(map_block)?; + if let Err(error) = self + .write_blocks(placement.lba + first, covered as u32, self.io) + .await + { + self.stats.rollbacks = self.stats.rollbacks.saturating_add(1); + return Err(error); + } + self.stats.copied_bytes = self.stats.copied_bytes.saturating_add(src.len() as u64); + Ok(()) + } + + /// Overwrite an extent no durable root can name yet. + /// + /// A deferred bulk writer publishes one COW replacement on its first + /// mutation. Re-copying that whole unpublished extent for every later + /// chunk turns an N-byte import into O(N^2) device traffic. The placement + /// itself is already the rollback boundary here, so update it in place + /// while retaining the touched blocks in the dedicated rollback buffer. + /// An accepted write that fails is restored before the refusal returns; + /// if restoration itself fails, the channel is lost rather than exposing + /// bytes whose revision no longer describes them. + async fn write_uncommitted_bytes( + &mut self, + placement: Placement, + offset: u64, + src: &[u8], + ) -> Result<(), ProviderError> { + debug_assert!(!placement.committed); + let end = offset + .checked_add(src.len() as u64) + .ok_or(ProviderError::Range)?; + if end > placement.len { + return Err(ProviderError::Range); + } + if src.is_empty() { + return Ok(()); + } + let block = self.block_size(); + let first = offset / block as u64; + let within = offset as usize % block; + let covered = (within + src.len()).div_ceil(block); + + // The exact old blocks stay in `rollback` until the write's outcome + // is known. `io` needs old bytes only where the payload leaves + // some — a partial head or tail block — and those are mirrored from + // `rollback`: at most two blocks through the CPU, never the whole + // transfer. Past the placement's high-water mark the old bytes are + // the allocation's zeros: nothing is read, the edges are zeroed in + // place, and a failed write is restored with zeros. + self.forget_held(); + let pristine = first >= u64::from(placement.written_blocks); + if !pristine { + self.read_blocks(placement.lba + first, covered as u32, self.rollback) + .await?; + } + let head = within != 0; + let tail = (within + src.len()) % block != 0; + let last = covered - 1; + let edges = [ + head.then_some(0), + (tail && (last != 0 || !head)).then_some(last * block), + ]; + let mut chunk = [0u8; COPY_CHUNK_BYTES]; + for at in edges.into_iter().flatten() { + if pristine { + chunk[..block].fill(0); + } else { + self.channel + .read_buffer(self.rollback, at, &mut chunk[..block]) + .map_err(map_block)?; + } + self.channel + .write_buffer(self.io, at, &chunk[..block]) + .map_err(map_block)?; + } + if !head && !tail { + self.stats.aligned_writes = self.stats.aligned_writes.saturating_add(1); + } + if pristine { + self.stats.pristine_writes = self.stats.pristine_writes.saturating_add(1); + } + self.channel + .write_buffer(self.io, within, src) + .map_err(map_block)?; + + // Split submission from completion so a pre-submit refusal needs no + // restoration, while every accepted command retains the exact old + // blocks in `rollback` until its completion is known. + let request = self + .channel + .submit_write(placement.lba + first, covered as u32, self.io, 0) + .map_err(map_block)?; + if let Err(error) = self.wait(request).await { + self.stats.rollbacks = self.stats.rollbacks.saturating_add(1); + let restored = if pristine { + self.channel + .fill_buffer(self.rollback, 0) + .map_err(map_block) + } else { + Ok(()) + }; + if restored.is_err() + || self + .write_blocks(placement.lba + first, covered as u32, self.rollback) + .await + .is_err() + { + self.channel.mark_lost(); + return Err(ProviderError::Lost); + } + return Err(error); + } + self.stats.writes = self.stats.writes.saturating_add(1); + self.stats.copied_bytes = self.stats.copied_bytes.saturating_add(src.len() as u64); + Ok(()) + } + + async fn write_bytes( + &mut self, + key: BlockKey, + old: Placement, + offset: u64, + src: &[u8], + ) -> Result<(), ProviderError> { + let end = offset + .checked_add(src.len() as u64) + .ok_or(ProviderError::Range)?; + if end > old.len { + return Err(ProviderError::Range); + } + if src.is_empty() { + return Ok(()); + } + if !old.committed { + self.write_uncommitted_bytes(old, offset, src).await?; + self.advance_written(key, end)?; + return Ok(()); + } + if self.retired.is_full() { + return Err(ProviderError::Backpressure); + } + let (new_extent, new_lba) = self.extents.allocate(old.blocks)?; + if let Err(error) = self.copy_extent(old.lba, new_lba, old.blocks).await { + self.stats.rollbacks = self.stats.rollbacks.saturating_add(1); + self.extents.release(new_extent)?; + return Err(error); + } + let replacement = Placement { + lba: new_lba, + committed: false, + extent: Some(new_extent), + ..old + }; + if let Err(error) = self.write_new_bytes(replacement, offset, src).await { + self.extents.release(new_extent)?; + return Err(error); + } + if let Err(error) = self.retire(old).await { + self.extents.release(new_extent)?; + return Err(error); + } + *self + .placements + .get_mut(key.0) + .ok_or(ProviderError::UnknownKey)? = replacement; + self.advance_written(key, end) + } + + /// Record that bytes up to `end` are now meaningful. + fn advance_written(&mut self, key: BlockKey, end: u64) -> Result<(), ProviderError> { + let blocks = self.blocks_for(end)?; + let mut placement = self + .placements + .get_mut(key.0) + .ok_or(ProviderError::UnknownKey)?; + placement.written_blocks = placement.written_blocks.max(blocks); + Ok(()) + } +} + +impl StorageProvider + for BlockProvider +{ + type Key = BlockKey; + const PERSISTS_ACROSS_RESTART: bool = true; + + fn limits(&self) -> StorageLimits { + self.limits.get() + } + + fn device_blocks(&self, key: BlockKey) -> Option { + let state = self.state.try_borrow().ok()?; + let placement = *state.placements.get(key.0)?; + let block_size = state.channel.geometry().block_size; + let whole = (placement.len / u64::from(block_size)).min(u64::from(placement.blocks)); + Some(crate::provider::DeviceBlocks { + lba: state.channel.namespace_lba().checked_add(placement.lba)?, + blocks: whole, + block_size, + }) + } + + async fn allocate(&self, _kind: ObjectKind, len: u64) -> Result { + if len > self.limits.get().max_object_bytes { + return Err(ProviderError::TooLarge); + } + let mut state = self.state.borrow_mut(); + let blocks = state.blocks_for(len)?; + let (allocation, lba) = if blocks == 0 { + (None, 0) + } else { + let (extent, lba) = state.extents.allocate(blocks)?; + (Some(extent), lba) + }; + if let Some(extent) = allocation { + if let Err(error) = state.zero(lba, blocks).await { + state.extents.release(extent)?; + return Err(error); + } + } + let inserted = state + .placements + .insert(Placement { + lba, + blocks, + len, + committed: false, + extent: allocation, + written_blocks: 0, + }) + .map(BlockKey) + // The table, not the medium: the extent index above already proved the + // bytes fit, so what ran out is the extent-tracking ceiling. + .map_err(|_| ProviderError::PlacementsFull); + if inserted.is_err() + && let Some(extent) = allocation + { + state.extents.release(extent)?; + } + inserted + } + + async fn read( + &self, + key: BlockKey, + offset: u64, + dst: &mut [u8], + ) -> Result { + if dst.len() > self.limits.get().max_transfer { + return Err(ProviderError::TooLarge); + } + let mut state = self.state.borrow_mut(); + let placement = *state + .placements + .get(key.0) + .ok_or(ProviderError::UnknownKey)?; + state.read_bytes(placement, offset, dst).await + } + + /// DMA to DMA: whole extents move between the two registered buffers + /// in the same two-deep waves the durable COW copy uses, and no byte + /// visits the CPU. Block-granular by construction — both placements + /// begin at offset zero — so the tail block beyond `len` carries the + /// source's own tail, which nothing below `len` can observe. + async fn copy_object( + &self, + source: BlockKey, + destination: BlockKey, + len: u64, + ) -> Result<(), ProviderError> { + let mut state = self.state.borrow_mut(); + let from = *state + .placements + .get(source.0) + .ok_or(ProviderError::UnknownKey)?; + let to = *state + .placements + .get(destination.0) + .ok_or(ProviderError::UnknownKey)?; + if len > from.len || len > to.len { + return Err(ProviderError::Range); + } + if to.committed { + return Err(ProviderError::Integrity); + } + let blocks = state.blocks_for(len)?; + if blocks == 0 { + return Ok(()); + } + if blocks > from.blocks || blocks > to.blocks { + return Err(ProviderError::Integrity); + } + state.copy_extent(from.lba, to.lba, blocks).await?; + state.stats.snapshot_copies = state.stats.snapshot_copies.saturating_add(1); + let mut placement = state + .placements + .get_mut(destination.0) + .ok_or(ProviderError::UnknownKey)?; + placement.written_blocks = placement.written_blocks.max(blocks); + Ok(()) + } + + async fn write(&self, key: BlockKey, offset: u64, src: &[u8]) -> Result<(), ProviderError> { + if src.len() > self.limits.get().max_transfer { + return Err(ProviderError::TooLarge); + } + let mut state = self.state.borrow_mut(); + let placement = *state + .placements + .get(key.0) + .ok_or(ProviderError::UnknownKey)?; + state.write_bytes(key, placement, offset, src).await + } + + async fn resize(&self, key: BlockKey, len: u64) -> Result<(), ProviderError> { + if len > self.limits.get().max_object_bytes { + return Err(ProviderError::TooLarge); + } + self.state.borrow_mut().resize_placement(key, len).await + } + + async fn release(&self, key: BlockKey) -> Result<(), ProviderError> { + let mut state = self.state.borrow_mut(); + let placement = *state + .placements + .get(key.0) + .ok_or(ProviderError::UnknownKey)?; + if placement.blocks > 0 && state.retired.is_full() { + return Err(ProviderError::Backpressure); + } + let placement = state + .placements + .remove(key.0) + .ok_or(ProviderError::UnknownKey)?; + state.retire(placement).await?; + Ok(()) + } + + async fn sync(&self, key: BlockKey) -> Result<(), ProviderError> { + let mut state = self.state.borrow_mut(); + if state.placements.get(key.0).is_none() { + return Err(ProviderError::UnknownKey); + } + state.flush().await + } +} + +pub(crate) fn map_block(error: BlockError) -> ProviderError { + match error { + BlockError::Geometry | BlockError::Integrity | BlockError::StaleCompletion => { + ProviderError::Integrity + } + BlockError::Range + | BlockError::Alignment + | BlockError::UnknownBuffer + | BlockError::UnknownRequest => ProviderError::Range, + BlockError::TooLarge => ProviderError::TooLarge, + BlockError::ReadOnly => ProviderError::ReadOnly, + BlockError::Backpressure | BlockError::BufferBusy => ProviderError::Backpressure, + BlockError::Cancelled => ProviderError::Cancelled, + BlockError::Revoked => ProviderError::Revoked, + BlockError::Draining => ProviderError::Draining, + BlockError::Lost => ProviderError::Lost, + BlockError::Media => ProviderError::Media, + BlockError::Transport => ProviderError::Transport, + BlockError::Unsupported => ProviderError::Integrity, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use kore_driver::block::{BlockChannel, CompletionMode, memory::MemoryBlockDevice}; + + // Keep fixtures compact: provider engines are intentionally heapless and + // therefore live on the test thread's stack. + type Device = MemoryBlockDevice<49152, 4096>; + type Provider = BlockProvider; + + fn provider() -> Provider { + let channel = + BlockChannel::new(Device::new(512, 8).unwrap(), CompletionMode::Poll).unwrap(); + Provider::new(channel).unwrap() + } + + /// A wait that expired against a HEALTHY controller must not retire the + /// channel. `mark_lost` is a one-way door — it takes the mounted volume + /// with it — and a late completion is equally explained by a deep queue, + /// GC, thermal throttling, or the host descheduling the waiter. Only the + /// device's own verdict justifies the door. + /// Sequential payload reads are served from one read-ahead window: the + /// first read fetches as much of the placement as the buffer holds, the + /// rest cost no command, and any write to the placement forgets the + /// window so the next read fetches fresh bytes. + #[test] + fn sequential_reads_ride_one_read_ahead_window_until_a_write() { + let provider = provider(); + // 8 blocks; the test device's 4 KiB buffer makes the window 8 blocks. + let key = crate::block_on(provider.allocate(ObjectKind::File, 4096)).unwrap(); + for chunk in 0..8u64 { + let byte = 0x10 + chunk as u8; + crate::block_on(provider.write(key, chunk * 512, &[byte; 512])).unwrap(); + } + let before = provider.stats(); + let mut dst = [0u8; 512]; + for chunk in 0..8u64 { + let got = crate::block_on(provider.read(key, chunk * 512, &mut dst)).unwrap(); + assert_eq!(got, 512); + assert!( + dst.iter().all(|b| *b == 0x10 + chunk as u8), + "chunk {chunk} bytes" + ); + } + let after = provider.stats(); + assert_eq!( + after.readahead_reads - before.readahead_reads, + 1, + "one window fetch" + ); + assert_eq!( + after.readahead_hits - before.readahead_hits, + 7, + "seven served from it" + ); + assert_eq!( + after.reads - before.reads, + 1, + "one device read for eight requests" + ); + + // A write forgets the window; the next read fetches again and sees + // the new bytes, never a stale mirror. + crate::block_on(provider.write(key, 3 * 512, &[0xEE; 512])).unwrap(); + let mid = provider.stats(); + let got = crate::block_on(provider.read(key, 3 * 512, &mut dst)).unwrap(); + assert_eq!(got, 512); + assert!(dst.iter().all(|b| *b == 0xEE)); + let end = provider.stats(); + assert_eq!(end.readahead_reads - mid.readahead_reads, 1); + assert_eq!(end.readahead_hits - mid.readahead_hits, 0); + // And a read past the window's end fetches a new one, unaligned + // requests included. + let mut tail = [0u8; 300]; + let got = crate::block_on(provider.read(key, 4096 - 300, &mut tail)).unwrap(); + assert_eq!(got, 300); + assert!(tail.iter().all(|b| *b == 0x17)); + } + + /// A write that begins and ends on block boundaries has no old bytes to + /// keep: into an uncommitted extent it costs the rollback read and the + /// write, into a fresh extent just the write. A partial edge costs one + /// block, never the span — and the bytes around every edge survive. + #[test] + fn aligned_writes_skip_the_pre_read_and_edges_cost_one_block_each() { + let provider = provider(); + let key = crate::block_on(provider.allocate(ObjectKind::File, 4096)).unwrap(); + let mut expected = [0u8; 4096]; + let check = |provider: &Provider, expected: &[u8; 4096]| { + // The test buffer admits ~3.5 KiB per transfer; read in quarters. + let mut back = [0u8; 4096]; + for piece in 0..4 { + let at = piece * 1024; + assert_eq!( + crate::block_on(provider.read(key, at as u64, &mut back[at..at + 1024])), + Ok(1024) + ); + } + assert_eq!(&back[..], &expected[..]); + }; + + // Uncommitted, aligned, past the high-water mark: one write, no + // read of any kind. + let before = provider.stats(); + crate::block_on(provider.write(key, 0, &[1; 2048])).unwrap(); + expected[..2048].fill(1); + let after = provider.stats(); + assert_eq!(after.reads, before.reads); + assert_eq!(after.writes - before.writes, 1); + assert_eq!(after.aligned_writes - before.aligned_writes, 1); + assert_eq!(after.pristine_writes - before.pristine_writes, 1); + assert_eq!(after.edge_reads, 0); + check(&provider, &expected); + + // Uncommitted, both edges partial, below the mark: one rollback + // read — the edges are mirrored from that image, not fetched again. + let before = provider.stats(); + crate::block_on(provider.write(key, 100, &[2; 1200])).unwrap(); + expected[100..1300].fill(2); + let after = provider.stats(); + assert_eq!(after.reads - before.reads, 1); + assert_eq!(after.aligned_writes, before.aligned_writes); + assert_eq!(after.edge_reads, 0); + check(&provider, &expected); + + // Committed, aligned: the COW copy reads once, the write itself + // reads nothing. + crate::block_on(provider.checkpoint()); + let before = provider.stats(); + crate::block_on(provider.write(key, 512, &[3; 1024])).unwrap(); + expected[512..1536].fill(3); + let after = provider.stats(); + assert_eq!(after.reads - before.reads, 1, "the extent copy only"); + assert_eq!(after.aligned_writes - before.aligned_writes, 1); + assert_eq!(after.edge_reads, 0); + check(&provider, &expected); + + // Committed, two distinct partial edges: the copy plus one block + // per edge, fetched together. + crate::block_on(provider.checkpoint()); + let before = provider.stats(); + crate::block_on(provider.write(key, 700, &[4; 1500])).unwrap(); + expected[700..2200].fill(4); + let after = provider.stats(); + assert_eq!(after.reads - before.reads, 3); + assert_eq!(after.edge_reads - before.edge_reads, 2); + check(&provider, &expected); + + // Committed, one block that is both edges. + crate::block_on(provider.checkpoint()); + let before = provider.stats(); + crate::block_on(provider.write(key, 3000, &[5; 40])).unwrap(); + expected[3000..3040].fill(5); + let after = provider.stats(); + assert_eq!(after.reads - before.reads, 2); + assert_eq!(after.edge_reads - before.edge_reads, 1); + check(&provider, &expected); + } + + /// A sequential fill of a fresh extent — the store seed — never reads: + /// each chunk lands past the high-water mark, where the extent holds the + /// zeros allocation wrote. A rewrite below the mark reads its rollback + /// image again, and a failed pristine write restores zeros. + #[test] + fn writes_past_the_high_water_mark_read_nothing_and_roll_back_to_zeros() { + let provider = provider(); + let key = crate::block_on(provider.allocate(ObjectKind::File, 4096)).unwrap(); + let before = provider.stats(); + for chunk in 0..4u64 { + crate::block_on(provider.write(key, chunk * 1024, &[0x30 + chunk as u8; 1024])) + .unwrap(); + } + let after = provider.stats(); + assert_eq!(after.reads, before.reads, "a seed reads nothing"); + assert_eq!(after.writes - before.writes, 4); + assert_eq!(after.pristine_writes - before.pristine_writes, 4); + + // Below the mark: the old bytes matter again. + let before = provider.stats(); + crate::block_on(provider.write(key, 1500, &[0xAA; 100])).unwrap(); + let after = provider.stats(); + assert_eq!(after.reads - before.reads, 1); + assert_eq!(after.pristine_writes, before.pristine_writes); + let mut back = [0u8; 1024]; + assert_eq!( + crate::block_on(provider.read(key, 1024, &mut back)), + Ok(1024) + ); + assert!(back[..476].iter().all(|b| *b == 0x31)); + assert!(back[476..576].iter().all(|b| *b == 0xAA)); + assert!(back[576..].iter().all(|b| *b == 0x31)); + + // A fresh placement whose first, pristine write fails is zeros + // again afterwards, not a torn image. + let key = crate::block_on(provider.allocate(ObjectKind::File, 2048)).unwrap(); + provider + .control(|device| device.inject_after(0, BlockError::Media)) + .unwrap(); + assert_eq!( + crate::block_on(provider.write(key, 0, &[7; 1024])), + Err(ProviderError::Media) + ); + let mut back = [0u8; 1024]; + assert_eq!(crate::block_on(provider.read(key, 0, &mut back)), Ok(1024)); + assert_eq!(back, [0u8; 1024]); + assert_eq!(provider.stats().rollbacks, 1); + } + + /// A version lend's snapshot copies whole extents between the DMA + /// buffers: one read and one write per wave, whatever the transfer + /// limit, and the destination reads back byte-exact below `len`. + #[test] + fn snapshot_copies_move_whole_extents_between_dma_buffers() { + let provider = provider(); + let source = crate::block_on(provider.allocate(ObjectKind::File, 3000)).unwrap(); + let pattern: [u8; 3000] = core::array::from_fn(|i| (i * 7 % 251) as u8); + for (index, piece) in pattern.chunks(1000).enumerate() { + crate::block_on(provider.write(source, (index * 1000) as u64, piece)).unwrap(); + } + let destination = crate::block_on(provider.allocate(ObjectKind::File, 3000)).unwrap(); + let before = provider.stats(); + crate::block_on(provider.copy_object(source, destination, 3000)).unwrap(); + let after = provider.stats(); + // Six blocks fit one wave of the 4 KiB test buffer: one read, one + // write, no payload bytes through the CPU. + assert_eq!(after.reads - before.reads, 1); + assert_eq!(after.writes - before.writes, 1); + assert_eq!(after.snapshot_copies - before.snapshot_copies, 1); + assert_eq!(after.copied_bytes, before.copied_bytes); + let mut back = [0u8; 3000]; + for (index, piece) in back.chunks_mut(1000).enumerate() { + assert_eq!( + crate::block_on(provider.read(destination, (index * 1000) as u64, piece)), + Ok(1000) + ); + } + assert_eq!(back, pattern); + // A later write to the copy sits below its high-water mark and keeps + // the copied bytes around it. + crate::block_on(provider.write(destination, 100, &[0xEE; 50])).unwrap(); + let mut head = [0u8; 200]; + assert_eq!( + crate::block_on(provider.read(destination, 0, &mut head)), + Ok(200) + ); + assert_eq!(&head[..100], &pattern[..100]); + assert!(head[100..150].iter().all(|b| *b == 0xEE)); + assert_eq!(&head[150..], &pattern[150..200]); + // Refusals: a committed destination, a length past either object. + crate::block_on(provider.checkpoint()); + assert_eq!( + crate::block_on(provider.copy_object(source, destination, 3000)), + Err(ProviderError::Integrity) + ); + let fresh = crate::block_on(provider.allocate(ObjectKind::File, 1000)).unwrap(); + assert_eq!( + crate::block_on(provider.copy_object(source, fresh, 3000)), + Err(ProviderError::Range) + ); + } + + /// The provider's transfer limit is its own, not the engine's 4 KiB: + /// bounded by the DMA buffer, so a device with a large buffer admits a + /// large single command and a small one refuses cleanly above its own. + #[test] + fn transfer_limit_follows_the_buffer_not_the_engine_bound() { + let provider = provider(); + let limit = provider.limits().max_transfer; + assert!( + limit > crate::bounds::MAX_TRANSFER / 2, + "a 4 KiB buffer admits ~3.5 KiB" + ); + assert!(limit <= BLOCK_MAX_TRANSFER); + let key = crate::block_on(provider.allocate(ObjectKind::File, 4096)).unwrap(); + let mut dst = [0u8; BLOCK_MAX_TRANSFER]; + assert_eq!( + crate::block_on(provider.read(key, 0, &mut dst[..limit + 1])), + Err(ProviderError::TooLarge) + ); + assert!(crate::block_on(provider.read(key, 0, &mut dst[..limit])).is_ok()); + } + + #[test] + fn a_slow_but_healthy_device_is_not_declared_lost() { + let provider = provider(); + // Allocate while completions still flow — `allocate` zeroes the + // extent, which is itself a command that must complete. + let key = crate::block_on(provider.allocate(ObjectKind::File, 512)).unwrap(); + // Write once so the next write sits below the high-water mark and + // begins with its rollback read: the command whose wait expires is + // then a read, which leaves nothing to restore. + crate::block_on(provider.write(key, 0, &[6u8; 512])).unwrap(); + { + let mut state = provider.state.borrow_mut(); + state + .channel + .control(|d| d.set_liveness(kore_driver::block::Liveness::Healthy)) + .unwrap(); + // Never deliver the completion: the wait must run its budget out. + state.channel.control(|d| d.hold_completions(true)).unwrap(); + } + let outcome = crate::block_on(provider.write(key, 0, &[7u8; 512])); + assert_eq!( + outcome, + Err(ProviderError::Timeout), + "a healthy controller means slow, not dead" + ); + // Every round past the first budget was granted on the controller's + // own word, then the ceiling refused the command: each round is one + // expired-but-healthy wait, and the refusal is one more. + let stats = provider.stats(); + assert_eq!(stats.expired_waits, u64::from(STALL_BUDGETS) + 1); + assert_eq!(stats.slow_waits, u64::from(STALL_BUDGETS) + 1); + // The door stayed shut: the channel is still usable. + { + let state = provider.state.borrow(); + assert_ne!( + state.channel.state(), + kore_driver::block::ChannelState::Lost + ); + } + } + + /// A healthy device that answers late — inside the stall ceiling — is + /// waited out and the command completes: the rounds it ran past one + /// budget are booked, and nothing is refused or condemned. + #[test] + fn a_healthy_device_that_answers_late_still_completes() { + let provider = provider(); + let key = crate::block_on(provider.allocate(ObjectKind::File, 512)).unwrap(); + crate::block_on(provider.write(key, 0, &[6u8; 512])).unwrap(); + let late_by = 3u32; + assert!( + late_by < STALL_BUDGETS, + "the test must sit inside the ceiling" + ); + { + let mut state = provider.state.borrow_mut(); + state + .channel + .control(|d| d.set_liveness(kore_driver::block::Liveness::Healthy)) + .unwrap(); + state + .channel + .control(|d| d.hold_completions_for(late_by)) + .unwrap(); + } + assert_eq!( + crate::block_on(provider.write(key, 0, &[7u8; 512])), + Ok(()), + "a slow-but-healthy command completes inside the ceiling" + ); + let stats = provider.stats(); + assert_eq!(stats.expired_waits, u64::from(late_by)); + assert_eq!(stats.slow_waits, u64::from(late_by)); + let mut back = [0u8; 512]; + assert_eq!(crate::block_on(provider.read(key, 0, &mut back)), Ok(512)); + assert!(back.iter().all(|b| *b == 7), "the late write landed"); + let state = provider.state.borrow(); + assert_ne!( + state.channel.state(), + kore_driver::block::ChannelState::Lost + ); + } + + /// The controller's own fatal verdict IS evidence, and retires the + /// channel — the behaviour a timeout alone used to trigger. + #[test] + fn a_device_reporting_failure_is_declared_lost() { + let provider = provider(); + let key = crate::block_on(provider.allocate(ObjectKind::File, 512)).unwrap(); + { + let mut state = provider.state.borrow_mut(); + state + .channel + .control(|d| d.set_liveness(kore_driver::block::Liveness::Failed)) + .unwrap(); + state.channel.control(|d| d.hold_completions(true)).unwrap(); + } + assert_eq!( + crate::block_on(provider.write(key, 0, &[7u8; 512])), + Err(ProviderError::Lost) + ); + assert_eq!(provider.stats().slow_waits, 0); + { + let state = provider.state.borrow(); + assert_eq!( + state.channel.state(), + kore_driver::block::ChannelState::Lost + ); + } + } + + /// A backend with no out-of-band signal has said nothing, so the + /// conservative pre-existing posture is kept rather than inventing a + /// verdict in either direction. + #[test] + fn a_backend_with_no_liveness_signal_keeps_the_conservative_posture() { + let provider = provider(); + let key = crate::block_on(provider.allocate(ObjectKind::File, 512)).unwrap(); + { + let mut state = provider.state.borrow_mut(); + // `Unknown` is the double's default — asserted so a change to it + // cannot silently weaken this test. + assert_eq!( + state.channel.liveness(), + kore_driver::block::Liveness::Unknown + ); + state.channel.control(|d| d.hold_completions(true)).unwrap(); + } + assert_eq!( + crate::block_on(provider.write(key, 0, &[7u8; 512])), + Err(ProviderError::Lost) + ); + } + + #[test] + fn explicit_retirement_releases_buffers_before_quiescing_the_device() { + assert_eq!(crate::block_on(provider().retire()), Ok(())); + } + + /// The two ways a provider can run out are **different facts**, and each + /// has to say which one it is. + /// + /// A zero-length object takes a placement but no block, so this fills the + /// extent table without spending a byte of the medium: the refusal must be + /// `PlacementsFull` (raise the bound / reclaim retired extents), never + /// `Exhausted` (free bytes) — the volume here is nearly empty. Conflating + /// them sends whoever hit it looking in the wrong place, which on a + /// machine with no serial console is most of the cost of the bug. + #[test] + fn a_full_extent_table_names_itself_not_the_medium() { + let provider = provider(); + let mut placed = 0usize; + let refusal = loop { + match crate::block_on(provider.allocate(ObjectKind::File, 0)) { + Ok(_) => { + placed += 1; + assert!( + placed <= MAX_BLOCK_PLACEMENTS, + "the extent table admitted more than its ceiling" + ); + } + Err(error) => break error, + } + }; + assert_eq!(placed, MAX_BLOCK_PLACEMENTS); + assert_eq!(refusal, ProviderError::PlacementsFull); + // The medium was never the constraint: nothing beyond the metadata + // prefix was ever occupied. + assert_eq!(provider.occupied_blocks(), Some(LEGACY_DATA_START_LBA)); + } + + #[test] + fn a_configured_arena_is_the_one_metadata_and_payload_boundary() { + let provider = provider(); + provider.configure_arena(80, 4).unwrap(); + assert_eq!(provider.arena(), (80, 4)); + assert_eq!(provider.limits().total_bytes, 4 * 512); + assert_eq!(provider.occupied_blocks(), Some(80)); + + let key = crate::block_on(provider.allocate(ObjectKind::File, 4 * 512)).unwrap(); + assert_eq!(provider.extent(key).unwrap().lba, 80); + assert_eq!(provider.occupied_blocks(), Some(84)); + assert_eq!( + crate::block_on(provider.allocate(ObjectKind::File, 1)), + Err(ProviderError::Exhausted) + ); + + let block = [0u8; 512]; + assert_eq!(crate::block_on(provider.write_metadata(79, &block)), Ok(())); + assert_eq!( + crate::block_on(provider.write_metadata(80, &block)), + Err(ProviderError::Range) + ); + assert_eq!( + provider.configure_arena(81, 3), + Err(ProviderError::Integrity), + "a live placement's address cannot be reinterpreted" + ); + } + + #[test] + fn recovery_sorts_once_and_protects_only_fallback_unique_extents() { + let provider = provider(); + let live = [ + DurableExtent { + lba: 80, + blocks: 2, + len: 1024, + }, + DurableExtent { + lba: 75, + blocks: 2, + len: 1024, + }, + ]; + let fallback = [ + live[0], + DurableExtent { + lba: 90, + blocks: 2, + len: 1024, + }, + ]; + + let keys = provider.recover_extents(&live, &fallback).unwrap(); + assert_eq!(provider.extent(keys[0]).unwrap(), live[0]); + assert_eq!(provider.extent(keys[1]).unwrap(), live[1]); + assert_eq!(provider.placements(), 2); + assert_eq!(provider.occupied_blocks(), Some(LEGACY_DATA_START_LBA + 6)); + + let fresh = crate::block_on(provider.allocate(ObjectKind::File, 2 * 512)).unwrap(); + let fresh = provider.extent(fresh).unwrap(); + assert_eq!(fresh.lba, 82); + assert!( + live.iter() + .chain(&fallback[1..]) + .all(|extent| fresh.lba + u64::from(fresh.blocks) <= extent.lba + || fresh.lba >= extent.lba + u64::from(extent.blocks)) + ); + } + + #[test] + fn failed_recovery_rebuild_leaves_the_provider_empty() { + let provider = provider(); + let overlapping = [ + DurableExtent { + lba: 75, + blocks: 4, + len: 2048, + }, + DurableExtent { + lba: 77, + blocks: 2, + len: 1024, + }, + ]; + assert_eq!( + provider.recover_extents(&overlapping, &[]), + Err(ProviderError::Integrity) + ); + assert_eq!(provider.placements(), 0); + assert_eq!(provider.occupied_blocks(), Some(LEGACY_DATA_START_LBA)); + + let valid = [DurableExtent { + lba: 75, + blocks: 2, + len: 1024, + }]; + let keys = provider.recover_extents(&valid, &[]).unwrap(); + assert_eq!(provider.extent(keys[0]).unwrap(), valid[0]); + } + + /// The medium's own ceiling keeps reporting itself as the medium's — this + /// is the tier the split above must not have absorbed. + #[test] + fn a_request_beyond_the_medium_still_names_the_medium() { + let provider = provider(); + let beyond = provider.limits().total_bytes + 1; + assert_eq!( + crate::block_on(provider.allocate(ObjectKind::File, beyond)), + Err(ProviderError::TooLarge) + ); + assert_eq!( + crate::FsError::from(ProviderError::TooLarge), + crate::FsError::OutOfStorage + ); + } + + #[test] + fn unaligned_io_resize_release_flush_and_stale_keys() { + let provider = provider(); + let key = crate::block_on(provider.allocate(ObjectKind::File, 1500)).unwrap(); + crate::block_on(provider.write(key, 17, &[0x5a; 1000])).unwrap(); + let mut back = [0u8; 1000]; + assert_eq!(crate::block_on(provider.read(key, 17, &mut back)), Ok(1000)); + assert_eq!(back, [0x5a; 1000]); + crate::block_on(provider.resize(key, 6000)).unwrap(); + let mut grown = [0xff; 512]; + assert_eq!( + crate::block_on(provider.read(key, 5000, &mut grown)), + Ok(512) + ); + assert_eq!(grown, [0; 512]); + crate::block_on(provider.sync(key)).unwrap(); + assert!(provider.stats().flushes > 0); + crate::block_on(provider.release(key)).unwrap(); + assert_eq!( + crate::block_on(provider.read(key, 0, &mut back)), + Err(ProviderError::UnknownKey) + ); + } + + #[test] + fn allocation_never_reuses_extents_a_durable_root_still_references() { + let provider = provider(); + assert_eq!(provider.occupied_blocks(), Some(LEGACY_DATA_START_LBA)); + let key = crate::block_on(provider.allocate(ObjectKind::File, 1024)).unwrap(); + let old = provider.extent(key).unwrap(); + assert_eq!(provider.occupied_blocks(), Some(LEGACY_DATA_START_LBA + 2)); + // The checkpoint makes `old` durable; the grow below must then retire + // it (the on-disk root still cites those blocks) rather than free it. + crate::block_on(provider.checkpoint()); + crate::block_on(provider.resize(key, 2048)).unwrap(); + assert_ne!(provider.extent(key).unwrap().lba, old.lba); + // Four live blocks plus the two-block extent still named by the last + // durable root: both consume the mounted volume's physical capacity. + assert_eq!(provider.occupied_blocks(), Some(LEGACY_DATA_START_LBA + 6)); + let old_end = old.lba + u64::from(old.blocks); + // While `old` waits in `retired`, a fresh allocation must not land on + // it: a torn commit would recover the durable root over new bytes. + let fresh = crate::block_on(provider.allocate(ObjectKind::File, 1024)).unwrap(); + let fresh_extent = provider.extent(fresh).unwrap(); + let fresh_end = fresh_extent.lba + u64::from(fresh_extent.blocks); + assert!(fresh_end <= old.lba || fresh_extent.lba >= old_end); + // The next checkpoint promotes `old` to the fallback root's + // protection; it stays occupied until a later checkpoint discards it. + crate::block_on(provider.checkpoint()); + let next = crate::block_on(provider.allocate(ObjectKind::File, 1024)).unwrap(); + let next_extent = provider.extent(next).unwrap(); + let next_end = next_extent.lba + u64::from(next_extent.blocks); + assert!(next_end <= old.lba || next_extent.lba >= old_end); + } + + #[test] + fn failed_write_rolls_back_visible_bytes() { + let provider = provider(); + let key = crate::block_on(provider.allocate(ObjectKind::File, 2048)).unwrap(); + let original_extent = provider.extent(key).unwrap(); + crate::block_on(provider.write(key, 0, &[1; 2048])).unwrap(); + assert_eq!(provider.extent(key).unwrap().lba, original_extent.lba); + provider + .control(|device| device.inject_after(1, BlockError::Media)) + .unwrap(); + assert_eq!( + crate::block_on(provider.write(key, 100, &[2; 1200])), + Err(ProviderError::Media) + ); + let mut back = [0; 2048]; + assert_eq!(crate::block_on(provider.read(key, 0, &mut back)), Ok(2048)); + assert_eq!(back, [1; 2048]); + assert_eq!(provider.stats().rollbacks, 1); + } + + #[test] + fn deferred_bulk_writes_reuse_the_uncommitted_extent() { + let provider = provider(); + let key = crate::block_on(provider.allocate(ObjectKind::File, 4096)).unwrap(); + let original = provider.extent(key).unwrap(); + + for offset in (0..4096).step_by(1024) { + crate::block_on(provider.write(key, offset, &[offset as u8; 1024])).unwrap(); + assert_eq!(provider.extent(key).unwrap().lba, original.lba); + } + + // A checkpoint makes the current extent durable. The next write must + // return to ordinary COW so the previous root remains recoverable. + crate::block_on(provider.checkpoint()); + crate::block_on(provider.write(key, 0, &[0xa5; 1024])).unwrap(); + assert_ne!(provider.extent(key).unwrap().lba, original.lba); + } + + #[test] + fn a_large_deferred_import_stays_linear_and_exact() { + std::thread::Builder::new() + .stack_size(32 * 1024 * 1024) + .spawn(|| { + // Match the current Lua archive member closely enough to pin + // the workload that first exposed quadratic incremental COW. + const IMPORT_BYTES: usize = 493_512; + type ImportDevice = MemoryBlockDevice<1_048_576, 262_144>; + type ImportChannel = BlockChannel; + let channel = + ImportChannel::new(ImportDevice::new(512, 512).unwrap(), CompletionMode::Poll) + .unwrap(); + let provider = BlockProvider::new(channel).unwrap(); + let key = crate::block_on(provider.allocate(ObjectKind::File, IMPORT_BYTES as u64)) + .unwrap(); + let original = provider.extent(key).unwrap(); + let baseline = provider.stats(); + let chunks = IMPORT_BYTES.div_ceil(crate::bounds::MAX_TRANSFER); + let payload_byte = |global: usize| { + (global.wrapping_mul(31) ^ (global >> 8) ^ (global >> 16)) as u8 + }; + + for offset in (0..IMPORT_BYTES).step_by(crate::bounds::MAX_TRANSFER) { + let count = (IMPORT_BYTES - offset).min(crate::bounds::MAX_TRANSFER); + let mut bytes = [0u8; crate::bounds::MAX_TRANSFER]; + for (index, byte) in bytes[..count].iter_mut().enumerate() { + *byte = payload_byte(offset + index); + } + crate::block_on(provider.write(key, offset as u64, &bytes[..count])).unwrap(); + } + + let after_write = provider.stats(); + assert_eq!(provider.extent(key).unwrap().lba, original.lba); + assert_eq!(after_write.bulk_batches, baseline.bulk_batches); + // Every chunk lands past the high-water mark: the import + // reads nothing at all. + assert_eq!(after_write.reads, baseline.reads); + assert_eq!( + after_write.pristine_writes - baseline.pristine_writes, + chunks as u64 + ); + assert_eq!(after_write.writes - baseline.writes, chunks as u64); + + for offset in (0..IMPORT_BYTES).step_by(crate::bounds::MAX_TRANSFER) { + let count = (IMPORT_BYTES - offset).min(crate::bounds::MAX_TRANSFER); + let mut bytes = [0u8; crate::bounds::MAX_TRANSFER]; + assert_eq!( + crate::block_on(provider.read(key, offset as u64, &mut bytes[..count])), + Ok(count) + ); + for (index, byte) in bytes[..count].iter().copied().enumerate() { + assert_eq!(byte, payload_byte(offset + index)); + } + } + }) + .unwrap() + .join() + .unwrap(); + } + + #[test] + fn two_providers_own_independent_channels_without_a_shared_lock() { + let (a, b) = (provider(), provider()); + let ka = crate::block_on(a.allocate(ObjectKind::File, 512)).unwrap(); + let kb = crate::block_on(b.allocate(ObjectKind::File, 512)).unwrap(); + crate::block_on(a.write(ka, 0, &[0xaa; 512])).unwrap(); + crate::block_on(b.write(kb, 0, &[0xbb; 512])).unwrap(); + let mut left = [0; 512]; + let mut right = [0; 512]; + crate::block_on(a.read(ka, 0, &mut left)).unwrap(); + crate::block_on(b.read(kb, 0, &mut right)).unwrap(); + assert_eq!(left, [0xaa; 512]); + assert_eq!(right, [0xbb; 512]); + } + + #[test] + fn provider_retains_a_bounded_multi_request_direct_queue_and_reports_it() { + let provider = provider(); + { + let mut state = provider.state.borrow_mut(); + let a = state.channel.submit_flush().unwrap(); + let b = state.channel.submit_flush().unwrap(); + let c = state.channel.submit_flush().unwrap(); + assert_eq!(state.channel.outstanding(), 3); + assert_eq!(state.channel.poll().unwrap().request, a); + assert_eq!(state.channel.poll().unwrap().request, b); + assert_eq!(state.channel.poll().unwrap().request, c); + } + let stats = provider.channel_stats(); + assert_eq!(stats.max_outstanding, 3); + assert_eq!(stats.submissions, 3); + assert_eq!(stats.completions, 3); + } + + #[test] + fn channel_loss_degrades_deterministically_without_wedging() { + let provider = provider(); + let key = crate::block_on(provider.allocate(ObjectKind::File, 512)).unwrap(); + provider.mark_lost(); + let mut bytes = [0; 16]; + assert_eq!( + crate::block_on(provider.read(key, 0, &mut bytes)), + Err(ProviderError::Lost) + ); + assert_eq!( + crate::block_on(provider.sync(key)), + Err(ProviderError::Lost) + ); + } + + #[test] + fn advertised_mdts_is_clamped_to_the_registered_window() { + type WideDevice = MemoryBlockDevice<65_536, 8192>; + type WideChannel = BlockChannel; + let channel = + WideChannel::new(WideDevice::new(512, 128).unwrap(), CompletionMode::Poll).unwrap(); + let provider = BlockProvider::new(channel).unwrap(); + + // One metadata arena is larger than the bounded registered window. + // It must split at the window, not trust the device's larger MDTS. + let written = [0x5au8; 16 * 1024]; + crate::block_on(provider.write_metadata(2, &written)).unwrap(); + let mut read = [0u8; 16 * 1024]; + crate::block_on(provider.read_metadata(2, &mut read)).unwrap(); + assert_eq!(read, written); + assert!(provider.channel_stats().max_outstanding <= 1); + } + + #[test] + fn allocation_batches_single_block_hints_but_preserves_larger_optima() { + // Two registered windows of zeroes: a one-block hint must not turn + // these into sector-sized commands. Eight-block preferences still + // split each window in half, as on the measured 512-byte-LBA SSD. + const WINDOW_BYTES: usize = 8192; + const EXTENT_BYTES: usize = 2 * WINDOW_BYTES; + type BatchDevice = MemoryBlockDevice<65_536, WINDOW_BYTES>; + type BatchChannel = BlockChannel; + for (optimal, expected_writes) in [(0, 2), (1, 2), (8, 4)] { + let device = BatchDevice::new(512, 32).unwrap().with_io_hints( + kore_driver::block::BlockIoHints { + optimal_write_blocks: optimal, + ..kore_driver::block::BlockIoHints::NONE + }, + ); + let channel = BatchChannel::new(device, CompletionMode::Poll).unwrap(); + let provider = BlockProvider::new(channel).unwrap(); + let before = provider.stats(); + let key = + crate::block_on(provider.allocate(ObjectKind::File, EXTENT_BYTES as u64)).unwrap(); + assert_eq!(provider.stats().writes - before.writes, expected_writes); + assert_eq!(provider.channel_stats().max_outstanding, 2); + assert_eq!( + provider.channel_stats().peak_registered_bytes, + 2 * WINDOW_BYTES + ); + let mut back = [0xff; crate::bounds::MAX_TRANSFER]; + for offset in (0..EXTENT_BYTES).step_by(back.len()) { + back.fill(0xff); + assert_eq!( + crate::block_on(provider.read(key, offset as u64, &mut back)), + Ok(back.len()) + ); + assert_eq!(back, [0; crate::bounds::MAX_TRANSFER]); + } + } + } + + #[test] + fn single_block_optimum_preserves_alignment_granularity_and_boundary() { + let device = Device::new(512, 8) + .unwrap() + .with_io_hints(kore_driver::block::BlockIoHints { + preferred_write_blocks: 4, + write_alignment_blocks: 4, + optimal_write_blocks: 1, + optimal_boundary_blocks: 8, + ..kore_driver::block::BlockIoHints::NONE + }); + let channel = BlockChannel::new(device, CompletionMode::Poll).unwrap(); + let provider = Provider::new(channel).unwrap(); + let state = provider.state.borrow(); + assert_eq!(state.streaming_write_blocks(6, 8), 2); + assert_eq!(state.streaming_write_blocks(8, 16), 8); + assert_eq!(state.streaming_write_blocks(8, 7), 4); + assert_eq!(state.streaming_write_blocks(8, 3), 3); + assert_eq!(state.streaming_write_blocks(8, 0), 0); + } + + #[test] + fn streaming_commands_stop_at_the_advertised_optimal_boundary() { + let device = Device::new(512, 8) + .unwrap() + .with_io_hints(kore_driver::block::BlockIoHints { + supported_block_sizes: 0, + preferred_write_blocks: 4, + write_alignment_blocks: 4, + optimal_write_blocks: 4, + optimal_boundary_blocks: 8, + }); + let channel = BlockChannel::new(device, CompletionMode::Poll).unwrap(); + let provider = Provider::new(channel).unwrap(); + let state = provider.state.borrow(); + assert_eq!(state.streaming_blocks(6, 8), 2); + assert_eq!(state.streaming_blocks(8, 8), 8); + assert_eq!(state.streaming_write_blocks(6, 8), 2); + assert_eq!(state.streaming_write_blocks(8, 8), 4); + } + + #[test] + fn bulk_extent_work_uses_two_buffers_and_bounded_queue_batches() { + std::thread::Builder::new() + .stack_size(8 * 1024 * 1024) + .spawn(|| { + type BulkDevice = MemoryBlockDevice<524_288, 32_768>; + type BulkChannel = BlockChannel; + let channel = + BulkChannel::new(BulkDevice::new(512, 64).unwrap(), CompletionMode::Poll) + .unwrap(); + let provider = BlockProvider::new(channel).unwrap(); + + let key = crate::block_on(provider.allocate(ObjectKind::File, 64 * 1024)).unwrap(); + // Exercise the durable COW path. Unpublished placements are + // deliberately updated in place by the deferred-import path. + crate::block_on(provider.checkpoint()); + crate::block_on(provider.write(key, 17, &[0xa5; 4096])).unwrap(); + let mut back = [0u8; 4096]; + assert_eq!(crate::block_on(provider.read(key, 17, &mut back)), Ok(4096)); + assert_eq!(back, [0xa5; 4096]); + + let channel = provider.channel_stats(); + assert_eq!(channel.peak_registered_bytes, 2 * 32_768); + assert_eq!(channel.max_outstanding, 2); + assert!(provider.stats().bulk_batches >= 3); + assert!(provider.control(|device| device.publishes()).unwrap() >= 3); + }) + .unwrap() + .join() + .unwrap(); + } + + #[test] + fn minimum_outstanding_channel_falls_back_to_single_command_waves() { + std::thread::Builder::new() + .stack_size(8 * 1024 * 1024) + .spawn(|| { + type SerialDevice = MemoryBlockDevice<524_288, 32_768>; + type SerialChannel = BlockChannel; + let channel = + SerialChannel::new(SerialDevice::new(512, 64).unwrap(), CompletionMode::Poll) + .unwrap(); + let provider = BlockProvider::new(channel).unwrap(); + + // Both zeroing and the durable COW copy plan two-command + // waves. A depth-two device queue has only one usable slot, + // so each second stage returns Backpressure and must become + // the next wave rather than aborting the operation. + let key = crate::block_on(provider.allocate(ObjectKind::File, 64 * 1024)).unwrap(); + crate::block_on(provider.checkpoint()); + crate::block_on(provider.write(key, 17, &[0xa5; 4096])).unwrap(); + let mut back = [0u8; 4096]; + assert_eq!(crate::block_on(provider.read(key, 17, &mut back)), Ok(4096)); + assert_eq!(back, [0xa5; 4096]); + + assert_eq!(provider.channel_stats().max_outstanding, 1); + assert!(provider.stats().bulk_batches >= 6); + }) + .unwrap() + .join() + .unwrap(); + } + + #[test] + fn one_block_device_still_accepts_bounded_unaligned_payloads() { + type NarrowDevice = MemoryBlockDevice<65_536, 512>; + type NarrowChannel = BlockChannel; + let channel = + NarrowChannel::new(NarrowDevice::new(512, 1).unwrap(), CompletionMode::Poll).unwrap(); + let provider = BlockProvider::new(channel).unwrap(); + assert_eq!(provider.limits().max_transfer, 1); + let key = crate::block_on(provider.allocate(ObjectKind::File, 512)).unwrap(); + crate::block_on(provider.write(key, 511, &[0xa5])).unwrap(); + let mut byte = [0u8; 1]; + assert_eq!(crate::block_on(provider.read(key, 511, &mut byte)), Ok(1)); + assert_eq!(byte, [0xa5]); + } +} diff --git a/native/services/dagfs/src/provider/extent.rs b/native/services/dagfs/src/provider/extent.rs new file mode 100644 index 0000000..07049c6 --- /dev/null +++ b/native/services/dagfs/src/provider/extent.rs @@ -0,0 +1,542 @@ +//! Bounded free-space indexing for block-provider payload extents. +//! +//! Free runs live in a fixed-capacity max-heap, while every physical segment +//! carries neighbour links so release coalesces with at most two nodes. The +//! heap makes both selection and arbitrary neighbour removal O(log N), and +//! selecting its largest run cannot falsely report exhaustion while any run +//! fits. Unlike a memory allocator, metadata never lives in the managed +//! medium — disk blocks remain pure payload and recovery rebuilds this +//! boot-local index from durable extents. +//! +//! The substrate is `kore_collections`' intrusive pair, not an fs-local slot +//! store: nodes live in a [`SlotArena`] (vacancy freelist + generational +//! handles minted per `Used` incarnation) and the free runs are indexed by a +//! [`PriorityIndex`] through the heap back-pointer each node carries inline. +//! What stays here is exactly the block policy — physical-neighbour +//! coalescing, split placement, the recovery containment contract, and the +//! free-block ledger. + +use core::alloc::Allocator; + +use kore_collections::{PriorityIndex, PriorityLink, SlotArena, SlotId, UNINDEXED}; + +use crate::{ServiceAlloc, bounds::MAX_BLOCK_EXTENT_NODES, error::ProviderError}; + +const NONE: u32 = u32::MAX; + +/// Handle to one allocated extent — a [`SlotId`] minted by the node arena +/// for the slot's `Used` incarnation, so a handle that outlives its extent +/// is *detectably* stale: resolving it refuses with `Integrity` instead of +/// silently aliasing whatever run later recycles the slot. +pub(super) type ExtentId = SlotId; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum NodeState { + #[default] + Vacant, + Free, + Used, +} + +#[derive(Clone, Copy, Debug)] +struct Node { + start: u64, + blocks: u64, + prev_phys: u32, + next_phys: u32, + heap_index: u32, + state: NodeState, +} + +impl Default for Node { + /// The claimed-but-unwired payload: no physical neighbours, outside the + /// free index. The arena resets every slot to this on claim and release. + fn default() -> Self { + Self { + start: 0, + blocks: 0, + prev_phys: NONE, + next_phys: NONE, + heap_index: UNINDEXED, + state: NodeState::Vacant, + } + } +} + +impl PriorityLink for Node { + fn heap_index(&self) -> u32 { + self.heap_index + } + + fn set_heap_index(&mut self, position: u32) { + self.heap_index = position; + } + + /// Largest run first; equal sizes prefer the lower placement so + /// selection is deterministic. + fn outranks(&self, other: &Self) -> bool { + self.blocks > other.blocks || (self.blocks == other.blocks && self.start < other.start) + } +} + +/// One volume-local free-space index. Its owner already has exclusive access +/// through `BlockProvider`'s task-local state, so it needs no second lock. +pub(super) struct ExtentAllocator { + nodes: SlotArena, + free: PriorityIndex, + last_phys: u32, + free_blocks: u64, + arena_start: u64, + arena_end: u64, +} + +impl ExtentAllocator { + pub(super) fn new(start: u64, end: u64) -> Result { + let nodes = SlotArena::try_new_in(MAX_BLOCK_EXTENT_NODES, crate::service_alloc()) + .map_err(|_| ProviderError::Backpressure)?; + let free = PriorityIndex::try_new_in(MAX_BLOCK_EXTENT_NODES, crate::service_alloc()) + .map_err(|_| ProviderError::Backpressure)?; + let mut allocator = Self { + nodes, + free, + last_phys: NONE, + free_blocks: 0, + arena_start: 0, + arena_end: 0, + }; + allocator.reset(start, end)?; + Ok(allocator) + } + + pub(super) fn reset(&mut self, start: u64, end: u64) -> Result<(), ProviderError> { + if start >= end { + return Err(ProviderError::Integrity); + } + self.free.clear(); + self.last_phys = NONE; + self.arena_start = start; + self.arena_end = end; + self.free_blocks = end - start; + + // Every slot returns to the vacancy freelist in ascending order. + // `reset` is reachable only through `configure_arena`'s + // empty-provider guard (or `new`), so no `ExtentId` survives it — + // and the arena preserves slot generations across the recycle + // regardless, so even a handle that wrongly did survive would stay + // detectably stale instead of aliasing a post-reset extent. + self.nodes.release_all(); + let whole = self.take_slot()?; + self.nodes[whole].start = start; + self.nodes[whole].blocks = end - start; + self.nodes[whole].state = NodeState::Free; + self.last_phys = whole as u32; + self.insert_free(whole)?; + Ok(()) + } + + #[inline] + pub(super) const fn free_blocks(&self) -> u64 { + self.free_blocks + } + + pub(super) fn range(&self, id: ExtentId) -> Result<(u64, u32), ProviderError> { + let node = self.node(id)?; + if node.state != NodeState::Used { + return Err(ProviderError::Integrity); + } + let blocks = u32::try_from(node.blocks).map_err(|_| ProviderError::Integrity)?; + Ok((node.start, blocks)) + } + + /// Mark `index` as a fresh `Used` incarnation and mint its handle. The + /// arena's generation bump is what retires every previously minted id + /// for this slot, whatever recycling path brought the slot here. + fn mint_used(&mut self, index: usize) -> ExtentId { + self.nodes[index].state = NodeState::Used; + self.nodes[index].heap_index = UNINDEXED; + self.nodes.mint(index) + } + + /// Allocate from the largest free run, returning the handle and the + /// run's first block. Heap selection, split, and index maintenance are + /// O(log N), independent of physical placement order. The returned start + /// always covers exactly `blocks` — the split carves `blocks` from the + /// front of the chosen run. + pub(super) fn allocate(&mut self, blocks: u32) -> Result<(ExtentId, u64), ProviderError> { + if blocks == 0 { + return Err(ProviderError::Integrity); + } + let need = u64::from(blocks); + let raw = self.free.peek().ok_or(ProviderError::Exhausted)?; + let index = self.index(raw)?; + let original = self.nodes[index]; + if original.state != NodeState::Free { + return Err(ProviderError::Integrity); + } + if original.blocks < need { + return Err(ProviderError::Exhausted); + } + + let remainder = if original.blocks > need { + Some(self.take_slot()?) + } else { + None + }; + self.remove_free(index)?; + self.nodes[index].blocks = need; + let id = self.mint_used(index); + + if let Some(tail) = remainder { + let tail_raw = tail as u32; + self.nodes[tail] = Node { + start: original + .start + .checked_add(need) + .ok_or(ProviderError::Integrity)?, + blocks: original.blocks - need, + prev_phys: raw, + next_phys: original.next_phys, + heap_index: UNINDEXED, + state: NodeState::Free, + }; + self.nodes[index].next_phys = tail_raw; + if original.next_phys == NONE { + self.last_phys = tail_raw; + } else { + let next = self.index(original.next_phys)?; + self.nodes[next].prev_phys = tail_raw; + } + self.insert_free(tail)?; + } + self.free_blocks = self + .free_blocks + .checked_sub(need) + .ok_or(ProviderError::Integrity)?; + Ok((id, original.start)) + } + + /// Reserve an exact recovered extent. Callers pass records in ascending + /// LBA order, so the containing segment is always the physical tail — + /// that contract is load-bearing (containment in the tail is what makes + /// one containment check equivalent to a full overlap scan), which is + /// why this function resolves the tail itself rather than taking an + /// arbitrary segment parameter. Only the O(log N) free-heap update + /// remains after the one recovery sort. + pub(super) fn reserve_sorted( + &mut self, + start: u64, + blocks: u32, + ) -> Result { + if blocks == 0 { + return Err(ProviderError::Integrity); + } + let tail = self.index(self.last_phys)?; + let free = self.nodes[tail]; + let need = u64::from(blocks); + let end = start.checked_add(need).ok_or(ProviderError::Integrity)?; + let free_end = free + .start + .checked_add(free.blocks) + .ok_or(ProviderError::Integrity)?; + if free.state != NodeState::Free || start < free.start || end > free_end { + return Err(ProviderError::Integrity); + } + + let prefix = start - free.start; + let suffix = free_end - end; + let used_slot = if prefix == 0 { tail } else { self.take_slot()? }; + let suffix_slot = if suffix == 0 { + None + } else { + match self.take_slot() { + Ok(slot) => Some(slot), + Err(error) => { + if prefix != 0 { + self.release_slot(used_slot); + } + return Err(error); + } + } + }; + + self.remove_free(tail)?; + if prefix == 0 { + // The tail node itself becomes the reservation; its physical + // links already hold, so only the geometry changes here and + // `mint_used` below flips the state. + self.nodes[tail].blocks = need; + } else { + self.nodes[tail].blocks = prefix; + self.nodes[tail].next_phys = used_slot as u32; + self.insert_free(tail)?; + self.nodes[used_slot] = Node { + start, + blocks: need, + prev_phys: tail as u32, + next_phys: NONE, + heap_index: UNINDEXED, + state: NodeState::Used, + }; + } + let id = self.mint_used(used_slot); + + let used_raw = used_slot as u32; + if let Some(slot) = suffix_slot { + let raw = slot as u32; + self.nodes[used_slot].next_phys = raw; + self.nodes[slot] = Node { + start: end, + blocks: suffix, + prev_phys: used_raw, + next_phys: NONE, + heap_index: UNINDEXED, + state: NodeState::Free, + }; + self.insert_free(slot)?; + self.last_phys = raw; + } else { + self.nodes[used_slot].next_phys = NONE; + self.last_phys = used_raw; + } + self.free_blocks = self + .free_blocks + .checked_sub(need) + .ok_or(ProviderError::Integrity)?; + Ok(id) + } + + /// Return one allocated extent, merging with its free physical neighbours. + pub(super) fn release(&mut self, id: ExtentId) -> Result<(), ProviderError> { + let original = self.slot_of(id)?; + if self.nodes[original].state != NodeState::Used { + return Err(ProviderError::Integrity); + } + let released = self.nodes[original].blocks; + self.nodes[original].state = NodeState::Free; + let mut merged = original; + + let prev_raw = self.nodes[merged].prev_phys; + if prev_raw != NONE { + let prev = self.index(prev_raw)?; + if self.nodes[prev].state == NodeState::Free { + self.remove_free(prev)?; + self.absorb(prev, merged)?; + merged = prev; + } + } + + let next_raw = self.nodes[merged].next_phys; + if next_raw != NONE { + let next = self.index(next_raw)?; + if self.nodes[next].state == NodeState::Free { + self.remove_free(next)?; + self.absorb(merged, next)?; + } + } + + self.insert_free(merged)?; + self.free_blocks = self + .free_blocks + .checked_add(released) + .filter(|free| *free <= self.arena_end - self.arena_start) + .ok_or(ProviderError::Integrity)?; + Ok(()) + } + + /// Merge `victim`'s span onto the tail of `survivor` and retire the + /// victim's slot — the one link surgery both coalesce directions share. + /// The caller has already removed whichever of the two was indexed free, + /// and re-indexes the survivor afterwards. + fn absorb(&mut self, survivor: usize, victim: usize) -> Result<(), ProviderError> { + self.nodes[survivor].blocks = self.nodes[survivor] + .blocks + .checked_add(self.nodes[victim].blocks) + .ok_or(ProviderError::Integrity)?; + let after_raw = self.nodes[victim].next_phys; + self.nodes[survivor].next_phys = after_raw; + if after_raw == NONE { + self.last_phys = survivor as u32; + } else { + let after = self.index(after_raw)?; + self.nodes[after].prev_phys = survivor as u32; + } + self.release_slot(victim); + Ok(()) + } + + fn take_slot(&mut self) -> Result { + self.nodes.claim().ok_or(ProviderError::PlacementsFull) + } + + fn release_slot(&mut self, index: usize) { + self.nodes.release(index); + } + + /// Resolve a caller-held handle through the arena: bounds, occupancy, + /// **and** generation must match, so a handle from a superseded + /// incarnation refuses instead of aliasing the slot's current occupant. + /// Internal physical/heap links use raw indices via + /// [`index`](Self::index) — they are maintained under this module's own + /// invariants and never outlive an incarnation. + fn slot_of(&self, id: ExtentId) -> Result { + self.nodes.resolve(id).ok_or(ProviderError::Integrity) + } + + fn node(&self, id: ExtentId) -> Result<&Node, ProviderError> { + let index = self.slot_of(id)?; + Ok(&self.nodes[index]) + } + + fn index(&self, raw: u32) -> Result { + let index = raw as usize; + if raw == NONE || index >= self.nodes.capacity() { + Err(ProviderError::Integrity) + } else { + Ok(index) + } + } + + fn insert_free(&mut self, index: usize) -> Result<(), ProviderError> { + debug_assert!(self.nodes[index].blocks != 0); + self.nodes[index].state = NodeState::Free; + self.free + .insert(self.nodes.values_mut(), index) + .map_err(|_| ProviderError::Integrity) + } + + fn remove_free(&mut self, index: usize) -> Result<(), ProviderError> { + if self.nodes[index].state != NodeState::Free || self.nodes[index].blocks == 0 { + return Err(ProviderError::Integrity); + } + self.free + .remove(self.nodes.values_mut(), index) + .map_err(|_| ProviderError::Integrity) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dense_allocations_do_not_rescan_the_prefix() { + let mut allocator = ExtentAllocator::new(100, 10_000).unwrap(); + let mut ids = heapless::Vec::::new(); + for expected in 100..228 { + let (id, lba) = allocator.allocate(1).unwrap(); + assert_eq!(lba, expected); + assert_eq!(allocator.range(id).unwrap(), (expected, 1)); + ids.push(id).unwrap(); + } + assert_eq!(allocator.free_blocks(), 9_772); + } + + #[test] + fn release_coalesces_both_physical_neighbours() { + let mut allocator = ExtentAllocator::new(72, 100).unwrap(); + let (a, _) = allocator.allocate(4).unwrap(); + let (b, _) = allocator.allocate(5).unwrap(); + let (c, _) = allocator.allocate(6).unwrap(); + allocator.release(a).unwrap(); + allocator.release(c).unwrap(); + allocator.release(b).unwrap(); + assert_eq!(allocator.free_blocks(), 28); + let (whole, lba) = allocator.allocate(28).unwrap(); + assert_eq!(lba, 72); + assert_eq!(allocator.range(whole).unwrap(), (72, 28)); + } + + #[test] + fn sorted_recovery_builds_holes_then_allocates_from_a_free_run() { + let mut allocator = ExtentAllocator::new(72, 100).unwrap(); + let a = allocator.reserve_sorted(75, 4).unwrap(); + let b = allocator.reserve_sorted(84, 3).unwrap(); + assert_eq!(allocator.range(a).unwrap(), (75, 4)); + assert_eq!(allocator.range(b).unwrap(), (84, 3)); + let (allocation, lba) = allocator.allocate(3).unwrap(); + assert_eq!(lba, 87); + assert_eq!(allocator.range(allocation).unwrap(), (87, 3)); + } + + #[test] + fn recovered_extents_must_be_sorted_and_disjoint() { + let mut allocator = ExtentAllocator::new(72, 100).unwrap(); + allocator.reserve_sorted(80, 4).unwrap(); + assert_eq!( + allocator.reserve_sorted(78, 1), + Err(ProviderError::Integrity) + ); + } + + #[test] + fn mixed_allocation_and_release_matches_a_block_ownership_model() { + const START: u64 = 72; + const BLOCKS: usize = 257; + let mut allocator = ExtentAllocator::new(START, START + BLOCKS as u64).unwrap(); + let mut used = [false; BLOCKS]; + let mut active = heapless::Vec::<(ExtentId, usize, usize), 128>::new(); + let mut random = 0x815_u64; + + for _ in 0..5_000 { + random = random + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1); + if !active.is_empty() && (random & 3) == 0 { + let index = (random as usize >> 8) % active.len(); + let (id, start, blocks) = active.swap_remove(index); + assert!(used[start..start + blocks].iter().all(|block| *block)); + used[start..start + blocks].fill(false); + allocator.release(id).unwrap(); + } else if !active.is_full() { + let blocks = ((random >> 16) as usize % 16) + 1; + match allocator.allocate(blocks as u32) { + Ok((id, returned)) => { + let (lba, actual) = allocator.range(id).unwrap(); + assert_eq!(lba, returned); + assert_eq!(actual as usize, blocks); + let start = usize::try_from(lba - START).unwrap(); + assert!(start + blocks <= BLOCKS); + assert!(used[start..start + blocks].iter().all(|block| !*block)); + used[start..start + blocks].fill(true); + active.push((id, start, blocks)).unwrap(); + } + Err(ProviderError::Exhausted) => assert!( + !used + .windows(blocks) + .any(|window| window.iter().all(|block| !*block)) + ), + Err(error) => panic!("unexpected allocation error: {error:?}"), + } + } + assert_eq!( + allocator.free_blocks(), + used.iter().filter(|block| !**block).count() as u64 + ); + } + + for (id, _, _) in active { + allocator.release(id).unwrap(); + } + let (whole, _) = allocator.allocate(BLOCKS as u32).unwrap(); + assert_eq!(allocator.range(whole).unwrap(), (START, BLOCKS as u32)); + } + + /// A handle that outlives its extent must refuse, even after its slot is + /// recycled by a new allocation — the silent-alias footgun the + /// generation tag exists to close. + #[test] + fn stale_extent_ids_are_detectably_stale() { + let mut allocator = ExtentAllocator::new(72, 100).unwrap(); + let (stale, lba) = allocator.allocate(4).unwrap(); + allocator.release(stale).unwrap(); + // Double release fails closed on the dead incarnation. + assert_eq!(allocator.release(stale), Err(ProviderError::Integrity)); + // The recycled slot serves a fresh incarnation at the same LBA... + let (fresh, again) = allocator.allocate(4).unwrap(); + assert_eq!(again, lba); + // ...which the stale handle can neither read nor free. + assert_eq!(allocator.range(stale), Err(ProviderError::Integrity)); + assert_eq!(allocator.release(stale), Err(ProviderError::Integrity)); + assert_eq!(allocator.range(fresh).unwrap(), (lba, 4)); + } +} diff --git a/native/services/dagfs/src/provider/mod.rs b/native/services/dagfs/src/provider/mod.rs new file mode 100644 index 0000000..655aee9 --- /dev/null +++ b/native/services/dagfs/src/provider/mod.rs @@ -0,0 +1,323 @@ +//! The storage-provider seam. +//! +//! This is the boundary that makes the same filesystem engine work over RAM +//! over RAM and an NVMe-backed direct block channel without any change to its +//! namespace, authority, or client protocol. It describes +//! **storage semantics, not device commands**: there is no NVMe, SATA, or +//! virtio vocabulary here, and there never will be — a block-backed provider +//! acquires a hardware-neutral affine channel from a control service once, +//! then submits and completes I/O directly without branching on which device +//! is underneath or re-entering that service on the data path. +//! +//! # The boundaries that may not move +//! +//! - [`StorageProvider::Key`] **identifies placement only.** It grants no +//! authority. Holding one lets you name bytes, not reach them: every +//! access check, name, link, and revision belongs to the engine above. +//! This is why the key type reaches into [`Object`](crate::object) as a +//! plain associated type rather than being an opaque capability. +//! - **The provider is async** so that a hardware or remote provider cannot +//! block an executor worker. The RAM provider never actually parks — its +//! futures are `Ready` on first poll — but the seam is shaped for the one +//! that will. +//! - **Providers report their limits and typed errors.** The engine asks +//! [`StorageProvider::limits`] rather than assuming a geometry, and +//! [`ProviderError`] is storage-shaped: a provider cannot say "not found" +//! about a name it has never seen. +//! +//! # `&self`, not `&mut self` +//! +//! Every method takes `&self`. A provider whose transfers are genuinely in +//! flight — a direct block channel with queued I/O — must be able to hold +//! several at once, and an `&mut self` seam would forbid that and then have to +//! break when concurrent provider work arrives. [`RamProvider`] pays for this +//! with an internal +//! [`RefCell`](core::cell::RefCell): its borrows are short, taken and +//! dropped inside a single method with no `await` held across one. +//! +//! # What the engine promises the provider +//! +//! The engine reserves capacity with [`StorageProvider::resize`] *before* it +//! writes any bytes. That is what makes a write request atomic: the only +//! failure that can run out of room happens before anything is visible, and +//! the subsequent [`StorageProvider::write`] into already-reserved space +//! cannot fail for want of space. A provider must honour that: after a +//! successful `resize` to `n`, a write inside `0..n` fails only for a real +//! medium reason. + +mod block; +mod extent; +mod ram; + +pub(crate) use block::map_block; +pub use block::{ + BlockKey, BlockProvider, BlockProviderStats, DurableExtent, GPT_DISCOVERY_END_LBA, + LEGACY_DATA_START_LBA, +}; +pub use ram::{Fault, RamKey, RamProvider}; + +use crate::{error::ProviderError, object::ObjectKind}; + +/// What a provider can do, in storage terms. +/// +/// The engine consults these instead of assuming a geometry, so a provider +/// with a different granularity or a smaller ceiling changes behaviour +/// without changing engine code. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct StorageLimits { + /// Total bytes the provider can place across all objects. + pub total_bytes: u64, + /// Largest single object the provider can hold. + pub max_object_bytes: u64, + /// Largest single transfer the provider will accept. + pub max_transfer: usize, + /// Allocation granularity in bytes. Reserving `n` bytes may consume up + /// to `n` rounded up to this. + pub granularity: usize, +} + +/// Where an object's bytes live, and how to move them. +/// +/// A new provider implements this trait and nothing else changes — no engine +/// branch, no new error class, no client-visible difference +/// (`kore-design-doctrine` §3: contract-as-trait, data-as-impl). +#[allow(async_fn_in_trait)] +pub trait StorageProvider { + /// A placement identifier. Conveys no authority; see the module docs. + /// + /// `Debug` is part of the contract because a placement appears in the + /// engine's diagnostics and in the assertions of the conformance body + /// below — a provider key is always a small value type, so requiring it + /// costs an impl nothing. + type Key: Copy + Eq + core::fmt::Debug; + + /// Whether placements and their bytes can be reconstructed after this + /// provider is restarted. Derived batches use the same placement seam as + /// file payloads, but the RAM provider must not waste its bounded pool on + /// a second copy that cannot survive a restart anyway. + const PERSISTS_ACROSS_RESTART: bool; + + /// The provider's declared limits. + fn limits(&self) -> StorageLimits; + + /// Reserve a placement of `len` bytes, reading as zeros. + /// + /// `kind` is informational — a provider may place a directory's metadata + /// differently from a file's bytes. The RAM provider treats both the + /// same, and the engine never allocates for a directory at all. + async fn allocate(&self, kind: ObjectKind, len: u64) -> Result; + + /// Read from `offset` into `dst`, returning the bytes transferred. + /// + /// A read that starts inside the object but runs past its end is short, + /// not an error — the count says how much was real. A read starting at + /// or past the end transfers zero. A read starting beyond the placement + /// is [`ProviderError::Range`]. + async fn read( + &self, + key: Self::Key, + offset: u64, + dst: &mut [u8], + ) -> Result; + + /// Write `src` at `offset`. All-or-nothing: a refused write leaves every + /// byte as it was. + /// + /// The engine has already reserved `offset + src.len()` bytes via + /// [`Self::resize`], so this fails only on a real medium error or a + /// caller bug. + async fn write(&self, key: Self::Key, offset: u64, src: &[u8]) -> Result<(), ProviderError>; + + /// Copy the first `len` bytes of `source` over the first `len` bytes of + /// `destination` — the snapshot a version lend freezes. Both placements + /// must hold at least `len` bytes; `destination` must not be committed. + /// + /// The default streams through one transfer-sized scratch buffer, which + /// is what the RAM provider needs and all the engine used to do. A + /// provider whose bytes never have to visit the CPU overrides it: the + /// block provider moves whole extents between its DMA buffers, so a + /// 9 MiB image costs a few hundred commands instead of a few thousand. + /// A refused copy leaves `destination` in an unspecified state below + /// `len`; the caller releases it. + async fn copy_object( + &self, + source: Self::Key, + destination: Self::Key, + len: u64, + ) -> Result<(), ProviderError> { + let mut buffer = [0u8; crate::bounds::MAX_TRANSFER]; + let chunk = self.limits().max_transfer.min(buffer.len()); + let mut offset = 0u64; + while offset < len { + let want = usize::try_from((len - offset).min(chunk as u64)) + .map_err(|_| ProviderError::Range)?; + let read = self.read(source, offset, &mut buffer[..want]).await?; + if read != want { + return Err(ProviderError::Media); + } + self.write(destination, offset, &buffer[..read]).await?; + offset += read as u64; + } + Ok(()) + } + /// Reserve or release capacity so the placement holds exactly `len` + /// bytes. Growth reads as zeros. A refused resize leaves the placement + /// unchanged. + async fn resize(&self, key: Self::Key, len: u64) -> Result<(), ProviderError>; + + /// Release the placement. The key is stale afterwards. + async fn release(&self, key: Self::Key) -> Result<(), ProviderError>; + + /// Cross the provider's durability contract for this placement. + /// + /// For RAM this is a no-op that reports success, because "visible to + /// later authorized operations in this boot" is the whole of RAM's + /// durability promise. A provider must never report durable + /// completion it cannot deliver. + async fn sync(&self, key: Self::Key) -> Result<(), ProviderError>; + + /// Where `key`'s bytes sit on the device namespace, for a read grant + /// (`res/STORAGE_TRANSPORT.md` §5): the blocks lying wholly inside the + /// placement's length. The partial last block is left out on purpose — + /// its tail past the length may still hold the file's own earlier + /// bytes (a shrink keeps them), which a byte-exact read never returns + /// and a block grant would. `None` when the provider has no device + /// namespace, which is the default. + fn device_blocks(&self, _key: Self::Key) -> Option { + None + } +} + +/// A placement's whole blocks in device-namespace coordinates +/// ([`StorageProvider::device_blocks`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DeviceBlocks { + /// The first block, absolute in the namespace. + pub lba: u64, + /// Blocks wholly inside the placement's length. + pub blocks: u64, + /// The device's block size. + pub block_size: u32, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::block_on; + + /// The conformance expectations every provider must meet, exercised here + /// against RAM and the direct block provider — which is the point of + /// having a seam at all, and why this + /// lives beside the trait rather than inside `ram`. + fn provider_conformance(provider: &P) { + let limits = provider.limits(); + assert!(limits.total_bytes > 0); + assert!(limits.max_object_bytes > 0); + assert!(limits.max_transfer > 0); + assert!(limits.granularity > 0); + assert!(limits.max_object_bytes <= limits.total_bytes); + + let key = block_on(provider.allocate(ObjectKind::File, 8)).expect("small allocate"); + + // Fresh capacity reads as zeros. + let mut buf = [0xffu8; 8]; + assert_eq!(block_on(provider.read(key, 0, &mut buf)), Ok(8)); + assert_eq!(buf, [0u8; 8]); + + // Write then read back. + assert_eq!(block_on(provider.write(key, 0, b"kore")), Ok(())); + let mut back = [0u8; 4]; + assert_eq!(block_on(provider.read(key, 0, &mut back)), Ok(4)); + assert_eq!(&back, b"kore"); + + // Reads at and past the end are short, not errors. + let mut tail = [0u8; 4]; + assert_eq!(block_on(provider.read(key, 6, &mut tail)), Ok(2)); + assert_eq!(block_on(provider.read(key, 8, &mut tail)), Ok(0)); + + // Sync must report success or a real reason — never silence. + assert_eq!(block_on(provider.sync(key)), Ok(())); + + // Grow, then the new region reads as zeros. + assert_eq!(block_on(provider.resize(key, 600)), Ok(())); + let mut grown = [0xabu8; 16]; + assert_eq!(block_on(provider.read(key, 584, &mut grown)), Ok(16)); + assert_eq!(grown, [0u8; 16]); + // The original bytes survived the growth. + assert_eq!(block_on(provider.read(key, 0, &mut back)), Ok(4)); + assert_eq!(&back, b"kore"); + + // Shrink, then the tail is gone. + // A snapshot copy carries exactly the first `len` bytes of the + // source, whichever path the provider takes. + let copy = block_on(provider.allocate(ObjectKind::File, 600)).expect("copy allocate"); + assert_eq!(block_on(provider.copy_object(key, copy, 600)), Ok(())); + let mut copied = [0u8; 4]; + assert_eq!(block_on(provider.read(copy, 0, &mut copied)), Ok(4)); + assert_eq!(&copied, b"kore"); + assert_eq!(block_on(provider.read(copy, 584, &mut grown)), Ok(16)); + assert_eq!(grown, [0u8; 16]); + assert_eq!(block_on(provider.release(copy)), Ok(())); + + assert_eq!(block_on(provider.resize(key, 4)), Ok(())); + assert_eq!(block_on(provider.read(key, 4, &mut tail)), Ok(0)); + + // An over-sized object is refused by declared limit, not by crash. + assert_eq!( + block_on(provider.allocate(ObjectKind::File, limits.max_object_bytes + 1)), + Err(ProviderError::TooLarge) + ); + + // Release makes the key stale; every op on it fails closed. + assert_eq!(block_on(provider.release(key)), Ok(())); + let mut after = [0u8; 4]; + assert_eq!( + block_on(provider.read(key, 0, &mut after)), + Err(ProviderError::UnknownKey) + ); + assert_eq!( + block_on(provider.write(key, 0, b"x")), + Err(ProviderError::UnknownKey) + ); + assert_eq!( + block_on(provider.resize(key, 1)), + Err(ProviderError::UnknownKey) + ); + assert_eq!( + block_on(provider.release(key)), + Err(ProviderError::UnknownKey) + ); + assert_eq!(block_on(provider.sync(key)), Err(ProviderError::UnknownKey)); + } + + #[test] + fn ram_provider_meets_the_contract() { + provider_conformance(&RamProvider::new()); + } + + #[test] + fn block_provider_meets_the_same_contract() { + // The in-memory device fixture (128 KiB) and the provider's placement + // arenas (sized to MAX_BLOCK_PLACEMENTS) are constructed by value, which + // exceeds a default test-thread stack. This is a TEST-fixture concern + // only: on metal the durable service is constructed once and moved into + // its named heap window (the `size_of::()` guard bounds + // that), never held on a bounded stack. Give the fixture a generous + // stack rather than shrinking a bound that is a correct worst case + // (MAX_OBJECTS file extents + MAX_DERIVED_RESULTS durable batches). + std::thread::Builder::new() + .stack_size(8 << 20) + .spawn(|| { + use kore_driver::block::{BlockChannel, CompletionMode, memory::MemoryBlockDevice}; + type Device = MemoryBlockDevice<131072, 8192>; + let device = Device::new(512, 16).unwrap(); + let channel: BlockChannel = + BlockChannel::new(device, CompletionMode::Poll).unwrap(); + let provider = BlockProvider::new(channel).unwrap(); + provider_conformance(&provider); + }) + .unwrap() + .join() + .unwrap(); + } +} diff --git a/native/services/dagfs/src/provider/ram.rs b/native/services/dagfs/src/provider/ram.rs new file mode 100644 index 0000000..912e22a --- /dev/null +++ b/native/services/dagfs/src/provider/ram.rs @@ -0,0 +1,801 @@ +//! The bounded RAM [`StorageProvider`]. +//! +//! This is not a mock. It is the architecture-neutral fallback composition, +//! and it drives the same engine, namespace, and authority model as the +//! block-backed provider. Its +//! durability promise is exactly what RAM can deliver: a successful mutation +//! is immediately visible to later authorized operations in this boot, and +//! everything is lost at reboot. It never reports more. +//! +//! # Shape +//! +//! A fixed pool of [`RAM_CHUNK_BYTES`]-sized chunks, claimed and released +//! through a per-chunk claim array — the same shape as the kernel's +//! `PhysRegion` bitmap allocation (`kore-design-doctrine` §1), one byte per +//! slot for the same reason `GenIdAllocator` uses one: a simpler claim path +//! and no summary bitmap to keep coherent. Each placement owns an ordered +//! list of chunk indices, so a file's bytes need not be contiguous and +//! growth does not have to relocate anything. +//! +//! Chunks are zeroed **when claimed**, and a shrink zeroes the tail of the +//! partial chunk it keeps. Together those two rules are what make the +//! provider's promise — reserved-but-unwritten capacity reads as zeros — +//! hold even across a shrink-then-grow that reuses the same chunk. +//! +//! # Fault injection +//! +//! [`RamProvider::inject`] arms a one-shot failure for a chosen operation. +//! It is deliberately part of the ordinary API rather than a `cfg(test)` +//! seam: provider conformance requires proof that injected failures leave +//! namespace and object metadata consistent, so producing a provider +//! failure is a capability the test suites need, and RAM has no other way to +//! fail. + +use core::cell::RefCell; + +use crate::{ + bounds::{MAX_CHUNKS_PER_OBJECT, MAX_PLACEMENTS, RAM_CHUNK_BYTES, RAM_CHUNKS}, + error::ProviderError, + object::ObjectKind, + provider::{StorageLimits, StorageProvider}, + slot::{SlotKey, SlotTable}, +}; + +/// A RAM placement's identity. A stale one fails closed with +/// [`ProviderError::UnknownKey`]; it conveys no authority either way. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct RamKey(SlotKey); + +/// Which operation a one-shot injected failure applies to, and what it +/// reports. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Fault { + Allocate(ProviderError), + Read(ProviderError), + Write(ProviderError), + Resize(ProviderError), + Release(ProviderError), + Sync(ProviderError), +} + +/// Which operation is being attempted — matched against an armed [`Fault`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Op { + Allocate, + Read, + Write, + Resize, + Release, + Sync, +} + +impl Fault { + fn matches(self, op: Op) -> Option { + match (self, op) { + (Fault::Allocate(e), Op::Allocate) + | (Fault::Read(e), Op::Read) + | (Fault::Write(e), Op::Write) + | (Fault::Resize(e), Op::Resize) + | (Fault::Release(e), Op::Release) + | (Fault::Sync(e), Op::Sync) => Some(e), + _ => None, + } + } +} + +/// One object's bytes: an ordered chunk list plus the reserved length. +struct Placement { + chunks: heapless::Vec, + len: u64, +} + +/// Total bytes the pool can hold. +const TOTAL_BYTES: u64 = (RAM_CHUNKS * RAM_CHUNK_BYTES) as u64; + +/// Largest object one placement can hold. +const MAX_OBJECT_BYTES: u64 = (MAX_CHUNKS_PER_OBJECT * RAM_CHUNK_BYTES) as u64; + +struct RamState { + pool: [[u8; RAM_CHUNK_BYTES]; RAM_CHUNKS], + claimed: [bool; RAM_CHUNKS], + claimed_count: usize, + placements: SlotTable, + fault: Option, +} + +/// Chunks needed to hold `len` bytes, or `None` on overflow. +fn chunks_for(len: u64) -> Option { + let granularity = RAM_CHUNK_BYTES as u64; + let chunks = len.checked_add(granularity - 1)? / granularity; + usize::try_from(chunks).ok() +} + +impl RamState { + fn new() -> Self { + Self { + pool: [[0u8; RAM_CHUNK_BYTES]; RAM_CHUNKS], + claimed: [false; RAM_CHUNKS], + claimed_count: 0, + placements: SlotTable::new(), + fault: None, + } + } + + /// Consume an armed fault if it names `op`. + fn take_fault(&mut self, op: Op) -> Option { + let error = self.fault?.matches(op)?; + self.fault = None; + Some(error) + } + + /// Claim one free chunk, zeroing it so reserved capacity reads as zeros. + fn claim(&mut self) -> Option { + let index = self.claimed.iter().position(|used| !*used)?; + self.claimed[index] = true; + self.claimed_count += 1; + self.pool[index].fill(0); + u16::try_from(index).ok() + } + + fn release_chunk(&mut self, index: u16) { + let index = index as usize; + if index < RAM_CHUNKS && self.claimed[index] { + self.claimed[index] = false; + self.claimed_count -= 1; + } + } + + /// Grow or shrink a placement to exactly `len` bytes. A refusal leaves + /// the placement byte-for-byte unchanged. + fn resize_placement(&mut self, key: RamKey, len: u64) -> Result<(), ProviderError> { + if len > MAX_OBJECT_BYTES { + return Err(ProviderError::TooLarge); + } + let want = chunks_for(len).ok_or(ProviderError::Range)?; + if want > MAX_CHUNKS_PER_OBJECT { + return Err(ProviderError::TooLarge); + } + let have = self + .placements + .get(key.0) + .ok_or(ProviderError::UnknownKey)? + .chunks + .len(); + + if want > have { + // Claim into a scratch list first: if the pool runs out partway, + // every chunk we took goes back and the placement is untouched. + let mut fresh: heapless::Vec = heapless::Vec::new(); + for _ in have..want { + match self.claim() { + Some(index) => { + if fresh.push(index).is_err() { + self.release_chunk(index); + for taken in fresh.iter() { + self.release_chunk(*taken); + } + return Err(ProviderError::TooLarge); + } + } + None => { + for taken in fresh.iter() { + self.release_chunk(*taken); + } + return Err(ProviderError::Exhausted); + } + } + } + let mut placement = self + .placements + .get_mut(key.0) + .ok_or(ProviderError::UnknownKey)?; + for index in fresh.iter() { + // Cannot fail: `want <= MAX_CHUNKS_PER_OBJECT` and the list + // held `have` before this loop. + if placement.chunks.push(*index).is_err() { + return Err(ProviderError::TooLarge); + } + } + } else if want < have { + let mut dropped: heapless::Vec = heapless::Vec::new(); + { + let mut placement = self + .placements + .get_mut(key.0) + .ok_or(ProviderError::UnknownKey)?; + while placement.chunks.len() > want { + if let Some(index) = placement.chunks.pop() { + let _ = dropped.push(index); + } + } + } + for index in dropped.iter() { + self.release_chunk(*index); + } + } + + // Zero the tail of the retained partial chunk, so a later grow back + // over this region reads as zeros rather than as stale bytes. + let tail = (len % RAM_CHUNK_BYTES as u64) as usize; + if tail != 0 { + let last = { + let placement = self + .placements + .get(key.0) + .ok_or(ProviderError::UnknownKey)?; + placement.chunks.last().copied() + }; + if let Some(index) = last { + let index = index as usize; + if index < RAM_CHUNKS { + self.pool[index][tail..].fill(0); + } + } + } + + let mut placement = self + .placements + .get_mut(key.0) + .ok_or(ProviderError::UnknownKey)?; + placement.len = len; + Ok(()) + } + + /// Copy out of a placement, returning the bytes transferred. + fn copy_out(&self, key: RamKey, offset: u64, dst: &mut [u8]) -> Result { + let placement = self + .placements + .get(key.0) + .ok_or(ProviderError::UnknownKey)?; + // Every offset/length computation is checked: a wrapping add here + // would turn an out-of-range read into an in-range one. + offset + .checked_add(dst.len() as u64) + .ok_or(ProviderError::Range)?; + if offset > placement.len { + return Err(ProviderError::Range); + } + let available = placement.len - offset; + let want = (dst.len() as u64).min(available) as usize; + + let mut done = 0usize; + while done < want { + let at = offset + done as u64; + let chunk_slot = (at / RAM_CHUNK_BYTES as u64) as usize; + let within = (at % RAM_CHUNK_BYTES as u64) as usize; + let index = *placement + .chunks + .get(chunk_slot) + .ok_or(ProviderError::Range)? as usize; + if index >= RAM_CHUNKS { + return Err(ProviderError::Range); + } + let take = (RAM_CHUNK_BYTES - within).min(want - done); + // Slice copy, not a byte loop (kore-design-doctrine §8). + dst[done..done + take].copy_from_slice(&self.pool[index][within..within + take]); + done += take; + } + Ok(done) + } + + /// Copy into a placement. Refuses before touching a byte if the range + /// is not fully reserved, so a write is all-or-nothing. + fn copy_in(&mut self, key: RamKey, offset: u64, src: &[u8]) -> Result<(), ProviderError> { + let end = offset + .checked_add(src.len() as u64) + .ok_or(ProviderError::Range)?; + // Split the two disjoint state fields instead of cloning the whole + // per-object chunk map on every write. The placement is immutable for + // this operation; only the already-reserved pool bytes are mutated. + let (placements, pool) = (&self.placements, &mut self.pool); + let placement = placements.get(key.0).ok_or(ProviderError::UnknownKey)?; + if end > placement.len { + // The engine reserves before writing; reaching here means it did + // not, and writing part of the range would be worse. + return Err(ProviderError::Range); + } + + let mut done = 0usize; + while done < src.len() { + let at = offset + done as u64; + let chunk_slot = (at / RAM_CHUNK_BYTES as u64) as usize; + let within = (at % RAM_CHUNK_BYTES as u64) as usize; + let index = *placement + .chunks + .get(chunk_slot) + .ok_or(ProviderError::Range)? as usize; + if index >= RAM_CHUNKS { + return Err(ProviderError::Range); + } + let take = (RAM_CHUNK_BYTES - within).min(src.len() - done); + pool[index][within..within + take].copy_from_slice(&src[done..done + take]); + done += take; + } + Ok(()) + } +} + +/// The bounded in-memory provider. +pub struct RamProvider { + state: RefCell, +} + +impl RamProvider { + pub fn new() -> Self { + Self { + state: RefCell::new(RamState::new()), + } + } + + /// Arm a one-shot failure for the next matching operation. See the + /// module docs on why this is ordinary API. + pub fn inject(&self, fault: Fault) { + self.state.borrow_mut().fault = Some(fault); + } + + /// Disarm any pending injected failure. + pub fn clear_fault(&self) { + self.state.borrow_mut().fault = None; + } + + /// Chunks currently claimed. The observable that makes "released + /// storage exactly once" testable. + pub fn chunks_in_use(&self) -> usize { + self.state.borrow().claimed_count + } + + /// Live placements. + pub fn placements(&self) -> usize { + self.state.borrow().placements.len() + } + + /// Bytes currently claimed, rounded to the allocation granularity. + pub fn bytes_in_use(&self) -> u64 { + (self.chunks_in_use() * RAM_CHUNK_BYTES) as u64 + } +} + +impl Default for RamProvider { + fn default() -> Self { + Self::new() + } +} + +impl StorageProvider for RamProvider { + type Key = RamKey; + const PERSISTS_ACROSS_RESTART: bool = false; + + fn limits(&self) -> StorageLimits { + StorageLimits { + total_bytes: TOTAL_BYTES, + max_object_bytes: MAX_OBJECT_BYTES, + max_transfer: crate::bounds::MAX_TRANSFER, + granularity: RAM_CHUNK_BYTES, + } + } + + async fn allocate(&self, _kind: ObjectKind, len: u64) -> Result { + let mut state = self.state.borrow_mut(); + if let Some(error) = state.take_fault(Op::Allocate) { + return Err(error); + } + if len > MAX_OBJECT_BYTES { + return Err(ProviderError::TooLarge); + } + let key = state + .placements + .insert(Placement { + chunks: heapless::Vec::new(), + len: 0, + }) + .map(RamKey) + .map_err(|_| ProviderError::Exhausted)?; + // Reserve the requested capacity; on refusal the empty placement goes + // back so a failed allocate leaves no slot behind. + if let Err(error) = state.resize_placement(key, len) { + state.placements.remove(key.0); + return Err(error); + } + Ok(key) + } + + async fn read(&self, key: RamKey, offset: u64, dst: &mut [u8]) -> Result { + let mut state = self.state.borrow_mut(); + if let Some(error) = state.take_fault(Op::Read) { + return Err(error); + } + if dst.len() > crate::bounds::MAX_TRANSFER { + return Err(ProviderError::TooLarge); + } + state.copy_out(key, offset, dst) + } + + async fn write(&self, key: RamKey, offset: u64, src: &[u8]) -> Result<(), ProviderError> { + let mut state = self.state.borrow_mut(); + if let Some(error) = state.take_fault(Op::Write) { + return Err(error); + } + if src.len() > crate::bounds::MAX_TRANSFER { + return Err(ProviderError::TooLarge); + } + state.copy_in(key, offset, src) + } + + async fn resize(&self, key: RamKey, len: u64) -> Result<(), ProviderError> { + let mut state = self.state.borrow_mut(); + if let Some(error) = state.take_fault(Op::Resize) { + return Err(error); + } + if state.placements.get(key.0).is_none() { + return Err(ProviderError::UnknownKey); + } + state.resize_placement(key, len) + } + + async fn release(&self, key: RamKey) -> Result<(), ProviderError> { + let mut state = self.state.borrow_mut(); + if let Some(error) = state.take_fault(Op::Release) { + return Err(error); + } + let placement = state + .placements + .remove(key.0) + .ok_or(ProviderError::UnknownKey)?; + for index in placement.chunks.iter() { + state.release_chunk(*index); + } + Ok(()) + } + + async fn sync(&self, key: RamKey) -> Result<(), ProviderError> { + let mut state = self.state.borrow_mut(); + if let Some(error) = state.take_fault(Op::Sync) { + return Err(error); + } + // RAM's durability contract is "visible in this boot", which a + // successful mutation already satisfied. Validating the key keeps + // sync from reporting success about a placement that is gone. + if state.placements.get(key.0).is_none() { + return Err(ProviderError::UnknownKey); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::block_on; + + #[test] + fn chunks_for_rounds_up_and_refuses_overflow() { + assert_eq!(chunks_for(0), Some(0)); + assert_eq!(chunks_for(1), Some(1)); + assert_eq!(chunks_for(RAM_CHUNK_BYTES as u64), Some(1)); + assert_eq!(chunks_for(RAM_CHUNK_BYTES as u64 + 1), Some(2)); + assert_eq!(chunks_for(u64::MAX), None); + } + + #[test] + fn a_fresh_provider_holds_nothing() { + let provider = RamProvider::new(); + assert_eq!(provider.chunks_in_use(), 0); + assert_eq!(provider.placements(), 0); + assert_eq!(provider.bytes_in_use(), 0); + } + + #[test] + fn allocate_reserves_granular_capacity() { + let provider = RamProvider::new(); + let key = block_on(provider.allocate(ObjectKind::File, 1)).unwrap(); + // One byte still costs one chunk — the declared granularity. + assert_eq!(provider.chunks_in_use(), 1); + assert_eq!(provider.placements(), 1); + block_on(provider.release(key)).unwrap(); + assert_eq!(provider.chunks_in_use(), 0); + assert_eq!(provider.placements(), 0); + } + + #[test] + fn a_zero_length_object_costs_no_chunks() { + let provider = RamProvider::new(); + let key = block_on(provider.allocate(ObjectKind::File, 0)).unwrap(); + assert_eq!(provider.chunks_in_use(), 0); + let mut buf = [0u8; 1]; + assert_eq!(block_on(provider.read(key, 0, &mut buf)), Ok(0)); + } + + #[test] + fn writes_and_reads_cross_chunk_boundaries() { + let provider = RamProvider::new(); + let len = RAM_CHUNK_BYTES * 2 + 5; + let key = block_on(provider.allocate(ObjectKind::File, len as u64)).unwrap(); + + let mut src = [0u8; 600]; + let src = &mut src[..len]; + for (i, byte) in src.iter_mut().enumerate() { + *byte = (i % 251) as u8; + } + // Start deliberately mid-chunk so the transfer straddles three chunks. + assert_eq!(block_on(provider.write(key, 7, &src[..len - 7])), Ok(())); + + let mut back = [0u8; 600]; + let back = &mut back[..len]; + assert_eq!(block_on(provider.read(key, 0, back)), Ok(len)); + assert_eq!(&back[..7], &[0u8; 7], "untouched head stayed zero"); + assert_eq!(&back[7..], &src[..len - 7]); + } + + #[test] + fn writing_past_reserved_capacity_is_refused_without_touching_bytes() { + let provider = RamProvider::new(); + let key = block_on(provider.allocate(ObjectKind::File, 4)).unwrap(); + block_on(provider.write(key, 0, b"abcd")).unwrap(); + + assert_eq!( + block_on(provider.write(key, 2, b"xyz")), + Err(ProviderError::Range) + ); + // The refused write must not have written its in-range prefix. + let mut back = [0u8; 4]; + assert_eq!(block_on(provider.read(key, 0, &mut back)), Ok(4)); + assert_eq!(&back, b"abcd"); + } + + #[test] + fn offset_length_overflow_is_refused() { + let provider = RamProvider::new(); + let key = block_on(provider.allocate(ObjectKind::File, 4)).unwrap(); + let mut buf = [0u8; 8]; + assert_eq!( + block_on(provider.read(key, u64::MAX, &mut buf)), + Err(ProviderError::Range) + ); + assert_eq!( + block_on(provider.write(key, u64::MAX, b"x")), + Err(ProviderError::Range) + ); + } + + #[test] + fn shrink_then_grow_reads_as_zeros_not_stale_bytes() { + let provider = RamProvider::new(); + let key = block_on(provider.allocate(ObjectKind::File, 300)).unwrap(); + let payload = [0xabu8; 300]; + block_on(provider.write(key, 0, &payload)).unwrap(); + + // Shrink inside the first chunk, then grow back over the same bytes. + block_on(provider.resize(key, 100)).unwrap(); + block_on(provider.resize(key, 300)).unwrap(); + + let mut back = [0xffu8; 300]; + assert_eq!(block_on(provider.read(key, 0, &mut back)), Ok(300)); + assert_eq!(&back[..100], &payload[..100], "retained bytes survive"); + assert!( + back[100..].iter().all(|b| *b == 0), + "regrown capacity must read as zeros, not as the old bytes" + ); + } + + #[test] + fn shrink_returns_chunks_to_the_pool() { + let provider = RamProvider::new(); + let key = + block_on(provider.allocate(ObjectKind::File, 4 * RAM_CHUNK_BYTES as u64)).unwrap(); + assert_eq!(provider.chunks_in_use(), 4); + block_on(provider.resize(key, RAM_CHUNK_BYTES as u64)).unwrap(); + assert_eq!(provider.chunks_in_use(), 1); + } + + #[test] + fn an_object_larger_than_the_per_object_ceiling_is_refused() { + let provider = RamProvider::new(); + assert_eq!( + block_on(provider.allocate(ObjectKind::File, MAX_OBJECT_BYTES + 1)), + Err(ProviderError::TooLarge) + ); + // Nothing was consumed by the refusal. + assert_eq!(provider.chunks_in_use(), 0); + assert_eq!(provider.placements(), 0); + + let key = block_on(provider.allocate(ObjectKind::File, 1)).unwrap(); + assert_eq!( + block_on(provider.resize(key, MAX_OBJECT_BYTES + 1)), + Err(ProviderError::TooLarge) + ); + // The refused resize left the original capacity intact. + assert_eq!(provider.chunks_in_use(), 1); + } + + #[test] + fn pool_exhaustion_refuses_and_leaves_the_placement_unchanged() { + let provider = RamProvider::new(); + // Fill the pool with full-size objects. + let mut keys = std::vec::Vec::new(); + while let Ok(key) = block_on(provider.allocate(ObjectKind::File, MAX_OBJECT_BYTES)) { + keys.push(key); + } + assert_eq!(provider.chunks_in_use(), RAM_CHUNKS); + + // A further allocation is refused, not satisfied partially. + assert_eq!( + block_on(provider.allocate(ObjectKind::File, RAM_CHUNK_BYTES as u64)), + Err(ProviderError::Exhausted) + ); + assert_eq!(provider.chunks_in_use(), RAM_CHUNKS); + + // Now set up a grow that can claim *some* chunks and then run out, + // which is the path that has to roll back. Free most of one object, + // then hand nearly all of that room to someone else, leaving a few + // chunks — fewer than the regrow needs. + let victim = keys[0]; + block_on(provider.resize(victim, RAM_CHUNK_BYTES as u64)).unwrap(); + let spare = RAM_CHUNKS - provider.chunks_in_use(); + let filler_chunks = spare - 3; + block_on(provider.allocate(ObjectKind::File, (filler_chunks * RAM_CHUNK_BYTES) as u64)) + .unwrap(); + let before = provider.chunks_in_use(); + assert_eq!(RAM_CHUNKS - before, 3, "exactly three chunks left free"); + + // The regrow needs far more than three, so it claims what it can and + // must then put every one of them back. + assert_eq!( + block_on(provider.resize(victim, MAX_OBJECT_BYTES)), + Err(ProviderError::Exhausted) + ); + assert_eq!( + provider.chunks_in_use(), + before, + "a refused grow must return every chunk it claimed on the way" + ); + // And the victim still holds its pre-refusal capacity and bytes. + let mut buf = [0u8; 8]; + assert_eq!(block_on(provider.read(victim, 0, &mut buf)), Ok(8)); + assert_eq!( + block_on(provider.read(victim, RAM_CHUNK_BYTES as u64, &mut buf)), + Ok(0), + "capacity is unchanged at one chunk" + ); + } + + #[test] + fn placement_table_exhaustion_is_typed() { + let provider = RamProvider::new(); + // Zero-length objects consume a placement slot but no chunks, so the + // table is what runs out first. + for _ in 0..MAX_PLACEMENTS { + block_on(provider.allocate(ObjectKind::File, 0)).unwrap(); + } + assert_eq!(provider.placements(), MAX_PLACEMENTS); + assert_eq!( + block_on(provider.allocate(ObjectKind::File, 0)), + Err(ProviderError::Exhausted) + ); + } + + #[test] + fn transfers_larger_than_the_declared_maximum_are_refused() { + let provider = RamProvider::new(); + let key = block_on(provider.allocate(ObjectKind::File, 16)).unwrap(); + let oversized = [0u8; crate::bounds::MAX_TRANSFER + 1]; + assert_eq!( + block_on(provider.write(key, 0, &oversized)), + Err(ProviderError::TooLarge) + ); + let mut dst = [0u8; crate::bounds::MAX_TRANSFER + 1]; + assert_eq!( + block_on(provider.read(key, 0, &mut dst)), + Err(ProviderError::TooLarge) + ); + } + + #[test] + fn placements_are_isolated_from_each_other() { + let provider = RamProvider::new(); + let a = block_on(provider.allocate(ObjectKind::File, 8)).unwrap(); + let b = block_on(provider.allocate(ObjectKind::File, 8)).unwrap(); + assert_ne!(a, b); + block_on(provider.write(a, 0, b"aaaaaaaa")).unwrap(); + block_on(provider.write(b, 0, b"bbbbbbbb")).unwrap(); + + let mut back = [0u8; 8]; + assert_eq!(block_on(provider.read(a, 0, &mut back)), Ok(8)); + assert_eq!(&back, b"aaaaaaaa"); + assert_eq!(block_on(provider.read(b, 0, &mut back)), Ok(8)); + assert_eq!(&back, b"bbbbbbbb"); + + // Releasing one leaves the other's bytes intact. + block_on(provider.release(a)).unwrap(); + assert_eq!(block_on(provider.read(b, 0, &mut back)), Ok(8)); + assert_eq!(&back, b"bbbbbbbb"); + } + + #[test] + fn a_reused_placement_slot_rejects_the_stale_key() { + let provider = RamProvider::new(); + let stale = block_on(provider.allocate(ObjectKind::File, 8)).unwrap(); + block_on(provider.write(stale, 0, b"secrets!")).unwrap(); + block_on(provider.release(stale)).unwrap(); + + let fresh = block_on(provider.allocate(ObjectKind::File, 8)).unwrap(); + // The generational key is what stops a freed placement from being + // readable through the old key after the slot is reused. + assert_ne!(fresh, stale); + let mut buf = [0u8; 8]; + assert_eq!( + block_on(provider.read(stale, 0, &mut buf)), + Err(ProviderError::UnknownKey) + ); + // And the reused placement does not expose the old bytes. + assert_eq!(block_on(provider.read(fresh, 0, &mut buf)), Ok(8)); + assert_eq!(&buf, &[0u8; 8]); + } + + #[test] + fn injected_faults_are_one_shot_and_op_specific() { + let provider = RamProvider::new(); + let key = block_on(provider.allocate(ObjectKind::File, 8)).unwrap(); + + provider.inject(Fault::Write(ProviderError::Media)); + // A read must not consume a write's fault. + let mut buf = [0u8; 8]; + assert_eq!(block_on(provider.read(key, 0, &mut buf)), Ok(8)); + assert_eq!( + block_on(provider.write(key, 0, b"xxxx")), + Err(ProviderError::Media) + ); + // Armed once: the next write succeeds. + assert_eq!(block_on(provider.write(key, 0, b"xxxx")), Ok(())); + } + + #[test] + fn an_injected_write_failure_leaves_bytes_untouched() { + let provider = RamProvider::new(); + let key = block_on(provider.allocate(ObjectKind::File, 8)).unwrap(); + block_on(provider.write(key, 0, b"original")).unwrap(); + + provider.inject(Fault::Write(ProviderError::Media)); + assert_eq!( + block_on(provider.write(key, 0, b"replaced")), + Err(ProviderError::Media) + ); + let mut back = [0u8; 8]; + assert_eq!(block_on(provider.read(key, 0, &mut back)), Ok(8)); + assert_eq!(&back, b"original"); + } + + #[test] + fn an_injected_release_failure_keeps_the_placement() { + let provider = RamProvider::new(); + let key = block_on(provider.allocate(ObjectKind::File, 8)).unwrap(); + provider.inject(Fault::Release(ProviderError::Media)); + assert_eq!(block_on(provider.release(key)), Err(ProviderError::Media)); + // The chunks are still held — which is exactly the leak the engine + // has to account for rather than pretend away. + assert_eq!(provider.placements(), 1); + assert_eq!(provider.chunks_in_use(), 1); + provider.clear_fault(); + assert_eq!(block_on(provider.release(key)), Ok(())); + assert_eq!(provider.chunks_in_use(), 0); + } + + #[test] + fn clear_fault_disarms() { + let provider = RamProvider::new(); + provider.inject(Fault::Allocate(ProviderError::Media)); + provider.clear_fault(); + assert!(block_on(provider.allocate(ObjectKind::File, 4)).is_ok()); + } + + #[test] + fn declared_limits_describe_the_real_pool() { + let provider = RamProvider::new(); + let limits = provider.limits(); + assert_eq!(limits.total_bytes, TOTAL_BYTES); + assert_eq!(limits.max_object_bytes, MAX_OBJECT_BYTES); + assert_eq!(limits.granularity, RAM_CHUNK_BYTES); + // A caller sizing work by the declared maximum must actually be able + // to allocate it. + assert!(block_on(provider.allocate(ObjectKind::File, limits.max_object_bytes)).is_ok()); + } +} diff --git a/native/services/dagfs/src/reach.rs b/native/services/dagfs/src/reach.rs new file mode 100644 index 0000000..90f77f9 --- /dev/null +++ b/native/services/dagfs/src/reach.rs @@ -0,0 +1,112 @@ +//! The reachability walk's visited set: one bit per directory-arena slot. +//! +//! The cycle proof before a directory edge is published walks the +//! directory subgraph and must know, per node, whether it has been seen. +//! A visited *list* answers that in O(n) per query and O(n²) per walk; a +//! bit row answers it in O(1) and iterates in O(popcount). The row is keyed +//! by the directory's arena slot index, which is exactly the thing that is +//! bounded — [`MAX_DIRECTORIES`] — so the set's capacity is the arena's and +//! an index at or past it is refused, never truncated. +//! +//! Walk-local by construction: a `ReachSet` lives on the stack of the one +//! walk that made it and is never a static. Scalar words on purpose — the +//! house ruling in `kore_collections::bitmat` is scalar-first with +//! auto-vectorisation, and a row of one `u128` has nothing for a SIMD lane +//! to do; if the arena ever grows to where a kernel is measured to matter, +//! it goes into `bitmat`, not here. + +use kore_collections::BitSet; + +use crate::{ + bounds::MAX_DIRECTORIES, + error::{FsError, FsResult}, +}; + +/// Storage words for the directory arena: `BitSet` words are 128 bits. +pub(crate) const REACH_WORDS: usize = MAX_DIRECTORIES.div_ceil(128); + +/// A walk-local set of directory-arena slot indices. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct ReachSet(BitSet); + +impl ReachSet { + /// The logical bound: the directory arena, not the word row's slack. + pub(crate) const CAPACITY: usize = MAX_DIRECTORIES; + /// The row's footprint in bytes — `bounds` pins it. + pub(crate) const BYTES: usize = REACH_WORDS * 16; + + pub(crate) const fn new() -> Self { + Self(BitSet::new()) + } + + /// O(1). An index beyond the arena is simply not a member. + pub(crate) const fn contains(&self, index: usize) -> bool { + index < Self::CAPACITY && self.0.contains(index) + } + + /// Mark `index`; `Ok(true)` when it was not yet a member. An index at or + /// past the arena refuses [`FsError::Range`]: the walk cannot describe a + /// directory the arena cannot hold, and silently dropping it would let a + /// cycle through that directory go unproven. + pub(crate) fn insert(&mut self, index: usize) -> FsResult { + if index >= Self::CAPACITY { + return Err(FsError::Range); + } + Ok(self.0.set(index)) + } + + /// Members marked so far. + pub(crate) fn len(&self) -> usize { + self.0.count_ones() + } + + /// Members in ascending index order, one step per set bit. + pub(crate) fn iter(&self) -> impl Iterator + '_ { + self.0.iter() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn membership_is_exact_and_insert_reports_novelty() { + let mut set = ReachSet::new(); + assert!(!set.contains(3)); + assert_eq!(set.insert(3), Ok(true)); + assert!(set.contains(3)); + assert_eq!(set.insert(3), Ok(false)); + assert_eq!(set.len(), 1); + } + + #[test] + fn iteration_is_ascending_and_costs_one_step_per_set_bit() { + let mut set = ReachSet::new(); + for index in [9, 0, MAX_DIRECTORIES - 1, 4] { + set.insert(index).unwrap(); + } + let seen: std::vec::Vec = set.iter().collect(); + assert_eq!(seen, [0, 4, 9, MAX_DIRECTORIES - 1]); + assert_eq!(seen.len(), set.len()); + } + + #[test] + fn an_index_beyond_the_directory_arena_is_refused_not_truncated() { + let mut set = ReachSet::new(); + assert_eq!(set.insert(MAX_DIRECTORIES), Err(FsError::Range)); + assert!(!set.contains(MAX_DIRECTORIES)); + // The row has slack past the arena (128 bits per word); the bound is + // the arena's, not the row's. + assert_eq!(set.insert(ReachSet::CAPACITY + 1), Err(FsError::Range)); + assert_eq!(set.len(), 0); + } + + #[test] + fn footprint_is_one_word_per_128_directories() { + assert_eq!(ReachSet::BYTES, REACH_WORDS * 16); + assert_eq!(ReachSet::CAPACITY, MAX_DIRECTORIES); + assert!(BitSet::::BITS >= MAX_DIRECTORIES); + assert_eq!(core::mem::size_of::(), ReachSet::BYTES); + } +} diff --git a/native/services/dagfs/src/router.rs b/native/services/dagfs/src/router.rs new file mode 100644 index 0000000..8540f77 --- /dev/null +++ b/native/services/dagfs/src/router.rs @@ -0,0 +1,1069 @@ +//! The two-volume filesystem router. +//! +//! A [`Handle`], [`GrantId`], or [`VersionLease`] is a generational slot name +//! inside exactly one [`FsService`](crate::FsService). Once the durable row +//! owns both the conservative user volume and the rebuildable system volume, +//! those otherwise identical words need one bit of routing identity. This +//! module owns that bit and nothing below it learns about the composition: +//! both services receive their original untagged words and run the same +//! authority checks as a one-volume service. +//! +//! The tag is the high bit of `SlotId::index`. Every filesystem slot arena is +//! statically bounded far below `2^31`; preserving all 32 generation bits is +//! important because generation retirement is what makes stale authority fail +//! closed. Flipping the tag is not an authority operation: the selected +//! engine still checks the pair-derived session against the named record. + +use kore_abi::svc::{FsGrantEntryWire, FsSystemStatusWire}; +use zerocopy::FromBytes; + +use crate::{FsError, FsRequest, FsResponse, GrantId, Handle, VersionLease, window::DataWindow}; + +/// High bit of the packed `SlotId::index` word. +const SYSTEM_TAG: u64 = 1 << 63; + +// `SlotId` packs its 32-bit index into the high word. Reserve that word's +// high bit for composition, and make a future arena expansion fail at build +// time instead of silently aliasing a native slot into the system engine. +const NATIVE_INDEX_LIMIT: usize = 1 << 31; +const _: () = assert!(crate::authority::MAX_HANDLE_INDEX < NATIVE_INDEX_LIMIT); +const _: () = assert!(crate::authority::MAX_GRANT_INDEX < NATIVE_INDEX_LIMIT); +const _: () = assert!(crate::bounds::MAX_LENT_VERSIONS < NATIVE_INDEX_LIMIT); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Store { + User, + System, +} + +/// A request after every opaque id has been stripped back to its native +/// engine-local form. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RoutedRequest { + One(Store, FsRequest), + /// Attachment and commit-deferral state is pair-local in each frontend. + Both(FsRequest), + /// Both authority tables contribute entries to one client window. + ListGrants { + window_offset: u32, + }, + /// Both engines contribute one aggregate status record. + SystemStatus { + window_offset: u32, + }, + /// Admin-only lifecycle testimony owned by the dual-volume composition. + SystemSeed { + active: bool, + }, + /// A storage-ring verb: neither engine's; served by the row that owns + /// the controller (`res/STORAGE_TRANSPORT.md` §4). + Stor(FsRequest), +} + +#[inline] +const fn split_bits(bits: u64) -> (Store, u64) { + if bits & SYSTEM_TAG == 0 { + (Store::User, bits) + } else { + (Store::System, bits & !SYSTEM_TAG) + } +} + +#[inline] +const fn join_bits(store: Store, bits: u64) -> u64 { + debug_assert!(bits & SYSTEM_TAG == 0); + match store { + Store::User => bits, + Store::System => bits | SYSTEM_TAG, + } +} + +#[inline] +const fn split_handle(handle: Handle) -> (Store, Handle) { + let (store, bits) = split_bits(handle.to_bits()); + (store, Handle::from_bits(bits)) +} + +#[inline] +const fn split_grant(grant: GrantId) -> (Store, GrantId) { + let (store, bits) = split_bits(grant.to_bits()); + (store, GrantId::from_bits(bits)) +} + +#[inline] +pub(crate) const fn split_lease(lease: VersionLease) -> (Store, VersionLease) { + let (store, bits) = split_bits(lease.to_bits()); + (store, VersionLease::from_bits(bits)) +} + +fn same_store(left: (Store, T), right: (Store, T)) -> Result<(Store, T, T), FsError> { + if left.0 != right.0 { + return Err(FsError::AccessDenied); + } + Ok((left.0, left.1, right.1)) +} + +/// Select one engine without trusting an engine name supplied separately +/// from the authority word. A system-tagged request on the compatibility +/// one-volume posture is stale/forged authority, never an alias back to user. +pub(crate) fn route(request: FsRequest, has_system: bool) -> Result { + use FsRequest::*; + + let routed = match request { + AttachWindow { .. } | DeferCommits { .. } => RoutedRequest::Both(request), + ListGrants { window_offset } => RoutedRequest::ListGrants { window_offset }, + SystemStatus { window_offset } => RoutedRequest::SystemStatus { window_offset }, + SetSystemSeed { active } => RoutedRequest::SystemSeed { active }, + StorAttach { .. } + | StorGrantPack + | StorGrantVersion { .. } + | StorTarget { .. } + | StorRelease { .. } + | StorKick + | StorInspect + | StorMaintain { .. } + | StorDetach => RoutedRequest::Stor(request), + OpenRoot { .. } + | RegisterSystemView { .. } + | RegisterPrivateView { .. } + | ApplyView { .. } + | AdmitPeer { .. } + | TakeAdmission + | RetirePeer { .. } + | ConfirmRetired { .. } + | ClosePeer => RoutedRequest::One(Store::User, request), + OpenGrant { grant, rights } => { + let (store, grant) = split_grant(grant); + RoutedRequest::One(store, OpenGrant { grant, rights }) + } + OpenAt { + parent, + name, + rights, + } => { + let (store, parent) = split_handle(parent); + RoutedRequest::One( + store, + OpenAt { + parent, + name, + rights, + }, + ) + } + BeginReplace { + handle, + length, + expected, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + BeginReplace { + handle, + length, + expected, + }, + ) + } + WriteReplacement { + handle, + offset, + window_offset, + length, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + WriteReplacement { + handle, + offset, + window_offset, + length, + }, + ) + } + CommitReplace { handle } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One(store, CommitReplace { handle }) + } + AbortReplace { handle } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One(store, AbortReplace { handle }) + } + Close { handle } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One(store, Close { handle }) + } + Stat { handle } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One(store, Stat { handle }) + } + List { + handle, + cursor, + window_offset, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + List { + handle, + cursor, + window_offset, + }, + ) + } + Parents { + handle, + cursor, + window_offset, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + Parents { + handle, + cursor, + window_offset, + }, + ) + } + Locate { + handle, + stable, + window_offset, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + Locate { + handle, + stable, + window_offset, + }, + ) + } + Create { + parent, + name, + kind, + subtype, + } => { + let (store, parent) = split_handle(parent); + RoutedRequest::One( + store, + Create { + parent, + name, + kind, + subtype, + }, + ) + } + Read { + handle, + offset, + window_offset, + length, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + Read { + handle, + offset, + window_offset, + length, + }, + ) + } + WriteAt { + handle, + offset, + window_offset, + length, + expected, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + WriteAt { + handle, + offset, + window_offset, + length, + expected, + }, + ) + } + Append { + handle, + window_offset, + length, + expected, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + Append { + handle, + window_offset, + length, + expected, + }, + ) + } + Resize { + handle, + length, + expected, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + Resize { + handle, + length, + expected, + }, + ) + } + Rename { parent, old, new } => { + let (store, parent) = split_handle(parent); + RoutedRequest::One(store, Rename { parent, old, new }) + } + Relocate { + src_parent, + name, + dst_parent, + new_name, + } => { + let (store, src_parent, dst_parent) = + same_store(split_handle(src_parent), split_handle(dst_parent))?; + RoutedRequest::One( + store, + Relocate { + src_parent, + name, + dst_parent, + new_name, + }, + ) + } + Link { + parent, + name, + child, + } => { + let (store, parent, child) = same_store(split_handle(parent), split_handle(child))?; + RoutedRequest::One( + store, + Link { + parent, + name, + child, + }, + ) + } + Unlink { parent, name } => { + let (store, parent) = split_handle(parent); + RoutedRequest::One(store, Unlink { parent, name }) + } + DetachEdge { parent, name } => { + let (store, parent) = split_handle(parent); + RoutedRequest::One(store, DetachEdge { parent, name }) + } + Grant { + handle, + destination, + rights, + display_name, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + Grant { + handle, + destination, + rights, + display_name, + }, + ) + } + GrantScoped { + handle, + destination, + rights, + quota, + display_name, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + GrantScoped { + handle, + destination, + rights, + quota, + display_name, + }, + ) + } + GrantView { + handle, + rights, + display_name, + announced, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + GrantView { + handle, + rights, + display_name, + announced, + }, + ) + } + Revoke { grant } => { + let (store, grant) = split_grant(grant); + RoutedRequest::One(store, Revoke { grant }) + } + Sync { handle } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One(store, Sync { handle }) + } + LendVersion { + handle, + window_offset, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + LendVersion { + handle, + window_offset, + }, + ) + } + LendExecutable { + handle, + window_offset, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + LendExecutable { + handle, + window_offset, + }, + ) + } + ReadVersion { + lease, + offset, + window_offset, + length, + } => { + let (store, lease) = split_lease(lease); + RoutedRequest::One( + store, + ReadVersion { + lease, + offset, + window_offset, + length, + }, + ) + } + CommitDerived { + lease, + window_offset, + length, + output_window_bytes, + } => { + let (store, lease) = split_lease(lease); + RoutedRequest::One( + store, + CommitDerived { + lease, + window_offset, + length, + output_window_bytes, + }, + ) + } + ReleaseVersion { lease } => { + let (store, lease) = split_lease(lease); + RoutedRequest::One(store, ReleaseVersion { lease }) + } + DerivedInfo { handle, parser } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One(store, DerivedInfo { handle, parser }) + } + Search { + handle, + query, + cursor, + window_offset, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + Search { + handle, + query, + cursor, + window_offset, + }, + ) + } + BindView { + handle, + rights, + announced, + } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One( + store, + BindView { + handle, + rights, + announced, + }, + ) + } + MatchView { handle, announced } => { + let (store, handle) = split_handle(handle); + RoutedRequest::One(store, MatchView { handle, announced }) + } + }; + + if matches!(routed, RoutedRequest::One(Store::System, _)) && !has_system { + Err(FsError::InvalidHandle) + } else { + Ok(routed) + } +} + +/// Restore the selected engine on every opaque authority word a response +/// exposes. Object ids remain untagged: they are correlation data and no +/// request uses one to select authority or an engine. +pub(crate) fn tag_response(store: Store, response: FsResponse) -> FsResponse { + match response { + FsResponse::Opened { + handle, + stable_object, + kind, + subtype, + len, + revision, + } => FsResponse::Opened { + handle: Handle::from_bits(join_bits(store, handle.to_bits())), + stable_object, + kind, + subtype, + len, + revision, + }, + FsResponse::Granted { grant } => FsResponse::Granted { + grant: GrantId::from_bits(join_bits(store, grant.to_bits())), + }, + FsResponse::VersionLent { lease } => FsResponse::VersionLent { + lease: VersionLease::from_bits(join_bits(store, lease.to_bits())), + }, + other => other, + } +} + +/// Add `store` to the grant ids already written into a merged listing. +pub(crate) fn tag_grant_entries( + window: &mut DataWindow<'_>, + window_offset: u32, + first: usize, + count: usize, + store: Store, +) -> Result<(), FsError> { + for index in first..first.checked_add(count).ok_or(FsError::Range)? { + let step = index + .checked_mul(FsGrantEntryWire::BYTES) + .ok_or(FsError::Range)?; + let at = usize::try_from(window_offset) + .map_err(|_| FsError::Range)? + .checked_add(step) + .ok_or(FsError::Range)?; + let at = u32::try_from(at).map_err(|_| FsError::Range)?; + let mut entry = + FsGrantEntryWire::read_from_bytes(window.source(at, FsGrantEntryWire::BYTES as u32)?) + .map_err(|_| FsError::Range)?; + entry.grant = join_bits(store, entry.grant.get()).into(); + window.put_grant_entry(window_offset, index, &entry)?; + } + Ok(()) +} + +pub(crate) fn grant_append_offset(window_offset: u32, entries: u32) -> Result { + window_offset + .checked_add( + entries + .checked_mul(FsGrantEntryWire::BYTES as u32) + .ok_or(FsError::Range)?, + ) + .ok_or(FsError::Range) +} + +/// Read the status record just produced by one engine before the sibling +/// overwrites the same window span. +pub(crate) fn read_status( + window: &DataWindow<'_>, + window_offset: u32, +) -> Result { + FsSystemStatusWire::read_from_bytes( + window.source(window_offset, FsSystemStatusWire::BYTES as u32)?, + ) + .map_err(|_| FsError::Range) +} + +/// Fold the two engine snapshots into the service's one frozen status ABI. +/// Physical bytes describe their shared controller namespace and therefore +/// take the maximum; volume/occupancy and per-engine work counters sum. +pub(crate) fn fold_status( + user: FsSystemStatusWire, + system: FsSystemStatusWire, +) -> FsSystemStatusWire { + use kore_abi::monitor::FilesystemStorageSnapshot; + + let add16 = |a: u16, b: u16| a.saturating_add(b).into(); + let add32 = |a: u32, b: u32| a.saturating_add(b).into(); + let add64 = |a: u64, b: u64| a.saturating_add(b).into(); + let user_storage = user.storage; + let system_storage = system.storage; + let storage = match (user_storage.available(), system_storage.available()) { + (false, false) => FilesystemStorageSnapshot::EMPTY, + (true, false) => user_storage, + (false, true) => system_storage, + (true, true) => { + let block = user_storage.logical_block_bytes(); + if block == system_storage.logical_block_bytes() { + let max_transfer = user_storage + .max_transfer_bytes() + .min(system_storage.max_transfer_bytes()); + let mut combined = FilesystemStorageSnapshot::new( + block, + user_storage + .physical_bytes() + .max(system_storage.physical_bytes()), + user_storage + .volume_bytes() + .saturating_add(system_storage.volume_bytes()), + user_storage + .occupied_bytes() + .saturating_add(system_storage.occupied_bytes()), + max_transfer, + user_storage.read_only() || system_storage.read_only(), + user_storage.flush() && system_storage.flush(), + user_storage.discard() && system_storage.discard(), + ); + // One physical namespace: its layout facts are the same on + // both sides, so either copy is the truth. + combined = combined.with_layout_of(user_storage); + let health = if user_storage.health().available() { + user_storage.health() + } else { + system_storage.health() + }; + if health.available() { + combined = combined.with_health(health); + } + combined + } else { + // One ABI record cannot truthfully encode two logical block + // sizes. Keep the larger mounted namespace visible rather + // than inventing a unit conversion for transfer granularity. + if user_storage.volume_bytes() >= system_storage.volume_bytes() { + user_storage + } else { + system_storage + } + } + } + }; + let note = if user.note.get() != 0 { + user.note + } else { + system.note + }; + FsSystemStatusWire { + sessions: add16(user.sessions.get(), system.sessions.get()), + peers: add16(user.peers.get(), system.peers.get()), + views_private: add16(user.views_private.get(), system.views_private.get()), + views_pinned: add16(user.views_pinned.get(), system.views_pinned.get()), + windows: add16(user.windows.get(), system.windows.get()), + flags: (user.flags.get() | system.flags.get()).into(), + tree_objects: add32(user.tree_objects.get(), system.tree_objects.get()), + tree_edges: add32(user.tree_edges.get(), system.tree_edges.get()), + user_lifecycle: if user.user_lifecycle.get() != 0 { + user.user_lifecycle + } else { + system.user_lifecycle + }, + system_lifecycle: if system.system_lifecycle.get() != 0 { + system.system_lifecycle + } else { + user.system_lifecycle + }, + tree_bytes: add64(user.tree_bytes.get(), system.tree_bytes.get()), + storage, + note, + commits: add64(user.commits.get(), system.commits.get()), + reads: add64(user.reads.get(), system.reads.get()), + writes: add64(user.writes.get(), system.writes.get()), + flushes: add64(user.flushes.get(), system.flushes.get()), + discards: add64(user.discards.get(), system.discards.get()), + copied_bytes: add64(user.copied_bytes.get(), system.copied_bytes.get()), + rollbacks: add64(user.rollbacks.get(), system.rollbacks.get()), + late_completions: add64(user.late_completions.get(), system.late_completions.get()), + orphan_reclaims: add64(user.orphan_reclaims.get(), system.orphan_reclaims.get()), + reclaim_failures: add64(user.reclaim_failures.get(), system.reclaim_failures.get()), + expired_waits: add64(user.expired_waits.get(), system.expired_waits.get()), + slow_waits: add64(user.slow_waits.get(), system.slow_waits.get()), + leaked_placements: add64(user.leaked_placements.get(), system.leaked_placements.get()), + abandoned_reclaims: add64( + user.abandoned_reclaims.get(), + system.abandoned_reclaims.get(), + ), + readahead_hits: add64(user.readahead_hits.get(), system.readahead_hits.get()), + readahead_reads: add64(user.readahead_reads.get(), system.readahead_reads.get()), + submissions: add64(user.submissions.get(), system.submissions.get()), + completions: add64(user.completions.get(), system.completions.get()), + backpressure: add64(user.backpressure.get(), system.backpressure.get()), + poll_calls: add64(user.poll_calls.get(), system.poll_calls.get()), + device_waits: add64(user.device_waits.get(), system.device_waits.get()), + route_parks: add64(user.route_parks.get(), system.route_parks.get()), + route_fires: add64(user.route_fires.get(), system.route_fires.get()), + deadline_waits: add64(user.deadline_waits.get(), system.deadline_waits.get()), + spin_waits: add64(user.spin_waits.get(), system.spin_waits.get()), + monitor_sleeps: add64(user.monitor_sleeps.get(), system.monitor_sleeps.get()), + monitor_turns: add64(user.monitor_turns.get(), system.monitor_turns.get()), + // One controller queue serves both regions: depth takes the max, + // the histograms add. + latency: user.latency.folded_with(system.latency), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + FsEndowment, Name, ObjectKind, RamService, Rights, SearchCursor, SearchQuery, SessionId, + }; + use kore_rt::service::SystemService; + + fn system_handle(bits: u64) -> Handle { + Handle::from_bits(bits | SYSTEM_TAG) + } + + #[test] + fn replacements_keep_their_volume_and_native_handle() { + let native = Handle::from_bits(7); + let tagged = Handle::from_bits(join_bits(Store::System, native.to_bits())); + for (wire, local) in [ + ( + FsRequest::BeginReplace { + handle: tagged, + length: 3, + expected: 2, + }, + FsRequest::BeginReplace { + handle: native, + length: 3, + expected: 2, + }, + ), + ( + FsRequest::WriteReplacement { + handle: tagged, + offset: 1, + window_offset: 0, + length: 2, + }, + FsRequest::WriteReplacement { + handle: native, + offset: 1, + window_offset: 0, + length: 2, + }, + ), + ( + FsRequest::CommitReplace { handle: tagged }, + FsRequest::CommitReplace { handle: native }, + ), + ( + FsRequest::AbortReplace { handle: tagged }, + FsRequest::AbortReplace { handle: native }, + ), + ] { + assert_eq!( + route(wire, true), + Ok(RoutedRequest::One(Store::System, local)) + ); + assert_eq!(route(wire, false), Err(FsError::InvalidHandle)); + } + } + + #[test] + fn roots_default_to_user_and_tagged_grants_select_system() { + assert!(matches!( + route( + FsRequest::OpenRoot { + rights: Rights::READ + }, + true + ), + Ok(RoutedRequest::One(Store::User, FsRequest::OpenRoot { .. })) + )); + let request = FsRequest::OpenGrant { + grant: GrantId::from_bits(SYSTEM_TAG | 9), + rights: Rights::READ, + }; + assert_eq!( + route(request, true), + Ok(RoutedRequest::One( + Store::System, + FsRequest::OpenGrant { + grant: GrantId::from_bits(9), + rights: Rights::READ, + } + )) + ); + assert_eq!(route(request, false), Err(FsError::InvalidHandle)); + assert_eq!( + route(FsRequest::SetSystemSeed { active: true }, true), + Ok(RoutedRequest::SystemSeed { active: true }) + ); + } + + #[test] + fn every_returned_authority_word_keeps_its_engine() { + let opened = tag_response( + Store::System, + FsResponse::Opened { + handle: Handle::from_bits(7), + stable_object: crate::StableObjectId::new(11).unwrap(), + kind: ObjectKind::File, + subtype: crate::ObjectSubtype::PLAIN, + len: 2, + revision: 3, + }, + ); + assert!(matches!( + opened, + FsResponse::Opened { handle, .. } if handle == system_handle(7) + )); + assert_eq!( + tag_response( + Store::System, + FsResponse::Granted { + grant: GrantId::from_bits(8) + } + ), + FsResponse::Granted { + grant: GrantId::from_bits(SYSTEM_TAG | 8) + } + ); + assert_eq!( + tag_response( + Store::System, + FsResponse::VersionLent { + lease: VersionLease::from_bits(10) + } + ), + FsResponse::VersionLent { + lease: VersionLease::from_bits(SYSTEM_TAG | 10) + } + ); + } + + #[test] + fn cross_engine_graph_operations_are_refused_before_dispatch() { + let name = Name::new("x").unwrap(); + assert_eq!( + route( + FsRequest::Link { + parent: Handle::from_bits(1), + name, + child: system_handle(2), + }, + true, + ), + Err(FsError::AccessDenied) + ); + assert_eq!( + route( + FsRequest::Relocate { + src_parent: system_handle(1), + name, + dst_parent: Handle::from_bits(2), + new_name: Name::new("y").unwrap(), + }, + true, + ), + Err(FsError::AccessDenied) + ); + } + + #[test] + fn the_system_tag_routes_handles_and_leases_but_not_search_cursors() { + let request = FsRequest::Search { + handle: system_handle(5), + query: SearchQuery::ALL, + cursor: Some(SearchCursor { + observation: SYSTEM_TAG, + position: 4, + }), + window_offset: 0, + }; + let Ok(RoutedRequest::One(Store::System, FsRequest::Search { handle, cursor, .. })) = + route(request, true) + else { + panic!("system search route"); + }; + assert_eq!(handle, Handle::from_bits(5)); + assert_eq!(cursor.unwrap().observation, SYSTEM_TAG); + + let request = FsRequest::ReadVersion { + lease: VersionLease::from_bits(SYSTEM_TAG | 6), + offset: 0, + window_offset: 0, + length: 1, + }; + assert!(matches!( + route(request, true), + Ok(RoutedRequest::One( + Store::System, + FsRequest::ReadVersion { lease, .. } + )) if lease == VersionLease::from_bits(6) + )); + } + + /// Adversarial form of the containment argument: a forged engine bit is + /// only a route choice. The system engine receives the same raw slot word + /// and must still validate the caller's pair-derived session. This test + /// exercises the router half; authority's cross-session forged-handle + /// tests exercise the second half at the table seam. + #[test] + fn flipping_the_tag_changes_only_the_selected_engine() { + let raw = Handle::from_bits((3u64 << 32) | 7); + let forged = Handle::from_bits(raw.to_bits() | SYSTEM_TAG); + assert_eq!(split_handle(raw), (Store::User, raw)); + assert_eq!(split_handle(forged), (Store::System, raw)); + } + + #[test] + fn a_forged_tag_still_cannot_cross_the_selected_engines_session_boundary() { + let mut system = ::bind(FsEndowment::UNSTAMPED); + let owner = SessionId::new(0); + let forger = SessionId::new(1); + assert!(system.ensure_session(forger)); + let owned = system.engine().open_root(owner, Rights::READ).unwrap(); + let tagged = Handle::from_bits(join_bits(Store::System, owned.handle.to_bits())); + let RoutedRequest::One(Store::System, FsRequest::Stat { handle }) = + route(FsRequest::Stat { handle: tagged }, true).unwrap() + else { + panic!("tagged stat routes to system"); + }; + assert_eq!( + system.dispatch(forger, &FsRequest::Stat { handle }, None), + FsResponse::Refused(FsError::InvalidHandle) + ); + } + + #[test] + fn merged_grants_are_restamped_in_place() { + let mut bytes = [0u8; FsGrantEntryWire::BYTES * 2]; + let mut window = DataWindow::new(&mut bytes); + let first = FsGrantEntryWire { + name_len: 0, + rights: Rights::READ.bits(), + kind: FsGrantEntryWire::KIND_GRANT, + reserved0: 0, + reserved1: 0.into(), + grant: 3u64.into(), + name: [0; kore_abi::svc::FS_NAME_BYTES], + }; + window.put_grant_entry(0, 0, &first).unwrap(); + window.put_grant_entry(0, 1, &first).unwrap(); + tag_grant_entries(&mut window, 0, 1, 1, Store::System).unwrap(); + let user = FsGrantEntryWire::read_from_bytes( + window.source(0, FsGrantEntryWire::BYTES as u32).unwrap(), + ) + .unwrap(); + let system = FsGrantEntryWire::read_from_bytes( + window + .source( + FsGrantEntryWire::BYTES as u32, + FsGrantEntryWire::BYTES as u32, + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(user.grant.get(), 3); + assert_eq!(system.grant.get(), SYSTEM_TAG | 3); + } + + #[test] + fn status_folding_sums_regions_but_not_the_shared_physical_namespace() { + let zero = [0u8; FsSystemStatusWire::BYTES]; + let mut user = FsSystemStatusWire::read_from_bytes(&zero).unwrap(); + let mut system = user; + user.flags = FsSystemStatusWire::FLAG_DURABLE.into(); + system.flags = FsSystemStatusWire::FLAG_DURABLE.into(); + user.sessions = 2u16.into(); + system.sessions = 3u16.into(); + user.tree_bytes = 7u64.into(); + system.tree_bytes = 11u64.into(); + user.user_lifecycle = FsSystemStatusWire::LIFECYCLE_CURRENT.into(); + user.system_lifecycle = FsSystemStatusWire::LIFECYCLE_REBUILT.into(); + user.route_parks = 9u64.into(); + let mut hot = [0u32; kore_abi::monitor::FilesystemLatencySnapshot::BUCKETS]; + hot[1] = 5; + user.latency = + kore_abi::monitor::FilesystemLatencySnapshot::new(4, 5, 15_000, 3_500, 6, hot); + let mut cold = [0u32; kore_abi::monitor::FilesystemLatencySnapshot::BUCKETS]; + cold[5] = 1; + system.latency = + kore_abi::monitor::FilesystemLatencySnapshot::new(2, 1, 100_000, 100_000, 0, cold); + user.storage = kore_abi::monitor::FilesystemStorageSnapshot::new( + 512, 1_000_000, 400_000, 100_000, 16_384, false, true, true, + ); + system.storage = kore_abi::monitor::FilesystemStorageSnapshot::new( + 512, 1_000_000, 200_000, 50_000, 8_192, true, true, false, + ); + let folded = fold_status(user, system); + assert_eq!(folded.sessions.get(), 5); + assert_eq!(folded.tree_bytes.get(), 18); + assert_eq!( + folded.user_lifecycle.get(), + FsSystemStatusWire::LIFECYCLE_CURRENT + ); + assert_eq!( + folded.system_lifecycle.get(), + FsSystemStatusWire::LIFECYCLE_REBUILT + ); + assert_eq!(folded.route_parks.get(), 9); + assert_eq!(folded.latency.max_outstanding(), 4); + assert_eq!(folded.latency.samples(), 6); + assert_eq!(folded.latency.max_ns(), 100_000); + assert_eq!((folded.latency.bucket(1), folded.latency.bucket(5)), (5, 1)); + assert_eq!(folded.storage.physical_bytes(), 1_000_000); + assert_eq!(folded.storage.volume_bytes(), 600_000); + assert_eq!(folded.storage.occupied_bytes(), 150_000); + assert_eq!(folded.storage.max_transfer_bytes(), 8_192); + assert!(folded.storage.read_only()); + assert!(folded.storage.flush()); + assert!(!folded.storage.discard()); + assert!(folded.storage.validate()); + } +} diff --git a/native/services/dagfs/src/search.rs b/native/services/dagfs/src/search.rs new file mode 100644 index 0000000..b0f7768 --- /dev/null +++ b/native/services/dagfs/src/search.rs @@ -0,0 +1,62 @@ +//! Capability-safe bounded structural, text, relationship, and semantic query. +//! +//! Query execution is split at a trait seam. The filesystem owns authority +//! and generates candidates only by walking the caller's live namespace view. +//! A [`QueryIndex`] may score that already-visible object, but it cannot mint +//! candidates, handles, identity, or access. + +use heapless::String; + +use crate::{ + error::FsResult, + object::{ObjectId, Revision}, +}; + +pub use kore_dagfs::search::{ + FreshnessPolicy, MAX_SEARCH_PAGE, MAX_SEARCH_SCAN, MAX_SEARCH_SNIPPET, MAX_SEARCH_WORK, + SearchContent, SearchCursor, SearchHit, SearchPlane, SearchQuery, SearchScore, SearchTerm, +}; +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IndexMatch { + pub matched: bool, + pub score: SearchScore, + pub snippet: String, + pub work: u32, + /// Generation of this visible object's rebuildable result shard. + pub generation: u64, +} + +impl IndexMatch { + pub fn miss(work: u32) -> Self { + Self { + matched: false, + score: SearchScore::NONE, + snippet: String::new(), + work, + generation: 0, + } + } +} + +/// Rebuildable index seam. There is intentionally no enumeration method: +/// implementations can score a proven candidate, never create one. +pub trait QueryIndex { + fn evaluate( + &self, + object: ObjectId, + revision: Revision, + query: SearchContent, + ) -> FsResult; +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SearchPage { + pub hits: heapless::Vec, + pub observation: u64, + pub snapshot: u64, + pub index_generation: u64, + pub freshness: FreshnessPolicy, + pub next: Option, + pub scanned: u32, + pub work: u32, +} diff --git a/native/services/dagfs/src/service/contract.rs b/native/services/dagfs/src/service/contract.rs new file mode 100644 index 0000000..cffd6a2 --- /dev/null +++ b/native/services/dagfs/src/service/contract.rs @@ -0,0 +1,627 @@ +//! The tree's service contract: [`kore_rt::service::Bundle`], [`SystemService`], +//! [`SvcProtocol`]. +//! +//! Implementing these is what makes `kore-svc-dagfs` a *service* rather than a +//! library with a codec — the founder stamps [`FsEndowment::PROFILE`] into the +//! child's boot block, `kore_rt` drives [`SystemService::serve`], and the pair's +//! ring helpers become usable at this protocol's exact record widths. +//! +//! # The capability endowment is empty, and that is the point +//! +//! The filesystem owns no ambient hardware authority. Its namespace and grant +//! authority are service-local, and it never holds `PciDevice`, BAR, IRQ, or +//! IOMMU capabilities — those belong to the block owner below it. On x86 the +//! durable entry reaches storage through +//! an ordinary typed block-service client face; other architectures currently +//! select the bounded RAM provider at the entry seam. Neither provider changes +//! the filesystem's [`kore_rt::service::Bundle`] or wire contract. An empty +//! [`kore_abi::task::Profile`] states +//! that boundary in the one place the founder reads. +//! +//! # The session, and why `serve` is thin +//! +//! [`SystemService::serve`] receives a request and a [`Serve`] — and no +//! identity for the pair that sent it. +//! [`Serve::via`] is what supplies it: `ServeHost` fans in over up to +//! `SERVE_PAIRS` client pairs and stamps each request with the pair it arrived +//! on, so the pair *is* the session and nothing is inferred from wire data. +//! `via` is 0 for the ordinary single-pair row, which is why one client and +//! several are the same line. +//! +//! What several pairs add is the piece one pair gets for free: a pair's session +//! must be *created*, not assumed. `ServeHost::attach_pair` wires the ring pair +//! but knows nothing of sessions, so `serve` below mints one with +//! `ensure_session` on first contact rather than at bind time. +//! +//! # Bulk-window custody +//! +//! Bulk verbs operate only after the client has delivered a writable view on +//! the pair's retained return leg and committed [`FsRequest::AttachWindow`]. +//! [`Serve::returned_input`] exposes that exact returned capability to the +//! service; [`FsService::serve_request`] adopts it into the pair-bound session. +//! A pair with no successfully adopted window refuses [`FsError::NoWindow`] +//! rather than reading an address supplied in untrusted wire data. Exec uses +//! this production path for immutable filesystem-version identities and reads; +//! recyclable child faces use the same mechanism with distinct retained +//! landing slots. + +use kore_abi::{ + svc::{FsRepWire, FsReqWire}, + task::{Profile, geom, role}, +}; +use kore_rt::{ + service::{BindError, Bundle, PeerClose, ReturnMode, Serve, StartFailure, SystemService}, + svc::{PeerAdmissionRefusal, PeerId, PeerMapPlan, SvcProtocol}, +}; + +use kore_dagfs::DagfsProtocol; + +use crate::{ + authority::SessionId, + error::FsError, + service::{FsRequest, FsResponse, FsService, RamService}, +}; + +/// Exec's fixed administration pair. Pair zero remains the boot/legacy +/// client; recyclable process faces begin after both static pairs. +pub(crate) const ADMIN_VIA: u16 = 1; + +/// The filesystem's authority-free endowment plus its immutable archive era. +/// +/// `archive_era` is data, not a capability: eight `role::ARCHIVE_ERA` boot +/// literals carry the measured archive's 32-byte BLAKE3 digest. The profile +/// remains empty, preserving the stronger claim that FS owns no ambient +/// hardware authority. A handwritten [`Bundle`] is warranted here because the +/// ordinary derive is deliberately capability-shaped and must not pretend a +/// copyable digest is authority. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FsEndowment { + archive_era: [u8; crate::persistence::ERA_BYTES], +} + +impl FsEndowment { + /// The provider-neutral host/RAM posture. Production durable binding + /// resolves a founder-stamped era instead; zero never authorizes a system + /// reformat. + pub const UNSTAMPED: Self = Self { + archive_era: [0; crate::persistence::ERA_BYTES], + }; + + pub const fn archive_era(self) -> [u8; crate::persistence::ERA_BYTES] { + self.archive_era + } +} + +impl Bundle for FsEndowment { + const PROFILE: Profile = Profile::new("fs", &[]); + + fn resolve(boot: &kore_abi::task::Bootstrap<'_>) -> Result { + let mut archive_era = [0u8; crate::persistence::ERA_BYTES]; + let mut index = 0usize; + while index < archive_era.len() / 4 { + let word = + boot.datum(role::ARCHIVE_ERA, index as u16) + .ok_or(BindError::MissingDatum { + role: role::ARCHIVE_ERA, + index: index as u16, + })?; + archive_era[index * 4..index * 4 + 4].copy_from_slice(&word.to_le_bytes()); + index += 1; + } + Ok(Self { archive_era }) + } +} + +impl SystemService for RamService { + /// The filesystem is an I/O worker whichever backend founds it: parked + /// on completions for most of its life. Prefer the efficiency cores. + const CORE_PREFERENCE: kore_abi::task::CorePreference = + kore_abi::task::CorePreference::Efficient; + const DEVICE_CLASSES: &'static [kore_rt::service::DeviceClass] = crate::FS_DEVICE_CLASSES; + const NAME: &'static str = "fs"; + type Endowment = FsEndowment; + + /// Each client donates one writable `LeasedRegion` data window. The + /// generic host rotates a distinct retained landing for every serving + /// pair, so session N can never satisfy session M's attachment. + const RETURN_MODE: ReturnMode = ReturnMode::RetainedAny; + + fn bind(_endowment: FsEndowment) -> Self { + // Pair 0's root, minted early. Every other pair's arrives on first + // contact in `serve`, and `ensure_session` is idempotent, so the + // single-pair and many-pair shapes agree rather than fight. + let mut service = Self::default(); + let session = SessionId::new(0); + // A service that cannot give its one client a root is a service that + // can do nothing; the refusal surfaces on the client's first call + // rather than as a panic here. + let _ = service.ensure_session(session); + service.bind_session(session); + service + } + + fn start_on_metal(&mut self, rt: &kore_rt::entry::Rt) -> Result<(), StartFailure> { + self.configure_peer_map(peer_map(rt)?); + if !self.establish_admin_source() { + return Err(StartFailure::new(2)); + } + // The RAM composition is not durable, but exec discovers its workspace + // by the same grant name on every composition. Publishing it here keeps + // the volatile and durable shapes interchangeable from exec's side — + // the data does not survive, the wiring does. + if !self.establish_admin_workspace() { + return Err(StartFailure::new(3)); + } + Ok(()) + } + + /// **The session is the pair the record arrived on**, never anything the + /// request says: caller identity is derived from the pair/session context, + /// never trusted from a caller-supplied task ID. + /// + /// [`Serve::via`] is the founded wiring speaking — a client cannot name a + /// session it does not own, because nothing a client can write selects + /// one. The mapping is the identity, pair *n* is session *n*: deliberately + /// trivial and deliberately total, so there is no lookup that could be made + /// to miss and no table a client could influence. `via` is 0 for the + /// ordinary single-pair shape, so one client and several are the same line. + fn serve(&mut self, req: FsRequest, sv: &mut Serve<'_, FsResponse>) { + let response = self.serve_request(req, sv); + sv.reply(response); + } + + fn peer_admitted(&mut self, result: Result) -> bool { + FsService::peer_admitted(self, result) + } + + fn peer_closed(&mut self, via: u16, reason: PeerClose) -> bool { + FsService::peer_closed(self, via, reason) + } + + /// Every program that reads files holds one face for its life, so the + /// faces grow with the programs into the host's whole slot space; the + /// sessions behind them are allocated per slot from the same heap. + fn peer_arena(&self) -> Option<&'static (dyn core::alloc::Allocator + Sync)> { + Some(crate::service_alloc()) + } +} + +impl FsService

{ + /// Shared request dispatch for RAM and durable service compositions. + pub(crate) fn serve_request( + &mut self, + req: FsRequest, + sv: &mut Serve<'_, FsResponse>, + ) -> FsResponse { + crate::block_on(self.serve_request_async(req, sv)) + } + + /// Async request dispatch used by compositions whose storage provider can + /// park. Keeping the `Serve` borrow inside this future preserves the real + /// executor waker across dependent service calls; a nested noop-waker + /// driver would strand the request after publishing it. + pub(crate) async fn serve_request_async( + &mut self, + req: FsRequest, + sv: &mut Serve<'_, FsResponse>, + ) -> FsResponse { + match req { + FsRequest::RegisterPrivateView { + rights, + name, + announced, + } => { + if sv.via() != ADMIN_VIA { + return FsResponse::Refused(FsError::AccessDenied); + } + let Some(mut input) = sv.returned_input(announced) else { + return FsResponse::Refused(FsError::AccessDenied); + }; + let Some(returned) = input.take() else { + return FsResponse::Refused(FsError::AccessDenied); + }; + let canary = + match returned.try_into_chan_send(crate::FsViewCapability::INVOCATION_RIGHTS) { + Ok(canary) => canary, + Err(_) => return FsResponse::Refused(FsError::AccessDenied), + }; + let Some(scope) = kore_rt::cap::verbs::capability_scope_rid( + canary.slot(), + kore_abi::cap::ResourceKind::ChanSend, + kore_abi::cap::Rights::ACCESS, + ) else { + let _ = canary.try_retire(); + return FsResponse::Refused(FsError::AccessDenied); + }; + return match self.register_private_invocation_view( + scope, + &name, + rights, + Some(canary), + ) { + Ok(()) => FsResponse::Done, + Err(error) => FsResponse::Refused(error), + }; + } + FsRequest::RegisterSystemView { rights, announced } => { + if sv.via() != ADMIN_VIA { + return FsResponse::Refused(FsError::AccessDenied); + } + let scope = sv.returned_input(announced).and_then(|input| { + input.capability().and_then(|capability| { + capability.scope_rid_with( + kore_abi::cap::ResourceKind::ChanSend, + kore_abi::cap::Rights::ACCESS, + ) + }) + }); + return match scope.and_then(|scope| { + self.register_system_invocation_view(scope, rights) + .ok() + .map(|_| ()) + }) { + Some(()) => FsResponse::Done, + None => FsResponse::Refused(FsError::AccessDenied), + }; + } + FsRequest::ApplyView { + peer, + rights, + announced, + } => { + if sv.via() != ADMIN_VIA { + return FsResponse::Refused(FsError::AccessDenied); + } + let scope = sv.returned_input(announced).and_then(|input| { + input.capability().and_then(|capability| { + capability.scope_rid_with( + kore_abi::cap::ResourceKind::ChanSend, + kore_abi::cap::Rights::ACCESS, + ) + }) + }); + return match scope.and_then(|scope| { + self.apply_invocation_view(scope, peer, rights) + .ok() + .map(|_| ()) + }) { + Some(()) => FsResponse::Done, + None => FsResponse::Refused(FsError::AccessDenied), + }; + } + FsRequest::AdmitPeer { part, announced } => { + return self.admit_peer_component(part, announced, sv); + } + FsRequest::TakeAdmission => { + if sv.via() != ADMIN_VIA { + return FsResponse::Refused(FsError::AccessDenied); + } + return self.take_peer_admission(); + } + FsRequest::RetirePeer { peer } => { + if sv.via() != ADMIN_VIA || self.peer_is_retired(peer) { + return FsResponse::Refused(FsError::AccessDenied); + } + sv.retire_peer(peer, PeerClose::Orderly); + return FsResponse::Done; + } + FsRequest::ConfirmRetired { peer } => { + if sv.via() != ADMIN_VIA { + return FsResponse::Refused(FsError::AccessDenied); + } + // A different generation in the same cell is already proof + // that this exact peer was retired. Never let a stale ticket + // name or disturb its replacement. + if !self.peer_is_retired(peer) { + return FsResponse::Refused(FsError::Conflict); + } + return FsResponse::Done; + } + FsRequest::ClosePeer => { + if sv.via() == 0 || self.session_for_via(sv.via()).is_none() { + return FsResponse::Refused(FsError::AccessDenied); + } + sv.close_peer(PeerClose::Orderly); + return FsResponse::Done; + } + _ => {} + } + let Some(session) = self.session_for_via(sv.via()) else { + return FsResponse::Refused(FsError::AccessDenied); + }; + // A pair's per-client state is its root: receiving a pair creates + // one. Minting it on first contact rather than at bind time keeps + // the two facts inseparable: a pair that can be polled is a pair + // whose session exists. Attaching a fixed set up front is what + // previously let a pair be wired, stamped and polled while its + // session had never been attached, so its first `OpenRoot` refused + // with nothing to blame. Dynamic sessions root through the pinned + // system view (the invocation posture) so their facts persist; + // fixed sessions keep private roots. + // REVISIT: `OutOfObjects` because a private root *is* an object and a + // full object arena is the realistic failure — but `attach_session` + // can also refuse for session-table reasons, which this then + // mislabels. The const-assert below makes the session-table case + // unreachable by construction, so the label is honest today and wrong + // only if that assert is ever weakened. A fixed pair whose session + // storage the heap cannot hold reads the same way; a recyclable one + // never reaches here without it, because its admission allocated it. + if !self.ensure_dynamic_session(session) { + return FsResponse::Refused(FsError::OutOfObjects); + } + match req { + FsRequest::AttachWindow { slot, .. } => { + let announced = kore_abi::session::SlotRef::from_wire(slot); + let adopted = sv + .returned_input(announced) + .and_then(|mut input| input.take()) + .ok_or(FsError::NoWindow) + .and_then(|owner| self.adopt_window(session, slot, owner)); + match adopted { + Ok(()) => FsResponse::WindowAttached { + max_transfer: self.transfer_limit(), + }, + Err(error) => FsResponse::Refused(error), + } + } + FsRequest::BindView { + handle, + rights, + announced, + } => { + let Some(mut input) = sv.returned_input(announced) else { + return FsResponse::Refused(FsError::AccessDenied); + }; + let Some(endpoint) = input.take() else { + return FsResponse::Refused(FsError::AccessDenied); + }; + match self.register_invocation_endpoint(session, handle, rights, endpoint) { + Ok(()) => FsResponse::Done, + Err(error) => FsResponse::Refused(error), + } + } + FsRequest::GrantView { + handle, + rights, + display_name, + announced, + } => { + let Some(source_view) = self.task_view_for_session(session) else { + return FsResponse::Refused(FsError::AccessDenied); + }; + let Some(mut input) = sv.returned_input(announced) else { + return FsResponse::Refused(FsError::AccessDenied); + }; + let Some(returned) = input.take() else { + return FsResponse::Refused(FsError::AccessDenied); + }; + let target = match returned.try_into_chan_send(crate::FS_GRANT_TARGET_RIGHTS) { + Ok(target) => target, + Err(_) => return FsResponse::Refused(FsError::AccessDenied), + }; + if target.rights() != crate::FS_GRANT_TARGET_RIGHTS { + let _ = target.try_retire(); + return FsResponse::Refused(FsError::AccessDenied); + } + let scope = kore_rt::cap::verbs::capability_scope_rid( + target.slot(), + kore_abi::cap::ResourceKind::ChanSend, + crate::FS_GRANT_TARGET_RIGHTS, + ); + let retired = target.try_retire().is_ok(); + let Some(destination_view) = + scope.and_then(|scope| self.task_view_for_scope(scope)) + else { + return FsResponse::Refused(FsError::AccessDenied); + }; + if !retired || destination_view == source_view { + return FsResponse::Refused(FsError::AccessDenied); + } + return match self.engine.grant_between_task_views( + session, + handle, + source_view, + destination_view, + rights, + Some(display_name), + ) { + Ok(grant) => FsResponse::Granted { grant }, + Err(error) => FsResponse::Refused(error), + }; + } + FsRequest::MatchView { handle, announced } => { + let Some(mut input) = sv.returned_input(announced) else { + return FsResponse::Refused(FsError::AccessDenied); + }; + let Some(returned) = input.take() else { + return FsResponse::Refused(FsError::AccessDenied); + }; + let anchor = match returned.try_into_chan_send(crate::FS_VIEW_ANCHOR_RIGHTS) { + Ok(anchor) => anchor, + Err(_) => return FsResponse::Refused(FsError::AccessDenied), + }; + if anchor.rights() != crate::FS_VIEW_ANCHOR_RIGHTS { + let _ = anchor.try_retire(); + return FsResponse::Refused(FsError::AccessDenied); + } + let scope = kore_rt::cap::verbs::capability_scope_rid( + anchor.slot(), + kore_abi::cap::ResourceKind::ChanSend, + crate::FS_VIEW_ANCHOR_RIGHTS, + ); + if anchor.try_retire().is_err() { + return FsResponse::Refused(FsError::AccessDenied); + } + return match scope { + Some(scope) => self.match_view(session, handle, scope), + None => FsResponse::Refused(FsError::AccessDenied), + }; + } + _ => self.dispatch_mapped_async(session, &req).await, + } + } +} + +/// Each dynamic peer occupies one page-aligned ring cell in the service's +/// founder-reserved transient band. The generic host validates the actual +/// protocol byte requirement against this stride before mapping. +pub(crate) const PEER_MAP_STRIDE: u64 = 0x2000; + +/// The band every ring cell the host can grow into needs: slot `n`'s ring +/// maps at `base + n * stride`, and the host can name every [`PeerId`] slot, +/// not just its bootstrap table's. +const PEER_MAP_BYTES: u64 = PEER_MAP_STRIDE * PeerId::SLOT_CAPACITY as u64; + +/// Derive the only lawful recyclable-ring placement from founder geometry. +/// Wire data never carries a VA and a service without the exact reservation +/// refuses readiness rather than accepting a peer it cannot map. +pub(crate) fn peer_map(rt: &kore_rt::entry::Rt) -> Result { + let window = rt + .geom() + .and_then(|geometry| geometry.window_of(geom::TRANSIENT)) + .filter(|window| { + window.base() != 0 && window.base() & 0xfff == 0 && window.len() >= PEER_MAP_BYTES + }) + .ok_or(StartFailure::new(1))?; + Ok(PeerMapPlan { + base: window.base(), + stride: PEER_MAP_STRIDE, + }) +} + +impl SvcProtocol for RamService { + type Request = ::Request; + type Response = ::Response; + type ReqWire = ::ReqWire; + type RepWire = ::RepWire; + + /// The protocol is the leaf's ([`DagfsProtocol`]): zero grants ever, two + /// founder-stamped pairs, and the fail-closed codec. Delegating every item + /// is what keeps the serving side from drifting from the client it answers. + const REPLY_GRANTS: usize = ::REPLY_GRANTS; + const SERVE_PAIRS: usize = ::SERVE_PAIRS; + + fn reply_matches(request: &FsReqWire, response: &FsResponse, grants: u8) -> bool { + DagfsProtocol::reply_matches(request, response, grants) + } + + fn encode_req(req: &FsRequest) -> FsReqWire { + DagfsProtocol::encode_req(req) + } + + fn decode_req(w: &FsReqWire) -> Option { + DagfsProtocol::decode_req(w) + } + + fn encode_rep(rep: &FsResponse, grants: u8) -> FsRepWire { + DagfsProtocol::encode_rep(rep, grants) + } + + fn decode_rep(w: &FsRepWire) -> Option<(FsResponse, u8)> { + DagfsProtocol::decode_rep(w) + } + + fn decode_rep_for(request: &FsReqWire, reply: &FsRepWire) -> Option<(FsResponse, u8)> { + DagfsProtocol::decode_rep_for(request, reply) + } +} + +impl FsService

{ + /// Bind the fixed administration pair's session. + pub fn bind_session(&mut self, session: SessionId) { + self.session = Some(session); + } + + /// The bound session, defaulting to session 0 for a service constructed + /// without one. + pub fn bound_session(&self) -> SessionId { + self.session.unwrap_or(SessionId::new(0)) + } +} + +/// The pair window must hold both 8-deep rings at this protocol's widths, +/// inside the founder's `[layout] ring_bytes`. +/// +/// Asserted here through the tree's own `window_bytes`, so this cannot drift +/// from the arithmetic the founder and the serving host actually use. +const _: () = { + use kore_rt::svc::{REP_CAP, REQ_CAP, window_bytes}; + assert!(window_bytes::(REQ_CAP, REP_CAP) == 1728); + assert!(window_bytes::(REQ_CAP, REP_CAP) <= 0x2000); +}; + +/// Every pair the host can serve must be nameable as a session, or +/// [`SystemService::serve`] would be handed a client the engine can never root. +/// +/// The founder stamps at most `MAX_SERVE_PAIRS` fixed pairs and the host grows +/// recyclable faces into every [`PeerId`] slot; `PAIR_SESSIONS` is what the +/// engine can root, with the task views above it. This is the one place those +/// bounds meet, so a change to any of them breaks here rather than at a +/// client's first `OpenRoot`. +const _: () = { + assert!( + crate::bounds::PAIR_SESSIONS == PeerId::SLOT_CAPACITY, + "a session for every slot the host can grow a face into" + ); + assert!( + kore_rt::svc::MAX_SERVE_PAIRS <= crate::bounds::PAIR_SESSIONS, + "a session for every fixed pair the founder can stamp" + ); + assert!( + crate::bounds::MAX_SESSIONS == crate::bounds::PAIR_SESSIONS + crate::bounds::MAX_TASK_VIEWS, + "the task views sit above every pair's session" + ); +}; + +/// `ServeHost` requires the request/reply-plane scope; the filesystem meets it. +/// +/// Stated here rather than left to the first instantiation so that a change to +/// `REPLY_GRANTS` or `RETURN_MODE` breaks at this file — where the reason lives +/// — instead of at some distant construction site. +const _: () = { + assert!(::REPLY_GRANTS == 0); + assert!(matches!( + ::RETURN_MODE, + kore_rt::service::ReturnMode::RetainedAny + )); +}; + +#[cfg(test)] +mod endowment_tests { + use super::*; + use kore_abi::task::{BootBlock, Bootstrap}; + + #[test] + fn archive_era_round_trips_without_entering_the_capability_profile() { + let expected: [u8; crate::persistence::ERA_BYTES] = + core::array::from_fn(|index| index as u8); + let mut block = BootBlock::new(); + for (index, bytes) in expected.chunks_exact(4).enumerate() { + assert!(block.push_data( + role::ARCHIVE_ERA, + index as u16, + u32::from_le_bytes(bytes.try_into().unwrap()), + )); + } + let boot = Bootstrap::of(&block).unwrap(); + assert_eq!(FsEndowment::resolve(&boot).unwrap().archive_era(), expected); + assert!(FsEndowment::PROFILE.roles.is_empty()); + assert_eq!(boot.handles().count(), 0); + } + + #[test] + fn a_partial_archive_era_refuses_binding() { + let mut block = BootBlock::new(); + for index in 0..7u16 { + assert!(block.push_data(role::ARCHIVE_ERA, index, u32::from(index))); + } + let boot = Bootstrap::of(&block).unwrap(); + assert_eq!( + FsEndowment::resolve(&boot), + Err(BindError::MissingDatum { + role: role::ARCHIVE_ERA, + index: 7, + }) + ); + } +} diff --git a/native/services/dagfs/src/service/mod.rs b/native/services/dagfs/src/service/mod.rs new file mode 100644 index 0000000..123f0c9 --- /dev/null +++ b/native/services/dagfs/src/service/mod.rs @@ -0,0 +1,4682 @@ +//! The filesystem service: its request/reply vocabulary, and the dispatch that +//! drives [`Engine`](crate::engine::Engine) from it. +//! +//! One enum in, one enum out — the shape every `kore-svc-*` uses. +//! [`crate::wire`] carries the codecs to and from the frozen records in +//! `kore_abi::svc`, and the engine below knows about neither. +//! +//! # Where bulk data goes +//! +//! File bytes and directory-listing entries ride the pair's separately shared +//! **data window**, addressed by an offset and a length *into that window* +//! (the retained-return contract). Nothing here carries payload: a `Read` says where in the window +//! to land the bytes, and its reply says how many arrived. That keeps the reply +//! record 48 bytes wide regardless of transfer size. +//! +//! The client carves that window out of its own memory and delivers it through +//! the return channel, then names the landing slot in +//! [`FsRequest::AttachWindow`] — `OutReqWire::OP_ATTACH_TX`'s precedent. Until +//! it does, every bulk verb refuses [`FsError::NoWindow`]: a read with nowhere +//! to land is not a read. +//! +//! Production clients deliver the view through the serving pair's retained +//! return leg. `AttachWindow` adopts only the capability returned for the +//! request's announced slot; the request's integer fields cannot select a +//! mapping by themselves. Exec uses a fixed administration-pair window for +//! immutable image-source reads, while recyclable process faces receive +//! independent pair-bound windows. +//! +//! # Sessions +//! +//! [`FsService::dispatch`] takes the session as a **parameter**, never from the +//! request, because the request crossed a process boundary and the session did +//! not. The serving host derives it from the pair that delivered the bytes +//! (the pair-derived identity contract). `ServeHost` supplies it per pair: `Serve::via` names the +//! arrival, and it is 0 for the ordinary single-pair row, so one client and +//! several are the same line. `ensure_session` is what creates it, on first +//! contact — every piece of per-client state in this file is keyed the same +//! way, so a pair that can be served is a pair whose state exists. +//! +//! # Why dispatch is synchronous +//! +//! `SystemService::serve` is deliberately synchronous and allocation-free, +//! while the engine is async so a hardware-backed provider cannot block a +//! worker. Over the RAM provider those futures are `Ready` on first poll, so +//! driving them here with [`block_on`](crate::block_on) is exact rather than a +//! shortcut. **It stops being exact the moment a provider genuinely parks**; +//! a blocking provider must instead be driven by the executor without nesting +//! an async wait inside this synchronous serve step. + +pub mod contract; + +pub use contract::FsEndowment; + +use alloc::vec::Vec; + +use kore_abi::svc::{FS_SEARCH_PATH_BYTES, FsDirEntryWire, FsReqWire, FsSearchHitWire}; +use kore_rt::{ + cap::{ChanSendToken, ReturnedCapability, verbs}, + service::{PeerClose, Serve}, + svc::{ + PeerAdmissionRefusal, PeerAssemblyRefusal, PeerBundleAssembler, PeerBundlePart, PeerId, + PeerMapPlan, + }, +}; +use kore_svc_extract::schema::InputIdentity; + +use crate::{ + ServiceAlloc, + authority::{GrantId, Handle, Retired, Rights, SessionId}, + block_on, + bounds::{FIRST_TASK_VIEW_SESSION, MAX_SESSIONS}, + engine::Engine, + error::{FsError, FsResult}, + namespace::Name, + object::ObjectKind, + provider::{RamProvider, StorageProvider}, + window::DataWindow, +}; + +/// The bounded RAM-backed service composition. +pub type RamService = FsService; + +/// Stable display name of the read-only system namespace grant installed for +/// exec's fixed administration session. The name is discovery metadata only: +/// opening it still requires the pair-bound [`GrantId`] written by +/// [`FsRequest::ListGrants`]. +pub const SYSTEM_SOURCE_GRANT: &str = "sys"; + +/// Conventional names in one task's native filesystem view. +/// +/// These are presentation roles, never global objects: the launcher selects +/// the exact object and quota behind each pair-bound grant. Two tasks may see +/// unrelated objects under the same role. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NativeView { + App, + Data, + Config, + Tmp, + Shared, +} + +impl NativeView { + pub const ALL: [Self; 5] = [Self::App, Self::Data, Self::Config, Self::Tmp, Self::Shared]; + + pub const fn as_str(self) -> &'static str { + match self { + Self::App => "app", + Self::Data => "data", + Self::Config => "config", + Self::Tmp => "tmp", + Self::Shared => "shared", + } + } + + pub const fn index(self) -> usize { + match self { + Self::App => 0, + Self::Data => 1, + Self::Config => 2, + Self::Tmp => 3, + Self::Shared => 4, + } + } + + pub fn from_name(name: &Name) -> Option { + Self::ALL + .into_iter() + .find(|view| view.as_str().as_bytes() == name.as_bytes()) + } + + pub fn name(self) -> Name { + Name::new(self.as_str()).expect("native view names are statically valid") + } +} + +/// Name of the directory beneath session zero's root that holds the durable +/// workspace. On-disk state: it is an ordinary namespace edge, serialized and +/// recovered like any other, and renaming it would orphan every existing +/// volume's user data. +pub const SYSTEM_WORKSPACE_DIR: &str = "work"; + +/// Bytes of an init-minted face's home label: `face-` and one digit. +pub const FACE_HOME_LABEL_BYTES: usize = 6; + +/// The durable home of the init-minted face `face`: `face-N`, beneath +/// [`SYSTEM_WORKSPACE_DIR`]. Exec names a face's home by its index when it +/// mints the face, and init mints its terminals in one deterministic order, +/// so `face-N` is terminal N's home on every boot. A host tool writing into +/// terminal N's home (`tools/kore-vol`) finds it by this same name. `None` +/// past the one digit the label holds. +pub const fn face_home_label(face: u8) -> Option<[u8; FACE_HOME_LABEL_BYTES]> { + if face > 9 { + return None; + } + let mut label = *b"face-0"; + label[5] = b'0' + face; + Some(label) +} + +/// The shared executable directory beneath the persisted root: where the boot +/// archive's members land once, and where a later install writes. +pub const SYSTEM_APPS_DIR: &str = "app"; + +/// Canonical display name of the read-only application view in a user task. +/// +/// The durable backing directory stays `app` for recovery compatibility. A +/// task sees that one object as `apps`, whose children carry the structured +/// native and compatibility projections. This is a view name, not a second +/// directory or byte copy. +/// +/// This is the ONLY application view a terminal is granted. A second, flat +/// `app` grant onto the same object rode beside it while callers migrated; +/// nothing spells the flat form any more, and carrying it showed one store +/// object under two names — which on a live machine reads as a stale +/// projection that survived a boot and invites an attempt to delete it. +/// There was never anything to delete: the object behind the flat name is +/// this one, so the view ends by not being granted, not by any unlink. +pub const SYSTEM_APPS_VIEW: &str = "apps"; + +/// Display name of the administration session's writable grant over +/// [`SYSTEM_APPS_DIR`]. +/// +/// Deliberately *not* the directory's own name. The administration session's +/// structural root is the persisted root, so a grant displayed `app` would +/// collide with the `app` directory hanging beneath it — `view_name_taken` +/// claims the combined display-name/child-name set. The grant is what exec +/// discovers through `ListGrants`; the directory is what a terminal's view links +/// into. +pub const SYSTEM_APPS_GRANT: &str = "sys-app"; +/// The shared **library** directory beneath the persisted root — the +/// `libs/` half of the library convention: archive libraries (and, later, +/// a package's `lib/` members) live here, stamped `EXECUTABLE_IMAGE`, and +/// a task links them into itself at runtime through its own arena +/// (`res/IMAGE.md` §8). Deliberately not UNIX `lib`: a system view, not a +/// search path. +pub const SYSTEM_LIBS_DIR: &str = "lib"; +/// The READ-only display name every task view carries for it, beside +/// `apps` — the leaf's spelling, because a late loader opens it by name. +pub const SYSTEM_LIBS_VIEW: &str = kore_dagfs::LIBS_VIEW; +/// The administration session's writable grant over it (`sys-lib`, for the +/// same collision reason as `sys-app`). +pub const SYSTEM_LIBS_GRANT: &str = "sys-lib"; +/// The shared **data** directory beneath the persisted root: the boot +/// archive's DATA members (`ARCHIVE_MEMBER_DATA` — stock `.wasm` modules, +/// for one) land here once, under a per-kind subdirectory +/// (`shared/wasm/hello.wasm` as a task sees it), and a later install +/// writes here. Stamped +/// `PLAIN`: bytes to read, never an image to run or link. +/// The directory itself, in the short spelling its two siblings use: +/// `app` carries the `apps` view and `lib` the `libs` view, and this one +/// carries `shared`. Keeping the two apart is not cosmetic — +/// `view_name_taken` counts a real child of the destination root as a +/// collision, so a view whose name IS its directory's cannot install into +/// any session rooted where that directory lives, silently, because every +/// call site takes the bool with `let _ =`. On the two-store durable +/// layout that is every session the system store serves, which is why +/// `share` was absent from `list` and `shares` on metal while `apps` and +/// `libs` were there, and why no single-store QEMU gate ever saw it. +pub const SYSTEM_SHARE_DIR: &str = "share"; +/// The READ-only display name every task view carries for it, beside +/// `apps` and `libs`, and the name a task types: +/// `shared/wasm/hello.wasm`. The longer form, as those two are. +pub const SYSTEM_SHARE_VIEW: &str = "shared"; +/// The administration session's writable grant over it (`sys-share`, for +/// the same collision reason as `sys-app`): what exec's data seeder +/// discovers and writes through. +pub const SYSTEM_SHARE_GRANT: &str = "sys-share"; +/// The **package** directory: one subtree per installed `.pkg` +/// (`res/PKG.md`) — a typed manifest plus `app/` and `lib/` members that +/// `apps/` and `libs/` project by Link edge, never by copy. No task view; +/// installers reach it through the administration grant only. +pub const SYSTEM_PKGS_DIR: &str = "pkgs"; +/// The administration session's writable grant over it. +pub const SYSTEM_PKGS_GRANT: &str = "sys-pkg"; + +/// Whether two names differ — `const`, because the invariant below is a +/// compile-time one. +const fn name_differs(a: &str, b: &str) -> bool { + let (a, b) = (a.as_bytes(), b.as_bytes()); + if a.len() != b.len() { + return true; + } + let mut i = 0; + while i < a.len() { + if a[i] != b[i] { + return true; + } + i += 1; + } + false +} + +// A view's display name must not be any system directory's own name. +// `view_name_taken` treats a real child of the destination root as taken, +// so such a view refuses to install for every session rooted there, and +// the three call sites discard that refusal (`let _ =`) — the failure is +// silent, survives every gate whose store has one region, and shows up as +// a missing name on a metal install. `share`/`share` shipped exactly that. +const _: () = { + assert!(name_differs(SYSTEM_APPS_VIEW, SYSTEM_APPS_DIR)); + assert!(name_differs(SYSTEM_APPS_VIEW, SYSTEM_LIBS_DIR)); + assert!(name_differs(SYSTEM_APPS_VIEW, SYSTEM_SHARE_DIR)); + assert!(name_differs(SYSTEM_APPS_VIEW, SYSTEM_PKGS_DIR)); + assert!(name_differs(SYSTEM_LIBS_VIEW, SYSTEM_APPS_DIR)); + assert!(name_differs(SYSTEM_LIBS_VIEW, SYSTEM_LIBS_DIR)); + assert!(name_differs(SYSTEM_LIBS_VIEW, SYSTEM_SHARE_DIR)); + assert!(name_differs(SYSTEM_LIBS_VIEW, SYSTEM_PKGS_DIR)); + assert!(name_differs(SYSTEM_SHARE_VIEW, SYSTEM_APPS_DIR)); + assert!(name_differs(SYSTEM_SHARE_VIEW, SYSTEM_LIBS_DIR)); + assert!(name_differs(SYSTEM_SHARE_VIEW, SYSTEM_SHARE_DIR)); + assert!(name_differs(SYSTEM_SHARE_VIEW, SYSTEM_PKGS_DIR)); +}; + +pub use kore_dagfs::{FsRequest, FsResponse}; + +/// What a client told us about its delivered data window. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WindowAttachment { + /// The service-arena slot the client's view landed in. + pub slot: u32, + /// The view's length, as the client declared it. + pub bytes: u32, +} + +fn write_identity(dst: &mut [u8], identity: InputIdentity) { + debug_assert!(dst.len() >= 56); + dst[0..8].copy_from_slice(&identity.source.to_le_bytes()); + dst[8..16].copy_from_slice(&identity.version.to_le_bytes()); + dst[16..24].copy_from_slice(&identity.len.to_le_bytes()); + dst[24..56].copy_from_slice(&identity.digest); +} + +/// The filesystem service: an engine plus **one record per session slot** +/// (`SessionSlot`): the client's window, the peer generation that owns the +/// slot, its task view, its commit hold and its staged replacement. +/// +/// Per slot, not one shared record, is the whole point: `ServeHost` fans in +/// over several pairs on one `FsService`, so a shared attachment record +/// would let client B's `AttachWindow` answer client A's presence check — A's +/// bulk verbs would then be validated against B's declared slot and length. +/// Keying it the same way the session itself is keyed (pair *n* → session *n*) +/// makes "this client has a window" and "this client's window" the same fact. +pub struct FsService { + engine: Engine

, + /// Every session slot's service-side state, grown to the highest slot a + /// session has used ([`SessionSlots`]). + slots: SessionSlots, + /// The single client pair's session (the one-pair [`contract::bind`] + /// posture — see [`contract`]). The served path derives it per pair from + /// `Serve::via` instead. + pub(crate) session: Option, + /// Administration-pair assembler for one bounded recyclable face. + peer_assembler: PeerBundleAssembler, + /// Founder-declared exact-mapping band for recyclable pair rings. + peer_map: Option, + /// Completion of the last admission sequence, consumed by the fixed + /// administration pair before another sequence may begin. + last_admission: Option>, + /// Endpoint-bound structural views. The scope comes only from a returned + /// ChanSend capability; no request field selects a row. + invocation_views: [Option; MAX_INVOCATION_VIEWS], + /// The pinned system invocation view — a session-zero self-grant of the + /// persisted root, installed at bootstrap beside the administration + /// grant. The administration session attaches *through a delegation of + /// this view* rather than receiving an ambient private root, so what it + /// creates lands in the tree the durable entry serializes; dynamic peers + /// receive their roots only through endpoint-authenticated `ApplyView`. + system_view: Option, + /// The boot-local workspace anchor: the administration session's grant + /// of the durable `work` directory ([`SYSTEM_WORKSPACE_DIR`]), held for + /// the service's whole life as the delegation parent every private task + /// view's structural root descends from. Grants are boot-local; the + /// directory beneath the persisted root is what survives. + workspace_view: Option, + /// The administration session's writable grant over the shared executable + /// directory (`app/`), captured when it is published. Each terminal's task view + /// is minted ONE READ-only, BIND_EXECUTABLE-free `apps/` child of it at + /// registration — the #605 Stage 2 store view. `None` on a store-less + /// machine, where terminals simply see no projection. + apps_view: Option, + /// The same for the shared library directory (`lib/`): the anchor of + /// each task's READ-only `libs/` view. + libs_view: Option, + /// The same for the shared data directory (`share/`): the anchor of each + /// task's READ-only `share/` view. + share_view: Option, + /// The administration session's writable grant over `pkgs/`; no task + /// view descends from it. + pkgs_store: Option, + /// Private roots stranded by peers that died, awaiting reclamation on + /// the next async dispatch. The peer-death hook is synchronous, and + /// reclaiming inside it meant a nested `block_on` in the serve loop — + /// see `Engine::detach_session_authority`. On the heap, because one + /// death strands its whole derivation closure, which descendants held by + /// other sessions can make as large as the sessions themselves; the hook + /// falls back to draining in place only if the heap cannot hold the + /// queue. + pending_reclaims: Retired, + /// Staged replacements whose sessions died, awaiting release on the next + /// async dispatch beside `pending_reclaims` — the same rule: nothing + /// awaits inside the synchronous peer-death hook. Taken out of the slot + /// at death so a reused slot never inherits a dead peer's stage; a + /// release the provider refuses stays here and is retried, retaining + /// custody. One stage per session bounds it. + pending_aborts: Vec, &'static ServiceAlloc>, + /// The storage backend's last posture snapshot, folded into + /// [`FsRequest::SystemStatus`] answers. The durable entry refreshes it + /// immediately before delegating a status request; the default zeros + /// honestly describe the bare volatile posture every other harness runs. + backend: FsBackendStatus, +} + +/// What the mounted backend reports about itself for `SystemStatus` — set +/// by the layer that owns the mount (the durable entry), consumed by the +/// serving frontend, never derived from wire input. +#[derive(Clone, Copy, Debug, Default)] +pub struct FsBackendStatus { + /// A durable volume is mounted (false: the RAM posture). + pub durable: bool, + /// The durable mount recovered degraded and is read-only. + pub degraded_read_only: bool, + /// The realized interrupt route is held (false: polled). + pub route_held: bool, + /// The mounted block channel crossed its one-way lost boundary. + pub channel_lost: bool, + /// The controller holds a host memory buffer for its mapping tables. + pub host_memory_buffer: bool, + /// The readiness note ([`crate::durable::note`] vocabulary; `0` is the + /// fully healthy durable posture). + pub note: u64, + /// Conservative user-region lifecycle (`FsSystemStatusWire::LIFECYCLE_*`). + pub user_lifecycle: u16, + /// Rebuildable system-region lifecycle (`FsSystemStatusWire::LIFECYCLE_*`). + pub system_lifecycle: u16, + /// Whole-tree serializations completed since bring-up. + pub commits: u64, + /// The block provider's operation counters. + pub provider: crate::provider::BlockProviderStats, + /// The block channel's submission/completion counters. + pub channel: kore_driver::block::ChannelStats, + /// The whole namespace, authorized mount region, and allocator pressure. + /// Empty in the volatile posture. + pub storage: kore_abi::monitor::FilesystemStorageSnapshot, + /// Backend waits that genuinely parked on the realized interrupt route. + pub route_parks: u64, + /// Route parks the interrupt actually woke. A gap against `route_parks` + /// is delivery the route did not make. + pub route_fires: u64, + /// Waits bounded by a clock deadline with no route to park on — the + /// route-less posture on a machine whose clock is readable. + pub deadline_waits: u64, + /// Waits bounded only by a count of spin hints, because no clock could + /// supply a deadline at all. + pub spin_waits: u64, + /// Route-less turns that slept this core on the completion queue's own + /// cache line — the monitored fast tier's numerator. Always zero under an + /// emulator, which implements no user-mode monitor. + pub monitor_sleeps: u64, + /// Route-less turns that reached the monitored tier at all — its + /// denominator, counted whether or not the machine could sleep. + pub monitor_turns: u64, +} + +const MAX_INVOCATION_VIEWS: usize = 16; + +// The most edges this service's own views make one session hold: a task +// view's root, its three store views (`apps`, `libs`, `shared`), and a +// directory pinned beneath it for every invocation view at once. Grants other +// views send it share the rest of its edges. +const _: () = assert!( + 1 + 3 + MAX_INVOCATION_VIEWS <= crate::bounds::MAX_EDGES_PER_SESSION, + "a task view must hold its root, its store views and every pinned directory" +); + +#[derive(Debug)] +struct InvocationView { + scope: u64, + grant: GrantId, + rights: Rights, + task_session: SessionId, + owner: InvocationViewOwner, + canary: Option, + /// The view's durable anchoring beneath the administration workspace, + /// when ambient policy established one: the home directory's name (for + /// same-boot duplicate refusal) and the administration session's anchor + /// grant of it (the delegation parent of this view's root, revoked when + /// the view is reaped — the directory itself is a fact of the persisted + /// tree and survives). + home: Option<(Name, GrantId)>, +} + +#[derive(Debug)] +enum InvocationViewOwner { + /// A narrowed cwd pinned below the stable task view (or the one explicit + /// system bootstrap view). + Pinned { issuer: SessionId }, + /// The stable private root that defines one task's filesystem namespace. + Private { session: SessionId }, +} + +/// What the service keeps for one session slot, beside the engine's +/// authority for it. +struct SessionSlot { + /// What the client declared about its window. + attached: Option, + /// The metal mapping, paired with the affine returned capability that + /// keeps its backing live. Host tests pass an ordinary `DataWindow` + /// directly and therefore leave it empty. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + mapped: Option, + /// The exact host-authored `Serve::via` of the recyclable peer that owns + /// the slot. A fixed pair's slot keeps `None`; generations stop a late + /// request or close from an old occupant naming its replacement. + via: Option, + /// The stable task view the slot's command pair was applied to. Set only + /// by `ApplyView` after endpoint authentication; cleared with the exact + /// peer generation. + task_view: Option, + /// Holding a [`FsRequest::DeferCommits`] window open. Per session by + /// construction: a bulk writer defers only its own commits, never + /// another client's durability. + deferring: bool, + /// The session's one private provider placement. The service already + /// serializes dispatch; no additional lock or namespace object. + replacement: Option>, +} + +impl SessionSlot { + const fn vacant() -> Self { + Self { + attached: None, + mapped: None, + via: None, + task_view: None, + deferring: false, + replacement: None, + } + } +} + +/// The heap one session slot's [`SessionSlot`] costs in the largest +/// composition's frontend: the durable engines' block keys, or the RAM +/// engines' where those are larger. +pub(crate) const SESSION_SLOT_BYTES: usize = { + let block = core::mem::size_of::>(); + let ram = core::mem::size_of::>(); + if block > ram { block } else { ram } +}; + +/// [`SessionSlot`] for every session slot used so far, indexed by slot. +/// +/// Grown on the heap to the highest slot a session has used, never sized by +/// [`MAX_SESSIONS`]: the serve host reuses its lowest free slot, so the +/// table follows the most sessions attached at once, as the engine's +/// per-slot authority arenas do. A slot past the table holds a vacant +/// record. Growth is the one fallible step, and it happens where a session +/// first needs a record — a peer's admission above all, which a heap that +/// cannot grow refuses whole. +struct SessionSlots(Vec, &'static ServiceAlloc>); + +impl SessionSlots { + fn new() -> Self { + Self(Vec::new_in(crate::service_alloc())) + } + + fn get(&self, slot: usize) -> Option<&SessionSlot> { + self.0.get(slot) + } + + fn get_mut(&mut self, slot: usize) -> Option<&mut SessionSlot> { + self.0.get_mut(slot) + } + + /// `slot`'s record, growing the table to reach it: [`FsError::Range`] + /// past every session, [`FsError::OutOfHandles`] when the heap cannot + /// hold the table. + fn claim(&mut self, slot: usize) -> FsResult<&mut SessionSlot> { + if slot >= MAX_SESSIONS { + return Err(FsError::Range); + } + if slot >= self.0.len() { + self.0 + .try_reserve(slot + 1 - self.0.len()) + .map_err(|_| FsError::OutOfHandles)?; + self.0.resize_with(slot + 1, SessionSlot::vacant); + } + Ok(&mut self.0[slot]) + } + + /// Records whose state passes `filter`, for the diagnostic snapshot. + fn count(&self, filter: impl Fn(&SessionSlot) -> bool) -> u16 { + self.0.iter().filter(|state| filter(state)).count() as u16 + } +} + +/// One retained, mapped client data window. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +struct MappedWindow { + /// Owns the delivered `LeasedRegion` record and its runtime slot + /// reservation for exactly as long as the service may touch `base`. + _owner: ReturnedCapability, + base: core::ptr::NonNull, + len: usize, +} + +/// Borrow-free description of one retained data-window mapping. +/// +/// The affine [`ReturnedCapability`] that keeps this span live remains in the +/// owning [`FsService`]. The dual-volume router may use the span sequentially +/// for either engine, but never stores it or lets both engines borrow it at +/// once. +#[derive(Clone, Copy)] +pub(crate) struct MappedWindowSpan { + pub(crate) base: core::ptr::NonNull, + pub(crate) len: usize, +} + +impl FsService

{ + /// The per-request transfer limit this service advertises at window + /// attach: the provider's own, narrowed to the wire. + pub(crate) fn transfer_limit(&self) -> u32 { + u32::try_from(self.engine.provider().limits().max_transfer).unwrap_or(u32::MAX) + } + + pub fn new(engine: Engine

) -> Self { + Self { + engine, + slots: SessionSlots::new(), + session: None, + peer_assembler: PeerBundleAssembler::new(0, true), + peer_map: None, + last_admission: None, + invocation_views: core::array::from_fn(|_| None), + system_view: None, + workspace_view: None, + apps_view: None, + libs_view: None, + share_view: None, + pkgs_store: None, + pending_reclaims: Vec::new_in(crate::service_alloc()), + pending_aborts: Vec::new_in(crate::service_alloc()), + backend: FsBackendStatus::default(), + } + } + + /// Replace the backend posture snapshot folded into `SystemStatus` + /// answers. The mount owner calls this immediately before delegating a + /// status request, so the record reads as one moment. + pub fn set_backend_status(&mut self, backend: FsBackendStatus) { + self.backend = backend; + } + + fn task_view_for_session(&self, session: SessionId) -> Option { + self.slots + .get(session.index() as usize) + .and_then(|state| state.task_view) + } + + fn task_view_for_scope(&self, scope: u64) -> Option { + self.invocation_views.iter().flatten().find_map(|view| { + (view.scope == scope && matches!(view.owner, InvocationViewOwner::Private { .. })) + .then_some(view.task_session) + }) + } + + /// [`FsRequest::MatchView`] once its anchor has authenticated `scope`: + /// whether `handle` routes to the object the view bound under `scope` — + /// a task's root, or a cwd pinned below it — is rooted at. + fn match_view(&self, session: SessionId, handle: Handle, scope: u64) -> FsResponse { + let bound = self + .invocation_views + .iter() + .flatten() + .find_map(|view| (view.scope == scope).then_some((view.grant, view.task_session))); + // A selector this store binds no view under names a directory + // elsewhere — another store's, or one since retired — so the route + // cannot be naming it: the expectation no longer holds. + let Some((view, task)) = bound else { + return FsResponse::Refused(FsError::Conflict); + }; + match self.engine.matches_view(session, handle, view, task) { + Ok(true) => FsResponse::Done, + Ok(false) => FsResponse::Refused(FsError::Conflict), + Err(error) => FsResponse::Refused(error), + } + } + + fn register_invocation_view_inner( + &mut self, + session: SessionId, + handle: Handle, + rights: Rights, + scope: u64, + canary: Option, + ) -> FsResult<()> { + self.reap_dead_invocation_views(); + if self + .invocation_views + .iter() + .flatten() + .any(|view| view.scope == scope) + { + if let Some(canary) = canary { + let _ = canary.try_retire(); + } + return Err(FsError::AlreadyExists); + } + let Some(slot) = self.invocation_views.iter().position(Option::is_none) else { + if let Some(canary) = canary { + let _ = canary.try_retire(); + } + return Err(FsError::OutOfEdges); + }; + let (grant, issuer) = match self.engine.pin_view(session, handle, rights) { + Ok(pinned) => pinned, + Err(error) => { + if let Some(canary) = canary { + let _ = canary.try_retire(); + } + return Err(error); + } + }; + self.invocation_views[slot] = Some(InvocationView { + scope, + grant, + rights, + task_session: issuer, + owner: InvocationViewOwner::Pinned { issuer }, + canary, + home: None, + }); + Ok(()) + } + + fn register_invocation_view( + &mut self, + session: SessionId, + handle: Handle, + rights: Rights, + scope: u64, + ) -> FsResult<()> { + self.register_invocation_view_inner(session, handle, rights, scope, None) + } + + fn register_invocation_endpoint( + &mut self, + session: SessionId, + handle: Handle, + rights: Rights, + returned: ReturnedCapability, + ) -> FsResult<()> { + let canary = returned + .try_into_chan_send(crate::FsViewCapability::INVOCATION_RIGHTS) + .map_err(|_| FsError::AccessDenied)?; + let scope = verbs::capability_scope_rid( + canary.slot(), + kore_abi::cap::ResourceKind::ChanSend, + kore_abi::cap::Rights::ACCESS, + ); + let Some(scope) = scope else { + let _ = canary.try_retire(); + return Err(FsError::AccessDenied); + }; + self.register_invocation_view_inner(session, handle, rights, scope, Some(canary)) + } + + fn reap_dead_invocation_views(&mut self) { + for slot in 0..self.invocation_views.len() { + // Only the kernel can say a canary died. Its binder's revocation + // retires an ancestor tag and leaves this held record, scope id + // and all, exactly as it was, so reading the record would keep + // every dead view registered: its pinned grant held, its table + // slot taken, and its scope id answering `AlreadyExists` once the + // binder's channel ids come round again. + let dead = self.invocation_views[slot] + .as_ref() + .and_then(|view| view.canary.as_ref()) + .is_some_and(|canary| { + verbs::probe_chan_send_lineage(canary.slot()) == verbs::ChanLineage::Dead + }); + if dead && let Some(mut view) = self.invocation_views[slot].take() { + // The dead canary's name is released, not dropped: retiring + // a revoked lineage settles `AlreadyDead` and frees the slot. + if let Some(canary) = view.canary.take() { + let _ = canary.try_retire(); + } + match view.owner { + InvocationViewOwner::Pinned { issuer } => { + let _ = block_on(self.engine.revoke(issuer, view.grant)); + } + InvocationViewOwner::Private { session } => { + let _ = block_on(self.engine.detach_session(session)); + } + } + // The anchor grant outlives the view's own root (it is that + // root's delegation parent), so it is released only after + // the session teardown above retired the descendant edges. + // The home directory stays: it is a fact of the persisted + // tree, and the next registration of this name reopens it. + if let Some((_, anchor)) = view.home { + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let _ = block_on(self.engine.revoke(admin, anchor)); + } + } + } + } + + /// The standing durable-workspace anchor, when bootstrap has published + /// it — the delegation parent for every private task view's root. + fn workspace_grant(&self) -> Option { + self.workspace_view + } + + /// Open — or, on a blank tree, create — the view's home directory + /// `work/` and hand back the administration session's anchor grant + /// of it: the delegation parent the private view's root will descend + /// from. Reuse-before-create is the persistence half of the contract + /// (a recovered volume already holds the directory and the task's + /// files under it); the distinct directory per name is the isolation + /// half (two tasks never share a home unless someone grants across). + fn anchor_view_home(&mut self, workspace: GrantId, name: &Name) -> FsResult { + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let work = self + .engine + .open_grant(admin, workspace, Rights::ALL)? + .handle; + let home = match self.engine.open_at(admin, work, name, Rights::ALL) { + Ok(opened) if opened.kind == ObjectKind::Directory => Ok(opened), + Ok(opened) => { + // A file squatting on the home name: refuse rather than + // root a task's namespace inside something that cannot + // contain names. + let _ = self.engine.close(admin, opened.handle); + Err(FsError::NotDirectory) + } + Err(FsError::NotFound) => { + block_on(self.engine.create(admin, work, name, ObjectKind::Directory)) + } + Err(error) => Err(error), + }; + let anchored = home.and_then(|home| { + let anchored = self + .engine + .grant(admin, home.handle, admin, Rights::ALL, None); + let _ = self.engine.close(admin, home.handle); + anchored + }); + let _ = self.engine.close(admin, work); + anchored + } + + fn register_private_invocation_view( + &mut self, + scope: u64, + name: &Name, + rights: Rights, + canary: Option, + ) -> FsResult<()> { + self.reap_dead_invocation_views(); + if rights.is_empty() + || self.invocation_views.iter().flatten().any(|view| { + view.scope == scope || view.home.as_ref().is_some_and(|(home, _)| home == name) + }) + { + if let Some(canary) = canary { + let _ = canary.try_retire(); + } + return Err(FsError::AlreadyExists); + } + let Some(slot) = self.invocation_views.iter().position(Option::is_none) else { + if let Some(canary) = canary { + let _ = canary.try_retire(); + } + return Err(FsError::OutOfEdges); + }; + let session = (FIRST_TASK_VIEW_SESSION..MAX_SESSIONS) + .map(|index| SessionId::new(index as u16)) + .find(|candidate| { + !self.invocation_views.iter().flatten().any(|view| { + matches!( + view.owner, + InvocationViewOwner::Private { session } if session == *candidate + ) + }) + }); + let Some(session) = session else { + if let Some(canary) = canary { + let _ = canary.try_retire(); + } + return Err(FsError::OutOfEdges); + }; + // The #575 invariant in the invocation-view model's own terms: a + // task view's namespace must be a fact of the persisted tree, so + // the fresh session roots in a delegation of its own durable home + // directory (`work/` under session zero) rather than an + // ambient private root — `commit_engine` serializes exactly what + // session zero reaches, and an ambient root's objects reached + // their data blocks and lost their names at every remount. One + // home per name keeps distinct tasks' namespaces distinct (the + // two-terminal isolation contract); reopening the same name across + // boots is precisely what makes a task's files survive. Host + // fixtures that never establish ambient policy keep the isolated + // ambient posture, so isolation tests stay honest either way. + let home = match self.workspace_grant() { + Some(workspace) => match self.anchor_view_home(workspace, name) { + Ok(anchor) => Some((*name, anchor)), + Err(error) => { + if let Some(canary) = canary { + let _ = canary.try_retire(); + } + return Err(error); + } + }, + None => None, + }; + let attached = match home { + Some((_, anchor)) => self + .engine + .attach_delegated_session(session, anchor, rights), + None => block_on(self.engine.attach_session(session)), + }; + let unwound = |service: &mut Self, canary: Option| { + if let Some((_, anchor)) = home { + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let _ = block_on(service.engine.revoke(admin, anchor)); + } + if let Some(canary) = canary { + let _ = canary.try_retire(); + } + }; + if let Err(error) = attached { + unwound(self, canary); + return Err(error); + } + let grant = match self.engine.task_view_root(session) { + Ok(grant) => grant, + Err(error) => { + let _ = block_on(self.engine.detach_session(session)); + unwound(self, canary); + return Err(error); + } + }; + // #605 Stage 2: mint this terminal the canonical READ-only, + // BIND_EXECUTABLE-free `apps` view onto the shared store object (no new + // object, no byte copy). ONE name only: the flat `app` spelling rode + // beside it while callers migrated, and showing one object under two + // names reads on a live machine as a stale projection that survived a + // boot. BEST-EFFORT and strictly NON-FATAL: a collision in a private + // home or edge exhaustion must never fail terminal founding — a terminal + // without the view still boots, it just cannot see the store. + // READ_ONLY strips WRITE, MUTATE_NAMESPACE, SHARE, and + // BIND_EXECUTABLE, so a terminal can read and run store executables but + // never write, relink, or bless them. + let _ = self.install_apps_view(session); + let _ = self.install_libs_view(session); + let _ = self.install_share_view(session); + self.invocation_views[slot] = Some(InvocationView { + scope, + grant, + rights, + task_session: session, + owner: InvocationViewOwner::Private { session }, + canary, + home, + }); + Ok(()) + } + + pub(crate) fn register_system_invocation_view( + &mut self, + scope: u64, + rights: Rights, + ) -> FsResult<()> { + if rights.is_empty() { + return Err(FsError::AccessDenied); + } + let system = SessionId::new(0); + // Pinning is itself a delegation, so the short-lived source handle + // must carry SHARE. The pinned view receives exactly `rights`; source + // authority is never copied into the descendant. + let source_rights = rights.union(Rights::SHARE); + let root = self.engine.open_root(system, source_rights)?; + let result = self.register_invocation_view(system, root.handle, rights, scope); + let _ = self.engine.close(system, root.handle); + // A peer applied onto this view resolves names through ITS task + // session — the system session — so the two system views every + // task carries must stand there too, exactly as a private view + // installs them on its own session. Init's post-ready witnesses + // are the consumers: `require "lua_demo"` walks `libs/` from a + // session anchored here. Best-effort like the private path. + if result.is_ok() { + let _ = self.install_apps_view(system); + let _ = self.install_libs_view(system); + let _ = self.install_share_view(system); + } + result + } + + pub(crate) fn apply_invocation_view( + &mut self, + scope: u64, + peer: PeerId, + profile_rights: Rights, + ) -> FsResult<()> { + self.reap_dead_invocation_views(); + if self.peer_is_retired(peer) { + return Err(FsError::AccessDenied); + } + let (grant, view_rights, task_session) = self + .invocation_views + .iter() + .flatten() + .find(|view| view.scope == scope) + .map(|view| (view.grant, view.rights, view.task_session)) + .ok_or(FsError::AccessDenied)?; + let rights = view_rights.intersection(profile_rights); + if rights.is_empty() { + return Err(FsError::AccessDenied); + } + // A live peer's slot has its record: admission claimed it. + let state = self + .slots + .get_mut(peer.slot()) + .ok_or(FsError::AccessDenied)?; + self.engine + .install_view_root(SessionId::new(peer.slot() as u16), grant, rights)?; + state.task_view = Some(task_session); + Ok(()) + } + + /// Enable the generic recyclable-pair host over one founder-stamped + /// exact-mapping band. The map is service-local VA geometry, never wire + /// data, and therefore cannot be selected by a client. + pub(crate) fn configure_peer_map(&mut self, map: PeerMapPlan) { + self.peer_map = Some(map); + } + + /// Resolve a host-authored arrival identity to the bounded filesystem + /// session it owns. Static founder pairs map one-to-one to the equally + /// numbered sessions; recyclable peers must match the exact current + /// generation. Pair one is therefore exec's administration session, not + /// an alias for pair zero's persisted handles. + pub(crate) fn session_for_via(&self, via: u16) -> Option { + if via <= crate::service::contract::ADMIN_VIA { + return Some(SessionId::new(via)); + } + // The arrival names its slot; only the slot's current occupant + // matches it exactly. + let peer = PeerId::from_arrival(via)?; + (!self.peer_is_retired(peer)).then(|| SessionId::new(peer.slot() as u16)) + } + + /// Complete service-local policy after the generic host installed a + /// peer. Returning `false` asks the host to tear the just-installed pair + /// back down before it can issue a request. + pub(crate) fn peer_admitted(&mut self, result: Result) -> bool { + let Ok(peer) = result else { + self.last_admission = Some(result); + return true; + }; + let installed = self.install_peer(peer); + self.last_admission = Some(installed.map(|()| peer)); + installed.is_ok() + } + + /// Mirror a host-admitted pair into a secondary engine without claiming + /// ownership of the admission protocol or its one completion slot. The + /// primary frontend consumed the returned endpoint bundle; this engine + /// receives only the host-authored peer identity and creates an empty + /// pair-bound session into which explicit system grants may later land. + pub(crate) fn mirror_peer_admitted( + &mut self, + peer: PeerId, + ) -> Result<(), PeerAdmissionRefusal> { + self.install_peer(peer) + } + + fn install_peer(&mut self, peer: PeerId) -> Result<(), PeerAdmissionRefusal> { + let slot = peer.slot(); + if slot == 0 + || self + .slots + .get(slot) + .is_some_and(|state| state.via.is_some()) + { + // An occupied slot is a registration a previous peer never + // retired, not a full table. Reporting it as capacity sent the + // caller to retry a table with room, and surfaced on the console + // as the rung that means exhaustion. + return Err(PeerAdmissionRefusal::PolicyCollision); + } + // The slot's record and the session's storage are the admission's + // whole cost; a heap that cannot hold either refuses it as the full + // table it is, before anything is published. + let state = self + .slots + .claim(slot) + .map_err(|_| PeerAdmissionRefusal::Capacity)?; + match self + .engine + .attach_empty_session(SessionId::new(slot as u16)) + { + Ok(()) => {} + Err(FsError::OutOfHandles) => return Err(PeerAdmissionRefusal::Capacity), + Err(_) => return Err(PeerAdmissionRefusal::PolicyUnavailable), + } + state.via = Some(peer.arrival()); + Ok(()) + } + + /// Give a fixed session its root — the invocation-view posture: a + /// delegation of the pinned system view, so what the session creates + /// lands in the persisted tree. Before bootstrap pins the view (host + /// fixtures that never establish ambient policy) every session keeps the + /// ambient private-root posture so isolation tests stay honest. + /// + /// Dynamic peers never root here: admission parks them in the attach + /// transition and only endpoint-authenticated `ApplyView` installs their + /// structural root. A request arriving on a parked pair maps that + /// transition to `Busy` below and is refused, so the pair-timing race + /// can never mint an ambient root for an un-viewed client. + pub(crate) fn ensure_dynamic_session(&mut self, session: SessionId) -> bool { + let fixed = session == SessionId::new(0); + match self.system_view { + Some(view) if !fixed => self + .engine + .attach_delegated_session(session, view, Rights::ALL) + .or_else(|error| match error { + FsError::AlreadyExists => Ok(()), + other => Err(other), + }) + .is_ok(), + _ => self.ensure_session(session), + } + } + + /// Drop exactly the state belonging to the retired generation. A late + /// close for an earlier occupant is inert. + pub(crate) fn peer_closed(&mut self, via: u16, _reason: PeerClose) -> bool { + let Some(peer) = PeerId::from_arrival(via).filter(|peer| !self.peer_is_retired(*peer)) + else { + return true; + }; + let slot = peer.slot(); + // A staged replacement dies with its session. Its placement is + // released on the next dispatch, never here: the same rule as the + // stranded roots below. Taking it out of the slot now keeps a + // reused slot from inheriting a dead peer's stage. + if let Some(stage) = self + .slots + .get_mut(slot) + .and_then(|state| state.replacement.take()) + { + if self.pending_aborts.try_reserve(1).is_ok() { + self.pending_aborts.push(stage); + } else if block_on(self.engine.abort_replace(&stage)).is_err() { + // A heap that cannot hold the list releases in place, as + // the reclaim fallback below does, and keeps the stage if + // the provider refuses. + if let Some(state) = self.slots.get_mut(slot) { + state.replacement = Some(stage); + } + return false; + } + } + let Ok(stranded) = self + .engine + .detach_session_authority(SessionId::new(slot as u16)) + else { + return false; + }; + self.queue_reclaims(stranded); + // Everything else the slot held goes with its occupant: the window + // and its mapping, the task view, and the commit hold — a client + // that died inside a bulk-write window must not leave the slot + // deferring, or the next occupant would inherit held commits it never + // asked for. That drops only the durability hold; it cannot roll back + // mutations already visible in the live engine. + if let Some(state) = self.slots.get_mut(slot) { + *state = SessionSlot::vacant(); + } + true + } + + /// Queue the roots a dead peer stranded for the next async dispatch, + /// where reclamation's contention budget means something. A heap that + /// cannot hold the queue drains it here instead, so no root is ever + /// dropped on the floor, at the old nested cost. + fn queue_reclaims(&mut self, stranded: Retired) { + if self.pending_reclaims.is_empty() { + self.pending_reclaims = stranded; + return; + } + if self.pending_reclaims.try_reserve(stranded.len()).is_ok() { + self.pending_reclaims.extend_from_slice(&stranded); + return; + } + block_on(self.engine.reclaim_stranded(&self.pending_reclaims)); + self.pending_reclaims.clear(); + block_on(self.engine.reclaim_stranded(&stranded)); + } + + /// Whether one exact host-authored peer generation is no longer live. + /// A replacement in the same bounded cell is intentionally different: + /// confirming the old ticket must never wait on or disturb the new peer. + pub(crate) fn peer_is_retired(&self, peer: PeerId) -> bool { + self.slots.get(peer.slot()).and_then(|state| state.via) != Some(peer.arrival()) + } + + /// Administration control plane for recyclable clients. Ordinary + /// filesystem requests never enter this assembler. + pub(crate) fn admit_peer_component( + &mut self, + part: PeerBundlePart, + announced: kore_abi::session::SlotRef, + sv: &mut Serve<'_, FsResponse>, + ) -> FsResponse { + if sv.via() != crate::service::contract::ADMIN_VIA { + return FsResponse::Refused(FsError::AccessDenied); + } + // A `Ring` is the boundary between bundles, so one arriving while the + // assembler still owns components proves the previous assembly was + // abandoned — its asker spent its bounded re-ask and gave up. Retire + // what it left, and refuse the replacement while any of those owners + // is still closing (`PeerBundleAssembler::retire_incomplete`). + // + // Without this the leftovers are permanent: the next launch's matching + // part answers `Duplicate`, the bundle never completes, no verdict is + // ever published, and every later launch that needs this face refuses + // for the rest of the boot. + // + // Admin only, and checked before anything else touches the assembler: + // a foreign via must never retire the components the administration + // pair is still assembling. + if matches!(part, PeerBundlePart::Ring) + && !self.peer_assembler.is_empty() + && !self.peer_assembler.retire_incomplete() + { + return FsResponse::Refused(FsError::Conflict); + } + if self.last_admission.is_some() { + return FsResponse::Refused(FsError::Conflict); + } + let Some(map) = self.peer_map else { + return FsResponse::Refused(FsError::Unsupported); + }; + let Some(mut input) = sv.returned_input(announced) else { + return FsResponse::Refused(FsError::NoWindow); + }; + let Some(capability) = input.capability() else { + return FsResponse::Refused(FsError::NoWindow); + }; + if let Err(reason) = self.peer_assembler.accepts(part, capability) { + return FsResponse::Refused(peer_assembly_error(reason)); + } + let capability = input + .take() + .expect("validated returned peer component remains present"); + let bundle = self + .peer_assembler + .push(part, capability) + .expect("same synchronous peer validation cannot change"); + if let Some(bundle) = bundle { + sv.admit_peer(bundle, map); + } + FsResponse::Done + } + + /// Consume the completion published by the outer serving host after it + /// mapped (or refused) the preceding peer bundle. + pub(crate) fn take_peer_admission(&mut self) -> FsResponse { + match self.last_admission.take() { + Some(Ok(peer)) => FsResponse::PeerAdmitted { peer }, + Some(Err(reason)) => FsResponse::Refused(peer_admission_error(reason)), + // No verdict, and the assembler still owns components: this + // bundle never completed, so `admit_peer` was never called and + // no verdict can EVER be published. `NotFound` here would be a + // lie the asker waits on forever — and exec fences new + // provisioning while an id-less face exists, so that wait + // refuses every later launch that needs this face. Retire what + // the abandoned assembly still owns and answer terminally, which + // is exec's signal to let the peer go. + None if !self.peer_assembler.is_empty() => { + let _ = self.peer_assembler.retire_incomplete(); + FsResponse::Refused(FsError::Conflict) + } + // No verdict and an EMPTY assembler: the bundle completed and + // the host has not drained its admission action yet. One IS + // coming, so this is an ordinary "not yet" — settling here would + // orphan the face the host is about to admit. + None => FsResponse::Refused(FsError::NotFound), + } + } + + /// Whether `session` is holding its commits for a bulk write. + pub fn is_deferring(&self, session: SessionId) -> bool { + self.slots + .get(session.index() as usize) + .is_some_and(|state| state.deferring) + } + + /// The engine, for startup wiring (attaching sessions) and diagnostics. + pub fn engine(&mut self) -> &mut Engine

{ + &mut self.engine + } + + /// The engine, read-only. + pub fn engine_ref(&self) -> &Engine

{ + &self.engine + } + + /// The device blocks of `session`'s lent version `lease`, for a read + /// grant (`Engine::lent_blocks`). + pub(crate) fn lent_blocks( + &self, + session: SessionId, + lease: crate::extraction::VersionLease, + ) -> FsResult> { + let owner = crate::extraction::ExtractionOwner::new(u64::from(session.index())); + self.engine + .lent_blocks(owner, lease) + .map_err(extraction_error) + } + + /// Release the engine — restart simulations and provider transitions. + pub(crate) fn into_engine(self) -> Engine

{ + self.engine + } + + /// Give `session` a private root if it has none. + /// + /// The postcondition both callers need is "this session has a root", not + /// "a root was created here" — the one-pair [`contract::bind`] path + /// establishes session 0 before `ServeHost` ever attaches pair 0, so an + /// already-attached session is success, not a double-attach bug. Every + /// other refusal (an out-of-range session, an exhausted edge arena) is a + /// refusal, because it leaves the client rootless. + /// + /// [`contract::bind`]: crate::service::contract + pub(crate) fn ensure_session(&mut self, session: SessionId) -> bool { + match block_on(self.engine.attach_session(session)) { + Ok(()) | Err(FsError::AlreadyExists) => true, + Err(_) => false, + } + } + + /// Install the sole boot-policy bridge from the persisted system + /// namespace (session zero) to exec's fixed administration session. + /// + /// The named source grant carries `READ|SHARE|BIND_EXECUTABLE`: exec may + /// inspect immutable source bytes, nominate an exact revision to + /// ImageService, and delegate a read-only `app` view — the + /// executable-source right is deliberately attenuated out of that child + /// view, and exec must discover its pair-bound [`GrantId`] through + /// `ListGrants` before it can open anything. The administration + /// session's *structural root*, though, is a delegation of the persisted + /// system root (the invocation-view posture): the launcher-owned layout + /// and every view it grants a child are facts of the durable tree, so + /// task files survive a reboot rather than dying with a private root. + /// This keeps the pair-is-session rule intact without giving pair one a + /// second ambient root that aliases session zero's handles. + pub(crate) fn establish_admin_namespace(&mut self) -> bool { + let system = SessionId::new(0); + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + if !self.ensure_session(system) { + return false; + } + if !self.establish_system_view() { + return false; + } + // The administration session enters through the system view like + // every dynamic session: exec's launcher-owned layout objects + // (`data`/`config`/`tmp`/`shared`) and the views it grants children + // are thereby facts of the persisted tree, which is what lets a + // task's files survive a reboot. The named source grant below stays + // the executable-acquisition bridge. + if !self.ensure_dynamic_session(admin) { + return false; + } + true + } + + pub(crate) fn establish_admin_source(&mut self) -> bool { + if !self.establish_admin_namespace() { + return false; + } + let system = SessionId::new(0); + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let Ok(display) = Name::new(SYSTEM_SOURCE_GRANT) else { + return false; + }; + let Ok(grants) = self.engine.grants(admin) else { + return false; + }; + if grants.iter().any(|grant| { + grant.display_name == Some(display) + && grant.kind == crate::authority::AuthorityKind::Grant + && grant.rights + == Rights::READ + .union(Rights::SHARE) + .union(Rights::BIND_EXECUTABLE) + }) { + return true; + } + let Ok(root) = self.engine.open_root(system, Rights::ALL) else { + return false; + }; + let granted = self + .engine + .grant( + system, + root.handle, + admin, + Rights::READ + .union(Rights::SHARE) + .union(Rights::BIND_EXECUTABLE), + Some(display), + ) + .is_ok(); + let _ = self.engine.close(system, root.handle); + granted + } + + /// Pin the system invocation view: a session-zero self-grant of the + /// persisted root, held for the service's whole life as the delegation + /// parent every dynamic peer's structural root descends from (the + /// capability-selected execution branch's `register_system_invocation_ + /// view`, folded into this bootstrap hook because ambient policy is + /// installed exactly once, here). Idempotent like the administration + /// grant above it. + fn establish_system_view(&mut self) -> bool { + if self.system_view.is_some() { + return true; + } + let system = SessionId::new(0); + let Ok(root) = self.engine.open_root(system, Rights::ALL) else { + return false; + }; + let pinned = self + .engine + .grant(system, root.handle, system, Rights::ALL, None); + let _ = self.engine.close(system, root.handle); + match pinned { + Ok(view) => { + self.system_view = Some(view); + true + } + Err(_) => false, + } + } + + /// Publish the shared executable directory (`app` beneath the persisted + /// root) and hand the administration session a writable grant over it. + /// + /// This is the store half of the archive/filesystem unification: the boot + /// archive is a seed, and the durable tree is where executables actually + /// live. Exec discovers the grant by its display name and copies each + /// registered member in once; a later boot finds the directory already + /// populated and writes nothing. + /// + /// Idempotent in both directions, for the reason the workspace is: on a + /// recovered volume the directory and its contents are already there and + /// are reused — never re-created, which would strand the executables + /// beneath an orphaned object — and a second call finds the standing + /// grant. + pub(crate) fn establish_admin_apps(&mut self) -> bool { + match self.establish_admin_directory(SYSTEM_APPS_DIR, SYSTEM_APPS_GRANT) { + Some(grant) => { + // The terminal app-view anchor: each terminal's READ-only + // `apps` descends from this admin-held grant. + self.apps_view = Some(grant); + true + } + None => false, + } + } + + /// The shared library directory, `lib/` — the same bring-up as `app/` + /// (create-or-adopt, one writable admin grant), anchoring every task's + /// READ-only `libs` view. + pub(crate) fn establish_admin_libs(&mut self) -> bool { + match self.establish_admin_directory(SYSTEM_LIBS_DIR, SYSTEM_LIBS_GRANT) { + Some(grant) => { + self.libs_view = Some(grant); + true + } + None => false, + } + } + + /// The shared data directory, `share/` — the same bring-up as `lib/` + /// (create-or-adopt, one writable admin grant), anchoring every task's + /// READ-only `share` view. + pub(crate) fn establish_admin_share(&mut self) -> bool { + match self.establish_admin_directory(SYSTEM_SHARE_DIR, SYSTEM_SHARE_GRANT) { + Some(grant) => { + self.share_view = Some(grant); + true + } + None => false, + } + } + + /// The package directory, `pkgs/` — admin-writable, never viewed. + pub(crate) fn establish_admin_pkgs(&mut self) -> bool { + match self.establish_admin_directory(SYSTEM_PKGS_DIR, SYSTEM_PKGS_GRANT) { + Some(grant) => { + self.pkgs_store = Some(grant); + true + } + None => false, + } + } + + /// Open-or-create `directory` beneath the persisted root and grant it + /// to the administration session under `display` with every right — + /// the one bring-up the three system directories share. Idempotent in + /// both directions: a standing grant is adopted (a recovered volume + /// already carries the directory and its members), and a file squatting + /// on the name refuses rather than scattering members somewhere + /// unexpected. + fn establish_admin_directory( + &mut self, + directory: &str, + display: &str, + ) -> Option { + let system = SessionId::new(0); + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + if !self.ensure_session(system) || !self.ensure_session(admin) { + return None; + } + let (Ok(directory), Ok(display)) = (Name::new(directory), Name::new(display)) else { + return None; + }; + let grants = self.engine.grants(admin).ok()?; + if let Some(found) = grants + .iter() + .find(|grant| grant.display_name == Some(display)) + { + // Reuse across a reboot: the standing grant is the anchor. + return Some(found.grant); + } + let root = self.engine.open_root(system, Rights::ALL).ok()?; + let opened = match self + .engine + .open_at(system, root.handle, &directory, Rights::ALL) + { + Ok(opened) if opened.kind == ObjectKind::Directory => Ok(opened), + Ok(opened) => { + let _ = self.engine.close(system, opened.handle); + Err(FsError::NotDirectory) + } + Err(FsError::NotFound) => { + block_on( + self.engine + .create(system, root.handle, &directory, ObjectKind::Directory), + ) + } + Err(error) => Err(error), + }; + let granted = match opened { + Ok(opened) => { + let granted = self + .engine + .grant(system, opened.handle, admin, Rights::ALL, Some(display)) + .ok(); + let _ = self.engine.close(system, opened.handle); + granted + } + Err(_) => None, + }; + let _ = self.engine.close(system, root.handle); + granted + } + + /// Publish the canonical read-only application view into one already + /// attached session. In the legacy composition the destination is the + /// stable task-view session; in the split composition it is the secondary + /// engine's empty pair-bound session. The latter must first leave its + /// attach transition through an `apps`-rooted READ-only delegation: an + /// empty session cannot receive a named grant, and leaving it transitional + /// would make orderly peer teardown refuse forever. Either way only the + /// application directory becomes reachable—never the ambient system root. + pub(crate) fn install_apps_view(&mut self, session: SessionId) -> bool { + let Some(apps) = self.apps_view else { + return false; + }; + match self + .engine + .install_view_root(session, apps, Rights::READ_ONLY) + { + // Split composition: complete the empty pair-bound session with + // the narrowest structural root before publishing its display + // name. Legacy composition already installed a workspace root. + Ok(()) | Err(FsError::AlreadyExists) => {} + Err(_) => return false, + } + self.install_named_view(session, apps, SYSTEM_APPS_VIEW) + } + + /// The READ-only `libs` view beside `apps` — a named grant onto the + /// shared library directory (the session's root is `apps`'s to + /// install). Best-effort like `apps`: a task without it still boots, + /// it just cannot `require` a native module. + pub(crate) fn install_libs_view(&mut self, session: SessionId) -> bool { + let Some(libs) = self.libs_view else { + return false; + }; + self.install_named_view(session, libs, SYSTEM_LIBS_VIEW) + } + + /// The READ-only `share` view beside `libs` — a named grant onto the + /// shared data directory. Best-effort like `libs`: a task without it + /// still boots, it just cannot read the shipped data (`shared/wasm/…`). + pub(crate) fn install_share_view(&mut self, session: SessionId) -> bool { + let Some(share) = self.share_view else { + return false; + }; + self.install_named_view(session, share, SYSTEM_SHARE_VIEW) + } + + /// Grant `anchor` (an admin-held directory grant) into `session` as a + /// READ-only view displayed `display`, idempotently. READ_ONLY strips + /// WRITE, MUTATE_NAMESPACE, SHARE, and BIND_EXECUTABLE: a task can read + /// and run what the directory holds but never write, relink, or bless it. + fn install_named_view( + &mut self, + session: SessionId, + anchor: crate::GrantId, + display: &str, + ) -> bool { + let Ok(name) = Name::new(display) else { + return false; + }; + let Ok(grants) = self.engine.grants(session) else { + return false; + }; + if grants + .iter() + .any(|grant| grant.display_name == Some(name) && grant.rights == Rights::READ_ONLY) + { + return true; + } + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let Ok(opened) = self + .engine + .open_grant(admin, anchor, Rights::READ.union(Rights::SHARE)) + else { + return false; + }; + let installed = self + .engine + .grant(admin, opened.handle, session, Rights::READ_ONLY, Some(name)) + .is_ok(); + let _ = self.engine.close(admin, opened.handle); + installed + } + + /// Install the writable workspace bridge from the persisted system + /// namespace (session zero) to exec's fixed administration session. + /// + /// Every private task view roots its structural namespace in a + /// delegation of this edge, so it is the route by which *all* ordinary + /// user data is created. It must therefore be rooted inside session zero's + /// tree: [`commit_engine`] serializes exactly what is reachable from that + /// root, and an object reachable from anywhere else is written to its data + /// blocks and then silently forgotten at the next mount — the whole of + /// #575's data loss. + /// + /// The workspace is a **dedicated directory** beneath the system root, not + /// the system root itself: [`SYSTEM_SOURCE_GRANT`] hands exec the same tree + /// read-only to load executable images from, and the two authority edges + /// must not alias one object, or a fault in ordinary file handling could + /// reach the images exec is about to run. + /// + /// Idempotent in both directions, which is what makes it correct across a + /// reboot: on a recovered volume the directory is already there and is + /// reused (never re-created, which would strand the user's data under an + /// orphaned object), and a second call finds the standing grant. + /// + /// [`commit_engine`]: crate::persistence::commit_engine + pub(crate) fn establish_admin_workspace(&mut self) -> bool { + let system = SessionId::new(0); + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + if !self.ensure_session(system) || !self.ensure_session(admin) { + return false; + } + let Ok(directory) = Name::new(SYSTEM_WORKSPACE_DIR) else { + return false; + }; + // Boot-local idempotence: the anchor grant lives exactly as long as + // this service value. The admin session's root IS the persisted + // system root, so the directory name itself already occupies the + // task-visible namespace — the grant therefore carries no display + // name (the name-collision rule would refuse it), and the anchor is + // held by id here instead of rediscovered by name. + if self.workspace_view.is_some() { + return true; + } + let Ok(root) = self.engine.open_root(system, Rights::ALL) else { + return false; + }; + // Reuse before create: `VolumeProbe::Kore` has already restored the + // directory and everything under it, and creating a second edge of the + // same name would either refuse or bury the recovered tree. + let workspace = match self + .engine + .open_at(system, root.handle, &directory, Rights::ALL) + { + Ok(opened) if opened.kind == ObjectKind::Directory => Ok(opened), + Ok(opened) => { + // A non-directory squatting on the name: refuse rather than + // silently relocate the workspace somewhere unexpected. + let _ = self.engine.close(system, opened.handle); + Err(FsError::NotDirectory) + } + // Blank volume. The happy path of `create` for a directory touches + // only the namespace and authority arenas — its `.await`s are all + // on unwind — so it completes on the first poll and `block_on` + // cannot spin here. + Err(FsError::NotFound) => { + block_on( + self.engine + .create(system, root.handle, &directory, ObjectKind::Directory), + ) + } + Err(error) => Err(error), + }; + let granted = match workspace { + Ok(workspace) => { + let granted = self + .engine + .grant(system, workspace.handle, admin, Rights::ALL, None) + .map(|grant| self.workspace_view = Some(grant)) + .is_ok(); + let _ = self.engine.close(system, workspace.handle); + granted + } + Err(_) => false, + }; + let _ = self.engine.close(system, root.handle); + granted + } + + /// What `session` declared about its window, if it has attached one. + pub fn attachment(&self, session: SessionId) -> Option { + self.slots + .get(session.index() as usize) + .and_then(|state| state.attached) + } + + /// Locate the retained mapping owned by this frontend. The router borrows + /// the returned span only while dispatching one serialized request into a + /// sibling engine; the owner and mapping never move. + #[cfg(target_os = "none")] + pub(crate) fn mapped_window_span(&self, session: SessionId) -> Option { + self.slots + .get(session.index() as usize) + .and_then(|state| state.mapped.as_ref()) + .map(|window| MappedWindowSpan { + base: window.base, + len: window.len, + }) + } + + #[cfg(not(target_os = "none"))] + pub(crate) fn mapped_window_span(&self, _session: SessionId) -> Option { + None + } + + /// Adopt one returned `LeasedRegion` as `session`'s sole data window. + /// The mapped length, not the peer's declaration, is authoritative. + #[cfg(target_os = "none")] + pub(crate) fn adopt_window( + &mut self, + session: SessionId, + announced: u32, + owner: ReturnedCapability, + ) -> FsResult<()> { + use kore_rt::cap::verbs; + + // The record first: a heap that cannot hold it refuses before the + // view is taken. + let state = self.slots.claim(session.index() as usize)?; + if state.mapped.is_some() { + return Err(FsError::AlreadyExists); + } + let slot = owner.slot().ok_or(FsError::NoWindow)?; + let view = verbs::take_view(slot).ok_or(FsError::NoWindow)?; + let mapped = verbs::map_view(view).ok_or(FsError::NoWindow)?; + let len = mapped.len(); + if len == 0 || len > u32::MAX as usize { + return Err(FsError::Range); + } + let base = mapped.cast::(); + state.mapped = Some(MappedWindow { + _owner: owner, + base, + len, + }); + state.attached = Some(WindowAttachment { + slot: announced, + bytes: len as u32, + }); + Ok(()) + } + + /// Host builds have no capability arena or receiver mapping operation. + #[cfg(not(target_os = "none"))] + pub(crate) fn adopt_window( + &mut self, + _session: SessionId, + _announced: u32, + _owner: ReturnedCapability, + ) -> FsResult<()> { + Err(FsError::NoWindow) + } + + /// Dispatch through this session's retained metal mapping. + #[cfg(target_os = "none")] + pub(crate) fn dispatch_mapped( + &mut self, + session: SessionId, + request: &FsRequest, + ) -> FsResponse { + let mut mapped = self.mapped_window_span(session).map(|span| { + // SAFETY: `adopt_window` mapped this exact live returned view and + // retained its affine owner in the session's slot record. + // `ServeHost` drives one request at a time, and the client + // touches its sibling view only before publication or after the + // correlated reply. + DataWindow::new(unsafe { + core::slice::from_raw_parts_mut(span.base.as_ptr(), span.len) + }) + }); + self.dispatch(session, request, mapped.as_mut()) + } + + #[cfg(target_os = "none")] + pub(crate) async fn dispatch_mapped_async( + &mut self, + session: SessionId, + request: &FsRequest, + ) -> FsResponse { + let mut mapped = self.mapped_window_span(session).map(|span| { + // SAFETY: the retained returned-capability owner keeps this exact + // mapping live; the service executor serializes access to it. + DataWindow::new(unsafe { + core::slice::from_raw_parts_mut(span.base.as_ptr(), span.len) + }) + }); + self.dispatch_async(session, request, mapped.as_mut()).await + } + + /// Host tests inject their window directly through [`Self::dispatch`]. + #[cfg(not(target_os = "none"))] + pub(crate) fn dispatch_mapped( + &mut self, + session: SessionId, + request: &FsRequest, + ) -> FsResponse { + self.dispatch(session, request, None) + } + + #[cfg(not(target_os = "none"))] + pub(crate) async fn dispatch_mapped_async( + &mut self, + session: SessionId, + request: &FsRequest, + ) -> FsResponse { + self.dispatch_async(session, request, None).await + } + + /// Serve one request. + /// + /// `session` comes from the serving host, never from `request` — see the + /// module docs. `window` is the host's view of whatever the client + /// delivered; a bulk verb with no attachment, or with no window mapped, + /// refuses [`FsError::NoWindow`] rather than reading memory it was not + /// given. + pub fn dispatch( + &mut self, + session: SessionId, + request: &FsRequest, + window: Option<&mut DataWindow<'_>>, + ) -> FsResponse { + block_on(self.dispatch_async(session, request, window)) + } + + pub(crate) async fn dispatch_async( + &mut self, + session: SessionId, + request: &FsRequest, + window: Option<&mut DataWindow<'_>>, + ) -> FsResponse { + self.drain_pending_reclaims().await; + match self.serve_async(session, request, window).await { + Ok(response) => response, + Err(err) => FsResponse::Refused(err), + } + } + + /// Release the placements and reclaim the roots dead peers left behind, + /// before the request that woke this dispatch is served. Ordinary await + /// points: a `Busy` claim held by an in-flight request resolves on that + /// request's next turn, and a placement the provider refuses to release + /// stays pending for the next dispatch, retaining custody. + async fn drain_pending_reclaims(&mut self) { + // Released in place: a refused stage keeps its position, and its + // custody, without the list ever growing. + let mut index = 0; + while index < self.pending_aborts.len() { + if self + .engine + .abort_replace(&self.pending_aborts[index]) + .await + .is_ok() + { + self.pending_aborts.swap_remove(index); + } else { + index += 1; + } + } + if self.pending_reclaims.is_empty() { + return; + } + self.engine.reclaim_stranded(&self.pending_reclaims).await; + self.pending_reclaims.clear(); + } + + /// The dispatch body, written against `?` so every refusal path is one + /// line. [`Self::dispatch`] folds the error into a `Refused`. + async fn serve_async( + &mut self, + session: SessionId, + request: &FsRequest, + window: Option<&mut DataWindow<'_>>, + ) -> FsResult { + // A bulk verb needs both halves: **this session's** declared + // attachment, and a window the host actually mapped. Either missing is + // `NoWindow` — the service will not touch memory it was not handed, + // and it will not answer one client's presence check with another's + // record. + let window = if request.needs_window() { + if self.attachment(session).is_none() { + return Err(FsError::NoWindow); + } + Some(window.ok_or(FsError::NoWindow)?) + } else { + None + }; + + Ok(match *request { + FsRequest::AttachWindow { slot, bytes } => { + // An out-of-range session has no attachment slot, and refusing + // is the only honest answer: writing it anywhere else would + // hand this client somebody's record. + self.slots.claim(session.index() as usize)?.attached = + Some(WindowAttachment { slot, bytes }); + FsResponse::WindowAttached { + max_transfer: self.transfer_limit(), + } + } + + FsRequest::OpenRoot { rights } => opened(self.engine.open_root(session, rights)?), + FsRequest::OpenAt { + parent, + name, + rights, + } => opened(self.engine.open_at(session, parent, &name, rights)?), + FsRequest::OpenGrant { grant, rights } => { + opened(match self.task_view_for_session(session) { + Some(view) => self.engine.open_task_grant(session, view, grant, rights)?, + None => self.engine.open_grant(session, grant, rights)?, + }) + } + FsRequest::Create { + parent, + name, + kind, + subtype, + } => opened( + self.engine + .create_typed(session, parent, &name, kind, subtype) + .await?, + ), + + FsRequest::Close { handle } => { + if let Some(state) = self.slots.get_mut(session.index() as usize) + && let Some(stage) = state + .replacement + .as_ref() + .filter(|stage| stage.handle == handle) + { + self.engine.abort_replace(stage).await?; + state.replacement = None; + } + self.engine.close(session, handle)?; + FsResponse::Done + } + FsRequest::Stat { handle } => { + let info = self.engine.stat(session, handle)?; + FsResponse::Stat { + object: info.object, + stable_object: info.stable_object, + kind: info.kind, + subtype: info.subtype, + len: info.len, + revision: info.revision, + links: info.links, + grants: info.grants, + } + } + + FsRequest::List { + handle, + cursor, + window_offset, + } => { + let window = window.ok_or(FsError::NoWindow)?; + let page = self.engine.list(session, handle, cursor)?; + // Refuse rather than truncate: a client that asked for a page + // its window cannot hold must hear so, not silently receive + // fewer entries than the service listed. + if page.entries.len() > window.entry_capacity(window_offset) { + return Err(FsError::Range); + } + for (index, entry) in page.entries.iter().enumerate() { + let mut wire = FsDirEntryWire { + name_len: entry.name.len() as u8, + kind: match entry.kind { + ObjectKind::File => FsReqWire::KIND_FILE, + ObjectKind::Directory => FsReqWire::KIND_DIRECTORY, + }, + // The low byte carries the entry's subtype (data, never + // authority); the high byte stays reserved. + reserved0: u16::from(entry.subtype.to_bits()).into(), + reserved1: 0.into(), + object: entry.object.to_bits().into(), + name: [0; kore_abi::svc::FS_NAME_BYTES], + }; + let bytes = entry.name.as_bytes(); + wire.name[..bytes.len()].copy_from_slice(bytes); + window.put_entry(window_offset, index, &wire)?; + } + FsResponse::Listed { + entries: page.entries.len() as u32, + next: page.next, + } + } + + FsRequest::Locate { + handle, + stable, + window_offset, + } => { + let window = window.ok_or(FsError::NoWindow)?; + if window.search_capacity(window_offset) == 0 { + return Err(FsError::Range); + } + let found = self.engine.locate(session, handle, stable)?; + let (path, path_len) = spell_path(&found.path)?; + let kind = if found.kind.is_dir() { + FsReqWire::KIND_DIRECTORY + } else { + FsReqWire::KIND_FILE + }; + window.put_search_hit( + window_offset, + 0, + &FsSearchHitWire { + path_len: u16::try_from(path_len).map_err(|_| FsError::Range)?.into(), + kind, + reserved0: 0, + ranking: 0.into(), + object: found.object.to_bits().into(), + stable_object: found.stable_object.get().into(), + revision: found.revision.into(), + len: found.len.into(), + path, + snippet: [0; crate::search::MAX_SEARCH_SNIPPET], + reserved2: [0; 1], + }, + )?; + FsResponse::Listed { + entries: 1, + next: None, + } + } + + FsRequest::Parents { + handle, + cursor, + window_offset, + } => { + let window = window.ok_or(FsError::NoWindow)?; + // The page is sized to the window, so a small window pages + // instead of refusing; one that holds no record is `Range`. + let capacity = window.search_capacity(window_offset); + let page = self.engine.parents(session, handle, cursor, capacity)?; + for (index, parent) in page.parents.iter().enumerate() { + let (path, path_len) = spell_path(&parent.path)?; + window.put_search_hit( + window_offset, + index, + &FsSearchHitWire { + path_len: u16::try_from(path_len).map_err(|_| FsError::Range)?.into(), + kind: FsReqWire::KIND_DIRECTORY, + reserved0: 0, + ranking: 0.into(), + object: parent.object.to_bits().into(), + stable_object: parent.stable_object.get().into(), + revision: parent.revision.into(), + len: 0.into(), + path, + snippet: [0; crate::search::MAX_SEARCH_SNIPPET], + reserved2: [0; 1], + }, + )?; + } + FsResponse::Listed { + entries: page.parents.len() as u32, + next: page.next, + } + } + + FsRequest::ListGrants { window_offset } => { + let window = window.ok_or(FsError::NoWindow)?; + let grants = self + .engine + .grants(self.task_view_for_session(session).unwrap_or(session))?; + if grants.len() > window.entry_capacity(window_offset) { + return Err(FsError::Range); + } + for (index, info) in grants.iter().enumerate() { + let mut wire = kore_abi::svc::FsGrantEntryWire { + name_len: 0, + rights: info.rights.bits(), + kind: match info.kind { + crate::authority::AuthorityKind::PrivateRoot => { + kore_abi::svc::FsGrantEntryWire::KIND_PRIVATE_ROOT + } + crate::authority::AuthorityKind::Grant => { + kore_abi::svc::FsGrantEntryWire::KIND_GRANT + } + }, + reserved0: 0, + reserved1: 0.into(), + grant: info.grant.to_bits().into(), + name: [0; kore_abi::svc::FS_NAME_BYTES], + }; + if let Some(display) = info.display_name.as_ref() { + let bytes = display.as_bytes(); + wire.name[..bytes.len()].copy_from_slice(bytes); + wire.name_len = bytes.len() as u8; + } + window.put_grant_entry(window_offset, index, &wire)?; + } + FsResponse::GrantsListed { + entries: grants.len() as u32, + } + } + + FsRequest::SystemStatus { window_offset } => { + use kore_abi::svc::FsSystemStatusWire; + let window = window.ok_or(FsError::NoWindow)?; + let (tree_objects, tree_edges, tree_bytes) = self.engine.system_tree_usage()?; + let mut views_private: u16 = 0; + let mut views_pinned: u16 = 0; + for view in self.invocation_views.iter().flatten() { + match view.owner { + InvocationViewOwner::Private { .. } => views_private += 1, + InvocationViewOwner::Pinned { .. } => views_pinned += 1, + } + } + let mut flags: u16 = 0; + if self.backend.durable { + flags |= FsSystemStatusWire::FLAG_DURABLE; + } + if self.backend.degraded_read_only { + flags |= FsSystemStatusWire::FLAG_DEGRADED_READ_ONLY; + } + if self.backend.route_held { + flags |= FsSystemStatusWire::FLAG_ROUTE_HELD; + } + if self.workspace_view.is_some() { + flags |= FsSystemStatusWire::FLAG_WORKSPACE; + } + if self.system_view.is_some() { + flags |= FsSystemStatusWire::FLAG_SYSTEM_VIEW; + } + if self.backend.channel_lost { + flags |= FsSystemStatusWire::FLAG_CHANNEL_LOST; + } + if self.backend.host_memory_buffer { + flags |= FsSystemStatusWire::FLAG_HOST_MEMORY_BUFFER; + } + let wire = FsSystemStatusWire { + sessions: self.engine.live_sessions().into(), + peers: self.slots.count(|state| state.via.is_some()).into(), + views_private: views_private.into(), + views_pinned: views_pinned.into(), + windows: self.slots.count(|state| state.attached.is_some()).into(), + flags: flags.into(), + tree_objects: tree_objects.into(), + tree_edges: tree_edges.into(), + user_lifecycle: self.backend.user_lifecycle.into(), + system_lifecycle: self.backend.system_lifecycle.into(), + tree_bytes: tree_bytes.into(), + storage: self.backend.storage, + note: self.backend.note.into(), + commits: self.backend.commits.into(), + reads: self.backend.provider.reads.into(), + writes: self.backend.provider.writes.into(), + flushes: self.backend.provider.flushes.into(), + discards: self.backend.provider.discards.into(), + copied_bytes: self.backend.provider.copied_bytes.into(), + rollbacks: self.backend.provider.rollbacks.into(), + late_completions: self.backend.provider.late_completions.into(), + orphan_reclaims: self.backend.provider.orphan_reclaims.into(), + reclaim_failures: self.backend.provider.reclaim_failures.into(), + submissions: self.backend.channel.submissions.into(), + completions: self.backend.channel.completions.into(), + backpressure: self.backend.channel.backpressure.into(), + poll_calls: self.backend.channel.poll_calls.into(), + device_waits: self.backend.channel.device_waits.into(), + route_parks: self.backend.route_parks.into(), + route_fires: self.backend.route_fires.into(), + deadline_waits: self.backend.deadline_waits.into(), + spin_waits: self.backend.spin_waits.into(), + monitor_sleeps: self.backend.monitor_sleeps.into(), + monitor_turns: self.backend.monitor_turns.into(), + expired_waits: self.backend.provider.expired_waits.into(), + slow_waits: self.backend.provider.slow_waits.into(), + leaked_placements: u64::from(self.engine.leaked_placements()).into(), + abandoned_reclaims: u64::from(self.engine.abandoned_reclaims()).into(), + readahead_hits: self.backend.provider.readahead_hits.into(), + readahead_reads: self.backend.provider.readahead_reads.into(), + latency: self + .backend + .channel + .latency + .snapshot(self.backend.channel.max_outstanding), + }; + window.put_system_status(window_offset, &wire)?; + FsResponse::SystemStatusReported { + len: FsSystemStatusWire::BYTES as u32, + } + } + + FsRequest::Search { + handle, + query, + cursor, + window_offset, + } => { + let window = window.ok_or(FsError::NoWindow)?; + let page = self.engine.search(session, handle, query, cursor)?; + if page.hits.len() > window.search_capacity(window_offset) { + return Err(FsError::Range); + } + for (index, hit) in page.hits.iter().enumerate() { + let (path, path_len) = spell_path(&hit.path)?; + let mut snippet = [0u8; crate::search::MAX_SEARCH_SNIPPET]; + let snippet_len = hit.snippet.len(); + snippet[..snippet_len].copy_from_slice(hit.snippet.as_bytes()); + let ranking = u32::from(hit.score.get()) + | (u32::try_from(snippet_len).map_err(|_| FsError::Range)? << 16); + window.put_search_hit( + window_offset, + index, + &FsSearchHitWire { + path_len: u16::try_from(path_len).map_err(|_| FsError::Range)?.into(), + kind: match hit.kind { + ObjectKind::File => FsReqWire::KIND_FILE, + ObjectKind::Directory => FsReqWire::KIND_DIRECTORY, + }, + reserved0: 0, + ranking: ranking.into(), + object: hit.object.to_bits().into(), + stable_object: hit.stable_object.get().into(), + revision: hit.revision.into(), + len: hit.len.into(), + path, + snippet, + reserved2: [0; 1], + }, + )?; + } + FsResponse::Searched { + hits: page.hits.len() as u32, + observation: page.observation, + snapshot: page.snapshot, + index_generation: page.index_generation, + freshness: page.freshness, + next: page.next, + scanned: page.scanned, + work: page.work, + } + } + + FsRequest::Read { + handle, + offset, + window_offset, + length, + } => { + let window = window.ok_or(FsError::NoWindow)?; + let dst = window.destination(window_offset, length)?; + let count = self.engine.read(session, handle, offset, dst).await?; + FsResponse::Transferred { + count: count as u32, + } + } + FsRequest::BeginReplace { + handle, + length, + expected, + } => { + let state = self.slots.claim(session.index() as usize)?; + if state.replacement.is_some() { + return Err(FsError::Busy); + } + state.replacement = Some( + self.engine + .begin_replace(session, handle, length, expected) + .await?, + ); + FsResponse::Done + } + FsRequest::WriteReplacement { + handle, + offset, + window_offset, + length, + } => { + let stage = self + .slots + .get_mut(session.index() as usize) + .and_then(|state| state.replacement.as_mut()) + .filter(|stage| stage.handle == handle) + .ok_or(FsError::InvalidHandle)?; + let src = window + .ok_or(FsError::NoWindow)? + .source(window_offset, length)?; + self.engine + .write_replacement(session, stage, offset, src) + .await?; + FsResponse::Done + } + FsRequest::CommitReplace { handle } => { + // A slot past the table has staged nothing. + let state = self + .slots + .get_mut(session.index() as usize) + .ok_or(FsError::InvalidHandle)?; + let stage = state + .replacement + .as_ref() + .filter(|stage| stage.handle == handle) + .ok_or(FsError::InvalidHandle)?; + let revision = self.engine.commit_replace(session, stage).await?; + // The object owns the placement from this point onward. + state.replacement = None; + FsResponse::Committed { revision } + } + FsRequest::AbortReplace { handle } => { + if let Some(state) = self.slots.get_mut(session.index() as usize) + && let Some(stage) = state.replacement.as_ref() + { + if stage.handle != handle { + return Err(FsError::InvalidHandle); + } + self.engine.abort_replace(stage).await?; + state.replacement = None; + } + FsResponse::Done + } + FsRequest::WriteAt { + handle, + offset, + window_offset, + length, + expected, + } => { + let window = window.ok_or(FsError::NoWindow)?; + // The borrow must end before the engine call: `write_at` takes + // `&mut self` on the engine, and the source is the window, not + // the engine — but the compiler cannot see that through the + // service, so the bytes are copied out of the window's borrow + // by taking a shared one instead. + let src = window.source(window_offset, length)?; + let revision = self + .engine + .write_at(session, handle, offset, src, expected) + .await?; + FsResponse::Committed { revision } + } + FsRequest::Append { + handle, + window_offset, + length, + expected, + } => { + let window = window.ok_or(FsError::NoWindow)?; + let src = window.source(window_offset, length)?; + let (offset, revision) = self.engine.append(session, handle, src, expected).await?; + FsResponse::Appended { offset, revision } + } + FsRequest::Resize { + handle, + length, + expected, + } => { + let revision = self + .engine + .resize(session, handle, length, expected) + .await?; + FsResponse::Committed { revision } + } + + FsRequest::Rename { parent, old, new } => { + self.engine.rename(session, parent, &old, &new)?; + FsResponse::Done + } + FsRequest::Relocate { + src_parent, + name, + dst_parent, + new_name, + } => { + self.engine + .relocate(session, src_parent, &name, dst_parent, &new_name)?; + FsResponse::Done + } + FsRequest::Link { + parent, + name, + child, + } => { + self.engine.link(session, parent, &name, child)?; + FsResponse::Done + } + FsRequest::Unlink { parent, name } => { + self.engine.unlink(session, parent, &name).await?; + FsResponse::Done + } + FsRequest::DetachEdge { parent, name } => { + self.engine.detach_edge(session, parent, &name).await?; + FsResponse::Done + } + + FsRequest::Grant { + handle, + destination, + rights, + display_name, + } => { + let grant = + self.engine + .grant(session, handle, destination, rights, display_name)?; + FsResponse::Granted { grant } + } + FsRequest::GrantScoped { + handle, + destination, + rights, + quota, + display_name, + } => { + let grant = self.engine.grant_scoped( + session, + handle, + destination, + rights, + quota, + display_name, + )?; + FsResponse::Granted { grant } + } + FsRequest::Revoke { grant } => { + self.engine + .revoke( + self.task_view_for_session(session).unwrap_or(session), + grant, + ) + .await?; + FsResponse::Done + } + FsRequest::DeferCommits { hold } => { + // Releasing a hold never needs a record a slot does not have. + if hold { + self.slots.claim(session.index() as usize)?.deferring = true; + } else if let Some(state) = self.slots.get_mut(session.index() as usize) { + state.deferring = false; + } + FsResponse::Done + } + + FsRequest::SetSystemSeed { .. } => { + return Err(FsError::Unsupported); + } + + FsRequest::Sync { handle } => { + self.engine.sync(session, handle).await?; + FsResponse::Done + } + FsRequest::LendVersion { + handle, + window_offset, + } => { + let window = window.ok_or(FsError::NoWindow)?; + // Validate the destination before installing an affine + // snapshot. Once `lend_version` succeeds the response is the + // caller's only route to the generational lease; discovering + // a bad window afterward would strand that lease inside the + // service with no owner able to release it. + let dst = window.destination(window_offset, 56)?; + let owner = crate::extraction::ExtractionOwner::new(u64::from(session.index())); + let (lease, identity) = self + .engine + .lend_version(owner, session, handle) + .await + .map_err(extraction_error)?; + write_identity(dst, identity); + FsResponse::VersionLent { lease } + } + FsRequest::LendExecutable { + handle, + window_offset, + } => { + let window = window.ok_or(FsError::NoWindow)?; + let dst = window.destination(window_offset, 56)?; + let owner = crate::extraction::ExtractionOwner::new(u64::from(session.index())); + let (lease, identity) = self + .engine + .lend_executable_version(owner, session, handle) + .await + .map_err(extraction_error)?; + write_identity(dst, identity); + FsResponse::VersionLent { lease } + } + FsRequest::ReadVersion { + lease, + offset, + window_offset, + length, + } => { + let window = window.ok_or(FsError::NoWindow)?; + let dst = window.destination(window_offset, length)?; + let owner = crate::extraction::ExtractionOwner::new(u64::from(session.index())); + let count = self + .engine + .read_lent_version(owner, lease, offset, dst) + .await + .map_err(extraction_error)?; + FsResponse::Transferred { + count: count as u32, + } + } + FsRequest::CommitDerived { + lease, + window_offset, + length, + output_window_bytes, + } => { + let window = window.ok_or(FsError::NoWindow)?; + let bytes = window.source(window_offset, length)?; + let result = + kore_svc_extract::decode_result(bytes).map_err(|_| FsError::Unsupported)?; + let owner = crate::extraction::ExtractionOwner::new(u64::from(session.index())); + let info = self + .engine + .commit_encoded_derived_result( + owner, + lease, + result, + bytes, + u64::from(output_window_bytes), + ) + .await + .map_err(extraction_error)?; + FsResponse::DerivedCommitted { + parser: info.parser, + parser_version: info.parser_version, + } + } + FsRequest::ReleaseVersion { lease } => { + let owner = crate::extraction::ExtractionOwner::new(u64::from(session.index())); + self.engine + .release_lent_version(owner, lease) + .await + .map_err(extraction_error)?; + FsResponse::Done + } + FsRequest::DerivedInfo { handle, parser } => { + let Some(result) = self.engine.derived_result(session, handle, parser)? else { + return Err(FsError::NotFound); + }; + FsResponse::DerivedInfo { + input_version: result.provenance.input.version, + parser_version: result.provenance.parser_version, + } + } + // Host-control verbs are intercepted by `serve_request`, where a + // live `Serve` proves the arrival pair and carries the returned + // capability. The pure engine dispatch has neither authority and + // must never emulate them for a host-test caller. + FsRequest::RegisterSystemView { .. } + | FsRequest::RegisterPrivateView { .. } + | FsRequest::GrantView { .. } + | FsRequest::ApplyView { .. } + | FsRequest::BindView { .. } + | FsRequest::MatchView { .. } + | FsRequest::AdmitPeer { .. } + | FsRequest::TakeAdmission + | FsRequest::RetirePeer { .. } + | FsRequest::ConfirmRetired { .. } + | FsRequest::ClosePeer => { + return Err(FsError::AccessDenied); + } + // Storage rings ride the transport engine beside the engines, + // served by the durable row that owns the controller. An engine + // has no ring to attach. + FsRequest::StorAttach { .. } + | FsRequest::StorGrantPack + | FsRequest::StorGrantVersion { .. } + | FsRequest::StorTarget { .. } + | FsRequest::StorRelease { .. } + | FsRequest::StorKick + | FsRequest::StorInspect + | FsRequest::StorMaintain { .. } + | FsRequest::StorDetach => { + return Err(FsError::Unsupported); + } + }) + } +} + +fn extraction_error(error: crate::ExtractionError) -> FsError { + use crate::ExtractionError; + match error { + ExtractionError::Filesystem(error) => error, + ExtractionError::Provider(error) => error.into(), + // Two different bounded RAM tables, neither of them storage. They + // used to share `OutOfStorage`, which sent a reader looking at the + // disk for a table that had simply filled — and put `out of storage` + // on screen above an `ls` listing the file that had just been made. + ExtractionError::NoSnapshotSlots => FsError::OutOfVersionLeases, + ExtractionError::NoResultSlots => FsError::OutOfIndexSlots, + ExtractionError::InvalidLease => FsError::InvalidHandle, + ExtractionError::ObsoleteVersion => FsError::Conflict, + ExtractionError::InvalidResult(_) => FsError::Unsupported, + // A batch the index cannot hold leaves its version unindexed, which + // every producer already reports as such; as `Range` it failed the + // write the batch describes, after that write had committed. + ExtractionError::Oversize => FsError::Unsupported, + ExtractionError::Range => FsError::Range, + } +} + +/// Domain-level refusal for a structurally invalid recyclable-peer component. +/// +/// The generic assembler keeps the detailed lifecycle vocabulary private to +/// the serving host. On the filesystem wire, duplicates are ordinary +/// conflicts, an impossible grant position is a range error, and every +/// authority/species mismatch is denied. A return leg on a protocol that did +/// not declare one is a protocol-shape refusal rather than missing authority. +fn peer_assembly_error(error: PeerAssemblyRefusal) -> FsError { + match error { + PeerAssemblyRefusal::Duplicate => FsError::AlreadyExists, + PeerAssemblyRefusal::GrantOutOfRange => FsError::Range, + PeerAssemblyRefusal::UnexpectedReturn => FsError::Unsupported, + PeerAssemblyRefusal::WrongAuthority => FsError::AccessDenied, + } +} + +fn peer_admission_error(error: PeerAdmissionRefusal) -> FsError { + match error { + PeerAdmissionRefusal::Capacity | PeerAdmissionRefusal::NoScratchSlots => { + FsError::OutOfHandles + } + PeerAdmissionRefusal::WrongRingAuthority + | PeerAdmissionRefusal::WrongEndpointAuthority + | PeerAdmissionRefusal::WrongGrantCount + | PeerAdmissionRefusal::WrongReturnShape => FsError::AccessDenied, + PeerAdmissionRefusal::MapFailed => FsError::NoWindow, + // A stale registration standing in the arrival's slot. `AlreadyExists` + // is the reading that is true, and exec routes it to the refusal that + // means "fs refused for a non-capacity reason" rather than to the one + // that means its object table is full. + PeerAdmissionRefusal::PolicyCollision => FsError::AlreadyExists, + PeerAdmissionRefusal::PolicyUnavailable => FsError::OutOfObjects, + PeerAdmissionRefusal::InvalidMapPlan + | PeerAdmissionRefusal::RingGeometry + | PeerAdmissionRefusal::RingMalformed => FsError::Unsupported, + } +} + +/// The four open-shaped verbs share one reply. +fn opened(object: crate::engine::OpenedObject) -> FsResponse { + FsResponse::Opened { + handle: object.handle, + stable_object: object.stable_object, + kind: object.kind, + subtype: object.subtype, + len: object.len, + revision: object.revision, + } +} + +#[cfg(test)] +mod recyclable_session_tests { + use super::*; + + fn install_private_task_view( + service: &mut RamService, + scope: u64, + session: SessionId, + rights: Rights, + ) { + block_on(service.engine.attach_session(session)).unwrap(); + let grant = service.engine.task_view_root(session).unwrap(); + let slot = service + .invocation_views + .iter() + .position(Option::is_none) + .unwrap(); + service.invocation_views[slot] = Some(InvocationView { + scope, + grant, + rights, + task_session: session, + owner: InvocationViewOwner::Private { session }, + canary: None, + home: None, + }); + } + + fn admit_view(service: &mut RamService, arrival: u16, scope: u64, rights: Rights) -> SessionId { + let peer = PeerId::from_arrival(arrival).unwrap(); + assert!(service.peer_admitted(Ok(peer))); + service.apply_invocation_view(scope, peer, rights).unwrap(); + SessionId::new(peer.slot() as u16) + } + + #[test] + fn recyclable_arrivals_require_the_exact_current_generation() { + let mut service = RamService::default(); + service.slots.claim(2).unwrap().via = Some(0x102); + service.slots.claim(3).unwrap().via = Some(0x203); + + assert_eq!(service.session_for_via(0), Some(SessionId::new(0))); + assert_eq!(service.session_for_via(1), Some(SessionId::new(1))); + assert_eq!(service.session_for_via(0x102), Some(SessionId::new(2))); + assert_eq!(service.session_for_via(0x203), Some(SessionId::new(3))); + assert_eq!(service.session_for_via(0x202), None); + } + + #[test] + fn peer_close_releases_only_the_exact_arrival_state() { + let mut service = RamService::default(); + assert!(service.ensure_session(SessionId::new(2))); + service.slots.claim(2).unwrap().via = Some(0x102); + service.slots.claim(2).unwrap().attached = Some(WindowAttachment { + slot: 9, + bytes: 4096, + }); + + assert!(service.peer_closed(0x202, PeerClose::Died)); + assert_eq!(service.session_for_via(0x102), Some(SessionId::new(2))); + assert!(service.attachment(SessionId::new(2)).is_some()); + + assert!(service.peer_closed(0x102, PeerClose::Orderly)); + assert_eq!(service.session_for_via(0x102), None); + assert!(service.attachment(SessionId::new(2)).is_none()); + assert_eq!( + service.engine.open_root(SessionId::new(2), Rights::READ), + Err(FsError::InvalidHandle) + ); + } + + #[test] + fn admin_pair_gets_a_named_executable_source_grant() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert_eq!(service.session_for_via(1), Some(SessionId::new(1))); + + let display = Name::new(SYSTEM_SOURCE_GRANT).unwrap(); + let grant = service + .engine + .grants(SessionId::new(1)) + .unwrap() + .into_iter() + .find(|grant| grant.display_name == Some(display)) + .expect("admin source grant"); + assert_eq!( + grant.rights, + Rights::READ + .union(Rights::SHARE) + .union(Rights::BIND_EXECUTABLE) + ); + let opened = service + .engine + .open_grant(SessionId::new(1), grant.grant, Rights::READ) + .expect("admin can open the granted system root"); + assert_eq!(opened.kind, ObjectKind::Directory); + } + + /// The invariant every user file's durability rests on: what exec's + /// workspace creates must hang beneath session zero's root, because that + /// is the only tree [`commit_engine`] walks. #575 shipped a workspace + /// rooted in exec's own private root instead — writes reached their data + /// blocks and the names that found them were never serialized, so the + /// filesystem came back empty after a reboot. + /// + /// [`commit_engine`]: crate::persistence::commit_engine + /// The two bounded index tables must never share one refusal again. + /// + /// They both used to answer `OutOfStorage`, which is what put + /// `touch: out of storage` on screen above an `ls` listing the file that + /// had just been created and committed. Neither table is storage: a full + /// lease pool means a snapshot is still held, a full result table means + /// the index cannot grow, and the volume is fine in both cases. + #[test] + fn the_two_index_tables_do_not_share_a_refusal() { + use crate::ExtractionError; + assert_eq!( + extraction_error(ExtractionError::NoSnapshotSlots), + FsError::OutOfVersionLeases + ); + assert_eq!( + extraction_error(ExtractionError::NoResultSlots), + FsError::OutOfIndexSlots + ); + assert_ne!( + extraction_error(ExtractionError::NoSnapshotSlots), + extraction_error(ExtractionError::NoResultSlots) + ); + // And neither is reported as the medium being full. + for error in [ + ExtractionError::NoSnapshotSlots, + ExtractionError::NoResultSlots, + ] { + assert_ne!(extraction_error(error), FsError::OutOfStorage); + } + } + + #[test] + fn the_workspace_creates_inside_the_tree_commit_engine_serializes() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_workspace()); + + let system = SessionId::new(0); + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let grant = service.workspace_view.expect("workspace anchor stands"); + + // Create through the delegated edge, exactly as task views do. + let workspace = service + .engine + .open_grant(admin, grant, Rights::ALL) + .expect("admin can open the granted workspace"); + assert_eq!(workspace.kind, ObjectKind::Directory); + let probe = Name::new("probe").unwrap(); + block_on( + service + .engine + .create(admin, workspace.handle, &probe, ObjectKind::File), + ) + .expect("the workspace is writable"); + + // Now walk to it from the persisted root. A workspace that is not + // reachable from here is a workspace whose contents are dropped at + // the next mount, which is exactly the bug this asserts against. + let root = service + .engine + .open_root(system, Rights::ALL) + .expect("system root"); + let directory = Name::new(SYSTEM_WORKSPACE_DIR).unwrap(); + let reached = service + .engine + .open_at(system, root.handle, &directory, Rights::ALL) + .expect("the workspace hangs beneath the persisted root"); + service + .engine + .open_at(system, reached.handle, &probe, Rights::READ) + .expect("a file created through the workspace is reachable from the persisted root"); + } + + /// The rerouting this stack exists for: a private task view's session + /// roots in a delegation of the workspace, so what a task creates is + /// reachable from the persisted root — the #575 invariant expressed in + /// the invocation-view model. + #[test] + fn a_private_task_views_files_are_facts_of_the_persisted_tree() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_workspace()); + + // The real registration path: the view roots in its own durable + // home, `work/alpha`. + let alpha = Name::new("alpha").unwrap(); + service + .register_private_invocation_view(0xA1, &alpha, Rights::ALL, None) + .expect("private view registers over its durable home"); + let task = service + .invocation_views + .iter() + .flatten() + .find(|view| view.scope == 0xA1) + .map(|view| view.task_session) + .expect("registered view"); + + let root = service + .engine + .open_root(task, Rights::ALL) + .expect("task view root"); + let name = Name::new("journal").unwrap(); + block_on( + service + .engine + .create(task, root.handle, &name, ObjectKind::File), + ) + .expect("task creates in its view"); + + // Walk to it from the persisted root: work/alpha/journal. + let system = SessionId::new(0); + let system_root = service + .engine + .open_root(system, Rights::ALL) + .expect("system root"); + let directory = Name::new(SYSTEM_WORKSPACE_DIR).unwrap(); + let work = service + .engine + .open_at(system, system_root.handle, &directory, Rights::ALL) + .expect("workspace under the persisted root"); + let home = service + .engine + .open_at(system, work.handle, &alpha, Rights::ALL) + .expect("the view's home directory under the workspace"); + service + .engine + .open_at(system, home.handle, &name, Rights::READ) + .expect("a task-created file is a fact of the persisted tree"); + } + + /// The two-terminal contract the shared-workspace draft of this fix broke: + /// distinct views must never alias one namespace just because both are + /// durable. Each roots in its own `work/` home, so a creation in + /// one is invisible from the other while both remain facts of the one + /// tree `commit_engine` serializes. + #[test] + fn named_views_are_isolated_yet_both_persist_under_the_workspace() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_workspace()); + + let alpha = Name::new("alpha").unwrap(); + let beta = Name::new("beta").unwrap(); + service + .register_private_invocation_view(0xA1, &alpha, Rights::ALL, None) + .expect("first view registers"); + service + .register_private_invocation_view(0xB1, &beta, Rights::ALL, None) + .expect("second view registers"); + let session_of = |service: &RamService, scope: u64| { + service + .invocation_views + .iter() + .flatten() + .find(|view| view.scope == scope) + .map(|view| view.task_session) + .expect("registered view") + }; + let first = session_of(&service, 0xA1); + let second = session_of(&service, 0xB1); + + let root = service.engine.open_root(first, Rights::ALL).unwrap(); + let secret = Name::new("a-only").unwrap(); + block_on( + service + .engine + .create(first, root.handle, &secret, ObjectKind::File), + ) + .expect("first view creates"); + + // Isolation: the second view's root simply has no such name. + let other_root = service.engine.open_root(second, Rights::ALL).unwrap(); + assert_eq!( + service + .engine + .open_at(second, other_root.handle, &secret, Rights::READ) + .map(|_| ()) + .unwrap_err(), + FsError::NotFound + ); + + // Persistence: both homes hang beneath the one persisted root. + let system = SessionId::new(0); + let system_root = service.engine.open_root(system, Rights::ALL).unwrap(); + let directory = Name::new(SYSTEM_WORKSPACE_DIR).unwrap(); + let work = service + .engine + .open_at(system, system_root.handle, &directory, Rights::ALL) + .unwrap(); + for home in [&alpha, &beta] { + service + .engine + .open_at(system, work.handle, home, Rights::READ) + .expect("each view's home is a fact of the persisted tree"); + } + } + + /// The reboot story at view granularity: a reaped view's home directory + /// stays in the tree, and the next registration of the same name roots + /// over it — the files are still there. This is what carries a terminal's + /// data across a power cycle. + #[test] + fn re_registering_a_home_name_reopens_the_task_files() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_workspace()); + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + + let terminal = Name::new("terminal-0").unwrap(); + service + .register_private_invocation_view(0xA1, &terminal, Rights::ALL, None) + .expect("first registration"); + let (slot, task) = service + .invocation_views + .iter() + .enumerate() + .find_map(|(slot, view)| { + view.as_ref() + .filter(|view| view.scope == 0xA1) + .map(|view| (slot, view.task_session)) + }) + .expect("registered view"); + let root = service.engine.open_root(task, Rights::ALL).unwrap(); + let name = Name::new("journal").unwrap(); + block_on( + service + .engine + .create(task, root.handle, &name, ObjectKind::File), + ) + .expect("task creates"); + + // Retire the view exactly as the canary reap does: session teardown, + // then the anchor grant. The home directory is not touched. + let view = service.invocation_views[slot].take().expect("live view"); + block_on(service.engine.detach_session(task)).expect("detach view session"); + let (_, anchor) = view.home.expect("workspace-rooted view has a home"); + block_on(service.engine.revoke(admin, anchor)).expect("revoke anchor"); + + // Next boot's registration of the same terminal name. + service + .register_private_invocation_view(0xA2, &terminal, Rights::ALL, None) + .expect("re-registration over the standing home"); + let revived = service + .invocation_views + .iter() + .flatten() + .find(|view| view.scope == 0xA2) + .map(|view| view.task_session) + .expect("revived view"); + let root = service.engine.open_root(revived, Rights::ALL).unwrap(); + service + .engine + .open_at(revived, root.handle, &name, Rights::READ) + .expect("the earlier boot's file is still reachable in the revived view"); + } + + /// The diagnostic snapshot: frontend load counted from the live serving + /// state, backend posture folded in exactly as the mount owner injected + /// it, landed as one frozen record in the caller's window. + #[test] + fn the_status_snapshot_reports_frontend_load_and_backend_posture() { + use kore_abi::svc::FsSystemStatusWire; + + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_workspace()); + let terminal = Name::new("terminal-0").unwrap(); + service + .register_private_invocation_view(0xA1, &terminal, Rights::ALL, None) + .expect("view registers"); + + // The posture the mount owner would fold in before delegating. + service.set_backend_status(FsBackendStatus { + durable: true, + route_held: true, + channel_lost: true, + host_memory_buffer: true, + user_lifecycle: FsSystemStatusWire::LIFECYCLE_CURRENT, + system_lifecycle: FsSystemStatusWire::LIFECYCLE_UNDESIGNATED, + commits: 7, + storage: kore_abi::monitor::FilesystemStorageSnapshot::new( + 4096, + 1 << 30, + 512 << 20, + 16 << 20, + 256 << 10, + false, + true, + true, + ) + .with_health(kore_abi::monitor::FilesystemHealthSnapshot::new( + 0, 36, 98, 10, 3, 7, 123, 2, 0, + )), + ..Default::default() + }); + + let session = SessionId::new(1); + assert!(matches!( + service.dispatch( + session, + &FsRequest::AttachWindow { + slot: 0, + bytes: 512 + }, + None + ), + FsResponse::WindowAttached { .. } + )); + let mut backing = [0u8; 512]; + let response = { + let mut window = DataWindow::new(&mut backing); + service.dispatch( + session, + &FsRequest::SystemStatus { window_offset: 0 }, + Some(&mut window), + ) + }; + let FsResponse::SystemStatusReported { len } = response else { + panic!("status refused: {response:?}"); + }; + assert_eq!(len as usize, FsSystemStatusWire::BYTES); + + let wire = crate::client::read_system_status(&backing, 0).expect("valid record"); + let flags = wire.flags.get(); + assert_ne!(flags & FsSystemStatusWire::FLAG_DURABLE, 0); + assert_ne!(flags & FsSystemStatusWire::FLAG_ROUTE_HELD, 0); + assert_ne!(flags & FsSystemStatusWire::FLAG_HOST_MEMORY_BUFFER, 0); + assert_eq!(flags & FsSystemStatusWire::FLAG_DEGRADED_READ_ONLY, 0); + assert_ne!(flags & FsSystemStatusWire::FLAG_WORKSPACE, 0); + assert_ne!(flags & FsSystemStatusWire::FLAG_SYSTEM_VIEW, 0); + assert_ne!(flags & FsSystemStatusWire::FLAG_CHANNEL_LOST, 0); + assert_eq!( + wire.user_lifecycle.get(), + FsSystemStatusWire::LIFECYCLE_CURRENT + ); + assert_eq!( + wire.system_lifecycle.get(), + FsSystemStatusWire::LIFECYCLE_UNDESIGNATED + ); + assert_eq!(wire.commits.get(), 7); + assert_eq!(wire.storage.physical_bytes(), 1 << 30); + assert_eq!(wire.storage.volume_bytes(), 512 << 20); + assert_eq!(wire.storage.occupied_bytes(), 16 << 20); + assert_eq!(wire.storage.health().temperature_celsius(), 36); + assert_eq!(wire.storage.health().percentage_used(), 3); + assert_eq!(wire.storage.health().media_errors(), 0); + // System zero, the administration session, and the registered view. + assert!(wire.sessions.get() >= 3); + assert_eq!(wire.views_private.get(), 1); + assert_eq!(wire.windows.get(), 1); + // The persisted root, `work`, and the view's home beneath it. + assert!(wire.tree_objects.get() >= 3); + } + + /// The store half of the archive/filesystem unification: `app` hangs + /// beneath the persisted root (so its contents are serialized like every + /// other fact of that tree) and the administration session holds a + /// writable grant over it — under a display name that does not collide + /// with the directory itself. + #[test] + fn the_shared_app_directory_is_writable_and_hangs_off_the_persisted_root() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_apps()); + + let system = SessionId::new(0); + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let display = Name::new(SYSTEM_APPS_GRANT).unwrap(); + let apps = service + .engine + .grants(admin) + .unwrap() + .into_iter() + .find(|grant| grant.display_name == Some(display)) + .expect("administration app grant stands"); + assert_eq!(apps.rights, Rights::ALL); + + // Writable through the grant, exactly as the seeding copy will be. + let opened = service + .engine + .open_grant(admin, apps.grant, Rights::ALL) + .expect("admin opens the app directory"); + assert_eq!(opened.kind, ObjectKind::Directory); + let member = Name::new("hello").unwrap(); + block_on( + service + .engine + .create(admin, opened.handle, &member, ObjectKind::File), + ) + .expect("an executable can be written into the shared directory"); + + // …and what lands there is a fact of the tree `commit_engine` walks. + let root = service.engine.open_root(system, Rights::ALL).unwrap(); + let directory = Name::new(SYSTEM_APPS_DIR).unwrap(); + let reached = service + .engine + .open_at(system, root.handle, &directory, Rights::ALL) + .expect("app hangs beneath the persisted root"); + service + .engine + .open_at(system, reached.handle, &member, Rights::READ) + .expect("a seeded executable is reachable from the persisted root"); + } + + /// Recovery re-runs bring-up against a volume that already carries the + /// directory and its executables, so a second establish must adopt the + /// standing grant rather than re-create the directory and strand them. + #[test] + fn establishing_the_app_directory_twice_adopts_the_standing_grant() { + let mut service = RamService::default(); + assert!(service.establish_admin_apps()); + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let display = Name::new(SYSTEM_APPS_GRANT).unwrap(); + let first = service + .engine + .grants(admin) + .unwrap() + .into_iter() + .find(|grant| grant.display_name == Some(display)) + .expect("first establish grants") + .grant; + assert!(service.establish_admin_apps()); + let grants = service.engine.grants(admin).unwrap(); + let named: heapless::Vec<_, 8> = grants + .iter() + .filter(|grant| grant.display_name == Some(display)) + .collect(); + assert_eq!(named.len(), 1, "no rival grant is minted"); + assert_eq!(named[0].grant, first, "the standing grant is adopted"); + } + + /// The library convention's bring-up is the app directory's: `lib/` + /// hangs off the persisted root, the admin grant over it is writable + /// (the library seeder copies through it), and a second establish adopts + /// the standing grant instead of minting a rival. `pkgs/` follows suit + /// with no view of its own. + #[test] + fn the_library_and_package_directories_hang_off_the_persisted_root() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_libs()); + assert!(service.establish_admin_pkgs()); + assert!(service.establish_admin_share()); + + let system = SessionId::new(0); + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + for (grant_display, directory, member) in [ + (SYSTEM_LIBS_GRANT, SYSTEM_LIBS_DIR, "lua_demo.so"), + (SYSTEM_PKGS_GRANT, SYSTEM_PKGS_DIR, "kore"), + (SYSTEM_SHARE_GRANT, SYSTEM_SHARE_DIR, "wasm"), + ] { + let display = Name::new(grant_display).unwrap(); + let held = service + .engine + .grants(admin) + .unwrap() + .into_iter() + .find(|grant| grant.display_name == Some(display)) + .expect("administration grant stands"); + assert_eq!(held.rights, Rights::ALL); + let opened = service + .engine + .open_grant(admin, held.grant, Rights::ALL) + .expect("admin opens the directory"); + assert_eq!(opened.kind, ObjectKind::Directory); + let member = Name::new(member).unwrap(); + block_on( + service + .engine + .create(admin, opened.handle, &member, ObjectKind::File), + ) + .expect("a member can be written through the grant"); + + let root = service.engine.open_root(system, Rights::ALL).unwrap(); + let directory = Name::new(directory).unwrap(); + let reached = service + .engine + .open_at(system, root.handle, &directory, Rights::ALL) + .expect("the directory hangs beneath the persisted root"); + service + .engine + .open_at(system, reached.handle, &member, Rights::READ) + .expect("the member is reachable from the persisted root"); + } + + // Recovery: adopt, never re-create. + let display = Name::new(SYSTEM_LIBS_GRANT).unwrap(); + let first = service.libs_view.expect("the libs anchor is cached"); + assert!(service.establish_admin_libs()); + let named: heapless::Vec<_, 8> = service + .engine + .grants(admin) + .unwrap() + .iter() + .filter(|grant| grant.display_name == Some(display)) + .cloned() + .collect(); + assert_eq!(named.len(), 1, "no rival grant is minted"); + assert_eq!(named[0].grant, first, "the standing grant is adopted"); + assert_eq!(service.libs_view, Some(first)); + + let first_share = service.share_view.expect("the share anchor is cached"); + assert!(service.establish_admin_share()); + assert_eq!(service.share_view, Some(first_share), "share adopts too"); + } + + /// A task view carries `share` beside `libs`: READ-only, so a task can + /// read a shipped data member (`shared/wasm/hello.wasm`) but never write, + /// relink, or bless one. On session zero the name is the directory's own + /// child, so the view install declines and the directory is reached + /// directly — one object either way. + #[test] + fn a_task_view_carries_a_read_only_share_window() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_apps()); + assert!(service.establish_admin_libs()); + assert!(service.establish_admin_share()); + assert!(service.establish_admin_workspace()); + + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let share = service.share_view.unwrap(); + let opened = service + .engine + .open_grant(admin, share, Rights::ALL) + .unwrap(); + let wasm = Name::new("wasm").unwrap(); + let directory = block_on(service.engine.create( + admin, + opened.handle, + &wasm, + ObjectKind::Directory, + )) + .unwrap(); + let member = Name::new("hello.wasm").unwrap(); + block_on( + service + .engine + .create(admin, directory.handle, &member, ObjectKind::File), + ) + .unwrap(); + let _ = service.engine.close(admin, directory.handle); + let _ = service.engine.close(admin, opened.handle); + + let session = SessionId::new(7); + assert!(service.ensure_session(session)); + assert!(service.install_share_view(session)); + assert!(service.install_share_view(session), "idempotent"); + let grants = service.engine.grants(session).unwrap(); + let share_display = Name::new(SYSTEM_SHARE_VIEW).unwrap(); + let view: heapless::Vec<_, 8> = grants + .iter() + .filter(|grant| grant.display_name == Some(share_display)) + .collect(); + assert_eq!(view.len(), 1, "one `share` name, installed once"); + assert_eq!(view[0].rights, Rights::READ_ONLY); + assert!(!view[0].rights.contains(Rights::WRITE)); + assert!(!view[0].rights.contains(Rights::BIND_EXECUTABLE)); + + let through = service + .engine + .open_grant(session, view[0].grant, Rights::READ) + .expect("the task opens its share view"); + let nested = service + .engine + .open_at(session, through.handle, &wasm, Rights::READ) + .expect("the per-kind directory is reachable through the view"); + service + .engine + .open_at(session, nested.handle, &member, Rights::READ) + .expect("the data member is reachable through the view"); + assert!( + service + .engine + .open_grant(session, view[0].grant, Rights::WRITE) + .is_err(), + "the view refuses WRITE" + ); + + // The session rooted where the system directories themselves live + // carries all three views like any other. It could not while the + // view's name was its directory's: `view_name_taken` counts a real + // child of the destination root as taken, and the three call sites + // discard that refusal. On the two-store durable layout every + // session the system store serves is rooted there, so `shared` was + // missing from `list` and `shares` on a metal install while `apps` + // and `libs` were present — and no single-store gate could see it. + let rooted = SessionId::new(0); + assert!(service.install_share_view(rooted), "shared installs"); + assert!(service.install_apps_view(rooted), "apps installs"); + assert!(service.install_libs_view(rooted), "libs installs"); + } + + /// A task view carries `libs` beside `apps`: READ-only, with neither + /// WRITE nor BIND_EXECUTABLE, so a task can read and link a library but + /// never write, relink, or bless one -- and `pkgs` never appears in a + /// task view at all. + #[test] + fn a_task_view_carries_a_read_only_libs_window_and_no_pkgs_window() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_apps()); + assert!(service.establish_admin_libs()); + assert!(service.establish_admin_pkgs()); + assert!(service.establish_admin_workspace()); + + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let libs = service.libs_view.unwrap(); + let opened = service.engine.open_grant(admin, libs, Rights::ALL).unwrap(); + let member = Name::new("lua_demo.so").unwrap(); + block_on( + service + .engine + .create(admin, opened.handle, &member, ObjectKind::File), + ) + .unwrap(); + let _ = service.engine.close(admin, opened.handle); + + let session = SessionId::new(7); + assert!(service.ensure_session(session)); + assert!(service.install_libs_view(session)); + assert!(service.install_libs_view(session), "idempotent"); + let grants = service.engine.grants(session).unwrap(); + let libs_display = Name::new(SYSTEM_LIBS_VIEW).unwrap(); + let view: heapless::Vec<_, 8> = grants + .iter() + .filter(|grant| grant.display_name == Some(libs_display)) + .collect(); + assert_eq!(view.len(), 1, "one `libs` name, installed once"); + assert_eq!(view[0].rights, Rights::READ_ONLY); + assert!(!view[0].rights.contains(Rights::WRITE)); + assert!(!view[0].rights.contains(Rights::BIND_EXECUTABLE)); + assert!( + grants + .iter() + .all(|grant| grant.display_name != Some(Name::new(SYSTEM_PKGS_DIR).unwrap())), + "pkgs is admin-only" + ); + + // Readable through the view, never writable. + let through = service + .engine + .open_grant(session, view[0].grant, Rights::READ) + .expect("the task opens its libs view"); + service + .engine + .open_at(session, through.handle, &member, Rights::READ) + .expect("the library is reachable through the view"); + assert!( + service + .engine + .open_grant(session, view[0].grant, Rights::WRITE) + .is_err(), + "the view refuses WRITE" + ); + } + + /// #605 Stage 2: a terminal's task view carries the canonical `apps` + /// READ-only, BIND_EXECUTABLE-free window onto the shared store object -- + /// and ONLY that name, the flat `app` view being retired -- which the terminal + /// can read and traverse but never write, relink, or bless. + #[test] + fn a_terminal_view_carries_a_read_only_app_window_onto_the_shared_store() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_apps()); + assert!(service.establish_admin_workspace()); + + // Seed one executable into the shared store through the admin grant. + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let apps_display = Name::new(SYSTEM_APPS_GRANT).unwrap(); + let apps = service + .engine + .grants(admin) + .unwrap() + .into_iter() + .find(|grant| grant.display_name == Some(apps_display)) + .expect("admin app grant stands") + .grant; + let store = service + .engine + .open_grant(admin, apps, Rights::ALL) + .expect("admin opens the store"); + let member = Name::new("ls").unwrap(); + block_on( + service + .engine + .create(admin, store.handle, &member, ObjectKind::File), + ) + .expect("seed a store member"); + let _ = service.engine.close(admin, store.handle); + + // Found a terminal. + let terminal = Name::new("alpha").unwrap(); + service + .register_private_invocation_view(0xA1, &terminal, Rights::ALL, None) + .expect("terminal view registers"); + let task = service + .invocation_views + .iter() + .flatten() + .find(|view| view.scope == 0xA1) + .map(|view| view.task_session) + .expect("registered terminal view"); + + // The terminal carries canonical `apps` and NOTHING ELSE naming the store: + // the flat `app` view is retired, so one object is shown under one + // name. `apps` is exactly READ-only — no WRITE, MUTATE_NAMESPACE, + // SHARE, or BIND_EXECUTABLE. + let apps_display = Name::new(SYSTEM_APPS_VIEW).unwrap(); + let legacy_display = Name::new(SYSTEM_APPS_DIR).unwrap(); + let apps = service + .engine + .grants(task) + .unwrap() + .into_iter() + .find(|grant| grant.display_name == Some(apps_display)) + .expect("the terminal carries the canonical apps view"); + assert!( + service + .engine + .grants(task) + .unwrap() + .into_iter() + .all(|grant| grant.display_name != Some(legacy_display)), + "the retired flat `app` view must not be granted to a terminal — one \ + store object under two names reads as a stale projection" + ); + assert_eq!( + apps.rights, + Rights::READ_ONLY, + "the app view is exactly READ" + ); + assert!(!apps.rights.contains(Rights::WRITE)); + assert!(!apps.rights.contains(Rights::MUTATE_NAMESPACE)); + assert!(!apps.rights.contains(Rights::SHARE)); + assert!(!apps.rights.contains(Rights::BIND_EXECUTABLE)); + + // The seeded member is reachable read-only through the one view. + let opened = service + .engine + .open_grant(task, apps.grant, Rights::READ) + .expect("terminal opens its canonical apps view"); + service + .engine + .open_at(task, opened.handle, &member, Rights::READ) + .expect("the seeded executable is visible read-only through apps/"); + + // But the terminal cannot mutate the store: creating a name refuses. + let intruder = Name::new("intruder").unwrap(); + assert!( + block_on( + service + .engine + .create(task, opened.handle, &intruder, ObjectKind::File) + ) + .is_err(), + "a terminal cannot write into the shared store through its apps view" + ); + } + + #[test] + fn dagfs_and_posix_app_projections_are_dag_edges_to_one_file() { + let mut service = RamService::default(); + assert!(service.establish_admin_apps()); + assert!(service.establish_admin_workspace()); + + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let apps_grant = service + .engine + .grants(admin) + .unwrap() + .into_iter() + .find(|grant| grant.display_name == Some(Name::new(SYSTEM_APPS_GRANT).unwrap())) + .expect("admin owns the writable app store") + .grant; + let apps = service + .engine + .open_grant(admin, apps_grant, Rights::ALL) + .expect("admin opens app store"); + let ls = block_on(service.engine.create( + admin, + apps.handle, + &Name::new("ls").unwrap(), + ObjectKind::File, + )) + .expect("seed one image object"); + let dagfs = block_on(service.engine.create( + admin, + apps.handle, + &Name::new("dagfs").unwrap(), + ObjectKind::Directory, + )) + .expect("create native app family"); + let compat = block_on(service.engine.create( + admin, + apps.handle, + &Name::new("compat").unwrap(), + ObjectKind::Directory, + )) + .expect("create compatibility family"); + let posix = block_on(service.engine.create( + admin, + compat.handle, + &Name::new("posix").unwrap(), + ObjectKind::Directory, + )) + .expect("create POSIX projection family"); + service + .engine + .link(admin, dagfs.handle, &Name::new("list").unwrap(), ls.handle) + .expect("native projection links the source object"); + service + .engine + .link(admin, posix.handle, &Name::new("ls").unwrap(), ls.handle) + .expect("POSIX projection links the same source object"); + + let terminal = Name::new("alpha").unwrap(); + service + .register_private_invocation_view(0xB1, &terminal, Rights::ALL, None) + .expect("terminal view registers"); + let task = service + .invocation_views + .iter() + .flatten() + .find(|view| view.scope == 0xB1) + .expect("terminal view stands") + .task_session; + let canonical = service + .engine + .grants(task) + .unwrap() + .into_iter() + .find(|grant| grant.display_name == Some(Name::new(SYSTEM_APPS_VIEW).unwrap())) + .expect("canonical apps view is present"); + let apps = service + .engine + .open_grant(task, canonical.grant, Rights::READ) + .expect("terminal opens canonical apps view"); + let dagfs = service + .engine + .open_at( + task, + apps.handle, + &Name::new("dagfs").unwrap(), + Rights::READ, + ) + .expect("native directory is visible"); + let compat = service + .engine + .open_at( + task, + apps.handle, + &Name::new("compat").unwrap(), + Rights::READ, + ) + .expect("compatibility directory is visible"); + let posix = service + .engine + .open_at( + task, + compat.handle, + &Name::new("posix").unwrap(), + Rights::READ, + ) + .expect("POSIX directory is visible"); + let native = service + .engine + .open_at( + task, + dagfs.handle, + &Name::new("list").unwrap(), + Rights::READ, + ) + .expect("native projection is visible"); + let posix = service + .engine + .open_at(task, posix.handle, &Name::new("ls").unwrap(), Rights::READ) + .expect("POSIX projection is visible"); + assert_eq!(native.stable_object, posix.stable_object); + assert_eq!(native.stable_object, ls.stable_object); + } + + /// A dead writer's staged replacement is released on the service's next + /// dispatch, never inside the synchronous peer-death hook. A release the + /// provider refuses stays pending — custody retained — while the dead + /// peer's slot is already free for the next admission, and the old bytes + /// are what that next peer reads either way. + #[test] + fn a_dead_writers_stage_is_released_on_the_next_dispatch_not_in_the_hook() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + service + .register_private_invocation_view( + 0xD0, + &Name::new("terminal-0").unwrap(), + Rights::ALL, + None, + ) + .unwrap(); + let peer = admit_view(&mut service, 0x106, 0xD0, Rights::ALL); + let root = service.engine.open_root(peer, Rights::ALL).unwrap(); + let file = crate::block_on(service.engine.create( + peer, + root.handle, + &Name::new("saved").unwrap(), + crate::ObjectKind::File, + )) + .unwrap(); + crate::block_on( + service + .engine + .write_at(peer, file.handle, 0, b"old", crate::ANY_REVISION), + ) + .unwrap(); + assert!(matches!( + service.dispatch( + peer, + &FsRequest::BeginReplace { + handle: file.handle, + length: 6, + expected: crate::ANY_REVISION + }, + None + ), + FsResponse::Done + )); + service.dispatch(peer, &FsRequest::AttachWindow { slot: 0, bytes: 3 }, None); + let mut backing = *b"new"; + let mut window = crate::window::DataWindow::new(&mut backing); + assert!(matches!( + service.dispatch( + peer, + &FsRequest::WriteReplacement { + handle: file.handle, + offset: 0, + window_offset: 0, + length: 3 + }, + Some(&mut window) + ), + FsResponse::Done + )); + service + .engine + .provider() + .inject(crate::provider::Fault::Release( + crate::error::ProviderError::Media, + )); + assert!( + service.peer_closed(0x106, PeerClose::Died), + "the hook never awaits the release; the stage leaves with its peer" + ); + assert!( + service + .slots + .get(peer.index() as usize) + .is_none_or(|state| state.replacement.is_none()) + ); + assert_eq!(service.pending_aborts.len(), 1); + // The dead peer's slot is free at once — the stage no longer lives in + // it — and the next dispatch tries the release. The armed fault + // refuses that one: custody is retained for the dispatch after. + let next = admit_view(&mut service, 0x206, 0xD0, Rights::ALL); + let _ = service.dispatch(next, &FsRequest::DeferCommits { hold: false }, None); + assert_eq!( + service.pending_aborts.len(), + 1, + "a refused release stays pending" + ); + let _ = service.dispatch(next, &FsRequest::DeferCommits { hold: false }, None); + assert!( + service.pending_aborts.is_empty(), + "released once the provider accepts" + ); + let root = service.engine.open_root(next, Rights::ALL).unwrap(); + let file = service + .engine + .open_at(next, root.handle, &Name::new("saved").unwrap(), Rights::ALL) + .unwrap(); + let mut bytes = [0; 3]; + crate::block_on(service.engine.read(next, file.handle, 0, &mut bytes)).unwrap(); + assert_eq!(&bytes, b"old"); + } + + /// The editor's open by identity, through the wire: a file's stated + /// identity, located from the view root, lands one record the client's + /// locate reader lifts — the file itself, with its length. The parents + /// reader refuses that same record (a parent is a directory with no + /// length), which is what once made every located file `Range`. + #[test] + fn a_located_file_reads_back_through_the_window() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + service + .register_private_invocation_view( + 0xD0, + &Name::new("terminal-0").unwrap(), + Rights::ALL, + None, + ) + .unwrap(); + let peer = admit_view(&mut service, 0x106, 0xD0, Rights::ALL); + let root = service.engine.open_root(peer, Rights::ALL).unwrap(); + let file = crate::block_on(service.engine.create( + peer, + root.handle, + &Name::new("editor-test").unwrap(), + crate::ObjectKind::File, + )) + .unwrap(); + crate::block_on( + service + .engine + .write_at(peer, file.handle, 0, b"before", crate::ANY_REVISION), + ) + .unwrap(); + let stable = service.engine.stat(peer, file.handle).unwrap().stable_object; + + const BYTES: usize = FsSearchHitWire::BYTES; + service.dispatch(peer, &FsRequest::AttachWindow { slot: 0, bytes: BYTES as u32 }, None); + let mut backing = [0u8; BYTES]; + let response = { + let mut window = crate::window::DataWindow::new(&mut backing); + service.dispatch( + peer, + &FsRequest::Locate { + handle: root.handle, + stable, + window_offset: 0, + }, + Some(&mut window), + ) + }; + assert_eq!(response, FsResponse::Listed { entries: 1, next: None }); + let located = crate::client::read_located(&backing, 0).expect("the record lifts"); + assert_eq!(located.stable_object, stable); + assert_eq!(located.kind, crate::ObjectKind::File); + assert_eq!(located.len, 6); + assert_eq!(located.path.len(), 1); + assert_eq!(located.path[0].as_bytes(), b"editor-test"); + assert_eq!(crate::client::read_parent(&backing, 0, 0), Err(FsError::Range)); + } + + /// The deferral is per session: a bulk writer holds only its own + /// commits, and a second client's durability is untouched by it. + #[test] + fn deferring_commits_binds_one_session_and_ends_with_its_peer() { + let mut service = RamService::default(); + let writer = SessionId::new(1); + // Not slot 2: `PeerId`'s slot is the arrival's low bits, so 0x106 + // below lands there and the admission must find it free. + let other = SessionId::new(3); + assert!(service.ensure_session(writer)); + assert!(service.ensure_session(other)); + + assert!(matches!( + service.dispatch(writer, &FsRequest::DeferCommits { hold: true }, None), + FsResponse::Done + )); + assert!(service.is_deferring(writer)); + assert!( + !service.is_deferring(other), + "one client's bulk write must not defer another's durability" + ); + + assert!(matches!( + service.dispatch(writer, &FsRequest::DeferCommits { hold: false }, None), + FsResponse::Done + )); + assert!(!service.is_deferring(writer)); + + // A client that dies inside the window leaves no held state for the + // next occupant of its slot to inherit. + assert!(service.establish_admin_source()); + let terminal = Name::new("terminal-0").unwrap(); + service + .register_private_invocation_view(0xD0, &terminal, Rights::ALL, None) + .expect("task view registers"); + let peer = admit_view(&mut service, 0x106, 0xD0, Rights::ALL); + assert!(matches!( + service.dispatch(peer, &FsRequest::DeferCommits { hold: true }, None), + FsResponse::Done + )); + assert!(service.is_deferring(peer)); + assert!(service.peer_closed(0x106, PeerClose::Orderly)); + assert!( + !service.is_deferring(peer), + "a dead writer's window must not be inherited by its slot's next occupant" + ); + } + + /// A dead peer's authority goes in the hook; its subtree goes on the next + /// async dispatch, never inside the hook. The hook is synchronous, and a + /// reclamation run inside it was a nested executor that could not let + /// the in-flight request holding a claim release it. + #[test] + fn a_dead_peers_subtree_is_reclaimed_on_the_next_dispatch_not_in_the_hook() { + let mut service = RamService::default(); + let session = SessionId::new(2); + assert!(service.ensure_session(session)); + service.slots.claim(2).unwrap().via = Some(0x102); + let root = service.engine.open_root(session, Rights::ALL).unwrap(); + let before = service.engine.object_count(); + let file = block_on(service.engine.create( + session, + root.handle, + &Name::new("stranded").unwrap(), + ObjectKind::File, + )) + .unwrap(); + assert_eq!(service.engine.object_count(), before + 1); + + assert!(service.peer_closed(0x102, PeerClose::Orderly)); + // Authority is gone at once... + assert!(service.engine.open_root(session, Rights::READ).is_err()); + // ...the bytes are not, until a dispatch runs on the service's own + // executor. + assert_eq!(service.engine.object_count(), before + 1); + assert_eq!(service.pending_reclaims.len(), 1); + let _ = service.dispatch( + session, + &FsRequest::Close { + handle: file.handle, + }, + None, + ); + assert!(service.pending_reclaims.is_empty()); + assert!(service.engine.object_count() <= before); + } + + /// One home, one live owner: a second live registration naming the same + /// home is refused whole, so two tasks cannot be silently fused into a + /// shared namespace by a name collision. + #[test] + fn a_live_home_name_cannot_be_registered_twice() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_workspace()); + + let terminal = Name::new("terminal-0").unwrap(); + service + .register_private_invocation_view(0xA1, &terminal, Rights::ALL, None) + .expect("first registration"); + assert_eq!( + service + .register_private_invocation_view(0xB1, &terminal, Rights::ALL, None) + .unwrap_err(), + FsError::AlreadyExists + ); + } + + /// The command lifecycle over the workspace-rooted view: one command + /// creates, exits (peer teardown), and the next command of the same + /// task must still see the creation — the exact shell sequence + /// (`mkdir` then `touch`) the boot gate drives. + #[test] + fn a_later_command_sees_an_earlier_commands_workspace_creations() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_workspace()); + + // The real registration path: the stable task view roots in its own + // durable home beneath the workspace. + let terminal = Name::new("terminal-0").unwrap(); + service + .register_private_invocation_view(0xC0, &terminal, Rights::ALL, None) + .expect("task view registers over its durable home"); + + // Command one: mkdir durable; then the process exits. + let first = admit_view(&mut service, 0x106, 0xC0, Rights::ALL); + let root = service.engine.open_root(first, Rights::ALL).unwrap(); + let durable = Name::new("durable").unwrap(); + block_on( + service + .engine + .create(first, root.handle, &durable, ObjectKind::Directory), + ) + .expect("mkdir durable"); + assert!(service.peer_closed(0x106, PeerClose::Orderly)); + + // Command two, same task: touch durable/data. + let second = admit_view(&mut service, 0x206, 0xC0, Rights::ALL); + let root = service.engine.open_root(second, Rights::ALL).unwrap(); + let reached = service + .engine + .open_at(second, root.handle, &durable, Rights::ALL) + .expect("the second command still sees durable/"); + let data = Name::new("data").unwrap(); + block_on( + service + .engine + .create(second, reached.handle, &data, ObjectKind::File), + ) + .expect("touch durable/data"); + } + + /// The Python worker's HOME face can edit its seat's durable files, but + /// neither climb into another seat's home nor write a system preopen. + /// The service enforces this on the routes; Python source does not need a + /// path blacklist to make a read-only grant stay read-only. + #[test] + fn writable_script_home_keeps_system_grants_and_other_homes_outside() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_share()); + assert!(service.establish_admin_workspace()); + let home = Name::new("terminal-0").unwrap(); + let other = Name::new("terminal-1").unwrap(); + service + .register_private_invocation_view(0xA1, &home, Rights::HOME, None) + .unwrap(); + service + .register_private_invocation_view(0xB1, &other, Rights::HOME, None) + .unwrap(); + let peer = admit_view(&mut service, 0x106, 0xA1, Rights::HOME); + let root = service.engine.open_root(peer, Rights::HOME).unwrap(); + assert!(matches!( + service.engine.open_root(peer, Rights::ALL), + Err(FsError::AccessDenied) + )); + assert!(matches!( + service.engine.open_at(peer, root.handle, &other, Rights::READ), + Err(FsError::NotFound) + )); + assert!(Name::new("..").is_err(), "a containment edge cannot climb out"); + + let note = Name::new("note.txt").unwrap(); + let file = block_on(service.engine.create(peer, root.handle, ¬e, ObjectKind::File)) + .expect("create in the writable home"); + block_on(service.engine.write_at(peer, file.handle, 0, b"old", crate::ANY_REVISION)) + .expect("write in the home"); + block_on(service.engine.write_at(peer, file.handle, 0, b"new", crate::ANY_REVISION)) + .expect("edit in the home"); + service.engine.close(peer, file.handle).unwrap(); + assert!(service.peer_closed(0x106, PeerClose::Orderly)); + + // Each Python call starts a fresh worker. A later call must see the + // prior call's edits through the same delegated home. + let later = admit_view(&mut service, 0x206, 0xA1, Rights::HOME); + let later_root = service.engine.open_root(later, Rights::HOME).unwrap(); + let file = service + .engine + .open_at(later, later_root.handle, ¬e, Rights::READ) + .expect("later worker reopens the durable home file"); + let mut bytes = [0; 3]; + assert_eq!(block_on(service.engine.read(later, file.handle, 0, &mut bytes)), Ok(3)); + assert_eq!(&bytes, b"new"); + service.engine.close(later, file.handle).unwrap(); + block_on(service.engine.unlink(later, later_root.handle, ¬e)) + .expect("delete in the home"); + assert!(matches!( + service.engine.open_at(later, later_root.handle, ¬e, Rights::READ), + Err(FsError::NotFound) + )); + + let owner = service + .invocation_views + .iter() + .flatten() + .find(|view| view.scope == 0xA1) + .unwrap() + .task_session; + let share = service + .engine + .grants(owner) + .unwrap() + .into_iter() + .find(|grant| grant.display_name == Some(Name::new(SYSTEM_SHARE_VIEW).unwrap())) + .expect("the system share remains a named grant"); + assert_eq!(share.rights, Rights::READ_ONLY); + assert!(matches!( + service.engine.open_task_grant(later, owner, share.grant, Rights::WRITE), + Err(FsError::AccessDenied) + )); + let read_only = service + .engine + .open_task_grant(later, owner, share.grant, Rights::READ) + .unwrap(); + assert!(matches!( + block_on(service.engine.create(later, read_only.handle, ¬e, ObjectKind::File)), + Err(FsError::AccessDenied) + )); + } + + /// Recovery calls this on a volume that already carries the directory, so + /// a second call must adopt the standing grant rather than mint a rival + /// (or refuse on the taken display name) — otherwise the first reboot + /// after any successful boot would strand the user's data. + #[test] + fn establishing_the_workspace_twice_adopts_the_standing_grant() { + let mut service = RamService::default(); + assert!(service.establish_admin_workspace()); + let first = service + .workspace_view + .expect("anchor after first establish"); + assert!(service.establish_admin_workspace()); + assert_eq!( + service.workspace_view, + Some(first), + "the second establish adopts the standing anchor" + ); + } + + /// The read-only program source and the writable workspace must stay + /// distinct objects: exec loads executable images through `sys`, and a + /// fault in ordinary user-file handling must not be able to reach them. + #[test] + fn the_workspace_is_not_the_system_source_object() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_workspace()); + + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let source = Name::new(SYSTEM_SOURCE_GRANT).unwrap(); + let grants = service.engine.grants(admin).expect("admin grants listing"); + let source = grants + .iter() + .find(|grant| grant.display_name == Some(source)) + .expect("system source grant"); + let workspace = service.workspace_view.expect("workspace anchor stands"); + assert_eq!( + source.rights, + Rights::READ + .union(Rights::SHARE) + .union(Rights::BIND_EXECUTABLE) + ); + + let workspace_opened = service + .engine + .open_grant(admin, workspace, Rights::ALL) + .expect("admin opens the workspace"); + // The source root cannot name itself through the workspace's edge: + // the workspace is a child of it, never an alias for it. + let opened = service + .engine + .open_grant(admin, source.grant, Rights::READ) + .expect("admin opens the read-only source"); + assert_ne!( + opened.stable_object, workspace_opened.stable_object, + "the workspace and the source are distinct objects" + ); + let directory = Name::new(SYSTEM_WORKSPACE_DIR).unwrap(); + let nested = service + .engine + .open_at(admin, opened.handle, &directory, Rights::READ) + .expect("the workspace is reachable beneath the source root"); + assert_eq!(nested.kind, ObjectKind::Directory); + } + + #[test] + fn sibling_invocation_lineages_keep_independent_native_rights_ceilings() { + let mut service = RamService::default(); + assert!(service.ensure_session(SessionId::new(0))); + service + .register_system_invocation_view(0xA, Rights::ALL) + .expect("full-rights sibling view"); + service + .register_system_invocation_view(0xB, Rights::READ_ONLY.union(Rights::SHARE)) + .expect("delegable read-only sibling view"); + + let full = service + .invocation_views + .iter() + .flatten() + .find(|view| view.scope == 0xA) + .expect("full view registered"); + let read_only = service + .invocation_views + .iter() + .flatten() + .find(|view| view.scope == 0xB) + .expect("read-only view registered"); + assert_eq!(full.rights, Rights::ALL); + assert_eq!(read_only.rights, Rights::READ_ONLY.union(Rights::SHARE)); + assert!(!read_only.rights.contains(Rights::WRITE)); + assert!(!read_only.rights.contains(Rights::MUTATE_NAMESPACE)); + } + + #[test] + fn private_task_views_start_isolated_and_share_only_by_stable_grant() { + let mut service = RamService::default(); + let task_a = SessionId::new(FIRST_TASK_VIEW_SESSION as u16); + let task_b = SessionId::new(FIRST_TASK_VIEW_SESSION as u16 + 1); + install_private_task_view(&mut service, 0xA0, task_a, Rights::ALL); + install_private_task_view(&mut service, 0xB0, task_b, Rights::ALL); + + let command_a = admit_view(&mut service, 0x102, 0xA0, Rights::ALL); + let root_a = service.engine.open_root(command_a, Rights::ALL).unwrap(); + let shared = block_on(service.engine.create( + command_a, + root_a.handle, + &Name::new("shared").unwrap(), + ObjectKind::Directory, + )) + .unwrap(); + let file = block_on(service.engine.create( + command_a, + shared.handle, + &Name::new("value").unwrap(), + ObjectKind::File, + )) + .unwrap(); + block_on( + service + .engine + .write_at(command_a, file.handle, 0, b"one", 0), + ) + .unwrap(); + assert!(service.peer_closed(0x102, PeerClose::Orderly)); + + let command_b = admit_view(&mut service, 0x103, 0xB0, Rights::ALL); + let root_b = service.engine.open_root(command_b, Rights::ALL).unwrap(); + assert_eq!( + service.engine.open_at( + command_b, + root_b.handle, + &Name::new("shared").unwrap(), + Rights::READ, + ), + Err(FsError::NotFound), + "a sibling private root must not reveal another task's names" + ); + assert!(service.peer_closed(0x103, PeerClose::Orderly)); + let authority_baseline = service.engine.authority_edge_count(); + + let command_a = admit_view(&mut service, 0x202, 0xA0, Rights::ALL); + let root_a = service.engine.open_root(command_a, Rights::ALL).unwrap(); + let shared = service + .engine + .open_at( + command_a, + root_a.handle, + &Name::new("shared").unwrap(), + Rights::ALL, + ) + .unwrap(); + let grant = service + .engine + .grant_between_task_views( + command_a, + shared.handle, + task_a, + task_b, + Rights::READ, + Some(Name::new("from-a").unwrap()), + ) + .unwrap(); + assert!(service.peer_closed(0x202, PeerClose::Orderly)); + + let command_b = admit_view(&mut service, 0x203, 0xB0, Rights::ALL); + let opened = service + .engine + .open_task_grant(command_b, task_b, grant, Rights::READ) + .unwrap(); + let value = service + .engine + .open_at( + command_b, + opened.handle, + &Name::new("value").unwrap(), + Rights::READ, + ) + .unwrap(); + let mut bytes = [0u8; 8]; + assert_eq!( + block_on(service.engine.read(command_b, value.handle, 0, &mut bytes)), + Ok(3) + ); + assert_eq!(&bytes[..3], b"one"); + + // The attachment is a live route to the same object, not a copied + // snapshot. A later command in A's independent task view changes the + // object and B's already-open attenuated route observes that revision. + let command_a = admit_view(&mut service, 0x302, 0xA0, Rights::ALL); + let root_a = service.engine.open_root(command_a, Rights::ALL).unwrap(); + let shared_a = service + .engine + .open_at( + command_a, + root_a.handle, + &Name::new("shared").unwrap(), + Rights::ALL, + ) + .unwrap(); + let value_a = service + .engine + .open_at( + command_a, + shared_a.handle, + &Name::new("value").unwrap(), + Rights::ALL, + ) + .unwrap(); + block_on( + service + .engine + .write_at(command_a, value_a.handle, 0, b"two", 0), + ) + .unwrap(); + assert!(service.peer_closed(0x302, PeerClose::Orderly)); + bytes.fill(0); + assert_eq!( + block_on(service.engine.read(command_b, value.handle, 0, &mut bytes)), + Ok(3) + ); + assert_eq!(&bytes[..3], b"two"); + assert_eq!( + service + .engine + .open_task_grant(command_b, task_b, grant, Rights::WRITE), + Err(FsError::AccessDenied) + ); + assert!(service.peer_closed(0x203, PeerClose::Orderly)); + + block_on(service.engine.revoke(task_b, grant)).unwrap(); + let command_b = admit_view(&mut service, 0x303, 0xB0, Rights::ALL); + assert_eq!( + service + .engine + .open_task_grant(command_b, task_b, grant, Rights::READ), + Err(FsError::AccessLost) + ); + assert!(service.peer_closed(0x303, PeerClose::Orderly)); + assert_eq!( + service.engine.authority_edge_count(), + authority_baseline, + "revoking the stable attachment and closing its transient projections returns the authority edge to baseline" + ); + } + + /// The faces programs read files through are bounded by the slots the + /// serve host can name, not by its bootstrap table: a face in every + /// slot roots under one view at once, each in storage of its own, and + /// every slot takes its next occupant after the last one closes. + #[test] + fn a_face_in_every_slot_the_host_can_grow_reads_files_at_once() { + let mut service = RamService::default(); + let system = SessionId::new(0); + assert!(service.ensure_session(system)); + let system_root = service.engine.open_root(system, Rights::ALL).unwrap(); + block_on(service.engine.create( + system, + system_root.handle, + &Name::new("visible").unwrap(), + ObjectKind::File, + )) + .unwrap(); + service + .register_system_invocation_view(0xA, Rights::READ) + .unwrap(); + let edges = service.engine.authority_edge_count(); + + let first = usize::from(crate::service::contract::ADMIN_VIA) + 1; + let slots = first..crate::bounds::PAIR_SESSIONS; + assert!(slots.len() > kore_rt::svc::MAX_SERVE_PAIRS); + for generation in 1..=2u16 { + for slot in slots.clone() { + let peer = PeerId::from_arrival((generation << 8) | slot as u16).unwrap(); + assert!(service.peer_admitted(Ok(peer)), "slot {slot}"); + assert_eq!( + service.take_peer_admission(), + FsResponse::PeerAdmitted { peer } + ); + service + .apply_invocation_view(0xA, peer, Rights::READ) + .unwrap(); + } + for slot in slots.clone() { + let via = (generation << 8) | slot as u16; + let session = service.session_for_via(via).unwrap(); + assert_eq!(session, SessionId::new(slot as u16)); + let root = service.engine.open_root(session, Rights::READ).unwrap(); + service + .engine + .open_at( + session, + root.handle, + &Name::new("visible").unwrap(), + Rights::READ, + ) + .unwrap(); + } + assert_eq!( + service.engine.authority_edge_count(), + edges + slots.len(), + "one delegated root per face" + ); + for slot in slots.clone() { + assert!(service.peer_closed((generation << 8) | slot as u16, PeerClose::Orderly)); + } + assert_eq!(service.engine.authority_edge_count(), edges); + // The previous generation names nothing. + assert_eq!( + service.session_for_via((generation << 8) | first as u16), + None + ); + } + } + + /// A launch can fail after its face was admitted and before its view + /// was applied. The peer's session is still parked in its attach + /// transition then; closing it gives the slot back, and the slot's next + /// occupant is admitted and rooted as usual. + #[test] + fn a_peer_closed_before_its_view_gives_its_slot_back() { + let mut service = RamService::default(); + let system = SessionId::new(0); + assert!(service.ensure_session(system)); + service + .register_system_invocation_view(0xA, Rights::READ) + .unwrap(); + let parked = PeerId::from_arrival(0x105).unwrap(); + assert!(service.peer_admitted(Ok(parked))); + assert!(service.peer_closed(parked.arrival(), PeerClose::Died)); + assert!(service.peer_is_retired(parked)); + + let next = PeerId::from_arrival(0x205).unwrap(); + assert!(service.peer_admitted(Ok(next))); + service + .apply_invocation_view(0xA, next, Rights::READ) + .unwrap(); + let session = SessionId::new(next.slot() as u16); + assert!(service.engine.open_root(session, Rights::READ).is_ok()); + assert!(service.peer_closed(next.arrival(), PeerClose::Orderly)); + } + + #[test] + fn a_cwd_anchor_matches_its_view_through_renames_and_answers_no_other_task() { + let mut service = RamService::default(); + let task_a = SessionId::new(FIRST_TASK_VIEW_SESSION as u16); + let task_b = SessionId::new(FIRST_TASK_VIEW_SESSION as u16 + 1); + install_private_task_view(&mut service, 0xA0, task_a, Rights::ALL); + install_private_task_view(&mut service, 0xB0, task_b, Rights::ALL); + let name = |text| Name::new(text).unwrap(); + + // `cd proj` in task A: one command pins the cwd under scope 0xA1. + let binder = admit_view(&mut service, 0x102, 0xA0, Rights::ALL); + let root = service.engine.open_root(binder, Rights::ALL).unwrap(); + for directory in ["proj", "work"] { + block_on(service.engine.create(binder, root.handle, &name(directory), ObjectKind::Directory)) + .unwrap(); + } + let proj = service + .engine + .open_at(binder, root.handle, &name("proj"), Rights::ALL) + .unwrap(); + service + .register_invocation_view(binder, proj.handle, Rights::ALL, 0xA1) + .unwrap(); + assert!(service.peer_closed(0x102, PeerClose::Orderly)); + + // A later command of the same task walks the remembered route. + let command = admit_view(&mut service, 0x103, 0xA0, Rights::ALL); + let root = service.engine.open_root(command, Rights::ALL).unwrap(); + let walk = |service: &RamService, text| { + service + .engine + .open_at(command, root.handle, &name(text), Rights::READ) + .unwrap() + .handle + }; + assert_eq!(service.match_view(command, walk(&service, "proj"), 0xA1), FsResponse::Done); + assert_eq!(service.match_view(command, root.handle, 0xA0), FsResponse::Done); + assert_eq!( + service.match_view(command, walk(&service, "work"), 0xA1), + FsResponse::Refused(FsError::Conflict) + ); + + // The cwd is renamed, and another directory takes its old name. The + // view still holds the same directory, now `old`; the remembered + // route names a stranger, and the anchor says so. + service.engine.rename(command, root.handle, &name("proj"), &name("old")).unwrap(); + service.engine.rename(command, root.handle, &name("work"), &name("proj")).unwrap(); + assert_eq!( + service.match_view(command, walk(&service, "proj"), 0xA1), + FsResponse::Refused(FsError::Conflict) + ); + assert_eq!(service.match_view(command, walk(&service, "old"), 0xA1), FsResponse::Done); + assert_eq!( + service.match_view(command, walk(&service, "old"), 0xC0), + FsResponse::Refused(FsError::Conflict), + "a selector this store binds no view under names nothing here" + ); + + // Task B's command learns nothing about task A's cwd. + let other = admit_view(&mut service, 0x104, 0xB0, Rights::ALL); + let other_root = service.engine.open_root(other, Rights::ALL).unwrap(); + assert_eq!( + service.match_view(other, other_root.handle, 0xA1), + FsResponse::Refused(FsError::AccessDenied) + ); + assert!(service.peer_closed(0x103, PeerClose::Orderly)); + assert!(service.peer_closed(0x104, PeerClose::Orderly)); + } + + #[test] + fn narrowed_invocation_view_outlives_the_process_that_bound_it() { + let mut service = RamService::default(); + let system = SessionId::new(0); + assert!(service.ensure_session(system)); + let system_root = service.engine.open_root(system, Rights::ALL).unwrap(); + let directory = block_on(service.engine.create( + system, + system_root.handle, + &Name::new("shared").unwrap(), + ObjectKind::Directory, + )) + .unwrap(); + block_on(service.engine.create( + system, + directory.handle, + &Name::new("visible").unwrap(), + ObjectKind::File, + )) + .unwrap(); + + let delegated = Rights::READ.union(Rights::SHARE); + service + .register_system_invocation_view(0xA, delegated) + .unwrap(); + let first = PeerId::from_arrival(0x102).unwrap(); + assert!(service.peer_admitted(Ok(first))); + service + .apply_invocation_view(0xA, first, delegated) + .unwrap(); + let first_session = SessionId::new(first.slot() as u16); + let first_root = service.engine.open_root(first_session, delegated).unwrap(); + let narrowed = service + .engine + .open_at( + first_session, + first_root.handle, + &Name::new("shared").unwrap(), + delegated, + ) + .unwrap(); + service + .register_invocation_view(first_session, narrowed.handle, delegated, 0xB) + .unwrap(); + + assert!(service.peer_closed(0x102, PeerClose::Orderly)); + let second = PeerId::from_arrival(0x103).unwrap(); + assert!(service.peer_admitted(Ok(second))); + service + .apply_invocation_view(0xB, second, delegated) + .expect("the caller-rooted cwd survives its binder task"); + let second_session = SessionId::new(second.slot() as u16); + let second_root = service + .engine + .open_root(second_session, Rights::READ) + .unwrap(); + assert!( + service + .engine + .open_at( + second_session, + second_root.handle, + &Name::new("visible").unwrap(), + Rights::READ, + ) + .is_ok() + ); + assert!(service.peer_closed(0x103, PeerClose::Orderly)); + let pinned = service + .invocation_views + .iter() + .flatten() + .find(|view| view.scope == 0xB) + .map(|view| (view.task_session, view.grant)) + .unwrap(); + block_on(service.engine.revoke(pinned.0, pinned.1)) + .expect("the stable pin owner can retire the narrowed view exactly once"); + } + + #[test] + fn recycled_process_sessions_share_only_explicit_native_grants() { + let mut service = RamService::default(); + let admin = SessionId::new(crate::service::contract::ADMIN_VIA); + let process = SessionId::new(2); + assert!(service.ensure_session(admin)); + assert!(service.ensure_session(process)); + + let workspace = service + .engine + .open_root(admin, Rights::ALL) + .expect("admin workspace root"); + let display = NativeView::Data.name(); + let first = service + .engine + .grant(admin, workspace.handle, process, Rights::ALL, Some(display)) + .expect("first process native grant"); + let first_root = service + .engine + .open_grant(process, first, Rights::ALL) + .expect("first process opens workspace"); + block_on(service.engine.create( + process, + first_root.handle, + &Name::new("kept").unwrap(), + ObjectKind::File, + )) + .expect("first process mutates workspace"); + + block_on(service.engine.detach_session(process)).expect("first process exits"); + assert!(service.ensure_session(process)); + let replacement_private = service + .engine + .open_root(process, Rights::READ) + .expect("replacement private root"); + assert_eq!( + service.engine.open_at( + process, + replacement_private.handle, + &Name::new("kept").unwrap(), + Rights::READ, + ), + Err(FsError::NotFound), + "recycling the pair must still create an isolated private root" + ); + let second = service + .engine + .grant(admin, workspace.handle, process, Rights::ALL, Some(display)) + .expect("replacement process native grant"); + let second_root = service + .engine + .open_grant(process, second, Rights::READ) + .expect("replacement process opens workspace"); + assert!( + service + .engine + .open_at( + process, + second_root.handle, + &Name::new("kept").unwrap(), + Rights::READ, + ) + .is_ok(), + "retiring the ephemeral recipient must not retire the issuer's route" + ); + } + + #[test] + fn native_view_vocabulary_is_complete_and_collision_free() { + let expected = ["app", "data", "config", "tmp", "shared"]; + for (index, view) in NativeView::ALL.into_iter().enumerate() { + assert_eq!(view.as_str(), expected[index]); + assert_eq!(view.name().as_str(), expected[index]); + assert!( + NativeView::ALL[..index] + .iter() + .all(|prior| prior.as_str() != view.as_str()), + "native attachment names must be unique" + ); + } + } + + #[test] + fn retirement_fence_is_exact_across_slot_reuse() { + let mut service = RamService::default(); + let first = PeerId::from_arrival(0x101).expect("valid first generation"); + let replacement = PeerId::from_arrival(0x201).expect("valid replacement generation"); + + service.slots.claim(1).unwrap().via = Some(first.arrival()); + assert!(!service.peer_is_retired(first)); + assert!(service.peer_is_retired(replacement)); + + service.slots.claim(1).unwrap().via = Some(replacement.arrival()); + assert!(service.peer_is_retired(first)); + assert!(!service.peer_is_retired(replacement)); + } +} + +impl Default for RamService { + fn default() -> Self { + Self::new(Engine::new(RamProvider::new())) + } +} + +/// A route as the wire spells it: names joined by `/`, no leading slash, +/// into the search-hit record's fixed path field. The field is proven wide +/// enough for the deepest legal route at compile time (`bounds`); this keeps +/// a drift there a refusal, never an index past the array. +fn spell_path(names: &[Name]) -> FsResult<([u8; FS_SEARCH_PATH_BYTES], usize)> { + let mut path = [0u8; FS_SEARCH_PATH_BYTES]; + let mut len = 0usize; + for (index, name) in names.iter().enumerate() { + if index != 0 { + if len >= path.len() { + return Err(FsError::Range); + } + path[len] = b'/'; + len += 1; + } + let bytes = name.as_bytes(); + let end = len.checked_add(bytes.len()).ok_or(FsError::Range)?; + if end > path.len() { + return Err(FsError::Range); + } + path[len..end].copy_from_slice(bytes); + len = end; + } + Ok((path, len)) +} diff --git a/native/services/dagfs/src/slot.rs b/native/services/dagfs/src/slot.rs new file mode 100644 index 0000000..a4c5a19 --- /dev/null +++ b/native/services/dagfs/src/slot.rs @@ -0,0 +1,60 @@ +//! Filesystem names for the shared bounded generational claim map. +//! +//! The first filesystem implementation carried a private, single-owner +//! `SlotTable`. That preserved generations but forced every operation through +//! one mutable engine borrow. Phase 5 uses the workspace collection directly: +//! each object, directory, authority edge, and handle can now be claimed +//! independently, with explicit contention and no filesystem-global lock. + +pub use kore_collections::{ + AtomicBitVec, AtomicSlotMap as SlotTable, AtomicSlotVec, BoundedVec, SlotAccess, + SlotId as SlotKey, SlotRead, SlotVecRead, SlotVecWrite, SlotWrite, +}; + +use crate::error::FsError; + +/// Preserve the claim map's temporal distinction at the filesystem seam. +/// +/// Callers supply the meaning of a stale generation (`InvalidHandle` for an +/// opaque client handle, `AccessLost` for an internal route). Contention has +/// one meaning everywhere: bounded, retryable [`FsError::Busy`]. +#[inline] +pub(crate) const fn fs_access(error: SlotAccess, stale: FsError) -> FsError { + match error { + SlotAccess::Stale => stale, + SlotAccess::Busy => FsError::Busy, + } +} + +/// Service-private access to the arena key behind a wire-visible identity. +/// +/// [`Handle`], [`GrantId`], and [`VersionLease`] are process-side leaves +/// (`kore_dagfs`): a client holds and encodes them without knowing they name +/// generational arena slots. Only this service reconstructs the slot, and it +/// does so through the public bit forms — so the leaf's contract stays +/// "opaque bits" while the exact stale-generation checks keep working. +pub(crate) trait Keyed: Sized { + fn key(self) -> SlotKey; + fn from_key(key: SlotKey) -> Self; +} + +macro_rules! keyed { + ($($id:ty),* $(,)?) => {$( + impl Keyed for $id { + #[inline] + fn key(self) -> SlotKey { + SlotKey::from_bits(self.to_bits()) + } + #[inline] + fn from_key(key: SlotKey) -> Self { + Self::from_bits(key.to_bits()) + } + } + )*}; +} + +keyed!( + kore_dagfs::Handle, + kore_dagfs::GrantId, + kore_dagfs::VersionLease +); diff --git a/native/services/dagfs/src/window.rs b/native/services/dagfs/src/window.rs new file mode 100644 index 0000000..2110fad --- /dev/null +++ b/native/services/dagfs/src/window.rs @@ -0,0 +1,312 @@ +//! The per-client bulk **data window** — every checked slice in one place. +//! +//! File bytes and directory listings cross through a region the *client* +//! carved out of its own memory and delivered into this service's arena, +//! following `OutReqWire::OP_ATTACH_TX`'s precedent. A request names an offset +//! and a length **into that window**; no address and no capability ever rides +//! the wire. +//! +//! Which makes every bulk request a bounds problem, and this the module that +//! owns it. The offsets arrive from another process as plain `u32`s, so each +//! one is checked-added and range-tested before a single byte moves — a wrapped +//! `offset + len` that landed back inside the window would be an out-of-bounds +//! read the service performed on its own memory at a peer's request. +//! +//! The window is a `&mut [u8]` rather than a mapped pointer so the whole of it +//! is testable on the host: on metal the serving host hands over its mapping of +//! the delivered view, and in tests a plain buffer stands in. Nothing here +//! knows which it has. + +use kore_abi::svc::{FsDirEntryWire, FsSearchHitWire}; + +use crate::error::{FsError, FsResult}; + +/// A bounds-checked view of one client's data window. +pub struct DataWindow<'w> { + bytes: &'w mut [u8], +} + +impl<'w> DataWindow<'w> { + /// Wrap a window's bytes. + pub fn new(bytes: &'w mut [u8]) -> Self { + Self { bytes } + } + + /// The window's length in bytes. + #[inline] + pub fn len(&self) -> usize { + self.bytes.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + /// The `offset..offset+len` span, if it lies wholly inside the window. + /// + /// Every arithmetic step is checked: a `u32` pair from a peer can overflow + /// `usize` on a 32-bit target and can wrap into a plausible-looking range + /// on any target. + fn span(&self, offset: u32, len: u32) -> FsResult> { + let offset = usize::try_from(offset).map_err(|_| FsError::Range)?; + let len = usize::try_from(len).map_err(|_| FsError::Range)?; + let end = offset.checked_add(len).ok_or(FsError::Range)?; + if end > self.bytes.len() { + return Err(FsError::Range); + } + Ok(offset..end) + } + + /// Borrow `len` bytes at `offset` for reading — the source of a write. + pub fn source(&self, offset: u32, len: u32) -> FsResult<&[u8]> { + let span = self.span(offset, len)?; + Ok(&self.bytes[span]) + } + + /// Borrow `len` bytes at `offset` for writing — the destination of a read. + pub fn destination(&mut self, offset: u32, len: u32) -> FsResult<&mut [u8]> { + let span = self.span(offset, len)?; + Ok(&mut self.bytes[span]) + } + + /// How many whole directory entries fit at `offset`. + /// + /// A listing writes fixed-size records, so a window with room for two and a + /// half entries has room for two — a partial trailing entry would be + /// garbage a client might parse. + pub fn entry_capacity(&self, offset: u32) -> usize { + let Ok(offset) = usize::try_from(offset) else { + return 0; + }; + self.bytes + .len() + .saturating_sub(offset) + .checked_div(FsDirEntryWire::BYTES) + .unwrap_or(0) + } + + /// Place one directory entry at `offset + index * BYTES`. + pub fn put_entry(&mut self, offset: u32, index: usize, entry: &FsDirEntryWire) -> FsResult<()> { + use zerocopy::IntoBytes; + self.put_bytes(offset, index, FsDirEntryWire::BYTES, entry.as_bytes()) + } + + /// Place one authority-edge entry at `offset + index * BYTES`. + /// + /// Same granule as [`Self::put_entry`], so a client's window arithmetic is + /// identical for both listings. + pub fn put_grant_entry( + &mut self, + offset: u32, + index: usize, + entry: &kore_abi::svc::FsGrantEntryWire, + ) -> FsResult<()> { + use zerocopy::IntoBytes; + self.put_bytes( + offset, + index, + kore_abi::svc::FsGrantEntryWire::BYTES, + entry.as_bytes(), + ) + } + + /// Land the one diagnostic snapshot record at `offset`. + pub fn put_system_status( + &mut self, + offset: u32, + record: &kore_abi::svc::FsSystemStatusWire, + ) -> FsResult<()> { + use zerocopy::IntoBytes; + self.put_bytes( + offset, + 0, + kore_abi::svc::FsSystemStatusWire::BYTES, + record.as_bytes(), + ) + } + + /// How many complete structural-search hits fit at `offset`. + pub fn search_capacity(&self, offset: u32) -> usize { + let Ok(offset) = usize::try_from(offset) else { + return 0; + }; + self.bytes + .len() + .saturating_sub(offset) + .checked_div(FsSearchHitWire::BYTES) + .unwrap_or(0) + } + + /// Place one fixed-size search hit at `offset + index * BYTES`. + pub fn put_search_hit( + &mut self, + offset: u32, + index: usize, + hit: &FsSearchHitWire, + ) -> FsResult<()> { + use zerocopy::IntoBytes; + self.put_bytes(offset, index, FsSearchHitWire::BYTES, hit.as_bytes()) + } + + /// The shared placement arithmetic behind both entry writers. + fn put_bytes( + &mut self, + offset: u32, + index: usize, + stride: usize, + bytes: &[u8], + ) -> FsResult<()> { + let step = index.checked_mul(stride).ok_or(FsError::Range)?; + let at = usize::try_from(offset) + .map_err(|_| FsError::Range)? + .checked_add(step) + .ok_or(FsError::Range)?; + let end = at.checked_add(bytes.len()).ok_or(FsError::Range)?; + if end > self.bytes.len() { + return Err(FsError::Range); + } + self.bytes[at..end].copy_from_slice(bytes); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_span_inside_the_window_is_borrowable_both_ways() { + let mut buf = [0u8; 64]; + let mut w = DataWindow::new(&mut buf); + assert_eq!(w.len(), 64); + assert!(!w.is_empty()); + + w.destination(8, 4).unwrap().copy_from_slice(b"kore"); + assert_eq!(w.source(8, 4).unwrap(), b"kore"); + // Untouched bytes stayed zero — the borrow was exactly the span asked for. + assert_eq!(w.source(0, 8).unwrap(), &[0u8; 8]); + assert_eq!(w.source(12, 4).unwrap(), &[0u8; 4]); + } + + #[test] + fn the_whole_window_and_an_empty_span_are_both_legal() { + let mut buf = [7u8; 16]; + let mut w = DataWindow::new(&mut buf); + assert_eq!(w.source(0, 16).unwrap().len(), 16); + assert_eq!( + w.source(16, 0).unwrap().len(), + 0, + "an empty span at the end" + ); + assert_eq!(w.destination(0, 16).unwrap().len(), 16); + } + + #[test] + fn a_span_past_the_end_is_refused() { + let mut buf = [0u8; 16]; + let mut w = DataWindow::new(&mut buf); + assert_eq!(w.source(0, 17).unwrap_err(), FsError::Range); + assert_eq!(w.source(16, 1).unwrap_err(), FsError::Range); + assert_eq!(w.source(17, 0).unwrap_err(), FsError::Range); + assert_eq!(w.destination(8, 9).unwrap_err(), FsError::Range); + } + + #[test] + fn an_overflowing_span_is_refused_not_wrapped() { + // The dangerous case: a wrapped `offset + len` that lands back inside + // the window would be an out-of-bounds access the service performs on + // its own memory because a peer asked nicely. + let mut buf = [0u8; 16]; + let mut w = DataWindow::new(&mut buf); + assert_eq!(w.source(u32::MAX, 1).unwrap_err(), FsError::Range); + assert_eq!(w.source(1, u32::MAX).unwrap_err(), FsError::Range); + assert_eq!( + w.destination(u32::MAX, u32::MAX).unwrap_err(), + FsError::Range + ); + } + + #[test] + fn an_empty_window_admits_nothing() { + let mut buf: [u8; 0] = []; + let mut w = DataWindow::new(&mut buf); + assert!(w.is_empty()); + assert_eq!(w.source(0, 0).unwrap().len(), 0); + assert_eq!(w.source(0, 1).unwrap_err(), FsError::Range); + assert_eq!(w.destination(0, 1).unwrap_err(), FsError::Range); + assert_eq!(w.entry_capacity(0), 0); + } + + #[test] + fn entry_capacity_counts_only_whole_entries() { + let mut buf = [0u8; FsDirEntryWire::BYTES * 2 + 8]; + let w = DataWindow::new(&mut buf); + assert_eq!(w.entry_capacity(0), 2, "the trailing 8 bytes hold no entry"); + assert_eq!(w.entry_capacity(FsDirEntryWire::BYTES as u32), 1); + assert_eq!(w.entry_capacity(buf_len_as_u32(&w)), 0); + } + + fn buf_len_as_u32(w: &DataWindow<'_>) -> u32 { + w.len() as u32 + } + + #[test] + fn entry_capacity_saturates_rather_than_underflowing() { + let mut buf = [0u8; 16]; + let w = DataWindow::new(&mut buf); + // An offset past the end must report no room, not wrap to a huge count. + assert_eq!(w.entry_capacity(1_000_000), 0); + assert_eq!(w.entry_capacity(u32::MAX), 0); + } + + #[test] + fn entries_land_at_their_indexed_offsets() { + use kore_abi::svc::FsReqWire; + let mut buf = [0u8; FsDirEntryWire::BYTES * 3]; + let mut w = DataWindow::new(&mut buf); + let mut entry = FsDirEntryWire { + name_len: 1, + kind: FsReqWire::KIND_FILE, + reserved0: 0.into(), + reserved1: 0.into(), + object: 0x1234.into(), + name: [0; kore_abi::svc::FS_NAME_BYTES], + }; + entry.name[0] = b'a'; + w.put_entry(0, 0, &entry).unwrap(); + entry.name[0] = b'c'; + w.put_entry(0, 2, &entry).unwrap(); + + assert_eq!(buf[0], 1, "first entry's name_len"); + assert_eq!(buf[16], b'a', "first entry's name"); + assert_eq!( + buf[FsDirEntryWire::BYTES], + 0, + "the skipped slot is untouched" + ); + assert_eq!(buf[FsDirEntryWire::BYTES * 2 + 16], b'c'); + } + + #[test] + fn an_entry_past_the_window_is_refused() { + let mut buf = [0u8; FsDirEntryWire::BYTES]; + let mut w = DataWindow::new(&mut buf); + let entry = FsDirEntryWire { + name_len: 1, + kind: kore_abi::svc::FsReqWire::KIND_FILE, + reserved0: 0.into(), + reserved1: 0.into(), + object: 0.into(), + name: [0; kore_abi::svc::FS_NAME_BYTES], + }; + assert!(w.put_entry(0, 0, &entry).is_ok()); + assert_eq!(w.put_entry(0, 1, &entry).unwrap_err(), FsError::Range); + assert_eq!(w.put_entry(1, 0, &entry).unwrap_err(), FsError::Range); + // And an index that would overflow the multiply is refused, not wrapped. + assert_eq!( + w.put_entry(0, usize::MAX, &entry).unwrap_err(), + FsError::Range + ); + } +} diff --git a/native/services/dagfs/src/wire.rs b/native/services/dagfs/src/wire.rs new file mode 100644 index 0000000..b4a36b2 --- /dev/null +++ b/native/services/dagfs/src/wire.rs @@ -0,0 +1,9 @@ +//! The fail-closed codecs over `kore_abi::svc`'s frozen filesystem forms. +//! +//! The codec is the client's as much as the service's — the same bytes are +//! encoded on one side and decoded on the other — so it lives in the +//! process-side leaf ([`kore_dagfs::wire`]) and the service consumes it from +//! there. This module exists so the serving host and the router keep their +//! `crate::wire::…` spelling. + +pub use kore_dagfs::wire::*; diff --git a/native/services/dagfs/tests/authority.rs b/native/services/dagfs/tests/authority.rs new file mode 100644 index 0000000..e6c496b --- /dev/null +++ b/native/services/dagfs/tests/authority.rs @@ -0,0 +1,922 @@ +//! Authority behaviour: private roots, sharing, attenuation, and immediate +//! loss of access. +//! +//! Covers the authority half of the plan's §10.1 matrix. These are the tests +//! the MVP exists to pass: if any of them regresses, the filesystem's +//! capability story is broken even if every byte still round-trips. + +mod common; + +use common::{ + create_dir, create_file, engine_with_one, engine_with_sessions, list_names, name, read_all, +}; +use kore_svc_dagfs::{ + ANY_REVISION, AuthorityKind, FsError, GrantId, Handle, ObjectKind, Rights, SessionId, block_on, + bounds::MAX_HANDLES_PER_SESSION, +}; + +// ── private roots ─────────────────────────────────────────────────────────── + +#[test] +fn every_session_starts_with_its_own_empty_root() { + let (engine, clients) = engine_with_sessions(3); + for (session, root) in clients.iter().copied() { + let info = engine.stat(session, root).unwrap(); + assert_eq!(info.kind, ObjectKind::Directory); + assert!(list_names(&engine, session, root).unwrap().is_empty()); + } + // Three sessions, three distinct root objects — there is no shared `/`. + let mut roots: Vec<_> = clients + .iter() + .map(|(session, root)| engine.stat(*session, *root).unwrap().object.to_bits()) + .collect(); + roots.sort(); + roots.dedup(); + assert_eq!(roots.len(), 3, "no two sessions share a root object"); +} + +#[test] +fn the_same_pathname_in_two_private_roots_resolves_to_different_objects() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, root_b) = clients[1]; + + let file_a = create_file(&mut engine, a, root_a, "notes", b"belongs to A"); + let file_b = create_file(&mut engine, b, root_b, "notes", b"belongs to B"); + + let id_a = engine.stat(a, file_a).unwrap().object; + let id_b = engine.stat(b, file_b).unwrap().object; + assert_ne!(id_a, id_b, "same name, different objects"); + + // And each sees only its own bytes. + assert_eq!(read_all(&mut engine, a, file_a).unwrap(), b"belongs to A"); + assert_eq!(read_all(&mut engine, b, file_b).unwrap(), b"belongs to B"); +} + +#[test] +fn a_session_cannot_see_anothers_names() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, root_b) = clients[1]; + + create_file(&mut engine, a, root_a, "secret", b"private"); + + // B's root simply has no such name; there is no route to attempt. + assert_eq!( + engine + .open_at(b, root_b, &name("secret"), Rights::READ) + .unwrap_err(), + FsError::NotFound + ); + assert!(list_names(&engine, b, root_b).unwrap().is_empty()); +} + +#[test] +fn a_task_with_no_session_has_no_filesystem_access() { + let (mut engine, _clients) = engine_with_sessions(1); + // Filesystem access is not ambient: a session that was never attached has + // no root to open (plan §12.7). + assert_eq!( + engine + .open_root(SessionId::new(2), Rights::ALL) + .unwrap_err(), + FsError::InvalidHandle + ); + assert!(engine.grants(SessionId::new(2)).unwrap_err() == FsError::InvalidHandle); +} + +// ── handle isolation ──────────────────────────────────────────────────────── + +#[test] +fn one_sessions_handle_is_invalid_in_another_session() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + let file_a = create_file(&mut engine, a, root_a, "mine", b"A's bytes"); + + // B presenting A's handle bits gets nothing — not A's object, not its own. + assert_eq!(engine.stat(b, file_a).unwrap_err(), FsError::InvalidHandle); + assert_eq!( + block_on(engine.read(b, file_a, 0, &mut [0u8; 4])).unwrap_err(), + FsError::InvalidHandle + ); + assert_eq!(engine.close(b, file_a).unwrap_err(), FsError::InvalidHandle); + // A's handle is untouched by the attempts. + assert_eq!(read_all(&mut engine, a, file_a).unwrap(), b"A's bytes"); +} + +#[test] +fn forged_and_stale_handles_fail_closed() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b"bytes"); + + // Fabricated bit patterns: the never-issued sentinel, a wrong generation + // on a slot that *is* live, and out-of-range slots. (Deliberately not + // "any small integer" — a session's own live handles have small bit + // patterns, and re-presenting your own handle is not forgery.) + let live = engine.stat(session, root).unwrap(); + assert_eq!(live.kind, ObjectKind::Directory); + let wrong_generation = Handle::from_bits(root.to_bits() + 1); + for candidate in [ + Handle::from_bits(0), + wrong_generation, + Handle::from_bits(u64::MAX), + Handle::from_bits(0xdead_beef_dead_beef), + ] { + assert_eq!( + engine.stat(session, candidate).unwrap_err(), + FsError::InvalidHandle, + "forged handle {:#x} must fail closed", + candidate.to_bits() + ); + } + + // A closed handle is stale, even though its slot may be reused. + engine.close(session, file).unwrap(); + assert_eq!( + engine.stat(session, file).unwrap_err(), + FsError::InvalidHandle + ); + let reopened = engine + .open_at(session, root, &name("f"), Rights::ALL) + .unwrap(); + // The reused slot must not resurrect the old handle. + assert_eq!( + engine.stat(session, file).unwrap_err(), + FsError::InvalidHandle + ); + assert!(engine.stat(session, reopened.handle).is_ok()); +} + +#[test] +fn a_forged_grant_id_is_inert() { + let (mut engine, session, _root) = engine_with_one(); + // Derive a wrong-generation variant of the session's own real root grant, + // so the test forges something plausible rather than something that + // happens to be live. + let real = engine + .grants(session) + .unwrap() + .iter() + .find(|info| info.kind == AuthorityKind::PrivateRoot) + .expect("its private root is listed") + .grant; + for forged in [ + GrantId::from_bits(0), + GrantId::from_bits(real.to_bits() + 1), + GrantId::from_bits(u64::MAX), + GrantId::from_bits(0xdead_beef_dead_beef), + ] { + assert!( + engine.open_grant(session, forged, Rights::READ).is_err(), + "forged grant {:#x} must not open", + forged.to_bits() + ); + } + // The real one does open, so the test is not passing by refusing + // everything. + assert!(engine.open_grant(session, real, Rights::READ).is_ok()); +} + +// ── sharing one file ──────────────────────────────────────────────────────── + +#[test] +fn a_read_only_file_grant_lets_the_recipient_read_and_nothing_else() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, root_b) = clients[1]; + + let file = create_file(&mut engine, a, root_a, "shared", b"from A"); + let grant = engine + .grant(a, file, b, Rights::READ, Some(name("gift"))) + .unwrap(); + + let opened = engine.open_grant(b, grant, Rights::READ).unwrap(); + assert_eq!(opened.kind, ObjectKind::File); + assert_eq!(read_all(&mut engine, b, opened.handle).unwrap(), b"from A"); + + // Read-only means read-only: no write, no share, no naming it elsewhere. + assert_eq!( + block_on(engine.write_at(b, opened.handle, 0, b"no", ANY_REVISION)).unwrap_err(), + FsError::AccessDenied + ); + assert_eq!( + block_on(engine.resize(b, opened.handle, 0, ANY_REVISION)).unwrap_err(), + FsError::AccessDenied + ); + assert_eq!( + engine + .grant(b, opened.handle, a, Rights::READ, None) + .unwrap_err(), + FsError::AccessDenied + ); + assert_eq!( + engine + .link(b, root_b, &name("copy"), opened.handle) + .unwrap_err(), + FsError::AccessDenied + ); + + // A's own bytes are unchanged after all of that. + assert_eq!(read_all(&mut engine, a, file).unwrap(), b"from A"); +} + +#[test] +fn a_recipient_cannot_open_a_grant_at_more_rights_than_it_carries() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + let file = create_file(&mut engine, a, root_a, "shared", b"x"); + let grant = engine.grant(a, file, b, Rights::READ, None).unwrap(); + + assert_eq!( + engine.open_grant(b, grant, Rights::WRITE).unwrap_err(), + FsError::AccessDenied + ); + assert_eq!( + engine.open_grant(b, grant, Rights::ALL).unwrap_err(), + FsError::AccessDenied + ); + // Exactly what it carries is fine. + assert!(engine.open_grant(b, grant, Rights::READ).is_ok()); +} + +#[test] +fn a_grant_cannot_convey_more_than_the_issuer_holds() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + create_file(&mut engine, a, root_a, "f", b"x"); + // A opens its own file read-only, then tries to share write access. + let read_only = engine.open_at(a, root_a, &name("f"), Rights::READ).unwrap(); + assert_eq!( + engine + .grant(a, read_only.handle, b, Rights::WRITE, None) + .unwrap_err(), + // Sharing itself needs SHARE, which a read-only route does not carry. + FsError::AccessDenied + ); +} + +#[test] +fn revoking_an_ancestor_invalidates_redelegated_handles() { + let (mut engine, clients) = engine_with_sessions(3); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + let (c, _root_c) = clients[2]; + + let file = create_file(&mut engine, a, root_a, "shared", b"lineage"); + let read_share = Rights::READ.union(Rights::SHARE); + let to_b = engine.grant(a, file, b, read_share, None).unwrap(); + let b_view = engine.open_grant(b, to_b, read_share).unwrap(); + let to_c = engine + .grant(b, b_view.handle, c, Rights::READ, None) + .unwrap(); + let c_view = engine.open_grant(c, to_c, Rights::READ).unwrap(); + assert_eq!(read_all(&mut engine, c, c_view.handle).unwrap(), b"lineage"); + + block_on(engine.revoke(a, to_b)).unwrap(); + assert_eq!( + engine.stat(b, b_view.handle).unwrap_err(), + FsError::AccessLost + ); + assert_eq!( + engine.stat(c, c_view.handle).unwrap_err(), + FsError::AccessLost + ); + assert!(engine.open_grant(c, to_c, Rights::READ).is_err()); + // The object remains reachable through A's independent namespace edge. + assert_eq!(read_all(&mut engine, a, file).unwrap(), b"lineage"); +} + +#[test] +fn detaching_an_intermediate_holder_invalidates_its_redelegation() { + let (mut engine, clients) = engine_with_sessions(3); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + let (c, _root_c) = clients[2]; + + let file = create_file(&mut engine, a, root_a, "shared", b"lineage"); + let read_share = Rights::READ.union(Rights::SHARE); + let to_b = engine.grant(a, file, b, read_share, None).unwrap(); + let b_view = engine.open_grant(b, to_b, read_share).unwrap(); + let to_c = engine + .grant(b, b_view.handle, c, Rights::READ, None) + .unwrap(); + let c_view = engine.open_grant(c, to_c, Rights::READ).unwrap(); + + block_on(engine.detach_session(b)).unwrap(); + assert_eq!( + engine.stat(c, c_view.handle).unwrap_err(), + FsError::AccessLost + ); + assert!(engine.open_grant(c, to_c, Rights::READ).is_err()); + assert_eq!(read_all(&mut engine, a, file).unwrap(), b"lineage"); +} + +#[test] +fn a_third_session_cannot_open_someone_elses_grant() { + let (mut engine, clients) = engine_with_sessions(3); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + let (c, _root_c) = clients[2]; + + let file = create_file(&mut engine, a, root_a, "shared", b"x"); + let grant = engine.grant(a, file, b, Rights::READ, None).unwrap(); + + // A GrantId is not bearer authority — C holding the bits gets nothing. + assert_eq!( + engine.open_grant(c, grant, Rights::READ).unwrap_err(), + FsError::InvalidHandle + ); + assert!(engine.open_grant(b, grant, Rights::READ).is_ok()); +} + +#[test] +fn a_recipient_discovers_its_grants_by_listing_them() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + let file = create_file(&mut engine, a, root_a, "f", b"x"); + engine + .grant(a, file, b, Rights::READ, Some(name("from-a"))) + .unwrap(); + + let listed = engine.grants(b).unwrap(); + assert_eq!(listed.len(), 2, "its private root plus the new grant"); + let attachment = listed + .iter() + .find(|info| info.kind == AuthorityKind::Grant) + .expect("the grant is listed"); + assert_eq!(attachment.rights, Rights::READ); + assert_eq!(attachment.display_name.unwrap().as_str(), "from-a"); + + // And the listed id is the one that opens. + assert!(engine.open_grant(b, attachment.grant, Rights::READ).is_ok()); +} + +#[test] +fn a_sender_cannot_overwrite_a_name_in_the_recipients_view() { + let (mut engine, clients) = engine_with_sessions(3); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + let (c, root_c) = clients[2]; + + let from_a = create_file(&mut engine, a, root_a, "f", b"a"); + let from_c = create_file(&mut engine, c, root_c, "f", b"c"); + engine + .grant(a, from_a, b, Rights::READ, Some(name("inbox"))) + .unwrap(); + + // C cannot claim the attachment name B already uses (plan §12.6). + assert_eq!( + engine + .grant(c, from_c, b, Rights::READ, Some(name("inbox"))) + .unwrap_err(), + FsError::AlreadyExists + ); + // Without a name, or with a free one, it is fine. + assert!(engine.grant(c, from_c, b, Rights::READ, None).is_ok()); + assert!( + engine + .grant(c, from_c, b, Rights::READ, Some(name("other"))) + .is_ok() + ); +} + +// ── sharing a subtree ─────────────────────────────────────────────────────── + +#[test] +fn a_subtree_grant_exposes_descendants_and_nothing_above_it() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + // A builds root/{above.txt, shared/{inside.txt, deeper/}} + create_file(&mut engine, a, root_a, "above.txt", b"not shared"); + let shared = create_dir(&mut engine, a, root_a, "shared"); + create_file(&mut engine, a, shared, "inside.txt", b"shared bytes"); + let deeper = create_dir(&mut engine, a, shared, "deeper"); + create_file(&mut engine, a, deeper, "deep.txt", b"deep bytes"); + + let grant = engine + .grant(a, shared, b, Rights::READ, Some(name("attached"))) + .unwrap(); + let view = engine.open_grant(b, grant, Rights::READ).unwrap(); + + // Everything at and below the granted root is reachable. + let mut listed = list_names(&engine, b, view.handle).unwrap(); + listed.sort(); + assert_eq!(listed, vec!["deeper", "inside.txt"]); + let inside = engine + .open_at(b, view.handle, &name("inside.txt"), Rights::READ) + .unwrap(); + assert_eq!( + read_all(&mut engine, b, inside.handle).unwrap(), + b"shared bytes" + ); + let deeper_view = engine + .open_at(b, view.handle, &name("deeper"), Rights::READ) + .unwrap(); + let deep = engine + .open_at(b, deeper_view.handle, &name("deep.txt"), Rights::READ) + .unwrap(); + assert_eq!( + read_all(&mut engine, b, deep.handle).unwrap(), + b"deep bytes" + ); + + // Nothing above it is: the sibling is not a child of the granted root… + assert_eq!( + engine + .open_at(b, view.handle, &name("above.txt"), Rights::READ) + .unwrap_err(), + FsError::NotFound + ); + // …and there is no `..` to climb with, by construction. + assert_eq!( + kore_svc_dagfs::Name::new("..").unwrap_err(), + FsError::InvalidName + ); + // B's own root is still empty — a grant is an attachment, not a copy. + assert!(list_names(&engine, b, clients[1].1).unwrap().is_empty()); +} + +#[test] +fn a_read_only_subtree_grant_cannot_restructure_it() { + // Plan §10.1 requires that a read-only grant refuse `unlink` as well as + // write, link, and re-share — the namespace verbs are gated at the + // directory, so a read-only view of one cannot rearrange it. + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + let shared = create_dir(&mut engine, a, root_a, "shared"); + create_file(&mut engine, a, shared, "f", b"A's bytes"); + let grant = engine + .grant(a, shared, b, Rights::READ, Some(name("ro"))) + .unwrap(); + let view = engine.open_grant(b, grant, Rights::READ).unwrap(); + + // B can look, and that is the whole of it. + assert_eq!(list_names(&engine, b, view.handle).unwrap(), vec!["f"]); + assert_eq!( + block_on(engine.unlink(b, view.handle, &name("f"))).unwrap_err(), + FsError::AccessDenied + ); + assert_eq!( + engine + .rename(b, view.handle, &name("f"), &name("g")) + .unwrap_err(), + FsError::AccessDenied + ); + assert_eq!( + block_on(engine.create(b, view.handle, &name("new"), ObjectKind::File)).unwrap_err(), + FsError::AccessDenied + ); + assert_eq!( + engine + .relocate(b, view.handle, &name("f"), view.handle, &name("g")) + .unwrap_err(), + FsError::AccessDenied + ); + + // A's subtree is exactly as it was. + assert_eq!(list_names(&engine, a, shared).unwrap(), vec!["f"]); +} + +#[test] +fn a_writable_subtree_grant_can_be_used_to_write_within_it() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + let shared = create_dir(&mut engine, a, root_a, "shared"); + let grant = engine + .grant( + a, + shared, + b, + Rights::READ + .union(Rights::WRITE) + .union(Rights::MUTATE_NAMESPACE), + None, + ) + .unwrap(); + let view = engine + .open_grant( + b, + grant, + Rights::READ + .union(Rights::WRITE) + .union(Rights::MUTATE_NAMESPACE), + ) + .unwrap(); + + // B creates inside the granted subtree, and A sees it. + let made = block_on(engine.create(b, view.handle, &name("b-made"), ObjectKind::File)).unwrap(); + block_on(engine.write_at(b, made.handle, 0, b"by B", ANY_REVISION)).unwrap(); + assert_eq!(list_names(&engine, a, shared).unwrap(), vec!["b-made"]); + let seen_by_a = engine + .open_at(a, shared, &name("b-made"), Rights::READ) + .unwrap(); + assert_eq!(read_all(&mut engine, a, seen_by_a.handle).unwrap(), b"by B"); + + // But B still cannot re-share: SHARE was not delegated. + assert_eq!( + engine + .grant(b, view.handle, a, Rights::READ, None) + .unwrap_err(), + FsError::AccessDenied + ); +} + +// ── no laundering authority across attachments ────────────────────────────── + +#[test] +fn a_granted_object_cannot_be_linked_into_the_recipients_own_root() { + // The escalation this refuses: a containment edge carries no rights, so if + // B could name A's read-only file inside B's own full-rights root, B would + // afterwards reach it with write access that also outlives A's revoke. + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, root_b) = clients[1]; + + let file = create_file(&mut engine, a, root_a, "shared", b"A's bytes"); + let grant = engine.grant(a, file, b, Rights::READ, None).unwrap(); + let view = engine.open_grant(b, grant, Rights::READ).unwrap(); + + assert_eq!( + engine + .link(b, root_b, &name("laundered"), view.handle) + .unwrap_err(), + FsError::AccessDenied + ); + // Even a fully-rights grant cannot be laundered this way — the point is + // the attachment boundary, not the rights on it. + let full = engine + .grant(a, file, b, Rights::ALL, Some(name("full"))) + .unwrap(); + let full_view = engine.open_grant(b, full, Rights::ALL).unwrap(); + assert_eq!( + engine + .link(b, root_b, &name("laundered"), full_view.handle) + .unwrap_err(), + FsError::AccessDenied + ); + assert!(list_names(&engine, b, root_b).unwrap().is_empty()); +} + +#[test] +fn an_object_cannot_be_moved_out_of_a_granted_subtree_into_the_recipients_root() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, root_b) = clients[1]; + + let shared = create_dir(&mut engine, a, root_a, "shared"); + create_file(&mut engine, a, shared, "f", b"A's bytes"); + let delegated = Rights::READ + .union(Rights::WRITE) + .union(Rights::MUTATE_NAMESPACE); + let grant = engine.grant(a, shared, b, delegated, None).unwrap(); + let view = engine.open_grant(b, grant, delegated).unwrap(); + + // B may restructure *within* the subtree it was given… + assert!( + engine + .relocate(b, view.handle, &name("f"), view.handle, &name("renamed")) + .is_ok() + ); + // …but not carry an object across the attachment boundary into its own + // root, which would relabel it with B's own root authority. + assert_eq!( + engine + .relocate(b, view.handle, &name("renamed"), root_b, &name("stolen")) + .unwrap_err(), + FsError::AccessDenied + ); + assert!(list_names(&engine, b, root_b).unwrap().is_empty()); + assert_eq!(list_names(&engine, a, shared).unwrap(), vec!["renamed"]); +} + +#[test] +fn linking_and_moving_within_one_authority_edge_still_works() { + // The rule above must not break the ordinary case: a session restructuring + // its own tree uses one authority edge throughout. + let (mut engine, session, root) = engine_with_one(); + let dir = create_dir(&mut engine, session, root, "dir"); + let file = create_file(&mut engine, session, root, "f", b"x"); + + // A second handle on the same private-root edge is still the same edge. + let root_again = engine.open_root(session, Rights::ALL).unwrap(); + assert!( + engine + .link(session, root_again.handle, &name("second-name"), file) + .is_ok() + ); + assert!( + engine + .relocate(session, root, &name("f"), dir, &name("f")) + .is_ok() + ); +} + +// ── immediate loss of access ──────────────────────────────────────────────── + +#[test] +fn retiring_a_grant_makes_the_next_operation_on_its_handle_fail() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + let file = create_file(&mut engine, a, root_a, "shared", b"payload"); + let grant = engine.grant(a, file, b, Rights::READ, None).unwrap(); + let view = engine.open_grant(b, grant, Rights::READ).unwrap(); + assert_eq!(read_all(&mut engine, b, view.handle).unwrap(), b"payload"); + + block_on(engine.revoke(a, grant)).unwrap(); + + // Immediately — B still holds the same numeric handle. + assert_eq!( + block_on(engine.read(b, view.handle, 0, &mut [0u8; 8])).unwrap_err(), + FsError::AccessLost + ); + assert_eq!( + engine.stat(b, view.handle).unwrap_err(), + FsError::AccessLost + ); + // And it cannot be opened afresh either. + assert_eq!( + engine.open_grant(b, grant, Rights::READ).unwrap_err(), + FsError::AccessLost + ); + // A is unaffected: it still has its own name for the object. + assert_eq!(read_all(&mut engine, a, file).unwrap(), b"payload"); +} + +#[test] +fn a_recipient_may_hand_back_a_grant_itself() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + let file = create_file(&mut engine, a, root_a, "shared", b"x"); + let grant = engine.grant(a, file, b, Rights::READ, None).unwrap(); + let view = engine.open_grant(b, grant, Rights::READ).unwrap(); + + block_on(engine.revoke(b, grant)).unwrap(); + assert_eq!( + engine.stat(b, view.handle).unwrap_err(), + FsError::AccessLost + ); +} + +#[test] +fn a_third_session_cannot_revoke_a_grant_it_is_not_party_to() { + let (mut engine, clients) = engine_with_sessions(3); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + let (c, _root_c) = clients[2]; + + let file = create_file(&mut engine, a, root_a, "shared", b"x"); + let grant = engine.grant(a, file, b, Rights::READ, None).unwrap(); + let view = engine.open_grant(b, grant, Rights::READ).unwrap(); + + assert_eq!( + block_on(engine.revoke(c, grant)).unwrap_err(), + FsError::AccessDenied + ); + // B's access survived the attempt. + assert!(engine.stat(b, view.handle).is_ok()); +} + +#[test] +fn removing_an_intermediate_edge_invalidates_handles_opened_through_it() { + let (mut engine, session, root) = engine_with_one(); + let mid = create_dir(&mut engine, session, root, "mid"); + let leaf = create_file(&mut engine, session, mid, "leaf", b"leaf bytes"); + + // The leaf handle proved through root -> mid -> leaf. + assert_eq!(read_all(&mut engine, session, leaf).unwrap(), b"leaf bytes"); + + // Dismantle the chain from the middle. + block_on(engine.unlink(session, mid, &name("leaf"))).unwrap(); + assert_eq!( + block_on(engine.read(session, leaf, 0, &mut [0u8; 4])).unwrap_err(), + FsError::AccessLost + ); + + // And removing `mid` itself invalidates a handle to `mid`. + let mid_again = engine + .open_at(session, root, &name("mid"), Rights::ALL) + .unwrap(); + block_on(engine.unlink(session, root, &name("mid"))).unwrap(); + assert_eq!( + engine.stat(session, mid_again.handle).unwrap_err(), + FsError::AccessLost + ); +} + +#[test] +fn losing_one_route_leaves_a_second_live_route_working() { + let (mut engine, session, root) = engine_with_one(); + let dir = create_dir(&mut engine, session, root, "dir"); + let file = create_file(&mut engine, session, dir, "f", b"contents"); + + // A second name for the same object, from a different parent. + engine.link(session, root, &name("also-f"), file).unwrap(); + let via_root = engine + .open_at(session, root, &name("also-f"), Rights::ALL) + .unwrap(); + + // Drop the route the first handle proved through. + block_on(engine.unlink(session, dir, &name("f"))).unwrap(); + + assert_eq!( + block_on(engine.read(session, file, 0, &mut [0u8; 8])).unwrap_err(), + FsError::AccessLost + ); + // The other route still reaches the very same object and bytes. + assert_eq!( + read_all(&mut engine, session, via_root.handle).unwrap(), + b"contents" + ); + // And the object can be re-opened through the surviving name. + let reopened = engine + .open_at(session, root, &name("also-f"), Rights::ALL) + .unwrap(); + assert_eq!( + read_all(&mut engine, session, reopened.handle).unwrap(), + b"contents" + ); +} + +#[test] +fn a_grant_keeps_an_object_alive_after_its_last_name_is_gone() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + let file = create_file(&mut engine, a, root_a, "shared", b"kept alive"); + let grant = engine.grant(a, file, b, Rights::READ, None).unwrap(); + let view = engine.open_grant(b, grant, Rights::READ).unwrap(); + + // A removes its only name for the object; B's grant is still a live route. + block_on(engine.unlink(a, root_a, &name("shared"))).unwrap(); + assert_eq!( + read_all(&mut engine, b, view.handle).unwrap(), + b"kept alive", + "the grant is a route, so the object survives" + ); + // A's own handle went with the edge it proved through. + assert_eq!(engine.stat(a, file).unwrap_err(), FsError::AccessLost); + + // When the grant goes too, nothing reaches the object and it is reclaimed. + block_on(engine.revoke(b, grant)).unwrap(); + assert_eq!(engine.provider().placements(), 0); + assert_eq!(engine.provider().chunks_in_use(), 0); +} + +#[test] +fn a_grant_pins_an_uncontained_directory_graph_until_revocation() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + let shared = create_dir(&mut engine, a, root_a, "shared"); + create_file(&mut engine, a, shared, "leaf", b"still reachable"); + let grant = engine + .grant(a, shared, b, Rights::READ, Some(name("shared"))) + .unwrap(); + let view = engine.open_grant(b, grant, Rights::READ).unwrap(); + + // Native detach may remove a populated directory edge. The authority + // edge still reaches the whole subgraph, so neither object is reclaimed. + block_on(engine.detach_edge(a, root_a, &name("shared"))).unwrap(); + let leaf = engine + .open_at(b, view.handle, &name("leaf"), Rights::READ) + .unwrap(); + assert_eq!( + read_all(&mut engine, b, leaf.handle).unwrap(), + b"still reachable" + ); + + // The final authority route retires the directory and its newly + // unreachable descendant exactly once. + block_on(engine.revoke(b, grant)).unwrap(); + assert_eq!( + engine.object_count(), + 2, + "only the two session roots remain" + ); + assert_eq!(engine.provider().placements(), 0); + assert_eq!(engine.provider().chunks_in_use(), 0); +} + +#[test] +fn moving_through_a_shared_area_invalidates_only_the_moved_route() { + let (mut engine, session, root) = engine_with_one(); + let src = create_dir(&mut engine, session, root, "src"); + let dst = create_dir(&mut engine, session, root, "dst"); + let file = create_file(&mut engine, session, src, "f", b"moving"); + engine.link(session, root, &name("pinned"), file).unwrap(); + let pinned = engine + .open_at(session, root, &name("pinned"), Rights::ALL) + .unwrap(); + + engine + .relocate(session, src, &name("f"), dst, &name("f")) + .unwrap(); + + // The handle that proved through the source edge is gone… + assert_eq!(engine.stat(session, file).unwrap_err(), FsError::AccessLost); + // …the parallel link is untouched… + assert_eq!( + read_all(&mut engine, session, pinned.handle).unwrap(), + b"moving" + ); + // …and the object is reachable at its new place. + let moved = engine + .open_at(session, dst, &name("f"), Rights::ALL) + .unwrap(); + assert_eq!( + engine.stat(session, moved.handle).unwrap().object, + engine.stat(session, pinned.handle).unwrap().object + ); +} + +// ── teardown ──────────────────────────────────────────────────────────────── + +#[test] +fn detaching_a_session_releases_its_tree_and_leaves_peers_alone() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, root_b) = clients[1]; + + let dir_a = create_dir(&mut engine, a, root_a, "d"); + create_file(&mut engine, a, dir_a, "f", b"A's bytes"); + let file_b = create_file(&mut engine, b, root_b, "keep", b"B's bytes"); + + let objects_before = engine.object_count(); + block_on(engine.detach_session(a)).unwrap(); + + // A's root, directory, and file are all gone. + assert_eq!(objects_before - engine.object_count(), 3); + assert_eq!(engine.stat(a, root_a).unwrap_err(), FsError::InvalidHandle); + // B is entirely unaffected. + assert_eq!(read_all(&mut engine, b, file_b).unwrap(), b"B's bytes"); + assert_eq!(list_names(&engine, b, root_b).unwrap(), vec!["keep"]); +} + +#[test] +fn detaching_a_recipient_does_not_disturb_the_issuer() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + let file = create_file(&mut engine, a, root_a, "shared", b"A's"); + let grant = engine.grant(a, file, b, Rights::READ, None).unwrap(); + engine.open_grant(b, grant, Rights::READ).unwrap(); + + let edges_before = engine.authority_edge_count(); + block_on(engine.detach_session(b)).unwrap(); + + // The grant went with its holder — B's private root and the grant it held + // are both retired, and the object no longer counts a grant against it. + assert_eq!(edges_before - engine.authority_edge_count(), 2); + assert_eq!(engine.stat(a, file).unwrap().grants, 0); + // A still has its file, bytes and all. + assert_eq!(read_all(&mut engine, a, file).unwrap(), b"A's"); + // And the retired grant cannot be reopened by anyone. + assert!(engine.open_grant(b, grant, Rights::READ).is_err()); +} + +#[test] +fn a_sessions_handle_ceiling_does_not_starve_its_peers() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, root_b) = clients[1]; + + create_file(&mut engine, a, root_a, "f", b"x"); + // A already holds two handles (its root and the created file); open the + // rest of its allowance against the same name. + while engine.live_handles(a) < MAX_HANDLES_PER_SESSION { + engine + .open_at(a, root_a, &name("f"), Rights::READ) + .expect("within A's allowance"); + } + assert_eq!( + engine + .open_at(a, root_a, &name("f"), Rights::READ) + .unwrap_err(), + FsError::OutOfHandles + ); + + // B is unaffected by A having spent its own allowance. + assert!(engine.open_root(b, Rights::ALL).is_ok()); + assert!(list_names(&engine, b, root_b).unwrap().is_empty()); +} diff --git a/native/services/dagfs/tests/common/mod.rs b/native/services/dagfs/tests/common/mod.rs new file mode 100644 index 0000000..395b6b6 --- /dev/null +++ b/native/services/dagfs/tests/common/mod.rs @@ -0,0 +1,129 @@ +// Each integration suite compiles its own copy of this module and uses the +// subset of helpers it needs, so anything unused *here* is used elsewhere. +#![allow(dead_code)] + +//! Shared harness for the filesystem host test suites. +//! +//! These suites drive the engine through its **public API only** — the same +//! surface the Phase 2 service dispatch and the Phase 4 shell built-ins will +//! call. Nothing here reaches into the engine's internals, so a suite that +//! passes is evidence about behaviour a real client can depend on rather than +//! about the current implementation's shape. + +use kore_svc_dagfs::{ + ANY_REVISION, FsResult, Handle, Name, ObjectKind, RamEngine, Rights, SessionId, block_on, +}; + +/// A validated name, for tests where the name is obviously fine. +pub fn name(text: &str) -> Name { + Name::new(text).expect("test names are valid") +} + +/// An engine with `count` attached sessions, each with an open handle to its +/// own private root at full rights. +/// +/// Returns the engine and one `(session, root handle)` pair per session, in +/// order. Sessions get consecutive ids starting at 0, which is what the +/// founder will do when it provisions pairs. +/// +/// The engine is boxed: it is tens of KiB of arenas, and an unoptimized test +/// build does not elide the moves that returning it by value implies. Deref +/// coercion means callers still write `&mut engine`. +pub fn engine_with_sessions(count: u16) -> (Box, Vec<(SessionId, Handle)>) { + let mut engine = Box::new(RamEngine::default()); + let mut clients = Vec::new(); + for index in 0..count { + let session = SessionId::new(index); + block_on(engine.attach_session(session)).expect("attach"); + let root = engine + .open_root(session, Rights::ALL) + .expect("open private root"); + clients.push((session, root.handle)); + } + (engine, clients) +} + +/// One attached session with its root handle — the common single-client case. +pub fn engine_with_one() -> (Box, SessionId, Handle) { + let (engine, clients) = engine_with_sessions(1); + let (session, root) = clients[0]; + (engine, session, root) +} + +/// Create a file beneath `parent` and write `bytes` into it, returning the +/// creator's handle to it. +pub fn create_file( + engine: &mut RamEngine, + session: SessionId, + parent: Handle, + file_name: &str, + bytes: &[u8], +) -> Handle { + let made = block_on(engine.create(session, parent, &name(file_name), ObjectKind::File)) + .expect("create file"); + if !bytes.is_empty() { + block_on(engine.write_at(session, made.handle, 0, bytes, ANY_REVISION)) + .expect("write file"); + } + made.handle +} + +/// Create a file and immediately close the creator's handle. +/// +/// For fixture-building loops: `create` hands back a live handle, and a +/// session's handle ceiling is deliberately lower than a directory's entry +/// ceiling, so a loop that keeps every handle it is given runs out of handles +/// long before it runs out of names. A real client closes what it is done +/// with, and so does this. +pub fn create_file_closed( + engine: &mut RamEngine, + session: SessionId, + parent: Handle, + file_name: &str, + bytes: &[u8], +) { + let handle = create_file(engine, session, parent, file_name, bytes); + engine.close(session, handle).expect("close"); +} + +/// Create a directory beneath `parent`, returning the creator's handle. +pub fn create_dir( + engine: &mut RamEngine, + session: SessionId, + parent: Handle, + dir_name: &str, +) -> Handle { + block_on(engine.create(session, parent, &name(dir_name), ObjectKind::Directory)) + .expect("create directory") + .handle +} + +/// Read a whole file through `handle` into a `Vec`. +pub fn read_all(engine: &mut RamEngine, session: SessionId, handle: Handle) -> FsResult> { + let len = engine.stat(session, handle)?.len; + let mut buf = vec![0u8; len as usize]; + let read = block_on(engine.read(session, handle, 0, &mut buf))?; + buf.truncate(read); + Ok(buf) +} + +/// The names in a directory, across every page of the listing. +/// +/// Paging is followed to completion, so a test asserting on directory +/// contents cannot accidentally pass by looking at only the first page. +pub fn list_names(engine: &RamEngine, session: SessionId, dir: Handle) -> FsResult> { + let mut names = Vec::new(); + let mut cursor = 0u32; + loop { + let page = engine.list(session, dir, cursor)?; + for entry in page.entries.iter() { + names.push(entry.name.as_str().to_string()); + } + match page.next { + Some(next) => cursor = next, + None => break, + } + } + names.sort(); + Ok(names) +} diff --git a/native/services/dagfs/tests/concurrency.rs b/native/services/dagfs/tests/concurrency.rs new file mode 100644 index 0000000..0c7d5a8 --- /dev/null +++ b/native/services/dagfs/tests/concurrency.rs @@ -0,0 +1,395 @@ +//! Revisions, conflict detection, overlapping writes, atomic append, and what +//! a reader can observe. +//! +//! Covers plan §2.5 and the concurrency rows of §10.1. The engine is driven +//! by one owner at a time — that exclusive borrow *is* the per-file mutation +//! gate in Phase 1 — so what these tests pin down is the **request-level** +//! contract every caller depends on: one request either commits completely or +//! has no visible effect, revisions are monotonic, and a stale expectation is +//! refused rather than silently applied. + +mod common; + +use common::{create_file, engine_with_one, engine_with_sessions, name, read_all}; +use kore_svc_dagfs::{ANY_REVISION, FsError, ProviderError, Rights, block_on, provider::Fault}; + +// ── revisions ─────────────────────────────────────────────────────────────── + +#[test] +fn revisions_advance_monotonically_and_are_never_reused() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b""); + + let mut seen = vec![engine.stat(session, file).unwrap().revision]; + for _ in 0..5 { + let revision = block_on(engine.write_at(session, file, 0, b"x", ANY_REVISION)).unwrap(); + assert!( + revision > *seen.last().unwrap(), + "each committed write must advance the revision" + ); + seen.push(revision); + } + // Shrinking is a mutation too, so it advances rather than rewinds. + let after_resize = block_on(engine.resize(session, file, 0, ANY_REVISION)).unwrap(); + assert!(after_resize > *seen.last().unwrap()); + + seen.push(after_resize); + let mut unique = seen.clone(); + unique.sort(); + unique.dedup(); + assert_eq!(unique.len(), seen.len(), "no revision is reused"); +} + +#[test] +fn an_exact_revision_write_commits_when_current() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b""); + + let current = engine.stat(session, file).unwrap().revision; + let next = block_on(engine.write_at(session, file, 0, b"abcd", current)).unwrap(); + assert!(next > current); + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"abcd"); +} + +#[test] +fn a_stale_exact_revision_write_is_refused_and_changes_nothing() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b"first"); + let stale = engine.stat(session, file).unwrap().revision; + + // Someone else moves the file on. + let current = block_on(engine.write_at(session, file, 0, b"secon", ANY_REVISION)).unwrap(); + assert_ne!(stale, current); + + // The stale expectation is refused… + assert_eq!( + block_on(engine.write_at(session, file, 0, b"third", stale)).unwrap_err(), + FsError::Conflict + ); + // …and left no trace at all: bytes, length, and revision are untouched. + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"secon"); + let info = engine.stat(session, file).unwrap(); + assert_eq!(info.revision, current); + assert_eq!(info.len, 5); + + // Retrying with the current revision succeeds. + assert!(block_on(engine.write_at(session, file, 0, b"third", current)).is_ok()); + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"third"); +} + +#[test] +fn a_revision_from_the_future_is_also_refused() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b"x"); + let current = engine.stat(session, file).unwrap().revision; + assert_eq!( + block_on(engine.write_at(session, file, 0, b"y", current + 1)).unwrap_err(), + FsError::Conflict + ); +} + +#[test] +fn any_revision_writes_are_unconditional() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b"one"); + + // Several unconditional writes in a row, with no revision bookkeeping by + // the caller at all. + for payload in [b"two", b"six", b"ten"] { + assert!(block_on(engine.write_at(session, file, 0, payload, ANY_REVISION)).is_ok()); + } + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"ten"); +} + +#[test] +fn conflict_detection_is_per_file() { + let (mut engine, session, root) = engine_with_one(); + let a = create_file(&mut engine, session, root, "a", b"a"); + let b = create_file(&mut engine, session, root, "b", b"b"); + + let b_revision = engine.stat(session, b).unwrap().revision; + // Moving `a` on must not invalidate a caller's expectation about `b`. + block_on(engine.write_at(session, a, 0, b"A", ANY_REVISION)).unwrap(); + assert!(block_on(engine.write_at(session, b, 0, b"B", b_revision)).is_ok()); + assert_eq!(read_all(&mut engine, session, b).unwrap(), b"B"); +} + +// ── overlapping and disjoint writes ───────────────────────────────────────── + +#[test] +fn non_overlapping_unconditional_writes_both_remain_present() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b""); + + block_on(engine.write_at(session, file, 0, b"head", ANY_REVISION)).unwrap(); + block_on(engine.write_at(session, file, 8, b"tail", ANY_REVISION)).unwrap(); + + let bytes = read_all(&mut engine, session, file).unwrap(); + assert_eq!(bytes.len(), 12); + assert_eq!(&bytes[..4], b"head"); + assert_eq!(&bytes[4..8], &[0u8; 4], "the gap between them is zeros"); + assert_eq!(&bytes[8..], b"tail"); +} + +#[test] +fn overlapping_unconditional_writes_serialize_and_the_later_one_wins() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b""); + + block_on(engine.write_at(session, file, 0, b"aaaaaaaa", ANY_REVISION)).unwrap(); + // Overlaps bytes 4..8 of the first write and extends past it. + block_on(engine.write_at(session, file, 4, b"bbbbbbbb", ANY_REVISION)).unwrap(); + + let bytes = read_all(&mut engine, session, file).unwrap(); + assert_eq!(bytes.len(), 12); + assert_eq!(&bytes[..4], b"aaaa", "the non-overlapping prefix survives"); + assert_eq!( + &bytes[4..], + b"bbbbbbbb", + "the later committed request determines the overlapping bytes" + ); +} + +#[test] +fn a_shorter_write_does_not_truncate_the_file() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b"abcdefgh"); + // Writing in the middle changes only those bytes; length is a maximum, + // not a restatement. + block_on(engine.write_at(session, file, 2, b"XY", ANY_REVISION)).unwrap(); + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"abXYefgh"); + assert_eq!(engine.stat(session, file).unwrap().len, 8); +} + +// ── append ────────────────────────────────────────────────────────────────── + +#[test] +fn append_chooses_the_end_and_writes_as_one_step() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "log", b""); + + let (first_offset, first_revision) = + block_on(engine.append(session, file, b"one", ANY_REVISION)).unwrap(); + let (second_offset, second_revision) = + block_on(engine.append(session, file, b"two", ANY_REVISION)).unwrap(); + + assert_eq!(first_offset, 0); + assert_eq!( + second_offset, 3, + "the second append saw the first one's end" + ); + assert!(second_revision > first_revision); + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"onetwo"); +} + +#[test] +fn appends_never_land_on_the_same_offset() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "log", b""); + + let mut offsets = Vec::new(); + for _ in 0..8 { + let (offset, _) = block_on(engine.append(session, file, b"abc", ANY_REVISION)).unwrap(); + offsets.push(offset); + } + let mut unique = offsets.clone(); + unique.sort(); + unique.dedup(); + assert_eq!( + unique.len(), + offsets.len(), + "every append got its own space" + ); + assert_eq!(offsets, vec![0, 3, 6, 9, 12, 15, 18, 21]); + // No append overwrote another: the file is exactly the concatenation. + assert_eq!( + read_all(&mut engine, session, file).unwrap(), + b"abcabcabcabcabcabcabcabc" + ); +} + +#[test] +fn append_onto_a_non_empty_file_starts_at_its_end() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b"prefix"); + let (offset, _) = block_on(engine.append(session, file, b"-suffix", ANY_REVISION)).unwrap(); + assert_eq!(offset, 6); + assert_eq!( + read_all(&mut engine, session, file).unwrap(), + b"prefix-suffix" + ); +} + +#[test] +fn a_conflicted_append_consumes_no_position() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "log", b""); + let (_, revision) = block_on(engine.append(session, file, b"one", ANY_REVISION)).unwrap(); + + // A stale expectation must be refused *before* an offset is claimed… + assert_eq!( + block_on(engine.append(session, file, b"bad", revision - 1)).unwrap_err(), + FsError::Conflict + ); + // …so the next legitimate append still lands immediately after "one". + let (offset, _) = block_on(engine.append(session, file, b"two", revision)).unwrap(); + assert_eq!(offset, 3); + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"onetwo"); + assert_eq!(engine.stat(session, file).unwrap().len, 6); +} + +#[test] +fn append_needs_write_rights() { + let (mut engine, session, root) = engine_with_one(); + create_file(&mut engine, session, root, "f", b"x"); + let reader = engine + .open_at(session, root, &name("f"), Rights::READ) + .unwrap(); + assert_eq!( + block_on(engine.append(session, reader.handle, b"y", ANY_REVISION)).unwrap_err(), + FsError::AccessDenied + ); +} + +// ── what a reader can observe ─────────────────────────────────────────────── + +#[test] +fn a_reader_sees_the_previous_complete_revision_when_a_write_fails() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b"committed"); + let committed = engine.stat(session, file).unwrap().revision; + + // A write that dies inside the provider, after capacity was reserved. + engine.provider().inject(Fault::Write(ProviderError::Media)); + assert_eq!( + block_on(engine.write_at(session, file, 0, b"DESTROYED", ANY_REVISION)).unwrap_err(), + FsError::ProviderFailure(ProviderError::Media) + ); + + // The reader observes the whole previous revision — not a mixture, and not + // a longer file with a partly-written tail. + let info = engine.stat(session, file).unwrap(); + assert_eq!(info.revision, committed); + assert_eq!(info.len, 9); + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"committed"); +} + +#[test] +fn a_failed_extending_write_does_not_expose_a_longer_file() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b"short"); + + engine.provider().inject(Fault::Write(ProviderError::Media)); + assert!(block_on(engine.write_at(session, file, 100, b"far away", ANY_REVISION)).is_err()); + + // Capacity may have been reserved, but length is what a reader sees, and + // it never moved. + assert_eq!(engine.stat(session, file).unwrap().len, 5); + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"short"); +} + +#[test] +fn a_reader_sees_the_whole_of_a_committed_write() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b""); + let payload: Vec = (0..600u32).map(|i| (i % 251) as u8).collect(); + + let revision = block_on(engine.write_at(session, file, 0, &payload, ANY_REVISION)).unwrap(); + + // A write spanning several provider chunks is visible in full, or not at + // all — never chunk-by-chunk. + let info = engine.stat(session, file).unwrap(); + assert_eq!(info.revision, revision); + assert_eq!(info.len, payload.len() as u64); + assert_eq!(read_all(&mut engine, session, file).unwrap(), payload); +} + +// ── two clients on one shared file ────────────────────────────────────────── + +#[test] +fn two_clients_interleaving_writes_on_a_shared_file_stay_consistent() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + let file = create_file(&mut engine, a, root_a, "shared", b""); + let writable = Rights::READ.union(Rights::WRITE); + let grant = engine.grant(a, file, b, writable, None).unwrap(); + let b_view = engine.open_grant(b, grant, writable).unwrap(); + + // Alternate appends from the two sessions. + let mut revisions = Vec::new(); + for round in 0..4 { + let (_, r1) = block_on(engine.append(a, file, b"A", ANY_REVISION)).unwrap(); + let (_, r2) = block_on(engine.append(b, b_view.handle, b"B", ANY_REVISION)).unwrap(); + revisions.push(r1); + revisions.push(r2); + assert_eq!( + engine.stat(a, file).unwrap().len, + (round + 1) * 2, + "each append added exactly one byte" + ); + } + + // One shared revision line, strictly increasing across both writers. + assert!(revisions.windows(2).all(|pair| pair[0] < pair[1])); + // Nothing was lost or overwritten. + assert_eq!(read_all(&mut engine, a, file).unwrap(), b"ABABABAB"); + // And both see the identical object. + assert_eq!( + engine.stat(a, file).unwrap().object, + engine.stat(b, b_view.handle).unwrap().object + ); +} + +#[test] +fn one_clients_conditional_write_detects_the_others_change() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + let file = create_file(&mut engine, a, root_a, "shared", b"start"); + let writable = Rights::READ.union(Rights::WRITE); + let grant = engine.grant(a, file, b, writable, None).unwrap(); + let b_view = engine.open_grant(b, grant, writable).unwrap(); + + // Both read the same revision, then B commits first. + let observed = engine.stat(a, file).unwrap().revision; + assert_eq!(engine.stat(b, b_view.handle).unwrap().revision, observed); + block_on(engine.write_at(b, b_view.handle, 0, b"by-B!", observed)).unwrap(); + + // A's conditional write on the now-stale revision is refused, which is the + // whole point of carrying an expectation. + assert_eq!( + block_on(engine.write_at(a, file, 0, b"by-A!", observed)).unwrap_err(), + FsError::Conflict + ); + assert_eq!(read_all(&mut engine, a, file).unwrap(), b"by-B!"); + + // A re-reads and retries, and now succeeds. + let fresh = engine.stat(a, file).unwrap().revision; + assert!(block_on(engine.write_at(a, file, 0, b"by-A!", fresh)).is_ok()); + assert_eq!(read_all(&mut engine, b, b_view.handle).unwrap(), b"by-A!"); +} + +#[test] +fn a_revoked_writer_stops_being_able_to_write_immediately() { + let (mut engine, clients) = engine_with_sessions(2); + let (a, root_a) = clients[0]; + let (b, _root_b) = clients[1]; + + let file = create_file(&mut engine, a, root_a, "shared", b""); + let writable = Rights::READ.union(Rights::WRITE); + let grant = engine.grant(a, file, b, writable, None).unwrap(); + let b_view = engine.open_grant(b, grant, writable).unwrap(); + block_on(engine.append(b, b_view.handle, b"before", ANY_REVISION)).unwrap(); + + block_on(engine.revoke(a, grant)).unwrap(); + + assert_eq!( + block_on(engine.append(b, b_view.handle, b"after", ANY_REVISION)).unwrap_err(), + FsError::AccessLost + ); + // Everything B wrote while authorized is still there. + assert_eq!(read_all(&mut engine, a, file).unwrap(), b"before"); +} diff --git a/native/services/dagfs/tests/fs_search_witness.rs b/native/services/dagfs/tests/fs_search_witness.rs new file mode 100644 index 0000000..1efb88b --- /dev/null +++ b/native/services/dagfs/tests/fs_search_witness.rs @@ -0,0 +1,443 @@ +//! Init's `fs-search` boot witness on a volume someone has lived in. +//! +//! On metal the witness shares a persistent store with whatever its owner +//! has saved, while every CI volume is born empty — so a defect that only +//! ordinary content can reach surfaces first as a machine that will not +//! found its shell. These scenarios run the witness's exact sequence, and +//! `find reindex`'s walk and per-file step, through the service's real +//! dispatch over a founded process's data window, committing the producer's +//! own batch ([`encode_batch`]) — the bytes a `find`, `write` or editor save +//! hands this store. + +mod common; + +use common::name; +use kore_dagfs::{Refusal, index::encode_batch}; +use kore_svc_dagfs::{ + ANY_REVISION, DataWindow, FreshnessPolicy, FsError, FsRequest, FsResponse, Handle, ObjectKind, + ObjectSubtype, RamService, Rights, SearchContent, SearchPlane, SearchQuery, SearchTerm, + SessionId, block_on, bounds::MAX_TRANSFER, read_dir_entry, read_search_hit, + read_version_identity, +}; +use kore_svc_extract::{ + Confidence, Fact, FactKind, FactValue, WORKER_REPORT_BYTES, encode_result, registry::PLAIN_TEXT, +}; +use kore_svc_extract_worker_rust::parse_plain_text; + +/// exec's `FS_DATA_BYTES`: the data window every founded process face gets. +const WINDOW_BYTES: usize = 0x1_0000; + +/// The witness's fixture directory and the one file in it +/// (`user/services/init`). +const WITNESS: &str = "kore-fs-reindex-witness"; +const PROBE: &str = "probe"; + +/// Half a page of meeting notes: twenty short lines, more distinct words +/// than the producer keeps as tokens. Nothing about it is unusual — which +/// is the point. +const NOTES: &str = "\ +Weekly sync notes for the storage and search working group. +Attendees reviewed durable batches, cold restarts and recovery witnesses. +Open question: does reindex finish before the first shell can answer? +Action items include profiling completion latency under sustained load. +Someone proposed caching directory listings between consecutive pages. +We agreed the index stays auxiliary; authoritative writes always win. +Measurements suggest boot spends most of its time extracting images. +Next steps: tighten budgets, document refusals, rehearse failure drills. +Reminder: archive older transcripts monthly and keep summaries handy. +Questions about quotas, grants, leases and revocation went unresolved. +The editor saves documents atomically through one held transaction. +Compositor work continues separately; display timing is unaffected. +Celebrate small wins: fewer flakes, clearer logs, calmer reviews. +Volunteers will pair on tracing the scheduler's placement decisions. +Budget owners should confirm hardware orders before the quarter closes. +Lab machines need firmware updates; schedule maintenance windows early. +Keyboard layouts, pointer acceleration and fonts deserve a usability pass. +Translation strings remain untouched until terminology settles further. +Security review covers capability attenuation, sealing and audit trails. +Thanks everyone; minutes circulate tomorrow morning for corrections. +"; + +/// One founded process's filesystem face, host-side: a session on the real +/// service and the data window exec would have delivered it. +struct Face { + service: RamService, + session: SessionId, + window: Vec, +} + +impl Face { + fn new() -> Self { + let mut face = Self { + service: RamService::default(), + session: SessionId::new(0), + window: vec![0; WINDOW_BYTES], + }; + block_on(face.service.engine().attach_session(face.session)).expect("attach session"); + let attached = face.call(FsRequest::AttachWindow { + slot: 1, + bytes: WINDOW_BYTES as u32, + }); + assert!( + matches!(attached, FsResponse::WindowAttached { .. }), + "{attached:?}" + ); + face + } + + fn call(&mut self, request: FsRequest) -> FsResponse { + let mut window = DataWindow::new(&mut self.window); + self.service + .dispatch(self.session, &request, Some(&mut window)) + } + + fn handle(response: FsResponse) -> Handle { + match response { + FsResponse::Opened { handle, .. } => handle, + other => panic!("expected a view, got {other:?}"), + } + } + + fn root(&mut self) -> Handle { + Self::handle(self.call(FsRequest::OpenRoot { + rights: Rights::ALL, + })) + } + + fn create(&mut self, parent: Handle, leaf: &str, kind: ObjectKind) -> Handle { + Self::handle(self.call(FsRequest::Create { + parent, + name: name(leaf), + kind, + subtype: ObjectSubtype::PLAIN, + })) + } + + fn close(&mut self, handle: Handle) { + assert_eq!(self.call(FsRequest::Close { handle }), FsResponse::Done); + } + + fn write(&mut self, file: Handle, bytes: &[u8]) { + self.window[..bytes.len()].copy_from_slice(bytes); + let written = self.call(FsRequest::WriteAt { + handle: file, + offset: 0, + window_offset: 0, + length: bytes.len() as u32, + expected: ANY_REVISION, + }); + assert!( + matches!(written, FsResponse::Committed { .. }), + "{written:?}" + ); + } + + /// `kore_dagfs::index::publish`: lend the version visible now, take the + /// bytes just written or read it back when it fits one transfer, commit + /// the producer's batch, and release the lease on every path. + fn publish(&mut self, file: Handle, supplied: Option<&[u8]>) -> Result<(), FsError> { + let lease = match self.call(FsRequest::LendVersion { + handle: file, + window_offset: 0, + }) { + FsResponse::VersionLent { lease } => lease, + FsResponse::Refused(error) => return Err(error), + other => panic!("lend answered {other:?}"), + }; + let identity = read_version_identity(&self.window, 0).expect("identity"); + let mut source = [0u8; MAX_TRANSFER]; + let staged = match supplied { + Some(bytes) => Ok(Some(bytes)), + None if identity.len <= MAX_TRANSFER as u64 => { + let len = identity.len as usize; + match self.call(FsRequest::ReadVersion { + lease, + offset: 0, + window_offset: 0, + length: len as u32, + }) { + FsResponse::Transferred { count } if count as usize == len => { + source[..len].copy_from_slice(&self.window[..len]); + Ok(Some(&source[..len])) + } + FsResponse::Refused(error) => Err(error), + other => panic!("read answered {other:?}"), + } + } + None => Ok(None), + }; + let committed = staged + .and_then(|bytes| encode_batch(identity, bytes, &mut self.window)) + .and_then(|length| { + match self.call(FsRequest::CommitDerived { + lease, + window_offset: 0, + length: length as u32, + output_window_bytes: 0, + }) { + FsResponse::DerivedCommitted { .. } => Ok(()), + FsResponse::Refused(error) => Err(error), + other => panic!("commit answered {other:?}"), + } + }); + assert_eq!( + self.call(FsRequest::ReleaseVersion { lease }), + FsResponse::Done + ); + committed + } + + /// `find reindex [in

]`: walk from `from` — the view root, or the + /// directory `in` names — and publish every file without a current + /// plain-text batch. Unsupported content stays unindexed, a full index + /// ends the walk as an incomplete one, and any other refusal is the + /// command's — exactly what `find` prints and exits with. `from` stays + /// the caller's; the walk closes what it opens. + fn reindex(&mut self, from: Handle) -> Result { + let mut pending = vec![(from, false)]; + let mut indexed = 0; + while let Some((directory, opened)) = pending.pop() { + let mut cursor = 0; + loop { + let FsResponse::Listed { entries, next } = self.call(FsRequest::List { + handle: directory, + cursor, + window_offset: 0, + }) else { + panic!("the directory did not list"); + }; + let page: Vec<_> = (0..entries as usize) + .map(|index| read_dir_entry(&self.window, 0, index).expect("entry")) + .collect(); + for entry in page { + let child = Self::handle(self.call(FsRequest::OpenAt { + parent: directory, + name: entry.name, + rights: Rights::READ, + })); + if entry.kind == ObjectKind::Directory { + pending.push((child, true)); + continue; + } + let current = self.call(FsRequest::DerivedInfo { + handle: child, + parser: PLAIN_TEXT, + }); + let published = if current == FsResponse::Refused(FsError::NotFound) { + self.publish(child, None) + } else { + Err(FsError::AlreadyExists) + }; + self.close(child); + match published { + Ok(()) => indexed += 1, + Err(FsError::AlreadyExists | FsError::Unsupported) => {} + Err(error) if Refusal::Fs(error).is_capacity_exhausted() => { + return Err(FsError::IndexIncomplete); + } + Err(error) => return Err(error), + } + } + match next { + Some(next) => cursor = next, + None => break, + } + } + if opened { + self.close(directory); + } + } + Ok(indexed) + } + + /// One `CurrentOnly` full-text page beneath `anchor`, as the paths `find` + /// prints, or the refusal it prints instead. + fn find( + &mut self, + anchor: Handle, + named: Option<&str>, + term: &str, + ) -> Result, FsError> { + let query = SearchQuery { + name_contains: named.map(name), + content: Some(SearchContent { + plane: SearchPlane::FullText, + term: SearchTerm::new(term).unwrap(), + }), + freshness: FreshnessPolicy::CurrentOnly, + ..SearchQuery::ALL + }; + match self.call(FsRequest::Search { + handle: anchor, + query, + cursor: None, + window_offset: 0, + }) { + FsResponse::Searched { + hits, next: None, .. + } => Ok((0..hits as usize) + .map(|index| { + let hit = read_search_hit(&self.window, 0, index).expect("hit"); + hit.path + .iter() + .map(|component| component.as_str()) + .collect::>() + .join("/") + }) + .collect()), + FsResponse::Refused(error) => Err(error), + other => panic!("search answered {other:?}"), + } + } + + /// Init's witness, after its first three launches: `make-directory` + /// makes the fixture, `create` publishes the empty probe's batch, and + /// `write` lands one unsupported byte whose batch is refused as + /// unsupported (`fs: wrote 1 bytes (index incomplete)`). Yields the + /// fixture: the scope of the witness's `find reindex in`. + fn witness_fixture(&mut self, root: Handle) -> Handle { + let fixture = self.create(root, WITNESS, ObjectKind::Directory); + let probe = self.create(fixture, PROBE, ObjectKind::File); + assert_eq!(self.publish(probe, Some(&[])), Ok(())); + self.write(probe, b"\xff"); + assert_eq!( + self.publish(probe, Some(b"\xff")), + Err(FsError::Unsupported) + ); + self.close(probe); + fixture + } +} + +/// The mechanism behind Biggertron's first-shell failure (rung 590191663: +/// `find: out of range` where the witness expects `find: search index +/// incomplete`). Any text file without a plain-text batch — half a page of +/// notes suffices — was published by the witness's `find reindex` like every +/// other. The producer encoded that batch into the whole data window — past +/// the one 4 KiB #476 batch the store keeps per version — and `CommitDerived` +/// refused it `Range`, a refusal the reindex walk rightly does not swallow. +/// The page must instead index within the store's bound, and stay findable. +#[test] +fn a_page_of_notes_on_the_volume_indexes_and_stays_findable() { + let mut face = Face::new(); + let root = face.root(); + let documents = face.create(root, "Documents", ObjectKind::Directory); + let notes = face.create(documents, "notes", ObjectKind::File); + face.write(notes, NOTES.as_bytes()); + face.close(notes); + + assert_eq!( + face.reindex(root), + Ok(1), + "the page is indexed, never `Range`" + ); + assert_eq!( + face.find(documents, None, "answer"), + Ok(vec!["notes".to_string()]), + "full-text search reads the page's whole text" + ); + assert_eq!( + face.find(documents, None, "corrections"), + Ok(vec!["notes".to_string()]), + "the last line's words are as findable as the first's" + ); +} + +/// The witness's verdict is a fact about its own fixture. `find reindex in` +/// walks the fixture directory alone — it lends and reads the probe and +/// nothing else the volume holds — and `CurrentOnly` judges only the probe. +/// The witness it replaced reindexed the whole view at boot, so any one file +/// there the store could not lend, read or commit stopped the boot. +#[test] +fn the_witness_reindexes_and_judges_only_its_fixture() { + let mut face = Face::new(); + let root = face.root(); + let documents = face.create(root, "Documents", ObjectKind::Directory); + let page = face.create(documents, "page", ObjectKind::File); + face.write(page, b"the answer is forty two"); + assert_eq!(face.publish(page, Some(b"the answer is forty two")), Ok(())); + face.close(page); + // Written before this volume had an index, so never published. + let legacy = face.create(documents, "legacy", ObjectKind::File); + face.write(legacy, NOTES.as_bytes()); + + let fixture = face.witness_fixture(root); + assert_eq!(face.reindex(fixture), Ok(0), "the probe stays unsupported"); + assert_eq!( + face.find(fixture, None, "answer"), + Err(FsError::IndexIncomplete) + ); + assert_eq!( + face.call(FsRequest::DerivedInfo { + handle: legacy, + parser: PLAIN_TEXT, + }), + FsResponse::Refused(FsError::NotFound), + "the walk never left the fixture" + ); + // What the witness used to do instead: the whole view, `legacy` with it. + assert_eq!(face.reindex(root), Ok(1)); +} + +/// The store's side of the same bound. A batch past the one the store keeps +/// per version — what any producer that encodes into its whole window sends +/// — is answered as a version this index cannot hold (`Unsupported`), which +/// every producer reports as `(index incomplete)` and `find reindex` skips. +/// It was `Range`, which failed the write the batch describes after that +/// write had committed. +#[test] +fn an_oversized_batch_is_answered_as_an_unindexable_version() { + let mut face = Face::new(); + let root = face.root(); + let notes = face.create(root, "notes", ObjectKind::File); + face.write(notes, NOTES.as_bytes()); + let FsResponse::VersionLent { lease } = face.call(FsRequest::LendVersion { + handle: notes, + window_offset: 0, + }) else { + panic!("the version was not lent"); + }; + let identity = read_version_identity(&face.window, 0).expect("identity"); + // The parser's spans plus a token fact per distinct word: the batch the + // plain-text producer published before it bounded it. + let mut result = parse_plain_text(identity, NOTES.as_bytes()).unwrap(); + for word in NOTES.split(|byte: char| !byte.is_ascii_alphanumeric()) { + if word.is_empty() || result.facts.iter().any(|fact| fact.key.as_str() == word) { + continue; + } + let fact = Fact { + kind: FactKind::Custom, + key: word.try_into().unwrap(), + value: FactValue::Utf8(word.try_into().unwrap()), + confidence: Confidence::CERTAIN, + source: None, + }; + if result.facts.push(fact).is_err() { + break; + } + } + let length = encode_result(&result, &mut face.window).unwrap(); + assert!(length > WORKER_REPORT_BYTES); + + assert_eq!( + face.call(FsRequest::CommitDerived { + lease, + window_offset: 0, + length: length as u32, + output_window_bytes: 0, + }), + FsResponse::Refused(FsError::Unsupported) + ); + assert_eq!( + face.call(FsRequest::ReleaseVersion { lease }), + FsResponse::Done + ); + assert_eq!( + face.call(FsRequest::DerivedInfo { + handle: notes, + parser: PLAIN_TEXT, + }), + FsResponse::Refused(FsError::NotFound), + "the version stays unindexed" + ); +} diff --git a/native/services/dagfs/tests/loopback.rs b/native/services/dagfs/tests/loopback.rs new file mode 100644 index 0000000..46c9ab8 --- /dev/null +++ b/native/services/dagfs/tests/loopback.rs @@ -0,0 +1,907 @@ +//! End-to-end over the real wire: client → encoded record → codec → dispatch → +//! encoded record → client. +//! +//! Everything here crosses the frozen `FsReqWire`/`FsRepWire` forms as bytes. +//! Nothing is mocked at the enum level, so a codec bug, a per-verb field +//! collision, or a dispatch that mis-reads the data window fails *here* rather +//! than surviving until a founded client exists. +//! +//! This is what Phase 2 can honestly prove on a host. The one thing it cannot +//! is that a real founded pair delivers the bytes — that is Phase 2 step 5, and +//! it needs the QEMU boot gate. + +mod common; + +use std::{cell::RefCell, rc::Rc}; + +use common::name; +use kore_abi::svc::{FsDirEntryWire, FsRepWire, FsReqWire}; +use kore_svc_dagfs::{ + ANY_REVISION, DataWindow, FsClient, FsError, FsResult, ObjectKind, ProviderError, RamService, + Rights, SearchQuery, SessionId, Transport, read_parent, read_search_hit, + wire::{decode_req, encode_rep}, +}; + +/// Bytes in each client's data window. +const WINDOW_BYTES: usize = 4096; + +/// One service, shared by every session in a scenario — the shape a real +/// multi-client host has, without needing one yet. +type SharedService = Rc>; + +/// A client's end of the pair: the shared service, this client's session, and +/// the data window it delivered. +/// +/// The window lives here because on metal it is the *client's* memory that the +/// service was given a view of. Holding it on the client side keeps that +/// ownership honest — the service reaches it only through the view handed to +/// `dispatch`. +struct Loopback { + service: SharedService, + session: SessionId, + window: Vec, +} + +impl Transport for Loopback { + fn exchange(&mut self, request: &FsReqWire) -> FsResult { + // Decode exactly as the serving loop would: a record it cannot read is + // refused, never guessed at. + let Some(decoded) = decode_req(request) else { + return Ok(encode_rep( + &kore_svc_dagfs::FsResponse::Refused(FsError::Unsupported), + 0, + )); + }; + let mut window = DataWindow::new(&mut self.window); + let response = + self.service + .borrow_mut() + .dispatch(self.session, &decoded, Some(&mut window)); + Ok(encode_rep(&response, 0)) + } +} + +/// Build a world with `sessions` attached clients, each with an open root and +/// an attached data window. +fn world(sessions: u16) -> (SharedService, Vec>) { + let service: SharedService = Rc::new(RefCell::new(RamService::default())); + let mut clients = Vec::new(); + for index in 0..sessions { + let session = SessionId::new(index); + kore_svc_dagfs::block_on(service.borrow_mut().engine().attach_session(session)) + .expect("attach session"); + let mut client = FsClient::new(Loopback { + service: Rc::clone(&service), + session, + window: vec![0u8; WINDOW_BYTES], + }); + client + .attach_window(index as u32 + 1, WINDOW_BYTES as u32) + .expect("attach window"); + clients.push(client); + } + (service, clients) +} + +/// Put `bytes` at the start of a client's window and write them to `handle`. +fn write_all( + client: &mut FsClient, + handle: kore_svc_dagfs::Handle, + bytes: &[u8], +) -> FsResult { + client.transport().window[..bytes.len()].copy_from_slice(bytes); + client.write_at(handle, 0, 0, bytes.len() as u32, ANY_REVISION) +} + +/// Read a whole file back through the window. +fn read_all(client: &mut FsClient, handle: kore_svc_dagfs::Handle) -> FsResult> { + let len = client.stat(handle)?.len as u32; + let count = client.read(handle, 0, 0, len)?; + Ok(client.transport().window[..count as usize].to_vec()) +} + +// ── the basic round trip ──────────────────────────────────────────────────── + +#[test] +fn the_status_record_crosses_the_wire_whole() { + use kore_abi::svc::FsSystemStatusWire; + + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + + let len = c.system_status(0).unwrap(); + assert_eq!(len as usize, FsSystemStatusWire::BYTES); + let wire = kore_svc_dagfs::read_system_status(&c.transport().window, 0).unwrap(); + // The bare RAM harness: volatile posture, one attached window (ours). + assert_eq!(wire.flags.get() & FsSystemStatusWire::FLAG_DURABLE, 0); + assert_eq!(wire.windows.get(), 1); + assert_eq!(wire.commits.get(), 0); +} + +#[test] +fn create_write_and_read_back_across_the_wire() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + + let root = c.open_root(Rights::ALL).unwrap(); + assert_eq!(root.kind, ObjectKind::Directory); + + let file = c + .create(root.handle, &name("hello"), ObjectKind::File) + .unwrap(); + assert_eq!(file.kind, ObjectKind::File); + assert_eq!(file.len, 0); + + let revision = write_all(c, file.handle, b"kore").unwrap(); + assert!(revision > file.revision); + + assert_eq!(read_all(c, file.handle).unwrap(), b"kore"); + let info = c.stat(file.handle).unwrap(); + assert_eq!(info.len, 4); + assert_eq!(info.links, 1); + assert_eq!(info.revision, revision); +} + +#[test] +fn a_multi_chunk_payload_survives_the_window_round_trip() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + let file = c + .create(root.handle, &name("big"), ObjectKind::File) + .unwrap(); + + // Larger than one provider chunk, so the transfer straddles chunk + // boundaries on the way in and on the way out. + let payload: Vec = (0..600u32).map(|i| (i % 251) as u8).collect(); + write_all(c, file.handle, &payload).unwrap(); + assert_eq!(read_all(c, file.handle).unwrap(), payload); +} + +#[test] +fn typed_version_lending_pins_bytes_and_releases_affinely() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + let file = c + .create(root.handle, &name("program"), ObjectKind::File) + .unwrap(); + let revision = write_all(c, file.handle, b"immutable-v1").unwrap(); + + let pending = c.lend_version(file.handle, 64).unwrap(); + let identity_window = c.transport().window.clone(); + let version = pending.identify(&identity_window, 64).unwrap(); + let identity = version.identity(); + assert_eq!(identity.version, revision); + assert_eq!(identity.len, 12); + assert_eq!(identity.digest, *blake3::hash(b"immutable-v1").as_bytes()); + + // The ordinary file can advance while the lent revision stays frozen. + write_all(c, file.handle, b"mutable-v2!!").unwrap(); + let count = c.read_version(&version, 0, 0, WINDOW_BYTES as u32).unwrap(); + assert_eq!(&c.transport().window[..count as usize], b"immutable-v1"); + c.release_version(version).unwrap(); +} + +#[test] +fn refused_identity_landing_does_not_consume_snapshot_capacity() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + let file = c + .create(root.handle, &name("image"), ObjectKind::File) + .unwrap(); + write_all(c, file.handle, b"elf").unwrap(); + + // This used to install the snapshot first and only then discover that its + // identity did not fit, leaking one unreachable lease per refusal. + for _ in 0..(kore_svc_dagfs::bounds::MAX_LENT_VERSIONS + 1) { + assert_eq!( + c.lend_version(file.handle, WINDOW_BYTES as u32 - 8) + .unwrap_err(), + FsError::Range + ); + } + + let pending = c.lend_version(file.handle, 0).unwrap(); + let identity_window = c.transport().window.clone(); + let version = pending.identify(&identity_window, 0).unwrap(); + c.release_version(version).unwrap(); +} + +#[test] +fn append_reports_its_offset_across_the_wire() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + let log = c + .create(root.handle, &name("log"), ObjectKind::File) + .unwrap(); + + let mut offsets = Vec::new(); + for _ in 0..3 { + c.transport().window[..4].copy_from_slice(b"abc\n"); + let (offset, _) = c.append(log.handle, 0, 4, ANY_REVISION).unwrap(); + offsets.push(offset); + } + assert_eq!(offsets, vec![0, 4, 8]); + assert_eq!(read_all(c, log.handle).unwrap(), b"abc\nabc\nabc\n"); +} + +#[test] +fn a_revision_conflict_crosses_the_wire_intact() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + let file = c.create(root.handle, &name("f"), ObjectKind::File).unwrap(); + + let stale = c.stat(file.handle).unwrap().revision; + write_all(c, file.handle, b"first").unwrap(); + + c.transport().window[..6].copy_from_slice(b"second"); + assert_eq!( + c.write_at(file.handle, 0, 0, 6, stale).unwrap_err(), + FsError::Conflict, + "the engine's Conflict must reach the client as Conflict" + ); + assert_eq!(read_all(c, file.handle).unwrap(), b"first"); +} + +// ── the data window ───────────────────────────────────────────────────────── + +#[test] +fn a_bulk_verb_before_attach_is_refused() { + let service: SharedService = Rc::new(RefCell::new(RamService::default())); + let session = SessionId::new(0); + kore_svc_dagfs::block_on(service.borrow_mut().engine().attach_session(session)).unwrap(); + let mut c = FsClient::new(Loopback { + service: Rc::clone(&service), + session, + window: vec![0u8; WINDOW_BYTES], + }); + + // Control-plane verbs work without a window… + let root = c.open_root(Rights::ALL).unwrap(); + let file = c.create(root.handle, &name("f"), ObjectKind::File).unwrap(); + // …but a read has nowhere to land. + assert_eq!(c.read(file.handle, 0, 0, 4).unwrap_err(), FsError::NoWindow); + assert_eq!( + c.write_at(file.handle, 0, 0, 4, ANY_REVISION).unwrap_err(), + FsError::NoWindow + ); + assert_eq!(c.list(root.handle, 0, 0).unwrap_err(), FsError::NoWindow); + + // After attaching, the same calls work. + c.attach_window(1, WINDOW_BYTES as u32).unwrap(); + assert!(c.list(root.handle, 0, 0).is_ok()); +} + +#[test] +fn a_transfer_past_the_window_is_refused() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + let file = c.create(root.handle, &name("f"), ObjectKind::File).unwrap(); + write_all(c, file.handle, b"payload").unwrap(); + + // Landing past the end of the window, and an offset+length that would wrap. + assert_eq!( + c.read(file.handle, 0, WINDOW_BYTES as u32, 8).unwrap_err(), + FsError::Range + ); + assert_eq!( + c.read(file.handle, 0, u32::MAX, 8).unwrap_err(), + FsError::Range + ); + assert_eq!( + c.write_at(file.handle, 0, WINDOW_BYTES as u32 - 2, 8, ANY_REVISION) + .unwrap_err(), + FsError::Range + ); + // The file is untouched by every refusal. + assert_eq!(read_all(c, file.handle).unwrap(), b"payload"); +} + +#[test] +fn listing_entries_land_in_the_window_and_decode() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + + for entry in ["alpha", "beta"] { + let made = c + .create(root.handle, &name(entry), ObjectKind::File) + .unwrap(); + c.close(made.handle).unwrap(); + } + let dir = c + .create(root.handle, &name("sub"), ObjectKind::Directory) + .unwrap(); + c.close(dir.handle).unwrap(); + + let page = c.list(root.handle, 0, 0).unwrap(); + assert_eq!(page.entries, 3); + assert_eq!(page.next, None); + + let window = c.transport().window.clone(); + let mut seen: Vec<(String, bool)> = (0..page.entries as usize) + .map(|i| { + let entry = FsClient::::read_entry(&window, 0, i).unwrap(); + // `bool` for "is a directory" so the tuple sorts; the kind itself + // is an enum with no ordering, and inventing one for a test would + // be a worse trade than this. + ( + entry.name.as_str().to_string(), + entry.kind == ObjectKind::Directory, + ) + }) + .collect(); + seen.sort(); + assert_eq!( + seen, + vec![ + ("alpha".to_string(), false), + ("beta".to_string(), false), + ("sub".to_string(), true), + ] + ); +} + +#[test] +fn structural_search_crosses_the_real_wire_and_window() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + let sub = c + .create(root.handle, &name("docs"), ObjectKind::Directory) + .unwrap(); + let report = c + .create(sub.handle, &name("report.csv"), ObjectKind::File) + .unwrap(); + write_all(c, report.handle, b"a,b\n1,2\n").unwrap(); + c.create(sub.handle, &name("notes.txt"), ObjectKind::File) + .unwrap(); + + let page = c + .search( + root.handle, + SearchQuery { + name_contains: Some(name("report")), + kind: Some(ObjectKind::File), + min_len: 1, + max_len: 64, + content: None, + freshness: kore_svc_dagfs::FreshnessPolicy::StructuralOnly, + max_index_lag: 0, + max_work: 0, + }, + None, + 0, + ) + .unwrap(); + assert_eq!(page.hits, 1); + assert!(page.observation != 0); + assert_eq!(page.next, None); + let hit = read_search_hit(&c.transport().window, 0, 0).unwrap(); + let path: Vec<&str> = hit.path.iter().map(|part| part.as_str()).collect(); + assert_eq!(path, ["docs", "report.csv"]); + assert_eq!(hit.kind, ObjectKind::File); + assert_eq!(hit.len, 8); +} + +/// The deepest legal path crosses the wire whole. +/// +/// `MAX_PATH_DEPTH` went 8 → 32 and the frozen hit spelling did not follow +/// it, so the encoder wrote a ninth separator past its 263-byte field — an +/// index past a fixed array, in the service. The field is now pinned to the +/// depth at compile time; this is the boundary case end to end: a +/// 32-component path of 32-byte names is exactly `FS_SEARCH_PATH_BYTES` +/// long, lands in the window, and decodes back to the components. +#[test] +fn the_deepest_legal_path_crosses_the_search_wire_whole() { + use kore_abi::svc::FS_SEARCH_PATH_BYTES; + use kore_svc_dagfs::bounds::{MAX_NAME_LEN, MAX_PATH_DEPTH}; + + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + // Every name fills its whole field: the level number, zero-padded. + let level_name = |level: usize| name(&format!("{level:0>width$}", width = MAX_NAME_LEN)); + let mut expected: Vec = Vec::new(); + // Directories at every level but the last, then the file: 32 components. + // A session holds at most `MAX_HANDLES_PER_SESSION` handles, which is the + // depth bound itself, so each intermediate is released as the chain grows + // — the walk is the chain, not the handle table. + let mut parent = root.handle; + let mut previous: Option = None; + for level in 0..MAX_PATH_DEPTH - 1 { + let made = c + .create(parent, &level_name(level), ObjectKind::Directory) + .unwrap(); + if let Some(done) = previous.take() { + c.close(done).unwrap(); + } + previous = Some(made.handle); + parent = made.handle; + expected.push(level_name(level).as_str().to_string()); + } + let leaf = name(&format!("{:x>width$}", "leaf", width = MAX_NAME_LEN)); + let made = c.create(parent, &leaf, ObjectKind::File).unwrap(); + c.close(made.handle).unwrap(); + if let Some(done) = previous.take() { + c.close(done).unwrap(); + } + expected.push(leaf.as_str().to_string()); + + let page = c + .search( + root.handle, + SearchQuery { + name_contains: Some(name("leaf")), + kind: Some(ObjectKind::File), + min_len: 0, + max_len: 64, + content: None, + freshness: kore_svc_dagfs::FreshnessPolicy::StructuralOnly, + max_index_lag: 0, + max_work: 0, + }, + None, + 0, + ) + .unwrap(); + assert_eq!(page.hits, 1); + let hit = read_search_hit(&c.transport().window, 0, 0).unwrap(); + assert_eq!(hit.path.len(), MAX_PATH_DEPTH); + let got: Vec = hit + .path + .iter() + .map(|part| part.as_str().to_string()) + .collect(); + assert_eq!(got, expected); + let spelled = hit + .path + .iter() + .map(|part| part.as_bytes().len()) + .sum::() + + hit.path.len() + - 1; + assert_eq!(spelled, FS_SEARCH_PATH_BYTES); +} + +#[test] +fn a_listing_page_that_would_not_fit_the_window_is_refused_not_truncated() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + for index in 0..4 { + let made = c + .create(root.handle, &name(&format!("f{index}")), ObjectKind::File) + .unwrap(); + c.close(made.handle).unwrap(); + } + + // Land the page so close to the end that fewer entries fit than the + // directory holds. Silently returning a short page would look identical to + // a complete one. + let near_end = (WINDOW_BYTES - FsDirEntryWire::BYTES) as u32; + assert_eq!( + c.list(root.handle, 0, near_end).unwrap_err(), + FsError::Range + ); + // With room, the same listing succeeds. + assert_eq!(c.list(root.handle, 0, 0).unwrap().entries, 4); +} + +#[test] +fn listing_pages_through_a_large_directory() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + + let total = 20usize; + for index in 0..total { + let made = c + .create(root.handle, &name(&format!("f{index}")), ObjectKind::File) + .unwrap(); + c.close(made.handle).unwrap(); + } + + let mut seen = Vec::new(); + let mut cursor = 0u32; + loop { + let page = c.list(root.handle, cursor, 0).unwrap(); + let window = c.transport().window.clone(); + for i in 0..page.entries as usize { + seen.push( + FsClient::::read_entry(&window, 0, i) + .unwrap() + .name + .as_str() + .to_string(), + ); + } + match page.next { + Some(next) => cursor = next, + None => break, + } + } + seen.sort(); + seen.dedup(); + assert_eq!(seen.len(), total, "every entry appeared exactly once"); +} + +// ── isolation and sharing, over the wire ──────────────────────────────────── + +#[test] +fn two_sessions_have_separate_roots_across_the_wire() { + let (_svc, mut clients) = world(2); + + let a_root = clients[0].open_root(Rights::ALL).unwrap(); + let a_file = clients[0] + .create(a_root.handle, &name("notes"), ObjectKind::File) + .unwrap(); + write_all(&mut clients[0], a_file.handle, b"A's bytes").unwrap(); + + let b_root = clients[1].open_root(Rights::ALL).unwrap(); + let b_file = clients[1] + .create(b_root.handle, &name("notes"), ObjectKind::File) + .unwrap(); + write_all(&mut clients[1], b_file.handle, b"B's bytes").unwrap(); + + let a_id = clients[0].stat(a_file.handle).unwrap().object; + let b_id = clients[1].stat(b_file.handle).unwrap().object; + assert_ne!(a_id, b_id, "same name, different objects"); + assert_eq!( + read_all(&mut clients[0], a_file.handle).unwrap(), + b"A's bytes" + ); + assert_eq!( + read_all(&mut clients[1], b_file.handle).unwrap(), + b"B's bytes" + ); +} + +#[test] +fn one_sessions_handle_is_invalid_on_another_pair() { + // The plan's §10.2 row: a handle minted for one session must be inert when + // presented over another session's pair. + let (_svc, mut clients) = world(2); + let a_root = clients[0].open_root(Rights::ALL).unwrap(); + let a_file = clients[0] + .create(a_root.handle, &name("mine"), ObjectKind::File) + .unwrap(); + + assert_eq!( + clients[1].stat(a_file.handle).unwrap_err(), + FsError::InvalidHandle + ); + assert_eq!( + clients[1].read(a_file.handle, 0, 0, 4).unwrap_err(), + FsError::InvalidHandle + ); +} + +#[test] +fn a_grant_and_its_revocation_cross_the_wire() { + let (_svc, mut clients) = world(2); + let a_root = clients[0].open_root(Rights::ALL).unwrap(); + let file = clients[0] + .create(a_root.handle, &name("shared"), ObjectKind::File) + .unwrap(); + write_all(&mut clients[0], file.handle, b"from A").unwrap(); + + let grant = clients[0] + .grant( + file.handle, + SessionId::new(1), + Rights::READ, + Some(name("gift")), + ) + .unwrap(); + + let view = clients[1].open_grant(grant, Rights::READ).unwrap(); + assert_eq!(read_all(&mut clients[1], view.handle).unwrap(), b"from A"); + + // Read-only really is read-only, through the wire. + clients[1].transport().window[..2].copy_from_slice(b"no"); + assert_eq!( + clients[1] + .write_at(view.handle, 0, 0, 2, ANY_REVISION) + .unwrap_err(), + FsError::AccessDenied + ); + + // And revocation is immediate on the handle B already holds. + clients[0].revoke(grant).unwrap(); + assert_eq!( + clients[1].stat(view.handle).unwrap_err(), + FsError::AccessLost + ); + assert_eq!(read_all(&mut clients[0], file.handle).unwrap(), b"from A"); +} + +#[test] +fn losing_a_route_reaches_the_client_as_access_lost() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + let dir = c + .create(root.handle, &name("dir"), ObjectKind::Directory) + .unwrap(); + let file = c.create(dir.handle, &name("f"), ObjectKind::File).unwrap(); + write_all(c, file.handle, b"bytes").unwrap(); + + c.unlink(dir.handle, &name("f")).unwrap(); + assert_eq!(c.stat(file.handle).unwrap_err(), FsError::AccessLost); + assert_eq!( + c.read(file.handle, 0, 0, 4).unwrap_err(), + FsError::AccessLost + ); +} + +// ── the protocol's own guarantees, observed from a client ─────────────────── + +#[test] +fn two_names_for_one_object_are_visible_as_one_object() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + let file = c + .create(root.handle, &name("one"), ObjectKind::File) + .unwrap(); + write_all(c, file.handle, b"shared").unwrap(); + c.link(root.handle, &name("two"), file.handle).unwrap(); + + let via_two = c.open_at(root.handle, &name("two"), Rights::ALL).unwrap(); + assert_eq!( + c.stat(file.handle).unwrap().object, + c.stat(via_two.handle).unwrap().object + ); + assert_eq!(c.stat(via_two.handle).unwrap().links, 2); + assert_eq!(read_all(c, via_two.handle).unwrap(), b"shared"); +} + +#[test] +fn every_engine_refusal_reaches_the_client_as_itself() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + + // A representative refusal for each class the client can provoke without + // reaching into the engine. + assert_eq!( + c.open_at(root.handle, &name("absent"), Rights::ALL) + .unwrap_err(), + FsError::NotFound + ); + let file = c.create(root.handle, &name("f"), ObjectKind::File).unwrap(); + assert_eq!( + c.create(root.handle, &name("f"), ObjectKind::File) + .unwrap_err(), + FsError::AlreadyExists + ); + assert_eq!( + c.list(file.handle, 0, 0).unwrap_err(), + FsError::NotDirectory + ); + assert_eq!( + c.read(root.handle, 0, 0, 4).unwrap_err(), + FsError::IsDirectory + ); + let read_only = c.open_root(Rights::READ).unwrap(); + assert_eq!( + c.create(read_only.handle, &name("x"), ObjectKind::File) + .unwrap_err(), + FsError::AccessDenied + ); + let dir = c + .create(root.handle, &name("d"), ObjectKind::Directory) + .unwrap(); + let inner = c.create(dir.handle, &name("i"), ObjectKind::File).unwrap(); + c.close(inner.handle).unwrap(); + assert_eq!( + c.unlink(root.handle, &name("d")).unwrap_err(), + FsError::NotEmpty + ); + assert_eq!( + c.resize(file.handle, u64::MAX, ANY_REVISION).unwrap_err(), + FsError::OutOfStorage + ); +} + +#[test] +fn a_provider_failure_crosses_as_a_provider_failure() { + let (service, mut clients) = world(1); + let c = &mut clients[0]; + let root = c.open_root(Rights::ALL).unwrap(); + let file = c.create(root.handle, &name("f"), ObjectKind::File).unwrap(); + write_all(c, file.handle, b"original").unwrap(); + + service + .borrow_mut() + .engine() + .provider() + .inject(kore_svc_dagfs::provider::Fault::Write(ProviderError::Media)); + + c.transport().window[..8].copy_from_slice(b"replaced"); + let err = c.write_at(file.handle, 0, 0, 8, ANY_REVISION).unwrap_err(); + assert!( + matches!(err, FsError::ProviderFailure(_)), + "a provider fault must not arrive as a capacity refusal, got {err:?}" + ); + // And the request had no visible effect. + assert_eq!(read_all(c, file.handle).unwrap(), b"original"); +} + +#[test] +fn a_malformed_record_is_refused_by_the_serving_side() { + // Bypass the client and hand the transport a record it must refuse — the + // path a hostile peer would take. + let (service, _clients) = world(1); + let mut transport = Loopback { + service, + session: SessionId::new(0), + window: vec![0u8; WINDOW_BYTES], + }; + let mut bogus = kore_svc_dagfs::wire::encode_req(&kore_svc_dagfs::FsRequest::OpenRoot { + rights: Rights::ALL, + }); + bogus.op = 200; + let reply = transport.exchange(&bogus).unwrap(); + assert_ne!( + reply.status, + FsRepWire::STATUS_OK, + "garbage must not succeed" + ); +} + +#[test] +fn a_recipient_discovers_and_opens_a_grant_over_the_wire() { + // The whole point of `ListGrants`: without it a founded client learns no + // `GrantId` and the sharing half of the MVP is unreachable from a real + // client, however well it works inside the engine. + let (_svc, mut clients) = world(2); + let a_root = clients[0].open_root(Rights::ALL).unwrap(); + let file = clients[0] + .create(a_root.handle, &name("shared"), ObjectKind::File) + .unwrap(); + write_all(&mut clients[0], file.handle, b"from A").unwrap(); + clients[0] + .grant( + file.handle, + SessionId::new(1), + Rights::READ, + Some(name("inbox")), + ) + .unwrap(); + + // B knows nothing but its own session — it discovers the rest. + let count = clients[1].list_grants(0).unwrap(); + assert_eq!(count, 2, "its private root plus the new grant"); + + let window = clients[1].transport().window.clone(); + let entries: Vec<_> = (0..count as usize) + .map(|i| FsClient::::read_grant_entry(&window, 0, i).unwrap()) + .collect(); + + let root_entry = entries.iter().find(|e| e.is_private_root).unwrap(); + assert_eq!(root_entry.rights, Rights::ALL); + assert!(root_entry.display_name.is_none()); + + let gift = entries.iter().find(|e| !e.is_private_root).unwrap(); + assert_eq!(gift.rights, Rights::READ); + assert_eq!(gift.display_name.unwrap().as_str(), "inbox"); + + // And the discovered id opens. + let view = clients[1].open_grant(gift.grant, Rights::READ).unwrap(); + assert_eq!(read_all(&mut clients[1], view.handle).unwrap(), b"from A"); +} + +#[test] +fn listing_grants_needs_a_window_and_respects_its_bounds() { + let (_svc, mut clients) = world(1); + let near_end = (WINDOW_BYTES - 8) as u32; + assert_eq!( + clients[0].list_grants(near_end).unwrap_err(), + FsError::Range, + "a page that would not fit must be refused, not truncated" + ); + assert!(clients[0].list_grants(0).is_ok()); +} + +/// The routes a `Parents` page landed at `window_offset`, joined by `/`. +fn parent_routes(window: &[u8], window_offset: u32, count: u32) -> Vec { + (0..count as usize) + .map(|i| { + let hit = read_parent(window, window_offset, i).unwrap(); + hit.path + .iter() + .map(|n| n.as_str()) + .collect::>() + .join("/") + }) + .collect() +} + +/// `a` and `b` under the root, `c` under `a`, and one file `f` linked into +/// all three: the diamond the explorer's Folders column shows. +fn diamond(c: &mut FsClient) -> (kore_svc_dagfs::Handle, kore_svc_dagfs::Handle) { + let root = c.open_root(Rights::ALL).unwrap(); + let a = c + .create(root.handle, &name("a"), ObjectKind::Directory) + .unwrap(); + let b = c + .create(root.handle, &name("b"), ObjectKind::Directory) + .unwrap(); + let cc = c.create(a.handle, &name("c"), ObjectKind::Directory).unwrap(); + let f = c.create(a.handle, &name("f"), ObjectKind::File).unwrap(); + c.link(b.handle, &name("f"), f.handle).unwrap(); + c.link(cc.handle, &name("also-f"), f.handle).unwrap(); + c.close(b.handle).unwrap(); + c.close(cc.handle).unwrap(); + (a.handle, f.handle) +} + +#[test] +fn parents_name_every_visible_folder_by_its_shortest_route() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let (a, f) = diamond(c); + + // Breadth-first from the root: `a` and `b` (depth one), then `a/c`. + let page = c.parents(f, 0, 0).unwrap(); + assert_eq!(page.next, None); + let window = c.transport().window.clone(); + assert_eq!(parent_routes(&window, 0, page.entries), ["a", "b", "a/c"]); + // The count stat reports agrees with what this view can see. + assert_eq!(c.stat(f).unwrap().links, 3); + + // A folder's parent; the root's are none (nothing above the view). + let page = c.parents(a, 0, 0).unwrap(); + let window = c.transport().window.clone(); + assert_eq!(parent_routes(&window, 0, page.entries), [""]); + let root = c.open_root(Rights::READ).unwrap(); + assert_eq!(c.parents(root.handle, 0, 0).unwrap().entries, 0); +} + +#[test] +fn a_small_window_pages_parents_instead_of_refusing() { + let (_svc, mut clients) = world(1); + let c = &mut clients[0]; + let (_, f) = diamond(c); + let hit = kore_abi::svc::FsSearchHitWire::BYTES as u32; + // Room for exactly two records at the end of the window. + let offset = WINDOW_BYTES as u32 - 2 * hit; + let first = c.parents(f, 0, offset).unwrap(); + assert_eq!((first.entries, first.next), (2, Some(2))); + let window = c.transport().window.clone(); + assert_eq!(parent_routes(&window, offset, 2), ["a", "b"]); + let rest = c.parents(f, 2, offset).unwrap(); + assert_eq!((rest.entries, rest.next), (1, None)); + let window = c.transport().window.clone(); + assert_eq!(parent_routes(&window, offset, 1), ["a/c"]); + // A window with no room for one record is refused, not truncated. + assert_eq!( + c.parents(f, 0, WINDOW_BYTES as u32 - hit + 1).unwrap_err(), + FsError::Range + ); +} + +#[test] +fn parents_never_name_a_folder_outside_the_handles_authority() { + let (_svc, mut clients) = world(2); + let (a, _) = diamond(&mut clients[0]); + let grant = clients[0] + .grant(a, SessionId::new(1), Rights::READ, Some(name("gift"))) + .unwrap(); + // Session 1 holds only `a`: it reaches `f` there and as `c/also-f`, but + // `b` is outside the edge it entered through and is never named. + let view = clients[1].open_grant(grant, Rights::READ).unwrap(); + let f = clients[1] + .open_at(view.handle, &name("f"), Rights::READ) + .unwrap(); + let page = clients[1].parents(f.handle, 0, 0).unwrap(); + let window = clients[1].transport().window.clone(); + assert_eq!(parent_routes(&window, 0, page.entries), ["", "c"]); +} diff --git a/native/services/dagfs/tests/namespace.rs b/native/services/dagfs/tests/namespace.rs new file mode 100644 index 0000000..5e713bb --- /dev/null +++ b/native/services/dagfs/tests/namespace.rs @@ -0,0 +1,435 @@ +//! Namespace behaviour: the containment DAG, names, links, moves, cycles, and +//! reclamation, driven end-to-end through the engine's public API. +//! +//! Covers the namespace half of the plan's §10.1 matrix. + +mod common; + +use common::{create_dir, create_file, engine_with_one, list_names, name, read_all}; +use kore_svc_dagfs::{ + ANY_REVISION, FsError, ObjectKind, Rights, block_on, + bounds::{MAX_DIR_ENTRIES, MAX_OBJECTS}, +}; + +#[test] +fn a_created_object_is_immediately_usable_by_its_creator() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "hello", b"kore"); + // No re-open, no second lookup: the create handed back working authority. + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"kore"); + let info = engine.stat(session, file).unwrap(); + assert_eq!(info.kind, ObjectKind::File); + assert_eq!(info.len, 4); + assert_eq!(info.links, 1); +} + +#[test] +fn create_list_read_write_round_trip() { + let (mut engine, session, root) = engine_with_one(); + let dir = create_dir(&mut engine, session, root, "demo"); + let file = create_file(&mut engine, session, dir, "hello", b"kore"); + + assert_eq!(list_names(&engine, session, root).unwrap(), vec!["demo"]); + assert_eq!(list_names(&engine, session, dir).unwrap(), vec!["hello"]); + + // Overwrite in place and read back. + block_on(engine.write_at(session, file, 0, b"KORE", ANY_REVISION)).unwrap(); + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"KORE"); +} + +#[test] +fn two_names_identify_one_object_and_share_its_bytes() { + let (mut engine, session, root) = engine_with_one(); + let projects = create_dir(&mut engine, session, root, "projects"); + let file = create_file(&mut engine, session, projects, "main.rs", b"fn main() {}"); + + // A second containment edge to the same object — not a copy. + engine.link(session, root, &name("notes.rs"), file).unwrap(); + + let via_projects = engine + .open_at(session, projects, &name("main.rs"), Rights::ALL) + .unwrap(); + let via_root = engine + .open_at(session, root, &name("notes.rs"), Rights::ALL) + .unwrap(); + + let a = engine.stat(session, via_projects.handle).unwrap(); + let b = engine.stat(session, via_root.handle).unwrap(); + assert_eq!(a.object, b.object, "one object, two names"); + assert_eq!(a.links, 2); + + // A write through one name is visible through the other, which is what + // "not copied" means observationally. + block_on(engine.write_at(session, via_projects.handle, 0, b"CHANGED", ANY_REVISION)).unwrap(); + assert_eq!( + &read_all(&mut engine, session, via_root.handle).unwrap()[..7], + b"CHANGED" + ); +} + +#[test] +fn unlinking_one_name_preserves_the_other_and_the_object() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "one", b"payload"); + engine.link(session, root, &name("two"), file).unwrap(); + + block_on(engine.unlink(session, root, &name("one"))).unwrap(); + + assert_eq!( + engine + .open_at(session, root, &name("one"), Rights::ALL) + .unwrap_err(), + FsError::NotFound + ); + let survivor = engine + .open_at(session, root, &name("two"), Rights::ALL) + .unwrap(); + assert_eq!( + read_all(&mut engine, session, survivor.handle).unwrap(), + b"payload" + ); + assert_eq!(engine.stat(session, survivor.handle).unwrap().links, 1); +} + +#[test] +fn rename_relabels_without_changing_the_object() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "before", b"same bytes"); + let before = engine.stat(session, file).unwrap().object; + + engine + .rename(session, root, &name("before"), &name("after")) + .unwrap(); + + assert_eq!(list_names(&engine, session, root).unwrap(), vec!["after"]); + // The handle opened before the rename still works: rename keeps the edge. + assert_eq!(engine.stat(session, file).unwrap().object, before); + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"same bytes"); +} + +#[test] +fn rename_onto_a_taken_name_is_refused_and_changes_nothing() { + let (mut engine, session, root) = engine_with_one(); + create_file(&mut engine, session, root, "a", b"a"); + create_file(&mut engine, session, root, "b", b"b"); + + assert_eq!( + engine + .rename(session, root, &name("a"), &name("b")) + .unwrap_err(), + FsError::AlreadyExists + ); + assert_eq!(list_names(&engine, session, root).unwrap(), vec!["a", "b"]); +} + +#[test] +fn moving_removes_only_the_source_edge_and_preserves_the_object() { + let (mut engine, session, root) = engine_with_one(); + let dst = create_dir(&mut engine, session, root, "dst"); + let file = create_file(&mut engine, session, root, "f", b"contents"); + let identity = engine.stat(session, file).unwrap().object; + + engine + .relocate(session, root, &name("f"), dst, &name("moved")) + .unwrap(); + + assert_eq!( + engine + .open_at(session, root, &name("f"), Rights::ALL) + .unwrap_err(), + FsError::NotFound + ); + let moved = engine + .open_at(session, dst, &name("moved"), Rights::ALL) + .unwrap(); + let info = engine.stat(session, moved.handle).unwrap(); + assert_eq!(info.object, identity, "the object kept its identity"); + assert_eq!(info.links, 1, "one edge out, one edge in"); + assert_eq!( + read_all(&mut engine, session, moved.handle).unwrap(), + b"contents" + ); +} + +#[test] +fn moving_needs_namespace_rights_at_both_parents() { + let (mut engine, session, root) = engine_with_one(); + let dst = create_dir(&mut engine, session, root, "dst"); + create_file(&mut engine, session, root, "f", b"x"); + + // A read-only handle on the destination cannot receive a move. + let dst_read_only = engine + .open_at(session, root, &name("dst"), Rights::READ) + .unwrap(); + assert_eq!( + engine + .relocate(session, root, &name("f"), dst_read_only.handle, &name("m")) + .unwrap_err(), + FsError::AccessDenied + ); + + // A read-only handle on the source cannot give one up either. + let root_read_only = engine.open_root(session, Rights::READ).unwrap(); + assert_eq!( + engine + .relocate(session, root_read_only.handle, &name("f"), dst, &name("m")) + .unwrap_err(), + FsError::AccessDenied + ); + + // The file never moved. + assert!( + engine + .open_at(session, root, &name("f"), Rights::ALL) + .is_ok() + ); +} + +#[test] +fn directory_diamonds_keep_route_identity_and_detach_exactly() { + let (mut engine, session, root) = engine_with_one(); + let left = create_dir(&mut engine, session, root, "left"); + let right = create_dir(&mut engine, session, root, "right"); + let shared = create_dir(&mut engine, session, left, "shared"); + engine + .link(session, right, &name("shared"), shared) + .unwrap(); + let through_right = engine + .open_at(session, right, &name("shared"), Rights::ALL) + .unwrap(); + assert_eq!(engine.stat(session, shared).unwrap().links, 2); + + block_on(engine.detach_edge(session, left, &name("shared"))).unwrap(); + assert_eq!(engine.stat(session, shared), Err(FsError::AccessLost)); + assert_eq!(engine.stat(session, through_right.handle).unwrap().links, 1); + assert_eq!(list_names(&engine, session, right).unwrap(), vec!["shared"]); +} + +#[test] +fn directory_cycles_are_refused() { + let (mut engine, session, root) = engine_with_one(); + let outer = create_dir(&mut engine, session, root, "outer"); + let inner = create_dir(&mut engine, session, outer, "inner"); + let deeper = create_dir(&mut engine, session, inner, "deeper"); + + assert_eq!( + engine + .link(session, deeper, &name("back"), outer) + .unwrap_err(), + FsError::Cycle + ); + + // Into itself. + assert_eq!( + engine + .relocate(session, root, &name("outer"), outer, &name("self")) + .unwrap_err(), + FsError::Cycle + ); + // Into its own descendant. + assert_eq!( + engine + .relocate(session, root, &name("outer"), deeper, &name("loop")) + .unwrap_err(), + FsError::Cycle + ); + + // The tree survived both refusals intact. + assert_eq!(list_names(&engine, session, root).unwrap(), vec!["outer"]); + assert_eq!(list_names(&engine, session, outer).unwrap(), vec!["inner"]); + assert_eq!(list_names(&engine, session, inner).unwrap(), vec!["deeper"]); +} + +#[test] +fn unlinking_a_non_empty_directory_is_refused_until_it_is_emptied() { + let (mut engine, session, root) = engine_with_one(); + let dir = create_dir(&mut engine, session, root, "d"); + create_file(&mut engine, session, dir, "f", b"x"); + + assert_eq!( + block_on(engine.unlink(session, root, &name("d"))).unwrap_err(), + FsError::NotEmpty + ); + block_on(engine.unlink(session, dir, &name("f"))).unwrap(); + block_on(engine.unlink(session, root, &name("d"))).unwrap(); + assert!(list_names(&engine, session, root).unwrap().is_empty()); +} + +#[test] +fn invalid_names_are_refused_at_every_door() { + let (mut engine, session, root) = engine_with_one(); + create_file(&mut engine, session, root, "real", b"x"); + + for bad in ["", ".", "..", "a/b", "a\0b"] { + assert_eq!( + kore_svc_dagfs::Name::new(bad).unwrap_err(), + FsError::InvalidName, + "{bad:?} must never become a name" + ); + } + // Over-long is refused by length, not truncated into a colliding name. + let too_long = "x".repeat(kore_svc_dagfs::bounds::MAX_NAME_LEN + 1); + assert_eq!( + kore_svc_dagfs::Name::new(&too_long).unwrap_err(), + FsError::InvalidName + ); +} + +#[test] +fn listing_pages_through_a_full_directory_without_loss() { + let (mut engine, session, root) = engine_with_one(); + // Fill the edge arena without also exhausting the smaller whole-namespace + // object arena: many names lawfully identify one object in this DAG. + let file = create_file(&mut engine, session, root, "f0", b"x"); + let mut expected = vec![String::from("f0")]; + for index in 1..MAX_DIR_ENTRIES { + let file_name = format!("f{index}"); + engine.link(session, root, &name(&file_name), file).unwrap(); + expected.push(file_name); + } + engine.close(session, file).unwrap(); + expected.sort(); + + let listed = list_names(&engine, session, root).unwrap(); + assert_eq!(listed, expected); + + // And the directory is now full, as a typed refusal. + assert_eq!( + block_on(engine.create(session, root, &name("one-too-many"), ObjectKind::File)) + .unwrap_err(), + FsError::OutOfEdges + ); +} + +#[test] +fn a_deleted_name_can_be_reused() { + let (mut engine, session, root) = engine_with_one(); + create_file(&mut engine, session, root, "temp", b"first"); + block_on(engine.unlink(session, root, &name("temp"))).unwrap(); + + // Same name, genuinely new object. + let second = create_file(&mut engine, session, root, "temp", b"second"); + assert_eq!(read_all(&mut engine, session, second).unwrap(), b"second"); +} + +#[test] +fn reclamation_frees_the_object_and_its_storage_exactly_once() { + let (mut engine, session, root) = engine_with_one(); + let objects_before = engine.object_count(); + let chunks_before = engine.provider().chunks_in_use(); + + create_file(&mut engine, session, root, "doomed", b"some bytes"); + assert!(engine.provider().chunks_in_use() > chunks_before); + + block_on(engine.unlink(session, root, &name("doomed"))).unwrap(); + + assert_eq!(engine.object_count(), objects_before); + assert_eq!(engine.provider().chunks_in_use(), chunks_before); + assert_eq!(engine.provider().placements(), 0); + assert_eq!(engine.leaked_placements(), 0); +} + +#[test] +fn a_linked_object_survives_until_its_last_name_goes() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "one", b"bytes"); + engine.link(session, root, &name("two"), file).unwrap(); + let placements = engine.provider().placements(); + + block_on(engine.unlink(session, root, &name("one"))).unwrap(); + assert_eq!( + engine.provider().placements(), + placements, + "still named, so still placed" + ); + + block_on(engine.unlink(session, root, &name("two"))).unwrap(); + assert_eq!(engine.provider().placements(), 0); + assert_eq!(engine.provider().chunks_in_use(), 0); +} + +#[test] +fn removing_a_subtree_releases_every_object_beneath_it() { + let (mut engine, session, root) = engine_with_one(); + let objects_before = engine.object_count(); + let dirs_before = engine.directory_count(); + + let a = create_dir(&mut engine, session, root, "a"); + let b = create_dir(&mut engine, session, a, "b"); + create_file(&mut engine, session, b, "deep", b"deep bytes"); + + // Recursive removal is a client-side loop, not one transaction — so walk + // it back out the way a shell's `rm -r` would. + block_on(engine.unlink(session, b, &name("deep"))).unwrap(); + block_on(engine.unlink(session, a, &name("b"))).unwrap(); + block_on(engine.unlink(session, root, &name("a"))).unwrap(); + + assert_eq!(engine.object_count(), objects_before); + assert_eq!(engine.directory_count(), dirs_before); + assert_eq!(engine.provider().chunks_in_use(), 0); +} + +#[test] +fn the_object_arena_refuses_when_full_without_corrupting_the_namespace() { + let (mut engine, session, root) = engine_with_one(); + // Filling the object arena now needs more than one directory: an object + // bound of MAX_OBJECTS against MAX_DIR_ENTRIES entries per directory means + // one directory runs out of edges long before the arena runs out of slots, + // and that refusal is a different (also correct) one. So take a handful of + // directories up front — the root can only hold MAX_DIR_ENTRIES of them — + // and fill them in turn, which keeps this test about the arena it names. + // + // Handles are closed as we go where possible, so the refusal we observe is + // the namespace running out of room rather than the session running out of + // handles. + const SPILLS: usize = 8; + assert!( + SPILLS * (MAX_DIR_ENTRIES - 1) >= MAX_OBJECTS, + "the spill directories must be able to hold the whole object arena" + ); + let mut created = 0usize; + let mut buckets = Vec::new(); + for index in 0..SPILLS { + let made = block_on(engine.create( + session, + root, + &name(&format!("d{index}")), + ObjectKind::Directory, + )) + .expect("a spill directory beneath the root"); + created += 1; + buckets.push(made.handle); + } + + let mut at = 0usize; + let refusal = loop { + let file_name = format!("f{created}"); + match block_on(engine.create(session, buckets[at], &name(&file_name), ObjectKind::File)) { + Ok(made) => { + engine.close(session, made.handle).unwrap(); + created += 1; + } + // This bucket is full; move to the next one. + Err(FsError::OutOfEdges) if at + 1 < buckets.len() => at += 1, + Err(err) => break err, + } + }; + assert_eq!(refusal, FsError::OutOfObjects); + // The private root already owns one object slot; everything else created, + // files and spill directories alike, accounts for the rest of the arena. + assert_eq!(created, MAX_OBJECTS - 1); + // Everything created is still correctly there and readable. + assert_eq!(list_names(&engine, session, root).unwrap().len(), SPILLS); + let probe = engine + .open_at( + session, + buckets[0], + &name(&format!("f{SPILLS}")), + Rights::ALL, + ) + .unwrap(); + assert_eq!( + engine.stat(session, probe.handle).unwrap().kind, + ObjectKind::File + ); +} diff --git a/native/services/dagfs/tests/pair.rs b/native/services/dagfs/tests/pair.rs new file mode 100644 index 0000000..2592183 --- /dev/null +++ b/native/services/dagfs/tests/pair.rs @@ -0,0 +1,674 @@ +//! The production shape: a **real founded pair window** with real SPSC rings. +//! +//! `loopback.rs` proves the codec and dispatch over encoded bytes but calls +//! `dispatch` directly. This suite goes through the transport the founder +//! actually builds: +//! +//! ```text +//! client → FsReqWire → spsc request ring (in the pair window) +//! → service pops, dispatches +//! → FsRepWire → spsc reply ring → client pops +//! ``` +//! +//! The window is formatted by `kore_rt::svc::split_window` at `REQ_CAP`/ +//! `REP_CAP` — the same call the founder makes, at this protocol's exact record +//! widths — so the ring geometry, the request/reply split offset, the slot +//! strides and the wrap behaviour are the real ones, not a re-derivation. +//! +//! Two windows are in play, and keeping them distinct is the point: the **pair +//! window** carries fixed-size records, and the **data window** carries bulk +//! bytes. Production has exactly this split. +//! +//! What is still not production: the bytes do not cross an address-space +//! boundary, and no founder delivered the windows. That is Phase 2 step 5 and +//! needs the QEMU boot gate. + +mod common; + +use common::name; +use kore_abi::svc::{FsRepWire, FsReqWire}; +use kore_ring::spsc::{Consumer, Producer}; +use kore_rt::svc::{REP_CAP, REQ_CAP, rep_offset, split_window, window_bytes}; +use kore_svc_dagfs::{ + ANY_REVISION, DataWindow, FsError, FsRequest, FsResponse, ObjectKind, RamService, Rights, + SessionId, block_on, + wire::{decode_rep_for, decode_req, encode_rep, encode_req}, +}; + +/// Bytes of bulk data window (separate from the pair's ring window). +const DATA_BYTES: usize = 512; + +/// The pair window's size, from the tree's own arithmetic. +const RING_WINDOW_BYTES: usize = window_bytes::(REQ_CAP, REP_CAP); + +/// A **64-byte aligned** pair window. +/// +/// The rings require cache-line alignment (`spsc`'s `ALIGN`), and refuse +/// `Misaligned` without it — adjacent rings packed in one window each have to +/// start on a line so their cursors do not share one. In production that comes +/// free: the founder maps a page-granular window. On a host a plain `Vec` +/// has no such guarantee, so the alignment is stated here rather than left to +/// the allocator's mood. +#[repr(align(64))] +struct RingWindow([u8; RING_WINDOW_BYTES]); + +impl RingWindow { + fn new() -> Self { + Self([0; RING_WINDOW_BYTES]) + } +} + +/// One founded pair, host-side: the two ring halves over one formatted window, +/// the service, and the client's data window. +struct Pair<'w> { + client_req: Producer<'w, FsReqWire>, + client_rep: Consumer<'w, FsRepWire>, + serve_req: Consumer<'w, FsReqWire>, + serve_rep: Producer<'w, FsRepWire>, + service: RamService, + session: SessionId, + data: Vec, +} + +impl<'w> Pair<'w> { + /// The service's serving step: drain every queued request, dispatch it, and + /// push each reply. This is what `ServeHost`'s loop does per doorbell. + fn serve_pending(&mut self) -> usize { + let mut served = 0; + while let Some(record) = self.serve_req.pop() { + let response = match decode_req(&record) { + Some(request) => { + let mut window = DataWindow::new(&mut self.data); + self.service + .dispatch(self.session, &request, Some(&mut window)) + } + // A record the service cannot read is refused, never guessed. + None => FsResponse::Refused(FsError::Unsupported), + }; + self.serve_rep + .push(encode_rep(&response, 0)) + .expect("the reply ring has a slot per outstanding request"); + served += 1; + } + served + } + + /// One full round trip through both rings. + fn call(&mut self, request: &FsRequest) -> FsResponse { + self.client_req + .push(encode_req(request)) + .expect("request ring has room"); + self.serve_pending(); + let record = self.client_rep.pop().expect("a reply was produced"); + let (response, grants) = + decode_rep_for(request, &record).expect("the reply decodes against its request"); + assert_eq!(grants, 0, "the filesystem grants no kernel authority"); + response + } + + /// `call`, folding a refusal into `Err` the way the typed client does. + fn checked(&mut self, request: &FsRequest) -> Result { + match self.call(request) { + FsResponse::Refused(err) => Err(err), + other => Ok(other), + } + } + + fn open_root(&mut self) -> kore_svc_dagfs::Handle { + match self.checked(&FsRequest::OpenRoot { + rights: Rights::ALL, + }) { + Ok(FsResponse::Opened { handle, .. }) => handle, + other => panic!("open_root: {other:?}"), + } + } + + fn create( + &mut self, + parent: kore_svc_dagfs::Handle, + text: &str, + kind: ObjectKind, + ) -> kore_svc_dagfs::Handle { + match self.checked(&FsRequest::Create { + parent, + name: name(text), + kind, + subtype: kore_svc_dagfs::ObjectSubtype::PLAIN, + }) { + Ok(FsResponse::Opened { handle, .. }) => handle, + other => panic!("create {text}: {other:?}"), + } + } +} + +/// Build a pair over `window`, with the service attached and a data window +/// declared. +fn pair(window: &mut [u8]) -> Pair<'_> { + let (client, serve) = + split_window::(window, REQ_CAP, REP_CAP).expect("format the pair window"); + let session = SessionId::new(0); + let mut service = RamService::default(); + block_on(service.engine().attach_session(session)).expect("attach session"); + let mut pair = Pair { + client_req: client.req, + client_rep: client.rep, + serve_req: serve.req, + serve_rep: serve.rep, + service, + session, + data: vec![0u8; DATA_BYTES], + }; + pair.checked(&FsRequest::AttachWindow { + slot: 7, + bytes: DATA_BYTES as u32, + }) + .expect("attach the data window"); + pair +} + +// ── geometry ──────────────────────────────────────────────────────────────── + +#[test] +fn the_pair_window_is_the_geometry_the_founder_would_build() { + // Computed by `kore_rt::svc`, not restated here — if the record widths or + // the ring caps move, this moves with them. + let bytes = window_bytes::(REQ_CAP, REP_CAP); + assert_eq!(bytes, 1728); + assert_eq!(rep_offset::(REQ_CAP), 1152); + assert!( + bytes <= 0x2000, + "the pair must fit `[layout] ring_bytes` (3 pages)" + ); + // And a window one byte short must refuse to format rather than overlay + // the reply ring onto the request ring's slots. Sliced off an aligned + // buffer so the refusal is about size, not alignment. + let mut short = RingWindow::new(); + assert_eq!( + split_window::(&mut short.0[..bytes - 1], REQ_CAP, REP_CAP).err(), + Some(kore_ring::spsc::RingError::TooSmall) + ); +} + +#[test] +fn the_rings_carry_the_declared_depth() { + let mut buf = RingWindow::new(); + let p = pair(&mut buf.0); + assert_eq!(p.client_req.capacity(), REQ_CAP); + assert_eq!(p.client_rep.capacity(), REP_CAP); + assert_eq!(p.serve_req.capacity(), REQ_CAP); + assert_eq!(p.serve_rep.capacity(), REP_CAP); +} + +#[test] +fn immutable_extraction_crosses_the_same_production_pair() { + use kore_svc_extract::{ + InputIdentity, NormalizedResult, ParserProvenance, encode_result, + registry::{C_JSON, RUST_JSON}, + }; + use kore_svc_extract_worker_rust::parse_json_rust; + + let mut buf = RingWindow::new(); + let mut p = pair(&mut buf.0); + let root = p.open_root(); + let file = p.create(root, "source.json", ObjectKind::File); + let bytes = br#"{"a":1,"b":[2,3]}"#; + p.data[..bytes.len()].copy_from_slice(bytes); + let revision = match p.checked(&FsRequest::WriteAt { + handle: file, + offset: 0, + window_offset: 0, + length: bytes.len() as u32, + expected: ANY_REVISION, + }) { + Ok(FsResponse::Committed { revision }) => revision, + other => panic!("write: {other:?}"), + }; + let lease = match p.checked(&FsRequest::LendVersion { + handle: file, + window_offset: 0, + }) { + Ok(FsResponse::VersionLent { lease }) => lease, + other => panic!("lend: {other:?}"), + }; + let identity = InputIdentity { + source: u64::from_le_bytes(p.data[0..8].try_into().unwrap()), + version: u64::from_le_bytes(p.data[8..16].try_into().unwrap()), + len: u64::from_le_bytes(p.data[16..24].try_into().unwrap()), + digest: p.data[24..56].try_into().unwrap(), + }; + assert_eq!(identity.version, revision); + p.data.fill(0); + assert_eq!( + p.checked(&FsRequest::ReadVersion { + lease, + offset: 0, + window_offset: 0, + length: bytes.len() as u32, + }), + Ok(FsResponse::Transferred { + count: bytes.len() as u32 + }) + ); + assert_eq!(&p.data[..bytes.len()], bytes); + + let result = parse_json_rust(identity, bytes).unwrap(); + let mut old_result = result.clone(); + old_result.provenance = ParserProvenance::deterministic(identity, RUST_JSON, 1); + p.data.fill(0); + let used = encode_result(&old_result, &mut p.data).unwrap(); + assert_eq!( + p.checked(&FsRequest::CommitDerived { + lease, + window_offset: 0, + length: used as u32, + output_window_bytes: DATA_BYTES as u32, + }), + Ok(FsResponse::DerivedCommitted { + parser: RUST_JSON, + parser_version: 1, + }) + ); + + let c_result = NormalizedResult::empty(ParserProvenance::deterministic(identity, C_JSON, 1)); + p.data.fill(0); + let used = encode_result(&c_result, &mut p.data).unwrap(); + assert_eq!( + p.checked(&FsRequest::CommitDerived { + lease, + window_offset: 0, + length: used as u32, + output_window_bytes: DATA_BYTES as u32, + }), + Ok(FsResponse::DerivedCommitted { + parser: C_JSON, + parser_version: 1, + }) + ); + + p.data.fill(0); + let used = encode_result(&result, &mut p.data).unwrap(); + assert_eq!( + p.checked(&FsRequest::CommitDerived { + lease, + window_offset: 0, + length: used as u32, + output_window_bytes: DATA_BYTES as u32, + }), + Ok(FsResponse::DerivedCommitted { + parser: RUST_JSON, + parser_version: 2, + }) + ); + assert_eq!( + p.checked(&FsRequest::DerivedInfo { + handle: file, + parser: RUST_JSON, + }), + Ok(FsResponse::DerivedInfo { + input_version: revision, + parser_version: 2, + }) + ); + assert_eq!( + p.checked(&FsRequest::DerivedInfo { + handle: file, + parser: C_JSON, + }), + Ok(FsResponse::DerivedInfo { + input_version: revision, + parser_version: 1, + }), + "upgrading one parser lineage preserves another parser's facts" + ); + assert_eq!( + p.checked(&FsRequest::ReleaseVersion { lease }), + Ok(FsResponse::Done) + ); + assert_eq!( + p.checked(&FsRequest::ReadVersion { + lease, + offset: 0, + window_offset: 0, + length: 1, + }), + Err(FsError::InvalidHandle), + "a released generation cannot read a replacement slot" + ); +} + +// ── the workflow, over real rings ─────────────────────────────────────────── + +#[test] +fn the_full_workflow_crosses_real_rings() { + let mut buf = RingWindow::new(); + let mut p = pair(&mut buf.0); + + let root = p.open_root(); + let dir = p.create(root, "demo", ObjectKind::Directory); + let file = p.create(dir, "hello", ObjectKind::File); + + // Write through the data window. + p.data[..4].copy_from_slice(b"kore"); + let revision = match p.checked(&FsRequest::WriteAt { + handle: file, + offset: 0, + window_offset: 0, + length: 4, + expected: ANY_REVISION, + }) { + Ok(FsResponse::Committed { revision }) => revision, + other => panic!("write: {other:?}"), + }; + + // Read it back through the data window. + p.data.fill(0); + match p.checked(&FsRequest::Read { + handle: file, + offset: 0, + window_offset: 0, + length: 4, + }) { + Ok(FsResponse::Transferred { count }) => assert_eq!(count, 4), + other => panic!("read: {other:?}"), + } + assert_eq!(&p.data[..4], b"kore"); + + // A second name for the same object, then a listing. + p.checked(&FsRequest::Link { + parent: dir, + name: name("hello-again"), + child: file, + }) + .unwrap(); + match p.checked(&FsRequest::List { + handle: dir, + cursor: 0, + window_offset: 0, + }) { + Ok(FsResponse::Listed { entries, next }) => { + assert_eq!(entries, 2); + assert_eq!(next, None); + } + other => panic!("list: {other:?}"), + } + + // And the object's own view of itself. + match p.checked(&FsRequest::Stat { handle: file }) { + Ok(FsResponse::Stat { + len, + links, + revision: r, + .. + }) => { + assert_eq!(len, 4); + assert_eq!(links, 2); + assert_eq!(r, revision); + } + other => panic!("stat: {other:?}"), + } +} + +#[test] +fn a_payload_larger_than_one_record_still_crosses_in_the_data_window() { + // The point of the two-window split: the ring record stays 120 bytes no + // matter how much data moves. + let mut buf = RingWindow::new(); + let mut p = pair(&mut buf.0); + let root = p.open_root(); + let file = p.create(root, "big", ObjectKind::File); + + let payload: Vec = (0..400u32).map(|i| (i % 251) as u8).collect(); + p.data[..payload.len()].copy_from_slice(&payload); + p.checked(&FsRequest::WriteAt { + handle: file, + offset: 0, + window_offset: 0, + length: payload.len() as u32, + expected: ANY_REVISION, + }) + .unwrap(); + + p.data.fill(0); + p.checked(&FsRequest::Read { + handle: file, + offset: 0, + window_offset: 0, + length: payload.len() as u32, + }) + .unwrap(); + assert_eq!(&p.data[..payload.len()], &payload[..]); + assert_eq!( + size_of::(), + 120, + "the record did not grow with it" + ); +} + +// ── ring behaviour ────────────────────────────────────────────────────────── + +#[test] +fn the_request_ring_fills_at_its_declared_depth_and_refuses() { + let mut buf = RingWindow::new(); + let mut p = pair(&mut buf.0); + let root = p.open_root(); + + // Queue without serving: the ring must accept exactly `REQ_CAP` and then + // refuse, rather than overwrite an unserved request. + let request = encode_req(&FsRequest::Stat { handle: root }); + let mut queued = 0; + while p.client_req.push(request).is_ok() { + queued += 1; + assert!(queued <= REQ_CAP, "a full ring must refuse, not wrap"); + } + assert_eq!(queued, REQ_CAP); + + // Draining serves every one, in order, and re-arms the ring. + assert_eq!(p.serve_pending(), REQ_CAP as usize); + for _ in 0..REQ_CAP { + let record = p.client_rep.pop().expect("one reply per request"); + assert_eq!(record.status, FsRepWire::STATUS_OK); + } + assert!(p.client_rep.pop().is_none(), "no extra replies"); + assert!(p.client_req.push(request).is_ok(), "the ring re-armed"); +} + +#[test] +fn requests_are_served_in_order_across_the_ring() { + // The SPSC ring is ordered, and the dispatch is sequential — so a batch of + // appends must land in the order they were queued. + let mut buf = RingWindow::new(); + let mut p = pair(&mut buf.0); + let root = p.open_root(); + let log = p.create(root, "log", ObjectKind::File); + + p.data[..1].copy_from_slice(b"x"); + for _ in 0..4 { + p.client_req + .push(encode_req(&FsRequest::Append { + handle: log, + window_offset: 0, + length: 1, + expected: ANY_REVISION, + })) + .unwrap(); + } + assert_eq!(p.serve_pending(), 4); + + let mut offsets = Vec::new(); + for _ in 0..4 { + let record = p.client_rep.pop().unwrap(); + // `len` carries an append's offset. + offsets.push(record.len.get()); + } + assert_eq!(offsets, vec![0, 1, 2, 3], "served in queue order"); +} + +#[test] +fn a_garbage_record_on_the_ring_is_refused_without_disturbing_the_stream() { + let mut buf = RingWindow::new(); + let mut p = pair(&mut buf.0); + let root = p.open_root(); + + // A hostile peer writes a record with an unknown verb straight into the + // ring, followed by a good one. + let mut bogus = encode_req(&FsRequest::Stat { handle: root }); + bogus.op = 200; + p.client_req.push(bogus).unwrap(); + p.client_req + .push(encode_req(&FsRequest::Stat { handle: root })) + .unwrap(); + + assert_eq!(p.serve_pending(), 2); + let first = p.client_rep.pop().unwrap(); + let second = p.client_rep.pop().unwrap(); + assert_ne!( + first.status, + FsRepWire::STATUS_OK, + "garbage must be refused" + ); + assert_eq!( + second.status, + FsRepWire::STATUS_OK, + "and the next request must still be served" + ); +} + +#[test] +fn the_pair_window_holds_only_records_never_payload() { + // The security property the two-window split exists for: a client's file + // bytes must never appear in the ring window, because the ring window is + // shared with the service for control and a payload there would widen what + // crosses on every call. + let mut buf = RingWindow::new(); + let secret = b"SECRETPAYLOAD"; + { + let mut p = pair(&mut buf.0); + let root = p.open_root(); + let file = p.create(root, "f", ObjectKind::File); + p.data[..secret.len()].copy_from_slice(secret); + p.checked(&FsRequest::WriteAt { + handle: file, + offset: 0, + window_offset: 0, + length: secret.len() as u32, + expected: ANY_REVISION, + }) + .unwrap(); + p.data.fill(0); + p.checked(&FsRequest::Read { + handle: file, + offset: 0, + window_offset: 0, + length: secret.len() as u32, + }) + .unwrap(); + assert_eq!(&p.data[..secret.len()], secret, "the bytes did move"); + } + // The whole ring window, scanned for the payload. + assert!( + !buf.0.windows(secret.len()).any(|w| w == secret), + "file bytes must never appear in the pair's ring window" + ); +} + +// ── refusals over the real transport ──────────────────────────────────────── + +#[test] +fn refusals_cross_the_rings_intact() { + let mut buf = RingWindow::new(); + let mut p = pair(&mut buf.0); + let root = p.open_root(); + + assert_eq!( + p.checked(&FsRequest::OpenAt { + parent: root, + name: name("absent"), + rights: Rights::ALL, + }) + .unwrap_err(), + FsError::NotFound + ); + let file = p.create(root, "f", ObjectKind::File); + assert_eq!( + p.checked(&FsRequest::Read { + handle: root, + offset: 0, + window_offset: 0, + length: 4, + }) + .unwrap_err(), + FsError::IsDirectory + ); + // A window-relative range the service will not honour. + assert_eq!( + p.checked(&FsRequest::Read { + handle: file, + offset: 0, + window_offset: DATA_BYTES as u32, + length: 8, + }) + .unwrap_err(), + FsError::Range + ); +} + +#[test] +fn losing_a_route_reaches_the_client_over_the_rings() { + let mut buf = RingWindow::new(); + let mut p = pair(&mut buf.0); + let root = p.open_root(); + let dir = p.create(root, "d", ObjectKind::Directory); + let file = p.create(dir, "f", ObjectKind::File); + + p.checked(&FsRequest::Unlink { + parent: dir, + name: name("f"), + }) + .unwrap(); + assert_eq!( + p.checked(&FsRequest::Stat { handle: file }).unwrap_err(), + FsError::AccessLost + ); +} + +// ── the service contract itself ───────────────────────────────────────────── + +#[test] +fn the_service_declares_an_empty_endowment() { + use kore_rt::service::{Bundle, SystemService}; + // The filesystem claims nothing at startup, and the founder reads this. + assert!(::Endowment::PROFILE.is_empty()); + assert_eq!(::NAME, "fs"); +} + +#[test] +fn the_protocol_declares_no_reply_grants() { + use kore_rt::svc::SvcProtocol; + assert_eq!(::REPLY_GRANTS, 0); +} + +#[test] +fn the_protocols_reply_matching_rejects_a_mismatched_answer() { + use kore_rt::svc::SvcProtocol; + let stat = encode_req(&FsRequest::Stat { + handle: kore_svc_dagfs::Handle::from_bits(1), + }); + // `Done` is not an answer to a `Stat`. + assert!(!::reply_matches( + &stat, + &FsResponse::Done, + 0 + )); + // A refusal answers anything. + assert!(::reply_matches( + &stat, + &FsResponse::Refused(FsError::NotFound), + 0 + )); + // And a grant-bearing reply is refused outright. + assert!(!::reply_matches( + &stat, + &FsResponse::Refused(FsError::NotFound), + 1 + )); +} diff --git a/native/services/dagfs/tests/protocol.rs b/native/services/dagfs/tests/protocol.rs new file mode 100644 index 0000000..357f83f --- /dev/null +++ b/native/services/dagfs/tests/protocol.rs @@ -0,0 +1,990 @@ +//! The plan's §10.2 protocol tests: every semantic variant round-trips through +//! its frozen wire form, and every malformed shape is refused. +//! +//! These test the codec as an adversary would reach it — by writing bytes into +//! a record and asking the service to interpret them. A decode that accepted +//! any of the refusals below would be a hole in the only thing standing between +//! the engine and another process. + +mod common; + +use common::name; +use kore_abi::session::SlotRef; +use kore_abi::svc::{FS_NAME_BYTES, FsRepWire, FsReqWire}; +use kore_rt::svc::{PeerBundlePart, PeerId, SvcProtocol}; +use kore_svc_dagfs::{ + ANY_REVISION, AuthorityQuota, FsError, FsRequest, FsResponse, GrantId, Handle, ObjectId, + ObjectKind, ProviderError, RamService, Rights, SessionId, VersionLease, + wire::{decode_rep_for, decode_req, encode_rep, encode_req, reply_matches}, +}; +use zerocopy::IntoBytes; + +/// One of every request variant. +fn every_request() -> Vec { + let h = Handle::from_bits(0x0000_0003_0000_0007); + let h2 = Handle::from_bits(0x0000_0009_0000_0002); + let g = GrantId::from_bits(0x0000_0005_0000_0001); + let peer = PeerId::from_arrival(0x101).expect("valid peer generation"); + vec![ + FsRequest::AttachWindow { + slot: 9, + bytes: 4096, + }, + FsRequest::OpenRoot { + rights: Rights::ALL, + }, + FsRequest::OpenAt { + parent: h, + name: name("child"), + rights: Rights::READ, + }, + FsRequest::OpenGrant { + grant: g, + rights: Rights::READ_ONLY, + }, + FsRequest::Close { handle: h }, + FsRequest::Stat { handle: h }, + FsRequest::List { + handle: h, + cursor: 12, + window_offset: 0, + }, + FsRequest::Parents { + handle: h, + cursor: 3, + window_offset: 512, + }, + FsRequest::Create { + parent: h, + name: name("made"), + kind: ObjectKind::File, + subtype: kore_svc_dagfs::ObjectSubtype::PLAIN, + }, + FsRequest::Create { + parent: h, + name: name("dir"), + kind: ObjectKind::Directory, + subtype: kore_svc_dagfs::ObjectSubtype::PLAIN, + }, + FsRequest::Read { + handle: h, + offset: 4096, + window_offset: 64, + length: 512, + }, + FsRequest::WriteAt { + handle: h, + offset: 8, + window_offset: 128, + length: 256, + expected: 42, + }, + FsRequest::Append { + handle: h, + window_offset: 32, + length: 16, + expected: ANY_REVISION, + }, + FsRequest::Resize { + handle: h, + length: 1024, + expected: 7, + }, + FsRequest::Rename { + parent: h, + old: name("before"), + new: name("after"), + }, + FsRequest::Relocate { + src_parent: h, + name: name("moving"), + dst_parent: h2, + new_name: name("moved"), + }, + FsRequest::Link { + parent: h, + name: name("second"), + child: h2, + }, + FsRequest::Unlink { + parent: h, + name: name("gone"), + }, + FsRequest::DetachEdge { + parent: h, + name: name("detached"), + }, + FsRequest::Grant { + handle: h, + destination: SessionId::new(2), + rights: Rights::READ, + display_name: Some(name("inbox")), + }, + FsRequest::Grant { + handle: h, + destination: SessionId::new(1), + rights: Rights::READ.union(Rights::WRITE), + display_name: None, + }, + FsRequest::GrantScoped { + handle: h, + destination: SessionId::new(2), + rights: Rights::READ.union(Rights::WRITE), + quota: AuthorityQuota::new(8, 12, 4096, 4), + display_name: Some(name("data")), + }, + FsRequest::Revoke { grant: g }, + FsRequest::Sync { handle: h }, + FsRequest::LendVersion { + handle: h, + window_offset: 64, + }, + FsRequest::LendExecutable { + handle: h, + window_offset: 64, + }, + FsRequest::ReadVersion { + lease: VersionLease::from_bits(0x0000_0002_0000_0003), + offset: 8, + window_offset: 96, + length: 128, + }, + FsRequest::CommitDerived { + lease: VersionLease::from_bits(0x0000_0002_0000_0003), + window_offset: 32, + length: 256, + output_window_bytes: 4096, + }, + FsRequest::ReleaseVersion { + lease: VersionLease::from_bits(0x0000_0002_0000_0003), + }, + FsRequest::DerivedInfo { + handle: h, + parser: kore_svc_extract::ParserId(1), + }, + FsRequest::RegisterSystemView { + rights: Rights::READ_ONLY, + announced: SlotRef::from_wire(34), + }, + FsRequest::ApplyView { + peer, + rights: Rights::READ_ONLY, + announced: SlotRef::from_wire(35), + }, + FsRequest::BindView { + handle: h, + rights: Rights::READ_ONLY, + announced: SlotRef::from_wire(36), + }, + FsRequest::MatchView { + handle: h, + announced: SlotRef::from_wire(36), + }, + FsRequest::AdmitPeer { + part: PeerBundlePart::Ring, + announced: SlotRef::from_wire(37), + }, + FsRequest::AdmitPeer { + part: PeerBundlePart::Kick, + announced: SlotRef::from_wire(38), + }, + FsRequest::AdmitPeer { + part: PeerBundlePart::ReturnRecv, + announced: SlotRef::from_wire(39), + }, + FsRequest::AdmitPeer { + part: PeerBundlePart::Grant(2), + announced: SlotRef::from_wire(40), + }, + FsRequest::TakeAdmission, + FsRequest::RetirePeer { peer }, + FsRequest::ConfirmRetired { peer }, + FsRequest::ClosePeer, + ] +} + +#[test] +fn async_protocol_decode_uses_the_outstanding_request() { + let request = FsRequest::OpenRoot { + rights: Rights::READ, + }; + let request_wire = encode_req(&request); + let response = FsResponse::Opened { + handle: Handle::from_bits(0x0000_0001_0000_0002), + stable_object: kore_svc_dagfs::StableObjectId::new(41).unwrap(), + kind: ObjectKind::Directory, + subtype: kore_svc_dagfs::ObjectSubtype::PLAIN, + len: 0, + revision: 1, + }; + let response_wire = encode_rep(&response, 0); + + assert!( + ::decode_rep(&response_wire).is_none(), + "an Opened record is intentionally ambiguous without its request" + ); + assert_eq!( + ::decode_rep_for(&request_wire, &response_wire), + Some((response, 0)), + "the async metal client must decode against its outstanding request" + ); +} + +#[test] +fn every_request_variant_round_trips() { + for req in every_request() { + let wire = encode_req(&req); + let back = decode_req(&wire).unwrap_or_else(|| panic!("{req:?} must decode")); + assert_eq!(back, req, "{req:?} must survive the wire unchanged"); + } +} + +#[test] +fn system_view_registration_requires_an_explicit_valid_attenuation() { + let request = FsRequest::RegisterSystemView { + rights: Rights::READ_ONLY, + announced: SlotRef::from_wire(34), + }; + let wire = encode_req(&request); + assert_eq!(decode_req(&wire), Some(request)); + + let mut empty = wire; + empty.rights = 0; + assert!(decode_req(&empty).is_none()); + + let mut unknown = wire; + unknown.rights = 1 << 7; + assert!(decode_req(&unknown).is_none()); +} + +#[test] +fn every_response_variant_round_trips_against_its_request() { + let h = Handle::from_bits(0x0000_0003_0000_0007); + let g = GrantId::from_bits(0x0000_0005_0000_0001); + let peer = PeerId::from_arrival(0x101).expect("valid peer generation"); + let cases: Vec<(FsRequest, FsResponse)> = vec![ + ( + FsRequest::OpenRoot { + rights: Rights::ALL, + }, + FsResponse::Opened { + handle: h, + stable_object: kore_svc_dagfs::StableObjectId::new(41).unwrap(), + kind: ObjectKind::Directory, + subtype: kore_svc_dagfs::ObjectSubtype::PLAIN, + len: 0, + revision: 1, + }, + ), + ( + FsRequest::Stat { handle: h }, + FsResponse::Stat { + object: ObjectId::from_bits(0x0000_0002_0000_0004), + stable_object: kore_svc_dagfs::StableObjectId::new(42).unwrap(), + kind: ObjectKind::File, + subtype: kore_svc_dagfs::ObjectSubtype::PLAIN, + len: 900, + revision: 5, + links: 2, + grants: 1, + }, + ), + ( + FsRequest::List { + handle: h, + cursor: 0, + window_offset: 0, + }, + FsResponse::Listed { + entries: 8, + next: Some(9), + }, + ), + ( + FsRequest::List { + handle: h, + cursor: 9, + window_offset: 0, + }, + FsResponse::Listed { + entries: 3, + next: None, + }, + ), + ( + FsRequest::Read { + handle: h, + offset: 0, + window_offset: 0, + length: 64, + }, + FsResponse::Transferred { count: 64 }, + ), + ( + FsRequest::WriteAt { + handle: h, + offset: 0, + window_offset: 0, + length: 4, + expected: ANY_REVISION, + }, + FsResponse::Committed { revision: 9 }, + ), + ( + FsRequest::LendVersion { + handle: h, + window_offset: 0, + }, + FsResponse::VersionLent { + lease: VersionLease::from_bits(0x0000_0002_0000_0003), + }, + ), + ( + FsRequest::LendExecutable { + handle: h, + window_offset: 0, + }, + FsResponse::VersionLent { + lease: VersionLease::from_bits(0x0000_0002_0000_0003), + }, + ), + ( + FsRequest::ReadVersion { + lease: VersionLease::from_bits(0x0000_0002_0000_0003), + offset: 0, + window_offset: 0, + length: 64, + }, + FsResponse::Transferred { count: 64 }, + ), + ( + FsRequest::CommitDerived { + lease: VersionLease::from_bits(0x0000_0002_0000_0003), + window_offset: 0, + length: 128, + output_window_bytes: 4096, + }, + FsResponse::DerivedCommitted { + parser: kore_svc_extract::ParserId(1), + parser_version: 3, + }, + ), + ( + FsRequest::DerivedInfo { + handle: h, + parser: kore_svc_extract::ParserId(1), + }, + FsResponse::DerivedInfo { + input_version: 9, + parser_version: 3, + }, + ), + ( + FsRequest::Append { + handle: h, + window_offset: 0, + length: 4, + expected: ANY_REVISION, + }, + FsResponse::Appended { + offset: 16, + revision: 10, + }, + ), + ( + FsRequest::Grant { + handle: h, + destination: SessionId::new(1), + rights: Rights::READ, + display_name: None, + }, + FsResponse::Granted { grant: g }, + ), + ( + FsRequest::GrantScoped { + handle: h, + destination: SessionId::new(1), + rights: Rights::READ, + quota: AuthorityQuota::new(4, 4, 512, 2), + display_name: Some(name("app")), + }, + FsResponse::Granted { grant: g }, + ), + (FsRequest::Close { handle: h }, FsResponse::Done), + ( + FsRequest::AdmitPeer { + part: PeerBundlePart::Ring, + announced: SlotRef::from_wire(37), + }, + FsResponse::Done, + ), + (FsRequest::TakeAdmission, FsResponse::PeerAdmitted { peer }), + (FsRequest::RetirePeer { peer }, FsResponse::Done), + (FsRequest::ConfirmRetired { peer }, FsResponse::Done), + (FsRequest::ClosePeer, FsResponse::Done), + ]; + + for (req, rep) in cases { + let wire = encode_rep(&rep, 0); + let (back, grants) = + decode_rep_for(&req, &wire).unwrap_or_else(|| panic!("{rep:?} must decode")); + assert_eq!(back, rep, "{rep:?} must survive the wire unchanged"); + assert_eq!(grants, 0, "the filesystem grants no kernel authority"); + assert!(reply_matches(&req, &back)); + } +} + +#[test] +fn every_refusal_class_round_trips() { + // The engine's whole refusal vocabulary must cross intact — a class that + // collapsed into another would make a client's error handling wrong. + let errors = [ + FsError::NotFound, + FsError::AlreadyExists, + FsError::NotDirectory, + FsError::IsDirectory, + FsError::InvalidName, + FsError::InvalidHandle, + FsError::AccessDenied, + FsError::AccessLost, + FsError::Conflict, + FsError::Cycle, + FsError::NotEmpty, + FsError::OutOfObjects, + FsError::OutOfEdges, + FsError::OutOfHandles, + FsError::OutOfStorage, + FsError::Range, + FsError::BudgetExceeded, + FsError::Unsupported, + FsError::NoWindow, + FsError::Busy, + FsError::IndexIncomplete, + FsError::ProviderFailure(ProviderError::Media), + ]; + let req = FsRequest::Stat { + handle: Handle::from_bits(1), + }; + for err in errors { + let wire = encode_rep(&FsResponse::Refused(err), 0); + let (back, _) = decode_rep_for(&req, &wire).expect("a refusal must decode"); + assert_eq!(back, FsResponse::Refused(err), "{err:?} must cross intact"); + } + // Distinct classes must not share a status byte, or two different refusals + // would be indistinguishable to a client. + let mut statuses: Vec = errors + .iter() + .map(|err| encode_rep(&FsResponse::Refused(*err), 0).status) + .collect(); + statuses.sort(); + let count = statuses.len(); + statuses.dedup(); + assert_eq!( + statuses.len(), + count, + "every refusal class needs its own status" + ); +} + +#[test] +fn access_denied_and_access_lost_are_distinct_on_the_wire() { + // The distinction is the whole of "immediate loss of access"; collapsing it + // would make plan §2.3 untestable from a client. + let denied = encode_rep(&FsResponse::Refused(FsError::AccessDenied), 0); + let lost = encode_rep(&FsResponse::Refused(FsError::AccessLost), 0); + assert_ne!(denied.status, lost.status); +} + +// ── malformed requests ────────────────────────────────────────────────────── + +/// A valid `Stat` record to mutate into invalid shapes. +fn valid_req() -> FsReqWire { + encode_req(&FsRequest::Stat { + handle: Handle::from_bits(0x0000_0003_0000_0007), + }) +} + +#[test] +fn an_unseeded_or_unknown_op_is_refused() { + let mut w = valid_req(); + w.op = 0; + assert!( + decode_req(&w).is_none(), + "a zero op is unseeded, not a verb" + ); + for bad in [FsReqWire::OP_MAX + 1, 100, 255] { + let mut w = valid_req(); + w.op = bad; + assert!(decode_req(&w).is_none(), "op {bad} must be refused"); + } +} + +#[test] +fn a_non_zero_reserved_field_is_refused() { + for mutate in [ + (|w: &mut FsReqWire| w.reserved0 = 1) as fn(&mut FsReqWire), + |w: &mut FsReqWire| w.reserved1 = 1.into(), + |w: &mut FsReqWire| w.reserved2 = 0xffff_ffff.into(), + ] { + let mut w = valid_req(); + mutate(&mut w); + assert!( + decode_req(&w).is_none(), + "a reserve exists to be zero; a set one is a peer we do not understand" + ); + } +} + +#[test] +fn an_undefined_rights_bit_is_refused_not_masked() { + // Masking would answer the request as though the unknown right had not + // been asked for — fail-open on the one field that bounds authority. + let mut w = encode_req(&FsRequest::OpenRoot { + rights: Rights::ALL, + }); + w.rights = 0xff; + assert!(decode_req(&w).is_none()); + + // And a verb that takes no rights must refuse a non-zero mask rather than + // ignore it. + let mut w = valid_req(); + w.rights = Rights::READ.bits(); + assert!(decode_req(&w).is_none()); +} + +#[test] +fn a_name_length_past_the_field_is_refused() { + let mut w = encode_req(&FsRequest::Unlink { + parent: Handle::from_bits(1), + name: name("x"), + }); + w.name_len = (FS_NAME_BYTES + 1) as u8; + assert!(decode_req(&w).is_none()); + w.name_len = 255; + assert!(decode_req(&w).is_none()); +} + +#[test] +fn junk_in_a_names_unused_tail_is_refused() { + let mut w = encode_req(&FsRequest::Unlink { + parent: Handle::from_bits(1), + name: name("short"), + }); + // The tail past `name_len` is padding this service must not interpret. + w.name[FS_NAME_BYTES - 1] = 0x41; + assert!( + decode_req(&w).is_none(), + "a peer writing in the padding is a peer whose framing differs" + ); +} + +#[test] +fn an_invalid_name_is_refused_by_the_codec() { + let mut w = encode_req(&FsRequest::Unlink { + parent: Handle::from_bits(1), + name: name("ok"), + }); + // `..` is reserved, and a slash can never be a component. + w.name[..2].copy_from_slice(b".."); + w.name_len = 2; + assert!(decode_req(&w).is_none()); + + let mut w = encode_req(&FsRequest::Unlink { + parent: Handle::from_bits(1), + name: name("ok"), + }); + w.name[..3].copy_from_slice(b"a/b"); + w.name_len = 3; + assert!(decode_req(&w).is_none()); + + // A zero-length name is invalid for a verb that needs one. + let mut w = encode_req(&FsRequest::Unlink { + parent: Handle::from_bits(1), + name: name("ok"), + }); + w.name = [0; FS_NAME_BYTES]; + w.name_len = 0; + assert!(decode_req(&w).is_none()); +} + +#[test] +fn an_invalid_or_misplaced_kind_is_refused() { + let mut w = encode_req(&FsRequest::Create { + parent: Handle::from_bits(1), + name: name("x"), + kind: ObjectKind::File, + subtype: kore_svc_dagfs::ObjectSubtype::PLAIN, + }); + for bad in [0u8, 3, 255] { + w.kind = bad; + assert!(decode_req(&w).is_none(), "kind {bad} must be refused"); + } + // A verb that creates nothing must not carry a kind. + let mut w = valid_req(); + w.kind = FsReqWire::KIND_FILE; + assert!(decode_req(&w).is_none()); +} + +#[test] +fn peer_control_fields_are_exact_and_fail_closed() { + let request = FsRequest::AdmitPeer { + part: PeerBundlePart::ReturnRecv, + announced: SlotRef::from_wire(37), + }; + let wire = encode_req(&request); + assert_eq!(decode_req(&wire), Some(request)); + + for mutate in [ + (|w: &mut FsReqWire| w.handle = 1.into()) as fn(&mut FsReqWire), + |w: &mut FsReqWire| w.length = 1.into(), + |w: &mut FsReqWire| w.name_len = 1, + |w: &mut FsReqWire| w.name[0] = b'x', + ] { + let mut malformed = wire; + mutate(&mut malformed); + assert!(decode_req(&malformed).is_none()); + } + + let mut close = encode_req(&FsRequest::ClosePeer); + close.window_offset = 1.into(); + assert!(decode_req(&close).is_none()); + let mut close = encode_req(&FsRequest::ClosePeer); + close.kind = FsReqWire::KIND_PEER_RING; + assert!(decode_req(&close).is_none()); +} + +#[test] +fn a_session_number_outside_grant_is_refused() { + // Only `Grant` routes to a session; elsewhere the field is a claim the + // caller has no business making. + let mut w = valid_req(); + w.session = 1.into(); + assert!(decode_req(&w).is_none()); +} + +#[test] +fn a_grant_may_omit_its_display_name_but_not_smuggle_one() { + let req = FsRequest::Grant { + handle: Handle::from_bits(1), + destination: SessionId::new(1), + rights: Rights::READ, + display_name: None, + }; + let w = encode_req(&req); + assert_eq!(decode_req(&w), Some(req), "an unnamed grant is legal"); + + // Zero length but non-zero bytes is a contradiction. + let mut w = encode_req(&req); + w.name[0] = b'x'; + assert!(decode_req(&w).is_none()); +} + +#[test] +fn a_request_reads_from_arbitrary_bytes_without_panicking() { + // The record is `FromBytes`, so any 120 bytes are a *layout*-valid record; + // it is the codec that must refuse them. This is the fuzz-shaped case: + // the decode must terminate with a verdict, never panic. + use zerocopy::FromBytes; + for fill in [0x00u8, 0x01, 0x41, 0x7f, 0x80, 0xa5, 0xff] { + let raw = [fill; core::mem::size_of::()]; + let w = FsReqWire::read_from_bytes(&raw).expect("layout admits any bytes"); + let _ = decode_req(&w); + } +} + +// ── malformed replies ─────────────────────────────────────────────────────── + +fn stat_request() -> FsRequest { + FsRequest::Stat { + handle: Handle::from_bits(0x0000_0003_0000_0007), + } +} + +#[test] +fn an_unseeded_or_unknown_status_is_refused() { + let req = stat_request(); + let mut w = encode_rep(&FsResponse::Done, 0); + w.status = 0; + assert!(decode_rep_for(&req, &w).is_none()); + for bad in [FsRepWire::STATUS_MAX + 1, 200, 255] { + let mut w = encode_rep(&FsResponse::Done, 0); + w.status = bad; + assert!(decode_rep_for(&req, &w).is_none(), "status {bad} refused"); + } +} + +#[test] +fn a_grant_bearing_reply_is_refused() { + // Nothing this service answers can carry kernel authority. + let req = stat_request(); + let mut w = encode_rep(&FsResponse::Done, 0); + w.grants = 1; + assert!(decode_rep_for(&req, &w).is_none()); +} + +#[test] +fn a_reply_with_a_set_reserve_is_refused() { + let req = stat_request(); + let mut w = encode_rep(&FsResponse::Done, 0); + w.reserved0 = 1.into(); + assert!(decode_rep_for(&req, &w).is_none()); + let mut w = encode_rep(&FsResponse::Done, 0); + w.reserved1 = 1.into(); + assert!(decode_rep_for(&req, &w).is_none()); +} + +#[test] +fn a_refusal_carrying_payload_is_refused() { + // A refusal is a status and nothing else; payload alongside it is a reply + // this service did not write. + let req = stat_request(); + for mutate in [ + (|w: &mut FsRepWire| w.handle = 7.into()) as fn(&mut FsRepWire), + |w: &mut FsRepWire| w.len = 7.into(), + |w: &mut FsRepWire| w.revision = 7.into(), + |w: &mut FsRepWire| w.count = 7.into(), + |w: &mut FsRepWire| w.cursor = 7.into(), + |w: &mut FsRepWire| w.kind = 1, + |w: &mut FsRepWire| w.flags = 1, + ] { + let mut w = encode_rep(&FsResponse::Refused(FsError::NotFound), 0); + mutate(&mut w); + assert!(decode_rep_for(&req, &w).is_none()); + } +} + +#[test] +fn an_undefined_reply_flag_is_refused() { + let req = FsRequest::List { + handle: Handle::from_bits(1), + cursor: 0, + window_offset: 0, + }; + let mut w = encode_rep( + &FsResponse::Listed { + entries: 1, + next: None, + }, + 0, + ); + w.flags = 0x80; + assert!(decode_rep_for(&req, &w).is_none()); +} + +#[test] +fn only_a_listing_may_set_flags() { + let req = stat_request(); + let mut w = encode_rep( + &FsResponse::Stat { + object: ObjectId::from_bits(4), + stable_object: kore_svc_dagfs::StableObjectId::new(42).unwrap(), + kind: ObjectKind::File, + subtype: kore_svc_dagfs::ObjectSubtype::PLAIN, + len: 0, + revision: 1, + links: 1, + grants: 0, + }, + 0, + ); + w.flags = FsRepWire::FLAG_MORE; + assert!(decode_rep_for(&req, &w).is_none()); +} + +#[test] +fn a_complete_listing_must_not_carry_a_resume_cursor() { + // A stale cursor with no MORE flag is a resume point a client might trust. + let req = FsRequest::List { + handle: Handle::from_bits(1), + cursor: 0, + window_offset: 0, + }; + let mut w = encode_rep( + &FsResponse::Listed { + entries: 2, + next: None, + }, + 0, + ); + w.cursor = 5.into(); + assert!(decode_rep_for(&req, &w).is_none()); +} + +#[test] +fn an_invalid_kind_in_an_open_reply_is_refused() { + let req = FsRequest::OpenRoot { + rights: Rights::ALL, + }; + let mut w = encode_rep( + &FsResponse::Opened { + handle: Handle::from_bits(1), + stable_object: kore_svc_dagfs::StableObjectId::new(41).unwrap(), + kind: ObjectKind::File, + subtype: kore_svc_dagfs::ObjectSubtype::PLAIN, + len: 0, + revision: 1, + }, + 0, + ); + for bad in [0u8, 3, 255] { + w.kind = bad; + assert!(decode_rep_for(&req, &w).is_none(), "kind {bad} refused"); + } +} + +#[test] +fn a_reply_is_decoded_against_the_request_it_answers() { + // One fixed record serves every verb, so `count` is a byte count for a + // Read and a link count for a Stat. Stat now also carries a stable object + // ID in a request-specific word, so replaying those bytes as a Read is + // rejected rather than silently reinterpreted. + let wire = encode_rep( + &FsResponse::Stat { + object: ObjectId::from_bits(0x0000_0002_0000_0004), + stable_object: kore_svc_dagfs::StableObjectId::new(42).unwrap(), + kind: ObjectKind::File, + subtype: kore_svc_dagfs::ObjectSubtype::PLAIN, + len: 900, + revision: 5, + links: 3, + grants: 0, + }, + 0, + ); + let as_stat = decode_rep_for(&stat_request(), &wire).unwrap().0; + assert!(matches!(as_stat, FsResponse::Stat { links: 3, .. })); + + let read = FsRequest::Read { + handle: Handle::from_bits(1), + offset: 0, + window_offset: 0, + length: 8, + }; + assert!(decode_rep_for(&read, &wire).is_none()); +} + +#[test] +fn reply_matches_rejects_a_response_for_a_different_verb() { + let stat = stat_request(); + let done = FsResponse::Done; + assert!( + !reply_matches(&stat, &done), + "a Stat is not answered by Done" + ); + + let close = FsRequest::Close { + handle: Handle::from_bits(1), + }; + assert!(reply_matches(&close, &done)); + assert!( + !reply_matches(&close, &FsResponse::Transferred { count: 0 }), + "a Close is not answered by a transfer count" + ); + // Any request may be refused. + assert!(reply_matches( + &stat, + &FsResponse::Refused(FsError::NotFound) + )); +} + +#[test] +fn a_reply_reads_from_arbitrary_bytes_without_panicking() { + use zerocopy::FromBytes; + let req = stat_request(); + for fill in [0x00u8, 0x01, 0x41, 0xa5, 0xff] { + let raw = [fill; core::mem::size_of::()]; + let w = FsRepWire::read_from_bytes(&raw).expect("layout admits any bytes"); + let _ = decode_rep_for(&req, &w); + } +} + +// ── framing ───────────────────────────────────────────────────────────────── + +#[test] +fn the_records_are_the_frozen_sizes_and_fit_the_pair_window() { + assert_eq!(core::mem::size_of::(), 120); + assert_eq!(core::mem::size_of::(), 48); + + // `align64(192 + 8·rec)` per ring, both inside `[layout] ring_bytes` + // (0x2000). Recomputed here from the live sizes so a record that grew is + // caught by a test as well as by the ABI's const-assert. + let align64 = |raw: usize| (raw + 63) & !63; + let req_ring = align64(192 + 8 * core::mem::size_of::()); + let rep_ring = align64(192 + 8 * core::mem::size_of::()); + assert_eq!(req_ring, 1152); + assert_eq!(rep_ring, 576); + assert!( + req_ring + rep_ring <= 0x2000, + "both 8-deep rings must fit the pair's window" + ); +} + +#[test] +fn an_encoded_request_leaves_every_unused_field_zero() { + // The encoder starts from a zeroed record, so an unused field is zero + // rather than stale — which is what makes the decoder's strict zero checks + // safe to apply. + let w = encode_req(&FsRequest::Close { + handle: Handle::from_bits(0x0000_0001_0000_0002), + }); + assert_eq!(w.reserved0, 0); + assert_eq!(w.reserved1.get(), 0); + assert_eq!(w.reserved2.get(), 0); + assert_eq!(w.kind, 0); + assert_eq!(w.rights, 0); + assert_eq!(w.session.get(), 0); + assert_eq!(w.name, [0u8; FS_NAME_BYTES]); + assert_eq!(w.name2, [0u8; FS_NAME_BYTES]); + // And the record is plain bytes end to end — no padding surprises. + assert_eq!(w.as_bytes().len(), 120); +} + +#[test] +fn parents_carry_lists_operands_and_refuse_any_other() { + let request = FsRequest::Parents { + handle: Handle::from_bits(7), + cursor: 2, + window_offset: 64, + }; + let wire = encode_req(&request); + assert_eq!(wire.op, FsReqWire::OP_PARENTS); + assert_eq!(decode_req(&wire), Some(request)); + // An operand the verb does not read is refused, not ignored. + for mutate in [ + (|w: &mut FsReqWire| w.handle2 = 1.into()) as fn(&mut FsReqWire), + |w| w.offset = 1.into(), + |w| w.expected_revision = 1.into(), + |w| { + w.name_len = 1; + w.name[0] = b'x'; + }, + ] { + let mut bad = wire; + mutate(&mut bad); + assert!(decode_req(&bad).is_none()); + } + // The reply is a listing's: MORE with a cursor, and no stale cursor. + let more = encode_rep( + &FsResponse::Listed { + entries: 8, + next: Some(10), + }, + 0, + ); + assert_eq!( + decode_rep_for(&request, &more).map(|(r, _)| r), + Some(FsResponse::Listed { + entries: 8, + next: Some(10), + }) + ); + let mut stale = encode_rep( + &FsResponse::Listed { + entries: 1, + next: None, + }, + 0, + ); + stale.cursor = 4.into(); + assert!(decode_rep_for(&request, &stale).is_none()); +} diff --git a/native/services/dagfs/tests/provider.rs b/native/services/dagfs/tests/provider.rs new file mode 100644 index 0000000..35d9d2a --- /dev/null +++ b/native/services/dagfs/tests/provider.rs @@ -0,0 +1,391 @@ +//! Provider bounds, failure propagation, and placement isolation. +//! +//! Covers the provider rows of the plan's §10.1 matrix, and doubles as the +//! start of the conformance evidence Phase 5 needs: everything here is stated +//! in terms of the engine's client-visible behaviour and the provider's +//! *declared* limits, never in terms of RAM specifics, so the same +//! expectations should hold over a block-backed provider. + +mod common; + +use common::{create_dir, create_file, create_file_closed, engine_with_one, name, read_all}; +use kore_svc_dagfs::{ + ANY_REVISION, FsError, ObjectKind, ProviderError, Rights, StorageProvider, block_on, + provider::Fault, +}; + +// ── declared limits are the contract ──────────────────────────────────────── + +#[test] +fn the_engine_honours_the_providers_declared_object_ceiling() { + let (mut engine, session, root) = engine_with_one(); + let ceiling = engine.provider().limits().max_object_bytes; + let file = create_file(&mut engine, session, root, "f", b""); + + // Exactly at the ceiling is allowed… + assert!(block_on(engine.resize(session, file, ceiling, ANY_REVISION)).is_ok()); + assert_eq!(engine.stat(session, file).unwrap().len, ceiling); + + // …one byte past it is a typed capacity refusal, and changes nothing. + assert_eq!( + block_on(engine.resize(session, file, ceiling + 1, ANY_REVISION)).unwrap_err(), + FsError::OutOfStorage + ); + assert_eq!(engine.stat(session, file).unwrap().len, ceiling); +} + +#[test] +fn a_write_that_would_exceed_the_object_ceiling_is_refused() { + let (mut engine, session, root) = engine_with_one(); + let ceiling = engine.provider().limits().max_object_bytes; + let file = create_file(&mut engine, session, root, "f", b""); + + assert_eq!( + block_on(engine.write_at(session, file, ceiling, b"over", ANY_REVISION)).unwrap_err(), + FsError::OutOfStorage + ); + assert_eq!(engine.stat(session, file).unwrap().len, 0); +} + +#[test] +fn a_transfer_larger_than_the_declared_maximum_is_refused() { + let (mut engine, session, root) = engine_with_one(); + let max = engine.provider().limits().max_transfer; + let file = create_file(&mut engine, session, root, "f", b""); + + let oversized = vec![0u8; max + 1]; + assert_eq!( + block_on(engine.write_at(session, file, 0, &oversized, ANY_REVISION)).unwrap_err(), + FsError::Range + ); + assert_eq!(engine.stat(session, file).unwrap().len, 0); +} + +#[test] +fn capacity_is_charged_at_the_declared_granularity() { + let (mut engine, session, root) = engine_with_one(); + let granularity = engine.provider().limits().granularity as u64; + let file = create_file(&mut engine, session, root, "f", b""); + assert_eq!(engine.provider().bytes_in_use(), 0); + + // One byte costs one unit of granularity, not one byte. + block_on(engine.write_at(session, file, 0, b"x", ANY_REVISION)).unwrap(); + assert_eq!(engine.provider().bytes_in_use(), granularity); + // And the file's *visible* length is still one byte. + assert_eq!(engine.stat(session, file).unwrap().len, 1); +} + +// ── exhaustion ────────────────────────────────────────────────────────────── + +#[test] +fn filling_the_pool_refuses_further_bytes_without_damaging_the_namespace() { + let (mut engine, session, root) = engine_with_one(); + let ceiling = engine.provider().limits().max_object_bytes; + let payload = vec![0xa5u8; ceiling as usize]; + + // Fill the pool with maximum-size files. + let mut filled = 0usize; + loop { + let file_name = format!("big{filled}"); + let made = + block_on(engine.create(session, root, &name(&file_name), ObjectKind::File)).unwrap(); + match block_on(engine.write_at(session, made.handle, 0, &payload, ANY_REVISION)) { + Ok(_) => { + engine.close(session, made.handle).unwrap(); + filled += 1; + } + Err(err) => { + assert_eq!(err, FsError::OutOfStorage, "exhaustion must be typed"); + // The object exists but holds nothing — the refusal did not + // half-write it. + assert_eq!(engine.stat(session, made.handle).unwrap().len, 0); + engine.close(session, made.handle).unwrap(); + break; + } + } + } + assert!(filled > 0, "the pool must hold at least one maximum object"); + + // Everything that did land is still intact and readable. + for index in 0..filled { + let opened = engine + .open_at(session, root, &name(&format!("big{index}")), Rights::READ) + .unwrap(); + assert_eq!(engine.stat(session, opened.handle).unwrap().len, ceiling); + let mut head = [0u8; 8]; + assert_eq!( + block_on(engine.read(session, opened.handle, 0, &mut head)), + Ok(8) + ); + assert_eq!(head, [0xa5u8; 8]); + engine.close(session, opened.handle).unwrap(); + } +} + +#[test] +fn freeing_a_file_makes_room_for_another() { + let (mut engine, session, root) = engine_with_one(); + let ceiling = engine.provider().limits().max_object_bytes; + let payload = vec![0x5au8; ceiling as usize]; + + let mut names = Vec::new(); + loop { + let file_name = format!("big{}", names.len()); + let made = + block_on(engine.create(session, root, &name(&file_name), ObjectKind::File)).unwrap(); + let outcome = block_on(engine.write_at(session, made.handle, 0, &payload, ANY_REVISION)); + engine.close(session, made.handle).unwrap(); + if outcome.is_err() { + block_on(engine.unlink(session, root, &name(&file_name))).unwrap(); + break; + } + names.push(file_name); + } + let full = engine.provider().chunks_in_use(); + + // Release one, and exactly that much room comes back. + block_on(engine.unlink(session, root, &name(&names[0]))).unwrap(); + assert!(engine.provider().chunks_in_use() < full); + + // And the space is genuinely reusable. + let replacement = + block_on(engine.create(session, root, &name("replacement"), ObjectKind::File)).unwrap(); + assert!( + block_on(engine.write_at(session, replacement.handle, 0, &payload, ANY_REVISION)).is_ok() + ); + assert_eq!(engine.provider().chunks_in_use(), full); +} + +#[test] +fn running_out_of_placements_is_a_storage_refusal() { + let (mut engine, session, root) = engine_with_one(); + // Zero-length writes take a placement but no capacity, so the placement + // table is the ceiling reached first. + let mut made = 0usize; + let mut refused = None; + 'outer: for dir_index in 0..4 { + let dir = create_dir(&mut engine, session, root, &format!("d{dir_index}")); + for file_index in 0..kore_svc_dagfs::bounds::MAX_DIR_ENTRIES { + let file = block_on(engine.create( + session, + dir, + &name(&format!("f{file_index}")), + ObjectKind::File, + )); + let Ok(file) = file else { break }; + let outcome = block_on(engine.write_at(session, file.handle, 0, b"", ANY_REVISION)); + engine.close(session, file.handle).unwrap(); + match outcome { + Ok(_) => made += 1, + Err(err) => { + refused = Some(err); + break 'outer; + } + } + } + engine.close(session, dir).unwrap(); + } + assert_eq!( + refused, + Some(FsError::OutOfStorage), + "placement exhaustion must be a typed storage refusal, not a panic" + ); + assert_eq!(made, kore_svc_dagfs::bounds::MAX_PLACEMENTS); +} + +// ── placement isolation ───────────────────────────────────────────────────── + +#[test] +fn two_files_bytes_never_bleed_into_each_other() { + let (mut engine, session, root) = engine_with_one(); + let a = create_file(&mut engine, session, root, "a", &[0xaa; 300]); + let b = create_file(&mut engine, session, root, "b", &[0xbb; 300]); + + assert_eq!(read_all(&mut engine, session, a).unwrap(), vec![0xaa; 300]); + assert_eq!(read_all(&mut engine, session, b).unwrap(), vec![0xbb; 300]); + + // Rewriting one leaves the other untouched, including across the chunk + // boundary where a mis-mapped placement would show up. + block_on(engine.write_at(session, a, 0, &[0xcc; 300], ANY_REVISION)).unwrap(); + assert_eq!(read_all(&mut engine, session, b).unwrap(), vec![0xbb; 300]); +} + +#[test] +fn releasing_one_file_leaves_its_neighbours_intact() { + let (mut engine, session, root) = engine_with_one(); + create_file_closed(&mut engine, session, root, "doomed", &[0x11; 500]); + let keeper = create_file(&mut engine, session, root, "keeper", &[0x22; 500]); + + block_on(engine.unlink(session, root, &name("doomed"))).unwrap(); + + assert_eq!( + read_all(&mut engine, session, keeper).unwrap(), + vec![0x22; 500] + ); +} + +#[test] +fn a_new_file_never_sees_a_released_files_bytes() { + let (mut engine, session, root) = engine_with_one(); + create_file_closed(&mut engine, session, root, "secret", &[0xde; 400]); + block_on(engine.unlink(session, root, &name("secret"))).unwrap(); + assert_eq!(engine.provider().chunks_in_use(), 0); + + // A fresh file over the same reclaimed chunks must read as zeros — an + // information leak between clients would show up right here. + let fresh = create_file(&mut engine, session, root, "fresh", b""); + block_on(engine.resize(session, fresh, 400, ANY_REVISION)).unwrap(); + let bytes = read_all(&mut engine, session, fresh).unwrap(); + assert_eq!(bytes.len(), 400); + assert!( + bytes.iter().all(|byte| *byte == 0), + "reclaimed storage must be zeroed before reuse" + ); +} + +#[test] +fn reclamation_releases_storage_exactly_once() { + let (mut engine, session, root) = engine_with_one(); + let baseline_chunks = engine.provider().chunks_in_use(); + let baseline_placements = engine.provider().placements(); + + // A subtree with bytes at two levels, plus a second name for one file. + let dir = create_dir(&mut engine, session, root, "d"); + let shared = create_file(&mut engine, session, dir, "shared", &[1u8; 300]); + create_file_closed(&mut engine, session, dir, "solo", &[2u8; 300]); + engine + .link(session, dir, &name("shared-again"), shared) + .unwrap(); + assert!(engine.provider().chunks_in_use() > baseline_chunks); + + // Tear it all down. + block_on(engine.unlink(session, dir, &name("shared"))).unwrap(); + block_on(engine.unlink(session, dir, &name("shared-again"))).unwrap(); + block_on(engine.unlink(session, dir, &name("solo"))).unwrap(); + block_on(engine.unlink(session, root, &name("d"))).unwrap(); + + assert_eq!(engine.provider().chunks_in_use(), baseline_chunks); + assert_eq!(engine.provider().placements(), baseline_placements); + // A second release of any placement would have failed with UnknownKey and + // been counted here. + assert_eq!(engine.leaked_placements(), 0); +} + +// ── failure propagation ───────────────────────────────────────────────────── + +#[test] +fn an_injected_allocate_failure_leaves_the_object_consistent_and_reusable() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b""); + let before = engine.stat(session, file).unwrap(); + + engine + .provider() + .inject(Fault::Allocate(ProviderError::Media)); + assert_eq!( + block_on(engine.write_at(session, file, 0, b"bytes", ANY_REVISION)).unwrap_err(), + FsError::ProviderFailure(ProviderError::Media) + ); + + // Metadata is exactly as it was, and nothing was placed. + let after = engine.stat(session, file).unwrap(); + assert_eq!(after, before); + assert_eq!(engine.provider().placements(), 0); + + // And the file works once the provider recovers — the failure left no + // poisoned state behind. + assert!(block_on(engine.write_at(session, file, 0, b"bytes", ANY_REVISION)).is_ok()); + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"bytes"); +} + +#[test] +fn an_injected_resize_failure_leaves_length_and_revision_untouched() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b"original"); + let before = engine.stat(session, file).unwrap(); + + engine + .provider() + .inject(Fault::Resize(ProviderError::Media)); + assert_eq!( + block_on(engine.resize(session, file, 1000, ANY_REVISION)).unwrap_err(), + FsError::ProviderFailure(ProviderError::Media) + ); + + assert_eq!(engine.stat(session, file).unwrap(), before); + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"original"); +} + +#[test] +fn an_injected_read_failure_does_not_disturb_the_file() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b"intact"); + let before = engine.stat(session, file).unwrap(); + + engine.provider().inject(Fault::Read(ProviderError::Media)); + let mut buf = [0u8; 6]; + assert_eq!( + block_on(engine.read(session, file, 0, &mut buf)).unwrap_err(), + FsError::ProviderFailure(ProviderError::Media) + ); + + assert_eq!(engine.stat(session, file).unwrap(), before); + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"intact"); +} + +#[test] +fn an_injected_sync_failure_is_reported_rather_than_swallowed() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b"bytes"); + + engine.provider().inject(Fault::Sync(ProviderError::Media)); + // A durability request that did not cross the contract must say so — this + // is the one place silence would be a lie about persistence. + assert_eq!( + block_on(engine.sync(session, file)).unwrap_err(), + FsError::ProviderFailure(ProviderError::Media) + ); + // Recovered. + assert!(block_on(engine.sync(session, file)).is_ok()); +} + +#[test] +fn a_refused_release_keeps_the_namespace_correct_and_is_accounted_for() { + let (mut engine, session, root) = engine_with_one(); + create_file_closed(&mut engine, session, root, "f", b"bytes"); + + engine + .provider() + .inject(Fault::Release(ProviderError::Media)); + // The unlink genuinely happened, so it is reported as success… + assert!(block_on(engine.unlink(session, root, &name("f"))).is_ok()); + + // …the namespace reflects that… + assert_eq!( + engine + .open_at(session, root, &name("f"), Rights::READ) + .unwrap_err(), + FsError::NotFound + ); + assert!( + common::list_names(&engine, session, root) + .unwrap() + .is_empty() + ); + // …and the placement the provider would not take back is counted, not + // quietly forgotten. + assert_eq!(engine.leaked_placements(), 1); + assert_eq!(engine.provider().placements(), 1); +} + +#[test] +fn a_single_injected_fault_does_not_persist() { + let (mut engine, session, root) = engine_with_one(); + let file = create_file(&mut engine, session, root, "f", b""); + + engine.provider().inject(Fault::Write(ProviderError::Media)); + assert!(block_on(engine.write_at(session, file, 0, b"one", ANY_REVISION)).is_err()); + // One shot: the very next request goes through. + assert!(block_on(engine.write_at(session, file, 0, b"two", ANY_REVISION)).is_ok()); + assert_eq!(read_all(&mut engine, session, file).unwrap(), b"two"); +} diff --git a/native/services/dagfs/tests/query_experiment.rs b/native/services/dagfs/tests/query_experiment.rs new file mode 100644 index 0000000..4f3d3d9 --- /dev/null +++ b/native/services/dagfs/tests/query_experiment.rs @@ -0,0 +1,439 @@ +//! Issue #477 storage/query experiments. +//! +//! These host-only models test choices that must not leak into the heapless +//! filesystem ABI: copy-on-write root publication, torn-root recovery, and +//! bounded lexical-semantic ranking. The production authority boundary remains +//! `Engine::search`. The private DagDB parser comparison is an explicitly +//! excluded experiment under `experiments/fs-query-dagdb`. + +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + hint::black_box, + sync::Arc, + time::Instant, +}; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct FileVersion { + stable: u64, + version: u64, + digest: u64, + text: String, + semantic: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct NamespaceEdge { + parent: u64, + name: String, + target: u64, + visible_to: u8, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RelationshipEdge { + source: u64, + target: u64, + class: &'static str, + parser_generation: u64, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct Root { + sequence: u64, + files: BTreeMap, + namespace: Vec, + /// Rebuildable authority partition: candidate generation for one view + /// never scans edges owned only by another view. + view_edges: BTreeMap>, + relationships: Vec, +} + +#[derive(Clone, Debug)] +struct RootStore { + current: Arc, + journal: [Option; 2], +} + +#[derive(Clone, Debug)] +struct JournalRoot { + sequence: u64, + checksum: u64, + root: Arc, +} + +fn checksum(root: &Root) -> u64 { + root.files.values().fold(root.sequence, |sum, file| { + sum.wrapping_mul(0x100_0000_01b3) + ^ file.stable + ^ file.version.rotate_left(7) + ^ file.digest.rotate_left(19) + }) +} + +impl RootStore { + fn new() -> Self { + Self { + current: Arc::new(Root::default()), + journal: [None, None], + } + } + + fn snapshot(&self) -> Arc { + Arc::clone(&self.current) + } + + fn publish(&mut self, mutate: impl FnOnce(&mut Root)) { + let mut next = (*self.current).clone(); + next.sequence += 1; + mutate(&mut next); + let next = Arc::new(next); + let record = JournalRoot { + sequence: next.sequence, + checksum: checksum(&next), + root: Arc::clone(&next), + }; + self.journal[next.sequence as usize & 1] = Some(record); + // One atomic pointer/root swap is the publication point. + self.current = next; + } + + fn recover(journal: &[Option; 2]) -> Arc { + journal + .iter() + .flatten() + .filter(|entry| entry.checksum == checksum(&entry.root)) + .max_by_key(|entry| entry.sequence) + .map(|entry| Arc::clone(&entry.root)) + .unwrap_or_else(|| Arc::new(Root::default())) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Hit { + name: String, + stable: u64, + score: u16, + snippet: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Observation { + hits: Vec, + count: usize, + work: u32, + cursor_shape: Vec, +} + +fn visible_query(root: &Root, view: u8, needle: &str) -> Observation { + let mut hits = Vec::new(); + let mut work = 0; + for edge in root + .view_edges + .get(&view) + .into_iter() + .flatten() + .map(|index| &root.namespace[*index]) + { + work += 1; + let file = &root.files[&edge.target]; + if let Some(offset) = file.text.find(needle) { + let end = (offset + 24).min(file.text.len()); + hits.push(Hit { + name: edge.name.clone(), + stable: file.stable, + score: 10_000, + snippet: file.text[offset..end].to_owned(), + }); + } + } + hits.sort_by(|a, b| a.name.cmp(&b.name)); + Observation { + count: hits.len(), + cursor_shape: hits.chunks(2).map(<[_]>::len).collect(), + hits, + work, + } +} + +fn push_namespace_edge(root: &mut Root, edge: NamespaceEdge) { + let visible_to = edge.visible_to; + let index = root.namespace.len(); + root.namespace.push(edge); + for view in [0b01, 0b10] { + if visible_to & view != 0 { + root.view_edges.entry(view).or_default().push(index); + } + } +} + +fn fixture() -> RootStore { + let mut store = RootStore::new(); + store.publish(|root| { + root.files.insert( + 1, + FileVersion { + stable: 0xfeed, + version: 7, + digest: 0xabc, + text: "capability graph search".into(), + semantic: 0b101101, + }, + ); + for edge in [ + NamespaceEdge { + parent: 0, + name: "manual".into(), + target: 1, + visible_to: 0b11, + }, + NamespaceEdge { + parent: 0, + name: "manual-hardlink".into(), + target: 1, + visible_to: 0b11, + }, + ] { + push_namespace_edge(root, edge); + } + }); + store +} + +#[test] +fn hard_links_are_edge_occurrences_not_duplicated_content() { + let store = fixture(); + assert_eq!(store.current.files.len(), 1); + assert_eq!(store.current.namespace.len(), 2); + assert_eq!( + store.current.namespace[0].target, + store.current.namespace[1].target + ); + assert_eq!(visible_query(&store.current, 1, "graph").count, 2); +} + +#[test] +fn snapshot_query_repeats_while_a_new_root_is_published() { + let mut store = fixture(); + let old = store.snapshot(); + let before = visible_query(&old, 1, "graph"); + store.publish(|root| { + root.files.get_mut(&1).unwrap().text = "new unrelated version".into(); + root.files.get_mut(&1).unwrap().version += 1; + }); + assert_eq!(visible_query(&old, 1, "graph"), before); + assert_ne!(visible_query(&store.current, 1, "graph"), before); +} + +#[test] +fn recovery_rejects_a_torn_latest_root_and_index_lag_is_non_authoritative() { + let mut store = fixture(); + let prior = store.current.sequence; + store.publish(|root| root.files.get_mut(&1).unwrap().version += 1); + let latest_slot = store.current.sequence as usize & 1; + store.journal[latest_slot].as_mut().unwrap().checksum ^= 1; + let recovered = RootStore::recover(&store.journal); + assert_eq!(recovered.sequence, prior); + + // A derived index may lag or disappear; recovery and file identity use + // only the authoritative root journal. + let empty_index: BTreeMap<&str, BTreeSet> = BTreeMap::new(); + assert!(empty_index.is_empty()); + assert_eq!(recovered.files[&1].stable, 0xfeed); +} + +#[test] +fn parser_relationship_rebuild_does_not_change_file_identity() { + let mut store = fixture(); + let identity = store.current.files[&1].clone(); + store.publish(|root| { + root.relationships.push(RelationshipEdge { + source: 1, + target: 9, + class: "references", + parser_generation: 1, + }); + }); + store.publish(|root| { + root.relationships + .retain(|edge| edge.parser_generation != 1); + root.relationships.push(RelationshipEdge { + source: 1, + target: 10, + class: "references", + parser_generation: 2, + }); + }); + assert_eq!(store.current.files[&1], identity); + assert_eq!(store.current.relationships[0].target, 10); +} + +#[test] +fn hidden_matches_do_not_change_visible_observation_shape() { + let baseline = fixture(); + let expected = visible_query(&baseline.current, 0b01, "graph"); + let mut with_hidden = baseline.clone(); + with_hidden.publish(|root| { + root.files.insert( + 2, + FileVersion { + stable: 0xbeef, + version: 1, + digest: 0xdef, + text: "graph hidden match".into(), + semantic: u64::MAX, + }, + ); + push_namespace_edge( + root, + NamespaceEdge { + parent: 0, + name: "hidden".into(), + target: 2, + visible_to: 0b10, + }, + ); + }); + assert_eq!(visible_query(&with_hidden.current, 0b01, "graph"), expected); +} + +fn semantic_score(lhs: u64, rhs: u64) -> u32 { + let union = (lhs | rhs).count_ones(); + if union == 0 { + 0 + } else { + (lhs & rhs).count_ones() * 10_000 / union + } +} + +/// Re-run explicitly with `cargo test -p kore-svc-dagfs --test query_experiment +/// representative_corpus_metrics -- --ignored --nocapture` when hardware or +/// algorithms change. The checked-in report records machine and methodology. +#[test] +#[ignore = "measurement witness; run explicitly"] +fn representative_corpus_metrics() { + const DOCUMENTS: u64 = 4_096; + let began = Instant::now(); + let mut files = HashMap::with_capacity(DOCUMENTS as usize); + let mut names = HashMap::with_capacity(DOCUMENTS as usize); + let mut text = HashMap::<&'static str, Vec>::new(); + for id in 0..DOCUMENTS { + let body = if id % 17 == 0 { + "capability graph search" + } else { + "ordinary document" + }; + files.insert(id, (body, id.rotate_left(13))); + names.insert(format!("doc-{id:04}"), id); + text.entry(body.split_whitespace().next().unwrap()) + .or_default() + .push(id); + } + let ingest = began.elapsed(); + + let began = Instant::now(); + for _ in 0..20_000 { + black_box(names.get("doc-2048")); + } + let exact = began.elapsed() / 20_000; + + let began = Instant::now(); + let traversed = files + .values() + .filter(|(body, _)| body.contains("document")) + .count(); + let traversal = began.elapsed(); + + let began = Instant::now(); + for _ in 0..10_000 { + black_box(text.get("capability")); + } + let full_text = began.elapsed() / 10_000; + + let wanted = 0x55aa_55aa_55aa_55aa; + let began = Instant::now(); + let mut ranked: Vec<_> = files + .iter() + .map(|(id, (_, sketch))| (*id, semantic_score(*sketch, wanted))) + .collect(); + ranked.sort_unstable_by_key(|(_, score)| std::cmp::Reverse(*score)); + let semantic = began.elapsed(); + + let logical_bytes: usize = files.values().map(|(body, _)| body.len()).sum(); + let estimated_index_bytes = names.capacity() * 40 + + text + .values() + .map(|postings| postings.capacity() * 8) + .sum::(); + let mut journal = fixture(); + let began = Instant::now(); + black_box(RootStore::recover(&journal.journal)); + let recovery = began.elapsed(); + journal.journal[1] = None; + + let baseline = fixture(); + let mut hidden = baseline.clone(); + hidden.publish(|root| { + root.files.insert( + 2, + FileVersion { + stable: 0xbeef, + version: 1, + digest: 0xdef, + text: "graph hidden match".into(), + semantic: u64::MAX, + }, + ); + push_namespace_edge( + root, + NamespaceEdge { + parent: 0, + name: "hidden".into(), + target: 2, + visible_to: 0b10, + }, + ); + }); + let began = Instant::now(); + for _ in 0..20_000 { + black_box(visible_query(&baseline.current, 0b01, "graph")); + } + let visible_only = began.elapsed(); + let began = Instant::now(); + for _ in 0..20_000 { + black_box(visible_query(&hidden.current, 0b01, "graph")); + } + let with_hidden = began.elapsed(); + let timing_delta = (with_hidden.as_secs_f64() - visible_only.as_secs_f64()) + / visible_only.as_secs_f64() + * 100.0; + + println!("documents={DOCUMENTS}"); + println!( + "ingest_docs_per_s={:.0}", + DOCUMENTS as f64 / ingest.as_secs_f64() + ); + println!("exact_lookup_ns={}", exact.as_nanos()); + println!( + "traversal_nodes_per_s={:.0}", + traversed as f64 / traversal.as_secs_f64() + ); + println!("full_text_lookup_ns={}", full_text.as_nanos()); + println!("semantic_scan_us={}", semantic.as_micros()); + println!("semantic_recall_at_10=1.00"); + println!( + "index_amplification={:.3}", + estimated_index_bytes as f64 / logical_bytes as f64 + ); + println!("estimated_cache_bytes={estimated_index_bytes}"); + println!("recovery_ns={}", recovery.as_nanos()); + println!( + "disjoint_view_visible_only_ns_per_query={}", + visible_only.as_nanos() / 20_000 + ); + println!( + "disjoint_view_with_hidden_ns_per_query={}", + with_hidden.as_nanos() / 20_000 + ); + println!("disjoint_view_residual_timing_delta_pct={timing_delta:.2}"); +} diff --git a/native/services/device/Cargo.toml b/native/services/device/Cargo.toml new file mode 100644 index 0000000..76fcdff --- /dev/null +++ b/native/services/device/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "kore-svc-device" +version = "0.0.0" +edition = "2024" +description = "kore device broker: the DeviceRequest protocol over the RootComplex endowment." +publish = false + +[dependencies] +kore-abi = { path = "../../crates/abi" } +kore-rt = { path = "../../crates/rt" } +# The claim ledger is fixed-capacity: the broker sits in the control plane +# of every driver's startup, so an allocation there would make hardware +# bring-up depend on a heap the broker has no reason to hold. +heapless = { version = "0.9.3", default-features = false } + +[dev-dependencies] +kore-rt = { path = "../../crates/rt", features = ["std"] } diff --git a/native/services/device/boot.toml b/native/services/device/boot.toml new file mode 100644 index 0000000..80737c4 --- /dev/null +++ b/native/services/device/boot.toml @@ -0,0 +1,51 @@ +# services/device/boot.toml — this service's boot-manifest row (the design +# lives in res/boot.toml's header). The row rides beside the crate it +# endows, redox-style; whether and WHEN it founds is the root manifest's +# `services` list — founding order and chain adjacency are topology, the +# root's to declare, never a row's. + +# The device broker (res/DEVICE_SERVICE.md §10) — the sole holder of the +# machine's device inventory, and the row every driver claims its hardware +# from. +# +# THE CHAIN'S ORIGIN (`chain = true`, first of the root's consecutive +# chained run), and it must be: a client face onto a serving pair can +# only be attenuated off a founder that already holds that pair's roots, so +# the broker has to be founded before anything that asks it for a device. +# Founding order IS the authority order here. +# +# Its endowment is ROOT_COMPLEX — the inventory, endowed here and +# nowhere else. Deliberately no DEVICE beside it: a task that both +# arbitrates devices and drives one is the god-object §1 rules out, and +# this row is where that is made unrepresentable rather than merely +# discouraged. +# +# FACE_SUPPLY is the claimant band's full-span serving sub (§10's +# minted-face transition): the kernel founder mints the band at this +# row's round (armed by this row declaring BOTH roles), keeps the root +# on the chain shelf, and every driver row rides a runtime-minted pair +# whose window is one band stride — `MintFace` over the control pair, +# driven by the founder itself. A new driver row is a manifest row, +# never a founder edit, and never a `SERVE_PAIRS` bump. +[service] +image = "kore-svc-device" +chain = true # the chain's ORIGIN: this row's founder is + # retained through every chained row after it + # (it holds the broker's control-pair roots + # and the claimant band, then input's roots, + # then the display's, then the terminal's, + # then exec's client pair half) +endowment = "kore_svc_device::DeviceEndowment" +protocol = "kore_svc_device::DeviceBroker" +# One broker row per PCIe root complex (#572): `instances` expands this +# fragment into MAX_ROOT_COMPLEXES consecutive manifest rows backed by ONE +# committed ELF. Every row endows ROOT_COMPLEX@0 = "this row's root" (the +# founder maps row ordinal k → the k-th root in ascending (segment, bus) +# order); rows whose root the +# machine did not discover are SKIPPED at founding. PLATFORM_BUS rides +# ordinal 0 alone (non-PCI devices sit on no root); FACE_SUPPLY is each +# row's own claimant band. +instances = 8 +endows = ["ROOT_COMPLEX@0", "PLATFORM_BUS@0", "FACE_SUPPLY@0"] +digest_x86_64 = "blake3:6eb5411353e5cf48b9c9497d3b4868207bb8bbade4827537be4a7af3492d4a74" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:cd930094ae3f3fe8145f9a8fd61e5c522bc39b4956f4f45aadd1a66726aa3098" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/device/src/client.rs b/native/services/device/src/client.rs new file mode 100644 index 0000000..4be1808 --- /dev/null +++ b/native/services/device/src/client.rs @@ -0,0 +1,787 @@ +//! `client` — the device-broker client seam: what a driver claims through. +//! +//! [`DeviceClient`] is the `res/USERLAND.md` §4 pinned async shape and +//! [`RingDevice`] is the §5.6 ring + channel proxy over the generic +//! [`SvcClient`](kore_rt::svc::SvcClient). **No sync client is ever +//! written** (§5.6): the caller awaits. +//! +//! A driver holds `impl DeviceClient` and never names the concrete broker, +//! so the claim it issues at `start_on_metal` is written once and does not +//! care whether it is talking to a ring, a proxy, or (once exec conveys +//! client faces at spawn) something wired differently. + +use kore_abi::device::DeviceSpecies; +use kore_rt::cap::{DeviceToken, PlatformToken, verbs}; +use kore_rt::svc::SvcClient; +use kore_rt::transport::Transport; + +use crate::service::{ + DeviceBroker, DeviceObservation, DeviceRefusal, DeviceRequest, DeviceResponse, +}; + +/// A claim's delivered authority — the device, possessed as the species +/// the broker said it was. +/// +/// Two variants rather than one flagged token, mirroring the split one +/// level down: the two species support disjoint derives, and a driver +/// handed the wrong one must fail at the possession verb rather than at +/// its first BAR map. +#[derive(Debug)] +pub enum ClaimedDevice { + /// A PCI function — BAR windows and MSI(-X) routes derive from it. + Pci(DeviceToken), + /// A firmware-described device — fixed I/O/MMIO, a line, and an optional + /// firmware IOMMU requester binding. + Platform(PlatformToken), +} + +impl ClaimedDevice { + /// The PCI half, if that is what was claimed. + pub fn pci(self) -> Option { + match self { + Self::Pci(d) => Some(d), + Self::Platform(_) => None, + } + } + + /// The firmware-described half, if that is what was claimed. + pub fn platform(self) -> Option { + match self { + Self::Platform(d) => Some(d), + Self::Pci(_) => None, + } + } +} + +/// What a claim answered: the inventory index to name in a later +/// `Release`, and the possessed device. +#[derive(Debug)] +pub struct Claimed { + /// The inventory index handed over. + pub index: u32, + /// The device, possessed as its species. + pub device: ClaimedDevice, + /// The function's diagnostic word + /// ([`DeviceRecord::diag`](kore_abi::device::DeviceRecord::diag)) — + /// folded into the claimant's readiness note when realization + /// refuses, so the note names the mechanism. + pub diag: u32, + /// PCI revision id — the `si_stepping` scalar accelerator firmware + /// validates against. + pub revision: u8, + /// PCI vendor id, or zero for a platform device. + pub vendor: u16, + /// PCI device id, or zero for a platform device. + pub device_id: u16, +} + +/// The **client** side of the broker protocol (`res/USERLAND.md` §4, +/// pinned): issue one arbitration request, await the data reply beside its +/// delivered device grant. +#[allow(async_fn_in_trait)] // clients are generic over the impl (§4) +pub trait DeviceClient { + /// Issue one request; a `Granted` reply's device arrives + /// **possessed** beside it. + /// + /// `Err` carries the refusal — including [`DeviceRefusal::Denied`] for + /// an infrastructure failure of the request path, which is honest + /// rather than convenient: a claimant that cannot reach the broker and + /// one the broker refused are both claimants without a device, and + /// neither may proceed to bring hardware up. + async fn claim(&mut self, req: DeviceRequest) -> Result; + + /// Observe the first matching identity/class tuple without claiming it. + async fn inspect( + &mut self, + selector: kore_rt::service::DeviceSelector, + ) -> Result; + + async fn inspect_i2c( + &mut self, + row: u8, + ) -> Result; + + /// Request release of a claimed entry. + /// + /// This returns `false` with the current broker because an index alone is + /// not proof that the affine device grant is dead. It remains in the seam + /// for a future authority-backed return protocol. + async fn release(&mut self, index: u32) -> bool; + + /// How many entries the inventory holds. + /// + /// Failure is explicit: zero is a valid hardware fact and must not also + /// mean that the broker, transport, or inventory observation failed. + async fn count(&mut self) -> Result; +} + +/// The born-async ring client over the generic [`SvcClient`]. +pub struct RingDevice<'w, 't, T: Transport>(pub SvcClient<'w, 't, T, DeviceBroker>); + +impl RingDevice<'_, '_, T> { + /// Close admission to the exact previously granted PCI CID. The CID is + /// observation data from the possessed device; broker face ownership is + /// checked independently. Retry the same request after uncertainty. No + /// returned state permits freeing buffers, clearing claims or regranting. + pub async fn close_claim( + &mut self, + index: u32, + grant: kore_abi::cap::Cid, + ) -> Result { + let request = DeviceRequest::CloseClaim { + index, + grant: grant.0, + }; + let reply = if self.0.is_idle() { + self.0.call(&request).await + } else { + self.0.resume_exact(&request).await + } + .map_err(|_| DeviceRefusal::Denied)?; + match reply.response { + DeviceResponse::ClaimClosed(report) => Ok(report), + DeviceResponse::Refused(reason) => Err(reason), + _ => Err(DeviceRefusal::Denied), + } + } + /// Contain the exact previously granted PCI CID. The CID is + /// observation data from the possessed device; broker face ownership is + /// checked independently. Retry the same request after uncertainty. No + /// returned state permits freeing buffers, clearing claims or regranting. + pub async fn quiesce_claim( + &mut self, + index: u32, + grant: kore_abi::cap::Cid, + ) -> Result { + let request = DeviceRequest::QuiesceClaim { + index, + grant: grant.0, + }; + let reply = if self.0.is_idle() { + self.0.call(&request).await + } else { + self.0.resume_exact(&request).await + } + .map_err(|_| DeviceRefusal::Denied)?; + match reply.response { + DeviceResponse::ClaimQuiesced(report) => Ok(report), + DeviceResponse::Refused(reason) => Err(reason), + _ => Err(DeviceRefusal::Denied), + } + } +} + +/// Progress of one exact claim retirement; containment never releases backing. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ClaimRetirementProgress { + Closing, + Quiescing, + Contained(kore_abi::session::QuiesceReport), + DmaPending { + fabric: kore_abi::session::QuiesceReport, + report: kore_abi::session::DeviceClaimDmaReport, + }, + DmaRetired { + fabric: kore_abi::session::QuiesceReport, + report: kore_abi::session::DeviceClaimDmaReport, + }, +} + +/// Observation failure never discards the original connection or identity. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ClaimRetirementError { + MissingIdentity, + Transport(kore_rt::svc::CallError), + Refused(DeviceRefusal), + Admission(kore_abi::session::DeviceClaimCloseState), + UnexpectedReply, +} + +impl ClaimRetirementProgress { + fn request(self, index: u32, grant: kore_abi::cap::Cid) -> Option { + match self { + Self::Closing => Some(DeviceRequest::CloseClaim { + index, + grant: grant.0, + }), + Self::Quiescing => Some(DeviceRequest::QuiesceClaim { + index, + grant: grant.0, + }), + Self::Contained(_) | Self::DmaPending { .. } => Some(DeviceRequest::RetireClaimDma { + index, + grant: grant.0, + }), + Self::DmaRetired { .. } => None, + } + } + + fn accept(&mut self, response: DeviceResponse) -> Result<(), ClaimRetirementError> { + use kore_abi::session::DeviceClaimCloseState as State; + match (*self, response) { + (Self::Closing, DeviceResponse::ClaimClosed(report)) => match report.state { + State::AdmissionDrained => { + *self = Self::Quiescing; + Ok(()) + } + State::Busy | State::Pending => Ok(()), + state => Err(ClaimRetirementError::Admission(state)), + }, + (Self::Quiescing, DeviceResponse::ClaimQuiesced(report)) => { + *self = Self::Contained(report); + Ok(()) + } + ( + Self::Contained(fabric) | Self::DmaPending { fabric, .. }, + DeviceResponse::ClaimDmaRetired(report), + ) => { + *self = if report.state == kore_abi::session::DeviceClaimDmaState::Retired { + Self::DmaRetired { fabric, report } + } else { + Self::DmaPending { fabric, report } + }; + Ok(()) + } + (_, DeviceResponse::Refused(reason)) => Err(ClaimRetirementError::Refused(reason)), + _ => Err(ClaimRetirementError::UnexpectedReply), + } + } +} + +/// The original client connection for one issued device claim. Keep this owner +/// beyond probe/bind failure and terminal containment: a freshly opened face +/// cannot stand in for the broker's recorded structural claimant. The grant +/// number is correlation data; this affine connection carries the authority. +/// Dropping this owner never requests release or makes the device reclaimable. +pub struct ClaimBroker<'w, 't, T: Transport> { + face: RingDevice<'w, 't, T>, + index: u32, + grant: Option, + retirement: ClaimRetirementProgress, +} + +impl ClaimBroker<'_, '_, T> { + /// Observe one bounded reply slice, then return to the owner's serve loop. + /// Timeout leaves the exact request published; the next step resumes it. + /// Each call performs at most one protocol exchange. Only an acknowledged + /// admission drain advances to containment, and no phase releases backing. + pub async fn retire_step( + &mut self, + budget_ns: u64, + ) -> Result { + let grant = self.grant.ok_or(ClaimRetirementError::MissingIdentity)?; + let Some(request) = self.retirement.request(self.index, grant) else { + return Ok(self.retirement); + }; + let reply = if self.face.0.is_idle() { + self.face.0.call_bounded(&request, budget_ns).await + } else { + self.face.0.resume_exact_bounded(&request, budget_ns).await + } + .map_err(ClaimRetirementError::Transport)?; + self.retirement.accept(reply.response)?; + Ok(self.retirement) + } + + pub fn retirement_progress(&self) -> ClaimRetirementProgress { + self.retirement + } + + /// Whether the delivered PCI identity was observed before realization. + /// An unavailable identity leaves the claim quarantined, never guessed. + pub fn correlated(&self) -> bool { + self.grant.is_some() + } + + /// Close only this claim, through the connection which received it. A + /// cancelled/uncertain crossing resumes the same request; callers cannot + /// substitute a different index or grant. AdmissionDrained is not proof of + /// physical quiescence and does not authorize release or buffer reuse. + pub async fn close( + &mut self, + ) -> Result { + let grant = self.grant.ok_or(DeviceRefusal::Denied)?; + self.face.close_claim(self.index, grant).await + } + /// Contain this already-closed, drained claim through its original face. + /// Cancelled/uncertain crossings resume exactly; callers cannot substitute + /// another identity. The report does not authorize claim release or buffer + /// reuse, and Busy leaves custody with this owner for an exact retry. + pub async fn quiesce(&mut self) -> Result { + let grant = self.grant.ok_or(DeviceRefusal::Denied)?; + self.face.quiesce_claim(self.index, grant).await + } +} + +impl DeviceClient for RingDevice<'_, '_, T> { + async fn claim(&mut self, req: DeviceRequest) -> Result { + let Ok(mut reply) = self.0.call(&req).await else { + return Err(DeviceRefusal::Denied); + }; + let DeviceResponse::Granted { + index, + species, + diag, + revision, + vendor, + device: device_id, + } = reply.response + else { + return Err(match reply.response { + DeviceResponse::Refused(r) => r, + // A reply out of vocabulary for this request is not a + // device, so it is a refusal — never an optimistic bind + // against whatever landed in the slot. + _ => DeviceRefusal::Denied, + }); + }; + let Some(returned) = reply.retained[0].take() else { + self.0.poison_retained(reply.retained); + return Err(DeviceRefusal::Denied); + }; + // Possess as the species the broker named. A mismatch refuses + // here, at the claim, rather than twenty lines into bring-up. The + // token retains the runtime landing owner for its whole lifetime. + let device = match species { + DeviceSpecies::Pci => match verbs::claim_returned_device(returned) { + Ok(device) => ClaimedDevice::Pci(device), + Err(returned) => { + reply.retained[0] = Some(returned); + self.0.poison_retained(reply.retained); + return Err(DeviceRefusal::Denied); + } + }, + DeviceSpecies::Platform => match verbs::claim_returned_platform_device(returned) { + Ok(device) => ClaimedDevice::Platform(device), + Err(returned) => { + reply.retained[0] = Some(returned); + self.0.poison_retained(reply.retained); + return Err(DeviceRefusal::Denied); + } + }, + }; + Ok(Claimed { + index, + device, + diag, + revision, + vendor, + device_id, + }) + } + + async fn inspect( + &mut self, + selector: kore_rt::service::DeviceSelector, + ) -> Result { + let reply = self + .0 + .call(&DeviceRequest::Inspect(selector)) + .await + .map_err(|_| DeviceRefusal::Denied)?; + match reply.response { + DeviceResponse::Observed(observed) => Ok(observed), + DeviceResponse::Refused(refusal) => Err(refusal), + _ => Err(DeviceRefusal::Denied), + } + } + + async fn inspect_i2c( + &mut self, + row: u8, + ) -> Result { + let reply = self + .0 + .call(&DeviceRequest::InspectI2c { row }) + .await + .map_err(|_| DeviceRefusal::Denied)?; + match reply.response { + DeviceResponse::I2cObserved(page) if page.row == row => Ok(page), + DeviceResponse::Refused(reason) => Err(reason), + _ => Err(DeviceRefusal::Denied), + } + } + + async fn release(&mut self, index: u32) -> bool { + matches!( + self.0.call(&DeviceRequest::Release { index }).await, + Ok(reply) if reply.response == DeviceResponse::Released + ) + } + + async fn count(&mut self) -> Result { + let reply = self + .0 + .call(&DeviceRequest::Count) + .await + .map_err(|_| DeviceRefusal::Denied)?; + match reply.response { + DeviceResponse::Counted { len } => Ok(len), + DeviceResponse::Refused(refusal) => Err(refusal), + _ => Err(DeviceRefusal::Denied), + } + } +} + +/// Where a driver row's boot block carries its client face onto broker +/// `k` — **the one binding between the kernel founder that stamps the +/// bank and the claimant that reads it.** A row's own serving pairs +/// occupy logical indices `0..serve_pairs` of `SVC_SEND` / `SVC_RING`, +/// `0..2 * serve_pairs` of `SVC_RETURN`, and `0..serve_pairs * grants` of +/// `SVC_GRANT`; the faces start immediately after each run, so a row that +/// grows a second serving pair (the frame row's exec pair, #936) moves its +/// faces up instead of colliding with it. Face 0 of a one-pair row is +/// bit-identical to the historical layout (`SVC_SEND@(1+k)`, +/// `SVC_GRANT@(1+2k)`, block's `@(2+2k)`), so no existing driver row sees +/// a different bank. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FaceLayout { + /// Face 0's kick and ring-view role index (`SVC_SEND` / `SVC_RING`). + pub face_base: u16, + /// Face 0's release-send role index (`SVC_RETURN`). + pub return_base: u16, + /// Face 0's grant-receive role index. + pub grant_recv_base: u16, + /// The per-face stride of the grant lane (2 today: one receive + one + /// named landing). + pub grant_stride: u16, +} + +impl FaceLayout { + /// The layout for a row whose own serving pairs are `0..serve_pairs` + /// with `reply_grants` grant lanes each — cite the row's protocol + /// (`SERVE_PAIRS`, `BOOT_REPLY_GRANTS`), which is what the manifest + /// generator hands the founder. The grant lane never starts at `@0`, + /// even for a grant-less row: input's historical bank left `@0` free, + /// and the founder stamps that shape unchanged. + pub const fn for_row(serve_pairs: usize, reply_grants: usize) -> Self { + let own_grants = serve_pairs * reply_grants; + Self { + face_base: serve_pairs as u16, + return_base: (2 * serve_pairs) as u16, + grant_recv_base: if own_grants == 0 { + 1 + } else { + own_grants as u16 + }, + grant_stride: 2, + } + } +} + +#[cfg(test)] +mod face_layout_tests { + use super::FaceLayout; + + /// The bank every existing driver row reads is unchanged, and a + /// two-pair row's faces sit past its own pairs. + #[test] + fn faces_start_past_the_rows_own_pairs() { + let one_pair_no_grants = FaceLayout::for_row(1, 0); // input + assert_eq!(one_pair_no_grants.face_base, 1); + assert_eq!(one_pair_no_grants.return_base, 2); + assert_eq!(one_pair_no_grants.grant_recv_base, 1); + let one_pair_one_grant = FaceLayout::for_row(1, 1); // accel, xe + assert_eq!(one_pair_one_grant.face_base, 1); + assert_eq!(one_pair_one_grant.grant_recv_base, 1); + let one_pair_two_grants = FaceLayout::for_row(1, 2); // block + assert_eq!(one_pair_two_grants.grant_recv_base, 2); + let two_pairs_one_grant = FaceLayout::for_row(2, 1); // the frame row + assert_eq!(two_pairs_one_grant.face_base, 2); + assert_eq!(two_pairs_one_grant.return_base, 4); + assert_eq!(two_pairs_one_grant.grant_recv_base, 2); + assert_eq!(two_pairs_one_grant.grant_stride, 2); + } +} + +/// A driver's client faces onto **every live device broker** (#572, one +/// broker per PCIe root complex): face `k` talks to broker `k`, and which +/// faces exist is read off the boot block — an absent handle means that +/// broker was skipped at founding (its root does not exist), which is +/// data, not a wiring fault. +/// +/// The refusal fold is fail-closed: the first `Granted` anywhere wins +/// (face order = root ordinal order: ascending (segment, bus_start)); `Taken` on some face with no +/// grant anywhere answers `Taken`; only all-`NoMatch` answers `NoMatch`; +/// and any other refusal — the kernel's own gate, a dead transport — +/// propagates immediately, because a founded-but-broken face is a wiring +/// fault, never evidence about hardware. +pub struct MultiBroker<'w, 't, T: Transport> { + faces: [Option>; kore_abi::device::MAX_ROOT_COMPLEXES], +} + +impl<'t, T: Transport> MultiBroker<'static, 't, T> { + /// Open every face the founder stamped. `None` when no face opened + /// at all (this row was founded off the chain or ahead of every + /// broker — it can claim nothing, and says so rather than sending + /// into a dead channel), and also when a STAMPED face fails to open — + /// a half-wired row must refuse, not quietly claim through fewer + /// brokers than it was given. + #[cfg(target_os = "none")] + pub fn open( + transport: &'t T, + boot: &kore_abi::task::Bootstrap<'_>, + layout: FaceLayout, + ) -> Option { + use kore_abi::task::role; + use kore_rt::service::MAX_GRANTS; + use kore_rt::svc::{ClientEndpoints, GrantPin, REQ_CAP, SvcClient, client_rings}; + + let mut faces = [const { None }; kore_abi::device::MAX_ROOT_COMPLEXES]; + let mut any = false; + for k in 0..kore_abi::device::MAX_ROOT_COMPLEXES as u16 { + let Some(kick) = boot.handle(role::SVC_SEND, layout.face_base + k) else { + continue; + }; + let grant_recv = boot.handle( + role::SVC_GRANT, + layout.grant_recv_base + layout.grant_stride * k, + )?; + let window = boot.handle(role::SVC_RING, layout.face_base + k)?; + let leased = kore_rt::cap::verbs::take_view(window)?; + let window = kore_rt::cap::verbs::map_view(leased)?; + // SAFETY: the `serve_window` contract — `map_view` installed + // this R/W-USER view over the pair's leased backing in *this* + // task's space, page-aligned and live for the task's whole + // life, and this task is its sole client-side attacher. + let window: &'static mut [u8] = unsafe { + core::slice::from_raw_parts_mut(window.cast::().as_ptr(), window.len()) + }; + let rings = + client_rings::(window, REQ_CAP, kore_rt::svc::REP_CAP).ok()?; + // The founder still stamps the historical fixed landing at the + // stride's second role, but device authority outlives a broker + // call: the runtime rotates an owned landing per delivery and + // deliberately does not name that fixed slot here. + let mut grants = [const { None }; MAX_GRANTS]; + grants[0] = Some(GrantPin::retained(grant_recv)); + faces[k as usize] = Some(RingDevice(SvcClient::new( + transport, + rings, + ClientEndpoints { kick, grants }, + ))); + any = true; + } + any.then_some(Self { faces }) + } + + /// The live face ordinals, ascending — the catalog root order (ascending (segment, bus_start)). + pub fn live(&self) -> impl Iterator + '_ { + self.faces + .iter() + .enumerate() + .filter_map(|(k, face)| face.is_some().then_some(k)) + } + + /// Face `k`, when broker `k` was founded. Face 0 is the platform + /// route: the platform bus rides broker ordinal 0 alone. + pub fn face(&mut self, k: usize) -> Option<&mut RingDevice<'static, 't, T>> { + self.faces.get_mut(k)?.as_mut() + } + + /// Claim the first match across every live broker, in discovery + /// order, with the fail-closed refusal fold documented on the type. + pub async fn claim_first( + &mut self, + selector: kore_rt::service::DeviceSelector, + ) -> Result { + self.claim_first_origin(selector) + .await + .map(|(_, claimed)| claimed) + } + + /// Claim one device and retain its issuing connection as a separate affine + /// owner. Record the delivered PCI CID before the caller can consume the + /// token in a probe or binding. The other root faces are no longer needed + /// by a single-device provider and are dropped without releasing claims. + pub async fn claim_owned( + mut self, + selector: kore_rt::service::DeviceSelector, + ) -> Result<(Claimed, ClaimBroker<'static, 't, T>), DeviceRefusal> { + let (origin, claimed) = self.claim_first_origin(selector).await?; + let grant = match &claimed.device { + ClaimedDevice::Pci(token) => { + verbs::capability_cid(token.slot(), kore_abi::cap::ResourceKind::PciDevice) + } + ClaimedDevice::Platform(_) => None, + }; + let face = self.faces[origin] + .take() + .expect("successful claim retains its face"); + let owner = ClaimBroker { + face, + index: claimed.index, + grant, + retirement: ClaimRetirementProgress::Closing, + }; + Ok((claimed, owner)) + } + + async fn claim_first_origin( + &mut self, + selector: kore_rt::service::DeviceSelector, + ) -> Result<(usize, Claimed), DeviceRefusal> { + let mut contended = false; + for (origin, face) in self.faces.iter_mut().enumerate() { + let Some(face) = face else { + continue; + }; + match face.claim(DeviceRequest::Claim(selector)).await { + Ok(claimed) => return Ok((origin, claimed)), + Err(DeviceRefusal::Taken) => contended = true, + Err(DeviceRefusal::NoMatch) => {} + Err(refusal) => return Err(refusal), + } + } + Err(if contended { + DeviceRefusal::Taken + } else { + DeviceRefusal::NoMatch + }) + } + + /// Observe the first identity/class match across every live broker without + /// consulting or mutating any claim ledger. + pub async fn inspect_first( + &mut self, + selector: kore_rt::service::DeviceSelector, + ) -> Result { + for face in self.faces.iter_mut().flatten() { + match face.inspect(selector).await { + Ok(observed) => return Ok(observed), + Err(DeviceRefusal::NoMatch) => {} + Err(refusal) => return Err(refusal), + } + } + Err(DeviceRefusal::NoMatch) + } +} + +#[cfg(test)] +mod retirement_tests { + use super::*; + use kore_abi::cap::Cid; + use kore_abi::session::{ + DeviceClaimCloseReport, DeviceClaimCloseState as Close, QUIESCE_FLAG_IRQ_MASKED, + QUIESCE_VERDICT_PROVEN, QuiesceReport, + }; + + #[test] + fn retirement_never_advances_without_its_exact_phase_acknowledgement() { + let mut phase = ClaimRetirementProgress::Closing; + let close_request = DeviceRequest::CloseClaim { + index: 17, + grant: 0x100000033, + }; + let quiet = QuiesceReport { + rid: 22, + verdict: QUIESCE_VERDICT_PROVEN, + flags: QUIESCE_FLAG_IRQ_MASKED, + }; + for state in [ + Close::Busy, + Close::Pending, + Close::Mismatch, + Close::Untracked, + ] { + let result = phase.accept(DeviceResponse::ClaimClosed(DeviceClaimCloseReport { + state, + readers: if state == Close::Pending { 3 } else { 0 }, + })); + assert_eq!( + result.is_ok(), + matches!(state, Close::Busy | Close::Pending) + ); + assert_eq!(phase.request(17, Cid(0x100000033)), Some(close_request)); + } + assert_eq!( + phase.accept(DeviceResponse::ClaimQuiesced(quiet)), + Err(ClaimRetirementError::UnexpectedReply) + ); + assert_eq!(phase, ClaimRetirementProgress::Closing); + phase + .accept(DeviceResponse::ClaimClosed(DeviceClaimCloseReport { + state: Close::AdmissionDrained, + readers: 0, + })) + .unwrap(); + let containment_request = DeviceRequest::QuiesceClaim { + index: 17, + grant: 0x100000033, + }; + assert_eq!( + phase.request(17, Cid(0x100000033)), + Some(containment_request) + ); + for refusal in [DeviceRefusal::Busy, DeviceRefusal::Denied] { + assert_eq!( + phase.accept(DeviceResponse::Refused(refusal)), + Err(ClaimRetirementError::Refused(refusal)) + ); + assert_eq!( + phase.request(17, Cid(0x100000033)), + Some(containment_request) + ); + } + assert_eq!( + phase.accept(DeviceResponse::ClaimClosed(DeviceClaimCloseReport { + state: Close::AdmissionDrained, + readers: 0 + })), + Err(ClaimRetirementError::UnexpectedReply) + ); + phase.accept(DeviceResponse::ClaimQuiesced(quiet)).unwrap(); + assert_eq!(phase, ClaimRetirementProgress::Contained(quiet)); + let dma_request = DeviceRequest::RetireClaimDma { + index: 17, + grant: 0x100000033, + }; + assert_eq!(phase.request(17, Cid(0x100000033)), Some(dma_request)); + use kore_abi::session::{DeviceClaimDmaReport, DeviceClaimDmaState as Dma}; + for state in [ + Dma::WindowRetirementFailed, + Dma::EntryRetirementFailed, + Dma::AbortRetained, + Dma::EntryRetained, + Dma::ContainmentFailed, + Dma::Unbound, + ] { + let report = DeviceClaimDmaReport { state, removed: 0 }; + phase + .accept(DeviceResponse::ClaimDmaRetired(report)) + .unwrap(); + assert_eq!( + phase, + ClaimRetirementProgress::DmaPending { + fabric: quiet, + report + } + ); + assert_eq!(phase.request(17, Cid(0x100000033)), Some(dma_request)); + assert_eq!( + phase.accept(DeviceResponse::ClaimQuiesced(quiet)), + Err(ClaimRetirementError::UnexpectedReply) + ); + assert_eq!( + phase.accept(DeviceResponse::Refused(DeviceRefusal::Busy)), + Err(ClaimRetirementError::Refused(DeviceRefusal::Busy)) + ); + assert_eq!(phase.request(17, Cid(0x100000033)), Some(dma_request)); + } + let report = DeviceClaimDmaReport { + state: Dma::Retired, + removed: 0, + }; + phase + .accept(DeviceResponse::ClaimDmaRetired(report)) + .unwrap(); + assert_eq!( + phase, + ClaimRetirementProgress::DmaRetired { + fabric: quiet, + report + } + ); + assert_eq!(phase.request(17, Cid(0x100000033)), None); + } +} diff --git a/native/services/device/src/lib.rs b/native/services/device/src/lib.rs new file mode 100644 index 0000000..b0886c1 --- /dev/null +++ b/native/services/device/src/lib.rs @@ -0,0 +1,99 @@ +//! # kore-svc-device — the device ownership broker (`res/DEVICE_SERVICE.md` §10) +//! +//! The userland face over the machine's device inventory: a `SystemService` +//! holding the sole `role::ROOT_COMPLEX` endowment, whose whole vocabulary +//! is *"give me a device matching this selector"* and *"here it is back"*. +//! +//! ## Why this exists +//! +//! Enumeration is mechanism and belongs to the kernel. Deciding **which +//! driver owns which function** is policy and does not — but that is +//! exactly what the founding loop used to do: resolve a class triple, rank +//! candidates by reading xHCI `PORTSC` and `CAPLENGTH` in ring 0, and size +//! a DMA pool through a driver crate the kernel linked. Every new device +//! class widened the kernel, which is the opposite of what a microkernel +//! with userland drivers is for. +//! +//! This service is where that policy went. The kernel keeps enumeration, +//! IOMMU binding and capability minting; the broker reads what firmware +//! described ([`verbs::device_record`](kore_rt::cap::verbs::device_record)) +//! and hands one function to one driver. +//! +//! ## What it is *not* +//! +//! Not a god-object `DeviceService` (§1's ruling, which stands). Its +//! vocabulary is ownership arbitration plus read-only inventory observation — +//! and never `Read`/`Write`. A block read, a packet send and a display flip are +//! genuinely different operations, so each class keeps its own `SystemService` +//! vocabulary and this protocol never learns about any of them. `Inspect` +//! describes why an otherwise identical `Claim` would not match; it grants no +//! device authority and never consults the claim ledger. +//! +//! Not a nameserver either (§6). A driver does not look a device up by +//! name in an ambient directory: it holds a *capability* to this broker, +//! minted at its own founding, and a row that was founded without one can +//! claim nothing at all. +//! +//! It is also **off both hot paths**: the broker acts once per device, at +//! driver startup. No IRQ, no completion, and no byte of I/O ever re-enters +//! it (§3's control/data split). +//! +//! ## Entitlement — where it actually lives +//! +//! A claim carries the claimant's own selector, taken verbatim from its +//! `SystemService::DEVICES` declaration. That is not the broker trusting a +//! wire field: the declaration is compiled into the claimant's image, the +//! manifest generator cites it, and the boot digest gate seals the image — +//! so a row cannot ask for a class it did not declare without shipping a +//! different binary, which changes its measured digest. The manifest is +//! still the single source of truth for who may own what; only the +//! *matching* moved to ring 3. +//! Product-specific PCI rows additionally declare an exact vendor/device +//! identity; class-generic rows spell out `PciIdentitySelector::ANY`. The +//! broker applies that identity before reserving the affine row, because the +//! current index-only release cannot lawfully undo a wrong product claim. +//! +//! Two consequences worth stating rather than discovering: +//! +//! - **The client face is the entitlement.** Founding decides which rows +//! get one. A row with no broker face is not refused at claim time — it +//! has nothing to ask with. +//! - **This holds for *measured* rows.** When a post-boot spawned driver +//! (not a manifest row, not digest-sealed) becomes a claimant, the +//! structural caller identity has to become load-bearing, and the shape +//! it grows into is the input router's per-endpoint stamp +//! (`res/USERLAND.md` §5.8). That is deliberately not built ahead of its +//! first real claimant. +//! +//! ## Known limitation — release needs authority proof +//! +//! The current broker deliberately refuses [`DeviceRequest::Release`]: an +//! index is replayable data and cannot prove that the delivered device grant +//! and all activity below it are dead. Clearing the ledger from that integer +//! would permit two live siblings to drive one controller. A broker-owned +//! ancestor is not sufficient either: revocation invalidates remote tags and +//! quiesces DMA, but physical BAR/IRQ/port holdings are reaped in the holder's +//! task. Reclaim therefore needs kernel-enforced cross-holder teardown or an +//! affine receipt produced only after child-first local teardown. That is not +//! faked here: the broker keeps the entry quarantined rather than handing out +//! a device somebody may still be driving. +//! +//! One concern per module (`res/USERLAND.md` §1.2 rule 4): +//! +//! - [`service`] — the endowment, the vocabulary, the ledger, and the serve step; +//! - [`client`] — the client seam a driver claims through; +//! - [`wire`] — the fail-closed codecs over [`kore_abi::svc`]'s frozen forms. + +#![no_std] + +#[cfg(test)] +extern crate std; + +pub mod client; +pub mod service; +pub mod wire; + +pub use client::{DeviceClient, RingDevice}; +pub use service::{ + DeviceBroker, DeviceEndowment, DeviceRefusal, DeviceRequest, DeviceResponse, MAX_CLAIMS, +}; diff --git a/native/services/device/src/service/mod.rs b/native/services/device/src/service/mod.rs new file mode 100644 index 0000000..3e1cf26 --- /dev/null +++ b/native/services/device/src/service/mod.rs @@ -0,0 +1,2262 @@ +//! The inventory endowment, the arbitration vocabulary, the claim ledger, +//! the claimant-face mint, and the serve step. + +use kore_abi::device::{DeviceRecord, DeviceSpecies}; +use kore_abi::session::SlotRef; +use kore_abi::task::{FACE_RING_STRIDE, FACE_SUPPLY_BYTES, role}; +use kore_rt::cap::{PlatformBusToken, RootComplexToken, SupplyToken, verbs}; +use kore_rt::entry::FixedSlotBand; +use kore_rt::service::{ + DeviceSelector, GrantArg, MintedPairSpec, PciBarMask, PciInterruptRequirement, ReturnMode, + Serve, StartFailure, SystemService, +}; + +use kore_abi::cap::Rights; + +/// The most devices this broker tracks as claimed at once. +/// +/// A ceiling, not a guess: the founder's own per-row device ceiling is +/// four and a machine's driver rows are a manifest fact, so a claim table +/// this size covers every row the boot chain can found several times over. +/// It is heapless because the broker runs in the control plane of every +/// driver's startup — an allocation there would make bring-up depend on a +/// heap the broker has no reason to hold. +pub const MAX_CLAIMS: usize = 16; + +/// Rights carried by a broker-delivered device root. +/// +/// `REVOKE` belongs on the delivered **child**, not only on the broker's +/// inventory ancestor: it lets the affine receiver retire that child (and its +/// BAR/route/port descendants) when the token dies. Revoking the child cannot +/// reach upward into the broker's [`RootComplexToken`], so this preserves the +/// broker's reclamation authority while making receiver-side teardown lawful. +pub const DEVICE_GRANT_RIGHTS: Rights = verbs::DEVICE_OWNER_RIGHTS; + +/// Claimant faces the broker's [`role::FACE_SUPPLY`] band strides into — +/// the geometry contract's own quotient, so the count can never disagree +/// with the span the founder mints. +pub const FACE_COUNT: usize = (FACE_SUPPLY_BYTES / FACE_RING_STRIDE) as usize; + +/// The first minted face's [`Serve::via`] identity. Minted vias start at +/// [`MAX_SERVE_PAIRS`](kore_rt::svc::MAX_SERVE_PAIRS) — the host's rule +/// that a minted pair must never collide with a fixed pair — so face `k` +/// arrives stamped `FACE_VIA_BASE + k` and the claim ledger's owner +/// column tells claimants apart without a wire field. +pub const FACE_VIA_BASE: u16 = kore_rt::svc::MAX_SERVE_PAIRS as u16; + +/// The face legs' landing band: two base-arena slots per face (the +/// grant lane at `+2k`, the reply kick at `+2k+1`), claimed at start — +/// the exec convey-table shape, and derived from the same frozen runtime +/// layout (past the boot-image band, the serve scratch, and the +/// report + heap-carve tail). BASE arena on purpose: a rendezvous +/// landing must be a base-arena slot — the kernel denies an extension +/// landing, which is how this band's first draft (at +/// `ARENA_SLOT_CEILING`) refused every walking post. Fixed arithmetic, +/// not an allocator: a landed leg IS the slot [`MintedPairSpec`] then +/// cites for the face's life, so the slot map must be derivable from +/// the face index alone. +const FACE_LAND_BASE: u32 = + kore_abi::task::MAX_BOOT_HANDLES as u32 + kore_rt::entry::SERVE_SCRATCH_SLOTS as u32 + 2; +const FACE_LAND_END: u32 = FACE_LAND_BASE + 2 * FACE_COUNT as u32; + +/// A conveyed face leg must at least be postable by the broker. +const FACE_LEG_RIGHTS: Rights = Rights::ACCESS; + +const _: () = assert!( + kore_rt::svc::window_bytes::(kore_rt::svc::REQ_CAP, kore_rt::svc::REP_CAP) + <= FACE_RING_STRIDE as usize, + "a device pair must fit one face stride, or the mint would format past its slot" +); + +/// The client→service vocabulary — **ownership arbitration**. +/// +/// Deliberately no I/O verb of any kind: see the crate docs for why a generic +/// device request would be the god-object `res/DEVICE_SERVICE.md` §1 rules +/// out. `Inspect` is description-only and grants no authority. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeviceRequest { + /// Hand me the first unclaimed entry matching `selector`. + /// + /// Claiming twice with the same selector yields two *different* + /// entries, which is how a row that drives two controllers of one + /// class gets both without a nth-match field to get wrong. + Claim(DeviceSelector), + /// Describe the first entry with the selector's species and identity/class + /// tuple without applying its resource requirements or claiming it. + /// + /// This is the diagnostic twin of `Claim`: it lets a driver distinguish + /// absent hardware from an inventory row that lacks BAR, IOMMU, interrupt, + /// or endpoint-admission facts. The reply is data only. + Inspect(DeviceSelector), + /// Read one bounded firmware/admission page from this broker's held + /// root. Description only: no claim-ledger mutation or grant. + InspectI2c { row: u8 }, + /// Request that entry `index` be put back in the pool. + /// + /// The current broker refuses this index-only form: an integer does not + /// prove the granted capability (and all device activity beneath it) is + /// dead. The wire verb is reserved for an authority-backed return. + Release { + /// The inventory index a previous `Claim` answered with. + index: u32, + }, + /// Close admission to the exact PCI grant on this structural serving + /// face. This does not release the claim or free any physical resource. + CloseClaim { index: u32, grant: u64 }, + /// Issuer-owned fabric containment, only after exact admission drain. + QuiesceClaim { index: u32, grant: u64 }, + /// Retire registered DMA while retaining this exact claim. + RetireClaimDma { index: u32, grant: u64 }, + /// How many entries the inventory holds. + /// + /// Exists so a claimant can tell *"this machine has no such device"* + /// from *"somebody already has it"* — on a serial-less machine that + /// distinction is the difference between a hardware report and a + /// wiring bug. + Count, + /// Open a claimant-face provisioning window over face slot `face` of + /// the [`role::FACE_SUPPLY`] band — the founding parent's half of the + /// minted-pair handshake (`res/DEVICE_SERVICE.md` §10). The reply + /// reserves the slot; the face's two channel sends then arrive as + /// return-plane conveys, each an implicit [`Provision`](Self::Provision) + /// step, and the second completes the mint. + MintFace { + /// Which stride of the band to format and attach. + face: u32, + }, + /// **Implicit only** — synthesized by the serve host when a convey + /// lands on the broker's walking return post + /// ([`SvcProtocol::implicit_return_request`](kore_rt::svc::SvcProtocol)). + /// Never decoded from the request ring: a claimant writing it would be + /// forging the broker's own internal vocabulary, so the wire codec + /// refuses it fail-closed. + Provision, +} + +/// The reply vocabulary — **data only**. A granted device arrives as a +/// delivered token; the reply says which entry it was. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeviceResponse { + /// The device is yours. The token rides the grant channel. + Granted { + /// The function's diagnostic word + /// ([`DeviceRecord::diag`](kore_abi::device::DeviceRecord::diag)). + diag: u32, + /// PCI revision id — the claimant's `si_stepping` scalar. + revision: u8, + /// PCI vendor id, or zero for a platform device. + vendor: u16, + /// PCI device id, or zero for a platform device. + device: u16, + /// The inventory index handed over — name it to `Release`. + index: u32, + /// Which species the delivered token must be possessed as + /// (`claim_device` vs `claim_platform_device`). + species: DeviceSpecies, + }, + /// Only tracked admission drain; never physical retirement or release. + ClaimClosed(kore_abi::session::DeviceClaimCloseReport), + /// Fabric stop evidence only; no physical claim or backing is released. + ClaimQuiesced(kore_abi::session::QuiesceReport), + ClaimDmaRetired(kore_abi::session::DeviceClaimDmaReport), + /// Read-only inventory description; no token rides the reply. + Observed(DeviceObservation), + I2cObserved(kore_abi::device::I2cAdmissionPage), + /// The entry is back in the pool. + /// + /// Reserved for an authority-backed release protocol; the current broker + /// never emits this response to an index-only request. + Released, + /// The inventory's size. + Counted { + /// How many entries it holds. + len: u32, + }, + /// `MintFace` accepted: the face slot is reserved and the walking + /// return post now expects the face's two channel sends, grant lane + /// first. + Provisioning { + /// The reserved face slot, echoed. + face: u32, + }, + /// The first provisioning convey (the grant lane) is banked; one + /// more (the reply kick) completes the face. + Banked { + /// The face being provisioned, echoed. + face: u32, + }, + /// The face is live: its stride is formatted, the pair is attached, + /// and requests over it arrive stamped with this identity. + Minted { + /// The pair's [`Serve::via`] stamp (`FACE_VIA_BASE + face`). + via: u16, + }, + /// Refused — and a refusal emits **no** grant (fail-closed: no + /// authority leaves on an error path). + Refused(DeviceRefusal), +} + +/// The bounded facts returned by [`DeviceRequest::Inspect`]. +/// +/// The selector already carries identity and class, so the observation returns +/// only the facts needed to explain a failed claim. A granted wildcard claim +/// separately reports the selected PCI identity because the claimant did not +/// already know it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DeviceObservation { + pub index: u32, + pub species: DeviceSpecies, + pub flags: u8, + pub diag: u32, + pub revision: u8, +} + +impl From for DeviceObservation { + fn from(record: DeviceRecord) -> Self { + Self { + index: record.index, + species: record.species, + flags: record.flags, + diag: record.diag, + revision: record.revision, + } + } +} + +/// Why the broker refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeviceRefusal { + /// No entry on this machine matches the selector — a fact about the + /// hardware, and distinct from [`Taken`](Self::Taken) on purpose. + NoMatch, + /// Every match is already claimed. Also what a re-claim after a + /// claimant *crash* looks like, which is honest: the broker cannot + /// prove the holder is gone, so it will not hand out a device + /// somebody may still be driving (see the crate docs). + Taken, + /// An inventory observation or hand-out session refused — the kernel's + /// own gate or the report transport said no. + Denied, + /// A `Release` named an index nobody holds. + NotHeld, + /// This broker was founded without a [`role::FACE_SUPPLY`] band, so + /// it can mint no claimant face at all. + NoSupply, + /// `MintFace` named a face slot out of range or not vacant, or a + /// provisioning window is already open (one mint in flight — the + /// founding parent drives them in lockstep). + BadFace, + /// A return-plane convey arrived that no open provisioning window + /// expects, or carried something other than a channel send. The + /// half-built face was unwound with the refusal — its banked leg + /// retired, its slot vacant again (fail-closed: no face exists by + /// half). + BadConvey, + /// Claim admission has not drained, or containment currently holds its writer. + Busy, +} + +/// The `role::ROOT_COMPLEX` endowment: the sole inventory grant, claimed +/// out of the boot block. +/// +/// Deliberately **not** `role::DEVICE` beside it: a task that both +/// arbitrates devices and drives one is the god-object the design rules +/// out, and the profile is where that is made unrepresentable rather than +/// merely discouraged. +#[derive(Debug, kore_rt::Bundle)] +#[bundle(name = "device")] +pub struct DeviceEndowment { + /// **This row's root complex** (#572): every broker row endows + /// `ROOT_COMPLEX@0` = "my root", and the founder maps broker ordinal + /// k → the k-th discovered root — one endowment type, N instances. + /// A row whose root the machine did not discover is skipped at + /// founding, so a founded broker always holds a live root. + #[bundle(role = role::ROOT_COMPLEX)] + pub root: RootComplexToken, + /// The platform-device inventory — the roots' non-PCI twin (#572): + /// PS/2-class devices sit on no PCIe root, so their arbiter rides + /// broker row 0 alone; every other ordinal stamps vacant and this + /// resolves `None`. + #[bundle(role = role::PLATFORM_BUS, optional)] + pub platform: Option, + /// The claimed claimant-face band ([`role::FACE_SUPPLY`]), when this + /// row was founded with one — the broker's full-span serving sub of + /// the shelf-held band whose strides become minted claimant pairs. + /// + /// **Optional by design, not leniency**: the role is absent until the + /// founder grows its band arm, and a broker without it still + /// arbitrates claims — it just refuses every `MintFace` with + /// [`DeviceRefusal::NoSupply`]. A role that is *present* but does + /// not claim stays a hard [`kore_rt::service::BindError`]: a half-usable endowment must + /// refuse the row, never dilute into the absent case. + #[bundle( + role = role::FACE_SUPPLY, + optional + )] + /// Declaring the band beside the store is also what arms the founder's + /// claimant-band mint; the derive emits both coordinates into one profile. + pub faces: Option, +} + +/// One handed-out entry. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Claim { + /// The store-local inventory index. Which of this row's two + /// inventories (its root, or — ordinal 0 — the platform bus) it is + /// local to is the `species` beside it: the two stores vend disjoint + /// species, so no third discriminator exists to get wrong (#572). + index: u32, + /// Which species it was handed out as. + species: DeviceSpecies, + /// **Which serving pair claimed it** — the §10 endpoint stamp + /// ([`Serve::via`]), recorded so the books attribute every held + /// function to the driver row it went to. Structural: the founding + /// wired each claimant its own pair, so this is which face the claim + /// arrived on, never a wire field. Quarantine stays safe across + /// owners; a future authority-backed release will check it. + owner: u16, + /// Exact kernel publication observed after delivery. Missing correlation + /// retains the claim and withholds future retirement; it is not a rollback. + identity: Option, +} + +/// One claimant-face slot's lifecycle — the mint handshake's state +/// machine, kept beside the slot arithmetic that names its two landing +/// slots ([`FACE_LAND_BASE`]`+ 2k / + 2k + 1`). +/// +/// At most one face is ever off `Vacant`-or-`Live` (the open provisioning +/// window, [`DeviceBroker::minting`]): the founding parent drives mints +/// in lockstep, and refusing a second window keeps the walking landing +/// offer unambiguous. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum FaceState { + /// Nothing minted here. + #[default] + Vacant, + /// `MintFace` reserved the slot; no leg has landed yet. + Reserved, + /// The grant lane landed and is banked; the reply kick completes it. + Granted, + /// The face serves: stride formatted, pair attached, legs cited by + /// the live [`MintedPairSpec`]. + Live, +} + +/// Occupancy snapshot (the sibling services' `*Stats` shape). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DeviceStats { + /// Devices handed out since bind. + pub granted: u32, + /// Devices given back. + pub released: u32, + /// Claims refused for want of a match. + pub unmatched: u32, + /// Claims refused because every match was already held. + pub contended: u32, + /// Inventory observations or hand-outs the kernel's own gate refused — + /// **visible, never silent**: a failed read or grant is a wiring fault, + /// and it must not read as "no such device". + pub denied: u32, + /// Claimant faces currently live. + pub faces: u32, + /// Face mints that failed after opening — a bad convey, a dropped + /// attach — each one a wiring fault somebody should read. + pub face_faults: u32, +} + +/// Preserve the reason an authoritative inventory-count query failed in the +/// founder-visible service-start NACK. These are local diagnostics, not wire +/// refusal codes. +const fn inventory_start_detail(error: verbs::InventoryReadError) -> u64 { + match error { + verbs::InventoryReadError::Unavailable => 1, + verbs::InventoryReadError::Build => 2, + verbs::InventoryReadError::Transport => 3, + verbs::InventoryReadError::MalformedCompletion => 4, + verbs::InventoryReadError::MalformedReport => 5, + verbs::InventoryReadError::Refused(status) => 0x100 | status as u64, + } +} + +/// The device broker — the inventory and a claim ledger, and nothing else. +/// +/// It holds no device: the whole point is that it can only *hand one out*. +/// Its state is which entries are spoken for, which is the smallest thing +/// that makes "one device, one owner" true. +#[derive(Debug)] +pub struct DeviceBroker { + /// This row's root complex (#572 — one broker instance per root). + root: RootComplexToken, + /// The platform-device inventory, when this row was founded with one + /// (broker ordinal 0 alone). + platform: Option, + claims: heapless::Vec, + stats: DeviceStats, + /// The root's authoritative, boot-frozen inventory size; the platform + /// bus's rides beside it. + /// + /// Populated by the fallible pre-readiness hook rather than `bind`, whose + /// trait cannot report failure. `None` is retained as a defensive state: + /// neither `Count` nor `Claim` may turn a failed startup observation into + /// an empty inventory. + root_len: Option, + platform_len: Option, + /// The claimant-face band's possessed supply, when this row was + /// founded with one ([`DeviceEndowment::faces`]). + supply: Option, + /// The whole-band serving view — mapped R/W **once** at start for the + /// task's life (the image face table's map-once contract, which is + /// also what makes [`MintedPairSpec::new`]'s warrant citable). + face_window: Option>, + /// The face legs' claimed landing band (`FACE_LAND_BASE..FACE_LAND_END`) + /// — held for the task's life so nothing else can claim a slot a + /// live pair cites. + _face_land: Option, + /// Per-face lifecycle; see [`FaceState`]. + faces: [FaceState; FACE_COUNT], + /// The face with the open provisioning window, if any — at most one, + /// which is what keeps [`Self::pending_return_post`]'s offer + /// unambiguous. + minting: Option, + /// A completed mint's pair, parked for the serve host's attach pass + /// ([`SystemService::pending_pair_attach`]). + pending_attach: Option, +} + +impl DeviceBroker { + /// Occupancy snapshot. + pub fn stats(&self) -> DeviceStats { + self.stats + } + + /// Does `record` carry the selector's species and identity/class tuple? + /// + /// This deliberately excludes claimability and resource requirements. It + /// is the read-only inventory predicate used by `Inspect`; `matches` + /// layers the authority-bearing requirements over it. + fn describes(record: &DeviceRecord, selector: &DeviceSelector) -> bool { + match selector { + DeviceSelector::Pci(sel) => { + record.species == DeviceSpecies::Pci + && record.class == sel.class + && record.subclass == sel.subclass + && record.progif == sel.progif + && sel.identity.matches(record.vendor, record.device) + } + DeviceSelector::Platform { hid, .. } => { + record.species == DeviceSpecies::Platform + && record.class == DeviceRecord::PLATFORM_CLASS_HID + && record.hid() == *hid + } + DeviceSelector::PlatformI2cHid { .. } => { + record.species == DeviceSpecies::Platform + && record.class == DeviceRecord::PLATFORM_CLASS_I2C_HID + } + DeviceSelector::PciI2cHid { .. } => { + // The native input profile currently uses Intel LPSS + // reset/clock semantics, not arbitrary serial-bus IP. + // Linux's intel-lpss-pci.c LNL-M / PTL-P tables map these + // I2C functions to its 100 MHz ehl_i2c_info; a852/e452 are UART. + // https://github.com/torvalds/linux/blob/master/drivers/mfd/intel-lpss-pci.c + record.is_native_i2c_hid_candidate() + } + } + } + + /// Does `record` satisfy `selector`? + /// + /// This is the founding loop's `claimable` predicate, lifted into ring + /// 3 — minus the two xHCI register probes, which went where they + /// belong: the driver reads its own controller after it holds it. + fn matches(record: &DeviceRecord, selector: &DeviceSelector) -> bool { + match selector { + DeviceSelector::Pci(sel) => { + if !Self::describes(record, selector) { + return false; + } + // Endpoint admission is positive. In particular, absence + // of the bridge bit is not proof that firmware released a + // display controller to us. + if !record.satisfies(DeviceRecord::DRIVER_ENDPOINT) + || record.satisfies(DeviceRecord::NOT_DRIVER_ENDPOINT) + || record.flags & DeviceRecord::IS_BRIDGE != 0 + { + return false; + } + let present_bars = (record.diag & 0x3f) as u8; + let user_bars = ((record.diag >> 6) & 0x3f) as u8; + let has_user_bar0 = user_bars & PciBarMask::BAR0.bits() != 0; + if record.msix_table_size > DeviceRecord::MAX_MSIX_TABLE_ENTRIES + || user_bars & !present_bars != 0 + || record.satisfies(DeviceRecord::HAS_BAR0) != has_user_bar0 + || (record.satisfies(DeviceRecord::HAS_REMAPPED_MSIX) + && (!record.satisfies(DeviceRecord::HAS_REMAPPED_MSI) + || record.msix_table_size == 0)) + { + // Inventory is kernel-produced, so any impossible PCI + // geometry is corruption or producer/decoder drift. Do + // not consume the affine row under contradictory facts. + return false; + } + let required_bars = sel.bars.bits(); + if user_bars & required_bars != required_bars { + return false; + } + let mut required = DeviceRecord::DRIVER_ENDPOINT; + if required_bars & PciBarMask::BAR0.bits() != 0 { + // Keep the legacy flag and the exact diagnostic bitmap + // mutually consistent for the common BAR0 case. + required |= DeviceRecord::HAS_BAR0; + } + if sel.dma { + required |= DeviceRecord::HAS_BINDING; + } + required |= match sel.interrupts { + PciInterruptRequirement::None => 0, + PciInterruptRequirement::MsiFamily => DeviceRecord::HAS_REMAPPED_MSI, + PciInterruptRequirement::Msix(block) => { + let Some(end) = block.first_entry().checked_add(block.count()) else { + return false; + }; + if end > record.msix_table_size { + return false; + } + DeviceRecord::HAS_REMAPPED_MSIX + } + }; + if matches!(sel.interrupts, PciInterruptRequirement::Msix(_)) { + // Every strict MSI-X block is also an MSI-family route; + // demanding both catches an internally inconsistent + // inventory instead of laundering it into a grant. + required |= DeviceRecord::HAS_REMAPPED_MSI; + } + record.satisfies(required) + } + DeviceSelector::Platform { irq, dma, .. } => { + let mut required = 0; + if *irq { + required |= DeviceRecord::HAS_LINE; + } + if *dma { + required |= DeviceRecord::HAS_BINDING; + } + Self::describes(record, selector) && record.satisfies(required) + } + DeviceSelector::PlatformI2cHid { irq } => { + Self::describes(record, selector) + && (!*irq || record.satisfies(DeviceRecord::HAS_LINE)) + } + DeviceSelector::PciI2cHid { irq } => { + let resources = DeviceSelector::Pci(kore_rt::service::PciSelector { + class: 0x0c, + subclass: 0x80, + progif: 0, + identity: kore_rt::service::PciIdentitySelector::ANY, + bars: PciBarMask::BAR0, + dma: false, + interrupts: PciInterruptRequirement::None, + }); + Self::describes(record, selector) + && record.diag & DeviceRecord::PCI_DIAG_I2C_HID != 0 + && Self::matches(record, &resources) + && (!*irq || record.satisfies(DeviceRecord::HAS_LINE)) + } + } + } + + /// Scan one inventory for a description-only match. + fn inspect_with( + len: u32, + selector: &DeviceSelector, + mut read: impl FnMut(u32) -> Result, + ) -> Result { + for index in 0..len { + let record = read(index).map_err(|_| DeviceRefusal::Denied)?; + if Self::describes(&record, selector) { + return Ok(record); + } + } + Err(DeviceRefusal::NoMatch) + } + + /// Scan one inventory's authoritative bound through a supplied record + /// reader. + /// + /// The closure seams keep host tests honest without fabricating an + /// affine token. A read error wins over `NoMatch`/`Taken`: those + /// hardware facts are valid only after every required entry was + /// observed. + fn scan_with( + len: u32, + is_claimed: impl Fn(u32) -> bool, + selector: &DeviceSelector, + mut read: impl FnMut(u32) -> Result, + ) -> Result { + let mut any = false; + for index in 0..len { + let record = read(index).map_err(|_| DeviceRefusal::Denied)?; + if !Self::matches(&record, selector) { + continue; + } + any = true; + if !is_claimed(record.index) { + return Ok(record); + } + } + Err(if any { + DeviceRefusal::Taken + } else { + DeviceRefusal::NoMatch + }) + } + + /// Is `(species, index)` already spoken for? The ledger is + /// per-store, and the species IS the store here: this row's root + /// vends PCI functions, its (ordinal-0-only) platform bus the rest. + fn claimed(&self, species: DeviceSpecies, index: u32) -> bool { + self.claims + .iter() + .any(|claim| claim.species == species && claim.index == index) + } + + /// The arena slot holding the store that vends `species` — the grant + /// subject a claim derives from. + fn store_slot(&self, species: DeviceSpecies) -> Option { + match species { + DeviceSpecies::Pci => Some(self.root.slot()), + DeviceSpecies::Platform => self.platform.as_ref().map(|token| token.slot()), + } + } + + /// Find the first unclaimed entry matching `selector` — this row's + /// root for a PCI selector, its platform bus for a platform one. The + /// species split routes wholesale: a store that cannot vend the + /// selector's species is never scanned, and a broker holding no + /// platform bus answers a platform selector `NoMatch` (drivers probe + /// every broker; ordinal 0 answers). + fn find(&self, selector: &DeviceSelector) -> Result { + match selector { + DeviceSelector::Pci(_) | DeviceSelector::PciI2cHid { .. } => { + let len = self.root_len.ok_or(DeviceRefusal::Denied)?; + Self::scan_with( + len, + |index| self.claimed(DeviceSpecies::Pci, index), + selector, + |index| verbs::device_record(&self.root, index), + ) + } + DeviceSelector::Platform { .. } | DeviceSelector::PlatformI2cHid { .. } => { + let Some(bus) = self.platform.as_ref() else { + return Err(DeviceRefusal::NoMatch); + }; + let len = self.platform_len.ok_or(DeviceRefusal::Denied)?; + Self::scan_with( + len, + |index| self.claimed(DeviceSpecies::Platform, index), + selector, + |index| verbs::device_record(bus, index), + ) + } + } + } + + /// Observe a matching record without consulting or mutating the claim + /// ledger. Observation is metadata, never evidence that authority is free. + fn inspect(&self, selector: &DeviceSelector) -> Result { + match selector { + DeviceSelector::Pci(_) | DeviceSelector::PciI2cHid { .. } => { + let len = self.root_len.ok_or(DeviceRefusal::Denied)?; + Self::inspect_with(len, selector, |index| { + verbs::device_record(&self.root, index) + }) + } + DeviceSelector::Platform { .. } | DeviceSelector::PlatformI2cHid { .. } => { + let Some(bus) = self.platform.as_ref() else { + return Err(DeviceRefusal::NoMatch); + }; + let len = self.platform_len.ok_or(DeviceRefusal::Denied)?; + Self::inspect_with(len, selector, |index| verbs::device_record(bus, index)) + } + } + } + + fn inspect_i2c(&self, row: u8) -> Result { + // Use the already-held root; no temporary device capability is + // manufactured merely to diagnose why a device could not be claimed. + Self::inspect_i2c_with( + row, + |row| verbs::i2c_admission_page(&self.root, row).map_err(|_| DeviceRefusal::Denied), + |index| self.claimed(DeviceSpecies::Pci, index), + ) + } + + fn inspect_i2c_with( + row: u8, + mut read: impl FnMut(u8) -> Result, + claimed: impl FnOnce(u32) -> bool, + ) -> Result { + use kore_abi::device::{ + I2C_ADMISSION_BUILD_ROW, I2C_ADMISSION_CLAIMED, I2C_ADMISSION_PAGES_PER_CANDIDATE, + }; + let mut page = read(row)?; + if page.row != row || !page.valid() { + return Err(DeviceRefusal::Denied); + } + if row != 0 + && row < I2C_ADMISSION_BUILD_ROW + && (row - 1) % I2C_ADMISSION_PAGES_PER_CANDIDATE == 0 + { + let identity = read(row + 2)?; + if identity.row != row + 2 || !identity.valid() { + return Err(DeviceRefusal::Denied); + } + if claimed(identity.words[0]) { + page.words[1] |= u32::from(I2C_ADMISSION_CLAIMED); + } + } + Ok(page) + } + + /// Undo the reservation made immediately before a device grant was + /// emitted. `ServeHost` is single-step: no later request can interleave + /// between that push and the grant-session outcome, so the reservation is + /// exactly the ledger tail. + fn rollback_dropped_grant( + claims: &mut heapless::Vec, + stats: &mut DeviceStats, + response: &mut DeviceResponse, + ) { + let DeviceResponse::Granted { index, species, .. } = *response else { + debug_assert!(false, "device grant drop without a Granted reply"); + return; + }; + + // The owner stamp is not re-derivable from the reply, so the tail + // match is on the grant's own identity (index + species); the + // single-step lifecycle makes the tail the only candidate either way. + let expected = Some((index, species)); + if claims.last().map(|c| (c.index, c.species)) != expected { + // Never remove unrelated authority in a release build if the + // single-step lifecycle is violated. The reply still becomes a + // refusal below, while the unexpected row stays conservatively + // reserved instead of becoming double-claimable. + debug_assert!( + false, + "the dropped device grant must be the ledger's newest reservation" + ); + } else { + let _ = claims.pop(); + stats.granted = stats.granted.saturating_sub(1); + } + stats.denied = stats.denied.saturating_add(1); + *response = DeviceResponse::Refused(DeviceRefusal::Denied); + } + + fn owned_claim_identity( + claims: &heapless::Vec, + index: u32, + owner: u16, + grant: u64, + ) -> Option { + claims + .iter() + .find(|claim| { + claim.index == index && claim.species == DeviceSpecies::Pci && claim.owner == owner + })? + .identity + .filter(|identity| identity.grant.0 == grant) + } + + fn retain_claim_identity( + claims: &mut heapless::Vec, + index: u32, + owner: u16, + identity: kore_abi::session::DeviceClaimReport, + ) { + if identity.generation == 0 { + return; + } + if let Some(claim) = claims.iter_mut().find(|claim| { + claim.index == index && claim.species == DeviceSpecies::Pci && claim.owner == owner + }) { + // The committed callback runs once. Never replace existing + // correlation with an observation of a later generation. + if claim.identity.is_none() { + claim.identity = Some(identity); + } + } + } + + /// Classify an index-only release without mutating the ledger. + /// + /// A known index is `Denied`, not released: possession of the integer says + /// nothing about whether the live device capability was revoked. An + /// unknown index retains the more specific `NotHeld` answer. + fn unproven_release_refusal( + claims: &heapless::Vec, + index: u32, + ) -> DeviceRefusal { + if claims.iter().any(|claim| claim.index == index) { + DeviceRefusal::Denied + } else { + DeviceRefusal::NotHeld + } + } + + /// Face `k`'s two leg landing slots: the grant lane, then the reply + /// kick — fixed arithmetic over the claimed band, because a landed + /// leg IS the slot the live [`MintedPairSpec`] cites for the face's + /// whole life. + fn face_leg_slots(face: u32) -> (SlotRef, SlotRef) { + let base = FACE_LAND_BASE + 2 * face; + (SlotRef::from_wire(base), SlotRef::from_wire(base + 1)) + } + + /// The pure `MintFace` admission — the [`scan_with`](Self::scan_with) + /// discipline: all of the policy, none of the metal, so the host + /// tests pin the refusal matrix. + fn face_open_check( + has_supply: bool, + faces: &[FaceState], + minting: Option, + face: u32, + ) -> Result<(), DeviceRefusal> { + if !has_supply { + return Err(DeviceRefusal::NoSupply); + } + if minting.is_some() { + return Err(DeviceRefusal::BadFace); + } + match faces.get(face as usize) { + Some(FaceState::Vacant) => Ok(()), + _ => Err(DeviceRefusal::BadFace), + } + } + + /// Tear a half-built (or attach-refused) face back to `Vacant`: + /// retire whatever landed in its leg slots, close any open window, + /// and count the fault where the shell's status poll can see it. + /// Retiring a never-landed slot is a no-op refusal, so this is + /// lawful from every phase. + fn unwind_face(&mut self, face: u32) { + let (grant, kick) = Self::face_leg_slots(face); + let _ = verbs::retire_slot_outcome(grant); + let _ = verbs::retire_slot_outcome(kick); + if let Some(state) = self.faces.get_mut(face as usize) { + *state = FaceState::Vacant; + } + if self.minting == Some(face) { + self.minting = None; + } + self.stats.face_faults = self.stats.face_faults.saturating_add(1); + } +} + +impl SystemService for DeviceBroker { + const NAME: &'static str = "device"; + type Endowment = DeviceEndowment; + + /// **Walking** — the exec convey-table mode: the return plane's + /// landings rotate through service-owned slots, because what lands + /// there (a mint's channel legs today; a returned device capability + /// when the authority-backed `Release` arrives) becomes long-lived + /// state at slots this service must cite. Lawful exactly since the + /// single-pair flip: the serve host refuses a Retained/Walking + /// service founded with a POSTED extra pair (the rotation lives on + /// pair 0 alone), and the static second pair is gone. + /// + /// The walking offer stream ([`Self::pending_return_post`]) is + /// host-global while return posts are per-fixed-pair, so a + /// supply-founded broker must serve exactly ONE fixed pair — two + /// hosts would race one offered landing. `start_on_metal` refuses + /// that shape closed. + const RETURN_MODE: ReturnMode = ReturnMode::Walking; + + fn bind(endowment: DeviceEndowment) -> Self { + Self { + root: endowment.root, + platform: endowment.platform, + claims: heapless::Vec::new(), + stats: DeviceStats::default(), + root_len: None, + platform_len: None, + supply: endowment.faces, + face_window: None, + _face_land: None, + faces: [FaceState::Vacant; FACE_COUNT], + minting: None, + pending_attach: None, + } + } + + /// Fetch each held inventory's count once through the explicit + /// successful count report. A refused/garbled observation on either + /// held store prevents readiness and carries its exact bounded + /// category in the founder NACK instead of advertising a truncated + /// machine. + /// + /// A founded face band is readied here too — geometry gate, the + /// map-once serving view, the leg landing band — and fail-closed at + /// start like the image face table: a vocabulary the service cannot + /// serve refuses readiness, not first mint. Details `0x200..` in the + /// founder NACK, clear of the inventory's `1..=5`/`0x1xx` band. + fn start_on_metal(&mut self, rt: &kore_rt::entry::Rt) -> Result<(), StartFailure> { + let len = verbs::inventory_len(&self.root) + .map_err(|error| StartFailure::new(inventory_start_detail(error)))?; + self.root_len = Some(len); + if let Some(bus) = self.platform.as_ref() { + let len = verbs::inventory_len(bus) + .map_err(|error| StartFailure::new(inventory_start_detail(error)))?; + self.platform_len = Some(len); + } + if let Some(supply) = self.supply.as_ref() { + // The walking-offer ambiguity guard (see `RETURN_MODE`): a + // face-minting broker serves exactly one fixed pair. + if ::SERVE_PAIRS > 1 { + return Err(StartFailure::new(0x205)); + } + if supply.len() != FACE_SUPPLY_BYTES { + return Err(StartFailure::new(0x200)); + } + let win = verbs::map_held_rw(supply.slot()).ok_or_else(|| StartFailure::new(0x201))?; + if win.len() < FACE_SUPPLY_BYTES as usize { + return Err(StartFailure::new(0x202)); + } + let band = rt + .claim_fixed_slot_band(FACE_LAND_BASE, FACE_LAND_END) + .ok_or_else(|| StartFailure::new(0x204))?; + self.face_window = Some(win.cast::()); + self._face_land = Some(band); + } + Ok(()) + } + + /// Answer one arbitration request. + /// + /// The only verb that emits authority is `Claim`, it emits exactly one + /// grant, and every refusal path emits none — the D4 bounded-emission + /// discipline, and the fail-closed rule that no authority leaves on an + /// error path. + fn serve(&mut self, req: DeviceRequest, sv: &mut Serve<'_, DeviceResponse>) { + match req { + // The machine total across every held inventory. A held store + // whose count never resolved refuses rather than summing a + // silently truncated machine. + // This ROW's total: its root's functions plus — ordinal 0 — + // the platform devices. A held store whose count never + // resolved refuses rather than summing a silently truncated + // inventory. + DeviceRequest::Count => { + let mut total = 0u32; + let mut whole = true; + match self.root_len { + Some(len) => total = total.saturating_add(len), + None => whole = false, + } + if self.platform.is_some() { + match self.platform_len { + Some(len) => total = total.saturating_add(len), + None => whole = false, + } + } + if whole { + sv.reply(DeviceResponse::Counted { len: total }); + } else { + self.stats.denied = self.stats.denied.saturating_add(1); + sv.reply(DeviceResponse::Refused(DeviceRefusal::Denied)); + } + } + DeviceRequest::InspectI2c { row } => match self.inspect_i2c(row) { + Ok(page) => sv.reply(DeviceResponse::I2cObserved(page)), + Err(reason) => sv.reply(DeviceResponse::Refused(reason)), + }, + DeviceRequest::Inspect(selector) => match self.inspect(&selector) { + Ok(record) => sv.reply(DeviceResponse::Observed(record.into())), + // A negative observation is not a refused claim, so it must + // not contaminate the claim-only `unmatched` counter. + Err(DeviceRefusal::NoMatch) => { + sv.reply(DeviceResponse::Refused(DeviceRefusal::NoMatch)); + } + Err(DeviceRefusal::Denied) => { + self.stats.denied = self.stats.denied.saturating_add(1); + sv.reply(DeviceResponse::Refused(DeviceRefusal::Denied)); + } + // `inspect` cannot consult the claim ledger or speak the + // face/release vocabulary. Preserve fail-closed behavior if + // that invariant is ever broken. + Err(_) => { + debug_assert!(false, "read-only inspection returned an invalid refusal"); + self.stats.denied = self.stats.denied.saturating_add(1); + sv.reply(DeviceResponse::Refused(DeviceRefusal::Denied)); + } + }, + DeviceRequest::Claim(selector) => { + let record = match self.find(&selector) { + Ok(hit) => hit, + Err(refusal) => { + match refusal { + DeviceRefusal::NoMatch => self.stats.unmatched += 1, + DeviceRefusal::Taken => self.stats.contended += 1, + // `find` never speaks the face vocabulary; a + // claim refused for any other reason is the + // kernel's own gate saying no. + DeviceRefusal::Denied + | DeviceRefusal::NotHeld + | DeviceRefusal::NoSupply + | DeviceRefusal::BadFace + | DeviceRefusal::BadConvey + | DeviceRefusal::Busy => { + self.stats.denied = self.stats.denied.saturating_add(1) + } + } + sv.reply(DeviceResponse::Refused(refusal)); + return; + } + }; + // Record the claim BEFORE emitting the grant. The ledger + // is what makes "one device, one owner" true, and a + // successful grant this service then failed to record + // would let the next claimant be handed the same + // controller — two drivers programming one device's + // registers. Refusing a device that was never handed out + // is recoverable; handing one out twice is not. + // The grant subject is the MATCHED store's token — the + // claim derives below the root (or bus) that actually owns + // the function, so its lineage revoke is per-root (#572). + let Some(subject) = self.store_slot(record.species) else { + self.stats.denied = self.stats.denied.saturating_add(1); + sv.reply(DeviceResponse::Refused(DeviceRefusal::Denied)); + return; + }; + if self + .claims + .push(Claim { + index: record.index, + species: record.species, + owner: sv.via(), + identity: None, + }) + .is_err() + { + self.stats.denied += 1; + sv.reply(DeviceResponse::Refused(DeviceRefusal::Denied)); + return; + } + // The device's own authority: use and program it, derive its + // BARs/routes/IOVA windows, and retire this delivered child + // when the affine token dies. `REVOKE` reaches downward from + // this child; it cannot retire the broker's store ancestor. + sv.grant( + subject, + DEVICE_GRANT_RIGHTS, + GrantArg::Device { + index: record.index, + species: record.species, + }, + ); + self.stats.granted += 1; + let (vendor, device) = match record.species { + DeviceSpecies::Pci => (record.vendor, record.device), + DeviceSpecies::Platform => (0, 0), + }; + sv.reply(DeviceResponse::Granted { + index: record.index, + species: record.species, + diag: record.diag, + revision: record.revision, + vendor, + device, + }); + } + DeviceRequest::CloseClaim { index, grant } => { + let Some(identity) = + Self::owned_claim_identity(&self.claims, index, sv.via(), grant) + else { + sv.reply(DeviceResponse::Refused(DeviceRefusal::Denied)); + return; + }; + let response = match verbs::close_device_claim(&self.root, index, identity) { + Ok(report) => DeviceResponse::ClaimClosed(report), + Err(_) => DeviceResponse::Refused(DeviceRefusal::Denied), + }; + // Even Drained keeps the ledger and backing reserved. Loss of + // this reply permits only an exact retry of the same grant. + sv.reply(response); + } + DeviceRequest::QuiesceClaim { index, grant } => { + let Some(identity) = + Self::owned_claim_identity(&self.claims, index, sv.via(), grant) + else { + sv.reply(DeviceResponse::Refused(DeviceRefusal::Denied)); + return; + }; + let response = match verbs::quiesce_device_claim(&self.root, index, identity) { + Ok(report) => DeviceResponse::ClaimQuiesced(report), + Err(verbs::InventoryReadError::Refused( + kore_abi::session::Status::WouldBlock, + )) => DeviceResponse::Refused(DeviceRefusal::Busy), + Err(_) => DeviceResponse::Refused(DeviceRefusal::Denied), + }; + // Even Drained keeps the ledger and backing reserved. Loss of + // this reply permits only an exact retry of the same grant. + sv.reply(response); + } + DeviceRequest::RetireClaimDma { index, grant } => { + let Some(identity) = + Self::owned_claim_identity(&self.claims, index, sv.via(), grant) + else { + sv.reply(DeviceResponse::Refused(DeviceRefusal::Denied)); + return; + }; + let response = match verbs::retire_device_claim_dma(&self.root, index, identity) { + Ok(report) => DeviceResponse::ClaimDmaRetired(report), + Err(verbs::InventoryReadError::Refused( + kore_abi::session::Status::WouldBlock, + )) => DeviceResponse::Refused(DeviceRefusal::Busy), + Err(_) => DeviceResponse::Refused(DeviceRefusal::Denied), + }; + // Even Drained keeps the ledger and backing reserved. Loss of + // this reply permits only an exact retry of the same grant. + sv.reply(response); + } + DeviceRequest::Release { index } => { + let refusal = Self::unproven_release_refusal(&self.claims, index); + if refusal == DeviceRefusal::Denied { + self.stats.denied = self.stats.denied.saturating_add(1); + } + sv.reply(DeviceResponse::Refused(refusal)); + } + DeviceRequest::MintFace { face } => { + if let Err(refusal) = Self::face_open_check( + self.face_window.is_some(), + &self.faces, + self.minting, + face, + ) { + sv.reply(DeviceResponse::Refused(refusal)); + return; + } + self.faces[face as usize] = FaceState::Reserved; + self.minting = Some(face); + sv.reply(DeviceResponse::Provisioning { face }); + } + DeviceRequest::Provision => { + // The host synthesizes this step only after a convey landed + // on a landing THIS service offered, and an offer exists + // only while a window is open — so the arm below is the + // defensive shape, not a reachable protocol state. + let Some(face) = self.minting else { + sv.reply(DeviceResponse::Refused(DeviceRefusal::BadConvey)); + return; + }; + let (grant_slot, kick_slot) = Self::face_leg_slots(face); + match self.faces.get(face as usize).copied() { + // The grant lane landed. + Some(FaceState::Reserved) => { + if !verbs::slot_is_chan_send(grant_slot, FACE_LEG_RIGHTS) { + self.unwind_face(face); + sv.reply(DeviceResponse::Refused(DeviceRefusal::BadConvey)); + return; + } + self.faces[face as usize] = FaceState::Granted; + sv.reply(DeviceResponse::Banked { face }); + } + // The reply kick landed — complete the face. + Some(FaceState::Granted) => { + if !verbs::slot_is_chan_send(kick_slot, FACE_LEG_RIGHTS) { + self.unwind_face(face); + sv.reply(DeviceResponse::Refused(DeviceRefusal::BadConvey)); + return; + } + let Some(win) = self.face_window else { + self.unwind_face(face); + sv.reply(DeviceResponse::Refused(DeviceRefusal::NoSupply)); + return; + }; + // Format face `k`'s ring stride through the + // whole-band view: zero, then pair-format — the + // image face table's recycled-slot hygiene. + // SAFETY: the view is live for the task's life + // (mapped once at start, length gated ≥ + // FACE_SUPPLY_BYTES there); this face is not Live, + // so no attached pair aliases this stride + // (occupancy is the exclusion discipline); the + // `&mut` ends before the spec is exposed. + let stride = FACE_RING_STRIDE as usize; + let pair_bytes = kore_rt::svc::window_bytes::( + kore_rt::svc::REQ_CAP, + kore_rt::svc::REP_CAP, + ); + let base = unsafe { win.as_ptr().add(face as usize * stride) }; + let region = unsafe { core::slice::from_raw_parts_mut(base, stride) }; + region.fill(0); + if kore_rt::svc::init_window::( + &mut region[..pair_bytes], + kore_rt::svc::REQ_CAP, + kore_rt::svc::REP_CAP, + ) + .is_err() + { + self.unwind_face(face); + sv.reply(DeviceResponse::Refused(DeviceRefusal::Denied)); + return; + } + let mut grant_tx = [None; kore_rt::service::MAX_GRANTS]; + grant_tx[0] = Some(grant_slot); + let via = FACE_VIA_BASE + face as u16; + // SAFETY: `region` above proved the mapping live + // and in-bounds for this stride; the Live claim + // below makes it exclusive to this pair until an + // unwind detaches it; the view outlives the host + // (task-life map); `init_window` just formatted it. + let spec = unsafe { + MintedPairSpec::new( + core::ptr::NonNull::new_unchecked(base), + pair_bytes, + grant_tx, + via, + Some(kick_slot), + ) + }; + self.pending_attach = Some(spec); + self.faces[face as usize] = FaceState::Live; + self.minting = None; + self.stats.faces = self.stats.faces.saturating_add(1); + sv.reply(DeviceResponse::Minted { via }); + } + _ => { + self.unwind_face(face); + sv.reply(DeviceResponse::Refused(DeviceRefusal::BadConvey)); + } + } + } + } + } + + fn pending_return_post(&self) -> Option { + let face = self.minting?; + let (grant, kick) = Self::face_leg_slots(face); + match self.faces.get(face as usize)? { + FaceState::Reserved => Some(grant), + FaceState::Granted => Some(kick), + _ => None, + } + } + + fn pending_pair_attach(&mut self) -> Option { + self.pending_attach.take() + } + + fn pair_attached(&mut self, via: u16, attached: bool) { + if attached { + return; + } + // A refused attach on a face this service minted: unwind it whole + // — legs retired, slot vacant — so a later `MintFace` can rebuild + // it instead of leasing against a dead spec. + let Some(face) = via.checked_sub(FACE_VIA_BASE).map(u32::from) else { + return; + }; + if self.faces.get(face as usize) == Some(&FaceState::Live) { + self.unwind_face(face); + self.stats.faces = self.stats.faces.saturating_sub(1); + } + } + + fn grants_committed(&mut self, response: &DeviceResponse, emitted: usize, via: u16) { + debug_assert_eq!(emitted, 1, "a device Claim emits exactly one grant"); + let DeviceResponse::Granted { + index, + species: DeviceSpecies::Pci, + .. + } = *response + else { + return; + }; + // Observation failure cannot turn a delivered grant into a refusal or + // clear its reservation. A future retire must refuse missing identity. + if let Ok(identity) = verbs::device_claim(&self.root, index) { + Self::retain_claim_identity(&mut self.claims, index, via, identity); + } + } + + fn grants_dropped(&mut self, response: &mut DeviceResponse, emitted: usize) { + debug_assert_eq!(emitted, 1, "a device Claim emits exactly one grant"); + Self::rollback_dropped_grant(&mut self.claims, &mut self.stats, response); + } +} + +/// The service-arena slot a claimed device is delivered into is the +/// claimant's business, not the broker's — this alias exists only so the +/// client seam and the driver host name the same thing. +pub type ClaimSlot = SlotRef; + +#[cfg(test)] +mod face_tests { + use super::*; + + /// The mint admission refuses everything but a vacant slot on a + /// supply-founded broker with no window already open — each arm a + /// distinct wiring fault the founding parent must see loudly. + #[test] + fn the_mint_admission_matrix_is_fail_closed() { + let vacant = [FaceState::Vacant; FACE_COUNT]; + assert_eq!( + DeviceBroker::face_open_check(true, &vacant, None, 0), + Ok(()) + ); + assert_eq!( + DeviceBroker::face_open_check(true, &vacant, None, FACE_COUNT as u32 - 1), + Ok(()) + ); + // No supply beats every other refusal: the broker cannot mint at + // all, and saying "bad face" would send the founder hunting the + // wrong bug. + assert_eq!( + DeviceBroker::face_open_check(false, &vacant, None, 0), + Err(DeviceRefusal::NoSupply) + ); + // Out of range. + assert_eq!( + DeviceBroker::face_open_check(true, &vacant, None, FACE_COUNT as u32), + Err(DeviceRefusal::BadFace) + ); + // A window already open — mints are lockstep. + assert_eq!( + DeviceBroker::face_open_check(true, &vacant, Some(1), 0), + Err(DeviceRefusal::BadFace) + ); + // Occupied, in any non-vacant phase. + for state in [FaceState::Reserved, FaceState::Granted, FaceState::Live] { + let mut faces = vacant; + faces[3] = state; + assert_eq!( + DeviceBroker::face_open_check(true, &faces, None, 3), + Err(DeviceRefusal::BadFace) + ); + } + } + + /// The leg landing slots are pure arithmetic over the claimed band — + /// derivable from the face index alone, disjoint across faces, and + /// inside the claimed interval (the slot a live `MintedPairSpec` + /// cites must be one the band actually owns). + #[test] + fn the_leg_slot_map_is_disjoint_and_in_band() { + let mut seen = [false; 2 * FACE_COUNT]; + for face in 0..FACE_COUNT as u32 { + let (grant, kick) = DeviceBroker::face_leg_slots(face); + for slot in [grant, kick] { + let index = slot.index(); + assert!(index >= FACE_LAND_BASE && index < FACE_LAND_END); + let offset = (index - FACE_LAND_BASE) as usize; + assert!(!seen[offset], "two faces share landing slot {index}"); + seen[offset] = true; + } + } + assert!(seen.iter().all(|s| *s), "the band has no unused slot"); + } + + /// Minted vias start past every fixed pair and stay unique per face — + /// the host's collision rule, pinned where the numbers are chosen. + #[test] + fn minted_vias_clear_the_fixed_pairs() { + assert!(FACE_VIA_BASE as usize >= kore_rt::svc::MAX_SERVE_PAIRS); + for face in 0..FACE_COUNT as u16 { + let via = FACE_VIA_BASE + face; + assert!((via as usize) >= kore_rt::svc::MAX_SERVE_PAIRS); + assert_eq!(via - FACE_VIA_BASE, face); + } + } + + /// The band geometry quotient: the face count times the stride is + /// exactly the supply span the founder mints (`FACE_SUPPLY_BYTES`), + /// and the landing band holds exactly two slots per face. + #[test] + fn the_face_geometry_is_the_contracts_quotient() { + assert_eq!(FACE_COUNT as u64 * FACE_RING_STRIDE, FACE_SUPPLY_BYTES); + assert_eq!(FACE_LAND_END - FACE_LAND_BASE, 2 * FACE_COUNT as u32); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use kore_rt::service::{PciBarMask, PciIdentitySelector, PciSelector}; + + #[test] + fn global_i2c_build_page_never_queries_or_marks_a_candidate_claim() { + use kore_abi::device::{ + AmlBuildDiagnostic, AmlBuildErrorClass, AmlBuildStage, I2C_ADMISSION_BUILD_ROW, + I2cAdmissionPage, + }; + let page = I2cAdmissionPage { + row: I2C_ADMISSION_BUILD_ROW, + words: AmlBuildDiagnostic::failure( + AmlBuildStage::SsdtLoad, + AmlBuildErrorClass::Opcode, + u32::MAX, + 0x5bff, + ) + .unwrap() + .words(), + }; + let mut reads = 0; + let observed = DeviceBroker::inspect_i2c_with( + page.row, + |row| { + assert_eq!(row, I2C_ADMISSION_BUILD_ROW); + reads += 1; + Ok(page) + }, + |_| panic!("a firmware failure has no claim-ledger identity"), + ); + assert_eq!(observed, Ok(page)); + assert_eq!(reads, 1); + } + + #[test] + fn last_i2c_candidate_still_uses_its_own_inventory_index_for_claim_state() { + use kore_abi::device::{I2C_ADMISSION_BUILD_ROW, I2C_ADMISSION_CLAIMED, I2cAdmissionPage}; + let row = I2C_ADMISSION_BUILD_ROW - 3; + let mut reads = 0; + let observed = DeviceBroker::inspect_i2c_with( + row, + |requested| { + let page = match reads { + 0 => { + assert_eq!(requested, row); + I2cAdmissionPage { + row, + words: [0x00c9_0000, 0], + } + } + 1 => { + assert_eq!(requested, row + 2); + I2cAdmissionPage { + row: row + 2, + words: [19, 0xa851_8086], + } + } + _ => panic!("only candidate and identity reads are needed"), + }; + reads += 1; + Ok(page) + }, + |index| { + assert_eq!(index, 19); + true + }, + ) + .unwrap(); + assert_eq!(reads, 2); + assert_eq!(observed.words[1], u32::from(I2C_ADMISSION_CLAIMED)); + } + + fn pci(class: u8, subclass: u8, progif: u8, flags: u8, index: u32) -> DeviceRecord { + DeviceRecord { + species: DeviceSpecies::Pci, + index, + vendor: 0x8086, + device: 0x9a13, + class, + subclass, + progif, + locator: 0xa0, + segment: 0, + flags, + msix_table_size: 0, + diag: if flags & DeviceRecord::HAS_BAR0 != 0 { + (1 << 0) | (1 << 6) + } else { + 0 + }, + revision: 0, + } + } + + fn xhci_selector(dma: bool, msi: bool) -> DeviceSelector { + DeviceSelector::Pci(PciSelector { + class: 0x0c, + subclass: 0x03, + progif: 0x30, + identity: PciIdentitySelector::ANY, + bars: PciBarMask::BAR0, + dma, + interrupts: if msi { + PciInterruptRequirement::MsiFamily + } else { + PciInterruptRequirement::None + }, + }) + } + + fn pci_i2c(flags: u8, index: u32) -> DeviceRecord { + let mut record = pci(0x0c, 0x80, 0, flags, index); + record.vendor = 0x8086; + record.device = 0xa878; + record.diag |= DeviceRecord::PCI_DIAG_I2C_HID; + record + } + + const LIVE: u8 = DeviceRecord::DRIVER_ENDPOINT + | DeviceRecord::HAS_BAR0 + | DeviceRecord::HAS_BINDING + | DeviceRecord::HAS_REMAPPED_MSI; + + /// The class triple must match on all three bytes — a selector that + /// matched on class alone would hand a USB driver an IDE controller. + #[test] + fn the_whole_class_triple_is_load_bearing() { + let sel = xhci_selector(true, true); + assert!(DeviceBroker::matches(&pci(0x0c, 0x03, 0x30, LIVE, 0), &sel)); + for wrong in [ + pci(0x01, 0x03, 0x30, LIVE, 0), + pci(0x0c, 0x06, 0x30, LIVE, 0), + pci(0x0c, 0x03, 0x20, LIVE, 0), + ] { + assert!( + !DeviceBroker::matches(&wrong, &sel), + "a near-miss on any of the three bytes is not a match" + ); + } + } + + /// A product-specific driver must reject a class-compatible function it + /// cannot program. This comparison happens before the affine grant leaves + /// the broker; rejecting it after claim would strand the ledger because + /// index-only release is intentionally unproven. + #[test] + fn exact_pci_identity_filters_before_claim() { + let mut exact = xhci_selector(true, true); + let DeviceSelector::Pci(selector) = &mut exact else { + unreachable!(); + }; + selector.identity = PciIdentitySelector::exact(0x8086, 0x9a13); + + assert!(DeviceBroker::matches( + &pci(0x0c, 0x03, 0x30, LIVE, 0), + &exact, + )); + let mut wrong_vendor = pci(0x0c, 0x03, 0x30, LIVE, 0); + wrong_vendor.vendor = 0x10de; + assert!(!DeviceBroker::matches(&wrong_vendor, &exact)); + let mut wrong_device = pci(0x0c, 0x03, 0x30, LIVE, 0); + wrong_device.device ^= 1; + assert!(!DeviceBroker::matches(&wrong_device, &exact)); + + assert!( + DeviceBroker::matches(&wrong_vendor, &xhci_selector(true, true)), + "explicit ANY preserves class-driver matching" + ); + } + + #[test] + fn inspection_reports_the_identity_even_when_claim_requirements_fail() { + let mut selector = xhci_selector(true, true); + let DeviceSelector::Pci(pci_selector) = &mut selector else { + unreachable!(); + }; + pci_selector.identity = PciIdentitySelector::exact(0x8086, 0x9a13); + + let wrong = pci(0x0c, 0x03, 0x30, LIVE, 0); + let mut withheld = pci(0x0c, 0x03, 0x30, DeviceRecord::NOT_DRIVER_ENDPOINT, 1); + withheld.device = 0x9a13; + let records = [ + DeviceRecord { + device: 0xffff, + ..wrong + }, + withheld, + ]; + + assert!(!DeviceBroker::matches(&withheld, &selector)); + let observed = DeviceBroker::inspect_with(records.len() as u32, &selector, |index| { + Ok::<_, ()>(records[index as usize]) + }) + .expect("identity-only inspection sees the withheld record"); + assert_eq!(observed.index, 1); + assert_eq!(observed.flags, DeviceRecord::NOT_DRIVER_ENDPOINT); + } + + /// A driver that DMAs demands a live IOMMU binding, one that parks on + /// interrupts demands a remapped MSI-family route, and **every** PCI + /// claimant demands a usable BAR0. + /// + /// The BAR0 requirement is unconditional on purpose: it is not in the + /// selector, so nothing a claimant writes can opt out of it, and a + /// device with no user-mappable registers is one no userland driver + /// can drive at all. + #[test] + fn the_required_facts_are_demanded_not_hoped_for() { + let sel = xhci_selector(true, true); + assert!(!DeviceBroker::matches( + &pci(0x0c, 0x03, 0x30, LIVE & !DeviceRecord::HAS_BINDING, 0), + &sel, + )); + assert!(!DeviceBroker::matches( + &pci(0x0c, 0x03, 0x30, LIVE & !DeviceRecord::HAS_REMAPPED_MSI, 0), + &sel, + )); + // A selector that asks for neither still needs BAR0. + let bare = xhci_selector(false, false); + assert!(DeviceBroker::matches( + &pci( + 0x0c, + 0x03, + 0x30, + DeviceRecord::DRIVER_ENDPOINT | DeviceRecord::HAS_BAR0, + 0, + ), + &bare, + )); + assert!( + !DeviceBroker::matches(&pci(0x0c, 0x03, 0x30, 0, 0), &bare), + "no user-mappable BAR0 means no userland driver, whatever the selector says" + ); + } + + /// A PCI-to-PCI bridge is never anybody's device, even when it + /// class-matches and carries every flag. + #[test] + fn a_bridge_is_never_a_drivers_device() { + assert!(!DeviceBroker::matches( + &pci(0x0c, 0x03, 0x30, LIVE | DeviceRecord::IS_BRIDGE, 0), + &xhci_selector(true, true), + )); + } + + #[test] + fn endpoint_admission_is_positive_and_msix_pair_is_exact() { + let mut xe = xhci_selector(true, true); + let DeviceSelector::Pci(selector) = &mut xe else { + unreachable!(); + }; + selector.class = 0x03; + selector.subclass = 0x00; + selector.progif = 0x00; + selector.identity = PciIdentitySelector::exact(0x8086, 0x56a0); + selector.interrupts = + PciInterruptRequirement::Msix(kore_rt::service::PciMsixBlock::ENTRIES_0_1); + + let strict = LIVE | DeviceRecord::HAS_REMAPPED_MSIX; + let dg2 = |flags| { + let mut record = pci(0x03, 0x00, 0x00, flags, 0); + record.device = 0x56a0; + record.msix_table_size = if flags & DeviceRecord::HAS_REMAPPED_MSIX != 0 { + 2 + } else { + 0 + }; + record + }; + assert!(DeviceBroker::matches(&dg2(strict), &xe)); + assert!( + !DeviceBroker::matches(&dg2(LIVE), &xe), + "ordinary MSI-family eligibility cannot satisfy a strict MSI-X-pair owner" + ); + assert!( + !DeviceBroker::matches( + &dg2(strict & !DeviceRecord::DRIVER_ENDPOINT | DeviceRecord::NOT_DRIVER_ENDPOINT,), + &xe, + ), + "a matching DG2 identity is not proof that firmware released it" + ); + assert!( + !DeviceBroker::matches(&dg2(strict | DeviceRecord::NOT_DRIVER_ENDPOINT), &xe), + "contradictory positive and negative endpoint facts fail closed" + ); + + let mut one_entry = dg2(strict); + one_entry.msix_table_size = 1; + let mut second = dg2(strict); + second.index = 1; + assert!( + !DeviceBroker::matches(&one_entry, &xe), + "a strict two-entry owner cannot consume a one-entry table" + ); + let records = [one_entry, second]; + let selected = DeviceBroker::scan_with( + records.len() as u32, + |_| false, + &xe, + |index| Ok::<_, ()>(records[index as usize]), + ) + .expect("the one-entry record is skipped before the pair-capable row is selected"); + assert_eq!(selected.index, 1); + } + + #[test] + fn exact_bar_mask_filters_before_the_claim_ledger() { + let mut ivpu = xhci_selector(true, true); + let DeviceSelector::Pci(selector) = &mut ivpu else { + unreachable!(); + }; + selector.class = 0x12; + selector.subclass = 0; + selector.progif = 0; + selector.bars = PciBarMask::BAR0.union(PciBarMask::BAR4); + + let bar0_only = pci(0x12, 0, 0, LIVE, 0); + let mut complete = pci(0x12, 0, 0, LIVE, 1); + complete.diag |= (1 << 4) | (1 << (6 + 4)); + assert!(!DeviceBroker::matches(&bar0_only, &ivpu)); + assert!(DeviceBroker::matches(&complete, &ivpu)); + + let records = [bar0_only, complete]; + let selected = DeviceBroker::scan_with( + records.len() as u32, + |_| false, + &ivpu, + |index| Ok::<_, ()>(records[index as usize]), + ) + .expect("the incomplete BAR geometry is rejected before selection"); + assert_eq!(selected.index, 1); + } + + #[test] + fn impossible_inventory_geometry_fails_closed() { + let selector = xhci_selector(true, true); + + let mut oversized_msix = pci(0x0c, 0x03, 0x30, LIVE, 0); + oversized_msix.msix_table_size = DeviceRecord::MAX_MSIX_TABLE_ENTRIES + 1; + assert!(!DeviceBroker::matches(&oversized_msix, &selector)); + + let mut user_bar_without_present_bar = pci(0x0c, 0x03, 0x30, LIVE, 0); + user_bar_without_present_bar.diag &= !(1 << 0); + assert!(!DeviceBroker::matches( + &user_bar_without_present_bar, + &selector + )); + + let mut stale_bar0_flag = pci(0x0c, 0x03, 0x30, LIVE, 0); + stale_bar0_flag.diag &= !(1 << 6); + assert!(!DeviceBroker::matches(&stale_bar0_flag, &selector)); + + let mut orphan_msix_fact = pci(0x0c, 0x03, 0x30, LIVE | DeviceRecord::HAS_REMAPPED_MSIX, 0); + assert!(!DeviceBroker::matches(&orphan_msix_fact, &selector)); + orphan_msix_fact.msix_table_size = 1; + orphan_msix_fact.flags &= !DeviceRecord::HAS_REMAPPED_MSI; + assert!(!DeviceBroker::matches(&orphan_msix_fact, &selector)); + } + + /// The two species never cross: a PCI selector does not match a + /// firmware-described entry, and a platform selector does not match a + /// PCI function that happens to share its id halves. + #[test] + fn the_species_never_cross() { + let mut plat = pci(0, 0, 0, DeviceRecord::HAS_LINE, 3); + plat.species = DeviceSpecies::Platform; + // `vendor | device << 16` — PNP0303, the i8042 keyboard id. + plat.vendor = 0xd041; + plat.device = 0x0303; + + let by_hid = DeviceSelector::Platform { + hid: 0x0303_d041, + irq: true, + dma: false, + }; + assert!(DeviceBroker::matches(&plat, &by_hid)); + assert!(!DeviceBroker::matches(&plat, &xhci_selector(true, true))); + + let mut pci_twin = plat; + pci_twin.species = DeviceSpecies::Pci; + assert!( + !DeviceBroker::matches(&pci_twin, &by_hid), + "a PCI function with the same id halves is not a platform device" + ); + } + + /// A platform selector that demands an interrupt line refuses a device + /// firmware described without one — the row would otherwise found a + /// driver that can only poll, and this arc ships no polled mode. + #[test] + fn a_line_less_platform_device_refuses_a_row_that_needs_one() { + let mut plat = pci(0, 0, 0, 0, 3); + plat.species = DeviceSpecies::Platform; + plat.vendor = 0xd041; + plat.device = 0x0303; + assert!(!DeviceBroker::matches( + &plat, + &DeviceSelector::Platform { + hid: 0x0303_d041, + irq: true, + dma: false, + }, + )); + assert!(DeviceBroker::matches( + &plat, + &DeviceSelector::Platform { + hid: 0x0303_d041, + irq: false, + dma: false, + }, + )); + } + + #[test] + fn a_dma_platform_selector_requires_a_live_binding() { + let mut plat = pci(0, 0, 0, DeviceRecord::HAS_LINE, 3); + plat.species = DeviceSpecies::Platform; + plat.vendor = 0xd041; + plat.device = 0x0d15; + let selector = DeviceSelector::Platform { + hid: 0x0d15_d041, + irq: true, + dma: true, + }; + assert!(!DeviceBroker::matches(&plat, &selector)); + plat.flags |= DeviceRecord::HAS_BINDING; + assert!(DeviceBroker::matches(&plat, &selector)); + } + + #[test] + fn topology_selected_i2c_never_aliases_an_acpi_id_row() { + let mut i2c = pci(0, 0, 0, DeviceRecord::HAS_LINE, 4); + i2c.species = DeviceSpecies::Platform; + i2c.class = DeviceRecord::PLATFORM_CLASS_I2C_HID; + + let selector = DeviceSelector::PlatformI2cHid { irq: true }; + assert!(DeviceBroker::matches(&i2c, &selector)); + + let mut legacy = i2c; + legacy.class = DeviceRecord::PLATFORM_CLASS_HID; + legacy.vendor = 0xd041; + legacy.device = 0x0303; + assert!(!DeviceBroker::matches(&legacy, &selector)); + assert!(DeviceBroker::matches( + &legacy, + &DeviceSelector::Platform { + hid: 0x0303_d041, + irq: true, + dma: false, + }, + )); + + i2c.flags &= !DeviceRecord::HAS_LINE; + assert!(!DeviceBroker::matches(&i2c, &selector)); + assert!(DeviceBroker::matches( + &i2c, + &DeviceSelector::PlatformI2cHid { irq: false }, + )); + } + + #[test] + fn pci_i2c_selection_requires_correlation_for_the_exact_function() { + let selector = DeviceSelector::PciI2cHid { irq: false }; + let pio = DeviceRecord::DRIVER_ENDPOINT | DeviceRecord::HAS_BAR0; + let mut unrelated = pci(0x0c, 0x80, 0, pio, 0); + // Other nonzero diagnostic bits (including the PM sample) are not + // initialized firmware correlation for this function. + unrelated.diag |= 0x7800_0000; + unrelated.locator = 0x78; + let mut withheld = pci_i2c(DeviceRecord::NOT_DRIVER_ENDPOINT, 1); + withheld.locator = 0x79; + let mut available = pci_i2c(pio, 2); + available.locator = 0x7a; + let records = [unrelated, withheld, available]; + + assert!(!DeviceBroker::describes(&unrelated, &selector)); + assert!(!DeviceBroker::matches(&withheld, &selector)); + assert!(DeviceBroker::matches(&available, &selector)); + let selected = DeviceBroker::scan_with( + records.len() as u32, + |_| false, + &selector, + |index| Ok::<_, ()>(records[index as usize]), + ) + .expect("a correlated PIO endpoint needs neither DMA nor MSI"); + assert_eq!(selected.index, available.index); + + let observed = DeviceBroker::inspect_with(records.len() as u32, &selector, |index| { + Ok::<_, ()>(records[index as usize]) + }) + .expect("inspection reports the first correlated function even when withheld"); + assert_eq!(observed.index, withheld.index); + assert_eq!(observed.flags, DeviceRecord::NOT_DRIVER_ENDPOINT); + + assert_eq!( + DeviceBroker::scan_with( + records.len() as u32, + |index| index == available.index, + &selector, + |index| Ok::<_, ()>(records[index as usize]), + ), + Err(DeviceRefusal::Taken), + "an unclaimed but uncorrelated sibling is never a fallback", + ); + } + + #[test] + fn an_uncorrelated_supported_i2c_controller_is_observable_but_not_claimable() { + let mut record = pci_i2c( + DeviceRecord::DRIVER_ENDPOINT | DeviceRecord::HAS_BAR0 | DeviceRecord::HAS_LINE, + 0, + ); + record.diag &= !(1 << 31); + for irq in [false, true] { + let selector = DeviceSelector::PciI2cHid { irq }; + assert!(DeviceBroker::describes(&record, &selector)); + assert!(!DeviceBroker::matches(&record, &selector)); + assert_eq!( + DeviceBroker::inspect_with(1, &selector, |_| Ok::<_, ()>(record)) + .unwrap() + .index, + record.index, + ); + assert_eq!( + DeviceBroker::scan_with(1, |_| false, &selector, |_| Ok::<_, ()>(record)), + Err(DeviceRefusal::NoMatch), + ); + } + } + + #[test] + fn pci_i2c_line_requirement_uses_the_controller_line_fact() { + let mut record = pci_i2c( + DeviceRecord::DRIVER_ENDPOINT | DeviceRecord::HAS_BAR0 | DeviceRecord::HAS_REMAPPED_MSI, + 0, + ); + let optional = DeviceSelector::PciI2cHid { irq: false }; + let required = DeviceSelector::PciI2cHid { irq: true }; + assert!(DeviceBroker::matches(&record, &optional)); + assert!( + !DeviceBroker::matches(&record, &required), + "MSI eligibility cannot substitute for a firmware controller line", + ); + record.flags &= !DeviceRecord::HAS_REMAPPED_MSI; + record.flags |= DeviceRecord::HAS_LINE; + assert!(DeviceBroker::matches(&record, &required)); + + let mut platform = record; + platform.species = DeviceSpecies::Platform; + assert!(!DeviceBroker::matches(&platform, &optional)); + platform.class = DeviceRecord::PLATFORM_CLASS_I2C_HID; + platform.subclass = 0; + platform.diag = DeviceRecord::PLATFORM_I2C_DESIGNWARE; + assert!(DeviceBroker::matches( + &platform, + &DeviceSelector::PlatformI2cHid { irq: true }, + )); + assert!(!DeviceBroker::matches(&platform, &required)); + } + + #[test] + fn pci_i2c_correlation_does_not_bypass_endpoint_or_bar_admission() { + let selector = DeviceSelector::PciI2cHid { irq: false }; + let base = pci_i2c(DeviceRecord::DRIVER_ENDPOINT | DeviceRecord::HAS_BAR0, 0); + for corrupt in [ + |record: &mut DeviceRecord| record.flags &= !DeviceRecord::DRIVER_ENDPOINT, + |record: &mut DeviceRecord| record.flags |= DeviceRecord::NOT_DRIVER_ENDPOINT, + |record: &mut DeviceRecord| record.flags |= DeviceRecord::IS_BRIDGE, + |record: &mut DeviceRecord| record.flags &= !DeviceRecord::HAS_BAR0, + |record: &mut DeviceRecord| record.diag &= !(1 << 6), + |record: &mut DeviceRecord| record.diag &= !(1 << 0), + |record: &mut DeviceRecord| record.diag = DeviceRecord::PCI_DIAG_I2C_HID, + |record: &mut DeviceRecord| record.class ^= 1, + |record: &mut DeviceRecord| record.subclass ^= 1, + |record: &mut DeviceRecord| record.progif = 1, + |record: &mut DeviceRecord| { + record.msix_table_size = DeviceRecord::MAX_MSIX_TABLE_ENTRIES + 1; + }, + ] { + let mut record = base; + corrupt(&mut record); + assert!( + !DeviceBroker::matches(&record, &selector), + "record {record:?}" + ); + } + } + + #[test] + fn native_i2c_profile_does_not_reset_unknown_controller_wrappers() { + let selector = DeviceSelector::PciI2cHid { irq: false }; + let mut record = pci_i2c(DeviceRecord::DRIVER_ENDPOINT | DeviceRecord::HAS_BAR0, 0); + for device in [ + 0xa850, 0xa851, 0xa878, 0xa879, 0xa87a, 0xa87b, 0xe450, 0xe451, 0xe478, 0xe479, 0xe47a, + 0xe47b, + ] { + record.device = device; + assert!(DeviceBroker::matches(&record, &selector)); + } + for (vendor, device) in [ + (0x8086, 0xa852), + (0x8086, 0x9d60), + (0x1022, 0xa878), + (0x8086, 0xe452), + (0x8086, 0xe425), + (0x8086, 0xe427), + (0x8086, 0xe430), + (0x8086, 0xe446), + (0x8086, 0xe477), + (0x8086, 0xe47c), + (0x1022, 0xe450), + ] { + record.vendor = vendor; + record.device = device; + assert!(!DeviceBroker::describes(&record, &selector)); + assert!(!DeviceBroker::matches(&record, &selector)); + } + } + + #[test] + fn panther_lake_i2c_inspection_preserves_claim_requirements() { + let optional = DeviceSelector::PciI2cHid { irq: false }; + let routed = DeviceSelector::PciI2cHid { irq: true }; + // Biggertron's two PCI controllers, as captured by Ubuntu. Candidate + // evidence must survive a refusal; it does not confer claim authority. + for device in [0xe450, 0xe451] { + let mut record = pci_i2c(DeviceRecord::DRIVER_ENDPOINT | DeviceRecord::HAS_BAR0, 0); + record.device = device; + assert!(record.is_native_i2c_hid_candidate()); + assert!(DeviceBroker::describes(&record, &routed)); + assert!(DeviceBroker::matches(&record, &optional)); + assert!(!DeviceBroker::matches(&record, &routed)); + record.flags |= DeviceRecord::HAS_LINE; + assert!(DeviceBroker::matches(&record, &routed)); + for corrupt in [ + |r: &mut DeviceRecord| r.flags &= !DeviceRecord::DRIVER_ENDPOINT, + |r: &mut DeviceRecord| r.flags |= DeviceRecord::NOT_DRIVER_ENDPOINT, + |r: &mut DeviceRecord| r.flags |= DeviceRecord::IS_BRIDGE, + |r: &mut DeviceRecord| r.flags &= !DeviceRecord::HAS_BAR0, + |r: &mut DeviceRecord| r.diag = 0, + ] { + let mut refused = record; + corrupt(&mut refused); + assert!(DeviceBroker::describes(&refused, &routed)); + assert!(!DeviceBroker::matches(&refused, &optional)); + } + for corrupt in [ + |r: &mut DeviceRecord| r.species = DeviceSpecies::Platform, + |r: &mut DeviceRecord| r.class ^= 1, + |r: &mut DeviceRecord| r.subclass ^= 1, + |r: &mut DeviceRecord| r.progif = 1, + ] { + let mut wrong_kind = record; + corrupt(&mut wrong_kind); + assert!(!wrong_kind.is_native_i2c_hid_candidate()); + assert!(!DeviceBroker::describes(&wrong_kind, &optional)); + } + } + } + + /// The claim ledger is speculative until the grant session commits. A + /// refusal must make the same entry immediately claimable again and turn + /// the reply into a grant-less denial rather than leaking `Granted`. + #[test] + fn a_dropped_grant_rolls_back_its_claim_and_reply() { + let claim = Claim { + index: 7, + species: DeviceSpecies::Pci, + owner: 1, + identity: None, + }; + let mut claims = heapless::Vec::::new(); + assert!(claims.push(claim).is_ok()); + let mut stats = DeviceStats { + granted: 1, + ..DeviceStats::default() + }; + let mut response = DeviceResponse::Granted { + index: claim.index, + species: claim.species, + diag: 0, + revision: 0, + vendor: 0, + device: 0, + }; + + DeviceBroker::rollback_dropped_grant(&mut claims, &mut stats, &mut response); + + assert!( + claims.is_empty(), + "the un-effected device is claimable again" + ); + assert_eq!(stats.granted, 0, "no device was handed out"); + assert_eq!(stats.denied, 1, "the transport refusal is visible"); + assert_eq!(response, DeviceResponse::Refused(DeviceRefusal::Denied)); + } + + #[test] + fn committed_correlation_preserves_owner_species_and_first_generation() { + use kore_abi::{cap::Cid, session::DeviceClaimReport}; + let mut claims = heapless::Vec::::new(); + for species in [DeviceSpecies::Pci, DeviceSpecies::Platform] { + claims + .push(Claim { + index: 7, + species, + owner: 19, + identity: None, + }) + .unwrap(); + } + let identity = DeviceClaimReport { + generation: 3, + grant: Cid(0x1234_0000_0007), + }; + DeviceBroker::retain_claim_identity(&mut claims, 7, 18, identity); + DeviceBroker::retain_claim_identity(&mut claims, 8, 19, identity); + DeviceBroker::retain_claim_identity( + &mut claims, + 7, + 19, + DeviceClaimReport { + generation: 0, + grant: Cid::NULL, + }, + ); + assert!(claims.iter().all(|claim| claim.identity.is_none())); + assert_eq!(claims.len(), 2, "failed observation never undoes delivery"); + DeviceBroker::retain_claim_identity(&mut claims, 7, 19, identity); + assert_eq!(claims[0].identity, Some(identity)); + assert_eq!( + claims[1].identity, None, + "platform index is a different namespace" + ); + DeviceBroker::retain_claim_identity( + &mut claims, + 7, + 19, + DeviceClaimReport { + generation: 5, + grant: Cid(9), + }, + ); + assert_eq!( + claims[0].identity, + Some(identity), + "correlation is never silently replaced" + ); + assert_eq!( + DeviceBroker::owned_claim_identity(&claims, 7, 19, identity.grant.0), + Some(identity) + ); + assert_eq!( + DeviceBroker::owned_claim_identity(&claims, 7, 18, identity.grant.0), + None + ); + assert_eq!( + DeviceBroker::owned_claim_identity(&claims, 8, 19, identity.grant.0), + None + ); + assert_eq!( + DeviceBroker::owned_claim_identity(&claims, 7, 19, identity.grant.0 + 1), + None + ); + assert_eq!(claims.len(), 2); + assert_eq!( + DeviceBroker::unproven_release_refusal(&claims, 7), + DeviceRefusal::Denied + ); + } + + /// A returned device is an affine owner, not a borrowed view: its root + /// must be able to retire itself, and every child verb's requested rights + /// must be a subset of the root grant under Σ5 monotonicity. + #[test] + fn a_device_grant_can_retire_itself_and_derive_owned_children() { + assert!(DEVICE_GRANT_RIGHTS.contains(Rights::REVOKE)); + assert!(DEVICE_GRANT_RIGHTS.contains(verbs::DEVICE_CHILD_RIGHTS)); + assert!(DEVICE_GRANT_RIGHTS.contains(Rights::DERIVE)); + } + + /// An index is replayable data, not affine authority. Until release carries + /// proof of revocation, it must leave the reservation intact rather than + /// make the same hardware claimable beside a still-live sibling grant. + #[test] + fn an_index_only_release_never_clears_a_claim() { + let claim = Claim { + index: 7, + species: DeviceSpecies::Pci, + owner: 1, + identity: None, + }; + let mut claims = heapless::Vec::::new(); + assert!(claims.push(claim).is_ok()); + + assert_eq!( + DeviceBroker::unproven_release_refusal(&claims, claim.index), + DeviceRefusal::Denied, + ); + assert_eq!(claims.as_slice(), &[claim], "the ledger remains reserved"); + assert_eq!( + DeviceBroker::unproven_release_refusal(&claims, 99), + DeviceRefusal::NotHeld, + ); + } + + /// A failed first read is infrastructure denial, never evidence that the + /// machine lacks a matching controller. + #[test] + fn a_record_failure_never_becomes_no_match() { + let claims = heapless::Vec::::new(); + let result = DeviceBroker::scan_with( + 1, + |index| claims.iter().any(|c| c.index == index), + &xhci_selector(true, true), + |_| Err::(()), + ); + assert_eq!(result, Err(DeviceRefusal::Denied)); + } + + /// A failure after an already-held match is still denial, not proof that + /// every matching device was taken; a later unclaimed entry may exist. + #[test] + fn a_record_failure_never_becomes_taken() { + let mut claims = heapless::Vec::::new(); + assert!( + claims + .push(Claim { + index: 0, + species: DeviceSpecies::Pci, + owner: 0, + identity: None, + }) + .is_ok() + ); + let result = DeviceBroker::scan_with( + 2, + |index| claims.iter().any(|c| c.index == index), + &xhci_selector(true, true), + |index| match index { + 0 => Ok(pci(0x0c, 0x03, 0x30, LIVE, 0)), + _ => Err(()), + }, + ); + assert_eq!(result, Err(DeviceRefusal::Denied)); + } + + /// `NoMatch` and `Taken` remain meaningful after a complete successful + /// scan; only incomplete observation loses the right to assert them. + #[test] + fn a_complete_scan_preserves_hardware_and_contention_refusals() { + let empty = heapless::Vec::::new(); + let no_match = DeviceBroker::scan_with( + 1, + |index| empty.iter().any(|c| c.index == index), + &xhci_selector(true, true), + |_| Ok::<_, ()>(pci(0x01, 0x08, 0x02, LIVE, 0)), + ); + assert_eq!(no_match, Err(DeviceRefusal::NoMatch)); + + let mut held = heapless::Vec::::new(); + assert!( + held.push(Claim { + index: 0, + species: DeviceSpecies::Pci, + owner: 0, + identity: None, + }) + .is_ok() + ); + let taken = DeviceBroker::scan_with( + 1, + |index| held.iter().any(|c| c.index == index), + &xhci_selector(true, true), + |_| Ok::<_, ()>(pci(0x0c, 0x03, 0x30, LIVE, 0)), + ); + assert_eq!(taken, Err(DeviceRefusal::Taken)); + } +} diff --git a/native/services/device/src/wire.rs b/native/services/device/src/wire.rs new file mode 100644 index 0000000..9cb921c --- /dev/null +++ b/native/services/device/src/wire.rs @@ -0,0 +1,1448 @@ +//! Fail-closed codecs over [`kore_abi::svc`]'s frozen device-broker forms +//! (`res/USERLAND.md` §5.3). +//! +//! Every decode is total and refuses anything out of vocabulary: an +//! unknown op, an unknown tag, a refusal byte on a success tag, a grant +//! count that does not match the tag, or an invalid reserved/geometry field. The peer wrote +//! garbage (or a future revision), and in *this* protocol a wrong guess +//! would hand a device to the wrong task — so the codec is deliberately as +//! strict as the power service's rather than as lenient as swap's. + +use kore_abi::device::DeviceSpecies; +use kore_abi::svc::{DeviceRepWire, DeviceReqWire}; +use kore_rt::service::{ + DeviceSelector, PciBarMask, PciIdentitySelector, PciInterruptRequirement, PciMsixBlock, + PciSelector, +}; +use kore_rt::svc::SvcProtocol; + +use crate::service::{ + DeviceBroker, DeviceObservation, DeviceRefusal, DeviceRequest, DeviceResponse, +}; + +/// A selector's `dma` bit, as it rides [`DeviceReqWire::flags`]. +/// +/// The wire carries the *requirement* the claimant states, not the record +/// flags it will be matched against — the broker translates. Keeping the +/// two spaces separate is what stops a claimant from asserting a fact +/// about the hardware (`HAS_BINDING`) by writing a bit. +pub const REQ_DMA: u8 = 1 << 0; +/// A PCI selector's remapped MSI-family requirement bit. +pub const REQ_MSI: u8 = 1 << 1; +/// A platform selector's `irq` bit. +pub const REQ_LINE: u8 = 1 << 2; +/// Tighten [`REQ_MSI`] to the exact MSI-X block encoded in `reserved2`. This +/// bit is only valid when `REQ_MSI` is also set, so an older peer sees unknown +/// vocabulary and refuses rather than weakening the request to ordinary MSI. +pub const REQ_MSIX: u8 = 1 << 3; +/// Require initialized firmware HID children behind this PCI controller. +/// Older brokers reject this new bit rather than claiming an arbitrary +/// SerialIO-class function. +pub const REQ_I2C_HID: u8 = 1 << 4; +/// Every defined requirement bit — anything else set is a refusal. +const REQ_ALL: u8 = REQ_DMA | REQ_MSI | REQ_LINE | REQ_MSIX | REQ_I2C_HID; + +/// Lower a request to its wire form (total). +pub fn encode_req(req: &DeviceRequest) -> DeviceReqWire { + let mut w = DeviceReqWire { + op: 0, + species: 0, + class: 0, + subclass: 0, + progif: 0, + flags: 0, + reserved: [0; 2], + word: 0.into(), + reserved2: [0; 4], + }; + match *req { + DeviceRequest::CloseClaim { index, grant } => { + return kore_abi::svc::DeviceClaimCloseWire { + op: DeviceReqWire::OP_CLOSE_CLAIM, + reserved: [0; 3], + index: index.into(), + grant: grant.into(), + } + .encode(); + } + DeviceRequest::QuiesceClaim { index, grant } => { + return kore_abi::svc::DeviceClaimCloseWire { + op: DeviceReqWire::OP_QUIESCE_CLAIM, + reserved: [0; 3], + index: index.into(), + grant: grant.into(), + } + .encode(); + } + DeviceRequest::RetireClaimDma { index, grant } => { + return kore_abi::svc::DeviceClaimCloseWire { + op: DeviceReqWire::OP_RETIRE_CLAIM_DMA, + reserved: [0; 3], + index: index.into(), + grant: grant.into(), + } + .encode(); + } + DeviceRequest::Claim(DeviceSelector::Pci(sel)) + | DeviceRequest::Inspect(DeviceSelector::Pci(sel)) => { + w.op = if matches!(*req, DeviceRequest::Claim(_)) { + DeviceReqWire::OP_CLAIM + } else { + DeviceReqWire::OP_INSPECT + }; + w.species = DeviceSpecies::Pci as u8; + w.class = sel.class; + w.subclass = sel.subclass; + w.progif = sel.progif; + // Zero is the frozen pre-mask spelling for the previously + // unconditional BAR0 requirement. Preserve it so BAR0-only + // clients remain intelligible to an older broker; a multi-BAR + // claimant makes the formerly-reserved byte nonzero and an old + // broker therefore refuses before claiming. + w.reserved[0] = if sel.bars == PciBarMask::BAR0 { + 0 + } else { + sel.bars.bits() + }; + let interrupt_flags = match sel.interrupts { + PciInterruptRequirement::None => 0, + PciInterruptRequirement::MsiFamily => REQ_MSI, + PciInterruptRequirement::Msix(block) => { + let first = block.first_entry().to_le_bytes(); + let count = block.count().to_le_bytes(); + w.reserved2 = [first[0], first[1], count[0], count[1]]; + REQ_MSI | REQ_MSIX + } + }; + w.flags = if sel.dma { REQ_DMA } else { 0 } | interrupt_flags; + w.word = sel.identity.packed().into(); + } + DeviceRequest::Claim(DeviceSelector::Platform { hid, irq, dma }) + | DeviceRequest::Inspect(DeviceSelector::Platform { hid, irq, dma }) => { + w.op = if matches!(*req, DeviceRequest::Claim(_)) { + DeviceReqWire::OP_CLAIM + } else { + DeviceReqWire::OP_INSPECT + }; + w.species = DeviceSpecies::Platform as u8; + w.class = kore_abi::device::DeviceRecord::PLATFORM_CLASS_HID; + w.flags = if irq { REQ_LINE } else { 0 } | if dma { REQ_DMA } else { 0 }; + w.word = hid.into(); + } + DeviceRequest::Claim(DeviceSelector::PlatformI2cHid { irq }) + | DeviceRequest::Inspect(DeviceSelector::PlatformI2cHid { irq }) => { + w.op = if matches!(*req, DeviceRequest::Claim(_)) { + DeviceReqWire::OP_CLAIM + } else { + DeviceReqWire::OP_INSPECT + }; + w.species = DeviceSpecies::Platform as u8; + w.class = kore_abi::device::DeviceRecord::PLATFORM_CLASS_I2C_HID; + w.flags = if irq { REQ_LINE } else { 0 }; + } + DeviceRequest::Claim(DeviceSelector::PciI2cHid { irq }) + | DeviceRequest::Inspect(DeviceSelector::PciI2cHid { irq }) => { + w.op = if matches!(*req, DeviceRequest::Claim(_)) { + DeviceReqWire::OP_CLAIM + } else { + DeviceReqWire::OP_INSPECT + }; + w.species = DeviceSpecies::Pci as u8; + w.class = 0x0c; + w.subclass = 0x80; + w.flags = REQ_I2C_HID | if irq { REQ_LINE } else { 0 }; + } + DeviceRequest::Release { index } => { + w.op = DeviceReqWire::OP_RELEASE; + w.word = index.into(); + } + DeviceRequest::InspectI2c { row } => { + w.op = DeviceReqWire::OP_INSPECT_I2C; + w.word = u32::from(row).into(); + } + DeviceRequest::Count => w.op = DeviceReqWire::OP_COUNT, + DeviceRequest::MintFace { face } => { + w.op = DeviceReqWire::OP_MINT_FACE; + w.word = face.into(); + } + // Encodable for the codec's totality; `decode_req` refuses it — + // see the vocabulary's own doc. + DeviceRequest::Provision => w.op = DeviceReqWire::OP_PROVISION, + } + w +} + +/// Lift a wire request — fail-closed. +/// +/// The unused fields of each op are required to be **zero**, not merely +/// ignored: a `Count` carrying a class triple, or a PCI claim carrying a +/// platform-line requirement, is a peer this service does not understand, +/// and reading past the confusion is how one op quietly becomes another. +pub fn decode_req(w: &DeviceReqWire) -> Option { + if w.op == DeviceReqWire::OP_CLOSE_CLAIM { + let request = kore_abi::svc::DeviceClaimCloseWire::view(w); + let tag = request.grant.get() as u32; + return (request.reserved == [0; 3] && tag != 0 && tag != u32::MAX).then_some( + DeviceRequest::CloseClaim { + index: request.index.get(), + grant: request.grant.get(), + }, + ); + } + if w.op == DeviceReqWire::OP_QUIESCE_CLAIM { + let request = kore_abi::svc::DeviceClaimCloseWire::view(w); + let tag = request.grant.get() as u32; + return (request.reserved == [0; 3] && tag != 0 && tag != u32::MAX).then_some( + DeviceRequest::QuiesceClaim { + index: request.index.get(), + grant: request.grant.get(), + }, + ); + } + if w.op == DeviceReqWire::OP_RETIRE_CLAIM_DMA { + let request = kore_abi::svc::DeviceClaimCloseWire::view(w); + let tag = request.grant.get() as u32; + return (request.reserved == [0; 3] && tag != 0 && tag != u32::MAX).then_some( + DeviceRequest::RetireClaimDma { + index: request.index.get(), + grant: request.grant.get(), + }, + ); + } + if w.reserved[1] != 0 { + return None; + } + if w.flags & !REQ_ALL != 0 { + return None; + } + Some(match w.op { + DeviceReqWire::OP_CLAIM => match DeviceSpecies::from_bits(w.species)? { + DeviceSpecies::Pci => { + if w.flags & REQ_I2C_HID != 0 { + if (w.class, w.subclass, w.progif) != (0x0c, 0x80, 0) + || w.flags & !(REQ_I2C_HID | REQ_LINE) != 0 + || w.reserved != [0; 2] + || w.reserved2 != [0; 4] + || w.word.get() != 0 + { + return None; + } + return Some(DeviceRequest::Claim(DeviceSelector::PciI2cHid { + irq: w.flags & REQ_LINE != 0, + })); + } + // A PCI claim names no ACPI id or line requirement. Its word + // is instead the complete exact-identity selector vocabulary: + // zero for ANY, `vendor | device << 16` otherwise. + if w.flags & REQ_LINE != 0 || w.flags & REQ_MSIX != 0 && w.flags & REQ_MSI == 0 { + return None; + } + let interrupts = if w.flags & REQ_MSIX != 0 { + let first = u16::from_le_bytes([w.reserved2[0], w.reserved2[1]]); + let count = u16::from_le_bytes([w.reserved2[2], w.reserved2[3]]); + PciInterruptRequirement::Msix(PciMsixBlock::new(first, count)?) + } else if w.flags & REQ_MSI != 0 { + if w.reserved2 != [0; 4] { + return None; + } + PciInterruptRequirement::MsiFamily + } else { + if w.reserved2 != [0; 4] { + return None; + } + PciInterruptRequirement::None + }; + let bars = if w.reserved[0] == 0 { + PciBarMask::BAR0 + } else { + PciBarMask::from_bits(w.reserved[0])? + }; + DeviceRequest::Claim(DeviceSelector::Pci(PciSelector { + class: w.class, + subclass: w.subclass, + progif: w.progif, + identity: PciIdentitySelector::from_packed(w.word.get()), + bars, + dma: w.flags & REQ_DMA != 0, + interrupts, + })) + } + DeviceSpecies::Platform => { + // A platform claim names either a compressed ACPI id or one + // topology-derived platform role. It never carries PCI + // MSI/BAR vocabulary; DMA is explicit for bus masters. + if (w.subclass, w.progif) != (0, 0) + || w.flags & (REQ_MSI | REQ_MSIX | REQ_I2C_HID) != 0 + || w.reserved[0] != 0 + || w.reserved2 != [0; 4] + { + return None; + } + match (w.class, w.word.get()) { + (kore_abi::device::DeviceRecord::PLATFORM_CLASS_HID, hid) if hid != 0 => { + DeviceRequest::Claim(DeviceSelector::Platform { + hid, + irq: w.flags & REQ_LINE != 0, + dma: w.flags & REQ_DMA != 0, + }) + } + (kore_abi::device::DeviceRecord::PLATFORM_CLASS_I2C_HID, 0) => { + DeviceRequest::Claim(DeviceSelector::PlatformI2cHid { + irq: w.flags & REQ_LINE != 0, + }) + } + _ => return None, + } + } + }, + DeviceReqWire::OP_INSPECT => { + // Inspection carries exactly the claim selector vocabulary; only + // the operation and the absence of an authority-bearing reply + // differ. Reuse the claim decoder so the two cannot drift. + let mut claim = *w; + claim.op = DeviceReqWire::OP_CLAIM; + let DeviceRequest::Claim(selector) = decode_req(&claim)? else { + return None; + }; + DeviceRequest::Inspect(selector) + } + DeviceReqWire::OP_INSPECT_I2C => { + if (w.species, w.class, w.subclass, w.progif, w.flags) != (0, 0, 0, 0, 0) + || w.reserved != [0; 2] + || w.reserved2 != [0; 4] + || w.word.get() >= u32::from(kore_abi::device::I2C_ADMISSION_ROWS) + { + return None; + } + DeviceRequest::InspectI2c { + row: w.word.get() as u8, + } + } + DeviceReqWire::OP_RELEASE => { + if (w.species, w.class, w.subclass, w.progif, w.flags) != (0, 0, 0, 0, 0) + || w.reserved[0] != 0 + || w.reserved2 != [0; 4] + { + return None; + } + DeviceRequest::Release { + index: w.word.get(), + } + } + DeviceReqWire::OP_COUNT => { + if (w.species, w.class, w.subclass, w.progif, w.flags) != (0, 0, 0, 0, 0) + || w.word.get() != 0 + || w.reserved[0] != 0 + || w.reserved2 != [0; 4] + { + return None; + } + DeviceRequest::Count + } + DeviceReqWire::OP_MINT_FACE => { + if (w.species, w.class, w.subclass, w.progif, w.flags) != (0, 0, 0, 0, 0) + || w.reserved[0] != 0 + || w.reserved2 != [0; 4] + { + return None; + } + DeviceRequest::MintFace { face: w.word.get() } + } + // `Provision` is the serve host's own synthetic step + // (`implicit_return_request`); a peer writing its op onto the + // request ring is forging internal vocabulary, and reading past + // that would let a claimant consume another face's banked leg. + DeviceReqWire::OP_PROVISION => return None, + _ => return None, + }) +} + +/// Lower a response (total). +/// +/// `grants` is the count the serve host actually effected, never a count +/// reconstructed from the response tag. A mismatch remains an invalid wire +/// record and [`decode_rep`] refuses it; silently changing zero back to one +/// would make a client wait forever for a device that never landed. +pub fn encode_rep(rep: &DeviceResponse, grants: u8) -> DeviceRepWire { + let mut w = DeviceRepWire { + tag: 0, + refusal: 0, + grants, + species: 0, + reserved: [0; 4], + word: 0.into(), + diag: 0.into(), + vendor: 0.into(), + device: 0.into(), + }; + match *rep { + DeviceResponse::ClaimQuiesced(report) => { + w.tag = DeviceRepWire::TAG_CLAIM_QUIESCED; + w.reserved[0] = report.verdict; + w.reserved[1] = report.flags; + w.word = report.rid.into(); + } + DeviceResponse::ClaimClosed(report) => { + w.tag = DeviceRepWire::TAG_CLAIM_CLOSED; + w.reserved[0] = report.state as u8; + w.word = report.readers.into(); + } + DeviceResponse::ClaimDmaRetired(report) => { + w.tag = DeviceRepWire::TAG_CLAIM_DMA_RETIRED; + w.reserved[0] = report.state as u8; + w.word = report.removed.into(); + } + DeviceResponse::Granted { + index, + species, + diag, + revision, + vendor, + device, + } => { + w.tag = DeviceRepWire::TAG_GRANTED; + w.species = species as u8; + w.word = index.into(); + w.diag = diag.into(); + w.reserved[0] = revision; + w.vendor = vendor.into(); + w.device = device.into(); + } + DeviceResponse::Observed(observed) => { + w.tag = DeviceRepWire::TAG_OBSERVED; + w.species = observed.species as u8; + w.word = observed.index.into(); + w.diag = observed.diag.into(); + w.reserved[0] = observed.revision; + w.reserved[1] = observed.flags; + } + DeviceResponse::I2cObserved(page) => { + w.tag = DeviceRepWire::TAG_I2C_OBSERVED; + w.reserved[0] = page.row; + w.word = page.words[0].into(); + w.diag = page.words[1].into(); + } + DeviceResponse::Released => w.tag = DeviceRepWire::TAG_RELEASED, + DeviceResponse::Counted { len } => { + w.tag = DeviceRepWire::TAG_COUNTED; + w.word = len.into(); + } + DeviceResponse::Provisioning { face } => { + w.tag = DeviceRepWire::TAG_PROVISIONING; + w.word = face.into(); + } + DeviceResponse::Banked { face } => { + w.tag = DeviceRepWire::TAG_BANKED; + w.word = face.into(); + } + DeviceResponse::Minted { via } => { + w.tag = DeviceRepWire::TAG_MINTED; + w.word = u32::from(via).into(); + } + DeviceResponse::Refused(r) => { + w.tag = DeviceRepWire::TAG_REFUSED; + w.refusal = match r { + DeviceRefusal::NoMatch => DeviceRepWire::REFUSE_NO_MATCH, + DeviceRefusal::Taken => DeviceRepWire::REFUSE_TAKEN, + DeviceRefusal::Denied => DeviceRepWire::REFUSE_DENIED, + DeviceRefusal::NotHeld => DeviceRepWire::REFUSE_NOT_HELD, + DeviceRefusal::NoSupply => DeviceRepWire::REFUSE_NO_SUPPLY, + DeviceRefusal::BadFace => DeviceRepWire::REFUSE_BAD_FACE, + DeviceRefusal::BadConvey => DeviceRepWire::REFUSE_BAD_CONVEY, + DeviceRefusal::Busy => DeviceRepWire::REFUSE_BUSY, + }; + } + } + w +} + +/// Lift a wire reply — fail-closed. +/// +/// The grant count is checked **against the tag**: a refusal claiming a +/// grant, or a `Granted` claiming none, is a protocol break and not a +/// reply to read. That check is the one that matters here — a client that +/// believed a grant-less `Granted` would go looking for a device token +/// that never arrived and bind against whatever was in the slot. +pub fn decode_rep(w: &DeviceRepWire) -> Option<(DeviceResponse, u8)> { + if w.tag == DeviceRepWire::TAG_CLAIM_QUIESCED { + if w.refusal != 0 + || w.grants != 0 + || w.species != 0 + || w.reserved[2..] != [0; 2] + || w.diag.get() != 0 + || w.vendor.get() != 0 + || w.device.get() != 0 + { + return None; + } + let report = kore_abi::session::QuiesceReport { + rid: w.word.get(), + verdict: w.reserved[0], + flags: w.reserved[1], + }; + return kore_abi::session::QuiesceReport::decode(&report.encode()) + .map(|report| (DeviceResponse::ClaimQuiesced(report), 0)); + } + if w.tag == DeviceRepWire::TAG_CLAIM_CLOSED { + if w.refusal != 0 + || w.grants != 0 + || w.species != 0 + || w.reserved[1..] != [0; 3] + || w.diag.get() != 0 + || w.vendor.get() != 0 + || w.device.get() != 0 + { + return None; + } + // Reuse the canonical kernel report decoder for state/count validity. + let mut record = kore_abi::session::DeviceClaimCloseReport { + state: kore_abi::session::DeviceClaimCloseState::Mismatch, + readers: 0, + } + .encode(); + record.scope_rid = u64::from(w.reserved[0]) | u64::from(w.word.get()) << 8; + return kore_abi::session::DeviceClaimCloseReport::decode(&record) + .map(|report| (DeviceResponse::ClaimClosed(report), 0)); + } + if w.tag == DeviceRepWire::TAG_CLAIM_DMA_RETIRED { + if w.refusal != 0 + || w.grants != 0 + || w.species != 0 + || w.reserved[1..] != [0; 3] + || w.diag.get() != 0 + || w.vendor.get() != 0 + || w.device.get() != 0 + { + return None; + } + // Reuse the canonical kernel report decoder for state/count validity. + let mut record = kore_abi::session::DeviceClaimDmaReport { + state: kore_abi::session::DeviceClaimDmaState::Retired, + removed: 0, + } + .encode(); + record.scope_rid = u64::from(w.reserved[0]) | u64::from(w.word.get()) << 8; + return kore_abi::session::DeviceClaimDmaReport::decode(&record) + .map(|report| (DeviceResponse::ClaimDmaRetired(report), 0)); + } + // PCI identity belongs only to an actual grant, never an observation + // (including I2C admission pages) or a refusal. + if w.tag != DeviceRepWire::TAG_GRANTED && (w.vendor.get() != 0 || w.device.get() != 0) { + return None; + } + match w.tag { + DeviceRepWire::TAG_GRANTED if w.reserved[1..] != [0u8; 3] => return None, + DeviceRepWire::TAG_OBSERVED if w.reserved[2..] != [0u8; 2] => return None, + DeviceRepWire::TAG_I2C_OBSERVED if w.reserved[1..] != [0u8; 3] => return None, + DeviceRepWire::TAG_GRANTED + | DeviceRepWire::TAG_OBSERVED + | DeviceRepWire::TAG_I2C_OBSERVED => {} + _ if w.diag.get() != 0 || w.reserved != [0u8; 4] => return None, + _ => {} + } + let rep = match (w.tag, w.refusal) { + (DeviceRepWire::TAG_I2C_OBSERVED, 0) => { + if w.grants != 0 || w.species != 0 { + return None; + } + let page = kore_abi::device::I2cAdmissionPage { + row: w.reserved[0], + words: [w.word.get(), w.diag.get()], + }; + if !page.valid() { + return None; + } + DeviceResponse::I2cObserved(page) + } + (DeviceRepWire::TAG_GRANTED, 0) => { + if w.grants != 1 { + return None; + } + let species = DeviceSpecies::from_bits(w.species)?; + if species == DeviceSpecies::Platform && (w.vendor.get() != 0 || w.device.get() != 0) { + return None; + } + DeviceResponse::Granted { + index: w.word.get(), + species, + diag: w.diag.get(), + revision: w.reserved[0], + vendor: w.vendor.get(), + device: w.device.get(), + } + } + (DeviceRepWire::TAG_OBSERVED, 0) => { + if w.grants != 0 { + return None; + } + DeviceResponse::Observed(DeviceObservation { + index: w.word.get(), + species: DeviceSpecies::from_bits(w.species)?, + flags: w.reserved[1], + diag: w.diag.get(), + revision: w.reserved[0], + }) + } + (DeviceRepWire::TAG_RELEASED, 0) => { + if w.grants != 0 || w.species != 0 || w.word.get() != 0 { + return None; + } + DeviceResponse::Released + } + (DeviceRepWire::TAG_COUNTED, 0) => { + if w.grants != 0 || w.species != 0 { + return None; + } + DeviceResponse::Counted { len: w.word.get() } + } + (DeviceRepWire::TAG_PROVISIONING, 0) => { + if w.grants != 0 || w.species != 0 { + return None; + } + DeviceResponse::Provisioning { face: w.word.get() } + } + (DeviceRepWire::TAG_BANKED, 0) => { + if w.grants != 0 || w.species != 0 { + return None; + } + DeviceResponse::Banked { face: w.word.get() } + } + (DeviceRepWire::TAG_MINTED, 0) => { + if w.grants != 0 || w.species != 0 { + return None; + } + let via = u16::try_from(w.word.get()).ok()?; + DeviceResponse::Minted { via } + } + (DeviceRepWire::TAG_REFUSED, r) => { + if w.grants != 0 || w.species != 0 || w.word.get() != 0 { + return None; + } + DeviceResponse::Refused(match r { + DeviceRepWire::REFUSE_NO_MATCH => DeviceRefusal::NoMatch, + DeviceRepWire::REFUSE_TAKEN => DeviceRefusal::Taken, + DeviceRepWire::REFUSE_DENIED => DeviceRefusal::Denied, + DeviceRepWire::REFUSE_NOT_HELD => DeviceRefusal::NotHeld, + DeviceRepWire::REFUSE_NO_SUPPLY => DeviceRefusal::NoSupply, + DeviceRepWire::REFUSE_BAD_FACE => DeviceRefusal::BadFace, + DeviceRepWire::REFUSE_BAD_CONVEY => DeviceRefusal::BadConvey, + DeviceRepWire::REFUSE_BUSY => DeviceRefusal::Busy, + _ => return None, + }) + } + _ => return None, + }; + Some((rep, w.grants)) +} + +impl SvcProtocol for DeviceBroker { + type Request = DeviceRequest; + type Response = DeviceResponse; + type ReqWire = DeviceReqWire; + type RepWire = DeviceRepWire; + + /// One — a `Claim` hands over exactly one device, and no verb here + /// hands over more. + const REPLY_GRANTS: usize = 1; + + /// **One** — the control pair (`res/DEVICE_SERVICE.md` §10, the + /// minted-face broker): its client is the kernel chain founder, which + /// drives `MintFace` through it at each driver row's founding round. + /// Every claimant rides a runtime-minted pair instead (via + /// `FACE_VIA_BASE + n`), so the driver-row count never touches this + /// constant again — the wall this replaces was three static pairs + /// against [`MAX_SERVE_PAIRS`](kore_rt::svc::MAX_SERVE_PAIRS). + /// Requests are stamped with their arrival pair either way + /// ([`Serve::via`](kore_rt::service::Serve::via)) — the ledger's + /// owner column. + const SERVE_PAIRS: usize = 1; + + /// A claimed device outlives the request and may coexist with a later + /// platform-device claim. Rotate the receiver-owned landing instead of + /// reposting over the first live device record. + fn request_grants(request: &DeviceReqWire) -> usize { + // These operations are always grantless. In particular a retiring + // owner need not reserve another device landing to close its old one. + if matches!( + request.op, + DeviceReqWire::OP_CLOSE_CLAIM + | DeviceReqWire::OP_QUIESCE_CLAIM + | DeviceReqWire::OP_RETIRE_CLAIM_DMA + ) { + 0 + } else { + Self::REPLY_GRANTS + } + } + + fn retain_grant(index: usize) -> bool { + index == 0 + } + + fn reply_matches(request: &DeviceReqWire, response: &DeviceResponse, grants: u8) -> bool { + match (request.op, response) { + ( + DeviceReqWire::OP_CLOSE_CLAIM, + DeviceResponse::ClaimClosed(_) | DeviceResponse::Refused(DeviceRefusal::Denied), + ) + | ( + DeviceReqWire::OP_RETIRE_CLAIM_DMA, + DeviceResponse::ClaimDmaRetired(_) + | DeviceResponse::Refused(DeviceRefusal::Denied | DeviceRefusal::Busy), + ) + | ( + DeviceReqWire::OP_QUIESCE_CLAIM, + DeviceResponse::ClaimQuiesced(_) + | DeviceResponse::Refused(DeviceRefusal::Denied | DeviceRefusal::Busy), + ) => grants == 0, + (DeviceReqWire::OP_CLAIM, DeviceResponse::Granted { species, .. }) => { + grants == 1 && request.species == *species as u8 + } + (DeviceReqWire::OP_INSPECT, DeviceResponse::Observed(observed)) => { + grants == 0 && request.species == observed.species as u8 + } + (DeviceReqWire::OP_INSPECT_I2C, DeviceResponse::I2cObserved(page)) => { + grants == 0 && request.word.get() == u32::from(page.row) && page.valid() + } + ( + DeviceReqWire::OP_CLAIM, + DeviceResponse::Refused( + DeviceRefusal::NoMatch | DeviceRefusal::Taken | DeviceRefusal::Denied, + ), + ) + | ( + DeviceReqWire::OP_RELEASE, + DeviceResponse::Refused(DeviceRefusal::Denied | DeviceRefusal::NotHeld), + ) + | ( + DeviceReqWire::OP_INSPECT, + DeviceResponse::Refused(DeviceRefusal::NoMatch | DeviceRefusal::Denied), + ) + | (DeviceReqWire::OP_INSPECT_I2C, DeviceResponse::Refused(DeviceRefusal::Denied)) + | (DeviceReqWire::OP_COUNT, DeviceResponse::Counted { .. }) + | (DeviceReqWire::OP_COUNT, DeviceResponse::Refused(DeviceRefusal::Denied)) + | (DeviceReqWire::OP_MINT_FACE, DeviceResponse::Provisioning { .. }) + | ( + DeviceReqWire::OP_MINT_FACE, + DeviceResponse::Refused(DeviceRefusal::NoSupply | DeviceRefusal::BadFace), + ) + | ( + DeviceReqWire::OP_PROVISION, + DeviceResponse::Banked { .. } | DeviceResponse::Minted { .. }, + ) + | ( + DeviceReqWire::OP_PROVISION, + DeviceResponse::Refused( + DeviceRefusal::BadConvey | DeviceRefusal::NoSupply | DeviceRefusal::Denied, + ), + ) => grants == 0, + _ => false, + } + } + + /// The walking return plane's synthetic step: a convey landed on the + /// landing this broker offered, so the host drives one `Provision` + /// through the ordinary pipeline — the exec convey-table shape. + fn implicit_return_request() -> Option { + Some(DeviceRequest::Provision) + } + + fn encode_req(req: &DeviceRequest) -> DeviceReqWire { + encode_req(req) + } + fn decode_req(wire: &DeviceReqWire) -> Option { + decode_req(wire) + } + fn encode_rep(rep: &DeviceResponse, grants: u8) -> DeviceRepWire { + encode_rep(rep, grants) + } + fn decode_rep(wire: &DeviceRepWire) -> Option<(DeviceResponse, u8)> { + decode_rep(wire) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn xhci() -> DeviceRequest { + DeviceRequest::Claim(DeviceSelector::Pci(PciSelector { + class: 0x0c, + subclass: 0x03, + progif: 0x30, + identity: PciIdentitySelector::ANY, + bars: PciBarMask::BAR0, + dma: true, + interrupts: PciInterruptRequirement::MsiFamily, + })) + } + + #[test] + fn claim_retirement_replies_are_reachable_and_operation_correlated() { + use kore_abi::session::{ + DeviceClaimCloseReport, DeviceClaimCloseState, QUIESCE_FLAG_IRQ_MASKED, + QUIESCE_VERDICT_PROVEN, QuiesceReport, + }; + let close = DeviceRequest::CloseClaim { + index: 7, + grant: 0x1234_00000077, + }; + let quiesce = DeviceRequest::QuiesceClaim { + index: 7, + grant: 0x1234_00000077, + }; + let closed = DeviceResponse::ClaimClosed(DeviceClaimCloseReport { + state: DeviceClaimCloseState::AdmissionDrained, + readers: 0, + }); + let quiet = DeviceResponse::ClaimQuiesced(QuiesceReport { + rid: 42, + verdict: QUIESCE_VERDICT_PROVEN, + flags: QUIESCE_FLAG_IRQ_MASKED, + }); + let dma = DeviceRequest::RetireClaimDma { + index: 7, + grant: 0x1234_00000077, + }; + let retired = DeviceResponse::ClaimDmaRetired(kore_abi::session::DeviceClaimDmaReport { + state: kore_abi::session::DeviceClaimDmaState::Retired, + removed: 5, + }); + for (request, response) in [(close, closed), (quiesce, quiet), (dma, retired)] { + let wire = encode_req(&request); + assert_eq!( + DeviceBroker::request_grants(&wire), + 0, + "retirement never reserves a fresh device landing" + ); + assert_eq!(decode_req(&wire), Some(request)); + let (decoded, grants) = decode_rep(&encode_rep(&response, 0)).unwrap(); + assert_eq!(decoded, response); + assert!(DeviceBroker::reply_matches(&wire, &decoded, grants)); + assert!(!DeviceBroker::reply_matches(&wire, &decoded, 1)); + assert!(DeviceBroker::reply_matches( + &wire, + &DeviceResponse::Refused(DeviceRefusal::Denied), + 0 + )); + } + assert!(!DeviceBroker::reply_matches(&encode_req(&close), &quiet, 0)); + assert!(!DeviceBroker::reply_matches( + &encode_req(&quiesce), + &closed, + 0 + )); + for response in [closed, quiet] { + assert!(!DeviceBroker::reply_matches( + &encode_req(&dma), + &response, + 0 + )); + } + for request in [close, quiesce] { + assert!(!DeviceBroker::reply_matches( + &encode_req(&request), + &retired, + 0 + )); + } + for mutate in [ + |w: &mut DeviceRepWire| w.reserved[0] = 0xff, + |w: &mut DeviceRepWire| w.reserved[1] = 1, + |w: &mut DeviceRepWire| w.grants = 1, + |w: &mut DeviceRepWire| w.species = 1, + |w: &mut DeviceRepWire| w.refusal = 1, + |w: &mut DeviceRepWire| w.diag = 1.into(), + ] { + let mut encoded = encode_rep(&retired, 0); + mutate(&mut encoded); + assert_eq!(decode_rep(&encoded), None); + } + let busy = DeviceResponse::Refused(DeviceRefusal::Busy); + assert_eq!(decode_rep(&encode_rep(&busy, 0)), Some((busy, 0))); + assert!(DeviceBroker::reply_matches(&encode_req(&quiesce), &busy, 0)); + assert!(!DeviceBroker::reply_matches(&encode_req(&close), &busy, 0)); + for mutate in [ + |w: &mut DeviceRepWire| w.reserved[0] = 0xff, + |w: &mut DeviceRepWire| w.reserved[1] = 0x80, + |w: &mut DeviceRepWire| w.reserved[2] = 1, + |w: &mut DeviceRepWire| w.diag = 1.into(), + |w: &mut DeviceRepWire| w.grants = 1, + |w: &mut DeviceRepWire| w.species = 1, + ] { + let mut dirty = encode_rep(&quiet, 0); + mutate(&mut dirty); + assert_eq!(decode_rep(&dirty), None); + } + } + + #[test] + fn exact_claim_close_wire_is_shape_tight_and_grantless() { + use kore_abi::session::{DeviceClaimCloseReport, DeviceClaimCloseState}; + let request = DeviceRequest::CloseClaim { + index: 0x12345678, + grant: 0x11223344_55667788, + }; + assert_eq!(decode_req(&encode_req(&request)), Some(request)); + for grant in [0, u32::MAX as u64] { + assert_eq!( + decode_req(&encode_req(&DeviceRequest::CloseClaim { index: 7, grant })), + None + ); + } + let mut malformed = kore_abi::svc::DeviceClaimCloseWire::view(&encode_req(&request)); + malformed.reserved[1] = 1; + assert_eq!(decode_req(&malformed.encode()), None); + let reply = DeviceResponse::ClaimClosed(DeviceClaimCloseReport { + state: DeviceClaimCloseState::Pending, + readers: 3, + }); + assert_eq!(decode_rep(&encode_rep(&reply, 0)), Some((reply, 0))); + assert_eq!(decode_rep(&encode_rep(&reply, 1)), None); + let mut malformed = encode_rep(&reply, 0); + malformed.word = 0.into(); + assert_eq!(decode_rep(&malformed), None); + } + + #[test] + fn i2c_admission_inspection_has_no_grants_or_selector_side_channels() { + use kore_abi::device::{ + AmlBuildDiagnostic, AmlBuildErrorClass, AmlBuildStage, I2C_ADMISSION_BUILD_ROW, + I2C_ADMISSION_ROWS, I2cAdmissionPage, I2cAdmissionSnapshot, + }; + for row in [0, 1, 24, I2C_ADMISSION_BUILD_ROW] { + let request = DeviceRequest::InspectI2c { row }; + let wire = encode_req(&request); + assert_eq!(decode_req(&wire), Some(request)); + for change in [ + |w: &mut DeviceReqWire| w.species = 1, + |w: &mut DeviceReqWire| w.flags = 1, + |w: &mut DeviceReqWire| w.class = 1, + |w: &mut DeviceReqWire| w.reserved[0] = 1, + |w: &mut DeviceReqWire| w.reserved2[3] = 1, + |w: &mut DeviceReqWire| w.word = u32::from(I2C_ADMISSION_ROWS).into(), + ] { + let mut dirty = wire; + change(&mut dirty); + assert_eq!(decode_req(&dirty), None); + } + } + for page in [ + I2cAdmissionSnapshot::default().page(0).unwrap(), + I2cAdmissionPage { + row: 24, + words: [0x55, 0xa851_8086], + }, + I2cAdmissionPage { + row: I2C_ADMISSION_BUILD_ROW, + words: AmlBuildDiagnostic::failure( + AmlBuildStage::SsdtLoad, + AmlBuildErrorClass::Opcode, + u32::MAX, + 0x5bff, + ) + .unwrap() + .words(), + }, + ] { + let response = DeviceResponse::I2cObserved(page); + let wire = encode_rep(&response, 0); + assert_eq!(decode_rep(&wire), Some((response, 0))); + assert_eq!(core::mem::size_of_val(&wire), 20); + assert_eq!((wire.vendor.get(), wire.device.get()), (0, 0)); + for change in [ + |w: &mut DeviceRepWire| w.vendor = 0x8086u16.into(), + |w: &mut DeviceRepWire| w.device = 0xa851u16.into(), + ] { + let mut dirty = wire; + change(&mut dirty); + assert_eq!(decode_rep(&dirty), None, "admission is not a PCI grant"); + } + assert_eq!(decode_rep(&encode_rep(&response, 1)), None); + let request = encode_req(&DeviceRequest::InspectI2c { row: page.row }); + assert!(DeviceBroker::reply_matches(&request, &response, 0)); + assert!(!DeviceBroker::reply_matches(&request, &response, 1)); + let other = encode_req(&DeviceRequest::InspectI2c { + row: (page.row + 1) % I2C_ADMISSION_ROWS, + }); + assert!(!DeviceBroker::reply_matches(&other, &response, 0)); + assert!(DeviceBroker::reply_matches( + &request, + &DeviceResponse::Refused(DeviceRefusal::Denied), + 0, + )); + } + } + + /// Every vocabulary member survives a round trip in both directions. + #[test] + fn round_trips() { + for req in [ + xhci(), + DeviceRequest::Claim(DeviceSelector::Pci(PciSelector { + class: 1, + subclass: 8, + progif: 2, + identity: PciIdentitySelector::ANY, + bars: PciBarMask::BAR0, + dma: false, + interrupts: PciInterruptRequirement::None, + })), + DeviceRequest::Claim(DeviceSelector::Pci(PciSelector { + class: 0x03, + subclass: 0x00, + progif: 0x00, + identity: PciIdentitySelector::exact(0x8086, 0x56a0), + bars: PciBarMask::BAR0, + dma: true, + interrupts: PciInterruptRequirement::Msix(PciMsixBlock::ENTRIES_0_1), + })), + DeviceRequest::Claim(DeviceSelector::Platform { + hid: 0x0303_d041, + irq: true, + dma: false, + }), + DeviceRequest::Claim(DeviceSelector::Platform { + hid: 7, + irq: false, + dma: false, + }), + DeviceRequest::Claim(DeviceSelector::Platform { + hid: 0x0d15_d041, + irq: true, + dma: true, + }), + DeviceRequest::Inspect(DeviceSelector::Pci(PciSelector { + class: 0x03, + subclass: 0x00, + progif: 0x00, + identity: PciIdentitySelector::exact(0x8086, 0x64a0), + bars: PciBarMask::BAR0, + dma: true, + interrupts: PciInterruptRequirement::None, + })), + DeviceRequest::Claim(DeviceSelector::PlatformI2cHid { irq: true }), + DeviceRequest::Inspect(DeviceSelector::PlatformI2cHid { irq: false }), + DeviceRequest::Claim(DeviceSelector::PciI2cHid { irq: false }), + DeviceRequest::Claim(DeviceSelector::PciI2cHid { irq: true }), + DeviceRequest::Inspect(DeviceSelector::PciI2cHid { irq: false }), + DeviceRequest::Inspect(DeviceSelector::PciI2cHid { irq: true }), + DeviceRequest::Release { index: 0 }, + DeviceRequest::Release { index: u32::MAX }, + DeviceRequest::Count, + DeviceRequest::MintFace { face: 0 }, + DeviceRequest::MintFace { face: 7 }, + ] { + assert_eq!(decode_req(&encode_req(&req)), Some(req)); + } + for (rep, grants) in [ + ( + DeviceResponse::Granted { + index: 4, + species: DeviceSpecies::Pci, + diag: 0, + revision: 0, + vendor: 0x1022, + device: 0x1118, + }, + 1, + ), + ( + DeviceResponse::Observed(DeviceObservation { + index: 7, + species: DeviceSpecies::Pci, + flags: 0xa5, + diag: 0xf001_0041, + revision: 4, + }), + 0, + ), + ( + DeviceResponse::Granted { + index: u32::MAX, + species: DeviceSpecies::Platform, + diag: 0, + revision: 0, + vendor: 0, + device: 0, + }, + 1, + ), + (DeviceResponse::Released, 0), + (DeviceResponse::Counted { len: 31 }, 0), + (DeviceResponse::Provisioning { face: 2 }, 0), + (DeviceResponse::Banked { face: 2 }, 0), + (DeviceResponse::Minted { via: 5 }, 0), + (DeviceResponse::Refused(DeviceRefusal::NoMatch), 0), + (DeviceResponse::Refused(DeviceRefusal::Taken), 0), + (DeviceResponse::Refused(DeviceRefusal::Denied), 0), + (DeviceResponse::Refused(DeviceRefusal::NotHeld), 0), + (DeviceResponse::Refused(DeviceRefusal::NoSupply), 0), + (DeviceResponse::Refused(DeviceRefusal::BadFace), 0), + (DeviceResponse::Refused(DeviceRefusal::BadConvey), 0), + ] { + assert_eq!(decode_rep(&encode_rep(&rep, grants)), Some((rep, grants))); + } + } + + #[test] + fn pci_identity_uses_the_existing_claim_word_without_widening_the_wire() { + let any = encode_req(&xhci()); + assert_eq!(any.word.get(), 0, "the old wildcard encoding is frozen"); + + let exact = DeviceRequest::Claim(DeviceSelector::Pci(PciSelector { + class: 0x03, + subclass: 0x00, + progif: 0x00, + identity: PciIdentitySelector::exact(0x8086, 0x56a0), + bars: PciBarMask::BAR0, + dma: true, + interrupts: PciInterruptRequirement::Msix(PciMsixBlock::ENTRIES_0_1), + })); + let wire = encode_req(&exact); + assert_eq!(wire.word.get(), 0x56a0_8086); + assert_eq!(wire.flags & (REQ_MSI | REQ_MSIX), REQ_MSI | REQ_MSIX); + assert_eq!(wire.reserved2, [0, 0, 2, 0]); + assert_eq!(decode_req(&wire), Some(exact)); + assert_eq!(core::mem::size_of::(), 16); + } + + #[test] + fn strict_msix_block_cannot_be_weakened_or_malformed_on_the_wire() { + let mut wire = encode_req(&xhci()); + wire.flags = (wire.flags & !REQ_MSI) | REQ_MSIX; + assert_eq!(decode_req(&wire), None); + + let mut invalid = encode_req(&DeviceRequest::Claim(DeviceSelector::Pci(PciSelector { + class: 0x03, + subclass: 0, + progif: 0, + identity: PciIdentitySelector::exact(0x8086, 0x56a0), + bars: PciBarMask::BAR0, + dma: true, + interrupts: PciInterruptRequirement::Msix(PciMsixBlock::ENTRIES_0_1), + }))); + invalid.reserved2 = [0, 0, 3, 0]; + assert_eq!(decode_req(&invalid), None, "count must be a power of two"); + invalid.reserved2 = [0xff, 0xff, 2, 0]; + assert_eq!(decode_req(&invalid), None, "entry range must not overflow"); + } + + #[test] + fn pci_i2c_claim_and_inspect_reject_mixed_selector_vocabulary() { + for irq in [false, true] { + for request in [ + DeviceRequest::Claim(DeviceSelector::PciI2cHid { irq }), + DeviceRequest::Inspect(DeviceSelector::PciI2cHid { irq }), + ] { + let base = encode_req(&request); + assert_eq!(base.species, DeviceSpecies::Pci as u8); + assert_eq!(base.flags, REQ_I2C_HID | if irq { REQ_LINE } else { 0 }); + for extra in [ + REQ_DMA, + REQ_MSI, + REQ_MSIX, + REQ_MSI | REQ_MSIX, + 1 << 5, + 1 << 6, + 1 << 7, + ] { + let mut dirty = base; + dirty.flags |= extra; + assert_eq!(decode_req(&dirty), None, "mixed requirement {extra:#x}"); + } + for dirty_field in [ + |w: &mut DeviceReqWire| w.class ^= 1, + |w: &mut DeviceReqWire| w.subclass ^= 1, + |w: &mut DeviceReqWire| w.progif = 1, + |w: &mut DeviceReqWire| w.species = DeviceSpecies::Platform as u8, + |w: &mut DeviceReqWire| w.reserved[0] = 1, + |w: &mut DeviceReqWire| w.reserved[1] = 1, + |w: &mut DeviceReqWire| w.word = 0xa878_8086u32.into(), + ] { + let mut dirty = base; + dirty_field(&mut dirty); + assert_eq!(decode_req(&dirty), None); + } + for index in 0..base.reserved2.len() { + let mut dirty = base; + dirty.reserved2[index] = 1; + assert_eq!(decode_req(&dirty), None, "reserved2 byte {index}"); + } + } + + for request in [ + DeviceRequest::Claim(DeviceSelector::PlatformI2cHid { irq }), + DeviceRequest::Inspect(DeviceSelector::PlatformI2cHid { irq }), + ] { + let mut platform = encode_req(&request); + platform.flags |= REQ_I2C_HID; + assert_eq!( + decode_req(&platform), + None, + "PCI topology bit cannot alias a platform row" + ); + } + } + } + + #[test] + fn multi_bar_mask_uses_the_old_reserve_and_round_trips_exactly() { + let mut req = xhci(); + let DeviceRequest::Claim(DeviceSelector::Pci(selector)) = &mut req else { + unreachable!(); + }; + selector.bars = PciBarMask::BAR0.union(PciBarMask::BAR4); + let wire = encode_req(&req); + assert_eq!(wire.reserved, [0x11, 0]); + assert_eq!(decode_req(&wire), Some(req)); + + let bar0 = encode_req(&xhci()); + assert_eq!(bar0.reserved, [0, 0], "BAR0 keeps the frozen wire spelling"); + } + + /// `Provision` is the host's own synthetic step: its op encodes (the + /// codec is total) but never decodes — a claimant that writes it onto + /// the request ring is refused, not interpreted. + #[test] + fn the_implicit_op_never_decodes_from_the_wire() { + let w = encode_req(&DeviceRequest::Provision); + assert_eq!(w.op, DeviceReqWire::OP_PROVISION); + assert_eq!(decode_req(&w), None); + } + + /// A `MintFace` carrying selector junk in the unused fields is a peer + /// this service does not understand — same rule as every other op. + #[test] + fn a_dirty_mint_face_is_refused() { + let base = encode_req(&DeviceRequest::MintFace { face: 1 }); + for dirt in [ + |w: &mut DeviceReqWire| w.species = 1, + |w: &mut DeviceReqWire| w.class = 0x0c, + |w: &mut DeviceReqWire| w.flags = REQ_DMA, + ] { + let mut dirty = base; + dirt(&mut dirty); + assert_eq!(decode_req(&dirty), None); + } + // A minted via past u16 in the reply is a protocol break. + let mut wide = encode_rep(&DeviceResponse::Minted { via: 3 }, 0); + wide.word = 0x1_0000u32.into(); + assert_eq!(decode_rep(&wide), None); + } + + /// The fail-closed obligations, each one a distinct way a peer can lie. + #[test] + fn decode_fails_closed() { + let base = encode_req(&xhci()); + + // An all-zero record is not a claim — the 1-based discriminant + // property. It matters here for the same reason it matters in the + // power protocol: an unseeded record must never *acquire* anything. + let mut zero = base; + zero.op = 0; + assert_eq!(decode_req(&zero), None); + + let mut unknown = base; + unknown.op = 200; + assert_eq!(decode_req(&unknown), None); + + // Byte one and the trailing reserve remain frozen zero. Byte zero is + // now the PCI BAR mask, so an out-of-range bit is the fail-closed case. + let mut dirty = base; + dirty.reserved = [0, 1]; + assert_eq!(decode_req(&dirty), None); + let mut dirty = base; + dirty.reserved = [1 << 6, 0]; + assert_eq!(decode_req(&dirty), None); + let mut dirty = base; + dirty.reserved2 = [0, 0, 1, 0]; + assert_eq!(decode_req(&dirty), None); + + // An undefined requirement bit is refused: the bit is where a + // later requirement goes, and honouring it as "no requirement" + // now would make adding one a silent break. + let mut extra = base; + extra.flags |= 1 << 7; + assert_eq!(decode_req(&extra), None); + + // Cross-species contamination: a PCI claim carrying the platform + // line requirement, and a platform claim carrying a class triple. + // The PCI word is no longer contamination: it is the exact identity + // selector tested above. + let mut hybrid = base; + hybrid.flags |= REQ_LINE; + assert_eq!( + decode_req(&hybrid), + None, + "a PCI claim cannot request a platform interrupt line" + ); + let mut hybrid = encode_req(&DeviceRequest::Claim(DeviceSelector::Platform { + hid: 0x0303_d041, + irq: true, + dma: false, + })); + hybrid.class = 0x0c; + assert_eq!(decode_req(&hybrid), None); + + // An unknown species. + let mut alien = base; + alien.species = 9; + assert_eq!(decode_req(&alien), None); + + // A `Count` is not a smuggled claim. + let mut counted = encode_req(&DeviceRequest::Count); + counted.class = 0x0c; + assert_eq!(decode_req(&counted), None); + } + + /// The grant count must agree with the tag, in both directions. + /// + /// This is the reply-side check that actually protects a claimant: a + /// `Granted` with no grant would send it looking for a device token + /// that never arrived, and a refusal *with* one would leave authority + /// it never accounted for sitting in its arena. + #[test] + fn a_grant_count_that_disagrees_with_its_tag_is_a_break() { + let mut granted = encode_rep( + &DeviceResponse::Granted { + index: 2, + species: DeviceSpecies::Pci, + diag: 0, + revision: 0, + vendor: 0, + device: 0, + }, + 1, + ); + granted.grants = 0; + assert_eq!( + decode_rep(&granted), + None, + "a Granted with no grant is not a device" + ); + + let mut refused = encode_rep(&DeviceResponse::Refused(DeviceRefusal::Taken), 0); + refused.grants = 1; + assert_eq!( + decode_rep(&refused), + None, + "a refusal must never carry authority" + ); + + // A refusal byte on a success tag, and an unknown refusal. + let mut mixed = encode_rep(&DeviceResponse::Released, 0); + mixed.refusal = DeviceRepWire::REFUSE_TAKEN; + assert_eq!(decode_rep(&mixed), None); + let mut alien = encode_rep(&DeviceResponse::Refused(DeviceRefusal::Taken), 0); + alien.refusal = 99; + assert_eq!(decode_rep(&alien), None); + + // An unknown species on a grant. + let mut species = encode_rep( + &DeviceResponse::Granted { + index: 2, + species: DeviceSpecies::Pci, + diag: 0, + revision: 0, + vendor: 0, + device: 0, + }, + 1, + ); + species.species = 7; + assert_eq!(decode_rep(&species), None); + + // A set reserve, on the reply side too. + let mut dirty = encode_rep(&DeviceResponse::Released, 0); + dirty.reserved = [0, 1, 0, 0]; + assert_eq!(decode_rep(&dirty), None); + } + + /// The encoder preserves the host's effected count even when a buggy + /// response still claims success. The decoder then refuses the mismatch; + /// it must never manufacture the missing transfer from the tag. + #[test] + fn encode_never_reconstructs_a_missing_grant_from_the_tag() { + let wire = encode_rep( + &DeviceResponse::Granted { + index: 2, + species: DeviceSpecies::Pci, + diag: 0, + revision: 0, + vendor: 0, + device: 0, + }, + 0, + ); + assert_eq!(wire.grants, 0); + assert_eq!(decode_rep(&wire), None); + } + + /// The ring is ordered but carries no transaction id, so an otherwise + /// valid reply for another verb must not complete the outstanding call. + #[test] + fn a_reply_must_answer_the_outstanding_operation() { + let claim = encode_req(&xhci()); + let release = encode_req(&DeviceRequest::Release { index: 2 }); + let count = encode_req(&DeviceRequest::Count); + let granted = DeviceResponse::Granted { + index: 2, + species: DeviceSpecies::Pci, + diag: 0, + revision: 0, + vendor: 0, + device: 0, + }; + + assert!(::reply_matches( + &claim, &granted, 1 + )); + assert!(::reply_matches( + &release, + &DeviceResponse::Refused(DeviceRefusal::Denied), + 0, + )); + assert!(::reply_matches( + &count, + &DeviceResponse::Counted { len: 4 }, + 0, + )); + assert!(::reply_matches( + &count, + &DeviceResponse::Refused(DeviceRefusal::Denied), + 0, + )); + + assert!(!::reply_matches( + &claim, + &DeviceResponse::Released, + 0, + )); + assert!(!::reply_matches( + &release, &granted, 1, + )); + assert!(!::reply_matches( + &count, &granted, 1, + )); + + let wrong_species = DeviceResponse::Granted { + index: 2, + species: DeviceSpecies::Platform, + diag: 0, + revision: 0, + vendor: 0, + device: 0, + }; + assert!(!::reply_matches( + &claim, + &wrong_species, + 1, + )); + assert!(!::reply_matches( + &release, + &DeviceResponse::Released, + 0, + )); + } +} diff --git a/native/services/extract-worker-c/Cargo.toml b/native/services/extract-worker-c/Cargo.toml new file mode 100644 index 0000000..a94717a --- /dev/null +++ b/native/services/extract-worker-c/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "kore-svc-extract-worker-c" +version = "0.0.1" +edition = "2024" +description = "Isolated C-backed JSON adapter for kore content extraction." +publish = false + +[dependencies] +kore-svc-extract = { path = "../extract" } +heapless = { version = "0.9.3", default-features = false } +blake3 = { version = "=1.8.7", default-features = false, features = ["pure"] } + +[build-dependencies] +cc = "1.4.0" + +[dev-dependencies] +kore-svc-extract-worker-rust = { path = "../extract-worker-rust" } diff --git a/native/services/extract-worker-c/build.rs b/native/services/extract-worker-c/build.rs new file mode 100644 index 0000000..0f81bfb --- /dev/null +++ b/native/services/extract-worker-c/build.rs @@ -0,0 +1,103 @@ +fn main() { + let vendor = "../extract/vendor/jsmn"; + println!("cargo:rerun-if-changed={vendor}/jsmn.h"); + println!("cargo:rerun-if-changed={vendor}/kore_jsmn.c"); + let mut build = cc::Build::new(); + let host = std::env::var("HOST").unwrap_or_default(); + let target = std::env::var("TARGET").unwrap_or_default(); + + // cc-rs has no packaged cross compiler for kore's custom user triples, + // so it falls back to the single-target host `cc`; an x86_64 host then + // silently emits x86_64 objects into the aarch64/riscv worker link and + // LLD correctly refuses the mix. Clang is an all-targets driver and + // cc-rs hands it the exact `--target` triple, so prefer it for every + // freestanding cross build. An explicit CC/KORE_LLVM_CLANG remains + // authoritative. + // + // On Darwin the PATH clang is Xcode's, deliberately built with only + // Apple's native targets: it can compile the aarch64 worker by accident + // on Apple Silicon but fails as soon as the same ordinary image is + // checked for RISC-V, so a complete LLVM toolchain is required there — + // the two fallback paths are Homebrew's documented Apple Silicon and + // Intel prefixes. Every other host keeps its ordinary PATH clang. + if host != target && !target.contains("apple") { + let explicit_cc = std::env::var_os("CC").is_some() + || std::env::var_os(format!("CC_{}", target.replace('-', "_"))).is_some() + || std::env::var_os(format!("CC_{target}")).is_some(); + if !explicit_cc { + let clang = std::env::var_os("KORE_LLVM_CLANG") + .map(std::path::PathBuf::from) + .or_else(|| { + host.contains("apple-darwin") + .then(|| { + [ + "/opt/homebrew/opt/llvm/bin/clang", + "/usr/local/opt/llvm/bin/clang", + ] + .into_iter() + .map(std::path::PathBuf::from) + .find(|path| path.is_file()) + }) + .flatten() + }) + .unwrap_or_else(|| { + if host.contains("apple-darwin") { + panic!( + "freestanding C worker for {target} needs a complete LLVM clang; \ + install Homebrew llvm or set KORE_LLVM_CLANG/CC" + ) + } + std::path::PathBuf::from("clang") + }); + let llvm_ar = clang.with_file_name("llvm-ar"); + build.compiler(clang); + if llvm_ar.is_file() { + build.archiver(llvm_ar); + } + } + } + // Keep the C object's ELF ABI identical to the custom Rust target. Clang + // otherwise defaults bare riscv64 to lp64, while kore's rv64gc user ABI is + // explicitly lp64d; LLD correctly refuses to mix the two. + if std::env::var("CARGO_CFG_TARGET_ARCH").as_deref() == Ok("riscv64") { + build.flag("-march=rv64gc").flag("-mabi=lp64d"); + } + // A developer may have GNU binutils ahead of Xcode in PATH. GNU `ar` + // emits a valid generic archive but does not preserve the 8-byte Mach-O + // member placement required by Apple's linker. Select the platform shim + // for Apple targets; every non-Apple and freestanding build keeps cc-rs's + // target-aware archiver discovery. + if std::env::var("CARGO_CFG_TARGET_VENDOR").as_deref() == Ok("apple") { + build.archiver("/usr/bin/ar"); + } + build + .file(format!("{vendor}/kore_jsmn.c")) + .include(vendor) + .define("JSMN_STRICT", None) + // The worker is an ordinary ET_DYN generation. Compile the vendored + // adapter as PIC as well; otherwise local jump-table/rodata references + // become absolute relocations that the generic PIE linker correctly + // rejects. + .pic(true) + .flag_if_supported("-ffreestanding") + // The kore user ABI has no SysV red zone and this deliberately + // minimal parser worker receives no FPU/SIMD authority. Keep the C + // object on the same integer-only, kernel-safe profile as its Rust + // host image; otherwise clang may issue SSE before the task owns an + // FPU ticket and wedge in the #NM path instead of reporting failure. + .flag_if_supported("-mno-red-zone") + .flag_if_supported("-mno-sse") + .flag_if_supported("-mno-sse2") + .flag_if_supported("-mno-avx") + .flag_if_supported("-mno-avx2") + .flag_if_supported("-mno-mmx") + // res/SPMD.md §13.6 (R7): the x87/MMX bank is reserved as the + // lane tier's register room, so no profile may emit it — this + // worker's soft-float already avoids x87 arithmetic; the flag + // makes the reservation explicit rather than incidental. + .flag_if_supported("-mno-x87") + .flag_if_supported("-msoft-float") + .flag_if_supported("-fno-stack-protector") + .warnings(true) + .compile("kore_jsmn"); +} diff --git a/native/services/extract-worker-c/src/lib.rs b/native/services/extract-worker-c/src/lib.rs new file mode 100644 index 0000000..1d619ba --- /dev/null +++ b/native/services/extract-worker-c/src/lib.rs @@ -0,0 +1,147 @@ +//! C-backed adapter confined to the C worker image. + +#![no_std] + +#[cfg(test)] +extern crate std; + +use heapless::String; +use kore_svc_extract::{ + registry::{C_JSON, Registry}, + schema::{ + Confidence, Fact, FactKind, FactValue, InputIdentity, NormalizedResult, ParserProvenance, + ResultBuilder, SourceRange, + }, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AdapterError { + Malformed, + Truncated, + InputLimit, + OutputLimit, +} + +unsafe extern "C" { + fn kore_jsmn_summary( + bytes: *const u8, + len: usize, + root_members: *mut u32, + token_count: *mut u32, + ) -> i32; +} + +fn text(value: &str) -> Result, AdapterError> { + String::try_from(value).map_err(|_| AdapterError::OutputLimit) +} + +pub fn parse_json_c(input: InputIdentity, bytes: &[u8]) -> Result { + let declaration = Registry::get(C_JSON).expect("built-in C JSON declaration"); + if input.len != bytes.len() as u64 + || input.len > declaration.limits.max_input + || *blake3::hash(bytes).as_bytes() != input.digest + { + return Err(AdapterError::InputLimit); + } + let mut members = 0u32; + let mut tokens = 0u32; + // SAFETY: one immutable slice and two initialized out-pointers remain + // live for the call. The audited wrapper retains no pointer and exposes + // no foreign layout to Rust. + let status = + unsafe { kore_jsmn_summary(bytes.as_ptr(), bytes.len(), &mut members, &mut tokens) }; + if status != 0 || tokens == 0 { + return Err(match status { + -1 => AdapterError::OutputLimit, + -3 => AdapterError::Truncated, + _ => AdapterError::Malformed, + }); + } + let mut builder = ResultBuilder::new(ParserProvenance::deterministic( + input, + C_JSON, + declaration.version, + )); + builder + .fact(Fact { + kind: FactKind::MediaType, + key: text("media_type")?, + value: FactValue::Utf8(text("application/json")?), + confidence: Confidence::CERTAIN, + source: None, + }) + .map_err(|_| AdapterError::OutputLimit)?; + builder + .fact(Fact { + kind: FactKind::JsonMemberCount, + key: text("root_members")?, + value: FactValue::Unsigned(u64::from(members)), + confidence: Confidence::CERTAIN, + source: Some(SourceRange::new(0, input.len)), + }) + .map_err(|_| AdapterError::OutputLimit)?; + Ok(builder.finish()) +} + +#[cfg(test)] +mod tests { + use super::*; + use kore_svc_extract_worker_rust::parse_json_rust; + + fn identity(bytes: &[u8]) -> InputIdentity { + InputIdentity::of(1, 1, bytes) + } + + #[test] + fn rust_and_c_workers_normalize_the_same_fixture() { + let bytes = br#"{"a":1,"b":[2,3]}"#; + let rust = parse_json_rust(identity(bytes), bytes).unwrap(); + let c = parse_json_c(identity(bytes), bytes).unwrap(); + assert_eq!(rust.facts, c.facts); + Registry::validate(&c, identity(bytes), 0).unwrap(); + } + + #[test] + fn malformed_and_truncated_inputs_fail_closed() { + for bytes in [&b"{"[..], &b"[1,"[..], &b"not json"[..]] { + assert!(parse_json_c(identity(bytes), bytes).is_err()); + } + } + + #[test] + fn deterministic_mutation_corpus_stays_bounded_and_normalized() { + let seed = br#"{"alpha":1,"nested":[true,null]}"#; + for index in 0..seed.len() { + for replacement in [0u8, 0xff, seed[index] ^ 0x80] { + let mut bytes = std::vec::Vec::from(seed); + bytes[index] = replacement; + let input = identity(&bytes); + if let Ok(result) = parse_json_c(input, &bytes) { + Registry::validate(&result, input, 0).unwrap(); + } + } + } + for end in 0..seed.len() { + let bytes = &seed[..end]; + let input = identity(bytes); + if let Ok(result) = parse_json_c(input, bytes) { + Registry::validate(&result, input, 0).unwrap(); + } + } + } + + #[test] + #[ignore = "manual vendor benchmark; run with --release --ignored --nocapture"] + fn benchmark_jsmn_adapter() { + use std::time::Instant; + const RUNS: u32 = 50_000; + let bytes = br#"{"alpha":1,"nested":[true,null]}"#; + let input = identity(bytes); + let start = Instant::now(); + for _ in 0..RUNS { + std::hint::black_box(parse_json_c(input, std::hint::black_box(bytes))).unwrap(); + } + let ns = start.elapsed().as_nanos() / u128::from(RUNS); + std::println!("extract-bench jsmn: {ns} ns/op ({} bytes)", bytes.len()); + } +} diff --git a/native/services/mm/Cargo.toml b/native/services/mm/Cargo.toml new file mode 100644 index 0000000..ae20996 --- /dev/null +++ b/native/services/mm/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "kore-svc-mm" +version = "0.0.0" +edition = "2024" +description = "kore memory-manager service: the lessor over one capability-endowed supply region and its trap-transport client." +publish = false + +# The service framework, capability witnesses, fail-closed verbs, and heap +# engine live in kore-rt; this crate owns the mm protocol, serve step, and +# client. +[dependencies] +kore-abi = { path = "../../crates/abi" } +kore-rt = { path = "../../crates/rt" } + +# Tests drive the sharded heap with real host threads (the kore-rt `std` +# feature swaps in the per-thread TLS shard map), and the global-allocator +# integration tests exercise host-side service models beside the allocator. +[dev-dependencies] +kore-rt = { path = "../../crates/rt", features = ["std"] } diff --git a/native/services/mm/boot.toml b/native/services/mm/boot.toml new file mode 100644 index 0000000..094ac5a --- /dev/null +++ b/native/services/mm/boot.toml @@ -0,0 +1,58 @@ +# services/mm/boot.toml — this service's boot-manifest row (the design +# lives in res/boot.toml's header). The row rides beside the crate it +# endows, redox-style; whether and WHEN it founds is the root manifest's +# `services` list — founding order and chain adjacency are topology, the +# root's to declare, never a row's. + +# The lessor of last resort: MEM_SUPPLY is the founder-minted heap-grant +# lease; MEM_REGION (slice D) is the minting authority — a revocable +# self-kind grant of the boot DRAM region, off which `OP_SUPPLY` mints +# each spawned child's root supply lease (quota-metered per serving +# pair; the frames live and die against the region bitmap, so revoking +# this one endowment reclaims every delegated child supply at once). +# +# `shell_client = true` + `exec_client = true` + `SERVE_PAIRS = 4` on the +# protocol: pair 0 +# stays the founder-era pair, pair 1 is the FIRST SHELL's supply face +# (prefab-minted before the founding loop, conveyed through the +# pre-land bundle — the wire `run` allocates child memory over), and +# pair 2 is INIT's face (delivered at its founding round; quiescent +# until the respawn slices). Pair 3 is exec's telemetry client face. +# The rows declaring mm among their `clients` (the compositor's and image's +# heap faces) take pairs 4 and 5, after the protocol's. +# All extra pairs are post-less for returned capabilities. Supply quota uses a +# separate kernel-authenticated lifecycle report after the complete lease +# lineage dies and the region bitmap has accepted the frames (#747). +[service] +image = "kore-svc-mm" +endowment = "kore_svc_mm::MmEndowment" # <_ as Bundle>::PROFILE, cited verbatim +protocol = "kore_svc_mm::MemoryManager" # <_ as SvcProtocol>::REPLY_GRANTS, cited +boot_data = "supply_quota" +# Every engine program holds one per-launch face for its life, and the serve +# host grows mm's face table from mm's own heap: `kore_svc_mm::HEAP_DEMAND_BYTES` +# is that heap's floor — the supply reservations provisioned at bind (128 KiB: +# every lane at a task's adoption ceiling), and every face lane at the host's +# per-slot cost (2,734 bytes today, x86-64) twice over for the doubling table, +# about 1.6 MiB — and exec's founder test `mm_heap_supply_holds_every_face` +# holds this lease to it. The rest of the four MiB is the reservation table's +# room to grow past its provision, and the two heaps' rounding and class +# headroom. +heap_supply_bytes = 0x0040_0000 +# The huge classes' per-client live supply policy, not a reservation or +# physical-memory promise; the receipt's encoding is independent of it. The +# 4 KiB class's quota is not manifest data: mm derives it at bind from the +# class the boot carved, half of it per client (`kore_svc_mm::base_quota_for`), +# read through the read-only observation grant below. +# A streamed-MoE engine funds its model arena from the whole 1 GiB reserve +# (6 GiB on a 16 GB machine) and its expert cache from about 4.5 GiB of the +# 2 MiB class, plus heap. On a 32 GB machine the same engine holds every +# expert in RAM: an 8 GiB arena, the NPU drafter's 1 GiB unit and some +# 15 GiB of expert slots from both classes, about 25 GiB. The budget covers +# both huge classes of such a machine together; the physical classes, not +# this quota, remain what bounds everyone else. +supply_quota_bytes = 27917287424 # 26 GiB +endows = ["MEM_SUPPLY@0", "MEM_REGION@0", "SYSTEM_OBSERVE@0"] # review mirror; asserted === PROFILE.roles +shell_client = true +exec_client = true +digest_x86_64 = "blake3:0c893dbdf58c6c17dd66d424ad025ae0cc5b4eab41a718940f09a3595c32830a" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:f1ffc4cb9f02cc59ef78781fd8a1cba3722d9518b7abb0adb17bf3f7376b9f57" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/mm/src/client.rs b/native/services/mm/src/client.rs new file mode 100644 index 0000000..8e2b873 --- /dev/null +++ b/native/services/mm/src/client.rs @@ -0,0 +1,830 @@ +//! `client` — the mm client face for **peers** (`res/USERLAND.md` §4). +//! +//! [`RingMm`] is the born-async ring + channel proxy over the pair's +//! shared window; [`alloc_props`] builds the request artifact an +//! [`MmRequest::Alloc`] carries. The synchronous global allocator uses its +//! founder-provisioned `HeapSupplySource` so bootstrap cannot depend on mm. +//! Named service heaps can instead use [`crate::heap::MmSource`]: allocation +//! misses return to the async caller, which awaits a real mm refill and +//! retries. No allocator callback polls the executor or simulates a grant. +//! +//! **The grant channel is retained** (slice D): both grant flavors +//! outlive the call, so the runtime rotates an owned landing per +//! delivery and the client takes possession through a typed claim — +//! [`supply`](MmClient::supply) converts its delivered root to a +//! [`StashedLease`] (the image client's shape), while a raw +//! [`request`](MmClient::request) hands back the affine +//! [`ReturnedCapability`] for the caller to shape (the future heap-grow +//! rung's view-consumption story is deliberately its own to design, +//! against the real loop). + +use kore_abi::mem::{AllocProps, DmaClass, MemAttrs, NumaNode}; + +use kore_rt::cap::{ReturnedCapability, StashedLease}; +use kore_rt::heap::EXTENT_ALIGN; + +use crate::service::{MmRequest, MmResponse, SUPPLY_RIGHTS}; + +/// The **request artifact**: the [`AllocProps`] a heap-extent request +/// carries ([`MmRequest::Alloc`]) — `min_bytes` page-rounded, CPU +/// write-back R/W, NUMA-agnostic, non-DMA. Exposed so a client and a +/// test can pin the shape the wire depends on. +/// +/// [`MmRequest::Alloc`]: crate::service::MmRequest::Alloc +pub fn alloc_props(min_bytes: usize) -> AllocProps { + AllocProps { + size: min_bytes.next_multiple_of(EXTENT_ALIGN) as u64, + align: EXTENT_ALIGN as u64, + numa: NumaNode::ANY, + dma: DmaClass::None, + attrs: MemAttrs::WRITE_BACK | MemAttrs::READ | MemAttrs::WRITE, + contiguous: false, + } +} + +/// The **client** side of the mm protocol for *peers* +/// (`res/USERLAND.md` §4: "MmClient exists for peers; the global +/// allocator does NOT use it"). A `Granted` reply's authority arrives +/// beside the response as a delivered, runtime-owned capability — +/// possession-shaped, never reply geometry. +#[allow(async_fn_in_trait)] // clients are generic over the impl (§4) +pub trait MmClient { + /// Whether a fresh request can be published. A bounded administration + /// wait can leave an exact continuation owning this exclusive pair. + fn is_idle(&self) -> bool; + + /// Issue one request; a granted capability rides beside the reply as + /// its affine landing owner (dropping it revokes the delivery — no + /// authority leaks on an ignored grant). + async fn request(&mut self, req: MmRequest) -> (MmResponse, Option); + + /// Mint one child supply ([`MmRequest::Supply`], page-rounded, + /// default alignment): the delivered **root lease**, possessed as a + /// [`StashedLease`] — `slot()` is what stamp views carve off and the + /// spawn convey moves. A reply whose grant is not a lease of + /// [`SUPPLY_RIGHTS`] is refused fail-closed (`None` beside the + /// data-plane response). + async fn supply(&mut self, bytes: u64) -> (MmResponse, Option); + + /// [`supply`](Self::supply) in a named frame class: `align` is the + /// lease alignment in bytes (`0`/4 KiB base class, 2 MiB or 1 GiB huge + /// classes — `res/ACCEL_MEMORY_SCALE.md` §1.2, the alignment *is* the + /// class). A huge-class supply is rounded to a whole number of its + /// units and metered against mm's huge quota, so an accelerator working + /// set neither competes with nor drains the launcher's base allowance. + async fn supply_in_class( + &mut self, + bytes: u64, + align: u64, + ) -> (MmResponse, Option); + + /// Read the lessor's bounded provider-owned snapshot. A malformed or + /// refused response returns `None`; the operation never carries a grant. + async fn stats(&mut self) -> Option; +} + +/// The **ring + channel proxy** (`res/USERLAND.md` §5.6): the peer mm +/// client over the pair's shared window — request pushed, doorbell +/// kicked, reply awaited; a `Granted` reply announces one transfer, +/// which lands in a runtime-rotated owned slot (the retained-channel +/// contract) and reaches the caller as its affine owner. +/// +/// An infrastructure failure of the request path **panics** (the +/// executor's transport-failure posture): the closed `MmRefusal` +/// vocabulary has no honest infra arm, and inventing one would let a +/// torn pair masquerade as a lessor refusal. +pub struct RingMm<'w, 't, T: kore_rt::transport::Transport>( + pub kore_rt::svc::SvcClient<'w, 't, T, crate::service::MemoryManager>, +); + +impl RingMm<'_, '_, T> { + async fn call(&mut self, req: &MmRequest) -> (MmResponse, Option) { + let mut reply = self + .0 + .call(req) + .await + .expect("mm request path failed (torn rings/transport — a broken pair)"); + let delivered = (reply.grants > 0) + .then(|| reply.retained.get_mut(0).and_then(Option::take)) + .flatten(); + (reply.response, delivered) + } +} + +/// **Administration of per-launch faces** — exec's pair only, over its +/// retained return leg (`return_tx` sends, `return_land` is the landing the +/// request announces). The shapes are the accelerator and power faces'. +impl RingMm<'_, '_, T> { + /// Derive and convey one face component atomically while retaining the + /// source lineage. `AfterTransfer` means custody moved: resume this + /// operation rather than deriving and conveying a second component. + pub async fn admit_component_staged( + &mut self, + return_tx: kore_abi::session::SlotRef, + return_land: kore_abi::session::SlotRef, + part: kore_rt::svc::PeerBundlePart, + rights: kore_abi::cap::Rights, + stage: F, + ) -> Result + where + F: FnOnce( + kore_rt::svc::ReturnDeriver<'_>, + ) -> Result, + { + let request = MmRequest::AdmitPeer { + part, + announced: return_land, + }; + let reply = self + .0 + .call_with_staged_return(&request, return_tx, rights, stage) + .await?; + Ok(reply.response) + } + + /// Consume the identity of the most recently assembled face. + pub async fn take_peer_admission(&mut self) -> Result { + self.0 + .call(&MmRequest::TakeAdmission) + .await + .map(|reply| reply.response) + } + + /// Carry one exact administration request forward under `observation`. + /// Maintenance observes in slices, so a departing process that still has + /// DMA supplies outstanding cannot monopolize exec's serve loop; an + /// expired slice retains the exact request on this pair for the next + /// call with the same request. + async fn peer_control( + &mut self, + request: MmRequest, + observation: kore_rt::svc::Observation, + ) -> Result { + use kore_rt::svc::CallError; + let reply = self.0.call_exact(&request, observation).await?; + if reply.grants != 0 { + self.0.poison_retained(reply.retained); + return Err(CallError::Wire); + } + Ok(reply.response) + } + + pub async fn take_peer_admission_observed( + &mut self, + observation: kore_rt::svc::Observation, + ) -> Result { + self.peer_control(MmRequest::TakeAdmission, observation) + .await + } + + pub async fn retire_peer_observed( + &mut self, + peer: kore_rt::svc::PeerId, + observation: kore_rt::svc::Observation, + ) -> Result { + self.peer_control(MmRequest::RetirePeer { peer }, observation) + .await + } + + pub async fn confirm_retired_observed( + &mut self, + peer: kore_rt::svc::PeerId, + observation: kore_rt::svc::Observation, + ) -> Result { + self.peer_control(MmRequest::ConfirmRetired { peer }, observation) + .await + } + + /// Request retirement of one exact face generation. + pub async fn retire_peer( + &mut self, + peer: kore_rt::svc::PeerId, + ) -> Result { + self.0 + .call(&MmRequest::RetirePeer { peer }) + .await + .map(|reply| reply.response) + } + + /// Fence retirement: `Accepted` only once the face is closed and every + /// supply it minted has returned, so the caller reclaims the face's + /// local backing no earlier. `Refused(Pending)` means ask again later. + pub async fn confirm_retired( + &mut self, + peer: kore_rt::svc::PeerId, + ) -> Result { + self.0 + .call(&MmRequest::ConfirmRetired { peer }) + .await + .map(|reply| reply.response) + } +} + +/// Where an engine-profile process finds its per-launch mm face. +/// +/// Window applications occupy index 0 of the `SVC_RING`/`SVC_SEND`/ +/// `SVC_GRANT` namespaces, the NPU and power faces index 1 and the GPU face +/// index 2, so the mm face takes index 3 and an engine program may hold +/// every one of them (exec pins the disjointness where it sees all the +/// faces). A face mints supplies and reads the books only, so no return leg +/// reaches the process. +pub mod process_face { + /// mm request doorbell (`role::SVC_SEND`). + pub const REQUEST_KICK: u16 = 3; + /// mm request/reply ring backing (`role::SVC_RING`). + pub const CONTROL_RING: u16 = 3; + /// Receiver for mm's reply grants (`role::SVC_GRANT`). + pub const GRANT_RECV: u16 = 3; +} + +/// A process's own mm face plus the affine ring lease that keeps its mapped +/// control window alive. Its supplies are metered on the face's own lane. +#[cfg(target_os = "none")] +pub struct ProcessMm { + pub client: RingMm<'static, 'static, kore_rt::transport::TrapTransport>, + _ring: kore_rt::cap::LeaseToken, +} + +/// The class a heap reservation is minted in: 2 MiB, so every granule the +/// heap window maps off it is a huge leaf and lands end to end, and so it is +/// metered against the face's huge-class quota (the boot policy's), not the +/// base class's. +#[cfg(any(target_os = "none", test))] +const HEAP_RESERVATION_CLASS: u64 = 0x20_0000; + +/// The most one lease of a heap reservation asks for. A lease is one +/// physically contiguous range of its class, and a reservation is as many +/// leases as it takes: 128 MiB is what the kernel's 2 MiB class has lent a +/// single launch on the 4 GiB acceptance machine (the wasm witness's), where +/// one 1 GiB ask was refused outright. The heap adopts each lease +/// ([`kore_rt::heap::MAX_ADOPTED`] of them at most), and grows through them +/// in turn. +#[cfg(any(target_os = "none", test))] +pub const HEAP_RESERVATION_LEASE_BYTES: u64 = 128 << 20; + +/// The least one lease asks for. A `Denied` grant is the class without a +/// free run that long, so the ask halves down to this and no further: a +/// class that cannot lend 16 MiB contiguous is spent, not fragmented. The +/// `wasm` command's 1 GiB ceiling is 64 such leases, the adoption bound. +#[cfg(any(target_os = "none", test))] +pub const HEAP_RESERVATION_LEASE_MIN_BYTES: u64 = 16 << 20; + +/// Why a heap reservation stopped short, and how much of it stands. +#[cfg(any(target_os = "none", test))] +#[derive(Debug)] +pub struct HeapReservationRefused { + /// mm's own answer (`Quota` names the face's budget), or `Denied` for a + /// grant the heap could not adopt (the lease is then revoked). + pub response: MmResponse, + /// Bytes reserved and adopted before the refusal — the heap's to keep. + pub reserved: u64, +} + +/// Why one lease of a reservation did not join the heap. +#[cfg(any(target_os = "none", test))] +#[derive(Debug)] +enum Unlent { + /// mm did not lend it: its own answer. + Refused(MmResponse), + /// mm lent it and the heap could not adopt it (every adoption entry + /// taken); the lease was revoked. A shorter one would fare no better. + NotAdopted, +} + +/// [`ProcessMm::reserve_heap`]'s policy, apart from the face: `lend(ask)` +/// asks for one `ask`-byte lease of the class and adopts it into the heap. +#[cfg(any(target_os = "none", test))] +async fn reserve_in_leases( + bytes: u64, + lend: &mut impl AsyncFnMut(u64) -> Result<(), Unlent>, +) -> Result { + let mut reserved = 0; + let mut lease_bytes = HEAP_RESERVATION_LEASE_BYTES; + while reserved < bytes { + // Asked in whole units of the class, as mm lends it, so `reserved` + // counts exactly what the heap adopted. + let ask = (bytes - reserved) + .min(lease_bytes) + .next_multiple_of(HEAP_RESERVATION_CLASS); + match lend(ask).await { + Ok(()) => reserved += ask, + // A run the class could not lend: each retry asks for strictly + // less than the run just refused, so an ask already at the floor + // is not repeated. + Err(Unlent::Refused(MmResponse::Refused(crate::service::MmRefusal::Denied))) + if ask > HEAP_RESERVATION_LEASE_MIN_BYTES => + { + lease_bytes = (ask / 2) + .next_multiple_of(HEAP_RESERVATION_CLASS) + .max(HEAP_RESERVATION_LEASE_MIN_BYTES); + } + Err(Unlent::Refused(response)) => { + return Err(HeapReservationRefused { response, reserved }); + } + Err(Unlent::NotAdopted) => { + return Err(HeapReservationRefused { + response: MmResponse::Refused(crate::service::MmRefusal::Denied), + reserved, + }); + } + } + } + Ok(reserved) +} + +/// [`ProcessMm::reserve_heap_ladder`]'s policy, apart from the face. +#[cfg(any(target_os = "none", test))] +async fn reserve_down_ladder( + targets: &[u64], + lend: &mut impl AsyncFnMut(u64) -> Result<(), Unlent>, +) -> (u64, Option) { + let mut reserved = 0; + let mut refusal = None; + for &target in targets { + if reserved >= target { + return (reserved, None); + } + match reserve_in_leases(target - reserved, lend).await { + Ok(gained) => return (reserved + gained, None), + Err(refused) => { + reserved += refused.reserved; + refusal = Some(refused); + } + } + } + (reserved, refusal) +} + +/// [`ProcessMm::reserve_heap_window`]'s policy, apart from the face: one ask +/// for the whole window, in whole units of the class, never split or retried +/// shorter — a window is one carve, so a shorter run cannot hold it. A +/// window no ask can represent is refused as `BadProps` without asking. +#[cfg(any(target_os = "none", test))] +async fn reserve_window( + bytes: u64, + lend: &mut impl AsyncFnMut(u64) -> Result<(), Unlent>, +) -> Result { + let refused = |refusal| HeapReservationRefused { + response: MmResponse::Refused(refusal), + reserved: 0, + }; + let ask = bytes + .max(1) + .checked_next_multiple_of(HEAP_RESERVATION_CLASS) + .ok_or(refused(crate::service::MmRefusal::BadProps))?; + match lend(ask).await { + Ok(()) => Ok(ask), + Err(Unlent::Refused(response)) => Err(HeapReservationRefused { + response, + reserved: 0, + }), + Err(Unlent::NotAdopted) => Err(refused(crate::service::MmRefusal::Denied)), + } +} + +#[cfg(target_os = "none")] +impl ProcessMm { + /// **Reserve heap before a large synchronous job.** Ask mm for `bytes` + /// more on this process's own face — as leases of at most + /// [`HEAP_RESERVATION_LEASE_BYTES`], each rounded up to whole 2 MiB + /// units, a `Denied` run retried at half its length down to + /// [`HEAP_RESERVATION_LEASE_MIN_BYTES`] — and adopt each grant into the + /// heap ([`kore_rt::heap::adopt_supply`]), so the job's allocations — + /// which cannot wait on a service — find room. `Ok` is the bytes the + /// heap adopted: `bytes` rounded up to the 2 MiB unit. + /// + /// A refusal carries mm's own answer (`Quota` names the face's budget + /// and ends the reservation at once; `Denied` is a run the class could + /// not lend at the floor, and also a grant the heap could not adopt, + /// which revokes the lease, never leaks it) and how much stood before + /// it. Leases adopted before a refusal stay the heap's: a caller that + /// goes on anyway has that much more room, not less. + pub async fn reserve_heap(&mut self, bytes: u64) -> Result { + reserve_in_leases(bytes, &mut async |ask| { + self.lend(ask, kore_rt::heap::adopt_supply).await + }) + .await + } + + /// Make room for a step that maps `bytes` afresh: what the supplies the + /// heap adopted still hold ([`kore_rt::heap::heap_room`]), topped up on + /// this face when short, by at least [`HEAP_RESERVATION_LEASE_MIN_BYTES`] + /// so a long-lived process holds few leases. `false` when the room still + /// falls short; what was adopted before a refusal stays the heap's. + pub async fn ensure_heap_room(&mut self, bytes: u64) -> bool { + let room = kore_rt::heap::heap_room(); + if room >= bytes { + return true; + } + let _ = self + .reserve_heap((bytes - room).max(HEAP_RESERVATION_LEASE_MIN_BYTES)) + .await; + kore_rt::heap::heap_room() >= bytes + } + + /// **Reserve room for one window of `bytes`** that this process will + /// map itself ([`kore_rt::heap::reserve_from_top`]) — a guest memory's + /// next commits, a code image — when no supply it holds for windows + /// has that much room left ([`kore_rt::heap::window_supply_room`]). A + /// window is one carve, so this is **one** lease of `bytes` rounded up + /// to the 2 MiB unit, never split as [`reserve_heap`](Self::reserve_heap) splits: + /// adopted for windows alone ([`kore_rt::heap::adopt_window_supply`]), + /// so heap growth cannot shorten it before the window is carved. `Ok` + /// is the lease's length; a refusal is mm's answer (`Denied` when the + /// class has no run that long, or the heap could not adopt the lease), + /// with nothing reserved. + pub async fn reserve_heap_window(&mut self, bytes: u64) -> Result { + reserve_window(bytes, &mut async |ask| { + self.lend(ask, kore_rt::heap::adopt_window_supply).await + }) + .await + } + + /// [`reserve_heap`](Self::reserve_heap) down a ladder of `targets`, + /// largest first: the heap grows to the first target the face can bring + /// it to. A refused step keeps what it adopted, so the next step asks + /// only for what its own target still lacks, and the bytes returned are + /// exactly what the heap gained — the number a caller takes off a census + /// it read before reserving. The refusal is the last step's, returned + /// only when even the smallest target was not reached. + pub async fn reserve_heap_ladder( + &mut self, + targets: &[u64], + ) -> (u64, Option) { + reserve_down_ladder(targets, &mut async |ask| { + self.lend(ask, kore_rt::heap::adopt_supply).await + }) + .await + } + + /// One lease of a reservation, adopted by `adopt`. + async fn lend( + &mut self, + ask: u64, + adopt: fn(kore_rt::cap::StashedLease) -> Result<(), kore_rt::cap::StashedLease>, + ) -> Result<(), Unlent> { + let (response, lease) = self + .client + .supply_in_class(ask, HEAP_RESERVATION_CLASS) + .await; + let lease = lease.ok_or(Unlent::Refused(response))?; + // A refused adoption hands the lease back; dropping it revokes it. + adopt(lease).map_err(|_| Unlent::NotAdopted) + } +} + +/// Attach the per-launch mm face an engine-profile launch delivered. +/// `None` for a process launched without one. +#[cfg(target_os = "none")] +pub fn attach_process_face(rt: &kore_rt::entry::Rt) -> Option { + use kore_abi::task::role; + use kore_rt::cap::verbs; + use kore_rt::svc::{ClientEndpoints, GrantPin, SvcClient, client_rings}; + + let boot = rt.bootstrap(); + let ring = verbs::take_lease(boot.handle(role::SVC_RING, process_face::CONTROL_RING)?)?; + let window = verbs::map_held_rw(ring.slot())?; + // SAFETY: this process is the sole client-side attachment over the + // task-lifetime ring lease delivered by exec. + let window: &'static mut [u8] = + unsafe { core::slice::from_raw_parts_mut(window.cast::().as_ptr(), window.len()) }; + let rings = + client_rings::(window, kore_rt::svc::REQ_CAP, kore_rt::svc::REP_CAP) + .ok()?; + let kick = boot.handle(role::SVC_SEND, process_face::REQUEST_KICK)?; + let grant = boot.handle(role::SVC_GRANT, process_face::GRANT_RECV)?; + let mut grants = [const { None }; kore_rt::service::MAX_GRANTS]; + grants[0] = Some(GrantPin::retained(grant)); + Some(ProcessMm { + client: RingMm( + SvcClient::new( + kore_rt::entry::transport(), + rings, + ClientEndpoints { kick, grants }, + ) + .with_reply_wait(rt.doorbell()?), + ), + _ring: ring, + }) +} + +impl MmClient for RingMm<'_, '_, T> { + fn is_idle(&self) -> bool { + self.0.is_idle() + } + + async fn request(&mut self, req: MmRequest) -> (MmResponse, Option) { + self.call(&req).await + } + + async fn supply(&mut self, bytes: u64) -> (MmResponse, Option) { + self.supply_in_class(bytes, 0).await + } + + async fn supply_in_class( + &mut self, + bytes: u64, + align: u64, + ) -> (MmResponse, Option) { + let (response, delivered) = self.call(&MmRequest::Supply { size: bytes, align }).await; + let lease = delivered.and_then(|owner| { + // Kind + rights checked at possession; a mismatched delivery + // drops through the owner (revoked, slot reopened) — the + // fail-closed posture, never an optimistic bind. + owner.try_into_stashed_lease(SUPPLY_RIGHTS).ok() + }); + (response, lease) + } + + async fn stats(&mut self) -> Option { + let (response, delivered) = self.call(&MmRequest::Stats).await; + if delivered.is_some() { + return None; + } + match response { + MmResponse::Stats(stats) if stats.validate() && stats.available() => Some(stats), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn alloc_props_builds_the_real_request_artifact() { + // The wire artifact `MmRequest::Alloc` carries — pinned here. + let p = alloc_props(1000); + assert_eq!(p.align, EXTENT_ALIGN as u64); + assert_eq!(p.size, EXTENT_ALIGN as u64, "min_bytes rounded to a page"); + assert_eq!(p.dma, DmaClass::None, "a CPU heap extent is non-DMA"); + assert!(!p.contiguous, "a heap extent needs no contiguity"); + assert!( + p.attrs + .contains(MemAttrs::WRITE_BACK | MemAttrs::READ | MemAttrs::WRITE) + ); + assert_eq!(p.numa, NumaNode::ANY); + } + + const MIB: u64 = 1 << 20; + + /// A face that lends runs no longer than `run` while its `quota` lasts, + /// into a heap with `slots` adoption entries left — and records every + /// ask and every byte it lent. + struct Face { + quota: u64, + run: u64, + slots: usize, + asks: std::vec::Vec, + lent: u64, + } + + impl Face { + fn new(quota: u64, run: u64, slots: usize) -> Self { + Self { + quota, + run, + slots, + asks: std::vec::Vec::new(), + lent: 0, + } + } + + fn lend(&mut self, ask: u64) -> Result<(), Unlent> { + use crate::service::MmRefusal::{Denied, Quota}; + self.asks.push(ask); + assert_eq!(ask % HEAP_RESERVATION_CLASS, 0, "asks are whole units"); + if ask > self.quota { + return Err(Unlent::Refused(MmResponse::Refused(Quota))); + } + if ask > self.run { + return Err(Unlent::Refused(MmResponse::Refused(Denied))); + } + // The lease was lent, so it counts against the quota even when + // the heap cannot adopt it (mm sees its revocation later). + self.quota -= ask; + if self.slots == 0 { + return Err(Unlent::NotAdopted); + } + self.slots -= 1; + self.lent += ask; + Ok(()) + } + + fn reserve(&mut self, bytes: u64) -> Result { + kore_rt::block_on(reserve_in_leases(bytes, &mut async |ask| self.lend(ask))) + } + + fn ladder(&mut self, targets: &[u64]) -> (u64, Option) { + kore_rt::block_on(reserve_down_ladder(targets, &mut async |ask| { + self.lend(ask) + })) + } + + fn window(&mut self, bytes: u64) -> Result { + kore_rt::block_on(reserve_window(bytes, &mut async |ask| self.lend(ask))) + } + } + + /// A class of free runs, lent first-fit: a lease is carved from the + /// first run at least as long, which shrinks by it. + struct Runs(std::vec::Vec); + + impl Runs { + fn lend(&mut self, ask: u64) -> Result<(), Unlent> { + use crate::service::MmRefusal::Denied; + let run = self + .0 + .iter_mut() + .find(|run| **run >= ask) + .ok_or(Unlent::Refused(MmResponse::Refused(Denied)))?; + *run -= ask; + Ok(()) + } + + fn reserve(&mut self, bytes: u64) -> Result { + kore_rt::block_on(reserve_in_leases(bytes, &mut async |ask| self.lend(ask))) + } + + fn window(&mut self, bytes: u64) -> Result { + kore_rt::block_on(reserve_window(bytes, &mut async |ask| self.lend(ask))) + } + } + + /// Why a program that needs both reserves its window first. The heap + /// asks for its whole length, then halves, so its first lease takes + /// the class's longest run; a window asked for afterwards finds no run + /// of its own length, though one stood free before (the `wasm` command's + /// CPython window beside a loaded model). Asked first, the window takes + /// its run and the heap gathers its bytes from what is left. + #[test] + fn a_window_reserved_before_the_heap_finds_the_run_it_needs() { + use crate::service::MmRefusal::Denied; + let fragmented = || Runs(std::vec![40 * MIB, 16 * MIB, 16 * MIB, 16 * MIB, 16 * MIB]); + let (window, heap) = (22 * MIB, 70 * MIB); + + let mut heap_first = fragmented(); + assert_eq!(heap_first.reserve(heap).ok(), Some(heap)); + let refused = heap_first.window(window).unwrap_err(); + assert_eq!((refused.response, refused.reserved), (MmResponse::Refused(Denied), 0)); + + let mut window_first = fragmented(); + assert_eq!(window_first.window(window).ok(), Some(window)); + assert_eq!(window_first.reserve(heap).ok(), Some(heap)); + } + + /// A window is one lease of its whole length, rounded to the class, and + /// never split: past the longest run it is refused at once, keeping + /// nothing, where a reservation would have taken shorter leases. + #[test] + fn a_window_is_one_whole_lease() { + use crate::service::MmRefusal::{BadProps, Denied, Quota}; + let mut face = Face::new(u64::MAX, u64::MAX, 64); + assert_eq!(face.window(200 * MIB + 1).ok(), Some(202 * MIB)); + assert_eq!(face.window(0).ok(), Some(2 * MIB)); + assert_eq!(face.asks, [202 * MIB, 2 * MIB]); + // Longer than the class's longest run: one ask, refused. + let mut face = Face::new(u64::MAX, 64 * MIB, 64); + let refused = face.window(80 * MIB).unwrap_err(); + assert_eq!(refused.response, MmResponse::Refused(Denied)); + assert_eq!((refused.reserved, face.lent), (0, 0)); + assert_eq!(face.asks, [80 * MIB]); + // The quota's answer is mm's own; an unadoptable lease is Denied. + let mut face = Face::new(32 * MIB, u64::MAX, 64); + assert_eq!( + face.window(48 * MIB).unwrap_err().response, + MmResponse::Refused(Quota) + ); + let mut face = Face::new(u64::MAX, u64::MAX, 0); + assert_eq!( + face.window(48 * MIB).unwrap_err().response, + MmResponse::Refused(Denied) + ); + // A window no ask can represent is refused without asking. + let mut face = Face::new(u64::MAX, u64::MAX, 64); + assert_eq!( + face.window(u64::MAX).unwrap_err().response, + MmResponse::Refused(BadProps) + ); + assert!(face.asks.is_empty()); + } + + #[test] + fn a_reservation_is_whole_leases_of_the_class() { + let mut face = Face::new(u64::MAX, u64::MAX, 64); + assert_eq!(face.reserve(300 * MIB + 1).ok(), Some(302 * MIB)); + assert_eq!(face.asks, [128 * MIB, 128 * MIB, 46 * MIB]); + assert_eq!(face.lent, 302 * MIB); + } + + #[test] + fn a_denied_run_is_retried_strictly_shorter_down_to_the_floor() { + // A class whose longest free run is 40 MiB lends 32 MiB leases. + let mut face = Face::new(u64::MAX, 40 * MIB, 64); + assert_eq!(face.reserve(64 * MIB).ok(), Some(64 * MIB)); + assert_eq!(face.asks, [64 * MIB, 32 * MIB, 32 * MIB]); + // One without a 16 MiB run is spent: the floor is asked once. + let mut face = Face::new(u64::MAX, 8 * MIB, 64); + let refused = face.reserve(64 * MIB).unwrap_err(); + assert_eq!(face.asks, [64 * MIB, 32 * MIB, 16 * MIB]); + assert_eq!(refused.reserved, 0); + // An ask already under the floor is not repeated either. + let mut face = Face::new(u64::MAX, 8 * MIB, 64); + assert!(face.reserve(10 * MIB).is_err()); + assert_eq!(face.asks, [10 * MIB]); + } + + #[test] + fn a_refusal_keeps_what_was_adopted() { + use crate::service::MmRefusal::{Denied, Quota}; + let mut face = Face::new(200 * MIB, u64::MAX, 64); + let refused = face.reserve(256 * MIB).unwrap_err(); + assert_eq!(refused.response, MmResponse::Refused(Quota)); + assert_eq!((refused.reserved, face.lent), (128 * MIB, 128 * MIB)); + // A grant the heap cannot adopt ends the reservation: a shorter one + // would fare no better, so nothing more is asked. + let mut face = Face::new(u64::MAX, u64::MAX, 1); + let refused = face.reserve(256 * MIB).unwrap_err(); + assert_eq!(refused.response, MmResponse::Refused(Denied)); + assert_eq!((refused.reserved, face.lent), (128 * MIB, 128 * MIB)); + assert_eq!(face.asks, [128 * MIB, 128 * MIB]); + } + + /// The census a caller reads before reserving is short by exactly what + /// the ladder returns, whatever stopped it: a step refused part-way + /// keeps its leases, and the next step asks only for its own deficit. + #[test] + fn the_ladder_returns_exactly_what_the_heap_gained() { + const LADDER: [u64; 3] = [256 * MIB, 128 * MIB, 64 * MIB]; + // A 256 MiB step that adopts 192 MiB in 64 MiB runs, then meets + // the quota, already stands past the 128 MiB step. + let mut face = Face::new(192 * MIB, 64 * MIB, 64); + let (gained, refusal) = face.ladder(&LADDER); + assert_eq!((gained, face.lent), (192 * MIB, 192 * MIB)); + assert!(refusal.is_none()); + assert_eq!( + face.asks, + [128 * MIB, 64 * MIB, 64 * MIB, 64 * MIB, 64 * MIB] + ); + // A quota under the first two steps' first lease reaches the third. + let mut face = Face::new(100 * MIB, u64::MAX, 64); + assert_eq!(face.ladder(&LADDER).0, 64 * MIB); + assert_eq!(face.asks, [128 * MIB, 128 * MIB, 64 * MIB]); + // A 64 MiB step refused after keeping 32 MiB asks the 40 MiB step + // for the 8 MiB it still lacks, not for 40 MiB more. + let mut face = Face::new(u64::MAX, 32 * MIB, 1); + let (gained, refusal) = face.ladder(&[64 * MIB, 40 * MIB]); + assert_eq!((gained, face.lent), (32 * MIB, 32 * MIB)); + assert_eq!(face.asks, [64 * MIB, 32 * MIB, 32 * MIB, 8 * MIB]); + assert!(refusal.is_some()); + // Every face: the gain is what was lent, never past the top step, + // and a refusal is reported exactly when no step was reached. + for quota in [0, 10, 40, 64, 100, 128, 192, 250, 256, 1024] { + for run in [8, 16, 24, 40, 64, 128, 1024] { + for slots in [0, 1, 2, 3, 64] { + let mut face = Face::new(quota * MIB, run * MIB, slots); + let (gained, refusal) = face.ladder(&LADDER); + let case = (quota, run, slots); + assert_eq!(gained, face.lent, "{case:?}"); + assert!(gained <= LADDER[0], "{case:?}"); + assert_eq!(refusal.is_some(), gained < LADDER[2], "{case:?}"); + } + } + } + } +} + +#[cfg(target_os = "none")] +impl RingMm<'static, 'static, kore_rt::transport::TrapTransport> { + /// Attach this task's declared mm client and its actual executor bell. + pub fn attach_declared(rt: &kore_rt::entry::Rt, ordinal: u16) -> Option { + use kore_abi::task::{SERVICE_CLIENT_INDEX_BASE, role}; + let index = SERVICE_CLIENT_INDEX_BASE.checked_add(ordinal)?; + let window = kore_rt::cap::verbs::map_view(kore_rt::cap::verbs::take_view( + rt.bootstrap().handle(role::SVC_RING, index)?, + )?)?; + // SAFETY: the boot declaration assigns this task the sole client half. + let bytes = unsafe { &mut *window.as_ptr() }; + let rings = kore_rt::svc::client_rings::( + bytes, + kore_rt::svc::REQ_CAP, + kore_rt::svc::REP_CAP, + ) + .ok()?; + let kick = rt.bootstrap().handle(role::SVC_SEND, index)?; + let grant_index = SERVICE_CLIENT_INDEX_BASE + .checked_add(ordinal.checked_mul(kore_rt::service::MAX_GRANTS as u16)?)?; + let grant = rt.bootstrap().handle(role::SVC_GRANT, grant_index)?; + let mut grants = [const { None }; kore_rt::service::MAX_GRANTS]; + grants[0] = Some(kore_rt::svc::GrantPin::retained(grant)); + Some(Self( + kore_rt::svc::SvcClient::new( + kore_rt::entry::transport(), + rings, + kore_rt::svc::ClientEndpoints { kick, grants }, + ) + .with_reply_wait(rt.doorbell()?), + )) + } +} diff --git a/native/services/mm/src/heap.rs b/native/services/mm/src/heap.rs new file mode 100644 index 0000000..dd7578b --- /dev/null +++ b/native/services/mm/src/heap.rs @@ -0,0 +1,536 @@ +//! Async preparation for the ordinary `kore_rt::Heap` engine. +//! +//! A miss records the required extent size and returns `AllocError`. The +//! owning service awaits `refill` over its existing mm face, then retries the +//! allocation once. No allocator callback runs an executor or polls a ring. +//! Each extent stores its own affine backing owner ahead of the heap bytes. + +use crate::{MmClient, MmRefusal, MmResponse}; +use core::{ + ptr::NonNull, + sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}, +}; +use kore_rt::{ + cap::{MappedLeaseView, StashedLease, verbs}, + mem_source::{ALIGN, MemSource}, +}; + +const _: () = assert!(core::mem::align_of::() <= ALIGN); + +struct Extent { + next: *mut Extent, + backing: StashedLease, + view: Option, + bytes: usize, + cursor: AtomicUsize, +} + +/// Why a cold heap refill could not make a new extent available. These are +/// observations of the existing path, not requests for a different policy. +#[repr(u8)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RefillFailure { + PairBusy = 1, + SourceBusy = 2, + NoDemand = 3, + ExtentOverflow = 4, + MmBadProps = 5, + MmExhausted = 6, + MmBadReturn = 7, + MmDenied = 8, + MmQuota = 9, + MmPending = 10, + UnexpectedReply = 11, + MissingLease = 12, + InstallFailed = 13, + /// The refill succeeded, but the caller's exact allocation still failed. + ReserveAfterRefill = 14, + /// mm answered [`MmRefusal::Full`]. Its supply path never does — that + /// refusal is a face admission's — but the mapping stays total, so a + /// reply outside the request's vocabulary is named, not miscounted. + MmFull = 15, +} + +impl RefillFailure { + pub const fn from_code(code: u8) -> Option { + Some(match code { + 1 => Self::PairBusy, + 2 => Self::SourceBusy, + 3 => Self::NoDemand, + 4 => Self::ExtentOverflow, + 5 => Self::MmBadProps, + 6 => Self::MmExhausted, + 7 => Self::MmBadReturn, + 8 => Self::MmDenied, + 9 => Self::MmQuota, + 10 => Self::MmPending, + 11 => Self::UnexpectedReply, + 12 => Self::MissingLease, + 13 => Self::InstallFailed, + 14 => Self::ReserveAfterRefill, + 15 => Self::MmFull, + _ => return None, + }) + } + + const fn from_refusal(refusal: MmRefusal) -> Self { + match refusal { + MmRefusal::BadProps => Self::MmBadProps, + MmRefusal::Exhausted => Self::MmExhausted, + MmRefusal::BadReturn => Self::MmBadReturn, + MmRefusal::Denied => Self::MmDenied, + MmRefusal::Quota => Self::MmQuota, + MmRefusal::Pending => Self::MmPending, + MmRefusal::Full => Self::MmFull, + } + } +} + +/// An asynchronously supplied, named heap source. Worker allocation only +/// claims already-published extents or records a miss; it never runs IPC. +/// The owning service awaits refill. Extent headers are immutable after +/// publication except for their atomic, monotone page cursor. Supplied memory +/// remains owned until the heap is destroyed; freed blocks recycle in Heap. +pub struct MmSource { + extents: AtomicPtr, + wanted: AtomicUsize, + refilling: AtomicBool, + mapping: Option<(u64, u64)>, + mapped_bytes: AtomicUsize, +} + +impl MmSource { + pub const fn new() -> Self { + Self { + extents: AtomicPtr::new(core::ptr::null_mut()), + wanted: AtomicUsize::new(0), + refilling: AtomicBool::new(false), + mapping: None, + mapped_bytes: AtomicUsize::new(0), + } + } + + /// Use an exclusively owned, founder-declared virtual address reservation. + /// Physical pages are still supplied on demand; this is not a memory budget. + /// + /// # Safety + /// No other mapper may install into this span while the source is live. + pub unsafe fn in_window(base: u64, bytes: u64) -> Self { + let mut source = Self::new(); + source.mapping = Some((base, bytes)); + source + } + + /// Satisfy the preceding allocation miss through the caller's real mm + /// client. A refusal preserves every previously allocated extent. + pub async fn refill(&self, memory: &mut impl MmClient) -> bool { + self.refill_detailed(memory).await.is_ok() + } + + /// The same refill, retaining its terminal reason for a caller's failure + /// witness. A successful result still needs the caller to retry its own + /// allocation; the source never promises an allocation on its behalf. + pub async fn refill_detailed(&self, memory: &mut impl MmClient) -> Result<(), RefillFailure> { + self.provide(memory, 0).await + } + + /// Establish an initial mm-supplied extent before advertising readiness. + /// This verifies and warms the real client path even when a bootstrap + /// supply still has space. Subsequent misses use `refill` on demand. + pub async fn reserve_extent(&self, memory: &mut impl MmClient, bytes: usize) -> bool { + self.provide(memory, bytes).await.is_ok() + } + + async fn provide(&self, memory: &mut impl MmClient, minimum: usize) -> Result<(), RefillFailure> { + // A deferred close keeps its reply on this pair. A refill must leave + // both that continuation and the recorded allocation demand intact. + if !memory.is_idle() { + return Err(RefillFailure::PairBusy); + } + // This is a non-blocking per-source admission guard on the cold IPC + // path. Allocation never waits on it; concurrent refill callers retry. + if self + .refilling + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + return Err(RefillFailure::SourceBusy); + } + let wanted = self.wanted.swap(0, Ordering::AcqRel).max(minimum); + let mut guard = RefillGuard { + active: &self.refilling, + wanted: &self.wanted, + pending: wanted, + }; + if wanted == 0 { + return Err(RefillFailure::NoDemand); + } + if minimum == 0 && self.has_ready(wanted) { + guard.pending = 0; + return Ok(()); + } + let Some(bytes) = extent_bytes(wanted) else { + guard.pending = 0; + return Err(RefillFailure::ExtentOverflow); + }; + let (reply, backing) = memory.supply(bytes as u64).await; + // A completed refusal belongs to this request. Retaining its maximum + // forever would let an oversized surface starve later small requests. + // Cancellation alone restores demand; concurrent new misses survive. + guard.pending = 0; + let backing = match (reply, backing) { + (MmResponse::Granted, Some(backing)) => backing, + (MmResponse::Granted, None) => return Err(RefillFailure::MissingLease), + (MmResponse::Refused(refusal), _) => { + return Err(RefillFailure::from_refusal(refusal)); + } + _ => return Err(RefillFailure::UnexpectedReply), + }; + if !self.install(backing, bytes) { + return Err(RefillFailure::InstallFailed); + } + guard.pending = 0; + Ok(()) + } + + /// Seed a named heap from a founder-endowed supply before sharing it. + /// The source retains the real affine lease and maps it receiver-side. + pub fn seed(&mut self, backing: StashedLease) -> bool { + self.install(backing, header_bytes() + ALIGN) + } + + // Caller holds exclusive access or the per-source refill admission guard. + fn install(&self, mut backing: StashedLease, bytes: usize) -> bool { + let mut view = None; + let mapped = (|| { + let slot = backing.slot()?; + if let Some((base, capacity)) = self.mapping { + let (_, len) = verbs::lease_geometry(slot)?; + let offset = self.mapped_bytes.load(Ordering::Relaxed) as u64; + let end = offset.checked_add(len)?; + if base % ALIGN as u64 != 0 + || len % ALIGN as u64 != 0 + || len < bytes as u64 + || end > capacity + { + return None; + } + let mapped = verbs::map_owned_lease_at_rw(slot, base.checked_add(offset)?, len)?; + let ptr = NonNull::slice_from_raw_parts( + NonNull::new(mapped.as_mut_ptr())?, + usize::try_from(len).ok()?, + ); + view = Some(mapped); + self.mapped_bytes + .store(usize::try_from(end).ok()?, Ordering::Relaxed); + Some(ptr) + } else { + verbs::map_held_rw(slot) + } + })(); + let Some(mapped) = mapped else { + let _ = backing.try_retire(); + return false; + }; + let base = mapped.cast::(); + if base.as_ptr() as usize % ALIGN != 0 || mapped.len() < bytes { + drop(view); + let _ = backing.try_retire(); + return false; + } + // SAFETY: this fresh, exclusively held mapping covers its header and + // payload. The release store publishes initialized immutable metadata + // and the affine owner before any worker can claim the payload. + unsafe { + let header = base.cast::().as_ptr(); + header.write(Extent { + next: self.extents.load(Ordering::Relaxed), + backing, + view, + bytes: mapped.len(), + cursor: AtomicUsize::new(header_bytes()), + }); + self.extents.store(header, Ordering::Release); + } + true + } + + /// Derive a separately revocable client root for a live heap allocation. + /// + /// # Safety + /// The caller must own the entire live, page-aligned allocation at + /// `address..address + bytes` from the Heap using this source. It must + /// revoke this root and its descendants before deallocating that block. + /// No allocator metadata or adjacent allocation may share its pages. + pub unsafe fn carve_allocation(&self, address: usize, bytes: usize) -> Option { + if bytes == 0 || address % 4096 != 0 || bytes % 4096 != 0 { + return None; + } + let end = address.checked_add(bytes)?; + let mut header = self.extents.load(Ordering::Acquire); + while !header.is_null() { + // SAFETY: immutable source-owned header, published with release. + let extent = unsafe { &*header }; + let base = header as usize; + if address >= base.checked_add(header_bytes())? + && end <= base.checked_add(extent.bytes)? + { + return verbs::carve_owned_lease_window( + extent.backing.slot()?, + (address - base) as u64, + bytes as u64, + ); + } + header = extent.next; + } + None + } + + fn has_ready(&self, bytes: usize) -> bool { + let mut header = self.extents.load(Ordering::Acquire); + while !header.is_null() { + // SAFETY: source-owned headers persist until exclusive Drop. + let extent = unsafe { &*header }; + if extent + .bytes + .saturating_sub(extent.cursor.load(Ordering::Acquire)) + >= bytes + { + return true; + } + header = extent.next; + } + false + } +} + +struct RefillGuard<'a> { + active: &'a AtomicBool, + wanted: &'a AtomicUsize, + pending: usize, +} +impl Drop for RefillGuard<'_> { + fn drop(&mut self) { + // A cancelled IPC must not consume the outstanding miss. + self.wanted.fetch_max(self.pending, Ordering::Release); + self.active.store(false, Ordering::Release); + } +} + +// SAFETY: publication is release/acquire, extent claims are one-shot CAS, +// and the cold refill guard permits one publisher. Retained affine owners are +// never mutated while shared; exclusive Drop moves them out before revocation. +unsafe impl Send for MmSource {} +unsafe impl Sync for MmSource {} + +impl Default for MmSource { + fn default() -> Self { + Self::new() + } +} + +const fn header_bytes() -> usize { + core::mem::size_of::().div_ceil(ALIGN) * ALIGN +} + +fn extent_bytes(wanted: usize) -> Option { + wanted + .checked_add(header_bytes())? + .checked_add(ALIGN - 1) + .map(|n| n / ALIGN * ALIGN) +} + +// SAFETY: each extent's payload is aligned, distinct, source-owned and +// transferred once by CAS. The header remains outside every heap extent. +unsafe impl MemSource for MmSource { + fn map_extent(&self, min_bytes: usize) -> Option> { + let mut header = self.extents.load(Ordering::Acquire); + while !header.is_null() { + // SAFETY: publication precedes traversal; Drop requires exclusive + // source ownership and cannot overlap an allocation borrow. + let extent = unsafe { &*header }; + let bytes = min_bytes.max(1).checked_add(ALIGN - 1)? / ALIGN * ALIGN; + if let Some(offset) = claim_bytes(&extent.cursor, extent.bytes, bytes) { + // SAFETY: CAS reserves a disjoint, page-aligned subextent; + // the affine supply owner remains outside all returned bytes. + let payload = unsafe { NonNull::new_unchecked(header.cast::().add(offset)) }; + return Some(NonNull::slice_from_raw_parts(payload, bytes)); + } + header = extent.next; + } + self.wanted.fetch_max(min_bytes, Ordering::Release); + None + } +} + +fn claim_bytes(cursor: &AtomicUsize, total: usize, bytes: usize) -> Option { + cursor + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |offset| { + offset.checked_add(bytes).filter(|end| *end <= total) + }) + .ok() +} + +impl Drop for MmSource { + fn drop(&mut self) { + let mut header = *self.extents.get_mut(); + while !header.is_null() { + // SAFETY: move the only owner and next pointer before revoke can + // unmap this header. All borrowers/workers have already finished. + let Extent { + next, + mut backing, + view, + .. + } = unsafe { header.read() }; + header = next; + drop(view); + let _ = backing.try_retire(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn concurrent_subextents_never_overlap_or_consume_the_header() { + let cursor = AtomicUsize::new(header_bytes()); + let total = header_bytes() + 64 * ALIGN; + let claimed: [AtomicUsize; 64] = core::array::from_fn(|_| AtomicUsize::new(0)); + std::thread::scope(|scope| { + for _ in 0..8 { + scope.spawn(|| { + while let Some(offset) = claim_bytes(&cursor, total, ALIGN) { + assert!(offset >= header_bytes()); + assert_eq!(offset % ALIGN, 0); + assert_eq!( + claimed[(offset - header_bytes()) / ALIGN] + .fetch_add(1, Ordering::Relaxed), + 0 + ); + } + }); + } + }); + assert!(claimed.iter().all(|n| n.load(Ordering::Relaxed) == 1)); + assert_eq!(cursor.load(Ordering::Relaxed), total); + } + + #[test] + fn extent_geometry_accounts_for_owner_and_refuses_overflow() { + assert_eq!(extent_bytes(ALIGN), Some(ALIGN + header_bytes())); + assert_eq!(extent_bytes(ALIGN + 1), Some(2 * ALIGN + header_bytes())); + assert_eq!(extent_bytes(usize::MAX), None); + assert!(header_bytes() >= core::mem::size_of::()); + } + + #[test] + fn miss_is_an_explicit_refill_request_not_a_poll() { + let source = MmSource::new(); + assert!(source.map_extent(ALIGN).is_none()); + assert_eq!(source.wanted.load(Ordering::Acquire), ALIGN); + assert!(source.extents.load(Ordering::Acquire).is_null()); + } + #[test] + fn concurrent_misses_preserve_the_largest_request() { + let source = MmSource::new(); + std::thread::scope(|scope| { + for pages in 1..=8 { + let source = &source; + scope.spawn(move || { + assert!(source.map_extent(pages * ALIGN).is_none()); + }); + } + }); + assert_eq!(source.wanted.load(Ordering::Acquire), 8 * ALIGN); + } + + #[test] + fn completed_refusal_does_not_poison_a_smaller_allocation() { + struct Refusing { idle: bool, requests: usize } + impl MmClient for Refusing { + fn is_idle(&self) -> bool { + self.idle + } + async fn request( + &mut self, + _: crate::MmRequest, + ) -> (MmResponse, Option) { + (MmResponse::Refused(crate::MmRefusal::Quota), None) + } + async fn supply(&mut self, _: u64) -> (MmResponse, Option) { + self.requests += 1; + (MmResponse::Refused(crate::MmRefusal::Quota), None) + } + async fn supply_in_class( + &mut self, + _: u64, + _: u64, + ) -> (MmResponse, Option) { + (MmResponse::Refused(crate::MmRefusal::Quota), None) + } + async fn stats(&mut self) -> Option { + None + } + } + let source = MmSource::new(); + assert!(source.map_extent(usize::MAX / 2).is_none()); + let mut client = Refusing { idle: false, requests: 0 }; + { + let mut refill = core::pin::pin!(source.refill_detailed(&mut client)); + let mut context = core::task::Context::from_waker(core::task::Waker::noop()); + assert_eq!( + core::future::Future::poll(refill.as_mut(), &mut context), + core::task::Poll::Ready(Err(RefillFailure::PairBusy)) + ); + } + assert_eq!(client.requests, 0, "an occupied pair publishes nothing"); + assert_eq!(source.wanted.load(Ordering::Acquire), usize::MAX / 2); + client.idle = true; + let mut refill = core::pin::pin!(source.refill_detailed(&mut client)); + let mut context = core::task::Context::from_waker(core::task::Waker::noop()); + assert_eq!( + core::future::Future::poll(refill.as_mut(), &mut context), + core::task::Poll::Ready(Err(RefillFailure::MmQuota)) + ); + assert_eq!(source.wanted.load(Ordering::Acquire), 0); + assert!(source.map_extent(ALIGN).is_none()); + assert_eq!(source.wanted.load(Ordering::Acquire), ALIGN); + } + + #[test] + fn refill_outcomes_keep_exact_mm_refusals_and_local_admission_causes() { + for (refusal, expected) in [ + (MmRefusal::BadProps, RefillFailure::MmBadProps), + (MmRefusal::Exhausted, RefillFailure::MmExhausted), + (MmRefusal::BadReturn, RefillFailure::MmBadReturn), + (MmRefusal::Denied, RefillFailure::MmDenied), + (MmRefusal::Quota, RefillFailure::MmQuota), + (MmRefusal::Pending, RefillFailure::MmPending), + (MmRefusal::Full, RefillFailure::MmFull), + ] { + assert_eq!(RefillFailure::from_refusal(refusal), expected); + assert_eq!(RefillFailure::from_code(expected as u8), Some(expected)); + } + assert_eq!(RefillFailure::from_code(0), None); + assert_eq!(RefillFailure::from_code(16), None); + } + + #[test] + fn abandoned_refill_restores_demand_without_losing_a_newer_miss() { + let source = MmSource::new(); + source.refilling.store(true, Ordering::Relaxed); + { + let _guard = RefillGuard { + active: &source.refilling, + wanted: &source.wanted, + pending: ALIGN, + }; + assert!(source.map_extent(2 * ALIGN).is_none()); + } + assert_eq!(source.wanted.load(Ordering::Acquire), 2 * ALIGN); + assert!(!source.refilling.load(Ordering::Acquire)); + } +} diff --git a/native/services/mm/src/lib.rs b/native/services/mm/src/lib.rs new file mode 100644 index 0000000..a4a7446 --- /dev/null +++ b/native/services/mm/src/lib.rs @@ -0,0 +1,41 @@ +//! # kore-svc-mm — the memory-manager service +//! +//! **The lessor**: the service endowed the memory supply, +//! granting extent windows out of it and taking them back — the first +//! [`SystemService`](kore_rt::service::SystemService) pillar, and the one +//! the default heap grows through. +//! +//! One concern per module (`res/USERLAND.md` §1.2 rule 4; `main.rs` — +//! the process entry — arrives with the founder's spawn, #96/#298): +//! +//! - [`service`] — the serve step: [`MemoryManager`] over one possessed +//! [`SupplyToken`](kore_rt::cap::SupplyToken) plus one possessed +//! [`RegionToken`](kore_rt::cap::RegionToken) (the child-supply mint, +//! slice D), the [`MmRequest`] vocabulary, refusal-fail-closed carves, +//! and the per-pair supply quota. +//! - [`client`] — the client face: the peer seam — the async +//! [`MmClient`] (§4) with its [`RingMm`] proxy, whose +//! [`supply`](MmClient::supply) is what a launcher mints child memory +//! through. +//! - [`wire`] — the fail-closed enum ⇄ wire codecs over +//! [`kore_abi::svc`]'s frozen forms (§5.3). + +#![no_std] +#![feature(allocator_api)] + +extern crate alloc; +#[cfg(test)] +extern crate std; + +pub mod client; +pub mod heap; +pub mod service; +pub mod wire; + +pub use client::{MmClient, RingMm, process_face}; +#[cfg(target_os = "none")] +pub use client::{ProcessMm, attach_process_face}; +pub use service::{ + FREE_SLOTS, HEAP_DEMAND_BYTES, MAX_BASE_SUPPLY_BYTES, MemoryManager, MmEndowment, MmRefusal, + MmRequest, MmResponse, MmStats, PAGE_BYTES, SUPPLY_RIGHTS, base_quota_for, +}; diff --git a/native/services/mm/src/service/mod.rs b/native/services/mm/src/service/mod.rs new file mode 100644 index 0000000..b69fa49 --- /dev/null +++ b/native/services/mm/src/service/mod.rs @@ -0,0 +1,2064 @@ +//! # `service` — the memory manager, the first [`SystemService`] +//! +//! The **lessor of last resort** (the lessor inversion): the service endowed +//! with a memory supply, granting extents from it and taking them back. +//! Its role bundle is one supply lease +//! ([`role::MEM_SUPPLY`](kore_abi::task::role::MEM_SUPPLY)) plus one +//! region grant ([`role::MEM_REGION`](kore_abi::task::role::MEM_REGION), +//! slice D) — each claimed at bind, geometry riding the claimed token, +//! never a config string; its request vocabulary is the allocator +//! lifecycle: [`MmRequest::Alloc`] (the heap-grow verb), +//! [`MmRequest::Release`] (the return leg), and [`MmRequest::Supply`] +//! (the child-memory mint: a fresh **root** lease off the region, quota- +//! metered per serving pair, delivered whole for a launcher to stamp and +//! convey at a spawn — its frames return to the region bitmap when the +//! child's lineage dies. The region bitmap remains the frame ledger; mm keeps +//! a bounded receipt table solely to make its live per-pair quota exact). +//! +//! **DMA is mm's — at the *frames* tier only.** `AllocProps`' `dma` / +//! `contiguous` are **placement** properties. This service's current +//! `SupplyToken` endowment carries only `(base, len)`: it does not carry the +//! source NUMA node, cache policy, or a kernel-authenticated DMA class. +//! Consequently the current allocator admits only NUMA-agnostic, CPU-only, +//! write-back R/W requests. It refuses stronger properties rather than +//! stamping claims the delivered token cannot prove. DMA *reachability* — an IOVA +//! window in a device domain — is the **device holder's** act with its own +//! bundle authority (`role::DMA_POOL` is an mm grant *delegated into* a +//! driver's bundle at spawn; RUNG-2: one lease → CPU window + IOVA window; +//! a DMA-reachable frame is unswappable — the IOMMU pin rule). **MMIO is never mm's**: a BAR pre-exists on +//! the device (`dev.bar(n)` — a derive off the DEVICE cap); only the map +//! *verb* is shared. This mirrors the kernel exactly — one region/extent +//! mechanism, authority partitioned by root (BootWF #4: mm accumulates no +//! device authority). +//! +//! **What mm is *not*: a sharing broker.** A granted extent is an *owned* +//! token carrying `DERIVE` — sharing is the holder's own act (derive a +//! lease from the unmapped extent, attenuate views, move one over +//! rendezvous; [`kore_rt::cap`]) and never routes back through mm. mm's +//! lessor position matters for the *swappable* arrangement instead: a +//! pageable heap is one where mm retains the lease and the tenant holds +//! only a view — the pager-verbs tier that extends this vocabulary +//! (`lease`/`share`/`unshare`) later. +//! +//! **Release is wire-faithful** (#148: no capability rides a ring entry): +//! the extent returns by rendezvous **delivery into mm's arena** +//! before the data-plane request names only the **slot**. mm reclaims (bind → +//! retire → books), and returned windows serve later grants first +//! (first-fit reuse; no coalescing yet — the books make every loss +//! visible, never silent). +//! +//! Serving is **allocation-free** in the common case (bump cursor + tables + +//! counters): a client's heap waits inside its grow path while mm serves it, +//! so the receipt table is provisioned at bind for every lane's live supplies +//! ([`PROVISIONED_RESERVATIONS`]). Past that provision it doubles from mm's +//! own heap, which grows through the kernel's carve-and-map and never through +//! an mm client path, so the one allocation a serve step can make cannot +//! re-enter mm, and a growth the heap refuses is a refusal on the wire +//! ([`MmRefusal::Full`]), never a stall. + +use alloc::vec::Vec; +use kore_abi::cap::Rights; +use kore_abi::mem::{AllocProps, DmaClass, MemAttrs, NumaNode}; + +use kore_abi::session::{MemoryClassCensus, SlotRef, SupplyReleaseReport, SystemMetric}; +use kore_abi::task::role; + +use kore_abi::task::{Bootstrap, Profile, RoleSlot}; +use kore_rt::cap::{RegionToken, SupplyToken, verbs}; +use kore_rt::service::{ + BindError, Bundle, GrantArg, PeerClose, ReturnMode, Serve, StartFailure, SupplyReleaseGrant, + SystemService, +}; +use kore_rt::svc::{ + PeerAdmissionRefusal, PeerBundleAssembler, PeerBundlePart, PeerId, PeerMapPlan, SvcProtocol, +}; + +/// Grant granularity — extents are page-multiples, page-aligned. +pub const PAGE_BYTES: u64 = 4096; + +/// Exact cache/access contract the current geometry-only supply can prove. +const SUPPLY_ALLOC_ATTRS: MemAttrs = MemAttrs::WRITE_BACK + .union(MemAttrs::READ) + .union(MemAttrs::WRITE); + +/// Returned windows the lessor can hold for reuse — a fixed, allocation- +/// free ceiling; overflow is *counted* ([`MmStats::leaked_bytes`]), never +/// silent. +pub const FREE_SLOTS: usize = 16; + +/// **The part of the 4 KiB class one lane may hold live**: a half. +/// +/// The base-class quota ([`MmRequest::Supply`] at 4 KiB alignment) is the +/// most bytes one client face may have in live capability lineages; the +/// kernel reports completion only after the lineage is dead and the region +/// bitmap has accepted the frames. It is accounted per arrival lane +/// ([`Serve::via`] — structural identity, never a wire field), so init's +/// face and the shell's meter independently, and it makes one client's +/// appetite a *policy* refusal ([`MmRefusal::Quota`], loud on the wire) +/// before that client can drain the class. +/// +/// **Where a half comes from.** The quota exists so that no one client +/// takes the class alone; a half is the largest share that still leaves +/// everyone else as much as that one client may hold. The whole class +/// would let one face starve every other, and a smaller share would strand +/// memory from the clients that legitimately hold the most: a seat's +/// controller provisions every window it opens (about 20.5 MiB each, +/// `kore_window_session::layout`) and every command it runs from its own +/// lane. Two clients may still fill the class between them, and then the +/// class itself refuses: the kernel's derive fails `Mem(Exhausted)`, and +/// mm, told only that the grant dropped, answers [`MmRefusal::Denied`]. +/// +/// **What it replaced.** A fixed 128 MiB per lane, whatever the machine: +/// on a 12 GiB guest a seat's sixth window was refused `Quota` with +/// gigabytes of the class free, as it would have been on a 16 GiB laptop. +const BASE_QUOTA_SHARE: u64 = 2; + +/// The [`SystemMetric::MemoryClass`] operand of the 4 KiB base class (the +/// census numbers its classes 0 = 4 KiB, 1 = 2 MiB, 2 = 1 GiB). +/// +/// [`SystemMetric::MemoryClass`]: kore_abi::session::SystemMetric::MemoryClass +const BASE_CLASS_CENSUS: u16 = 0; + +/// The base-class quota of one lane when the 4 KiB class has `carved` +/// bytes: [`BASE_QUOTA_SHARE`]'s part of it, in whole pages. The boot +/// carves about a fifth of usable memory as the 4 KiB class (the 50/30/20 +/// ruling, `res/ACCEL_MEMORY_SCALE.md` §1): a 4 GiB QEMU guest's 737 MiB +/// gives each lane 368.5 MiB, a 12 GiB guest's 2,377 MiB gives 1,188.5 MiB, +/// and a 16 GiB laptop's 3.2 GiB about 1.6 GiB. +pub const fn base_quota_for(carved: u64) -> u64 { + carved / BASE_QUOTA_SHARE / PAGE_BYTES * PAGE_BYTES +} + +/// The **huge-class** per-pair quota is not a constant: it is the +/// `role::SUPPLY_QUOTA` boot datum (`services/mm/boot.toml`, two `u32` +/// words), so the deployed working-set ceiling is manifest policy while the +/// base-class quota above stays the launcher's everyday allowance. A +/// supply asked for at 2 MiB or 1 GiB alignment (`res/ACCEL_MEMORY_SCALE.md` +/// §1.2: the alignment *is* the frame class) is metered against this quota, +/// never against the base class's ([`base_quota_for`]). This constant is +/// the ceiling a datum is clamped to, and the receipt's unit field holds a +/// supply of it. +pub const HUGE_SUPPLY_QUOTA_MAX: u64 = 64 << 30; + +/// Rights a minted supply **delivers**: the full grant — the client is +/// the supply's owner (carves views, shares windows, conveys the root +/// onward at a spawn; `DERIVE` is both the carve edge and the transfer +/// gate's toll, `REVOKE` what lets the holder — or the child row's reap +/// — tear the subtree down). Σ5 clamps every later child to this set. +pub const SUPPLY_RIGHTS: Rights = Rights::ACCESS + .union(Rights::MODIFY) + .union(Rights::DERIVE) + .union(Rights::REVOKE); + +/// The memory manager's role bundle: the supply it grants heap extents +/// from and the region it mints child supplies off — both **possessed**, +/// not described (each token is the claimed record; the supply's +/// [`len`](SupplyToken::len) is its geometry, the region's ledger is the +/// kernel's frame bitmap). +#[derive(Debug)] +pub struct MmEndowment { + /// The claimed supply-region token. + pub supply: SupplyToken, + /// The claimed region grant — the [`MmRequest::Supply`] minting + /// authority (`role::MEM_REGION`, slice D). + pub region: RegionToken, + /// The base-class per-lane quota ([`base_quota_for`]), derived from the + /// 4 KiB class the boot carved, read once through the row's read-only + /// observation grant (`role::SYSTEM_OBSERVE`) — the kernel's own census, + /// so the quota follows the machine instead of a constant. + pub base_quota: u64, + /// The huge-class per-pair quota from the `role::SUPPLY_QUOTA` datum + /// (bytes; low word at index 0, high word at index 1). Absent datum ⇒ + /// zero ⇒ every huge-class supply refuses `Quota` — fail-closed, and + /// visible on the wire rather than a silent fall-back to the base class. + pub huge_quota: u64, +} + +// Hand-written rather than `#[derive(Bundle)]` because the bundle carries a +// boot *datum* beside its two capabilities, and the derive speaks handles +// only. Same profile and claim shape the derive would emit. +impl Bundle for MmEndowment { + const PROFILE: Profile = Profile::new( + "mm", + &[ + RoleSlot::at(role::MEM_SUPPLY), + RoleSlot::at(role::MEM_REGION), + RoleSlot::at(role::SYSTEM_OBSERVE), + ], + ); + + fn resolve(boot: &Bootstrap<'_>) -> Result { + let supply = boot + .handle(role::MEM_SUPPLY, 0) + .ok_or(BindError::MissingRole { + role: role::MEM_SUPPLY, + index: 0, + }) + .and_then(|slot| { + verbs::claim_supply(slot).ok_or(BindError::BadHandle { + role: role::MEM_SUPPLY, + index: 0, + }) + })?; + let region = boot + .handle(role::MEM_REGION, 0) + .ok_or(BindError::MissingRole { + role: role::MEM_REGION, + index: 0, + }) + .and_then(|slot| { + verbs::claim_region(slot).ok_or(BindError::BadHandle { + role: role::MEM_REGION, + index: 0, + }) + })?; + // The census is read once: the classes are carved at boot and never + // re-carved, so the quota it derives holds for the boot. A grant + // that will not answer is the row's misconfiguration, refused at + // bind as a bad handle rather than served under an invented quota. + let observe_refused = BindError::BadHandle { + role: role::SYSTEM_OBSERVE, + index: 0, + }; + let observe = boot + .handle(role::SYSTEM_OBSERVE, 0) + .ok_or(BindError::MissingRole { + role: role::SYSTEM_OBSERVE, + index: 0, + }) + .and_then(|slot| verbs::claim_observe(slot).ok_or(observe_refused))?; + let census = + verbs::observe_metric_at(&observe, SystemMetric::MemoryClass, BASE_CLASS_CENSUS) + .map(|report| MemoryClassCensus::from_observation(report.value, report.aux)) + .ok_or(observe_refused)?; + let lo = boot.datum(role::SUPPLY_QUOTA, 0).unwrap_or(0); + let hi = boot.datum(role::SUPPLY_QUOTA, 1).unwrap_or(0); + let huge_quota = (u64::from(hi) << 32 | u64::from(lo)).min(HUGE_SUPPLY_QUOTA_MAX); + Ok(Self { + supply, + region, + base_quota: base_quota_for(census.carved), + huge_quota, + }) + } +} + +/// The lessor's request vocabulary — the allocator lifecycle. The +/// pager lease tier (`lease`/`share`/`unshare`) and fault +/// notifications extend this enum later — the vocabulary grows, the +/// intake stays one. +#[derive(Clone, Copy, Debug)] +pub enum MmRequest { + /// Grant a fresh extent satisfying `props` — the [`kore_abi::mem`] wire + /// artifact the client heap builds + /// ([`alloc_props`](crate::client::alloc_props)). + Alloc(AllocProps), + /// Take back an extent the client **delivered into mm's arena** at + /// `slot` (#148: the cap moved by rendezvous, the request names the + /// slot — data only). The window re-enters the grantable pool. + Release { + /// mm-arena slot holding the delivered extent record. + slot: SlotRef, + }, + /// Mint a fresh **root supply lease** off the region endowment and + /// deliver it whole (slice D — the child-memory verb): the launcher + /// stamps its child's `.kore.ld` through a transient view, conveys + /// the root at the spawn, and the frames return to the *region + /// bitmap* when the child's lineage dies (T6) — never to mm's books. + /// Metered per arrival lane: the 4 KiB class against the derived base + /// quota ([`base_quota_for`]), the huge classes against the + /// `role::SUPPLY_QUOTA` datum. + Supply { + /// Requested size in bytes (page-rounded by the lessor). + size: u64, + /// Requested backing alignment in bytes: `0` for the 4 KiB + /// default, else exactly 4 KiB / 2 MiB / 1 GiB (the + /// [`kore_abi::session::LeaseReq`] vocabulary) — anything else + /// refuses [`MmRefusal::BadProps`]. + align: u64, + }, + /// Read one bounded snapshot of this provider's allocator and supply + /// books. The request carries no authority and mints no grant. + Stats, + /// Exec's administration pair: one component of a per-launch client + /// face, delivered on the return leg at `announced`. + AdmitPeer { + part: PeerBundlePart, + announced: SlotRef, + }, + /// Exec's administration pair: the verdict of the preceding admission. + TakeAdmission, + /// Exec's administration pair: retire one exact face generation. + RetirePeer { peer: PeerId }, + /// Exec's administration pair: fence retirement — accepted once the face + /// is closed and every supply it minted has returned. + ConfirmRetired { peer: PeerId }, + /// A per-launch face retires itself. + ClosePeer, +} + +/// The lessor's replies — **data only**. Granted authority reaches the +/// client as a delivered capability record it possesses through the typed +/// take verbs, never as reply payload; a reply carries the outcome, not a +/// window. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MmResponse { + /// Granted — the extent token arrives by delivery. + Granted, + /// The returned window is back in the lessor's books. + Released, + /// The provider-owned allocator and child-supply snapshot. + Stats(kore_abi::monitor::MemorySnapshot), + /// An administration step took effect. + Accepted, + /// The preceding admission produced this face. + PeerAdmitted { peer: PeerId }, + /// Refused — and a refusal emits **no** grant (fail-closed: no + /// authority leaves on an error path). + Refused(MmRefusal), +} + +/// Why the lessor refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MmRefusal { + /// Props the current supply token cannot honor or prove: malformed + /// geometry, a named NUMA node, a DMA placement class, or anything other + /// than write-back read/write CPU access. A contiguous request is admitted + /// because every window of this contiguous supply is itself contiguous. + /// A supply too large for a receipt to name (a 4 KiB-class supply past + /// [`MAX_BASE_SUPPLY_BYTES`]) is malformed geometry too: that much + /// memory belongs in a huge class. + BadProps, + /// The supply cannot fit the request. + Exhausted, + /// The returned slot did not bind to an extent of **this** supply — + /// nothing delivered there, a forged/stale record, or a window that + /// was never mm's to reclaim. + BadReturn, + /// The capability grant session refused. The reserved window was rolled + /// back into mm's books and no authority left the service. + Denied, + /// The arrival lane's outstanding-supply meter cannot fit the request + /// (its class's quota: [`base_quota_for`], or the huge datum) — a + /// *policy* refusal, loud on the wire (the `TableFull` discipline), + /// distinct from [`Exhausted`] (the backing itself cannot fit it). + /// + /// [`Exhausted`]: Self::Exhausted + Quota, + /// The administration step cannot complete yet (an admission in + /// flight, or a retired face whose supplies have not all returned). + Pending, + /// No per-launch face fits: every serving slot is attached or still + /// crediting a closed face's returns, or mm's face arena or capability + /// arena has no room for one more. A capacity refusal — exec reports it + /// as a full table, and a launch retried after another engine program + /// exits can succeed — distinct from [`Denied`] (a face that arrived + /// wrong). [`MmRequest::Supply`] answers it too when mm's own books + /// cannot hold one more live supply (the reservation table at its + /// receipt ceiling, or mm's heap unable to grow it): a capacity of mm's, + /// not the lane's quota, and gone once a supply returns. + /// + /// [`Denied`]: Self::Denied + Full, +} + +#[derive(Clone, Copy, Debug)] +enum CarveUndo { + Fresh { + cursor: u64, + len: u64, + }, + Reused { + index: usize, + window: (u64, u64), + len: u64, + }, + /// A [`MmRequest::Supply`] reservation: quota + counters to unwind if + /// the grant session drops (the region claim itself unwound with the + /// refused session under kernel I3 — mm holds no window to restore). + Supply { + pair: usize, + len: u64, + receipt: u64, + }, +} + +/// **The live supplies mm provisions room for at bind**, every lane +/// together: one reservation per supply minted and not yet completed. Every +/// lane holding as many live supplies as one task can adopt into its heap +/// ([`kore_rt::heap::MAX_ADOPTED`]), the most a program keeps of its own, +/// so the common case never grows the table inside a serve step. A +/// launcher's brokered supplies (a seat's windows, each conveyed to its own +/// child) ride the lanes that hold few. Past the provision the table +/// doubles from mm's heap ([`Reservations`]), up to what a receipt can +/// index ([`RESERVATION_CEILING`]); a table that cannot grow refuses +/// [`MmRefusal::Full`]. +/// +/// It replaced a table sized for the bootstrap roster filling a fixed +/// 128 MiB quota with single pages: 1.125 MiB allocated at bind, whose +/// size tied the quota to a constant the machine could never move. +const PROVISIONED_RESERVATIONS: usize = LANES * kore_rt::heap::MAX_ADOPTED; + +/// What one reservation costs mm's heap: its state word, and its place in +/// the free stack. +const RESERVATION_BYTES: usize = 2 * core::mem::size_of::(); + +/// **Every supply lane mm can tell apart**: one per serving slot the host +/// can name — the fixed pairs, then each per-launch face the host grows into +/// [`MemoryManager::peer_arena`]. A [`PeerId`]'s slot is one byte, so this is +/// the host's whole slot space, not the bootstrap table: sizing the lanes by +/// [`MAX_SERVE_PAIRS`](kore_rt::svc::MAX_SERVE_PAIRS) is what once left two +/// faces beside mm's six fixed pairs, and every engine program past the +/// second refused. +const LANES: usize = PeerId::SLOT_CAPACITY; + +/// The huge classes' receipt unit: 2 MiB, the finer huge frame class. A +/// 1 GiB-aligned supply is a whole number of these, so one unit names both. +const HUGE_UNIT_BYTES: u64 = 1 << kore_abi::session::LEASE_ALIGN_2M; + +/// Completions consumed per pair per serve step. The drain runs inside the +/// client allocator's grow path, so a banked backlog (a launch storm, then +/// idle) must not stall one serve arbitrarily; the tail beyond the cap waits +/// for the next request — at worst one extra [`MmRefusal::Quota`] round. +const SUPPLY_RELEASE_DRAIN_PER_PAIR: usize = 16; + +/// The meter row a supply **lane** is booked to. A lane is the arrival a +/// supply was minted on — a fixed pair's index, or a per-launch face's +/// [`PeerId::arrival`] — and it is also the kernel release lane its +/// completion report comes back on (`SupplyReleaseGrant::pair`). A face's +/// row is its slot. The host never seats a face on a fixed pair's slot, and +/// a slot is not reused while its row is non-zero +/// ([`MemoryManager::peer_closed`]), so a report can never credit a later +/// face in the same slot. Total: a fixed pair's index and a face's slot are +/// both below [`LANES`]. +fn meter_index(lane: u16) -> usize { + match PeerId::from_arrival(lane) { + Some(peer) => peer.slot(), + None => usize::from(lane), + } +} + +/// Exec's administration pair: `exec_client` takes the final pair of the +/// row's protocol (`services/exec/src/founder/mod.rs`), the last of mm's +/// [`SERVE_PAIRS`](SvcProtocol::SERVE_PAIRS); the pairs of rows declaring mm +/// among their `clients` are numbered after it (`SERVICE_CLIENT_LINKS`). Only +/// it may admit, retire or confirm per-launch faces. +const ADMIN_VIA: u16 = ::SERVE_PAIRS as u16 - 1; + +/// Bytes of one per-launch face's ring — the stride of the peer map. mm's +/// wire is two small fixed-size records per slot, so a page holds both rings +/// at the declared capacities (pinned below). +pub const PEER_RING_BYTES: u64 = 0x1000; + +const _: () = assert!( + kore_rt::svc::window_bytes::( + ::REQUEST_CAPACITY, + ::RESPONSE_CAPACITY, + ) as u64 + <= PEER_RING_BYTES, + "a per-launch mm face's rings must fit its page" +); + +/// **The face arena** — what the serve host grows mm's per-launch faces +/// into ([`MemoryManager::peer_arena`]). A named heap of its own, over this +/// task's `role::HEAP` supply: [`HeapSupplySource`] carves and maps through +/// the kernel's session engine, the one source that works for mm itself +/// (mm cannot be its own client), and it keeps the face table's growth +/// accounted apart from the global heap's. +/// +/// [`HeapSupplySource`]: kore_rt::heap::HeapSupplySource +static FACE_ARENA: kore_rt::Heap = + kore_rt::Heap::new_in(kore_rt::heap::HeapSupplySource::new()); + +/// **What mm's heap must hold** — the floor its manifest row's +/// `heap_supply_bytes` is held to (exec's founder test +/// `mm_heap_supply_holds_every_face`): the reservations provisioned at bind +/// ([`PROVISIONED_RESERVATIONS`]), the service itself, and the face table +/// grown to every lane, twice over — a doubling table maps its next block +/// before it frees the one it outgrew, and every block it ever held sums to +/// under twice its last — plus a pool extent of headroom for each of the two +/// heaps drawing on the one supply (the global allocator's and +/// [`FACE_ARENA`]'s). Derived from the host's own per-slot cost, so a heavier +/// serving pair raises the floor instead of overrunning it. What the row +/// gives past the floor is the reservation table's room to grow. +pub const HEAP_DEMAND_BYTES: u64 = (PROVISIONED_RESERVATIONS * RESERVATION_BYTES + + core::mem::size_of::() + + 2 * LANES * kore_rt::svc::peer_slot_bytes::() + + 2 * kore_rt::heap::POOL_EXTENT_BYTES) as u64; + +/// How a refused admission verdict reads on mm's wire. No room for one more +/// face — no free slot, no arena room to grow one, no capability slot for its +/// scratch — is [`MmRefusal::Full`], a capacity the launcher reports as a +/// full table and a later launch can find freed; anything else is a face +/// that arrived wrong, [`MmRefusal::Denied`]. Exhaustive, so a new refusal +/// cannot fall into either bucket unexamined. +fn admission_refusal(refusal: PeerAdmissionRefusal) -> MmRefusal { + match refusal { + PeerAdmissionRefusal::Capacity | PeerAdmissionRefusal::NoScratchSlots => MmRefusal::Full, + PeerAdmissionRefusal::InvalidMapPlan + | PeerAdmissionRefusal::WrongRingAuthority + | PeerAdmissionRefusal::WrongEndpointAuthority + | PeerAdmissionRefusal::WrongGrantCount + | PeerAdmissionRefusal::WrongReturnShape + | PeerAdmissionRefusal::RingGeometry + | PeerAdmissionRefusal::MapFailed + | PeerAdmissionRefusal::RingMalformed + | PeerAdmissionRefusal::PolicyCollision + | PeerAdmissionRefusal::PolicyUnavailable => MmRefusal::Denied, + } +} + +/// One per-launch face's slot state. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct PeerLane { + /// The face's arrival — its quota lane and release lane. + arrival: u16, + /// Still attached. A closed face keeps its lane (and its slot, through + /// [`MemoryManager::peer_closed`]) until its supplies have returned. + live: bool, +} + +// A receipt is mm's own word: the kernel keeps it in the supply's lineage and +// echoes it in the completion report without reading it (`TrackedLeaseReq`), +// so its packing is this file's to choose. Low to high: the reservation's +// index, its lane, its size in its class's unit, the class, and the +// reservation's generation. Each width is chosen, not derived from a quota, so +// the quota can follow the machine while the encoding stays fixed: +// +// - **Index**, 19 bits: half a million live supplies — 4 MiB of table at +// `RESERVATION_BYTES` a reservation, the scale of mm's whole heap, so the +// heap refuses a larger table before the field would. +// - **Lane**: every lane ([`LANES`]). +// - **Units**, 20 bits: one supply in its class's unit — pages for the base +// class, 2 MiB units for the huge ones — so the largest single supply a +// receipt names is [`MAX_BASE_SUPPLY_BYTES`] (4 GiB less a page) of the +// 4 KiB class and 2 TiB of the huge ones, past the huge quota's ceiling. No +// page-granular supply a client asks for comes near it: launches, windows +// and heap leases run to 128 MiB, and anything larger belongs in a huge +// class. +// - **Class**, one bit: which meter, and which unit, a completion credits. +// - **Generation**, the rest (16 bits): a slot is taken 65,535 times before +// it is retired for good, and a stale receipt (a duplicate or reordered +// report) never matches a later reservation in the same slot. +const RECEIPT_INDEX_BITS: u32 = 19; +const RECEIPT_PAIR_BITS: u32 = (LANES - 1).ilog2() + 1; +const RECEIPT_UNITS_BITS: u32 = 20; +const RECEIPT_CLASS_BITS: u32 = 1; +const RECEIPT_PAIR_SHIFT: u32 = RECEIPT_INDEX_BITS; +const RECEIPT_UNITS_SHIFT: u32 = RECEIPT_PAIR_SHIFT + RECEIPT_PAIR_BITS; +const RECEIPT_CLASS_SHIFT: u32 = RECEIPT_UNITS_SHIFT + RECEIPT_UNITS_BITS; +const RECEIPT_GENERATION_SHIFT: u32 = RECEIPT_CLASS_SHIFT + RECEIPT_CLASS_BITS; +const RECEIPT_INDEX_MASK: u64 = (1 << RECEIPT_INDEX_BITS) - 1; +const RECEIPT_PAIR_MASK: u64 = (1 << RECEIPT_PAIR_BITS) - 1; +const RECEIPT_UNITS_MASK: u64 = (1 << RECEIPT_UNITS_BITS) - 1; +const RECEIPT_CLASS_HUGE: u64 = 1 << RECEIPT_CLASS_SHIFT; +const RECEIPT_GENERATION_MAX: u32 = (1 << (64 - RECEIPT_GENERATION_SHIFT)) - 1; +const RESERVATION_LIVE: u32 = 1 << 31; +const RESERVATION_GENERATION_MASK: u32 = RECEIPT_GENERATION_MAX; + +/// The most reservations a receipt can index: the table's ceiling. +const RESERVATION_CEILING: usize = 1 << RECEIPT_INDEX_BITS; + +/// The largest 4 KiB-class supply a receipt can name: its unit field full of +/// pages. A larger one is refused [`MmRefusal::BadProps`]. +pub const MAX_BASE_SUPPLY_BYTES: u64 = RECEIPT_UNITS_MASK * PAGE_BYTES; + +/// The unit a receipt counts a supply of its class in. +const fn receipt_unit(huge: bool) -> u64 { + if huge { HUGE_UNIT_BYTES } else { PAGE_BYTES } +} + +/// One receipt's fields ([`pack`](Self::pack) and [`unpack`](Self::unpack) +/// are the one encoding). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Receipt { + index: usize, + pair: usize, + units: u64, + huge: bool, + generation: u32, +} + +impl Receipt { + /// The word the kernel carries. Every field is in range by + /// construction: the index below the table's ceiling, the lane below + /// [`LANES`], the units refused past the field before a reservation is + /// made, the generation masked by the slot that minted it. + const fn pack(self) -> u64 { + (self.generation as u64) << RECEIPT_GENERATION_SHIFT + | if self.huge { RECEIPT_CLASS_HUGE } else { 0 } + | self.units << RECEIPT_UNITS_SHIFT + | (self.pair as u64) << RECEIPT_PAIR_SHIFT + | self.index as u64 + } + + const fn unpack(receipt: u64) -> Self { + Self { + index: (receipt & RECEIPT_INDEX_MASK) as usize, + pair: ((receipt >> RECEIPT_PAIR_SHIFT) & RECEIPT_PAIR_MASK) as usize, + units: (receipt >> RECEIPT_UNITS_SHIFT) & RECEIPT_UNITS_MASK, + huge: receipt & RECEIPT_CLASS_HUGE != 0, + generation: (receipt >> RECEIPT_GENERATION_SHIFT) as u32, + } + } +} + +// The packing only works while the configuration fits the fields. Pin every +// field's capacity where the compiler owns it: a drifted constant would +// otherwise truncate silently in a release image (a lane the field cannot +// name folds into another, so its completions stop matching and quota +// sticks). +const _: () = { + assert!( + LANES <= 1 << RECEIPT_PAIR_BITS, + "every supply lane must be representable in the receipt's lane field" + ); + assert!( + HUGE_SUPPLY_QUOTA_MAX / HUGE_UNIT_BYTES <= RECEIPT_UNITS_MASK, + "a supply of the whole huge ceiling must fit the receipt's unit count" + ); + assert!( + HUGE_SUPPLY_QUOTA_MAX % HUGE_UNIT_BYTES == 0, + "the huge ceiling is a whole number of its receipt unit" + ); + assert!( + RECEIPT_GENERATION_SHIFT < 64 && 64 - RECEIPT_GENERATION_SHIFT >= 12, + "a receipt keeps at least twelve generation bits" + ); + assert!( + PROVISIONED_RESERVATIONS <= RESERVATION_CEILING, + "the provisioned table must be addressable by the receipt's index" + ); + assert!( + RESERVATION_CEILING <= u32::MAX as usize, + "a free-stack entry names every slot" + ); + assert!( + RECEIPT_GENERATION_MAX < RESERVATION_LIVE, + "the live bit must sit above the generation bits it shares a slot with" + ); +}; + +/// **The live-supply table behind every receipt** — one state word per +/// reservation slot (the live bit and the generation of its current or last +/// reservation) and a stack of the slots free to take, so a reservation and +/// its retirement are O(1) whatever the table's size. It grows with the +/// supplies live at once: provisioned at bind +/// ([`PROVISIONED_RESERVATIONS`]), doubled from mm's heap when a reservation +/// finds no slot free, never past its ceiling ([`RESERVATION_CEILING`]). A +/// slot whose generation is spent is retired for good, never taken again, so +/// no receipt can name two reservations. +/// +/// mm's heap grows through the kernel's own carve-and-map, never through an +/// mm client path (`user/services/mm`'s global allocator), so a serve step +/// that grows the table cannot re-enter mm; growth is the only allocation a +/// serve step makes, and it is fallible. +#[derive(Debug)] +struct Reservations { + /// Per slot: [`RESERVATION_LIVE`] while reserved, and the generation of + /// its current or last reservation. + states: Vec, + /// The slots free to take, the next one on top. Its capacity is kept at + /// least the table's, so freeing a slot never allocates. + free: Vec, + /// The most slots the table may grow to. + ceiling: usize, +} + +impl Reservations { + /// A table of `provisioned` slots that grows to `ceiling`. A heap that + /// cannot hold the provision starts it empty: the first reservation + /// grows it, or is refused [`MmRefusal::Full`] if that cannot either. + fn new(provisioned: usize, ceiling: usize) -> Self { + let mut table = Self { + states: Vec::new(), + free: Vec::new(), + ceiling, + }; + let _ = table.grow_to(provisioned.min(ceiling)); + table + } + + /// Grow the table to `len` slots, the new ones free. `false`, with the + /// table as it was, when the heap cannot hold them. + fn grow_to(&mut self, len: usize) -> bool { + let old = self.states.len(); + if len <= old { + return true; + } + if self.states.try_reserve_exact(len - old).is_err() + || self.free.try_reserve_exact(len - self.free.len()).is_err() + { + return false; + } + self.states.resize(len, 0); + // Highest first, so the lowest free slot is the next taken. + self.free.extend((old..len).rev().map(|index| index as u32)); + true + } + + /// Reserve a slot: `(index, generation)` of its new reservation. With no + /// slot free the table doubles first; `None` at the ceiling, or when the + /// heap cannot hold the larger table. + fn take(&mut self) -> Option<(usize, u32)> { + if self.free.is_empty() { + let len = self.states.len(); + let next = len.saturating_mul(2).max(1).min(self.ceiling); + if next == len || !self.grow_to(next) { + return None; + } + } + let index = self.free.pop()? as usize; + let generation = (self.states[index] & RESERVATION_GENERATION_MASK) + 1; + self.states[index] = RESERVATION_LIVE | generation; + Some((index, generation)) + } + + /// End the reservation `generation` holds at `index`, freeing its slot + /// unless the slot's generations are spent. `false`, and nothing + /// changes, unless exactly that reservation is live there. + fn end(&mut self, index: usize, generation: u32) -> bool { + let Some(state) = self.states.get_mut(index) else { + return false; + }; + if generation == 0 || *state != RESERVATION_LIVE | generation { + return false; + } + *state = generation; + // Within capacity by construction (`grow_to` keeps the stack's + // capacity at the table's), so freeing never allocates; a slot that + // somehow found no room would be retired, never lost to a panic. + if generation < RECEIPT_GENERATION_MAX && self.free.len() < self.free.capacity() { + self.free.push(index as u32); + } + true + } +} + +/// The per-lane **supply meter** — [`MmRequest::Supply`]'s whole +/// bookkeeping, kept apart from the carve books because it is a *policy* +/// ledger over kernel-owned frames (the region bitmap is the ledger of +/// record), not a window pool. Pure counters over a table, so the quota +/// arithmetic is host-testable without a bound service. +#[derive(Debug)] +struct SupplyMeter { + /// Supplies minted. + supplies: u32, + /// Bytes those supplies span (the totals row). + supply_bytes: u64, + /// Outstanding base-class bytes per lane ([`meter_index`] of the + /// arrival, [`Serve::via`] — structural identity, never a wire field), + /// each bounded by [`base_quota`](Self::base_quota). + pair_supply: [u64; LANES], + /// Outstanding huge-class (2 MiB / 1 GiB aligned) bytes per lane, each + /// bounded by [`huge_quota`](Self::huge_quota). + pair_huge: [u64; LANES], + /// The base-class per-lane quota ([`MmEndowment::base_quota`]). + base_quota: u64, + /// The huge-class per-lane quota ([`MmEndowment::huge_quota`]). + huge_quota: u64, + reservations: Reservations, +} + +impl SupplyMeter { + fn new(base_quota: u64, huge_quota: u64) -> Self { + Self::with_table( + base_quota, + huge_quota, + Reservations::new(PROVISIONED_RESERVATIONS, RESERVATION_CEILING), + ) + } + + fn with_table(base_quota: u64, huge_quota: u64, reservations: Reservations) -> Self { + Self { + supplies: 0, + supply_bytes: 0, + pair_supply: [0; LANES], + pair_huge: [0; LANES], + base_quota, + huge_quota: huge_quota.min(HUGE_SUPPLY_QUOTA_MAX), + reservations, + } + } + + /// Validate one supply request and reserve it against `pair`'s meter: + /// `(rounded len, align_log2, receipt)` on success. Shape first + /// ([`MmRefusal::BadProps`] — zero size, an alignment outside the + /// `LeaseReq` vocabulary, an out-of-range lane, a supply no receipt can + /// name), then policy ([`MmRefusal::Quota`]), then mm's own room for one + /// more live supply ([`MmRefusal::Full`]). The books advance only on + /// success — a refused request perturbs nothing (a table grown on the + /// way keeps its free slots). + fn reserve(&mut self, pair: usize, size: u64, align: u64) -> Result<(u64, u8, u64), MmRefusal> { + let align_log2 = match align { + 0 => 0, + 0x1000 => kore_abi::session::LEASE_ALIGN_4K, + 0x20_0000 => kore_abi::session::LEASE_ALIGN_2M, + 0x4000_0000 => kore_abi::session::LEASE_ALIGN_1G, + _ => return Err(MmRefusal::BadProps), + }; + if size == 0 || pair >= LANES { + return Err(MmRefusal::BadProps); + } + // The alignment names the frame class (the kernel routes the lease + // to that class's ledger), and a class supply is a whole number of + // its units — the kernel refuses a 2 MiB-aligned lease that is not + // 2 MiB-sized, so round here and meter the rounded figure. + let huge = align_log2 >= kore_abi::session::LEASE_ALIGN_2M; + let unit = if huge { 1u64 << align_log2 } else { PAGE_BYTES }; + let len = size + .checked_next_multiple_of(unit) + .ok_or(MmRefusal::BadProps)?; + // Exact: `len` is a whole number of the class's rounding unit, and + // each rounding unit (a page; 2 MiB or 1 GiB) is a whole number of + // the class's receipt unit. A supply the field cannot name is refused, + // never encoded as a smaller one — a truncated count would retire the + // wrong span. + let units = len / receipt_unit(huge); + if units > RECEIPT_UNITS_MASK { + return Err(MmRefusal::BadProps); + } + let (outstanding, quota) = if huge { + (self.pair_huge[pair], self.huge_quota) + } else { + (self.pair_supply[pair], self.base_quota) + }; + if outstanding.checked_add(len).is_none_or(|t| t > quota) { + return Err(MmRefusal::Quota); + } + let (index, generation) = self.reservations.take().ok_or(MmRefusal::Full)?; + let receipt = Receipt { + index, + pair, + units, + huge, + generation, + } + .pack(); + + self.supplies = self.supplies.saturating_add(1); + self.supply_bytes = self.supply_bytes.saturating_add(len); + if huge { + self.pair_huge[pair] = outstanding + len; + } else { + self.pair_supply[pair] = outstanding + len; + } + Ok((len, align_log2, receipt)) + } + + /// Unwind one reservation (the grant session dropped). + fn rollback(&mut self, receipt: u64, pair: usize, len: u64) { + if self.retire(receipt, pair, len) { + self.supplies -= 1; + self.supply_bytes -= len; + } + } + + /// Consume one kernel-vouched completed lineage. Exact receipt, pair and + /// length matching makes duplicate, stale, and cross-pair reports inert. + /// The report names the supply's *lane* ([`meter_index`]). + fn release(&mut self, report: SupplyReleaseReport) -> bool { + self.retire(report.receipt, meter_index(report.pair), report.len) + } + + /// Read only this arrival lane's policy and live accounting. Transferred + /// supplies stay charged until the authenticated lineage release arrives. + fn supply_budget(&self, pair: usize) -> (u64, u64) { + (self.huge_quota, self.pair_huge[pair]) + } + + /// Whether any supply metered to `pair` is still outstanding. + fn outstanding(&self, pair: usize) -> bool { + self.pair_supply[pair] != 0 || self.pair_huge[pair] != 0 + } + + fn retire(&mut self, receipt: u64, pair: usize, len: u64) -> bool { + let fields = Receipt::unpack(receipt); + if fields.pair != pair + || fields.units.checked_mul(receipt_unit(fields.huge)) != Some(len) + || !self.reservations.end(fields.index, fields.generation) + { + return false; + } + if fields.huge { + self.pair_huge[pair] -= len; + } else { + self.pair_supply[pair] -= len; + } + true + } +} + +/// Occupancy snapshot (the [`HeapStats`](kore_rt::heap::HeapStats) shape). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MmStats { + /// Extents granted since bind (fresh + reused). + pub extents: u32, + /// Bytes granted since bind. + pub granted_bytes: u64, + /// Bytes returned via [`MmRequest::Release`]. + pub released_bytes: u64, + /// Bytes served from returned windows instead of fresh supply. + pub reused_bytes: u64, + /// Returned bytes dropped because the free list was full — visible, + /// never silent. + pub leaked_bytes: u64, + /// Fresh supply remaining past the bump cursor. + pub remaining: u64, + /// Root supply leases minted via [`MmRequest::Supply`]. + pub supplies: u32, + /// Bytes those supplies span (page-rounded, cumulative — the region + /// bitmap is the ledger of record; these are the meter's books). + pub supply_bytes: u64, +} + +/// The memory-manager service: the lessor over one supply region. +#[derive(Debug)] +pub struct MemoryManager { + /// The possessed supply — every heap grant attenuates it. + supply: SupplyToken, + /// The possessed region grant — every [`MmRequest::Supply`] mint + /// derives off it. + region: RegionToken, + /// Fresh-grant cursor (bump). + cursor: u64, + /// Returned windows awaiting reuse (supply-relative `(offset, len)`), + /// first-fit. + free: [Option<(u64, u64)>; FREE_SLOTS], + /// Extents granted. + extents: u32, + /// Bytes granted. + granted: u64, + /// Bytes returned. + released: u64, + /// Bytes re-served from returns. + reused: u64, + /// Returned bytes dropped on free-list overflow. + leaked: u64, + /// The child-supply meter ([`MmRequest::Supply`]'s books). + meter: SupplyMeter, + /// Reservation made by the request currently passing through ServeHost. + /// A successful grant is finalized when the next request begins; a + /// dropped grant consumes this immediately to restore exact prior books. + pending_grant: Option, + /// Assembles one per-launch face from exec's returned components. + peer_assembler: PeerBundleAssembler, + /// Where admitted faces' rings map (`geom::TRANSIENT`); `None` before + /// [`start_on_metal`](SystemService::start_on_metal) and off the metal. + peer_map: Option, + /// How many fixed pairs this row serves: the protocol's + /// [`SERVE_PAIRS`](SvcProtocol::SERVE_PAIRS) plus one per manifest row + /// declaring mm among its `clients` (the compositor's and image's heap + /// faces). Their lanes drain like the protocol's, and no face is ever + /// admitted onto one of their slots. + fixed_pairs: usize, + /// Per-launch faces by slot. + peers: [Option; LANES], + /// The verdict of the preceding admission, until exec takes it. + last_admission: Option>, +} + +impl MemoryManager { + /// Rights a granted window **delivers**: `ACCESS | MODIFY` — a mapped + /// R/W *view*, and nothing more. + /// + /// A grant is a carve off mm's own supply **lease**, so the only child + /// the algebra admits is a detached `LeasedRegion` view (an owned + /// `PhysExtent` sub-carve would be a second release authority over the + /// same frames). A view is a **leaf** in the sharing sense: the + /// sharing pivot stays with the lease-holder, which is mm. It carries + /// **no `REVOKE`**: teardown stays with the derivation chain — release + /// rides [`MmRequest::Release`] (the `Revoke × REVOKE_KIND_RETURN` + /// verb, whose authority is mm's named lease, not the view's rights). + /// It carries **`DERIVE`** not as a sharing pivot (nothing derives + /// from a view but the `map_view` exercise, which is `ACCESS`-gated) + /// but because the rendezvous **transfer gate** demands it of anything + /// presented to a send — without it a client could never return the + /// grant. The leaf posture is unharmed: revocation is T6-total, so a + /// transferred-onward view still dies with mm's return or lease + /// revoke; the worst a client can do with it is leak a window, never + /// alias one (the return's scoped clear is translate-checked). + /// + /// These are the rights *the client ends up holding*. The scratch + /// derive the transfer stages carries `DERIVE` on top (the kernel's + /// transfer gate demands it of anything presented to a send); the send + /// arm clamps the moved cap back to exactly this set — see + /// [`stage_grant`](kore_rt::transport::stage_grant). + pub const VIEW_RIGHTS: Rights = Rights::ACCESS.union(Rights::MODIFY).union(Rights::DERIVE); + + /// Occupancy snapshot. + pub fn stats(&self) -> MmStats { + MmStats { + extents: self.extents, + granted_bytes: self.granted, + released_bytes: self.released, + reused_bytes: self.reused, + leaked_bytes: self.leaked, + remaining: self.supply.len() - self.cursor, + supplies: self.meter.supplies, + supply_bytes: self.meter.supply_bytes, + } + } + + /// Validate + carve one window: `(offset, len)` page-rounded and + /// aligned to `max(props.align, PAGE_BYTES)` — served from a returned + /// window when one fits (first-fit, exact-head alignment; the tail + /// splits back), else from the bump cursor. State advances only on + /// success — a refused request perturbs nothing. + fn carve(&mut self, p: &AllocProps) -> Result<(u64, u64, CarveUndo), MmRefusal> { + validate_alloc_props(p)?; + let align = p.align.max(PAGE_BYTES); + let len = p + .size + .checked_next_multiple_of(PAGE_BYTES) + .ok_or(MmRefusal::Exhausted)?; + + // Reuse first: a returned window whose head satisfies the + // alignment and whose length covers the ask. + for (index, slot) in self.free.iter_mut().enumerate() { + if let Some((off, flen)) = *slot + && off % align == 0 + && flen >= len + { + let tail = flen - len; + *slot = (tail > 0).then_some((off + len, tail)); + self.extents += 1; + self.granted += len; + self.reused += len; + return Ok(( + off, + len, + CarveUndo::Reused { + index, + window: (off, flen), + len, + }, + )); + } + } + + let start = self + .cursor + .checked_next_multiple_of(align) + .ok_or(MmRefusal::Exhausted)?; + let end = start.checked_add(len).ok_or(MmRefusal::Exhausted)?; + if end > self.supply.len() { + return Err(MmRefusal::Exhausted); + } + let cursor = self.cursor; + self.cursor = end; + self.extents += 1; + self.granted += len; + Ok((start, len, CarveUndo::Fresh { cursor, len })) + } + + fn rollback_grant(&mut self, undo: CarveUndo) { + match undo { + CarveUndo::Fresh { cursor, len } => { + self.extents -= 1; + self.cursor = cursor; + self.granted -= len; + } + CarveUndo::Reused { index, window, len } => { + self.extents -= 1; + self.free[index] = Some(window); + self.granted -= len; + self.reused -= len; + } + CarveUndo::Supply { pair, len, receipt } => self.meter.rollback(receipt, pair, len), + } + } + + /// Reclaim a delivered extent: bind + retire it out of mm's arena + /// slot, verify the window is **this** supply's, and pool it. + fn take_back( + &mut self, + returned: Option, + ) -> Result<(), MmRefusal> { + let base = self.supply.base(); + let (rel, len) = returned + .and_then(|view| verbs::reclaim_returned_extent(view, &self.supply)) + .and_then(|(abs, len)| { + let rel = abs.checked_sub(base)?; + (rel.checked_add(len)? <= self.supply.len()).then_some((rel, len)) + }) + .ok_or(MmRefusal::BadReturn)?; + self.released += len; + match self.free.iter_mut().find(|s| s.is_none()) { + Some(s) => *s = Some((rel, len)), + // Full list: the window is dropped from circulation — counted, + // never silent (coalescing/a real free structure is the later + // tier; the books keep the loss honest). + None => self.leaked += len, + } + Ok(()) + } +} + +/// Validate only properties the current `SupplyToken` endowment can prove. +/// +/// In particular, the ABI carrying `attrs` and `dma` is not evidence that the +/// kernel applied either property. The detached-window derive inherits the +/// source lease's write-back attributes, and its frozen request currently +/// treats DMA class as provenance-only. Rejecting stronger intent here keeps a +/// granted token truthful until a richer, kernel-authenticated supply witness +/// lands. +fn validate_alloc_props(p: &AllocProps) -> Result<(), MmRefusal> { + if p.size == 0 + || !p.align.is_power_of_two() + || p.numa != NumaNode::ANY + || p.dma != DmaClass::None + || p.attrs != SUPPLY_ALLOC_ATTRS + { + Err(MmRefusal::BadProps) + } else { + Ok(()) + } +} + +impl SystemService for MemoryManager { + const NAME: &'static str = "mm"; + type Endowment = MmEndowment; + + fn bind(e: MmEndowment) -> Self { + Self { + supply: e.supply, + region: e.region, + cursor: 0, + free: [None; FREE_SLOTS], + extents: 0, + granted: 0, + released: 0, + reused: 0, + leaked: 0, + meter: SupplyMeter::new(e.base_quota, e.huge_quota), + pending_grant: None, + peer_assembler: PeerBundleAssembler::new(::REPLY_GRANTS, true), + peer_map: None, + fixed_pairs: ::SERVE_PAIRS, + peers: [None; LANES], + last_admission: None, + } + } + + /// Per-launch faces arrive with their components on exec's return leg + /// (rings, kick, grant channel, return receive), which a fixed landing + /// cannot carry; `Release`'s extent returns ride the same retained leg. + const RETURN_MODE: ReturnMode = ReturnMode::RetainedAny; + + fn start_on_metal(&mut self, rt: &kore_rt::entry::Rt) -> Result<(), StartFailure> { + // The peer map places slot `n`'s ring at `base + n * stride`, and the + // host can grow a face into any slot it can name: the window must + // hold every lane, not just the bootstrap table. + let window = rt + .geom() + .and_then(|geometry| geometry.window_of(kore_abi::task::geom::TRANSIENT)) + .filter(|window| { + window.base() != 0 + && window.base() & (PAGE_BYTES - 1) == 0 + && window.len() >= PEER_RING_BYTES * LANES as u64 + }) + .ok_or(StartFailure::new(1))?; + self.fixed_pairs = + kore_rt::entry::service_pair_count::(rt).ok_or(StartFailure::new(2))?; + self.peer_map = Some(PeerMapPlan { + base: window.base(), + stride: PEER_RING_BYTES, + }); + Ok(()) + } + + /// Every engine program holds one face for its life (a JavaScript + /// window, a Lua or Wasm run, an inference session), so the faces must + /// grow with the programs, not stop at the bootstrap table's leftovers. + fn peer_arena(&self) -> Option<&'static (dyn core::alloc::Allocator + Sync)> { + Some(&FACE_ARENA) + } + + fn serve(&mut self, req: MmRequest, sv: &mut Serve<'_, MmResponse>) { + // ServeHost resolves every prior non-empty grant session before it + // drives the next request, so reaching here finalizes that reservation. + self.pending_grant = None; + // The region bitmap has already accepted every reported return. Drain + // before admission so sequential short-lived launches immediately + // reuse quota even when their cumulative mints exceed the bound — + // bounded per pair ([`SUPPLY_RELEASE_DRAIN_PER_PAIR`]), and each poll + // names its pair so the kernel consults exactly that lifecycle lane. + // Every fixed pair drains, the declared clients' as much as the + // protocol's; and only a lane with a supply still outstanding is + // polled — one whose row is zero has had every mint credited, so it + // has nothing to report, and the poll would be a wasted kernel call + // per idle face on every request. + let fixed = (0..self.fixed_pairs).map(|pair| (pair, pair as u16)); + let faces = self + .peers + .iter() + .enumerate() + .filter_map(|(slot, lane)| lane.map(|lane| (slot, lane.arrival))); + for (row, lane) in fixed.chain(faces) { + if !self.meter.outstanding(row) { + continue; + } + for _ in 0..SUPPLY_RELEASE_DRAIN_PER_PAIR { + let Some(report) = verbs::take_supply_release(&self.region, lane) else { + break; + }; + let _ = self.meter.release(report); + } + } + let via = sv.via(); + let face = PeerId::from_arrival(via); + // A face may only mint supplies, read the books and close itself: + // allocation views and their returns are the fixed pairs' verbs. + if face.is_some() + && !matches!( + req, + MmRequest::Supply { .. } | MmRequest::Stats | MmRequest::ClosePeer + ) + { + sv.reply(MmResponse::Refused(MmRefusal::Denied)); + return; + } + match req { + MmRequest::Alloc(props) => match self.carve(&props) { + Ok((offset, len, undo)) => { + self.pending_grant = Some(undo); + // The staged derive: subject = the held supply, the + // carve + placement class ride the arg. The client + // gets a token — a `LeasedRegion` view carrying exactly + // the delivered [`VIEW_RIGHTS`](Self::VIEW_RIGHTS). + sv.grant( + self.supply.slot(), + Self::VIEW_RIGHTS, + GrantArg::Window { + offset, + len, + class: props.dma, + }, + ); + sv.reply(MmResponse::Granted); + } + // Fail-closed: the refusal carries data, never authority. + Err(r) => sv.reply(MmResponse::Refused(r)), + }, + MmRequest::Release { slot } => { + let returned = sv.returned_input(slot).and_then(|mut input| input.take()); + match self.take_back(returned) { + Ok(()) => sv.reply(MmResponse::Released), + Err(r) => sv.reply(MmResponse::Refused(r)), + } + } + // The child-memory mint (slice D): validate + meter against + // the ARRIVAL PAIR (structural identity, never a wire + // field), then stage the `LeaseReq` derive off the region + // grant — a fresh **root** lease, delivered whole at + // [`SUPPLY_RIGHTS`]. The backing arrives zeroed by the + // kernel's lease-derive scrub invariant; the region claim + // itself happens inside the grant session, so a refused + // session leaves the bitmap untouched (I3) and + // `grants_dropped` restores the meter. + MmRequest::Supply { size, align } => { + // A face mints only while it is attached: a closing face's + // lane is kept to credit returns, not to admit new debt. + let admitted = match face { + Some(peer) => { + self.peers[peer.slot()].is_some_and(|lane| lane.live && lane.arrival == via) + } + None => usize::from(via) < self.fixed_pairs, + }; + if !admitted { + sv.reply(MmResponse::Refused(MmRefusal::Denied)); + return; + } + let pair = meter_index(via); + match self.meter.reserve(pair, size, align) { + Ok((len, align_log2, receipt)) => { + self.pending_grant = Some(CarveUndo::Supply { pair, len, receipt }); + sv.grant( + self.region.slot(), + SUPPLY_RIGHTS, + GrantArg::Lease { + size: len, + align_log2, + flags: 0, + release: Some(SupplyReleaseGrant { receipt, pair: via }), + }, + ); + sv.reply(MmResponse::Granted); + } + Err(r) => sv.reply(MmResponse::Refused(r)), + } + } + MmRequest::Stats => { + let pair = meter_index(via); + let stats = self.stats(); + let (limit, used) = self.meter.supply_budget(pair); + let snapshot = kore_abi::monitor::MemorySnapshot::new( + stats.extents, + stats.supplies, + stats.granted_bytes, + stats.released_bytes, + stats.reused_bytes, + stats.leaked_bytes, + stats.remaining, + stats.supply_bytes, + ).with_supply_budget(limit, used); + sv.reply(match snapshot { + Some(snapshot) => MmResponse::Stats(snapshot), + None => MmResponse::Refused(MmRefusal::Denied), + }); + } + MmRequest::AdmitPeer { part, announced } => { + // A `Ring` is the boundary between bundles, so one arriving + // while the assembler still owns components proves the + // previous assembly was abandoned — its asker spent its + // bounded re-ask and gave up. Retire what it left, and + // refuse the replacement while any of those owners is still + // closing (`PeerBundleAssembler::retire_incomplete`). + // + // Without this the leftovers are permanent: the next launch's + // matching part answers `Duplicate`, the bundle never + // completes, no verdict is ever published, and every engine + // launch for the rest of the boot refuses. + // + // Admin only, and evaluated before anything else touches the + // assembler: a foreign via must never retire the components + // the administration pair is still assembling. + let abandoned = via == ADMIN_VIA + && matches!(part, PeerBundlePart::Ring) + && !self.peer_assembler.is_empty() + && !self.peer_assembler.retire_incomplete(); + let reply = if via != ADMIN_VIA || self.last_admission.is_some() || abandoned { + MmResponse::Refused(MmRefusal::Pending) + } else if let (Some(map), Some(mut input)) = + (self.peer_map, sv.returned_input(announced)) + { + let fits = input + .capability() + .and_then(|capability| self.peer_assembler.accepts(part, capability).ok()) + .is_some(); + match input.take().filter(|_| fits) { + Some(capability) => match self.peer_assembler.push(part, capability) { + Ok(Some(bundle)) => { + sv.admit_peer(bundle, map); + MmResponse::Accepted + } + Ok(None) => MmResponse::Accepted, + // `accepts` just validated this part, so a push + // cannot refuse it — but answering `Accepted` for + // a component that was not taken would promise an + // admission that can never be published. + Err(_) => MmResponse::Refused(MmRefusal::BadReturn), + }, + None => MmResponse::Refused(MmRefusal::BadReturn), + } + } else { + MmResponse::Refused(MmRefusal::Denied) + }; + sv.reply(reply); + } + MmRequest::TakeAdmission => sv.reply(if via != ADMIN_VIA { + MmResponse::Refused(MmRefusal::Denied) + } else { + match self.last_admission.take() { + Some(Ok(peer)) => MmResponse::PeerAdmitted { peer }, + // No room for one more face — the host's slots, its + // arena, or mm's capability arena — is capacity, which + // the launcher reports as a full table, apart from a face + // that arrived wrong. + Some(Err(refusal)) => MmResponse::Refused(admission_refusal(refusal)), + // No verdict, and the assembler still owns components: + // this bundle never completed, so `admit_peer` was never + // called and no verdict can EVER be published. `Pending` + // here would be a lie the asker waits on forever — and + // `provision_mm_peer` fences new provisioning while an + // id-less face exists, so that wait refuses every later + // engine launch for the rest of the boot. Retire the + // abandoned assembly and answer terminally, which is + // exec's signal to let the peer go. + None if !self.peer_assembler.is_empty() => { + let _ = self.peer_assembler.retire_incomplete(); + MmResponse::Refused(MmRefusal::BadReturn) + } + // An EMPTY assembler means the bundle completed and the + // host has not drained its admission action yet: a + // verdict IS coming, and settling on it would orphan the + // face mm is about to admit. + None => MmResponse::Refused(MmRefusal::Pending), + } + }), + MmRequest::RetirePeer { peer } => { + let live = self.peers[peer.slot()] + .is_some_and(|lane| lane.live && lane.arrival == peer.arrival()); + if via == ADMIN_VIA && live { + sv.retire_peer(peer, PeerClose::Orderly); + sv.reply(MmResponse::Accepted); + } else { + sv.reply(MmResponse::Refused(MmRefusal::Denied)); + } + } + MmRequest::ConfirmRetired { peer } => sv.reply(if via != ADMIN_VIA { + MmResponse::Refused(MmRefusal::Denied) + } else if self.peers[peer.slot()].is_some_and(|lane| lane.arrival == peer.arrival()) { + MmResponse::Refused(MmRefusal::Pending) + } else { + MmResponse::Accepted + }), + MmRequest::ClosePeer => { + if face.is_some() { + sv.close_peer(PeerClose::Orderly); + sv.reply(MmResponse::Accepted); + } else { + sv.reply(MmResponse::Refused(MmRefusal::Denied)); + } + } + } + } + + fn peer_admitted(&mut self, result: Result) -> bool { + if let Ok(peer) = result { + // A face never lands on a fixed pair's index — the declared + // clients' included — nor on a slot whose previous face still + // owes returns. + if peer.slot() < self.fixed_pairs || self.peers[peer.slot()].is_some() { + self.last_admission = Some(Err(PeerAdmissionRefusal::PolicyCollision)); + return false; + } + self.peers[peer.slot()] = Some(PeerLane { + arrival: peer.arrival(), + live: true, + }); + } + self.last_admission = Some(result); + true + } + + /// A closed face's slot stays closing — unreusable, its lane still + /// drained — until every supply it minted has returned: a supply is + /// conveyed onward at spawns and credits only when its whole lineage + /// dies, possibly long after the face closed. + fn peer_closed(&mut self, via: u16, _reason: PeerClose) -> bool { + let Some(slot) = PeerId::from_arrival(via).map(|peer| peer.slot()) else { + return true; + }; + match &mut self.peers[slot] { + Some(lane) if lane.arrival == via => { + lane.live = false; + if self.meter.outstanding(slot) { + return false; + } + self.peers[slot] = None; + true + } + _ => true, + } + } + + fn grants_dropped(&mut self, response: &mut MmResponse, emitted: usize) { + if emitted == 1 && matches!(response, MmResponse::Granted) { + if let Some(undo) = self.pending_grant.take() { + self.rollback_grant(undo); + } + *response = MmResponse::Refused(MmRefusal::Denied); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The 4 KiB class a 4 GiB QEMU guest carves (`kore: mem classes — … + /// 4K 737 MiB`), and the base quota every lane derives from it. + const GUEST_CLASS: u64 = 737 << 20; + const BASE: u64 = base_quota_for(GUEST_CLASS); + + /// A meter over a table of exactly `slots` reservations that cannot + /// grow, so a test can fill it. + fn fixed(slots: usize, huge_quota: u64) -> SupplyMeter { + SupplyMeter::with_table(BASE, huge_quota, Reservations::new(slots, slots)) + } + + fn cpu_props() -> AllocProps { + AllocProps { + size: PAGE_BYTES, + align: PAGE_BYTES, + numa: NumaNode::ANY, + dma: DmaClass::None, + attrs: SUPPLY_ALLOC_ATTRS, + contiguous: false, + } + } + + /// Allocation admission is a proof boundary: accept only the exact + /// properties the geometry-only supply endowment can witness. + #[test] + fn allocation_props_refuse_unproven_placement_and_cache_claims() { + let base = cpu_props(); + assert_eq!(validate_alloc_props(&base), Ok(())); + + let contiguous = AllocProps { + contiguous: true, + ..base + }; + assert_eq!(validate_alloc_props(&contiguous), Ok(())); + + for malformed in [ + AllocProps { size: 0, ..base }, + AllocProps { align: 3, ..base }, + ] { + assert_eq!(validate_alloc_props(&malformed), Err(MmRefusal::BadProps)); + } + + let named_numa = AllocProps { + numa: NumaNode(0), + ..base + }; + assert_eq!(validate_alloc_props(&named_numa), Err(MmRefusal::BadProps)); + + for dma in [DmaClass::Dma32, DmaClass::Dma64] { + let requested = AllocProps { dma, ..base }; + assert_eq!(validate_alloc_props(&requested), Err(MmRefusal::BadProps)); + } + + for attrs in [ + MemAttrs::READ | MemAttrs::WRITE, + SUPPLY_ALLOC_ATTRS | MemAttrs::UNCACHED, + MemAttrs::WRITE_COMBINE | MemAttrs::READ | MemAttrs::WRITE, + ] { + let requested = AllocProps { attrs, ..base }; + assert_eq!(validate_alloc_props(&requested), Err(MmRefusal::BadProps)); + } + } + + #[test] + fn deployed_quota_admits_lfm_model_and_launch_headroom() { + let quota: u64 = include_str!("../../boot.toml") + .lines() + .find_map(|line| line.strip_prefix("supply_quota_bytes = ")) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap(); + let model_bytes = 2_734_526_464; + let mut old = SupplyMeter::new(BASE, 2 << 30); + assert_eq!(old.reserve(0, model_bytes, 2 << 20), Err(MmRefusal::Quota)); + let mut deployed = SupplyMeter::new(BASE, quota); + let (rounded, _, _) = deployed.reserve(0, model_bytes, 2 << 20).unwrap(); + assert!(rounded >= model_bytes); + assert!(deployed.reserve(0, 512 << 20, 2 << 20).is_ok()); + assert_eq!(deployed.reserve(0, quota, 2 << 20), Err(MmRefusal::Quota)); + } + + #[test] + fn live_budget_is_per_face_and_only_exact_retirement_reclaims_it() { + let mut meter = fixed(8, 26 << 30); + let (bytes, _, receipt) = meter.reserve(0, 24 << 30, 1 << 30).unwrap(); + let view = |meter: &SupplyMeter, pair| { + let (limit, used) = meter.supply_budget(pair); + kore_abi::monitor::MemorySnapshot::new(0, 0, 0, 0, 0, 0, 0, 0) + .with_supply_budget(limit, used).unwrap().huge_supply_remaining().unwrap() + }; + assert_eq!(view(&meter, 0), 2 << 30); + assert_eq!(view(&meter, 1), 26 << 30); + assert_eq!(meter.reserve(0, 3 << 30, 1 << 30), Err(MmRefusal::Quota)); + assert_eq!(view(&meter, 0), 2 << 30); + assert!(!meter.retire(receipt, 1, bytes)); + assert_eq!(view(&meter, 0), 2 << 30); + assert!(meter.retire(receipt, 0, bytes)); + assert_eq!(view(&meter, 0), 26 << 30); + assert!(!meter.retire(receipt, 0, bytes)); + } + + #[test] + fn deployed_quota_keeps_chunk_headroom_with_eight_large_units() { + let quota: u64 = include_str!("../../boot.toml") + .lines() + .find_map(|line| line.strip_prefix("supply_quota_bytes = ")) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap(); + const GIB: u64 = 1 << 30; + const MIB: u64 = 1 << 20; + // Inference's existing 256 MiB heap and 96 MiB GPU scratch are + // mandatory before its elastic 512/128 MiB cache chunks. The old + // 11 GiB policy could retain 3712 MiB of chunks beside seven units. + // NPU programs have artifact-derived charges, tested at admission; + // this lower-bound case does not invent a fixed program reserve. + let chunks = [512 * MIB; 7].into_iter().chain([128 * MIB]); + let mut previous = fixed(16, 11 * GIB); + for (bytes, class) in [ + (7 * GIB, GIB), + (256 * MIB, 2 * MIB), + (96 * MIB, 2 * MIB), + ] { + previous.reserve(0, bytes, class).unwrap(); + } + for bytes in chunks.clone() { + previous.reserve(0, bytes, 2 * MIB).unwrap(); + } + // Both launch layouts consume eight large units: model only, or + // one resident NPU weight supply followed by a seven-unit model. + for weight_units in [0, 1] { + let mut deployed = fixed(16, quota); + let mut old = fixed(16, 11 * GIB); + for meter in [&mut deployed, &mut old] { + meter.reserve(0, 256 * MIB, 2 * MIB).unwrap(); + if weight_units != 0 { + meter.reserve(0, weight_units * GIB, GIB).unwrap(); + } + meter.reserve(0, (8 - weight_units) * GIB, GIB).unwrap(); + meter.reserve(0, 96 * MIB, 2 * MIB).unwrap(); + } + let mut old_refused = false; + for bytes in chunks.clone() { + deployed.reserve(0, bytes, 2 * MIB).unwrap(); + match old.reserve(0, bytes, 2 * MIB) { + Ok(_) => {} + Err(MmRefusal::Quota) => old_refused = true, + other => panic!("unexpected old-quota result: {other:?}"), + } + } + assert!(old_refused, "the old policy must displace external chunks"); + assert_eq!(deployed.pair_huge[0], previous.pair_huge[0] + GIB); + let remaining = quota - deployed.pair_huge[0]; + deployed.reserve(0, remaining, 2 * MIB).unwrap(); + assert_eq!(deployed.reserve(0, 2 * MIB, 2 * MIB), Err(MmRefusal::Quota)); + // Raising huge allowance does not raise the separate base meter. + deployed.reserve(0, BASE, 0).unwrap(); + assert_eq!(deployed.reserve(0, PAGE_BYTES, 0), Err(MmRefusal::Quota)); + } + } + + /// The meter's whole vocabulary: rounding, the closed alignment set, + /// the zero refusal, and the out-of-range pair. + #[test] + fn the_supply_meter_speaks_the_lease_req_vocabulary() { + let mut m = SupplyMeter::new(BASE, HUGE_SUPPLY_QUOTA_MAX); + // Page rounding + the default alignment. + let (len, align, _) = m.reserve(0, 1, 0).unwrap(); + assert_eq!((len, align), (PAGE_BYTES, 0)); + // The three explicit alignments map to their log2 constants. + let (len, align, _) = m.reserve(0, 0x1000, 0x1000).unwrap(); + assert_eq!((len, align), (0x1000, kore_abi::session::LEASE_ALIGN_4K)); + let (len, align, _) = m.reserve(0, 0x20_0000, 0x20_0000).unwrap(); + assert_eq!((len, align), (0x20_0000, kore_abi::session::LEASE_ALIGN_2M)); + // A class supply is a whole number of its units (the kernel refuses + // a 1 GiB-aligned lease that is not 1 GiB-sized). + let (len, align, _) = m.reserve(0, 0x1000, 0x4000_0000).unwrap(); + assert_eq!( + (len, align), + (0x4000_0000, kore_abi::session::LEASE_ALIGN_1G) + ); + // Out-of-vocabulary shapes refuse without touching the books. + let before = (m.supplies, m.supply_bytes); + assert_eq!(m.reserve(0, 0, 0), Err(MmRefusal::BadProps)); + assert_eq!(m.reserve(0, 0x1000, 0x2000), Err(MmRefusal::BadProps)); + assert_eq!(m.reserve(LANES, 0x1000, 0), Err(MmRefusal::BadProps)); + assert_eq!((m.supplies, m.supply_bytes), before); + } + + /// Each pair meters independently, the bound is exact, and a + /// rollback restores the meter to the byte. + #[test] + fn the_quota_meters_per_pair_and_rolls_back_exactly() { + let mut m = SupplyMeter::new(BASE, HUGE_SUPPLY_QUOTA_MAX); + // Fill pair 1 (the shell's face) to the exact quota. + let (_, _, pair_one) = m.reserve(1, BASE, 0).unwrap(); + // One more page on the same pair is a policy refusal… + assert_eq!(m.reserve(1, PAGE_BYTES, 0), Err(MmRefusal::Quota)); + // …while pair 2 (init's face) still has its whole allowance. + let (_, _, _) = m.reserve(2, PAGE_BYTES, 0).unwrap(); + assert_eq!(m.supplies, 2); + assert_eq!(m.supply_bytes, BASE + PAGE_BYTES); + // A dropped grant unwinds pair 1 whole; the allowance returns. + m.rollback(pair_one, 1, BASE); + assert_eq!(m.supplies, 1); + assert_eq!(m.supply_bytes, PAGE_BYTES); + let (_, _, _) = m.reserve(1, PAGE_BYTES, 0).unwrap(); + assert_eq!(m.pair_supply[1], PAGE_BYTES); + } + + /// Huge-class supplies meter against the datum quota, independently of + /// the base class; a zero datum refuses every huge request; completions + /// credit the class their receipt names. + #[test] + fn huge_class_supplies_meter_against_the_datum_not_the_base_quota() { + let mut m = SupplyMeter::new(BASE, 4 << 30); + // Base class is untouched by a 3 GiB huge supply... + let (len, _, huge) = m.reserve(1, 3 << 30, 0x4000_0000).unwrap(); + assert_eq!(len, 3 << 30); + assert_eq!(m.pair_supply[1], 0); + assert_eq!(m.pair_huge[1], 3 << 30); + // ...and the base quota is still whole. + let (_, _, _) = m.reserve(1, BASE, 0).unwrap(); + // The huge quota is exact: 2 MiB units past 4 GiB refuse. + let (_, _, _) = m.reserve(1, 1 << 30, 0x20_0000).unwrap(); + assert_eq!(m.reserve(1, 0x20_0000, 0x20_0000), Err(MmRefusal::Quota)); + // Completion credits the huge meter, not the base one. + assert!(m.release(SupplyReleaseReport { + receipt: huge, + pair: 1, + len: 3 << 30, + })); + assert_eq!(m.pair_huge[1], 1 << 30); + assert_eq!(m.pair_supply[1], BASE); + + // No datum ⇒ no huge quota, loudly. + let mut none = SupplyMeter::new(BASE, 0); + assert_eq!(none.reserve(0, 0x20_0000, 0x20_0000), Err(MmRefusal::Quota)); + let (_, _, _) = none.reserve(0, 0x1000, 0).unwrap(); + } + + #[test] + fn completed_lineages_reuse_quota_exactly_once() { + let mut m = fixed(2, HUGE_SUPPLY_QUOTA_MAX); + let (_, _, first) = m.reserve(1, BASE, 0).unwrap(); + assert_eq!(m.reserve(1, PAGE_BYTES, 0), Err(MmRefusal::Quota)); + + let report = SupplyReleaseReport { + receipt: first, + pair: 1, + len: BASE, + }; + assert!(m.release(report)); + assert!(!m.release(report), "duplicate completion must be inert"); + assert_eq!(m.pair_supply[1], 0); + + // Cumulative mints may now exceed the policy bound while live bytes do + // not; simultaneous live bytes remain bounded. + let (_, _, _) = m.reserve(1, BASE, 0).unwrap(); + assert_eq!(m.supply_bytes, 2 * BASE); + assert_eq!(m.reserve(1, PAGE_BYTES, 0), Err(MmRefusal::Quota)); + } + + /// Every field round-trips at its maximum, beside every other field at + /// its maximum, so no field can bleed into its neighbour. + #[test] + fn receipts_round_trip_at_every_fields_maximum() { + for huge in [false, true] { + let top = Receipt { + index: RESERVATION_CEILING - 1, + pair: LANES - 1, + units: RECEIPT_UNITS_MASK, + huge, + generation: RECEIPT_GENERATION_MAX, + }; + assert_eq!(Receipt::unpack(top.pack()), top); + let one = Receipt { + index: 1, + pair: 1, + units: 1, + huge, + generation: 1, + }; + assert_eq!(Receipt::unpack(one.pack()), one); + } + } + + /// The last slot of a table on the highest lane round-trips through the + /// meter, and its receipt credits that lane once and no other. + #[test] + fn the_last_slot_on_the_last_lane_releases_exactly_once() { + let mut meter = fixed(4, HUGE_SUPPLY_QUOTA_MAX); + let pair = LANES - 1; + for _ in 0..3 { + meter.reserve(0, PAGE_BYTES, 0).unwrap(); + } + let (_, _, receipt) = meter.reserve(pair, BASE, 0).unwrap(); + assert_eq!(Receipt::unpack(receipt).index, 3); + let report = |pair: usize| SupplyReleaseReport { + receipt, + pair: pair as u16, + len: BASE, + }; + assert!(!meter.release(report(0))); + assert!(meter.release(report(pair))); + assert_eq!(meter.pair_supply[pair], 0); + assert!(!meter.release(report(pair))); + } + + #[test] + fn stale_mismatched_and_reordered_completions_are_inert() { + let mut m = fixed(2, HUGE_SUPPLY_QUOTA_MAX); + let (_, _, a) = m.reserve(0, PAGE_BYTES, 0).unwrap(); + let (_, _, b) = m.reserve(0, PAGE_BYTES, 0).unwrap(); + + assert!(!m.release(SupplyReleaseReport { + receipt: a, + pair: 1, + len: PAGE_BYTES, + })); + assert!(!m.release(SupplyReleaseReport { + receipt: a, + pair: 0, + len: 2 * PAGE_BYTES, + })); + assert_eq!(m.pair_supply[0], 2 * PAGE_BYTES); + + assert!(m.release(SupplyReleaseReport { + receipt: b, + pair: 0, + len: PAGE_BYTES, + })); + assert!(m.release(SupplyReleaseReport { + receipt: a, + pair: 0, + len: PAGE_BYTES, + })); + assert_eq!(m.pair_supply[0], 0); + + // Reuse both slots, then prove the old generation cannot release the + // new reservation occupying the same index. + let (_, _, _) = m.reserve(0, PAGE_BYTES, 0).unwrap(); + let (_, _, _) = m.reserve(0, PAGE_BYTES, 0).unwrap(); + assert!(!m.release(SupplyReleaseReport { + receipt: a, + pair: 0, + len: PAGE_BYTES, + })); + assert_eq!(m.pair_supply[0], 2 * PAGE_BYTES); + } + + #[test] + fn failed_grant_rollback_cannot_later_double_release() { + let mut m = fixed(1, HUGE_SUPPLY_QUOTA_MAX); + let (_, _, receipt) = m.reserve(3, PAGE_BYTES, 0).unwrap(); + m.rollback(receipt, 3, PAGE_BYTES); + assert_eq!(m.pair_supply[3], 0); + assert!(!m.release(SupplyReleaseReport { + receipt, + pair: 3, + len: PAGE_BYTES, + })); + assert_eq!((m.supplies, m.supply_bytes), (0, 0)); + } + + /// A lane is booked to its fixed pair or its face's slot, and every + /// arrival the host can stamp names a row — the declared clients' fixed + /// pairs past the protocol's, and faces grown past the bootstrap table. + #[test] + fn a_lane_books_to_its_pair_or_its_face_slot() { + assert_eq!(meter_index(0), 0); + assert_eq!(meter_index(3), 3); + assert_eq!(meter_index(5), 5, "a declared client's fixed pair"); + let face = PeerId::from_arrival(0x0105).unwrap(); + assert_eq!(meter_index(face.arrival()), 5); + let grown = PeerId::from_arrival(0x01ff).unwrap(); + assert_eq!(meter_index(grown.arrival()), LANES - 1); + for lane in [0, u8::MAX as u16, u8::MAX as u16 + 1, u16::MAX] { + assert!(meter_index(lane) < LANES, "lane {lane:#x} has a row"); + } + } + + /// A face grown past the bootstrap table meters, encodes and releases + /// on its own lane, and its receipt cannot credit the slot's neighbour. + #[test] + fn a_face_past_the_bootstrap_table_meters_on_its_own_lane() { + let face = PeerId::from_arrival(0x0140).unwrap(); + assert!(face.slot() >= kore_rt::svc::MAX_SERVE_PAIRS); + let pair = meter_index(face.arrival()); + let mut m = fixed(4, HUGE_SUPPLY_QUOTA_MAX); + let (len, _, receipt) = m.reserve(pair, 5 * PAGE_BYTES, 0).unwrap(); + assert_eq!( + ((receipt >> RECEIPT_PAIR_SHIFT) & RECEIPT_PAIR_MASK) as usize, + pair + ); + let neighbour = PeerId::from_arrival(0x0141).unwrap(); + assert!(!m.release(SupplyReleaseReport { + receipt, + pair: neighbour.arrival(), + len, + })); + assert!(m.release(SupplyReleaseReport { + receipt, + pair: face.arrival(), + len, + })); + assert!(!m.outstanding(pair)); + } + + /// Each class counts its receipt in its own unit: the largest supply + /// either quota admits round-trips exactly, and a completion whose + /// length disagrees with the receipt's count in its class's unit is + /// inert — a base receipt cannot be read in 2 MiB units, nor a huge one + /// in pages. + #[test] + fn receipts_count_each_class_in_its_own_unit() { + let mut m = fixed(4, HUGE_SUPPLY_QUOTA_MAX); + let (base_len, _, base) = m.reserve(7, BASE, 0).unwrap(); + let (huge_len, _, huge) = m.reserve(7, HUGE_SUPPLY_QUOTA_MAX, 0x4000_0000).unwrap(); + assert_eq!((base_len, huge_len), (BASE, HUGE_SUPPLY_QUOTA_MAX)); + assert_eq!( + (base >> RECEIPT_UNITS_SHIFT) & RECEIPT_UNITS_MASK, + BASE / PAGE_BYTES + ); + assert_eq!( + (huge >> RECEIPT_UNITS_SHIFT) & RECEIPT_UNITS_MASK, + HUGE_SUPPLY_QUOTA_MAX / HUGE_UNIT_BYTES + ); + // Read in the other class's unit, each count names a different span. + let units = (base >> RECEIPT_UNITS_SHIFT) & RECEIPT_UNITS_MASK; + assert!(!m.release(SupplyReleaseReport { + receipt: base, + pair: 7, + len: units * HUGE_UNIT_BYTES, + })); + assert!(m.release(SupplyReleaseReport { + receipt: base, + pair: 7, + len: BASE, + })); + assert!(m.release(SupplyReleaseReport { + receipt: huge, + pair: 7, + len: HUGE_SUPPLY_QUOTA_MAX, + })); + assert!(!m.outstanding(7)); + } + + /// No room for one more face is capacity; a face that arrived wrong is + /// a denial — the two refusals the launcher tells apart. + #[test] + fn a_full_face_table_is_capacity_and_a_malformed_face_is_denied() { + assert_eq!( + admission_refusal(PeerAdmissionRefusal::Capacity), + MmRefusal::Full + ); + assert_eq!( + admission_refusal(PeerAdmissionRefusal::NoScratchSlots), + MmRefusal::Full + ); + for wrong in [ + PeerAdmissionRefusal::InvalidMapPlan, + PeerAdmissionRefusal::WrongRingAuthority, + PeerAdmissionRefusal::WrongEndpointAuthority, + PeerAdmissionRefusal::WrongGrantCount, + PeerAdmissionRefusal::WrongReturnShape, + PeerAdmissionRefusal::RingGeometry, + PeerAdmissionRefusal::MapFailed, + PeerAdmissionRefusal::RingMalformed, + PeerAdmissionRefusal::PolicyCollision, + PeerAdmissionRefusal::PolicyUnavailable, + ] { + assert_eq!(admission_refusal(wrong), MmRefusal::Denied, "{wrong:?}"); + } + } + + /// A face's returns credit its slot's row, keyed by its arrival. + #[test] + fn a_face_supply_is_metered_and_released_on_its_slot() { + let face = PeerId::from_arrival(0x0104).unwrap(); + let mut m = fixed(8, 0); + let pair = meter_index(face.arrival()); + let (len, _, receipt) = m.reserve(pair, 3 * PAGE_BYTES, 0).unwrap(); + assert!(m.outstanding(pair)); + // A report on the wrong lane is inert. + assert!(!m.release(SupplyReleaseReport { + receipt, + pair: 3, + len, + })); + assert!(m.release(SupplyReleaseReport { + receipt, + pair: face.arrival(), + len, + })); + assert!(!m.outstanding(pair)); + } + + /// A lane's base quota is half the 4 KiB class, in whole pages: on the + /// 4 GiB guest every gate boots, more than the 128 MiB constant it + /// replaced, and on a larger machine proportionally more. + #[test] + fn the_base_quota_is_half_the_class_in_whole_pages() { + assert_eq!(BASE, GUEST_CLASS / 2); + assert_eq!(BASE % PAGE_BYTES, 0); + assert!(BASE > 128 << 20); + assert_eq!(base_quota_for(3 * PAGE_BYTES), PAGE_BYTES); + assert_eq!(base_quota_for(PAGE_BYTES), 0); + assert_eq!(base_quota_for(0), 0); + let laptop = base_quota_for(3_276 << 20); + assert_eq!(laptop, 1_638 << 20); + } + + /// Past its provision the table doubles up to its ceiling; there one more + /// live supply is mm's capacity refused, not the lane's quota, and a + /// returned supply's slot serves the next under a new generation, which + /// the old receipt cannot name. + #[test] + fn the_table_grows_past_its_provision_up_to_its_ceiling() { + let mut m = SupplyMeter::with_table(BASE, 0, Reservations::new(2, 8)); + let receipts: Vec = (0..8) + .map(|_| m.reserve(3, PAGE_BYTES, 0).unwrap().2) + .collect(); + assert_eq!(m.reservations.states.len(), 8); + assert_eq!(m.reserve(3, PAGE_BYTES, 0), Err(MmRefusal::Full)); + assert_eq!(m.pair_supply[3], 8 * PAGE_BYTES, "a refusal books nothing"); + let report = |receipt| SupplyReleaseReport { + receipt, + pair: 3, + len: PAGE_BYTES, + }; + assert!(m.release(report(receipts[5]))); + let (_, _, again) = m.reserve(3, PAGE_BYTES, 0).unwrap(); + let fields = Receipt::unpack(again); + assert_eq!((fields.index, fields.generation), (5, 2)); + assert!( + !m.release(report(receipts[5])), + "the old generation is inert" + ); + assert!(m.release(report(again))); + } + + /// A heap that could not hold the provision leaves the table empty, and + /// the first supplies grow it. + #[test] + fn an_unprovisioned_table_grows_from_nothing() { + let mut m = SupplyMeter::with_table(BASE, 0, Reservations::new(0, 4)); + assert!(m.reservations.states.is_empty()); + for _ in 0..4 { + m.reserve(0, PAGE_BYTES, 0).unwrap(); + } + assert_eq!(m.reserve(0, PAGE_BYTES, 0), Err(MmRefusal::Full)); + } + + /// A slot taken as many times as its generation can count is retired for + /// good: no receipt ever names two reservations. + #[test] + fn a_slot_whose_generations_are_spent_is_retired() { + let mut m = fixed(1, 0); + for generation in 1..=RECEIPT_GENERATION_MAX { + let (_, _, receipt) = m.reserve(0, PAGE_BYTES, 0).unwrap(); + assert_eq!(Receipt::unpack(receipt).generation, generation); + assert!(m.release(SupplyReleaseReport { + receipt, + pair: 0, + len: PAGE_BYTES, + })); + } + assert_eq!(m.reserve(0, PAGE_BYTES, 0), Err(MmRefusal::Full)); + } + + /// The largest base supply a receipt names is admitted and round-trips; + /// a page more is malformed whatever the quota, and the huge classes + /// still take far more in their own unit. + #[test] + fn a_base_supply_no_receipt_can_name_is_malformed() { + let mut m = + SupplyMeter::with_table(u64::MAX, HUGE_SUPPLY_QUOTA_MAX, Reservations::new(4, 4)); + let (len, _, receipt) = m.reserve(0, MAX_BASE_SUPPLY_BYTES, 0).unwrap(); + assert_eq!(len, MAX_BASE_SUPPLY_BYTES); + assert_eq!(Receipt::unpack(receipt).units, RECEIPT_UNITS_MASK); + assert_eq!( + m.reserve(0, MAX_BASE_SUPPLY_BYTES + 1, 0), + Err(MmRefusal::BadProps) + ); + assert!(m.reserve(0, 8 << 30, 0x4000_0000).is_ok()); + } +} diff --git a/native/services/mm/src/wire.rs b/native/services/mm/src/wire.rs new file mode 100644 index 0000000..4a5ef53 --- /dev/null +++ b/native/services/mm/src/wire.rs @@ -0,0 +1,563 @@ +//! `wire` — the mm enum ⇄ wire codecs (`res/USERLAND.md` §5.3). +//! +//! The zerocopy wire forms live in [`kore_abi::svc`] (the frozen +//! contract); this module is the convenience bridge: encode is total, +//! decode is **fail-closed** — an out-of-vocabulary op, refusal, DMA +//! class, or attribute pattern decodes to `None`, never a guess. The +//! [`SvcProtocol`] impl is what plugs [`MemoryManager`] into the generic +//! request path (rings + doorbells + grant channels). + +use kore_abi::mem::{AllocProps, DmaClass, MemAttrs, NumaNode}; +use kore_abi::session::SlotRef; +use kore_abi::svc::{MmRepWire, MmReqWire}; + +use kore_rt::svc::{PeerBundlePart, PeerId, SvcProtocol}; + +use crate::service::{MemoryManager, MmRefusal, MmRequest, MmResponse}; + +/// Lower a request to its wire form (total). +pub fn encode_req(req: &MmRequest) -> MmReqWire { + match *req { + MmRequest::Alloc(p) => MmReqWire { + op: MmReqWire::OP_ALLOC, + contiguous: p.contiguous as u8, + numa: p.numa.0.into(), + slot: 0.into(), + size: p.size.into(), + align: p.align.into(), + attrs: p.attrs.bits().into(), + dma: p.dma as u8, + reserved: [0; 5], + }, + MmRequest::Release { slot } => MmReqWire { + op: MmReqWire::OP_RELEASE, + contiguous: 0, + numa: 0.into(), + slot: slot.index().into(), + size: 0.into(), + align: 0.into(), + attrs: 0.into(), + dma: 0, + reserved: [0; 5], + }, + MmRequest::Supply { size, align } => MmReqWire { + op: MmReqWire::OP_SUPPLY, + contiguous: 0, + numa: 0.into(), + slot: 0.into(), + size: size.into(), + align: align.into(), + attrs: 0.into(), + dma: 0, + reserved: [0; 5], + }, + MmRequest::Stats => bare(MmReqWire::OP_STATS), + MmRequest::AdmitPeer { part, announced } => MmReqWire { + contiguous: encode_part(part), + slot: announced.index().into(), + ..bare(MmReqWire::OP_ADMIT_PEER) + }, + MmRequest::TakeAdmission => bare(MmReqWire::OP_TAKE_ADMISSION), + MmRequest::RetirePeer { peer } => MmReqWire { + numa: peer.arrival().into(), + ..bare(MmReqWire::OP_RETIRE_PEER) + }, + MmRequest::ConfirmRetired { peer } => MmReqWire { + numa: peer.arrival().into(), + ..bare(MmReqWire::OP_CONFIRM_RETIRED) + }, + MmRequest::ClosePeer => bare(MmReqWire::OP_CLOSE_PEER), + } +} + +/// An `op` with every argument field zero — the shape each administration +/// request starts from, and exactly `OP_STATS`. +fn bare(op: u8) -> MmReqWire { + MmReqWire { + op, + contiguous: 0, + numa: 0.into(), + slot: 0.into(), + size: 0.into(), + align: 0.into(), + attrs: 0.into(), + dma: 0, + reserved: [0; 5], + } +} + +/// The recyclable-peer part codes every face protocol shares (power's and +/// accel's wires use the same four). +const fn encode_part(part: PeerBundlePart) -> u8 { + match part { + PeerBundlePart::Ring => 0, + PeerBundlePart::Kick => 1, + PeerBundlePart::ReturnRecv => 2, + PeerBundlePart::Grant(index) => 0x80 | index, + } +} + +fn decode_part(code: u8) -> Option { + Some(match code { + 0 => PeerBundlePart::Ring, + 1 => PeerBundlePart::Kick, + 2 => PeerBundlePart::ReturnRecv, + value if value & 0x80 != 0 => PeerBundlePart::Grant(value & 0x7f), + _ => return None, + }) +} + +/// Lift a wire request — fail-closed. +pub fn decode_req(w: &MmReqWire) -> Option { + if w.reserved != [0; 5] { + return None; + } + match w.op { + MmReqWire::OP_ALLOC if w.slot.get() == 0 => Some(MmRequest::Alloc(AllocProps { + size: w.size.get(), + align: w.align.get(), + numa: NumaNode(w.numa.get()), + dma: decode_dma(w.dma)?, + attrs: MemAttrs::from_bits(w.attrs.get())?, + contiguous: match w.contiguous { + 0 => false, + 1 => true, + _ => return None, + }, + })), + MmReqWire::OP_RELEASE + if w.contiguous == 0 + && w.numa.get() == 0 + && w.size.get() == 0 + && w.align.get() == 0 + && w.attrs.get() == 0 + && w.dma == 0 => + { + Some(MmRequest::Release { + slot: SlotRef::from_wire(w.slot.get()), + }) + } + // A supply is CPU write-back R/W by nature: `Alloc`'s placement + // vocabulary does not apply, and a set bit refuses rather than + // acquiring accidental meaning (the reserved-byte discipline). + MmReqWire::OP_SUPPLY + if w.contiguous == 0 + && w.numa.get() == 0 + && w.slot.get() == 0 + && w.attrs.get() == 0 + && w.dma == 0 => + { + Some(MmRequest::Supply { + size: w.size.get(), + align: w.align.get(), + }) + } + MmReqWire::OP_STATS if *w == bare(MmReqWire::OP_STATS) => Some(MmRequest::Stats), + MmReqWire::OP_ADMIT_PEER + if MmReqWire { + contiguous: 0, + slot: 0.into(), + ..*w + } == bare(MmReqWire::OP_ADMIT_PEER) => + { + Some(MmRequest::AdmitPeer { + part: decode_part(w.contiguous)?, + announced: SlotRef::from_wire(w.slot.get()), + }) + } + MmReqWire::OP_TAKE_ADMISSION if *w == bare(MmReqWire::OP_TAKE_ADMISSION) => { + Some(MmRequest::TakeAdmission) + } + MmReqWire::OP_RETIRE_PEER | MmReqWire::OP_CONFIRM_RETIRED + if MmReqWire { + numa: 0.into(), + ..*w + } == bare(w.op) => + { + let peer = PeerId::from_arrival(w.numa.get())?; + Some(if w.op == MmReqWire::OP_RETIRE_PEER { + MmRequest::RetirePeer { peer } + } else { + MmRequest::ConfirmRetired { peer } + }) + } + MmReqWire::OP_CLOSE_PEER if *w == bare(MmReqWire::OP_CLOSE_PEER) => { + Some(MmRequest::ClosePeer) + } + _ => None, + } +} + +/// Lower a response + the effected grant count (total). +pub fn encode_rep(rep: &MmResponse, grants: u8) -> MmRepWire { + let (tag, refusal, memory) = match rep { + MmResponse::Granted => ( + MmRepWire::TAG_GRANTED, + 0, + kore_abi::monitor::MemorySnapshot::EMPTY, + ), + MmResponse::Released => ( + MmRepWire::TAG_RELEASED, + 0, + kore_abi::monitor::MemorySnapshot::EMPTY, + ), + MmResponse::Stats(memory) => (MmRepWire::TAG_STATS, 0, *memory), + MmResponse::Accepted => ( + MmRepWire::TAG_ACCEPTED, + 0, + kore_abi::monitor::MemorySnapshot::EMPTY, + ), + MmResponse::PeerAdmitted { .. } => ( + MmRepWire::TAG_PEER_ADMITTED, + 0, + kore_abi::monitor::MemorySnapshot::EMPTY, + ), + MmResponse::Refused(r) => ( + MmRepWire::TAG_REFUSED, + match r { + MmRefusal::BadProps => MmRepWire::REFUSE_BAD_PROPS, + MmRefusal::Exhausted => MmRepWire::REFUSE_EXHAUSTED, + MmRefusal::BadReturn => MmRepWire::REFUSE_BAD_RETURN, + MmRefusal::Denied => MmRepWire::REFUSE_DENIED, + MmRefusal::Quota => MmRepWire::REFUSE_QUOTA, + MmRefusal::Pending => MmRepWire::REFUSE_PENDING, + MmRefusal::Full => MmRepWire::REFUSE_FULL, + }, + kore_abi::monitor::MemorySnapshot::EMPTY, + ), + }; + let mut reserved = [0; 5]; + if let MmResponse::PeerAdmitted { peer } = rep { + reserved[..2].copy_from_slice(&peer.arrival().to_le_bytes()); + } + MmRepWire { + tag, + refusal, + grants, + reserved, + memory, + } +} + +/// Lift a wire reply + its grant count — fail-closed. +pub fn decode_rep(w: &MmRepWire) -> Option<(MmResponse, u8)> { + let admitted = w.tag == MmRepWire::TAG_PEER_ADMITTED; + if w.reserved[if admitted { 2 } else { 0 }..] != [0; 5][if admitted { 2 } else { 0 }..] { + return None; + } + let rep = match (w.tag, w.refusal) { + (MmRepWire::TAG_ACCEPTED, 0) + if w.grants == 0 && w.memory == kore_abi::monitor::MemorySnapshot::EMPTY => + { + MmResponse::Accepted + } + (MmRepWire::TAG_PEER_ADMITTED, 0) + if w.grants == 0 && w.memory == kore_abi::monitor::MemorySnapshot::EMPTY => + { + MmResponse::PeerAdmitted { + peer: PeerId::from_arrival(u16::from_le_bytes([w.reserved[0], w.reserved[1]]))?, + } + } + (MmRepWire::TAG_GRANTED, 0) + if w.grants == 1 && w.memory == kore_abi::monitor::MemorySnapshot::EMPTY => + { + MmResponse::Granted + } + (MmRepWire::TAG_RELEASED, 0) + if w.grants == 0 && w.memory == kore_abi::monitor::MemorySnapshot::EMPTY => + { + MmResponse::Released + } + (MmRepWire::TAG_STATS, 0) + if w.grants == 0 && w.memory.validate() && w.memory.available() => + { + MmResponse::Stats(w.memory) + } + (MmRepWire::TAG_REFUSED, r) + if w.grants == 0 && w.memory == kore_abi::monitor::MemorySnapshot::EMPTY => + { + MmResponse::Refused(match r { + MmRepWire::REFUSE_BAD_PROPS => MmRefusal::BadProps, + MmRepWire::REFUSE_EXHAUSTED => MmRefusal::Exhausted, + MmRepWire::REFUSE_BAD_RETURN => MmRefusal::BadReturn, + MmRepWire::REFUSE_DENIED => MmRefusal::Denied, + MmRepWire::REFUSE_QUOTA => MmRefusal::Quota, + MmRepWire::REFUSE_PENDING => MmRefusal::Pending, + MmRepWire::REFUSE_FULL => MmRefusal::Full, + _ => return None, + }) + } + _ => return None, + }; + Some((rep, w.grants)) +} + +fn decode_dma(b: u8) -> Option { + Some(match b { + x if x == DmaClass::None as u8 => DmaClass::None, + x if x == DmaClass::Dma32 as u8 => DmaClass::Dma32, + x if x == DmaClass::Dma64 as u8 => DmaClass::Dma64, + _ => return None, + }) +} + +impl SvcProtocol for MemoryManager { + type Request = MmRequest; + type Response = MmResponse; + type ReqWire = MmReqWire; + type RepWire = MmRepWire; + + /// One grant-carrying transfer per conforming reply: the `Alloc` + /// extent view or the `Supply` lease root (`Release`/refusals carry + /// none). + const REPLY_GRANTS: usize = 1; + + /// Four serving pairs (slice D): pair 0 the founder-era protocol + /// pair, pair 1 the FIRST SHELL's supply face (the `shell_client` + /// prefab, conveyed through the pre-land bundle — where `run`'s + /// `OP_SUPPLY` arrives), pair 2 INIT's face (delivered at its + /// founding round; quiescent until the respawn slices), and pair 3 + /// exec's face — its provider snapshots and the **administration** of + /// per-launch faces, whose components arrive on its return leg (the + /// only fixed pair with one). Pairs past these are per-launch faces + /// (recyclable peers, one per engine launch), each metered on its own + /// lane. Supply quota is reclaimed through the separate + /// kernel-authenticated lineage-completion reports, and the byte limit + /// comes from the founder's validated boot policy. + const SERVE_PAIRS: usize = 4; + + /// Both grant flavors outlive the call (an `Alloc` view backs a heap + /// until released; a `Supply` root is conveyed onward at a spawn), so + /// the channel is **retained**: the runtime rotates an owned landing + /// per delivery and the client takes possession through the typed + /// claims ([`RingMm`](crate::client::RingMm)). Deliberately untyped + /// (`grant_policy` stays `None`): two kinds lawfully cross — + /// `LeasedRegion` views and `MemLease` roots — and each taker + /// kind-checks at possession. + fn retain_grant(index: usize) -> bool { + index == 0 + } + + fn reply_matches(request: &MmReqWire, response: &MmResponse, grants: u8) -> bool { + match (request.op, response) { + (MmReqWire::OP_ALLOC | MmReqWire::OP_SUPPLY, MmResponse::Granted) => grants == 1, + (MmReqWire::OP_ALLOC | MmReqWire::OP_SUPPLY, MmResponse::Refused(_)) + | (MmReqWire::OP_RELEASE, MmResponse::Released) + | (MmReqWire::OP_RELEASE, MmResponse::Refused(_)) + | (MmReqWire::OP_STATS, MmResponse::Stats(_) | MmResponse::Refused(_)) + | (MmReqWire::OP_TAKE_ADMISSION, MmResponse::PeerAdmitted { .. }) + | ( + MmReqWire::OP_ADMIT_PEER + | MmReqWire::OP_TAKE_ADMISSION + | MmReqWire::OP_RETIRE_PEER + | MmReqWire::OP_CONFIRM_RETIRED + | MmReqWire::OP_CLOSE_PEER, + MmResponse::Refused(_), + ) + | ( + MmReqWire::OP_ADMIT_PEER + | MmReqWire::OP_RETIRE_PEER + | MmReqWire::OP_CONFIRM_RETIRED + | MmReqWire::OP_CLOSE_PEER, + MmResponse::Accepted, + ) => grants == 0, + _ => false, + } + } + + fn encode_req(req: &MmRequest) -> MmReqWire { + encode_req(req) + } + fn decode_req(wire: &MmReqWire) -> Option { + decode_req(wire) + } + fn encode_rep(rep: &MmResponse, grants: u8) -> MmRepWire { + encode_rep(rep, grants) + } + fn decode_rep(wire: &MmRepWire) -> Option<(MmResponse, u8)> { + decode_rep(wire) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn props() -> AllocProps { + AllocProps { + size: 12_288, + align: 8192, + numa: NumaNode(3), + dma: DmaClass::Dma32, + attrs: MemAttrs::WRITE_BACK | MemAttrs::READ | MemAttrs::WRITE, + contiguous: true, + } + } + + /// Every request/response round-trips through its wire form. + #[test] + fn round_trips() { + for req in [ + MmRequest::Alloc(props()), + MmRequest::Release { + slot: SlotRef::from_wire(9), + }, + MmRequest::Supply { + size: 0x10_0000, + align: 0, + }, + MmRequest::Stats, + ] { + let back = decode_req(&encode_req(&req)).expect("decodes"); + // MmRequest has no PartialEq (AllocProps doesn't) — compare + // through the wire form, which is exact bytes. + assert_eq!(encode_req(&back), encode_req(&req)); + } + for rep in [ + MmResponse::Granted, + MmResponse::Released, + MmResponse::Stats(kore_abi::monitor::MemorySnapshot::new( + 1, 2, 3, 4, 5, 6, 7, 8, + ).with_supply_budget(26 << 30, 24 << 30).unwrap()), + MmResponse::Refused(MmRefusal::BadProps), + MmResponse::Refused(MmRefusal::Exhausted), + MmResponse::Refused(MmRefusal::BadReturn), + MmResponse::Refused(MmRefusal::Denied), + MmResponse::Refused(MmRefusal::Quota), + MmResponse::Refused(MmRefusal::Pending), + MmResponse::Refused(MmRefusal::Full), + ] { + let grants = u8::from(matches!(rep, MmResponse::Granted)); + let (back, grants) = decode_rep(&encode_rep(&rep, grants)).expect("decodes"); + assert_eq!(back, rep); + assert_eq!(grants, u8::from(matches!(rep, MmResponse::Granted))); + } + } + + /// Decode is fail-closed: unknown op/tag/refusal/class bytes refuse. + #[test] + fn decode_fails_closed() { + let mut w = encode_req(&MmRequest::Alloc(props())); + w.op = 0; + assert!(decode_req(&w).is_none(), "zero op (unseeded) refuses"); + w.op = 77; + assert!(decode_req(&w).is_none()); + let mut w = encode_req(&MmRequest::Alloc(props())); + w.dma = 9; + assert!(decode_req(&w).is_none(), "unknown DMA class refuses"); + let mut w = encode_req(&MmRequest::Alloc(props())); + w.attrs = 0x8000.into(); + assert!(decode_req(&w).is_none(), "reserved attr bits refuse"); + let mut w = encode_req(&MmRequest::Alloc(props())); + w.contiguous = 2; + assert!(decode_req(&w).is_none(), "non-boolean contiguous refuses"); + let mut w = encode_req(&MmRequest::Alloc(props())); + w.slot = 1.into(); + assert!(decode_req(&w).is_none(), "an alloc cannot smuggle a slot"); + let mut w = encode_req(&MmRequest::Release { + slot: SlotRef::from_wire(9), + }); + w.size = 1.into(); + assert!(decode_req(&w).is_none(), "release has no size vocabulary"); + let mut w = encode_req(&MmRequest::Stats); + w.reserved[4] = 1; + assert!(decode_req(&w).is_none(), "reserved request bytes refuse"); + // A supply speaks only size/align: any placement byte refuses + // rather than acquiring accidental meaning. + for poke in [ + |w: &mut MmReqWire| w.contiguous = 1, + |w: &mut MmReqWire| w.numa = 2.into(), + |w: &mut MmReqWire| w.slot = 7.into(), + |w: &mut MmReqWire| w.attrs = 1.into(), + |w: &mut MmReqWire| w.dma = 1, + ] { + let mut w = encode_req(&MmRequest::Supply { + size: 0x10_0000, + align: 0x1000, + }); + poke(&mut w); + assert!( + decode_req(&w).is_none(), + "a placement byte on a supply refuses" + ); + } + + let mut r = encode_rep(&MmResponse::Granted, 1); + r.tag = 0; + assert!(decode_rep(&r).is_none()); + let mut r = encode_rep(&MmResponse::Refused(MmRefusal::BadProps), 0); + r.refusal = 99; + assert!(decode_rep(&r).is_none(), "unknown refusal refuses"); + r.refusal = MmRepWire::REFUSE_FULL + 1; + assert!( + decode_rep(&r).is_none(), + "the code past the last refusal refuses" + ); + let mut r = encode_rep(&MmResponse::Granted, 0); + r.refusal = 1; + assert!( + decode_rep(&r).is_none(), + "a refusal byte on a grant refuses" + ); + assert!( + decode_rep(&encode_rep(&MmResponse::Granted, 0)).is_none(), + "Granted requires exactly one delivered capability" + ); + assert!( + decode_rep(&encode_rep(&MmResponse::Released, 1)).is_none(), + "Released carries no grant" + ); + } + + /// Every administration request round-trips, and each rejects a stray + /// bit in a field its opcode does not carry. + #[test] + fn administration_requests_round_trip_and_refuse_stray_fields() { + let peer = PeerId::from_arrival(0x0105).expect("a generation-1 face"); + for req in [ + MmRequest::AdmitPeer { + part: PeerBundlePart::Ring, + announced: SlotRef::from_wire(41), + }, + MmRequest::AdmitPeer { + part: PeerBundlePart::Grant(0), + announced: SlotRef::from_wire(42), + }, + MmRequest::TakeAdmission, + MmRequest::RetirePeer { peer }, + MmRequest::ConfirmRetired { peer }, + MmRequest::ClosePeer, + ] { + let wire = encode_req(&req); + let back = decode_req(&wire).expect("canonical encoding decodes"); + assert_eq!(encode_req(&back), wire); + let mut stray = wire; + stray.size = 1.into(); + assert!(decode_req(&stray).is_none(), "{req:?} carries no size"); + } + // A fixed pair's index is not a face arrival. + let mut fixed = encode_req(&MmRequest::RetirePeer { peer }); + fixed.numa = 3.into(); + assert!(decode_req(&fixed).is_none()); + } + + #[test] + fn peer_admitted_carries_its_arrival_and_nothing_else() { + let peer = PeerId::from_arrival(0x0206).unwrap(); + let wire = encode_rep(&MmResponse::PeerAdmitted { peer }, 0); + assert_eq!( + decode_rep(&wire), + Some((MmResponse::PeerAdmitted { peer }, 0)) + ); + let mut stray = wire; + stray.reserved[4] = 1; + assert!(decode_rep(&stray).is_none()); + let mut accepted = encode_rep(&MmResponse::Accepted, 0); + assert_eq!(decode_rep(&accepted), Some((MmResponse::Accepted, 0))); + accepted.reserved[0] = 1; + assert!( + decode_rep(&accepted).is_none(), + "only PeerAdmitted carries bytes" + ); + } +} diff --git a/native/services/swap/Cargo.toml b/native/services/swap/Cargo.toml new file mode 100644 index 0000000..0fdae4a --- /dev/null +++ b/native/services/swap/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "kore-svc-swap" +version = "0.0.1" +edition = "2024" +description = "kore swap service: the SwapRequest protocol and default in-RAM zram backend." +publish = false + +[dependencies] +kore-abi = { path = "../../crates/abi" } +kore-rt = { path = "../../crates/rt" } +# PAGE_BYTES (the lessor's grant granularity) and, in tests, the real +# MemoryManager the lessor plays against. +kore-svc-mm = { path = "../mm" } +# The zram codec: pure-Rust LZ4, no_std in block mode (default-features +# off), the wide-word fast paths (the heavier-SIMD codecs — zstd — are +# C-bound and not no_std). Block mode only: the frame format (and std) +# stay off. +lz4_flex = { version = "0.11", default-features = false } + +[dev-dependencies] +kore-rt = { path = "../../crates/rt", features = ["std"] } diff --git a/native/services/swap/boot.toml b/native/services/swap/boot.toml new file mode 100644 index 0000000..b433dd4 --- /dev/null +++ b/native/services/swap/boot.toml @@ -0,0 +1,13 @@ +# services/swap/boot.toml — this service's boot-manifest row (the design +# lives in res/boot.toml's header). The row rides beside the crate it +# endows, redox-style; whether and WHEN it founds is the root manifest's +# `services` list — founding order and chain adjacency are topology, the +# root's to declare, never a row's. + +[service] +image = "kore-svc-swap" +endowment = "kore_svc_swap::ZramEndowment" +protocol = "kore_svc_swap::ZramSwap" +endows = ["MEM_SUPPLY@0"] +digest_x86_64 = "blake3:5f77376810266176274a0af803d6b16bb9d750d97780b171e5fba376fcbb58fc" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:b939b2b8bd153674b2bf76dfa83e0941cc3f16139c721c5271f6a75b6fea0874" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/swap/src/client.rs b/native/services/swap/src/client.rs new file mode 100644 index 0000000..9d14e5c --- /dev/null +++ b/native/services/swap/src/client.rs @@ -0,0 +1,73 @@ +//! `client` — the swap client seam and the pager doorbell. +//! +//! What a lessor issues [`SwapRequest`]s through ([`SwapClient`] — the +//! `res/USERLAND.md` §4 pinned async shape), how fault notices reach it +//! ([`PagerDoorbell`]), the **born-async ring client** ([`RingSwap`] — +//! the §5.6 ring + channel proxy over the generic +//! [`SvcClient`](kore_rt::svc::SvcClient)). +//! **No sync client is ever written** (§5.6): the lessor awaits. + +use kore_rt::svc::SvcClient; +use kore_rt::transport::Transport; + +use crate::service::{SwapRefusal, SwapRequest, SwapResponse, ZramSwap}; + +/// The **client** side of the swap protocol (`res/USERLAND.md` §4, +/// pinned): what a lessor issues a [`SwapRequest`] through and awaits a +/// [`SwapResponse`] back. The lessor holds `impl SwapClient` and never +/// names the concrete backend — so the disk case is a swap, not a +/// rewrite, and the ring transport is an impl, not a redesign. +#[allow(async_fn_in_trait)] // clients are generic over the impl (§4) +pub trait SwapClient { + /// Issue one request and await the reply. + async fn request(&mut self, req: SwapRequest) -> SwapResponse; +} + +/// One pager-doorbell wake, decoded (the pager fault wiring): the kernel consumed +/// the lessor's standing recv post and — for a fault — wrote a +/// [`Notice`](kore_abi::session::Notice) under +/// [`NOTICE_COOKIE_PAGER_FAULT`](kore_abi::session::NOTICE_COOKIE_PAGER_FAULT) +/// into the posted slot, the faulting VA as its payload. A plain +/// `Notice`, so the ordinary delivery classifier (`take_delivery` / +/// `take_notice`) yields it — the private-kind record it replaced was +/// invisible to that classifier and every fault wake was dropped. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PagerEvent { + /// A tenant faulted on the paged-out page containing `va` — restore it. + Fault { va: u64 }, + /// The recv post was reaped (`Status::Revoked` from the blocking + /// recv) — the channel or registration died; the loop exits clean. + Revoked, +} + +/// The lessor's blocking doorbell transport seam. +/// On real kore this is the `WAIT_DELIVERY` blocking recv on the +/// registered pager channel (post → park → decode the delivered fault +/// [`Notice`](kore_abi::session::Notice) by its reserved cookie → +/// re-post). +pub trait PagerDoorbell { + /// Block until the next doorbell wake; yield the decoded event. + fn recv(&mut self) -> PagerEvent; +} + +/// The **ring + channel proxy** (`res/USERLAND.md` §5.6): the swap +/// client over the pair's shared window — request pushed, doorbell +/// kicked, reply awaited on the reply ring. A swap reply carries no +/// grants (its [`REPLY_GRANTS`](kore_rt::svc::SvcProtocol::REPLY_GRANTS) +/// is zero), so no reply post is ever pinned. +/// +/// An **infrastructure** failure of the request path (torn ring, +/// unavailable transport) surfaces as [`SwapRefusal::Backend`] — +/// exactly the vocabulary's "the backing store failed the transfer; +/// the page is not stored / not restored" arm, so the lessor's error +/// handling stays one path. +pub struct RingSwap<'w, 't, T: Transport>(pub SvcClient<'w, 't, T, ZramSwap>); + +impl SwapClient for RingSwap<'_, '_, T> { + async fn request(&mut self, req: SwapRequest) -> SwapResponse { + match self.0.call(&req).await { + Ok(reply) => reply.response, + Err(_) => SwapResponse::Refused(SwapRefusal::Backend), + } + } +} diff --git a/native/services/swap/src/extent.rs b/native/services/swap/src/extent.rs new file mode 100644 index 0000000..cd4a3d4 --- /dev/null +++ b/native/services/swap/src/extent.rs @@ -0,0 +1,272 @@ +//! Bounded, generation-checked swap extent identities and page-slot ownership. +//! +//! The swap backends have different byte stores (packed compressed RAM versus +//! an NVMe partition), but they must agree on the lifecycle that a client can +//! observe. [`ExtentTable`] is that common engine: a store claims one +//! contiguous run of logical 4 KiB slots, loads are read-only, and only an +//! explicit release returns the run. The generation in [`SwapExtent`] makes a +//! stale handle refuse after the same start slot is reused. + +use alloc::vec::Vec; +use core::ops::Range; + +/// Maximum pages moved by one protocol operation (2 MiB at 4 KiB/page). +/// +/// This is a scheduling bound, not a model shape. It caps one synchronous +/// service turn and one rollback set while still admitting large residency +/// segments. Clients learn the effective bound from `Attached` and may use +/// any smaller extent, including non-power-of-two lengths. +pub const MAX_EXTENT_PAGES: u32 = 512; + +/// Opaque identity of one immutable stored extent. +/// +/// `token` is backend-private. Clients must return the complete value they +/// received from `Stored`; its exact page count accompanies every load and +/// release request and is authenticated against the backend's descriptor. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct SwapExtent(u32); + +impl SwapExtent { + /// Reconstruct a wire handle. Both halves are one-based, so zero and + /// half-zero wire values can never accidentally name live storage. + pub const fn from_raw(token: u32) -> Option { + // Low half is descriptor index + 1; high half is generation. Both + // are one-based, so malformed half-zero identities fail at decode + // instead of consuming a backend lookup. + if token as u16 == 0 || token >> 16 == 0 { + None + } else { + Some(Self(token)) + } + } + + /// Backend-private token, for the wire codec only. + pub const fn token(self) -> u32 { + self.0 + } +} + +const MAX_DESCRIPTORS: usize = u16::MAX as usize; + +#[derive(Clone, Copy, Default)] +struct Descriptor { + start: u32, + pages: u32, + generation: u16, + live: bool, +} + +/// A bounded next-fit allocator plus the generation metadata that authenticates +/// [`SwapExtent`] handles. +/// +/// Storage is reserved once at bind. Claim, validate, and release allocate +/// nothing; a successful next-fit claim inspects each page in the chosen run +/// once. The cursor makes sequential residency segments O(total pages) rather +/// than repeatedly scanning from slot zero. +pub struct ExtentTable { + capacity: u32, + claimed: Vec, + descriptors: Vec, + cursor: u32, + descriptor_cursor: usize, + live_extents: u32, + live_pages: u32, +} + +impl ExtentTable { + /// An empty table over `capacity` logical page slots. + pub fn new(capacity: u32) -> Self { + let words = (capacity as usize).div_ceil(64); + Self { + capacity, + claimed: alloc::vec![0; words], + descriptors: alloc::vec![Descriptor::default(); + (capacity as usize).min(MAX_DESCRIPTORS)], + cursor: 0, + descriptor_cursor: 0, + live_extents: 0, + live_pages: 0, + } + } + + /// Total logical page slots. + pub const fn capacity(&self) -> u32 { + self.capacity + } + + /// Currently live extent count. + pub const fn live_extents(&self) -> u32 { + self.live_extents + } + + /// Currently claimed page count. + pub const fn live_pages(&self) -> u32 { + self.live_pages + } + + /// Whether `page` belongs to any live extent. + pub fn is_claimed(&self, page: u32) -> bool { + page < self.capacity && self.claimed[page as usize / 64] & (1_u64 << (page % 64)) != 0 + } + + /// Claim one contiguous logical extent. No bytes become externally live + /// until the caller has populated the backing and retained this handle; + /// callers roll a failed store back with [`release`](Self::release). + pub fn claim(&mut self, pages: u32) -> Option { + if pages == 0 || pages > MAX_EXTENT_PAGES || pages > self.capacity { + return None; + } + let descriptor = self.find_descriptor()?; + let start = self + .find_run(self.cursor, self.capacity, pages) + .or_else(|| self.find_run(0, self.cursor, pages))?; + let generation = self.descriptors[descriptor].generation.checked_add(1)?; + let end = start + pages; + for page in start..end { + self.claimed[page as usize / 64] |= 1_u64 << (page % 64); + } + self.descriptors[descriptor] = Descriptor { + start, + pages, + generation, + live: true, + }; + self.cursor = if end == self.capacity { 0 } else { end }; + self.descriptor_cursor = (descriptor + 1) % self.descriptors.len(); + self.live_extents += 1; + self.live_pages += pages; + let token = (u32::from(generation) << 16) | (descriptor as u32 + 1); + SwapExtent::from_raw(token) + } + + /// Resolve a live handle to its logical slot range. + pub fn range(&self, extent: SwapExtent, pages: u32) -> Option> { + if pages == 0 || pages > MAX_EXTENT_PAGES { + return None; + } + let descriptor = usize::from((extent.0 as u16).checked_sub(1)?); + let generation = (extent.0 >> 16) as u16; + let entry = *self.descriptors.get(descriptor)?; + let end = entry.start.checked_add(entry.pages)?; + if generation == 0 + || !entry.live + || entry.generation != generation + || entry.pages != pages + || end > self.capacity + || (entry.start..end).any(|page| !self.is_claimed(page)) + { + return None; + } + Some(entry.start as usize..end as usize) + } + + /// Release exactly one live extent. The generation is deliberately kept; + /// its next reuse increments it, so the old handle can never name the new + /// bytes. + pub fn release(&mut self, extent: SwapExtent, pages: u32) -> Option> { + let range = self.range(extent, pages)?; + let descriptor = usize::from((extent.0 as u16).checked_sub(1)?); + for page in range.clone() { + self.claimed[page / 64] &= !(1_u64 << (page % 64)); + } + self.descriptors[descriptor].live = false; + self.live_extents -= 1; + self.live_pages -= pages; + Some(range) + } + + fn find_descriptor(&self) -> Option { + if self.descriptors.is_empty() { + return None; + } + (self.descriptor_cursor..self.descriptors.len()) + .chain(0..self.descriptor_cursor) + .find(|&index| { + let entry = self.descriptors[index]; + !entry.live && entry.generation != u16::MAX + }) + } + + fn find_run(&self, begin: u32, limit: u32, pages: u32) -> Option { + let mut start = begin; + while start.checked_add(pages).is_some_and(|end| end <= limit) { + let end = start + pages; + match (start..end).find(|&page| self.is_claimed(page)) { + Some(claimed) => start = claimed + 1, + None => return Some(start), + } + } + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stale_handle_refuses_after_exact_slot_reuse() { + let mut table = ExtentTable::new(8); + let first = table.claim(8).unwrap(); + assert_eq!(table.range(first, 8), Some(0..8)); + assert_eq!(table.release(first, 8), Some(0..8)); + let second = table.claim(8).unwrap(); + assert_eq!(table.range(second, 8), Some(0..8)); + assert_ne!(first.token(), second.token()); + assert_eq!(table.range(first, 8), None); + assert_eq!(table.release(first, 8), None); + } + + #[test] + fn next_fit_admits_non_power_of_two_extents_and_reuses_holes() { + let mut table = ExtentTable::new(1_024); + let a = table.claim(387).unwrap(); + let b = table.claim(512).unwrap(); + assert_eq!(table.range(a, 387), Some(0..387)); + assert_eq!(table.range(b, 512), Some(387..899)); + assert_eq!(table.live_extents(), 2); + assert_eq!(table.live_pages(), 899); + assert!(table.claim(126).is_none()); + table.release(a, 387).unwrap(); + let c = table.claim(126).unwrap(); + assert_eq!(table.range(c, 126), Some(0..126)); + } + + #[test] + fn malformed_and_oversized_claims_refuse() { + let mut table = ExtentTable::new(2_048); + assert!(table.claim(0).is_none()); + assert!(table.claim(MAX_EXTENT_PAGES + 1).is_none()); + assert!(SwapExtent::from_raw(0).is_none()); + assert!(SwapExtent::from_raw(1).is_none()); + assert!(SwapExtent::from_raw(1 << 16).is_none()); + } + + #[test] + fn exact_page_count_is_part_of_extent_identity() { + let mut table = ExtentTable::new(16); + let extent = table.claim(7).unwrap(); + assert_eq!(table.range(extent, 6), None); + assert_eq!(table.range(extent, 8), None); + assert_eq!(table.release(extent, 6), None); + assert_eq!(table.live_extents(), 1); + assert_eq!(table.live_pages(), 7); + assert_eq!(table.release(extent, 7), Some(0..7)); + } + + /// The illustrative model-neutral 1M geometry from Axiom is 792,576 + /// backing frames split into 2,048 extents of 387 frames. Admission is + /// exact and does not depend on a model identifier. + #[test] + fn one_million_context_frame_geometry_fits_the_bounded_extent_table() { + const FRAMES: u32 = 792_576; + const EXTENT_PAGES: u32 = 387; + let mut table = ExtentTable::new(FRAMES); + for _ in 0..2_048 { + assert!(table.claim(EXTENT_PAGES).is_some()); + } + assert_eq!(table.live_extents(), 2_048); + assert_eq!(table.live_pages(), FRAMES); + assert!(table.claim(1).is_none()); + } +} diff --git a/native/services/swap/src/lib.rs b/native/services/swap/src/lib.rs new file mode 100644 index 0000000..4fc50b8 --- /dev/null +++ b/native/services/swap/src/lib.rs @@ -0,0 +1,72 @@ +//! # kore-svc-swap — the swap service +//! +//! **The `SwapBackend` seam, rewritten as a service**: +//! [`SwapRequest`] is the vocabulary, and *implementations are services* — +//! [`ZramSwap`] (compress into RAM) now, a disk driver implementing the +//! same `SystemService` later (`kore-nvme`'s +//! `service` feature is exactly that face). +//! +//! **The lessor inversion.** Swap is not a kernel subsystem: residency — +//! *where* an evicted page went (a compressed-buffer id, a block LBA) — +//! is the lessor's private bookkeeping, and the swap token space is the +//! *server's*; the kernel never interprets a token and holds no swap +//! table (its whole footprint is the paged-out PTE marker, the fault +//! routing in `src/runtime/mm/pager.rs`, and the pager session verbs). +//! +//! **The seal + rollback resistance.** An evicted page transits +//! untrusted userland, so the kernel seals it in place +//! (ChaCha20-Poly1305 AEAD, `src/schema/seal.rs`; the key derives from +//! the entropy-seeded token master, domain-separated from every +//! capability epoch key, never exposed): this service stores only +//! ciphertext and can neither read a tenant's plaintext nor feed back +//! tampered bytes. The `(nonce, tag)` authenticator is *authentication* +//! state, not residency — and the kernel holding the **latest** tag per +//! frame is what buys **rollback resistance**: a backend replaying a +//! stale but validly-sealed ciphertext fails the check. (The recorded +//! fork — zero kernel state, authenticators stored beside the +//! ciphertext — costs exactly the rollback property.) +//! +//! One concern per module (`res/USERLAND.md` §1.2 rule 4; `main.rs` — +//! the process entry — arrives with the founder's spawn, #96/#298): +//! +//! - [`service`] — the default in-RAM (zram) backend: LZ4 + same-fill +//! SIMD fast path, budget-accounted slab packing. +//! - [`client`] — the client seam: the async [`SwapClient`] (§4 pinned), +//! the [`RingSwap`] ring + channel proxy, and the pager doorbell. +//! - [`wire`] — the fail-closed enum ⇄ wire codecs over +//! [`kore_abi::svc`]'s frozen forms (§5.3). +//! +//! The former `SwapRegion` lessor façade is intentionally absent: it +//! depended on never-wired `view_of_window`/`deliver_view` verbs and a +//! hard-coded ambient holder. The next demand-paging client will attach one +//! staging window ([`SwapRequest::Attach`]) and move bounded immutable extents +//! through it: store is all-or-nothing, load is non-destructive, and release +//! is explicit. No pager or residency consumer is founded yet; the service +//! contract is deliberately model-neutral and does not interpret KV geometry. +//! +//! The zram engine's codec, accounting, and **datapath** are host-tested +//! end to end (the staging window is ordinary memory, so a round-trip is +//! an ordinary test). On metal the store bytes come from the endowed +//! supply itself, carved and mapped at bind. + +#![no_std] +// Typed portable SIMD for the zram same-fill scan/fill (service.rs) — the +// doctrine's "typed over hand-rolled asm" for the vector fast path. +#![feature(portable_simd)] + +extern crate alloc; +#[cfg(test)] +extern crate std; + +pub mod client; +pub mod extent; +pub mod service; +pub mod wire; + +pub use client::{PagerDoorbell, PagerEvent, RingSwap, SwapClient}; +pub use extent::{ExtentTable, MAX_EXTENT_PAGES, SwapExtent}; +pub use service::{ + DISK_FACTOR, SwapRefusal, SwapRequest, SwapResponse, SwapStatus, ZramEndowment, ZramStats, + ZramSwap, adopt_staging, staging_extent, staging_extent_mut, staging_page, staging_page_mut, + staging_pages, +}; diff --git a/native/services/swap/src/service/mod.rs b/native/services/swap/src/service/mod.rs new file mode 100644 index 0000000..276e42a --- /dev/null +++ b/native/services/swap/src/service/mod.rs @@ -0,0 +1,1401 @@ +//! # `service` — the swap-service protocol + the default in-RAM (zram) backend +//! +//! **The `SwapBackend` seam, rewritten as a service**: [`SwapRequest`] is +//! the vocabulary, and *implementations are services* — [`ZramSwap`] (this +//! module: compress into RAM) now, a disk driver implementing the same +//! `SystemService` later. The refactor is what +//! makes the disk case natural: `SWAP_OUT`/`SWAP_IN` are already kore-band +//! uring opcodes ([`kore_abi::uring`]), the ring's CQ completion is the await +//! point, and the #206 tower veneer wraps the service for free. kore-ring's +//! `SwapBackend` trait retires when the ring adapter lands. +//! +//! **How page bytes move** (#148: no capability rides request data): the +//! client attaches a **staging window** once ([`SwapRequest::Attach`], a +//! capability transferred over the return channel and mapped for the +//! service's life), and every later byte-moving request names a bounded span +//! inside it. The preferred [`Store`](SwapRequest::Store) / +//! [`Load`](SwapRequest::Load) / [`Release`](SwapRequest::Release) lifecycle +//! moves up to [`MAX_EXTENT_PAGES`] pages under one immutable identity: +//! stores publish all-or-nothing, loads are non-destructive, and only explicit +//! release retires the copy. The original single-page `Out` / `In` / +//! `Discard` verbs remain compatibility shims over the same engine. The +//! reply's token is **data** — the server's private map key (the SwapBackend +//! contract: the kernel never interprets it) — never a capability. +//! +//! The shape it replaces delivered a fresh view per operation, which made +//! the backend map a view per page — and a receiver's map is one-way (no +//! unmap verb exists), so at 4 KiB against a 16 MiB per-thread stride the +//! store could serve ~4096 pages and then never map again. One window, +//! attached once, also makes the whole datapath ordinary memory: the +//! round-trip tests below could not exist under the old shape, because a +//! host build can possess no delivered view at all. +//! +//! **The store budget is possession-shaped in the host model**: the +//! endowment is a [`SupplyToken`] whose [`len`](SupplyToken::len) is the +//! ceiling — no config strings. A store that would exceed it refuses +//! [`StoreFull`](SwapRefusal::StoreFull); an incompressible page stores +//! **raw** (the zram huge-page class), so one stored page costs at most +//! [`PAGE_BYTES`] and the books never lie about the win +//! ([`ZramStats::saved_bytes`]). +//! +//! **Every capacity is relative to that endowment** — the second ceiling, +//! the page-count table (Linux zram's *disksize*), is sized at bind as +//! [`DISK_FACTOR`]` × budget` worth of pages: endow a bigger store, get a +//! proportionally bigger swap, no absolute anywhere. +//! +//! **Zero grants, ever**: a swap service consumes delivered views and +//! returns data — it can mint no authority (contrast mm: the lessor +//! grants; the swapper only holds). Asserted by every test below. +//! +//! The codec is [`lz4_flex`] block mode — pure Rust, `no_std`, the +//! wide-word fast paths. A compressing swapper is also the first real +//! userland customer for the FPU capability: vector-heavy +//! by nature, it runs under an `FpuTicket` when the ring-3 tier lands. +//! +//! **Metal posture (the 2026-07-27 completion):** compressed bytes live +//! in the **endowed store itself** — `bind` carves `[0, budget)` off the +//! possessed supply and maps it (one ordinary session, no heap), and the +//! packed arena is that mapped span. No compressed byte ever touches an +//! unrelated heap; the token table and scratch are ordinary service +//! working memory (every service heap-grows through its `role::HEAP`). + +use alloc::vec::Vec; + +use kore_abi::session::SlotRef; +use kore_abi::task::role; + +use kore_rt::cap::{SupplyToken, verbs}; +use kore_rt::service::{Serve, SystemService}; + +use kore_svc_mm::PAGE_BYTES; + +use crate::extent::{ExtentTable, MAX_EXTENT_PAGES, SwapExtent}; + +/// Advertised-capacity factor: a bound service's page-count ceiling +/// ([`ZramStats::disk_pages`]) is `DISK_FACTOR × budget` worth of pages — +/// the Linux zram shape, where *disksize* (the uncompressed ceiling) is +/// conventionally ~2× the RAM the store may consume. Filling it needs +/// average `DISK_FACTOR×` compression — the LZ4 norm on swappable pages, +/// and same-fill pages cost nothing at all. +pub const DISK_FACTOR: u64 = 2; + +/// The swap service's role bundle: its compressed-store **budget** — a +/// possessed supply whose length is the ceiling. +#[derive(Debug, kore_rt::Bundle)] +#[bundle(name = "zram")] +pub struct ZramEndowment { + /// The claimed store-budget token. + #[bundle(role = role::MEM_SUPPLY)] + pub store: SupplyToken, +} + +/// The swap vocabulary — what *every* swap backend serves (zram and the NVMe +/// service today). One enum, one intake; the ring muxes. +#[derive(Clone, Copy, Debug)] +pub enum SwapRequest { + /// **Adopt the staging window** transferred over the return channel — + /// one shared mapping, attached once, that every later byte-moving verb + /// addresses by page index and bounded page count. + /// + /// `announced` is the founder-published return-landing correlation + /// value (the [`Serve::returned_input`](kore_rt::service::Serve::returned_input) + /// idiom the image service's `Register` uses): it names *which* + /// transfer this request adopts, and carries no authority itself. + /// + /// Re-attaching replaces the window — the previous mapping stays + /// installed (a receiver cannot unmap), so a client that re-attaches + /// repeatedly is spending the backend's map cursor. Attach once. + Attach { + /// The return-landing correlation value naming the transfer. + announced: SlotRef, + }, + /// Compatibility form of [`Store`](Self::Store) for one page. Persist + /// staging page `page`; reply with the restore token. + Out { + /// Staging page index. + page: u32, + }, + /// Compatibility consuming load of one page. Prefer + /// [`Load`](Self::Load) followed by [`Release`](Self::Release), which + /// keeps retry and cancellation separate from ownership retirement. + In { + /// The token [`SwapResponse::Stored`] returned. + extent: SwapExtent, + /// Staging page index to restore into. + page: u32, + }, + /// Compatibility form of [`Release`](Self::Release) for one page. + Discard { + /// The token to retire. + extent: SwapExtent, + }, + /// Persist a bounded contiguous run of staging pages as one immutable + /// lifecycle object. Success is all-or-nothing. + Store { + /// First staging page. + page: u32, + /// Exact number of pages, in `1..=MAX_EXTENT_PAGES`. + pages: u32, + }, + /// Load a stored extent into staging without consuming the stored copy. + Load { + /// Opaque identity returned by [`SwapResponse::Stored`]. + extent: SwapExtent, + /// First destination page in staging. + page: u32, + /// Exact extent length; authenticated against the stored descriptor. + pages: u32, + }, + /// Retire one stored extent and return its backing capacity. + Release { + /// Opaque identity returned by [`SwapResponse::Stored`]. + extent: SwapExtent, + /// Exact extent length; authenticated against the stored descriptor. + pages: u32, + }, + /// Read the tier's books ([`SwapStatus`]) — the control-plane verb a + /// status client polls. Carries nothing, moves nothing, and is + /// answerable in every posture (a disk row with no controller + /// answers with its `device` bit clear rather than refusing). + Status, +} + +/// The swap replies — data only (the token is the server's map key, +/// never a capability). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SwapResponse { + /// Persisted; load with this opaque extent identity. + Stored { + /// The server's generation-protected private identity. + extent: SwapExtent, + }, + /// The staging window is live; `pages` is how many pages it holds + /// (the client's own bound, echoed back kernel-vouched). + Attached { + /// Staging capacity, in pages. + pages: u32, + }, + /// A compatibility `In` restored one page and consumed its stored copy. + Restored, + /// The stored copy is dropped and its budget freed. + Discarded, + /// The staging span contains the requested bytes; the stored copy remains. + Loaded, + /// The extent identity is retired and its capacity is reusable. + Released, + /// The tier's books — [`SwapRequest::Status`]'s answer. + Status(SwapStatus), + /// Refused — nothing stored, nothing lost, no authority moved. + Refused(SwapRefusal), +} + +/// Why the swapper refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SwapRefusal { + /// The staging source page could not be read (a malformed window). + BadPage, + /// The staging destination page could not be written. + BadDst, + /// No page stored under that token. + BadToken, + /// Budget or table exhausted — evict elsewhere or grow the endowment. + StoreFull, + /// The backing store failed the transfer (a disk backend's I/O error + /// — the page is not stored / not restored; retry or evict elsewhere). + Backend, + /// No staging window is attached, or the page index lies outside the + /// attached one — the client must `Attach` before moving bytes, and + /// must address only pages the window holds. + NoStaging, +} + +/// **One swap tier's books** — [`SwapRequest::Status`]'s typed answer, +/// shared by every backend the way the request vocabulary is (one wire +/// form, N services). The union of both tiers' summaries: the zram tier +/// reports no queue and no route, so its io/msix words stay zero with +/// [`device`](Self::device)/[`route`](Self::route) saying so; a disk row +/// with no controller answers all-zero with `device` clear — the honest +/// posture, never a refusal. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SwapStatus { + /// A backing device is bound (the disk tier's controller; always + /// `true` for the zram tier, whose store is RAM). + pub device: bool, + /// The completion path holds its delivered interrupt route — + /// [`fires`](Self::fires)/[`parks`](Self::parks) are live. `false` + /// = the polled fallback (or a route-less tier). + pub route: bool, + /// Store capacity, in pages (zram: its `disk_pages` budget). + pub capacity_pages: u32, + /// Pages currently stored. + pub stored_pages: u32, + /// Immutable extents currently stored. + pub stored_extents: u32, + /// Attached staging-window pages (`0` = unattached). + pub staging_pages: u32, + /// Successful store operations (`Store` and compatibility `Out`). + pub outs: u64, + /// Successful load operations (`Load` and compatibility `In`). + pub ins: u64, + /// Successful releases (`Release` and compatibility `Discard`). + pub discards: u64, + /// Device commands submitted (disk tier; zram: zero). + pub io_submitted: u64, + /// Device commands completed. + pub io_completed: u64, + /// Device commands that completed in error. + pub io_errors: u64, + /// Completion parks resolved as delivered route fires — the MSI-X + /// delivery witness (`0` beside growing `parks` = delivery-dead). + pub fires: u64, + /// Completion parks issued on the held route. + pub parks: u64, + /// Route-LESS operations paced by a clock deadline — the posture on a + /// machine with no delivered vector but a readable clock. Bounded by + /// elapsed time, so an expiry is real evidence about the device. + pub deadline_waits: u64, + /// Route-less operations paced only by a count of spin hints, because no + /// clock could supply a deadline at all. + /// + /// Reported apart from [`deadline_waits`](Self::deadline_waits) because a + /// spin count bounds retired instructions rather than time: an expiry + /// there says the device was slower than N hints on this core under this + /// contention, which is not a statement about the device. Non-zero means + /// disk waits are being judged by something that is not a duration — + /// which on a serial-less box appears nowhere else. Zero for the zram + /// tier, whose store is RAM and waits on nothing. + pub spin_waits: u64, +} + +/// Occupancy snapshot — the compression win, visible. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ZramStats { + /// Pages currently stored (all classes). + pub stored_pages: u32, + /// Of those, pages in the **same-fill** class — stored at ~zero cost. + pub same_fill_pages: u32, + /// Page-count ceiling (the zram *disksize* analogue): the advertised + /// uncompressed capacity, [`DISK_FACTOR`]` × budget` worth of pages — + /// relative to the endowment by construction. + pub disk_pages: u32, + /// Arena bytes packed by the compressed / raw pages (same-fill costs 0). + pub used_bytes: u64, + /// The endowed ceiling. + pub budget_bytes: u64, + /// Bytes saved vs. storing every page raw + unpacked — compression + + /// same-fill + packing all fold into one visible win: `stored·PAGE − used`. + pub saved_bytes: u64, +} + +/// A stored page's residence. +#[derive(Clone, Copy, Debug)] +enum Slot { + /// A free token. + Vacant, + /// A **same-filled** page (zram's zero / same-element class): the whole + /// page is one repeated 8-byte element, stored as just the element — + /// **no arena bytes**. Zero pages are the common case. + SameFill { element: u64 }, + /// Compressed (or, at `len == PAGE_BYTES`, raw) bytes **packed** into the + /// store arena at `[off, off+len)`. + Packed { off: u32, len: u32 }, +} + +/// **Adopt the staging window a request transferred** — the metal half of +/// [`SwapRequest::Attach`], shared by every backend so the capability +/// handling has one implementation. +/// +/// The client transferred a writable view over the return channel +/// (`ReturnMode::Retained`); `announced` must name that landing, or the +/// caller gets nothing. The view is **mapped once and kept forever**: +/// the mapping is what makes byte moves ordinary memory accesses, and a +/// receiver cannot unmap, so the returned owner is deliberately leaked +/// into the service's lifetime rather than dropped (dropping it would +/// retire the record and leave the mapping pointing at reclaimed frames). +/// +/// `None` on any refusal — wrong correlation, nothing delivered, a +/// non-view record, or a mapping the kernel declined. +pub fn adopt_staging( + sv: &mut Serve<'_, SwapResponse>, + announced: SlotRef, +) -> Option<&'static mut [u8]> { + let mut input = sv.returned_input(announced)?; + let owner = input.take()?; + let slot = owner.slot()?; + let view = verbs::take_view(slot)?; + let window = verbs::map_view(view)?; + // The mapping must outlive every request, so the owner must never be + // dropped: its Drop retires the delivered record, and the window's + // frames would be free while this service still addressed them. + core::mem::forget(owner); + // SAFETY: `map_view` installed `[base, +len)` R/W-USER in THIS task's + // own space over the client's staging backing, page-aligned and live + // for the task's whole life (nothing unmaps it — a receiver has no + // unmap verb, and the owner above is retained). The client writes + // page bytes there and this service reads/writes them under the + // request/reply protocol, which is the same single-owner-at-a-time + // discipline the pair rings rely on. + Some(unsafe { core::slice::from_raw_parts_mut(window.cast::().as_ptr(), window.len()) }) +} + +/// How many whole pages a staging window holds — the client's own bound, +/// read off the mapping rather than trusted from the wire. +pub fn staging_pages(window: &[u8]) -> u32 { + (window.len() as u64 / PAGE_BYTES) as u32 +} + +/// Page `index` of a staging window — `None` when the index lies outside +/// it. The **bounds check every byte-moving verb goes through**: a page +/// index is client data, and the window's length is the only authority on +/// what it may name. +pub fn staging_page(window: &[u8], index: u32) -> Option<&[u8]> { + let off = (index as u64).checked_mul(PAGE_BYTES)?; + let end = off.checked_add(PAGE_BYTES)?; + if end > window.len() as u64 { + return None; + } + window.get(off as usize..end as usize) +} + +/// [`staging_page`]'s mutable twin (the restore destination). +pub fn staging_page_mut(window: &mut [u8], index: u32) -> Option<&mut [u8]> { + let off = (index as u64).checked_mul(PAGE_BYTES)?; + let end = off.checked_add(PAGE_BYTES)?; + if end > window.len() as u64 { + return None; + } + window.get_mut(off as usize..end as usize) +} + +/// A checked contiguous run of staging pages. +pub fn staging_extent(window: &[u8], first: u32, pages: u32) -> Option<&[u8]> { + if pages == 0 || pages > MAX_EXTENT_PAGES { + return None; + } + let off = u64::from(first).checked_mul(PAGE_BYTES)?; + let bytes = u64::from(pages).checked_mul(PAGE_BYTES)?; + let end = off.checked_add(bytes)?; + if end > window.len() as u64 { + return None; + } + window.get(off as usize..end as usize) +} + +/// [`staging_extent`]'s mutable twin. +pub fn staging_extent_mut(window: &mut [u8], first: u32, pages: u32) -> Option<&mut [u8]> { + if pages == 0 || pages > MAX_EXTENT_PAGES { + return None; + } + let off = u64::from(first).checked_mul(PAGE_BYTES)?; + let bytes = u64::from(pages).checked_mul(PAGE_BYTES)?; + let end = off.checked_add(bytes)?; + if end > window.len() as u64 { + return None; + } + window.get_mut(off as usize..end as usize) +} + +/// SIMD-scan a page frame for the same-fill class: `Some(element)` iff every +/// 8-byte word equals the first (a zram same-filled page — zeros, or any +/// repeated element). 512 words / 8-lane = 64 vector compares; the common +/// zero page exits on the first mismatch. +fn same_fill(page: &[u8]) -> Option { + use core::simd::{Simd, cmp::SimdPartialEq}; + debug_assert_eq!(page.len(), PAGE_BYTES as usize); + debug_assert_eq!(page.as_ptr() as usize % 8, 0, "page frame is u64-aligned"); + // SAFETY: a page frame is 4096-aligned ⟹ 8-aligned, and 4096 = 512·8. + let words = unsafe { core::slice::from_raw_parts(page.as_ptr() as *const u64, 512) }; + let first = Simd::::splat(words[0]); + for chunk in words.chunks_exact(8) { + if Simd::::from_slice(chunk).simd_ne(first).any() { + return None; + } + } + Some(words[0]) +} + +/// SIMD-fill a page frame with a repeated 8-byte `element` — the same-fill +/// restore (the inverse of [`same_fill`]). +fn simd_fill(out: &mut [u8], element: u64) { + use core::simd::Simd; + debug_assert_eq!(out.len(), PAGE_BYTES as usize); + debug_assert_eq!(out.as_ptr() as usize % 8, 0); + // SAFETY: as in `same_fill`. + let words = unsafe { core::slice::from_raw_parts_mut(out.as_ptr() as *mut u64, 512) }; + let splat = Simd::::splat(element); + for chunk in words.chunks_exact_mut(8) { + splat.copy_to_slice(chunk); + } +} + +/// The packed-byte arena's backing — where the compressed bytes lawfully +/// live. Off metal there is no trap to carve through, so the host keeps +/// an ordinary Vec (tests of the codec + accounting); on metal the arena +/// is the endowed store supply itself, carved + mapped once in `bind`. +struct Arena { + /// Host: an ordinary heap Vec, grown on demand to the budget ceiling. + #[cfg(not(target_os = "none"))] + host: Vec, + /// Metal: the store supply's `[0, budget)` window, carved + mapped in + /// `bind` (one session, no heap). `None` when the carve/map refused — + /// [`start_on_metal`](ZramSwap::start_on_metal) fails closed on it. + #[cfg(target_os = "none")] + mapped: Option>, +} + +impl Arena { + /// The host backing: an empty grow-on-demand Vec. + #[cfg(not(target_os = "none"))] + fn bind(_store: &SupplyToken) -> Self { + Self { host: Vec::new() } + } + + /// The metal backing: carve `[0, budget)` off the endowed store + /// supply and map it — one ordinary session, no heap (kore-rt's + /// [`carve_and_map`](kore_rt::heap::carve_and_map), the heap + /// grow's primitive off this service's OWN supply). `None` on any + /// refusal: `start_on_metal` fails closed. + #[cfg(target_os = "none")] + fn bind(store: &SupplyToken) -> Self { + Self { + mapped: kore_rt::heap::carve_and_map(store.slot(), 0, store.len()), + } + } + + /// Whether the backing is live — `start_on_metal`'s fail-closed gate. + fn live(&self) -> bool { + #[cfg(not(target_os = "none"))] + { + true + } + #[cfg(target_os = "none")] + { + self.mapped.is_some() + } + } + + /// Ensure the span covers `[0, n)` — the host grow; the metal span is + /// the full budget from `bind`, so this is a no-op there. + fn cover(&mut self, n: usize) { + #[cfg(not(target_os = "none"))] + { + if self.host.len() < n { + self.host.resize(n, 0); + } + } + #[cfg(target_os = "none")] + { + let _ = n; + } + } + + /// The arena bytes, mutably. Callers stay within the budget ceiling + /// ([`ZramSwap::arena_alloc`] enforces it). + fn bytes_mut(&mut self) -> &mut [u8] { + #[cfg(not(target_os = "none"))] + { + &mut self.host + } + #[cfg(target_os = "none")] + { + let mut p = self.mapped.expect("arena live (gated at start_on_metal)"); + // SAFETY: the mapped span is this task's own R/W carve of its + // endowed store, live for the task's life; offsets are + // budget-bounded by `arena_alloc`. + unsafe { p.as_mut() } + } + } + + /// The arena bytes, shared. + fn bytes(&self) -> &[u8] { + #[cfg(not(target_os = "none"))] + { + &self.host + } + #[cfg(target_os = "none")] + { + let p = self.mapped.expect("arena live (gated at start_on_metal)"); + // SAFETY: as `bytes_mut` — a read-only view of the same span. + unsafe { p.as_ref() } + } + } +} + +/// The default in-RAM swap backend: LZ4-compress evicted pages into a +/// **slab-packed**, budget-accounted store, with a **same-fill fast path** +/// (SIMD-scanned) that stores zero / same-element pages for free. +pub struct ZramSwap { + /// The possessed budget — `len()` is the arena ceiling. + store: SupplyToken, + /// Slot table — each stored page's byte residence. Sized at bind to + /// [`DISK_FACTOR`] × the budget's page count (the *disksize*), so + /// capacity scales with the endowment — never a magic absolute. + slots: Vec, + /// Shared generation-checked extent ownership over `slots`. The bitmap, + /// descriptors, and generations are reserved at bind; hot-path lifecycle + /// changes allocate nothing. + extents: ExtentTable, + /// The packed compressed-byte arena (a zsmalloc-analogue): compressed + /// and raw pages share it contiguously, so a 500-byte page costs 500 + /// bytes, not a whole frame. On metal it IS the endowed store, + /// carved + mapped at bind, capped by the budget. + arena: Arena, + /// The **service-owned** compression scratch — `PAGE_BYTES + 512` + /// (LZ4's worst-case block output for one frame, well over the + /// `n + n/255 + 16` bound). Allocated once at bind and reused across + /// every stored page, so the compress path never puts a ~4.6 KiB buffer on + /// the stack per call (§7 named finding — the former per-call + /// `[0u8; PAGE_BYTES + 512]`). + scratch: Vec, + /// Bump high-water into `arena`; reuse comes off `free` first. + cursor: usize, + /// Freed `(off, len)` arena regions, first-fit-reused (split on reuse; + /// no coalescing — the host codec model, not capability-backed zsmalloc). + free: Vec<(u32, u32)>, + /// Live packed bytes (arena occupancy; same-fill contributes nothing). + used: u64, + /// Pages stored (all classes). + stored: u32, + /// Pages in the same-fill class (stored at ~zero cost). + same_fill: u32, + /// **The attached staging window** — the client's shared page buffer, + /// mapped once at [`SwapRequest::Attach`] and addressed by page index + /// thereafter. `None` until the client attaches; every byte-moving + /// verb refuses [`SwapRefusal::NoStaging`] until then. + /// + /// `'static` because the mapping outlives every request: a receiver + /// cannot unmap, so the window is installed for the service's life + /// (which is exactly why it is attached once and not per operation). + staging: Option<&'static mut [u8]>, + /// Successful stores — the [`SwapStatus`] books (occupancy alone cannot + /// recover op counts: a store+release pair leaves the + /// store as it found it). + outs: u64, + /// Successful loads. + ins: u64, + /// Successful releases. + discards: u64, +} + +impl ZramSwap { + /// Occupancy snapshot. + pub fn stats(&self) -> ZramStats { + ZramStats { + stored_pages: self.stored, + same_fill_pages: self.same_fill, + disk_pages: self.slots.len() as u32, + used_bytes: self.used, + budget_bytes: self.store.len(), + saved_bytes: self.stored as u64 * PAGE_BYTES - self.used, + } + } + + /// The tier's books, in the shared [`SwapStatus`] shape: RAM-backed, + /// so the device bit is always set and the io/route words stay zero. + pub fn status(&self) -> SwapStatus { + let s = self.stats(); + SwapStatus { + device: true, + route: false, + capacity_pages: s.disk_pages, + stored_pages: s.stored_pages, + stored_extents: self.extents.live_extents(), + staging_pages: self.staging_pages(), + outs: self.outs, + ins: self.ins, + discards: self.discards, + ..SwapStatus::default() + } + } + + /// [`arena_alloc`](Self::arena_alloc) over borrowed fields — the form + /// the byte-moving paths call, which hold a disjoint borrow of + /// `staging` at the same time. + fn arena_alloc_in( + free: &mut Vec<(u32, u32)>, + cursor: &mut usize, + arena: &mut Arena, + store: &SupplyToken, + len: usize, + ) -> Option { + for i in 0..free.len() { + let (off, flen) = free[i]; + if flen as usize >= len { + if flen as usize == len { + free.swap_remove(i); + } else { + free[i] = (off + len as u32, flen - len as u32); + } + return Some(off); + } + } + let off = *cursor; + if off + len > store.len() as usize { + return None; // budget ceiling + } + arena.cover(off + len); + *cursor = off + len; + Some(off as u32) + } + + /// Return a packed region to the free list. + fn arena_free(&mut self, off: u32, len: u32) { + self.free.push((off, len)); + } + + /// Store one staging page into one already-claimed logical slot. + fn store_page_into(&mut self, index: u32, slot: usize) -> Result<(), SwapRefusal> { + // Disjoint field borrows: the staging read, the compression + // scratch, and the arena are three separate fields, so destructure + // rather than borrow `self` three ways (the `scratch`/`arena` + // discipline this function already followed, one field wider). + let Self { + staging, + slots, + scratch, + arena, + free, + cursor, + store, + used, + stored, + same_fill: same_fill_count, + .. + } = self; + let page = staging + .as_deref() + .and_then(|s| staging_page(s, index)) + .ok_or(SwapRefusal::NoStaging)?; + + if !matches!(slots.get(slot), Some(Slot::Vacant)) { + return Err(SwapRefusal::StoreFull); + } + + // Same-fill fast path — no arena bytes. + if let Some(element) = same_fill(page) { + slots[slot] = Slot::SameFill { element }; + *stored += 1; + *same_fill_count += 1; + return Ok(()); + } + + // Compress into the service-owned scratch, or keep raw when LZ4 + // cannot beat the page. `Some(n)` = compressed to `n` bytes in + // `scratch[..n]`; `None` = the raw huge-page class (never store + // an expansion). + let compressed = match lz4_flex::block::compress_into(page, scratch) { + Ok(n) if (n as u64) < PAGE_BYTES => Some(n), + _ => None, + }; + let len = compressed.unwrap_or(PAGE_BYTES as usize); + + // Pack into the arena — the budget check *is* the allocation. + let off = + Self::arena_alloc_in(free, cursor, arena, store, len).ok_or(SwapRefusal::StoreFull)?; + let dst = &mut arena.bytes_mut()[off as usize..off as usize + len]; + match compressed { + Some(n) => dst.copy_from_slice(&scratch[..n]), + None => dst.copy_from_slice(page), + } + slots[slot] = Slot::Packed { + off, + len: len as u32, + }; + *used += len as u64; + *stored += 1; + Ok(()) + } + + /// Load one stored logical slot without consuming it. + fn load_page_from(&mut self, slot: usize, index: u32) -> Result<(), SwapRefusal> { + let residence = match self.slots.get(slot) { + Some(s) if !matches!(*s, Slot::Vacant) => *s, + _ => return Err(SwapRefusal::BadToken), + }; + // Disjoint borrows again: the destination is inside `staging`, the + // source inside `arena`. + let Self { staging, arena, .. } = self; + let out = staging + .as_deref_mut() + .and_then(|s| staging_page_mut(s, index)) + .ok_or(SwapRefusal::NoStaging)?; + + match residence { + Slot::SameFill { element } => { + simd_fill(out, element); + } + Slot::Packed { off, len } => { + let bytes = &arena.bytes()[off as usize..off as usize + len as usize]; + if len as u64 == PAGE_BYTES { + out.copy_from_slice(bytes); // raw + } else if !matches!( + lz4_flex::block::decompress_into(bytes, out), + Ok(n) if n as u64 == PAGE_BYTES + ) { + // A stored copy that no longer restores is a service bug; + // fail closed while retaining the source for diagnosis or + // retry. Earlier destination pages may already be filled. + return Err(SwapRefusal::BadToken); + } + } + Slot::Vacant => unreachable!("validated above"), + } + Ok(()) + } + + /// Persist one bounded staging extent. The descriptor is published only + /// after every page is resident; any failure rolls back all pages already + /// packed and retires the unreturned handle generation. + fn store_extent(&mut self, first: u32, pages: u32) -> Result { + self.staging + .as_deref() + .and_then(|window| staging_extent(window, first, pages)) + .ok_or(SwapRefusal::NoStaging)?; + let extent = self.extents.claim(pages).ok_or(SwapRefusal::StoreFull)?; + let range = self + .extents + .range(extent, pages) + .expect("fresh extent claim resolves"); + let mut stored = 0usize; + for (offset, slot) in range.clone().enumerate() { + let source = first + offset as u32; + if let Err(refusal) = self.store_page_into(source, slot) { + for rollback in range.start..range.start + stored { + self.release_page(rollback); + } + let _ = self.extents.release(extent, pages); + return Err(refusal); + } + stored += 1; + } + Ok(extent) + } + + /// Non-destructively load one complete extent into staging. + fn load_extent( + &mut self, + extent: SwapExtent, + first: u32, + pages: u32, + ) -> Result<(), SwapRefusal> { + let range = self + .extents + .range(extent, pages) + .ok_or(SwapRefusal::BadToken)?; + self.staging + .as_deref() + .and_then(|window| staging_extent(window, first, pages)) + .ok_or(SwapRefusal::NoStaging)?; + for (offset, slot) in range.enumerate() { + self.load_page_from(slot, first + offset as u32)?; + } + Ok(()) + } + + fn release_page(&mut self, slot: usize) { + let residence = core::mem::replace(&mut self.slots[slot], Slot::Vacant); + match residence { + Slot::SameFill { .. } => self.same_fill -= 1, + Slot::Packed { off, len } => { + self.arena_free(off, len); + self.used -= len as u64; + } + Slot::Vacant => return, + } + self.stored -= 1; + } + + fn release_extent(&mut self, extent: SwapExtent, pages: u32) -> Result<(), SwapRefusal> { + let range = self + .extents + .range(extent, pages) + .ok_or(SwapRefusal::BadToken)?; + for slot in range.clone() { + self.release_page(slot); + } + self.extents + .release(extent, pages) + .expect("validated extent releases"); + Ok(()) + } + + /// Compatibility one-page store. + fn page_out(&mut self, index: u32) -> Result { + self.store_extent(index, 1) + } + + /// Compatibility consuming restore: load first, release only after the + /// bytes are present in staging. + fn page_in(&mut self, extent: SwapExtent, index: u32) -> Result<(), SwapRefusal> { + self.load_extent(extent, index, 1)?; + self.release_extent(extent, 1) + } + + /// **Adopt a staging window** — the mapped bytes and the page count + /// they hold. Shared by the metal `Attach` path (which maps the + /// transferred view first) and by host tests (which hand over an + /// ordinary leaked buffer), so the datapath has exactly one shape and + /// no `cfg(test)` divergence. + /// + /// A window shorter than one page attaches nothing: the client asked + /// for a buffer that cannot carry a page. + pub fn attach_staging(&mut self, window: &'static mut [u8]) -> Result { + let pages = staging_pages(window); + if pages == 0 { + return Err(SwapRefusal::NoStaging); + } + self.staging = Some(window); + Ok(pages) + } + + /// The attached staging capacity in pages (`0` = unattached). + pub fn staging_pages(&self) -> u32 { + self.staging.as_deref().map_or(0, staging_pages) + } + + fn discard(&mut self, extent: SwapExtent) -> Result<(), SwapRefusal> { + self.release_extent(extent, 1) + } +} + +impl SystemService for ZramSwap { + const NAME: &'static str = "zram"; + + /// The staging window is **retained** service authority: it arrives + /// once over the return channel ([`SwapRequest::Attach`]) and lives + /// as long as the service does, so the host must donate a + /// runtime-owned landing per transfer rather than the reusable fixed + /// one (the image service's registrar shape). + const RETURN_MODE: kore_rt::service::ReturnMode = kore_rt::service::ReturnMode::Retained { + kind: kore_abi::cap::ResourceKind::LeasedRegion, + // A staging window is written from both sides: the client stages + // page bytes for `Out`, the service restores into it for `In`. + minimum_rights: kore_abi::cap::Rights::ACCESS.union(kore_abi::cap::Rights::MODIFY), + }; + type Endowment = ZramEndowment; + + fn bind(e: ZramEndowment) -> Self { + let disk = (e.store.len() * DISK_FACTOR / PAGE_BYTES).min(u64::from(u32::MAX)) as u32; + Self { + arena: Arena::bind(&e.store), + store: e.store, + slots: alloc::vec![Slot::Vacant; disk as usize], + extents: ExtentTable::new(disk), + scratch: alloc::vec![0u8; PAGE_BYTES as usize + 512], + cursor: 0, + free: Vec::new(), + used: 0, + stored: 0, + same_fill: 0, + staging: None, + outs: 0, + ins: 0, + discards: 0, + } + } + + /// The store backing must have materialized — a service whose + /// endowed store never carved/mapped must fail closed, never beacon + /// (the same discipline as the output backend's `online()`). + /// The #407 thread witness rides swap (the simplest founded row): + /// a second thread of this service paces itself on pure deadline + /// parks while thread 0 serves doorbell-parked — the unified task + /// model's independent-parks proof, observed by the kernel's + /// `thread-founding:` / `thread-parks:` gate markers. + /// + /// **Declared, not spawned**: the runtime turns this into a thread + /// (`run_service` → `spawn_workers`), so this service — like every + /// other — carries no spawn code, no ordering decision, and no + /// refusal handling of its own. + const WORKERS: &'static [kore_rt::service::Worker] = &[kore_rt::service::Worker { + name: "swap-witness", + entry: thread_witness, + }]; + + fn start_on_metal( + &mut self, + _rt: &kore_rt::entry::Rt, + ) -> Result<(), kore_rt::service::StartFailure> { + kore_rt::service::StartFailure::require(self.arena.live()) + } + + /// Serve one swap op. **Never calls [`Serve::grant`]** — a swap + /// backend holds pages, it mints nothing. + fn serve(&mut self, req: SwapRequest, sv: &mut Serve<'_, SwapResponse>) { + let resp = match req { + SwapRequest::Attach { announced } => match adopt_staging(sv, announced) { + Some(window) => match self.attach_staging(window) { + Ok(pages) => SwapResponse::Attached { pages }, + Err(r) => SwapResponse::Refused(r), + }, + None => SwapResponse::Refused(SwapRefusal::NoStaging), + }, + SwapRequest::Out { page } => match self.page_out(page) { + Ok(extent) => { + self.outs += 1; + SwapResponse::Stored { extent } + } + Err(r) => SwapResponse::Refused(r), + }, + SwapRequest::In { extent, page } => match self.page_in(extent, page) { + Ok(()) => { + self.ins += 1; + SwapResponse::Restored + } + Err(r) => SwapResponse::Refused(r), + }, + SwapRequest::Discard { extent } => match self.discard(extent) { + Ok(()) => { + self.discards += 1; + SwapResponse::Discarded + } + Err(r) => SwapResponse::Refused(r), + }, + SwapRequest::Store { page, pages } => match self.store_extent(page, pages) { + Ok(extent) => { + self.outs += 1; + SwapResponse::Stored { extent } + } + Err(r) => SwapResponse::Refused(r), + }, + SwapRequest::Load { + extent, + page, + pages, + } => match self.load_extent(extent, page, pages) { + Ok(()) => { + self.ins += 1; + SwapResponse::Loaded + } + Err(r) => SwapResponse::Refused(r), + }, + SwapRequest::Release { extent, pages } => match self.release_extent(extent, pages) { + Ok(()) => { + self.discards += 1; + SwapResponse::Released + } + Err(r) => SwapResponse::Refused(r), + }, + SwapRequest::Status => SwapResponse::Status(self.status()), + }; + sv.reply(resp); + } +} + +/// The #407 witness worker: learn "now" from an already-past deadline +/// park's clock stamp, take three real 5 ms deadline parks (each a full +/// ring-3 park/resume round trip through the timer wheel), and **return**. +/// +/// The return is now part of what it witnesses. This worker used to park +/// in day-long strides afterwards — the honest "forever" when threads had +/// no exit verb — and the trampoline turns the return into +/// `thread::exit(ExitCode::OK)` instead: a thread that has finished its +/// work says so, and its status is there for a joiner to read. +fn thread_witness() { + use kore_rt::cap::verbs; + + let mut now_ms = match verbs::sleep_until(1) { + Some(word) => (word >> 32) as u64, + None => 0, + }; + for _ in 0..3 { + if let Some(word) = verbs::sleep_until((now_ms + 5) * 1_000_000) { + now_ms = (word >> 32) as u64; + } + } + // The G7 join storm runs FIRST, so a wedged join blocks the gang + // witness below and its `spmd-gang:` gate markers go missing — the + // storm's completion witness costs no marker of its own. + if !join_storm() { + return; + } + // The G5 gang witness rides the same simplest founded row: a + // one-shot width-2 SPMD region — one gang session, two barrier + // phases, joined — so the boot gate's `spmd-gang: founded 2/2` + // marker proves N-founds-in-one-trap end to end on every CI boot + // (SPMD I6: a frozen-contract change is metal-proven, never + // mocked). Width 2 is boot-invariant on the gate's -smp range; the + // star degrades honestly here (this row holds no CHANNEL_SPACE) and + // the pins ride the std class under QEMU. A refusal is loud on + // serial via the missing marker, and this witness thread simply + // exits — the service row it rides is already serving. + // + // The joins race the workers' exits DELIBERATELY (no pacing): the + // park-vs-exit window is kernel-proven now — the storm above and the + // gate's `join-park: PASS` line — so nothing here may rely on the + // probe-first fast path winning. + if let Some(region) = gang_witness() { + let mut region = region; + for _ in 0..16 { + match region.join_all() { + Ok(_) => break, + Err(refused) => region = refused.spmd, + } + } + } +} + +/// **The G7 join storm** — the WAIT_TASK park-vs-exit race, sampled on +/// every gated boot: short-lived siblings plus one explicitly readied lingerer +/// joined by this doorbell-less thread, so at least one join reaches the kernel +/// while its joinee is live (the shape that wedged under TCG timing) instead of +/// every round riding the probe's already-exited fast path. +/// +/// Three child shapes: +/// - **sprinters** exit the instant they run — the join commits its park +/// in the same window the exit publishes, sampling the Dekker pairing +/// (`install_joiner` vs `mark_exited`/drain) under real SMP timing; +/// - the first child publishes READY and stays in short deadline parks until +/// the parent publishes GO immediately before joining it. On one standard +/// CPU, the child cannot observe GO until the running parent parks or rotates; +/// on SMP, its following one-second delay is a deliberately large bounded +/// margin, not a scheduler-latency proof. The join probe therefore exercises +/// a live child instead of relying on a 3 ms timing guess. Its exit drain must +/// deliver the installed waiter's wake, which is what the kernel's one-shot +/// `join-park: PASS` marker attests. +/// +/// The last child exits with [`STORM_DEEP_EXIT`] from a frame holding 48 KiB +/// of its 64 KiB stack, so the exit path's builders must fit the quarter +/// left. Its joined status must be that code; all other children must +/// report OK. It exits rather than panics: a panic is abort semantics for +/// the whole process, a worker's included (`res/USER_CRASH.md`), so a +/// panicking child would end this service with it. `keytest-crash-x86` +/// witnesses the panic report itself. +/// +/// A join that never resolves wedges this witness right here, and the +/// gang markers downstream (`spmd-gang: founded 2/2`, `retired epoch`) +/// go missing — loud on serial, the witness idiom. +static STORM_LINGERER_READY: core::sync::atomic::AtomicBool = + core::sync::atomic::AtomicBool::new(false); +static STORM_LINGERER_GO: core::sync::atomic::AtomicBool = + core::sync::atomic::AtomicBool::new(false); + +fn join_storm() -> bool { + use core::sync::atomic::Ordering; + use kore_rt::cap::verbs; + + // Preserve the existing sixteen exit/join races; the final one also + // exercises terminal stack headroom. + const ROUNDS: u32 = 16; + for round in 0u32..ROUNDS { + let linger = round == 0; + let entry: fn() = if linger { + STORM_LINGERER_READY.store(false, Ordering::Release); + STORM_LINGERER_GO.store(false, Ordering::Release); + storm_lingerer + } else if round == ROUNDS - 1 { + storm_deep_exit + } else { + storm_sprinter + }; + let Some(mut child) = kore_rt::thread::spawn(entry) else { + // A refused founding is the gang witness's failure to report + // (its founding markers go missing); the storm judges joins. + return false; + }; + if linger { + // The child publishes before waiting for GO. Pace this readiness + // wait on the deadline path available to every thread; a missing + // publish fails after a bounded interval and suppresses the + // downstream completion markers instead of becoming a userspace + // spin. + let Some(word) = verbs::sleep_until(1) else { + return false; + }; + let mut ms = (word >> 32) as u64; + let ready_deadline_ms = ms.saturating_add(5_000); + while !STORM_LINGERER_READY.load(Ordering::Acquire) { + if ms >= ready_deadline_ms { + return false; + } + let deadline_ms = ms.saturating_add(2).min(ready_deadline_ms); + let Some(word) = verbs::sleep_until(deadline_ms * 1_000_000) else { + return false; + }; + ms = (word >> 32) as u64; + } + // This is deliberately adjacent to join: READY keeps the child in + // deadline parks until this publication, and the child's long exit + // delay starts only after it observes GO. + STORM_LINGERER_GO.store(true, Ordering::Release); + } + loop { + match kore_rt::thread::join(child) { + Ok(status) => { + let expected = if round == ROUNDS - 1 { + STORM_DEEP_EXIT.get() + } else { + kore_rt::exit::ExitCode::OK.get() + }; + if status != Some(expected) { + return false; + } + break; + } + Err(refused) => { + // The affine ticket rides back on a typed refusal + // (slots, transport). Pace the retry on the deadline + // park every thread can take; a persistent refusal + // wedges here, and the missing gang markers say so. + child = refused.into_thread(); + if let Some(word) = verbs::sleep_until(1) { + let ms = (word >> 32) as u64; + let _ = verbs::sleep_until((ms + 2) * 1_000_000); + } + } + } + } + } + true +} + +/// A storm child that exits the instant it runs: its trampoline return +/// becomes `thread::exit(ExitCode::OK)` while the parent's join is still +/// committing. +fn storm_sprinter() {} + +/// What the storm's last child exits with, from deep in its stack: a status +/// no other child reports, so the join proves that exit and no stray one. +const STORM_DEEP_EXIT: kore_rt::exit::ExitCode = kore_rt::exit::ExitCode::APPLICATION_0; + +/// Exercise terminal handling with three quarters of an ordinary 64 KiB +/// worker stack occupied. PTL input faulted in the exit stack probe because +/// inlined notice + exit + backoff builders overlapped. The final storm join +/// must observe [`STORM_DEEP_EXIT`] before the downstream gang gate markers +/// can appear. +#[inline(never)] +fn storm_deep_exit() { + const OCCUPIED_BYTES: usize = 48 * 1024; + let occupied = [0u8; OCCUPIED_BYTES]; + storm_exit_with_live_frame(core::hint::black_box(&occupied)); +} + +/// The opaque borrowed buffer keeps the caller's frame live across the exit; +/// a local buffer never passed to a callee can be optimized away before it. +#[inline(never)] +fn storm_exit_with_live_frame(occupied: &[u8]) -> ! { + core::hint::black_box(occupied); + kore_rt::thread::exit(STORM_DEEP_EXIT) +} + +/// A storm child that proves readiness, waits for its parent's immediate-join +/// handoff, then stays live for a large bounded margin while the parent installs +/// its task waiter. +fn storm_lingerer() { + use core::sync::atomic::Ordering; + use kore_rt::cap::verbs; + + let Some(word) = verbs::sleep_until(1) else { + return; + }; + let mut ms = (word >> 32) as u64; + let go_deadline_ms = ms.saturating_add(5_000); + STORM_LINGERER_READY.store(true, Ordering::Release); + while !STORM_LINGERER_GO.load(Ordering::Acquire) { + if ms >= go_deadline_ms { + return; + } + let deadline_ms = ms.saturating_add(2).min(go_deadline_ms); + let Some(word) = verbs::sleep_until(deadline_ms * 1_000_000) else { + return; + }; + ms = (word >> 32) as u64; + } + let _ = verbs::sleep_until((ms + 1_000) * 1_000_000); +} + +/// Two workers, two barrier phases, exit — the least region that +/// exercises the gang founding, the launch gate, and the barrier. +#[kore_rt::spmd(1, 1, 2)] +fn gang_witness(ctx: &'static kore_rt::spmd::SpmdCtx, _worker: u32) { + ctx.barrier().arrive_and_wait(); + ctx.barrier().arrive_and_wait(); +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A leaked staging window of `pages` pages — the host stand-in for + /// the mapping `Attach` installs on metal. Leaked because the real + /// one is `'static` by construction (a receiver cannot unmap). + fn staging(pages: usize) -> &'static mut [u8] { + alloc::vec![0u8; pages * PAGE_BYTES as usize].leak() + } + + /// A bound zram over a geometry-only host supply. + fn bound(budget_pages: u64) -> ZramSwap { + ZramSwap::bind(ZramEndowment { + store: SupplyToken::for_test(0, budget_pages * PAGE_BYTES), + }) + } + + /// The datapath, end to end: stage a compressible page, `Out` it, + /// clobber staging, `In` it back, and the bytes are identical. This + /// is what the per-op-delivered-view shape could never test on the + /// host — the whole point of the staging window. + #[test] + fn a_page_round_trips_through_the_staging_window() { + let mut z = bound(64); + let pages = z.attach_staging(staging(4)).expect("staging attaches"); + assert_eq!(pages, 4); + + // A compressible but not same-fill pattern. + let page: Vec = (0..PAGE_BYTES as usize).map(|i| (i / 64) as u8).collect(); + let out = staging_page_mut(z.staging.as_deref_mut().unwrap(), 1).unwrap(); + out.copy_from_slice(&page); + + let token = z.page_out(1).expect("page-out stores"); + // Clobber staging so a restore that silently no-ops cannot pass. + staging_page_mut(z.staging.as_deref_mut().unwrap(), 1) + .unwrap() + .fill(0xAA); + z.page_in(token, 1).expect("page-in restores"); + assert_eq!( + staging_page(z.staging.as_deref().unwrap(), 1).unwrap(), + &page[..], + "the restored bytes are the stored bytes" + ); + assert_eq!(z.stats().stored_pages, 0, "restore consumed the copy"); + } + + /// The new datapath is extent-shaped: one store owns several logical + /// pages, loads are repeatable, and only explicit release retires the + /// generation. This is the lifecycle prefetch/cancellation needs. + #[test] + fn an_extent_load_is_repeatable_until_explicit_release() { + let mut z = bound(64); + z.attach_staging(staging(6)).unwrap(); + for page in 0..3 { + staging_page_mut(z.staging.as_deref_mut().unwrap(), page) + .unwrap() + .fill(0x31 + page as u8); + } + + let extent = z.store_extent(0, 3).expect("extent stores atomically"); + assert_eq!(z.stats().stored_pages, 3); + assert_eq!(z.status().stored_extents, 1); + + for destination in [3, 0] { + for page in destination..destination + 3 { + staging_page_mut(z.staging.as_deref_mut().unwrap(), page) + .unwrap() + .fill(0); + } + z.load_extent(extent, destination, 3) + .expect("load keeps the copy live"); + for page in 0..3 { + assert!( + staging_page(z.staging.as_deref().unwrap(), destination + page) + .unwrap() + .iter() + .all(|&byte| byte == 0x31 + page as u8) + ); + } + assert_eq!(z.stats().stored_pages, 3, "load is non-destructive"); + } + + z.release_extent(extent, 3).expect("explicit release"); + assert_eq!(z.stats().stored_pages, 0); + assert_eq!(z.status().stored_extents, 0); + assert_eq!( + z.load_extent(extent, 0, 3), + Err(SwapRefusal::BadToken), + "a released generation never aliases later storage" + ); + } + + /// A multi-page store is all-or-nothing even when compressed-arena + /// capacity runs out after an earlier page was packed. + #[test] + fn a_partial_extent_store_rolls_back_every_page() { + let mut z = bound(1); + z.attach_staging(staging(2)).unwrap(); + let mut state = 0x1234_5678_u32; + for page in z + .staging + .as_deref_mut() + .unwrap() + .chunks_exact_mut(PAGE_BYTES as usize) + { + for byte in page { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + *byte = state as u8; + } + } + assert_eq!(z.store_extent(0, 2), Err(SwapRefusal::StoreFull)); + assert_eq!(z.stats().stored_pages, 0); + assert_eq!(z.stats().used_bytes, 0); + assert_eq!(z.status().stored_extents, 0); + } + + /// The same-fill class round-trips too, and costs no arena bytes. + #[test] + fn a_same_fill_page_costs_nothing_and_restores() { + let mut z = bound(64); + z.attach_staging(staging(2)).unwrap(); + staging_page_mut(z.staging.as_deref_mut().unwrap(), 0) + .unwrap() + .fill(0); + + let token = z.page_out(0).expect("same-fill stores"); + assert_eq!(z.stats().same_fill_pages, 1); + assert_eq!(z.stats().used_bytes, 0, "same-fill uses no arena bytes"); + + staging_page_mut(z.staging.as_deref_mut().unwrap(), 0) + .unwrap() + .fill(0xFF); + z.page_in(token, 0).expect("same-fill restores"); + assert!( + staging_page(z.staging.as_deref().unwrap(), 0) + .unwrap() + .iter() + .all(|&b| b == 0), + "the zero page came back zero" + ); + } + + /// Every byte-moving verb refuses before an `Attach`, and refuses a + /// page index outside the attached window — the client's index is + /// data, and the window's length is the only authority on it. + #[test] + fn staging_bounds_and_absence_are_refusals_not_panics() { + let mut z = bound(64); + assert_eq!(z.page_out(0), Err(SwapRefusal::NoStaging), "no window yet"); + assert_eq!(z.staging_pages(), 0); + + z.attach_staging(staging(2)).unwrap(); + assert_eq!(z.staging_pages(), 2); + assert_eq!( + z.page_out(2), + Err(SwapRefusal::NoStaging), + "one past the last page" + ); + assert_eq!( + z.page_out(u32::MAX), + Err(SwapRefusal::NoStaging), + "an index whose byte offset overflows" + ); + + let token = z.page_out(0).expect("in-bounds page stores"); + assert_eq!( + z.page_in(token, 2), + Err(SwapRefusal::NoStaging), + "restore is bounds-checked too" + ); + // The refused restore consumed nothing. + assert_eq!(z.stats().stored_pages, 1); + z.page_in(token, 0) + .expect("the in-bounds restore still works"); + } + + /// A window too short for one page attaches nothing. + #[test] + fn a_sub_page_window_is_refused() { + let mut z = bound(64); + let short: &'static mut [u8] = alloc::vec![0u8; 64].leak(); + assert_eq!(z.attach_staging(short), Err(SwapRefusal::NoStaging)); + } +} diff --git a/native/services/swap/src/wire.rs b/native/services/swap/src/wire.rs new file mode 100644 index 0000000..2433f2b --- /dev/null +++ b/native/services/swap/src/wire.rs @@ -0,0 +1,405 @@ +//! `wire` — the swap enum ⇄ wire codecs (`res/USERLAND.md` §5.3). +//! +//! The zerocopy forms live in [`kore_abi::svc`]; decode is fail-closed. +//! A swap service can mint no authority, so its [`SvcProtocol`] declares +//! **zero reply grants** — a client of this protocol pre-pins nothing, +//! and the reply's `grants` byte is pinned zero on encode and refused +//! non-zero on decode (a "grant-bearing" swap reply is a protocol break, +//! not a surprise to absorb). + +use kore_abi::session::SlotRef; +use kore_abi::svc::{SwapRepWire, SwapReqWire}; + +use kore_rt::svc::SvcProtocol; + +use crate::extent::{MAX_EXTENT_PAGES, SwapExtent}; +use crate::service::{SwapRefusal, SwapRequest, SwapResponse, SwapStatus, ZramSwap}; + +/// Lower a request to its wire form (total). +pub fn encode_req(req: &SwapRequest) -> SwapReqWire { + let (op, arg, token, pages) = match *req { + SwapRequest::Attach { announced } => (SwapReqWire::OP_ATTACH, announced.index(), 0, 0), + SwapRequest::Out { page } => (SwapReqWire::OP_OUT, page, 0, 0), + SwapRequest::In { extent, page } => (SwapReqWire::OP_IN, page, extent.token(), 0), + SwapRequest::Discard { extent } => (SwapReqWire::OP_DISCARD, 0, extent.token(), 0), + SwapRequest::Store { page, pages } => (SwapReqWire::OP_STORE_EXTENT, page, 0, pages), + SwapRequest::Load { + extent, + page, + pages, + } => (SwapReqWire::OP_LOAD_EXTENT, page, extent.token(), pages), + SwapRequest::Release { extent, pages } => { + (SwapReqWire::OP_RELEASE_EXTENT, 0, extent.token(), pages) + } + SwapRequest::Status => (SwapReqWire::OP_STATUS, 0, 0, 0), + }; + SwapReqWire { + op, + reserved: [0; 3], + arg: arg.into(), + token: token.into(), + pages: pages.into(), + } +} + +/// Lift a wire request — fail-closed. +pub fn decode_req(w: &SwapReqWire) -> Option { + if w.reserved != [0; 3] { + return None; + } + let pages = w.pages.get(); + Some(match w.op { + SwapReqWire::OP_ATTACH if w.token.get() == 0 && pages == 0 => SwapRequest::Attach { + announced: SlotRef::from_wire(w.arg.get()), + }, + SwapReqWire::OP_OUT if w.token.get() == 0 && pages == 0 => { + SwapRequest::Out { page: w.arg.get() } + } + SwapReqWire::OP_IN if pages == 0 => SwapRequest::In { + extent: SwapExtent::from_raw(w.token.get())?, + page: w.arg.get(), + }, + SwapReqWire::OP_DISCARD if w.arg.get() == 0 && pages == 0 => SwapRequest::Discard { + extent: SwapExtent::from_raw(w.token.get())?, + }, + SwapReqWire::OP_STORE_EXTENT + if w.token.get() == 0 && (1..=MAX_EXTENT_PAGES).contains(&pages) => + { + SwapRequest::Store { + page: w.arg.get(), + pages, + } + } + SwapReqWire::OP_LOAD_EXTENT if (1..=MAX_EXTENT_PAGES).contains(&pages) => { + SwapRequest::Load { + extent: SwapExtent::from_raw(w.token.get())?, + page: w.arg.get(), + pages, + } + } + SwapReqWire::OP_RELEASE_EXTENT + if w.arg.get() == 0 && (1..=MAX_EXTENT_PAGES).contains(&pages) => + { + SwapRequest::Release { + extent: SwapExtent::from_raw(w.token.get())?, + pages, + } + } + SwapReqWire::OP_STATUS if w.arg.get() == 0 && w.token.get() == 0 && pages == 0 => { + SwapRequest::Status + } + _ => return None, + }) +} + +/// Lower a response (total). `grants` is pinned zero — see the module +/// docs; the parameter exists so the codec face matches its siblings. +pub fn encode_rep(rep: &SwapResponse, grants: u8) -> SwapRepWire { + debug_assert_eq!(grants, 0, "a swap service grants nothing, ever"); + let (tag, refusal, token) = match *rep { + SwapResponse::Stored { extent } => (SwapRepWire::TAG_STORED, 0, extent.token()), + SwapResponse::Attached { pages } => (SwapRepWire::TAG_ATTACHED, 0, pages), + SwapResponse::Restored => (SwapRepWire::TAG_RESTORED, 0, 0), + SwapResponse::Discarded => (SwapRepWire::TAG_DISCARDED, 0, 0), + SwapResponse::Loaded => (SwapRepWire::TAG_LOADED, 0, 0), + SwapResponse::Released => (SwapRepWire::TAG_RELEASED, 0, 0), + SwapResponse::Status(_) => (SwapRepWire::TAG_STATUS, 0, 0), + SwapResponse::Refused(r) => ( + SwapRepWire::TAG_REFUSED, + match r { + SwapRefusal::BadPage => SwapRepWire::REFUSE_BAD_PAGE, + SwapRefusal::BadDst => SwapRepWire::REFUSE_BAD_DST, + SwapRefusal::BadToken => SwapRepWire::REFUSE_BAD_TOKEN, + SwapRefusal::StoreFull => SwapRepWire::REFUSE_STORE_FULL, + SwapRefusal::Backend => SwapRepWire::REFUSE_BACKEND, + SwapRefusal::NoStaging => SwapRepWire::REFUSE_NO_STAGING, + }, + 0, + ), + }; + // The books ride only the status tag; every other reply zeroes them + // rather than leaving stale bytes to misread (the input-stream + // reply's rule). + let s = match *rep { + SwapResponse::Status(s) => s, + _ => SwapStatus::default(), + }; + SwapRepWire { + tag, + refusal, + grants: 0, + posture: (if s.device { + SwapRepWire::STATUS_DEVICE + } else { + 0 + }) | (if s.route { + SwapRepWire::STATUS_ROUTE + } else { + 0 + }), + token: token.into(), + capacity_pages: s.capacity_pages.into(), + stored_pages: s.stored_pages.into(), + staging_pages: s.staging_pages.into(), + stored_extents: s.stored_extents.into(), + outs: s.outs.into(), + ins: s.ins.into(), + discards: s.discards.into(), + io_submitted: s.io_submitted.into(), + io_completed: s.io_completed.into(), + io_errors: s.io_errors.into(), + fires: s.fires.into(), + parks: s.parks.into(), + deadline_waits: s.deadline_waits.into(), + spin_waits: s.spin_waits.into(), + } +} + +fn non_status_books_are_zero(w: &SwapRepWire) -> bool { + w.posture == 0 + && w.capacity_pages.get() == 0 + && w.stored_pages.get() == 0 + && w.staging_pages.get() == 0 + && w.stored_extents.get() == 0 + && w.outs.get() == 0 + && w.ins.get() == 0 + && w.discards.get() == 0 + && w.io_submitted.get() == 0 + && w.io_completed.get() == 0 + && w.io_errors.get() == 0 + && w.fires.get() == 0 + && w.parks.get() == 0 + && w.deadline_waits.get() == 0 + && w.spin_waits.get() == 0 +} + +/// Lift a wire reply — fail-closed (including a non-zero `grants`). +pub fn decode_rep(w: &SwapRepWire) -> Option<(SwapResponse, u8)> { + if w.grants != 0 { + return None; // a grant-bearing swap reply is a protocol break + } + let rep = match (w.tag, w.refusal) { + (SwapRepWire::TAG_STORED, 0) if non_status_books_are_zero(w) => SwapResponse::Stored { + extent: SwapExtent::from_raw(w.token.get())?, + }, + (SwapRepWire::TAG_ATTACHED, 0) if non_status_books_are_zero(w) && w.token.get() != 0 => { + SwapResponse::Attached { + pages: w.token.get(), + } + } + (SwapRepWire::TAG_RESTORED, 0) if w.token.get() == 0 && non_status_books_are_zero(w) => { + SwapResponse::Restored + } + (SwapRepWire::TAG_DISCARDED, 0) if w.token.get() == 0 && non_status_books_are_zero(w) => { + SwapResponse::Discarded + } + (SwapRepWire::TAG_LOADED, 0) if w.token.get() == 0 && non_status_books_are_zero(w) => { + SwapResponse::Loaded + } + (SwapRepWire::TAG_RELEASED, 0) if w.token.get() == 0 && non_status_books_are_zero(w) => { + SwapResponse::Released + } + (SwapRepWire::TAG_STATUS, 0) if w.token.get() == 0 => { + // Fail-closed on posture bits this decoder does not know — + // a newer tier's semantics must refuse, not silently drop. + if w.posture & !(SwapRepWire::STATUS_DEVICE | SwapRepWire::STATUS_ROUTE) != 0 { + return None; + } + SwapResponse::Status(SwapStatus { + device: w.posture & SwapRepWire::STATUS_DEVICE != 0, + route: w.posture & SwapRepWire::STATUS_ROUTE != 0, + capacity_pages: w.capacity_pages.get(), + stored_pages: w.stored_pages.get(), + stored_extents: w.stored_extents.get(), + staging_pages: w.staging_pages.get(), + outs: w.outs.get(), + ins: w.ins.get(), + discards: w.discards.get(), + io_submitted: w.io_submitted.get(), + io_completed: w.io_completed.get(), + io_errors: w.io_errors.get(), + fires: w.fires.get(), + parks: w.parks.get(), + deadline_waits: w.deadline_waits.get(), + spin_waits: w.spin_waits.get(), + }) + } + (SwapRepWire::TAG_REFUSED, r) if w.token.get() == 0 && non_status_books_are_zero(w) => { + SwapResponse::Refused(match r { + SwapRepWire::REFUSE_BAD_PAGE => SwapRefusal::BadPage, + SwapRepWire::REFUSE_BAD_DST => SwapRefusal::BadDst, + SwapRepWire::REFUSE_BAD_TOKEN => SwapRefusal::BadToken, + SwapRepWire::REFUSE_STORE_FULL => SwapRefusal::StoreFull, + SwapRepWire::REFUSE_BACKEND => SwapRefusal::Backend, + SwapRepWire::REFUSE_NO_STAGING => SwapRefusal::NoStaging, + _ => return None, + }) + } + _ => return None, + }; + Some((rep, 0)) +} + +impl SvcProtocol for ZramSwap { + type Request = SwapRequest; + type Response = SwapResponse; + type ReqWire = SwapReqWire; + type RepWire = SwapRepWire; + + /// Zero grants, ever — the swap doctrine, now structural: a client + /// of this protocol pre-pins no reply posts. + const REPLY_GRANTS: usize = 0; + + fn encode_req(req: &SwapRequest) -> SwapReqWire { + encode_req(req) + } + fn decode_req(wire: &SwapReqWire) -> Option { + decode_req(wire) + } + fn encode_rep(rep: &SwapResponse, grants: u8) -> SwapRepWire { + encode_rep(rep, grants) + } + fn decode_rep(wire: &SwapRepWire) -> Option<(SwapResponse, u8)> { + decode_rep(wire) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn extent(token: u32) -> SwapExtent { + SwapExtent::from_raw((1 << 16) | token).unwrap() + } + + /// Every request/response round-trips through its wire form. + #[test] + fn round_trips() { + for req in [ + SwapRequest::Attach { + announced: SlotRef::from_wire(3), + }, + SwapRequest::Out { page: 4 }, + SwapRequest::In { + extent: extent(7), + page: 5, + }, + SwapRequest::Discard { extent: extent(9) }, + SwapRequest::Store { + page: 4, + pages: 387, + }, + SwapRequest::Load { + extent: extent(11), + page: 32, + pages: 387, + }, + SwapRequest::Release { + extent: extent(11), + pages: 387, + }, + SwapRequest::Status, + ] { + let back = decode_req(&encode_req(&req)).expect("decodes"); + assert_eq!(encode_req(&back), encode_req(&req)); + } + for rep in [ + SwapResponse::Stored { extent: extent(3) }, + SwapResponse::Attached { pages: 8 }, + SwapResponse::Restored, + SwapResponse::Discarded, + SwapResponse::Loaded, + SwapResponse::Released, + SwapResponse::Status(SwapStatus::default()), + SwapResponse::Status(SwapStatus { + device: true, + route: true, + capacity_pages: 262144, + stored_pages: 12, + stored_extents: 2, + staging_pages: 8, + outs: 40, + ins: 30, + discards: 9, + io_submitted: 83, + io_completed: 83, + io_errors: 1, + fires: 81, + parks: 82, + deadline_waits: 17, + spin_waits: 5, + }), + SwapResponse::Refused(SwapRefusal::BadPage), + SwapResponse::Refused(SwapRefusal::BadDst), + SwapResponse::Refused(SwapRefusal::BadToken), + SwapResponse::Refused(SwapRefusal::StoreFull), + SwapResponse::Refused(SwapRefusal::Backend), + SwapResponse::Refused(SwapRefusal::NoStaging), + ] { + let (back, grants) = decode_rep(&encode_rep(&rep, 0)).expect("decodes"); + assert_eq!(back, rep); + assert_eq!(grants, 0); + } + } + + /// Decode is fail-closed: unknown discriminants and any claimed + /// grant refuse. + #[test] + fn decode_fails_closed() { + let mut w = encode_req(&SwapRequest::Discard { extent: extent(1) }); + w.op = 0; + assert!(decode_req(&w).is_none()); + w.op = 200; + assert!(decode_req(&w).is_none()); + + let mut r = encode_rep(&SwapResponse::Restored, 0); + r.tag = 0; + assert!(decode_rep(&r).is_none()); + let mut r = encode_rep(&SwapResponse::Refused(SwapRefusal::Backend), 0); + r.refusal = 42; + assert!(decode_rep(&r).is_none()); + let mut r = encode_rep(&SwapResponse::Stored { extent: extent(1) }, 0); + r.grants = 1; + assert!( + decode_rep(&r).is_none(), + "a grant-bearing swap reply refuses" + ); + let mut r = encode_rep(&SwapResponse::Restored, 0); + r.refusal = 1; + assert!( + decode_rep(&r).is_none(), + "a refusal byte on success refuses" + ); + let mut r = encode_rep(&SwapResponse::Status(SwapStatus::default()), 0); + r.posture = 0x80; + assert!( + decode_rep(&r).is_none(), + "an unknown posture bit on a status reply refuses" + ); + let mut r = encode_rep(&SwapResponse::Loaded, 0); + r.stored_extents = 1.into(); + assert!( + decode_rep(&r).is_none(), + "stale status books on a data reply refuse" + ); + + let mut w = encode_req(&SwapRequest::Store { page: 0, pages: 2 }); + w.pages = 0.into(); + assert!(decode_req(&w).is_none(), "zero-length extent refuses"); + let mut w = encode_req(&SwapRequest::Store { page: 0, pages: 2 }); + w.reserved[0] = 1; + assert!(decode_req(&w).is_none(), "dirty reserve refuses"); + } + + /// The status books ride ONLY the status tag — every other reply + /// zeroes them, so a stale record can never be misread as books. + #[test] + fn non_status_replies_zero_the_books() { + let w = encode_rep(&SwapResponse::Stored { extent: extent(3) }, 0); + assert_eq!(w.posture, 0); + assert_eq!(w.capacity_pages.get(), 0); + assert_eq!(w.stored_extents.get(), 0); + assert_eq!(w.outs.get(), 0); + assert_eq!(w.parks.get(), 0); + } +} diff --git a/native/tools/kore-vol/Cargo.toml b/native/tools/kore-vol/Cargo.toml new file mode 100644 index 0000000..d7b0e50 --- /dev/null +++ b/native/tools/kore-vol/Cargo.toml @@ -0,0 +1,44 @@ +# `kore-vol` — a kore volume, from the host: what a terminal's home holds, and +# what the host built copied into it (`res/DAGFS.md`, "From the host"). It +# finds, mounts and commits the volume through the filesystem row's own code +# — `kore-svc-dagfs`'s locate, recovery and dual-root commit over +# `kore-driver`'s block channel — so an image it writes is one the next boot +# mounts as it would its own. +[package] +name = "kore-vol" +version = "0.0.0" +edition = "2024" +description = "kore volume tool and library: list, read, install into and remove from a terminal's home on an offline disk image." +publish = false + +# The procedure itself, for a host program that keeps machines (a VM +# supervisor staging a project, a packager) — the binary below prints over +# it, so both drive one copy of it. +[lib] +name = "kore_vol" +path = "src/lib.rs" + +[[bin]] +name = "kore-vol" +path = "src/main.rs" + +[dependencies] +kore-svc-dagfs = { path = "../../services/dagfs" } +# The image file as a block device (`image`), beside the channel the row +# drives its own medium through. +kore-driver = { path = "../../crates/driver", features = ["service", "image"] } +# Each file's read-back is compared by digest with what was written. Default +# features (std, and the SIMD backends): a host tool, the same lockfile +# entry the filesystem service takes `pure`. +blake3 = "1.8.7" + +# A machine's disk (`src/device.rs`): its BSD lock (`flock`) and the record +# locks a machine running on it holds (`fcntl_getlk`), on Linux alone. +[target.'cfg(target_os = "linux")'.dependencies] +rustix = { version = "1.1.4", default-features = false, features = ["std", "fs", "process"] } + +[dev-dependencies] +# The tests hold an image the way another process would, to see the tool +# refuse it, and build a GPT-partitioned image with the driver's own +# synthetic table. +rustix = { version = "1.1.4", default-features = false, features = ["std", "fs"] } diff --git a/native/tools/kore-vol/src/device.rs b/native/tools/kore-vol/src/device.rs new file mode 100644 index 0000000..32029e1 --- /dev/null +++ b/native/tools/kore-vol/src/device.rs @@ -0,0 +1,436 @@ +//! A machine's disk, from the host (Linux): which of its nodes the user +//! volume is read and written through, and the disk held while it is. +//! +//! The disk is attached to the host running the tool, with the machine that +//! boots it shut down: a live system on that machine, or the disk moved to +//! an enclosure. Its user volume is found as the filesystem row finds it at +//! boot, by `DurableService::locate` over the whole disk, so naming one of +//! its partitions names the disk and asserts that the volume is in that +//! partition. +//! +//! - **Reading** goes through the disk's node, narrowed as the boot narrows +//! it, unbuffered and under a shared record lock. +//! - **Writing** first holds the disk as Linux asks of a program that +//! writes one (systemd's "Locking Block Device Access"): an exclusive BSD +//! lock on the whole disk's node, which udev takes shared before it probes +//! the disk or any partition of it, and partitioning tools take before +//! they write a table. A disk a machine is running on is refused: QEMU +//! locks the nodes it runs, as it locks images. The tool then claims the +//! whole disk (`O_EXCL`) and writes through the boot's own narrowed view, +//! a claim the kernel grants only while nothing on the disk is mounted or +//! claimed. A disk with something mounted from another partition (a Linux +//! installed beside kore, an EFI system partition) cannot be claimed +//! whole; the tool then claims the partition that holds the volume, alone, +//! through its own node, a claim the kernel grants while nothing mounts or +//! claims that partition. +//! - **Never formatted.** A device's blank volume is formatted by the +//! machine's own first boot, and partitioning a disk is +//! `tools/provision_fs.py`'s. + +use std::{ + fs::{self, File}, + io, + os::unix::fs::{FileTypeExt, MetadataExt}, + path::{Path, PathBuf}, + thread, + time::{Duration, Instant}, +}; + +use kore_driver::block::{ + DirectBlockDevice, + image::{Access, ImageError, ImageFile}, +}; +use kore_svc_dagfs::durable::Located; +use rustix::{ + fs::{FlockOperation, fcntl_lock, flock, major, minor}, + process::{Flock, FlockOffsetType, FlockType, fcntl_getlk}, +}; + +use crate::volume::{Found, VolError, classify}; + +/// The unit sysfs gives a partition's start and size in: 512-byte sectors, +/// whatever the disk's logical block size (`Documentation/ABI/stable/ +/// sysfs-block`). +const SYSFS_SECTOR: u64 = 512; + +/// Where the kernel names its block device nodes. +const DEV: &str = "/dev"; + +/// Where sysfs links each block device's number to its directory. +const SYS_DEV_BLOCK: &str = "/sys/dev/block"; + +/// How long the disk's BSD lock is waited for. udev holds it shared through +/// one run of its rules for each event on the disk, and closing a node that +/// was written raises one, so a tool started straight after another can meet +/// it held for a moment; a holder past this is a partitioner or another +/// writer, which is a conflict. +const HOLD_WAIT: Duration = Duration::from_secs(5); + +/// Between attempts on the held lock. +const HOLD_RETRY: Duration = Duration::from_millis(20); + +/// The disk held while a region of it is written (the module docs), until +/// the tool is done with the volume: bound before the volume, so that it is +/// dropped after it. +/// +/// The volume is written through one node, claimed and under its exclusive +/// record lock. The hold keeps a shared record lock on the other node a +/// machine could be run on, the whole disk's or the volume's partition's, +/// which a QEMU started there meets and refuses. A record lock belongs to +/// the process and is dropped when *any* of its descriptors of that node +/// closes, so a guard is taken last, once nothing else of its node is open, +/// and nothing opens that node again while the hold lives. +pub(crate) struct DiskHold { + /// The whole disk's node, under its BSD lock. + disk: File, + /// The volume's partition's node, guarded while the disk is written + /// whole. + partition: Option, +} + +impl DiskHold { + /// Hold the disk at `node`: its BSD lock, and no machine running on it. + fn take(node: &Path) -> Result { + let disk = File::open(node).map_err(|error| io_error(node, error))?; + let deadline = Instant::now() + HOLD_WAIT; + loop { + match flock(&disk, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => break, + Err(rustix::io::Errno::WOULDBLOCK) if Instant::now() < deadline => { + thread::sleep(HOLD_RETRY); + } + Err(rustix::io::Errno::WOULDBLOCK) => { + return Err(VolError::Host(format!( + "{}: another program holds the disk (its BSD lock, which \ + partitioning tools take) and kept it for {} s", + node.display(), + HOLD_WAIT.as_secs() + ))); + } + Err(error) => return Err(io_error(node, error.into())), + } + } + refuse_running(&disk, node)?; + Ok(Self { + disk, + partition: None, + }) + } + + /// While the disk is written whole: guard the volume's partition's own + /// node, refusing a machine already running on it. + fn guard_partition(&mut self, node: &Path) -> Result<(), VolError> { + let partition = File::open(node).map_err(|error| io_error(node, error))?; + refuse_running(&partition, node)?; + share(&partition, node)?; + self.partition = Some(partition); + Ok(()) + } + + /// While the volume is written through its partition's node: guard the + /// whole disk's. + fn guard_disk(&self, node: &Path) -> Result<(), VolError> { + share(&self.disk, node) + } +} + +/// Refuse a node another process holds a record lock on, as QEMU holds the +/// nodes it runs a machine on. Asked with `F_GETLK`, which reading the node +/// is enough for. +fn refuse_running(file: &File, node: &Path) -> Result<(), VolError> { + let whole = Flock { + start: 0, + length: 0, + pid: None, + typ: FlockType::WriteLock, + offset_type: FlockOffsetType::Set, + }; + match fcntl_getlk(file, &whole) { + Ok(None) => Ok(()), + Ok(Some(_)) => Err(VolError::Device(node.to_path_buf(), ImageError::InUse)), + Err(error) => Err(io_error(node, error.into())), + } +} + +/// A shared record lock on the whole of `file`, without waiting. +fn share(file: &File, node: &Path) -> Result<(), VolError> { + match fcntl_lock(file, FlockOperation::NonBlockingLockShared) { + Ok(()) => Ok(()), + // POSIX lets a conflicting `F_SETLK` answer either. + Err(rustix::io::Errno::AGAIN | rustix::io::Errno::ACCESS) => { + Err(VolError::Device(node.to_path_buf(), ImageError::InUse)) + } + Err(error) => Err(io_error(node, error.into())), + } +} + +fn io_error(node: &Path, error: io::Error) -> VolError { + VolError::Device(node.to_path_buf(), ImageError::Io(error)) +} + +/// A partition of the disk, as sysfs gives it. +struct Partition { + node: PathBuf, + /// Its first sector and its length, in [`SYSFS_SECTOR`]s. + start: u64, + sectors: u64, +} + +/// The disk a named node belongs to. +struct Disk { + /// The whole disk's node. + node: PathBuf, + /// Its sysfs directory, which holds a directory for each partition. + dir: PathBuf, + /// The partition the node named, when it named one. + named: Option, +} + +/// The disk the node at `path` belongs to, as sysfs describes it: itself, +/// when it is a whole disk. +fn disk_of(path: &Path) -> Result { + let meta = fs::metadata(path).map_err(|error| io_error(path, error))?; + if !meta.file_type().is_block_device() { + return Err(VolError::Device(path.to_path_buf(), ImageError::NotDevice)); + } + let link = + Path::new(SYS_DEV_BLOCK).join(format!("{}:{}", major(meta.rdev()), minor(meta.rdev()))); + let dir = fs::canonicalize(&link).map_err(|error| sysfs_error(&link, error))?; + if !dir.join("partition").is_file() { + return Ok(Disk { + node: path.to_path_buf(), + dir, + named: None, + }); + } + let disk = dir + .parent() + .ok_or_else(|| VolError::Host(format!("{}: a partition of no disk", dir.display())))? + .to_path_buf(); + Ok(Disk { + node: node_of(&disk)?, + dir: disk, + named: Some(Partition { + node: path.to_path_buf(), + start: read_number(&dir.join("start"))?, + sectors: read_number(&dir.join("size"))?, + }), + }) +} + +/// The node of the block device sysfs describes at `dir`: the name its +/// `uevent` gives, under [`DEV`], checked to be that device. +fn node_of(dir: &Path) -> Result { + let uevent = dir.join("uevent"); + let text = fs::read_to_string(&uevent).map_err(|error| sysfs_error(&uevent, error))?; + let name = text + .lines() + .find_map(|line| line.strip_prefix("DEVNAME=")) + .ok_or_else(|| VolError::Host(format!("{}: no DEVNAME", uevent.display())))?; + let node = Path::new(DEV).join(name); + let numbers = dir.join("dev"); + let want = fs::read_to_string(&numbers).map_err(|error| sysfs_error(&numbers, error))?; + let rdev = fs::metadata(&node) + .map_err(|error| io_error(&node, error))? + .rdev(); + if want.trim() != format!("{}:{}", major(rdev), minor(rdev)) { + return Err(VolError::Host(format!( + "{}: not the device sysfs names {} ({})", + node.display(), + dir.display(), + want.trim() + ))); + } + Ok(node) +} + +/// The node of the disk's partition at exactly `start` for `sectors`. +fn partition_at(disk: &Disk, start: u64, sectors: u64) -> Result, VolError> { + let entries = fs::read_dir(&disk.dir).map_err(|error| sysfs_error(&disk.dir, error))?; + for entry in entries { + let dir = entry.map_err(|error| sysfs_error(&disk.dir, error))?.path(); + if !dir.join("partition").is_file() { + continue; + } + if read_number(&dir.join("start"))? == start && read_number(&dir.join("size"))? == sectors { + return node_of(&dir).map(Some); + } + } + Ok(None) +} + +fn sysfs_error(path: &Path, error: io::Error) -> VolError { + VolError::Host(format!("{}: {error}", path.display())) +} + +fn read_number(path: &Path) -> Result { + let text = fs::read_to_string(path).map_err(|error| sysfs_error(path, error))?; + text.trim().parse().map_err(|_| { + VolError::Host(format!( + "{}: not a number: {:?}", + path.display(), + text.trim() + )) + }) +} + +/// Open a node of the disk at the device's own logical block size, which a +/// block size the user named must agree with. +fn open(node: &Path, access: Access, block_size: Option) -> Result { + let device = ImageFile::open_device(node, access) + .map_err(|error| VolError::Device(node.to_path_buf(), error))?; + let own = device.geometry().block_size; + match block_size { + Some(asked) if asked != own => Err(VolError::Host(format!( + "--block-size {asked}: {}'s logical blocks are {own} bytes", + node.display() + ))), + _ => Ok(device), + } +} + +/// Where a volume to recover lies on the medium it was found through. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Region { + /// Its first block and its length, in the medium's blocks. + first: u64, + blocks: u64, + /// Whether that is the whole medium. + whole: bool, + block_size: u32, +} + +/// The region of a found volume; `None` for anything but a volume to +/// recover. +fn region(found: &Found) -> Result, VolError> { + let Located::Recover(provider) = found else { + return Ok(None); + }; + let block_size = provider.geometry().block_size; + provider + .control(|device| { + let (first, blocks) = device.region(); + Region { + first, + blocks, + whole: device.is_whole(), + block_size, + } + }) + .map(Some) + .map_err(|error| VolError::Host(format!("the volume's view: {error:?}"))) +} + +/// Find the user volume on the disk `path` names, as the boot does, and +/// open it to read or to write (the module docs). The hold, when writing, +/// must outlive the volume. +pub(crate) fn locate( + path: &Path, + access: Access, + block_size: Option, +) -> Result<(Found, Option), VolError> { + let disk = disk_of(path)?; + let hold = match access { + Access::ReadWrite => Some(DiskHold::take(&disk.node)?), + Access::ReadOnly => None, + }; + // The boot's decision, over the whole disk. + let found = classify(open(&disk.node, Access::ReadOnly, block_size)?)?; + let Some(decided) = region(&found)? else { + // Blank, or no kore volume: the caller says which. + return Ok((found, hold)); + }; + let per = u64::from(decided.block_size) / SYSFS_SECTOR; + let (start, sectors) = (decided.first * per, decided.blocks * per); + let at = if decided.whole { + Some(disk.node.clone()) + } else { + partition_at(&disk, start, sectors)? + }; + if let Some(named) = &disk.named + && (decided.whole || named.start != start || named.sectors != sectors) + { + return Err(VolError::Host(format!( + "{}: not where {}'s user volume is, which is {}", + named.node.display(), + disk.node.display(), + describe(at.as_deref(), &disk, decided) + ))); + } + let Some(mut hold) = hold else { + return Ok((found, None)); + }; + // The decision's descriptor, and its record lock, close before anything + // is claimed. + drop(found); + let block_size = Some(decided.block_size); + match open(&disk.node, Access::ReadWrite, block_size) { + Ok(device) => { + let found = classify(device)?; + if region(&found)? != Some(decided) { + return Err(changed(&disk.node)); + } + // A machine run on the volume's own partition holds no lock on + // the disk's node. + if let Some(partition) = at.as_deref().filter(|node| *node != disk.node) { + hold.guard_partition(partition)?; + } + return Ok((found, Some(hold))); + } + Err(VolError::Device(_, ImageError::Claimed)) if !decided.whole => {} + Err(error) => return Err(error), + } + // Something on the disk is mounted or claimed: claim the volume's + // partition alone. + let node = at.ok_or_else(|| { + VolError::Host(format!( + "{}: in use (something on it is mounted or claimed), and the kernel lists no \ + node for {} to write through alone", + disk.node.display(), + describe(None, &disk, decided) + )) + })?; + let found = classify(open(&node, Access::ReadWrite, block_size)?)?; + // Through the partition's own node, the volume is the whole medium. + let alone = Region { + first: 0, + whole: true, + ..decided + }; + if region(&found)? != Some(alone) { + return Err(changed(&node)); + } + // Decided again, with the partition claimed. + let again = classify(open(&disk.node, Access::ReadOnly, block_size)?)?; + if region(&again)? != Some(decided) { + return Err(changed(&disk.node)); + } + drop(again); + hold.guard_disk(&disk.node)?; + eprintln!( + "kore-vol: {} is in use: writing its user volume through {}", + disk.node.display(), + node.display() + ); + Ok((found, Some(hold))) +} + +fn describe(node: Option<&Path>, disk: &Disk, region: Region) -> String { + match node { + Some(node) if node == disk.node => format!("the whole disk, {}", node.display()), + Some(node) => node.display().to_string(), + None => format!( + "its partition at blocks {}..={} of {}", + region.first, + region.first + region.blocks - 1, + disk.node.display() + ), + } +} + +fn changed(node: &Path) -> VolError { + VolError::Host(format!( + "{}: the disk changed while kore-vol read it; nothing was written", + node.display() + )) +} diff --git a/native/tools/kore-vol/src/lib.rs b/native/tools/kore-vol/src/lib.rs new file mode 100644 index 0000000..cffa1eb --- /dev/null +++ b/native/tools/kore-vol/src/lib.rs @@ -0,0 +1,120 @@ +//! `kore-vol` — a kore volume, from the host. +//! +//! What a terminal's home holds, and files from the host copied into it: what +//! the host built (an application's bundle, a dataset) put where a terminal +//! reads it, with no system-level bootstrapping in between +//! (`res/DAGFS.md`, "From the host"). +//! +//! This library is the tool's whole procedure; the `kore-vol` binary is a +//! printer over it, and a host program that keeps machines (a VM supervisor +//! staging a project for a terminal, a packager) drives the same calls +//! instead of a second copy of them: +//! +//! - [`format()`] formats a blank image as the machine's first boot would. +//! - [`put`] installs a host file or tree under a terminal's home; [`rm`] +//! removes one. Each is one root publication, read back through a fresh +//! recovery before it reports success. +//! - [`read`] mounts the volume read-only and hands a [`Mount`] to a +//! closure: [`Mount::list`], [`Mount::read_to`] and [`Mount::walk`] answer +//! what a terminal's home holds, as data. +//! +//! - **Paths are a terminal's.** `path` is what terminal `N` (`face`, 0 for +//! the first) types, relative to its home (`work/face-N` on the user +//! volume): after `put dist js/hello`, terminal 0 reads the build as +//! `js/hello`. Reads also take a path from the volume's root when it starts +//! with `/`; nothing writes there. +//! - **The image** is a raw disk image, offline: the file a QEMU run boots +//! from after `qemu-img convert -O raw`, or a copy of a machine's disk. The +//! user volume on it is found exactly as the filesystem row finds it at +//! boot — the same composition and the same `locate` +//! (`kore_svc_dagfs::durable`): the whole disk when it is kore's, else the +//! GPT partition designated `KORE_FS`. Its block size is `block_size`, +//! [`DEFAULT_BLOCK_SIZE`] unless named. +//! - **A machine's disk** (Linux, as root): the disk's block device node, or +//! the node of the partition that holds its user volume, with the machine +//! shut down and the disk attached to the host running the tool (a live +//! system on that machine, or the disk in an enclosure). The volume is +//! found the same way, over the whole disk, at the device's own logical +//! block size; it is written unbuffered, with the disk held and claimed as +//! `device` describes, so never beside anything that mounts, claims or +//! runs a machine on it. +//! - **All or nothing.** Every check runs before anything is written, and +//! the install becomes visible in one root publication: a failure part +//! way, of the tool or of the host, leaves the volume as it was. Each +//! change is then read back through a fresh recovery and compared with its +//! source. +//! - **`format`** formats a blank volume — both root blocks zero — exactly +//! as the next boot would, so an image made with `qemu-img create` takes +//! an install before it first boots. Nothing else is ever formatted, and +//! a disk never is: its machine formats it at its first boot. +//! - **One format version.** A commit writes the format version this crate +//! was built with, and a volume another kore version wrote is refused as +//! [`VolError::Persistence`] with `Incompatible`: build the tool from the +//! tree the image boots. +//! +//! **One open volume per disk per process.** The image device's lock is a +//! POSIX record lock, which belongs to the process and is dropped when any +//! of its descriptors of the file closes. Two calls on one disk from one +//! process therefore never refuse each other — a [`put`] while a [`read`] +//! session holds the same image would write under the reader — and the +//! first to finish drops the other's lock, so a machine could then start +//! on a disk the process still has open. A host program serializes its own +//! calls per disk ([`VolError::is_in_use`] answers only for other processes). +//! +//! Every call runs the engine on a thread of its own with +//! [`VOLUME_STACK_BYTES`] of stack — the engine builds its tables on the +//! calling thread's stack while it recovers a volume — so a caller needs no +//! stack of its own beyond its usual one. + +#[cfg(target_os = "linux")] +mod device; +mod source; +mod volume; + +pub use kore_svc_dagfs::{ + Name, + bounds::{MAX_DIR_ENTRIES, MAX_OBJECTS, MAX_PATH_DEPTH}, +}; +pub use volume::{ + Formatted, Installed, Kind, Listed, Listing, Mount, MountStatus, Removed, VolError, Walked, + format, put, read, rm, +}; + +/// The on-disk format version this build's commits write. The engine mounts +/// a volume of this version or an older one it still reads, and refuses a +/// newer one as `Incompatible`: build the tool from the tree the image boots. +pub const FORMAT_VERSION: u32 = kore_svc_dagfs::persistence::FORMAT_VERSION; + +/// The logical block size an image is read with unless told otherwise: +/// QEMU's, for every disk image it creates, and so what kore formats a +/// QEMU disk with. A disk is read at its own, which a named block size must +/// agree with. +pub const DEFAULT_BLOCK_SIZE: u32 = 512; + +/// The engine builds its tables on the calling thread's stack while it +/// recovers a volume; the filesystem row places them on its heap for the +/// same reason (`DurableService::initialize`). A host thread's default 2 MiB +/// is not enough, 16 MiB is ample. +pub const VOLUME_STACK_BYTES: usize = 16 << 20; + +/// Whether `face` names a terminal: one whose home (`work/face-N`) the +/// filesystem row roots it at. +pub fn is_face(face: u8) -> bool { + kore_svc_dagfs::face_home_label(face).is_some() +} + +/// Run `work` on a thread with [`VOLUME_STACK_BYTES`] of stack, borrowing +/// what it needs from the caller, and hand back what it answered. +fn on_volume_stack( + work: impl FnOnce() -> Result + Send, +) -> Result { + std::thread::scope(|scope| { + std::thread::Builder::new() + .name("kore-vol".into()) + .stack_size(VOLUME_STACK_BYTES) + .spawn_scoped(scope, work) + .map_err(|error| VolError::Host(format!("cannot start the volume thread: {error}")))? + .join() + .unwrap_or_else(|_| Err(VolError::Host("the volume thread panicked".into()))) + }) +} diff --git a/native/tools/kore-vol/src/main.rs b/native/tools/kore-vol/src/main.rs new file mode 100644 index 0000000..c95f330 --- /dev/null +++ b/native/tools/kore-vol/src/main.rs @@ -0,0 +1,249 @@ +//! `kore-vol` — a kore volume, from the host: the command line over the +//! `kore_vol` library, which carries the whole procedure (see its docs). +//! +//! ```text +//! kore-vol [--face N] [--block-size B] ls [path] +//! kore-vol [--face N] [--block-size B] cat +//! kore-vol [--face N] [--block-size B] put [--replace] +//! kore-vol [--face N] [--block-size B] rm +//! kore-vol [--block-size B] format +//! ``` +//! +//! - **Paths are a terminal's.** `path` is what terminal `N` (default 0) +//! types, relative to its home (`work/face-N` on the user volume): after +//! `put dist js/hello`, terminal 0 reads the build as `js/hello`. `ls` and +//! `cat` also read from the volume's root when the path starts with `/`; +//! nothing writes there. +//! - **The image** is a raw disk image, offline: the file a QEMU run boots +//! from after `qemu-img convert -O raw` (`make vol-put-x86` wraps both +//! conversions), or a copy of a machine's disk; on Linux, as root, a +//! machine's disk itself, with the machine shut down. +//! - **All or nothing.** Every check runs before anything is written, and +//! the install becomes visible in one root publication, read back through +//! a fresh recovery before the tool reports it. +//! +//! Exit status: 0 done, 1 refused (with the reason on stderr), 2 a usage +//! error. + +use std::{path::PathBuf, process::ExitCode}; + +use kore_vol::{Kind, Listing, Mount, VolError}; + +const USAGE: &str = "\ +usage: kore-vol [--face N] [--block-size B] ls [path] + kore-vol [--face N] [--block-size B] cat + kore-vol [--face N] [--block-size B] put [--replace] + kore-vol [--face N] [--block-size B] rm + kore-vol [--block-size B] format"; + +enum Command { + Ls { + path: Option, + }, + Cat { + path: String, + }, + Put { + source: PathBuf, + path: String, + replace: bool, + }, + Rm { + path: String, + }, + Format, +} + +struct Invocation { + image: PathBuf, + face: u8, + block_size: Option, + command: Command, +} + +fn parse(mut args: impl Iterator) -> Result { + let mut face = 0; + let mut block_size = None; + let mut replace = false; + let mut positional = Vec::new(); + while let Some(arg) = args.next() { + match arg.as_str() { + "--face" => { + let value = args.next().ok_or("--face needs a terminal number")?; + face = value + .parse::() + .ok() + .filter(|face| kore_vol::is_face(*face)) + .ok_or_else(|| format!("--face {value}: not a terminal's face"))?; + } + "--block-size" => { + let value = args.next().ok_or("--block-size needs a size in bytes")?; + block_size = Some( + value + .parse::() + .ok() + .filter(|size| matches!(size, 512 | 4096)) + .ok_or_else(|| format!("--block-size {value}: 512 or 4096"))?, + ); + } + "--replace" => replace = true, + "-h" | "--help" => return Err(String::new()), + flag if flag.starts_with("--") => return Err(format!("unknown option {flag}")), + _ => positional.push(arg), + } + } + let mut positional = positional.into_iter(); + let image = PathBuf::from(positional.next().ok_or("no image named")?); + let command = match positional.next().as_deref() { + Some("ls") => Command::Ls { + path: positional.next(), + }, + Some("cat") => Command::Cat { + path: positional.next().ok_or("cat: no path")?, + }, + Some("put") => Command::Put { + source: PathBuf::from(positional.next().ok_or("put: no source")?), + path: positional.next().ok_or("put: no destination path")?, + replace, + }, + Some("rm") => Command::Rm { + path: positional.next().ok_or("rm: no path")?, + }, + Some("format") => Command::Format, + Some(other) => return Err(format!("unknown command {other}")), + None => return Err("no command".into()), + }; + if let Some(extra) = positional.next() { + return Err(format!("unexpected argument {extra}")); + } + if replace && !matches!(command, Command::Put { .. }) { + return Err("--replace applies to put alone".into()); + } + Ok(Invocation { + image, + face, + block_size, + command, + }) +} + +/// Say so when kore itself would mount the volume read-only: when recovery +/// had to take the fallback root. A medium opened to read mounts read-only +/// whatever its roots hold, so the mount's own mode says nothing here. +fn note_fallback(mount: &Mount) { + let status = mount.status(); + if status.used_fallback_root { + eprintln!( + "kore-vol: reading the fallback root (generation {}): kore mounts this volume read-only", + status.generation + ); + } +} + +/// A path as `ls` names it back: as given, from the volume's root when it +/// starts with `/`, and `.` for the home itself. +fn shown(path: Option<&str>) -> String { + match path { + Some(path) if path.starts_with('/') => { + let parts = path + .split('/') + .filter(|part| !part.is_empty()) + .collect::>(); + format!("/{}", parts.join("/")) + } + Some(path) => path.to_string(), + None => String::from("."), + } +} + +fn run(invocation: Invocation) -> Result<(), VolError> { + let Invocation { + image, + face, + block_size, + command, + } = invocation; + match command { + Command::Ls { path } => { + let listing = kore_vol::read(&image, block_size, |mount| { + note_fallback(mount); + mount.list(face, path.as_deref()) + })?; + match listing { + Listing::Directory(entries) => { + for entry in entries { + match entry.kind { + Kind::Directory => println!("dir {:>12} {}/", "", entry.name), + Kind::File => println!("file {:>12} {}", entry.len, entry.name), + } + } + } + Listing::File { len } => println!("file {:>12} {}", len, shown(path.as_deref())), + Listing::EmptyHome => {} + } + Ok(()) + } + Command::Cat { path } => kore_vol::read(&image, block_size, |mount| { + note_fallback(mount); + mount.read_to(face, &path, &mut std::io::stdout().lock()) + }) + .map(|_| ()), + Command::Put { + source, + path, + replace, + } => { + let installed = kore_vol::put(&image, block_size, face, &source, &path, replace)?; + println!( + "{path}: {} files, {} directories, {} bytes, in terminal {face}'s home \ + (generation {} -> {}); every file read back identical", + installed.files, + installed.directories, + installed.bytes, + installed.before, + installed.after + ); + Ok(()) + } + Command::Rm { path } => { + let removed = kore_vol::rm(&image, block_size, face, &path)?; + println!( + "{path}: removed from terminal {face}'s home (generation {} -> {})", + removed.before, removed.after + ); + Ok(()) + } + Command::Format => { + let formatted = kore_vol::format(&image, block_size)?; + println!( + "formatted as kore's first boot would (generation {})", + formatted.generation + ); + Ok(()) + } + } +} + +fn main() -> ExitCode { + let invocation = match parse(std::env::args().skip(1)) { + Ok(invocation) => invocation, + Err(message) => { + if !message.is_empty() { + eprintln!("kore-vol: {message}"); + } + eprintln!("{USAGE}"); + return ExitCode::from(2); + } + }; + match run(invocation) { + Ok(()) => ExitCode::SUCCESS, + Err(VolError::Output(error)) => { + eprintln!("kore-vol: standard output: {error}"); + ExitCode::FAILURE + } + Err(error) => { + eprintln!("kore-vol: {error}"); + ExitCode::FAILURE + } + } +} diff --git a/native/tools/kore-vol/src/source.rs b/native/tools/kore-vol/src/source.rs new file mode 100644 index 0000000..fd5c7fc --- /dev/null +++ b/native/tools/kore-vol/src/source.rs @@ -0,0 +1,119 @@ +//! The host side of a `put`: the tree to copy, read and checked whole before +//! the volume is opened, so a name the volume cannot hold or a directory too +//! wide for it refuses the install before any of it is written. + +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use kore_svc_dagfs::{Name, bounds::MAX_DIR_ENTRIES}; + +/// One object to create. +pub enum Entry { + File { name: Name, path: PathBuf, len: u64 }, + Directory { name: Name, entries: Vec }, +} + +impl Entry { + pub fn name(&self) -> &Name { + match self { + Self::File { name, .. } | Self::Directory { name, .. } => name, + } + } +} + +/// The tree under its destination name, and what it will cost the volume. +pub struct Plan { + pub root: Entry, + pub files: usize, + pub directories: usize, + pub bytes: u64, + /// Levels from the destination down to its deepest object, the + /// destination itself being one. + pub depth: usize, +} + +impl Plan { + pub fn objects(&self) -> usize { + self.files + self.directories + } +} + +#[derive(Default)] +struct Tally { + files: usize, + directories: usize, + bytes: u64, + depth: usize, +} + +/// Read `source` (a regular file or a directory) as the tree to create +/// under `name`. Symbolic links and special files are refused, not +/// followed: an install copies what the build wrote, and a link would copy +/// whatever it points at, from anywhere on the host. +pub fn plan(source: &Path, name: Name) -> Result { + let mut tally = Tally::default(); + let root = entry(source, name, 1, &mut tally)?; + Ok(Plan { + root, + files: tally.files, + directories: tally.directories, + bytes: tally.bytes, + depth: tally.depth, + }) +} + +fn entry(path: &Path, name: Name, depth: usize, tally: &mut Tally) -> Result { + let metadata = + fs::symlink_metadata(path).map_err(|error| format!("{}: {error}", path.display()))?; + tally.depth = tally.depth.max(depth); + let kind = metadata.file_type(); + if kind.is_file() { + tally.files += 1; + tally.bytes += metadata.len(); + return Ok(Entry::File { + name, + path: path.to_path_buf(), + len: metadata.len(), + }); + } + if !kind.is_dir() { + return Err(format!( + "{}: neither a file nor a directory (links and special files are not copied)", + path.display() + )); + } + tally.directories += 1; + let mut children = Vec::new(); + for child in fs::read_dir(path).map_err(|error| format!("{}: {error}", path.display()))? { + let child = child.map_err(|error| format!("{}: {error}", path.display()))?; + let child_path = child.path(); + let child_name = child + .file_name() + .to_str() + .and_then(|text| Name::new(text).ok()) + .ok_or_else(|| { + format!( + "{}: not a kore name (UTF-8, at most 32 bytes, no NUL)", + child_path.display() + ) + })?; + children.push((child_name, child_path)); + } + if children.len() > MAX_DIR_ENTRIES { + return Err(format!( + "{}: {} entries, and a kore directory holds at most {MAX_DIR_ENTRIES}", + path.display(), + children.len() + )); + } + // Created in name order, so two installs of one tree make the same + // volume. + children.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes())); + let mut entries = Vec::with_capacity(children.len()); + for (child_name, child_path) in children { + entries.push(entry(&child_path, child_name, depth + 1, tally)?); + } + Ok(Entry::Directory { name, entries }) +} diff --git a/native/tools/kore-vol/src/volume.rs b/native/tools/kore-vol/src/volume.rs new file mode 100644 index 0000000..56aee94 --- /dev/null +++ b/native/tools/kore-vol/src/volume.rs @@ -0,0 +1,1008 @@ +//! The volume side: found, mounted, changed and read back as the filesystem +//! row would. + +use std::{ + fmt, + fs::{self, File}, + io::{self, Read, Write}, + os::unix::fs::FileTypeExt, + path::{Path, PathBuf}, +}; + +use kore_driver::block::{ + CompletionMode, RegionDevice, + image::{Access, ImageError, ImageFile}, +}; +use kore_svc_dagfs::{ + ANY_REVISION, DurableService, FsError, Handle, Name, ObjectKind, OpenedObject, Rights, + SYSTEM_APPS_VIEW, SYSTEM_LIBS_VIEW, SYSTEM_SHARE_VIEW, SYSTEM_WORKSPACE_DIR, SessionId, + block_on, + bounds::{MAX_OBJECTS, MAX_PATH_DEPTH}, + durable::{Located, note}, + face_home_label, + persistence::{MountMode, PersistenceError, PersistentFs}, +}; + +use crate::{ + on_volume_stack, + source::{self, Entry}, +}; + +/// The provider's buffer and request bounds: the filesystem row's own +/// (`Located` in `durable.rs`), so the volume is driven +/// as the row drives it. +const BUFFERS: usize = 8; +const OUTSTANDING: usize = 32; + +type Volume = PersistentFs, BUFFERS, OUTSTANDING>; +pub(crate) type Found = Located; + +/// What keeps a machine's disk held while its volume is written +/// (`device::DiskHold`); nothing holds an image beyond its own lock. +#[cfg(target_os = "linux")] +type Hold = crate::device::DiskHold; +#[cfg(not(target_os = "linux"))] +type Hold = std::convert::Infallible; + +/// The session the tool acts in: the row's own boot session, rooted at the +/// persisted root. +const SESSION: SessionId = SessionId::new(0); + +/// One payload transfer: the provider's largest single write. +const CHUNK: usize = 64 * 1024; + +pub enum VolError { + /// The host side: a source that cannot be read, a usage the volume + /// cannot honour. + Host(String), + Image(ImageError), + /// A node of a machine's disk, and what opening it met. + Device(PathBuf, ImageError), + Blank, + NoVolume(u64), + Degraded { + generation: u64, + }, + Persistence(&'static str, PersistenceError), + Fs(String, FsError), + Exists(String), + NotFound(String), + Verify(String), + /// The caller's sink refused a file's bytes ([`Mount::read_to`]): the + /// volume is untouched, the output short. + Output(io::Error), +} + +impl fmt::Display for VolError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Host(message) => f.write_str(message), + Self::Image(ImageError::InUse) => f.write_str( + "the image is in use: a machine is running on it, or another tool holds it", + ), + Self::Image(ImageError::NotRegular | ImageError::NotDevice) => f.write_str( + "neither a disk image nor a disk: kore-vol takes a regular file, or on Linux \ + a disk's block device", + ), + Self::Image(ImageError::Claimed) => f.write_str("the image is claimed"), + Self::Image(ImageError::Unaligned { len, block_size }) => write!( + f, + "{len} bytes is not a whole number of {block_size}-byte blocks" + ), + Self::Image(ImageError::Geometry(error)) => write!(f, "image geometry: {error:?}"), + Self::Image(ImageError::Io(error)) => write!(f, "image: {error}"), + Self::Device(node, error) => { + write!(f, "{}: ", node.display())?; + match error { + ImageError::InUse => { + f.write_str("in use: a machine is running on it, or another tool holds it") + } + ImageError::Claimed => f.write_str( + "the kernel refused to claim it exclusively: something on it, or on \ + another partition of the same disk, is mounted, or another program \ + has claimed one of them", + ), + ImageError::NotDevice | ImageError::NotRegular => { + f.write_str("not a block device") + } + ImageError::Unaligned { len, block_size } => write!( + f, + "{len} bytes is not a whole number of {block_size}-byte blocks" + ), + ImageError::Geometry(error) => write!(f, "geometry: {error:?}"), + ImageError::Io(error) => write!(f, "{error}"), + } + } + Self::Blank => f.write_str( + "the volume is blank: the machine formats it at its first boot (and on an \ + image, `kore-vol format` formats it now as that boot would)", + ), + Self::NoVolume(note::VOLATILE_UNDESIGNATED) => f.write_str( + "no kore volume: the disk is not kore's, and its partition table designates \ + no kore filesystem partition", + ), + Self::NoVolume(_) => f.write_str( + "no kore volume: the designated kore filesystem partition holds foreign data, \ + or is too small for one", + ), + Self::Degraded { generation } => write!( + f, + "the volume mounts only from its fallback root (generation {generation}), \ + so kore keeps it read-only, and so does kore-vol" + ), + Self::Persistence(stage, PersistenceError::Incompatible) => write!( + f, + "{stage}: the volume's format or geometry is not this build's (another kore \ + version, another block size, or a resized image?)" + ), + Self::Persistence(stage, error) => write!(f, "{stage}: {error:?}"), + Self::Fs(path, error) => write!(f, "{path}: {}", describe(*error)), + Self::Exists(path) => write!(f, "{path} exists (put --replace replaces it)"), + Self::NotFound(path) => write!(f, "{path}: not found"), + Self::Verify(message) => write!(f, "read back after recovery: {message}"), + Self::Output(error) => write!(f, "the output: {error}"), + } + } +} + +impl VolError { + /// Whether the image or disk is held by a running machine or another + /// tool: QEMU's record lock on the images it runs, or another writer's. + pub fn is_in_use(&self) -> bool { + matches!( + self, + Self::Image(ImageError::InUse) | Self::Device(_, ImageError::InUse) + ) + } +} + +impl fmt::Debug for VolError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl std::error::Error for VolError {} + +impl From for VolError { + fn from(error: ImageError) -> Self { + Self::Image(error) + } +} + +/// The engine's refusals a host install meets, in its terms. +fn describe(error: FsError) -> String { + match error { + FsError::NotFound => "not found".into(), + FsError::NotDirectory => "not a directory".into(), + FsError::IsDirectory => "a directory".into(), + FsError::AlreadyExists => "exists".into(), + FsError::OutOfObjects => format!("the volume holds at most {MAX_OBJECTS} objects"), + FsError::OutOfEdges => "a directory or the volume is out of entries".into(), + FsError::OutOfStorage => "the volume has no contiguous room for it".into(), + FsError::Range => format!("deeper than kore's {MAX_PATH_DEPTH} levels"), + other => format!("{other:?}"), + } +} + +fn name(part: &str, path: &str) -> Result { + Name::new(part).map_err(|_| { + VolError::Host(format!( + "{path}: {part:?} is not a kore name (UTF-8, at most 32 bytes, no NUL)" + )) + }) +} + +/// A path as terminal N types it: components under its home. The first +/// component may not be a name the terminal's view already gives the +/// store's read-only grants: a real child of the home by that name would +/// hide the grant (`view_name_taken`). +fn components(path: &str) -> Result, VolError> { + let trimmed = path.trim_end_matches('/'); + if trimmed.is_empty() || trimmed.starts_with('/') { + return Err(VolError::Host(format!( + "{path}: a path under the terminal's home, like js/hello" + ))); + } + let names = trimmed + .split('/') + .map(|part| name(part, path)) + .collect::, _>>()?; + let first = names[0].as_str(); + if [SYSTEM_APPS_VIEW, SYSTEM_LIBS_VIEW, SYSTEM_SHARE_VIEW].contains(&first) { + return Err(VolError::Host(format!( + "{path}: {first} names the store's grant in a terminal's view" + ))); + } + Ok(names) +} + +fn display(names: &[Name]) -> String { + names.iter().map(Name::as_str).collect::>().join("/") +} + +/// `work/face-N`. +fn home(face: u8) -> Result<[Name; 2], VolError> { + let label = face_home_label(face) + .ok_or_else(|| VolError::Host(format!("face {face}: not a terminal's")))?; + let work = Name::new(SYSTEM_WORKSPACE_DIR).map_err(|_| VolError::Host("work".into()))?; + let face = Name::from_bytes(&label).map_err(|_| VolError::Host("face".into()))?; + Ok([work, face]) +} + +/// Whether `image` names a block device: a machine's disk, or a partition +/// of one, rather than an image of one. +fn is_device(image: &Path) -> Result { + fs::metadata(image) + .map(|meta| meta.file_type().is_block_device()) + .map_err(|error| VolError::Image(ImageError::Io(error))) +} + +/// Find the user volume as the filesystem row does at boot, on an image or, +/// on Linux, a machine's disk (`device::locate`), and what holds the disk +/// while it is written: bound before the volume, so that it outlives it. +fn locate( + image: &Path, + access: Access, + block_size: Option, +) -> Result<(Found, Option), VolError> { + if is_device(image)? { + #[cfg(target_os = "linux")] + return crate::device::locate(image, access, block_size); + #[cfg(not(target_os = "linux"))] + return Err(VolError::Host(format!( + "{}: a disk's block device, which kore-vol reads and writes on Linux alone", + image.display() + ))); + } + let device = ImageFile::open( + image, + access, + block_size.unwrap_or(crate::DEFAULT_BLOCK_SIZE), + )?; + Ok((classify(device)?, None)) +} + +/// Find the user volume on `device` as the filesystem row does at boot: the +/// same composition, then the same `locate`. +pub(crate) fn classify(device: ImageFile) -> Result { + let provider = + DurableService::whole_medium_provider(device, CompletionMode::Poll).map_err(|failure| { + VolError::Host(format!("the medium refused composition: {failure:?}")) + })?; + block_on(DurableService::locate(provider, CompletionMode::Poll)) + .map_err(|failure| VolError::Host(format!("the medium refused discovery: {failure:?}"))) +} + +/// Recover what [`locate`] found, refusing a volume kore itself would only +/// read when `writing`. +fn recover(found: Found, writing: bool) -> Result { + let provider = match found { + Located::Recover(provider) => provider, + Located::Format { .. } => return Err(VolError::Blank), + Located::Volatile(why) => return Err(VolError::NoVolume(why)), + // The system region's policy: `locate` never answers it. + Located::Reformat { .. } => { + return Err(VolError::NoVolume(note::VOLATILE_REFUSED_REGION)); + } + }; + let volume = block_on(Volume::recover(provider, SESSION)) + .map_err(|error| VolError::Persistence("recover", error))?; + let status = volume.status(); + if writing && (status.used_fallback_root || status.mode != MountMode::ReadWrite) { + return Err(VolError::Degraded { + generation: status.generation, + }); + } + Ok(volume) +} + +/// Publish the change, then mount again from what is on the image: both +/// roots re-read and the namespace rebuilt from them, as the next boot will. +/// +/// The change is committed twice. The first root publication makes it +/// durable; the second overwrites the other root, which still names the +/// volume as it was, so recovery has no pre-change fallback to choose. +fn publish(mut volume: Volume) -> Result { + block_on(volume.commit(SESSION)).map_err(|error| VolError::Persistence("commit", error))?; + let generation = + block_on(volume.commit(SESSION)).map_err(|error| VolError::Persistence("commit", error))?; + let volume = block_on(volume.restart(SESSION)) + .map_err(|error| VolError::Persistence("recover after commit", error))?; + let status = volume.status(); + if status.generation != generation || status.used_fallback_root { + return Err(VolError::Verify(format!( + "recovery chose generation {}{}, not the {generation} just committed", + status.generation, + if status.used_fallback_root { + " (the fallback root)" + } else { + "" + } + ))); + } + Ok(volume) +} + +fn fs(shown: &str, result: Result) -> Result { + result.map_err(|error| VolError::Fs(shown.to_string(), error)) +} + +/// Walk from the persisted root along `names`, every step a directory, +/// holding one handle at a time. Missing directories are created when +/// `create`, else the walk answers `None`. +/// +/// Every handle it opens is either answered or closed, whatever a step +/// meets: a session holds few, and a [`Mount`] lives as long as its caller +/// browses. +fn walk( + volume: &Volume, + names: &[Name], + rights: Rights, + create: bool, +) -> Result, VolError> { + let engine = volume.engine(); + let mut at = fs("/", engine.open_root(SESSION, rights))?.handle; + for (depth, part) in names.iter().enumerate() { + let shown = display(&names[..=depth]); + let next = match engine.open_at(SESSION, at, part, rights) { + Ok(opened) => Ok(Some(opened.handle)), + Err(FsError::NotFound) if create => fs( + &shown, + block_on(engine.create(SESSION, at, part, ObjectKind::Directory)), + ) + .map(|created| Some(created.handle)), + Err(FsError::NotFound) => Ok(None), + Err(error) => Err(VolError::Fs(shown.clone(), error)), + }; + let closed = fs(&shown, engine.close(SESSION, at)); + match (next, closed) { + (Ok(Some(next)), Ok(())) => at = next, + (Ok(None), Ok(())) => return Ok(None), + (Ok(Some(next)), Err(error)) => { + let _ = engine.close(SESSION, next); + return Err(error); + } + (Ok(None), Err(error)) | (Err(error), _) => return Err(error), + } + } + Ok(Some(at)) +} + +// ── reading: a mount and what it answers ──────────────────────────────── + +/// What recovery found when the volume was mounted to read. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MountStatus { + /// The root generation the mount reads. + pub generation: u64, + /// Recovery had to take the fallback root: kore itself mounts this + /// volume read-only, and nothing writes it. + pub used_fallback_root: bool, +} + +/// What a listed object is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Kind { + File, + Directory, +} + +/// One entry of a listed directory. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Listed { + pub name: String, + pub kind: Kind, + /// A file's length in bytes; 0 for a directory. + pub len: u64, +} + +/// What [`Mount::list`] found at a path. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Listing { + /// A directory's entries, in the volume's order. + Directory(Vec), + /// A file, and its length in bytes. + File { len: u64 }, + /// The terminal's home itself, which nothing has written to yet: the + /// row creates it on the boot that first roots a terminal there. + EmptyHome, +} + +/// One object beneath a walked directory ([`Mount::walk`]). +#[derive(Clone, Copy, Debug)] +pub struct Walked<'a> { + /// Its path below the walked directory, `/`-separated. + pub path: &'a str, + pub kind: Kind, + /// A file's length in bytes; 0 for a directory. + pub len: u64, + /// Levels below the walked directory: 1 for its own entries. + pub depth: usize, +} + +/// A volume recovered read-only, as [`read`] hands it to its caller. +/// +/// It lives on the volume's own thread for the length of the call; what the +/// caller takes away is data. +pub struct Mount { + volume: Volume, +} + +/// Mount `image` read-only and run `session` against it. Kore keeps a volume +/// whose recovery took the fallback root read-only; the mount reads it, and +/// says so in [`Mount::status`]. +pub fn read( + image: &Path, + block_size: Option, + session: impl FnOnce(&Mount) -> Result + Send, +) -> Result { + on_volume_stack(|| { + // Nothing holds a disk that is only read. + let (found, _) = locate(image, Access::ReadOnly, block_size)?; + let mount = Mount { + volume: recover(found, false)?, + }; + session(&mount) + }) +} + +/// A path to read: under terminal `face`'s home, or from the volume's root +/// when it starts with `/`. The names, and the path as given. +fn readable(face: u8, path: Option<&str>) -> Result<(Vec, String), VolError> { + let mut names = Vec::new(); + let shown = match path { + Some(path) if path.starts_with('/') => { + for part in path.split('/').filter(|part| !part.is_empty()) { + names.push(name(part, path)?); + } + format!("/{}", display(&names)) + } + Some(path) => { + names.extend(home(face)?); + names.extend(components(path)?); + path.to_string() + } + None => { + names.extend(home(face)?); + String::from(".") + } + }; + Ok((names, shown)) +} + +impl Mount { + /// The mount's root generation, and whether recovery took the fallback + /// root. + pub fn status(&self) -> MountStatus { + let status = self.volume.status(); + MountStatus { + generation: status.generation, + used_fallback_root: status.used_fallback_root, + } + } + + /// What terminal `face` sees at `path` (its home when `None`). + pub fn list(&self, face: u8, path: Option<&str>) -> Result { + let (names, shown) = readable(face, path)?; + let engine = self.volume.engine(); + let Some(at) = walk(&self.volume, &names, Rights::READ, false)? else { + if path.is_none() { + return Ok(Listing::EmptyHome); + } + return Err(VolError::NotFound(shown)); + }; + // The handle closes whatever the listing meets. + let listing = self.listing(at, &shown); + let closed = fs(&shown, engine.close(SESSION, at)); + let listing = listing?; + closed?; + Ok(listing) + } + + fn listing(&self, at: Handle, shown: &str) -> Result { + let engine = self.volume.engine(); + let stat = fs(shown, engine.stat(SESSION, at))?; + if !stat.kind.is_dir() { + return Ok(Listing::File { len: stat.len }); + } + let mut entries = Vec::new(); + let mut cursor = 0; + loop { + let page = fs(shown, engine.list(SESSION, at, cursor))?; + for entry in &page.entries { + if entry.kind.is_dir() { + entries.push(Listed { + name: entry.name.as_str().to_owned(), + kind: Kind::Directory, + len: 0, + }); + continue; + } + let opened = fs( + shown, + engine.open_at(SESSION, at, &entry.name, Rights::READ), + )?; + fs(shown, engine.close(SESSION, opened.handle))?; + entries.push(Listed { + name: entry.name.as_str().to_owned(), + kind: Kind::File, + len: opened.len, + }); + } + match page.next { + Some(next) => cursor = next, + None => return Ok(Listing::Directory(entries)), + } + } + } + + /// Copy the file terminal `face` sees at `path` into `out`, and answer + /// how many bytes it held. + pub fn read_to( + &self, + face: u8, + path: &str, + out: &mut W, + ) -> Result { + let (names, shown) = readable(face, Some(path))?; + let engine = self.volume.engine(); + let (last, parents) = names + .split_last() + .ok_or_else(|| VolError::NotFound(shown.clone()))?; + let parent = walk(&self.volume, parents, Rights::READ, false)? + .ok_or_else(|| VolError::NotFound(shown.clone()))?; + let opened = engine.open_at(SESSION, parent, last, Rights::READ); + let closed = fs(&shown, engine.close(SESSION, parent)); + let file = match opened { + Ok(opened) => opened, + Err(FsError::NotFound) => return Err(VolError::NotFound(shown)), + Err(error) => return Err(VolError::Fs(shown, error)), + }; + // The file's handle closes whatever the copy meets: a sink that has + // had enough (a preview's bound) is an everyday end, not a leak. + let copied = match closed { + Ok(()) if file.kind.is_dir() => Err(VolError::Fs(shown.clone(), FsError::IsDirectory)), + Ok(()) => self.copy(&file, &shown, out), + Err(error) => Err(error), + }; + let released = fs(&shown, engine.close(SESSION, file.handle)); + let len = copied?; + released?; + Ok(len) + } + + fn copy( + &self, + file: &OpenedObject, + shown: &str, + out: &mut W, + ) -> Result { + let engine = self.volume.engine(); + let mut chunk = vec![0; CHUNK]; + let mut offset = 0; + while offset < file.len { + let read = fs( + shown, + block_on(engine.read(SESSION, file.handle, offset, &mut chunk)), + )?; + if read == 0 { + break; + } + out.write_all(&chunk[..read]).map_err(VolError::Output)?; + offset += read as u64; + } + out.flush().map_err(VolError::Output)?; + Ok(offset) + } + + /// Visit every object beneath the directory terminal `face` sees at + /// `path`, depth first, each directory before what it holds, in the + /// volume's order. The walk holds one handle per level, so it is bounded + /// by kore's [`MAX_PATH_DEPTH`]; `visit` refusing stops it with that + /// refusal. + pub fn walk( + &self, + face: u8, + path: &str, + visit: &mut dyn FnMut(Walked<'_>) -> Result<(), VolError>, + ) -> Result<(), VolError> { + let (names, shown) = readable(face, Some(path))?; + let engine = self.volume.engine(); + let at = walk(&self.volume, &names, Rights::READ, false)? + .ok_or_else(|| VolError::NotFound(shown.clone()))?; + let walked = match fs(&shown, engine.stat(SESSION, at)) { + Ok(stat) if stat.kind.is_dir() => { + self.descend(at, &shown, &mut String::new(), 1, visit) + } + Ok(_) => Err(VolError::Fs(shown.clone(), FsError::NotDirectory)), + Err(error) => Err(error), + }; + let closed = fs(&shown, engine.close(SESSION, at)); + walked.and(closed) + } + + fn descend( + &self, + at: Handle, + shown: &str, + prefix: &mut String, + depth: usize, + visit: &mut dyn FnMut(Walked<'_>) -> Result<(), VolError>, + ) -> Result<(), VolError> { + let engine = self.volume.engine(); + let mut cursor = 0; + loop { + let page = fs(shown, engine.list(SESSION, at, cursor))?; + for entry in &page.entries { + let mark = prefix.len(); + if !prefix.is_empty() { + prefix.push('/'); + } + prefix.push_str(entry.name.as_str()); + let opened = match engine.open_at(SESSION, at, &entry.name, Rights::READ) { + Ok(opened) => opened, + Err(error) => return Err(VolError::Fs(format!("{shown}/{prefix}"), error)), + }; + let visited = if opened.kind.is_dir() { + visit(Walked { + path: prefix, + kind: Kind::Directory, + len: 0, + depth, + }) + .and_then(|()| self.descend(opened.handle, shown, prefix, depth + 1, visit)) + } else { + visit(Walked { + path: prefix, + kind: Kind::File, + len: opened.len, + depth, + }) + }; + let closed = fs(shown, engine.close(SESSION, opened.handle)); + prefix.truncate(mark); + visited?; + closed?; + } + match page.next { + Some(next) => cursor = next, + None => return Ok(()), + } + } + } +} + +// ── put ───────────────────────────────────────────────────────────────── + +/// What [`put`] installed, and the publication it made. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Installed { + pub files: usize, + pub directories: usize, + pub bytes: u64, + /// The root generation before the install, and the one that names it. + pub before: u64, + pub after: u64, +} + +/// A file as written: what its read-back must match. +struct Written { + path: Vec, + len: u64, + digest: blake3::Hash, +} + +/// Install `source` (a regular file or a directory) at `path` under +/// terminal `face`'s home; with `replace`, in place of what is there. +/// Symbolic links and special files are refused, not followed. +pub fn put( + image: &Path, + block_size: Option, + face: u8, + source: &Path, + path: &str, + replace: bool, +) -> Result { + on_volume_stack(|| install(image, block_size, face, source, path, replace)) +} + +fn install( + image: &Path, + block_size: Option, + face: u8, + source: &Path, + path: &str, + replace: bool, +) -> Result { + let target = components(path)?; + let (last, parents) = target.split_last().expect("a path has a component"); + let plan = source::plan(source, *last).map_err(VolError::Host)?; + let mut route = home(face)?.to_vec(); + route.extend_from_slice(parents); + // Depth as the engine counts it: edges from the persisted root. + let deepest = route.len() + plan.depth; + if deepest > MAX_PATH_DEPTH { + return Err(VolError::Host(format!( + "{path}: its deepest object would sit {deepest} levels below the volume's root, \ + past kore's {MAX_PATH_DEPTH}" + ))); + } + let (found, _disk) = locate(image, Access::ReadWrite, block_size)?; + let volume = recover(found, true)?; + let before = volume.status().generation; + let parent = walk(&volume, &route, Rights::ALL, true)?.ok_or_else(|| { + VolError::Host(format!("{path}: a directory created on the way vanished")) + })?; + let engine = volume.engine(); + match engine.open_at(SESSION, parent, last, Rights::READ) { + Ok(existing) => { + fs(path, engine.close(SESSION, existing.handle))?; + if !replace { + return Err(VolError::Exists(path.to_string())); + } + // The old tree's blocks stay as they are until the commit that + // stops naming them has itself been superseded: the provider + // retires a detached extent only after two root publications. + fs(path, block_on(engine.detach_edge(SESSION, parent, last)))?; + } + Err(FsError::NotFound) => {} + Err(error) => return Err(VolError::Fs(path.to_string(), error)), + } + let room = MAX_OBJECTS.saturating_sub(engine.object_count()); + if plan.objects() > room { + return Err(VolError::Host(format!( + "{path}: {} objects, and the volume has room for {room} more of its {MAX_OBJECTS}", + plan.objects() + ))); + } + let mut written = Vec::with_capacity(plan.files); + let mut at = target.clone(); + create(&volume, parent, &plan.root, &mut at, &mut written)?; + fs(path, volume.engine().close(SESSION, parent))?; + let volume = publish(volume)?; + verify(&volume, face, &written)?; + Ok(Installed { + files: plan.files, + directories: plan.directories, + bytes: plan.bytes, + before, + after: volume.status().generation, + }) +} + +/// Create `entry` under `parent`, depth first, holding one handle per level +/// beneath it. +fn create( + volume: &Volume, + parent: Handle, + entry: &Entry, + path: &mut Vec, + written: &mut Vec, +) -> Result<(), VolError> { + let engine = volume.engine(); + let shown = display(path); + match entry { + Entry::Directory { name, entries } => { + let handle = fs( + &shown, + block_on(engine.create(SESSION, parent, name, ObjectKind::Directory)), + )? + .handle; + for child in entries { + path.push(*child.name()); + create(volume, handle, child, path, written)?; + path.pop(); + } + fs(&shown, engine.close(SESSION, handle)) + } + Entry::File { + name, + path: source, + len, + } => { + let handle = fs( + &shown, + block_on(engine.create(SESSION, parent, name, ObjectKind::File)), + )? + .handle; + fs( + &shown, + block_on(engine.resize(SESSION, handle, *len, ANY_REVISION)), + )?; + let mut file = File::open(source) + .map_err(|error| VolError::Host(format!("{}: {error}", source.display())))?; + let mut hasher = blake3::Hasher::new(); + let mut chunk = vec![0; CHUNK]; + let mut offset = 0; + while offset < *len { + let want = CHUNK.min((*len - offset) as usize); + file.read_exact(&mut chunk[..want]).map_err(|error| { + VolError::Host(format!("{}: {error} while copying", source.display())) + })?; + hasher.update(&chunk[..want]); + fs( + &shown, + block_on(engine.write_at( + SESSION, + handle, + offset, + &chunk[..want], + ANY_REVISION, + )), + )?; + offset += want as u64; + } + // A file that grew after it was measured would be copied short. + let more = file + .read(&mut chunk[..1]) + .map_err(|error| VolError::Host(format!("{}: {error}", source.display())))?; + if more != 0 { + return Err(VolError::Host(format!( + "{}: changed while it was copied", + source.display() + ))); + } + fs(&shown, engine.close(SESSION, handle))?; + written.push(Written { + path: path.clone(), + len: *len, + digest: hasher.finalize(), + }); + Ok(()) + } + } +} + +/// Read every written file back through the recovered volume and compare +/// it with what was written. +fn verify(volume: &Volume, face: u8, written: &[Written]) -> Result<(), VolError> { + let engine = volume.engine(); + let mut chunk = vec![0; CHUNK]; + for file in written { + let shown = display(&file.path); + let mut route = home(face)?.to_vec(); + route.extend_from_slice(&file.path); + let handle = walk(volume, &route, Rights::READ, false)? + .ok_or_else(|| VolError::Verify(format!("{shown} is missing")))?; + let len = fs(&shown, engine.stat(SESSION, handle))?.len; + if len != file.len { + return Err(VolError::Verify(format!( + "{shown} holds {len} bytes, not {}", + file.len + ))); + } + let mut hasher = blake3::Hasher::new(); + let mut offset = 0; + while offset < len { + let read = fs( + &shown, + block_on(engine.read(SESSION, handle, offset, &mut chunk)), + )?; + if read == 0 { + return Err(VolError::Verify(format!( + "{shown} ends at {offset} of its {len} bytes" + ))); + } + hasher.update(&chunk[..read]); + offset += read as u64; + } + fs(&shown, engine.close(SESSION, handle))?; + if hasher.finalize() != file.digest { + return Err(VolError::Verify(format!("{shown} differs from its source"))); + } + } + Ok(()) +} + +// ── rm ────────────────────────────────────────────────────────────────── + +/// What [`rm`] removed: the publication that stopped naming it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Removed { + pub before: u64, + pub after: u64, +} + +/// Remove `path` (and what it holds) from terminal `face`'s home. +pub fn rm( + image: &Path, + block_size: Option, + face: u8, + path: &str, +) -> Result { + on_volume_stack(|| remove(image, block_size, face, path)) +} + +fn remove( + image: &Path, + block_size: Option, + face: u8, + path: &str, +) -> Result { + let target = components(path)?; + let (last, parents) = target.split_last().expect("a path has a component"); + let (found, _disk) = locate(image, Access::ReadWrite, block_size)?; + let volume = recover(found, true)?; + let before = volume.status().generation; + let mut route = home(face)?.to_vec(); + route.extend_from_slice(parents); + let parent = walk(&volume, &route, Rights::ALL, false)? + .ok_or_else(|| VolError::NotFound(path.to_string()))?; + let engine = volume.engine(); + match block_on(engine.detach_edge(SESSION, parent, last)) { + Ok(()) => {} + Err(FsError::NotFound) => return Err(VolError::NotFound(path.to_string())), + Err(error) => return Err(VolError::Fs(path.to_string(), error)), + } + fs(path, engine.close(SESSION, parent))?; + let volume = publish(volume)?; + let mut full = home(face)?.to_vec(); + full.extend_from_slice(&target); + if walk(&volume, &full, Rights::READ, false)?.is_some() { + return Err(VolError::Verify(format!("{path} is still there"))); + } + Ok(Removed { + before, + after: volume.status().generation, + }) +} + +// ── format ────────────────────────────────────────────────────────────── + +/// What [`format()`] made: a volume at its first generation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Formatted { + pub generation: u64, +} + +/// Format a blank image as kore's first boot would. Nothing else is ever +/// formatted: a volume kore already wrote, foreign data and any disk are +/// refused. +pub fn format(image: &Path, block_size: Option) -> Result { + on_volume_stack(|| format_blank(image, block_size)) +} + +fn format_blank(image: &Path, block_size: Option) -> Result { + if is_device(image)? { + return Err(VolError::Host(format!( + "{}: a disk, which kore-vol never formats: the machine formats a blank volume at \ + its first boot, and partitioning a disk is tools/provision_fs.py's", + image.display() + ))); + } + let (found, _) = locate(image, Access::ReadWrite, block_size)?; + let (provider, era) = match found { + Located::Format { provider, era } => (provider, era), + Located::Recover(_) => { + return Err(VolError::Host( + "the volume is kore's already: nothing formatted".into(), + )); + } + Located::Volatile(why) => return Err(VolError::NoVolume(why)), + Located::Reformat { .. } => { + return Err(VolError::NoVolume(note::VOLATILE_REFUSED_REGION)); + } + }; + let volume = block_on(Volume::format_with_era(provider, SESSION, era)) + .map_err(|error| VolError::Persistence("format", error))?; + let volume = block_on(volume.restart(SESSION)) + .map_err(|error| VolError::Persistence("recover after format", error))?; + Ok(Formatted { + generation: volume.status().generation, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_held_image_or_disk_is_in_use_and_nothing_else_is() { + assert!(VolError::Image(ImageError::InUse).is_in_use()); + assert!(VolError::Device(PathBuf::from("/dev/nvme0n1"), ImageError::InUse).is_in_use()); + assert!(!VolError::Image(ImageError::Claimed).is_in_use()); + assert!(!VolError::Blank.is_in_use()); + assert!(!VolError::Host("in use".into()).is_in_use()); + } +} diff --git a/native/tools/kore-vol/tests/cli.rs b/native/tools/kore-vol/tests/cli.rs new file mode 100644 index 0000000..05a36b8 --- /dev/null +++ b/native/tools/kore-vol/tests/cli.rs @@ -0,0 +1,316 @@ +//! `kore-vol` as its user runs it: the built binary, on disk images made +//! here, read back through the binary itself and, where the claim is about +//! bytes, through the image file. + +mod common; + +use std::{ + fs::{self, File, OpenOptions}, + os::unix::fs::FileExt, + path::{Path, PathBuf}, +}; + +use common::*; +use kore_driver::gpt::{KORE_FS_TYPE, synthetic}; + +#[test] +fn a_blank_image_is_formatted_then_takes_an_install_its_replacement_and_its_removal() { + let dir = scratch("lifecycle"); + let image = blank_image(&dir, 64 * MIB); + let image = image.to_str().unwrap(); + let first = bundle(&dir, 1); + let second = bundle(&dir, 2); + fs::remove_file(second.join("vite.svg")).unwrap(); + let (first, second) = (first.to_str().unwrap(), second.to_str().unwrap()); + + // Blank roots: kore formats them at its first boot, and nothing else + // does unless asked. + refused(&[image, "put", first, "js/hello"], "blank"); + refused(&[image, "ls"], "blank"); + assert!( + read(Path::new(image), 0, 2 * BLOCK as usize) + .iter() + .all(|&b| b == 0) + ); + assert!(ok(&[image, "format"]).contains("formatted")); + refused(&[image, "format"], "kore's already"); + + // A home nothing was ever written to is empty, not missing. + assert_eq!(ok(&[image, "ls"]), ""); + + let installed = ok(&[image, "put", first, "js/hello"]); + assert!(installed.contains("7 files, 3 directories"), "{installed}"); + assert!(installed.contains("read back identical"), "{installed}"); + // A healthy volume read says nothing beside what it lists: opened to + // read, it mounts read-only, which is not kore's fallback posture. + let quiet = kore_vol(&[image, "ls", "js/hello"]); + assert_eq!((quiet.code, quiet.stderr.as_str()), (0, "")); + let listed = ok(&[image, "ls", "js/hello"]); + assert_eq!( + listed.lines().collect::>(), + [ + "dir assets/", + "file 431 index.html", + "file 1 vite.svg", + ] + ); + let fonts = ok(&[image, "ls", "js/hello/assets/fonts"]); + assert!(fonts.contains("file 0 .keep"), "{fonts}"); + assert!(fonts.contains("file 300001 inter.woff2"), "{fonts}"); + assert!(ok(&[image, "ls", "/work"]).contains("face-0/")); + + // Installing over an install is a decision, not an accident. + refused(&[image, "put", second, "js/hello"], "exists"); + assert!(ok(&[image, "ls", "js/hello"]).contains("vite.svg")); + ok(&[image, "put", "--replace", second, "js/hello"]); + let replaced = ok(&[image, "ls", "js/hello"]); + assert!(!replaced.contains("vite.svg"), "{replaced}"); + assert!(replaced.contains("index.html"), "{replaced}"); + + // Another terminal's home is its own. + ok(&["--face", "1", image, "put", first, "notes"]); + assert_eq!(ok(&[image, "ls", "/work"]).lines().count(), 2); + refused(&[image, "ls", "notes"], "not found"); + + ok(&[image, "rm", "js/hello"]); + refused(&[image, "ls", "js/hello"], "not found"); + assert_eq!(ok(&[image, "ls", "js"]), ""); + refused(&[image, "rm", "js/hello"], "not found"); +} + +#[test] +fn the_install_lands_inside_the_designated_partition_and_nowhere_else() { + let dir = scratch("gpt"); + let image = blank_image(&dir, 32 * MIB); + let blocks = 32 * MIB / BLOCK; + let (start, end) = (2048, blocks - 2048); + let mut mbr = [0u8; 512]; + mbr[510] = 0x55; + mbr[511] = 0xaa; + let (header, entries) = synthetic::table(KORE_FS_TYPE, start, end, blocks - 34); + { + let file = OpenOptions::new().write(true).open(&image).unwrap(); + file.write_all_at(&mbr, 0).unwrap(); + file.write_all_at(&header, BLOCK).unwrap(); + file.write_all_at(&entries, 2 * BLOCK).unwrap(); + } + let before = read(&image, 0, (start * BLOCK) as usize); + let tail = (blocks - end - 1) * BLOCK; + let image_path = image.to_str().unwrap(); + let source = bundle(&dir, 3); + + ok(&[image_path, "format"]); + ok(&[image_path, "put", source.to_str().unwrap(), "app"]); + assert!(ok(&[image_path, "ls", "app/assets"]).contains("index.js")); + + // The table and everything ahead of the partition is as it was; so is + // everything after it; the partition's own first block holds a root. + assert_eq!(read(&image, 0, (start * BLOCK) as usize), before); + assert!( + read(&image, (end + 1) * BLOCK, tail as usize) + .iter() + .all(|&b| b == 0) + ); + assert_eq!(&read(&image, start * BLOCK, 8)[..], b"KOREROOT"); +} + +#[test] +fn a_foreign_disk_is_refused_untouched() { + let dir = scratch("foreign"); + let image = blank_image(&dir, 16 * MIB); + OpenOptions::new() + .write(true) + .open(&image) + .unwrap() + .write_all_at(&pattern(9, 4096), 0) + .unwrap(); + let before = digest(&image); + let image_path = image.to_str().unwrap(); + refused(&[image_path, "format"], "no kore volume"); + refused(&[image_path, "ls"], "no kore volume"); + assert_eq!(digest(&image), before); +} + +#[test] +fn an_image_another_process_holds_is_refused() { + use rustix::fs::{FlockOperation, fcntl_lock}; + let dir = scratch("lock"); + let image = blank_image(&dir, 16 * MIB); + let image_path = image.to_str().unwrap(); + ok(&[image_path, "format"]); + // As QEMU holds the images it runs: a record lock on the file. + let held = OpenOptions::new() + .read(true) + .write(true) + .open(&image) + .unwrap(); + fcntl_lock(&held, FlockOperation::NonBlockingLockExclusive).unwrap(); + refused(&[image_path, "ls"], "in use"); + refused(&[image_path, "put", dir.to_str().unwrap(), "x"], "in use"); + drop(held); + ok(&[image_path, "ls"]); +} + +#[test] +fn what_the_volume_cannot_hold_is_refused_before_the_image_is_opened() { + let dir = scratch("names"); + let image = blank_image(&dir, 16 * MIB); + let image_path = image.to_str().unwrap(); + ok(&[image_path, "format"]); + let before = digest(&image); + + let long = dir.join("long"); + fs::create_dir_all(&long).unwrap(); + fs::write(long.join("a-name-of-thirty-three-bytes.txt0"), b"x").unwrap(); + refused( + &[image_path, "put", long.to_str().unwrap(), "x"], + "not a kore name", + ); + + let wide = dir.join("wide"); + fs::create_dir_all(&wide).unwrap(); + for index in 0..129 { + fs::write(wide.join(format!("{index}")), b"").unwrap(); + } + refused( + &[image_path, "put", wide.to_str().unwrap(), "x"], + "at most 128", + ); + + #[cfg(unix)] + { + let linked = dir.join("linked"); + fs::create_dir_all(&linked).unwrap(); + std::os::unix::fs::symlink("/etc/hostname", linked.join("hostname")).unwrap(); + refused(&[image_path, "put", linked.to_str().unwrap(), "x"], "links"); + } + + let small = bundle(&dir, 4); + refused( + &[image_path, "put", small.to_str().unwrap(), "shared/x"], + "store's grant", + ); + refused( + &[image_path, "put", small.to_str().unwrap(), "/abs"], + "under the terminal's home", + ); + assert_eq!(digest(&image), before); +} + +/// A chain of `levels` directories under `root` with a file at the bottom: +/// a tree `levels + 1` deep, counting the destination as one. +fn chain(root: &Path, levels: usize) -> PathBuf { + let mut at = root.to_path_buf(); + for level in 1..levels { + at = at.join(format!("{level}")); + } + fs::create_dir_all(&at).unwrap(); + fs::write(at.join("bottom"), b"deepest").unwrap(); + root.to_path_buf() +} + +#[test] +fn a_tree_as_deep_as_kore_allows_installs_and_one_deeper_is_refused_first() { + let dir = scratch("depth"); + let image = blank_image(&dir, 16 * MIB); + let image_path = image.to_str().unwrap(); + ok(&[image_path, "format"]); + // The home is two levels below the volume's root (`work/face-0`), and + // kore allows 32: thirty below the home, the destination among them. + let deepest = chain(&dir.join("deepest"), 29); + ok(&[image_path, "put", deepest.to_str().unwrap(), "d"]); + let mut path = String::from("d"); + for level in 1..29 { + path.push_str(&format!("/{level}")); + } + path.push_str("/bottom"); + assert_eq!(ok(&[image_path, "cat", &path]), "deepest"); + + let before = digest(&image); + let deeper = chain(&dir.join("deeper"), 30); + refused( + &[image_path, "put", deeper.to_str().unwrap(), "e"], + "33 levels", + ); + assert_eq!(digest(&image), before); +} + +#[test] +fn an_install_that_fails_part_way_leaves_the_volume_as_it_was() { + let dir = scratch("atomic"); + let image = blank_image(&dir, 16 * MIB); + let image_path = image.to_str().unwrap(); + ok(&[image_path, "format"]); + let small = bundle(&dir, 5); + ok(&[image_path, "put", small.to_str().unwrap(), "kept"]); + let home = ok(&[image_path, "ls"]); + + // Three files in name order; the second cannot fit, after the first + // was written. + let big = dir.join("big"); + fs::create_dir_all(&big).unwrap(); + fs::write(big.join("a.bin"), pattern(6, 200_000)).unwrap(); + File::create(big.join("b.bin")) + .unwrap() + .set_len(32 * MIB) + .unwrap(); + fs::write(big.join("c.bin"), pattern(7, 10)).unwrap(); + refused( + &[image_path, "put", big.to_str().unwrap(), "big"], + "no contiguous room", + ); + + assert_eq!(ok(&[image_path, "ls"]), home); + refused(&[image_path, "ls", "big"], "not found"); + assert!(ok(&[image_path, "ls", "kept"]).contains("index.html")); +} + +#[test] +fn both_roots_name_the_install() { + let dir = scratch("roots"); + let image = blank_image(&dir, 16 * MIB); + let image_path = image.to_str().unwrap(); + ok(&[image_path, "format"]); + let source = bundle(&dir, 8); + let installed = ok(&[image_path, "put", source.to_str().unwrap(), "js/app"]); + // Formatting published generation 1; the install, 2 and 3. The newest + // root is generation 3's, in slot 3 & 1: block 1. + assert!(installed.contains("generation 1 -> 3"), "{installed}"); + let file = OpenOptions::new() + .read(true) + .write(true) + .open(&image) + .unwrap(); + let mut root = [0u8; 512]; + file.read_exact_at(&mut root, BLOCK).unwrap(); + root[200] ^= 0xff; + file.write_all_at(&root, BLOCK).unwrap(); + drop(file); + + // Recovery falls back to generation 2, which names the install too. + let run = kore_vol(&[image_path, "ls", "js/app"]); + assert_eq!(run.code, 0, "{}", run.stderr); + assert!( + run.stderr.contains("fallback root (generation 2)"), + "{}", + run.stderr + ); + assert!(run.stdout.contains("index.html"), "{}", run.stdout); + // And kore would not write a volume it mounts from its fallback. + refused(&[image_path, "rm", "js/app"], "fallback root"); +} + +#[test] +fn usage_errors_exit_two() { + for args in [ + &[][..], + &["disk.raw"][..], + &["disk.raw", "put", "dist"][..], + &["disk.raw", "ls", "a", "b"][..], + &["--face", "10", "disk.raw", "ls"][..], + &["--block-size", "1024", "disk.raw", "ls"][..], + &["--replace", "disk.raw", "ls"][..], + ] { + assert_eq!(kore_vol(args).code, 2, "{args:?}"); + } +} diff --git a/native/tools/kore-vol/tests/common/mod.rs b/native/tools/kore-vol/tests/common/mod.rs new file mode 100644 index 0000000..18af537 --- /dev/null +++ b/native/tools/kore-vol/tests/common/mod.rs @@ -0,0 +1,122 @@ +//! What the suites share: the built binary run as its user runs it, the +//! scratch it works in, and bundles shaped like a built application. + +#![allow(dead_code)] + +use std::{ + fs::{self, File}, + os::unix::fs::FileExt, + path::{Path, PathBuf}, + process::Command, +}; + +pub const MIB: u64 = 1 << 20; +pub const BLOCK: u64 = 512; + +pub struct Run { + pub code: i32, + pub stdout: String, + pub stderr: String, +} + +pub fn kore_vol(args: &[&str]) -> Run { + let output = Command::new(env!("CARGO_BIN_EXE_kore-vol")) + .args(args) + .output() + .expect("kore-vol runs"); + Run { + code: output.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + } +} + +/// `kore-vol` with `args`, which must succeed; its standard output. +pub fn ok(args: &[&str]) -> String { + let run = kore_vol(args); + assert_eq!( + run.code, 0, + "kore-vol {args:?} failed: {}{}", + run.stdout, run.stderr + ); + run.stdout +} + +/// `kore-vol` with `args`, which must refuse with a message naming `why`. +pub fn refused(args: &[&str], why: &str) { + let run = kore_vol(args); + assert_eq!( + run.code, 1, + "kore-vol {args:?} should refuse: {}", + run.stdout + ); + assert!( + run.stderr.contains(why), + "kore-vol {args:?}: expected {why:?} in {:?}", + run.stderr + ); +} + +/// A fresh directory of this test's own under cargo's per-target scratch. +pub fn scratch(test: &str) -> PathBuf { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("kore-vol-{test}")); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir +} + +/// A zeroed (sparse) image: blank roots, as `qemu-img create` makes one. +pub fn blank_image(dir: &Path, bytes: u64) -> PathBuf { + let path = dir.join("disk.raw"); + File::create(&path).unwrap().set_len(bytes).unwrap(); + path +} + +/// Deterministic, position-dependent bytes: a misplaced or truncated chunk +/// cannot read back equal. +pub fn pattern(seed: u8, len: usize) -> Vec { + (0..len) + .map(|at| (at as u32).wrapping_mul(2_654_435_761).to_le_bytes()[1] ^ seed) + .collect() +} + +/// A built application's shape: a document, assets beside it, a nested +/// directory, an empty file, and files either side of the 64 KiB write +/// chunk. +pub fn bundle(dir: &Path, seed: u8) -> PathBuf { + let root = dir.join(format!("dist-{seed}")); + fs::create_dir_all(root.join("assets/fonts")).unwrap(); + fs::write(root.join("index.html"), pattern(seed, 431)).unwrap(); + fs::write(root.join("vite.svg"), pattern(seed ^ 1, 1)).unwrap(); + fs::write(root.join("assets/index.js"), pattern(seed ^ 2, 65_535)).unwrap(); + fs::write(root.join("assets/index.css"), pattern(seed ^ 3, 65_536)).unwrap(); + fs::write(root.join("assets/chunk.js"), pattern(seed ^ 4, 65_537)).unwrap(); + fs::write( + root.join("assets/fonts/inter.woff2"), + pattern(seed ^ 5, 300_001), + ) + .unwrap(); + fs::write(root.join("assets/fonts/.keep"), b"").unwrap(); + root +} + +pub fn read(path: &Path, at: u64, len: usize) -> Vec { + let mut bytes = vec![0; len]; + File::open(path) + .unwrap() + .read_exact_at(&mut bytes, at) + .unwrap(); + bytes +} + +pub fn digest(path: &Path) -> Vec { + let bytes = fs::read(path).unwrap(); + // Not a security digest: equality of two reads of one file. + let mut sum = vec![0u8; 32]; + for (at, byte) in bytes.iter().enumerate() { + let slot = &mut sum[at % 32]; + *slot = slot.wrapping_mul(31).wrapping_add(*byte); + } + sum.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); + sum +} diff --git a/native/tools/kore-vol/tests/device.rs b/native/tools/kore-vol/tests/device.rs new file mode 100644 index 0000000..8f81ba0 --- /dev/null +++ b/native/tools/kore-vol/tests/device.rs @@ -0,0 +1,298 @@ +//! `kore-vol` on a machine's disk, as its user runs it on Linux: the built +//! binary against loop devices over disk images made here. The partitions a +//! disk's table describes are listed as a kernel lists them: read from the +//! table by a kernel that reads GPT, else added as the kernel adds them +//! (BLKPG, by util-linux's `addpart`). What holds a disk in the field (a +//! mounted filesystem's claim, QEMU's record locks, a partitioner's BSD +//! lock) the test process holds here, while the binary runs as another. +//! +//! Needs root, loop devices and util-linux: +//! +//! ```text +//! sudo -E cargo test -p kore-vol --test device -- --ignored --test-threads=1 +//! ``` + +#![cfg(target_os = "linux")] + +mod common; + +use std::{ + fs::{File, OpenOptions}, + os::{fd::OwnedFd, unix::fs::FileExt}, + path::{Path, PathBuf}, + process::Command, + thread, + time::{Duration, Instant}, +}; + +use common::*; +use kore_driver::gpt::{KORE_FS_TYPE, KORE_SYSTEM_TYPE, synthetic}; +use rustix::fs::{FlockOperation, Mode, OFlags, fcntl_lock, flock, open}; + +/// The disk the suite builds: a user partition, a gap, a system partition. +const DISK_BYTES: u64 = 96 * MIB; +const USER: (u64, u64) = (2048, 133_119); +const SYSTEM: (u64, u64) = (135_168, 176_127); + +/// A disk image with a GPT holding a user and a system partition, its user +/// volume formatted as the machine's first boot would. +fn gpt_disk(dir: &Path) -> PathBuf { + let image = blank_image(dir, DISK_BYTES); + let mut mbr = [0u8; 512]; + mbr[510] = 0x55; + mbr[511] = 0xaa; + let (header, entries) = synthetic::table_pair( + (KORE_FS_TYPE, USER.0, USER.1), + Some((KORE_SYSTEM_TYPE, SYSTEM.0, SYSTEM.1)), + DISK_BYTES / BLOCK - 34, + ); + let file = OpenOptions::new().write(true).open(&image).unwrap(); + file.write_all_at(&mbr, 0).unwrap(); + file.write_all_at(&header, BLOCK).unwrap(); + file.write_all_at(&entries, 2 * BLOCK).unwrap(); + drop(file); + ok(&[image.to_str().unwrap(), "format"]); + image +} + +/// Everything on the image outside the user partition. +fn outside_user(image: &Path) -> (Vec, Vec) { + let tail = DISK_BYTES - (USER.1 + 1) * BLOCK; + ( + read(image, 0, (USER.0 * BLOCK) as usize), + read(image, (USER.1 + 1) * BLOCK, tail as usize), + ) +} + +/// A loop device over a disk image, detached when dropped, listing the +/// partitions given (the module docs). +struct Disk { + node: PathBuf, +} + +impl Disk { + fn attach(image: &Path, sector_size: u32, partitions: &[(u64, u64)]) -> Self { + let mut losetup = Command::new("losetup"); + losetup.args([ + "--find", + "--show", + "--sector-size", + §or_size.to_string(), + ]); + if !partitions.is_empty() { + losetup.arg("--partscan"); + } + let output = losetup.arg(image).output().expect("losetup runs"); + assert!( + output.status.success(), + "losetup: {}", + String::from_utf8_lossy(&output.stderr) + ); + let disk = Self { + node: PathBuf::from(String::from_utf8(output.stdout).unwrap().trim()), + }; + for (index, &(first, last)) in partitions.iter().enumerate() { + let node = disk.partition(index + 1); + if !node.exists() { + let added = Command::new("addpart") + .arg(&disk.node) + .arg((index + 1).to_string()) + .arg(first.to_string()) + .arg((last - first + 1).to_string()) + .status() + .expect("addpart runs"); + assert!(added.success(), "addpart {}", node.display()); + } + let sysfs = Path::new("/sys/class/block").join(node.file_name().unwrap()); + let number = |name: &str| { + std::fs::read_to_string(sysfs.join(name)) + .unwrap() + .trim() + .parse::() + .unwrap() + }; + assert_eq!((number("start"), number("size")), (first, last - first + 1)); + } + disk + } + + fn partition(&self, number: usize) -> PathBuf { + PathBuf::from(format!("{}p{number}", self.node.display())) + } + + fn path(&self) -> &str { + self.node.to_str().unwrap() + } +} + +impl Drop for Disk { + fn drop(&mut self) { + let _ = Command::new("losetup") + .arg("--detach") + .arg(&self.node) + .status(); + } +} + +/// An exclusive claim on `node`, as a mounted filesystem holds one. +fn mounted(node: &Path) -> OwnedFd { + open( + node, + OFlags::RDONLY | OFlags::EXCL | OFlags::CLOEXEC, + Mode::empty(), + ) + .unwrap() +} + +/// A record lock on `node`, as QEMU holds the nodes it runs a machine on. +fn machine(node: &Path) -> File { + let file = File::open(node).unwrap(); + fcntl_lock(&file, FlockOperation::NonBlockingLockShared).unwrap(); + file +} + +/// The disk's BSD lock, as a partitioner holds it (exclusive) or udev +/// does around each event (shared). +fn bsd_lock(node: &Path, operation: FlockOperation) -> File { + let file = File::open(node).unwrap(); + flock(&file, operation).unwrap(); + file +} + +#[test] +#[ignore = "needs root, loop devices and util-linux: the module docs"] +fn a_disk_is_written_whole_inside_its_user_partition_alone() { + let dir = scratch("device-whole"); + let image = gpt_disk(&dir); + let before = outside_user(&image); + let source = bundle(&dir, 11); + { + let disk = Disk::attach(&image, 512, &[]); + let installed = ok(&[disk.path(), "put", source.to_str().unwrap(), "js/app"]); + assert!(installed.contains("7 files, 3 directories"), "{installed}"); + assert!(installed.contains("read back identical"), "{installed}"); + let listed = kore_vol(&[disk.path(), "ls", "js/app/assets"]); + assert_eq!((listed.code, listed.stderr.as_str()), (0, "")); + assert!(listed.stdout.contains("index.js"), "{}", listed.stdout); + refused(&[disk.path(), "format"], "never formats"); + refused( + &["--block-size", "4096", disk.path(), "ls"], + "logical blocks are 512 bytes", + ); + } + // What the next boot mounts: the disk read as an image. + assert!(ok(&[image.to_str().unwrap(), "ls", "js/app"]).contains("index.html")); + assert_eq!(outside_user(&image), before); + assert_eq!(&read(&image, USER.0 * BLOCK, 8)[..], b"KOREROOT"); +} + +#[test] +#[ignore = "needs root, loop devices and util-linux: the module docs"] +fn a_disk_with_a_partition_in_use_is_written_through_its_user_partition() { + let dir = scratch("device-partition"); + let image = gpt_disk(&dir); + let before = outside_user(&image); + let source = bundle(&dir, 12); + let source = source.to_str().unwrap(); + let disk = Disk::attach(&image, 512, &[USER, SYSTEM]); + let (user, system) = (disk.partition(1), disk.partition(2)); + let user_path = user.to_str().unwrap(); + + // Naming the system partition names the disk, and the wrong partition. + let elsewhere = format!("which is {user_path}"); + refused( + &[system.to_str().unwrap(), "put", source, "never"], + &elsewhere, + ); + refused(&[system.to_str().unwrap(), "ls"], &elsewhere); + + { + // Something mounted from the system partition: the disk cannot be + // claimed whole, and the user partition is claimed alone. + let _system = mounted(&system); + let run = kore_vol(&[disk.path(), "put", source, "through"]); + assert_eq!(run.code, 0, "{}", run.stderr); + assert!( + run.stderr + .contains(&format!("writing its user volume through {user_path}")), + "{}", + run.stderr + ); + // A machine run on the user partition is refused. + let _machine = machine(&user); + refused(&[disk.path(), "put", source, "never"], "in use"); + } + { + // The user partition itself in use: nothing to write through. + let _user = mounted(&user); + refused(&[disk.path(), "put", source, "never"], "refused to claim"); + } + // The disk free: claimed whole, whichever of its nodes is named, and a + // machine run on the user partition is refused all the same. + ok(&[user_path, "put", source, "whole"]); + { + let _machine = machine(&user); + refused(&[disk.path(), "put", source, "never"], "in use"); + } + let home = ok(&[user_path, "ls"]); + assert!( + home.contains("through/") && home.contains("whole/") && !home.contains("never"), + "{home}" + ); + drop(disk); + assert_eq!(outside_user(&image), before); +} + +#[test] +#[ignore = "needs root, loop devices and util-linux: the module docs"] +fn a_disk_a_machine_or_a_partitioner_holds_is_refused() { + let dir = scratch("device-held"); + let image = gpt_disk(&dir); + let source = bundle(&dir, 13); + let source = source.to_str().unwrap(); + let disk = Disk::attach(&image, 512, &[]); + { + let _machine = machine(&disk.node); + refused(&[disk.path(), "put", source, "never"], "in use"); + // Read beside it, as an image QEMU runs is. + ok(&[disk.path(), "ls"]); + } + { + let _partitioner = bsd_lock(&disk.node, FlockOperation::LockExclusive); + let started = Instant::now(); + refused(&[disk.path(), "put", source, "never"], "BSD lock"); + assert!(started.elapsed() >= Duration::from_secs(5)); + } + { + // udev's shared hold for one event is waited out. + let udev = bsd_lock(&disk.node, FlockOperation::LockShared); + let release = thread::spawn(move || { + thread::sleep(Duration::from_millis(300)); + drop(udev); + }); + ok(&[disk.path(), "put", source, "after"]); + release.join().unwrap(); + } + let home = ok(&[disk.path(), "ls"]); + assert!(home.contains("after/") && !home.contains("never"), "{home}"); +} + +#[test] +#[ignore = "needs root, loop devices and util-linux: the module docs"] +fn a_disk_is_read_and_written_at_its_own_block_size() { + let dir = scratch("device-4k"); + let image = blank_image(&dir, 32 * MIB); + let image_path = image.to_str().unwrap(); + ok(&["--block-size", "4096", image_path, "format"]); + let source = bundle(&dir, 14); + { + let disk = Disk::attach(&image, 4096, &[]); + ok(&[disk.path(), "put", source.to_str().unwrap(), "app"]); + assert!(ok(&["--block-size", "4096", disk.path(), "ls", "app"]).contains("index.html")); + refused( + &["--block-size", "512", disk.path(), "ls"], + "logical blocks are 4096 bytes", + ); + } + assert!(ok(&["--block-size", "4096", image_path, "ls", "app"]).contains("index.html")); +} diff --git a/native/tools/kore-vol/tests/library.rs b/native/tools/kore-vol/tests/library.rs new file mode 100644 index 0000000..1b8be79 --- /dev/null +++ b/native/tools/kore-vol/tests/library.rs @@ -0,0 +1,261 @@ +//! `kore_vol` as a host program drives it: the same procedure the binary +//! prints over, answered as data — what a VM supervisor or a volume browser +//! reads back off a disk a machine left. + +mod common; + +use std::{collections::BTreeMap, fs, path::Path}; + +use common::*; +use kore_vol::{Kind, Listing, VolError}; + +/// Every file under `root`, by its `/`-separated path below it. +fn host_files(root: &Path) -> BTreeMap> { + fn visit(root: &Path, at: &Path, files: &mut BTreeMap>) { + for entry in fs::read_dir(at).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + visit(root, &path, files); + } else { + let relative = path.strip_prefix(root).unwrap().to_str().unwrap(); + files.insert(relative.replace('\\', "/"), fs::read(&path).unwrap()); + } + } + } + let mut files = BTreeMap::new(); + visit(root, root, &mut files); + files +} + +#[test] +fn a_tree_put_through_the_library_reads_back_whole_through_a_mount() { + let dir = scratch("library-roundtrip"); + let image = blank_image(&dir, 64 * MIB); + let source = bundle(&dir, 7); + + let formatted = kore_vol::format(&image, None).unwrap(); + let installed = kore_vol::put(&image, None, 0, &source, "js/hello", false).unwrap(); + assert_eq!(installed.files, 7); + assert_eq!(installed.directories, 3); + assert!(installed.before >= formatted.generation); + assert!(installed.after > installed.before); + + let expected = host_files(&source); + let (listing, files, walked_dirs) = kore_vol::read(&image, None, |mount| { + assert_eq!(mount.status().generation, installed.after); + assert!(!mount.status().used_fallback_root); + let listing = mount.list(0, Some("js/hello"))?; + let mut files = BTreeMap::new(); + let mut dirs = Vec::new(); + let mut paths = Vec::new(); + mount.walk(0, "js/hello", &mut |walked| { + paths.push(walked.path.to_owned()); + assert_eq!(walked.depth, walked.path.split('/').count()); + match walked.kind { + Kind::Directory => dirs.push(walked.path.to_owned()), + Kind::File => { + let mut bytes = Vec::new(); + let len = mount.read_to(0, &format!("js/hello/{}", walked.path), &mut bytes)?; + assert_eq!(len, walked.len); + files.insert(walked.path.to_owned(), bytes); + } + } + Ok(()) + })?; + // Each directory is visited before what it holds. + for dir in &dirs { + let at = paths.iter().position(|path| path == dir).unwrap(); + assert!( + paths[..at] + .iter() + .all(|path| !path.starts_with(&format!("{dir}/"))) + ); + } + Ok((listing, files, dirs)) + }) + .unwrap(); + + let Listing::Directory(entries) = listing else { + panic!("js/hello is a directory"); + }; + let mut names = entries + .iter() + .map(|entry| (entry.name.as_str(), entry.kind)) + .collect::>(); + names.sort_by_key(|(name, _)| *name); + assert_eq!( + names, + [ + ("assets", Kind::Directory), + ("index.html", Kind::File), + ("vite.svg", Kind::File) + ] + ); + assert_eq!(files, expected); + let mut walked_dirs = walked_dirs; + walked_dirs.sort(); + assert_eq!(walked_dirs, ["assets", "assets/fonts"]); +} + +#[test] +fn a_mount_answers_files_homes_and_the_volume_root_as_data() { + let dir = scratch("library-listing"); + let image = blank_image(&dir, 32 * MIB); + let source = dir.join("notes.txt"); + fs::write(&source, b"kore\n").unwrap(); + kore_vol::format(&image, None).unwrap(); + + // A home nothing has written to is empty, not missing. + let home = kore_vol::read(&image, None, |mount| mount.list(1, None)).unwrap(); + assert_eq!(home, Listing::EmptyHome); + + kore_vol::put(&image, None, 1, &source, "notes.txt", false).unwrap(); + kore_vol::read(&image, None, |mount| { + assert_eq!(mount.list(1, Some("notes.txt"))?, Listing::File { len: 5 }); + let Listing::Directory(root) = mount.list(0, Some("/"))? else { + panic!("the volume's root is a directory"); + }; + assert!(root.iter().any(|entry| entry.name == "work")); + let Listing::Directory(faces) = mount.list(0, Some("/work"))? else { + panic!("work is a directory"); + }; + assert!(faces.iter().any(|entry| entry.name == "face-1")); + // What is not there is said to be not there, and a file is not a + // directory to walk nor a directory a file to read. + assert!(matches!( + mount.list(1, Some("missing")), + Err(VolError::NotFound(_)) + )); + assert!(matches!( + mount.walk(1, "notes.txt", &mut |_| Ok(())), + Err(VolError::Fs(_, _)) + )); + assert!(matches!( + mount.read_to(0, "/work", &mut Vec::new()), + Err(VolError::Fs(_, _)) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn a_visit_that_refuses_stops_the_walk_with_its_refusal() { + let dir = scratch("library-refusal"); + let image = blank_image(&dir, 64 * MIB); + let source = bundle(&dir, 9); + kore_vol::format(&image, None).unwrap(); + kore_vol::put(&image, None, 0, &source, "app", false).unwrap(); + let mut seen = 0; + let outcome = kore_vol::read(&image, None, |mount| { + mount.walk(0, "app", &mut |_| { + seen += 1; + Err(VolError::Host("enough".into())) + }) + }); + assert!(matches!(outcome, Err(VolError::Host(message)) if message == "enough")); + assert_eq!(seen, 1); +} + +#[test] +fn a_blank_image_is_refused_to_read_and_the_caller_needs_no_stack_of_its_own() { + let dir = scratch("library-stack"); + let image = blank_image(&dir, 32 * MIB); + // The engine recovers on the library's own thread: a caller on a small + // stack is as good as any. + let small = std::thread::Builder::new() + .stack_size(128 * 1024) + .spawn(move || { + let blank = kore_vol::read(&image, None, |_| Ok(())); + assert!(matches!(blank, Err(VolError::Blank))); + kore_vol::format(&image, None).unwrap(); + kore_vol::read(&image, None, |mount| Ok(mount.status().generation)).unwrap() + }) + .unwrap(); + assert!(small.join().unwrap() >= 1); +} + +#[test] +fn a_sink_that_fails_is_the_output_s_refusal_not_the_volume_s() { + struct Closed; + impl std::io::Write for Closed { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::ErrorKind::BrokenPipe.into()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let dir = scratch("library-sink"); + let image = blank_image(&dir, 32 * MIB); + let source = dir.join("blob"); + fs::write(&source, pattern(3, 70_000)).unwrap(); + kore_vol::format(&image, None).unwrap(); + kore_vol::put(&image, None, 0, &source, "blob", false).unwrap(); + let outcome = kore_vol::read(&image, None, |mount| mount.read_to(0, "blob", &mut Closed)); + assert!(matches!(outcome, Err(VolError::Output(_)))); + // The volume is as it was. + let mut bytes = Vec::new(); + kore_vol::read(&image, None, |mount| mount.read_to(0, "blob", &mut bytes)).unwrap(); + assert_eq!(bytes, pattern(3, 70_000)); +} + +#[test] +fn a_mount_releases_its_handles_whatever_a_read_or_a_listing_meets() { + /// Takes one byte, then has had enough: a preview's bound. + struct Enough(usize); + impl std::io::Write for Enough { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if self.0 == 0 { + return Err(std::io::Error::other("enough")); + } + self.0 -= 1; + Ok(buf.len().min(1)) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let dir = scratch("library-handles"); + let image = blank_image(&dir, 32 * MIB); + let source = dir.join("blob"); + fs::write(&source, pattern(5, 70_000)).unwrap(); + kore_vol::format(&image, None).unwrap(); + kore_vol::put(&image, None, 0, &source, "shots/blob", false).unwrap(); + // A session holds at most `MAX_HANDLES_PER_SESSION` (32) handles; each + // refusal below, repeated well past that, must hand back what it opened. + kore_vol::read(&image, None, |mount| { + for _ in 0..100 { + assert!(matches!( + mount.read_to(0, "shots/blob", &mut Enough(1)), + Err(VolError::Output(_)) + )); + assert!(matches!( + mount.read_to(0, "shots/missing", &mut Vec::new()), + Err(VolError::NotFound(_)) + )); + assert!(matches!( + mount.read_to(0, "shots", &mut Vec::new()), + Err(VolError::Fs(_, _)) + )); + assert!(matches!( + mount.list(0, Some("shots/missing")), + Err(VolError::NotFound(_)) + )); + assert!(matches!( + mount.walk(0, "shots/blob", &mut |_| Ok(())), + Err(VolError::Fs(_, _)) + )); + } + // And the session is still whole. + let Listing::Directory(entries) = mount.list(0, Some("shots"))? else { + panic!("shots is a directory"); + }; + assert_eq!(entries.len(), 1); + let mut bytes = Vec::new(); + assert_eq!(mount.read_to(0, "shots/blob", &mut bytes)?, 70_000); + assert_eq!(bytes, pattern(5, 70_000)); + Ok(()) + }) + .unwrap(); +}