diff --git a/core/continuum-core/src/ipc/start_server/resource_authority.rs b/core/continuum-core/src/ipc/start_server/resource_authority.rs index 5de2edb661..327f007e21 100644 --- a/core/continuum-core/src/ipc/start_server/resource_authority.rs +++ b/core/continuum-core/src/ipc/start_server/resource_authority.rs @@ -109,15 +109,27 @@ pub(super) fn build_model_catalog_and_resource_daemon( // responsible for not double-counting") — measured live as VRAM // advertising 16.9 GB while RAM correctly reported 0. Both axes come // from UnifiedMemoryPool below instead. - crate::gpu::monitor::MemoryMode::Unified => Some(monitor.total_bytes()), + Some(crate::gpu::monitor::MemoryMode::Unified) => Some(monitor.total_bytes()), // Discrete: the axes really are separate pools. Unchanged. - crate::gpu::monitor::MemoryMode::Discrete => { + Some(crate::gpu::monitor::MemoryMode::Discrete) => { capacity_sources.push(Arc::new(crate::resources::GpuCapacitySource::new( monitor, GPU_SAFETY_RESERVE_BYTES, ))); None } + // The backend did not report its memory mode: whether VRAM and RAM are one pool + // or two is unknown, so VRAM stays UNGOVERNED (serving fails closed), named at + // every boot. Never a guessed topology (card 13c81ced). + None => { + continuum_base::probe!( + class = "resources.memory_mode_unknown", + platform = monitor.platform(), + device = monitor.device_name(), + "the GPU backend reports no memory mode: VRAM is ungoverned until its adapter reports one" + ); + None + } }; } None => { diff --git a/core/continuum-kernel/src/gpu/backends/metal/mod.rs b/core/continuum-kernel/src/gpu/backends/metal/mod.rs index 806a978a13..6ed818b203 100644 --- a/core/continuum-kernel/src/gpu/backends/metal/mod.rs +++ b/core/continuum-kernel/src/gpu/backends/metal/mod.rs @@ -167,8 +167,8 @@ impl GpuDeviceProbe for MetalProbe { /// signal that tells the ResourceGovernor whether its VRAM and RAM ledgers /// describe one physical pool or two — never inferred from `target_os`, since /// an Intel Mac with a discrete AMD card also runs macOS and is NOT unified. - fn memory_mode(&self) -> MemoryMode { - self.memory_mode + fn memory_mode(&self) -> Option { + Some(self.memory_mode) } /// Two Mach syscalls (Unified) or one Metal property read (Discrete). In-process @@ -386,7 +386,7 @@ mod tests { }; assert_eq!( monitor.memory_mode(), - expected, + Some(expected), "MetalMonitor memory_mode must agree with the device's hasUnifiedMemory at construction time" ); } diff --git a/core/continuum-kernel/src/gpu/backends/nvidia.rs b/core/continuum-kernel/src/gpu/backends/nvidia.rs index bf0a35a1a1..5ce4e00126 100644 --- a/core/continuum-kernel/src/gpu/backends/nvidia.rs +++ b/core/continuum-kernel/src/gpu/backends/nvidia.rs @@ -120,6 +120,12 @@ impl GpuDeviceProbe for NvidiaProbe { self.total_bytes } + /// nvidia-smi reports the card's own framebuffer (`memory.total`), memory the host does not + /// share: a dedicated pool. + fn memory_mode(&self) -> Option { + Some(crate::gpu::monitor::MemoryMode::Discrete) + } + /// Two `nvidia-smi` child processes, awaited off the reactor /// (`tokio::process`), so this never blocks a runtime thread. /// diff --git a/core/continuum-kernel/src/gpu/device_probe.rs b/core/continuum-kernel/src/gpu/device_probe.rs index 4197bce5f9..cb2f0f102b 100644 --- a/core/continuum-kernel/src/gpu/device_probe.rs +++ b/core/continuum-kernel/src/gpu/device_probe.rs @@ -95,8 +95,9 @@ pub trait GpuDeviceProbe: Send + Sync + 'static { /// ARM iGPUs) or owns separate VRAM. The governor's VRAM and RAM ledgers are the /// same pool or two pools depending on this answer, so it is DETECTED by the /// backend, never inferred from `target_os`. - fn memory_mode(&self) -> MemoryMode { - MemoryMode::Discrete + /// `None`: not reported, so unknown (see [`super::monitor::GpuMonitor::memory_mode`]). + fn memory_mode(&self) -> Option { + None } /// How often to sample. See [`DEFAULT_TICK`]. @@ -240,7 +241,7 @@ impl GpuMonitor for MonitoredGpu

{ self.probe.platform() } - fn memory_mode(&self) -> MemoryMode { + fn memory_mode(&self) -> Option { self.probe.memory_mode() } diff --git a/core/continuum-kernel/src/gpu/monitor.rs b/core/continuum-kernel/src/gpu/monitor.rs index 513cc8b5ea..1a14cf0f37 100644 --- a/core/continuum-kernel/src/gpu/monitor.rs +++ b/core/continuum-kernel/src/gpu/monitor.rs @@ -67,15 +67,16 @@ pub trait GpuMonitor: Send + Sync { /// Platform identifier — "metal" | "cuda" | "vulkan" | "cpu" | "mock". fn platform(&self) -> &'static str; - /// Whether this device shares the host's physical memory. + /// Whether this device shares the host's physical memory, as the BACKEND reports it. /// - /// Defaults to [`MemoryMode::Discrete`] — the correct answer for every discrete - /// card, and the behavior the governor had before unified memory was modeled at - /// all, so an adapter that does not override this is unchanged rather than - /// silently wrong. Metal overrides it with the real `hasUnifiedMemory` answer. - /// A future UMA backend (an ARM iGPU via Vulkan) MUST override it. - fn memory_mode(&self) -> MemoryMode { - MemoryMode::Discrete + /// `None` (the default) is UNKNOWN, never a guess: a backend that does not report its mode + /// gets no governed VRAM at all, named at boot, so nothing is planned on an assumed topology + /// (Codex on card 13c81ced: "unknown mode must remain explicit, no guessed Apple/APU + /// defaults"). It used to default to `Discrete`, which on a shared-memory device (an AMD APU, + /// an ARM iGPU) silently built two ledgers over one pool. Metal reports `hasUnifiedMemory`; + /// NVIDIA reports its card's dedicated memory; every other backend MUST report its own. + fn memory_mode(&self) -> Option { + None } /// Human-readable device name (e.g. "Apple M5 Pro", "NVIDIA RTX 5090", @@ -250,6 +251,10 @@ impl GpuMonitor for MockMonitor { fn platform(&self) -> &'static str { "mock" } + /// A scripted dedicated card: the scenarios that use this mock model one. + fn memory_mode(&self) -> Option { + Some(MemoryMode::Discrete) + } fn device_name(&self) -> &str { &self.device_name } diff --git a/core/continuum-kernel/src/resources/capacity.rs b/core/continuum-kernel/src/resources/capacity.rs index bd74f64463..236fb6b37b 100644 --- a/core/continuum-kernel/src/resources/capacity.rs +++ b/core/continuum-kernel/src/resources/capacity.rs @@ -59,6 +59,14 @@ pub trait CapacitySource: Send + Sync { fn used_bytes(&self) -> u64 { 0 } + + /// The most ONE process may hold of this kind, when the device bounds a process rather than + /// the pool: Metal's working-set hint on unified memory bounds each GPU process's own + /// resident bytes (the engine's, the embedder's), never what the pool has left. `None`: no + /// per-process bound beyond the pool (a dedicated card, host RAM). + fn per_process_cap(&self) -> Option { + None + } } /// VRAM source from a [`GpuMonitor`]. Reports the device's fixed usable total @@ -326,8 +334,12 @@ impl CapacitySource for HostRamCapacitySource { /// /// - **Pool ceiling** = host physical RAM − reserve. NOT `recommendedMaxWorkingSetSize`, /// which is a Metal *hint* about GPU working sets and is smaller than physical memory. -/// - **VRAM view ceiling** = `min(pool ceiling, GPU working-set hint)`. The hint keeps its -/// meaning as a cap on GPU-side allocation without becoming a second pool. +/// - **VRAM view ceiling** = the pool ceiling, the same as RAM's: one pool, one available. +/// - **The GPU working-set hint** bounds each GPU process's own residency +/// ([`CapacitySource::per_process_cap`]), never the pool. As a pool ceiling it was netted +/// against the WHOLE system's usage, CPU-only processes included, and read 0 available while +/// RAM read 3,165 MiB of the same memory (the M5, 2026-11-10: 53,084 MiB hint, 54,179 MiB +/// used system-wide), so a 4 MB learning step waited and every VRAM lease beyond a floor did. /// - **Both views' used** = the one host measurement (`total − available`). On Apple /// Silicon the host VM free count already includes GPU buffers — `metal_monitor`'s /// Unified arm reads exactly this number for `free_bytes`, so using it here is agreeing @@ -409,12 +421,12 @@ impl CapacitySource for UnifiedPoolView { } fn ceiling_bytes(&self) -> u64 { - let pool = self.pool.pool_ceiling(); - match self.kind { - // The GPU hint caps GPU-side allocation without minting a second pool. - ResourceKind::Vram => pool.min(self.pool.gpu_working_set_hint), - _ => pool, - } + self.pool.pool_ceiling() + } + + fn per_process_cap(&self) -> Option { + // The GPU hint bounds one GPU process, never the pool (module doc). + (self.kind == ResourceKind::Vram).then_some(self.pool.gpu_working_set_hint) } fn used_bytes(&self) -> u64 { @@ -682,22 +694,40 @@ mod tests { } // what this catches: the pool ceiling silently becoming the GPU's working-set - // hint. That hint is smaller than physical memory, so adopting it pool-wide would - // shrink RAM for no reason; ignoring it entirely would let a GPU allocation blow - // past what Metal recommends. It caps VRAM only. + // hint (VRAM read 0 available while RAM read 3,165 MiB of the same memory on the + // M5, 2026-11-10), or the hint ignored (a GPU process growing past what Metal + // recommends). Both views carry the pool's ceiling; the hint is VRAM's per-process cap. #[test] - fn the_gpu_hint_caps_vram_without_shrinking_the_pool() { + fn the_gpu_hint_caps_a_gpu_process_never_the_pool() { let views = m5_pool().views(); + let ram = view_of(&views, ResourceKind::Ram); + let vram = view_of(&views, ResourceKind::Vram); // 64 GiB − 8 GiB reserve. Matches the board's reported ram capacity exactly. - assert_eq!( - view_of(&views, ResourceKind::Ram).ceiling_bytes(), - 60_129_542_144 - ); - // min(pool, hint) — and on this box the hint binds, matching the board's vram. - assert_eq!( - view_of(&views, ResourceKind::Vram).ceiling_bytes(), - M5_GPU_HINT - ); + assert_eq!(ram.ceiling_bytes(), 60_129_542_144); + assert_eq!(vram.ceiling_bytes(), ram.ceiling_bytes(), "one pool, one ceiling"); + assert_eq!(vram.per_process_cap(), Some(M5_GPU_HINT), "the hint bounds a GPU process"); + assert_eq!(ram.per_process_cap(), None); + } + + // what this catches (the M5, 2026-11-10, resources/board): the two rows of one + // unified pool disagreeing. 64 GiB, 54,179 MiB used system-wide, a 53,084 MiB hint: + // VRAM read 0 available while RAM read 3,165 MiB, and a 4,096,000-byte learning step + // waited. Both rows must offer the same free memory. + #[test] + fn both_rows_of_a_unified_pool_offer_the_same_free_memory() { + const MIB: u64 = 1024 * 1024; + let pool = Arc::new(UnifiedMemoryPool::new( + FakeHost { total: M5_TOTAL, available: Some(M5_TOTAL - 54_179 * MIB) }, + 53_084 * MIB, + 8 * GIB, + )); + let views = pool.views(); + let free = |kind| { + let view = view_of(&views, kind); + view.ceiling_bytes().saturating_sub(view.used_bytes()) + }; + assert_eq!(free(ResourceKind::Ram), 3_165 * MIB); + assert_eq!(free(ResourceKind::Vram), free(ResourceKind::Ram)); } // what this catches: a GPU whose hint EXCEEDS the pool handing out bytes the host diff --git a/core/continuum-kernel/src/resources/daemon.rs b/core/continuum-kernel/src/resources/daemon.rs index b0d00d6727..6ec06052b5 100644 --- a/core/continuum-kernel/src/resources/daemon.rs +++ b/core/continuum-kernel/src/resources/daemon.rs @@ -322,6 +322,13 @@ impl ResourceDaemon { g.reserve(consumer_id, kind, min_bytes); } + /// The most one process may hold of `kind` (`CapacitySource::per_process_cap`): what a planner + /// sizing ONE process (the engine) stays under beside what the pool has left. `None`: the pool + /// is the only bound. A cached source read, non-blocking. + pub fn per_process_cap(&self, kind: ResourceKind) -> Option { + self.sources.iter().filter(|source| source.kind() == kind).find_map(|source| source.per_process_cap()) + } + /// This consumer's plannable headroom: global available minus every OTHER /// consumer's unmet reservation floor — the same math `acquire` enforces, /// exposed for BUDGETING so a planner sizes itself only into bytes it could diff --git a/core/continuum-kernel/src/system_resources/monitor.rs b/core/continuum-kernel/src/system_resources/monitor.rs index 58cc800169..a990d3749c 100644 --- a/core/continuum-kernel/src/system_resources/monitor.rs +++ b/core/continuum-kernel/src/system_resources/monitor.rs @@ -228,7 +228,7 @@ impl SystemResourceMonitor { self.inner .lock() .ok() - .and_then(|inner| inner.gpu.as_ref().map(|g| g.memory_mode())) + .and_then(|inner| inner.gpu.as_ref().and_then(|g| g.memory_mode())) } /// Refresh all readings. Call this from the TypeScript watcher on each poll. diff --git a/core/continuum-serving/src/modules/serving_daemon/device_budget.rs b/core/continuum-serving/src/modules/serving_daemon/device_budget.rs index b14745bc23..b7eefa4f7a 100644 --- a/core/continuum-serving/src/modules/serving_daemon/device_budget.rs +++ b/core/continuum-serving/src/modules/serving_daemon/device_budget.rs @@ -170,7 +170,12 @@ pub(super) fn governed_pool_ceiling( // it abandoned the 27B it was ALREADY RUNNING for a 7B, then a 0.5B. `acquire` // still uses available_for — a lease must never be granted against bytes not yet // released; only the replace-myself DECISION gets the add-back. - .map(|_| resource_daemon.budget_for_replacing(SERVING_CONSUMER_ID, kind)) + // The engine is ONE process: where the device bounds a process (Metal's working-set hint on + // unified memory), the plan stays under that bound as well as under what the pool has left. + .map(|_| { + let pool = resource_daemon.budget_for_replacing(SERVING_CONSUMER_ID, kind); + resource_daemon.per_process_cap(kind).map_or(pool, |cap| pool.min(cap)) // map_or: no per-process bound, the pool's own + }) } /// The governed VRAM ceiling for a planner that has no way to represent "unknown", diff --git a/core/continuum-serving/src/modules/serving_daemon/tests/budget.rs b/core/continuum-serving/src/modules/serving_daemon/tests/budget.rs index 8a81aa0b4a..c3effe8058 100644 --- a/core/continuum-serving/src/modules/serving_daemon/tests/budget.rs +++ b/core/continuum-serving/src/modules/serving_daemon/tests/budget.rs @@ -587,6 +587,42 @@ fn downshift_gate_classifies_the_transient_lobotomy() { ); } +// what this catches (the M5, 2026-11-10: VRAM read 0 available while RAM read 3,165 MiB of the +// same unified memory, because the GPU working-set hint was the pool's ceiling): the engine's plan +// reading the pool's free memory, and staying under the hint because the engine is ONE GPU +// process. The hosts are SIMULATIONS (a 32 GiB unified host), not measurements. +#[tokio::test] +async fn the_engine_plan_reads_the_pool_and_stays_under_its_gpu_process_cap() { + use crate::resources::{DaemonConfig, GovernorConfig, HostMemoryReader, ResourceDaemon, ResourceKind, UnifiedMemoryPool}; + const GIB: u64 = 1 << 30; + struct Host(u64, u64); + impl HostMemoryReader for Host { + fn total_bytes(&self) -> u64 { + self.0 + } + fn available_bytes(&self) -> Option { + Some(self.1) + } + } + let plan = |hint: u64| { + let pool = Arc::new(UnifiedMemoryPool::new(Host(32 * GIB, 20 * GIB), hint, 4 * GIB)); + let daemon = ResourceDaemon::start( + pool.views(), + vec![], + DaemonConfig { + tick_interval: std::time::Duration::from_millis(20), + min_reclaim_budget: std::time::Duration::from_millis(100), + governor: GovernorConfig { min_dwell_ms: 0, graceful_grace_ms: 50 }, + }, + ); + (governed_pool_ceiling(&daemon, ResourceKind::Vram), governed_pool_ceiling(&daemon, ResourceKind::Ram)) + }; + // 28 GiB pool, 12 GiB used: 16 GiB free on both rows; a 21 GiB hint does not bind. + assert_eq!(plan(21 * GIB), (Some(16 * GIB), Some(16 * GIB)), "one pool, one free"); + // A 10 GiB hint binds the engine (one GPU process), never the RAM row. + assert_eq!(plan(10 * GIB), (Some(10 * GIB), Some(16 * GIB)), "the hint caps the engine only"); +} + // what this catches (#225, Joel 2026-08-08 "the budgeter just has all its // parts figure it out"): serving's plan budget (`governed_vram_ceiling`) // reads serving's OWN view of the board — global available minus every