diff --git a/.agents/skills/rust-resource-guard-must-use/SKILL.md b/.agents/skills/rust-resource-guard-must-use/SKILL.md index d48309e4bd..fb9385a972 100644 --- a/.agents/skills/rust-resource-guard-must-use/SKILL.md +++ b/.agents/skills/rust-resource-guard-must-use/SKILL.md @@ -23,9 +23,9 @@ The following types in `kernel/` already have type-level `#[must_use]` attribute | Types | Source | Guard behavior | | --- | --- | --- | -| `DisableInterruptGuard`, `PlainDisableInterruptGuard` | [support.rs](../../../kernel/kernel/src/support.rs) | The former restores the saved local interrupt state in `Drop`; the latter re-enables local interrupts in `Drop`. | +| `DisableInterruptGuard` | [support.rs](../../../kernel/kernel/src/support.rs) | Drops one counted IRQ guard and restores the saved mask on the last release. | | `PreemptGuard` | [thread/mod.rs](../../../kernel/kernel/src/thread/mod.rs) | Calls `enable_preempt()` on its thread in `Drop`. | -| `SpinLockGuard`, `SpinLockReadGuard` | [sync/spinlock.rs](../../../kernel/kernel/src/sync/spinlock.rs) | Hold the underlying lock guard and an optional interrupt guard. Cleanup occurs through their fields when dropped. | +| `SpinLockGuard`, `OwnedIrqSpinLockGuard`, `SpinLockReadGuard` | [sync/spinlock.rs](../../../kernel/kernel/src/sync/spinlock.rs) | Write guards release the payload before their preemption or IRQ protection. Read guards carry the protection selected at acquisition. | | `RwLockReadGuard`, `RwLockWriteGuard`, `RwLockUpgradableGuard` | [tinyrwlock.rs](../../../kernel/infra/src/tinyrwlock.rs) | Update the reader count or clear lock state in `Drop`. | | `__BlueOSTestDoneGuard` | [test_harness/src/lib.rs](../../../kernel/test_harness/src/lib.rs) | Generated inside a `quote!` block with `#[must_use]`; increments the completion counter and wakes waiters in `Drop`. | @@ -34,9 +34,7 @@ Actual type definition from `kernel/kernel/src/support.rs`: ```rust #[derive(Debug)] #[must_use] -pub struct DisableInterruptGuard { - old: usize, -} +pub struct DisableInterruptGuard(NotThreadSafe); ``` Before delivery, check every resource guard generated or modified in the task. Confirm that the attribute is on the type definition, including types generated by macros. diff --git a/external/CMSIS-RTOS2_Validation/Source/Config/RV2_Config.c b/external/CMSIS-RTOS2_Validation/Source/Config/RV2_Config.c index 8401bae628..44a8ea92d1 100644 --- a/external/CMSIS-RTOS2_Validation/Source/Config/RV2_Config.c +++ b/external/CMSIS-RTOS2_Validation/Source/Config/RV2_Config.c @@ -33,7 +33,7 @@ extern int stdout_putchar (int ch); typedef unsigned short IRQn_Type; /* forward to rust */ -extern void NVIC_SetPriority(IRQn_Type IRQn, char priority); +extern void configure_validation_irqs(IRQn_Type primary, IRQn_Type secondary); extern void NVIC_EnableIRQ(IRQn_Type IRQn); extern void NVIC_DisableIRQ(IRQn_Type IRQn); extern uint32_t NVIC_GetPendingIRQ(IRQn_Type IRQn); @@ -65,9 +65,8 @@ void TST_IRQ_HANDLER_B (void) { void TS_Init (void) { #if defined(__CORTEX_M) - /* Set primary and secondary interrupt priority */ - NVIC_SetPriority ((IRQn_Type)TST_IRQ_NUM_A, 5U); - NVIC_SetPriority ((IRQn_Type)TST_IRQ_NUM_B, 4U); + /* Install the validation handlers and set their kernel-callable priorities. */ + configure_validation_irqs (TST_IRQ_NUM_A, TST_IRQ_NUM_B); /* Enable interrupts */ NVIC_EnableIRQ((IRQn_Type)TST_IRQ_NUM_A); diff --git a/kernel/adapter/cmsis/src/os2/thread.rs b/kernel/adapter/cmsis/src/os2/thread.rs index 8178f85055..91e2903fcb 100644 --- a/kernel/adapter/cmsis/src/os2/thread.rs +++ b/kernel/adapter/cmsis/src/os2/thread.rs @@ -26,7 +26,7 @@ use blueos::{ sync::{ atomic_wait, atomic_wake, event_flags::{EventFlags, EventFlagsMode}, - ConstBarrier, SpinLockGuard, + ConstBarrier, OwnedIrqSpinLockGuard, }, thread::{self, Entry, GlobalQueueVisitor, Thread}, time, @@ -68,7 +68,7 @@ impl Os2Thread { to self.inner() { pub fn priority(&self) -> ThreadPriority; pub fn state(&self) -> Uint; - pub fn lock(&self) -> SpinLockGuard<'_, Thread>; + pub fn lock(&self) -> OwnedIrqSpinLockGuard<'_, Thread>; pub fn stack_size(&self) -> usize; pub fn stack_base(&self) -> usize; pub fn saved_stack_usage(&self) -> usize; diff --git a/kernel/adapter/cmsis/src/validation.checker b/kernel/adapter/cmsis/src/validation.checker index 80b1664fe0..a482a59822 100644 --- a/kernel/adapter/cmsis/src/validation.checker +++ b/kernel/adapter/cmsis/src/validation.checker @@ -1,3 +1,4 @@ // TOTAL-TIMEOUT: 25 // ASSERT-SUCC: cmsis_rv2 unittest finished -// ASSERT-FAIL: Oops: \ No newline at end of file +// ASSERT-FAIL: Oops: +// ASSERT-FAIL: Test Result: FAILED diff --git a/kernel/adapter/cmsis/src/validation.rs b/kernel/adapter/cmsis/src/validation.rs index bf5412b394..c70fb24b9c 100644 --- a/kernel/adapter/cmsis/src/validation.rs +++ b/kernel/adapter/cmsis/src/validation.rs @@ -126,12 +126,21 @@ mod tests { } } #[unsafe(no_mangle)] - pub unsafe extern "C" fn NVIC_SetPriority(irq: u16, priority: u8) { + pub unsafe extern "C" fn configure_validation_irqs(primary: u16, secondary: u16) { + use arch::irq::{disable_irq, register_raw_isr, set_irq_priority, IrqNumber, Priority}; + let primary = IrqNumber::new(primary); + let secondary = IrqNumber::new(secondary); + disable_irq(primary); + disable_irq(secondary); + // SAFETY: TS_Init installs the validation vectors before enabling either IRQ. unsafe { - cortex_m::Peripherals::steal() - .NVIC - .set_priority(IrqNumber::new(irq), priority) - }; + register_raw_isr(primary, uart::uart1rx_handler); + register_raw_isr(secondary, uart::uart1tx_handler); + } + // Both ISRs call RTOS APIs. Keep the secondary IRQ above the primary + // while allowing the kernel's IRQ mask to exclude either one. + set_irq_priority(primary, Priority::Low as u8); + set_irq_priority(secondary, Priority::Normal as u8); } #[unsafe(no_mangle)] pub unsafe extern "C" fn NVIC_EnableIRQ(irq: u16) { diff --git a/kernel/adapter/thread_metric/BUILD.gn b/kernel/adapter/thread_metric/BUILD.gn index c37e1058ca..be385c35a0 100644 --- a/kernel/adapter/thread_metric/BUILD.gn +++ b/kernel/adapter/thread_metric/BUILD.gn @@ -87,3 +87,43 @@ group("run_tests") { ":run_tm_synchronization_processing_test", ] } + +# Real-kernel lock contract checks, separate from measured benchmarks. +foreach(probe, + [ + "local_lock_contract_probe", + "local_lock_reject_yield_probe", + "local_lock_high_irq_probe", + ]) { + build_rust(probe) { + crate_type = "bin" + sources = [ "src/local_lock_probe.rs" ] + configs += [ "//build/boards/${board}:kernel_config" ] + deps = board_deps + [ + "//kernel/kernel:blueos", + "//kernel/rsrt", + "//libs/libc", + "//libs/librs", + ] + linker_script = default_linker_script + if (probe == "local_lock_high_irq_probe") { + rustflags = [ "--cfg=probe_high_irq" ] + } + if (probe == "local_lock_reject_yield_probe") { + rustflags = [ "--cfg=probe_yield_in_lock" ] + } + } +} + +build_rust("contended_semaphore_probe") { + crate_type = "bin" + sources = [ "src/contended_semaphore_probe.rs" ] + configs += [ "//build/boards/${board}:kernel_config" ] + deps = board_deps + [ + "//kernel/kernel:blueos", + "//kernel/rsrt", + "//libs/libc", + "//libs/librs", + ] + linker_script = default_linker_script +} diff --git a/kernel/adapter/thread_metric/src/contended_semaphore_probe.rs b/kernel/adapter/thread_metric/src/contended_semaphore_probe.rs new file mode 100644 index 0000000000..559a48dd9b --- /dev/null +++ b/kernel/adapter/thread_metric/src/contended_semaphore_probe.rs @@ -0,0 +1,63 @@ +// Copyright (c) 2026 vivo Mobile Communication Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![no_std] +#![no_main] + +extern crate librs; +extern crate rsrt; + +use blueos::{scheduler::InsertToEnd, sync::Semaphore, thread, time::Tick}; +use core::sync::atomic::{AtomicBool, Ordering}; + +static PING: Semaphore = Semaphore::new(); +static PONG: Semaphore = Semaphore::new(); +static STOP: AtomicBool = AtomicBool::new(false); + +#[unsafe(no_mangle)] +pub extern "C" fn main() -> i32 { + librs::stdio::init(); + librs::pthread::register_my_posix_tcb(); + + assert!(PING.init(1)); + assert!(PONG.init(0)); + + thread::spawn(|| loop { + assert!(PONG.acquire_notimeout::()); + if STOP.load(Ordering::Relaxed) { + break; + } + PING.release(); + }) + .unwrap(); + + for _ in 0..6 { + let end = Tick::after(Tick::from_millis(2000)); + let mut count = 0u32; + while Tick::now().0 < end.0 { + assert!(PING.acquire_notimeout::()); + PONG.release(); + count += 1; + } + unsafe { + libc::printf(c"Time Period Total: %u\n".as_ptr(), count); + } + } + STOP.store(true, Ordering::Relaxed); + PONG.release(); + + loop { + core::hint::spin_loop(); + } +} diff --git a/kernel/adapter/thread_metric/src/local_lock_probe.rs b/kernel/adapter/thread_metric/src/local_lock_probe.rs new file mode 100644 index 0000000000..05eb3ef1e7 --- /dev/null +++ b/kernel/adapter/thread_metric/src/local_lock_probe.rs @@ -0,0 +1,231 @@ +// Copyright (c) 2026 vivo Mobile Communication Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![no_std] +#![no_main] +extern crate librs; +extern crate rsrt; + +use blueos::{arch, scheduler, support::DisableInterruptGuard, sync::SpinLock}; +use core::sync::atomic::{AtomicUsize, Ordering}; + +static IRQ_LOCK: SpinLock = SpinLock::new(0); +static IRQ_RESULT: AtomicUsize = AtomicUsize::new(0); +const TEST_IRQ: arch::arm::irq::IrqNumber = arch::arm::irq::IrqNumber::new(0); + +unsafe extern "C" fn test_irq() { + IRQ_RESULT.store( + if IRQ_LOCK.try_lock().is_some() { 1 } else { 2 }, + Ordering::Relaxed, + ); +} + +fn pend_test_irq() { + unsafe { + core::ptr::write_volatile(0xe000e200 as *mut u32, 1); + core::arch::asm!("dsb", "isb", options(nostack)); + } +} + +fn irq_checks() { + let irq = TEST_IRQ; + unsafe { + arch::arm::irq::register_raw_isr(irq, test_irq); + } + #[cfg(probe_high_irq)] + arch::arm::irq::enable_irq_with_priority(irq, arch::arm::irq::Priority::High); + #[cfg(not(probe_high_irq))] + arch::arm::irq::enable_irq_with_priority(irq, arch::arm::irq::Priority::Normal); + let g = IRQ_LOCK.lock(); + pend_test_irq(); + while IRQ_RESULT.load(Ordering::Relaxed) == 0 { + core::hint::spin_loop(); + } + assert_eq!(IRQ_RESULT.load(Ordering::Relaxed), 2); + drop(g); + IRQ_RESULT.store(0, Ordering::Relaxed); + let g = IRQ_LOCK.irqsave_lock(); + pend_test_irq(); + assert_eq!(IRQ_RESULT.load(Ordering::Relaxed), 0); + drop(g); + while IRQ_RESULT.load(Ordering::Relaxed) == 0 { + core::hint::spin_loop(); + } + assert_eq!(IRQ_RESULT.load(Ordering::Relaxed), 1); + arch::arm::irq::disable_irq(irq); +} + +fn irq_wakes_semaphore_waiter() { + use blueos::{sync::Semaphore, thread, time::Tick}; + use core::sync::atomic::AtomicBool; + + static SEMAPHORE: Semaphore = Semaphore::new(); + static RELEASE_RETURNED: AtomicBool = AtomicBool::new(false); + + unsafe extern "C" fn release_from_irq() { + assert!(arch::arm::irq::is_irq_active(TEST_IRQ)); + SEMAPHORE.release(); + RELEASE_RETURNED.store(true, Ordering::Release); + } + + assert!(SEMAPHORE.init(0)); + // SAFETY: irq_checks disabled this probe's IRQ before we replace its handler. + unsafe { arch::arm::irq::register_raw_isr(TEST_IRQ, release_from_irq) }; + arch::arm::irq::enable_irq_with_priority(TEST_IRQ, arch::arm::irq::Priority::Normal); + + let waiter = scheduler::current_thread(); + thread::spawn(move || { + while waiter.state() != thread::SUSPENDED { + scheduler::suspend_me_for::<()>(Tick(1), None); + } + pend_test_irq(); + }) + .unwrap(); + + assert!( + SEMAPHORE.acquire_timeout::(Tick(100)), + "IRQ did not wake the semaphore waiter before timeout" + ); + assert!(RELEASE_RETURNED.load(Ordering::Acquire)); + assert_eq!(SEMAPHORE.count(), 0); + arch::arm::irq::disable_irq(TEST_IRQ); +} + +fn wait_checks() { + use blueos::{ + sync::{atomic_wait, atomic_wake, mqueue::SendMode, MessageQueue, Semaphore}, + thread, + time::Tick, + types::Arc, + }; + let sema = Arc::new(Semaphore::new()); + assert!(sema.init(0)); + let producer = sema.clone(); + thread::spawn(move || { + scheduler::suspend_me_for::<()>(Tick(2), None); + producer.release(); + }) + .unwrap(); + assert!(sema.acquire::(Tick(100))); + let atom = Arc::new(AtomicUsize::new(0)); + let producer = atom.clone(); + thread::spawn(move || { + scheduler::suspend_me_for::<()>(Tick(2), None); + producer.store(1, Ordering::Release); + atomic_wake(&producer, 1).unwrap(); + }) + .unwrap(); + atomic_wait(&atom, 0, Tick(100)).unwrap(); + assert_eq!(atom.load(Ordering::Acquire), 1); + let queue = Arc::new(MessageQueue::new(4, 1, core::ptr::null_mut()).unwrap()); + assert!(queue.init()); + let producer = queue.clone(); + thread::spawn(move || { + scheduler::suspend_me_for::<()>(Tick(2), None); + producer + .send(&[1, 2, 3, 4], 4, Tick(100), SendMode::Normal) + .unwrap(); + }) + .unwrap(); + let mut buf = [0; 4]; + queue.recv(&mut buf, 4, Tick(100)).unwrap(); + assert_eq!(buf, [1, 2, 3, 4]); + queue + .send(&[5, 6, 7, 8], 4, Tick(100), SendMode::Normal) + .unwrap(); + let consumer = queue.clone(); + thread::spawn(move || { + scheduler::suspend_me_for::<()>(Tick(2), None); + let mut buf = [0; 4]; + consumer.recv(&mut buf, 4, Tick(100)).unwrap(); + assert_eq!(buf, [5, 6, 7, 8]); + }) + .unwrap(); + queue + .send(&[9, 10, 11, 12], 4, Tick(100), SendMode::Normal) + .unwrap(); + queue.recv(&mut buf, 4, Tick(100)).unwrap(); + assert_eq!(buf, [9, 10, 11, 12]); +} + +#[unsafe(no_mangle)] +pub extern "C" fn main() -> i32 { + librs::stdio::init(); + librs::pthread::register_my_posix_tcb(); + let lock = SpinLock::new(0usize); + assert!(arch::local_irq_enabled()); + let mut w = lock.lock(); + assert!(arch::local_irq_enabled()); + assert!(lock.try_write().is_none()); + assert!(lock.try_read().is_none()); + *w = 42; + #[cfg(probe_yield_in_lock)] + scheduler::yield_me(); + scheduler::yield_me_now_or_later(); + assert_eq!(*w, 42); + drop(w); + let a = lock.read(); + let b = lock.read(); + assert_eq!(*a, 42); + assert!(lock.try_write().is_none()); + drop(a); + assert!(lock.try_write().is_none()); + drop(b); + assert!(lock.try_write().is_some()); + let first = DisableInterruptGuard::new(); + let second = DisableInterruptGuard::new(); + assert!(!arch::local_irq_enabled()); + drop(first); + assert!(!arch::local_irq_enabled()); + drop(second); + assert!(arch::local_irq_enabled()); + let other = SpinLock::new(1); + let a = lock.irqsave_lock(); + let b = other.irqsave_lock(); + assert!(lock.try_irqsave_read().is_none()); + assert!(lock.try_irqsave_lock().is_none()); + drop(a); + assert!(!arch::local_irq_enabled()); + drop(b); + assert!(arch::local_irq_enabled()); + irq_checks(); + irq_wakes_semaphore_waiter(); + wait_checks(); + // Stronger pre-existing masks survive entry and final restoration. + unsafe { + core::arch::asm!("msr basepri, {}", in(reg) 0x40u32, options(nostack)); + } + let g = DisableInterruptGuard::new(); + let mask: u32; + unsafe { + core::arch::asm!("mrs {}, basepri", out(reg) mask, options(nostack)); + } + assert_eq!(mask, 0x40); + drop(g); + unsafe { + arch::enable_local_irq(); + } + // A leaked plain guard must never restore access to the payload. + let leaked = SpinLock::new(3); + core::mem::forget(leaked.lock()); + assert!(leaked.try_write().is_none()); + assert!(leaked.try_read().is_none()); + // No scheduling is attempted after the deliberately leaked preemption lease. + unsafe { + libc::printf(c"LOCAL_LOCK_CONTRACT_PASS\n".as_ptr()); + } + loop { + core::hint::spin_loop(); + } +} diff --git a/kernel/infra/src/lib.rs b/kernel/infra/src/lib.rs index ef4859edb1..7220eeb2f4 100644 --- a/kernel/infra/src/lib.rs +++ b/kernel/infra/src/lib.rs @@ -31,6 +31,7 @@ pub mod intrusive; pub mod lifetime; pub mod list; +pub mod marker; pub mod no_let_underscore; pub mod rbtree; pub mod ringbuffer; @@ -40,5 +41,5 @@ pub mod string; pub mod tinyarc; pub use tinyarc::sorted_list; pub mod iheap; -pub mod nolock; +pub mod local_rwlock; pub mod tinyrwlock; diff --git a/kernel/infra/src/local_rwlock.rs b/kernel/infra/src/local_rwlock.rs new file mode 100644 index 0000000000..36c61c286d --- /dev/null +++ b/kernel/infra/src/local_rwlock.rs @@ -0,0 +1,282 @@ +// Copyright (c) 2026 vivo Mobile Communication Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Non-atomic read/write borrow checking with externally synchronized state. +//! +//! This module does not mask interrupts or prevent scheduling. Its caller must +//! serialize borrow-state access, including guard destruction. The kernel's UP +//! `SpinLock` wrapper supplies this protection; other users must supply their own. + +use crate::marker::NotThreadSafe; +use core::{ + cell::{Cell, UnsafeCell}, + ops::{Deref, DerefMut}, +}; + +/// A non-atomic read/write lock whose borrow bookkeeping needs external protection. +/// +/// The lock is not `Sync`; it does not provide cross-thread or interrupt exclusion +/// by itself. Obtaining guards requires an unsafe call that establishes the +/// protection needed by both acquisition and subsequent guard destruction. +pub struct LocalRwLock { + state: BorrowState, + data: UnsafeCell, +} + +// Zero is idle, WRITER is exclusive, and all other values count readers. +struct BorrowState(Cell); + +impl LocalRwLock { + pub const fn new(data: T) -> Self { + Self { + state: BorrowState::new(), + data: UnsafeCell::new(data), + } + } +} + +impl LocalRwLock { + /// Attempts to acquire an exclusive payload borrow. + /// + /// # Safety + /// The caller must serialize every access to this lock's borrow state, + /// including acquisitions, count queries, and destruction of any returned + /// guard. No concurrent or reentrant state access may interrupt one of these + /// operations. Protection must also be established when the guard is dropped; + /// protecting only this call is insufficient. + #[inline(always)] + pub unsafe fn try_write(&self) -> Option> { + self.state.try_write().then(|| WriteGuard { + state: &self.state, + data: self.data.get(), + _not_send: NotThreadSafe, + }) + } + + /// Attempts to acquire a shared payload borrow. + /// + /// # Safety + /// The caller must uphold the state-access and guard-destruction protection + /// requirements of [`Self::try_write`]. + #[inline(always)] + pub unsafe fn try_read(&self) -> Option> { + self.state.try_read().then(|| ReadGuard { + state: &self.state, + data: self.data.get(), + _not_send: NotThreadSafe, + }) + } + + /// Returns the number of active read guards. + /// + /// # Safety + /// This query must be serialized with every other access to the borrow state, + /// including guard destruction, as described in [`Self::try_write`]. + #[inline(always)] + pub unsafe fn reader_count(&self) -> usize { + self.state.reader_count() + } + + /// Returns one if a write guard is active, otherwise zero. + /// + /// # Safety + /// This query must be serialized with every other access to the borrow state, + /// including guard destruction, as described in [`Self::try_write`]. + #[inline(always)] + pub unsafe fn writer_count(&self) -> usize { + self.state.writer_count() + } +} + +impl LocalRwLock<()> { + /// Acquires a write borrow for an externally supplied payload. + /// + /// This is a transitional adapter for existing intrusive callers. Borrow-state + /// exclusion alone does not establish the validity of an owner projection. + /// + /// # Safety + /// The caller must uphold the protection requirements of [`Self::try_write`]. + /// In addition, `data` must be aligned, non-null, point to an initialized `T`, + /// and remain valid for the returned guard's lifetime. It must permit the + /// shared and exclusive references exposed by the guard, with no conflicting + /// accesses through other pointers or references. The payload must not overlap + /// the borrow bookkeeping accessed through this lock. + #[inline(always)] + pub unsafe fn project_write(&self, data: *mut T) -> Option> { + self.state.try_write().then(|| WriteGuard { + state: &self.state, + data, + _not_send: NotThreadSafe, + }) + } +} + +impl BorrowState { + const WRITER: usize = usize::MAX; + + const fn new() -> Self { + Self(Cell::new(0)) + } + + #[inline(always)] + fn try_read(&self) -> bool { + let n = self.0.get(); + if n >= Self::WRITER - 1 { + return false; + } + self.0.set(n + 1); + true + } + + #[inline(always)] + fn try_write(&self) -> bool { + if self.0.get() != 0 { + return false; + } + self.0.set(Self::WRITER); + true + } + + #[inline(always)] + fn release_read(&self) { + let n = self.0.get(); + assert!(n != 0 && n != Self::WRITER); + self.0.set(n - 1); + } + + #[inline(always)] + fn release_write(&self) { + assert_eq!(self.0.get(), Self::WRITER); + self.0.set(0); + } + + #[inline(always)] + fn reader_count(&self) -> usize { + match self.0.get() { + Self::WRITER => 0, + n => n, + } + } + + #[inline(always)] + fn writer_count(&self) -> usize { + usize::from(self.0.get() == Self::WRITER) + } +} + +/// An exclusive payload borrow. Dropping it updates externally protected state. +#[must_use] +pub struct WriteGuard<'a, T: ?Sized> { + state: &'a BorrowState, + data: *mut T, + _not_send: NotThreadSafe, +} + +/// A shared payload borrow. Dropping it updates externally protected state. +#[must_use] +pub struct ReadGuard<'a, T: ?Sized> { + state: &'a BorrowState, + data: *const T, + _not_send: NotThreadSafe, +} + +impl Deref for WriteGuard<'_, T> { + type Target = T; + #[inline(always)] + fn deref(&self) -> &T { + unsafe { &*self.data } + } +} + +impl DerefMut for WriteGuard<'_, T> { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + unsafe { &mut *self.data } + } +} + +impl Deref for ReadGuard<'_, T> { + type Target = T; + #[inline(always)] + fn deref(&self) -> &T { + unsafe { &*self.data } + } +} + +impl Drop for WriteGuard<'_, T> { + #[inline(always)] + fn drop(&mut self) { + self.state.release_write(); + } +} + +impl Drop for ReadGuard<'_, T> { + #[inline(always)] + fn drop(&mut self) { + self.state.release_read(); + } +} + +impl core::fmt::Debug for LocalRwLock { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("LocalRwLock { .. }") + } +} + +impl core::fmt::Debug for WriteGuard<'_, T> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("LocalWriteGuard { .. }") + } +} + +impl core::fmt::Debug for ReadGuard<'_, T> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("LocalReadGuard { .. }") + } +} + +#[cfg(test)] +mod tests { + use super::{BorrowState, Cell}; + + #[test] + fn exclusion_and_release() { + let s = BorrowState::new(); + assert!(s.try_write()); + assert!(!s.try_write()); + assert!(!s.try_read()); + s.release_write(); + assert!(s.try_read()); + assert!(s.try_read()); + assert!(!s.try_write()); + s.release_read(); + assert!(!s.try_write()); + s.release_read(); + assert!(s.try_write()); + s.release_write(); + assert_eq!(s.reader_count(), 0); + assert_eq!(s.writer_count(), 0); + } + + #[test] + fn reader_overflow_cannot_unlock() { + let s = BorrowState(Cell::new(BorrowState::WRITER - 2)); + assert!(s.try_read()); + assert!(!s.try_read()); + assert!(!s.try_write()); + assert_eq!(s.reader_count(), BorrowState::WRITER - 1); + s.release_read(); + assert!(s.try_read()); + } +} diff --git a/kernel/infra/src/marker.rs b/kernel/infra/src/marker.rs new file mode 100644 index 0000000000..ff314b9da6 --- /dev/null +++ b/kernel/infra/src/marker.rs @@ -0,0 +1,28 @@ +// Copyright (c) 2025 vivo Mobile Communication Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Marker types for expressing thread contracts. + +use core::marker::PhantomData; + +/// A zero-sized field that prevents automatic `Send` and `Sync` implementations. +/// +/// Use this for guards that must stay in their acquiring execution context. +/// A containing type may explicitly implement `Sync` if sharing references to it +/// is safe, while leaving ownership transfer disabled. +pub type NotThreadSafe = PhantomData<*mut ()>; + +/// Constructs a [`NotThreadSafe`](type@NotThreadSafe) marker. +#[allow(non_upper_case_globals)] +pub const NotThreadSafe: NotThreadSafe = PhantomData; diff --git a/kernel/infra/src/nolock.rs b/kernel/infra/src/nolock.rs deleted file mode 100644 index d9e648ed61..0000000000 --- a/kernel/infra/src/nolock.rs +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright (c) 2026 vivo Mobile Communication Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Implement a lock based on UnsafeCell to optimize performance on single core. -// The APIs of this lock should be compatible with that in tinyrwlock. - -use crate::intrusive::Adapter; -use core::{ - cell::UnsafeCell, - marker::PhantomData, - ops::{Deref, DerefMut}, -}; - -#[derive(Debug)] -pub struct NoLock { - data: UnsafeCell, -} - -impl Default for NoLock { - fn default() -> Self { - Self::new(Default::default()) - } -} - -#[derive(Debug)] -#[must_use] -pub struct NoLockWriteGuard<'a, T: 'a + ?Sized> { - inner: *mut T, - _a: PhantomData<&'a mut T>, -} - -impl Deref for NoLockWriteGuard<'_, T> { - type Target = T; - fn deref(&self) -> &T { - unsafe { &*self.inner } - } -} - -impl DerefMut for NoLockWriteGuard<'_, T> { - fn deref_mut(&mut self) -> &mut T { - unsafe { &mut *self.inner } - } -} - -#[derive(Debug)] -#[must_use] -pub struct NoLockReadGuard<'a, T: 'a + ?Sized> { - inner: *const T, - _a: PhantomData<&'a mut T>, -} - -impl Deref for NoLockReadGuard<'_, T> { - type Target = T; - fn deref(&self) -> &T { - unsafe { &*self.inner } - } -} - -impl NoLock { - pub const fn new(val: T) -> Self { - Self { - data: UnsafeCell::new(val), - } - } -} - -impl NoLock { - pub fn try_write(&self) -> Option> { - Some(NoLockWriteGuard { - inner: self.data.get(), - _a: PhantomData, - }) - } - - pub fn try_read(&self) -> Option> { - Some(NoLockReadGuard { - inner: self.data.get() as *const _, - _a: PhantomData, - }) - } - - pub fn write(&self) -> NoLockWriteGuard<'_, T> { - self.try_write().unwrap() - } - - pub fn read(&self) -> NoLockReadGuard<'_, T> { - self.try_read().unwrap() - } - - pub fn reader_count(&self) -> usize { - usize::MAX - } - - pub fn writer_count(&self) -> usize { - usize::MAX - } -} - -unsafe impl Send for NoLock {} -unsafe impl Sync for NoLock {} -unsafe impl Send for NoLockWriteGuard<'_, T> {} -unsafe impl Sync for NoLockWriteGuard<'_, T> {} -unsafe impl Send for NoLockReadGuard<'_, T> {} -unsafe impl Sync for NoLockReadGuard<'_, T> {} - -#[derive(Default, Debug)] -pub struct INoLock> { - lock: NoLock<()>, - _a: PhantomData<(T, A)>, -} - -impl> INoLock { - #[inline] - pub const fn new() -> Self { - Self { - lock: NoLock::new(()), - _a: PhantomData, - } - } - - #[inline] - fn this_mut(&self) -> &mut T { - let ptr = (&raw const *self).cast::().cast_mut(); - let base = unsafe { ptr.sub(A::offset()) as *mut T }; - unsafe { &mut *base } - } - - #[inline] - pub fn write(&self) -> NoLockWriteGuard<'_, T> { - let inner = self.this_mut() as *mut T; - NoLockWriteGuard { - inner, - _a: PhantomData, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_read() { - let l = NoLock::new(42); - let r = l.read(); - assert_eq!(*r, 42); - } - - #[test] - fn test_write() { - let l = NoLock::new(0); - let mut w = l.write(); - *w = 42; - drop(w); - let r = l.read(); - assert_eq!(*r, 42); - } -} diff --git a/kernel/infra/src/tinyarc/mod.rs b/kernel/infra/src/tinyarc/mod.rs index edd43b0443..a551c3759d 100644 --- a/kernel/infra/src/tinyarc/mod.rs +++ b/kernel/infra/src/tinyarc/mod.rs @@ -354,6 +354,10 @@ impl> TinyArcList { Some(unsafe { Self::clone_from(next.as_ref()) }) } + // TODO(#559): Rework how detach updates head.next through the first node's prev + // pointer. Until then, inlining on RV32 release can elide the caller's post-pop + // emptiness check and leave an empty scheduler queue marked active. + #[inline(never)] pub fn pop_front(&mut self) -> Option> { if self.is_empty() { return None; diff --git a/kernel/infra/src/tinyrwlock.rs b/kernel/infra/src/tinyrwlock.rs index be46854921..792f68e548 100644 --- a/kernel/infra/src/tinyrwlock.rs +++ b/kernel/infra/src/tinyrwlock.rs @@ -17,8 +17,11 @@ // SPDX-License: MIT //! A lock that provides data access to either one writer or many readers. +//! +//! Guards stay on the thread that acquired them. Upgrading or downgrading a +//! guard preserves this contract. -use crate::intrusive::Adapter; +use crate::{intrusive::Adapter, marker::NotThreadSafe}; use core::{ cell::UnsafeCell, fmt, @@ -99,6 +102,11 @@ pub struct RwLock { data: UnsafeCell, } +// SAFETY: Moving the lock transfers ownership of its protected data. +unsafe impl Send for RwLock {} +// SAFETY: Writers can move T out by replacement; concurrent readers share &T. +unsafe impl Sync for RwLock {} + const READER: Usize = 1 << 2; const UPGRADED: Usize = 1 << 1; const WRITER: Usize = 1; @@ -111,8 +119,12 @@ const WRITER: Usize = 1; pub struct RwLockReadGuard<'a, T: 'a + ?Sized> { lock: &'a AtomicUsize, data: *const T, + _not_send: NotThreadSafe, } +// SAFETY: Sharing a read guard exposes only &T; releasing it consumes the guard. +unsafe impl Sync for RwLockReadGuard<'_, T> {} + /// A guard that provides mutable data access. /// /// When the guard falls out of scope it will release the lock. @@ -120,8 +132,13 @@ pub struct RwLockReadGuard<'a, T: 'a + ?Sized> { pub struct RwLockWriteGuard<'a, T: 'a + ?Sized> { lock: &'a AtomicUsize, data: *mut T, + _not_send: NotThreadSafe, } +// SAFETY: Sharing a write guard exposes only &T. Mutation requires &mut Guard, +// and releasing or changing lock mode consumes it. Keep the existing Send bound. +unsafe impl Sync for RwLockWriteGuard<'_, T> {} + /// A guard that provides immutable data access but can be upgraded to [`RwLockWriteGuard`]. /// /// No writers or other upgradeable guards can exist while this is in scope. New reader @@ -133,19 +150,11 @@ pub struct RwLockWriteGuard<'a, T: 'a + ?Sized> { pub struct RwLockUpgradableGuard<'a, T: 'a + ?Sized> { lock: &'a AtomicUsize, data: *const T, + _not_send: NotThreadSafe, } -// Same unsafe impls as `std::sync::RwLock` -unsafe impl Send for RwLock {} -unsafe impl Sync for RwLock {} - -unsafe impl Send for RwLockWriteGuard<'_, T> {} -unsafe impl Sync for RwLockWriteGuard<'_, T> {} - -unsafe impl Send for RwLockReadGuard<'_, T> {} -unsafe impl Sync for RwLockReadGuard<'_, T> {} - -unsafe impl Send for RwLockUpgradableGuard<'_, T> {} +// SAFETY: Sharing this guard exposes only &T; upgrading, downgrading or releasing +// it consumes the guard. Keep the existing Send bound. unsafe impl Sync for RwLockUpgradableGuard<'_, T> {} impl RwLock { @@ -339,6 +348,7 @@ impl RwLock { Some(RwLockReadGuard { lock: &self.lock, data: self.data.get(), + _not_send: NotThreadSafe, }) } } @@ -409,6 +419,7 @@ impl RwLock { Some(RwLockWriteGuard { lock: &self.lock, data: self.data.get(), + _not_send: NotThreadSafe, }) } else { None @@ -446,6 +457,7 @@ impl RwLock { Some(RwLockUpgradableGuard { lock: &self.lock, data: self.data.get(), + _not_send: NotThreadSafe, }) } else { // We can't unflip the UPGRADED bit back just yet as there is another upgradeable or write lock. @@ -573,6 +585,7 @@ impl<'rwlock, T: ?Sized> RwLockUpgradableGuard<'rwlock, T> { Ok(RwLockWriteGuard { lock, data: data as *mut T, + _not_send: NotThreadSafe, }) } else { Err(self) @@ -619,7 +632,11 @@ impl<'rwlock, T: ?Sized> RwLockUpgradableGuard<'rwlock, T> { // Dropping self removes the UPGRADED bit mem::drop(self); - RwLockReadGuard { lock, data } + RwLockReadGuard { + lock, + data, + _not_send: NotThreadSafe, + } } /// Leak the lock guard, yielding a reference to the underlying data. @@ -677,7 +694,11 @@ impl<'rwlock, T: ?Sized> RwLockWriteGuard<'rwlock, T> { // Dropping self removes the UPGRADED bit mem::drop(self); - RwLockReadGuard { lock, data } + RwLockReadGuard { + lock, + data, + _not_send: NotThreadSafe, + } } /// Downgrades the writable lock guard to an upgradable, shared lock guard. Cannot fail and is guaranteed not to spin. @@ -707,7 +728,11 @@ impl<'rwlock, T: ?Sized> RwLockWriteGuard<'rwlock, T> { // Dropping self removes the UPGRADED bit mem::forget(self); - RwLockUpgradableGuard { lock, data } + RwLockUpgradableGuard { + lock, + data, + _not_send: NotThreadSafe, + } } /// Leak the lock guard, yielding a mutable reference to the underlying data. @@ -848,7 +873,11 @@ impl> IRwLock { let lock = g.lock; let data = self.this_mut() as *mut T; core::mem::forget(g); - RwLockWriteGuard { lock, data } + RwLockWriteGuard { + lock, + data, + _not_send: NotThreadSafe, + } } } diff --git a/kernel/kernel/src/arch/aarch64/exception.rs b/kernel/kernel/src/arch/aarch64/exception.rs index d7e6fa56b7..f015db8e68 100644 --- a/kernel/kernel/src/arch/aarch64/exception.rs +++ b/kernel/kernel/src/arch/aarch64/exception.rs @@ -168,7 +168,10 @@ extern "C" fn handle_svc(context: &mut Context) -> usize { context.x0, context.x1, context.x2, context.x3, context.x4, context.x5, ], }; - enable_local_irq(); + crate::sync::local_context::assert_can_switch(); + unsafe { + enable_local_irq(); + } context.x0 = dispatch_syscall(&sc); disable_local_irq(); compiler_fence(Ordering::SeqCst); diff --git a/kernel/kernel/src/arch/aarch64/irq.rs b/kernel/kernel/src/arch/aarch64/irq.rs index 63b941781e..094a597202 100644 --- a/kernel/kernel/src/arch/aarch64/irq.rs +++ b/kernel/kernel/src/arch/aarch64/irq.rs @@ -98,7 +98,8 @@ pub const INTERRUPT_TABLE_LEN: usize = 1024; pub struct IrqContext { pub irq: IrqNumber, - pub handler: Box, + // The registry owns the handler, which another thread can replace or remove. + pub handler: Box, } impl core::fmt::Display for IrqContext { @@ -108,7 +109,7 @@ impl core::fmt::Display for IrqContext { } impl IrqContext { - fn new(irq: IrqNumber, handler: Box) -> Self { + fn new(irq: IrqNumber, handler: Box) -> Self { Self { irq, handler } } } @@ -130,7 +131,7 @@ impl IrqManager { fn register_handler( &mut self, irq: IrqNumber, - handler: Box, + handler: Box, ) -> Result<(), &'static str> { if u32::from(irq) >= INTERRUPT_TABLE_LEN as u32 { return Err("IRQ number out of range"); @@ -149,7 +150,10 @@ impl IrqManager { } // Register interrupt handler -pub fn register_handler(irq: IrqNumber, handler: Box) -> Result<(), &'static str> { +pub fn register_handler( + irq: IrqNumber, + handler: Box, +) -> Result<(), &'static str> { IRQ_MANAGER.lock().register_handler(irq, handler) } diff --git a/kernel/kernel/src/arch/aarch64/mod.rs b/kernel/kernel/src/arch/aarch64/mod.rs index da77883f32..31c423c77a 100644 --- a/kernel/kernel/src/arch/aarch64/mod.rs +++ b/kernel/kernel/src/arch/aarch64/mod.rs @@ -463,6 +463,7 @@ pub(crate) extern "C" fn svc_switch_context_with_hook(hook: *mut ContextSwitchHo #[inline] pub(crate) extern "C" fn switch_context_with_hook(hook: *mut ContextSwitchHookHolder) { + crate::sync::local_context::assert_can_switch(); svc_switch_context_with_hook(hook) } #[unsafe(naked)] @@ -524,7 +525,10 @@ pub extern "C" fn disable_local_irq() { } #[inline] -pub extern "C" fn enable_local_irq() { +/// # Safety +/// No live execution lease may require IRQs to remain masked. Only architecture +/// entry/exit code may bypass the nesting-aware IRQ guard. +pub unsafe extern "C" fn enable_local_irq() { unsafe { core::arch::asm!("msr daifclr, #3", options(nostack, nomem)) } } @@ -552,7 +556,7 @@ pub extern "C" fn disable_local_irq_save() -> usize { } #[inline] -pub extern "C" fn enable_local_irq_restore(old: usize) { +pub unsafe extern "C" fn enable_local_irq_restore(old: usize) { atomic::compiler_fence(Ordering::SeqCst); DAIF.set(old as u64); } diff --git a/kernel/kernel/src/arch/arm/mod.rs b/kernel/kernel/src/arch/arm/mod.rs index 80ea637c48..5d4989de60 100644 --- a/kernel/kernel/src/arch/arm/mod.rs +++ b/kernel/kernel/src/arch/arm/mod.rs @@ -568,7 +568,10 @@ impl Context { } #[inline] -pub extern "C" fn enable_local_irq() { +/// # Safety +/// No live execution lease may require IRQs to remain masked. Only architecture +/// entry/exit code may bypass the nesting-aware IRQ guard. +pub unsafe extern "C" fn enable_local_irq() { unsafe { core::arch::asm!( "msr basepri, {}", @@ -578,11 +581,11 @@ pub extern "C" fn enable_local_irq() { } } -#[inline] +#[inline(always)] pub extern "C" fn disable_local_irq() { unsafe { core::arch::asm!( - "msr basepri, {}", + "msr basepri_max, {}", "isb", in(reg) DISABLE_LOCAL_IRQ_BASEPRI, options(nostack), @@ -599,7 +602,7 @@ pub extern "C" fn disable_local_irq_save() -> usize { concat!( " mrs {old}, basepri - msr basepri, {val} + msr basepri_max, {val} isb ", ), @@ -614,7 +617,7 @@ pub extern "C" fn disable_local_irq_save() -> usize { #[coverage(off)] #[cfg_attr(debug, inline(never))] -pub extern "C" fn enable_local_irq_restore(old: usize) { +pub unsafe extern "C" fn enable_local_irq_restore(old: usize) { atomic::compiler_fence(Ordering::SeqCst); unsafe { core::arch::asm!( @@ -652,6 +655,7 @@ pub extern "C" fn current_psp() -> usize { #[inline(never)] pub(crate) extern "C" fn switch_context_with_hook(hook: *mut ContextSwitchHookHolder) { + crate::sync::local_context::assert_can_switch(); unsafe { core::arch::asm!( "movs {tmp}, r7", diff --git a/kernel/kernel/src/arch/riscv/mod.rs b/kernel/kernel/src/arch/riscv/mod.rs index b325732324..8af435504e 100644 --- a/kernel/kernel/src/arch/riscv/mod.rs +++ b/kernel/kernel/src/arch/riscv/mod.rs @@ -53,7 +53,9 @@ pub(crate) extern "C" fn pend_switch_context() { let level = disable_local_irq_save(); let id = current_cpu_id(); unsafe { PENDING_SWITCH_CONTEXT[id].set(true) }; - enable_local_irq_restore(level); + unsafe { + enable_local_irq_restore(level); + } } #[inline] @@ -62,7 +64,9 @@ pub(crate) extern "C" fn claim_switch_context() -> bool { let id = current_cpu_id(); let ok = unsafe { PENDING_SWITCH_CONTEXT[id].get() }; unsafe { PENDING_SWITCH_CONTEXT[id].set(false) }; - enable_local_irq_restore(level); + unsafe { + enable_local_irq_restore(level); + } ok } @@ -339,7 +343,10 @@ pub(crate) extern "C" fn disable_local_irq() { } #[inline] -pub(crate) extern "C" fn enable_local_irq() { +/// # Safety +/// No live execution lease may require IRQs to remain masked. Only architecture +/// entry/exit code may bypass the nesting-aware IRQ guard. +pub(crate) unsafe extern "C" fn enable_local_irq() { unsafe { core::arch::asm!(set_mstatus_mie!(), options(nostack)) }; compiler_fence(Ordering::SeqCst); } @@ -364,7 +371,7 @@ pub(crate) extern "C" fn disable_local_irq_save() -> usize { } #[inline] -pub(crate) extern "C" fn enable_local_irq_restore(old: usize) { +pub(crate) unsafe extern "C" fn enable_local_irq_restore(old: usize) { unsafe { core::arch::asm!("csrw mstatus, {old}", old = in(reg) old, options(nostack)) @@ -380,6 +387,7 @@ pub extern "C" fn current_sp() -> usize { } pub(crate) extern "C" fn ecall_switch_context_with_hook(hook: *mut ContextSwitchHookHolder) { + crate::sync::local_context::assert_can_switch(); unsafe { core::arch::asm!( "ecall", @@ -392,6 +400,7 @@ pub(crate) extern "C" fn ecall_switch_context_with_hook(hook: *mut ContextSwitch #[inline(always)] pub(crate) extern "C" fn switch_context_with_hook(hook: *mut ContextSwitchHookHolder) { + crate::sync::local_context::assert_can_switch(); ecall_switch_context_with_hook(hook) } diff --git a/kernel/kernel/src/arch/riscv/trap.rs b/kernel/kernel/src/arch/riscv/trap.rs index 6c1f4d497b..9922afe76b 100644 --- a/kernel/kernel/src/arch/riscv/trap.rs +++ b/kernel/kernel/src/arch/riscv/trap.rs @@ -27,7 +27,7 @@ use crate::{ syscalls::{dispatch_syscall, Context as ScContext}, thread, thread::Thread, - types::Arc, + types::{Arc, NotThreadSafe}, }; use core::{ mem::offset_of, @@ -123,10 +123,16 @@ pub(crate) unsafe extern "C" fn trap_entry() { #[must_use] struct SyscallGuard { isr_ctx: IsrContext, + _not_send: NotThreadSafe, } +// SAFETY: Shared references only expose saved register values; restoring local +// IRQ and trap state happens in Drop, in the acquiring execution context. +unsafe impl Sync for SyscallGuard {} + impl SyscallGuard { pub fn new() -> Self { + crate::sync::local_context::assert_can_switch(); let mut g = Self::default(); unsafe { core::arch::asm!( @@ -143,7 +149,9 @@ impl SyscallGuard { } compiler_fence(Ordering::SeqCst); leave_irq(); - enable_local_irq(); + unsafe { + enable_local_irq(); + } g } } diff --git a/kernel/kernel/src/asynk/mod.rs b/kernel/kernel/src/asynk/mod.rs index 2550283f69..c6e443a32c 100644 --- a/kernel/kernel/src/asynk/mod.rs +++ b/kernel/kernel/src/asynk/mod.rs @@ -22,7 +22,7 @@ use crate::{ scheduler::WaitQueue, static_arc, support::ArcBufferingQueue, - sync::{atomic_wait, SpinLock, SpinLockGuard}, + sync::{atomic_wait, OwnedIrqSpinLockGuard, SpinLock}, thread::{self, Entry, SystemThreadStorage, ThreadKind, ThreadNode}, time::Tick, types::{impl_simple_intrusive_adapter, Arc, IlistHead}, @@ -61,7 +61,7 @@ impl Tasklet { } } - fn lock_inner(&self) -> SpinLockGuard<'_, TaskletInner> { + fn lock_inner(&self) -> OwnedIrqSpinLockGuard<'_, TaskletInner> { self.inner.irqsave_lock() } } diff --git a/kernel/kernel/src/drivers/flash/esp32_rom.rs b/kernel/kernel/src/drivers/flash/esp32_rom.rs index d9e86ce5ae..9510cb3e90 100644 --- a/kernel/kernel/src/drivers/flash/esp32_rom.rs +++ b/kernel/kernel/src/drivers/flash/esp32_rom.rs @@ -72,7 +72,9 @@ pub(crate) fn with_flash_op(body: impl FnOnce() -> R) -> R { let cache_state = unsafe { Cache_Suspend_ICache() }; let result = body(); unsafe { Cache_Resume_ICache(cache_state) }; - enable_local_irq_restore(flags); + unsafe { + enable_local_irq_restore(flags); + } result } #[unsafe(link_section = ".rwtext")] diff --git a/kernel/kernel/src/drivers/flash/internal_flash.rs b/kernel/kernel/src/drivers/flash/internal_flash.rs index 060db63751..01096e672c 100644 --- a/kernel/kernel/src/drivers/flash/internal_flash.rs +++ b/kernel/kernel/src/drivers/flash/internal_flash.rs @@ -16,7 +16,7 @@ //! `write()` does NOT auto-erase; callers must `erase_region` first. use super::esp32_rom; -use crate::{scheduler, sync::Mutex, time::Tick}; +use crate::{scheduler, sync::Mutex, time::Tick, types::NotThreadSafe}; use core::sync::atomic::{AtomicU32, Ordering}; pub const ESP_FLASH_SECTOR_SIZE: usize = 4096; @@ -50,17 +50,26 @@ crate::static_arc! { } static INTERNAL_FLASH_CAPACITY: AtomicU32 = AtomicU32::new(0); +#[must_use] struct InternalFlashGuard { locked: bool, + _not_send: NotThreadSafe, } +// SAFETY: Shared references cannot release the mutex. Drop calls Mutex::post +// on the acquiring thread because ownership cannot be sent to another thread. +unsafe impl Sync for InternalFlashGuard {} + impl InternalFlashGuard { fn acquire() -> Result { let locked = scheduler::is_schedule_ready(); if locked && !INTERNAL_FLASH_LOCK.pend_for(Tick::MAX) { return Err(EspFlashError::Busy); } - Ok(Self { locked }) + Ok(Self { + locked, + _not_send: NotThreadSafe, + }) } } diff --git a/kernel/kernel/src/ffi.rs b/kernel/kernel/src/ffi.rs index 5a40f97d39..cc135bb444 100644 --- a/kernel/kernel/src/ffi.rs +++ b/kernel/kernel/src/ffi.rs @@ -23,8 +23,8 @@ pub extern "C" fn disable_local_irq_save() -> usize { #[coverage(off)] #[unsafe(no_mangle)] -pub extern "C" fn enable_local_irq_restore(val: usize) { - crate::arch::enable_local_irq_restore(val) +pub unsafe extern "C" fn enable_local_irq_restore(val: usize) { + unsafe { crate::arch::enable_local_irq_restore(val) } } #[coverage(off)] diff --git a/kernel/kernel/src/irq.rs b/kernel/kernel/src/irq.rs index d931cd5af2..8cd29a0f68 100644 --- a/kernel/kernel/src/irq.rs +++ b/kernel/kernel/src/irq.rs @@ -12,7 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{arch, support::DisableInterruptGuard, time, types::Uint}; +use crate::{ + arch, + support::DisableInterruptGuard, + time, + types::{NotThreadSafe, Uint}, +}; use blueos_kconfig::CONFIG_NUM_CORES as NUM_CORES; use core::sync::atomic::Ordering; @@ -20,13 +25,23 @@ use core::sync::atomic::Ordering; pub(crate) static mut IRQ_NESTING_COUNT: [Uint; NUM_CORES as usize] = [const { 0 }; NUM_CORES as usize]; +/// Tracks an interrupt until the guard is dropped in the same IRQ context. +#[must_use] pub struct IrqTrace { irq_number: arch::irq::IrqNumber, + _not_send: NotThreadSafe, } +// SAFETY: A shared reference exposes no operations on the CPU-local nesting +// count. Dropping the guard decrements that count and requires ownership. +unsafe impl Sync for IrqTrace {} + impl IrqTrace { pub fn new(irq_number: arch::irq::IrqNumber) -> Self { - let irq_trace = Self { irq_number }; + let irq_trace = Self { + irq_number, + _not_send: NotThreadSafe, + }; irq_trace.enter(); irq_trace } diff --git a/kernel/kernel/src/net/link/esp32_wlan/api.rs b/kernel/kernel/src/net/link/esp32_wlan/api.rs index 1fa948b8ac..e4ff58d5fc 100644 --- a/kernel/kernel/src/net/link/esp32_wlan/api.rs +++ b/kernel/kernel/src/net/link/esp32_wlan/api.rs @@ -348,7 +348,9 @@ pub unsafe extern "C" fn wifi_int_disable(_wifi_int_mux: *mut c_void) -> u32 { } pub unsafe extern "C" fn wifi_int_restore(_wifi_int_mux: *mut c_void, tmp: u32) { - crate::arch::enable_local_irq_restore(tmp as usize); + unsafe { + crate::arch::enable_local_irq_restore(tmp as usize); + } } pub unsafe extern "C" fn task_yield_from_isr() { diff --git a/kernel/kernel/src/scheduler/global_scheduler.rs b/kernel/kernel/src/scheduler/global_scheduler.rs index 49636538dc..608ed4569a 100644 --- a/kernel/kernel/src/scheduler/global_scheduler.rs +++ b/kernel/kernel/src/scheduler/global_scheduler.rs @@ -15,7 +15,8 @@ use crate::{ arch, config::MAX_THREAD_PRIORITY, - sync::spinlock::{SpinLock, SpinLockGuard}, + support::DisableInterruptGuard, + sync::spinlock::SpinLock, thread, thread::{Thread, ThreadNode}, types::{ArcList, ThreadPriority, Uint}, @@ -69,7 +70,7 @@ impl ReadyTable { } #[inline] -fn inner_next_thread(mut tbl: SpinLockGuard<'_, ReadyTable>, index: usize) -> Option { +fn inner_next_thread(tbl: &mut ReadyTable, index: usize) -> Option { let q = &mut tbl.tables[index]; let next = q.pop_front()?; debug_assert_eq!(next.state(), thread::READY); @@ -90,7 +91,16 @@ pub enum PreferMode { #[inline(always)] pub fn next_preferred_thread(prio: ThreadPriority, mode: PreferMode) -> Option { - let mut tbl = unsafe { READY_TABLE.assume_init_ref().irqsave_lock() }; + let irq = DisableInterruptGuard::new(); + next_preferred_thread_with(prio, mode, &irq) +} +#[inline(always)] +pub(crate) fn next_preferred_thread_with( + prio: ThreadPriority, + mode: PreferMode, + irq: &DisableInterruptGuard, +) -> Option { + let mut tbl = unsafe { READY_TABLE.assume_init_ref().lock_with(irq) }; let highest_active = tbl.highest_active(); // highest_active is trailing_zeros(): 0 = highest priority. When a // same-priority ready thread exists, highest_active == prio. @@ -106,11 +116,16 @@ pub fn next_preferred_thread(prio: ThreadPriority, mode: PreferMode) -> Option Option { - let mut tbl = unsafe { READY_TABLE.assume_init_ref().irqsave_lock() }; + let irq = DisableInterruptGuard::new(); + next_ready_thread_with(&irq) +} +#[inline(always)] +pub(crate) fn next_ready_thread_with(irq: &DisableInterruptGuard) -> Option { + let mut tbl = unsafe { READY_TABLE.assume_init_ref().lock_with(irq) }; let highest_active = tbl.highest_active(); #[cfg(debugging_scheduler)] @@ -121,7 +136,7 @@ pub fn next_ready_thread() -> Option { if highest_active > MAX_THREAD_PRIORITY as u32 { return None; } - inner_next_thread(tbl, highest_active as usize) + inner_next_thread(&mut tbl, highest_active as usize) } pub fn queue_ready_thread_with_post_action( @@ -149,7 +164,7 @@ where } #[inline] -fn queue_ready_thread_inner(tbl: &mut SpinLockGuard<'_, ReadyTable>, t: &ThreadNode) -> bool { +fn queue_ready_thread_inner(tbl: &mut ReadyTable, t: &ThreadNode) -> bool { let priority = t.priority(); debug_assert!(priority <= MAX_THREAD_PRIORITY); let q = &mut tbl.tables[priority as usize]; @@ -171,12 +186,21 @@ fn queue_ready_thread_inner(tbl: &mut SpinLockGuard<'_, ReadyTable>, t: &ThreadN // We only queue the thread if old_state equals thread's current state. pub fn queue_ready_thread(old_state: Uint, t: ThreadNode) -> Result<(), Uint> { + let irq = DisableInterruptGuard::new(); + queue_ready_thread_with(old_state, &t, &irq) +} +#[inline(always)] +pub(crate) fn queue_ready_thread_with( + old_state: Uint, + t: &ThreadNode, + irq: &DisableInterruptGuard, +) -> Result<(), Uint> { if old_state == thread::READY { return Err(thread::READY); } - let mut tbl = unsafe { READY_TABLE.assume_init_ref().irqsave_lock() }; + let mut tbl = unsafe { READY_TABLE.assume_init_ref().lock_with(irq) }; t.transfer_state(old_state, thread::READY)?; - let ok = queue_ready_thread_inner(&mut tbl, &t); + let ok = queue_ready_thread_inner(&mut tbl, t); if !ok { // push_back failed: thread is in READY state with sched_node still // detached. Roll back to old_state so the thread stays in its wait @@ -188,7 +212,7 @@ pub fn queue_ready_thread(old_state: Uint, t: ThreadNode) -> Result<(), Uint> { debug_assert!( rollback.is_ok(), "queue_ready_thread rollback failed for TH:0x{:x}: state was {:?}, expected READY; alien_ptr={:?}, priority={}, thread is now ghost", - Thread::id(&t), + Thread::id(t), rollback, t.get_alien_ptr(), t.priority(), @@ -201,7 +225,7 @@ pub fn queue_ready_thread(old_state: Uint, t: ThreadNode) -> Result<(), Uint> { Ok(()) } -fn remove_from_ready_queue_inner(tbl: &mut SpinLockGuard<'_, ReadyTable>, t: &ThreadNode) -> bool { +fn remove_from_ready_queue_inner(tbl: &mut ReadyTable, t: &ThreadNode) -> bool { let priority = t.priority(); debug_assert!(priority <= MAX_THREAD_PRIORITY); let q = &mut tbl.tables[priority as usize]; diff --git a/kernel/kernel/src/scheduler/mod.rs b/kernel/kernel/src/scheduler/mod.rs index 4a1f7284b6..32dec3db98 100644 --- a/kernel/kernel/src/scheduler/mod.rs +++ b/kernel/kernel/src/scheduler/mod.rs @@ -18,7 +18,7 @@ use crate::{ arch::{self, Context}, signal, support::DisableInterruptGuard, - sync::SpinLockGuard, + sync::OwnedIrqSpinLockGuard, thread, thread::{Entry, GlobalQueueVisitor, Thread, ThreadNode}, time, @@ -111,24 +111,24 @@ fn init_timers() { } } -fn set_per_cpu_timer(id: usize, deadline: Tick) { +fn set_per_cpu_timer(id: usize, deadline: Tick, irq: &DisableInterruptGuard) { debug_assert_ne!(deadline, Tick::MAX); debug_assert!(!arch::local_irq_enabled()); unsafe { let iou = TimerIou::from_mut(PER_CPU_TIMER[id].assume_init_mut()); - timer::remove_hard_timer(iou); + timer::remove_hard_timer_with(iou, irq); let tm_mut = PER_CPU_TIMER[id].assume_init_mut(); tm_mut.reset(); let TimerMode::Deadline(d) = &mut tm_mut.mode else { panic!("PER_CPU_TIMER is in a wrong mode") }; *d = deadline; - let _ = timer::add_hard_timer(tm_mut).unwrap(); + let _ = timer::add_hard_timer_with(tm_mut, irq).unwrap(); } } -fn set_current_timer(deadline: Tick) { - set_per_cpu_timer(arch::current_cpu_id(), deadline); +fn set_current_timer(deadline: Tick, irq: &DisableInterruptGuard) { + set_per_cpu_timer(arch::current_cpu_id(), deadline, irq); } pub(crate) struct ContextSwitchHookHolder { @@ -199,12 +199,20 @@ fn switch_current_thread(next: ThreadNode, old_sp: usize) -> usize { next.set_this_round_start_at(now); let time_slices = next.refresh_time_slices(); let deadline = now.add(time_slices); - set_current_timer(deadline); + // SAFETY: all switch_current_thread callers are architecture hooks + // running with the kernel IRQ mask held through this timer update. + let irq = unsafe { DisableInterruptGuard::assume_disabled() }; + set_current_timer(deadline, irq); } #[cfg(thread_stats)] let cycles = time::current_clock_cycles(); #[cfg(thread_stats)] - next.lock().set_start_cycles(cycles); + { + // SAFETY: switch hook runs under the architecture's IRQ mask; this + // short stats update neither schedules nor invokes callbacks. + let irq = unsafe { DisableInterruptGuard::assume_disabled() }; + next.lock_with(irq).set_start_cycles(cycles); + } let next_id = Thread::id(&next); let next_priority = next.priority(); let next_saved_sp = spin_until_ready_to_run(&next); @@ -217,7 +225,11 @@ fn switch_current_thread(next: ThreadNode, old_sp: usize) -> usize { debug_assert_eq!(ok, Ok(())); let mut old = set_current_thread(next); #[cfg(thread_stats)] - old.lock().increment_cycles(cycles); + { + // SAFETY: same masked switch-hook context as above. + let irq = unsafe { DisableInterruptGuard::assume_disabled() }; + old.lock_with(irq).increment_cycles(cycles); + } #[cfg(debugging_scheduler)] crate::trace!( "Switching from 0x{:x}: {{ SP: 0x{:x} PRI: {} }} to 0x{:x}: {{ SP: 0x{:x} PRI: {} }}", @@ -267,12 +279,19 @@ fn cleanup_retired_thread(old: &mut ThreadNode) { // It's usually used in cortex-m's pendsv handler. pub(crate) extern "C" fn relinquish_me_and_return_next_sp(old_sp: usize) -> usize { + if crate::sync::local_context::defer_if_held() { + return old_sp; + } debug_assert_eq!(old_sp % core::mem::align_of::(), 0); debug_assert!(!arch::local_irq_enabled()); debug_assert!(!crate::irq::is_in_irq()); let old = current_thread_ref(); debug_assert_eq!(old.preempt_count(), 0); - let Some(next) = next_preferred_thread(old.priority(), PreferMode::StrictHigher) else { + // SAFETY: PendSV masked kernel IRQs before this hook and restores them only + // after it returns; this hook does not invoke a synchronous context switch. + let irq = unsafe { DisableInterruptGuard::assume_disabled() }; + let Some(next) = next_preferred_thread_with(old.priority(), PreferMode::StrictHigher, irq) + else { #[cfg(debugging_scheduler)] crate::trace!("[TH:0x{:x}] keeps running", Thread::id(old)); return old_sp; @@ -282,13 +301,19 @@ pub(crate) extern "C" fn relinquish_me_and_return_next_sp(old_sp: usize) -> usiz let ok = old.transfer_state(thread::RUNNING, thread::READY); debug_assert_eq!(ok, Ok(())); } else { - let ok = queue_ready_thread(thread::RUNNING, unsafe { Arc::clone_from(old) }); + let ok = queue_ready_thread_with(thread::RUNNING, &unsafe { Arc::clone_from(old) }, irq); debug_assert_eq!(ok, Ok(())); }; switch_current_thread(next, old_sp) } pub fn retire_me() -> ! { + crate::sync::local_context::assert_can_switch(); + assert!( + arch::local_irq_enabled(), + "cannot retire in a masked context" + ); + assert!(!crate::irq::is_in_irq(), "cannot retire in an interrupt"); let retiring = current_thread_ref(); #[cfg(procfs)] { @@ -317,15 +342,16 @@ fn inner_yield(next: ThreadNode) { debug_assert_eq!(ok, Ok(())); }; arch::switch_context_with_hook(&mut hook_holder as *mut _); - debug_assert!(arch::local_irq_enabled()); + assert!(arch::local_irq_enabled()); old.finish_resumption(); } pub fn yield_me() { + crate::sync::local_context::assert_can_switch(); // We don't allow thread yielding with irq disabled. // The scheduler assumes every thread should be resumed with local // irq enabled. - debug_assert!(arch::local_irq_enabled()); + assert!(arch::local_irq_enabled()); let Some(next) = next_ready_thread() else { return; }; @@ -334,7 +360,8 @@ pub fn yield_me() { } pub fn relinquish_me() { - debug_assert!(arch::local_irq_enabled()); + crate::sync::local_context::assert_can_switch(); + assert!(arch::local_irq_enabled()); let old = current_thread_ref(); let Some(next) = next_preferred_thread(old.priority(), PreferMode::SameOrHigher) else { return; @@ -343,7 +370,8 @@ pub fn relinquish_me() { inner_yield(next); } -pub fn suspend_me_for(mut timeout: Tick, wq: Option>) -> bool { +/// Suspends with an optional queue lock acquired by [`SpinLock::irqsave_lock`](crate::sync::SpinLock::irqsave_lock). +pub fn suspend_me_for(mut timeout: Tick, wq: Option>) -> bool { if timeout != Tick::MAX { timeout = Tick::after(timeout); } @@ -354,7 +382,8 @@ pub fn suspend_me_for(mut timeout: Tick, wq: Option>) -> // exception, we have to implement waking up the thread via signal. Currently, // the timer is detached when the thread is re-scheduled, we have to implement // detaching from a waitqueue uniformly. -pub fn suspend_me_until(deadline: Tick, wq: Option>) -> bool { +/// Publishes the suspended state before releasing the supplied queue lock and IRQ mask. +pub fn suspend_me_until(deadline: Tick, wq: Option>) -> bool { if unlikely(!is_schedule_ready()) { return false; } @@ -363,10 +392,15 @@ pub fn suspend_me_until(deadline: Tick, wq: Option>) -> "[TH:0x{:x}] is looking for the next thread", current_thread_id() ); + assert!(!crate::irq::is_in_irq(), "cannot suspend in an interrupt"); + crate::sync::local_context::assert_can_suspend(wq.is_some()); let old = current_thread_ref(); - let current_idle = idle::current_idle_thread_ref(); - debug_assert_ne!(Thread::id(old), Thread::id(current_idle)); - let next = next_ready_thread().unwrap_or_else(|| unsafe { Arc::clone_from(current_idle) }); + debug_assert_ne!(Thread::id(old), Thread::id(idle::current_idle_thread_ref())); + let next = match &wq { + Some(guard) => next_ready_thread_with(guard.irq_guard()), + None => next_ready_thread(), + } + .unwrap_or_else(idle::current_idle_thread); debug_assert_eq!(next.state(), thread::READY); #[cfg(debugging_scheduler)] crate::trace!( @@ -396,7 +430,7 @@ pub fn suspend_me_until(deadline: Tick, wq: Option>) -> } else { arch::switch_context_with_hook(&mut hook_holder as *mut _); } - debug_assert!(arch::local_irq_enabled()); + assert!(arch::local_irq_enabled()); old.finish_resumption(); reached_deadline } @@ -409,13 +443,16 @@ pub fn yield_me_now_or_later() { if unlikely(!is_schedule_ready()) { return; } + if crate::sync::local_context::defer_if_held() { + return; + } if current_thread_ref().preempt_count() != 0 { return; } arch::pend_switch_context(); } -pub fn wake_up_all(mut w: SpinLockGuard<'_, WaitQueue>) -> usize { +pub fn wake_up_all(mut w: OwnedIrqSpinLockGuard<'_, WaitQueue>) -> usize { #[cfg(debugging_scheduler)] crate::trace!("[TH:0x{:x}] is waking up all threads", current_thread_id()); let woken = wait_queue::wake_up_all(&mut w); @@ -427,7 +464,7 @@ pub fn wake_up_all(mut w: SpinLockGuard<'_, WaitQueue>) -> usize { } // wake_up_all, but injects `err` into each woken waiter. Same post-wake yield. -pub fn wake_up_all_with_error(mut w: SpinLockGuard<'_, WaitQueue>, err: i32) -> usize { +pub fn wake_up_all_with_error(mut w: OwnedIrqSpinLockGuard<'_, WaitQueue>, err: i32) -> usize { #[cfg(debugging_scheduler)] crate::trace!( "[TH:0x{:x}] is waking up all threads with error {}", @@ -456,8 +493,10 @@ pub fn wait_and_then_start_schedule() { // Entry of system idle threads. pub extern "C" fn schedule() -> ! { READY_CORES.fetch_add(1, Ordering::Relaxed); - arch::enable_local_irq(); - debug_assert!(arch::local_irq_enabled()); + unsafe { + arch::enable_local_irq(); + } + assert!(arch::local_irq_enabled()); loop { yield_me(); idle::get_idle_hook()(); @@ -466,9 +505,14 @@ pub extern "C" fn schedule() -> ! { #[inline] pub fn current_thread() -> ThreadNode { - let _guard = DisableInterruptGuard::new(); - let my_id = arch::current_cpu_id(); + let guard = DisableInterruptGuard::new(); + current_thread_with(&guard) +} +#[inline] +pub(crate) fn current_thread_with(_irq: &DisableInterruptGuard) -> ThreadNode { + let my_id = arch::current_cpu_id(); + // SAFETY: the IRQ proof keeps this CPU's running thread stable for the clone. unsafe { RUNNING_THREADS[my_id].assume_init_ref().clone() } } diff --git a/kernel/kernel/src/support.rs b/kernel/kernel/src/support.rs index 9786b39ebe..bca83f9ec2 100644 --- a/kernel/kernel/src/support.rs +++ b/kernel/kernel/src/support.rs @@ -14,9 +14,9 @@ use crate::{ arch, - sync::spinlock::{SpinLock, SpinLockGuard}, + sync::spinlock::{OwnedIrqSpinLockGuard, SpinLock}, thread::ThreadNode, - types::{Arc, ArcList, AtomicUint, IntrusiveAdapter, Uint}, + types::{Arc, ArcList, AtomicUint, IntrusiveAdapter, NotThreadSafe, Uint}, }; pub use blueos_infra::storage::Storage; use core::{ @@ -26,43 +26,36 @@ use core::{ sync::atomic::{compiler_fence, AtomicUsize, Ordering}, }; +/// Owns one counted kernel IRQ mask on the current CPU. A shared reference +/// proves that the mask is held for the duration of the borrow. #[derive(Debug)] #[must_use] -pub struct DisableInterruptGuard { - old: usize, -} +pub struct DisableInterruptGuard(NotThreadSafe); impl DisableInterruptGuard { - #[inline] - pub fn new() -> Self { - Self { - old: arch::disable_local_irq_save(), - } + /// Borrows proof of an IRQ mask established by architecture code. + /// + /// # Safety + /// For all of `'a` the current CPU must stay in an admitted context with + /// kernel-capable IRQs masked and without switching execution context. + #[inline(always)] + pub(crate) unsafe fn assume_disabled<'a>() -> &'a Self { + // Borrow this zero-sized marker without creating a counted owner to drop. + const MASKED: &DisableInterruptGuard = &DisableInterruptGuard(NotThreadSafe); + MASKED } -} -impl Drop for DisableInterruptGuard { - #[inline] - fn drop(&mut self) { - arch::enable_local_irq_restore(self.old); - } -} - -#[must_use] -pub struct PlainDisableInterruptGuard; - -impl PlainDisableInterruptGuard { - #[inline] + #[inline(always)] pub fn new() -> Self { - arch::disable_local_irq(); - Self + crate::sync::local_context::enter_irq(); + Self(NotThreadSafe) } } -impl Drop for PlainDisableInterruptGuard { - #[inline] +impl Drop for DisableInterruptGuard { + #[inline(always)] fn drop(&mut self) { - arch::enable_local_irq(); + crate::sync::local_context::exit_irq(); } } @@ -243,8 +236,13 @@ pub fn nonnull_slice_end(ptr: NonNull<[T]>) -> *mut T { pub(crate) struct PerCpuVarAccessGuard { t: ThreadNode, id: u8, + _not_send: NotThreadSafe, } +// SAFETY: Shared access only reads the captured CPU id. Releasing preemption +// protection requires ownership, which stays in the acquiring context. +unsafe impl Sync for PerCpuVarAccessGuard {} + impl PerCpuVarAccessGuard { pub fn new() -> Self { let dig = DisableInterruptGuard::new(); @@ -257,7 +255,11 @@ impl PerCpuVarAccessGuard { t.disable_preempt(); compiler_fence(Ordering::SeqCst); drop(dig); - Self { t, id: id as u8 } + Self { + t, + id: id as u8, + _not_send: NotThreadSafe, + } } #[inline(always)] @@ -307,14 +309,14 @@ impl, const N: usize> ArcBufferingQueue SpinLockGuard<'_, ArcList> { + pub fn advance_active_queue(&self) -> OwnedIrqSpinLockGuard<'_, ArcList> { let i = self.active.fetch_add(1, Ordering::AcqRel) as usize; self.queues[i % N].irqsave_lock() } #[inline] #[allow(clippy::unnecessary_cast)] - pub fn get_active_queue(&self) -> SpinLockGuard<'_, ArcList> { + pub fn get_active_queue(&self) -> OwnedIrqSpinLockGuard<'_, ArcList> { let i = self.active.load(Ordering::Acquire) as usize; self.queues[i % N].irqsave_lock() } diff --git a/kernel/kernel/src/sync/atomic_wait.rs b/kernel/kernel/src/sync/atomic_wait.rs index 8e4af05271..9d17366a6e 100644 --- a/kernel/kernel/src/sync/atomic_wait.rs +++ b/kernel/kernel/src/sync/atomic_wait.rs @@ -110,8 +110,7 @@ pub fn atomic_wait(atom: &AtomicUsize, val: usize, timeout: Tick) -> Result<(), |e| unsafe { Arc::clone_from(e) }, ); let t = scheduler::current_thread(); - let mut we = entry.pending.lock(); - we.take_irq_guard(w.get_guard_mut()); + let mut we = entry.pending.irqsave_lock(); drop(w); #[cfg(debugging_scheduler)] crate::trace!( @@ -126,7 +125,7 @@ pub fn atomic_wait(atom: &AtomicUsize, val: usize, timeout: Tick) -> Result<(), wait_queue::insert(we.deref_mut(), &mut wait_entry, InsertMode::InsertToEnd).unwrap(); reached_deadline = scheduler::suspend_me_for(timeout, Some(we)); w = EntryListHead::lock(); - we = entry.pending.lock(); + we = entry.pending.irqsave_lock(); borrowed_wait_entry = we.pop(borrowed_wait_entry).unwrap(); }); if we.is_empty() { @@ -161,8 +160,7 @@ pub fn atomic_wake(atom: &AtomicUsize, how_many: usize) -> Result let Some(entry) = entry else { return Ok(woken); }; - let mut we = entry.pending.lock(); - we.take_irq_guard(w.get_guard_mut()); + let mut we = entry.pending.irqsave_lock(); drop(w); for next in we.iter() { if scheduler::queue_ready_thread(thread::SUSPENDED, next.thread.clone()).is_err() { diff --git a/kernel/kernel/src/sync/local_context.rs b/kernel/kernel/src/sync/local_context.rs new file mode 100644 index 0000000000..3ed1a95032 --- /dev/null +++ b/kernel/kernel/src/sync/local_context.rs @@ -0,0 +1,210 @@ +// Copyright (c) 2026 vivo Mobile Communication Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! CPU-local execution leases. All bookkeeping is protected by the kernel IRQ +//! mask. No allocation, thread reference counting, or atomic RMW is required. +use crate::{arch, types::NotThreadSafe}; +use core::{ + cell::{Cell, UnsafeCell}, + sync::atomic::{compiler_fence, Ordering}, +}; + +const CPUS: usize = blueos_kconfig::CONFIG_NUM_CORES as usize; +#[cfg(not(smp))] +const _: () = assert!(CPUS == 1, "the local lock backend requires exactly one CPU"); + +struct CpuState { + irq: Cell, + plain: Cell, + saved: Cell, + pending: Cell, +} + +impl CpuState { + const fn new() -> Self { + Self { + irq: Cell::new(0), + plain: Cell::new(0), + saved: Cell::new(0), + pending: Cell::new(false), + } + } +} + +struct States(UnsafeCell<[CpuState; CPUS]>); +// SAFETY: each CPU accesses only its slot, with all kernel-capable asynchronous +// interrupts masked. Scheduling cannot move a live lease to another CPU. +unsafe impl Sync for States {} +static STATES: States = States(UnsafeCell::new([const { CpuState::new() }; CPUS])); + +#[cold] +fn invalid_context() -> ! { + // Do not enter the ordinary logger or panic handler from an excluded IRQ. + #[cfg(target_board = "qemu_mps2_an385")] + semihosting::println!("LOCAL_LOCK_INVALID_CONTEXT"); + core::intrinsics::abort() +} + +#[inline(always)] +fn admit() { + #[cfg(cortex_m)] + { + let ipsr: u32; + unsafe { + core::arch::asm!("mrs {}, ipsr", out(reg) ipsr, options(nomem, nostack, preserves_flags)); + } + match ipsr & 0x1ff { + // SVC is synchronous. Borrow-state/lease updates never invoke SVC + // or callbacks. The actual switch gateway separately checks leases. + 0 | 11 | 14 => (), + 15 => { + if unsafe { + cortex_m::peripheral::SCB::get_priority( + cortex_m::peripheral::scb::SystemHandler::SysTick, + ) + } < arch::irq::IRQ_PRIORITY_FOR_SCHEDULER + { + invalid_context(); + } + } + n if n >= 16 => { + if arch::irq::get_irq_priority(arch::irq::IrqNumber::new((n - 16) as u16)) + < arch::irq::IRQ_PRIORITY_FOR_SCHEDULER + { + invalid_context(); + } + } + _ => invalid_context(), + } + } +} + +#[inline(always)] +unsafe fn state() -> &'static CpuState { + unsafe { &(*STATES.0.get())[arch::current_cpu_id()] } +} + +#[inline(always)] +pub(crate) fn enter_irq() { + admit(); + let old = arch::disable_local_irq_save(); + // SAFETY: mask established before selecting the CPU slot. + let s = unsafe { state() }; + let n = s.irq.get(); + if n == usize::MAX { + invalid_context(); + } + if n == 0 { + s.saved.set(old); + } + s.irq.set(n + 1); +} + +#[inline(always)] +pub(crate) fn exit_irq() { + let s = unsafe { state() }; + let n = s.irq.get(); + assert_ne!(n, 0); + s.irq.set(n - 1); + if n != 1 { + return; + } + let old = s.saved.get(); + let kick = s.plain.get() == 0 && s.pending.replace(false); + compiler_fence(Ordering::SeqCst); + unsafe { + arch::enable_local_irq_restore(old); + } + if kick { + crate::scheduler::yield_me_now_or_later(); + } +} + +#[derive(Debug)] +#[must_use] +pub(crate) struct PreemptLease { + _not_send: NotThreadSafe, +} +impl PreemptLease { + #[inline(always)] + pub(crate) fn new() -> Self { + let _irq = crate::support::DisableInterruptGuard::new(); + let s = unsafe { state() }; + s.plain.set( + s.plain + .get() + .checked_add(1) + .unwrap_or_else(|| invalid_context()), + ); + Self { + _not_send: NotThreadSafe, + } + } +} + +impl Drop for PreemptLease { + #[inline(always)] + fn drop(&mut self) { + let _irq = crate::support::DisableInterruptGuard::new(); + let s = unsafe { state() }; + s.plain.set( + s.plain + .get() + .checked_sub(1) + .expect("unbalanced preemption lease"), + ); + } +} + +// Called with the hardware kernel mask held, including in PendSV before any +// ordinary guard exists. A queued request is retried when the last lease exits. +#[inline(always)] +pub(crate) fn defer_if_held() -> bool { + admit(); + let old = arch::disable_local_irq_save(); + let s = unsafe { state() }; + let held = s.irq.get() != 0 || s.plain.get() != 0; + if held { + s.pending.set(true); + } + unsafe { + arch::enable_local_irq_restore(old); + } + held +} + +pub(crate) fn assert_can_suspend(irq_guard: bool) { + admit(); + let old = arch::disable_local_irq_save(); + let s = unsafe { state() }; + let resume_mask = if irq_guard { s.saved.get() } else { old }; + #[cfg(cortex_m)] + let resumes_enabled = resume_mask == 0; + #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] + let resumes_enabled = resume_mask & (1 << 3) != 0; + #[cfg(target_arch = "aarch64")] + let resumes_enabled = resume_mask & (1 << 7) == 0; + let ok = s.irq.get() == usize::from(irq_guard) && s.plain.get() == 0 && resumes_enabled; + unsafe { + arch::enable_local_irq_restore(old); + } + assert!(ok, "cannot suspend while other execution leases are held"); +} + +pub(crate) fn assert_can_switch() { + assert!( + !defer_if_held(), + "cannot switch while an execution lease is held" + ); +} diff --git a/kernel/kernel/src/sync/mod.rs b/kernel/kernel/src/sync/mod.rs index 35c9cc3204..ad2b20dc56 100644 --- a/kernel/kernel/src/sync/mod.rs +++ b/kernel/kernel/src/sync/mod.rs @@ -16,6 +16,7 @@ pub mod atomic_wait; pub use atomic_wait::{atomic_wait, atomic_wake}; pub mod delay; pub use delay::KernelDelay; +pub(crate) mod local_context; pub mod mqueue; pub mod mutex; pub mod posix_mqueue; @@ -24,7 +25,10 @@ pub mod spinlock; pub use mqueue::MessageQueue; pub use mutex::Mutex; pub use semaphore::Semaphore; -pub use spinlock::{ISpinLock, SpinLock, SpinLockGuard, SpinLockReadGuard, SpinLockWriteGuard}; +pub use spinlock::{ + ISpinLock, OwnedIrqSpinLockGuard, SpinLock, SpinLockGuard, SpinLockReadGuard, + SpinLockWriteGuard, +}; pub mod barrier; pub use barrier::ConstBarrier; #[cfg(event_flags)] diff --git a/kernel/kernel/src/sync/mqueue.rs b/kernel/kernel/src/sync/mqueue.rs index 61648b7549..40417c52f1 100644 --- a/kernel/kernel/src/sync/mqueue.rs +++ b/kernel/kernel/src/sync/mqueue.rs @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -use super::{SpinLock, SpinLockGuard}; +use super::{OwnedIrqSpinLockGuard, SpinLock}; use crate::{ allocator, error::{code, Error}, @@ -118,7 +118,7 @@ impl MessageQueue { } #[inline] - fn lock_buffer(&self) -> SpinLockGuard<'_, BoxedRingBuffer> { + fn lock_buffer(&self) -> OwnedIrqSpinLockGuard<'_, BoxedRingBuffer> { self.queue_buffer.irqsave_lock() } @@ -157,7 +157,7 @@ impl MessageQueue { self.accessible_count[RECV_TYPE].fetch_sub(1, Ordering::Relaxed); } - fn wakeup_pend_receiver(w: &mut SpinLockGuard<'_, WaitQueue>) -> bool { + fn wakeup_pend_receiver(w: &mut OwnedIrqSpinLockGuard<'_, WaitQueue>) -> bool { for next in w.iter() { let t = next.thread.clone(); if scheduler::queue_ready_thread(thread::SUSPENDED, t).is_ok() { @@ -197,7 +197,6 @@ impl MessageQueue { if irq::is_in_irq() { return Err(code::ENOTSUP); } - send_queue.take_irq_guard(&mut buffer_guard); drop(buffer_guard); let reached_deadline; with_iou!(|borrowed_wait_entry| { @@ -287,7 +286,6 @@ impl MessageQueue { if irq::is_in_irq() { return Err(code::ENOTSUP); } - recv_queue.take_irq_guard(&mut buffer_guard); drop(buffer_guard); let reached_deadline; with_iou!(|borrowed_wait_entry| { diff --git a/kernel/kernel/src/sync/mutex.rs b/kernel/kernel/src/sync/mutex.rs index 7990899c98..724f4243ca 100644 --- a/kernel/kernel/src/sync/mutex.rs +++ b/kernel/kernel/src/sync/mutex.rs @@ -26,7 +26,7 @@ // consistency, instead it utilizes a local approach, while keeping the above // invariance, to achieve a not bad result. -use super::{SpinLock, SpinLockGuard}; +use super::{OwnedIrqSpinLockGuard, SpinLock}; use crate::{ irq, scheduler::{self, wait_queue, InsertMode, OffsetOfWait, WaitEntry, WaitQueue}, @@ -77,7 +77,7 @@ impl Mutex { self.pending.irqsave_lock().init() } - pub fn waitqueue_guard(&self) -> SpinLockGuard<'_, WaitQueue> { + pub fn waitqueue_guard(&self) -> OwnedIrqSpinLockGuard<'_, WaitQueue> { self.pending.irqsave_lock() } @@ -199,10 +199,10 @@ impl Mutex { fn inner_pend_for<'a>( ticks: Tick, this_mutex: &'a Arc, - mut this_lock: SpinLockGuard<'a, WaitQueue>, + mut this_lock: OwnedIrqSpinLockGuard<'a, WaitQueue>, this_thread: &'a ThreadNode, owner_thread: &ThreadNode, - ) -> (bool, SpinLockGuard<'a, WaitQueue>) { + ) -> (bool, OwnedIrqSpinLockGuard<'a, WaitQueue>) { // Publish the wait edge before propagating priority. A concurrent donor // either sees this edge and follows it, or promotes us before we acquire // our thread lock below, so this walk observes the inherited priority. @@ -309,7 +309,7 @@ impl Mutex { _target_prio: ThreadPriority, who: &ThreadNode, mutex: &Arc, - mutex_lock: &mut SpinLockGuard<'_, WaitQueue>, + mutex_lock: &mut OwnedIrqSpinLockGuard<'_, WaitQueue>, ) -> bool { let Some(pending_on_mutex) = who.pending_on_mutex() else { return false; diff --git a/kernel/kernel/src/sync/semaphore.rs b/kernel/kernel/src/sync/semaphore.rs index b02cc13d95..27e583b7cd 100644 --- a/kernel/kernel/src/sync/semaphore.rs +++ b/kernel/kernel/src/sync/semaphore.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{SpinLock, SpinLockGuard}; +use super::{OwnedIrqSpinLockGuard, SpinLock}; use crate::{ error::code, irq, scheduler, @@ -71,12 +71,29 @@ impl Semaphore { true } + #[inline(always)] pub fn acquire_notimeout(&self) -> bool where M: InsertModeTrait, { debug_assert!(!irq::is_in_irq()); - let mut w = self.pending.irqsave_lock(); + let w = self.pending.irqsave_lock(); + let old = self.counter.get(); + if old != 0 { + self.counter.set(old - 1); + return true; + } + self.acquire_notimeout_slow::(w) + } + + // Keep wait-entry/timer setup out of the uncontended path's stack frame. + // The existing payload borrow and IRQ lease are passed in, never reacquired. + #[cold] + #[inline(never)] + fn acquire_notimeout_slow<'a, M>(&'a self, mut w: OwnedIrqSpinLockGuard<'a, WaitQueue>) -> bool + where + M: InsertModeTrait, + { loop { let old = self.counter.get(); #[cfg(debugging_scheduler)] @@ -94,8 +111,7 @@ impl Semaphore { return true; } with_iou!(|borrowed_wait_entry| { - let this_thread = scheduler::current_thread(); - let mut wait_entry = WaitEntry::new(this_thread.clone()); + let mut wait_entry = WaitEntry::new(scheduler::current_thread_with(w.irq_guard())); borrowed_wait_entry = wait_queue::insert(w.deref_mut(), &mut wait_entry, M::MODE).unwrap(); let timeout = scheduler::suspend_me_for(Tick::MAX, Some(w)); @@ -103,7 +119,7 @@ impl Semaphore { w = self.pending.irqsave_lock(); borrowed_wait_entry = w.pop(borrowed_wait_entry).unwrap(); // ERESET injected by reset()/delete()? Bail out. - if this_thread.take_wait_error() != 0 { + if wait_entry.thread.take_wait_error() != 0 { return false; } }); @@ -138,15 +154,14 @@ impl Semaphore { } let timeout; with_iou!(|borrowed_wait_entry| { - let this_thread = scheduler::current_thread(); - let mut wait_entry = WaitEntry::new(this_thread.clone()); + let mut wait_entry = WaitEntry::new(scheduler::current_thread_with(w.irq_guard())); borrowed_wait_entry = wait_queue::insert(w.deref_mut(), &mut wait_entry, M::MODE).unwrap(); timeout = scheduler::suspend_me_for(ticks, Some(w)); w = self.pending.irqsave_lock(); borrowed_wait_entry = w.pop(borrowed_wait_entry).unwrap(); // ERESET injected by reset()/delete()? Bail out. - if this_thread.take_wait_error() != 0 { + if wait_entry.thread.take_wait_error() != 0 { return false; } }); @@ -171,7 +186,7 @@ impl Semaphore { self.acquire_timeout::(timeout) } - #[inline] + #[inline(always)] pub fn release(&self) { let w = self.pending.irqsave_lock(); let old = self.counter.get(); @@ -193,11 +208,11 @@ impl Semaphore { #[cold] #[inline(never)] - fn wake_waiter(w: SpinLockGuard<'_, WaitQueue>) { + fn wake_waiter(w: OwnedIrqSpinLockGuard<'_, WaitQueue>) { let mut woken = false; + let irq = w.irq_guard(); for next in w.iter() { - let t = next.thread.clone(); - if scheduler::queue_ready_thread(thread::SUSPENDED, t).is_ok() { + if scheduler::queue_ready_thread_with(thread::SUSPENDED, &next.thread, irq).is_ok() { woken = true; break; } diff --git a/kernel/kernel/src/sync/spinlock.rs b/kernel/kernel/src/sync/spinlock.rs index 23e7fde63c..27b544883a 100644 --- a/kernel/kernel/src/sync/spinlock.rs +++ b/kernel/kernel/src/sync/spinlock.rs @@ -12,220 +12,219 @@ // See the License for the specific language governing permissions and // limitations under the License. +use super::local_context::PreemptLease; use crate::{ support::DisableInterruptGuard, - types::{IRwLock, IntrusiveAdapter, NestedAdapter, RwLock, RwLockReadGuard, RwLockWriteGuard}, + types::{IntrusiveAdapter, NestedAdapter, NotThreadSafe}, }; -use blueos_infra::nolock::{INoLock, NoLock, NoLockReadGuard, NoLockWriteGuard}; use core::{ marker::PhantomData, + mem::ManuallyDrop, ops::{Deref, DerefMut}, - sync::atomic::{compiler_fence, Ordering}, }; +core::cfg_select! { + smp => { + use crate::types::{IRwLock, RwLock, RwLockReadGuard as ReadGuard, RwLockWriteGuard as WriteGuard}; + } + _ => { + use blueos_infra::local_rwlock::{LocalRwLock as RwLock, ReadGuard, WriteGuard}; + } +} + +/// Only SMP can make progress by spinning on a competing guard. +#[inline(always)] +fn acquire(mut attempt: impl FnMut() -> Option, _conflict: &'static str) -> G { + core::cfg_select! { + smp => { + loop { + if let Some(guard) = attempt() { + break guard; + } + core::hint::spin_loop(); + } + } + _ => attempt().expect(_conflict), + } +} + #[derive(Debug)] pub struct SpinLock { - #[cfg(smp)] lock: RwLock, - #[cfg(not(smp))] - lock: NoLock, } +// SAFETY: ownership of the protected value can transfer iff T can. +unsafe impl Send for SpinLock {} +// SAFETY: SMP uses atomic exclusion; UP state transitions are protected by +// kernel IRQ leases. Readers share &T and writers may replace T. +unsafe impl Sync for SpinLock {} pub type SpinLockWriteGuard<'a, T> = SpinLockGuard<'a, T>; - -// See https://doc.rust-lang.org/reference/destructors.html#r-destructors.operation for dropping orders. #[derive(Debug)] -#[repr(C)] #[must_use] pub struct SpinLockGuard<'a, T: ?Sized> { - #[cfg(smp)] - lock_guard: RwLockWriteGuard<'a, T>, - #[cfg(not(smp))] - lock_guard: NoLockWriteGuard<'a, T>, - irq_guard: Option, + lock_guard: ManuallyDrop>, + _preempt: PreemptLease, } - -impl SpinLockGuard<'_, T> { - #[inline] - pub fn take_irq_guard(&mut self, other: &mut SpinLockGuard<'_, S>) { - self.irq_guard = other.irq_guard.take(); - } +// SAFETY: shared access exposes only &T; mutation and release need ownership +// or &mut Guard. Preserve the existing conservative bound. +unsafe impl Sync for SpinLockGuard<'_, T> {} +#[derive(Debug)] +#[must_use] +pub struct SpinLockReadGuard<'a, T: ?Sized> { + lock_guard: ManuallyDrop>, + irq_guard: Option, + _preempt: Option, + _not_send: NotThreadSafe, } - -impl<'a, T: 'a + ?Sized> Deref for SpinLockGuard<'a, T> { +// SAFETY: sharing the guard exposes only &T. +unsafe impl Sync for SpinLockReadGuard<'_, T> {} +impl Deref for SpinLockGuard<'_, T> { type Target = T; - #[inline] + #[inline(always)] fn deref(&self) -> &T { - self.lock_guard.deref() + &self.lock_guard } } - -impl<'a, T: 'a + ?Sized> DerefMut for SpinLockGuard<'a, T> { - #[inline] +impl DerefMut for SpinLockGuard<'_, T> { + #[inline(always)] fn deref_mut(&mut self) -> &mut T { - self.lock_guard.deref_mut() + &mut self.lock_guard } } - -#[derive(Debug)] -#[repr(C)] -#[must_use] -pub struct SpinLockReadGuard<'a, T: ?Sized> { - #[cfg(smp)] - lock_guard: RwLockReadGuard<'a, T>, - #[cfg(not(smp))] - lock_guard: NoLockReadGuard<'a, T>, - irq_guard: Option, +impl Deref for SpinLockReadGuard<'_, T> { + type Target = T; + #[inline(always)] + fn deref(&self) -> &T { + &self.lock_guard + } } - -impl SpinLockReadGuard<'_, T> { - #[inline] - pub fn take_irq_guard(&mut self, other: &mut SpinLockReadGuard<'_, S>) { - self.irq_guard = other.irq_guard.take(); +impl Drop for SpinLockGuard<'_, T> { + #[inline(always)] + fn drop(&mut self) { + // On UP the borrow count must be updated with kernel IRQs masked. + #[cfg(not(smp))] + let _irq = DisableInterruptGuard::new(); + unsafe { + ManuallyDrop::drop(&mut self.lock_guard); + } } } - -impl<'a, T: 'a + ?Sized> Deref for SpinLockReadGuard<'a, T> { - type Target = T; - #[inline] - fn deref(&self) -> &T { - self.lock_guard.deref() +impl Drop for SpinLockReadGuard<'_, T> { + #[inline(always)] + fn drop(&mut self) { + #[cfg(not(smp))] + let _irq = self.irq_guard.is_none().then(DisableInterruptGuard::new); + unsafe { + ManuallyDrop::drop(&mut self.lock_guard); + } } } - impl SpinLock { - pub const fn new(val: T) -> Self { + pub const fn new(data: T) -> Self { Self { - #[cfg(smp)] - lock: RwLock::new(val), - #[cfg(not(smp))] - lock: NoLock::new(val), + lock: RwLock::new(data), } } } - impl SpinLock { - pub fn try_irqsave_lock(&self) -> Option> { + #[inline(always)] + pub fn try_irqsave_lock(&self) -> Option> { let irq_guard = DisableInterruptGuard::new(); - compiler_fence(Ordering::SeqCst); - let mut guard = self.try_lock()?; - debug_assert!(guard.irq_guard.is_none()); - guard.irq_guard = Some(irq_guard); - Some(guard) - } - - pub fn irqsave_lock(&self) -> SpinLockGuard<'_, T> { - let irq_guard = DisableInterruptGuard::new(); - loop { - let Some(mut l) = self.try_lock() else { - core::hint::spin_loop(); - continue; - }; - debug_assert!(l.irq_guard.is_none()); - l.irq_guard = Some(irq_guard); - return l; - } + // SAFETY: the IRQ mask protects the UP borrow count through guard release. + let lock_guard = unsafe { self.lock.try_write()? }; + Some(OwnedIrqSpinLockGuard { + lock_guard, + irq_guard, + }) } - + #[inline(always)] pub fn try_lock(&self) -> Option> { - let lock_guard = self.lock.try_write()?; + let preempt = PreemptLease::new(); + #[cfg(not(smp))] + let _irq = DisableInterruptGuard::new(); + let g = unsafe { self.lock.try_write()? }; Some(SpinLockGuard { + lock_guard: ManuallyDrop::new(g), + _preempt: preempt, + }) + } + #[inline(always)] + pub fn try_irqsave_read(&self) -> Option> { + let irq_guard = DisableInterruptGuard::new(); + let g = unsafe { self.lock.try_read()? }; + Some(SpinLockReadGuard { + lock_guard: ManuallyDrop::new(g), + irq_guard: Some(irq_guard), + _preempt: None, + _not_send: NotThreadSafe, + }) + } + #[inline(always)] + pub fn try_read(&self) -> Option> { + let preempt = PreemptLease::new(); + #[cfg(not(smp))] + let _irq = DisableInterruptGuard::new(); + let g = unsafe { self.lock.try_read()? }; + Some(SpinLockReadGuard { + lock_guard: ManuallyDrop::new(g), irq_guard: None, - lock_guard, + _preempt: Some(preempt), + _not_send: NotThreadSafe, }) } - + #[inline(always)] pub fn lock(&self) -> SpinLockGuard<'_, T> { - loop { - let Some(l) = self.try_lock() else { - core::hint::spin_loop(); - continue; - }; - return l; - } + acquire(|| self.try_lock(), "conflicting UP lock borrow") } - + /// Takes a write guard that owns the local IRQ mask through unlock. #[inline] + pub fn irqsave_lock(&self) -> OwnedIrqSpinLockGuard<'_, T> { + core::cfg_select! { + smp => acquire(|| self.try_irqsave_lock(), "conflicting UP IRQ lock borrow"), + _ => { + let irq_guard = DisableInterruptGuard::new(); + // SAFETY: the IRQ mask protects the UP borrow count through guard release. + let lock_guard = unsafe { self.lock.try_write() }.expect("conflicting UP IRQ lock borrow"); + OwnedIrqSpinLockGuard { lock_guard, irq_guard } + }, + } + } + #[inline(always)] + pub fn read(&self) -> SpinLockReadGuard<'_, T> { + acquire(|| self.try_read(), "conflicting UP read borrow") + } + #[inline(always)] + pub fn irqsave_read(&self) -> SpinLockReadGuard<'_, T> { + acquire(|| self.try_irqsave_read(), "conflicting UP IRQ read borrow") + } + #[inline(always)] pub fn write(&self) -> SpinLockWriteGuard<'_, T> { self.lock() } - - #[inline] + #[inline(always)] pub fn try_write(&self) -> Option> { self.try_lock() } - - pub fn try_irqsave_write(&self) -> Option> { - self.try_irqsave_lock() - } - - pub fn irqsave_write(&self) -> SpinLockWriteGuard<'_, T> { + #[inline(always)] + pub fn irqsave_write(&self) -> OwnedIrqSpinLockGuard<'_, T> { self.irqsave_lock() } - - #[inline] - pub fn try_read(&self) -> Option> { - let lock_guard = self.lock.try_read()?; - Some(SpinLockReadGuard { - irq_guard: None, - lock_guard, - }) - } - - #[inline] - pub fn read(&self) -> SpinLockReadGuard<'_, T> { - loop { - let Some(l) = self.try_read() else { - core::hint::spin_loop(); - continue; - }; - return l; - } - } - - pub fn try_irqsave_read(&self) -> Option> { - let irq_guard = DisableInterruptGuard::new(); - compiler_fence(Ordering::SeqCst); - let mut guard = self.try_read()?; - debug_assert!(guard.irq_guard.is_none()); - guard.irq_guard = Some(irq_guard); - Some(guard) - } - - pub fn irqsave_read(&self) -> SpinLockReadGuard<'_, T> { - loop { - let Some(l) = self.try_irqsave_read() else { - core::hint::spin_loop(); - continue; - }; - return l; - } + #[inline(always)] + pub fn try_irqsave_write(&self) -> Option> { + self.try_irqsave_lock() } - pub fn reader_count(&self) -> usize { - self.lock.reader_count() as usize + let _irq = DisableInterruptGuard::new(); + unsafe { self.lock.reader_count() as usize } } - pub fn writer_count(&self) -> usize { - self.lock.writer_count() as usize - } -} - -impl<'a, T: 'a + ?Sized> SpinLockGuard<'a, T> { - pub fn forget_irq(&mut self) { - if self.irq_guard.is_none() { - return; - } - core::mem::forget(self.irq_guard.take()) + let _irq = DisableInterruptGuard::new(); + unsafe { self.lock.writer_count() as usize } } } -unsafe impl Send for SpinLock {} -unsafe impl Sync for SpinLock {} - #[derive(Default, Debug)] struct ISpinLockOffset>(PhantomData, PhantomData); - impl> const IntrusiveAdapter> for ISpinLockOffset { @@ -234,35 +233,212 @@ impl> const IntrusiveAdapter> { + #[allow(clippy::type_complexity)] #[cfg(smp)] - lock: IRwLock, ISpinLockOffset>>, + lock: IRwLock>>, #[cfg(not(smp))] - lock: INoLock, ISpinLockOffset>>, + lock: RwLock<()>, + _adapter: PhantomData<(T, A)>, } +impl> Default for ISpinLock { + fn default() -> Self { + Self::new() + } +} impl> ISpinLock { pub const fn new() -> Self { Self { - #[cfg(smp)] - lock: IRwLock::new(), - #[cfg(not(smp))] - lock: INoLock::new(), + lock: core::cfg_select! { + smp => IRwLock::new(), + _ => RwLock::new(()), + }, + _adapter: PhantomData, } } - - #[inline] - pub fn irqsave_lock(&self) -> SpinLockGuard<'_, T> { + #[inline(always)] + pub fn irqsave_lock(&self) -> OwnedIrqSpinLockGuard<'_, T> { let irq_guard = DisableInterruptGuard::new(); - compiler_fence(Ordering::SeqCst); - SpinLockGuard { - lock_guard: self.lock.write(), - irq_guard: Some(irq_guard), + let g = core::cfg_select! { + smp => self.lock.write(), + _ => unsafe { + let data = (&raw const *self) + .cast::() + .cast_mut() + .sub(A::offset()) + .cast::(); + self.lock + .project_write(data) + .expect("conflicting UP intrusive lock borrow") + }, + }; + OwnedIrqSpinLockGuard { + lock_guard: g, + irq_guard, } } } - unsafe impl> Send for ISpinLock {} -unsafe impl> Sync for ISpinLock {} +// A write guard can move the projected T out by replacement, requiring T: Send. +// Keep the existing conservative T: Sync bound for this intrusive lock. +unsafe impl> Sync for ISpinLock {} + +/// Payload guard borrowing an existing IRQ proof. Dropping it only unlocks the +/// payload; the proof's owner determines when interrupts can be restored. +#[derive(Debug)] +#[must_use] +pub(crate) struct BorrowedSpinLockGuard<'a, T: ?Sized> { + lock_guard: WriteGuard<'a, T>, + _context: PhantomData<&'a DisableInterruptGuard>, +} +impl Deref for BorrowedSpinLockGuard<'_, T> { + type Target = T; + #[inline(always)] + fn deref(&self) -> &T { + &self.lock_guard + } +} +impl DerefMut for BorrowedSpinLockGuard<'_, T> { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + &mut self.lock_guard + } +} +impl SpinLock { + #[inline(always)] + pub(crate) fn try_lock_with<'a>( + &'a self, + _: &'a DisableInterruptGuard, + ) -> Option> { + // SAFETY: the proof masks UP transitions including the guard's Drop. + let lock_guard = unsafe { self.lock.try_write()? }; + Some(BorrowedSpinLockGuard { + lock_guard, + _context: PhantomData, + }) + } + #[inline(always)] + pub(crate) fn lock_with<'a>( + &'a self, + context: &'a DisableInterruptGuard, + ) -> BorrowedSpinLockGuard<'a, T> { + acquire( + || self.try_lock_with(context), + "conflicting UP borrowed lock", + ) + } +} +impl> ISpinLock { + #[inline(always)] + pub(crate) fn lock_with<'a>( + &'a self, + _: &'a DisableInterruptGuard, + ) -> BorrowedSpinLockGuard<'a, T> { + let lock_guard = core::cfg_select! { + smp => self.lock.write(), + _ => unsafe { + // Existing intrusive projection remains a separate migration task. + let data = (&raw const *self) + .cast::() + .cast_mut() + .sub(A::offset()) + .cast::(); + self.lock + .project_write(data) + .expect("conflicting borrowed intrusive lock") + }, + }; + BorrowedSpinLockGuard { + lock_guard, + _context: PhantomData, + } + } +} + +#[cfg(test)] +mod context_tests { + use super::*; + use blueos_test_macro::test; + + #[test] + fn borrowed_irq_guard_excludes_and_preserves_outer_mask() { + let initially_enabled = crate::arch::local_irq_enabled(); + let a = SpinLock::new(1usize); + let b = SpinLock::new(2usize); + let irq = DisableInterruptGuard::new(); + let context = &irq; + let mut ga = a.lock_with(context); + let gb = b.lock_with(context); + assert!(a.try_lock_with(context).is_none()); + assert!(a.try_irqsave_read().is_none()); + *ga = *gb + 1; + drop(ga); + assert!(!crate::arch::local_irq_enabled()); + assert_eq!(*a.lock_with(context), 3); + drop(gb); + drop(irq); + assert_eq!(crate::arch::local_irq_enabled(), initially_enabled); + } + + #[test] + fn owned_irq_guard_keeps_lock_and_mask_until_drop() { + let initially_enabled = crate::arch::local_irq_enabled(); + let a = SpinLock::new(0usize); + let b = SpinLock::new(7usize); + let waiting = a.irqsave_lock(); + let second = b.irqsave_lock(); + assert!(!crate::arch::local_irq_enabled()); + assert!(a.try_irqsave_lock().is_none()); + drop(second); + assert!(!crate::arch::local_irq_enabled()); + assert_eq!(*b.lock_with(waiting.irq_guard()), 7); + drop(waiting); + assert_eq!(crate::arch::local_irq_enabled(), initially_enabled); + assert!(a.try_irqsave_lock().is_some()); + } + + #[test] + fn forgotten_borrowed_guard_keeps_payload_inaccessible() { + let initially_enabled = crate::arch::local_irq_enabled(); + let a = SpinLock::new(1usize); + let irq = DisableInterruptGuard::new(); + core::mem::forget(a.lock_with(&irq)); + drop(irq); + assert_eq!(crate::arch::local_irq_enabled(), initially_enabled); + assert!(a.try_irqsave_lock().is_none()); + assert!(a.try_irqsave_read().is_none()); + } +} + +/// Holds an exclusive payload borrow and owns the IRQ mask needed for a wait handoff. +#[derive(Debug)] +#[must_use] +pub struct OwnedIrqSpinLockGuard<'a, T: ?Sized> { + // Field drop order releases the payload before restoring IRQs can run the scheduler. + lock_guard: WriteGuard<'a, T>, + irq_guard: DisableInterruptGuard, +} +impl Deref for OwnedIrqSpinLockGuard<'_, T> { + type Target = T; + #[inline(always)] + fn deref(&self) -> &T { + &self.lock_guard + } +} +impl DerefMut for OwnedIrqSpinLockGuard<'_, T> { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + &mut self.lock_guard + } +} +impl OwnedIrqSpinLockGuard<'_, T> { + #[inline(always)] + pub(crate) fn irq_guard(&self) -> &DisableInterruptGuard { + &self.irq_guard + } +} + +#[cfg(all(test, not(smp)))] +mod up_tests; diff --git a/kernel/kernel/src/sync/spinlock/up_tests.rs b/kernel/kernel/src/sync/spinlock/up_tests.rs new file mode 100644 index 0000000000..ca7c91d74f --- /dev/null +++ b/kernel/kernel/src/sync/spinlock/up_tests.rs @@ -0,0 +1,121 @@ +// Copyright (c) 2026 vivo Mobile Communication Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::SpinLock; +use crate::arch; +use blueos_test_macro::test; + +#[test] +fn test_up_spinlock_shared_reads_and_reacquisition() { + let lock = SpinLock::new(7); + let first = lock.try_read().unwrap(); + let second = lock.try_read().unwrap(); + assert_eq!((*first, *second), (7, 7)); + drop(first); + drop(second); + + let mut writer = lock.try_lock().unwrap(); + *writer = 9; + drop(writer); + assert_eq!(*lock.try_read().unwrap(), 9); +} + +#[test] +fn test_up_spinlock_writer_excludes_other_access() { + let lock = SpinLock::new(0); + let writer = lock.try_lock().unwrap(); + // Only inspect acquisition results. Never dereference a conflicting guard, + // so a broken backend is reported by assertions without exercising UB. + let rejects_writer = lock.try_lock().is_none(); + let rejects_reader = lock.try_read().is_none(); + drop(writer); + + assert!(rejects_writer, "UP SpinLock admitted a second writer"); + assert!( + rejects_reader, + "UP SpinLock admitted a reader during a write" + ); + assert!(lock.try_lock().is_some()); +} + +#[test] +fn test_up_spinlock_readers_exclude_writer_until_last_drop() { + let lock = SpinLock::new(0); + let first = lock.try_read().unwrap(); + let second = lock.try_read().unwrap(); + let rejects_with_two = lock.try_lock().is_none(); + drop(first); + let rejects_with_one = lock.try_lock().is_none(); + drop(second); + + assert!( + rejects_with_two, + "UP SpinLock admitted a writer during reads" + ); + assert!( + rejects_with_one, + "UP SpinLock released access before its last reader" + ); + assert!(lock.try_lock().is_some()); +} + +#[test] +fn test_up_spinlock_irqsave_writer_excludes_other_access() { + let lock = SpinLock::new(0); + let initially_enabled = arch::local_irq_enabled(); + let writer = lock.try_irqsave_lock().unwrap(); + let masked = !arch::local_irq_enabled(); + let rejects_writer = lock.try_irqsave_lock().is_none(); + let rejects_reader = lock.try_irqsave_read().is_none(); + let still_masked = !arch::local_irq_enabled(); + // Restore IRQ state before reporting a failure. + drop(writer); + let restored = arch::local_irq_enabled() == initially_enabled; + + assert!( + masked && still_masked && restored, + "IRQ state was not preserved" + ); + assert!( + rejects_writer, + "UP IRQ-save SpinLock admitted a second writer" + ); + assert!( + rejects_reader, + "UP IRQ-save SpinLock admitted a reader during a write" + ); + assert!(lock.try_irqsave_lock().is_some()); +} + +#[test] +fn test_up_spinlock_irqsave_reader_excludes_writer() { + let lock = SpinLock::new(0); + let initially_enabled = arch::local_irq_enabled(); + let reader = lock.try_irqsave_read().unwrap(); + let masked = !arch::local_irq_enabled(); + let rejects_writer = lock.try_irqsave_lock().is_none(); + let still_masked = !arch::local_irq_enabled(); + drop(reader); + let restored = arch::local_irq_enabled() == initially_enabled; + + assert!( + masked && still_masked && restored, + "IRQ state was not preserved" + ); + assert!( + rejects_writer, + "UP IRQ-save SpinLock admitted a writer during a read" + ); + assert!(lock.try_irqsave_lock().is_some()); +} diff --git a/kernel/kernel/src/thread/builder.rs b/kernel/kernel/src/thread/builder.rs index 1f433f9a8a..a29adc2426 100644 --- a/kernel/kernel/src/thread/builder.rs +++ b/kernel/kernel/src/thread/builder.rs @@ -19,7 +19,7 @@ use crate::{ arch::Context, config, debug, scheduler, static_arc, support::Storage, - sync::spinlock::{SpinLock, SpinLockGuard}, + sync::spinlock::{OwnedIrqSpinLockGuard, SpinLock}, thread, trace, types::{ Arc, ArcInner, ArcList, ArcListIterator, AtomicIlistHead as ListHead, StaticListOwner, Uint, @@ -41,7 +41,7 @@ static_arc! { } pub struct GlobalQueueVisitor<'a> { - lock: SpinLockGuard<'a, Head>, + lock: OwnedIrqSpinLockGuard<'a, Head>, it: ArcListIterator<'a, Thread, OffsetOfGlobal>, } diff --git a/kernel/kernel/src/thread/mod.rs b/kernel/kernel/src/thread/mod.rs index 095a22bf74..0dd9ef6f2f 100644 --- a/kernel/kernel/src/thread/mod.rs +++ b/kernel/kernel/src/thread/mod.rs @@ -26,7 +26,7 @@ use crate::{ support::{Region, RegionalObjectBuilder, Storage}, sync::{ mutex::{MutexList, MutexListIterator}, - ISpinLock, Mutex, SpinLock, SpinLockGuard, SpinLockWriteGuard, + ISpinLock, Mutex, OwnedIrqSpinLockGuard, SpinLock, }, thread::builder::GlobalQueue, time::Tick, @@ -311,9 +311,16 @@ impl Thread { // FIXME: rustc miscompiles it if not inlined. #[inline] - pub fn lock(&self) -> SpinLockGuard<'_, Self> { + pub fn lock(&self) -> OwnedIrqSpinLockGuard<'_, Self> { self.lock.irqsave_lock() } + #[inline(always)] + pub(crate) fn lock_with<'a>( + &'a self, + irq: &'a crate::support::DisableInterruptGuard, + ) -> crate::sync::spinlock::BorrowedSpinLockGuard<'a, Self> { + self.lock.lock_with(irq) + } #[inline(always)] pub fn stack_usage(&self) -> usize { @@ -458,7 +465,7 @@ impl Thread { } #[inline] - pub fn acquired_mutexes_mut(&self) -> SpinLockWriteGuard<'_, MutexList> { + pub fn acquired_mutexes_mut(&self) -> OwnedIrqSpinLockGuard<'_, MutexList> { self.acquired_mutexes.irqsave_write() } diff --git a/kernel/kernel/src/time.rs b/kernel/kernel/src/time.rs index 7aa43a47f7..2f4352cec1 100644 --- a/kernel/kernel/src/time.rs +++ b/kernel/kernel/src/time.rs @@ -91,7 +91,12 @@ impl Tick { } pub fn interrupt_at(n: Tick) { - let _guard = DisableInterruptGuard::new(); + let guard = DisableInterruptGuard::new(); + Self::interrupt_at_with(n, &guard); + } + + #[inline] + pub(crate) fn interrupt_at_with(n: Tick, _irq: &DisableInterruptGuard) { if n == Self::MAX { ClockImpl::stop(); return; diff --git a/kernel/kernel/src/time/timer.rs b/kernel/kernel/src/time/timer.rs index d2fc436138..b1b819f1b7 100644 --- a/kernel/kernel/src/time/timer.rs +++ b/kernel/kernel/src/time/timer.rs @@ -17,6 +17,7 @@ use crate::{ boards::ClockImpl, config::MAX_THREAD_PRIORITY, scheduler, static_arc, + support::DisableInterruptGuard, sync::{atomic_wait, atomic_wake, SpinLock}, thread, thread::{Entry, SystemThreadStorage, ThreadKind, ThreadNode}, @@ -231,25 +232,35 @@ pub fn remove_soft_timer<'a>(iou: Iou<'_>) -> Option> { } pub fn add_hard_timer(tm: &mut Timer) -> Option> { - let res; - { - let mut w = HW_TIMERS.irqsave_lock(); - res = w.add(tm); - } + let res = HW_TIMERS.irqsave_lock().add(tm); update_clock_interrupt(); res } +pub(crate) fn add_hard_timer_with<'tm>( + tm: &'tm mut Timer, + irq: &DisableInterruptGuard, +) -> Option> { + let res = HW_TIMERS.lock_with(irq).add(tm); + update_clock_interrupt_with(irq); + res +} + pub fn remove_hard_timer<'a>(iou: Iou<'_>) -> Option> { - let res; - { - let mut w = HW_TIMERS.irqsave_lock(); - res = w.remove(iou); - } + let res = HW_TIMERS.irqsave_lock().remove(iou); update_clock_interrupt(); res } +pub(crate) fn remove_hard_timer_with<'a>( + iou: Iou<'_>, + irq: &DisableInterruptGuard, +) -> Option> { + let res = HW_TIMERS.lock_with(irq).remove(iou); + update_clock_interrupt_with(irq); + res +} + pub fn is_active_hard_timer(iou: &Iou<'_>) -> bool { let mut w = HW_TIMERS.irqsave_lock(); w.is_active(iou) @@ -301,6 +312,20 @@ fn update_clock_interrupt() -> Tick { next_deadline } +fn update_clock_interrupt_with(irq: &DisableInterruptGuard) -> Tick { + let soft_deadline = { + let mut w = unsafe { &SW_TIMER_WORKER.timers }.lock_with(irq); + w.next_deadline().unwrap_or(Tick::MAX) + }; + let hard_deadline = HW_TIMERS + .lock_with(irq) + .next_deadline() + .unwrap_or(Tick::MAX); + let next_deadline = core::cmp::min(soft_deadline, hard_deadline); + Tick::interrupt_at_with(next_deadline, irq); + next_deadline +} + #[cfg(test)] mod tests { use super::*; diff --git a/kernel/kernel/src/types.rs b/kernel/kernel/src/types.rs index b2a0c98da9..1046841ce7 100644 --- a/kernel/kernel/src/types.rs +++ b/kernel/kernel/src/types.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sync::{SpinLock, SpinLockGuard}; +use crate::sync::{OwnedIrqSpinLockGuard, SpinLock}; pub use blueos_infra::{ iheap::{ IouMinHeapNodeMut as IouIntrusiveBinaryHeapNodeMut, MinHeap as IntrusiveBinaryHeap, @@ -30,6 +30,7 @@ pub use blueos_infra::{ }, GenericList, }, + marker::NotThreadSafe, tinyarc::{ TinyArc as Arc, TinyArcCas as ArcCas, TinyArcInner as ArcInner, TinyArcList as ArcList, TinyArcListIterator as ArcListIterator, @@ -89,7 +90,7 @@ pub(crate) struct UniqueListHeadAccessGuard< A: IntrusiveAdapter + 'static, O: StaticListOwner, >( - SpinLockGuard<'static, AtomicIlistHead>, + OwnedIrqSpinLockGuard<'static, AtomicIlistHead>, PhantomData, ); @@ -97,7 +98,7 @@ impl + 'static, O: StaticListOwner> UniqueListHeadAccessGuard { #[inline] - pub fn new(w: SpinLockGuard<'static, AtomicIlistHead>) -> Self { + pub fn new(w: OwnedIrqSpinLockGuard<'static, AtomicIlistHead>) -> Self { Self(w, PhantomData) } @@ -117,7 +118,7 @@ impl + 'static, O: StaticListOwner> } #[inline] - pub fn get_guard_mut(&mut self) -> &mut SpinLockGuard<'static, AtomicIlistHead> { + pub fn get_guard_mut(&mut self) -> &mut OwnedIrqSpinLockGuard<'static, AtomicIlistHead> { &mut self.0 } }