From 4446e13de17ddd3e25bf46d32589c225b40b1c0f Mon Sep 17 00:00:00 2001 From: Hegui Dai Date: Fri, 20 Nov 2026 09:49:36 +0800 Subject: [PATCH 01/10] fix(infra): keep RwLock guards on the acquiring thread Introduce NotThreadSafe and carry it through read, write and upgradable guards, including conversions. Express when a shared reference to each guard may cross threads. --- kernel/infra/src/lib.rs | 1 + kernel/infra/src/marker.rs | 28 ++++++++++++++++ kernel/infra/src/tinyrwlock.rs | 61 +++++++++++++++++++++++++--------- kernel/kernel/src/types.rs | 1 + 4 files changed, 75 insertions(+), 16 deletions(-) create mode 100644 kernel/infra/src/marker.rs diff --git a/kernel/infra/src/lib.rs b/kernel/infra/src/lib.rs index ef4859edb..437e32836 100644 --- a/kernel/infra/src/lib.rs +++ b/kernel/infra/src/lib.rs @@ -31,6 +31,7 @@ pub mod intrusive; pub mod lifetime; pub mod list; +pub mod marker; pub mod no_let_underscore; pub mod rbtree; pub mod ringbuffer; diff --git a/kernel/infra/src/marker.rs b/kernel/infra/src/marker.rs new file mode 100644 index 000000000..ff314b9da --- /dev/null +++ b/kernel/infra/src/marker.rs @@ -0,0 +1,28 @@ +// Copyright (c) 2025 vivo Mobile Communication Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Marker types for expressing thread contracts. + +use core::marker::PhantomData; + +/// A zero-sized field that prevents automatic `Send` and `Sync` implementations. +/// +/// Use this for guards that must stay in their acquiring execution context. +/// A containing type may explicitly implement `Sync` if sharing references to it +/// is safe, while leaving ownership transfer disabled. +pub type NotThreadSafe = PhantomData<*mut ()>; + +/// Constructs a [`NotThreadSafe`](type@NotThreadSafe) marker. +#[allow(non_upper_case_globals)] +pub const NotThreadSafe: NotThreadSafe = PhantomData; diff --git a/kernel/infra/src/tinyrwlock.rs b/kernel/infra/src/tinyrwlock.rs index be4685492..792f68e54 100644 --- a/kernel/infra/src/tinyrwlock.rs +++ b/kernel/infra/src/tinyrwlock.rs @@ -17,8 +17,11 @@ // SPDX-License: MIT //! A lock that provides data access to either one writer or many readers. +//! +//! Guards stay on the thread that acquired them. Upgrading or downgrading a +//! guard preserves this contract. -use crate::intrusive::Adapter; +use crate::{intrusive::Adapter, marker::NotThreadSafe}; use core::{ cell::UnsafeCell, fmt, @@ -99,6 +102,11 @@ pub struct RwLock { data: UnsafeCell, } +// SAFETY: Moving the lock transfers ownership of its protected data. +unsafe impl Send for RwLock {} +// SAFETY: Writers can move T out by replacement; concurrent readers share &T. +unsafe impl Sync for RwLock {} + const READER: Usize = 1 << 2; const UPGRADED: Usize = 1 << 1; const WRITER: Usize = 1; @@ -111,8 +119,12 @@ const WRITER: Usize = 1; pub struct RwLockReadGuard<'a, T: 'a + ?Sized> { lock: &'a AtomicUsize, data: *const T, + _not_send: NotThreadSafe, } +// SAFETY: Sharing a read guard exposes only &T; releasing it consumes the guard. +unsafe impl Sync for RwLockReadGuard<'_, T> {} + /// A guard that provides mutable data access. /// /// When the guard falls out of scope it will release the lock. @@ -120,8 +132,13 @@ pub struct RwLockReadGuard<'a, T: 'a + ?Sized> { pub struct RwLockWriteGuard<'a, T: 'a + ?Sized> { lock: &'a AtomicUsize, data: *mut T, + _not_send: NotThreadSafe, } +// SAFETY: Sharing a write guard exposes only &T. Mutation requires &mut Guard, +// and releasing or changing lock mode consumes it. Keep the existing Send bound. +unsafe impl Sync for RwLockWriteGuard<'_, T> {} + /// A guard that provides immutable data access but can be upgraded to [`RwLockWriteGuard`]. /// /// No writers or other upgradeable guards can exist while this is in scope. New reader @@ -133,19 +150,11 @@ pub struct RwLockWriteGuard<'a, T: 'a + ?Sized> { pub struct RwLockUpgradableGuard<'a, T: 'a + ?Sized> { lock: &'a AtomicUsize, data: *const T, + _not_send: NotThreadSafe, } -// Same unsafe impls as `std::sync::RwLock` -unsafe impl Send for RwLock {} -unsafe impl Sync for RwLock {} - -unsafe impl Send for RwLockWriteGuard<'_, T> {} -unsafe impl Sync for RwLockWriteGuard<'_, T> {} - -unsafe impl Send for RwLockReadGuard<'_, T> {} -unsafe impl Sync for RwLockReadGuard<'_, T> {} - -unsafe impl Send for RwLockUpgradableGuard<'_, T> {} +// SAFETY: Sharing this guard exposes only &T; upgrading, downgrading or releasing +// it consumes the guard. Keep the existing Send bound. unsafe impl Sync for RwLockUpgradableGuard<'_, T> {} impl RwLock { @@ -339,6 +348,7 @@ impl RwLock { Some(RwLockReadGuard { lock: &self.lock, data: self.data.get(), + _not_send: NotThreadSafe, }) } } @@ -409,6 +419,7 @@ impl RwLock { Some(RwLockWriteGuard { lock: &self.lock, data: self.data.get(), + _not_send: NotThreadSafe, }) } else { None @@ -446,6 +457,7 @@ impl RwLock { Some(RwLockUpgradableGuard { lock: &self.lock, data: self.data.get(), + _not_send: NotThreadSafe, }) } else { // We can't unflip the UPGRADED bit back just yet as there is another upgradeable or write lock. @@ -573,6 +585,7 @@ impl<'rwlock, T: ?Sized> RwLockUpgradableGuard<'rwlock, T> { Ok(RwLockWriteGuard { lock, data: data as *mut T, + _not_send: NotThreadSafe, }) } else { Err(self) @@ -619,7 +632,11 @@ impl<'rwlock, T: ?Sized> RwLockUpgradableGuard<'rwlock, T> { // Dropping self removes the UPGRADED bit mem::drop(self); - RwLockReadGuard { lock, data } + RwLockReadGuard { + lock, + data, + _not_send: NotThreadSafe, + } } /// Leak the lock guard, yielding a reference to the underlying data. @@ -677,7 +694,11 @@ impl<'rwlock, T: ?Sized> RwLockWriteGuard<'rwlock, T> { // Dropping self removes the UPGRADED bit mem::drop(self); - RwLockReadGuard { lock, data } + RwLockReadGuard { + lock, + data, + _not_send: NotThreadSafe, + } } /// Downgrades the writable lock guard to an upgradable, shared lock guard. Cannot fail and is guaranteed not to spin. @@ -707,7 +728,11 @@ impl<'rwlock, T: ?Sized> RwLockWriteGuard<'rwlock, T> { // Dropping self removes the UPGRADED bit mem::forget(self); - RwLockUpgradableGuard { lock, data } + RwLockUpgradableGuard { + lock, + data, + _not_send: NotThreadSafe, + } } /// Leak the lock guard, yielding a mutable reference to the underlying data. @@ -848,7 +873,11 @@ impl> IRwLock { let lock = g.lock; let data = self.this_mut() as *mut T; core::mem::forget(g); - RwLockWriteGuard { lock, data } + RwLockWriteGuard { + lock, + data, + _not_send: NotThreadSafe, + } } } diff --git a/kernel/kernel/src/types.rs b/kernel/kernel/src/types.rs index b2a0c98da..239332c6a 100644 --- a/kernel/kernel/src/types.rs +++ b/kernel/kernel/src/types.rs @@ -30,6 +30,7 @@ pub use blueos_infra::{ }, GenericList, }, + marker::NotThreadSafe, tinyarc::{ TinyArc as Arc, TinyArcCas as ArcCas, TinyArcInner as ArcInner, TinyArcList as ArcList, TinyArcListIterator as ArcListIterator, From e493e8f5f0fb2669f8276bb45bb39406e8b2f659 Mon Sep 17 00:00:00 2001 From: Hegui Dai Date: Fri, 20 Nov 2026 09:49:36 +0800 Subject: [PATCH 02/10] refactor(infra): replace NoLock with checked local read/write borrows Track outstanding reads and exclusive writes in LocalRwLock. Its callers supply the execution-context exclusion needed to update the borrow state. --- kernel/infra/src/lib.rs | 2 +- kernel/infra/src/local_rwlock.rs | 282 +++++++++++++++++++++++++++++++ kernel/infra/src/nolock.rs | 169 ------------------ 3 files changed, 283 insertions(+), 170 deletions(-) create mode 100644 kernel/infra/src/local_rwlock.rs delete mode 100644 kernel/infra/src/nolock.rs diff --git a/kernel/infra/src/lib.rs b/kernel/infra/src/lib.rs index 437e32836..7220eeb2f 100644 --- a/kernel/infra/src/lib.rs +++ b/kernel/infra/src/lib.rs @@ -41,5 +41,5 @@ pub mod string; pub mod tinyarc; pub use tinyarc::sorted_list; pub mod iheap; -pub mod nolock; +pub mod local_rwlock; pub mod tinyrwlock; diff --git a/kernel/infra/src/local_rwlock.rs b/kernel/infra/src/local_rwlock.rs new file mode 100644 index 000000000..36c61c286 --- /dev/null +++ b/kernel/infra/src/local_rwlock.rs @@ -0,0 +1,282 @@ +// Copyright (c) 2026 vivo Mobile Communication Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Non-atomic read/write borrow checking with externally synchronized state. +//! +//! This module does not mask interrupts or prevent scheduling. Its caller must +//! serialize borrow-state access, including guard destruction. The kernel's UP +//! `SpinLock` wrapper supplies this protection; other users must supply their own. + +use crate::marker::NotThreadSafe; +use core::{ + cell::{Cell, UnsafeCell}, + ops::{Deref, DerefMut}, +}; + +/// A non-atomic read/write lock whose borrow bookkeeping needs external protection. +/// +/// The lock is not `Sync`; it does not provide cross-thread or interrupt exclusion +/// by itself. Obtaining guards requires an unsafe call that establishes the +/// protection needed by both acquisition and subsequent guard destruction. +pub struct LocalRwLock { + state: BorrowState, + data: UnsafeCell, +} + +// Zero is idle, WRITER is exclusive, and all other values count readers. +struct BorrowState(Cell); + +impl LocalRwLock { + pub const fn new(data: T) -> Self { + Self { + state: BorrowState::new(), + data: UnsafeCell::new(data), + } + } +} + +impl LocalRwLock { + /// Attempts to acquire an exclusive payload borrow. + /// + /// # Safety + /// The caller must serialize every access to this lock's borrow state, + /// including acquisitions, count queries, and destruction of any returned + /// guard. No concurrent or reentrant state access may interrupt one of these + /// operations. Protection must also be established when the guard is dropped; + /// protecting only this call is insufficient. + #[inline(always)] + pub unsafe fn try_write(&self) -> Option> { + self.state.try_write().then(|| WriteGuard { + state: &self.state, + data: self.data.get(), + _not_send: NotThreadSafe, + }) + } + + /// Attempts to acquire a shared payload borrow. + /// + /// # Safety + /// The caller must uphold the state-access and guard-destruction protection + /// requirements of [`Self::try_write`]. + #[inline(always)] + pub unsafe fn try_read(&self) -> Option> { + self.state.try_read().then(|| ReadGuard { + state: &self.state, + data: self.data.get(), + _not_send: NotThreadSafe, + }) + } + + /// Returns the number of active read guards. + /// + /// # Safety + /// This query must be serialized with every other access to the borrow state, + /// including guard destruction, as described in [`Self::try_write`]. + #[inline(always)] + pub unsafe fn reader_count(&self) -> usize { + self.state.reader_count() + } + + /// Returns one if a write guard is active, otherwise zero. + /// + /// # Safety + /// This query must be serialized with every other access to the borrow state, + /// including guard destruction, as described in [`Self::try_write`]. + #[inline(always)] + pub unsafe fn writer_count(&self) -> usize { + self.state.writer_count() + } +} + +impl LocalRwLock<()> { + /// Acquires a write borrow for an externally supplied payload. + /// + /// This is a transitional adapter for existing intrusive callers. Borrow-state + /// exclusion alone does not establish the validity of an owner projection. + /// + /// # Safety + /// The caller must uphold the protection requirements of [`Self::try_write`]. + /// In addition, `data` must be aligned, non-null, point to an initialized `T`, + /// and remain valid for the returned guard's lifetime. It must permit the + /// shared and exclusive references exposed by the guard, with no conflicting + /// accesses through other pointers or references. The payload must not overlap + /// the borrow bookkeeping accessed through this lock. + #[inline(always)] + pub unsafe fn project_write(&self, data: *mut T) -> Option> { + self.state.try_write().then(|| WriteGuard { + state: &self.state, + data, + _not_send: NotThreadSafe, + }) + } +} + +impl BorrowState { + const WRITER: usize = usize::MAX; + + const fn new() -> Self { + Self(Cell::new(0)) + } + + #[inline(always)] + fn try_read(&self) -> bool { + let n = self.0.get(); + if n >= Self::WRITER - 1 { + return false; + } + self.0.set(n + 1); + true + } + + #[inline(always)] + fn try_write(&self) -> bool { + if self.0.get() != 0 { + return false; + } + self.0.set(Self::WRITER); + true + } + + #[inline(always)] + fn release_read(&self) { + let n = self.0.get(); + assert!(n != 0 && n != Self::WRITER); + self.0.set(n - 1); + } + + #[inline(always)] + fn release_write(&self) { + assert_eq!(self.0.get(), Self::WRITER); + self.0.set(0); + } + + #[inline(always)] + fn reader_count(&self) -> usize { + match self.0.get() { + Self::WRITER => 0, + n => n, + } + } + + #[inline(always)] + fn writer_count(&self) -> usize { + usize::from(self.0.get() == Self::WRITER) + } +} + +/// An exclusive payload borrow. Dropping it updates externally protected state. +#[must_use] +pub struct WriteGuard<'a, T: ?Sized> { + state: &'a BorrowState, + data: *mut T, + _not_send: NotThreadSafe, +} + +/// A shared payload borrow. Dropping it updates externally protected state. +#[must_use] +pub struct ReadGuard<'a, T: ?Sized> { + state: &'a BorrowState, + data: *const T, + _not_send: NotThreadSafe, +} + +impl Deref for WriteGuard<'_, T> { + type Target = T; + #[inline(always)] + fn deref(&self) -> &T { + unsafe { &*self.data } + } +} + +impl DerefMut for WriteGuard<'_, T> { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + unsafe { &mut *self.data } + } +} + +impl Deref for ReadGuard<'_, T> { + type Target = T; + #[inline(always)] + fn deref(&self) -> &T { + unsafe { &*self.data } + } +} + +impl Drop for WriteGuard<'_, T> { + #[inline(always)] + fn drop(&mut self) { + self.state.release_write(); + } +} + +impl Drop for ReadGuard<'_, T> { + #[inline(always)] + fn drop(&mut self) { + self.state.release_read(); + } +} + +impl core::fmt::Debug for LocalRwLock { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("LocalRwLock { .. }") + } +} + +impl core::fmt::Debug for WriteGuard<'_, T> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("LocalWriteGuard { .. }") + } +} + +impl core::fmt::Debug for ReadGuard<'_, T> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("LocalReadGuard { .. }") + } +} + +#[cfg(test)] +mod tests { + use super::{BorrowState, Cell}; + + #[test] + fn exclusion_and_release() { + let s = BorrowState::new(); + assert!(s.try_write()); + assert!(!s.try_write()); + assert!(!s.try_read()); + s.release_write(); + assert!(s.try_read()); + assert!(s.try_read()); + assert!(!s.try_write()); + s.release_read(); + assert!(!s.try_write()); + s.release_read(); + assert!(s.try_write()); + s.release_write(); + assert_eq!(s.reader_count(), 0); + assert_eq!(s.writer_count(), 0); + } + + #[test] + fn reader_overflow_cannot_unlock() { + let s = BorrowState(Cell::new(BorrowState::WRITER - 2)); + assert!(s.try_read()); + assert!(!s.try_read()); + assert!(!s.try_write()); + assert_eq!(s.reader_count(), BorrowState::WRITER - 1); + s.release_read(); + assert!(s.try_read()); + } +} diff --git a/kernel/infra/src/nolock.rs b/kernel/infra/src/nolock.rs deleted file mode 100644 index d9e648ed6..000000000 --- a/kernel/infra/src/nolock.rs +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright (c) 2026 vivo Mobile Communication Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Implement a lock based on UnsafeCell to optimize performance on single core. -// The APIs of this lock should be compatible with that in tinyrwlock. - -use crate::intrusive::Adapter; -use core::{ - cell::UnsafeCell, - marker::PhantomData, - ops::{Deref, DerefMut}, -}; - -#[derive(Debug)] -pub struct NoLock { - data: UnsafeCell, -} - -impl Default for NoLock { - fn default() -> Self { - Self::new(Default::default()) - } -} - -#[derive(Debug)] -#[must_use] -pub struct NoLockWriteGuard<'a, T: 'a + ?Sized> { - inner: *mut T, - _a: PhantomData<&'a mut T>, -} - -impl Deref for NoLockWriteGuard<'_, T> { - type Target = T; - fn deref(&self) -> &T { - unsafe { &*self.inner } - } -} - -impl DerefMut for NoLockWriteGuard<'_, T> { - fn deref_mut(&mut self) -> &mut T { - unsafe { &mut *self.inner } - } -} - -#[derive(Debug)] -#[must_use] -pub struct NoLockReadGuard<'a, T: 'a + ?Sized> { - inner: *const T, - _a: PhantomData<&'a mut T>, -} - -impl Deref for NoLockReadGuard<'_, T> { - type Target = T; - fn deref(&self) -> &T { - unsafe { &*self.inner } - } -} - -impl NoLock { - pub const fn new(val: T) -> Self { - Self { - data: UnsafeCell::new(val), - } - } -} - -impl NoLock { - pub fn try_write(&self) -> Option> { - Some(NoLockWriteGuard { - inner: self.data.get(), - _a: PhantomData, - }) - } - - pub fn try_read(&self) -> Option> { - Some(NoLockReadGuard { - inner: self.data.get() as *const _, - _a: PhantomData, - }) - } - - pub fn write(&self) -> NoLockWriteGuard<'_, T> { - self.try_write().unwrap() - } - - pub fn read(&self) -> NoLockReadGuard<'_, T> { - self.try_read().unwrap() - } - - pub fn reader_count(&self) -> usize { - usize::MAX - } - - pub fn writer_count(&self) -> usize { - usize::MAX - } -} - -unsafe impl Send for NoLock {} -unsafe impl Sync for NoLock {} -unsafe impl Send for NoLockWriteGuard<'_, T> {} -unsafe impl Sync for NoLockWriteGuard<'_, T> {} -unsafe impl Send for NoLockReadGuard<'_, T> {} -unsafe impl Sync for NoLockReadGuard<'_, T> {} - -#[derive(Default, Debug)] -pub struct INoLock> { - lock: NoLock<()>, - _a: PhantomData<(T, A)>, -} - -impl> INoLock { - #[inline] - pub const fn new() -> Self { - Self { - lock: NoLock::new(()), - _a: PhantomData, - } - } - - #[inline] - fn this_mut(&self) -> &mut T { - let ptr = (&raw const *self).cast::().cast_mut(); - let base = unsafe { ptr.sub(A::offset()) as *mut T }; - unsafe { &mut *base } - } - - #[inline] - pub fn write(&self) -> NoLockWriteGuard<'_, T> { - let inner = self.this_mut() as *mut T; - NoLockWriteGuard { - inner, - _a: PhantomData, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_read() { - let l = NoLock::new(42); - let r = l.read(); - assert_eq!(*r, 42); - } - - #[test] - fn test_write() { - let l = NoLock::new(0); - let mut w = l.write(); - *w = 42; - drop(w); - let r = l.read(); - assert_eq!(*r, 42); - } -} From c0a6e16599cb6f1105942b66d4c4dda4b6c92612 Mon Sep 17 00:00:00 2001 From: Hegui Dai Date: Fri, 20 Nov 2026 09:49:36 +0800 Subject: [PATCH 03/10] feat(sync): track CPU-local guards and lend proof of IRQ masking Count nested IRQ leases, restore the saved mask on the last release and track live preemption leases. Borrow DisableInterruptGuard as proof of masking and remove PlainDisableInterruptGuard. Keep guards that release IRQ nesting, per-CPU access or the flash mutex in their acquiring context. --- .../rust-resource-guard-must-use/SKILL.md | 6 +- .../src/drivers/flash/internal_flash.rs | 13 +- kernel/kernel/src/irq.rs | 19 +- kernel/kernel/src/support.rs | 58 ++--- kernel/kernel/src/sync/local_context.rs | 210 ++++++++++++++++++ kernel/kernel/src/sync/mod.rs | 1 + 6 files changed, 271 insertions(+), 36 deletions(-) create mode 100644 kernel/kernel/src/sync/local_context.rs diff --git a/.agents/skills/rust-resource-guard-must-use/SKILL.md b/.agents/skills/rust-resource-guard-must-use/SKILL.md index d48309e4b..c5a5eaf5d 100644 --- a/.agents/skills/rust-resource-guard-must-use/SKILL.md +++ b/.agents/skills/rust-resource-guard-must-use/SKILL.md @@ -23,7 +23,7 @@ The following types in `kernel/` already have type-level `#[must_use]` attribute | Types | Source | Guard behavior | | --- | --- | --- | -| `DisableInterruptGuard`, `PlainDisableInterruptGuard` | [support.rs](../../../kernel/kernel/src/support.rs) | The former restores the saved local interrupt state in `Drop`; the latter re-enables local interrupts in `Drop`. | +| `DisableInterruptGuard` | [support.rs](../../../kernel/kernel/src/support.rs) | Drops one counted IRQ guard and restores the saved mask on the last release. | | `PreemptGuard` | [thread/mod.rs](../../../kernel/kernel/src/thread/mod.rs) | Calls `enable_preempt()` on its thread in `Drop`. | | `SpinLockGuard`, `SpinLockReadGuard` | [sync/spinlock.rs](../../../kernel/kernel/src/sync/spinlock.rs) | Hold the underlying lock guard and an optional interrupt guard. Cleanup occurs through their fields when dropped. | | `RwLockReadGuard`, `RwLockWriteGuard`, `RwLockUpgradableGuard` | [tinyrwlock.rs](../../../kernel/infra/src/tinyrwlock.rs) | Update the reader count or clear lock state in `Drop`. | @@ -34,9 +34,7 @@ Actual type definition from `kernel/kernel/src/support.rs`: ```rust #[derive(Debug)] #[must_use] -pub struct DisableInterruptGuard { - old: usize, -} +pub struct DisableInterruptGuard(NotThreadSafe); ``` Before delivery, check every resource guard generated or modified in the task. Confirm that the attribute is on the type definition, including types generated by macros. diff --git a/kernel/kernel/src/drivers/flash/internal_flash.rs b/kernel/kernel/src/drivers/flash/internal_flash.rs index 060db6375..01096e672 100644 --- a/kernel/kernel/src/drivers/flash/internal_flash.rs +++ b/kernel/kernel/src/drivers/flash/internal_flash.rs @@ -16,7 +16,7 @@ //! `write()` does NOT auto-erase; callers must `erase_region` first. use super::esp32_rom; -use crate::{scheduler, sync::Mutex, time::Tick}; +use crate::{scheduler, sync::Mutex, time::Tick, types::NotThreadSafe}; use core::sync::atomic::{AtomicU32, Ordering}; pub const ESP_FLASH_SECTOR_SIZE: usize = 4096; @@ -50,17 +50,26 @@ crate::static_arc! { } static INTERNAL_FLASH_CAPACITY: AtomicU32 = AtomicU32::new(0); +#[must_use] struct InternalFlashGuard { locked: bool, + _not_send: NotThreadSafe, } +// SAFETY: Shared references cannot release the mutex. Drop calls Mutex::post +// on the acquiring thread because ownership cannot be sent to another thread. +unsafe impl Sync for InternalFlashGuard {} + impl InternalFlashGuard { fn acquire() -> Result { let locked = scheduler::is_schedule_ready(); if locked && !INTERNAL_FLASH_LOCK.pend_for(Tick::MAX) { return Err(EspFlashError::Busy); } - Ok(Self { locked }) + Ok(Self { + locked, + _not_send: NotThreadSafe, + }) } } diff --git a/kernel/kernel/src/irq.rs b/kernel/kernel/src/irq.rs index d931cd5af..8cd29a0f6 100644 --- a/kernel/kernel/src/irq.rs +++ b/kernel/kernel/src/irq.rs @@ -12,7 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{arch, support::DisableInterruptGuard, time, types::Uint}; +use crate::{ + arch, + support::DisableInterruptGuard, + time, + types::{NotThreadSafe, Uint}, +}; use blueos_kconfig::CONFIG_NUM_CORES as NUM_CORES; use core::sync::atomic::Ordering; @@ -20,13 +25,23 @@ use core::sync::atomic::Ordering; pub(crate) static mut IRQ_NESTING_COUNT: [Uint; NUM_CORES as usize] = [const { 0 }; NUM_CORES as usize]; +/// Tracks an interrupt until the guard is dropped in the same IRQ context. +#[must_use] pub struct IrqTrace { irq_number: arch::irq::IrqNumber, + _not_send: NotThreadSafe, } +// SAFETY: A shared reference exposes no operations on the CPU-local nesting +// count. Dropping the guard decrements that count and requires ownership. +unsafe impl Sync for IrqTrace {} + impl IrqTrace { pub fn new(irq_number: arch::irq::IrqNumber) -> Self { - let irq_trace = Self { irq_number }; + let irq_trace = Self { + irq_number, + _not_send: NotThreadSafe, + }; irq_trace.enter(); irq_trace } diff --git a/kernel/kernel/src/support.rs b/kernel/kernel/src/support.rs index 9786b39eb..f66b82a30 100644 --- a/kernel/kernel/src/support.rs +++ b/kernel/kernel/src/support.rs @@ -16,7 +16,7 @@ use crate::{ arch, sync::spinlock::{SpinLock, SpinLockGuard}, thread::ThreadNode, - types::{Arc, ArcList, AtomicUint, IntrusiveAdapter, Uint}, + types::{Arc, ArcList, AtomicUint, IntrusiveAdapter, NotThreadSafe, Uint}, }; pub use blueos_infra::storage::Storage; use core::{ @@ -26,43 +26,36 @@ use core::{ sync::atomic::{compiler_fence, AtomicUsize, Ordering}, }; +/// Owns one counted kernel IRQ mask on the current CPU. A shared reference +/// proves that the mask is held for the duration of the borrow. #[derive(Debug)] #[must_use] -pub struct DisableInterruptGuard { - old: usize, -} +pub struct DisableInterruptGuard(NotThreadSafe); impl DisableInterruptGuard { - #[inline] - pub fn new() -> Self { - Self { - old: arch::disable_local_irq_save(), - } + /// Borrows proof of an IRQ mask established by architecture code. + /// + /// # Safety + /// For all of `'a` the current CPU must stay in an admitted context with + /// kernel-capable IRQs masked and without switching execution context. + #[inline(always)] + pub(crate) unsafe fn assume_disabled<'a>() -> &'a Self { + // Borrow this zero-sized marker without creating a counted owner to drop. + const MASKED: &DisableInterruptGuard = &DisableInterruptGuard(NotThreadSafe); + MASKED } -} -impl Drop for DisableInterruptGuard { - #[inline] - fn drop(&mut self) { - arch::enable_local_irq_restore(self.old); - } -} - -#[must_use] -pub struct PlainDisableInterruptGuard; - -impl PlainDisableInterruptGuard { - #[inline] + #[inline(always)] pub fn new() -> Self { - arch::disable_local_irq(); - Self + crate::sync::local_context::enter_irq(); + Self(NotThreadSafe) } } -impl Drop for PlainDisableInterruptGuard { - #[inline] +impl Drop for DisableInterruptGuard { + #[inline(always)] fn drop(&mut self) { - arch::enable_local_irq(); + crate::sync::local_context::exit_irq(); } } @@ -243,8 +236,13 @@ pub fn nonnull_slice_end(ptr: NonNull<[T]>) -> *mut T { pub(crate) struct PerCpuVarAccessGuard { t: ThreadNode, id: u8, + _not_send: NotThreadSafe, } +// SAFETY: Shared access only reads the captured CPU id. Releasing preemption +// protection requires ownership, which stays in the acquiring context. +unsafe impl Sync for PerCpuVarAccessGuard {} + impl PerCpuVarAccessGuard { pub fn new() -> Self { let dig = DisableInterruptGuard::new(); @@ -257,7 +255,11 @@ impl PerCpuVarAccessGuard { t.disable_preempt(); compiler_fence(Ordering::SeqCst); drop(dig); - Self { t, id: id as u8 } + Self { + t, + id: id as u8, + _not_send: NotThreadSafe, + } } #[inline(always)] diff --git a/kernel/kernel/src/sync/local_context.rs b/kernel/kernel/src/sync/local_context.rs new file mode 100644 index 000000000..3ed1a9503 --- /dev/null +++ b/kernel/kernel/src/sync/local_context.rs @@ -0,0 +1,210 @@ +// Copyright (c) 2026 vivo Mobile Communication Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! CPU-local execution leases. All bookkeeping is protected by the kernel IRQ +//! mask. No allocation, thread reference counting, or atomic RMW is required. +use crate::{arch, types::NotThreadSafe}; +use core::{ + cell::{Cell, UnsafeCell}, + sync::atomic::{compiler_fence, Ordering}, +}; + +const CPUS: usize = blueos_kconfig::CONFIG_NUM_CORES as usize; +#[cfg(not(smp))] +const _: () = assert!(CPUS == 1, "the local lock backend requires exactly one CPU"); + +struct CpuState { + irq: Cell, + plain: Cell, + saved: Cell, + pending: Cell, +} + +impl CpuState { + const fn new() -> Self { + Self { + irq: Cell::new(0), + plain: Cell::new(0), + saved: Cell::new(0), + pending: Cell::new(false), + } + } +} + +struct States(UnsafeCell<[CpuState; CPUS]>); +// SAFETY: each CPU accesses only its slot, with all kernel-capable asynchronous +// interrupts masked. Scheduling cannot move a live lease to another CPU. +unsafe impl Sync for States {} +static STATES: States = States(UnsafeCell::new([const { CpuState::new() }; CPUS])); + +#[cold] +fn invalid_context() -> ! { + // Do not enter the ordinary logger or panic handler from an excluded IRQ. + #[cfg(target_board = "qemu_mps2_an385")] + semihosting::println!("LOCAL_LOCK_INVALID_CONTEXT"); + core::intrinsics::abort() +} + +#[inline(always)] +fn admit() { + #[cfg(cortex_m)] + { + let ipsr: u32; + unsafe { + core::arch::asm!("mrs {}, ipsr", out(reg) ipsr, options(nomem, nostack, preserves_flags)); + } + match ipsr & 0x1ff { + // SVC is synchronous. Borrow-state/lease updates never invoke SVC + // or callbacks. The actual switch gateway separately checks leases. + 0 | 11 | 14 => (), + 15 => { + if unsafe { + cortex_m::peripheral::SCB::get_priority( + cortex_m::peripheral::scb::SystemHandler::SysTick, + ) + } < arch::irq::IRQ_PRIORITY_FOR_SCHEDULER + { + invalid_context(); + } + } + n if n >= 16 => { + if arch::irq::get_irq_priority(arch::irq::IrqNumber::new((n - 16) as u16)) + < arch::irq::IRQ_PRIORITY_FOR_SCHEDULER + { + invalid_context(); + } + } + _ => invalid_context(), + } + } +} + +#[inline(always)] +unsafe fn state() -> &'static CpuState { + unsafe { &(*STATES.0.get())[arch::current_cpu_id()] } +} + +#[inline(always)] +pub(crate) fn enter_irq() { + admit(); + let old = arch::disable_local_irq_save(); + // SAFETY: mask established before selecting the CPU slot. + let s = unsafe { state() }; + let n = s.irq.get(); + if n == usize::MAX { + invalid_context(); + } + if n == 0 { + s.saved.set(old); + } + s.irq.set(n + 1); +} + +#[inline(always)] +pub(crate) fn exit_irq() { + let s = unsafe { state() }; + let n = s.irq.get(); + assert_ne!(n, 0); + s.irq.set(n - 1); + if n != 1 { + return; + } + let old = s.saved.get(); + let kick = s.plain.get() == 0 && s.pending.replace(false); + compiler_fence(Ordering::SeqCst); + unsafe { + arch::enable_local_irq_restore(old); + } + if kick { + crate::scheduler::yield_me_now_or_later(); + } +} + +#[derive(Debug)] +#[must_use] +pub(crate) struct PreemptLease { + _not_send: NotThreadSafe, +} +impl PreemptLease { + #[inline(always)] + pub(crate) fn new() -> Self { + let _irq = crate::support::DisableInterruptGuard::new(); + let s = unsafe { state() }; + s.plain.set( + s.plain + .get() + .checked_add(1) + .unwrap_or_else(|| invalid_context()), + ); + Self { + _not_send: NotThreadSafe, + } + } +} + +impl Drop for PreemptLease { + #[inline(always)] + fn drop(&mut self) { + let _irq = crate::support::DisableInterruptGuard::new(); + let s = unsafe { state() }; + s.plain.set( + s.plain + .get() + .checked_sub(1) + .expect("unbalanced preemption lease"), + ); + } +} + +// Called with the hardware kernel mask held, including in PendSV before any +// ordinary guard exists. A queued request is retried when the last lease exits. +#[inline(always)] +pub(crate) fn defer_if_held() -> bool { + admit(); + let old = arch::disable_local_irq_save(); + let s = unsafe { state() }; + let held = s.irq.get() != 0 || s.plain.get() != 0; + if held { + s.pending.set(true); + } + unsafe { + arch::enable_local_irq_restore(old); + } + held +} + +pub(crate) fn assert_can_suspend(irq_guard: bool) { + admit(); + let old = arch::disable_local_irq_save(); + let s = unsafe { state() }; + let resume_mask = if irq_guard { s.saved.get() } else { old }; + #[cfg(cortex_m)] + let resumes_enabled = resume_mask == 0; + #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] + let resumes_enabled = resume_mask & (1 << 3) != 0; + #[cfg(target_arch = "aarch64")] + let resumes_enabled = resume_mask & (1 << 7) == 0; + let ok = s.irq.get() == usize::from(irq_guard) && s.plain.get() == 0 && resumes_enabled; + unsafe { + arch::enable_local_irq_restore(old); + } + assert!(ok, "cannot suspend while other execution leases are held"); +} + +pub(crate) fn assert_can_switch() { + assert!( + !defer_if_held(), + "cannot switch while an execution lease is held" + ); +} diff --git a/kernel/kernel/src/sync/mod.rs b/kernel/kernel/src/sync/mod.rs index 35c9cc320..5d036633f 100644 --- a/kernel/kernel/src/sync/mod.rs +++ b/kernel/kernel/src/sync/mod.rs @@ -16,6 +16,7 @@ pub mod atomic_wait; pub use atomic_wait::{atomic_wait, atomic_wake}; pub mod delay; pub use delay::KernelDelay; +pub(crate) mod local_context; pub mod mqueue; pub mod mutex; pub mod posix_mqueue; From 5a3b35e93b4b7a45f67a6685877677ee2a6244ca Mon Sep 17 00:00:00 2001 From: Hegui Dai Date: Fri, 20 Nov 2026 09:49:36 +0800 Subject: [PATCH 04/10] refactor(sync): distinguish owned and borrowed SpinLock guards Return an owned IRQ guard from IRQ-save writes and tie borrowed acquisitions to the caller's IRQ mask. Keep the preemption lease in the ordinary write guard. Tighten shared lock and AArch64 handler bounds and cover conflicts, guard transfer and IRQ restoration in the lock tests. --- .../rust-resource-guard-must-use/SKILL.md | 2 +- kernel/kernel/src/arch/aarch64/irq.rs | 12 +- kernel/kernel/src/sync/mod.rs | 5 +- kernel/kernel/src/sync/spinlock.rs | 510 ++++++++++++------ kernel/kernel/src/sync/spinlock/up_tests.rs | 121 +++++ 5 files changed, 477 insertions(+), 173 deletions(-) create mode 100644 kernel/kernel/src/sync/spinlock/up_tests.rs diff --git a/.agents/skills/rust-resource-guard-must-use/SKILL.md b/.agents/skills/rust-resource-guard-must-use/SKILL.md index c5a5eaf5d..fb9385a97 100644 --- a/.agents/skills/rust-resource-guard-must-use/SKILL.md +++ b/.agents/skills/rust-resource-guard-must-use/SKILL.md @@ -25,7 +25,7 @@ The following types in `kernel/` already have type-level `#[must_use]` attribute | --- | --- | --- | | `DisableInterruptGuard` | [support.rs](../../../kernel/kernel/src/support.rs) | Drops one counted IRQ guard and restores the saved mask on the last release. | | `PreemptGuard` | [thread/mod.rs](../../../kernel/kernel/src/thread/mod.rs) | Calls `enable_preempt()` on its thread in `Drop`. | -| `SpinLockGuard`, `SpinLockReadGuard` | [sync/spinlock.rs](../../../kernel/kernel/src/sync/spinlock.rs) | Hold the underlying lock guard and an optional interrupt guard. Cleanup occurs through their fields when dropped. | +| `SpinLockGuard`, `OwnedIrqSpinLockGuard`, `SpinLockReadGuard` | [sync/spinlock.rs](../../../kernel/kernel/src/sync/spinlock.rs) | Write guards release the payload before their preemption or IRQ protection. Read guards carry the protection selected at acquisition. | | `RwLockReadGuard`, `RwLockWriteGuard`, `RwLockUpgradableGuard` | [tinyrwlock.rs](../../../kernel/infra/src/tinyrwlock.rs) | Update the reader count or clear lock state in `Drop`. | | `__BlueOSTestDoneGuard` | [test_harness/src/lib.rs](../../../kernel/test_harness/src/lib.rs) | Generated inside a `quote!` block with `#[must_use]`; increments the completion counter and wakes waiters in `Drop`. | diff --git a/kernel/kernel/src/arch/aarch64/irq.rs b/kernel/kernel/src/arch/aarch64/irq.rs index 63b941781..094a59720 100644 --- a/kernel/kernel/src/arch/aarch64/irq.rs +++ b/kernel/kernel/src/arch/aarch64/irq.rs @@ -98,7 +98,8 @@ pub const INTERRUPT_TABLE_LEN: usize = 1024; pub struct IrqContext { pub irq: IrqNumber, - pub handler: Box, + // The registry owns the handler, which another thread can replace or remove. + pub handler: Box, } impl core::fmt::Display for IrqContext { @@ -108,7 +109,7 @@ impl core::fmt::Display for IrqContext { } impl IrqContext { - fn new(irq: IrqNumber, handler: Box) -> Self { + fn new(irq: IrqNumber, handler: Box) -> Self { Self { irq, handler } } } @@ -130,7 +131,7 @@ impl IrqManager { fn register_handler( &mut self, irq: IrqNumber, - handler: Box, + handler: Box, ) -> Result<(), &'static str> { if u32::from(irq) >= INTERRUPT_TABLE_LEN as u32 { return Err("IRQ number out of range"); @@ -149,7 +150,10 @@ impl IrqManager { } // Register interrupt handler -pub fn register_handler(irq: IrqNumber, handler: Box) -> Result<(), &'static str> { +pub fn register_handler( + irq: IrqNumber, + handler: Box, +) -> Result<(), &'static str> { IRQ_MANAGER.lock().register_handler(irq, handler) } diff --git a/kernel/kernel/src/sync/mod.rs b/kernel/kernel/src/sync/mod.rs index 5d036633f..ad2b20dc5 100644 --- a/kernel/kernel/src/sync/mod.rs +++ b/kernel/kernel/src/sync/mod.rs @@ -25,7 +25,10 @@ pub mod spinlock; pub use mqueue::MessageQueue; pub use mutex::Mutex; pub use semaphore::Semaphore; -pub use spinlock::{ISpinLock, SpinLock, SpinLockGuard, SpinLockReadGuard, SpinLockWriteGuard}; +pub use spinlock::{ + ISpinLock, OwnedIrqSpinLockGuard, SpinLock, SpinLockGuard, SpinLockReadGuard, + SpinLockWriteGuard, +}; pub mod barrier; pub use barrier::ConstBarrier; #[cfg(event_flags)] diff --git a/kernel/kernel/src/sync/spinlock.rs b/kernel/kernel/src/sync/spinlock.rs index 23e7fde63..27b544883 100644 --- a/kernel/kernel/src/sync/spinlock.rs +++ b/kernel/kernel/src/sync/spinlock.rs @@ -12,220 +12,219 @@ // See the License for the specific language governing permissions and // limitations under the License. +use super::local_context::PreemptLease; use crate::{ support::DisableInterruptGuard, - types::{IRwLock, IntrusiveAdapter, NestedAdapter, RwLock, RwLockReadGuard, RwLockWriteGuard}, + types::{IntrusiveAdapter, NestedAdapter, NotThreadSafe}, }; -use blueos_infra::nolock::{INoLock, NoLock, NoLockReadGuard, NoLockWriteGuard}; use core::{ marker::PhantomData, + mem::ManuallyDrop, ops::{Deref, DerefMut}, - sync::atomic::{compiler_fence, Ordering}, }; +core::cfg_select! { + smp => { + use crate::types::{IRwLock, RwLock, RwLockReadGuard as ReadGuard, RwLockWriteGuard as WriteGuard}; + } + _ => { + use blueos_infra::local_rwlock::{LocalRwLock as RwLock, ReadGuard, WriteGuard}; + } +} + +/// Only SMP can make progress by spinning on a competing guard. +#[inline(always)] +fn acquire(mut attempt: impl FnMut() -> Option, _conflict: &'static str) -> G { + core::cfg_select! { + smp => { + loop { + if let Some(guard) = attempt() { + break guard; + } + core::hint::spin_loop(); + } + } + _ => attempt().expect(_conflict), + } +} + #[derive(Debug)] pub struct SpinLock { - #[cfg(smp)] lock: RwLock, - #[cfg(not(smp))] - lock: NoLock, } +// SAFETY: ownership of the protected value can transfer iff T can. +unsafe impl Send for SpinLock {} +// SAFETY: SMP uses atomic exclusion; UP state transitions are protected by +// kernel IRQ leases. Readers share &T and writers may replace T. +unsafe impl Sync for SpinLock {} pub type SpinLockWriteGuard<'a, T> = SpinLockGuard<'a, T>; - -// See https://doc.rust-lang.org/reference/destructors.html#r-destructors.operation for dropping orders. #[derive(Debug)] -#[repr(C)] #[must_use] pub struct SpinLockGuard<'a, T: ?Sized> { - #[cfg(smp)] - lock_guard: RwLockWriteGuard<'a, T>, - #[cfg(not(smp))] - lock_guard: NoLockWriteGuard<'a, T>, - irq_guard: Option, + lock_guard: ManuallyDrop>, + _preempt: PreemptLease, } - -impl SpinLockGuard<'_, T> { - #[inline] - pub fn take_irq_guard(&mut self, other: &mut SpinLockGuard<'_, S>) { - self.irq_guard = other.irq_guard.take(); - } +// SAFETY: shared access exposes only &T; mutation and release need ownership +// or &mut Guard. Preserve the existing conservative bound. +unsafe impl Sync for SpinLockGuard<'_, T> {} +#[derive(Debug)] +#[must_use] +pub struct SpinLockReadGuard<'a, T: ?Sized> { + lock_guard: ManuallyDrop>, + irq_guard: Option, + _preempt: Option, + _not_send: NotThreadSafe, } - -impl<'a, T: 'a + ?Sized> Deref for SpinLockGuard<'a, T> { +// SAFETY: sharing the guard exposes only &T. +unsafe impl Sync for SpinLockReadGuard<'_, T> {} +impl Deref for SpinLockGuard<'_, T> { type Target = T; - #[inline] + #[inline(always)] fn deref(&self) -> &T { - self.lock_guard.deref() + &self.lock_guard } } - -impl<'a, T: 'a + ?Sized> DerefMut for SpinLockGuard<'a, T> { - #[inline] +impl DerefMut for SpinLockGuard<'_, T> { + #[inline(always)] fn deref_mut(&mut self) -> &mut T { - self.lock_guard.deref_mut() + &mut self.lock_guard } } - -#[derive(Debug)] -#[repr(C)] -#[must_use] -pub struct SpinLockReadGuard<'a, T: ?Sized> { - #[cfg(smp)] - lock_guard: RwLockReadGuard<'a, T>, - #[cfg(not(smp))] - lock_guard: NoLockReadGuard<'a, T>, - irq_guard: Option, +impl Deref for SpinLockReadGuard<'_, T> { + type Target = T; + #[inline(always)] + fn deref(&self) -> &T { + &self.lock_guard + } } - -impl SpinLockReadGuard<'_, T> { - #[inline] - pub fn take_irq_guard(&mut self, other: &mut SpinLockReadGuard<'_, S>) { - self.irq_guard = other.irq_guard.take(); +impl Drop for SpinLockGuard<'_, T> { + #[inline(always)] + fn drop(&mut self) { + // On UP the borrow count must be updated with kernel IRQs masked. + #[cfg(not(smp))] + let _irq = DisableInterruptGuard::new(); + unsafe { + ManuallyDrop::drop(&mut self.lock_guard); + } } } - -impl<'a, T: 'a + ?Sized> Deref for SpinLockReadGuard<'a, T> { - type Target = T; - #[inline] - fn deref(&self) -> &T { - self.lock_guard.deref() +impl Drop for SpinLockReadGuard<'_, T> { + #[inline(always)] + fn drop(&mut self) { + #[cfg(not(smp))] + let _irq = self.irq_guard.is_none().then(DisableInterruptGuard::new); + unsafe { + ManuallyDrop::drop(&mut self.lock_guard); + } } } - impl SpinLock { - pub const fn new(val: T) -> Self { + pub const fn new(data: T) -> Self { Self { - #[cfg(smp)] - lock: RwLock::new(val), - #[cfg(not(smp))] - lock: NoLock::new(val), + lock: RwLock::new(data), } } } - impl SpinLock { - pub fn try_irqsave_lock(&self) -> Option> { + #[inline(always)] + pub fn try_irqsave_lock(&self) -> Option> { let irq_guard = DisableInterruptGuard::new(); - compiler_fence(Ordering::SeqCst); - let mut guard = self.try_lock()?; - debug_assert!(guard.irq_guard.is_none()); - guard.irq_guard = Some(irq_guard); - Some(guard) - } - - pub fn irqsave_lock(&self) -> SpinLockGuard<'_, T> { - let irq_guard = DisableInterruptGuard::new(); - loop { - let Some(mut l) = self.try_lock() else { - core::hint::spin_loop(); - continue; - }; - debug_assert!(l.irq_guard.is_none()); - l.irq_guard = Some(irq_guard); - return l; - } + // SAFETY: the IRQ mask protects the UP borrow count through guard release. + let lock_guard = unsafe { self.lock.try_write()? }; + Some(OwnedIrqSpinLockGuard { + lock_guard, + irq_guard, + }) } - + #[inline(always)] pub fn try_lock(&self) -> Option> { - let lock_guard = self.lock.try_write()?; + let preempt = PreemptLease::new(); + #[cfg(not(smp))] + let _irq = DisableInterruptGuard::new(); + let g = unsafe { self.lock.try_write()? }; Some(SpinLockGuard { + lock_guard: ManuallyDrop::new(g), + _preempt: preempt, + }) + } + #[inline(always)] + pub fn try_irqsave_read(&self) -> Option> { + let irq_guard = DisableInterruptGuard::new(); + let g = unsafe { self.lock.try_read()? }; + Some(SpinLockReadGuard { + lock_guard: ManuallyDrop::new(g), + irq_guard: Some(irq_guard), + _preempt: None, + _not_send: NotThreadSafe, + }) + } + #[inline(always)] + pub fn try_read(&self) -> Option> { + let preempt = PreemptLease::new(); + #[cfg(not(smp))] + let _irq = DisableInterruptGuard::new(); + let g = unsafe { self.lock.try_read()? }; + Some(SpinLockReadGuard { + lock_guard: ManuallyDrop::new(g), irq_guard: None, - lock_guard, + _preempt: Some(preempt), + _not_send: NotThreadSafe, }) } - + #[inline(always)] pub fn lock(&self) -> SpinLockGuard<'_, T> { - loop { - let Some(l) = self.try_lock() else { - core::hint::spin_loop(); - continue; - }; - return l; - } + acquire(|| self.try_lock(), "conflicting UP lock borrow") } - + /// Takes a write guard that owns the local IRQ mask through unlock. #[inline] + pub fn irqsave_lock(&self) -> OwnedIrqSpinLockGuard<'_, T> { + core::cfg_select! { + smp => acquire(|| self.try_irqsave_lock(), "conflicting UP IRQ lock borrow"), + _ => { + let irq_guard = DisableInterruptGuard::new(); + // SAFETY: the IRQ mask protects the UP borrow count through guard release. + let lock_guard = unsafe { self.lock.try_write() }.expect("conflicting UP IRQ lock borrow"); + OwnedIrqSpinLockGuard { lock_guard, irq_guard } + }, + } + } + #[inline(always)] + pub fn read(&self) -> SpinLockReadGuard<'_, T> { + acquire(|| self.try_read(), "conflicting UP read borrow") + } + #[inline(always)] + pub fn irqsave_read(&self) -> SpinLockReadGuard<'_, T> { + acquire(|| self.try_irqsave_read(), "conflicting UP IRQ read borrow") + } + #[inline(always)] pub fn write(&self) -> SpinLockWriteGuard<'_, T> { self.lock() } - - #[inline] + #[inline(always)] pub fn try_write(&self) -> Option> { self.try_lock() } - - pub fn try_irqsave_write(&self) -> Option> { - self.try_irqsave_lock() - } - - pub fn irqsave_write(&self) -> SpinLockWriteGuard<'_, T> { + #[inline(always)] + pub fn irqsave_write(&self) -> OwnedIrqSpinLockGuard<'_, T> { self.irqsave_lock() } - - #[inline] - pub fn try_read(&self) -> Option> { - let lock_guard = self.lock.try_read()?; - Some(SpinLockReadGuard { - irq_guard: None, - lock_guard, - }) - } - - #[inline] - pub fn read(&self) -> SpinLockReadGuard<'_, T> { - loop { - let Some(l) = self.try_read() else { - core::hint::spin_loop(); - continue; - }; - return l; - } - } - - pub fn try_irqsave_read(&self) -> Option> { - let irq_guard = DisableInterruptGuard::new(); - compiler_fence(Ordering::SeqCst); - let mut guard = self.try_read()?; - debug_assert!(guard.irq_guard.is_none()); - guard.irq_guard = Some(irq_guard); - Some(guard) - } - - pub fn irqsave_read(&self) -> SpinLockReadGuard<'_, T> { - loop { - let Some(l) = self.try_irqsave_read() else { - core::hint::spin_loop(); - continue; - }; - return l; - } + #[inline(always)] + pub fn try_irqsave_write(&self) -> Option> { + self.try_irqsave_lock() } - pub fn reader_count(&self) -> usize { - self.lock.reader_count() as usize + let _irq = DisableInterruptGuard::new(); + unsafe { self.lock.reader_count() as usize } } - pub fn writer_count(&self) -> usize { - self.lock.writer_count() as usize - } -} - -impl<'a, T: 'a + ?Sized> SpinLockGuard<'a, T> { - pub fn forget_irq(&mut self) { - if self.irq_guard.is_none() { - return; - } - core::mem::forget(self.irq_guard.take()) + let _irq = DisableInterruptGuard::new(); + unsafe { self.lock.writer_count() as usize } } } -unsafe impl Send for SpinLock {} -unsafe impl Sync for SpinLock {} - #[derive(Default, Debug)] struct ISpinLockOffset>(PhantomData, PhantomData); - impl> const IntrusiveAdapter> for ISpinLockOffset { @@ -234,35 +233,212 @@ impl> const IntrusiveAdapter> { + #[allow(clippy::type_complexity)] #[cfg(smp)] - lock: IRwLock, ISpinLockOffset>>, + lock: IRwLock>>, #[cfg(not(smp))] - lock: INoLock, ISpinLockOffset>>, + lock: RwLock<()>, + _adapter: PhantomData<(T, A)>, } +impl> Default for ISpinLock { + fn default() -> Self { + Self::new() + } +} impl> ISpinLock { pub const fn new() -> Self { Self { - #[cfg(smp)] - lock: IRwLock::new(), - #[cfg(not(smp))] - lock: INoLock::new(), + lock: core::cfg_select! { + smp => IRwLock::new(), + _ => RwLock::new(()), + }, + _adapter: PhantomData, } } - - #[inline] - pub fn irqsave_lock(&self) -> SpinLockGuard<'_, T> { + #[inline(always)] + pub fn irqsave_lock(&self) -> OwnedIrqSpinLockGuard<'_, T> { let irq_guard = DisableInterruptGuard::new(); - compiler_fence(Ordering::SeqCst); - SpinLockGuard { - lock_guard: self.lock.write(), - irq_guard: Some(irq_guard), + let g = core::cfg_select! { + smp => self.lock.write(), + _ => unsafe { + let data = (&raw const *self) + .cast::() + .cast_mut() + .sub(A::offset()) + .cast::(); + self.lock + .project_write(data) + .expect("conflicting UP intrusive lock borrow") + }, + }; + OwnedIrqSpinLockGuard { + lock_guard: g, + irq_guard, } } } - unsafe impl> Send for ISpinLock {} -unsafe impl> Sync for ISpinLock {} +// A write guard can move the projected T out by replacement, requiring T: Send. +// Keep the existing conservative T: Sync bound for this intrusive lock. +unsafe impl> Sync for ISpinLock {} + +/// Payload guard borrowing an existing IRQ proof. Dropping it only unlocks the +/// payload; the proof's owner determines when interrupts can be restored. +#[derive(Debug)] +#[must_use] +pub(crate) struct BorrowedSpinLockGuard<'a, T: ?Sized> { + lock_guard: WriteGuard<'a, T>, + _context: PhantomData<&'a DisableInterruptGuard>, +} +impl Deref for BorrowedSpinLockGuard<'_, T> { + type Target = T; + #[inline(always)] + fn deref(&self) -> &T { + &self.lock_guard + } +} +impl DerefMut for BorrowedSpinLockGuard<'_, T> { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + &mut self.lock_guard + } +} +impl SpinLock { + #[inline(always)] + pub(crate) fn try_lock_with<'a>( + &'a self, + _: &'a DisableInterruptGuard, + ) -> Option> { + // SAFETY: the proof masks UP transitions including the guard's Drop. + let lock_guard = unsafe { self.lock.try_write()? }; + Some(BorrowedSpinLockGuard { + lock_guard, + _context: PhantomData, + }) + } + #[inline(always)] + pub(crate) fn lock_with<'a>( + &'a self, + context: &'a DisableInterruptGuard, + ) -> BorrowedSpinLockGuard<'a, T> { + acquire( + || self.try_lock_with(context), + "conflicting UP borrowed lock", + ) + } +} +impl> ISpinLock { + #[inline(always)] + pub(crate) fn lock_with<'a>( + &'a self, + _: &'a DisableInterruptGuard, + ) -> BorrowedSpinLockGuard<'a, T> { + let lock_guard = core::cfg_select! { + smp => self.lock.write(), + _ => unsafe { + // Existing intrusive projection remains a separate migration task. + let data = (&raw const *self) + .cast::() + .cast_mut() + .sub(A::offset()) + .cast::(); + self.lock + .project_write(data) + .expect("conflicting borrowed intrusive lock") + }, + }; + BorrowedSpinLockGuard { + lock_guard, + _context: PhantomData, + } + } +} + +#[cfg(test)] +mod context_tests { + use super::*; + use blueos_test_macro::test; + + #[test] + fn borrowed_irq_guard_excludes_and_preserves_outer_mask() { + let initially_enabled = crate::arch::local_irq_enabled(); + let a = SpinLock::new(1usize); + let b = SpinLock::new(2usize); + let irq = DisableInterruptGuard::new(); + let context = &irq; + let mut ga = a.lock_with(context); + let gb = b.lock_with(context); + assert!(a.try_lock_with(context).is_none()); + assert!(a.try_irqsave_read().is_none()); + *ga = *gb + 1; + drop(ga); + assert!(!crate::arch::local_irq_enabled()); + assert_eq!(*a.lock_with(context), 3); + drop(gb); + drop(irq); + assert_eq!(crate::arch::local_irq_enabled(), initially_enabled); + } + + #[test] + fn owned_irq_guard_keeps_lock_and_mask_until_drop() { + let initially_enabled = crate::arch::local_irq_enabled(); + let a = SpinLock::new(0usize); + let b = SpinLock::new(7usize); + let waiting = a.irqsave_lock(); + let second = b.irqsave_lock(); + assert!(!crate::arch::local_irq_enabled()); + assert!(a.try_irqsave_lock().is_none()); + drop(second); + assert!(!crate::arch::local_irq_enabled()); + assert_eq!(*b.lock_with(waiting.irq_guard()), 7); + drop(waiting); + assert_eq!(crate::arch::local_irq_enabled(), initially_enabled); + assert!(a.try_irqsave_lock().is_some()); + } + + #[test] + fn forgotten_borrowed_guard_keeps_payload_inaccessible() { + let initially_enabled = crate::arch::local_irq_enabled(); + let a = SpinLock::new(1usize); + let irq = DisableInterruptGuard::new(); + core::mem::forget(a.lock_with(&irq)); + drop(irq); + assert_eq!(crate::arch::local_irq_enabled(), initially_enabled); + assert!(a.try_irqsave_lock().is_none()); + assert!(a.try_irqsave_read().is_none()); + } +} + +/// Holds an exclusive payload borrow and owns the IRQ mask needed for a wait handoff. +#[derive(Debug)] +#[must_use] +pub struct OwnedIrqSpinLockGuard<'a, T: ?Sized> { + // Field drop order releases the payload before restoring IRQs can run the scheduler. + lock_guard: WriteGuard<'a, T>, + irq_guard: DisableInterruptGuard, +} +impl Deref for OwnedIrqSpinLockGuard<'_, T> { + type Target = T; + #[inline(always)] + fn deref(&self) -> &T { + &self.lock_guard + } +} +impl DerefMut for OwnedIrqSpinLockGuard<'_, T> { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + &mut self.lock_guard + } +} +impl OwnedIrqSpinLockGuard<'_, T> { + #[inline(always)] + pub(crate) fn irq_guard(&self) -> &DisableInterruptGuard { + &self.irq_guard + } +} + +#[cfg(all(test, not(smp)))] +mod up_tests; diff --git a/kernel/kernel/src/sync/spinlock/up_tests.rs b/kernel/kernel/src/sync/spinlock/up_tests.rs new file mode 100644 index 000000000..ca7c91d74 --- /dev/null +++ b/kernel/kernel/src/sync/spinlock/up_tests.rs @@ -0,0 +1,121 @@ +// Copyright (c) 2026 vivo Mobile Communication Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::SpinLock; +use crate::arch; +use blueos_test_macro::test; + +#[test] +fn test_up_spinlock_shared_reads_and_reacquisition() { + let lock = SpinLock::new(7); + let first = lock.try_read().unwrap(); + let second = lock.try_read().unwrap(); + assert_eq!((*first, *second), (7, 7)); + drop(first); + drop(second); + + let mut writer = lock.try_lock().unwrap(); + *writer = 9; + drop(writer); + assert_eq!(*lock.try_read().unwrap(), 9); +} + +#[test] +fn test_up_spinlock_writer_excludes_other_access() { + let lock = SpinLock::new(0); + let writer = lock.try_lock().unwrap(); + // Only inspect acquisition results. Never dereference a conflicting guard, + // so a broken backend is reported by assertions without exercising UB. + let rejects_writer = lock.try_lock().is_none(); + let rejects_reader = lock.try_read().is_none(); + drop(writer); + + assert!(rejects_writer, "UP SpinLock admitted a second writer"); + assert!( + rejects_reader, + "UP SpinLock admitted a reader during a write" + ); + assert!(lock.try_lock().is_some()); +} + +#[test] +fn test_up_spinlock_readers_exclude_writer_until_last_drop() { + let lock = SpinLock::new(0); + let first = lock.try_read().unwrap(); + let second = lock.try_read().unwrap(); + let rejects_with_two = lock.try_lock().is_none(); + drop(first); + let rejects_with_one = lock.try_lock().is_none(); + drop(second); + + assert!( + rejects_with_two, + "UP SpinLock admitted a writer during reads" + ); + assert!( + rejects_with_one, + "UP SpinLock released access before its last reader" + ); + assert!(lock.try_lock().is_some()); +} + +#[test] +fn test_up_spinlock_irqsave_writer_excludes_other_access() { + let lock = SpinLock::new(0); + let initially_enabled = arch::local_irq_enabled(); + let writer = lock.try_irqsave_lock().unwrap(); + let masked = !arch::local_irq_enabled(); + let rejects_writer = lock.try_irqsave_lock().is_none(); + let rejects_reader = lock.try_irqsave_read().is_none(); + let still_masked = !arch::local_irq_enabled(); + // Restore IRQ state before reporting a failure. + drop(writer); + let restored = arch::local_irq_enabled() == initially_enabled; + + assert!( + masked && still_masked && restored, + "IRQ state was not preserved" + ); + assert!( + rejects_writer, + "UP IRQ-save SpinLock admitted a second writer" + ); + assert!( + rejects_reader, + "UP IRQ-save SpinLock admitted a reader during a write" + ); + assert!(lock.try_irqsave_lock().is_some()); +} + +#[test] +fn test_up_spinlock_irqsave_reader_excludes_writer() { + let lock = SpinLock::new(0); + let initially_enabled = arch::local_irq_enabled(); + let reader = lock.try_irqsave_read().unwrap(); + let masked = !arch::local_irq_enabled(); + let rejects_writer = lock.try_irqsave_lock().is_none(); + let still_masked = !arch::local_irq_enabled(); + drop(reader); + let restored = arch::local_irq_enabled() == initially_enabled; + + assert!( + masked && still_masked && restored, + "IRQ state was not preserved" + ); + assert!( + rejects_writer, + "UP IRQ-save SpinLock admitted a writer during a read" + ); + assert!(lock.try_irqsave_lock().is_some()); +} From e659e06aa44b4a136a09f7cc945cba2c3cc4fc20 Mon Sep 17 00:00:00 2001 From: Hegui Dai Date: Fri, 20 Nov 2026 09:49:36 +0800 Subject: [PATCH 05/10] fix(arch): enforce guard contracts at IRQ and context-switch entry points Check switching and interrupt restoration at architecture boundaries. Keep the RISC-V syscall guard on its acquiring thread and make the raw IRQ enable and restore calls explicit. --- kernel/kernel/src/arch/aarch64/exception.rs | 5 ++++- kernel/kernel/src/arch/aarch64/mod.rs | 8 ++++++-- kernel/kernel/src/arch/arm/mod.rs | 14 +++++++++----- kernel/kernel/src/arch/riscv/mod.rs | 17 +++++++++++++---- kernel/kernel/src/arch/riscv/trap.rs | 12 ++++++++++-- kernel/kernel/src/drivers/flash/esp32_rom.rs | 4 +++- kernel/kernel/src/ffi.rs | 4 ++-- kernel/kernel/src/net/link/esp32_wlan/api.rs | 4 +++- 8 files changed, 50 insertions(+), 18 deletions(-) diff --git a/kernel/kernel/src/arch/aarch64/exception.rs b/kernel/kernel/src/arch/aarch64/exception.rs index d7e6fa56b..f015db8e6 100644 --- a/kernel/kernel/src/arch/aarch64/exception.rs +++ b/kernel/kernel/src/arch/aarch64/exception.rs @@ -168,7 +168,10 @@ extern "C" fn handle_svc(context: &mut Context) -> usize { context.x0, context.x1, context.x2, context.x3, context.x4, context.x5, ], }; - enable_local_irq(); + crate::sync::local_context::assert_can_switch(); + unsafe { + enable_local_irq(); + } context.x0 = dispatch_syscall(&sc); disable_local_irq(); compiler_fence(Ordering::SeqCst); diff --git a/kernel/kernel/src/arch/aarch64/mod.rs b/kernel/kernel/src/arch/aarch64/mod.rs index da77883f3..31c423c77 100644 --- a/kernel/kernel/src/arch/aarch64/mod.rs +++ b/kernel/kernel/src/arch/aarch64/mod.rs @@ -463,6 +463,7 @@ pub(crate) extern "C" fn svc_switch_context_with_hook(hook: *mut ContextSwitchHo #[inline] pub(crate) extern "C" fn switch_context_with_hook(hook: *mut ContextSwitchHookHolder) { + crate::sync::local_context::assert_can_switch(); svc_switch_context_with_hook(hook) } #[unsafe(naked)] @@ -524,7 +525,10 @@ pub extern "C" fn disable_local_irq() { } #[inline] -pub extern "C" fn enable_local_irq() { +/// # Safety +/// No live execution lease may require IRQs to remain masked. Only architecture +/// entry/exit code may bypass the nesting-aware IRQ guard. +pub unsafe extern "C" fn enable_local_irq() { unsafe { core::arch::asm!("msr daifclr, #3", options(nostack, nomem)) } } @@ -552,7 +556,7 @@ pub extern "C" fn disable_local_irq_save() -> usize { } #[inline] -pub extern "C" fn enable_local_irq_restore(old: usize) { +pub unsafe extern "C" fn enable_local_irq_restore(old: usize) { atomic::compiler_fence(Ordering::SeqCst); DAIF.set(old as u64); } diff --git a/kernel/kernel/src/arch/arm/mod.rs b/kernel/kernel/src/arch/arm/mod.rs index 80ea637c4..5d4989de6 100644 --- a/kernel/kernel/src/arch/arm/mod.rs +++ b/kernel/kernel/src/arch/arm/mod.rs @@ -568,7 +568,10 @@ impl Context { } #[inline] -pub extern "C" fn enable_local_irq() { +/// # Safety +/// No live execution lease may require IRQs to remain masked. Only architecture +/// entry/exit code may bypass the nesting-aware IRQ guard. +pub unsafe extern "C" fn enable_local_irq() { unsafe { core::arch::asm!( "msr basepri, {}", @@ -578,11 +581,11 @@ pub extern "C" fn enable_local_irq() { } } -#[inline] +#[inline(always)] pub extern "C" fn disable_local_irq() { unsafe { core::arch::asm!( - "msr basepri, {}", + "msr basepri_max, {}", "isb", in(reg) DISABLE_LOCAL_IRQ_BASEPRI, options(nostack), @@ -599,7 +602,7 @@ pub extern "C" fn disable_local_irq_save() -> usize { concat!( " mrs {old}, basepri - msr basepri, {val} + msr basepri_max, {val} isb ", ), @@ -614,7 +617,7 @@ pub extern "C" fn disable_local_irq_save() -> usize { #[coverage(off)] #[cfg_attr(debug, inline(never))] -pub extern "C" fn enable_local_irq_restore(old: usize) { +pub unsafe extern "C" fn enable_local_irq_restore(old: usize) { atomic::compiler_fence(Ordering::SeqCst); unsafe { core::arch::asm!( @@ -652,6 +655,7 @@ pub extern "C" fn current_psp() -> usize { #[inline(never)] pub(crate) extern "C" fn switch_context_with_hook(hook: *mut ContextSwitchHookHolder) { + crate::sync::local_context::assert_can_switch(); unsafe { core::arch::asm!( "movs {tmp}, r7", diff --git a/kernel/kernel/src/arch/riscv/mod.rs b/kernel/kernel/src/arch/riscv/mod.rs index b32573232..8af435504 100644 --- a/kernel/kernel/src/arch/riscv/mod.rs +++ b/kernel/kernel/src/arch/riscv/mod.rs @@ -53,7 +53,9 @@ pub(crate) extern "C" fn pend_switch_context() { let level = disable_local_irq_save(); let id = current_cpu_id(); unsafe { PENDING_SWITCH_CONTEXT[id].set(true) }; - enable_local_irq_restore(level); + unsafe { + enable_local_irq_restore(level); + } } #[inline] @@ -62,7 +64,9 @@ pub(crate) extern "C" fn claim_switch_context() -> bool { let id = current_cpu_id(); let ok = unsafe { PENDING_SWITCH_CONTEXT[id].get() }; unsafe { PENDING_SWITCH_CONTEXT[id].set(false) }; - enable_local_irq_restore(level); + unsafe { + enable_local_irq_restore(level); + } ok } @@ -339,7 +343,10 @@ pub(crate) extern "C" fn disable_local_irq() { } #[inline] -pub(crate) extern "C" fn enable_local_irq() { +/// # Safety +/// No live execution lease may require IRQs to remain masked. Only architecture +/// entry/exit code may bypass the nesting-aware IRQ guard. +pub(crate) unsafe extern "C" fn enable_local_irq() { unsafe { core::arch::asm!(set_mstatus_mie!(), options(nostack)) }; compiler_fence(Ordering::SeqCst); } @@ -364,7 +371,7 @@ pub(crate) extern "C" fn disable_local_irq_save() -> usize { } #[inline] -pub(crate) extern "C" fn enable_local_irq_restore(old: usize) { +pub(crate) unsafe extern "C" fn enable_local_irq_restore(old: usize) { unsafe { core::arch::asm!("csrw mstatus, {old}", old = in(reg) old, options(nostack)) @@ -380,6 +387,7 @@ pub extern "C" fn current_sp() -> usize { } pub(crate) extern "C" fn ecall_switch_context_with_hook(hook: *mut ContextSwitchHookHolder) { + crate::sync::local_context::assert_can_switch(); unsafe { core::arch::asm!( "ecall", @@ -392,6 +400,7 @@ pub(crate) extern "C" fn ecall_switch_context_with_hook(hook: *mut ContextSwitch #[inline(always)] pub(crate) extern "C" fn switch_context_with_hook(hook: *mut ContextSwitchHookHolder) { + crate::sync::local_context::assert_can_switch(); ecall_switch_context_with_hook(hook) } diff --git a/kernel/kernel/src/arch/riscv/trap.rs b/kernel/kernel/src/arch/riscv/trap.rs index 6c1f4d497..9922afe76 100644 --- a/kernel/kernel/src/arch/riscv/trap.rs +++ b/kernel/kernel/src/arch/riscv/trap.rs @@ -27,7 +27,7 @@ use crate::{ syscalls::{dispatch_syscall, Context as ScContext}, thread, thread::Thread, - types::Arc, + types::{Arc, NotThreadSafe}, }; use core::{ mem::offset_of, @@ -123,10 +123,16 @@ pub(crate) unsafe extern "C" fn trap_entry() { #[must_use] struct SyscallGuard { isr_ctx: IsrContext, + _not_send: NotThreadSafe, } +// SAFETY: Shared references only expose saved register values; restoring local +// IRQ and trap state happens in Drop, in the acquiring execution context. +unsafe impl Sync for SyscallGuard {} + impl SyscallGuard { pub fn new() -> Self { + crate::sync::local_context::assert_can_switch(); let mut g = Self::default(); unsafe { core::arch::asm!( @@ -143,7 +149,9 @@ impl SyscallGuard { } compiler_fence(Ordering::SeqCst); leave_irq(); - enable_local_irq(); + unsafe { + enable_local_irq(); + } g } } diff --git a/kernel/kernel/src/drivers/flash/esp32_rom.rs b/kernel/kernel/src/drivers/flash/esp32_rom.rs index d9e86ce5a..9510cb3e9 100644 --- a/kernel/kernel/src/drivers/flash/esp32_rom.rs +++ b/kernel/kernel/src/drivers/flash/esp32_rom.rs @@ -72,7 +72,9 @@ pub(crate) fn with_flash_op(body: impl FnOnce() -> R) -> R { let cache_state = unsafe { Cache_Suspend_ICache() }; let result = body(); unsafe { Cache_Resume_ICache(cache_state) }; - enable_local_irq_restore(flags); + unsafe { + enable_local_irq_restore(flags); + } result } #[unsafe(link_section = ".rwtext")] diff --git a/kernel/kernel/src/ffi.rs b/kernel/kernel/src/ffi.rs index 5a40f97d3..cc135bb44 100644 --- a/kernel/kernel/src/ffi.rs +++ b/kernel/kernel/src/ffi.rs @@ -23,8 +23,8 @@ pub extern "C" fn disable_local_irq_save() -> usize { #[coverage(off)] #[unsafe(no_mangle)] -pub extern "C" fn enable_local_irq_restore(val: usize) { - crate::arch::enable_local_irq_restore(val) +pub unsafe extern "C" fn enable_local_irq_restore(val: usize) { + unsafe { crate::arch::enable_local_irq_restore(val) } } #[coverage(off)] diff --git a/kernel/kernel/src/net/link/esp32_wlan/api.rs b/kernel/kernel/src/net/link/esp32_wlan/api.rs index 1fa948b8a..e4ff58d5f 100644 --- a/kernel/kernel/src/net/link/esp32_wlan/api.rs +++ b/kernel/kernel/src/net/link/esp32_wlan/api.rs @@ -348,7 +348,9 @@ pub unsafe extern "C" fn wifi_int_disable(_wifi_int_mux: *mut c_void) -> u32 { } pub unsafe extern "C" fn wifi_int_restore(_wifi_int_mux: *mut c_void, tmp: u32) { - crate::arch::enable_local_irq_restore(tmp as usize); + unsafe { + crate::arch::enable_local_irq_restore(tmp as usize); + } } pub unsafe extern "C" fn task_yield_from_isr() { From 80553e54d76d5393e7ee0fb288dc310a1baaeb18 Mon Sep 17 00:00:00 2001 From: Hegui Dai Date: Fri, 20 Nov 2026 09:49:36 +0800 Subject: [PATCH 06/10] fix(scheduler): honor live guards and accept owned IRQ queue guards Pass the IRQ proof through thread selection and timer operations. Honor live guards at context switches and require an owned IRQ guard for suspension with a queue lock. Release the payload borrow before restoring the saved mask. --- .../kernel/src/scheduler/global_scheduler.rs | 46 ++++++--- kernel/kernel/src/scheduler/mod.rs | 96 ++++++++++++++----- kernel/kernel/src/thread/builder.rs | 4 +- kernel/kernel/src/thread/mod.rs | 13 ++- kernel/kernel/src/time.rs | 7 +- kernel/kernel/src/time/timer.rs | 45 +++++++-- 6 files changed, 158 insertions(+), 53 deletions(-) diff --git a/kernel/kernel/src/scheduler/global_scheduler.rs b/kernel/kernel/src/scheduler/global_scheduler.rs index 49636538d..608ed4569 100644 --- a/kernel/kernel/src/scheduler/global_scheduler.rs +++ b/kernel/kernel/src/scheduler/global_scheduler.rs @@ -15,7 +15,8 @@ use crate::{ arch, config::MAX_THREAD_PRIORITY, - sync::spinlock::{SpinLock, SpinLockGuard}, + support::DisableInterruptGuard, + sync::spinlock::SpinLock, thread, thread::{Thread, ThreadNode}, types::{ArcList, ThreadPriority, Uint}, @@ -69,7 +70,7 @@ impl ReadyTable { } #[inline] -fn inner_next_thread(mut tbl: SpinLockGuard<'_, ReadyTable>, index: usize) -> Option { +fn inner_next_thread(tbl: &mut ReadyTable, index: usize) -> Option { let q = &mut tbl.tables[index]; let next = q.pop_front()?; debug_assert_eq!(next.state(), thread::READY); @@ -90,7 +91,16 @@ pub enum PreferMode { #[inline(always)] pub fn next_preferred_thread(prio: ThreadPriority, mode: PreferMode) -> Option { - let mut tbl = unsafe { READY_TABLE.assume_init_ref().irqsave_lock() }; + let irq = DisableInterruptGuard::new(); + next_preferred_thread_with(prio, mode, &irq) +} +#[inline(always)] +pub(crate) fn next_preferred_thread_with( + prio: ThreadPriority, + mode: PreferMode, + irq: &DisableInterruptGuard, +) -> Option { + let mut tbl = unsafe { READY_TABLE.assume_init_ref().lock_with(irq) }; let highest_active = tbl.highest_active(); // highest_active is trailing_zeros(): 0 = highest priority. When a // same-priority ready thread exists, highest_active == prio. @@ -106,11 +116,16 @@ pub fn next_preferred_thread(prio: ThreadPriority, mode: PreferMode) -> Option Option { - let mut tbl = unsafe { READY_TABLE.assume_init_ref().irqsave_lock() }; + let irq = DisableInterruptGuard::new(); + next_ready_thread_with(&irq) +} +#[inline(always)] +pub(crate) fn next_ready_thread_with(irq: &DisableInterruptGuard) -> Option { + let mut tbl = unsafe { READY_TABLE.assume_init_ref().lock_with(irq) }; let highest_active = tbl.highest_active(); #[cfg(debugging_scheduler)] @@ -121,7 +136,7 @@ pub fn next_ready_thread() -> Option { if highest_active > MAX_THREAD_PRIORITY as u32 { return None; } - inner_next_thread(tbl, highest_active as usize) + inner_next_thread(&mut tbl, highest_active as usize) } pub fn queue_ready_thread_with_post_action( @@ -149,7 +164,7 @@ where } #[inline] -fn queue_ready_thread_inner(tbl: &mut SpinLockGuard<'_, ReadyTable>, t: &ThreadNode) -> bool { +fn queue_ready_thread_inner(tbl: &mut ReadyTable, t: &ThreadNode) -> bool { let priority = t.priority(); debug_assert!(priority <= MAX_THREAD_PRIORITY); let q = &mut tbl.tables[priority as usize]; @@ -171,12 +186,21 @@ fn queue_ready_thread_inner(tbl: &mut SpinLockGuard<'_, ReadyTable>, t: &ThreadN // We only queue the thread if old_state equals thread's current state. pub fn queue_ready_thread(old_state: Uint, t: ThreadNode) -> Result<(), Uint> { + let irq = DisableInterruptGuard::new(); + queue_ready_thread_with(old_state, &t, &irq) +} +#[inline(always)] +pub(crate) fn queue_ready_thread_with( + old_state: Uint, + t: &ThreadNode, + irq: &DisableInterruptGuard, +) -> Result<(), Uint> { if old_state == thread::READY { return Err(thread::READY); } - let mut tbl = unsafe { READY_TABLE.assume_init_ref().irqsave_lock() }; + let mut tbl = unsafe { READY_TABLE.assume_init_ref().lock_with(irq) }; t.transfer_state(old_state, thread::READY)?; - let ok = queue_ready_thread_inner(&mut tbl, &t); + let ok = queue_ready_thread_inner(&mut tbl, t); if !ok { // push_back failed: thread is in READY state with sched_node still // detached. Roll back to old_state so the thread stays in its wait @@ -188,7 +212,7 @@ pub fn queue_ready_thread(old_state: Uint, t: ThreadNode) -> Result<(), Uint> { debug_assert!( rollback.is_ok(), "queue_ready_thread rollback failed for TH:0x{:x}: state was {:?}, expected READY; alien_ptr={:?}, priority={}, thread is now ghost", - Thread::id(&t), + Thread::id(t), rollback, t.get_alien_ptr(), t.priority(), @@ -201,7 +225,7 @@ pub fn queue_ready_thread(old_state: Uint, t: ThreadNode) -> Result<(), Uint> { Ok(()) } -fn remove_from_ready_queue_inner(tbl: &mut SpinLockGuard<'_, ReadyTable>, t: &ThreadNode) -> bool { +fn remove_from_ready_queue_inner(tbl: &mut ReadyTable, t: &ThreadNode) -> bool { let priority = t.priority(); debug_assert!(priority <= MAX_THREAD_PRIORITY); let q = &mut tbl.tables[priority as usize]; diff --git a/kernel/kernel/src/scheduler/mod.rs b/kernel/kernel/src/scheduler/mod.rs index 4a1f7284b..32dec3db9 100644 --- a/kernel/kernel/src/scheduler/mod.rs +++ b/kernel/kernel/src/scheduler/mod.rs @@ -18,7 +18,7 @@ use crate::{ arch::{self, Context}, signal, support::DisableInterruptGuard, - sync::SpinLockGuard, + sync::OwnedIrqSpinLockGuard, thread, thread::{Entry, GlobalQueueVisitor, Thread, ThreadNode}, time, @@ -111,24 +111,24 @@ fn init_timers() { } } -fn set_per_cpu_timer(id: usize, deadline: Tick) { +fn set_per_cpu_timer(id: usize, deadline: Tick, irq: &DisableInterruptGuard) { debug_assert_ne!(deadline, Tick::MAX); debug_assert!(!arch::local_irq_enabled()); unsafe { let iou = TimerIou::from_mut(PER_CPU_TIMER[id].assume_init_mut()); - timer::remove_hard_timer(iou); + timer::remove_hard_timer_with(iou, irq); let tm_mut = PER_CPU_TIMER[id].assume_init_mut(); tm_mut.reset(); let TimerMode::Deadline(d) = &mut tm_mut.mode else { panic!("PER_CPU_TIMER is in a wrong mode") }; *d = deadline; - let _ = timer::add_hard_timer(tm_mut).unwrap(); + let _ = timer::add_hard_timer_with(tm_mut, irq).unwrap(); } } -fn set_current_timer(deadline: Tick) { - set_per_cpu_timer(arch::current_cpu_id(), deadline); +fn set_current_timer(deadline: Tick, irq: &DisableInterruptGuard) { + set_per_cpu_timer(arch::current_cpu_id(), deadline, irq); } pub(crate) struct ContextSwitchHookHolder { @@ -199,12 +199,20 @@ fn switch_current_thread(next: ThreadNode, old_sp: usize) -> usize { next.set_this_round_start_at(now); let time_slices = next.refresh_time_slices(); let deadline = now.add(time_slices); - set_current_timer(deadline); + // SAFETY: all switch_current_thread callers are architecture hooks + // running with the kernel IRQ mask held through this timer update. + let irq = unsafe { DisableInterruptGuard::assume_disabled() }; + set_current_timer(deadline, irq); } #[cfg(thread_stats)] let cycles = time::current_clock_cycles(); #[cfg(thread_stats)] - next.lock().set_start_cycles(cycles); + { + // SAFETY: switch hook runs under the architecture's IRQ mask; this + // short stats update neither schedules nor invokes callbacks. + let irq = unsafe { DisableInterruptGuard::assume_disabled() }; + next.lock_with(irq).set_start_cycles(cycles); + } let next_id = Thread::id(&next); let next_priority = next.priority(); let next_saved_sp = spin_until_ready_to_run(&next); @@ -217,7 +225,11 @@ fn switch_current_thread(next: ThreadNode, old_sp: usize) -> usize { debug_assert_eq!(ok, Ok(())); let mut old = set_current_thread(next); #[cfg(thread_stats)] - old.lock().increment_cycles(cycles); + { + // SAFETY: same masked switch-hook context as above. + let irq = unsafe { DisableInterruptGuard::assume_disabled() }; + old.lock_with(irq).increment_cycles(cycles); + } #[cfg(debugging_scheduler)] crate::trace!( "Switching from 0x{:x}: {{ SP: 0x{:x} PRI: {} }} to 0x{:x}: {{ SP: 0x{:x} PRI: {} }}", @@ -267,12 +279,19 @@ fn cleanup_retired_thread(old: &mut ThreadNode) { // It's usually used in cortex-m's pendsv handler. pub(crate) extern "C" fn relinquish_me_and_return_next_sp(old_sp: usize) -> usize { + if crate::sync::local_context::defer_if_held() { + return old_sp; + } debug_assert_eq!(old_sp % core::mem::align_of::(), 0); debug_assert!(!arch::local_irq_enabled()); debug_assert!(!crate::irq::is_in_irq()); let old = current_thread_ref(); debug_assert_eq!(old.preempt_count(), 0); - let Some(next) = next_preferred_thread(old.priority(), PreferMode::StrictHigher) else { + // SAFETY: PendSV masked kernel IRQs before this hook and restores them only + // after it returns; this hook does not invoke a synchronous context switch. + let irq = unsafe { DisableInterruptGuard::assume_disabled() }; + let Some(next) = next_preferred_thread_with(old.priority(), PreferMode::StrictHigher, irq) + else { #[cfg(debugging_scheduler)] crate::trace!("[TH:0x{:x}] keeps running", Thread::id(old)); return old_sp; @@ -282,13 +301,19 @@ pub(crate) extern "C" fn relinquish_me_and_return_next_sp(old_sp: usize) -> usiz let ok = old.transfer_state(thread::RUNNING, thread::READY); debug_assert_eq!(ok, Ok(())); } else { - let ok = queue_ready_thread(thread::RUNNING, unsafe { Arc::clone_from(old) }); + let ok = queue_ready_thread_with(thread::RUNNING, &unsafe { Arc::clone_from(old) }, irq); debug_assert_eq!(ok, Ok(())); }; switch_current_thread(next, old_sp) } pub fn retire_me() -> ! { + crate::sync::local_context::assert_can_switch(); + assert!( + arch::local_irq_enabled(), + "cannot retire in a masked context" + ); + assert!(!crate::irq::is_in_irq(), "cannot retire in an interrupt"); let retiring = current_thread_ref(); #[cfg(procfs)] { @@ -317,15 +342,16 @@ fn inner_yield(next: ThreadNode) { debug_assert_eq!(ok, Ok(())); }; arch::switch_context_with_hook(&mut hook_holder as *mut _); - debug_assert!(arch::local_irq_enabled()); + assert!(arch::local_irq_enabled()); old.finish_resumption(); } pub fn yield_me() { + crate::sync::local_context::assert_can_switch(); // We don't allow thread yielding with irq disabled. // The scheduler assumes every thread should be resumed with local // irq enabled. - debug_assert!(arch::local_irq_enabled()); + assert!(arch::local_irq_enabled()); let Some(next) = next_ready_thread() else { return; }; @@ -334,7 +360,8 @@ pub fn yield_me() { } pub fn relinquish_me() { - debug_assert!(arch::local_irq_enabled()); + crate::sync::local_context::assert_can_switch(); + assert!(arch::local_irq_enabled()); let old = current_thread_ref(); let Some(next) = next_preferred_thread(old.priority(), PreferMode::SameOrHigher) else { return; @@ -343,7 +370,8 @@ pub fn relinquish_me() { inner_yield(next); } -pub fn suspend_me_for(mut timeout: Tick, wq: Option>) -> bool { +/// Suspends with an optional queue lock acquired by [`SpinLock::irqsave_lock`](crate::sync::SpinLock::irqsave_lock). +pub fn suspend_me_for(mut timeout: Tick, wq: Option>) -> bool { if timeout != Tick::MAX { timeout = Tick::after(timeout); } @@ -354,7 +382,8 @@ pub fn suspend_me_for(mut timeout: Tick, wq: Option>) -> // exception, we have to implement waking up the thread via signal. Currently, // the timer is detached when the thread is re-scheduled, we have to implement // detaching from a waitqueue uniformly. -pub fn suspend_me_until(deadline: Tick, wq: Option>) -> bool { +/// Publishes the suspended state before releasing the supplied queue lock and IRQ mask. +pub fn suspend_me_until(deadline: Tick, wq: Option>) -> bool { if unlikely(!is_schedule_ready()) { return false; } @@ -363,10 +392,15 @@ pub fn suspend_me_until(deadline: Tick, wq: Option>) -> "[TH:0x{:x}] is looking for the next thread", current_thread_id() ); + assert!(!crate::irq::is_in_irq(), "cannot suspend in an interrupt"); + crate::sync::local_context::assert_can_suspend(wq.is_some()); let old = current_thread_ref(); - let current_idle = idle::current_idle_thread_ref(); - debug_assert_ne!(Thread::id(old), Thread::id(current_idle)); - let next = next_ready_thread().unwrap_or_else(|| unsafe { Arc::clone_from(current_idle) }); + debug_assert_ne!(Thread::id(old), Thread::id(idle::current_idle_thread_ref())); + let next = match &wq { + Some(guard) => next_ready_thread_with(guard.irq_guard()), + None => next_ready_thread(), + } + .unwrap_or_else(idle::current_idle_thread); debug_assert_eq!(next.state(), thread::READY); #[cfg(debugging_scheduler)] crate::trace!( @@ -396,7 +430,7 @@ pub fn suspend_me_until(deadline: Tick, wq: Option>) -> } else { arch::switch_context_with_hook(&mut hook_holder as *mut _); } - debug_assert!(arch::local_irq_enabled()); + assert!(arch::local_irq_enabled()); old.finish_resumption(); reached_deadline } @@ -409,13 +443,16 @@ pub fn yield_me_now_or_later() { if unlikely(!is_schedule_ready()) { return; } + if crate::sync::local_context::defer_if_held() { + return; + } if current_thread_ref().preempt_count() != 0 { return; } arch::pend_switch_context(); } -pub fn wake_up_all(mut w: SpinLockGuard<'_, WaitQueue>) -> usize { +pub fn wake_up_all(mut w: OwnedIrqSpinLockGuard<'_, WaitQueue>) -> usize { #[cfg(debugging_scheduler)] crate::trace!("[TH:0x{:x}] is waking up all threads", current_thread_id()); let woken = wait_queue::wake_up_all(&mut w); @@ -427,7 +464,7 @@ pub fn wake_up_all(mut w: SpinLockGuard<'_, WaitQueue>) -> usize { } // wake_up_all, but injects `err` into each woken waiter. Same post-wake yield. -pub fn wake_up_all_with_error(mut w: SpinLockGuard<'_, WaitQueue>, err: i32) -> usize { +pub fn wake_up_all_with_error(mut w: OwnedIrqSpinLockGuard<'_, WaitQueue>, err: i32) -> usize { #[cfg(debugging_scheduler)] crate::trace!( "[TH:0x{:x}] is waking up all threads with error {}", @@ -456,8 +493,10 @@ pub fn wait_and_then_start_schedule() { // Entry of system idle threads. pub extern "C" fn schedule() -> ! { READY_CORES.fetch_add(1, Ordering::Relaxed); - arch::enable_local_irq(); - debug_assert!(arch::local_irq_enabled()); + unsafe { + arch::enable_local_irq(); + } + assert!(arch::local_irq_enabled()); loop { yield_me(); idle::get_idle_hook()(); @@ -466,9 +505,14 @@ pub extern "C" fn schedule() -> ! { #[inline] pub fn current_thread() -> ThreadNode { - let _guard = DisableInterruptGuard::new(); - let my_id = arch::current_cpu_id(); + let guard = DisableInterruptGuard::new(); + current_thread_with(&guard) +} +#[inline] +pub(crate) fn current_thread_with(_irq: &DisableInterruptGuard) -> ThreadNode { + let my_id = arch::current_cpu_id(); + // SAFETY: the IRQ proof keeps this CPU's running thread stable for the clone. unsafe { RUNNING_THREADS[my_id].assume_init_ref().clone() } } diff --git a/kernel/kernel/src/thread/builder.rs b/kernel/kernel/src/thread/builder.rs index 1f433f9a8..a29adc242 100644 --- a/kernel/kernel/src/thread/builder.rs +++ b/kernel/kernel/src/thread/builder.rs @@ -19,7 +19,7 @@ use crate::{ arch::Context, config, debug, scheduler, static_arc, support::Storage, - sync::spinlock::{SpinLock, SpinLockGuard}, + sync::spinlock::{OwnedIrqSpinLockGuard, SpinLock}, thread, trace, types::{ Arc, ArcInner, ArcList, ArcListIterator, AtomicIlistHead as ListHead, StaticListOwner, Uint, @@ -41,7 +41,7 @@ static_arc! { } pub struct GlobalQueueVisitor<'a> { - lock: SpinLockGuard<'a, Head>, + lock: OwnedIrqSpinLockGuard<'a, Head>, it: ArcListIterator<'a, Thread, OffsetOfGlobal>, } diff --git a/kernel/kernel/src/thread/mod.rs b/kernel/kernel/src/thread/mod.rs index 095a22bf7..0dd9ef6f2 100644 --- a/kernel/kernel/src/thread/mod.rs +++ b/kernel/kernel/src/thread/mod.rs @@ -26,7 +26,7 @@ use crate::{ support::{Region, RegionalObjectBuilder, Storage}, sync::{ mutex::{MutexList, MutexListIterator}, - ISpinLock, Mutex, SpinLock, SpinLockGuard, SpinLockWriteGuard, + ISpinLock, Mutex, OwnedIrqSpinLockGuard, SpinLock, }, thread::builder::GlobalQueue, time::Tick, @@ -311,9 +311,16 @@ impl Thread { // FIXME: rustc miscompiles it if not inlined. #[inline] - pub fn lock(&self) -> SpinLockGuard<'_, Self> { + pub fn lock(&self) -> OwnedIrqSpinLockGuard<'_, Self> { self.lock.irqsave_lock() } + #[inline(always)] + pub(crate) fn lock_with<'a>( + &'a self, + irq: &'a crate::support::DisableInterruptGuard, + ) -> crate::sync::spinlock::BorrowedSpinLockGuard<'a, Self> { + self.lock.lock_with(irq) + } #[inline(always)] pub fn stack_usage(&self) -> usize { @@ -458,7 +465,7 @@ impl Thread { } #[inline] - pub fn acquired_mutexes_mut(&self) -> SpinLockWriteGuard<'_, MutexList> { + pub fn acquired_mutexes_mut(&self) -> OwnedIrqSpinLockGuard<'_, MutexList> { self.acquired_mutexes.irqsave_write() } diff --git a/kernel/kernel/src/time.rs b/kernel/kernel/src/time.rs index 7aa43a47f..2f4352cec 100644 --- a/kernel/kernel/src/time.rs +++ b/kernel/kernel/src/time.rs @@ -91,7 +91,12 @@ impl Tick { } pub fn interrupt_at(n: Tick) { - let _guard = DisableInterruptGuard::new(); + let guard = DisableInterruptGuard::new(); + Self::interrupt_at_with(n, &guard); + } + + #[inline] + pub(crate) fn interrupt_at_with(n: Tick, _irq: &DisableInterruptGuard) { if n == Self::MAX { ClockImpl::stop(); return; diff --git a/kernel/kernel/src/time/timer.rs b/kernel/kernel/src/time/timer.rs index d2fc43613..b1b819f1b 100644 --- a/kernel/kernel/src/time/timer.rs +++ b/kernel/kernel/src/time/timer.rs @@ -17,6 +17,7 @@ use crate::{ boards::ClockImpl, config::MAX_THREAD_PRIORITY, scheduler, static_arc, + support::DisableInterruptGuard, sync::{atomic_wait, atomic_wake, SpinLock}, thread, thread::{Entry, SystemThreadStorage, ThreadKind, ThreadNode}, @@ -231,25 +232,35 @@ pub fn remove_soft_timer<'a>(iou: Iou<'_>) -> Option> { } pub fn add_hard_timer(tm: &mut Timer) -> Option> { - let res; - { - let mut w = HW_TIMERS.irqsave_lock(); - res = w.add(tm); - } + let res = HW_TIMERS.irqsave_lock().add(tm); update_clock_interrupt(); res } +pub(crate) fn add_hard_timer_with<'tm>( + tm: &'tm mut Timer, + irq: &DisableInterruptGuard, +) -> Option> { + let res = HW_TIMERS.lock_with(irq).add(tm); + update_clock_interrupt_with(irq); + res +} + pub fn remove_hard_timer<'a>(iou: Iou<'_>) -> Option> { - let res; - { - let mut w = HW_TIMERS.irqsave_lock(); - res = w.remove(iou); - } + let res = HW_TIMERS.irqsave_lock().remove(iou); update_clock_interrupt(); res } +pub(crate) fn remove_hard_timer_with<'a>( + iou: Iou<'_>, + irq: &DisableInterruptGuard, +) -> Option> { + let res = HW_TIMERS.lock_with(irq).remove(iou); + update_clock_interrupt_with(irq); + res +} + pub fn is_active_hard_timer(iou: &Iou<'_>) -> bool { let mut w = HW_TIMERS.irqsave_lock(); w.is_active(iou) @@ -301,6 +312,20 @@ fn update_clock_interrupt() -> Tick { next_deadline } +fn update_clock_interrupt_with(irq: &DisableInterruptGuard) -> Tick { + let soft_deadline = { + let mut w = unsafe { &SW_TIMER_WORKER.timers }.lock_with(irq); + w.next_deadline().unwrap_or(Tick::MAX) + }; + let hard_deadline = HW_TIMERS + .lock_with(irq) + .next_deadline() + .unwrap_or(Tick::MAX); + let next_deadline = core::cmp::min(soft_deadline, hard_deadline); + Tick::interrupt_at_with(next_deadline, irq); + next_deadline +} + #[cfg(test)] mod tests { use super::*; From 1dc32bda5d038e147df1b24097bd7c1827640b41 Mon Sep 17 00:00:00 2001 From: Hegui Dai Date: Fri, 20 Nov 2026 09:49:36 +0800 Subject: [PATCH 07/10] fix(sync): carry owned IRQ guards through wait queue handoffs Carry IRQ protection from queue acquisition through suspension and wakeup. Update semaphore and message queue operations, atomic waits and lock accessors to use the final guard types. --- kernel/adapter/cmsis/src/os2/thread.rs | 4 +-- kernel/kernel/src/asynk/mod.rs | 4 +-- kernel/kernel/src/support.rs | 6 ++-- kernel/kernel/src/sync/atomic_wait.rs | 8 ++---- kernel/kernel/src/sync/mqueue.rs | 8 ++---- kernel/kernel/src/sync/mutex.rs | 10 +++---- kernel/kernel/src/sync/semaphore.rs | 39 ++++++++++++++++++-------- kernel/kernel/src/types.rs | 8 +++--- 8 files changed, 49 insertions(+), 38 deletions(-) diff --git a/kernel/adapter/cmsis/src/os2/thread.rs b/kernel/adapter/cmsis/src/os2/thread.rs index 8178f8505..91e2903fc 100644 --- a/kernel/adapter/cmsis/src/os2/thread.rs +++ b/kernel/adapter/cmsis/src/os2/thread.rs @@ -26,7 +26,7 @@ use blueos::{ sync::{ atomic_wait, atomic_wake, event_flags::{EventFlags, EventFlagsMode}, - ConstBarrier, SpinLockGuard, + ConstBarrier, OwnedIrqSpinLockGuard, }, thread::{self, Entry, GlobalQueueVisitor, Thread}, time, @@ -68,7 +68,7 @@ impl Os2Thread { to self.inner() { pub fn priority(&self) -> ThreadPriority; pub fn state(&self) -> Uint; - pub fn lock(&self) -> SpinLockGuard<'_, Thread>; + pub fn lock(&self) -> OwnedIrqSpinLockGuard<'_, Thread>; pub fn stack_size(&self) -> usize; pub fn stack_base(&self) -> usize; pub fn saved_stack_usage(&self) -> usize; diff --git a/kernel/kernel/src/asynk/mod.rs b/kernel/kernel/src/asynk/mod.rs index 2550283f6..c6e443a32 100644 --- a/kernel/kernel/src/asynk/mod.rs +++ b/kernel/kernel/src/asynk/mod.rs @@ -22,7 +22,7 @@ use crate::{ scheduler::WaitQueue, static_arc, support::ArcBufferingQueue, - sync::{atomic_wait, SpinLock, SpinLockGuard}, + sync::{atomic_wait, OwnedIrqSpinLockGuard, SpinLock}, thread::{self, Entry, SystemThreadStorage, ThreadKind, ThreadNode}, time::Tick, types::{impl_simple_intrusive_adapter, Arc, IlistHead}, @@ -61,7 +61,7 @@ impl Tasklet { } } - fn lock_inner(&self) -> SpinLockGuard<'_, TaskletInner> { + fn lock_inner(&self) -> OwnedIrqSpinLockGuard<'_, TaskletInner> { self.inner.irqsave_lock() } } diff --git a/kernel/kernel/src/support.rs b/kernel/kernel/src/support.rs index f66b82a30..bca83f9ec 100644 --- a/kernel/kernel/src/support.rs +++ b/kernel/kernel/src/support.rs @@ -14,7 +14,7 @@ use crate::{ arch, - sync::spinlock::{SpinLock, SpinLockGuard}, + sync::spinlock::{OwnedIrqSpinLockGuard, SpinLock}, thread::ThreadNode, types::{Arc, ArcList, AtomicUint, IntrusiveAdapter, NotThreadSafe, Uint}, }; @@ -309,14 +309,14 @@ impl, const N: usize> ArcBufferingQueue SpinLockGuard<'_, ArcList> { + pub fn advance_active_queue(&self) -> OwnedIrqSpinLockGuard<'_, ArcList> { let i = self.active.fetch_add(1, Ordering::AcqRel) as usize; self.queues[i % N].irqsave_lock() } #[inline] #[allow(clippy::unnecessary_cast)] - pub fn get_active_queue(&self) -> SpinLockGuard<'_, ArcList> { + pub fn get_active_queue(&self) -> OwnedIrqSpinLockGuard<'_, ArcList> { let i = self.active.load(Ordering::Acquire) as usize; self.queues[i % N].irqsave_lock() } diff --git a/kernel/kernel/src/sync/atomic_wait.rs b/kernel/kernel/src/sync/atomic_wait.rs index 8e4af0527..9d17366a6 100644 --- a/kernel/kernel/src/sync/atomic_wait.rs +++ b/kernel/kernel/src/sync/atomic_wait.rs @@ -110,8 +110,7 @@ pub fn atomic_wait(atom: &AtomicUsize, val: usize, timeout: Tick) -> Result<(), |e| unsafe { Arc::clone_from(e) }, ); let t = scheduler::current_thread(); - let mut we = entry.pending.lock(); - we.take_irq_guard(w.get_guard_mut()); + let mut we = entry.pending.irqsave_lock(); drop(w); #[cfg(debugging_scheduler)] crate::trace!( @@ -126,7 +125,7 @@ pub fn atomic_wait(atom: &AtomicUsize, val: usize, timeout: Tick) -> Result<(), wait_queue::insert(we.deref_mut(), &mut wait_entry, InsertMode::InsertToEnd).unwrap(); reached_deadline = scheduler::suspend_me_for(timeout, Some(we)); w = EntryListHead::lock(); - we = entry.pending.lock(); + we = entry.pending.irqsave_lock(); borrowed_wait_entry = we.pop(borrowed_wait_entry).unwrap(); }); if we.is_empty() { @@ -161,8 +160,7 @@ pub fn atomic_wake(atom: &AtomicUsize, how_many: usize) -> Result let Some(entry) = entry else { return Ok(woken); }; - let mut we = entry.pending.lock(); - we.take_irq_guard(w.get_guard_mut()); + let mut we = entry.pending.irqsave_lock(); drop(w); for next in we.iter() { if scheduler::queue_ready_thread(thread::SUSPENDED, next.thread.clone()).is_err() { diff --git a/kernel/kernel/src/sync/mqueue.rs b/kernel/kernel/src/sync/mqueue.rs index 61648b754..40417c52f 100644 --- a/kernel/kernel/src/sync/mqueue.rs +++ b/kernel/kernel/src/sync/mqueue.rs @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -use super::{SpinLock, SpinLockGuard}; +use super::{OwnedIrqSpinLockGuard, SpinLock}; use crate::{ allocator, error::{code, Error}, @@ -118,7 +118,7 @@ impl MessageQueue { } #[inline] - fn lock_buffer(&self) -> SpinLockGuard<'_, BoxedRingBuffer> { + fn lock_buffer(&self) -> OwnedIrqSpinLockGuard<'_, BoxedRingBuffer> { self.queue_buffer.irqsave_lock() } @@ -157,7 +157,7 @@ impl MessageQueue { self.accessible_count[RECV_TYPE].fetch_sub(1, Ordering::Relaxed); } - fn wakeup_pend_receiver(w: &mut SpinLockGuard<'_, WaitQueue>) -> bool { + fn wakeup_pend_receiver(w: &mut OwnedIrqSpinLockGuard<'_, WaitQueue>) -> bool { for next in w.iter() { let t = next.thread.clone(); if scheduler::queue_ready_thread(thread::SUSPENDED, t).is_ok() { @@ -197,7 +197,6 @@ impl MessageQueue { if irq::is_in_irq() { return Err(code::ENOTSUP); } - send_queue.take_irq_guard(&mut buffer_guard); drop(buffer_guard); let reached_deadline; with_iou!(|borrowed_wait_entry| { @@ -287,7 +286,6 @@ impl MessageQueue { if irq::is_in_irq() { return Err(code::ENOTSUP); } - recv_queue.take_irq_guard(&mut buffer_guard); drop(buffer_guard); let reached_deadline; with_iou!(|borrowed_wait_entry| { diff --git a/kernel/kernel/src/sync/mutex.rs b/kernel/kernel/src/sync/mutex.rs index 7990899c9..724f4243c 100644 --- a/kernel/kernel/src/sync/mutex.rs +++ b/kernel/kernel/src/sync/mutex.rs @@ -26,7 +26,7 @@ // consistency, instead it utilizes a local approach, while keeping the above // invariance, to achieve a not bad result. -use super::{SpinLock, SpinLockGuard}; +use super::{OwnedIrqSpinLockGuard, SpinLock}; use crate::{ irq, scheduler::{self, wait_queue, InsertMode, OffsetOfWait, WaitEntry, WaitQueue}, @@ -77,7 +77,7 @@ impl Mutex { self.pending.irqsave_lock().init() } - pub fn waitqueue_guard(&self) -> SpinLockGuard<'_, WaitQueue> { + pub fn waitqueue_guard(&self) -> OwnedIrqSpinLockGuard<'_, WaitQueue> { self.pending.irqsave_lock() } @@ -199,10 +199,10 @@ impl Mutex { fn inner_pend_for<'a>( ticks: Tick, this_mutex: &'a Arc, - mut this_lock: SpinLockGuard<'a, WaitQueue>, + mut this_lock: OwnedIrqSpinLockGuard<'a, WaitQueue>, this_thread: &'a ThreadNode, owner_thread: &ThreadNode, - ) -> (bool, SpinLockGuard<'a, WaitQueue>) { + ) -> (bool, OwnedIrqSpinLockGuard<'a, WaitQueue>) { // Publish the wait edge before propagating priority. A concurrent donor // either sees this edge and follows it, or promotes us before we acquire // our thread lock below, so this walk observes the inherited priority. @@ -309,7 +309,7 @@ impl Mutex { _target_prio: ThreadPriority, who: &ThreadNode, mutex: &Arc, - mutex_lock: &mut SpinLockGuard<'_, WaitQueue>, + mutex_lock: &mut OwnedIrqSpinLockGuard<'_, WaitQueue>, ) -> bool { let Some(pending_on_mutex) = who.pending_on_mutex() else { return false; diff --git a/kernel/kernel/src/sync/semaphore.rs b/kernel/kernel/src/sync/semaphore.rs index b02cc13d9..27e583b7c 100644 --- a/kernel/kernel/src/sync/semaphore.rs +++ b/kernel/kernel/src/sync/semaphore.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{SpinLock, SpinLockGuard}; +use super::{OwnedIrqSpinLockGuard, SpinLock}; use crate::{ error::code, irq, scheduler, @@ -71,12 +71,29 @@ impl Semaphore { true } + #[inline(always)] pub fn acquire_notimeout(&self) -> bool where M: InsertModeTrait, { debug_assert!(!irq::is_in_irq()); - let mut w = self.pending.irqsave_lock(); + let w = self.pending.irqsave_lock(); + let old = self.counter.get(); + if old != 0 { + self.counter.set(old - 1); + return true; + } + self.acquire_notimeout_slow::(w) + } + + // Keep wait-entry/timer setup out of the uncontended path's stack frame. + // The existing payload borrow and IRQ lease are passed in, never reacquired. + #[cold] + #[inline(never)] + fn acquire_notimeout_slow<'a, M>(&'a self, mut w: OwnedIrqSpinLockGuard<'a, WaitQueue>) -> bool + where + M: InsertModeTrait, + { loop { let old = self.counter.get(); #[cfg(debugging_scheduler)] @@ -94,8 +111,7 @@ impl Semaphore { return true; } with_iou!(|borrowed_wait_entry| { - let this_thread = scheduler::current_thread(); - let mut wait_entry = WaitEntry::new(this_thread.clone()); + let mut wait_entry = WaitEntry::new(scheduler::current_thread_with(w.irq_guard())); borrowed_wait_entry = wait_queue::insert(w.deref_mut(), &mut wait_entry, M::MODE).unwrap(); let timeout = scheduler::suspend_me_for(Tick::MAX, Some(w)); @@ -103,7 +119,7 @@ impl Semaphore { w = self.pending.irqsave_lock(); borrowed_wait_entry = w.pop(borrowed_wait_entry).unwrap(); // ERESET injected by reset()/delete()? Bail out. - if this_thread.take_wait_error() != 0 { + if wait_entry.thread.take_wait_error() != 0 { return false; } }); @@ -138,15 +154,14 @@ impl Semaphore { } let timeout; with_iou!(|borrowed_wait_entry| { - let this_thread = scheduler::current_thread(); - let mut wait_entry = WaitEntry::new(this_thread.clone()); + let mut wait_entry = WaitEntry::new(scheduler::current_thread_with(w.irq_guard())); borrowed_wait_entry = wait_queue::insert(w.deref_mut(), &mut wait_entry, M::MODE).unwrap(); timeout = scheduler::suspend_me_for(ticks, Some(w)); w = self.pending.irqsave_lock(); borrowed_wait_entry = w.pop(borrowed_wait_entry).unwrap(); // ERESET injected by reset()/delete()? Bail out. - if this_thread.take_wait_error() != 0 { + if wait_entry.thread.take_wait_error() != 0 { return false; } }); @@ -171,7 +186,7 @@ impl Semaphore { self.acquire_timeout::(timeout) } - #[inline] + #[inline(always)] pub fn release(&self) { let w = self.pending.irqsave_lock(); let old = self.counter.get(); @@ -193,11 +208,11 @@ impl Semaphore { #[cold] #[inline(never)] - fn wake_waiter(w: SpinLockGuard<'_, WaitQueue>) { + fn wake_waiter(w: OwnedIrqSpinLockGuard<'_, WaitQueue>) { let mut woken = false; + let irq = w.irq_guard(); for next in w.iter() { - let t = next.thread.clone(); - if scheduler::queue_ready_thread(thread::SUSPENDED, t).is_ok() { + if scheduler::queue_ready_thread_with(thread::SUSPENDED, &next.thread, irq).is_ok() { woken = true; break; } diff --git a/kernel/kernel/src/types.rs b/kernel/kernel/src/types.rs index 239332c6a..1046841ce 100644 --- a/kernel/kernel/src/types.rs +++ b/kernel/kernel/src/types.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::sync::{SpinLock, SpinLockGuard}; +use crate::sync::{OwnedIrqSpinLockGuard, SpinLock}; pub use blueos_infra::{ iheap::{ IouMinHeapNodeMut as IouIntrusiveBinaryHeapNodeMut, MinHeap as IntrusiveBinaryHeap, @@ -90,7 +90,7 @@ pub(crate) struct UniqueListHeadAccessGuard< A: IntrusiveAdapter + 'static, O: StaticListOwner, >( - SpinLockGuard<'static, AtomicIlistHead>, + OwnedIrqSpinLockGuard<'static, AtomicIlistHead>, PhantomData, ); @@ -98,7 +98,7 @@ impl + 'static, O: StaticListOwner> UniqueListHeadAccessGuard { #[inline] - pub fn new(w: SpinLockGuard<'static, AtomicIlistHead>) -> Self { + pub fn new(w: OwnedIrqSpinLockGuard<'static, AtomicIlistHead>) -> Self { Self(w, PhantomData) } @@ -118,7 +118,7 @@ impl + 'static, O: StaticListOwner> } #[inline] - pub fn get_guard_mut(&mut self) -> &mut SpinLockGuard<'static, AtomicIlistHead> { + pub fn get_guard_mut(&mut self) -> &mut OwnedIrqSpinLockGuard<'static, AtomicIlistHead> { &mut self.0 } } From e93c9d17321ad463866b0229c255619979c5bfc1 Mon Sep 17 00:00:00 2001 From: Hegui Dai Date: Fri, 20 Nov 2026 10:39:52 +0800 Subject: [PATCH 08/10] fix(infra): keep TinyArcList::pop_front out of line on the RV32 scheduler path RV32 release builds can lose the empty-queue check after an inlined pop_front, leaving the priority active after its last thread is removed. Keep the function out of line pending the intrusive list aliasing work tracked in #559. --- kernel/infra/src/tinyarc/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/infra/src/tinyarc/mod.rs b/kernel/infra/src/tinyarc/mod.rs index edd43b044..a551c3759 100644 --- a/kernel/infra/src/tinyarc/mod.rs +++ b/kernel/infra/src/tinyarc/mod.rs @@ -354,6 +354,10 @@ impl> TinyArcList { Some(unsafe { Self::clone_from(next.as_ref()) }) } + // TODO(#559): Rework how detach updates head.next through the first node's prev + // pointer. Until then, inlining on RV32 release can elide the caller's post-pop + // emptiness check and leave an empty scheduler queue marked active. + #[inline(never)] pub fn pop_front(&mut self) -> Option> { if self.is_empty() { return None; From d50cabd27a4c4b812adcd9db2cccadf1641a2d3e Mon Sep 17 00:00:00 2001 From: Hegui Dai Date: Fri, 20 Nov 2026 09:49:36 +0800 Subject: [PATCH 09/10] test(sync): exercise UP lock contracts and semaphore contention Add thread-metric probes for local lock behavior and the contended semaphore handoff. --- kernel/adapter/thread_metric/BUILD.gn | 40 +++ .../src/contended_semaphore_probe.rs | 63 +++++ .../thread_metric/src/local_lock_probe.rs | 231 ++++++++++++++++++ 3 files changed, 334 insertions(+) create mode 100644 kernel/adapter/thread_metric/src/contended_semaphore_probe.rs create mode 100644 kernel/adapter/thread_metric/src/local_lock_probe.rs diff --git a/kernel/adapter/thread_metric/BUILD.gn b/kernel/adapter/thread_metric/BUILD.gn index c37e1058c..be385c35a 100644 --- a/kernel/adapter/thread_metric/BUILD.gn +++ b/kernel/adapter/thread_metric/BUILD.gn @@ -87,3 +87,43 @@ group("run_tests") { ":run_tm_synchronization_processing_test", ] } + +# Real-kernel lock contract checks, separate from measured benchmarks. +foreach(probe, + [ + "local_lock_contract_probe", + "local_lock_reject_yield_probe", + "local_lock_high_irq_probe", + ]) { + build_rust(probe) { + crate_type = "bin" + sources = [ "src/local_lock_probe.rs" ] + configs += [ "//build/boards/${board}:kernel_config" ] + deps = board_deps + [ + "//kernel/kernel:blueos", + "//kernel/rsrt", + "//libs/libc", + "//libs/librs", + ] + linker_script = default_linker_script + if (probe == "local_lock_high_irq_probe") { + rustflags = [ "--cfg=probe_high_irq" ] + } + if (probe == "local_lock_reject_yield_probe") { + rustflags = [ "--cfg=probe_yield_in_lock" ] + } + } +} + +build_rust("contended_semaphore_probe") { + crate_type = "bin" + sources = [ "src/contended_semaphore_probe.rs" ] + configs += [ "//build/boards/${board}:kernel_config" ] + deps = board_deps + [ + "//kernel/kernel:blueos", + "//kernel/rsrt", + "//libs/libc", + "//libs/librs", + ] + linker_script = default_linker_script +} diff --git a/kernel/adapter/thread_metric/src/contended_semaphore_probe.rs b/kernel/adapter/thread_metric/src/contended_semaphore_probe.rs new file mode 100644 index 000000000..559a48dd9 --- /dev/null +++ b/kernel/adapter/thread_metric/src/contended_semaphore_probe.rs @@ -0,0 +1,63 @@ +// Copyright (c) 2026 vivo Mobile Communication Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![no_std] +#![no_main] + +extern crate librs; +extern crate rsrt; + +use blueos::{scheduler::InsertToEnd, sync::Semaphore, thread, time::Tick}; +use core::sync::atomic::{AtomicBool, Ordering}; + +static PING: Semaphore = Semaphore::new(); +static PONG: Semaphore = Semaphore::new(); +static STOP: AtomicBool = AtomicBool::new(false); + +#[unsafe(no_mangle)] +pub extern "C" fn main() -> i32 { + librs::stdio::init(); + librs::pthread::register_my_posix_tcb(); + + assert!(PING.init(1)); + assert!(PONG.init(0)); + + thread::spawn(|| loop { + assert!(PONG.acquire_notimeout::()); + if STOP.load(Ordering::Relaxed) { + break; + } + PING.release(); + }) + .unwrap(); + + for _ in 0..6 { + let end = Tick::after(Tick::from_millis(2000)); + let mut count = 0u32; + while Tick::now().0 < end.0 { + assert!(PING.acquire_notimeout::()); + PONG.release(); + count += 1; + } + unsafe { + libc::printf(c"Time Period Total: %u\n".as_ptr(), count); + } + } + STOP.store(true, Ordering::Relaxed); + PONG.release(); + + loop { + core::hint::spin_loop(); + } +} diff --git a/kernel/adapter/thread_metric/src/local_lock_probe.rs b/kernel/adapter/thread_metric/src/local_lock_probe.rs new file mode 100644 index 000000000..05eb3ef1e --- /dev/null +++ b/kernel/adapter/thread_metric/src/local_lock_probe.rs @@ -0,0 +1,231 @@ +// Copyright (c) 2026 vivo Mobile Communication Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![no_std] +#![no_main] +extern crate librs; +extern crate rsrt; + +use blueos::{arch, scheduler, support::DisableInterruptGuard, sync::SpinLock}; +use core::sync::atomic::{AtomicUsize, Ordering}; + +static IRQ_LOCK: SpinLock = SpinLock::new(0); +static IRQ_RESULT: AtomicUsize = AtomicUsize::new(0); +const TEST_IRQ: arch::arm::irq::IrqNumber = arch::arm::irq::IrqNumber::new(0); + +unsafe extern "C" fn test_irq() { + IRQ_RESULT.store( + if IRQ_LOCK.try_lock().is_some() { 1 } else { 2 }, + Ordering::Relaxed, + ); +} + +fn pend_test_irq() { + unsafe { + core::ptr::write_volatile(0xe000e200 as *mut u32, 1); + core::arch::asm!("dsb", "isb", options(nostack)); + } +} + +fn irq_checks() { + let irq = TEST_IRQ; + unsafe { + arch::arm::irq::register_raw_isr(irq, test_irq); + } + #[cfg(probe_high_irq)] + arch::arm::irq::enable_irq_with_priority(irq, arch::arm::irq::Priority::High); + #[cfg(not(probe_high_irq))] + arch::arm::irq::enable_irq_with_priority(irq, arch::arm::irq::Priority::Normal); + let g = IRQ_LOCK.lock(); + pend_test_irq(); + while IRQ_RESULT.load(Ordering::Relaxed) == 0 { + core::hint::spin_loop(); + } + assert_eq!(IRQ_RESULT.load(Ordering::Relaxed), 2); + drop(g); + IRQ_RESULT.store(0, Ordering::Relaxed); + let g = IRQ_LOCK.irqsave_lock(); + pend_test_irq(); + assert_eq!(IRQ_RESULT.load(Ordering::Relaxed), 0); + drop(g); + while IRQ_RESULT.load(Ordering::Relaxed) == 0 { + core::hint::spin_loop(); + } + assert_eq!(IRQ_RESULT.load(Ordering::Relaxed), 1); + arch::arm::irq::disable_irq(irq); +} + +fn irq_wakes_semaphore_waiter() { + use blueos::{sync::Semaphore, thread, time::Tick}; + use core::sync::atomic::AtomicBool; + + static SEMAPHORE: Semaphore = Semaphore::new(); + static RELEASE_RETURNED: AtomicBool = AtomicBool::new(false); + + unsafe extern "C" fn release_from_irq() { + assert!(arch::arm::irq::is_irq_active(TEST_IRQ)); + SEMAPHORE.release(); + RELEASE_RETURNED.store(true, Ordering::Release); + } + + assert!(SEMAPHORE.init(0)); + // SAFETY: irq_checks disabled this probe's IRQ before we replace its handler. + unsafe { arch::arm::irq::register_raw_isr(TEST_IRQ, release_from_irq) }; + arch::arm::irq::enable_irq_with_priority(TEST_IRQ, arch::arm::irq::Priority::Normal); + + let waiter = scheduler::current_thread(); + thread::spawn(move || { + while waiter.state() != thread::SUSPENDED { + scheduler::suspend_me_for::<()>(Tick(1), None); + } + pend_test_irq(); + }) + .unwrap(); + + assert!( + SEMAPHORE.acquire_timeout::(Tick(100)), + "IRQ did not wake the semaphore waiter before timeout" + ); + assert!(RELEASE_RETURNED.load(Ordering::Acquire)); + assert_eq!(SEMAPHORE.count(), 0); + arch::arm::irq::disable_irq(TEST_IRQ); +} + +fn wait_checks() { + use blueos::{ + sync::{atomic_wait, atomic_wake, mqueue::SendMode, MessageQueue, Semaphore}, + thread, + time::Tick, + types::Arc, + }; + let sema = Arc::new(Semaphore::new()); + assert!(sema.init(0)); + let producer = sema.clone(); + thread::spawn(move || { + scheduler::suspend_me_for::<()>(Tick(2), None); + producer.release(); + }) + .unwrap(); + assert!(sema.acquire::(Tick(100))); + let atom = Arc::new(AtomicUsize::new(0)); + let producer = atom.clone(); + thread::spawn(move || { + scheduler::suspend_me_for::<()>(Tick(2), None); + producer.store(1, Ordering::Release); + atomic_wake(&producer, 1).unwrap(); + }) + .unwrap(); + atomic_wait(&atom, 0, Tick(100)).unwrap(); + assert_eq!(atom.load(Ordering::Acquire), 1); + let queue = Arc::new(MessageQueue::new(4, 1, core::ptr::null_mut()).unwrap()); + assert!(queue.init()); + let producer = queue.clone(); + thread::spawn(move || { + scheduler::suspend_me_for::<()>(Tick(2), None); + producer + .send(&[1, 2, 3, 4], 4, Tick(100), SendMode::Normal) + .unwrap(); + }) + .unwrap(); + let mut buf = [0; 4]; + queue.recv(&mut buf, 4, Tick(100)).unwrap(); + assert_eq!(buf, [1, 2, 3, 4]); + queue + .send(&[5, 6, 7, 8], 4, Tick(100), SendMode::Normal) + .unwrap(); + let consumer = queue.clone(); + thread::spawn(move || { + scheduler::suspend_me_for::<()>(Tick(2), None); + let mut buf = [0; 4]; + consumer.recv(&mut buf, 4, Tick(100)).unwrap(); + assert_eq!(buf, [5, 6, 7, 8]); + }) + .unwrap(); + queue + .send(&[9, 10, 11, 12], 4, Tick(100), SendMode::Normal) + .unwrap(); + queue.recv(&mut buf, 4, Tick(100)).unwrap(); + assert_eq!(buf, [9, 10, 11, 12]); +} + +#[unsafe(no_mangle)] +pub extern "C" fn main() -> i32 { + librs::stdio::init(); + librs::pthread::register_my_posix_tcb(); + let lock = SpinLock::new(0usize); + assert!(arch::local_irq_enabled()); + let mut w = lock.lock(); + assert!(arch::local_irq_enabled()); + assert!(lock.try_write().is_none()); + assert!(lock.try_read().is_none()); + *w = 42; + #[cfg(probe_yield_in_lock)] + scheduler::yield_me(); + scheduler::yield_me_now_or_later(); + assert_eq!(*w, 42); + drop(w); + let a = lock.read(); + let b = lock.read(); + assert_eq!(*a, 42); + assert!(lock.try_write().is_none()); + drop(a); + assert!(lock.try_write().is_none()); + drop(b); + assert!(lock.try_write().is_some()); + let first = DisableInterruptGuard::new(); + let second = DisableInterruptGuard::new(); + assert!(!arch::local_irq_enabled()); + drop(first); + assert!(!arch::local_irq_enabled()); + drop(second); + assert!(arch::local_irq_enabled()); + let other = SpinLock::new(1); + let a = lock.irqsave_lock(); + let b = other.irqsave_lock(); + assert!(lock.try_irqsave_read().is_none()); + assert!(lock.try_irqsave_lock().is_none()); + drop(a); + assert!(!arch::local_irq_enabled()); + drop(b); + assert!(arch::local_irq_enabled()); + irq_checks(); + irq_wakes_semaphore_waiter(); + wait_checks(); + // Stronger pre-existing masks survive entry and final restoration. + unsafe { + core::arch::asm!("msr basepri, {}", in(reg) 0x40u32, options(nostack)); + } + let g = DisableInterruptGuard::new(); + let mask: u32; + unsafe { + core::arch::asm!("mrs {}, basepri", out(reg) mask, options(nostack)); + } + assert_eq!(mask, 0x40); + drop(g); + unsafe { + arch::enable_local_irq(); + } + // A leaked plain guard must never restore access to the payload. + let leaked = SpinLock::new(3); + core::mem::forget(leaked.lock()); + assert!(leaked.try_write().is_none()); + assert!(leaked.try_read().is_none()); + // No scheduling is attempted after the deliberately leaked preemption lease. + unsafe { + libc::printf(c"LOCAL_LOCK_CONTRACT_PASS\n".as_ptr()); + } + loop { + core::hint::spin_loop(); + } +} From f6c311e4f26e1a5f023d2bbab4243b4548fd212f Mon Sep 17 00:00:00 2001 From: Hegui Dai Date: Fri, 20 Nov 2026 09:49:36 +0800 Subject: [PATCH 10/10] fix(cmsis): install validation IRQ handlers and propagate test failures Install the handlers used by the CMSIS validation suite and make validation failures visible to the test runner. --- .../Source/Config/RV2_Config.c | 7 +++---- kernel/adapter/cmsis/src/validation.checker | 3 ++- kernel/adapter/cmsis/src/validation.rs | 19 ++++++++++++++----- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/external/CMSIS-RTOS2_Validation/Source/Config/RV2_Config.c b/external/CMSIS-RTOS2_Validation/Source/Config/RV2_Config.c index 8401bae62..44a8ea92d 100644 --- a/external/CMSIS-RTOS2_Validation/Source/Config/RV2_Config.c +++ b/external/CMSIS-RTOS2_Validation/Source/Config/RV2_Config.c @@ -33,7 +33,7 @@ extern int stdout_putchar (int ch); typedef unsigned short IRQn_Type; /* forward to rust */ -extern void NVIC_SetPriority(IRQn_Type IRQn, char priority); +extern void configure_validation_irqs(IRQn_Type primary, IRQn_Type secondary); extern void NVIC_EnableIRQ(IRQn_Type IRQn); extern void NVIC_DisableIRQ(IRQn_Type IRQn); extern uint32_t NVIC_GetPendingIRQ(IRQn_Type IRQn); @@ -65,9 +65,8 @@ void TST_IRQ_HANDLER_B (void) { void TS_Init (void) { #if defined(__CORTEX_M) - /* Set primary and secondary interrupt priority */ - NVIC_SetPriority ((IRQn_Type)TST_IRQ_NUM_A, 5U); - NVIC_SetPriority ((IRQn_Type)TST_IRQ_NUM_B, 4U); + /* Install the validation handlers and set their kernel-callable priorities. */ + configure_validation_irqs (TST_IRQ_NUM_A, TST_IRQ_NUM_B); /* Enable interrupts */ NVIC_EnableIRQ((IRQn_Type)TST_IRQ_NUM_A); diff --git a/kernel/adapter/cmsis/src/validation.checker b/kernel/adapter/cmsis/src/validation.checker index 80b1664fe..a482a5982 100644 --- a/kernel/adapter/cmsis/src/validation.checker +++ b/kernel/adapter/cmsis/src/validation.checker @@ -1,3 +1,4 @@ // TOTAL-TIMEOUT: 25 // ASSERT-SUCC: cmsis_rv2 unittest finished -// ASSERT-FAIL: Oops: \ No newline at end of file +// ASSERT-FAIL: Oops: +// ASSERT-FAIL: Test Result: FAILED diff --git a/kernel/adapter/cmsis/src/validation.rs b/kernel/adapter/cmsis/src/validation.rs index bf5412b39..c70fb24b9 100644 --- a/kernel/adapter/cmsis/src/validation.rs +++ b/kernel/adapter/cmsis/src/validation.rs @@ -126,12 +126,21 @@ mod tests { } } #[unsafe(no_mangle)] - pub unsafe extern "C" fn NVIC_SetPriority(irq: u16, priority: u8) { + pub unsafe extern "C" fn configure_validation_irqs(primary: u16, secondary: u16) { + use arch::irq::{disable_irq, register_raw_isr, set_irq_priority, IrqNumber, Priority}; + let primary = IrqNumber::new(primary); + let secondary = IrqNumber::new(secondary); + disable_irq(primary); + disable_irq(secondary); + // SAFETY: TS_Init installs the validation vectors before enabling either IRQ. unsafe { - cortex_m::Peripherals::steal() - .NVIC - .set_priority(IrqNumber::new(irq), priority) - }; + register_raw_isr(primary, uart::uart1rx_handler); + register_raw_isr(secondary, uart::uart1tx_handler); + } + // Both ISRs call RTOS APIs. Keep the secondary IRQ above the primary + // while allowing the kernel's IRQ mask to exclude either one. + set_irq_priority(primary, Priority::Low as u8); + set_irq_priority(secondary, Priority::Normal as u8); } #[unsafe(no_mangle)] pub unsafe extern "C" fn NVIC_EnableIRQ(irq: u16) {