+1 -1

Cargo.toml

··· 101 101 cfg-if = "1.0.0" 102 102 log = "0.4.27" 103 103 bitflags = "2.9.1" 104 104 - lock_api = "0.4.12" 105 104 xmas-elf = "0.10.0" 105 105 + lock_api = "0.4.13" 106 106 static_assertions = "1.1.0" 107 107 rand = { version = "0.9.1", default-features = false } 108 108 rand_chacha = { version = "0.9.0", default-features = false }

+1 -1

kernel/Cargo.toml

··· 13 13 loader-api.workspace = true 14 14 cpu-local.workspace = true 15 15 trap.workspace = true 16 16 - spin = { workspace = true, features = ["thread-local", "lock_api"] } 16 16 + spin = { workspace = true, features = ["thread-local"] } 17 17 unwind2.workspace = true 18 18 wavltree = { workspace = true, features = ["dot"] } 19 19 fdt.workspace = true

+1 -1

kernel/src/allocator.rs

··· 15 15 use crate::{INITIAL_HEAP_SIZE_PAGES, arch}; 16 16 17 17 #[global_allocator] 18 18 - static KERNEL_ALLOCATOR: Talck<spin::Mutex<()>, ErrOnOom> = Talc::new(ErrOnOom).lock(); 18 18 + static KERNEL_ALLOCATOR: Talck<spin::RawMutex, ErrOnOom> = Talc::new(ErrOnOom).lock(); 19 19 20 20 pub fn init(boot_alloc: &mut BootstrapAllocator, boot_info: &BootInfo) { 21 21 let layout =

+1 -2

libs/spin/Cargo.toml

··· 11 11 12 12 [dependencies] 13 13 cfg-if.workspace = true 14 14 - lock_api = { workspace = true, optional = true } 14 14 + lock_api.workspace = true 15 15 util.workspace = true 16 16 17 17 [target.'cfg(loom)'.dependencies] ··· 19 19 20 20 [features] 21 21 thread-local = [] 22 22 - lock_api = ["dep:lock_api"]

+2 -1

libs/spin/src/barrier.rs

··· 74 74 use std::sync::mpsc::{TryRecvError, channel}; 75 75 76 76 use super::*; 77 77 - use crate::loom::{Arc, thread}; 77 77 + use crate::loom::sync::Arc; 78 78 + use crate::loom::thread; 78 79 79 80 #[test] 80 81 fn test_barrier() {

+3 -2

libs/spin/src/lazy_lock.rs

··· 14 14 15 15 use super::Once; 16 16 use super::once::ExclusiveState; 17 17 - use crate::loom::UnsafeCell; 17 17 + use crate::loom::cell::UnsafeCell; 18 18 19 19 union Data<T, F> { 20 20 value: ManuallyDrop<T>, ··· 170 170 use std::cell::LazyCell; 171 171 172 172 use super::*; 173 173 - use crate::loom::{AtomicUsize, Ordering, thread}; 173 173 + use crate::loom::sync::atomic::{AtomicUsize, Ordering}; 174 174 + use crate::loom::thread; 174 175 use crate::{Mutex, OnceLock}; 175 176 176 177 fn spawn_and_wait<R: Send + 'static>(f: impl FnOnce() -> R + Send + 'static) -> R {

+1 -1

libs/spin/src/lib.rs

··· 27 27 pub use backoff::Backoff; 28 28 pub use barrier::{Barrier, BarrierWaitResult}; 29 29 pub use lazy_lock::LazyLock; 30 30 - pub use mutex::{Mutex, MutexGuard}; 30 30 + pub use mutex::{Mutex, MutexGuard, RawMutex}; 31 31 pub use once::{ExclusiveState, Once}; 32 32 pub use once_lock::OnceLock; 33 33 #[cfg(feature = "thread-local")]

+47 -72

libs/spin/src/loom.rs

··· 1 1 - // Copyright 2025 Jonas Kruckenberg 1 1 + // Copyright 2025. Jonas Kruckenberg 2 2 // 3 3 // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or 4 4 // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or 5 5 // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 // copied, modified, or distributed except according to those terms. 7 7 - 8 8 - // #![allow(unused_imports,)] 9 7 10 8 cfg_if::cfg_if! { 11 9 if #[cfg(loom)] { 12 12 - pub(crate) use loom::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering}; 13 13 - pub(crate) use loom::cell::UnsafeCell; 10 10 + pub(crate) use loom::sync; 11 11 + pub(crate) use loom::cell; 14 12 pub(crate) use loom::thread; 15 15 - pub(crate) use loom::model; 16 16 - pub(crate) use loom::sync::Arc; 17 13 } else { 18 18 - pub(crate) use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; 19 19 - 20 20 - #[derive(Debug)] 21 21 - #[repr(transparent)] 22 22 - pub(crate) struct UnsafeCell<T: ?Sized>(core::cell::UnsafeCell<T>); 23 23 - 24 24 - impl<T> UnsafeCell<T> { 25 25 - pub const fn new(data: T) -> UnsafeCell<T> { 26 26 - UnsafeCell(core::cell::UnsafeCell::new(data)) 27 27 - } 28 28 - } 29 29 - 30 30 - impl<T: ?Sized> UnsafeCell<T> { 31 31 - #[inline(always)] 32 32 - pub fn with<F, R>(&self, f: F) -> R 33 33 - where 34 34 - F: FnOnce(*const T) -> R, 35 35 - { 36 36 - f(self.0.get()) 37 37 - } 38 38 - #[inline(always)] 39 39 - pub fn with_mut<F, R>(&self, f: F) -> R 40 40 - where 41 41 - F: FnOnce(*mut T) -> R, 42 42 - { 43 43 - f(self.0.get()) 44 44 - } 45 45 - } 46 46 - impl<T> UnsafeCell<T> { 47 47 - #[inline(always)] 48 48 - #[must_use] 49 49 - pub(crate) fn into_inner(self) -> T { 50 50 - self.0.into_inner() 51 51 - } 52 52 - } 53 53 - 54 54 - #[derive(Debug)] 55 55 - #[repr(transparent)] 56 56 - pub(crate) struct AtomicU8(core::sync::atomic::AtomicU8); 57 57 - 58 58 - impl AtomicU8 { 59 59 - #[inline(always)] 60 60 - pub const fn new(val: u8) -> Self { 61 61 - Self(core::sync::atomic::AtomicU8::new(val)) 62 62 - } 63 63 - #[inline(always)] 64 64 - pub fn load(&self, order: Ordering) -> u8 { 65 65 - self.0.load(order) 66 66 - } 67 67 - #[inline(always)] 68 68 - pub fn store(& self, val: u8, order: Ordering) { 69 69 - self.0.store(val, order); 70 70 - } 71 71 - #[inline(always)] 72 72 - pub fn compare_exchange(& self, current: u8, new: u8, success: Ordering, failure: Ordering) -> Result<u8, u8> { 73 73 - self.0.compare_exchange(current, new,success, failure) 74 74 - } 75 75 - #[inline(always)] 76 76 - pub fn with_mut<R>(&mut self, f: impl FnOnce(&mut u8) -> R) -> R { 77 77 - f(self.0.get_mut()) 78 78 - } 79 79 - } 80 80 - 81 81 - #[cfg(test)] 82 82 - pub(crate) use std::sync::Arc; 83 14 #[cfg(test)] 84 15 pub(crate) use std::thread; 85 16 ··· 90 21 F: Fn() + Sync + Send + 'static, 91 22 { 92 23 f() 24 24 + } 25 25 + 26 26 + pub(crate) mod sync { 27 27 + pub(crate) use core::sync::*; 28 28 + 29 29 + 30 30 + #[cfg(test)] 31 31 + pub(crate) use std::sync::*; 32 32 + } 33 33 + 34 34 + pub(crate) mod cell { 35 35 + #[derive(Debug)] 36 36 + #[repr(transparent)] 37 37 + pub(crate) struct UnsafeCell<T: ?Sized>(core::cell::UnsafeCell<T>); 38 38 + 39 39 + impl<T> UnsafeCell<T> { 40 40 + pub const fn new(data: T) -> UnsafeCell<T> { 41 41 + UnsafeCell(core::cell::UnsafeCell::new(data)) 42 42 + } 43 43 + } 44 44 + 45 45 + impl<T: ?Sized> UnsafeCell<T> { 46 46 + #[inline(always)] 47 47 + pub fn with<F, R>(&self, f: F) -> R 48 48 + where 49 49 + F: FnOnce(*const T) -> R, 50 50 + { 51 51 + f(self.0.get()) 52 52 + } 53 53 + #[inline(always)] 54 54 + pub fn with_mut<F, R>(&self, f: F) -> R 55 55 + where 56 56 + F: FnOnce(*mut T) -> R, 57 57 + { 58 58 + f(self.0.get()) 59 59 + } 60 60 + } 61 61 + impl<T> UnsafeCell<T> { 62 62 + #[inline(always)] 63 63 + #[must_use] 64 64 + pub(crate) fn into_inner(self) -> T { 65 65 + self.0.into_inner() 66 66 + } 67 67 + } 93 68 } 94 69 } 95 70 }

+153 -429

libs/spin/src/mutex.rs

··· 5 5 // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 // copied, modified, or distributed except according to those terms. 7 7 8 8 - // Mutex 9 9 - // A mutual exclusion primitive useful for protecting shared data 10 10 - // MutexGuard 11 11 - // An RAII implementation of a “scoped lock” of a mutex. When this structure is dropped (falls out of scope), the lock will be unlocked. 12 12 - // MappedMutexGuard 13 13 - // An RAII mutex guard returned by MutexGuard::map, which can point to a subfield of the protected data. 14 14 - // ArcMutexGuardarc_lock 15 15 - // An RAII mutex guard returned by the Arc locking operations on Mutex. 8 8 + use core::sync::atomic::{AtomicBool, Ordering}; 16 9 17 17 - use core::marker::PhantomData; 18 18 - use core::ops::{Deref, DerefMut}; 19 19 - use core::{fmt, mem}; 10 10 + use crate::Backoff; 20 11 21 21 - use util::loom_const_fn; 12 12 + pub type Mutex<T> = lock_api::Mutex<RawMutex, T>; 13 13 + pub type MutexGuard<'a, T> = lock_api::MutexGuard<'a, RawMutex, T>; 22 14 23 23 - use crate::backoff::Backoff; 24 24 - use crate::loom::{AtomicBool, Ordering, UnsafeCell}; 25 25 - 26 26 - /// A mutual exclusion primitive useful for protecting shared data 27 27 - /// 28 28 - /// This mutex will block threads waiting for the lock to become available. The 29 29 - /// mutex can also be statically initialized or created via a `new` 30 30 - /// constructor. Each mutex has a type parameter which represents the data that 31 31 - /// it is protecting. The data can only be accessed through the RAII guards 32 32 - /// returned from `lock` and `try_lock`, which guarantees that the data is only 33 33 - /// ever accessed when the mutex is locked. 34 34 - pub struct Mutex<T: ?Sized> { 15 15 + pub struct RawMutex { 35 16 lock: AtomicBool, 36 36 - data: UnsafeCell<T>, 37 17 } 38 18 39 39 - /// An RAII implementation of a "scoped lock" of a mutex. When this structure is 40 40 - /// dropped (falls out of scope), the lock will be unlocked. 41 41 - /// 42 42 - /// The data protected by the mutex can be accessed through this guard via its 43 43 - /// `Deref` and `DerefMut` implementations. 44 44 - #[clippy::has_significant_drop] 45 45 - #[must_use = "if unused the Mutex will immediately unlock"] 46 46 - pub struct MutexGuard<'a, T: ?Sized> { 47 47 - mutex: &'a Mutex<T>, 48 48 - marker: PhantomData<&'a mut T>, 49 49 - } 19 19 + #[allow(clippy::undocumented_unsafe_blocks, reason = "TODO")] 20 20 + unsafe impl lock_api::RawMutex for RawMutex { 21 21 + type GuardMarker = lock_api::GuardSend; 50 22 51 51 - #[expect(clippy::undocumented_unsafe_blocks, reason = "")] 52 52 - unsafe impl<T: ?Sized + Send> Send for Mutex<T> {} 53 53 - #[expect(clippy::undocumented_unsafe_blocks, reason = "")] 54 54 - unsafe impl<T: ?Sized + Send> Sync for Mutex<T> {} 55 55 - 56 56 - impl<T> Mutex<T> { 57 57 - loom_const_fn! { 58 58 - pub const fn new(val: T) -> Mutex<T> { 59 59 - Mutex { 60 60 - lock: AtomicBool::new(false), 61 61 - data: UnsafeCell::new(val), 62 62 - } 63 63 - } 64 64 - } 65 65 - 66 66 - /// Consumes this mutex, returning the underlying data. 67 67 - #[inline] 68 68 - pub fn into_inner(self) -> T { 69 69 - self.data.into_inner() 70 70 - } 71 71 - } 72 72 - 73 73 - impl<T: ?Sized> Mutex<T> { 74 74 - /// Creates a new `MutexGuard` without checking if the mutex is locked. 75 75 - /// 76 76 - /// # Safety 77 77 - /// 78 78 - /// This method must only be called if the thread logically holds the lock. 79 79 - /// 80 80 - /// Calling this function when a guard has already been produced is undefined behaviour unless 81 81 - /// the guard was forgotten with `mem::forget`. 82 82 - #[inline] 83 83 - pub unsafe fn make_guard_unchecked(&self) -> MutexGuard<'_, T> { 84 84 - MutexGuard { 85 85 - mutex: self, 86 86 - marker: PhantomData, 87 87 - } 88 88 - } 23 23 + const INIT: Self = Self { 24 24 + lock: AtomicBool::new(false), 25 25 + }; 89 26 90 90 - /// Acquires a mutex, blocking the current thread until it is able to do so. 91 91 - /// 92 92 - /// This function will block the local thread until it is available to acquire 93 93 - /// the mutex. Upon returning, the thread is the only thread with the mutex 94 94 - /// held. An RAII guard is returned to allow scoped unlock of the lock. When 95 95 - /// the guard goes out of scope, the mutex will be unlocked. 96 96 - /// 97 97 - /// Attempts to lock a mutex in the thread which already holds the lock will 98 98 - /// result in a deadlock. 99 99 - #[inline] 100 100 - pub fn lock(&self) -> MutexGuard<'_, T> { 27 27 + fn lock(&self) { 101 28 let mut boff = Backoff::default(); 102 29 while self 103 30 .lock ··· 108 35 boff.spin(); 109 36 } 110 37 } 111 111 - 112 112 - // Safety: The lock is held, as required. 113 113 - unsafe { self.make_guard_unchecked() } 114 38 } 115 39 116 116 - /// Try to lock this mutex, returning a lock guard if successful. 117 117 - /// 118 118 - /// Like [`Self::lock`] the lock will be unlocked when the guard is dropped, but *unlike* 119 119 - /// [`Self::lock`] this method never blocks. 120 120 - #[inline] 121 121 - pub fn try_lock(&self) -> Option<MutexGuard<'_, T>> { 122 122 - if self 123 123 - .lock 40 40 + fn try_lock(&self) -> bool { 41 41 + self.lock 124 42 .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) 125 43 .is_ok() 126 126 - { 127 127 - // SAFETY: The lock is held, as required. 128 128 - Some(unsafe { self.make_guard_unchecked() }) 129 129 - } else { 130 130 - None 131 131 - } 132 44 } 133 45 134 134 - /// Try to lock this mutex, returning a lock guard if successful. 135 135 - /// 136 136 - /// Like [`Self::lock`] the lock will be unlocked when the guard is dropped, but *unlike* 137 137 - /// [`Self::lock`] this method never blocks. 138 138 - /// 139 139 - /// Unlike [`Self::try_lock`] this method can spuriously fail even if the mutex is unlocked, 140 140 - /// this makes this method only suitable to be called in loops or similar scenarios, but might 141 141 - /// result in more efficient code on some platforms. 142 142 - #[inline] 143 143 - pub fn try_lock_weak(&self) -> Option<MutexGuard<'_, T>> { 144 144 - if self 145 145 - .lock 146 146 - .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) 147 147 - .is_ok() 148 148 - { 149 149 - // SAFETY: The lock is held, as required. 150 150 - Some(unsafe { self.make_guard_unchecked() }) 151 151 - } else { 152 152 - None 153 153 - } 154 154 - } 155 155 - 156 156 - /// Returns a mutable reference to the underlying data. 157 157 - /// 158 158 - /// Since this call borrows the `Mutex` mutably, no actual locking needs to 159 159 - /// take place---the mutable borrow statically guarantees no locks exist. 160 160 - #[inline] 161 161 - pub fn get_mut(&mut self) -> &mut T { 162 162 - // Safety: We hold a mutable reference to the Mutex so getting a mutable reference to the 163 163 - // data is safe 164 164 - self.data.with_mut(|data| unsafe { &mut *data }) 165 165 - } 166 166 - 167 167 - /// Checks whether the mutex is currently locked. 168 168 - #[inline] 169 169 - pub fn is_locked(&self) -> bool { 170 170 - self.lock.load(Ordering::Relaxed) 171 171 - } 172 172 - 173 173 - /// Forcibly unlocks the mutex. 174 174 - /// 175 175 - /// This is useful when combined with `mem::forget` to hold a lock without 176 176 - /// the need to maintain a `MutexGuard` object alive, for example when 177 177 - /// dealing with FFI. 178 178 - /// 179 179 - /// # Safety 180 180 - /// 181 181 - /// This method must only be called if the current thread logically owns a 182 182 - /// `MutexGuard` but that guard has been discarded using `mem::forget`. 183 183 - /// Behavior is undefined if a mutex is unlocked when not locked. 184 184 - #[inline] 185 185 - pub unsafe fn force_unlock(&self) { 186 186 - self.lock.store(false, Ordering::Release); 187 187 - } 188 188 - } 189 189 - 190 190 - impl<T: Default> Default for Mutex<T> { 191 191 - #[inline] 192 192 - fn default() -> Mutex<T> { 193 193 - Mutex::new(Default::default()) 194 194 - } 195 195 - } 196 196 - 197 197 - impl<T> From<T> for Mutex<T> { 198 198 - #[inline] 199 199 - fn from(t: T) -> Mutex<T> { 200 200 - Mutex::new(t) 201 201 - } 202 202 - } 203 203 - 204 204 - impl<T: ?Sized + fmt::Debug> fmt::Debug for Mutex<T> { 205 205 - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 206 206 - match self.try_lock() { 207 207 - Some(guard) => f.debug_struct("Mutex").field("data", &&*guard).finish(), 208 208 - None => { 209 209 - struct LockedPlaceholder; 210 210 - impl fmt::Debug for LockedPlaceholder { 211 211 - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 212 212 - f.write_str("<locked>") 213 213 - } 214 214 - } 215 215 - 216 216 - f.debug_struct("Mutex") 217 217 - .field("data", &LockedPlaceholder) 218 218 - .finish() 219 219 - } 220 220 - } 221 221 - } 222 222 - } 223 223 - 224 224 - #[expect(clippy::undocumented_unsafe_blocks, reason = "")] 225 225 - unsafe impl<'a, T: ?Sized + Sync + 'a> Sync for MutexGuard<'a, T> {} 226 226 - 227 227 - impl<'a, T: ?Sized + 'a> MutexGuard<'a, T> { 228 228 - /// Returns a reference to the original `Mutex` object. 229 229 - pub fn mutex(s: &Self) -> &'a Mutex<T> { 230 230 - s.mutex 231 231 - } 232 232 - 233 233 - /// Leaks the mutex guard and returns a mutable reference to the data 234 234 - /// protected by the mutex. 235 235 - /// 236 236 - /// This will leave the `Mutex` in a locked state. 237 237 - #[inline] 238 238 - pub fn leak(s: Self) -> &'a mut T { 239 239 - // Safety: MutexGuard always holds the lock, so it is safe to access the data 240 240 - let r = s.mutex.data.with_mut(|r| unsafe { &mut *r }); 241 241 - mem::forget(s); 242 242 - r 243 243 - } 244 244 - 245 245 - pub fn unlocked<F, U>(s: &mut Self, f: F) -> U 246 246 - where 247 247 - F: FnOnce() -> U, 248 248 - { 249 249 - struct DropGuard<'a, T: ?Sized> { 250 250 - mutex: &'a Mutex<T>, 251 251 - } 252 252 - impl<T: ?Sized> Drop for DropGuard<'_, T> { 253 253 - fn drop(&mut self) { 254 254 - mem::forget(self.mutex.lock()); 255 255 - } 256 256 - } 257 257 - 258 258 - // Safety: A MutexGuard always holds the lock. 259 259 - unsafe { 260 260 - s.mutex.force_unlock(); 261 261 - } 262 262 - let _guard = DropGuard { mutex: s.mutex }; 263 263 - f() 264 264 - } 265 265 - } 266 266 - 267 267 - impl<'a, T: ?Sized + 'a> Deref for MutexGuard<'a, T> { 268 268 - type Target = T; 269 269 - #[inline] 270 270 - fn deref(&self) -> &T { 271 271 - // Safety: MutexGuard always holds the lock, so it is safe to access the data 272 272 - self.mutex.data.with(|data| unsafe { &*data }) 273 273 - } 274 274 - } 275 275 - 276 276 - impl<'a, T: ?Sized + 'a> DerefMut for MutexGuard<'a, T> { 277 277 - #[inline] 278 278 - fn deref_mut(&mut self) -> &mut T { 279 279 - // Safety: MutexGuard always holds the lock, so it is safe to access the data 280 280 - self.mutex.data.with_mut(|data| unsafe { &mut *data }) 281 281 - } 282 282 - } 283 283 - 284 284 - impl<'a, T: ?Sized + 'a> Drop for MutexGuard<'a, T> { 285 285 - #[inline] 286 286 - fn drop(&mut self) { 287 287 - // Safety: A MutexGuard always holds the lock. 288 288 - unsafe { 289 289 - self.mutex.force_unlock(); 290 290 - } 291 291 - } 292 292 - } 293 293 - 294 294 - impl<'a, T: fmt::Debug + ?Sized + 'a> fmt::Debug for MutexGuard<'a, T> { 295 295 - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 296 296 - fmt::Debug::fmt(&**self, f) 297 297 - } 298 298 - } 299 299 - 300 300 - impl<'a, T: fmt::Display + ?Sized + 'a> fmt::Display for MutexGuard<'a, T> { 301 301 - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 302 302 - (**self).fmt(f) 303 303 - } 304 304 - } 305 305 - 306 306 - #[cfg(feature = "lock_api")] 307 307 - #[expect(clippy::undocumented_unsafe_blocks, reason = "")] 308 308 - unsafe impl lock_api::RawMutex for Mutex<()> { 309 309 - #[allow(clippy::declare_interior_mutable_const, reason = "TODO")] 310 310 - const INIT: Self = Mutex::new(()); 311 311 - type GuardMarker = lock_api::GuardSend; 312 312 - 313 313 - fn lock(&self) { 314 314 - let g = Mutex::lock(self); 315 315 - mem::forget(g); 316 316 - } 317 317 - 318 318 - fn try_lock(&self) -> bool { 319 319 - let g = Mutex::try_lock(self); 320 320 - let ret = g.is_some(); 321 321 - mem::forget(g); 322 322 - ret 323 323 - } 46 46 + // fn try_lock_weak(&self) -> bool { 47 47 + // self.lock 48 48 + // .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) 49 49 + // .is_ok() 50 50 + // } 324 51 325 52 unsafe fn unlock(&self) { 326 326 - // Safety: ensured by caller 327 327 - unsafe { 328 328 - Mutex::force_unlock(self); 329 329 - } 53 53 + self.lock.store(false, Ordering::Release); 330 54 } 331 55 332 56 fn is_locked(&self) -> bool { 333 333 - Mutex::is_locked(self) 57 57 + self.lock.load(Ordering::Relaxed) 334 58 } 335 59 } 336 60 337 337 - #[cfg(test)] 338 338 - mod tests { 339 339 - use core::fmt::Debug; 340 340 - use std::{hint, mem}; 341 341 - 342 342 - use super::*; 343 343 - use crate::loom::{Arc, AtomicUsize}; 344 344 - 345 345 - #[derive(Eq, PartialEq, Debug)] 346 346 - struct NonCopy(i32); 347 347 - 348 348 - #[derive(Eq, PartialEq, Debug)] 349 349 - struct NonCopyNeedsDrop(i32); 350 350 - 351 351 - impl Drop for NonCopyNeedsDrop { 352 352 - fn drop(&mut self) { 353 353 - hint::black_box(()); 354 354 - } 355 355 - } 356 356 - 357 357 - #[test] 358 358 - fn test_needs_drop() { 359 359 - assert!(!mem::needs_drop::<NonCopy>()); 360 360 - assert!(mem::needs_drop::<NonCopyNeedsDrop>()); 361 361 - } 362 362 - 363 363 - #[test] 364 364 - fn smoke() { 365 365 - let m = Mutex::new(()); 366 366 - drop(m.lock()); 367 367 - drop(m.lock()); 368 368 - } 369 369 - 370 370 - #[test] 371 371 - fn try_lock() { 372 372 - let mutex = Mutex::<_>::new(42); 373 373 - 374 374 - // First lock succeeds 375 375 - let a = mutex.try_lock(); 376 376 - assert_eq!(a.as_ref().map(|r| **r), Some(42)); 377 377 - 378 378 - // Additional lock fails 379 379 - let b = mutex.try_lock(); 380 380 - assert!(b.is_none()); 381 381 - 382 382 - // After dropping lock, it succeeds again 383 383 - drop(a); 384 384 - let c = mutex.try_lock(); 385 385 - assert_eq!(c.as_ref().map(|r| **r), Some(42)); 386 386 - } 387 387 - 388 388 - #[test] 389 389 - fn test_into_inner() { 390 390 - let m = Mutex::<_>::new(NonCopy(10)); 391 391 - assert_eq!(m.into_inner(), NonCopy(10)); 392 392 - } 393 393 - 394 394 - #[test] 395 395 - fn test_into_inner_drop() { 396 396 - struct Foo(Arc<AtomicUsize>); 397 397 - impl Drop for Foo { 398 398 - fn drop(&mut self) { 399 399 - self.0.fetch_add(1, Ordering::SeqCst); 400 400 - } 401 401 - } 402 402 - let num_drops = Arc::new(AtomicUsize::new(0)); 403 403 - let m = Mutex::<_>::new(Foo(num_drops.clone())); 404 404 - assert_eq!(num_drops.load(Ordering::SeqCst), 0); 405 405 - { 406 406 - let _inner = m.into_inner(); 407 407 - assert_eq!(num_drops.load(Ordering::SeqCst), 0); 408 408 - } 409 409 - assert_eq!(num_drops.load(Ordering::SeqCst), 1); 410 410 - } 411 411 - 412 412 - #[test] 413 413 - fn test_mutex_unsized() { 414 414 - let mutex: &Mutex<[i32]> = &Mutex::<_>::new([1, 2, 3]); 415 415 - { 416 416 - let b = &mut *mutex.lock(); 417 417 - b[0] = 4; 418 418 - b[2] = 5; 419 419 - } 420 420 - let comp: &[i32] = &[4, 2, 5]; 421 421 - assert_eq!(&*mutex.lock(), comp); 422 422 - } 423 423 - 424 424 - #[test] 425 425 - fn test_mutex_force_lock() { 426 426 - let lock = Mutex::<_>::new(()); 427 427 - mem::forget(lock.lock()); 428 428 - unsafe { 429 429 - lock.force_unlock(); 430 430 - } 431 431 - assert!(lock.try_lock().is_some()); 432 432 - } 433 433 - 434 434 - #[test] 435 435 - fn test_get_mut() { 436 436 - let mut m = Mutex::new(NonCopy(10)); 437 437 - *m.get_mut() = NonCopy(20); 438 438 - assert_eq!(m.into_inner(), NonCopy(20)); 439 439 - } 440 440 - 441 441 - #[test] 442 442 - fn basic_multi_threaded() { 443 443 - use crate::loom::{self, Arc, thread}; 444 444 - 445 445 - #[allow(tail_expr_drop_order)] 446 446 - fn incr(lock: &Arc<Mutex<i32>>) -> thread::JoinHandle<()> { 447 447 - let lock = lock.clone(); 448 448 - thread::spawn(move || { 449 449 - let mut lock = lock.lock(); 450 450 - *lock += 1; 451 451 - }) 452 452 - } 453 453 - 454 454 - loom::model(|| { 455 455 - let lock = Arc::new(Mutex::new(0)); 456 456 - let t1 = incr(&lock); 457 457 - let t2 = incr(&lock); 458 458 - 459 459 - t1.join().unwrap(); 460 460 - t2.join().unwrap(); 461 461 - 462 462 - thread::spawn(move || { 463 463 - let lock = lock.lock(); 464 464 - assert_eq!(*lock, 2) 465 465 - }); 466 466 - }); 467 467 - } 468 468 - } 61 61 + // #[cfg(test)] 62 62 + // mod tests { 63 63 + // use core::fmt::Debug; 64 64 + // use std::{hint, mem}; 65 65 + // 66 66 + // use super::*; 67 67 + // use crate::loom::{Arc, AtomicUsize}; 68 68 + // 69 69 + // #[derive(Eq, PartialEq, Debug)] 70 70 + // struct NonCopy(i32); 71 71 + // 72 72 + // #[derive(Eq, PartialEq, Debug)] 73 73 + // struct NonCopyNeedsDrop(i32); 74 74 + // 75 75 + // impl Drop for NonCopyNeedsDrop { 76 76 + // fn drop(&mut self) { 77 77 + // hint::black_box(()); 78 78 + // } 79 79 + // } 80 80 + // 81 81 + // #[test] 82 82 + // fn test_needs_drop() { 83 83 + // assert!(!mem::needs_drop::<NonCopy>()); 84 84 + // assert!(mem::needs_drop::<NonCopyNeedsDrop>()); 85 85 + // } 86 86 + // 87 87 + // #[test] 88 88 + // fn smoke() { 89 89 + // let m = Mutex::new(()); 90 90 + // drop(m.lock()); 91 91 + // drop(m.lock()); 92 92 + // } 93 93 + // 94 94 + // #[test] 95 95 + // fn try_lock() { 96 96 + // let mutex = Mutex::<_>::new(42); 97 97 + // 98 98 + // // First lock succeeds 99 99 + // let a = mutex.try_lock(); 100 100 + // assert_eq!(a.as_ref().map(|r| **r), Some(42)); 101 101 + // 102 102 + // // Additional lock fails 103 103 + // let b = mutex.try_lock(); 104 104 + // assert!(b.is_none()); 105 105 + // 106 106 + // // After dropping lock, it succeeds again 107 107 + // drop(a); 108 108 + // let c = mutex.try_lock(); 109 109 + // assert_eq!(c.as_ref().map(|r| **r), Some(42)); 110 110 + // } 111 111 + // 112 112 + // #[test] 113 113 + // fn test_into_inner() { 114 114 + // let m = Mutex::<_>::new(NonCopy(10)); 115 115 + // assert_eq!(m.into_inner(), NonCopy(10)); 116 116 + // } 117 117 + // 118 118 + // #[test] 119 119 + // fn test_into_inner_drop() { 120 120 + // struct Foo(Arc<AtomicUsize>); 121 121 + // impl Drop for Foo { 122 122 + // fn drop(&mut self) { 123 123 + // self.0.fetch_add(1, Ordering::SeqCst); 124 124 + // } 125 125 + // } 126 126 + // let num_drops = Arc::new(AtomicUsize::new(0)); 127 127 + // let m = Mutex::<_>::new(Foo(num_drops.clone())); 128 128 + // assert_eq!(num_drops.load(Ordering::SeqCst), 0); 129 129 + // { 130 130 + // let _inner = m.into_inner(); 131 131 + // assert_eq!(num_drops.load(Ordering::SeqCst), 0); 132 132 + // } 133 133 + // assert_eq!(num_drops.load(Ordering::SeqCst), 1); 134 134 + // } 135 135 + // 136 136 + // #[test] 137 137 + // fn test_mutex_unsized() { 138 138 + // let mutex: &Mutex<[i32]> = &Mutex::<_>::new([1, 2, 3]); 139 139 + // { 140 140 + // let b = &mut *mutex.lock(); 141 141 + // b[0] = 4; 142 142 + // b[2] = 5; 143 143 + // } 144 144 + // let comp: &[i32] = &[4, 2, 5]; 145 145 + // assert_eq!(&*mutex.lock(), comp); 146 146 + // } 147 147 + // 148 148 + // #[test] 149 149 + // fn test_mutex_force_lock() { 150 150 + // let lock = Mutex::<_>::new(()); 151 151 + // mem::forget(lock.lock()); 152 152 + // unsafe { 153 153 + // lock.force_unlock(); 154 154 + // } 155 155 + // assert!(lock.try_lock().is_some()); 156 156 + // } 157 157 + // 158 158 + // #[test] 159 159 + // fn test_get_mut() { 160 160 + // let mut m = Mutex::new(NonCopy(10)); 161 161 + // *m.get_mut() = NonCopy(20); 162 162 + // assert_eq!(m.into_inner(), NonCopy(20)); 163 163 + // } 164 164 + // 165 165 + // #[test] 166 166 + // fn basic_multi_threaded() { 167 167 + // use crate::loom::{self, Arc, thread}; 168 168 + // 169 169 + // #[allow(tail_expr_drop_order)] 170 170 + // fn incr(lock: &Arc<Mutex<i32>>) -> thread::JoinHandle<()> { 171 171 + // let lock = lock.clone(); 172 172 + // thread::spawn(move || { 173 173 + // let mut lock = lock.lock(); 174 174 + // *lock += 1; 175 175 + // }) 176 176 + // } 177 177 + // 178 178 + // loom::model(|| { 179 179 + // let lock = Arc::new(Mutex::new(0)); 180 180 + // let t1 = incr(&lock); 181 181 + // let t2 = incr(&lock); 182 182 + // 183 183 + // t1.join().unwrap(); 184 184 + // t2.join().unwrap(); 185 185 + // 186 186 + // thread::spawn(move || { 187 187 + // let lock = lock.lock(); 188 188 + // assert_eq!(*lock, 2) 189 189 + // }); 190 190 + // }); 191 191 + // } 192 192 + // }

+19 -8

libs/spin/src/once.rs

··· 9 9 10 10 use util::loom_const_fn; 11 11 12 12 - use crate::loom::{AtomicU8, Ordering}; 12 12 + use crate::loom::sync::atomic::{AtomicU8, Ordering}; 13 13 14 14 /// No initialization has run yet, and no thread is currently using the Once. 15 15 const STATUS_INCOMPLETE: u8 = 0; ··· 51 51 } 52 52 53 53 pub fn state(&mut self) -> ExclusiveState { 54 54 - self.status.with_mut(|status| match *status { 55 55 - STATUS_INCOMPLETE => ExclusiveState::Incomplete, 56 56 - STATUS_POISONED => ExclusiveState::Poisoned, 57 57 - STATUS_COMPLETE => ExclusiveState::Complete, 58 58 - _ => unreachable!("invalid Once state"), 59 59 - }) 54 54 + cfg_if::cfg_if! { 55 55 + if #[cfg(loom)] { 56 56 + self.status.with_mut(|status| match *status { 57 57 + STATUS_INCOMPLETE => ExclusiveState::Incomplete, 58 58 + STATUS_POISONED => ExclusiveState::Poisoned, 59 59 + STATUS_COMPLETE => ExclusiveState::Complete, 60 60 + _ => unreachable!("invalid Once state"), 61 61 + }) 62 62 + } else { 63 63 + match *self.status.get_mut() { 64 64 + STATUS_INCOMPLETE => ExclusiveState::Incomplete, 65 65 + STATUS_POISONED => ExclusiveState::Poisoned, 66 66 + STATUS_COMPLETE => ExclusiveState::Complete, 67 67 + _ => unreachable!("invalid Once state"), 68 68 + } 69 69 + } 70 70 + } 60 71 } 61 72 62 73 /// # Panics ··· 208 219 #[cfg(not(loom))] 209 220 #[test] 210 221 fn wait() { 211 211 - use crate::loom::{AtomicBool, Ordering}; 222 222 + use crate::loom::sync::atomic::{AtomicBool, Ordering}; 212 223 213 224 for _ in 0..50 { 214 225 let val = AtomicBool::new(false);

+3 -2

libs/spin/src/once_lock.rs

··· 12 12 use util::loom_const_fn; 13 13 14 14 use super::Once; 15 15 - use crate::loom::UnsafeCell; 15 15 + use crate::loom::cell::UnsafeCell; 16 16 17 17 /// A synchronization primitive which can be written to only once. 18 18 /// ··· 319 319 use std::sync::mpsc::channel; 320 320 321 321 use super::*; 322 322 - use crate::loom::{AtomicUsize, Ordering, thread}; 322 322 + use crate::loom::sync::atomic::{AtomicUsize, Ordering}; 323 323 + use crate::loom::thread; 323 324 324 325 fn spawn_and_wait<R: Send + 'static>(f: impl FnOnce() -> R + Send + 'static) -> R { 325 326 thread::spawn(f).join().unwrap()

+2 -1

libs/spin/src/remutex.rs

··· 15 15 16 16 use util::loom_const_fn; 17 17 18 18 - use crate::loom::{AtomicUsize, Ordering, UnsafeCell}; 18 18 + use crate::loom::cell::UnsafeCell; 19 19 + use crate::loom::sync::atomic::{AtomicUsize, Ordering}; 19 20 use crate::{Backoff, GuardNoSend}; 20 21 21 22 /// A mutex which can be recursively locked by a single thread.

+95 -492

libs/spin/src/rw_lock.rs

··· 5 5 // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 // copied, modified, or distributed except according to those terms. 7 7 8 8 - use core::ops::{Deref, DerefMut}; 9 9 - use core::ptr::NonNull; 10 10 - use core::{fmt, mem}; 8 8 + use core::sync::atomic::{AtomicUsize, Ordering}; 11 9 12 12 - use util::loom_const_fn; 10 10 + use lock_api::GuardSend; 13 11 14 12 use crate::Backoff; 15 15 - use crate::loom::{AtomicUsize, Ordering, UnsafeCell}; 16 13 17 14 const READER: usize = 1 << 2; 18 15 const UPGRADED: usize = 1 << 1; 19 16 const WRITER: usize = 1; 20 17 21 21 - pub struct RwLock<T: ?Sized> { 22 22 - lock: AtomicUsize, 23 23 - data: UnsafeCell<T>, 24 24 - } 25 25 - 26 26 - /// RAII structure used to release the shared read access of a lock when 27 27 - /// dropped. 28 28 - #[clippy::has_significant_drop] 29 29 - #[must_use = "if unused the RwLock will immediately unlock"] 30 30 - pub struct RwLockReadGuard<'a, T: ?Sized + 'a> { 31 31 - // NB: we use a pointer instead of `&'a T` to avoid `noalias` violations, because a 32 32 - // `RwLockReadGuard` argument doesn't hold immutability for its whole scope, only until it drops. 33 33 - // `NonNull` is also covariant over `T`, just like we would have with `&T`. `NonNull` 34 34 - // is preferable over `const* T` to allow for niche optimization. 35 35 - _data: NonNull<T>, 36 36 - rwlock: &'a RwLock<T>, 37 37 - } 38 38 - 39 39 - /// RAII structure used to release the exclusive write access of a lock when 40 40 - /// dropped. 41 41 - #[clippy::has_significant_drop] 42 42 - #[must_use = "if unused the RwLock will immediately unlock"] 43 43 - pub struct RwLockWriteGuard<'a, T: ?Sized> { 44 44 - rwlock: &'a RwLock<T>, 45 45 - } 46 46 - 47 47 - /// RAII structure used to release the upgradable read access of a lock when 48 48 - /// dropped. 49 49 - #[clippy::has_significant_drop] 50 50 - #[must_use = "if unused the RwLock will immediately unlock"] 51 51 - pub struct RwLockUpgradableReadGuard<'a, T: ?Sized + 'a> { 52 52 - // NB: we use a pointer instead of `&'a T` to avoid `noalias` violations, because a 53 53 - // `RwLockReadGuard` argument doesn't hold immutability for its whole scope, only until it drops. 54 54 - // `NonNull` is also covariant over `T`, just like we would have with `&T`. `NonNull` 55 55 - // is preferable over `const* T` to allow for niche optimization. 56 56 - data: NonNull<T>, 57 57 - rwlock: &'a RwLock<T>, 58 58 - } 18 18 + pub type RwLock<T> = lock_api::RwLock<RawRwLock, T>; 19 19 + pub type RwLockReadGuard<'a, T> = lock_api::RwLockReadGuard<'a, RawRwLock, T>; 20 20 + pub type RwLockWriteGuard<'a, T> = lock_api::RwLockWriteGuard<'a, RawRwLock, T>; 21 21 + pub type RwLockUpgradableReadGuard<'a, T> = lock_api::RwLockUpgradableReadGuard<'a, RawRwLock, T>; 59 22 60 60 - #[expect(clippy::undocumented_unsafe_blocks, reason = "")] 61 61 - unsafe impl<T: ?Sized + Send> Send for RwLock<T> {} 62 62 - #[expect(clippy::undocumented_unsafe_blocks, reason = "")] 63 63 - unsafe impl<T: ?Sized + Send + Sync> Sync for RwLock<T> {} 64 64 - 65 65 - impl<T> RwLock<T> { 66 66 - loom_const_fn! { 67 67 - /// Creates a new instance of an `RwLock<T>` which is unlocked. 68 68 - #[inline] 69 69 - pub const fn new(val: T) -> RwLock<T> { 70 70 - RwLock { 71 71 - data: UnsafeCell::new(val), 72 72 - lock: AtomicUsize::new(0), 73 73 - } 74 74 - } 75 75 - } 76 76 - 77 77 - /// Consumes this `RwLock`, returning the underlying data. 78 78 - #[inline] 79 79 - pub fn into_inner(self) -> T { 80 80 - self.data.into_inner() 81 81 - } 23 23 + pub struct RawRwLock { 24 24 + lock: AtomicUsize, 82 25 } 83 26 84 84 - impl<T: ?Sized> RwLock<T> { 85 85 - /// Creates a new `RwLockReadGuard` without checking if the lock is held. 86 86 - /// 87 87 - /// # Safety 88 88 - /// 89 89 - /// This method must only be called if the thread logically holds a read lock. 90 90 - /// 91 91 - /// This function does not increment the read count of the lock. Calling this function when a 92 92 - /// guard has already been produced is undefined behaviour unless the guard was forgotten 93 93 - /// with `mem::forget`. 94 94 - #[inline] 95 95 - pub unsafe fn make_read_guard_unchecked(&self) -> RwLockReadGuard<'_, T> { 96 96 - RwLockReadGuard { 97 97 - _data: self.data.with_mut(|data| { 98 98 - // Safety: ensured by caller 99 99 - unsafe { NonNull::new_unchecked(data) } 100 100 - }), 101 101 - rwlock: self, 102 102 - } 103 103 - } 104 104 - 105 105 - /// Creates a new `RwLockReadGuard` without checking if the lock is held. 106 106 - /// 107 107 - /// # Safety 108 108 - /// 109 109 - /// This method must only be called if the thread logically holds a write lock. 110 110 - /// 111 111 - /// Calling this function when a guard has already been produced is undefined behaviour unless 112 112 - /// the guard was forgotten with `mem::forget`. 113 113 - #[inline] 114 114 - pub unsafe fn make_write_guard_unchecked(&self) -> RwLockWriteGuard<'_, T> { 115 115 - RwLockWriteGuard { rwlock: self } 116 116 - } 117 117 - 118 118 - /// Locks this `RwLock` with shared read access, blocking the current thread 119 119 - /// until it can be acquired. 120 120 - /// 121 121 - /// The calling thread will be blocked until there are no more writers which 122 122 - /// hold the lock. There may be other readers currently inside the lock when 123 123 - /// this method returns. 124 124 - /// 125 125 - /// Note that attempts to recursively acquire a read lock on a `RwLock` when 126 126 - /// the current thread already holds one may result in a deadlock. 127 127 - /// 128 128 - /// Returns an RAII guard which will release this thread's shared access 129 129 - /// once it is dropped. 130 130 - #[inline] 131 131 - pub fn read(&self) -> RwLockReadGuard<'_, T> { 132 132 - self.lock_shared(); 133 133 - 134 134 - // SAFETY: The lock is held, as required. 135 135 - unsafe { self.make_read_guard_unchecked() } 136 136 - } 137 137 - 138 138 - /// Attempts to acquire this `RwLock` with shared read access. 139 139 - /// 140 140 - /// If the access could not be granted at this time, then `None` is returned. 141 141 - /// Otherwise, an RAII guard is returned which will release the shared access 142 142 - /// when it is dropped. 143 143 - /// 144 144 - /// This function does not block. 145 145 - #[inline] 146 146 - pub fn try_read(&self) -> Option<RwLockReadGuard<'_, T>> { 147 147 - if self.try_lock_shared() { 148 148 - // SAFETY: The lock is held, as required. 149 149 - Some(unsafe { self.make_read_guard_unchecked() }) 150 150 - } else { 151 151 - None 152 152 - } 153 153 - } 154 154 - 155 155 - fn acquire_reader(&self) -> usize { 156 156 - // An arbitrary cap that allows us to catch overflows long before they happen 157 157 - const MAX_READERS: usize = usize::MAX / READER / 2; 158 158 - 159 159 - let value = self.lock.fetch_add(READER, Ordering::Acquire); 160 160 - 161 161 - if value > MAX_READERS * READER { 162 162 - self.lock.fetch_sub(READER, Ordering::Relaxed); 163 163 - panic!("Too many lock readers, cannot safely proceed"); 164 164 - } else { 165 165 - value 166 166 - } 167 167 - } 168 168 - 169 169 - /// Locks this `RwLock` with exclusive write access, blocking the current 170 170 - /// thread until it can be acquired. 171 171 - /// 172 172 - /// This function will not return while other writers or other readers 173 173 - /// currently have access to the lock. 174 174 - /// 175 175 - /// Returns an RAII guard which will drop the write access of this `RwLock` 176 176 - /// when dropped. 177 177 - #[inline] 178 178 - pub fn write(&self) -> RwLockWriteGuard<'_, T> { 179 179 - self.lock_exclusive(); 180 180 - // SAFETY: The lock is held, as required. 181 181 - unsafe { self.make_write_guard_unchecked() } 182 182 - } 183 183 - 184 184 - /// Attempts to lock this `RwLock` with exclusive write access. 185 185 - /// 186 186 - /// If the lock could not be acquired at this time, then `None` is returned. 187 187 - /// Otherwise, an RAII guard is returned which will release the lock when 188 188 - /// it is dropped. 189 189 - /// 190 190 - /// This function does not block. 191 191 - #[inline] 192 192 - pub fn try_write(&self) -> Option<RwLockWriteGuard<'_, T>> { 193 193 - if self.try_lock_exclusive() { 194 194 - // SAFETY: The lock is held, as required. 195 195 - Some(unsafe { self.make_write_guard_unchecked() }) 196 196 - } else { 197 197 - None 198 198 - } 199 199 - } 200 200 - 201 201 - /// Returns a mutable reference to the underlying data. 202 202 - /// 203 203 - /// Since this call borrows the `RwLock` mutably, no actual locking needs to 204 204 - /// take place---the mutable borrow statically guarantees no locks exist. 205 205 - #[inline] 206 206 - pub fn get_mut(&mut self) -> &mut T { 207 207 - self.data.with_mut(|data| { 208 208 - // Safety: We hold a mutable reference to the RwLock so getting a mutable reference to the 209 209 - // data is safe 210 210 - unsafe { &mut *data } 211 211 - }) 212 212 - } 213 213 - 214 214 - /// Checks whether this `RwLock` is currently locked in any way. 215 215 - #[inline] 216 216 - pub fn is_locked(&self) -> bool { 217 217 - self.lock.load(Ordering::Relaxed) != 0 218 218 - } 27 27 + #[allow(clippy::undocumented_unsafe_blocks, reason = "TODO")] 28 28 + unsafe impl lock_api::RawRwLock for RawRwLock { 29 29 + const INIT: Self = Self { 30 30 + lock: AtomicUsize::new(0), 31 31 + }; 32 32 + type GuardMarker = GuardSend; 219 33 220 34 fn lock_shared(&self) { 221 35 while !self.try_lock_shared() { ··· 239 53 } 240 54 } 241 55 56 56 + // fn try_lock_shared_weak(&self) -> bool { 57 57 + // self.try_lock_shared() 58 58 + // } 59 59 + 60 60 + unsafe fn unlock_shared(&self) { 61 61 + debug_assert!(self.lock.load(Ordering::Relaxed) & !(WRITER | UPGRADED) > 0); 62 62 + self.lock.fetch_sub(READER, Ordering::Release); 63 63 + } 64 64 + 242 65 fn lock_exclusive(&self) { 243 66 let mut boff = Backoff::default(); 244 67 while !self.try_lock_exclusive_internal(false) { ··· 250 73 self.try_lock_exclusive_internal(true) 251 74 } 252 75 253 253 - fn try_lock_exclusive_internal(&self, strong: bool) -> bool { 254 254 - compare_exchange( 255 255 - &self.lock, 256 256 - 0, 257 257 - WRITER, 258 258 - Ordering::Acquire, 259 259 - Ordering::Relaxed, 260 260 - strong, 261 261 - ) 262 262 - .is_ok() 263 263 - } 264 264 - 265 265 - unsafe fn unlock_shared(&self) { 266 266 - debug_assert!(self.lock.load(Ordering::Relaxed) & !(WRITER | UPGRADED) > 0); 267 267 - self.lock.fetch_sub(READER, Ordering::Release); 268 268 - } 76 76 + // fn try_lock_exclusive_weak(&self) -> bool { 77 77 + // self.try_lock_exclusive_internal(false) 78 78 + // } 269 79 270 80 unsafe fn unlock_exclusive(&self) { 271 81 debug_assert_eq!(self.lock.load(Ordering::Relaxed) & WRITER, WRITER); ··· 274 84 // The UPGRADED bit may be set if an upgradeable lock attempts an upgrade while this lock is held. 275 85 self.lock.fetch_and(!(WRITER | UPGRADED), Ordering::Release); 276 86 } 87 87 + } 277 88 278 278 - fn try_upgrade_internal(&self, strong: bool) -> bool { 279 279 - compare_exchange( 280 280 - &self.lock, 281 281 - UPGRADED, 282 282 - WRITER, 283 283 - Ordering::Acquire, 284 284 - Ordering::Relaxed, 285 285 - strong, 286 286 - ) 287 287 - .is_ok() 89 89 + #[allow(clippy::undocumented_unsafe_blocks, reason = "TODO")] 90 90 + unsafe impl lock_api::RawRwLockUpgrade for RawRwLock { 91 91 + fn lock_upgradable(&self) { 92 92 + todo!() 93 93 + } 94 94 + 95 95 + fn try_lock_upgradable(&self) -> bool { 96 96 + todo!() 288 97 } 98 98 + 99 99 + // fn try_lock_upgradable_weak(&self) -> bool { 100 100 + // todo!() 101 101 + // } 289 102 290 103 unsafe fn unlock_upgradable(&self) { 291 104 debug_assert_eq!( ··· 296 109 } 297 110 298 111 unsafe fn upgrade(&self) { 299 299 - while !self.try_upgrade_internal(false) { 112 112 + while self.try_upgrade_internal(false) { 300 113 #[cfg(loom)] 301 114 crate::loom::thread::yield_now(); 302 115 core::hint::spin_loop(); ··· 307 120 self.try_upgrade_internal(true) 308 121 } 309 122 310 310 - unsafe fn downgrade_upgradable(&self) { 123 123 + // unsafe fn try_upgrade_weak(&self) -> bool { 124 124 + // self.try_upgrade_internal(false) 125 125 + // } 126 126 + } 127 127 + 128 128 + #[allow(clippy::undocumented_unsafe_blocks, reason = "TODO")] 129 129 + unsafe impl lock_api::RawRwLockDowngrade for RawRwLock { 130 130 + unsafe fn downgrade(&self) { 311 131 // Reserve the read guard for ourselves 312 132 self.acquire_reader(); 313 133 314 314 - // Safety: we just acquired the lock 315 315 - unsafe { 316 316 - self.unlock_upgradable(); 317 317 - } 318 318 - } 319 319 - } 134 134 + debug_assert_eq!(self.lock.load(Ordering::Relaxed) & WRITER, WRITER); 320 135 321 321 - impl<T: Default> Default for RwLock<T> { 322 322 - #[inline] 323 323 - fn default() -> RwLock<T> { 324 324 - RwLock::new(Default::default()) 136 136 + // Writer is responsible for clearing both WRITER and UPGRADED bits. 137 137 + // The UPGRADED bit may be set if an upgradeable lock attempts an upgrade while this lock is held. 138 138 + self.lock.fetch_and(!(WRITER | UPGRADED), Ordering::Release); 325 139 } 326 140 } 327 141 328 328 - impl<T> From<T> for RwLock<T> { 329 329 - #[inline] 330 330 - fn from(t: T) -> RwLock<T> { 331 331 - RwLock::new(t) 332 332 - } 333 333 - } 334 334 - 335 335 - impl<T: ?Sized + fmt::Debug> fmt::Debug for RwLock<T> { 336 336 - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 337 337 - let mut d = f.debug_struct("RwLock"); 338 338 - match self.try_read() { 339 339 - Some(guard) => d.field("data", &&*guard), 340 340 - None => { 341 341 - // Additional format_args! here is to remove quotes around <locked> in debug output. 342 342 - d.field("data", &format_args!("<locked>")) 343 343 - } 344 344 - }; 345 345 - d.finish() 346 346 - } 347 347 - } 348 348 - 349 349 - impl<T: ?Sized> !Send for RwLockReadGuard<'_, T> {} 350 350 - #[expect(clippy::undocumented_unsafe_blocks, reason = "")] 351 351 - unsafe impl<T: Sync + ?Sized> Sync for RwLockReadGuard<'_, T> {} 352 352 - 353 353 - impl<'a, T: ?Sized + 'a> RwLockReadGuard<'a, T> { 354 354 - /// Returns a reference to the original reader-writer lock object. 355 355 - pub fn rwlock(s: &Self) -> &'a RwLock<T> { 356 356 - s.rwlock 357 357 - } 358 358 - } 359 359 - 360 360 - impl<'a, T: ?Sized + 'a> Deref for RwLockReadGuard<'a, T> { 361 361 - type Target = T; 362 362 - #[inline] 363 363 - fn deref(&self) -> &T { 364 364 - self.rwlock.data.with(|data| { 365 365 - // Safety: RwLockReadGuard always holds a read lock, so obtaining an immutable reference 366 366 - // is safe 367 367 - unsafe { &*data } 368 368 - }) 369 369 - } 370 370 - } 142 142 + #[allow(clippy::undocumented_unsafe_blocks, reason = "TODO")] 143 143 + unsafe impl lock_api::RawRwLockUpgradeDowngrade for RawRwLock { 144 144 + unsafe fn downgrade_upgradable(&self) { 145 145 + // Reserve the read guard for ourselves 146 146 + self.acquire_reader(); 371 147 372 372 - impl<'a, T: ?Sized + 'a> Drop for RwLockReadGuard<'a, T> { 373 373 - #[inline] 374 374 - fn drop(&mut self) { 375 375 - // Safety: An RwLockReadGuard always holds a shared lock. 148 148 + // Safety: we just acquired the lock 376 149 unsafe { 377 377 - self.rwlock.unlock_shared(); 150 150 + <Self as lock_api::RawRwLockUpgrade>::unlock_upgradable(self); 378 151 } 379 152 } 380 380 - } 381 153 382 382 - impl<'a, T: fmt::Debug + ?Sized + 'a> fmt::Debug for RwLockReadGuard<'a, T> { 383 383 - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 384 384 - fmt::Debug::fmt(&**self, f) 385 385 - } 386 386 - } 387 387 - 388 388 - impl<'a, T: fmt::Display + ?Sized + 'a> fmt::Display for RwLockReadGuard<'a, T> { 389 389 - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 390 390 - (**self).fmt(f) 391 391 - } 392 392 - } 393 393 - 394 394 - impl<T: ?Sized> !Send for RwLockWriteGuard<'_, T> {} 395 395 - #[expect(clippy::undocumented_unsafe_blocks, reason = "")] 396 396 - unsafe impl<T: Sync + ?Sized> Sync for RwLockWriteGuard<'_, T> {} 397 397 - 398 398 - impl<'a, T: ?Sized + 'a> RwLockWriteGuard<'a, T> { 399 399 - /// Returns a reference to the original reader-writer lock object. 400 400 - pub fn rwlock(s: &Self) -> &'a RwLock<T> { 401 401 - s.rwlock 402 402 - } 403 403 - 404 404 - /// Atomically downgrades a write lock into a read lock without allowing any 405 405 - /// writers to take exclusive access of the lock in the meantime. 406 406 - /// 407 407 - /// Note that if there are any writers currently waiting to take the lock 408 408 - /// then other readers may not be able to acquire the lock even if it was 409 409 - /// downgraded. 410 410 - pub fn downgrade(s: Self) -> RwLockReadGuard<'a, T> { 411 411 - let rwlock = s.rwlock; 412 412 - 413 413 - // Reserve the read guard for ourselves 414 414 - rwlock.acquire_reader(); 415 415 - 416 416 - debug_assert_eq!(rwlock.lock.load(Ordering::Relaxed) & WRITER, WRITER); 417 417 - 418 418 - // Writer is responsible for clearing both WRITER and UPGRADED bits. 419 419 - // The UPGRADED bit may be set if an upgradeable lock attempts an upgrade while this lock is held. 420 420 - rwlock 421 421 - .lock 422 422 - .fetch_and(!(WRITER | UPGRADED), Ordering::Release); 423 423 - 424 424 - mem::forget(s); 425 425 - RwLockReadGuard { 426 426 - _data: rwlock.data.with_mut(|data| { 427 427 - // Safety: RwLockWriteGuard holds mutable access to the data 428 428 - unsafe { NonNull::new_unchecked(data) } 429 429 - }), 430 430 - rwlock, 431 431 - } 432 432 - } 433 433 - 434 434 - /// Atomically downgrades a write lock into an upgradable read lock without allowing any 435 435 - /// writers to take exclusive access of the lock in the meantime. 436 436 - /// 437 437 - /// Note that if there are any writers currently waiting to take the lock 438 438 - /// then other readers may not be able to acquire the lock even if it was 439 439 - /// downgraded. 440 440 - pub fn downgrade_to_upgradable(s: Self) -> RwLockUpgradableReadGuard<'a, T> { 441 441 - let rwlock = s.rwlock; 442 442 - 154 154 + unsafe fn downgrade_to_upgradable(&self) { 443 155 debug_assert_eq!( 444 444 - rwlock.lock.load(Ordering::Acquire) & (WRITER | UPGRADED), 156 156 + self.lock.load(Ordering::Acquire) & (WRITER | UPGRADED), 445 157 WRITER 446 158 ); 447 159 448 160 // Reserve the read guard for ourselves 449 449 - rwlock.lock.store(UPGRADED, Ordering::Release); 161 161 + self.lock.store(UPGRADED, Ordering::Release); 450 162 451 451 - debug_assert_eq!(rwlock.lock.load(Ordering::Relaxed) & WRITER, WRITER); 163 163 + debug_assert_eq!(self.lock.load(Ordering::Relaxed) & WRITER, WRITER); 452 164 453 165 // Writer is responsible for clearing both WRITER and UPGRADED bits. 454 166 // The UPGRADED bit may be set if an upgradeable lock attempts an upgrade while this lock is held. 455 455 - rwlock 456 456 - .lock 457 457 - .fetch_and(!(WRITER | UPGRADED), Ordering::Release); 458 458 - 459 459 - mem::forget(s); 460 460 - RwLockUpgradableReadGuard { 461 461 - data: rwlock.data.with_mut(|data| { 462 462 - // Safety: RwLockWriteGuard holds mutable access to the data 463 463 - unsafe { NonNull::new_unchecked(data) } 464 464 - }), 465 465 - rwlock, 466 466 - } 167 167 + self.lock.fetch_and(!(WRITER | UPGRADED), Ordering::Release); 467 168 } 468 169 } 469 170 470 470 - impl<'a, T: ?Sized + 'a> Deref for RwLockWriteGuard<'a, T> { 471 471 - type Target = T; 472 472 - #[inline] 473 473 - fn deref(&self) -> &T { 474 474 - self.rwlock.data.with(|data| { 475 475 - // Safety: RwLockWriteGuard always holds a read lock, so obtaining an immutable reference 476 476 - // is safe 477 477 - unsafe { &*data } 478 478 - }) 479 479 - } 480 480 - } 171 171 + impl RawRwLock { 172 172 + fn acquire_reader(&self) -> usize { 173 173 + // An arbitrary cap that allows us to catch overflows long before they happen 174 174 + const MAX_READERS: usize = usize::MAX / READER / 2; 481 175 482 482 - impl<'a, T: ?Sized + 'a> DerefMut for RwLockWriteGuard<'a, T> { 483 483 - #[inline] 484 484 - fn deref_mut(&mut self) -> &mut T { 485 485 - self.rwlock.data.with_mut(|data| { 486 486 - // Safety: RwLockWriteGuard always holds a write lock, so obtaining a mutable reference 487 487 - // is safe 488 488 - unsafe { &mut *data } 489 489 - }) 490 490 - } 491 491 - } 176 176 + let value = self.lock.fetch_add(READER, Ordering::Acquire); 492 177 493 493 - impl<'a, T: ?Sized + 'a> Drop for RwLockWriteGuard<'a, T> { 494 494 - #[inline] 495 495 - fn drop(&mut self) { 496 496 - // Safety: An RwLockWriteGuard always holds an exclusive lock. 497 497 - unsafe { 498 498 - self.rwlock.unlock_exclusive(); 499 499 - } 500 500 - } 501 501 - } 502 502 - 503 503 - impl<'a, T: fmt::Debug + ?Sized + 'a> fmt::Debug for RwLockWriteGuard<'a, T> { 504 504 - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 505 505 - fmt::Debug::fmt(&**self, f) 506 506 - } 507 507 - } 508 508 - 509 509 - impl<'a, T: fmt::Display + ?Sized + 'a> fmt::Display for RwLockWriteGuard<'a, T> { 510 510 - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 511 511 - (**self).fmt(f) 512 512 - } 513 513 - } 514 514 - 515 515 - impl<T: ?Sized> !Send for RwLockUpgradableReadGuard<'_, T> {} 516 516 - #[expect(clippy::undocumented_unsafe_blocks, reason = "")] 517 517 - unsafe impl<'a, T: ?Sized + Sync + 'a> Sync for RwLockUpgradableReadGuard<'a, T> {} 518 518 - 519 519 - impl<'a, T: ?Sized + 'a> RwLockUpgradableReadGuard<'a, T> { 520 520 - /// Returns a reference to the original reader-writer lock object. 521 521 - pub fn rwlock(s: &Self) -> &'a RwLock<T> { 522 522 - s.rwlock 523 523 - } 524 524 - 525 525 - /// Atomically upgrades an upgradable read lock lock into an exclusive write lock, 526 526 - /// blocking the current thread until it can be acquired. 527 527 - pub fn upgrade(s: Self) -> RwLockWriteGuard<'a, T> { 528 528 - // Safety: An RwLockUpgradableReadGuard always holds an upgradable lock. 529 529 - unsafe { 530 530 - s.rwlock.upgrade(); 531 531 - } 532 532 - let rwlock = s.rwlock; 533 533 - mem::forget(s); 534 534 - RwLockWriteGuard { rwlock } 535 535 - } 536 536 - 537 537 - /// Tries to atomically upgrade an upgradable read lock into an exclusive write lock. 538 538 - /// 539 539 - /// # Errors 540 540 - /// 541 541 - /// If the access could not be granted at this time, then the current guard is returned. 542 542 - pub fn try_upgrade(s: Self) -> Result<RwLockWriteGuard<'a, T>, Self> { 543 543 - // Safety: An RwLockUpgradableReadGuard always holds an upgradable lock. 544 544 - if unsafe { s.rwlock.try_upgrade() } { 545 545 - let rwlock = s.rwlock; 546 546 - mem::forget(s); 547 547 - Ok(RwLockWriteGuard { rwlock }) 178 178 + if value > MAX_READERS * READER { 179 179 + self.lock.fetch_sub(READER, Ordering::Relaxed); 180 180 + panic!("Too many lock readers, cannot safely proceed"); 548 181 } else { 549 549 - Err(s) 550 550 - } 551 551 - } 552 552 - 553 553 - /// Atomically downgrades an upgradable read lock lock into a shared read lock 554 554 - /// without allowing any writers to take exclusive access of the lock in the 555 555 - /// meantime. 556 556 - /// 557 557 - /// Note that if there are any writers currently waiting to take the lock 558 558 - /// then other readers may not be able to acquire the lock even if it was 559 559 - /// downgraded. 560 560 - pub fn downgrade(s: Self) -> RwLockReadGuard<'a, T> { 561 561 - let data = s.data; 562 562 - // Safety: An RwLockUpgradableReadGuard always holds an upgradable lock. 563 563 - unsafe { 564 564 - s.rwlock.downgrade_upgradable(); 565 565 - } 566 566 - let rwlock = s.rwlock; 567 567 - mem::forget(s); 568 568 - RwLockReadGuard { 569 569 - _data: data, 570 570 - rwlock, 182 182 + value 571 183 } 572 184 } 573 573 - } 574 185 575 575 - impl<'a, T: ?Sized + 'a> Deref for RwLockUpgradableReadGuard<'a, T> { 576 576 - type Target = T; 577 577 - #[inline] 578 578 - fn deref(&self) -> &T { 579 579 - self.rwlock.data.with(|data| { 580 580 - // Safety: RwLockUpgradableReadGuard always holds a read lock, so obtaining an immutable reference 581 581 - // is safe 582 582 - unsafe { &*data } 583 583 - }) 186 186 + fn try_lock_exclusive_internal(&self, strong: bool) -> bool { 187 187 + compare_exchange( 188 188 + &self.lock, 189 189 + 0, 190 190 + WRITER, 191 191 + Ordering::Acquire, 192 192 + Ordering::Relaxed, 193 193 + strong, 194 194 + ) 195 195 + .is_ok() 584 196 } 585 585 - } 586 197 587 587 - impl<'a, T: ?Sized + 'a> Drop for RwLockUpgradableReadGuard<'a, T> { 588 588 - #[inline] 589 589 - fn drop(&mut self) { 590 590 - // Safety: An RwLockUpgradableReadGuard always holds an upgradable lock. 591 591 - unsafe { 592 592 - self.rwlock.unlock_upgradable(); 593 593 - } 594 594 - } 595 595 - } 596 596 - 597 597 - impl<'a, T: fmt::Debug + ?Sized + 'a> fmt::Debug for RwLockUpgradableReadGuard<'a, T> { 598 598 - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 599 599 - fmt::Debug::fmt(&**self, f) 600 600 - } 601 601 - } 602 602 - 603 603 - impl<'a, T: fmt::Display + ?Sized + 'a> fmt::Display for RwLockUpgradableReadGuard<'a, T> { 604 604 - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 605 605 - (**self).fmt(f) 198 198 + fn try_upgrade_internal(&self, strong: bool) -> bool { 199 199 + compare_exchange( 200 200 + &self.lock, 201 201 + UPGRADED, 202 202 + WRITER, 203 203 + Ordering::Acquire, 204 204 + Ordering::Relaxed, 205 205 + strong, 206 206 + ) 207 207 + .is_ok() 606 208 } 607 209 } 608 210 ··· 629 231 use std::sync::mpsc::channel; 630 232 631 233 use super::*; 632 632 - use crate::loom::{Arc, thread}; 234 234 + use crate::loom::sync::Arc; 235 235 + use crate::loom::thread; 633 236 634 237 #[derive(Eq, PartialEq, Debug)] 635 238 struct NonCopy(i32);