refactor(kmem): fix FrameAllocator::allocate/allocate_zeroed (#626)

+44 -1

Cargo.lock

··· 582 582 checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" 583 583 584 584 [[package]] 585 + name = "env_filter" 586 + version = "0.1.4" 587 + source = "registry+https://github.com/rust-lang/crates.io-index" 588 + checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" 589 + dependencies = [ 590 + "log", 591 + ] 592 + 593 + [[package]] 594 + name = "env_logger" 595 + version = "0.11.8" 596 + source = "registry+https://github.com/rust-lang/crates.io-index" 597 + checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" 598 + dependencies = [ 599 + "anstream", 600 + "anstyle", 601 + "env_filter", 602 + "log", 603 + ] 604 + 605 + [[package]] 585 606 name = "equivalent" 586 607 version = "1.0.2" 587 608 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1098 1119 dependencies = [ 1099 1120 "arrayvec", 1100 1121 "cpu-local", 1101 - "fallible-iterator", 1102 1122 "kmem", 1103 1123 "lock_api", 1104 1124 "log", ··· 1108 1128 "proptest-derive", 1109 1129 "riscv", 1110 1130 "spin", 1131 + "test-log", 1111 1132 ] 1112 1133 1113 1134 [[package]] ··· 1971 1992 "once_cell", 1972 1993 "rustix", 1973 1994 "windows-sys", 1995 + ] 1996 + 1997 + [[package]] 1998 + name = "test-log" 1999 + version = "0.2.19" 2000 + source = "registry+https://github.com/rust-lang/crates.io-index" 2001 + checksum = "37d53ac171c92a39e4769491c4b4dde7022c60042254b5fc044ae409d34a24d4" 2002 + dependencies = [ 2003 + "env_logger", 2004 + "test-log-macros", 2005 + "tracing-subscriber 0.3.20", 2006 + ] 2007 + 2008 + [[package]] 2009 + name = "test-log-macros" 2010 + version = "0.2.19" 2011 + source = "registry+https://github.com/rust-lang/crates.io-index" 2012 + checksum = "be35209fd0781c5401458ab66e4f98accf63553e8fae7425503e92fdd319783b" 2013 + dependencies = [ 2014 + "proc-macro2", 2015 + "quote", 2016 + "syn", 1974 2017 ] 1975 2018 1976 2019 [[package]]

+1 -1

libs/kmem/Cargo.toml

··· 15 15 16 16 # 3rd-party dependencies 17 17 mycelium-bitfield.workspace = true 18 - fallible-iterator.workspace = true 19 18 log.workspace = true 20 19 lock_api.workspace = true 21 20 22 21 [dev-dependencies] 23 22 kmem = { workspace = true, features = ["test_utils"] } 24 23 parking_lot = "0.12.5" 24 + test-log = "0.2.19" 25 25 26 26 [features] 27 27 test_utils = ["cpu-local", "spin", "proptest", "proptest-derive"]

+16

libs/kmem/proptest-regressions/bootstrap/frame_allocator.txt

··· 1 + # Seeds for failure cases proptest has generated in the past. It is 2 + # automatically read and these particular cases re-run before any 3 + # novel cases are generated. 4 + # 5 + # It is recommended to check this file in to source control so that 6 + # everyone who runs the test benefits from these saved cases. 7 + cc fb3867557e4125c9af34128bc2d2b86f878f5a1a40db8db2380262cbb51571cd # shrinks to region_sizes = [] 8 + cc a5d30f8381e0d4dcab3cf645f308d1998dc2dd7802e0e47a985e74babf2fc543 # shrinks to region_sizes = [] 9 + cc 7849807d5ff072a5df694489ecd12668ee5f7aa50a281a94fec2b58142cb1566 # shrinks to region_sizes = [] 10 + cc 589e03024578c3fc3583201c824e75d7947a96c14153416c4c260415a69c0a28 # shrinks to region_sizes = [4096], alignment_pot = 13 11 + cc ff2dcd7413a420740b44940b64bda8956711aab2bcd68c9771b8e06361f7b9fc # shrinks to region_sizes = [4096, 4096], alignment_pot = 13 12 + cc 4acc0a82561e26e57b9c37aee7ba9f3ee03df2a399a444f96e8d402ffc4ebe90 # shrinks to region_sizes = [4096], alignment_pot = 13 13 + cc cffd8e7f33b1804f3ca8808dc85381851df16ade0431000006b49f31f69dfd18 # shrinks to region_sizes = [4294971392, 4294975488, 4096], alignment_pot = 32 14 + cc 791f5e19cd5c7aff666d2823dbca4be2f1b4d41b52cd7d5847ea5c2be795e18d # shrinks to region_sizes = [4096], alignment_pot = 13 15 + cc 423b7f20d0692df9842b5e45736a8d9b7732362eddea413f2955d20a8cce549a # shrinks to region_sizes = [1073741824], alignment_pot = 30 16 + cc 230d6d336cc87025a79f56c49cd98699f02940b892ec15c0f586660dd0a191ce # shrinks to region_sizes = [1073741824], size = 536870913, alignment_pot = 29

+18 -20

libs/kmem/src/address_space.rs

··· 2 2 use core::convert::Infallible; 3 3 use core::ops::Range; 4 4 5 - use fallible_iterator::FallibleIterator; 6 - 7 5 use crate::arch::{Arch, PageTableEntry, PageTableLevel}; 8 6 use crate::bootstrap::{Bootstrap, BootstrapAllocator}; 9 7 use crate::flush::Flush; ··· 50 48 /// # Errors 51 49 /// 52 50 /// Returns `Err(AllocError)` when allocating the root page table fails. 53 - pub fn new_bootstrap<R: lock_api::RawMutex>( 51 + pub fn new_bootstrap<R: lock_api::RawMutex, const MAX_REGIONS: usize>( 54 52 arch: A, 55 53 future_physmap: PhysMap, 56 - frame_allocator: &BootstrapAllocator<R>, 54 + frame_allocator: &BootstrapAllocator<R, MAX_REGIONS>, 57 55 flush: &mut Flush, 58 56 ) -> Result<Bootstrap<Self>, AllocError> { 59 57 let address_space = Self::new(arch, PhysMap::new_bootstrap(), frame_allocator, flush)?; ··· 134 132 unreachable!() 135 133 } 136 134 137 - /// Maps the virtual address range `virt` to *possibly discontiguous* chunk(s) of physical memory 135 + /// Maps the virtual address range `virt` to *possibly discontiguous* block(s) of physical memory 138 136 /// `phys` with the specified memory attributes. 139 137 /// 140 138 /// If this returns `Ok`, the mapping is added to the address space. ··· 151 149 /// 152 150 /// 1. The entire range `virt` must be unmapped. 153 151 /// 2. `virt` must be aligned to at least the smallest architecture block size. 154 - /// 3. `phys` chunks must be aligned to at least the smallest architecture block size. 155 - /// 4. `phys` chunks must in-total be at least as large as `virt`. 152 + /// 3. `phys` blocks must be aligned to at least the smallest architecture block size. 153 + /// 4. `phys` blocks must in-total be at least as large as `virt`. 156 154 /// 157 155 /// # Errors 158 156 /// ··· 161 159 pub unsafe fn map( 162 160 &mut self, 163 161 mut virt: Range<VirtualAddress>, 164 - mut phys: impl FallibleIterator<Item = Range<PhysicalAddress>, Error = AllocError>, 162 + phys: impl ExactSizeIterator<Item = Range<PhysicalAddress>>, 165 163 attributes: MemoryAttributes, 166 164 frame_allocator: impl FrameAllocator, 167 165 flush: &mut Flush, 168 166 ) -> Result<(), AllocError> { 169 - while let Some(chunk_phys) = phys.next()? { 167 + for block_phys in phys { 170 168 debug_assert!(!virt.is_empty()); 171 169 172 170 // Safety: ensured by caller 173 171 unsafe { 174 172 self.map_contiguous( 175 - Range::from_start_len(virt.start, chunk_phys.len()), 176 - chunk_phys.start, 173 + Range::from_start_len(virt.start, block_phys.len()), 174 + block_phys.start, 177 175 attributes, 178 176 frame_allocator.by_ref(), 179 177 flush, 180 178 )?; 181 179 } 182 180 183 - virt.start = virt.start.add(chunk_phys.len()); 181 + virt.start = virt.start.add(block_phys.len()); 184 182 } 185 183 186 184 Ok(()) ··· 275 273 Ok(()) 276 274 } 277 275 278 - /// Remaps the virtual address range `virt` to new *possibly discontiguous* chunk(s) of physical 276 + /// Remaps the virtual address range `virt` to new *possibly discontiguous* block(s) of physical 279 277 /// memory `phys`. The old physical memory region is not freed. 280 278 /// 281 279 /// Note that this method **does not** establish any ordering between address space modification ··· 289 287 /// 290 288 /// 1. The entire range `virt` must be mapped. 291 289 /// 2. `virt` must be aligned to at least the smallest architecture block size. 292 - /// 3. `phys` chunks must be aligned to `at least the smallest architecture block size. 293 - /// 4. `phys` chunks must in-total be at least as large as `virt`. 290 + /// 3. `phys` blocks must be aligned to `at least the smallest architecture block size. 291 + /// 4. `phys` blocks must in-total be at least as large as `virt`. 294 292 /// 295 293 /// # Errors 296 294 /// ··· 299 297 pub unsafe fn remap( 300 298 &mut self, 301 299 mut virt: Range<VirtualAddress>, 302 - mut phys: impl FallibleIterator<Item = Range<PhysicalAddress>, Error = AllocError>, 300 + phys: impl ExactSizeIterator<Item = Range<PhysicalAddress>>, 303 301 flush: &mut Flush, 304 302 ) -> Result<(), AllocError> { 305 - while let Some(chunk_phys) = phys.next()? { 303 + for block_phys in phys { 306 304 debug_assert!(!virt.is_empty()); 307 305 308 306 // Safety: ensured by caller 309 307 unsafe { 310 308 self.remap_contiguous( 311 - Range::from_start_len(virt.start, chunk_phys.len()), 312 - chunk_phys.start, 309 + Range::from_start_len(virt.start, block_phys.len()), 310 + block_phys.start, 313 311 flush, 314 312 ); 315 313 } 316 314 317 - virt.start = virt.start.add(chunk_phys.len()); 315 + virt.start = virt.start.add(block_phys.len()); 318 316 } 319 317 320 318 Ok(())

+18 -3

libs/kmem/src/arch/mod.rs

··· 2 2 3 3 use core::alloc::Layout; 4 4 use core::ops::Range; 5 - use core::{fmt, ptr}; 5 + use core::{fmt, ptr, slice}; 6 6 7 7 use crate::{MemoryAttributes, PhysicalAddress, VirtualAddress}; 8 8 ··· 138 138 unsafe { address.as_mut_ptr().cast::<T>().write(value) } 139 139 } 140 140 141 + /// Reads `count` bytes of memory starting at `address`. This leaves the memory in `address` unchanged. 142 + /// 143 + /// # Safety 144 + /// 145 + /// This method largely inherits the safety requirements of [`slice::from_raw_parts`], namely 146 + /// behavior is undefined if any of the following conditions are violated: 147 + /// 148 + /// - `address` must be non-null and [valid] for reads of `count` bytes. 149 + /// - `address` must be properly aligned. 150 + /// - The memory referenced by the returned slice must not be mutated for the duration its lifetime. 151 + unsafe fn read_bytes(&self, address: VirtualAddress, count: usize) -> &[u8] { 152 + // Safety: ensured by the caller. 153 + unsafe { slice::from_raw_parts(address.as_ptr(), count) } 154 + } 155 + 141 156 /// Sets `count` bytes of memory starting at `address` to `val`. 142 157 /// 143 158 /// `write_bytes` behaves like C's [`memset`]. ··· 149 164 /// This method largely inherits the safety requirements of [`ptr::write_bytes`], namely 150 165 /// behavior is undefined if any of the following conditions are violated: 151 166 /// 152 - /// - `address` must be [valid] for writes of `count` bytes. 167 + /// - `address` must be non-null and [valid] for writes of `count` bytes. 153 168 /// - `address` must be properly aligned. 154 169 /// 155 - /// Note that even if the effectively copied sizeis 0, the pointer must be properly aligned. 170 + /// Note that even if the effectively copied size is 0, the pointer must be properly aligned. 156 171 /// 157 172 /// [valid]: 158 173 /// [`ptr::write_bytes`]: core::ptr::write_bytes()

+3 -3

libs/kmem/src/bootstrap.rs

··· 2 2 3 3 use core::ops::Range; 4 4 5 - pub use frame_allocator::{BootstrapAllocator, DEFAULT_MAX_REGIONS, FreeRegions, UsedRegions}; 5 + pub use frame_allocator::{BootstrapAllocator, DEFAULT_MAX_REGIONS}; 6 6 7 7 use crate::arch::Arch; 8 8 use crate::flush::Flush; ··· 34 34 /// 35 35 /// Returning `Err` indicates the mapping cannot be established and the address space remains 36 36 /// unaltered. 37 - pub fn map_physical_memory<R: lock_api::RawMutex>( 37 + pub fn map_physical_memory<R: lock_api::RawMutex, const MAX_REGIONS: usize>( 38 38 &mut self, 39 - frame_allocator: &BootstrapAllocator<R>, 39 + frame_allocator: &BootstrapAllocator<R, MAX_REGIONS>, 40 40 flush: &mut Flush, 41 41 ) -> Result<(), AllocError> { 42 42 let attrs = MemoryAttributes::new()

+616 -134

libs/kmem/src/bootstrap/frame_allocator.rs

··· 1 1 use core::alloc::Layout; 2 - use core::fmt; 2 + use core::cmp::Ordering; 3 3 use core::num::NonZeroUsize; 4 4 use core::ops::Range; 5 + use core::{cmp, fmt, iter}; 5 6 6 7 use arrayvec::ArrayVec; 7 8 use lock_api::Mutex; ··· 22 23 R: lock_api::RawMutex, 23 24 { 24 25 inner: Mutex<R, BootstrapAllocatorInner<MAX_REGIONS>>, 25 - // we make a "snapshot" of the translation granule size during construction so that the allocator 26 - // itself doesn't need to be generic over `Arch`. 27 - frame_size: usize, 26 + min_align: NonZeroUsize, 28 27 } 29 28 30 29 #[derive(Debug)] 31 30 struct BootstrapAllocatorInner<const MAX_REGIONS: usize> { 32 - /// The discontiguous regions of "regular" physical memory that we can use for allocation. 33 - regions: ArrayVec<Range<PhysicalAddress>, MAX_REGIONS>, 34 - /// offset from the top of memory regions 35 - offset: usize, 31 + arenas: ArrayVec<Arena, MAX_REGIONS>, 32 + current_arena_hint: usize, 36 33 } 37 34 38 35 impl<R, const MAX_REGIONS: usize> fmt::Debug for BootstrapAllocator<R, MAX_REGIONS> ··· 41 38 { 42 39 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 43 40 f.debug_struct("BootstrapAllocator") 44 - .field("regions", &self.inner.lock()) 45 - .field("frame_size", &self.frame_size) 41 + .field("inner", &self.inner.lock()) 42 + .field("min_align", &self.min_align) 46 43 .finish() 47 44 } 48 45 } ··· 70 67 } 71 68 } 72 69 73 - regions 74 - .iter_mut() 75 - .for_each(|region| *region = region.clone().align_in(A::GRANULE_SIZE)); 70 + let mut largest_region_idx = 0; 71 + let mut largest_region_size = 0; 72 + let arenas: ArrayVec<_, MAX_REGIONS> = regions 73 + .into_iter() 74 + .enumerate() 75 + .map(|(i, region)| { 76 + let region = region.align_in(A::GRANULE_SIZE); 77 + 78 + if region.len() > largest_region_size { 79 + largest_region_size = region.len(); 80 + largest_region_idx = i; 81 + } 82 + 83 + Arena { 84 + // we allocate from the top of each region downward 85 + ptr: region.end, 86 + region, 87 + } 88 + }) 89 + .collect(); 76 90 77 91 Self { 78 - inner: Mutex::new(BootstrapAllocatorInner { regions, offset: 0 }), 79 - frame_size: A::GRANULE_SIZE, 92 + inner: Mutex::new(BootstrapAllocatorInner { 93 + arenas, 94 + current_arena_hint: largest_region_idx, 95 + }), 96 + min_align: NonZeroUsize::new(A::GRANULE_SIZE).unwrap(), 80 97 } 81 98 } 82 99 83 100 /// Returns the array of "regular" physical memory regions managed by this allocator. 101 + #[inline] 84 102 pub fn regions(&self) -> ArrayVec<Range<PhysicalAddress>, MAX_REGIONS> { 85 - self.inner.lock().regions.clone() 103 + self.inner 104 + .lock() 105 + .arenas 106 + .iter() 107 + .map(|arena| arena.region.clone()) 108 + .collect() 86 109 } 87 110 88 - /// Returns an iterator over the "free" (not allocated) portions of physical memory regions 89 - /// managed by this allocator. 90 - pub fn free_regions(&self) -> impl Iterator<Item = Range<PhysicalAddress>> { 91 - let inner = self.inner.lock(); 92 - 93 - FreeRegions { 94 - offset: inner.offset, 95 - inner: inner.regions.clone().into_iter(), 96 - } 111 + /// Returns the remaining capacity (free bytes) of this allocator in bytes 112 + #[inline] 113 + pub fn capacity(&self) -> usize { 114 + self.capacities().into_iter().sum() 97 115 } 98 116 99 - /// Returns an iterator over the "used" (allocated) portions of physical memory regions 100 - /// managed by this allocator. 101 - pub fn used_regions(&self) -> impl Iterator<Item = Range<PhysicalAddress>> { 102 - let inner = self.inner.lock(); 103 - 104 - UsedRegions { 105 - offset: inner.offset, 106 - inner: inner.regions.clone().into_iter(), 107 - } 117 + /// Returns the remaining capacity of each physical memory region. 118 + #[inline] 119 + pub fn capacities(&self) -> ArrayVec<usize, MAX_REGIONS> { 120 + self.inner 121 + .lock() 122 + .arenas 123 + .iter() 124 + .map(|region| region.capacity()) 125 + .collect() 108 126 } 109 127 110 128 /// Returns the number of allocated bytes. 129 + #[inline] 111 130 pub fn usage(&self) -> usize { 112 - self.inner.lock().offset 131 + self.usages().into_iter().sum() 132 + } 133 + 134 + /// Returns the number of allocated bytes of each physical memory region. 135 + #[inline] 136 + pub fn usages(&self) -> ArrayVec<usize, MAX_REGIONS> { 137 + self.inner 138 + .lock() 139 + .arenas 140 + .iter() 141 + .map(|region| region.usage()) 142 + .collect() 113 143 } 114 144 } 115 145 ··· 119 149 where 120 150 R: lock_api::RawMutex, 121 151 { 152 + fn allocate( 153 + &self, 154 + layout: Layout, 155 + ) -> Result<impl ExactSizeIterator<Item = Range<PhysicalAddress>>, AllocError> { 156 + let mut inner = self.inner.lock(); 157 + 158 + if let Some(p) = inner.allocate_contiguous_fast(self.min_align, layout) { 159 + let block = Range::from_start_len(p, layout.size()); 160 + 161 + Ok(Blocks::One(iter::once(block))) 162 + } else { 163 + let blocks = inner 164 + .allocate_slow(self.min_align, layout) 165 + .ok_or(AllocError)?; 166 + 167 + Ok(Blocks::Multiple(blocks.into_iter())) 168 + } 169 + } 170 + 171 + #[inline(always)] 122 172 fn allocate_contiguous(&self, layout: Layout) -> Result<PhysicalAddress, AllocError> { 123 - assert_eq!( 124 - layout.align(), 125 - self.frame_size, 126 - "BootstrapAllocator only supports page-aligned allocations" 127 - ); 173 + let mut inner = self.inner.lock(); 128 174 129 - self.inner.lock().allocate(layout) 175 + if let Some(p) = inner.allocate_contiguous_fast(self.min_align, layout) { 176 + Ok(p) 177 + } else { 178 + inner 179 + .allocate_contiguous_slow(self.min_align, layout) 180 + .ok_or(AllocError) 181 + } 130 182 } 131 183 132 184 unsafe fn deallocate(&self, _block: PhysicalAddress, _layout: Layout) { 133 - unimplemented!("BootstrapAllocator does not support deallocation") 185 + unimplemented!("BootstrapAllocator does not support deallocation"); 134 186 } 187 + } 135 188 136 - fn size_hint(&self) -> (NonZeroUsize, Option<NonZeroUsize>) { 137 - (NonZeroUsize::new(self.frame_size).unwrap(), None) 189 + impl<const MAX_REGIONS: usize> BootstrapAllocatorInner<MAX_REGIONS> { 190 + /// Fast-path for allocation from the "current" arena. Most modern machines have a single large 191 + /// physical memory region. During creation, we determine the largest physical memory region 192 + /// and designate it as the "current" arena. 193 + /// 194 + /// This means this fast-path can fulfill the vast majority of requests with a single allocation 195 + /// from this "main" arena. 196 + #[inline(always)] 197 + fn allocate_contiguous_fast( 198 + &mut self, 199 + min_align: NonZeroUsize, 200 + layout: Layout, 201 + ) -> Option<PhysicalAddress> { 202 + self.arenas[self.current_arena_hint].allocate(min_align, layout) 138 203 } 139 - } 204 + 205 + /// Cold-path when we have exhausted the capacity of the current region and need to consider 206 + /// other regions. 207 + #[inline(never)] 208 + #[cold] 209 + fn allocate_contiguous_slow( 210 + &mut self, 211 + min_align: NonZeroUsize, 212 + layout: Layout, 213 + ) -> Option<PhysicalAddress> { 214 + let current_arena_hint = self.current_arena_hint; 215 + 216 + // NB: we know this method is called when the "current region" (as indicated by the current region hint) 217 + // is exhausted. We therefore begin our search at the next region (offset 1..) wrapping around 218 + // at the end to double-check previous regions (they might have capacity still). But we still 219 + // don't double-check the "current region" as we know it cant fit `layout`. 220 + for offset in 1..self.arenas.len() { 221 + let i = (current_arena_hint + offset) % self.arenas.len(); 222 + 223 + // only attempt to allocate if the region has any capacity 224 + if self.arenas[i].has_capacity() 225 + && let Some(block) = self.arenas[i].allocate(min_align, layout) 226 + { 227 + self.current_arena_hint = i; 228 + 229 + return Some(block); 230 + } 231 + } 232 + 233 + None 234 + } 235 + 236 + /// Cold-path for discontiguous allocations when we have exhausted the capacity of the current 237 + /// region and need to consider other regions. 238 + #[inline(never)] 239 + #[cold] 240 + fn allocate_slow( 241 + &mut self, 242 + min_align: NonZeroUsize, 243 + layout: Layout, 244 + ) -> Option<ArrayVec<Range<PhysicalAddress>, MAX_REGIONS>> { 245 + let mut blocks: ArrayVec<_, MAX_REGIONS> = ArrayVec::new(); 246 + let mut remaining_bytes = layout.size(); 247 + let current_arena_hint = self.current_arena_hint; 140 248 141 - impl<const MAX_REGIONS: usize> BootstrapAllocatorInner<MAX_REGIONS> { 142 - fn allocate(&mut self, layout: Layout) -> Result<PhysicalAddress, AllocError> { 143 - let requested_size = layout.pad_to_align().size(); 144 - let mut offset = self.offset; 249 + for offset in 0..self.arenas.len() { 250 + let i = (current_arena_hint + offset) % self.arenas.len(); 145 251 146 - for region in self.regions.iter().rev() { 147 - let region_size = region.len(); 252 + // only attempt to allocate if the region has any capacity 253 + if self.arenas[i].has_capacity() { 254 + // attempt to allocate as big of a block as we can 255 + let requested_size = cmp::min(remaining_bytes, self.arenas[i].capacity()); 256 + let layout = Layout::from_size_align(requested_size, layout.align()).unwrap(); 148 257 149 - // only consider regions that we haven't already exhausted 150 - if offset < region_size { 151 - // Allocating a contiguous range has different requirements than "regular" allocation 152 - // contiguous are rare and often happen in very critical paths where e.g. virtual 153 - // memory is not available yet. So we rather waste some memory than outright crash. 154 - if region_size - offset < requested_size { 155 - log::warn!( 156 - "Skipped memory region {region:?} since it was too small to fulfill request for {requested_size} bytes. Wasted {} bytes in the process...", 157 - region_size - offset 158 - ); 258 + if let Some(block) = self.arenas[i].allocate(min_align, layout) { 259 + self.current_arena_hint = i; 260 + remaining_bytes -= requested_size; 159 261 160 - self.offset += region_size - offset; 161 - offset = 0; 162 - continue; 262 + blocks.push(Range::from_start_len(block, requested_size)); 163 263 } 264 + } 164 265 165 - let frame = region.end.sub(offset + requested_size); 166 - self.offset += requested_size; 167 - return Ok(frame); 266 + // if - through this loop iteration - we fully allocated the required memory 267 + // return the blocks 268 + if remaining_bytes == 0 { 269 + return Some(blocks); 168 270 } 271 + } 169 272 170 - offset -= region_size; 273 + log::trace!( 274 + "failed to allocate layout {layout:?} capacities={:#?} usages={:#?}", 275 + self.arenas 276 + .iter() 277 + .map(|arena| arena.capacity()) 278 + .collect::<ArrayVec<_, MAX_REGIONS>>(), 279 + self.arenas 280 + .iter() 281 + .map(|arena| arena.usage()) 282 + .collect::<ArrayVec<_, MAX_REGIONS>>() 283 + ); 284 + 285 + // if we've gone through all regions without fully allocating the required memory we cant 286 + // satisfy this allocation request. 287 + // we might have allocated some blocks though, so lets go and clean them up now 288 + 289 + for block in blocks { 290 + let region = self 291 + .arenas 292 + .iter_mut() 293 + .find(|region| region.region.overlaps(&block)) 294 + .unwrap_or_else(|| { 295 + panic!("block {block:?} must belong to an arena. this is a bug!") 296 + }); 297 + 298 + region.deallocate_if_last(block); 171 299 } 172 300 173 - Err(AllocError) 301 + None 174 302 } 175 303 } 176 304 177 - pub struct FreeRegions<const MAX_REGIONS: usize> { 178 - offset: usize, 179 - inner: arrayvec::IntoIter<Range<PhysicalAddress>, { MAX_REGIONS }>, 305 + /// Manages a contiguous region of physical memory. 306 + struct Arena { 307 + region: Range<PhysicalAddress>, 308 + ptr: PhysicalAddress, 309 + } 310 + 311 + impl fmt::Debug for Arena { 312 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 313 + f.debug_struct("Arena") 314 + .field("region", &self.region) 315 + .field("ptr", &self.ptr) 316 + .field("<capacity>", &(self.free(), self.capacity())) 317 + .field("<usage>", &(self.used(), self.usage())) 318 + .finish() 319 + } 180 320 } 181 321 182 - impl<const MAX_REGIONS: usize> Iterator for FreeRegions<MAX_REGIONS> { 183 - type Item = Range<PhysicalAddress>; 322 + impl Arena { 323 + /// Returns the number of bytes left to allocate 324 + #[inline] 325 + pub fn capacity(&self) -> usize { 326 + self.ptr.offset_from_unsigned(self.region.start) 327 + } 328 + 329 + /// Returns true if this arena has any capacity left 330 + #[inline] 331 + fn has_capacity(&self) -> bool { 332 + self.capacity() > 0 333 + } 334 + 335 + /// Returns the number of bytes allocated from this arena 336 + #[inline] 337 + pub fn usage(&self) -> usize { 338 + self.region.end.offset_from_unsigned(self.ptr) 339 + } 340 + 341 + /// Returns the used (allocated) slice of the physical memory region managed by this arena 342 + #[inline] 343 + pub fn used(&self) -> Range<PhysicalAddress> { 344 + self.ptr..self.region.end 345 + } 346 + 347 + /// Returns the free (not allocated) slice of the physical memory region managed by this arena 348 + #[inline] 349 + pub fn free(&self) -> Range<PhysicalAddress> { 350 + self.region.start..self.ptr 351 + } 352 + 353 + /// Deallocates a given memory block IF it is the last block that was allocated from this arena. 354 + /// 355 + /// # Panics 356 + /// 357 + /// Panics if the block was not the last allocated block. 358 + fn deallocate_if_last(&mut self, block: Range<PhysicalAddress>) { 359 + if self.ptr == block.start { 360 + self.ptr = block.end; 361 + } else { 362 + panic!("can only free last allocated block"); 363 + } 364 + } 365 + 366 + /// Attempt to allocate enough memory to satisfy the size and alignment requirements of `layout`. 367 + #[inline] 368 + fn allocate(&mut self, min_align: NonZeroUsize, layout: Layout) -> Option<PhysicalAddress> { 369 + debug_assert!( 370 + self.region.start <= self.ptr && self.ptr <= self.region.end, 371 + "bump pointer {:?} should in region range {}..={}", 372 + self.ptr, 373 + self.region.start, 374 + self.region.end 375 + ); 376 + debug_assert!( 377 + self.ptr.is_aligned_to(min_align.get()), 378 + "bump pointer {:?} should be aligned to the minimum alignment of {min_align:#x}", 379 + self.ptr 380 + ); 184 381 185 - fn next(&mut self) -> Option<Self::Item> { 186 - loop { 187 - let mut region = self.inner.next()?; 188 - // keep advancing past already fully used memory regions 189 - let region_size = region.len(); 382 + let aligned_ptr = match layout.align().cmp(&min_align.get()) { 383 + Ordering::Less => { 384 + // the requested alignment is smaller than our minimum alignment 385 + // we need to round up the requested size to our minimum alignment 386 + let aligned_size = round_up_to(layout.size(), min_align.get())?; 190 387 191 - if self.offset >= region_size { 192 - self.offset -= region_size; 193 - continue; 194 - } else if self.offset > 0 { 195 - region.end = region.end.sub(self.offset); 196 - self.offset = 0; 388 + if self.capacity() < aligned_size { 389 + return None; 390 + } 391 + 392 + self.ptr.wrapping_sub(aligned_size) 197 393 } 394 + Ordering::Equal => { 395 + // the requested alignment is equal to our minimum alignment 198 396 199 - return Some(region); 397 + // round up the layout size to be a multiple of the layout's alignment 398 + // Safety: `Layout` guarantees that rounding the size up to its align cannot overflow 399 + let aligned_size = unsafe { round_up_to_unchecked(layout.size(), layout.align()) }; 400 + 401 + if self.capacity() < aligned_size { 402 + return None; 403 + } 404 + 405 + self.ptr.wrapping_sub(aligned_size) 406 + } 407 + Ordering::Greater => { 408 + // the requested alignment is greater than our minimum alignment 409 + 410 + // round up the layout size to be a multiple of the layout's alignment. 411 + // Safety: `Layout` guarantees that rounding the size up to its align cannot overflow 412 + let aligned_size = unsafe { round_up_to_unchecked(layout.size(), layout.align()) }; 413 + 414 + let aligned_ptr = self.ptr.align_down(layout.align()); 415 + 416 + // NB: we're not using .capacity() here because we actually care about the capacity 417 + // that's left *after* aligning the bump pointer down 418 + let capacity = aligned_ptr.offset_from_unsigned(self.region.start); 419 + 420 + if aligned_ptr < self.region.start || capacity < aligned_size { 421 + return None; 422 + } 423 + 424 + aligned_ptr.wrapping_sub(aligned_size) 425 + } 426 + }; 427 + 428 + debug_assert!( 429 + aligned_ptr.is_aligned_to(layout.align()), 430 + "pointer {aligned_ptr:?} should be aligned to layout alignment of {}", 431 + layout.align() 432 + ); 433 + debug_assert!( 434 + aligned_ptr.is_aligned_to(min_align.get()), 435 + "pointer {aligned_ptr:?} should be aligned to minimum alignment of {min_align}", 436 + ); 437 + debug_assert!( 438 + self.region.start <= aligned_ptr && aligned_ptr <= self.ptr, 439 + "pointer {aligned_ptr:?} should be in range {:?}..={:?}", 440 + self.region.start, 441 + self.ptr 442 + ); 443 + 444 + self.ptr = aligned_ptr; 445 + 446 + Some(aligned_ptr) 447 + } 448 + } 449 + 450 + #[inline] 451 + const fn round_up_to(n: usize, divisor: usize) -> Option<usize> { 452 + debug_assert!(divisor > 0); 453 + debug_assert!(divisor.is_power_of_two()); 454 + 455 + match n.checked_add(divisor - 1) { 456 + Some(x) => Some(x & !(divisor - 1)), 457 + None => None, 458 + } 459 + } 460 + 461 + /// Like `round_up_to` but turns overflow into undefined behavior rather than 462 + /// returning `None`. 463 + /// 464 + /// # Safety: 465 + /// 466 + /// This results in undefined behavior when `n + (divisor - 1) > usize::MAX` or n + (divisor - 1) < usize::MIN` i.e. when round_up_to would return None. 467 + #[inline] 468 + unsafe fn round_up_to_unchecked(n: usize, divisor: usize) -> usize { 469 + match round_up_to(n, divisor) { 470 + Some(x) => x, 471 + None => { 472 + debug_assert!(false, "round_up_to_unchecked failed"); 473 + 474 + // Safety: ensured by caller 475 + unsafe { core::hint::unreachable_unchecked() } 200 476 } 201 477 } 202 478 } 203 479 204 - pub struct UsedRegions<const MAX_REGIONS: usize> { 205 - offset: usize, 206 - inner: arrayvec::IntoIter<Range<PhysicalAddress>, { MAX_REGIONS }>, 480 + enum Blocks<const MAX: usize> { 481 + One(iter::Once<Range<PhysicalAddress>>), 482 + Multiple(arrayvec::IntoIter<Range<PhysicalAddress>, MAX>), 207 483 } 208 484 209 - impl<const MAX_REGIONS: usize> Iterator for UsedRegions<MAX_REGIONS> { 485 + impl<const MAX: usize> Iterator for Blocks<MAX> { 210 486 type Item = Range<PhysicalAddress>; 211 487 212 488 fn next(&mut self) -> Option<Self::Item> { 213 - let mut region = self.inner.next()?; 489 + match self { 490 + Blocks::One(iter) => iter.next(), 491 + Blocks::Multiple(iter) => iter.next(), 492 + } 493 + } 214 494 215 - if self.offset >= region.len() { 216 - Some(region) 217 - } else if self.offset > 0 { 218 - region.start = region.end.sub(self.offset); 219 - self.offset = 0; 220 - 221 - Some(region) 222 - } else { 223 - None 495 + fn size_hint(&self) -> (usize, Option<usize>) { 496 + match self { 497 + Blocks::One(iter) => iter.size_hint(), 498 + Blocks::Multiple(iter) => iter.size_hint(), 224 499 } 225 500 } 226 501 } 227 502 503 + impl<const MAX: usize> ExactSizeIterator for Blocks<MAX> {} 504 + 228 505 #[cfg(test)] 229 506 mod tests { 507 + use core::alloc::Layout; 508 + 509 + use crate::address_range::AddressRangeExt; 230 510 use crate::arch::Arch; 231 511 use crate::bootstrap::BootstrapAllocator; 232 512 use crate::frame_allocator::FrameAllocator; 233 513 use crate::test_utils::{EmulateArch, MachineBuilder}; 234 - use crate::{PhysMap, archtest}; 514 + use crate::{GIB, PhysMap, PhysicalAddress, archtest}; 515 + 516 + fn assert_zeroed(frame: PhysicalAddress, bytes: usize, physmap: &PhysMap, arch: &impl Arch) { 517 + let frame = unsafe { arch.read_bytes(physmap.phys_to_virt(frame), bytes) }; 518 + 519 + assert!(frame.iter().all(|byte| *byte == 0)); 520 + } 235 521 236 522 archtest! { 237 523 // Assert that the BootstrapAllocator can allocate frames 238 - #[test] 239 - fn allocate_contiguous<A: Arch>() { 524 + #[test_log::test] 525 + fn allocate_contiguous_smoke<A: Arch>() { 240 526 let (machine, _) = MachineBuilder::<A, parking_lot::RawMutex, _>::new() 241 - .with_memory_regions([0x3000]) 527 + .with_memory_regions([0x2000, 0x1000]) 242 528 .finish(); 243 529 244 530 let frame_allocator: BootstrapAllocator<parking_lot::RawMutex> = ··· 247 533 // Based on the memory of the machine we set up above, we expect the allocator to 248 534 // yield 3 pages. 249 535 250 - frame_allocator 536 + let frame = frame_allocator 251 537 .allocate_contiguous(A::GRANULE_LAYOUT) 252 538 .unwrap(); 539 + assert!(frame.is_aligned_to(A::GRANULE_SIZE)); 253 540 254 - frame_allocator 541 + let frame = frame_allocator 255 542 .allocate_contiguous(A::GRANULE_LAYOUT) 256 543 .unwrap(); 544 + assert!(frame.is_aligned_to(A::GRANULE_SIZE)); 257 545 258 - frame_allocator 546 + let frame = frame_allocator 259 547 .allocate_contiguous(A::GRANULE_LAYOUT) 260 548 .unwrap(); 549 + assert!(frame.is_aligned_to(A::GRANULE_SIZE)); 261 550 262 551 // assert that we're out of memory 263 552 frame_allocator ··· 267 556 268 557 // Assert that the BootstrapAllocator can allocate zeroed frames in 269 558 // bootstrap (bare, before paging is enabled) mode. 270 - #[test] 271 - fn allocate_contiguous_zeroed_bare<A: Arch>() { 559 + #[test_log::test] 560 + fn allocate_contiguous_zeroed_smoke<A: Arch>() { 272 561 let (machine, _) = MachineBuilder::<A, parking_lot::RawMutex, _>::new() 273 - .with_memory_regions([0x3000]) 562 + .with_memory_regions([0x2000, 0x1000]) 274 563 .finish(); 275 - 276 - println!("{machine:?}"); 277 564 278 565 let frame_allocator: BootstrapAllocator<parking_lot::RawMutex> = 279 566 BootstrapAllocator::new::<A>(machine.memory_regions()); ··· 285 572 // Based on the memory of the machine we set up above, we expect the allocator to 286 573 // yield 3 pages. 287 574 288 - frame_allocator 575 + let frame = frame_allocator 289 576 .allocate_contiguous_zeroed(A::GRANULE_LAYOUT, &physmap, &arch) 290 577 .unwrap(); 578 + assert!(frame.is_aligned_to(A::GRANULE_SIZE)); 579 + assert_zeroed(frame, A::GRANULE_SIZE, &physmap, &arch); 291 580 292 - frame_allocator 581 + let frame = frame_allocator 293 582 .allocate_contiguous_zeroed(A::GRANULE_LAYOUT, &physmap, &arch) 294 583 .unwrap(); 584 + assert!(frame.is_aligned_to(A::GRANULE_SIZE)); 585 + assert_zeroed(frame, A::GRANULE_SIZE, &physmap, &arch); 295 586 296 - frame_allocator 587 + let frame = frame_allocator 297 588 .allocate_contiguous_zeroed(A::GRANULE_LAYOUT, &physmap, &arch) 298 589 .unwrap(); 590 + assert!(frame.is_aligned_to(A::GRANULE_SIZE)); 591 + assert_zeroed(frame, A::GRANULE_SIZE, &physmap, &arch); 299 592 300 593 // assert that we're out of memory 301 594 frame_allocator ··· 303 596 .unwrap_err(); 304 597 } 305 598 306 - // Assert that the BootstrapAllocator can allocate frames 307 - #[test] 308 - fn allocate_contiguous_multiple_regions<A: Arch>() { 599 + #[test_log::test] 600 + fn allocate_smoke<A: Arch>() { 601 + let (machine, _) = MachineBuilder::<A, parking_lot::RawMutex, _>::new() 602 + .with_memory_regions([0x3000, 0x1000]) 603 + .finish(); 604 + 605 + let frame_allocator: BootstrapAllocator<parking_lot::RawMutex> = 606 + BootstrapAllocator::new::<A>(machine.memory_regions()); 607 + 608 + let blocks: Vec<_> = frame_allocator 609 + .allocate(Layout::from_size_align(0x4000, A::GRANULE_SIZE).unwrap()) 610 + .unwrap() 611 + .collect(); 612 + 613 + // assert the total size is what we expect 614 + let allocated_size: usize = blocks.iter().map(|block| block.len()).sum(); 615 + assert!(allocated_size >= 0x4000); 616 + 617 + // assert each block is aligned correctly 618 + for block in blocks.iter() { 619 + assert!(block.start.is_aligned_to(A::GRANULE_SIZE)); 620 + } 621 + } 622 + 623 + #[test_log::test] 624 + fn allocate_zeroed_smoke<A: Arch>() { 625 + let (machine, _) = MachineBuilder::<A, parking_lot::RawMutex, _>::new() 626 + .with_memory_regions([0x3000, 0x1000]) 627 + .finish(); 628 + 629 + let arch = EmulateArch::new(machine.clone()); 630 + 631 + let physmap = PhysMap::new_bootstrap(); 632 + 633 + let frame_allocator: BootstrapAllocator<parking_lot::RawMutex> = 634 + BootstrapAllocator::new::<A>(machine.memory_regions()); 635 + 636 + let blocks: Vec<_> = frame_allocator 637 + .allocate_zeroed(Layout::from_size_align(0x4000, A::GRANULE_SIZE).unwrap(), &physmap, &arch) 638 + .unwrap() 639 + .collect(); 640 + 641 + // assert the total size is what we expect 642 + let allocated_size: usize = blocks.iter().map(|block| block.len()).sum(); 643 + assert!(allocated_size >= 0x4000); 644 + 645 + // assert each block is aligned correctly 646 + for block in blocks.iter() { 647 + assert!(block.start.is_aligned_to(A::GRANULE_SIZE)); 648 + 649 + assert_zeroed(block.start, block.len(), &physmap, &arch); 650 + } 651 + } 652 + 653 + #[test_log::test] 654 + fn allocate_contiguous_small_alignment<A: Arch>() { 309 655 let (machine, _) = MachineBuilder::<A, parking_lot::RawMutex, _>::new() 310 - .with_memory_regions([0x3000]) 656 + .with_memory_regions([0x4000, 0x1000]) 311 657 .finish(); 312 658 313 659 let frame_allocator: BootstrapAllocator<parking_lot::RawMutex> = 314 660 BootstrapAllocator::new::<A>(machine.memory_regions()); 315 661 316 - // Based on the memory of the machine we set up above, we expect the allocator to 317 - // yield 3 pages. 662 + let frame = frame_allocator.allocate_contiguous(Layout::from_size_align(A::GRANULE_SIZE, 1).unwrap()).unwrap(); 318 663 319 - frame_allocator 320 - .allocate_contiguous(A::GRANULE_LAYOUT) 321 - .unwrap(); 664 + assert!(frame.is_aligned_to(1)); 665 + assert!(frame.is_aligned_to(A::GRANULE_SIZE)); 666 + } 322 667 323 - frame_allocator 324 - .allocate_contiguous(A::GRANULE_LAYOUT) 325 - .unwrap(); 668 + #[test_log::test] 669 + fn allocate_small_alignment<A: Arch>() { 670 + let (machine, _) = MachineBuilder::<A, parking_lot::RawMutex, _>::new() 671 + .with_memory_regions([0x4000, 0x1000]) 672 + .finish(); 326 673 327 - frame_allocator 328 - .allocate_contiguous(A::GRANULE_LAYOUT) 329 - .unwrap(); 674 + let frame_allocator: BootstrapAllocator<parking_lot::RawMutex> = 675 + BootstrapAllocator::new::<A>(machine.memory_regions()); 330 676 331 - // assert that we're out of memory 332 - frame_allocator 333 - .allocate_contiguous(A::GRANULE_LAYOUT) 334 - .unwrap_err(); 677 + let blocks = frame_allocator.allocate(Layout::from_size_align(A::GRANULE_SIZE, 1).unwrap()).unwrap(); 678 + 679 + for block in blocks { 680 + assert!(block.start.is_aligned_to(1)); 681 + assert!(block.start.is_aligned_to(A::GRANULE_SIZE)); 682 + } 683 + } 684 + 685 + #[test_log::test] 686 + fn allocate_contiguous_large_alignment<A: Arch>() { 687 + let (machine, _) = MachineBuilder::<A, parking_lot::RawMutex, _>::new() 688 + .with_memory_regions([2*GIB, 0x1000]) 689 + .finish(); 690 + 691 + let frame_allocator: BootstrapAllocator<parking_lot::RawMutex> = 692 + BootstrapAllocator::new::<A>(machine.memory_regions()); 693 + 694 + let frame = frame_allocator.allocate_contiguous(Layout::from_size_align(A::GRANULE_SIZE, 1*GIB).unwrap()).unwrap(); 695 + 696 + assert!(frame.is_aligned_to(1*GIB)); 697 + } 698 + 699 + #[test_log::test] 700 + fn allocate_large_alignment<A: Arch>() { 701 + let (machine, _) = MachineBuilder::<A, parking_lot::RawMutex, _>::new() 702 + .with_memory_regions([2*GIB, 0x1000]) 703 + .finish(); 704 + 705 + let frame_allocator: BootstrapAllocator<parking_lot::RawMutex> = 706 + BootstrapAllocator::new::<A>(machine.memory_regions()); 707 + 708 + let blocks = frame_allocator.allocate(Layout::from_size_align(A::GRANULE_SIZE, 1*GIB).unwrap()).unwrap(); 709 + 710 + for block in blocks { 711 + assert!(block.start.is_aligned_to(1*GIB)); 712 + } 335 713 } 336 714 } 337 715 } 716 + 717 + #[cfg(test)] 718 + mod proptests { 719 + use core::alloc::Layout; 720 + 721 + use proptest::prelude::*; 722 + 723 + use crate::address_range::AddressRangeExt; 724 + use crate::arch::Arch; 725 + use crate::bootstrap::{BootstrapAllocator, DEFAULT_MAX_REGIONS}; 726 + use crate::frame_allocator::FrameAllocator; 727 + use crate::test_utils::MachineBuilder; 728 + use crate::test_utils::proptest::region_sizes; 729 + use crate::{GIB, KIB, for_every_arch}; 730 + 731 + for_every_arch!(A => { 732 + proptest! { 733 + #[test_log::test] 734 + fn allocate(region_sizes in region_sizes(1..DEFAULT_MAX_REGIONS, 4*KIB, 16*GIB)) { 735 + let (machine, _) = MachineBuilder::<A, parking_lot::RawMutex, _>::new() 736 + .with_memory_regions(region_sizes.clone()) 737 + .finish(); 738 + 739 + let frame_allocator: BootstrapAllocator<parking_lot::RawMutex> = 740 + BootstrapAllocator::new::<A>(machine.memory_regions()); 741 + 742 + let total_size = region_sizes.iter().sum(); 743 + 744 + let res = frame_allocator 745 + .allocate(Layout::from_size_align(total_size, A::GRANULE_SIZE).unwrap()); 746 + prop_assert!(res.is_ok(), "failed to allocate {} bytes with alignment {}. capacities left {:?}", total_size, A::GRANULE_SIZE, frame_allocator.capacities()); 747 + let blocks = res.unwrap(); 748 + 749 + let blocks: Vec<_> = blocks.collect(); 750 + 751 + // assert the total size is what we expect 752 + let allocated_size: usize = blocks.iter().map(|block| block.len()).sum(); 753 + prop_assert!(allocated_size >= total_size); 754 + 755 + // assert each block is aligned correctly 756 + for block in blocks.iter() { 757 + prop_assert!(block.start.is_aligned_to(A::GRANULE_SIZE)); 758 + } 759 + } 760 + 761 + #[test_log::test] 762 + fn allocate_contiguous(region_sizes in region_sizes(1..DEFAULT_MAX_REGIONS, 1*GIB, 16*GIB)) { 763 + let (machine, _) = MachineBuilder::<A, parking_lot::RawMutex, _>::new() 764 + .with_memory_regions(region_sizes.clone()) 765 + .finish(); 766 + 767 + let frame_allocator: BootstrapAllocator<parking_lot::RawMutex> = 768 + BootstrapAllocator::new::<A>(machine.memory_regions()); 769 + 770 + let total_size = region_sizes.iter().sum(); 771 + 772 + for _ in (0..total_size).step_by(1*GIB) { 773 + let res = frame_allocator.allocate_contiguous(Layout::from_size_align(1*GIB, A::GRANULE_SIZE).unwrap()); 774 + prop_assert!(res.is_ok()); 775 + let base = res.unwrap(); 776 + prop_assert!(base.is_aligned_to(A::GRANULE_SIZE)); 777 + } 778 + } 779 + 780 + #[test_log::test] 781 + fn allocate_contiguous_alignments(region_sizes in region_sizes(1..DEFAULT_MAX_REGIONS, 1*GIB, 16*GIB), alignment_pot in 1..30) { 782 + let (machine, _) = MachineBuilder::<A, parking_lot::RawMutex, _>::new() 783 + .with_memory_regions(region_sizes.clone()) 784 + .finish(); 785 + 786 + let frame_allocator: BootstrapAllocator<parking_lot::RawMutex> = 787 + BootstrapAllocator::new::<A>(machine.memory_regions()); 788 + 789 + let alignment = 1usize << alignment_pot; 790 + 791 + let res = frame_allocator.allocate_contiguous(Layout::from_size_align(A::GRANULE_SIZE, alignment).unwrap()); 792 + prop_assert!(res.is_ok()); 793 + let base = res.unwrap(); 794 + 795 + prop_assert!(base.is_aligned_to(alignment)); 796 + } 797 + 798 + #[test_log::test] 799 + fn allocate_alignments(region_sizes in region_sizes(1..DEFAULT_MAX_REGIONS, 1*GIB, 16*GIB), alignment_pot in 1..30) { 800 + let (machine, _) = MachineBuilder::<A, parking_lot::RawMutex, _>::new() 801 + .with_memory_regions(region_sizes.clone()) 802 + .finish(); 803 + 804 + let frame_allocator: BootstrapAllocator<parking_lot::RawMutex> = 805 + BootstrapAllocator::new::<A>(machine.memory_regions()); 806 + 807 + let alignment = 1usize << alignment_pot; 808 + 809 + let res = frame_allocator.allocate(Layout::from_size_align(A::GRANULE_SIZE, alignment).unwrap()); 810 + prop_assert!(res.is_ok()); 811 + let blocks = res.unwrap(); 812 + 813 + for block in blocks { 814 + prop_assert!(block.start.is_aligned_to(alignment)); 815 + } 816 + } 817 + } 818 + }); 819 + }

+68 -94

libs/kmem/src/frame_allocator.rs

··· 1 1 use core::alloc::Layout; 2 - use core::num::NonZeroUsize; 2 + use core::fmt; 3 3 use core::ops::Range; 4 - use core::{cmp, fmt}; 5 - 6 - use fallible_iterator::FallibleIterator; 7 4 8 5 use crate::arch::Arch; 9 6 use crate::physmap::PhysMap; ··· 42 39 /// A memory block which is currently allocated may be passed to any method of the allocator that 43 40 /// accepts such an argument. 44 41 pub unsafe trait FrameAllocator { 45 - fn allocate(&self, layout: Layout) -> FrameIter<'_, Self> 46 - where 47 - Self: Sized, 48 - { 49 - FrameIter { 50 - alloc: self, 51 - remaining: layout.size(), 52 - alignment: layout.align(), 53 - } 54 - } 42 + /// Attempts to allocate physical memory. 43 + /// 44 + /// On success, returns an iterator over the allocated blocks of physical memory. The combined 45 + /// size of all blocks will meet the size required by `Layout` and each block will individually 46 + /// meet the alignment required by `Layout`. 47 + /// 48 + /// The returned blocks may have a larger size than specified by `layout.size()`, and may or may 49 + /// not have its contents initialized. 50 + /// 51 + /// # Errors 52 + /// 53 + /// Returning `Err` indicates that either memory is exhausted or `layout` does not meet 54 + /// allocator's size or alignment constraints. You can check [`Self::max_alignment_hint`] for 55 + /// the largest alignment possibly supported by this allocator. 56 + fn allocate( 57 + &self, 58 + layout: Layout, 59 + ) -> Result<impl ExactSizeIterator<Item = Range<PhysicalAddress>>, AllocError>; 55 60 56 - fn allocate_zeroed<'a, A: Arch>( 61 + /// Attempts to allocate physical memory. 62 + /// 63 + /// On success, returns an iterator over the allocated blocks of physical memory. The combined 64 + /// size of all blocks will meet the size required by `Layout` and each block will individually 65 + /// meet the alignment required by `Layout`. 66 + /// 67 + /// The returned blocks may have a larger size than specified by `layout.size()`. 68 + /// The contents of each block will be initialized to zero. 69 + /// 70 + /// # Errors 71 + /// 72 + /// Returning `Err` indicates that either memory is exhausted or `layout` does not meet 73 + /// allocator's size or alignment constraints. You can check [`Self::max_alignment_hint`] for 74 + /// the largest alignment possibly supported by this allocator. 75 + fn allocate_zeroed( 57 76 &self, 58 77 layout: Layout, 59 - physmap: &'a PhysMap, 60 - arch: &'a A, 61 - ) -> FrameIterZeroed<'_, 'a, Self, A> 62 - where 63 - Self: Sized, 64 - { 65 - FrameIterZeroed { 66 - inner: self.allocate(layout), 67 - physmap, 68 - arch, 69 - } 78 + physmap: &PhysMap, 79 + arch: &impl Arch, 80 + ) -> Result<impl ExactSizeIterator<Item = Range<PhysicalAddress>>, AllocError> { 81 + let blocks = self.allocate(layout)?; 82 + 83 + let blocks = blocks.inspect(|block_phys| { 84 + let block_virt = physmap.phys_to_virt_range(block_phys.clone()); 85 + debug_assert_eq!(block_phys.len(), block_virt.len()); 86 + 87 + // Safety: we just allocated the block 88 + unsafe { 89 + arch.write_bytes(block_virt.start, 0, block_phys.len()); 90 + } 91 + }); 92 + 93 + Ok(blocks) 70 94 } 71 95 72 96 /// Attempts to allocate a contiguous block of physical memory. ··· 89 113 /// On success, returns a [`PhysicalAddress`] meeting the size and alignment guarantees 90 114 /// of `layout`. 91 115 /// 92 - /// The returned block may have a larger size than specified by `layout.size()`, and may or may 93 - /// not have its contents initialized. 116 + /// The returned block may have a larger size than specified by `layout.size()`. 117 + /// The contents of the returned block will be initialized to zero. 94 118 /// 95 119 /// # Errors 96 120 /// ··· 134 158 { 135 159 self 136 160 } 137 - 138 - fn size_hint(&self) -> (NonZeroUsize, Option<NonZeroUsize>); 139 161 } 140 162 141 163 // Safety: we just forward to the inner implementation ··· 143 165 where 144 166 F: FrameAllocator + ?Sized, 145 167 { 168 + fn allocate( 169 + &self, 170 + layout: Layout, 171 + ) -> Result<impl ExactSizeIterator<Item = Range<PhysicalAddress>>, AllocError> { 172 + (**self).allocate(layout) 173 + } 174 + 175 + fn allocate_zeroed( 176 + &self, 177 + layout: Layout, 178 + physmap: &PhysMap, 179 + arch: &impl Arch, 180 + ) -> Result<impl ExactSizeIterator<Item = Range<PhysicalAddress>>, AllocError> { 181 + (**self).allocate_zeroed(layout, physmap, arch) 182 + } 183 + 146 184 fn allocate_contiguous(&self, layout: Layout) -> Result<PhysicalAddress, AllocError> { 147 185 (**self).allocate_contiguous(layout) 148 186 } ··· 159 197 unsafe fn deallocate(&self, block: PhysicalAddress, layout: Layout) { 160 198 // Safety: ensured by caller 161 199 unsafe { (**self).deallocate(block, layout) } 162 - } 163 - 164 - fn size_hint(&self) -> (NonZeroUsize, Option<NonZeroUsize>) { 165 - (**self).size_hint() 166 - } 167 - } 168 - 169 - pub struct FrameIter<'alloc, F: ?Sized> { 170 - alloc: &'alloc F, 171 - remaining: usize, 172 - alignment: usize, 173 - } 174 - 175 - impl<F: FrameAllocator> FallibleIterator for FrameIter<'_, F> { 176 - type Item = Range<PhysicalAddress>; 177 - type Error = AllocError; 178 - 179 - fn next(&mut self) -> Result<Option<Self::Item>, Self::Error> { 180 - let Some(remaining) = NonZeroUsize::new(self.remaining) else { 181 - return Ok(None); 182 - }; 183 - 184 - let (min_size, max_size) = self.alloc.size_hint(); 185 - 186 - let requested_size = cmp::min(remaining, max_size.unwrap_or(NonZeroUsize::MAX)); 187 - let alloc_size = cmp::max(requested_size, min_size); 188 - 189 - log::trace!( 190 - "requested_size={requested_size:?} alloc_size={alloc_size:?} align={:?}", 191 - self.alignment 192 - ); 193 - let layout = Layout::from_size_align(alloc_size.get(), self.alignment).unwrap(); 194 - 195 - let addr = self.alloc.allocate_contiguous(layout)?; 196 - 197 - self.remaining -= requested_size.get(); 198 - 199 - Ok(Some(Range::from_start_len(addr, requested_size.get()))) 200 - } 201 - } 202 - 203 - pub struct FrameIterZeroed<'alloc, 'a, F: ?Sized, A: Arch> { 204 - inner: FrameIter<'alloc, F>, 205 - physmap: &'a PhysMap, 206 - arch: &'a A, 207 - } 208 - 209 - impl<F: FrameAllocator, A: Arch> FallibleIterator for FrameIterZeroed<'_, '_, F, A> { 210 - type Item = Range<PhysicalAddress>; 211 - type Error = AllocError; 212 - 213 - fn next(&mut self) -> Result<Option<Self::Item>, Self::Error> { 214 - let Some(range) = self.inner.next()? else { 215 - return Ok(None); 216 - }; 217 - 218 - let virt = self.physmap.phys_to_virt_range(range.clone()); 219 - 220 - // Safety: we just allocated the frame 221 - unsafe { 222 - self.arch.write_bytes(virt.start, 0, virt.len()); 223 - } 224 - 225 - Ok(Some(range)) 226 200 } 227 201 }

+3 -1

libs/kmem/src/lib.rs

··· 3 3 #![feature(step_trait)] 4 4 #![feature(debug_closure_helpers)] 5 5 #![feature(allocator_api)] 6 + #![feature(alloc_layout_extra)] 7 + extern crate core; 6 8 7 9 mod address; 8 10 mod address_range; ··· 23 25 pub use address_space::HardwareAddressSpace; 24 26 pub use arch::Arch; 25 27 pub use flush::Flush; 26 - pub use frame_allocator::{AllocError, FrameAllocator, FrameIter}; 28 + pub use frame_allocator::{AllocError, FrameAllocator}; 27 29 pub use memory_attributes::{MemoryAttributes, WriteOrExecute}; 28 30 pub use physmap::PhysMap; 29 31

+1

libs/kmem/src/physmap.rs

··· 111 111 ); 112 112 113 113 prop_assert_eq!(map.translation_offset, base.get().wrapping_sub(region_start.get()) as isize); 114 + #[cfg(debug_assertions)] 114 115 prop_assert_eq!( 115 116 map.range, 116 117 Some(base.get() as u128..base.add(region_size).get() as u128)

+24

libs/kmem/src/test_utils.rs

··· 8 8 pub use memory::Memory; 9 9 10 10 #[macro_export] 11 + macro_rules! for_every_arch { 12 + ($arch:ident => {$($body:item)*}) => { 13 + mod riscv64_sv39 { 14 + use super::*; 15 + type $arch = $crate::arch::riscv64::Riscv64Sv39; 16 + 17 + $($body)* 18 + } 19 + mod riscv64_sv48 { 20 + use super::*; 21 + type $arch = $crate::arch::riscv64::Riscv64Sv48; 22 + 23 + $($body)* 24 + } 25 + mod riscv64_sv57 { 26 + use super::*; 27 + type $arch = $crate::arch::riscv64::Riscv64Sv57; 28 + 29 + $($body)* 30 + } 31 + }; 32 + } 33 + 34 + #[macro_export] 11 35 macro_rules! archtest { 12 36 ($($(#[$meta:meta])* fn $test_name:ident$(<$ge:ident: $gen_ty:tt>)*() $body:block)*) => { 13 37 mod riscv64_sv39 {

+14

libs/kmem/src/test_utils/arch.rs

··· 92 92 } 93 93 } 94 94 95 + unsafe fn read_bytes(&self, address: VirtualAddress, count: usize) -> &[u8] { 96 + // NB: if there is no active page table on this CPU, we are in "bare" translation mode. 97 + // In which case we need to use `write_bytes_phys` instead of `write_bytes`, bypassing 98 + // translation checks. 99 + if self.active_table().is_some() { 100 + self.machine.read_bytes(self.asid, address, count) 101 + } else { 102 + // Safety: We checked for the absence of an active translation table, meaning we're in 103 + // "bare" mode and VirtualAddress==PhysicalAddress. 104 + let address = unsafe { mem::transmute::<VirtualAddress, PhysicalAddress>(address) }; 105 + self.machine.read_bytes_phys(address, count) 106 + } 107 + } 108 + 95 109 unsafe fn write_bytes(&self, address: VirtualAddress, value: u8, count: usize) { 96 110 // NB: if there is no active page table on this CPU, we are in "bare" translation mode. 97 111 // In which case we need to use `write_bytes_phys` instead of `write_bytes`, bypassing

+11 -4

libs/kmem/src/test_utils/memory.rs

··· 47 47 self.regions.iter().map(|(end, (start, _, _))| *start..*end) 48 48 } 49 49 50 - fn get_region_containing(&self, address: PhysicalAddress) -> Option<(NonNull<[u8]>, usize)> { 51 - let (_end, (start, region, _)) = self.regions.range(address..).next()?; 50 + fn get_region_containing( 51 + &self, 52 + range: Range<PhysicalAddress>, 53 + ) -> Option<(NonNull<[u8]>, usize)> { 54 + let (_end, (start, region, _)) = self.regions.range(range.start.add(1)..).next()?; 55 + 56 + let offset = range.start.get().checked_sub(start.get())?; 52 57 53 - let offset = address.get().checked_sub(start.get())?; 58 + if offset + range.len() > region.len() { 59 + return None; 60 + } 54 61 55 62 Some((*region, offset)) 56 63 } 57 64 58 65 pub fn region(&self, range: Range<PhysicalAddress>, will_write: bool) -> &mut [u8] { 59 - let Some((mut region, offset)) = self.get_region_containing(range.start) else { 66 + let Some((mut region, offset)) = self.get_region_containing(range.clone()) else { 60 67 let access_ty = if will_write { "write" } else { "read" }; 61 68 62 69 panic!(

+22

libs/kmem/src/test_utils/proptest.rs

··· 32 32 addr.prop_map(move |value| value.align_down(alignment)) 33 33 } 34 34 35 + pub fn region_sizes( 36 + num_regions: Range<usize>, 37 + alignment: usize, 38 + max_region_size: usize, 39 + ) -> impl Strategy<Value = Vec<usize>> { 40 + proptest::collection::vec( 41 + // Size of the region (will be aligned) 42 + alignment..=max_region_size, 43 + num_regions, 44 + ) 45 + .prop_map(move |mut regions| { 46 + regions.iter_mut().for_each(|size| { 47 + let align_minus_one = unsafe { alignment.unchecked_sub(1) }; 48 + 49 + *size = size.wrapping_add(align_minus_one) & 0usize.wrapping_sub(alignment); 50 + 51 + debug_assert_ne!(*size, 0); 52 + }); 53 + regions 54 + }) 55 + } 56 + 35 57 /// Produces a set of *sorted*, *non-overlapping* regions of physical memory aligned to `alignment`. 36 58 /// Most useful for initializing an emulated machine. 37 59 pub fn regions(