refactor: separate memory subsystem into own crate #1

+136 -2

Cargo.lock

reviewed

··· 135 135 "windows-targets", 136 136 ] 137 137 138 138 + [[package]] 139 139 + name = "bit-set" 140 140 + version = "0.8.0" 141 141 + source = "registry+https://github.com/rust-lang/crates.io-index" 142 142 + checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" 143 143 + dependencies = [ 144 144 + "bit-vec", 145 145 + ] 146 146 + 147 147 + [[package]] 148 148 + name = "bit-vec" 149 149 + version = "0.8.0" 150 150 + source = "registry+https://github.com/rust-lang/crates.io-index" 151 151 + checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" 152 152 + 138 153 [[package]] 139 154 name = "bitflags" 140 155 version = "2.9.1" ··· 562 577 source = "registry+https://github.com/rust-lang/crates.io-index" 563 578 checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" 564 579 580 580 + [[package]] 581 581 + name = "errno" 582 582 + version = "0.3.13" 583 583 + source = "registry+https://github.com/rust-lang/crates.io-index" 584 584 + checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" 585 585 + dependencies = [ 586 586 + "libc", 587 587 + "windows-sys 0.59.0", 588 588 + ] 589 589 + 565 590 [[package]] 566 591 name = "escape8259" 567 592 version = "0.5.3" ··· 591 616 "criterion", 592 617 ] 593 618 619 619 + [[package]] 620 620 + name = "fastrand" 621 621 + version = "2.3.0" 622 622 + source = "registry+https://github.com/rust-lang/crates.io-index" 623 623 + checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" 624 624 + 594 625 [[package]] 595 626 name = "fdt" 596 627 version = "0.1.0" ··· 952 983 "cordyceps", 953 984 "cpu-local", 954 985 "criterion", 955 955 - "fastrand", 986 986 + "fastrand 0.1.0", 956 987 "futures", 957 988 "lazy_static", 958 989 "loom", ··· 995 1026 "cranelift-entity", 996 1027 "cranelift-frontend", 997 1028 "fallible-iterator", 998 998 - "fastrand", 1029 1029 + "fastrand 0.1.0", 999 1030 "fdt", 1000 1031 "futures", 1001 1032 "gimli", ··· 1119 1150 "vcpkg", 1120 1151 ] 1121 1152 1153 1153 + [[package]] 1154 1154 + name = "linux-raw-sys" 1155 1155 + version = "0.9.4" 1156 1156 + source = "registry+https://github.com/rust-lang/crates.io-index" 1157 1157 + checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" 1158 1158 + 1122 1159 [[package]] 1123 1160 name = "litemap" 1124 1161 version = "0.8.0" ··· 1195 1232 "regex-automata 0.4.9", 1196 1233 ] 1197 1234 1235 1235 + [[package]] 1236 1236 + name = "mem" 1237 1237 + version = "0.1.0" 1238 1238 + dependencies = [ 1239 1239 + "anyhow", 1240 1240 + "cordyceps", 1241 1241 + "cpu-local", 1242 1242 + "fallible-iterator", 1243 1243 + "lock_api", 1244 1244 + "mycelium-bitfield", 1245 1245 + "pin-project", 1246 1246 + "proptest", 1247 1247 + "rand", 1248 1248 + "rand_chacha", 1249 1249 + "smallvec", 1250 1250 + "wavltree", 1251 1251 + ] 1252 1252 + 1198 1253 [[package]] 1199 1254 name = "memchr" 1200 1255 version = "2.7.5" ··· 1467 1522 "yansi", 1468 1523 ] 1469 1524 1525 1525 + [[package]] 1526 1526 + name = "proptest" 1527 1527 + version = "1.7.0" 1528 1528 + source = "registry+https://github.com/rust-lang/crates.io-index" 1529 1529 + checksum = "6fcdab19deb5195a31cf7726a210015ff1496ba1464fd42cb4f537b8b01b471f" 1530 1530 + dependencies = [ 1531 1531 + "bit-set", 1532 1532 + "bit-vec", 1533 1533 + "bitflags", 1534 1534 + "lazy_static", 1535 1535 + "num-traits", 1536 1536 + "rand", 1537 1537 + "rand_chacha", 1538 1538 + "rand_xorshift", 1539 1539 + "regex-syntax 0.8.5", 1540 1540 + "rusty-fork", 1541 1541 + "tempfile", 1542 1542 + "unarray", 1543 1543 + ] 1544 1544 + 1545 1545 + [[package]] 1546 1546 + name = "quick-error" 1547 1547 + version = "1.2.3" 1548 1548 + source = "registry+https://github.com/rust-lang/crates.io-index" 1549 1549 + checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" 1550 1550 + 1470 1551 [[package]] 1471 1552 name = "quote" 1472 1553 version = "1.0.40" ··· 1511 1592 "getrandom", 1512 1593 ] 1513 1594 1595 1595 + [[package]] 1596 1596 + name = "rand_xorshift" 1597 1597 + version = "0.4.0" 1598 1598 + source = "registry+https://github.com/rust-lang/crates.io-index" 1599 1599 + checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" 1600 1600 + dependencies = [ 1601 1601 + "rand_core", 1602 1602 + ] 1603 1603 + 1514 1604 [[package]] 1515 1605 name = "rayon" 1516 1606 version = "1.10.0" ··· 1619 1709 "semver", 1620 1710 ] 1621 1711 1712 1712 + [[package]] 1713 1713 + name = "rustix" 1714 1714 + version = "1.0.8" 1715 1715 + source = "registry+https://github.com/rust-lang/crates.io-index" 1716 1716 + checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" 1717 1717 + dependencies = [ 1718 1718 + "bitflags", 1719 1719 + "errno", 1720 1720 + "libc", 1721 1721 + "linux-raw-sys", 1722 1722 + "windows-sys 0.59.0", 1723 1723 + ] 1724 1724 + 1622 1725 [[package]] 1623 1726 name = "rustversion" 1624 1727 version = "1.0.21" 1625 1728 source = "registry+https://github.com/rust-lang/crates.io-index" 1626 1729 checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" 1627 1730 1731 1731 + [[package]] 1732 1732 + name = "rusty-fork" 1733 1733 + version = "0.3.0" 1734 1734 + source = "registry+https://github.com/rust-lang/crates.io-index" 1735 1735 + checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" 1736 1736 + dependencies = [ 1737 1737 + "fnv", 1738 1738 + "quick-error", 1739 1739 + "tempfile", 1740 1740 + "wait-timeout", 1741 1741 + ] 1742 1742 + 1628 1743 [[package]] 1629 1744 name = "ryu" 1630 1745 version = "1.0.20" ··· 1788 1903 source = "registry+https://github.com/rust-lang/crates.io-index" 1789 1904 checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" 1790 1905 1906 1906 + [[package]] 1907 1907 + name = "tempfile" 1908 1908 + version = "3.20.0" 1909 1909 + source = "registry+https://github.com/rust-lang/crates.io-index" 1910 1910 + checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" 1911 1911 + dependencies = [ 1912 1912 + "fastrand 2.3.0", 1913 1913 + "getrandom", 1914 1914 + "once_cell", 1915 1915 + "rustix", 1916 1916 + "windows-sys 0.59.0", 1917 1917 + ] 1918 1918 + 1791 1919 [[package]] 1792 1920 name = "thiserror" 1793 1921 version = "2.0.12" ··· 2093 2221 "spin", 2094 2222 ] 2095 2223 2224 2224 + [[package]] 2225 2225 + name = "unarray" 2226 2226 + version = "0.1.4" 2227 2227 + source = "registry+https://github.com/rust-lang/crates.io-index" 2228 2228 + checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" 2229 2229 + 2096 2230 [[package]] 2097 2231 name = "unicode-ident" 2098 2232 version = "1.0.18"

+27

libs/mem/Cargo.toml

reviewed

··· 1 1 + [package] 2 2 + name = "mem" 3 3 + version.workspace = true 4 4 + edition.workspace = true 5 5 + authors.workspace = true 6 6 + license.workspace = true 7 7 + 8 8 + [dependencies] 9 9 + cpu-local.workspace = true 10 10 + 11 11 + # 3rd-party dependencies 12 12 + mycelium-bitfield.workspace = true 13 13 + anyhow.workspace = true 14 14 + cordyceps.workspace = true 15 15 + pin-project.workspace = true 16 16 + lock_api.workspace = true 17 17 + fallible-iterator.workspace = true 18 18 + smallvec.workspace = true 19 19 + wavltree.workspace = true 20 20 + rand_chacha.workspace = true 21 21 + rand.workspace = true 22 22 + 23 23 + [dev-dependencies] 24 24 + proptest = "1.7.0" 25 25 + 26 26 + [lints] 27 27 + workspace = true

+13

libs/mem/proptest-regressions/frame.txt

reviewed

··· 1 1 + # Seeds for failure cases proptest has generated in the past. It is 2 2 + # automatically read and these particular cases re-run before any 3 3 + # novel cases are generated. 4 4 + # 5 5 + # It is recommended to check this file in to source control so that 6 6 + # everyone who runs the test benefits from these saved cases. 7 7 + cc 4cf994999dd04e4312e6dd0f9601044b488e1eda3d9c18cdfd57ac4a3e1b00fc # shrinks to num_frames = 0, area_start = 0, alloc_frames = 1 8 8 + cc 3a702a85b8b8ece9062ec02861bb17665fa95817c7b65a2897b2a7db347db322 # shrinks to num_frames = 292, area_start = 0, alloc_frames = 257 9 9 + cc 3065cda233769bdf9b16f3f134e65dcfe170c9a9462cfb013139b9203a43c6c7 # shrinks to num_frames = 512, area_start = 4096, alloc_frames = 257 10 10 + cc d333ce22c6888222b53fa6d21bd2c29aece2aaf1266c7251b2deb86f679221c5 # shrinks to num_frames = 2357, area_start = 3814267094354915328, alloc_frames = 354 11 11 + cc 14f06bd08feb57c49cd25113a630c65e48383d6666178b7b3c157099b40d6286 # shrinks to num_frames = 1421, area_start = 12923327278880337920, alloc_frames = 257 12 12 + cc 007d0fba2f9391c80693c16b411362c67d3be3995856f30e7352aa40e70bb7cc # shrinks to num_frames = 82, area_start = 5938167848445603840, alloc_frames = 20 13 13 + cc 88599b677f8f36a1f4cc363c75d296624989cbefa59b120d7195e209a1a8e897 # shrinks to num_frames = 741, area_start = 9374927382302433280, alloc_frames = 231

+69

libs/mem/src/access_rules.rs

reviewed

··· 1 1 + // Copyright 2025. Jonas Kruckenberg 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or 4 4 + // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or 5 5 + // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 + // copied, modified, or distributed except according to those terms. 7 7 + 8 8 + mycelium_bitfield::bitfield! { 9 9 + /// Rules that dictate how a region of virtual memory may be accessed. 10 10 + /// 11 11 + /// # W^X 12 12 + /// 13 13 + /// In order to prevent malicious code execution as proactively as possible, 14 14 + /// [`AccessRules`] can either allow *writes* OR *execution* but never both. This is enforced 15 15 + /// through the [`WriteOrExecute`] enum field. 16 16 + #[derive(PartialEq, Eq)] 17 17 + pub struct AccessRules<u8> { 18 18 + /// If set, reading from the memory region is allowed. 19 19 + pub const READ: bool; 20 20 + /// Whether executing, or writing this memory region is allowed (or neither). 21 21 + pub const WRITE_OR_EXECUTE: WriteOrExecute; 22 22 + /// If set, requires code in the memory region to use aarch64 Branch Target Identification. 23 23 + /// Does nothing on non-aarch64 architectures. 24 24 + pub const BTI: bool; 25 25 + } 26 26 + } 27 27 + 28 28 + /// Whether executing, or writing this memory region is allowed (or neither). 29 29 + /// 30 30 + /// This is an enum to enforce [`W^X`] at the type-level. 31 31 + /// 32 32 + /// [`W^X`]: AccessRules 33 33 + #[derive(Copy, Clone, Debug, Eq, PartialEq)] 34 34 + #[repr(u8)] 35 35 + pub enum WriteOrExecute { 36 36 + /// Neither writing nor execution of the memory region is allowed. 37 37 + Neither = 0b00, 38 38 + /// Writing to the memory region is allowed. 39 39 + Write = 0b01, 40 40 + /// Executing code from the memory region is allowed. 41 41 + Execute = 0b10, 42 42 + } 43 43 + 44 44 + // ===== impl WriteOrExecute ===== 45 45 + 46 46 + impl mycelium_bitfield::FromBits<u8> for WriteOrExecute { 47 47 + type Error = core::convert::Infallible; 48 48 + 49 49 + /// The number of bits required to represent a value of this type. 50 50 + const BITS: u32 = 2; 51 51 + 52 52 + #[inline] 53 53 + fn try_from_bits(bits: u8) -> Result<Self, Self::Error> { 54 54 + match bits { 55 55 + b if b == Self::Neither as u8 => Ok(Self::Neither), 56 56 + b if b == Self::Write as u8 => Ok(Self::Write), 57 57 + b if b == Self::Execute as u8 => Ok(Self::Execute), 58 58 + _ => { 59 59 + // this should never happen unless the bitpacking code is broken 60 60 + unreachable!("invalid memory region access rules {bits:#b}") 61 61 + } 62 62 + } 63 63 + } 64 64 + 65 65 + #[inline] 66 66 + fn into_bits(self) -> u8 { 67 67 + self as u8 68 68 + } 69 69 + }

+1011

libs/mem/src/address_space.rs

reviewed

··· 1 1 + // Copyright 2025. Jonas Kruckenberg 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or 4 4 + // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or 5 5 + // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 + // copied, modified, or distributed except according to those terms. 7 7 + 8 8 + mod batch; 9 9 + mod region; 10 10 + 11 11 + use alloc::boxed::Box; 12 12 + use core::alloc::Layout; 13 13 + use core::num::NonZeroUsize; 14 14 + use core::ops::{Bound, ControlFlow, Range}; 15 15 + use core::pin::Pin; 16 16 + use core::ptr::NonNull; 17 17 + 18 18 + use anyhow::{format_err, Context}; 19 19 + use rand::distr::Uniform; 20 20 + use rand::Rng; 21 21 + use rand_chacha::ChaCha20Rng; 22 22 + use region::AddressSpaceRegion; 23 23 + use wavltree::{CursorMut, WAVLTree}; 24 24 + 25 25 + use crate::access_rules::AccessRules; 26 26 + use crate::address_space::batch::Batch; 27 27 + use crate::utils::assert_unsafe_precondition_; 28 28 + use crate::{AddressRangeExt, PhysicalAddress, VirtualAddress}; 29 29 + 30 30 + pub unsafe trait RawAddressSpace { 31 31 + /// The smallest addressable chunk of memory of this address space. All address argument provided 32 32 + /// to methods of this type (both virtual and physical) must be aligned to this. 33 33 + const PAGE_SIZE: usize; 34 34 + const VIRT_ADDR_BITS: u32; 35 35 + 36 36 + const PAGE_SIZE_LOG_2: u8 = (Self::PAGE_SIZE - 1).count_ones() as u8; 37 37 + const CANONICAL_ADDRESS_MASK: usize = !((1 << (Self::VIRT_ADDR_BITS)) - 1); 38 38 + 39 39 + /// The [`Flush`] implementation for this address space. 40 40 + type Flush: Flush; 41 41 + 42 42 + /// Return a new, empty flush for this address space. 43 43 + fn flush(&self) -> Self::Flush; 44 44 + 45 45 + /// Return the corresponding [`PhysicalAddress`] and [`AccessRules`] for the given 46 46 + /// [`VirtualAddress`] if mapped. 47 47 + fn lookup(&self, virt: VirtualAddress) -> Option<(PhysicalAddress, AccessRules)>; 48 48 + 49 49 + /// Map a contiguous range of `len` virtual addresses to `len` physical addresses with the 50 50 + /// specified access rules. 51 51 + /// 52 52 + /// If this returns `Ok`, the mapping is added to the raw address space and all future 53 53 + /// accesses to the virtual address range will translate to accesses of the physical address 54 54 + /// range. 55 55 + /// 56 56 + /// # Safety 57 57 + /// 58 58 + /// - `virt` must be aligned to `Self::PAGE_SIZE` 59 59 + /// - `phys` must be aligned to `Self::PAGE_SIZE` 60 60 + /// - `len` must an integer multiple of `Self::PAGE_SIZE` 61 61 + /// 62 62 + /// # Errors 63 63 + /// 64 64 + /// Returning `Err` indicates the mapping cannot be established and the virtual address range 65 65 + /// remains unaltered. 66 66 + unsafe fn map( 67 67 + &mut self, 68 68 + virt: VirtualAddress, 69 69 + phys: PhysicalAddress, 70 70 + len: NonZeroUsize, 71 71 + access_rules: AccessRules, 72 72 + flush: &mut Self::Flush, 73 73 + ) -> crate::Result<()>; 74 74 + 75 75 + /// Unmap a contiguous range of `len` virtual addresses. 76 76 + /// 77 77 + /// After this returns all accesses to the virtual address region will cause a fault. 78 78 + /// 79 79 + /// # Safety 80 80 + /// 81 81 + /// - `virt..virt+len` must be mapped 82 82 + /// - `virt` must be aligned to `Self::PAGE_SIZE` 83 83 + /// - `phys` must be aligned to `Self::PAGE_SIZE` 84 84 + /// - `len` must an integer multiple of `Self::PAGE_SIZE` 85 85 + unsafe fn unmap(&mut self, virt: VirtualAddress, len: NonZeroUsize, flush: &mut Self::Flush); 86 86 + 87 87 + /// Set the [`AccessRules`] for a contiguous range of `len` virtual addresses. 88 88 + /// 89 89 + /// After this returns all accesses to the virtual address region must follow the 90 90 + /// specified `AccessRules` or cause a fault. 91 91 + /// 92 92 + /// # Safety 93 93 + /// 94 94 + /// - `virt..virt+len` must be mapped 95 95 + /// - `virt` must be aligned to `Self::PAGE_SIZE` 96 96 + /// - `phys` must be aligned to `Self::PAGE_SIZE` 97 97 + /// - `len` must an integer multiple of `Self::PAGE_SIZE` 98 98 + unsafe fn set_access_rules( 99 99 + &mut self, 100 100 + virt: VirtualAddress, 101 101 + len: NonZeroUsize, 102 102 + access_rules: AccessRules, 103 103 + flush: &mut Self::Flush, 104 104 + ); 105 105 + } 106 106 + 107 107 + /// A type that can flush changes made to a [`RawAddressSpace`]. 108 108 + /// 109 109 + /// Note: [`Flush`] is purely optional, it exists so implementation MAY batch 110 110 + /// Note that the implementation is not required to delay materializing changes until [`Flush::flush`] 111 111 + /// is called. 112 112 + pub trait Flush { 113 113 + /// Flush changes made to its [`RawAddressSpace`]. 114 114 + /// 115 115 + /// If this returns `Ok`, changes made to the address space are REQUIRED to take effect across 116 116 + /// all affected threads/CPUs. 117 117 + /// 118 118 + /// # Errors 119 119 + /// 120 120 + /// If this returns `Err`, if flushing the changes failed. The changes, or a subset of them, might 121 121 + /// still have taken effect across all or some of the threads/CPUs. 122 122 + fn flush(self) -> crate::Result<()>; 123 123 + } 124 124 + 125 125 + pub struct AddressSpace<R: RawAddressSpace> { 126 126 + raw: R, 127 127 + regions: WAVLTree<AddressSpaceRegion>, 128 128 + batched_raw: Batch, 129 129 + max_range: Range<VirtualAddress>, 130 130 + rng: Option<ChaCha20Rng>, 131 131 + } 132 132 + 133 133 + impl<R: RawAddressSpace> AddressSpace<R> { 134 134 + pub fn new(raw: R, rng: Option<ChaCha20Rng>) -> Self { 135 135 + Self { 136 136 + raw, 137 137 + regions: WAVLTree::new(), 138 138 + batched_raw: Batch::new(), 139 139 + max_range: VirtualAddress::MIN..VirtualAddress::MAX, 140 140 + rng, 141 141 + } 142 142 + } 143 143 + 144 144 + /// Attempts to reserve a region of virtual memory. 145 145 + /// 146 146 + /// On success, returns a [`NonNull<[u8]>`][NonNull] meeting the size and alignment guarantees 147 147 + /// of `layout`. Access to this region must obey the provided `rules` or cause a hardware fault. 148 148 + /// 149 149 + /// The returned region may have a larger size than specified by `layout.size()`, and may or may 150 150 + /// not have its contents initialized. 151 151 + /// 152 152 + /// The returned region of virtual memory remains mapped as long as it is [*currently mapped*] 153 153 + /// and the address space type itself has not been dropped. 154 154 + /// 155 155 + /// [*currently mapped*]: #currently-mapped-memory 156 156 + /// 157 157 + /// # Errors 158 158 + /// 159 159 + /// Returning `Err` indicates the layout does not meet the address space's size or alignment 160 160 + /// constraints, virtual memory is exhausted, or mapping otherwise fails. 161 161 + pub fn map( 162 162 + &mut self, 163 163 + layout: Layout, 164 164 + access_rules: AccessRules, 165 165 + ) -> crate::Result<NonNull<[u8]>> { 166 166 + #[cfg(debug_assertions)] 167 167 + self.assert_valid("[AddressSpace::map]"); 168 168 + 169 169 + let layout = layout.align_to(R::PAGE_SIZE).unwrap(); 170 170 + 171 171 + let spot = self 172 172 + .find_spot_for(layout) 173 173 + .context(format_err!("cannot find free spot for layout {layout:?}"))?; 174 174 + 175 175 + // TODO "relaxed" frame provider 176 176 + let region = AddressSpaceRegion::new(spot, layout, access_rules); 177 177 + 178 178 + let region = self.regions.insert(Box::pin(region)); 179 179 + 180 180 + // TODO OPTIONAL eagerly commit a few pages 181 181 + 182 182 + self.batched_raw.flush_changes(&mut self.raw)?; 183 183 + 184 184 + Ok(region.as_non_null()) 185 185 + } 186 186 + 187 187 + /// Behaves like [`map`][AddressSpace::map], but also *guarantees* the virtual memory region 188 188 + /// is zero-initialized. 189 189 + /// 190 190 + /// # Errors 191 191 + /// 192 192 + /// Returning `Err` indicates the layout does not meet the address space's size or alignment 193 193 + /// constraints, virtual memory is exhausted, or mapping otherwise fails. 194 194 + pub fn map_zeroed( 195 195 + &mut self, 196 196 + layout: Layout, 197 197 + access_rules: AccessRules, 198 198 + ) -> crate::Result<NonNull<[u8]>> { 199 199 + #[cfg(debug_assertions)] 200 200 + self.assert_valid("[AddressSpace::map_zeroed]"); 201 201 + 202 202 + let layout = layout.align_to(R::PAGE_SIZE).unwrap(); 203 203 + 204 204 + let spot = self 205 205 + .find_spot_for(layout) 206 206 + .context(format_err!("cannot find free spot for layout {layout:?}"))?; 207 207 + 208 208 + // TODO "zeroed" frame provider 209 209 + let region = AddressSpaceRegion::new(spot, layout, access_rules); 210 210 + 211 211 + let region = self.regions.insert(Box::pin(region)); 212 212 + 213 213 + // TODO OPTIONAL eagerly commit a few pages 214 214 + 215 215 + self.batched_raw.flush_changes(&mut self.raw)?; 216 216 + 217 217 + Ok(region.as_non_null()) 218 218 + } 219 219 + 220 220 + /// Unmaps the virtual memory region referenced by `ptr`. 221 221 + /// 222 222 + /// # Safety 223 223 + /// 224 224 + /// * `ptr` must denote a region of memory [*currently mapped*] in this address space, and 225 225 + /// * `layout` must [*fit*] that region of memory. 226 226 + /// 227 227 + /// [*currently mapped*]: #currently-mapped-memory 228 228 + /// [*fit*]: #memory-fitting 229 229 + pub unsafe fn unmap(&mut self, ptr: NonNull<u8>, layout: Layout) { 230 230 + #[cfg(debug_assertions)] 231 231 + self.assert_valid("[AddressSpace::unmap]"); 232 232 + 233 233 + // Safety: responsibility of caller 234 234 + let mut cursor = unsafe { get_region_containing_ptr(&mut self.regions, ptr, layout) }; 235 235 + 236 236 + // Safety: responsibility of caller 237 237 + let mut region = unsafe { cursor.remove().unwrap_unchecked() }; 238 238 + 239 239 + region.decommit(.., &mut self.batched_raw).unwrap(); 240 240 + 241 241 + self.batched_raw.flush_changes(&mut self.raw).unwrap(); 242 242 + } 243 243 + 244 244 + /// Attempts to extend the virtual memory reservation. 245 245 + /// 246 246 + /// Returns a new [`NonNull<[u8]>`][NonNull] containing a pointer and the actual size of the 247 247 + /// mapped region. The pointer is suitable for holding data described by `new_layout`. To accomplish 248 248 + /// this, the address space may extend the mapping referenced by `ptr` to fit the new layout. 249 249 + /// 250 250 + /// TODO describe how extending a file-backed, of DMA-backed mapping works 251 251 + /// 252 252 + /// The [`AccessRules`] of the new virtual memory region are *the same* at the old ones. 253 253 + /// 254 254 + /// If this returns `Ok`, then ownership of the memory region referenced by `ptr` has been 255 255 + /// transferred to this address space. Any access to the old `ptr` is [*Undefined Behavior*], 256 256 + /// even if the mapping was grown in-place. The newly returned pointer is the only valid pointer 257 257 + /// for accessing this region now. 258 258 + /// 259 259 + /// If this method returns `Err`, then ownership of the memory region has not been transferred to 260 260 + /// this address space, and the contents of the region are unaltered. 261 261 + /// 262 262 + /// [*Undefined Behavior*] 263 263 + /// 264 264 + /// # Safety 265 265 + /// 266 266 + /// * `ptr` must denote a region of memory [*currently mapped*] in this address space. 267 267 + /// * `old_layout` must [*fit*] that region (The `new_layout` argument need not fit it.). 268 268 + /// * `new_layout.size()` must be greater than or equal to `old_layout.size()`. 269 269 + /// 270 270 + /// Note that `new_layout.align()` need not be the same as `old_layout.align()`. 271 271 + /// 272 272 + /// [*currently mapped*]: #currently-mapped-memory 273 273 + /// [*fit*]: #memory-fitting 274 274 + /// 275 275 + /// # Errors 276 276 + /// 277 277 + /// Returning `Err` indicates the layout does not meet the address space's size or alignment 278 278 + /// constraints, virtual memory is exhausted, or growing otherwise fails. 279 279 + pub unsafe fn grow( 280 280 + &mut self, 281 281 + ptr: NonNull<u8>, 282 282 + old_layout: Layout, 283 283 + new_layout: Layout, 284 284 + ) -> crate::Result<NonNull<[u8]>> { 285 285 + #[cfg(debug_assertions)] 286 286 + self.assert_valid("[AddressSpace::grow]"); 287 287 + 288 288 + assert_unsafe_precondition_!( 289 289 + "TODO", 290 290 + (old_layout: Layout = old_layout, page_size: usize = R::PAGE_SIZE) => { 291 291 + old_layout.align().is_multiple_of(page_size) 292 292 + } 293 293 + ); 294 294 + 295 295 + assert_unsafe_precondition_!( 296 296 + "TODO", 297 297 + (new_layout: Layout = new_layout, page_size: usize = R::PAGE_SIZE) => { 298 298 + new_layout.align().is_multiple_of(page_size) 299 299 + } 300 300 + ); 301 301 + 302 302 + if new_layout == old_layout { 303 303 + return Ok(NonNull::slice_from_raw_parts(ptr, new_layout.size())); 304 304 + } 305 305 + 306 306 + assert_unsafe_precondition_!( 307 307 + "TODO", 308 308 + (old_layout: Layout = old_layout, new_layout: Layout = new_layout) => { 309 309 + new_layout.size() >= old_layout.size() 310 310 + } 311 311 + ); 312 312 + 313 313 + if let Ok(ptr) = unsafe { self.grow_in_place_inner(ptr, old_layout, new_layout) } { 314 314 + Ok(ptr) 315 315 + } else { 316 316 + unsafe { self.reallocate_region(ptr, old_layout, new_layout) } 317 317 + } 318 318 + } 319 319 + 320 320 + /// Behaves like [`grow`][AddressSpace::grow], only grows the region if it can be grown in-place. 321 321 + /// 322 322 + /// # Safety 323 323 + /// 324 324 + /// * `ptr` must denote a region of memory [*currently mapped*] in this address space. 325 325 + /// * `old_layout` must [*fit*] that region (The `new_layout` argument need not fit it.). 326 326 + /// * `new_layout.size()` must be greater than or equal to `old_layout.size()`. 327 327 + /// 328 328 + /// Note that `new_layout.align()` need not be the same as `old_layout.align()`. 329 329 + /// 330 330 + /// [*currently mapped*]: #currently-mapped-memory 331 331 + /// [*fit*]: #memory-fitting 332 332 + /// 333 333 + /// # Errors 334 334 + /// 335 335 + /// Returning `Err` indicates the layout does not meet the address space's size or alignment 336 336 + /// constraints, virtual memory is exhausted, or growing otherwise fails. 337 337 + pub unsafe fn grow_in_place( 338 338 + &mut self, 339 339 + ptr: NonNull<u8>, 340 340 + old_layout: Layout, 341 341 + new_layout: Layout, 342 342 + ) -> crate::Result<NonNull<[u8]>> { 343 343 + #[cfg(debug_assertions)] 344 344 + self.assert_valid("[AddressSpace::grow_in_place]"); 345 345 + 346 346 + assert_unsafe_precondition_!( 347 347 + "TODO", 348 348 + (old_layout: Layout = old_layout, page_size: usize = R::PAGE_SIZE) => { 349 349 + old_layout.align().is_multiple_of(page_size) 350 350 + } 351 351 + ); 352 352 + 353 353 + assert_unsafe_precondition_!( 354 354 + "TODO", 355 355 + (new_layout: Layout = new_layout, page_size: usize = R::PAGE_SIZE) => { 356 356 + new_layout.align().is_multiple_of(page_size) 357 357 + } 358 358 + ); 359 359 + 360 360 + if new_layout == old_layout { 361 361 + return Ok(NonNull::slice_from_raw_parts(ptr, new_layout.size())); 362 362 + } 363 363 + 364 364 + assert_unsafe_precondition_!( 365 365 + "TODO", 366 366 + (old_layout: Layout = old_layout, new_layout: Layout = new_layout) => { 367 367 + new_layout.size() >= old_layout.size() 368 368 + } 369 369 + ); 370 370 + 371 371 + unsafe { self.grow_in_place_inner(ptr, old_layout, new_layout) } 372 372 + } 373 373 + 374 374 + /// Attempts to shrink the virtual memory reservation. 375 375 + /// 376 376 + /// Returns a new [`NonNull<[u8]>`][NonNull] containing a pointer and the actual size of the 377 377 + /// mapped region. The pointer is suitable for holding data described by `new_layout`. To accomplish 378 378 + /// this, the address space may shrink the mapping referenced by `ptr` to fit the new layout. 379 379 + /// 380 380 + /// TODO describe how shrinking a file-backed, of DMA-backed mapping works 381 381 + /// 382 382 + /// The [`AccessRules`] of the new virtual memory region are *the same* at the old ones. 383 383 + /// 384 384 + /// If this returns `Ok`, then ownership of the memory region referenced by `ptr` has been 385 385 + /// transferred to this address space. Any access to the old `ptr` is [*Undefined Behavior*], 386 386 + /// even if the mapping was shrunk in-place. The newly returned pointer is the only valid pointer 387 387 + /// for accessing this region now. 388 388 + /// 389 389 + /// If this method returns `Err`, then ownership of the memory region has not been transferred to 390 390 + /// this address space, and the contents of the region are unaltered. 391 391 + /// 392 392 + /// [*Undefined Behavior*] 393 393 + /// 394 394 + /// # Safety 395 395 + /// 396 396 + /// * `ptr` must denote a region of memory [*currently mapped*] in this address space. 397 397 + /// * `old_layout` must [*fit*] that region (The `new_layout` argument need not fit it.). 398 398 + /// * `new_layout.size()` must be smaller than or equal to `old_layout.size()`. 399 399 + /// 400 400 + /// Note that `new_layout.align()` need not be the same as `old_layout.align()`. 401 401 + /// 402 402 + /// [*currently mapped*]: #currently-mapped-memory 403 403 + /// [*fit*]: #memory-fitting 404 404 + /// 405 405 + /// # Errors 406 406 + /// 407 407 + /// Returning `Err` indicates the layout does not meet the address space's size or alignment 408 408 + /// constraints, virtual memory is exhausted, or shrinking otherwise fails. 409 409 + pub unsafe fn shrink( 410 410 + &mut self, 411 411 + ptr: NonNull<u8>, 412 412 + old_layout: Layout, 413 413 + new_layout: Layout, 414 414 + ) -> crate::Result<NonNull<[u8]>> { 415 415 + #[cfg(debug_assertions)] 416 416 + self.assert_valid("[AddressSpace::shrink]"); 417 417 + 418 418 + assert_unsafe_precondition_!( 419 419 + "TODO", 420 420 + (old_layout: Layout = old_layout, page_size: usize = R::PAGE_SIZE) => { 421 421 + old_layout.align().is_multiple_of(page_size) 422 422 + } 423 423 + ); 424 424 + 425 425 + assert_unsafe_precondition_!( 426 426 + "TODO", 427 427 + (new_layout: Layout = new_layout, page_size: usize = R::PAGE_SIZE) => { 428 428 + new_layout.align().is_multiple_of(page_size) 429 429 + } 430 430 + ); 431 431 + 432 432 + if new_layout == old_layout { 433 433 + return Ok(NonNull::slice_from_raw_parts(ptr, new_layout.size())); 434 434 + } 435 435 + 436 436 + assert_unsafe_precondition_!( 437 437 + "TODO", 438 438 + (old_layout: Layout = old_layout, new_layout: Layout = new_layout) => { 439 439 + new_layout.size() <= old_layout.size() 440 440 + } 441 441 + ); 442 442 + 443 443 + if let Ok(ptr) = unsafe { self.shrink_in_place_inner(ptr, old_layout, new_layout) } { 444 444 + Ok(ptr) 445 445 + } else { 446 446 + unsafe { self.reallocate_region(ptr, old_layout, new_layout) } 447 447 + } 448 448 + } 449 449 + 450 450 + /// Behaves like [`shrink`][AddressSpace::shrink], but *guarantees* that the region will be 451 451 + /// shrunk in-place. Both `old_layout` and `new_layout` need to be at least page aligned. 452 452 + /// 453 453 + /// # Safety 454 454 + /// 455 455 + /// * `ptr` must denote a region of memory [*currently mapped*] in this address space. 456 456 + /// * `old_layout` must [*fit*] that region (The `new_layout` argument need not fit it.). 457 457 + /// * `new_layout.size()` must be smaller than or equal to `old_layout.size()`. 458 458 + /// 459 459 + /// Note that `new_layout.align()` need not be the same as `old_layout.align()`. 460 460 + /// 461 461 + /// [*currently mapped*]: #currently-mapped-memory 462 462 + /// [*fit*]: #memory-fitting 463 463 + /// 464 464 + /// # Errors 465 465 + /// 466 466 + /// Returning `Err` indicates the layout does not meet the address space's size or alignment 467 467 + /// constraints, virtual memory is exhausted, or growing otherwise fails. 468 468 + pub unsafe fn shrink_in_place( 469 469 + &mut self, 470 470 + ptr: NonNull<u8>, 471 471 + old_layout: Layout, 472 472 + new_layout: Layout, 473 473 + ) -> crate::Result<NonNull<[u8]>> { 474 474 + #[cfg(debug_assertions)] 475 475 + self.assert_valid("[AddressSpace::shrink_in_place]"); 476 476 + 477 477 + assert_unsafe_precondition_!( 478 478 + "TODO", 479 479 + (old_layout: Layout = old_layout, page_size: usize = R::PAGE_SIZE) => { 480 480 + old_layout.align().is_multiple_of(page_size) 481 481 + } 482 482 + ); 483 483 + 484 484 + assert_unsafe_precondition_!( 485 485 + "TODO", 486 486 + (new_layout: Layout = new_layout, page_size: usize = R::PAGE_SIZE) => { 487 487 + new_layout.align().is_multiple_of(page_size) 488 488 + } 489 489 + ); 490 490 + 491 491 + if new_layout == old_layout { 492 492 + return Ok(NonNull::slice_from_raw_parts(ptr, new_layout.size())); 493 493 + } 494 494 + 495 495 + assert_unsafe_precondition_!( 496 496 + "TODO", 497 497 + (old_layout: Layout = old_layout, new_layout: Layout = new_layout) => { 498 498 + new_layout.size() <= old_layout.size() 499 499 + } 500 500 + ); 501 501 + 502 502 + unsafe { self.shrink_in_place_inner(ptr, old_layout, new_layout) } 503 503 + } 504 504 + 505 505 + /// Updates the access rules for the virtual memory region referenced by `ptr`. 506 506 + /// 507 507 + /// If this returns `Ok`, access to this region must obey the new `rules` or cause a hardware fault. 508 508 + /// 509 509 + /// If this method returns `Err`, the access rules of the memory region are unaltered. 510 510 + /// 511 511 + /// # Safety 512 512 + /// 513 513 + /// * `ptr` must denote a region of memory [*currently mapped*] in this address space, and 514 514 + /// * `layout` must [*fit*] that region of memory. 515 515 + /// 516 516 + /// [*currently mapped*]: #currently-mapped-memory 517 517 + /// [*fit*]: #memory-fitting 518 518 + pub unsafe fn update_access_rules( 519 519 + &mut self, 520 520 + ptr: NonNull<u8>, 521 521 + layout: Layout, 522 522 + access_rules: AccessRules, 523 523 + ) -> crate::Result<()> { 524 524 + #[cfg(debug_assertions)] 525 525 + self.assert_valid("[AddressSpace::update_access_rules]"); 526 526 + 527 527 + // Safety: responsibility of caller 528 528 + let mut cursor = unsafe { get_region_containing_ptr(&mut self.regions, ptr, layout) }; 529 529 + 530 530 + // Safety: responsibility of caller 531 531 + let mut region = unsafe { cursor.get_mut().unwrap_unchecked() }; 532 532 + 533 533 + region.update_access_rules(access_rules, &mut self.batched_raw)?; 534 534 + 535 535 + self.batched_raw.flush_changes(&mut self.raw)?; 536 536 + 537 537 + Ok(()) 538 538 + } 539 539 + 540 540 + /// Attempts to fill the virtual memory region referenced by `ptr` with zeroes. 541 541 + /// 542 542 + /// Returns a new [`NonNull<[u8]>`][NonNull] containing a pointer and the actual size of the 543 543 + /// mapped region. The pointer is suitable for holding data described by `new_layout` and is 544 544 + /// *guaranteed* to be zero-initialized. To accomplish this, the address space may remap the 545 545 + /// virtual memory region. 546 546 + /// 547 547 + /// TODO describe how clearing a file-backed, of DMA-backed mapping works 548 548 + /// 549 549 + /// The [`AccessRules`] of the new virtual memory region are *the same* at the old ones. 550 550 + /// 551 551 + /// If this returns `Ok`, then ownership of the memory region referenced by `ptr` has been 552 552 + /// transferred to this address space. Any access to the old `ptr` is [*Undefined Behavior*], 553 553 + /// even if the mapping was cleared in-place. The newly returned pointer is the only valid pointer 554 554 + /// for accessing this region now. 555 555 + /// 556 556 + /// If this method returns `Err`, then ownership of the memory region has not been transferred to 557 557 + /// this address space, and the contents of the region are unaltered. 558 558 + /// 559 559 + /// [*Undefined Behavior*] 560 560 + /// 561 561 + /// # Safety 562 562 + /// 563 563 + /// * `ptr` must denote a region of memory [*currently mapped*] in this address space, and 564 564 + /// * `layout` must [*fit*] that region of memory. 565 565 + /// 566 566 + /// [*currently mapped*]: #currently-mapped-memory 567 567 + /// [*fit*]: #memory-fitting 568 568 + /// 569 569 + /// # Errors 570 570 + /// 571 571 + /// Returning `Err` indicates the layout does not meet the address space's size or alignment 572 572 + /// constraints, clearing a virtual memory region is not supported by the backing storage, or 573 573 + /// clearing otherwise fails. 574 574 + pub unsafe fn clear( 575 575 + &mut self, 576 576 + ptr: NonNull<u8>, 577 577 + layout: Layout, 578 578 + ) -> crate::Result<NonNull<[u8]>> { 579 579 + #[cfg(debug_assertions)] 580 580 + self.assert_valid("[AddressSpace::clear]"); 581 581 + 582 582 + // Safety: responsibility of caller 583 583 + let mut cursor = unsafe { get_region_containing_ptr(&mut self.regions, ptr, layout) }; 584 584 + 585 585 + // Safety: responsibility of caller 586 586 + let mut region = unsafe { cursor.get_mut().unwrap_unchecked() }; 587 587 + 588 588 + region.clear(&mut self.batched_raw)?; 589 589 + 590 590 + self.batched_raw.flush_changes(&mut self.raw)?; 591 591 + 592 592 + Ok(region.as_non_null()) 593 593 + } 594 594 + 595 595 + pub fn assert_valid(&self, msg: &str) { 596 596 + let mut regions = self.regions.iter(); 597 597 + 598 598 + let Some(first_region) = regions.next() else { 599 599 + assert!( 600 600 + self.regions.is_empty(), 601 601 + "{msg}region iterator is empty but tree is not." 602 602 + ); 603 603 + 604 604 + return; 605 605 + }; 606 606 + 607 607 + first_region.assert_valid(msg); 608 608 + 609 609 + let mut seen_range = first_region.range().clone(); 610 610 + 611 611 + while let Some(region) = regions.next() { 612 612 + assert!( 613 613 + !region.range().is_overlapping(&seen_range), 614 614 + "{msg}region cannot overlap previous region; region={region:?}" 615 615 + ); 616 616 + assert!( 617 617 + region.range().start >= self.max_range.start 618 618 + && region.range().end <= self.max_range.end, 619 619 + "{msg}region cannot lie outside of max address space range; region={region:?}" 620 620 + ); 621 621 + 622 622 + seen_range = seen_range.start..region.range().end; 623 623 + 624 624 + region.assert_valid(msg); 625 625 + 626 626 + // TODO assert validity of of VMO against phys addresses 627 627 + // let (_phys, access_rules) = self 628 628 + // .batched_raw 629 629 + // .raw_address_space() 630 630 + // .lookup(region.range().start) 631 631 + // .unwrap_or_else(|| { 632 632 + // panic!("{msg}region base address is not mapped in raw address space region={region:?}") 633 633 + // }); 634 634 + // 635 635 + // assert_eq!( 636 636 + // access_rules, 637 637 + // region.access_rules(), 638 638 + // "{msg}region's access rules do not match access rules in raw address space; region={region:?}, expected={:?}, actual={access_rules:?}", 639 639 + // region.access_rules(), 640 640 + // ); 641 641 + } 642 642 + } 643 643 + 644 644 + /// Attempts to grow a virtual memory region in-place. This method is shared between [`Self::shrink`] 645 645 + /// and [`Self::shrink_in_place`]. 646 646 + /// 647 647 + /// # Safety 648 648 + /// 649 649 + /// * `ptr` must denote a region of memory [*currently mapped*] in this address space. 650 650 + /// * `old_layout` must [*fit*] that region (The `new_layout` argument need not fit it.). 651 651 + /// * `new_layout.size()` must be greater than or equal to `old_layout.size()`. 652 652 + /// * `new_layout.align()` must be multiple of PAGE_SIZE 653 653 + unsafe fn grow_in_place_inner( 654 654 + &mut self, 655 655 + ptr: NonNull<u8>, 656 656 + old_layout: Layout, 657 657 + new_layout: Layout, 658 658 + ) -> crate::Result<NonNull<[u8]>> { 659 659 + // Safety: responsibility of caller 660 660 + let mut cursor = unsafe { get_region_containing_ptr(&mut self.regions, ptr, old_layout) }; 661 661 + 662 662 + let next_range = cursor.peek_next().map(|region| region.range().clone()); 663 663 + 664 664 + // Safety: responsibility of caller 665 665 + let mut region = unsafe { cursor.get_mut().unwrap_unchecked() }; 666 666 + 667 667 + region.grow_in_place(new_layout, next_range, &mut self.batched_raw)?; 668 668 + 669 669 + self.batched_raw.flush_changes(&mut self.raw)?; 670 670 + 671 671 + Ok(region.as_non_null()) 672 672 + } 673 673 + 674 674 + /// Attempts to shrink a virtual memory region in-place. This method is shared between [`Self::grow`] 675 675 + /// and [`Self::grow_in_place`]. 676 676 + /// 677 677 + /// # Safety 678 678 + /// 679 679 + /// * `ptr` must denote a region of memory [*currently mapped*] in this address space. 680 680 + /// * `old_layout` must [*fit*] that region (The `new_layout` argument need not fit it.). 681 681 + /// * `new_layout.size()` must be smaller than or equal to `old_layout.size()`. 682 682 + /// * `new_layout.align()` must be multiple of PAGE_SIZE 683 683 + unsafe fn shrink_in_place_inner( 684 684 + &mut self, 685 685 + ptr: NonNull<u8>, 686 686 + old_layout: Layout, 687 687 + new_layout: Layout, 688 688 + ) -> crate::Result<NonNull<[u8]>> { 689 689 + // Safety: responsibility of caller 690 690 + let mut cursor = unsafe { get_region_containing_ptr(&mut self.regions, ptr, old_layout) }; 691 691 + 692 692 + // Safety: responsibility of caller 693 693 + let mut region = unsafe { cursor.get_mut().unwrap_unchecked() }; 694 694 + 695 695 + region.shrink(new_layout, &mut self.batched_raw)?; 696 696 + 697 697 + self.batched_raw.flush_changes(&mut self.raw)?; 698 698 + 699 699 + Ok(region.as_non_null()) 700 700 + } 701 701 + 702 702 + /// Reallocates a virtual address region. This will unmap and remove the old region, allocating 703 703 + /// a new region that will be backed the old regions physical memory. 704 704 + /// 705 705 + /// # Safety 706 706 + /// 707 707 + /// * `ptr` must denote a region of memory [*currently mapped*] in this address space. 708 708 + /// * `old_layout` must [*fit*] that region (The `new_layout` argument need not fit it.). 709 709 + /// * `new_layout.size()` must be smaller than or equal to `old_layout.size()`. 710 710 + /// * `new_layout.align()` must be multiple of PAGE_SIZE 711 711 + unsafe fn reallocate_region( 712 712 + &mut self, 713 713 + ptr: NonNull<u8>, 714 714 + old_layout: Layout, 715 715 + new_layout: Layout, 716 716 + ) -> crate::Result<NonNull<[u8]>> { 717 717 + // Safety: responsibility of caller 718 718 + let mut cursor = unsafe { get_region_containing_ptr(&mut self.regions, ptr, old_layout) }; 719 719 + let mut region = unsafe { cursor.remove().unwrap_unchecked() }; 720 720 + 721 721 + region.decommit(.., &mut self.batched_raw).unwrap(); 722 722 + 723 723 + let (_, _, access_rules) = Pin::into_inner(region).into_parts(); 724 724 + 725 725 + let spot = self.find_spot_for(new_layout).context(format_err!( 726 726 + "cannot find free spot for layout {new_layout:?}" 727 727 + ))?; 728 728 + 729 729 + // TODO resize VMO 730 730 + 731 731 + let region = AddressSpaceRegion::new(spot, new_layout, access_rules); 732 732 + let region = self.regions.insert(Box::pin(region)); 733 733 + 734 734 + // TODO replicate old regions committed frames? 735 735 + 736 736 + self.batched_raw.flush_changes(&mut self.raw)?; 737 737 + 738 738 + Ok(region.as_non_null()) 739 739 + } 740 740 + 741 741 + /// Find a spot in the address space that satisfies the given `layout` requirements. 742 742 + /// 743 743 + /// If a spot suitable for holding data described by `layout` is found, the base address of the 744 744 + /// address range is returned in `Some`. The returned address is already correct aligned to 745 745 + /// `layout.align()`. 746 746 + /// 747 747 + /// Returns `None` if no suitable spot was found. This *does not* mean there are no more gaps in 748 748 + /// the address space just that the *combination* of `layout.size()` and `layout.align()` cannot 749 749 + /// be satisfied *at the moment*. Calls to this method will a different size, alignment, or at a 750 750 + /// different time might still succeed. 751 751 + fn find_spot_for(&mut self, layout: Layout) -> Option<VirtualAddress> { 752 752 + // The algorithm we use here - loosely based on Zircon's (Fuchsia's) implementation - is 753 753 + // guaranteed to find a spot (if any even exist) with max 2 attempts. Additionally, it works 754 754 + // elegantly *with* AND *without* ASLR, picking a random spot or the lowest free spot respectively. 755 755 + // Here is how it works: 756 756 + // 1. We set up two counters: (see the GapVisitor) 757 757 + // - `candidate_spot_count` which we initialize to zero 758 758 + // - `target_index` which we either set to a random value between 0..<the maximum number of 759 759 + // possible addresses in the address space> if ASLR is requested OR to zero otherwise. 760 760 + // 2. We then iterate over all `AddressSpaceRegion`s from lowest to highest looking at the 761 761 + // gaps between regions. We count the number of addresses in each gap that satisfy the 762 762 + // requested `Layout`s size and alignment and add that to the `candidate_spot_count`. 763 763 + // IF the number of spots in the gap is greater than our chosen target index, we pick the 764 764 + // spot at the target index and finish. ELSE we *decrement* the target index by the number 765 765 + // of spots and continue to the next gap. 766 766 + // 3. After we have processed all the gaps, we have EITHER found a suitable spot OR our original 767 767 + // guess for `target_index` was too big, in which case we need to retry. 768 768 + // 4. When retrying we iterate over all `AddressSpaceRegion`s *again*, but this time we know 769 769 + // the *actual* number of possible spots in the address space since we just counted them 770 770 + // during the first attempt. We initialize `target_index` to `0..candidate_spot_count` 771 771 + // which is guaranteed to return us a spot. 772 772 + // IF `candidate_spot_count` is ZERO after the first attempt, there is no point in 773 773 + // retrying since we cannot fulfill the requested layout. 774 774 + // 775 775 + // Note that in practice, we use a binary tree to keep track of regions, and we use binary search 776 776 + // to optimize the search for a suitable gap instead of linear iteration. 777 777 + 778 778 + let layout = layout.pad_to_align(); 779 779 + 780 780 + // First attempt: guess a random target index 781 781 + let max_candidate_spots = self.max_range.size(); 782 782 + 783 783 + let target_index: usize = self 784 784 + .rng 785 785 + .as_mut() 786 786 + .map(|prng| prng.sample(Uniform::new(0, max_candidate_spots).unwrap())) 787 787 + .unwrap_or_default(); 788 788 + 789 789 + // First attempt: visit the binary search tree to find a gap 790 790 + let mut v = GapVisitor::new(layout, target_index); 791 791 + self.visit_gaps(&mut v); 792 792 + 793 793 + // if we found a spot already we're done 794 794 + if let Some(chosen) = v.chosen { 795 795 + return Some(chosen); 796 796 + } 797 797 + 798 798 + // otherwise, Second attempt: we need to retry with the correct candidate spot count 799 799 + // but if we counted no suitable candidate spots during the first attempt, we cannot fulfill 800 800 + // the request. 801 801 + if v.candidate_spots == 0 { 802 802 + return None; 803 803 + } 804 804 + 805 805 + // Second attempt: pick a new target_index that's actually fulfillable 806 806 + let target_index: usize = self 807 807 + .rng 808 808 + .as_mut() 809 809 + .map(|prng| prng.sample(Uniform::new(0, v.candidate_spots).unwrap())) 810 810 + .unwrap_or_default(); 811 811 + 812 812 + // Second attempt: visit the binary search tree to find a gap 813 813 + let mut v = GapVisitor::new(layout, target_index); 814 814 + self.visit_gaps(&mut v); 815 815 + 816 816 + let chosen = v 817 817 + .chosen 818 818 + .expect("There must be a chosen spot after the first attempt. This is a bug!"); 819 819 + 820 820 + // assert!(chosen.is_canonical()); 821 821 + 822 822 + Some(chosen) 823 823 + } 824 824 + 825 825 + /// Visit all gaps (address ranges not covered by an [`AddressSpaceRegion`]) in this address space 826 826 + /// from lowest to highest addresses. 827 827 + fn visit_gaps(&self, v: &mut GapVisitor) { 828 828 + let Some(root) = self.regions.root().get() else { 829 829 + // if the tree is empty, we treat the entire max_range as the gap 830 830 + // note that we do not care about the returned ControlFlow, as there is nothing else we 831 831 + // could try to find a spot anyway 832 832 + let _ = v.visit(self.max_range.clone()); 833 833 + 834 834 + return; 835 835 + }; 836 836 + 837 837 + // see if there is a suitable gap between BEFORE the first address space region 838 838 + if v.visit(self.max_range.start..root.subtree_range().start) 839 839 + .is_break() 840 840 + { 841 841 + return; 842 842 + } 843 843 + 844 844 + // now comes the main part of the search. we start at the WAVLTree root node and do a 845 845 + // binary search for a suitable gap. We use special metadata on each `AddressSpaceRegion` 846 846 + // to speed up this search. See `AddressSpaceRegion` for details on how this works. 847 847 + 848 848 + let mut maybe_current = self.regions.root().get(); 849 849 + let mut already_visited = VirtualAddress::MIN; 850 850 + 851 851 + while let Some(current) = maybe_current { 852 852 + // If there is no suitable gap in this entire 853 853 + if current.suitable_gap_in_subtree(v.layout()) { 854 854 + // First, look at the left subtree 855 855 + if let Some(left) = current.left_child() { 856 856 + if left.suitable_gap_in_subtree(v.layout()) 857 857 + && left.subtree_range().end > already_visited 858 858 + { 859 859 + maybe_current = Some(left); 860 860 + continue; 861 861 + } 862 862 + 863 863 + if v.visit(left.subtree_range().end..current.range().start) 864 864 + .is_break() 865 865 + { 866 866 + return; 867 867 + } 868 868 + } 869 869 + 870 870 + if let Some(right) = current.right_child() { 871 871 + if v.visit(current.range().end..right.subtree_range().start) 872 872 + .is_break() 873 873 + { 874 874 + return; 875 875 + } 876 876 + 877 877 + if right.suitable_gap_in_subtree(v.layout()) 878 878 + && right.subtree_range().end > already_visited 879 879 + { 880 880 + maybe_current = Some(right); 881 881 + continue; 882 882 + } 883 883 + } 884 884 + } 885 885 + 886 886 + already_visited = current.subtree_range().end; 887 887 + maybe_current = current.parent(); 888 888 + } 889 889 + 890 890 + // see if there is a suitable gap between AFTER the last address space region 891 891 + if v.visit(root.subtree_range().end..self.max_range.end) 892 892 + .is_break() 893 893 + { 894 894 + return; 895 895 + } 896 896 + } 897 897 + } 898 898 + 899 899 + /// # Safety 900 900 + /// 901 901 + /// * `ptr` must denote a region of memory [*currently mapped*] in this address space, and 902 902 + /// * `layout` must [*fit*] that region of memory. 903 903 + /// 904 904 + /// [*currently mapped*]: #currently-mapped-memory 905 905 + /// [*fit*]: #memory-fitting 906 906 + unsafe fn get_region_containing_ptr( 907 907 + regions: &mut WAVLTree<AddressSpaceRegion>, 908 908 + ptr: NonNull<u8>, 909 909 + layout: Layout, 910 910 + ) -> CursorMut<'_, AddressSpaceRegion> { 911 911 + let addr = VirtualAddress::from_non_null(ptr); 912 912 + 913 913 + let cursor = regions.lower_bound_mut(Bound::Included(&addr)); 914 914 + 915 915 + assert_unsafe_precondition_!( 916 916 + "TODO", 917 917 + (cursor: &CursorMut<AddressSpaceRegion> = &cursor) => cursor.get().is_some() 918 918 + ); 919 919 + 920 920 + // Safety: The caller guarantees the pointer is currently mapped which means we must have 921 921 + // a corresponding address space region for it 922 922 + let region = unsafe { cursor.get().unwrap_unchecked() }; 923 923 + 924 924 + assert_unsafe_precondition_!( 925 925 + "TODO", 926 926 + (region: &AddressSpaceRegion = region, addr: VirtualAddress = addr) => { 927 927 + let range = region.range(); 928 928 + 929 929 + range.start.get() <= addr.get() && addr.get() < range.end.get() 930 930 + } 931 931 + ); 932 932 + 933 933 + assert_unsafe_precondition_!( 934 934 + "`layout` does not fit memory region", 935 935 + (layout: Layout = layout, region: &AddressSpaceRegion = &region) => region.layout_fits_region(layout) 936 936 + ); 937 937 + 938 938 + cursor 939 939 + } 940 940 + 941 941 + pub(crate) struct GapVisitor { 942 942 + layout: Layout, 943 943 + target_index: usize, 944 944 + candidate_spots: usize, 945 945 + chosen: Option<VirtualAddress>, 946 946 + } 947 947 + 948 948 + impl GapVisitor { 949 949 + fn new(layout: Layout, target_index: usize) -> Self { 950 950 + Self { 951 951 + layout, 952 952 + target_index, 953 953 + candidate_spots: 0, 954 954 + chosen: None, 955 955 + } 956 956 + } 957 957 + 958 958 + pub fn layout(&self) -> Layout { 959 959 + self.layout 960 960 + } 961 961 + 962 962 + /// Returns the number of spots in the given range that satisfy the layout we require 963 963 + fn spots_in_range(&self, range: &Range<VirtualAddress>) -> usize { 964 964 + debug_assert!( 965 965 + range.start.is_aligned_to(self.layout.align()) 966 966 + && range.end.is_aligned_to(self.layout.align()) 967 967 + ); 968 968 + 969 969 + // ranges passed in here can become empty for a number of reasons (aligning might produce ranges 970 970 + // where end > start, or the range might be empty to begin with) in either case an empty 971 971 + // range means no spots are available 972 972 + if range.is_empty() { 973 973 + return 0; 974 974 + } 975 975 + 976 976 + let range_size = range.size(); 977 977 + if range_size >= self.layout.size() { 978 978 + ((range_size - self.layout.size()) >> self.layout.align().ilog2()) + 1 979 979 + } else { 980 980 + 0 981 981 + } 982 982 + } 983 983 + 984 984 + pub fn visit(&mut self, gap: Range<VirtualAddress>) -> ControlFlow<()> { 985 985 + // if we have already chosen a spot, signal the caller to stop 986 986 + if self.chosen.is_some() { 987 987 + return ControlFlow::Break(()); 988 988 + } 989 989 + 990 990 + let aligned_gap = gap.checked_align_in(self.layout.align()).unwrap(); 991 991 + 992 992 + let spot_count = self.spots_in_range(&aligned_gap); 993 993 + 994 994 + self.candidate_spots += spot_count; 995 995 + 996 996 + if self.target_index < spot_count { 997 997 + self.chosen = Some( 998 998 + aligned_gap 999 999 + .start 1000 1000 + .checked_add(self.target_index << self.layout.align().ilog2()) 1001 1001 + .unwrap(), 1002 1002 + ); 1003 1003 + 1004 1004 + ControlFlow::Break(()) 1005 1005 + } else { 1006 1006 + self.target_index -= spot_count; 1007 1007 + 1008 1008 + ControlFlow::Continue(()) 1009 1009 + } 1010 1010 + } 1011 1011 + }

+336

libs/mem/src/address_space/batch.rs

reviewed

··· 1 1 + // Copyright 2025. Jonas Kruckenberg 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or 4 4 + // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or 5 5 + // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 + // copied, modified, or distributed except according to those terms. 7 7 + 8 8 + use core::cmp; 9 9 + use core::num::{NonZero, NonZeroUsize}; 10 10 + 11 11 + use smallvec::SmallVec; 12 12 + 13 13 + use crate::address_space::{Flush, RawAddressSpace}; 14 14 + use crate::{AccessRules, PhysicalAddress, VirtualAddress}; 15 15 + 16 16 + /// [`Batch`] maintains an *unordered* set of batched operations over an `RawAddressSpace`. 17 17 + /// 18 18 + /// Operations are "enqueued" (but unordered) into the batch and executed against the raw address space 19 19 + /// when [`Self::flush_changes`] is called. This helps to reduce the number and size of (expensive) TLB 20 20 + /// flushes we need to perform. Internally, `Batch` will merge operations if possible to further reduce 21 21 + /// this number. 22 22 + pub struct Batch { 23 23 + ops: SmallVec<[BatchOperation; 4]>, 24 24 + } 25 25 + 26 26 + enum BatchOperation { 27 27 + Map(MapOperation), 28 28 + Unmap(UnmapOperation), 29 29 + SetAccessRules(SetAccessRulesOperation), 30 30 + } 31 31 + 32 32 + struct MapOperation { 33 33 + virt: VirtualAddress, 34 34 + phys: PhysicalAddress, 35 35 + len: NonZeroUsize, 36 36 + access_rules: AccessRules, 37 37 + } 38 38 + 39 39 + struct UnmapOperation { 40 40 + virt: VirtualAddress, 41 41 + len: NonZeroUsize, 42 42 + } 43 43 + 44 44 + struct SetAccessRulesOperation { 45 45 + virt: VirtualAddress, 46 46 + len: NonZeroUsize, 47 47 + access_rules: AccessRules, 48 48 + } 49 49 + 50 50 + // ===== impl Batch ===== 51 51 + 52 52 + impl Batch { 53 53 + /// Construct a new empty [`Batch`]. 54 54 + pub fn new() -> Self { 55 55 + Self { 56 56 + ops: SmallVec::new(), 57 57 + } 58 58 + } 59 59 + 60 60 + /// Add a [`map`] operation to the set of batched operations. 61 61 + /// 62 62 + /// # Safety 63 63 + /// 64 64 + /// - `virt` must be aligned to `Self::PAGE_SIZE` 65 65 + /// - `phys` must be aligned to `Self::PAGE_SIZE` 66 66 + /// - `len` must an integer multiple of `Self::PAGE_SIZE` 67 67 + /// 68 68 + /// [`map`]: RawAddressSpace::map 69 69 + pub fn map( 70 70 + &mut self, 71 71 + virt: VirtualAddress, 72 72 + phys: PhysicalAddress, 73 73 + len: NonZeroUsize, 74 74 + access_rules: AccessRules, 75 75 + ) { 76 76 + let mut new = MapOperation { 77 77 + virt, 78 78 + phys, 79 79 + len, 80 80 + access_rules, 81 81 + }; 82 82 + 83 83 + let ops = self.ops.iter_mut().filter_map(|op| match op { 84 84 + BatchOperation::Map(op) => Some(op), 85 85 + _ => None, 86 86 + }); 87 87 + 88 88 + for op in ops { 89 89 + match op.try_merge_with(new) { 90 90 + Ok(()) => return, 91 91 + Err(new_) => new = new_, 92 92 + } 93 93 + } 94 94 + 95 95 + self.ops.push(BatchOperation::Map(new)); 96 96 + } 97 97 + 98 98 + /// Add an [`unmap`] operation to the set of batched operations. 99 99 + /// 100 100 + /// # Safety 101 101 + /// 102 102 + /// - virt..virt+len must be mapped 103 103 + /// - `virt` must be aligned to `Self::PAGE_SIZE` 104 104 + /// - `phys` must be aligned to `Self::PAGE_SIZE` 105 105 + /// - `len` must an integer multiple of `Self::PAGE_SIZE` 106 106 + /// 107 107 + /// [`unmap`]: RawAddressSpace::unmap 108 108 + pub unsafe fn unmap(&mut self, virt: VirtualAddress, len: NonZeroUsize) { 109 109 + let mut new = UnmapOperation { virt, len }; 110 110 + 111 111 + let ops = self.ops.iter_mut().filter_map(|op| match op { 112 112 + BatchOperation::Unmap(op) => Some(op), 113 113 + _ => None, 114 114 + }); 115 115 + 116 116 + for op in ops { 117 117 + match op.try_merge_with(new) { 118 118 + Ok(()) => return, 119 119 + Err(new_) => new = new_, 120 120 + } 121 121 + } 122 122 + 123 123 + self.ops.push(BatchOperation::Unmap(new)); 124 124 + } 125 125 + 126 126 + /// Add a [`set_access_rules`] operation to the set of batched operations. 127 127 + /// 128 128 + /// # Safety 129 129 + /// 130 130 + /// - virt..virt+len must be mapped 131 131 + /// - `virt` must be aligned to `Self::PAGE_SIZE` 132 132 + /// - `phys` must be aligned to `Self::PAGE_SIZE` 133 133 + /// - `len` must an integer multiple of `Self::PAGE_SIZE` 134 134 + /// 135 135 + /// [`set_access_rules`]: RawAddressSpace::set_access_rules 136 136 + pub fn set_access_rules( 137 137 + &mut self, 138 138 + virt: VirtualAddress, 139 139 + len: NonZeroUsize, 140 140 + access_rules: AccessRules, 141 141 + ) { 142 142 + let mut new = SetAccessRulesOperation { 143 143 + virt, 144 144 + len, 145 145 + access_rules, 146 146 + }; 147 147 + 148 148 + let ops = self.ops.iter_mut().filter_map(|op| match op { 149 149 + BatchOperation::SetAccessRules(op) => Some(op), 150 150 + _ => None, 151 151 + }); 152 152 + 153 153 + for op in ops { 154 154 + match op.try_merge_with(new) { 155 155 + Ok(()) => return, 156 156 + Err(new_) => new = new_, 157 157 + } 158 158 + } 159 159 + 160 160 + self.ops.push(BatchOperation::SetAccessRules(new)); 161 161 + } 162 162 + 163 163 + /// Flushes the `Batch` ensuring all changes are materialized into the raw address space. 164 164 + pub fn flush_changes<A: RawAddressSpace>(&mut self, raw_aspace: &mut A) -> crate::Result<()> { 165 165 + let mut flush = raw_aspace.flush(); 166 166 + for op in self.ops.drain(..) { 167 167 + match op { 168 168 + BatchOperation::Map(op) => { 169 169 + debug_assert!(op.virt.is_aligned_to(A::PAGE_SIZE)); 170 170 + debug_assert!(op.phys.is_aligned_to(A::PAGE_SIZE)); 171 171 + debug_assert!(op.len.get().is_multiple_of(A::PAGE_SIZE)); 172 172 + 173 173 + // Safety: the caller promised the correctness of the values on construction of 174 174 + // the operation. 175 175 + unsafe { 176 176 + raw_aspace.map(op.virt, op.phys, op.len, op.access_rules, &mut flush)?; 177 177 + } 178 178 + } 179 179 + BatchOperation::Unmap(op) => { 180 180 + debug_assert!(op.virt.is_aligned_to(A::PAGE_SIZE)); 181 181 + debug_assert!(op.len.get().is_multiple_of(A::PAGE_SIZE)); 182 182 + 183 183 + // Safety: the caller promised the correctness of the values on construction of 184 184 + // the operation. 185 185 + unsafe { 186 186 + raw_aspace.unmap(op.virt, op.len, &mut flush); 187 187 + } 188 188 + } 189 189 + BatchOperation::SetAccessRules(op) => { 190 190 + debug_assert!(op.virt.is_aligned_to(A::PAGE_SIZE)); 191 191 + debug_assert!(op.len.get().is_multiple_of(A::PAGE_SIZE)); 192 192 + 193 193 + // Safety: the caller promised the correctness of the values on construction of 194 194 + // the operation. 195 195 + unsafe { 196 196 + raw_aspace.set_access_rules(op.virt, op.len, op.access_rules, &mut flush); 197 197 + } 198 198 + } 199 199 + }; 200 200 + } 201 201 + flush.flush() 202 202 + } 203 203 + } 204 204 + 205 205 + // ===== impl MapOperation ===== 206 206 + 207 207 + impl MapOperation { 208 208 + /// Returns true if this operation can be merged with `other`. 209 209 + /// 210 210 + /// Map operations can be merged if: 211 211 + /// - their [`AccessRules`] are the same 212 212 + /// - their virtual address ranges are contiguous (no gap between self and other) 213 213 + /// - their physical address ranges are contiguous 214 214 + /// - the resulting virtual address range still has the same size as the resulting 215 215 + /// physical address range 216 216 + const fn can_merge_with(&self, other: &Self) -> bool { 217 217 + // the access rules need to be the same 218 218 + let same_rules = self.access_rules.bits() == other.access_rules.bits(); 219 219 + 220 220 + let overlap_virt = self.virt.get() <= other.len.get() 221 221 + && other.virt.get() <= self.virt.get() + self.len.get(); 222 222 + 223 223 + let overlap_phys = self.phys.get() <= other.len.get() 224 224 + && other.phys.get() <= self.phys.get() + self.len.get(); 225 225 + 226 226 + let offset_virt = self.virt.get().wrapping_sub(other.virt.get()); 227 227 + let offset_phys = self.virt.get().wrapping_sub(other.virt.get()); 228 228 + let same_offset = offset_virt == offset_phys; 229 229 + 230 230 + same_rules && overlap_virt && overlap_phys && same_offset 231 231 + } 232 232 + 233 233 + /// Attempt to merge this operation with `other`. 234 234 + /// 235 235 + /// If this returns `Ok`, `other` has been merged into `self`. 236 236 + /// 237 237 + /// If this returns `Err`, `other` cannot be merged and is returned in the `Err` variant. 238 238 + fn try_merge_with(&mut self, other: Self) -> Result<(), Self> { 239 239 + if self.can_merge_with(&other) { 240 240 + let offset = self.virt.get().wrapping_sub(other.virt.get()); 241 241 + let len = self 242 242 + .len 243 243 + .get() 244 244 + .checked_add(other.len.get()) 245 245 + .unwrap() 246 246 + .wrapping_add(offset); 247 247 + 248 248 + self.virt = cmp::min(self.virt, other.virt); 249 249 + self.phys = cmp::min(self.phys, other.phys); 250 250 + self.len = NonZero::new(len).ok_or(other)?; 251 251 + 252 252 + Ok(()) 253 253 + } else { 254 254 + Err(other) 255 255 + } 256 256 + } 257 257 + } 258 258 + 259 259 + // ===== impl UnmapOperation ===== 260 260 + 261 261 + impl UnmapOperation { 262 262 + /// Returns true if this operation can be merged with `other`. 263 263 + /// 264 264 + /// Unmap operations can be merged if: 265 265 + /// - their virtual address ranges are contiguous (no gap between self and other) 266 266 + const fn can_merge_with(&self, other: &Self) -> bool { 267 267 + self.virt.get() <= other.len.get() && other.virt.get() <= self.virt.get() + self.len.get() 268 268 + } 269 269 + 270 270 + /// Attempt to merge this operation with `other`. 271 271 + /// 272 272 + /// If this returns `Ok`, `other` has been merged into `self`. 273 273 + /// 274 274 + /// If this returns `Err`, `other` cannot be merged and is returned in the `Err` variant. 275 275 + fn try_merge_with(&mut self, other: Self) -> Result<(), Self> { 276 276 + if self.can_merge_with(&other) { 277 277 + let offset = self.virt.get().wrapping_sub(other.virt.get()); 278 278 + let len = self 279 279 + .len 280 280 + .get() 281 281 + .checked_add(other.len.get()) 282 282 + .unwrap() 283 283 + .wrapping_add(offset); 284 284 + 285 285 + self.virt = cmp::min(self.virt, other.virt); 286 286 + self.len = NonZero::new(len).ok_or(other)?; 287 287 + 288 288 + Ok(()) 289 289 + } else { 290 290 + Err(other) 291 291 + } 292 292 + } 293 293 + } 294 294 + 295 295 + // ===== impl ProtectOperation ===== 296 296 + 297 297 + impl SetAccessRulesOperation { 298 298 + /// Returns true if this operation can be merged with `other`. 299 299 + /// 300 300 + /// Protect operations can be merged if: 301 301 + /// - their [`AccessRules`] are the same 302 302 + /// - their virtual address ranges are contiguous (no gap between self and other) 303 303 + const fn can_merge_with(&self, other: &Self) -> bool { 304 304 + // the access rules need to be the same 305 305 + let same_rules = self.access_rules.bits() == other.access_rules.bits(); 306 306 + 307 307 + let overlap = self.virt.get() <= other.len.get() 308 308 + && other.virt.get() <= self.virt.get() + self.len.get(); 309 309 + 310 310 + same_rules && overlap 311 311 + } 312 312 + 313 313 + /// Attempt to merge this operation with `other`. 314 314 + /// 315 315 + /// If this returns `Ok`, `other` has been merged into `self`. 316 316 + /// 317 317 + /// If this returns `Err`, `other` cannot be merged and is returned in the `Err` variant. 318 318 + fn try_merge_with(&mut self, other: Self) -> Result<(), Self> { 319 319 + if self.can_merge_with(&other) { 320 320 + let offset = self.virt.get().wrapping_sub(other.virt.get()); 321 321 + let len = self 322 322 + .len 323 323 + .get() 324 324 + .checked_add(other.len.get()) 325 325 + .unwrap() 326 326 + .wrapping_add(offset); 327 327 + 328 328 + self.virt = cmp::min(self.virt, other.virt); 329 329 + self.len = NonZero::new(len).ok_or(other)?; 330 330 + 331 331 + Ok(()) 332 332 + } else { 333 333 + Err(other) 334 334 + } 335 335 + } 336 336 + }

+376

libs/mem/src/address_space/region.rs

reviewed

··· 1 1 + // Copyright 2025. Jonas Kruckenberg 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or 4 4 + // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or 5 5 + // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 + // copied, modified, or distributed except according to those terms. 7 7 + 8 8 + use alloc::boxed::Box; 9 9 + use core::alloc::Layout; 10 10 + use core::mem::offset_of; 11 11 + use core::ops::{Range, RangeBounds}; 12 12 + use core::pin::Pin; 13 13 + use core::ptr::NonNull; 14 14 + use core::{cmp, mem, slice}; 15 15 + 16 16 + use anyhow::bail; 17 17 + use pin_project::pin_project; 18 18 + 19 19 + use crate::address_space::batch::Batch; 20 20 + use crate::{AccessRules, AddressRangeExt, VirtualAddress}; 21 21 + 22 22 + #[pin_project] 23 23 + #[derive(Debug)] 24 24 + pub struct AddressSpaceRegion { 25 25 + access_rules: AccessRules, 26 26 + layout: Layout, 27 27 + range: Range<VirtualAddress>, 28 28 + /// The address range covered by this region and its WAVL tree subtree, used when allocating new regions 29 29 + subtree_range: Range<VirtualAddress>, 30 30 + /// The largest gap in this subtree, used when allocating new regions 31 31 + max_gap: usize, 32 32 + /// Links to other regions in the WAVL tree 33 33 + links: wavltree::Links<AddressSpaceRegion>, 34 34 + } 35 35 + 36 36 + impl AddressSpaceRegion { 37 37 + pub const fn new(spot: VirtualAddress, layout: Layout, access_rules: AccessRules) -> Self { 38 38 + Self { 39 39 + range: spot..spot.checked_add(layout.size()).unwrap(), 40 40 + access_rules, 41 41 + layout, 42 42 + 43 43 + max_gap: 0, 44 44 + subtree_range: spot..spot.checked_add(layout.size()).unwrap(), 45 45 + links: wavltree::Links::new(), 46 46 + } 47 47 + } 48 48 + 49 49 + pub const fn into_parts(self) -> (VirtualAddress, Layout, AccessRules) { 50 50 + debug_assert!(!self.links.is_linked()); 51 51 + 52 52 + // TODO add VMO here 53 53 + (self.range.start, self.layout, self.access_rules) 54 54 + } 55 55 + 56 56 + pub const fn range(&self) -> &Range<VirtualAddress> { 57 57 + &self.range 58 58 + } 59 59 + 60 60 + pub const fn subtree_range(&self) -> &Range<VirtualAddress> { 61 61 + &self.subtree_range 62 62 + } 63 63 + 64 64 + pub const fn access_rules(&self) -> AccessRules { 65 65 + self.access_rules 66 66 + } 67 67 + 68 68 + pub fn as_slice(&self) -> &[u8] { 69 69 + let ptr = self.range.start.as_ptr(); 70 70 + let len = self.range.size(); 71 71 + 72 72 + unsafe { slice::from_raw_parts(ptr, len) } 73 73 + } 74 74 + 75 75 + pub fn as_slice_mut(&mut self) -> &mut [u8] { 76 76 + let ptr = self.range.start.as_mut_ptr(); 77 77 + let len = self.range.size(); 78 78 + 79 79 + unsafe { slice::from_raw_parts_mut(ptr, len) } 80 80 + } 81 81 + 82 82 + pub fn as_non_null(&self) -> NonNull<[u8]> { 83 83 + let ptr = self.range.start.as_non_null().unwrap(); 84 84 + NonNull::slice_from_raw_parts(ptr, self.range.size()) 85 85 + } 86 86 + 87 87 + pub const fn layout_fits_region(&self, layout: Layout) -> bool { 88 88 + self.range.start.is_aligned_to(layout.align()) 89 89 + && layout.size() >= self.layout.size() 90 90 + && layout.size() <= self.range.end.get() - self.range.start.get() 91 91 + } 92 92 + 93 93 + /// grow region to `new_len`, attempting to grow the VMO accordingly 94 94 + /// `new_layout.size()` mut be greater than or equal to `self.layout.size()` 95 95 + pub fn grow_in_place( 96 96 + &mut self, 97 97 + new_layout: Layout, 98 98 + next_range: Option<Range<VirtualAddress>>, 99 99 + _batch: &mut Batch, 100 100 + ) -> crate::Result<()> { 101 101 + if new_layout.align() > self.layout.align() { 102 102 + bail!("cannot grow in-place: New alignment greater than current"); 103 103 + } 104 104 + 105 105 + let new_range = self.range.start..self.range.start.checked_add(new_layout.size()).unwrap(); 106 106 + 107 107 + if let Some(next_range) = next_range 108 108 + && next_range.is_overlapping(&new_range) 109 109 + { 110 110 + bail!("cannot grow in-place: New overlapping with next range"); 111 111 + } 112 112 + 113 113 + // TODO attempt to resize VMO 114 114 + self.update_range(new_range); 115 115 + 116 116 + Ok(()) 117 117 + } 118 118 + 119 119 + /// shrink region to the first `len` bytes, dropping the rest frames. 120 120 + /// `new_layout.size()` mut be smaller than or equal to `self.layout.size()` 121 121 + pub fn shrink(&mut self, new_layout: Layout, _batch: &mut Batch) -> crate::Result<()> { 122 122 + if new_layout.align() > self.layout.align() { 123 123 + bail!("cannot grow in-place: New alignment greater than current"); 124 124 + } 125 125 + 126 126 + let new_range = self.range.start..self.range.start.checked_add(new_layout.size()).unwrap(); 127 127 + 128 128 + // TODO drop rest pages in VMO if possible (add unmaps to batch) 129 129 + self.update_range(new_range); 130 130 + 131 131 + Ok(()) 132 132 + } 133 133 + 134 134 + /// move the entire region to the new base address, remapping any already mapped frames 135 135 + pub fn move_to(&mut self, base: VirtualAddress, batch: &mut Batch) -> crate::Result<()> { 136 136 + // TODO 137 137 + // - attempt to resize VMO 138 138 + // - update self range 139 139 + // - for every frame in VMO 140 140 + // - attempt to map at new offset (add maps to batch) 141 141 + 142 142 + todo!() 143 143 + } 144 144 + 145 145 + pub fn commit<R>(&mut self, range: R, will_write: bool, batch: &mut Batch) -> crate::Result<()> 146 146 + where 147 147 + R: RangeBounds<VirtualAddress>, 148 148 + { 149 149 + // TODO 150 150 + // - for every *uncommited* frame in range 151 151 + // - request frame from VMO (add map to batch) 152 152 + 153 153 + todo!() 154 154 + } 155 155 + 156 156 + pub fn decommit<R>(&mut self, range: R, batch: &mut Batch) -> crate::Result<()> 157 157 + where 158 158 + R: RangeBounds<VirtualAddress>, 159 159 + { 160 160 + // TODO 161 161 + // - for every *committed* frame in range 162 162 + // - drop pages in VMO if possible (add unmaps to batch) 163 163 + 164 164 + todo!() 165 165 + } 166 166 + 167 167 + /// updates the access rules fo this region 168 168 + pub fn update_access_rules( 169 169 + &mut self, 170 170 + access_rules: AccessRules, 171 171 + batch: &mut Batch, 172 172 + ) -> crate::Result<()> { 173 173 + // TODO 174 174 + // - for every frame in VMO 175 175 + // - update access rules (add protects to batch) 176 176 + // - update self access rules 177 177 + 178 178 + todo!() 179 179 + } 180 180 + 181 181 + pub fn clear(&mut self, batch: &mut Batch) -> crate::Result<()> { 182 182 + // TODO 183 183 + // - replace VMO with "zeroed" VMO 184 184 + // - drop pages in VMO if possible (add unmaps to batch) 185 185 + 186 186 + todo!() 187 187 + } 188 188 + 189 189 + pub fn assert_valid(&self, msg: &str) { 190 190 + assert!(!self.range.is_empty(), "{msg}region range cannot be empty"); 191 191 + assert!( 192 192 + self.subtree_range.start <= self.range.start 193 193 + && self.range.end <= self.subtree_range.end, 194 194 + "{msg}region range cannot be bigger than its subtree range; region={self:?}" 195 195 + ); 196 196 + assert!( 197 197 + self.max_gap < self.subtree_range.size(), 198 198 + "{msg}region's subtree max_gap cannot be bigger than its subtree range; region={self:?}" 199 199 + ); 200 200 + assert!( 201 201 + self.range.start.is_aligned_to(self.layout.align()), 202 202 + "{msg}region range is not aligned to its layout; region={self:?}" 203 203 + ); 204 204 + assert!( 205 205 + self.range.size() >= self.layout.size(), 206 206 + "{msg}region range is smaller than its layout; region={self:?}" 207 207 + ); 208 208 + 209 209 + self.links.assert_valid(); 210 210 + } 211 211 + 212 212 + /// Returns `true` if this nodes subtree contains a gap suitable for the given `layout`, used 213 213 + /// during gap-searching. 214 214 + pub fn suitable_gap_in_subtree(&self, layout: Layout) -> bool { 215 215 + // we need the layout to be padded to alignment 216 216 + debug_assert!(layout.size().is_multiple_of(layout.align())); 217 217 + 218 218 + self.max_gap >= layout.size() 219 219 + } 220 220 + 221 221 + /// Returns the left child node in the search tree of regions, used during gap-searching. 222 222 + pub fn left_child(&self) -> Option<&Self> { 223 223 + Some(unsafe { self.links.left()?.as_ref() }) 224 224 + } 225 225 + 226 226 + /// Returns the right child node in the search tree of regions, used during gap-searching. 227 227 + pub fn right_child(&self) -> Option<&Self> { 228 228 + Some(unsafe { self.links.right()?.as_ref() }) 229 229 + } 230 230 + 231 231 + /// Returns the parent node in the search tree of regions, used during gap-searching. 232 232 + pub fn parent(&self) -> Option<&Self> { 233 233 + Some(unsafe { self.links.parent()?.as_ref() }) 234 234 + } 235 235 + 236 236 + fn update_range(&mut self, new_range: Range<VirtualAddress>) { 237 237 + self.range = new_range; 238 238 + // We also must propagate the information about our changed range to the rest of the tree 239 239 + // so searching for a free spot returns the correct results. 240 240 + Self::propagate_update_to_parent(Some(NonNull::from(self))); 241 241 + } 242 242 + 243 243 + /// Update the gap search metadata of this region. This method is called in the [`wavltree::Linked`] 244 244 + /// implementation below after each tree mutation that impacted this node or its subtree in some way 245 245 + /// (insertion, rotation, deletion). 246 246 + /// 247 247 + /// Returns `true` if this nodes metadata changed. 248 248 + #[expect(clippy::undocumented_unsafe_blocks, reason = "intrusive tree access")] 249 249 + fn update_gap_metadata( 250 250 + mut node: NonNull<Self>, 251 251 + left: Option<NonNull<Self>>, 252 252 + right: Option<NonNull<Self>>, 253 253 + ) -> bool { 254 254 + fn gap(left_last_byte: VirtualAddress, right_first_byte: VirtualAddress) -> usize { 255 255 + right_first_byte 256 256 + .checked_sub_addr(left_last_byte) 257 257 + .unwrap_or_default() // TODO use saturating_sub_addr 258 258 + } 259 259 + 260 260 + let node = unsafe { node.as_mut() }; 261 261 + let mut left_max_gap = 0; 262 262 + let mut right_max_gap = 0; 263 263 + 264 264 + // recalculate the subtree_range start 265 265 + let old_subtree_range_start = if let Some(left) = left { 266 266 + let left = unsafe { left.as_ref() }; 267 267 + let left_gap = gap(left.subtree_range.end, node.range.start); 268 268 + left_max_gap = cmp::max(left_gap, left.max_gap); 269 269 + mem::replace(&mut node.subtree_range.start, left.subtree_range.start) 270 270 + } else { 271 271 + mem::replace(&mut node.subtree_range.start, node.range.start) 272 272 + }; 273 273 + 274 274 + // recalculate the subtree range end 275 275 + let old_subtree_range_end = if let Some(right) = right { 276 276 + let right = unsafe { right.as_ref() }; 277 277 + let right_gap = gap(node.range.end, right.subtree_range.start); 278 278 + right_max_gap = cmp::max(right_gap, right.max_gap); 279 279 + mem::replace(&mut node.subtree_range.end, right.subtree_range.end) 280 280 + } else { 281 281 + mem::replace(&mut node.subtree_range.end, node.range.end) 282 282 + }; 283 283 + 284 284 + // recalculate the map_gap 285 285 + let old_max_gap = mem::replace(&mut node.max_gap, cmp::max(left_max_gap, right_max_gap)); 286 286 + 287 287 + old_max_gap != node.max_gap 288 288 + || old_subtree_range_start != node.subtree_range.start 289 289 + || old_subtree_range_end != node.subtree_range.end 290 290 + } 291 291 + 292 292 + // Propagate metadata updates to this regions parent in the search tree. If we had to update 293 293 + // our metadata the parent must update its metadata too. 294 294 + #[expect(clippy::undocumented_unsafe_blocks, reason = "intrusive tree access")] 295 295 + fn propagate_update_to_parent(mut maybe_node: Option<NonNull<Self>>) { 296 296 + while let Some(node) = maybe_node { 297 297 + let links = unsafe { &node.as_ref().links }; 298 298 + let changed = Self::update_gap_metadata(node, links.left(), links.right()); 299 299 + 300 300 + // if the metadata didn't actually change, we don't need to recalculate parents 301 301 + if !changed { 302 302 + return; 303 303 + } 304 304 + 305 305 + maybe_node = links.parent(); 306 306 + } 307 307 + } 308 308 + } 309 309 + 310 310 + unsafe impl wavltree::Linked for AddressSpaceRegion { 311 311 + /// Any heap-allocated type that owns an element may be used. 312 312 + /// 313 313 + /// An element *must not* move while part of an intrusive data 314 314 + /// structure. In many cases, `Pin` may be used to enforce this. 315 315 + type Handle = Pin<Box<Self>>; // TODO better handle type 316 316 + 317 317 + type Key = VirtualAddress; 318 318 + 319 319 + /// Convert an owned `Handle` into a raw pointer 320 320 + fn into_ptr(handle: Self::Handle) -> NonNull<Self> { 321 321 + // Safety: wavltree treats the ptr as pinned 322 322 + unsafe { NonNull::from(Box::leak(Pin::into_inner_unchecked(handle))) } 323 323 + } 324 324 + 325 325 + /// Convert a raw pointer back into an owned `Handle`. 326 326 + unsafe fn from_ptr(ptr: NonNull<Self>) -> Self::Handle { 327 327 + // Safety: `NonNull` *must* be constructed from a pinned reference 328 328 + // which the tree implementation upholds. 329 329 + unsafe { Pin::new_unchecked(Box::from_raw(ptr.as_ptr())) } 330 330 + } 331 331 + 332 332 + unsafe fn links(ptr: NonNull<Self>) -> NonNull<wavltree::Links<Self>> { 333 333 + ptr.map_addr(|addr| { 334 334 + let offset = offset_of!(Self, links); 335 335 + addr.checked_add(offset).unwrap() 336 336 + }) 337 337 + .cast() 338 338 + } 339 339 + 340 340 + fn get_key(&self) -> &Self::Key { 341 341 + &self.range.start 342 342 + } 343 343 + 344 344 + fn after_insert(self: Pin<&mut Self>) { 345 345 + debug_assert_eq!(self.subtree_range.start, self.range.start); 346 346 + debug_assert_eq!(self.subtree_range.end, self.range.end); 347 347 + debug_assert_eq!(self.max_gap, 0); 348 348 + Self::propagate_update_to_parent(self.links.parent()); 349 349 + } 350 350 + 351 351 + fn after_remove(self: Pin<&mut Self>, parent: Option<NonNull<Self>>) { 352 352 + Self::propagate_update_to_parent(parent); 353 353 + } 354 354 + 355 355 + fn after_rotate( 356 356 + self: Pin<&mut Self>, 357 357 + parent: NonNull<Self>, 358 358 + sibling: Option<NonNull<Self>>, 359 359 + lr_child: Option<NonNull<Self>>, 360 360 + side: wavltree::Side, 361 361 + ) { 362 362 + let this = self.project(); 363 363 + // Safety: caller ensures ptr is valid 364 364 + let _parent = unsafe { parent.as_ref() }; 365 365 + 366 366 + this.subtree_range.start = _parent.subtree_range.start; 367 367 + this.subtree_range.end = _parent.subtree_range.end; 368 368 + *this.max_gap = _parent.max_gap; 369 369 + 370 370 + if side == wavltree::Side::Left { 371 371 + Self::update_gap_metadata(parent, sibling, lr_child); 372 372 + } else { 373 373 + Self::update_gap_metadata(parent, lr_child, sibling); 374 374 + } 375 375 + } 376 376 + }

+414

libs/mem/src/addresses.rs

reviewed

··· 1 1 + // Copyright 2025. Jonas Kruckenberg 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or 4 4 + // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or 5 5 + // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 + // copied, modified, or distributed except according to those terms. 7 7 + 8 8 + use core::alloc::{Layout, LayoutError}; 9 9 + use core::ops::Range; 10 10 + 11 11 + use crate::address_space::RawAddressSpace; 12 12 + 13 13 + macro_rules! impl_address { 14 14 + ($address_ty:ident) => { 15 15 + impl $address_ty { 16 16 + pub const MAX: Self = Self(usize::MAX); 17 17 + pub const MIN: Self = Self(0); 18 18 + pub const ZERO: Self = Self(0); 19 19 + pub const BITS: u32 = usize::BITS; 20 20 + 21 21 + #[inline] 22 22 + pub const fn get(&self) -> usize { 23 23 + self.0 24 24 + } 25 25 + 26 26 + #[must_use] 27 27 + #[inline] 28 28 + pub fn from_ptr<T: ?Sized>(ptr: *const T) -> Self { 29 29 + Self(ptr.expose_provenance()) 30 30 + } 31 31 + 32 32 + #[must_use] 33 33 + #[inline] 34 34 + pub fn from_mut_ptr<T: ?Sized>(ptr: *mut T) -> Self { 35 35 + Self(ptr.expose_provenance()) 36 36 + } 37 37 + 38 38 + #[must_use] 39 39 + #[inline] 40 40 + pub fn from_non_null<T: ?Sized>(ptr: ::core::ptr::NonNull<T>) -> Self { 41 41 + Self(ptr.addr().get()) 42 42 + } 43 43 + 44 44 + #[inline] 45 45 + pub fn as_ptr(self) -> *const u8 { 46 46 + ::core::ptr::with_exposed_provenance(self.0) 47 47 + } 48 48 + 49 49 + #[inline] 50 50 + pub fn as_mut_ptr(self) -> *mut u8 { 51 51 + ::core::ptr::with_exposed_provenance_mut(self.0) 52 52 + } 53 53 + 54 54 + #[inline] 55 55 + pub fn as_non_null(self) -> Option<::core::ptr::NonNull<u8>> { 56 56 + ::core::num::NonZeroUsize::new(self.0) 57 57 + .map(::core::ptr::NonNull::with_exposed_provenance) 58 58 + } 59 59 + 60 60 + #[must_use] 61 61 + #[inline] 62 62 + pub const fn checked_add(self, rhs: usize) -> Option<Self> { 63 63 + if let Some(out) = self.0.checked_add(rhs) { 64 64 + Some(Self(out)) 65 65 + } else { 66 66 + None 67 67 + } 68 68 + } 69 69 + 70 70 + #[must_use] 71 71 + #[inline] 72 72 + pub const fn checked_add_signed(self, rhs: isize) -> Option<Self> { 73 73 + if let Some(out) = self.0.checked_add_signed(rhs) { 74 74 + Some(Self(out)) 75 75 + } else { 76 76 + None 77 77 + } 78 78 + } 79 79 + 80 80 + #[must_use] 81 81 + #[inline] 82 82 + pub const fn checked_sub(self, rhs: usize) -> Option<Self> { 83 83 + if let Some(out) = self.0.checked_sub(rhs) { 84 84 + Some(Self(out)) 85 85 + } else { 86 86 + None 87 87 + } 88 88 + } 89 89 + #[must_use] 90 90 + #[inline] 91 91 + pub const fn checked_div(self, rhs: usize) -> Option<Self> { 92 92 + if let Some(out) = self.0.checked_div(rhs) { 93 93 + Some(Self(out)) 94 94 + } else { 95 95 + None 96 96 + } 97 97 + } 98 98 + #[must_use] 99 99 + #[inline] 100 100 + pub const fn checked_mul(self, rhs: usize) -> Option<Self> { 101 101 + if let Some(out) = self.0.checked_mul(rhs) { 102 102 + Some(Self(out)) 103 103 + } else { 104 104 + None 105 105 + } 106 106 + } 107 107 + #[must_use] 108 108 + #[inline] 109 109 + pub const fn checked_shl(self, rhs: u32) -> Option<Self> { 110 110 + if let Some(out) = self.0.checked_shl(rhs) { 111 111 + Some(Self(out)) 112 112 + } else { 113 113 + None 114 114 + } 115 115 + } 116 116 + #[must_use] 117 117 + #[inline] 118 118 + pub const fn checked_shr(self, rhs: u32) -> Option<Self> { 119 119 + if let Some(out) = self.0.checked_shr(rhs) { 120 120 + Some(Self(out)) 121 121 + } else { 122 122 + None 123 123 + } 124 124 + } 125 125 + // #[must_use] 126 126 + // #[inline] 127 127 + // pub const fn saturating_add(self, rhs: usize) -> Self { 128 128 + // Self(self.0.saturating_add(rhs)) 129 129 + // } 130 130 + // #[must_use] 131 131 + // #[inline] 132 132 + // pub const fn saturating_add_signed(self, rhs: isize) -> Self { 133 133 + // Self(self.0.saturating_add_signed(rhs)) 134 134 + // } 135 135 + // #[must_use] 136 136 + // #[inline] 137 137 + // pub const fn saturating_div(self, rhs: usize) -> Self { 138 138 + // Self(self.0.saturating_div(rhs)) 139 139 + // } 140 140 + // #[must_use] 141 141 + // #[inline] 142 142 + // pub const fn saturating_sub(self, rhs: usize) -> Self { 143 143 + // Self(self.0.saturating_sub(rhs)) 144 144 + // } 145 145 + // #[must_use] 146 146 + // #[inline] 147 147 + // pub const fn saturating_mul(self, rhs: usize) -> Self { 148 148 + // Self(self.0.saturating_mul(rhs)) 149 149 + // } 150 150 + #[must_use] 151 151 + #[inline] 152 152 + pub const fn overflowing_shl(self, rhs: u32) -> (Self, bool) { 153 153 + let (a, b) = self.0.overflowing_shl(rhs); 154 154 + (Self(a), b) 155 155 + } 156 156 + #[must_use] 157 157 + #[inline] 158 158 + pub const fn overflowing_shr(self, rhs: u32) -> (Self, bool) { 159 159 + let (a, b) = self.0.overflowing_shr(rhs); 160 160 + (Self(a), b) 161 161 + } 162 162 + 163 163 + #[must_use] 164 164 + #[inline] 165 165 + pub const fn checked_sub_addr(self, rhs: Self) -> Option<usize> { 166 166 + self.0.checked_sub(rhs.0) 167 167 + } 168 168 + 169 169 + // #[must_use] 170 170 + // #[inline] 171 171 + // pub const fn saturating_sub_addr(self, rhs: Self) -> usize { 172 172 + // self.0.saturating_sub(rhs.0) 173 173 + // } 174 174 + 175 175 + #[must_use] 176 176 + #[inline] 177 177 + pub const fn is_aligned_to(&self, align: usize) -> bool { 178 178 + assert!( 179 179 + align.is_power_of_two(), 180 180 + "is_aligned_to: align is not a power-of-two" 181 181 + ); 182 182 + 183 183 + self.0 & (align - 1) == 0 184 184 + } 185 185 + 186 186 + #[must_use] 187 187 + #[inline] 188 188 + pub const fn checked_align_up(self, align: usize) -> Option<Self> { 189 189 + if !align.is_power_of_two() { 190 190 + panic!("checked_align_up: align is not a power-of-two"); 191 191 + } 192 192 + 193 193 + // SAFETY: `align` has been checked to be a power of 2 above 194 194 + let align_minus_one = unsafe { align.unchecked_sub(1) }; 195 195 + 196 196 + // addr.wrapping_add(align_minus_one) & 0usize.wrapping_sub(align) 197 197 + if let Some(addr_plus_align) = self.0.checked_add(align_minus_one) { 198 198 + let aligned = Self(addr_plus_align & 0usize.wrapping_sub(align)); 199 199 + debug_assert!(aligned.is_aligned_to(align)); 200 200 + debug_assert!(aligned.0 >= self.0); 201 201 + Some(aligned) 202 202 + } else { 203 203 + None 204 204 + } 205 205 + } 206 206 + 207 207 + // #[must_use] 208 208 + // #[inline] 209 209 + // pub const fn wrapping_align_up(self, align: usize) -> Self { 210 210 + // if !align.is_power_of_two() { 211 211 + // panic!("checked_align_up: align is not a power-of-two"); 212 212 + // } 213 213 + // 214 214 + // // SAFETY: `align` has been checked to be a power of 2 above 215 215 + // let align_minus_one = unsafe { align.unchecked_sub(1) }; 216 216 + // 217 217 + // // addr.wrapping_add(align_minus_one) & 0usize.wrapping_sub(align) 218 218 + // let out = addr.wrapping_add(align_minus_one) & 0usize.wrapping_sub(align); 219 219 + // debug_assert!(out.is_aligned_to(align)); 220 220 + // out 221 221 + // } 222 222 + 223 223 + #[inline] 224 224 + pub const fn alignment(&self) -> usize { 225 225 + self.0 & (!self.0 + 1) 226 226 + } 227 227 + 228 228 + #[must_use] 229 229 + #[inline] 230 230 + pub const fn align_down(self, align: usize) -> Self { 231 231 + if !align.is_power_of_two() { 232 232 + panic!("checked_align_up: align is not a power-of-two"); 233 233 + } 234 234 + 235 235 + let aligned = Self(self.0 & 0usize.wrapping_sub(align)); 236 236 + debug_assert!(aligned.is_aligned_to(align)); 237 237 + debug_assert!(aligned.0 <= self.0); 238 238 + aligned 239 239 + } 240 240 + } 241 241 + 242 242 + impl ::core::fmt::Display for $address_ty { 243 243 + fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { 244 244 + f.write_fmt(format_args!("{:#018x}", self.0)) // 18 digits to account for the leading 0x 245 245 + } 246 246 + } 247 247 + 248 248 + impl ::core::fmt::Debug for $address_ty { 249 249 + fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { 250 250 + f.debug_tuple(stringify!($address_ty)) 251 251 + .field(&format_args!("{:#018x}", self.0)) // 18 digits to account for the leading 0x 252 252 + .finish() 253 253 + } 254 254 + } 255 255 + }; 256 256 + } 257 257 + 258 258 + #[repr(transparent)] 259 259 + #[derive(Default, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] 260 260 + pub struct VirtualAddress(usize); 261 261 + impl_address!(VirtualAddress); 262 262 + 263 263 + impl VirtualAddress { 264 264 + #[must_use] 265 265 + pub const fn new(n: usize) -> Self { 266 266 + Self(n) 267 267 + } 268 268 + 269 269 + pub const fn is_canonical<A: RawAddressSpace>(self) -> bool { 270 270 + (self.0 & A::CANONICAL_ADDRESS_MASK).wrapping_sub(1) >= A::CANONICAL_ADDRESS_MASK - 1 271 271 + } 272 272 + 273 273 + #[inline] 274 274 + pub const fn is_user_accessible<A: RawAddressSpace>(self) -> bool { 275 275 + // This address refers to userspace if it is in the lower half of the 276 276 + // canonical addresses. IOW - if all of the bits in the canonical address 277 277 + // mask are zero. 278 278 + (self.0 & A::CANONICAL_ADDRESS_MASK) == 0 279 279 + } 280 280 + } 281 281 + 282 282 + #[repr(transparent)] 283 283 + #[derive(Default, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] 284 284 + pub struct PhysicalAddress(usize); 285 285 + impl_address!(PhysicalAddress); 286 286 + 287 287 + impl PhysicalAddress { 288 288 + pub const fn new(n: usize) -> Self { 289 289 + Self(n) 290 290 + } 291 291 + } 292 292 + 293 293 + macro_rules! address_range_impl { 294 294 + () => { 295 295 + fn size(&self) -> usize { 296 296 + debug_assert!(self.start <= self.end); 297 297 + let is = self.end.checked_sub_addr(self.start).unwrap_or_default(); 298 298 + let should = if self.is_empty() { 299 299 + 0 300 300 + } else { 301 301 + self.end.get() - self.start.get() 302 302 + }; 303 303 + debug_assert_eq!(is, should); 304 304 + is 305 305 + } 306 306 + fn checked_add(self, offset: usize) -> Option<Self> { 307 307 + Some(Range::from( 308 308 + self.start.checked_add(offset)?..self.end.checked_add(offset)?, 309 309 + )) 310 310 + } 311 311 + fn as_ptr_range(&self) -> Range<*const u8> { 312 312 + Range::from(self.start.as_ptr()..self.end.as_ptr()) 313 313 + } 314 314 + fn as_mut_ptr_range(&self) -> Range<*mut u8> { 315 315 + Range::from(self.start.as_mut_ptr()..self.end.as_mut_ptr()) 316 316 + } 317 317 + fn checked_align_in(self, align: usize) -> Option<Self> 318 318 + where 319 319 + Self: Sized, 320 320 + { 321 321 + let res = Range::from(self.start.checked_align_up(align)?..self.end.align_down(align)); 322 322 + Some(res) 323 323 + } 324 324 + fn checked_align_out(self, align: usize) -> Option<Self> 325 325 + where 326 326 + Self: Sized, 327 327 + { 328 328 + let res = Range::from(self.start.align_down(align)..self.end.checked_align_up(align)?); 329 329 + // aligning outwards can only increase the size 330 330 + debug_assert!(res.start.0 <= res.end.0); 331 331 + Some(res) 332 332 + } 333 333 + // fn saturating_align_in(self, align: usize) -> Self { 334 334 + // self.start.saturating_align_up(align)..self.end.saturating_align_down(align) 335 335 + // } 336 336 + // fn saturating_align_out(self, align: usize) -> Self { 337 337 + // self.start.saturating_align_down(align)..self.end.saturating_align_up(align) 338 338 + // } 339 339 + 340 340 + // TODO test 341 341 + fn alignment(&self) -> usize { 342 342 + self.start.alignment() 343 343 + } 344 344 + fn into_layout(self) -> core::result::Result<Layout, core::alloc::LayoutError> { 345 345 + Layout::from_size_align(self.size(), self.alignment()) 346 346 + } 347 347 + fn is_overlapping(&self, other: &Self) -> bool { 348 348 + (self.start < other.end) & (other.start < self.end) 349 349 + } 350 350 + fn difference(&self, other: Self) -> (Option<Self>, Option<Self>) { 351 351 + debug_assert!(self.is_overlapping(&other)); 352 352 + let a = Range::from(self.start..other.start); 353 353 + let b = Range::from(other.end..self.end); 354 354 + ((!a.is_empty()).then_some(a), (!b.is_empty()).then_some(b)) 355 355 + } 356 356 + fn clamp(&self, range: Self) -> Self { 357 357 + Range::from(self.start.max(range.start)..self.end.min(range.end)) 358 358 + } 359 359 + }; 360 360 + } 361 361 + 362 362 + pub trait AddressRangeExt { 363 363 + fn size(&self) -> usize; 364 364 + #[must_use] 365 365 + fn checked_add(self, offset: usize) -> Option<Self> 366 366 + where 367 367 + Self: Sized; 368 368 + #[must_use] 369 369 + fn as_ptr_range(&self) -> Range<*const u8>; 370 370 + #[must_use] 371 371 + fn as_mut_ptr_range(&self) -> Range<*mut u8>; 372 372 + #[must_use] 373 373 + fn checked_align_in(self, align: usize) -> Option<Self> 374 374 + where 375 375 + Self: Sized; 376 376 + #[must_use] 377 377 + fn checked_align_out(self, align: usize) -> Option<Self> 378 378 + where 379 379 + Self: Sized; 380 380 + // #[must_use] 381 381 + // fn saturating_align_in(self, align: usize) -> Self; 382 382 + // #[must_use] 383 383 + // fn saturating_align_out(self, align: usize) -> Self; 384 384 + fn alignment(&self) -> usize; 385 385 + fn into_layout(self) -> Result<Layout, LayoutError>; 386 386 + fn is_overlapping(&self, other: &Self) -> bool; 387 387 + fn difference(&self, other: Self) -> (Option<Self>, Option<Self>) 388 388 + where 389 389 + Self: Sized; 390 390 + fn clamp(&self, range: Self) -> Self; 391 391 + fn is_user_accessible<A: RawAddressSpace>(&self) -> bool; 392 392 + } 393 393 + 394 394 + impl AddressRangeExt for Range<PhysicalAddress> { 395 395 + address_range_impl!(); 396 396 + fn is_user_accessible<A: RawAddressSpace>(&self) -> bool { 397 397 + unimplemented!("PhysicalAddress is never user accessible") 398 398 + } 399 399 + } 400 400 + 401 401 + impl AddressRangeExt for Range<VirtualAddress> { 402 402 + address_range_impl!(); 403 403 + 404 404 + fn is_user_accessible<A: RawAddressSpace>(&self) -> bool { 405 405 + if self.is_empty() { 406 406 + return false; 407 407 + } 408 408 + let Some(end_minus_one) = self.end.checked_sub(1) else { 409 409 + return false; 410 410 + }; 411 411 + 412 412 + self.start.is_user_accessible::<A>() && end_minus_one.is_user_accessible::<A>() 413 413 + } 414 414 + }

+178

libs/mem/src/frame.rs

reviewed

··· 1 1 + // Copyright 2025. Jonas Kruckenberg 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or 4 4 + // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or 5 5 + // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 + // copied, modified, or distributed except according to those terms. 7 7 + 8 8 + use core::alloc::Layout; 9 9 + use core::cmp::PartialEq; 10 10 + use core::fmt::Debug; 11 11 + use core::mem::offset_of; 12 12 + use core::ptr::NonNull; 13 13 + use core::sync::atomic; 14 14 + use core::sync::atomic::{AtomicUsize, Ordering}; 15 15 + 16 16 + use cordyceps::{list, Linked}; 17 17 + use pin_project::pin_project; 18 18 + 19 19 + use crate::frame_alloc::FrameAllocator; 20 20 + use crate::PhysicalAddress; 21 21 + 22 22 + /// Soft limit on the amount of references that may be made to a `Frame`. 23 23 + const MAX_REFCOUNT: usize = isize::MAX as usize; 24 24 + 25 25 + pub struct FrameRef<A: FrameAllocator> { 26 26 + frame: NonNull<Frame>, 27 27 + alloc: A, 28 28 + } 29 29 + 30 30 + #[pin_project(!Unpin)] 31 31 + #[derive(Debug)] 32 32 + pub struct Frame { 33 33 + addr: PhysicalAddress, 34 34 + refcount: AtomicUsize, 35 35 + #[pin] 36 36 + links: list::Links<Self>, 37 37 + } 38 38 + 39 39 + // ===== impl FrameRef ===== 40 40 + 41 41 + // Safety: assert_impl_all! above ensures that `FrameInfo` is `Send` 42 42 + unsafe impl Send for Frame {} 43 43 + 44 44 + // Safety: assert_impl_all! above ensures that `FrameInfo` is `Sync` 45 45 + unsafe impl Sync for Frame {} 46 46 + 47 47 + impl<A: FrameAllocator + Clone> Clone for FrameRef<A> { 48 48 + /// Makes a clone of the `Frame`. 49 49 + /// 50 50 + /// This creates reference to the same `FrameInfo`, increasing the reference count by one. 51 51 + fn clone(&self) -> Self { 52 52 + // Increase the reference count by one. Using relaxed ordering, as knowledge of the 53 53 + // original reference prevents other threads from erroneously deleting 54 54 + // the object. 55 55 + // 56 56 + // Again, restating what the `Arc` implementation quotes from the 57 57 + // [Boost documentation][1]: 58 58 + // 59 59 + // > Increasing the reference counter can always be done with memory_order_relaxed: New 60 60 + // > references to an object can only be formed from an existing 61 61 + // > reference, and passing an existing reference from one thread to 62 62 + // > another must already provide any required synchronization. 63 63 + // 64 64 + // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html) 65 65 + let old_size = self.frame().refcount.fetch_add(1, Ordering::Relaxed); 66 66 + debug_assert_ne!(old_size, 0); 67 67 + 68 68 + // Just like with `Arc` we want to prevent excessive refcounts in the case that we are leaking 69 69 + // `Frame`s somewhere (which we really shouldn't but just in case). Overflowing the refcount 70 70 + // would *really* bad as it would treat the frame as free and potentially cause a use-after-free 71 71 + // scenario. Realistically this branch should never be taken. 72 72 + // 73 73 + // Also worth noting: Just like `Arc`, the refcount could still overflow when in between 74 74 + // the load above and this check some other cpu increased the refcount from `isize::MAX` to 75 75 + // `usize::MAX` but that seems unlikely. The other option, doing the comparison and update in 76 76 + // one conditional atomic operation produces much worse code, so if its good enough for the 77 77 + // standard library, it is good enough for us. 78 78 + assert!(old_size <= MAX_REFCOUNT, "Frame refcount overflow"); 79 79 + 80 80 + unsafe { Self::from_raw_parts(self.frame, self.alloc.clone()) } 81 81 + } 82 82 + } 83 83 + 84 84 + impl<A: FrameAllocator> Drop for FrameRef<A> { 85 85 + /// Drops the `Frame`. 86 86 + /// 87 87 + /// This will decrement the reference count. If the reference count reaches zero 88 88 + /// then this frame will be marked as free and returned to the frame allocator. 89 89 + fn drop(&mut self) { 90 90 + if self.frame().refcount.fetch_sub(1, Ordering::Release) != 1 { 91 91 + return; 92 92 + } 93 93 + 94 94 + // Ensure uses of `FrameInfo` happen before freeing it. 95 95 + // Because it is marked `Release`, the decreasing of the reference count synchronizes 96 96 + // with this `Acquire` fence. This means that use of `FrameInfo` happens before decreasing 97 97 + // the reference count, which happens before this fence, which happens before freeing `FrameInfo`. 98 98 + // 99 99 + // This section of the [Boost documentation][1] as quoted in Rusts `Arc` implementation and 100 100 + // may explain further: 101 101 + // 102 102 + // > It is important to enforce any possible access to the object in one 103 103 + // > thread (through an existing reference) to *happen before* deleting 104 104 + // > the object in a different thread. This is achieved by a "release" 105 105 + // > operation after dropping a reference (any access to the object 106 106 + // > through this reference must obviously happened before), and an 107 107 + // > "acquire" operation before deleting the object. 108 108 + // 109 109 + // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html) 110 110 + atomic::fence(Ordering::Acquire); 111 111 + 112 112 + self.drop_slow(); 113 113 + } 114 114 + } 115 115 + 116 116 + impl<A: FrameAllocator> FrameRef<A> { 117 117 + unsafe fn from_raw_parts(frame: NonNull<Frame>, alloc: A) -> Self { 118 118 + Self { frame, alloc } 119 119 + } 120 120 + 121 121 + fn frame(&self) -> &Frame { 122 122 + unsafe { self.frame.as_ref() } 123 123 + } 124 124 + 125 125 + #[inline(never)] 126 126 + fn drop_slow(&mut self) { 127 127 + let layout = unsafe { Layout::from_size_align_unchecked(A::FRAME_SIZE, A::FRAME_SIZE) }; 128 128 + unsafe { 129 129 + self.alloc.deallocate(self.frame, layout); 130 130 + } 131 131 + } 132 132 + } 133 133 + 134 134 + // ===== impl Frame ===== 135 135 + 136 136 + impl PartialEq<Frame> for &Frame { 137 137 + fn eq(&self, other: &Frame) -> bool { 138 138 + self.refcount() == other.refcount() && self.addr == other.addr 139 139 + } 140 140 + } 141 141 + 142 142 + impl Frame { 143 143 + pub fn new(addr: PhysicalAddress, initial_refcount: usize) -> Self { 144 144 + Self { 145 145 + addr, 146 146 + refcount: AtomicUsize::new(initial_refcount), 147 147 + links: list::Links::new(), 148 148 + } 149 149 + } 150 150 + 151 151 + pub fn refcount(&self) -> usize { 152 152 + self.refcount.load(Ordering::Relaxed) 153 153 + } 154 154 + 155 155 + pub fn addr(&self) -> PhysicalAddress { 156 156 + self.addr 157 157 + } 158 158 + } 159 159 + 160 160 + unsafe impl Linked<list::Links<Self>> for Frame { 161 161 + type Handle = NonNull<Self>; 162 162 + 163 163 + fn into_ptr(r: Self::Handle) -> NonNull<Self> { 164 164 + r 165 165 + } 166 166 + 167 167 + unsafe fn from_ptr(ptr: NonNull<Self>) -> Self::Handle { 168 168 + ptr 169 169 + } 170 170 + 171 171 + unsafe fn links(ptr: NonNull<Self>) -> NonNull<list::Links<Self>> { 172 172 + ptr.map_addr(|addr| { 173 173 + let offset = offset_of!(Self, links); 174 174 + addr.checked_add(offset).unwrap() 175 175 + }) 176 176 + .cast() 177 177 + } 178 178 + }

+135

libs/mem/src/frame_alloc.rs

reviewed

··· 1 1 + // Copyright 2025. Jonas Kruckenberg 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or 4 4 + // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or 5 5 + // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 + // copied, modified, or distributed except according to those terms. 7 7 + 8 8 + mod area; 9 9 + mod area_selection; 10 10 + 11 11 + use core::alloc::Layout; 12 12 + use core::cell::RefCell; 13 13 + use core::cmp; 14 14 + use core::ops::Range; 15 15 + use core::ptr::NonNull; 16 16 + use core::sync::atomic::{AtomicUsize, Ordering}; 17 17 + 18 18 + use cordyceps::List; 19 19 + use cpu_local::collection::CpuLocal; 20 20 + use fallible_iterator::FallibleIterator; 21 21 + use lock_api::Mutex; 22 22 + use smallvec::SmallVec; 23 23 + 24 24 + use crate::address_space::RawAddressSpace; 25 25 + use crate::frame_alloc::area::Area; 26 26 + use crate::frame_alloc::area_selection::select_areas; 27 27 + use crate::{Frame, PhysicalAddress}; 28 28 + 29 29 + #[derive(Debug)] 30 30 + pub struct AllocError; 31 31 + 32 32 + pub unsafe trait FrameAllocator: Send + Sync + 'static { 33 33 + const FRAME_SIZE: usize; 34 34 + fn allocate(&self, layout: Layout) -> Result<NonNull<[Frame]>, AllocError>; 35 35 + unsafe fn deallocate(&self, block: NonNull<Frame>, layout: Layout); 36 36 + } 37 37 + 38 38 + const MAX_FRAMES_IN_CACHE: usize = 256; 39 39 + 40 40 + pub struct FrameAlloc<L: lock_api::RawMutex, A: RawAddressSpace> { 41 41 + areas: Mutex<L, SmallVec<[Area<A>; 4]>>, 42 42 + cpu_local_cache: CpuLocal<RefCell<List<Frame>>>, 43 43 + max_alignment_hint: AtomicUsize, 44 44 + } 45 45 + 46 46 + impl<L: lock_api::RawMutex, A: RawAddressSpace> FrameAlloc<L, A> { 47 47 + pub fn new(allocatable_regions: SmallVec<[Range<PhysicalAddress>; 4]>) -> crate::Result<Self> { 48 48 + let mut max_alignment_hint = 0; 49 49 + let mut areas = SmallVec::new(); 50 50 + 51 51 + let mut selections = select_areas::<A>(allocatable_regions); 52 52 + while let Some(selection) = selections.next()? { 53 53 + let area = Area::new(selection.area, selection.bookkeeping); 54 54 + max_alignment_hint = cmp::max(max_alignment_hint, area.max_alignment_hint()); 55 55 + areas.push(area); 56 56 + } 57 57 + 58 58 + Ok(Self { 59 59 + areas: Mutex::new(areas), 60 60 + cpu_local_cache: CpuLocal::new(), 61 61 + max_alignment_hint: AtomicUsize::new(max_alignment_hint), 62 62 + }) 63 63 + } 64 64 + 65 65 + pub fn max_alignment_hint(&self) -> usize { 66 66 + self.max_alignment_hint.load(Ordering::Relaxed) 67 67 + } 68 68 + 69 69 + fn allocate_local(&self, layout: Layout) -> Option<NonNull<Frame>> { 70 70 + if layout.size() == A::PAGE_SIZE && layout.align() == A::PAGE_SIZE { 71 71 + let mut cache = self.cpu_local_cache.get_or_default().borrow_mut(); 72 72 + cache.pop_back() 73 73 + } else { 74 74 + None 75 75 + } 76 76 + } 77 77 + 78 78 + fn deallocate_local(&self, block: NonNull<Frame>, layout: Layout) -> bool { 79 79 + if layout.size() == A::PAGE_SIZE && layout.align() == A::PAGE_SIZE { 80 80 + let mut cache = self.cpu_local_cache.get_or_default().borrow_mut(); 81 81 + 82 82 + if cache.len() < MAX_FRAMES_IN_CACHE { 83 83 + cache.push_back(block); 84 84 + return true; 85 85 + } 86 86 + } 87 87 + 88 88 + false 89 89 + } 90 90 + } 91 91 + 92 92 + unsafe impl<L: lock_api::RawMutex + Send + Sync, A: RawAddressSpace + Send + Sync> FrameAllocator 93 93 + for &'static FrameAlloc<L, A> 94 94 + { 95 95 + const FRAME_SIZE: usize = A::PAGE_SIZE; 96 96 + 97 97 + fn allocate(&self, layout: Layout) -> Result<NonNull<[Frame]>, AllocError> { 98 98 + // attempt to allocate from the CPU-local cache first 99 99 + if let Some(frame) = self.allocate_local(layout) { 100 100 + return Ok(NonNull::slice_from_raw_parts(frame.cast(), 1)); 101 101 + } 102 102 + 103 103 + let mut areas = self.areas.lock(); 104 104 + for area in areas.iter_mut() { 105 105 + if let Ok(frames) = area.allocate(layout) { 106 106 + return Ok(frames); 107 107 + } 108 108 + } 109 109 + 110 110 + Err(AllocError) 111 111 + } 112 112 + 113 113 + unsafe fn deallocate(&self, block: NonNull<Frame>, layout: Layout) { 114 114 + // attempt to place the frame into the CPU-local cache first 115 115 + if self.deallocate_local(block, layout) { 116 116 + return; 117 117 + } 118 118 + 119 119 + let mut areas = self.areas.lock(); 120 120 + for area in areas.iter_mut() { 121 121 + let block_ = unsafe { block.as_ref() }; 122 122 + 123 123 + if area.contains_frame(block_.addr()) { 124 124 + unsafe { area.deallocate(block, layout) }; 125 125 + 126 126 + self.max_alignment_hint 127 127 + .fetch_max(area.max_alignment_hint(), Ordering::Relaxed); 128 128 + 129 129 + return; 130 130 + } 131 131 + } 132 132 + 133 133 + unreachable!(); 134 134 + } 135 135 + }

+444

libs/mem/src/frame_alloc/area.rs

reviewed

··· 1 1 + // Copyright 2025. Jonas Kruckenberg 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or 4 4 + // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or 5 5 + // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 + // copied, modified, or distributed except according to those terms. 7 7 + 8 8 + use core::alloc::Layout; 9 9 + use core::marker::PhantomData; 10 10 + use core::mem::MaybeUninit; 11 11 + use core::ops::Range; 12 12 + use core::ptr::NonNull; 13 13 + use core::{cmp, fmt}; 14 14 + 15 15 + use cordyceps::List; 16 16 + 17 17 + use crate::address_space::RawAddressSpace; 18 18 + use crate::frame_alloc::AllocError; 19 19 + use crate::{AddressRangeExt, Frame, PhysicalAddress}; 20 20 + 21 21 + const MAX_ORDER: usize = 11; 22 22 + 23 23 + pub struct Area<A: RawAddressSpace> { 24 24 + area: Range<PhysicalAddress>, 25 25 + frames: &'static mut [MaybeUninit<Frame>], 26 26 + 27 27 + free_lists: [List<Frame>; MAX_ORDER], 28 28 + 29 29 + max_order: usize, 30 30 + total_frames: usize, 31 31 + used_frames: usize, 32 32 + 33 33 + _aspace: PhantomData<A>, 34 34 + } 35 35 + 36 36 + unsafe impl<A: RawAddressSpace + Send> Send for Area<A> {} 37 37 + unsafe impl<A: RawAddressSpace + Sync> Sync for Area<A> {} 38 38 + 39 39 + impl<A: RawAddressSpace> fmt::Debug for Area<A> { 40 40 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 41 41 + f.debug_struct("Area") 42 42 + .field("area", &self.area) 43 43 + .field( 44 44 + "frames", 45 45 + &format_args!("&[MaybeUninit<FrameInner>; {}]", self.frames.len()), 46 46 + ) 47 47 + .field("free_lists", &self.free_lists) 48 48 + .field("max_order", &self.max_order) 49 49 + .field("total_frames", &self.total_frames) 50 50 + .field("used_frames", &self.used_frames) 51 51 + .finish() 52 52 + } 53 53 + } 54 54 + 55 55 + impl<A: RawAddressSpace> Area<A> { 56 56 + pub fn new(area: Range<PhysicalAddress>, frames: &'static mut [MaybeUninit<Frame>]) -> Self { 57 57 + let mut free_lists = [const { List::new() }; MAX_ORDER]; 58 58 + let mut total_frames = 0; 59 59 + let mut max_order = 0; 60 60 + 61 61 + let mut remaining_bytes = area.size(); 62 62 + let mut addr = area.start; 63 63 + 64 64 + // This is the main area initialization loop. We loop through the `area` "chopping off" the 65 65 + // largest possible min_block_size-aligned block from the area and add that to its corresponding 66 66 + // free list. 67 67 + // 68 68 + // Note: Remember that for buddy allocators `size == align`. That means we both need to check 69 69 + // the alignment and size of our remaining area and can only chop off whatever is smaller. 70 70 + while remaining_bytes > 0 { 71 71 + // println!("processing next chunk. remaining_bytes={remaining_bytes};addr={addr:?}"); 72 72 + 73 73 + // the largest size we can chop off given the alignment of the remaining area 74 74 + let max_align = if addr == PhysicalAddress::ZERO { 75 75 + // if area happens to start exactly at address 0x0 our calculation below doesn't work. 76 76 + // address 0x0 actually supports *any* alignment so we special-case it and return `MAX` 77 77 + usize::MAX 78 78 + } else { 79 79 + // otherwise mask out the least significant bit of the address to figure out its alignment 80 80 + addr.get() & (!addr.get() + 1) 81 81 + }; 82 82 + // the largest size we can chop off given the size of the remaining area 83 83 + // which is the next smaller power of two 84 84 + let max_size = 1 << remaining_bytes.ilog2(); 85 85 + 86 86 + // our chosen size will be the smallest of 87 87 + // - the maximum size by remaining areas alignment 88 88 + // - the maximum size by remaining areas size 89 89 + // - the maximum block size supported by this allocator 90 90 + let size = cmp::min( 91 91 + cmp::min(max_align, max_size), 92 92 + A::PAGE_SIZE << (MAX_ORDER - 1), 93 93 + ); 94 94 + debug_assert!(size.is_multiple_of(A::PAGE_SIZE)); 95 95 + 96 96 + let order = (size.trailing_zeros() as u8 - A::PAGE_SIZE_LOG_2) as usize; 97 97 + 98 98 + { 99 99 + let frame = frames[total_frames].write(Frame::new(addr, 0)); 100 100 + 101 101 + free_lists[order].push_back(NonNull::from(frame)); 102 102 + } 103 103 + 104 104 + total_frames += 1 << order; 105 105 + max_order = cmp::max(max_order, order); 106 106 + addr = addr.checked_add(size).unwrap(); 107 107 + remaining_bytes -= size; 108 108 + } 109 109 + 110 110 + // Make sure we've accounted for all frames 111 111 + debug_assert_eq!(total_frames, area.size() / A::PAGE_SIZE); 112 112 + 113 113 + Self { 114 114 + area, 115 115 + frames, 116 116 + 117 117 + free_lists, 118 118 + 119 119 + max_order, 120 120 + total_frames, 121 121 + used_frames: 0, 122 122 + 123 123 + _aspace: PhantomData, 124 124 + } 125 125 + } 126 126 + 127 127 + pub fn allocate(&mut self, layout: Layout) -> Result<NonNull<[Frame]>, AllocError> { 128 128 + #[cfg(debug_assertions)] 129 129 + self.assert_valid(); 130 130 + 131 131 + let min_order = self.allocation_order(layout)?; 132 132 + 133 133 + // Starting at the smallest sufficient size class, search for a free block. If we find one in 134 134 + // a free list, return it and its order. 135 135 + let (block, block_order) = self.free_lists[min_order..] 136 136 + .iter_mut() 137 137 + .enumerate() 138 138 + .find_map(|(i, list)| list.pop_back().map(|block| (block, i + min_order))) 139 139 + .ok_or(AllocError)?; 140 140 + 141 141 + // if the block we found is larger than the `min_order` we need, we repeatedly split off 142 142 + // the upper half (of decreasing size) until we reach the desired size. The split off blocks 143 143 + // are returned to their appropriate free lists. 144 144 + for order in (min_order..block_order).rev() { 145 145 + let block = unsafe { block.as_ref() }; 146 146 + let buddy_addr = block.addr().checked_add(A::PAGE_SIZE << order).unwrap(); 147 147 + let buddy = self.frame_for_addr(buddy_addr).unwrap(); 148 148 + 149 149 + let buddy = buddy.write(Frame::new(buddy_addr, 0)); 150 150 + let buddy = NonNull::from(buddy); 151 151 + 152 152 + self.free_lists[order].push_back(buddy); 153 153 + } 154 154 + 155 155 + let alloc_size_frames = 1 << min_order; 156 156 + 157 157 + // lazily initialize all frames 158 158 + for idx in 0..alloc_size_frames { 159 159 + let block = unsafe { block.as_ref() }; 160 160 + let addr = block.addr().checked_add(A::PAGE_SIZE * idx).unwrap(); 161 161 + 162 162 + let frame = self.frame_for_addr(addr).unwrap(); 163 163 + frame.write(Frame::new(addr, 1)); 164 164 + } 165 165 + 166 166 + self.used_frames += alloc_size_frames; 167 167 + 168 168 + #[cfg(debug_assertions)] 169 169 + self.assert_valid(); 170 170 + 171 171 + Ok(NonNull::slice_from_raw_parts(block, alloc_size_frames)) 172 172 + } 173 173 + 174 174 + pub unsafe fn deallocate(&mut self, mut block: NonNull<Frame>, layout: Layout) { 175 175 + #[cfg(debug_assertions)] 176 176 + self.assert_valid(); 177 177 + 178 178 + let initial_order = self.allocation_order(layout).unwrap(); 179 179 + let mut order = initial_order; 180 180 + 181 181 + while order < self.free_lists.len() - 1 { 182 182 + let block_ = unsafe { block.as_ref() }; 183 183 + if let Some(buddy) = self.buddy_addr(order, block_.addr()) 184 184 + && cmp::min(block_.addr(), buddy).is_aligned_to(A::PAGE_SIZE << (order + 1)) 185 185 + && self.remove_from_free_list(order, buddy) 186 186 + { 187 187 + let buddy: NonNull<Frame> = 188 188 + NonNull::from(self.frame_for_addr(buddy).unwrap()).cast(); 189 189 + block = cmp::min(buddy, block); 190 190 + order += 1; 191 191 + } else { 192 192 + break; 193 193 + } 194 194 + } 195 195 + 196 196 + self.free_lists[order].push_back(block); 197 197 + self.used_frames -= 1 << initial_order; 198 198 + self.max_order = cmp::max(self.max_order, order); 199 199 + 200 200 + #[cfg(debug_assertions)] 201 201 + self.assert_valid(); 202 202 + } 203 203 + 204 204 + pub fn max_alignment_hint(&self) -> usize { 205 205 + self.order_size(self.max_order) 206 206 + } 207 207 + 208 208 + fn frame_for_addr(&mut self, addr: PhysicalAddress) -> Option<&mut MaybeUninit<Frame>> { 209 209 + let relative = addr.checked_sub_addr(self.area.start).unwrap(); 210 210 + let idx = relative >> A::PAGE_SIZE_LOG_2; 211 211 + Some(&mut self.frames[idx]) 212 212 + } 213 213 + 214 214 + pub(crate) fn contains_frame(&self, addr: PhysicalAddress) -> bool { 215 215 + self.area.contains(&addr) 216 216 + } 217 217 + 218 218 + fn buddy_addr(&self, order: usize, block: PhysicalAddress) -> Option<PhysicalAddress> { 219 219 + assert!(block >= self.area.start); 220 220 + assert!(block.is_aligned_to(A::PAGE_SIZE << order)); 221 221 + 222 222 + let relative = block.checked_sub_addr(self.area.start).unwrap(); 223 223 + let size = self.order_size(order); 224 224 + if size >= self.area.size() { 225 225 + // MAX_ORDER blocks do not have buddies 226 226 + None 227 227 + } else { 228 228 + // Fun: We can find our buddy by xoring the right bit in our 229 229 + // offset from the base of the heap. 230 230 + Some(self.area.start.checked_add(relative ^ size).unwrap()) 231 231 + } 232 232 + } 233 233 + 234 234 + fn remove_from_free_list(&mut self, order: usize, to_remove: PhysicalAddress) -> bool { 235 235 + let mut c = self.free_lists[order].cursor_front_mut(); 236 236 + 237 237 + while let Some(candidate) = c.current() { 238 238 + if candidate.addr() == to_remove { 239 239 + c.remove_current().unwrap(); 240 240 + return true; 241 241 + } 242 242 + 243 243 + c.move_next(); 244 244 + } 245 245 + 246 246 + false 247 247 + } 248 248 + 249 249 + // The size of the blocks we allocate for a given order. 250 250 + const fn order_size(&self, order: usize) -> usize { 251 251 + 1 << (A::PAGE_SIZE_LOG_2 as usize + order) 252 252 + } 253 253 + 254 254 + const fn allocation_size(&self, layout: Layout) -> Result<usize, AllocError> { 255 255 + // We can only allocate blocks that are at least one page 256 256 + if !layout.size().is_multiple_of(A::PAGE_SIZE) { 257 257 + return Err(AllocError); 258 258 + } 259 259 + 260 260 + // We can only allocate blocks that are at least page aligned 261 261 + if !layout.align().is_multiple_of(A::PAGE_SIZE) { 262 262 + return Err(AllocError); 263 263 + } 264 264 + 265 265 + let size = layout.size().next_power_of_two(); 266 266 + 267 267 + // We cannot allocate blocks larger than our largest size class 268 268 + if size > self.order_size(self.free_lists.len()) { 269 269 + return Err(AllocError); 270 270 + } 271 271 + 272 272 + Ok(size) 273 273 + } 274 274 + 275 275 + const fn allocation_order(&self, layout: Layout) -> Result<usize, AllocError> { 276 276 + if let Ok(size) = self.allocation_size(layout) { 277 277 + Ok((size.ilog2() as u8 - A::PAGE_SIZE_LOG_2) as usize) 278 278 + } else { 279 279 + Err(AllocError) 280 280 + } 281 281 + } 282 282 + 283 283 + fn assert_valid(&self) { 284 284 + for (order, l) in self.free_lists.iter().enumerate() { 285 285 + l.assert_valid(); 286 286 + 287 287 + for f in l { 288 288 + assert!( 289 289 + f.addr().is_aligned_to(A::PAGE_SIZE << order), 290 290 + "frame {f:?} is not aligned to order {order}" 291 291 + ); 292 292 + } 293 293 + } 294 294 + 295 295 + assert_eq!(frames_in_area(self) + self.used_frames, self.total_frames); 296 296 + } 297 297 + } 298 298 + 299 299 + fn frames_in_area<A: RawAddressSpace>(area: &Area<A>) -> usize { 300 300 + let mut frames = 0; 301 301 + for (order, l) in area.free_lists.iter().enumerate() { 302 302 + frames += l.len() << order; 303 303 + } 304 304 + frames 305 305 + } 306 306 + 307 307 + #[cfg(test)] 308 308 + mod tests { 309 309 + use alloc::vec::Vec; 310 310 + 311 311 + use proptest::{prop_assert, prop_assert_eq, prop_assume, prop_compose, proptest}; 312 312 + 313 313 + use super::*; 314 314 + use crate::test_utils::TestAddressSpace; 315 315 + 316 316 + const PAGE_SIZE: usize = 4096; 317 317 + 318 318 + prop_compose! { 319 319 + // Generate arbitrary integers up to half the maximum desired value, 320 320 + // then multiply them by 2, thus producing only even integers in the 321 321 + // desired range. 322 322 + fn page_aligned(max: usize)(base in 0..max/PAGE_SIZE) -> usize { base * PAGE_SIZE } 323 323 + } 324 324 + 325 325 + proptest! { 326 326 + #[test] 327 327 + fn new_fixed_base(num_frames in 0..50_000usize) { 328 328 + let mut area: Area<TestAddressSpace<PAGE_SIZE, 38>> = Area::new( 329 329 + PhysicalAddress::ZERO..PhysicalAddress::new(num_frames * PAGE_SIZE), 330 330 + { 331 331 + let mut frames: Vec<MaybeUninit<Frame>> = Vec::with_capacity(num_frames); 332 332 + frames.resize_with(num_frames, || MaybeUninit::uninit()); 333 333 + frames.leak() 334 334 + } 335 335 + ); 336 336 + area.assert_valid(); 337 337 + 338 338 + // let's check whether the area correctly initialized itself 339 339 + // 340 340 + // since we start on an aligned base address (0x0) we expect it have split off chunks 341 341 + // largest-to-smallest. We replicate the process here, but take a block from its free list. 342 342 + let mut frames_remaining = num_frames; 343 343 + while frames_remaining > 0 { 344 344 + // clamp the order we calculate at the max possible order 345 345 + let chunk_order = cmp::min(frames_remaining.ilog2() as usize, MAX_ORDER - 1); 346 346 + 347 347 + let chunk = area.free_lists[chunk_order].pop_back(); 348 348 + prop_assert!(chunk.is_some(), "expected chunk of order {chunk_order}"); 349 349 + 350 350 + frames_remaining -= 1 << chunk_order; 351 351 + } 352 352 + // At the end of this process we expect all free lists to be empty 353 353 + prop_assert!(area.free_lists.iter().all(|list| list.is_empty())); 354 354 + } 355 355 + 356 356 + #[test] 357 357 + fn new_arbitrary_base(num_frames in 0..50_000usize, area_start in page_aligned(usize::MAX)) { 358 358 + 359 359 + let area = { 360 360 + let area_end = area_start.checked_add(num_frames * PAGE_SIZE); 361 361 + prop_assume!(area_end.is_some()); 362 362 + PhysicalAddress::new(area_start)..PhysicalAddress::new(area_end.unwrap()) 363 363 + }; 364 364 + 365 365 + let area: Area<TestAddressSpace<PAGE_SIZE, 38>> = Area::new( 366 366 + area, 367 367 + { 368 368 + let mut frames: Vec<MaybeUninit<Frame>> = Vec::with_capacity(num_frames); 369 369 + frames.resize_with(num_frames, || MaybeUninit::uninit()); 370 370 + frames.leak() 371 371 + } 372 372 + ); 373 373 + area.assert_valid(); 374 374 + 375 375 + // TODO figure out if we can test the free lists in a sensible way 376 376 + } 377 377 + 378 378 + #[test] 379 379 + fn alloc_exhaustion(num_frames in 0..5_000usize, area_start in page_aligned(usize::MAX)) { 380 380 + let area = { 381 381 + let area_end = area_start.checked_add(num_frames * PAGE_SIZE); 382 382 + prop_assume!(area_end.is_some()); 383 383 + PhysicalAddress::new(area_start)..PhysicalAddress::new(area_end.unwrap()) 384 384 + }; 385 385 + 386 386 + let mut area: Area<TestAddressSpace<PAGE_SIZE, 38>> = Area::new( 387 387 + area, 388 388 + { 389 389 + let mut frames: Vec<MaybeUninit<Frame>> = Vec::with_capacity(num_frames); 390 390 + frames.resize_with(num_frames, || MaybeUninit::uninit()); 391 391 + frames.leak() 392 392 + } 393 393 + ); 394 394 + area.assert_valid(); 395 395 + 396 396 + debug_assert_eq!(frames_in_area(&mut area), num_frames); 397 397 + } 398 398 + 399 399 + #[test] 400 400 + fn alloc_dealloc(num_frames in 0..5_000usize, area_start in page_aligned(usize::MAX), alloc_frames in 1..500usize) { 401 401 + let area = { 402 402 + let area_end = area_start.checked_add(num_frames * PAGE_SIZE); 403 403 + prop_assume!(area_end.is_some()); 404 404 + PhysicalAddress::new(area_start)..PhysicalAddress::new(area_end.unwrap()) 405 405 + }; 406 406 + 407 407 + let area1: Area<TestAddressSpace<PAGE_SIZE, 38>> = Area::new( 408 408 + area.clone(), 409 409 + { 410 410 + let mut frames: Vec<MaybeUninit<Frame>> = Vec::with_capacity(num_frames); 411 411 + frames.resize_with(num_frames, || MaybeUninit::uninit()); 412 412 + frames.leak() 413 413 + } 414 414 + ); 415 415 + area1.assert_valid(); 416 416 + 417 417 + let mut area2: Area<TestAddressSpace<PAGE_SIZE, 38>> = Area::new( 418 418 + area, 419 419 + { 420 420 + let mut frames: Vec<MaybeUninit<Frame>> = Vec::with_capacity(num_frames); 421 421 + frames.resize_with(num_frames, || MaybeUninit::uninit()); 422 422 + frames.leak() 423 423 + } 424 424 + ); 425 425 + area2.assert_valid(); 426 426 + 427 427 + // we can only allocate contiguous blocks of the largest order available 428 428 + prop_assume!(alloc_frames < (area2.max_alignment_hint() / PAGE_SIZE)); 429 429 + 430 430 + let layout = Layout::from_size_align(alloc_frames * PAGE_SIZE, PAGE_SIZE).unwrap(); 431 431 + 432 432 + let block = area2.allocate(layout).unwrap(); 433 433 + prop_assert!(block.len() >= alloc_frames); 434 434 + 435 435 + unsafe { area2.deallocate(block.cast(), layout); } 436 436 + 437 437 + assert_eq!(frames_in_area(&area2), num_frames); 438 438 + 439 439 + for (order, (f1, f2)) in area1.free_lists.iter().zip(area2.free_lists.iter()).enumerate() { 440 440 + prop_assert_eq!(f1.len(), f2.len(), "free lists at order {} have different lengths {} vs {}", order, f1.len(), f2.len()); 441 441 + } 442 442 + } 443 443 + } 444 444 + }

+133

libs/mem/src/frame_alloc/area_selection.rs

reviewed

··· 1 1 + // Copyright 2025. Jonas Kruckenberg 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or 4 4 + // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or 5 5 + // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 + // copied, modified, or distributed except according to those terms. 7 7 + 8 8 + use alloc::slice; 9 9 + use core::fmt::Formatter; 10 10 + use core::marker::PhantomData; 11 11 + use core::mem; 12 12 + use core::mem::MaybeUninit; 13 13 + use core::ops::Range; 14 14 + 15 15 + use fallible_iterator::FallibleIterator; 16 16 + use smallvec::SmallVec; 17 17 + 18 18 + use crate::address_space::RawAddressSpace; 19 19 + use crate::{AddressRangeExt, Frame, PhysicalAddress}; 20 20 + 21 21 + const MAX_WASTED_AREA_BYTES: usize = 0x8_4000; // 528 KiB 22 22 + 23 23 + #[derive(Debug)] 24 24 + pub struct AreaSelection { 25 25 + pub area: Range<PhysicalAddress>, 26 26 + pub bookkeeping: &'static mut [MaybeUninit<Frame>], 27 27 + pub wasted_bytes: usize, 28 28 + } 29 29 + 30 30 + #[derive(Debug)] 31 31 + pub struct SelectionError { 32 32 + pub range: Range<PhysicalAddress>, 33 33 + } 34 34 + 35 35 + pub struct ArenaSelections<A: RawAddressSpace> { 36 36 + allocatable_regions: SmallVec<[Range<PhysicalAddress>; 4]>, 37 37 + wasted_bytes: usize, 38 38 + 39 39 + _aspace: PhantomData<A>, 40 40 + } 41 41 + 42 42 + pub fn select_areas<A: RawAddressSpace>( 43 43 + allocatable_regions: SmallVec<[Range<PhysicalAddress>; 4]>, 44 44 + ) -> ArenaSelections<A> { 45 45 + ArenaSelections { 46 46 + allocatable_regions, 47 47 + wasted_bytes: 0, 48 48 + 49 49 + _aspace: PhantomData, 50 50 + } 51 51 + } 52 52 + 53 53 + impl<A: RawAddressSpace> FallibleIterator for ArenaSelections<A> { 54 54 + type Item = AreaSelection; 55 55 + type Error = SelectionError; 56 56 + 57 57 + fn next(&mut self) -> Result<Option<Self::Item>, Self::Error> { 58 58 + let Some(mut area) = self.allocatable_regions.pop() else { 59 59 + return Ok(None); 60 60 + }; 61 61 + 62 62 + while let Some(region) = self.allocatable_regions.pop() { 63 63 + debug_assert!(!area.is_overlapping(&region)); 64 64 + 65 65 + let pages_in_hole = if area.end <= region.start { 66 66 + // the region is higher than the current area 67 67 + region.start.checked_sub_addr(area.end).unwrap() / A::PAGE_SIZE 68 68 + } else { 69 69 + debug_assert!(region.end <= area.start); 70 70 + // the region is lower than the current area 71 71 + area.start.checked_sub_addr(region.end).unwrap() / A::PAGE_SIZE 72 72 + }; 73 73 + 74 74 + let waste_from_hole = size_of::<Frame>() * pages_in_hole; 75 75 + 76 76 + if self.wasted_bytes + waste_from_hole > MAX_WASTED_AREA_BYTES { 77 77 + self.allocatable_regions.push(region); 78 78 + break; 79 79 + } else { 80 80 + self.wasted_bytes += waste_from_hole; 81 81 + 82 82 + if area.end <= region.start { 83 83 + area.end = region.end; 84 84 + } else { 85 85 + area.start = region.start; 86 86 + } 87 87 + } 88 88 + } 89 89 + 90 90 + let mut aligned = area.checked_align_in(A::PAGE_SIZE).unwrap(); 91 91 + // We can't use empty areas anyway 92 92 + if aligned.is_empty() { 93 93 + return Err(SelectionError { range: aligned }); 94 94 + } 95 95 + 96 96 + let bookkeeping_size_frames = aligned.size() / A::PAGE_SIZE; 97 97 + 98 98 + let bookkeeping_start = aligned 99 99 + .end 100 100 + .checked_sub(bookkeeping_size_frames * size_of::<Frame>()) 101 101 + .unwrap() 102 102 + .align_down(A::PAGE_SIZE); 103 103 + 104 104 + // The area has no space to hold its own bookkeeping 105 105 + if bookkeeping_start < aligned.start { 106 106 + return Err(SelectionError { range: aligned }); 107 107 + } 108 108 + 109 109 + let bookkeeping = unsafe { 110 110 + slice::from_raw_parts_mut( 111 111 + bookkeeping_start.as_mut_ptr().cast(), 112 112 + bookkeeping_size_frames, 113 113 + ) 114 114 + }; 115 115 + aligned.end = bookkeeping_start; 116 116 + 117 117 + Ok(Some(AreaSelection { 118 118 + area: aligned, 119 119 + bookkeeping, 120 120 + wasted_bytes: mem::take(&mut self.wasted_bytes), 121 121 + })) 122 122 + } 123 123 + } 124 124 + 125 125 + // ===== impl SelectionError ===== 126 126 + 127 127 + impl core::fmt::Display for SelectionError { 128 128 + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { 129 129 + todo!() 130 130 + } 131 131 + } 132 132 + 133 133 + impl core::error::Error for SelectionError {}

+15

libs/mem/src/lib.rs

reviewed

··· 1 1 + #![cfg_attr(not(test), no_std)] 2 2 + extern crate alloc; 3 3 + 4 4 + mod access_rules; 5 5 + mod addresses; 6 6 + mod frame; 7 7 + mod test_utils; 8 8 + pub mod address_space; 9 9 + pub mod frame_alloc; 10 10 + mod utils; 11 11 + 12 12 + pub type Result<T> = anyhow::Result<T>; 13 13 + pub use frame::{FrameRef, Frame}; 14 14 + pub use addresses::{PhysicalAddress, VirtualAddress, AddressRangeExt}; 15 15 + pub use access_rules::{AccessRules, WriteOrExecute};

+169

libs/mem/src/test_utils.rs

reviewed

··· 1 1 + // Copyright 2025. Jonas Kruckenberg 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or 4 4 + // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or 5 5 + // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 + // copied, modified, or distributed except according to those terms. 7 7 + 8 8 + extern crate std; 9 9 + 10 10 + use alloc::collections::BTreeMap; 11 11 + use core::marker::PhantomData; 12 12 + use core::num::NonZeroUsize; 13 13 + 14 14 + use crate::address_space::{Flush, RawAddressSpace}; 15 15 + use crate::{AccessRules, PhysicalAddress, VirtualAddress}; 16 16 + 17 17 + #[derive(Debug)] 18 18 + pub struct TestAddressSpace<const PAGE_SIZE: usize, const ADDR_BITS: u32> { 19 19 + mappings: BTreeMap<VirtualAddress, Mapping>, 20 20 + } 21 21 + 22 22 + #[derive(Debug)] 23 23 + pub struct Mapping { 24 24 + pub virt: VirtualAddress, 25 25 + pub phys: PhysicalAddress, 26 26 + pub len: NonZeroUsize, 27 27 + pub access_rules: AccessRules, 28 28 + } 29 29 + 30 30 + pub struct TestFlush { 31 31 + _priv: PhantomData<()>, 32 32 + } 33 33 + 34 34 + impl<const PAGE_SIZE: usize, const ADDR_BITS: u32> TestAddressSpace<PAGE_SIZE, ADDR_BITS> { 35 35 + pub const fn new() -> Self { 36 36 + Self { 37 37 + mappings: BTreeMap::new(), 38 38 + } 39 39 + } 40 40 + 41 41 + pub fn get_mapping_containing(&self, addr: VirtualAddress) -> Option<&Mapping> { 42 42 + let (end, mapping) = self.mappings.range(addr..).next()?; 43 43 + 44 44 + if addr > *end { None } else { Some(mapping) } 45 45 + } 46 46 + 47 47 + pub fn get_mapping_mut_containing(&mut self, addr: VirtualAddress) -> Option<&mut Mapping> { 48 48 + let (end, mapping) = self.mappings.range_mut(addr..).next()?; 49 49 + 50 50 + if addr > *end { None } else { Some(mapping) } 51 51 + } 52 52 + 53 53 + pub fn remove_mapping_containing(&mut self, addr: VirtualAddress) -> Option<Mapping> { 54 54 + let (key, _) = self.mappings.range_mut(addr..).next()?; 55 55 + let key = *key; 56 56 + 57 57 + Some(self.mappings.remove(&key).unwrap()) 58 58 + } 59 59 + } 60 60 + 61 61 + unsafe impl<const PAGE_SIZE: usize, const ADDR_BITS: u32> RawAddressSpace for TestAddressSpace<PAGE_SIZE, ADDR_BITS> { 62 62 + const PAGE_SIZE: usize = PAGE_SIZE; 63 63 + const VIRT_ADDR_BITS: u32 = ADDR_BITS; 64 64 + 65 65 + type Flush = TestFlush; 66 66 + 67 67 + fn flush(&self) -> Self::Flush { 68 68 + TestFlush { _priv: PhantomData } 69 69 + } 70 70 + 71 71 + fn lookup(&self, virt: VirtualAddress) -> Option<(PhysicalAddress, AccessRules)> { 72 72 + let mapping = self.get_mapping_containing(virt)?; 73 73 + 74 74 + let offset = virt.checked_sub_addr(mapping.virt).unwrap(); 75 75 + 76 76 + Some(( 77 77 + mapping.phys.checked_add(offset).unwrap(), 78 78 + mapping.access_rules, 79 79 + )) 80 80 + } 81 81 + 82 82 + unsafe fn map( 83 83 + &mut self, 84 84 + virt: VirtualAddress, 85 85 + phys: PhysicalAddress, 86 86 + len: NonZeroUsize, 87 87 + access_rules: AccessRules, 88 88 + _flush: &mut Self::Flush, 89 89 + ) -> crate::Result<()> { 90 90 + assert!(virt.is_aligned_to(Self::PAGE_SIZE)); 91 91 + assert!(phys.is_aligned_to(Self::PAGE_SIZE)); 92 92 + assert!(self.get_mapping_containing(virt).is_none()); 93 93 + 94 94 + let end_virt = virt.checked_add(len.get() - 1).unwrap(); 95 95 + assert!(end_virt.is_aligned_to(Self::PAGE_SIZE)); 96 96 + 97 97 + let prev = self.mappings.insert( 98 98 + end_virt, 99 99 + Mapping { 100 100 + virt, 101 101 + phys, 102 102 + len, 103 103 + access_rules, 104 104 + }, 105 105 + ); 106 106 + assert!(prev.is_none()); 107 107 + 108 108 + Ok(()) 109 109 + } 110 110 + 111 111 + unsafe fn unmap( 112 112 + &mut self, 113 113 + mut virt: VirtualAddress, 114 114 + len: NonZeroUsize, 115 115 + _flush: &mut Self::Flush, 116 116 + ) { 117 117 + assert!(virt.is_aligned_to(Self::PAGE_SIZE)); 118 118 + assert!( 119 119 + virt.checked_add(len.get()) 120 120 + .unwrap() 121 121 + .is_aligned_to(Self::PAGE_SIZE) 122 122 + ); 123 123 + 124 124 + let mut bytes_remaining = len.get(); 125 125 + 126 126 + while bytes_remaining > 0 { 127 127 + let mapping = self.remove_mapping_containing(virt).unwrap(); 128 128 + assert_eq!(mapping.virt, virt); 129 129 + 130 130 + bytes_remaining -= mapping.len.get(); 131 131 + virt = virt.checked_sub(mapping.len.get()).unwrap(); 132 132 + } 133 133 + } 134 134 + 135 135 + unsafe fn set_access_rules( 136 136 + &mut self, 137 137 + mut virt: VirtualAddress, 138 138 + len: NonZeroUsize, 139 139 + access_rules: AccessRules, 140 140 + _flush: &mut Self::Flush, 141 141 + ) { 142 142 + assert!(virt.is_aligned_to(Self::PAGE_SIZE)); 143 143 + assert!( 144 144 + virt.checked_add(len.get()) 145 145 + .unwrap() 146 146 + .is_aligned_to(Self::PAGE_SIZE) 147 147 + ); 148 148 + 149 149 + let mut bytes_remaining = len.get(); 150 150 + 151 151 + while bytes_remaining > 0 { 152 152 + let mapping = self.get_mapping_mut_containing(virt).unwrap(); 153 153 + assert_eq!(mapping.virt, virt); 154 154 + 155 155 + mapping.access_rules = access_rules; 156 156 + 157 157 + bytes_remaining -= mapping.len.get(); 158 158 + virt = virt.checked_sub(mapping.len.get()).unwrap(); 159 159 + } 160 160 + } 161 161 + } 162 162 + 163 163 + // ===== impl TestFlush ===== 164 164 + 165 165 + impl Flush for TestFlush { 166 166 + fn flush(self) -> crate::Result<()> { 167 167 + Ok(()) 168 168 + } 169 169 + }

+31

libs/mem/src/utils.rs

reviewed

··· 1 1 + // Copyright 2025. Jonas Kruckenberg 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or 4 4 + // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or 5 5 + // http://opensource.org/licenses/MIT>, at your option. This file may not be 6 6 + // copied, modified, or distributed except according to those terms. 7 7 + 8 8 + macro_rules! assert_unsafe_precondition_ { 9 9 + ($message:expr, ($($name:ident:$ty:ty = $arg:expr),*$(,)?) => $e:expr $(,)?) => { 10 10 + { 11 11 + // This check is inlineable, but not by the MIR inliner. 12 12 + // The reason for this is that the MIR inliner is in an exceptionally bad position 13 13 + // to think about whether or not to inline this. In MIR, this call is gated behind `debug_assertions`, 14 14 + // which will codegen to `false` in release builds. Inlining the check would be wasted work in that case and 15 15 + // would be bad for compile times. 16 16 + // 17 17 + // LLVM on the other hand sees the constant branch, so if it's `false`, it can immediately delete it without 18 18 + // inlining the check. If it's `true`, it can inline it and get significantly better performance. 19 19 + #[inline] 20 20 + const fn precondition_check($($name:$ty),*) { 21 21 + assert!($e, concat!("unsafe precondition(s) violated: ", $message, 22 22 + "\n\nThis indicates a bug in the program. \ 23 23 + This Undefined Behavior check is optional, and cannot be relied on for safety.")) 24 24 + } 25 25 + 26 26 + #[cfg(debug_assertions)] 27 27 + precondition_check($($arg,)*); 28 28 + } 29 29 + }; 30 30 + } 31 31 + pub(crate) use assert_unsafe_precondition_;

+6 -2

libs/wavltree/src/cursor.rs

reviewed

··· 88 88 pub unsafe fn get_ptr(&self) -> Link<T> { 89 89 self.current 90 90 } 91 91 - pub fn get(&self) -> Option<&'a T> { 92 92 - unsafe { self.current.map(|ptr| ptr.as_ref()) } 91 91 + pub const fn get(&self) -> Option<&'a T> { 92 92 + if let Some(ptr) = self.current { 93 93 + Some(unsafe { ptr.as_ref() }) 94 94 + } else { 95 95 + None 96 96 + } 93 97 } 94 98 pub fn get_mut(&mut self) -> Option<Pin<&'a mut T>> { 95 99 unsafe { self.current.map(|mut ptr| Pin::new_unchecked(ptr.as_mut())) }

+1 -1

libs/wavltree/src/lib.rs

reviewed

··· 1510 1510 } 1511 1511 1512 1512 /// Returns `true` if this node is currently linked to a [WAVLTree]. 1513 1513 - pub fn is_linked(&self) -> bool { 1513 1513 + pub const fn is_linked(&self) -> bool { 1514 1514 let inner = unsafe { &*self.inner.get() }; 1515 1515 inner.up.is_some() || inner.left.is_some() || inner.right.is_some() 1516 1516 }