From 7cedf7345cce7a12c28cce8df0bdc8a62d4ca438 Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Thu, 31 Jul 2025 10:50:06 +0800 Subject: rust: alloc: kvec: add doc example for as_slice method Add a practical usage example to the documentation of KVec::as_slice() showing how to: Create a new KVec. Push elements into it. Convert to a slice via as_slice(). Co-developed-by: Geliang Tang Signed-off-by: Geliang Tang Signed-off-by: Hui Zhu Reviewed-by: Alice Ryhl Reviewed-by: Kunwu Chan Reviewed-by: David Gow Link: https://lore.kernel.org/r/4e7f396f38ed8a780f863384bfc3d7de135ef3ea.1753929369.git.zhuhui@kylinos.cn Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/kvec.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index 3c72e0bdddb8..fa04cc0987d6 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -224,6 +224,16 @@ where } /// Returns a slice of the entire vector. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// v.push(2, GFP_KERNEL)?; + /// assert_eq!(v.as_slice(), &[1, 2]); + /// # Ok::<(), Error>(()) + /// ``` #[inline] pub fn as_slice(&self) -> &[T] { self -- cgit From a55498c7d8a651ca5c7b8169e84d48af77e60723 Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Thu, 31 Jul 2025 10:50:07 +0800 Subject: rust: alloc: kvec: simplify KUnit test module name to "rust_kvec" Remove redundant "_kunit" suffix from test module name. The naming is now consistent with other Rust components as the test context is already implied by the #[kunit_tests] macro and test module location. Co-developed-by: Geliang Tang Signed-off-by: Geliang Tang Signed-off-by: Hui Zhu Reviewed-by: David Gow Link: https://lore.kernel.org/r/4eb554c3bf03dd4f9e6dea659497938baab61dba.1753929369.git.zhuhui@kylinos.cn Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/kvec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index fa04cc0987d6..7316f48aaa90 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -1304,7 +1304,7 @@ impl<'vec, T> Drop for DrainAll<'vec, T> { } } -#[macros::kunit_tests(rust_kvec_kunit)] +#[macros::kunit_tests(rust_kvec)] mod tests { use super::*; use crate::prelude::*; -- cgit From 8d3e8057eeadab134a71d6a495d6f1b6818144fe Mon Sep 17 00:00:00 2001 From: Onur Özkan Date: Sun, 20 Jul 2025 12:48:37 +0300 Subject: rust: make `ArrayLayout::new_unchecked` a `const fn` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes `ArrayLayout::new_unchecked` a `const fn` to allow compile-time evaluation. Signed-off-by: Onur Özkan Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20250720094838.29530-3-work@onurozkan.dev Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/layout.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/layout.rs b/rust/kernel/alloc/layout.rs index 93ed514f7cc7..52cbf61c4539 100644 --- a/rust/kernel/alloc/layout.rs +++ b/rust/kernel/alloc/layout.rs @@ -80,7 +80,7 @@ impl ArrayLayout { /// # Safety /// /// `len` must be a value, for which `len * size_of::() <= isize::MAX` is true. - pub unsafe fn new_unchecked(len: usize) -> Self { + pub const unsafe fn new_unchecked(len: usize) -> Self { // INVARIANT: By the safety requirements of this function // `len * size_of::() <= isize::MAX`. Self { -- cgit From f87de6919dc126f22c7b5bf92e378f0346f190a8 Mon Sep 17 00:00:00 2001 From: Onur Özkan Date: Sun, 20 Jul 2025 12:48:38 +0300 Subject: rust: make `kvec::Vec` functions `const fn` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes various `kvec::Vec` functions `const fn` to allow compile-time evaluation. Signed-off-by: Onur Özkan Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20250720094838.29530-4-work@onurozkan.dev Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/kvec.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index 7316f48aaa90..d42dbdc44f0f 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -175,7 +175,7 @@ where /// Returns the number of elements that can be stored within the vector without allocating /// additional memory. - pub fn capacity(&self) -> usize { + pub const fn capacity(&self) -> usize { if const { Self::is_zst() } { usize::MAX } else { @@ -185,7 +185,7 @@ where /// Returns the number of elements stored within the vector. #[inline] - pub fn len(&self) -> usize { + pub const fn len(&self) -> usize { self.len } @@ -196,7 +196,7 @@ where /// - `additional` must be less than or equal to `self.capacity - self.len`. /// - All elements within the interval [`self.len`,`self.len + additional`) must be initialized. #[inline] - pub unsafe fn inc_len(&mut self, additional: usize) { + pub const unsafe fn inc_len(&mut self, additional: usize) { // Guaranteed by the type invariant to never underflow. debug_assert!(additional <= self.capacity() - self.len()); // INVARIANT: By the safety requirements of this method this represents the exact number of @@ -255,7 +255,7 @@ where /// Returns a raw pointer to the vector's backing buffer, or, if `T` is a ZST, a dangling raw /// pointer. #[inline] - pub fn as_ptr(&self) -> *const T { + pub const fn as_ptr(&self) -> *const T { self.ptr.as_ptr() } @@ -271,7 +271,7 @@ where /// assert!(!v.is_empty()); /// ``` #[inline] - pub fn is_empty(&self) -> bool { + pub const fn is_empty(&self) -> bool { self.len() == 0 } -- cgit From 1b1a946dc2b535785663742f9e4f15fd64bece60 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 11 Aug 2025 12:31:50 +0000 Subject: rust: alloc: specify the minimum alignment of each allocator The kernel's allocators sometimes provide a higher alignment than the end-user requested, so add a new constant on the Allocator trait to let the allocator specify what its minimum guaranteed alignment is. This allows the ForeignOwnable trait to provide a more accurate value of FOREIGN_ALIGN when using a pointer type such as Box, which will be useful with certain collections such as XArray that store its own data in the low bits of pointers. Reviewed-by: Benno Lossin Signed-off-by: Alice Ryhl Acked-by: Liam R. Howlett Reviewed-by: Andreas Hindborg Link: https://lore.kernel.org/r/20250811-align-min-allocator-v2-1-3386cc94f4fc@google.com [ Add helper for ARCH_KMALLOC_MINALIGN; remove cast to usize. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/allocator.rs | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index aa2dfa9dca4c..5003907f0240 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -17,6 +17,8 @@ use crate::alloc::{AllocError, Allocator}; use crate::bindings; use crate::pr_warn; +const ARCH_KMALLOC_MINALIGN: usize = bindings::ARCH_KMALLOC_MINALIGN; + /// The contiguous kernel allocator. /// /// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also @@ -128,6 +130,8 @@ impl ReallocFunc { // - passing a pointer to a valid memory allocation is OK, // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. unsafe impl Allocator for Kmalloc { + const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN; + #[inline] unsafe fn realloc( ptr: Option>, @@ -145,6 +149,8 @@ unsafe impl Allocator for Kmalloc { // - passing a pointer to a valid memory allocation is OK, // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. unsafe impl Allocator for Vmalloc { + const MIN_ALIGN: usize = kernel::page::PAGE_SIZE; + #[inline] unsafe fn realloc( ptr: Option>, @@ -169,6 +175,8 @@ unsafe impl Allocator for Vmalloc { // - passing a pointer to a valid memory allocation is OK, // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. unsafe impl Allocator for KVmalloc { + const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN; + #[inline] unsafe fn realloc( ptr: Option>, -- cgit From bb9749f32ad38cf343e6650a34125be6aacd65d1 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 11 Aug 2025 12:31:51 +0000 Subject: rust: alloc: take the allocator into account for FOREIGN_ALIGN When converting a Box into a void pointer, the allocator might guarantee a higher alignment than the type itself does, and in that case it is guaranteed that the void pointer has that higher alignment. This is quite useful when combined with the XArray, which you can only create using a ForeignOwnable whose FOREIGN_ALIGN is at least 4. This means that you can now always use a Box with the XArray no matter the alignment of T. Reviewed-by: Benno Lossin Signed-off-by: Alice Ryhl Acked-by: Liam R. Howlett Reviewed-by: Andreas Hindborg Link: https://lore.kernel.org/r/20250811-align-min-allocator-v2-2-3386cc94f4fc@google.com Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/kbox.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index 856d05aa60f1..eedab0be1eff 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -401,12 +401,17 @@ where } // SAFETY: The pointer returned by `into_foreign` comes from a well aligned -// pointer to `T`. +// pointer to `T` allocated by `A`. unsafe impl ForeignOwnable for Box where A: Allocator, { - const FOREIGN_ALIGN: usize = core::mem::align_of::(); + const FOREIGN_ALIGN: usize = if core::mem::align_of::() < A::MIN_ALIGN { + A::MIN_ALIGN + } else { + core::mem::align_of::() + }; + type Borrowed<'a> = &'a T; type BorrowedMut<'a> = &'a mut T; @@ -435,12 +440,12 @@ where } // SAFETY: The pointer returned by `into_foreign` comes from a well aligned -// pointer to `T`. +// pointer to `T` allocated by `A`. unsafe impl ForeignOwnable for Pin> where A: Allocator, { - const FOREIGN_ALIGN: usize = core::mem::align_of::(); + const FOREIGN_ALIGN: usize = as ForeignOwnable>::FOREIGN_ALIGN; type Borrowed<'a> = Pin<&'a T>; type BorrowedMut<'a> = Pin<&'a mut T>; -- cgit From ac9eea3d08c25fb213deb113d246ff5dadb31fbc Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 11 Aug 2025 12:14:56 +0200 Subject: rust: alloc: implement Box::pin_slice() Add a new constructor to Box to facilitate Box creation from a pinned slice of elements. This allows to efficiently allocate memory for e.g. slices of structrures containing spinlocks or mutexes. Such slices may be used in kmemcache like or zpool API implementations. Signed-off-by: Alice Ryhl Signed-off-by: Vitaly Wool Link: https://lore.kernel.org/r/20250811101456.2901694-1-vitaly.wool@konsulko.se [ Add empty lines after struct definitions in the example; end sentences with a period. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/kbox.rs | 77 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index eedab0be1eff..1aa83d751f10 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -290,6 +290,83 @@ where Ok(Self::new(x, flags)?.into()) } + /// Construct a pinned slice of elements `Pin>`. + /// + /// This is a convenient means for creation of e.g. slices of structrures containing spinlocks + /// or mutexes. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::{new_spinlock, SpinLock}; + /// + /// struct Inner { + /// a: u32, + /// b: u32, + /// } + /// + /// #[pin_data] + /// struct Example { + /// c: u32, + /// #[pin] + /// d: SpinLock, + /// } + /// + /// impl Example { + /// fn new() -> impl PinInit { + /// try_pin_init!(Self { + /// c: 10, + /// d <- new_spinlock!(Inner { a: 20, b: 30 }), + /// }) + /// } + /// } + /// + /// // Allocate a boxed slice of 10 `Example`s. + /// let s = KBox::pin_slice( + /// | _i | Example::new(), + /// 10, + /// GFP_KERNEL + /// )?; + /// + /// assert_eq!(s[5].c, 10); + /// assert_eq!(s[3].d.lock().a, 20); + /// # Ok::<(), Error>(()) + /// ``` + pub fn pin_slice( + mut init: Func, + len: usize, + flags: Flags, + ) -> Result>, E> + where + Func: FnMut(usize) -> Item, + Item: PinInit, + E: From, + { + let mut buffer = super::Vec::::with_capacity(len, flags)?; + for i in 0..len { + let ptr = buffer.spare_capacity_mut().as_mut_ptr().cast(); + // SAFETY: + // - `ptr` is a valid pointer to uninitialized memory. + // - `ptr` is not used if an error is returned. + // - `ptr` won't be moved until it is dropped, i.e. it is pinned. + unsafe { init(i).__pinned_init(ptr)? }; + + // SAFETY: + // - `i + 1 <= len`, hence we don't exceed the capacity, due to the call to + // `with_capacity()` above. + // - The new value at index buffer.len() + 1 is the only element being added here, and + // it has been initialized above by `init(i).__pinned_init(ptr)`. + unsafe { buffer.inc_len(1) }; + } + + let (ptr, _, _) = buffer.into_raw_parts(); + let slice = core::ptr::slice_from_raw_parts_mut(ptr, len); + + // SAFETY: `slice` points to an allocation allocated with `A` (`buffer`) and holds a valid + // `[T]`. + Ok(Pin::from(unsafe { Box::from_raw(slice) })) + } + /// Convert a [`Box`] to a [`Pin>`]. If `T` does not implement /// [`Unpin`], then `x` will be pinned in memory and can't be moved. pub fn into_pin(this: Self) -> Pin { -- cgit From fe927defbb4f31c15a52f0372d7f5d608f161086 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Sat, 16 Aug 2025 23:19:00 +0200 Subject: rust: alloc: remove `allocator_test` Given we do not have tests that rely on it anymore, remove `allocator_test`, which simplifies the complexity of the build. In particular, it avoids potential issues with `rusttest`, such as the one fixed at [1], where a public function was added to `Kmalloc` and used elsewhere, but it was not added to `Cmalloc`; or trivial issues like a missing import [2] due to not many people testing that target. The only downside is that we cannot use it in the `macros`' crate examples anymore, but we did not feel a need for that so far, and anyway we could support that by running those within the kernel too, which we may do regardless. Link: https://lore.kernel.org/rust-for-linux/20250816204215.2719559-1-ojeda@kernel.org/ [1] Link: https://lore.kernel.org/rust-for-linux/20250816210214.2729269-1-ojeda@kernel.org/ [2] Signed-off-by: Miguel Ojeda Link: https://lore.kernel.org/r/20250816211900.2731720-1-ojeda@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/allocator_test.rs | 113 ------------------------------------ 1 file changed, 113 deletions(-) delete mode 100644 rust/kernel/alloc/allocator_test.rs (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs deleted file mode 100644 index a3074480bd8d..000000000000 --- a/rust/kernel/alloc/allocator_test.rs +++ /dev/null @@ -1,113 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! So far the kernel's `Box` and `Vec` types can't be used by userspace test cases, since all users -//! of those types (e.g. `CString`) use kernel allocators for instantiation. -//! -//! In order to allow userspace test cases to make use of such types as well, implement the -//! `Cmalloc` allocator within the `allocator_test` module and type alias all kernel allocators to -//! `Cmalloc`. The `Cmalloc` allocator uses libc's `realloc()` function as allocator backend. - -#![allow(missing_docs)] - -use super::{flags::*, AllocError, Allocator, Flags}; -use core::alloc::Layout; -use core::cmp; -use core::ptr; -use core::ptr::NonNull; - -/// The userspace allocator based on libc. -pub struct Cmalloc; - -pub type Kmalloc = Cmalloc; -pub type Vmalloc = Kmalloc; -pub type KVmalloc = Kmalloc; - -extern "C" { - #[link_name = "aligned_alloc"] - fn libc_aligned_alloc(align: usize, size: usize) -> *mut crate::ffi::c_void; - - #[link_name = "free"] - fn libc_free(ptr: *mut crate::ffi::c_void); -} - -// SAFETY: -// - memory remains valid until it is explicitly freed, -// - passing a pointer to a valid memory allocation created by this `Allocator` is always OK, -// - `realloc` provides the guarantees as provided in the `# Guarantees` section. -unsafe impl Allocator for Cmalloc { - unsafe fn realloc( - ptr: Option>, - layout: Layout, - old_layout: Layout, - flags: Flags, - ) -> Result, AllocError> { - let src = match ptr { - Some(src) => { - if old_layout.size() == 0 { - ptr::null_mut() - } else { - src.as_ptr() - } - } - None => ptr::null_mut(), - }; - - if layout.size() == 0 { - // SAFETY: `src` is either NULL or was previously allocated with this `Allocator` - unsafe { libc_free(src.cast()) }; - - return Ok(NonNull::slice_from_raw_parts( - crate::alloc::dangling_from_layout(layout), - 0, - )); - } - - // ISO C (ISO/IEC 9899:2011) defines `aligned_alloc`: - // - // > The value of alignment shall be a valid alignment supported by the implementation - // [...]. - // - // As an example of the "supported by the implementation" requirement, POSIX.1-2001 (IEEE - // 1003.1-2001) defines `posix_memalign`: - // - // > The value of alignment shall be a power of two multiple of sizeof (void *). - // - // and POSIX-based implementations of `aligned_alloc` inherit this requirement. At the time - // of writing, this is known to be the case on macOS (but not in glibc). - // - // Satisfy the stricter requirement to avoid spurious test failures on some platforms. - let min_align = core::mem::size_of::<*const crate::ffi::c_void>(); - let layout = layout.align_to(min_align).map_err(|_| AllocError)?; - let layout = layout.pad_to_align(); - - // SAFETY: Returns either NULL or a pointer to a memory allocation that satisfies or - // exceeds the given size and alignment requirements. - let dst = unsafe { libc_aligned_alloc(layout.align(), layout.size()) }.cast::(); - let dst = NonNull::new(dst).ok_or(AllocError)?; - - if flags.contains(__GFP_ZERO) { - // SAFETY: The preceding calls to `libc_aligned_alloc` and `NonNull::new` - // guarantee that `dst` points to memory of at least `layout.size()` bytes. - unsafe { dst.as_ptr().write_bytes(0, layout.size()) }; - } - - if !src.is_null() { - // SAFETY: - // - `src` has previously been allocated with this `Allocator`; `dst` has just been - // newly allocated, hence the memory regions do not overlap. - // - both` src` and `dst` are properly aligned and valid for reads and writes - unsafe { - ptr::copy_nonoverlapping( - src, - dst.as_ptr(), - cmp::min(layout.size(), old_layout.size()), - ) - }; - } - - // SAFETY: `src` is either NULL or was previously allocated with this `Allocator` - unsafe { libc_free(src.cast()) }; - - Ok(NonNull::slice_from_raw_parts(dst, layout.size())) - } -} -- cgit From 8e92c9902ff11a1c2aab229a3d7d4c1d7e5b698f Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Wed, 20 Aug 2025 16:53:38 +0200 Subject: rust: alloc: vmalloc: implement Vmalloc::to_page() Implement an abstraction of vmalloc_to_page() for subsequent use in the AsPageIter implementation of VBox and VVec. Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Reviewed-by: Alice Ryhl Reviewed-by: Daniel Almeida Link: https://lore.kernel.org/r/20250820145434.94745-3-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/allocator.rs | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index aa2dfa9dca4c..2315f5063011 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -15,6 +15,7 @@ use core::ptr::NonNull; use crate::alloc::{AllocError, Allocator}; use crate::bindings; +use crate::page; use crate::pr_warn; /// The contiguous kernel allocator. @@ -140,6 +141,54 @@ unsafe impl Allocator for Kmalloc { } } +impl Vmalloc { + /// Convert a pointer to a [`Vmalloc`] allocation to a [`page::BorrowedPage`]. + /// + /// # Examples + /// + /// ``` + /// # use core::ptr::{NonNull, from_mut}; + /// # use kernel::{page, prelude::*}; + /// use kernel::alloc::allocator::Vmalloc; + /// + /// let mut vbox = VBox::<[u8; page::PAGE_SIZE]>::new_uninit(GFP_KERNEL)?; + /// + /// { + /// // SAFETY: By the type invariant of `Box` the inner pointer of `vbox` is non-null. + /// let ptr = unsafe { NonNull::new_unchecked(from_mut(&mut *vbox)) }; + /// + /// // SAFETY: + /// // `ptr` is a valid pointer to a `Vmalloc` allocation. + /// // `ptr` is valid for the entire lifetime of `page`. + /// let page = unsafe { Vmalloc::to_page(ptr.cast()) }; + /// + /// // SAFETY: There is no concurrent read or write to the same page. + /// unsafe { page.fill_zero_raw(0, page::PAGE_SIZE)? }; + /// } + /// # Ok::<(), Error>(()) + /// ``` + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to a [`Vmalloc`] allocation. + /// - `ptr` must remain valid for the entire duration of `'a`. + pub unsafe fn to_page<'a>(ptr: NonNull) -> page::BorrowedPage<'a> { + // SAFETY: `ptr` is a valid pointer to `Vmalloc` memory. + let page = unsafe { bindings::vmalloc_to_page(ptr.as_ptr().cast()) }; + + // SAFETY: `vmalloc_to_page` returns a valid pointer to a `struct page` for a valid pointer + // to `Vmalloc` memory. + let page = unsafe { NonNull::new_unchecked(page) }; + + // SAFETY: + // - `page` is a valid pointer to a `struct page`, given that by the safety requirements of + // this function `ptr` is a valid pointer to a `Vmalloc` allocation. + // - By the safety requirements of this function `ptr` is valid for the entire lifetime of + // `'a`. + unsafe { page::BorrowedPage::from_raw(page) } + } +} + // SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that // - memory remains valid until it is explicitly freed, // - passing a pointer to a valid memory allocation is OK, -- cgit From 7937dca770393d34d1ad217580c5446d2e45417f Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Wed, 20 Aug 2025 16:53:39 +0200 Subject: rust: alloc: implement VmallocPageIter Introduce the VmallocPageIter type; an instance of VmallocPageIter may be exposed by owners of vmalloc allocations to provide borrowed access to its backing pages. For instance, this is useful to access and borrow the backing pages of allocation primitives, such as Box and Vec, backing a scatterlist. Reviewed-by: Daniel Almeida Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Reviewed-by: Alice Ryhl Suggested-by: Alice Ryhl Link: https://lore.kernel.org/r/20250820145434.94745-4-dakr@kernel.org [ Drop VmallocPageIter::base_address(), move to allocator/iter.rs and stub VmallocPageIter for allocator_test.rs. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/allocator.rs | 3 ++ rust/kernel/alloc/allocator/iter.rs | 102 ++++++++++++++++++++++++++++++++++++ rust/kernel/alloc/allocator_test.rs | 29 ++++++++++ 3 files changed, 134 insertions(+) create mode 100644 rust/kernel/alloc/allocator/iter.rs (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 2315f5063011..90e03ad15760 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -18,6 +18,9 @@ use crate::bindings; use crate::page; use crate::pr_warn; +mod iter; +pub use self::iter::VmallocPageIter; + /// The contiguous kernel allocator. /// /// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also diff --git a/rust/kernel/alloc/allocator/iter.rs b/rust/kernel/alloc/allocator/iter.rs new file mode 100644 index 000000000000..5759f86029b7 --- /dev/null +++ b/rust/kernel/alloc/allocator/iter.rs @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0 + +use super::Vmalloc; +use crate::page; +use core::marker::PhantomData; +use core::ptr::NonNull; + +/// An [`Iterator`] of [`page::BorrowedPage`] items owned by a [`Vmalloc`] allocation. +/// +/// # Guarantees +/// +/// The pages iterated by the [`Iterator`] appear in the order as they are mapped in the CPU's +/// virtual address space ascendingly. +/// +/// # Invariants +/// +/// - `buf` is a valid and [`page::PAGE_SIZE`] aligned pointer into a [`Vmalloc`] allocation. +/// - `size` is the number of bytes from `buf` until the end of the [`Vmalloc`] allocation `buf` +/// points to. +pub struct VmallocPageIter<'a> { + /// The base address of the [`Vmalloc`] buffer. + buf: NonNull, + /// The size of the buffer pointed to by `buf` in bytes. + size: usize, + /// The current page index of the [`Iterator`]. + index: usize, + _p: PhantomData>, +} + +impl<'a> Iterator for VmallocPageIter<'a> { + type Item = page::BorrowedPage<'a>; + + fn next(&mut self) -> Option { + let offset = self.index.checked_mul(page::PAGE_SIZE)?; + + // Even though `self.size()` may be smaller than `Self::page_count() * page::PAGE_SIZE`, it + // is always a number between `(Self::page_count() - 1) * page::PAGE_SIZE` and + // `Self::page_count() * page::PAGE_SIZE`, hence the check below is sufficient. + if offset < self.size() { + self.index += 1; + } else { + return None; + } + + // TODO: Use `NonNull::add()` instead, once the minimum supported compiler version is + // bumped to 1.80 or later. + // + // SAFETY: `offset` is in the interval `[0, (self.page_count() - 1) * page::PAGE_SIZE]`, + // hence the resulting pointer is guaranteed to be within the same allocation. + let ptr = unsafe { self.buf.as_ptr().add(offset) }; + + // SAFETY: `ptr` is guaranteed to be non-null given that it is derived from `self.buf`. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + // SAFETY: + // - `ptr` is a valid pointer to a `Vmalloc` allocation. + // - `ptr` is valid for the duration of `'a`. + Some(unsafe { Vmalloc::to_page(ptr) }) + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = self.page_count().saturating_sub(self.index); + + (remaining, Some(remaining)) + } +} + +impl<'a> VmallocPageIter<'a> { + /// Creates a new [`VmallocPageIter`] instance. + /// + /// # Safety + /// + /// - `buf` must be a [`page::PAGE_SIZE`] aligned pointer into a [`Vmalloc`] allocation. + /// - `buf` must be valid for at least the lifetime of `'a`. + /// - `size` must be the number of bytes from `buf` until the end of the [`Vmalloc`] allocation + /// `buf` points to. + pub unsafe fn new(buf: NonNull, size: usize) -> Self { + // INVARIANT: By the safety requirements, `buf` is a valid and `page::PAGE_SIZE` aligned + // pointer into a [`Vmalloc`] allocation. + Self { + buf, + size, + index: 0, + _p: PhantomData, + } + } + + /// Returns the size of the backing [`Vmalloc`] allocation in bytes. + /// + /// Note that this is the size the [`Vmalloc`] allocation has been allocated with. Hence, this + /// number may be smaller than `[`Self::page_count`] * [`page::PAGE_SIZE`]`. + #[inline] + pub fn size(&self) -> usize { + self.size + } + + /// Returns the number of pages owned by the backing [`Vmalloc`] allocation. + #[inline] + pub fn page_count(&self) -> usize { + self.size().div_ceil(page::PAGE_SIZE) + } +} diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs index a3074480bd8d..f46b4b671389 100644 --- a/rust/kernel/alloc/allocator_test.rs +++ b/rust/kernel/alloc/allocator_test.rs @@ -12,8 +12,10 @@ use super::{flags::*, AllocError, Allocator, Flags}; use core::alloc::Layout; use core::cmp; +use core::marker::PhantomData; use core::ptr; use core::ptr::NonNull; +use kernel::page; /// The userspace allocator based on libc. pub struct Cmalloc; @@ -22,6 +24,33 @@ pub type Kmalloc = Cmalloc; pub type Vmalloc = Kmalloc; pub type KVmalloc = Kmalloc; +pub struct VmallocPageIter<'a> { + _p: PhantomData>, +} + +impl<'a> Iterator for VmallocPageIter<'a> { + type Item = page::BorrowedPage<'a>; + + fn next(&mut self) -> Option { + None + } +} + +impl<'a> VmallocPageIter<'a> { + #[allow(clippy::missing_safety_doc)] + pub unsafe fn new(_buf: NonNull, _size: usize) -> Self { + Self { _p: PhantomData } + } + + pub fn size(&self) -> usize { + 0 + } + + pub fn page_count(&self) -> usize { + 0 + } +} + extern "C" { #[link_name = "aligned_alloc"] fn libc_aligned_alloc(align: usize, size: usize) -> *mut crate::ffi::c_void; -- cgit From 671618432f46c553d9b82ad701ac054e16c8d55a Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Wed, 20 Aug 2025 16:53:41 +0200 Subject: rust: alloc: kbox: implement AsPageIter for VBox Implement AsPageIter for VBox; this allows to iterate and borrow the backing pages of a VBox. This, for instance, is useful in combination with VBox backing a scatterlist. Reviewed-by: Alice Ryhl Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://lore.kernel.org/r/20250820145434.94745-6-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/kbox.rs | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index 856d05aa60f1..b69ff4a1d748 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -3,7 +3,7 @@ //! Implementation of [`Box`]. #[allow(unused_imports)] // Used in doc comments. -use super::allocator::{KVmalloc, Kmalloc, Vmalloc}; +use super::allocator::{KVmalloc, Kmalloc, Vmalloc, VmallocPageIter}; use super::{AllocError, Allocator, Flags}; use core::alloc::Layout; use core::borrow::{Borrow, BorrowMut}; @@ -18,6 +18,7 @@ use core::result::Result; use crate::ffi::c_void; use crate::init::InPlaceInit; +use crate::page::AsPageIter; use crate::types::ForeignOwnable; use pin_init::{InPlaceWrite, Init, PinInit, ZeroableOption}; @@ -598,3 +599,40 @@ where unsafe { A::free(self.0.cast(), layout) }; } } + +/// # Examples +/// +/// ``` +/// # use kernel::prelude::*; +/// use kernel::alloc::allocator::VmallocPageIter; +/// use kernel::page::{AsPageIter, PAGE_SIZE}; +/// +/// let mut vbox = VBox::new((), GFP_KERNEL)?; +/// +/// assert!(vbox.page_iter().next().is_none()); +/// +/// let mut vbox = VBox::<[u8; PAGE_SIZE]>::new_uninit(GFP_KERNEL)?; +/// +/// let page = vbox.page_iter().next().expect("At least one page should be available.\n"); +/// +/// // SAFETY: There is no concurrent read or write to the same page. +/// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? }; +/// # Ok::<(), Error>(()) +/// ``` +impl AsPageIter for VBox { + type Iter<'a> + = VmallocPageIter<'a> + where + T: 'a; + + fn page_iter(&mut self) -> Self::Iter<'_> { + let ptr = self.0.cast(); + let size = core::mem::size_of::(); + + // SAFETY: + // - `ptr` is a valid pointer to the beginning of a `Vmalloc` allocation. + // - `ptr` is guaranteed to be valid for the lifetime of `'a`. + // - `size` is the size of the `Vmalloc` allocation `ptr` points to. + unsafe { VmallocPageIter::new(ptr, size) } + } +} -- cgit From 9acb4e630c3f3fc070b9962683fdde5ba1c4c70c Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Wed, 20 Aug 2025 16:53:42 +0200 Subject: rust: alloc: layout: implement ArrayLayout::size() Provide a convenience method for ArrayLayout to calculate the size of the ArrayLayout in bytes. Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Reviewed-by: Alice Ryhl Reviewed-by: Abdiel Janulgue Link: https://lore.kernel.org/r/20250820145434.94745-7-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/layout.rs | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/layout.rs b/rust/kernel/alloc/layout.rs index 93ed514f7cc7..666accb7859c 100644 --- a/rust/kernel/alloc/layout.rs +++ b/rust/kernel/alloc/layout.rs @@ -98,6 +98,11 @@ impl ArrayLayout { pub const fn is_empty(&self) -> bool { self.len == 0 } + + /// Returns the size of the [`ArrayLayout`] in bytes. + pub const fn size(&self) -> usize { + self.len() * core::mem::size_of::() + } } impl From> for Layout { -- cgit From 779db37373a38f23e2534c7f4ffe8188f5237988 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Wed, 20 Aug 2025 16:53:43 +0200 Subject: rust: alloc: kvec: implement AsPageIter for VVec Implement AsPageIter for VVec; this allows to iterate and borrow the backing pages of a VVec. This, for instance, is useful in combination with VVec backing a scatterlist. Reviewed-by: Alice Ryhl Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Reviewed-by: Daniel Almeida Link: https://lore.kernel.org/r/20250820145434.94745-8-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/kvec.rs | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index 3c72e0bdddb8..ac438e70a1ed 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -3,10 +3,11 @@ //! Implementation of [`Vec`]. use super::{ - allocator::{KVmalloc, Kmalloc, Vmalloc}, + allocator::{KVmalloc, Kmalloc, Vmalloc, VmallocPageIter}, layout::ArrayLayout, AllocError, Allocator, Box, Flags, }; +use crate::page::AsPageIter; use core::{ borrow::{Borrow, BorrowMut}, fmt, @@ -1017,6 +1018,43 @@ where } } +/// # Examples +/// +/// ``` +/// # use kernel::prelude::*; +/// use kernel::alloc::allocator::VmallocPageIter; +/// use kernel::page::{AsPageIter, PAGE_SIZE}; +/// +/// let mut vec = VVec::::new(); +/// +/// assert!(vec.page_iter().next().is_none()); +/// +/// vec.reserve(PAGE_SIZE, GFP_KERNEL)?; +/// +/// let page = vec.page_iter().next().expect("At least one page should be available.\n"); +/// +/// // SAFETY: There is no concurrent read or write to the same page. +/// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? }; +/// # Ok::<(), Error>(()) +/// ``` +impl AsPageIter for VVec { + type Iter<'a> + = VmallocPageIter<'a> + where + T: 'a; + + fn page_iter(&mut self) -> Self::Iter<'_> { + let ptr = self.ptr.cast(); + let size = self.layout.size(); + + // SAFETY: + // - `ptr` is a valid pointer to the beginning of a `Vmalloc` allocation. + // - `ptr` is guaranteed to be valid for the lifetime of `'a`. + // - `size` is the size of the `Vmalloc` allocation `ptr` points to. + unsafe { VmallocPageIter::new(ptr, size) } + } +} + /// An [`Iterator`] implementation for [`Vec`] that moves elements out of a vector. /// /// This structure is created by the [`Vec::into_iter`] method on [`Vec`] (provided by the -- cgit From 7760b6421b6c1b49550885ecdfa9cf720ead6eed Mon Sep 17 00:00:00 2001 From: Vitaly Wool Date: Wed, 6 Aug 2025 14:55:22 +0200 Subject: rust: add support for NUMA ids in allocations Add a new type to support specifying NUMA identifiers in Rust allocators and extend the allocators to have NUMA id as a parameter. Thus, modify ReallocFunc to use the new extended realloc primitives from the C side of the kernel (i.e. k[v]realloc_node_align/vrealloc_node_align) and add the new function alloc_node to the Allocator trait while keeping the existing one (alloc) for backward compatibility. This will allow to specify node to use for allocation of e. g. {KV}Box, as well as for future NUMA aware users of the API. [ojeda@kernel.org: fix missing import needed for `rusttest`] Link: https://lkml.kernel.org/r/20250816210214.2729269-1-ojeda@kernel.org Link: https://lkml.kernel.org/r/20250806125522.1726992-1-vitaly.wool@konsulko.se Signed-off-by: Vitaly Wool Signed-off-by: Miguel Ojeda Acked-by: Danilo Krummrich Acked-by: Alice Ryhl Cc: Herbert Xu Cc: Jann Horn Cc: Kent Overstreet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Uladzislau Rezki (Sony) Cc: Vlastimil Babka Cc: Miguel Ojeda Signed-off-by: Andrew Morton --- rust/kernel/alloc/allocator.rs | 35 ++++++++++++++++++++++------------- rust/kernel/alloc/allocator_test.rs | 3 ++- rust/kernel/alloc/kbox.rs | 4 ++-- rust/kernel/alloc/kvec.rs | 11 +++++++++-- 4 files changed, 35 insertions(+), 18 deletions(-) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 2692cf90c948..14510a9e4502 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -13,7 +13,7 @@ use core::alloc::Layout; use core::ptr; use core::ptr::NonNull; -use crate::alloc::{AllocError, Allocator}; +use crate::alloc::{AllocError, Allocator, NumaNode}; use crate::bindings; use crate::pr_warn; @@ -45,20 +45,25 @@ pub struct KVmalloc; /// # Invariants /// -/// One of the following: `krealloc`, `vrealloc`, `kvrealloc`. +/// One of the following: `krealloc_node`, `vrealloc_node`, `kvrealloc_node`. struct ReallocFunc( - unsafe extern "C" fn(*const crate::ffi::c_void, usize, u32) -> *mut crate::ffi::c_void, + unsafe extern "C" fn( + *const crate::ffi::c_void, + usize, + u32, + crate::ffi::c_int, + ) -> *mut crate::ffi::c_void, ); impl ReallocFunc { - // INVARIANT: `krealloc` satisfies the type invariants. - const KREALLOC: Self = Self(bindings::krealloc); + // INVARIANT: `krealloc_node` satisfies the type invariants. + const KREALLOC: Self = Self(bindings::krealloc_node); - // INVARIANT: `vrealloc` satisfies the type invariants. - const VREALLOC: Self = Self(bindings::vrealloc); + // INVARIANT: `vrealloc_node` satisfies the type invariants. + const VREALLOC: Self = Self(bindings::vrealloc_node); - // INVARIANT: `kvrealloc` satisfies the type invariants. - const KVREALLOC: Self = Self(bindings::kvrealloc); + // INVARIANT: `kvrealloc_node` satisfies the type invariants. + const KVREALLOC: Self = Self(bindings::kvrealloc_node); /// # Safety /// @@ -76,6 +81,7 @@ impl ReallocFunc { layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError> { let size = layout.size(); let ptr = match ptr { @@ -99,7 +105,7 @@ impl ReallocFunc { // - Those functions provide the guarantees of this function. let raw_ptr = unsafe { // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. - self.0(ptr.cast(), size, flags.0).cast() + self.0(ptr.cast(), size, flags.0, nid.0).cast() }; let ptr = if size == 0 { @@ -134,11 +140,12 @@ unsafe impl Allocator for Kmalloc { layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError> { let layout = Kmalloc::aligned_layout(layout); // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. - unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) } + unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags, nid) } } } @@ -153,6 +160,7 @@ unsafe impl Allocator for Vmalloc { layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError> { // TODO: Support alignments larger than PAGE_SIZE. if layout.align() > bindings::PAGE_SIZE { @@ -162,7 +170,7 @@ unsafe impl Allocator for Vmalloc { // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously // allocated with this `Allocator`. - unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) } + unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags, nid) } } } @@ -177,6 +185,7 @@ unsafe impl Allocator for KVmalloc { layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError> { // `KVmalloc` may use the `Kmalloc` backend, hence we have to enforce a `Kmalloc` // compatible layout. @@ -190,6 +199,6 @@ unsafe impl Allocator for KVmalloc { // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously // allocated with this `Allocator`. - unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) } + unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags, nid) } } } diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs index 90dd987d40e4..2e61cdbd2303 100644 --- a/rust/kernel/alloc/allocator_test.rs +++ b/rust/kernel/alloc/allocator_test.rs @@ -9,7 +9,7 @@ #![allow(missing_docs)] -use super::{flags::*, AllocError, Allocator, Flags}; +use super::{flags::*, AllocError, Allocator, Flags, NumaNode}; use core::alloc::Layout; use core::cmp; use core::ptr; @@ -51,6 +51,7 @@ unsafe impl Allocator for Cmalloc { layout: Layout, old_layout: Layout, flags: Flags, + _nid: NumaNode, ) -> Result, AllocError> { let src = match ptr { Some(src) => { diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index 856d05aa60f1..1fef9beb57c8 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -4,7 +4,7 @@ #[allow(unused_imports)] // Used in doc comments. use super::allocator::{KVmalloc, Kmalloc, Vmalloc}; -use super::{AllocError, Allocator, Flags}; +use super::{AllocError, Allocator, Flags, NumaNode}; use core::alloc::Layout; use core::borrow::{Borrow, BorrowMut}; use core::fmt; @@ -273,7 +273,7 @@ where /// ``` pub fn new_uninit(flags: Flags) -> Result, A>, AllocError> { let layout = Layout::new::>(); - let ptr = A::alloc(layout, flags)?; + let ptr = A::alloc(layout, flags, NumaNode::NO_NODE)?; // INVARIANT: `ptr` is either a dangling pointer or points to memory allocated with `A`, // which is sufficient in size and alignment for storing a `T`. diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index 3c72e0bdddb8..92d0ed3f302e 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -5,7 +5,7 @@ use super::{ allocator::{KVmalloc, Kmalloc, Vmalloc}, layout::ArrayLayout, - AllocError, Allocator, Box, Flags, + AllocError, Allocator, Box, Flags, NumaNode, }; use core::{ borrow::{Borrow, BorrowMut}, @@ -634,6 +634,7 @@ where layout.into(), self.layout.into(), flags, + NumaNode::NO_NODE, )? }; @@ -1111,7 +1112,13 @@ where // the type invariant to be smaller than `cap`. Depending on `realloc` this operation // may shrink the buffer or leave it as it is. ptr = match unsafe { - A::realloc(Some(buf.cast()), layout.into(), old_layout.into(), flags) + A::realloc( + Some(buf.cast()), + layout.into(), + old_layout.into(), + flags, + NumaNode::NO_NODE, + ) } { // If we fail to shrink, which likely can't even happen, continue with the existing // buffer. -- cgit From 1738796994a439b0ea796847e3ceb8688dacd93d Mon Sep 17 00:00:00 2001 From: Vitaly Wool Date: Wed, 6 Aug 2025 14:55:52 +0200 Subject: rust: support large alignments in allocations Add support for large (> PAGE_SIZE) alignments in Rust allocators. All the preparations on the C side are already done, we just need to add bindings for _node_align() functions and start using those. Link: https://lkml.kernel.org/r/20250806125552.1727073-1-vitaly.wool@konsulko.se Signed-off-by: Vitaly Wool Acked-by: Danilo Krummrich Acked-by: Alice Ryhl Cc: Herbert Xu Cc: Jann Horn Cc: Kent Overstreet Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Uladzislau Rezki (Sony) Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- rust/kernel/alloc/allocator.rs | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 14510a9e4502..f4ae0cf0a594 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -15,7 +15,6 @@ use core::ptr::NonNull; use crate::alloc::{AllocError, Allocator, NumaNode}; use crate::bindings; -use crate::pr_warn; /// The contiguous kernel allocator. /// @@ -45,25 +44,26 @@ pub struct KVmalloc; /// # Invariants /// -/// One of the following: `krealloc_node`, `vrealloc_node`, `kvrealloc_node`. +/// One of the following: `krealloc_node_align`, `vrealloc_node_align`, `kvrealloc_node_align`. struct ReallocFunc( unsafe extern "C" fn( *const crate::ffi::c_void, usize, + crate::ffi::c_ulong, u32, crate::ffi::c_int, ) -> *mut crate::ffi::c_void, ); impl ReallocFunc { - // INVARIANT: `krealloc_node` satisfies the type invariants. - const KREALLOC: Self = Self(bindings::krealloc_node); + // INVARIANT: `krealloc_node_align` satisfies the type invariants. + const KREALLOC: Self = Self(bindings::krealloc_node_align); - // INVARIANT: `vrealloc_node` satisfies the type invariants. - const VREALLOC: Self = Self(bindings::vrealloc_node); + // INVARIANT: `vrealloc_node_align` satisfies the type invariants. + const VREALLOC: Self = Self(bindings::vrealloc_node_align); - // INVARIANT: `kvrealloc_node` satisfies the type invariants. - const KVREALLOC: Self = Self(bindings::kvrealloc_node); + // INVARIANT: `kvrealloc_node_align` satisfies the type invariants. + const KVREALLOC: Self = Self(bindings::kvrealloc_node_align); /// # Safety /// @@ -105,7 +105,7 @@ impl ReallocFunc { // - Those functions provide the guarantees of this function. let raw_ptr = unsafe { // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. - self.0(ptr.cast(), size, flags.0, nid.0).cast() + self.0(ptr.cast(), size, layout.align(), flags.0, nid.0).cast() }; let ptr = if size == 0 { @@ -162,12 +162,6 @@ unsafe impl Allocator for Vmalloc { flags: Flags, nid: NumaNode, ) -> Result, AllocError> { - // TODO: Support alignments larger than PAGE_SIZE. - if layout.align() > bindings::PAGE_SIZE { - pr_warn!("Vmalloc does not support alignments larger than PAGE_SIZE yet.\n"); - return Err(AllocError); - } - // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously // allocated with this `Allocator`. unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags, nid) } @@ -191,12 +185,6 @@ unsafe impl Allocator for KVmalloc { // compatible layout. let layout = Kmalloc::aligned_layout(layout); - // TODO: Support alignments larger than PAGE_SIZE. - if layout.align() > bindings::PAGE_SIZE { - pr_warn!("KVmalloc does not support alignments larger than PAGE_SIZE yet.\n"); - return Err(AllocError); - } - // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously // allocated with this `Allocator`. unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags, nid) } -- cgit From 868ade323e9deff67b8be3e93876596e4d2c71d3 Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Thu, 31 Jul 2025 10:50:05 +0800 Subject: rust: allocator: add KUnit tests for alignment guarantees MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a test module to verify memory alignment guarantees for Rust kernel allocators. The tests cover `Kmalloc`, `Vmalloc` and `KVmalloc` allocators with both standard and large page-aligned allocations. Key features of the tests: 1. Creates alignment-constrained types: - 128-byte aligned `Blob` - 8192-byte (4-page) aligned `LargeAlignBlob` 2. Validates allocators using `TestAlign` helper which: - Checks address alignment masks - Supports uninitialized allocations 3. Tests all three allocators with both alignment requirements: - Kmalloc with 128B and 8192B - Vmalloc with 128B and 8192B - KVmalloc with 128B and 8192B Link: https://lkml.kernel.org/r/d2e3d6454c1435713be0fe3c0dc444d2c60bba51.1753929369.git.zhuhui@kylinos.cn Co-developed-by: Geliang Tang Signed-off-by: Geliang Tang Signed-off-by: Hui Zhu Reviewed-by: Kunwu Chan Acked-by: Danilo Krummrich Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Björn Roy Baron Cc: Boqun Feng Cc: Gary Guo Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Miguel Ojeda Cc: Trevor Gross Cc: "Uladzislau Rezki (Sony)" Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- rust/kernel/alloc/allocator.rs | 56 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index f4ae0cf0a594..b561e7a57bb8 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -190,3 +190,59 @@ unsafe impl Allocator for KVmalloc { unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags, nid) } } } + +#[macros::kunit_tests(rust_allocator)] +mod tests { + use super::*; + use core::mem::MaybeUninit; + use kernel::prelude::*; + + #[test] + fn test_alignment() -> Result { + const TEST_SIZE: usize = 1024; + const TEST_LARGE_ALIGN_SIZE: usize = kernel::page::PAGE_SIZE * 4; + + // These two structs are used to test allocating aligned memory. + // they don't need to be accessed, so they're marked as dead_code. + #[expect(dead_code)] + #[repr(align(128))] + struct Blob([u8; TEST_SIZE]); + #[expect(dead_code)] + #[repr(align(8192))] + struct LargeAlignBlob([u8; TEST_LARGE_ALIGN_SIZE]); + + struct TestAlign(Box, A>); + impl TestAlign { + fn new() -> Result { + Ok(Self(Box::<_, A>::new_uninit(GFP_KERNEL)?)) + } + + fn is_aligned_to(&self, align: usize) -> bool { + assert!(align.is_power_of_two()); + + let addr = self.0.as_ptr() as usize; + addr & (align - 1) == 0 + } + } + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(128)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(8192)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(128)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(8192)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(128)); + + let ta = TestAlign::::new()?; + assert!(ta.is_aligned_to(8192)); + + Ok(()) + } +} -- cgit From 1f96115f502abae7d08ba35dd02e6b0381f265a4 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Wed, 13 Aug 2025 11:39:47 -0400 Subject: rust: alloc: use `kernel::{fmt,prelude::fmt!}` Reduce coupling to implementation details of the formatting machinery by avoiding direct use for `core`'s formatting traits and macros. Acked-by: Greg Kroah-Hartman Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Acked-by: Danilo Krummrich Signed-off-by: Tamir Duberstein Signed-off-by: Miguel Ojeda --- rust/kernel/alloc/kbox.rs | 2 +- rust/kernel/alloc/kvec.rs | 2 +- rust/kernel/alloc/kvec/errors.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'rust/kernel/alloc') diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index 1aa83d751f10..27c4b5a9b61d 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -7,7 +7,6 @@ use super::allocator::{KVmalloc, Kmalloc, Vmalloc}; use super::{AllocError, Allocator, Flags}; use core::alloc::Layout; use core::borrow::{Borrow, BorrowMut}; -use core::fmt; use core::marker::PhantomData; use core::mem::ManuallyDrop; use core::mem::MaybeUninit; @@ -17,6 +16,7 @@ use core::ptr::NonNull; use core::result::Result; use crate::ffi::c_void; +use crate::fmt; use crate::init::InPlaceInit; use crate::types::ForeignOwnable; use pin_init::{InPlaceWrite, Init, PinInit, ZeroableOption}; diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index d42dbdc44f0f..dfc101e03f35 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -7,9 +7,9 @@ use super::{ layout::ArrayLayout, AllocError, Allocator, Box, Flags, }; +use crate::fmt; use core::{ borrow::{Borrow, BorrowMut}, - fmt, marker::PhantomData, mem::{ManuallyDrop, MaybeUninit}, ops::Deref, diff --git a/rust/kernel/alloc/kvec/errors.rs b/rust/kernel/alloc/kvec/errors.rs index 348b8d27e102..21a920a4b09b 100644 --- a/rust/kernel/alloc/kvec/errors.rs +++ b/rust/kernel/alloc/kvec/errors.rs @@ -2,7 +2,7 @@ //! Errors for the [`Vec`] type. -use core::fmt::{self, Debug, Formatter}; +use kernel::fmt::{self, Debug, Formatter}; use kernel::prelude::*; /// Error type for [`Vec::push_within_capacity`]. -- cgit