page.rs

  1// SPDX-License-Identifier: GPL-2.0
  2
  3//! Kernel page allocation and management.
  4
  5use crate::{
  6    alloc::{AllocError, Flags},
  7    bindings,
  8    error::code::*,
  9    error::Result,
 10    uaccess::UserSliceReader,
 11};
 12use core::{
 13    marker::PhantomData,
 14    mem::ManuallyDrop,
 15    ops::Deref,
 16    ptr::{self, NonNull},
 17};
 18
 19/// A bitwise shift for the page size.
 20pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize;
 21
 22/// The number of bytes in a page.
 23pub const PAGE_SIZE: usize = bindings::PAGE_SIZE;
 24
 25/// A bitmask that gives the page containing a given address.
 26pub const PAGE_MASK: usize = !(PAGE_SIZE - 1);
 27
 28/// Round up the given number to the next multiple of [`PAGE_SIZE`].
 29///
 30/// It is incorrect to pass an address where the next multiple of [`PAGE_SIZE`] doesn't fit in a
 31/// [`usize`].
 32pub const fn page_align(addr: usize) -> usize {
 33    // Parentheses around `PAGE_SIZE - 1` to avoid triggering overflow sanitizers in the wrong
 34    // cases.
 35    (addr + (PAGE_SIZE - 1)) & PAGE_MASK
 36}
 37
 38/// Representation of a non-owning reference to a [`Page`].
 39///
 40/// This type provides a borrowed version of a [`Page`] that is owned by some other entity, e.g. a
 41/// [`Vmalloc`] allocation such as [`VBox`].
 42///
 43/// # Example
 44///
 45/// ```
 46/// # use kernel::{bindings, prelude::*};
 47/// use kernel::page::{BorrowedPage, Page, PAGE_SIZE};
 48/// # use core::{mem::MaybeUninit, ptr, ptr::NonNull };
 49///
 50/// fn borrow_page<'a>(vbox: &'a mut VBox<MaybeUninit<[u8; PAGE_SIZE]>>) -> BorrowedPage<'a> {
 51///     let ptr = ptr::from_ref(&**vbox);
 52///
 53///     // SAFETY: `ptr` is a valid pointer to `Vmalloc` memory.
 54///     let page = unsafe { bindings::vmalloc_to_page(ptr.cast()) };
 55///
 56///     // SAFETY: `vmalloc_to_page` returns a valid pointer to a `struct page` for a valid
 57///     // pointer to `Vmalloc` memory.
 58///     let page = unsafe { NonNull::new_unchecked(page) };
 59///
 60///     // SAFETY:
 61///     // - `self.0` is a valid pointer to a `struct page`.
 62///     // - `self.0` is valid for the entire lifetime of `self`.
 63///     unsafe { BorrowedPage::from_raw(page) }
 64/// }
 65///
 66/// let mut vbox = VBox::<[u8; PAGE_SIZE]>::new_uninit(GFP_KERNEL)?;
 67/// let page = borrow_page(&mut vbox);
 68///
 69/// // SAFETY: There is no concurrent read or write to this page.
 70/// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? };
 71/// # Ok::<(), Error>(())
 72/// ```
 73///
 74/// # Invariants
 75///
 76/// The borrowed underlying pointer to a `struct page` is valid for the entire lifetime `'a`.
 77///
 78/// [`VBox`]: kernel::alloc::VBox
 79/// [`Vmalloc`]: kernel::alloc::allocator::Vmalloc
 80pub struct BorrowedPage<'a>(ManuallyDrop<Page>, PhantomData<&'a Page>);
 81
 82impl<'a> BorrowedPage<'a> {
 83    /// Constructs a [`BorrowedPage`] from a raw pointer to a `struct page`.
 84    ///
 85    /// # Safety
 86    ///
 87    /// - `ptr` must point to a valid `bindings::page`.
 88    /// - `ptr` must remain valid for the entire lifetime `'a`.
 89    pub unsafe fn from_raw(ptr: NonNull<bindings::page>) -> Self {
 90        let page = Page { page: ptr };
 91
 92        // INVARIANT: The safety requirements guarantee that `ptr` is valid for the entire lifetime
 93        // `'a`.
 94        Self(ManuallyDrop::new(page), PhantomData)
 95    }
 96}
 97
 98impl<'a> Deref for BorrowedPage<'a> {
 99    type Target = Page;
100
101    fn deref(&self) -> &Self::Target {
102        &self.0
103    }
104}
105
106/// Trait to be implemented by types which provide an [`Iterator`] implementation of
107/// [`BorrowedPage`] items, such as [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter).
108pub trait AsPageIter {
109    /// The [`Iterator`] type, e.g. [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter).
110    type Iter<'a>: Iterator<Item = BorrowedPage<'a>>
111    where
112        Self: 'a;
113
114    /// Returns an [`Iterator`] of [`BorrowedPage`] items over all pages owned by `self`.
115    fn page_iter(&mut self) -> Self::Iter<'_>;
116}
117
118/// A pointer to a page that owns the page allocation.
119///
120/// # Invariants
121///
122/// The pointer is valid, and has ownership over the page.
123pub struct Page {
124    page: NonNull<bindings::page>,
125}
126
127// SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across
128// threads is safe.
129unsafe impl Send for Page {}
130
131// SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing
132// them concurrently is safe.
133unsafe impl Sync for Page {}
134
135impl Page {
136    /// Allocates a new page.
137    ///
138    /// # Examples
139    ///
140    /// Allocate memory for a page.
141    ///
142    /// ```
143    /// use kernel::page::Page;
144    ///
145    /// let page = Page::alloc_page(GFP_KERNEL)?;
146    /// # Ok::<(), kernel::alloc::AllocError>(())
147    /// ```
148    ///
149    /// Allocate memory for a page and zero its contents.
150    ///
151    /// ```
152    /// use kernel::page::Page;
153    ///
154    /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?;
155    /// # Ok::<(), kernel::alloc::AllocError>(())
156    /// ```
157    #[inline]
158    pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> {
159        // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it
160        // is always safe to call this method.
161        let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) };
162        let page = NonNull::new(page).ok_or(AllocError)?;
163        // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly
164        // allocated page. We transfer that ownership to the new `Page` object.
165        Ok(Self { page })
166    }
167
168    /// Returns a raw pointer to the page.
169    pub fn as_ptr(&self) -> *mut bindings::page {
170        self.page.as_ptr()
171    }
172
173    /// Get the node id containing this page.
174    pub fn nid(&self) -> i32 {
175        // SAFETY: Always safe to call with a valid page.
176        unsafe { bindings::page_to_nid(self.as_ptr()) }
177    }
178
179    /// Runs a piece of code with this page mapped to an address.
180    ///
181    /// The page is unmapped when this call returns.
182    ///
183    /// # Using the raw pointer
184    ///
185    /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for
186    /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might
187    /// only be mapped on the current thread, and when that is the case, dereferencing it on other
188    /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't
189    /// cause data races, the memory may be uninitialized, and so on.
190    ///
191    /// If multiple threads map the same page at the same time, then they may reference with
192    /// different addresses. However, even if the addresses are different, the underlying memory is
193    /// still the same for these purposes (e.g., it's still a data race if they both write to the
194    /// same underlying byte at the same time).
195    fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T {
196        // SAFETY: `page` is valid due to the type invariants on `Page`.
197        let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) };
198
199        let res = f(mapped_addr.cast());
200
201        // This unmaps the page mapped above.
202        //
203        // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner
204        // where the pages are unmapped in reverse order. This is as required by `kunmap_local`.
205        //
206        // In other words, if this call to `kunmap_local` happens when a different page should be
207        // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the
208        // call just above in `with_page_mapped` that made that possible. In this case, it is the
209        // unsafe block that wraps that other call that is incorrect.
210        unsafe { bindings::kunmap_local(mapped_addr) };
211
212        res
213    }
214
215    /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking.
216    ///
217    /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the
218    /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on
219    /// this task, as this method uses a local mapping.
220    ///
221    /// If `off` and `len` refers to a region outside of this page, then this method returns
222    /// [`EINVAL`] and does not call `f`.
223    ///
224    /// # Using the raw pointer
225    ///
226    /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for
227    /// `len` bytes and for the duration in which the closure is called. The pointer might only be
228    /// mapped on the current thread, and when that is the case, dereferencing it on other threads
229    /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause
230    /// data races, the memory may be uninitialized, and so on.
231    ///
232    /// If multiple threads map the same page at the same time, then they may reference with
233    /// different addresses. However, even if the addresses are different, the underlying memory is
234    /// still the same for these purposes (e.g., it's still a data race if they both write to the
235    /// same underlying byte at the same time).
236    fn with_pointer_into_page<T>(
237        &self,
238        off: usize,
239        len: usize,
240        f: impl FnOnce(*mut u8) -> Result<T>,
241    ) -> Result<T> {
242        let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE;
243
244        if bounds_ok {
245            self.with_page_mapped(move |page_addr| {
246                // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will
247                // result in a pointer that is in bounds or one off the end of the page.
248                f(unsafe { page_addr.add(off) })
249            })
250        } else {
251            Err(EINVAL)
252        }
253    }
254
255    /// Maps the page and reads from it into the given buffer.
256    ///
257    /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
258    /// outside of the page, then this call returns [`EINVAL`].
259    ///
260    /// # Safety
261    ///
262    /// * Callers must ensure that `dst` is valid for writing `len` bytes.
263    /// * Callers must ensure that this call does not race with a write to the same page that
264    ///   overlaps with this read.
265    pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result {
266        self.with_pointer_into_page(offset, len, move |src| {
267            // SAFETY: If `with_pointer_into_page` calls into this closure, then
268            // it has performed a bounds check and guarantees that `src` is
269            // valid for `len` bytes.
270            //
271            // There caller guarantees that there is no data race.
272            unsafe { ptr::copy_nonoverlapping(src, dst, len) };
273            Ok(())
274        })
275    }
276
277    /// Maps the page and writes into it from the given buffer.
278    ///
279    /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
280    /// outside of the page, then this call returns [`EINVAL`].
281    ///
282    /// # Safety
283    ///
284    /// * Callers must ensure that `src` is valid for reading `len` bytes.
285    /// * Callers must ensure that this call does not race with a read or write to the same page
286    ///   that overlaps with this write.
287    pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result {
288        self.with_pointer_into_page(offset, len, move |dst| {
289            // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
290            // bounds check and guarantees that `dst` is valid for `len` bytes.
291            //
292            // There caller guarantees that there is no data race.
293            unsafe { ptr::copy_nonoverlapping(src, dst, len) };
294            Ok(())
295        })
296    }
297
298    /// Maps the page and zeroes the given slice.
299    ///
300    /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
301    /// outside of the page, then this call returns [`EINVAL`].
302    ///
303    /// # Safety
304    ///
305    /// Callers must ensure that this call does not race with a read or write to the same page that
306    /// overlaps with this write.
307    pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result {
308        self.with_pointer_into_page(offset, len, move |dst| {
309            // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
310            // bounds check and guarantees that `dst` is valid for `len` bytes.
311            //
312            // There caller guarantees that there is no data race.
313            unsafe { ptr::write_bytes(dst, 0u8, len) };
314            Ok(())
315        })
316    }
317
318    /// Copies data from userspace into this page.
319    ///
320    /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
321    /// outside of the page, then this call returns [`EINVAL`].
322    ///
323    /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address.
324    /// However, they are not allowed on the page you are copying into.
325    ///
326    /// # Safety
327    ///
328    /// Callers must ensure that this call does not race with a read or write to the same page that
329    /// overlaps with this write.
330    pub unsafe fn copy_from_user_slice_raw(
331        &self,
332        reader: &mut UserSliceReader,
333        offset: usize,
334        len: usize,
335    ) -> Result {
336        self.with_pointer_into_page(offset, len, move |dst| {
337            // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
338            // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have
339            // exclusive access to the slice since the caller guarantees that there are no races.
340            reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) })
341        })
342    }
343}
344
345impl Drop for Page {
346    #[inline]
347    fn drop(&mut self) {
348        // SAFETY: By the type invariants, we have ownership of the page and can free it.
349        unsafe { bindings::__free_pages(self.page.as_ptr(), 0) };
350    }
351}