page_range.rs

  1// SPDX-License-Identifier: GPL-2.0
  2
  3// Copyright (C) 2025 Google LLC.
  4
  5//! This module has utilities for managing a page range where unused pages may be reclaimed by a
  6//! vma shrinker.
  7
  8// To avoid deadlocks, locks are taken in the order:
  9//
 10//  1. mmap lock
 11//  2. spinlock
 12//  3. lru spinlock
 13//
 14// The shrinker will use trylock methods because it locks them in a different order.
 15
 16use crate::AssertSync;
 17
 18use core::{
 19    marker::PhantomPinned,
 20    mem::{size_of, size_of_val, MaybeUninit},
 21    ptr,
 22};
 23
 24use kernel::{
 25    bindings,
 26    error::Result,
 27    ffi::{c_ulong, c_void},
 28    mm::{virt, Mm, MmWithUser},
 29    new_mutex, new_spinlock,
 30    page::{Page, PAGE_SHIFT, PAGE_SIZE},
 31    prelude::*,
 32    str::CStr,
 33    sync::{aref::ARef, Mutex, SpinLock},
 34    task::Pid,
 35    transmute::FromBytes,
 36    types::Opaque,
 37    uaccess::UserSliceReader,
 38};
 39
 40/// Represents a shrinker that can be registered with the kernel.
 41///
 42/// Each shrinker can be used by many `ShrinkablePageRange` objects.
 43#[repr(C)]
 44pub(crate) struct Shrinker {
 45    inner: Opaque<*mut bindings::shrinker>,
 46    list_lru: Opaque<bindings::list_lru>,
 47}
 48
 49// SAFETY: The shrinker and list_lru are thread safe.
 50unsafe impl Send for Shrinker {}
 51// SAFETY: The shrinker and list_lru are thread safe.
 52unsafe impl Sync for Shrinker {}
 53
 54impl Shrinker {
 55    /// Create a new shrinker.
 56    ///
 57    /// # Safety
 58    ///
 59    /// Before using this shrinker with a `ShrinkablePageRange`, the `register` method must have
 60    /// been called exactly once, and it must not have returned an error.
 61    pub(crate) const unsafe fn new() -> Self {
 62        Self {
 63            inner: Opaque::uninit(),
 64            list_lru: Opaque::uninit(),
 65        }
 66    }
 67
 68    /// Register this shrinker with the kernel.
 69    pub(crate) fn register(&'static self, name: &CStr) -> Result<()> {
 70        // SAFETY: These fields are not yet used, so it's okay to zero them.
 71        unsafe {
 72            self.inner.get().write(ptr::null_mut());
 73            self.list_lru.get().write_bytes(0, 1);
 74        }
 75
 76        // SAFETY: The field is not yet used, so we can initialize it.
 77        let ret = unsafe { bindings::__list_lru_init(self.list_lru.get(), false, ptr::null_mut()) };
 78        if ret != 0 {
 79            return Err(Error::from_errno(ret));
 80        }
 81
 82        // SAFETY: The `name` points at a valid c string.
 83        let shrinker = unsafe { bindings::shrinker_alloc(0, name.as_char_ptr()) };
 84        if shrinker.is_null() {
 85            // SAFETY: We initialized it, so its okay to destroy it.
 86            unsafe { bindings::list_lru_destroy(self.list_lru.get()) };
 87            return Err(Error::from_errno(ret));
 88        }
 89
 90        // SAFETY: We're about to register the shrinker, and these are the fields we need to
 91        // initialize. (All other fields are already zeroed.)
 92        unsafe {
 93            (&raw mut (*shrinker).count_objects).write(Some(rust_shrink_count));
 94            (&raw mut (*shrinker).scan_objects).write(Some(rust_shrink_scan));
 95            (&raw mut (*shrinker).private_data).write(self.list_lru.get().cast());
 96        }
 97
 98        // SAFETY: The new shrinker has been fully initialized, so we can register it.
 99        unsafe { bindings::shrinker_register(shrinker) };
100
101        // SAFETY: This initializes the pointer to the shrinker so that we can use it.
102        unsafe { self.inner.get().write(shrinker) };
103
104        Ok(())
105    }
106}
107
108/// A container that manages a page range in a vma.
109///
110/// The pages can be thought of as an array of booleans of whether the pages are usable. The
111/// methods `use_range` and `stop_using_range` set all booleans in a range to true or false
112/// respectively. Initially, no pages are allocated. When a page is not used, it is not freed
113/// immediately. Instead, it is made available to the memory shrinker to free it if the device is
114/// under memory pressure.
115///
116/// It's okay for `use_range` and `stop_using_range` to race with each other, although there's no
117/// way to know whether an index ends up with true or false if a call to `use_range` races with
118/// another call to `stop_using_range` on a given index.
119///
120/// It's also okay for the two methods to race with themselves, e.g. if two threads call
121/// `use_range` on the same index, then that's fine and neither call will return until the page is
122/// allocated and mapped.
123///
124/// The methods that read or write to a range require that the page is marked as in use. So it is
125/// _not_ okay to call `stop_using_range` on a page that is in use by the methods that read or
126/// write to the page.
127#[pin_data(PinnedDrop)]
128pub(crate) struct ShrinkablePageRange {
129    /// Shrinker object registered with the kernel.
130    shrinker: &'static Shrinker,
131    /// Pid using this page range. Only used as debugging information.
132    pid: Pid,
133    /// The mm for the relevant process.
134    mm: ARef<Mm>,
135    /// Used to synchronize calls to `vm_insert_page` and `zap_page_range_single`.
136    #[pin]
137    mm_lock: Mutex<()>,
138    /// Spinlock protecting changes to pages.
139    #[pin]
140    lock: SpinLock<Inner>,
141
142    /// Must not move, since page info has pointers back.
143    #[pin]
144    _pin: PhantomPinned,
145}
146
147// We do not define any ops. For now, used only to check identity of vmas.
148static BINDER_VM_OPS: AssertSync<bindings::vm_operations_struct> = AssertSync(pin_init::zeroed());
149
150// To ensure that we do not accidentally install pages into or zap pages from the wrong vma, we
151// check its vm_ops and private data before using it.
152fn check_vma(vma: &virt::VmaRef, owner: *const ShrinkablePageRange) -> Option<&virt::VmaMixedMap> {
153    // SAFETY: Just reading the vm_ops pointer of any active vma is safe.
154    let vm_ops = unsafe { (*vma.as_ptr()).vm_ops };
155    if !ptr::eq(vm_ops, &BINDER_VM_OPS.0) {
156        return None;
157    }
158
159    // SAFETY: Reading the vm_private_data pointer of a binder-owned vma is safe.
160    let vm_private_data = unsafe { (*vma.as_ptr()).vm_private_data };
161    // The ShrinkablePageRange is only dropped when the Process is dropped, which only happens once
162    // the file's ->release handler is invoked, which means the ShrinkablePageRange outlives any
163    // VMA associated with it, so there can't be any false positives due to pointer reuse here.
164    if !ptr::eq(vm_private_data, owner.cast()) {
165        return None;
166    }
167
168    vma.as_mixedmap_vma()
169}
170
171struct Inner {
172    /// Array of pages.
173    ///
174    /// Since this is also accessed by the shrinker, we can't use a `Box`, which asserts exclusive
175    /// ownership. To deal with that, we manage it using raw pointers.
176    pages: *mut PageInfo,
177    /// Length of the `pages` array.
178    size: usize,
179    /// The address of the vma to insert the pages into.
180    vma_addr: usize,
181}
182
183// SAFETY: proper locking is in place for `Inner`
184unsafe impl Send for Inner {}
185
186type StableMmGuard =
187    kernel::sync::lock::Guard<'static, (), kernel::sync::lock::mutex::MutexBackend>;
188
189/// An array element that describes the current state of a page.
190///
191/// There are three states:
192///
193///  * Free. The page is None. The `lru` element is not queued.
194///  * Available. The page is Some. The `lru` element is queued to the shrinker's lru.
195///  * Used. The page is Some. The `lru` element is not queued.
196///
197/// When an element is available, the shrinker is able to free the page.
198#[repr(C)]
199struct PageInfo {
200    lru: bindings::list_head,
201    page: Option<Page>,
202    range: *const ShrinkablePageRange,
203}
204
205impl PageInfo {
206    /// # Safety
207    ///
208    /// The caller ensures that writing to `me.page` is ok, and that the page is not currently set.
209    unsafe fn set_page(me: *mut PageInfo, page: Page) {
210        // SAFETY: This pointer offset is in bounds.
211        let ptr = unsafe { &raw mut (*me).page };
212
213        // SAFETY: The pointer is valid for writing, so also valid for reading.
214        if unsafe { (*ptr).is_some() } {
215            pr_err!("set_page called when there is already a page");
216            // SAFETY: We will initialize the page again below.
217            unsafe { ptr::drop_in_place(ptr) };
218        }
219
220        // SAFETY: The pointer is valid for writing.
221        unsafe { ptr::write(ptr, Some(page)) };
222    }
223
224    /// # Safety
225    ///
226    /// The caller ensures that reading from `me.page` is ok for the duration of 'a.
227    unsafe fn get_page<'a>(me: *const PageInfo) -> Option<&'a Page> {
228        // SAFETY: This pointer offset is in bounds.
229        let ptr = unsafe { &raw const (*me).page };
230
231        // SAFETY: The pointer is valid for reading.
232        unsafe { (*ptr).as_ref() }
233    }
234
235    /// # Safety
236    ///
237    /// The caller ensures that writing to `me.page` is ok for the duration of 'a.
238    unsafe fn take_page(me: *mut PageInfo) -> Option<Page> {
239        // SAFETY: This pointer offset is in bounds.
240        let ptr = unsafe { &raw mut (*me).page };
241
242        // SAFETY: The pointer is valid for reading.
243        unsafe { (*ptr).take() }
244    }
245
246    /// Add this page to the lru list, if not already in the list.
247    ///
248    /// # Safety
249    ///
250    /// The pointer must be valid, and it must be the right shrinker and nid.
251    unsafe fn list_lru_add(me: *mut PageInfo, nid: i32, shrinker: &'static Shrinker) {
252        // SAFETY: This pointer offset is in bounds.
253        let lru_ptr = unsafe { &raw mut (*me).lru };
254        // SAFETY: The lru pointer is valid, and we're not using it with any other lru list.
255        unsafe { bindings::list_lru_add(shrinker.list_lru.get(), lru_ptr, nid, ptr::null_mut()) };
256    }
257
258    /// Remove this page from the lru list, if it is in the list.
259    ///
260    /// # Safety
261    ///
262    /// The pointer must be valid, and it must be the right shrinker and nid.
263    unsafe fn list_lru_del(me: *mut PageInfo, nid: i32, shrinker: &'static Shrinker) {
264        // SAFETY: This pointer offset is in bounds.
265        let lru_ptr = unsafe { &raw mut (*me).lru };
266        // SAFETY: The lru pointer is valid, and we're not using it with any other lru list.
267        unsafe { bindings::list_lru_del(shrinker.list_lru.get(), lru_ptr, nid, ptr::null_mut()) };
268    }
269}
270
271impl ShrinkablePageRange {
272    /// Create a new `ShrinkablePageRange` using the given shrinker.
273    pub(crate) fn new(shrinker: &'static Shrinker) -> impl PinInit<Self, Error> {
274        try_pin_init!(Self {
275            shrinker,
276            pid: kernel::current!().pid(),
277            mm: ARef::from(&**kernel::current!().mm().ok_or(ESRCH)?),
278            mm_lock <- new_mutex!((), "ShrinkablePageRange::mm"),
279            lock <- new_spinlock!(Inner {
280                pages: ptr::null_mut(),
281                size: 0,
282                vma_addr: 0,
283            }, "ShrinkablePageRange"),
284            _pin: PhantomPinned,
285        })
286    }
287
288    pub(crate) fn stable_trylock_mm(&self) -> Option<StableMmGuard> {
289        // SAFETY: This extends the duration of the reference. Since this call happens before
290        // `mm_lock` is taken in the destructor of `ShrinkablePageRange`, the destructor will block
291        // until the returned guard is dropped. This ensures that the guard is valid until dropped.
292        let mm_lock = unsafe { &*ptr::from_ref(&self.mm_lock) };
293
294        mm_lock.try_lock()
295    }
296
297    /// Register a vma with this page range. Returns the size of the region.
298    pub(crate) fn register_with_vma(&self, vma: &virt::VmaNew) -> Result<usize> {
299        let num_bytes = usize::min(vma.end() - vma.start(), bindings::SZ_4M as usize);
300        let num_pages = num_bytes >> PAGE_SHIFT;
301
302        if !ptr::eq::<Mm>(&*self.mm, &**vma.mm()) {
303            pr_debug!("Failed to register with vma: invalid vma->vm_mm");
304            return Err(EINVAL);
305        }
306        if num_pages == 0 {
307            pr_debug!("Failed to register with vma: size zero");
308            return Err(EINVAL);
309        }
310
311        let mut pages = KVVec::<PageInfo>::with_capacity(num_pages, GFP_KERNEL)?;
312
313        // SAFETY: This just initializes the pages array.
314        unsafe {
315            let self_ptr = self as *const ShrinkablePageRange;
316            for i in 0..num_pages {
317                let info = pages.as_mut_ptr().add(i);
318                (&raw mut (*info).range).write(self_ptr);
319                (&raw mut (*info).page).write(None);
320                let lru = &raw mut (*info).lru;
321                (&raw mut (*lru).next).write(lru);
322                (&raw mut (*lru).prev).write(lru);
323            }
324        }
325
326        let mut inner = self.lock.lock();
327        if inner.size > 0 {
328            pr_debug!("Failed to register with vma: already registered");
329            drop(inner);
330            return Err(EBUSY);
331        }
332
333        inner.pages = pages.into_raw_parts().0;
334        inner.size = num_pages;
335        inner.vma_addr = vma.start();
336
337        // This pointer is only used for comparison - it's not dereferenced.
338        //
339        // SAFETY: We own the vma, and we don't use any methods on VmaNew that rely on
340        // `vm_private_data`.
341        unsafe {
342            (*vma.as_ptr()).vm_private_data = ptr::from_ref(self).cast_mut().cast::<c_void>()
343        };
344
345        // SAFETY: We own the vma, and we don't use any methods on VmaNew that rely on
346        // `vm_ops`.
347        unsafe { (*vma.as_ptr()).vm_ops = &BINDER_VM_OPS.0 };
348
349        Ok(num_pages)
350    }
351
352    /// Make sure that the given pages are allocated and mapped.
353    ///
354    /// Must not be called from an atomic context.
355    pub(crate) fn use_range(&self, start: usize, end: usize) -> Result<()> {
356        if start >= end {
357            return Ok(());
358        }
359        let mut inner = self.lock.lock();
360        assert!(end <= inner.size);
361
362        for i in start..end {
363            // SAFETY: This pointer offset is in bounds.
364            let page_info = unsafe { inner.pages.add(i) };
365
366            // SAFETY: The pointer is valid, and we hold the lock so reading from the page is okay.
367            if let Some(page) = unsafe { PageInfo::get_page(page_info) } {
368                // Since we're going to use the page, we should remove it from the lru list so that
369                // the shrinker will not free it.
370                //
371                // SAFETY: The pointer is valid, and this is the right shrinker.
372                //
373                // The shrinker can't free the page between the check and this call to
374                // `list_lru_del` because we hold the lock.
375                unsafe { PageInfo::list_lru_del(page_info, page.nid(), self.shrinker) };
376            } else {
377                // We have to allocate a new page. Use the slow path.
378                drop(inner);
379                // SAFETY: `i < end <= inner.size` so `i` is in bounds.
380                match unsafe { self.use_page_slow(i) } {
381                    Ok(()) => {}
382                    Err(err) => {
383                        pr_warn!("Error in use_page_slow: {:?}", err);
384                        return Err(err);
385                    }
386                }
387                inner = self.lock.lock();
388            }
389        }
390        Ok(())
391    }
392
393    /// Mark the given page as in use, slow path.
394    ///
395    /// Must not be called from an atomic context.
396    ///
397    /// # Safety
398    ///
399    /// Assumes that `i` is in bounds.
400    #[cold]
401    unsafe fn use_page_slow(&self, i: usize) -> Result<()> {
402        let new_page = Page::alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO)?;
403
404        let mm_mutex = self.mm_lock.lock();
405        let inner = self.lock.lock();
406
407        // SAFETY: This pointer offset is in bounds.
408        let page_info = unsafe { inner.pages.add(i) };
409
410        // SAFETY: The pointer is valid, and we hold the lock so reading from the page is okay.
411        if let Some(page) = unsafe { PageInfo::get_page(page_info) } {
412            // The page was already there, or someone else added the page while we didn't hold the
413            // spinlock.
414            //
415            // SAFETY: The pointer is valid, and this is the right shrinker.
416            //
417            // The shrinker can't free the page between the check and this call to
418            // `list_lru_del` because we hold the lock.
419            unsafe { PageInfo::list_lru_del(page_info, page.nid(), self.shrinker) };
420            return Ok(());
421        }
422
423        let vma_addr = inner.vma_addr;
424        // Release the spinlock while we insert the page into the vma.
425        drop(inner);
426
427        // No overflow since we stay in bounds of the vma.
428        let user_page_addr = vma_addr + (i << PAGE_SHIFT);
429
430        // We use `mmput_async` when dropping the `mm` because `use_page_slow` is usually used from
431        // a remote process. If the call to `mmput` races with the process shutting down, then the
432        // caller of `use_page_slow` becomes responsible for cleaning up the `mm`, which doesn't
433        // happen until it returns to userspace. However, the caller might instead go to sleep and
434        // wait for the owner of the `mm` to wake it up, which doesn't happen because it's in the
435        // middle of a shutdown process that won't complete until the `mm` is dropped. This can
436        // amount to a deadlock.
437        //
438        // Using `mmput_async` avoids this, because then the `mm` cleanup is instead queued to a
439        // workqueue.
440        check_vma(
441            MmWithUser::into_mmput_async(self.mm.mmget_not_zero().ok_or(ESRCH)?)
442                .mmap_read_lock()
443                .vma_lookup(vma_addr)
444                .ok_or(ESRCH)?,
445            self,
446        )
447        .ok_or(ESRCH)?
448        .vm_insert_page(user_page_addr, &new_page)
449        .inspect_err(|err| {
450            pr_warn!(
451                "Failed to vm_insert_page({}): vma_addr:{} i:{} err:{:?}",
452                user_page_addr,
453                vma_addr,
454                i,
455                err
456            )
457        })?;
458
459        let inner = self.lock.lock();
460
461        // SAFETY: The `page_info` pointer is valid and currently does not have a page. The page
462        // can be written to since we hold the lock.
463        //
464        // We released and reacquired the spinlock since we checked that the page is null, but we
465        // always hold the mm_lock mutex when setting the page to a non-null value, so it's not
466        // possible for someone else to have changed it since our check.
467        unsafe { PageInfo::set_page(page_info, new_page) };
468
469        drop(inner);
470        drop(mm_mutex);
471
472        Ok(())
473    }
474
475    /// If the given page is in use, then mark it as available so that the shrinker can free it.
476    ///
477    /// May be called from an atomic context.
478    pub(crate) fn stop_using_range(&self, start: usize, end: usize) {
479        if start >= end {
480            return;
481        }
482        let inner = self.lock.lock();
483        assert!(end <= inner.size);
484
485        for i in (start..end).rev() {
486            // SAFETY: The pointer is in bounds.
487            let page_info = unsafe { inner.pages.add(i) };
488
489            // SAFETY: Okay for reading since we have the lock.
490            if let Some(page) = unsafe { PageInfo::get_page(page_info) } {
491                // SAFETY: The pointer is valid, and it's the right shrinker.
492                unsafe { PageInfo::list_lru_add(page_info, page.nid(), self.shrinker) };
493            }
494        }
495    }
496
497    /// Helper for reading or writing to a range of bytes that may overlap with several pages.
498    ///
499    /// # Safety
500    ///
501    /// All pages touched by this operation must be in use for the duration of this call.
502    unsafe fn iterate<T>(&self, mut offset: usize, mut size: usize, mut cb: T) -> Result
503    where
504        T: FnMut(&Page, usize, usize) -> Result,
505    {
506        if size == 0 {
507            return Ok(());
508        }
509
510        let (pages, num_pages) = {
511            let inner = self.lock.lock();
512            (inner.pages, inner.size)
513        };
514        let num_bytes = num_pages << PAGE_SHIFT;
515
516        // Check that the request is within the buffer.
517        if offset.checked_add(size).ok_or(EFAULT)? > num_bytes {
518            return Err(EFAULT);
519        }
520
521        let mut page_index = offset >> PAGE_SHIFT;
522        offset &= PAGE_SIZE - 1;
523        while size > 0 {
524            let available = usize::min(size, PAGE_SIZE - offset);
525            // SAFETY: The pointer is in bounds.
526            let page_info = unsafe { pages.add(page_index) };
527            // SAFETY: The caller guarantees that this page is in the "in use" state for the
528            // duration of this call to `iterate`, so nobody will change the page.
529            let page = unsafe { PageInfo::get_page(page_info) };
530            if page.is_none() {
531                pr_warn!("Page is null!");
532            }
533            let page = page.ok_or(EFAULT)?;
534            cb(page, offset, available)?;
535            size -= available;
536            page_index += 1;
537            offset = 0;
538        }
539        Ok(())
540    }
541
542    /// Copy from userspace into this page range.
543    ///
544    /// # Safety
545    ///
546    /// All pages touched by this operation must be in use for the duration of this call.
547    pub(crate) unsafe fn copy_from_user_slice(
548        &self,
549        reader: &mut UserSliceReader,
550        offset: usize,
551        size: usize,
552    ) -> Result {
553        // SAFETY: `self.iterate` has the same safety requirements as `copy_from_user_slice`.
554        unsafe {
555            self.iterate(offset, size, |page, offset, to_copy| {
556                page.copy_from_user_slice_raw(reader, offset, to_copy)
557            })
558        }
559    }
560
561    /// Copy from this page range into kernel space.
562    ///
563    /// # Safety
564    ///
565    /// All pages touched by this operation must be in use for the duration of this call.
566    pub(crate) unsafe fn read<T: FromBytes>(&self, offset: usize) -> Result<T> {
567        let mut out = MaybeUninit::<T>::uninit();
568        let mut out_offset = 0;
569        // SAFETY: `self.iterate` has the same safety requirements as `read`.
570        unsafe {
571            self.iterate(offset, size_of::<T>(), |page, offset, to_copy| {
572                // SAFETY: The sum of `offset` and `to_copy` is bounded by the size of T.
573                let obj_ptr = (out.as_mut_ptr() as *mut u8).add(out_offset);
574                // SAFETY: The pointer points is in-bounds of the `out` variable, so it is valid.
575                page.read_raw(obj_ptr, offset, to_copy)?;
576                out_offset += to_copy;
577                Ok(())
578            })?;
579        }
580        // SAFETY: We just initialised the data.
581        Ok(unsafe { out.assume_init() })
582    }
583
584    /// Copy from kernel space into this page range.
585    ///
586    /// # Safety
587    ///
588    /// All pages touched by this operation must be in use for the duration of this call.
589    pub(crate) unsafe fn write<T: ?Sized>(&self, offset: usize, obj: &T) -> Result {
590        let mut obj_offset = 0;
591        // SAFETY: `self.iterate` has the same safety requirements as `write`.
592        unsafe {
593            self.iterate(offset, size_of_val(obj), |page, offset, to_copy| {
594                // SAFETY: The sum of `offset` and `to_copy` is bounded by the size of T.
595                let obj_ptr = (obj as *const T as *const u8).add(obj_offset);
596                // SAFETY: We have a reference to the object, so the pointer is valid.
597                page.write_raw(obj_ptr, offset, to_copy)?;
598                obj_offset += to_copy;
599                Ok(())
600            })
601        }
602    }
603
604    /// Write zeroes to the given range.
605    ///
606    /// # Safety
607    ///
608    /// All pages touched by this operation must be in use for the duration of this call.
609    pub(crate) unsafe fn fill_zero(&self, offset: usize, size: usize) -> Result {
610        // SAFETY: `self.iterate` has the same safety requirements as `copy_into`.
611        unsafe {
612            self.iterate(offset, size, |page, offset, len| {
613                page.fill_zero_raw(offset, len)
614            })
615        }
616    }
617}
618
619#[pinned_drop]
620impl PinnedDrop for ShrinkablePageRange {
621    fn drop(self: Pin<&mut Self>) {
622        let (pages, size) = {
623            let lock = self.lock.lock();
624            (lock.pages, lock.size)
625        };
626
627        if size == 0 {
628            return;
629        }
630
631        // Note: This call is also necessary for the safety of `stable_trylock_mm`.
632        let mm_lock = self.mm_lock.lock();
633
634        // This is the destructor, so unlike the other methods, we only need to worry about races
635        // with the shrinker here. Since we hold the `mm_lock`, we also can't race with the
636        // shrinker, and after this loop, the shrinker will not access any of our pages since we
637        // removed them from the lru list.
638        for i in 0..size {
639            // SAFETY: Loop is in-bounds of the size.
640            let p_ptr = unsafe { pages.add(i) };
641            // SAFETY: No other readers, so we can read.
642            if let Some(p) = unsafe { PageInfo::get_page(p_ptr) } {
643                // SAFETY: The pointer is valid and it's the right shrinker.
644                unsafe { PageInfo::list_lru_del(p_ptr, p.nid(), self.shrinker) };
645            }
646        }
647
648        drop(mm_lock);
649
650        // SAFETY: `pages` was allocated as an `KVVec<PageInfo>` with capacity `size`. Furthermore,
651        // all `size` elements are initialized. Also, the array is no longer shared with the
652        // shrinker due to the above loop.
653        drop(unsafe { KVVec::from_raw_parts(pages, size, size) });
654    }
655}
656
657/// # Safety
658/// Called by the shrinker.
659#[no_mangle]
660unsafe extern "C" fn rust_shrink_count(
661    shrink: *mut bindings::shrinker,
662    _sc: *mut bindings::shrink_control,
663) -> c_ulong {
664    // SAFETY: We can access our own private data.
665    let list_lru = unsafe { (*shrink).private_data.cast::<bindings::list_lru>() };
666    // SAFETY: Accessing the lru list is okay. Just an FFI call.
667    unsafe { bindings::list_lru_count(list_lru) }
668}
669
670/// # Safety
671/// Called by the shrinker.
672#[no_mangle]
673unsafe extern "C" fn rust_shrink_scan(
674    shrink: *mut bindings::shrinker,
675    sc: *mut bindings::shrink_control,
676) -> c_ulong {
677    // SAFETY: We can access our own private data.
678    let list_lru = unsafe { (*shrink).private_data.cast::<bindings::list_lru>() };
679    // SAFETY: Caller guarantees that it is safe to read this field.
680    let nr_to_scan = unsafe { (*sc).nr_to_scan };
681    // SAFETY: Accessing the lru list is okay. Just an FFI call.
682    unsafe {
683        bindings::list_lru_walk(
684            list_lru,
685            Some(bindings::rust_shrink_free_page_wrap),
686            ptr::null_mut(),
687            nr_to_scan,
688        )
689    }
690}
691
692const LRU_SKIP: bindings::lru_status = bindings::lru_status_LRU_SKIP;
693const LRU_REMOVED_ENTRY: bindings::lru_status = bindings::lru_status_LRU_REMOVED_RETRY;
694
695/// # Safety
696/// Called by the shrinker.
697#[no_mangle]
698unsafe extern "C" fn rust_shrink_free_page(
699    item: *mut bindings::list_head,
700    lru: *mut bindings::list_lru_one,
701    _cb_arg: *mut c_void,
702) -> bindings::lru_status {
703    // Fields that should survive after unlocking the lru lock.
704    let page;
705    let page_index;
706    let mm;
707    let mmap_read;
708    let mm_mutex;
709    let vma_addr;
710    let range_ptr;
711
712    {
713        // CAST: The `list_head` field is first in `PageInfo`.
714        let info = item as *mut PageInfo;
715        // SAFETY: The `range` field of `PageInfo` is immutable.
716        range_ptr = unsafe { (*info).range };
717        // SAFETY: The `range` outlives its `PageInfo` values.
718        let range = unsafe { &*range_ptr };
719
720        mm = match range.mm.mmget_not_zero() {
721            Some(mm) => MmWithUser::into_mmput_async(mm),
722            None => return LRU_SKIP,
723        };
724
725        mm_mutex = match range.stable_trylock_mm() {
726            Some(guard) => guard,
727            None => return LRU_SKIP,
728        };
729
730        mmap_read = match mm.mmap_read_trylock() {
731            Some(guard) => guard,
732            None => return LRU_SKIP,
733        };
734
735        // We can't lock it normally here, since we hold the lru lock.
736        let inner = match range.lock.try_lock() {
737            Some(inner) => inner,
738            None => return LRU_SKIP,
739        };
740
741        // SAFETY: The item is in this lru list, so it's okay to remove it.
742        unsafe { bindings::list_lru_isolate(lru, item) };
743
744        // SAFETY: Both pointers are in bounds of the same allocation.
745        page_index = unsafe { info.offset_from(inner.pages) } as usize;
746
747        // SAFETY: We hold the spinlock, so we can take the page.
748        //
749        // This sets the page pointer to zero before we unmap it from the vma. However, we call
750        // `zap_page_range` before we release the mmap lock, so `use_page_slow` will not be able to
751        // insert a new page until after our call to `zap_page_range`.
752        page = unsafe { PageInfo::take_page(info) };
753        vma_addr = inner.vma_addr;
754
755        // From this point on, we don't access this PageInfo or ShrinkablePageRange again, because
756        // they can be freed at any point after we unlock `lru_lock`. This is with the exception of
757        // `mm_mutex` which is kept alive by holding the lock.
758    }
759
760    // SAFETY: The lru lock is locked when this method is called.
761    unsafe { bindings::spin_unlock(&raw mut (*lru).lock) };
762
763    if let Some(unchecked_vma) = mmap_read.vma_lookup(vma_addr) {
764        if let Some(vma) = check_vma(unchecked_vma, range_ptr) {
765            let user_page_addr = vma_addr + (page_index << PAGE_SHIFT);
766            vma.zap_page_range_single(user_page_addr, PAGE_SIZE);
767        }
768    }
769
770    drop(mmap_read);
771    drop(mm_mutex);
772    drop(mm);
773    drop(page);
774
775    LRU_REMOVED_ENTRY
776}