arc.rs

  1// SPDX-License-Identifier: GPL-2.0
  2
  3//! A reference-counted pointer.
  4//!
  5//! This module implements a way for users to create reference-counted objects and pointers to
  6//! them. Such a pointer automatically increments and decrements the count, and drops the
  7//! underlying object when it reaches zero. It is also safe to use concurrently from multiple
  8//! threads.
  9//!
 10//! It is different from the standard library's [`Arc`] in a few ways:
 11//! 1. It is backed by the kernel's [`Refcount`] type.
 12//! 2. It does not support weak references, which allows it to be half the size.
 13//! 3. It saturates the reference count instead of aborting when it goes over a threshold.
 14//! 4. It does not provide a `get_mut` method, so the ref counted object is pinned.
 15//! 5. The object in [`Arc`] is pinned implicitly.
 16//!
 17//! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html
 18
 19use crate::{
 20    alloc::{AllocError, Flags, KBox},
 21    ffi::c_void,
 22    fmt,
 23    init::InPlaceInit,
 24    sync::Refcount,
 25    try_init,
 26    types::ForeignOwnable,
 27};
 28use core::{
 29    alloc::Layout,
 30    borrow::{Borrow, BorrowMut},
 31    marker::PhantomData,
 32    mem::{ManuallyDrop, MaybeUninit},
 33    ops::{Deref, DerefMut},
 34    pin::Pin,
 35    ptr::NonNull,
 36};
 37use pin_init::{self, pin_data, InPlaceWrite, Init, PinInit};
 38
 39mod std_vendor;
 40
 41/// A reference-counted pointer to an instance of `T`.
 42///
 43/// The reference count is incremented when new instances of [`Arc`] are created, and decremented
 44/// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.
 45///
 46/// # Invariants
 47///
 48/// The reference count on an instance of [`Arc`] is always non-zero.
 49/// The object pointed to by [`Arc`] is always pinned.
 50///
 51/// # Examples
 52///
 53/// ```
 54/// use kernel::sync::Arc;
 55///
 56/// struct Example {
 57///     a: u32,
 58///     b: u32,
 59/// }
 60///
 61/// // Create a refcounted instance of `Example`.
 62/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
 63///
 64/// // Get a new pointer to `obj` and increment the refcount.
 65/// let cloned = obj.clone();
 66///
 67/// // Assert that both `obj` and `cloned` point to the same underlying object.
 68/// assert!(core::ptr::eq(&*obj, &*cloned));
 69///
 70/// // Destroy `obj` and decrement its refcount.
 71/// drop(obj);
 72///
 73/// // Check that the values are still accessible through `cloned`.
 74/// assert_eq!(cloned.a, 10);
 75/// assert_eq!(cloned.b, 20);
 76///
 77/// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed.
 78/// # Ok::<(), Error>(())
 79/// ```
 80///
 81/// Using `Arc<T>` as the type of `self`:
 82///
 83/// ```
 84/// use kernel::sync::Arc;
 85///
 86/// struct Example {
 87///     a: u32,
 88///     b: u32,
 89/// }
 90///
 91/// impl Example {
 92///     fn take_over(self: Arc<Self>) {
 93///         // ...
 94///     }
 95///
 96///     fn use_reference(self: &Arc<Self>) {
 97///         // ...
 98///     }
 99/// }
100///
101/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
102/// obj.use_reference();
103/// obj.take_over();
104/// # Ok::<(), Error>(())
105/// ```
106///
107/// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`:
108///
109/// ```
110/// use kernel::sync::{Arc, ArcBorrow};
111///
112/// trait MyTrait {
113///     // Trait has a function whose `self` type is `Arc<Self>`.
114///     fn example1(self: Arc<Self>) {}
115///
116///     // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`.
117///     fn example2(self: ArcBorrow<'_, Self>) {}
118/// }
119///
120/// struct Example;
121/// impl MyTrait for Example {}
122///
123/// // `obj` has type `Arc<Example>`.
124/// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?;
125///
126/// // `coerced` has type `Arc<dyn MyTrait>`.
127/// let coerced: Arc<dyn MyTrait> = obj;
128/// # Ok::<(), Error>(())
129/// ```
130#[repr(transparent)]
131#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))]
132pub struct Arc<T: ?Sized> {
133    ptr: NonNull<ArcInner<T>>,
134    // NB: this informs dropck that objects of type `ArcInner<T>` may be used in `<Arc<T> as
135    // Drop>::drop`. Note that dropck already assumes that objects of type `T` may be used in
136    // `<Arc<T> as Drop>::drop` and the distinction between `T` and `ArcInner<T>` is not presently
137    // meaningful with respect to dropck - but this may change in the future so this is left here
138    // out of an abundance of caution.
139    //
140    // See <https://doc.rust-lang.org/nomicon/phantom-data.html#generic-parameters-and-drop-checking>
141    // for more detail on the semantics of dropck in the presence of `PhantomData`.
142    _p: PhantomData<ArcInner<T>>,
143}
144
145#[pin_data]
146#[repr(C)]
147struct ArcInner<T: ?Sized> {
148    refcount: Refcount,
149    data: T,
150}
151
152impl<T: ?Sized> ArcInner<T> {
153    /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`].
154    ///
155    /// # Safety
156    ///
157    /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must
158    /// not yet have been destroyed.
159    unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> {
160        let refcount_layout = Layout::new::<Refcount>();
161        // SAFETY: The caller guarantees that the pointer is valid.
162        let val_layout = Layout::for_value(unsafe { &*ptr });
163        // SAFETY: We're computing the layout of a real struct that existed when compiling this
164        // binary, so its layout is not so large that it can trigger arithmetic overflow.
165        let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 };
166
167        // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and
168        // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field.
169        //
170        // This is documented at:
171        // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>.
172        let ptr = ptr as *const ArcInner<T>;
173
174        // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the
175        // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is
176        // still valid.
177        let ptr = unsafe { ptr.byte_sub(val_offset) };
178
179        // SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null
180        // address.
181        unsafe { NonNull::new_unchecked(ptr.cast_mut()) }
182    }
183}
184
185// This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the
186// dynamically-sized type (DST) `U`.
187#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
188impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {}
189
190// This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`.
191#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
192impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {}
193
194// SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because
195// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
196// `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a
197// mutable reference when the reference count reaches zero and `T` is dropped.
198unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
199
200// SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync`
201// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally,
202// it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an
203// `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when
204// the reference count reaches zero and `T` is dropped.
205unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
206
207impl<T> InPlaceInit<T> for Arc<T> {
208    type PinnedSelf = Self;
209
210    #[inline]
211    fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
212    where
213        E: From<AllocError>,
214    {
215        UniqueArc::try_pin_init(init, flags).map(|u| u.into())
216    }
217
218    #[inline]
219    fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
220    where
221        E: From<AllocError>,
222    {
223        UniqueArc::try_init(init, flags).map(|u| u.into())
224    }
225}
226
227impl<T> Arc<T> {
228    /// Constructs a new reference counted instance of `T`.
229    pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> {
230        // INVARIANT: The refcount is initialised to a non-zero value.
231        let value = ArcInner {
232            refcount: Refcount::new(1),
233            data: contents,
234        };
235
236        let inner = KBox::new(value, flags)?;
237        let inner = KBox::leak(inner).into();
238
239        // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new
240        // `Arc` object.
241        Ok(unsafe { Self::from_inner(inner) })
242    }
243}
244
245impl<T: ?Sized> Arc<T> {
246    /// Constructs a new [`Arc`] from an existing [`ArcInner`].
247    ///
248    /// # Safety
249    ///
250    /// The caller must ensure that `inner` points to a valid location and has a non-zero reference
251    /// count, one of which will be owned by the new [`Arc`] instance.
252    unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self {
253        // INVARIANT: By the safety requirements, the invariants hold.
254        Arc {
255            ptr: inner,
256            _p: PhantomData,
257        }
258    }
259
260    /// Convert the [`Arc`] into a raw pointer.
261    ///
262    /// The raw pointer has ownership of the refcount that this Arc object owned.
263    pub fn into_raw(self) -> *const T {
264        let ptr = self.ptr.as_ptr();
265        core::mem::forget(self);
266        // SAFETY: The pointer is valid.
267        unsafe { core::ptr::addr_of!((*ptr).data) }
268    }
269
270    /// Return a raw pointer to the data in this arc.
271    pub fn as_ptr(this: &Self) -> *const T {
272        let ptr = this.ptr.as_ptr();
273
274        // SAFETY: As `ptr` points to a valid allocation of type `ArcInner`,
275        // field projection to `data`is within bounds of the allocation.
276        unsafe { core::ptr::addr_of!((*ptr).data) }
277    }
278
279    /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`].
280    ///
281    /// # Safety
282    ///
283    /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it
284    /// must not be called more than once for each previous call to [`Arc::into_raw`].
285    pub unsafe fn from_raw(ptr: *const T) -> Self {
286        // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
287        // `Arc` that is still valid.
288        let ptr = unsafe { ArcInner::container_of(ptr) };
289
290        // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the
291        // reference count held then will be owned by the new `Arc` object.
292        unsafe { Self::from_inner(ptr) }
293    }
294
295    /// Returns an [`ArcBorrow`] from the given [`Arc`].
296    ///
297    /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method
298    /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised.
299    #[inline]
300    pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> {
301        // SAFETY: The constraint that the lifetime of the shared reference must outlive that of
302        // the returned `ArcBorrow` ensures that the object remains alive and that no mutable
303        // reference can be created.
304        unsafe { ArcBorrow::new(self.ptr) }
305    }
306
307    /// Compare whether two [`Arc`] pointers reference the same underlying object.
308    pub fn ptr_eq(this: &Self, other: &Self) -> bool {
309        core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr())
310    }
311
312    /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique.
313    ///
314    /// When this destroys the `Arc`, it does so while properly avoiding races. This means that
315    /// this method will never call the destructor of the value.
316    ///
317    /// # Examples
318    ///
319    /// ```
320    /// use kernel::sync::{Arc, UniqueArc};
321    ///
322    /// let arc = Arc::new(42, GFP_KERNEL)?;
323    /// let unique_arc = Arc::into_unique_or_drop(arc);
324    ///
325    /// // The above conversion should succeed since refcount of `arc` is 1.
326    /// assert!(unique_arc.is_some());
327    ///
328    /// assert_eq!(*(unique_arc.unwrap()), 42);
329    ///
330    /// # Ok::<(), Error>(())
331    /// ```
332    ///
333    /// ```
334    /// use kernel::sync::{Arc, UniqueArc};
335    ///
336    /// let arc = Arc::new(42, GFP_KERNEL)?;
337    /// let another = arc.clone();
338    ///
339    /// let unique_arc = Arc::into_unique_or_drop(arc);
340    ///
341    /// // The above conversion should fail since refcount of `arc` is >1.
342    /// assert!(unique_arc.is_none());
343    ///
344    /// # Ok::<(), Error>(())
345    /// ```
346    pub fn into_unique_or_drop(this: Self) -> Option<Pin<UniqueArc<T>>> {
347        // We will manually manage the refcount in this method, so we disable the destructor.
348        let this = ManuallyDrop::new(this);
349        // SAFETY: We own a refcount, so the pointer is still valid.
350        let refcount = unsafe { &this.ptr.as_ref().refcount };
351
352        // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will
353        // return without further touching the `Arc`. If the refcount reaches zero, then there are
354        // no other arcs, and we can create a `UniqueArc`.
355        if refcount.dec_and_test() {
356            refcount.set(1);
357
358            // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We
359            // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin
360            // their values.
361            Some(Pin::from(UniqueArc {
362                inner: ManuallyDrop::into_inner(this),
363            }))
364        } else {
365            None
366        }
367    }
368}
369
370// SAFETY: The pointer returned by `into_foreign` was originally allocated as an
371// `KBox<ArcInner<T>>`, so that type is what determines the alignment.
372unsafe impl<T: 'static> ForeignOwnable for Arc<T> {
373    const FOREIGN_ALIGN: usize = <KBox<ArcInner<T>> as ForeignOwnable>::FOREIGN_ALIGN;
374
375    type Borrowed<'a> = ArcBorrow<'a, T>;
376    type BorrowedMut<'a> = Self::Borrowed<'a>;
377
378    fn into_foreign(self) -> *mut c_void {
379        ManuallyDrop::new(self).ptr.as_ptr().cast()
380    }
381
382    unsafe fn from_foreign(ptr: *mut c_void) -> Self {
383        // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
384        // call to `Self::into_foreign`.
385        let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) };
386
387        // SAFETY: By the safety requirement of this function, we know that `ptr` came from
388        // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and
389        // holds a reference count increment that is transferrable to us.
390        unsafe { Self::from_inner(inner) }
391    }
392
393    unsafe fn borrow<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> {
394        // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
395        // call to `Self::into_foreign`.
396        let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) };
397
398        // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive
399        // for the lifetime of the returned value.
400        unsafe { ArcBorrow::new(inner) }
401    }
402
403    unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> {
404        // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety
405        // requirements for `borrow`.
406        unsafe { <Self as ForeignOwnable>::borrow(ptr) }
407    }
408}
409
410impl<T: ?Sized> Deref for Arc<T> {
411    type Target = T;
412
413    fn deref(&self) -> &Self::Target {
414        // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
415        // safe to dereference it.
416        unsafe { &self.ptr.as_ref().data }
417    }
418}
419
420impl<T: ?Sized> AsRef<T> for Arc<T> {
421    fn as_ref(&self) -> &T {
422        self.deref()
423    }
424}
425
426/// # Examples
427///
428/// ```
429/// # use core::borrow::Borrow;
430/// # use kernel::sync::Arc;
431/// struct Foo<B: Borrow<u32>>(B);
432///
433/// // Owned instance.
434/// let owned = Foo(1);
435///
436/// // Shared instance.
437/// let arc = Arc::new(1, GFP_KERNEL)?;
438/// let shared = Foo(arc.clone());
439///
440/// let i = 1;
441/// // Borrowed from `i`.
442/// let borrowed = Foo(&i);
443/// # Ok::<(), Error>(())
444/// ```
445impl<T: ?Sized> Borrow<T> for Arc<T> {
446    fn borrow(&self) -> &T {
447        self.deref()
448    }
449}
450
451impl<T: ?Sized> Clone for Arc<T> {
452    fn clone(&self) -> Self {
453        // INVARIANT: `Refcount` saturates the refcount, so it cannot overflow to zero.
454        // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
455        // safe to increment the refcount.
456        unsafe { self.ptr.as_ref() }.refcount.inc();
457
458        // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`.
459        unsafe { Self::from_inner(self.ptr) }
460    }
461}
462
463impl<T: ?Sized> Drop for Arc<T> {
464    fn drop(&mut self) {
465        // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and
466        // this instance is being dropped, so the broken invariant is not observable.
467        // SAFETY: By the type invariant, there is necessarily a reference to the object.
468        let is_zero = unsafe { self.ptr.as_ref() }.refcount.dec_and_test();
469        if is_zero {
470            // The count reached zero, we must free the memory.
471            //
472            // SAFETY: The pointer was initialised from the result of `KBox::leak`.
473            unsafe { drop(KBox::from_raw(self.ptr.as_ptr())) };
474        }
475    }
476}
477
478impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> {
479    fn from(item: UniqueArc<T>) -> Self {
480        item.inner
481    }
482}
483
484impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {
485    fn from(item: Pin<UniqueArc<T>>) -> Self {
486        // SAFETY: The type invariants of `Arc` guarantee that the data is pinned.
487        unsafe { Pin::into_inner_unchecked(item).inner }
488    }
489}
490
491/// A borrowed reference to an [`Arc`] instance.
492///
493/// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler
494/// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance.
495///
496/// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>`
497/// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference)
498/// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double
499/// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if
500/// needed.
501///
502/// # Invariants
503///
504/// There are no mutable references to the underlying [`Arc`], and it remains valid for the
505/// lifetime of the [`ArcBorrow`] instance.
506///
507/// # Examples
508///
509/// ```
510/// use kernel::sync::{Arc, ArcBorrow};
511///
512/// struct Example;
513///
514/// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> {
515///     e.into()
516/// }
517///
518/// let obj = Arc::new(Example, GFP_KERNEL)?;
519/// let cloned = do_something(obj.as_arc_borrow());
520///
521/// // Assert that both `obj` and `cloned` point to the same underlying object.
522/// assert!(core::ptr::eq(&*obj, &*cloned));
523/// # Ok::<(), Error>(())
524/// ```
525///
526/// Using `ArcBorrow<T>` as the type of `self`:
527///
528/// ```
529/// use kernel::sync::{Arc, ArcBorrow};
530///
531/// struct Example {
532///     a: u32,
533///     b: u32,
534/// }
535///
536/// impl Example {
537///     fn use_reference(self: ArcBorrow<'_, Self>) {
538///         // ...
539///     }
540/// }
541///
542/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
543/// obj.as_arc_borrow().use_reference();
544/// # Ok::<(), Error>(())
545/// ```
546#[repr(transparent)]
547#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))]
548pub struct ArcBorrow<'a, T: ?Sized + 'a> {
549    inner: NonNull<ArcInner<T>>,
550    _p: PhantomData<&'a ()>,
551}
552
553// This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into
554// `ArcBorrow<U>`.
555#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
556impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>>
557    for ArcBorrow<'_, T>
558{
559}
560
561impl<T: ?Sized> Clone for ArcBorrow<'_, T> {
562    fn clone(&self) -> Self {
563        *self
564    }
565}
566
567impl<T: ?Sized> Copy for ArcBorrow<'_, T> {}
568
569impl<T: ?Sized> ArcBorrow<'_, T> {
570    /// Creates a new [`ArcBorrow`] instance.
571    ///
572    /// # Safety
573    ///
574    /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance:
575    /// 1. That `inner` remains valid;
576    /// 2. That no mutable references to `inner` are created.
577    unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self {
578        // INVARIANT: The safety requirements guarantee the invariants.
579        Self {
580            inner,
581            _p: PhantomData,
582        }
583    }
584
585    /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with
586    /// [`Arc::into_raw`] or [`Arc::as_ptr`].
587    ///
588    /// # Safety
589    ///
590    /// * The provided pointer must originate from a call to [`Arc::into_raw`] or [`Arc::as_ptr`].
591    /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must
592    ///   not hit zero.
593    /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a
594    ///   [`UniqueArc`] reference to this value.
595    pub unsafe fn from_raw(ptr: *const T) -> Self {
596        // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
597        // `Arc` that is still valid.
598        let ptr = unsafe { ArcInner::container_of(ptr) };
599
600        // SAFETY: The caller promises that the value remains valid since the reference count must
601        // not hit zero, and no mutable reference will be created since that would involve a
602        // `UniqueArc`.
603        unsafe { Self::new(ptr) }
604    }
605}
606
607impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> {
608    fn from(b: ArcBorrow<'_, T>) -> Self {
609        // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop`
610        // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the
611        // increment.
612        ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) })
613            .deref()
614            .clone()
615    }
616}
617
618impl<T: ?Sized> Deref for ArcBorrow<'_, T> {
619    type Target = T;
620
621    fn deref(&self) -> &Self::Target {
622        // SAFETY: By the type invariant, the underlying object is still alive with no mutable
623        // references to it, so it is safe to create a shared reference.
624        unsafe { &self.inner.as_ref().data }
625    }
626}
627
628/// A refcounted object that is known to have a refcount of 1.
629///
630/// It is mutable and can be converted to an [`Arc`] so that it can be shared.
631///
632/// # Invariants
633///
634/// `inner` always has a reference count of 1.
635///
636/// # Examples
637///
638/// In the following example, we make changes to the inner object before turning it into an
639/// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()`
640/// cannot fail.
641///
642/// ```
643/// use kernel::sync::{Arc, UniqueArc};
644///
645/// struct Example {
646///     a: u32,
647///     b: u32,
648/// }
649///
650/// fn test() -> Result<Arc<Example>> {
651///     let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
652///     x.a += 1;
653///     x.b += 1;
654///     Ok(x.into())
655/// }
656///
657/// # test().unwrap();
658/// ```
659///
660/// In the following example we first allocate memory for a refcounted `Example` but we don't
661/// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`],
662/// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens
663/// in one context (e.g., sleepable) and initialisation in another (e.g., atomic):
664///
665/// ```
666/// use kernel::sync::{Arc, UniqueArc};
667///
668/// struct Example {
669///     a: u32,
670///     b: u32,
671/// }
672///
673/// fn test() -> Result<Arc<Example>> {
674///     let x = UniqueArc::new_uninit(GFP_KERNEL)?;
675///     Ok(x.write(Example { a: 10, b: 20 }).into())
676/// }
677///
678/// # test().unwrap();
679/// ```
680///
681/// In the last example below, the caller gets a pinned instance of `Example` while converting to
682/// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during
683/// initialisation, for example, when initialising fields that are wrapped in locks.
684///
685/// ```
686/// use kernel::sync::{Arc, UniqueArc};
687///
688/// struct Example {
689///     a: u32,
690///     b: u32,
691/// }
692///
693/// fn test() -> Result<Arc<Example>> {
694///     let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?);
695///     // We can modify `pinned` because it is `Unpin`.
696///     pinned.as_mut().a += 1;
697///     Ok(pinned.into())
698/// }
699///
700/// # test().unwrap();
701/// ```
702pub struct UniqueArc<T: ?Sized> {
703    inner: Arc<T>,
704}
705
706impl<T> InPlaceInit<T> for UniqueArc<T> {
707    type PinnedSelf = Pin<Self>;
708
709    #[inline]
710    fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
711    where
712        E: From<AllocError>,
713    {
714        UniqueArc::new_uninit(flags)?.write_pin_init(init)
715    }
716
717    #[inline]
718    fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
719    where
720        E: From<AllocError>,
721    {
722        UniqueArc::new_uninit(flags)?.write_init(init)
723    }
724}
725
726impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> {
727    type Initialized = UniqueArc<T>;
728
729    fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
730        let slot = self.as_mut_ptr();
731        // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
732        // slot is valid.
733        unsafe { init.__init(slot)? };
734        // SAFETY: All fields have been initialized.
735        Ok(unsafe { self.assume_init() })
736    }
737
738    fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
739        let slot = self.as_mut_ptr();
740        // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
741        // slot is valid and will not be moved, because we pin it later.
742        unsafe { init.__pinned_init(slot)? };
743        // SAFETY: All fields have been initialized.
744        Ok(unsafe { self.assume_init() }.into())
745    }
746}
747
748impl<T> UniqueArc<T> {
749    /// Tries to allocate a new [`UniqueArc`] instance.
750    pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> {
751        Ok(Self {
752            // INVARIANT: The newly-created object has a refcount of 1.
753            inner: Arc::new(value, flags)?,
754        })
755    }
756
757    /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet.
758    pub fn new_uninit(flags: Flags) -> Result<UniqueArc<MaybeUninit<T>>, AllocError> {
759        // INVARIANT: The refcount is initialised to a non-zero value.
760        let inner = KBox::try_init::<AllocError>(
761            try_init!(ArcInner {
762                refcount: Refcount::new(1),
763                data <- pin_init::uninit::<T, AllocError>(),
764            }? AllocError),
765            flags,
766        )?;
767        Ok(UniqueArc {
768            // INVARIANT: The newly-created object has a refcount of 1.
769            // SAFETY: The pointer from the `KBox` is valid.
770            inner: unsafe { Arc::from_inner(KBox::leak(inner).into()) },
771        })
772    }
773}
774
775impl<T> UniqueArc<MaybeUninit<T>> {
776    /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it.
777    pub fn write(mut self, value: T) -> UniqueArc<T> {
778        self.deref_mut().write(value);
779        // SAFETY: We just wrote the value to be initialized.
780        unsafe { self.assume_init() }
781    }
782
783    /// Unsafely assume that `self` is initialized.
784    ///
785    /// # Safety
786    ///
787    /// The caller guarantees that the value behind this pointer has been initialized. It is
788    /// *immediate* UB to call this when the value is not initialized.
789    pub unsafe fn assume_init(self) -> UniqueArc<T> {
790        let inner = ManuallyDrop::new(self).inner.ptr;
791        UniqueArc {
792            // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be
793            // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`.
794            inner: unsafe { Arc::from_inner(inner.cast()) },
795        }
796    }
797
798    /// Initialize `self` using the given initializer.
799    pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> {
800        // SAFETY: The supplied pointer is valid for initialization.
801        match unsafe { init.__init(self.as_mut_ptr()) } {
802            // SAFETY: Initialization completed successfully.
803            Ok(()) => Ok(unsafe { self.assume_init() }),
804            Err(err) => Err(err),
805        }
806    }
807
808    /// Pin-initialize `self` using the given pin-initializer.
809    pub fn pin_init_with<E>(
810        mut self,
811        init: impl PinInit<T, E>,
812    ) -> core::result::Result<Pin<UniqueArc<T>>, E> {
813        // SAFETY: The supplied pointer is valid for initialization and we will later pin the value
814        // to ensure it does not move.
815        match unsafe { init.__pinned_init(self.as_mut_ptr()) } {
816            // SAFETY: Initialization completed successfully.
817            Ok(()) => Ok(unsafe { self.assume_init() }.into()),
818            Err(err) => Err(err),
819        }
820    }
821}
822
823impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> {
824    fn from(obj: UniqueArc<T>) -> Self {
825        // SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T`
826        // is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`.
827        unsafe { Pin::new_unchecked(obj) }
828    }
829}
830
831impl<T: ?Sized> Deref for UniqueArc<T> {
832    type Target = T;
833
834    fn deref(&self) -> &Self::Target {
835        self.inner.deref()
836    }
837}
838
839impl<T: ?Sized> DerefMut for UniqueArc<T> {
840    fn deref_mut(&mut self) -> &mut Self::Target {
841        // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so
842        // it is safe to dereference it. Additionally, we know there is only one reference when
843        // it's inside a `UniqueArc`, so it is safe to get a mutable reference.
844        unsafe { &mut self.inner.ptr.as_mut().data }
845    }
846}
847
848/// # Examples
849///
850/// ```
851/// # use core::borrow::Borrow;
852/// # use kernel::sync::UniqueArc;
853/// struct Foo<B: Borrow<u32>>(B);
854///
855/// // Owned instance.
856/// let owned = Foo(1);
857///
858/// // Owned instance using `UniqueArc`.
859/// let arc = UniqueArc::new(1, GFP_KERNEL)?;
860/// let shared = Foo(arc);
861///
862/// let i = 1;
863/// // Borrowed from `i`.
864/// let borrowed = Foo(&i);
865/// # Ok::<(), Error>(())
866/// ```
867impl<T: ?Sized> Borrow<T> for UniqueArc<T> {
868    fn borrow(&self) -> &T {
869        self.deref()
870    }
871}
872
873/// # Examples
874///
875/// ```
876/// # use core::borrow::BorrowMut;
877/// # use kernel::sync::UniqueArc;
878/// struct Foo<B: BorrowMut<u32>>(B);
879///
880/// // Owned instance.
881/// let owned = Foo(1);
882///
883/// // Owned instance using `UniqueArc`.
884/// let arc = UniqueArc::new(1, GFP_KERNEL)?;
885/// let shared = Foo(arc);
886///
887/// let mut i = 1;
888/// // Borrowed from `i`.
889/// let borrowed = Foo(&mut i);
890/// # Ok::<(), Error>(())
891/// ```
892impl<T: ?Sized> BorrowMut<T> for UniqueArc<T> {
893    fn borrow_mut(&mut self) -> &mut T {
894        self.deref_mut()
895    }
896}
897
898impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> {
899    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
900        fmt::Display::fmt(self.deref(), f)
901    }
902}
903
904impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> {
905    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
906        fmt::Display::fmt(self.deref(), f)
907    }
908}
909
910impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> {
911    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
912        fmt::Debug::fmt(self.deref(), f)
913    }
914}
915
916impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> {
917    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
918        fmt::Debug::fmt(self.deref(), f)
919    }
920}