kernel/sync/
arc.rs

1// SPDX-License-Identifier: GPL-2.0
2
3//! A reference-counted pointer.
4//!
5//! This module implements a way for users to create reference-counted objects and pointers to
6//! them. Such a pointer automatically increments and decrements the count, and drops the
7//! underlying object when it reaches zero. It is also safe to use concurrently from multiple
8//! threads.
9//!
10//! It is different from the standard library's [`Arc`] in a few ways:
11//! 1. It is backed by the kernel's `refcount_t` type.
12//! 2. It does not support weak references, which allows it to be half the size.
13//! 3. It saturates the reference count instead of aborting when it goes over a threshold.
14//! 4. It does not provide a `get_mut` method, so the ref counted object is pinned.
15//! 5. The object in [`Arc`] is pinned implicitly.
16//!
17//! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html
18
19use crate::{
20    alloc::{AllocError, Flags, KBox},
21    bindings,
22    ffi::c_void,
23    init::InPlaceInit,
24    try_init,
25    types::{ForeignOwnable, Opaque},
26};
27use core::{
28    alloc::Layout,
29    borrow::{Borrow, BorrowMut},
30    fmt,
31    marker::PhantomData,
32    mem::{ManuallyDrop, MaybeUninit},
33    ops::{Deref, DerefMut},
34    pin::Pin,
35    ptr::NonNull,
36};
37use pin_init::{self, pin_data, InPlaceWrite, Init, PinInit};
38
39mod std_vendor;
40
41/// A reference-counted pointer to an instance of `T`.
42///
43/// The reference count is incremented when new instances of [`Arc`] are created, and decremented
44/// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.
45///
46/// # Invariants
47///
48/// The reference count on an instance of [`Arc`] is always non-zero.
49/// The object pointed to by [`Arc`] is always pinned.
50///
51/// # Examples
52///
53/// ```
54/// use kernel::sync::Arc;
55///
56/// struct Example {
57///     a: u32,
58///     b: u32,
59/// }
60///
61/// // Create a refcounted instance of `Example`.
62/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
63///
64/// // Get a new pointer to `obj` and increment the refcount.
65/// let cloned = obj.clone();
66///
67/// // Assert that both `obj` and `cloned` point to the same underlying object.
68/// assert!(core::ptr::eq(&*obj, &*cloned));
69///
70/// // Destroy `obj` and decrement its refcount.
71/// drop(obj);
72///
73/// // Check that the values are still accessible through `cloned`.
74/// assert_eq!(cloned.a, 10);
75/// assert_eq!(cloned.b, 20);
76///
77/// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed.
78/// # Ok::<(), Error>(())
79/// ```
80///
81/// Using `Arc<T>` as the type of `self`:
82///
83/// ```
84/// use kernel::sync::Arc;
85///
86/// struct Example {
87///     a: u32,
88///     b: u32,
89/// }
90///
91/// impl Example {
92///     fn take_over(self: Arc<Self>) {
93///         // ...
94///     }
95///
96///     fn use_reference(self: &Arc<Self>) {
97///         // ...
98///     }
99/// }
100///
101/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
102/// obj.use_reference();
103/// obj.take_over();
104/// # Ok::<(), Error>(())
105/// ```
106///
107/// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`:
108///
109/// ```
110/// use kernel::sync::{Arc, ArcBorrow};
111///
112/// trait MyTrait {
113///     // Trait has a function whose `self` type is `Arc<Self>`.
114///     fn example1(self: Arc<Self>) {}
115///
116///     // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`.
117///     fn example2(self: ArcBorrow<'_, Self>) {}
118/// }
119///
120/// struct Example;
121/// impl MyTrait for Example {}
122///
123/// // `obj` has type `Arc<Example>`.
124/// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?;
125///
126/// // `coerced` has type `Arc<dyn MyTrait>`.
127/// let coerced: Arc<dyn MyTrait> = obj;
128/// # Ok::<(), Error>(())
129/// ```
130#[repr(transparent)]
131#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))]
132pub struct Arc<T: ?Sized> {
133    ptr: NonNull<ArcInner<T>>,
134    // NB: this informs dropck that objects of type `ArcInner<T>` may be used in `<Arc<T> as
135    // Drop>::drop`. Note that dropck already assumes that objects of type `T` may be used in
136    // `<Arc<T> as Drop>::drop` and the distinction between `T` and `ArcInner<T>` is not presently
137    // meaningful with respect to dropck - but this may change in the future so this is left here
138    // out of an abundance of caution.
139    //
140    // See <https://doc.rust-lang.org/nomicon/phantom-data.html#generic-parameters-and-drop-checking>
141    // for more detail on the semantics of dropck in the presence of `PhantomData`.
142    _p: PhantomData<ArcInner<T>>,
143}
144
145#[pin_data]
146#[repr(C)]
147struct ArcInner<T: ?Sized> {
148    refcount: Opaque<bindings::refcount_t>,
149    data: T,
150}
151
152impl<T: ?Sized> ArcInner<T> {
153    /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`].
154    ///
155    /// # Safety
156    ///
157    /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must
158    /// not yet have been destroyed.
159    unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> {
160        let refcount_layout = Layout::new::<bindings::refcount_t>();
161        // SAFETY: The caller guarantees that the pointer is valid.
162        let val_layout = Layout::for_value(unsafe { &*ptr });
163        // SAFETY: We're computing the layout of a real struct that existed when compiling this
164        // binary, so its layout is not so large that it can trigger arithmetic overflow.
165        let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 };
166
167        // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and
168        // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field.
169        //
170        // This is documented at:
171        // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>.
172        let ptr = ptr as *const ArcInner<T>;
173
174        // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the
175        // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is
176        // still valid.
177        let ptr = unsafe { ptr.byte_sub(val_offset) };
178
179        // SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null
180        // address.
181        unsafe { NonNull::new_unchecked(ptr.cast_mut()) }
182    }
183}
184
185// This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the
186// dynamically-sized type (DST) `U`.
187#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
188impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {}
189
190// This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`.
191#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
192impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {}
193
194// SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because
195// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
196// `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a
197// mutable reference when the reference count reaches zero and `T` is dropped.
198unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
199
200// SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync`
201// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally,
202// it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an
203// `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when
204// the reference count reaches zero and `T` is dropped.
205unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
206
207impl<T> InPlaceInit<T> for Arc<T> {
208    type PinnedSelf = Self;
209
210    #[inline]
211    fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
212    where
213        E: From<AllocError>,
214    {
215        UniqueArc::try_pin_init(init, flags).map(|u| u.into())
216    }
217
218    #[inline]
219    fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
220    where
221        E: From<AllocError>,
222    {
223        UniqueArc::try_init(init, flags).map(|u| u.into())
224    }
225}
226
227impl<T> Arc<T> {
228    /// Constructs a new reference counted instance of `T`.
229    pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> {
230        // INVARIANT: The refcount is initialised to a non-zero value.
231        let value = ArcInner {
232            // SAFETY: There are no safety requirements for this FFI call.
233            refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
234            data: contents,
235        };
236
237        let inner = KBox::new(value, flags)?;
238        let inner = KBox::leak(inner).into();
239
240        // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new
241        // `Arc` object.
242        Ok(unsafe { Self::from_inner(inner) })
243    }
244}
245
246impl<T: ?Sized> Arc<T> {
247    /// Constructs a new [`Arc`] from an existing [`ArcInner`].
248    ///
249    /// # Safety
250    ///
251    /// The caller must ensure that `inner` points to a valid location and has a non-zero reference
252    /// count, one of which will be owned by the new [`Arc`] instance.
253    unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self {
254        // INVARIANT: By the safety requirements, the invariants hold.
255        Arc {
256            ptr: inner,
257            _p: PhantomData,
258        }
259    }
260
261    /// Convert the [`Arc`] into a raw pointer.
262    ///
263    /// The raw pointer has ownership of the refcount that this Arc object owned.
264    pub fn into_raw(self) -> *const T {
265        let ptr = self.ptr.as_ptr();
266        core::mem::forget(self);
267        // SAFETY: The pointer is valid.
268        unsafe { core::ptr::addr_of!((*ptr).data) }
269    }
270
271    /// Return a raw pointer to the data in this arc.
272    pub fn as_ptr(this: &Self) -> *const T {
273        let ptr = this.ptr.as_ptr();
274
275        // SAFETY: As `ptr` points to a valid allocation of type `ArcInner`,
276        // field projection to `data`is within bounds of the allocation.
277        unsafe { core::ptr::addr_of!((*ptr).data) }
278    }
279
280    /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`].
281    ///
282    /// # Safety
283    ///
284    /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it
285    /// must not be called more than once for each previous call to [`Arc::into_raw`].
286    pub unsafe fn from_raw(ptr: *const T) -> Self {
287        // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
288        // `Arc` that is still valid.
289        let ptr = unsafe { ArcInner::container_of(ptr) };
290
291        // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the
292        // reference count held then will be owned by the new `Arc` object.
293        unsafe { Self::from_inner(ptr) }
294    }
295
296    /// Returns an [`ArcBorrow`] from the given [`Arc`].
297    ///
298    /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method
299    /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised.
300    #[inline]
301    pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> {
302        // SAFETY: The constraint that the lifetime of the shared reference must outlive that of
303        // the returned `ArcBorrow` ensures that the object remains alive and that no mutable
304        // reference can be created.
305        unsafe { ArcBorrow::new(self.ptr) }
306    }
307
308    /// Compare whether two [`Arc`] pointers reference the same underlying object.
309    pub fn ptr_eq(this: &Self, other: &Self) -> bool {
310        core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr())
311    }
312
313    /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique.
314    ///
315    /// When this destroys the `Arc`, it does so while properly avoiding races. This means that
316    /// this method will never call the destructor of the value.
317    ///
318    /// # Examples
319    ///
320    /// ```
321    /// use kernel::sync::{Arc, UniqueArc};
322    ///
323    /// let arc = Arc::new(42, GFP_KERNEL)?;
324    /// let unique_arc = arc.into_unique_or_drop();
325    ///
326    /// // The above conversion should succeed since refcount of `arc` is 1.
327    /// assert!(unique_arc.is_some());
328    ///
329    /// assert_eq!(*(unique_arc.unwrap()), 42);
330    ///
331    /// # Ok::<(), Error>(())
332    /// ```
333    ///
334    /// ```
335    /// use kernel::sync::{Arc, UniqueArc};
336    ///
337    /// let arc = Arc::new(42, GFP_KERNEL)?;
338    /// let another = arc.clone();
339    ///
340    /// let unique_arc = arc.into_unique_or_drop();
341    ///
342    /// // The above conversion should fail since refcount of `arc` is >1.
343    /// assert!(unique_arc.is_none());
344    ///
345    /// # Ok::<(), Error>(())
346    /// ```
347    pub fn into_unique_or_drop(self) -> Option<Pin<UniqueArc<T>>> {
348        // We will manually manage the refcount in this method, so we disable the destructor.
349        let me = ManuallyDrop::new(self);
350        // SAFETY: We own a refcount, so the pointer is still valid.
351        let refcount = unsafe { me.ptr.as_ref() }.refcount.get();
352
353        // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will
354        // return without further touching the `Arc`. If the refcount reaches zero, then there are
355        // no other arcs, and we can create a `UniqueArc`.
356        //
357        // SAFETY: We own a refcount, so the pointer is not dangling.
358        let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };
359        if is_zero {
360            // SAFETY: We have exclusive access to the arc, so we can perform unsynchronized
361            // accesses to the refcount.
362            unsafe { core::ptr::write(refcount, bindings::REFCOUNT_INIT(1)) };
363
364            // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We
365            // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin
366            // their values.
367            Some(Pin::from(UniqueArc {
368                inner: ManuallyDrop::into_inner(me),
369            }))
370        } else {
371            None
372        }
373    }
374}
375
376// SAFETY: The pointer returned by `into_foreign` comes from a well aligned
377// pointer to `ArcInner<T>`.
378unsafe impl<T: 'static> ForeignOwnable for Arc<T> {
379    const FOREIGN_ALIGN: usize = core::mem::align_of::<ArcInner<T>>();
380
381    type Borrowed<'a> = ArcBorrow<'a, T>;
382    type BorrowedMut<'a> = Self::Borrowed<'a>;
383
384    fn into_foreign(self) -> *mut c_void {
385        ManuallyDrop::new(self).ptr.as_ptr().cast()
386    }
387
388    unsafe fn from_foreign(ptr: *mut c_void) -> Self {
389        // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
390        // call to `Self::into_foreign`.
391        let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) };
392
393        // SAFETY: By the safety requirement of this function, we know that `ptr` came from
394        // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and
395        // holds a reference count increment that is transferrable to us.
396        unsafe { Self::from_inner(inner) }
397    }
398
399    unsafe fn borrow<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> {
400        // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
401        // call to `Self::into_foreign`.
402        let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) };
403
404        // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive
405        // for the lifetime of the returned value.
406        unsafe { ArcBorrow::new(inner) }
407    }
408
409    unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> {
410        // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety
411        // requirements for `borrow`.
412        unsafe { <Self as ForeignOwnable>::borrow(ptr) }
413    }
414}
415
416impl<T: ?Sized> Deref for Arc<T> {
417    type Target = T;
418
419    fn deref(&self) -> &Self::Target {
420        // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
421        // safe to dereference it.
422        unsafe { &self.ptr.as_ref().data }
423    }
424}
425
426impl<T: ?Sized> AsRef<T> for Arc<T> {
427    fn as_ref(&self) -> &T {
428        self.deref()
429    }
430}
431
432/// # Examples
433///
434/// ```
435/// # use core::borrow::Borrow;
436/// # use kernel::sync::Arc;
437/// struct Foo<B: Borrow<u32>>(B);
438///
439/// // Owned instance.
440/// let owned = Foo(1);
441///
442/// // Shared instance.
443/// let arc = Arc::new(1, GFP_KERNEL)?;
444/// let shared = Foo(arc.clone());
445///
446/// let i = 1;
447/// // Borrowed from `i`.
448/// let borrowed = Foo(&i);
449/// # Ok::<(), Error>(())
450/// ```
451impl<T: ?Sized> Borrow<T> for Arc<T> {
452    fn borrow(&self) -> &T {
453        self.deref()
454    }
455}
456
457impl<T: ?Sized> Clone for Arc<T> {
458    fn clone(&self) -> Self {
459        // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
460        // safe to dereference it.
461        let refcount = unsafe { self.ptr.as_ref() }.refcount.get();
462
463        // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero.
464        // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
465        // safe to increment the refcount.
466        unsafe { bindings::refcount_inc(refcount) };
467
468        // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`.
469        unsafe { Self::from_inner(self.ptr) }
470    }
471}
472
473impl<T: ?Sized> Drop for Arc<T> {
474    fn drop(&mut self) {
475        // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot
476        // touch `refcount` after it's decremented to a non-zero value because another thread/CPU
477        // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to
478        // freed/invalid memory as long as it is never dereferenced.
479        let refcount = unsafe { self.ptr.as_ref() }.refcount.get();
480
481        // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and
482        // this instance is being dropped, so the broken invariant is not observable.
483        // SAFETY: Also by the type invariant, we are allowed to decrement the refcount.
484        let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };
485        if is_zero {
486            // The count reached zero, we must free the memory.
487            //
488            // SAFETY: The pointer was initialised from the result of `KBox::leak`.
489            unsafe { drop(KBox::from_raw(self.ptr.as_ptr())) };
490        }
491    }
492}
493
494impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> {
495    fn from(item: UniqueArc<T>) -> Self {
496        item.inner
497    }
498}
499
500impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {
501    fn from(item: Pin<UniqueArc<T>>) -> Self {
502        // SAFETY: The type invariants of `Arc` guarantee that the data is pinned.
503        unsafe { Pin::into_inner_unchecked(item).inner }
504    }
505}
506
507/// A borrowed reference to an [`Arc`] instance.
508///
509/// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler
510/// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance.
511///
512/// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>`
513/// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference)
514/// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double
515/// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if
516/// needed.
517///
518/// # Invariants
519///
520/// There are no mutable references to the underlying [`Arc`], and it remains valid for the
521/// lifetime of the [`ArcBorrow`] instance.
522///
523/// # Examples
524///
525/// ```
526/// use kernel::sync::{Arc, ArcBorrow};
527///
528/// struct Example;
529///
530/// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> {
531///     e.into()
532/// }
533///
534/// let obj = Arc::new(Example, GFP_KERNEL)?;
535/// let cloned = do_something(obj.as_arc_borrow());
536///
537/// // Assert that both `obj` and `cloned` point to the same underlying object.
538/// assert!(core::ptr::eq(&*obj, &*cloned));
539/// # Ok::<(), Error>(())
540/// ```
541///
542/// Using `ArcBorrow<T>` as the type of `self`:
543///
544/// ```
545/// use kernel::sync::{Arc, ArcBorrow};
546///
547/// struct Example {
548///     a: u32,
549///     b: u32,
550/// }
551///
552/// impl Example {
553///     fn use_reference(self: ArcBorrow<'_, Self>) {
554///         // ...
555///     }
556/// }
557///
558/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
559/// obj.as_arc_borrow().use_reference();
560/// # Ok::<(), Error>(())
561/// ```
562#[repr(transparent)]
563#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))]
564pub struct ArcBorrow<'a, T: ?Sized + 'a> {
565    inner: NonNull<ArcInner<T>>,
566    _p: PhantomData<&'a ()>,
567}
568
569// This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into
570// `ArcBorrow<U>`.
571#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
572impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>>
573    for ArcBorrow<'_, T>
574{
575}
576
577impl<T: ?Sized> Clone for ArcBorrow<'_, T> {
578    fn clone(&self) -> Self {
579        *self
580    }
581}
582
583impl<T: ?Sized> Copy for ArcBorrow<'_, T> {}
584
585impl<T: ?Sized> ArcBorrow<'_, T> {
586    /// Creates a new [`ArcBorrow`] instance.
587    ///
588    /// # Safety
589    ///
590    /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance:
591    /// 1. That `inner` remains valid;
592    /// 2. That no mutable references to `inner` are created.
593    unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self {
594        // INVARIANT: The safety requirements guarantee the invariants.
595        Self {
596            inner,
597            _p: PhantomData,
598        }
599    }
600
601    /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with
602    /// [`Arc::into_raw`] or [`Arc::as_ptr`].
603    ///
604    /// # Safety
605    ///
606    /// * The provided pointer must originate from a call to [`Arc::into_raw`] or [`Arc::as_ptr`].
607    /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must
608    ///   not hit zero.
609    /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a
610    ///   [`UniqueArc`] reference to this value.
611    pub unsafe fn from_raw(ptr: *const T) -> Self {
612        // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
613        // `Arc` that is still valid.
614        let ptr = unsafe { ArcInner::container_of(ptr) };
615
616        // SAFETY: The caller promises that the value remains valid since the reference count must
617        // not hit zero, and no mutable reference will be created since that would involve a
618        // `UniqueArc`.
619        unsafe { Self::new(ptr) }
620    }
621}
622
623impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> {
624    fn from(b: ArcBorrow<'_, T>) -> Self {
625        // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop`
626        // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the
627        // increment.
628        ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) })
629            .deref()
630            .clone()
631    }
632}
633
634impl<T: ?Sized> Deref for ArcBorrow<'_, T> {
635    type Target = T;
636
637    fn deref(&self) -> &Self::Target {
638        // SAFETY: By the type invariant, the underlying object is still alive with no mutable
639        // references to it, so it is safe to create a shared reference.
640        unsafe { &self.inner.as_ref().data }
641    }
642}
643
644/// A refcounted object that is known to have a refcount of 1.
645///
646/// It is mutable and can be converted to an [`Arc`] so that it can be shared.
647///
648/// # Invariants
649///
650/// `inner` always has a reference count of 1.
651///
652/// # Examples
653///
654/// In the following example, we make changes to the inner object before turning it into an
655/// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()`
656/// cannot fail.
657///
658/// ```
659/// use kernel::sync::{Arc, UniqueArc};
660///
661/// struct Example {
662///     a: u32,
663///     b: u32,
664/// }
665///
666/// fn test() -> Result<Arc<Example>> {
667///     let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
668///     x.a += 1;
669///     x.b += 1;
670///     Ok(x.into())
671/// }
672///
673/// # test().unwrap();
674/// ```
675///
676/// In the following example we first allocate memory for a refcounted `Example` but we don't
677/// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`],
678/// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens
679/// in one context (e.g., sleepable) and initialisation in another (e.g., atomic):
680///
681/// ```
682/// use kernel::sync::{Arc, UniqueArc};
683///
684/// struct Example {
685///     a: u32,
686///     b: u32,
687/// }
688///
689/// fn test() -> Result<Arc<Example>> {
690///     let x = UniqueArc::new_uninit(GFP_KERNEL)?;
691///     Ok(x.write(Example { a: 10, b: 20 }).into())
692/// }
693///
694/// # test().unwrap();
695/// ```
696///
697/// In the last example below, the caller gets a pinned instance of `Example` while converting to
698/// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during
699/// initialisation, for example, when initialising fields that are wrapped in locks.
700///
701/// ```
702/// use kernel::sync::{Arc, UniqueArc};
703///
704/// struct Example {
705///     a: u32,
706///     b: u32,
707/// }
708///
709/// fn test() -> Result<Arc<Example>> {
710///     let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?);
711///     // We can modify `pinned` because it is `Unpin`.
712///     pinned.as_mut().a += 1;
713///     Ok(pinned.into())
714/// }
715///
716/// # test().unwrap();
717/// ```
718pub struct UniqueArc<T: ?Sized> {
719    inner: Arc<T>,
720}
721
722impl<T> InPlaceInit<T> for UniqueArc<T> {
723    type PinnedSelf = Pin<Self>;
724
725    #[inline]
726    fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
727    where
728        E: From<AllocError>,
729    {
730        UniqueArc::new_uninit(flags)?.write_pin_init(init)
731    }
732
733    #[inline]
734    fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
735    where
736        E: From<AllocError>,
737    {
738        UniqueArc::new_uninit(flags)?.write_init(init)
739    }
740}
741
742impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> {
743    type Initialized = UniqueArc<T>;
744
745    fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
746        let slot = self.as_mut_ptr();
747        // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
748        // slot is valid.
749        unsafe { init.__init(slot)? };
750        // SAFETY: All fields have been initialized.
751        Ok(unsafe { self.assume_init() })
752    }
753
754    fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
755        let slot = self.as_mut_ptr();
756        // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
757        // slot is valid and will not be moved, because we pin it later.
758        unsafe { init.__pinned_init(slot)? };
759        // SAFETY: All fields have been initialized.
760        Ok(unsafe { self.assume_init() }.into())
761    }
762}
763
764impl<T> UniqueArc<T> {
765    /// Tries to allocate a new [`UniqueArc`] instance.
766    pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> {
767        Ok(Self {
768            // INVARIANT: The newly-created object has a refcount of 1.
769            inner: Arc::new(value, flags)?,
770        })
771    }
772
773    /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet.
774    pub fn new_uninit(flags: Flags) -> Result<UniqueArc<MaybeUninit<T>>, AllocError> {
775        // INVARIANT: The refcount is initialised to a non-zero value.
776        let inner = KBox::try_init::<AllocError>(
777            try_init!(ArcInner {
778                // SAFETY: There are no safety requirements for this FFI call.
779                refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
780                data <- pin_init::uninit::<T, AllocError>(),
781            }? AllocError),
782            flags,
783        )?;
784        Ok(UniqueArc {
785            // INVARIANT: The newly-created object has a refcount of 1.
786            // SAFETY: The pointer from the `KBox` is valid.
787            inner: unsafe { Arc::from_inner(KBox::leak(inner).into()) },
788        })
789    }
790}
791
792impl<T> UniqueArc<MaybeUninit<T>> {
793    /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it.
794    pub fn write(mut self, value: T) -> UniqueArc<T> {
795        self.deref_mut().write(value);
796        // SAFETY: We just wrote the value to be initialized.
797        unsafe { self.assume_init() }
798    }
799
800    /// Unsafely assume that `self` is initialized.
801    ///
802    /// # Safety
803    ///
804    /// The caller guarantees that the value behind this pointer has been initialized. It is
805    /// *immediate* UB to call this when the value is not initialized.
806    pub unsafe fn assume_init(self) -> UniqueArc<T> {
807        let inner = ManuallyDrop::new(self).inner.ptr;
808        UniqueArc {
809            // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be
810            // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`.
811            inner: unsafe { Arc::from_inner(inner.cast()) },
812        }
813    }
814
815    /// Initialize `self` using the given initializer.
816    pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> {
817        // SAFETY: The supplied pointer is valid for initialization.
818        match unsafe { init.__init(self.as_mut_ptr()) } {
819            // SAFETY: Initialization completed successfully.
820            Ok(()) => Ok(unsafe { self.assume_init() }),
821            Err(err) => Err(err),
822        }
823    }
824
825    /// Pin-initialize `self` using the given pin-initializer.
826    pub fn pin_init_with<E>(
827        mut self,
828        init: impl PinInit<T, E>,
829    ) -> core::result::Result<Pin<UniqueArc<T>>, E> {
830        // SAFETY: The supplied pointer is valid for initialization and we will later pin the value
831        // to ensure it does not move.
832        match unsafe { init.__pinned_init(self.as_mut_ptr()) } {
833            // SAFETY: Initialization completed successfully.
834            Ok(()) => Ok(unsafe { self.assume_init() }.into()),
835            Err(err) => Err(err),
836        }
837    }
838}
839
840impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> {
841    fn from(obj: UniqueArc<T>) -> Self {
842        // SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T`
843        // is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`.
844        unsafe { Pin::new_unchecked(obj) }
845    }
846}
847
848impl<T: ?Sized> Deref for UniqueArc<T> {
849    type Target = T;
850
851    fn deref(&self) -> &Self::Target {
852        self.inner.deref()
853    }
854}
855
856impl<T: ?Sized> DerefMut for UniqueArc<T> {
857    fn deref_mut(&mut self) -> &mut Self::Target {
858        // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so
859        // it is safe to dereference it. Additionally, we know there is only one reference when
860        // it's inside a `UniqueArc`, so it is safe to get a mutable reference.
861        unsafe { &mut self.inner.ptr.as_mut().data }
862    }
863}
864
865/// # Examples
866///
867/// ```
868/// # use core::borrow::Borrow;
869/// # use kernel::sync::UniqueArc;
870/// struct Foo<B: Borrow<u32>>(B);
871///
872/// // Owned instance.
873/// let owned = Foo(1);
874///
875/// // Owned instance using `UniqueArc`.
876/// let arc = UniqueArc::new(1, GFP_KERNEL)?;
877/// let shared = Foo(arc);
878///
879/// let i = 1;
880/// // Borrowed from `i`.
881/// let borrowed = Foo(&i);
882/// # Ok::<(), Error>(())
883/// ```
884impl<T: ?Sized> Borrow<T> for UniqueArc<T> {
885    fn borrow(&self) -> &T {
886        self.deref()
887    }
888}
889
890/// # Examples
891///
892/// ```
893/// # use core::borrow::BorrowMut;
894/// # use kernel::sync::UniqueArc;
895/// struct Foo<B: BorrowMut<u32>>(B);
896///
897/// // Owned instance.
898/// let owned = Foo(1);
899///
900/// // Owned instance using `UniqueArc`.
901/// let arc = UniqueArc::new(1, GFP_KERNEL)?;
902/// let shared = Foo(arc);
903///
904/// let mut i = 1;
905/// // Borrowed from `i`.
906/// let borrowed = Foo(&mut i);
907/// # Ok::<(), Error>(())
908/// ```
909impl<T: ?Sized> BorrowMut<T> for UniqueArc<T> {
910    fn borrow_mut(&mut self) -> &mut T {
911        self.deref_mut()
912    }
913}
914
915impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> {
916    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
917        fmt::Display::fmt(self.deref(), f)
918    }
919}
920
921impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> {
922    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
923        fmt::Display::fmt(self.deref(), f)
924    }
925}
926
927impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> {
928    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
929        fmt::Debug::fmt(self.deref(), f)
930    }
931}
932
933impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> {
934    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
935        fmt::Debug::fmt(self.deref(), f)
936    }
937}