kernel/sync/arc.rs
1// SPDX-License-Identifier: GPL-2.0
2
3//! A reference-counted pointer.
4//!
5//! This module implements a way for users to create reference-counted objects and pointers to
6//! them. Such a pointer automatically increments and decrements the count, and drops the
7//! underlying object when it reaches zero. It is also safe to use concurrently from multiple
8//! threads.
9//!
10//! It is different from the standard library's [`Arc`] in a few ways:
11//! 1. It is backed by the kernel's [`Refcount`] type.
12//! 2. It does not support weak references, which allows it to be half the size.
13//! 3. It saturates the reference count instead of aborting when it goes over a threshold.
14//! 4. It does not provide a `get_mut` method, so the ref counted object is pinned.
15//! 5. The object in [`Arc`] is pinned implicitly.
16//!
17//! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html
18
19use crate::{
20 alloc::{AllocError, Flags, KBox},
21 ffi::c_void,
22 fmt,
23 init::InPlaceInit,
24 sync::Refcount,
25 try_init,
26 types::ForeignOwnable,
27};
28use core::{
29 alloc::Layout,
30 borrow::{Borrow, BorrowMut},
31 marker::PhantomData,
32 mem::{ManuallyDrop, MaybeUninit},
33 ops::{Deref, DerefMut},
34 pin::Pin,
35 ptr::NonNull,
36};
37use pin_init::{self, pin_data, InPlaceWrite, Init, PinInit};
38
39mod std_vendor;
40
41/// A reference-counted pointer to an instance of `T`.
42///
43/// The reference count is incremented when new instances of [`Arc`] are created, and decremented
44/// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.
45///
46/// # Invariants
47///
48/// The reference count on an instance of [`Arc`] is always non-zero.
49/// The object pointed to by [`Arc`] is always pinned.
50///
51/// # Examples
52///
53/// ```
54/// use kernel::sync::Arc;
55///
56/// struct Example {
57/// a: u32,
58/// b: u32,
59/// }
60///
61/// // Create a refcounted instance of `Example`.
62/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
63///
64/// // Get a new pointer to `obj` and increment the refcount.
65/// let cloned = obj.clone();
66///
67/// // Assert that both `obj` and `cloned` point to the same underlying object.
68/// assert!(core::ptr::eq(&*obj, &*cloned));
69///
70/// // Destroy `obj` and decrement its refcount.
71/// drop(obj);
72///
73/// // Check that the values are still accessible through `cloned`.
74/// assert_eq!(cloned.a, 10);
75/// assert_eq!(cloned.b, 20);
76///
77/// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed.
78/// # Ok::<(), Error>(())
79/// ```
80///
81/// Using `Arc<T>` as the type of `self`:
82///
83/// ```
84/// use kernel::sync::Arc;
85///
86/// struct Example {
87/// a: u32,
88/// b: u32,
89/// }
90///
91/// impl Example {
92/// fn take_over(self: Arc<Self>) {
93/// // ...
94/// }
95///
96/// fn use_reference(self: &Arc<Self>) {
97/// // ...
98/// }
99/// }
100///
101/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
102/// obj.use_reference();
103/// obj.take_over();
104/// # Ok::<(), Error>(())
105/// ```
106///
107/// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`:
108///
109/// ```
110/// use kernel::sync::{Arc, ArcBorrow};
111///
112/// trait MyTrait {
113/// // Trait has a function whose `self` type is `Arc<Self>`.
114/// fn example1(self: Arc<Self>) {}
115///
116/// // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`.
117/// fn example2(self: ArcBorrow<'_, Self>) {}
118/// }
119///
120/// struct Example;
121/// impl MyTrait for Example {}
122///
123/// // `obj` has type `Arc<Example>`.
124/// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?;
125///
126/// // `coerced` has type `Arc<dyn MyTrait>`.
127/// let coerced: Arc<dyn MyTrait> = obj;
128/// # Ok::<(), Error>(())
129/// ```
130#[repr(transparent)]
131#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))]
132pub struct Arc<T: ?Sized> {
133 ptr: NonNull<ArcInner<T>>,
134 // NB: this informs dropck that objects of type `ArcInner<T>` may be used in `<Arc<T> as
135 // Drop>::drop`. Note that dropck already assumes that objects of type `T` may be used in
136 // `<Arc<T> as Drop>::drop` and the distinction between `T` and `ArcInner<T>` is not presently
137 // meaningful with respect to dropck - but this may change in the future so this is left here
138 // out of an abundance of caution.
139 //
140 // See <https://doc.rust-lang.org/nomicon/phantom-data.html#generic-parameters-and-drop-checking>
141 // for more detail on the semantics of dropck in the presence of `PhantomData`.
142 _p: PhantomData<ArcInner<T>>,
143}
144
145#[pin_data]
146#[repr(C)]
147struct ArcInner<T: ?Sized> {
148 refcount: Refcount,
149 data: T,
150}
151
152impl<T: ?Sized> ArcInner<T> {
153 /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`].
154 ///
155 /// # Safety
156 ///
157 /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must
158 /// not yet have been destroyed.
159 unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> {
160 let refcount_layout = Layout::new::<Refcount>();
161 // SAFETY: The caller guarantees that the pointer is valid.
162 let val_layout = Layout::for_value(unsafe { &*ptr });
163 // SAFETY: We're computing the layout of a real struct that existed when compiling this
164 // binary, so its layout is not so large that it can trigger arithmetic overflow.
165 let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 };
166
167 // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and
168 // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field.
169 //
170 // This is documented at:
171 // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>.
172 let ptr = ptr as *const ArcInner<T>;
173
174 // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the
175 // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is
176 // still valid.
177 let ptr = unsafe { ptr.byte_sub(val_offset) };
178
179 // SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null
180 // address.
181 unsafe { NonNull::new_unchecked(ptr.cast_mut()) }
182 }
183}
184
185// This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the
186// dynamically-sized type (DST) `U`.
187#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
188impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {}
189
190// This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`.
191#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
192impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {}
193
194// SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because
195// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
196// `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a
197// mutable reference when the reference count reaches zero and `T` is dropped.
198unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
199
200// SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync`
201// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally,
202// it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an
203// `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when
204// the reference count reaches zero and `T` is dropped.
205unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
206
207impl<T> InPlaceInit<T> for Arc<T> {
208 type PinnedSelf = Self;
209
210 #[inline]
211 fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
212 where
213 E: From<AllocError>,
214 {
215 UniqueArc::try_pin_init(init, flags).map(|u| u.into())
216 }
217
218 #[inline]
219 fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
220 where
221 E: From<AllocError>,
222 {
223 UniqueArc::try_init(init, flags).map(|u| u.into())
224 }
225}
226
227impl<T> Arc<T> {
228 /// Constructs a new reference counted instance of `T`.
229 pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> {
230 // INVARIANT: The refcount is initialised to a non-zero value.
231 let value = ArcInner {
232 refcount: Refcount::new(1),
233 data: contents,
234 };
235
236 let inner = KBox::new(value, flags)?;
237 let inner = KBox::leak(inner).into();
238
239 // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new
240 // `Arc` object.
241 Ok(unsafe { Self::from_inner(inner) })
242 }
243}
244
245impl<T: ?Sized> Arc<T> {
246 /// Constructs a new [`Arc`] from an existing [`ArcInner`].
247 ///
248 /// # Safety
249 ///
250 /// The caller must ensure that `inner` points to a valid location and has a non-zero reference
251 /// count, one of which will be owned by the new [`Arc`] instance.
252 unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self {
253 // INVARIANT: By the safety requirements, the invariants hold.
254 Arc {
255 ptr: inner,
256 _p: PhantomData,
257 }
258 }
259
260 /// Convert the [`Arc`] into a raw pointer.
261 ///
262 /// The raw pointer has ownership of the refcount that this Arc object owned.
263 pub fn into_raw(self) -> *const T {
264 let ptr = self.ptr.as_ptr();
265 core::mem::forget(self);
266 // SAFETY: The pointer is valid.
267 unsafe { core::ptr::addr_of!((*ptr).data) }
268 }
269
270 /// Return a raw pointer to the data in this arc.
271 pub fn as_ptr(this: &Self) -> *const T {
272 let ptr = this.ptr.as_ptr();
273
274 // SAFETY: As `ptr` points to a valid allocation of type `ArcInner`,
275 // field projection to `data`is within bounds of the allocation.
276 unsafe { core::ptr::addr_of!((*ptr).data) }
277 }
278
279 /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`].
280 ///
281 /// # Safety
282 ///
283 /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it
284 /// must not be called more than once for each previous call to [`Arc::into_raw`].
285 pub unsafe fn from_raw(ptr: *const T) -> Self {
286 // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
287 // `Arc` that is still valid.
288 let ptr = unsafe { ArcInner::container_of(ptr) };
289
290 // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the
291 // reference count held then will be owned by the new `Arc` object.
292 unsafe { Self::from_inner(ptr) }
293 }
294
295 /// Returns an [`ArcBorrow`] from the given [`Arc`].
296 ///
297 /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method
298 /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised.
299 #[inline]
300 pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> {
301 // SAFETY: The constraint that the lifetime of the shared reference must outlive that of
302 // the returned `ArcBorrow` ensures that the object remains alive and that no mutable
303 // reference can be created.
304 unsafe { ArcBorrow::new(self.ptr) }
305 }
306
307 /// Compare whether two [`Arc`] pointers reference the same underlying object.
308 pub fn ptr_eq(this: &Self, other: &Self) -> bool {
309 core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr())
310 }
311
312 /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique.
313 ///
314 /// When this destroys the `Arc`, it does so while properly avoiding races. This means that
315 /// this method will never call the destructor of the value.
316 ///
317 /// # Examples
318 ///
319 /// ```
320 /// use kernel::sync::{Arc, UniqueArc};
321 ///
322 /// let arc = Arc::new(42, GFP_KERNEL)?;
323 /// let unique_arc = Arc::into_unique_or_drop(arc);
324 ///
325 /// // The above conversion should succeed since refcount of `arc` is 1.
326 /// assert!(unique_arc.is_some());
327 ///
328 /// assert_eq!(*(unique_arc.unwrap()), 42);
329 ///
330 /// # Ok::<(), Error>(())
331 /// ```
332 ///
333 /// ```
334 /// use kernel::sync::{Arc, UniqueArc};
335 ///
336 /// let arc = Arc::new(42, GFP_KERNEL)?;
337 /// let another = arc.clone();
338 ///
339 /// let unique_arc = Arc::into_unique_or_drop(arc);
340 ///
341 /// // The above conversion should fail since refcount of `arc` is >1.
342 /// assert!(unique_arc.is_none());
343 ///
344 /// # Ok::<(), Error>(())
345 /// ```
346 pub fn into_unique_or_drop(this: Self) -> Option<Pin<UniqueArc<T>>> {
347 // We will manually manage the refcount in this method, so we disable the destructor.
348 let this = ManuallyDrop::new(this);
349 // SAFETY: We own a refcount, so the pointer is still valid.
350 let refcount = unsafe { &this.ptr.as_ref().refcount };
351
352 // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will
353 // return without further touching the `Arc`. If the refcount reaches zero, then there are
354 // no other arcs, and we can create a `UniqueArc`.
355 if refcount.dec_and_test() {
356 refcount.set(1);
357
358 // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We
359 // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin
360 // their values.
361 Some(Pin::from(UniqueArc {
362 inner: ManuallyDrop::into_inner(this),
363 }))
364 } else {
365 None
366 }
367 }
368}
369
370// SAFETY: The pointer returned by `into_foreign` was originally allocated as an
371// `KBox<ArcInner<T>>`, so that type is what determines the alignment.
372unsafe impl<T: 'static> ForeignOwnable for Arc<T> {
373 const FOREIGN_ALIGN: usize = <KBox<ArcInner<T>> as ForeignOwnable>::FOREIGN_ALIGN;
374
375 type Borrowed<'a> = ArcBorrow<'a, T>;
376 type BorrowedMut<'a> = Self::Borrowed<'a>;
377
378 fn into_foreign(self) -> *mut c_void {
379 ManuallyDrop::new(self).ptr.as_ptr().cast()
380 }
381
382 unsafe fn from_foreign(ptr: *mut c_void) -> Self {
383 // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
384 // call to `Self::into_foreign`.
385 let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) };
386
387 // SAFETY: By the safety requirement of this function, we know that `ptr` came from
388 // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and
389 // holds a reference count increment that is transferrable to us.
390 unsafe { Self::from_inner(inner) }
391 }
392
393 unsafe fn borrow<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> {
394 // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
395 // call to `Self::into_foreign`.
396 let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) };
397
398 // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive
399 // for the lifetime of the returned value.
400 unsafe { ArcBorrow::new(inner) }
401 }
402
403 unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> {
404 // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety
405 // requirements for `borrow`.
406 unsafe { <Self as ForeignOwnable>::borrow(ptr) }
407 }
408}
409
410impl<T: ?Sized> Deref for Arc<T> {
411 type Target = T;
412
413 fn deref(&self) -> &Self::Target {
414 // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
415 // safe to dereference it.
416 unsafe { &self.ptr.as_ref().data }
417 }
418}
419
420impl<T: ?Sized> AsRef<T> for Arc<T> {
421 fn as_ref(&self) -> &T {
422 self.deref()
423 }
424}
425
426/// # Examples
427///
428/// ```
429/// # use core::borrow::Borrow;
430/// # use kernel::sync::Arc;
431/// struct Foo<B: Borrow<u32>>(B);
432///
433/// // Owned instance.
434/// let owned = Foo(1);
435///
436/// // Shared instance.
437/// let arc = Arc::new(1, GFP_KERNEL)?;
438/// let shared = Foo(arc.clone());
439///
440/// let i = 1;
441/// // Borrowed from `i`.
442/// let borrowed = Foo(&i);
443/// # Ok::<(), Error>(())
444/// ```
445impl<T: ?Sized> Borrow<T> for Arc<T> {
446 fn borrow(&self) -> &T {
447 self.deref()
448 }
449}
450
451impl<T: ?Sized> Clone for Arc<T> {
452 fn clone(&self) -> Self {
453 // INVARIANT: `Refcount` saturates the refcount, so it cannot overflow to zero.
454 // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
455 // safe to increment the refcount.
456 unsafe { self.ptr.as_ref() }.refcount.inc();
457
458 // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`.
459 unsafe { Self::from_inner(self.ptr) }
460 }
461}
462
463impl<T: ?Sized> Drop for Arc<T> {
464 fn drop(&mut self) {
465 // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and
466 // this instance is being dropped, so the broken invariant is not observable.
467 // SAFETY: By the type invariant, there is necessarily a reference to the object.
468 let is_zero = unsafe { self.ptr.as_ref() }.refcount.dec_and_test();
469 if is_zero {
470 // The count reached zero, we must free the memory.
471 //
472 // SAFETY: The pointer was initialised from the result of `KBox::leak`.
473 unsafe { drop(KBox::from_raw(self.ptr.as_ptr())) };
474 }
475 }
476}
477
478impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> {
479 fn from(item: UniqueArc<T>) -> Self {
480 item.inner
481 }
482}
483
484impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {
485 fn from(item: Pin<UniqueArc<T>>) -> Self {
486 // SAFETY: The type invariants of `Arc` guarantee that the data is pinned.
487 unsafe { Pin::into_inner_unchecked(item).inner }
488 }
489}
490
491/// A borrowed reference to an [`Arc`] instance.
492///
493/// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler
494/// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance.
495///
496/// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>`
497/// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference)
498/// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double
499/// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if
500/// needed.
501///
502/// # Invariants
503///
504/// There are no mutable references to the underlying [`Arc`], and it remains valid for the
505/// lifetime of the [`ArcBorrow`] instance.
506///
507/// # Examples
508///
509/// ```
510/// use kernel::sync::{Arc, ArcBorrow};
511///
512/// struct Example;
513///
514/// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> {
515/// e.into()
516/// }
517///
518/// let obj = Arc::new(Example, GFP_KERNEL)?;
519/// let cloned = do_something(obj.as_arc_borrow());
520///
521/// // Assert that both `obj` and `cloned` point to the same underlying object.
522/// assert!(core::ptr::eq(&*obj, &*cloned));
523/// # Ok::<(), Error>(())
524/// ```
525///
526/// Using `ArcBorrow<T>` as the type of `self`:
527///
528/// ```
529/// use kernel::sync::{Arc, ArcBorrow};
530///
531/// struct Example {
532/// a: u32,
533/// b: u32,
534/// }
535///
536/// impl Example {
537/// fn use_reference(self: ArcBorrow<'_, Self>) {
538/// // ...
539/// }
540/// }
541///
542/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
543/// obj.as_arc_borrow().use_reference();
544/// # Ok::<(), Error>(())
545/// ```
546#[repr(transparent)]
547#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))]
548pub struct ArcBorrow<'a, T: ?Sized + 'a> {
549 inner: NonNull<ArcInner<T>>,
550 _p: PhantomData<&'a ()>,
551}
552
553// This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into
554// `ArcBorrow<U>`.
555#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
556impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>>
557 for ArcBorrow<'_, T>
558{
559}
560
561impl<T: ?Sized> Clone for ArcBorrow<'_, T> {
562 fn clone(&self) -> Self {
563 *self
564 }
565}
566
567impl<T: ?Sized> Copy for ArcBorrow<'_, T> {}
568
569impl<T: ?Sized> ArcBorrow<'_, T> {
570 /// Creates a new [`ArcBorrow`] instance.
571 ///
572 /// # Safety
573 ///
574 /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance:
575 /// 1. That `inner` remains valid;
576 /// 2. That no mutable references to `inner` are created.
577 unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self {
578 // INVARIANT: The safety requirements guarantee the invariants.
579 Self {
580 inner,
581 _p: PhantomData,
582 }
583 }
584
585 /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with
586 /// [`Arc::into_raw`] or [`Arc::as_ptr`].
587 ///
588 /// # Safety
589 ///
590 /// * The provided pointer must originate from a call to [`Arc::into_raw`] or [`Arc::as_ptr`].
591 /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must
592 /// not hit zero.
593 /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a
594 /// [`UniqueArc`] reference to this value.
595 pub unsafe fn from_raw(ptr: *const T) -> Self {
596 // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
597 // `Arc` that is still valid.
598 let ptr = unsafe { ArcInner::container_of(ptr) };
599
600 // SAFETY: The caller promises that the value remains valid since the reference count must
601 // not hit zero, and no mutable reference will be created since that would involve a
602 // `UniqueArc`.
603 unsafe { Self::new(ptr) }
604 }
605}
606
607impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> {
608 fn from(b: ArcBorrow<'_, T>) -> Self {
609 // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop`
610 // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the
611 // increment.
612 ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) })
613 .deref()
614 .clone()
615 }
616}
617
618impl<T: ?Sized> Deref for ArcBorrow<'_, T> {
619 type Target = T;
620
621 fn deref(&self) -> &Self::Target {
622 // SAFETY: By the type invariant, the underlying object is still alive with no mutable
623 // references to it, so it is safe to create a shared reference.
624 unsafe { &self.inner.as_ref().data }
625 }
626}
627
628/// A refcounted object that is known to have a refcount of 1.
629///
630/// It is mutable and can be converted to an [`Arc`] so that it can be shared.
631///
632/// # Invariants
633///
634/// `inner` always has a reference count of 1.
635///
636/// # Examples
637///
638/// In the following example, we make changes to the inner object before turning it into an
639/// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()`
640/// cannot fail.
641///
642/// ```
643/// use kernel::sync::{Arc, UniqueArc};
644///
645/// struct Example {
646/// a: u32,
647/// b: u32,
648/// }
649///
650/// fn test() -> Result<Arc<Example>> {
651/// let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
652/// x.a += 1;
653/// x.b += 1;
654/// Ok(x.into())
655/// }
656///
657/// # test().unwrap();
658/// ```
659///
660/// In the following example we first allocate memory for a refcounted `Example` but we don't
661/// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`],
662/// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens
663/// in one context (e.g., sleepable) and initialisation in another (e.g., atomic):
664///
665/// ```
666/// use kernel::sync::{Arc, UniqueArc};
667///
668/// struct Example {
669/// a: u32,
670/// b: u32,
671/// }
672///
673/// fn test() -> Result<Arc<Example>> {
674/// let x = UniqueArc::new_uninit(GFP_KERNEL)?;
675/// Ok(x.write(Example { a: 10, b: 20 }).into())
676/// }
677///
678/// # test().unwrap();
679/// ```
680///
681/// In the last example below, the caller gets a pinned instance of `Example` while converting to
682/// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during
683/// initialisation, for example, when initialising fields that are wrapped in locks.
684///
685/// ```
686/// use kernel::sync::{Arc, UniqueArc};
687///
688/// struct Example {
689/// a: u32,
690/// b: u32,
691/// }
692///
693/// fn test() -> Result<Arc<Example>> {
694/// let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?);
695/// // We can modify `pinned` because it is `Unpin`.
696/// pinned.as_mut().a += 1;
697/// Ok(pinned.into())
698/// }
699///
700/// # test().unwrap();
701/// ```
702pub struct UniqueArc<T: ?Sized> {
703 inner: Arc<T>,
704}
705
706impl<T> InPlaceInit<T> for UniqueArc<T> {
707 type PinnedSelf = Pin<Self>;
708
709 #[inline]
710 fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
711 where
712 E: From<AllocError>,
713 {
714 UniqueArc::new_uninit(flags)?.write_pin_init(init)
715 }
716
717 #[inline]
718 fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
719 where
720 E: From<AllocError>,
721 {
722 UniqueArc::new_uninit(flags)?.write_init(init)
723 }
724}
725
726impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> {
727 type Initialized = UniqueArc<T>;
728
729 fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
730 let slot = self.as_mut_ptr();
731 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
732 // slot is valid.
733 unsafe { init.__init(slot)? };
734 // SAFETY: All fields have been initialized.
735 Ok(unsafe { self.assume_init() })
736 }
737
738 fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
739 let slot = self.as_mut_ptr();
740 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
741 // slot is valid and will not be moved, because we pin it later.
742 unsafe { init.__pinned_init(slot)? };
743 // SAFETY: All fields have been initialized.
744 Ok(unsafe { self.assume_init() }.into())
745 }
746}
747
748impl<T> UniqueArc<T> {
749 /// Tries to allocate a new [`UniqueArc`] instance.
750 pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> {
751 Ok(Self {
752 // INVARIANT: The newly-created object has a refcount of 1.
753 inner: Arc::new(value, flags)?,
754 })
755 }
756
757 /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet.
758 pub fn new_uninit(flags: Flags) -> Result<UniqueArc<MaybeUninit<T>>, AllocError> {
759 // INVARIANT: The refcount is initialised to a non-zero value.
760 let inner = KBox::try_init::<AllocError>(
761 try_init!(ArcInner {
762 refcount: Refcount::new(1),
763 data <- pin_init::uninit::<T, AllocError>(),
764 }? AllocError),
765 flags,
766 )?;
767 Ok(UniqueArc {
768 // INVARIANT: The newly-created object has a refcount of 1.
769 // SAFETY: The pointer from the `KBox` is valid.
770 inner: unsafe { Arc::from_inner(KBox::leak(inner).into()) },
771 })
772 }
773}
774
775impl<T> UniqueArc<MaybeUninit<T>> {
776 /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it.
777 pub fn write(mut self, value: T) -> UniqueArc<T> {
778 self.deref_mut().write(value);
779 // SAFETY: We just wrote the value to be initialized.
780 unsafe { self.assume_init() }
781 }
782
783 /// Unsafely assume that `self` is initialized.
784 ///
785 /// # Safety
786 ///
787 /// The caller guarantees that the value behind this pointer has been initialized. It is
788 /// *immediate* UB to call this when the value is not initialized.
789 pub unsafe fn assume_init(self) -> UniqueArc<T> {
790 let inner = ManuallyDrop::new(self).inner.ptr;
791 UniqueArc {
792 // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be
793 // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`.
794 inner: unsafe { Arc::from_inner(inner.cast()) },
795 }
796 }
797
798 /// Initialize `self` using the given initializer.
799 pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> {
800 // SAFETY: The supplied pointer is valid for initialization.
801 match unsafe { init.__init(self.as_mut_ptr()) } {
802 // SAFETY: Initialization completed successfully.
803 Ok(()) => Ok(unsafe { self.assume_init() }),
804 Err(err) => Err(err),
805 }
806 }
807
808 /// Pin-initialize `self` using the given pin-initializer.
809 pub fn pin_init_with<E>(
810 mut self,
811 init: impl PinInit<T, E>,
812 ) -> core::result::Result<Pin<UniqueArc<T>>, E> {
813 // SAFETY: The supplied pointer is valid for initialization and we will later pin the value
814 // to ensure it does not move.
815 match unsafe { init.__pinned_init(self.as_mut_ptr()) } {
816 // SAFETY: Initialization completed successfully.
817 Ok(()) => Ok(unsafe { self.assume_init() }.into()),
818 Err(err) => Err(err),
819 }
820 }
821}
822
823impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> {
824 fn from(obj: UniqueArc<T>) -> Self {
825 // SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T`
826 // is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`.
827 unsafe { Pin::new_unchecked(obj) }
828 }
829}
830
831impl<T: ?Sized> Deref for UniqueArc<T> {
832 type Target = T;
833
834 fn deref(&self) -> &Self::Target {
835 self.inner.deref()
836 }
837}
838
839impl<T: ?Sized> DerefMut for UniqueArc<T> {
840 fn deref_mut(&mut self) -> &mut Self::Target {
841 // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so
842 // it is safe to dereference it. Additionally, we know there is only one reference when
843 // it's inside a `UniqueArc`, so it is safe to get a mutable reference.
844 unsafe { &mut self.inner.ptr.as_mut().data }
845 }
846}
847
848/// # Examples
849///
850/// ```
851/// # use core::borrow::Borrow;
852/// # use kernel::sync::UniqueArc;
853/// struct Foo<B: Borrow<u32>>(B);
854///
855/// // Owned instance.
856/// let owned = Foo(1);
857///
858/// // Owned instance using `UniqueArc`.
859/// let arc = UniqueArc::new(1, GFP_KERNEL)?;
860/// let shared = Foo(arc);
861///
862/// let i = 1;
863/// // Borrowed from `i`.
864/// let borrowed = Foo(&i);
865/// # Ok::<(), Error>(())
866/// ```
867impl<T: ?Sized> Borrow<T> for UniqueArc<T> {
868 fn borrow(&self) -> &T {
869 self.deref()
870 }
871}
872
873/// # Examples
874///
875/// ```
876/// # use core::borrow::BorrowMut;
877/// # use kernel::sync::UniqueArc;
878/// struct Foo<B: BorrowMut<u32>>(B);
879///
880/// // Owned instance.
881/// let owned = Foo(1);
882///
883/// // Owned instance using `UniqueArc`.
884/// let arc = UniqueArc::new(1, GFP_KERNEL)?;
885/// let shared = Foo(arc);
886///
887/// let mut i = 1;
888/// // Borrowed from `i`.
889/// let borrowed = Foo(&mut i);
890/// # Ok::<(), Error>(())
891/// ```
892impl<T: ?Sized> BorrowMut<T> for UniqueArc<T> {
893 fn borrow_mut(&mut self) -> &mut T {
894 self.deref_mut()
895 }
896}
897
898impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> {
899 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
900 fmt::Display::fmt(self.deref(), f)
901 }
902}
903
904impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> {
905 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
906 fmt::Display::fmt(self.deref(), f)
907 }
908}
909
910impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> {
911 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
912 fmt::Debug::fmt(self.deref(), f)
913 }
914}
915
916impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> {
917 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
918 fmt::Debug::fmt(self.deref(), f)
919 }
920}