pin_init/
lib.rs

1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3//! Library to safely and fallibly initialize pinned `struct`s using in-place constructors.
4//!
5//! [Pinning][pinning] is Rust's way of ensuring data does not move.
6//!
7//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
8//! overflow.
9//!
10//! This library's main use-case is in [Rust-for-Linux]. Although this version can be used
11//! standalone.
12//!
13//! There are cases when you want to in-place initialize a struct. For example when it is very big
14//! and moving it from the stack is not an option, because it is bigger than the stack itself.
15//! Another reason would be that you need the address of the object to initialize it. This stands
16//! in direct conflict with Rust's normal process of first initializing an object and then moving
17//! it into it's final memory location. For more information, see
18//! <https://rust-for-linux.com/the-safe-pinned-initialization-problem>.
19//!
20//! This library allows you to do in-place initialization safely.
21//!
22//! ## Nightly Needed for `alloc` feature
23//!
24//! This library requires the [`allocator_api` unstable feature] when the `alloc` feature is
25//! enabled and thus this feature can only be used with a nightly compiler. When enabling the
26//! `alloc` feature, the user will be required to activate `allocator_api` as well.
27//!
28//! [`allocator_api` unstable feature]: https://doc.rust-lang.org/nightly/unstable-book/library-features/allocator-api.html
29//!
30//! The feature is enabled by default, thus by default `pin-init` will require a nightly compiler.
31//! However, using the crate on stable compilers is possible by disabling `alloc`. In practice this
32//! will require the `std` feature, because stable compilers have neither `Box` nor `Arc` in no-std
33//! mode.
34//!
35//! ## Nightly needed for `unsafe-pinned` feature
36//!
37//! This feature enables the `Wrapper` implementation on the unstable `core::pin::UnsafePinned` type.
38//! This requires the [`unsafe_pinned` unstable feature](https://github.com/rust-lang/rust/issues/125735)
39//! and therefore a nightly compiler. Note that this feature is not enabled by default.
40//!
41//! # Overview
42//!
43//! To initialize a `struct` with an in-place constructor you will need two things:
44//! - an in-place constructor,
45//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`],
46//!   [`Box<T>`] or any other smart pointer that supports this library).
47//!
48//! To get an in-place constructor there are generally three options:
49//! - directly creating an in-place constructor using the [`pin_init!`] macro,
50//! - a custom function/macro returning an in-place constructor provided by someone else,
51//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
52//!
53//! Aside from pinned initialization, this library also supports in-place construction without
54//! pinning, the macros/types/functions are generally named like the pinned variants without the
55//! `pin_` prefix.
56//!
57//! # Examples
58//!
59//! Throughout the examples we will often make use of the `CMutex` type which can be found in
60//! `../examples/mutex.rs`. It is essentially a userland rebuild of the `struct mutex` type from
61//! the Linux kernel. It also uses a wait list and a basic spinlock. Importantly the wait list
62//! requires it to be pinned to be locked and thus is a prime candidate for using this library.
63//!
64//! ## Using the [`pin_init!`] macro
65//!
66//! If you want to use [`PinInit`], then you will have to annotate your `struct` with
67//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
68//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
69//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
70//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
71//!
72//! ```rust
73//! # #![expect(clippy::disallowed_names)]
74//! # #![feature(allocator_api)]
75//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
76//! # use core::pin::Pin;
77//! use pin_init::{pin_data, pin_init, InPlaceInit};
78//!
79//! #[pin_data]
80//! struct Foo {
81//!     #[pin]
82//!     a: CMutex<usize>,
83//!     b: u32,
84//! }
85//!
86//! let foo = pin_init!(Foo {
87//!     a <- CMutex::new(42),
88//!     b: 24,
89//! });
90//! # let _ = Box::pin_init(foo);
91//! ```
92//!
93//! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like
94//! (or just the stack) to actually initialize a `Foo`:
95//!
96//! ```rust
97//! # #![expect(clippy::disallowed_names)]
98//! # #![feature(allocator_api)]
99//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
100//! # use core::{alloc::AllocError, pin::Pin};
101//! # use pin_init::*;
102//! #
103//! # #[pin_data]
104//! # struct Foo {
105//! #     #[pin]
106//! #     a: CMutex<usize>,
107//! #     b: u32,
108//! # }
109//! #
110//! # let foo = pin_init!(Foo {
111//! #     a <- CMutex::new(42),
112//! #     b: 24,
113//! # });
114//! let foo: Result<Pin<Box<Foo>>, AllocError> = Box::pin_init(foo);
115//! ```
116//!
117//! For more information see the [`pin_init!`] macro.
118//!
119//! ## Using a custom function/macro that returns an initializer
120//!
121//! Many types that use this library supply a function/macro that returns an initializer, because
122//! the above method only works for types where you can access the fields.
123//!
124//! ```rust
125//! # #![feature(allocator_api)]
126//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
127//! # use pin_init::*;
128//! # use std::sync::Arc;
129//! # use core::pin::Pin;
130//! let mtx: Result<Pin<Arc<CMutex<usize>>>, _> = Arc::pin_init(CMutex::new(42));
131//! ```
132//!
133//! To declare an init macro/function you just return an [`impl PinInit<T, E>`]:
134//!
135//! ```rust
136//! # #![feature(allocator_api)]
137//! # use pin_init::*;
138//! # #[path = "../examples/error.rs"] mod error; use error::Error;
139//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
140//! #[pin_data]
141//! struct DriverData {
142//!     #[pin]
143//!     status: CMutex<i32>,
144//!     buffer: Box<[u8; 1_000_000]>,
145//! }
146//!
147//! impl DriverData {
148//!     fn new() -> impl PinInit<Self, Error> {
149//!         try_pin_init!(Self {
150//!             status <- CMutex::new(0),
151//!             buffer: Box::init(pin_init::init_zeroed())?,
152//!         }? Error)
153//!     }
154//! }
155//! ```
156//!
157//! ## Manual creation of an initializer
158//!
159//! Often when working with primitives the previous approaches are not sufficient. That is where
160//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a
161//! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure
162//! actually does the initialization in the correct way. Here are the things to look out for
163//! (we are calling the parameter to the closure `slot`):
164//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so
165//!   `slot` now contains a valid bit pattern for the type `T`,
166//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so
167//!   you need to take care to clean up anything if your initialization fails mid-way,
168//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of
169//!   `slot` gets called.
170//!
171//! ```rust
172//! # #![feature(extern_types)]
173//! use pin_init::{pin_data, pinned_drop, PinInit, PinnedDrop, pin_init_from_closure};
174//! use core::{
175//!     ptr::addr_of_mut,
176//!     marker::PhantomPinned,
177//!     cell::UnsafeCell,
178//!     pin::Pin,
179//!     mem::MaybeUninit,
180//! };
181//! mod bindings {
182//!     #[repr(C)]
183//!     pub struct foo {
184//!         /* fields from C ... */
185//!     }
186//!     extern "C" {
187//!         pub fn init_foo(ptr: *mut foo);
188//!         pub fn destroy_foo(ptr: *mut foo);
189//!         #[must_use = "you must check the error return code"]
190//!         pub fn enable_foo(ptr: *mut foo, flags: u32) -> i32;
191//!     }
192//! }
193//!
194//! /// # Invariants
195//! ///
196//! /// `foo` is always initialized
197//! #[pin_data(PinnedDrop)]
198//! pub struct RawFoo {
199//!     #[pin]
200//!     _p: PhantomPinned,
201//!     #[pin]
202//!     foo: UnsafeCell<MaybeUninit<bindings::foo>>,
203//! }
204//!
205//! impl RawFoo {
206//!     pub fn new(flags: u32) -> impl PinInit<Self, i32> {
207//!         // SAFETY:
208//!         // - when the closure returns `Ok(())`, then it has successfully initialized and
209//!         //   enabled `foo`,
210//!         // - when it returns `Err(e)`, then it has cleaned up before
211//!         unsafe {
212//!             pin_init_from_closure(move |slot: *mut Self| {
213//!                 // `slot` contains uninit memory, avoid creating a reference.
214//!                 let foo = addr_of_mut!((*slot).foo);
215//!                 let foo = UnsafeCell::raw_get(foo).cast::<bindings::foo>();
216//!
217//!                 // Initialize the `foo`
218//!                 bindings::init_foo(foo);
219//!
220//!                 // Try to enable it.
221//!                 let err = bindings::enable_foo(foo, flags);
222//!                 if err != 0 {
223//!                     // Enabling has failed, first clean up the foo and then return the error.
224//!                     bindings::destroy_foo(foo);
225//!                     Err(err)
226//!                 } else {
227//!                     // All fields of `RawFoo` have been initialized, since `_p` is a ZST.
228//!                     Ok(())
229//!                 }
230//!             })
231//!         }
232//!     }
233//! }
234//!
235//! #[pinned_drop]
236//! impl PinnedDrop for RawFoo {
237//!     fn drop(self: Pin<&mut Self>) {
238//!         // SAFETY: Since `foo` is initialized, destroying is safe.
239//!         unsafe { bindings::destroy_foo(self.foo.get().cast::<bindings::foo>()) };
240//!     }
241//! }
242//! ```
243//!
244//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside
245//! the `kernel` crate. The [`sync`] module is a good starting point.
246//!
247//! [`sync`]: https://rust.docs.kernel.org/kernel/sync/index.html
248//! [pinning]: https://doc.rust-lang.org/std/pin/index.html
249//! [structurally pinned fields]:
250//!     https://doc.rust-lang.org/std/pin/index.html#projections-and-structural-pinning
251//! [stack]: crate::stack_pin_init
252#![cfg_attr(
253    kernel,
254    doc = "[`Arc<T>`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html"
255)]
256#![cfg_attr(
257    kernel,
258    doc = "[`Box<T>`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html"
259)]
260#![cfg_attr(not(kernel), doc = "[`Arc<T>`]: alloc::alloc::sync::Arc")]
261#![cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")]
262//! [`impl PinInit<Foo>`]: crate::PinInit
263//! [`impl PinInit<T, E>`]: crate::PinInit
264//! [`impl Init<T, E>`]: crate::Init
265//! [Rust-for-Linux]: https://rust-for-linux.com/
266
267#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))]
268#![cfg_attr(
269    all(
270        any(feature = "alloc", feature = "std"),
271        not(RUSTC_NEW_UNINIT_IS_STABLE)
272    ),
273    feature(new_uninit)
274)]
275#![forbid(missing_docs, unsafe_op_in_unsafe_fn)]
276#![cfg_attr(not(feature = "std"), no_std)]
277#![cfg_attr(feature = "alloc", feature(allocator_api))]
278#![cfg_attr(
279    all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED),
280    feature(unsafe_pinned)
281)]
282
283use core::{
284    cell::UnsafeCell,
285    convert::Infallible,
286    marker::PhantomData,
287    mem::MaybeUninit,
288    num::*,
289    pin::Pin,
290    ptr::{self, NonNull},
291};
292
293#[doc(hidden)]
294pub mod __internal;
295#[doc(hidden)]
296pub mod macros;
297
298#[cfg(any(feature = "std", feature = "alloc"))]
299mod alloc;
300#[cfg(any(feature = "std", feature = "alloc"))]
301pub use alloc::InPlaceInit;
302
303/// Used to specify the pinning information of the fields of a struct.
304///
305/// This is somewhat similar in purpose as
306/// [pin-project-lite](https://crates.io/crates/pin-project-lite).
307/// Place this macro on a struct definition and then `#[pin]` in front of the attributes of each
308/// field you want to structurally pin.
309///
310/// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`,
311/// then `#[pin]` directs the type of initializer that is required.
312///
313/// If your `struct` implements `Drop`, then you need to add `PinnedDrop` as arguments to this
314/// macro, and change your `Drop` implementation to `PinnedDrop` annotated with
315/// `#[`[`macro@pinned_drop`]`]`, since dropping pinned values requires extra care.
316///
317/// # Examples
318///
319/// ```
320/// # #![feature(allocator_api)]
321/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
322/// use pin_init::pin_data;
323///
324/// enum Command {
325///     /* ... */
326/// }
327///
328/// #[pin_data]
329/// struct DriverData {
330///     #[pin]
331///     queue: CMutex<Vec<Command>>,
332///     buf: Box<[u8; 1024 * 1024]>,
333/// }
334/// ```
335///
336/// ```
337/// # #![feature(allocator_api)]
338/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
339/// # mod bindings { pub struct info; pub unsafe fn destroy_info(_: *mut info) {} }
340/// use core::pin::Pin;
341/// use pin_init::{pin_data, pinned_drop, PinnedDrop};
342///
343/// enum Command {
344///     /* ... */
345/// }
346///
347/// #[pin_data(PinnedDrop)]
348/// struct DriverData {
349///     #[pin]
350///     queue: CMutex<Vec<Command>>,
351///     buf: Box<[u8; 1024 * 1024]>,
352///     raw_info: *mut bindings::info,
353/// }
354///
355/// #[pinned_drop]
356/// impl PinnedDrop for DriverData {
357///     fn drop(self: Pin<&mut Self>) {
358///         unsafe { bindings::destroy_info(self.raw_info) };
359///     }
360/// }
361/// ```
362pub use ::pin_init_internal::pin_data;
363
364/// Used to implement `PinnedDrop` safely.
365///
366/// Only works on structs that are annotated via `#[`[`macro@pin_data`]`]`.
367///
368/// # Examples
369///
370/// ```
371/// # #![feature(allocator_api)]
372/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
373/// # mod bindings { pub struct info; pub unsafe fn destroy_info(_: *mut info) {} }
374/// use core::pin::Pin;
375/// use pin_init::{pin_data, pinned_drop, PinnedDrop};
376///
377/// enum Command {
378///     /* ... */
379/// }
380///
381/// #[pin_data(PinnedDrop)]
382/// struct DriverData {
383///     #[pin]
384///     queue: CMutex<Vec<Command>>,
385///     buf: Box<[u8; 1024 * 1024]>,
386///     raw_info: *mut bindings::info,
387/// }
388///
389/// #[pinned_drop]
390/// impl PinnedDrop for DriverData {
391///     fn drop(self: Pin<&mut Self>) {
392///         unsafe { bindings::destroy_info(self.raw_info) };
393///     }
394/// }
395/// ```
396pub use ::pin_init_internal::pinned_drop;
397
398/// Derives the [`Zeroable`] trait for the given `struct` or `union`.
399///
400/// This can only be used for `struct`s/`union`s where every field implements the [`Zeroable`]
401/// trait.
402///
403/// # Examples
404///
405/// ```
406/// use pin_init::Zeroable;
407///
408/// #[derive(Zeroable)]
409/// pub struct DriverData {
410///     pub(crate) id: i64,
411///     buf_ptr: *mut u8,
412///     len: usize,
413/// }
414/// ```
415///
416/// ```
417/// use pin_init::Zeroable;
418///
419/// #[derive(Zeroable)]
420/// pub union SignCast {
421///     signed: i64,
422///     unsigned: u64,
423/// }
424/// ```
425pub use ::pin_init_internal::Zeroable;
426
427/// Derives the [`Zeroable`] trait for the given `struct` or `union` if all fields implement
428/// [`Zeroable`].
429///
430/// Contrary to the derive macro named [`macro@Zeroable`], this one silently fails when a field
431/// doesn't implement [`Zeroable`].
432///
433/// # Examples
434///
435/// ```
436/// use pin_init::MaybeZeroable;
437///
438/// // implmements `Zeroable`
439/// #[derive(MaybeZeroable)]
440/// pub struct DriverData {
441///     pub(crate) id: i64,
442///     buf_ptr: *mut u8,
443///     len: usize,
444/// }
445///
446/// // does not implmement `Zeroable`
447/// #[derive(MaybeZeroable)]
448/// pub struct DriverData2 {
449///     pub(crate) id: i64,
450///     buf_ptr: *mut u8,
451///     len: usize,
452///     // this field doesn't implement `Zeroable`
453///     other_data: &'static i32,
454/// }
455/// ```
456pub use ::pin_init_internal::MaybeZeroable;
457
458/// Initialize and pin a type directly on the stack.
459///
460/// # Examples
461///
462/// ```rust
463/// # #![expect(clippy::disallowed_names)]
464/// # #![feature(allocator_api)]
465/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
466/// # use pin_init::*;
467/// # use core::pin::Pin;
468/// #[pin_data]
469/// struct Foo {
470///     #[pin]
471///     a: CMutex<usize>,
472///     b: Bar,
473/// }
474///
475/// #[pin_data]
476/// struct Bar {
477///     x: u32,
478/// }
479///
480/// stack_pin_init!(let foo = pin_init!(Foo {
481///     a <- CMutex::new(42),
482///     b: Bar {
483///         x: 64,
484///     },
485/// }));
486/// let foo: Pin<&mut Foo> = foo;
487/// println!("a: {}", &*foo.a.lock());
488/// ```
489///
490/// # Syntax
491///
492/// A normal `let` binding with optional type annotation. The expression is expected to implement
493/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error
494/// type, then use [`stack_try_pin_init!`].
495#[macro_export]
496macro_rules! stack_pin_init {
497    (let $var:ident $(: $t:ty)? = $val:expr) => {
498        let val = $val;
499        let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit());
500        let mut $var = match $crate::__internal::StackInit::init($var, val) {
501            Ok(res) => res,
502            Err(x) => {
503                let x: ::core::convert::Infallible = x;
504                match x {}
505            }
506        };
507    };
508}
509
510/// Initialize and pin a type directly on the stack.
511///
512/// # Examples
513///
514/// ```rust
515/// # #![expect(clippy::disallowed_names)]
516/// # #![feature(allocator_api)]
517/// # #[path = "../examples/error.rs"] mod error; use error::Error;
518/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
519/// # use pin_init::*;
520/// #[pin_data]
521/// struct Foo {
522///     #[pin]
523///     a: CMutex<usize>,
524///     b: Box<Bar>,
525/// }
526///
527/// struct Bar {
528///     x: u32,
529/// }
530///
531/// stack_try_pin_init!(let foo: Foo = try_pin_init!(Foo {
532///     a <- CMutex::new(42),
533///     b: Box::try_new(Bar {
534///         x: 64,
535///     })?,
536/// }? Error));
537/// let foo = foo.unwrap();
538/// println!("a: {}", &*foo.a.lock());
539/// ```
540///
541/// ```rust
542/// # #![expect(clippy::disallowed_names)]
543/// # #![feature(allocator_api)]
544/// # #[path = "../examples/error.rs"] mod error; use error::Error;
545/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
546/// # use pin_init::*;
547/// #[pin_data]
548/// struct Foo {
549///     #[pin]
550///     a: CMutex<usize>,
551///     b: Box<Bar>,
552/// }
553///
554/// struct Bar {
555///     x: u32,
556/// }
557///
558/// stack_try_pin_init!(let foo: Foo =? try_pin_init!(Foo {
559///     a <- CMutex::new(42),
560///     b: Box::try_new(Bar {
561///         x: 64,
562///     })?,
563/// }? Error));
564/// println!("a: {}", &*foo.a.lock());
565/// # Ok::<_, Error>(())
566/// ```
567///
568/// # Syntax
569///
570/// A normal `let` binding with optional type annotation. The expression is expected to implement
571/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the
572/// `=` will propagate this error.
573#[macro_export]
574macro_rules! stack_try_pin_init {
575    (let $var:ident $(: $t:ty)? = $val:expr) => {
576        let val = $val;
577        let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit());
578        let mut $var = $crate::__internal::StackInit::init($var, val);
579    };
580    (let $var:ident $(: $t:ty)? =? $val:expr) => {
581        let val = $val;
582        let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit());
583        let mut $var = $crate::__internal::StackInit::init($var, val)?;
584    };
585}
586
587/// Construct an in-place, pinned initializer for `struct`s.
588///
589/// This macro defaults the error to [`Infallible`]. If you need a different error, then use
590/// [`try_pin_init!`].
591///
592/// The syntax is almost identical to that of a normal `struct` initializer:
593///
594/// ```rust
595/// # use pin_init::*;
596/// # use core::pin::Pin;
597/// #[pin_data]
598/// struct Foo {
599///     a: usize,
600///     b: Bar,
601/// }
602///
603/// #[pin_data]
604/// struct Bar {
605///     x: u32,
606/// }
607///
608/// # fn demo() -> impl PinInit<Foo> {
609/// let a = 42;
610///
611/// let initializer = pin_init!(Foo {
612///     a,
613///     b: Bar {
614///         x: 64,
615///     },
616/// });
617/// # initializer }
618/// # Box::pin_init(demo()).unwrap();
619/// ```
620///
621/// Arbitrary Rust expressions can be used to set the value of a variable.
622///
623/// The fields are initialized in the order that they appear in the initializer. So it is possible
624/// to read already initialized fields using raw pointers.
625///
626/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
627/// initializer.
628///
629/// # Init-functions
630///
631/// When working with this library it is often desired to let others construct your types without
632/// giving access to all fields. This is where you would normally write a plain function `new` that
633/// would return a new instance of your type. With this library that is also possible. However,
634/// there are a few extra things to keep in mind.
635///
636/// To create an initializer function, simply declare it like this:
637///
638/// ```rust
639/// # use pin_init::*;
640/// # use core::pin::Pin;
641/// # #[pin_data]
642/// # struct Foo {
643/// #     a: usize,
644/// #     b: Bar,
645/// # }
646/// # #[pin_data]
647/// # struct Bar {
648/// #     x: u32,
649/// # }
650/// impl Foo {
651///     fn new() -> impl PinInit<Self> {
652///         pin_init!(Self {
653///             a: 42,
654///             b: Bar {
655///                 x: 64,
656///             },
657///         })
658///     }
659/// }
660/// ```
661///
662/// Users of `Foo` can now create it like this:
663///
664/// ```rust
665/// # #![expect(clippy::disallowed_names)]
666/// # use pin_init::*;
667/// # use core::pin::Pin;
668/// # #[pin_data]
669/// # struct Foo {
670/// #     a: usize,
671/// #     b: Bar,
672/// # }
673/// # #[pin_data]
674/// # struct Bar {
675/// #     x: u32,
676/// # }
677/// # impl Foo {
678/// #     fn new() -> impl PinInit<Self> {
679/// #         pin_init!(Self {
680/// #             a: 42,
681/// #             b: Bar {
682/// #                 x: 64,
683/// #             },
684/// #         })
685/// #     }
686/// # }
687/// let foo = Box::pin_init(Foo::new());
688/// ```
689///
690/// They can also easily embed it into their own `struct`s:
691///
692/// ```rust
693/// # use pin_init::*;
694/// # use core::pin::Pin;
695/// # #[pin_data]
696/// # struct Foo {
697/// #     a: usize,
698/// #     b: Bar,
699/// # }
700/// # #[pin_data]
701/// # struct Bar {
702/// #     x: u32,
703/// # }
704/// # impl Foo {
705/// #     fn new() -> impl PinInit<Self> {
706/// #         pin_init!(Self {
707/// #             a: 42,
708/// #             b: Bar {
709/// #                 x: 64,
710/// #             },
711/// #         })
712/// #     }
713/// # }
714/// #[pin_data]
715/// struct FooContainer {
716///     #[pin]
717///     foo1: Foo,
718///     #[pin]
719///     foo2: Foo,
720///     other: u32,
721/// }
722///
723/// impl FooContainer {
724///     fn new(other: u32) -> impl PinInit<Self> {
725///         pin_init!(Self {
726///             foo1 <- Foo::new(),
727///             foo2 <- Foo::new(),
728///             other,
729///         })
730///     }
731/// }
732/// ```
733///
734/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
735/// This signifies that the given field is initialized in-place. As with `struct` initializers, just
736/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
737///
738/// # Syntax
739///
740/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
741/// the following modifications is expected:
742/// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
743/// - You can use `_: { /* run any user-code here */ },` anywhere where you can place fields in
744///   order to run arbitrary code.
745/// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
746///   pointer named `this` inside of the initializer.
747/// - Using struct update syntax one can place `..Zeroable::init_zeroed()` at the very end of the
748///   struct, this initializes every field with 0 and then runs all initializers specified in the
749///   body. This can only be done if [`Zeroable`] is implemented for the struct.
750///
751/// For instance:
752///
753/// ```rust
754/// # use pin_init::*;
755/// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
756/// #[pin_data]
757/// #[derive(Zeroable)]
758/// struct Buf {
759///     // `ptr` points into `buf`.
760///     ptr: *mut u8,
761///     buf: [u8; 64],
762///     #[pin]
763///     pin: PhantomPinned,
764/// }
765///
766/// let init = pin_init!(&this in Buf {
767///     buf: [0; 64],
768///     // SAFETY: TODO.
769///     ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
770///     pin: PhantomPinned,
771/// });
772/// let init = pin_init!(Buf {
773///     buf: [1; 64],
774///     ..Zeroable::init_zeroed()
775/// });
776/// ```
777///
778/// [`NonNull<Self>`]: core::ptr::NonNull
779// For a detailed example of how this macro works, see the module documentation of the hidden
780// module `macros` inside of `macros.rs`.
781#[macro_export]
782macro_rules! pin_init {
783    ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
784        $($fields:tt)*
785    }) => {
786        $crate::try_pin_init!($(&$this in)? $t $(::<$($generics),*>)? {
787            $($fields)*
788        }? ::core::convert::Infallible)
789    };
790}
791
792/// Construct an in-place, fallible pinned initializer for `struct`s.
793///
794/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`].
795///
796/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop
797/// initialization and return the error.
798///
799/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when
800/// initialization fails, the memory can be safely deallocated without any further modifications.
801///
802/// The syntax is identical to [`pin_init!`] with the following exception: you must append `? $type`
803/// after the `struct` initializer to specify the error type you want to use.
804///
805/// # Examples
806///
807/// ```rust
808/// # #![feature(allocator_api)]
809/// # #[path = "../examples/error.rs"] mod error; use error::Error;
810/// use pin_init::{pin_data, try_pin_init, PinInit, InPlaceInit, init_zeroed};
811///
812/// #[pin_data]
813/// struct BigBuf {
814///     big: Box<[u8; 1024 * 1024 * 1024]>,
815///     small: [u8; 1024 * 1024],
816///     ptr: *mut u8,
817/// }
818///
819/// impl BigBuf {
820///     fn new() -> impl PinInit<Self, Error> {
821///         try_pin_init!(Self {
822///             big: Box::init(init_zeroed())?,
823///             small: [0; 1024 * 1024],
824///             ptr: core::ptr::null_mut(),
825///         }? Error)
826///     }
827/// }
828/// # let _ = Box::pin_init(BigBuf::new());
829/// ```
830// For a detailed example of how this macro works, see the module documentation of the hidden
831// module `macros` inside of `macros.rs`.
832#[macro_export]
833macro_rules! try_pin_init {
834    ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
835        $($fields:tt)*
836    }? $err:ty) => {
837        $crate::__init_internal!(
838            @this($($this)?),
839            @typ($t $(::<$($generics),*>)? ),
840            @fields($($fields)*),
841            @error($err),
842            @data(PinData, use_data),
843            @has_data(HasPinData, __pin_data),
844            @construct_closure(pin_init_from_closure),
845            @munch_fields($($fields)*),
846        )
847    }
848}
849
850/// Construct an in-place initializer for `struct`s.
851///
852/// This macro defaults the error to [`Infallible`]. If you need a different error, then use
853/// [`try_init!`].
854///
855/// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
856/// - `unsafe` code must guarantee either full initialization or return an error and allow
857///   deallocation of the memory.
858/// - the fields are initialized in the order given in the initializer.
859/// - no references to fields are allowed to be created inside of the initializer.
860///
861/// This initializer is for initializing data in-place that might later be moved. If you want to
862/// pin-initialize, use [`pin_init!`].
863///
864/// # Examples
865///
866/// ```rust
867/// # #![feature(allocator_api)]
868/// # #[path = "../examples/error.rs"] mod error; use error::Error;
869/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
870/// # use pin_init::InPlaceInit;
871/// use pin_init::{init, Init, init_zeroed};
872///
873/// struct BigBuf {
874///     small: [u8; 1024 * 1024],
875/// }
876///
877/// impl BigBuf {
878///     fn new() -> impl Init<Self> {
879///         init!(Self {
880///             small <- init_zeroed(),
881///         })
882///     }
883/// }
884/// # let _ = Box::init(BigBuf::new());
885/// ```
886// For a detailed example of how this macro works, see the module documentation of the hidden
887// module `macros` inside of `macros.rs`.
888#[macro_export]
889macro_rules! init {
890    ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
891        $($fields:tt)*
892    }) => {
893        $crate::try_init!($(&$this in)? $t $(::<$($generics),*>)? {
894            $($fields)*
895        }? ::core::convert::Infallible)
896    }
897}
898
899/// Construct an in-place fallible initializer for `struct`s.
900///
901/// If the initialization can complete without error (or [`Infallible`]), then use
902/// [`init!`].
903///
904/// The syntax is identical to [`try_pin_init!`]. You need to specify a custom error
905/// via `? $type` after the `struct` initializer.
906/// The safety caveats from [`try_pin_init!`] also apply:
907/// - `unsafe` code must guarantee either full initialization or return an error and allow
908///   deallocation of the memory.
909/// - the fields are initialized in the order given in the initializer.
910/// - no references to fields are allowed to be created inside of the initializer.
911///
912/// # Examples
913///
914/// ```rust
915/// # #![feature(allocator_api)]
916/// # use core::alloc::AllocError;
917/// # use pin_init::InPlaceInit;
918/// use pin_init::{try_init, Init, init_zeroed};
919///
920/// struct BigBuf {
921///     big: Box<[u8; 1024 * 1024 * 1024]>,
922///     small: [u8; 1024 * 1024],
923/// }
924///
925/// impl BigBuf {
926///     fn new() -> impl Init<Self, AllocError> {
927///         try_init!(Self {
928///             big: Box::init(init_zeroed())?,
929///             small: [0; 1024 * 1024],
930///         }? AllocError)
931///     }
932/// }
933/// # let _ = Box::init(BigBuf::new());
934/// ```
935// For a detailed example of how this macro works, see the module documentation of the hidden
936// module `macros` inside of `macros.rs`.
937#[macro_export]
938macro_rules! try_init {
939    ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
940        $($fields:tt)*
941    }? $err:ty) => {
942        $crate::__init_internal!(
943            @this($($this)?),
944            @typ($t $(::<$($generics),*>)?),
945            @fields($($fields)*),
946            @error($err),
947            @data(InitData, /*no use_data*/),
948            @has_data(HasInitData, __init_data),
949            @construct_closure(init_from_closure),
950            @munch_fields($($fields)*),
951        )
952    }
953}
954
955/// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is
956/// structurally pinned.
957///
958/// # Examples
959///
960/// This will succeed:
961/// ```
962/// use pin_init::{pin_data, assert_pinned};
963///
964/// #[pin_data]
965/// struct MyStruct {
966///     #[pin]
967///     some_field: u64,
968/// }
969///
970/// assert_pinned!(MyStruct, some_field, u64);
971/// ```
972///
973/// This will fail:
974/// ```compile_fail
975/// use pin_init::{pin_data, assert_pinned};
976///
977/// #[pin_data]
978/// struct MyStruct {
979///     some_field: u64,
980/// }
981///
982/// assert_pinned!(MyStruct, some_field, u64);
983/// ```
984///
985/// Some uses of the macro may trigger the `can't use generic parameters from outer item` error. To
986/// work around this, you may pass the `inline` parameter to the macro. The `inline` parameter can
987/// only be used when the macro is invoked from a function body.
988/// ```
989/// # use core::pin::Pin;
990/// use pin_init::{pin_data, assert_pinned};
991///
992/// #[pin_data]
993/// struct Foo<T> {
994///     #[pin]
995///     elem: T,
996/// }
997///
998/// impl<T> Foo<T> {
999///     fn project_this(self: Pin<&mut Self>) -> Pin<&mut T> {
1000///         assert_pinned!(Foo<T>, elem, T, inline);
1001///
1002///         // SAFETY: The field is structurally pinned.
1003///         unsafe { self.map_unchecked_mut(|me| &mut me.elem) }
1004///     }
1005/// }
1006/// ```
1007#[macro_export]
1008macro_rules! assert_pinned {
1009    ($ty:ty, $field:ident, $field_ty:ty, inline) => {
1010        let _ = move |ptr: *mut $field_ty| {
1011            // SAFETY: This code is unreachable.
1012            let data = unsafe { <$ty as $crate::__internal::HasPinData>::__pin_data() };
1013            let init = $crate::__internal::AlwaysFail::<$field_ty>::new();
1014            // SAFETY: This code is unreachable.
1015            unsafe { data.$field(ptr, init) }.ok();
1016        };
1017    };
1018
1019    ($ty:ty, $field:ident, $field_ty:ty) => {
1020        const _: () = {
1021            $crate::assert_pinned!($ty, $field, $field_ty, inline);
1022        };
1023    };
1024}
1025
1026/// A pin-initializer for the type `T`.
1027///
1028/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
1029/// be [`Box<T>`], [`Arc<T>`] or even the stack (see [`stack_pin_init!`]).
1030///
1031/// Also see the [module description](self).
1032///
1033/// # Safety
1034///
1035/// When implementing this trait you will need to take great care. Also there are probably very few
1036/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
1037///
1038/// The [`PinInit::__pinned_init`] function:
1039/// - returns `Ok(())` if it initialized every field of `slot`,
1040/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1041///     - `slot` can be deallocated without UB occurring,
1042///     - `slot` does not need to be dropped,
1043///     - `slot` is not partially initialized.
1044/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1045///
1046#[cfg_attr(
1047    kernel,
1048    doc = "[`Arc<T>`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html"
1049)]
1050#[cfg_attr(
1051    kernel,
1052    doc = "[`Box<T>`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html"
1053)]
1054#[cfg_attr(not(kernel), doc = "[`Arc<T>`]: alloc::alloc::sync::Arc")]
1055#[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")]
1056#[must_use = "An initializer must be used in order to create its value."]
1057pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
1058    /// Initializes `slot`.
1059    ///
1060    /// # Safety
1061    ///
1062    /// - `slot` is a valid pointer to uninitialized memory.
1063    /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
1064    ///   deallocate.
1065    /// - `slot` will not move until it is dropped, i.e. it will be pinned.
1066    unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
1067
1068    /// First initializes the value using `self` then calls the function `f` with the initialized
1069    /// value.
1070    ///
1071    /// If `f` returns an error the value is dropped and the initializer will forward the error.
1072    ///
1073    /// # Examples
1074    ///
1075    /// ```rust
1076    /// # #![feature(allocator_api)]
1077    /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
1078    /// # use pin_init::*;
1079    /// let mtx_init = CMutex::new(42);
1080    /// // Make the initializer print the value.
1081    /// let mtx_init = mtx_init.pin_chain(|mtx| {
1082    ///     println!("{:?}", mtx.get_data_mut());
1083    ///     Ok(())
1084    /// });
1085    /// ```
1086    fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E>
1087    where
1088        F: FnOnce(Pin<&mut T>) -> Result<(), E>,
1089    {
1090        ChainPinInit(self, f, PhantomData)
1091    }
1092}
1093
1094/// An initializer returned by [`PinInit::pin_chain`].
1095pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, T)>);
1096
1097// SAFETY: The `__pinned_init` function is implemented such that it
1098// - returns `Ok(())` on successful initialization,
1099// - returns `Err(err)` on error and in this case `slot` will be dropped.
1100// - considers `slot` pinned.
1101unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E>
1102where
1103    I: PinInit<T, E>,
1104    F: FnOnce(Pin<&mut T>) -> Result<(), E>,
1105{
1106    unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
1107        // SAFETY: All requirements fulfilled since this function is `__pinned_init`.
1108        unsafe { self.0.__pinned_init(slot)? };
1109        // SAFETY: The above call initialized `slot` and we still have unique access.
1110        let val = unsafe { &mut *slot };
1111        // SAFETY: `slot` is considered pinned.
1112        let val = unsafe { Pin::new_unchecked(val) };
1113        // SAFETY: `slot` was initialized above.
1114        (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) })
1115    }
1116}
1117
1118/// An initializer for `T`.
1119///
1120/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
1121/// be [`Box<T>`], [`Arc<T>`] or even the stack (see [`stack_pin_init!`]). Because
1122/// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well.
1123///
1124/// Also see the [module description](self).
1125///
1126/// # Safety
1127///
1128/// When implementing this trait you will need to take great care. Also there are probably very few
1129/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
1130///
1131/// The [`Init::__init`] function:
1132/// - returns `Ok(())` if it initialized every field of `slot`,
1133/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1134///     - `slot` can be deallocated without UB occurring,
1135///     - `slot` does not need to be dropped,
1136///     - `slot` is not partially initialized.
1137/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1138///
1139/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
1140/// code as `__init`.
1141///
1142/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
1143/// move the pointee after initialization.
1144///
1145#[cfg_attr(
1146    kernel,
1147    doc = "[`Arc<T>`]: https://rust.docs.kernel.org/kernel/sync/struct.Arc.html"
1148)]
1149#[cfg_attr(
1150    kernel,
1151    doc = "[`Box<T>`]: https://rust.docs.kernel.org/kernel/alloc/kbox/struct.Box.html"
1152)]
1153#[cfg_attr(not(kernel), doc = "[`Arc<T>`]: alloc::alloc::sync::Arc")]
1154#[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")]
1155#[must_use = "An initializer must be used in order to create its value."]
1156pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
1157    /// Initializes `slot`.
1158    ///
1159    /// # Safety
1160    ///
1161    /// - `slot` is a valid pointer to uninitialized memory.
1162    /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
1163    ///   deallocate.
1164    unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
1165
1166    /// First initializes the value using `self` then calls the function `f` with the initialized
1167    /// value.
1168    ///
1169    /// If `f` returns an error the value is dropped and the initializer will forward the error.
1170    ///
1171    /// # Examples
1172    ///
1173    /// ```rust
1174    /// # #![expect(clippy::disallowed_names)]
1175    /// use pin_init::{init, init_zeroed, Init};
1176    ///
1177    /// struct Foo {
1178    ///     buf: [u8; 1_000_000],
1179    /// }
1180    ///
1181    /// impl Foo {
1182    ///     fn setup(&mut self) {
1183    ///         println!("Setting up foo");
1184    ///     }
1185    /// }
1186    ///
1187    /// let foo = init!(Foo {
1188    ///     buf <- init_zeroed()
1189    /// }).chain(|foo| {
1190    ///     foo.setup();
1191    ///     Ok(())
1192    /// });
1193    /// ```
1194    fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E>
1195    where
1196        F: FnOnce(&mut T) -> Result<(), E>,
1197    {
1198        ChainInit(self, f, PhantomData)
1199    }
1200}
1201
1202/// An initializer returned by [`Init::chain`].
1203pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, T)>);
1204
1205// SAFETY: The `__init` function is implemented such that it
1206// - returns `Ok(())` on successful initialization,
1207// - returns `Err(err)` on error and in this case `slot` will be dropped.
1208unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E>
1209where
1210    I: Init<T, E>,
1211    F: FnOnce(&mut T) -> Result<(), E>,
1212{
1213    unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
1214        // SAFETY: All requirements fulfilled since this function is `__init`.
1215        unsafe { self.0.__pinned_init(slot)? };
1216        // SAFETY: The above call initialized `slot` and we still have unique access.
1217        (self.1)(unsafe { &mut *slot }).inspect_err(|_|
1218            // SAFETY: `slot` was initialized above.
1219            unsafe { core::ptr::drop_in_place(slot) })
1220    }
1221}
1222
1223// SAFETY: `__pinned_init` behaves exactly the same as `__init`.
1224unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E>
1225where
1226    I: Init<T, E>,
1227    F: FnOnce(&mut T) -> Result<(), E>,
1228{
1229    unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
1230        // SAFETY: `__init` has less strict requirements compared to `__pinned_init`.
1231        unsafe { self.__init(slot) }
1232    }
1233}
1234
1235/// Creates a new [`PinInit<T, E>`] from the given closure.
1236///
1237/// # Safety
1238///
1239/// The closure:
1240/// - returns `Ok(())` if it initialized every field of `slot`,
1241/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1242///     - `slot` can be deallocated without UB occurring,
1243///     - `slot` does not need to be dropped,
1244///     - `slot` is not partially initialized.
1245/// - may assume that the `slot` does not move if `T: !Unpin`,
1246/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1247#[inline]
1248pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
1249    f: impl FnOnce(*mut T) -> Result<(), E>,
1250) -> impl PinInit<T, E> {
1251    __internal::InitClosure(f, PhantomData)
1252}
1253
1254/// Creates a new [`Init<T, E>`] from the given closure.
1255///
1256/// # Safety
1257///
1258/// The closure:
1259/// - returns `Ok(())` if it initialized every field of `slot`,
1260/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1261///     - `slot` can be deallocated without UB occurring,
1262///     - `slot` does not need to be dropped,
1263///     - `slot` is not partially initialized.
1264/// - the `slot` may move after initialization.
1265/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1266#[inline]
1267pub const unsafe fn init_from_closure<T: ?Sized, E>(
1268    f: impl FnOnce(*mut T) -> Result<(), E>,
1269) -> impl Init<T, E> {
1270    __internal::InitClosure(f, PhantomData)
1271}
1272
1273/// Changes the to be initialized type.
1274///
1275/// # Safety
1276///
1277/// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a
1278///   pointer must result in a valid `U`.
1279#[expect(clippy::let_and_return)]
1280pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl PinInit<U, E> {
1281    // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety
1282    // requirements.
1283    let res = unsafe { pin_init_from_closure(|ptr: *mut U| init.__pinned_init(ptr.cast::<T>())) };
1284    // FIXME: remove the let statement once the nightly-MSRV allows it (1.78 otherwise encounters a
1285    // cycle when computing the type returned by this function)
1286    res
1287}
1288
1289/// Changes the to be initialized type.
1290///
1291/// # Safety
1292///
1293/// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a
1294///   pointer must result in a valid `U`.
1295#[expect(clippy::let_and_return)]
1296pub const unsafe fn cast_init<T, U, E>(init: impl Init<T, E>) -> impl Init<U, E> {
1297    // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety
1298    // requirements.
1299    let res = unsafe { init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) };
1300    // FIXME: remove the let statement once the nightly-MSRV allows it (1.78 otherwise encounters a
1301    // cycle when computing the type returned by this function)
1302    res
1303}
1304
1305/// An initializer that leaves the memory uninitialized.
1306///
1307/// The initializer is a no-op. The `slot` memory is not changed.
1308#[inline]
1309pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
1310    // SAFETY: The memory is allowed to be uninitialized.
1311    unsafe { init_from_closure(|_| Ok(())) }
1312}
1313
1314/// Initializes an array by initializing each element via the provided initializer.
1315///
1316/// # Examples
1317///
1318/// ```rust
1319/// # use pin_init::*;
1320/// use pin_init::init_array_from_fn;
1321/// let array: Box<[usize; 1_000]> = Box::init(init_array_from_fn(|i| i)).unwrap();
1322/// assert_eq!(array.len(), 1_000);
1323/// ```
1324pub fn init_array_from_fn<I, const N: usize, T, E>(
1325    mut make_init: impl FnMut(usize) -> I,
1326) -> impl Init<[T; N], E>
1327where
1328    I: Init<T, E>,
1329{
1330    let init = move |slot: *mut [T; N]| {
1331        let slot = slot.cast::<T>();
1332        for i in 0..N {
1333            let init = make_init(i);
1334            // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
1335            let ptr = unsafe { slot.add(i) };
1336            // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
1337            // requirements.
1338            if let Err(e) = unsafe { init.__init(ptr) } {
1339                // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return
1340                // `Err` below, `slot` will be considered uninitialized memory.
1341                unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
1342                return Err(e);
1343            }
1344        }
1345        Ok(())
1346    };
1347    // SAFETY: The initializer above initializes every element of the array. On failure it drops
1348    // any initialized elements and returns `Err`.
1349    unsafe { init_from_closure(init) }
1350}
1351
1352/// Initializes an array by initializing each element via the provided initializer.
1353///
1354/// # Examples
1355///
1356/// ```rust
1357/// # #![feature(allocator_api)]
1358/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
1359/// # use pin_init::*;
1360/// # use core::pin::Pin;
1361/// use pin_init::pin_init_array_from_fn;
1362/// use std::sync::Arc;
1363/// let array: Pin<Arc<[CMutex<usize>; 1_000]>> =
1364///     Arc::pin_init(pin_init_array_from_fn(|i| CMutex::new(i))).unwrap();
1365/// assert_eq!(array.len(), 1_000);
1366/// ```
1367pub fn pin_init_array_from_fn<I, const N: usize, T, E>(
1368    mut make_init: impl FnMut(usize) -> I,
1369) -> impl PinInit<[T; N], E>
1370where
1371    I: PinInit<T, E>,
1372{
1373    let init = move |slot: *mut [T; N]| {
1374        let slot = slot.cast::<T>();
1375        for i in 0..N {
1376            let init = make_init(i);
1377            // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
1378            let ptr = unsafe { slot.add(i) };
1379            // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
1380            // requirements.
1381            if let Err(e) = unsafe { init.__pinned_init(ptr) } {
1382                // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return
1383                // `Err` below, `slot` will be considered uninitialized memory.
1384                unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
1385                return Err(e);
1386            }
1387        }
1388        Ok(())
1389    };
1390    // SAFETY: The initializer above initializes every element of the array. On failure it drops
1391    // any initialized elements and returns `Err`.
1392    unsafe { pin_init_from_closure(init) }
1393}
1394
1395// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of `slot`.
1396unsafe impl<T> Init<T> for T {
1397    unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> {
1398        // SAFETY: `slot` is valid for writes by the safety requirements of this function.
1399        unsafe { slot.write(self) };
1400        Ok(())
1401    }
1402}
1403
1404// SAFETY: the `__pinned_init` function always returns `Ok(())` and initializes every field of
1405// `slot`. Additionally, all pinning invariants of `T` are upheld.
1406unsafe impl<T> PinInit<T> for T {
1407    unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), Infallible> {
1408        // SAFETY: `slot` is valid for writes by the safety requirements of this function.
1409        unsafe { slot.write(self) };
1410        Ok(())
1411    }
1412}
1413
1414// SAFETY: when the `__init` function returns with
1415// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld.
1416// - `Err(err)`, slot was not written to.
1417unsafe impl<T, E> Init<T, E> for Result<T, E> {
1418    unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
1419        // SAFETY: `slot` is valid for writes by the safety requirements of this function.
1420        unsafe { slot.write(self?) };
1421        Ok(())
1422    }
1423}
1424
1425// SAFETY: when the `__pinned_init` function returns with
1426// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld.
1427// - `Err(err)`, slot was not written to.
1428unsafe impl<T, E> PinInit<T, E> for Result<T, E> {
1429    unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
1430        // SAFETY: `slot` is valid for writes by the safety requirements of this function.
1431        unsafe { slot.write(self?) };
1432        Ok(())
1433    }
1434}
1435
1436/// Smart pointer containing uninitialized memory and that can write a value.
1437pub trait InPlaceWrite<T> {
1438    /// The type `Self` turns into when the contents are initialized.
1439    type Initialized;
1440
1441    /// Use the given initializer to write a value into `self`.
1442    ///
1443    /// Does not drop the current value and considers it as uninitialized memory.
1444    fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E>;
1445
1446    /// Use the given pin-initializer to write a value into `self`.
1447    ///
1448    /// Does not drop the current value and considers it as uninitialized memory.
1449    fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E>;
1450}
1451
1452/// Trait facilitating pinned destruction.
1453///
1454/// Use [`pinned_drop`] to implement this trait safely:
1455///
1456/// ```rust
1457/// # #![feature(allocator_api)]
1458/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
1459/// # use pin_init::*;
1460/// use core::pin::Pin;
1461/// #[pin_data(PinnedDrop)]
1462/// struct Foo {
1463///     #[pin]
1464///     mtx: CMutex<usize>,
1465/// }
1466///
1467/// #[pinned_drop]
1468/// impl PinnedDrop for Foo {
1469///     fn drop(self: Pin<&mut Self>) {
1470///         println!("Foo is being dropped!");
1471///     }
1472/// }
1473/// ```
1474///
1475/// # Safety
1476///
1477/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl.
1478pub unsafe trait PinnedDrop: __internal::HasPinData {
1479    /// Executes the pinned destructor of this type.
1480    ///
1481    /// While this function is marked safe, it is actually unsafe to call it manually. For this
1482    /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code
1483    /// and thus prevents this function from being called where it should not.
1484    ///
1485    /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute
1486    /// automatically.
1487    fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop);
1488}
1489
1490/// Marker trait for types that can be initialized by writing just zeroes.
1491///
1492/// # Safety
1493///
1494/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words,
1495/// this is not UB:
1496///
1497/// ```rust,ignore
1498/// let val: Self = unsafe { core::mem::zeroed() };
1499/// ```
1500pub unsafe trait Zeroable {
1501    /// Create a new zeroed `Self`.
1502    ///
1503    /// The returned initializer will write `0x00` to every byte of the given `slot`.
1504    #[inline]
1505    fn init_zeroed() -> impl Init<Self>
1506    where
1507        Self: Sized,
1508    {
1509        init_zeroed()
1510    }
1511
1512    /// Create a `Self` consisting of all zeroes.
1513    ///
1514    /// Whenever a type implements [`Zeroable`], this function should be preferred over
1515    /// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`.
1516    ///
1517    /// # Examples
1518    ///
1519    /// ```
1520    /// use pin_init::{Zeroable, zeroed};
1521    ///
1522    /// #[derive(Zeroable)]
1523    /// struct Point {
1524    ///     x: u32,
1525    ///     y: u32,
1526    /// }
1527    ///
1528    /// let point: Point = zeroed();
1529    /// assert_eq!(point.x, 0);
1530    /// assert_eq!(point.y, 0);
1531    /// ```
1532    fn zeroed() -> Self
1533    where
1534        Self: Sized,
1535    {
1536        zeroed()
1537    }
1538}
1539
1540/// Marker trait for types that allow `Option<Self>` to be set to all zeroes in order to write
1541/// `None` to that location.
1542///
1543/// # Safety
1544///
1545/// The implementer needs to ensure that `unsafe impl Zeroable for Option<Self> {}` is sound.
1546pub unsafe trait ZeroableOption {}
1547
1548// SAFETY: by the safety requirement of `ZeroableOption`, this is valid.
1549unsafe impl<T: ZeroableOption> Zeroable for Option<T> {}
1550
1551// SAFETY: `Option<&T>` is part of the option layout optimization guarantee:
1552// <https://doc.rust-lang.org/stable/std/option/index.html#representation>.
1553unsafe impl<T> ZeroableOption for &T {}
1554// SAFETY: `Option<&mut T>` is part of the option layout optimization guarantee:
1555// <https://doc.rust-lang.org/stable/std/option/index.html#representation>.
1556unsafe impl<T> ZeroableOption for &mut T {}
1557// SAFETY: `Option<NonNull<T>>` is part of the option layout optimization guarantee:
1558// <https://doc.rust-lang.org/stable/std/option/index.html#representation>.
1559unsafe impl<T> ZeroableOption for NonNull<T> {}
1560
1561/// Create an initializer for a zeroed `T`.
1562///
1563/// The returned initializer will write `0x00` to every byte of the given `slot`.
1564#[inline]
1565pub fn init_zeroed<T: Zeroable>() -> impl Init<T> {
1566    // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T`
1567    // and because we write all zeroes, the memory is initialized.
1568    unsafe {
1569        init_from_closure(|slot: *mut T| {
1570            slot.write_bytes(0, 1);
1571            Ok(())
1572        })
1573    }
1574}
1575
1576/// Create a `T` consisting of all zeroes.
1577///
1578/// Whenever a type implements [`Zeroable`], this function should be preferred over
1579/// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`.
1580///
1581/// # Examples
1582///
1583/// ```
1584/// use pin_init::{Zeroable, zeroed};
1585///
1586/// #[derive(Zeroable)]
1587/// struct Point {
1588///     x: u32,
1589///     y: u32,
1590/// }
1591///
1592/// let point: Point = zeroed();
1593/// assert_eq!(point.x, 0);
1594/// assert_eq!(point.y, 0);
1595/// ```
1596pub const fn zeroed<T: Zeroable>() -> T {
1597    // SAFETY:By the type invariants of `Zeroable`, all zeroes is a valid bit pattern for `T`.
1598    unsafe { core::mem::zeroed() }
1599}
1600
1601macro_rules! impl_zeroable {
1602    ($($({$($generics:tt)*})? $t:ty, )*) => {
1603        // SAFETY: Safety comments written in the macro invocation.
1604        $(unsafe impl$($($generics)*)? Zeroable for $t {})*
1605    };
1606}
1607
1608impl_zeroable! {
1609    // SAFETY: All primitives that are allowed to be zero.
1610    bool,
1611    char,
1612    u8, u16, u32, u64, u128, usize,
1613    i8, i16, i32, i64, i128, isize,
1614    f32, f64,
1615
1616    // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list;
1617    // creating an instance of an uninhabited type is immediate undefined behavior. For more on
1618    // uninhabited/empty types, consult The Rustonomicon:
1619    // <https://doc.rust-lang.org/stable/nomicon/exotic-sizes.html#empty-types>. The Rust Reference
1620    // also has information on undefined behavior:
1621    // <https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html>.
1622    //
1623    // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists.
1624    {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, (),
1625
1626    // SAFETY: Type is allowed to take any value, including all zeros.
1627    {<T>} MaybeUninit<T>,
1628
1629    // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`.
1630    {<T: ?Sized + Zeroable>} UnsafeCell<T>,
1631
1632    // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee:
1633    // <https://doc.rust-lang.org/stable/std/option/index.html#representation>).
1634    Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>,
1635    Option<NonZeroU128>, Option<NonZeroUsize>,
1636    Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>,
1637    Option<NonZeroI128>, Option<NonZeroIsize>,
1638
1639    // SAFETY: `null` pointer is valid.
1640    //
1641    // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be
1642    // null.
1643    //
1644    // When `Pointee` gets stabilized, we could use
1645    // `T: ?Sized where <T as Pointee>::Metadata: Zeroable`
1646    {<T>} *mut T, {<T>} *const T,
1647
1648    // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be
1649    // zero.
1650    {<T>} *mut [T], {<T>} *const [T], *mut str, *const str,
1651
1652    // SAFETY: `T` is `Zeroable`.
1653    {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>,
1654}
1655
1656macro_rules! impl_tuple_zeroable {
1657    ($(,)?) => {};
1658    ($first:ident, $($t:ident),* $(,)?) => {
1659        // SAFETY: All elements are zeroable and padding can be zero.
1660        unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {}
1661        impl_tuple_zeroable!($($t),* ,);
1662    }
1663}
1664
1665impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);
1666
1667macro_rules! impl_fn_zeroable_option {
1668    ([$($abi:literal),* $(,)?] $args:tt) => {
1669        $(impl_fn_zeroable_option!({extern $abi} $args);)*
1670        $(impl_fn_zeroable_option!({unsafe extern $abi} $args);)*
1671    };
1672    ({$($prefix:tt)*} {$(,)?}) => {};
1673    ({$($prefix:tt)*} {$ret:ident, $($rest:ident),* $(,)?}) => {
1674        // SAFETY: function pointers are part of the option layout optimization:
1675        // <https://doc.rust-lang.org/stable/std/option/index.html#representation>.
1676        unsafe impl<$ret, $($rest),*> ZeroableOption for $($prefix)* fn($($rest),*) -> $ret {}
1677        impl_fn_zeroable_option!({$($prefix)*} {$($rest),*,});
1678    };
1679}
1680
1681impl_fn_zeroable_option!(["Rust", "C"] { A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U });
1682
1683/// This trait allows creating an instance of `Self` which contains exactly one
1684/// [structurally pinned value](https://doc.rust-lang.org/std/pin/index.html#projections-and-structural-pinning).
1685///
1686/// This is useful when using wrapper `struct`s like [`UnsafeCell`] or with new-type `struct`s.
1687///
1688/// # Examples
1689///
1690/// ```
1691/// # use core::cell::UnsafeCell;
1692/// # use pin_init::{pin_data, pin_init, Wrapper};
1693///
1694/// #[pin_data]
1695/// struct Foo {}
1696///
1697/// #[pin_data]
1698/// struct Bar {
1699///     #[pin]
1700///     content: UnsafeCell<Foo>
1701/// };
1702///
1703/// let foo_initializer = pin_init!(Foo{});
1704/// let initializer = pin_init!(Bar {
1705///     content <- UnsafeCell::pin_init(foo_initializer)
1706/// });
1707/// ```
1708pub trait Wrapper<T> {
1709    /// Creates an pin-initializer for a [`Self`] containing `T` from the `value_init` initializer.
1710    fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E>;
1711}
1712
1713impl<T> Wrapper<T> for UnsafeCell<T> {
1714    fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
1715        // SAFETY: `UnsafeCell<T>` has a compatible layout to `T`.
1716        unsafe { cast_pin_init(value_init) }
1717    }
1718}
1719
1720impl<T> Wrapper<T> for MaybeUninit<T> {
1721    fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
1722        // SAFETY: `MaybeUninit<T>` has a compatible layout to `T`.
1723        unsafe { cast_pin_init(value_init) }
1724    }
1725}
1726
1727#[cfg(all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED))]
1728impl<T> Wrapper<T> for core::pin::UnsafePinned<T> {
1729    fn pin_init<E>(init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
1730        // SAFETY: `UnsafePinned<T>` has a compatible layout to `T`.
1731        unsafe { cast_pin_init(init) }
1732    }
1733}