pin_init/macros.rs
1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3//! This module provides the macros that actually implement the proc-macros `pin_data` and
4//! `pinned_drop`. It also contains `__init_internal`, the implementation of the
5//! `{try_}{pin_}init!` macros.
6//!
7//! These macros should never be called directly, since they expect their input to be
8//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in
9//! safe code! Use the public facing macros instead.
10//!
11//! This architecture has been chosen because the kernel does not yet have access to `syn` which
12//! would make matters a lot easier for implementing these as proc-macros.
13//!
14//! Since this library and the kernel implementation should diverge as little as possible, the same
15//! approach has been taken here.
16//!
17//! # Macro expansion example
18//!
19//! This section is intended for readers trying to understand the macros in this module and the
20//! `[try_][pin_]init!` macros from `lib.rs`.
21//!
22//! We will look at the following example:
23//!
24//! ```rust,ignore
25//! #[pin_data]
26//! #[repr(C)]
27//! struct Bar<T> {
28//! #[pin]
29//! t: T,
30//! pub x: usize,
31//! }
32//!
33//! impl<T> Bar<T> {
34//! fn new(t: T) -> impl PinInit<Self> {
35//! pin_init!(Self { t, x: 0 })
36//! }
37//! }
38//!
39//! #[pin_data(PinnedDrop)]
40//! struct Foo {
41//! a: usize,
42//! #[pin]
43//! b: Bar<u32>,
44//! }
45//!
46//! #[pinned_drop]
47//! impl PinnedDrop for Foo {
48//! fn drop(self: Pin<&mut Self>) {
49//! println!("{self:p} is getting dropped.");
50//! }
51//! }
52//!
53//! let a = 42;
54//! let initializer = pin_init!(Foo {
55//! a,
56//! b <- Bar::new(36),
57//! });
58//! ```
59//!
60//! This example includes the most common and important features of the pin-init API.
61//!
62//! Below you can find individual section about the different macro invocations. Here are some
63//! general things we need to take into account when designing macros:
64//! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`
65//! this ensures that the correct item is used, since users could define their own `mod core {}`
66//! and then their own `panic!` inside to execute arbitrary code inside of our macro.
67//! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied
68//! expressions inside of an `unsafe` block in the macro, because this would allow users to do
69//! `unsafe` operations without an associated `unsafe` block.
70//!
71//! ## `#[pin_data]` on `Bar`
72//!
73//! This macro is used to specify which fields are structurally pinned and which fields are not. It
74//! is placed on the struct definition and allows `#[pin]` to be placed on the fields.
75//!
76//! Here is the definition of `Bar` from our example:
77//!
78//! ```rust,ignore
79//! #[pin_data]
80//! #[repr(C)]
81//! struct Bar<T> {
82//! #[pin]
83//! t: T,
84//! pub x: usize,
85//! }
86//! ```
87//!
88//! This expands to the following code:
89//!
90//! ```rust,ignore
91//! // Firstly the normal definition of the struct, attributes are preserved:
92//! #[repr(C)]
93//! struct Bar<T> {
94//! t: T,
95//! pub x: usize,
96//! }
97//! // Then an anonymous constant is defined, this is because we do not want any code to access the
98//! // types that we define inside:
99//! const _: () = {
100//! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,
101//! // since we need to implement access functions for each field and thus need to know its
102//! // type.
103//! struct __ThePinData<T> {
104//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
105//! }
106//! // We implement `Copy` for the pin-data struct, since all functions it defines will take
107//! // `self` by value.
108//! impl<T> ::core::clone::Clone for __ThePinData<T> {
109//! fn clone(&self) -> Self {
110//! *self
111//! }
112//! }
113//! impl<T> ::core::marker::Copy for __ThePinData<T> {}
114//! // For every field of `Bar`, the pin-data struct will define a function with the same name
115//! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the
116//! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field
117//! // (if pinning is structural for the field, then `PinInit` otherwise `Init`).
118//! #[allow(dead_code)]
119//! impl<T> __ThePinData<T> {
120//! unsafe fn t<E>(
121//! self,
122//! slot: *mut T,
123//! // Since `t` is `#[pin]`, this is `PinInit`.
124//! init: impl ::pin_init::PinInit<T, E>,
125//! ) -> ::core::result::Result<(), E> {
126//! unsafe { ::pin_init::PinInit::__pinned_init(init, slot) }
127//! }
128//! pub unsafe fn x<E>(
129//! self,
130//! slot: *mut usize,
131//! // Since `x` is not `#[pin]`, this is `Init`.
132//! init: impl ::pin_init::Init<usize, E>,
133//! ) -> ::core::result::Result<(), E> {
134//! unsafe { ::pin_init::Init::__init(init, slot) }
135//! }
136//! }
137//! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct
138//! // that we constructed above.
139//! unsafe impl<T> ::pin_init::__internal::HasPinData for Bar<T> {
140//! type PinData = __ThePinData<T>;
141//! unsafe fn __pin_data() -> Self::PinData {
142//! __ThePinData {
143//! __phantom: ::core::marker::PhantomData,
144//! }
145//! }
146//! }
147//! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data
148//! // struct. This is important to ensure that no user can implement a rogue `__pin_data`
149//! // function without using `unsafe`.
150//! unsafe impl<T> ::pin_init::__internal::PinData for __ThePinData<T> {
151//! type Datee = Bar<T>;
152//! }
153//! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is
154//! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned
155//! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our
156//! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist
157//! // for two reasons:
158//! // - `__phantom`: every generic must be used, since we cannot really know which generics
159//! // are used, we declare all and then use everything here once.
160//! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant
161//! // over it. The lifetime is needed to work around the limitation that trait bounds must
162//! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is
163//! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler
164//! // into accepting these bounds regardless.
165//! #[allow(dead_code)]
166//! struct __Unpin<'__pin, T> {
167//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
168//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
169//! // Our only `#[pin]` field is `t`.
170//! t: T,
171//! }
172//! #[doc(hidden)]
173//! impl<'__pin, T> ::core::marker::Unpin for Bar<T>
174//! where
175//! __Unpin<'__pin, T>: ::core::marker::Unpin,
176//! {}
177//! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users
178//! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to
179//! // UB with only safe code, so we disallow this by giving a trait implementation error using
180//! // a direct impl and a blanket implementation.
181//! trait MustNotImplDrop {}
182//! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do
183//! // (normally people want to know if a type has any kind of drop glue at all, here we want
184//! // to know if it has any kind of custom drop glue, which is exactly what this bound does).
185//! #[expect(drop_bounds)]
186//! impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
187//! impl<T> MustNotImplDrop for Bar<T> {}
188//! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to
189//! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed
190//! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.
191//! #[expect(non_camel_case_types)]
192//! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
193//! impl<
194//! T: ::pin_init::PinnedDrop,
195//! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
196//! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}
197//! };
198//! ```
199//!
200//! ## `pin_init!` in `impl Bar`
201//!
202//! This macro creates an pin-initializer for the given struct. It requires that the struct is
203//! annotated by `#[pin_data]`.
204//!
205//! Here is the impl on `Bar` defining the new function:
206//!
207//! ```rust,ignore
208//! impl<T> Bar<T> {
209//! fn new(t: T) -> impl PinInit<Self> {
210//! pin_init!(Self { t, x: 0 })
211//! }
212//! }
213//! ```
214//!
215//! This expands to the following code:
216//!
217//! ```rust,ignore
218//! impl<T> Bar<T> {
219//! fn new(t: T) -> impl PinInit<Self> {
220//! {
221//! // We do not want to allow arbitrary returns, so we declare this type as the `Ok`
222//! // return type and shadow it later when we insert the arbitrary user code. That way
223//! // there will be no possibility of returning without `unsafe`.
224//! struct __InitOk;
225//! // Get the data about fields from the supplied type.
226//! // - the function is unsafe, hence the unsafe block
227//! // - we `use` the `HasPinData` trait in the block, it is only available in that
228//! // scope.
229//! let data = unsafe {
230//! use ::pin_init::__internal::HasPinData;
231//! Self::__pin_data()
232//! };
233//! // Ensure that `data` really is of type `PinData` and help with type inference:
234//! let init = ::pin_init::__internal::PinData::make_closure::<
235//! _,
236//! __InitOk,
237//! ::core::convert::Infallible,
238//! >(data, move |slot| {
239//! {
240//! // Shadow the structure so it cannot be used to return early. If a user
241//! // tries to write `return Ok(__InitOk)`, then they get a type error,
242//! // since that will refer to this struct instead of the one defined
243//! // above.
244//! struct __InitOk;
245//! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.
246//! {
247//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };
248//! }
249//! // Since initialization could fail later (not in this case, since the
250//! // error type is `Infallible`) we will need to drop this field if there
251//! // is an error later. This `DropGuard` will drop the field when it gets
252//! // dropped and has not yet been forgotten.
253//! let __t_guard = unsafe {
254//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t))
255//! };
256//! // Expansion of `x: 0,`:
257//! // Since this can be an arbitrary expression we cannot place it inside
258//! // of the `unsafe` block, so we bind it here.
259//! {
260//! let x = 0;
261//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };
262//! }
263//! // We again create a `DropGuard`.
264//! let __x_guard = unsafe {
265//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x))
266//! };
267//! // Since initialization has successfully completed, we can now forget
268//! // the guards. This is not `mem::forget`, since we only have
269//! // `&DropGuard`.
270//! ::core::mem::forget(__x_guard);
271//! ::core::mem::forget(__t_guard);
272//! // Here we use the type checker to ensure that every field has been
273//! // initialized exactly once, since this is `if false` it will never get
274//! // executed, but still type-checked.
275//! // Additionally we abuse `slot` to automatically infer the correct type
276//! // for the struct. This is also another check that every field is
277//! // accessible from this scope.
278//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
279//! let _ = || {
280//! unsafe {
281//! ::core::ptr::write(
282//! slot,
283//! Self {
284//! // We only care about typecheck finding every field
285//! // here, the expression does not matter, just conjure
286//! // one using `panic!()`:
287//! t: ::core::panic!(),
288//! x: ::core::panic!(),
289//! },
290//! );
291//! };
292//! };
293//! }
294//! // We leave the scope above and gain access to the previously shadowed
295//! // `__InitOk` that we need to return.
296//! Ok(__InitOk)
297//! });
298//! // Change the return type from `__InitOk` to `()`.
299//! let init = move |
300//! slot,
301//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
302//! init(slot).map(|__InitOk| ())
303//! };
304//! // Construct the initializer.
305//! let init = unsafe {
306//! ::pin_init::pin_init_from_closure::<
307//! _,
308//! ::core::convert::Infallible,
309//! >(init)
310//! };
311//! init
312//! }
313//! }
314//! }
315//! ```
316//!
317//! ## `#[pin_data]` on `Foo`
318//!
319//! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the
320//! differences/new things in the expansion of the `Foo` definition:
321//!
322//! ```rust,ignore
323//! #[pin_data(PinnedDrop)]
324//! struct Foo {
325//! a: usize,
326//! #[pin]
327//! b: Bar<u32>,
328//! }
329//! ```
330//!
331//! This expands to the following code:
332//!
333//! ```rust,ignore
334//! struct Foo {
335//! a: usize,
336//! b: Bar<u32>,
337//! }
338//! const _: () = {
339//! struct __ThePinData {
340//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
341//! }
342//! impl ::core::clone::Clone for __ThePinData {
343//! fn clone(&self) -> Self {
344//! *self
345//! }
346//! }
347//! impl ::core::marker::Copy for __ThePinData {}
348//! #[allow(dead_code)]
349//! impl __ThePinData {
350//! unsafe fn b<E>(
351//! self,
352//! slot: *mut Bar<u32>,
353//! init: impl ::pin_init::PinInit<Bar<u32>, E>,
354//! ) -> ::core::result::Result<(), E> {
355//! unsafe { ::pin_init::PinInit::__pinned_init(init, slot) }
356//! }
357//! unsafe fn a<E>(
358//! self,
359//! slot: *mut usize,
360//! init: impl ::pin_init::Init<usize, E>,
361//! ) -> ::core::result::Result<(), E> {
362//! unsafe { ::pin_init::Init::__init(init, slot) }
363//! }
364//! }
365//! unsafe impl ::pin_init::__internal::HasPinData for Foo {
366//! type PinData = __ThePinData;
367//! unsafe fn __pin_data() -> Self::PinData {
368//! __ThePinData {
369//! __phantom: ::core::marker::PhantomData,
370//! }
371//! }
372//! }
373//! unsafe impl ::pin_init::__internal::PinData for __ThePinData {
374//! type Datee = Foo;
375//! }
376//! #[allow(dead_code)]
377//! struct __Unpin<'__pin> {
378//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
379//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
380//! b: Bar<u32>,
381//! }
382//! #[doc(hidden)]
383//! impl<'__pin> ::core::marker::Unpin for Foo
384//! where
385//! __Unpin<'__pin>: ::core::marker::Unpin,
386//! {}
387//! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to
388//! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like
389//! // before, instead we implement `Drop` here and delegate to `PinnedDrop`.
390//! impl ::core::ops::Drop for Foo {
391//! fn drop(&mut self) {
392//! // Since we are getting dropped, no one else has a reference to `self` and thus we
393//! // can assume that we never move.
394//! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
395//! // Create the unsafe token that proves that we are inside of a destructor, this
396//! // type is only allowed to be created in a destructor.
397//! let token = unsafe { ::pin_init::__internal::OnlyCallFromDrop::new() };
398//! ::pin_init::PinnedDrop::drop(pinned, token);
399//! }
400//! }
401//! };
402//! ```
403//!
404//! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`
405//!
406//! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an
407//! extra parameter that should not be used at all. The macro hides that parameter.
408//!
409//! Here is the `PinnedDrop` impl for `Foo`:
410//!
411//! ```rust,ignore
412//! #[pinned_drop]
413//! impl PinnedDrop for Foo {
414//! fn drop(self: Pin<&mut Self>) {
415//! println!("{self:p} is getting dropped.");
416//! }
417//! }
418//! ```
419//!
420//! This expands to the following code:
421//!
422//! ```rust,ignore
423//! // `unsafe`, full path and the token parameter are added, everything else stays the same.
424//! unsafe impl ::pin_init::PinnedDrop for Foo {
425//! fn drop(self: Pin<&mut Self>, _: ::pin_init::__internal::OnlyCallFromDrop) {
426//! println!("{self:p} is getting dropped.");
427//! }
428//! }
429//! ```
430//!
431//! ## `pin_init!` on `Foo`
432//!
433//! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion
434//! of `pin_init!` on `Foo`:
435//!
436//! ```rust,ignore
437//! let a = 42;
438//! let initializer = pin_init!(Foo {
439//! a,
440//! b <- Bar::new(36),
441//! });
442//! ```
443//!
444//! This expands to the following code:
445//!
446//! ```rust,ignore
447//! let a = 42;
448//! let initializer = {
449//! struct __InitOk;
450//! let data = unsafe {
451//! use ::pin_init::__internal::HasPinData;
452//! Foo::__pin_data()
453//! };
454//! let init = ::pin_init::__internal::PinData::make_closure::<
455//! _,
456//! __InitOk,
457//! ::core::convert::Infallible,
458//! >(data, move |slot| {
459//! {
460//! struct __InitOk;
461//! {
462//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };
463//! }
464//! let __a_guard = unsafe {
465//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))
466//! };
467//! let init = Bar::new(36);
468//! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };
469//! let __b_guard = unsafe {
470//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))
471//! };
472//! ::core::mem::forget(__b_guard);
473//! ::core::mem::forget(__a_guard);
474//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
475//! let _ = || {
476//! unsafe {
477//! ::core::ptr::write(
478//! slot,
479//! Foo {
480//! a: ::core::panic!(),
481//! b: ::core::panic!(),
482//! },
483//! );
484//! };
485//! };
486//! }
487//! Ok(__InitOk)
488//! });
489//! let init = move |
490//! slot,
491//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
492//! init(slot).map(|__InitOk| ())
493//! };
494//! let init = unsafe {
495//! ::pin_init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
496//! };
497//! init
498//! };
499//! ```
500
501#[cfg(kernel)]
502pub use ::macros::paste;
503#[cfg(not(kernel))]
504pub use ::paste::paste;
505
506/// Creates a `unsafe impl<...> PinnedDrop for $type` block.
507///
508/// See [`PinnedDrop`] for more information.
509#[doc(hidden)]
510#[macro_export]
511macro_rules! __pinned_drop {
512 (
513 @impl_sig($($impl_sig:tt)*),
514 @impl_body(
515 $(#[$($attr:tt)*])*
516 fn drop($($sig:tt)*) {
517 $($inner:tt)*
518 }
519 ),
520 ) => {
521 // SAFETY: TODO.
522 unsafe $($impl_sig)* {
523 // Inherit all attributes and the type/ident tokens for the signature.
524 $(#[$($attr)*])*
525 fn drop($($sig)*, _: $crate::__internal::OnlyCallFromDrop) {
526 $($inner)*
527 }
528 }
529 }
530}
531
532/// This macro first parses the struct definition such that it separates pinned and not pinned
533/// fields. Afterwards it declares the struct and implement the `PinData` trait safely.
534#[doc(hidden)]
535#[macro_export]
536macro_rules! __pin_data {
537 // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.
538 (parse_input:
539 @args($($pinned_drop:ident)?),
540 @sig(
541 $(#[$($struct_attr:tt)*])*
542 $vis:vis struct $name:ident
543 $(where $($whr:tt)*)?
544 ),
545 @impl_generics($($impl_generics:tt)*),
546 @ty_generics($($ty_generics:tt)*),
547 @decl_generics($($decl_generics:tt)*),
548 @body({ $($fields:tt)* }),
549 ) => {
550 // We now use token munching to iterate through all of the fields. While doing this we
551 // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user
552 // wants these to be structurally pinned. The rest of the fields are the
553 // 'not pinned fields'. Additionally we collect all fields, since we need them in the right
554 // order to declare the struct.
555 //
556 // In this call we also put some explaining comments for the parameters.
557 $crate::__pin_data!(find_pinned_fields:
558 // Attributes on the struct itself, these will just be propagated to be put onto the
559 // struct definition.
560 @struct_attrs($(#[$($struct_attr)*])*),
561 // The visibility of the struct.
562 @vis($vis),
563 // The name of the struct.
564 @name($name),
565 // The 'impl generics', the generics that will need to be specified on the struct inside
566 // of an `impl<$ty_generics>` block.
567 @impl_generics($($impl_generics)*),
568 // The 'ty generics', the generics that will need to be specified on the impl blocks.
569 @ty_generics($($ty_generics)*),
570 // The 'decl generics', the generics that need to be specified on the struct
571 // definition.
572 @decl_generics($($decl_generics)*),
573 // The where clause of any impl block and the declaration.
574 @where($($($whr)*)?),
575 // The remaining fields tokens that need to be processed.
576 // We add a `,` at the end to ensure correct parsing.
577 @fields_munch($($fields)* ,),
578 // The pinned fields.
579 @pinned(),
580 // The not pinned fields.
581 @not_pinned(),
582 // All fields.
583 @fields(),
584 // The accumulator containing all attributes already parsed.
585 @accum(),
586 // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.
587 @is_pinned(),
588 // The proc-macro argument, this should be `PinnedDrop` or ``.
589 @pinned_drop($($pinned_drop)?),
590 );
591 };
592 (find_pinned_fields:
593 @struct_attrs($($struct_attrs:tt)*),
594 @vis($vis:vis),
595 @name($name:ident),
596 @impl_generics($($impl_generics:tt)*),
597 @ty_generics($($ty_generics:tt)*),
598 @decl_generics($($decl_generics:tt)*),
599 @where($($whr:tt)*),
600 // We found a PhantomPinned field, this should generally be pinned!
601 @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),
602 @pinned($($pinned:tt)*),
603 @not_pinned($($not_pinned:tt)*),
604 @fields($($fields:tt)*),
605 @accum($($accum:tt)*),
606 // This field is not pinned.
607 @is_pinned(),
608 @pinned_drop($($pinned_drop:ident)?),
609 ) => {
610 ::core::compile_error!(concat!(
611 "The field `",
612 stringify!($field),
613 "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",
614 ));
615 $crate::__pin_data!(find_pinned_fields:
616 @struct_attrs($($struct_attrs)*),
617 @vis($vis),
618 @name($name),
619 @impl_generics($($impl_generics)*),
620 @ty_generics($($ty_generics)*),
621 @decl_generics($($decl_generics)*),
622 @where($($whr)*),
623 @fields_munch($($rest)*),
624 @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),
625 @not_pinned($($not_pinned)*),
626 @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),
627 @accum(),
628 @is_pinned(),
629 @pinned_drop($($pinned_drop)?),
630 );
631 };
632 (find_pinned_fields:
633 @struct_attrs($($struct_attrs:tt)*),
634 @vis($vis:vis),
635 @name($name:ident),
636 @impl_generics($($impl_generics:tt)*),
637 @ty_generics($($ty_generics:tt)*),
638 @decl_generics($($decl_generics:tt)*),
639 @where($($whr:tt)*),
640 // We reached the field declaration.
641 @fields_munch($field:ident : $type:ty, $($rest:tt)*),
642 @pinned($($pinned:tt)*),
643 @not_pinned($($not_pinned:tt)*),
644 @fields($($fields:tt)*),
645 @accum($($accum:tt)*),
646 // This field is pinned.
647 @is_pinned(yes),
648 @pinned_drop($($pinned_drop:ident)?),
649 ) => {
650 $crate::__pin_data!(find_pinned_fields:
651 @struct_attrs($($struct_attrs)*),
652 @vis($vis),
653 @name($name),
654 @impl_generics($($impl_generics)*),
655 @ty_generics($($ty_generics)*),
656 @decl_generics($($decl_generics)*),
657 @where($($whr)*),
658 @fields_munch($($rest)*),
659 @pinned($($pinned)* $($accum)* $field: $type,),
660 @not_pinned($($not_pinned)*),
661 @fields($($fields)* $($accum)* $field: $type,),
662 @accum(),
663 @is_pinned(),
664 @pinned_drop($($pinned_drop)?),
665 );
666 };
667 (find_pinned_fields:
668 @struct_attrs($($struct_attrs:tt)*),
669 @vis($vis:vis),
670 @name($name:ident),
671 @impl_generics($($impl_generics:tt)*),
672 @ty_generics($($ty_generics:tt)*),
673 @decl_generics($($decl_generics:tt)*),
674 @where($($whr:tt)*),
675 // We reached the field declaration.
676 @fields_munch($field:ident : $type:ty, $($rest:tt)*),
677 @pinned($($pinned:tt)*),
678 @not_pinned($($not_pinned:tt)*),
679 @fields($($fields:tt)*),
680 @accum($($accum:tt)*),
681 // This field is not pinned.
682 @is_pinned(),
683 @pinned_drop($($pinned_drop:ident)?),
684 ) => {
685 $crate::__pin_data!(find_pinned_fields:
686 @struct_attrs($($struct_attrs)*),
687 @vis($vis),
688 @name($name),
689 @impl_generics($($impl_generics)*),
690 @ty_generics($($ty_generics)*),
691 @decl_generics($($decl_generics)*),
692 @where($($whr)*),
693 @fields_munch($($rest)*),
694 @pinned($($pinned)*),
695 @not_pinned($($not_pinned)* $($accum)* $field: $type,),
696 @fields($($fields)* $($accum)* $field: $type,),
697 @accum(),
698 @is_pinned(),
699 @pinned_drop($($pinned_drop)?),
700 );
701 };
702 (find_pinned_fields:
703 @struct_attrs($($struct_attrs:tt)*),
704 @vis($vis:vis),
705 @name($name:ident),
706 @impl_generics($($impl_generics:tt)*),
707 @ty_generics($($ty_generics:tt)*),
708 @decl_generics($($decl_generics:tt)*),
709 @where($($whr:tt)*),
710 // We found the `#[pin]` attr.
711 @fields_munch(#[pin] $($rest:tt)*),
712 @pinned($($pinned:tt)*),
713 @not_pinned($($not_pinned:tt)*),
714 @fields($($fields:tt)*),
715 @accum($($accum:tt)*),
716 @is_pinned($($is_pinned:ident)?),
717 @pinned_drop($($pinned_drop:ident)?),
718 ) => {
719 $crate::__pin_data!(find_pinned_fields:
720 @struct_attrs($($struct_attrs)*),
721 @vis($vis),
722 @name($name),
723 @impl_generics($($impl_generics)*),
724 @ty_generics($($ty_generics)*),
725 @decl_generics($($decl_generics)*),
726 @where($($whr)*),
727 @fields_munch($($rest)*),
728 // We do not include `#[pin]` in the list of attributes, since it is not actually an
729 // attribute that is defined somewhere.
730 @pinned($($pinned)*),
731 @not_pinned($($not_pinned)*),
732 @fields($($fields)*),
733 @accum($($accum)*),
734 // Set this to `yes`.
735 @is_pinned(yes),
736 @pinned_drop($($pinned_drop)?),
737 );
738 };
739 (find_pinned_fields:
740 @struct_attrs($($struct_attrs:tt)*),
741 @vis($vis:vis),
742 @name($name:ident),
743 @impl_generics($($impl_generics:tt)*),
744 @ty_generics($($ty_generics:tt)*),
745 @decl_generics($($decl_generics:tt)*),
746 @where($($whr:tt)*),
747 // We reached the field declaration with visibility, for simplicity we only munch the
748 // visibility and put it into `$accum`.
749 @fields_munch($fvis:vis $field:ident $($rest:tt)*),
750 @pinned($($pinned:tt)*),
751 @not_pinned($($not_pinned:tt)*),
752 @fields($($fields:tt)*),
753 @accum($($accum:tt)*),
754 @is_pinned($($is_pinned:ident)?),
755 @pinned_drop($($pinned_drop:ident)?),
756 ) => {
757 $crate::__pin_data!(find_pinned_fields:
758 @struct_attrs($($struct_attrs)*),
759 @vis($vis),
760 @name($name),
761 @impl_generics($($impl_generics)*),
762 @ty_generics($($ty_generics)*),
763 @decl_generics($($decl_generics)*),
764 @where($($whr)*),
765 @fields_munch($field $($rest)*),
766 @pinned($($pinned)*),
767 @not_pinned($($not_pinned)*),
768 @fields($($fields)*),
769 @accum($($accum)* $fvis),
770 @is_pinned($($is_pinned)?),
771 @pinned_drop($($pinned_drop)?),
772 );
773 };
774 (find_pinned_fields:
775 @struct_attrs($($struct_attrs:tt)*),
776 @vis($vis:vis),
777 @name($name:ident),
778 @impl_generics($($impl_generics:tt)*),
779 @ty_generics($($ty_generics:tt)*),
780 @decl_generics($($decl_generics:tt)*),
781 @where($($whr:tt)*),
782 // Some other attribute, just put it into `$accum`.
783 @fields_munch(#[$($attr:tt)*] $($rest:tt)*),
784 @pinned($($pinned:tt)*),
785 @not_pinned($($not_pinned:tt)*),
786 @fields($($fields:tt)*),
787 @accum($($accum:tt)*),
788 @is_pinned($($is_pinned:ident)?),
789 @pinned_drop($($pinned_drop:ident)?),
790 ) => {
791 $crate::__pin_data!(find_pinned_fields:
792 @struct_attrs($($struct_attrs)*),
793 @vis($vis),
794 @name($name),
795 @impl_generics($($impl_generics)*),
796 @ty_generics($($ty_generics)*),
797 @decl_generics($($decl_generics)*),
798 @where($($whr)*),
799 @fields_munch($($rest)*),
800 @pinned($($pinned)*),
801 @not_pinned($($not_pinned)*),
802 @fields($($fields)*),
803 @accum($($accum)* #[$($attr)*]),
804 @is_pinned($($is_pinned)?),
805 @pinned_drop($($pinned_drop)?),
806 );
807 };
808 (find_pinned_fields:
809 @struct_attrs($($struct_attrs:tt)*),
810 @vis($vis:vis),
811 @name($name:ident),
812 @impl_generics($($impl_generics:tt)*),
813 @ty_generics($($ty_generics:tt)*),
814 @decl_generics($($decl_generics:tt)*),
815 @where($($whr:tt)*),
816 // We reached the end of the fields, plus an optional additional comma, since we added one
817 // before and the user is also allowed to put a trailing comma.
818 @fields_munch($(,)?),
819 @pinned($($pinned:tt)*),
820 @not_pinned($($not_pinned:tt)*),
821 @fields($($fields:tt)*),
822 @accum(),
823 @is_pinned(),
824 @pinned_drop($($pinned_drop:ident)?),
825 ) => {
826 // Declare the struct with all fields in the correct order.
827 $($struct_attrs)*
828 $vis struct $name <$($decl_generics)*>
829 where $($whr)*
830 {
831 $($fields)*
832 }
833
834 $crate::__pin_data!(make_pin_projections:
835 @vis($vis),
836 @name($name),
837 @impl_generics($($impl_generics)*),
838 @ty_generics($($ty_generics)*),
839 @decl_generics($($decl_generics)*),
840 @where($($whr)*),
841 @pinned($($pinned)*),
842 @not_pinned($($not_pinned)*),
843 );
844
845 // We put the rest into this const item, because it then will not be accessible to anything
846 // outside.
847 const _: () = {
848 // We declare this struct which will host all of the projection function for our type.
849 // it will be invariant over all generic parameters which are inherited from the
850 // struct.
851 $vis struct __ThePinData<$($impl_generics)*>
852 where $($whr)*
853 {
854 __phantom: ::core::marker::PhantomData<
855 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
856 >,
857 }
858
859 impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>
860 where $($whr)*
861 {
862 fn clone(&self) -> Self { *self }
863 }
864
865 impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>
866 where $($whr)*
867 {}
868
869 // Make all projection functions.
870 $crate::__pin_data!(make_pin_data:
871 @pin_data(__ThePinData),
872 @impl_generics($($impl_generics)*),
873 @ty_generics($($ty_generics)*),
874 @where($($whr)*),
875 @pinned($($pinned)*),
876 @not_pinned($($not_pinned)*),
877 );
878
879 // SAFETY: We have added the correct projection functions above to `__ThePinData` and
880 // we also use the least restrictive generics possible.
881 unsafe impl<$($impl_generics)*>
882 $crate::__internal::HasPinData for $name<$($ty_generics)*>
883 where $($whr)*
884 {
885 type PinData = __ThePinData<$($ty_generics)*>;
886
887 unsafe fn __pin_data() -> Self::PinData {
888 __ThePinData { __phantom: ::core::marker::PhantomData }
889 }
890 }
891
892 // SAFETY: TODO.
893 unsafe impl<$($impl_generics)*>
894 $crate::__internal::PinData for __ThePinData<$($ty_generics)*>
895 where $($whr)*
896 {
897 type Datee = $name<$($ty_generics)*>;
898 }
899
900 // This struct will be used for the unpin analysis. Since only structurally pinned
901 // fields are relevant whether the struct should implement `Unpin`.
902 #[allow(dead_code)]
903 struct __Unpin <'__pin, $($impl_generics)*>
904 where $($whr)*
905 {
906 __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
907 __phantom: ::core::marker::PhantomData<
908 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
909 >,
910 // Only the pinned fields.
911 $($pinned)*
912 }
913
914 #[doc(hidden)]
915 impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>
916 where
917 __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,
918 $($whr)*
919 {}
920
921 // We need to disallow normal `Drop` implementation, the exact behavior depends on
922 // whether `PinnedDrop` was specified as the parameter.
923 $crate::__pin_data!(drop_prevention:
924 @name($name),
925 @impl_generics($($impl_generics)*),
926 @ty_generics($($ty_generics)*),
927 @where($($whr)*),
928 @pinned_drop($($pinned_drop)?),
929 );
930 };
931 };
932 // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
933 (drop_prevention:
934 @name($name:ident),
935 @impl_generics($($impl_generics:tt)*),
936 @ty_generics($($ty_generics:tt)*),
937 @where($($whr:tt)*),
938 @pinned_drop(),
939 ) => {
940 // We prevent this by creating a trait that will be implemented for all types implementing
941 // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
942 // if it also implements `Drop`
943 trait MustNotImplDrop {}
944 #[expect(drop_bounds)]
945 impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
946 impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>
947 where $($whr)* {}
948 // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
949 // They might implement `PinnedDrop` correctly for the struct, but forget to give
950 // `PinnedDrop` as the parameter to `#[pin_data]`.
951 #[expect(non_camel_case_types)]
952 trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
953 impl<T: $crate::PinnedDrop>
954 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
955 impl<$($impl_generics)*>
956 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>
957 where $($whr)* {}
958 };
959 // When `PinnedDrop` was specified we just implement `Drop` and delegate.
960 (drop_prevention:
961 @name($name:ident),
962 @impl_generics($($impl_generics:tt)*),
963 @ty_generics($($ty_generics:tt)*),
964 @where($($whr:tt)*),
965 @pinned_drop(PinnedDrop),
966 ) => {
967 impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>
968 where $($whr)*
969 {
970 fn drop(&mut self) {
971 // SAFETY: Since this is a destructor, `self` will not move after this function
972 // terminates, since it is inaccessible.
973 let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
974 // SAFETY: Since this is a drop function, we can create this token to call the
975 // pinned destructor of this type.
976 let token = unsafe { $crate::__internal::OnlyCallFromDrop::new() };
977 $crate::PinnedDrop::drop(pinned, token);
978 }
979 }
980 };
981 // If some other parameter was specified, we emit a readable error.
982 (drop_prevention:
983 @name($name:ident),
984 @impl_generics($($impl_generics:tt)*),
985 @ty_generics($($ty_generics:tt)*),
986 @where($($whr:tt)*),
987 @pinned_drop($($rest:tt)*),
988 ) => {
989 compile_error!(
990 "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",
991 stringify!($($rest)*),
992 );
993 };
994 (make_pin_projections:
995 @vis($vis:vis),
996 @name($name:ident),
997 @impl_generics($($impl_generics:tt)*),
998 @ty_generics($($ty_generics:tt)*),
999 @decl_generics($($decl_generics:tt)*),
1000 @where($($whr:tt)*),
1001 @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
1002 @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
1003 ) => {
1004 $crate::macros::paste! {
1005 #[doc(hidden)]
1006 $vis struct [< $name Projection >] <'__pin, $($decl_generics)*> {
1007 $($(#[$($p_attr)*])* $pvis $p_field : ::core::pin::Pin<&'__pin mut $p_type>,)*
1008 $($(#[$($attr)*])* $fvis $field : &'__pin mut $type,)*
1009 ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>,
1010 }
1011
1012 impl<$($impl_generics)*> $name<$($ty_generics)*>
1013 where $($whr)*
1014 {
1015 /// Pin-projects all fields of `Self`.
1016 ///
1017 /// These fields are structurally pinned:
1018 $(#[doc = ::core::concat!(" - `", ::core::stringify!($p_field), "`")])*
1019 ///
1020 /// These fields are **not** structurally pinned:
1021 $(#[doc = ::core::concat!(" - `", ::core::stringify!($field), "`")])*
1022 #[inline]
1023 $vis fn project<'__pin>(
1024 self: ::core::pin::Pin<&'__pin mut Self>,
1025 ) -> [< $name Projection >] <'__pin, $($ty_generics)*> {
1026 // SAFETY: we only give access to `&mut` for fields not structurally pinned.
1027 let this = unsafe { ::core::pin::Pin::get_unchecked_mut(self) };
1028 [< $name Projection >] {
1029 $(
1030 // SAFETY: `$p_field` is structurally pinned.
1031 $(#[$($p_attr)*])*
1032 $p_field : unsafe { ::core::pin::Pin::new_unchecked(&mut this.$p_field) },
1033 )*
1034 $(
1035 $(#[$($attr)*])*
1036 $field : &mut this.$field,
1037 )*
1038 ___pin_phantom_data: ::core::marker::PhantomData,
1039 }
1040 }
1041 }
1042 }
1043 };
1044 (make_pin_data:
1045 @pin_data($pin_data:ident),
1046 @impl_generics($($impl_generics:tt)*),
1047 @ty_generics($($ty_generics:tt)*),
1048 @where($($whr:tt)*),
1049 @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
1050 @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
1051 ) => {
1052 $crate::macros::paste! {
1053 // For every field, we create a projection function according to its projection type. If a
1054 // field is structurally pinned, then it must be initialized via `PinInit`, if it is not
1055 // structurally pinned, then it can be initialized via `Init`.
1056 //
1057 // The functions are `unsafe` to prevent accidentally calling them.
1058 #[allow(dead_code)]
1059 #[expect(clippy::missing_safety_doc)]
1060 impl<$($impl_generics)*> $pin_data<$($ty_generics)*>
1061 where $($whr)*
1062 {
1063 $(
1064 $(#[$($p_attr)*])*
1065 $pvis unsafe fn $p_field<E>(
1066 self,
1067 slot: *mut $p_type,
1068 init: impl $crate::PinInit<$p_type, E>,
1069 ) -> ::core::result::Result<(), E> {
1070 // SAFETY: TODO.
1071 unsafe { $crate::PinInit::__pinned_init(init, slot) }
1072 }
1073
1074 $(#[$($p_attr)*])*
1075 $pvis unsafe fn [<__project_ $p_field>]<'__slot>(
1076 self,
1077 slot: &'__slot mut $p_type,
1078 ) -> ::core::pin::Pin<&'__slot mut $p_type> {
1079 ::core::pin::Pin::new_unchecked(slot)
1080 }
1081 )*
1082 $(
1083 $(#[$($attr)*])*
1084 $fvis unsafe fn $field<E>(
1085 self,
1086 slot: *mut $type,
1087 init: impl $crate::Init<$type, E>,
1088 ) -> ::core::result::Result<(), E> {
1089 // SAFETY: TODO.
1090 unsafe { $crate::Init::__init(init, slot) }
1091 }
1092
1093 $(#[$($attr)*])*
1094 $fvis unsafe fn [<__project_ $field>]<'__slot>(
1095 self,
1096 slot: &'__slot mut $type,
1097 ) -> &'__slot mut $type {
1098 slot
1099 }
1100 )*
1101 }
1102 }
1103 };
1104}
1105
1106/// The internal init macro. Do not call manually!
1107///
1108/// This is called by the `{try_}{pin_}init!` macros with various inputs.
1109///
1110/// This macro has multiple internal call configurations, these are always the very first ident:
1111/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros.
1112/// - `with_update_parsed`: when the `..Zeroable::init_zeroed()` syntax has been handled.
1113/// - `init_slot`: recursively creates the code that initializes all fields in `slot`.
1114/// - `make_initializer`: recursively create the struct initializer that guarantees that every
1115/// field has been initialized exactly once.
1116#[doc(hidden)]
1117#[macro_export]
1118macro_rules! __init_internal {
1119 (
1120 @this($($this:ident)?),
1121 @typ($t:path),
1122 @fields($($fields:tt)*),
1123 @error($err:ty),
1124 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1125 // case.
1126 @data($data:ident, $($use_data:ident)?),
1127 // `HasPinData` or `HasInitData`.
1128 @has_data($has_data:ident, $get_data:ident),
1129 // `pin_init_from_closure` or `init_from_closure`.
1130 @construct_closure($construct_closure:ident),
1131 @munch_fields(),
1132 ) => {
1133 $crate::__init_internal!(with_update_parsed:
1134 @this($($this)?),
1135 @typ($t),
1136 @fields($($fields)*),
1137 @error($err),
1138 @data($data, $($use_data)?),
1139 @has_data($has_data, $get_data),
1140 @construct_closure($construct_closure),
1141 @init_zeroed(), // Nothing means default behavior.
1142 )
1143 };
1144 (
1145 @this($($this:ident)?),
1146 @typ($t:path),
1147 @fields($($fields:tt)*),
1148 @error($err:ty),
1149 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1150 // case.
1151 @data($data:ident, $($use_data:ident)?),
1152 // `HasPinData` or `HasInitData`.
1153 @has_data($has_data:ident, $get_data:ident),
1154 // `pin_init_from_closure` or `init_from_closure`.
1155 @construct_closure($construct_closure:ident),
1156 @munch_fields(..Zeroable::init_zeroed()),
1157 ) => {
1158 $crate::__init_internal!(with_update_parsed:
1159 @this($($this)?),
1160 @typ($t),
1161 @fields($($fields)*),
1162 @error($err),
1163 @data($data, $($use_data)?),
1164 @has_data($has_data, $get_data),
1165 @construct_closure($construct_closure),
1166 @init_zeroed(()), // `()` means zero all fields not mentioned.
1167 )
1168 };
1169 (
1170 @this($($this:ident)?),
1171 @typ($t:path),
1172 @fields($($fields:tt)*),
1173 @error($err:ty),
1174 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1175 // case.
1176 @data($data:ident, $($use_data:ident)?),
1177 // `HasPinData` or `HasInitData`.
1178 @has_data($has_data:ident, $get_data:ident),
1179 // `pin_init_from_closure` or `init_from_closure`.
1180 @construct_closure($construct_closure:ident),
1181 @munch_fields($ignore:tt $($rest:tt)*),
1182 ) => {
1183 $crate::__init_internal!(
1184 @this($($this)?),
1185 @typ($t),
1186 @fields($($fields)*),
1187 @error($err),
1188 @data($data, $($use_data)?),
1189 @has_data($has_data, $get_data),
1190 @construct_closure($construct_closure),
1191 @munch_fields($($rest)*),
1192 )
1193 };
1194 (with_update_parsed:
1195 @this($($this:ident)?),
1196 @typ($t:path),
1197 @fields($($fields:tt)*),
1198 @error($err:ty),
1199 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1200 // case.
1201 @data($data:ident, $($use_data:ident)?),
1202 // `HasPinData` or `HasInitData`.
1203 @has_data($has_data:ident, $get_data:ident),
1204 // `pin_init_from_closure` or `init_from_closure`.
1205 @construct_closure($construct_closure:ident),
1206 @init_zeroed($($init_zeroed:expr)?),
1207 ) => {{
1208 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
1209 // type and shadow it later when we insert the arbitrary user code. That way there will be
1210 // no possibility of returning without `unsafe`.
1211 struct __InitOk;
1212 // Get the data about fields from the supplied type.
1213 //
1214 // SAFETY: TODO.
1215 let data = unsafe {
1216 use $crate::__internal::$has_data;
1217 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1218 // information that is associated to already parsed fragments, so a path fragment
1219 // cannot be used in this position. Doing the retokenization results in valid rust
1220 // code.
1221 $crate::macros::paste!($t::$get_data())
1222 };
1223 // Ensure that `data` really is of type `$data` and help with type inference:
1224 let init = $crate::__internal::$data::make_closure::<_, __InitOk, $err>(
1225 data,
1226 move |slot| {
1227 {
1228 // Shadow the structure so it cannot be used to return early.
1229 struct __InitOk;
1230 // If `$init_zeroed` is present we should zero the slot now and not emit an
1231 // error when fields are missing (since they will be zeroed). We also have to
1232 // check that the type actually implements `Zeroable`.
1233 $({
1234 fn assert_zeroable<T: $crate::Zeroable>(_: *mut T) {}
1235 // Ensure that the struct is indeed `Zeroable`.
1236 assert_zeroable(slot);
1237 // SAFETY: The type implements `Zeroable` by the check above.
1238 unsafe { ::core::ptr::write_bytes(slot, 0, 1) };
1239 $init_zeroed // This will be `()` if set.
1240 })?
1241 // Create the `this` so it can be referenced by the user inside of the
1242 // expressions creating the individual fields.
1243 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
1244 // Initialize every field.
1245 $crate::__init_internal!(init_slot($($use_data)?):
1246 @data(data),
1247 @slot(slot),
1248 @guards(),
1249 @munch_fields($($fields)*,),
1250 );
1251 // We use unreachable code to ensure that all fields have been mentioned exactly
1252 // once, this struct initializer will still be type-checked and complain with a
1253 // very natural error message if a field is forgotten/mentioned more than once.
1254 #[allow(unreachable_code, clippy::diverging_sub_expression)]
1255 let _ = || {
1256 $crate::__init_internal!(make_initializer:
1257 @slot(slot),
1258 @type_name($t),
1259 @munch_fields($($fields)*,),
1260 @acc(),
1261 );
1262 };
1263 }
1264 Ok(__InitOk)
1265 }
1266 );
1267 let init = move |slot| -> ::core::result::Result<(), $err> {
1268 init(slot).map(|__InitOk| ())
1269 };
1270 // SAFETY: TODO.
1271 let init = unsafe { $crate::$construct_closure::<_, $err>(init) };
1272 init
1273 }};
1274 (init_slot($($use_data:ident)?):
1275 @data($data:ident),
1276 @slot($slot:ident),
1277 @guards($($guards:ident,)*),
1278 @munch_fields($(..Zeroable::init_zeroed())? $(,)?),
1279 ) => {
1280 // Endpoint of munching, no fields are left. If execution reaches this point, all fields
1281 // have been initialized. Therefore we can now dismiss the guards by forgetting them.
1282 $(::core::mem::forget($guards);)*
1283 };
1284 (init_slot($($use_data:ident)?):
1285 @data($data:ident),
1286 @slot($slot:ident),
1287 @guards($($guards:ident,)*),
1288 // arbitrary code block
1289 @munch_fields(_: { $($code:tt)* }, $($rest:tt)*),
1290 ) => {
1291 { $($code)* }
1292 $crate::__init_internal!(init_slot($($use_data)?):
1293 @data($data),
1294 @slot($slot),
1295 @guards($($guards,)*),
1296 @munch_fields($($rest)*),
1297 );
1298 };
1299 (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields.
1300 @data($data:ident),
1301 @slot($slot:ident),
1302 @guards($($guards:ident,)*),
1303 // In-place initialization syntax.
1304 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1305 ) => {
1306 let init = $val;
1307 // Call the initializer.
1308 //
1309 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1310 // return when an error/panic occurs.
1311 // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
1312 unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? };
1313 // SAFETY:
1314 // - the project function does the correct field projection,
1315 // - the field has been initialized,
1316 // - the reference is only valid until the end of the initializer.
1317 #[allow(unused_variables)]
1318 let $field = $crate::macros::paste!(unsafe { $data.[< __project_ $field >](&mut (*$slot).$field) });
1319
1320 // Create the drop guard:
1321 //
1322 // We rely on macro hygiene to make it impossible for users to access this local variable.
1323 // We use `paste!` to create new hygiene for `$field`.
1324 $crate::macros::paste! {
1325 // SAFETY: We forget the guard later when initialization has succeeded.
1326 let [< __ $field _guard >] = unsafe {
1327 $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1328 };
1329
1330 $crate::__init_internal!(init_slot($use_data):
1331 @data($data),
1332 @slot($slot),
1333 @guards([< __ $field _guard >], $($guards,)*),
1334 @munch_fields($($rest)*),
1335 );
1336 }
1337 };
1338 (init_slot(): // No `use_data`, so we use `Init::__init` directly.
1339 @data($data:ident),
1340 @slot($slot:ident),
1341 @guards($($guards:ident,)*),
1342 // In-place initialization syntax.
1343 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1344 ) => {
1345 let init = $val;
1346 // Call the initializer.
1347 //
1348 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1349 // return when an error/panic occurs.
1350 unsafe { $crate::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? };
1351
1352 // SAFETY:
1353 // - the field is not structurally pinned, since the line above must compile,
1354 // - the field has been initialized,
1355 // - the reference is only valid until the end of the initializer.
1356 #[allow(unused_variables)]
1357 let $field = unsafe { &mut (*$slot).$field };
1358
1359 // Create the drop guard:
1360 //
1361 // We rely on macro hygiene to make it impossible for users to access this local variable.
1362 // We use `paste!` to create new hygiene for `$field`.
1363 $crate::macros::paste! {
1364 // SAFETY: We forget the guard later when initialization has succeeded.
1365 let [< __ $field _guard >] = unsafe {
1366 $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1367 };
1368
1369 $crate::__init_internal!(init_slot():
1370 @data($data),
1371 @slot($slot),
1372 @guards([< __ $field _guard >], $($guards,)*),
1373 @munch_fields($($rest)*),
1374 );
1375 }
1376 };
1377 (init_slot(): // No `use_data`, so all fields are not structurally pinned
1378 @data($data:ident),
1379 @slot($slot:ident),
1380 @guards($($guards:ident,)*),
1381 // Init by-value.
1382 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1383 ) => {
1384 {
1385 $(let $field = $val;)?
1386 // Initialize the field.
1387 //
1388 // SAFETY: The memory at `slot` is uninitialized.
1389 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
1390 }
1391
1392 #[allow(unused_variables)]
1393 // SAFETY:
1394 // - the field is not structurally pinned, since no `use_data` was required to create this
1395 // initializer,
1396 // - the field has been initialized,
1397 // - the reference is only valid until the end of the initializer.
1398 let $field = unsafe { &mut (*$slot).$field };
1399
1400 // Create the drop guard:
1401 //
1402 // We rely on macro hygiene to make it impossible for users to access this local variable.
1403 // We use `paste!` to create new hygiene for `$field`.
1404 $crate::macros::paste! {
1405 // SAFETY: We forget the guard later when initialization has succeeded.
1406 let [< __ $field _guard >] = unsafe {
1407 $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1408 };
1409
1410 $crate::__init_internal!(init_slot():
1411 @data($data),
1412 @slot($slot),
1413 @guards([< __ $field _guard >], $($guards,)*),
1414 @munch_fields($($rest)*),
1415 );
1416 }
1417 };
1418 (init_slot($use_data:ident):
1419 @data($data:ident),
1420 @slot($slot:ident),
1421 @guards($($guards:ident,)*),
1422 // Init by-value.
1423 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1424 ) => {
1425 {
1426 $(let $field = $val;)?
1427 // Initialize the field.
1428 //
1429 // SAFETY: The memory at `slot` is uninitialized.
1430 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
1431 }
1432 // SAFETY:
1433 // - the project function does the correct field projection,
1434 // - the field has been initialized,
1435 // - the reference is only valid until the end of the initializer.
1436 #[allow(unused_variables)]
1437 let $field = $crate::macros::paste!(unsafe { $data.[< __project_ $field >](&mut (*$slot).$field) });
1438
1439 // Create the drop guard:
1440 //
1441 // We rely on macro hygiene to make it impossible for users to access this local variable.
1442 // We use `paste!` to create new hygiene for `$field`.
1443 $crate::macros::paste! {
1444 // SAFETY: We forget the guard later when initialization has succeeded.
1445 let [< __ $field _guard >] = unsafe {
1446 $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1447 };
1448
1449 $crate::__init_internal!(init_slot($use_data):
1450 @data($data),
1451 @slot($slot),
1452 @guards([< __ $field _guard >], $($guards,)*),
1453 @munch_fields($($rest)*),
1454 );
1455 }
1456 };
1457 (make_initializer:
1458 @slot($slot:ident),
1459 @type_name($t:path),
1460 @munch_fields(_: { $($code:tt)* }, $($rest:tt)*),
1461 @acc($($acc:tt)*),
1462 ) => {
1463 // code blocks are ignored for the initializer check
1464 $crate::__init_internal!(make_initializer:
1465 @slot($slot),
1466 @type_name($t),
1467 @munch_fields($($rest)*),
1468 @acc($($acc)*),
1469 );
1470 };
1471 (make_initializer:
1472 @slot($slot:ident),
1473 @type_name($t:path),
1474 @munch_fields(..Zeroable::init_zeroed() $(,)?),
1475 @acc($($acc:tt)*),
1476 ) => {
1477 // Endpoint, nothing more to munch, create the initializer. Since the users specified
1478 // `..Zeroable::init_zeroed()`, the slot will already have been zeroed and all field that have
1479 // not been overwritten are thus zero and initialized. We still check that all fields are
1480 // actually accessible by using the struct update syntax ourselves.
1481 // We are inside of a closure that is never executed and thus we can abuse `slot` to
1482 // get the correct type inference here:
1483 #[allow(unused_assignments)]
1484 unsafe {
1485 let mut zeroed = ::core::mem::zeroed();
1486 // We have to use type inference here to make zeroed have the correct type. This does
1487 // not get executed, so it has no effect.
1488 ::core::ptr::write($slot, zeroed);
1489 zeroed = ::core::mem::zeroed();
1490 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1491 // information that is associated to already parsed fragments, so a path fragment
1492 // cannot be used in this position. Doing the retokenization results in valid rust
1493 // code.
1494 $crate::macros::paste!(
1495 ::core::ptr::write($slot, $t {
1496 $($acc)*
1497 ..zeroed
1498 });
1499 );
1500 }
1501 };
1502 (make_initializer:
1503 @slot($slot:ident),
1504 @type_name($t:path),
1505 @munch_fields($(,)?),
1506 @acc($($acc:tt)*),
1507 ) => {
1508 // Endpoint, nothing more to munch, create the initializer.
1509 // Since we are in the closure that is never called, this will never get executed.
1510 // We abuse `slot` to get the correct type inference here:
1511 //
1512 // SAFETY: TODO.
1513 unsafe {
1514 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1515 // information that is associated to already parsed fragments, so a path fragment
1516 // cannot be used in this position. Doing the retokenization results in valid rust
1517 // code.
1518 $crate::macros::paste!(
1519 ::core::ptr::write($slot, $t {
1520 $($acc)*
1521 });
1522 );
1523 }
1524 };
1525 (make_initializer:
1526 @slot($slot:ident),
1527 @type_name($t:path),
1528 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1529 @acc($($acc:tt)*),
1530 ) => {
1531 $crate::__init_internal!(make_initializer:
1532 @slot($slot),
1533 @type_name($t),
1534 @munch_fields($($rest)*),
1535 @acc($($acc)* $field: ::core::panic!(),),
1536 );
1537 };
1538 (make_initializer:
1539 @slot($slot:ident),
1540 @type_name($t:path),
1541 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1542 @acc($($acc:tt)*),
1543 ) => {
1544 $crate::__init_internal!(make_initializer:
1545 @slot($slot),
1546 @type_name($t),
1547 @munch_fields($($rest)*),
1548 @acc($($acc)* $field: ::core::panic!(),),
1549 );
1550 };
1551}
1552
1553#[doc(hidden)]
1554#[macro_export]
1555macro_rules! __derive_zeroable {
1556 (parse_input:
1557 @sig(
1558 $(#[$($struct_attr:tt)*])*
1559 $vis:vis struct $name:ident
1560 $(where $($whr:tt)*)?
1561 ),
1562 @impl_generics($($impl_generics:tt)*),
1563 @ty_generics($($ty_generics:tt)*),
1564 @body({
1565 $(
1566 $(#[$($field_attr:tt)*])*
1567 $field_vis:vis $field:ident : $field_ty:ty
1568 ),* $(,)?
1569 }),
1570 ) => {
1571 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1572 #[automatically_derived]
1573 unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*>
1574 where
1575 $($($whr)*)?
1576 {}
1577 const _: () = {
1578 fn assert_zeroable<T: ?::core::marker::Sized + $crate::Zeroable>() {}
1579 fn ensure_zeroable<$($impl_generics)*>()
1580 where $($($whr)*)?
1581 {
1582 $(assert_zeroable::<$field_ty>();)*
1583 }
1584 };
1585 };
1586 (parse_input:
1587 @sig(
1588 $(#[$($struct_attr:tt)*])*
1589 $vis:vis union $name:ident
1590 $(where $($whr:tt)*)?
1591 ),
1592 @impl_generics($($impl_generics:tt)*),
1593 @ty_generics($($ty_generics:tt)*),
1594 @body({
1595 $(
1596 $(#[$($field_attr:tt)*])*
1597 $field_vis:vis $field:ident : $field_ty:ty
1598 ),* $(,)?
1599 }),
1600 ) => {
1601 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1602 #[automatically_derived]
1603 unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*>
1604 where
1605 $($($whr)*)?
1606 {}
1607 const _: () = {
1608 fn assert_zeroable<T: ?::core::marker::Sized + $crate::Zeroable>() {}
1609 fn ensure_zeroable<$($impl_generics)*>()
1610 where $($($whr)*)?
1611 {
1612 $(assert_zeroable::<$field_ty>();)*
1613 }
1614 };
1615 };
1616}
1617
1618#[doc(hidden)]
1619#[macro_export]
1620macro_rules! __maybe_derive_zeroable {
1621 (parse_input:
1622 @sig(
1623 $(#[$($struct_attr:tt)*])*
1624 $vis:vis struct $name:ident
1625 $(where $($whr:tt)*)?
1626 ),
1627 @impl_generics($($impl_generics:tt)*),
1628 @ty_generics($($ty_generics:tt)*),
1629 @body({
1630 $(
1631 $(#[$($field_attr:tt)*])*
1632 $field_vis:vis $field:ident : $field_ty:ty
1633 ),* $(,)?
1634 }),
1635 ) => {
1636 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1637 #[automatically_derived]
1638 unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*>
1639 where
1640 $(
1641 // the `for<'__dummy>` HRTB makes this not error without the `trivial_bounds`
1642 // feature <https://github.com/rust-lang/rust/issues/48214#issuecomment-2557829956>.
1643 $field_ty: for<'__dummy> $crate::Zeroable,
1644 )*
1645 $($($whr)*)?
1646 {}
1647 };
1648 (parse_input:
1649 @sig(
1650 $(#[$($struct_attr:tt)*])*
1651 $vis:vis union $name:ident
1652 $(where $($whr:tt)*)?
1653 ),
1654 @impl_generics($($impl_generics:tt)*),
1655 @ty_generics($($ty_generics:tt)*),
1656 @body({
1657 $(
1658 $(#[$($field_attr:tt)*])*
1659 $field_vis:vis $field:ident : $field_ty:ty
1660 ),* $(,)?
1661 }),
1662 ) => {
1663 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1664 #[automatically_derived]
1665 unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*>
1666 where
1667 $(
1668 // the `for<'__dummy>` HRTB makes this not error without the `trivial_bounds`
1669 // feature <https://github.com/rust-lang/rust/issues/48214#issuecomment-2557829956>.
1670 $field_ty: for<'__dummy> $crate::Zeroable,
1671 )*
1672 $($($whr)*)?
1673 {}
1674 };
1675}