pin_init/macros.rs
1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3//! This module provides the macros that actually implement the proc-macros `pin_data` and
4//! `pinned_drop`. It also contains `__init_internal`, the implementation of the
5//! `{try_}{pin_}init!` macros.
6//!
7//! These macros should never be called directly, since they expect their input to be
8//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in
9//! safe code! Use the public facing macros instead.
10//!
11//! This architecture has been chosen because the kernel does not yet have access to `syn` which
12//! would make matters a lot easier for implementing these as proc-macros.
13//!
14//! Since this library and the kernel implementation should diverge as little as possible, the same
15//! approach has been taken here.
16//!
17//! # Macro expansion example
18//!
19//! This section is intended for readers trying to understand the macros in this module and the
20//! `[try_][pin_]init!` macros from `lib.rs`.
21//!
22//! We will look at the following example:
23//!
24//! ```rust,ignore
25//! #[pin_data]
26//! #[repr(C)]
27//! struct Bar<T> {
28//! #[pin]
29//! t: T,
30//! pub x: usize,
31//! }
32//!
33//! impl<T> Bar<T> {
34//! fn new(t: T) -> impl PinInit<Self> {
35//! pin_init!(Self { t, x: 0 })
36//! }
37//! }
38//!
39//! #[pin_data(PinnedDrop)]
40//! struct Foo {
41//! a: usize,
42//! #[pin]
43//! b: Bar<u32>,
44//! }
45//!
46//! #[pinned_drop]
47//! impl PinnedDrop for Foo {
48//! fn drop(self: Pin<&mut Self>) {
49//! println!("{self:p} is getting dropped.");
50//! }
51//! }
52//!
53//! let a = 42;
54//! let initializer = pin_init!(Foo {
55//! a,
56//! b <- Bar::new(36),
57//! });
58//! ```
59//!
60//! This example includes the most common and important features of the pin-init API.
61//!
62//! Below you can find individual section about the different macro invocations. Here are some
63//! general things we need to take into account when designing macros:
64//! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`
65//! this ensures that the correct item is used, since users could define their own `mod core {}`
66//! and then their own `panic!` inside to execute arbitrary code inside of our macro.
67//! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied
68//! expressions inside of an `unsafe` block in the macro, because this would allow users to do
69//! `unsafe` operations without an associated `unsafe` block.
70//!
71//! ## `#[pin_data]` on `Bar`
72//!
73//! This macro is used to specify which fields are structurally pinned and which fields are not. It
74//! is placed on the struct definition and allows `#[pin]` to be placed on the fields.
75//!
76//! Here is the definition of `Bar` from our example:
77//!
78//! ```rust,ignore
79//! #[pin_data]
80//! #[repr(C)]
81//! struct Bar<T> {
82//! #[pin]
83//! t: T,
84//! pub x: usize,
85//! }
86//! ```
87//!
88//! This expands to the following code:
89//!
90//! ```rust,ignore
91//! // Firstly the normal definition of the struct, attributes are preserved:
92//! #[repr(C)]
93//! struct Bar<T> {
94//! t: T,
95//! pub x: usize,
96//! }
97//! // Then an anonymous constant is defined, this is because we do not want any code to access the
98//! // types that we define inside:
99//! const _: () = {
100//! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,
101//! // since we need to implement access functions for each field and thus need to know its
102//! // type.
103//! struct __ThePinData<T> {
104//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
105//! }
106//! // We implement `Copy` for the pin-data struct, since all functions it defines will take
107//! // `self` by value.
108//! impl<T> ::core::clone::Clone for __ThePinData<T> {
109//! fn clone(&self) -> Self {
110//! *self
111//! }
112//! }
113//! impl<T> ::core::marker::Copy for __ThePinData<T> {}
114//! // For every field of `Bar`, the pin-data struct will define a function with the same name
115//! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the
116//! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field
117//! // (if pinning is structural for the field, then `PinInit` otherwise `Init`).
118//! #[allow(dead_code)]
119//! impl<T> __ThePinData<T> {
120//! unsafe fn t<E>(
121//! self,
122//! slot: *mut T,
123//! // Since `t` is `#[pin]`, this is `PinInit`.
124//! init: impl ::pin_init::PinInit<T, E>,
125//! ) -> ::core::result::Result<(), E> {
126//! unsafe { ::pin_init::PinInit::__pinned_init(init, slot) }
127//! }
128//! pub unsafe fn x<E>(
129//! self,
130//! slot: *mut usize,
131//! // Since `x` is not `#[pin]`, this is `Init`.
132//! init: impl ::pin_init::Init<usize, E>,
133//! ) -> ::core::result::Result<(), E> {
134//! unsafe { ::pin_init::Init::__init(init, slot) }
135//! }
136//! }
137//! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct
138//! // that we constructed above.
139//! unsafe impl<T> ::pin_init::__internal::HasPinData for Bar<T> {
140//! type PinData = __ThePinData<T>;
141//! unsafe fn __pin_data() -> Self::PinData {
142//! __ThePinData {
143//! __phantom: ::core::marker::PhantomData,
144//! }
145//! }
146//! }
147//! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data
148//! // struct. This is important to ensure that no user can implement a rogue `__pin_data`
149//! // function without using `unsafe`.
150//! unsafe impl<T> ::pin_init::__internal::PinData for __ThePinData<T> {
151//! type Datee = Bar<T>;
152//! }
153//! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is
154//! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned
155//! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our
156//! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist
157//! // for two reasons:
158//! // - `__phantom`: every generic must be used, since we cannot really know which generics
159//! // are used, we declare all and then use everything here once.
160//! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant
161//! // over it. The lifetime is needed to work around the limitation that trait bounds must
162//! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is
163//! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler
164//! // into accepting these bounds regardless.
165//! #[allow(dead_code)]
166//! struct __Unpin<'__pin, T> {
167//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
168//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
169//! // Our only `#[pin]` field is `t`.
170//! t: T,
171//! }
172//! #[doc(hidden)]
173//! impl<'__pin, T> ::core::marker::Unpin for Bar<T>
174//! where
175//! __Unpin<'__pin, T>: ::core::marker::Unpin,
176//! {}
177//! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users
178//! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to
179//! // UB with only safe code, so we disallow this by giving a trait implementation error using
180//! // a direct impl and a blanket implementation.
181//! trait MustNotImplDrop {}
182//! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do
183//! // (normally people want to know if a type has any kind of drop glue at all, here we want
184//! // to know if it has any kind of custom drop glue, which is exactly what this bound does).
185//! #[expect(drop_bounds)]
186//! impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
187//! impl<T> MustNotImplDrop for Bar<T> {}
188//! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to
189//! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed
190//! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.
191//! #[expect(non_camel_case_types)]
192//! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
193//! impl<
194//! T: ::pin_init::PinnedDrop,
195//! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
196//! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}
197//! };
198//! ```
199//!
200//! ## `pin_init!` in `impl Bar`
201//!
202//! This macro creates an pin-initializer for the given struct. It requires that the struct is
203//! annotated by `#[pin_data]`.
204//!
205//! Here is the impl on `Bar` defining the new function:
206//!
207//! ```rust,ignore
208//! impl<T> Bar<T> {
209//! fn new(t: T) -> impl PinInit<Self> {
210//! pin_init!(Self { t, x: 0 })
211//! }
212//! }
213//! ```
214//!
215//! This expands to the following code:
216//!
217//! ```rust,ignore
218//! impl<T> Bar<T> {
219//! fn new(t: T) -> impl PinInit<Self> {
220//! {
221//! // We do not want to allow arbitrary returns, so we declare this type as the `Ok`
222//! // return type and shadow it later when we insert the arbitrary user code. That way
223//! // there will be no possibility of returning without `unsafe`.
224//! struct __InitOk;
225//! // Get the data about fields from the supplied type.
226//! // - the function is unsafe, hence the unsafe block
227//! // - we `use` the `HasPinData` trait in the block, it is only available in that
228//! // scope.
229//! let data = unsafe {
230//! use ::pin_init::__internal::HasPinData;
231//! Self::__pin_data()
232//! };
233//! // Ensure that `data` really is of type `PinData` and help with type inference:
234//! let init = ::pin_init::__internal::PinData::make_closure::<
235//! _,
236//! __InitOk,
237//! ::core::convert::Infallible,
238//! >(data, move |slot| {
239//! {
240//! // Shadow the structure so it cannot be used to return early. If a user
241//! // tries to write `return Ok(__InitOk)`, then they get a type error,
242//! // since that will refer to this struct instead of the one defined
243//! // above.
244//! struct __InitOk;
245//! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.
246//! {
247//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };
248//! }
249//! // Since initialization could fail later (not in this case, since the
250//! // error type is `Infallible`) we will need to drop this field if there
251//! // is an error later. This `DropGuard` will drop the field when it gets
252//! // dropped and has not yet been forgotten.
253//! let __t_guard = unsafe {
254//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t))
255//! };
256//! // Expansion of `x: 0,`:
257//! // Since this can be an arbitrary expression we cannot place it inside
258//! // of the `unsafe` block, so we bind it here.
259//! {
260//! let x = 0;
261//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };
262//! }
263//! // We again create a `DropGuard`.
264//! let __x_guard = unsafe {
265//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x))
266//! };
267//! // Since initialization has successfully completed, we can now forget
268//! // the guards. This is not `mem::forget`, since we only have
269//! // `&DropGuard`.
270//! ::core::mem::forget(__x_guard);
271//! ::core::mem::forget(__t_guard);
272//! // Here we use the type checker to ensure that every field has been
273//! // initialized exactly once, since this is `if false` it will never get
274//! // executed, but still type-checked.
275//! // Additionally we abuse `slot` to automatically infer the correct type
276//! // for the struct. This is also another check that every field is
277//! // accessible from this scope.
278//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
279//! let _ = || {
280//! unsafe {
281//! ::core::ptr::write(
282//! slot,
283//! Self {
284//! // We only care about typecheck finding every field
285//! // here, the expression does not matter, just conjure
286//! // one using `panic!()`:
287//! t: ::core::panic!(),
288//! x: ::core::panic!(),
289//! },
290//! );
291//! };
292//! };
293//! }
294//! // We leave the scope above and gain access to the previously shadowed
295//! // `__InitOk` that we need to return.
296//! Ok(__InitOk)
297//! });
298//! // Change the return type from `__InitOk` to `()`.
299//! let init = move |
300//! slot,
301//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
302//! init(slot).map(|__InitOk| ())
303//! };
304//! // Construct the initializer.
305//! let init = unsafe {
306//! ::pin_init::pin_init_from_closure::<
307//! _,
308//! ::core::convert::Infallible,
309//! >(init)
310//! };
311//! init
312//! }
313//! }
314//! }
315//! ```
316//!
317//! ## `#[pin_data]` on `Foo`
318//!
319//! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the
320//! differences/new things in the expansion of the `Foo` definition:
321//!
322//! ```rust,ignore
323//! #[pin_data(PinnedDrop)]
324//! struct Foo {
325//! a: usize,
326//! #[pin]
327//! b: Bar<u32>,
328//! }
329//! ```
330//!
331//! This expands to the following code:
332//!
333//! ```rust,ignore
334//! struct Foo {
335//! a: usize,
336//! b: Bar<u32>,
337//! }
338//! const _: () = {
339//! struct __ThePinData {
340//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
341//! }
342//! impl ::core::clone::Clone for __ThePinData {
343//! fn clone(&self) -> Self {
344//! *self
345//! }
346//! }
347//! impl ::core::marker::Copy for __ThePinData {}
348//! #[allow(dead_code)]
349//! impl __ThePinData {
350//! unsafe fn b<E>(
351//! self,
352//! slot: *mut Bar<u32>,
353//! init: impl ::pin_init::PinInit<Bar<u32>, E>,
354//! ) -> ::core::result::Result<(), E> {
355//! unsafe { ::pin_init::PinInit::__pinned_init(init, slot) }
356//! }
357//! unsafe fn a<E>(
358//! self,
359//! slot: *mut usize,
360//! init: impl ::pin_init::Init<usize, E>,
361//! ) -> ::core::result::Result<(), E> {
362//! unsafe { ::pin_init::Init::__init(init, slot) }
363//! }
364//! }
365//! unsafe impl ::pin_init::__internal::HasPinData for Foo {
366//! type PinData = __ThePinData;
367//! unsafe fn __pin_data() -> Self::PinData {
368//! __ThePinData {
369//! __phantom: ::core::marker::PhantomData,
370//! }
371//! }
372//! }
373//! unsafe impl ::pin_init::__internal::PinData for __ThePinData {
374//! type Datee = Foo;
375//! }
376//! #[allow(dead_code)]
377//! struct __Unpin<'__pin> {
378//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
379//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
380//! b: Bar<u32>,
381//! }
382//! #[doc(hidden)]
383//! impl<'__pin> ::core::marker::Unpin for Foo
384//! where
385//! __Unpin<'__pin>: ::core::marker::Unpin,
386//! {}
387//! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to
388//! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like
389//! // before, instead we implement `Drop` here and delegate to `PinnedDrop`.
390//! impl ::core::ops::Drop for Foo {
391//! fn drop(&mut self) {
392//! // Since we are getting dropped, no one else has a reference to `self` and thus we
393//! // can assume that we never move.
394//! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
395//! // Create the unsafe token that proves that we are inside of a destructor, this
396//! // type is only allowed to be created in a destructor.
397//! let token = unsafe { ::pin_init::__internal::OnlyCallFromDrop::new() };
398//! ::pin_init::PinnedDrop::drop(pinned, token);
399//! }
400//! }
401//! };
402//! ```
403//!
404//! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`
405//!
406//! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an
407//! extra parameter that should not be used at all. The macro hides that parameter.
408//!
409//! Here is the `PinnedDrop` impl for `Foo`:
410//!
411//! ```rust,ignore
412//! #[pinned_drop]
413//! impl PinnedDrop for Foo {
414//! fn drop(self: Pin<&mut Self>) {
415//! println!("{self:p} is getting dropped.");
416//! }
417//! }
418//! ```
419//!
420//! This expands to the following code:
421//!
422//! ```rust,ignore
423//! // `unsafe`, full path and the token parameter are added, everything else stays the same.
424//! unsafe impl ::pin_init::PinnedDrop for Foo {
425//! fn drop(self: Pin<&mut Self>, _: ::pin_init::__internal::OnlyCallFromDrop) {
426//! println!("{self:p} is getting dropped.");
427//! }
428//! }
429//! ```
430//!
431//! ## `pin_init!` on `Foo`
432//!
433//! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion
434//! of `pin_init!` on `Foo`:
435//!
436//! ```rust,ignore
437//! let a = 42;
438//! let initializer = pin_init!(Foo {
439//! a,
440//! b <- Bar::new(36),
441//! });
442//! ```
443//!
444//! This expands to the following code:
445//!
446//! ```rust,ignore
447//! let a = 42;
448//! let initializer = {
449//! struct __InitOk;
450//! let data = unsafe {
451//! use ::pin_init::__internal::HasPinData;
452//! Foo::__pin_data()
453//! };
454//! let init = ::pin_init::__internal::PinData::make_closure::<
455//! _,
456//! __InitOk,
457//! ::core::convert::Infallible,
458//! >(data, move |slot| {
459//! {
460//! struct __InitOk;
461//! {
462//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };
463//! }
464//! let __a_guard = unsafe {
465//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))
466//! };
467//! let init = Bar::new(36);
468//! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };
469//! let __b_guard = unsafe {
470//! ::pin_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))
471//! };
472//! ::core::mem::forget(__b_guard);
473//! ::core::mem::forget(__a_guard);
474//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
475//! let _ = || {
476//! unsafe {
477//! ::core::ptr::write(
478//! slot,
479//! Foo {
480//! a: ::core::panic!(),
481//! b: ::core::panic!(),
482//! },
483//! );
484//! };
485//! };
486//! }
487//! Ok(__InitOk)
488//! });
489//! let init = move |
490//! slot,
491//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
492//! init(slot).map(|__InitOk| ())
493//! };
494//! let init = unsafe {
495//! ::pin_init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
496//! };
497//! init
498//! };
499//! ```
500
501#[cfg(kernel)]
502pub use ::macros::paste;
503#[cfg(not(kernel))]
504pub use ::paste::paste;
505
506/// Creates a `unsafe impl<...> PinnedDrop for $type` block.
507///
508/// See [`PinnedDrop`] for more information.
509///
510/// [`PinnedDrop`]: crate::PinnedDrop
511#[doc(hidden)]
512#[macro_export]
513macro_rules! __pinned_drop {
514 (
515 @impl_sig($($impl_sig:tt)*),
516 @impl_body(
517 $(#[$($attr:tt)*])*
518 fn drop($($sig:tt)*) {
519 $($inner:tt)*
520 }
521 ),
522 ) => {
523 // SAFETY: TODO.
524 unsafe $($impl_sig)* {
525 // Inherit all attributes and the type/ident tokens for the signature.
526 $(#[$($attr)*])*
527 fn drop($($sig)*, _: $crate::__internal::OnlyCallFromDrop) {
528 $($inner)*
529 }
530 }
531 }
532}
533
534/// This macro first parses the struct definition such that it separates pinned and not pinned
535/// fields. Afterwards it declares the struct and implement the `PinData` trait safely.
536#[doc(hidden)]
537#[macro_export]
538macro_rules! __pin_data {
539 // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.
540 (parse_input:
541 @args($($pinned_drop:ident)?),
542 @sig(
543 $(#[$($struct_attr:tt)*])*
544 $vis:vis struct $name:ident
545 $(where $($whr:tt)*)?
546 ),
547 @impl_generics($($impl_generics:tt)*),
548 @ty_generics($($ty_generics:tt)*),
549 @decl_generics($($decl_generics:tt)*),
550 @body({ $($fields:tt)* }),
551 ) => {
552 // We now use token munching to iterate through all of the fields. While doing this we
553 // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user
554 // wants these to be structurally pinned. The rest of the fields are the
555 // 'not pinned fields'. Additionally we collect all fields, since we need them in the right
556 // order to declare the struct.
557 //
558 // In this call we also put some explaining comments for the parameters.
559 $crate::__pin_data!(find_pinned_fields:
560 // Attributes on the struct itself, these will just be propagated to be put onto the
561 // struct definition.
562 @struct_attrs($(#[$($struct_attr)*])*),
563 // The visibility of the struct.
564 @vis($vis),
565 // The name of the struct.
566 @name($name),
567 // The 'impl generics', the generics that will need to be specified on the struct inside
568 // of an `impl<$ty_generics>` block.
569 @impl_generics($($impl_generics)*),
570 // The 'ty generics', the generics that will need to be specified on the impl blocks.
571 @ty_generics($($ty_generics)*),
572 // The 'decl generics', the generics that need to be specified on the struct
573 // definition.
574 @decl_generics($($decl_generics)*),
575 // The where clause of any impl block and the declaration.
576 @where($($($whr)*)?),
577 // The remaining fields tokens that need to be processed.
578 // We add a `,` at the end to ensure correct parsing.
579 @fields_munch($($fields)* ,),
580 // The pinned fields.
581 @pinned(),
582 // The not pinned fields.
583 @not_pinned(),
584 // All fields.
585 @fields(),
586 // The accumulator containing all attributes already parsed.
587 @accum(),
588 // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.
589 @is_pinned(),
590 // The proc-macro argument, this should be `PinnedDrop` or ``.
591 @pinned_drop($($pinned_drop)?),
592 );
593 };
594 (find_pinned_fields:
595 @struct_attrs($($struct_attrs:tt)*),
596 @vis($vis:vis),
597 @name($name:ident),
598 @impl_generics($($impl_generics:tt)*),
599 @ty_generics($($ty_generics:tt)*),
600 @decl_generics($($decl_generics:tt)*),
601 @where($($whr:tt)*),
602 // We found a PhantomPinned field, this should generally be pinned!
603 @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),
604 @pinned($($pinned:tt)*),
605 @not_pinned($($not_pinned:tt)*),
606 @fields($($fields:tt)*),
607 @accum($($accum:tt)*),
608 // This field is not pinned.
609 @is_pinned(),
610 @pinned_drop($($pinned_drop:ident)?),
611 ) => {
612 ::core::compile_error!(concat!(
613 "The field `",
614 stringify!($field),
615 "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",
616 ));
617 $crate::__pin_data!(find_pinned_fields:
618 @struct_attrs($($struct_attrs)*),
619 @vis($vis),
620 @name($name),
621 @impl_generics($($impl_generics)*),
622 @ty_generics($($ty_generics)*),
623 @decl_generics($($decl_generics)*),
624 @where($($whr)*),
625 @fields_munch($($rest)*),
626 @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),
627 @not_pinned($($not_pinned)*),
628 @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),
629 @accum(),
630 @is_pinned(),
631 @pinned_drop($($pinned_drop)?),
632 );
633 };
634 (find_pinned_fields:
635 @struct_attrs($($struct_attrs:tt)*),
636 @vis($vis:vis),
637 @name($name:ident),
638 @impl_generics($($impl_generics:tt)*),
639 @ty_generics($($ty_generics:tt)*),
640 @decl_generics($($decl_generics:tt)*),
641 @where($($whr:tt)*),
642 // We reached the field declaration.
643 @fields_munch($field:ident : $type:ty, $($rest:tt)*),
644 @pinned($($pinned:tt)*),
645 @not_pinned($($not_pinned:tt)*),
646 @fields($($fields:tt)*),
647 @accum($($accum:tt)*),
648 // This field is pinned.
649 @is_pinned(yes),
650 @pinned_drop($($pinned_drop:ident)?),
651 ) => {
652 $crate::__pin_data!(find_pinned_fields:
653 @struct_attrs($($struct_attrs)*),
654 @vis($vis),
655 @name($name),
656 @impl_generics($($impl_generics)*),
657 @ty_generics($($ty_generics)*),
658 @decl_generics($($decl_generics)*),
659 @where($($whr)*),
660 @fields_munch($($rest)*),
661 @pinned($($pinned)* $($accum)* $field: $type,),
662 @not_pinned($($not_pinned)*),
663 @fields($($fields)* $($accum)* $field: $type,),
664 @accum(),
665 @is_pinned(),
666 @pinned_drop($($pinned_drop)?),
667 );
668 };
669 (find_pinned_fields:
670 @struct_attrs($($struct_attrs:tt)*),
671 @vis($vis:vis),
672 @name($name:ident),
673 @impl_generics($($impl_generics:tt)*),
674 @ty_generics($($ty_generics:tt)*),
675 @decl_generics($($decl_generics:tt)*),
676 @where($($whr:tt)*),
677 // We reached the field declaration.
678 @fields_munch($field:ident : $type:ty, $($rest:tt)*),
679 @pinned($($pinned:tt)*),
680 @not_pinned($($not_pinned:tt)*),
681 @fields($($fields:tt)*),
682 @accum($($accum:tt)*),
683 // This field is not pinned.
684 @is_pinned(),
685 @pinned_drop($($pinned_drop:ident)?),
686 ) => {
687 $crate::__pin_data!(find_pinned_fields:
688 @struct_attrs($($struct_attrs)*),
689 @vis($vis),
690 @name($name),
691 @impl_generics($($impl_generics)*),
692 @ty_generics($($ty_generics)*),
693 @decl_generics($($decl_generics)*),
694 @where($($whr)*),
695 @fields_munch($($rest)*),
696 @pinned($($pinned)*),
697 @not_pinned($($not_pinned)* $($accum)* $field: $type,),
698 @fields($($fields)* $($accum)* $field: $type,),
699 @accum(),
700 @is_pinned(),
701 @pinned_drop($($pinned_drop)?),
702 );
703 };
704 (find_pinned_fields:
705 @struct_attrs($($struct_attrs:tt)*),
706 @vis($vis:vis),
707 @name($name:ident),
708 @impl_generics($($impl_generics:tt)*),
709 @ty_generics($($ty_generics:tt)*),
710 @decl_generics($($decl_generics:tt)*),
711 @where($($whr:tt)*),
712 // We found the `#[pin]` attr.
713 @fields_munch(#[pin] $($rest:tt)*),
714 @pinned($($pinned:tt)*),
715 @not_pinned($($not_pinned:tt)*),
716 @fields($($fields:tt)*),
717 @accum($($accum:tt)*),
718 @is_pinned($($is_pinned:ident)?),
719 @pinned_drop($($pinned_drop:ident)?),
720 ) => {
721 $crate::__pin_data!(find_pinned_fields:
722 @struct_attrs($($struct_attrs)*),
723 @vis($vis),
724 @name($name),
725 @impl_generics($($impl_generics)*),
726 @ty_generics($($ty_generics)*),
727 @decl_generics($($decl_generics)*),
728 @where($($whr)*),
729 @fields_munch($($rest)*),
730 // We do not include `#[pin]` in the list of attributes, since it is not actually an
731 // attribute that is defined somewhere.
732 @pinned($($pinned)*),
733 @not_pinned($($not_pinned)*),
734 @fields($($fields)*),
735 @accum($($accum)*),
736 // Set this to `yes`.
737 @is_pinned(yes),
738 @pinned_drop($($pinned_drop)?),
739 );
740 };
741 (find_pinned_fields:
742 @struct_attrs($($struct_attrs:tt)*),
743 @vis($vis:vis),
744 @name($name:ident),
745 @impl_generics($($impl_generics:tt)*),
746 @ty_generics($($ty_generics:tt)*),
747 @decl_generics($($decl_generics:tt)*),
748 @where($($whr:tt)*),
749 // We reached the field declaration with visibility, for simplicity we only munch the
750 // visibility and put it into `$accum`.
751 @fields_munch($fvis:vis $field:ident $($rest:tt)*),
752 @pinned($($pinned:tt)*),
753 @not_pinned($($not_pinned:tt)*),
754 @fields($($fields:tt)*),
755 @accum($($accum:tt)*),
756 @is_pinned($($is_pinned:ident)?),
757 @pinned_drop($($pinned_drop:ident)?),
758 ) => {
759 $crate::__pin_data!(find_pinned_fields:
760 @struct_attrs($($struct_attrs)*),
761 @vis($vis),
762 @name($name),
763 @impl_generics($($impl_generics)*),
764 @ty_generics($($ty_generics)*),
765 @decl_generics($($decl_generics)*),
766 @where($($whr)*),
767 @fields_munch($field $($rest)*),
768 @pinned($($pinned)*),
769 @not_pinned($($not_pinned)*),
770 @fields($($fields)*),
771 @accum($($accum)* $fvis),
772 @is_pinned($($is_pinned)?),
773 @pinned_drop($($pinned_drop)?),
774 );
775 };
776 (find_pinned_fields:
777 @struct_attrs($($struct_attrs:tt)*),
778 @vis($vis:vis),
779 @name($name:ident),
780 @impl_generics($($impl_generics:tt)*),
781 @ty_generics($($ty_generics:tt)*),
782 @decl_generics($($decl_generics:tt)*),
783 @where($($whr:tt)*),
784 // Some other attribute, just put it into `$accum`.
785 @fields_munch(#[$($attr:tt)*] $($rest:tt)*),
786 @pinned($($pinned:tt)*),
787 @not_pinned($($not_pinned:tt)*),
788 @fields($($fields:tt)*),
789 @accum($($accum:tt)*),
790 @is_pinned($($is_pinned:ident)?),
791 @pinned_drop($($pinned_drop:ident)?),
792 ) => {
793 $crate::__pin_data!(find_pinned_fields:
794 @struct_attrs($($struct_attrs)*),
795 @vis($vis),
796 @name($name),
797 @impl_generics($($impl_generics)*),
798 @ty_generics($($ty_generics)*),
799 @decl_generics($($decl_generics)*),
800 @where($($whr)*),
801 @fields_munch($($rest)*),
802 @pinned($($pinned)*),
803 @not_pinned($($not_pinned)*),
804 @fields($($fields)*),
805 @accum($($accum)* #[$($attr)*]),
806 @is_pinned($($is_pinned)?),
807 @pinned_drop($($pinned_drop)?),
808 );
809 };
810 (find_pinned_fields:
811 @struct_attrs($($struct_attrs:tt)*),
812 @vis($vis:vis),
813 @name($name:ident),
814 @impl_generics($($impl_generics:tt)*),
815 @ty_generics($($ty_generics:tt)*),
816 @decl_generics($($decl_generics:tt)*),
817 @where($($whr:tt)*),
818 // We reached the end of the fields, plus an optional additional comma, since we added one
819 // before and the user is also allowed to put a trailing comma.
820 @fields_munch($(,)?),
821 @pinned($($pinned:tt)*),
822 @not_pinned($($not_pinned:tt)*),
823 @fields($($fields:tt)*),
824 @accum(),
825 @is_pinned(),
826 @pinned_drop($($pinned_drop:ident)?),
827 ) => {
828 // Declare the struct with all fields in the correct order.
829 $($struct_attrs)*
830 $vis struct $name <$($decl_generics)*>
831 where $($whr)*
832 {
833 $($fields)*
834 }
835
836 $crate::__pin_data!(make_pin_projections:
837 @vis($vis),
838 @name($name),
839 @impl_generics($($impl_generics)*),
840 @ty_generics($($ty_generics)*),
841 @decl_generics($($decl_generics)*),
842 @where($($whr)*),
843 @pinned($($pinned)*),
844 @not_pinned($($not_pinned)*),
845 );
846
847 // We put the rest into this const item, because it then will not be accessible to anything
848 // outside.
849 const _: () = {
850 // We declare this struct which will host all of the projection function for our type.
851 // it will be invariant over all generic parameters which are inherited from the
852 // struct.
853 $vis struct __ThePinData<$($impl_generics)*>
854 where $($whr)*
855 {
856 __phantom: ::core::marker::PhantomData<
857 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
858 >,
859 }
860
861 impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>
862 where $($whr)*
863 {
864 fn clone(&self) -> Self { *self }
865 }
866
867 impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>
868 where $($whr)*
869 {}
870
871 // Make all projection functions.
872 $crate::__pin_data!(make_pin_data:
873 @pin_data(__ThePinData),
874 @impl_generics($($impl_generics)*),
875 @ty_generics($($ty_generics)*),
876 @where($($whr)*),
877 @pinned($($pinned)*),
878 @not_pinned($($not_pinned)*),
879 );
880
881 // SAFETY: We have added the correct projection functions above to `__ThePinData` and
882 // we also use the least restrictive generics possible.
883 unsafe impl<$($impl_generics)*>
884 $crate::__internal::HasPinData for $name<$($ty_generics)*>
885 where $($whr)*
886 {
887 type PinData = __ThePinData<$($ty_generics)*>;
888
889 unsafe fn __pin_data() -> Self::PinData {
890 __ThePinData { __phantom: ::core::marker::PhantomData }
891 }
892 }
893
894 // SAFETY: TODO.
895 unsafe impl<$($impl_generics)*>
896 $crate::__internal::PinData for __ThePinData<$($ty_generics)*>
897 where $($whr)*
898 {
899 type Datee = $name<$($ty_generics)*>;
900 }
901
902 // This struct will be used for the unpin analysis. Since only structurally pinned
903 // fields are relevant whether the struct should implement `Unpin`.
904 #[allow(dead_code)]
905 struct __Unpin <'__pin, $($impl_generics)*>
906 where $($whr)*
907 {
908 __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
909 __phantom: ::core::marker::PhantomData<
910 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
911 >,
912 // Only the pinned fields.
913 $($pinned)*
914 }
915
916 #[doc(hidden)]
917 impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>
918 where
919 __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,
920 $($whr)*
921 {}
922
923 // We need to disallow normal `Drop` implementation, the exact behavior depends on
924 // whether `PinnedDrop` was specified as the parameter.
925 $crate::__pin_data!(drop_prevention:
926 @name($name),
927 @impl_generics($($impl_generics)*),
928 @ty_generics($($ty_generics)*),
929 @where($($whr)*),
930 @pinned_drop($($pinned_drop)?),
931 );
932 };
933 };
934 // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
935 (drop_prevention:
936 @name($name:ident),
937 @impl_generics($($impl_generics:tt)*),
938 @ty_generics($($ty_generics:tt)*),
939 @where($($whr:tt)*),
940 @pinned_drop(),
941 ) => {
942 // We prevent this by creating a trait that will be implemented for all types implementing
943 // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
944 // if it also implements `Drop`
945 trait MustNotImplDrop {}
946 #[expect(drop_bounds)]
947 impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
948 impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>
949 where $($whr)* {}
950 // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
951 // They might implement `PinnedDrop` correctly for the struct, but forget to give
952 // `PinnedDrop` as the parameter to `#[pin_data]`.
953 #[expect(non_camel_case_types)]
954 trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
955 impl<T: $crate::PinnedDrop>
956 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
957 impl<$($impl_generics)*>
958 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>
959 where $($whr)* {}
960 };
961 // When `PinnedDrop` was specified we just implement `Drop` and delegate.
962 (drop_prevention:
963 @name($name:ident),
964 @impl_generics($($impl_generics:tt)*),
965 @ty_generics($($ty_generics:tt)*),
966 @where($($whr:tt)*),
967 @pinned_drop(PinnedDrop),
968 ) => {
969 impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>
970 where $($whr)*
971 {
972 fn drop(&mut self) {
973 // SAFETY: Since this is a destructor, `self` will not move after this function
974 // terminates, since it is inaccessible.
975 let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
976 // SAFETY: Since this is a drop function, we can create this token to call the
977 // pinned destructor of this type.
978 let token = unsafe { $crate::__internal::OnlyCallFromDrop::new() };
979 $crate::PinnedDrop::drop(pinned, token);
980 }
981 }
982 };
983 // If some other parameter was specified, we emit a readable error.
984 (drop_prevention:
985 @name($name:ident),
986 @impl_generics($($impl_generics:tt)*),
987 @ty_generics($($ty_generics:tt)*),
988 @where($($whr:tt)*),
989 @pinned_drop($($rest:tt)*),
990 ) => {
991 compile_error!(
992 "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",
993 stringify!($($rest)*),
994 );
995 };
996 (make_pin_projections:
997 @vis($vis:vis),
998 @name($name:ident),
999 @impl_generics($($impl_generics:tt)*),
1000 @ty_generics($($ty_generics:tt)*),
1001 @decl_generics($($decl_generics:tt)*),
1002 @where($($whr:tt)*),
1003 @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
1004 @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
1005 ) => {
1006 $crate::macros::paste! {
1007 #[doc(hidden)]
1008 $vis struct [< $name Projection >] <'__pin, $($decl_generics)*> {
1009 $($(#[$($p_attr)*])* $pvis $p_field : ::core::pin::Pin<&'__pin mut $p_type>,)*
1010 $($(#[$($attr)*])* $fvis $field : &'__pin mut $type,)*
1011 ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>,
1012 }
1013
1014 impl<$($impl_generics)*> $name<$($ty_generics)*>
1015 where $($whr)*
1016 {
1017 /// Pin-projects all fields of `Self`.
1018 ///
1019 /// These fields are structurally pinned:
1020 $(#[doc = ::core::concat!(" - `", ::core::stringify!($p_field), "`")])*
1021 ///
1022 /// These fields are **not** structurally pinned:
1023 $(#[doc = ::core::concat!(" - `", ::core::stringify!($field), "`")])*
1024 #[inline]
1025 $vis fn project<'__pin>(
1026 self: ::core::pin::Pin<&'__pin mut Self>,
1027 ) -> [< $name Projection >] <'__pin, $($ty_generics)*> {
1028 // SAFETY: we only give access to `&mut` for fields not structurally pinned.
1029 let this = unsafe { ::core::pin::Pin::get_unchecked_mut(self) };
1030 [< $name Projection >] {
1031 $(
1032 // SAFETY: `$p_field` is structurally pinned.
1033 $(#[$($p_attr)*])*
1034 $p_field : unsafe { ::core::pin::Pin::new_unchecked(&mut this.$p_field) },
1035 )*
1036 $(
1037 $(#[$($attr)*])*
1038 $field : &mut this.$field,
1039 )*
1040 ___pin_phantom_data: ::core::marker::PhantomData,
1041 }
1042 }
1043 }
1044 }
1045 };
1046 (make_pin_data:
1047 @pin_data($pin_data:ident),
1048 @impl_generics($($impl_generics:tt)*),
1049 @ty_generics($($ty_generics:tt)*),
1050 @where($($whr:tt)*),
1051 @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
1052 @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
1053 ) => {
1054 $crate::macros::paste! {
1055 // For every field, we create a projection function according to its projection type. If a
1056 // field is structurally pinned, then it must be initialized via `PinInit`, if it is not
1057 // structurally pinned, then it can be initialized via `Init`.
1058 //
1059 // The functions are `unsafe` to prevent accidentally calling them.
1060 #[allow(dead_code)]
1061 #[expect(clippy::missing_safety_doc)]
1062 impl<$($impl_generics)*> $pin_data<$($ty_generics)*>
1063 where $($whr)*
1064 {
1065 $(
1066 $(#[$($p_attr)*])*
1067 $pvis unsafe fn $p_field<E>(
1068 self,
1069 slot: *mut $p_type,
1070 init: impl $crate::PinInit<$p_type, E>,
1071 ) -> ::core::result::Result<(), E> {
1072 // SAFETY: TODO.
1073 unsafe { $crate::PinInit::__pinned_init(init, slot) }
1074 }
1075
1076 $(#[$($p_attr)*])*
1077 $pvis unsafe fn [<__project_ $p_field>]<'__slot>(
1078 self,
1079 slot: &'__slot mut $p_type,
1080 ) -> ::core::pin::Pin<&'__slot mut $p_type> {
1081 ::core::pin::Pin::new_unchecked(slot)
1082 }
1083 )*
1084 $(
1085 $(#[$($attr)*])*
1086 $fvis unsafe fn $field<E>(
1087 self,
1088 slot: *mut $type,
1089 init: impl $crate::Init<$type, E>,
1090 ) -> ::core::result::Result<(), E> {
1091 // SAFETY: TODO.
1092 unsafe { $crate::Init::__init(init, slot) }
1093 }
1094
1095 $(#[$($attr)*])*
1096 $fvis unsafe fn [<__project_ $field>]<'__slot>(
1097 self,
1098 slot: &'__slot mut $type,
1099 ) -> &'__slot mut $type {
1100 slot
1101 }
1102 )*
1103 }
1104 }
1105 };
1106}
1107
1108/// The internal init macro. Do not call manually!
1109///
1110/// This is called by the `{try_}{pin_}init!` macros with various inputs.
1111///
1112/// This macro has multiple internal call configurations, these are always the very first ident:
1113/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros.
1114/// - `with_update_parsed`: when the `..Zeroable::init_zeroed()` syntax has been handled.
1115/// - `init_slot`: recursively creates the code that initializes all fields in `slot`.
1116/// - `make_initializer`: recursively create the struct initializer that guarantees that every
1117/// field has been initialized exactly once.
1118#[doc(hidden)]
1119#[macro_export]
1120macro_rules! __init_internal {
1121 (
1122 @this($($this:ident)?),
1123 @typ($t:path),
1124 @fields($($fields:tt)*),
1125 @error($err:ty),
1126 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1127 // case.
1128 @data($data:ident, $($use_data:ident)?),
1129 // `HasPinData` or `HasInitData`.
1130 @has_data($has_data:ident, $get_data:ident),
1131 // `pin_init_from_closure` or `init_from_closure`.
1132 @construct_closure($construct_closure:ident),
1133 @munch_fields(),
1134 ) => {
1135 $crate::__init_internal!(with_update_parsed:
1136 @this($($this)?),
1137 @typ($t),
1138 @fields($($fields)*),
1139 @error($err),
1140 @data($data, $($use_data)?),
1141 @has_data($has_data, $get_data),
1142 @construct_closure($construct_closure),
1143 @init_zeroed(), // Nothing means default behavior.
1144 )
1145 };
1146 (
1147 @this($($this:ident)?),
1148 @typ($t:path),
1149 @fields($($fields:tt)*),
1150 @error($err:ty),
1151 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1152 // case.
1153 @data($data:ident, $($use_data:ident)?),
1154 // `HasPinData` or `HasInitData`.
1155 @has_data($has_data:ident, $get_data:ident),
1156 // `pin_init_from_closure` or `init_from_closure`.
1157 @construct_closure($construct_closure:ident),
1158 @munch_fields(..Zeroable::init_zeroed()),
1159 ) => {
1160 $crate::__init_internal!(with_update_parsed:
1161 @this($($this)?),
1162 @typ($t),
1163 @fields($($fields)*),
1164 @error($err),
1165 @data($data, $($use_data)?),
1166 @has_data($has_data, $get_data),
1167 @construct_closure($construct_closure),
1168 @init_zeroed(()), // `()` means zero all fields not mentioned.
1169 )
1170 };
1171 (
1172 @this($($this:ident)?),
1173 @typ($t:path),
1174 @fields($($fields:tt)*),
1175 @error($err:ty),
1176 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1177 // case.
1178 @data($data:ident, $($use_data:ident)?),
1179 // `HasPinData` or `HasInitData`.
1180 @has_data($has_data:ident, $get_data:ident),
1181 // `pin_init_from_closure` or `init_from_closure`.
1182 @construct_closure($construct_closure:ident),
1183 @munch_fields($ignore:tt $($rest:tt)*),
1184 ) => {
1185 $crate::__init_internal!(
1186 @this($($this)?),
1187 @typ($t),
1188 @fields($($fields)*),
1189 @error($err),
1190 @data($data, $($use_data)?),
1191 @has_data($has_data, $get_data),
1192 @construct_closure($construct_closure),
1193 @munch_fields($($rest)*),
1194 )
1195 };
1196 (with_update_parsed:
1197 @this($($this:ident)?),
1198 @typ($t:path),
1199 @fields($($fields:tt)*),
1200 @error($err:ty),
1201 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1202 // case.
1203 @data($data:ident, $($use_data:ident)?),
1204 // `HasPinData` or `HasInitData`.
1205 @has_data($has_data:ident, $get_data:ident),
1206 // `pin_init_from_closure` or `init_from_closure`.
1207 @construct_closure($construct_closure:ident),
1208 @init_zeroed($($init_zeroed:expr)?),
1209 ) => {{
1210 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
1211 // type and shadow it later when we insert the arbitrary user code. That way there will be
1212 // no possibility of returning without `unsafe`.
1213 struct __InitOk;
1214 // Get the data about fields from the supplied type.
1215 //
1216 // SAFETY: TODO.
1217 let data = unsafe {
1218 use $crate::__internal::$has_data;
1219 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1220 // information that is associated to already parsed fragments, so a path fragment
1221 // cannot be used in this position. Doing the retokenization results in valid rust
1222 // code.
1223 $crate::macros::paste!($t::$get_data())
1224 };
1225 // Ensure that `data` really is of type `$data` and help with type inference:
1226 let init = $crate::__internal::$data::make_closure::<_, __InitOk, $err>(
1227 data,
1228 move |slot| {
1229 {
1230 // Shadow the structure so it cannot be used to return early.
1231 struct __InitOk;
1232 // If `$init_zeroed` is present we should zero the slot now and not emit an
1233 // error when fields are missing (since they will be zeroed). We also have to
1234 // check that the type actually implements `Zeroable`.
1235 $({
1236 fn assert_zeroable<T: $crate::Zeroable>(_: *mut T) {}
1237 // Ensure that the struct is indeed `Zeroable`.
1238 assert_zeroable(slot);
1239 // SAFETY: The type implements `Zeroable` by the check above.
1240 unsafe { ::core::ptr::write_bytes(slot, 0, 1) };
1241 $init_zeroed // This will be `()` if set.
1242 })?
1243 // Create the `this` so it can be referenced by the user inside of the
1244 // expressions creating the individual fields.
1245 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
1246 // Initialize every field.
1247 $crate::__init_internal!(init_slot($($use_data)?):
1248 @data(data),
1249 @slot(slot),
1250 @guards(),
1251 @munch_fields($($fields)*,),
1252 );
1253 // We use unreachable code to ensure that all fields have been mentioned exactly
1254 // once, this struct initializer will still be type-checked and complain with a
1255 // very natural error message if a field is forgotten/mentioned more than once.
1256 #[allow(unreachable_code, clippy::diverging_sub_expression)]
1257 let _ = || {
1258 $crate::__init_internal!(make_initializer:
1259 @slot(slot),
1260 @type_name($t),
1261 @munch_fields($($fields)*,),
1262 @acc(),
1263 );
1264 };
1265 }
1266 Ok(__InitOk)
1267 }
1268 );
1269 let init = move |slot| -> ::core::result::Result<(), $err> {
1270 init(slot).map(|__InitOk| ())
1271 };
1272 // SAFETY: TODO.
1273 let init = unsafe { $crate::$construct_closure::<_, $err>(init) };
1274 init
1275 }};
1276 (init_slot($($use_data:ident)?):
1277 @data($data:ident),
1278 @slot($slot:ident),
1279 @guards($($guards:ident,)*),
1280 @munch_fields($(..Zeroable::init_zeroed())? $(,)?),
1281 ) => {
1282 // Endpoint of munching, no fields are left. If execution reaches this point, all fields
1283 // have been initialized. Therefore we can now dismiss the guards by forgetting them.
1284 $(::core::mem::forget($guards);)*
1285 };
1286 (init_slot($($use_data:ident)?):
1287 @data($data:ident),
1288 @slot($slot:ident),
1289 @guards($($guards:ident,)*),
1290 // arbitrary code block
1291 @munch_fields(_: { $($code:tt)* }, $($rest:tt)*),
1292 ) => {
1293 { $($code)* }
1294 $crate::__init_internal!(init_slot($($use_data)?):
1295 @data($data),
1296 @slot($slot),
1297 @guards($($guards,)*),
1298 @munch_fields($($rest)*),
1299 );
1300 };
1301 (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields.
1302 @data($data:ident),
1303 @slot($slot:ident),
1304 @guards($($guards:ident,)*),
1305 // In-place initialization syntax.
1306 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1307 ) => {
1308 let init = $val;
1309 // Call the initializer.
1310 //
1311 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1312 // return when an error/panic occurs.
1313 // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
1314 unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? };
1315 // SAFETY:
1316 // - the project function does the correct field projection,
1317 // - the field has been initialized,
1318 // - the reference is only valid until the end of the initializer.
1319 #[allow(unused_variables)]
1320 let $field = $crate::macros::paste!(unsafe { $data.[< __project_ $field >](&mut (*$slot).$field) });
1321
1322 // Create the drop guard:
1323 //
1324 // We rely on macro hygiene to make it impossible for users to access this local variable.
1325 // We use `paste!` to create new hygiene for `$field`.
1326 $crate::macros::paste! {
1327 // SAFETY: We forget the guard later when initialization has succeeded.
1328 let [< __ $field _guard >] = unsafe {
1329 $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1330 };
1331
1332 $crate::__init_internal!(init_slot($use_data):
1333 @data($data),
1334 @slot($slot),
1335 @guards([< __ $field _guard >], $($guards,)*),
1336 @munch_fields($($rest)*),
1337 );
1338 }
1339 };
1340 (init_slot(): // No `use_data`, so we use `Init::__init` directly.
1341 @data($data:ident),
1342 @slot($slot:ident),
1343 @guards($($guards:ident,)*),
1344 // In-place initialization syntax.
1345 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1346 ) => {
1347 let init = $val;
1348 // Call the initializer.
1349 //
1350 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1351 // return when an error/panic occurs.
1352 unsafe { $crate::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? };
1353
1354 // SAFETY:
1355 // - the field is not structurally pinned, since the line above must compile,
1356 // - the field has been initialized,
1357 // - the reference is only valid until the end of the initializer.
1358 #[allow(unused_variables)]
1359 let $field = unsafe { &mut (*$slot).$field };
1360
1361 // Create the drop guard:
1362 //
1363 // We rely on macro hygiene to make it impossible for users to access this local variable.
1364 // We use `paste!` to create new hygiene for `$field`.
1365 $crate::macros::paste! {
1366 // SAFETY: We forget the guard later when initialization has succeeded.
1367 let [< __ $field _guard >] = unsafe {
1368 $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1369 };
1370
1371 $crate::__init_internal!(init_slot():
1372 @data($data),
1373 @slot($slot),
1374 @guards([< __ $field _guard >], $($guards,)*),
1375 @munch_fields($($rest)*),
1376 );
1377 }
1378 };
1379 (init_slot(): // No `use_data`, so all fields are not structurally pinned
1380 @data($data:ident),
1381 @slot($slot:ident),
1382 @guards($($guards:ident,)*),
1383 // Init by-value.
1384 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1385 ) => {
1386 {
1387 $(let $field = $val;)?
1388 // Initialize the field.
1389 //
1390 // SAFETY: The memory at `slot` is uninitialized.
1391 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
1392 }
1393
1394 #[allow(unused_variables)]
1395 // SAFETY:
1396 // - the field is not structurally pinned, since no `use_data` was required to create this
1397 // initializer,
1398 // - the field has been initialized,
1399 // - the reference is only valid until the end of the initializer.
1400 let $field = unsafe { &mut (*$slot).$field };
1401
1402 // Create the drop guard:
1403 //
1404 // We rely on macro hygiene to make it impossible for users to access this local variable.
1405 // We use `paste!` to create new hygiene for `$field`.
1406 $crate::macros::paste! {
1407 // SAFETY: We forget the guard later when initialization has succeeded.
1408 let [< __ $field _guard >] = unsafe {
1409 $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1410 };
1411
1412 $crate::__init_internal!(init_slot():
1413 @data($data),
1414 @slot($slot),
1415 @guards([< __ $field _guard >], $($guards,)*),
1416 @munch_fields($($rest)*),
1417 );
1418 }
1419 };
1420 (init_slot($use_data:ident):
1421 @data($data:ident),
1422 @slot($slot:ident),
1423 @guards($($guards:ident,)*),
1424 // Init by-value.
1425 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1426 ) => {
1427 {
1428 $(let $field = $val;)?
1429 // Initialize the field.
1430 //
1431 // SAFETY: The memory at `slot` is uninitialized.
1432 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
1433 }
1434 // SAFETY:
1435 // - the project function does the correct field projection,
1436 // - the field has been initialized,
1437 // - the reference is only valid until the end of the initializer.
1438 #[allow(unused_variables)]
1439 let $field = $crate::macros::paste!(unsafe { $data.[< __project_ $field >](&mut (*$slot).$field) });
1440
1441 // Create the drop guard:
1442 //
1443 // We rely on macro hygiene to make it impossible for users to access this local variable.
1444 // We use `paste!` to create new hygiene for `$field`.
1445 $crate::macros::paste! {
1446 // SAFETY: We forget the guard later when initialization has succeeded.
1447 let [< __ $field _guard >] = unsafe {
1448 $crate::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1449 };
1450
1451 $crate::__init_internal!(init_slot($use_data):
1452 @data($data),
1453 @slot($slot),
1454 @guards([< __ $field _guard >], $($guards,)*),
1455 @munch_fields($($rest)*),
1456 );
1457 }
1458 };
1459 (make_initializer:
1460 @slot($slot:ident),
1461 @type_name($t:path),
1462 @munch_fields(_: { $($code:tt)* }, $($rest:tt)*),
1463 @acc($($acc:tt)*),
1464 ) => {
1465 // code blocks are ignored for the initializer check
1466 $crate::__init_internal!(make_initializer:
1467 @slot($slot),
1468 @type_name($t),
1469 @munch_fields($($rest)*),
1470 @acc($($acc)*),
1471 );
1472 };
1473 (make_initializer:
1474 @slot($slot:ident),
1475 @type_name($t:path),
1476 @munch_fields(..Zeroable::init_zeroed() $(,)?),
1477 @acc($($acc:tt)*),
1478 ) => {
1479 // Endpoint, nothing more to munch, create the initializer. Since the users specified
1480 // `..Zeroable::init_zeroed()`, the slot will already have been zeroed and all field that have
1481 // not been overwritten are thus zero and initialized. We still check that all fields are
1482 // actually accessible by using the struct update syntax ourselves.
1483 // We are inside of a closure that is never executed and thus we can abuse `slot` to
1484 // get the correct type inference here:
1485 #[allow(unused_assignments)]
1486 unsafe {
1487 let mut zeroed = ::core::mem::zeroed();
1488 // We have to use type inference here to make zeroed have the correct type. This does
1489 // not get executed, so it has no effect.
1490 ::core::ptr::write($slot, zeroed);
1491 zeroed = ::core::mem::zeroed();
1492 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1493 // information that is associated to already parsed fragments, so a path fragment
1494 // cannot be used in this position. Doing the retokenization results in valid rust
1495 // code.
1496 $crate::macros::paste!(
1497 ::core::ptr::write($slot, $t {
1498 $($acc)*
1499 ..zeroed
1500 });
1501 );
1502 }
1503 };
1504 (make_initializer:
1505 @slot($slot:ident),
1506 @type_name($t:path),
1507 @munch_fields($(,)?),
1508 @acc($($acc:tt)*),
1509 ) => {
1510 // Endpoint, nothing more to munch, create the initializer.
1511 // Since we are in the closure that is never called, this will never get executed.
1512 // We abuse `slot` to get the correct type inference here:
1513 //
1514 // SAFETY: TODO.
1515 unsafe {
1516 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1517 // information that is associated to already parsed fragments, so a path fragment
1518 // cannot be used in this position. Doing the retokenization results in valid rust
1519 // code.
1520 $crate::macros::paste!(
1521 ::core::ptr::write($slot, $t {
1522 $($acc)*
1523 });
1524 );
1525 }
1526 };
1527 (make_initializer:
1528 @slot($slot:ident),
1529 @type_name($t:path),
1530 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1531 @acc($($acc:tt)*),
1532 ) => {
1533 $crate::__init_internal!(make_initializer:
1534 @slot($slot),
1535 @type_name($t),
1536 @munch_fields($($rest)*),
1537 @acc($($acc)* $field: ::core::panic!(),),
1538 );
1539 };
1540 (make_initializer:
1541 @slot($slot:ident),
1542 @type_name($t:path),
1543 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1544 @acc($($acc:tt)*),
1545 ) => {
1546 $crate::__init_internal!(make_initializer:
1547 @slot($slot),
1548 @type_name($t),
1549 @munch_fields($($rest)*),
1550 @acc($($acc)* $field: ::core::panic!(),),
1551 );
1552 };
1553}
1554
1555#[doc(hidden)]
1556#[macro_export]
1557macro_rules! __derive_zeroable {
1558 (parse_input:
1559 @sig(
1560 $(#[$($struct_attr:tt)*])*
1561 $vis:vis struct $name:ident
1562 $(where $($whr:tt)*)?
1563 ),
1564 @impl_generics($($impl_generics:tt)*),
1565 @ty_generics($($ty_generics:tt)*),
1566 @body({
1567 $(
1568 $(#[$($field_attr:tt)*])*
1569 $field_vis:vis $field:ident : $field_ty:ty
1570 ),* $(,)?
1571 }),
1572 ) => {
1573 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1574 #[automatically_derived]
1575 unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*>
1576 where
1577 $($($whr)*)?
1578 {}
1579 const _: () = {
1580 fn assert_zeroable<T: ?::core::marker::Sized + $crate::Zeroable>() {}
1581 fn ensure_zeroable<$($impl_generics)*>()
1582 where $($($whr)*)?
1583 {
1584 $(assert_zeroable::<$field_ty>();)*
1585 }
1586 };
1587 };
1588 (parse_input:
1589 @sig(
1590 $(#[$($struct_attr:tt)*])*
1591 $vis:vis union $name:ident
1592 $(where $($whr:tt)*)?
1593 ),
1594 @impl_generics($($impl_generics:tt)*),
1595 @ty_generics($($ty_generics:tt)*),
1596 @body({
1597 $(
1598 $(#[$($field_attr:tt)*])*
1599 $field_vis:vis $field:ident : $field_ty:ty
1600 ),* $(,)?
1601 }),
1602 ) => {
1603 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1604 #[automatically_derived]
1605 unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*>
1606 where
1607 $($($whr)*)?
1608 {}
1609 const _: () = {
1610 fn assert_zeroable<T: ?::core::marker::Sized + $crate::Zeroable>() {}
1611 fn ensure_zeroable<$($impl_generics)*>()
1612 where $($($whr)*)?
1613 {
1614 $(assert_zeroable::<$field_ty>();)*
1615 }
1616 };
1617 };
1618}
1619
1620#[doc(hidden)]
1621#[macro_export]
1622macro_rules! __maybe_derive_zeroable {
1623 (parse_input:
1624 @sig(
1625 $(#[$($struct_attr:tt)*])*
1626 $vis:vis struct $name:ident
1627 $(where $($whr:tt)*)?
1628 ),
1629 @impl_generics($($impl_generics:tt)*),
1630 @ty_generics($($ty_generics:tt)*),
1631 @body({
1632 $(
1633 $(#[$($field_attr:tt)*])*
1634 $field_vis:vis $field:ident : $field_ty:ty
1635 ),* $(,)?
1636 }),
1637 ) => {
1638 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1639 #[automatically_derived]
1640 unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*>
1641 where
1642 $(
1643 // the `for<'__dummy>` HRTB makes this not error without the `trivial_bounds`
1644 // feature <https://github.com/rust-lang/rust/issues/48214#issuecomment-2557829956>.
1645 $field_ty: for<'__dummy> $crate::Zeroable,
1646 )*
1647 $($($whr)*)?
1648 {}
1649 };
1650 (parse_input:
1651 @sig(
1652 $(#[$($struct_attr:tt)*])*
1653 $vis:vis union $name:ident
1654 $(where $($whr:tt)*)?
1655 ),
1656 @impl_generics($($impl_generics:tt)*),
1657 @ty_generics($($ty_generics:tt)*),
1658 @body({
1659 $(
1660 $(#[$($field_attr:tt)*])*
1661 $field_vis:vis $field:ident : $field_ty:ty
1662 ),* $(,)?
1663 }),
1664 ) => {
1665 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1666 #[automatically_derived]
1667 unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*>
1668 where
1669 $(
1670 // the `for<'__dummy>` HRTB makes this not error without the `trivial_bounds`
1671 // feature <https://github.com/rust-lang/rust/issues/48214#issuecomment-2557829956>.
1672 $field_ty: for<'__dummy> $crate::Zeroable,
1673 )*
1674 $($($whr)*)?
1675 {}
1676 };
1677}