kernel/sync/
poll.rs

1// SPDX-License-Identifier: GPL-2.0
2
3// Copyright (C) 2024 Google LLC.
4
5//! Utilities for working with `struct poll_table`.
6
7use crate::{
8    alloc::AllocError,
9    bindings,
10    fs::File,
11    prelude::*,
12    sync::{CondVar, LockClassKey},
13    types::Opaque, //
14};
15use core::{
16    marker::PhantomData,
17    mem::ManuallyDrop,
18    ops::Deref, //
19};
20
21/// Creates a [`PollCondVar`] initialiser with the given name and a newly-created lock class.
22#[macro_export]
23macro_rules! new_poll_condvar {
24    ($($name:literal)?) => {
25        $crate::sync::poll::PollCondVar::new(
26            $crate::optional_name!($($name)?), $crate::static_lock_class!()
27        )
28    };
29}
30
31/// Wraps the kernel's `poll_table`.
32///
33/// # Invariants
34///
35/// The pointer must be null or reference a valid `poll_table`.
36#[repr(transparent)]
37pub struct PollTable<'a> {
38    table: *mut bindings::poll_table,
39    _lifetime: PhantomData<&'a bindings::poll_table>,
40}
41
42impl<'a> PollTable<'a> {
43    /// Creates a [`PollTable`] from a valid pointer.
44    ///
45    /// # Safety
46    ///
47    /// The pointer must be null or reference a valid `poll_table` for the duration of `'a`.
48    pub unsafe fn from_raw(table: *mut bindings::poll_table) -> Self {
49        // INVARIANTS: The safety requirements are the same as the struct invariants.
50        PollTable {
51            table,
52            _lifetime: PhantomData,
53        }
54    }
55
56    /// Register this [`PollTable`] with the provided [`PollCondVar`], so that it can be notified
57    /// using the condition variable.
58    pub fn register_wait(&self, file: &File, cv: &PollCondVar) {
59        // SAFETY:
60        // * `file.as_ptr()` references a valid file for the duration of this call.
61        // * `self.table` is null or references a valid poll_table for the duration of this call.
62        // * Since `PollCondVar` is pinned, its destructor is guaranteed to run before the memory
63        //   containing `cv.wait_queue_head` is invalidated. Since the destructor clears all
64        //   waiters and then waits for an rcu grace period, it's guaranteed that
65        //   `cv.wait_queue_head` remains valid for at least an rcu grace period after the removal
66        //   of the last waiter.
67        unsafe { bindings::poll_wait(file.as_ptr(), cv.wait_queue_head.get(), self.table) }
68    }
69}
70
71/// A wrapper around [`CondVar`] that makes it usable with [`PollTable`].
72///
73/// [`CondVar`]: crate::sync::CondVar
74#[pin_data(PinnedDrop)]
75#[repr(transparent)]
76pub struct PollCondVar {
77    #[pin]
78    inner: CondVar,
79}
80
81impl PollCondVar {
82    /// Constructs a new condvar initialiser.
83    pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> {
84        pin_init!(Self {
85            inner <- CondVar::new(name, key),
86        })
87    }
88}
89
90// Make the `CondVar` methods callable on `PollCondVar`.
91impl Deref for PollCondVar {
92    type Target = CondVar;
93
94    fn deref(&self) -> &CondVar {
95        &self.inner
96    }
97}
98
99#[pinned_drop]
100impl PinnedDrop for PollCondVar {
101    #[inline]
102    fn drop(self: Pin<&mut Self>) {
103        // Clear anything registered using `register_wait`.
104        //
105        // SAFETY: The pointer points at a valid `wait_queue_head`.
106        unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.get()) };
107
108        // Wait for epoll items to be properly removed.
109        //
110        // SAFETY: Just an FFI call.
111        unsafe { bindings::synchronize_rcu() };
112    }
113}
114
115/// A [`KBox<PollCondVar>`] that uses `kfree_rcu`.
116///
117/// [`KBox<PollCondVar>`]: PollCondVar
118pub struct PollCondVarBox {
119    inner: ManuallyDrop<Pin<KBox<PollCondVarBoxInner>>>,
120}
121
122#[pin_data]
123#[repr(C)]
124struct PollCondVarBoxInner {
125    #[pin]
126    inner: PollCondVar,
127    rcu: Opaque<bindings::callback_head>,
128}
129
130// SAFETY: PollCondVar is Send
131unsafe impl Send for PollCondVarBoxInner {}
132// SAFETY: PollCondVar is Sync
133unsafe impl Sync for PollCondVarBoxInner {}
134
135impl PollCondVarBox {
136    /// Constructs a new boxed [`PollCondVar`].
137    pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> Result<Self, AllocError> {
138        let b = KBox::pin_init(
139            pin_init!(PollCondVarBoxInner {
140                inner <- PollCondVar::new(name, key),
141                rcu: Opaque::uninit(),
142            }),
143            GFP_KERNEL,
144        )
145        .map_err(|_| AllocError)?;
146
147        Ok(PollCondVarBox {
148            inner: ManuallyDrop::new(b),
149        })
150    }
151}
152
153impl Deref for PollCondVarBox {
154    type Target = PollCondVar;
155    fn deref(&self) -> &PollCondVar {
156        &self.inner.inner
157    }
158}
159
160impl Drop for PollCondVarBox {
161    #[inline]
162    fn drop(&mut self) {
163        // SAFETY: ManuallyDrop::take ok because not already taken.
164        let boxed = unsafe { ManuallyDrop::take(&mut self.inner) };
165
166        // SAFETY: The code below frees the box without calling the actual destructor of the type,
167        // but it's okay because it re-implements the destructor using `kfree_rcu()` in place of
168        // `synchronize_rcu()`.
169        let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(boxed) });
170
171        // SAFETY: The pointer points at a valid `wait_queue_head`.
172        unsafe { bindings::__wake_up_pollfree((*ptr).inner.inner.wait_queue_head.get()) };
173
174        // SAFETY: This was allocated using `KBox::pin_init`, so it can be freed with `kvfree`.
175        unsafe { bindings::kvfree_call_rcu((*ptr).rcu.get(), ptr.cast::<ffi::c_void>()) };
176    }
177}
kernel/sync/poll.rs

kernel/sync/
poll.rs
