kernel/sync/
poll.rs

1// SPDX-License-Identifier: GPL-2.0
2
3// Copyright (C) 2024 Google LLC.
4
5//! Utilities for working with `struct poll_table`.
6
7use crate::{
8    alloc::AllocError,
9    bindings,
10    fs::File,
11    prelude::*,
12    sync::{
13        rcu::synchronize_rcu,
14        CondVar,
15        LockClassKey, //
16    },
17    types::Opaque, //
18};
19use core::{
20    marker::PhantomData,
21    mem::ManuallyDrop,
22    ops::Deref, //
23};
24
25/// Creates a [`PollCondVar`] initialiser with the given name and a newly-created lock class.
26#[macro_export]
27macro_rules! new_poll_condvar {
28    ($($name:literal)?) => {
29        $crate::sync::poll::PollCondVar::new(
30            $crate::optional_name!($($name)?), $crate::static_lock_class!()
31        )
32    };
33}
34
35/// Wraps the kernel's `poll_table`.
36///
37/// # Invariants
38///
39/// The pointer must be null or reference a valid `poll_table`.
40#[repr(transparent)]
41pub struct PollTable<'a> {
42    table: *mut bindings::poll_table,
43    _lifetime: PhantomData<&'a bindings::poll_table>,
44}
45
46impl<'a> PollTable<'a> {
47    /// Creates a [`PollTable`] from a valid pointer.
48    ///
49    /// # Safety
50    ///
51    /// The pointer must be null or reference a valid `poll_table` for the duration of `'a`.
52    pub unsafe fn from_raw(table: *mut bindings::poll_table) -> Self {
53        // INVARIANTS: The safety requirements are the same as the struct invariants.
54        PollTable {
55            table,
56            _lifetime: PhantomData,
57        }
58    }
59
60    /// Register this [`PollTable`] with the provided [`PollCondVar`], so that it can be notified
61    /// using the condition variable.
62    pub fn register_wait(&self, file: &File, cv: &PollCondVar) {
63        // SAFETY:
64        // * `file.as_ptr()` references a valid file for the duration of this call.
65        // * `self.table` is null or references a valid poll_table for the duration of this call.
66        // * Since `PollCondVar` is pinned, its destructor is guaranteed to run before the memory
67        //   containing `cv.wait_queue_head` is invalidated. Since the destructor clears all
68        //   waiters and then waits for an rcu grace period, it's guaranteed that
69        //   `cv.wait_queue_head` remains valid for at least an rcu grace period after the removal
70        //   of the last waiter.
71        unsafe { bindings::poll_wait(file.as_ptr(), cv.wait_queue_head.get(), self.table) }
72    }
73}
74
75/// A wrapper around [`CondVar`] that makes it usable with [`PollTable`].
76///
77/// [`CondVar`]: crate::sync::CondVar
78#[pin_data(PinnedDrop)]
79#[repr(transparent)]
80pub struct PollCondVar {
81    #[pin]
82    inner: CondVar,
83}
84
85impl PollCondVar {
86    /// Constructs a new condvar initialiser.
87    pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> {
88        pin_init!(Self {
89            inner <- CondVar::new(name, key),
90        })
91    }
92}
93
94// Make the `CondVar` methods callable on `PollCondVar`.
95impl Deref for PollCondVar {
96    type Target = CondVar;
97
98    fn deref(&self) -> &CondVar {
99        &self.inner
100    }
101}
102
103#[pinned_drop]
104impl PinnedDrop for PollCondVar {
105    #[inline]
106    fn drop(self: Pin<&mut Self>) {
107        // Clear anything registered using `register_wait`.
108        //
109        // SAFETY: The pointer points at a valid `wait_queue_head`.
110        unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.get()) };
111
112        // Wait for epoll items to be properly removed.
113        synchronize_rcu();
114    }
115}
116
117/// A [`KBox<PollCondVar>`] that uses `kfree_rcu`.
118///
119/// [`KBox<PollCondVar>`]: PollCondVar
120pub struct PollCondVarBox {
121    inner: ManuallyDrop<Pin<KBox<PollCondVarBoxInner>>>,
122}
123
124#[pin_data]
125#[repr(C)]
126struct PollCondVarBoxInner {
127    #[pin]
128    inner: PollCondVar,
129    rcu: Opaque<bindings::kvfree_rcu_head>,
130}
131
132// SAFETY: PollCondVar is Send
133unsafe impl Send for PollCondVarBoxInner {}
134// SAFETY: PollCondVar is Sync
135unsafe impl Sync for PollCondVarBoxInner {}
136
137impl PollCondVarBox {
138    /// Constructs a new boxed [`PollCondVar`].
139    pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> Result<Self, AllocError> {
140        let b = KBox::pin_init(
141            pin_init!(PollCondVarBoxInner {
142                inner <- PollCondVar::new(name, key),
143                rcu: Opaque::uninit(),
144            }),
145            GFP_KERNEL,
146        )
147        .map_err(|_| AllocError)?;
148
149        Ok(PollCondVarBox {
150            inner: ManuallyDrop::new(b),
151        })
152    }
153}
154
155impl Deref for PollCondVarBox {
156    type Target = PollCondVar;
157    fn deref(&self) -> &PollCondVar {
158        &self.inner.inner
159    }
160}
161
162impl Drop for PollCondVarBox {
163    #[inline]
164    fn drop(&mut self) {
165        // SAFETY: ManuallyDrop::take ok because not already taken.
166        let boxed = unsafe { ManuallyDrop::take(&mut self.inner) };
167
168        // SAFETY: The code below frees the box without calling the actual destructor of the type,
169        // but it's okay because it re-implements the destructor using `kfree_rcu()` in place of
170        // `synchronize_rcu()`.
171        let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(boxed) });
172
173        // SAFETY: The pointer points at a valid `wait_queue_head`.
174        unsafe { bindings::__wake_up_pollfree((*ptr).inner.inner.wait_queue_head.get()) };
175
176        // SAFETY: This was allocated using `KBox::pin_init`, so it can be freed with `kvfree`.
177        unsafe { bindings::kvfree_call_rcu((*ptr).rcu.get(), ptr.cast::<ffi::c_void>()) };
178    }
179}
kernel/sync/poll.rs

kernel/sync/
poll.rs
