kernel/sync/lock/
spinlock.rs

1// SPDX-License-Identifier: GPL-2.0
2
3//! A kernel spinlock.
4//!
5//! This module allows Rust code to use the kernel's `spinlock_t`.
6use super::*;
7use crate::interrupt::LocalInterruptDisabled;
8
9/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class.
10///
11/// It uses the name if one is given, otherwise it generates one based on the file name and line
12/// number.
13#[macro_export]
14macro_rules! new_spinlock {
15    ($inner:expr $(, $name:literal)? $(,)?) => {
16        $crate::sync::SpinLock::new(
17            $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!())
18    };
19}
20pub use new_spinlock;
21
22/// A spinlock.
23///
24/// Exposes the kernel's [`spinlock_t`]. When multiple CPUs attempt to lock the same spinlock, only
25/// one at a time is allowed to progress, the others will block (spinning) until the spinlock is
26/// unlocked, at which point another CPU will be allowed to make progress.
27///
28/// Instances of [`SpinLock`] need a lock class and to be pinned. The recommended way to create such
29/// instances is with the [`pin_init`](pin_init::pin_init) and [`new_spinlock`] macros.
30///
31/// # Examples
32///
33/// The following example shows how to declare, allocate and initialise a struct (`Example`) that
34/// contains an inner struct (`Inner`) that is protected by a spinlock.
35///
36/// ```
37/// use kernel::sync::{new_spinlock, SpinLock};
38///
39/// struct Inner {
40///     a: u32,
41///     b: u32,
42/// }
43///
44/// #[pin_data]
45/// struct Example {
46///     c: u32,
47///     #[pin]
48///     d: SpinLock<Inner>,
49/// }
50///
51/// impl Example {
52///     fn new() -> impl PinInit<Self> {
53///         pin_init!(Self {
54///             c: 10,
55///             d <- new_spinlock!(Inner { a: 20, b: 30 }),
56///         })
57///     }
58/// }
59///
60/// // Allocate a boxed `Example`.
61/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
62/// assert_eq!(e.c, 10);
63/// assert_eq!(e.d.lock().a, 20);
64/// assert_eq!(e.d.lock().b, 30);
65/// # Ok::<(), Error>(())
66/// ```
67///
68/// The following example shows how to use interior mutability to modify the contents of a struct
69/// protected by a spinlock despite only having a shared reference:
70///
71/// ```
72/// use kernel::sync::SpinLock;
73///
74/// struct Example {
75///     a: u32,
76///     b: u32,
77/// }
78///
79/// fn example(m: &SpinLock<Example>) {
80///     let mut guard = m.lock();
81///     guard.a += 10;
82///     guard.b += 20;
83/// }
84/// ```
85///
86/// [`spinlock_t`]: srctree/include/linux/spinlock.h
87pub type SpinLock<T> = Lock<T, SpinLockBackend>;
88
89/// A kernel `spinlock_t` lock backend.
90pub struct SpinLockBackend;
91
92/// A [`Guard`] acquired from locking a [`SpinLock`].
93///
94/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock
95/// the [`SpinLock`] upon being dropped.
96pub type SpinLockGuard<'a, T> = Guard<'a, T, SpinLockBackend>;
97
98// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the
99// default implementation that always calls the same locking method.
100unsafe impl Backend for SpinLockBackend {
101    type State = bindings::spinlock_t;
102    type GuardState = ();
103
104    #[inline]
105    unsafe fn init(
106        ptr: *mut Self::State,
107        name: *const crate::ffi::c_char,
108        key: *mut bindings::lock_class_key,
109    ) {
110        // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and
111        // `key` are valid for read indefinitely.
112        unsafe { bindings::__spin_lock_init(ptr, name, key) }
113    }
114
115    #[inline]
116    unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState {
117        // SAFETY: The safety requirements of this function ensure that `ptr` points to valid
118        // memory, and that it has been initialised before.
119        unsafe { bindings::spin_lock(ptr) }
120    }
121
122    #[inline]
123    unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) {
124        // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the
125        // caller is the owner of the spinlock.
126        unsafe { bindings::spin_unlock(ptr) }
127    }
128
129    #[inline]
130    unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState> {
131        // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
132        let result = unsafe { bindings::spin_trylock(ptr) };
133
134        if result != 0 {
135            Some(())
136        } else {
137            None
138        }
139    }
140
141    #[inline]
142    unsafe fn assert_is_held(ptr: *mut Self::State) {
143        // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
144        unsafe { bindings::spin_assert_is_held(ptr) }
145    }
146}
147
148/// Creates a [`SpinLockIrq`] initialiser with the given name and a newly-created lock class.
149///
150/// It uses the name if one is given, otherwise it generates one based on the file name and line
151/// number.
152#[macro_export]
153macro_rules! new_spinlock_irq {
154    ($inner:expr $(, $name:literal)? $(,)?) => {
155        $crate::sync::SpinLockIrq::new(
156            $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!())
157    };
158}
159pub use new_spinlock_irq;
160
161/// A variant of `SpinLock` that ensures interrupts are disabled in the critical section.
162///
163/// This lock can be acquired in two ways:
164///
165/// - Using [`lock()`] like any other type of lock, in which case the bindings will modify the
166///   interrupt state to ensure that local processor interrupts remain disabled for at least as
167///   long as the [`SpinLockIrqGuard`] exists.
168/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`] token is present and
169///   local processor interrupts are already known to be disabled, in which case the local
170///   interrupt state will not be touched. This method should be preferred if a
171///   [`LocalInterruptDisabled`] token is present in the scope.
172///
173/// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts,
174/// [see the interrupt module](kernel::interrupt).
175///
176/// # Examples
177///
178/// The following example shows how to declare, allocate initialise and access a struct (`Example`)
179/// that contains an inner struct (`Inner`) that is protected by a spinlock that requires local
180/// processor interrupts to be disabled.
181///
182/// ```
183/// use kernel::sync::{new_spinlock_irq, SpinLockIrq};
184///
185/// struct Inner {
186///     a: u32,
187///     b: u32,
188/// }
189///
190/// #[pin_data]
191/// struct Example {
192///     #[pin]
193///     c: SpinLockIrq<Inner>,
194///     #[pin]
195///     d: SpinLockIrq<Inner>,
196/// }
197///
198/// impl Example {
199///     fn new() -> impl PinInit<Self> {
200///         pin_init!(Self {
201///             c <- new_spinlock_irq!(Inner { a: 0, b: 10 }),
202///             d <- new_spinlock_irq!(Inner { a: 20, b: 30 }),
203///         })
204///     }
205/// }
206///
207/// // Allocate a boxed `Example`
208/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
209///
210/// // Accessing an `Example` from a context where interrupts may not be disabled already.
211/// let c_guard = e.c.lock(); // interrupts are disabled now, +1 interrupt disable refcount
212/// let d_guard = e.d.lock(); // no interrupt state change, +1 interrupt disable refcount
213///
214/// assert_eq!(c_guard.a, 0);
215/// assert_eq!(c_guard.b, 10);
216/// assert_eq!(d_guard.a, 20);
217/// assert_eq!(d_guard.b, 30);
218///
219/// drop(c_guard); // Dropping c_guard will not re-enable interrupts just yet, since d_guard is
220///                // still in scope.
221/// drop(d_guard); // Last interrupt disable reference dropped here, so interrupts are re-enabled
222///                // now
223/// # Ok::<(), Error>(())
224/// ```
225///
226/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_with()`] in a function
227/// which can only be called when local processor interrupts are already disabled.
228///
229/// ```
230/// use kernel::sync::{new_spinlock_irq, SpinLockIrq};
231/// use kernel::interrupt::*;
232///
233/// struct Inner {
234///     a: u32,
235/// }
236///
237/// #[pin_data]
238/// struct Example {
239///     #[pin]
240///     inner: SpinLockIrq<Inner>,
241/// }
242///
243/// impl Example {
244///     fn new() -> impl PinInit<Self> {
245///         pin_init!(Self {
246///             inner <- new_spinlock_irq!(Inner { a: 20 }),
247///         })
248///     }
249/// }
250///
251/// // Accessing an `Example` from a function that can only be called in no-interrupt contexts.
252/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled) {
253///     // Because we know interrupts are disabled from interrupt_disable, we can skip toggling
254///     // interrupt state using lock_with() and the provided token
255///     assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20);
256/// }
257///
258/// # let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
259/// # let interrupt_guard = local_interrupt_disable();
260/// # noirq_work(&e, &interrupt_guard);
261/// #
262/// # Ok::<(), Error>(())
263/// ```
264///
265/// [`lock()`]: SpinLockIrq::lock
266/// [`lock_with()`]: SpinLockIrq::lock_with
267pub type SpinLockIrq<T> = super::Lock<T, SpinLockIrqBackend>;
268
269/// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts.
270pub struct SpinLockIrqBackend;
271
272/// A [`Guard`] acquired from locking a [`SpinLockIrq`] using [`lock()`].
273///
274/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLockIrq`] using
275/// [`lock()`]. It will unlock the [`SpinLockIrq`] and decrement the local processor's interrupt
276/// disablement refcount upon being dropped.
277///
278/// [`lock()`]: SpinLockIrq::lock
279pub type SpinLockIrqGuard<'a, T> = Guard<'a, T, SpinLockIrqBackend>;
280
281// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the
282// default implementation that always calls the same locking method.
283unsafe impl Backend for SpinLockIrqBackend {
284    type State = bindings::spinlock_t;
285    type GuardState = ();
286
287    #[inline]
288    unsafe fn init(
289        ptr: *mut Self::State,
290        name: *const crate::ffi::c_char,
291        key: *mut bindings::lock_class_key,
292    ) {
293        // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and
294        // `key` are valid for read indefinitely.
295        unsafe { bindings::__spin_lock_init(ptr, name, key) }
296    }
297
298    #[inline]
299    unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState {
300        // SAFETY: The safety requirements of this function ensure that `ptr` points to valid
301        // memory, and that it has been initialised before.
302        unsafe { bindings::spin_lock_irq_disable(ptr) }
303    }
304
305    #[inline]
306    unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) {
307        // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the
308        // caller is the owner of the spinlock.
309        unsafe { bindings::spin_unlock_irq_enable(ptr) }
310    }
311
312    #[inline]
313    unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState> {
314        // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
315        let result = unsafe { bindings::spin_trylock_irq_disable(ptr) };
316
317        if result != 0 {
318            Some(())
319        } else {
320            None
321        }
322    }
323
324    #[inline]
325    unsafe fn assert_is_held(ptr: *mut Self::State) {
326        // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
327        unsafe { bindings::spin_assert_is_held(ptr) }
328    }
329}
330
331impl<T: ?Sized> Lock<T, SpinLockIrqBackend> {
332    /// Casts the lock as a `Lock<T, SpinLockBackend>`.
333    #[inline]
334    fn as_lock_in_interrupt<'a>(&'a self, _context: &'a LocalInterruptDisabled) -> &'a SpinLock<T> {
335        // SAFETY:
336        // - `Lock<T, SpinLockBackend>` and `Lock<T, SpinLockIrqBackend>` both have identical data
337        //   layouts.
338        // - As long as local interrupts are disabled (which is proven to be true by _context), it
339        //   is safe to treat a lock with SpinLockIrqBackend as a SpinLockBackend lock.
340        unsafe { core::mem::transmute(self) }
341    }
342
343    /// Acquires the lock without modifying local interrupt state.
344    ///
345    /// This function should be used in place of the more expensive [`Lock::lock()`] function when
346    /// possible for [`SpinLockIrq`] locks.
347    #[inline]
348    pub fn lock_with<'a>(&'a self, context: &'a LocalInterruptDisabled) -> SpinLockGuard<'a, T> {
349        self.as_lock_in_interrupt(context).lock()
350    }
351
352    /// Tries to acquire the lock without modifying local interrupt state.
353    ///
354    /// This function should be used in place of the more expensive [`Lock::try_lock()`] function
355    /// when possible for [`SpinLockIrq`] locks.
356    ///
357    /// Returns a guard that can be used to access the data protected by the lock if successful.
358    #[must_use = "if unused, the lock will be immediately unlocked"]
359    #[inline]
360    pub fn try_lock_with<'a>(
361        &'a self,
362        context: &'a LocalInterruptDisabled,
363    ) -> Option<SpinLockGuard<'a, T>> {
364        self.as_lock_in_interrupt(context).try_lock()
365    }
366}
367
368#[kunit_tests(rust_spinlock_irq_condvar)]
369mod tests {
370    use super::*;
371    use crate::{
372        sync::*,
373        workqueue::{
374            self,
375            impl_has_work,
376            new_work,
377            Work,
378            WorkItem, //
379        },
380    };
381
382    struct TestState {
383        value: u32,
384        waiter_ready: bool,
385    }
386
387    #[pin_data]
388    struct Test {
389        #[pin]
390        state: SpinLockIrq<TestState>,
391
392        #[pin]
393        state_changed: CondVar,
394
395        #[pin]
396        waiter_state_changed: CondVar,
397
398        #[pin]
399        wait_work: Work<Self>,
400    }
401
402    impl_has_work! {
403        impl HasWork<Self> for Test { self.wait_work }
404    }
405
406    impl Test {
407        pub(crate) fn new() -> Result<Arc<Self>> {
408            Arc::try_pin_init(
409                try_pin_init!(
410                    Self {
411                        state <- new_spinlock_irq!(TestState {
412                            value: 1,
413                            waiter_ready: false
414                        }),
415                        state_changed <- new_condvar!(),
416                        waiter_state_changed <- new_condvar!(),
417                        wait_work <- new_work!("IrqCondvarTest::wait_work")
418                    }
419                ),
420                GFP_KERNEL,
421            )
422        }
423    }
424
425    impl WorkItem for Test {
426        type Pointer = Arc<Self>;
427
428        fn run(this: Arc<Self>) {
429            // Wait for the test to be ready to wait for us
430            let mut state = this.state.lock();
431
432            // Make sure the interrupts actually turned off
433            // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()`
434            unsafe { bindings::lockdep_assert_irqs_disabled() };
435
436            while !state.waiter_ready {
437                this.waiter_state_changed.wait(&mut state);
438            }
439
440            // Deliver the exciting value update our test has been waiting for
441            state.value += 1;
442            this.state_changed.notify_sync();
443        }
444    }
445
446    #[test]
447    fn spinlock_irq_condvar() -> Result {
448        let testdata = Test::new()?;
449
450        let _ = workqueue::system_dfl().enqueue(testdata.clone());
451
452        // Let the updater know when we're ready to wait
453        let mut state = testdata.state.lock();
454        state.waiter_ready = true;
455        testdata.waiter_state_changed.notify_sync();
456
457        // Wait for the exciting value update
458        testdata.state_changed.wait(&mut state);
459        assert_eq!(state.value, 2);
460        Ok(())
461    }
462}
463
464/// Helper for creating a raw unlocked `bindings::raw_spinlock_t`.
465///
466/// For use in statics containing raw spinlocks.
467#[doc(alias("__SPIN_LOCK_UNLOCKED", "DEFINE_SPINLOCK"))]
468pub(crate) const fn raw_spin_lock_unlocked(name: &'static CStr) -> bindings::raw_spinlock_t {
469    // Silence unused variable warnings.
470    #[cfg(not(CONFIG_DEBUG_LOCK_ALLOC))]
471    let _ = name;
472
473    bindings::raw_spinlock_t {
474        raw_lock: bindings::__ARCH_SPIN_LOCK_UNLOCKED,
475
476        #[cfg(CONFIG_DEBUG_SPINLOCK)]
477        magic: bindings::SPINLOCK_MAGIC,
478        #[cfg(CONFIG_DEBUG_SPINLOCK)]
479        owner_cpu: u32::MAX,
480        #[cfg(CONFIG_DEBUG_SPINLOCK)]
481        owner: usize::MAX as *mut c_void,
482
483        #[cfg(CONFIG_DEBUG_LOCK_ALLOC)]
484        dep_map: kernel::sync::lockdep::raw_lockdep_map(
485            name,
486            kernel::sync::lockdep::LD_WAIT_SPIN,
487            kernel::sync::lockdep::LD_WAIT_INV,
488        ),
489    }
490}
kernel/sync/lock/spinlock.rs

kernel/sync/lock/
spinlock.rs
