kernel/sync/
barrier.rs

1// SPDX-License-Identifier: GPL-2.0
2
3//! Memory barriers.
4//!
5//! These primitives have the same semantics as their C counterparts: and the precise definitions
6//! of semantics can be found at [`LKMM`].
7//!
8//! [`LKMM`]: srctree/tools/memory-model/
9
10#![expect(private_bounds, reason = "sealed implementation")]
11
12/// Memory barrier orderings.
13///
14/// The semantics of these orderings follows the [`LKMM`] definitions and rules.
15///
16/// - [`Read`] provides ordering between preceding load operations and succeeding load operations.
17/// - [`Write`] provides ordering between preceding store operations and succeeding store
18///   operations.
19/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory
20///   accesses.
21///
22/// [`LKMM`]: srctree/tools/memory-model/
23pub mod ordering {
24    pub use crate::sync::atomic::ordering::Full;
25
26    /// The annotation type for read-read barrier ordering.
27    pub struct Read;
28
29    /// The annotation type for write-write barrier ordering.
30    pub struct Write;
31}
32
33pub use ordering::{
34    Full,
35    Read,
36    Write, //
37};
38
39struct Smp;
40struct Dma;
41
42/// A compiler barrier.
43///
44/// A barrier that prevents compiler from reordering memory accesses across the barrier.
45#[inline(always)]
46pub(crate) fn barrier() {
47    // By default, Rust inline asms are treated as being able to access any memory or flags, hence
48    // it suffices as a compiler barrier.
49    //
50    // SAFETY: An empty asm block.
51    unsafe { core::arch::asm!("") };
52}
53
54trait MemoryBarrier<Flavour = ()> {
55    fn run();
56}
57
58macro_rules! define_barrier {
59    ($([$flavour:ident])? $ordering:ident, $binding:ident) => {
60        impl MemoryBarrier$(<$flavour>)? for $ordering {
61            #[inline]
62            fn run() {
63                // SAFETY: barrier methods are safe to call.
64                unsafe { bindings::$binding() };
65            }
66        }
67    };
68}
69
70define_barrier!(Full, mb);
71define_barrier!(Read, rmb);
72define_barrier!(Write, wmb);
73define_barrier!([Dma] Full, dma_mb);
74define_barrier!([Dma] Read, dma_rmb);
75define_barrier!([Dma] Write, dma_wmb);
76define_barrier!([Smp] Full, smp_mb);
77define_barrier!([Smp] Read, smp_rmb);
78define_barrier!([Smp] Write, smp_wmb);
79
80/// Memory barrier.
81///
82/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
83///
84/// The specific forms of reordering can be specified using the parameter.
85/// - `mb(Read)` provides a read-read barrier.
86/// - `mb(Write)` provides a write-write barrier.
87/// - `mb(Full)` provides a full barrier.
88///
89/// # Examples
90///
91/// ```
92/// # use kernel::sync::barrier::*;
93/// mb(Read);
94/// mb(Write);
95/// mb(Full);
96/// ```
97#[inline]
98#[doc(alias = "rmb")]
99#[doc(alias = "wmb")]
100pub fn mb<T: MemoryBarrier>(_: T) {
101    T::run()
102}
103
104/// Memory barrier between CPUs.
105///
106/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
107/// Does not prevent re-ordering with respect to other bus-mastering devices.
108///
109/// See [`mb`] for usage.
110#[inline]
111#[doc(alias = "smp_rmb")]
112#[doc(alias = "smp_wmb")]
113pub fn smp_mb<T: MemoryBarrier<Smp>>(_: T) {
114    if cfg!(CONFIG_SMP) {
115        T::run()
116    } else {
117        barrier()
118    }
119}
120
121/// Memory barrier between local CPU and bus-mastering devices.
122///
123/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
124/// Does not prevent re-ordering with respect to other CPUs.
125///
126/// See [`mb`] for usage.
127#[inline]
128#[doc(alias = "dma_rmb")]
129#[doc(alias = "dma_wmb")]
130pub fn dma_mb<T: MemoryBarrier<Dma>>(_: T) {
131    T::run()
132}
kernel/sync/barrier.rs

kernel/sync/
barrier.rs
