kernel/iommu/
pgtable.rs

1// SPDX-License-Identifier: GPL-2.0
2
3//! IOMMU page table management.
4//!
5//! C header: [`include/linux/io-pgtable.h`](srctree/include/linux/io-pgtable.h)
6
7use core::{
8    marker::PhantomData,
9    ptr::NonNull, //
10};
11
12use crate::{
13    alloc,
14    bindings,
15    device::{
16        Bound,
17        Device, //
18    },
19    devres::Devres,
20    error::to_result,
21    io::PhysAddr,
22    prelude::*, //
23};
24
25use bindings::io_pgtable_fmt;
26
27/// Protection flags used with IOMMU mappings.
28pub mod prot {
29    /// Read access.
30    pub const READ: u32 = bindings::IOMMU_READ;
31    /// Write access.
32    pub const WRITE: u32 = bindings::IOMMU_WRITE;
33    /// Request cache coherency.
34    pub const CACHE: u32 = bindings::IOMMU_CACHE;
35    /// Request no-execute permission.
36    pub const NOEXEC: u32 = bindings::IOMMU_NOEXEC;
37    /// MMIO peripheral mapping.
38    pub const MMIO: u32 = bindings::IOMMU_MMIO;
39    /// Privileged mapping.
40    pub const PRIVILEGED: u32 = bindings::IOMMU_PRIV;
41}
42
43/// Represents a requested `io_pgtable` configuration.
44pub struct Config {
45    /// Quirk bitmask (type-specific).
46    pub quirks: usize,
47    /// Valid page sizes, as a bitmask of powers of two.
48    pub pgsize_bitmap: usize,
49    /// Input address space size in bits.
50    pub ias: u32,
51    /// Output address space size in bits.
52    pub oas: u32,
53    /// IOMMU uses coherent accesses for page table walks.
54    pub coherent_walk: bool,
55}
56
57/// An io page table using a specific format.
58///
59/// # Invariants
60///
61/// The pointer references a valid io page table.
62pub struct IoPageTable<F: IoPageTableFmt> {
63    ptr: NonNull<bindings::io_pgtable_ops>,
64    _marker: PhantomData<F>,
65}
66
67// SAFETY: `struct io_pgtable_ops` is not restricted to a single thread.
68unsafe impl<F: IoPageTableFmt> Send for IoPageTable<F> {}
69// SAFETY: `struct io_pgtable_ops` may be accessed concurrently.
70unsafe impl<F: IoPageTableFmt> Sync for IoPageTable<F> {}
71
72/// The format used by this page table.
73pub trait IoPageTableFmt: 'static {
74    /// The value representing this format.
75    const FORMAT: io_pgtable_fmt;
76}
77
78impl<F: IoPageTableFmt> IoPageTable<F> {
79    /// Create a new `IoPageTable` as a device resource.
80    #[inline]
81    pub fn new(
82        dev: &Device<Bound>,
83        config: Config,
84    ) -> impl PinInit<Devres<IoPageTable<F>>, Error> + '_ {
85        // SAFETY: Devres ensures that the value is dropped during device unbind.
86        Devres::new(dev, unsafe { Self::new_raw(dev, config) })
87    }
88
89    /// Create a new `IoPageTable`.
90    ///
91    /// # Safety
92    ///
93    /// If successful, then the returned `IoPageTable` must be dropped before the device is
94    /// unbound.
95    #[inline]
96    pub unsafe fn new_raw(dev: &Device<Bound>, config: Config) -> Result<IoPageTable<F>> {
97        let mut raw_cfg = bindings::io_pgtable_cfg {
98            quirks: config.quirks,
99            pgsize_bitmap: config.pgsize_bitmap,
100            ias: config.ias,
101            oas: config.oas,
102            coherent_walk: config.coherent_walk,
103            tlb: &raw const NOOP_FLUSH_OPS,
104            iommu_dev: dev.as_raw(),
105            ..Zeroable::zeroed()
106        };
107
108        // SAFETY:
109        // * The raw_cfg pointer is valid for the duration of this call.
110        // * The provided `FLUSH_OPS` contains valid function pointers that accept a null pointer
111        //   as cookie.
112        // * The caller ensures that the io pgtable does not outlive the device.
113        let ops = unsafe {
114            bindings::alloc_io_pgtable_ops(F::FORMAT, &mut raw_cfg, core::ptr::null_mut())
115        };
116
117        // INVARIANT: We successfully created a valid page table.
118        Ok(IoPageTable {
119            ptr: NonNull::new(ops).ok_or(ENOMEM)?,
120            _marker: PhantomData,
121        })
122    }
123
124    /// Obtain a raw pointer to the underlying `struct io_pgtable_ops`.
125    #[inline]
126    pub fn raw_ops(&self) -> *mut bindings::io_pgtable_ops {
127        self.ptr.as_ptr()
128    }
129
130    /// Obtain a raw pointer to the underlying `struct io_pgtable`.
131    #[inline]
132    pub fn raw_pgtable(&self) -> *mut bindings::io_pgtable {
133        // SAFETY: The io_pgtable_ops of an io-pgtable is always the ops field of a io_pgtable.
134        unsafe { kernel::container_of!(self.raw_ops(), bindings::io_pgtable, ops) }
135    }
136
137    /// Obtain a raw pointer to the underlying `struct io_pgtable_cfg`.
138    #[inline]
139    pub fn raw_cfg(&self) -> *mut bindings::io_pgtable_cfg {
140        // SAFETY: The `raw_pgtable()` method returns a valid pointer.
141        unsafe { &raw mut (*self.raw_pgtable()).cfg }
142    }
143
144    /// Map a physically contiguous range of pages of the same size.
145    ///
146    /// Even if successful, this operation may not map the entire range. In that case, only a
147    /// prefix of the range is mapped, and the returned integer indicates its length in bytes. In
148    /// this case, the caller will usually call `map_pages` again for the remaining range.
149    ///
150    /// The returned [`Result`] indicates whether an error was encountered while mapping pages.
151    /// Note that this may return a non-zero length even if an error was encountered. The caller
152    /// will usually [unmap the relevant pages](Self::unmap_pages) on error.
153    ///
154    /// The caller must flush the TLB before using the pgtable to access the newly created mapping.
155    ///
156    /// # Safety
157    ///
158    /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while
159    ///   this `map_pages` operation executes.
160    /// * This page table must not contain any mapping that overlaps with the mapping created by
161    ///   this call.
162    /// * If this page table is live, then the caller must ensure that it's okay to access the
163    ///   physical address being mapped for the duration in which it is mapped.
164    #[inline]
165    pub unsafe fn map_pages(
166        &self,
167        iova: usize,
168        paddr: PhysAddr,
169        pgsize: usize,
170        pgcount: usize,
171        prot: u32,
172        flags: alloc::Flags,
173    ) -> (usize, Result) {
174        let mut mapped: usize = 0;
175
176        // SAFETY: The `map_pages` function in `io_pgtable_ops` is never null.
177        let map_pages = unsafe { (*self.raw_ops()).map_pages.unwrap_unchecked() };
178
179        // SAFETY: The safety requirements of this method are sufficient to call `map_pages`.
180        let ret = to_result(unsafe {
181            (map_pages)(
182                self.raw_ops(),
183                iova,
184                paddr,
185                pgsize,
186                pgcount,
187                prot as i32,
188                flags.as_raw(),
189                &mut mapped,
190            )
191        });
192
193        (mapped, ret)
194    }
195
196    /// Unmap a range of virtually contiguous pages of the same size.
197    ///
198    /// This may not unmap the entire range, and returns the length of the unmapped prefix in
199    /// bytes.
200    ///
201    /// # Safety
202    ///
203    /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while
204    ///   this `unmap_pages` operation executes.
205    /// * This page table must contain one or more consecutive mappings starting at `iova` whose
206    ///   total size is `pgcount * pgsize`.
207    #[inline]
208    #[must_use]
209    pub unsafe fn unmap_pages(&self, iova: usize, pgsize: usize, pgcount: usize) -> usize {
210        // SAFETY: The `unmap_pages` function in `io_pgtable_ops` is never null.
211        let unmap_pages = unsafe { (*self.raw_ops()).unmap_pages.unwrap_unchecked() };
212
213        // SAFETY: The safety requirements of this method are sufficient to call `unmap_pages`.
214        unsafe { (unmap_pages)(self.raw_ops(), iova, pgsize, pgcount, core::ptr::null_mut()) }
215    }
216}
217
218// For the initial users of these rust bindings, the GPU FW is managing the IOTLB and performs all
219// required invalidations using a range. There is no need for it get ARM style invalidation
220// instructions from the page table code.
221//
222// Support for flushing the TLB with ARM style invalidation instructions may be added in the
223// future.
224static NOOP_FLUSH_OPS: bindings::iommu_flush_ops = bindings::iommu_flush_ops {
225    tlb_flush_all: Some(rust_tlb_flush_all_noop),
226    tlb_flush_walk: Some(rust_tlb_flush_walk_noop),
227    tlb_add_page: None,
228};
229
230#[no_mangle]
231extern "C" fn rust_tlb_flush_all_noop(_cookie: *mut core::ffi::c_void) {}
232
233#[no_mangle]
234extern "C" fn rust_tlb_flush_walk_noop(
235    _iova: usize,
236    _size: usize,
237    _granule: usize,
238    _cookie: *mut core::ffi::c_void,
239) {
240}
241
242impl<F: IoPageTableFmt> Drop for IoPageTable<F> {
243    fn drop(&mut self) {
244        // SAFETY: The caller of `Self::ttbr()` promised that the page table is not live when this
245        // destructor runs.
246        unsafe { bindings::free_io_pgtable_ops(self.raw_ops()) };
247    }
248}
249
250/// The `ARM_64_LPAE_S1` page table format.
251pub enum ARM64LPAES1 {}
252
253impl IoPageTableFmt for ARM64LPAES1 {
254    const FORMAT: io_pgtable_fmt = bindings::io_pgtable_fmt_ARM_64_LPAE_S1 as io_pgtable_fmt;
255}
256
257impl IoPageTable<ARM64LPAES1> {
258    /// Access the `ttbr` field of the configuration.
259    ///
260    /// This is the physical address of the page table, which may be passed to the device that
261    /// needs to use it.
262    ///
263    /// # Safety
264    ///
265    /// The caller must ensure that the device stops using the page table before dropping it.
266    #[inline]
267    pub unsafe fn ttbr(&self) -> u64 {
268        // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`.
269        unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.ttbr }
270    }
271
272    /// Access the `mair` field of the configuration.
273    #[inline]
274    pub fn mair(&self) -> u64 {
275        // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`.
276        unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.mair }
277    }
278}
kernel/iommu/pgtable.rs

kernel/iommu/
pgtable.rs
