kernel/dma.rs
1// SPDX-License-Identifier: GPL-2.0
2
3//! Direct memory access (DMA).
4//!
5//! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h)
6
7use crate::{
8 bindings, build_assert, device,
9 device::{Bound, Core},
10 error::{to_result, Result},
11 prelude::*,
12 sync::aref::ARef,
13 transmute::{AsBytes, FromBytes},
14};
15
16/// DMA address type.
17///
18/// Represents a bus address used for Direct Memory Access (DMA) operations.
19///
20/// This is an alias of the kernel's `dma_addr_t`, which may be `u32` or `u64` depending on
21/// `CONFIG_ARCH_DMA_ADDR_T_64BIT`.
22///
23/// Note that this may be `u64` even on 32-bit architectures.
24pub type DmaAddress = bindings::dma_addr_t;
25
26/// Trait to be implemented by DMA capable bus devices.
27///
28/// The [`dma::Device`](Device) trait should be implemented by bus specific device representations,
29/// where the underlying bus is DMA capable, such as [`pci::Device`](::kernel::pci::Device) or
30/// [`platform::Device`](::kernel::platform::Device).
31pub trait Device: AsRef<device::Device<Core>> {
32 /// Set up the device's DMA streaming addressing capabilities.
33 ///
34 /// This method is usually called once from `probe()` as soon as the device capabilities are
35 /// known.
36 ///
37 /// # Safety
38 ///
39 /// This method must not be called concurrently with any DMA allocation or mapping primitives,
40 /// such as [`CoherentAllocation::alloc_attrs`].
41 unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result {
42 // SAFETY:
43 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
44 // - The safety requirement of this function guarantees that there are no concurrent calls
45 // to DMA allocation and mapping primitives using this mask.
46 to_result(unsafe { bindings::dma_set_mask(self.as_ref().as_raw(), mask.value()) })
47 }
48
49 /// Set up the device's DMA coherent addressing capabilities.
50 ///
51 /// This method is usually called once from `probe()` as soon as the device capabilities are
52 /// known.
53 ///
54 /// # Safety
55 ///
56 /// This method must not be called concurrently with any DMA allocation or mapping primitives,
57 /// such as [`CoherentAllocation::alloc_attrs`].
58 unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result {
59 // SAFETY:
60 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
61 // - The safety requirement of this function guarantees that there are no concurrent calls
62 // to DMA allocation and mapping primitives using this mask.
63 to_result(unsafe { bindings::dma_set_coherent_mask(self.as_ref().as_raw(), mask.value()) })
64 }
65
66 /// Set up the device's DMA addressing capabilities.
67 ///
68 /// This is a combination of [`Device::dma_set_mask`] and [`Device::dma_set_coherent_mask`].
69 ///
70 /// This method is usually called once from `probe()` as soon as the device capabilities are
71 /// known.
72 ///
73 /// # Safety
74 ///
75 /// This method must not be called concurrently with any DMA allocation or mapping primitives,
76 /// such as [`CoherentAllocation::alloc_attrs`].
77 unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result {
78 // SAFETY:
79 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
80 // - The safety requirement of this function guarantees that there are no concurrent calls
81 // to DMA allocation and mapping primitives using this mask.
82 to_result(unsafe {
83 bindings::dma_set_mask_and_coherent(self.as_ref().as_raw(), mask.value())
84 })
85 }
86}
87
88/// A DMA mask that holds a bitmask with the lowest `n` bits set.
89///
90/// Use [`DmaMask::new`] or [`DmaMask::try_new`] to construct a value. Values
91/// are guaranteed to never exceed the bit width of `u64`.
92///
93/// This is the Rust equivalent of the C macro `DMA_BIT_MASK()`.
94#[derive(Debug, Clone, Copy, PartialEq, Eq)]
95pub struct DmaMask(u64);
96
97impl DmaMask {
98 /// Constructs a `DmaMask` with the lowest `n` bits set to `1`.
99 ///
100 /// For `n <= 64`, sets exactly the lowest `n` bits.
101 /// For `n > 64`, results in a build error.
102 ///
103 /// # Examples
104 ///
105 /// ```
106 /// use kernel::dma::DmaMask;
107 ///
108 /// let mask0 = DmaMask::new::<0>();
109 /// assert_eq!(mask0.value(), 0);
110 ///
111 /// let mask1 = DmaMask::new::<1>();
112 /// assert_eq!(mask1.value(), 0b1);
113 ///
114 /// let mask64 = DmaMask::new::<64>();
115 /// assert_eq!(mask64.value(), u64::MAX);
116 ///
117 /// // Build failure.
118 /// // let mask_overflow = DmaMask::new::<100>();
119 /// ```
120 #[inline]
121 pub const fn new<const N: u32>() -> Self {
122 let Ok(mask) = Self::try_new(N) else {
123 build_error!("Invalid DMA Mask.");
124 };
125
126 mask
127 }
128
129 /// Constructs a `DmaMask` with the lowest `n` bits set to `1`.
130 ///
131 /// For `n <= 64`, sets exactly the lowest `n` bits.
132 /// For `n > 64`, returns [`EINVAL`].
133 ///
134 /// # Examples
135 ///
136 /// ```
137 /// use kernel::dma::DmaMask;
138 ///
139 /// let mask0 = DmaMask::try_new(0)?;
140 /// assert_eq!(mask0.value(), 0);
141 ///
142 /// let mask1 = DmaMask::try_new(1)?;
143 /// assert_eq!(mask1.value(), 0b1);
144 ///
145 /// let mask64 = DmaMask::try_new(64)?;
146 /// assert_eq!(mask64.value(), u64::MAX);
147 ///
148 /// let mask_overflow = DmaMask::try_new(100);
149 /// assert!(mask_overflow.is_err());
150 /// # Ok::<(), Error>(())
151 /// ```
152 #[inline]
153 pub const fn try_new(n: u32) -> Result<Self> {
154 Ok(Self(match n {
155 0 => 0,
156 1..=64 => u64::MAX >> (64 - n),
157 _ => return Err(EINVAL),
158 }))
159 }
160
161 /// Returns the underlying `u64` bitmask value.
162 #[inline]
163 pub const fn value(&self) -> u64 {
164 self.0
165 }
166}
167
168/// Possible attributes associated with a DMA mapping.
169///
170/// They can be combined with the operators `|`, `&`, and `!`.
171///
172/// Values can be used from the [`attrs`] module.
173///
174/// # Examples
175///
176/// ```
177/// # use kernel::device::{Bound, Device};
178/// use kernel::dma::{attrs::*, CoherentAllocation};
179///
180/// # fn test(dev: &Device<Bound>) -> Result {
181/// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN;
182/// let c: CoherentAllocation<u64> =
183/// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, attribs)?;
184/// # Ok::<(), Error>(()) }
185/// ```
186#[derive(Clone, Copy, PartialEq)]
187#[repr(transparent)]
188pub struct Attrs(u32);
189
190impl Attrs {
191 /// Get the raw representation of this attribute.
192 pub(crate) fn as_raw(self) -> crate::ffi::c_ulong {
193 self.0 as crate::ffi::c_ulong
194 }
195
196 /// Check whether `flags` is contained in `self`.
197 pub fn contains(self, flags: Attrs) -> bool {
198 (self & flags) == flags
199 }
200}
201
202impl core::ops::BitOr for Attrs {
203 type Output = Self;
204 fn bitor(self, rhs: Self) -> Self::Output {
205 Self(self.0 | rhs.0)
206 }
207}
208
209impl core::ops::BitAnd for Attrs {
210 type Output = Self;
211 fn bitand(self, rhs: Self) -> Self::Output {
212 Self(self.0 & rhs.0)
213 }
214}
215
216impl core::ops::Not for Attrs {
217 type Output = Self;
218 fn not(self) -> Self::Output {
219 Self(!self.0)
220 }
221}
222
223/// DMA mapping attributes.
224pub mod attrs {
225 use super::Attrs;
226
227 /// Specifies that reads and writes to the mapping may be weakly ordered, that is that reads
228 /// and writes may pass each other.
229 pub const DMA_ATTR_WEAK_ORDERING: Attrs = Attrs(bindings::DMA_ATTR_WEAK_ORDERING);
230
231 /// Specifies that writes to the mapping may be buffered to improve performance.
232 pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE);
233
234 /// Lets the platform to avoid creating a kernel virtual mapping for the allocated buffer.
235 pub const DMA_ATTR_NO_KERNEL_MAPPING: Attrs = Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING);
236
237 /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming
238 /// that it has been already transferred to 'device' domain.
239 pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC);
240
241 /// Forces contiguous allocation of the buffer in physical memory.
242 pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS);
243
244 /// Hints DMA-mapping subsystem that it's probably not worth the time to try
245 /// to allocate memory to in a way that gives better TLB efficiency.
246 pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES);
247
248 /// This tells the DMA-mapping subsystem to suppress allocation failure reports (similarly to
249 /// `__GFP_NOWARN`).
250 pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN);
251
252 /// Indicates that the buffer is fully accessible at an elevated privilege level (and
253 /// ideally inaccessible or at least read-only at lesser-privileged levels).
254 pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED);
255
256 /// Indicates that the buffer is MMIO memory.
257 pub const DMA_ATTR_MMIO: Attrs = Attrs(bindings::DMA_ATTR_MMIO);
258}
259
260/// DMA data direction.
261///
262/// Corresponds to the C [`enum dma_data_direction`].
263///
264/// [`enum dma_data_direction`]: srctree/include/linux/dma-direction.h
265#[derive(Copy, Clone, PartialEq, Eq, Debug)]
266#[repr(u32)]
267pub enum DataDirection {
268 /// The DMA mapping is for bidirectional data transfer.
269 ///
270 /// This is used when the buffer can be both read from and written to by the device.
271 /// The cache for the corresponding memory region is both flushed and invalidated.
272 Bidirectional = Self::const_cast(bindings::dma_data_direction_DMA_BIDIRECTIONAL),
273
274 /// The DMA mapping is for data transfer from memory to the device (write).
275 ///
276 /// The CPU has prepared data in the buffer, and the device will read it.
277 /// The cache for the corresponding memory region is flushed before device access.
278 ToDevice = Self::const_cast(bindings::dma_data_direction_DMA_TO_DEVICE),
279
280 /// The DMA mapping is for data transfer from the device to memory (read).
281 ///
282 /// The device will write data into the buffer for the CPU to read.
283 /// The cache for the corresponding memory region is invalidated before CPU access.
284 FromDevice = Self::const_cast(bindings::dma_data_direction_DMA_FROM_DEVICE),
285
286 /// The DMA mapping is not for data transfer.
287 ///
288 /// This is primarily for debugging purposes. With this direction, the DMA mapping API
289 /// will not perform any cache coherency operations.
290 None = Self::const_cast(bindings::dma_data_direction_DMA_NONE),
291}
292
293impl DataDirection {
294 /// Casts the bindgen-generated enum type to a `u32` at compile time.
295 ///
296 /// This function will cause a compile-time error if the underlying value of the
297 /// C enum is out of bounds for `u32`.
298 const fn const_cast(val: bindings::dma_data_direction) -> u32 {
299 // CAST: The C standard allows compilers to choose different integer types for enums.
300 // To safely check the value, we cast it to a wide signed integer type (`i128`)
301 // which can hold any standard C integer enum type without truncation.
302 let wide_val = val as i128;
303
304 // Check if the value is outside the valid range for the target type `u32`.
305 // CAST: `u32::MAX` is cast to `i128` to match the type of `wide_val` for the comparison.
306 if wide_val < 0 || wide_val > u32::MAX as i128 {
307 // Trigger a compile-time error in a const context.
308 build_error!("C enum value is out of bounds for the target type `u32`.");
309 }
310
311 // CAST: This cast is valid because the check above guarantees that `wide_val`
312 // is within the representable range of `u32`.
313 wide_val as u32
314 }
315}
316
317impl From<DataDirection> for bindings::dma_data_direction {
318 /// Returns the raw representation of [`enum dma_data_direction`].
319 fn from(direction: DataDirection) -> Self {
320 // CAST: `direction as u32` gets the underlying representation of our `#[repr(u32)]` enum.
321 // The subsequent cast to `Self` (the bindgen type) assumes the C enum is compatible
322 // with the enum variants of `DataDirection`, which is a valid assumption given our
323 // compile-time checks.
324 direction as u32 as Self
325 }
326}
327
328/// An abstraction of the `dma_alloc_coherent` API.
329///
330/// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map
331/// large coherent DMA regions.
332///
333/// A [`CoherentAllocation`] instance contains a pointer to the allocated region (in the
334/// processor's virtual address space) and the device address which can be given to the device
335/// as the DMA address base of the region. The region is released once [`CoherentAllocation`]
336/// is dropped.
337///
338/// # Invariants
339///
340/// - For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer
341/// to an allocated region of coherent memory and `dma_handle` is the DMA address base of the
342/// region.
343/// - The size in bytes of the allocation is equal to `size_of::<T> * count`.
344/// - `size_of::<T> * count` fits into a `usize`.
345// TODO
346//
347// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness
348// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure
349// that device resources can never survive device unbind.
350//
351// However, it is neither desirable nor necessary to protect the allocated memory of the DMA
352// allocation from surviving device unbind; it would require RCU read side critical sections to
353// access the memory, which may require subsequent unnecessary copies.
354//
355// Hence, find a way to revoke the device resources of a `CoherentAllocation`, but not the
356// entire `CoherentAllocation` including the allocated memory itself.
357pub struct CoherentAllocation<T: AsBytes + FromBytes> {
358 dev: ARef<device::Device>,
359 dma_handle: DmaAddress,
360 count: usize,
361 cpu_addr: *mut T,
362 dma_attrs: Attrs,
363}
364
365impl<T: AsBytes + FromBytes> CoherentAllocation<T> {
366 /// Allocates a region of `size_of::<T> * count` of coherent memory.
367 ///
368 /// # Examples
369 ///
370 /// ```
371 /// # use kernel::device::{Bound, Device};
372 /// use kernel::dma::{attrs::*, CoherentAllocation};
373 ///
374 /// # fn test(dev: &Device<Bound>) -> Result {
375 /// let c: CoherentAllocation<u64> =
376 /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
377 /// # Ok::<(), Error>(()) }
378 /// ```
379 pub fn alloc_attrs(
380 dev: &device::Device<Bound>,
381 count: usize,
382 gfp_flags: kernel::alloc::Flags,
383 dma_attrs: Attrs,
384 ) -> Result<CoherentAllocation<T>> {
385 build_assert!(
386 core::mem::size_of::<T>() > 0,
387 "It doesn't make sense for the allocated type to be a ZST"
388 );
389
390 let size = count
391 .checked_mul(core::mem::size_of::<T>())
392 .ok_or(EOVERFLOW)?;
393 let mut dma_handle = 0;
394 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
395 let ret = unsafe {
396 bindings::dma_alloc_attrs(
397 dev.as_raw(),
398 size,
399 &mut dma_handle,
400 gfp_flags.as_raw(),
401 dma_attrs.as_raw(),
402 )
403 };
404 if ret.is_null() {
405 return Err(ENOMEM);
406 }
407 // INVARIANT:
408 // - We just successfully allocated a coherent region which is accessible for
409 // `count` elements, hence the cpu address is valid. We also hold a refcounted reference
410 // to the device.
411 // - The allocated `size` is equal to `size_of::<T> * count`.
412 // - The allocated `size` fits into a `usize`.
413 Ok(Self {
414 dev: dev.into(),
415 dma_handle,
416 count,
417 cpu_addr: ret.cast::<T>(),
418 dma_attrs,
419 })
420 }
421
422 /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the
423 /// `dma_attrs` is 0 by default.
424 pub fn alloc_coherent(
425 dev: &device::Device<Bound>,
426 count: usize,
427 gfp_flags: kernel::alloc::Flags,
428 ) -> Result<CoherentAllocation<T>> {
429 CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0))
430 }
431
432 /// Returns the number of elements `T` in this allocation.
433 ///
434 /// Note that this is not the size of the allocation in bytes, which is provided by
435 /// [`Self::size`].
436 pub fn count(&self) -> usize {
437 self.count
438 }
439
440 /// Returns the size in bytes of this allocation.
441 pub fn size(&self) -> usize {
442 // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits into
443 // a `usize`.
444 self.count * core::mem::size_of::<T>()
445 }
446
447 /// Returns the base address to the allocated region in the CPU's virtual address space.
448 pub fn start_ptr(&self) -> *const T {
449 self.cpu_addr
450 }
451
452 /// Returns the base address to the allocated region in the CPU's virtual address space as
453 /// a mutable pointer.
454 pub fn start_ptr_mut(&mut self) -> *mut T {
455 self.cpu_addr
456 }
457
458 /// Returns a DMA handle which may be given to the device as the DMA address base of
459 /// the region.
460 pub fn dma_handle(&self) -> DmaAddress {
461 self.dma_handle
462 }
463
464 /// Returns a DMA handle starting at `offset` (in units of `T`) which may be given to the
465 /// device as the DMA address base of the region.
466 ///
467 /// Returns `EINVAL` if `offset` is not within the bounds of the allocation.
468 pub fn dma_handle_with_offset(&self, offset: usize) -> Result<DmaAddress> {
469 if offset >= self.count {
470 Err(EINVAL)
471 } else {
472 // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits
473 // into a `usize`, and `offset` is inferior to `count`.
474 Ok(self.dma_handle + (offset * core::mem::size_of::<T>()) as DmaAddress)
475 }
476 }
477
478 /// Common helper to validate a range applied from the allocated region in the CPU's virtual
479 /// address space.
480 fn validate_range(&self, offset: usize, count: usize) -> Result {
481 if offset.checked_add(count).ok_or(EOVERFLOW)? > self.count {
482 return Err(EINVAL);
483 }
484 Ok(())
485 }
486
487 /// Returns the data from the region starting from `offset` as a slice.
488 /// `offset` and `count` are in units of `T`, not the number of bytes.
489 ///
490 /// For ringbuffer type of r/w access or use-cases where the pointer to the live data is needed,
491 /// [`CoherentAllocation::start_ptr`] or [`CoherentAllocation::start_ptr_mut`] could be used
492 /// instead.
493 ///
494 /// # Safety
495 ///
496 /// * Callers must ensure that the device does not read/write to/from memory while the returned
497 /// slice is live.
498 /// * Callers must ensure that this call does not race with a write to the same region while
499 /// the returned slice is live.
500 pub unsafe fn as_slice(&self, offset: usize, count: usize) -> Result<&[T]> {
501 self.validate_range(offset, count)?;
502 // SAFETY:
503 // - The pointer is valid due to type invariant on `CoherentAllocation`,
504 // we've just checked that the range and index is within bounds. The immutability of the
505 // data is also guaranteed by the safety requirements of the function.
506 // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked
507 // that `self.count` won't overflow early in the constructor.
508 Ok(unsafe { core::slice::from_raw_parts(self.cpu_addr.add(offset), count) })
509 }
510
511 /// Performs the same functionality as [`CoherentAllocation::as_slice`], except that a mutable
512 /// slice is returned.
513 ///
514 /// # Safety
515 ///
516 /// * Callers must ensure that the device does not read/write to/from memory while the returned
517 /// slice is live.
518 /// * Callers must ensure that this call does not race with a read or write to the same region
519 /// while the returned slice is live.
520 pub unsafe fn as_slice_mut(&mut self, offset: usize, count: usize) -> Result<&mut [T]> {
521 self.validate_range(offset, count)?;
522 // SAFETY:
523 // - The pointer is valid due to type invariant on `CoherentAllocation`,
524 // we've just checked that the range and index is within bounds. The immutability of the
525 // data is also guaranteed by the safety requirements of the function.
526 // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked
527 // that `self.count` won't overflow early in the constructor.
528 Ok(unsafe { core::slice::from_raw_parts_mut(self.cpu_addr.add(offset), count) })
529 }
530
531 /// Writes data to the region starting from `offset`. `offset` is in units of `T`, not the
532 /// number of bytes.
533 ///
534 /// # Safety
535 ///
536 /// * Callers must ensure that the device does not read/write to/from memory while the returned
537 /// slice is live.
538 /// * Callers must ensure that this call does not race with a read or write to the same region
539 /// that overlaps with this write.
540 ///
541 /// # Examples
542 ///
543 /// ```
544 /// # fn test(alloc: &mut kernel::dma::CoherentAllocation<u8>) -> Result {
545 /// let somedata: [u8; 4] = [0xf; 4];
546 /// let buf: &[u8] = &somedata;
547 /// // SAFETY: There is no concurrent HW operation on the device and no other R/W access to the
548 /// // region.
549 /// unsafe { alloc.write(buf, 0)?; }
550 /// # Ok::<(), Error>(()) }
551 /// ```
552 pub unsafe fn write(&mut self, src: &[T], offset: usize) -> Result {
553 self.validate_range(offset, src.len())?;
554 // SAFETY:
555 // - The pointer is valid due to type invariant on `CoherentAllocation`
556 // and we've just checked that the range and index is within bounds.
557 // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked
558 // that `self.count` won't overflow early in the constructor.
559 unsafe {
560 core::ptr::copy_nonoverlapping(src.as_ptr(), self.cpu_addr.add(offset), src.len())
561 };
562 Ok(())
563 }
564
565 /// Returns a pointer to an element from the region with bounds checking. `offset` is in
566 /// units of `T`, not the number of bytes.
567 ///
568 /// Public but hidden since it should only be used from [`dma_read`] and [`dma_write`] macros.
569 #[doc(hidden)]
570 pub fn item_from_index(&self, offset: usize) -> Result<*mut T> {
571 if offset >= self.count {
572 return Err(EINVAL);
573 }
574 // SAFETY:
575 // - The pointer is valid due to type invariant on `CoherentAllocation`
576 // and we've just checked that the range and index is within bounds.
577 // - `offset` can't overflow since it is smaller than `self.count` and we've checked
578 // that `self.count` won't overflow early in the constructor.
579 Ok(unsafe { self.cpu_addr.add(offset) })
580 }
581
582 /// Reads the value of `field` and ensures that its type is [`FromBytes`].
583 ///
584 /// # Safety
585 ///
586 /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is
587 /// validated beforehand.
588 ///
589 /// Public but hidden since it should only be used from [`dma_read`] macro.
590 #[doc(hidden)]
591 pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F {
592 // SAFETY:
593 // - By the safety requirements field is valid.
594 // - Using read_volatile() here is not sound as per the usual rules, the usage here is
595 // a special exception with the following notes in place. When dealing with a potential
596 // race from a hardware or code outside kernel (e.g. user-space program), we need that
597 // read on a valid memory is not UB. Currently read_volatile() is used for this, and the
598 // rationale behind is that it should generate the same code as READ_ONCE() which the
599 // kernel already relies on to avoid UB on data races. Note that the usage of
600 // read_volatile() is limited to this particular case, it cannot be used to prevent
601 // the UB caused by racing between two kernel functions nor do they provide atomicity.
602 unsafe { field.read_volatile() }
603 }
604
605 /// Writes a value to `field` and ensures that its type is [`AsBytes`].
606 ///
607 /// # Safety
608 ///
609 /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is
610 /// validated beforehand.
611 ///
612 /// Public but hidden since it should only be used from [`dma_write`] macro.
613 #[doc(hidden)]
614 pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) {
615 // SAFETY:
616 // - By the safety requirements field is valid.
617 // - Using write_volatile() here is not sound as per the usual rules, the usage here is
618 // a special exception with the following notes in place. When dealing with a potential
619 // race from a hardware or code outside kernel (e.g. user-space program), we need that
620 // write on a valid memory is not UB. Currently write_volatile() is used for this, and the
621 // rationale behind is that it should generate the same code as WRITE_ONCE() which the
622 // kernel already relies on to avoid UB on data races. Note that the usage of
623 // write_volatile() is limited to this particular case, it cannot be used to prevent
624 // the UB caused by racing between two kernel functions nor do they provide atomicity.
625 unsafe { field.write_volatile(val) }
626 }
627}
628
629/// Note that the device configured to do DMA must be halted before this object is dropped.
630impl<T: AsBytes + FromBytes> Drop for CoherentAllocation<T> {
631 fn drop(&mut self) {
632 let size = self.count * core::mem::size_of::<T>();
633 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
634 // The cpu address, and the dma handle are valid due to the type invariants on
635 // `CoherentAllocation`.
636 unsafe {
637 bindings::dma_free_attrs(
638 self.dev.as_raw(),
639 size,
640 self.cpu_addr.cast(),
641 self.dma_handle,
642 self.dma_attrs.as_raw(),
643 )
644 }
645 }
646}
647
648// SAFETY: It is safe to send a `CoherentAllocation` to another thread if `T`
649// can be sent to another thread.
650unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {}
651
652/// Reads a field of an item from an allocated region of structs.
653///
654/// # Examples
655///
656/// ```
657/// use kernel::device::Device;
658/// use kernel::dma::{attrs::*, CoherentAllocation};
659///
660/// struct MyStruct { field: u32, }
661///
662/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
663/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
664/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
665/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
666///
667/// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
668/// let whole = kernel::dma_read!(alloc[2]);
669/// let field = kernel::dma_read!(alloc[1].field);
670/// # Ok::<(), Error>(()) }
671/// ```
672#[macro_export]
673macro_rules! dma_read {
674 ($dma:expr, $idx: expr, $($field:tt)*) => {{
675 (|| -> ::core::result::Result<_, $crate::error::Error> {
676 let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
677 // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be
678 // dereferenced. The compiler also further validates the expression on whether `field`
679 // is a member of `item` when expanded by the macro.
680 unsafe {
681 let ptr_field = ::core::ptr::addr_of!((*item) $($field)*);
682 ::core::result::Result::Ok(
683 $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field)
684 )
685 }
686 })()
687 }};
688 ($dma:ident [ $idx:expr ] $($field:tt)* ) => {
689 $crate::dma_read!($dma, $idx, $($field)*)
690 };
691 ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {
692 $crate::dma_read!($($dma).*, $idx, $($field)*)
693 };
694}
695
696/// Writes to a field of an item from an allocated region of structs.
697///
698/// # Examples
699///
700/// ```
701/// use kernel::device::Device;
702/// use kernel::dma::{attrs::*, CoherentAllocation};
703///
704/// struct MyStruct { member: u32, }
705///
706/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
707/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
708/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
709/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
710///
711/// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
712/// kernel::dma_write!(alloc[2].member = 0xf);
713/// kernel::dma_write!(alloc[1] = MyStruct { member: 0xf });
714/// # Ok::<(), Error>(()) }
715/// ```
716#[macro_export]
717macro_rules! dma_write {
718 ($dma:ident [ $idx:expr ] $($field:tt)*) => {{
719 $crate::dma_write!($dma, $idx, $($field)*)
720 }};
721 ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{
722 $crate::dma_write!($($dma).*, $idx, $($field)*)
723 }};
724 ($dma:expr, $idx: expr, = $val:expr) => {
725 (|| -> ::core::result::Result<_, $crate::error::Error> {
726 let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
727 // SAFETY: `item_from_index` ensures that `item` is always a valid item.
728 unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) }
729 ::core::result::Result::Ok(())
730 })()
731 };
732 ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => {
733 (|| -> ::core::result::Result<_, $crate::error::Error> {
734 let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
735 // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be
736 // dereferenced. The compiler also further validates the expression on whether `field`
737 // is a member of `item` when expanded by the macro.
738 unsafe {
739 let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*);
740 $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val)
741 }
742 ::core::result::Result::Ok(())
743 })()
744 };
745}