kernel/
pci.rs

1// SPDX-License-Identifier: GPL-2.0
2
3//! Abstractions for the PCI bus.
4//!
5//! C header: [`include/linux/pci.h`](srctree/include/linux/pci.h)
6
7use crate::{
8    bindings, container_of, device,
9    device_id::{RawDeviceId, RawDeviceIdIndex},
10    devres::Devres,
11    driver,
12    error::{from_result, to_result, Result},
13    io::Io,
14    io::IoRaw,
15    str::CStr,
16    types::{ARef, Opaque},
17    ThisModule,
18};
19use core::{
20    marker::PhantomData,
21    ops::Deref,
22    ptr::{addr_of_mut, NonNull},
23};
24use kernel::prelude::*;
25
26/// An adapter for the registration of PCI drivers.
27pub struct Adapter<T: Driver>(T);
28
29// SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if
30// a preceding call to `register` has been successful.
31unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {
32    type RegType = bindings::pci_driver;
33
34    unsafe fn register(
35        pdrv: &Opaque<Self::RegType>,
36        name: &'static CStr,
37        module: &'static ThisModule,
38    ) -> Result {
39        // SAFETY: It's safe to set the fields of `struct pci_driver` on initialization.
40        unsafe {
41            (*pdrv.get()).name = name.as_char_ptr();
42            (*pdrv.get()).probe = Some(Self::probe_callback);
43            (*pdrv.get()).remove = Some(Self::remove_callback);
44            (*pdrv.get()).id_table = T::ID_TABLE.as_ptr();
45        }
46
47        // SAFETY: `pdrv` is guaranteed to be a valid `RegType`.
48        to_result(unsafe {
49            bindings::__pci_register_driver(pdrv.get(), module.0, name.as_char_ptr())
50        })
51    }
52
53    unsafe fn unregister(pdrv: &Opaque<Self::RegType>) {
54        // SAFETY: `pdrv` is guaranteed to be a valid `RegType`.
55        unsafe { bindings::pci_unregister_driver(pdrv.get()) }
56    }
57}
58
59impl<T: Driver + 'static> Adapter<T> {
60    extern "C" fn probe_callback(
61        pdev: *mut bindings::pci_dev,
62        id: *const bindings::pci_device_id,
63    ) -> kernel::ffi::c_int {
64        // SAFETY: The PCI bus only ever calls the probe callback with a valid pointer to a
65        // `struct pci_dev`.
66        //
67        // INVARIANT: `pdev` is valid for the duration of `probe_callback()`.
68        let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() };
69
70        // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct pci_device_id` and
71        // does not add additional invariants, so it's safe to transmute.
72        let id = unsafe { &*id.cast::<DeviceId>() };
73        let info = T::ID_TABLE.info(id.index());
74
75        from_result(|| {
76            let data = T::probe(pdev, info)?;
77
78            pdev.as_ref().set_drvdata(data);
79            Ok(0)
80        })
81    }
82
83    extern "C" fn remove_callback(pdev: *mut bindings::pci_dev) {
84        // SAFETY: The PCI bus only ever calls the remove callback with a valid pointer to a
85        // `struct pci_dev`.
86        //
87        // INVARIANT: `pdev` is valid for the duration of `remove_callback()`.
88        let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() };
89
90        // SAFETY: `remove_callback` is only ever called after a successful call to
91        // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called
92        // and stored a `Pin<KBox<T>>`.
93        let data = unsafe { pdev.as_ref().drvdata_obtain::<Pin<KBox<T>>>() };
94
95        T::unbind(pdev, data.as_ref());
96    }
97}
98
99/// Declares a kernel module that exposes a single PCI driver.
100///
101/// # Examples
102///
103///```ignore
104/// kernel::module_pci_driver! {
105///     type: MyDriver,
106///     name: "Module name",
107///     authors: ["Author name"],
108///     description: "Description",
109///     license: "GPL v2",
110/// }
111///```
112#[macro_export]
113macro_rules! module_pci_driver {
114($($f:tt)*) => {
115    $crate::module_driver!(<T>, $crate::pci::Adapter<T>, { $($f)* });
116};
117}
118
119/// Abstraction for the PCI device ID structure ([`struct pci_device_id`]).
120///
121/// [`struct pci_device_id`]: https://docs.kernel.org/PCI/pci.html#c.pci_device_id
122#[repr(transparent)]
123#[derive(Clone, Copy)]
124pub struct DeviceId(bindings::pci_device_id);
125
126impl DeviceId {
127    const PCI_ANY_ID: u32 = !0;
128
129    /// Equivalent to C's `PCI_DEVICE` macro.
130    ///
131    /// Create a new `pci::DeviceId` from a vendor and device ID number.
132    pub const fn from_id(vendor: u32, device: u32) -> Self {
133        Self(bindings::pci_device_id {
134            vendor,
135            device,
136            subvendor: DeviceId::PCI_ANY_ID,
137            subdevice: DeviceId::PCI_ANY_ID,
138            class: 0,
139            class_mask: 0,
140            driver_data: 0,
141            override_only: 0,
142        })
143    }
144
145    /// Equivalent to C's `PCI_DEVICE_CLASS` macro.
146    ///
147    /// Create a new `pci::DeviceId` from a class number and mask.
148    pub const fn from_class(class: u32, class_mask: u32) -> Self {
149        Self(bindings::pci_device_id {
150            vendor: DeviceId::PCI_ANY_ID,
151            device: DeviceId::PCI_ANY_ID,
152            subvendor: DeviceId::PCI_ANY_ID,
153            subdevice: DeviceId::PCI_ANY_ID,
154            class,
155            class_mask,
156            driver_data: 0,
157            override_only: 0,
158        })
159    }
160}
161
162// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `pci_device_id` and does not add
163// additional invariants, so it's safe to transmute to `RawType`.
164unsafe impl RawDeviceId for DeviceId {
165    type RawType = bindings::pci_device_id;
166}
167
168// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field.
169unsafe impl RawDeviceIdIndex for DeviceId {
170    const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::pci_device_id, driver_data);
171
172    fn index(&self) -> usize {
173        self.0.driver_data
174    }
175}
176
177/// `IdTable` type for PCI.
178pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>;
179
180/// Create a PCI `IdTable` with its alias for modpost.
181#[macro_export]
182macro_rules! pci_device_table {
183    ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => {
184        const $table_name: $crate::device_id::IdArray<
185            $crate::pci::DeviceId,
186            $id_info_type,
187            { $table_data.len() },
188        > = $crate::device_id::IdArray::new($table_data);
189
190        $crate::module_device_table!("pci", $module_table_name, $table_name);
191    };
192}
193
194/// The PCI driver trait.
195///
196/// # Examples
197///
198///```
199/// # use kernel::{bindings, device::Core, pci};
200///
201/// struct MyDriver;
202///
203/// kernel::pci_device_table!(
204///     PCI_TABLE,
205///     MODULE_PCI_TABLE,
206///     <MyDriver as pci::Driver>::IdInfo,
207///     [
208///         (
209///             pci::DeviceId::from_id(bindings::PCI_VENDOR_ID_REDHAT, bindings::PCI_ANY_ID as u32),
210///             (),
211///         )
212///     ]
213/// );
214///
215/// impl pci::Driver for MyDriver {
216///     type IdInfo = ();
217///     const ID_TABLE: pci::IdTable<Self::IdInfo> = &PCI_TABLE;
218///
219///     fn probe(
220///         _pdev: &pci::Device<Core>,
221///         _id_info: &Self::IdInfo,
222///     ) -> Result<Pin<KBox<Self>>> {
223///         Err(ENODEV)
224///     }
225/// }
226///```
227/// Drivers must implement this trait in order to get a PCI driver registered. Please refer to the
228/// `Adapter` documentation for an example.
229pub trait Driver: Send {
230    /// The type holding information about each device id supported by the driver.
231    // TODO: Use `associated_type_defaults` once stabilized:
232    //
233    // ```
234    // type IdInfo: 'static = ();
235    // ```
236    type IdInfo: 'static;
237
238    /// The table of device ids supported by the driver.
239    const ID_TABLE: IdTable<Self::IdInfo>;
240
241    /// PCI driver probe.
242    ///
243    /// Called when a new platform device is added or discovered.
244    /// Implementers should attempt to initialize the device here.
245    fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> Result<Pin<KBox<Self>>>;
246
247    /// Platform driver unbind.
248    ///
249    /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback
250    /// is optional.
251    ///
252    /// This callback serves as a place for drivers to perform teardown operations that require a
253    /// `&Device<Core>` or `&Device<Bound>` reference. For instance, drivers may try to perform I/O
254    /// operations to gracefully tear down the device.
255    ///
256    /// Otherwise, release operations for driver resources should be performed in `Self::drop`.
257    fn unbind(dev: &Device<device::Core>, this: Pin<&Self>) {
258        let _ = (dev, this);
259    }
260}
261
262/// The PCI device representation.
263///
264/// This structure represents the Rust abstraction for a C `struct pci_dev`. The implementation
265/// abstracts the usage of an already existing C `struct pci_dev` within Rust code that we get
266/// passed from the C side.
267///
268/// # Invariants
269///
270/// A [`Device`] instance represents a valid `struct pci_dev` created by the C portion of the
271/// kernel.
272#[repr(transparent)]
273pub struct Device<Ctx: device::DeviceContext = device::Normal>(
274    Opaque<bindings::pci_dev>,
275    PhantomData<Ctx>,
276);
277
278/// A PCI BAR to perform I/O-Operations on.
279///
280/// # Invariants
281///
282/// `Bar` always holds an `IoRaw` inststance that holds a valid pointer to the start of the I/O
283/// memory mapped PCI bar and its size.
284pub struct Bar<const SIZE: usize = 0> {
285    pdev: ARef<Device>,
286    io: IoRaw<SIZE>,
287    num: i32,
288}
289
290impl<const SIZE: usize> Bar<SIZE> {
291    fn new(pdev: &Device, num: u32, name: &CStr) -> Result<Self> {
292        let len = pdev.resource_len(num)?;
293        if len == 0 {
294            return Err(ENOMEM);
295        }
296
297        // Convert to `i32`, since that's what all the C bindings use.
298        let num = i32::try_from(num)?;
299
300        // SAFETY:
301        // `pdev` is valid by the invariants of `Device`.
302        // `num` is checked for validity by a previous call to `Device::resource_len`.
303        // `name` is always valid.
304        let ret = unsafe { bindings::pci_request_region(pdev.as_raw(), num, name.as_char_ptr()) };
305        if ret != 0 {
306            return Err(EBUSY);
307        }
308
309        // SAFETY:
310        // `pdev` is valid by the invariants of `Device`.
311        // `num` is checked for validity by a previous call to `Device::resource_len`.
312        // `name` is always valid.
313        let ioptr: usize = unsafe { bindings::pci_iomap(pdev.as_raw(), num, 0) } as usize;
314        if ioptr == 0 {
315            // SAFETY:
316            // `pdev` valid by the invariants of `Device`.
317            // `num` is checked for validity by a previous call to `Device::resource_len`.
318            unsafe { bindings::pci_release_region(pdev.as_raw(), num) };
319            return Err(ENOMEM);
320        }
321
322        let io = match IoRaw::new(ioptr, len as usize) {
323            Ok(io) => io,
324            Err(err) => {
325                // SAFETY:
326                // `pdev` is valid by the invariants of `Device`.
327                // `ioptr` is guaranteed to be the start of a valid I/O mapped memory region.
328                // `num` is checked for validity by a previous call to `Device::resource_len`.
329                unsafe { Self::do_release(pdev, ioptr, num) };
330                return Err(err);
331            }
332        };
333
334        Ok(Bar {
335            pdev: pdev.into(),
336            io,
337            num,
338        })
339    }
340
341    /// # Safety
342    ///
343    /// `ioptr` must be a valid pointer to the memory mapped PCI bar number `num`.
344    unsafe fn do_release(pdev: &Device, ioptr: usize, num: i32) {
345        // SAFETY:
346        // `pdev` is valid by the invariants of `Device`.
347        // `ioptr` is valid by the safety requirements.
348        // `num` is valid by the safety requirements.
349        unsafe {
350            bindings::pci_iounmap(pdev.as_raw(), ioptr as *mut kernel::ffi::c_void);
351            bindings::pci_release_region(pdev.as_raw(), num);
352        }
353    }
354
355    fn release(&self) {
356        // SAFETY: The safety requirements are guaranteed by the type invariant of `self.pdev`.
357        unsafe { Self::do_release(&self.pdev, self.io.addr(), self.num) };
358    }
359}
360
361impl Bar {
362    fn index_is_valid(index: u32) -> bool {
363        // A `struct pci_dev` owns an array of resources with at most `PCI_NUM_RESOURCES` entries.
364        index < bindings::PCI_NUM_RESOURCES
365    }
366}
367
368impl<const SIZE: usize> Drop for Bar<SIZE> {
369    fn drop(&mut self) {
370        self.release();
371    }
372}
373
374impl<const SIZE: usize> Deref for Bar<SIZE> {
375    type Target = Io<SIZE>;
376
377    fn deref(&self) -> &Self::Target {
378        // SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped.
379        unsafe { Io::from_raw(&self.io) }
380    }
381}
382
383impl<Ctx: device::DeviceContext> Device<Ctx> {
384    fn as_raw(&self) -> *mut bindings::pci_dev {
385        self.0.get()
386    }
387}
388
389impl Device {
390    /// Returns the PCI vendor ID.
391    pub fn vendor_id(&self) -> u16 {
392        // SAFETY: `self.as_raw` is a valid pointer to a `struct pci_dev`.
393        unsafe { (*self.as_raw()).vendor }
394    }
395
396    /// Returns the PCI device ID.
397    pub fn device_id(&self) -> u16 {
398        // SAFETY: `self.as_raw` is a valid pointer to a `struct pci_dev`.
399        unsafe { (*self.as_raw()).device }
400    }
401
402    /// Returns the size of the given PCI bar resource.
403    pub fn resource_len(&self, bar: u32) -> Result<bindings::resource_size_t> {
404        if !Bar::index_is_valid(bar) {
405            return Err(EINVAL);
406        }
407
408        // SAFETY:
409        // - `bar` is a valid bar number, as guaranteed by the above call to `Bar::index_is_valid`,
410        // - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`.
411        Ok(unsafe { bindings::pci_resource_len(self.as_raw(), bar.try_into()?) })
412    }
413}
414
415impl Device<device::Bound> {
416    /// Mapps an entire PCI-BAR after performing a region-request on it. I/O operation bound checks
417    /// can be performed on compile time for offsets (plus the requested type size) < SIZE.
418    pub fn iomap_region_sized<'a, const SIZE: usize>(
419        &'a self,
420        bar: u32,
421        name: &'a CStr,
422    ) -> impl PinInit<Devres<Bar<SIZE>>, Error> + 'a {
423        Devres::new(self.as_ref(), Bar::<SIZE>::new(self, bar, name))
424    }
425
426    /// Mapps an entire PCI-BAR after performing a region-request on it.
427    pub fn iomap_region<'a>(
428        &'a self,
429        bar: u32,
430        name: &'a CStr,
431    ) -> impl PinInit<Devres<Bar>, Error> + 'a {
432        self.iomap_region_sized::<0>(bar, name)
433    }
434}
435
436impl Device<device::Core> {
437    /// Enable memory resources for this device.
438    pub fn enable_device_mem(&self) -> Result {
439        // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`.
440        to_result(unsafe { bindings::pci_enable_device_mem(self.as_raw()) })
441    }
442
443    /// Enable bus-mastering for this device.
444    pub fn set_master(&self) {
445        // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`.
446        unsafe { bindings::pci_set_master(self.as_raw()) };
447    }
448}
449
450// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic
451// argument.
452kernel::impl_device_context_deref!(unsafe { Device });
453kernel::impl_device_context_into_aref!(Device);
454
455impl crate::dma::Device for Device<device::Core> {}
456
457// SAFETY: Instances of `Device` are always reference-counted.
458unsafe impl crate::types::AlwaysRefCounted for Device {
459    fn inc_ref(&self) {
460        // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero.
461        unsafe { bindings::pci_dev_get(self.as_raw()) };
462    }
463
464    unsafe fn dec_ref(obj: NonNull<Self>) {
465        // SAFETY: The safety requirements guarantee that the refcount is non-zero.
466        unsafe { bindings::pci_dev_put(obj.cast().as_ptr()) }
467    }
468}
469
470impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> {
471    fn as_ref(&self) -> &device::Device<Ctx> {
472        // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid
473        // `struct pci_dev`.
474        let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) };
475
476        // SAFETY: `dev` points to a valid `struct device`.
477        unsafe { device::Device::from_raw(dev) }
478    }
479}
480
481impl<Ctx: device::DeviceContext> TryFrom<&device::Device<Ctx>> for &Device<Ctx> {
482    type Error = kernel::error::Error;
483
484    fn try_from(dev: &device::Device<Ctx>) -> Result<Self, Self::Error> {
485        // SAFETY: By the type invariant of `Device`, `dev.as_raw()` is a valid pointer to a
486        // `struct device`.
487        if !unsafe { bindings::dev_is_pci(dev.as_raw()) } {
488            return Err(EINVAL);
489        }
490
491        // SAFETY: We've just verified that the bus type of `dev` equals `bindings::pci_bus_type`,
492        // hence `dev` must be embedded in a valid `struct pci_dev` as guaranteed by the
493        // corresponding C code.
494        let pdev = unsafe { container_of!(dev.as_raw(), bindings::pci_dev, dev) };
495
496        // SAFETY: `pdev` is a valid pointer to a `struct pci_dev`.
497        Ok(unsafe { &*pdev.cast() })
498    }
499}
500
501// SAFETY: A `Device` is always reference-counted and can be released from any thread.
502unsafe impl Send for Device {}
503
504// SAFETY: `Device` can be shared among threads because all methods of `Device`
505// (i.e. `Device<Normal>) are thread safe.
506unsafe impl Sync for Device {}