kernel/
uaccess.rs

1// SPDX-License-Identifier: GPL-2.0
2
3//! Slices to user space memory regions.
4//!
5//! C header: [`include/linux/uaccess.h`](srctree/include/linux/uaccess.h)
6
7use crate::{
8    alloc::{Allocator, Flags},
9    bindings,
10    error::Result,
11    ffi::{c_char, c_void},
12    prelude::*,
13    transmute::{AsBytes, FromBytes},
14};
15use core::mem::{size_of, MaybeUninit};
16
17/// A pointer into userspace.
18///
19/// This is the Rust equivalent to C pointers tagged with `__user`.
20#[repr(transparent)]
21#[derive(Copy, Clone)]
22pub struct UserPtr(*mut c_void);
23
24impl UserPtr {
25    /// Create a `UserPtr` from an integer representing the userspace address.
26    #[inline]
27    pub fn from_addr(addr: usize) -> Self {
28        Self(addr as *mut c_void)
29    }
30
31    /// Create a `UserPtr` from a pointer representing the userspace address.
32    #[inline]
33    pub fn from_ptr(addr: *mut c_void) -> Self {
34        Self(addr)
35    }
36
37    /// Cast this userspace pointer to a raw const void pointer.
38    ///
39    /// It is up to the caller to use the returned pointer correctly.
40    #[inline]
41    pub fn as_const_ptr(self) -> *const c_void {
42        self.0
43    }
44
45    /// Cast this userspace pointer to a raw mutable void pointer.
46    ///
47    /// It is up to the caller to use the returned pointer correctly.
48    #[inline]
49    pub fn as_mut_ptr(self) -> *mut c_void {
50        self.0
51    }
52
53    /// Increment this user pointer by `add` bytes.
54    ///
55    /// This addition is wrapping, so wrapping around the address space does not result in a panic
56    /// even if `CONFIG_RUST_OVERFLOW_CHECKS` is enabled.
57    #[inline]
58    pub fn wrapping_byte_add(self, add: usize) -> UserPtr {
59        UserPtr(self.0.wrapping_byte_add(add))
60    }
61}
62
63/// A pointer to an area in userspace memory, which can be either read-only or read-write.
64///
65/// All methods on this struct are safe: attempting to read or write on bad addresses (either out of
66/// the bound of the slice or unmapped addresses) will return [`EFAULT`]. Concurrent access,
67/// *including data races to/from userspace memory*, is permitted, because fundamentally another
68/// userspace thread/process could always be modifying memory at the same time (in the same way that
69/// userspace Rust's [`std::io`] permits data races with the contents of files on disk). In the
70/// presence of a race, the exact byte values read/written are unspecified but the operation is
71/// well-defined. Kernelspace code should validate its copy of data after completing a read, and not
72/// expect that multiple reads of the same address will return the same value.
73///
74/// These APIs are designed to make it difficult to accidentally write TOCTOU (time-of-check to
75/// time-of-use) bugs. Every time a memory location is read, the reader's position is advanced by
76/// the read length and the next read will start from there. This helps prevent accidentally reading
77/// the same location twice and causing a TOCTOU bug.
78///
79/// Creating a [`UserSliceReader`] and/or [`UserSliceWriter`] consumes the `UserSlice`, helping
80/// ensure that there aren't multiple readers or writers to the same location.
81///
82/// If double-fetching a memory location is necessary for some reason, then that is done by creating
83/// multiple readers to the same memory location, e.g. using [`clone_reader`].
84///
85/// # Examples
86///
87/// Takes a region of userspace memory from the current process, and modify it by adding one to
88/// every byte in the region.
89///
90/// ```no_run
91/// use kernel::ffi::c_void;
92/// use kernel::uaccess::{UserPtr, UserSlice};
93///
94/// fn bytes_add_one(uptr: UserPtr, len: usize) -> Result {
95///     let (read, mut write) = UserSlice::new(uptr, len).reader_writer();
96///
97///     let mut buf = KVec::new();
98///     read.read_all(&mut buf, GFP_KERNEL)?;
99///
100///     for b in &mut buf {
101///         *b = b.wrapping_add(1);
102///     }
103///
104///     write.write_slice(&buf)?;
105///     Ok(())
106/// }
107/// ```
108///
109/// Example illustrating a TOCTOU (time-of-check to time-of-use) bug.
110///
111/// ```no_run
112/// use kernel::ffi::c_void;
113/// use kernel::uaccess::{UserPtr, UserSlice};
114///
115/// /// Returns whether the data in this region is valid.
116/// fn is_valid(uptr: UserPtr, len: usize) -> Result<bool> {
117///     let read = UserSlice::new(uptr, len).reader();
118///
119///     let mut buf = KVec::new();
120///     read.read_all(&mut buf, GFP_KERNEL)?;
121///
122///     todo!()
123/// }
124///
125/// /// Returns the bytes behind this user pointer if they are valid.
126/// fn get_bytes_if_valid(uptr: UserPtr, len: usize) -> Result<KVec<u8>> {
127///     if !is_valid(uptr, len)? {
128///         return Err(EINVAL);
129///     }
130///
131///     let read = UserSlice::new(uptr, len).reader();
132///
133///     let mut buf = KVec::new();
134///     read.read_all(&mut buf, GFP_KERNEL)?;
135///
136///     // THIS IS A BUG! The bytes could have changed since we checked them.
137///     //
138///     // To avoid this kind of bug, don't call `UserSlice::new` multiple
139///     // times with the same address.
140///     Ok(buf)
141/// }
142/// ```
143///
144/// [`std::io`]: https://doc.rust-lang.org/std/io/index.html
145/// [`clone_reader`]: UserSliceReader::clone_reader
146pub struct UserSlice {
147    ptr: UserPtr,
148    length: usize,
149}
150
151impl UserSlice {
152    /// Constructs a user slice from a raw pointer and a length in bytes.
153    ///
154    /// Constructing a [`UserSlice`] performs no checks on the provided address and length, it can
155    /// safely be constructed inside a kernel thread with no current userspace process. Reads and
156    /// writes wrap the kernel APIs `copy_from_user` and `copy_to_user`, which check the memory map
157    /// of the current process and enforce that the address range is within the user range (no
158    /// additional calls to `access_ok` are needed). Validity of the pointer is checked when you
159    /// attempt to read or write, not in the call to `UserSlice::new`.
160    ///
161    /// Callers must be careful to avoid time-of-check-time-of-use (TOCTOU) issues. The simplest way
162    /// is to create a single instance of [`UserSlice`] per user memory block as it reads each byte
163    /// at most once.
164    pub fn new(ptr: UserPtr, length: usize) -> Self {
165        UserSlice { ptr, length }
166    }
167
168    /// Reads the entirety of the user slice, appending it to the end of the provided buffer.
169    ///
170    /// Fails with [`EFAULT`] if the read happens on a bad address.
171    pub fn read_all<A: Allocator>(self, buf: &mut Vec<u8, A>, flags: Flags) -> Result {
172        self.reader().read_all(buf, flags)
173    }
174
175    /// Constructs a [`UserSliceReader`].
176    pub fn reader(self) -> UserSliceReader {
177        UserSliceReader {
178            ptr: self.ptr,
179            length: self.length,
180        }
181    }
182
183    /// Constructs a [`UserSliceWriter`].
184    pub fn writer(self) -> UserSliceWriter {
185        UserSliceWriter {
186            ptr: self.ptr,
187            length: self.length,
188        }
189    }
190
191    /// Constructs both a [`UserSliceReader`] and a [`UserSliceWriter`].
192    ///
193    /// Usually when this is used, you will first read the data, and then overwrite it afterwards.
194    pub fn reader_writer(self) -> (UserSliceReader, UserSliceWriter) {
195        (
196            UserSliceReader {
197                ptr: self.ptr,
198                length: self.length,
199            },
200            UserSliceWriter {
201                ptr: self.ptr,
202                length: self.length,
203            },
204        )
205    }
206}
207
208/// A reader for [`UserSlice`].
209///
210/// Used to incrementally read from the user slice.
211pub struct UserSliceReader {
212    ptr: UserPtr,
213    length: usize,
214}
215
216impl UserSliceReader {
217    /// Skip the provided number of bytes.
218    ///
219    /// Returns an error if skipping more than the length of the buffer.
220    pub fn skip(&mut self, num_skip: usize) -> Result {
221        // Update `self.length` first since that's the fallible part of this operation.
222        self.length = self.length.checked_sub(num_skip).ok_or(EFAULT)?;
223        self.ptr = self.ptr.wrapping_byte_add(num_skip);
224        Ok(())
225    }
226
227    /// Create a reader that can access the same range of data.
228    ///
229    /// Reading from the clone does not advance the current reader.
230    ///
231    /// The caller should take care to not introduce TOCTOU issues, as described in the
232    /// documentation for [`UserSlice`].
233    pub fn clone_reader(&self) -> UserSliceReader {
234        UserSliceReader {
235            ptr: self.ptr,
236            length: self.length,
237        }
238    }
239
240    /// Returns the number of bytes left to be read from this reader.
241    ///
242    /// Note that even reading less than this number of bytes may fail.
243    pub fn len(&self) -> usize {
244        self.length
245    }
246
247    /// Returns `true` if no data is available in the io buffer.
248    pub fn is_empty(&self) -> bool {
249        self.length == 0
250    }
251
252    /// Reads raw data from the user slice into a kernel buffer.
253    ///
254    /// For a version that uses `&mut [u8]`, please see [`UserSliceReader::read_slice`].
255    ///
256    /// Fails with [`EFAULT`] if the read happens on a bad address, or if the read goes out of
257    /// bounds of this [`UserSliceReader`]. This call may modify `out` even if it returns an error.
258    ///
259    /// # Guarantees
260    ///
261    /// After a successful call to this method, all bytes in `out` are initialized.
262    pub fn read_raw(&mut self, out: &mut [MaybeUninit<u8>]) -> Result {
263        let len = out.len();
264        let out_ptr = out.as_mut_ptr().cast::<c_void>();
265        if len > self.length {
266            return Err(EFAULT);
267        }
268        // SAFETY: `out_ptr` points into a mutable slice of length `len`, so we may write
269        // that many bytes to it.
270        let res = unsafe { bindings::copy_from_user(out_ptr, self.ptr.as_const_ptr(), len) };
271        if res != 0 {
272            return Err(EFAULT);
273        }
274        self.ptr = self.ptr.wrapping_byte_add(len);
275        self.length -= len;
276        Ok(())
277    }
278
279    /// Reads raw data from the user slice into a kernel buffer.
280    ///
281    /// Fails with [`EFAULT`] if the read happens on a bad address, or if the read goes out of
282    /// bounds of this [`UserSliceReader`]. This call may modify `out` even if it returns an error.
283    pub fn read_slice(&mut self, out: &mut [u8]) -> Result {
284        // SAFETY: The types are compatible and `read_raw` doesn't write uninitialized bytes to
285        // `out`.
286        let out = unsafe { &mut *(core::ptr::from_mut(out) as *mut [MaybeUninit<u8>]) };
287        self.read_raw(out)
288    }
289
290    /// Reads a value of the specified type.
291    ///
292    /// Fails with [`EFAULT`] if the read happens on a bad address, or if the read goes out of
293    /// bounds of this [`UserSliceReader`].
294    pub fn read<T: FromBytes>(&mut self) -> Result<T> {
295        let len = size_of::<T>();
296        if len > self.length {
297            return Err(EFAULT);
298        }
299        let mut out: MaybeUninit<T> = MaybeUninit::uninit();
300        // SAFETY: The local variable `out` is valid for writing `size_of::<T>()` bytes.
301        //
302        // By using the _copy_from_user variant, we skip the check_object_size check that verifies
303        // the kernel pointer. This mirrors the logic on the C side that skips the check when the
304        // length is a compile-time constant.
305        let res = unsafe {
306            bindings::_copy_from_user(
307                out.as_mut_ptr().cast::<c_void>(),
308                self.ptr.as_const_ptr(),
309                len,
310            )
311        };
312        if res != 0 {
313            return Err(EFAULT);
314        }
315        self.ptr = self.ptr.wrapping_byte_add(len);
316        self.length -= len;
317        // SAFETY: The read above has initialized all bytes in `out`, and since `T` implements
318        // `FromBytes`, any bit-pattern is a valid value for this type.
319        Ok(unsafe { out.assume_init() })
320    }
321
322    /// Reads the entirety of the user slice, appending it to the end of the provided buffer.
323    ///
324    /// Fails with [`EFAULT`] if the read happens on a bad address.
325    pub fn read_all<A: Allocator>(mut self, buf: &mut Vec<u8, A>, flags: Flags) -> Result {
326        let len = self.length;
327        buf.reserve(len, flags)?;
328
329        // The call to `reserve` was successful, so the spare capacity is at least `len` bytes long.
330        self.read_raw(&mut buf.spare_capacity_mut()[..len])?;
331
332        // SAFETY: Since the call to `read_raw` was successful, so the next `len` bytes of the
333        // vector have been initialized.
334        unsafe { buf.inc_len(len) };
335        Ok(())
336    }
337
338    /// Read a NUL-terminated string from userspace and return it.
339    ///
340    /// The string is read into `buf` and a NUL-terminator is added if the end of `buf` is reached.
341    /// Since there must be space to add a NUL-terminator, the buffer must not be empty. The
342    /// returned `&CStr` points into `buf`.
343    ///
344    /// Fails with [`EFAULT`] if the read happens on a bad address (some data may have been
345    /// copied).
346    #[doc(alias = "strncpy_from_user")]
347    pub fn strcpy_into_buf<'buf>(self, buf: &'buf mut [u8]) -> Result<&'buf CStr> {
348        if buf.is_empty() {
349            return Err(EINVAL);
350        }
351
352        // SAFETY: The types are compatible and `strncpy_from_user` doesn't write uninitialized
353        // bytes to `buf`.
354        let mut dst = unsafe { &mut *(core::ptr::from_mut(buf) as *mut [MaybeUninit<u8>]) };
355
356        // We never read more than `self.length` bytes.
357        if dst.len() > self.length {
358            dst = &mut dst[..self.length];
359        }
360
361        let mut len = raw_strncpy_from_user(dst, self.ptr)?;
362        if len < dst.len() {
363            // Add one to include the NUL-terminator.
364            len += 1;
365        } else if len < buf.len() {
366            // This implies that `len == dst.len() < buf.len()`.
367            //
368            // This means that we could not fill the entire buffer, but we had to stop reading
369            // because we hit the `self.length` limit of this `UserSliceReader`. Since we did not
370            // fill the buffer, we treat this case as if we tried to read past the `self.length`
371            // limit and received a page fault, which is consistent with other `UserSliceReader`
372            // methods that also return page faults when you exceed `self.length`.
373            return Err(EFAULT);
374        } else {
375            // This implies that `len == buf.len()`.
376            //
377            // This means that we filled the buffer exactly. In this case, we add a NUL-terminator
378            // and return it. Unlike the `len < dst.len()` branch, don't modify `len` because it
379            // already represents the length including the NUL-terminator.
380            //
381            // SAFETY: Due to the check at the beginning, the buffer is not empty.
382            unsafe { *buf.last_mut().unwrap_unchecked() = 0 };
383        }
384
385        // This method consumes `self`, so it can only be called once, thus we do not need to
386        // update `self.length`. This sidesteps concerns such as whether `self.length` should be
387        // incremented by `len` or `len-1` in the `len == buf.len()` case.
388
389        // SAFETY: There are two cases:
390        // * If we hit the `len < dst.len()` case, then `raw_strncpy_from_user` guarantees that
391        //   this slice contains exactly one NUL byte at the end of the string.
392        // * Otherwise, `raw_strncpy_from_user` guarantees that the string contained no NUL bytes,
393        //   and we have since added a NUL byte at the end.
394        Ok(unsafe { CStr::from_bytes_with_nul_unchecked(&buf[..len]) })
395    }
396}
397
398/// A writer for [`UserSlice`].
399///
400/// Used to incrementally write into the user slice.
401pub struct UserSliceWriter {
402    ptr: UserPtr,
403    length: usize,
404}
405
406impl UserSliceWriter {
407    /// Returns the amount of space remaining in this buffer.
408    ///
409    /// Note that even writing less than this number of bytes may fail.
410    pub fn len(&self) -> usize {
411        self.length
412    }
413
414    /// Returns `true` if no more data can be written to this buffer.
415    pub fn is_empty(&self) -> bool {
416        self.length == 0
417    }
418
419    /// Writes raw data to this user pointer from a kernel buffer.
420    ///
421    /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of
422    /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even
423    /// if it returns an error.
424    pub fn write_slice(&mut self, data: &[u8]) -> Result {
425        let len = data.len();
426        let data_ptr = data.as_ptr().cast::<c_void>();
427        if len > self.length {
428            return Err(EFAULT);
429        }
430        // SAFETY: `data_ptr` points into an immutable slice of length `len`, so we may read
431        // that many bytes from it.
432        let res = unsafe { bindings::copy_to_user(self.ptr.as_mut_ptr(), data_ptr, len) };
433        if res != 0 {
434            return Err(EFAULT);
435        }
436        self.ptr = self.ptr.wrapping_byte_add(len);
437        self.length -= len;
438        Ok(())
439    }
440
441    /// Writes the provided Rust value to this userspace pointer.
442    ///
443    /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of
444    /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even
445    /// if it returns an error.
446    pub fn write<T: AsBytes>(&mut self, value: &T) -> Result {
447        let len = size_of::<T>();
448        if len > self.length {
449            return Err(EFAULT);
450        }
451        // SAFETY: The reference points to a value of type `T`, so it is valid for reading
452        // `size_of::<T>()` bytes.
453        //
454        // By using the _copy_to_user variant, we skip the check_object_size check that verifies the
455        // kernel pointer. This mirrors the logic on the C side that skips the check when the length
456        // is a compile-time constant.
457        let res = unsafe {
458            bindings::_copy_to_user(
459                self.ptr.as_mut_ptr(),
460                core::ptr::from_ref(value).cast::<c_void>(),
461                len,
462            )
463        };
464        if res != 0 {
465            return Err(EFAULT);
466        }
467        self.ptr = self.ptr.wrapping_byte_add(len);
468        self.length -= len;
469        Ok(())
470    }
471}
472
473/// Reads a nul-terminated string into `dst` and returns the length.
474///
475/// This reads from userspace until a NUL byte is encountered, or until `dst.len()` bytes have been
476/// read. Fails with [`EFAULT`] if a read happens on a bad address (some data may have been
477/// copied). When the end of the buffer is encountered, no NUL byte is added, so the string is
478/// *not* guaranteed to be NUL-terminated when `Ok(dst.len())` is returned.
479///
480/// # Guarantees
481///
482/// When this function returns `Ok(len)`, it is guaranteed that the first `len` bytes of `dst` are
483/// initialized and non-zero. Furthermore, if `len < dst.len()`, then `dst[len]` is a NUL byte.
484#[inline]
485fn raw_strncpy_from_user(dst: &mut [MaybeUninit<u8>], src: UserPtr) -> Result<usize> {
486    // CAST: Slice lengths are guaranteed to be `<= isize::MAX`.
487    let len = dst.len() as isize;
488
489    // SAFETY: `dst` is valid for writing `dst.len()` bytes.
490    let res = unsafe {
491        bindings::strncpy_from_user(
492            dst.as_mut_ptr().cast::<c_char>(),
493            src.as_const_ptr().cast::<c_char>(),
494            len,
495        )
496    };
497
498    if res < 0 {
499        return Err(Error::from_errno(res as i32));
500    }
501
502    #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)]
503    assert!(res <= len);
504
505    // GUARANTEES: `strncpy_from_user` was successful, so `dst` has contents in accordance with the
506    // guarantees of this function.
507    Ok(res as usize)
508}