diff options
Diffstat (limited to 'io_uring')
| -rw-r--r-- | io_uring/alloc_cache.h | 5 | ||||
| -rw-r--r-- | io_uring/io_uring.c | 4 | ||||
| -rw-r--r-- | io_uring/poll.c | 20 | ||||
| -rw-r--r-- | io_uring/splice.c | 4 | ||||
| -rw-r--r-- | io_uring/uring_cmd.c | 3 |
5 files changed, 24 insertions, 12 deletions
diff --git a/io_uring/alloc_cache.h b/io_uring/alloc_cache.h index 241245cb54a6b..bf2fb26a65398 100644 --- a/io_uring/alloc_cache.h +++ b/io_uring/alloc_cache.h @@ -16,8 +16,7 @@ static inline bool io_alloc_cache_put(struct io_alloc_cache *cache, if (cache->nr_cached < cache->max_cached) { cache->nr_cached++; wq_stack_add_head(&entry->node, &cache->list); - /* KASAN poisons object */ - kasan_slab_free_mempool(entry); + kasan_mempool_poison_object(entry); return true; } return false; @@ -34,7 +33,7 @@ static inline struct io_cache_entry *io_alloc_cache_get(struct io_alloc_cache *c struct io_cache_entry *entry; entry = container_of(cache->list.next, struct io_cache_entry, node); - kasan_unpoison_range(entry, cache->elem_size); + kasan_mempool_unpoison_object(entry, cache->elem_size); cache->list.next = cache->list.next->next; cache->nr_cached--; return entry; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 4afb911fc042b..09b6d860deba3 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -540,7 +540,7 @@ void io_eventfd_ops(struct rcu_head *rcu) int ops = atomic_xchg(&ev_fd->ops, 0); if (ops & BIT(IO_EVENTFD_OP_SIGNAL_BIT)) - eventfd_signal_mask(ev_fd->cq_ev_fd, 1, EPOLL_URING_WAKE); + eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE); /* IO_EVENTFD_OP_FREE_BIT may not be set here depending on callback * ordering in a race but if references are 0 we know we have to free @@ -576,7 +576,7 @@ static void io_eventfd_signal(struct io_ring_ctx *ctx) goto out; if (likely(eventfd_signal_allowed())) { - eventfd_signal_mask(ev_fd->cq_ev_fd, 1, EPOLL_URING_WAKE); + eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE); } else { atomic_inc(&ev_fd->refs); if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops)) diff --git a/io_uring/poll.c b/io_uring/poll.c index d38d05edb4fa2..d59b74a99d4e4 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -366,11 +366,16 @@ void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts) static void __io_poll_execute(struct io_kiocb *req, int mask) { + unsigned flags = 0; + io_req_set_res(req, mask, 0); req->io_task_work.func = io_poll_task_func; trace_io_uring_task_add(req, mask); - __io_req_task_work_add(req, IOU_F_TWQ_LAZY_WAKE); + + if (!(req->flags & REQ_F_POLL_NO_LAZY)) + flags = IOU_F_TWQ_LAZY_WAKE; + __io_req_task_work_add(req, flags); } static inline void io_poll_execute(struct io_kiocb *req, int res) @@ -526,10 +531,19 @@ static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt, poll->head = head; poll->wait.private = (void *) wqe_private; - if (poll->events & EPOLLEXCLUSIVE) + if (poll->events & EPOLLEXCLUSIVE) { + /* + * Exclusive waits may only wake a limited amount of entries + * rather than all of them, this may interfere with lazy + * wake if someone does wait(events > 1). Ensure we don't do + * lazy wake for those, as we need to process each one as they + * come in. + */ + req->flags |= REQ_F_POLL_NO_LAZY; add_wait_queue_exclusive(head, &poll->wait); - else + } else { add_wait_queue(head, &poll->wait); + } } static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head, diff --git a/io_uring/splice.c b/io_uring/splice.c index 7c4469e9540e0..3b659cd23e9da 100644 --- a/io_uring/splice.c +++ b/io_uring/splice.c @@ -51,7 +51,7 @@ int io_tee(struct io_kiocb *req, unsigned int issue_flags) struct file *out = sp->file_out; unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; struct file *in; - long ret = 0; + ssize_t ret = 0; WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); @@ -92,7 +92,7 @@ int io_splice(struct io_kiocb *req, unsigned int issue_flags) unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; loff_t *poff_in, *poff_out; struct file *in; - long ret = 0; + ssize_t ret = 0; WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 34030583b9b25..c33fca585dde5 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -7,7 +7,7 @@ #include <linux/nospec.h> #include <uapi/linux/io_uring.h> -#include <uapi/asm-generic/ioctls.h> +#include <asm/ioctls.h> #include "io_uring.h" #include "rsrc.h" @@ -169,7 +169,6 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) return -EOPNOTSUPP; issue_flags |= IO_URING_F_IOPOLL; req->iopoll_completed = 0; - WRITE_ONCE(ioucmd->cookie, NULL); } ret = file->f_op->uring_cmd(ioucmd, issue_flags); |