diff options
| author | Stefan Roesch <shr@devkernel.io> | 2023-06-08 09:38:37 -0700 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2024-02-09 11:54:28 -0700 |
| commit | ff183d427da0a733b0dbe11bd7acaf2dcb37b4cc (patch) | |
| tree | 608885d59ca17fa553a9a9c9edfae6776b63545b /io_uring | |
| parent | 8d0c12a80cdeb80d5e0510e96d38fe551ed8e9b5 (diff) | |
| download | linux-ff183d427da0a733b0dbe11bd7acaf2dcb37b4cc.tar.gz | |
io-uring: add sqpoll support for napi busy poll
This adds the sqpoll support to the io-uring napi.
Signed-off-by: Stefan Roesch <shr@devkernel.io>
Suggested-by: Olivier Langlois <olivier@trillion01.com>
Link: https://lore.kernel.org/r/20230608163839.2891748-6-shr@devkernel.io
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'io_uring')
| -rw-r--r-- | io_uring/napi.c | 24 | ||||
| -rw-r--r-- | io_uring/napi.h | 6 | ||||
| -rw-r--r-- | io_uring/sqpoll.c | 4 |
3 files changed, 33 insertions, 1 deletions
diff --git a/io_uring/napi.c b/io_uring/napi.c index 1112cc39153cb5..3e578df36cc510 100644 --- a/io_uring/napi.c +++ b/io_uring/napi.c @@ -252,4 +252,28 @@ void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq) io_napi_blocking_busy_loop(ctx, iowq); } +/* + * io_napi_sqpoll_busy_poll() - busy poll loop for sqpoll + * @ctx: pointer to io-uring context structure + * + * Splice of the napi list and execute the napi busy poll loop. + */ +int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx) +{ + LIST_HEAD(napi_list); + bool is_stale = false; + + if (!READ_ONCE(ctx->napi_busy_poll_to)) + return 0; + if (list_empty_careful(&ctx->napi_list)) + return 0; + + rcu_read_lock(); + is_stale = __io_napi_do_busy_loop(ctx, NULL); + rcu_read_unlock(); + + io_napi_remove_stale(ctx, is_stale); + return 1; +} + #endif diff --git a/io_uring/napi.h b/io_uring/napi.h index be8aa8ee32d940..b6d6243fc7fe02 100644 --- a/io_uring/napi.h +++ b/io_uring/napi.h @@ -17,6 +17,7 @@ void __io_napi_add(struct io_ring_ctx *ctx, struct socket *sock); void __io_napi_adjust_timeout(struct io_ring_ctx *ctx, struct io_wait_queue *iowq, struct timespec64 *ts); void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq); +int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx); static inline bool io_napi(struct io_ring_ctx *ctx) { @@ -83,7 +84,10 @@ static inline void io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq) { } - +static inline int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx) +{ + return 0; +} #endif /* CONFIG_NET_RX_BUSY_POLL */ #endif diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c index 28bf0e085d3108..f3979cacda13b3 100644 --- a/io_uring/sqpoll.c +++ b/io_uring/sqpoll.c @@ -15,6 +15,7 @@ #include <uapi/linux/io_uring.h> #include "io_uring.h" +#include "napi.h" #include "sqpoll.h" #define IORING_SQPOLL_CAP_ENTRIES_VALUE 8 @@ -194,6 +195,9 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) ret = io_submit_sqes(ctx, to_submit); mutex_unlock(&ctx->uring_lock); + if (io_napi(ctx)) + ret += io_napi_sqpoll_busy_poll(ctx); + if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait)) wake_up(&ctx->sqo_sq_wait); if (creds) |