diff options
| author | Jens Axboe <axboe@kernel.dk> | 2022-09-14 03:47:58 -0600 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2022-09-14 03:49:10 -0600 |
| commit | 995f16f44e4488234774d1b826a4b294b15f61fc (patch) | |
| tree | d642eec246eaa58901befb9becb7b61eda241b53 | |
| parent | e7a4a472d6e335bf87e2033477c862b870605a2b (diff) | |
| download | liburing-sq-khead.tar.gz | |
testsq-khead
Signed-off-by: Jens Axboe <axboe@kernel.dk>
| -rw-r--r-- | src/include/liburing.h | 9 | ||||
| -rw-r--r-- | src/queue.c | 3 | ||||
| -rw-r--r-- | src/setup.c | 1 |
3 files changed, 10 insertions, 3 deletions
diff --git a/src/include/liburing.h b/src/include/liburing.h index ae25e219..aff3e9cf 100644 --- a/src/include/liburing.h +++ b/src/include/liburing.h @@ -97,7 +97,8 @@ struct io_uring_sq { unsigned ring_mask; unsigned ring_entries; - unsigned pad[2]; + unsigned sq_head_cache; + unsigned pad; }; struct io_uring_cq { @@ -1240,14 +1241,16 @@ static inline int io_uring_wait_cqe(struct io_uring *ring, static inline struct io_uring_sqe *_io_uring_get_sqe(struct io_uring *ring) { struct io_uring_sq *sq = &ring->sq; - unsigned int head = io_uring_smp_load_acquire(sq->khead); unsigned int next = sq->sqe_tail + 1; int shift = 0; if (ring->flags & IORING_SETUP_SQE128) shift = 1; - if (next - head <= sq->ring_entries) { + if (!sq->sq_head_cache) + sq->sq_head_cache = io_uring_smp_load_acquire(sq->khead); + + if (next - sq->sq_head_cache <= sq->ring_entries) { struct io_uring_sqe *sqe; sqe = &sq->sqes[(sq->sqe_tail & sq->ring_mask) << shift]; diff --git a/src/queue.c b/src/queue.c index b012a3dd..b6fd64a4 100644 --- a/src/queue.c +++ b/src/queue.c @@ -197,6 +197,9 @@ unsigned __io_uring_flush_sq(struct io_uring *ring) */ io_uring_smp_store_release(sq->ktail, tail); } + + sq->sq_head_cache = 0; + /* * This _may_ look problematic, as we're not supposed to be reading * SQ->head without acquire semantics. When we're in SQPOLL mode, the diff --git a/src/setup.c b/src/setup.c index 21283eb9..549d28fc 100644 --- a/src/setup.c +++ b/src/setup.c @@ -170,6 +170,7 @@ __cold int io_uring_queue_init_params(unsigned entries, struct io_uring *ring, sq_array[index] = index; ring->features = p->features; + ring->sq.sq_head_cache = 0; return 0; } |