Commit 7b235dd8 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: separate wq for ring polling 



Don't use ->cq_wait for ring polling but add a separate wait queue for
it. We need it for following patches.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/dea0be0bf990503443c5c6c337fc66824af7d590.1673274244.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 360173ab

include/linux/io_uring_types.h

+1 −1
Original line number Diff line number Diff line
@@ -319,7 +319,7 @@ struct io_ring_ctx { 
	} ____cacheline_aligned_in_smp;



	/* Keep this last, we don't need it for the fast path */



	struct wait_queue_head		poll_wq;

	struct io_restriction		restrictions;



	/* slow path rsrc auxilary data, used by update/register */

io_uring/io_uring.c

+2 −1
Original line number Diff line number Diff line
@@ -316,6 +316,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) 
	xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);

	mutex_init(&ctx->uring_lock);

	init_waitqueue_head(&ctx->cq_wait);

	init_waitqueue_head(&ctx->poll_wq);

	spin_lock_init(&ctx->completion_lock);

	spin_lock_init(&ctx->timeout_lock);

	INIT_WQ_LIST(&ctx->iopoll_list);
@@ -2786,7 +2787,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) 
	struct io_ring_ctx *ctx = file->private_data;

	__poll_t mask = 0;



	poll_wait(file, &ctx->cq_wait, wait);

	poll_wait(file, &ctx->poll_wq, wait);

	/*

	 * synchronizes with barrier from wq_has_sleeper call in

	 * io_commit_cqring

io_uring/io_uring.h

+9 −0
Original line number Diff line number Diff line
@@ -220,9 +220,18 @@ static inline void io_commit_cqring(struct io_ring_ctx *ctx) 
	smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);

}



static inline void io_poll_wq_wake(struct io_ring_ctx *ctx)

{

	if (waitqueue_active(&ctx->poll_wq))

		__wake_up(&ctx->poll_wq, TASK_NORMAL, 0,

				poll_to_key(EPOLL_URING_WAKE | EPOLLIN));

}



/* requires smb_mb() prior, see wq_has_sleeper() */

static inline void __io_cqring_wake(struct io_ring_ctx *ctx)

{

	io_poll_wq_wake(ctx);



	/*

	 * Trigger waitqueue handler on all waiters on our waitqueue. This

	 * won't necessarily wake up all the tasks, io_should_wake() will make