Commit d80c0f00 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: add io_req_local_work_add wake fast path 



Don't wake the master task after queueing a deferred tw unless it's
currently waiting in io_cqring_wait.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/717702d772825a6647e6c315b4690277ba84c3fc.1673274244.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 130bd686

include/linux/io_uring_types.h

+1 −0
Original line number Diff line number Diff line
@@ -295,6 +295,7 @@ struct io_ring_ctx { 
		spinlock_t		completion_lock;



		bool			poll_multi_queue;

		bool			cq_waiting;



		/*

		 * ->iopoll_list is protected by the ctx->uring_lock for

io_uring/io_uring.c

+6 −1
Original line number Diff line number Diff line
@@ -1276,6 +1276,8 @@ static void io_req_local_work_add(struct io_kiocb *req) 
		atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);

	if (ctx->has_evfd)

		io_eventfd_signal(ctx);



	if (READ_ONCE(ctx->cq_waiting))

		wake_up_state(ctx->submitter_task, TASK_INTERRUPTIBLE);

	percpu_ref_put(&ctx->refs);

}
@@ -2576,6 +2578,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, 
		unsigned long check_cq;



		if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {

			WRITE_ONCE(ctx->cq_waiting, 1);

			set_current_state(TASK_INTERRUPTIBLE);

		} else {

			prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
@@ -2584,6 +2587,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, 


		ret = io_cqring_wait_schedule(ctx, &iowq);

		__set_current_state(TASK_RUNNING);

		WRITE_ONCE(ctx->cq_waiting, 0);



		if (ret < 0)

			break;

		/*