Commit e6aeb272 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: complete all requests in task context 



This patch adds ctx->task_complete flag. If set, we'll complete all
requests in the context of the original task. Note, this extends to
completion CQE posting only but not io_kiocb cleanup / free, e.g. io-wq
may free the requests in the free calllback. This flag will be used
later for optimisations purposes.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/21ece72953f76bb2e77659a72a14326227ab6460.1670384893.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 1b346e4a

include/linux/io_uring.h

+2 −0
Original line number Diff line number Diff line
@@ -11,6 +11,8 @@ enum io_uring_cmd_flags { 
	IO_URING_F_UNLOCKED		= 2,

	/* the request is executed from poll, it should not be freed */

	IO_URING_F_MULTISHOT		= 4,

	/* executed by io-wq */

	IO_URING_F_IOWQ			= 8,

	/* int's last bit, sign checks are usually faster than a bit test */

	IO_URING_F_NONBLOCK		= INT_MIN,

include/linux/io_uring_types.h

+2 −0
Original line number Diff line number Diff line
@@ -208,6 +208,8 @@ struct io_ring_ctx { 
		unsigned int		drain_disabled: 1;

		unsigned int		has_evfd: 1;

		unsigned int		syscall_iopoll: 1;

		/* all CQEs should be posted only by the submitter task */

		unsigned int		task_complete: 1;

	} ____cacheline_aligned_in_smp;



	/* submission data */

io_uring/io_uring.c

+11 −3
Original line number Diff line number Diff line
@@ -921,7 +921,10 @@ static void __io_req_complete_post(struct io_kiocb *req) 


void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)

{

	if (!(issue_flags & IO_URING_F_UNLOCKED) ||

	if (req->ctx->task_complete && (issue_flags & IO_URING_F_IOWQ)) {

		req->io_task_work.func = io_req_task_complete;

		io_req_task_work_add(req);

	} else if (!(issue_flags & IO_URING_F_UNLOCKED) ||

		   !(req->ctx->flags & IORING_SETUP_IOPOLL)) {

		__io_req_complete_post(req);

	} else {
@@ -1830,7 +1833,7 @@ void io_wq_submit_work(struct io_wq_work *work) 
{

	struct io_kiocb *req = container_of(work, struct io_kiocb, work);

	const struct io_op_def *def = &io_op_defs[req->opcode];

	unsigned int issue_flags = IO_URING_F_UNLOCKED;

	unsigned int issue_flags = IO_URING_F_UNLOCKED | IO_URING_F_IOWQ;

	bool needs_poll = false;

	int ret = 0, err = -ECANCELED;
@@ -3490,6 +3493,11 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, 
	if (!ctx)

		return -ENOMEM;



	if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&

	    !(ctx->flags & IORING_SETUP_IOPOLL) &&

	    !(ctx->flags & IORING_SETUP_SQPOLL))

		ctx->task_complete = true;



	/*

	 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user

	 * space applications don't need to do io completion events