Commit 2ad4c6d0 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: don't put nodes under spinlocks 



io_req_put_rsrc() doesn't need any locking, so move it out of
a spinlock section in __io_req_complete_post() and adjust helpers.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/d5b87a5f31270dade6805f7acafc4cc34b84b241.1680576071.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 8e15c0e7

io_uring/io_uring.c

+5 −2
Original line number Diff line number Diff line
@@ -970,6 +970,7 @@ bool io_aux_cqe(struct io_ring_ctx *ctx, bool defer, u64 user_data, s32 res, u32 
static void __io_req_complete_post(struct io_kiocb *req)

{

	struct io_ring_ctx *ctx = req->ctx;

	struct io_rsrc_node *rsrc_node = NULL;



	io_cq_lock(ctx);

	if (!(req->flags & REQ_F_CQE_SKIP))
@@ -990,7 +991,7 @@ static void __io_req_complete_post(struct io_kiocb *req) 
		}

		io_put_kbuf_comp(req);

		io_dismantle_req(req);

		io_req_put_rsrc(req);

		rsrc_node = req->rsrc_node;

		/*

		 * Selected buffer deallocation in io_clean_op() assumes that

		 * we don't hold ->completion_lock. Clean them here to avoid
@@ -1001,6 +1002,8 @@ static void __io_req_complete_post(struct io_kiocb *req) 
		ctx->locked_free_nr++;

	}

	io_cq_unlock_post(ctx);



	io_put_rsrc_node(rsrc_node);

}



void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
@@ -1117,7 +1120,7 @@ __cold void io_free_req(struct io_kiocb *req) 
{

	struct io_ring_ctx *ctx = req->ctx;



	io_req_put_rsrc(req);

	io_put_rsrc_node(req->rsrc_node);

	io_dismantle_req(req);

	io_put_task_remote(req->task, 1);

io_uring/rsrc.h

+3 −3
Original line number Diff line number Diff line
@@ -115,10 +115,10 @@ static inline void io_rsrc_put_node(struct io_rsrc_node *node, int nr) 
		io_rsrc_node_ref_zero(node);

}



static inline void io_req_put_rsrc(struct io_kiocb *req)

static inline void io_put_rsrc_node(struct io_rsrc_node *node)

{

	if (req->rsrc_node)

		io_rsrc_put_node(req->rsrc_node, 1);

	if (node)

		io_rsrc_put_node(node, 1);

}



static inline void io_req_put_rsrc_locked(struct io_kiocb *req,