Commit ef8ae64f authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring/rsrc: protect node refs with uring_lock 



Currently, for nodes we have an atomic counter and some cached
(non-atomic) refs protected by uring_lock. Let's put all ref
manipulations under uring_lock and get rid of the atomic part.
It's free as in all cases we care about we already hold the lock.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/25b142feed7d831008257d90c8b17c0115d4fc15.1680576071.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 03adabe8

io_uring/io_uring.c

+12 −6
Original line number Diff line number Diff line
@@ -967,7 +967,7 @@ bool io_aux_cqe(struct io_ring_ctx *ctx, bool defer, u64 user_data, s32 res, u32 
	return true;

}



static void __io_req_complete_post(struct io_kiocb *req)

static void __io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)

{

	struct io_ring_ctx *ctx = req->ctx;

	struct io_rsrc_node *rsrc_node = NULL;
@@ -1003,7 +1003,11 @@ static void __io_req_complete_post(struct io_kiocb *req) 
	}

	io_cq_unlock_post(ctx);



	if (rsrc_node) {

		io_ring_submit_lock(ctx, issue_flags);

		io_put_rsrc_node(rsrc_node);

		io_ring_submit_unlock(ctx, issue_flags);

	}

}



void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
@@ -1013,12 +1017,12 @@ void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) 
		io_req_task_work_add(req);

	} else if (!(issue_flags & IO_URING_F_UNLOCKED) ||

		   !(req->ctx->flags & IORING_SETUP_IOPOLL)) {

		__io_req_complete_post(req);

		__io_req_complete_post(req, issue_flags);

	} else {

		struct io_ring_ctx *ctx = req->ctx;



		mutex_lock(&ctx->uring_lock);

		__io_req_complete_post(req);

		__io_req_complete_post(req, issue_flags & ~IO_URING_F_UNLOCKED);

		mutex_unlock(&ctx->uring_lock);

	}

}
@@ -1120,7 +1124,10 @@ static __cold void io_free_req_tw(struct io_kiocb *req, struct io_tw_state *ts) 
{

	struct io_ring_ctx *ctx = req->ctx;



	if (req->rsrc_node) {

		io_tw_lock(ctx, ts);

		io_put_rsrc_node(req->rsrc_node);

	}

	io_dismantle_req(req);

	io_put_task_remote(req->task, 1);
@@ -2790,7 +2797,6 @@ static void io_req_caches_free(struct io_ring_ctx *ctx) 
static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)

{

	io_sq_thread_finish(ctx);

	io_rsrc_refs_drop(ctx);

	/* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */

	io_wait_rsrc_data(ctx->buf_data);

	io_wait_rsrc_data(ctx->file_data);

io_uring/rsrc.c

+4 −26
Original line number Diff line number Diff line
@@ -27,23 +27,10 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, 
				  struct io_mapped_ubuf **pimu,

				  struct page **last_hpage);



#define IO_RSRC_REF_BATCH	100



/* only define max */

#define IORING_MAX_FIXED_FILES	(1U << 20)

#define IORING_MAX_REG_BUFFERS	(1U << 14)



void io_rsrc_refs_drop(struct io_ring_ctx *ctx)

	__must_hold(&ctx->uring_lock)

{

	struct io_rsrc_node *node = ctx->rsrc_node;



	if (node && node->cached_refs) {

		io_rsrc_put_node(node, node->cached_refs);

		node->cached_refs = 0;

	}

}



int __io_account_mem(struct user_struct *user, unsigned long nr_pages)

{

	unsigned long page_limit, cur_pages, new_pages;
@@ -153,13 +140,6 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slo 
	*slot = NULL;

}



void io_rsrc_refs_refill(struct io_ring_ctx *ctx, struct io_rsrc_node *node)

	__must_hold(&ctx->uring_lock)

{

	node->cached_refs += IO_RSRC_REF_BATCH;

	refcount_add(IO_RSRC_REF_BATCH, &node->refs);

}



static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)

{

	struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
@@ -225,7 +205,8 @@ void io_rsrc_node_destroy(struct io_rsrc_node *ref_node) 
	kfree(ref_node);

}



__cold void io_rsrc_node_ref_zero(struct io_rsrc_node *node)

void io_rsrc_node_ref_zero(struct io_rsrc_node *node)

	__must_hold(&node->rsrc_data->ctx->uring_lock)

{

	struct io_ring_ctx *ctx = node->rsrc_data->ctx;

	unsigned long flags;
@@ -269,7 +250,7 @@ static struct io_rsrc_node *io_rsrc_node_alloc(void) 
	if (!ref_node)

		return NULL;



	refcount_set(&ref_node->refs, 1);

	ref_node->refs = 1;

	INIT_LIST_HEAD(&ref_node->node);

	INIT_LIST_HEAD(&ref_node->rsrc_list);

	ref_node->done = false;
@@ -283,8 +264,6 @@ void io_rsrc_node_switch(struct io_ring_ctx *ctx, 
	WARN_ON_ONCE(!ctx->rsrc_backup_node);

	WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node);



	io_rsrc_refs_drop(ctx);



	if (data_to_kill) {

		struct io_rsrc_node *rsrc_node = ctx->rsrc_node;
@@ -295,14 +274,13 @@ void io_rsrc_node_switch(struct io_ring_ctx *ctx, 


		atomic_inc(&data_to_kill->refs);

		/* put master ref */

		io_rsrc_put_node(rsrc_node, 1);

		io_put_rsrc_node(rsrc_node);

		ctx->rsrc_node = NULL;

	}



	if (!ctx->rsrc_node) {

		ctx->rsrc_node = ctx->rsrc_backup_node;

		ctx->rsrc_backup_node = NULL;

		ctx->rsrc_node->cached_refs = 0;

	}

}

io_uring/rsrc.h

+5 −24
Original line number Diff line number Diff line
@@ -37,13 +37,12 @@ struct io_rsrc_data { 
};



struct io_rsrc_node {

	refcount_t			refs;

	struct list_head		node;

	struct list_head		rsrc_list;

	struct io_rsrc_data		*rsrc_data;

	struct llist_node		llist;

	int				refs;

	bool				done;

	int				cached_refs;

};



struct io_mapped_ubuf {
@@ -57,10 +56,8 @@ struct io_mapped_ubuf { 
void io_rsrc_put_tw(struct callback_head *cb);

void io_rsrc_node_ref_zero(struct io_rsrc_node *node);

void io_rsrc_put_work(struct work_struct *work);

void io_rsrc_refs_refill(struct io_ring_ctx *ctx, struct io_rsrc_node *node);

void io_wait_rsrc_data(struct io_rsrc_data *data);

void io_rsrc_node_destroy(struct io_rsrc_node *ref_node);

void io_rsrc_refs_drop(struct io_ring_ctx *ctx);

int io_rsrc_node_switch_start(struct io_ring_ctx *ctx);

int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,

			  struct io_rsrc_node *node, void *rsrc);
@@ -109,38 +106,22 @@ int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, 
int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,

			unsigned int size, unsigned int type);



static inline void io_rsrc_put_node(struct io_rsrc_node *node, int nr)

{

	if (refcount_sub_and_test(nr, &node->refs))

		io_rsrc_node_ref_zero(node);

}



static inline void io_put_rsrc_node(struct io_rsrc_node *node)

{

	if (node)

		io_rsrc_put_node(node, 1);

	if (node && !--node->refs)

		io_rsrc_node_ref_zero(node);

}



static inline void io_req_put_rsrc_locked(struct io_kiocb *req,

					  struct io_ring_ctx *ctx)

	__must_hold(&ctx->uring_lock)

{

	struct io_rsrc_node *node = req->rsrc_node;



	if (node) {

		if (node == ctx->rsrc_node)

			node->cached_refs++;

		else

			io_rsrc_put_node(node, 1);

	}

	io_put_rsrc_node(req->rsrc_node);

}



static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx,

				       struct io_rsrc_node *node)

{

	node->cached_refs--;

	if (unlikely(node->cached_refs < 0))

		io_rsrc_refs_refill(ctx, node);

	node->refs++;

}



static inline void io_req_set_rsrc_node(struct io_kiocb *req,