Commit c8576f3e authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: refactor req allocation 



Follow the io_get_sqe pattern returning the result via a pointer
and hide request cache refill inside io_alloc_req().

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/8c37c2e8a3cb5e4cd6a8ae3b91371227a92708a6.1674484266.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent b5083dfa

io_uring/io_uring.c

+3 −4
Original line number Diff line number Diff line
@@ -2415,9 +2415,8 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) 
		const struct io_uring_sqe *sqe;

		struct io_kiocb *req;



		if (unlikely(!io_alloc_req_refill(ctx)))

		if (unlikely(!io_alloc_req(ctx, &req)))

			break;

		req = io_alloc_req(ctx);

		if (unlikely(!io_get_sqe(ctx, &sqe))) {

			io_req_add_to_cache(req, ctx);

			break;
@@ -2736,14 +2735,14 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) 


static void io_req_caches_free(struct io_ring_ctx *ctx)

{

	struct io_kiocb *req;

	int nr = 0;



	mutex_lock(&ctx->uring_lock);

	io_flush_cached_locked_reqs(ctx, &ctx->submit_state);



	while (!io_req_cache_empty(ctx)) {

		struct io_kiocb *req = io_alloc_req(ctx);



		req = io_extract_req(ctx);

		kmem_cache_free(req_cachep, req);

		nr++;

	}

io_uring/io_uring.h

+11 −8
Original line number Diff line number Diff line
@@ -341,16 +341,9 @@ static inline bool io_req_cache_empty(struct io_ring_ctx *ctx) 
	return !ctx->submit_state.free_list.next;

}



static inline bool io_alloc_req_refill(struct io_ring_ctx *ctx)

{

	if (unlikely(io_req_cache_empty(ctx)))

		return __io_alloc_req_refill(ctx);

	return true;

}



extern struct kmem_cache *req_cachep;



static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)

static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx)

{

	struct io_kiocb *req;
@@ -360,6 +353,16 @@ static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) 
	return req;

}



static inline bool io_alloc_req(struct io_ring_ctx *ctx, struct io_kiocb **req)

{

	if (unlikely(io_req_cache_empty(ctx))) {

		if (!__io_alloc_req_refill(ctx))

			return false;

	}

	*req = io_extract_req(ctx);

	return true;

}



static inline bool io_allowed_defer_tw_run(struct io_ring_ctx *ctx)

{

	return likely(ctx->submitter_task == current);

io_uring/notif.c

+1 −2
Original line number Diff line number Diff line
@@ -68,9 +68,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) 
	struct io_kiocb *notif;

	struct io_notif_data *nd;



	if (unlikely(!io_alloc_req_refill(ctx)))

	if (unlikely(!io_alloc_req(ctx, &notif)))

		return NULL;

	notif = io_alloc_req(ctx);

	notif->opcode = IORING_OP_NOP;

	notif->flags = 0;

	notif->file = NULL;