Commit 77e443ab authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe
Browse files

io_uring: never run tw and fallback in parallel 



Once we fallback a tw we want all requests to that task to be given to
the fallback wq so we dont run it in parallel with the last, i.e. post
PF_EXITING, tw run of the task.

Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/96f4987265c4312f376f206511c6af3e77aaf5ac.1670384893.git.asml.silence@gmail.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent d34b1b0b

io_uring/io_uring.c

+10 −2
Original line number Diff line number Diff line
@@ -149,6 +149,7 @@ static void io_clean_op(struct io_kiocb *req); 
static void io_queue_sqe(struct io_kiocb *req);

static void io_move_task_work_from_local(struct io_ring_ctx *ctx);

static void __io_submit_flush_completions(struct io_ring_ctx *ctx);

static __cold void io_fallback_tw(struct io_uring_task *tctx);



static struct kmem_cache *req_cachep;
@@ -1149,10 +1150,17 @@ void tctx_task_work(struct callback_head *cb) 
	struct io_uring_task *tctx = container_of(cb, struct io_uring_task,

						  task_work);

	struct llist_node fake = {};

	struct llist_node *node = io_llist_xchg(&tctx->task_list, &fake);

	struct llist_node *node;

	unsigned int loops = 1;

	unsigned int count = handle_tw_list(node, &ctx, &uring_locked, NULL);

	unsigned int count;



	if (unlikely(current->flags & PF_EXITING)) {

		io_fallback_tw(tctx);

		return;

	}



	node = io_llist_xchg(&tctx->task_list, &fake);

	count = handle_tw_list(node, &ctx, &uring_locked, NULL);

	node = io_llist_cmpxchg(&tctx->task_list, &fake, NULL);

	while (node != &fake) {

		loops++;