Commit c1ef5003 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'for-6.3/iter-ubuf-2023-02-16' of git://git.kernel.dk/linux 

Pull io_uring ITER_UBUF conversion from Jens Axboe:
 "Since we now have ITER_UBUF available, switch to using it for single
  ranges as it's more efficient than ITER_IOVEC for that"

* tag 'for-6.3/iter-ubuf-2023-02-16' of git://git.kernel.dk/linux:
  block: use iter_ubuf for single range
  iov_iter: move iter_ubuf check inside restore WARN
  io_uring: use iter_ubuf for single range imports
  io_uring: switch network send/recv to ITER_UBUF
  iov: add import_ubuf()
parents cce5fe5e d46aa786

block/blk-map.c

+4 −4
Original line number Diff line number Diff line
@@ -31,6 +31,7 @@ static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, 
		return NULL;

	memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs);

	bmd->iter = *data;

	if (iter_is_iovec(data))

		bmd->iter.iov = bmd->iov;

	return bmd;

}
@@ -641,7 +642,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, 
		copy = true;

	else if (iov_iter_is_bvec(iter))

		map_bvec = true;

	else if (!iter_is_iovec(iter))

	else if (!user_backed_iter(iter))

		copy = true;

	else if (queue_virt_boundary(q))

		copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter);
@@ -682,9 +683,8 @@ int blk_rq_map_user(struct request_queue *q, struct request *rq, 
		    struct rq_map_data *map_data, void __user *ubuf,

		    unsigned long len, gfp_t gfp_mask)

{

	struct iovec iov;

	struct iov_iter i;

	int ret = import_single_range(rq_data_dir(rq), ubuf, len, &iov, &i);

	int ret = import_ubuf(rq_data_dir(rq), ubuf, len, &i);



	if (unlikely(ret < 0))

		return ret;

include/linux/uio.h

+1 −0
Original line number Diff line number Diff line
@@ -346,6 +346,7 @@ ssize_t __import_iovec(int type, const struct iovec __user *uvec, 
		 struct iov_iter *i, bool compat);

int import_single_range(int type, void __user *buf, size_t len,

		 struct iovec *iov, struct iov_iter *i);

int import_ubuf(int type, void __user *buf, size_t len, struct iov_iter *i);



static inline void iov_iter_ubuf(struct iov_iter *i, unsigned int direction,

			void __user *buf, size_t count)

io_uring/net.c

+5 −12
Original line number Diff line number Diff line
@@ -182,7 +182,7 @@ static int io_setup_async_msg(struct io_kiocb *req, 
	if (async_msg->msg.msg_name)

		async_msg->msg.msg_name = &async_msg->addr;

	/* if were using fast_iov, set it to the new one */

	if (!kmsg->free_iov) {

	if (iter_is_iovec(&kmsg->msg.msg_iter) && !kmsg->free_iov) {

		size_t fast_idx = kmsg->msg.msg_iter.iov - kmsg->fast_iov;

		async_msg->msg.msg_iter.iov = &async_msg->fast_iov[fast_idx];

	}
@@ -345,7 +345,6 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) 
	struct sockaddr_storage __address;

	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

	struct msghdr msg;

	struct iovec iov;

	struct socket *sock;

	unsigned flags;

	int min_ret = 0;
@@ -379,7 +378,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) 
	if (unlikely(!sock))

		return -ENOTSOCK;



	ret = import_single_range(ITER_SOURCE, sr->buf, sr->len, &iov, &msg.msg_iter);

	ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, &msg.msg_iter);

	if (unlikely(ret))

		return ret;
@@ -775,10 +774,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 
			}

		}



		kmsg->fast_iov[0].iov_base = buf;

		kmsg->fast_iov[0].iov_len = len;

		iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, kmsg->fast_iov, 1,

				len);

		iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, buf, len);

	}



	flags = sr->msg_flags;
@@ -846,7 +842,6 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) 
	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);

	struct msghdr msg;

	struct socket *sock;

	struct iovec iov;

	unsigned int cflags;

	unsigned flags;

	int ret, min_ret = 0;
@@ -874,7 +869,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) 
		sr->buf = buf;

	}



	ret = import_single_range(ITER_DEST, sr->buf, len, &iov, &msg.msg_iter);

	ret = import_ubuf(ITER_DEST, sr->buf, len, &msg.msg_iter);

	if (unlikely(ret))

		goto out_free;
@@ -1085,7 +1080,6 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) 
	struct sockaddr_storage __address;

	struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);

	struct msghdr msg;

	struct iovec iov;

	struct socket *sock;

	unsigned msg_flags;

	int ret, min_ret = 0;
@@ -1127,8 +1121,7 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) 
		msg.sg_from_iter = io_sg_from_iter;

	} else {

		io_notif_set_extended(zc->notif);

		ret = import_single_range(ITER_SOURCE, zc->buf, zc->len, &iov,

					  &msg.msg_iter);

		ret = import_ubuf(ITER_SOURCE, zc->buf, zc->len, &msg.msg_iter);

		if (unlikely(ret))

			return ret;

		ret = io_notif_account_mem(zc->notif, zc->len);

io_uring/rw.c

+6 −3
Original line number Diff line number Diff line
@@ -391,7 +391,7 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, 
			rw->len = sqe_len;

		}



		ret = import_single_range(ddir, buf, sqe_len, s->fast_iov, iter);

		ret = import_ubuf(ddir, buf, sqe_len, iter);

		if (ret)

			return ERR_PTR(ret);

		return NULL;
@@ -450,7 +450,10 @@ static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter) 
		struct iovec iovec;

		ssize_t nr;



		if (!iov_iter_is_bvec(iter)) {

		if (iter_is_ubuf(iter)) {

			iovec.iov_base = iter->ubuf + iter->iov_offset;

			iovec.iov_len = iov_iter_count(iter);

		} else if (!iov_iter_is_bvec(iter)) {

			iovec = iov_iter_iovec(iter);

		} else {

			iovec.iov_base = u64_to_user_ptr(rw->addr);
@@ -495,7 +498,7 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, 
	io->free_iovec = iovec;

	io->bytes_done = 0;

	/* can only be fixed buffers, no need to do anything */

	if (iov_iter_is_bvec(iter))

	if (iov_iter_is_bvec(iter) || iter_is_ubuf(iter))

		return;

	if (!iovec) {

		unsigned iov_off = 0;

lib/iov_iter.c

+13 −2
Original line number Diff line number Diff line
@@ -1877,6 +1877,17 @@ int import_single_range(int rw, void __user *buf, size_t len, 
}

EXPORT_SYMBOL(import_single_range);



int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i)

{

	if (len > MAX_RW_COUNT)

		len = MAX_RW_COUNT;

	if (unlikely(!access_ok(buf, len)))

		return -EFAULT;



	iov_iter_ubuf(i, rw, buf, len);

	return 0;

}



/**

 * iov_iter_restore() - Restore a &struct iov_iter to the same state as when

 *     iov_iter_save_state() was called.
@@ -1891,8 +1902,8 @@ EXPORT_SYMBOL(import_single_range); 
 */

void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state)

{

	if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i)) &&

			 !iov_iter_is_kvec(i) && !iter_is_ubuf(i))

	if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i) &&

			 !iter_is_ubuf(i)) && !iov_iter_is_kvec(i))

		return;

	i->iov_offset = state->iov_offset;

	i->count = state->count;