Commit de17c685 authored by David Howells's avatar David Howells Committed by Jakub Kicinski
Browse files

tcp_bpf: Make tcp_bpf_sendpage() go through tcp_bpf_sendmsg(MSG_SPLICE_PAGES) 



Make tcp_bpf_sendpage() a wrapper around tcp_bpf_sendmsg(MSG_SPLICE_PAGES)
rather than a loop calling tcp_sendpage().  sendpage() will be removed in
the future.

Signed-off-by: default avatarDavid Howells <dhowells@redhat.com>
cc: John Fastabend <john.fastabend@gmail.com>
cc: Jakub Sitnicki <jakub@cloudflare.com>
cc: Jens Axboe <axboe@kernel.dk>
cc: Matthew Wilcox <willy@infradead.org>
Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 5df5dd03

net/ipv4/tcp_bpf.c

+9 −40
Original line number Diff line number Diff line
@@ -568,49 +568,18 @@ static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 
static int tcp_bpf_sendpage(struct sock *sk, struct page *page, int offset,

			    size_t size, int flags)

{

	struct sk_msg tmp, *msg = NULL;

	int err = 0, copied = 0;

	struct sk_psock *psock;

	bool enospc = false;



	psock = sk_psock_get(sk);

	if (unlikely(!psock))

		return tcp_sendpage(sk, page, offset, size, flags);



	lock_sock(sk);

	if (psock->cork) {

		msg = psock->cork;

	} else {

		msg = &tmp;

		sk_msg_init(msg);

	}

	struct bio_vec bvec;

	struct msghdr msg = {

		.msg_flags = flags | MSG_SPLICE_PAGES,

	};



	/* Catch case where ring is full and sendpage is stalled. */

	if (unlikely(sk_msg_full(msg)))

		goto out_err;

	bvec_set_page(&bvec, page, size, offset);

	iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, size);



	sk_msg_page_add(msg, page, size, offset);

	sk_mem_charge(sk, size);

	copied = size;

	if (sk_msg_full(msg))

		enospc = true;

	if (psock->cork_bytes) {

		if (size > psock->cork_bytes)

			psock->cork_bytes = 0;

		else

			psock->cork_bytes -= size;

		if (psock->cork_bytes && !enospc)

			goto out_err;

		/* All cork bytes are accounted, rerun the prog. */

		psock->eval = __SK_NONE;

		psock->cork_bytes = 0;

	}

	if (flags & MSG_SENDPAGE_NOTLAST)

		msg.msg_flags |= MSG_MORE;



	err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags);

out_err:

	release_sock(sk);

	sk_psock_put(sk, psock);

	return copied ? copied : err;

	return tcp_bpf_sendmsg(sk, &msg, size);

}



enum {