Commit 636d549f authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf: misc performance improvements for cgroup' 



Stanislav Fomichev says:

====================

First patch adds custom getsockopt for TCP_ZEROCOPY_RECEIVE
to remove kmalloc and lock_sock overhead from the dat path.

Second patch removes kzalloc/kfree from getsockopt for the common cases.

Third patch switches cgroup_bpf_enabled to be per-attach to
to add only overhead for the cgroup attach types used on the system.

No visible user-side changes.

v9:
- include linux/tcp.h instead of netinet/tcp.h in sockopt_sk.c
- note that v9 depends on the commit 4be34f3d ("bpf: Don't leak
  memory in bpf getsockopt when optlen == 0") from bpf tree

v8:
- add bpi.h to tools/include/uapi in the same patch (Martin KaFai Lau)
- kmalloc instead of kzalloc when exporting buffer (Martin KaFai Lau)
- note that v8 depends on the commit 4be34f3d ("bpf: Don't leak
  memory in bpf getsockopt when optlen == 0") from bpf tree

v7:
- add comment about buffer contents for retval != 0 (Martin KaFai Lau)
- export tcp.h into tools/include/uapi (Martin KaFai Lau)
- note that v7 depends on the commit 4be34f3d ("bpf: Don't leak
  memory in bpf getsockopt when optlen == 0") from bpf tree

v6:
- avoid indirect cost for new bpf_bypass_getsockopt (Eric Dumazet)

v5:
- reorder patches to reduce the churn (Martin KaFai Lau)

v4:
- update performance numbers
- bypass_bpf_getsockopt (Martin KaFai Lau)

v3:
- remove extra newline, add comment about sizeof tcp_zerocopy_receive
  (Martin KaFai Lau)
- add another patch to remove lock_sock overhead from
  TCP_ZEROCOPY_RECEIVE; technically, this makes patch #1 obsolete,
  but I'd still prefer to keep it to help with other socket
  options

v2:
- perf numbers for getsockopt kmalloc reduction (Song Liu)
- (sk) in BPF_CGROUP_PRE_CONNECT_ENABLED (Song Liu)
- 128 -> 64 buffer size, BUILD_BUG_ON (Martin KaFai Lau)
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 13ca51d5 a9ed15da

include/linux/bpf-cgroup.h

+42 −21
Original line number Diff line number Diff line
@@ -23,8 +23,8 @@ struct ctl_table_header; 


#ifdef CONFIG_CGROUP_BPF



extern struct static_key_false cgroup_bpf_enabled_key;

#define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key)

extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE];

#define cgroup_bpf_enabled(type) static_branch_unlikely(&cgroup_bpf_enabled_key[type])



DECLARE_PER_CPU(struct bpf_cgroup_storage*,

		bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
@@ -147,6 +147,10 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, 
				       int __user *optlen, int max_optlen,

				       int retval);



int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level,

					    int optname, void *optval,

					    int *optlen, int retval);



static inline enum bpf_cgroup_storage_type cgroup_storage_type(

	struct bpf_map *map)

{
@@ -185,7 +189,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 
#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb)			      \

({									      \

	int __ret = 0;							      \

	if (cgroup_bpf_enabled)						      \

	if (cgroup_bpf_enabled(BPF_CGROUP_INET_INGRESS))		      \

		__ret = __cgroup_bpf_run_filter_skb(sk, skb,		      \

						    BPF_CGROUP_INET_INGRESS); \

									      \
@@ -195,7 +199,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 
#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb)			       \

({									       \

	int __ret = 0;							       \

	if (cgroup_bpf_enabled && sk && sk == skb->sk) {		       \

	if (cgroup_bpf_enabled(BPF_CGROUP_INET_EGRESS) && sk && sk == skb->sk) { \

		typeof(sk) __sk = sk_to_full_sk(sk);			       \

		if (sk_fullsock(__sk))					       \

			__ret = __cgroup_bpf_run_filter_skb(__sk, skb,	       \
@@ -207,7 +211,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 
#define BPF_CGROUP_RUN_SK_PROG(sk, type)				       \

({									       \

	int __ret = 0;							       \

	if (cgroup_bpf_enabled) {					       \

	if (cgroup_bpf_enabled(type)) {					       \

		__ret = __cgroup_bpf_run_filter_sk(sk, type);		       \

	}								       \

	__ret;								       \
@@ -228,7 +232,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 
#define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, type)				       \

({									       \

	int __ret = 0;							       \

	if (cgroup_bpf_enabled)						       \

	if (cgroup_bpf_enabled(type))					       \

		__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type,     \

							  NULL);	       \

	__ret;								       \
@@ -237,7 +241,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 
#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx)		       \

({									       \

	int __ret = 0;							       \

	if (cgroup_bpf_enabled)	{					       \

	if (cgroup_bpf_enabled(type))	{				       \

		lock_sock(sk);						       \

		__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type,     \

							  t_ctx);	       \
@@ -252,8 +256,10 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 
#define BPF_CGROUP_RUN_PROG_INET6_BIND_LOCK(sk, uaddr)			       \

	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_BIND, NULL)



#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (cgroup_bpf_enabled && \

					    sk->sk_prot->pre_connect)

#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk)				       \

	((cgroup_bpf_enabled(BPF_CGROUP_INET4_CONNECT) ||		       \

	  cgroup_bpf_enabled(BPF_CGROUP_INET6_CONNECT)) &&		       \

	 (sk)->sk_prot->pre_connect)



#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr)			       \

	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT)
@@ -297,7 +303,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 
#define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk)			\

({									\

	int __ret = 0;							\

	if (cgroup_bpf_enabled)						\

	if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS))			\

		__ret = __cgroup_bpf_run_filter_sock_ops(sk,		\

							 sock_ops,	\

							 BPF_CGROUP_SOCK_OPS); \
@@ -307,7 +313,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 
#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops)				       \

({									       \

	int __ret = 0;							       \

	if (cgroup_bpf_enabled && (sock_ops)->sk) {	       \

	if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS) && (sock_ops)->sk) {       \

		typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk);	       \

		if (__sk && sk_fullsock(__sk))				       \

			__ret = __cgroup_bpf_run_filter_sock_ops(__sk,	       \
@@ -320,7 +326,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 
#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type, major, minor, access)	      \

({									      \

	int __ret = 0;							      \

	if (cgroup_bpf_enabled)						      \

	if (cgroup_bpf_enabled(BPF_CGROUP_DEVICE))			      \

		__ret = __cgroup_bpf_check_dev_permission(type, major, minor, \

							  access,	      \

							  BPF_CGROUP_DEVICE); \
@@ -332,7 +338,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 
#define BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, buf, count, pos)  \

({									       \

	int __ret = 0;							       \

	if (cgroup_bpf_enabled)						       \

	if (cgroup_bpf_enabled(BPF_CGROUP_SYSCTL))			       \

		__ret = __cgroup_bpf_run_filter_sysctl(head, table, write,     \

						       buf, count, pos,        \

						       BPF_CGROUP_SYSCTL);     \
@@ -343,7 +349,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 
				       kernel_optval)			       \

({									       \

	int __ret = 0;							       \

	if (cgroup_bpf_enabled)						       \

	if (cgroup_bpf_enabled(BPF_CGROUP_SETSOCKOPT))			       \

		__ret = __cgroup_bpf_run_filter_setsockopt(sock, level,	       \

							   optname, optval,    \

							   optlen,	       \
@@ -354,7 +360,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 
#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen)			       \

({									       \

	int __ret = 0;							       \

	if (cgroup_bpf_enabled)						       \

	if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT))			       \

		get_user(__ret, optlen);				       \

	__ret;								       \

})
@@ -363,11 +369,24 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 
				       max_optlen, retval)		       \

({									       \

	int __ret = retval;						       \

	if (cgroup_bpf_enabled)						       \

		__ret = __cgroup_bpf_run_filter_getsockopt(sock, level,	       \

							   optname, optval,    \

							   optlen, max_optlen, \

							   retval);	       \

	if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT))			       \

		if (!(sock)->sk_prot->bpf_bypass_getsockopt ||		       \

		    !INDIRECT_CALL_INET_1((sock)->sk_prot->bpf_bypass_getsockopt, \

					tcp_bpf_bypass_getsockopt,	       \

					level, optname))		       \

			__ret = __cgroup_bpf_run_filter_getsockopt(	       \

				sock, level, optname, optval, optlen,	       \

				max_optlen, retval);			       \

	__ret;								       \

})



#define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval,      \

					    optlen, retval)		       \

({									       \

	int __ret = retval;						       \

	if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT))			       \

		__ret = __cgroup_bpf_run_filter_getsockopt_kern(	       \

			sock, level, optname, optval, optlen, retval);	       \

	__ret;								       \

})
@@ -427,7 +446,7 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, 
	return 0;

}



#define cgroup_bpf_enabled (0)

#define cgroup_bpf_enabled(type) (0)

#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) ({ 0; })

#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0)

#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; })
@@ -452,6 +471,8 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, 
#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; })

#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \

				       optlen, max_optlen, retval) ({ retval; })

#define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval, \

					    optlen, retval) ({ retval; })

#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \

				       kernel_optval) ({ 0; })

include/linux/filter.h

+5 −0
Original line number Diff line number Diff line
@@ -1298,6 +1298,11 @@ struct bpf_sysctl_kern { 
	u64 tmp_reg;

};



#define BPF_SOCKOPT_KERN_BUF_SIZE	32

struct bpf_sockopt_buf {

	u8		data[BPF_SOCKOPT_KERN_BUF_SIZE];

};



struct bpf_sockopt_kern {

	struct sock	*sk;

	u8		*optval;

include/linux/indirect_call_wrapper.h

+6 −0
Original line number Diff line number Diff line
@@ -60,4 +60,10 @@ 
#define INDIRECT_CALL_INET(f, f2, f1, ...) f(__VA_ARGS__)

#endif



#if IS_ENABLED(CONFIG_INET)

#define INDIRECT_CALL_INET_1(f, f1, ...) INDIRECT_CALL_1(f, f1, __VA_ARGS__)

#else

#define INDIRECT_CALL_INET_1(f, f1, ...) f(__VA_ARGS__)

#endif



#endif

include/net/sock.h

+2 −0
Original line number Diff line number Diff line
@@ -1174,6 +1174,8 @@ struct proto { 


	int			(*backlog_rcv) (struct sock *sk,

						struct sk_buff *skb);

	bool			(*bpf_bypass_getsockopt)(int level,

							 int optname);



	void		(*release_cb)(struct sock *sk);

include/net/tcp.h

+1 −0
Original line number Diff line number Diff line
@@ -403,6 +403,7 @@ __poll_t tcp_poll(struct file *file, struct socket *sock, 
		      struct poll_table_struct *wait);

int tcp_getsockopt(struct sock *sk, int level, int optname,

		   char __user *optval, int __user *optlen);

bool tcp_bpf_bypass_getsockopt(int level, int optname);

int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,

		   unsigned int optlen);

void tcp_set_keepalive(struct sock *sk, int val);