Loading net/sunrpc/xprtrdma/backchannel.c +3 −4 Original line number Diff line number Diff line Loading @@ -27,7 +27,7 @@ static void rpcrdma_bc_free_rqst(struct rpcrdma_xprt *r_xprt, list_del(&req->rl_all); spin_unlock(&buf->rb_reqslock); rpcrdma_destroy_req(&r_xprt->rx_ia, req); rpcrdma_destroy_req(req); kfree(rqst); } Loading @@ -35,7 +35,6 @@ static void rpcrdma_bc_free_rqst(struct rpcrdma_xprt *r_xprt, static int rpcrdma_bc_setup_rqst(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst) { struct rpcrdma_ia *ia = &r_xprt->rx_ia; struct rpcrdma_regbuf *rb; struct rpcrdma_req *req; size_t size; Loading @@ -45,14 +44,14 @@ static int rpcrdma_bc_setup_rqst(struct rpcrdma_xprt *r_xprt, return PTR_ERR(req); req->rl_backchannel = true; rb = rpcrdma_alloc_regbuf(ia, RPCRDMA_HDRBUF_SIZE, rb = rpcrdma_alloc_regbuf(RPCRDMA_HDRBUF_SIZE, DMA_TO_DEVICE, GFP_KERNEL); if (IS_ERR(rb)) goto out_fail; req->rl_rdmabuf = rb; size = r_xprt->rx_data.inline_rsize; rb = rpcrdma_alloc_regbuf(ia, size, DMA_TO_DEVICE, GFP_KERNEL); rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, GFP_KERNEL); if (IS_ERR(rb)) goto out_fail; req->rl_sendbuf = rb; Loading net/sunrpc/xprtrdma/transport.c +5 −6 Original line number Diff line number Diff line Loading @@ -490,7 +490,7 @@ rpcrdma_get_rdmabuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, if (req->rl_rdmabuf) return true; rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, size, DMA_TO_DEVICE, flags); rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, flags); if (IS_ERR(rb)) return false; Loading @@ -517,12 +517,11 @@ rpcrdma_get_sendbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, return true; min_size = max_t(size_t, size, r_xprt->rx_data.inline_wsize); rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, min_size, DMA_TO_DEVICE, flags); rb = rpcrdma_alloc_regbuf(min_size, DMA_TO_DEVICE, flags); if (IS_ERR(rb)) return false; rpcrdma_free_regbuf(&r_xprt->rx_ia, req->rl_sendbuf); rpcrdma_free_regbuf(req->rl_sendbuf); r_xprt->rx_stats.hardway_register_count += min_size; req->rl_sendbuf = rb; return true; Loading @@ -548,11 +547,11 @@ rpcrdma_get_recvbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, if (req->rl_recvbuf && rdmab_length(req->rl_recvbuf) >= size) return true; rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, size, DMA_NONE, flags); rb = rpcrdma_alloc_regbuf(size, DMA_NONE, flags); if (IS_ERR(rb)) return false; rpcrdma_free_regbuf(&r_xprt->rx_ia, req->rl_recvbuf); rpcrdma_free_regbuf(req->rl_recvbuf); r_xprt->rx_stats.hardway_register_count += size; req->rl_recvbuf = rb; return true; Loading net/sunrpc/xprtrdma/verbs.c +12 −16 Original line number Diff line number Diff line Loading @@ -865,7 +865,7 @@ rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt) if (rep == NULL) goto out; rep->rr_rdmabuf = rpcrdma_alloc_regbuf(ia, cdata->inline_rsize, rep->rr_rdmabuf = rpcrdma_alloc_regbuf(cdata->inline_rsize, DMA_FROM_DEVICE, GFP_KERNEL); if (IS_ERR(rep->rr_rdmabuf)) { rc = PTR_ERR(rep->rr_rdmabuf); Loading Loading @@ -966,18 +966,18 @@ rpcrdma_buffer_get_rep_locked(struct rpcrdma_buffer *buf) } static void rpcrdma_destroy_rep(struct rpcrdma_ia *ia, struct rpcrdma_rep *rep) rpcrdma_destroy_rep(struct rpcrdma_rep *rep) { rpcrdma_free_regbuf(ia, rep->rr_rdmabuf); rpcrdma_free_regbuf(rep->rr_rdmabuf); kfree(rep); } void rpcrdma_destroy_req(struct rpcrdma_ia *ia, struct rpcrdma_req *req) rpcrdma_destroy_req(struct rpcrdma_req *req) { rpcrdma_free_regbuf(ia, req->rl_recvbuf); rpcrdma_free_regbuf(ia, req->rl_sendbuf); rpcrdma_free_regbuf(ia, req->rl_rdmabuf); rpcrdma_free_regbuf(req->rl_recvbuf); rpcrdma_free_regbuf(req->rl_sendbuf); rpcrdma_free_regbuf(req->rl_rdmabuf); kfree(req); } Loading Loading @@ -1010,15 +1010,13 @@ rpcrdma_destroy_mrs(struct rpcrdma_buffer *buf) void rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) { struct rpcrdma_ia *ia = rdmab_to_ia(buf); cancel_delayed_work_sync(&buf->rb_recovery_worker); while (!list_empty(&buf->rb_recv_bufs)) { struct rpcrdma_rep *rep; rep = rpcrdma_buffer_get_rep_locked(buf); rpcrdma_destroy_rep(ia, rep); rpcrdma_destroy_rep(rep); } buf->rb_send_count = 0; Loading @@ -1031,7 +1029,7 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) list_del(&req->rl_all); spin_unlock(&buf->rb_reqslock); rpcrdma_destroy_req(ia, req); rpcrdma_destroy_req(req); spin_lock(&buf->rb_reqslock); } spin_unlock(&buf->rb_reqslock); Loading Loading @@ -1174,7 +1172,6 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) /** * rpcrdma_alloc_regbuf - allocate and DMA-map memory for SEND/RECV buffers * @ia: controlling rpcrdma_ia * @size: size of buffer to be allocated, in bytes * @direction: direction of data movement * @flags: GFP flags Loading @@ -1187,8 +1184,8 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) * or Replies they may be registered externally via ro_map. */ struct rpcrdma_regbuf * rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, enum dma_data_direction direction, gfp_t flags) rpcrdma_alloc_regbuf(size_t size, enum dma_data_direction direction, gfp_t flags) { struct rpcrdma_regbuf *rb; Loading Loading @@ -1239,11 +1236,10 @@ rpcrdma_dma_unmap_regbuf(struct rpcrdma_regbuf *rb) /** * rpcrdma_free_regbuf - deregister and free registered buffer * @ia: controlling rpcrdma_ia * @rb: regbuf to be deregistered and freed */ void rpcrdma_free_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb) rpcrdma_free_regbuf(struct rpcrdma_regbuf *rb) { if (!rb) return; Loading net/sunrpc/xprtrdma/xprt_rdma.h +3 −5 Original line number Diff line number Diff line Loading @@ -465,7 +465,7 @@ int rpcrdma_ep_post_recv(struct rpcrdma_ia *, struct rpcrdma_ep *, */ struct rpcrdma_req *rpcrdma_create_req(struct rpcrdma_xprt *); struct rpcrdma_rep *rpcrdma_create_rep(struct rpcrdma_xprt *); void rpcrdma_destroy_req(struct rpcrdma_ia *, struct rpcrdma_req *); void rpcrdma_destroy_req(struct rpcrdma_req *); int rpcrdma_buffer_create(struct rpcrdma_xprt *); void rpcrdma_buffer_destroy(struct rpcrdma_buffer *); Loading @@ -478,12 +478,10 @@ void rpcrdma_recv_buffer_put(struct rpcrdma_rep *); void rpcrdma_defer_mr_recovery(struct rpcrdma_mw *); struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(struct rpcrdma_ia *, size_t, enum dma_data_direction, struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(size_t, enum dma_data_direction, gfp_t); bool __rpcrdma_dma_map_regbuf(struct rpcrdma_ia *, struct rpcrdma_regbuf *); void rpcrdma_free_regbuf(struct rpcrdma_ia *, struct rpcrdma_regbuf *); void rpcrdma_free_regbuf(struct rpcrdma_regbuf *); static inline bool rpcrdma_regbuf_is_mapped(struct rpcrdma_regbuf *rb) Loading Loading
net/sunrpc/xprtrdma/backchannel.c +3 −4 Original line number Diff line number Diff line Loading @@ -27,7 +27,7 @@ static void rpcrdma_bc_free_rqst(struct rpcrdma_xprt *r_xprt, list_del(&req->rl_all); spin_unlock(&buf->rb_reqslock); rpcrdma_destroy_req(&r_xprt->rx_ia, req); rpcrdma_destroy_req(req); kfree(rqst); } Loading @@ -35,7 +35,6 @@ static void rpcrdma_bc_free_rqst(struct rpcrdma_xprt *r_xprt, static int rpcrdma_bc_setup_rqst(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst) { struct rpcrdma_ia *ia = &r_xprt->rx_ia; struct rpcrdma_regbuf *rb; struct rpcrdma_req *req; size_t size; Loading @@ -45,14 +44,14 @@ static int rpcrdma_bc_setup_rqst(struct rpcrdma_xprt *r_xprt, return PTR_ERR(req); req->rl_backchannel = true; rb = rpcrdma_alloc_regbuf(ia, RPCRDMA_HDRBUF_SIZE, rb = rpcrdma_alloc_regbuf(RPCRDMA_HDRBUF_SIZE, DMA_TO_DEVICE, GFP_KERNEL); if (IS_ERR(rb)) goto out_fail; req->rl_rdmabuf = rb; size = r_xprt->rx_data.inline_rsize; rb = rpcrdma_alloc_regbuf(ia, size, DMA_TO_DEVICE, GFP_KERNEL); rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, GFP_KERNEL); if (IS_ERR(rb)) goto out_fail; req->rl_sendbuf = rb; Loading
net/sunrpc/xprtrdma/transport.c +5 −6 Original line number Diff line number Diff line Loading @@ -490,7 +490,7 @@ rpcrdma_get_rdmabuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, if (req->rl_rdmabuf) return true; rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, size, DMA_TO_DEVICE, flags); rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, flags); if (IS_ERR(rb)) return false; Loading @@ -517,12 +517,11 @@ rpcrdma_get_sendbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, return true; min_size = max_t(size_t, size, r_xprt->rx_data.inline_wsize); rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, min_size, DMA_TO_DEVICE, flags); rb = rpcrdma_alloc_regbuf(min_size, DMA_TO_DEVICE, flags); if (IS_ERR(rb)) return false; rpcrdma_free_regbuf(&r_xprt->rx_ia, req->rl_sendbuf); rpcrdma_free_regbuf(req->rl_sendbuf); r_xprt->rx_stats.hardway_register_count += min_size; req->rl_sendbuf = rb; return true; Loading @@ -548,11 +547,11 @@ rpcrdma_get_recvbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, if (req->rl_recvbuf && rdmab_length(req->rl_recvbuf) >= size) return true; rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, size, DMA_NONE, flags); rb = rpcrdma_alloc_regbuf(size, DMA_NONE, flags); if (IS_ERR(rb)) return false; rpcrdma_free_regbuf(&r_xprt->rx_ia, req->rl_recvbuf); rpcrdma_free_regbuf(req->rl_recvbuf); r_xprt->rx_stats.hardway_register_count += size; req->rl_recvbuf = rb; return true; Loading
net/sunrpc/xprtrdma/verbs.c +12 −16 Original line number Diff line number Diff line Loading @@ -865,7 +865,7 @@ rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt) if (rep == NULL) goto out; rep->rr_rdmabuf = rpcrdma_alloc_regbuf(ia, cdata->inline_rsize, rep->rr_rdmabuf = rpcrdma_alloc_regbuf(cdata->inline_rsize, DMA_FROM_DEVICE, GFP_KERNEL); if (IS_ERR(rep->rr_rdmabuf)) { rc = PTR_ERR(rep->rr_rdmabuf); Loading Loading @@ -966,18 +966,18 @@ rpcrdma_buffer_get_rep_locked(struct rpcrdma_buffer *buf) } static void rpcrdma_destroy_rep(struct rpcrdma_ia *ia, struct rpcrdma_rep *rep) rpcrdma_destroy_rep(struct rpcrdma_rep *rep) { rpcrdma_free_regbuf(ia, rep->rr_rdmabuf); rpcrdma_free_regbuf(rep->rr_rdmabuf); kfree(rep); } void rpcrdma_destroy_req(struct rpcrdma_ia *ia, struct rpcrdma_req *req) rpcrdma_destroy_req(struct rpcrdma_req *req) { rpcrdma_free_regbuf(ia, req->rl_recvbuf); rpcrdma_free_regbuf(ia, req->rl_sendbuf); rpcrdma_free_regbuf(ia, req->rl_rdmabuf); rpcrdma_free_regbuf(req->rl_recvbuf); rpcrdma_free_regbuf(req->rl_sendbuf); rpcrdma_free_regbuf(req->rl_rdmabuf); kfree(req); } Loading Loading @@ -1010,15 +1010,13 @@ rpcrdma_destroy_mrs(struct rpcrdma_buffer *buf) void rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) { struct rpcrdma_ia *ia = rdmab_to_ia(buf); cancel_delayed_work_sync(&buf->rb_recovery_worker); while (!list_empty(&buf->rb_recv_bufs)) { struct rpcrdma_rep *rep; rep = rpcrdma_buffer_get_rep_locked(buf); rpcrdma_destroy_rep(ia, rep); rpcrdma_destroy_rep(rep); } buf->rb_send_count = 0; Loading @@ -1031,7 +1029,7 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) list_del(&req->rl_all); spin_unlock(&buf->rb_reqslock); rpcrdma_destroy_req(ia, req); rpcrdma_destroy_req(req); spin_lock(&buf->rb_reqslock); } spin_unlock(&buf->rb_reqslock); Loading Loading @@ -1174,7 +1172,6 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) /** * rpcrdma_alloc_regbuf - allocate and DMA-map memory for SEND/RECV buffers * @ia: controlling rpcrdma_ia * @size: size of buffer to be allocated, in bytes * @direction: direction of data movement * @flags: GFP flags Loading @@ -1187,8 +1184,8 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) * or Replies they may be registered externally via ro_map. */ struct rpcrdma_regbuf * rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, enum dma_data_direction direction, gfp_t flags) rpcrdma_alloc_regbuf(size_t size, enum dma_data_direction direction, gfp_t flags) { struct rpcrdma_regbuf *rb; Loading Loading @@ -1239,11 +1236,10 @@ rpcrdma_dma_unmap_regbuf(struct rpcrdma_regbuf *rb) /** * rpcrdma_free_regbuf - deregister and free registered buffer * @ia: controlling rpcrdma_ia * @rb: regbuf to be deregistered and freed */ void rpcrdma_free_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb) rpcrdma_free_regbuf(struct rpcrdma_regbuf *rb) { if (!rb) return; Loading
net/sunrpc/xprtrdma/xprt_rdma.h +3 −5 Original line number Diff line number Diff line Loading @@ -465,7 +465,7 @@ int rpcrdma_ep_post_recv(struct rpcrdma_ia *, struct rpcrdma_ep *, */ struct rpcrdma_req *rpcrdma_create_req(struct rpcrdma_xprt *); struct rpcrdma_rep *rpcrdma_create_rep(struct rpcrdma_xprt *); void rpcrdma_destroy_req(struct rpcrdma_ia *, struct rpcrdma_req *); void rpcrdma_destroy_req(struct rpcrdma_req *); int rpcrdma_buffer_create(struct rpcrdma_xprt *); void rpcrdma_buffer_destroy(struct rpcrdma_buffer *); Loading @@ -478,12 +478,10 @@ void rpcrdma_recv_buffer_put(struct rpcrdma_rep *); void rpcrdma_defer_mr_recovery(struct rpcrdma_mw *); struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(struct rpcrdma_ia *, size_t, enum dma_data_direction, struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(size_t, enum dma_data_direction, gfp_t); bool __rpcrdma_dma_map_regbuf(struct rpcrdma_ia *, struct rpcrdma_regbuf *); void rpcrdma_free_regbuf(struct rpcrdma_ia *, struct rpcrdma_regbuf *); void rpcrdma_free_regbuf(struct rpcrdma_regbuf *); static inline bool rpcrdma_regbuf_is_mapped(struct rpcrdma_regbuf *rb) Loading