| .. | .. |
|---|
| 5 | 5 | * Support for backward direction RPCs on RPC/RDMA (server-side). |
|---|
| 6 | 6 | */ |
|---|
| 7 | 7 | |
|---|
| 8 | | -#include <linux/module.h> |
|---|
| 9 | | - |
|---|
| 10 | 8 | #include <linux/sunrpc/svc_rdma.h> |
|---|
| 11 | 9 | |
|---|
| 12 | 10 | #include "xprt_rdma.h" |
|---|
| 13 | 11 | #include <trace/events/rpcrdma.h> |
|---|
| 14 | 12 | |
|---|
| 15 | | -#define RPCDBG_FACILITY RPCDBG_SVCXPRT |
|---|
| 16 | | - |
|---|
| 17 | | -#undef SVCRDMA_BACKCHANNEL_DEBUG |
|---|
| 18 | | - |
|---|
| 19 | 13 | /** |
|---|
| 20 | | - * svc_rdma_handle_bc_reply - Process incoming backchannel reply |
|---|
| 21 | | - * @xprt: controlling backchannel transport |
|---|
| 22 | | - * @rdma_resp: pointer to incoming transport header |
|---|
| 23 | | - * @rcvbuf: XDR buffer into which to decode the reply |
|---|
| 14 | + * svc_rdma_handle_bc_reply - Process incoming backchannel Reply |
|---|
| 15 | + * @rqstp: resources for handling the Reply |
|---|
| 16 | + * @rctxt: Received message |
|---|
| 24 | 17 | * |
|---|
| 25 | | - * Returns: |
|---|
| 26 | | - * %0 if @rcvbuf is filled in, xprt_complete_rqst called, |
|---|
| 27 | | - * %-EAGAIN if server should call ->recvfrom again. |
|---|
| 28 | 18 | */ |
|---|
| 29 | | -int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, __be32 *rdma_resp, |
|---|
| 30 | | - struct xdr_buf *rcvbuf) |
|---|
| 19 | +void svc_rdma_handle_bc_reply(struct svc_rqst *rqstp, |
|---|
| 20 | + struct svc_rdma_recv_ctxt *rctxt) |
|---|
| 31 | 21 | { |
|---|
| 22 | + struct svc_xprt *sxprt = rqstp->rq_xprt; |
|---|
| 23 | + struct rpc_xprt *xprt = sxprt->xpt_bc_xprt; |
|---|
| 32 | 24 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); |
|---|
| 25 | + struct xdr_buf *rcvbuf = &rqstp->rq_arg; |
|---|
| 33 | 26 | struct kvec *dst, *src = &rcvbuf->head[0]; |
|---|
| 27 | + __be32 *rdma_resp = rctxt->rc_recv_buf; |
|---|
| 34 | 28 | struct rpc_rqst *req; |
|---|
| 35 | | - unsigned long cwnd; |
|---|
| 36 | 29 | u32 credits; |
|---|
| 37 | | - size_t len; |
|---|
| 38 | | - __be32 xid; |
|---|
| 39 | | - __be32 *p; |
|---|
| 40 | | - int ret; |
|---|
| 41 | 30 | |
|---|
| 42 | | - p = (__be32 *)src->iov_base; |
|---|
| 43 | | - len = src->iov_len; |
|---|
| 44 | | - xid = *rdma_resp; |
|---|
| 45 | | - |
|---|
| 46 | | -#ifdef SVCRDMA_BACKCHANNEL_DEBUG |
|---|
| 47 | | - pr_info("%s: xid=%08x, length=%zu\n", |
|---|
| 48 | | - __func__, be32_to_cpu(xid), len); |
|---|
| 49 | | - pr_info("%s: RPC/RDMA: %*ph\n", |
|---|
| 50 | | - __func__, (int)RPCRDMA_HDRLEN_MIN, rdma_resp); |
|---|
| 51 | | - pr_info("%s: RPC: %*ph\n", |
|---|
| 52 | | - __func__, (int)len, p); |
|---|
| 53 | | -#endif |
|---|
| 54 | | - |
|---|
| 55 | | - ret = -EAGAIN; |
|---|
| 56 | | - if (src->iov_len < 24) |
|---|
| 57 | | - goto out_shortreply; |
|---|
| 58 | | - |
|---|
| 59 | | - spin_lock(&xprt->recv_lock); |
|---|
| 60 | | - req = xprt_lookup_rqst(xprt, xid); |
|---|
| 31 | + spin_lock(&xprt->queue_lock); |
|---|
| 32 | + req = xprt_lookup_rqst(xprt, *rdma_resp); |
|---|
| 61 | 33 | if (!req) |
|---|
| 62 | | - goto out_notfound; |
|---|
| 34 | + goto out_unlock; |
|---|
| 63 | 35 | |
|---|
| 64 | 36 | dst = &req->rq_private_buf.head[0]; |
|---|
| 65 | 37 | memcpy(&req->rq_private_buf, &req->rq_rcv_buf, sizeof(struct xdr_buf)); |
|---|
| 66 | | - if (dst->iov_len < len) |
|---|
| 38 | + if (dst->iov_len < src->iov_len) |
|---|
| 67 | 39 | goto out_unlock; |
|---|
| 68 | | - memcpy(dst->iov_base, p, len); |
|---|
| 40 | + memcpy(dst->iov_base, src->iov_base, src->iov_len); |
|---|
| 41 | + xprt_pin_rqst(req); |
|---|
| 42 | + spin_unlock(&xprt->queue_lock); |
|---|
| 69 | 43 | |
|---|
| 70 | 44 | credits = be32_to_cpup(rdma_resp + 2); |
|---|
| 71 | 45 | if (credits == 0) |
|---|
| 72 | 46 | credits = 1; /* don't deadlock */ |
|---|
| 73 | 47 | else if (credits > r_xprt->rx_buf.rb_bc_max_requests) |
|---|
| 74 | 48 | credits = r_xprt->rx_buf.rb_bc_max_requests; |
|---|
| 75 | | - |
|---|
| 76 | | - spin_lock_bh(&xprt->transport_lock); |
|---|
| 77 | | - cwnd = xprt->cwnd; |
|---|
| 49 | + spin_lock(&xprt->transport_lock); |
|---|
| 78 | 50 | xprt->cwnd = credits << RPC_CWNDSHIFT; |
|---|
| 79 | | - if (xprt->cwnd > cwnd) |
|---|
| 80 | | - xprt_release_rqst_cong(req->rq_task); |
|---|
| 81 | | - spin_unlock_bh(&xprt->transport_lock); |
|---|
| 51 | + spin_unlock(&xprt->transport_lock); |
|---|
| 82 | 52 | |
|---|
| 83 | | - |
|---|
| 84 | | - ret = 0; |
|---|
| 53 | + spin_lock(&xprt->queue_lock); |
|---|
| 85 | 54 | xprt_complete_rqst(req->rq_task, rcvbuf->len); |
|---|
| 55 | + xprt_unpin_rqst(req); |
|---|
| 86 | 56 | rcvbuf->len = 0; |
|---|
| 87 | 57 | |
|---|
| 88 | 58 | out_unlock: |
|---|
| 89 | | - spin_unlock(&xprt->recv_lock); |
|---|
| 90 | | -out: |
|---|
| 91 | | - return ret; |
|---|
| 92 | | - |
|---|
| 93 | | -out_shortreply: |
|---|
| 94 | | - dprintk("svcrdma: short bc reply: xprt=%p, len=%zu\n", |
|---|
| 95 | | - xprt, src->iov_len); |
|---|
| 96 | | - goto out; |
|---|
| 97 | | - |
|---|
| 98 | | -out_notfound: |
|---|
| 99 | | - dprintk("svcrdma: unrecognized bc reply: xprt=%p, xid=%08x\n", |
|---|
| 100 | | - xprt, be32_to_cpu(xid)); |
|---|
| 101 | | - goto out_unlock; |
|---|
| 59 | + spin_unlock(&xprt->queue_lock); |
|---|
| 102 | 60 | } |
|---|
| 103 | 61 | |
|---|
| 104 | 62 | /* Send a backwards direction RPC call. |
|---|
| .. | .. |
|---|
| 120 | 78 | { |
|---|
| 121 | 79 | int ret; |
|---|
| 122 | 80 | |
|---|
| 123 | | - ret = svc_rdma_map_reply_msg(rdma, ctxt, &rqst->rq_snd_buf, NULL); |
|---|
| 81 | + ret = svc_rdma_map_reply_msg(rdma, ctxt, NULL, &rqst->rq_snd_buf); |
|---|
| 124 | 82 | if (ret < 0) |
|---|
| 125 | 83 | return -EIO; |
|---|
| 126 | 84 | |
|---|
| .. | .. |
|---|
| 129 | 87 | */ |
|---|
| 130 | 88 | get_page(virt_to_page(rqst->rq_buffer)); |
|---|
| 131 | 89 | ctxt->sc_send_wr.opcode = IB_WR_SEND; |
|---|
| 132 | | - return svc_rdma_send(rdma, &ctxt->sc_send_wr); |
|---|
| 90 | + return svc_rdma_send(rdma, ctxt); |
|---|
| 133 | 91 | } |
|---|
| 134 | 92 | |
|---|
| 135 | 93 | /* Server-side transport endpoint wants a whole page for its send |
|---|
| .. | .. |
|---|
| 184 | 142 | if (!ctxt) |
|---|
| 185 | 143 | goto drop_connection; |
|---|
| 186 | 144 | |
|---|
| 187 | | - p = ctxt->sc_xprt_buf; |
|---|
| 145 | + p = xdr_reserve_space(&ctxt->sc_stream, RPCRDMA_HDRLEN_MIN); |
|---|
| 146 | + if (!p) |
|---|
| 147 | + goto put_ctxt; |
|---|
| 188 | 148 | *p++ = rqst->rq_xid; |
|---|
| 189 | 149 | *p++ = rpcrdma_version; |
|---|
| 190 | 150 | *p++ = cpu_to_be32(r_xprt->rx_buf.rb_bc_max_requests); |
|---|
| .. | .. |
|---|
| 192 | 152 | *p++ = xdr_zero; |
|---|
| 193 | 153 | *p++ = xdr_zero; |
|---|
| 194 | 154 | *p = xdr_zero; |
|---|
| 195 | | - svc_rdma_sync_reply_hdr(rdma, ctxt, RPCRDMA_HDRLEN_MIN); |
|---|
| 196 | 155 | |
|---|
| 197 | | -#ifdef SVCRDMA_BACKCHANNEL_DEBUG |
|---|
| 198 | | - pr_info("%s: %*ph\n", __func__, 64, rqst->rq_buffer); |
|---|
| 199 | | -#endif |
|---|
| 200 | | - |
|---|
| 156 | + rqst->rq_xtime = ktime_get(); |
|---|
| 201 | 157 | rc = svc_rdma_bc_sendto(rdma, rqst, ctxt); |
|---|
| 202 | | - if (rc) { |
|---|
| 203 | | - svc_rdma_send_ctxt_put(rdma, ctxt); |
|---|
| 204 | | - goto drop_connection; |
|---|
| 205 | | - } |
|---|
| 206 | | - return rc; |
|---|
| 158 | + if (rc) |
|---|
| 159 | + goto put_ctxt; |
|---|
| 160 | + return 0; |
|---|
| 161 | + |
|---|
| 162 | +put_ctxt: |
|---|
| 163 | + svc_rdma_send_ctxt_put(rdma, ctxt); |
|---|
| 207 | 164 | |
|---|
| 208 | 165 | drop_connection: |
|---|
| 209 | | - dprintk("svcrdma: failed to send bc call\n"); |
|---|
| 210 | | - xprt_disconnect_done(xprt); |
|---|
| 211 | 166 | return -ENOTCONN; |
|---|
| 212 | 167 | } |
|---|
| 213 | 168 | |
|---|
| 214 | | -/* Send an RPC call on the passive end of a transport |
|---|
| 215 | | - * connection. |
|---|
| 169 | +/** |
|---|
| 170 | + * xprt_rdma_bc_send_request - Send a reverse-direction Call |
|---|
| 171 | + * @rqst: rpc_rqst containing Call message to be sent |
|---|
| 172 | + * |
|---|
| 173 | + * Return values: |
|---|
| 174 | + * %0 if the message was sent successfully |
|---|
| 175 | + * %ENOTCONN if the message was not sent |
|---|
| 216 | 176 | */ |
|---|
| 217 | | -static int |
|---|
| 218 | | -xprt_rdma_bc_send_request(struct rpc_task *task) |
|---|
| 177 | +static int xprt_rdma_bc_send_request(struct rpc_rqst *rqst) |
|---|
| 219 | 178 | { |
|---|
| 220 | | - struct rpc_rqst *rqst = task->tk_rqstp; |
|---|
| 221 | 179 | struct svc_xprt *sxprt = rqst->rq_xprt->bc_xprt; |
|---|
| 222 | | - struct svcxprt_rdma *rdma; |
|---|
| 180 | + struct svcxprt_rdma *rdma = |
|---|
| 181 | + container_of(sxprt, struct svcxprt_rdma, sc_xprt); |
|---|
| 223 | 182 | int ret; |
|---|
| 224 | 183 | |
|---|
| 225 | | - dprintk("svcrdma: sending bc call with xid: %08x\n", |
|---|
| 226 | | - be32_to_cpu(rqst->rq_xid)); |
|---|
| 184 | + if (test_bit(XPT_DEAD, &sxprt->xpt_flags)) |
|---|
| 185 | + return -ENOTCONN; |
|---|
| 227 | 186 | |
|---|
| 228 | | - if (!mutex_trylock(&sxprt->xpt_mutex)) { |
|---|
| 229 | | - rpc_sleep_on(&sxprt->xpt_bc_pending, task, NULL); |
|---|
| 230 | | - if (!mutex_trylock(&sxprt->xpt_mutex)) |
|---|
| 231 | | - return -EAGAIN; |
|---|
| 232 | | - rpc_wake_up_queued_task(&sxprt->xpt_bc_pending, task); |
|---|
| 233 | | - } |
|---|
| 234 | | - |
|---|
| 235 | | - ret = -ENOTCONN; |
|---|
| 236 | | - rdma = container_of(sxprt, struct svcxprt_rdma, sc_xprt); |
|---|
| 237 | | - if (!test_bit(XPT_DEAD, &sxprt->xpt_flags)) |
|---|
| 238 | | - ret = rpcrdma_bc_send_request(rdma, rqst); |
|---|
| 239 | | - |
|---|
| 240 | | - mutex_unlock(&sxprt->xpt_mutex); |
|---|
| 241 | | - |
|---|
| 242 | | - if (ret < 0) |
|---|
| 243 | | - return ret; |
|---|
| 244 | | - return 0; |
|---|
| 187 | + ret = rpcrdma_bc_send_request(rdma, rqst); |
|---|
| 188 | + if (ret == -ENOTCONN) |
|---|
| 189 | + svc_close_xprt(sxprt); |
|---|
| 190 | + return ret; |
|---|
| 245 | 191 | } |
|---|
| 246 | 192 | |
|---|
| 247 | 193 | static void |
|---|
| 248 | 194 | xprt_rdma_bc_close(struct rpc_xprt *xprt) |
|---|
| 249 | 195 | { |
|---|
| 250 | | - dprintk("svcrdma: %s: xprt %p\n", __func__, xprt); |
|---|
| 196 | + xprt_disconnect_done(xprt); |
|---|
| 251 | 197 | xprt->cwnd = RPC_CWNDSHIFT; |
|---|
| 252 | 198 | } |
|---|
| 253 | 199 | |
|---|
| 254 | 200 | static void |
|---|
| 255 | 201 | xprt_rdma_bc_put(struct rpc_xprt *xprt) |
|---|
| 256 | 202 | { |
|---|
| 257 | | - dprintk("svcrdma: %s: xprt %p\n", __func__, xprt); |
|---|
| 258 | | - |
|---|
| 259 | 203 | xprt_rdma_free_addresses(xprt); |
|---|
| 260 | 204 | xprt_free(xprt); |
|---|
| 261 | | - module_put(THIS_MODULE); |
|---|
| 262 | 205 | } |
|---|
| 263 | 206 | |
|---|
| 264 | 207 | static const struct rpc_xprt_ops xprt_rdma_bc_procs = { |
|---|
| .. | .. |
|---|
| 270 | 213 | .buf_alloc = xprt_rdma_bc_allocate, |
|---|
| 271 | 214 | .buf_free = xprt_rdma_bc_free, |
|---|
| 272 | 215 | .send_request = xprt_rdma_bc_send_request, |
|---|
| 273 | | - .set_retrans_timeout = xprt_set_retrans_timeout_def, |
|---|
| 216 | + .wait_for_reply_request = xprt_wait_for_reply_request_def, |
|---|
| 274 | 217 | .close = xprt_rdma_bc_close, |
|---|
| 275 | 218 | .destroy = xprt_rdma_bc_put, |
|---|
| 276 | 219 | .print_stats = xprt_rdma_print_stats |
|---|
| .. | .. |
|---|
| 291 | 234 | struct rpc_xprt *xprt; |
|---|
| 292 | 235 | struct rpcrdma_xprt *new_xprt; |
|---|
| 293 | 236 | |
|---|
| 294 | | - if (args->addrlen > sizeof(xprt->addr)) { |
|---|
| 295 | | - dprintk("RPC: %s: address too large\n", __func__); |
|---|
| 237 | + if (args->addrlen > sizeof(xprt->addr)) |
|---|
| 296 | 238 | return ERR_PTR(-EBADF); |
|---|
| 297 | | - } |
|---|
| 298 | 239 | |
|---|
| 299 | 240 | xprt = xprt_alloc(args->net, sizeof(*new_xprt), |
|---|
| 300 | 241 | RPCRDMA_MAX_BC_REQUESTS, |
|---|
| 301 | 242 | RPCRDMA_MAX_BC_REQUESTS); |
|---|
| 302 | | - if (!xprt) { |
|---|
| 303 | | - dprintk("RPC: %s: couldn't allocate rpc_xprt\n", |
|---|
| 304 | | - __func__); |
|---|
| 243 | + if (!xprt) |
|---|
| 305 | 244 | return ERR_PTR(-ENOMEM); |
|---|
| 306 | | - } |
|---|
| 307 | 245 | |
|---|
| 308 | 246 | xprt->timeout = &xprt_rdma_bc_timeout; |
|---|
| 309 | 247 | xprt_set_bound(xprt); |
|---|
| .. | .. |
|---|
| 313 | 251 | xprt->idle_timeout = 0; |
|---|
| 314 | 252 | |
|---|
| 315 | 253 | xprt->prot = XPRT_TRANSPORT_BC_RDMA; |
|---|
| 316 | | - xprt->tsh_size = 0; |
|---|
| 317 | 254 | xprt->ops = &xprt_rdma_bc_procs; |
|---|
| 318 | 255 | |
|---|
| 319 | 256 | memcpy(&xprt->addr, args->dstaddr, args->addrlen); |
|---|
| .. | .. |
|---|
| 330 | 267 | args->bc_xprt->xpt_bc_xprt = xprt; |
|---|
| 331 | 268 | xprt->bc_xprt = args->bc_xprt; |
|---|
| 332 | 269 | |
|---|
| 333 | | - if (!try_module_get(THIS_MODULE)) |
|---|
| 334 | | - goto out_fail; |
|---|
| 335 | | - |
|---|
| 336 | 270 | /* Final put for backchannel xprt is in __svc_rdma_free */ |
|---|
| 337 | 271 | xprt_get(xprt); |
|---|
| 338 | 272 | return xprt; |
|---|
| 339 | | - |
|---|
| 340 | | -out_fail: |
|---|
| 341 | | - xprt_rdma_free_addresses(xprt); |
|---|
| 342 | | - args->bc_xprt->xpt_bc_xprt = NULL; |
|---|
| 343 | | - args->bc_xprt->xpt_bc_xps = NULL; |
|---|
| 344 | | - xprt_put(xprt); |
|---|
| 345 | | - xprt_free(xprt); |
|---|
| 346 | | - return ERR_PTR(-EINVAL); |
|---|
| 347 | 273 | } |
|---|
| 348 | 274 | |
|---|
| 349 | 275 | struct xprt_class xprt_rdma_bc = { |
|---|