.. | .. |
---|
5 | 5 | * Support for backward direction RPCs on RPC/RDMA (server-side). |
---|
6 | 6 | */ |
---|
7 | 7 | |
---|
8 | | -#include <linux/module.h> |
---|
9 | | - |
---|
10 | 8 | #include <linux/sunrpc/svc_rdma.h> |
---|
11 | 9 | |
---|
12 | 10 | #include "xprt_rdma.h" |
---|
13 | 11 | #include <trace/events/rpcrdma.h> |
---|
14 | 12 | |
---|
15 | | -#define RPCDBG_FACILITY RPCDBG_SVCXPRT |
---|
16 | | - |
---|
17 | | -#undef SVCRDMA_BACKCHANNEL_DEBUG |
---|
18 | | - |
---|
19 | 13 | /** |
---|
20 | | - * svc_rdma_handle_bc_reply - Process incoming backchannel reply |
---|
21 | | - * @xprt: controlling backchannel transport |
---|
22 | | - * @rdma_resp: pointer to incoming transport header |
---|
23 | | - * @rcvbuf: XDR buffer into which to decode the reply |
---|
| 14 | + * svc_rdma_handle_bc_reply - Process incoming backchannel Reply |
---|
| 15 | + * @rqstp: resources for handling the Reply |
---|
| 16 | + * @rctxt: Received message |
---|
24 | 17 | * |
---|
25 | | - * Returns: |
---|
26 | | - * %0 if @rcvbuf is filled in, xprt_complete_rqst called, |
---|
27 | | - * %-EAGAIN if server should call ->recvfrom again. |
---|
28 | 18 | */ |
---|
29 | | -int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, __be32 *rdma_resp, |
---|
30 | | - struct xdr_buf *rcvbuf) |
---|
| 19 | +void svc_rdma_handle_bc_reply(struct svc_rqst *rqstp, |
---|
| 20 | + struct svc_rdma_recv_ctxt *rctxt) |
---|
31 | 21 | { |
---|
| 22 | + struct svc_xprt *sxprt = rqstp->rq_xprt; |
---|
| 23 | + struct rpc_xprt *xprt = sxprt->xpt_bc_xprt; |
---|
32 | 24 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); |
---|
| 25 | + struct xdr_buf *rcvbuf = &rqstp->rq_arg; |
---|
33 | 26 | struct kvec *dst, *src = &rcvbuf->head[0]; |
---|
| 27 | + __be32 *rdma_resp = rctxt->rc_recv_buf; |
---|
34 | 28 | struct rpc_rqst *req; |
---|
35 | | - unsigned long cwnd; |
---|
36 | 29 | u32 credits; |
---|
37 | | - size_t len; |
---|
38 | | - __be32 xid; |
---|
39 | | - __be32 *p; |
---|
40 | | - int ret; |
---|
41 | 30 | |
---|
42 | | - p = (__be32 *)src->iov_base; |
---|
43 | | - len = src->iov_len; |
---|
44 | | - xid = *rdma_resp; |
---|
45 | | - |
---|
46 | | -#ifdef SVCRDMA_BACKCHANNEL_DEBUG |
---|
47 | | - pr_info("%s: xid=%08x, length=%zu\n", |
---|
48 | | - __func__, be32_to_cpu(xid), len); |
---|
49 | | - pr_info("%s: RPC/RDMA: %*ph\n", |
---|
50 | | - __func__, (int)RPCRDMA_HDRLEN_MIN, rdma_resp); |
---|
51 | | - pr_info("%s: RPC: %*ph\n", |
---|
52 | | - __func__, (int)len, p); |
---|
53 | | -#endif |
---|
54 | | - |
---|
55 | | - ret = -EAGAIN; |
---|
56 | | - if (src->iov_len < 24) |
---|
57 | | - goto out_shortreply; |
---|
58 | | - |
---|
59 | | - spin_lock(&xprt->recv_lock); |
---|
60 | | - req = xprt_lookup_rqst(xprt, xid); |
---|
| 31 | + spin_lock(&xprt->queue_lock); |
---|
| 32 | + req = xprt_lookup_rqst(xprt, *rdma_resp); |
---|
61 | 33 | if (!req) |
---|
62 | | - goto out_notfound; |
---|
| 34 | + goto out_unlock; |
---|
63 | 35 | |
---|
64 | 36 | dst = &req->rq_private_buf.head[0]; |
---|
65 | 37 | memcpy(&req->rq_private_buf, &req->rq_rcv_buf, sizeof(struct xdr_buf)); |
---|
66 | | - if (dst->iov_len < len) |
---|
| 38 | + if (dst->iov_len < src->iov_len) |
---|
67 | 39 | goto out_unlock; |
---|
68 | | - memcpy(dst->iov_base, p, len); |
---|
| 40 | + memcpy(dst->iov_base, src->iov_base, src->iov_len); |
---|
| 41 | + xprt_pin_rqst(req); |
---|
| 42 | + spin_unlock(&xprt->queue_lock); |
---|
69 | 43 | |
---|
70 | 44 | credits = be32_to_cpup(rdma_resp + 2); |
---|
71 | 45 | if (credits == 0) |
---|
72 | 46 | credits = 1; /* don't deadlock */ |
---|
73 | 47 | else if (credits > r_xprt->rx_buf.rb_bc_max_requests) |
---|
74 | 48 | credits = r_xprt->rx_buf.rb_bc_max_requests; |
---|
75 | | - |
---|
76 | | - spin_lock_bh(&xprt->transport_lock); |
---|
77 | | - cwnd = xprt->cwnd; |
---|
| 49 | + spin_lock(&xprt->transport_lock); |
---|
78 | 50 | xprt->cwnd = credits << RPC_CWNDSHIFT; |
---|
79 | | - if (xprt->cwnd > cwnd) |
---|
80 | | - xprt_release_rqst_cong(req->rq_task); |
---|
81 | | - spin_unlock_bh(&xprt->transport_lock); |
---|
| 51 | + spin_unlock(&xprt->transport_lock); |
---|
82 | 52 | |
---|
83 | | - |
---|
84 | | - ret = 0; |
---|
| 53 | + spin_lock(&xprt->queue_lock); |
---|
85 | 54 | xprt_complete_rqst(req->rq_task, rcvbuf->len); |
---|
| 55 | + xprt_unpin_rqst(req); |
---|
86 | 56 | rcvbuf->len = 0; |
---|
87 | 57 | |
---|
88 | 58 | out_unlock: |
---|
89 | | - spin_unlock(&xprt->recv_lock); |
---|
90 | | -out: |
---|
91 | | - return ret; |
---|
92 | | - |
---|
93 | | -out_shortreply: |
---|
94 | | - dprintk("svcrdma: short bc reply: xprt=%p, len=%zu\n", |
---|
95 | | - xprt, src->iov_len); |
---|
96 | | - goto out; |
---|
97 | | - |
---|
98 | | -out_notfound: |
---|
99 | | - dprintk("svcrdma: unrecognized bc reply: xprt=%p, xid=%08x\n", |
---|
100 | | - xprt, be32_to_cpu(xid)); |
---|
101 | | - goto out_unlock; |
---|
| 59 | + spin_unlock(&xprt->queue_lock); |
---|
102 | 60 | } |
---|
103 | 61 | |
---|
104 | 62 | /* Send a backwards direction RPC call. |
---|
.. | .. |
---|
120 | 78 | { |
---|
121 | 79 | int ret; |
---|
122 | 80 | |
---|
123 | | - ret = svc_rdma_map_reply_msg(rdma, ctxt, &rqst->rq_snd_buf, NULL); |
---|
| 81 | + ret = svc_rdma_map_reply_msg(rdma, ctxt, NULL, &rqst->rq_snd_buf); |
---|
124 | 82 | if (ret < 0) |
---|
125 | 83 | return -EIO; |
---|
126 | 84 | |
---|
.. | .. |
---|
129 | 87 | */ |
---|
130 | 88 | get_page(virt_to_page(rqst->rq_buffer)); |
---|
131 | 89 | ctxt->sc_send_wr.opcode = IB_WR_SEND; |
---|
132 | | - return svc_rdma_send(rdma, &ctxt->sc_send_wr); |
---|
| 90 | + return svc_rdma_send(rdma, ctxt); |
---|
133 | 91 | } |
---|
134 | 92 | |
---|
135 | 93 | /* Server-side transport endpoint wants a whole page for its send |
---|
.. | .. |
---|
184 | 142 | if (!ctxt) |
---|
185 | 143 | goto drop_connection; |
---|
186 | 144 | |
---|
187 | | - p = ctxt->sc_xprt_buf; |
---|
| 145 | + p = xdr_reserve_space(&ctxt->sc_stream, RPCRDMA_HDRLEN_MIN); |
---|
| 146 | + if (!p) |
---|
| 147 | + goto put_ctxt; |
---|
188 | 148 | *p++ = rqst->rq_xid; |
---|
189 | 149 | *p++ = rpcrdma_version; |
---|
190 | 150 | *p++ = cpu_to_be32(r_xprt->rx_buf.rb_bc_max_requests); |
---|
.. | .. |
---|
192 | 152 | *p++ = xdr_zero; |
---|
193 | 153 | *p++ = xdr_zero; |
---|
194 | 154 | *p = xdr_zero; |
---|
195 | | - svc_rdma_sync_reply_hdr(rdma, ctxt, RPCRDMA_HDRLEN_MIN); |
---|
196 | 155 | |
---|
197 | | -#ifdef SVCRDMA_BACKCHANNEL_DEBUG |
---|
198 | | - pr_info("%s: %*ph\n", __func__, 64, rqst->rq_buffer); |
---|
199 | | -#endif |
---|
200 | | - |
---|
| 156 | + rqst->rq_xtime = ktime_get(); |
---|
201 | 157 | rc = svc_rdma_bc_sendto(rdma, rqst, ctxt); |
---|
202 | | - if (rc) { |
---|
203 | | - svc_rdma_send_ctxt_put(rdma, ctxt); |
---|
204 | | - goto drop_connection; |
---|
205 | | - } |
---|
206 | | - return rc; |
---|
| 158 | + if (rc) |
---|
| 159 | + goto put_ctxt; |
---|
| 160 | + return 0; |
---|
| 161 | + |
---|
| 162 | +put_ctxt: |
---|
| 163 | + svc_rdma_send_ctxt_put(rdma, ctxt); |
---|
207 | 164 | |
---|
208 | 165 | drop_connection: |
---|
209 | | - dprintk("svcrdma: failed to send bc call\n"); |
---|
210 | | - xprt_disconnect_done(xprt); |
---|
211 | 166 | return -ENOTCONN; |
---|
212 | 167 | } |
---|
213 | 168 | |
---|
214 | | -/* Send an RPC call on the passive end of a transport |
---|
215 | | - * connection. |
---|
| 169 | +/** |
---|
| 170 | + * xprt_rdma_bc_send_request - Send a reverse-direction Call |
---|
| 171 | + * @rqst: rpc_rqst containing Call message to be sent |
---|
| 172 | + * |
---|
| 173 | + * Return values: |
---|
| 174 | + * %0 if the message was sent successfully |
---|
| 175 | + * %ENOTCONN if the message was not sent |
---|
216 | 176 | */ |
---|
217 | | -static int |
---|
218 | | -xprt_rdma_bc_send_request(struct rpc_task *task) |
---|
| 177 | +static int xprt_rdma_bc_send_request(struct rpc_rqst *rqst) |
---|
219 | 178 | { |
---|
220 | | - struct rpc_rqst *rqst = task->tk_rqstp; |
---|
221 | 179 | struct svc_xprt *sxprt = rqst->rq_xprt->bc_xprt; |
---|
222 | | - struct svcxprt_rdma *rdma; |
---|
| 180 | + struct svcxprt_rdma *rdma = |
---|
| 181 | + container_of(sxprt, struct svcxprt_rdma, sc_xprt); |
---|
223 | 182 | int ret; |
---|
224 | 183 | |
---|
225 | | - dprintk("svcrdma: sending bc call with xid: %08x\n", |
---|
226 | | - be32_to_cpu(rqst->rq_xid)); |
---|
| 184 | + if (test_bit(XPT_DEAD, &sxprt->xpt_flags)) |
---|
| 185 | + return -ENOTCONN; |
---|
227 | 186 | |
---|
228 | | - if (!mutex_trylock(&sxprt->xpt_mutex)) { |
---|
229 | | - rpc_sleep_on(&sxprt->xpt_bc_pending, task, NULL); |
---|
230 | | - if (!mutex_trylock(&sxprt->xpt_mutex)) |
---|
231 | | - return -EAGAIN; |
---|
232 | | - rpc_wake_up_queued_task(&sxprt->xpt_bc_pending, task); |
---|
233 | | - } |
---|
234 | | - |
---|
235 | | - ret = -ENOTCONN; |
---|
236 | | - rdma = container_of(sxprt, struct svcxprt_rdma, sc_xprt); |
---|
237 | | - if (!test_bit(XPT_DEAD, &sxprt->xpt_flags)) |
---|
238 | | - ret = rpcrdma_bc_send_request(rdma, rqst); |
---|
239 | | - |
---|
240 | | - mutex_unlock(&sxprt->xpt_mutex); |
---|
241 | | - |
---|
242 | | - if (ret < 0) |
---|
243 | | - return ret; |
---|
244 | | - return 0; |
---|
| 187 | + ret = rpcrdma_bc_send_request(rdma, rqst); |
---|
| 188 | + if (ret == -ENOTCONN) |
---|
| 189 | + svc_close_xprt(sxprt); |
---|
| 190 | + return ret; |
---|
245 | 191 | } |
---|
246 | 192 | |
---|
247 | 193 | static void |
---|
248 | 194 | xprt_rdma_bc_close(struct rpc_xprt *xprt) |
---|
249 | 195 | { |
---|
250 | | - dprintk("svcrdma: %s: xprt %p\n", __func__, xprt); |
---|
| 196 | + xprt_disconnect_done(xprt); |
---|
251 | 197 | xprt->cwnd = RPC_CWNDSHIFT; |
---|
252 | 198 | } |
---|
253 | 199 | |
---|
254 | 200 | static void |
---|
255 | 201 | xprt_rdma_bc_put(struct rpc_xprt *xprt) |
---|
256 | 202 | { |
---|
257 | | - dprintk("svcrdma: %s: xprt %p\n", __func__, xprt); |
---|
258 | | - |
---|
259 | 203 | xprt_rdma_free_addresses(xprt); |
---|
260 | 204 | xprt_free(xprt); |
---|
261 | | - module_put(THIS_MODULE); |
---|
262 | 205 | } |
---|
263 | 206 | |
---|
264 | 207 | static const struct rpc_xprt_ops xprt_rdma_bc_procs = { |
---|
.. | .. |
---|
270 | 213 | .buf_alloc = xprt_rdma_bc_allocate, |
---|
271 | 214 | .buf_free = xprt_rdma_bc_free, |
---|
272 | 215 | .send_request = xprt_rdma_bc_send_request, |
---|
273 | | - .set_retrans_timeout = xprt_set_retrans_timeout_def, |
---|
| 216 | + .wait_for_reply_request = xprt_wait_for_reply_request_def, |
---|
274 | 217 | .close = xprt_rdma_bc_close, |
---|
275 | 218 | .destroy = xprt_rdma_bc_put, |
---|
276 | 219 | .print_stats = xprt_rdma_print_stats |
---|
.. | .. |
---|
291 | 234 | struct rpc_xprt *xprt; |
---|
292 | 235 | struct rpcrdma_xprt *new_xprt; |
---|
293 | 236 | |
---|
294 | | - if (args->addrlen > sizeof(xprt->addr)) { |
---|
295 | | - dprintk("RPC: %s: address too large\n", __func__); |
---|
| 237 | + if (args->addrlen > sizeof(xprt->addr)) |
---|
296 | 238 | return ERR_PTR(-EBADF); |
---|
297 | | - } |
---|
298 | 239 | |
---|
299 | 240 | xprt = xprt_alloc(args->net, sizeof(*new_xprt), |
---|
300 | 241 | RPCRDMA_MAX_BC_REQUESTS, |
---|
301 | 242 | RPCRDMA_MAX_BC_REQUESTS); |
---|
302 | | - if (!xprt) { |
---|
303 | | - dprintk("RPC: %s: couldn't allocate rpc_xprt\n", |
---|
304 | | - __func__); |
---|
| 243 | + if (!xprt) |
---|
305 | 244 | return ERR_PTR(-ENOMEM); |
---|
306 | | - } |
---|
307 | 245 | |
---|
308 | 246 | xprt->timeout = &xprt_rdma_bc_timeout; |
---|
309 | 247 | xprt_set_bound(xprt); |
---|
.. | .. |
---|
313 | 251 | xprt->idle_timeout = 0; |
---|
314 | 252 | |
---|
315 | 253 | xprt->prot = XPRT_TRANSPORT_BC_RDMA; |
---|
316 | | - xprt->tsh_size = 0; |
---|
317 | 254 | xprt->ops = &xprt_rdma_bc_procs; |
---|
318 | 255 | |
---|
319 | 256 | memcpy(&xprt->addr, args->dstaddr, args->addrlen); |
---|
.. | .. |
---|
330 | 267 | args->bc_xprt->xpt_bc_xprt = xprt; |
---|
331 | 268 | xprt->bc_xprt = args->bc_xprt; |
---|
332 | 269 | |
---|
333 | | - if (!try_module_get(THIS_MODULE)) |
---|
334 | | - goto out_fail; |
---|
335 | | - |
---|
336 | 270 | /* Final put for backchannel xprt is in __svc_rdma_free */ |
---|
337 | 271 | xprt_get(xprt); |
---|
338 | 272 | return xprt; |
---|
339 | | - |
---|
340 | | -out_fail: |
---|
341 | | - xprt_rdma_free_addresses(xprt); |
---|
342 | | - args->bc_xprt->xpt_bc_xprt = NULL; |
---|
343 | | - args->bc_xprt->xpt_bc_xps = NULL; |
---|
344 | | - xprt_put(xprt); |
---|
345 | | - xprt_free(xprt); |
---|
346 | | - return ERR_PTR(-EINVAL); |
---|
347 | 273 | } |
---|
348 | 274 | |
---|
349 | 275 | struct xprt_class xprt_rdma_bc = { |
---|