.. | .. |
---|
1 | 1 | /* |
---|
| 2 | + * Copyright (c) 2012 - 2019 Intel Corporation. All rights reserved. |
---|
2 | 3 | * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved. |
---|
3 | 4 | * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. |
---|
4 | 5 | * |
---|
.. | .. |
---|
63 | 64 | enum ib_qp_type sqptype, dqptype; |
---|
64 | 65 | |
---|
65 | 66 | rcu_read_lock(); |
---|
66 | | - qp = rvt_lookup_qpn(rdi, &ibp->rvp, swqe->ud_wr.remote_qpn); |
---|
| 67 | + qp = rvt_lookup_qpn(rdi, &ibp->rvp, rvt_get_swqe_remote_qpn(swqe)); |
---|
67 | 68 | if (!qp) { |
---|
68 | 69 | ibp->rvp.n_pkt_drops++; |
---|
69 | 70 | goto drop; |
---|
.. | .. |
---|
80 | 81 | goto drop; |
---|
81 | 82 | } |
---|
82 | 83 | |
---|
83 | | - ah_attr = &ibah_to_rvtah(swqe->ud_wr.ah)->attr; |
---|
| 84 | + ah_attr = rvt_get_swqe_ah_attr(swqe); |
---|
84 | 85 | ppd = ppd_from_ibp(ibp); |
---|
85 | 86 | |
---|
86 | 87 | if (qp->ibqp.qp_num > 1) { |
---|
.. | .. |
---|
110 | 111 | if (qp->ibqp.qp_num) { |
---|
111 | 112 | u32 qkey; |
---|
112 | 113 | |
---|
113 | | - qkey = (int)swqe->ud_wr.remote_qkey < 0 ? |
---|
114 | | - sqp->qkey : swqe->ud_wr.remote_qkey; |
---|
| 114 | + qkey = (int)rvt_get_swqe_remote_qkey(swqe) < 0 ? |
---|
| 115 | + sqp->qkey : rvt_get_swqe_remote_qkey(swqe); |
---|
115 | 116 | if (unlikely(qkey != qp->qkey)) |
---|
116 | 117 | goto drop; |
---|
117 | 118 | } |
---|
.. | .. |
---|
162 | 163 | const struct ib_global_route *grd = rdma_ah_read_grh(ah_attr); |
---|
163 | 164 | |
---|
164 | 165 | qib_make_grh(ibp, &grh, grd, 0, 0); |
---|
165 | | - qib_copy_sge(&qp->r_sge, &grh, |
---|
166 | | - sizeof(grh), 1); |
---|
| 166 | + rvt_copy_sge(qp, &qp->r_sge, &grh, |
---|
| 167 | + sizeof(grh), true, false); |
---|
167 | 168 | wc.wc_flags |= IB_WC_GRH; |
---|
168 | 169 | } else |
---|
169 | 170 | rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); |
---|
.. | .. |
---|
172 | 173 | ssge.num_sge = swqe->wr.num_sge; |
---|
173 | 174 | sge = &ssge.sge; |
---|
174 | 175 | while (length) { |
---|
175 | | - u32 len = sge->length; |
---|
| 176 | + u32 len = rvt_get_sge_length(sge, length); |
---|
176 | 177 | |
---|
177 | | - if (len > length) |
---|
178 | | - len = length; |
---|
179 | | - if (len > sge->sge_length) |
---|
180 | | - len = sge->sge_length; |
---|
181 | | - BUG_ON(len == 0); |
---|
182 | | - qib_copy_sge(&qp->r_sge, sge->vaddr, len, 1); |
---|
| 178 | + rvt_copy_sge(qp, &qp->r_sge, sge->vaddr, len, true, false); |
---|
183 | 179 | sge->vaddr += len; |
---|
184 | 180 | sge->length -= len; |
---|
185 | 181 | sge->sge_length -= len; |
---|
.. | .. |
---|
208 | 204 | wc.qp = &qp->ibqp; |
---|
209 | 205 | wc.src_qp = sqp->ibqp.qp_num; |
---|
210 | 206 | wc.pkey_index = qp->ibqp.qp_type == IB_QPT_GSI ? |
---|
211 | | - swqe->ud_wr.pkey_index : 0; |
---|
| 207 | + rvt_get_swqe_pkey_index(swqe) : 0; |
---|
212 | 208 | wc.slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) & |
---|
213 | 209 | ((1 << ppd->lmc) - 1)); |
---|
214 | 210 | wc.sl = rdma_ah_get_sl(ah_attr); |
---|
215 | 211 | wc.dlid_path_bits = rdma_ah_get_dlid(ah_attr) & ((1 << ppd->lmc) - 1); |
---|
216 | 212 | wc.port_num = qp->port_num; |
---|
217 | 213 | /* Signal completion event if the solicited bit is set. */ |
---|
218 | | - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, |
---|
219 | | - swqe->wr.send_flags & IB_SEND_SOLICITED); |
---|
| 214 | + rvt_recv_cq(qp, &wc, swqe->wr.send_flags & IB_SEND_SOLICITED); |
---|
220 | 215 | ibp->rvp.n_loop_pkts++; |
---|
221 | 216 | bail_unlock: |
---|
222 | 217 | spin_unlock_irqrestore(&qp->r_lock, flags); |
---|
.. | .. |
---|
260 | 255 | goto bail; |
---|
261 | 256 | } |
---|
262 | 257 | wqe = rvt_get_swqe_ptr(qp, qp->s_last); |
---|
263 | | - qib_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); |
---|
| 258 | + rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); |
---|
264 | 259 | goto done; |
---|
265 | 260 | } |
---|
266 | 261 | |
---|
.. | .. |
---|
276 | 271 | /* Construct the header. */ |
---|
277 | 272 | ibp = to_iport(qp->ibqp.device, qp->port_num); |
---|
278 | 273 | ppd = ppd_from_ibp(ibp); |
---|
279 | | - ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; |
---|
| 274 | + ah_attr = rvt_get_swqe_ah_attr(wqe); |
---|
280 | 275 | if (rdma_ah_get_dlid(ah_attr) >= be16_to_cpu(IB_MULTICAST_LID_BASE)) { |
---|
281 | 276 | if (rdma_ah_get_dlid(ah_attr) != |
---|
282 | 277 | be16_to_cpu(IB_LID_PERMISSIVE)) |
---|
.. | .. |
---|
304 | 299 | qib_ud_loopback(qp, wqe); |
---|
305 | 300 | spin_lock_irqsave(&qp->s_lock, tflags); |
---|
306 | 301 | *flags = tflags; |
---|
307 | | - qib_send_complete(qp, wqe, IB_WC_SUCCESS); |
---|
| 302 | + rvt_send_complete(qp, wqe, IB_WC_SUCCESS); |
---|
308 | 303 | goto done; |
---|
309 | 304 | } |
---|
310 | 305 | } |
---|
.. | .. |
---|
368 | 363 | bth0 |= extra_bytes << 20; |
---|
369 | 364 | bth0 |= qp->ibqp.qp_type == IB_QPT_SMI ? QIB_DEFAULT_P_KEY : |
---|
370 | 365 | qib_get_pkey(ibp, qp->ibqp.qp_type == IB_QPT_GSI ? |
---|
371 | | - wqe->ud_wr.pkey_index : qp->s_pkey_index); |
---|
| 366 | + rvt_get_swqe_pkey_index(wqe) : qp->s_pkey_index); |
---|
372 | 367 | ohdr->bth[0] = cpu_to_be32(bth0); |
---|
373 | 368 | /* |
---|
374 | 369 | * Use the multicast QP if the destination LID is a multicast LID. |
---|
.. | .. |
---|
377 | 372 | be16_to_cpu(IB_MULTICAST_LID_BASE) && |
---|
378 | 373 | rdma_ah_get_dlid(ah_attr) != be16_to_cpu(IB_LID_PERMISSIVE) ? |
---|
379 | 374 | cpu_to_be32(QIB_MULTICAST_QPN) : |
---|
380 | | - cpu_to_be32(wqe->ud_wr.remote_qpn); |
---|
| 375 | + cpu_to_be32(rvt_get_swqe_remote_qpn(wqe)); |
---|
381 | 376 | ohdr->bth[2] = cpu_to_be32(wqe->psn & QIB_PSN_MASK); |
---|
382 | 377 | /* |
---|
383 | 378 | * Qkeys with the high order bit set mean use the |
---|
384 | 379 | * qkey from the QP context instead of the WR (see 10.2.5). |
---|
385 | 380 | */ |
---|
386 | | - ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.remote_qkey < 0 ? |
---|
387 | | - qp->qkey : wqe->ud_wr.remote_qkey); |
---|
| 381 | + ohdr->u.ud.deth[0] = |
---|
| 382 | + cpu_to_be32((int)rvt_get_swqe_remote_qkey(wqe) < 0 ? qp->qkey : |
---|
| 383 | + rvt_get_swqe_remote_qkey(wqe)); |
---|
388 | 384 | ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num); |
---|
389 | 385 | |
---|
390 | 386 | done: |
---|
.. | .. |
---|
550 | 546 | goto drop; |
---|
551 | 547 | } |
---|
552 | 548 | if (has_grh) { |
---|
553 | | - qib_copy_sge(&qp->r_sge, &hdr->u.l.grh, |
---|
554 | | - sizeof(struct ib_grh), 1); |
---|
| 549 | + rvt_copy_sge(qp, &qp->r_sge, &hdr->u.l.grh, |
---|
| 550 | + sizeof(struct ib_grh), true, false); |
---|
555 | 551 | wc.wc_flags |= IB_WC_GRH; |
---|
556 | 552 | } else |
---|
557 | 553 | rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); |
---|
558 | | - qib_copy_sge(&qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh), 1); |
---|
| 554 | + rvt_copy_sge(qp, &qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh), |
---|
| 555 | + true, false); |
---|
559 | 556 | rvt_put_ss(&qp->r_sge); |
---|
560 | 557 | if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) |
---|
561 | 558 | return; |
---|
.. | .. |
---|
577 | 574 | dlid & ((1 << ppd_from_ibp(ibp)->lmc) - 1); |
---|
578 | 575 | wc.port_num = qp->port_num; |
---|
579 | 576 | /* Signal completion event if the solicited bit is set. */ |
---|
580 | | - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, |
---|
581 | | - ib_bth_is_solicited(ohdr)); |
---|
| 577 | + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); |
---|
582 | 578 | return; |
---|
583 | 579 | |
---|
584 | 580 | drop: |
---|