| .. | .. |
|---|
| 1 | 1 | /* |
|---|
| 2 | + * Copyright (c) 2012 - 2019 Intel Corporation. All rights reserved. |
|---|
| 2 | 3 | * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved. |
|---|
| 3 | 4 | * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. |
|---|
| 4 | 5 | * |
|---|
| .. | .. |
|---|
| 63 | 64 | enum ib_qp_type sqptype, dqptype; |
|---|
| 64 | 65 | |
|---|
| 65 | 66 | rcu_read_lock(); |
|---|
| 66 | | - qp = rvt_lookup_qpn(rdi, &ibp->rvp, swqe->ud_wr.remote_qpn); |
|---|
| 67 | + qp = rvt_lookup_qpn(rdi, &ibp->rvp, rvt_get_swqe_remote_qpn(swqe)); |
|---|
| 67 | 68 | if (!qp) { |
|---|
| 68 | 69 | ibp->rvp.n_pkt_drops++; |
|---|
| 69 | 70 | goto drop; |
|---|
| .. | .. |
|---|
| 80 | 81 | goto drop; |
|---|
| 81 | 82 | } |
|---|
| 82 | 83 | |
|---|
| 83 | | - ah_attr = &ibah_to_rvtah(swqe->ud_wr.ah)->attr; |
|---|
| 84 | + ah_attr = rvt_get_swqe_ah_attr(swqe); |
|---|
| 84 | 85 | ppd = ppd_from_ibp(ibp); |
|---|
| 85 | 86 | |
|---|
| 86 | 87 | if (qp->ibqp.qp_num > 1) { |
|---|
| .. | .. |
|---|
| 110 | 111 | if (qp->ibqp.qp_num) { |
|---|
| 111 | 112 | u32 qkey; |
|---|
| 112 | 113 | |
|---|
| 113 | | - qkey = (int)swqe->ud_wr.remote_qkey < 0 ? |
|---|
| 114 | | - sqp->qkey : swqe->ud_wr.remote_qkey; |
|---|
| 114 | + qkey = (int)rvt_get_swqe_remote_qkey(swqe) < 0 ? |
|---|
| 115 | + sqp->qkey : rvt_get_swqe_remote_qkey(swqe); |
|---|
| 115 | 116 | if (unlikely(qkey != qp->qkey)) |
|---|
| 116 | 117 | goto drop; |
|---|
| 117 | 118 | } |
|---|
| .. | .. |
|---|
| 162 | 163 | const struct ib_global_route *grd = rdma_ah_read_grh(ah_attr); |
|---|
| 163 | 164 | |
|---|
| 164 | 165 | qib_make_grh(ibp, &grh, grd, 0, 0); |
|---|
| 165 | | - qib_copy_sge(&qp->r_sge, &grh, |
|---|
| 166 | | - sizeof(grh), 1); |
|---|
| 166 | + rvt_copy_sge(qp, &qp->r_sge, &grh, |
|---|
| 167 | + sizeof(grh), true, false); |
|---|
| 167 | 168 | wc.wc_flags |= IB_WC_GRH; |
|---|
| 168 | 169 | } else |
|---|
| 169 | 170 | rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); |
|---|
| .. | .. |
|---|
| 172 | 173 | ssge.num_sge = swqe->wr.num_sge; |
|---|
| 173 | 174 | sge = &ssge.sge; |
|---|
| 174 | 175 | while (length) { |
|---|
| 175 | | - u32 len = sge->length; |
|---|
| 176 | + u32 len = rvt_get_sge_length(sge, length); |
|---|
| 176 | 177 | |
|---|
| 177 | | - if (len > length) |
|---|
| 178 | | - len = length; |
|---|
| 179 | | - if (len > sge->sge_length) |
|---|
| 180 | | - len = sge->sge_length; |
|---|
| 181 | | - BUG_ON(len == 0); |
|---|
| 182 | | - qib_copy_sge(&qp->r_sge, sge->vaddr, len, 1); |
|---|
| 178 | + rvt_copy_sge(qp, &qp->r_sge, sge->vaddr, len, true, false); |
|---|
| 183 | 179 | sge->vaddr += len; |
|---|
| 184 | 180 | sge->length -= len; |
|---|
| 185 | 181 | sge->sge_length -= len; |
|---|
| .. | .. |
|---|
| 208 | 204 | wc.qp = &qp->ibqp; |
|---|
| 209 | 205 | wc.src_qp = sqp->ibqp.qp_num; |
|---|
| 210 | 206 | wc.pkey_index = qp->ibqp.qp_type == IB_QPT_GSI ? |
|---|
| 211 | | - swqe->ud_wr.pkey_index : 0; |
|---|
| 207 | + rvt_get_swqe_pkey_index(swqe) : 0; |
|---|
| 212 | 208 | wc.slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) & |
|---|
| 213 | 209 | ((1 << ppd->lmc) - 1)); |
|---|
| 214 | 210 | wc.sl = rdma_ah_get_sl(ah_attr); |
|---|
| 215 | 211 | wc.dlid_path_bits = rdma_ah_get_dlid(ah_attr) & ((1 << ppd->lmc) - 1); |
|---|
| 216 | 212 | wc.port_num = qp->port_num; |
|---|
| 217 | 213 | /* Signal completion event if the solicited bit is set. */ |
|---|
| 218 | | - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, |
|---|
| 219 | | - swqe->wr.send_flags & IB_SEND_SOLICITED); |
|---|
| 214 | + rvt_recv_cq(qp, &wc, swqe->wr.send_flags & IB_SEND_SOLICITED); |
|---|
| 220 | 215 | ibp->rvp.n_loop_pkts++; |
|---|
| 221 | 216 | bail_unlock: |
|---|
| 222 | 217 | spin_unlock_irqrestore(&qp->r_lock, flags); |
|---|
| .. | .. |
|---|
| 260 | 255 | goto bail; |
|---|
| 261 | 256 | } |
|---|
| 262 | 257 | wqe = rvt_get_swqe_ptr(qp, qp->s_last); |
|---|
| 263 | | - qib_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); |
|---|
| 258 | + rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); |
|---|
| 264 | 259 | goto done; |
|---|
| 265 | 260 | } |
|---|
| 266 | 261 | |
|---|
| .. | .. |
|---|
| 276 | 271 | /* Construct the header. */ |
|---|
| 277 | 272 | ibp = to_iport(qp->ibqp.device, qp->port_num); |
|---|
| 278 | 273 | ppd = ppd_from_ibp(ibp); |
|---|
| 279 | | - ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; |
|---|
| 274 | + ah_attr = rvt_get_swqe_ah_attr(wqe); |
|---|
| 280 | 275 | if (rdma_ah_get_dlid(ah_attr) >= be16_to_cpu(IB_MULTICAST_LID_BASE)) { |
|---|
| 281 | 276 | if (rdma_ah_get_dlid(ah_attr) != |
|---|
| 282 | 277 | be16_to_cpu(IB_LID_PERMISSIVE)) |
|---|
| .. | .. |
|---|
| 304 | 299 | qib_ud_loopback(qp, wqe); |
|---|
| 305 | 300 | spin_lock_irqsave(&qp->s_lock, tflags); |
|---|
| 306 | 301 | *flags = tflags; |
|---|
| 307 | | - qib_send_complete(qp, wqe, IB_WC_SUCCESS); |
|---|
| 302 | + rvt_send_complete(qp, wqe, IB_WC_SUCCESS); |
|---|
| 308 | 303 | goto done; |
|---|
| 309 | 304 | } |
|---|
| 310 | 305 | } |
|---|
| .. | .. |
|---|
| 368 | 363 | bth0 |= extra_bytes << 20; |
|---|
| 369 | 364 | bth0 |= qp->ibqp.qp_type == IB_QPT_SMI ? QIB_DEFAULT_P_KEY : |
|---|
| 370 | 365 | qib_get_pkey(ibp, qp->ibqp.qp_type == IB_QPT_GSI ? |
|---|
| 371 | | - wqe->ud_wr.pkey_index : qp->s_pkey_index); |
|---|
| 366 | + rvt_get_swqe_pkey_index(wqe) : qp->s_pkey_index); |
|---|
| 372 | 367 | ohdr->bth[0] = cpu_to_be32(bth0); |
|---|
| 373 | 368 | /* |
|---|
| 374 | 369 | * Use the multicast QP if the destination LID is a multicast LID. |
|---|
| .. | .. |
|---|
| 377 | 372 | be16_to_cpu(IB_MULTICAST_LID_BASE) && |
|---|
| 378 | 373 | rdma_ah_get_dlid(ah_attr) != be16_to_cpu(IB_LID_PERMISSIVE) ? |
|---|
| 379 | 374 | cpu_to_be32(QIB_MULTICAST_QPN) : |
|---|
| 380 | | - cpu_to_be32(wqe->ud_wr.remote_qpn); |
|---|
| 375 | + cpu_to_be32(rvt_get_swqe_remote_qpn(wqe)); |
|---|
| 381 | 376 | ohdr->bth[2] = cpu_to_be32(wqe->psn & QIB_PSN_MASK); |
|---|
| 382 | 377 | /* |
|---|
| 383 | 378 | * Qkeys with the high order bit set mean use the |
|---|
| 384 | 379 | * qkey from the QP context instead of the WR (see 10.2.5). |
|---|
| 385 | 380 | */ |
|---|
| 386 | | - ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.remote_qkey < 0 ? |
|---|
| 387 | | - qp->qkey : wqe->ud_wr.remote_qkey); |
|---|
| 381 | + ohdr->u.ud.deth[0] = |
|---|
| 382 | + cpu_to_be32((int)rvt_get_swqe_remote_qkey(wqe) < 0 ? qp->qkey : |
|---|
| 383 | + rvt_get_swqe_remote_qkey(wqe)); |
|---|
| 388 | 384 | ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num); |
|---|
| 389 | 385 | |
|---|
| 390 | 386 | done: |
|---|
| .. | .. |
|---|
| 550 | 546 | goto drop; |
|---|
| 551 | 547 | } |
|---|
| 552 | 548 | if (has_grh) { |
|---|
| 553 | | - qib_copy_sge(&qp->r_sge, &hdr->u.l.grh, |
|---|
| 554 | | - sizeof(struct ib_grh), 1); |
|---|
| 549 | + rvt_copy_sge(qp, &qp->r_sge, &hdr->u.l.grh, |
|---|
| 550 | + sizeof(struct ib_grh), true, false); |
|---|
| 555 | 551 | wc.wc_flags |= IB_WC_GRH; |
|---|
| 556 | 552 | } else |
|---|
| 557 | 553 | rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); |
|---|
| 558 | | - qib_copy_sge(&qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh), 1); |
|---|
| 554 | + rvt_copy_sge(qp, &qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh), |
|---|
| 555 | + true, false); |
|---|
| 559 | 556 | rvt_put_ss(&qp->r_sge); |
|---|
| 560 | 557 | if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) |
|---|
| 561 | 558 | return; |
|---|
| .. | .. |
|---|
| 577 | 574 | dlid & ((1 << ppd_from_ibp(ibp)->lmc) - 1); |
|---|
| 578 | 575 | wc.port_num = qp->port_num; |
|---|
| 579 | 576 | /* Signal completion event if the solicited bit is set. */ |
|---|
| 580 | | - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, |
|---|
| 581 | | - ib_bth_is_solicited(ohdr)); |
|---|
| 577 | + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); |
|---|
| 582 | 578 | return; |
|---|
| 583 | 579 | |
|---|
| 584 | 580 | drop: |
|---|