| .. | .. |
|---|
| 1 | 1 | /* |
|---|
| 2 | | - * Copyright(c) 2015 - 2018 Intel Corporation. |
|---|
| 2 | + * Copyright(c) 2015 - 2019 Intel Corporation. |
|---|
| 3 | 3 | * |
|---|
| 4 | 4 | * This file is provided under a dual BSD/GPLv2 license. When using or |
|---|
| 5 | 5 | * redistributing this file, you may do so under either license. |
|---|
| .. | .. |
|---|
| 87 | 87 | rcu_read_lock(); |
|---|
| 88 | 88 | |
|---|
| 89 | 89 | qp = rvt_lookup_qpn(ib_to_rvt(sqp->ibqp.device), &ibp->rvp, |
|---|
| 90 | | - swqe->ud_wr.remote_qpn); |
|---|
| 90 | + rvt_get_swqe_remote_qpn(swqe)); |
|---|
| 91 | 91 | if (!qp) { |
|---|
| 92 | 92 | ibp->rvp.n_pkt_drops++; |
|---|
| 93 | 93 | rcu_read_unlock(); |
|---|
| .. | .. |
|---|
| 105 | 105 | goto drop; |
|---|
| 106 | 106 | } |
|---|
| 107 | 107 | |
|---|
| 108 | | - ah_attr = &ibah_to_rvtah(swqe->ud_wr.ah)->attr; |
|---|
| 108 | + ah_attr = rvt_get_swqe_ah_attr(swqe); |
|---|
| 109 | 109 | ppd = ppd_from_ibp(ibp); |
|---|
| 110 | 110 | |
|---|
| 111 | 111 | if (qp->ibqp.qp_num > 1) { |
|---|
| .. | .. |
|---|
| 135 | 135 | if (qp->ibqp.qp_num) { |
|---|
| 136 | 136 | u32 qkey; |
|---|
| 137 | 137 | |
|---|
| 138 | | - qkey = (int)swqe->ud_wr.remote_qkey < 0 ? |
|---|
| 139 | | - sqp->qkey : swqe->ud_wr.remote_qkey; |
|---|
| 138 | + qkey = (int)rvt_get_swqe_remote_qkey(swqe) < 0 ? |
|---|
| 139 | + sqp->qkey : rvt_get_swqe_remote_qkey(swqe); |
|---|
| 140 | 140 | if (unlikely(qkey != qp->qkey)) |
|---|
| 141 | 141 | goto drop; /* silently drop per IBTA spec */ |
|---|
| 142 | 142 | } |
|---|
| .. | .. |
|---|
| 211 | 211 | } |
|---|
| 212 | 212 | |
|---|
| 213 | 213 | hfi1_make_grh(ibp, &grh, &grd, 0, 0); |
|---|
| 214 | | - hfi1_copy_sge(&qp->r_sge, &grh, |
|---|
| 215 | | - sizeof(grh), true, false); |
|---|
| 214 | + rvt_copy_sge(qp, &qp->r_sge, &grh, |
|---|
| 215 | + sizeof(grh), true, false); |
|---|
| 216 | 216 | wc.wc_flags |= IB_WC_GRH; |
|---|
| 217 | 217 | } else { |
|---|
| 218 | 218 | rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); |
|---|
| .. | .. |
|---|
| 222 | 222 | ssge.num_sge = swqe->wr.num_sge; |
|---|
| 223 | 223 | sge = &ssge.sge; |
|---|
| 224 | 224 | while (length) { |
|---|
| 225 | | - u32 len = sge->length; |
|---|
| 225 | + u32 len = rvt_get_sge_length(sge, length); |
|---|
| 226 | 226 | |
|---|
| 227 | | - if (len > length) |
|---|
| 228 | | - len = length; |
|---|
| 229 | | - if (len > sge->sge_length) |
|---|
| 230 | | - len = sge->sge_length; |
|---|
| 231 | 227 | WARN_ON_ONCE(len == 0); |
|---|
| 232 | | - hfi1_copy_sge(&qp->r_sge, sge->vaddr, len, true, false); |
|---|
| 233 | | - sge->vaddr += len; |
|---|
| 234 | | - sge->length -= len; |
|---|
| 235 | | - sge->sge_length -= len; |
|---|
| 236 | | - if (sge->sge_length == 0) { |
|---|
| 237 | | - if (--ssge.num_sge) |
|---|
| 238 | | - *sge = *ssge.sg_list++; |
|---|
| 239 | | - } else if (sge->length == 0 && sge->mr->lkey) { |
|---|
| 240 | | - if (++sge->n >= RVT_SEGSZ) { |
|---|
| 241 | | - if (++sge->m >= sge->mr->mapsz) |
|---|
| 242 | | - break; |
|---|
| 243 | | - sge->n = 0; |
|---|
| 244 | | - } |
|---|
| 245 | | - sge->vaddr = |
|---|
| 246 | | - sge->mr->map[sge->m]->segs[sge->n].vaddr; |
|---|
| 247 | | - sge->length = |
|---|
| 248 | | - sge->mr->map[sge->m]->segs[sge->n].length; |
|---|
| 249 | | - } |
|---|
| 228 | + rvt_copy_sge(qp, &qp->r_sge, sge->vaddr, len, true, false); |
|---|
| 229 | + rvt_update_sge(&ssge, len, false); |
|---|
| 250 | 230 | length -= len; |
|---|
| 251 | 231 | } |
|---|
| 252 | 232 | rvt_put_ss(&qp->r_sge); |
|---|
| .. | .. |
|---|
| 260 | 240 | if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI) { |
|---|
| 261 | 241 | if (sqp->ibqp.qp_type == IB_QPT_GSI || |
|---|
| 262 | 242 | sqp->ibqp.qp_type == IB_QPT_SMI) |
|---|
| 263 | | - wc.pkey_index = swqe->ud_wr.pkey_index; |
|---|
| 243 | + wc.pkey_index = rvt_get_swqe_pkey_index(swqe); |
|---|
| 264 | 244 | else |
|---|
| 265 | 245 | wc.pkey_index = sqp->s_pkey_index; |
|---|
| 266 | 246 | } else { |
|---|
| .. | .. |
|---|
| 275 | 255 | wc.dlid_path_bits = rdma_ah_get_dlid(ah_attr) & ((1 << ppd->lmc) - 1); |
|---|
| 276 | 256 | wc.port_num = qp->port_num; |
|---|
| 277 | 257 | /* Signal completion event if the solicited bit is set. */ |
|---|
| 278 | | - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, |
|---|
| 279 | | - swqe->wr.send_flags & IB_SEND_SOLICITED); |
|---|
| 258 | + rvt_recv_cq(qp, &wc, swqe->wr.send_flags & IB_SEND_SOLICITED); |
|---|
| 280 | 259 | ibp->rvp.n_loop_pkts++; |
|---|
| 281 | 260 | bail_unlock: |
|---|
| 282 | 261 | spin_unlock_irqrestore(&qp->r_lock, flags); |
|---|
| .. | .. |
|---|
| 303 | 282 | bth0 |= IB_BTH_SOLICITED; |
|---|
| 304 | 283 | bth0 |= extra_bytes << 20; |
|---|
| 305 | 284 | if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI) |
|---|
| 306 | | - *pkey = hfi1_get_pkey(ibp, wqe->ud_wr.pkey_index); |
|---|
| 285 | + *pkey = hfi1_get_pkey(ibp, rvt_get_swqe_pkey_index(wqe)); |
|---|
| 307 | 286 | else |
|---|
| 308 | 287 | *pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); |
|---|
| 309 | 288 | if (!bypass) |
|---|
| 310 | 289 | bth0 |= *pkey; |
|---|
| 311 | 290 | ohdr->bth[0] = cpu_to_be32(bth0); |
|---|
| 312 | | - ohdr->bth[1] = cpu_to_be32(wqe->ud_wr.remote_qpn); |
|---|
| 291 | + ohdr->bth[1] = cpu_to_be32(rvt_get_swqe_remote_qpn(wqe)); |
|---|
| 313 | 292 | ohdr->bth[2] = cpu_to_be32(mask_psn(wqe->psn)); |
|---|
| 314 | 293 | /* |
|---|
| 315 | 294 | * Qkeys with the high order bit set mean use the |
|---|
| 316 | 295 | * qkey from the QP context instead of the WR (see 10.2.5). |
|---|
| 317 | 296 | */ |
|---|
| 318 | | - ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.remote_qkey < 0 ? |
|---|
| 319 | | - qp->qkey : wqe->ud_wr.remote_qkey); |
|---|
| 297 | + ohdr->u.ud.deth[0] = |
|---|
| 298 | + cpu_to_be32((int)rvt_get_swqe_remote_qkey(wqe) < 0 ? qp->qkey : |
|---|
| 299 | + rvt_get_swqe_remote_qkey(wqe)); |
|---|
| 320 | 300 | ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num); |
|---|
| 321 | 301 | } |
|---|
| 322 | 302 | |
|---|
| .. | .. |
|---|
| 336 | 316 | |
|---|
| 337 | 317 | ibp = to_iport(qp->ibqp.device, qp->port_num); |
|---|
| 338 | 318 | ppd = ppd_from_ibp(ibp); |
|---|
| 339 | | - ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; |
|---|
| 319 | + ah_attr = rvt_get_swqe_ah_attr(wqe); |
|---|
| 340 | 320 | |
|---|
| 341 | 321 | extra_bytes = -wqe->length & 3; |
|---|
| 342 | 322 | nwords = ((wqe->length + extra_bytes) >> 2) + SIZE_OF_CRC; |
|---|
| .. | .. |
|---|
| 400 | 380 | struct hfi1_pportdata *ppd; |
|---|
| 401 | 381 | struct hfi1_ibport *ibp; |
|---|
| 402 | 382 | u32 dlid, slid, nwords, extra_bytes; |
|---|
| 403 | | - u32 dest_qp = wqe->ud_wr.remote_qpn; |
|---|
| 383 | + u32 dest_qp = rvt_get_swqe_remote_qpn(wqe); |
|---|
| 404 | 384 | u32 src_qp = qp->ibqp.qp_num; |
|---|
| 405 | 385 | u16 len, pkey; |
|---|
| 406 | 386 | u8 l4, sc5; |
|---|
| .. | .. |
|---|
| 408 | 388 | |
|---|
| 409 | 389 | ibp = to_iport(qp->ibqp.device, qp->port_num); |
|---|
| 410 | 390 | ppd = ppd_from_ibp(ibp); |
|---|
| 411 | | - ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; |
|---|
| 391 | + ah_attr = rvt_get_swqe_ah_attr(wqe); |
|---|
| 412 | 392 | |
|---|
| 413 | 393 | /* |
|---|
| 414 | 394 | * Build 16B Management Packet if either the destination |
|---|
| .. | .. |
|---|
| 470 | 450 | |
|---|
| 471 | 451 | if (is_mgmt) { |
|---|
| 472 | 452 | l4 = OPA_16B_L4_FM; |
|---|
| 473 | | - pkey = hfi1_get_pkey(ibp, wqe->ud_wr.pkey_index); |
|---|
| 453 | + pkey = hfi1_get_pkey(ibp, rvt_get_swqe_pkey_index(wqe)); |
|---|
| 474 | 454 | hfi1_16B_set_qpn(&ps->s_txreq->phdr.hdr.opah.u.mgmt, |
|---|
| 475 | 455 | dest_qp, src_qp); |
|---|
| 476 | 456 | } else { |
|---|
| .. | .. |
|---|
| 519 | 499 | goto bail; |
|---|
| 520 | 500 | } |
|---|
| 521 | 501 | wqe = rvt_get_swqe_ptr(qp, qp->s_last); |
|---|
| 522 | | - hfi1_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); |
|---|
| 502 | + rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); |
|---|
| 523 | 503 | goto done_free_tx; |
|---|
| 524 | 504 | } |
|---|
| 525 | 505 | |
|---|
| .. | .. |
|---|
| 535 | 515 | /* Construct the header. */ |
|---|
| 536 | 516 | ibp = to_iport(qp->ibqp.device, qp->port_num); |
|---|
| 537 | 517 | ppd = ppd_from_ibp(ibp); |
|---|
| 538 | | - ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; |
|---|
| 518 | + ah_attr = rvt_get_swqe_ah_attr(wqe); |
|---|
| 539 | 519 | priv->hdr_type = hfi1_get_hdr_type(ppd->lid, ah_attr); |
|---|
| 540 | 520 | if ((!hfi1_check_mcast(rdma_ah_get_dlid(ah_attr))) || |
|---|
| 541 | 521 | (rdma_ah_get_dlid(ah_attr) == be32_to_cpu(OPA_LID_PERMISSIVE))) { |
|---|
| .. | .. |
|---|
| 561 | 541 | ud_loopback(qp, wqe); |
|---|
| 562 | 542 | spin_lock_irqsave(&qp->s_lock, tflags); |
|---|
| 563 | 543 | ps->flags = tflags; |
|---|
| 564 | | - hfi1_send_complete(qp, wqe, IB_WC_SUCCESS); |
|---|
| 544 | + rvt_send_complete(qp, wqe, IB_WC_SUCCESS); |
|---|
| 565 | 545 | goto done_free_tx; |
|---|
| 566 | 546 | } |
|---|
| 567 | 547 | } |
|---|
| .. | .. |
|---|
| 1025 | 1005 | goto drop; |
|---|
| 1026 | 1006 | } |
|---|
| 1027 | 1007 | if (packet->grh) { |
|---|
| 1028 | | - hfi1_copy_sge(&qp->r_sge, packet->grh, |
|---|
| 1029 | | - sizeof(struct ib_grh), true, false); |
|---|
| 1008 | + rvt_copy_sge(qp, &qp->r_sge, packet->grh, |
|---|
| 1009 | + sizeof(struct ib_grh), true, false); |
|---|
| 1030 | 1010 | wc.wc_flags |= IB_WC_GRH; |
|---|
| 1031 | 1011 | } else if (packet->etype == RHF_RCV_TYPE_BYPASS) { |
|---|
| 1032 | 1012 | struct ib_grh grh; |
|---|
| .. | .. |
|---|
| 1036 | 1016 | * out when creating 16B, add back the GRH here. |
|---|
| 1037 | 1017 | */ |
|---|
| 1038 | 1018 | hfi1_make_ext_grh(packet, &grh, slid, dlid); |
|---|
| 1039 | | - hfi1_copy_sge(&qp->r_sge, &grh, |
|---|
| 1040 | | - sizeof(struct ib_grh), true, false); |
|---|
| 1019 | + rvt_copy_sge(qp, &qp->r_sge, &grh, |
|---|
| 1020 | + sizeof(struct ib_grh), true, false); |
|---|
| 1041 | 1021 | wc.wc_flags |= IB_WC_GRH; |
|---|
| 1042 | 1022 | } else { |
|---|
| 1043 | 1023 | rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true); |
|---|
| 1044 | 1024 | } |
|---|
| 1045 | | - hfi1_copy_sge(&qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh), |
|---|
| 1046 | | - true, false); |
|---|
| 1025 | + rvt_copy_sge(qp, &qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh), |
|---|
| 1026 | + true, false); |
|---|
| 1047 | 1027 | rvt_put_ss(&qp->r_sge); |
|---|
| 1048 | 1028 | if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) |
|---|
| 1049 | 1029 | return; |
|---|
| .. | .. |
|---|
| 1081 | 1061 | dlid & ((1 << ppd_from_ibp(ibp)->lmc) - 1); |
|---|
| 1082 | 1062 | wc.port_num = qp->port_num; |
|---|
| 1083 | 1063 | /* Signal completion event if the solicited bit is set. */ |
|---|
| 1084 | | - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, solicited); |
|---|
| 1064 | + rvt_recv_cq(qp, &wc, solicited); |
|---|
| 1085 | 1065 | return; |
|---|
| 1086 | 1066 | |
|---|
| 1087 | 1067 | drop: |
|---|