| .. | .. |
|---|
| 45 | 45 | u32 len; |
|---|
| 46 | 46 | |
|---|
| 47 | 47 | len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu; |
|---|
| 48 | | - ss->sge = wqe->sg_list[0]; |
|---|
| 49 | | - ss->sg_list = wqe->sg_list + 1; |
|---|
| 50 | | - ss->num_sge = wqe->wr.num_sge; |
|---|
| 51 | | - ss->total_len = wqe->length; |
|---|
| 52 | | - rvt_skip_sge(ss, len, false); |
|---|
| 53 | | - return wqe->length - len; |
|---|
| 48 | + return rvt_restart_sge(ss, wqe, len); |
|---|
| 54 | 49 | } |
|---|
| 55 | 50 | |
|---|
| 56 | 51 | /** |
|---|
| .. | .. |
|---|
| 88 | 83 | rvt_put_mr(e->rdma_sge.mr); |
|---|
| 89 | 84 | e->rdma_sge.mr = NULL; |
|---|
| 90 | 85 | } |
|---|
| 91 | | - /* FALLTHROUGH */ |
|---|
| 86 | + fallthrough; |
|---|
| 92 | 87 | case OP(ATOMIC_ACKNOWLEDGE): |
|---|
| 93 | 88 | /* |
|---|
| 94 | 89 | * We can increment the tail pointer now that the last |
|---|
| .. | .. |
|---|
| 97 | 92 | */ |
|---|
| 98 | 93 | if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC) |
|---|
| 99 | 94 | qp->s_tail_ack_queue = 0; |
|---|
| 100 | | - /* FALLTHROUGH */ |
|---|
| 95 | + fallthrough; |
|---|
| 101 | 96 | case OP(SEND_ONLY): |
|---|
| 102 | 97 | case OP(ACKNOWLEDGE): |
|---|
| 103 | 98 | /* Check for no next entry in the queue. */ |
|---|
| .. | .. |
|---|
| 154 | 149 | |
|---|
| 155 | 150 | case OP(RDMA_READ_RESPONSE_FIRST): |
|---|
| 156 | 151 | qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); |
|---|
| 157 | | - /* FALLTHROUGH */ |
|---|
| 152 | + fallthrough; |
|---|
| 158 | 153 | case OP(RDMA_READ_RESPONSE_MIDDLE): |
|---|
| 159 | 154 | qp->s_cur_sge = &qp->s_ack_rdma_sge; |
|---|
| 160 | 155 | qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr; |
|---|
| .. | .. |
|---|
| 254 | 249 | goto bail; |
|---|
| 255 | 250 | } |
|---|
| 256 | 251 | wqe = rvt_get_swqe_ptr(qp, qp->s_last); |
|---|
| 257 | | - qib_send_complete(qp, wqe, qp->s_last != qp->s_acked ? |
|---|
| 252 | + rvt_send_complete(qp, wqe, qp->s_last != qp->s_acked ? |
|---|
| 258 | 253 | IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR); |
|---|
| 259 | 254 | /* will get called again */ |
|---|
| 260 | 255 | goto done; |
|---|
| .. | .. |
|---|
| 318 | 313 | case IB_WR_SEND: |
|---|
| 319 | 314 | case IB_WR_SEND_WITH_IMM: |
|---|
| 320 | 315 | /* If no credit, return. */ |
|---|
| 321 | | - if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) && |
|---|
| 322 | | - rvt_cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) { |
|---|
| 323 | | - qp->s_flags |= RVT_S_WAIT_SSN_CREDIT; |
|---|
| 316 | + if (!rvt_rc_credit_avail(qp, wqe)) |
|---|
| 324 | 317 | goto bail; |
|---|
| 325 | | - } |
|---|
| 326 | 318 | if (len > pmtu) { |
|---|
| 327 | 319 | qp->s_state = OP(SEND_FIRST); |
|---|
| 328 | 320 | len = pmtu; |
|---|
| .. | .. |
|---|
| 349 | 341 | goto no_flow_control; |
|---|
| 350 | 342 | case IB_WR_RDMA_WRITE_WITH_IMM: |
|---|
| 351 | 343 | /* If no credit, return. */ |
|---|
| 352 | | - if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) && |
|---|
| 353 | | - rvt_cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) { |
|---|
| 354 | | - qp->s_flags |= RVT_S_WAIT_SSN_CREDIT; |
|---|
| 344 | + if (!rvt_rc_credit_avail(qp, wqe)) |
|---|
| 355 | 345 | goto bail; |
|---|
| 356 | | - } |
|---|
| 357 | 346 | no_flow_control: |
|---|
| 358 | 347 | ohdr->u.rc.reth.vaddr = |
|---|
| 359 | 348 | cpu_to_be64(wqe->rdma_wr.remote_addr); |
|---|
| .. | .. |
|---|
| 482 | 471 | * See qib_restart_rc(). |
|---|
| 483 | 472 | */ |
|---|
| 484 | 473 | qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); |
|---|
| 485 | | - /* FALLTHROUGH */ |
|---|
| 474 | + fallthrough; |
|---|
| 486 | 475 | case OP(SEND_FIRST): |
|---|
| 487 | 476 | qp->s_state = OP(SEND_MIDDLE); |
|---|
| 488 | | - /* FALLTHROUGH */ |
|---|
| 477 | + fallthrough; |
|---|
| 489 | 478 | case OP(SEND_MIDDLE): |
|---|
| 490 | 479 | bth2 = qp->s_psn++ & QIB_PSN_MASK; |
|---|
| 491 | 480 | ss = &qp->s_sge; |
|---|
| .. | .. |
|---|
| 521 | 510 | * See qib_restart_rc(). |
|---|
| 522 | 511 | */ |
|---|
| 523 | 512 | qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); |
|---|
| 524 | | - /* FALLTHROUGH */ |
|---|
| 513 | + fallthrough; |
|---|
| 525 | 514 | case OP(RDMA_WRITE_FIRST): |
|---|
| 526 | 515 | qp->s_state = OP(RDMA_WRITE_MIDDLE); |
|---|
| 527 | | - /* FALLTHROUGH */ |
|---|
| 516 | + fallthrough; |
|---|
| 528 | 517 | case OP(RDMA_WRITE_MIDDLE): |
|---|
| 529 | 518 | bth2 = qp->s_psn++ & QIB_PSN_MASK; |
|---|
| 530 | 519 | ss = &qp->s_sge; |
|---|
| .. | .. |
|---|
| 838 | 827 | qib_migrate_qp(qp); |
|---|
| 839 | 828 | qp->s_retry = qp->s_retry_cnt; |
|---|
| 840 | 829 | } else if (qp->s_last == qp->s_acked) { |
|---|
| 841 | | - qib_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); |
|---|
| 830 | + rvt_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); |
|---|
| 842 | 831 | rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); |
|---|
| 843 | 832 | return; |
|---|
| 844 | 833 | } else /* XXX need to handle delayed completion */ |
|---|
| .. | .. |
|---|
| 926 | 915 | rvt_add_retry_timer(qp); |
|---|
| 927 | 916 | |
|---|
| 928 | 917 | while (qp->s_last != qp->s_acked) { |
|---|
| 929 | | - u32 s_last; |
|---|
| 930 | | - |
|---|
| 931 | 918 | wqe = rvt_get_swqe_ptr(qp, qp->s_last); |
|---|
| 932 | 919 | if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && |
|---|
| 933 | 920 | qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) |
|---|
| 934 | 921 | break; |
|---|
| 935 | | - s_last = qp->s_last; |
|---|
| 936 | | - if (++s_last >= qp->s_size) |
|---|
| 937 | | - s_last = 0; |
|---|
| 938 | | - qp->s_last = s_last; |
|---|
| 939 | | - /* see post_send() */ |
|---|
| 940 | | - barrier(); |
|---|
| 941 | | - rvt_put_swqe(wqe); |
|---|
| 942 | | - rvt_qp_swqe_complete(qp, |
|---|
| 922 | + rvt_qp_complete_swqe(qp, |
|---|
| 943 | 923 | wqe, |
|---|
| 944 | 924 | ib_qib_wc_opcode[wqe->wr.opcode], |
|---|
| 945 | 925 | IB_WC_SUCCESS); |
|---|
| .. | .. |
|---|
| 977 | 957 | * is finished. |
|---|
| 978 | 958 | */ |
|---|
| 979 | 959 | if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || |
|---|
| 980 | | - qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { |
|---|
| 981 | | - u32 s_last; |
|---|
| 982 | | - |
|---|
| 983 | | - rvt_put_swqe(wqe); |
|---|
| 984 | | - s_last = qp->s_last; |
|---|
| 985 | | - if (++s_last >= qp->s_size) |
|---|
| 986 | | - s_last = 0; |
|---|
| 987 | | - qp->s_last = s_last; |
|---|
| 988 | | - /* see post_send() */ |
|---|
| 989 | | - barrier(); |
|---|
| 990 | | - rvt_qp_swqe_complete(qp, |
|---|
| 960 | + qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) |
|---|
| 961 | + rvt_qp_complete_swqe(qp, |
|---|
| 991 | 962 | wqe, |
|---|
| 992 | 963 | ib_qib_wc_opcode[wqe->wr.opcode], |
|---|
| 993 | 964 | IB_WC_SUCCESS); |
|---|
| 994 | | - } else |
|---|
| 965 | + else |
|---|
| 995 | 966 | this_cpu_inc(*ibp->rvp.rc_delayed_comp); |
|---|
| 996 | 967 | |
|---|
| 997 | 968 | qp->s_retry = qp->s_retry_cnt; |
|---|
| .. | .. |
|---|
| 1221 | 1192 | ibp->rvp.n_other_naks++; |
|---|
| 1222 | 1193 | class_b: |
|---|
| 1223 | 1194 | if (qp->s_last == qp->s_acked) { |
|---|
| 1224 | | - qib_send_complete(qp, wqe, status); |
|---|
| 1195 | + rvt_send_complete(qp, wqe, status); |
|---|
| 1225 | 1196 | rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); |
|---|
| 1226 | 1197 | } |
|---|
| 1227 | 1198 | break; |
|---|
| .. | .. |
|---|
| 1425 | 1396 | qp->s_rdma_read_len -= pmtu; |
|---|
| 1426 | 1397 | update_last_psn(qp, psn); |
|---|
| 1427 | 1398 | spin_unlock_irqrestore(&qp->s_lock, flags); |
|---|
| 1428 | | - qib_copy_sge(&qp->s_rdma_read_sge, data, pmtu, 0); |
|---|
| 1399 | + rvt_copy_sge(qp, &qp->s_rdma_read_sge, |
|---|
| 1400 | + data, pmtu, false, false); |
|---|
| 1429 | 1401 | goto bail; |
|---|
| 1430 | 1402 | |
|---|
| 1431 | 1403 | case OP(RDMA_READ_RESPONSE_ONLY): |
|---|
| .. | .. |
|---|
| 1471 | 1443 | if (unlikely(tlen != qp->s_rdma_read_len)) |
|---|
| 1472 | 1444 | goto ack_len_err; |
|---|
| 1473 | 1445 | aeth = be32_to_cpu(ohdr->u.aeth); |
|---|
| 1474 | | - qib_copy_sge(&qp->s_rdma_read_sge, data, tlen, 0); |
|---|
| 1446 | + rvt_copy_sge(qp, &qp->s_rdma_read_sge, |
|---|
| 1447 | + data, tlen, false, false); |
|---|
| 1475 | 1448 | WARN_ON(qp->s_rdma_read_sge.num_sge); |
|---|
| 1476 | 1449 | (void) do_rc_ack(qp, aeth, psn, |
|---|
| 1477 | 1450 | OP(RDMA_READ_RESPONSE_LAST), 0, rcd); |
|---|
| .. | .. |
|---|
| 1490 | 1463 | status = IB_WC_LOC_LEN_ERR; |
|---|
| 1491 | 1464 | ack_err: |
|---|
| 1492 | 1465 | if (qp->s_last == qp->s_acked) { |
|---|
| 1493 | | - qib_send_complete(qp, wqe, status); |
|---|
| 1466 | + rvt_send_complete(qp, wqe, status); |
|---|
| 1494 | 1467 | rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); |
|---|
| 1495 | 1468 | } |
|---|
| 1496 | 1469 | ack_done: |
|---|
| .. | .. |
|---|
| 1834 | 1807 | if (!ret) |
|---|
| 1835 | 1808 | goto rnr_nak; |
|---|
| 1836 | 1809 | qp->r_rcv_len = 0; |
|---|
| 1837 | | - /* FALLTHROUGH */ |
|---|
| 1810 | + fallthrough; |
|---|
| 1838 | 1811 | case OP(SEND_MIDDLE): |
|---|
| 1839 | 1812 | case OP(RDMA_WRITE_MIDDLE): |
|---|
| 1840 | 1813 | send_middle: |
|---|
| .. | .. |
|---|
| 1844 | 1817 | qp->r_rcv_len += pmtu; |
|---|
| 1845 | 1818 | if (unlikely(qp->r_rcv_len > qp->r_len)) |
|---|
| 1846 | 1819 | goto nack_inv; |
|---|
| 1847 | | - qib_copy_sge(&qp->r_sge, data, pmtu, 1); |
|---|
| 1820 | + rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); |
|---|
| 1848 | 1821 | break; |
|---|
| 1849 | 1822 | |
|---|
| 1850 | 1823 | case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): |
|---|
| .. | .. |
|---|
| 1866 | 1839 | qp->r_rcv_len = 0; |
|---|
| 1867 | 1840 | if (opcode == OP(SEND_ONLY)) |
|---|
| 1868 | 1841 | goto no_immediate_data; |
|---|
| 1869 | | - /* fall through -- for SEND_ONLY_WITH_IMMEDIATE */ |
|---|
| 1842 | + fallthrough; /* for SEND_ONLY_WITH_IMMEDIATE */ |
|---|
| 1870 | 1843 | case OP(SEND_LAST_WITH_IMMEDIATE): |
|---|
| 1871 | 1844 | send_last_imm: |
|---|
| 1872 | 1845 | wc.ex.imm_data = ohdr->u.imm_data; |
|---|
| .. | .. |
|---|
| 1890 | 1863 | wc.byte_len = tlen + qp->r_rcv_len; |
|---|
| 1891 | 1864 | if (unlikely(wc.byte_len > qp->r_len)) |
|---|
| 1892 | 1865 | goto nack_inv; |
|---|
| 1893 | | - qib_copy_sge(&qp->r_sge, data, tlen, 1); |
|---|
| 1866 | + rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); |
|---|
| 1894 | 1867 | rvt_put_ss(&qp->r_sge); |
|---|
| 1895 | 1868 | qp->r_msn++; |
|---|
| 1896 | 1869 | if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) |
|---|
| .. | .. |
|---|
| 1912 | 1885 | wc.dlid_path_bits = 0; |
|---|
| 1913 | 1886 | wc.port_num = 0; |
|---|
| 1914 | 1887 | /* Signal completion event if the solicited bit is set. */ |
|---|
| 1915 | | - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, |
|---|
| 1916 | | - ib_bth_is_solicited(ohdr)); |
|---|
| 1888 | + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); |
|---|
| 1917 | 1889 | break; |
|---|
| 1918 | 1890 | |
|---|
| 1919 | 1891 | case OP(RDMA_WRITE_FIRST): |
|---|