.. | .. |
---|
45 | 45 | * |
---|
46 | 46 | */ |
---|
47 | 47 | |
---|
48 | | -#include <rdma/rdma_vt.h> |
---|
| 48 | +#include <rdma/rdmavt_qp.h> |
---|
49 | 49 | #include <rdma/ib_hdrs.h> |
---|
50 | 50 | |
---|
51 | 51 | /* |
---|
.. | .. |
---|
104 | 104 | } else { |
---|
105 | 105 | u32 min, max, x; |
---|
106 | 106 | u32 credits; |
---|
107 | | - struct rvt_rwq *wq = qp->r_rq.wq; |
---|
108 | 107 | u32 head; |
---|
109 | 108 | u32 tail; |
---|
110 | 109 | |
---|
111 | | - /* sanity check pointers before trusting them */ |
---|
112 | | - head = wq->head; |
---|
113 | | - if (head >= qp->r_rq.size) |
---|
114 | | - head = 0; |
---|
115 | | - tail = wq->tail; |
---|
116 | | - if (tail >= qp->r_rq.size) |
---|
117 | | - tail = 0; |
---|
118 | | - /* |
---|
119 | | - * Compute the number of credits available (RWQEs). |
---|
120 | | - * There is a small chance that the pair of reads are |
---|
121 | | - * not atomic, which is OK, since the fuzziness is |
---|
122 | | - * resolved as further ACKs go out. |
---|
123 | | - */ |
---|
124 | | - credits = head - tail; |
---|
125 | | - if ((int)credits < 0) |
---|
126 | | - credits += qp->r_rq.size; |
---|
| 110 | + credits = READ_ONCE(qp->r_rq.kwq->count); |
---|
| 111 | + if (credits == 0) { |
---|
| 112 | + /* sanity check pointers before trusting them */ |
---|
| 113 | + if (qp->ip) { |
---|
| 114 | + head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head); |
---|
| 115 | + tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail); |
---|
| 116 | + } else { |
---|
| 117 | + head = READ_ONCE(qp->r_rq.kwq->head); |
---|
| 118 | + tail = READ_ONCE(qp->r_rq.kwq->tail); |
---|
| 119 | + } |
---|
| 120 | + if (head >= qp->r_rq.size) |
---|
| 121 | + head = 0; |
---|
| 122 | + if (tail >= qp->r_rq.size) |
---|
| 123 | + tail = 0; |
---|
| 124 | + /* |
---|
| 125 | + * Compute the number of credits available (RWQEs). |
---|
| 126 | + * There is a small chance that the pair of reads are |
---|
| 127 | + * not atomic, which is OK, since the fuzziness is |
---|
| 128 | + * resolved as further ACKs go out. |
---|
| 129 | + */ |
---|
| 130 | + credits = rvt_get_rq_count(&qp->r_rq, head, tail); |
---|
| 131 | + } |
---|
127 | 132 | /* |
---|
128 | 133 | * Binary search the credit table to find the code to |
---|
129 | 134 | * use. |
---|
.. | .. |
---|
187 | 192 | } |
---|
188 | 193 | } |
---|
189 | 194 | EXPORT_SYMBOL(rvt_get_credit); |
---|
| 195 | + |
---|
| 196 | +/** |
---|
| 197 | + * rvt_restart_sge - rewind the sge state for a wqe |
---|
| 198 | + * @ss: the sge state pointer |
---|
| 199 | + * @wqe: the wqe to rewind |
---|
| 200 | + * @len: the data length from the start of the wqe in bytes |
---|
| 201 | + * |
---|
| 202 | + * Returns the remaining data length. |
---|
| 203 | + */ |
---|
| 204 | +u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len) |
---|
| 205 | +{ |
---|
| 206 | + ss->sge = wqe->sg_list[0]; |
---|
| 207 | + ss->sg_list = wqe->sg_list + 1; |
---|
| 208 | + ss->num_sge = wqe->wr.num_sge; |
---|
| 209 | + ss->total_len = wqe->length; |
---|
| 210 | + rvt_skip_sge(ss, len, false); |
---|
| 211 | + return wqe->length - len; |
---|
| 212 | +} |
---|
| 213 | +EXPORT_SYMBOL(rvt_restart_sge); |
---|
| 214 | + |
---|