~hc/RK356X_SDK_RELEASE.git

..	..	@@ -36,12 +36,16 @@
36	36	* Description: Fast Path Operators
37	37	*/
38	38
	39	+#define dev_fmt(fmt) "QPLIB: " fmt
	40	+
39	41	#include <linux/interrupt.h>
40	42	#include <linux/spinlock.h>
41	43	#include <linux/sched.h>
42	44	#include <linux/slab.h>
43	45	#include <linux/pci.h>
	46	+#include <linux/delay.h>
44	47	#include <linux/prefetch.h>
	48	+#include <linux/if_ether.h>
45	49
46	50	#include "roce_hsi.h"
47	51
..	..	@@ -50,9 +54,7 @@
50	54	#include "qplib_sp.h"
51	55	#include "qplib_fp.h"
52	56
53		-static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
54	57	static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);
55		-static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type);
56	58
57	59	static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp)
58	60	{
..	..	@@ -71,8 +73,7 @@
71	73
72	74	if (!qp->sq.flushed) {
73	75	dev_dbg(&scq->hwq.pdev->dev,
74		- "QPLIB: FP: Adding to SQ Flush list = %p",
75		- qp);
	76	+ "FP: Adding to SQ Flush list = %p\n", qp);
76	77	bnxt_qplib_cancel_phantom_processing(qp);
77	78	list_add_tail(&qp->sq_flush, &scq->sqf_head);
78	79	qp->sq.flushed = true;
..	..	@@ -80,8 +81,7 @@
80	81	if (!qp->srq) {
81	82	if (!qp->rq.flushed) {
82	83	dev_dbg(&rcq->hwq.pdev->dev,
83		- "QPLIB: FP: Adding to RQ Flush list = %p",
84		- qp);
	84	+ "FP: Adding to RQ Flush list = %p\n", qp);
85	85	list_add_tail(&qp->rq_flush, &rcq->rqf_head);
86	86	qp->rq.flushed = true;
87	87	}
..	..	@@ -178,11 +178,11 @@
178	178
179	179	if (qp->rq_hdr_buf)
180	180	dma_free_coherent(&res->pdev->dev,
181		- rq->hwq.max_elements * qp->rq_hdr_buf_size,
	181	+ rq->max_wqe * qp->rq_hdr_buf_size,
182	182	qp->rq_hdr_buf, qp->rq_hdr_buf_map);
183	183	if (qp->sq_hdr_buf)
184	184	dma_free_coherent(&res->pdev->dev,
185		- sq->hwq.max_elements * qp->sq_hdr_buf_size,
	185	+ sq->max_wqe * qp->sq_hdr_buf_size,
186	186	qp->sq_hdr_buf, qp->sq_hdr_buf_map);
187	187	qp->rq_hdr_buf = NULL;
188	188	qp->sq_hdr_buf = NULL;
..	..	@@ -199,29 +199,28 @@
199	199	struct bnxt_qplib_q *sq = &qp->sq;
200	200	int rc = 0;
201	201
202		- if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {
	202	+ if (qp->sq_hdr_buf_size && sq->max_wqe) {
203	203	qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
204		- sq->hwq.max_elements *
205		- qp->sq_hdr_buf_size,
	204	+ sq->max_wqe * qp->sq_hdr_buf_size,
206	205	&qp->sq_hdr_buf_map, GFP_KERNEL);
207	206	if (!qp->sq_hdr_buf) {
208	207	rc = -ENOMEM;
209	208	dev_err(&res->pdev->dev,
210		- "QPLIB: Failed to create sq_hdr_buf");
	209	+ "Failed to create sq_hdr_buf\n");
211	210	goto fail;
212	211	}
213	212	}
214	213
215		- if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {
	214	+ if (qp->rq_hdr_buf_size && rq->max_wqe) {
216	215	qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
217		- rq->hwq.max_elements *
	216	+ rq->max_wqe *
218	217	qp->rq_hdr_buf_size,
219	218	&qp->rq_hdr_buf_map,
220	219	GFP_KERNEL);
221	220	if (!qp->rq_hdr_buf) {
222	221	rc = -ENOMEM;
223	222	dev_err(&res->pdev->dev,
224		- "QPLIB: Failed to create rq_hdr_buf");
	223	+ "Failed to create rq_hdr_buf\n");
225	224	goto fail;
226	225	}
227	226	}
..	..	@@ -232,19 +231,16 @@
232	231	return rc;
233	232	}
234	233
235		-static void bnxt_qplib_service_nq(unsigned long data)
	234	+static void clean_nq(struct bnxt_qplib_nq nq, struct bnxt_qplib_cq cq)
236	235	{
237		- struct bnxt_qplib_nq nq = (struct bnxt_qplib_nq )data;
238	236	struct bnxt_qplib_hwq *hwq = &nq->hwq;
239	237	struct nq_base nqe, *nq_ptr;
240		- struct bnxt_qplib_cq *cq;
241		- int num_cqne_processed = 0;
242		- int num_srqne_processed = 0;
243		- u32 sw_cons, raw_cons;
244		- u16 type;
245	238	int budget = nq->budget;
	239	+ u32 sw_cons, raw_cons;
246	240	uintptr_t q_handle;
	241	+ u16 type;
247	242
	243	+ spin_lock_bh(&hwq->lock);
248	244	/* Service the NQ until empty */
249	245	raw_cons = hwq->cons;
250	246	while (budget--) {
..	..	@@ -269,36 +265,108 @@
269	265	q_handle = le32_to_cpu(nqcne->cq_handle_low);
270	266	q_handle \|= (u64)le32_to_cpu(nqcne->cq_handle_high)
271	267	<< 32;
	268	+ if ((unsigned long)cq == q_handle) {
	269	+ nqcne->cq_handle_low = 0;
	270	+ nqcne->cq_handle_high = 0;
	271	+ cq->cnq_events++;
	272	+ }
	273	+ break;
	274	+ }
	275	+ default:
	276	+ break;
	277	+ }
	278	+ raw_cons++;
	279	+ }
	280	+ spin_unlock_bh(&hwq->lock);
	281	+}
	282	+
	283	+/* Wait for receiving all NQEs for this CQ and clean the NQEs associated with
	284	+ * this CQ.
	285	+ */
	286	+static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events)
	287	+{
	288	+ u32 retry_cnt = 100;
	289	+
	290	+ while (retry_cnt--) {
	291	+ if (cnq_events == cq->cnq_events)
	292	+ return;
	293	+ usleep_range(50, 100);
	294	+ clean_nq(cq->nq, cq);
	295	+ }
	296	+}
	297	+
	298	+static void bnxt_qplib_service_nq(struct tasklet_struct *t)
	299	+{
	300	+ struct bnxt_qplib_nq *nq = from_tasklet(nq, t, nq_tasklet);
	301	+ struct bnxt_qplib_hwq *hwq = &nq->hwq;
	302	+ int num_srqne_processed = 0;
	303	+ int num_cqne_processed = 0;
	304	+ struct bnxt_qplib_cq *cq;
	305	+ int budget = nq->budget;
	306	+ u32 sw_cons, raw_cons;
	307	+ struct nq_base *nqe;
	308	+ uintptr_t q_handle;
	309	+ u16 type;
	310	+
	311	+ spin_lock_bh(&hwq->lock);
	312	+ /* Service the NQ until empty */
	313	+ raw_cons = hwq->cons;
	314	+ while (budget--) {
	315	+ sw_cons = HWQ_CMP(raw_cons, hwq);
	316	+ nqe = bnxt_qplib_get_qe(hwq, sw_cons, NULL);
	317	+ if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
	318	+ break;
	319	+
	320	+ /*
	321	+ * The valid test of the entry must be done first before
	322	+ * reading any further.
	323	+ */
	324	+ dma_rmb();
	325	+
	326	+ type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
	327	+ switch (type) {
	328	+ case NQ_BASE_TYPE_CQ_NOTIFICATION:
	329	+ {
	330	+ struct nq_cn nqcne = (struct nq_cn )nqe;
	331	+
	332	+ q_handle = le32_to_cpu(nqcne->cq_handle_low);
	333	+ q_handle \|= (u64)le32_to_cpu(nqcne->cq_handle_high)
	334	+ << 32;
272	335	cq = (struct bnxt_qplib_cq *)(unsigned long)q_handle;
273		- bnxt_qplib_arm_cq_enable(cq);
	336	+ if (!cq)
	337	+ break;
	338	+ bnxt_qplib_armen_db(&cq->dbinfo,
	339	+ DBC_DBC_TYPE_CQ_ARMENA);
274	340	spin_lock_bh(&cq->compl_lock);
275	341	atomic_set(&cq->arm_state, 0);
276	342	if (!nq->cqn_handler(nq, (cq)))
277	343	num_cqne_processed++;
278	344	else
279	345	dev_warn(&nq->pdev->dev,
280		- "QPLIB: cqn - type 0x%x not handled",
281		- type);
	346	+ "cqn - type 0x%x not handled\n", type);
	347	+ cq->cnq_events++;
282	348	spin_unlock_bh(&cq->compl_lock);
283	349	break;
284	350	}
285	351	case NQ_BASE_TYPE_SRQ_EVENT:
286	352	{
	353	+ struct bnxt_qplib_srq *srq;
287	354	struct nq_srq_event *nqsrqe =
288	355	(struct nq_srq_event *)nqe;
289	356
290	357	q_handle = le32_to_cpu(nqsrqe->srq_handle_low);
291	358	q_handle \|= (u64)le32_to_cpu(nqsrqe->srq_handle_high)
292	359	<< 32;
293		- bnxt_qplib_arm_srq((struct bnxt_qplib_srq *)q_handle,
294		- DBR_DBR_TYPE_SRQ_ARMENA);
	360	+ srq = (struct bnxt_qplib_srq *)q_handle;
	361	+ bnxt_qplib_armen_db(&srq->dbinfo,
	362	+ DBC_DBC_TYPE_SRQ_ARMENA);
295	363	if (!nq->srqn_handler(nq,
296	364	(struct bnxt_qplib_srq *)q_handle,
297	365	nqsrqe->event))
298	366	num_srqne_processed++;
299	367	else
300	368	dev_warn(&nq->pdev->dev,
301		- "QPLIB: SRQ event 0x%x not handled",
	369	+ "SRQ event 0x%x not handled\n",
302	370	nqsrqe->event);
303	371	break;
304	372	}
..	..	@@ -306,50 +374,52 @@
306	374	break;
307	375	default:
308	376	dev_warn(&nq->pdev->dev,
309		- "QPLIB: nqe with type = 0x%x not handled",
310		- type);
	377	+ "nqe with type = 0x%x not handled\n", type);
311	378	break;
312	379	}
313	380	raw_cons++;
314	381	}
315	382	if (hwq->cons != raw_cons) {
316	383	hwq->cons = raw_cons;
317		- NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements);
	384	+ bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, true);
318	385	}
	386	+ spin_unlock_bh(&hwq->lock);
319	387	}
320	388
321	389	static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
322	390	{
323	391	struct bnxt_qplib_nq *nq = dev_instance;
324	392	struct bnxt_qplib_hwq *hwq = &nq->hwq;
325		- struct nq_base **nq_ptr;
326	393	u32 sw_cons;
327	394
328	395	/* Prefetch the NQ element */
329	396	sw_cons = HWQ_CMP(hwq->cons, hwq);
330		- nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr;
331		- prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);
	397	+ prefetch(bnxt_qplib_get_qe(hwq, sw_cons, NULL));
332	398
333	399	/* Fan out to CPU affinitized kthreads? */
334		- tasklet_schedule(&nq->worker);
	400	+ tasklet_schedule(&nq->nq_tasklet);
335	401
336	402	return IRQ_HANDLED;
337	403	}
338	404
339	405	void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill)
340	406	{
341		- tasklet_disable(&nq->worker);
	407	+ if (!nq->requested)
	408	+ return;
	409	+
	410	+ tasklet_disable(&nq->nq_tasklet);
342	411	/* Mask h/w interrupt */
343		- NQ_DB(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
	412	+ bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, false);
344	413	/* Sync with last running IRQ handler */
345		- synchronize_irq(nq->vector);
	414	+ synchronize_irq(nq->msix_vec);
346	415	if (kill)
347		- tasklet_kill(&nq->worker);
348		- if (nq->requested) {
349		- irq_set_affinity_hint(nq->vector, NULL);
350		- free_irq(nq->vector, nq);
351		- nq->requested = false;
352		- }
	416	+ tasklet_kill(&nq->nq_tasklet);
	417	+
	418	+ irq_set_affinity_hint(nq->msix_vec, NULL);
	419	+ free_irq(nq->msix_vec, nq);
	420	+ kfree(nq->name);
	421	+ nq->name = NULL;
	422	+ nq->requested = false;
353	423	}
354	424
355	425	void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
..	..	@@ -360,91 +430,120 @@
360	430	}
361	431
362	432	/* Make sure the HW is stopped! */
363		- if (nq->requested)
364		- bnxt_qplib_nq_stop_irq(nq, true);
	433	+ bnxt_qplib_nq_stop_irq(nq, true);
365	434
366		- if (nq->bar_reg_iomem)
367		- iounmap(nq->bar_reg_iomem);
368		- nq->bar_reg_iomem = NULL;
	435	+ if (nq->nq_db.reg.bar_reg) {
	436	+ iounmap(nq->nq_db.reg.bar_reg);
	437	+ nq->nq_db.reg.bar_reg = NULL;
	438	+ }
369	439
370	440	nq->cqn_handler = NULL;
371	441	nq->srqn_handler = NULL;
372		- nq->vector = 0;
	442	+ nq->msix_vec = 0;
373	443	}
374	444
375	445	int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
376	446	int msix_vector, bool need_init)
377	447	{
	448	+ struct bnxt_qplib_res *res = nq->res;
378	449	int rc;
379	450
380	451	if (nq->requested)
381	452	return -EFAULT;
382	453
383		- nq->vector = msix_vector;
	454	+ nq->msix_vec = msix_vector;
384	455	if (need_init)
385		- tasklet_init(&nq->worker, bnxt_qplib_service_nq,
386		- (unsigned long)nq);
	456	+ tasklet_setup(&nq->nq_tasklet, bnxt_qplib_service_nq);
387	457	else
388		- tasklet_enable(&nq->worker);
	458	+ tasklet_enable(&nq->nq_tasklet);
389	459
390		- snprintf(nq->name, sizeof(nq->name), "bnxt_qplib_nq-%d", nq_indx);
391		- rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, nq->name, nq);
392		- if (rc)
	460	+ nq->name = kasprintf(GFP_KERNEL, "bnxt_re-nq-%d@pci:%s",
	461	+ nq_indx, pci_name(res->pdev));
	462	+ if (!nq->name)
	463	+ return -ENOMEM;
	464	+ rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq);
	465	+ if (rc) {
	466	+ kfree(nq->name);
	467	+ nq->name = NULL;
	468	+ tasklet_disable(&nq->nq_tasklet);
393	469	return rc;
	470	+ }
394	471
395	472	cpumask_clear(&nq->mask);
396	473	cpumask_set_cpu(nq_indx, &nq->mask);
397		- rc = irq_set_affinity_hint(nq->vector, &nq->mask);
	474	+ rc = irq_set_affinity_hint(nq->msix_vec, &nq->mask);
398	475	if (rc) {
399	476	dev_warn(&nq->pdev->dev,
400		- "QPLIB: set affinity failed; vector: %d nq_idx: %d\n",
401		- nq->vector, nq_indx);
	477	+ "set affinity failed; vector: %d nq_idx: %d\n",
	478	+ nq->msix_vec, nq_indx);
402	479	}
403	480	nq->requested = true;
404		- NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
	481	+ bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true);
405	482
	483	+ return rc;
	484	+}
	485	+
	486	+static int bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq, u32 reg_offt)
	487	+{
	488	+ resource_size_t reg_base;
	489	+ struct bnxt_qplib_nq_db *nq_db;
	490	+ struct pci_dev *pdev;
	491	+ int rc = 0;
	492	+
	493	+ pdev = nq->pdev;
	494	+ nq_db = &nq->nq_db;
	495	+
	496	+ nq_db->reg.bar_id = NQ_CONS_PCI_BAR_REGION;
	497	+ nq_db->reg.bar_base = pci_resource_start(pdev, nq_db->reg.bar_id);
	498	+ if (!nq_db->reg.bar_base) {
	499	+ dev_err(&pdev->dev, "QPLIB: NQ BAR region %d resc start is 0!",
	500	+ nq_db->reg.bar_id);
	501	+ rc = -ENOMEM;
	502	+ goto fail;
	503	+ }
	504	+
	505	+ reg_base = nq_db->reg.bar_base + reg_offt;
	506	+ /* Unconditionally map 8 bytes to support 57500 series */
	507	+ nq_db->reg.len = 8;
	508	+ nq_db->reg.bar_reg = ioremap(reg_base, nq_db->reg.len);
	509	+ if (!nq_db->reg.bar_reg) {
	510	+ dev_err(&pdev->dev, "QPLIB: NQ BAR region %d mapping failed",
	511	+ nq_db->reg.bar_id);
	512	+ rc = -ENOMEM;
	513	+ goto fail;
	514	+ }
	515	+
	516	+ nq_db->dbinfo.db = nq_db->reg.bar_reg;
	517	+ nq_db->dbinfo.hwq = &nq->hwq;
	518	+ nq_db->dbinfo.xid = nq->ring_id;
	519	+fail:
406	520	return rc;
407	521	}
408	522
409	523	int bnxt_qplib_enable_nq(struct pci_dev pdev, struct bnxt_qplib_nq nq,
410	524	int nq_idx, int msix_vector, int bar_reg_offset,
411		- int (cqn_handler)(struct bnxt_qplib_nq nq,
412		- struct bnxt_qplib_cq *),
413		- int (srqn_handler)(struct bnxt_qplib_nq nq,
414		- struct bnxt_qplib_srq *,
415		- u8 event))
	525	+ cqn_handler_t cqn_handler,
	526	+ srqn_handler_t srqn_handler)
416	527	{
417		- resource_size_t nq_base;
418	528	int rc = -1;
419	529
420		- if (cqn_handler)
421		- nq->cqn_handler = cqn_handler;
422		-
423		- if (srqn_handler)
424		- nq->srqn_handler = srqn_handler;
	530	+ nq->pdev = pdev;
	531	+ nq->cqn_handler = cqn_handler;
	532	+ nq->srqn_handler = srqn_handler;
425	533
426	534	/* Have a task to schedule CQ notifiers in post send case */
427	535	nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq");
428	536	if (!nq->cqn_wq)
429	537	return -ENOMEM;
430	538
431		- nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
432		- nq->bar_reg_off = bar_reg_offset;
433		- nq_base = pci_resource_start(pdev, nq->bar_reg);
434		- if (!nq_base) {
435		- rc = -ENOMEM;
	539	+ rc = bnxt_qplib_map_nq_db(nq, bar_reg_offset);
	540	+ if (rc)
436	541	goto fail;
437		- }
438		- nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4);
439		- if (!nq->bar_reg_iomem) {
440		- rc = -ENOMEM;
441		- goto fail;
442		- }
443	542
444	543	rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true);
445	544	if (rc) {
446	545	dev_err(&nq->pdev->dev,
447		- "QPLIB: Failed to request irq for nq-idx %d", nq_idx);
	546	+ "Failed to request irq for nq-idx %d\n", nq_idx);
448	547	goto fail;
449	548	}
450	549
..	..	@@ -457,50 +556,39 @@
457	556	void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
458	557	{
459	558	if (nq->hwq.max_elements) {
460		- bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);
	559	+ bnxt_qplib_free_hwq(nq->res, &nq->hwq);
461	560	nq->hwq.max_elements = 0;
462	561	}
463	562	}
464	563
465		-int bnxt_qplib_alloc_nq(struct pci_dev pdev, struct bnxt_qplib_nq nq)
	564	+int bnxt_qplib_alloc_nq(struct bnxt_qplib_res res, struct bnxt_qplib_nq nq)
466	565	{
467		- nq->pdev = pdev;
	566	+ struct bnxt_qplib_hwq_attr hwq_attr = {};
	567	+ struct bnxt_qplib_sg_info sginfo = {};
	568	+
	569	+ nq->pdev = res->pdev;
	570	+ nq->res = res;
468	571	if (!nq->hwq.max_elements \|\|
469	572	nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
470	573	nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
471	574
472		- if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0,
473		- &nq->hwq.max_elements,
474		- BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,
475		- PAGE_SIZE, HWQ_TYPE_L2_CMPL))
	575	+ sginfo.pgsize = PAGE_SIZE;
	576	+ sginfo.pgshft = PAGE_SHIFT;
	577	+ hwq_attr.res = res;
	578	+ hwq_attr.sginfo = &sginfo;
	579	+ hwq_attr.depth = nq->hwq.max_elements;
	580	+ hwq_attr.stride = sizeof(struct nq_base);
	581	+ hwq_attr.type = bnxt_qplib_get_hwq_type(nq->res);
	582	+ if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) {
	583	+ dev_err(&nq->pdev->dev, "FP NQ allocation failed");
476	584	return -ENOMEM;
477		-
	585	+ }
478	586	nq->budget = 8;
479	587	return 0;
480	588	}
481	589
482	590	/* SRQ */
483		-static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type)
484		-{
485		- struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
486		- struct dbr_dbr db_msg = { 0 };
487		- void __iomem *db;
488		- u32 sw_prod = 0;
489		-
490		- /* Ring DB */
491		- sw_prod = (arm_type == DBR_DBR_TYPE_SRQ_ARM) ? srq->threshold :
492		- HWQ_CMP(srq_hwq->prod, srq_hwq);
493		- db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
494		- DBR_DBR_INDEX_MASK);
495		- db_msg.type_xid = cpu_to_le32(((srq->id << DBR_DBR_XID_SFT) &
496		- DBR_DBR_XID_MASK) \| arm_type);
497		- db = (arm_type == DBR_DBR_TYPE_SRQ_ARMENA) ?
498		- srq->dbr_base : srq->dpi->dbr;
499		- wmb(); /* barrier before db ring */
500		- __iowrite64_copy(db, &db_msg, sizeof(db_msg) / sizeof(u64));
501		-}
502		-
503		-int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
	591	+void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
504	592	struct bnxt_qplib_srq *srq)
505	593	{
506	594	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
..	..	@@ -514,31 +602,32 @@
514	602	/* Configure the request */
515	603	req.srq_cid = cpu_to_le32(srq->id);
516	604
517		- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
518		- (void *)&resp, NULL, 0);
519		- if (rc)
520		- return rc;
521		-
522		- bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
	605	+ rc = bnxt_qplib_rcfw_send_message(rcfw, (struct cmdq_base *)&req,
	606	+ (struct creq_base *)&resp, NULL, 0);
523	607	kfree(srq->swq);
524		- return 0;
	608	+ if (rc)
	609	+ return;
	610	+ bnxt_qplib_free_hwq(res, &srq->hwq);
525	611	}
526	612
527	613	int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
528	614	struct bnxt_qplib_srq *srq)
529	615	{
530	616	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
531		- struct cmdq_create_srq req;
	617	+ struct bnxt_qplib_hwq_attr hwq_attr = {};
532	618	struct creq_create_srq_resp resp;
	619	+ struct cmdq_create_srq req;
533	620	struct bnxt_qplib_pbl *pbl;
534	621	u16 cmd_flags = 0;
	622	+ u16 pg_sz_lvl;
535	623	int rc, idx;
536	624
537		- srq->hwq.max_elements = srq->max_wqe;
538		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, &srq->hwq, srq->sglist,
539		- srq->nmap, &srq->hwq.max_elements,
540		- BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
541		- PAGE_SIZE, HWQ_TYPE_QUEUE);
	625	+ hwq_attr.res = res;
	626	+ hwq_attr.sginfo = &srq->sg_info;
	627	+ hwq_attr.depth = srq->max_wqe;
	628	+ hwq_attr.stride = srq->wqe_size;
	629	+ hwq_attr.type = HWQ_TYPE_QUEUE;
	630	+ rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr);
542	631	if (rc)
543	632	goto exit;
544	633
..	..	@@ -557,22 +646,11 @@
557	646
558	647	req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements);
559	648	pbl = &srq->hwq.pbl[PBL_LVL_0];
560		- req.pg_size_lvl = cpu_to_le16((((u16)srq->hwq.level &
561		- CMDQ_CREATE_SRQ_LVL_MASK) <<
562		- CMDQ_CREATE_SRQ_LVL_SFT) \|
563		- (pbl->pg_size == ROCE_PG_SIZE_4K ?
564		- CMDQ_CREATE_SRQ_PG_SIZE_PG_4K :
565		- pbl->pg_size == ROCE_PG_SIZE_8K ?
566		- CMDQ_CREATE_SRQ_PG_SIZE_PG_8K :
567		- pbl->pg_size == ROCE_PG_SIZE_64K ?
568		- CMDQ_CREATE_SRQ_PG_SIZE_PG_64K :
569		- pbl->pg_size == ROCE_PG_SIZE_2M ?
570		- CMDQ_CREATE_SRQ_PG_SIZE_PG_2M :
571		- pbl->pg_size == ROCE_PG_SIZE_8M ?
572		- CMDQ_CREATE_SRQ_PG_SIZE_PG_8M :
573		- pbl->pg_size == ROCE_PG_SIZE_1G ?
574		- CMDQ_CREATE_SRQ_PG_SIZE_PG_1G :
575		- CMDQ_CREATE_SRQ_PG_SIZE_PG_4K));
	649	+ pg_sz_lvl = ((u16)bnxt_qplib_base_pg_size(&srq->hwq) <<
	650	+ CMDQ_CREATE_SRQ_PG_SIZE_SFT);
	651	+ pg_sz_lvl \|= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK) <<
	652	+ CMDQ_CREATE_SRQ_LVL_SFT;
	653	+ req.pg_size_lvl = cpu_to_le16(pg_sz_lvl);
576	654	req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
577	655	req.pd_id = cpu_to_le32(srq->pd->id);
578	656	req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);
..	..	@@ -590,14 +668,18 @@
590	668	srq->swq[srq->last_idx].next_idx = -1;
591	669
592	670	srq->id = le32_to_cpu(resp.xid);
593		- srq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
	671	+ srq->dbinfo.hwq = &srq->hwq;
	672	+ srq->dbinfo.xid = srq->id;
	673	+ srq->dbinfo.db = srq->dpi->dbr;
	674	+ srq->dbinfo.max_slot = 1;
	675	+ srq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem;
594	676	if (srq->threshold)
595		- bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARMENA);
	677	+ bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA);
596	678	srq->arm_req = false;
597	679
598	680	return 0;
599	681	fail:
600		- bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
	682	+ bnxt_qplib_free_hwq(res, &srq->hwq);
601	683	kfree(srq->swq);
602	684	exit:
603	685	return rc;
..	..	@@ -616,7 +698,7 @@
616	698	srq_hwq->max_elements - sw_cons + sw_prod;
617	699	if (count > srq->threshold) {
618	700	srq->arm_req = false;
619		- bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM);
	701	+ bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold);
620	702	} else {
621	703	/* Deferred arming */
622	704	srq->arm_req = true;
..	..	@@ -657,15 +739,15 @@
657	739	struct bnxt_qplib_swqe *wqe)
658	740	{
659	741	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
660		- struct rq_wqe srqe, *srqe_ptr;
	742	+ struct rq_wqe *srqe;
661	743	struct sq_sge *hw_sge;
662	744	u32 sw_prod, sw_cons, count = 0;
663	745	int i, rc = 0, next;
664	746
665	747	spin_lock(&srq_hwq->lock);
666	748	if (srq->start_idx == srq->last_idx) {
667		- dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!",
668		- srq->id);
	749	+ dev_err(&srq_hwq->pdev->dev,
	750	+ "FP: SRQ (0x%x) is full!\n", srq->id);
669	751	rc = -EINVAL;
670	752	spin_unlock(&srq_hwq->lock);
671	753	goto done;
..	..	@@ -675,9 +757,8 @@
675	757	spin_unlock(&srq_hwq->lock);
676	758
677	759	sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
678		- srqe_ptr = (struct rq_wqe **)srq_hwq->pbl_ptr;
679		- srqe = &srqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
680		- memset(srqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
	760	+ srqe = bnxt_qplib_get_qe(srq_hwq, sw_prod, NULL);
	761	+ memset(srqe, 0, srq->wqe_size);
681	762	/* Calculate wqe_size16 and data_len */
682	763	for (i = 0, hw_sge = (struct sq_sge *)srqe->data;
683	764	i < wqe->num_sge; i++, hw_sge++) {
..	..	@@ -705,27 +786,52 @@
705	786	srq_hwq->max_elements - sw_cons + sw_prod;
706	787	spin_unlock(&srq_hwq->lock);
707	788	/* Ring DB */
708		- bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ);
	789	+ bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ);
709	790	if (srq->arm_req == true && count > srq->threshold) {
710	791	srq->arm_req = false;
711		- bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM);
	792	+ bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold);
712	793	}
713	794	done:
714	795	return rc;
715	796	}
716	797
717	798	/* QP */
	799	+
	800	+static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que)
	801	+{
	802	+ int rc = 0;
	803	+ int indx;
	804	+
	805	+ que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL);
	806	+ if (!que->swq) {
	807	+ rc = -ENOMEM;
	808	+ goto out;
	809	+ }
	810	+
	811	+ que->swq_start = 0;
	812	+ que->swq_last = que->max_wqe - 1;
	813	+ for (indx = 0; indx < que->max_wqe; indx++)
	814	+ que->swq[indx].next_idx = indx + 1;
	815	+ que->swq[que->swq_last].next_idx = 0; /* Make it circular */
	816	+ que->swq_last = 0;
	817	+out:
	818	+ return rc;
	819	+}
	820	+
718	821	int bnxt_qplib_create_qp1(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp)
719	822	{
	823	+ struct bnxt_qplib_hwq_attr hwq_attr = {};
720	824	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
721		- struct cmdq_create_qp1 req;
722		- struct creq_create_qp1_resp resp;
723		- struct bnxt_qplib_pbl *pbl;
724	825	struct bnxt_qplib_q *sq = &qp->sq;
725	826	struct bnxt_qplib_q *rq = &qp->rq;
726		- int rc;
	827	+ struct creq_create_qp1_resp resp;
	828	+ struct cmdq_create_qp1 req;
	829	+ struct bnxt_qplib_pbl *pbl;
727	830	u16 cmd_flags = 0;
728	831	u32 qp_flags = 0;
	832	+ u8 pg_sz_lvl;
	833	+ u32 tbl_indx;
	834	+ int rc;
729	835
730	836	RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
731	837
..	..	@@ -735,98 +841,65 @@
735	841	req.qp_handle = cpu_to_le64(qp->qp_handle);
736	842
737	843	/* SQ */
738		- sq->hwq.max_elements = sq->max_wqe;
739		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0,
740		- &sq->hwq.max_elements,
741		- BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,
742		- PAGE_SIZE, HWQ_TYPE_QUEUE);
	844	+ hwq_attr.res = res;
	845	+ hwq_attr.sginfo = &sq->sg_info;
	846	+ hwq_attr.stride = sizeof(struct sq_sge);
	847	+ hwq_attr.depth = bnxt_qplib_get_depth(sq);
	848	+ hwq_attr.type = HWQ_TYPE_QUEUE;
	849	+ rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
743	850	if (rc)
744	851	goto exit;
745	852
746		- sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
747		- if (!sq->swq) {
748		- rc = -ENOMEM;
	853	+ rc = bnxt_qplib_alloc_init_swq(sq);
	854	+ if (rc)
749	855	goto fail_sq;
750		- }
	856	+
	857	+ req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode));
751	858	pbl = &sq->hwq.pbl[PBL_LVL_0];
752	859	req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
753		- req.sq_pg_size_sq_lvl =
754		- ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK)
755		- << CMDQ_CREATE_QP1_SQ_LVL_SFT) \|
756		- (pbl->pg_size == ROCE_PG_SIZE_4K ?
757		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K :
758		- pbl->pg_size == ROCE_PG_SIZE_8K ?
759		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K :
760		- pbl->pg_size == ROCE_PG_SIZE_64K ?
761		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K :
762		- pbl->pg_size == ROCE_PG_SIZE_2M ?
763		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M :
764		- pbl->pg_size == ROCE_PG_SIZE_8M ?
765		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M :
766		- pbl->pg_size == ROCE_PG_SIZE_1G ?
767		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G :
768		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K);
769		-
770		- if (qp->scq)
771		- req.scq_cid = cpu_to_le32(qp->scq->id);
772		-
773		- qp_flags \|= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
	860	+ pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
	861	+ CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT);
	862	+ pg_sz_lvl \|= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK);
	863	+ req.sq_pg_size_sq_lvl = pg_sz_lvl;
	864	+ req.sq_fwo_sq_sge =
	865	+ cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
	866	+ CMDQ_CREATE_QP1_SQ_SGE_SFT);
	867	+ req.scq_cid = cpu_to_le32(qp->scq->id);
774	868
775	869	/* RQ */
776	870	if (rq->max_wqe) {
777		- rq->hwq.max_elements = qp->rq.max_wqe;
778		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0,
779		- &rq->hwq.max_elements,
780		- BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
781		- PAGE_SIZE, HWQ_TYPE_QUEUE);
	871	+ hwq_attr.res = res;
	872	+ hwq_attr.sginfo = &rq->sg_info;
	873	+ hwq_attr.stride = sizeof(struct sq_sge);
	874	+ hwq_attr.depth = bnxt_qplib_get_depth(rq);
	875	+ hwq_attr.type = HWQ_TYPE_QUEUE;
	876	+ rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
782	877	if (rc)
783		- goto fail_sq;
784		-
785		- rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
786		- GFP_KERNEL);
787		- if (!rq->swq) {
788		- rc = -ENOMEM;
	878	+ goto sq_swq;
	879	+ rc = bnxt_qplib_alloc_init_swq(rq);
	880	+ if (rc)
789	881	goto fail_rq;
790		- }
	882	+ req.rq_size = cpu_to_le32(rq->max_wqe);
791	883	pbl = &rq->hwq.pbl[PBL_LVL_0];
792	884	req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
793		- req.rq_pg_size_rq_lvl =
794		- ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
795		- CMDQ_CREATE_QP1_RQ_LVL_SFT) \|
796		- (pbl->pg_size == ROCE_PG_SIZE_4K ?
797		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K :
798		- pbl->pg_size == ROCE_PG_SIZE_8K ?
799		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K :
800		- pbl->pg_size == ROCE_PG_SIZE_64K ?
801		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K :
802		- pbl->pg_size == ROCE_PG_SIZE_2M ?
803		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M :
804		- pbl->pg_size == ROCE_PG_SIZE_8M ?
805		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M :
806		- pbl->pg_size == ROCE_PG_SIZE_1G ?
807		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G :
808		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K);
809		- if (qp->rcq)
810		- req.rcq_cid = cpu_to_le32(qp->rcq->id);
	885	+ pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
	886	+ CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT);
	887	+ pg_sz_lvl \|= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK);
	888	+ req.rq_pg_size_rq_lvl = pg_sz_lvl;
	889	+ req.rq_fwo_rq_sge =
	890	+ cpu_to_le16((rq->max_sge &
	891	+ CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
	892	+ CMDQ_CREATE_QP1_RQ_SGE_SFT);
811	893	}
812		-
	894	+ req.rcq_cid = cpu_to_le32(qp->rcq->id);
813	895	/* Header buffer - allow hdr_buf pass in */
814	896	rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
815	897	if (rc) {
816	898	rc = -ENOMEM;
817		- goto fail;
	899	+ goto rq_rwq;
818	900	}
	901	+ qp_flags \|= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
819	902	req.qp_flags = cpu_to_le32(qp_flags);
820		- req.sq_size = cpu_to_le32(sq->hwq.max_elements);
821		- req.rq_size = cpu_to_le32(rq->hwq.max_elements);
822		-
823		- req.sq_fwo_sq_sge =
824		- cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
825		- CMDQ_CREATE_QP1_SQ_SGE_SFT);
826		- req.rq_fwo_rq_sge =
827		- cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
828		- CMDQ_CREATE_QP1_RQ_SGE_SFT);
829		-
830	903	req.pd_id = cpu_to_le32(qp->pd->id);
831	904
832	905	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
..	..	@@ -836,38 +909,72 @@
836	909
837	910	qp->id = le32_to_cpu(resp.xid);
838	911	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
839		- rcfw->qp_tbl[qp->id].qp_id = qp->id;
840		- rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
	912	+ qp->cctx = res->cctx;
	913	+ sq->dbinfo.hwq = &sq->hwq;
	914	+ sq->dbinfo.xid = qp->id;
	915	+ sq->dbinfo.db = qp->dpi->dbr;
	916	+ sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode);
	917	+ if (rq->max_wqe) {
	918	+ rq->dbinfo.hwq = &rq->hwq;
	919	+ rq->dbinfo.xid = qp->id;
	920	+ rq->dbinfo.db = qp->dpi->dbr;
	921	+ rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size);
	922	+ }
	923	+ tbl_indx = map_qp_id_to_tbl_indx(qp->id, rcfw);
	924	+ rcfw->qp_tbl[tbl_indx].qp_id = qp->id;
	925	+ rcfw->qp_tbl[tbl_indx].qp_handle = (void *)qp;
841	926
842	927	return 0;
843	928
844	929	fail:
845	930	bnxt_qplib_free_qp_hdr_buf(res, qp);
846		-fail_rq:
847		- bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
	931	+rq_rwq:
848	932	kfree(rq->swq);
849		-fail_sq:
850		- bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
	933	+fail_rq:
	934	+ bnxt_qplib_free_hwq(res, &rq->hwq);
	935	+sq_swq:
851	936	kfree(sq->swq);
	937	+fail_sq:
	938	+ bnxt_qplib_free_hwq(res, &sq->hwq);
852	939	exit:
853	940	return rc;
	941	+}
	942	+
	943	+static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size)
	944	+{
	945	+ struct bnxt_qplib_hwq *hwq;
	946	+ struct bnxt_qplib_q *sq;
	947	+ u64 fpsne, psn_pg;
	948	+ u16 indx_pad = 0;
	949	+
	950	+ sq = &qp->sq;
	951	+ hwq = &sq->hwq;
	952	+ /* First psn entry */
	953	+ fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->depth, &psn_pg);
	954	+ if (!IS_ALIGNED(fpsne, PAGE_SIZE))
	955	+ indx_pad = (fpsne & ~PAGE_MASK) / size;
	956	+ hwq->pad_pgofft = indx_pad;
	957	+ hwq->pad_pg = (u64 *)psn_pg;
	958	+ hwq->pad_stride = size;
854	959	}
855	960
856	961	int bnxt_qplib_create_qp(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp)
857	962	{
858	963	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
859		- struct sq_send hw_sq_send_hdr, *hw_sq_send_ptr;
860		- struct cmdq_create_qp req;
861		- struct creq_create_qp_resp resp;
862		- struct bnxt_qplib_pbl *pbl;
863		- struct sq_psn_search **psn_search_ptr;
864		- unsigned long int psn_search, poff = 0;
	964	+ struct bnxt_qplib_hwq_attr hwq_attr = {};
	965	+ struct bnxt_qplib_sg_info sginfo = {};
865	966	struct bnxt_qplib_q *sq = &qp->sq;
866	967	struct bnxt_qplib_q *rq = &qp->rq;
	968	+ struct creq_create_qp_resp resp;
	969	+ int rc, req_size, psn_sz = 0;
867	970	struct bnxt_qplib_hwq *xrrq;
868		- int i, rc, req_size, psn_sz;
869		- u16 cmd_flags = 0, max_ssge;
870		- u32 sw_prod, qp_flags = 0;
	971	+ struct bnxt_qplib_pbl *pbl;
	972	+ struct cmdq_create_qp req;
	973	+ u16 cmd_flags = 0;
	974	+ u32 qp_flags = 0;
	975	+ u8 pg_sz_lvl;
	976	+ u32 tbl_indx;
	977	+ u16 nsge;
871	978
872	979	RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
873	980
..	..	@@ -877,142 +984,86 @@
877	984	req.qp_handle = cpu_to_le64(qp->qp_handle);
878	985
879	986	/* SQ */
880		- psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ?
881		- sizeof(struct sq_psn_search) : 0;
882		- sq->hwq.max_elements = sq->max_wqe;
883		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist,
884		- sq->nmap, &sq->hwq.max_elements,
885		- BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,
886		- psn_sz,
887		- PAGE_SIZE, HWQ_TYPE_QUEUE);
	987	+ if (qp->type == CMDQ_CREATE_QP_TYPE_RC) {
	988	+ psn_sz = bnxt_qplib_is_chip_gen_p5(res->cctx) ?
	989	+ sizeof(struct sq_psn_search_ext) :
	990	+ sizeof(struct sq_psn_search);
	991	+ }
	992	+
	993	+ hwq_attr.res = res;
	994	+ hwq_attr.sginfo = &sq->sg_info;
	995	+ hwq_attr.stride = sizeof(struct sq_sge);
	996	+ hwq_attr.depth = bnxt_qplib_get_depth(sq);
	997	+ hwq_attr.aux_stride = psn_sz;
	998	+ hwq_attr.aux_depth = bnxt_qplib_set_sq_size(sq, qp->wqe_mode);
	999	+ hwq_attr.type = HWQ_TYPE_QUEUE;
	1000	+ rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
888	1001	if (rc)
889	1002	goto exit;
890	1003
891		- sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
892		- if (!sq->swq) {
893		- rc = -ENOMEM;
	1004	+ rc = bnxt_qplib_alloc_init_swq(sq);
	1005	+ if (rc)
894	1006	goto fail_sq;
895		- }
896		- hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
897		- if (psn_sz) {
898		- psn_search_ptr = (struct sq_psn_search **)
899		- &hw_sq_send_ptr[get_sqe_pg
900		- (sq->hwq.max_elements)];
901		- psn_search = (unsigned long int)
902		- &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)]
903		- [get_sqe_idx(sq->hwq.max_elements)];
904		- if (psn_search & ~PAGE_MASK) {
905		- /* If the psn_search does not start on a page boundary,
906		- * then calculate the offset
907		- */
908		- poff = (psn_search & ~PAGE_MASK) /
909		- BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;
910		- }
911		- for (i = 0; i < sq->hwq.max_elements; i++)
912		- sq->swq[i].psn_search =
913		- &psn_search_ptr[get_psne_pg(i + poff)]
914		- [get_psne_idx(i + poff)];
915		- }
	1007	+
	1008	+ if (psn_sz)
	1009	+ bnxt_qplib_init_psn_ptr(qp, psn_sz);
	1010	+
	1011	+ req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode));
916	1012	pbl = &sq->hwq.pbl[PBL_LVL_0];
917	1013	req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
918		- req.sq_pg_size_sq_lvl =
919		- ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK)
920		- << CMDQ_CREATE_QP_SQ_LVL_SFT) \|
921		- (pbl->pg_size == ROCE_PG_SIZE_4K ?
922		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K :
923		- pbl->pg_size == ROCE_PG_SIZE_8K ?
924		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K :
925		- pbl->pg_size == ROCE_PG_SIZE_64K ?
926		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K :
927		- pbl->pg_size == ROCE_PG_SIZE_2M ?
928		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M :
929		- pbl->pg_size == ROCE_PG_SIZE_8M ?
930		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M :
931		- pbl->pg_size == ROCE_PG_SIZE_1G ?
932		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G :
933		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K);
	1014	+ pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
	1015	+ CMDQ_CREATE_QP_SQ_PG_SIZE_SFT);
	1016	+ pg_sz_lvl \|= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK);
	1017	+ req.sq_pg_size_sq_lvl = pg_sz_lvl;
	1018	+ req.sq_fwo_sq_sge =
	1019	+ cpu_to_le16(((sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK) <<
	1020	+ CMDQ_CREATE_QP_SQ_SGE_SFT) \| 0);
	1021	+ req.scq_cid = cpu_to_le32(qp->scq->id);
934	1022
935		- /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */
936		- hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
937		- for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) {
938		- hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
939		- [get_sqe_idx(sw_prod)];
940		- hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID;
	1023	+ /* RQ */
	1024	+ if (!qp->srq) {
	1025	+ hwq_attr.res = res;
	1026	+ hwq_attr.sginfo = &rq->sg_info;
	1027	+ hwq_attr.stride = sizeof(struct sq_sge);
	1028	+ hwq_attr.depth = bnxt_qplib_get_depth(rq);
	1029	+ hwq_attr.aux_stride = 0;
	1030	+ hwq_attr.aux_depth = 0;
	1031	+ hwq_attr.type = HWQ_TYPE_QUEUE;
	1032	+ rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
	1033	+ if (rc)
	1034	+ goto sq_swq;
	1035	+ rc = bnxt_qplib_alloc_init_swq(rq);
	1036	+ if (rc)
	1037	+ goto fail_rq;
	1038	+
	1039	+ req.rq_size = cpu_to_le32(rq->max_wqe);
	1040	+ pbl = &rq->hwq.pbl[PBL_LVL_0];
	1041	+ req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
	1042	+ pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
	1043	+ CMDQ_CREATE_QP_RQ_PG_SIZE_SFT);
	1044	+ pg_sz_lvl \|= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK);
	1045	+ req.rq_pg_size_rq_lvl = pg_sz_lvl;
	1046	+ nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
	1047	+ 6 : rq->max_sge;
	1048	+ req.rq_fwo_rq_sge =
	1049	+ cpu_to_le16(((nsge &
	1050	+ CMDQ_CREATE_QP_RQ_SGE_MASK) <<
	1051	+ CMDQ_CREATE_QP_RQ_SGE_SFT) \| 0);
	1052	+ } else {
	1053	+ /* SRQ */
	1054	+ qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
	1055	+ req.srq_cid = cpu_to_le32(qp->srq->id);
941	1056	}
942		-
943		- if (qp->scq)
944		- req.scq_cid = cpu_to_le32(qp->scq->id);
	1057	+ req.rcq_cid = cpu_to_le32(qp->rcq->id);
945	1058
946	1059	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
947	1060	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
948	1061	if (qp->sig_type)
949	1062	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
950		-
951		- /* RQ */
952		- if (rq->max_wqe) {
953		- rq->hwq.max_elements = rq->max_wqe;
954		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist,
955		- rq->nmap, &rq->hwq.max_elements,
956		- BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
957		- PAGE_SIZE, HWQ_TYPE_QUEUE);
958		- if (rc)
959		- goto fail_sq;
960		-
961		- rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
962		- GFP_KERNEL);
963		- if (!rq->swq) {
964		- rc = -ENOMEM;
965		- goto fail_rq;
966		- }
967		- pbl = &rq->hwq.pbl[PBL_LVL_0];
968		- req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
969		- req.rq_pg_size_rq_lvl =
970		- ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
971		- CMDQ_CREATE_QP_RQ_LVL_SFT) \|
972		- (pbl->pg_size == ROCE_PG_SIZE_4K ?
973		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K :
974		- pbl->pg_size == ROCE_PG_SIZE_8K ?
975		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K :
976		- pbl->pg_size == ROCE_PG_SIZE_64K ?
977		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K :
978		- pbl->pg_size == ROCE_PG_SIZE_2M ?
979		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M :
980		- pbl->pg_size == ROCE_PG_SIZE_8M ?
981		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M :
982		- pbl->pg_size == ROCE_PG_SIZE_1G ?
983		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :
984		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);
985		- } else {
986		- /* SRQ */
987		- if (qp->srq) {
988		- qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
989		- req.srq_cid = cpu_to_le32(qp->srq->id);
990		- }
991		- }
992		-
993		- if (qp->rcq)
994		- req.rcq_cid = cpu_to_le32(qp->rcq->id);
	1063	+ if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
	1064	+ qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED;
995	1065	req.qp_flags = cpu_to_le32(qp_flags);
996		- req.sq_size = cpu_to_le32(sq->hwq.max_elements);
997		- req.rq_size = cpu_to_le32(rq->hwq.max_elements);
998		- qp->sq_hdr_buf = NULL;
999		- qp->rq_hdr_buf = NULL;
1000	1066
1001		- rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
1002		- if (rc)
1003		- goto fail_rq;
1004		-
1005		- /* CTRL-22434: Irrespective of the requested SGE count on the SQ
1006		- * always create the QP with max send sges possible if the requested
1007		- * inline size is greater than 0.
1008		- */
1009		- max_ssge = qp->max_inline_data ? 6 : sq->max_sge;
1010		- req.sq_fwo_sq_sge = cpu_to_le16(
1011		- ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
1012		- << CMDQ_CREATE_QP_SQ_SGE_SFT) \| 0);
1013		- req.rq_fwo_rq_sge = cpu_to_le16(
1014		- ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK)
1015		- << CMDQ_CREATE_QP_RQ_SGE_SFT) \| 0);
1016	1067	/* ORRQ and IRRQ */
1017	1068	if (psn_sz) {
1018	1069	xrrq = &qp->orrq;
..	..	@@ -1021,12 +1072,19 @@
1021	1072	req_size = xrrq->max_elements *
1022	1073	BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
1023	1074	req_size &= ~(PAGE_SIZE - 1);
1024		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
1025		- &xrrq->max_elements,
1026		- BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,
1027		- 0, req_size, HWQ_TYPE_CTX);
	1075	+ sginfo.pgsize = req_size;
	1076	+ sginfo.pgshft = PAGE_SHIFT;
	1077	+
	1078	+ hwq_attr.res = res;
	1079	+ hwq_attr.sginfo = &sginfo;
	1080	+ hwq_attr.depth = xrrq->max_elements;
	1081	+ hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE;
	1082	+ hwq_attr.aux_stride = 0;
	1083	+ hwq_attr.aux_depth = 0;
	1084	+ hwq_attr.type = HWQ_TYPE_CTX;
	1085	+ rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
1028	1086	if (rc)
1029		- goto fail_buf_free;
	1087	+ goto rq_swq;
1030	1088	pbl = &xrrq->pbl[PBL_LVL_0];
1031	1089	req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
1032	1090
..	..	@@ -1036,11 +1094,10 @@
1036	1094	req_size = xrrq->max_elements *
1037	1095	BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
1038	1096	req_size &= ~(PAGE_SIZE - 1);
1039		-
1040		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
1041		- &xrrq->max_elements,
1042		- BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,
1043		- 0, req_size, HWQ_TYPE_CTX);
	1097	+ sginfo.pgsize = req_size;
	1098	+ hwq_attr.depth = xrrq->max_elements;
	1099	+ hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE;
	1100	+ rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
1044	1101	if (rc)
1045	1102	goto fail_orrq;
1046	1103
..	..	@@ -1058,25 +1115,34 @@
1058	1115	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
1059	1116	INIT_LIST_HEAD(&qp->sq_flush);
1060	1117	INIT_LIST_HEAD(&qp->rq_flush);
1061		- rcfw->qp_tbl[qp->id].qp_id = qp->id;
1062		- rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
	1118	+ qp->cctx = res->cctx;
	1119	+ sq->dbinfo.hwq = &sq->hwq;
	1120	+ sq->dbinfo.xid = qp->id;
	1121	+ sq->dbinfo.db = qp->dpi->dbr;
	1122	+ sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode);
	1123	+ if (rq->max_wqe) {
	1124	+ rq->dbinfo.hwq = &rq->hwq;
	1125	+ rq->dbinfo.xid = qp->id;
	1126	+ rq->dbinfo.db = qp->dpi->dbr;
	1127	+ rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size);
	1128	+ }
	1129	+ tbl_indx = map_qp_id_to_tbl_indx(qp->id, rcfw);
	1130	+ rcfw->qp_tbl[tbl_indx].qp_id = qp->id;
	1131	+ rcfw->qp_tbl[tbl_indx].qp_handle = (void *)qp;
1063	1132
1064	1133	return 0;
1065		-
1066	1134	fail:
1067		- if (qp->irrq.max_elements)
1068		- bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
	1135	+ bnxt_qplib_free_hwq(res, &qp->irrq);
1069	1136	fail_orrq:
1070		- if (qp->orrq.max_elements)
1071		- bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
1072		-fail_buf_free:
1073		- bnxt_qplib_free_qp_hdr_buf(res, qp);
1074		-fail_rq:
1075		- bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
	1137	+ bnxt_qplib_free_hwq(res, &qp->orrq);
	1138	+rq_swq:
1076	1139	kfree(rq->swq);
1077		-fail_sq:
1078		- bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
	1140	+fail_rq:
	1141	+ bnxt_qplib_free_hwq(res, &rq->hwq);
	1142	+sq_swq:
1079	1143	kfree(sq->swq);
	1144	+fail_sq:
	1145	+ bnxt_qplib_free_hwq(res, &sq->hwq);
1080	1146	exit:
1081	1147	return rc;
1082	1148	}
..	..	@@ -1326,7 +1392,7 @@
1326	1392	}
1327	1393	}
1328	1394	if (i == res->sgid_tbl.max)
1329		- dev_warn(&res->pdev->dev, "QPLIB: SGID not found??");
	1395	+ dev_warn(&res->pdev->dev, "SGID not found??\n");
1330	1396
1331	1397	qp->ah.hop_limit = sb->hop_limit;
1332	1398	qp->ah.traffic_class = sb->traffic_class;
..	..	@@ -1362,12 +1428,11 @@
1362	1428	static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
1363	1429	{
1364	1430	struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1365		- struct cq_base hw_cqe, *hw_cqe_ptr;
	1431	+ struct cq_base *hw_cqe;
1366	1432	int i;
1367	1433
1368	1434	for (i = 0; i < cq_hwq->max_elements; i++) {
1369		- hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr;
1370		- hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)];
	1435	+ hw_cqe = bnxt_qplib_get_qe(cq_hwq, i, NULL);
1371	1436	if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
1372	1437	continue;
1373	1438	/*
..	..	@@ -1408,10 +1473,12 @@
1408	1473	struct cmdq_destroy_qp req;
1409	1474	struct creq_destroy_qp_resp resp;
1410	1475	u16 cmd_flags = 0;
	1476	+ u32 tbl_indx;
1411	1477	int rc;
1412	1478
1413		- rcfw->qp_tbl[qp->id].qp_id = BNXT_QPLIB_QP_ID_INVALID;
1414		- rcfw->qp_tbl[qp->id].qp_handle = NULL;
	1479	+ tbl_indx = map_qp_id_to_tbl_indx(qp->id, rcfw);
	1480	+ rcfw->qp_tbl[tbl_indx].qp_id = BNXT_QPLIB_QP_ID_INVALID;
	1481	+ rcfw->qp_tbl[tbl_indx].qp_handle = NULL;
1415	1482
1416	1483	RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
1417	1484
..	..	@@ -1419,8 +1486,8 @@
1419	1486	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1420	1487	(void *)&resp, NULL, 0);
1421	1488	if (rc) {
1422		- rcfw->qp_tbl[qp->id].qp_id = qp->id;
1423		- rcfw->qp_tbl[qp->id].qp_handle = qp;
	1489	+ rcfw->qp_tbl[tbl_indx].qp_id = qp->id;
	1490	+ rcfw->qp_tbl[tbl_indx].qp_handle = qp;
1424	1491	return rc;
1425	1492	}
1426	1493
..	..	@@ -1431,16 +1498,16 @@
1431	1498	struct bnxt_qplib_qp *qp)
1432	1499	{
1433	1500	bnxt_qplib_free_qp_hdr_buf(res, qp);
1434		- bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
	1501	+ bnxt_qplib_free_hwq(res, &qp->sq.hwq);
1435	1502	kfree(qp->sq.swq);
1436	1503
1437		- bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq);
	1504	+ bnxt_qplib_free_hwq(res, &qp->rq.hwq);
1438	1505	kfree(qp->rq.swq);
1439	1506
1440	1507	if (qp->irrq.max_elements)
1441		- bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
	1508	+ bnxt_qplib_free_hwq(res, &qp->irrq);
1442	1509	if (qp->orrq.max_elements)
1443		- bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
	1510	+ bnxt_qplib_free_hwq(res, &qp->orrq);
1444	1511
1445	1512	}
1446	1513
..	..	@@ -1453,7 +1520,7 @@
1453	1520	memset(sge, 0, sizeof(*sge));
1454	1521
1455	1522	if (qp->sq_hdr_buf) {
1456		- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
	1523	+ sw_prod = sq->swq_start;
1457	1524	sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
1458	1525	sw_prod * qp->sq_hdr_buf_size);
1459	1526	sge->lkey = 0xFFFFFFFF;
..	..	@@ -1467,7 +1534,7 @@
1467	1534	{
1468	1535	struct bnxt_qplib_q *rq = &qp->rq;
1469	1536
1470		- return HWQ_CMP(rq->hwq.prod, &rq->hwq);
	1537	+ return rq->swq_start;
1471	1538	}
1472	1539
1473	1540	dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
..	..	@@ -1484,7 +1551,7 @@
1484	1551	memset(sge, 0, sizeof(*sge));
1485	1552
1486	1553	if (qp->rq_hdr_buf) {
1487		- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
	1554	+ sw_prod = rq->swq_start;
1488	1555	sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
1489	1556	sw_prod * qp->rq_hdr_buf_size);
1490	1557	sge->lkey = 0xFFFFFFFF;
..	..	@@ -1494,144 +1561,262 @@
1494	1561	return NULL;
1495	1562	}
1496	1563
	1564	+static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp,
	1565	+ struct bnxt_qplib_swqe *wqe,
	1566	+ struct bnxt_qplib_swq *swq)
	1567	+{
	1568	+ struct sq_psn_search_ext *psns_ext;
	1569	+ struct sq_psn_search *psns;
	1570	+ u32 flg_npsn;
	1571	+ u32 op_spsn;
	1572	+
	1573	+ if (!swq->psn_search)
	1574	+ return;
	1575	+ psns = swq->psn_search;
	1576	+ psns_ext = swq->psn_ext;
	1577	+
	1578	+ op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
	1579	+ SQ_PSN_SEARCH_START_PSN_MASK);
	1580	+ op_spsn \|= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
	1581	+ SQ_PSN_SEARCH_OPCODE_MASK);
	1582	+ flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
	1583	+ SQ_PSN_SEARCH_NEXT_PSN_MASK);
	1584	+
	1585	+ if (bnxt_qplib_is_chip_gen_p5(qp->cctx)) {
	1586	+ psns_ext->opcode_start_psn = cpu_to_le32(op_spsn);
	1587	+ psns_ext->flags_next_psn = cpu_to_le32(flg_npsn);
	1588	+ psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx);
	1589	+ } else {
	1590	+ psns->opcode_start_psn = cpu_to_le32(op_spsn);
	1591	+ psns->flags_next_psn = cpu_to_le32(flg_npsn);
	1592	+ }
	1593	+}
	1594	+
	1595	+static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp,
	1596	+ struct bnxt_qplib_swqe *wqe,
	1597	+ u16 *idx)
	1598	+{
	1599	+ struct bnxt_qplib_hwq *hwq;
	1600	+ int len, t_len, offt;
	1601	+ bool pull_dst = true;
	1602	+ void *il_dst = NULL;
	1603	+ void *il_src = NULL;
	1604	+ int t_cplen, cplen;
	1605	+ int indx;
	1606	+
	1607	+ hwq = &qp->sq.hwq;
	1608	+ t_len = 0;
	1609	+ for (indx = 0; indx < wqe->num_sge; indx++) {
	1610	+ len = wqe->sg_list[indx].size;
	1611	+ il_src = (void *)wqe->sg_list[indx].addr;
	1612	+ t_len += len;
	1613	+ if (t_len > qp->max_inline_data)
	1614	+ return -ENOMEM;
	1615	+ while (len) {
	1616	+ if (pull_dst) {
	1617	+ pull_dst = false;
	1618	+ il_dst = bnxt_qplib_get_prod_qe(hwq, *idx);
	1619	+ (*idx)++;
	1620	+ t_cplen = 0;
	1621	+ offt = 0;
	1622	+ }
	1623	+ cplen = min_t(int, len, sizeof(struct sq_sge));
	1624	+ cplen = min_t(int, cplen,
	1625	+ (sizeof(struct sq_sge) - offt));
	1626	+ memcpy(il_dst, il_src, cplen);
	1627	+ t_cplen += cplen;
	1628	+ il_src += cplen;
	1629	+ il_dst += cplen;
	1630	+ offt += cplen;
	1631	+ len -= cplen;
	1632	+ if (t_cplen == sizeof(struct sq_sge))
	1633	+ pull_dst = true;
	1634	+ }
	1635	+ }
	1636	+
	1637	+ return t_len;
	1638	+}
	1639	+
	1640	+static u32 bnxt_qplib_put_sges(struct bnxt_qplib_hwq *hwq,
	1641	+ struct bnxt_qplib_sge *ssge,
	1642	+ u16 nsge, u16 *idx)
	1643	+{
	1644	+ struct sq_sge *dsge;
	1645	+ int indx, len = 0;
	1646	+
	1647	+ for (indx = 0; indx < nsge; indx++, (*idx)++) {
	1648	+ dsge = bnxt_qplib_get_prod_qe(hwq, *idx);
	1649	+ dsge->va_or_pa = cpu_to_le64(ssge[indx].addr);
	1650	+ dsge->l_key = cpu_to_le32(ssge[indx].lkey);
	1651	+ dsge->size = cpu_to_le32(ssge[indx].size);
	1652	+ len += ssge[indx].size;
	1653	+ }
	1654	+
	1655	+ return len;
	1656	+}
	1657	+
	1658	+static u16 bnxt_qplib_required_slots(struct bnxt_qplib_qp *qp,
	1659	+ struct bnxt_qplib_swqe *wqe,
	1660	+ u16 wqe_sz, u16 qdf, u8 mode)
	1661	+{
	1662	+ u32 ilsize, bytes;
	1663	+ u16 nsge;
	1664	+ u16 slot;
	1665	+
	1666	+ nsge = wqe->num_sge;
	1667	+ /* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */
	1668	+ bytes = sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge);
	1669	+ if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
	1670	+ ilsize = bnxt_qplib_calc_ilsize(wqe, qp->max_inline_data);
	1671	+ bytes = ALIGN(ilsize, sizeof(struct sq_sge));
	1672	+ bytes += sizeof(struct sq_send_hdr);
	1673	+ }
	1674	+
	1675	+ *qdf = __xlate_qfd(qp->sq.q_full_delta, bytes);
	1676	+ slot = bytes >> 4;
	1677	+ *wqe_sz = slot;
	1678	+ if (mode == BNXT_QPLIB_WQE_MODE_STATIC)
	1679	+ slot = 8;
	1680	+ return slot;
	1681	+}
	1682	+
	1683	+static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_q *sq,
	1684	+ struct bnxt_qplib_swq *swq)
	1685	+{
	1686	+ struct bnxt_qplib_hwq *hwq;
	1687	+ u32 pg_num, pg_indx;
	1688	+ void *buff;
	1689	+ u32 tail;
	1690	+
	1691	+ hwq = &sq->hwq;
	1692	+ if (!hwq->pad_pg)
	1693	+ return;
	1694	+ tail = swq->slot_idx / sq->dbinfo.max_slot;
	1695	+ pg_num = (tail + hwq->pad_pgofft) / (PAGE_SIZE / hwq->pad_stride);
	1696	+ pg_indx = (tail + hwq->pad_pgofft) % (PAGE_SIZE / hwq->pad_stride);
	1697	+ buff = (void )(hwq->pad_pg[pg_num] + pg_indx hwq->pad_stride);
	1698	+ swq->psn_ext = buff;
	1699	+ swq->psn_search = buff;
	1700	+}
	1701	+
1497	1702	void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1498	1703	{
1499	1704	struct bnxt_qplib_q *sq = &qp->sq;
1500		- struct dbr_dbr db_msg = { 0 };
1501		- u32 sw_prod;
1502	1705
1503		- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1504		-
1505		- db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1506		- DBR_DBR_INDEX_MASK);
1507		- db_msg.type_xid =
1508		- cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) \|
1509		- DBR_DBR_TYPE_SQ);
1510		- /* Flush all the WQE writes to HW */
1511		- wmb();
1512		- __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
	1706	+ bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ);
1513	1707	}
1514	1708
1515	1709	int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
1516	1710	struct bnxt_qplib_swqe *wqe)
1517	1711	{
1518		- struct bnxt_qplib_q *sq = &qp->sq;
1519		- struct bnxt_qplib_swq *swq;
1520		- struct sq_send hw_sq_send_hdr, *hw_sq_send_ptr;
1521		- struct sq_sge *hw_sge;
1522	1712	struct bnxt_qplib_nq_work *nq_work = NULL;
1523		- bool sch_handler = false;
1524		- u32 sw_prod;
1525		- u8 wqe_size16;
1526	1713	int i, rc = 0, data_len = 0, pkt_num = 0;
	1714	+ struct bnxt_qplib_q *sq = &qp->sq;
	1715	+ struct bnxt_qplib_hwq *hwq;
	1716	+ struct bnxt_qplib_swq *swq;
	1717	+ bool sch_handler = false;
	1718	+ u16 wqe_sz, qdf = 0;
	1719	+ void *base_hdr;
	1720	+ void *ext_hdr;
1527	1721	__le32 temp32;
	1722	+ u32 wqe_idx;
	1723	+ u32 slots;
	1724	+ u16 idx;
1528	1725
1529		- if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
1530		- if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1531		- sch_handler = true;
1532		- dev_dbg(&sq->hwq.pdev->dev,
1533		- "%s Error QP. Scheduling for poll_cq\n",
1534		- __func__);
1535		- goto queue_err;
1536		- }
	1726	+ hwq = &sq->hwq;
	1727	+ if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS &&
	1728	+ qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) {
	1729	+ dev_err(&hwq->pdev->dev,
	1730	+ "QPLIB: FP: QP (0x%x) is in the 0x%x state",
	1731	+ qp->id, qp->state);
	1732	+ rc = -EINVAL;
	1733	+ goto done;
1537	1734	}
1538	1735
1539		- if (bnxt_qplib_queue_full(sq)) {
1540		- dev_err(&sq->hwq.pdev->dev,
1541		- "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x",
1542		- sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements,
1543		- sq->q_full_delta);
	1736	+ slots = bnxt_qplib_required_slots(qp, wqe, &wqe_sz, &qdf, qp->wqe_mode);
	1737	+ if (bnxt_qplib_queue_full(sq, slots + qdf)) {
	1738	+ dev_err(&hwq->pdev->dev,
	1739	+ "prod = %#x cons = %#x qdepth = %#x delta = %#x\n",
	1740	+ hwq->prod, hwq->cons, hwq->depth, sq->q_full_delta);
1544	1741	rc = -ENOMEM;
1545	1742	goto done;
1546	1743	}
1547		- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1548		- swq = &sq->swq[sw_prod];
	1744	+
	1745	+ swq = bnxt_qplib_get_swqe(sq, &wqe_idx);
	1746	+ bnxt_qplib_pull_psn_buff(sq, swq);
	1747	+
	1748	+ idx = 0;
	1749	+ swq->slot_idx = hwq->prod;
	1750	+ swq->slots = slots;
1549	1751	swq->wr_id = wqe->wr_id;
1550	1752	swq->type = wqe->type;
1551	1753	swq->flags = wqe->flags;
	1754	+ swq->start_psn = sq->psn & BTH_PSN_MASK;
1552	1755	if (qp->sig_type)
1553	1756	swq->flags \|= SQ_SEND_FLAGS_SIGNAL_COMP;
1554		- swq->start_psn = sq->psn & BTH_PSN_MASK;
1555	1757
1556		- hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
1557		- hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
1558		- [get_sqe_idx(sw_prod)];
1559		-
1560		- memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
1561		-
1562		- if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
1563		- /* Copy the inline data */
1564		- if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1565		- dev_warn(&sq->hwq.pdev->dev,
1566		- "QPLIB: Inline data length > 96 detected");
1567		- data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;
1568		- } else {
1569		- data_len = wqe->inline_len;
1570		- }
1571		- memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len);
1572		- wqe_size16 = (data_len + 15) >> 4;
1573		- } else {
1574		- for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data;
1575		- i < wqe->num_sge; i++, hw_sge++) {
1576		- hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1577		- hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1578		- hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1579		- data_len += wqe->sg_list[i].size;
1580		- }
1581		- /* Each SGE entry = 1 WQE size16 */
1582		- wqe_size16 = wqe->num_sge;
1583		- /* HW requires wqe size has room for atleast one SGE even if
1584		- * none was supplied by ULP
1585		- */
1586		- if (!wqe->num_sge)
1587		- wqe_size16++;
	1758	+ if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
	1759	+ sch_handler = true;
	1760	+ dev_dbg(&hwq->pdev->dev,
	1761	+ "%s Error QP. Scheduling for poll_cq\n", __func__);
	1762	+ goto queue_err;
1588	1763	}
1589	1764
	1765	+ base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
	1766	+ ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
	1767	+ memset(base_hdr, 0, sizeof(struct sq_sge));
	1768	+ memset(ext_hdr, 0, sizeof(struct sq_sge));
	1769	+
	1770	+ if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE)
	1771	+ /* Copy the inline data */
	1772	+ data_len = bnxt_qplib_put_inline(qp, wqe, &idx);
	1773	+ else
	1774	+ data_len = bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge,
	1775	+ &idx);
	1776	+ if (data_len < 0)
	1777	+ goto queue_err;
1590	1778	/* Specifics */
1591	1779	switch (wqe->type) {
1592	1780	case BNXT_QPLIB_SWQE_TYPE_SEND:
1593	1781	if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
	1782	+ struct sq_send_raweth_qp1_hdr *sqe = base_hdr;
	1783	+ struct sq_raw_ext_hdr *ext_sqe = ext_hdr;
1594	1784	/* Assemble info for Raw Ethertype QPs */
1595		- struct sq_send_raweth_qp1 *sqe =
1596		- (struct sq_send_raweth_qp1 *)hw_sq_send_hdr;
1597	1785
1598	1786	sqe->wqe_type = wqe->type;
1599	1787	sqe->flags = wqe->flags;
1600		- sqe->wqe_size = wqe_size16 +
1601		- ((offsetof(typeof(*sqe), data) + 15) >> 4);
	1788	+ sqe->wqe_size = wqe_sz;
1602	1789	sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
1603	1790	sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
1604	1791	sqe->length = cpu_to_le32(data_len);
1605		- sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
	1792	+ ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
1606	1793	SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
1607	1794	SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
1608	1795
1609	1796	break;
1610	1797	}
1611		- /* fall thru */
	1798	+ fallthrough;
1612	1799	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
1613	1800	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
1614	1801	{
1615		- struct sq_send sqe = (struct sq_send )hw_sq_send_hdr;
	1802	+ struct sq_ud_ext_hdr *ext_sqe = ext_hdr;
	1803	+ struct sq_send_hdr *sqe = base_hdr;
1616	1804
1617	1805	sqe->wqe_type = wqe->type;
1618	1806	sqe->flags = wqe->flags;
1619		- sqe->wqe_size = wqe_size16 +
1620		- ((offsetof(typeof(*sqe), data) + 15) >> 4);
1621		- sqe->inv_key_or_imm_data = cpu_to_le32(
1622		- wqe->send.inv_key);
1623		- if (qp->type == CMDQ_CREATE_QP_TYPE_UD) {
	1807	+ sqe->wqe_size = wqe_sz;
	1808	+ sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key);
	1809	+ if (qp->type == CMDQ_CREATE_QP_TYPE_UD \|\|
	1810	+ qp->type == CMDQ_CREATE_QP_TYPE_GSI) {
1624	1811	sqe->q_key = cpu_to_le32(wqe->send.q_key);
1625		- sqe->dst_qp = cpu_to_le32(
1626		- wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
1627	1812	sqe->length = cpu_to_le32(data_len);
1628		- sqe->avid = cpu_to_le32(wqe->send.avid &
1629		- SQ_SEND_AVID_MASK);
1630	1813	sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
	1814	+ ext_sqe->dst_qp = cpu_to_le32(wqe->send.dst_qp &
	1815	+ SQ_SEND_DST_QP_MASK);
	1816	+ ext_sqe->avid = cpu_to_le32(wqe->send.avid &
	1817	+ SQ_SEND_AVID_MASK);
1631	1818	} else {
1632	1819	sqe->length = cpu_to_le32(data_len);
1633		- sqe->dst_qp = 0;
1634		- sqe->avid = 0;
1635	1820	if (qp->mtu)
1636	1821	pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1637	1822	if (!pkt_num)
..	..	@@ -1644,16 +1829,16 @@
1644	1829	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
1645	1830	case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
1646	1831	{
1647		- struct sq_rdma sqe = (struct sq_rdma )hw_sq_send_hdr;
	1832	+ struct sq_rdma_ext_hdr *ext_sqe = ext_hdr;
	1833	+ struct sq_rdma_hdr *sqe = base_hdr;
1648	1834
1649	1835	sqe->wqe_type = wqe->type;
1650	1836	sqe->flags = wqe->flags;
1651		- sqe->wqe_size = wqe_size16 +
1652		- ((offsetof(typeof(*sqe), data) + 15) >> 4);
	1837	+ sqe->wqe_size = wqe_sz;
1653	1838	sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
1654	1839	sqe->length = cpu_to_le32((u32)data_len);
1655		- sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
1656		- sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
	1840	+ ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
	1841	+ ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
1657	1842	if (qp->mtu)
1658	1843	pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1659	1844	if (!pkt_num)
..	..	@@ -1664,14 +1849,15 @@
1664	1849	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
1665	1850	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
1666	1851	{
1667		- struct sq_atomic sqe = (struct sq_atomic )hw_sq_send_hdr;
	1852	+ struct sq_atomic_ext_hdr *ext_sqe = ext_hdr;
	1853	+ struct sq_atomic_hdr *sqe = base_hdr;
1668	1854
1669	1855	sqe->wqe_type = wqe->type;
1670	1856	sqe->flags = wqe->flags;
1671	1857	sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
1672	1858	sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
1673		- sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
1674		- sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
	1859	+ ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
	1860	+ ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
1675	1861	if (qp->mtu)
1676	1862	pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1677	1863	if (!pkt_num)
..	..	@@ -1681,8 +1867,7 @@
1681	1867	}
1682	1868	case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
1683	1869	{
1684		- struct sq_localinvalidate *sqe =
1685		- (struct sq_localinvalidate *)hw_sq_send_hdr;
	1870	+ struct sq_localinvalidate *sqe = base_hdr;
1686	1871
1687	1872	sqe->wqe_type = wqe->type;
1688	1873	sqe->flags = wqe->flags;
..	..	@@ -1692,7 +1877,8 @@
1692	1877	}
1693	1878	case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
1694	1879	{
1695		- struct sq_fr_pmr sqe = (struct sq_fr_pmr )hw_sq_send_hdr;
	1880	+ struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr;
	1881	+ struct sq_fr_pmr_hdr *sqe = base_hdr;
1696	1882
1697	1883	sqe->wqe_type = wqe->type;
1698	1884	sqe->flags = wqe->flags;
..	..	@@ -1716,14 +1902,15 @@
1716	1902	wqe->frmr.pbl_ptr[i] = cpu_to_le64(
1717	1903	wqe->frmr.page_list[i] \|
1718	1904	PTU_PTE_VALID);
1719		- sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
1720		- sqe->va = cpu_to_le64(wqe->frmr.va);
	1905	+ ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
	1906	+ ext_sqe->va = cpu_to_le64(wqe->frmr.va);
1721	1907
1722	1908	break;
1723	1909	}
1724	1910	case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
1725	1911	{
1726		- struct sq_bind sqe = (struct sq_bind )hw_sq_send_hdr;
	1912	+ struct sq_bind_ext_hdr *ext_sqe = ext_hdr;
	1913	+ struct sq_bind_hdr *sqe = base_hdr;
1727	1914
1728	1915	sqe->wqe_type = wqe->type;
1729	1916	sqe->flags = wqe->flags;
..	..	@@ -1732,9 +1919,8 @@
1732	1919	(wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
1733	1920	sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
1734	1921	sqe->l_key = cpu_to_le32(wqe->bind.r_key);
1735		- sqe->va = cpu_to_le64(wqe->bind.va);
1736		- temp32 = cpu_to_le32(wqe->bind.length);
1737		- memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length));
	1922	+ ext_sqe->va = cpu_to_le64(wqe->bind.va);
	1923	+ ext_sqe->length_lo = cpu_to_le32(wqe->bind.length);
1738	1924	break;
1739	1925	}
1740	1926	default:
..	..	@@ -1743,31 +1929,11 @@
1743	1929	goto done;
1744	1930	}
1745	1931	swq->next_psn = sq->psn & BTH_PSN_MASK;
1746		- if (swq->psn_search) {
1747		- swq->psn_search->opcode_start_psn = cpu_to_le32(
1748		- ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
1749		- SQ_PSN_SEARCH_START_PSN_MASK) \|
1750		- ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
1751		- SQ_PSN_SEARCH_OPCODE_MASK));
1752		- swq->psn_search->flags_next_psn = cpu_to_le32(
1753		- ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
1754		- SQ_PSN_SEARCH_NEXT_PSN_MASK));
1755		- }
	1932	+ bnxt_qplib_fill_psn_search(qp, wqe, swq);
1756	1933	queue_err:
1757		- if (sch_handler) {
1758		- /* Store the ULP info in the software structures */
1759		- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1760		- swq = &sq->swq[sw_prod];
1761		- swq->wr_id = wqe->wr_id;
1762		- swq->type = wqe->type;
1763		- swq->flags = wqe->flags;
1764		- if (qp->sig_type)
1765		- swq->flags \|= SQ_SEND_FLAGS_SIGNAL_COMP;
1766		- swq->start_psn = sq->psn & BTH_PSN_MASK;
1767		- }
1768		- sq->hwq.prod++;
	1934	+ bnxt_qplib_swq_mod_start(sq, wqe_idx);
	1935	+ bnxt_qplib_hwq_incr_prod(hwq, swq->slots);
1769	1936	qp->wqe_cnt++;
1770		-
1771	1937	done:
1772	1938	if (sch_handler) {
1773	1939	nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
..	..	@@ -1777,8 +1943,8 @@
1777	1943	INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
1778	1944	queue_work(qp->scq->nq->cqn_wq, &nq_work->work);
1779	1945	} else {
1780		- dev_err(&sq->hwq.pdev->dev,
1781		- "QPLIB: FP: Failed to allocate SQ nq_work!");
	1946	+ dev_err(&hwq->pdev->dev,
	1947	+ "FP: Failed to allocate SQ nq_work!\n");
1782	1948	rc = -ENOMEM;
1783	1949	}
1784	1950	}
..	..	@@ -1788,81 +1954,74 @@
1788	1954	void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
1789	1955	{
1790	1956	struct bnxt_qplib_q *rq = &qp->rq;
1791		- struct dbr_dbr db_msg = { 0 };
1792		- u32 sw_prod;
1793	1957
1794		- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1795		- db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1796		- DBR_DBR_INDEX_MASK);
1797		- db_msg.type_xid =
1798		- cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) \|
1799		- DBR_DBR_TYPE_RQ);
1800		-
1801		- /* Flush the writes to HW Rx WQE before the ringing Rx DB */
1802		- wmb();
1803		- __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
	1958	+ bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ);
1804	1959	}
1805	1960
1806	1961	int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
1807	1962	struct bnxt_qplib_swqe *wqe)
1808	1963	{
1809		- struct bnxt_qplib_q *rq = &qp->rq;
1810		- struct rq_wqe rqe, *rqe_ptr;
1811		- struct sq_sge *hw_sge;
1812	1964	struct bnxt_qplib_nq_work *nq_work = NULL;
	1965	+ struct bnxt_qplib_q *rq = &qp->rq;
	1966	+ struct rq_wqe_hdr *base_hdr;
	1967	+ struct rq_ext_hdr *ext_hdr;
	1968	+ struct bnxt_qplib_hwq *hwq;
	1969	+ struct bnxt_qplib_swq *swq;
1813	1970	bool sch_handler = false;
1814		- u32 sw_prod;
1815		- int i, rc = 0;
	1971	+ u16 wqe_sz, idx;
	1972	+ u32 wqe_idx;
	1973	+ int rc = 0;
1816	1974
1817		- if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1818		- sch_handler = true;
1819		- dev_dbg(&rq->hwq.pdev->dev,
1820		- "%s Error QP. Scheduling for poll_cq\n",
1821		- __func__);
1822		- goto queue_err;
1823		- }
1824		- if (bnxt_qplib_queue_full(rq)) {
1825		- dev_err(&rq->hwq.pdev->dev,
1826		- "QPLIB: FP: QP (0x%x) RQ is full!", qp->id);
	1975	+ hwq = &rq->hwq;
	1976	+ if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
	1977	+ dev_err(&hwq->pdev->dev,
	1978	+ "QPLIB: FP: QP (0x%x) is in the 0x%x state",
	1979	+ qp->id, qp->state);
1827	1980	rc = -EINVAL;
1828	1981	goto done;
1829	1982	}
1830		- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1831		- rq->swq[sw_prod].wr_id = wqe->wr_id;
1832	1983
1833		- rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr;
1834		- rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
1835		-
1836		- memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
1837		-
1838		- /* Calculate wqe_size16 and data_len */
1839		- for (i = 0, hw_sge = (struct sq_sge *)rqe->data;
1840		- i < wqe->num_sge; i++, hw_sge++) {
1841		- hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1842		- hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1843		- hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
	1984	+ if (bnxt_qplib_queue_full(rq, rq->dbinfo.max_slot)) {
	1985	+ dev_err(&hwq->pdev->dev,
	1986	+ "FP: QP (0x%x) RQ is full!\n", qp->id);
	1987	+ rc = -EINVAL;
	1988	+ goto done;
1844	1989	}
1845		- rqe->wqe_type = wqe->type;
1846		- rqe->flags = wqe->flags;
1847		- rqe->wqe_size = wqe->num_sge +
1848		- ((offsetof(typeof(*rqe), data) + 15) >> 4);
1849		- /* HW requires wqe size has room for atleast one SGE even if none
1850		- * was supplied by ULP
1851		- */
1852		- if (!wqe->num_sge)
1853		- rqe->wqe_size++;
1854	1990
1855		- /* Supply the rqe->wr_id index to the wr_id_tbl for now */
1856		- rqe->wr_id[0] = cpu_to_le32(sw_prod);
	1991	+ swq = bnxt_qplib_get_swqe(rq, &wqe_idx);
	1992	+ swq->wr_id = wqe->wr_id;
	1993	+ swq->slots = rq->dbinfo.max_slot;
1857	1994
	1995	+ if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
	1996	+ sch_handler = true;
	1997	+ dev_dbg(&hwq->pdev->dev,
	1998	+ "%s: Error QP. Scheduling for poll_cq\n", __func__);
	1999	+ goto queue_err;
	2000	+ }
	2001	+
	2002	+ idx = 0;
	2003	+ base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
	2004	+ ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
	2005	+ memset(base_hdr, 0, sizeof(struct sq_sge));
	2006	+ memset(ext_hdr, 0, sizeof(struct sq_sge));
	2007	+ wqe_sz = (sizeof(struct rq_wqe_hdr) +
	2008	+ wqe->num_sge * sizeof(struct sq_sge)) >> 4;
	2009	+ bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge, &idx);
	2010	+ if (!wqe->num_sge) {
	2011	+ struct sq_sge *sge;
	2012	+
	2013	+ sge = bnxt_qplib_get_prod_qe(hwq, idx++);
	2014	+ sge->size = 0;
	2015	+ wqe_sz++;
	2016	+ }
	2017	+ base_hdr->wqe_type = wqe->type;
	2018	+ base_hdr->flags = wqe->flags;
	2019	+ base_hdr->wqe_size = wqe_sz;
	2020	+ base_hdr->wr_id[0] = cpu_to_le32(wqe_idx);
1858	2021	queue_err:
1859		- if (sch_handler) {
1860		- /* Store the ULP info in the software structures */
1861		- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1862		- rq->swq[sw_prod].wr_id = wqe->wr_id;
1863		- }
1864		-
1865		- rq->hwq.prod++;
	2022	+ bnxt_qplib_swq_mod_start(rq, wqe_idx);
	2023	+ bnxt_qplib_hwq_incr_prod(hwq, swq->slots);
	2024	+done:
1866	2025	if (sch_handler) {
1867	2026	nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
1868	2027	if (nq_work) {
..	..	@@ -1871,90 +2030,53 @@
1871	2030	INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
1872	2031	queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);
1873	2032	} else {
1874		- dev_err(&rq->hwq.pdev->dev,
1875		- "QPLIB: FP: Failed to allocate RQ nq_work!");
	2033	+ dev_err(&hwq->pdev->dev,
	2034	+ "FP: Failed to allocate RQ nq_work!\n");
1876	2035	rc = -ENOMEM;
1877	2036	}
1878	2037	}
1879		-done:
	2038	+
1880	2039	return rc;
1881	2040	}
1882	2041
1883	2042	/* CQ */
1884		-
1885		-/* Spinlock must be held */
1886		-static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq)
1887		-{
1888		- struct dbr_dbr db_msg = { 0 };
1889		-
1890		- db_msg.type_xid =
1891		- cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) \|
1892		- DBR_DBR_TYPE_CQ_ARMENA);
1893		- /* Flush memory writes before enabling the CQ */
1894		- wmb();
1895		- __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64));
1896		-}
1897		-
1898		-static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
1899		-{
1900		- struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1901		- struct dbr_dbr db_msg = { 0 };
1902		- u32 sw_cons;
1903		-
1904		- /* Ring DB */
1905		- sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq);
1906		- db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) &
1907		- DBR_DBR_INDEX_MASK);
1908		- db_msg.type_xid =
1909		- cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) \|
1910		- arm_type);
1911		- /* flush memory writes before arming the CQ */
1912		- wmb();
1913		- __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1914		-}
1915		-
1916	2043	int bnxt_qplib_create_cq(struct bnxt_qplib_res res, struct bnxt_qplib_cq cq)
1917	2044	{
1918	2045	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1919		- struct cmdq_create_cq req;
	2046	+ struct bnxt_qplib_hwq_attr hwq_attr = {};
1920	2047	struct creq_create_cq_resp resp;
1921	2048	struct bnxt_qplib_pbl *pbl;
	2049	+ struct cmdq_create_cq req;
1922	2050	u16 cmd_flags = 0;
	2051	+ u32 pg_sz_lvl;
1923	2052	int rc;
1924		-
1925		- cq->hwq.max_elements = cq->max_wqe;
1926		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead,
1927		- cq->nmap, &cq->hwq.max_elements,
1928		- BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,
1929		- PAGE_SIZE, HWQ_TYPE_QUEUE);
1930		- if (rc)
1931		- goto exit;
1932		-
1933		- RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);
1934	2053
1935	2054	if (!cq->dpi) {
1936	2055	dev_err(&rcfw->pdev->dev,
1937		- "QPLIB: FP: CREATE_CQ failed due to NULL DPI");
	2056	+ "FP: CREATE_CQ failed due to NULL DPI\n");
1938	2057	return -EINVAL;
1939	2058	}
	2059	+
	2060	+ hwq_attr.res = res;
	2061	+ hwq_attr.depth = cq->max_wqe;
	2062	+ hwq_attr.stride = sizeof(struct cq_base);
	2063	+ hwq_attr.type = HWQ_TYPE_QUEUE;
	2064	+ hwq_attr.sginfo = &cq->sg_info;
	2065	+ rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr);
	2066	+ if (rc)
	2067	+ return rc;
	2068	+
	2069	+ RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);
	2070	+
1940	2071	req.dpi = cpu_to_le32(cq->dpi->dpi);
1941	2072	req.cq_handle = cpu_to_le64(cq->cq_handle);
1942		-
1943	2073	req.cq_size = cpu_to_le32(cq->hwq.max_elements);
1944	2074	pbl = &cq->hwq.pbl[PBL_LVL_0];
1945		- req.pg_size_lvl = cpu_to_le32(
1946		- ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
1947		- CMDQ_CREATE_CQ_LVL_SFT) \|
1948		- (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K :
1949		- pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K :
1950		- pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K :
1951		- pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M :
1952		- pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M :
1953		- pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G :
1954		- CMDQ_CREATE_CQ_PG_SIZE_PG_4K));
1955		-
	2075	+ pg_sz_lvl = (bnxt_qplib_base_pg_size(&cq->hwq) <<
	2076	+ CMDQ_CREATE_CQ_PG_SIZE_SFT);
	2077	+ pg_sz_lvl \|= (cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK);
	2078	+ req.pg_size_lvl = cpu_to_le32(pg_sz_lvl);
1956	2079	req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
1957		-
1958	2080	req.cq_fco_cnq_id = cpu_to_le32(
1959	2081	(cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
1960	2082	CMDQ_CREATE_CQ_CNQ_ID_SFT);
..	..	@@ -1965,7 +2087,6 @@
1965	2087	goto fail;
1966	2088
1967	2089	cq->id = le32_to_cpu(resp.xid);
1968		- cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
1969	2090	cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
1970	2091	init_waitqueue_head(&cq->waitq);
1971	2092	INIT_LIST_HEAD(&cq->sqf_head);
..	..	@@ -1973,12 +2094,17 @@
1973	2094	spin_lock_init(&cq->compl_lock);
1974	2095	spin_lock_init(&cq->flush_lock);
1975	2096
1976		- bnxt_qplib_arm_cq_enable(cq);
	2097	+ cq->dbinfo.hwq = &cq->hwq;
	2098	+ cq->dbinfo.xid = cq->id;
	2099	+ cq->dbinfo.db = cq->dpi->dbr;
	2100	+ cq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem;
	2101	+
	2102	+ bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA);
	2103	+
1977	2104	return 0;
1978	2105
1979	2106	fail:
1980		- bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1981		-exit:
	2107	+ bnxt_qplib_free_hwq(res, &cq->hwq);
1982	2108	return rc;
1983	2109	}
1984	2110
..	..	@@ -1987,6 +2113,7 @@
1987	2113	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1988	2114	struct cmdq_destroy_cq req;
1989	2115	struct creq_destroy_cq_resp resp;
	2116	+ u16 total_cnq_events;
1990	2117	u16 cmd_flags = 0;
1991	2118	int rc;
1992	2119
..	..	@@ -1997,27 +2124,28 @@
1997	2124	(void *)&resp, NULL, 0);
1998	2125	if (rc)
1999	2126	return rc;
2000		- bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
	2127	+ total_cnq_events = le16_to_cpu(resp.total_cnq_events);
	2128	+ __wait_for_all_nqes(cq, total_cnq_events);
	2129	+ bnxt_qplib_free_hwq(res, &cq->hwq);
2001	2130	return 0;
2002	2131	}
2003	2132
2004	2133	static int __flush_sq(struct bnxt_qplib_q sq, struct bnxt_qplib_qp qp,
2005	2134	struct bnxt_qplib_cqe *pcqe, int budget)
2006	2135	{
2007		- u32 sw_prod, sw_cons;
2008	2136	struct bnxt_qplib_cqe *cqe;
	2137	+ u32 start, last;
2009	2138	int rc = 0;
2010	2139
2011	2140	/* Now complete all outstanding SQEs with FLUSHED_ERR */
2012		- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
	2141	+ start = sq->swq_start;
2013	2142	cqe = *pcqe;
2014	2143	while (*budget) {
2015		- sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2016		- if (sw_cons == sw_prod) {
	2144	+ last = sq->swq_last;
	2145	+ if (start == last)
2017	2146	break;
2018		- }
2019	2147	/* Skip the FENCE WQE completions */
2020		- if (sq->swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) {
	2148	+ if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) {
2021	2149	bnxt_qplib_cancel_phantom_processing(qp);
2022	2150	goto skip_compl;
2023	2151	}
..	..	@@ -2025,16 +2153,17 @@
2025	2153	cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
2026	2154	cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2027	2155	cqe->qp_handle = (u64)(unsigned long)qp;
2028		- cqe->wr_id = sq->swq[sw_cons].wr_id;
	2156	+ cqe->wr_id = sq->swq[last].wr_id;
2029	2157	cqe->src_qp = qp->id;
2030		- cqe->type = sq->swq[sw_cons].type;
	2158	+ cqe->type = sq->swq[last].type;
2031	2159	cqe++;
2032	2160	(*budget)--;
2033	2161	skip_compl:
2034		- sq->hwq.cons++;
	2162	+ bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[last].slots);
	2163	+ sq->swq_last = sq->swq[last].next_idx;
2035	2164	}
2036	2165	*pcqe = cqe;
2037		- if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)
	2166	+ if (!(*budget) && sq->swq_last != start)
2038	2167	/* Out of budget */
2039	2168	rc = -EAGAIN;
2040	2169
..	..	@@ -2045,9 +2174,9 @@
2045	2174	struct bnxt_qplib_cqe *pcqe, int budget)
2046	2175	{
2047	2176	struct bnxt_qplib_cqe *cqe;
2048		- u32 sw_prod, sw_cons;
2049		- int rc = 0;
	2177	+ u32 start, last;
2050	2178	int opcode = 0;
	2179	+ int rc = 0;
2051	2180
2052	2181	switch (qp->type) {
2053	2182	case CMDQ_CREATE_QP1_TYPE_GSI:
..	..	@@ -2057,29 +2186,31 @@
2057	2186	opcode = CQ_BASE_CQE_TYPE_RES_RC;
2058	2187	break;
2059	2188	case CMDQ_CREATE_QP_TYPE_UD:
	2189	+ case CMDQ_CREATE_QP_TYPE_GSI:
2060	2190	opcode = CQ_BASE_CQE_TYPE_RES_UD;
2061	2191	break;
2062	2192	}
2063	2193
2064	2194	/* Flush the rest of the RQ */
2065		- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
	2195	+ start = rq->swq_start;
2066	2196	cqe = *pcqe;
2067	2197	while (*budget) {
2068		- sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);
2069		- if (sw_cons == sw_prod)
	2198	+ last = rq->swq_last;
	2199	+ if (last == start)
2070	2200	break;
2071	2201	memset(cqe, 0, sizeof(*cqe));
2072	2202	cqe->status =
2073	2203	CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
2074	2204	cqe->opcode = opcode;
2075	2205	cqe->qp_handle = (unsigned long)qp;
2076		- cqe->wr_id = rq->swq[sw_cons].wr_id;
	2206	+ cqe->wr_id = rq->swq[last].wr_id;
2077	2207	cqe++;
2078	2208	(*budget)--;
2079		- rq->hwq.cons++;
	2209	+ bnxt_qplib_hwq_incr_cons(&rq->hwq, rq->swq[last].slots);
	2210	+ rq->swq_last = rq->swq[last].next_idx;
2080	2211	}
2081	2212	*pcqe = cqe;
2082		- if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)
	2213	+ if (!*budget && rq->swq_last != start)
2083	2214	/* Out of budget */
2084	2215	rc = -EAGAIN;
2085	2216
..	..	@@ -2102,20 +2233,20 @@
2102	2233	* CQE is track from sw_cq_cons to max_element but valid only if VALID=1
2103	2234	*/
2104	2235	static int do_wa9060(struct bnxt_qplib_qp qp, struct bnxt_qplib_cq cq,
2105		- u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons)
	2236	+ u32 cq_cons, u32 swq_last, u32 cqe_sq_cons)
2106	2237	{
2107		- struct bnxt_qplib_q *sq = &qp->sq;
2108		- struct bnxt_qplib_swq *swq;
2109	2238	u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;
2110		- struct cq_base peek_hwcqe, *peek_hw_cqe_ptr;
	2239	+ struct bnxt_qplib_q *sq = &qp->sq;
2111	2240	struct cq_req *peek_req_hwcqe;
2112	2241	struct bnxt_qplib_qp *peek_qp;
2113	2242	struct bnxt_qplib_q *peek_sq;
	2243	+ struct bnxt_qplib_swq *swq;
	2244	+ struct cq_base *peek_hwcqe;
2114	2245	int i, rc = 0;
2115	2246
2116	2247	/* Normal mode */
2117	2248	/* Check for the psn_search marking before completing */
2118		- swq = &sq->swq[sw_sq_cons];
	2249	+ swq = &sq->swq[swq_last];
2119	2250	if (swq->psn_search &&
2120	2251	le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
2121	2252	/* Unmark */
..	..	@@ -2124,13 +2255,12 @@
2124	2255	& ~0x80000000);
2125	2256	dev_dbg(&cq->hwq.pdev->dev,
2126	2257	"FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
2127		- cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
	2258	+ cq_cons, qp->id, swq_last, cqe_sq_cons);
2128	2259	sq->condition = true;
2129	2260	sq->send_phantom = true;
2130	2261
2131	2262	/* TODO: Only ARM if the previous SQE is ARMALL */
2132		- bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ_ARMALL);
2133		-
	2263	+ bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL);
2134	2264	rc = -EAGAIN;
2135	2265	goto out;
2136	2266	}
..	..	@@ -2141,9 +2271,8 @@
2141	2271	i = cq->hwq.max_elements;
2142	2272	while (i--) {
2143	2273	peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq);
2144		- peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2145		- peek_hwcqe = &peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)]
2146		- [CQE_IDX(peek_sw_cq_cons)];
	2274	+ peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq,
	2275	+ peek_sw_cq_cons, NULL);
2147	2276	/* If the next hwcqe is VALID */
2148	2277	if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons,
2149	2278	cq->hwq.max_elements)) {
..	..	@@ -2163,9 +2292,10 @@
2163	2292	le64_to_cpu
2164	2293	(peek_req_hwcqe->qp_handle));
2165	2294	peek_sq = &peek_qp->sq;
2166		- peek_sq_cons_idx = HWQ_CMP(le16_to_cpu(
2167		- peek_req_hwcqe->sq_cons_idx) - 1
2168		- , &sq->hwq);
	2295	+ peek_sq_cons_idx =
	2296	+ ((le16_to_cpu(
	2297	+ peek_req_hwcqe->sq_cons_idx)
	2298	+ - 1) % sq->max_wqe);
2169	2299	/* If the hwcqe's sq's wr_id matches */
2170	2300	if (peek_sq == sq &&
2171	2301	sq->swq[peek_sq_cons_idx].wr_id ==
..	..	@@ -2175,7 +2305,7 @@
2175	2305	* comes back
2176	2306	*/
2177	2307	dev_dbg(&cq->hwq.pdev->dev,
2178		- "FP:Got Phantom CQE");
	2308	+ "FP: Got Phantom CQE\n");
2179	2309	sq->condition = false;
2180	2310	sq->single = true;
2181	2311	rc = 0;
..	..	@@ -2192,8 +2322,8 @@
2192	2322	peek_raw_cq_cons++;
2193	2323	}
2194	2324	dev_err(&cq->hwq.pdev->dev,
2195		- "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x",
2196		- cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
	2325	+ "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n",
	2326	+ cq_cons, qp->id, swq_last, cqe_sq_cons);
2197	2327	rc = -EINVAL;
2198	2328	}
2199	2329	out:
..	..	@@ -2205,35 +2335,26 @@
2205	2335	struct bnxt_qplib_cqe *pcqe, int budget,
2206	2336	u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
2207	2337	{
	2338	+ struct bnxt_qplib_swq *swq;
	2339	+ struct bnxt_qplib_cqe *cqe;
2208	2340	struct bnxt_qplib_qp *qp;
2209	2341	struct bnxt_qplib_q *sq;
2210		- struct bnxt_qplib_cqe *cqe;
2211		- u32 sw_sq_cons, cqe_sq_cons;
2212		- struct bnxt_qplib_swq *swq;
	2342	+ u32 cqe_sq_cons;
2213	2343	int rc = 0;
2214	2344
2215	2345	qp = (struct bnxt_qplib_qp *)((unsigned long)
2216	2346	le64_to_cpu(hwcqe->qp_handle));
2217	2347	if (!qp) {
2218	2348	dev_err(&cq->hwq.pdev->dev,
2219		- "QPLIB: FP: Process Req qp is NULL");
	2349	+ "FP: Process Req qp is NULL\n");
2220	2350	return -EINVAL;
2221	2351	}
2222	2352	sq = &qp->sq;
2223	2353
2224		- cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
2225		- if (cqe_sq_cons > sq->hwq.max_elements) {
2226		- dev_err(&cq->hwq.pdev->dev,
2227		- "QPLIB: FP: CQ Process req reported ");
2228		- dev_err(&cq->hwq.pdev->dev,
2229		- "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
2230		- cqe_sq_cons, sq->hwq.max_elements);
2231		- return -EINVAL;
2232		- }
2233		-
	2354	+ cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe;
2234	2355	if (qp->sq.flushed) {
2235	2356	dev_dbg(&cq->hwq.pdev->dev,
2236		- "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
	2357	+ "%s: QP in Flush QP = %p\n", __func__, qp);
2237	2358	goto done;
2238	2359	}
2239	2360	/* Require to walk the sq's swq to fabricate CQEs for all previously
..	..	@@ -2242,12 +2363,11 @@
2242	2363	*/
2243	2364	cqe = *pcqe;
2244	2365	while (*budget) {
2245		- sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2246		- if (sw_sq_cons == cqe_sq_cons)
	2366	+ if (sq->swq_last == cqe_sq_cons)
2247	2367	/* Done */
2248	2368	break;
2249	2369
2250		- swq = &sq->swq[sw_sq_cons];
	2370	+ swq = &sq->swq[sq->swq_last];
2251	2371	memset(cqe, 0, sizeof(*cqe));
2252	2372	cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2253	2373	cqe->qp_handle = (u64)(unsigned long)qp;
..	..	@@ -2261,14 +2381,12 @@
2261	2381	* of the request being signaled or not, it must complete with
2262	2382	* the hwcqe error status
2263	2383	*/
2264		- if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons &&
	2384	+ if (swq->next_idx == cqe_sq_cons &&
2265	2385	hwcqe->status != CQ_REQ_STATUS_OK) {
2266	2386	cqe->status = hwcqe->status;
2267	2387	dev_err(&cq->hwq.pdev->dev,
2268		- "QPLIB: FP: CQ Processed Req ");
2269		- dev_err(&cq->hwq.pdev->dev,
2270		- "QPLIB: wr_id[%d] = 0x%llx with status 0x%x",
2271		- sw_sq_cons, cqe->wr_id, cqe->status);
	2388	+ "FP: CQ Processed Req wr_id[%d] = 0x%llx with status 0x%x\n",
	2389	+ sq->swq_last, cqe->wr_id, cqe->status);
2272	2390	cqe++;
2273	2391	(*budget)--;
2274	2392	bnxt_qplib_mark_qp_error(qp);
..	..	@@ -2276,7 +2394,7 @@
2276	2394	bnxt_qplib_add_flush_qp(qp);
2277	2395	} else {
2278	2396	/* Before we complete, do WA 9060 */
2279		- if (do_wa9060(qp, cq, cq_cons, sw_sq_cons,
	2397	+ if (do_wa9060(qp, cq, cq_cons, sq->swq_last,
2280	2398	cqe_sq_cons)) {
2281	2399	*lib_qp = qp;
2282	2400	goto out;
..	..	@@ -2288,13 +2406,14 @@
2288	2406	}
2289	2407	}
2290	2408	skip:
2291		- sq->hwq.cons++;
	2409	+ bnxt_qplib_hwq_incr_cons(&sq->hwq, swq->slots);
	2410	+ sq->swq_last = swq->next_idx;
2292	2411	if (sq->single)
2293	2412	break;
2294	2413	}
2295	2414	out:
2296	2415	*pcqe = cqe;
2297		- if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) {
	2416	+ if (sq->swq_last != cqe_sq_cons) {
2298	2417	/* Out of budget */
2299	2418	rc = -EAGAIN;
2300	2419	goto done;
..	..	@@ -2323,22 +2442,22 @@
2323	2442	struct bnxt_qplib_cqe **pcqe,
2324	2443	int *budget)
2325	2444	{
2326		- struct bnxt_qplib_qp *qp;
2327		- struct bnxt_qplib_q *rq;
2328	2445	struct bnxt_qplib_srq *srq;
2329	2446	struct bnxt_qplib_cqe *cqe;
	2447	+ struct bnxt_qplib_qp *qp;
	2448	+ struct bnxt_qplib_q *rq;
2330	2449	u32 wr_id_idx;
2331	2450	int rc = 0;
2332	2451
2333	2452	qp = (struct bnxt_qplib_qp *)((unsigned long)
2334	2453	le64_to_cpu(hwcqe->qp_handle));
2335	2454	if (!qp) {
2336		- dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL");
	2455	+ dev_err(&cq->hwq.pdev->dev, "process_cq RC qp is NULL\n");
2337	2456	return -EINVAL;
2338	2457	}
2339	2458	if (qp->rq.flushed) {
2340	2459	dev_dbg(&cq->hwq.pdev->dev,
2341		- "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
	2460	+ "%s: QP in Flush QP = %p\n", __func__, qp);
2342	2461	goto done;
2343	2462	}
2344	2463
..	..	@@ -2359,9 +2478,7 @@
2359	2478	return -EINVAL;
2360	2479	if (wr_id_idx >= srq->hwq.max_elements) {
2361	2480	dev_err(&cq->hwq.pdev->dev,
2362		- "QPLIB: FP: CQ Process RC ");
2363		- dev_err(&cq->hwq.pdev->dev,
2364		- "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
	2481	+ "FP: CQ Process RC wr_id idx 0x%x exceeded SRQ max 0x%x\n",
2365	2482	wr_id_idx, srq->hwq.max_elements);
2366	2483	return -EINVAL;
2367	2484	}
..	..	@@ -2371,19 +2488,23 @@
2371	2488	(*budget)--;
2372	2489	*pcqe = cqe;
2373	2490	} else {
	2491	+ struct bnxt_qplib_swq *swq;
	2492	+
2374	2493	rq = &qp->rq;
2375		- if (wr_id_idx >= rq->hwq.max_elements) {
	2494	+ if (wr_id_idx > (rq->max_wqe - 1)) {
2376	2495	dev_err(&cq->hwq.pdev->dev,
2377		- "QPLIB: FP: CQ Process RC ");
2378		- dev_err(&cq->hwq.pdev->dev,
2379		- "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
2380		- wr_id_idx, rq->hwq.max_elements);
	2496	+ "FP: CQ Process RC wr_id idx 0x%x exceeded RQ max 0x%x\n",
	2497	+ wr_id_idx, rq->max_wqe);
2381	2498	return -EINVAL;
2382	2499	}
2383		- cqe->wr_id = rq->swq[wr_id_idx].wr_id;
	2500	+ if (wr_id_idx != rq->swq_last)
	2501	+ return -EINVAL;
	2502	+ swq = &rq->swq[rq->swq_last];
	2503	+ cqe->wr_id = swq->wr_id;
2384	2504	cqe++;
2385	2505	(*budget)--;
2386		- rq->hwq.cons++;
	2506	+ bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
	2507	+ rq->swq_last = swq->next_idx;
2387	2508	*pcqe = cqe;
2388	2509
2389	2510	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
..	..	@@ -2402,32 +2523,34 @@
2402	2523	struct bnxt_qplib_cqe **pcqe,
2403	2524	int *budget)
2404	2525	{
2405		- struct bnxt_qplib_qp *qp;
2406		- struct bnxt_qplib_q *rq;
2407	2526	struct bnxt_qplib_srq *srq;
2408	2527	struct bnxt_qplib_cqe *cqe;
	2528	+ struct bnxt_qplib_qp *qp;
	2529	+ struct bnxt_qplib_q *rq;
2409	2530	u32 wr_id_idx;
2410	2531	int rc = 0;
2411	2532
2412	2533	qp = (struct bnxt_qplib_qp *)((unsigned long)
2413	2534	le64_to_cpu(hwcqe->qp_handle));
2414	2535	if (!qp) {
2415		- dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL");
	2536	+ dev_err(&cq->hwq.pdev->dev, "process_cq UD qp is NULL\n");
2416	2537	return -EINVAL;
2417	2538	}
2418	2539	if (qp->rq.flushed) {
2419	2540	dev_dbg(&cq->hwq.pdev->dev,
2420		- "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
	2541	+ "%s: QP in Flush QP = %p\n", __func__, qp);
2421	2542	goto done;
2422	2543	}
2423	2544	cqe = *pcqe;
2424	2545	cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2425		- cqe->length = le32_to_cpu(hwcqe->length);
	2546	+ cqe->length = le16_to_cpu(hwcqe->length) & CQ_RES_UD_LENGTH_MASK;
	2547	+ cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata);
2426	2548	cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
2427	2549	cqe->flags = le16_to_cpu(hwcqe->flags);
2428	2550	cqe->status = hwcqe->status;
2429	2551	cqe->qp_handle = (u64)(unsigned long)qp;
2430		- memcpy(cqe->smac, hwcqe->src_mac, 6);
	2552	+ /FIXME: Endianness fix needed for smace /
	2553	+ memcpy(cqe->smac, hwcqe->src_mac, ETH_ALEN);
2431	2554	wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
2432	2555	& CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK;
2433	2556	cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) \|
..	..	@@ -2442,9 +2565,7 @@
2442	2565
2443	2566	if (wr_id_idx >= srq->hwq.max_elements) {
2444	2567	dev_err(&cq->hwq.pdev->dev,
2445		- "QPLIB: FP: CQ Process UD ");
2446		- dev_err(&cq->hwq.pdev->dev,
2447		- "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
	2568	+ "FP: CQ Process UD wr_id idx 0x%x exceeded SRQ max 0x%x\n",
2448	2569	wr_id_idx, srq->hwq.max_elements);
2449	2570	return -EINVAL;
2450	2571	}
..	..	@@ -2454,20 +2575,24 @@
2454	2575	(*budget)--;
2455	2576	*pcqe = cqe;
2456	2577	} else {
	2578	+ struct bnxt_qplib_swq *swq;
	2579	+
2457	2580	rq = &qp->rq;
2458		- if (wr_id_idx >= rq->hwq.max_elements) {
	2581	+ if (wr_id_idx > (rq->max_wqe - 1)) {
2459	2582	dev_err(&cq->hwq.pdev->dev,
2460		- "QPLIB: FP: CQ Process UD ");
2461		- dev_err(&cq->hwq.pdev->dev,
2462		- "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
2463		- wr_id_idx, rq->hwq.max_elements);
	2583	+ "FP: CQ Process UD wr_id idx 0x%x exceeded RQ max 0x%x\n",
	2584	+ wr_id_idx, rq->max_wqe);
2464	2585	return -EINVAL;
2465	2586	}
2466	2587
2467		- cqe->wr_id = rq->swq[wr_id_idx].wr_id;
	2588	+ if (rq->swq_last != wr_id_idx)
	2589	+ return -EINVAL;
	2590	+ swq = &rq->swq[rq->swq_last];
	2591	+ cqe->wr_id = swq->wr_id;
2468	2592	cqe++;
2469	2593	(*budget)--;
2470		- rq->hwq.cons++;
	2594	+ bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
	2595	+ rq->swq_last = swq->next_idx;
2471	2596	*pcqe = cqe;
2472	2597
2473	2598	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
..	..	@@ -2482,15 +2607,13 @@
2482	2607
2483	2608	bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
2484	2609	{
2485		- struct cq_base hw_cqe, *hw_cqe_ptr;
	2610	+ struct cq_base *hw_cqe;
2486	2611	u32 sw_cons, raw_cons;
2487	2612	bool rc = true;
2488	2613
2489	2614	raw_cons = cq->hwq.cons;
2490	2615	sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2491		- hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2492		- hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2493		-
	2616	+ hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL);
2494	2617	/* Check for Valid bit. If the CQE is valid, return false */
2495	2618	rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements);
2496	2619	return rc;
..	..	@@ -2511,13 +2634,12 @@
2511	2634	qp = (struct bnxt_qplib_qp *)((unsigned long)
2512	2635	le64_to_cpu(hwcqe->qp_handle));
2513	2636	if (!qp) {
2514		- dev_err(&cq->hwq.pdev->dev,
2515		- "QPLIB: process_cq Raw/QP1 qp is NULL");
	2637	+ dev_err(&cq->hwq.pdev->dev, "process_cq Raw/QP1 qp is NULL\n");
2516	2638	return -EINVAL;
2517	2639	}
2518	2640	if (qp->rq.flushed) {
2519	2641	dev_dbg(&cq->hwq.pdev->dev,
2520		- "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
	2642	+ "%s: QP in Flush QP = %p\n", __func__, qp);
2521	2643	goto done;
2522	2644	}
2523	2645	cqe = *pcqe;
..	..	@@ -2546,14 +2668,12 @@
2546	2668	srq = qp->srq;
2547	2669	if (!srq) {
2548	2670	dev_err(&cq->hwq.pdev->dev,
2549		- "QPLIB: FP: SRQ used but not defined??");
	2671	+ "FP: SRQ used but not defined??\n");
2550	2672	return -EINVAL;
2551	2673	}
2552	2674	if (wr_id_idx >= srq->hwq.max_elements) {
2553	2675	dev_err(&cq->hwq.pdev->dev,
2554		- "QPLIB: FP: CQ Process Raw/QP1 ");
2555		- dev_err(&cq->hwq.pdev->dev,
2556		- "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
	2676	+ "FP: CQ Process Raw/QP1 wr_id idx 0x%x exceeded SRQ max 0x%x\n",
2557	2677	wr_id_idx, srq->hwq.max_elements);
2558	2678	return -EINVAL;
2559	2679	}
..	..	@@ -2563,19 +2683,23 @@
2563	2683	(*budget)--;
2564	2684	*pcqe = cqe;
2565	2685	} else {
	2686	+ struct bnxt_qplib_swq *swq;
	2687	+
2566	2688	rq = &qp->rq;
2567		- if (wr_id_idx >= rq->hwq.max_elements) {
	2689	+ if (wr_id_idx > (rq->max_wqe - 1)) {
2568	2690	dev_err(&cq->hwq.pdev->dev,
2569		- "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
2570		- dev_err(&cq->hwq.pdev->dev,
2571		- "QPLIB: ix 0x%x exceeded RQ max 0x%x",
2572		- wr_id_idx, rq->hwq.max_elements);
	2691	+ "FP: CQ Process Raw/QP1 RQ wr_id idx 0x%x exceeded RQ max 0x%x\n",
	2692	+ wr_id_idx, rq->max_wqe);
2573	2693	return -EINVAL;
2574	2694	}
2575		- cqe->wr_id = rq->swq[wr_id_idx].wr_id;
	2695	+ if (rq->swq_last != wr_id_idx)
	2696	+ return -EINVAL;
	2697	+ swq = &rq->swq[rq->swq_last];
	2698	+ cqe->wr_id = swq->wr_id;
2576	2699	cqe++;
2577	2700	(*budget)--;
2578		- rq->hwq.cons++;
	2701	+ bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
	2702	+ rq->swq_last = swq->next_idx;
2579	2703	*pcqe = cqe;
2580	2704
2581	2705	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
..	..	@@ -2597,22 +2721,19 @@
2597	2721	struct bnxt_qplib_qp *qp;
2598	2722	struct bnxt_qplib_q sq, rq;
2599	2723	struct bnxt_qplib_cqe *cqe;
2600		- u32 sw_cons = 0, cqe_cons;
	2724	+ u32 swq_last = 0, cqe_cons;
2601	2725	int rc = 0;
2602	2726
2603	2727	/* Check the Status */
2604	2728	if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
2605	2729	dev_warn(&cq->hwq.pdev->dev,
2606		- "QPLIB: FP: CQ Process Terminal Error status = 0x%x",
	2730	+ "FP: CQ Process Terminal Error status = 0x%x\n",
2607	2731	hwcqe->status);
2608	2732
2609	2733	qp = (struct bnxt_qplib_qp *)((unsigned long)
2610	2734	le64_to_cpu(hwcqe->qp_handle));
2611		- if (!qp) {
2612		- dev_err(&cq->hwq.pdev->dev,
2613		- "QPLIB: FP: CQ Process terminal qp is NULL");
	2735	+ if (!qp)
2614	2736	return -EINVAL;
2615		- }
2616	2737
2617	2738	/* Must block new posting of SQ and RQ */
2618	2739	qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
..	..	@@ -2623,19 +2744,11 @@
2623	2744	cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
2624	2745	if (cqe_cons == 0xFFFF)
2625	2746	goto do_rq;
2626		-
2627		- if (cqe_cons > sq->hwq.max_elements) {
2628		- dev_err(&cq->hwq.pdev->dev,
2629		- "QPLIB: FP: CQ Process terminal reported ");
2630		- dev_err(&cq->hwq.pdev->dev,
2631		- "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
2632		- cqe_cons, sq->hwq.max_elements);
2633		- goto do_rq;
2634		- }
	2747	+ cqe_cons %= sq->max_wqe;
2635	2748
2636	2749	if (qp->sq.flushed) {
2637	2750	dev_dbg(&cq->hwq.pdev->dev,
2638		- "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
	2751	+ "%s: QP in Flush QP = %p\n", __func__, qp);
2639	2752	goto sq_done;
2640	2753	}
2641	2754
..	..	@@ -2645,24 +2758,25 @@
2645	2758	*/
2646	2759	cqe = *pcqe;
2647	2760	while (*budget) {
2648		- sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2649		- if (sw_cons == cqe_cons)
	2761	+ swq_last = sq->swq_last;
	2762	+ if (swq_last == cqe_cons)
2650	2763	break;
2651		- if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
	2764	+ if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2652	2765	memset(cqe, 0, sizeof(*cqe));
2653	2766	cqe->status = CQ_REQ_STATUS_OK;
2654	2767	cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2655	2768	cqe->qp_handle = (u64)(unsigned long)qp;
2656	2769	cqe->src_qp = qp->id;
2657		- cqe->wr_id = sq->swq[sw_cons].wr_id;
2658		- cqe->type = sq->swq[sw_cons].type;
	2770	+ cqe->wr_id = sq->swq[swq_last].wr_id;
	2771	+ cqe->type = sq->swq[swq_last].type;
2659	2772	cqe++;
2660	2773	(*budget)--;
2661	2774	}
2662		- sq->hwq.cons++;
	2775	+ bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[swq_last].slots);
	2776	+ sq->swq_last = sq->swq[swq_last].next_idx;
2663	2777	}
2664	2778	*pcqe = cqe;
2665		- if (!(*budget) && sw_cons != cqe_cons) {
	2779	+ if (!(*budget) && swq_last != cqe_cons) {
2666	2780	/* Out of budget */
2667	2781	rc = -EAGAIN;
2668	2782	goto sq_done;
..	..	@@ -2674,18 +2788,17 @@
2674	2788	cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
2675	2789	if (cqe_cons == 0xFFFF) {
2676	2790	goto done;
2677		- } else if (cqe_cons > rq->hwq.max_elements) {
	2791	+ } else if (cqe_cons > rq->max_wqe - 1) {
2678	2792	dev_err(&cq->hwq.pdev->dev,
2679		- "QPLIB: FP: CQ Processed terminal ");
2680		- dev_err(&cq->hwq.pdev->dev,
2681		- "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x",
2682		- cqe_cons, rq->hwq.max_elements);
	2793	+ "FP: CQ Processed terminal reported rq_cons_idx 0x%x exceeds max 0x%x\n",
	2794	+ cqe_cons, rq->max_wqe);
	2795	+ rc = -EINVAL;
2683	2796	goto done;
2684	2797	}
2685	2798
2686	2799	if (qp->rq.flushed) {
2687	2800	dev_dbg(&cq->hwq.pdev->dev,
2688		- "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
	2801	+ "%s: QP in Flush QP = %p\n", __func__, qp);
2689	2802	rc = 0;
2690	2803	goto done;
2691	2804	}
..	..	@@ -2707,7 +2820,7 @@
2707	2820	/* Check the Status */
2708	2821	if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
2709	2822	dev_err(&cq->hwq.pdev->dev,
2710		- "QPLIB: FP: CQ Process Cutoff Error status = 0x%x",
	2823	+ "FP: CQ Process Cutoff Error status = 0x%x\n",
2711	2824	hwcqe->status);
2712	2825	return -EINVAL;
2713	2826	}
..	..	@@ -2727,16 +2840,12 @@
2727	2840
2728	2841	spin_lock_irqsave(&cq->flush_lock, flags);
2729	2842	list_for_each_entry(qp, &cq->sqf_head, sq_flush) {
2730		- dev_dbg(&cq->hwq.pdev->dev,
2731		- "QPLIB: FP: Flushing SQ QP= %p",
2732		- qp);
	2843	+ dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing SQ QP= %p\n", qp);
2733	2844	__flush_sq(&qp->sq, qp, &cqe, &budget);
2734	2845	}
2735	2846
2736	2847	list_for_each_entry(qp, &cq->rqf_head, rq_flush) {
2737		- dev_dbg(&cq->hwq.pdev->dev,
2738		- "QPLIB: FP: Flushing RQ QP= %p",
2739		- qp);
	2848	+ dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing RQ QP= %p\n", qp);
2740	2849	__flush_rq(&qp->rq, qp, &cqe, &budget);
2741	2850	}
2742	2851	spin_unlock_irqrestore(&cq->flush_lock, flags);
..	..	@@ -2747,7 +2856,7 @@
2747	2856	int bnxt_qplib_poll_cq(struct bnxt_qplib_cq cq, struct bnxt_qplib_cqe cqe,
2748	2857	int num_cqes, struct bnxt_qplib_qp **lib_qp)
2749	2858	{
2750		- struct cq_base hw_cqe, *hw_cqe_ptr;
	2859	+ struct cq_base *hw_cqe;
2751	2860	u32 sw_cons, raw_cons;
2752	2861	int budget, rc = 0;
2753	2862
..	..	@@ -2756,8 +2865,7 @@
2756	2865
2757	2866	while (budget) {
2758	2867	sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2759		- hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2760		- hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
	2868	+ hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL);
2761	2869
2762	2870	/* Check for Valid bit */
2763	2871	if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
..	..	@@ -2804,7 +2912,7 @@
2804	2912	goto exit;
2805	2913	default:
2806	2914	dev_err(&cq->hwq.pdev->dev,
2807		- "QPLIB: process_cq unknown type 0x%lx",
	2915	+ "process_cq unknown type 0x%lx\n",
2808	2916	hw_cqe->cqe_type_toggle &
2809	2917	CQ_BASE_CQE_TYPE_MASK);
2810	2918	rc = -EINVAL;
..	..	@@ -2817,13 +2925,13 @@
2817	2925	* next one
2818	2926	*/
2819	2927	dev_err(&cq->hwq.pdev->dev,
2820		- "QPLIB: process_cqe error rc = 0x%x", rc);
	2928	+ "process_cqe error rc = 0x%x\n", rc);
2821	2929	}
2822	2930	raw_cons++;
2823	2931	}
2824	2932	if (cq->hwq.cons != raw_cons) {
2825	2933	cq->hwq.cons = raw_cons;
2826		- bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ);
	2934	+ bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ);
2827	2935	}
2828	2936	exit:
2829	2937	return num_cqes - budget;
..	..	@@ -2832,7 +2940,7 @@
2832	2940	void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
2833	2941	{
2834	2942	if (arm_type)
2835		- bnxt_qplib_arm_cq(cq, arm_type);
	2943	+ bnxt_qplib_ring_db(&cq->dbinfo, arm_type);
2836	2944	/* Using cq->arm_state variable to track whether to issue cq handler */
2837	2945	atomic_set(&cq->arm_state, 1);
2838	2946	}