~hc/RK356X_SDK_RELEASE.git

..	..	@@ -36,12 +36,16 @@
36	36	* Description: Fast Path Operators
37	37	*/
38	38
	39	+#define dev_fmt(fmt) "QPLIB: " fmt
	40	+
39	41	#include <linux/interrupt.h>
40	42	#include <linux/spinlock.h>
41	43	#include <linux/sched.h>
42	44	#include <linux/slab.h>
43	45	#include <linux/pci.h>
	46	+#include <linux/delay.h>
44	47	#include <linux/prefetch.h>
	48	+#include <linux/if_ether.h>
45	49
46	50	#include "roce_hsi.h"
47	51
..	..	@@ -50,9 +54,7 @@
50	54	#include "qplib_sp.h"
51	55	#include "qplib_fp.h"
52	56
53		-static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
54	57	static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);
55		-static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type);
56	58
57	59	static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp)
58	60	{
..	..	@@ -71,8 +73,7 @@
71	73
72	74	if (!qp->sq.flushed) {
73	75	dev_dbg(&scq->hwq.pdev->dev,
74		- "QPLIB: FP: Adding to SQ Flush list = %p",
75		- qp);
	76	+ "FP: Adding to SQ Flush list = %p\n", qp);
76	77	bnxt_qplib_cancel_phantom_processing(qp);
77	78	list_add_tail(&qp->sq_flush, &scq->sqf_head);
78	79	qp->sq.flushed = true;
..	..	@@ -80,8 +81,7 @@
80	81	if (!qp->srq) {
81	82	if (!qp->rq.flushed) {
82	83	dev_dbg(&rcq->hwq.pdev->dev,
83		- "QPLIB: FP: Adding to RQ Flush list = %p",
84		- qp);
	84	+ "FP: Adding to RQ Flush list = %p\n", qp);
85	85	list_add_tail(&qp->rq_flush, &rcq->rqf_head);
86	86	qp->rq.flushed = true;
87	87	}
..	..	@@ -178,11 +178,11 @@
178	178
179	179	if (qp->rq_hdr_buf)
180	180	dma_free_coherent(&res->pdev->dev,
181		- rq->hwq.max_elements * qp->rq_hdr_buf_size,
	181	+ rq->max_wqe * qp->rq_hdr_buf_size,
182	182	qp->rq_hdr_buf, qp->rq_hdr_buf_map);
183	183	if (qp->sq_hdr_buf)
184	184	dma_free_coherent(&res->pdev->dev,
185		- sq->hwq.max_elements * qp->sq_hdr_buf_size,
	185	+ sq->max_wqe * qp->sq_hdr_buf_size,
186	186	qp->sq_hdr_buf, qp->sq_hdr_buf_map);
187	187	qp->rq_hdr_buf = NULL;
188	188	qp->sq_hdr_buf = NULL;
..	..	@@ -199,29 +199,28 @@
199	199	struct bnxt_qplib_q *sq = &qp->sq;
200	200	int rc = 0;
201	201
202		- if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {
	202	+ if (qp->sq_hdr_buf_size && sq->max_wqe) {
203	203	qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
204		- sq->hwq.max_elements *
205		- qp->sq_hdr_buf_size,
	204	+ sq->max_wqe * qp->sq_hdr_buf_size,
206	205	&qp->sq_hdr_buf_map, GFP_KERNEL);
207	206	if (!qp->sq_hdr_buf) {
208	207	rc = -ENOMEM;
209	208	dev_err(&res->pdev->dev,
210		- "QPLIB: Failed to create sq_hdr_buf");
	209	+ "Failed to create sq_hdr_buf\n");
211	210	goto fail;
212	211	}
213	212	}
214	213
215		- if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {
	214	+ if (qp->rq_hdr_buf_size && rq->max_wqe) {
216	215	qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
217		- rq->hwq.max_elements *
	216	+ rq->max_wqe *
218	217	qp->rq_hdr_buf_size,
219	218	&qp->rq_hdr_buf_map,
220	219	GFP_KERNEL);
221	220	if (!qp->rq_hdr_buf) {
222	221	rc = -ENOMEM;
223	222	dev_err(&res->pdev->dev,
224		- "QPLIB: Failed to create rq_hdr_buf");
	223	+ "Failed to create rq_hdr_buf\n");
225	224	goto fail;
226	225	}
227	226	}
..	..	@@ -232,19 +231,16 @@
232	231	return rc;
233	232	}
234	233
235		-static void bnxt_qplib_service_nq(unsigned long data)
	234	+static void clean_nq(struct bnxt_qplib_nq nq, struct bnxt_qplib_cq cq)
236	235	{
237		- struct bnxt_qplib_nq nq = (struct bnxt_qplib_nq )data;
238	236	struct bnxt_qplib_hwq *hwq = &nq->hwq;
239	237	struct nq_base nqe, *nq_ptr;
240		- struct bnxt_qplib_cq *cq;
241		- int num_cqne_processed = 0;
242		- int num_srqne_processed = 0;
243		- u32 sw_cons, raw_cons;
244		- u16 type;
245	238	int budget = nq->budget;
	239	+ u32 sw_cons, raw_cons;
246	240	uintptr_t q_handle;
	241	+ u16 type;
247	242
	243	+ spin_lock_bh(&hwq->lock);
248	244	/* Service the NQ until empty */
249	245	raw_cons = hwq->cons;
250	246	while (budget--) {
..	..	@@ -269,36 +265,108 @@
269	265	q_handle = le32_to_cpu(nqcne->cq_handle_low);
270	266	q_handle \|= (u64)le32_to_cpu(nqcne->cq_handle_high)
271	267	<< 32;
	268	+ if ((unsigned long)cq == q_handle) {
	269	+ nqcne->cq_handle_low = 0;
	270	+ nqcne->cq_handle_high = 0;
	271	+ cq->cnq_events++;
	272	+ }
	273	+ break;
	274	+ }
	275	+ default:
	276	+ break;
	277	+ }
	278	+ raw_cons++;
	279	+ }
	280	+ spin_unlock_bh(&hwq->lock);
	281	+}
	282	+
	283	+/* Wait for receiving all NQEs for this CQ and clean the NQEs associated with
	284	+ * this CQ.
	285	+ */
	286	+static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events)
	287	+{
	288	+ u32 retry_cnt = 100;
	289	+
	290	+ while (retry_cnt--) {
	291	+ if (cnq_events == cq->cnq_events)
	292	+ return;
	293	+ usleep_range(50, 100);
	294	+ clean_nq(cq->nq, cq);
	295	+ }
	296	+}
	297	+
	298	+static void bnxt_qplib_service_nq(struct tasklet_struct *t)
	299	+{
	300	+ struct bnxt_qplib_nq *nq = from_tasklet(nq, t, nq_tasklet);
	301	+ struct bnxt_qplib_hwq *hwq = &nq->hwq;
	302	+ int num_srqne_processed = 0;
	303	+ int num_cqne_processed = 0;
	304	+ struct bnxt_qplib_cq *cq;
	305	+ int budget = nq->budget;
	306	+ u32 sw_cons, raw_cons;
	307	+ struct nq_base *nqe;
	308	+ uintptr_t q_handle;
	309	+ u16 type;
	310	+
	311	+ spin_lock_bh(&hwq->lock);
	312	+ /* Service the NQ until empty */
	313	+ raw_cons = hwq->cons;
	314	+ while (budget--) {
	315	+ sw_cons = HWQ_CMP(raw_cons, hwq);
	316	+ nqe = bnxt_qplib_get_qe(hwq, sw_cons, NULL);
	317	+ if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
	318	+ break;
	319	+
	320	+ /*
	321	+ * The valid test of the entry must be done first before
	322	+ * reading any further.
	323	+ */
	324	+ dma_rmb();
	325	+
	326	+ type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
	327	+ switch (type) {
	328	+ case NQ_BASE_TYPE_CQ_NOTIFICATION:
	329	+ {
	330	+ struct nq_cn nqcne = (struct nq_cn )nqe;
	331	+
	332	+ q_handle = le32_to_cpu(nqcne->cq_handle_low);
	333	+ q_handle \|= (u64)le32_to_cpu(nqcne->cq_handle_high)
	334	+ << 32;
272	335	cq = (struct bnxt_qplib_cq *)(unsigned long)q_handle;
273		- bnxt_qplib_arm_cq_enable(cq);
	336	+ if (!cq)
	337	+ break;
	338	+ bnxt_qplib_armen_db(&cq->dbinfo,
	339	+ DBC_DBC_TYPE_CQ_ARMENA);
274	340	spin_lock_bh(&cq->compl_lock);
275	341	atomic_set(&cq->arm_state, 0);
276	342	if (!nq->cqn_handler(nq, (cq)))
277	343	num_cqne_processed++;
278	344	else
279	345	dev_warn(&nq->pdev->dev,
280		- "QPLIB: cqn - type 0x%x not handled",
281		- type);
	346	+ "cqn - type 0x%x not handled\n", type);
	347	+ cq->cnq_events++;
282	348	spin_unlock_bh(&cq->compl_lock);
283	349	break;
284	350	}
285	351	case NQ_BASE_TYPE_SRQ_EVENT:
286	352	{
	353	+ struct bnxt_qplib_srq *srq;
287	354	struct nq_srq_event *nqsrqe =
288	355	(struct nq_srq_event *)nqe;
289	356
290	357	q_handle = le32_to_cpu(nqsrqe->srq_handle_low);
291	358	q_handle \|= (u64)le32_to_cpu(nqsrqe->srq_handle_high)
292	359	<< 32;
293		- bnxt_qplib_arm_srq((struct bnxt_qplib_srq *)q_handle,
294		- DBR_DBR_TYPE_SRQ_ARMENA);
	360	+ srq = (struct bnxt_qplib_srq *)q_handle;
	361	+ bnxt_qplib_armen_db(&srq->dbinfo,
	362	+ DBC_DBC_TYPE_SRQ_ARMENA);
295	363	if (!nq->srqn_handler(nq,
296	364	(struct bnxt_qplib_srq *)q_handle,
297	365	nqsrqe->event))
298	366	num_srqne_processed++;
299	367	else
300	368	dev_warn(&nq->pdev->dev,
301		- "QPLIB: SRQ event 0x%x not handled",
	369	+ "SRQ event 0x%x not handled\n",
302	370	nqsrqe->event);
303	371	break;
304	372	}
..	..	@@ -306,48 +374,46 @@
306	374	break;
307	375	default:
308	376	dev_warn(&nq->pdev->dev,
309		- "QPLIB: nqe with type = 0x%x not handled",
310		- type);
	377	+ "nqe with type = 0x%x not handled\n", type);
311	378	break;
312	379	}
313	380	raw_cons++;
314	381	}
315	382	if (hwq->cons != raw_cons) {
316	383	hwq->cons = raw_cons;
317		- NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements);
	384	+ bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, true);
318	385	}
	386	+ spin_unlock_bh(&hwq->lock);
319	387	}
320	388
321	389	static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
322	390	{
323	391	struct bnxt_qplib_nq *nq = dev_instance;
324	392	struct bnxt_qplib_hwq *hwq = &nq->hwq;
325		- struct nq_base **nq_ptr;
326	393	u32 sw_cons;
327	394
328	395	/* Prefetch the NQ element */
329	396	sw_cons = HWQ_CMP(hwq->cons, hwq);
330		- nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr;
331		- prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);
	397	+ prefetch(bnxt_qplib_get_qe(hwq, sw_cons, NULL));
332	398
333	399	/* Fan out to CPU affinitized kthreads? */
334		- tasklet_schedule(&nq->worker);
	400	+ tasklet_schedule(&nq->nq_tasklet);
335	401
336	402	return IRQ_HANDLED;
337	403	}
338	404
339	405	void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill)
340	406	{
341		- tasklet_disable(&nq->worker);
	407	+ tasklet_disable(&nq->nq_tasklet);
342	408	/* Mask h/w interrupt */
343		- NQ_DB(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
	409	+ bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, false);
344	410	/* Sync with last running IRQ handler */
345		- synchronize_irq(nq->vector);
	411	+ synchronize_irq(nq->msix_vec);
346	412	if (kill)
347		- tasklet_kill(&nq->worker);
	413	+ tasklet_kill(&nq->nq_tasklet);
348	414	if (nq->requested) {
349		- irq_set_affinity_hint(nq->vector, NULL);
350		- free_irq(nq->vector, nq);
	415	+ irq_set_affinity_hint(nq->msix_vec, NULL);
	416	+ free_irq(nq->msix_vec, nq);
351	417	nq->requested = false;
352	418	}
353	419	}
..	..	@@ -360,16 +426,16 @@
360	426	}
361	427
362	428	/* Make sure the HW is stopped! */
363		- if (nq->requested)
364		- bnxt_qplib_nq_stop_irq(nq, true);
	429	+ bnxt_qplib_nq_stop_irq(nq, true);
365	430
366		- if (nq->bar_reg_iomem)
367		- iounmap(nq->bar_reg_iomem);
368		- nq->bar_reg_iomem = NULL;
	431	+ if (nq->nq_db.reg.bar_reg) {
	432	+ iounmap(nq->nq_db.reg.bar_reg);
	433	+ nq->nq_db.reg.bar_reg = NULL;
	434	+ }
369	435
370	436	nq->cqn_handler = NULL;
371	437	nq->srqn_handler = NULL;
372		- nq->vector = 0;
	438	+ nq->msix_vec = 0;
373	439	}
374	440
375	441	int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
..	..	@@ -380,71 +446,92 @@
380	446	if (nq->requested)
381	447	return -EFAULT;
382	448
383		- nq->vector = msix_vector;
	449	+ nq->msix_vec = msix_vector;
384	450	if (need_init)
385		- tasklet_init(&nq->worker, bnxt_qplib_service_nq,
386		- (unsigned long)nq);
	451	+ tasklet_setup(&nq->nq_tasklet, bnxt_qplib_service_nq);
387	452	else
388		- tasklet_enable(&nq->worker);
	453	+ tasklet_enable(&nq->nq_tasklet);
389	454
390	455	snprintf(nq->name, sizeof(nq->name), "bnxt_qplib_nq-%d", nq_indx);
391		- rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, nq->name, nq);
	456	+ rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq);
392	457	if (rc)
393	458	return rc;
394	459
395	460	cpumask_clear(&nq->mask);
396	461	cpumask_set_cpu(nq_indx, &nq->mask);
397		- rc = irq_set_affinity_hint(nq->vector, &nq->mask);
	462	+ rc = irq_set_affinity_hint(nq->msix_vec, &nq->mask);
398	463	if (rc) {
399	464	dev_warn(&nq->pdev->dev,
400		- "QPLIB: set affinity failed; vector: %d nq_idx: %d\n",
401		- nq->vector, nq_indx);
	465	+ "set affinity failed; vector: %d nq_idx: %d\n",
	466	+ nq->msix_vec, nq_indx);
402	467	}
403	468	nq->requested = true;
404		- NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
	469	+ bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, true);
405	470
	471	+ return rc;
	472	+}
	473	+
	474	+static int bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq, u32 reg_offt)
	475	+{
	476	+ resource_size_t reg_base;
	477	+ struct bnxt_qplib_nq_db *nq_db;
	478	+ struct pci_dev *pdev;
	479	+ int rc = 0;
	480	+
	481	+ pdev = nq->pdev;
	482	+ nq_db = &nq->nq_db;
	483	+
	484	+ nq_db->reg.bar_id = NQ_CONS_PCI_BAR_REGION;
	485	+ nq_db->reg.bar_base = pci_resource_start(pdev, nq_db->reg.bar_id);
	486	+ if (!nq_db->reg.bar_base) {
	487	+ dev_err(&pdev->dev, "QPLIB: NQ BAR region %d resc start is 0!",
	488	+ nq_db->reg.bar_id);
	489	+ rc = -ENOMEM;
	490	+ goto fail;
	491	+ }
	492	+
	493	+ reg_base = nq_db->reg.bar_base + reg_offt;
	494	+ /* Unconditionally map 8 bytes to support 57500 series */
	495	+ nq_db->reg.len = 8;
	496	+ nq_db->reg.bar_reg = ioremap(reg_base, nq_db->reg.len);
	497	+ if (!nq_db->reg.bar_reg) {
	498	+ dev_err(&pdev->dev, "QPLIB: NQ BAR region %d mapping failed",
	499	+ nq_db->reg.bar_id);
	500	+ rc = -ENOMEM;
	501	+ goto fail;
	502	+ }
	503	+
	504	+ nq_db->dbinfo.db = nq_db->reg.bar_reg;
	505	+ nq_db->dbinfo.hwq = &nq->hwq;
	506	+ nq_db->dbinfo.xid = nq->ring_id;
	507	+fail:
406	508	return rc;
407	509	}
408	510
409	511	int bnxt_qplib_enable_nq(struct pci_dev pdev, struct bnxt_qplib_nq nq,
410	512	int nq_idx, int msix_vector, int bar_reg_offset,
411		- int (cqn_handler)(struct bnxt_qplib_nq nq,
412		- struct bnxt_qplib_cq *),
413		- int (srqn_handler)(struct bnxt_qplib_nq nq,
414		- struct bnxt_qplib_srq *,
415		- u8 event))
	513	+ cqn_handler_t cqn_handler,
	514	+ srqn_handler_t srqn_handler)
416	515	{
417		- resource_size_t nq_base;
418	516	int rc = -1;
419	517
420		- if (cqn_handler)
421		- nq->cqn_handler = cqn_handler;
422		-
423		- if (srqn_handler)
424		- nq->srqn_handler = srqn_handler;
	518	+ nq->pdev = pdev;
	519	+ nq->cqn_handler = cqn_handler;
	520	+ nq->srqn_handler = srqn_handler;
425	521
426	522	/* Have a task to schedule CQ notifiers in post send case */
427	523	nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq");
428	524	if (!nq->cqn_wq)
429	525	return -ENOMEM;
430	526
431		- nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
432		- nq->bar_reg_off = bar_reg_offset;
433		- nq_base = pci_resource_start(pdev, nq->bar_reg);
434		- if (!nq_base) {
435		- rc = -ENOMEM;
	527	+ rc = bnxt_qplib_map_nq_db(nq, bar_reg_offset);
	528	+ if (rc)
436	529	goto fail;
437		- }
438		- nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4);
439		- if (!nq->bar_reg_iomem) {
440		- rc = -ENOMEM;
441		- goto fail;
442		- }
443	530
444	531	rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true);
445	532	if (rc) {
446	533	dev_err(&nq->pdev->dev,
447		- "QPLIB: Failed to request irq for nq-idx %d", nq_idx);
	534	+ "Failed to request irq for nq-idx %d\n", nq_idx);
448	535	goto fail;
449	536	}
450	537
..	..	@@ -457,50 +544,39 @@
457	544	void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
458	545	{
459	546	if (nq->hwq.max_elements) {
460		- bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);
	547	+ bnxt_qplib_free_hwq(nq->res, &nq->hwq);
461	548	nq->hwq.max_elements = 0;
462	549	}
463	550	}
464	551
465		-int bnxt_qplib_alloc_nq(struct pci_dev pdev, struct bnxt_qplib_nq nq)
	552	+int bnxt_qplib_alloc_nq(struct bnxt_qplib_res res, struct bnxt_qplib_nq nq)
466	553	{
467		- nq->pdev = pdev;
	554	+ struct bnxt_qplib_hwq_attr hwq_attr = {};
	555	+ struct bnxt_qplib_sg_info sginfo = {};
	556	+
	557	+ nq->pdev = res->pdev;
	558	+ nq->res = res;
468	559	if (!nq->hwq.max_elements \|\|
469	560	nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
470	561	nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
471	562
472		- if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0,
473		- &nq->hwq.max_elements,
474		- BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,
475		- PAGE_SIZE, HWQ_TYPE_L2_CMPL))
	563	+ sginfo.pgsize = PAGE_SIZE;
	564	+ sginfo.pgshft = PAGE_SHIFT;
	565	+ hwq_attr.res = res;
	566	+ hwq_attr.sginfo = &sginfo;
	567	+ hwq_attr.depth = nq->hwq.max_elements;
	568	+ hwq_attr.stride = sizeof(struct nq_base);
	569	+ hwq_attr.type = bnxt_qplib_get_hwq_type(nq->res);
	570	+ if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) {
	571	+ dev_err(&nq->pdev->dev, "FP NQ allocation failed");
476	572	return -ENOMEM;
477		-
	573	+ }
478	574	nq->budget = 8;
479	575	return 0;
480	576	}
481	577
482	578	/* SRQ */
483		-static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type)
484		-{
485		- struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
486		- struct dbr_dbr db_msg = { 0 };
487		- void __iomem *db;
488		- u32 sw_prod = 0;
489		-
490		- /* Ring DB */
491		- sw_prod = (arm_type == DBR_DBR_TYPE_SRQ_ARM) ? srq->threshold :
492		- HWQ_CMP(srq_hwq->prod, srq_hwq);
493		- db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
494		- DBR_DBR_INDEX_MASK);
495		- db_msg.type_xid = cpu_to_le32(((srq->id << DBR_DBR_XID_SFT) &
496		- DBR_DBR_XID_MASK) \| arm_type);
497		- db = (arm_type == DBR_DBR_TYPE_SRQ_ARMENA) ?
498		- srq->dbr_base : srq->dpi->dbr;
499		- wmb(); /* barrier before db ring */
500		- __iowrite64_copy(db, &db_msg, sizeof(db_msg) / sizeof(u64));
501		-}
502		-
503		-int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
	579	+void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
504	580	struct bnxt_qplib_srq *srq)
505	581	{
506	582	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
..	..	@@ -514,31 +590,32 @@
514	590	/* Configure the request */
515	591	req.srq_cid = cpu_to_le32(srq->id);
516	592
517		- rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
518		- (void *)&resp, NULL, 0);
519		- if (rc)
520		- return rc;
521		-
522		- bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
	593	+ rc = bnxt_qplib_rcfw_send_message(rcfw, (struct cmdq_base *)&req,
	594	+ (struct creq_base *)&resp, NULL, 0);
523	595	kfree(srq->swq);
524		- return 0;
	596	+ if (rc)
	597	+ return;
	598	+ bnxt_qplib_free_hwq(res, &srq->hwq);
525	599	}
526	600
527	601	int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
528	602	struct bnxt_qplib_srq *srq)
529	603	{
530	604	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
531		- struct cmdq_create_srq req;
	605	+ struct bnxt_qplib_hwq_attr hwq_attr = {};
532	606	struct creq_create_srq_resp resp;
	607	+ struct cmdq_create_srq req;
533	608	struct bnxt_qplib_pbl *pbl;
534	609	u16 cmd_flags = 0;
	610	+ u16 pg_sz_lvl;
535	611	int rc, idx;
536	612
537		- srq->hwq.max_elements = srq->max_wqe;
538		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, &srq->hwq, srq->sglist,
539		- srq->nmap, &srq->hwq.max_elements,
540		- BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
541		- PAGE_SIZE, HWQ_TYPE_QUEUE);
	613	+ hwq_attr.res = res;
	614	+ hwq_attr.sginfo = &srq->sg_info;
	615	+ hwq_attr.depth = srq->max_wqe;
	616	+ hwq_attr.stride = srq->wqe_size;
	617	+ hwq_attr.type = HWQ_TYPE_QUEUE;
	618	+ rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr);
542	619	if (rc)
543	620	goto exit;
544	621
..	..	@@ -557,22 +634,11 @@
557	634
558	635	req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements);
559	636	pbl = &srq->hwq.pbl[PBL_LVL_0];
560		- req.pg_size_lvl = cpu_to_le16((((u16)srq->hwq.level &
561		- CMDQ_CREATE_SRQ_LVL_MASK) <<
562		- CMDQ_CREATE_SRQ_LVL_SFT) \|
563		- (pbl->pg_size == ROCE_PG_SIZE_4K ?
564		- CMDQ_CREATE_SRQ_PG_SIZE_PG_4K :
565		- pbl->pg_size == ROCE_PG_SIZE_8K ?
566		- CMDQ_CREATE_SRQ_PG_SIZE_PG_8K :
567		- pbl->pg_size == ROCE_PG_SIZE_64K ?
568		- CMDQ_CREATE_SRQ_PG_SIZE_PG_64K :
569		- pbl->pg_size == ROCE_PG_SIZE_2M ?
570		- CMDQ_CREATE_SRQ_PG_SIZE_PG_2M :
571		- pbl->pg_size == ROCE_PG_SIZE_8M ?
572		- CMDQ_CREATE_SRQ_PG_SIZE_PG_8M :
573		- pbl->pg_size == ROCE_PG_SIZE_1G ?
574		- CMDQ_CREATE_SRQ_PG_SIZE_PG_1G :
575		- CMDQ_CREATE_SRQ_PG_SIZE_PG_4K));
	637	+ pg_sz_lvl = ((u16)bnxt_qplib_base_pg_size(&srq->hwq) <<
	638	+ CMDQ_CREATE_SRQ_PG_SIZE_SFT);
	639	+ pg_sz_lvl \|= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK) <<
	640	+ CMDQ_CREATE_SRQ_LVL_SFT;
	641	+ req.pg_size_lvl = cpu_to_le16(pg_sz_lvl);
576	642	req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
577	643	req.pd_id = cpu_to_le32(srq->pd->id);
578	644	req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);
..	..	@@ -590,14 +656,18 @@
590	656	srq->swq[srq->last_idx].next_idx = -1;
591	657
592	658	srq->id = le32_to_cpu(resp.xid);
593		- srq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
	659	+ srq->dbinfo.hwq = &srq->hwq;
	660	+ srq->dbinfo.xid = srq->id;
	661	+ srq->dbinfo.db = srq->dpi->dbr;
	662	+ srq->dbinfo.max_slot = 1;
	663	+ srq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem;
594	664	if (srq->threshold)
595		- bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARMENA);
	665	+ bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA);
596	666	srq->arm_req = false;
597	667
598	668	return 0;
599	669	fail:
600		- bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
	670	+ bnxt_qplib_free_hwq(res, &srq->hwq);
601	671	kfree(srq->swq);
602	672	exit:
603	673	return rc;
..	..	@@ -616,7 +686,7 @@
616	686	srq_hwq->max_elements - sw_cons + sw_prod;
617	687	if (count > srq->threshold) {
618	688	srq->arm_req = false;
619		- bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM);
	689	+ bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold);
620	690	} else {
621	691	/* Deferred arming */
622	692	srq->arm_req = true;
..	..	@@ -657,15 +727,15 @@
657	727	struct bnxt_qplib_swqe *wqe)
658	728	{
659	729	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
660		- struct rq_wqe srqe, *srqe_ptr;
	730	+ struct rq_wqe *srqe;
661	731	struct sq_sge *hw_sge;
662	732	u32 sw_prod, sw_cons, count = 0;
663	733	int i, rc = 0, next;
664	734
665	735	spin_lock(&srq_hwq->lock);
666	736	if (srq->start_idx == srq->last_idx) {
667		- dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!",
668		- srq->id);
	737	+ dev_err(&srq_hwq->pdev->dev,
	738	+ "FP: SRQ (0x%x) is full!\n", srq->id);
669	739	rc = -EINVAL;
670	740	spin_unlock(&srq_hwq->lock);
671	741	goto done;
..	..	@@ -675,9 +745,8 @@
675	745	spin_unlock(&srq_hwq->lock);
676	746
677	747	sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
678		- srqe_ptr = (struct rq_wqe **)srq_hwq->pbl_ptr;
679		- srqe = &srqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
680		- memset(srqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
	748	+ srqe = bnxt_qplib_get_qe(srq_hwq, sw_prod, NULL);
	749	+ memset(srqe, 0, srq->wqe_size);
681	750	/* Calculate wqe_size16 and data_len */
682	751	for (i = 0, hw_sge = (struct sq_sge *)srqe->data;
683	752	i < wqe->num_sge; i++, hw_sge++) {
..	..	@@ -705,27 +774,52 @@
705	774	srq_hwq->max_elements - sw_cons + sw_prod;
706	775	spin_unlock(&srq_hwq->lock);
707	776	/* Ring DB */
708		- bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ);
	777	+ bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ);
709	778	if (srq->arm_req == true && count > srq->threshold) {
710	779	srq->arm_req = false;
711		- bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM);
	780	+ bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold);
712	781	}
713	782	done:
714	783	return rc;
715	784	}
716	785
717	786	/* QP */
	787	+
	788	+static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que)
	789	+{
	790	+ int rc = 0;
	791	+ int indx;
	792	+
	793	+ que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL);
	794	+ if (!que->swq) {
	795	+ rc = -ENOMEM;
	796	+ goto out;
	797	+ }
	798	+
	799	+ que->swq_start = 0;
	800	+ que->swq_last = que->max_wqe - 1;
	801	+ for (indx = 0; indx < que->max_wqe; indx++)
	802	+ que->swq[indx].next_idx = indx + 1;
	803	+ que->swq[que->swq_last].next_idx = 0; /* Make it circular */
	804	+ que->swq_last = 0;
	805	+out:
	806	+ return rc;
	807	+}
	808	+
718	809	int bnxt_qplib_create_qp1(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp)
719	810	{
	811	+ struct bnxt_qplib_hwq_attr hwq_attr = {};
720	812	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
721		- struct cmdq_create_qp1 req;
722		- struct creq_create_qp1_resp resp;
723		- struct bnxt_qplib_pbl *pbl;
724	813	struct bnxt_qplib_q *sq = &qp->sq;
725	814	struct bnxt_qplib_q *rq = &qp->rq;
726		- int rc;
	815	+ struct creq_create_qp1_resp resp;
	816	+ struct cmdq_create_qp1 req;
	817	+ struct bnxt_qplib_pbl *pbl;
727	818	u16 cmd_flags = 0;
728	819	u32 qp_flags = 0;
	820	+ u8 pg_sz_lvl;
	821	+ u32 tbl_indx;
	822	+ int rc;
729	823
730	824	RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
731	825
..	..	@@ -735,98 +829,65 @@
735	829	req.qp_handle = cpu_to_le64(qp->qp_handle);
736	830
737	831	/* SQ */
738		- sq->hwq.max_elements = sq->max_wqe;
739		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0,
740		- &sq->hwq.max_elements,
741		- BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,
742		- PAGE_SIZE, HWQ_TYPE_QUEUE);
	832	+ hwq_attr.res = res;
	833	+ hwq_attr.sginfo = &sq->sg_info;
	834	+ hwq_attr.stride = sizeof(struct sq_sge);
	835	+ hwq_attr.depth = bnxt_qplib_get_depth(sq);
	836	+ hwq_attr.type = HWQ_TYPE_QUEUE;
	837	+ rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
743	838	if (rc)
744	839	goto exit;
745	840
746		- sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
747		- if (!sq->swq) {
748		- rc = -ENOMEM;
	841	+ rc = bnxt_qplib_alloc_init_swq(sq);
	842	+ if (rc)
749	843	goto fail_sq;
750		- }
	844	+
	845	+ req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode));
751	846	pbl = &sq->hwq.pbl[PBL_LVL_0];
752	847	req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
753		- req.sq_pg_size_sq_lvl =
754		- ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK)
755		- << CMDQ_CREATE_QP1_SQ_LVL_SFT) \|
756		- (pbl->pg_size == ROCE_PG_SIZE_4K ?
757		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K :
758		- pbl->pg_size == ROCE_PG_SIZE_8K ?
759		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K :
760		- pbl->pg_size == ROCE_PG_SIZE_64K ?
761		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K :
762		- pbl->pg_size == ROCE_PG_SIZE_2M ?
763		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M :
764		- pbl->pg_size == ROCE_PG_SIZE_8M ?
765		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M :
766		- pbl->pg_size == ROCE_PG_SIZE_1G ?
767		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G :
768		- CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K);
769		-
770		- if (qp->scq)
771		- req.scq_cid = cpu_to_le32(qp->scq->id);
772		-
773		- qp_flags \|= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
	848	+ pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
	849	+ CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT);
	850	+ pg_sz_lvl \|= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK);
	851	+ req.sq_pg_size_sq_lvl = pg_sz_lvl;
	852	+ req.sq_fwo_sq_sge =
	853	+ cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
	854	+ CMDQ_CREATE_QP1_SQ_SGE_SFT);
	855	+ req.scq_cid = cpu_to_le32(qp->scq->id);
774	856
775	857	/* RQ */
776	858	if (rq->max_wqe) {
777		- rq->hwq.max_elements = qp->rq.max_wqe;
778		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0,
779		- &rq->hwq.max_elements,
780		- BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
781		- PAGE_SIZE, HWQ_TYPE_QUEUE);
	859	+ hwq_attr.res = res;
	860	+ hwq_attr.sginfo = &rq->sg_info;
	861	+ hwq_attr.stride = sizeof(struct sq_sge);
	862	+ hwq_attr.depth = bnxt_qplib_get_depth(rq);
	863	+ hwq_attr.type = HWQ_TYPE_QUEUE;
	864	+ rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
782	865	if (rc)
783		- goto fail_sq;
784		-
785		- rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
786		- GFP_KERNEL);
787		- if (!rq->swq) {
788		- rc = -ENOMEM;
	866	+ goto sq_swq;
	867	+ rc = bnxt_qplib_alloc_init_swq(rq);
	868	+ if (rc)
789	869	goto fail_rq;
790		- }
	870	+ req.rq_size = cpu_to_le32(rq->max_wqe);
791	871	pbl = &rq->hwq.pbl[PBL_LVL_0];
792	872	req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
793		- req.rq_pg_size_rq_lvl =
794		- ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
795		- CMDQ_CREATE_QP1_RQ_LVL_SFT) \|
796		- (pbl->pg_size == ROCE_PG_SIZE_4K ?
797		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K :
798		- pbl->pg_size == ROCE_PG_SIZE_8K ?
799		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K :
800		- pbl->pg_size == ROCE_PG_SIZE_64K ?
801		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K :
802		- pbl->pg_size == ROCE_PG_SIZE_2M ?
803		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M :
804		- pbl->pg_size == ROCE_PG_SIZE_8M ?
805		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M :
806		- pbl->pg_size == ROCE_PG_SIZE_1G ?
807		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G :
808		- CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K);
809		- if (qp->rcq)
810		- req.rcq_cid = cpu_to_le32(qp->rcq->id);
	873	+ pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
	874	+ CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT);
	875	+ pg_sz_lvl \|= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK);
	876	+ req.rq_pg_size_rq_lvl = pg_sz_lvl;
	877	+ req.rq_fwo_rq_sge =
	878	+ cpu_to_le16((rq->max_sge &
	879	+ CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
	880	+ CMDQ_CREATE_QP1_RQ_SGE_SFT);
811	881	}
812		-
	882	+ req.rcq_cid = cpu_to_le32(qp->rcq->id);
813	883	/* Header buffer - allow hdr_buf pass in */
814	884	rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
815	885	if (rc) {
816	886	rc = -ENOMEM;
817		- goto fail;
	887	+ goto rq_rwq;
818	888	}
	889	+ qp_flags \|= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
819	890	req.qp_flags = cpu_to_le32(qp_flags);
820		- req.sq_size = cpu_to_le32(sq->hwq.max_elements);
821		- req.rq_size = cpu_to_le32(rq->hwq.max_elements);
822		-
823		- req.sq_fwo_sq_sge =
824		- cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
825		- CMDQ_CREATE_QP1_SQ_SGE_SFT);
826		- req.rq_fwo_rq_sge =
827		- cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
828		- CMDQ_CREATE_QP1_RQ_SGE_SFT);
829		-
830	891	req.pd_id = cpu_to_le32(qp->pd->id);
831	892
832	893	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
..	..	@@ -836,38 +897,72 @@
836	897
837	898	qp->id = le32_to_cpu(resp.xid);
838	899	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
839		- rcfw->qp_tbl[qp->id].qp_id = qp->id;
840		- rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
	900	+ qp->cctx = res->cctx;
	901	+ sq->dbinfo.hwq = &sq->hwq;
	902	+ sq->dbinfo.xid = qp->id;
	903	+ sq->dbinfo.db = qp->dpi->dbr;
	904	+ sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode);
	905	+ if (rq->max_wqe) {
	906	+ rq->dbinfo.hwq = &rq->hwq;
	907	+ rq->dbinfo.xid = qp->id;
	908	+ rq->dbinfo.db = qp->dpi->dbr;
	909	+ rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size);
	910	+ }
	911	+ tbl_indx = map_qp_id_to_tbl_indx(qp->id, rcfw);
	912	+ rcfw->qp_tbl[tbl_indx].qp_id = qp->id;
	913	+ rcfw->qp_tbl[tbl_indx].qp_handle = (void *)qp;
841	914
842	915	return 0;
843	916
844	917	fail:
845	918	bnxt_qplib_free_qp_hdr_buf(res, qp);
846		-fail_rq:
847		- bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
	919	+rq_rwq:
848	920	kfree(rq->swq);
849		-fail_sq:
850		- bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
	921	+fail_rq:
	922	+ bnxt_qplib_free_hwq(res, &rq->hwq);
	923	+sq_swq:
851	924	kfree(sq->swq);
	925	+fail_sq:
	926	+ bnxt_qplib_free_hwq(res, &sq->hwq);
852	927	exit:
853	928	return rc;
	929	+}
	930	+
	931	+static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size)
	932	+{
	933	+ struct bnxt_qplib_hwq *hwq;
	934	+ struct bnxt_qplib_q *sq;
	935	+ u64 fpsne, psn_pg;
	936	+ u16 indx_pad = 0;
	937	+
	938	+ sq = &qp->sq;
	939	+ hwq = &sq->hwq;
	940	+ /* First psn entry */
	941	+ fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->depth, &psn_pg);
	942	+ if (!IS_ALIGNED(fpsne, PAGE_SIZE))
	943	+ indx_pad = (fpsne & ~PAGE_MASK) / size;
	944	+ hwq->pad_pgofft = indx_pad;
	945	+ hwq->pad_pg = (u64 *)psn_pg;
	946	+ hwq->pad_stride = size;
854	947	}
855	948
856	949	int bnxt_qplib_create_qp(struct bnxt_qplib_res res, struct bnxt_qplib_qp qp)
857	950	{
858	951	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
859		- struct sq_send hw_sq_send_hdr, *hw_sq_send_ptr;
860		- struct cmdq_create_qp req;
861		- struct creq_create_qp_resp resp;
862		- struct bnxt_qplib_pbl *pbl;
863		- struct sq_psn_search **psn_search_ptr;
864		- unsigned long int psn_search, poff = 0;
	952	+ struct bnxt_qplib_hwq_attr hwq_attr = {};
	953	+ struct bnxt_qplib_sg_info sginfo = {};
865	954	struct bnxt_qplib_q *sq = &qp->sq;
866	955	struct bnxt_qplib_q *rq = &qp->rq;
	956	+ struct creq_create_qp_resp resp;
	957	+ int rc, req_size, psn_sz = 0;
867	958	struct bnxt_qplib_hwq *xrrq;
868		- int i, rc, req_size, psn_sz;
869		- u16 cmd_flags = 0, max_ssge;
870		- u32 sw_prod, qp_flags = 0;
	959	+ struct bnxt_qplib_pbl *pbl;
	960	+ struct cmdq_create_qp req;
	961	+ u16 cmd_flags = 0;
	962	+ u32 qp_flags = 0;
	963	+ u8 pg_sz_lvl;
	964	+ u32 tbl_indx;
	965	+ u16 nsge;
871	966
872	967	RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
873	968
..	..	@@ -877,142 +972,86 @@
877	972	req.qp_handle = cpu_to_le64(qp->qp_handle);
878	973
879	974	/* SQ */
880		- psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ?
881		- sizeof(struct sq_psn_search) : 0;
882		- sq->hwq.max_elements = sq->max_wqe;
883		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist,
884		- sq->nmap, &sq->hwq.max_elements,
885		- BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,
886		- psn_sz,
887		- PAGE_SIZE, HWQ_TYPE_QUEUE);
	975	+ if (qp->type == CMDQ_CREATE_QP_TYPE_RC) {
	976	+ psn_sz = bnxt_qplib_is_chip_gen_p5(res->cctx) ?
	977	+ sizeof(struct sq_psn_search_ext) :
	978	+ sizeof(struct sq_psn_search);
	979	+ }
	980	+
	981	+ hwq_attr.res = res;
	982	+ hwq_attr.sginfo = &sq->sg_info;
	983	+ hwq_attr.stride = sizeof(struct sq_sge);
	984	+ hwq_attr.depth = bnxt_qplib_get_depth(sq);
	985	+ hwq_attr.aux_stride = psn_sz;
	986	+ hwq_attr.aux_depth = bnxt_qplib_set_sq_size(sq, qp->wqe_mode);
	987	+ hwq_attr.type = HWQ_TYPE_QUEUE;
	988	+ rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
888	989	if (rc)
889	990	goto exit;
890	991
891		- sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
892		- if (!sq->swq) {
893		- rc = -ENOMEM;
	992	+ rc = bnxt_qplib_alloc_init_swq(sq);
	993	+ if (rc)
894	994	goto fail_sq;
895		- }
896		- hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
897		- if (psn_sz) {
898		- psn_search_ptr = (struct sq_psn_search **)
899		- &hw_sq_send_ptr[get_sqe_pg
900		- (sq->hwq.max_elements)];
901		- psn_search = (unsigned long int)
902		- &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)]
903		- [get_sqe_idx(sq->hwq.max_elements)];
904		- if (psn_search & ~PAGE_MASK) {
905		- /* If the psn_search does not start on a page boundary,
906		- * then calculate the offset
907		- */
908		- poff = (psn_search & ~PAGE_MASK) /
909		- BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;
910		- }
911		- for (i = 0; i < sq->hwq.max_elements; i++)
912		- sq->swq[i].psn_search =
913		- &psn_search_ptr[get_psne_pg(i + poff)]
914		- [get_psne_idx(i + poff)];
915		- }
	995	+
	996	+ if (psn_sz)
	997	+ bnxt_qplib_init_psn_ptr(qp, psn_sz);
	998	+
	999	+ req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode));
916	1000	pbl = &sq->hwq.pbl[PBL_LVL_0];
917	1001	req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
918		- req.sq_pg_size_sq_lvl =
919		- ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK)
920		- << CMDQ_CREATE_QP_SQ_LVL_SFT) \|
921		- (pbl->pg_size == ROCE_PG_SIZE_4K ?
922		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K :
923		- pbl->pg_size == ROCE_PG_SIZE_8K ?
924		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K :
925		- pbl->pg_size == ROCE_PG_SIZE_64K ?
926		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K :
927		- pbl->pg_size == ROCE_PG_SIZE_2M ?
928		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M :
929		- pbl->pg_size == ROCE_PG_SIZE_8M ?
930		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M :
931		- pbl->pg_size == ROCE_PG_SIZE_1G ?
932		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G :
933		- CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K);
	1002	+ pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
	1003	+ CMDQ_CREATE_QP_SQ_PG_SIZE_SFT);
	1004	+ pg_sz_lvl \|= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK);
	1005	+ req.sq_pg_size_sq_lvl = pg_sz_lvl;
	1006	+ req.sq_fwo_sq_sge =
	1007	+ cpu_to_le16(((sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK) <<
	1008	+ CMDQ_CREATE_QP_SQ_SGE_SFT) \| 0);
	1009	+ req.scq_cid = cpu_to_le32(qp->scq->id);
934	1010
935		- /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */
936		- hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
937		- for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) {
938		- hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
939		- [get_sqe_idx(sw_prod)];
940		- hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID;
	1011	+ /* RQ */
	1012	+ if (!qp->srq) {
	1013	+ hwq_attr.res = res;
	1014	+ hwq_attr.sginfo = &rq->sg_info;
	1015	+ hwq_attr.stride = sizeof(struct sq_sge);
	1016	+ hwq_attr.depth = bnxt_qplib_get_depth(rq);
	1017	+ hwq_attr.aux_stride = 0;
	1018	+ hwq_attr.aux_depth = 0;
	1019	+ hwq_attr.type = HWQ_TYPE_QUEUE;
	1020	+ rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
	1021	+ if (rc)
	1022	+ goto sq_swq;
	1023	+ rc = bnxt_qplib_alloc_init_swq(rq);
	1024	+ if (rc)
	1025	+ goto fail_rq;
	1026	+
	1027	+ req.rq_size = cpu_to_le32(rq->max_wqe);
	1028	+ pbl = &rq->hwq.pbl[PBL_LVL_0];
	1029	+ req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
	1030	+ pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
	1031	+ CMDQ_CREATE_QP_RQ_PG_SIZE_SFT);
	1032	+ pg_sz_lvl \|= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK);
	1033	+ req.rq_pg_size_rq_lvl = pg_sz_lvl;
	1034	+ nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
	1035	+ 6 : rq->max_sge;
	1036	+ req.rq_fwo_rq_sge =
	1037	+ cpu_to_le16(((nsge &
	1038	+ CMDQ_CREATE_QP_RQ_SGE_MASK) <<
	1039	+ CMDQ_CREATE_QP_RQ_SGE_SFT) \| 0);
	1040	+ } else {
	1041	+ /* SRQ */
	1042	+ qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
	1043	+ req.srq_cid = cpu_to_le32(qp->srq->id);
941	1044	}
942		-
943		- if (qp->scq)
944		- req.scq_cid = cpu_to_le32(qp->scq->id);
	1045	+ req.rcq_cid = cpu_to_le32(qp->rcq->id);
945	1046
946	1047	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
947	1048	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
948	1049	if (qp->sig_type)
949	1050	qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
950		-
951		- /* RQ */
952		- if (rq->max_wqe) {
953		- rq->hwq.max_elements = rq->max_wqe;
954		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist,
955		- rq->nmap, &rq->hwq.max_elements,
956		- BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
957		- PAGE_SIZE, HWQ_TYPE_QUEUE);
958		- if (rc)
959		- goto fail_sq;
960		-
961		- rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
962		- GFP_KERNEL);
963		- if (!rq->swq) {
964		- rc = -ENOMEM;
965		- goto fail_rq;
966		- }
967		- pbl = &rq->hwq.pbl[PBL_LVL_0];
968		- req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
969		- req.rq_pg_size_rq_lvl =
970		- ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
971		- CMDQ_CREATE_QP_RQ_LVL_SFT) \|
972		- (pbl->pg_size == ROCE_PG_SIZE_4K ?
973		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K :
974		- pbl->pg_size == ROCE_PG_SIZE_8K ?
975		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K :
976		- pbl->pg_size == ROCE_PG_SIZE_64K ?
977		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K :
978		- pbl->pg_size == ROCE_PG_SIZE_2M ?
979		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M :
980		- pbl->pg_size == ROCE_PG_SIZE_8M ?
981		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M :
982		- pbl->pg_size == ROCE_PG_SIZE_1G ?
983		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :
984		- CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);
985		- } else {
986		- /* SRQ */
987		- if (qp->srq) {
988		- qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
989		- req.srq_cid = cpu_to_le32(qp->srq->id);
990		- }
991		- }
992		-
993		- if (qp->rcq)
994		- req.rcq_cid = cpu_to_le32(qp->rcq->id);
	1051	+ if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
	1052	+ qp_flags \|= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED;
995	1053	req.qp_flags = cpu_to_le32(qp_flags);
996		- req.sq_size = cpu_to_le32(sq->hwq.max_elements);
997		- req.rq_size = cpu_to_le32(rq->hwq.max_elements);
998		- qp->sq_hdr_buf = NULL;
999		- qp->rq_hdr_buf = NULL;
1000	1054
1001		- rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
1002		- if (rc)
1003		- goto fail_rq;
1004		-
1005		- /* CTRL-22434: Irrespective of the requested SGE count on the SQ
1006		- * always create the QP with max send sges possible if the requested
1007		- * inline size is greater than 0.
1008		- */
1009		- max_ssge = qp->max_inline_data ? 6 : sq->max_sge;
1010		- req.sq_fwo_sq_sge = cpu_to_le16(
1011		- ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
1012		- << CMDQ_CREATE_QP_SQ_SGE_SFT) \| 0);
1013		- req.rq_fwo_rq_sge = cpu_to_le16(
1014		- ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK)
1015		- << CMDQ_CREATE_QP_RQ_SGE_SFT) \| 0);
1016	1055	/* ORRQ and IRRQ */
1017	1056	if (psn_sz) {
1018	1057	xrrq = &qp->orrq;
..	..	@@ -1021,12 +1060,19 @@
1021	1060	req_size = xrrq->max_elements *
1022	1061	BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
1023	1062	req_size &= ~(PAGE_SIZE - 1);
1024		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
1025		- &xrrq->max_elements,
1026		- BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,
1027		- 0, req_size, HWQ_TYPE_CTX);
	1063	+ sginfo.pgsize = req_size;
	1064	+ sginfo.pgshft = PAGE_SHIFT;
	1065	+
	1066	+ hwq_attr.res = res;
	1067	+ hwq_attr.sginfo = &sginfo;
	1068	+ hwq_attr.depth = xrrq->max_elements;
	1069	+ hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE;
	1070	+ hwq_attr.aux_stride = 0;
	1071	+ hwq_attr.aux_depth = 0;
	1072	+ hwq_attr.type = HWQ_TYPE_CTX;
	1073	+ rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
1028	1074	if (rc)
1029		- goto fail_buf_free;
	1075	+ goto rq_swq;
1030	1076	pbl = &xrrq->pbl[PBL_LVL_0];
1031	1077	req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
1032	1078
..	..	@@ -1036,11 +1082,10 @@
1036	1082	req_size = xrrq->max_elements *
1037	1083	BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
1038	1084	req_size &= ~(PAGE_SIZE - 1);
1039		-
1040		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
1041		- &xrrq->max_elements,
1042		- BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,
1043		- 0, req_size, HWQ_TYPE_CTX);
	1085	+ sginfo.pgsize = req_size;
	1086	+ hwq_attr.depth = xrrq->max_elements;
	1087	+ hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE;
	1088	+ rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
1044	1089	if (rc)
1045	1090	goto fail_orrq;
1046	1091
..	..	@@ -1058,25 +1103,34 @@
1058	1103	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
1059	1104	INIT_LIST_HEAD(&qp->sq_flush);
1060	1105	INIT_LIST_HEAD(&qp->rq_flush);
1061		- rcfw->qp_tbl[qp->id].qp_id = qp->id;
1062		- rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
	1106	+ qp->cctx = res->cctx;
	1107	+ sq->dbinfo.hwq = &sq->hwq;
	1108	+ sq->dbinfo.xid = qp->id;
	1109	+ sq->dbinfo.db = qp->dpi->dbr;
	1110	+ sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode);
	1111	+ if (rq->max_wqe) {
	1112	+ rq->dbinfo.hwq = &rq->hwq;
	1113	+ rq->dbinfo.xid = qp->id;
	1114	+ rq->dbinfo.db = qp->dpi->dbr;
	1115	+ rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size);
	1116	+ }
	1117	+ tbl_indx = map_qp_id_to_tbl_indx(qp->id, rcfw);
	1118	+ rcfw->qp_tbl[tbl_indx].qp_id = qp->id;
	1119	+ rcfw->qp_tbl[tbl_indx].qp_handle = (void *)qp;
1063	1120
1064	1121	return 0;
1065		-
1066	1122	fail:
1067		- if (qp->irrq.max_elements)
1068		- bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
	1123	+ bnxt_qplib_free_hwq(res, &qp->irrq);
1069	1124	fail_orrq:
1070		- if (qp->orrq.max_elements)
1071		- bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
1072		-fail_buf_free:
1073		- bnxt_qplib_free_qp_hdr_buf(res, qp);
1074		-fail_rq:
1075		- bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
	1125	+ bnxt_qplib_free_hwq(res, &qp->orrq);
	1126	+rq_swq:
1076	1127	kfree(rq->swq);
1077		-fail_sq:
1078		- bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
	1128	+fail_rq:
	1129	+ bnxt_qplib_free_hwq(res, &rq->hwq);
	1130	+sq_swq:
1079	1131	kfree(sq->swq);
	1132	+fail_sq:
	1133	+ bnxt_qplib_free_hwq(res, &sq->hwq);
1080	1134	exit:
1081	1135	return rc;
1082	1136	}
..	..	@@ -1326,7 +1380,7 @@
1326	1380	}
1327	1381	}
1328	1382	if (i == res->sgid_tbl.max)
1329		- dev_warn(&res->pdev->dev, "QPLIB: SGID not found??");
	1383	+ dev_warn(&res->pdev->dev, "SGID not found??\n");
1330	1384
1331	1385	qp->ah.hop_limit = sb->hop_limit;
1332	1386	qp->ah.traffic_class = sb->traffic_class;
..	..	@@ -1362,12 +1416,11 @@
1362	1416	static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
1363	1417	{
1364	1418	struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1365		- struct cq_base hw_cqe, *hw_cqe_ptr;
	1419	+ struct cq_base *hw_cqe;
1366	1420	int i;
1367	1421
1368	1422	for (i = 0; i < cq_hwq->max_elements; i++) {
1369		- hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr;
1370		- hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)];
	1423	+ hw_cqe = bnxt_qplib_get_qe(cq_hwq, i, NULL);
1371	1424	if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
1372	1425	continue;
1373	1426	/*
..	..	@@ -1408,10 +1461,12 @@
1408	1461	struct cmdq_destroy_qp req;
1409	1462	struct creq_destroy_qp_resp resp;
1410	1463	u16 cmd_flags = 0;
	1464	+ u32 tbl_indx;
1411	1465	int rc;
1412	1466
1413		- rcfw->qp_tbl[qp->id].qp_id = BNXT_QPLIB_QP_ID_INVALID;
1414		- rcfw->qp_tbl[qp->id].qp_handle = NULL;
	1467	+ tbl_indx = map_qp_id_to_tbl_indx(qp->id, rcfw);
	1468	+ rcfw->qp_tbl[tbl_indx].qp_id = BNXT_QPLIB_QP_ID_INVALID;
	1469	+ rcfw->qp_tbl[tbl_indx].qp_handle = NULL;
1415	1470
1416	1471	RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
1417	1472
..	..	@@ -1419,8 +1474,8 @@
1419	1474	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1420	1475	(void *)&resp, NULL, 0);
1421	1476	if (rc) {
1422		- rcfw->qp_tbl[qp->id].qp_id = qp->id;
1423		- rcfw->qp_tbl[qp->id].qp_handle = qp;
	1477	+ rcfw->qp_tbl[tbl_indx].qp_id = qp->id;
	1478	+ rcfw->qp_tbl[tbl_indx].qp_handle = qp;
1424	1479	return rc;
1425	1480	}
1426	1481
..	..	@@ -1431,16 +1486,16 @@
1431	1486	struct bnxt_qplib_qp *qp)
1432	1487	{
1433	1488	bnxt_qplib_free_qp_hdr_buf(res, qp);
1434		- bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
	1489	+ bnxt_qplib_free_hwq(res, &qp->sq.hwq);
1435	1490	kfree(qp->sq.swq);
1436	1491
1437		- bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq);
	1492	+ bnxt_qplib_free_hwq(res, &qp->rq.hwq);
1438	1493	kfree(qp->rq.swq);
1439	1494
1440	1495	if (qp->irrq.max_elements)
1441		- bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
	1496	+ bnxt_qplib_free_hwq(res, &qp->irrq);
1442	1497	if (qp->orrq.max_elements)
1443		- bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
	1498	+ bnxt_qplib_free_hwq(res, &qp->orrq);
1444	1499
1445	1500	}
1446	1501
..	..	@@ -1453,7 +1508,7 @@
1453	1508	memset(sge, 0, sizeof(*sge));
1454	1509
1455	1510	if (qp->sq_hdr_buf) {
1456		- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
	1511	+ sw_prod = sq->swq_start;
1457	1512	sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
1458	1513	sw_prod * qp->sq_hdr_buf_size);
1459	1514	sge->lkey = 0xFFFFFFFF;
..	..	@@ -1467,7 +1522,7 @@
1467	1522	{
1468	1523	struct bnxt_qplib_q *rq = &qp->rq;
1469	1524
1470		- return HWQ_CMP(rq->hwq.prod, &rq->hwq);
	1525	+ return rq->swq_start;
1471	1526	}
1472	1527
1473	1528	dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
..	..	@@ -1484,7 +1539,7 @@
1484	1539	memset(sge, 0, sizeof(*sge));
1485	1540
1486	1541	if (qp->rq_hdr_buf) {
1487		- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
	1542	+ sw_prod = rq->swq_start;
1488	1543	sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
1489	1544	sw_prod * qp->rq_hdr_buf_size);
1490	1545	sge->lkey = 0xFFFFFFFF;
..	..	@@ -1494,144 +1549,264 @@
1494	1549	return NULL;
1495	1550	}
1496	1551
	1552	+static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp,
	1553	+ struct bnxt_qplib_swqe *wqe,
	1554	+ struct bnxt_qplib_swq *swq)
	1555	+{
	1556	+ struct sq_psn_search_ext *psns_ext;
	1557	+ struct sq_psn_search *psns;
	1558	+ u32 flg_npsn;
	1559	+ u32 op_spsn;
	1560	+
	1561	+ if (!swq->psn_search)
	1562	+ return;
	1563	+ psns = swq->psn_search;
	1564	+ psns_ext = swq->psn_ext;
	1565	+
	1566	+ op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
	1567	+ SQ_PSN_SEARCH_START_PSN_MASK);
	1568	+ op_spsn \|= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
	1569	+ SQ_PSN_SEARCH_OPCODE_MASK);
	1570	+ flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
	1571	+ SQ_PSN_SEARCH_NEXT_PSN_MASK);
	1572	+
	1573	+ if (bnxt_qplib_is_chip_gen_p5(qp->cctx)) {
	1574	+ psns_ext->opcode_start_psn = cpu_to_le32(op_spsn);
	1575	+ psns_ext->flags_next_psn = cpu_to_le32(flg_npsn);
	1576	+ psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx);
	1577	+ } else {
	1578	+ psns->opcode_start_psn = cpu_to_le32(op_spsn);
	1579	+ psns->flags_next_psn = cpu_to_le32(flg_npsn);
	1580	+ }
	1581	+}
	1582	+
	1583	+static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp,
	1584	+ struct bnxt_qplib_swqe *wqe,
	1585	+ u16 *idx)
	1586	+{
	1587	+ struct bnxt_qplib_hwq *hwq;
	1588	+ int len, t_len, offt;
	1589	+ bool pull_dst = true;
	1590	+ void *il_dst = NULL;
	1591	+ void *il_src = NULL;
	1592	+ int t_cplen, cplen;
	1593	+ int indx;
	1594	+
	1595	+ hwq = &qp->sq.hwq;
	1596	+ t_len = 0;
	1597	+ for (indx = 0; indx < wqe->num_sge; indx++) {
	1598	+ len = wqe->sg_list[indx].size;
	1599	+ il_src = (void *)wqe->sg_list[indx].addr;
	1600	+ t_len += len;
	1601	+ if (t_len > qp->max_inline_data)
	1602	+ goto bad;
	1603	+ while (len) {
	1604	+ if (pull_dst) {
	1605	+ pull_dst = false;
	1606	+ il_dst = bnxt_qplib_get_prod_qe(hwq, *idx);
	1607	+ (*idx)++;
	1608	+ t_cplen = 0;
	1609	+ offt = 0;
	1610	+ }
	1611	+ cplen = min_t(int, len, sizeof(struct sq_sge));
	1612	+ cplen = min_t(int, cplen,
	1613	+ (sizeof(struct sq_sge) - offt));
	1614	+ memcpy(il_dst, il_src, cplen);
	1615	+ t_cplen += cplen;
	1616	+ il_src += cplen;
	1617	+ il_dst += cplen;
	1618	+ offt += cplen;
	1619	+ len -= cplen;
	1620	+ if (t_cplen == sizeof(struct sq_sge))
	1621	+ pull_dst = true;
	1622	+ }
	1623	+ }
	1624	+
	1625	+ return t_len;
	1626	+bad:
	1627	+ return -ENOMEM;
	1628	+}
	1629	+
	1630	+static u32 bnxt_qplib_put_sges(struct bnxt_qplib_hwq *hwq,
	1631	+ struct bnxt_qplib_sge *ssge,
	1632	+ u16 nsge, u16 *idx)
	1633	+{
	1634	+ struct sq_sge *dsge;
	1635	+ int indx, len = 0;
	1636	+
	1637	+ for (indx = 0; indx < nsge; indx++, (*idx)++) {
	1638	+ dsge = bnxt_qplib_get_prod_qe(hwq, *idx);
	1639	+ dsge->va_or_pa = cpu_to_le64(ssge[indx].addr);
	1640	+ dsge->l_key = cpu_to_le32(ssge[indx].lkey);
	1641	+ dsge->size = cpu_to_le32(ssge[indx].size);
	1642	+ len += ssge[indx].size;
	1643	+ }
	1644	+
	1645	+ return len;
	1646	+}
	1647	+
	1648	+static u16 bnxt_qplib_required_slots(struct bnxt_qplib_qp *qp,
	1649	+ struct bnxt_qplib_swqe *wqe,
	1650	+ u16 wqe_sz, u16 qdf, u8 mode)
	1651	+{
	1652	+ u32 ilsize, bytes;
	1653	+ u16 nsge;
	1654	+ u16 slot;
	1655	+
	1656	+ nsge = wqe->num_sge;
	1657	+ /* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */
	1658	+ bytes = sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge);
	1659	+ if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
	1660	+ ilsize = bnxt_qplib_calc_ilsize(wqe, qp->max_inline_data);
	1661	+ bytes = ALIGN(ilsize, sizeof(struct sq_sge));
	1662	+ bytes += sizeof(struct sq_send_hdr);
	1663	+ }
	1664	+
	1665	+ *qdf = __xlate_qfd(qp->sq.q_full_delta, bytes);
	1666	+ slot = bytes >> 4;
	1667	+ *wqe_sz = slot;
	1668	+ if (mode == BNXT_QPLIB_WQE_MODE_STATIC)
	1669	+ slot = 8;
	1670	+ return slot;
	1671	+}
	1672	+
	1673	+static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_q *sq,
	1674	+ struct bnxt_qplib_swq *swq)
	1675	+{
	1676	+ struct bnxt_qplib_hwq *hwq;
	1677	+ u32 pg_num, pg_indx;
	1678	+ void *buff;
	1679	+ u32 tail;
	1680	+
	1681	+ hwq = &sq->hwq;
	1682	+ if (!hwq->pad_pg)
	1683	+ return;
	1684	+ tail = swq->slot_idx / sq->dbinfo.max_slot;
	1685	+ pg_num = (tail + hwq->pad_pgofft) / (PAGE_SIZE / hwq->pad_stride);
	1686	+ pg_indx = (tail + hwq->pad_pgofft) % (PAGE_SIZE / hwq->pad_stride);
	1687	+ buff = (void )(hwq->pad_pg[pg_num] + pg_indx hwq->pad_stride);
	1688	+ swq->psn_ext = buff;
	1689	+ swq->psn_search = buff;
	1690	+}
	1691	+
1497	1692	void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1498	1693	{
1499	1694	struct bnxt_qplib_q *sq = &qp->sq;
1500		- struct dbr_dbr db_msg = { 0 };
1501		- u32 sw_prod;
1502	1695
1503		- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1504		-
1505		- db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1506		- DBR_DBR_INDEX_MASK);
1507		- db_msg.type_xid =
1508		- cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) \|
1509		- DBR_DBR_TYPE_SQ);
1510		- /* Flush all the WQE writes to HW */
1511		- wmb();
1512		- __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
	1696	+ bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ);
1513	1697	}
1514	1698
1515	1699	int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
1516	1700	struct bnxt_qplib_swqe *wqe)
1517	1701	{
1518		- struct bnxt_qplib_q *sq = &qp->sq;
1519		- struct bnxt_qplib_swq *swq;
1520		- struct sq_send hw_sq_send_hdr, *hw_sq_send_ptr;
1521		- struct sq_sge *hw_sge;
1522	1702	struct bnxt_qplib_nq_work *nq_work = NULL;
1523		- bool sch_handler = false;
1524		- u32 sw_prod;
1525		- u8 wqe_size16;
1526	1703	int i, rc = 0, data_len = 0, pkt_num = 0;
	1704	+ struct bnxt_qplib_q *sq = &qp->sq;
	1705	+ struct bnxt_qplib_hwq *hwq;
	1706	+ struct bnxt_qplib_swq *swq;
	1707	+ bool sch_handler = false;
	1708	+ u16 wqe_sz, qdf = 0;
	1709	+ void *base_hdr;
	1710	+ void *ext_hdr;
1527	1711	__le32 temp32;
	1712	+ u32 wqe_idx;
	1713	+ u32 slots;
	1714	+ u16 idx;
1528	1715
1529		- if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
1530		- if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1531		- sch_handler = true;
1532		- dev_dbg(&sq->hwq.pdev->dev,
1533		- "%s Error QP. Scheduling for poll_cq\n",
1534		- __func__);
1535		- goto queue_err;
1536		- }
	1716	+ hwq = &sq->hwq;
	1717	+ if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS &&
	1718	+ qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) {
	1719	+ dev_err(&hwq->pdev->dev,
	1720	+ "QPLIB: FP: QP (0x%x) is in the 0x%x state",
	1721	+ qp->id, qp->state);
	1722	+ rc = -EINVAL;
	1723	+ goto done;
1537	1724	}
1538	1725
1539		- if (bnxt_qplib_queue_full(sq)) {
1540		- dev_err(&sq->hwq.pdev->dev,
1541		- "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x",
1542		- sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements,
1543		- sq->q_full_delta);
	1726	+ slots = bnxt_qplib_required_slots(qp, wqe, &wqe_sz, &qdf, qp->wqe_mode);
	1727	+ if (bnxt_qplib_queue_full(sq, slots + qdf)) {
	1728	+ dev_err(&hwq->pdev->dev,
	1729	+ "prod = %#x cons = %#x qdepth = %#x delta = %#x\n",
	1730	+ hwq->prod, hwq->cons, hwq->depth, sq->q_full_delta);
1544	1731	rc = -ENOMEM;
1545	1732	goto done;
1546	1733	}
1547		- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1548		- swq = &sq->swq[sw_prod];
	1734	+
	1735	+ swq = bnxt_qplib_get_swqe(sq, &wqe_idx);
	1736	+ bnxt_qplib_pull_psn_buff(sq, swq);
	1737	+
	1738	+ idx = 0;
	1739	+ swq->slot_idx = hwq->prod;
	1740	+ swq->slots = slots;
1549	1741	swq->wr_id = wqe->wr_id;
1550	1742	swq->type = wqe->type;
1551	1743	swq->flags = wqe->flags;
	1744	+ swq->start_psn = sq->psn & BTH_PSN_MASK;
1552	1745	if (qp->sig_type)
1553	1746	swq->flags \|= SQ_SEND_FLAGS_SIGNAL_COMP;
1554		- swq->start_psn = sq->psn & BTH_PSN_MASK;
1555	1747
1556		- hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
1557		- hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
1558		- [get_sqe_idx(sw_prod)];
1559		-
1560		- memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
1561		-
1562		- if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
1563		- /* Copy the inline data */
1564		- if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1565		- dev_warn(&sq->hwq.pdev->dev,
1566		- "QPLIB: Inline data length > 96 detected");
1567		- data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;
1568		- } else {
1569		- data_len = wqe->inline_len;
1570		- }
1571		- memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len);
1572		- wqe_size16 = (data_len + 15) >> 4;
1573		- } else {
1574		- for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data;
1575		- i < wqe->num_sge; i++, hw_sge++) {
1576		- hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1577		- hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1578		- hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1579		- data_len += wqe->sg_list[i].size;
1580		- }
1581		- /* Each SGE entry = 1 WQE size16 */
1582		- wqe_size16 = wqe->num_sge;
1583		- /* HW requires wqe size has room for atleast one SGE even if
1584		- * none was supplied by ULP
1585		- */
1586		- if (!wqe->num_sge)
1587		- wqe_size16++;
	1748	+ if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
	1749	+ sch_handler = true;
	1750	+ dev_dbg(&hwq->pdev->dev,
	1751	+ "%s Error QP. Scheduling for poll_cq\n", __func__);
	1752	+ goto queue_err;
1588	1753	}
1589	1754
	1755	+ base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
	1756	+ ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
	1757	+ memset(base_hdr, 0, sizeof(struct sq_sge));
	1758	+ memset(ext_hdr, 0, sizeof(struct sq_sge));
	1759	+
	1760	+ if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE)
	1761	+ /* Copy the inline data */
	1762	+ data_len = bnxt_qplib_put_inline(qp, wqe, &idx);
	1763	+ else
	1764	+ data_len = bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge,
	1765	+ &idx);
	1766	+ if (data_len < 0)
	1767	+ goto queue_err;
1590	1768	/* Specifics */
1591	1769	switch (wqe->type) {
1592	1770	case BNXT_QPLIB_SWQE_TYPE_SEND:
1593	1771	if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
	1772	+ struct sq_send_raweth_qp1_hdr *sqe = base_hdr;
	1773	+ struct sq_raw_ext_hdr *ext_sqe = ext_hdr;
1594	1774	/* Assemble info for Raw Ethertype QPs */
1595		- struct sq_send_raweth_qp1 *sqe =
1596		- (struct sq_send_raweth_qp1 *)hw_sq_send_hdr;
1597	1775
1598	1776	sqe->wqe_type = wqe->type;
1599	1777	sqe->flags = wqe->flags;
1600		- sqe->wqe_size = wqe_size16 +
1601		- ((offsetof(typeof(*sqe), data) + 15) >> 4);
	1778	+ sqe->wqe_size = wqe_sz;
1602	1779	sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
1603	1780	sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
1604	1781	sqe->length = cpu_to_le32(data_len);
1605		- sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
	1782	+ ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
1606	1783	SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
1607	1784	SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
1608	1785
1609	1786	break;
1610	1787	}
1611		- /* fall thru */
	1788	+ fallthrough;
1612	1789	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
1613	1790	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
1614	1791	{
1615		- struct sq_send sqe = (struct sq_send )hw_sq_send_hdr;
	1792	+ struct sq_ud_ext_hdr *ext_sqe = ext_hdr;
	1793	+ struct sq_send_hdr *sqe = base_hdr;
1616	1794
1617	1795	sqe->wqe_type = wqe->type;
1618	1796	sqe->flags = wqe->flags;
1619		- sqe->wqe_size = wqe_size16 +
1620		- ((offsetof(typeof(*sqe), data) + 15) >> 4);
1621		- sqe->inv_key_or_imm_data = cpu_to_le32(
1622		- wqe->send.inv_key);
1623		- if (qp->type == CMDQ_CREATE_QP_TYPE_UD) {
	1797	+ sqe->wqe_size = wqe_sz;
	1798	+ sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key);
	1799	+ if (qp->type == CMDQ_CREATE_QP_TYPE_UD \|\|
	1800	+ qp->type == CMDQ_CREATE_QP_TYPE_GSI) {
1624	1801	sqe->q_key = cpu_to_le32(wqe->send.q_key);
1625		- sqe->dst_qp = cpu_to_le32(
1626		- wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
1627	1802	sqe->length = cpu_to_le32(data_len);
1628		- sqe->avid = cpu_to_le32(wqe->send.avid &
1629		- SQ_SEND_AVID_MASK);
1630	1803	sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
	1804	+ ext_sqe->dst_qp = cpu_to_le32(wqe->send.dst_qp &
	1805	+ SQ_SEND_DST_QP_MASK);
	1806	+ ext_sqe->avid = cpu_to_le32(wqe->send.avid &
	1807	+ SQ_SEND_AVID_MASK);
1631	1808	} else {
1632	1809	sqe->length = cpu_to_le32(data_len);
1633		- sqe->dst_qp = 0;
1634		- sqe->avid = 0;
1635	1810	if (qp->mtu)
1636	1811	pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1637	1812	if (!pkt_num)
..	..	@@ -1644,16 +1819,16 @@
1644	1819	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
1645	1820	case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
1646	1821	{
1647		- struct sq_rdma sqe = (struct sq_rdma )hw_sq_send_hdr;
	1822	+ struct sq_rdma_ext_hdr *ext_sqe = ext_hdr;
	1823	+ struct sq_rdma_hdr *sqe = base_hdr;
1648	1824
1649	1825	sqe->wqe_type = wqe->type;
1650	1826	sqe->flags = wqe->flags;
1651		- sqe->wqe_size = wqe_size16 +
1652		- ((offsetof(typeof(*sqe), data) + 15) >> 4);
	1827	+ sqe->wqe_size = wqe_sz;
1653	1828	sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
1654	1829	sqe->length = cpu_to_le32((u32)data_len);
1655		- sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
1656		- sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
	1830	+ ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
	1831	+ ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
1657	1832	if (qp->mtu)
1658	1833	pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1659	1834	if (!pkt_num)
..	..	@@ -1664,14 +1839,15 @@
1664	1839	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
1665	1840	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
1666	1841	{
1667		- struct sq_atomic sqe = (struct sq_atomic )hw_sq_send_hdr;
	1842	+ struct sq_atomic_ext_hdr *ext_sqe = ext_hdr;
	1843	+ struct sq_atomic_hdr *sqe = base_hdr;
1668	1844
1669	1845	sqe->wqe_type = wqe->type;
1670	1846	sqe->flags = wqe->flags;
1671	1847	sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
1672	1848	sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
1673		- sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
1674		- sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
	1849	+ ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
	1850	+ ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
1675	1851	if (qp->mtu)
1676	1852	pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1677	1853	if (!pkt_num)
..	..	@@ -1681,8 +1857,7 @@
1681	1857	}
1682	1858	case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
1683	1859	{
1684		- struct sq_localinvalidate *sqe =
1685		- (struct sq_localinvalidate *)hw_sq_send_hdr;
	1860	+ struct sq_localinvalidate *sqe = base_hdr;
1686	1861
1687	1862	sqe->wqe_type = wqe->type;
1688	1863	sqe->flags = wqe->flags;
..	..	@@ -1692,7 +1867,8 @@
1692	1867	}
1693	1868	case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
1694	1869	{
1695		- struct sq_fr_pmr sqe = (struct sq_fr_pmr )hw_sq_send_hdr;
	1870	+ struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr;
	1871	+ struct sq_fr_pmr_hdr *sqe = base_hdr;
1696	1872
1697	1873	sqe->wqe_type = wqe->type;
1698	1874	sqe->flags = wqe->flags;
..	..	@@ -1716,14 +1892,15 @@
1716	1892	wqe->frmr.pbl_ptr[i] = cpu_to_le64(
1717	1893	wqe->frmr.page_list[i] \|
1718	1894	PTU_PTE_VALID);
1719		- sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
1720		- sqe->va = cpu_to_le64(wqe->frmr.va);
	1895	+ ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
	1896	+ ext_sqe->va = cpu_to_le64(wqe->frmr.va);
1721	1897
1722	1898	break;
1723	1899	}
1724	1900	case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
1725	1901	{
1726		- struct sq_bind sqe = (struct sq_bind )hw_sq_send_hdr;
	1902	+ struct sq_bind_ext_hdr *ext_sqe = ext_hdr;
	1903	+ struct sq_bind_hdr *sqe = base_hdr;
1727	1904
1728	1905	sqe->wqe_type = wqe->type;
1729	1906	sqe->flags = wqe->flags;
..	..	@@ -1732,9 +1909,8 @@
1732	1909	(wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
1733	1910	sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
1734	1911	sqe->l_key = cpu_to_le32(wqe->bind.r_key);
1735		- sqe->va = cpu_to_le64(wqe->bind.va);
1736		- temp32 = cpu_to_le32(wqe->bind.length);
1737		- memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length));
	1912	+ ext_sqe->va = cpu_to_le64(wqe->bind.va);
	1913	+ ext_sqe->length_lo = cpu_to_le32(wqe->bind.length);
1738	1914	break;
1739	1915	}
1740	1916	default:
..	..	@@ -1743,31 +1919,11 @@
1743	1919	goto done;
1744	1920	}
1745	1921	swq->next_psn = sq->psn & BTH_PSN_MASK;
1746		- if (swq->psn_search) {
1747		- swq->psn_search->opcode_start_psn = cpu_to_le32(
1748		- ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
1749		- SQ_PSN_SEARCH_START_PSN_MASK) \|
1750		- ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
1751		- SQ_PSN_SEARCH_OPCODE_MASK));
1752		- swq->psn_search->flags_next_psn = cpu_to_le32(
1753		- ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
1754		- SQ_PSN_SEARCH_NEXT_PSN_MASK));
1755		- }
	1922	+ bnxt_qplib_fill_psn_search(qp, wqe, swq);
1756	1923	queue_err:
1757		- if (sch_handler) {
1758		- /* Store the ULP info in the software structures */
1759		- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1760		- swq = &sq->swq[sw_prod];
1761		- swq->wr_id = wqe->wr_id;
1762		- swq->type = wqe->type;
1763		- swq->flags = wqe->flags;
1764		- if (qp->sig_type)
1765		- swq->flags \|= SQ_SEND_FLAGS_SIGNAL_COMP;
1766		- swq->start_psn = sq->psn & BTH_PSN_MASK;
1767		- }
1768		- sq->hwq.prod++;
	1924	+ bnxt_qplib_swq_mod_start(sq, wqe_idx);
	1925	+ bnxt_qplib_hwq_incr_prod(hwq, swq->slots);
1769	1926	qp->wqe_cnt++;
1770		-
1771	1927	done:
1772	1928	if (sch_handler) {
1773	1929	nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
..	..	@@ -1777,8 +1933,8 @@
1777	1933	INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
1778	1934	queue_work(qp->scq->nq->cqn_wq, &nq_work->work);
1779	1935	} else {
1780		- dev_err(&sq->hwq.pdev->dev,
1781		- "QPLIB: FP: Failed to allocate SQ nq_work!");
	1936	+ dev_err(&hwq->pdev->dev,
	1937	+ "FP: Failed to allocate SQ nq_work!\n");
1782	1938	rc = -ENOMEM;
1783	1939	}
1784	1940	}
..	..	@@ -1788,81 +1944,74 @@
1788	1944	void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
1789	1945	{
1790	1946	struct bnxt_qplib_q *rq = &qp->rq;
1791		- struct dbr_dbr db_msg = { 0 };
1792		- u32 sw_prod;
1793	1947
1794		- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1795		- db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1796		- DBR_DBR_INDEX_MASK);
1797		- db_msg.type_xid =
1798		- cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) \|
1799		- DBR_DBR_TYPE_RQ);
1800		-
1801		- /* Flush the writes to HW Rx WQE before the ringing Rx DB */
1802		- wmb();
1803		- __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
	1948	+ bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ);
1804	1949	}
1805	1950
1806	1951	int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
1807	1952	struct bnxt_qplib_swqe *wqe)
1808	1953	{
1809		- struct bnxt_qplib_q *rq = &qp->rq;
1810		- struct rq_wqe rqe, *rqe_ptr;
1811		- struct sq_sge *hw_sge;
1812	1954	struct bnxt_qplib_nq_work *nq_work = NULL;
	1955	+ struct bnxt_qplib_q *rq = &qp->rq;
	1956	+ struct rq_wqe_hdr *base_hdr;
	1957	+ struct rq_ext_hdr *ext_hdr;
	1958	+ struct bnxt_qplib_hwq *hwq;
	1959	+ struct bnxt_qplib_swq *swq;
1813	1960	bool sch_handler = false;
1814		- u32 sw_prod;
1815		- int i, rc = 0;
	1961	+ u16 wqe_sz, idx;
	1962	+ u32 wqe_idx;
	1963	+ int rc = 0;
1816	1964
1817		- if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1818		- sch_handler = true;
1819		- dev_dbg(&rq->hwq.pdev->dev,
1820		- "%s Error QP. Scheduling for poll_cq\n",
1821		- __func__);
1822		- goto queue_err;
1823		- }
1824		- if (bnxt_qplib_queue_full(rq)) {
1825		- dev_err(&rq->hwq.pdev->dev,
1826		- "QPLIB: FP: QP (0x%x) RQ is full!", qp->id);
	1965	+ hwq = &rq->hwq;
	1966	+ if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
	1967	+ dev_err(&hwq->pdev->dev,
	1968	+ "QPLIB: FP: QP (0x%x) is in the 0x%x state",
	1969	+ qp->id, qp->state);
1827	1970	rc = -EINVAL;
1828	1971	goto done;
1829	1972	}
1830		- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1831		- rq->swq[sw_prod].wr_id = wqe->wr_id;
1832	1973
1833		- rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr;
1834		- rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
1835		-
1836		- memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
1837		-
1838		- /* Calculate wqe_size16 and data_len */
1839		- for (i = 0, hw_sge = (struct sq_sge *)rqe->data;
1840		- i < wqe->num_sge; i++, hw_sge++) {
1841		- hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1842		- hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1843		- hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
	1974	+ if (bnxt_qplib_queue_full(rq, rq->dbinfo.max_slot)) {
	1975	+ dev_err(&hwq->pdev->dev,
	1976	+ "FP: QP (0x%x) RQ is full!\n", qp->id);
	1977	+ rc = -EINVAL;
	1978	+ goto done;
1844	1979	}
1845		- rqe->wqe_type = wqe->type;
1846		- rqe->flags = wqe->flags;
1847		- rqe->wqe_size = wqe->num_sge +
1848		- ((offsetof(typeof(*rqe), data) + 15) >> 4);
1849		- /* HW requires wqe size has room for atleast one SGE even if none
1850		- * was supplied by ULP
1851		- */
1852		- if (!wqe->num_sge)
1853		- rqe->wqe_size++;
1854	1980
1855		- /* Supply the rqe->wr_id index to the wr_id_tbl for now */
1856		- rqe->wr_id[0] = cpu_to_le32(sw_prod);
	1981	+ swq = bnxt_qplib_get_swqe(rq, &wqe_idx);
	1982	+ swq->wr_id = wqe->wr_id;
	1983	+ swq->slots = rq->dbinfo.max_slot;
1857	1984
	1985	+ if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
	1986	+ sch_handler = true;
	1987	+ dev_dbg(&hwq->pdev->dev,
	1988	+ "%s: Error QP. Scheduling for poll_cq\n", __func__);
	1989	+ goto queue_err;
	1990	+ }
	1991	+
	1992	+ idx = 0;
	1993	+ base_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
	1994	+ ext_hdr = bnxt_qplib_get_prod_qe(hwq, idx++);
	1995	+ memset(base_hdr, 0, sizeof(struct sq_sge));
	1996	+ memset(ext_hdr, 0, sizeof(struct sq_sge));
	1997	+ wqe_sz = (sizeof(struct rq_wqe_hdr) +
	1998	+ wqe->num_sge * sizeof(struct sq_sge)) >> 4;
	1999	+ bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge, &idx);
	2000	+ if (!wqe->num_sge) {
	2001	+ struct sq_sge *sge;
	2002	+
	2003	+ sge = bnxt_qplib_get_prod_qe(hwq, idx++);
	2004	+ sge->size = 0;
	2005	+ wqe_sz++;
	2006	+ }
	2007	+ base_hdr->wqe_type = wqe->type;
	2008	+ base_hdr->flags = wqe->flags;
	2009	+ base_hdr->wqe_size = wqe_sz;
	2010	+ base_hdr->wr_id[0] = cpu_to_le32(wqe_idx);
1858	2011	queue_err:
1859		- if (sch_handler) {
1860		- /* Store the ULP info in the software structures */
1861		- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1862		- rq->swq[sw_prod].wr_id = wqe->wr_id;
1863		- }
1864		-
1865		- rq->hwq.prod++;
	2012	+ bnxt_qplib_swq_mod_start(rq, wqe_idx);
	2013	+ bnxt_qplib_hwq_incr_prod(hwq, swq->slots);
	2014	+done:
1866	2015	if (sch_handler) {
1867	2016	nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
1868	2017	if (nq_work) {
..	..	@@ -1871,62 +2020,33 @@
1871	2020	INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
1872	2021	queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);
1873	2022	} else {
1874		- dev_err(&rq->hwq.pdev->dev,
1875		- "QPLIB: FP: Failed to allocate RQ nq_work!");
	2023	+ dev_err(&hwq->pdev->dev,
	2024	+ "FP: Failed to allocate RQ nq_work!\n");
1876	2025	rc = -ENOMEM;
1877	2026	}
1878	2027	}
1879		-done:
	2028	+
1880	2029	return rc;
1881	2030	}
1882	2031
1883	2032	/* CQ */
1884		-
1885		-/* Spinlock must be held */
1886		-static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq)
1887		-{
1888		- struct dbr_dbr db_msg = { 0 };
1889		-
1890		- db_msg.type_xid =
1891		- cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) \|
1892		- DBR_DBR_TYPE_CQ_ARMENA);
1893		- /* Flush memory writes before enabling the CQ */
1894		- wmb();
1895		- __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64));
1896		-}
1897		-
1898		-static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
1899		-{
1900		- struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1901		- struct dbr_dbr db_msg = { 0 };
1902		- u32 sw_cons;
1903		-
1904		- /* Ring DB */
1905		- sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq);
1906		- db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) &
1907		- DBR_DBR_INDEX_MASK);
1908		- db_msg.type_xid =
1909		- cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) \|
1910		- arm_type);
1911		- /* flush memory writes before arming the CQ */
1912		- wmb();
1913		- __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1914		-}
1915		-
1916	2033	int bnxt_qplib_create_cq(struct bnxt_qplib_res res, struct bnxt_qplib_cq cq)
1917	2034	{
1918	2035	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1919		- struct cmdq_create_cq req;
	2036	+ struct bnxt_qplib_hwq_attr hwq_attr = {};
1920	2037	struct creq_create_cq_resp resp;
1921	2038	struct bnxt_qplib_pbl *pbl;
	2039	+ struct cmdq_create_cq req;
1922	2040	u16 cmd_flags = 0;
	2041	+ u32 pg_sz_lvl;
1923	2042	int rc;
1924	2043
1925		- cq->hwq.max_elements = cq->max_wqe;
1926		- rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead,
1927		- cq->nmap, &cq->hwq.max_elements,
1928		- BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,
1929		- PAGE_SIZE, HWQ_TYPE_QUEUE);
	2044	+ hwq_attr.res = res;
	2045	+ hwq_attr.depth = cq->max_wqe;
	2046	+ hwq_attr.stride = sizeof(struct cq_base);
	2047	+ hwq_attr.type = HWQ_TYPE_QUEUE;
	2048	+ hwq_attr.sginfo = &cq->sg_info;
	2049	+ rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr);
1930	2050	if (rc)
1931	2051	goto exit;
1932	2052
..	..	@@ -1934,27 +2054,18 @@
1934	2054
1935	2055	if (!cq->dpi) {
1936	2056	dev_err(&rcfw->pdev->dev,
1937		- "QPLIB: FP: CREATE_CQ failed due to NULL DPI");
	2057	+ "FP: CREATE_CQ failed due to NULL DPI\n");
1938	2058	return -EINVAL;
1939	2059	}
1940	2060	req.dpi = cpu_to_le32(cq->dpi->dpi);
1941	2061	req.cq_handle = cpu_to_le64(cq->cq_handle);
1942		-
1943	2062	req.cq_size = cpu_to_le32(cq->hwq.max_elements);
1944	2063	pbl = &cq->hwq.pbl[PBL_LVL_0];
1945		- req.pg_size_lvl = cpu_to_le32(
1946		- ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
1947		- CMDQ_CREATE_CQ_LVL_SFT) \|
1948		- (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K :
1949		- pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K :
1950		- pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K :
1951		- pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M :
1952		- pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M :
1953		- pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G :
1954		- CMDQ_CREATE_CQ_PG_SIZE_PG_4K));
1955		-
	2064	+ pg_sz_lvl = (bnxt_qplib_base_pg_size(&cq->hwq) <<
	2065	+ CMDQ_CREATE_CQ_PG_SIZE_SFT);
	2066	+ pg_sz_lvl \|= (cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK);
	2067	+ req.pg_size_lvl = cpu_to_le32(pg_sz_lvl);
1956	2068	req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
1957		-
1958	2069	req.cq_fco_cnq_id = cpu_to_le32(
1959	2070	(cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
1960	2071	CMDQ_CREATE_CQ_CNQ_ID_SFT);
..	..	@@ -1965,7 +2076,6 @@
1965	2076	goto fail;
1966	2077
1967	2078	cq->id = le32_to_cpu(resp.xid);
1968		- cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
1969	2079	cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
1970	2080	init_waitqueue_head(&cq->waitq);
1971	2081	INIT_LIST_HEAD(&cq->sqf_head);
..	..	@@ -1973,11 +2083,17 @@
1973	2083	spin_lock_init(&cq->compl_lock);
1974	2084	spin_lock_init(&cq->flush_lock);
1975	2085
1976		- bnxt_qplib_arm_cq_enable(cq);
	2086	+ cq->dbinfo.hwq = &cq->hwq;
	2087	+ cq->dbinfo.xid = cq->id;
	2088	+ cq->dbinfo.db = cq->dpi->dbr;
	2089	+ cq->dbinfo.priv_db = res->dpi_tbl.dbr_bar_reg_iomem;
	2090	+
	2091	+ bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA);
	2092	+
1977	2093	return 0;
1978	2094
1979	2095	fail:
1980		- bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
	2096	+ bnxt_qplib_free_hwq(res, &cq->hwq);
1981	2097	exit:
1982	2098	return rc;
1983	2099	}
..	..	@@ -1987,6 +2103,7 @@
1987	2103	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1988	2104	struct cmdq_destroy_cq req;
1989	2105	struct creq_destroy_cq_resp resp;
	2106	+ u16 total_cnq_events;
1990	2107	u16 cmd_flags = 0;
1991	2108	int rc;
1992	2109
..	..	@@ -1997,27 +2114,28 @@
1997	2114	(void *)&resp, NULL, 0);
1998	2115	if (rc)
1999	2116	return rc;
2000		- bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
	2117	+ total_cnq_events = le16_to_cpu(resp.total_cnq_events);
	2118	+ __wait_for_all_nqes(cq, total_cnq_events);
	2119	+ bnxt_qplib_free_hwq(res, &cq->hwq);
2001	2120	return 0;
2002	2121	}
2003	2122
2004	2123	static int __flush_sq(struct bnxt_qplib_q sq, struct bnxt_qplib_qp qp,
2005	2124	struct bnxt_qplib_cqe *pcqe, int budget)
2006	2125	{
2007		- u32 sw_prod, sw_cons;
2008	2126	struct bnxt_qplib_cqe *cqe;
	2127	+ u32 start, last;
2009	2128	int rc = 0;
2010	2129
2011	2130	/* Now complete all outstanding SQEs with FLUSHED_ERR */
2012		- sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
	2131	+ start = sq->swq_start;
2013	2132	cqe = *pcqe;
2014	2133	while (*budget) {
2015		- sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2016		- if (sw_cons == sw_prod) {
	2134	+ last = sq->swq_last;
	2135	+ if (start == last)
2017	2136	break;
2018		- }
2019	2137	/* Skip the FENCE WQE completions */
2020		- if (sq->swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) {
	2138	+ if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) {
2021	2139	bnxt_qplib_cancel_phantom_processing(qp);
2022	2140	goto skip_compl;
2023	2141	}
..	..	@@ -2025,16 +2143,17 @@
2025	2143	cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
2026	2144	cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2027	2145	cqe->qp_handle = (u64)(unsigned long)qp;
2028		- cqe->wr_id = sq->swq[sw_cons].wr_id;
	2146	+ cqe->wr_id = sq->swq[last].wr_id;
2029	2147	cqe->src_qp = qp->id;
2030		- cqe->type = sq->swq[sw_cons].type;
	2148	+ cqe->type = sq->swq[last].type;
2031	2149	cqe++;
2032	2150	(*budget)--;
2033	2151	skip_compl:
2034		- sq->hwq.cons++;
	2152	+ bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[last].slots);
	2153	+ sq->swq_last = sq->swq[last].next_idx;
2035	2154	}
2036	2155	*pcqe = cqe;
2037		- if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)
	2156	+ if (!(*budget) && sq->swq_last != start)
2038	2157	/* Out of budget */
2039	2158	rc = -EAGAIN;
2040	2159
..	..	@@ -2045,9 +2164,9 @@
2045	2164	struct bnxt_qplib_cqe *pcqe, int budget)
2046	2165	{
2047	2166	struct bnxt_qplib_cqe *cqe;
2048		- u32 sw_prod, sw_cons;
2049		- int rc = 0;
	2167	+ u32 start, last;
2050	2168	int opcode = 0;
	2169	+ int rc = 0;
2051	2170
2052	2171	switch (qp->type) {
2053	2172	case CMDQ_CREATE_QP1_TYPE_GSI:
..	..	@@ -2057,29 +2176,31 @@
2057	2176	opcode = CQ_BASE_CQE_TYPE_RES_RC;
2058	2177	break;
2059	2178	case CMDQ_CREATE_QP_TYPE_UD:
	2179	+ case CMDQ_CREATE_QP_TYPE_GSI:
2060	2180	opcode = CQ_BASE_CQE_TYPE_RES_UD;
2061	2181	break;
2062	2182	}
2063	2183
2064	2184	/* Flush the rest of the RQ */
2065		- sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
	2185	+ start = rq->swq_start;
2066	2186	cqe = *pcqe;
2067	2187	while (*budget) {
2068		- sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);
2069		- if (sw_cons == sw_prod)
	2188	+ last = rq->swq_last;
	2189	+ if (last == start)
2070	2190	break;
2071	2191	memset(cqe, 0, sizeof(*cqe));
2072	2192	cqe->status =
2073	2193	CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
2074	2194	cqe->opcode = opcode;
2075	2195	cqe->qp_handle = (unsigned long)qp;
2076		- cqe->wr_id = rq->swq[sw_cons].wr_id;
	2196	+ cqe->wr_id = rq->swq[last].wr_id;
2077	2197	cqe++;
2078	2198	(*budget)--;
2079		- rq->hwq.cons++;
	2199	+ bnxt_qplib_hwq_incr_cons(&rq->hwq, rq->swq[last].slots);
	2200	+ rq->swq_last = rq->swq[last].next_idx;
2080	2201	}
2081	2202	*pcqe = cqe;
2082		- if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)
	2203	+ if (!*budget && rq->swq_last != start)
2083	2204	/* Out of budget */
2084	2205	rc = -EAGAIN;
2085	2206
..	..	@@ -2102,20 +2223,20 @@
2102	2223	* CQE is track from sw_cq_cons to max_element but valid only if VALID=1
2103	2224	*/
2104	2225	static int do_wa9060(struct bnxt_qplib_qp qp, struct bnxt_qplib_cq cq,
2105		- u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons)
	2226	+ u32 cq_cons, u32 swq_last, u32 cqe_sq_cons)
2106	2227	{
2107		- struct bnxt_qplib_q *sq = &qp->sq;
2108		- struct bnxt_qplib_swq *swq;
2109	2228	u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;
2110		- struct cq_base peek_hwcqe, *peek_hw_cqe_ptr;
	2229	+ struct bnxt_qplib_q *sq = &qp->sq;
2111	2230	struct cq_req *peek_req_hwcqe;
2112	2231	struct bnxt_qplib_qp *peek_qp;
2113	2232	struct bnxt_qplib_q *peek_sq;
	2233	+ struct bnxt_qplib_swq *swq;
	2234	+ struct cq_base *peek_hwcqe;
2114	2235	int i, rc = 0;
2115	2236
2116	2237	/* Normal mode */
2117	2238	/* Check for the psn_search marking before completing */
2118		- swq = &sq->swq[sw_sq_cons];
	2239	+ swq = &sq->swq[swq_last];
2119	2240	if (swq->psn_search &&
2120	2241	le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
2121	2242	/* Unmark */
..	..	@@ -2124,13 +2245,12 @@
2124	2245	& ~0x80000000);
2125	2246	dev_dbg(&cq->hwq.pdev->dev,
2126	2247	"FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
2127		- cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
	2248	+ cq_cons, qp->id, swq_last, cqe_sq_cons);
2128	2249	sq->condition = true;
2129	2250	sq->send_phantom = true;
2130	2251
2131	2252	/* TODO: Only ARM if the previous SQE is ARMALL */
2132		- bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ_ARMALL);
2133		-
	2253	+ bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL);
2134	2254	rc = -EAGAIN;
2135	2255	goto out;
2136	2256	}
..	..	@@ -2141,9 +2261,8 @@
2141	2261	i = cq->hwq.max_elements;
2142	2262	while (i--) {
2143	2263	peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq);
2144		- peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2145		- peek_hwcqe = &peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)]
2146		- [CQE_IDX(peek_sw_cq_cons)];
	2264	+ peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq,
	2265	+ peek_sw_cq_cons, NULL);
2147	2266	/* If the next hwcqe is VALID */
2148	2267	if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons,
2149	2268	cq->hwq.max_elements)) {
..	..	@@ -2163,9 +2282,10 @@
2163	2282	le64_to_cpu
2164	2283	(peek_req_hwcqe->qp_handle));
2165	2284	peek_sq = &peek_qp->sq;
2166		- peek_sq_cons_idx = HWQ_CMP(le16_to_cpu(
2167		- peek_req_hwcqe->sq_cons_idx) - 1
2168		- , &sq->hwq);
	2285	+ peek_sq_cons_idx =
	2286	+ ((le16_to_cpu(
	2287	+ peek_req_hwcqe->sq_cons_idx)
	2288	+ - 1) % sq->max_wqe);
2169	2289	/* If the hwcqe's sq's wr_id matches */
2170	2290	if (peek_sq == sq &&
2171	2291	sq->swq[peek_sq_cons_idx].wr_id ==
..	..	@@ -2175,7 +2295,7 @@
2175	2295	* comes back
2176	2296	*/
2177	2297	dev_dbg(&cq->hwq.pdev->dev,
2178		- "FP:Got Phantom CQE");
	2298	+ "FP: Got Phantom CQE\n");
2179	2299	sq->condition = false;
2180	2300	sq->single = true;
2181	2301	rc = 0;
..	..	@@ -2192,8 +2312,8 @@
2192	2312	peek_raw_cq_cons++;
2193	2313	}
2194	2314	dev_err(&cq->hwq.pdev->dev,
2195		- "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x",
2196		- cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
	2315	+ "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n",
	2316	+ cq_cons, qp->id, swq_last, cqe_sq_cons);
2197	2317	rc = -EINVAL;
2198	2318	}
2199	2319	out:
..	..	@@ -2205,35 +2325,26 @@
2205	2325	struct bnxt_qplib_cqe *pcqe, int budget,
2206	2326	u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
2207	2327	{
	2328	+ struct bnxt_qplib_swq *swq;
	2329	+ struct bnxt_qplib_cqe *cqe;
2208	2330	struct bnxt_qplib_qp *qp;
2209	2331	struct bnxt_qplib_q *sq;
2210		- struct bnxt_qplib_cqe *cqe;
2211		- u32 sw_sq_cons, cqe_sq_cons;
2212		- struct bnxt_qplib_swq *swq;
	2332	+ u32 cqe_sq_cons;
2213	2333	int rc = 0;
2214	2334
2215	2335	qp = (struct bnxt_qplib_qp *)((unsigned long)
2216	2336	le64_to_cpu(hwcqe->qp_handle));
2217	2337	if (!qp) {
2218	2338	dev_err(&cq->hwq.pdev->dev,
2219		- "QPLIB: FP: Process Req qp is NULL");
	2339	+ "FP: Process Req qp is NULL\n");
2220	2340	return -EINVAL;
2221	2341	}
2222	2342	sq = &qp->sq;
2223	2343
2224		- cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
2225		- if (cqe_sq_cons > sq->hwq.max_elements) {
2226		- dev_err(&cq->hwq.pdev->dev,
2227		- "QPLIB: FP: CQ Process req reported ");
2228		- dev_err(&cq->hwq.pdev->dev,
2229		- "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
2230		- cqe_sq_cons, sq->hwq.max_elements);
2231		- return -EINVAL;
2232		- }
2233		-
	2344	+ cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe;
2234	2345	if (qp->sq.flushed) {
2235	2346	dev_dbg(&cq->hwq.pdev->dev,
2236		- "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
	2347	+ "%s: QP in Flush QP = %p\n", __func__, qp);
2237	2348	goto done;
2238	2349	}
2239	2350	/* Require to walk the sq's swq to fabricate CQEs for all previously
..	..	@@ -2242,12 +2353,11 @@
2242	2353	*/
2243	2354	cqe = *pcqe;
2244	2355	while (*budget) {
2245		- sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2246		- if (sw_sq_cons == cqe_sq_cons)
	2356	+ if (sq->swq_last == cqe_sq_cons)
2247	2357	/* Done */
2248	2358	break;
2249	2359
2250		- swq = &sq->swq[sw_sq_cons];
	2360	+ swq = &sq->swq[sq->swq_last];
2251	2361	memset(cqe, 0, sizeof(*cqe));
2252	2362	cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2253	2363	cqe->qp_handle = (u64)(unsigned long)qp;
..	..	@@ -2261,14 +2371,12 @@
2261	2371	* of the request being signaled or not, it must complete with
2262	2372	* the hwcqe error status
2263	2373	*/
2264		- if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons &&
	2374	+ if (swq->next_idx == cqe_sq_cons &&
2265	2375	hwcqe->status != CQ_REQ_STATUS_OK) {
2266	2376	cqe->status = hwcqe->status;
2267	2377	dev_err(&cq->hwq.pdev->dev,
2268		- "QPLIB: FP: CQ Processed Req ");
2269		- dev_err(&cq->hwq.pdev->dev,
2270		- "QPLIB: wr_id[%d] = 0x%llx with status 0x%x",
2271		- sw_sq_cons, cqe->wr_id, cqe->status);
	2378	+ "FP: CQ Processed Req wr_id[%d] = 0x%llx with status 0x%x\n",
	2379	+ sq->swq_last, cqe->wr_id, cqe->status);
2272	2380	cqe++;
2273	2381	(*budget)--;
2274	2382	bnxt_qplib_mark_qp_error(qp);
..	..	@@ -2276,7 +2384,7 @@
2276	2384	bnxt_qplib_add_flush_qp(qp);
2277	2385	} else {
2278	2386	/* Before we complete, do WA 9060 */
2279		- if (do_wa9060(qp, cq, cq_cons, sw_sq_cons,
	2387	+ if (do_wa9060(qp, cq, cq_cons, sq->swq_last,
2280	2388	cqe_sq_cons)) {
2281	2389	*lib_qp = qp;
2282	2390	goto out;
..	..	@@ -2288,13 +2396,14 @@
2288	2396	}
2289	2397	}
2290	2398	skip:
2291		- sq->hwq.cons++;
	2399	+ bnxt_qplib_hwq_incr_cons(&sq->hwq, swq->slots);
	2400	+ sq->swq_last = swq->next_idx;
2292	2401	if (sq->single)
2293	2402	break;
2294	2403	}
2295	2404	out:
2296	2405	*pcqe = cqe;
2297		- if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) {
	2406	+ if (sq->swq_last != cqe_sq_cons) {
2298	2407	/* Out of budget */
2299	2408	rc = -EAGAIN;
2300	2409	goto done;
..	..	@@ -2323,22 +2432,22 @@
2323	2432	struct bnxt_qplib_cqe **pcqe,
2324	2433	int *budget)
2325	2434	{
2326		- struct bnxt_qplib_qp *qp;
2327		- struct bnxt_qplib_q *rq;
2328	2435	struct bnxt_qplib_srq *srq;
2329	2436	struct bnxt_qplib_cqe *cqe;
	2437	+ struct bnxt_qplib_qp *qp;
	2438	+ struct bnxt_qplib_q *rq;
2330	2439	u32 wr_id_idx;
2331	2440	int rc = 0;
2332	2441
2333	2442	qp = (struct bnxt_qplib_qp *)((unsigned long)
2334	2443	le64_to_cpu(hwcqe->qp_handle));
2335	2444	if (!qp) {
2336		- dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL");
	2445	+ dev_err(&cq->hwq.pdev->dev, "process_cq RC qp is NULL\n");
2337	2446	return -EINVAL;
2338	2447	}
2339	2448	if (qp->rq.flushed) {
2340	2449	dev_dbg(&cq->hwq.pdev->dev,
2341		- "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
	2450	+ "%s: QP in Flush QP = %p\n", __func__, qp);
2342	2451	goto done;
2343	2452	}
2344	2453
..	..	@@ -2359,9 +2468,7 @@
2359	2468	return -EINVAL;
2360	2469	if (wr_id_idx >= srq->hwq.max_elements) {
2361	2470	dev_err(&cq->hwq.pdev->dev,
2362		- "QPLIB: FP: CQ Process RC ");
2363		- dev_err(&cq->hwq.pdev->dev,
2364		- "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
	2471	+ "FP: CQ Process RC wr_id idx 0x%x exceeded SRQ max 0x%x\n",
2365	2472	wr_id_idx, srq->hwq.max_elements);
2366	2473	return -EINVAL;
2367	2474	}
..	..	@@ -2371,19 +2478,23 @@
2371	2478	(*budget)--;
2372	2479	*pcqe = cqe;
2373	2480	} else {
	2481	+ struct bnxt_qplib_swq *swq;
	2482	+
2374	2483	rq = &qp->rq;
2375		- if (wr_id_idx >= rq->hwq.max_elements) {
	2484	+ if (wr_id_idx > (rq->max_wqe - 1)) {
2376	2485	dev_err(&cq->hwq.pdev->dev,
2377		- "QPLIB: FP: CQ Process RC ");
2378		- dev_err(&cq->hwq.pdev->dev,
2379		- "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
2380		- wr_id_idx, rq->hwq.max_elements);
	2486	+ "FP: CQ Process RC wr_id idx 0x%x exceeded RQ max 0x%x\n",
	2487	+ wr_id_idx, rq->max_wqe);
2381	2488	return -EINVAL;
2382	2489	}
2383		- cqe->wr_id = rq->swq[wr_id_idx].wr_id;
	2490	+ if (wr_id_idx != rq->swq_last)
	2491	+ return -EINVAL;
	2492	+ swq = &rq->swq[rq->swq_last];
	2493	+ cqe->wr_id = swq->wr_id;
2384	2494	cqe++;
2385	2495	(*budget)--;
2386		- rq->hwq.cons++;
	2496	+ bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
	2497	+ rq->swq_last = swq->next_idx;
2387	2498	*pcqe = cqe;
2388	2499
2389	2500	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
..	..	@@ -2402,32 +2513,34 @@
2402	2513	struct bnxt_qplib_cqe **pcqe,
2403	2514	int *budget)
2404	2515	{
2405		- struct bnxt_qplib_qp *qp;
2406		- struct bnxt_qplib_q *rq;
2407	2516	struct bnxt_qplib_srq *srq;
2408	2517	struct bnxt_qplib_cqe *cqe;
	2518	+ struct bnxt_qplib_qp *qp;
	2519	+ struct bnxt_qplib_q *rq;
2409	2520	u32 wr_id_idx;
2410	2521	int rc = 0;
2411	2522
2412	2523	qp = (struct bnxt_qplib_qp *)((unsigned long)
2413	2524	le64_to_cpu(hwcqe->qp_handle));
2414	2525	if (!qp) {
2415		- dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL");
	2526	+ dev_err(&cq->hwq.pdev->dev, "process_cq UD qp is NULL\n");
2416	2527	return -EINVAL;
2417	2528	}
2418	2529	if (qp->rq.flushed) {
2419	2530	dev_dbg(&cq->hwq.pdev->dev,
2420		- "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
	2531	+ "%s: QP in Flush QP = %p\n", __func__, qp);
2421	2532	goto done;
2422	2533	}
2423	2534	cqe = *pcqe;
2424	2535	cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2425		- cqe->length = le32_to_cpu(hwcqe->length);
	2536	+ cqe->length = le16_to_cpu(hwcqe->length) & CQ_RES_UD_LENGTH_MASK;
	2537	+ cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata);
2426	2538	cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
2427	2539	cqe->flags = le16_to_cpu(hwcqe->flags);
2428	2540	cqe->status = hwcqe->status;
2429	2541	cqe->qp_handle = (u64)(unsigned long)qp;
2430		- memcpy(cqe->smac, hwcqe->src_mac, 6);
	2542	+ /FIXME: Endianness fix needed for smace /
	2543	+ memcpy(cqe->smac, hwcqe->src_mac, ETH_ALEN);
2431	2544	wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
2432	2545	& CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK;
2433	2546	cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) \|
..	..	@@ -2442,9 +2555,7 @@
2442	2555
2443	2556	if (wr_id_idx >= srq->hwq.max_elements) {
2444	2557	dev_err(&cq->hwq.pdev->dev,
2445		- "QPLIB: FP: CQ Process UD ");
2446		- dev_err(&cq->hwq.pdev->dev,
2447		- "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
	2558	+ "FP: CQ Process UD wr_id idx 0x%x exceeded SRQ max 0x%x\n",
2448	2559	wr_id_idx, srq->hwq.max_elements);
2449	2560	return -EINVAL;
2450	2561	}
..	..	@@ -2454,20 +2565,24 @@
2454	2565	(*budget)--;
2455	2566	*pcqe = cqe;
2456	2567	} else {
	2568	+ struct bnxt_qplib_swq *swq;
	2569	+
2457	2570	rq = &qp->rq;
2458		- if (wr_id_idx >= rq->hwq.max_elements) {
	2571	+ if (wr_id_idx > (rq->max_wqe - 1)) {
2459	2572	dev_err(&cq->hwq.pdev->dev,
2460		- "QPLIB: FP: CQ Process UD ");
2461		- dev_err(&cq->hwq.pdev->dev,
2462		- "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
2463		- wr_id_idx, rq->hwq.max_elements);
	2573	+ "FP: CQ Process UD wr_id idx 0x%x exceeded RQ max 0x%x\n",
	2574	+ wr_id_idx, rq->max_wqe);
2464	2575	return -EINVAL;
2465	2576	}
2466	2577
2467		- cqe->wr_id = rq->swq[wr_id_idx].wr_id;
	2578	+ if (rq->swq_last != wr_id_idx)
	2579	+ return -EINVAL;
	2580	+ swq = &rq->swq[rq->swq_last];
	2581	+ cqe->wr_id = swq->wr_id;
2468	2582	cqe++;
2469	2583	(*budget)--;
2470		- rq->hwq.cons++;
	2584	+ bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
	2585	+ rq->swq_last = swq->next_idx;
2471	2586	*pcqe = cqe;
2472	2587
2473	2588	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
..	..	@@ -2482,15 +2597,13 @@
2482	2597
2483	2598	bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
2484	2599	{
2485		- struct cq_base hw_cqe, *hw_cqe_ptr;
	2600	+ struct cq_base *hw_cqe;
2486	2601	u32 sw_cons, raw_cons;
2487	2602	bool rc = true;
2488	2603
2489	2604	raw_cons = cq->hwq.cons;
2490	2605	sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2491		- hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2492		- hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2493		-
	2606	+ hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL);
2494	2607	/* Check for Valid bit. If the CQE is valid, return false */
2495	2608	rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements);
2496	2609	return rc;
..	..	@@ -2511,13 +2624,12 @@
2511	2624	qp = (struct bnxt_qplib_qp *)((unsigned long)
2512	2625	le64_to_cpu(hwcqe->qp_handle));
2513	2626	if (!qp) {
2514		- dev_err(&cq->hwq.pdev->dev,
2515		- "QPLIB: process_cq Raw/QP1 qp is NULL");
	2627	+ dev_err(&cq->hwq.pdev->dev, "process_cq Raw/QP1 qp is NULL\n");
2516	2628	return -EINVAL;
2517	2629	}
2518	2630	if (qp->rq.flushed) {
2519	2631	dev_dbg(&cq->hwq.pdev->dev,
2520		- "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
	2632	+ "%s: QP in Flush QP = %p\n", __func__, qp);
2521	2633	goto done;
2522	2634	}
2523	2635	cqe = *pcqe;
..	..	@@ -2546,14 +2658,12 @@
2546	2658	srq = qp->srq;
2547	2659	if (!srq) {
2548	2660	dev_err(&cq->hwq.pdev->dev,
2549		- "QPLIB: FP: SRQ used but not defined??");
	2661	+ "FP: SRQ used but not defined??\n");
2550	2662	return -EINVAL;
2551	2663	}
2552	2664	if (wr_id_idx >= srq->hwq.max_elements) {
2553	2665	dev_err(&cq->hwq.pdev->dev,
2554		- "QPLIB: FP: CQ Process Raw/QP1 ");
2555		- dev_err(&cq->hwq.pdev->dev,
2556		- "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
	2666	+ "FP: CQ Process Raw/QP1 wr_id idx 0x%x exceeded SRQ max 0x%x\n",
2557	2667	wr_id_idx, srq->hwq.max_elements);
2558	2668	return -EINVAL;
2559	2669	}
..	..	@@ -2563,19 +2673,23 @@
2563	2673	(*budget)--;
2564	2674	*pcqe = cqe;
2565	2675	} else {
	2676	+ struct bnxt_qplib_swq *swq;
	2677	+
2566	2678	rq = &qp->rq;
2567		- if (wr_id_idx >= rq->hwq.max_elements) {
	2679	+ if (wr_id_idx > (rq->max_wqe - 1)) {
2568	2680	dev_err(&cq->hwq.pdev->dev,
2569		- "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
2570		- dev_err(&cq->hwq.pdev->dev,
2571		- "QPLIB: ix 0x%x exceeded RQ max 0x%x",
2572		- wr_id_idx, rq->hwq.max_elements);
	2681	+ "FP: CQ Process Raw/QP1 RQ wr_id idx 0x%x exceeded RQ max 0x%x\n",
	2682	+ wr_id_idx, rq->max_wqe);
2573	2683	return -EINVAL;
2574	2684	}
2575		- cqe->wr_id = rq->swq[wr_id_idx].wr_id;
	2685	+ if (rq->swq_last != wr_id_idx)
	2686	+ return -EINVAL;
	2687	+ swq = &rq->swq[rq->swq_last];
	2688	+ cqe->wr_id = swq->wr_id;
2576	2689	cqe++;
2577	2690	(*budget)--;
2578		- rq->hwq.cons++;
	2691	+ bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
	2692	+ rq->swq_last = swq->next_idx;
2579	2693	*pcqe = cqe;
2580	2694
2581	2695	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
..	..	@@ -2597,20 +2711,20 @@
2597	2711	struct bnxt_qplib_qp *qp;
2598	2712	struct bnxt_qplib_q sq, rq;
2599	2713	struct bnxt_qplib_cqe *cqe;
2600		- u32 sw_cons = 0, cqe_cons;
	2714	+ u32 swq_last = 0, cqe_cons;
2601	2715	int rc = 0;
2602	2716
2603	2717	/* Check the Status */
2604	2718	if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
2605	2719	dev_warn(&cq->hwq.pdev->dev,
2606		- "QPLIB: FP: CQ Process Terminal Error status = 0x%x",
	2720	+ "FP: CQ Process Terminal Error status = 0x%x\n",
2607	2721	hwcqe->status);
2608	2722
2609	2723	qp = (struct bnxt_qplib_qp *)((unsigned long)
2610	2724	le64_to_cpu(hwcqe->qp_handle));
2611	2725	if (!qp) {
2612	2726	dev_err(&cq->hwq.pdev->dev,
2613		- "QPLIB: FP: CQ Process terminal qp is NULL");
	2727	+ "FP: CQ Process terminal qp is NULL\n");
2614	2728	return -EINVAL;
2615	2729	}
2616	2730
..	..	@@ -2623,19 +2737,11 @@
2623	2737	cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
2624	2738	if (cqe_cons == 0xFFFF)
2625	2739	goto do_rq;
2626		-
2627		- if (cqe_cons > sq->hwq.max_elements) {
2628		- dev_err(&cq->hwq.pdev->dev,
2629		- "QPLIB: FP: CQ Process terminal reported ");
2630		- dev_err(&cq->hwq.pdev->dev,
2631		- "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
2632		- cqe_cons, sq->hwq.max_elements);
2633		- goto do_rq;
2634		- }
	2740	+ cqe_cons %= sq->max_wqe;
2635	2741
2636	2742	if (qp->sq.flushed) {
2637	2743	dev_dbg(&cq->hwq.pdev->dev,
2638		- "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
	2744	+ "%s: QP in Flush QP = %p\n", __func__, qp);
2639	2745	goto sq_done;
2640	2746	}
2641	2747
..	..	@@ -2645,24 +2751,25 @@
2645	2751	*/
2646	2752	cqe = *pcqe;
2647	2753	while (*budget) {
2648		- sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2649		- if (sw_cons == cqe_cons)
	2754	+ swq_last = sq->swq_last;
	2755	+ if (swq_last == cqe_cons)
2650	2756	break;
2651		- if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
	2757	+ if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2652	2758	memset(cqe, 0, sizeof(*cqe));
2653	2759	cqe->status = CQ_REQ_STATUS_OK;
2654	2760	cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2655	2761	cqe->qp_handle = (u64)(unsigned long)qp;
2656	2762	cqe->src_qp = qp->id;
2657		- cqe->wr_id = sq->swq[sw_cons].wr_id;
2658		- cqe->type = sq->swq[sw_cons].type;
	2763	+ cqe->wr_id = sq->swq[swq_last].wr_id;
	2764	+ cqe->type = sq->swq[swq_last].type;
2659	2765	cqe++;
2660	2766	(*budget)--;
2661	2767	}
2662		- sq->hwq.cons++;
	2768	+ bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[swq_last].slots);
	2769	+ sq->swq_last = sq->swq[swq_last].next_idx;
2663	2770	}
2664	2771	*pcqe = cqe;
2665		- if (!(*budget) && sw_cons != cqe_cons) {
	2772	+ if (!(*budget) && swq_last != cqe_cons) {
2666	2773	/* Out of budget */
2667	2774	rc = -EAGAIN;
2668	2775	goto sq_done;
..	..	@@ -2674,18 +2781,17 @@
2674	2781	cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
2675	2782	if (cqe_cons == 0xFFFF) {
2676	2783	goto done;
2677		- } else if (cqe_cons > rq->hwq.max_elements) {
	2784	+ } else if (cqe_cons > rq->max_wqe - 1) {
2678	2785	dev_err(&cq->hwq.pdev->dev,
2679		- "QPLIB: FP: CQ Processed terminal ");
2680		- dev_err(&cq->hwq.pdev->dev,
2681		- "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x",
2682		- cqe_cons, rq->hwq.max_elements);
	2786	+ "FP: CQ Processed terminal reported rq_cons_idx 0x%x exceeds max 0x%x\n",
	2787	+ cqe_cons, rq->max_wqe);
	2788	+ rc = -EINVAL;
2683	2789	goto done;
2684	2790	}
2685	2791
2686	2792	if (qp->rq.flushed) {
2687	2793	dev_dbg(&cq->hwq.pdev->dev,
2688		- "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
	2794	+ "%s: QP in Flush QP = %p\n", __func__, qp);
2689	2795	rc = 0;
2690	2796	goto done;
2691	2797	}
..	..	@@ -2707,7 +2813,7 @@
2707	2813	/* Check the Status */
2708	2814	if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
2709	2815	dev_err(&cq->hwq.pdev->dev,
2710		- "QPLIB: FP: CQ Process Cutoff Error status = 0x%x",
	2816	+ "FP: CQ Process Cutoff Error status = 0x%x\n",
2711	2817	hwcqe->status);
2712	2818	return -EINVAL;
2713	2819	}
..	..	@@ -2727,16 +2833,12 @@
2727	2833
2728	2834	spin_lock_irqsave(&cq->flush_lock, flags);
2729	2835	list_for_each_entry(qp, &cq->sqf_head, sq_flush) {
2730		- dev_dbg(&cq->hwq.pdev->dev,
2731		- "QPLIB: FP: Flushing SQ QP= %p",
2732		- qp);
	2836	+ dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing SQ QP= %p\n", qp);
2733	2837	__flush_sq(&qp->sq, qp, &cqe, &budget);
2734	2838	}
2735	2839
2736	2840	list_for_each_entry(qp, &cq->rqf_head, rq_flush) {
2737		- dev_dbg(&cq->hwq.pdev->dev,
2738		- "QPLIB: FP: Flushing RQ QP= %p",
2739		- qp);
	2841	+ dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing RQ QP= %p\n", qp);
2740	2842	__flush_rq(&qp->rq, qp, &cqe, &budget);
2741	2843	}
2742	2844	spin_unlock_irqrestore(&cq->flush_lock, flags);
..	..	@@ -2747,7 +2849,7 @@
2747	2849	int bnxt_qplib_poll_cq(struct bnxt_qplib_cq cq, struct bnxt_qplib_cqe cqe,
2748	2850	int num_cqes, struct bnxt_qplib_qp **lib_qp)
2749	2851	{
2750		- struct cq_base hw_cqe, *hw_cqe_ptr;
	2852	+ struct cq_base *hw_cqe;
2751	2853	u32 sw_cons, raw_cons;
2752	2854	int budget, rc = 0;
2753	2855
..	..	@@ -2756,8 +2858,7 @@
2756	2858
2757	2859	while (budget) {
2758	2860	sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2759		- hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2760		- hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
	2861	+ hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL);
2761	2862
2762	2863	/* Check for Valid bit */
2763	2864	if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
..	..	@@ -2804,7 +2905,7 @@
2804	2905	goto exit;
2805	2906	default:
2806	2907	dev_err(&cq->hwq.pdev->dev,
2807		- "QPLIB: process_cq unknown type 0x%lx",
	2908	+ "process_cq unknown type 0x%lx\n",
2808	2909	hw_cqe->cqe_type_toggle &
2809	2910	CQ_BASE_CQE_TYPE_MASK);
2810	2911	rc = -EINVAL;
..	..	@@ -2817,13 +2918,13 @@
2817	2918	* next one
2818	2919	*/
2819	2920	dev_err(&cq->hwq.pdev->dev,
2820		- "QPLIB: process_cqe error rc = 0x%x", rc);
	2921	+ "process_cqe error rc = 0x%x\n", rc);
2821	2922	}
2822	2923	raw_cons++;
2823	2924	}
2824	2925	if (cq->hwq.cons != raw_cons) {
2825	2926	cq->hwq.cons = raw_cons;
2826		- bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ);
	2927	+ bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ);
2827	2928	}
2828	2929	exit:
2829	2930	return num_cqes - budget;
..	..	@@ -2832,7 +2933,7 @@
2832	2933	void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
2833	2934	{
2834	2935	if (arm_type)
2835		- bnxt_qplib_arm_cq(cq, arm_type);
	2936	+ bnxt_qplib_ring_db(&cq->dbinfo, arm_type);
2836	2937	/* Using cq->arm_state variable to track whether to issue cq handler */
2837	2938	atomic_set(&cq->arm_state, 1);
2838	2939	}