~hc/RK356X_SDK_RELEASE.git

..	..	@@ -67,7 +67,7 @@
67	67	#include "hw_counters.h"
68	68
69	69	static char version[] =
70		- BNXT_RE_DESC " v" ROCE_DRV_MODULE_VERSION "\n";
	70	+ BNXT_RE_DESC "\n";
71	71
72	72	MODULE_AUTHOR("Eddie Wai <eddie.wai@broadcom.com>");
73	73	MODULE_DESCRIPTION(BNXT_RE_DESC " Driver");
..	..	@@ -78,7 +78,58 @@
78	78	/* Mutex to protect the list of bnxt_re devices added */
79	79	static DEFINE_MUTEX(bnxt_re_dev_lock);
80	80	static struct workqueue_struct *bnxt_re_wq;
81		-static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev);
	81	+static void bnxt_re_remove_device(struct bnxt_re_dev *rdev);
	82	+static void bnxt_re_dealloc_driver(struct ib_device *ib_dev);
	83	+static void bnxt_re_stop_irq(void *handle);
	84	+
	85	+static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode)
	86	+{
	87	+ struct bnxt_qplib_chip_ctx *cctx;
	88	+
	89	+ cctx = rdev->chip_ctx;
	90	+ cctx->modes.wqe_mode = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
	91	+ mode : BNXT_QPLIB_WQE_MODE_STATIC;
	92	+}
	93	+
	94	+static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev)
	95	+{
	96	+ struct bnxt_qplib_chip_ctx *chip_ctx;
	97	+
	98	+ if (!rdev->chip_ctx)
	99	+ return;
	100	+ chip_ctx = rdev->chip_ctx;
	101	+ rdev->chip_ctx = NULL;
	102	+ rdev->rcfw.res = NULL;
	103	+ rdev->qplib_res.cctx = NULL;
	104	+ rdev->qplib_res.pdev = NULL;
	105	+ rdev->qplib_res.netdev = NULL;
	106	+ kfree(chip_ctx);
	107	+}
	108	+
	109	+static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode)
	110	+{
	111	+ struct bnxt_qplib_chip_ctx *chip_ctx;
	112	+ struct bnxt_en_dev *en_dev;
	113	+ struct bnxt *bp;
	114	+
	115	+ en_dev = rdev->en_dev;
	116	+ bp = netdev_priv(en_dev->net);
	117	+
	118	+ chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL);
	119	+ if (!chip_ctx)
	120	+ return -ENOMEM;
	121	+ chip_ctx->chip_num = bp->chip_num;
	122	+ chip_ctx->hw_stats_size = bp->hw_ring_stats_size;
	123	+
	124	+ rdev->chip_ctx = chip_ctx;
	125	+ /* rest members to follow eventually */
	126	+
	127	+ rdev->qplib_res.cctx = rdev->chip_ctx;
	128	+ rdev->rcfw.res = &rdev->qplib_res;
	129	+
	130	+ bnxt_re_set_drv_mode(rdev, wqe_mode);
	131	+ return 0;
	132	+}
82	133
83	134	/* SR-IOV helper functions */
84	135
..	..	@@ -96,61 +147,76 @@
96	147	* reserved for the function. The driver may choose to allocate fewer
97	148	* resources than the firmware maximum.
98	149	*/
	150	+static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev)
	151	+{
	152	+ struct bnxt_qplib_dev_attr *attr;
	153	+ struct bnxt_qplib_ctx *ctx;
	154	+ int i;
	155	+
	156	+ attr = &rdev->dev_attr;
	157	+ ctx = &rdev->qplib_ctx;
	158	+
	159	+ ctx->qpc_count = min_t(u32, BNXT_RE_MAX_QPC_COUNT,
	160	+ attr->max_qp);
	161	+ ctx->mrw_count = BNXT_RE_MAX_MRW_COUNT_256K;
	162	+ /* Use max_mr from fw since max_mrw does not get set */
	163	+ ctx->mrw_count = min_t(u32, ctx->mrw_count, attr->max_mr);
	164	+ ctx->srqc_count = min_t(u32, BNXT_RE_MAX_SRQC_COUNT,
	165	+ attr->max_srq);
	166	+ ctx->cq_count = min_t(u32, BNXT_RE_MAX_CQ_COUNT, attr->max_cq);
	167	+ if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx))
	168	+ for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
	169	+ rdev->qplib_ctx.tqm_ctx.qcount[i] =
	170	+ rdev->dev_attr.tqm_alloc_reqs[i];
	171	+}
	172	+
	173	+static void bnxt_re_limit_vf_res(struct bnxt_qplib_ctx *qplib_ctx, u32 num_vf)
	174	+{
	175	+ struct bnxt_qplib_vf_res *vf_res;
	176	+ u32 mrws = 0;
	177	+ u32 vf_pct;
	178	+ u32 nvfs;
	179	+
	180	+ vf_res = &qplib_ctx->vf_res;
	181	+ /*
	182	+ * Reserve a set of resources for the PF. Divide the remaining
	183	+ * resources among the VFs
	184	+ */
	185	+ vf_pct = 100 - BNXT_RE_PCT_RSVD_FOR_PF;
	186	+ nvfs = num_vf;
	187	+ num_vf = 100 * num_vf;
	188	+ vf_res->max_qp_per_vf = (qplib_ctx->qpc_count * vf_pct) / num_vf;
	189	+ vf_res->max_srq_per_vf = (qplib_ctx->srqc_count * vf_pct) / num_vf;
	190	+ vf_res->max_cq_per_vf = (qplib_ctx->cq_count * vf_pct) / num_vf;
	191	+ /*
	192	+ * The driver allows many more MRs than other resources. If the
	193	+ * firmware does also, then reserve a fixed amount for the PF and
	194	+ * divide the rest among VFs. VFs may use many MRs for NFS
	195	+ * mounts, ISER, NVME applications, etc. If the firmware severely
	196	+ * restricts the number of MRs, then let PF have half and divide
	197	+ * the rest among VFs, as for the other resource types.
	198	+ */
	199	+ if (qplib_ctx->mrw_count < BNXT_RE_MAX_MRW_COUNT_64K) {
	200	+ mrws = qplib_ctx->mrw_count * vf_pct;
	201	+ nvfs = num_vf;
	202	+ } else {
	203	+ mrws = qplib_ctx->mrw_count - BNXT_RE_RESVD_MR_FOR_PF;
	204	+ }
	205	+ vf_res->max_mrw_per_vf = (mrws / nvfs);
	206	+ vf_res->max_gid_per_vf = BNXT_RE_MAX_GID_PER_VF;
	207	+}
	208	+
99	209	static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)
100	210	{
101		- u32 vf_qps = 0, vf_srqs = 0, vf_cqs = 0, vf_mrws = 0, vf_gids = 0;
102		- u32 i;
103		- u32 vf_pct;
104	211	u32 num_vfs;
105		- struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
106	212
107		- rdev->qplib_ctx.qpc_count = min_t(u32, BNXT_RE_MAX_QPC_COUNT,
108		- dev_attr->max_qp);
	213	+ memset(&rdev->qplib_ctx.vf_res, 0, sizeof(struct bnxt_qplib_vf_res));
	214	+ bnxt_re_limit_pf_res(rdev);
109	215
110		- rdev->qplib_ctx.mrw_count = BNXT_RE_MAX_MRW_COUNT_256K;
111		- /* Use max_mr from fw since max_mrw does not get set */
112		- rdev->qplib_ctx.mrw_count = min_t(u32, rdev->qplib_ctx.mrw_count,
113		- dev_attr->max_mr);
114		- rdev->qplib_ctx.srqc_count = min_t(u32, BNXT_RE_MAX_SRQC_COUNT,
115		- dev_attr->max_srq);
116		- rdev->qplib_ctx.cq_count = min_t(u32, BNXT_RE_MAX_CQ_COUNT,
117		- dev_attr->max_cq);
118		-
119		- for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
120		- rdev->qplib_ctx.tqm_count[i] =
121		- rdev->dev_attr.tqm_alloc_reqs[i];
122		-
123		- if (rdev->num_vfs) {
124		- /*
125		- * Reserve a set of resources for the PF. Divide the remaining
126		- * resources among the VFs
127		- */
128		- vf_pct = 100 - BNXT_RE_PCT_RSVD_FOR_PF;
129		- num_vfs = 100 * rdev->num_vfs;
130		- vf_qps = (rdev->qplib_ctx.qpc_count * vf_pct) / num_vfs;
131		- vf_srqs = (rdev->qplib_ctx.srqc_count * vf_pct) / num_vfs;
132		- vf_cqs = (rdev->qplib_ctx.cq_count * vf_pct) / num_vfs;
133		- /*
134		- * The driver allows many more MRs than other resources. If the
135		- * firmware does also, then reserve a fixed amount for the PF
136		- * and divide the rest among VFs. VFs may use many MRs for NFS
137		- * mounts, ISER, NVME applications, etc. If the firmware
138		- * severely restricts the number of MRs, then let PF have
139		- * half and divide the rest among VFs, as for the other
140		- * resource types.
141		- */
142		- if (rdev->qplib_ctx.mrw_count < BNXT_RE_MAX_MRW_COUNT_64K)
143		- vf_mrws = rdev->qplib_ctx.mrw_count * vf_pct / num_vfs;
144		- else
145		- vf_mrws = (rdev->qplib_ctx.mrw_count -
146		- BNXT_RE_RESVD_MR_FOR_PF) / rdev->num_vfs;
147		- vf_gids = BNXT_RE_MAX_GID_PER_VF;
148		- }
149		- rdev->qplib_ctx.vf_res.max_mrw_per_vf = vf_mrws;
150		- rdev->qplib_ctx.vf_res.max_gid_per_vf = vf_gids;
151		- rdev->qplib_ctx.vf_res.max_qp_per_vf = vf_qps;
152		- rdev->qplib_ctx.vf_res.max_srq_per_vf = vf_srqs;
153		- rdev->qplib_ctx.vf_res.max_cq_per_vf = vf_cqs;
	216	+ num_vfs = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
	217	+ BNXT_RE_GEN_P5_MAX_VF : rdev->num_vfs;
	218	+ if (num_vfs)
	219	+ bnxt_re_limit_vf_res(&rdev->qplib_ctx, num_vfs);
154	220	}
155	221
156	222	/* for handling bnxt_en callbacks later */
..	..	@@ -170,9 +236,11 @@
170	236	return;
171	237
172	238	rdev->num_vfs = num_vfs;
173		- bnxt_re_set_resource_limits(rdev);
174		- bnxt_qplib_set_func_resources(&rdev->qplib_res, &rdev->rcfw,
175		- &rdev->qplib_ctx);
	239	+ if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
	240	+ bnxt_re_set_resource_limits(rdev);
	241	+ bnxt_qplib_set_func_resources(&rdev->qplib_res, &rdev->rcfw,
	242	+ &rdev->qplib_ctx);
	243	+ }
176	244	}
177	245
178	246	static void bnxt_re_shutdown(void *p)
..	..	@@ -181,8 +249,10 @@
181	249
182	250	if (!rdev)
183	251	return;
184		-
185		- bnxt_re_ib_unreg(rdev);
	252	+ ASSERT_RTNL();
	253	+ /* Release the MSIx vectors before queuing unregister */
	254	+ bnxt_re_stop_irq(rdev);
	255	+ ib_unregister_device_queued(&rdev->ibdev);
186	256	}
187	257
188	258	static void bnxt_re_stop_irq(void *handle)
..	..	@@ -214,7 +284,7 @@
214	284	* to f/w will timeout and that will set the
215	285	* timeout bit.
216	286	*/
217		- dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n");
	287	+ ibdev_err(&rdev->ibdev, "Failed to re-start IRQs\n");
218	288	return;
219	289	}
220	290
..	..	@@ -231,8 +301,8 @@
231	301	rc = bnxt_qplib_nq_start_irq(nq, indx - 1,
232	302	msix_ent[indx].vector, false);
233	303	if (rc)
234		- dev_warn(rdev_to_dev(rdev),
235		- "Failed to reinit NQ index %d\n", indx - 1);
	304	+ ibdev_warn(&rdev->ibdev, "Failed to reinit NQ index %d\n",
	305	+ indx - 1);
236	306	}
237	307	}
238	308
..	..	@@ -278,6 +348,7 @@
278	348
279	349	rc = en_dev->en_ops->bnxt_register_device(en_dev, BNXT_ROCE_ULP,
280	350	&bnxt_re_ulp_ops, rdev);
	351	+ rdev->qplib_res.pdev = rdev->en_dev->pdev;
281	352	return rc;
282	353	}
283	354
..	..	@@ -317,9 +388,9 @@
317	388	goto done;
318	389	}
319	390	if (num_msix_got != num_msix_want) {
320		- dev_warn(rdev_to_dev(rdev),
321		- "Requested %d MSI-X vectors, got %d\n",
322		- num_msix_want, num_msix_got);
	391	+ ibdev_warn(&rdev->ibdev,
	392	+ "Requested %d MSI-X vectors, got %d\n",
	393	+ num_msix_want, num_msix_got);
323	394	}
324	395	rdev->num_msix = num_msix_got;
325	396	done:
..	..	@@ -345,7 +416,8 @@
345	416	fw_msg->timeout = timeout;
346	417	}
347	418
348		-static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id)
	419	+static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev,
	420	+ u16 fw_ring_id, int type)
349	421	{
350	422	struct bnxt_en_dev *en_dev = rdev->en_dev;
351	423	struct hwrm_ring_free_input req = {0};
..	..	@@ -359,20 +431,20 @@
359	431	memset(&fw_msg, 0, sizeof(fw_msg));
360	432
361	433	bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1);
362		- req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
	434	+ req.ring_type = type;
363	435	req.ring_id = cpu_to_le16(fw_ring_id);
364	436	bnxt_re_fill_fw_msg(&fw_msg, (void )&req, sizeof(req), (void )&resp,
365	437	sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
366	438	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
367	439	if (rc)
368		- dev_err(rdev_to_dev(rdev),
369		- "Failed to free HW ring:%d :%#x", req.ring_id, rc);
	440	+ ibdev_err(&rdev->ibdev, "Failed to free HW ring:%d :%#x",
	441	+ req.ring_id, rc);
370	442	return rc;
371	443	}
372	444
373		-static int bnxt_re_net_ring_alloc(struct bnxt_re_dev rdev, dma_addr_t dma_arr,
374		- int pages, int type, u32 ring_mask,
375		- u32 map_index, u16 *fw_ring_id)
	445	+static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev,
	446	+ struct bnxt_re_ring_attr *ring_attr,
	447	+ u16 *fw_ring_id)
376	448	{
377	449	struct bnxt_en_dev *en_dev = rdev->en_dev;
378	450	struct hwrm_ring_alloc_input req = {0};
..	..	@@ -386,18 +458,18 @@
386	458	memset(&fw_msg, 0, sizeof(fw_msg));
387	459	bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1);
388	460	req.enables = 0;
389		- req.page_tbl_addr = cpu_to_le64(dma_arr[0]);
390		- if (pages > 1) {
	461	+ req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]);
	462	+ if (ring_attr->pages > 1) {
391	463	/* Page size is in log2 units */
392	464	req.page_size = BNXT_PAGE_SHIFT;
393	465	req.page_tbl_depth = 1;
394	466	}
395	467	req.fbo = 0;
396	468	/* Association of ring index with doorbell index and MSIX number */
397		- req.logical_id = cpu_to_le16(map_index);
398		- req.length = cpu_to_le32(ring_mask + 1);
399		- req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
400		- req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
	469	+ req.logical_id = cpu_to_le16(ring_attr->lrid);
	470	+ req.length = cpu_to_le32(ring_attr->depth + 1);
	471	+ req.ring_type = ring_attr->type;
	472	+ req.int_mode = ring_attr->mode;
401	473	bnxt_re_fill_fw_msg(&fw_msg, (void )&req, sizeof(req), (void )&resp,
402	474	sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
403	475	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
..	..	@@ -426,8 +498,8 @@
426	498	sizeof(req), DFLT_HWRM_CMD_TIMEOUT);
427	499	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
428	500	if (rc)
429		- dev_err(rdev_to_dev(rdev),
430		- "Failed to free HW stats context %#x", rc);
	501	+ ibdev_err(&rdev->ibdev, "Failed to free HW stats context %#x",
	502	+ rc);
431	503
432	504	return rc;
433	505	}
..	..	@@ -436,6 +508,7 @@
436	508	dma_addr_t dma_map,
437	509	u32 *fw_stats_ctx_id)
438	510	{
	511	+ struct bnxt_qplib_chip_ctx *chip_ctx = rdev->chip_ctx;
439	512	struct hwrm_stat_ctx_alloc_output resp = {0};
440	513	struct hwrm_stat_ctx_alloc_input req = {0};
441	514	struct bnxt_en_dev *en_dev = rdev->en_dev;
..	..	@@ -452,6 +525,7 @@
452	525	bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_ALLOC, -1, -1);
453	526	req.update_period_ms = cpu_to_le32(1000);
454	527	req.stats_dma_addr = cpu_to_le64(dma_map);
	528	+ req.stats_dma_length = cpu_to_le16(chip_ctx->hw_stats_size);
455	529	req.stat_ctx_flags = STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE;
456	530	bnxt_re_fill_fw_msg(&fw_msg, (void )&req, sizeof(req), (void )&resp,
457	531	sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
..	..	@@ -481,17 +555,12 @@
481	555
482	556	static struct bnxt_re_dev bnxt_re_from_netdev(struct net_device netdev)
483	557	{
484		- struct bnxt_re_dev *rdev;
	558	+ struct ib_device *ibdev =
	559	+ ib_device_get_by_netdev(netdev, RDMA_DRIVER_BNXT_RE);
	560	+ if (!ibdev)
	561	+ return NULL;
485	562
486		- rcu_read_lock();
487		- list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) {
488		- if (rdev->netdev == netdev) {
489		- rcu_read_unlock();
490		- return rdev;
491		- }
492		- }
493		- rcu_read_unlock();
494		- return NULL;
	563	+ return container_of(ibdev, struct bnxt_re_dev, ibdev);
495	564	}
496	565
497	566	static void bnxt_re_dev_unprobe(struct net_device *netdev,
..	..	@@ -535,31 +604,106 @@
535	604	return en_dev;
536	605	}
537	606
538		-static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev)
	607	+static ssize_t hw_rev_show(struct device device, struct device_attribute attr,
	608	+ char *buf)
539	609	{
540		- ib_unregister_device(&rdev->ibdev);
	610	+ struct bnxt_re_dev *rdev =
	611	+ rdma_device_to_drv_device(device, struct bnxt_re_dev, ibdev);
	612	+
	613	+ return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor);
541	614	}
	615	+static DEVICE_ATTR_RO(hw_rev);
	616	+
	617	+static ssize_t hca_type_show(struct device *device,
	618	+ struct device_attribute attr, char buf)
	619	+{
	620	+ struct bnxt_re_dev *rdev =
	621	+ rdma_device_to_drv_device(device, struct bnxt_re_dev, ibdev);
	622	+
	623	+ return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc);
	624	+}
	625	+static DEVICE_ATTR_RO(hca_type);
	626	+
	627	+static struct attribute *bnxt_re_attributes[] = {
	628	+ &dev_attr_hw_rev.attr,
	629	+ &dev_attr_hca_type.attr,
	630	+ NULL
	631	+};
	632	+
	633	+static const struct attribute_group bnxt_re_dev_attr_group = {
	634	+ .attrs = bnxt_re_attributes,
	635	+};
	636	+
	637	+static const struct ib_device_ops bnxt_re_dev_ops = {
	638	+ .owner = THIS_MODULE,
	639	+ .driver_id = RDMA_DRIVER_BNXT_RE,
	640	+ .uverbs_abi_ver = BNXT_RE_ABI_VERSION,
	641	+
	642	+ .add_gid = bnxt_re_add_gid,
	643	+ .alloc_hw_stats = bnxt_re_ib_alloc_hw_stats,
	644	+ .alloc_mr = bnxt_re_alloc_mr,
	645	+ .alloc_pd = bnxt_re_alloc_pd,
	646	+ .alloc_ucontext = bnxt_re_alloc_ucontext,
	647	+ .create_ah = bnxt_re_create_ah,
	648	+ .create_cq = bnxt_re_create_cq,
	649	+ .create_qp = bnxt_re_create_qp,
	650	+ .create_srq = bnxt_re_create_srq,
	651	+ .dealloc_driver = bnxt_re_dealloc_driver,
	652	+ .dealloc_pd = bnxt_re_dealloc_pd,
	653	+ .dealloc_ucontext = bnxt_re_dealloc_ucontext,
	654	+ .del_gid = bnxt_re_del_gid,
	655	+ .dereg_mr = bnxt_re_dereg_mr,
	656	+ .destroy_ah = bnxt_re_destroy_ah,
	657	+ .destroy_cq = bnxt_re_destroy_cq,
	658	+ .destroy_qp = bnxt_re_destroy_qp,
	659	+ .destroy_srq = bnxt_re_destroy_srq,
	660	+ .get_dev_fw_str = bnxt_re_query_fw_str,
	661	+ .get_dma_mr = bnxt_re_get_dma_mr,
	662	+ .get_hw_stats = bnxt_re_ib_get_hw_stats,
	663	+ .get_link_layer = bnxt_re_get_link_layer,
	664	+ .get_port_immutable = bnxt_re_get_port_immutable,
	665	+ .map_mr_sg = bnxt_re_map_mr_sg,
	666	+ .mmap = bnxt_re_mmap,
	667	+ .modify_ah = bnxt_re_modify_ah,
	668	+ .modify_qp = bnxt_re_modify_qp,
	669	+ .modify_srq = bnxt_re_modify_srq,
	670	+ .poll_cq = bnxt_re_poll_cq,
	671	+ .post_recv = bnxt_re_post_recv,
	672	+ .post_send = bnxt_re_post_send,
	673	+ .post_srq_recv = bnxt_re_post_srq_recv,
	674	+ .query_ah = bnxt_re_query_ah,
	675	+ .query_device = bnxt_re_query_device,
	676	+ .query_pkey = bnxt_re_query_pkey,
	677	+ .query_port = bnxt_re_query_port,
	678	+ .query_qp = bnxt_re_query_qp,
	679	+ .query_srq = bnxt_re_query_srq,
	680	+ .reg_user_mr = bnxt_re_reg_user_mr,
	681	+ .req_notify_cq = bnxt_re_req_notify_cq,
	682	+ INIT_RDMA_OBJ_SIZE(ib_ah, bnxt_re_ah, ib_ah),
	683	+ INIT_RDMA_OBJ_SIZE(ib_cq, bnxt_re_cq, ib_cq),
	684	+ INIT_RDMA_OBJ_SIZE(ib_pd, bnxt_re_pd, ib_pd),
	685	+ INIT_RDMA_OBJ_SIZE(ib_srq, bnxt_re_srq, ib_srq),
	686	+ INIT_RDMA_OBJ_SIZE(ib_ucontext, bnxt_re_ucontext, ib_uctx),
	687	+};
542	688
543	689	static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
544	690	{
545	691	struct ib_device *ibdev = &rdev->ibdev;
	692	+ int ret;
546	693
547	694	/* ib device init */
548		- ibdev->owner = THIS_MODULE;
549	695	ibdev->node_type = RDMA_NODE_IB_CA;
550		- strlcpy(ibdev->name, "bnxt_re%d", IB_DEVICE_NAME_MAX);
551	696	strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA",
552	697	strlen(BNXT_RE_DESC) + 5);
553	698	ibdev->phys_port_cnt = 1;
554	699
555	700	bnxt_qplib_get_guid(rdev->netdev->dev_addr, (u8 *)&ibdev->node_guid);
556	701
557		- ibdev->num_comp_vectors = 1;
	702	+ ibdev->num_comp_vectors = rdev->num_msix - 1;
558	703	ibdev->dev.parent = &rdev->en_dev->pdev->dev;
559	704	ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY;
560	705
561	706	/* User space */
562		- ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION;
563	707	ibdev->uverbs_cmd_mask =
564	708	(1ull << IB_USER_VERBS_CMD_GET_CONTEXT) \|
565	709	(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) \|
..	..	@@ -587,99 +731,26 @@
587	731	(1ull << IB_USER_VERBS_CMD_DESTROY_AH);
588	732	/* POLL_CQ and REQ_NOTIFY_CQ is directly handled in libbnxt_re */
589	733
590		- /* Kernel verbs */
591		- ibdev->query_device = bnxt_re_query_device;
592		- ibdev->modify_device = bnxt_re_modify_device;
593	734
594		- ibdev->query_port = bnxt_re_query_port;
595		- ibdev->get_port_immutable = bnxt_re_get_port_immutable;
596		- ibdev->get_dev_fw_str = bnxt_re_query_fw_str;
597		- ibdev->query_pkey = bnxt_re_query_pkey;
598		- ibdev->get_netdev = bnxt_re_get_netdev;
599		- ibdev->add_gid = bnxt_re_add_gid;
600		- ibdev->del_gid = bnxt_re_del_gid;
601		- ibdev->get_link_layer = bnxt_re_get_link_layer;
	735	+ rdma_set_device_sysfs_group(ibdev, &bnxt_re_dev_attr_group);
	736	+ ib_set_device_ops(ibdev, &bnxt_re_dev_ops);
	737	+ ret = ib_device_set_netdev(&rdev->ibdev, rdev->netdev, 1);
	738	+ if (ret)
	739	+ return ret;
602	740
603		- ibdev->alloc_pd = bnxt_re_alloc_pd;
604		- ibdev->dealloc_pd = bnxt_re_dealloc_pd;
605		-
606		- ibdev->create_ah = bnxt_re_create_ah;
607		- ibdev->modify_ah = bnxt_re_modify_ah;
608		- ibdev->query_ah = bnxt_re_query_ah;
609		- ibdev->destroy_ah = bnxt_re_destroy_ah;
610		-
611		- ibdev->create_srq = bnxt_re_create_srq;
612		- ibdev->modify_srq = bnxt_re_modify_srq;
613		- ibdev->query_srq = bnxt_re_query_srq;
614		- ibdev->destroy_srq = bnxt_re_destroy_srq;
615		- ibdev->post_srq_recv = bnxt_re_post_srq_recv;
616		-
617		- ibdev->create_qp = bnxt_re_create_qp;
618		- ibdev->modify_qp = bnxt_re_modify_qp;
619		- ibdev->query_qp = bnxt_re_query_qp;
620		- ibdev->destroy_qp = bnxt_re_destroy_qp;
621		-
622		- ibdev->post_send = bnxt_re_post_send;
623		- ibdev->post_recv = bnxt_re_post_recv;
624		-
625		- ibdev->create_cq = bnxt_re_create_cq;
626		- ibdev->destroy_cq = bnxt_re_destroy_cq;
627		- ibdev->poll_cq = bnxt_re_poll_cq;
628		- ibdev->req_notify_cq = bnxt_re_req_notify_cq;
629		-
630		- ibdev->get_dma_mr = bnxt_re_get_dma_mr;
631		- ibdev->dereg_mr = bnxt_re_dereg_mr;
632		- ibdev->alloc_mr = bnxt_re_alloc_mr;
633		- ibdev->map_mr_sg = bnxt_re_map_mr_sg;
634		-
635		- ibdev->reg_user_mr = bnxt_re_reg_user_mr;
636		- ibdev->alloc_ucontext = bnxt_re_alloc_ucontext;
637		- ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext;
638		- ibdev->mmap = bnxt_re_mmap;
639		- ibdev->get_hw_stats = bnxt_re_ib_get_hw_stats;
640		- ibdev->alloc_hw_stats = bnxt_re_ib_alloc_hw_stats;
641		-
642		- ibdev->driver_id = RDMA_DRIVER_BNXT_RE;
643		- return ib_register_device(ibdev, NULL);
	741	+ dma_set_max_seg_size(&rdev->en_dev->pdev->dev, UINT_MAX);
	742	+ return ib_register_device(ibdev, "bnxt_re%d", &rdev->en_dev->pdev->dev);
644	743	}
645		-
646		-static ssize_t show_rev(struct device device, struct device_attribute attr,
647		- char *buf)
648		-{
649		- struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
650		-
651		- return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor);
652		-}
653		-
654		-static ssize_t show_hca(struct device device, struct device_attribute attr,
655		- char *buf)
656		-{
657		- struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
658		-
659		- return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc);
660		-}
661		-
662		-static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL);
663		-static DEVICE_ATTR(hca_type, 0444, show_hca, NULL);
664		-
665		-static struct device_attribute *bnxt_re_attributes[] = {
666		- &dev_attr_hw_rev,
667		- &dev_attr_hca_type
668		-};
669	744
670	745	static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev)
671	746	{
672	747	dev_put(rdev->netdev);
673	748	rdev->netdev = NULL;
674		-
675	749	mutex_lock(&bnxt_re_dev_lock);
676	750	list_del_rcu(&rdev->list);
677	751	mutex_unlock(&bnxt_re_dev_lock);
678	752
679	753	synchronize_rcu();
680		-
681		- ib_dealloc_device(&rdev->ibdev);
682		- /* rdev is gone */
683	754	}
684	755
685	756	static struct bnxt_re_dev bnxt_re_dev_add(struct net_device netdev,
..	..	@@ -688,10 +759,10 @@
688	759	struct bnxt_re_dev *rdev;
689	760
690	761	/* Allocate bnxt_re_dev instance here */
691		- rdev = (struct bnxt_re_dev )ib_alloc_device(sizeof(rdev));
	762	+ rdev = ib_alloc_device(bnxt_re_dev, ibdev);
692	763	if (!rdev) {
693		- dev_err(NULL, "%s: bnxt_re_dev allocation failure!",
694		- ROCE_DRV_MODULE_NAME);
	764	+ ibdev_err(NULL, "%s: bnxt_re_dev allocation failure!",
	765	+ ROCE_DRV_MODULE_NAME);
695	766	return NULL;
696	767	}
697	768	/* Default values */
..	..	@@ -821,8 +892,8 @@
821	892	int rc = 0;
822	893
823	894	if (!srq) {
824		- dev_err(NULL, "%s: SRQ is NULL, SRQN not handled",
825		- ROCE_DRV_MODULE_NAME);
	895	+ ibdev_err(NULL, "%s: SRQ is NULL, SRQN not handled",
	896	+ ROCE_DRV_MODULE_NAME);
826	897	rc = -EINVAL;
827	898	goto done;
828	899	}
..	..	@@ -849,8 +920,8 @@
849	920	qplib_cq);
850	921
851	922	if (!cq) {
852		- dev_err(NULL, "%s: CQ is NULL, CQN not handled",
853		- ROCE_DRV_MODULE_NAME);
	923	+ ibdev_err(NULL, "%s: CQ is NULL, CQN not handled",
	924	+ ROCE_DRV_MODULE_NAME);
854	925	return -EINVAL;
855	926	}
856	927	if (cq->ib_cq.comp_handler) {
..	..	@@ -859,6 +930,16 @@
859	930	}
860	931
861	932	return 0;
	933	+}
	934	+
	935	+#define BNXT_RE_GEN_P5_PF_NQ_DB 0x10000
	936	+#define BNXT_RE_GEN_P5_VF_NQ_DB 0x4000
	937	+static u32 bnxt_re_get_nqdb_offset(struct bnxt_re_dev *rdev, u16 indx)
	938	+{
	939	+ return bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
	940	+ (rdev->is_virtfn ? BNXT_RE_GEN_P5_VF_NQ_DB :
	941	+ BNXT_RE_GEN_P5_PF_NQ_DB) :
	942	+ rdev->msix_entries[indx].db_offset;
862	943	}
863	944
864	945	static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev)
..	..	@@ -874,21 +955,21 @@
874	955
875	956	static int bnxt_re_init_res(struct bnxt_re_dev *rdev)
876	957	{
877		- int rc = 0, i;
878	958	int num_vec_enabled = 0;
	959	+ int rc = 0, i;
	960	+ u32 db_offt;
879	961
880	962	bnxt_qplib_init_res(&rdev->qplib_res);
881	963
882	964	for (i = 1; i < rdev->num_msix ; i++) {
	965	+ db_offt = bnxt_re_get_nqdb_offset(rdev, i);
883	966	rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq[i - 1],
884	967	i - 1, rdev->msix_entries[i].vector,
885		- rdev->msix_entries[i].db_offset,
886		- &bnxt_re_cqn_handler,
	968	+ db_offt, &bnxt_re_cqn_handler,
887	969	&bnxt_re_srqn_handler);
888		-
889	970	if (rc) {
890		- dev_err(rdev_to_dev(rdev),
891		- "Failed to enable NQ with rc = 0x%x", rc);
	971	+ ibdev_err(&rdev->ibdev,
	972	+ "Failed to enable NQ with rc = 0x%x", rc);
892	973	goto fail;
893	974	}
894	975	num_vec_enabled++;
..	..	@@ -897,17 +978,19 @@
897	978	fail:
898	979	for (i = num_vec_enabled; i >= 0; i--)
899	980	bnxt_qplib_disable_nq(&rdev->nq[i]);
900		-
901	981	return rc;
902	982	}
903	983
904	984	static void bnxt_re_free_nq_res(struct bnxt_re_dev *rdev)
905	985	{
	986	+ u8 type;
906	987	int i;
907	988
908	989	for (i = 0; i < rdev->num_msix - 1; i++) {
909		- bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id);
	990	+ type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
	991	+ bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id, type);
910	992	bnxt_qplib_free_nq(&rdev->nq[i]);
	993	+ rdev->nq[i].res = NULL;
911	994	}
912	995	}
913	996
..	..	@@ -928,8 +1011,10 @@
928	1011
929	1012	static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
930	1013	{
931		- int rc = 0, i;
	1014	+ struct bnxt_re_ring_attr rattr = {};
932	1015	int num_vec_created = 0;
	1016	+ int rc = 0, i;
	1017	+ u8 type;
933	1018
934	1019	/* Configure and allocate resources for qplib */
935	1020	rdev->qplib_res.rcfw = &rdev->rcfw;
..	..	@@ -950,25 +1035,28 @@
950	1035	goto dealloc_res;
951	1036
952	1037	for (i = 0; i < rdev->num_msix - 1; i++) {
953		- rdev->nq[i].hwq.max_elements = BNXT_RE_MAX_CQ_COUNT +
954		- BNXT_RE_MAX_SRQC_COUNT + 2;
955		- rc = bnxt_qplib_alloc_nq(rdev->en_dev->pdev, &rdev->nq[i]);
	1038	+ struct bnxt_qplib_nq *nq;
	1039	+
	1040	+ nq = &rdev->nq[i];
	1041	+ nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
	1042	+ rc = bnxt_qplib_alloc_nq(&rdev->qplib_res, &rdev->nq[i]);
956	1043	if (rc) {
957		- dev_err(rdev_to_dev(rdev), "Alloc Failed NQ%d rc:%#x",
958		- i, rc);
	1044	+ ibdev_err(&rdev->ibdev, "Alloc Failed NQ%d rc:%#x",
	1045	+ i, rc);
959	1046	goto free_nq;
960	1047	}
961		- rc = bnxt_re_net_ring_alloc
962		- (rdev, rdev->nq[i].hwq.pbl[PBL_LVL_0].pg_map_arr,
963		- rdev->nq[i].hwq.pbl[rdev->nq[i].hwq.level].pg_count,
964		- HWRM_RING_ALLOC_CMPL,
965		- BNXT_QPLIB_NQE_MAX_CNT - 1,
966		- rdev->msix_entries[i + 1].ring_idx,
967		- &rdev->nq[i].ring_id);
	1048	+ type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
	1049	+ rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr;
	1050	+ rattr.pages = nq->hwq.pbl[rdev->nq[i].hwq.level].pg_count;
	1051	+ rattr.type = type;
	1052	+ rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
	1053	+ rattr.depth = BNXT_QPLIB_NQE_MAX_CNT - 1;
	1054	+ rattr.lrid = rdev->msix_entries[i + 1].ring_idx;
	1055	+ rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id);
968	1056	if (rc) {
969		- dev_err(rdev_to_dev(rdev),
970		- "Failed to allocate NQ fw id with rc = 0x%x",
971		- rc);
	1057	+ ibdev_err(&rdev->ibdev,
	1058	+ "Failed to allocate NQ fw id with rc = 0x%x",
	1059	+ rc);
972	1060	bnxt_qplib_free_nq(&rdev->nq[i]);
973	1061	goto free_nq;
974	1062	}
..	..	@@ -976,8 +1064,9 @@
976	1064	}
977	1065	return 0;
978	1066	free_nq:
979		- for (i = num_vec_created; i >= 0; i--) {
980		- bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id);
	1067	+ for (i = num_vec_created - 1; i >= 0; i--) {
	1068	+ type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
	1069	+ bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id, type);
981	1070	bnxt_qplib_free_nq(&rdev->nq[i]);
982	1071	}
983	1072	bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
..	..	@@ -1041,10 +1130,10 @@
1041	1130	return rc;
1042	1131
1043	1132	if (resp.queue_cfg_info) {
1044		- dev_warn(rdev_to_dev(rdev),
1045		- "Asymmetric cos queue configuration detected");
1046		- dev_warn(rdev_to_dev(rdev),
1047		- " on device, QoS may not be fully functional\n");
	1133	+ ibdev_warn(&rdev->ibdev,
	1134	+ "Asymmetric cos queue configuration detected");
	1135	+ ibdev_warn(&rdev->ibdev,
	1136	+ " on device, QoS may not be fully functional\n");
1048	1137	}
1049	1138	qcfgmap = &resp.pri0_cos_queue_id;
1050	1139	tmp_map = (u8 *)cid_map;
..	..	@@ -1057,7 +1146,8 @@
1057	1146	static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev,
1058	1147	struct bnxt_re_qp *qp)
1059	1148	{
1060		- return (qp->ib_qp.qp_type == IB_QPT_GSI) \|\| (qp == rdev->qp1_sqp);
	1149	+ return (qp->ib_qp.qp_type == IB_QPT_GSI) \|\|
	1150	+ (qp == rdev->gsi_ctx.gsi_sqp);
1061	1151	}
1062	1152
1063	1153	static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev)
..	..	@@ -1092,12 +1182,13 @@
1092	1182	u16 gid_idx, index;
1093	1183	int rc = 0;
1094	1184
1095		- if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
	1185	+ if (!ib_device_try_get(&rdev->ibdev))
1096	1186	return 0;
1097	1187
1098	1188	if (!sgid_tbl) {
1099		- dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated");
1100		- return -EINVAL;
	1189	+ ibdev_err(&rdev->ibdev, "QPLIB: SGID table not allocated");
	1190	+ rc = -EINVAL;
	1191	+ goto out;
1101	1192	}
1102	1193
1103	1194	for (index = 0; index < sgid_tbl->active; index++) {
..	..	@@ -1117,7 +1208,8 @@
1117	1208	rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx,
1118	1209	rdev->qplib_res.netdev->dev_addr);
1119	1210	}
1120		-
	1211	+out:
	1212	+ ib_device_put(&rdev->ibdev);
1121	1213	return rc;
1122	1214	}
1123	1215
..	..	@@ -1173,7 +1265,7 @@
1173	1265	/* Get cosq id for this priority */
1174	1266	rc = bnxt_re_query_hwrm_pri2cos(rdev, 0, &cid_map);
1175	1267	if (rc) {
1176		- dev_warn(rdev_to_dev(rdev), "no cos for p_mask %x\n", prio_map);
	1268	+ ibdev_warn(&rdev->ibdev, "no cos for p_mask %x\n", prio_map);
1177	1269	return rc;
1178	1270	}
1179	1271	/* Parse CoS IDs for app priority */
..	..	@@ -1182,8 +1274,8 @@
1182	1274	/* Config BONO. */
1183	1275	rc = bnxt_qplib_map_tc2cos(&rdev->qplib_res, rdev->cosq);
1184	1276	if (rc) {
1185		- dev_warn(rdev_to_dev(rdev), "no tc for cos{%x, %x}\n",
1186		- rdev->cosq[0], rdev->cosq[1]);
	1277	+ ibdev_warn(&rdev->ibdev, "no tc for cos{%x, %x}\n",
	1278	+ rdev->cosq[0], rdev->cosq[1]);
1187	1279	return rc;
1188	1280	}
1189	1281
..	..	@@ -1200,19 +1292,66 @@
1200	1292	return 0;
1201	1293	}
1202	1294
1203		-static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev)
	1295	+static void bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev)
1204	1296	{
1205		- int i, rc;
	1297	+ struct bnxt_en_dev *en_dev = rdev->en_dev;
	1298	+ struct hwrm_ver_get_output resp = {0};
	1299	+ struct hwrm_ver_get_input req = {0};
	1300	+ struct bnxt_fw_msg fw_msg;
	1301	+ int rc = 0;
1206	1302
1207		- if (test_and_clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) {
1208		- for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++)
1209		- device_remove_file(&rdev->ibdev.dev,
1210		- bnxt_re_attributes[i]);
1211		- /* Cleanup ib dev */
1212		- bnxt_re_unregister_ib(rdev);
	1303	+ memset(&fw_msg, 0, sizeof(fw_msg));
	1304	+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
	1305	+ HWRM_VER_GET, -1, -1);
	1306	+ req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
	1307	+ req.hwrm_intf_min = HWRM_VERSION_MINOR;
	1308	+ req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
	1309	+ bnxt_re_fill_fw_msg(&fw_msg, (void )&req, sizeof(req), (void )&resp,
	1310	+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
	1311	+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
	1312	+ if (rc) {
	1313	+ ibdev_err(&rdev->ibdev, "Failed to query HW version, rc = 0x%x",
	1314	+ rc);
	1315	+ return;
1213	1316	}
	1317	+ rdev->qplib_ctx.hwrm_intf_ver =
	1318	+ (u64)le16_to_cpu(resp.hwrm_intf_major) << 48 \|
	1319	+ (u64)le16_to_cpu(resp.hwrm_intf_minor) << 32 \|
	1320	+ (u64)le16_to_cpu(resp.hwrm_intf_build) << 16 \|
	1321	+ le16_to_cpu(resp.hwrm_intf_patch);
	1322	+}
	1323	+
	1324	+static int bnxt_re_ib_init(struct bnxt_re_dev *rdev)
	1325	+{
	1326	+ int rc = 0;
	1327	+ u32 event;
	1328	+
	1329	+ /* Register ib dev */
	1330	+ rc = bnxt_re_register_ib(rdev);
	1331	+ if (rc) {
	1332	+ pr_err("Failed to register with IB: %#x\n", rc);
	1333	+ return rc;
	1334	+ }
	1335	+ dev_info(rdev_to_dev(rdev), "Device registered successfully");
	1336	+ ib_get_eth_speed(&rdev->ibdev, 1, &rdev->active_speed,
	1337	+ &rdev->active_width);
	1338	+ set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags);
	1339	+
	1340	+ event = netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev) ?
	1341	+ IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
	1342	+
	1343	+ bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, event);
	1344	+
	1345	+ return rc;
	1346	+}
	1347	+
	1348	+static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev)
	1349	+{
	1350	+ u8 type;
	1351	+ int rc;
	1352	+
1214	1353	if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags))
1215		- cancel_delayed_work(&rdev->worker);
	1354	+ cancel_delayed_work_sync(&rdev->worker);
1216	1355
1217	1356	if (test_and_clear_bit(BNXT_RE_FLAG_RESOURCES_INITIALIZED,
1218	1357	&rdev->flags))
..	..	@@ -1223,25 +1362,28 @@
1223	1362	if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) {
1224	1363	rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw);
1225	1364	if (rc)
1226		- dev_warn(rdev_to_dev(rdev),
1227		- "Failed to deinitialize RCFW: %#x", rc);
	1365	+ ibdev_warn(&rdev->ibdev,
	1366	+ "Failed to deinitialize RCFW: %#x", rc);
1228	1367	bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id);
1229		- bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx);
	1368	+ bnxt_qplib_free_ctx(&rdev->qplib_res, &rdev->qplib_ctx);
1230	1369	bnxt_qplib_disable_rcfw_channel(&rdev->rcfw);
1231		- bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id);
	1370	+ type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
	1371	+ bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type);
1232	1372	bnxt_qplib_free_rcfw_channel(&rdev->rcfw);
1233	1373	}
1234	1374	if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) {
1235	1375	rc = bnxt_re_free_msix(rdev);
1236	1376	if (rc)
1237		- dev_warn(rdev_to_dev(rdev),
1238		- "Failed to free MSI-X vectors: %#x", rc);
	1377	+ ibdev_warn(&rdev->ibdev,
	1378	+ "Failed to free MSI-X vectors: %#x", rc);
1239	1379	}
	1380	+
	1381	+ bnxt_re_destroy_chip_ctx(rdev);
1240	1382	if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) {
1241	1383	rc = bnxt_re_unregister_netdev(rdev);
1242	1384	if (rc)
1243		- dev_warn(rdev_to_dev(rdev),
1244		- "Failed to unregister with netdev: %#x", rc);
	1385	+ ibdev_warn(&rdev->ibdev,
	1386	+ "Failed to unregister with netdev: %#x", rc);
1245	1387	}
1246	1388	}
1247	1389
..	..	@@ -1255,62 +1397,78 @@
1255	1397	schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
1256	1398	}
1257	1399
1258		-static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
	1400	+static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode)
1259	1401	{
1260		- int i, j, rc;
1261		-
1262		- bool locked;
1263		-
1264		- /* Acquire rtnl lock through out this function */
1265		- rtnl_lock();
1266		- locked = true;
	1402	+ struct bnxt_qplib_creq_ctx *creq;
	1403	+ struct bnxt_re_ring_attr rattr;
	1404	+ u32 db_offt;
	1405	+ int vid;
	1406	+ u8 type;
	1407	+ int rc;
1267	1408
1268	1409	/* Registered a new RoCE device instance to netdev */
	1410	+ memset(&rattr, 0, sizeof(rattr));
1269	1411	rc = bnxt_re_register_netdev(rdev);
1270	1412	if (rc) {
1271		- rtnl_unlock();
1272		- pr_err("Failed to register with netedev: %#x\n", rc);
	1413	+ ibdev_err(&rdev->ibdev,
	1414	+ "Failed to register with netedev: %#x\n", rc);
1273	1415	return -EINVAL;
1274	1416	}
1275	1417	set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
	1418	+
	1419	+ rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode);
	1420	+ if (rc) {
	1421	+ ibdev_err(&rdev->ibdev, "Failed to get chip context\n");
	1422	+ return -EINVAL;
	1423	+ }
1276	1424
1277	1425	/* Check whether VF or PF */
1278	1426	bnxt_re_get_sriov_func_type(rdev);
1279	1427
1280	1428	rc = bnxt_re_request_msix(rdev);
1281	1429	if (rc) {
1282		- pr_err("Failed to get MSI-X vectors: %#x\n", rc);
	1430	+ ibdev_err(&rdev->ibdev,
	1431	+ "Failed to get MSI-X vectors: %#x\n", rc);
1283	1432	rc = -EINVAL;
1284	1433	goto fail;
1285	1434	}
1286	1435	set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags);
1287	1436
	1437	+ bnxt_re_query_hwrm_intf_version(rdev);
	1438	+
1288	1439	/* Establish RCFW Communication Channel to initialize the context
1289	1440	* memory for the function and all child VFs
1290	1441	*/
1291		- rc = bnxt_qplib_alloc_rcfw_channel(rdev->en_dev->pdev, &rdev->rcfw,
	1442	+ rc = bnxt_qplib_alloc_rcfw_channel(&rdev->qplib_res, &rdev->rcfw,
	1443	+ &rdev->qplib_ctx,
1292	1444	BNXT_RE_MAX_QPC_COUNT);
1293	1445	if (rc) {
1294		- pr_err("Failed to allocate RCFW Channel: %#x\n", rc);
	1446	+ ibdev_err(&rdev->ibdev,
	1447	+ "Failed to allocate RCFW Channel: %#x\n", rc);
1295	1448	goto fail;
1296	1449	}
1297		- rc = bnxt_re_net_ring_alloc
1298		- (rdev, rdev->rcfw.creq.pbl[PBL_LVL_0].pg_map_arr,
1299		- rdev->rcfw.creq.pbl[rdev->rcfw.creq.level].pg_count,
1300		- HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_CREQE_MAX_CNT - 1,
1301		- rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx,
1302		- &rdev->rcfw.creq_ring_id);
	1450	+
	1451	+ type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
	1452	+ creq = &rdev->rcfw.creq;
	1453	+ rattr.dma_arr = creq->hwq.pbl[PBL_LVL_0].pg_map_arr;
	1454	+ rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count;
	1455	+ rattr.type = type;
	1456	+ rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
	1457	+ rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1;
	1458	+ rattr.lrid = rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx;
	1459	+ rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id);
1303	1460	if (rc) {
1304		- pr_err("Failed to allocate CREQ: %#x\n", rc);
	1461	+ ibdev_err(&rdev->ibdev, "Failed to allocate CREQ: %#x\n", rc);
1305	1462	goto free_rcfw;
1306	1463	}
1307		- rc = bnxt_qplib_enable_rcfw_channel
1308		- (rdev->en_dev->pdev, &rdev->rcfw,
1309		- rdev->msix_entries[BNXT_RE_AEQ_IDX].vector,
1310		- rdev->msix_entries[BNXT_RE_AEQ_IDX].db_offset,
1311		- rdev->is_virtfn, &bnxt_re_aeq_handler);
	1464	+ db_offt = bnxt_re_get_nqdb_offset(rdev, BNXT_RE_AEQ_IDX);
	1465	+ vid = rdev->msix_entries[BNXT_RE_AEQ_IDX].vector;
	1466	+ rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw,
	1467	+ vid, db_offt, rdev->is_virtfn,
	1468	+ &bnxt_re_aeq_handler);
1312	1469	if (rc) {
1313		- pr_err("Failed to enable RCFW channel: %#x\n", rc);
	1470	+ ibdev_err(&rdev->ibdev, "Failed to enable RCFW channel: %#x\n",
	1471	+ rc);
1314	1472	goto free_ring;
1315	1473	}
1316	1474
..	..	@@ -1318,26 +1476,30 @@
1318	1476	rdev->is_virtfn);
1319	1477	if (rc)
1320	1478	goto disable_rcfw;
1321		- if (!rdev->is_virtfn)
1322		- bnxt_re_set_resource_limits(rdev);
1323	1479
1324		- rc = bnxt_qplib_alloc_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx, 0);
	1480	+ bnxt_re_set_resource_limits(rdev);
	1481	+
	1482	+ rc = bnxt_qplib_alloc_ctx(&rdev->qplib_res, &rdev->qplib_ctx, 0,
	1483	+ bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx));
1325	1484	if (rc) {
1326		- pr_err("Failed to allocate QPLIB context: %#x\n", rc);
	1485	+ ibdev_err(&rdev->ibdev,
	1486	+ "Failed to allocate QPLIB context: %#x\n", rc);
1327	1487	goto disable_rcfw;
1328	1488	}
1329	1489	rc = bnxt_re_net_stats_ctx_alloc(rdev,
1330	1490	rdev->qplib_ctx.stats.dma_map,
1331	1491	&rdev->qplib_ctx.stats.fw_id);
1332	1492	if (rc) {
1333		- pr_err("Failed to allocate stats context: %#x\n", rc);
	1493	+ ibdev_err(&rdev->ibdev,
	1494	+ "Failed to allocate stats context: %#x\n", rc);
1334	1495	goto free_ctx;
1335	1496	}
1336	1497
1337	1498	rc = bnxt_qplib_init_rcfw(&rdev->rcfw, &rdev->qplib_ctx,
1338	1499	rdev->is_virtfn);
1339	1500	if (rc) {
1340		- pr_err("Failed to initialize RCFW: %#x\n", rc);
	1501	+ ibdev_err(&rdev->ibdev,
	1502	+ "Failed to initialize RCFW: %#x\n", rc);
1341	1503	goto free_sctx;
1342	1504	}
1343	1505	set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags);
..	..	@@ -1345,13 +1507,15 @@
1345	1507	/* Resources based on the 'new' device caps */
1346	1508	rc = bnxt_re_alloc_res(rdev);
1347	1509	if (rc) {
1348		- pr_err("Failed to allocate resources: %#x\n", rc);
	1510	+ ibdev_err(&rdev->ibdev,
	1511	+ "Failed to allocate resources: %#x\n", rc);
1349	1512	goto fail;
1350	1513	}
1351	1514	set_bit(BNXT_RE_FLAG_RESOURCES_ALLOCATED, &rdev->flags);
1352	1515	rc = bnxt_re_init_res(rdev);
1353	1516	if (rc) {
1354		- pr_err("Failed to initialize resources: %#x\n", rc);
	1517	+ ibdev_err(&rdev->ibdev,
	1518	+ "Failed to initialize resources: %#x\n", rc);
1355	1519	goto fail;
1356	1520	}
1357	1521
..	..	@@ -1360,60 +1524,28 @@
1360	1524	if (!rdev->is_virtfn) {
1361	1525	rc = bnxt_re_setup_qos(rdev);
1362	1526	if (rc)
1363		- pr_info("RoCE priority not yet configured\n");
	1527	+ ibdev_info(&rdev->ibdev,
	1528	+ "RoCE priority not yet configured\n");
1364	1529
1365	1530	INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker);
1366	1531	set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags);
1367	1532	schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
1368	1533	}
1369	1534
1370		- rtnl_unlock();
1371		- locked = false;
1372		-
1373		- /* Register ib dev */
1374		- rc = bnxt_re_register_ib(rdev);
1375		- if (rc) {
1376		- pr_err("Failed to register with IB: %#x\n", rc);
1377		- goto fail;
1378		- }
1379		- set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
1380		- dev_info(rdev_to_dev(rdev), "Device registered successfully");
1381		- for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) {
1382		- rc = device_create_file(&rdev->ibdev.dev,
1383		- bnxt_re_attributes[i]);
1384		- if (rc) {
1385		- dev_err(rdev_to_dev(rdev),
1386		- "Failed to create IB sysfs: %#x", rc);
1387		- /* Must clean up all created device files */
1388		- for (j = 0; j < i; j++)
1389		- device_remove_file(&rdev->ibdev.dev,
1390		- bnxt_re_attributes[j]);
1391		- bnxt_re_unregister_ib(rdev);
1392		- goto fail;
1393		- }
1394		- }
1395		- ib_get_eth_speed(&rdev->ibdev, 1, &rdev->active_speed,
1396		- &rdev->active_width);
1397		- set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags);
1398		- bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE);
1399		- bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE);
1400		-
1401	1535	return 0;
1402	1536	free_sctx:
1403	1537	bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id);
1404	1538	free_ctx:
1405		- bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx);
	1539	+ bnxt_qplib_free_ctx(&rdev->qplib_res, &rdev->qplib_ctx);
1406	1540	disable_rcfw:
1407	1541	bnxt_qplib_disable_rcfw_channel(&rdev->rcfw);
1408	1542	free_ring:
1409		- bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id);
	1543	+ type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
	1544	+ bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type);
1410	1545	free_rcfw:
1411	1546	bnxt_qplib_free_rcfw_channel(&rdev->rcfw);
1412	1547	fail:
1413		- if (!locked)
1414		- rtnl_lock();
1415		- bnxt_re_ib_unreg(rdev);
1416		- rtnl_unlock();
	1548	+ bnxt_re_dev_uninit(rdev);
1417	1549
1418	1550	return rc;
1419	1551	}
..	..	@@ -1440,7 +1572,8 @@
1440	1572	en_dev = bnxt_re_dev_probe(netdev);
1441	1573	if (IS_ERR(en_dev)) {
1442	1574	if (en_dev != ERR_PTR(-ENODEV))
1443		- pr_err("%s: Failed to probe\n", ROCE_DRV_MODULE_NAME);
	1575	+ ibdev_err(&(*rdev)->ibdev, "%s: Failed to probe\n",
	1576	+ ROCE_DRV_MODULE_NAME);
1444	1577	rc = PTR_ERR(en_dev);
1445	1578	goto exit;
1446	1579	}
..	..	@@ -1454,9 +1587,47 @@
1454	1587	return rc;
1455	1588	}
1456	1589
1457		-static void bnxt_re_remove_one(struct bnxt_re_dev *rdev)
	1590	+static void bnxt_re_remove_device(struct bnxt_re_dev *rdev)
1458	1591	{
	1592	+ bnxt_re_dev_uninit(rdev);
1459	1593	pci_dev_put(rdev->en_dev->pdev);
	1594	+ bnxt_re_dev_unreg(rdev);
	1595	+}
	1596	+
	1597	+static int bnxt_re_add_device(struct bnxt_re_dev **rdev,
	1598	+ struct net_device *netdev, u8 wqe_mode)
	1599	+{
	1600	+ int rc;
	1601	+
	1602	+ rc = bnxt_re_dev_reg(rdev, netdev);
	1603	+ if (rc == -ENODEV)
	1604	+ return rc;
	1605	+ if (rc) {
	1606	+ pr_err("Failed to register with the device %s: %#x\n",
	1607	+ netdev->name, rc);
	1608	+ return rc;
	1609	+ }
	1610	+
	1611	+ pci_dev_get((*rdev)->en_dev->pdev);
	1612	+ rc = bnxt_re_dev_init(*rdev, wqe_mode);
	1613	+ if (rc) {
	1614	+ pci_dev_put((*rdev)->en_dev->pdev);
	1615	+ bnxt_re_dev_unreg(*rdev);
	1616	+ }
	1617	+
	1618	+ return rc;
	1619	+}
	1620	+
	1621	+static void bnxt_re_dealloc_driver(struct ib_device *ib_dev)
	1622	+{
	1623	+ struct bnxt_re_dev *rdev =
	1624	+ container_of(ib_dev, struct bnxt_re_dev, ibdev);
	1625	+
	1626	+ dev_info(rdev_to_dev(rdev), "Unregistering Device");
	1627	+
	1628	+ rtnl_lock();
	1629	+ bnxt_re_remove_device(rdev);
	1630	+ rtnl_unlock();
1460	1631	}
1461	1632
1462	1633	/* Handle all deferred netevents tasks */
..	..	@@ -1469,21 +1640,23 @@
1469	1640	re_work = container_of(work, struct bnxt_re_work, work);
1470	1641	rdev = re_work->rdev;
1471	1642
1472		- if (re_work->event != NETDEV_REGISTER &&
1473		- !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
1474		- return;
1475		-
1476		- switch (re_work->event) {
1477		- case NETDEV_REGISTER:
1478		- rc = bnxt_re_ib_reg(rdev);
	1643	+ if (re_work->event == NETDEV_REGISTER) {
	1644	+ rc = bnxt_re_ib_init(rdev);
1479	1645	if (rc) {
1480		- dev_err(rdev_to_dev(rdev),
1481		- "Failed to register with IB: %#x", rc);
1482		- bnxt_re_remove_one(rdev);
1483		- bnxt_re_dev_unreg(rdev);
	1646	+ ibdev_err(&rdev->ibdev,
	1647	+ "Failed to register with IB: %#x", rc);
	1648	+ rtnl_lock();
	1649	+ bnxt_re_remove_device(rdev);
	1650	+ rtnl_unlock();
1484	1651	goto exit;
1485	1652	}
1486		- break;
	1653	+ goto exit;
	1654	+ }
	1655	+
	1656	+ if (!ib_device_try_get(&rdev->ibdev))
	1657	+ goto exit;
	1658	+
	1659	+ switch (re_work->event) {
1487	1660	case NETDEV_UP:
1488	1661	bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
1489	1662	IB_EVENT_PORT_ACTIVE);
..	..	@@ -1503,15 +1676,10 @@
1503	1676	default:
1504	1677	break;
1505	1678	}
1506		- smp_mb__before_atomic();
1507		- atomic_dec(&rdev->sched_count);
	1679	+ ib_device_put(&rdev->ibdev);
1508	1680	exit:
	1681	+ put_device(&rdev->ibdev.dev);
1509	1682	kfree(re_work);
1510		-}
1511		-
1512		-static void bnxt_re_init_one(struct bnxt_re_dev *rdev)
1513		-{
1514		- pci_dev_get(rdev->en_dev->pdev);
1515	1683	}
1516	1684
1517	1685	/*
..	..	@@ -1536,6 +1704,7 @@
1536	1704	struct bnxt_re_dev *rdev;
1537	1705	int rc = 0;
1538	1706	bool sch_work = false;
	1707	+ bool release = true;
1539	1708
1540	1709	real_dev = rdma_vlan_dev_real_dev(netdev);
1541	1710	if (!real_dev)
..	..	@@ -1543,7 +1712,8 @@
1543	1712
1544	1713	rdev = bnxt_re_from_netdev(real_dev);
1545	1714	if (!rdev && event != NETDEV_REGISTER)
1546		- goto exit;
	1715	+ return NOTIFY_OK;
	1716	+
1547	1717	if (real_dev != netdev)
1548	1718	goto exit;
1549	1719
..	..	@@ -1551,27 +1721,15 @@
1551	1721	case NETDEV_REGISTER:
1552	1722	if (rdev)
1553	1723	break;
1554		- rc = bnxt_re_dev_reg(&rdev, real_dev);
1555		- if (rc == -ENODEV)
1556		- break;
1557		- if (rc) {
1558		- pr_err("Failed to register with the device %s: %#x\n",
1559		- real_dev->name, rc);
1560		- break;
1561		- }
1562		- bnxt_re_init_one(rdev);
1563		- sch_work = true;
	1724	+ rc = bnxt_re_add_device(&rdev, real_dev,
	1725	+ BNXT_QPLIB_WQE_MODE_STATIC);
	1726	+ if (!rc)
	1727	+ sch_work = true;
	1728	+ release = false;
1564	1729	break;
1565	1730
1566	1731	case NETDEV_UNREGISTER:
1567		- /* netdev notifier will call NETDEV_UNREGISTER again later since
1568		- * we are still holding the reference to the netdev
1569		- */
1570		- if (atomic_read(&rdev->sched_count) > 0)
1571		- goto exit;
1572		- bnxt_re_ib_unreg(rdev);
1573		- bnxt_re_remove_one(rdev);
1574		- bnxt_re_dev_unreg(rdev);
	1732	+ ib_unregister_device_queued(&rdev->ibdev);
1575	1733	break;
1576	1734
1577	1735	default:
..	..	@@ -1582,17 +1740,19 @@
1582	1740	/* Allocate for the deferred task */
1583	1741	re_work = kzalloc(sizeof(*re_work), GFP_ATOMIC);
1584	1742	if (re_work) {
	1743	+ get_device(&rdev->ibdev.dev);
1585	1744	re_work->rdev = rdev;
1586	1745	re_work->event = event;
1587	1746	re_work->vlan_dev = (real_dev == netdev ?
1588	1747	NULL : netdev);
1589	1748	INIT_WORK(&re_work->work, bnxt_re_task);
1590		- atomic_inc(&rdev->sched_count);
1591	1749	queue_work(bnxt_re_wq, &re_work->work);
1592	1750	}
1593	1751	}
1594	1752
1595	1753	exit:
	1754	+ if (rdev && release)
	1755	+ ib_device_put(&rdev->ibdev);
1596	1756	return NOTIFY_DONE;
1597	1757	}
1598	1758
..	..	@@ -1628,36 +1788,21 @@
1628	1788
1629	1789	static void __exit bnxt_re_mod_exit(void)
1630	1790	{
1631		- struct bnxt_re_dev rdev, next;
1632		- LIST_HEAD(to_be_deleted);
	1791	+ struct bnxt_re_dev *rdev;
1633	1792
1634		- mutex_lock(&bnxt_re_dev_lock);
1635		- /* Free all adapter allocated resources */
1636		- if (!list_empty(&bnxt_re_dev_list))
1637		- list_splice_init(&bnxt_re_dev_list, &to_be_deleted);
1638		- mutex_unlock(&bnxt_re_dev_lock);
1639		- /*
1640		- * Cleanup the devices in reverse order so that the VF device
1641		- * cleanup is done before PF cleanup
1642		- */
1643		- list_for_each_entry_safe_reverse(rdev, next, &to_be_deleted, list) {
1644		- dev_info(rdev_to_dev(rdev), "Unregistering Device");
1645		- /*
1646		- * Flush out any scheduled tasks before destroying the
1647		- * resources
1648		- */
1649		- flush_workqueue(bnxt_re_wq);
1650		- bnxt_re_dev_stop(rdev);
1651		- /* Acquire the rtnl_lock as the L2 resources are freed here */
1652		- rtnl_lock();
1653		- bnxt_re_ib_unreg(rdev);
1654		- rtnl_unlock();
1655		- bnxt_re_remove_one(rdev);
1656		- bnxt_re_dev_unreg(rdev);
1657		- }
1658	1793	unregister_netdevice_notifier(&bnxt_re_netdev_notifier);
1659	1794	if (bnxt_re_wq)
1660	1795	destroy_workqueue(bnxt_re_wq);
	1796	+ list_for_each_entry(rdev, &bnxt_re_dev_list, list) {
	1797	+ /* VF device removal should be called before the removal
	1798	+ * of PF device. Queue VFs unregister first, so that VFs
	1799	+ * shall be removed before the PF during the call of
	1800	+ * ib_unregister_driver.
	1801	+ */
	1802	+ if (rdev->is_virtfn)
	1803	+ ib_unregister_device(&rdev->ibdev);
	1804	+ }
	1805	+ ib_unregister_driver(RDMA_DRIVER_BNXT_RE);
1661	1806	}
1662	1807
1663	1808	module_init(bnxt_re_mod_init);