~hc/RK356X_SDK_RELEASE.git

..	..	@@ -50,8 +50,10 @@
50	50	#include <rdma/ib_cache.h>
51	51	#include <rdma/ib_addr.h>
52	52	#include <rdma/rw.h>
	53	+#include <rdma/lag.h>
53	54
54	55	#include "core_priv.h"
	56	+#include <trace/events/rdma_core.h>
55	57
56	58	static int ib_resolve_eth_dmac(struct ib_device *device,
57	59	struct rdma_ah_attr *ah_attr);
..	..	@@ -141,6 +143,10 @@
141	143	case IB_RATE_100_GBPS: return 40;
142	144	case IB_RATE_200_GBPS: return 80;
143	145	case IB_RATE_300_GBPS: return 120;
	146	+ case IB_RATE_28_GBPS: return 11;
	147	+ case IB_RATE_50_GBPS: return 20;
	148	+ case IB_RATE_400_GBPS: return 160;
	149	+ case IB_RATE_600_GBPS: return 240;
144	150	default: return -1;
145	151	}
146	152	}
..	..	@@ -166,6 +172,10 @@
166	172	case 40: return IB_RATE_100_GBPS;
167	173	case 80: return IB_RATE_200_GBPS;
168	174	case 120: return IB_RATE_300_GBPS;
	175	+ case 11: return IB_RATE_28_GBPS;
	176	+ case 20: return IB_RATE_50_GBPS;
	177	+ case 160: return IB_RATE_400_GBPS;
	178	+ case 240: return IB_RATE_600_GBPS;
169	179	default: return IB_RATE_PORT_CURRENT;
170	180	}
171	181	}
..	..	@@ -191,13 +201,17 @@
191	201	case IB_RATE_100_GBPS: return 103125;
192	202	case IB_RATE_200_GBPS: return 206250;
193	203	case IB_RATE_300_GBPS: return 309375;
	204	+ case IB_RATE_28_GBPS: return 28125;
	205	+ case IB_RATE_50_GBPS: return 53125;
	206	+ case IB_RATE_400_GBPS: return 425000;
	207	+ case IB_RATE_600_GBPS: return 637500;
194	208	default: return -1;
195	209	}
196	210	}
197	211	EXPORT_SYMBOL(ib_rate_to_mbps);
198	212
199	213	__attribute_const__ enum rdma_transport_type
200		-rdma_node_get_transport(enum rdma_node_type node_type)
	214	+rdma_node_get_transport(unsigned int node_type)
201	215	{
202	216
203	217	if (node_type == RDMA_NODE_USNIC)
..	..	@@ -206,6 +220,8 @@
206	220	return RDMA_TRANSPORT_USNIC_UDP;
207	221	if (node_type == RDMA_NODE_RNIC)
208	222	return RDMA_TRANSPORT_IWARP;
	223	+ if (node_type == RDMA_NODE_UNSPECIFIED)
	224	+ return RDMA_TRANSPORT_UNSPECIFIED;
209	225
210	226	return RDMA_TRANSPORT_IB;
211	227	}
..	..	@@ -214,8 +230,8 @@
214	230	enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)
215	231	{
216	232	enum rdma_transport_type lt;
217		- if (device->get_link_layer)
218		- return device->get_link_layer(device, port_num);
	233	+ if (device->ops.get_link_layer)
	234	+ return device->ops.get_link_layer(device, port_num);
219	235
220	236	lt = rdma_node_get_transport(device->node_type);
221	237	if (lt == RDMA_TRANSPORT_IB)
..	..	@@ -230,6 +246,8 @@
230	246	/**
231	247	* ib_alloc_pd - Allocates an unused protection domain.
232	248	* @device: The device on which to allocate the protection domain.
	249	+ * @flags: protection domain flags
	250	+ * @caller: caller's build-time module name
233	251	*
234	252	* A protection domain object provides an association between QPs, shared
235	253	* receive queues, address handles, memory regions, and memory windows.
..	..	@@ -242,16 +260,28 @@
242	260	{
243	261	struct ib_pd *pd;
244	262	int mr_access_flags = 0;
	263	+ int ret;
245	264
246		- pd = device->alloc_pd(device, NULL, NULL);
247		- if (IS_ERR(pd))
248		- return pd;
	265	+ pd = rdma_zalloc_drv_obj(device, ib_pd);
	266	+ if (!pd)
	267	+ return ERR_PTR(-ENOMEM);
249	268
250	269	pd->device = device;
251	270	pd->uobject = NULL;
252	271	pd->__internal_mr = NULL;
253	272	atomic_set(&pd->usecnt, 0);
254	273	pd->flags = flags;
	274	+
	275	+ rdma_restrack_new(&pd->res, RDMA_RESTRACK_PD);
	276	+ rdma_restrack_set_name(&pd->res, caller);
	277	+
	278	+ ret = device->ops.alloc_pd(pd, NULL);
	279	+ if (ret) {
	280	+ rdma_restrack_put(&pd->res);
	281	+ kfree(pd);
	282	+ return ERR_PTR(ret);
	283	+ }
	284	+ rdma_restrack_add(&pd->res);
255	285
256	286	if (device->attrs.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY)
257	287	pd->local_dma_lkey = device->local_dma_lkey;
..	..	@@ -263,14 +293,10 @@
263	293	mr_access_flags \|= IB_ACCESS_REMOTE_READ \| IB_ACCESS_REMOTE_WRITE;
264	294	}
265	295
266		- pd->res.type = RDMA_RESTRACK_PD;
267		- pd->res.kern_name = caller;
268		- rdma_restrack_add(&pd->res);
269		-
270	296	if (mr_access_flags) {
271	297	struct ib_mr *mr;
272	298
273		- mr = pd->device->get_dma_mr(pd, mr_access_flags);
	299	+ mr = pd->device->ops.get_dma_mr(pd, mr_access_flags);
274	300	if (IS_ERR(mr)) {
275	301	ib_dealloc_pd(pd);
276	302	return ERR_CAST(mr);
..	..	@@ -278,6 +304,7 @@
278	304
279	305	mr->device = pd->device;
280	306	mr->pd = pd;
	307	+ mr->type = IB_MR_TYPE_DMA;
281	308	mr->uobject = NULL;
282	309	mr->need_inval = false;
283	310
..	..	@@ -295,19 +322,20 @@
295	322	EXPORT_SYMBOL(__ib_alloc_pd);
296	323
297	324	/**
298		- * ib_dealloc_pd - Deallocates a protection domain.
	325	+ * ib_dealloc_pd_user - Deallocates a protection domain.
299	326	* @pd: The protection domain to deallocate.
	327	+ * @udata: Valid user data or NULL for kernel object
300	328	*
301	329	* It is an error to call this function while any resources in the pd still
302	330	* exist. The caller is responsible to synchronously destroy them and
303	331	* guarantee no new allocations will happen.
304	332	*/
305		-void ib_dealloc_pd(struct ib_pd *pd)
	333	+int ib_dealloc_pd_user(struct ib_pd pd, struct ib_udata udata)
306	334	{
307	335	int ret;
308	336
309	337	if (pd->__internal_mr) {
310		- ret = pd->device->dereg_mr(pd->__internal_mr);
	338	+ ret = pd->device->ops.dereg_mr(pd->__internal_mr, NULL);
311	339	WARN_ON(ret);
312	340	pd->__internal_mr = NULL;
313	341	}
..	..	@@ -316,13 +344,15 @@
316	344	requires the caller to guarantee we can't race here. */
317	345	WARN_ON(atomic_read(&pd->usecnt));
318	346
	347	+ ret = pd->device->ops.dealloc_pd(pd, udata);
	348	+ if (ret)
	349	+ return ret;
	350	+
319	351	rdma_restrack_del(&pd->res);
320		- /* Making delalloc_pd a void return is a WIP, no driver should return
321		- an error here. */
322		- ret = pd->device->dealloc_pd(pd);
323		- WARN_ONCE(ret, "Infiniband HW driver failed dealloc_pd");
	352	+ kfree(pd);
	353	+ return ret;
324	354	}
325		-EXPORT_SYMBOL(ib_dealloc_pd);
	355	+EXPORT_SYMBOL(ib_dealloc_pd_user);
326	356
327	357	/* Address handles */
328	358
..	..	@@ -475,25 +505,41 @@
475	505
476	506	static struct ib_ah _rdma_create_ah(struct ib_pd pd,
477	507	struct rdma_ah_attr *ah_attr,
478		- struct ib_udata *udata)
	508	+ u32 flags,
	509	+ struct ib_udata *udata,
	510	+ struct net_device *xmit_slave)
479	511	{
	512	+ struct rdma_ah_init_attr init_attr = {};
	513	+ struct ib_device *device = pd->device;
480	514	struct ib_ah *ah;
	515	+ int ret;
481	516
482		- if (!pd->device->create_ah)
	517	+ might_sleep_if(flags & RDMA_CREATE_AH_SLEEPABLE);
	518	+
	519	+ if (!device->ops.create_ah)
483	520	return ERR_PTR(-EOPNOTSUPP);
484	521
485		- ah = pd->device->create_ah(pd, ah_attr, udata);
	522	+ ah = rdma_zalloc_drv_obj_gfp(
	523	+ device, ib_ah,
	524	+ (flags & RDMA_CREATE_AH_SLEEPABLE) ? GFP_KERNEL : GFP_ATOMIC);
	525	+ if (!ah)
	526	+ return ERR_PTR(-ENOMEM);
486	527
487		- if (!IS_ERR(ah)) {
488		- ah->device = pd->device;
489		- ah->pd = pd;
490		- ah->uobject = NULL;
491		- ah->type = ah_attr->type;
492		- ah->sgid_attr = rdma_update_sgid_attr(ah_attr, NULL);
	528	+ ah->device = device;
	529	+ ah->pd = pd;
	530	+ ah->type = ah_attr->type;
	531	+ ah->sgid_attr = rdma_update_sgid_attr(ah_attr, NULL);
	532	+ init_attr.ah_attr = ah_attr;
	533	+ init_attr.flags = flags;
	534	+ init_attr.xmit_slave = xmit_slave;
493	535
494		- atomic_inc(&pd->usecnt);
	536	+ ret = device->ops.create_ah(ah, &init_attr, udata);
	537	+ if (ret) {
	538	+ kfree(ah);
	539	+ return ERR_PTR(ret);
495	540	}
496	541
	542	+ atomic_inc(&pd->usecnt);
497	543	return ah;
498	544	}
499	545
..	..	@@ -502,23 +548,32 @@
502	548	* given address vector.
503	549	* @pd: The protection domain associated with the address handle.
504	550	* @ah_attr: The attributes of the address vector.
	551	+ * @flags: Create address handle flags (see enum rdma_create_ah_flags).
505	552	*
506	553	* It returns 0 on success and returns appropriate error code on error.
507	554	* The address handle is used to reference a local or global destination
508	555	* in all UD QP post sends.
509	556	*/
510		-struct ib_ah rdma_create_ah(struct ib_pd pd, struct rdma_ah_attr *ah_attr)
	557	+struct ib_ah rdma_create_ah(struct ib_pd pd, struct rdma_ah_attr *ah_attr,
	558	+ u32 flags)
511	559	{
512	560	const struct ib_gid_attr *old_sgid_attr;
	561	+ struct net_device *slave;
513	562	struct ib_ah *ah;
514	563	int ret;
515	564
516	565	ret = rdma_fill_sgid_attr(pd->device, ah_attr, &old_sgid_attr);
517	566	if (ret)
518	567	return ERR_PTR(ret);
519		-
520		- ah = _rdma_create_ah(pd, ah_attr, NULL);
521		-
	568	+ slave = rdma_lag_get_ah_roce_slave(pd->device, ah_attr,
	569	+ (flags & RDMA_CREATE_AH_SLEEPABLE) ?
	570	+ GFP_KERNEL : GFP_ATOMIC);
	571	+ if (IS_ERR(slave)) {
	572	+ rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
	573	+ return (void *)slave;
	574	+ }
	575	+ ah = _rdma_create_ah(pd, ah_attr, flags, NULL, slave);
	576	+ rdma_lag_put_ah_roce_slave(slave);
522	577	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
523	578	return ah;
524	579	}
..	..	@@ -557,7 +612,8 @@
557	612	}
558	613	}
559	614
560		- ah = _rdma_create_ah(pd, ah_attr, udata);
	615	+ ah = _rdma_create_ah(pd, ah_attr, RDMA_CREATE_AH_SLEEPABLE,
	616	+ udata, NULL);
561	617
562	618	out:
563	619	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
..	..	@@ -628,16 +684,17 @@
628	684	void *context)
629	685	{
630	686	struct find_gid_index_context *ctx = context;
	687	+ u16 vlan_id = 0xffff;
	688	+ int ret;
631	689
632	690	if (ctx->gid_type != gid_attr->gid_type)
633	691	return false;
634	692
635		- if ((!!(ctx->vlan_id != 0xffff) == !is_vlan_dev(gid_attr->ndev)) \|\|
636		- (is_vlan_dev(gid_attr->ndev) &&
637		- vlan_dev_vlan_id(gid_attr->ndev) != ctx->vlan_id))
	693	+ ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
	694	+ if (ret)
638	695	return false;
639	696
640		- return true;
	697	+ return ctx->vlan_id == vlan_id;
641	698	}
642	699
643	700	static const struct ib_gid_attr *
..	..	@@ -676,7 +733,7 @@
676	733	(struct in6_addr *)dgid);
677	734	return 0;
678	735	} else if (net_type == RDMA_NETWORK_IPV6 \|\|
679		- net_type == RDMA_NETWORK_IB) {
	736	+ net_type == RDMA_NETWORK_IB \|\| RDMA_NETWORK_ROCE_V1) {
680	737	*dgid = hdr->ibgrh.dgid;
681	738	*sgid = hdr->ibgrh.sgid;
682	739	return 0;
..	..	@@ -710,7 +767,7 @@
710	767
711	768	ret = rdma_addr_find_l2_eth_by_grh(&sgid_attr->gid, &grh->dgid,
712	769	ah_attr->roce.dmac,
713		- sgid_attr->ndev, &hop_limit);
	770	+ sgid_attr, &hop_limit);
714	771
715	772	grh->hop_limit = hop_limit;
716	773	return ret;
..	..	@@ -869,7 +926,7 @@
869	926	if (ret)
870	927	return ERR_PTR(ret);
871	928
872		- ah = rdma_create_ah(pd, &ah_attr);
	929	+ ah = rdma_create_ah(pd, &ah_attr, RDMA_CREATE_AH_SLEEPABLE);
873	930
874	931	rdma_destroy_ah_attr(&ah_attr);
875	932	return ah;
..	..	@@ -888,8 +945,8 @@
888	945	if (ret)
889	946	return ret;
890	947
891		- ret = ah->device->modify_ah ?
892		- ah->device->modify_ah(ah, ah_attr) :
	948	+ ret = ah->device->ops.modify_ah ?
	949	+ ah->device->ops.modify_ah(ah, ah_attr) :
893	950	-EOPNOTSUPP;
894	951
895	952	ah->sgid_attr = rdma_update_sgid_attr(ah_attr, ah->sgid_attr);
..	..	@@ -902,113 +959,135 @@
902	959	{
903	960	ah_attr->grh.sgid_attr = NULL;
904	961
905		- return ah->device->query_ah ?
906		- ah->device->query_ah(ah, ah_attr) :
	962	+ return ah->device->ops.query_ah ?
	963	+ ah->device->ops.query_ah(ah, ah_attr) :
907	964	-EOPNOTSUPP;
908	965	}
909	966	EXPORT_SYMBOL(rdma_query_ah);
910	967
911		-int rdma_destroy_ah(struct ib_ah *ah)
	968	+int rdma_destroy_ah_user(struct ib_ah ah, u32 flags, struct ib_udata udata)
912	969	{
913	970	const struct ib_gid_attr *sgid_attr = ah->sgid_attr;
914	971	struct ib_pd *pd;
915	972	int ret;
916	973
917		- pd = ah->pd;
918		- ret = ah->device->destroy_ah(ah);
919		- if (!ret) {
920		- atomic_dec(&pd->usecnt);
921		- if (sgid_attr)
922		- rdma_put_gid_attr(sgid_attr);
923		- }
	974	+ might_sleep_if(flags & RDMA_DESTROY_AH_SLEEPABLE);
924	975
	976	+ pd = ah->pd;
	977	+
	978	+ ret = ah->device->ops.destroy_ah(ah, flags);
	979	+ if (ret)
	980	+ return ret;
	981	+
	982	+ atomic_dec(&pd->usecnt);
	983	+ if (sgid_attr)
	984	+ rdma_put_gid_attr(sgid_attr);
	985	+
	986	+ kfree(ah);
925	987	return ret;
926	988	}
927		-EXPORT_SYMBOL(rdma_destroy_ah);
	989	+EXPORT_SYMBOL(rdma_destroy_ah_user);
928	990
929	991	/* Shared receive queues */
930	992
931		-struct ib_srq ib_create_srq(struct ib_pd pd,
932		- struct ib_srq_init_attr *srq_init_attr)
	993	+/**
	994	+ * ib_create_srq_user - Creates a SRQ associated with the specified protection
	995	+ * domain.
	996	+ * @pd: The protection domain associated with the SRQ.
	997	+ * @srq_init_attr: A list of initial attributes required to create the
	998	+ * SRQ. If SRQ creation succeeds, then the attributes are updated to
	999	+ * the actual capabilities of the created SRQ.
	1000	+ * @uobject: uobject pointer if this is not a kernel SRQ
	1001	+ * @udata: udata pointer if this is not a kernel SRQ
	1002	+ *
	1003	+ * srq_attr->max_wr and srq_attr->max_sge are read the determine the
	1004	+ * requested size of the SRQ, and set to the actual values allocated
	1005	+ * on return. If ib_create_srq() succeeds, then max_wr and max_sge
	1006	+ * will always be at least as large as the requested values.
	1007	+ */
	1008	+struct ib_srq ib_create_srq_user(struct ib_pd pd,
	1009	+ struct ib_srq_init_attr *srq_init_attr,
	1010	+ struct ib_usrq_object *uobject,
	1011	+ struct ib_udata *udata)
933	1012	{
934	1013	struct ib_srq *srq;
	1014	+ int ret;
935	1015
936		- if (!pd->device->create_srq)
937		- return ERR_PTR(-EOPNOTSUPP);
	1016	+ srq = rdma_zalloc_drv_obj(pd->device, ib_srq);
	1017	+ if (!srq)
	1018	+ return ERR_PTR(-ENOMEM);
938	1019
939		- srq = pd->device->create_srq(pd, srq_init_attr, NULL);
	1020	+ srq->device = pd->device;
	1021	+ srq->pd = pd;
	1022	+ srq->event_handler = srq_init_attr->event_handler;
	1023	+ srq->srq_context = srq_init_attr->srq_context;
	1024	+ srq->srq_type = srq_init_attr->srq_type;
	1025	+ srq->uobject = uobject;
940	1026
941		- if (!IS_ERR(srq)) {
942		- srq->device = pd->device;
943		- srq->pd = pd;
944		- srq->uobject = NULL;
945		- srq->event_handler = srq_init_attr->event_handler;
946		- srq->srq_context = srq_init_attr->srq_context;
947		- srq->srq_type = srq_init_attr->srq_type;
948		- if (ib_srq_has_cq(srq->srq_type)) {
949		- srq->ext.cq = srq_init_attr->ext.cq;
950		- atomic_inc(&srq->ext.cq->usecnt);
951		- }
952		- if (srq->srq_type == IB_SRQT_XRC) {
953		- srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
954		- atomic_inc(&srq->ext.xrc.xrcd->usecnt);
955		- }
956		- atomic_inc(&pd->usecnt);
957		- atomic_set(&srq->usecnt, 0);
	1027	+ if (ib_srq_has_cq(srq->srq_type)) {
	1028	+ srq->ext.cq = srq_init_attr->ext.cq;
	1029	+ atomic_inc(&srq->ext.cq->usecnt);
	1030	+ }
	1031	+ if (srq->srq_type == IB_SRQT_XRC) {
	1032	+ srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
	1033	+ atomic_inc(&srq->ext.xrc.xrcd->usecnt);
	1034	+ }
	1035	+ atomic_inc(&pd->usecnt);
	1036	+
	1037	+ ret = pd->device->ops.create_srq(srq, srq_init_attr, udata);
	1038	+ if (ret) {
	1039	+ atomic_dec(&srq->pd->usecnt);
	1040	+ if (srq->srq_type == IB_SRQT_XRC)
	1041	+ atomic_dec(&srq->ext.xrc.xrcd->usecnt);
	1042	+ if (ib_srq_has_cq(srq->srq_type))
	1043	+ atomic_dec(&srq->ext.cq->usecnt);
	1044	+ kfree(srq);
	1045	+ return ERR_PTR(ret);
958	1046	}
959	1047
960	1048	return srq;
961	1049	}
962		-EXPORT_SYMBOL(ib_create_srq);
	1050	+EXPORT_SYMBOL(ib_create_srq_user);
963	1051
964	1052	int ib_modify_srq(struct ib_srq *srq,
965	1053	struct ib_srq_attr *srq_attr,
966	1054	enum ib_srq_attr_mask srq_attr_mask)
967	1055	{
968		- return srq->device->modify_srq ?
969		- srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) :
970		- -EOPNOTSUPP;
	1056	+ return srq->device->ops.modify_srq ?
	1057	+ srq->device->ops.modify_srq(srq, srq_attr, srq_attr_mask,
	1058	+ NULL) : -EOPNOTSUPP;
971	1059	}
972	1060	EXPORT_SYMBOL(ib_modify_srq);
973	1061
974	1062	int ib_query_srq(struct ib_srq *srq,
975	1063	struct ib_srq_attr *srq_attr)
976	1064	{
977		- return srq->device->query_srq ?
978		- srq->device->query_srq(srq, srq_attr) : -EOPNOTSUPP;
	1065	+ return srq->device->ops.query_srq ?
	1066	+ srq->device->ops.query_srq(srq, srq_attr) : -EOPNOTSUPP;
979	1067	}
980	1068	EXPORT_SYMBOL(ib_query_srq);
981	1069
982		-int ib_destroy_srq(struct ib_srq *srq)
	1070	+int ib_destroy_srq_user(struct ib_srq srq, struct ib_udata udata)
983	1071	{
984		- struct ib_pd *pd;
985		- enum ib_srq_type srq_type;
986		- struct ib_xrcd *uninitialized_var(xrcd);
987		- struct ib_cq *uninitialized_var(cq);
988	1072	int ret;
989	1073
990	1074	if (atomic_read(&srq->usecnt))
991	1075	return -EBUSY;
992	1076
993		- pd = srq->pd;
994		- srq_type = srq->srq_type;
995		- if (ib_srq_has_cq(srq_type))
996		- cq = srq->ext.cq;
997		- if (srq_type == IB_SRQT_XRC)
998		- xrcd = srq->ext.xrc.xrcd;
	1077	+ ret = srq->device->ops.destroy_srq(srq, udata);
	1078	+ if (ret)
	1079	+ return ret;
999	1080
1000		- ret = srq->device->destroy_srq(srq);
1001		- if (!ret) {
1002		- atomic_dec(&pd->usecnt);
1003		- if (srq_type == IB_SRQT_XRC)
1004		- atomic_dec(&xrcd->usecnt);
1005		- if (ib_srq_has_cq(srq_type))
1006		- atomic_dec(&cq->usecnt);
1007		- }
	1081	+ atomic_dec(&srq->pd->usecnt);
	1082	+ if (srq->srq_type == IB_SRQT_XRC)
	1083	+ atomic_dec(&srq->ext.xrc.xrcd->usecnt);
	1084	+ if (ib_srq_has_cq(srq->srq_type))
	1085	+ atomic_dec(&srq->ext.cq->usecnt);
	1086	+ kfree(srq);
1008	1087
1009	1088	return ret;
1010	1089	}
1011		-EXPORT_SYMBOL(ib_destroy_srq);
	1090	+EXPORT_SYMBOL(ib_destroy_srq_user);
1012	1091
1013	1092	/* Queue pairs */
1014	1093
..	..	@@ -1017,18 +1096,11 @@
1017	1096	struct ib_qp *qp = context;
1018	1097	unsigned long flags;
1019	1098
1020		- spin_lock_irqsave(&qp->device->event_handler_lock, flags);
	1099	+ spin_lock_irqsave(&qp->device->qp_open_list_lock, flags);
1021	1100	list_for_each_entry(event->element.qp, &qp->open_list, open_list)
1022	1101	if (event->element.qp->event_handler)
1023	1102	event->element.qp->event_handler(event, event->element.qp->qp_context);
1024		- spin_unlock_irqrestore(&qp->device->event_handler_lock, flags);
1025		-}
1026		-
1027		-static void __ib_insert_xrcd_qp(struct ib_xrcd xrcd, struct ib_qp qp)
1028		-{
1029		- mutex_lock(&xrcd->tgt_qp_mutex);
1030		- list_add(&qp->xrcd_list, &xrcd->tgt_qp_list);
1031		- mutex_unlock(&xrcd->tgt_qp_mutex);
	1103	+ spin_unlock_irqrestore(&qp->device->qp_open_list_lock, flags);
1032	1104	}
1033	1105
1034	1106	static struct ib_qp __ib_open_qp(struct ib_qp real_qp,
..	..	@@ -1058,9 +1130,9 @@
1058	1130	qp->qp_num = real_qp->qp_num;
1059	1131	qp->qp_type = real_qp->qp_type;
1060	1132
1061		- spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
	1133	+ spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags);
1062	1134	list_add(&qp->open_list, &real_qp->open_list);
1063		- spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
	1135	+ spin_unlock_irqrestore(&real_qp->device->qp_open_list_lock, flags);
1064	1136
1065	1137	return qp;
1066	1138	}
..	..	@@ -1073,24 +1145,24 @@
1073	1145	if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
1074	1146	return ERR_PTR(-EINVAL);
1075	1147
1076		- qp = ERR_PTR(-EINVAL);
1077		- mutex_lock(&xrcd->tgt_qp_mutex);
1078		- list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) {
1079		- if (real_qp->qp_num == qp_open_attr->qp_num) {
1080		- qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
1081		- qp_open_attr->qp_context);
1082		- break;
1083		- }
	1148	+ down_read(&xrcd->tgt_qps_rwsem);
	1149	+ real_qp = xa_load(&xrcd->tgt_qps, qp_open_attr->qp_num);
	1150	+ if (!real_qp) {
	1151	+ up_read(&xrcd->tgt_qps_rwsem);
	1152	+ return ERR_PTR(-EINVAL);
1084	1153	}
1085		- mutex_unlock(&xrcd->tgt_qp_mutex);
	1154	+ qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
	1155	+ qp_open_attr->qp_context);
	1156	+ up_read(&xrcd->tgt_qps_rwsem);
1086	1157	return qp;
1087	1158	}
1088	1159	EXPORT_SYMBOL(ib_open_qp);
1089	1160
1090		-static struct ib_qp create_xrc_qp(struct ib_qp qp,
1091		- struct ib_qp_init_attr *qp_init_attr)
	1161	+static struct ib_qp create_xrc_qp_user(struct ib_qp qp,
	1162	+ struct ib_qp_init_attr *qp_init_attr)
1092	1163	{
1093	1164	struct ib_qp *real_qp = qp;
	1165	+ int err;
1094	1166
1095	1167	qp->event_handler = __ib_shared_qp_event_handler;
1096	1168	qp->qp_context = qp;
..	..	@@ -1106,10 +1178,25 @@
1106	1178	if (IS_ERR(qp))
1107	1179	return qp;
1108	1180
1109		- __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp);
	1181	+ err = xa_err(xa_store(&qp_init_attr->xrcd->tgt_qps, real_qp->qp_num,
	1182	+ real_qp, GFP_KERNEL));
	1183	+ if (err) {
	1184	+ ib_close_qp(qp);
	1185	+ return ERR_PTR(err);
	1186	+ }
1110	1187	return qp;
1111	1188	}
1112	1189
	1190	+/**
	1191	+ * ib_create_qp - Creates a kernel QP associated with the specified protection
	1192	+ * domain.
	1193	+ * @pd: The protection domain associated with the QP.
	1194	+ * @qp_init_attr: A list of initial attributes required to create the
	1195	+ * QP. If QP creation succeeds, then the attributes are updated to
	1196	+ * the actual capabilities of the created QP.
	1197	+ *
	1198	+ * NOTE: for user qp use ib_create_qp_user with valid udata!
	1199	+ */
1113	1200	struct ib_qp ib_create_qp(struct ib_pd pd,
1114	1201	struct ib_qp_init_attr *qp_init_attr)
1115	1202	{
..	..	@@ -1121,6 +1208,10 @@
1121	1208	(qp_init_attr->recv_cq \|\|
1122	1209	qp_init_attr->srq \|\| qp_init_attr->cap.max_recv_wr \|\|
1123	1210	qp_init_attr->cap.max_recv_sge))
	1211	+ return ERR_PTR(-EINVAL);
	1212	+
	1213	+ if ((qp_init_attr->create_flags & IB_QP_CREATE_INTEGRITY_EN) &&
	1214	+ !(device->attrs.device_cap_flags & IB_DEVICE_INTEGRITY_HANDOVER))
1124	1215	return ERR_PTR(-EINVAL);
1125	1216
1126	1217	/*
..	..	@@ -1140,19 +1231,9 @@
1140	1231	if (ret)
1141	1232	goto err;
1142	1233
1143		- qp->real_qp = qp;
1144		- qp->qp_type = qp_init_attr->qp_type;
1145		- qp->rwq_ind_tbl = qp_init_attr->rwq_ind_tbl;
1146		-
1147		- atomic_set(&qp->usecnt, 0);
1148		- qp->mrs_used = 0;
1149		- spin_lock_init(&qp->mr_lock);
1150		- INIT_LIST_HEAD(&qp->rdma_mrs);
1151		- INIT_LIST_HEAD(&qp->sig_mrs);
1152		- qp->port = 0;
1153		-
1154	1234	if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
1155		- struct ib_qp *xrc_qp = create_xrc_qp(qp, qp_init_attr);
	1235	+ struct ib_qp *xrc_qp =
	1236	+ create_xrc_qp_user(qp, qp_init_attr);
1156	1237
1157	1238	if (IS_ERR(xrc_qp)) {
1158	1239	ret = PTR_ERR(xrc_qp);
..	..	@@ -1198,6 +1279,8 @@
1198	1279	qp->max_write_sge = qp_init_attr->cap.max_send_sge;
1199	1280	qp->max_read_sge = min_t(u32, qp_init_attr->cap.max_send_sge,
1200	1281	device->attrs.max_sge_rd);
	1282	+ if (qp_init_attr->create_flags & IB_QP_CREATE_INTEGRITY_EN)
	1283	+ qp->integrity_en = true;
1201	1284
1202	1285	return qp;
1203	1286
..	..	@@ -1516,8 +1599,7 @@
1516	1599	};
1517	1600
1518	1601	bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
1519		- enum ib_qp_type type, enum ib_qp_attr_mask mask,
1520		- enum rdma_link_layer ll)
	1602	+ enum ib_qp_type type, enum ib_qp_attr_mask mask)
1521	1603	{
1522	1604	enum ib_qp_attr_mask req_param, opt_param;
1523	1605
..	..	@@ -1591,11 +1673,37 @@
1591	1673	const struct ib_gid_attr *old_sgid_attr_alt_av;
1592	1674	int ret;
1593	1675
	1676	+ attr->xmit_slave = NULL;
1594	1677	if (attr_mask & IB_QP_AV) {
1595	1678	ret = rdma_fill_sgid_attr(qp->device, &attr->ah_attr,
1596	1679	&old_sgid_attr_av);
1597	1680	if (ret)
1598	1681	return ret;
	1682	+
	1683	+ if (attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE &&
	1684	+ is_qp_type_connected(qp)) {
	1685	+ struct net_device *slave;
	1686	+
	1687	+ /*
	1688	+ * If the user provided the qp_attr then we have to
	1689	+ * resolve it. Kerne users have to provide already
	1690	+ * resolved rdma_ah_attr's.
	1691	+ */
	1692	+ if (udata) {
	1693	+ ret = ib_resolve_eth_dmac(qp->device,
	1694	+ &attr->ah_attr);
	1695	+ if (ret)
	1696	+ goto out_av;
	1697	+ }
	1698	+ slave = rdma_lag_get_ah_roce_slave(qp->device,
	1699	+ &attr->ah_attr,
	1700	+ GFP_KERNEL);
	1701	+ if (IS_ERR(slave)) {
	1702	+ ret = PTR_ERR(slave);
	1703	+ goto out_av;
	1704	+ }
	1705	+ attr->xmit_slave = slave;
	1706	+ }
1599	1707	}
1600	1708	if (attr_mask & IB_QP_ALT_PATH) {
1601	1709	/*
..	..	@@ -1622,31 +1730,29 @@
1622	1730	}
1623	1731	}
1624	1732
1625		- /*
1626		- * If the user provided the qp_attr then we have to resolve it. Kernel
1627		- * users have to provide already resolved rdma_ah_attr's
1628		- */
1629		- if (udata && (attr_mask & IB_QP_AV) &&
1630		- attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE &&
1631		- is_qp_type_connected(qp)) {
1632		- ret = ib_resolve_eth_dmac(qp->device, &attr->ah_attr);
1633		- if (ret)
1634		- goto out;
1635		- }
1636		-
1637	1733	if (rdma_ib_or_roce(qp->device, port)) {
1638	1734	if (attr_mask & IB_QP_RQ_PSN && attr->rq_psn & ~0xffffff) {
1639		- pr_warn("%s: %s rq_psn overflow, masking to 24 bits\n",
1640		- __func__, qp->device->name);
	1735	+ dev_warn(&qp->device->dev,
	1736	+ "%s rq_psn overflow, masking to 24 bits\n",
	1737	+ __func__);
1641	1738	attr->rq_psn &= 0xffffff;
1642	1739	}
1643	1740
1644	1741	if (attr_mask & IB_QP_SQ_PSN && attr->sq_psn & ~0xffffff) {
1645		- pr_warn("%s: %s sq_psn overflow, masking to 24 bits\n",
1646		- __func__, qp->device->name);
	1742	+ dev_warn(&qp->device->dev,
	1743	+ " %s sq_psn overflow, masking to 24 bits\n",
	1744	+ __func__);
1647	1745	attr->sq_psn &= 0xffffff;
1648	1746	}
1649	1747	}
	1748	+
	1749	+ /*
	1750	+ * Bind this qp to a counter automatically based on the rdma counter
	1751	+ * rules. This only set in RST2INIT with port specified
	1752	+ */
	1753	+ if (!qp->counter && (attr_mask & IB_QP_PORT) &&
	1754	+ ((attr_mask & IB_QP_STATE) && attr->qp_state == IB_QPS_INIT))
	1755	+ rdma_counter_bind_qp_auto(qp, attr->port_num);
1650	1756
1651	1757	ret = ib_security_modify_qp(qp, attr, attr_mask, udata);
1652	1758	if (ret)
..	..	@@ -1665,8 +1771,10 @@
1665	1771	if (attr_mask & IB_QP_ALT_PATH)
1666	1772	rdma_unfill_sgid_attr(&attr->alt_ah_attr, old_sgid_attr_alt_av);
1667	1773	out_av:
1668		- if (attr_mask & IB_QP_AV)
	1774	+ if (attr_mask & IB_QP_AV) {
	1775	+ rdma_lag_put_ah_roce_slave(attr->xmit_slave);
1669	1776	rdma_unfill_sgid_attr(&attr->ah_attr, old_sgid_attr_av);
	1777	+ }
1670	1778	return ret;
1671	1779	}
1672	1780
..	..	@@ -1688,7 +1796,7 @@
1688	1796	}
1689	1797	EXPORT_SYMBOL(ib_modify_qp_with_udata);
1690	1798
1691		-int ib_get_eth_speed(struct ib_device dev, u8 port_num, u8 speed, u8 *width)
	1799	+int ib_get_eth_speed(struct ib_device dev, u8 port_num, u16 speed, u8 *width)
1692	1800	{
1693	1801	int rc;
1694	1802	u32 netdev_speed;
..	..	@@ -1698,10 +1806,7 @@
1698	1806	if (rdma_port_get_link_layer(dev, port_num) != IB_LINK_LAYER_ETHERNET)
1699	1807	return -EINVAL;
1700	1808
1701		- if (!dev->get_netdev)
1702		- return -EOPNOTSUPP;
1703		-
1704		- netdev = dev->get_netdev(dev, port_num);
	1809	+ netdev = ib_device_get_netdev(dev, port_num);
1705	1810	if (!netdev)
1706	1811	return -ENODEV;
1707	1812
..	..	@@ -1759,9 +1864,9 @@
1759	1864	qp_attr->ah_attr.grh.sgid_attr = NULL;
1760	1865	qp_attr->alt_ah_attr.grh.sgid_attr = NULL;
1761	1866
1762		- return qp->device->query_qp ?
1763		- qp->device->query_qp(qp->real_qp, qp_attr, qp_attr_mask, qp_init_attr) :
1764		- -EOPNOTSUPP;
	1867	+ return qp->device->ops.query_qp ?
	1868	+ qp->device->ops.query_qp(qp->real_qp, qp_attr, qp_attr_mask,
	1869	+ qp_init_attr) : -EOPNOTSUPP;
1765	1870	}
1766	1871	EXPORT_SYMBOL(ib_query_qp);
1767	1872
..	..	@@ -1774,9 +1879,9 @@
1774	1879	if (real_qp == qp)
1775	1880	return -EINVAL;
1776	1881
1777		- spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
	1882	+ spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags);
1778	1883	list_del(&qp->open_list);
1779		- spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
	1884	+ spin_unlock_irqrestore(&real_qp->device->qp_open_list_lock, flags);
1780	1885
1781	1886	atomic_dec(&real_qp->usecnt);
1782	1887	if (qp->qp_sec)
..	..	@@ -1795,27 +1900,24 @@
1795	1900
1796	1901	real_qp = qp->real_qp;
1797	1902	xrcd = real_qp->xrcd;
1798		-
1799		- mutex_lock(&xrcd->tgt_qp_mutex);
	1903	+ down_write(&xrcd->tgt_qps_rwsem);
1800	1904	ib_close_qp(qp);
1801	1905	if (atomic_read(&real_qp->usecnt) == 0)
1802		- list_del(&real_qp->xrcd_list);
	1906	+ xa_erase(&xrcd->tgt_qps, real_qp->qp_num);
1803	1907	else
1804	1908	real_qp = NULL;
1805		- mutex_unlock(&xrcd->tgt_qp_mutex);
	1909	+ up_write(&xrcd->tgt_qps_rwsem);
1806	1910
1807	1911	if (real_qp) {
1808	1912	ret = ib_destroy_qp(real_qp);
1809	1913	if (!ret)
1810	1914	atomic_dec(&xrcd->usecnt);
1811		- else
1812		- __ib_insert_xrcd_qp(xrcd, real_qp);
1813	1915	}
1814	1916
1815	1917	return 0;
1816	1918	}
1817	1919
1818		-int ib_destroy_qp(struct ib_qp *qp)
	1920	+int ib_destroy_qp_user(struct ib_qp qp, struct ib_udata udata)
1819	1921	{
1820	1922	const struct ib_gid_attr *alt_path_sgid_attr = qp->alt_path_sgid_attr;
1821	1923	const struct ib_gid_attr *av_sgid_attr = qp->av_sgid_attr;
..	..	@@ -1846,8 +1948,9 @@
1846	1948	if (!qp->uobject)
1847	1949	rdma_rw_cleanup_mrs(qp);
1848	1950
	1951	+ rdma_counter_unbind_qp(qp, true);
1849	1952	rdma_restrack_del(&qp->res);
1850		- ret = qp->device->destroy_qp(qp);
	1953	+ ret = qp->device->ops.destroy_qp(qp, udata);
1851	1954	if (!ret) {
1852	1955	if (alt_path_sgid_attr)
1853	1956	rdma_put_gid_attr(alt_path_sgid_attr);
..	..	@@ -1872,7 +1975,7 @@
1872	1975
1873	1976	return ret;
1874	1977	}
1875		-EXPORT_SYMBOL(ib_destroy_qp);
	1978	+EXPORT_SYMBOL(ib_destroy_qp_user);
1876	1979
1877	1980	/* Completion queues */
1878	1981
..	..	@@ -1884,68 +1987,146 @@
1884	1987	const char *caller)
1885	1988	{
1886	1989	struct ib_cq *cq;
	1990	+ int ret;
1887	1991
1888		- cq = device->create_cq(device, cq_attr, NULL, NULL);
	1992	+ cq = rdma_zalloc_drv_obj(device, ib_cq);
	1993	+ if (!cq)
	1994	+ return ERR_PTR(-ENOMEM);
1889	1995
1890		- if (!IS_ERR(cq)) {
1891		- cq->device = device;
1892		- cq->uobject = NULL;
1893		- cq->comp_handler = comp_handler;
1894		- cq->event_handler = event_handler;
1895		- cq->cq_context = cq_context;
1896		- atomic_set(&cq->usecnt, 0);
1897		- cq->res.type = RDMA_RESTRACK_CQ;
1898		- cq->res.kern_name = caller;
1899		- rdma_restrack_add(&cq->res);
	1996	+ cq->device = device;
	1997	+ cq->uobject = NULL;
	1998	+ cq->comp_handler = comp_handler;
	1999	+ cq->event_handler = event_handler;
	2000	+ cq->cq_context = cq_context;
	2001	+ atomic_set(&cq->usecnt, 0);
	2002	+
	2003	+ rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
	2004	+ rdma_restrack_set_name(&cq->res, caller);
	2005	+
	2006	+ ret = device->ops.create_cq(cq, cq_attr, NULL);
	2007	+ if (ret) {
	2008	+ rdma_restrack_put(&cq->res);
	2009	+ kfree(cq);
	2010	+ return ERR_PTR(ret);
1900	2011	}
1901	2012
	2013	+ rdma_restrack_add(&cq->res);
1902	2014	return cq;
1903	2015	}
1904	2016	EXPORT_SYMBOL(__ib_create_cq);
1905	2017
1906	2018	int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period)
1907	2019	{
1908		- return cq->device->modify_cq ?
1909		- cq->device->modify_cq(cq, cq_count, cq_period) : -EOPNOTSUPP;
	2020	+ if (cq->shared)
	2021	+ return -EOPNOTSUPP;
	2022	+
	2023	+ return cq->device->ops.modify_cq ?
	2024	+ cq->device->ops.modify_cq(cq, cq_count,
	2025	+ cq_period) : -EOPNOTSUPP;
1910	2026	}
1911	2027	EXPORT_SYMBOL(rdma_set_cq_moderation);
1912	2028
1913		-int ib_destroy_cq(struct ib_cq *cq)
	2029	+int ib_destroy_cq_user(struct ib_cq cq, struct ib_udata udata)
1914	2030	{
	2031	+ int ret;
	2032	+
	2033	+ if (WARN_ON_ONCE(cq->shared))
	2034	+ return -EOPNOTSUPP;
	2035	+
1915	2036	if (atomic_read(&cq->usecnt))
1916	2037	return -EBUSY;
1917	2038
	2039	+ ret = cq->device->ops.destroy_cq(cq, udata);
	2040	+ if (ret)
	2041	+ return ret;
	2042	+
1918	2043	rdma_restrack_del(&cq->res);
1919		- return cq->device->destroy_cq(cq);
	2044	+ kfree(cq);
	2045	+ return ret;
1920	2046	}
1921		-EXPORT_SYMBOL(ib_destroy_cq);
	2047	+EXPORT_SYMBOL(ib_destroy_cq_user);
1922	2048
1923	2049	int ib_resize_cq(struct ib_cq *cq, int cqe)
1924	2050	{
1925		- return cq->device->resize_cq ?
1926		- cq->device->resize_cq(cq, cqe, NULL) : -EOPNOTSUPP;
	2051	+ if (cq->shared)
	2052	+ return -EOPNOTSUPP;
	2053	+
	2054	+ return cq->device->ops.resize_cq ?
	2055	+ cq->device->ops.resize_cq(cq, cqe, NULL) : -EOPNOTSUPP;
1927	2056	}
1928	2057	EXPORT_SYMBOL(ib_resize_cq);
1929	2058
1930	2059	/* Memory regions */
1931	2060
1932		-int ib_dereg_mr(struct ib_mr *mr)
	2061	+struct ib_mr ib_reg_user_mr(struct ib_pd pd, u64 start, u64 length,
	2062	+ u64 virt_addr, int access_flags)
	2063	+{
	2064	+ struct ib_mr *mr;
	2065	+
	2066	+ if (access_flags & IB_ACCESS_ON_DEMAND) {
	2067	+ if (!(pd->device->attrs.device_cap_flags &
	2068	+ IB_DEVICE_ON_DEMAND_PAGING)) {
	2069	+ pr_debug("ODP support not available\n");
	2070	+ return ERR_PTR(-EINVAL);
	2071	+ }
	2072	+ }
	2073	+
	2074	+ mr = pd->device->ops.reg_user_mr(pd, start, length, virt_addr,
	2075	+ access_flags, NULL);
	2076	+
	2077	+ if (IS_ERR(mr))
	2078	+ return mr;
	2079	+
	2080	+ mr->device = pd->device;
	2081	+ mr->type = IB_MR_TYPE_USER;
	2082	+ mr->pd = pd;
	2083	+ mr->dm = NULL;
	2084	+ atomic_inc(&pd->usecnt);
	2085	+ mr->iova = virt_addr;
	2086	+ mr->length = length;
	2087	+
	2088	+ rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
	2089	+ rdma_restrack_parent_name(&mr->res, &pd->res);
	2090	+ rdma_restrack_add(&mr->res);
	2091	+
	2092	+ return mr;
	2093	+}
	2094	+EXPORT_SYMBOL(ib_reg_user_mr);
	2095	+
	2096	+int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,
	2097	+ u32 flags, struct ib_sge *sg_list, u32 num_sge)
	2098	+{
	2099	+ if (!pd->device->ops.advise_mr)
	2100	+ return -EOPNOTSUPP;
	2101	+
	2102	+ if (!num_sge)
	2103	+ return 0;
	2104	+
	2105	+ return pd->device->ops.advise_mr(pd, advice, flags, sg_list, num_sge,
	2106	+ NULL);
	2107	+}
	2108	+EXPORT_SYMBOL(ib_advise_mr);
	2109	+
	2110	+int ib_dereg_mr_user(struct ib_mr mr, struct ib_udata udata)
1933	2111	{
1934	2112	struct ib_pd *pd = mr->pd;
1935	2113	struct ib_dm *dm = mr->dm;
	2114	+ struct ib_sig_attrs *sig_attrs = mr->sig_attrs;
1936	2115	int ret;
1937	2116
	2117	+ trace_mr_dereg(mr);
1938	2118	rdma_restrack_del(&mr->res);
1939		- ret = mr->device->dereg_mr(mr);
	2119	+ ret = mr->device->ops.dereg_mr(mr, udata);
1940	2120	if (!ret) {
1941	2121	atomic_dec(&pd->usecnt);
1942	2122	if (dm)
1943	2123	atomic_dec(&dm->usecnt);
	2124	+ kfree(sig_attrs);
1944	2125	}
1945	2126
1946	2127	return ret;
1947	2128	}
1948		-EXPORT_SYMBOL(ib_dereg_mr);
	2129	+EXPORT_SYMBOL(ib_dereg_mr_user);
1949	2130
1950	2131	/**
1951	2132	* ib_alloc_mr() - Allocates a memory region
..	..	@@ -1959,78 +2140,104 @@
1959	2140	* max_num_sg * used_page_size.
1960	2141	*
1961	2142	*/
1962		-struct ib_mr ib_alloc_mr(struct ib_pd pd,
1963		- enum ib_mr_type mr_type,
	2143	+struct ib_mr ib_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
1964	2144	u32 max_num_sg)
1965	2145	{
1966	2146	struct ib_mr *mr;
1967	2147
1968		- if (!pd->device->alloc_mr)
1969		- return ERR_PTR(-EOPNOTSUPP);
1970		-
1971		- mr = pd->device->alloc_mr(pd, mr_type, max_num_sg);
1972		- if (!IS_ERR(mr)) {
1973		- mr->device = pd->device;
1974		- mr->pd = pd;
1975		- mr->dm = NULL;
1976		- mr->uobject = NULL;
1977		- atomic_inc(&pd->usecnt);
1978		- mr->need_inval = false;
1979		- mr->res.type = RDMA_RESTRACK_MR;
1980		- rdma_restrack_add(&mr->res);
	2148	+ if (!pd->device->ops.alloc_mr) {
	2149	+ mr = ERR_PTR(-EOPNOTSUPP);
	2150	+ goto out;
1981	2151	}
1982	2152
	2153	+ if (mr_type == IB_MR_TYPE_INTEGRITY) {
	2154	+ WARN_ON_ONCE(1);
	2155	+ mr = ERR_PTR(-EINVAL);
	2156	+ goto out;
	2157	+ }
	2158	+
	2159	+ mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg);
	2160	+ if (IS_ERR(mr))
	2161	+ goto out;
	2162	+
	2163	+ mr->device = pd->device;
	2164	+ mr->pd = pd;
	2165	+ mr->dm = NULL;
	2166	+ mr->uobject = NULL;
	2167	+ atomic_inc(&pd->usecnt);
	2168	+ mr->need_inval = false;
	2169	+ mr->type = mr_type;
	2170	+ mr->sig_attrs = NULL;
	2171	+
	2172	+ rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
	2173	+ rdma_restrack_parent_name(&mr->res, &pd->res);
	2174	+ rdma_restrack_add(&mr->res);
	2175	+out:
	2176	+ trace_mr_alloc(pd, mr_type, max_num_sg, mr);
1983	2177	return mr;
1984	2178	}
1985	2179	EXPORT_SYMBOL(ib_alloc_mr);
1986	2180
1987		-/* "Fast" memory regions */
1988		-
1989		-struct ib_fmr ib_alloc_fmr(struct ib_pd pd,
1990		- int mr_access_flags,
1991		- struct ib_fmr_attr *fmr_attr)
	2181	+/**
	2182	+ * ib_alloc_mr_integrity() - Allocates an integrity memory region
	2183	+ * @pd: protection domain associated with the region
	2184	+ * @max_num_data_sg: maximum data sg entries available for registration
	2185	+ * @max_num_meta_sg: maximum metadata sg entries available for
	2186	+ * registration
	2187	+ *
	2188	+ * Notes:
	2189	+ * Memory registration page/sg lists must not exceed max_num_sg,
	2190	+ * also the integrity page/sg lists must not exceed max_num_meta_sg.
	2191	+ *
	2192	+ */
	2193	+struct ib_mr ib_alloc_mr_integrity(struct ib_pd pd,
	2194	+ u32 max_num_data_sg,
	2195	+ u32 max_num_meta_sg)
1992	2196	{
1993		- struct ib_fmr *fmr;
	2197	+ struct ib_mr *mr;
	2198	+ struct ib_sig_attrs *sig_attrs;
1994	2199
1995		- if (!pd->device->alloc_fmr)
1996		- return ERR_PTR(-EOPNOTSUPP);
1997		-
1998		- fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
1999		- if (!IS_ERR(fmr)) {
2000		- fmr->device = pd->device;
2001		- fmr->pd = pd;
2002		- atomic_inc(&pd->usecnt);
	2200	+ if (!pd->device->ops.alloc_mr_integrity \|\|
	2201	+ !pd->device->ops.map_mr_sg_pi) {
	2202	+ mr = ERR_PTR(-EOPNOTSUPP);
	2203	+ goto out;
2003	2204	}
2004	2205
2005		- return fmr;
	2206	+ if (!max_num_meta_sg) {
	2207	+ mr = ERR_PTR(-EINVAL);
	2208	+ goto out;
	2209	+ }
	2210	+
	2211	+ sig_attrs = kzalloc(sizeof(struct ib_sig_attrs), GFP_KERNEL);
	2212	+ if (!sig_attrs) {
	2213	+ mr = ERR_PTR(-ENOMEM);
	2214	+ goto out;
	2215	+ }
	2216	+
	2217	+ mr = pd->device->ops.alloc_mr_integrity(pd, max_num_data_sg,
	2218	+ max_num_meta_sg);
	2219	+ if (IS_ERR(mr)) {
	2220	+ kfree(sig_attrs);
	2221	+ goto out;
	2222	+ }
	2223	+
	2224	+ mr->device = pd->device;
	2225	+ mr->pd = pd;
	2226	+ mr->dm = NULL;
	2227	+ mr->uobject = NULL;
	2228	+ atomic_inc(&pd->usecnt);
	2229	+ mr->need_inval = false;
	2230	+ mr->type = IB_MR_TYPE_INTEGRITY;
	2231	+ mr->sig_attrs = sig_attrs;
	2232	+
	2233	+ rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
	2234	+ rdma_restrack_parent_name(&mr->res, &pd->res);
	2235	+ rdma_restrack_add(&mr->res);
	2236	+out:
	2237	+ trace_mr_integ_alloc(pd, max_num_data_sg, max_num_meta_sg, mr);
	2238	+ return mr;
2006	2239	}
2007		-EXPORT_SYMBOL(ib_alloc_fmr);
2008		-
2009		-int ib_unmap_fmr(struct list_head *fmr_list)
2010		-{
2011		- struct ib_fmr *fmr;
2012		-
2013		- if (list_empty(fmr_list))
2014		- return 0;
2015		-
2016		- fmr = list_entry(fmr_list->next, struct ib_fmr, list);
2017		- return fmr->device->unmap_fmr(fmr_list);
2018		-}
2019		-EXPORT_SYMBOL(ib_unmap_fmr);
2020		-
2021		-int ib_dealloc_fmr(struct ib_fmr *fmr)
2022		-{
2023		- struct ib_pd *pd;
2024		- int ret;
2025		-
2026		- pd = fmr->pd;
2027		- ret = fmr->device->dealloc_fmr(fmr);
2028		- if (!ret)
2029		- atomic_dec(&pd->usecnt);
2030		-
2031		- return ret;
2032		-}
2033		-EXPORT_SYMBOL(ib_dealloc_fmr);
	2240	+EXPORT_SYMBOL(ib_alloc_mr_integrity);
2034	2241
2035	2242	/* Multicast groups */
2036	2243
..	..	@@ -2076,14 +2283,14 @@
2076	2283	{
2077	2284	int ret;
2078	2285
2079		- if (!qp->device->attach_mcast)
	2286	+ if (!qp->device->ops.attach_mcast)
2080	2287	return -EOPNOTSUPP;
2081	2288
2082	2289	if (!rdma_is_multicast_addr((struct in6_addr *)gid->raw) \|\|
2083	2290	qp->qp_type != IB_QPT_UD \|\| !is_valid_mcast_lid(qp, lid))
2084	2291	return -EINVAL;
2085	2292
2086		- ret = qp->device->attach_mcast(qp, gid, lid);
	2293	+ ret = qp->device->ops.attach_mcast(qp, gid, lid);
2087	2294	if (!ret)
2088	2295	atomic_inc(&qp->usecnt);
2089	2296	return ret;
..	..	@@ -2094,58 +2301,75 @@
2094	2301	{
2095	2302	int ret;
2096	2303
2097		- if (!qp->device->detach_mcast)
	2304	+ if (!qp->device->ops.detach_mcast)
2098	2305	return -EOPNOTSUPP;
2099	2306
2100	2307	if (!rdma_is_multicast_addr((struct in6_addr *)gid->raw) \|\|
2101	2308	qp->qp_type != IB_QPT_UD \|\| !is_valid_mcast_lid(qp, lid))
2102	2309	return -EINVAL;
2103	2310
2104		- ret = qp->device->detach_mcast(qp, gid, lid);
	2311	+ ret = qp->device->ops.detach_mcast(qp, gid, lid);
2105	2312	if (!ret)
2106	2313	atomic_dec(&qp->usecnt);
2107	2314	return ret;
2108	2315	}
2109	2316	EXPORT_SYMBOL(ib_detach_mcast);
2110	2317
2111		-struct ib_xrcd __ib_alloc_xrcd(struct ib_device device, const char *caller)
	2318	+/**
	2319	+ * ib_alloc_xrcd_user - Allocates an XRC domain.
	2320	+ * @device: The device on which to allocate the XRC domain.
	2321	+ * @inode: inode to connect XRCD
	2322	+ * @udata: Valid user data or NULL for kernel object
	2323	+ */
	2324	+struct ib_xrcd ib_alloc_xrcd_user(struct ib_device device,
	2325	+ struct inode inode, struct ib_udata udata)
2112	2326	{
2113	2327	struct ib_xrcd *xrcd;
	2328	+ int ret;
2114	2329
2115		- if (!device->alloc_xrcd)
	2330	+ if (!device->ops.alloc_xrcd)
2116	2331	return ERR_PTR(-EOPNOTSUPP);
2117	2332
2118		- xrcd = device->alloc_xrcd(device, NULL, NULL);
2119		- if (!IS_ERR(xrcd)) {
2120		- xrcd->device = device;
2121		- xrcd->inode = NULL;
2122		- atomic_set(&xrcd->usecnt, 0);
2123		- mutex_init(&xrcd->tgt_qp_mutex);
2124		- INIT_LIST_HEAD(&xrcd->tgt_qp_list);
2125		- }
	2333	+ xrcd = rdma_zalloc_drv_obj(device, ib_xrcd);
	2334	+ if (!xrcd)
	2335	+ return ERR_PTR(-ENOMEM);
2126	2336
	2337	+ xrcd->device = device;
	2338	+ xrcd->inode = inode;
	2339	+ atomic_set(&xrcd->usecnt, 0);
	2340	+ init_rwsem(&xrcd->tgt_qps_rwsem);
	2341	+ xa_init(&xrcd->tgt_qps);
	2342	+
	2343	+ ret = device->ops.alloc_xrcd(xrcd, udata);
	2344	+ if (ret)
	2345	+ goto err;
2127	2346	return xrcd;
	2347	+err:
	2348	+ kfree(xrcd);
	2349	+ return ERR_PTR(ret);
2128	2350	}
2129		-EXPORT_SYMBOL(__ib_alloc_xrcd);
	2351	+EXPORT_SYMBOL(ib_alloc_xrcd_user);
2130	2352
2131		-int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
	2353	+/**
	2354	+ * ib_dealloc_xrcd_user - Deallocates an XRC domain.
	2355	+ * @xrcd: The XRC domain to deallocate.
	2356	+ * @udata: Valid user data or NULL for kernel object
	2357	+ */
	2358	+int ib_dealloc_xrcd_user(struct ib_xrcd xrcd, struct ib_udata udata)
2132	2359	{
2133		- struct ib_qp *qp;
2134	2360	int ret;
2135	2361
2136	2362	if (atomic_read(&xrcd->usecnt))
2137	2363	return -EBUSY;
2138	2364
2139		- while (!list_empty(&xrcd->tgt_qp_list)) {
2140		- qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list);
2141		- ret = ib_destroy_qp(qp);
2142		- if (ret)
2143		- return ret;
2144		- }
2145		-
2146		- return xrcd->device->dealloc_xrcd(xrcd);
	2365	+ WARN_ON(!xa_empty(&xrcd->tgt_qps));
	2366	+ ret = xrcd->device->ops.dealloc_xrcd(xrcd, udata);
	2367	+ if (ret)
	2368	+ return ret;
	2369	+ kfree(xrcd);
	2370	+ return ret;
2147	2371	}
2148		-EXPORT_SYMBOL(ib_dealloc_xrcd);
	2372	+EXPORT_SYMBOL(ib_dealloc_xrcd_user);
2149	2373
2150	2374	/**
2151	2375	* ib_create_wq - Creates a WQ associated with the specified protection
..	..	@@ -2166,10 +2390,10 @@
2166	2390	{
2167	2391	struct ib_wq *wq;
2168	2392
2169		- if (!pd->device->create_wq)
	2393	+ if (!pd->device->ops.create_wq)
2170	2394	return ERR_PTR(-EOPNOTSUPP);
2171	2395
2172		- wq = pd->device->create_wq(pd, wq_attr, NULL);
	2396	+ wq = pd->device->ops.create_wq(pd, wq_attr, NULL);
2173	2397	if (!IS_ERR(wq)) {
2174	2398	wq->event_handler = wq_attr->event_handler;
2175	2399	wq->wq_context = wq_attr->wq_context;
..	..	@@ -2187,26 +2411,28 @@
2187	2411	EXPORT_SYMBOL(ib_create_wq);
2188	2412
2189	2413	/**
2190		- * ib_destroy_wq - Destroys the specified WQ.
	2414	+ * ib_destroy_wq_user - Destroys the specified user WQ.
2191	2415	* @wq: The WQ to destroy.
	2416	+ * @udata: Valid user data
2192	2417	*/
2193		-int ib_destroy_wq(struct ib_wq *wq)
	2418	+int ib_destroy_wq_user(struct ib_wq wq, struct ib_udata udata)
2194	2419	{
2195		- int err;
2196	2420	struct ib_cq *cq = wq->cq;
2197	2421	struct ib_pd *pd = wq->pd;
	2422	+ int ret;
2198	2423
2199	2424	if (atomic_read(&wq->usecnt))
2200	2425	return -EBUSY;
2201	2426
2202		- err = wq->device->destroy_wq(wq);
2203		- if (!err) {
2204		- atomic_dec(&pd->usecnt);
2205		- atomic_dec(&cq->usecnt);
2206		- }
2207		- return err;
	2427	+ ret = wq->device->ops.destroy_wq(wq, udata);
	2428	+ if (ret)
	2429	+ return ret;
	2430	+
	2431	+ atomic_dec(&pd->usecnt);
	2432	+ atomic_dec(&cq->usecnt);
	2433	+ return ret;
2208	2434	}
2209		-EXPORT_SYMBOL(ib_destroy_wq);
	2435	+EXPORT_SYMBOL(ib_destroy_wq_user);
2210	2436
2211	2437	/**
2212	2438	* ib_modify_wq - Modifies the specified WQ.
..	..	@@ -2221,123 +2447,110 @@
2221	2447	{
2222	2448	int err;
2223	2449
2224		- if (!wq->device->modify_wq)
	2450	+ if (!wq->device->ops.modify_wq)
2225	2451	return -EOPNOTSUPP;
2226	2452
2227		- err = wq->device->modify_wq(wq, wq_attr, wq_attr_mask, NULL);
	2453	+ err = wq->device->ops.modify_wq(wq, wq_attr, wq_attr_mask, NULL);
2228	2454	return err;
2229	2455	}
2230	2456	EXPORT_SYMBOL(ib_modify_wq);
2231	2457
2232		-/*
2233		- * ib_create_rwq_ind_table - Creates a RQ Indirection Table.
2234		- * @device: The device on which to create the rwq indirection table.
2235		- * @ib_rwq_ind_table_init_attr: A list of initial attributes required to
2236		- * create the Indirection Table.
2237		- *
2238		- * Note: The life time of ib_rwq_ind_table_init_attr->ind_tbl is not less
2239		- * than the created ib_rwq_ind_table object and the caller is responsible
2240		- * for its memory allocation/free.
2241		- */
2242		-struct ib_rwq_ind_table ib_create_rwq_ind_table(struct ib_device device,
2243		- struct ib_rwq_ind_table_init_attr *init_attr)
2244		-{
2245		- struct ib_rwq_ind_table *rwq_ind_table;
2246		- int i;
2247		- u32 table_size;
2248		-
2249		- if (!device->create_rwq_ind_table)
2250		- return ERR_PTR(-EOPNOTSUPP);
2251		-
2252		- table_size = (1 << init_attr->log_ind_tbl_size);
2253		- rwq_ind_table = device->create_rwq_ind_table(device,
2254		- init_attr, NULL);
2255		- if (IS_ERR(rwq_ind_table))
2256		- return rwq_ind_table;
2257		-
2258		- rwq_ind_table->ind_tbl = init_attr->ind_tbl;
2259		- rwq_ind_table->log_ind_tbl_size = init_attr->log_ind_tbl_size;
2260		- rwq_ind_table->device = device;
2261		- rwq_ind_table->uobject = NULL;
2262		- atomic_set(&rwq_ind_table->usecnt, 0);
2263		-
2264		- for (i = 0; i < table_size; i++)
2265		- atomic_inc(&rwq_ind_table->ind_tbl[i]->usecnt);
2266		-
2267		- return rwq_ind_table;
2268		-}
2269		-EXPORT_SYMBOL(ib_create_rwq_ind_table);
2270		-
2271		-/*
2272		- * ib_destroy_rwq_ind_table - Destroys the specified Indirection Table.
2273		- * @wq_ind_table: The Indirection Table to destroy.
2274		-*/
2275		-int ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table)
2276		-{
2277		- int err, i;
2278		- u32 table_size = (1 << rwq_ind_table->log_ind_tbl_size);
2279		- struct ib_wq **ind_tbl = rwq_ind_table->ind_tbl;
2280		-
2281		- if (atomic_read(&rwq_ind_table->usecnt))
2282		- return -EBUSY;
2283		-
2284		- err = rwq_ind_table->device->destroy_rwq_ind_table(rwq_ind_table);
2285		- if (!err) {
2286		- for (i = 0; i < table_size; i++)
2287		- atomic_dec(&ind_tbl[i]->usecnt);
2288		- }
2289		-
2290		- return err;
2291		-}
2292		-EXPORT_SYMBOL(ib_destroy_rwq_ind_table);
2293		-
2294	2458	int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
2295	2459	struct ib_mr_status *mr_status)
2296	2460	{
2297		- return mr->device->check_mr_status ?
2298		- mr->device->check_mr_status(mr, check_mask, mr_status) : -EOPNOTSUPP;
	2461	+ if (!mr->device->ops.check_mr_status)
	2462	+ return -EOPNOTSUPP;
	2463	+
	2464	+ return mr->device->ops.check_mr_status(mr, check_mask, mr_status);
2299	2465	}
2300	2466	EXPORT_SYMBOL(ib_check_mr_status);
2301	2467
2302	2468	int ib_set_vf_link_state(struct ib_device *device, int vf, u8 port,
2303	2469	int state)
2304	2470	{
2305		- if (!device->set_vf_link_state)
	2471	+ if (!device->ops.set_vf_link_state)
2306	2472	return -EOPNOTSUPP;
2307	2473
2308		- return device->set_vf_link_state(device, vf, port, state);
	2474	+ return device->ops.set_vf_link_state(device, vf, port, state);
2309	2475	}
2310	2476	EXPORT_SYMBOL(ib_set_vf_link_state);
2311	2477
2312	2478	int ib_get_vf_config(struct ib_device *device, int vf, u8 port,
2313	2479	struct ifla_vf_info *info)
2314	2480	{
2315		- if (!device->get_vf_config)
	2481	+ if (!device->ops.get_vf_config)
2316	2482	return -EOPNOTSUPP;
2317	2483
2318		- return device->get_vf_config(device, vf, port, info);
	2484	+ return device->ops.get_vf_config(device, vf, port, info);
2319	2485	}
2320	2486	EXPORT_SYMBOL(ib_get_vf_config);
2321	2487
2322	2488	int ib_get_vf_stats(struct ib_device *device, int vf, u8 port,
2323	2489	struct ifla_vf_stats *stats)
2324	2490	{
2325		- if (!device->get_vf_stats)
	2491	+ if (!device->ops.get_vf_stats)
2326	2492	return -EOPNOTSUPP;
2327	2493
2328		- return device->get_vf_stats(device, vf, port, stats);
	2494	+ return device->ops.get_vf_stats(device, vf, port, stats);
2329	2495	}
2330	2496	EXPORT_SYMBOL(ib_get_vf_stats);
2331	2497
2332	2498	int ib_set_vf_guid(struct ib_device *device, int vf, u8 port, u64 guid,
2333	2499	int type)
2334	2500	{
2335		- if (!device->set_vf_guid)
	2501	+ if (!device->ops.set_vf_guid)
2336	2502	return -EOPNOTSUPP;
2337	2503
2338		- return device->set_vf_guid(device, vf, port, guid, type);
	2504	+ return device->ops.set_vf_guid(device, vf, port, guid, type);
2339	2505	}
2340	2506	EXPORT_SYMBOL(ib_set_vf_guid);
	2507	+
	2508	+int ib_get_vf_guid(struct ib_device *device, int vf, u8 port,
	2509	+ struct ifla_vf_guid *node_guid,
	2510	+ struct ifla_vf_guid *port_guid)
	2511	+{
	2512	+ if (!device->ops.get_vf_guid)
	2513	+ return -EOPNOTSUPP;
	2514	+
	2515	+ return device->ops.get_vf_guid(device, vf, port, node_guid, port_guid);
	2516	+}
	2517	+EXPORT_SYMBOL(ib_get_vf_guid);
	2518	+/**
	2519	+ * ib_map_mr_sg_pi() - Map the dma mapped SG lists for PI (protection
	2520	+ * information) and set an appropriate memory region for registration.
	2521	+ * @mr: memory region
	2522	+ * @data_sg: dma mapped scatterlist for data
	2523	+ * @data_sg_nents: number of entries in data_sg
	2524	+ * @data_sg_offset: offset in bytes into data_sg
	2525	+ * @meta_sg: dma mapped scatterlist for metadata
	2526	+ * @meta_sg_nents: number of entries in meta_sg
	2527	+ * @meta_sg_offset: offset in bytes into meta_sg
	2528	+ * @page_size: page vector desired page size
	2529	+ *
	2530	+ * Constraints:
	2531	+ * - The MR must be allocated with type IB_MR_TYPE_INTEGRITY.
	2532	+ *
	2533	+ * Return: 0 on success.
	2534	+ *
	2535	+ * After this completes successfully, the memory region
	2536	+ * is ready for registration.
	2537	+ */
	2538	+int ib_map_mr_sg_pi(struct ib_mr mr, struct scatterlist data_sg,
	2539	+ int data_sg_nents, unsigned int *data_sg_offset,
	2540	+ struct scatterlist *meta_sg, int meta_sg_nents,
	2541	+ unsigned int *meta_sg_offset, unsigned int page_size)
	2542	+{
	2543	+ if (unlikely(!mr->device->ops.map_mr_sg_pi \|\|
	2544	+ WARN_ON_ONCE(mr->type != IB_MR_TYPE_INTEGRITY)))
	2545	+ return -EOPNOTSUPP;
	2546	+
	2547	+ mr->page_size = page_size;
	2548	+
	2549	+ return mr->device->ops.map_mr_sg_pi(mr, data_sg, data_sg_nents,
	2550	+ data_sg_offset, meta_sg,
	2551	+ meta_sg_nents, meta_sg_offset);
	2552	+}
	2553	+EXPORT_SYMBOL(ib_map_mr_sg_pi);
2341	2554
2342	2555	/**
2343	2556	* ib_map_mr_sg() - Map the largest prefix of a dma mapped SG list
..	..	@@ -2349,6 +2562,7 @@
2349	2562	* @page_size: page vector desired page size
2350	2563	*
2351	2564	* Constraints:
	2565	+ *
2352	2566	* - The first sg element is allowed to have an offset.
2353	2567	* - Each sg element must either be aligned to page_size or virtually
2354	2568	* contiguous to the previous element. In case an sg element has a
..	..	@@ -2367,12 +2581,12 @@
2367	2581	int ib_map_mr_sg(struct ib_mr mr, struct scatterlist sg, int sg_nents,
2368	2582	unsigned int *sg_offset, unsigned int page_size)
2369	2583	{
2370		- if (unlikely(!mr->device->map_mr_sg))
	2584	+ if (unlikely(!mr->device->ops.map_mr_sg))
2371	2585	return -EOPNOTSUPP;
2372	2586
2373	2587	mr->page_size = page_size;
2374	2588
2375		- return mr->device->map_mr_sg(mr, sg, sg_nents, sg_offset);
	2589	+ return mr->device->ops.map_mr_sg(mr, sg, sg_nents, sg_offset);
2376	2590	}
2377	2591	EXPORT_SYMBOL(ib_map_mr_sg);
2378	2592
..	..	@@ -2382,10 +2596,12 @@
2382	2596	* @mr: memory region
2383	2597	* @sgl: dma mapped scatterlist
2384	2598	* @sg_nents: number of entries in sg
2385		- * @sg_offset_p: IN: start offset in bytes into sg
2386		- * OUT: offset in bytes for element n of the sg of the first
	2599	+ * @sg_offset_p: ==== =======================================================
	2600	+ * IN start offset in bytes into sg
	2601	+ * OUT offset in bytes for element n of the sg of the first
2387	2602	* byte that has not been processed where n is the return
2388	2603	* value of this function.
	2604	+ * ==== =======================================================
2389	2605	* @set_page: driver page assignment function pointer
2390	2606	*
2391	2607	* Core service helper for drivers to convert the largest
..	..	@@ -2571,10 +2787,11 @@
2571	2787	*/
2572	2788	void ib_drain_sq(struct ib_qp *qp)
2573	2789	{
2574		- if (qp->device->drain_sq)
2575		- qp->device->drain_sq(qp);
	2790	+ if (qp->device->ops.drain_sq)
	2791	+ qp->device->ops.drain_sq(qp);
2576	2792	else
2577	2793	__ib_drain_sq(qp);
	2794	+ trace_cq_drain_complete(qp->send_cq);
2578	2795	}
2579	2796	EXPORT_SYMBOL(ib_drain_sq);
2580	2797
..	..	@@ -2599,10 +2816,11 @@
2599	2816	*/
2600	2817	void ib_drain_rq(struct ib_qp *qp)
2601	2818	{
2602		- if (qp->device->drain_rq)
2603		- qp->device->drain_rq(qp);
	2819	+ if (qp->device->ops.drain_rq)
	2820	+ qp->device->ops.drain_rq(qp);
2604	2821	else
2605	2822	__ib_drain_rq(qp);
	2823	+ trace_cq_drain_complete(qp->recv_cq);
2606	2824	}
2607	2825	EXPORT_SYMBOL(ib_drain_rq);
2608	2826
..	..	@@ -2628,3 +2846,85 @@
2628	2846	ib_drain_rq(qp);
2629	2847	}
2630	2848	EXPORT_SYMBOL(ib_drain_qp);
	2849	+
	2850	+struct net_device rdma_alloc_netdev(struct ib_device device, u8 port_num,
	2851	+ enum rdma_netdev_t type, const char *name,
	2852	+ unsigned char name_assign_type,
	2853	+ void (setup)(struct net_device ))
	2854	+{
	2855	+ struct rdma_netdev_alloc_params params;
	2856	+ struct net_device *netdev;
	2857	+ int rc;
	2858	+
	2859	+ if (!device->ops.rdma_netdev_get_params)
	2860	+ return ERR_PTR(-EOPNOTSUPP);
	2861	+
	2862	+ rc = device->ops.rdma_netdev_get_params(device, port_num, type,
	2863	+ &params);
	2864	+ if (rc)
	2865	+ return ERR_PTR(rc);
	2866	+
	2867	+ netdev = alloc_netdev_mqs(params.sizeof_priv, name, name_assign_type,
	2868	+ setup, params.txqs, params.rxqs);
	2869	+ if (!netdev)
	2870	+ return ERR_PTR(-ENOMEM);
	2871	+
	2872	+ return netdev;
	2873	+}
	2874	+EXPORT_SYMBOL(rdma_alloc_netdev);
	2875	+
	2876	+int rdma_init_netdev(struct ib_device *device, u8 port_num,
	2877	+ enum rdma_netdev_t type, const char *name,
	2878	+ unsigned char name_assign_type,
	2879	+ void (setup)(struct net_device ),
	2880	+ struct net_device *netdev)
	2881	+{
	2882	+ struct rdma_netdev_alloc_params params;
	2883	+ int rc;
	2884	+
	2885	+ if (!device->ops.rdma_netdev_get_params)
	2886	+ return -EOPNOTSUPP;
	2887	+
	2888	+ rc = device->ops.rdma_netdev_get_params(device, port_num, type,
	2889	+ &params);
	2890	+ if (rc)
	2891	+ return rc;
	2892	+
	2893	+ return params.initialize_rdma_netdev(device, port_num,
	2894	+ netdev, params.param);
	2895	+}
	2896	+EXPORT_SYMBOL(rdma_init_netdev);
	2897	+
	2898	+void __rdma_block_iter_start(struct ib_block_iter *biter,
	2899	+ struct scatterlist *sglist, unsigned int nents,
	2900	+ unsigned long pgsz)
	2901	+{
	2902	+ memset(biter, 0, sizeof(struct ib_block_iter));
	2903	+ biter->__sg = sglist;
	2904	+ biter->__sg_nents = nents;
	2905	+
	2906	+ /* Driver provides best block size to use */
	2907	+ biter->__pg_bit = __fls(pgsz);
	2908	+}
	2909	+EXPORT_SYMBOL(__rdma_block_iter_start);
	2910	+
	2911	+bool __rdma_block_iter_next(struct ib_block_iter *biter)
	2912	+{
	2913	+ unsigned int block_offset;
	2914	+
	2915	+ if (!biter->__sg_nents \|\| !biter->__sg)
	2916	+ return false;
	2917	+
	2918	+ biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance;
	2919	+ block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1);
	2920	+ biter->__sg_advance += BIT_ULL(biter->__pg_bit) - block_offset;
	2921	+
	2922	+ if (biter->__sg_advance >= sg_dma_len(biter->__sg)) {
	2923	+ biter->__sg_advance = 0;
	2924	+ biter->__sg = sg_next(biter->__sg);
	2925	+ biter->__sg_nents--;
	2926	+ }
	2927	+
	2928	+ return true;
	2929	+}
	2930	+EXPORT_SYMBOL(__rdma_block_iter_next);