~hc/RK356X_SDK_RELEASE.git

..	..	@@ -41,6 +41,7 @@
41	41	#include <rdma/ib_pack.h>
42	42	#include <rdma/ib_addr.h>
43	43	#include <rdma/ib_mad.h>
	44	+#include <rdma/uverbs_ioctl.h>
44	45
45	46	#include <linux/mlx4/driver.h>
46	47	#include <linux/mlx4/qp.h>
..	..	@@ -52,7 +53,8 @@
52	53	struct mlx4_ib_cq *recv_cq);
53	54	static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq,
54	55	struct mlx4_ib_cq *recv_cq);
55		-static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state);
	56	+static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state,
	57	+ struct ib_udata *udata);
56	58
57	59	enum {
58	60	MLX4_IB_ACK_REQ_FREQ = 8,
..	..	@@ -63,27 +65,6 @@
63	65	MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
64	66	MLX4_IB_LINK_TYPE_IB = 0,
65	67	MLX4_IB_LINK_TYPE_ETH = 1
66		-};
67		-
68		-enum {
69		- /*
70		- * Largest possible UD header: send with GRH and immediate
71		- * data plus 18 bytes for an Ethernet header with VLAN/802.1Q
72		- * tag. (LRH would only use 8 bytes, so Ethernet is the
73		- * biggest case)
74		- */
75		- MLX4_IB_UD_HEADER_SIZE = 82,
76		- MLX4_IB_LSO_HEADER_SPARE = 128,
77		-};
78		-
79		-struct mlx4_ib_sqp {
80		- struct mlx4_ib_qp qp;
81		- int pkey_index;
82		- u32 qkey;
83		- u32 send_psn;
84		- struct ib_ud_header ud_header;
85		- u8 header_buf[MLX4_IB_UD_HEADER_SIZE];
86		- struct ib_qp *roce_v2_gsi;
87	68	};
88	69
89	70	enum {
..	..	@@ -120,11 +101,6 @@
120	101	MLX4_IB_QP_SRC = 0,
121	102	MLX4_IB_RWQ_SRC = 1,
122	103	};
123		-
124		-static struct mlx4_ib_sqp to_msqp(struct mlx4_ib_qp mqp)
125		-{
126		- return container_of(mqp, struct mlx4_ib_sqp, qp);
127		-}
128	104
129	105	static int is_tunnel_qp(struct mlx4_ib_dev dev, struct mlx4_ib_qp qp)
130	106	{
..	..	@@ -323,7 +299,7 @@
323	299	}
324	300
325	301	static int set_rq_size(struct mlx4_ib_dev dev, struct ib_qp_cap cap,
326		- int is_user, int has_rq, struct mlx4_ib_qp *qp,
	302	+ bool is_user, bool has_rq, struct mlx4_ib_qp *qp,
327	303	u32 inl_recv_sz)
328	304	{
329	305	/* Sanity check RQ size before proceeding */
..	..	@@ -401,7 +377,7 @@
401	377	* We need to leave 2 KB + 1 WR of headroom in the SQ to
402	378	* allow HW to prefetch.
403	379	*/
404		- qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + 1;
	380	+ qp->sq_spare_wqes = MLX4_IB_SQ_HEADROOM(qp->sq.wqe_shift);
405	381	qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr +
406	382	qp->sq_spare_wqes);
407	383
..	..	@@ -436,9 +412,13 @@
436	412	struct mlx4_ib_qp *qp,
437	413	struct mlx4_ib_create_qp *ucmd)
438	414	{
	415	+ u32 cnt;
	416	+
439	417	/* Sanity check SQ size before proceeding */
440		- if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes \|\|
441		- ucmd->log_sq_stride >
	418	+ if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) \|\|
	419	+ cnt > dev->dev->caps.max_wqes)
	420	+ return -EINVAL;
	421	+ if (ucmd->log_sq_stride >
442	422	ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) \|\|
443	423	ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)
444	424	return -EINVAL;
..	..	@@ -504,10 +484,10 @@
504	484	kfree(qp->sqp_proxy_rcv);
505	485	}
506	486
507		-static int qp_has_rq(struct ib_qp_init_attr *attr)
	487	+static bool qp_has_rq(struct ib_qp_init_attr *attr)
508	488	{
509	489	if (attr->qp_type == IB_QPT_XRC_INI \|\| attr->qp_type == IB_QPT_XRC_TGT)
510		- return 0;
	490	+ return false;
511	491
512	492	return !attr->srq;
513	493	}
..	..	@@ -550,15 +530,15 @@
550	530	return (-EOPNOTSUPP);
551	531	}
552	532
553		- if (ucmd->rx_hash_fields_mask & ~(MLX4_IB_RX_HASH_SRC_IPV4 \|
554		- MLX4_IB_RX_HASH_DST_IPV4 \|
555		- MLX4_IB_RX_HASH_SRC_IPV6 \|
556		- MLX4_IB_RX_HASH_DST_IPV6 \|
557		- MLX4_IB_RX_HASH_SRC_PORT_TCP \|
558		- MLX4_IB_RX_HASH_DST_PORT_TCP \|
559		- MLX4_IB_RX_HASH_SRC_PORT_UDP \|
560		- MLX4_IB_RX_HASH_DST_PORT_UDP \|
561		- MLX4_IB_RX_HASH_INNER)) {
	533	+ if (ucmd->rx_hash_fields_mask & ~(u64)(MLX4_IB_RX_HASH_SRC_IPV4 \|
	534	+ MLX4_IB_RX_HASH_DST_IPV4 \|
	535	+ MLX4_IB_RX_HASH_SRC_IPV6 \|
	536	+ MLX4_IB_RX_HASH_DST_IPV6 \|
	537	+ MLX4_IB_RX_HASH_SRC_PORT_TCP \|
	538	+ MLX4_IB_RX_HASH_DST_PORT_TCP \|
	539	+ MLX4_IB_RX_HASH_SRC_PORT_UDP \|
	540	+ MLX4_IB_RX_HASH_DST_PORT_UDP \|
	541	+ MLX4_IB_RX_HASH_INNER)) {
562	542	pr_debug("RX Hash fields_mask has unsupported mask (0x%llx)\n",
563	543	ucmd->rx_hash_fields_mask);
564	544	return (-EOPNOTSUPP);
..	..	@@ -654,8 +634,6 @@
654	634	if (err)
655	635	goto err_qpn;
656	636
657		- mutex_init(&qp->mutex);
658		-
659	637	INIT_LIST_HEAD(&qp->gid_list);
660	638	INIT_LIST_HEAD(&qp->steering_rules);
661	639
..	..	@@ -694,80 +672,72 @@
694	672	return err;
695	673	}
696	674
697		-static struct ib_qp _mlx4_ib_create_qp_rss(struct ib_pd pd,
698		- struct ib_qp_init_attr *init_attr,
699		- struct ib_udata *udata)
	675	+static int _mlx4_ib_create_qp_rss(struct ib_pd pd, struct mlx4_ib_qp qp,
	676	+ struct ib_qp_init_attr *init_attr,
	677	+ struct ib_udata *udata)
700	678	{
701		- struct mlx4_ib_qp *qp;
702	679	struct mlx4_ib_create_qp_rss ucmd = {};
703	680	size_t required_cmd_sz;
704	681	int err;
705	682
706	683	if (!udata) {
707	684	pr_debug("RSS QP with NULL udata\n");
708		- return ERR_PTR(-EINVAL);
	685	+ return -EINVAL;
709	686	}
710	687
711	688	if (udata->outlen)
712		- return ERR_PTR(-EOPNOTSUPP);
	689	+ return -EOPNOTSUPP;
713	690
714	691	required_cmd_sz = offsetof(typeof(ucmd), reserved1) +
715	692	sizeof(ucmd.reserved1);
716	693	if (udata->inlen < required_cmd_sz) {
717	694	pr_debug("invalid inlen\n");
718		- return ERR_PTR(-EINVAL);
	695	+ return -EINVAL;
719	696	}
720	697
721	698	if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
722	699	pr_debug("copy failed\n");
723		- return ERR_PTR(-EFAULT);
	700	+ return -EFAULT;
724	701	}
725	702
726	703	if (memchr_inv(ucmd.reserved, 0, sizeof(ucmd.reserved)))
727		- return ERR_PTR(-EOPNOTSUPP);
	704	+ return -EOPNOTSUPP;
728	705
729	706	if (ucmd.comp_mask \|\| ucmd.reserved1)
730		- return ERR_PTR(-EOPNOTSUPP);
	707	+ return -EOPNOTSUPP;
731	708
732	709	if (udata->inlen > sizeof(ucmd) &&
733	710	!ib_is_udata_cleared(udata, sizeof(ucmd),
734	711	udata->inlen - sizeof(ucmd))) {
735	712	pr_debug("inlen is not supported\n");
736		- return ERR_PTR(-EOPNOTSUPP);
	713	+ return -EOPNOTSUPP;
737	714	}
738	715
739	716	if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
740	717	pr_debug("RSS QP with unsupported QP type %d\n",
741	718	init_attr->qp_type);
742		- return ERR_PTR(-EOPNOTSUPP);
	719	+ return -EOPNOTSUPP;
743	720	}
744	721
745	722	if (init_attr->create_flags) {
746	723	pr_debug("RSS QP doesn't support create flags\n");
747		- return ERR_PTR(-EOPNOTSUPP);
	724	+ return -EOPNOTSUPP;
748	725	}
749	726
750	727	if (init_attr->send_cq \|\| init_attr->cap.max_send_wr) {
751	728	pr_debug("RSS QP with unsupported send attributes\n");
752		- return ERR_PTR(-EOPNOTSUPP);
	729	+ return -EOPNOTSUPP;
753	730	}
754		-
755		- qp = kzalloc(sizeof(*qp), GFP_KERNEL);
756		- if (!qp)
757		- return ERR_PTR(-ENOMEM);
758	731
759	732	qp->pri.vid = 0xFFFF;
760	733	qp->alt.vid = 0xFFFF;
761	734
762	735	err = create_qp_rss(to_mdev(pd->device), init_attr, &ucmd, qp);
763		- if (err) {
764		- kfree(qp);
765		- return ERR_PTR(err);
766		- }
	736	+ if (err)
	737	+ return err;
767	738
768	739	qp->ibqp.qp_num = qp->mqp.qpn;
769		-
770		- return &qp->ibqp;
	740	+ return 0;
771	741	}
772	742
773	743	/*
..	..	@@ -847,26 +817,154 @@
847	817	* reused for further WQN allocations.
848	818	* The next created WQ will allocate a new range.
849	819	*/
850		- range->dirty = 1;
	820	+ range->dirty = true;
851	821	}
852	822
853	823	mutex_unlock(&context->wqn_ranges_mutex);
854	824	}
855	825
856		-static int create_qp_common(struct mlx4_ib_dev dev, struct ib_pd pd,
857		- enum mlx4_ib_source_type src,
858		- struct ib_qp_init_attr *init_attr,
859		- struct ib_udata *udata, int sqpn,
860		- struct mlx4_ib_qp **caller_qp)
	826	+static int create_rq(struct ib_pd pd, struct ib_qp_init_attr init_attr,
	827	+ struct ib_udata udata, struct mlx4_ib_qp qp)
861	828	{
	829	+ struct mlx4_ib_dev *dev = to_mdev(pd->device);
862	830	int qpn;
863	831	int err;
864		- struct mlx4_ib_sqp *sqp = NULL;
865		- struct mlx4_ib_qp *qp;
	832	+ struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
	833	+ udata, struct mlx4_ib_ucontext, ibucontext);
	834	+ struct mlx4_ib_cq *mcq;
	835	+ unsigned long flags;
	836	+ int range_size;
	837	+ struct mlx4_ib_create_wq wq;
	838	+ size_t copy_len;
	839	+ int shift;
	840	+ int n;
	841	+
	842	+ qp->mlx4_ib_qp_type = MLX4_IB_QPT_RAW_PACKET;
	843	+
	844	+ spin_lock_init(&qp->sq.lock);
	845	+ spin_lock_init(&qp->rq.lock);
	846	+ INIT_LIST_HEAD(&qp->gid_list);
	847	+ INIT_LIST_HEAD(&qp->steering_rules);
	848	+
	849	+ qp->state = IB_QPS_RESET;
	850	+
	851	+ copy_len = min(sizeof(struct mlx4_ib_create_wq), udata->inlen);
	852	+
	853	+ if (ib_copy_from_udata(&wq, udata, copy_len)) {
	854	+ err = -EFAULT;
	855	+ goto err;
	856	+ }
	857	+
	858	+ if (wq.comp_mask \|\| wq.reserved[0] \|\| wq.reserved[1] \|\|
	859	+ wq.reserved[2]) {
	860	+ pr_debug("user command isn't supported\n");
	861	+ err = -EOPNOTSUPP;
	862	+ goto err;
	863	+ }
	864	+
	865	+ if (wq.log_range_size > ilog2(dev->dev->caps.max_rss_tbl_sz)) {
	866	+ pr_debug("WQN range size must be equal or smaller than %d\n",
	867	+ dev->dev->caps.max_rss_tbl_sz);
	868	+ err = -EOPNOTSUPP;
	869	+ goto err;
	870	+ }
	871	+ range_size = 1 << wq.log_range_size;
	872	+
	873	+ if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS)
	874	+ qp->flags \|= MLX4_IB_QP_SCATTER_FCS;
	875	+
	876	+ err = set_rq_size(dev, &init_attr->cap, true, true, qp, qp->inl_recv_sz);
	877	+ if (err)
	878	+ goto err;
	879	+
	880	+ qp->sq_no_prefetch = 1;
	881	+ qp->sq.wqe_cnt = 1;
	882	+ qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
	883	+ qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
	884	+ (qp->sq.wqe_cnt << qp->sq.wqe_shift);
	885	+
	886	+ qp->umem = ib_umem_get(pd->device, wq.buf_addr, qp->buf_size, 0);
	887	+ if (IS_ERR(qp->umem)) {
	888	+ err = PTR_ERR(qp->umem);
	889	+ goto err;
	890	+ }
	891	+
	892	+ shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n);
	893	+ err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt);
	894	+
	895	+ if (err)
	896	+ goto err_buf;
	897	+
	898	+ err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
	899	+ if (err)
	900	+ goto err_mtt;
	901	+
	902	+ err = mlx4_ib_db_map_user(udata, wq.db_addr, &qp->db);
	903	+ if (err)
	904	+ goto err_mtt;
	905	+ qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
	906	+
	907	+ err = mlx4_ib_alloc_wqn(context, qp, range_size, &qpn);
	908	+ if (err)
	909	+ goto err_wrid;
	910	+
	911	+ err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
	912	+ if (err)
	913	+ goto err_qpn;
	914	+
	915	+ /*
	916	+ * Hardware wants QPN written in big-endian order (after
	917	+ * shifting) for send doorbell. Precompute this value to save
	918	+ * a little bit when posting sends.
	919	+ */
	920	+ qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
	921	+
	922	+ qp->mqp.event = mlx4_ib_wq_event;
	923	+
	924	+ spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
	925	+ mlx4_ib_lock_cqs(to_mcq(init_attr->send_cq),
	926	+ to_mcq(init_attr->recv_cq));
	927	+ /* Maintain device to QPs access, needed for further handling
	928	+ * via reset flow
	929	+ */
	930	+ list_add_tail(&qp->qps_list, &dev->qp_list);
	931	+ /* Maintain CQ to QPs access, needed for further handling
	932	+ * via reset flow
	933	+ */
	934	+ mcq = to_mcq(init_attr->send_cq);
	935	+ list_add_tail(&qp->cq_send_list, &mcq->send_qp_list);
	936	+ mcq = to_mcq(init_attr->recv_cq);
	937	+ list_add_tail(&qp->cq_recv_list, &mcq->recv_qp_list);
	938	+ mlx4_ib_unlock_cqs(to_mcq(init_attr->send_cq),
	939	+ to_mcq(init_attr->recv_cq));
	940	+ spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
	941	+ return 0;
	942	+
	943	+err_qpn:
	944	+ mlx4_ib_release_wqn(context, qp, 0);
	945	+err_wrid:
	946	+ mlx4_ib_db_unmap_user(context, &qp->db);
	947	+
	948	+err_mtt:
	949	+ mlx4_mtt_cleanup(dev->dev, &qp->mtt);
	950	+err_buf:
	951	+ ib_umem_release(qp->umem);
	952	+err:
	953	+ return err;
	954	+}
	955	+
	956	+static int create_qp_common(struct ib_pd pd, struct ib_qp_init_attr init_attr,
	957	+ struct ib_udata *udata, int sqpn,
	958	+ struct mlx4_ib_qp *qp)
	959	+{
	960	+ struct mlx4_ib_dev *dev = to_mdev(pd->device);
	961	+ int qpn;
	962	+ int err;
	963	+ struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
	964	+ udata, struct mlx4_ib_ucontext, ibucontext);
866	965	enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type;
867	966	struct mlx4_ib_cq *mcq;
868	967	unsigned long flags;
869		- int range_size = 0;
870	968
871	969	/* When tunneling special qps, we use a plain UD qp */
872	970	if (sqpn) {
..	..	@@ -909,76 +1007,41 @@
909	1007	sqpn = qpn;
910	1008	}
911	1009
912		- if (!*caller_qp) {
913		- if (qp_type == MLX4_IB_QPT_SMI \|\| qp_type == MLX4_IB_QPT_GSI \|\|
914		- (qp_type & (MLX4_IB_QPT_PROXY_SMI \| MLX4_IB_QPT_PROXY_SMI_OWNER \|
915		- MLX4_IB_QPT_PROXY_GSI \| MLX4_IB_QPT_TUN_SMI_OWNER))) {
916		- sqp = kzalloc(sizeof(struct mlx4_ib_sqp), GFP_KERNEL);
917		- if (!sqp)
918		- return -ENOMEM;
919		- qp = &sqp->qp;
920		- qp->pri.vid = 0xFFFF;
921		- qp->alt.vid = 0xFFFF;
922		- } else {
923		- qp = kzalloc(sizeof(struct mlx4_ib_qp), GFP_KERNEL);
924		- if (!qp)
925		- return -ENOMEM;
926		- qp->pri.vid = 0xFFFF;
927		- qp->alt.vid = 0xFFFF;
928		- }
929		- } else
930		- qp = *caller_qp;
	1010	+ if (init_attr->qp_type == IB_QPT_SMI \|\|
	1011	+ init_attr->qp_type == IB_QPT_GSI \|\| qp_type == MLX4_IB_QPT_SMI \|\|
	1012	+ qp_type == MLX4_IB_QPT_GSI \|\|
	1013	+ (qp_type & (MLX4_IB_QPT_PROXY_SMI \| MLX4_IB_QPT_PROXY_SMI_OWNER \|
	1014	+ MLX4_IB_QPT_PROXY_GSI \| MLX4_IB_QPT_TUN_SMI_OWNER))) {
	1015	+ qp->sqp = kzalloc(sizeof(struct mlx4_ib_sqp), GFP_KERNEL);
	1016	+ if (!qp->sqp)
	1017	+ return -ENOMEM;
	1018	+ }
931	1019
932	1020	qp->mlx4_ib_qp_type = qp_type;
933	1021
934		- mutex_init(&qp->mutex);
935	1022	spin_lock_init(&qp->sq.lock);
936	1023	spin_lock_init(&qp->rq.lock);
937	1024	INIT_LIST_HEAD(&qp->gid_list);
938	1025	INIT_LIST_HEAD(&qp->steering_rules);
939	1026
940		- qp->state = IB_QPS_RESET;
	1027	+ qp->state = IB_QPS_RESET;
941	1028	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
942	1029	qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
943	1030
944		-
945		- if (pd->uobject) {
946		- union {
947		- struct mlx4_ib_create_qp qp;
948		- struct mlx4_ib_create_wq wq;
949		- } ucmd;
	1031	+ if (udata) {
	1032	+ struct mlx4_ib_create_qp ucmd;
950	1033	size_t copy_len;
951	1034	int shift;
952	1035	int n;
953	1036
954		- copy_len = (src == MLX4_IB_QP_SRC) ?
955		- sizeof(struct mlx4_ib_create_qp) :
956		- min(sizeof(struct mlx4_ib_create_wq), udata->inlen);
	1037	+ copy_len = sizeof(struct mlx4_ib_create_qp);
957	1038
958	1039	if (ib_copy_from_udata(&ucmd, udata, copy_len)) {
959	1040	err = -EFAULT;
960	1041	goto err;
961	1042	}
962	1043
963		- if (src == MLX4_IB_RWQ_SRC) {
964		- if (ucmd.wq.comp_mask \|\| ucmd.wq.reserved[0] \|\|
965		- ucmd.wq.reserved[1] \|\| ucmd.wq.reserved[2]) {
966		- pr_debug("user command isn't supported\n");
967		- err = -EOPNOTSUPP;
968		- goto err;
969		- }
970		-
971		- if (ucmd.wq.log_range_size >
972		- ilog2(dev->dev->caps.max_rss_tbl_sz)) {
973		- pr_debug("WQN range size must be equal or smaller than %d\n",
974		- dev->dev->caps.max_rss_tbl_sz);
975		- err = -EOPNOTSUPP;
976		- goto err;
977		- }
978		- range_size = 1 << ucmd.wq.log_range_size;
979		- } else {
980		- qp->inl_recv_sz = ucmd.qp.inl_recv_sz;
981		- }
	1044	+ qp->inl_recv_sz = ucmd.inl_recv_sz;
982	1045
983	1046	if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS) {
984	1047	if (!(dev->dev->caps.flags &
..	..	@@ -991,39 +1054,24 @@
991	1054	qp->flags \|= MLX4_IB_QP_SCATTER_FCS;
992	1055	}
993	1056
994		- err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
	1057	+ err = set_rq_size(dev, &init_attr->cap, udata,
995	1058	qp_has_rq(init_attr), qp, qp->inl_recv_sz);
996	1059	if (err)
997	1060	goto err;
998	1061
999		- if (src == MLX4_IB_QP_SRC) {
1000		- qp->sq_no_prefetch = ucmd.qp.sq_no_prefetch;
	1062	+ qp->sq_no_prefetch = ucmd.sq_no_prefetch;
1001	1063
1002		- err = set_user_sq_size(dev, qp,
1003		- (struct mlx4_ib_create_qp *)
1004		- &ucmd);
1005		- if (err)
1006		- goto err;
1007		- } else {
1008		- qp->sq_no_prefetch = 1;
1009		- qp->sq.wqe_cnt = 1;
1010		- qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
1011		- /* Allocated buffer expects to have at least that SQ
1012		- * size.
1013		- */
1014		- qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
1015		- (qp->sq.wqe_cnt << qp->sq.wqe_shift);
1016		- }
	1064	+ err = set_user_sq_size(dev, qp, &ucmd);
	1065	+ if (err)
	1066	+ goto err;
1017	1067
1018		- qp->umem = ib_umem_get(pd->uobject->context,
1019		- (src == MLX4_IB_QP_SRC) ? ucmd.qp.buf_addr :
1020		- ucmd.wq.buf_addr, qp->buf_size, 0, 0);
	1068	+ qp->umem =
	1069	+ ib_umem_get(pd->device, ucmd.buf_addr, qp->buf_size, 0);
1021	1070	if (IS_ERR(qp->umem)) {
1022	1071	err = PTR_ERR(qp->umem);
1023	1072	goto err;
1024	1073	}
1025	1074
1026		- n = ib_umem_page_count(qp->umem);
1027	1075	shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n);
1028	1076	err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt);
1029	1077
..	..	@@ -1035,15 +1083,13 @@
1035	1083	goto err_mtt;
1036	1084
1037	1085	if (qp_has_rq(init_attr)) {
1038		- err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
1039		- (src == MLX4_IB_QP_SRC) ? ucmd.qp.db_addr :
1040		- ucmd.wq.db_addr, &qp->db);
	1086	+ err = mlx4_ib_db_map_user(udata, ucmd.db_addr, &qp->db);
1041	1087	if (err)
1042	1088	goto err_mtt;
1043	1089	}
1044	1090	qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
1045	1091	} else {
1046		- err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
	1092	+ err = set_rq_size(dev, &init_attr->cap, udata,
1047	1093	qp_has_rq(init_attr), qp, 0);
1048	1094	if (err)
1049	1095	goto err;
..	..	@@ -1109,11 +1155,6 @@
1109	1155	goto err_wrid;
1110	1156	}
1111	1157	}
1112		- } else if (src == MLX4_IB_RWQ_SRC) {
1113		- err = mlx4_ib_alloc_wqn(to_mucontext(pd->uobject->context), qp,
1114		- range_size, &qpn);
1115		- if (err)
1116		- goto err_wrid;
1117	1158	} else {
1118	1159	/* Raw packet QPNs may not have bits 6,7 set in their qp_num;
1119	1160	* otherwise, the WQE BlueFlame setup flow wrongly causes
..	..	@@ -1152,11 +1193,7 @@
1152	1193	*/
1153	1194	qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
1154	1195
1155		- qp->mqp.event = (src == MLX4_IB_QP_SRC) ? mlx4_ib_qp_event :
1156		- mlx4_ib_wq_event;
1157		-
1158		- if (!*caller_qp)
1159		- *caller_qp = qp;
	1196	+ qp->mqp.event = mlx4_ib_qp_event;
1160	1197
1161	1198	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1162	1199	mlx4_ib_lock_cqs(to_mcq(init_attr->send_cq),
..	..	@@ -1181,9 +1218,6 @@
1181	1218	if (!sqpn) {
1182	1219	if (qp->flags & MLX4_IB_QP_NETIF)
1183	1220	mlx4_ib_steer_qp_free(dev, qpn, 1);
1184		- else if (src == MLX4_IB_RWQ_SRC)
1185		- mlx4_ib_release_wqn(to_mucontext(pd->uobject->context),
1186		- qp, 0);
1187	1221	else
1188	1222	mlx4_qp_release_range(dev->dev, qpn, 1);
1189	1223	}
..	..	@@ -1191,9 +1225,9 @@
1191	1225	if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
1192	1226	free_proxy_bufs(pd->device, qp);
1193	1227	err_wrid:
1194		- if (pd->uobject) {
	1228	+ if (udata) {
1195	1229	if (qp_has_rq(init_attr))
1196		- mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db);
	1230	+ mlx4_ib_db_unmap_user(context, &qp->db);
1197	1231	} else {
1198	1232	kvfree(qp->sq.wrid);
1199	1233	kvfree(qp->rq.wrid);
..	..	@@ -1203,20 +1237,16 @@
1203	1237	mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1204	1238
1205	1239	err_buf:
1206		- if (pd->uobject)
1207		- ib_umem_release(qp->umem);
1208		- else
	1240	+ if (!qp->umem)
1209	1241	mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
	1242	+ ib_umem_release(qp->umem);
1210	1243
1211	1244	err_db:
1212		- if (!pd->uobject && qp_has_rq(init_attr))
	1245	+ if (!udata && qp_has_rq(init_attr))
1213	1246	mlx4_db_free(dev->dev, &qp->db);
1214	1247
1215	1248	err:
1216		- if (sqp)
1217		- kfree(sqp);
1218		- else if (!*caller_qp)
1219		- kfree(qp);
	1249	+ kfree(qp->sqp);
1220	1250	return err;
1221	1251	}
1222	1252
..	..	@@ -1330,11 +1360,11 @@
1330	1360	mlx4_qp_free(dev->dev, &qp->mqp);
1331	1361	mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1332	1362	del_gid_entries(qp);
1333		- kfree(qp->rss_ctx);
1334	1363	}
1335	1364
1336	1365	static void destroy_qp_common(struct mlx4_ib_dev dev, struct mlx4_ib_qp qp,
1337		- enum mlx4_ib_source_type src, int is_user)
	1366	+ enum mlx4_ib_source_type src,
	1367	+ struct ib_udata *udata)
1338	1368	{
1339	1369	struct mlx4_ib_cq send_cq, recv_cq;
1340	1370	unsigned long flags;
..	..	@@ -1376,7 +1406,7 @@
1376	1406	list_del(&qp->qps_list);
1377	1407	list_del(&qp->cq_send_list);
1378	1408	list_del(&qp->cq_recv_list);
1379		- if (!is_user) {
	1409	+ if (!udata) {
1380	1410	__mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
1381	1411	qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
1382	1412	if (send_cq != recv_cq)
..	..	@@ -1394,22 +1424,28 @@
1394	1424	if (qp->flags & MLX4_IB_QP_NETIF)
1395	1425	mlx4_ib_steer_qp_free(dev, qp->mqp.qpn, 1);
1396	1426	else if (src == MLX4_IB_RWQ_SRC)
1397		- mlx4_ib_release_wqn(to_mucontext(
1398		- qp->ibwq.uobject->context), qp, 1);
	1427	+ mlx4_ib_release_wqn(
	1428	+ rdma_udata_to_drv_context(
	1429	+ udata,
	1430	+ struct mlx4_ib_ucontext,
	1431	+ ibucontext),
	1432	+ qp, 1);
1399	1433	else
1400	1434	mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1401	1435	}
1402	1436
1403	1437	mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1404	1438
1405		- if (is_user) {
	1439	+ if (udata) {
1406	1440	if (qp->rq.wqe_cnt) {
1407		- struct mlx4_ib_ucontext *mcontext = !src ?
1408		- to_mucontext(qp->ibqp.uobject->context) :
1409		- to_mucontext(qp->ibwq.uobject->context);
	1441	+ struct mlx4_ib_ucontext *mcontext =
	1442	+ rdma_udata_to_drv_context(
	1443	+ udata,
	1444	+ struct mlx4_ib_ucontext,
	1445	+ ibucontext);
	1446	+
1410	1447	mlx4_ib_db_unmap_user(mcontext, &qp->db);
1411	1448	}
1412		- ib_umem_release(qp->umem);
1413	1449	} else {
1414	1450	kvfree(qp->sq.wrid);
1415	1451	kvfree(qp->rq.wrid);
..	..	@@ -1420,6 +1456,7 @@
1420	1456	if (qp->rq.wqe_cnt)
1421	1457	mlx4_db_free(dev->dev, &qp->db);
1422	1458	}
	1459	+ ib_umem_release(qp->umem);
1423	1460
1424	1461	del_gid_entries(qp);
1425	1462	}
..	..	@@ -1441,17 +1478,16 @@
1441	1478	return dev->dev->caps.spec_qps[attr->port_num - 1].qp1_proxy;
1442	1479	}
1443	1480
1444		-static struct ib_qp _mlx4_ib_create_qp(struct ib_pd pd,
1445		- struct ib_qp_init_attr *init_attr,
1446		- struct ib_udata *udata)
	1481	+static int _mlx4_ib_create_qp(struct ib_pd pd, struct mlx4_ib_qp qp,
	1482	+ struct ib_qp_init_attr *init_attr,
	1483	+ struct ib_udata *udata)
1447	1484	{
1448		- struct mlx4_ib_qp *qp = NULL;
1449	1485	int err;
1450	1486	int sup_u_create_flags = MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1451	1487	u16 xrcdn = 0;
1452	1488
1453	1489	if (init_attr->rwq_ind_tbl)
1454		- return _mlx4_ib_create_qp_rss(pd, init_attr, udata);
	1490	+ return _mlx4_ib_create_qp_rss(pd, qp, init_attr, udata);
1455	1491
1456	1492	/*
1457	1493	* We only support LSO, vendor flag1, and multicast loopback blocking,
..	..	@@ -1463,16 +1499,16 @@
1463	1499	MLX4_IB_SRIOV_SQP \|
1464	1500	MLX4_IB_QP_NETIF \|
1465	1501	MLX4_IB_QP_CREATE_ROCE_V2_GSI))
1466		- return ERR_PTR(-EINVAL);
	1502	+ return -EINVAL;
1467	1503
1468	1504	if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
1469	1505	if (init_attr->qp_type != IB_QPT_UD)
1470		- return ERR_PTR(-EINVAL);
	1506	+ return -EINVAL;
1471	1507	}
1472	1508
1473	1509	if (init_attr->create_flags) {
1474	1510	if (udata && init_attr->create_flags & ~(sup_u_create_flags))
1475		- return ERR_PTR(-EINVAL);
	1511	+ return -EINVAL;
1476	1512
1477	1513	if ((init_attr->create_flags & ~(MLX4_IB_SRIOV_SQP \|
1478	1514	MLX4_IB_QP_CREATE_ROCE_V2_GSI \|
..	..	@@ -1482,7 +1518,7 @@
1482	1518	init_attr->qp_type > IB_QPT_GSI) \|\|
1483	1519	(init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI &&
1484	1520	init_attr->qp_type != IB_QPT_GSI))
1485		- return ERR_PTR(-EINVAL);
	1521	+ return -EINVAL;
1486	1522	}
1487	1523
1488	1524	switch (init_attr->qp_type) {
..	..	@@ -1490,58 +1526,46 @@
1490	1526	pd = to_mxrcd(init_attr->xrcd)->pd;
1491	1527	xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
1492	1528	init_attr->send_cq = to_mxrcd(init_attr->xrcd)->cq;
1493		- /* fall through */
	1529	+ fallthrough;
1494	1530	case IB_QPT_XRC_INI:
1495	1531	if (!(to_mdev(pd->device)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
1496		- return ERR_PTR(-ENOSYS);
	1532	+ return -ENOSYS;
1497	1533	init_attr->recv_cq = init_attr->send_cq;
1498		- /* fall through */
	1534	+ fallthrough;
1499	1535	case IB_QPT_RC:
1500	1536	case IB_QPT_UC:
1501	1537	case IB_QPT_RAW_PACKET:
1502		- qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1503		- if (!qp)
1504		- return ERR_PTR(-ENOMEM);
	1538	+ case IB_QPT_UD:
1505	1539	qp->pri.vid = 0xFFFF;
1506	1540	qp->alt.vid = 0xFFFF;
1507		- /* fall through */
1508		- case IB_QPT_UD:
1509		- {
1510		- err = create_qp_common(to_mdev(pd->device), pd, MLX4_IB_QP_SRC,
1511		- init_attr, udata, 0, &qp);
1512		- if (err) {
1513		- kfree(qp);
1514		- return ERR_PTR(err);
1515		- }
	1541	+ err = create_qp_common(pd, init_attr, udata, 0, qp);
	1542	+ if (err)
	1543	+ return err;
1516	1544
1517	1545	qp->ibqp.qp_num = qp->mqp.qpn;
1518	1546	qp->xrcdn = xrcdn;
1519		-
1520	1547	break;
1521		- }
1522	1548	case IB_QPT_SMI:
1523	1549	case IB_QPT_GSI:
1524	1550	{
1525	1551	int sqpn;
1526	1552
1527		- /* Userspace is not allowed to create special QPs: */
1528		- if (udata)
1529		- return ERR_PTR(-EINVAL);
1530	1553	if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) {
1531	1554	int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev,
1532	1555	1, 1, &sqpn, 0,
1533	1556	MLX4_RES_USAGE_DRIVER);
1534	1557
1535	1558	if (res)
1536		- return ERR_PTR(res);
	1559	+ return res;
1537	1560	} else {
1538	1561	sqpn = get_sqp_num(to_mdev(pd->device), init_attr);
1539	1562	}
1540	1563
1541		- err = create_qp_common(to_mdev(pd->device), pd, MLX4_IB_QP_SRC,
1542		- init_attr, udata, sqpn, &qp);
	1564	+ qp->pri.vid = 0xFFFF;
	1565	+ qp->alt.vid = 0xFFFF;
	1566	+ err = create_qp_common(pd, init_attr, udata, sqpn, qp);
1543	1567	if (err)
1544		- return ERR_PTR(err);
	1568	+ return err;
1545	1569
1546	1570	qp->port = init_attr->port_num;
1547	1571	qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 :
..	..	@@ -1550,25 +1574,33 @@
1550	1574	}
1551	1575	default:
1552	1576	/* Don't support raw QPs */
1553		- return ERR_PTR(-EINVAL);
	1577	+ return -EOPNOTSUPP;
1554	1578	}
1555		-
1556		- return &qp->ibqp;
	1579	+ return 0;
1557	1580	}
1558	1581
1559	1582	struct ib_qp mlx4_ib_create_qp(struct ib_pd pd,
1560	1583	struct ib_qp_init_attr *init_attr,
1561	1584	struct ib_udata *udata) {
1562	1585	struct ib_device *device = pd ? pd->device : init_attr->xrcd->device;
1563		- struct ib_qp *ibqp;
1564	1586	struct mlx4_ib_dev *dev = to_mdev(device);
	1587	+ struct mlx4_ib_qp *qp;
	1588	+ int ret;
1565	1589
1566		- ibqp = _mlx4_ib_create_qp(pd, init_attr, udata);
	1590	+ qp = kzalloc(sizeof(*qp), GFP_KERNEL);
	1591	+ if (!qp)
	1592	+ return ERR_PTR(-ENOMEM);
1567	1593
1568		- if (!IS_ERR(ibqp) &&
1569		- (init_attr->qp_type == IB_QPT_GSI) &&
	1594	+ mutex_init(&qp->mutex);
	1595	+ ret = _mlx4_ib_create_qp(pd, qp, init_attr, udata);
	1596	+ if (ret) {
	1597	+ kfree(qp);
	1598	+ return ERR_PTR(ret);
	1599	+ }
	1600	+
	1601	+ if (init_attr->qp_type == IB_QPT_GSI &&
1570	1602	!(init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI)) {
1571		- struct mlx4_ib_sqp *sqp = to_msqp((to_mqp(ibqp)));
	1603	+ struct mlx4_ib_sqp *sqp = qp->sqp;
1572	1604	int is_eth = rdma_cap_eth_ah(&dev->ib_dev, init_attr->port_num);
1573	1605
1574	1606	if (is_eth &&
..	..	@@ -1580,17 +1612,17 @@
1580	1612	pr_err("Failed to create GSI QP for RoCEv2 (%ld)\n", PTR_ERR(sqp->roce_v2_gsi));
1581	1613	sqp->roce_v2_gsi = NULL;
1582	1614	} else {
1583		- sqp = to_msqp(to_mqp(sqp->roce_v2_gsi));
1584		- sqp->qp.flags \|= MLX4_IB_ROCE_V2_GSI_QP;
	1615	+ to_mqp(sqp->roce_v2_gsi)->flags \|=
	1616	+ MLX4_IB_ROCE_V2_GSI_QP;
1585	1617	}
1586	1618
1587	1619	init_attr->create_flags &= ~MLX4_IB_QP_CREATE_ROCE_V2_GSI;
1588	1620	}
1589	1621	}
1590		- return ibqp;
	1622	+ return &qp->ibqp;
1591	1623	}
1592	1624
1593		-static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
	1625	+static int _mlx4_ib_destroy_qp(struct ib_qp qp, struct ib_udata udata)
1594	1626	{
1595	1627	struct mlx4_ib_dev *dev = to_mdev(qp->device);
1596	1628	struct mlx4_ib_qp *mqp = to_mqp(qp);
..	..	@@ -1611,32 +1643,27 @@
1611	1643	if (qp->rwq_ind_tbl) {
1612	1644	destroy_qp_rss(dev, mqp);
1613	1645	} else {
1614		- struct mlx4_ib_pd *pd;
1615		-
1616		- pd = get_pd(mqp);
1617		- destroy_qp_common(dev, mqp, MLX4_IB_QP_SRC, !!pd->ibpd.uobject);
	1646	+ destroy_qp_common(dev, mqp, MLX4_IB_QP_SRC, udata);
1618	1647	}
1619	1648
1620		- if (is_sqp(dev, mqp))
1621		- kfree(to_msqp(mqp));
1622		- else
1623		- kfree(mqp);
	1649	+ kfree(mqp->sqp);
	1650	+ kfree(mqp);
1624	1651
1625	1652	return 0;
1626	1653	}
1627	1654
1628		-int mlx4_ib_destroy_qp(struct ib_qp *qp)
	1655	+int mlx4_ib_destroy_qp(struct ib_qp qp, struct ib_udata udata)
1629	1656	{
1630	1657	struct mlx4_ib_qp *mqp = to_mqp(qp);
1631	1658
1632	1659	if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
1633		- struct mlx4_ib_sqp *sqp = to_msqp(mqp);
	1660	+ struct mlx4_ib_sqp *sqp = mqp->sqp;
1634	1661
1635	1662	if (sqp->roce_v2_gsi)
1636	1663	ib_destroy_qp(sqp->roce_v2_gsi);
1637	1664	}
1638	1665
1639		- return _mlx4_ib_destroy_qp(qp);
	1666	+ return _mlx4_ib_destroy_qp(qp, udata);
1640	1667	}
1641	1668
1642	1669	static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type)
..	..	@@ -1943,7 +1970,8 @@
1943	1970	* Go over all RSS QP's childes (WQs) and apply their HW state according to
1944	1971	* their logic state if the RSS QP is the first RSS QP associated for the WQ.
1945	1972	*/
1946		-static int bringup_rss_rwqs(struct ib_rwq_ind_table *ind_tbl, u8 port_num)
	1973	+static int bringup_rss_rwqs(struct ib_rwq_ind_table *ind_tbl, u8 port_num,
	1974	+ struct ib_udata *udata)
1947	1975	{
1948	1976	int err = 0;
1949	1977	int i;
..	..	@@ -1967,7 +1995,7 @@
1967	1995	}
1968	1996	wq->port = port_num;
1969	1997	if ((wq->rss_usecnt == 0) && (ibwq->state == IB_WQS_RDY)) {
1970		- err = _mlx4_ib_modify_wq(ibwq, IB_WQS_RDY);
	1998	+ err = _mlx4_ib_modify_wq(ibwq, IB_WQS_RDY, udata);
1971	1999	if (err) {
1972	2000	mutex_unlock(&wq->mutex);
1973	2001	break;
..	..	@@ -1989,7 +2017,8 @@
1989	2017
1990	2018	if ((wq->rss_usecnt == 1) &&
1991	2019	(ibwq->state == IB_WQS_RDY))
1992		- if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET))
	2020	+ if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET,
	2021	+ udata))
1993	2022	pr_warn("failed to reverse WQN=0x%06x\n",
1994	2023	ibwq->wq_num);
1995	2024	wq->rss_usecnt--;
..	..	@@ -2001,7 +2030,8 @@
2001	2030	return err;
2002	2031	}
2003	2032
2004		-static void bring_down_rss_rwqs(struct ib_rwq_ind_table *ind_tbl)
	2033	+static void bring_down_rss_rwqs(struct ib_rwq_ind_table *ind_tbl,
	2034	+ struct ib_udata *udata)
2005	2035	{
2006	2036	int i;
2007	2037
..	..	@@ -2012,7 +2042,7 @@
2012	2042	mutex_lock(&wq->mutex);
2013	2043
2014	2044	if ((wq->rss_usecnt == 1) && (ibwq->state == IB_WQS_RDY))
2015		- if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET))
	2045	+ if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET, udata))
2016	2046	pr_warn("failed to reverse WQN=%x\n",
2017	2047	ibwq->wq_num);
2018	2048	wq->rss_usecnt--;
..	..	@@ -2044,9 +2074,10 @@
2044	2074
2045	2075	static int __mlx4_ib_modify_qp(void *src, enum mlx4_ib_source_type src_type,
2046	2076	const struct ib_qp_attr *attr, int attr_mask,
2047		- enum ib_qp_state cur_state, enum ib_qp_state new_state)
	2077	+ enum ib_qp_state cur_state,
	2078	+ enum ib_qp_state new_state,
	2079	+ struct ib_udata *udata)
2048	2080	{
2049		- struct ib_uobject *ibuobject;
2050	2081	struct ib_srq *ibsrq;
2051	2082	const struct ib_gid_attr *gid_attr = NULL;
2052	2083	struct ib_rwq_ind_table *rwq_ind_tbl;
..	..	@@ -2055,6 +2086,8 @@
2055	2086	struct mlx4_ib_qp *qp;
2056	2087	struct mlx4_ib_pd *pd;
2057	2088	struct mlx4_ib_cq send_cq, recv_cq;
	2089	+ struct mlx4_ib_ucontext *ucontext = rdma_udata_to_drv_context(
	2090	+ udata, struct mlx4_ib_ucontext, ibucontext);
2058	2091	struct mlx4_qp_context *context;
2059	2092	enum mlx4_qp_optpar optpar = 0;
2060	2093	int sqd_event;
..	..	@@ -2066,7 +2099,6 @@
2066	2099	struct ib_wq *ibwq;
2067	2100
2068	2101	ibwq = (struct ib_wq *)src;
2069		- ibuobject = ibwq->uobject;
2070	2102	ibsrq = NULL;
2071	2103	rwq_ind_tbl = NULL;
2072	2104	qp_type = IB_QPT_RAW_PACKET;
..	..	@@ -2077,7 +2109,6 @@
2077	2109	struct ib_qp *ibqp;
2078	2110
2079	2111	ibqp = (struct ib_qp *)src;
2080		- ibuobject = ibqp->uobject;
2081	2112	ibsrq = ibqp->srq;
2082	2113	rwq_ind_tbl = ibqp->rwq_ind_tbl;
2083	2114	qp_type = ibqp->qp_type;
..	..	@@ -2162,11 +2193,9 @@
2162	2193	context->param3 \|= cpu_to_be32(1 << 30);
2163	2194	}
2164	2195
2165		- if (ibuobject)
	2196	+ if (ucontext)
2166	2197	context->usr_page = cpu_to_be32(
2167		- mlx4_to_hw_uar_index(dev->dev,
2168		- to_mucontext(ibuobject->context)
2169		- ->uar.index));
	2198	+ mlx4_to_hw_uar_index(dev->dev, ucontext->uar.index));
2170	2199	else
2171	2200	context->usr_page = cpu_to_be32(
2172	2201	mlx4_to_hw_uar_index(dev->dev, dev->priv_uar.index));
..	..	@@ -2237,8 +2266,10 @@
2237	2266
2238	2267	if (is_eth) {
2239	2268	gid_attr = attr->ah_attr.grh.sgid_attr;
2240		- vlan = rdma_vlan_dev_vlan_id(gid_attr->ndev);
2241		- memcpy(smac, gid_attr->ndev->dev_addr, ETH_ALEN);
	2269	+ err = rdma_read_gid_l2_fields(gid_attr, &vlan,
	2270	+ &smac[0]);
	2271	+ if (err)
	2272	+ goto out;
2242	2273	}
2243	2274
2244	2275	if (mlx4_set_path(dev, attr, attr_mask, qp, &context->pri_path,
..	..	@@ -2298,7 +2329,7 @@
2298	2329	context->cqn_recv = cpu_to_be32(recv_cq->mcq.cqn);
2299	2330
2300	2331	/* Set "fast registration enabled" for all kernel QPs */
2301		- if (!ibuobject)
	2332	+ if (!ucontext)
2302	2333	context->params1 \|= cpu_to_be32(1 << 11);
2303	2334
2304	2335	if (attr_mask & IB_QP_RNR_RETRY) {
..	..	@@ -2435,7 +2466,7 @@
2435	2466	else
2436	2467	sqd_event = 0;
2437	2468
2438		- if (!ibuobject &&
	2469	+ if (!ucontext &&
2439	2470	cur_state == IB_QPS_RESET &&
2440	2471	new_state == IB_QPS_INIT)
2441	2472	context->rlkey_roce_mode \|= (1 << 4);
..	..	@@ -2446,7 +2477,7 @@
2446	2477	* headroom is stamped so that the hardware doesn't start
2447	2478	* processing stale work requests.
2448	2479	*/
2449		- if (!ibuobject &&
	2480	+ if (!ucontext &&
2450	2481	cur_state == IB_QPS_RESET &&
2451	2482	new_state == IB_QPS_INIT) {
2452	2483	struct mlx4_wqe_ctrl_seg *ctrl;
..	..	@@ -2488,7 +2519,7 @@
2488	2519	qp->alt_port = attr->alt_port_num;
2489	2520
2490	2521	if (is_sqp(dev, qp))
2491		- store_sqp_attrs(to_msqp(qp), attr, attr_mask);
	2522	+ store_sqp_attrs(qp->sqp, attr, attr_mask);
2492	2523
2493	2524	/*
2494	2525	* If we moved QP0 to RTR, bring the IB link up; if we moved
..	..	@@ -2510,7 +2541,7 @@
2510	2541	* entries and reinitialize the QP.
2511	2542	*/
2512	2543	if (new_state == IB_QPS_RESET) {
2513		- if (!ibuobject) {
	2544	+ if (!ucontext) {
2514	2545	mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
2515	2546	ibsrq ? to_msrq(ibsrq) : NULL);
2516	2547	if (send_cq != recv_cq)
..	..	@@ -2631,7 +2662,6 @@
2631	2662	static int _mlx4_ib_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
2632	2663	int attr_mask, struct ib_udata *udata)
2633	2664	{
2634		- enum rdma_link_layer ll = IB_LINK_LAYER_UNSPECIFIED;
2635	2665	struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
2636	2666	struct mlx4_ib_qp *qp = to_mqp(ibqp);
2637	2667	enum ib_qp_state cur_state, new_state;
..	..	@@ -2641,13 +2671,8 @@
2641	2671	cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
2642	2672	new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
2643	2673
2644		- if (cur_state != new_state \|\| cur_state != IB_QPS_RESET) {
2645		- int port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2646		- ll = rdma_port_get_link_layer(&dev->ib_dev, port);
2647		- }
2648		-
2649	2674	if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
2650		- attr_mask, ll)) {
	2675	+ attr_mask)) {
2651	2676	pr_debug("qpn 0x%x: invalid attribute mask specified "
2652	2677	"for transition %d to %d. qp_type %d,"
2653	2678	" attr_mask 0x%x\n",
..	..	@@ -2742,16 +2767,17 @@
2742	2767	}
2743	2768
2744	2769	if (ibqp->rwq_ind_tbl && (new_state == IB_QPS_INIT)) {
2745		- err = bringup_rss_rwqs(ibqp->rwq_ind_tbl, attr->port_num);
	2770	+ err = bringup_rss_rwqs(ibqp->rwq_ind_tbl, attr->port_num,
	2771	+ udata);
2746	2772	if (err)
2747	2773	goto out;
2748	2774	}
2749	2775
2750	2776	err = __mlx4_ib_modify_qp(ibqp, MLX4_IB_QP_SRC, attr, attr_mask,
2751		- cur_state, new_state);
	2777	+ cur_state, new_state, udata);
2752	2778
2753	2779	if (ibqp->rwq_ind_tbl && err)
2754		- bring_down_rss_rwqs(ibqp->rwq_ind_tbl);
	2780	+ bring_down_rss_rwqs(ibqp->rwq_ind_tbl, udata);
2755	2781
2756	2782	if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT))
2757	2783	attr->port_num = 1;
..	..	@@ -2770,7 +2796,7 @@
2770	2796	ret = _mlx4_ib_modify_qp(ibqp, attr, attr_mask, udata);
2771	2797
2772	2798	if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
2773		- struct mlx4_ib_sqp *sqp = to_msqp(mqp);
	2799	+ struct mlx4_ib_sqp *sqp = mqp->sqp;
2774	2800	int err = 0;
2775	2801
2776	2802	if (sqp->roce_v2_gsi)
..	..	@@ -2795,12 +2821,13 @@
2795	2821	return -EINVAL;
2796	2822	}
2797	2823
2798		-static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,
	2824	+static int build_sriov_qp0_header(struct mlx4_ib_qp *qp,
2799	2825	const struct ib_ud_wr *wr,
2800	2826	void wqe, unsigned mlx_seg_len)
2801	2827	{
2802		- struct mlx4_ib_dev *mdev = to_mdev(sqp->qp.ibqp.device);
2803		- struct ib_device *ib_dev = &mdev->ib_dev;
	2828	+ struct mlx4_ib_dev *mdev = to_mdev(qp->ibqp.device);
	2829	+ struct mlx4_ib_sqp *sqp = qp->sqp;
	2830	+ struct ib_device *ib_dev = qp->ibqp.device;
2804	2831	struct mlx4_wqe_mlx_seg *mlx = wqe;
2805	2832	struct mlx4_wqe_inline_seg inl = wqe + sizeof mlx;
2806	2833	struct mlx4_ib_ah *ah = to_mah(wr->ah);
..	..	@@ -2822,12 +2849,12 @@
2822	2849
2823	2850	/* for proxy-qp0 sends, need to add in size of tunnel header */
2824	2851	/* for tunnel-qp0 sends, tunnel header is already in s/g list */
2825		- if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER)
	2852	+ if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER)
2826	2853	send_size += sizeof (struct mlx4_ib_tunnel_header);
2827	2854
2828	2855	ib_ud_header_init(send_size, 1, 0, 0, 0, 0, 0, 0, &sqp->ud_header);
2829	2856
2830		- if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) {
	2857	+ if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) {
2831	2858	sqp->ud_header.lrh.service_level =
2832	2859	be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
2833	2860	sqp->ud_header.lrh.destination_lid =
..	..	@@ -2844,26 +2871,26 @@
2844	2871
2845	2872	sqp->ud_header.lrh.virtual_lane = 0;
2846	2873	sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
2847		- err = ib_get_cached_pkey(ib_dev, sqp->qp.port, 0, &pkey);
	2874	+ err = ib_get_cached_pkey(ib_dev, qp->port, 0, &pkey);
2848	2875	if (err)
2849	2876	return err;
2850	2877	sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
2851		- if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER)
	2878	+ if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER)
2852	2879	sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn);
2853	2880	else
2854	2881	sqp->ud_header.bth.destination_qpn =
2855		- cpu_to_be32(mdev->dev->caps.spec_qps[sqp->qp.port - 1].qp0_tunnel);
	2882	+ cpu_to_be32(mdev->dev->caps.spec_qps[qp->port - 1].qp0_tunnel);
2856	2883
2857	2884	sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
2858	2885	if (mlx4_is_master(mdev->dev)) {
2859		- if (mlx4_get_parav_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
	2886	+ if (mlx4_get_parav_qkey(mdev->dev, qp->mqp.qpn, &qkey))
2860	2887	return -EINVAL;
2861	2888	} else {
2862		- if (vf_get_qp0_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
	2889	+ if (vf_get_qp0_qkey(mdev->dev, qp->mqp.qpn, &qkey))
2863	2890	return -EINVAL;
2864	2891	}
2865	2892	sqp->ud_header.deth.qkey = cpu_to_be32(qkey);
2866		- sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.mqp.qpn);
	2893	+ sqp->ud_header.deth.source_qpn = cpu_to_be32(qp->mqp.qpn);
2867	2894
2868	2895	sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2869	2896	sqp->ud_header.immediate_present = 0;
..	..	@@ -2947,10 +2974,11 @@
2947	2974	}
2948	2975
2949	2976	#define MLX4_ROCEV2_QP1_SPORT 0xC000
2950		-static int build_mlx_header(struct mlx4_ib_sqp sqp, const struct ib_ud_wr wr,
	2977	+static int build_mlx_header(struct mlx4_ib_qp qp, const struct ib_ud_wr wr,
2951	2978	void wqe, unsigned mlx_seg_len)
2952	2979	{
2953		- struct ib_device *ib_dev = sqp->qp.ibqp.device;
	2980	+ struct mlx4_ib_sqp *sqp = qp->sqp;
	2981	+ struct ib_device *ib_dev = qp->ibqp.device;
2954	2982	struct mlx4_ib_dev *ibdev = to_mdev(ib_dev);
2955	2983	struct mlx4_wqe_mlx_seg *mlx = wqe;
2956	2984	struct mlx4_wqe_ctrl_seg *ctrl = wqe;
..	..	@@ -2974,7 +3002,7 @@
2974	3002	for (i = 0; i < wr->wr.num_sge; ++i)
2975	3003	send_size += wr->wr.sg_list[i].length;
2976	3004
2977		- is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET;
	3005	+ is_eth = rdma_port_get_link_layer(qp->ibqp.device, qp->port) == IB_LINK_LAYER_ETHERNET;
2978	3006	is_grh = mlx4_ib_ah_grh_present(ah);
2979	3007	if (is_eth) {
2980	3008	enum ib_gid_type gid_type;
..	..	@@ -2988,9 +3016,9 @@
2988	3016	if (err)
2989	3017	return err;
2990	3018	} else {
2991		- err = fill_gid_by_hw_index(ibdev, sqp->qp.port,
2992		- ah->av.ib.gid_index,
2993		- &sgid, &gid_type);
	3019	+ err = fill_gid_by_hw_index(ibdev, qp->port,
	3020	+ ah->av.ib.gid_index, &sgid,
	3021	+ &gid_type);
2994	3022	if (!err) {
2995	3023	is_udp = gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
2996	3024	if (is_udp) {
..	..	@@ -3006,7 +3034,7 @@
3006	3034	}
3007	3035	if (ah->av.eth.vlan != cpu_to_be16(0xffff)) {
3008	3036	vlan = be16_to_cpu(ah->av.eth.vlan) & 0x0fff;
3009		- is_vlan = 1;
	3037	+ is_vlan = true;
3010	3038	}
3011	3039	}
3012	3040	err = ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh,
..	..	@@ -3035,13 +3063,18 @@
3035	3063	* indexes don't necessarily match the hw ones, so
3036	3064	* we must use our own cache
3037	3065	*/
3038		- sqp->ud_header.grh.source_gid.global.subnet_prefix =
3039		- cpu_to_be64(atomic64_read(&(to_mdev(ib_dev)->sriov.
3040		- demux[sqp->qp.port - 1].
3041		- subnet_prefix)));
3042		- sqp->ud_header.grh.source_gid.global.interface_id =
3043		- to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1].
3044		- guid_cache[ah->av.ib.gid_index];
	3066	+ sqp->ud_header.grh.source_gid.global
	3067	+ .subnet_prefix =
	3068	+ cpu_to_be64(atomic64_read(
	3069	+ &(to_mdev(ib_dev)
	3070	+ ->sriov
	3071	+ .demux[qp->port - 1]
	3072	+ .subnet_prefix)));
	3073	+ sqp->ud_header.grh.source_gid.global
	3074	+ .interface_id =
	3075	+ to_mdev(ib_dev)
	3076	+ ->sriov.demux[qp->port - 1]
	3077	+ .guid_cache[ah->av.ib.gid_index];
3045	3078	} else {
3046	3079	sqp->ud_header.grh.source_gid =
3047	3080	ah->ibah.sgid_attr->gid;
..	..	@@ -3073,10 +3106,13 @@
3073	3106	mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
3074	3107
3075	3108	if (!is_eth) {
3076		- mlx->flags \|= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) \|
3077		- (sqp->ud_header.lrh.destination_lid ==
3078		- IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) \|
3079		- (sqp->ud_header.lrh.service_level << 8));
	3109	+ mlx->flags \|=
	3110	+ cpu_to_be32((!qp->ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) \|
	3111	+ (sqp->ud_header.lrh.destination_lid ==
	3112	+ IB_LID_PERMISSIVE ?
	3113	+ MLX4_WQE_MLX_SLR :
	3114	+ 0) \|
	3115	+ (sqp->ud_header.lrh.service_level << 8));
3080	3116	if (ah->av.ib.port_pd & cpu_to_be32(0x80000000))
3081	3117	mlx->flags \|= cpu_to_be32(0x1); /* force loopback */
3082	3118	mlx->rlid = sqp->ud_header.lrh.destination_lid;
..	..	@@ -3122,21 +3158,23 @@
3122	3158	sqp->ud_header.vlan.tag = cpu_to_be16(vlan \| pcp);
3123	3159	}
3124	3160	} else {
3125		- sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 :
3126		- sl_to_vl(to_mdev(ib_dev),
3127		- sqp->ud_header.lrh.service_level,
3128		- sqp->qp.port);
3129		- if (sqp->qp.ibqp.qp_num && sqp->ud_header.lrh.virtual_lane == 15)
	3161	+ sqp->ud_header.lrh.virtual_lane =
	3162	+ !qp->ibqp.qp_num ?
	3163	+ 15 :
	3164	+ sl_to_vl(to_mdev(ib_dev),
	3165	+ sqp->ud_header.lrh.service_level,
	3166	+ qp->port);
	3167	+ if (qp->ibqp.qp_num && sqp->ud_header.lrh.virtual_lane == 15)
3130	3168	return -EINVAL;
3131	3169	if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
3132	3170	sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
3133	3171	}
3134	3172	sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
3135		- if (!sqp->qp.ibqp.qp_num)
3136		- err = ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index,
	3173	+ if (!qp->ibqp.qp_num)
	3174	+ err = ib_get_cached_pkey(ib_dev, qp->port, sqp->pkey_index,
3137	3175	&pkey);
3138	3176	else
3139		- err = ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->pkey_index,
	3177	+ err = ib_get_cached_pkey(ib_dev, qp->port, wr->pkey_index,
3140	3178	&pkey);
3141	3179	if (err)
3142	3180	return err;
..	..	@@ -3146,7 +3184,7 @@
3146	3184	sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
3147	3185	sqp->ud_header.deth.qkey = cpu_to_be32(wr->remote_qkey & 0x80000000 ?
3148	3186	sqp->qkey : wr->remote_qkey);
3149		- sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num);
	3187	+ sqp->ud_header.deth.source_qpn = cpu_to_be32(qp->ibqp.qp_num);
3150	3188
3151	3189	header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
3152	3190
..	..	@@ -3459,24 +3497,24 @@
3459	3497	int nreq;
3460	3498	int err = 0;
3461	3499	unsigned ind;
3462		- int uninitialized_var(size);
3463		- unsigned uninitialized_var(seglen);
	3500	+ int size;
	3501	+ unsigned seglen;
3464	3502	__be32 dummy;
3465	3503	__be32 *lso_wqe;
3466		- __be32 uninitialized_var(lso_hdr_sz);
	3504	+ __be32 lso_hdr_sz;
3467	3505	__be32 blh;
3468	3506	int i;
3469	3507	struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
3470	3508
3471	3509	if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
3472		- struct mlx4_ib_sqp *sqp = to_msqp(qp);
	3510	+ struct mlx4_ib_sqp *sqp = qp->sqp;
3473	3511
3474	3512	if (sqp->roce_v2_gsi) {
3475	3513	struct mlx4_ib_ah *ah = to_mah(ud_wr(wr)->ah);
3476	3514	enum ib_gid_type gid_type;
3477	3515	union ib_gid gid;
3478	3516
3479		- if (!fill_gid_by_hw_index(mdev, sqp->qp.port,
	3517	+ if (!fill_gid_by_hw_index(mdev, qp->port,
3480	3518	ah->av.ib.gid_index,
3481	3519	&gid, &gid_type))
3482	3520	qp = (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
..	..	@@ -3596,8 +3634,8 @@
3596	3634	break;
3597	3635
3598	3636	case MLX4_IB_QPT_TUN_SMI_OWNER:
3599		- err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),
3600		- ctrl, &seglen);
	3637	+ err = build_sriov_qp0_header(qp, ud_wr(wr), ctrl,
	3638	+ &seglen);
3601	3639	if (unlikely(err)) {
3602	3640	*bad_wr = wr;
3603	3641	goto out;
..	..	@@ -3633,8 +3671,8 @@
3633	3671	break;
3634	3672
3635	3673	case MLX4_IB_QPT_PROXY_SMI_OWNER:
3636		- err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),
3637		- ctrl, &seglen);
	3674	+ err = build_sriov_qp0_header(qp, ud_wr(wr), ctrl,
	3675	+ &seglen);
3638	3676	if (unlikely(err)) {
3639	3677	*bad_wr = wr;
3640	3678	goto out;
..	..	@@ -3667,8 +3705,7 @@
3667	3705
3668	3706	case MLX4_IB_QPT_SMI:
3669	3707	case MLX4_IB_QPT_GSI:
3670		- err = build_mlx_header(to_msqp(qp), ud_wr(wr), ctrl,
3671		- &seglen);
	3708	+ err = build_mlx_header(qp, ud_wr(wr), ctrl, &seglen);
3672	3709	if (unlikely(err)) {
3673	3710	*bad_wr = wr;
3674	3711	goto out;
..	..	@@ -3753,12 +3790,6 @@
3753	3790
3754	3791	writel_relaxed(qp->doorbell_qpn,
3755	3792	to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL);
3756		-
3757		- /*
3758		- * Make sure doorbells don't leak out of SQ spinlock
3759		- * and reach the HCA out of order.
3760		- */
3761		- mmiowb();
3762	3793
3763	3794	stamp_send_wqe(qp, ind + qp->sq_spare_wqes - 1);
3764	3795
..	..	@@ -4054,13 +4085,13 @@
4054	4085	struct ib_wq_init_attr *init_attr,
4055	4086	struct ib_udata *udata)
4056	4087	{
4057		- struct mlx4_ib_dev *dev;
4058		- struct ib_qp_init_attr ib_qp_init_attr;
	4088	+ struct mlx4_dev *dev = to_mdev(pd->device)->dev;
	4089	+ struct ib_qp_init_attr ib_qp_init_attr = {};
4059	4090	struct mlx4_ib_qp *qp;
4060	4091	struct mlx4_ib_create_wq ucmd;
4061	4092	int err, required_cmd_sz;
4062	4093
4063		- if (!(udata && pd->uobject))
	4094	+ if (!udata)
4064	4095	return ERR_PTR(-EINVAL);
4065	4096
4066	4097	required_cmd_sz = offsetof(typeof(ucmd), comp_mask) +
..	..	@@ -4080,14 +4111,13 @@
4080	4111	if (udata->outlen)
4081	4112	return ERR_PTR(-EOPNOTSUPP);
4082	4113
4083		- dev = to_mdev(pd->device);
4084		-
4085	4114	if (init_attr->wq_type != IB_WQT_RQ) {
4086	4115	pr_debug("unsupported wq type %d\n", init_attr->wq_type);
4087	4116	return ERR_PTR(-EOPNOTSUPP);
4088	4117	}
4089	4118
4090		- if (init_attr->create_flags & ~IB_WQ_FLAGS_SCATTER_FCS) {
	4119	+ if (init_attr->create_flags & ~IB_WQ_FLAGS_SCATTER_FCS \|\|
	4120	+ !(dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP)) {
4091	4121	pr_debug("unsupported create_flags %u\n",
4092	4122	init_attr->create_flags);
4093	4123	return ERR_PTR(-EOPNOTSUPP);
..	..	@@ -4097,10 +4127,10 @@
4097	4127	if (!qp)
4098	4128	return ERR_PTR(-ENOMEM);
4099	4129
	4130	+ mutex_init(&qp->mutex);
4100	4131	qp->pri.vid = 0xFFFF;
4101	4132	qp->alt.vid = 0xFFFF;
4102	4133
4103		- memset(&ib_qp_init_attr, 0, sizeof(ib_qp_init_attr));
4104	4134	ib_qp_init_attr.qp_context = init_attr->wq_context;
4105	4135	ib_qp_init_attr.qp_type = IB_QPT_RAW_PACKET;
4106	4136	ib_qp_init_attr.cap.max_recv_wr = init_attr->max_wr;
..	..	@@ -4111,8 +4141,7 @@
4111	4141	if (init_attr->create_flags & IB_WQ_FLAGS_SCATTER_FCS)
4112	4142	ib_qp_init_attr.create_flags \|= IB_QP_CREATE_SCATTER_FCS;
4113	4143
4114		- err = create_qp_common(dev, pd, MLX4_IB_RWQ_SRC, &ib_qp_init_attr,
4115		- udata, 0, &qp);
	4144	+ err = create_rq(pd, &ib_qp_init_attr, udata, qp);
4116	4145	if (err) {
4117	4146	kfree(qp);
4118	4147	return ERR_PTR(err);
..	..	@@ -4137,7 +4166,8 @@
4137	4166	}
4138	4167	}
4139	4168
4140		-static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state)
	4169	+static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state,
	4170	+ struct ib_udata *udata)
4141	4171	{
4142	4172	struct mlx4_ib_qp qp = to_mqp((struct ib_qp )ibwq);
4143	4173	enum ib_qp_state qp_cur_state;
..	..	@@ -4161,7 +4191,8 @@
4161	4191	attr_mask = IB_QP_PORT;
4162	4192
4163	4193	err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, &attr,
4164		- attr_mask, IB_QPS_RESET, IB_QPS_INIT);
	4194	+ attr_mask, IB_QPS_RESET, IB_QPS_INIT,
	4195	+ udata);
4165	4196	if (err) {
4166	4197	pr_debug("WQN=0x%06x failed to apply RST->INIT on the HW QP\n",
4167	4198	ibwq->wq_num);
..	..	@@ -4173,12 +4204,13 @@
4173	4204
4174	4205	attr_mask = 0;
4175	4206	err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL, attr_mask,
4176		- qp_cur_state, qp_new_state);
	4207	+ qp_cur_state, qp_new_state, udata);
4177	4208
4178	4209	if (err && (qp_cur_state == IB_QPS_INIT)) {
4179	4210	qp_new_state = IB_QPS_RESET;
4180	4211	if (__mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL,
4181		- attr_mask, IB_QPS_INIT, IB_QPS_RESET)) {
	4212	+ attr_mask, IB_QPS_INIT, IB_QPS_RESET,
	4213	+ udata)) {
4182	4214	pr_warn("WQN=0x%06x failed with reverting HW's resources failure\n",
4183	4215	ibwq->wq_num);
4184	4216	qp_new_state = IB_QPS_INIT;
..	..	@@ -4218,13 +4250,8 @@
4218	4250	if (wq_attr_mask & IB_WQ_FLAGS)
4219	4251	return -EOPNOTSUPP;
4220	4252
4221		- cur_state = wq_attr_mask & IB_WQ_CUR_STATE ? wq_attr->curr_wq_state :
4222		- ibwq->state;
4223		- new_state = wq_attr_mask & IB_WQ_STATE ? wq_attr->wq_state : cur_state;
4224		-
4225		- if (cur_state < IB_WQS_RESET \|\| cur_state > IB_WQS_ERR \|\|
4226		- new_state < IB_WQS_RESET \|\| new_state > IB_WQS_ERR)
4227		- return -EINVAL;
	4253	+ cur_state = wq_attr->curr_wq_state;
	4254	+ new_state = wq_attr->wq_state;
4228	4255
4229	4256	if ((new_state == IB_WQS_RDY) && (cur_state == IB_WQS_ERR))
4230	4257	return -EINVAL;
..	..	@@ -4241,7 +4268,7 @@
4241	4268	* WQ, so we can apply its port on the WQ.
4242	4269	*/
4243	4270	if (qp->rss_usecnt)
4244		- err = _mlx4_ib_modify_wq(ibwq, new_state);
	4271	+ err = _mlx4_ib_modify_wq(ibwq, new_state, udata);
4245	4272
4246	4273	if (!err)
4247	4274	ibwq->state = new_state;
..	..	@@ -4251,7 +4278,7 @@
4251	4278	return err;
4252	4279	}
4253	4280
4254		-int mlx4_ib_destroy_wq(struct ib_wq *ibwq)
	4281	+int mlx4_ib_destroy_wq(struct ib_wq ibwq, struct ib_udata udata)
4255	4282	{
4256	4283	struct mlx4_ib_dev *dev = to_mdev(ibwq->device);
4257	4284	struct mlx4_ib_qp qp = to_mqp((struct ib_qp )ibwq);
..	..	@@ -4259,41 +4286,38 @@
4259	4286	if (qp->counter_index)
4260	4287	mlx4_ib_free_qp_counter(dev, qp);
4261	4288
4262		- destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, 1);
	4289	+ destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, udata);
4263	4290
4264	4291	kfree(qp);
4265		-
4266	4292	return 0;
4267	4293	}
4268	4294
4269		-struct ib_rwq_ind_table
4270		-mlx4_ib_create_rwq_ind_table(struct ib_device device,
4271		- struct ib_rwq_ind_table_init_attr *init_attr,
4272		- struct ib_udata *udata)
	4295	+int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table,
	4296	+ struct ib_rwq_ind_table_init_attr *init_attr,
	4297	+ struct ib_udata *udata)
4273	4298	{
4274		- struct ib_rwq_ind_table *rwq_ind_table;
4275	4299	struct mlx4_ib_create_rwq_ind_tbl_resp resp = {};
4276	4300	unsigned int ind_tbl_size = 1 << init_attr->log_ind_tbl_size;
	4301	+ struct ib_device *device = rwq_ind_table->device;
4277	4302	unsigned int base_wqn;
4278	4303	size_t min_resp_len;
4279		- int i;
4280		- int err;
	4304	+ int i, err = 0;
4281	4305
4282	4306	if (udata->inlen > 0 &&
4283	4307	!ib_is_udata_cleared(udata, 0,
4284	4308	udata->inlen))
4285		- return ERR_PTR(-EOPNOTSUPP);
	4309	+ return -EOPNOTSUPP;
4286	4310
4287	4311	min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
4288	4312	if (udata->outlen && udata->outlen < min_resp_len)
4289		- return ERR_PTR(-EINVAL);
	4313	+ return -EINVAL;
4290	4314
4291	4315	if (ind_tbl_size >
4292	4316	device->attrs.rss_caps.max_rwq_indirection_table_size) {
4293	4317	pr_debug("log_ind_tbl_size = %d is bigger than supported = %d\n",
4294	4318	ind_tbl_size,
4295	4319	device->attrs.rss_caps.max_rwq_indirection_table_size);
4296		- return ERR_PTR(-EINVAL);
	4320	+ return -EINVAL;
4297	4321	}
4298	4322
4299	4323	base_wqn = init_attr->ind_tbl[0]->wq_num;
..	..	@@ -4301,39 +4325,23 @@
4301	4325	if (base_wqn % ind_tbl_size) {
4302	4326	pr_debug("WQN=0x%x isn't aligned with indirection table size\n",
4303	4327	base_wqn);
4304		- return ERR_PTR(-EINVAL);
	4328	+ return -EINVAL;
4305	4329	}
4306	4330
4307	4331	for (i = 1; i < ind_tbl_size; i++) {
4308	4332	if (++base_wqn != init_attr->ind_tbl[i]->wq_num) {
4309	4333	pr_debug("indirection table's WQNs aren't consecutive\n");
4310		- return ERR_PTR(-EINVAL);
	4334	+ return -EINVAL;
4311	4335	}
4312	4336	}
4313		-
4314		- rwq_ind_table = kzalloc(sizeof(*rwq_ind_table), GFP_KERNEL);
4315		- if (!rwq_ind_table)
4316		- return ERR_PTR(-ENOMEM);
4317	4337
4318	4338	if (udata->outlen) {
4319	4339	resp.response_length = offsetof(typeof(resp), response_length) +
4320	4340	sizeof(resp.response_length);
4321	4341	err = ib_copy_to_udata(udata, &resp, resp.response_length);
4322		- if (err)
4323		- goto err;
4324	4342	}
4325	4343
4326		- return rwq_ind_table;
4327		-
4328		-err:
4329		- kfree(rwq_ind_table);
4330		- return ERR_PTR(err);
4331		-}
4332		-
4333		-int mlx4_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
4334		-{
4335		- kfree(ib_rwq_ind_tbl);
4336		- return 0;
	4344	+ return err;
4337	4345	}
4338	4346
4339	4347	struct mlx4_ib_drain_cqe {