~hc/RK356X_SDK_RELEASE.git

..	..	@@ -41,6 +41,7 @@
41	41	#include <rdma/ib_pack.h>
42	42	#include <rdma/ib_addr.h>
43	43	#include <rdma/ib_mad.h>
	44	+#include <rdma/uverbs_ioctl.h>
44	45
45	46	#include <linux/mlx4/driver.h>
46	47	#include <linux/mlx4/qp.h>
..	..	@@ -52,7 +53,8 @@
52	53	struct mlx4_ib_cq *recv_cq);
53	54	static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq,
54	55	struct mlx4_ib_cq *recv_cq);
55		-static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state);
	56	+static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state,
	57	+ struct ib_udata *udata);
56	58
57	59	enum {
58	60	MLX4_IB_ACK_REQ_FREQ = 8,
..	..	@@ -63,27 +65,6 @@
63	65	MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
64	66	MLX4_IB_LINK_TYPE_IB = 0,
65	67	MLX4_IB_LINK_TYPE_ETH = 1
66		-};
67		-
68		-enum {
69		- /*
70		- * Largest possible UD header: send with GRH and immediate
71		- * data plus 18 bytes for an Ethernet header with VLAN/802.1Q
72		- * tag. (LRH would only use 8 bytes, so Ethernet is the
73		- * biggest case)
74		- */
75		- MLX4_IB_UD_HEADER_SIZE = 82,
76		- MLX4_IB_LSO_HEADER_SPARE = 128,
77		-};
78		-
79		-struct mlx4_ib_sqp {
80		- struct mlx4_ib_qp qp;
81		- int pkey_index;
82		- u32 qkey;
83		- u32 send_psn;
84		- struct ib_ud_header ud_header;
85		- u8 header_buf[MLX4_IB_UD_HEADER_SIZE];
86		- struct ib_qp *roce_v2_gsi;
87	68	};
88	69
89	70	enum {
..	..	@@ -120,11 +101,6 @@
120	101	MLX4_IB_QP_SRC = 0,
121	102	MLX4_IB_RWQ_SRC = 1,
122	103	};
123		-
124		-static struct mlx4_ib_sqp to_msqp(struct mlx4_ib_qp mqp)
125		-{
126		- return container_of(mqp, struct mlx4_ib_sqp, qp);
127		-}
128	104
129	105	static int is_tunnel_qp(struct mlx4_ib_dev dev, struct mlx4_ib_qp qp)
130	106	{
..	..	@@ -323,7 +299,7 @@
323	299	}
324	300
325	301	static int set_rq_size(struct mlx4_ib_dev dev, struct ib_qp_cap cap,
326		- int is_user, int has_rq, struct mlx4_ib_qp *qp,
	302	+ bool is_user, bool has_rq, struct mlx4_ib_qp *qp,
327	303	u32 inl_recv_sz)
328	304	{
329	305	/* Sanity check RQ size before proceeding */
..	..	@@ -401,7 +377,7 @@
401	377	* We need to leave 2 KB + 1 WR of headroom in the SQ to
402	378	* allow HW to prefetch.
403	379	*/
404		- qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + 1;
	380	+ qp->sq_spare_wqes = MLX4_IB_SQ_HEADROOM(qp->sq.wqe_shift);
405	381	qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr +
406	382	qp->sq_spare_wqes);
407	383
..	..	@@ -504,10 +480,10 @@
504	480	kfree(qp->sqp_proxy_rcv);
505	481	}
506	482
507		-static int qp_has_rq(struct ib_qp_init_attr *attr)
	483	+static bool qp_has_rq(struct ib_qp_init_attr *attr)
508	484	{
509	485	if (attr->qp_type == IB_QPT_XRC_INI \|\| attr->qp_type == IB_QPT_XRC_TGT)
510		- return 0;
	486	+ return false;
511	487
512	488	return !attr->srq;
513	489	}
..	..	@@ -654,8 +630,6 @@
654	630	if (err)
655	631	goto err_qpn;
656	632
657		- mutex_init(&qp->mutex);
658		-
659	633	INIT_LIST_HEAD(&qp->gid_list);
660	634	INIT_LIST_HEAD(&qp->steering_rules);
661	635
..	..	@@ -694,80 +668,72 @@
694	668	return err;
695	669	}
696	670
697		-static struct ib_qp _mlx4_ib_create_qp_rss(struct ib_pd pd,
698		- struct ib_qp_init_attr *init_attr,
699		- struct ib_udata *udata)
	671	+static int _mlx4_ib_create_qp_rss(struct ib_pd pd, struct mlx4_ib_qp qp,
	672	+ struct ib_qp_init_attr *init_attr,
	673	+ struct ib_udata *udata)
700	674	{
701		- struct mlx4_ib_qp *qp;
702	675	struct mlx4_ib_create_qp_rss ucmd = {};
703	676	size_t required_cmd_sz;
704	677	int err;
705	678
706	679	if (!udata) {
707	680	pr_debug("RSS QP with NULL udata\n");
708		- return ERR_PTR(-EINVAL);
	681	+ return -EINVAL;
709	682	}
710	683
711	684	if (udata->outlen)
712		- return ERR_PTR(-EOPNOTSUPP);
	685	+ return -EOPNOTSUPP;
713	686
714	687	required_cmd_sz = offsetof(typeof(ucmd), reserved1) +
715	688	sizeof(ucmd.reserved1);
716	689	if (udata->inlen < required_cmd_sz) {
717	690	pr_debug("invalid inlen\n");
718		- return ERR_PTR(-EINVAL);
	691	+ return -EINVAL;
719	692	}
720	693
721	694	if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
722	695	pr_debug("copy failed\n");
723		- return ERR_PTR(-EFAULT);
	696	+ return -EFAULT;
724	697	}
725	698
726	699	if (memchr_inv(ucmd.reserved, 0, sizeof(ucmd.reserved)))
727		- return ERR_PTR(-EOPNOTSUPP);
	700	+ return -EOPNOTSUPP;
728	701
729	702	if (ucmd.comp_mask \|\| ucmd.reserved1)
730		- return ERR_PTR(-EOPNOTSUPP);
	703	+ return -EOPNOTSUPP;
731	704
732	705	if (udata->inlen > sizeof(ucmd) &&
733	706	!ib_is_udata_cleared(udata, sizeof(ucmd),
734	707	udata->inlen - sizeof(ucmd))) {
735	708	pr_debug("inlen is not supported\n");
736		- return ERR_PTR(-EOPNOTSUPP);
	709	+ return -EOPNOTSUPP;
737	710	}
738	711
739	712	if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
740	713	pr_debug("RSS QP with unsupported QP type %d\n",
741	714	init_attr->qp_type);
742		- return ERR_PTR(-EOPNOTSUPP);
	715	+ return -EOPNOTSUPP;
743	716	}
744	717
745	718	if (init_attr->create_flags) {
746	719	pr_debug("RSS QP doesn't support create flags\n");
747		- return ERR_PTR(-EOPNOTSUPP);
	720	+ return -EOPNOTSUPP;
748	721	}
749	722
750	723	if (init_attr->send_cq \|\| init_attr->cap.max_send_wr) {
751	724	pr_debug("RSS QP with unsupported send attributes\n");
752		- return ERR_PTR(-EOPNOTSUPP);
	725	+ return -EOPNOTSUPP;
753	726	}
754		-
755		- qp = kzalloc(sizeof(*qp), GFP_KERNEL);
756		- if (!qp)
757		- return ERR_PTR(-ENOMEM);
758	727
759	728	qp->pri.vid = 0xFFFF;
760	729	qp->alt.vid = 0xFFFF;
761	730
762	731	err = create_qp_rss(to_mdev(pd->device), init_attr, &ucmd, qp);
763		- if (err) {
764		- kfree(qp);
765		- return ERR_PTR(err);
766		- }
	732	+ if (err)
	733	+ return err;
767	734
768	735	qp->ibqp.qp_num = qp->mqp.qpn;
769		-
770		- return &qp->ibqp;
	736	+ return 0;
771	737	}
772	738
773	739	/*
..	..	@@ -847,26 +813,154 @@
847	813	* reused for further WQN allocations.
848	814	* The next created WQ will allocate a new range.
849	815	*/
850		- range->dirty = 1;
	816	+ range->dirty = true;
851	817	}
852	818
853	819	mutex_unlock(&context->wqn_ranges_mutex);
854	820	}
855	821
856		-static int create_qp_common(struct mlx4_ib_dev dev, struct ib_pd pd,
857		- enum mlx4_ib_source_type src,
858		- struct ib_qp_init_attr *init_attr,
859		- struct ib_udata *udata, int sqpn,
860		- struct mlx4_ib_qp **caller_qp)
	822	+static int create_rq(struct ib_pd pd, struct ib_qp_init_attr init_attr,
	823	+ struct ib_udata udata, struct mlx4_ib_qp qp)
861	824	{
	825	+ struct mlx4_ib_dev *dev = to_mdev(pd->device);
862	826	int qpn;
863	827	int err;
864		- struct mlx4_ib_sqp *sqp = NULL;
865		- struct mlx4_ib_qp *qp;
	828	+ struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
	829	+ udata, struct mlx4_ib_ucontext, ibucontext);
	830	+ struct mlx4_ib_cq *mcq;
	831	+ unsigned long flags;
	832	+ int range_size;
	833	+ struct mlx4_ib_create_wq wq;
	834	+ size_t copy_len;
	835	+ int shift;
	836	+ int n;
	837	+
	838	+ qp->mlx4_ib_qp_type = MLX4_IB_QPT_RAW_PACKET;
	839	+
	840	+ spin_lock_init(&qp->sq.lock);
	841	+ spin_lock_init(&qp->rq.lock);
	842	+ INIT_LIST_HEAD(&qp->gid_list);
	843	+ INIT_LIST_HEAD(&qp->steering_rules);
	844	+
	845	+ qp->state = IB_QPS_RESET;
	846	+
	847	+ copy_len = min(sizeof(struct mlx4_ib_create_wq), udata->inlen);
	848	+
	849	+ if (ib_copy_from_udata(&wq, udata, copy_len)) {
	850	+ err = -EFAULT;
	851	+ goto err;
	852	+ }
	853	+
	854	+ if (wq.comp_mask \|\| wq.reserved[0] \|\| wq.reserved[1] \|\|
	855	+ wq.reserved[2]) {
	856	+ pr_debug("user command isn't supported\n");
	857	+ err = -EOPNOTSUPP;
	858	+ goto err;
	859	+ }
	860	+
	861	+ if (wq.log_range_size > ilog2(dev->dev->caps.max_rss_tbl_sz)) {
	862	+ pr_debug("WQN range size must be equal or smaller than %d\n",
	863	+ dev->dev->caps.max_rss_tbl_sz);
	864	+ err = -EOPNOTSUPP;
	865	+ goto err;
	866	+ }
	867	+ range_size = 1 << wq.log_range_size;
	868	+
	869	+ if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS)
	870	+ qp->flags \|= MLX4_IB_QP_SCATTER_FCS;
	871	+
	872	+ err = set_rq_size(dev, &init_attr->cap, true, true, qp, qp->inl_recv_sz);
	873	+ if (err)
	874	+ goto err;
	875	+
	876	+ qp->sq_no_prefetch = 1;
	877	+ qp->sq.wqe_cnt = 1;
	878	+ qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
	879	+ qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
	880	+ (qp->sq.wqe_cnt << qp->sq.wqe_shift);
	881	+
	882	+ qp->umem = ib_umem_get(pd->device, wq.buf_addr, qp->buf_size, 0);
	883	+ if (IS_ERR(qp->umem)) {
	884	+ err = PTR_ERR(qp->umem);
	885	+ goto err;
	886	+ }
	887	+
	888	+ shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n);
	889	+ err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt);
	890	+
	891	+ if (err)
	892	+ goto err_buf;
	893	+
	894	+ err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
	895	+ if (err)
	896	+ goto err_mtt;
	897	+
	898	+ err = mlx4_ib_db_map_user(udata, wq.db_addr, &qp->db);
	899	+ if (err)
	900	+ goto err_mtt;
	901	+ qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
	902	+
	903	+ err = mlx4_ib_alloc_wqn(context, qp, range_size, &qpn);
	904	+ if (err)
	905	+ goto err_wrid;
	906	+
	907	+ err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
	908	+ if (err)
	909	+ goto err_qpn;
	910	+
	911	+ /*
	912	+ * Hardware wants QPN written in big-endian order (after
	913	+ * shifting) for send doorbell. Precompute this value to save
	914	+ * a little bit when posting sends.
	915	+ */
	916	+ qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
	917	+
	918	+ qp->mqp.event = mlx4_ib_wq_event;
	919	+
	920	+ spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
	921	+ mlx4_ib_lock_cqs(to_mcq(init_attr->send_cq),
	922	+ to_mcq(init_attr->recv_cq));
	923	+ /* Maintain device to QPs access, needed for further handling
	924	+ * via reset flow
	925	+ */
	926	+ list_add_tail(&qp->qps_list, &dev->qp_list);
	927	+ /* Maintain CQ to QPs access, needed for further handling
	928	+ * via reset flow
	929	+ */
	930	+ mcq = to_mcq(init_attr->send_cq);
	931	+ list_add_tail(&qp->cq_send_list, &mcq->send_qp_list);
	932	+ mcq = to_mcq(init_attr->recv_cq);
	933	+ list_add_tail(&qp->cq_recv_list, &mcq->recv_qp_list);
	934	+ mlx4_ib_unlock_cqs(to_mcq(init_attr->send_cq),
	935	+ to_mcq(init_attr->recv_cq));
	936	+ spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
	937	+ return 0;
	938	+
	939	+err_qpn:
	940	+ mlx4_ib_release_wqn(context, qp, 0);
	941	+err_wrid:
	942	+ mlx4_ib_db_unmap_user(context, &qp->db);
	943	+
	944	+err_mtt:
	945	+ mlx4_mtt_cleanup(dev->dev, &qp->mtt);
	946	+err_buf:
	947	+ ib_umem_release(qp->umem);
	948	+err:
	949	+ return err;
	950	+}
	951	+
	952	+static int create_qp_common(struct ib_pd pd, struct ib_qp_init_attr init_attr,
	953	+ struct ib_udata *udata, int sqpn,
	954	+ struct mlx4_ib_qp *qp)
	955	+{
	956	+ struct mlx4_ib_dev *dev = to_mdev(pd->device);
	957	+ int qpn;
	958	+ int err;
	959	+ struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
	960	+ udata, struct mlx4_ib_ucontext, ibucontext);
866	961	enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type;
867	962	struct mlx4_ib_cq *mcq;
868	963	unsigned long flags;
869		- int range_size = 0;
870	964
871	965	/* When tunneling special qps, we use a plain UD qp */
872	966	if (sqpn) {
..	..	@@ -909,76 +1003,41 @@
909	1003	sqpn = qpn;
910	1004	}
911	1005
912		- if (!*caller_qp) {
913		- if (qp_type == MLX4_IB_QPT_SMI \|\| qp_type == MLX4_IB_QPT_GSI \|\|
914		- (qp_type & (MLX4_IB_QPT_PROXY_SMI \| MLX4_IB_QPT_PROXY_SMI_OWNER \|
915		- MLX4_IB_QPT_PROXY_GSI \| MLX4_IB_QPT_TUN_SMI_OWNER))) {
916		- sqp = kzalloc(sizeof(struct mlx4_ib_sqp), GFP_KERNEL);
917		- if (!sqp)
918		- return -ENOMEM;
919		- qp = &sqp->qp;
920		- qp->pri.vid = 0xFFFF;
921		- qp->alt.vid = 0xFFFF;
922		- } else {
923		- qp = kzalloc(sizeof(struct mlx4_ib_qp), GFP_KERNEL);
924		- if (!qp)
925		- return -ENOMEM;
926		- qp->pri.vid = 0xFFFF;
927		- qp->alt.vid = 0xFFFF;
928		- }
929		- } else
930		- qp = *caller_qp;
	1006	+ if (init_attr->qp_type == IB_QPT_SMI \|\|
	1007	+ init_attr->qp_type == IB_QPT_GSI \|\| qp_type == MLX4_IB_QPT_SMI \|\|
	1008	+ qp_type == MLX4_IB_QPT_GSI \|\|
	1009	+ (qp_type & (MLX4_IB_QPT_PROXY_SMI \| MLX4_IB_QPT_PROXY_SMI_OWNER \|
	1010	+ MLX4_IB_QPT_PROXY_GSI \| MLX4_IB_QPT_TUN_SMI_OWNER))) {
	1011	+ qp->sqp = kzalloc(sizeof(struct mlx4_ib_sqp), GFP_KERNEL);
	1012	+ if (!qp->sqp)
	1013	+ return -ENOMEM;
	1014	+ }
931	1015
932	1016	qp->mlx4_ib_qp_type = qp_type;
933	1017
934		- mutex_init(&qp->mutex);
935	1018	spin_lock_init(&qp->sq.lock);
936	1019	spin_lock_init(&qp->rq.lock);
937	1020	INIT_LIST_HEAD(&qp->gid_list);
938	1021	INIT_LIST_HEAD(&qp->steering_rules);
939	1022
940		- qp->state = IB_QPS_RESET;
	1023	+ qp->state = IB_QPS_RESET;
941	1024	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
942	1025	qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
943	1026
944		-
945		- if (pd->uobject) {
946		- union {
947		- struct mlx4_ib_create_qp qp;
948		- struct mlx4_ib_create_wq wq;
949		- } ucmd;
	1027	+ if (udata) {
	1028	+ struct mlx4_ib_create_qp ucmd;
950	1029	size_t copy_len;
951	1030	int shift;
952	1031	int n;
953	1032
954		- copy_len = (src == MLX4_IB_QP_SRC) ?
955		- sizeof(struct mlx4_ib_create_qp) :
956		- min(sizeof(struct mlx4_ib_create_wq), udata->inlen);
	1033	+ copy_len = sizeof(struct mlx4_ib_create_qp);
957	1034
958	1035	if (ib_copy_from_udata(&ucmd, udata, copy_len)) {
959	1036	err = -EFAULT;
960	1037	goto err;
961	1038	}
962	1039
963		- if (src == MLX4_IB_RWQ_SRC) {
964		- if (ucmd.wq.comp_mask \|\| ucmd.wq.reserved[0] \|\|
965		- ucmd.wq.reserved[1] \|\| ucmd.wq.reserved[2]) {
966		- pr_debug("user command isn't supported\n");
967		- err = -EOPNOTSUPP;
968		- goto err;
969		- }
970		-
971		- if (ucmd.wq.log_range_size >
972		- ilog2(dev->dev->caps.max_rss_tbl_sz)) {
973		- pr_debug("WQN range size must be equal or smaller than %d\n",
974		- dev->dev->caps.max_rss_tbl_sz);
975		- err = -EOPNOTSUPP;
976		- goto err;
977		- }
978		- range_size = 1 << ucmd.wq.log_range_size;
979		- } else {
980		- qp->inl_recv_sz = ucmd.qp.inl_recv_sz;
981		- }
	1040	+ qp->inl_recv_sz = ucmd.inl_recv_sz;
982	1041
983	1042	if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS) {
984	1043	if (!(dev->dev->caps.flags &
..	..	@@ -991,39 +1050,24 @@
991	1050	qp->flags \|= MLX4_IB_QP_SCATTER_FCS;
992	1051	}
993	1052
994		- err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
	1053	+ err = set_rq_size(dev, &init_attr->cap, udata,
995	1054	qp_has_rq(init_attr), qp, qp->inl_recv_sz);
996	1055	if (err)
997	1056	goto err;
998	1057
999		- if (src == MLX4_IB_QP_SRC) {
1000		- qp->sq_no_prefetch = ucmd.qp.sq_no_prefetch;
	1058	+ qp->sq_no_prefetch = ucmd.sq_no_prefetch;
1001	1059
1002		- err = set_user_sq_size(dev, qp,
1003		- (struct mlx4_ib_create_qp *)
1004		- &ucmd);
1005		- if (err)
1006		- goto err;
1007		- } else {
1008		- qp->sq_no_prefetch = 1;
1009		- qp->sq.wqe_cnt = 1;
1010		- qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
1011		- /* Allocated buffer expects to have at least that SQ
1012		- * size.
1013		- */
1014		- qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
1015		- (qp->sq.wqe_cnt << qp->sq.wqe_shift);
1016		- }
	1060	+ err = set_user_sq_size(dev, qp, &ucmd);
	1061	+ if (err)
	1062	+ goto err;
1017	1063
1018		- qp->umem = ib_umem_get(pd->uobject->context,
1019		- (src == MLX4_IB_QP_SRC) ? ucmd.qp.buf_addr :
1020		- ucmd.wq.buf_addr, qp->buf_size, 0, 0);
	1064	+ qp->umem =
	1065	+ ib_umem_get(pd->device, ucmd.buf_addr, qp->buf_size, 0);
1021	1066	if (IS_ERR(qp->umem)) {
1022	1067	err = PTR_ERR(qp->umem);
1023	1068	goto err;
1024	1069	}
1025	1070
1026		- n = ib_umem_page_count(qp->umem);
1027	1071	shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n);
1028	1072	err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt);
1029	1073
..	..	@@ -1035,15 +1079,13 @@
1035	1079	goto err_mtt;
1036	1080
1037	1081	if (qp_has_rq(init_attr)) {
1038		- err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
1039		- (src == MLX4_IB_QP_SRC) ? ucmd.qp.db_addr :
1040		- ucmd.wq.db_addr, &qp->db);
	1082	+ err = mlx4_ib_db_map_user(udata, ucmd.db_addr, &qp->db);
1041	1083	if (err)
1042	1084	goto err_mtt;
1043	1085	}
1044	1086	qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
1045	1087	} else {
1046		- err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
	1088	+ err = set_rq_size(dev, &init_attr->cap, udata,
1047	1089	qp_has_rq(init_attr), qp, 0);
1048	1090	if (err)
1049	1091	goto err;
..	..	@@ -1109,11 +1151,6 @@
1109	1151	goto err_wrid;
1110	1152	}
1111	1153	}
1112		- } else if (src == MLX4_IB_RWQ_SRC) {
1113		- err = mlx4_ib_alloc_wqn(to_mucontext(pd->uobject->context), qp,
1114		- range_size, &qpn);
1115		- if (err)
1116		- goto err_wrid;
1117	1154	} else {
1118	1155	/* Raw packet QPNs may not have bits 6,7 set in their qp_num;
1119	1156	* otherwise, the WQE BlueFlame setup flow wrongly causes
..	..	@@ -1152,11 +1189,7 @@
1152	1189	*/
1153	1190	qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
1154	1191
1155		- qp->mqp.event = (src == MLX4_IB_QP_SRC) ? mlx4_ib_qp_event :
1156		- mlx4_ib_wq_event;
1157		-
1158		- if (!*caller_qp)
1159		- *caller_qp = qp;
	1192	+ qp->mqp.event = mlx4_ib_qp_event;
1160	1193
1161	1194	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1162	1195	mlx4_ib_lock_cqs(to_mcq(init_attr->send_cq),
..	..	@@ -1181,9 +1214,6 @@
1181	1214	if (!sqpn) {
1182	1215	if (qp->flags & MLX4_IB_QP_NETIF)
1183	1216	mlx4_ib_steer_qp_free(dev, qpn, 1);
1184		- else if (src == MLX4_IB_RWQ_SRC)
1185		- mlx4_ib_release_wqn(to_mucontext(pd->uobject->context),
1186		- qp, 0);
1187	1217	else
1188	1218	mlx4_qp_release_range(dev->dev, qpn, 1);
1189	1219	}
..	..	@@ -1191,9 +1221,9 @@
1191	1221	if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
1192	1222	free_proxy_bufs(pd->device, qp);
1193	1223	err_wrid:
1194		- if (pd->uobject) {
	1224	+ if (udata) {
1195	1225	if (qp_has_rq(init_attr))
1196		- mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db);
	1226	+ mlx4_ib_db_unmap_user(context, &qp->db);
1197	1227	} else {
1198	1228	kvfree(qp->sq.wrid);
1199	1229	kvfree(qp->rq.wrid);
..	..	@@ -1203,20 +1233,16 @@
1203	1233	mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1204	1234
1205	1235	err_buf:
1206		- if (pd->uobject)
1207		- ib_umem_release(qp->umem);
1208		- else
	1236	+ if (!qp->umem)
1209	1237	mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
	1238	+ ib_umem_release(qp->umem);
1210	1239
1211	1240	err_db:
1212		- if (!pd->uobject && qp_has_rq(init_attr))
	1241	+ if (!udata && qp_has_rq(init_attr))
1213	1242	mlx4_db_free(dev->dev, &qp->db);
1214	1243
1215	1244	err:
1216		- if (sqp)
1217		- kfree(sqp);
1218		- else if (!*caller_qp)
1219		- kfree(qp);
	1245	+ kfree(qp->sqp);
1220	1246	return err;
1221	1247	}
1222	1248
..	..	@@ -1330,11 +1356,11 @@
1330	1356	mlx4_qp_free(dev->dev, &qp->mqp);
1331	1357	mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1332	1358	del_gid_entries(qp);
1333		- kfree(qp->rss_ctx);
1334	1359	}
1335	1360
1336	1361	static void destroy_qp_common(struct mlx4_ib_dev dev, struct mlx4_ib_qp qp,
1337		- enum mlx4_ib_source_type src, int is_user)
	1362	+ enum mlx4_ib_source_type src,
	1363	+ struct ib_udata *udata)
1338	1364	{
1339	1365	struct mlx4_ib_cq send_cq, recv_cq;
1340	1366	unsigned long flags;
..	..	@@ -1376,7 +1402,7 @@
1376	1402	list_del(&qp->qps_list);
1377	1403	list_del(&qp->cq_send_list);
1378	1404	list_del(&qp->cq_recv_list);
1379		- if (!is_user) {
	1405	+ if (!udata) {
1380	1406	__mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
1381	1407	qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
1382	1408	if (send_cq != recv_cq)
..	..	@@ -1394,22 +1420,28 @@
1394	1420	if (qp->flags & MLX4_IB_QP_NETIF)
1395	1421	mlx4_ib_steer_qp_free(dev, qp->mqp.qpn, 1);
1396	1422	else if (src == MLX4_IB_RWQ_SRC)
1397		- mlx4_ib_release_wqn(to_mucontext(
1398		- qp->ibwq.uobject->context), qp, 1);
	1423	+ mlx4_ib_release_wqn(
	1424	+ rdma_udata_to_drv_context(
	1425	+ udata,
	1426	+ struct mlx4_ib_ucontext,
	1427	+ ibucontext),
	1428	+ qp, 1);
1399	1429	else
1400	1430	mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1401	1431	}
1402	1432
1403	1433	mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1404	1434
1405		- if (is_user) {
	1435	+ if (udata) {
1406	1436	if (qp->rq.wqe_cnt) {
1407		- struct mlx4_ib_ucontext *mcontext = !src ?
1408		- to_mucontext(qp->ibqp.uobject->context) :
1409		- to_mucontext(qp->ibwq.uobject->context);
	1437	+ struct mlx4_ib_ucontext *mcontext =
	1438	+ rdma_udata_to_drv_context(
	1439	+ udata,
	1440	+ struct mlx4_ib_ucontext,
	1441	+ ibucontext);
	1442	+
1410	1443	mlx4_ib_db_unmap_user(mcontext, &qp->db);
1411	1444	}
1412		- ib_umem_release(qp->umem);
1413	1445	} else {
1414	1446	kvfree(qp->sq.wrid);
1415	1447	kvfree(qp->rq.wrid);
..	..	@@ -1420,6 +1452,7 @@
1420	1452	if (qp->rq.wqe_cnt)
1421	1453	mlx4_db_free(dev->dev, &qp->db);
1422	1454	}
	1455	+ ib_umem_release(qp->umem);
1423	1456
1424	1457	del_gid_entries(qp);
1425	1458	}
..	..	@@ -1441,17 +1474,16 @@
1441	1474	return dev->dev->caps.spec_qps[attr->port_num - 1].qp1_proxy;
1442	1475	}
1443	1476
1444		-static struct ib_qp _mlx4_ib_create_qp(struct ib_pd pd,
1445		- struct ib_qp_init_attr *init_attr,
1446		- struct ib_udata *udata)
	1477	+static int _mlx4_ib_create_qp(struct ib_pd pd, struct mlx4_ib_qp qp,
	1478	+ struct ib_qp_init_attr *init_attr,
	1479	+ struct ib_udata *udata)
1447	1480	{
1448		- struct mlx4_ib_qp *qp = NULL;
1449	1481	int err;
1450	1482	int sup_u_create_flags = MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1451	1483	u16 xrcdn = 0;
1452	1484
1453	1485	if (init_attr->rwq_ind_tbl)
1454		- return _mlx4_ib_create_qp_rss(pd, init_attr, udata);
	1486	+ return _mlx4_ib_create_qp_rss(pd, qp, init_attr, udata);
1455	1487
1456	1488	/*
1457	1489	* We only support LSO, vendor flag1, and multicast loopback blocking,
..	..	@@ -1463,16 +1495,16 @@
1463	1495	MLX4_IB_SRIOV_SQP \|
1464	1496	MLX4_IB_QP_NETIF \|
1465	1497	MLX4_IB_QP_CREATE_ROCE_V2_GSI))
1466		- return ERR_PTR(-EINVAL);
	1498	+ return -EINVAL;
1467	1499
1468	1500	if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
1469	1501	if (init_attr->qp_type != IB_QPT_UD)
1470		- return ERR_PTR(-EINVAL);
	1502	+ return -EINVAL;
1471	1503	}
1472	1504
1473	1505	if (init_attr->create_flags) {
1474	1506	if (udata && init_attr->create_flags & ~(sup_u_create_flags))
1475		- return ERR_PTR(-EINVAL);
	1507	+ return -EINVAL;
1476	1508
1477	1509	if ((init_attr->create_flags & ~(MLX4_IB_SRIOV_SQP \|
1478	1510	MLX4_IB_QP_CREATE_ROCE_V2_GSI \|
..	..	@@ -1482,7 +1514,7 @@
1482	1514	init_attr->qp_type > IB_QPT_GSI) \|\|
1483	1515	(init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI &&
1484	1516	init_attr->qp_type != IB_QPT_GSI))
1485		- return ERR_PTR(-EINVAL);
	1517	+ return -EINVAL;
1486	1518	}
1487	1519
1488	1520	switch (init_attr->qp_type) {
..	..	@@ -1490,58 +1522,46 @@
1490	1522	pd = to_mxrcd(init_attr->xrcd)->pd;
1491	1523	xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
1492	1524	init_attr->send_cq = to_mxrcd(init_attr->xrcd)->cq;
1493		- /* fall through */
	1525	+ fallthrough;
1494	1526	case IB_QPT_XRC_INI:
1495	1527	if (!(to_mdev(pd->device)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
1496		- return ERR_PTR(-ENOSYS);
	1528	+ return -ENOSYS;
1497	1529	init_attr->recv_cq = init_attr->send_cq;
1498		- /* fall through */
	1530	+ fallthrough;
1499	1531	case IB_QPT_RC:
1500	1532	case IB_QPT_UC:
1501	1533	case IB_QPT_RAW_PACKET:
1502		- qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1503		- if (!qp)
1504		- return ERR_PTR(-ENOMEM);
	1534	+ case IB_QPT_UD:
1505	1535	qp->pri.vid = 0xFFFF;
1506	1536	qp->alt.vid = 0xFFFF;
1507		- /* fall through */
1508		- case IB_QPT_UD:
1509		- {
1510		- err = create_qp_common(to_mdev(pd->device), pd, MLX4_IB_QP_SRC,
1511		- init_attr, udata, 0, &qp);
1512		- if (err) {
1513		- kfree(qp);
1514		- return ERR_PTR(err);
1515		- }
	1537	+ err = create_qp_common(pd, init_attr, udata, 0, qp);
	1538	+ if (err)
	1539	+ return err;
1516	1540
1517	1541	qp->ibqp.qp_num = qp->mqp.qpn;
1518	1542	qp->xrcdn = xrcdn;
1519		-
1520	1543	break;
1521		- }
1522	1544	case IB_QPT_SMI:
1523	1545	case IB_QPT_GSI:
1524	1546	{
1525	1547	int sqpn;
1526	1548
1527		- /* Userspace is not allowed to create special QPs: */
1528		- if (udata)
1529		- return ERR_PTR(-EINVAL);
1530	1549	if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) {
1531	1550	int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev,
1532	1551	1, 1, &sqpn, 0,
1533	1552	MLX4_RES_USAGE_DRIVER);
1534	1553
1535	1554	if (res)
1536		- return ERR_PTR(res);
	1555	+ return res;
1537	1556	} else {
1538	1557	sqpn = get_sqp_num(to_mdev(pd->device), init_attr);
1539	1558	}
1540	1559
1541		- err = create_qp_common(to_mdev(pd->device), pd, MLX4_IB_QP_SRC,
1542		- init_attr, udata, sqpn, &qp);
	1560	+ qp->pri.vid = 0xFFFF;
	1561	+ qp->alt.vid = 0xFFFF;
	1562	+ err = create_qp_common(pd, init_attr, udata, sqpn, qp);
1543	1563	if (err)
1544		- return ERR_PTR(err);
	1564	+ return err;
1545	1565
1546	1566	qp->port = init_attr->port_num;
1547	1567	qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 :
..	..	@@ -1550,25 +1570,33 @@
1550	1570	}
1551	1571	default:
1552	1572	/* Don't support raw QPs */
1553		- return ERR_PTR(-EINVAL);
	1573	+ return -EOPNOTSUPP;
1554	1574	}
1555		-
1556		- return &qp->ibqp;
	1575	+ return 0;
1557	1576	}
1558	1577
1559	1578	struct ib_qp mlx4_ib_create_qp(struct ib_pd pd,
1560	1579	struct ib_qp_init_attr *init_attr,
1561	1580	struct ib_udata *udata) {
1562	1581	struct ib_device *device = pd ? pd->device : init_attr->xrcd->device;
1563		- struct ib_qp *ibqp;
1564	1582	struct mlx4_ib_dev *dev = to_mdev(device);
	1583	+ struct mlx4_ib_qp *qp;
	1584	+ int ret;
1565	1585
1566		- ibqp = _mlx4_ib_create_qp(pd, init_attr, udata);
	1586	+ qp = kzalloc(sizeof(*qp), GFP_KERNEL);
	1587	+ if (!qp)
	1588	+ return ERR_PTR(-ENOMEM);
1567	1589
1568		- if (!IS_ERR(ibqp) &&
1569		- (init_attr->qp_type == IB_QPT_GSI) &&
	1590	+ mutex_init(&qp->mutex);
	1591	+ ret = _mlx4_ib_create_qp(pd, qp, init_attr, udata);
	1592	+ if (ret) {
	1593	+ kfree(qp);
	1594	+ return ERR_PTR(ret);
	1595	+ }
	1596	+
	1597	+ if (init_attr->qp_type == IB_QPT_GSI &&
1570	1598	!(init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI)) {
1571		- struct mlx4_ib_sqp *sqp = to_msqp((to_mqp(ibqp)));
	1599	+ struct mlx4_ib_sqp *sqp = qp->sqp;
1572	1600	int is_eth = rdma_cap_eth_ah(&dev->ib_dev, init_attr->port_num);
1573	1601
1574	1602	if (is_eth &&
..	..	@@ -1580,17 +1608,17 @@
1580	1608	pr_err("Failed to create GSI QP for RoCEv2 (%ld)\n", PTR_ERR(sqp->roce_v2_gsi));
1581	1609	sqp->roce_v2_gsi = NULL;
1582	1610	} else {
1583		- sqp = to_msqp(to_mqp(sqp->roce_v2_gsi));
1584		- sqp->qp.flags \|= MLX4_IB_ROCE_V2_GSI_QP;
	1611	+ to_mqp(sqp->roce_v2_gsi)->flags \|=
	1612	+ MLX4_IB_ROCE_V2_GSI_QP;
1585	1613	}
1586	1614
1587	1615	init_attr->create_flags &= ~MLX4_IB_QP_CREATE_ROCE_V2_GSI;
1588	1616	}
1589	1617	}
1590		- return ibqp;
	1618	+ return &qp->ibqp;
1591	1619	}
1592	1620
1593		-static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
	1621	+static int _mlx4_ib_destroy_qp(struct ib_qp qp, struct ib_udata udata)
1594	1622	{
1595	1623	struct mlx4_ib_dev *dev = to_mdev(qp->device);
1596	1624	struct mlx4_ib_qp *mqp = to_mqp(qp);
..	..	@@ -1611,32 +1639,27 @@
1611	1639	if (qp->rwq_ind_tbl) {
1612	1640	destroy_qp_rss(dev, mqp);
1613	1641	} else {
1614		- struct mlx4_ib_pd *pd;
1615		-
1616		- pd = get_pd(mqp);
1617		- destroy_qp_common(dev, mqp, MLX4_IB_QP_SRC, !!pd->ibpd.uobject);
	1642	+ destroy_qp_common(dev, mqp, MLX4_IB_QP_SRC, udata);
1618	1643	}
1619	1644
1620		- if (is_sqp(dev, mqp))
1621		- kfree(to_msqp(mqp));
1622		- else
1623		- kfree(mqp);
	1645	+ kfree(mqp->sqp);
	1646	+ kfree(mqp);
1624	1647
1625	1648	return 0;
1626	1649	}
1627	1650
1628		-int mlx4_ib_destroy_qp(struct ib_qp *qp)
	1651	+int mlx4_ib_destroy_qp(struct ib_qp qp, struct ib_udata udata)
1629	1652	{
1630	1653	struct mlx4_ib_qp *mqp = to_mqp(qp);
1631	1654
1632	1655	if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
1633		- struct mlx4_ib_sqp *sqp = to_msqp(mqp);
	1656	+ struct mlx4_ib_sqp *sqp = mqp->sqp;
1634	1657
1635	1658	if (sqp->roce_v2_gsi)
1636	1659	ib_destroy_qp(sqp->roce_v2_gsi);
1637	1660	}
1638	1661
1639		- return _mlx4_ib_destroy_qp(qp);
	1662	+ return _mlx4_ib_destroy_qp(qp, udata);
1640	1663	}
1641	1664
1642	1665	static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type)
..	..	@@ -1943,7 +1966,8 @@
1943	1966	* Go over all RSS QP's childes (WQs) and apply their HW state according to
1944	1967	* their logic state if the RSS QP is the first RSS QP associated for the WQ.
1945	1968	*/
1946		-static int bringup_rss_rwqs(struct ib_rwq_ind_table *ind_tbl, u8 port_num)
	1969	+static int bringup_rss_rwqs(struct ib_rwq_ind_table *ind_tbl, u8 port_num,
	1970	+ struct ib_udata *udata)
1947	1971	{
1948	1972	int err = 0;
1949	1973	int i;
..	..	@@ -1967,7 +1991,7 @@
1967	1991	}
1968	1992	wq->port = port_num;
1969	1993	if ((wq->rss_usecnt == 0) && (ibwq->state == IB_WQS_RDY)) {
1970		- err = _mlx4_ib_modify_wq(ibwq, IB_WQS_RDY);
	1994	+ err = _mlx4_ib_modify_wq(ibwq, IB_WQS_RDY, udata);
1971	1995	if (err) {
1972	1996	mutex_unlock(&wq->mutex);
1973	1997	break;
..	..	@@ -1989,7 +2013,8 @@
1989	2013
1990	2014	if ((wq->rss_usecnt == 1) &&
1991	2015	(ibwq->state == IB_WQS_RDY))
1992		- if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET))
	2016	+ if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET,
	2017	+ udata))
1993	2018	pr_warn("failed to reverse WQN=0x%06x\n",
1994	2019	ibwq->wq_num);
1995	2020	wq->rss_usecnt--;
..	..	@@ -2001,7 +2026,8 @@
2001	2026	return err;
2002	2027	}
2003	2028
2004		-static void bring_down_rss_rwqs(struct ib_rwq_ind_table *ind_tbl)
	2029	+static void bring_down_rss_rwqs(struct ib_rwq_ind_table *ind_tbl,
	2030	+ struct ib_udata *udata)
2005	2031	{
2006	2032	int i;
2007	2033
..	..	@@ -2012,7 +2038,7 @@
2012	2038	mutex_lock(&wq->mutex);
2013	2039
2014	2040	if ((wq->rss_usecnt == 1) && (ibwq->state == IB_WQS_RDY))
2015		- if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET))
	2041	+ if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET, udata))
2016	2042	pr_warn("failed to reverse WQN=%x\n",
2017	2043	ibwq->wq_num);
2018	2044	wq->rss_usecnt--;
..	..	@@ -2044,9 +2070,10 @@
2044	2070
2045	2071	static int __mlx4_ib_modify_qp(void *src, enum mlx4_ib_source_type src_type,
2046	2072	const struct ib_qp_attr *attr, int attr_mask,
2047		- enum ib_qp_state cur_state, enum ib_qp_state new_state)
	2073	+ enum ib_qp_state cur_state,
	2074	+ enum ib_qp_state new_state,
	2075	+ struct ib_udata *udata)
2048	2076	{
2049		- struct ib_uobject *ibuobject;
2050	2077	struct ib_srq *ibsrq;
2051	2078	const struct ib_gid_attr *gid_attr = NULL;
2052	2079	struct ib_rwq_ind_table *rwq_ind_tbl;
..	..	@@ -2055,6 +2082,8 @@
2055	2082	struct mlx4_ib_qp *qp;
2056	2083	struct mlx4_ib_pd *pd;
2057	2084	struct mlx4_ib_cq send_cq, recv_cq;
	2085	+ struct mlx4_ib_ucontext *ucontext = rdma_udata_to_drv_context(
	2086	+ udata, struct mlx4_ib_ucontext, ibucontext);
2058	2087	struct mlx4_qp_context *context;
2059	2088	enum mlx4_qp_optpar optpar = 0;
2060	2089	int sqd_event;
..	..	@@ -2066,7 +2095,6 @@
2066	2095	struct ib_wq *ibwq;
2067	2096
2068	2097	ibwq = (struct ib_wq *)src;
2069		- ibuobject = ibwq->uobject;
2070	2098	ibsrq = NULL;
2071	2099	rwq_ind_tbl = NULL;
2072	2100	qp_type = IB_QPT_RAW_PACKET;
..	..	@@ -2077,7 +2105,6 @@
2077	2105	struct ib_qp *ibqp;
2078	2106
2079	2107	ibqp = (struct ib_qp *)src;
2080		- ibuobject = ibqp->uobject;
2081	2108	ibsrq = ibqp->srq;
2082	2109	rwq_ind_tbl = ibqp->rwq_ind_tbl;
2083	2110	qp_type = ibqp->qp_type;
..	..	@@ -2162,11 +2189,9 @@
2162	2189	context->param3 \|= cpu_to_be32(1 << 30);
2163	2190	}
2164	2191
2165		- if (ibuobject)
	2192	+ if (ucontext)
2166	2193	context->usr_page = cpu_to_be32(
2167		- mlx4_to_hw_uar_index(dev->dev,
2168		- to_mucontext(ibuobject->context)
2169		- ->uar.index));
	2194	+ mlx4_to_hw_uar_index(dev->dev, ucontext->uar.index));
2170	2195	else
2171	2196	context->usr_page = cpu_to_be32(
2172	2197	mlx4_to_hw_uar_index(dev->dev, dev->priv_uar.index));
..	..	@@ -2237,8 +2262,10 @@
2237	2262
2238	2263	if (is_eth) {
2239	2264	gid_attr = attr->ah_attr.grh.sgid_attr;
2240		- vlan = rdma_vlan_dev_vlan_id(gid_attr->ndev);
2241		- memcpy(smac, gid_attr->ndev->dev_addr, ETH_ALEN);
	2265	+ err = rdma_read_gid_l2_fields(gid_attr, &vlan,
	2266	+ &smac[0]);
	2267	+ if (err)
	2268	+ goto out;
2242	2269	}
2243	2270
2244	2271	if (mlx4_set_path(dev, attr, attr_mask, qp, &context->pri_path,
..	..	@@ -2298,7 +2325,7 @@
2298	2325	context->cqn_recv = cpu_to_be32(recv_cq->mcq.cqn);
2299	2326
2300	2327	/* Set "fast registration enabled" for all kernel QPs */
2301		- if (!ibuobject)
	2328	+ if (!ucontext)
2302	2329	context->params1 \|= cpu_to_be32(1 << 11);
2303	2330
2304	2331	if (attr_mask & IB_QP_RNR_RETRY) {
..	..	@@ -2435,7 +2462,7 @@
2435	2462	else
2436	2463	sqd_event = 0;
2437	2464
2438		- if (!ibuobject &&
	2465	+ if (!ucontext &&
2439	2466	cur_state == IB_QPS_RESET &&
2440	2467	new_state == IB_QPS_INIT)
2441	2468	context->rlkey_roce_mode \|= (1 << 4);
..	..	@@ -2446,7 +2473,7 @@
2446	2473	* headroom is stamped so that the hardware doesn't start
2447	2474	* processing stale work requests.
2448	2475	*/
2449		- if (!ibuobject &&
	2476	+ if (!ucontext &&
2450	2477	cur_state == IB_QPS_RESET &&
2451	2478	new_state == IB_QPS_INIT) {
2452	2479	struct mlx4_wqe_ctrl_seg *ctrl;
..	..	@@ -2488,7 +2515,7 @@
2488	2515	qp->alt_port = attr->alt_port_num;
2489	2516
2490	2517	if (is_sqp(dev, qp))
2491		- store_sqp_attrs(to_msqp(qp), attr, attr_mask);
	2518	+ store_sqp_attrs(qp->sqp, attr, attr_mask);
2492	2519
2493	2520	/*
2494	2521	* If we moved QP0 to RTR, bring the IB link up; if we moved
..	..	@@ -2510,7 +2537,7 @@
2510	2537	* entries and reinitialize the QP.
2511	2538	*/
2512	2539	if (new_state == IB_QPS_RESET) {
2513		- if (!ibuobject) {
	2540	+ if (!ucontext) {
2514	2541	mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
2515	2542	ibsrq ? to_msrq(ibsrq) : NULL);
2516	2543	if (send_cq != recv_cq)
..	..	@@ -2631,7 +2658,6 @@
2631	2658	static int _mlx4_ib_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
2632	2659	int attr_mask, struct ib_udata *udata)
2633	2660	{
2634		- enum rdma_link_layer ll = IB_LINK_LAYER_UNSPECIFIED;
2635	2661	struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
2636	2662	struct mlx4_ib_qp *qp = to_mqp(ibqp);
2637	2663	enum ib_qp_state cur_state, new_state;
..	..	@@ -2641,13 +2667,8 @@
2641	2667	cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
2642	2668	new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
2643	2669
2644		- if (cur_state != new_state \|\| cur_state != IB_QPS_RESET) {
2645		- int port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2646		- ll = rdma_port_get_link_layer(&dev->ib_dev, port);
2647		- }
2648		-
2649	2670	if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
2650		- attr_mask, ll)) {
	2671	+ attr_mask)) {
2651	2672	pr_debug("qpn 0x%x: invalid attribute mask specified "
2652	2673	"for transition %d to %d. qp_type %d,"
2653	2674	" attr_mask 0x%x\n",
..	..	@@ -2742,16 +2763,17 @@
2742	2763	}
2743	2764
2744	2765	if (ibqp->rwq_ind_tbl && (new_state == IB_QPS_INIT)) {
2745		- err = bringup_rss_rwqs(ibqp->rwq_ind_tbl, attr->port_num);
	2766	+ err = bringup_rss_rwqs(ibqp->rwq_ind_tbl, attr->port_num,
	2767	+ udata);
2746	2768	if (err)
2747	2769	goto out;
2748	2770	}
2749	2771
2750	2772	err = __mlx4_ib_modify_qp(ibqp, MLX4_IB_QP_SRC, attr, attr_mask,
2751		- cur_state, new_state);
	2773	+ cur_state, new_state, udata);
2752	2774
2753	2775	if (ibqp->rwq_ind_tbl && err)
2754		- bring_down_rss_rwqs(ibqp->rwq_ind_tbl);
	2776	+ bring_down_rss_rwqs(ibqp->rwq_ind_tbl, udata);
2755	2777
2756	2778	if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT))
2757	2779	attr->port_num = 1;
..	..	@@ -2770,7 +2792,7 @@
2770	2792	ret = _mlx4_ib_modify_qp(ibqp, attr, attr_mask, udata);
2771	2793
2772	2794	if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
2773		- struct mlx4_ib_sqp *sqp = to_msqp(mqp);
	2795	+ struct mlx4_ib_sqp *sqp = mqp->sqp;
2774	2796	int err = 0;
2775	2797
2776	2798	if (sqp->roce_v2_gsi)
..	..	@@ -2795,12 +2817,13 @@
2795	2817	return -EINVAL;
2796	2818	}
2797	2819
2798		-static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,
	2820	+static int build_sriov_qp0_header(struct mlx4_ib_qp *qp,
2799	2821	const struct ib_ud_wr *wr,
2800	2822	void wqe, unsigned mlx_seg_len)
2801	2823	{
2802		- struct mlx4_ib_dev *mdev = to_mdev(sqp->qp.ibqp.device);
2803		- struct ib_device *ib_dev = &mdev->ib_dev;
	2824	+ struct mlx4_ib_dev *mdev = to_mdev(qp->ibqp.device);
	2825	+ struct mlx4_ib_sqp *sqp = qp->sqp;
	2826	+ struct ib_device *ib_dev = qp->ibqp.device;
2804	2827	struct mlx4_wqe_mlx_seg *mlx = wqe;
2805	2828	struct mlx4_wqe_inline_seg inl = wqe + sizeof mlx;
2806	2829	struct mlx4_ib_ah *ah = to_mah(wr->ah);
..	..	@@ -2822,12 +2845,12 @@
2822	2845
2823	2846	/* for proxy-qp0 sends, need to add in size of tunnel header */
2824	2847	/* for tunnel-qp0 sends, tunnel header is already in s/g list */
2825		- if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER)
	2848	+ if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER)
2826	2849	send_size += sizeof (struct mlx4_ib_tunnel_header);
2827	2850
2828	2851	ib_ud_header_init(send_size, 1, 0, 0, 0, 0, 0, 0, &sqp->ud_header);
2829	2852
2830		- if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) {
	2853	+ if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) {
2831	2854	sqp->ud_header.lrh.service_level =
2832	2855	be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
2833	2856	sqp->ud_header.lrh.destination_lid =
..	..	@@ -2844,26 +2867,26 @@
2844	2867
2845	2868	sqp->ud_header.lrh.virtual_lane = 0;
2846	2869	sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
2847		- err = ib_get_cached_pkey(ib_dev, sqp->qp.port, 0, &pkey);
	2870	+ err = ib_get_cached_pkey(ib_dev, qp->port, 0, &pkey);
2848	2871	if (err)
2849	2872	return err;
2850	2873	sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
2851		- if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER)
	2874	+ if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER)
2852	2875	sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn);
2853	2876	else
2854	2877	sqp->ud_header.bth.destination_qpn =
2855		- cpu_to_be32(mdev->dev->caps.spec_qps[sqp->qp.port - 1].qp0_tunnel);
	2878	+ cpu_to_be32(mdev->dev->caps.spec_qps[qp->port - 1].qp0_tunnel);
2856	2879
2857	2880	sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
2858	2881	if (mlx4_is_master(mdev->dev)) {
2859		- if (mlx4_get_parav_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
	2882	+ if (mlx4_get_parav_qkey(mdev->dev, qp->mqp.qpn, &qkey))
2860	2883	return -EINVAL;
2861	2884	} else {
2862		- if (vf_get_qp0_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
	2885	+ if (vf_get_qp0_qkey(mdev->dev, qp->mqp.qpn, &qkey))
2863	2886	return -EINVAL;
2864	2887	}
2865	2888	sqp->ud_header.deth.qkey = cpu_to_be32(qkey);
2866		- sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.mqp.qpn);
	2889	+ sqp->ud_header.deth.source_qpn = cpu_to_be32(qp->mqp.qpn);
2867	2890
2868	2891	sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2869	2892	sqp->ud_header.immediate_present = 0;
..	..	@@ -2947,10 +2970,11 @@
2947	2970	}
2948	2971
2949	2972	#define MLX4_ROCEV2_QP1_SPORT 0xC000
2950		-static int build_mlx_header(struct mlx4_ib_sqp sqp, const struct ib_ud_wr wr,
	2973	+static int build_mlx_header(struct mlx4_ib_qp qp, const struct ib_ud_wr wr,
2951	2974	void wqe, unsigned mlx_seg_len)
2952	2975	{
2953		- struct ib_device *ib_dev = sqp->qp.ibqp.device;
	2976	+ struct mlx4_ib_sqp *sqp = qp->sqp;
	2977	+ struct ib_device *ib_dev = qp->ibqp.device;
2954	2978	struct mlx4_ib_dev *ibdev = to_mdev(ib_dev);
2955	2979	struct mlx4_wqe_mlx_seg *mlx = wqe;
2956	2980	struct mlx4_wqe_ctrl_seg *ctrl = wqe;
..	..	@@ -2974,7 +2998,7 @@
2974	2998	for (i = 0; i < wr->wr.num_sge; ++i)
2975	2999	send_size += wr->wr.sg_list[i].length;
2976	3000
2977		- is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET;
	3001	+ is_eth = rdma_port_get_link_layer(qp->ibqp.device, qp->port) == IB_LINK_LAYER_ETHERNET;
2978	3002	is_grh = mlx4_ib_ah_grh_present(ah);
2979	3003	if (is_eth) {
2980	3004	enum ib_gid_type gid_type;
..	..	@@ -2988,9 +3012,9 @@
2988	3012	if (err)
2989	3013	return err;
2990	3014	} else {
2991		- err = fill_gid_by_hw_index(ibdev, sqp->qp.port,
2992		- ah->av.ib.gid_index,
2993		- &sgid, &gid_type);
	3015	+ err = fill_gid_by_hw_index(ibdev, qp->port,
	3016	+ ah->av.ib.gid_index, &sgid,
	3017	+ &gid_type);
2994	3018	if (!err) {
2995	3019	is_udp = gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
2996	3020	if (is_udp) {
..	..	@@ -3006,7 +3030,7 @@
3006	3030	}
3007	3031	if (ah->av.eth.vlan != cpu_to_be16(0xffff)) {
3008	3032	vlan = be16_to_cpu(ah->av.eth.vlan) & 0x0fff;
3009		- is_vlan = 1;
	3033	+ is_vlan = true;
3010	3034	}
3011	3035	}
3012	3036	err = ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh,
..	..	@@ -3035,13 +3059,18 @@
3035	3059	* indexes don't necessarily match the hw ones, so
3036	3060	* we must use our own cache
3037	3061	*/
3038		- sqp->ud_header.grh.source_gid.global.subnet_prefix =
3039		- cpu_to_be64(atomic64_read(&(to_mdev(ib_dev)->sriov.
3040		- demux[sqp->qp.port - 1].
3041		- subnet_prefix)));
3042		- sqp->ud_header.grh.source_gid.global.interface_id =
3043		- to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1].
3044		- guid_cache[ah->av.ib.gid_index];
	3062	+ sqp->ud_header.grh.source_gid.global
	3063	+ .subnet_prefix =
	3064	+ cpu_to_be64(atomic64_read(
	3065	+ &(to_mdev(ib_dev)
	3066	+ ->sriov
	3067	+ .demux[qp->port - 1]
	3068	+ .subnet_prefix)));
	3069	+ sqp->ud_header.grh.source_gid.global
	3070	+ .interface_id =
	3071	+ to_mdev(ib_dev)
	3072	+ ->sriov.demux[qp->port - 1]
	3073	+ .guid_cache[ah->av.ib.gid_index];
3045	3074	} else {
3046	3075	sqp->ud_header.grh.source_gid =
3047	3076	ah->ibah.sgid_attr->gid;
..	..	@@ -3073,10 +3102,13 @@
3073	3102	mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
3074	3103
3075	3104	if (!is_eth) {
3076		- mlx->flags \|= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) \|
3077		- (sqp->ud_header.lrh.destination_lid ==
3078		- IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) \|
3079		- (sqp->ud_header.lrh.service_level << 8));
	3105	+ mlx->flags \|=
	3106	+ cpu_to_be32((!qp->ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) \|
	3107	+ (sqp->ud_header.lrh.destination_lid ==
	3108	+ IB_LID_PERMISSIVE ?
	3109	+ MLX4_WQE_MLX_SLR :
	3110	+ 0) \|
	3111	+ (sqp->ud_header.lrh.service_level << 8));
3080	3112	if (ah->av.ib.port_pd & cpu_to_be32(0x80000000))
3081	3113	mlx->flags \|= cpu_to_be32(0x1); /* force loopback */
3082	3114	mlx->rlid = sqp->ud_header.lrh.destination_lid;
..	..	@@ -3122,21 +3154,23 @@
3122	3154	sqp->ud_header.vlan.tag = cpu_to_be16(vlan \| pcp);
3123	3155	}
3124	3156	} else {
3125		- sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 :
3126		- sl_to_vl(to_mdev(ib_dev),
3127		- sqp->ud_header.lrh.service_level,
3128		- sqp->qp.port);
3129		- if (sqp->qp.ibqp.qp_num && sqp->ud_header.lrh.virtual_lane == 15)
	3157	+ sqp->ud_header.lrh.virtual_lane =
	3158	+ !qp->ibqp.qp_num ?
	3159	+ 15 :
	3160	+ sl_to_vl(to_mdev(ib_dev),
	3161	+ sqp->ud_header.lrh.service_level,
	3162	+ qp->port);
	3163	+ if (qp->ibqp.qp_num && sqp->ud_header.lrh.virtual_lane == 15)
3130	3164	return -EINVAL;
3131	3165	if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
3132	3166	sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
3133	3167	}
3134	3168	sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
3135		- if (!sqp->qp.ibqp.qp_num)
3136		- err = ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index,
	3169	+ if (!qp->ibqp.qp_num)
	3170	+ err = ib_get_cached_pkey(ib_dev, qp->port, sqp->pkey_index,
3137	3171	&pkey);
3138	3172	else
3139		- err = ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->pkey_index,
	3173	+ err = ib_get_cached_pkey(ib_dev, qp->port, wr->pkey_index,
3140	3174	&pkey);
3141	3175	if (err)
3142	3176	return err;
..	..	@@ -3146,7 +3180,7 @@
3146	3180	sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
3147	3181	sqp->ud_header.deth.qkey = cpu_to_be32(wr->remote_qkey & 0x80000000 ?
3148	3182	sqp->qkey : wr->remote_qkey);
3149		- sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num);
	3183	+ sqp->ud_header.deth.source_qpn = cpu_to_be32(qp->ibqp.qp_num);
3150	3184
3151	3185	header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
3152	3186
..	..	@@ -3459,24 +3493,24 @@
3459	3493	int nreq;
3460	3494	int err = 0;
3461	3495	unsigned ind;
3462		- int uninitialized_var(size);
3463		- unsigned uninitialized_var(seglen);
	3496	+ int size;
	3497	+ unsigned seglen;
3464	3498	__be32 dummy;
3465	3499	__be32 *lso_wqe;
3466		- __be32 uninitialized_var(lso_hdr_sz);
	3500	+ __be32 lso_hdr_sz;
3467	3501	__be32 blh;
3468	3502	int i;
3469	3503	struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
3470	3504
3471	3505	if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
3472		- struct mlx4_ib_sqp *sqp = to_msqp(qp);
	3506	+ struct mlx4_ib_sqp *sqp = qp->sqp;
3473	3507
3474	3508	if (sqp->roce_v2_gsi) {
3475	3509	struct mlx4_ib_ah *ah = to_mah(ud_wr(wr)->ah);
3476	3510	enum ib_gid_type gid_type;
3477	3511	union ib_gid gid;
3478	3512
3479		- if (!fill_gid_by_hw_index(mdev, sqp->qp.port,
	3513	+ if (!fill_gid_by_hw_index(mdev, qp->port,
3480	3514	ah->av.ib.gid_index,
3481	3515	&gid, &gid_type))
3482	3516	qp = (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
..	..	@@ -3596,8 +3630,8 @@
3596	3630	break;
3597	3631
3598	3632	case MLX4_IB_QPT_TUN_SMI_OWNER:
3599		- err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),
3600		- ctrl, &seglen);
	3633	+ err = build_sriov_qp0_header(qp, ud_wr(wr), ctrl,
	3634	+ &seglen);
3601	3635	if (unlikely(err)) {
3602	3636	*bad_wr = wr;
3603	3637	goto out;
..	..	@@ -3633,8 +3667,8 @@
3633	3667	break;
3634	3668
3635	3669	case MLX4_IB_QPT_PROXY_SMI_OWNER:
3636		- err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),
3637		- ctrl, &seglen);
	3670	+ err = build_sriov_qp0_header(qp, ud_wr(wr), ctrl,
	3671	+ &seglen);
3638	3672	if (unlikely(err)) {
3639	3673	*bad_wr = wr;
3640	3674	goto out;
..	..	@@ -3667,8 +3701,7 @@
3667	3701
3668	3702	case MLX4_IB_QPT_SMI:
3669	3703	case MLX4_IB_QPT_GSI:
3670		- err = build_mlx_header(to_msqp(qp), ud_wr(wr), ctrl,
3671		- &seglen);
	3704	+ err = build_mlx_header(qp, ud_wr(wr), ctrl, &seglen);
3672	3705	if (unlikely(err)) {
3673	3706	*bad_wr = wr;
3674	3707	goto out;
..	..	@@ -3753,12 +3786,6 @@
3753	3786
3754	3787	writel_relaxed(qp->doorbell_qpn,
3755	3788	to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL);
3756		-
3757		- /*
3758		- * Make sure doorbells don't leak out of SQ spinlock
3759		- * and reach the HCA out of order.
3760		- */
3761		- mmiowb();
3762	3789
3763	3790	stamp_send_wqe(qp, ind + qp->sq_spare_wqes - 1);
3764	3791
..	..	@@ -4054,13 +4081,13 @@
4054	4081	struct ib_wq_init_attr *init_attr,
4055	4082	struct ib_udata *udata)
4056	4083	{
4057		- struct mlx4_ib_dev *dev;
4058		- struct ib_qp_init_attr ib_qp_init_attr;
	4084	+ struct mlx4_dev *dev = to_mdev(pd->device)->dev;
	4085	+ struct ib_qp_init_attr ib_qp_init_attr = {};
4059	4086	struct mlx4_ib_qp *qp;
4060	4087	struct mlx4_ib_create_wq ucmd;
4061	4088	int err, required_cmd_sz;
4062	4089
4063		- if (!(udata && pd->uobject))
	4090	+ if (!udata)
4064	4091	return ERR_PTR(-EINVAL);
4065	4092
4066	4093	required_cmd_sz = offsetof(typeof(ucmd), comp_mask) +
..	..	@@ -4080,14 +4107,13 @@
4080	4107	if (udata->outlen)
4081	4108	return ERR_PTR(-EOPNOTSUPP);
4082	4109
4083		- dev = to_mdev(pd->device);
4084		-
4085	4110	if (init_attr->wq_type != IB_WQT_RQ) {
4086	4111	pr_debug("unsupported wq type %d\n", init_attr->wq_type);
4087	4112	return ERR_PTR(-EOPNOTSUPP);
4088	4113	}
4089	4114
4090		- if (init_attr->create_flags & ~IB_WQ_FLAGS_SCATTER_FCS) {
	4115	+ if (init_attr->create_flags & ~IB_WQ_FLAGS_SCATTER_FCS \|\|
	4116	+ !(dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP)) {
4091	4117	pr_debug("unsupported create_flags %u\n",
4092	4118	init_attr->create_flags);
4093	4119	return ERR_PTR(-EOPNOTSUPP);
..	..	@@ -4097,10 +4123,10 @@
4097	4123	if (!qp)
4098	4124	return ERR_PTR(-ENOMEM);
4099	4125
	4126	+ mutex_init(&qp->mutex);
4100	4127	qp->pri.vid = 0xFFFF;
4101	4128	qp->alt.vid = 0xFFFF;
4102	4129
4103		- memset(&ib_qp_init_attr, 0, sizeof(ib_qp_init_attr));
4104	4130	ib_qp_init_attr.qp_context = init_attr->wq_context;
4105	4131	ib_qp_init_attr.qp_type = IB_QPT_RAW_PACKET;
4106	4132	ib_qp_init_attr.cap.max_recv_wr = init_attr->max_wr;
..	..	@@ -4111,8 +4137,7 @@
4111	4137	if (init_attr->create_flags & IB_WQ_FLAGS_SCATTER_FCS)
4112	4138	ib_qp_init_attr.create_flags \|= IB_QP_CREATE_SCATTER_FCS;
4113	4139
4114		- err = create_qp_common(dev, pd, MLX4_IB_RWQ_SRC, &ib_qp_init_attr,
4115		- udata, 0, &qp);
	4140	+ err = create_rq(pd, &ib_qp_init_attr, udata, qp);
4116	4141	if (err) {
4117	4142	kfree(qp);
4118	4143	return ERR_PTR(err);
..	..	@@ -4137,7 +4162,8 @@
4137	4162	}
4138	4163	}
4139	4164
4140		-static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state)
	4165	+static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state,
	4166	+ struct ib_udata *udata)
4141	4167	{
4142	4168	struct mlx4_ib_qp qp = to_mqp((struct ib_qp )ibwq);
4143	4169	enum ib_qp_state qp_cur_state;
..	..	@@ -4161,7 +4187,8 @@
4161	4187	attr_mask = IB_QP_PORT;
4162	4188
4163	4189	err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, &attr,
4164		- attr_mask, IB_QPS_RESET, IB_QPS_INIT);
	4190	+ attr_mask, IB_QPS_RESET, IB_QPS_INIT,
	4191	+ udata);
4165	4192	if (err) {
4166	4193	pr_debug("WQN=0x%06x failed to apply RST->INIT on the HW QP\n",
4167	4194	ibwq->wq_num);
..	..	@@ -4173,12 +4200,13 @@
4173	4200
4174	4201	attr_mask = 0;
4175	4202	err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL, attr_mask,
4176		- qp_cur_state, qp_new_state);
	4203	+ qp_cur_state, qp_new_state, udata);
4177	4204
4178	4205	if (err && (qp_cur_state == IB_QPS_INIT)) {
4179	4206	qp_new_state = IB_QPS_RESET;
4180	4207	if (__mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL,
4181		- attr_mask, IB_QPS_INIT, IB_QPS_RESET)) {
	4208	+ attr_mask, IB_QPS_INIT, IB_QPS_RESET,
	4209	+ udata)) {
4182	4210	pr_warn("WQN=0x%06x failed with reverting HW's resources failure\n",
4183	4211	ibwq->wq_num);
4184	4212	qp_new_state = IB_QPS_INIT;
..	..	@@ -4218,13 +4246,8 @@
4218	4246	if (wq_attr_mask & IB_WQ_FLAGS)
4219	4247	return -EOPNOTSUPP;
4220	4248
4221		- cur_state = wq_attr_mask & IB_WQ_CUR_STATE ? wq_attr->curr_wq_state :
4222		- ibwq->state;
4223		- new_state = wq_attr_mask & IB_WQ_STATE ? wq_attr->wq_state : cur_state;
4224		-
4225		- if (cur_state < IB_WQS_RESET \|\| cur_state > IB_WQS_ERR \|\|
4226		- new_state < IB_WQS_RESET \|\| new_state > IB_WQS_ERR)
4227		- return -EINVAL;
	4249	+ cur_state = wq_attr->curr_wq_state;
	4250	+ new_state = wq_attr->wq_state;
4228	4251
4229	4252	if ((new_state == IB_WQS_RDY) && (cur_state == IB_WQS_ERR))
4230	4253	return -EINVAL;
..	..	@@ -4241,7 +4264,7 @@
4241	4264	* WQ, so we can apply its port on the WQ.
4242	4265	*/
4243	4266	if (qp->rss_usecnt)
4244		- err = _mlx4_ib_modify_wq(ibwq, new_state);
	4267	+ err = _mlx4_ib_modify_wq(ibwq, new_state, udata);
4245	4268
4246	4269	if (!err)
4247	4270	ibwq->state = new_state;
..	..	@@ -4251,7 +4274,7 @@
4251	4274	return err;
4252	4275	}
4253	4276
4254		-int mlx4_ib_destroy_wq(struct ib_wq *ibwq)
	4277	+int mlx4_ib_destroy_wq(struct ib_wq ibwq, struct ib_udata udata)
4255	4278	{
4256	4279	struct mlx4_ib_dev *dev = to_mdev(ibwq->device);
4257	4280	struct mlx4_ib_qp qp = to_mqp((struct ib_qp )ibwq);
..	..	@@ -4259,41 +4282,38 @@
4259	4282	if (qp->counter_index)
4260	4283	mlx4_ib_free_qp_counter(dev, qp);
4261	4284
4262		- destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, 1);
	4285	+ destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, udata);
4263	4286
4264	4287	kfree(qp);
4265		-
4266	4288	return 0;
4267	4289	}
4268	4290
4269		-struct ib_rwq_ind_table
4270		-mlx4_ib_create_rwq_ind_table(struct ib_device device,
4271		- struct ib_rwq_ind_table_init_attr *init_attr,
4272		- struct ib_udata *udata)
	4291	+int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table,
	4292	+ struct ib_rwq_ind_table_init_attr *init_attr,
	4293	+ struct ib_udata *udata)
4273	4294	{
4274		- struct ib_rwq_ind_table *rwq_ind_table;
4275	4295	struct mlx4_ib_create_rwq_ind_tbl_resp resp = {};
4276	4296	unsigned int ind_tbl_size = 1 << init_attr->log_ind_tbl_size;
	4297	+ struct ib_device *device = rwq_ind_table->device;
4277	4298	unsigned int base_wqn;
4278	4299	size_t min_resp_len;
4279		- int i;
4280		- int err;
	4300	+ int i, err = 0;
4281	4301
4282	4302	if (udata->inlen > 0 &&
4283	4303	!ib_is_udata_cleared(udata, 0,
4284	4304	udata->inlen))
4285		- return ERR_PTR(-EOPNOTSUPP);
	4305	+ return -EOPNOTSUPP;
4286	4306
4287	4307	min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
4288	4308	if (udata->outlen && udata->outlen < min_resp_len)
4289		- return ERR_PTR(-EINVAL);
	4309	+ return -EINVAL;
4290	4310
4291	4311	if (ind_tbl_size >
4292	4312	device->attrs.rss_caps.max_rwq_indirection_table_size) {
4293	4313	pr_debug("log_ind_tbl_size = %d is bigger than supported = %d\n",
4294	4314	ind_tbl_size,
4295	4315	device->attrs.rss_caps.max_rwq_indirection_table_size);
4296		- return ERR_PTR(-EINVAL);
	4316	+ return -EINVAL;
4297	4317	}
4298	4318
4299	4319	base_wqn = init_attr->ind_tbl[0]->wq_num;
..	..	@@ -4301,39 +4321,23 @@
4301	4321	if (base_wqn % ind_tbl_size) {
4302	4322	pr_debug("WQN=0x%x isn't aligned with indirection table size\n",
4303	4323	base_wqn);
4304		- return ERR_PTR(-EINVAL);
	4324	+ return -EINVAL;
4305	4325	}
4306	4326
4307	4327	for (i = 1; i < ind_tbl_size; i++) {
4308	4328	if (++base_wqn != init_attr->ind_tbl[i]->wq_num) {
4309	4329	pr_debug("indirection table's WQNs aren't consecutive\n");
4310		- return ERR_PTR(-EINVAL);
	4330	+ return -EINVAL;
4311	4331	}
4312	4332	}
4313		-
4314		- rwq_ind_table = kzalloc(sizeof(*rwq_ind_table), GFP_KERNEL);
4315		- if (!rwq_ind_table)
4316		- return ERR_PTR(-ENOMEM);
4317	4333
4318	4334	if (udata->outlen) {
4319	4335	resp.response_length = offsetof(typeof(resp), response_length) +
4320	4336	sizeof(resp.response_length);
4321	4337	err = ib_copy_to_udata(udata, &resp, resp.response_length);
4322		- if (err)
4323		- goto err;
4324	4338	}
4325	4339
4326		- return rwq_ind_table;
4327		-
4328		-err:
4329		- kfree(rwq_ind_table);
4330		- return ERR_PTR(err);
4331		-}
4332		-
4333		-int mlx4_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
4334		-{
4335		- kfree(ib_rwq_ind_tbl);
4336		- return 0;
	4340	+ return err;
4337	4341	}
4338	4342
4339	4343	struct mlx4_ib_drain_cqe {