~hc/RK356X_SDK_RELEASE.git

..	..	@@ -34,12 +34,14 @@
34	34	#include <rdma/ib_umem.h>
35	35	#include <rdma/ib_cache.h>
36	36	#include <rdma/ib_user_verbs.h>
	37	+#include <rdma/rdma_counter.h>
37	38	#include <linux/mlx5/fs.h>
38	39	#include "mlx5_ib.h"
39	40	#include "ib_rep.h"
40		-
41		-/* not supported currently */
42		-static int wq_signature;
	41	+#include "counters.h"
	42	+#include "cmd.h"
	43	+#include "qp.h"
	44	+#include "wr.h"
43	45
44	46	enum {
45	47	MLX5_IB_ACK_REQ_FREQ = 8,
..	..	@@ -50,32 +52,6 @@
50	52	MLX5_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
51	53	MLX5_IB_LINK_TYPE_IB = 0,
52	54	MLX5_IB_LINK_TYPE_ETH = 1
53		-};
54		-
55		-enum {
56		- MLX5_IB_SQ_STRIDE = 6,
57		- MLX5_IB_SQ_UMR_INLINE_THRESHOLD = 64,
58		-};
59		-
60		-static const u32 mlx5_ib_opcode[] = {
61		- [IB_WR_SEND] = MLX5_OPCODE_SEND,
62		- [IB_WR_LSO] = MLX5_OPCODE_LSO,
63		- [IB_WR_SEND_WITH_IMM] = MLX5_OPCODE_SEND_IMM,
64		- [IB_WR_RDMA_WRITE] = MLX5_OPCODE_RDMA_WRITE,
65		- [IB_WR_RDMA_WRITE_WITH_IMM] = MLX5_OPCODE_RDMA_WRITE_IMM,
66		- [IB_WR_RDMA_READ] = MLX5_OPCODE_RDMA_READ,
67		- [IB_WR_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_CS,
68		- [IB_WR_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_FA,
69		- [IB_WR_SEND_WITH_INV] = MLX5_OPCODE_SEND_INVAL,
70		- [IB_WR_LOCAL_INV] = MLX5_OPCODE_UMR,
71		- [IB_WR_REG_MR] = MLX5_OPCODE_UMR,
72		- [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_MASKED_CS,
73		- [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_MASKED_FA,
74		- [MLX5_IB_WR_UMR] = MLX5_OPCODE_UMR,
75		-};
76		-
77		-struct mlx5_wqe_eth_pad {
78		- u8 rsvd0[16];
79	55	};
80	56
81	57	enum raw_qp_set_mask_map {
..	..	@@ -91,6 +67,7 @@
91	67	struct mlx5_rate_limit rl;
92	68
93	69	u8 rq_q_ctr_id;
	70	+ u16 port;
94	71	};
95	72
96	73	static void get_cqs(enum ib_qp_type qp_type,
..	..	@@ -107,91 +84,221 @@
107	84	return is_qp0(qp_type) \|\| is_qp1(qp_type);
108	85	}
109	86
110		-static void get_wqe(struct mlx5_ib_qp qp, int offset)
111		-{
112		- return mlx5_buf_offset(&qp->buf, offset);
113		-}
114		-
115		-static void get_recv_wqe(struct mlx5_ib_qp qp, int n)
116		-{
117		- return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
118		-}
119		-
120		-void mlx5_get_send_wqe(struct mlx5_ib_qp qp, int n)
121		-{
122		- return get_wqe(qp, qp->sq.offset + (n << MLX5_IB_SQ_STRIDE));
123		-}
124		-
125	87	/**
126		- * mlx5_ib_read_user_wqe() - Copy a user-space WQE to kernel space.
	88	+ * mlx5_ib_read_user_wqe_common() - Copy a WQE (or part of) from user WQ
	89	+ * to kernel buffer
127	90	*
128		- * @qp: QP to copy from.
129		- * @send: copy from the send queue when non-zero, use the receive queue
130		- * otherwise.
131		- * @wqe_index: index to start copying from. For send work queues, the
132		- * wqe_index is in units of MLX5_SEND_WQE_BB.
133		- * For receive work queue, it is the number of work queue
134		- * element in the queue.
135		- * @buffer: destination buffer.
136		- * @length: maximum number of bytes to copy.
	91	+ * @umem: User space memory where the WQ is
	92	+ * @buffer: buffer to copy to
	93	+ * @buflen: buffer length
	94	+ * @wqe_index: index of WQE to copy from
	95	+ * @wq_offset: offset to start of WQ
	96	+ * @wq_wqe_cnt: number of WQEs in WQ
	97	+ * @wq_wqe_shift: log2 of WQE size
	98	+ * @bcnt: number of bytes to copy
	99	+ * @bytes_copied: number of bytes to copy (return value)
137	100	*
138		- * Copies at least a single WQE, but may copy more data.
	101	+ * Copies from start of WQE bcnt or less bytes.
	102	+ * Does not gurantee to copy the entire WQE.
139	103	*
140		- * Return: the number of bytes copied, or an error code.
	104	+ * Return: zero on success, or an error code.
141	105	*/
142		-int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index,
143		- void *buffer, u32 length,
144		- struct mlx5_ib_qp_base *base)
	106	+static int mlx5_ib_read_user_wqe_common(struct ib_umem umem, void buffer,
	107	+ size_t buflen, int wqe_index,
	108	+ int wq_offset, int wq_wqe_cnt,
	109	+ int wq_wqe_shift, int bcnt,
	110	+ size_t *bytes_copied)
145	111	{
146		- struct ib_device *ibdev = qp->ibqp.device;
147		- struct mlx5_ib_dev *dev = to_mdev(ibdev);
148		- struct mlx5_ib_wq *wq = send ? &qp->sq : &qp->rq;
149		- size_t offset;
150		- size_t wq_end;
151		- struct ib_umem *umem = base->ubuffer.umem;
152		- u32 first_copy_length;
153		- int wqe_length;
	112	+ size_t offset = wq_offset + ((wqe_index % wq_wqe_cnt) << wq_wqe_shift);
	113	+ size_t wq_end = wq_offset + (wq_wqe_cnt << wq_wqe_shift);
	114	+ size_t copy_length;
154	115	int ret;
155	116
156		- if (wq->wqe_cnt == 0) {
157		- mlx5_ib_dbg(dev, "mlx5_ib_read_user_wqe for a QP with wqe_cnt == 0. qp_type: 0x%x\n",
158		- qp->ibqp.qp_type);
159		- return -EINVAL;
160		- }
	117	+ /* don't copy more than requested, more than buffer length or
	118	+ * beyond WQ end
	119	+ */
	120	+ copy_length = min_t(u32, buflen, wq_end - offset);
	121	+ copy_length = min_t(u32, copy_length, bcnt);
161	122
162		- offset = wq->offset + ((wqe_index % wq->wqe_cnt) << wq->wqe_shift);
163		- wq_end = wq->offset + (wq->wqe_cnt << wq->wqe_shift);
164		-
165		- if (send && length < sizeof(struct mlx5_wqe_ctrl_seg))
166		- return -EINVAL;
167		-
168		- if (offset > umem->length \|\|
169		- (send && offset + sizeof(struct mlx5_wqe_ctrl_seg) > umem->length))
170		- return -EINVAL;
171		-
172		- first_copy_length = min_t(u32, offset + length, wq_end) - offset;
173		- ret = ib_umem_copy_from(buffer, umem, offset, first_copy_length);
	123	+ ret = ib_umem_copy_from(buffer, umem, offset, copy_length);
174	124	if (ret)
175	125	return ret;
176	126
177		- if (send) {
178		- struct mlx5_wqe_ctrl_seg *ctrl = buffer;
179		- int ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
	127	+ if (!ret && bytes_copied)
	128	+ *bytes_copied = copy_length;
180	129
181		- wqe_length = ds * MLX5_WQE_DS_UNITS;
182		- } else {
183		- wqe_length = 1 << wq->wqe_shift;
	130	+ return 0;
	131	+}
	132	+
	133	+static int mlx5_ib_read_kernel_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
	134	+ void buffer, size_t buflen, size_t bc)
	135	+{
	136	+ struct mlx5_wqe_ctrl_seg *ctrl;
	137	+ size_t bytes_copied = 0;
	138	+ size_t wqe_length;
	139	+ void *p;
	140	+ int ds;
	141	+
	142	+ wqe_index = wqe_index & qp->sq.fbc.sz_m1;
	143	+
	144	+ /* read the control segment first */
	145	+ p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, wqe_index);
	146	+ ctrl = p;
	147	+ ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
	148	+ wqe_length = ds * MLX5_WQE_DS_UNITS;
	149	+
	150	+ /* read rest of WQE if it spreads over more than one stride */
	151	+ while (bytes_copied < wqe_length) {
	152	+ size_t copy_length =
	153	+ min_t(size_t, buflen - bytes_copied, MLX5_SEND_WQE_BB);
	154	+
	155	+ if (!copy_length)
	156	+ break;
	157	+
	158	+ memcpy(buffer + bytes_copied, p, copy_length);
	159	+ bytes_copied += copy_length;
	160	+
	161	+ wqe_index = (wqe_index + 1) & qp->sq.fbc.sz_m1;
	162	+ p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, wqe_index);
184	163	}
	164	+ *bc = bytes_copied;
	165	+ return 0;
	166	+}
185	167
186		- if (wqe_length <= first_copy_length)
187		- return first_copy_length;
	168	+static int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
	169	+ void buffer, size_t buflen, size_t bc)
	170	+{
	171	+ struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
	172	+ struct ib_umem *umem = base->ubuffer.umem;
	173	+ struct mlx5_ib_wq *wq = &qp->sq;
	174	+ struct mlx5_wqe_ctrl_seg *ctrl;
	175	+ size_t bytes_copied;
	176	+ size_t bytes_copied2;
	177	+ size_t wqe_length;
	178	+ int ret;
	179	+ int ds;
188	180
189		- ret = ib_umem_copy_from(buffer + first_copy_length, umem, wq->offset,
190		- wqe_length - first_copy_length);
	181	+ /* at first read as much as possible */
	182	+ ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,
	183	+ wq->offset, wq->wqe_cnt,
	184	+ wq->wqe_shift, buflen,
	185	+ &bytes_copied);
191	186	if (ret)
192	187	return ret;
193	188
194		- return wqe_length;
	189	+ /* we need at least control segment size to proceed */
	190	+ if (bytes_copied < sizeof(*ctrl))
	191	+ return -EINVAL;
	192	+
	193	+ ctrl = buffer;
	194	+ ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
	195	+ wqe_length = ds * MLX5_WQE_DS_UNITS;
	196	+
	197	+ /* if we copied enough then we are done */
	198	+ if (bytes_copied >= wqe_length) {
	199	+ *bc = bytes_copied;
	200	+ return 0;
	201	+ }
	202	+
	203	+ /* otherwise this a wrapped around wqe
	204	+ * so read the remaining bytes starting
	205	+ * from wqe_index 0
	206	+ */
	207	+ ret = mlx5_ib_read_user_wqe_common(umem, buffer + bytes_copied,
	208	+ buflen - bytes_copied, 0, wq->offset,
	209	+ wq->wqe_cnt, wq->wqe_shift,
	210	+ wqe_length - bytes_copied,
	211	+ &bytes_copied2);
	212	+
	213	+ if (ret)
	214	+ return ret;
	215	+ *bc = bytes_copied + bytes_copied2;
	216	+ return 0;
	217	+}
	218	+
	219	+int mlx5_ib_read_wqe_sq(struct mlx5_ib_qp qp, int wqe_index, void buffer,
	220	+ size_t buflen, size_t *bc)
	221	+{
	222	+ struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
	223	+ struct ib_umem *umem = base->ubuffer.umem;
	224	+
	225	+ if (buflen < sizeof(struct mlx5_wqe_ctrl_seg))
	226	+ return -EINVAL;
	227	+
	228	+ if (!umem)
	229	+ return mlx5_ib_read_kernel_wqe_sq(qp, wqe_index, buffer,
	230	+ buflen, bc);
	231	+
	232	+ return mlx5_ib_read_user_wqe_sq(qp, wqe_index, buffer, buflen, bc);
	233	+}
	234	+
	235	+static int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index,
	236	+ void buffer, size_t buflen, size_t bc)
	237	+{
	238	+ struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
	239	+ struct ib_umem *umem = base->ubuffer.umem;
	240	+ struct mlx5_ib_wq *wq = &qp->rq;
	241	+ size_t bytes_copied;
	242	+ int ret;
	243	+
	244	+ ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,
	245	+ wq->offset, wq->wqe_cnt,
	246	+ wq->wqe_shift, buflen,
	247	+ &bytes_copied);
	248	+
	249	+ if (ret)
	250	+ return ret;
	251	+ *bc = bytes_copied;
	252	+ return 0;
	253	+}
	254	+
	255	+int mlx5_ib_read_wqe_rq(struct mlx5_ib_qp qp, int wqe_index, void buffer,
	256	+ size_t buflen, size_t *bc)
	257	+{
	258	+ struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
	259	+ struct ib_umem *umem = base->ubuffer.umem;
	260	+ struct mlx5_ib_wq *wq = &qp->rq;
	261	+ size_t wqe_size = 1 << wq->wqe_shift;
	262	+
	263	+ if (buflen < wqe_size)
	264	+ return -EINVAL;
	265	+
	266	+ if (!umem)
	267	+ return -EOPNOTSUPP;
	268	+
	269	+ return mlx5_ib_read_user_wqe_rq(qp, wqe_index, buffer, buflen, bc);
	270	+}
	271	+
	272	+static int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index,
	273	+ void buffer, size_t buflen, size_t bc)
	274	+{
	275	+ struct ib_umem *umem = srq->umem;
	276	+ size_t bytes_copied;
	277	+ int ret;
	278	+
	279	+ ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index, 0,
	280	+ srq->msrq.max, srq->msrq.wqe_shift,
	281	+ buflen, &bytes_copied);
	282	+
	283	+ if (ret)
	284	+ return ret;
	285	+ *bc = bytes_copied;
	286	+ return 0;
	287	+}
	288	+
	289	+int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq srq, int wqe_index, void buffer,
	290	+ size_t buflen, size_t *bc)
	291	+{
	292	+ struct ib_umem *umem = srq->umem;
	293	+ size_t wqe_size = 1 << srq->msrq.wqe_shift;
	294	+
	295	+ if (buflen < wqe_size)
	296	+ return -EINVAL;
	297	+
	298	+ if (!umem)
	299	+ return -EOPNOTSUPP;
	300	+
	301	+ return mlx5_ib_read_user_wqe_srq(srq, wqe_index, buffer, buflen, bc);
195	302	}
196	303
197	304	static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)
..	..	@@ -258,17 +365,26 @@
258	365	cap->max_recv_wr = 0;
259	366	cap->max_recv_sge = 0;
260	367	} else {
	368	+ int wq_sig = !!(qp->flags_en & MLX5_QP_FLAG_SIGNATURE);
	369	+
261	370	if (ucmd) {
262	371	qp->rq.wqe_cnt = ucmd->rq_wqe_count;
263	372	if (ucmd->rq_wqe_shift > BITS_PER_BYTE * sizeof(ucmd->rq_wqe_shift))
264	373	return -EINVAL;
265	374	qp->rq.wqe_shift = ucmd->rq_wqe_shift;
266		- if ((1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) < qp->wq_sig)
	375	+ if ((1 << qp->rq.wqe_shift) /
	376	+ sizeof(struct mlx5_wqe_data_seg) <
	377	+ wq_sig)
267	378	return -EINVAL;
268		- qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig;
	379	+ qp->rq.max_gs =
	380	+ (1 << qp->rq.wqe_shift) /
	381	+ sizeof(struct mlx5_wqe_data_seg) -
	382	+ wq_sig;
269	383	qp->rq.max_post = qp->rq.wqe_cnt;
270	384	} else {
271		- wqe_size = qp->wq_sig ? sizeof(struct mlx5_wqe_signature_seg) : 0;
	385	+ wqe_size =
	386	+ wq_sig ? sizeof(struct mlx5_wqe_signature_seg) :
	387	+ 0;
272	388	wqe_size += cap->max_recv_sge * sizeof(struct mlx5_wqe_data_seg);
273	389	wqe_size = roundup_pow_of_two(wqe_size);
274	390	wq_size = roundup_pow_of_two(cap->max_recv_wr) * wqe_size;
..	..	@@ -282,7 +398,10 @@
282	398	return -EINVAL;
283	399	}
284	400	qp->rq.wqe_shift = ilog2(wqe_size);
285		- qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig;
	401	+ qp->rq.max_gs =
	402	+ (1 << qp->rq.wqe_shift) /
	403	+ sizeof(struct mlx5_wqe_data_seg) -
	404	+ wq_sig;
286	405	qp->rq.max_post = qp->rq.wqe_cnt;
287	406	}
288	407	}
..	..	@@ -297,7 +416,7 @@
297	416	switch (attr->qp_type) {
298	417	case IB_QPT_XRC_INI:
299	418	size += sizeof(struct mlx5_wqe_xrc_seg);
300		- /* fall through */
	419	+ fallthrough;
301	420	case IB_QPT_RC:
302	421	size += sizeof(struct mlx5_wqe_ctrl_seg) +
303	422	max(sizeof(struct mlx5_wqe_atomic_seg) +
..	..	@@ -322,7 +441,7 @@
322	441	if (attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
323	442	size += sizeof(struct mlx5_wqe_eth_pad) +
324	443	sizeof(struct mlx5_wqe_eth_seg);
325		- /* fall through */
	444	+ fallthrough;
326	445	case IB_QPT_SMI:
327	446	case MLX5_IB_QPT_HW_GSI:
328	447	size += sizeof(struct mlx5_wqe_ctrl_seg) +
..	..	@@ -357,9 +476,9 @@
357	476	}
358	477
359	478	size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);
360		- if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN &&
	479	+ if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN &&
361	480	ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE)
362		- return MLX5_SIG_WQE_SIZE;
	481	+ return MLX5_SIG_WQE_SIZE;
363	482	else
364	483	return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
365	484	}
..	..	@@ -411,9 +530,6 @@
411	530	sizeof(struct mlx5_wqe_inline_seg);
412	531	attr->cap.max_inline_data = qp->max_inline_data;
413	532
414		- if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN)
415		- qp->signature_en = true;
416		-
417	533	wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
418	534	qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
419	535	if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
..	..	@@ -449,9 +565,9 @@
449	565	return -EINVAL;
450	566	}
451	567
452		- if (ucmd->sq_wqe_count && ((1 << ilog2(ucmd->sq_wqe_count)) != ucmd->sq_wqe_count)) {
453		- mlx5_ib_warn(dev, "sq_wqe_count %d, sq_wqe_count %d\n",
454		- ucmd->sq_wqe_count, ucmd->sq_wqe_count);
	568	+ if (ucmd->sq_wqe_count && !is_power_of_2(ucmd->sq_wqe_count)) {
	569	+ mlx5_ib_warn(dev, "sq_wqe_count %d is not a power of two\n",
	570	+ ucmd->sq_wqe_count);
455	571	return -EINVAL;
456	572	}
457	573
..	..	@@ -465,7 +581,7 @@
465	581	}
466	582
467	583	if (attr->qp_type == IB_QPT_RAW_PACKET \|\|
468		- qp->flags & MLX5_IB_QP_UNDERLAY) {
	584	+ qp->flags & IB_QP_CREATE_SOURCE_QPN) {
469	585	base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift;
470	586	qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6;
471	587	} else {
..	..	@@ -567,6 +683,9 @@
567	683	{
568	684	int bfregn = -ENOMEM;
569	685
	686	+ if (bfregi->lib_uar_dyn)
	687	+ return -EINVAL;
	688	+
570	689	mutex_lock(&bfregi->lock);
571	690	if (bfregi->ver >= 2) {
572	691	bfregn = alloc_high_class_bfreg(dev, bfregi);
..	..	@@ -617,10 +736,7 @@
617	736	case IB_QPT_SMI: return MLX5_QP_ST_QP0;
618	737	case MLX5_IB_QPT_HW_GSI: return MLX5_QP_ST_QP1;
619	738	case MLX5_IB_QPT_DCI: return MLX5_QP_ST_DCI;
620		- case IB_QPT_RAW_IPV6: return MLX5_QP_ST_RAW_IPV6;
621		- case IB_QPT_RAW_PACKET:
622		- case IB_QPT_RAW_ETHERTYPE: return MLX5_QP_ST_RAW_ETHERTYPE;
623		- case IB_QPT_MAX:
	739	+ case IB_QPT_RAW_PACKET: return MLX5_QP_ST_RAW_ETHERTYPE;
624	740	default: return -EINVAL;
625	741	}
626	742	}
..	..	@@ -637,6 +753,9 @@
637	753	unsigned int bfregs_per_sys_page;
638	754	u32 index_of_sys_page;
639	755	u32 offset;
	756	+
	757	+ if (bfregi->lib_uar_dyn)
	758	+ return -EINVAL;
640	759
641	760	bfregs_per_sys_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
642	761	MLX5_NON_FP_BFREGS_PER_UAR;
..	..	@@ -659,16 +778,14 @@
659	778	return bfregi->sys_pages[index_of_sys_page] + offset;
660	779	}
661	780
662		-static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev,
663		- struct ib_pd *pd,
	781	+static int mlx5_ib_umem_get(struct mlx5_ib_dev dev, struct ib_udata udata,
664	782	unsigned long addr, size_t size,
665		- struct ib_umem **umem,
666		- int npages, int page_shift, int *ncont,
667		- u32 *offset)
	783	+ struct ib_umem *umem, int npages, int *page_shift,
	784	+ int ncont, u32 offset)
668	785	{
669	786	int err;
670	787
671		- *umem = ib_umem_get(pd->uobject->context, addr, size, 0, 0);
	788	+ *umem = ib_umem_get(&dev->ib_dev, addr, size, 0);
672	789	if (IS_ERR(*umem)) {
673	790	mlx5_ib_dbg(dev, "umem_get failed\n");
674	791	return PTR_ERR(*umem);
..	..	@@ -695,24 +812,27 @@
695	812	}
696	813
697	814	static void destroy_user_rq(struct mlx5_ib_dev dev, struct ib_pd pd,
698		- struct mlx5_ib_rwq *rwq)
	815	+ struct mlx5_ib_rwq rwq, struct ib_udata udata)
699	816	{
700		- struct mlx5_ib_ucontext *context;
	817	+ struct mlx5_ib_ucontext *context =
	818	+ rdma_udata_to_drv_context(
	819	+ udata,
	820	+ struct mlx5_ib_ucontext,
	821	+ ibucontext);
701	822
702	823	if (rwq->create_flags & MLX5_IB_WQ_FLAGS_DELAY_DROP)
703	824	atomic_dec(&dev->delay_drop.rqs_cnt);
704	825
705		- context = to_mucontext(pd->uobject->context);
706	826	mlx5_ib_db_unmap_user(context, &rwq->db);
707		- if (rwq->umem)
708		- ib_umem_release(rwq->umem);
	827	+ ib_umem_release(rwq->umem);
709	828	}
710	829
711	830	static int create_user_rq(struct mlx5_ib_dev dev, struct ib_pd pd,
712		- struct mlx5_ib_rwq *rwq,
	831	+ struct ib_udata udata, struct mlx5_ib_rwq rwq,
713	832	struct mlx5_ib_create_wq *ucmd)
714	833	{
715		- struct mlx5_ib_ucontext *context;
	834	+ struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
	835	+ udata, struct mlx5_ib_ucontext, ibucontext);
716	836	int page_shift = 0;
717	837	int npages;
718	838	u32 offset = 0;
..	..	@@ -722,9 +842,7 @@
722	842	if (!ucmd->buf_addr)
723	843	return -EINVAL;
724	844
725		- context = to_mucontext(pd->uobject->context);
726		- rwq->umem = ib_umem_get(pd->uobject->context, ucmd->buf_addr,
727		- rwq->buf_size, 0, 0);
	845	+ rwq->umem = ib_umem_get(&dev->ib_dev, ucmd->buf_addr, rwq->buf_size, 0);
728	846	if (IS_ERR(rwq->umem)) {
729	847	mlx5_ib_dbg(dev, "umem_get failed\n");
730	848	err = PTR_ERR(rwq->umem);
..	..	@@ -749,13 +867,12 @@
749	867	(unsigned long long)ucmd->buf_addr, rwq->buf_size,
750	868	npages, page_shift, ncont, offset);
751	869
752		- err = mlx5_ib_db_map_user(context, ucmd->db_addr, &rwq->db);
	870	+ err = mlx5_ib_db_map_user(ucontext, udata, ucmd->db_addr, &rwq->db);
753	871	if (err) {
754	872	mlx5_ib_dbg(dev, "map failed\n");
755	873	goto err_umem;
756	874	}
757	875
758		- rwq->create_type = MLX5_WQ_USER;
759	876	return 0;
760	877
761	878	err_umem:
..	..	@@ -770,15 +887,14 @@
770	887	bfregn % MLX5_NON_FP_BFREGS_PER_UAR;
771	888	}
772	889
773		-static int create_user_qp(struct mlx5_ib_dev dev, struct ib_pd pd,
774		- struct mlx5_ib_qp qp, struct ib_udata udata,
775		- struct ib_qp_init_attr *attr,
776		- u32 **in,
777		- struct mlx5_ib_create_qp_resp resp, int inlen,
778		- struct mlx5_ib_qp_base *base)
	890	+static int _create_user_qp(struct mlx5_ib_dev dev, struct ib_pd pd,
	891	+ struct mlx5_ib_qp qp, struct ib_udata udata,
	892	+ struct ib_qp_init_attr attr, u32 *in,
	893	+ struct mlx5_ib_create_qp_resp resp, int inlen,
	894	+ struct mlx5_ib_qp_base *base,
	895	+ struct mlx5_ib_create_qp *ucmd)
779	896	{
780	897	struct mlx5_ib_ucontext *context;
781		- struct mlx5_ib_create_qp ucmd;
782	898	struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer;
783	899	int page_shift = 0;
784	900	int uar_index = 0;
..	..	@@ -789,32 +905,34 @@
789	905	__be64 *pas;
790	906	void *qpc;
791	907	int err;
	908	+ u16 uid;
	909	+ u32 uar_flags;
792	910
793		- err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
794		- if (err) {
795		- mlx5_ib_dbg(dev, "copy failed\n");
796		- return err;
797		- }
798		-
799		- context = to_mucontext(pd->uobject->context);
800		- if (ucmd.flags & MLX5_QP_FLAG_BFREG_INDEX) {
	911	+ context = rdma_udata_to_drv_context(udata, struct mlx5_ib_ucontext,
	912	+ ibucontext);
	913	+ uar_flags = qp->flags_en &
	914	+ (MLX5_QP_FLAG_UAR_PAGE_INDEX \| MLX5_QP_FLAG_BFREG_INDEX);
	915	+ switch (uar_flags) {
	916	+ case MLX5_QP_FLAG_UAR_PAGE_INDEX:
	917	+ uar_index = ucmd->bfreg_index;
	918	+ bfregn = MLX5_IB_INVALID_BFREG;
	919	+ break;
	920	+ case MLX5_QP_FLAG_BFREG_INDEX:
801	921	uar_index = bfregn_to_uar_index(dev, &context->bfregi,
802		- ucmd.bfreg_index, true);
	922	+ ucmd->bfreg_index, true);
803	923	if (uar_index < 0)
804	924	return uar_index;
805		-
806	925	bfregn = MLX5_IB_INVALID_BFREG;
807		- } else if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL) {
808		- /*
809		- * TBD: should come from the verbs when we have the API
810		- */
811		- /* In CROSS_CHANNEL CQ and QP must use the same UAR */
812		- bfregn = MLX5_CROSS_CHANNEL_BFREG;
813		- }
814		- else {
	926	+ break;
	927	+ case 0:
	928	+ if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
	929	+ return -EINVAL;
815	930	bfregn = alloc_bfreg(dev, &context->bfregi);
816	931	if (bfregn < 0)
817	932	return bfregn;
	933	+ break;
	934	+ default:
	935	+ return -EINVAL;
818	936	}
819	937
820	938	mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);
..	..	@@ -826,16 +944,15 @@
826	944	qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
827	945	qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
828	946
829		- err = set_user_buf_size(dev, qp, &ucmd, base, attr);
	947	+ err = set_user_buf_size(dev, qp, ucmd, base, attr);
830	948	if (err)
831	949	goto err_bfreg;
832	950
833		- if (ucmd.buf_addr && ubuffer->buf_size) {
834		- ubuffer->buf_addr = ucmd.buf_addr;
835		- err = mlx5_ib_umem_get(dev, pd, ubuffer->buf_addr,
836		- ubuffer->buf_size,
837		- &ubuffer->umem, &npages, &page_shift,
838		- &ncont, &offset);
	951	+ if (ucmd->buf_addr && ubuffer->buf_size) {
	952	+ ubuffer->buf_addr = ucmd->buf_addr;
	953	+ err = mlx5_ib_umem_get(dev, udata, ubuffer->buf_addr,
	954	+ ubuffer->buf_size, &ubuffer->umem,
	955	+ &npages, &page_shift, &ncont, &offset);
839	956	if (err)
840	957	goto err_bfreg;
841	958	} else {
..	..	@@ -850,6 +967,8 @@
850	967	goto err_umem;
851	968	}
852	969
	970	+ uid = (attr->qp_type != IB_QPT_XRC_INI) ? to_mpd(pd)->uid : 0;
	971	+ MLX5_SET(create_qp_in, *in, uid, uid);
853	972	pas = (__be64 )MLX5_ADDR_OF(create_qp_in, in, pas);
854	973	if (ubuffer->umem)
855	974	mlx5_ib_populate_pas(dev, ubuffer->umem, page_shift, pas, 0);
..	..	@@ -866,30 +985,19 @@
866	985	resp->bfreg_index = MLX5_IB_INVALID_BFREG;
867	986	qp->bfregn = bfregn;
868	987
869		- err = mlx5_ib_db_map_user(context, ucmd.db_addr, &qp->db);
	988	+ err = mlx5_ib_db_map_user(context, udata, ucmd->db_addr, &qp->db);
870	989	if (err) {
871	990	mlx5_ib_dbg(dev, "map failed\n");
872	991	goto err_free;
873	992	}
874	993
875		- err = ib_copy_to_udata(udata, resp, min(udata->outlen, sizeof(*resp)));
876		- if (err) {
877		- mlx5_ib_dbg(dev, "copy failed\n");
878		- goto err_unmap;
879		- }
880		- qp->create_type = MLX5_QP_USER;
881		-
882	994	return 0;
883		-
884		-err_unmap:
885		- mlx5_ib_db_unmap_user(context, &qp->db);
886	995
887	996	err_free:
888	997	kvfree(*in);
889	998
890	999	err_umem:
891		- if (ubuffer->umem)
892		- ib_umem_release(ubuffer->umem);
	1000	+ ib_umem_release(ubuffer->umem);
893	1001
894	1002	err_bfreg:
895	1003	if (bfregn != MLX5_IB_INVALID_BFREG)
..	..	@@ -897,43 +1005,51 @@
897	1005	return err;
898	1006	}
899	1007
900		-static void destroy_qp_user(struct mlx5_ib_dev dev, struct ib_pd pd,
901		- struct mlx5_ib_qp qp, struct mlx5_ib_qp_base base)
	1008	+static void destroy_qp(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp,
	1009	+ struct mlx5_ib_qp_base base, struct ib_udata udata)
902	1010	{
903		- struct mlx5_ib_ucontext *context;
	1011	+ struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
	1012	+ udata, struct mlx5_ib_ucontext, ibucontext);
904	1013
905		- context = to_mucontext(pd->uobject->context);
906		- mlx5_ib_db_unmap_user(context, &qp->db);
907		- if (base->ubuffer.umem)
	1014	+ if (udata) {
	1015	+ /* User QP */
	1016	+ mlx5_ib_db_unmap_user(context, &qp->db);
908	1017	ib_umem_release(base->ubuffer.umem);
909	1018
910		- /*
911		- * Free only the BFREGs which are handled by the kernel.
912		- * BFREGs of UARs allocated dynamically are handled by user.
913		- */
914		- if (qp->bfregn != MLX5_IB_INVALID_BFREG)
915		- mlx5_ib_free_bfreg(dev, &context->bfregi, qp->bfregn);
	1019	+ /*
	1020	+ * Free only the BFREGs which are handled by the kernel.
	1021	+ * BFREGs of UARs allocated dynamically are handled by user.
	1022	+ */
	1023	+ if (qp->bfregn != MLX5_IB_INVALID_BFREG)
	1024	+ mlx5_ib_free_bfreg(dev, &context->bfregi, qp->bfregn);
	1025	+ return;
	1026	+ }
	1027	+
	1028	+ /* Kernel QP */
	1029	+ kvfree(qp->sq.wqe_head);
	1030	+ kvfree(qp->sq.w_list);
	1031	+ kvfree(qp->sq.wrid);
	1032	+ kvfree(qp->sq.wr_data);
	1033	+ kvfree(qp->rq.wrid);
	1034	+ if (qp->db.db)
	1035	+ mlx5_db_free(dev->mdev, &qp->db);
	1036	+ if (qp->buf.frags)
	1037	+ mlx5_frag_buf_free(dev->mdev, &qp->buf);
916	1038	}
917	1039
918		-static int create_kernel_qp(struct mlx5_ib_dev *dev,
919		- struct ib_qp_init_attr *init_attr,
920		- struct mlx5_ib_qp *qp,
921		- u32 *in, int inlen,
922		- struct mlx5_ib_qp_base *base)
	1040	+static int _create_kernel_qp(struct mlx5_ib_dev *dev,
	1041	+ struct ib_qp_init_attr *init_attr,
	1042	+ struct mlx5_ib_qp qp, u32 in, int inlen,
	1043	+ struct mlx5_ib_qp_base *base)
923	1044	{
924	1045	int uar_index;
925	1046	void *qpc;
926	1047	int err;
927	1048
928		- if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN \|
929		- IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK \|
930		- IB_QP_CREATE_IPOIB_UD_LSO \|
931		- IB_QP_CREATE_NETIF_QP \|
932		- mlx5_ib_create_qp_sqpn_qp1()))
933		- return -EINVAL;
934		-
935	1049	if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
936	1050	qp->bf.bfreg = &dev->fp_bfreg;
	1051	+ else if (qp->flags & MLX5_IB_QP_CREATE_WC_TEST)
	1052	+ qp->bf.bfreg = &dev->wc_bfreg;
937	1053	else
938	1054	qp->bf.bfreg = &dev->bfreg;
939	1055
..	..	@@ -953,13 +1069,29 @@
953	1069	qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
954	1070	base->ubuffer.buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift);
955	1071
956		- err = mlx5_buf_alloc(dev->mdev, base->ubuffer.buf_size, &qp->buf);
	1072	+ err = mlx5_frag_buf_alloc_node(dev->mdev, base->ubuffer.buf_size,
	1073	+ &qp->buf, dev->mdev->priv.numa_node);
957	1074	if (err) {
958	1075	mlx5_ib_dbg(dev, "err %d\n", err);
959	1076	return err;
960	1077	}
961	1078
962		- qp->sq.qend = mlx5_get_send_wqe(qp, qp->sq.wqe_cnt);
	1079	+ if (qp->rq.wqe_cnt)
	1080	+ mlx5_init_fbc(qp->buf.frags, qp->rq.wqe_shift,
	1081	+ ilog2(qp->rq.wqe_cnt), &qp->rq.fbc);
	1082	+
	1083	+ if (qp->sq.wqe_cnt) {
	1084	+ int sq_strides_offset = (qp->sq.offset & (PAGE_SIZE - 1)) /
	1085	+ MLX5_SEND_WQE_BB;
	1086	+ mlx5_init_fbc_offset(qp->buf.frags +
	1087	+ (qp->sq.offset / PAGE_SIZE),
	1088	+ ilog2(MLX5_SEND_WQE_BB),
	1089	+ ilog2(qp->sq.wqe_cnt),
	1090	+ sq_strides_offset, &qp->sq.fbc);
	1091	+
	1092	+ qp->sq.cur_edge = get_sq_edge(&qp->sq, 0);
	1093	+ }
	1094	+
963	1095	*inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
964	1096	MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * qp->buf.npages;
965	1097	in = kvzalloc(inlen, GFP_KERNEL);
..	..	@@ -976,13 +1108,12 @@
976	1108	MLX5_SET(qpc, qpc, fre, 1);
977	1109	MLX5_SET(qpc, qpc, rlky, 1);
978	1110
979		- if (init_attr->create_flags & mlx5_ib_create_qp_sqpn_qp1()) {
	1111	+ if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
980	1112	MLX5_SET(qpc, qpc, deth_sqpn, 1);
981		- qp->flags \|= MLX5_IB_QP_SQPN_QP1;
982		- }
983	1113
984		- mlx5_fill_page_array(&qp->buf,
985		- (__be64 )MLX5_ADDR_OF(create_qp_in, in, pas));
	1114	+ mlx5_fill_page_frag_array(&qp->buf,
	1115	+ (__be64 *)MLX5_ADDR_OF(create_qp_in,
	1116	+ *in, pas));
986	1117
987	1118	err = mlx5_db_alloc(dev->mdev, &qp->db);
988	1119	if (err) {
..	..	@@ -1006,7 +1137,6 @@
1006	1137	err = -ENOMEM;
1007	1138	goto err_wrid;
1008	1139	}
1009		- qp->create_type = MLX5_QP_KERNEL;
1010	1140
1011	1141	return 0;
1012	1142
..	..	@@ -1022,69 +1152,52 @@
1022	1152	kvfree(*in);
1023	1153
1024	1154	err_buf:
1025		- mlx5_buf_free(dev->mdev, &qp->buf);
	1155	+ mlx5_frag_buf_free(dev->mdev, &qp->buf);
1026	1156	return err;
1027		-}
1028		-
1029		-static void destroy_qp_kernel(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp)
1030		-{
1031		- kvfree(qp->sq.wqe_head);
1032		- kvfree(qp->sq.w_list);
1033		- kvfree(qp->sq.wrid);
1034		- kvfree(qp->sq.wr_data);
1035		- kvfree(qp->rq.wrid);
1036		- mlx5_db_free(dev->mdev, &qp->db);
1037		- mlx5_buf_free(dev->mdev, &qp->buf);
1038	1157	}
1039	1158
1040	1159	static u32 get_rx_type(struct mlx5_ib_qp qp, struct ib_qp_init_attr attr)
1041	1160	{
1042		- if (attr->srq \|\| (attr->qp_type == IB_QPT_XRC_TGT) \|\|
1043		- (attr->qp_type == MLX5_IB_QPT_DCI) \|\|
1044		- (attr->qp_type == IB_QPT_XRC_INI))
	1161	+ if (attr->srq \|\| (qp->type == IB_QPT_XRC_TGT) \|\|
	1162	+ (qp->type == MLX5_IB_QPT_DCI) \|\| (qp->type == IB_QPT_XRC_INI))
1045	1163	return MLX5_SRQ_RQ;
1046	1164	else if (!qp->has_rq)
1047	1165	return MLX5_ZERO_LEN_RQ;
1048		- else
1049		- return MLX5_NON_ZERO_RQ;
1050		-}
1051	1166
1052		-static int is_connected(enum ib_qp_type qp_type)
1053		-{
1054		- if (qp_type == IB_QPT_RC \|\| qp_type == IB_QPT_UC)
1055		- return 1;
1056		-
1057		- return 0;
	1167	+ return MLX5_NON_ZERO_RQ;
1058	1168	}
1059	1169
1060	1170	static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1061	1171	struct mlx5_ib_qp *qp,
1062		- struct mlx5_ib_sq *sq, u32 tdn)
	1172	+ struct mlx5_ib_sq *sq, u32 tdn,
	1173	+ struct ib_pd *pd)
1063	1174	{
1064		- u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
	1175	+ u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
1065	1176	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
1066	1177
	1178	+ MLX5_SET(create_tis_in, in, uid, to_mpd(pd)->uid);
1067	1179	MLX5_SET(tisc, tisc, transport_domain, tdn);
1068		- if (qp->flags & MLX5_IB_QP_UNDERLAY)
	1180	+ if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
1069	1181	MLX5_SET(tisc, tisc, underlay_qpn, qp->underlay_qpn);
1070	1182
1071		- return mlx5_core_create_tis(dev->mdev, in, sizeof(in), &sq->tisn);
	1183	+ return mlx5_core_create_tis(dev->mdev, in, &sq->tisn);
1072	1184	}
1073	1185
1074	1186	static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1075		- struct mlx5_ib_sq *sq)
	1187	+ struct mlx5_ib_sq sq, struct ib_pd pd)
1076	1188	{
1077		- mlx5_core_destroy_tis(dev->mdev, sq->tisn);
	1189	+ mlx5_cmd_destroy_tis(dev->mdev, sq->tisn, to_mpd(pd)->uid);
1078	1190	}
1079	1191
1080		-static void destroy_flow_rule_vport_sq(struct mlx5_ib_dev *dev,
1081		- struct mlx5_ib_sq *sq)
	1192	+static void destroy_flow_rule_vport_sq(struct mlx5_ib_sq *sq)
1082	1193	{
1083	1194	if (sq->flow_rule)
1084	1195	mlx5_del_flow_rules(sq->flow_rule);
	1196	+ sq->flow_rule = NULL;
1085	1197	}
1086	1198
1087	1199	static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
	1200	+ struct ib_udata *udata,
1088	1201	struct mlx5_ib_sq sq, void qpin,
1089	1202	struct ib_pd *pd)
1090	1203	{
..	..	@@ -1101,9 +1214,9 @@
1101	1214	int ncont = 0;
1102	1215	u32 offset = 0;
1103	1216
1104		- err = mlx5_ib_umem_get(dev, pd, ubuffer->buf_addr, ubuffer->buf_size,
1105		- &sq->ubuffer.umem, &npages, &page_shift,
1106		- &ncont, &offset);
	1217	+ err = mlx5_ib_umem_get(dev, udata, ubuffer->buf_addr, ubuffer->buf_size,
	1218	+ &sq->ubuffer.umem, &npages, &page_shift, &ncont,
	1219	+ &offset);
1107	1220	if (err)
1108	1221	return err;
1109	1222
..	..	@@ -1114,6 +1227,7 @@
1114	1227	goto err_umem;
1115	1228	}
1116	1229
	1230	+ MLX5_SET(create_sq_in, in, uid, to_mpd(pd)->uid);
1117	1231	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1118	1232	MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1119	1233	if (MLX5_CAP_ETH(dev->mdev, multi_pkt_send_wqe))
..	..	@@ -1140,21 +1254,14 @@
1140	1254	pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1141	1255	mlx5_ib_populate_pas(dev, sq->ubuffer.umem, page_shift, pas, 0);
1142	1256
1143		- err = mlx5_core_create_sq_tracked(dev->mdev, in, inlen, &sq->base.mqp);
	1257	+ err = mlx5_core_create_sq_tracked(dev, in, inlen, &sq->base.mqp);
1144	1258
1145	1259	kvfree(in);
1146	1260
1147	1261	if (err)
1148	1262	goto err_umem;
1149	1263
1150		- err = create_flow_rule_vport_sq(dev, sq);
1151		- if (err)
1152		- goto err_flow;
1153		-
1154	1264	return 0;
1155		-
1156		-err_flow:
1157		- mlx5_core_destroy_sq_tracked(dev->mdev, &sq->base.mqp);
1158	1265
1159	1266	err_umem:
1160	1267	ib_umem_release(sq->ubuffer.umem);
..	..	@@ -1166,8 +1273,8 @@
1166	1273	static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1167	1274	struct mlx5_ib_sq *sq)
1168	1275	{
1169		- destroy_flow_rule_vport_sq(dev, sq);
1170		- mlx5_core_destroy_sq_tracked(dev->mdev, &sq->base.mqp);
	1276	+ destroy_flow_rule_vport_sq(sq);
	1277	+ mlx5_core_destroy_sq_tracked(dev, &sq->base.mqp);
1171	1278	ib_umem_release(sq->ubuffer.umem);
1172	1279	}
1173	1280
..	..	@@ -1188,7 +1295,7 @@
1188	1295
1189	1296	static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1190	1297	struct mlx5_ib_rq rq, void qpin,
1191		- size_t qpinlen)
	1298	+ size_t qpinlen, struct ib_pd *pd)
1192	1299	{
1193	1300	struct mlx5_ib_qp *mqp = rq->base.container_mibqp;
1194	1301	__be64 *pas;
..	..	@@ -1209,6 +1316,7 @@
1209	1316	if (!in)
1210	1317	return -ENOMEM;
1211	1318
	1319	+ MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
1212	1320	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1213	1321	if (!(rq->flags & MLX5_IB_RQ_CVLAN_STRIPPING))
1214	1322	MLX5_SET(rqc, rqc, vsd, 1);
..	..	@@ -1218,7 +1326,7 @@
1218	1326	MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index));
1219	1327	MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv));
1220	1328
1221		- if (mqp->flags & MLX5_IB_QP_CAP_SCATTER_FCS)
	1329	+ if (mqp->flags & IB_QP_CREATE_SCATTER_FCS)
1222	1330	MLX5_SET(rqc, rqc, scatter_fcs, 1);
1223	1331
1224	1332	wq = MLX5_ADDR_OF(rqc, rqc, wq);
..	..	@@ -1236,7 +1344,7 @@
1236	1344	qp_pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, qpin, pas);
1237	1345	memcpy(pas, qp_pas, rq_pas_size);
1238	1346
1239		- err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rq->base.mqp);
	1347	+ err = mlx5_core_create_rq_tracked(dev, in, inlen, &rq->base.mqp);
1240	1348
1241	1349	kvfree(in);
1242	1350
..	..	@@ -1246,20 +1354,26 @@
1246	1354	static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1247	1355	struct mlx5_ib_rq *rq)
1248	1356	{
1249		- mlx5_core_destroy_rq_tracked(dev->mdev, &rq->base.mqp);
	1357	+ mlx5_core_destroy_rq_tracked(dev, &rq->base.mqp);
1250	1358	}
1251	1359
1252		-static bool tunnel_offload_supported(struct mlx5_core_dev *dev)
	1360	+static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
	1361	+ struct mlx5_ib_rq *rq,
	1362	+ u32 qp_flags_en,
	1363	+ struct ib_pd *pd)
1253	1364	{
1254		- return (MLX5_CAP_ETH(dev, tunnel_stateless_vxlan) \|\|
1255		- MLX5_CAP_ETH(dev, tunnel_stateless_gre) \|\|
1256		- MLX5_CAP_ETH(dev, tunnel_stateless_geneve_rx));
	1365	+ if (qp_flags_en & (MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC \|
	1366	+ MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC))
	1367	+ mlx5_ib_disable_lb(dev, false, true);
	1368	+ mlx5_cmd_destroy_tir(dev->mdev, rq->tirn, to_mpd(pd)->uid);
1257	1369	}
1258	1370
1259	1371	static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1260	1372	struct mlx5_ib_rq *rq, u32 tdn,
1261		- bool tunnel_offload_en)
	1373	+ u32 qp_flags_en, struct ib_pd pd,
	1374	+ u32 *out)
1262	1375	{
	1376	+ u8 lb_flag = 0;
1263	1377	u32 *in;
1264	1378	void *tirc;
1265	1379	int inlen;
..	..	@@ -1270,51 +1384,73 @@
1270	1384	if (!in)
1271	1385	return -ENOMEM;
1272	1386
	1387	+ MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
1273	1388	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1274	1389	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
1275	1390	MLX5_SET(tirc, tirc, inline_rqn, rq->base.mqp.qpn);
1276	1391	MLX5_SET(tirc, tirc, transport_domain, tdn);
1277		- if (tunnel_offload_en)
	1392	+ if (*qp_flags_en & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
1278	1393	MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
1279	1394
1280		- if (dev->rep)
1281		- MLX5_SET(tirc, tirc, self_lb_block,
1282		- MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST_);
	1395	+ if (*qp_flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)
	1396	+ lb_flag \|= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1283	1397
1284		- err = mlx5_core_create_tir(dev->mdev, in, inlen, &rq->tirn);
	1398	+ if (*qp_flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
	1399	+ lb_flag \|= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
1285	1400
	1401	+ if (dev->is_rep) {
	1402	+ lb_flag \|= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
	1403	+ *qp_flags_en \|= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
	1404	+ }
	1405	+
	1406	+ MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
	1407	+ MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
	1408	+ err = mlx5_cmd_exec_inout(dev->mdev, create_tir, in, out);
	1409	+ rq->tirn = MLX5_GET(create_tir_out, out, tirn);
	1410	+ if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
	1411	+ err = mlx5_ib_enable_lb(dev, false, true);
	1412	+
	1413	+ if (err)
	1414	+ destroy_raw_packet_qp_tir(dev, rq, 0, pd);
	1415	+ }
1286	1416	kvfree(in);
1287	1417
1288	1418	return err;
1289	1419	}
1290	1420
1291		-static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1292		- struct mlx5_ib_rq *rq)
1293		-{
1294		- mlx5_core_destroy_tir(dev->mdev, rq->tirn);
1295		-}
1296		-
1297	1421	static int create_raw_packet_qp(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp,
1298	1422	u32 *in, size_t inlen,
1299		- struct ib_pd *pd)
	1423	+ struct ib_pd *pd,
	1424	+ struct ib_udata *udata,
	1425	+ struct mlx5_ib_create_qp_resp *resp)
1300	1426	{
1301	1427	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1302	1428	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1303	1429	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1304		- struct ib_uobject *uobj = pd->uobject;
1305		- struct ib_ucontext *ucontext = uobj->context;
1306		- struct mlx5_ib_ucontext *mucontext = to_mucontext(ucontext);
	1430	+ struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(
	1431	+ udata, struct mlx5_ib_ucontext, ibucontext);
1307	1432	int err;
1308	1433	u32 tdn = mucontext->tdn;
	1434	+ u16 uid = to_mpd(pd)->uid;
	1435	+ u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
1309	1436
	1437	+ if (!qp->sq.wqe_cnt && !qp->rq.wqe_cnt)
	1438	+ return -EINVAL;
1310	1439	if (qp->sq.wqe_cnt) {
1311		- err = create_raw_packet_qp_tis(dev, qp, sq, tdn);
	1440	+ err = create_raw_packet_qp_tis(dev, qp, sq, tdn, pd);
1312	1441	if (err)
1313	1442	return err;
1314	1443
1315		- err = create_raw_packet_qp_sq(dev, sq, in, pd);
	1444	+ err = create_raw_packet_qp_sq(dev, udata, sq, in, pd);
1316	1445	if (err)
1317	1446	goto err_destroy_tis;
	1447	+
	1448	+ if (uid) {
	1449	+ resp->tisn = sq->tisn;
	1450	+ resp->comp_mask \|= MLX5_IB_CREATE_QP_RESP_MASK_TISN;
	1451	+ resp->sqn = sq->base.mqp.qpn;
	1452	+ resp->comp_mask \|= MLX5_IB_CREATE_QP_RESP_MASK_SQN;
	1453	+ }
1318	1454
1319	1455	sq->base.container_mibqp = qp;
1320	1456	sq->base.mqp.event = mlx5_ib_qp_event;
..	..	@@ -1323,24 +1459,44 @@
1323	1459	if (qp->rq.wqe_cnt) {
1324	1460	rq->base.container_mibqp = qp;
1325	1461
1326		- if (qp->flags & MLX5_IB_QP_CVLAN_STRIPPING)
	1462	+ if (qp->flags & IB_QP_CREATE_CVLAN_STRIPPING)
1327	1463	rq->flags \|= MLX5_IB_RQ_CVLAN_STRIPPING;
1328		- if (qp->flags & MLX5_IB_QP_PCI_WRITE_END_PADDING)
	1464	+ if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING)
1329	1465	rq->flags \|= MLX5_IB_RQ_PCI_WRITE_END_PADDING;
1330		- err = create_raw_packet_qp_rq(dev, rq, in, inlen);
	1466	+ err = create_raw_packet_qp_rq(dev, rq, in, inlen, pd);
1331	1467	if (err)
1332	1468	goto err_destroy_sq;
1333	1469
1334		-
1335		- err = create_raw_packet_qp_tir(dev, rq, tdn,
1336		- qp->tunnel_offload_en);
	1470	+ err = create_raw_packet_qp_tir(dev, rq, tdn, &qp->flags_en, pd,
	1471	+ out);
1337	1472	if (err)
1338	1473	goto err_destroy_rq;
	1474	+
	1475	+ if (uid) {
	1476	+ resp->rqn = rq->base.mqp.qpn;
	1477	+ resp->comp_mask \|= MLX5_IB_CREATE_QP_RESP_MASK_RQN;
	1478	+ resp->tirn = rq->tirn;
	1479	+ resp->comp_mask \|= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
	1480	+ if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) \|\|
	1481	+ MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2)) {
	1482	+ resp->tir_icm_addr = MLX5_GET(
	1483	+ create_tir_out, out, icm_address_31_0);
	1484	+ resp->tir_icm_addr \|=
	1485	+ (u64)MLX5_GET(create_tir_out, out,
	1486	+ icm_address_39_32)
	1487	+ << 32;
	1488	+ resp->tir_icm_addr \|=
	1489	+ (u64)MLX5_GET(create_tir_out, out,
	1490	+ icm_address_63_40)
	1491	+ << 40;
	1492	+ resp->comp_mask \|=
	1493	+ MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
	1494	+ }
	1495	+ }
1339	1496	}
1340	1497
1341	1498	qp->trans_qp.base.mqp.qpn = qp->sq.wqe_cnt ? sq->base.mqp.qpn :
1342	1499	rq->base.mqp.qpn;
1343		-
1344	1500	return 0;
1345	1501
1346	1502	err_destroy_rq:
..	..	@@ -1350,7 +1506,7 @@
1350	1506	return err;
1351	1507	destroy_raw_packet_qp_sq(dev, sq);
1352	1508	err_destroy_tis:
1353		- destroy_raw_packet_qp_tis(dev, sq);
	1509	+ destroy_raw_packet_qp_tis(dev, sq, pd);
1354	1510
1355	1511	return err;
1356	1512	}
..	..	@@ -1363,13 +1519,13 @@
1363	1519	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1364	1520
1365	1521	if (qp->rq.wqe_cnt) {
1366		- destroy_raw_packet_qp_tir(dev, rq);
	1522	+ destroy_raw_packet_qp_tir(dev, rq, qp->flags_en, qp->ibqp.pd);
1367	1523	destroy_raw_packet_qp_rq(dev, rq);
1368	1524	}
1369	1525
1370	1526	if (qp->sq.wqe_cnt) {
1371	1527	destroy_raw_packet_qp_sq(dev, sq);
1372		- destroy_raw_packet_qp_tis(dev, sq);
	1528	+ destroy_raw_packet_qp_tis(dev, sq, qp->ibqp.pd);
1373	1529	}
1374	1530	}
1375	1531
..	..	@@ -1387,91 +1543,74 @@
1387	1543
1388	1544	static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp)
1389	1545	{
1390		- mlx5_core_destroy_tir(dev->mdev, qp->rss_qp.tirn);
	1546	+ if (qp->flags_en & (MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC \|
	1547	+ MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC))
	1548	+ mlx5_ib_disable_lb(dev, false, true);
	1549	+ mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn,
	1550	+ to_mpd(qp->ibqp.pd)->uid);
1391	1551	}
1392	1552
1393		-static int create_rss_raw_qp_tir(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp,
1394		- struct ib_pd *pd,
1395		- struct ib_qp_init_attr *init_attr,
1396		- struct ib_udata *udata)
	1553	+struct mlx5_create_qp_params {
	1554	+ struct ib_udata *udata;
	1555	+ size_t inlen;
	1556	+ size_t outlen;
	1557	+ size_t ucmd_size;
	1558	+ void *ucmd;
	1559	+ u8 is_rss_raw : 1;
	1560	+ struct ib_qp_init_attr *attr;
	1561	+ u32 uidx;
	1562	+ struct mlx5_ib_create_qp_resp resp;
	1563	+};
	1564	+
	1565	+static int create_rss_raw_qp_tir(struct mlx5_ib_dev dev, struct ib_pd pd,
	1566	+ struct mlx5_ib_qp *qp,
	1567	+ struct mlx5_create_qp_params *params)
1397	1568	{
1398		- struct ib_uobject *uobj = pd->uobject;
1399		- struct ib_ucontext *ucontext = uobj->context;
1400		- struct mlx5_ib_ucontext *mucontext = to_mucontext(ucontext);
1401		- struct mlx5_ib_create_qp_resp resp = {};
	1569	+ struct ib_qp_init_attr *init_attr = params->attr;
	1570	+ struct mlx5_ib_create_qp_rss *ucmd = params->ucmd;
	1571	+ struct ib_udata *udata = params->udata;
	1572	+ struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(
	1573	+ udata, struct mlx5_ib_ucontext, ibucontext);
1402	1574	int inlen;
	1575	+ int outlen;
1403	1576	int err;
1404	1577	u32 *in;
	1578	+ u32 *out;
1405	1579	void *tirc;
1406	1580	void *hfso;
1407	1581	u32 selected_fields = 0;
1408	1582	u32 outer_l4;
1409		- size_t min_resp_len;
1410	1583	u32 tdn = mucontext->tdn;
1411		- struct mlx5_ib_create_qp_rss ucmd = {};
1412		- size_t required_cmd_sz;
	1584	+ u8 lb_flag = 0;
1413	1585
1414		- if (init_attr->qp_type != IB_QPT_RAW_PACKET)
1415		- return -EOPNOTSUPP;
1416		-
1417		- if (init_attr->create_flags \|\| init_attr->send_cq)
1418		- return -EINVAL;
1419		-
1420		- min_resp_len = offsetof(typeof(resp), bfreg_index) + sizeof(resp.bfreg_index);
1421		- if (udata->outlen < min_resp_len)
1422		- return -EINVAL;
1423		-
1424		- required_cmd_sz = offsetof(typeof(ucmd), flags) + sizeof(ucmd.flags);
1425		- if (udata->inlen < required_cmd_sz) {
1426		- mlx5_ib_dbg(dev, "invalid inlen\n");
1427		- return -EINVAL;
1428		- }
1429		-
1430		- if (udata->inlen > sizeof(ucmd) &&
1431		- !ib_is_udata_cleared(udata, sizeof(ucmd),
1432		- udata->inlen - sizeof(ucmd))) {
1433		- mlx5_ib_dbg(dev, "inlen is not supported\n");
1434		- return -EOPNOTSUPP;
1435		- }
1436		-
1437		- if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
1438		- mlx5_ib_dbg(dev, "copy failed\n");
1439		- return -EFAULT;
1440		- }
1441		-
1442		- if (ucmd.comp_mask) {
	1586	+ if (ucmd->comp_mask) {
1443	1587	mlx5_ib_dbg(dev, "invalid comp mask\n");
1444	1588	return -EOPNOTSUPP;
1445	1589	}
1446	1590
1447		- if (ucmd.flags & ~MLX5_QP_FLAG_TUNNEL_OFFLOADS) {
1448		- mlx5_ib_dbg(dev, "invalid flags\n");
1449		- return -EOPNOTSUPP;
1450		- }
1451		-
1452		- if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS &&
1453		- !tunnel_offload_supported(dev->mdev)) {
1454		- mlx5_ib_dbg(dev, "tunnel offloads isn't supported\n");
1455		- return -EOPNOTSUPP;
1456		- }
1457		-
1458		- if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_INNER &&
1459		- !(ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {
	1591	+ if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER &&
	1592	+ !(ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {
1460	1593	mlx5_ib_dbg(dev, "Tunnel offloads must be set for inner RSS\n");
1461	1594	return -EOPNOTSUPP;
1462	1595	}
1463	1596
1464		- err = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
1465		- if (err) {
1466		- mlx5_ib_dbg(dev, "copy failed\n");
1467		- return -EINVAL;
1468		- }
	1597	+ if (dev->is_rep)
	1598	+ qp->flags_en \|= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
	1599	+
	1600	+ if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)
	1601	+ lb_flag \|= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
	1602	+
	1603	+ if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
	1604	+ lb_flag \|= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
1469	1605
1470	1606	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1471		- in = kvzalloc(inlen, GFP_KERNEL);
	1607	+ outlen = MLX5_ST_SZ_BYTES(create_tir_out);
	1608	+ in = kvzalloc(inlen + outlen, GFP_KERNEL);
1472	1609	if (!in)
1473	1610	return -ENOMEM;
1474	1611
	1612	+ out = in + MLX5_ST_SZ_DW(create_tir_in);
	1613	+ MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
1475	1614	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1476	1615	MLX5_SET(tirc, tirc, disp_type,
1477	1616	MLX5_TIRC_DISP_TYPE_INDIRECT);
..	..	@@ -1481,27 +1620,29 @@
1481	1620
1482	1621	hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1483	1622
1484		- if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
	1623	+ if (ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
1485	1624	MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
1486	1625
1487		- if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_INNER)
	1626	+ MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
	1627	+
	1628	+ if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER)
1488	1629	hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
1489	1630	else
1490	1631	hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1491	1632
1492		- switch (ucmd.rx_hash_function) {
	1633	+ switch (ucmd->rx_hash_function) {
1493	1634	case MLX5_RX_HASH_FUNC_TOEPLITZ:
1494	1635	{
1495	1636	void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
1496	1637	size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key);
1497	1638
1498		- if (len != ucmd.rx_key_len) {
	1639	+ if (len != ucmd->rx_key_len) {
1499	1640	err = -EINVAL;
1500	1641	goto err;
1501	1642	}
1502	1643
1503	1644	MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ);
1504		- memcpy(rss_key, ucmd.rx_hash_key, len);
	1645	+ memcpy(rss_key, ucmd->rx_hash_key, len);
1505	1646	break;
1506	1647	}
1507	1648	default:
..	..	@@ -1509,7 +1650,7 @@
1509	1650	goto err;
1510	1651	}
1511	1652
1512		- if (!ucmd.rx_hash_fields_mask) {
	1653	+ if (!ucmd->rx_hash_fields_mask) {
1513	1654	/* special case when this TIR serves as steering entry without hashing */
1514	1655	if (!init_attr->rwq_ind_tbl->log_ind_tbl_size)
1515	1656	goto create_tir;
..	..	@@ -1517,29 +1658,31 @@
1517	1658	goto err;
1518	1659	}
1519	1660
1520		- if (((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) \|\|
1521		- (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) &&
1522		- ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) \|\|
1523		- (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) {
	1661	+ if (((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) \|\|
	1662	+ (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) &&
	1663	+ ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) \|\|
	1664	+ (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) {
1524	1665	err = -EINVAL;
1525	1666	goto err;
1526	1667	}
1527	1668
1528	1669	/* If none of IPV4 & IPV6 SRC/DST was set - this bit field is ignored */
1529		- if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) \|\|
1530		- (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4))
	1670	+ if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) \|\|
	1671	+ (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4))
1531	1672	MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1532	1673	MLX5_L3_PROT_TYPE_IPV4);
1533		- else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) \|\|
1534		- (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
	1674	+ else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) \|\|
	1675	+ (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1535	1676	MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1536	1677	MLX5_L3_PROT_TYPE_IPV6);
1537	1678
1538		- outer_l4 = ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) \|\|
1539		- (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP)) << 0 \|
1540		- ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) \|\|
1541		- (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP)) << 1 \|
1542		- (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI) << 2;
	1679	+ outer_l4 = ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) \|\|
	1680	+ (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
	1681	+ << 0 \|
	1682	+ ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) \|\|
	1683	+ (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
	1684	+ << 1 \|
	1685	+ (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI) << 2;
1543	1686
1544	1687	/* Check that only one l4 protocol is set */
1545	1688	if (outer_l4 & (outer_l4 - 1)) {
..	..	@@ -1548,50 +1691,76 @@
1548	1691	}
1549	1692
1550	1693	/* If none of TCP & UDP SRC/DST was set - this bit field is ignored */
1551		- if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) \|\|
1552		- (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
	1694	+ if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) \|\|
	1695	+ (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
1553	1696	MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1554	1697	MLX5_L4_PROT_TYPE_TCP);
1555		- else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) \|\|
1556		- (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
	1698	+ else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) \|\|
	1699	+ (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1557	1700	MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1558	1701	MLX5_L4_PROT_TYPE_UDP);
1559	1702
1560		- if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) \|\|
1561		- (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6))
	1703	+ if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) \|\|
	1704	+ (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6))
1562	1705	selected_fields \|= MLX5_HASH_FIELD_SEL_SRC_IP;
1563	1706
1564		- if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) \|\|
1565		- (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
	1707	+ if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) \|\|
	1708	+ (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1566	1709	selected_fields \|= MLX5_HASH_FIELD_SEL_DST_IP;
1567	1710
1568		- if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) \|\|
1569		- (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP))
	1711	+ if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) \|\|
	1712	+ (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP))
1570	1713	selected_fields \|= MLX5_HASH_FIELD_SEL_L4_SPORT;
1571	1714
1572		- if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) \|\|
1573		- (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
	1715	+ if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) \|\|
	1716	+ (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1574	1717	selected_fields \|= MLX5_HASH_FIELD_SEL_L4_DPORT;
1575	1718
1576		- if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI)
	1719	+ if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI)
1577	1720	selected_fields \|= MLX5_HASH_FIELD_SEL_IPSEC_SPI;
1578	1721
1579	1722	MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields);
1580	1723
1581	1724	create_tir:
1582		- if (dev->rep)
1583		- MLX5_SET(tirc, tirc, self_lb_block,
1584		- MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST_);
	1725	+ MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
	1726	+ err = mlx5_cmd_exec_inout(dev->mdev, create_tir, in, out);
1585	1727
1586		- err = mlx5_core_create_tir(dev->mdev, in, inlen, &qp->rss_qp.tirn);
	1728	+ qp->rss_qp.tirn = MLX5_GET(create_tir_out, out, tirn);
	1729	+ if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
	1730	+ err = mlx5_ib_enable_lb(dev, false, true);
	1731	+
	1732	+ if (err)
	1733	+ mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn,
	1734	+ to_mpd(pd)->uid);
	1735	+ }
1587	1736
1588	1737	if (err)
1589	1738	goto err;
1590	1739
	1740	+ if (mucontext->devx_uid) {
	1741	+ params->resp.comp_mask \|= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
	1742	+ params->resp.tirn = qp->rss_qp.tirn;
	1743	+ if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) \|\|
	1744	+ MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2)) {
	1745	+ params->resp.tir_icm_addr =
	1746	+ MLX5_GET(create_tir_out, out, icm_address_31_0);
	1747	+ params->resp.tir_icm_addr \|=
	1748	+ (u64)MLX5_GET(create_tir_out, out,
	1749	+ icm_address_39_32)
	1750	+ << 32;
	1751	+ params->resp.tir_icm_addr \|=
	1752	+ (u64)MLX5_GET(create_tir_out, out,
	1753	+ icm_address_63_40)
	1754	+ << 40;
	1755	+ params->resp.comp_mask \|=
	1756	+ MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
	1757	+ }
	1758	+ }
	1759	+
1591	1760	kvfree(in);
1592	1761	/* qpn is reserved for that QP */
1593	1762	qp->trans_qp.base.mqp.qpn = 0;
1594		- qp->flags \|= MLX5_IB_QP_RSS;
	1763	+ qp->is_rss = true;
1595	1764	return 0;
1596	1765
1597	1766	err:
..	..	@@ -1599,235 +1768,247 @@
1599	1768	return err;
1600	1769	}
1601	1770
1602		-static int create_qp_common(struct mlx5_ib_dev dev, struct ib_pd pd,
1603		- struct ib_qp_init_attr *init_attr,
1604		- struct ib_udata udata, struct mlx5_ib_qp qp)
	1771	+static void configure_requester_scat_cqe(struct mlx5_ib_dev *dev,
	1772	+ struct mlx5_ib_qp *qp,
	1773	+ struct ib_qp_init_attr *init_attr,
	1774	+ void *qpc)
1605	1775	{
	1776	+ int scqe_sz;
	1777	+ bool allow_scat_cqe = false;
	1778	+
	1779	+ allow_scat_cqe = qp->flags_en & MLX5_QP_FLAG_ALLOW_SCATTER_CQE;
	1780	+
	1781	+ if (!allow_scat_cqe && init_attr->sq_sig_type != IB_SIGNAL_ALL_WR)
	1782	+ return;
	1783	+
	1784	+ scqe_sz = mlx5_ib_get_cqe_size(init_attr->send_cq);
	1785	+ if (scqe_sz == 128) {
	1786	+ MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA64_CQE);
	1787	+ return;
	1788	+ }
	1789	+
	1790	+ if (init_attr->qp_type != MLX5_IB_QPT_DCI \|\|
	1791	+ MLX5_CAP_GEN(dev->mdev, dc_req_scat_data_cqe))
	1792	+ MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA32_CQE);
	1793	+}
	1794	+
	1795	+static int atomic_size_to_mode(int size_mask)
	1796	+{
	1797	+ /* driver does not support atomic_size > 256B
	1798	+ * and does not know how to translate bigger sizes
	1799	+ */
	1800	+ int supported_size_mask = size_mask & 0x1ff;
	1801	+ int log_max_size;
	1802	+
	1803	+ if (!supported_size_mask)
	1804	+ return -EOPNOTSUPP;
	1805	+
	1806	+ log_max_size = __fls(supported_size_mask);
	1807	+
	1808	+ if (log_max_size > 3)
	1809	+ return log_max_size;
	1810	+
	1811	+ return MLX5_ATOMIC_MODE_8B;
	1812	+}
	1813	+
	1814	+static int get_atomic_mode(struct mlx5_ib_dev *dev,
	1815	+ enum ib_qp_type qp_type)
	1816	+{
	1817	+ u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
	1818	+ u8 atomic = MLX5_CAP_GEN(dev->mdev, atomic);
	1819	+ int atomic_mode = -EOPNOTSUPP;
	1820	+ int atomic_size_mask;
	1821	+
	1822	+ if (!atomic)
	1823	+ return -EOPNOTSUPP;
	1824	+
	1825	+ if (qp_type == MLX5_IB_QPT_DCT)
	1826	+ atomic_size_mask = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_dc);
	1827	+ else
	1828	+ atomic_size_mask = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
	1829	+
	1830	+ if ((atomic_operations & MLX5_ATOMIC_OPS_EXTENDED_CMP_SWAP) \|\|
	1831	+ (atomic_operations & MLX5_ATOMIC_OPS_EXTENDED_FETCH_ADD))
	1832	+ atomic_mode = atomic_size_to_mode(atomic_size_mask);
	1833	+
	1834	+ if (atomic_mode <= 0 &&
	1835	+ (atomic_operations & MLX5_ATOMIC_OPS_CMP_SWAP &&
	1836	+ atomic_operations & MLX5_ATOMIC_OPS_FETCH_ADD))
	1837	+ atomic_mode = MLX5_ATOMIC_MODE_IB_COMP;
	1838	+
	1839	+ return atomic_mode;
	1840	+}
	1841	+
	1842	+static int create_xrc_tgt_qp(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp,
	1843	+ struct mlx5_create_qp_params *params)
	1844	+{
	1845	+ struct ib_qp_init_attr *attr = params->attr;
	1846	+ u32 uidx = params->uidx;
	1847	+ struct mlx5_ib_resources *devr = &dev->devr;
	1848	+ u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
	1849	+ int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
	1850	+ struct mlx5_core_dev *mdev = dev->mdev;
	1851	+ struct mlx5_ib_qp_base *base;
	1852	+ unsigned long flags;
	1853	+ void *qpc;
	1854	+ u32 *in;
	1855	+ int err;
	1856	+
	1857	+ if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
	1858	+ qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
	1859	+
	1860	+ in = kvzalloc(inlen, GFP_KERNEL);
	1861	+ if (!in)
	1862	+ return -ENOMEM;
	1863	+
	1864	+ qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
	1865	+
	1866	+ MLX5_SET(qpc, qpc, st, MLX5_QP_ST_XRC);
	1867	+ MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
	1868	+ MLX5_SET(qpc, qpc, pd, to_mpd(devr->p0)->pdn);
	1869	+
	1870	+ if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
	1871	+ MLX5_SET(qpc, qpc, block_lb_mc, 1);
	1872	+ if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
	1873	+ MLX5_SET(qpc, qpc, cd_master, 1);
	1874	+ if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
	1875	+ MLX5_SET(qpc, qpc, cd_slave_send, 1);
	1876	+ if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
	1877	+ MLX5_SET(qpc, qpc, cd_slave_receive, 1);
	1878	+
	1879	+ MLX5_SET(qpc, qpc, rq_type, MLX5_SRQ_RQ);
	1880	+ MLX5_SET(qpc, qpc, no_sq, 1);
	1881	+ MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
	1882	+ MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);
	1883	+ MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
	1884	+ MLX5_SET(qpc, qpc, xrcd, to_mxrcd(attr->xrcd)->xrcdn);
	1885	+ MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
	1886	+
	1887	+ /* 0xffffff means we ask to work with cqe version 0 */
	1888	+ if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
	1889	+ MLX5_SET(qpc, qpc, user_index, uidx);
	1890	+
	1891	+ if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
	1892	+ MLX5_SET(qpc, qpc, end_padding_mode,
	1893	+ MLX5_WQ_END_PAD_MODE_ALIGN);
	1894	+ /* Special case to clean flag */
	1895	+ qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
	1896	+ }
	1897	+
	1898	+ base = &qp->trans_qp.base;
	1899	+ err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
	1900	+ kvfree(in);
	1901	+ if (err)
	1902	+ return err;
	1903	+
	1904	+ base->container_mibqp = qp;
	1905	+ base->mqp.event = mlx5_ib_qp_event;
	1906	+ if (MLX5_CAP_GEN(mdev, ece_support))
	1907	+ params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
	1908	+
	1909	+ spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
	1910	+ list_add_tail(&qp->qps_list, &dev->qp_list);
	1911	+ spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
	1912	+
	1913	+ qp->trans_qp.xrcdn = to_mxrcd(attr->xrcd)->xrcdn;
	1914	+ return 0;
	1915	+}
	1916	+
	1917	+static int create_user_qp(struct mlx5_ib_dev dev, struct ib_pd pd,
	1918	+ struct mlx5_ib_qp *qp,
	1919	+ struct mlx5_create_qp_params *params)
	1920	+{
	1921	+ struct ib_qp_init_attr *init_attr = params->attr;
	1922	+ struct mlx5_ib_create_qp *ucmd = params->ucmd;
	1923	+ u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
	1924	+ struct ib_udata *udata = params->udata;
	1925	+ u32 uidx = params->uidx;
1606	1926	struct mlx5_ib_resources *devr = &dev->devr;
1607	1927	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
1608	1928	struct mlx5_core_dev *mdev = dev->mdev;
1609		- struct mlx5_ib_create_qp_resp resp = {};
1610	1929	struct mlx5_ib_cq *send_cq;
1611	1930	struct mlx5_ib_cq *recv_cq;
1612	1931	unsigned long flags;
1613		- u32 uidx = MLX5_IB_DEFAULT_UIDX;
1614		- struct mlx5_ib_create_qp ucmd;
1615	1932	struct mlx5_ib_qp_base *base;
1616	1933	int mlx5_st;
1617	1934	void *qpc;
1618	1935	u32 *in;
1619	1936	int err;
1620	1937
1621		- mutex_init(&qp->mutex);
1622	1938	spin_lock_init(&qp->sq.lock);
1623	1939	spin_lock_init(&qp->rq.lock);
1624	1940
1625		- mlx5_st = to_mlx5_st(init_attr->qp_type);
	1941	+ mlx5_st = to_mlx5_st(qp->type);
1626	1942	if (mlx5_st < 0)
1627	1943	return -EINVAL;
1628		-
1629		- if (init_attr->rwq_ind_tbl) {
1630		- if (!udata)
1631		- return -ENOSYS;
1632		-
1633		- err = create_rss_raw_qp_tir(dev, qp, pd, init_attr, udata);
1634		- return err;
1635		- }
1636		-
1637		- if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
1638		- if (!MLX5_CAP_GEN(mdev, block_lb_mc)) {
1639		- mlx5_ib_dbg(dev, "block multicast loopback isn't supported\n");
1640		- return -EINVAL;
1641		- } else {
1642		- qp->flags \|= MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1643		- }
1644		- }
1645		-
1646		- if (init_attr->create_flags &
1647		- (IB_QP_CREATE_CROSS_CHANNEL \|
1648		- IB_QP_CREATE_MANAGED_SEND \|
1649		- IB_QP_CREATE_MANAGED_RECV)) {
1650		- if (!MLX5_CAP_GEN(mdev, cd)) {
1651		- mlx5_ib_dbg(dev, "cross-channel isn't supported\n");
1652		- return -EINVAL;
1653		- }
1654		- if (init_attr->create_flags & IB_QP_CREATE_CROSS_CHANNEL)
1655		- qp->flags \|= MLX5_IB_QP_CROSS_CHANNEL;
1656		- if (init_attr->create_flags & IB_QP_CREATE_MANAGED_SEND)
1657		- qp->flags \|= MLX5_IB_QP_MANAGED_SEND;
1658		- if (init_attr->create_flags & IB_QP_CREATE_MANAGED_RECV)
1659		- qp->flags \|= MLX5_IB_QP_MANAGED_RECV;
1660		- }
1661		-
1662		- if (init_attr->qp_type == IB_QPT_UD &&
1663		- (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO))
1664		- if (!MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
1665		- mlx5_ib_dbg(dev, "ipoib UD lso qp isn't supported\n");
1666		- return -EOPNOTSUPP;
1667		- }
1668		-
1669		- if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS) {
1670		- if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
1671		- mlx5_ib_dbg(dev, "Scatter FCS is supported only for Raw Packet QPs");
1672		- return -EOPNOTSUPP;
1673		- }
1674		- if (!MLX5_CAP_GEN(dev->mdev, eth_net_offloads) \|\|
1675		- !MLX5_CAP_ETH(dev->mdev, scatter_fcs)) {
1676		- mlx5_ib_dbg(dev, "Scatter FCS isn't supported\n");
1677		- return -EOPNOTSUPP;
1678		- }
1679		- qp->flags \|= MLX5_IB_QP_CAP_SCATTER_FCS;
1680		- }
1681	1944
1682	1945	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
1683	1946	qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
1684	1947
1685		- if (init_attr->create_flags & IB_QP_CREATE_CVLAN_STRIPPING) {
1686		- if (!(MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
1687		- MLX5_CAP_ETH(dev->mdev, vlan_cap)) \|\|
1688		- (init_attr->qp_type != IB_QPT_RAW_PACKET))
1689		- return -EOPNOTSUPP;
1690		- qp->flags \|= MLX5_IB_QP_CVLAN_STRIPPING;
1691		- }
1692		-
1693		- if (pd && pd->uobject) {
1694		- if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
1695		- mlx5_ib_dbg(dev, "copy failed\n");
1696		- return -EFAULT;
1697		- }
1698		-
1699		- err = get_qp_user_index(to_mucontext(pd->uobject->context),
1700		- &ucmd, udata->inlen, &uidx);
1701		- if (err)
1702		- return err;
1703		-
1704		- qp->wq_sig = !!(ucmd.flags & MLX5_QP_FLAG_SIGNATURE);
1705		- qp->scat_cqe = !!(ucmd.flags & MLX5_QP_FLAG_SCATTER_CQE);
1706		- if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS) {
1707		- if (init_attr->qp_type != IB_QPT_RAW_PACKET \|\|
1708		- !tunnel_offload_supported(mdev)) {
1709		- mlx5_ib_dbg(dev, "Tunnel offload isn't supported\n");
1710		- return -EOPNOTSUPP;
1711		- }
1712		- qp->tunnel_offload_en = true;
1713		- }
1714		-
1715		- if (init_attr->create_flags & IB_QP_CREATE_SOURCE_QPN) {
1716		- if (init_attr->qp_type != IB_QPT_UD \|\|
1717		- (MLX5_CAP_GEN(dev->mdev, port_type) !=
1718		- MLX5_CAP_PORT_TYPE_IB) \|\|
1719		- !mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS)) {
1720		- mlx5_ib_dbg(dev, "Source QP option isn't supported\n");
1721		- return -EOPNOTSUPP;
1722		- }
1723		-
1724		- qp->flags \|= MLX5_IB_QP_UNDERLAY;
1725		- qp->underlay_qpn = init_attr->source_qpn;
1726		- }
1727		- } else {
1728		- qp->wq_sig = !!wq_signature;
1729		- }
	1948	+ if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
	1949	+ qp->underlay_qpn = init_attr->source_qpn;
1730	1950
1731	1951	base = (init_attr->qp_type == IB_QPT_RAW_PACKET \|\|
1732		- qp->flags & MLX5_IB_QP_UNDERLAY) ?
	1952	+ qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
1733	1953	&qp->raw_packet_qp.rq.base :
1734	1954	&qp->trans_qp.base;
1735	1955
1736	1956	qp->has_rq = qp_has_rq(init_attr);
1737		- err = set_rq_size(dev, &init_attr->cap, qp->has_rq,
1738		- qp, (pd && pd->uobject) ? &ucmd : NULL);
	1957	+ err = set_rq_size(dev, &init_attr->cap, qp->has_rq, qp, ucmd);
1739	1958	if (err) {
1740	1959	mlx5_ib_dbg(dev, "err %d\n", err);
1741	1960	return err;
1742	1961	}
1743	1962
1744		- if (pd) {
1745		- if (pd->uobject) {
1746		- __u32 max_wqes =
1747		- 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
1748		- mlx5_ib_dbg(dev, "requested sq_wqe_count (%d)\n", ucmd.sq_wqe_count);
1749		- if (ucmd.rq_wqe_shift != qp->rq.wqe_shift \|\|
1750		- ucmd.rq_wqe_count != qp->rq.wqe_cnt) {
1751		- mlx5_ib_dbg(dev, "invalid rq params\n");
1752		- return -EINVAL;
1753		- }
1754		- if (ucmd.sq_wqe_count > max_wqes) {
1755		- mlx5_ib_dbg(dev, "requested sq_wqe_count (%d) > max allowed (%d)\n",
1756		- ucmd.sq_wqe_count, max_wqes);
1757		- return -EINVAL;
1758		- }
1759		- if (init_attr->create_flags &
1760		- mlx5_ib_create_qp_sqpn_qp1()) {
1761		- mlx5_ib_dbg(dev, "user-space is not allowed to create UD QPs spoofing as QP1\n");
1762		- return -EINVAL;
1763		- }
1764		- err = create_user_qp(dev, pd, qp, udata, init_attr, &in,
1765		- &resp, &inlen, base);
1766		- if (err)
1767		- mlx5_ib_dbg(dev, "err %d\n", err);
1768		- } else {
1769		- err = create_kernel_qp(dev, init_attr, qp, &in, &inlen,
1770		- base);
1771		- if (err)
1772		- mlx5_ib_dbg(dev, "err %d\n", err);
1773		- }
	1963	+ if (ucmd->rq_wqe_shift != qp->rq.wqe_shift \|\|
	1964	+ ucmd->rq_wqe_count != qp->rq.wqe_cnt)
	1965	+ return -EINVAL;
1774	1966
1775		- if (err)
1776		- return err;
1777		- } else {
1778		- in = kvzalloc(inlen, GFP_KERNEL);
1779		- if (!in)
1780		- return -ENOMEM;
	1967	+ if (ucmd->sq_wqe_count > (1 << MLX5_CAP_GEN(mdev, log_max_qp_sz)))
	1968	+ return -EINVAL;
1781	1969
1782		- qp->create_type = MLX5_QP_EMPTY;
1783		- }
	1970	+ err = _create_user_qp(dev, pd, qp, udata, init_attr, &in, &params->resp,
	1971	+ &inlen, base, ucmd);
	1972	+ if (err)
	1973	+ return err;
1784	1974
1785	1975	if (is_sqp(init_attr->qp_type))
1786	1976	qp->port = init_attr->port_num;
1787	1977
	1978	+ if (MLX5_CAP_GEN(mdev, ece_support))
	1979	+ MLX5_SET(create_qp_in, in, ece, ucmd->ece_options);
1788	1980	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
1789	1981
1790	1982	MLX5_SET(qpc, qpc, st, mlx5_st);
1791	1983	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
	1984	+ MLX5_SET(qpc, qpc, pd, to_mpd(pd)->pdn);
1792	1985
1793		- if (init_attr->qp_type != MLX5_IB_QPT_REG_UMR)
1794		- MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);
1795		- else
1796		- MLX5_SET(qpc, qpc, latency_sensitive, 1);
1797		-
1798		-
1799		- if (qp->wq_sig)
	1986	+ if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
1800	1987	MLX5_SET(qpc, qpc, wq_signature, 1);
1801	1988
1802		- if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
	1989	+ if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
1803	1990	MLX5_SET(qpc, qpc, block_lb_mc, 1);
1804	1991
1805		- if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
	1992	+ if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
1806	1993	MLX5_SET(qpc, qpc, cd_master, 1);
1807		- if (qp->flags & MLX5_IB_QP_MANAGED_SEND)
	1994	+ if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
1808	1995	MLX5_SET(qpc, qpc, cd_slave_send, 1);
1809		- if (qp->flags & MLX5_IB_QP_MANAGED_RECV)
	1996	+ if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
1810	1997	MLX5_SET(qpc, qpc, cd_slave_receive, 1);
	1998	+ if (qp->flags_en & MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE)
	1999	+ MLX5_SET(qpc, qpc, req_e2e_credit_mode, 1);
	2000	+ if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
	2001	+ (init_attr->qp_type == IB_QPT_RC \|\|
	2002	+ init_attr->qp_type == IB_QPT_UC)) {
	2003	+ int rcqe_sz = mlx5_ib_get_cqe_size(init_attr->recv_cq);
1811	2004
1812		- if (qp->scat_cqe && is_connected(init_attr->qp_type)) {
1813		- int rcqe_sz;
1814		- int scqe_sz;
1815		-
1816		- rcqe_sz = mlx5_ib_get_cqe_size(dev, init_attr->recv_cq);
1817		- scqe_sz = mlx5_ib_get_cqe_size(dev, init_attr->send_cq);
1818		-
1819		- if (rcqe_sz == 128)
1820		- MLX5_SET(qpc, qpc, cs_res, MLX5_RES_SCAT_DATA64_CQE);
1821		- else
1822		- MLX5_SET(qpc, qpc, cs_res, MLX5_RES_SCAT_DATA32_CQE);
1823		-
1824		- if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) {
1825		- if (scqe_sz == 128)
1826		- MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA64_CQE);
1827		- else
1828		- MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA32_CQE);
1829		- }
	2005	+ MLX5_SET(qpc, qpc, cs_res,
	2006	+ rcqe_sz == 128 ? MLX5_RES_SCAT_DATA64_CQE :
	2007	+ MLX5_RES_SCAT_DATA32_CQE);
1830	2008	}
	2009	+ if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
	2010	+ (qp->type == MLX5_IB_QPT_DCI \|\| qp->type == IB_QPT_RC))
	2011	+ configure_requester_scat_cqe(dev, qp, init_attr, qpc);
1831	2012
1832	2013	if (qp->rq.wqe_cnt) {
1833	2014	MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
..	..	@@ -1848,23 +2029,17 @@
1848	2029
1849	2030	/* Set default resources */
1850	2031	switch (init_attr->qp_type) {
1851		- case IB_QPT_XRC_TGT:
1852		- MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
1853		- MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);
1854		- MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
1855		- MLX5_SET(qpc, qpc, xrcd, to_mxrcd(init_attr->xrcd)->xrcdn);
1856		- break;
1857	2032	case IB_QPT_XRC_INI:
1858	2033	MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
1859		- MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
	2034	+ MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
1860	2035	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
1861	2036	break;
1862	2037	default:
1863	2038	if (init_attr->srq) {
1864		- MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn);
	2039	+ MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
1865	2040	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);
1866	2041	} else {
1867		- MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
	2042	+ MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
1868	2043	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);
1869	2044	}
1870	2045	}
..	..	@@ -1881,51 +2056,33 @@
1881	2056	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
1882	2057	MLX5_SET(qpc, qpc, user_index, uidx);
1883	2058
1884		- /* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
1885		- if (init_attr->qp_type == IB_QPT_UD &&
1886		- (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)) {
1887		- MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
1888		- qp->flags \|= MLX5_IB_QP_LSO;
1889		- }
1890		-
1891		- if (init_attr->create_flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
1892		- if (!MLX5_CAP_GEN(dev->mdev, end_pad)) {
1893		- mlx5_ib_dbg(dev, "scatter end padding is not supported\n");
1894		- err = -EOPNOTSUPP;
1895		- goto err;
1896		- } else if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
1897		- MLX5_SET(qpc, qpc, end_padding_mode,
1898		- MLX5_WQ_END_PAD_MODE_ALIGN);
1899		- } else {
1900		- qp->flags \|= MLX5_IB_QP_PCI_WRITE_END_PADDING;
1901		- }
1902		- }
1903		-
1904		- if (inlen < 0) {
1905		- err = -EINVAL;
1906		- goto err;
	2059	+ if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING &&
	2060	+ init_attr->qp_type != IB_QPT_RAW_PACKET) {
	2061	+ MLX5_SET(qpc, qpc, end_padding_mode,
	2062	+ MLX5_WQ_END_PAD_MODE_ALIGN);
	2063	+ /* Special case to clean flag */
	2064	+ qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
1907	2065	}
1908	2066
1909	2067	if (init_attr->qp_type == IB_QPT_RAW_PACKET \|\|
1910		- qp->flags & MLX5_IB_QP_UNDERLAY) {
1911		- qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr;
	2068	+ qp->flags & IB_QP_CREATE_SOURCE_QPN) {
	2069	+ qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr;
1912	2070	raw_packet_qp_copy_info(qp, &qp->raw_packet_qp);
1913		- err = create_raw_packet_qp(dev, qp, in, inlen, pd);
1914		- } else {
1915		- err = mlx5_core_create_qp(dev->mdev, &base->mqp, in, inlen);
1916		- }
1917		-
1918		- if (err) {
1919		- mlx5_ib_dbg(dev, "create qp failed\n");
1920		- goto err_create;
1921		- }
	2071	+ err = create_raw_packet_qp(dev, qp, in, inlen, pd, udata,
	2072	+ &params->resp);
	2073	+ } else
	2074	+ err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
1922	2075
1923	2076	kvfree(in);
	2077	+ if (err)
	2078	+ goto err_create;
1924	2079
1925	2080	base->container_mibqp = qp;
1926	2081	base->mqp.event = mlx5_ib_qp_event;
	2082	+ if (MLX5_CAP_GEN(mdev, ece_support))
	2083	+ params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
1927	2084
1928		- get_cqs(init_attr->qp_type, init_attr->send_cq, init_attr->recv_cq,
	2085	+ get_cqs(qp->type, init_attr->send_cq, init_attr->recv_cq,
1929	2086	&send_cq, &recv_cq);
1930	2087	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1931	2088	mlx5_ib_lock_cqs(send_cq, recv_cq);
..	..	@@ -1945,13 +2102,136 @@
1945	2102	return 0;
1946	2103
1947	2104	err_create:
1948		- if (qp->create_type == MLX5_QP_USER)
1949		- destroy_qp_user(dev, pd, qp, base);
1950		- else if (qp->create_type == MLX5_QP_KERNEL)
1951		- destroy_qp_kernel(dev, qp);
	2105	+ destroy_qp(dev, qp, base, udata);
	2106	+ return err;
	2107	+}
1952	2108
1953		-err:
	2109	+static int create_kernel_qp(struct mlx5_ib_dev dev, struct ib_pd pd,
	2110	+ struct mlx5_ib_qp *qp,
	2111	+ struct mlx5_create_qp_params *params)
	2112	+{
	2113	+ struct ib_qp_init_attr *attr = params->attr;
	2114	+ u32 uidx = params->uidx;
	2115	+ struct mlx5_ib_resources *devr = &dev->devr;
	2116	+ u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
	2117	+ int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
	2118	+ struct mlx5_core_dev *mdev = dev->mdev;
	2119	+ struct mlx5_ib_cq *send_cq;
	2120	+ struct mlx5_ib_cq *recv_cq;
	2121	+ unsigned long flags;
	2122	+ struct mlx5_ib_qp_base *base;
	2123	+ int mlx5_st;
	2124	+ void *qpc;
	2125	+ u32 *in;
	2126	+ int err;
	2127	+
	2128	+ spin_lock_init(&qp->sq.lock);
	2129	+ spin_lock_init(&qp->rq.lock);
	2130	+
	2131	+ mlx5_st = to_mlx5_st(qp->type);
	2132	+ if (mlx5_st < 0)
	2133	+ return -EINVAL;
	2134	+
	2135	+ if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
	2136	+ qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
	2137	+
	2138	+ base = &qp->trans_qp.base;
	2139	+
	2140	+ qp->has_rq = qp_has_rq(attr);
	2141	+ err = set_rq_size(dev, &attr->cap, qp->has_rq, qp, NULL);
	2142	+ if (err) {
	2143	+ mlx5_ib_dbg(dev, "err %d\n", err);
	2144	+ return err;
	2145	+ }
	2146	+
	2147	+ err = _create_kernel_qp(dev, attr, qp, &in, &inlen, base);
	2148	+ if (err)
	2149	+ return err;
	2150	+
	2151	+ if (is_sqp(attr->qp_type))
	2152	+ qp->port = attr->port_num;
	2153	+
	2154	+ qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
	2155	+
	2156	+ MLX5_SET(qpc, qpc, st, mlx5_st);
	2157	+ MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
	2158	+
	2159	+ if (attr->qp_type != MLX5_IB_QPT_REG_UMR)
	2160	+ MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);
	2161	+ else
	2162	+ MLX5_SET(qpc, qpc, latency_sensitive, 1);
	2163	+
	2164	+
	2165	+ if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
	2166	+ MLX5_SET(qpc, qpc, block_lb_mc, 1);
	2167	+
	2168	+ if (qp->rq.wqe_cnt) {
	2169	+ MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
	2170	+ MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
	2171	+ }
	2172	+
	2173	+ MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, attr));
	2174	+
	2175	+ if (qp->sq.wqe_cnt)
	2176	+ MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
	2177	+ else
	2178	+ MLX5_SET(qpc, qpc, no_sq, 1);
	2179	+
	2180	+ if (attr->srq) {
	2181	+ MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
	2182	+ MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
	2183	+ to_msrq(attr->srq)->msrq.srqn);
	2184	+ } else {
	2185	+ MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
	2186	+ MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
	2187	+ to_msrq(devr->s1)->msrq.srqn);
	2188	+ }
	2189	+
	2190	+ if (attr->send_cq)
	2191	+ MLX5_SET(qpc, qpc, cqn_snd, to_mcq(attr->send_cq)->mcq.cqn);
	2192	+
	2193	+ if (attr->recv_cq)
	2194	+ MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(attr->recv_cq)->mcq.cqn);
	2195	+
	2196	+ MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
	2197	+
	2198	+ /* 0xffffff means we ask to work with cqe version 0 */
	2199	+ if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
	2200	+ MLX5_SET(qpc, qpc, user_index, uidx);
	2201	+
	2202	+ /* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
	2203	+ if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO)
	2204	+ MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
	2205	+
	2206	+ err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
1954	2207	kvfree(in);
	2208	+ if (err)
	2209	+ goto err_create;
	2210	+
	2211	+ base->container_mibqp = qp;
	2212	+ base->mqp.event = mlx5_ib_qp_event;
	2213	+
	2214	+ get_cqs(qp->type, attr->send_cq, attr->recv_cq,
	2215	+ &send_cq, &recv_cq);
	2216	+ spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
	2217	+ mlx5_ib_lock_cqs(send_cq, recv_cq);
	2218	+ /* Maintain device to QPs access, needed for further handling via reset
	2219	+ * flow
	2220	+ */
	2221	+ list_add_tail(&qp->qps_list, &dev->qp_list);
	2222	+ /* Maintain CQ to QPs access, needed for further handling via reset flow
	2223	+ */
	2224	+ if (send_cq)
	2225	+ list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
	2226	+ if (recv_cq)
	2227	+ list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
	2228	+ mlx5_ib_unlock_cqs(send_cq, recv_cq);
	2229	+ spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
	2230	+
	2231	+ return 0;
	2232	+
	2233	+err_create:
	2234	+ destroy_qp(dev, qp, base, NULL);
1955	2235	return err;
1956	2236	}
1957	2237
..	..	@@ -2013,11 +2293,6 @@
2013	2293	}
2014	2294	}
2015	2295
2016		-static struct mlx5_ib_pd get_pd(struct mlx5_ib_qp qp)
2017		-{
2018		- return to_mpd(qp->ibqp.pd);
2019		-}
2020		-
2021	2296	static void get_cqs(enum ib_qp_type qp_type,
2022	2297	struct ib_cq ib_send_cq, struct ib_cq ib_recv_cq,
2023	2298	struct mlx5_ib_cq send_cq, struct mlx5_ib_cq recv_cq)
..	..	@@ -2038,14 +2313,10 @@
2038	2313	case IB_QPT_RC:
2039	2314	case IB_QPT_UC:
2040	2315	case IB_QPT_UD:
2041		- case IB_QPT_RAW_IPV6:
2042		- case IB_QPT_RAW_ETHERTYPE:
2043	2316	case IB_QPT_RAW_PACKET:
2044	2317	*send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
2045	2318	*recv_cq = ib_recv_cq ? to_mcq(ib_recv_cq) : NULL;
2046	2319	break;
2047		-
2048		- case IB_QPT_MAX:
2049	2320	default:
2050	2321	*send_cq = NULL;
2051	2322	*recv_cq = NULL;
..	..	@@ -2057,29 +2328,29 @@
2057	2328	const struct mlx5_modify_raw_qp_param *raw_qp_param,
2058	2329	u8 lag_tx_affinity);
2059	2330
2060		-static void destroy_qp_common(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp)
	2331	+static void destroy_qp_common(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp,
	2332	+ struct ib_udata *udata)
2061	2333	{
2062	2334	struct mlx5_ib_cq send_cq, recv_cq;
2063	2335	struct mlx5_ib_qp_base *base;
2064	2336	unsigned long flags;
2065	2337	int err;
2066	2338
2067		- if (qp->ibqp.rwq_ind_tbl) {
	2339	+ if (qp->is_rss) {
2068	2340	destroy_rss_raw_qp_tir(dev, qp);
2069	2341	return;
2070	2342	}
2071	2343
2072		- base = (qp->ibqp.qp_type == IB_QPT_RAW_PACKET \|\|
2073		- qp->flags & MLX5_IB_QP_UNDERLAY) ?
2074		- &qp->raw_packet_qp.rq.base :
2075		- &qp->trans_qp.base;
	2344	+ base = (qp->type == IB_QPT_RAW_PACKET \|\|
	2345	+ qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
	2346	+ &qp->raw_packet_qp.rq.base :
	2347	+ &qp->trans_qp.base;
2076	2348
2077	2349	if (qp->state != IB_QPS_RESET) {
2078		- if (qp->ibqp.qp_type != IB_QPT_RAW_PACKET &&
2079		- !(qp->flags & MLX5_IB_QP_UNDERLAY)) {
2080		- err = mlx5_core_qp_modify(dev->mdev,
2081		- MLX5_CMD_OP_2RST_QP, 0,
2082		- NULL, &base->mqp);
	2350	+ if (qp->type != IB_QPT_RAW_PACKET &&
	2351	+ !(qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
	2352	+ err = mlx5_core_qp_modify(dev, MLX5_CMD_OP_2RST_QP, 0,
	2353	+ NULL, &base->mqp, NULL);
2083	2354	} else {
2084	2355	struct mlx5_modify_raw_qp_param raw_qp_param = {
2085	2356	.operation = MLX5_CMD_OP_2RST_QP
..	..	@@ -2092,8 +2363,8 @@
2092	2363	base->mqp.qpn);
2093	2364	}
2094	2365
2095		- get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
2096		- &send_cq, &recv_cq);
	2366	+ get_cqs(qp->type, qp->ibqp.send_cq, qp->ibqp.recv_cq, &send_cq,
	2367	+ &recv_cq);
2097	2368
2098	2369	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2099	2370	mlx5_ib_lock_cqs(send_cq, recv_cq);
..	..	@@ -2105,7 +2376,7 @@
2105	2376	if (recv_cq)
2106	2377	list_del(&qp->cq_recv_list);
2107	2378
2108		- if (qp->create_type == MLX5_QP_KERNEL) {
	2379	+ if (!udata) {
2109	2380	__mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
2110	2381	qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
2111	2382	if (send_cq != recv_cq)
..	..	@@ -2115,256 +2386,465 @@
2115	2386	mlx5_ib_unlock_cqs(send_cq, recv_cq);
2116	2387	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2117	2388
2118		- if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET \|\|
2119		- qp->flags & MLX5_IB_QP_UNDERLAY) {
	2389	+ if (qp->type == IB_QPT_RAW_PACKET \|\|
	2390	+ qp->flags & IB_QP_CREATE_SOURCE_QPN) {
2120	2391	destroy_raw_packet_qp(dev, qp);
2121	2392	} else {
2122		- err = mlx5_core_destroy_qp(dev->mdev, &base->mqp);
	2393	+ err = mlx5_core_destroy_qp(dev, &base->mqp);
2123	2394	if (err)
2124	2395	mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",
2125	2396	base->mqp.qpn);
2126	2397	}
2127	2398
2128		- if (qp->create_type == MLX5_QP_KERNEL)
2129		- destroy_qp_kernel(dev, qp);
2130		- else if (qp->create_type == MLX5_QP_USER)
2131		- destroy_qp_user(dev, &get_pd(qp)->ibpd, qp, base);
	2399	+ destroy_qp(dev, qp, base, udata);
2132	2400	}
2133	2401
2134		-static const char *ib_qp_type_str(enum ib_qp_type type)
	2402	+static int create_dct(struct mlx5_ib_dev dev, struct ib_pd pd,
	2403	+ struct mlx5_ib_qp *qp,
	2404	+ struct mlx5_create_qp_params *params)
2135	2405	{
2136		- switch (type) {
2137		- case IB_QPT_SMI:
2138		- return "IB_QPT_SMI";
2139		- case IB_QPT_GSI:
2140		- return "IB_QPT_GSI";
2141		- case IB_QPT_RC:
2142		- return "IB_QPT_RC";
2143		- case IB_QPT_UC:
2144		- return "IB_QPT_UC";
2145		- case IB_QPT_UD:
2146		- return "IB_QPT_UD";
2147		- case IB_QPT_RAW_IPV6:
2148		- return "IB_QPT_RAW_IPV6";
2149		- case IB_QPT_RAW_ETHERTYPE:
2150		- return "IB_QPT_RAW_ETHERTYPE";
2151		- case IB_QPT_XRC_INI:
2152		- return "IB_QPT_XRC_INI";
2153		- case IB_QPT_XRC_TGT:
2154		- return "IB_QPT_XRC_TGT";
2155		- case IB_QPT_RAW_PACKET:
2156		- return "IB_QPT_RAW_PACKET";
2157		- case MLX5_IB_QPT_REG_UMR:
2158		- return "MLX5_IB_QPT_REG_UMR";
2159		- case IB_QPT_DRIVER:
2160		- return "IB_QPT_DRIVER";
2161		- case IB_QPT_MAX:
2162		- default:
2163		- return "Invalid QP type";
2164		- }
2165		-}
2166		-
2167		-static struct ib_qp mlx5_ib_create_dct(struct ib_pd pd,
2168		- struct ib_qp_init_attr *attr,
2169		- struct mlx5_ib_create_qp *ucmd)
2170		-{
2171		- struct mlx5_ib_qp *qp;
2172		- int err = 0;
2173		- u32 uidx = MLX5_IB_DEFAULT_UIDX;
	2406	+ struct ib_qp_init_attr *attr = params->attr;
	2407	+ struct mlx5_ib_create_qp *ucmd = params->ucmd;
	2408	+ u32 uidx = params->uidx;
2174	2409	void *dctc;
2175	2410
2176		- if (!attr->srq \|\| !attr->recv_cq)
2177		- return ERR_PTR(-EINVAL);
2178		-
2179		- err = get_qp_user_index(to_mucontext(pd->uobject->context),
2180		- ucmd, sizeof(*ucmd), &uidx);
2181		- if (err)
2182		- return ERR_PTR(err);
2183		-
2184		- qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2185		- if (!qp)
2186		- return ERR_PTR(-ENOMEM);
	2411	+ if (mlx5_lag_is_active(dev->mdev) && !MLX5_CAP_GEN(dev->mdev, lag_dct))
	2412	+ return -EOPNOTSUPP;
2187	2413
2188	2414	qp->dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL);
2189		- if (!qp->dct.in) {
2190		- err = -ENOMEM;
2191		- goto err_free;
2192		- }
	2415	+ if (!qp->dct.in)
	2416	+ return -ENOMEM;
2193	2417
	2418	+ MLX5_SET(create_dct_in, qp->dct.in, uid, to_mpd(pd)->uid);
2194	2419	dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
2195		- qp->qp_sub_type = MLX5_IB_QPT_DCT;
2196	2420	MLX5_SET(dctc, dctc, pd, to_mpd(pd)->pdn);
2197	2421	MLX5_SET(dctc, dctc, srqn_xrqn, to_msrq(attr->srq)->msrq.srqn);
2198	2422	MLX5_SET(dctc, dctc, cqn, to_mcq(attr->recv_cq)->mcq.cqn);
2199	2423	MLX5_SET64(dctc, dctc, dc_access_key, ucmd->access_key);
2200	2424	MLX5_SET(dctc, dctc, user_index, uidx);
	2425	+ if (MLX5_CAP_GEN(dev->mdev, ece_support))
	2426	+ MLX5_SET(dctc, dctc, ece, ucmd->ece_options);
	2427	+
	2428	+ if (qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) {
	2429	+ int rcqe_sz = mlx5_ib_get_cqe_size(attr->recv_cq);
	2430	+
	2431	+ if (rcqe_sz == 128)
	2432	+ MLX5_SET(dctc, dctc, cs_res, MLX5_RES_SCAT_DATA64_CQE);
	2433	+ }
2201	2434
2202	2435	qp->state = IB_QPS_RESET;
2203		-
2204		- return &qp->ibqp;
2205		-err_free:
2206		- kfree(qp);
2207		- return ERR_PTR(err);
2208		-}
2209		-
2210		-static int set_mlx_qp_type(struct mlx5_ib_dev *dev,
2211		- struct ib_qp_init_attr *init_attr,
2212		- struct mlx5_ib_create_qp *ucmd,
2213		- struct ib_udata *udata)
2214		-{
2215		- enum { MLX_QP_FLAGS = MLX5_QP_FLAG_TYPE_DCT \| MLX5_QP_FLAG_TYPE_DCI };
2216		- int err;
2217		-
2218		- if (!udata)
2219		- return -EINVAL;
2220		-
2221		- if (udata->inlen < sizeof(*ucmd)) {
2222		- mlx5_ib_dbg(dev, "create_qp user command is smaller than expected\n");
2223		- return -EINVAL;
2224		- }
2225		- err = ib_copy_from_udata(ucmd, udata, sizeof(*ucmd));
2226		- if (err)
2227		- return err;
2228		-
2229		- if ((ucmd->flags & MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCI) {
2230		- init_attr->qp_type = MLX5_IB_QPT_DCI;
2231		- } else {
2232		- if ((ucmd->flags & MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCT) {
2233		- init_attr->qp_type = MLX5_IB_QPT_DCT;
2234		- } else {
2235		- mlx5_ib_dbg(dev, "Invalid QP flags\n");
2236		- return -EINVAL;
2237		- }
2238		- }
2239		-
2240		- if (!MLX5_CAP_GEN(dev->mdev, dct)) {
2241		- mlx5_ib_dbg(dev, "DC transport is not supported\n");
2242		- return -EOPNOTSUPP;
2243		- }
2244	2436
2245	2437	return 0;
2246	2438	}
2247	2439
2248		-struct ib_qp mlx5_ib_create_qp(struct ib_pd pd,
2249		- struct ib_qp_init_attr *verbs_init_attr,
2250		- struct ib_udata *udata)
	2440	+static int check_qp_type(struct mlx5_ib_dev dev, struct ib_qp_init_attr attr,
	2441	+ enum ib_qp_type *type)
2251	2442	{
2252		- struct mlx5_ib_dev *dev;
2253		- struct mlx5_ib_qp *qp;
2254		- u16 xrcdn = 0;
2255		- int err;
2256		- struct ib_qp_init_attr mlx_init_attr;
2257		- struct ib_qp_init_attr *init_attr = verbs_init_attr;
	2443	+ if (attr->qp_type == IB_QPT_DRIVER && !MLX5_CAP_GEN(dev->mdev, dct))
	2444	+ goto out;
2258	2445
2259		- if (pd) {
2260		- dev = to_mdev(pd->device);
2261		-
2262		- if (init_attr->qp_type == IB_QPT_RAW_PACKET) {
2263		- if (!pd->uobject) {
2264		- mlx5_ib_dbg(dev, "Raw Packet QP is not supported for kernel consumers\n");
2265		- return ERR_PTR(-EINVAL);
2266		- } else if (!to_mucontext(pd->uobject->context)->cqe_version) {
2267		- mlx5_ib_dbg(dev, "Raw Packet QP is only supported for CQE version > 0\n");
2268		- return ERR_PTR(-EINVAL);
2269		- }
2270		- }
2271		- } else {
2272		- /* being cautious here */
2273		- if (init_attr->qp_type != IB_QPT_XRC_TGT &&
2274		- init_attr->qp_type != MLX5_IB_QPT_REG_UMR) {
2275		- pr_warn("%s: no PD for transport %s\n", __func__,
2276		- ib_qp_type_str(init_attr->qp_type));
2277		- return ERR_PTR(-EINVAL);
2278		- }
2279		- dev = to_mdev(to_mxrcd(init_attr->xrcd)->ibxrcd.device);
2280		- }
2281		-
2282		- if (init_attr->qp_type == IB_QPT_DRIVER) {
2283		- struct mlx5_ib_create_qp ucmd;
2284		-
2285		- init_attr = &mlx_init_attr;
2286		- memcpy(init_attr, verbs_init_attr, sizeof(*verbs_init_attr));
2287		- err = set_mlx_qp_type(dev, init_attr, &ucmd, udata);
2288		- if (err)
2289		- return ERR_PTR(err);
2290		-
2291		- if (init_attr->qp_type == MLX5_IB_QPT_DCI) {
2292		- if (init_attr->cap.max_recv_wr \|\|
2293		- init_attr->cap.max_recv_sge) {
2294		- mlx5_ib_dbg(dev, "DCI QP requires zero size receive queue\n");
2295		- return ERR_PTR(-EINVAL);
2296		- }
2297		- } else {
2298		- return mlx5_ib_create_dct(pd, init_attr, &ucmd);
2299		- }
2300		- }
2301		-
2302		- switch (init_attr->qp_type) {
	2446	+ switch (attr->qp_type) {
2303	2447	case IB_QPT_XRC_TGT:
2304	2448	case IB_QPT_XRC_INI:
2305		- if (!MLX5_CAP_GEN(dev->mdev, xrc)) {
2306		- mlx5_ib_dbg(dev, "XRC not supported\n");
2307		- return ERR_PTR(-ENOSYS);
2308		- }
2309		- init_attr->recv_cq = NULL;
2310		- if (init_attr->qp_type == IB_QPT_XRC_TGT) {
2311		- xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
2312		- init_attr->send_cq = NULL;
2313		- }
2314		-
2315		- /* fall through */
2316		- case IB_QPT_RAW_PACKET:
	2449	+ if (!MLX5_CAP_GEN(dev->mdev, xrc))
	2450	+ goto out;
	2451	+ fallthrough;
2317	2452	case IB_QPT_RC:
2318	2453	case IB_QPT_UC:
2319		- case IB_QPT_UD:
2320	2454	case IB_QPT_SMI:
2321	2455	case MLX5_IB_QPT_HW_GSI:
2322		- case MLX5_IB_QPT_REG_UMR:
2323		- case MLX5_IB_QPT_DCI:
2324		- qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2325		- if (!qp)
2326		- return ERR_PTR(-ENOMEM);
2327		-
2328		- err = create_qp_common(dev, pd, init_attr, udata, qp);
2329		- if (err) {
2330		- mlx5_ib_dbg(dev, "create_qp_common failed\n");
2331		- kfree(qp);
2332		- return ERR_PTR(err);
2333		- }
2334		-
2335		- if (is_qp0(init_attr->qp_type))
2336		- qp->ibqp.qp_num = 0;
2337		- else if (is_qp1(init_attr->qp_type))
2338		- qp->ibqp.qp_num = 1;
2339		- else
2340		- qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn;
2341		-
2342		- mlx5_ib_dbg(dev, "ib qpnum 0x%x, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x\n",
2343		- qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn,
2344		- init_attr->recv_cq ? to_mcq(init_attr->recv_cq)->mcq.cqn : -1,
2345		- init_attr->send_cq ? to_mcq(init_attr->send_cq)->mcq.cqn : -1);
2346		-
2347		- qp->trans_qp.xrcdn = xrcdn;
2348		-
2349		- break;
2350		-
	2456	+ case IB_QPT_DRIVER:
2351	2457	case IB_QPT_GSI:
2352		- return mlx5_ib_gsi_create_qp(pd, init_attr);
2353		-
2354		- case IB_QPT_RAW_IPV6:
2355		- case IB_QPT_RAW_ETHERTYPE:
2356		- case IB_QPT_MAX:
	2458	+ case IB_QPT_RAW_PACKET:
	2459	+ case IB_QPT_UD:
	2460	+ case MLX5_IB_QPT_REG_UMR:
	2461	+ break;
2357	2462	default:
2358		- mlx5_ib_dbg(dev, "unsupported qp type %d\n",
2359		- init_attr->qp_type);
2360		- /* Don't support raw QPs */
2361		- return ERR_PTR(-EINVAL);
	2463	+ goto out;
2362	2464	}
2363	2465
2364		- if (verbs_init_attr->qp_type == IB_QPT_DRIVER)
2365		- qp->qp_sub_type = init_attr->qp_type;
	2466	+ *type = attr->qp_type;
	2467	+ return 0;
2366	2468
2367		- return &qp->ibqp;
	2469	+out:
	2470	+ mlx5_ib_dbg(dev, "Unsupported QP type %d\n", attr->qp_type);
	2471	+ return -EOPNOTSUPP;
	2472	+}
	2473	+
	2474	+static int check_valid_flow(struct mlx5_ib_dev dev, struct ib_pd pd,
	2475	+ struct ib_qp_init_attr *attr,
	2476	+ struct ib_udata *udata)
	2477	+{
	2478	+ struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
	2479	+ udata, struct mlx5_ib_ucontext, ibucontext);
	2480	+
	2481	+ if (!udata) {
	2482	+ /* Kernel create_qp callers */
	2483	+ if (attr->rwq_ind_tbl)
	2484	+ return -EOPNOTSUPP;
	2485	+
	2486	+ switch (attr->qp_type) {
	2487	+ case IB_QPT_RAW_PACKET:
	2488	+ case IB_QPT_DRIVER:
	2489	+ return -EOPNOTSUPP;
	2490	+ default:
	2491	+ return 0;
	2492	+ }
	2493	+ }
	2494	+
	2495	+ /* Userspace create_qp callers */
	2496	+ if (attr->qp_type == IB_QPT_RAW_PACKET && !ucontext->cqe_version) {
	2497	+ mlx5_ib_dbg(dev,
	2498	+ "Raw Packet QP is only supported for CQE version > 0\n");
	2499	+ return -EINVAL;
	2500	+ }
	2501	+
	2502	+ if (attr->qp_type != IB_QPT_RAW_PACKET && attr->rwq_ind_tbl) {
	2503	+ mlx5_ib_dbg(dev,
	2504	+ "Wrong QP type %d for the RWQ indirect table\n",
	2505	+ attr->qp_type);
	2506	+ return -EINVAL;
	2507	+ }
	2508	+
	2509	+ /*
	2510	+ * We don't need to see this warning, it means that kernel code
	2511	+ * missing ib_pd. Placed here to catch developer's mistakes.
	2512	+ */
	2513	+ WARN_ONCE(!pd && attr->qp_type != IB_QPT_XRC_TGT,
	2514	+ "There is a missing PD pointer assignment\n");
	2515	+ return 0;
	2516	+}
	2517	+
	2518	+static void process_vendor_flag(struct mlx5_ib_dev dev, int flags, int flag,
	2519	+ bool cond, struct mlx5_ib_qp *qp)
	2520	+{
	2521	+ if (!(*flags & flag))
	2522	+ return;
	2523	+
	2524	+ if (cond) {
	2525	+ qp->flags_en \|= flag;
	2526	+ *flags &= ~flag;
	2527	+ return;
	2528	+ }
	2529	+
	2530	+ switch (flag) {
	2531	+ case MLX5_QP_FLAG_SCATTER_CQE:
	2532	+ case MLX5_QP_FLAG_ALLOW_SCATTER_CQE:
	2533	+ /*
	2534	+ * We don't return error if these flags were provided,
	2535	+ * and mlx5 doesn't have right capability.
	2536	+ */
	2537	+ *flags &= ~(MLX5_QP_FLAG_SCATTER_CQE \|
	2538	+ MLX5_QP_FLAG_ALLOW_SCATTER_CQE);
	2539	+ return;
	2540	+ default:
	2541	+ break;
	2542	+ }
	2543	+ mlx5_ib_dbg(dev, "Vendor create QP flag 0x%X is not supported\n", flag);
	2544	+}
	2545	+
	2546	+static int process_vendor_flags(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp,
	2547	+ void ucmd, struct ib_qp_init_attr attr)
	2548	+{
	2549	+ struct mlx5_core_dev *mdev = dev->mdev;
	2550	+ bool cond;
	2551	+ int flags;
	2552	+
	2553	+ if (attr->rwq_ind_tbl)
	2554	+ flags = ((struct mlx5_ib_create_qp_rss *)ucmd)->flags;
	2555	+ else
	2556	+ flags = ((struct mlx5_ib_create_qp *)ucmd)->flags;
	2557	+
	2558	+ switch (flags & (MLX5_QP_FLAG_TYPE_DCT \| MLX5_QP_FLAG_TYPE_DCI)) {
	2559	+ case MLX5_QP_FLAG_TYPE_DCI:
	2560	+ qp->type = MLX5_IB_QPT_DCI;
	2561	+ break;
	2562	+ case MLX5_QP_FLAG_TYPE_DCT:
	2563	+ qp->type = MLX5_IB_QPT_DCT;
	2564	+ break;
	2565	+ default:
	2566	+ if (qp->type != IB_QPT_DRIVER)
	2567	+ break;
	2568	+ /*
	2569	+ * It is IB_QPT_DRIVER and or no subtype or
	2570	+ * wrong subtype were provided.
	2571	+ */
	2572	+ return -EINVAL;
	2573	+ }
	2574	+
	2575	+ process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TYPE_DCI, true, qp);
	2576	+ process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TYPE_DCT, true, qp);
	2577	+
	2578	+ process_vendor_flag(dev, &flags, MLX5_QP_FLAG_SIGNATURE, true, qp);
	2579	+ process_vendor_flag(dev, &flags, MLX5_QP_FLAG_SCATTER_CQE,
	2580	+ MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);
	2581	+ process_vendor_flag(dev, &flags, MLX5_QP_FLAG_ALLOW_SCATTER_CQE,
	2582	+ MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);
	2583	+
	2584	+ if (qp->type == IB_QPT_RAW_PACKET) {
	2585	+ cond = MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan) \|\|
	2586	+ MLX5_CAP_ETH(mdev, tunnel_stateless_gre) \|\|
	2587	+ MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx);
	2588	+ process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TUNNEL_OFFLOADS,
	2589	+ cond, qp);
	2590	+ process_vendor_flag(dev, &flags,
	2591	+ MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC, true,
	2592	+ qp);
	2593	+ process_vendor_flag(dev, &flags,
	2594	+ MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC, true,
	2595	+ qp);
	2596	+ }
	2597	+
	2598	+ if (qp->type == IB_QPT_RC)
	2599	+ process_vendor_flag(dev, &flags,
	2600	+ MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE,
	2601	+ MLX5_CAP_GEN(mdev, qp_packet_based), qp);
	2602	+
	2603	+ process_vendor_flag(dev, &flags, MLX5_QP_FLAG_BFREG_INDEX, true, qp);
	2604	+ process_vendor_flag(dev, &flags, MLX5_QP_FLAG_UAR_PAGE_INDEX, true, qp);
	2605	+
	2606	+ cond = qp->flags_en & ~(MLX5_QP_FLAG_TUNNEL_OFFLOADS \|
	2607	+ MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC \|
	2608	+ MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC);
	2609	+ if (attr->rwq_ind_tbl && cond) {
	2610	+ mlx5_ib_dbg(dev, "RSS RAW QP has unsupported flags 0x%X\n",
	2611	+ cond);
	2612	+ return -EINVAL;
	2613	+ }
	2614	+
	2615	+ if (flags)
	2616	+ mlx5_ib_dbg(dev, "udata has unsupported flags 0x%X\n", flags);
	2617	+
	2618	+ return (flags) ? -EINVAL : 0;
	2619	+ }
	2620	+
	2621	+static void process_create_flag(struct mlx5_ib_dev dev, int flags, int flag,
	2622	+ bool cond, struct mlx5_ib_qp *qp)
	2623	+{
	2624	+ if (!(*flags & flag))
	2625	+ return;
	2626	+
	2627	+ if (cond) {
	2628	+ qp->flags \|= flag;
	2629	+ *flags &= ~flag;
	2630	+ return;
	2631	+ }
	2632	+
	2633	+ if (flag == MLX5_IB_QP_CREATE_WC_TEST) {
	2634	+ /*
	2635	+ * Special case, if condition didn't meet, it won't be error,
	2636	+ * just different in-kernel flow.
	2637	+ */
	2638	+ *flags &= ~MLX5_IB_QP_CREATE_WC_TEST;
	2639	+ return;
	2640	+ }
	2641	+ mlx5_ib_dbg(dev, "Verbs create QP flag 0x%X is not supported\n", flag);
	2642	+}
	2643	+
	2644	+static int process_create_flags(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp,
	2645	+ struct ib_qp_init_attr *attr)
	2646	+{
	2647	+ enum ib_qp_type qp_type = qp->type;
	2648	+ struct mlx5_core_dev *mdev = dev->mdev;
	2649	+ int create_flags = attr->create_flags;
	2650	+ bool cond;
	2651	+
	2652	+ if (qp_type == MLX5_IB_QPT_DCT)
	2653	+ return (create_flags) ? -EINVAL : 0;
	2654	+
	2655	+ if (qp_type == IB_QPT_RAW_PACKET && attr->rwq_ind_tbl)
	2656	+ return (create_flags) ? -EINVAL : 0;
	2657	+
	2658	+ process_create_flag(dev, &create_flags, IB_QP_CREATE_NETIF_QP,
	2659	+ mlx5_get_flow_namespace(dev->mdev,
	2660	+ MLX5_FLOW_NAMESPACE_BYPASS),
	2661	+ qp);
	2662	+ process_create_flag(dev, &create_flags,
	2663	+ IB_QP_CREATE_INTEGRITY_EN,
	2664	+ MLX5_CAP_GEN(mdev, sho), qp);
	2665	+ process_create_flag(dev, &create_flags,
	2666	+ IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
	2667	+ MLX5_CAP_GEN(mdev, block_lb_mc), qp);
	2668	+ process_create_flag(dev, &create_flags, IB_QP_CREATE_CROSS_CHANNEL,
	2669	+ MLX5_CAP_GEN(mdev, cd), qp);
	2670	+ process_create_flag(dev, &create_flags, IB_QP_CREATE_MANAGED_SEND,
	2671	+ MLX5_CAP_GEN(mdev, cd), qp);
	2672	+ process_create_flag(dev, &create_flags, IB_QP_CREATE_MANAGED_RECV,
	2673	+ MLX5_CAP_GEN(mdev, cd), qp);
	2674	+
	2675	+ if (qp_type == IB_QPT_UD) {
	2676	+ process_create_flag(dev, &create_flags,
	2677	+ IB_QP_CREATE_IPOIB_UD_LSO,
	2678	+ MLX5_CAP_GEN(mdev, ipoib_basic_offloads),
	2679	+ qp);
	2680	+ cond = MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_IB;
	2681	+ process_create_flag(dev, &create_flags, IB_QP_CREATE_SOURCE_QPN,
	2682	+ cond, qp);
	2683	+ }
	2684	+
	2685	+ if (qp_type == IB_QPT_RAW_PACKET) {
	2686	+ cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
	2687	+ MLX5_CAP_ETH(mdev, scatter_fcs);
	2688	+ process_create_flag(dev, &create_flags,
	2689	+ IB_QP_CREATE_SCATTER_FCS, cond, qp);
	2690	+
	2691	+ cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
	2692	+ MLX5_CAP_ETH(mdev, vlan_cap);
	2693	+ process_create_flag(dev, &create_flags,
	2694	+ IB_QP_CREATE_CVLAN_STRIPPING, cond, qp);
	2695	+ }
	2696	+
	2697	+ process_create_flag(dev, &create_flags,
	2698	+ IB_QP_CREATE_PCI_WRITE_END_PADDING,
	2699	+ MLX5_CAP_GEN(mdev, end_pad), qp);
	2700	+
	2701	+ process_create_flag(dev, &create_flags, MLX5_IB_QP_CREATE_WC_TEST,
	2702	+ qp_type != MLX5_IB_QPT_REG_UMR, qp);
	2703	+ process_create_flag(dev, &create_flags, MLX5_IB_QP_CREATE_SQPN_QP1,
	2704	+ true, qp);
	2705	+
	2706	+ if (create_flags)
	2707	+ mlx5_ib_dbg(dev, "Create QP has unsupported flags 0x%X\n",
	2708	+ create_flags);
	2709	+
	2710	+ return (create_flags) ? -EINVAL : 0;
	2711	+}
	2712	+
	2713	+static int process_udata_size(struct mlx5_ib_dev *dev,
	2714	+ struct mlx5_create_qp_params *params)
	2715	+{
	2716	+ size_t ucmd = sizeof(struct mlx5_ib_create_qp);
	2717	+ struct ib_udata *udata = params->udata;
	2718	+ size_t outlen = udata->outlen;
	2719	+ size_t inlen = udata->inlen;
	2720	+
	2721	+ params->outlen = min(outlen, sizeof(struct mlx5_ib_create_qp_resp));
	2722	+ params->ucmd_size = ucmd;
	2723	+ if (!params->is_rss_raw) {
	2724	+ /* User has old rdma-core, which doesn't support ECE */
	2725	+ size_t min_inlen =
	2726	+ offsetof(struct mlx5_ib_create_qp, ece_options);
	2727	+
	2728	+ /*
	2729	+ * We will check in check_ucmd_data() that user
	2730	+ * cleared everything after inlen.
	2731	+ */
	2732	+ params->inlen = (inlen < min_inlen) ? 0 : min(inlen, ucmd);
	2733	+ goto out;
	2734	+ }
	2735	+
	2736	+ /* RSS RAW QP */
	2737	+ if (inlen < offsetofend(struct mlx5_ib_create_qp_rss, flags))
	2738	+ return -EINVAL;
	2739	+
	2740	+ if (outlen < offsetofend(struct mlx5_ib_create_qp_resp, bfreg_index))
	2741	+ return -EINVAL;
	2742	+
	2743	+ ucmd = sizeof(struct mlx5_ib_create_qp_rss);
	2744	+ params->ucmd_size = ucmd;
	2745	+ if (inlen > ucmd && !ib_is_udata_cleared(udata, ucmd, inlen - ucmd))
	2746	+ return -EINVAL;
	2747	+
	2748	+ params->inlen = min(ucmd, inlen);
	2749	+out:
	2750	+ if (!params->inlen)
	2751	+ mlx5_ib_dbg(dev, "udata is too small\n");
	2752	+
	2753	+ return (params->inlen) ? 0 : -EINVAL;
	2754	+}
	2755	+
	2756	+static int create_qp(struct mlx5_ib_dev dev, struct ib_pd pd,
	2757	+ struct mlx5_ib_qp *qp,
	2758	+ struct mlx5_create_qp_params *params)
	2759	+{
	2760	+ int err;
	2761	+
	2762	+ if (params->is_rss_raw) {
	2763	+ err = create_rss_raw_qp_tir(dev, pd, qp, params);
	2764	+ goto out;
	2765	+ }
	2766	+
	2767	+ switch (qp->type) {
	2768	+ case MLX5_IB_QPT_DCT:
	2769	+ err = create_dct(dev, pd, qp, params);
	2770	+ break;
	2771	+ case IB_QPT_XRC_TGT:
	2772	+ err = create_xrc_tgt_qp(dev, qp, params);
	2773	+ break;
	2774	+ case IB_QPT_GSI:
	2775	+ err = mlx5_ib_create_gsi(pd, qp, params->attr);
	2776	+ break;
	2777	+ default:
	2778	+ if (params->udata)
	2779	+ err = create_user_qp(dev, pd, qp, params);
	2780	+ else
	2781	+ err = create_kernel_qp(dev, pd, qp, params);
	2782	+ }
	2783	+
	2784	+out:
	2785	+ if (err) {
	2786	+ mlx5_ib_err(dev, "Create QP type %d failed\n", qp->type);
	2787	+ return err;
	2788	+ }
	2789	+
	2790	+ if (is_qp0(qp->type))
	2791	+ qp->ibqp.qp_num = 0;
	2792	+ else if (is_qp1(qp->type))
	2793	+ qp->ibqp.qp_num = 1;
	2794	+ else
	2795	+ qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn;
	2796	+
	2797	+ mlx5_ib_dbg(dev,
	2798	+ "QP type %d, ib qpn 0x%X, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x, ece 0x%x\n",
	2799	+ qp->type, qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn,
	2800	+ params->attr->recv_cq ? to_mcq(params->attr->recv_cq)->mcq.cqn :
	2801	+ -1,
	2802	+ params->attr->send_cq ? to_mcq(params->attr->send_cq)->mcq.cqn :
	2803	+ -1,
	2804	+ params->resp.ece_options);
	2805	+
	2806	+ return 0;
	2807	+}
	2808	+
	2809	+static int check_qp_attr(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp,
	2810	+ struct ib_qp_init_attr *attr)
	2811	+{
	2812	+ int ret = 0;
	2813	+
	2814	+ switch (qp->type) {
	2815	+ case MLX5_IB_QPT_DCT:
	2816	+ ret = (!attr->srq \|\| !attr->recv_cq) ? -EINVAL : 0;
	2817	+ break;
	2818	+ case MLX5_IB_QPT_DCI:
	2819	+ ret = (attr->cap.max_recv_wr \|\| attr->cap.max_recv_sge) ?
	2820	+ -EINVAL :
	2821	+ 0;
	2822	+ break;
	2823	+ case IB_QPT_RAW_PACKET:
	2824	+ ret = (attr->rwq_ind_tbl && attr->send_cq) ? -EINVAL : 0;
	2825	+ break;
	2826	+ default:
	2827	+ break;
	2828	+ }
	2829	+
	2830	+ if (ret)
	2831	+ mlx5_ib_dbg(dev, "QP type %d has wrong attributes\n", qp->type);
	2832	+
	2833	+ return ret;
	2834	+}
	2835	+
	2836	+static int get_qp_uidx(struct mlx5_ib_qp *qp,
	2837	+ struct mlx5_create_qp_params *params)
	2838	+{
	2839	+ struct mlx5_ib_create_qp *ucmd = params->ucmd;
	2840	+ struct ib_udata *udata = params->udata;
	2841	+ struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
	2842	+ udata, struct mlx5_ib_ucontext, ibucontext);
	2843	+
	2844	+ if (params->is_rss_raw)
	2845	+ return 0;
	2846	+
	2847	+ return get_qp_user_index(ucontext, ucmd, sizeof(*ucmd), &params->uidx);
2368	2848	}
2369	2849
2370	2850	static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)
..	..	@@ -2374,7 +2854,7 @@
2374	2854	if (mqp->state == IB_QPS_RTR) {
2375	2855	int err;
2376	2856
2377		- err = mlx5_core_destroy_dct(dev->mdev, &mqp->dct.mdct);
	2857	+ err = mlx5_core_destroy_dct(dev, &mqp->dct.mdct);
2378	2858	if (err) {
2379	2859	mlx5_ib_warn(dev, "failed to destroy DCT %d\n", err);
2380	2860	return err;
..	..	@@ -2386,28 +2866,177 @@
2386	2866	return 0;
2387	2867	}
2388	2868
2389		-int mlx5_ib_destroy_qp(struct ib_qp *qp)
	2869	+static int check_ucmd_data(struct mlx5_ib_dev *dev,
	2870	+ struct mlx5_create_qp_params *params)
	2871	+{
	2872	+ struct ib_udata *udata = params->udata;
	2873	+ size_t size, last;
	2874	+ int ret;
	2875	+
	2876	+ if (params->is_rss_raw)
	2877	+ /*
	2878	+ * These QPs don't have "reserved" field in their
	2879	+ * create_qp input struct, so their data is always valid.
	2880	+ */
	2881	+ last = sizeof(struct mlx5_ib_create_qp_rss);
	2882	+ else
	2883	+ last = offsetof(struct mlx5_ib_create_qp, reserved);
	2884	+
	2885	+ if (udata->inlen <= last)
	2886	+ return 0;
	2887	+
	2888	+ /*
	2889	+ * User provides different create_qp structures based on the
	2890	+ * flow and we need to know if he cleared memory after our
	2891	+ * struct create_qp ends.
	2892	+ */
	2893	+ size = udata->inlen - last;
	2894	+ ret = ib_is_udata_cleared(params->udata, last, size);
	2895	+ if (!ret)
	2896	+ mlx5_ib_dbg(
	2897	+ dev,
	2898	+ "udata is not cleared, inlen = %zu, ucmd = %zu, last = %zu, size = %zu\n",
	2899	+ udata->inlen, params->ucmd_size, last, size);
	2900	+ return ret ? 0 : -EINVAL;
	2901	+}
	2902	+
	2903	+struct ib_qp mlx5_ib_create_qp(struct ib_pd pd, struct ib_qp_init_attr *attr,
	2904	+ struct ib_udata *udata)
	2905	+{
	2906	+ struct mlx5_create_qp_params params = {};
	2907	+ struct mlx5_ib_dev *dev;
	2908	+ struct mlx5_ib_qp *qp;
	2909	+ enum ib_qp_type type;
	2910	+ int err;
	2911	+
	2912	+ dev = pd ? to_mdev(pd->device) :
	2913	+ to_mdev(to_mxrcd(attr->xrcd)->ibxrcd.device);
	2914	+
	2915	+ err = check_qp_type(dev, attr, &type);
	2916	+ if (err)
	2917	+ return ERR_PTR(err);
	2918	+
	2919	+ err = check_valid_flow(dev, pd, attr, udata);
	2920	+ if (err)
	2921	+ return ERR_PTR(err);
	2922	+
	2923	+ params.udata = udata;
	2924	+ params.uidx = MLX5_IB_DEFAULT_UIDX;
	2925	+ params.attr = attr;
	2926	+ params.is_rss_raw = !!attr->rwq_ind_tbl;
	2927	+
	2928	+ if (udata) {
	2929	+ err = process_udata_size(dev, &params);
	2930	+ if (err)
	2931	+ return ERR_PTR(err);
	2932	+
	2933	+ err = check_ucmd_data(dev, &params);
	2934	+ if (err)
	2935	+ return ERR_PTR(err);
	2936	+
	2937	+ params.ucmd = kzalloc(params.ucmd_size, GFP_KERNEL);
	2938	+ if (!params.ucmd)
	2939	+ return ERR_PTR(-ENOMEM);
	2940	+
	2941	+ err = ib_copy_from_udata(params.ucmd, udata, params.inlen);
	2942	+ if (err)
	2943	+ goto free_ucmd;
	2944	+ }
	2945	+
	2946	+ qp = kzalloc(sizeof(*qp), GFP_KERNEL);
	2947	+ if (!qp) {
	2948	+ err = -ENOMEM;
	2949	+ goto free_ucmd;
	2950	+ }
	2951	+
	2952	+ mutex_init(&qp->mutex);
	2953	+ qp->type = type;
	2954	+ if (udata) {
	2955	+ err = process_vendor_flags(dev, qp, params.ucmd, attr);
	2956	+ if (err)
	2957	+ goto free_qp;
	2958	+
	2959	+ err = get_qp_uidx(qp, &params);
	2960	+ if (err)
	2961	+ goto free_qp;
	2962	+ }
	2963	+ err = process_create_flags(dev, qp, attr);
	2964	+ if (err)
	2965	+ goto free_qp;
	2966	+
	2967	+ err = check_qp_attr(dev, qp, attr);
	2968	+ if (err)
	2969	+ goto free_qp;
	2970	+
	2971	+ err = create_qp(dev, pd, qp, &params);
	2972	+ if (err)
	2973	+ goto free_qp;
	2974	+
	2975	+ kfree(params.ucmd);
	2976	+ params.ucmd = NULL;
	2977	+
	2978	+ if (udata)
	2979	+ /*
	2980	+ * It is safe to copy response for all user create QP flows,
	2981	+ * including MLX5_IB_QPT_DCT, which doesn't need it.
	2982	+ * In that case, resp will be filled with zeros.
	2983	+ */
	2984	+ err = ib_copy_to_udata(udata, &params.resp, params.outlen);
	2985	+ if (err)
	2986	+ goto destroy_qp;
	2987	+
	2988	+ return &qp->ibqp;
	2989	+
	2990	+destroy_qp:
	2991	+ switch (qp->type) {
	2992	+ case MLX5_IB_QPT_DCT:
	2993	+ mlx5_ib_destroy_dct(qp);
	2994	+ break;
	2995	+ case IB_QPT_GSI:
	2996	+ mlx5_ib_destroy_gsi(qp);
	2997	+ break;
	2998	+ default:
	2999	+ /*
	3000	+ * These lines below are temp solution till QP allocation
	3001	+ * will be moved to be under IB/core responsiblity.
	3002	+ */
	3003	+ qp->ibqp.send_cq = attr->send_cq;
	3004	+ qp->ibqp.recv_cq = attr->recv_cq;
	3005	+ qp->ibqp.pd = pd;
	3006	+ destroy_qp_common(dev, qp, udata);
	3007	+ }
	3008	+
	3009	+ qp = NULL;
	3010	+free_qp:
	3011	+ kfree(qp);
	3012	+free_ucmd:
	3013	+ kfree(params.ucmd);
	3014	+ return ERR_PTR(err);
	3015	+}
	3016	+
	3017	+int mlx5_ib_destroy_qp(struct ib_qp qp, struct ib_udata udata)
2390	3018	{
2391	3019	struct mlx5_ib_dev *dev = to_mdev(qp->device);
2392	3020	struct mlx5_ib_qp *mqp = to_mqp(qp);
2393	3021
2394	3022	if (unlikely(qp->qp_type == IB_QPT_GSI))
2395		- return mlx5_ib_gsi_destroy_qp(qp);
	3023	+ return mlx5_ib_destroy_gsi(mqp);
2396	3024
2397		- if (mqp->qp_sub_type == MLX5_IB_QPT_DCT)
	3025	+ if (mqp->type == MLX5_IB_QPT_DCT)
2398	3026	return mlx5_ib_destroy_dct(mqp);
2399	3027
2400		- destroy_qp_common(dev, mqp);
	3028	+ destroy_qp_common(dev, mqp, udata);
2401	3029
2402	3030	kfree(mqp);
2403	3031
2404	3032	return 0;
2405	3033	}
2406	3034
2407		-static __be32 to_mlx5_access_flags(struct mlx5_ib_qp qp, const struct ib_qp_attr attr,
2408		- int attr_mask)
	3035	+static int set_qpc_atomic_flags(struct mlx5_ib_qp *qp,
	3036	+ const struct ib_qp_attr *attr, int attr_mask,
	3037	+ void *qpc)
2409	3038	{
2410		- u32 hw_access_flags = 0;
	3039	+ struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device);
2411	3040	u8 dest_rd_atomic;
2412	3041	u32 access_flags;
2413	3042
..	..	@@ -2424,14 +3053,21 @@
2424	3053	if (!dest_rd_atomic)
2425	3054	access_flags &= IB_ACCESS_REMOTE_WRITE;
2426	3055
2427		- if (access_flags & IB_ACCESS_REMOTE_READ)
2428		- hw_access_flags \|= MLX5_QP_BIT_RRE;
2429		- if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
2430		- hw_access_flags \|= (MLX5_QP_BIT_RAE \| MLX5_ATOMIC_MODE_CX);
2431		- if (access_flags & IB_ACCESS_REMOTE_WRITE)
2432		- hw_access_flags \|= MLX5_QP_BIT_RWE;
	3056	+ MLX5_SET(qpc, qpc, rre, !!(access_flags & IB_ACCESS_REMOTE_READ));
2433	3057
2434		- return cpu_to_be32(hw_access_flags);
	3058	+ if (access_flags & IB_ACCESS_REMOTE_ATOMIC) {
	3059	+ int atomic_mode;
	3060	+
	3061	+ atomic_mode = get_atomic_mode(dev, qp->ibqp.qp_type);
	3062	+ if (atomic_mode < 0)
	3063	+ return -EOPNOTSUPP;
	3064	+
	3065	+ MLX5_SET(qpc, qpc, rae, 1);
	3066	+ MLX5_SET(qpc, qpc, atomic_mode, atomic_mode);
	3067	+ }
	3068	+
	3069	+ MLX5_SET(qpc, qpc, rwe, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
	3070	+ return 0;
2435	3071	}
2436	3072
2437	3073	enum {
..	..	@@ -2440,24 +3076,62 @@
2440	3076	MLX5_PATH_FLAG_COUNTER = 1 << 2,
2441	3077	};
2442	3078
	3079	+static int mlx5_to_ib_rate_map(u8 rate)
	3080	+{
	3081	+ static const int rates[] = { IB_RATE_PORT_CURRENT, IB_RATE_56_GBPS,
	3082	+ IB_RATE_25_GBPS, IB_RATE_100_GBPS,
	3083	+ IB_RATE_200_GBPS, IB_RATE_50_GBPS,
	3084	+ IB_RATE_400_GBPS };
	3085	+
	3086	+ if (rate < ARRAY_SIZE(rates))
	3087	+ return rates[rate];
	3088	+
	3089	+ return rate - MLX5_STAT_RATE_OFFSET;
	3090	+}
	3091	+
	3092	+static int ib_to_mlx5_rate_map(u8 rate)
	3093	+{
	3094	+ switch (rate) {
	3095	+ case IB_RATE_PORT_CURRENT:
	3096	+ return 0;
	3097	+ case IB_RATE_56_GBPS:
	3098	+ return 1;
	3099	+ case IB_RATE_25_GBPS:
	3100	+ return 2;
	3101	+ case IB_RATE_100_GBPS:
	3102	+ return 3;
	3103	+ case IB_RATE_200_GBPS:
	3104	+ return 4;
	3105	+ case IB_RATE_50_GBPS:
	3106	+ return 5;
	3107	+ default:
	3108	+ return rate + MLX5_STAT_RATE_OFFSET;
	3109	+ };
	3110	+
	3111	+ return 0;
	3112	+}
	3113	+
2443	3114	static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate)
2444	3115	{
	3116	+ u32 stat_rate_support;
	3117	+
2445	3118	if (rate == IB_RATE_PORT_CURRENT)
2446	3119	return 0;
2447	3120
2448		- if (rate < IB_RATE_2_5_GBPS \|\| rate > IB_RATE_300_GBPS)
	3121	+ if (rate < IB_RATE_2_5_GBPS \|\| rate > IB_RATE_600_GBPS)
2449	3122	return -EINVAL;
2450	3123
	3124	+ stat_rate_support = MLX5_CAP_GEN(dev->mdev, stat_rate_support);
2451	3125	while (rate != IB_RATE_PORT_CURRENT &&
2452		- !(1 << (rate + MLX5_STAT_RATE_OFFSET) &
2453		- MLX5_CAP_GEN(dev->mdev, stat_rate_support)))
	3126	+ !(1 << ib_to_mlx5_rate_map(rate) & stat_rate_support))
2454	3127	--rate;
2455	3128
2456		- return rate ? rate + MLX5_STAT_RATE_OFFSET : rate;
	3129	+ return ib_to_mlx5_rate_map(rate);
2457	3130	}
2458	3131
2459	3132	static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
2460		- struct mlx5_ib_sq *sq, u8 sl)
	3133	+ struct mlx5_ib_sq *sq, u8 sl,
	3134	+ struct ib_pd *pd)
2461	3135	{
2462	3136	void *in;
2463	3137	void *tisc;
..	..	@@ -2470,11 +3144,12 @@
2470	3144	return -ENOMEM;
2471	3145
2472	3146	MLX5_SET(modify_tis_in, in, bitmask.prio, 1);
	3147	+ MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
2473	3148
2474	3149	tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
2475	3150	MLX5_SET(tisc, tisc, prio, ((sl & 0x7) << 1));
2476	3151
2477		- err = mlx5_core_modify_tis(dev, sq->tisn, in, inlen);
	3152	+ err = mlx5_core_modify_tis(dev, sq->tisn, in);
2478	3153
2479	3154	kvfree(in);
2480	3155
..	..	@@ -2482,7 +3157,8 @@
2482	3157	}
2483	3158
2484	3159	static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
2485		- struct mlx5_ib_sq *sq, u8 tx_affinity)
	3160	+ struct mlx5_ib_sq *sq, u8 tx_affinity,
	3161	+ struct ib_pd *pd)
2486	3162	{
2487	3163	void *in;
2488	3164	void *tisc;
..	..	@@ -2495,22 +3171,34 @@
2495	3171	return -ENOMEM;
2496	3172
2497	3173	MLX5_SET(modify_tis_in, in, bitmask.lag_tx_port_affinity, 1);
	3174	+ MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
2498	3175
2499	3176	tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
2500	3177	MLX5_SET(tisc, tisc, lag_tx_port_affinity, tx_affinity);
2501	3178
2502		- err = mlx5_core_modify_tis(dev, sq->tisn, in, inlen);
	3179	+ err = mlx5_core_modify_tis(dev, sq->tisn, in);
2503	3180
2504	3181	kvfree(in);
2505	3182
2506	3183	return err;
2507	3184	}
2508	3185
	3186	+static void mlx5_set_path_udp_sport(void path, const struct rdma_ah_attr ah,
	3187	+ u32 lqpn, u32 rqpn)
	3188	+
	3189	+{
	3190	+ u32 fl = ah->grh.flow_label;
	3191	+
	3192	+ if (!fl)
	3193	+ fl = rdma_calc_flow_label(lqpn, rqpn);
	3194	+
	3195	+ MLX5_SET(ads, path, udp_sport, rdma_flow_label_to_udp_sport(fl));
	3196	+}
	3197	+
2509	3198	static int mlx5_set_path(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp,
2510		- const struct rdma_ah_attr *ah,
2511		- struct mlx5_qp_path *path, u8 port, int attr_mask,
2512		- u32 path_flags, const struct ib_qp_attr *attr,
2513		- bool alt)
	3199	+ const struct rdma_ah_attr ah, void path, u8 port,
	3200	+ int attr_mask, u32 path_flags,
	3201	+ const struct ib_qp_attr *attr, bool alt)
2514	3202	{
2515	3203	const struct ib_global_route *grh = rdma_ah_read_grh(ah);
2516	3204	int err;
..	..	@@ -2519,8 +3207,8 @@
2519	3207	u8 sl = rdma_ah_get_sl(ah);
2520	3208
2521	3209	if (attr_mask & IB_QP_PKEY_INDEX)
2522		- path->pkey_index = cpu_to_be16(alt ? attr->alt_pkey_index :
2523		- attr->pkey_index);
	3210	+ MLX5_SET(ads, path, pkey_index,
	3211	+ alt ? attr->alt_pkey_index : attr->pkey_index);
2524	3212
2525	3213	if (ah_flags & IB_AH_GRH) {
2526	3214	if (grh->sgid_index >=
..	..	@@ -2536,50 +3224,54 @@
2536	3224	if (!(ah_flags & IB_AH_GRH))
2537	3225	return -EINVAL;
2538	3226
2539		- memcpy(path->rmac, ah->roce.dmac, sizeof(ah->roce.dmac));
2540		- if (qp->ibqp.qp_type == IB_QPT_RC \|\|
2541		- qp->ibqp.qp_type == IB_QPT_UC \|\|
2542		- qp->ibqp.qp_type == IB_QPT_XRC_INI \|\|
2543		- qp->ibqp.qp_type == IB_QPT_XRC_TGT)
2544		- path->udp_sport =
2545		- mlx5_get_roce_udp_sport(dev, ah->grh.sgid_attr);
2546		- path->dci_cfi_prio_sl = (sl & 0x7) << 4;
	3227	+ ether_addr_copy(MLX5_ADDR_OF(ads, path, rmac_47_32),
	3228	+ ah->roce.dmac);
	3229	+ if ((qp->ibqp.qp_type == IB_QPT_RC \|\|
	3230	+ qp->ibqp.qp_type == IB_QPT_UC \|\|
	3231	+ qp->ibqp.qp_type == IB_QPT_XRC_INI \|\|
	3232	+ qp->ibqp.qp_type == IB_QPT_XRC_TGT) &&
	3233	+ (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) &&
	3234	+ (attr_mask & IB_QP_DEST_QPN))
	3235	+ mlx5_set_path_udp_sport(path, ah,
	3236	+ qp->ibqp.qp_num,
	3237	+ attr->dest_qp_num);
	3238	+ MLX5_SET(ads, path, eth_prio, sl & 0x7);
2547	3239	gid_type = ah->grh.sgid_attr->gid_type;
2548	3240	if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
2549		- path->ecn_dscp = (grh->traffic_class >> 2) & 0x3f;
	3241	+ MLX5_SET(ads, path, dscp, grh->traffic_class >> 2);
2550	3242	} else {
2551		- path->fl_free_ar = (path_flags & MLX5_PATH_FLAG_FL) ? 0x80 : 0;
2552		- path->fl_free_ar \|=
2553		- (path_flags & MLX5_PATH_FLAG_FREE_AR) ? 0x40 : 0;
2554		- path->rlid = cpu_to_be16(rdma_ah_get_dlid(ah));
2555		- path->grh_mlid = rdma_ah_get_path_bits(ah) & 0x7f;
2556		- if (ah_flags & IB_AH_GRH)
2557		- path->grh_mlid \|= 1 << 7;
2558		- path->dci_cfi_prio_sl = sl & 0xf;
	3243	+ MLX5_SET(ads, path, fl, !!(path_flags & MLX5_PATH_FLAG_FL));
	3244	+ MLX5_SET(ads, path, free_ar,
	3245	+ !!(path_flags & MLX5_PATH_FLAG_FREE_AR));
	3246	+ MLX5_SET(ads, path, rlid, rdma_ah_get_dlid(ah));
	3247	+ MLX5_SET(ads, path, mlid, rdma_ah_get_path_bits(ah));
	3248	+ MLX5_SET(ads, path, grh, !!(ah_flags & IB_AH_GRH));
	3249	+ MLX5_SET(ads, path, sl, sl);
2559	3250	}
2560	3251
2561	3252	if (ah_flags & IB_AH_GRH) {
2562		- path->mgid_index = grh->sgid_index;
2563		- path->hop_limit = grh->hop_limit;
2564		- path->tclass_flowlabel =
2565		- cpu_to_be32((grh->traffic_class << 20) \|
2566		- (grh->flow_label));
2567		- memcpy(path->rgid, grh->dgid.raw, 16);
	3253	+ MLX5_SET(ads, path, src_addr_index, grh->sgid_index);
	3254	+ MLX5_SET(ads, path, hop_limit, grh->hop_limit);
	3255	+ MLX5_SET(ads, path, tclass, grh->traffic_class);
	3256	+ MLX5_SET(ads, path, flow_label, grh->flow_label);
	3257	+ memcpy(MLX5_ADDR_OF(ads, path, rgid_rip), grh->dgid.raw,
	3258	+ sizeof(grh->dgid.raw));
2568	3259	}
2569	3260
2570	3261	err = ib_rate_to_mlx5(dev, rdma_ah_get_static_rate(ah));
2571	3262	if (err < 0)
2572	3263	return err;
2573		- path->static_rate = err;
2574		- path->port = port;
	3264	+ MLX5_SET(ads, path, stat_rate, err);
	3265	+ MLX5_SET(ads, path, vhca_port_num, port);
2575	3266
2576	3267	if (attr_mask & IB_QP_TIMEOUT)
2577		- path->ackto_lt = (alt ? attr->alt_timeout : attr->timeout) << 3;
	3268	+ MLX5_SET(ads, path, ack_timeout,
	3269	+ alt ? attr->alt_timeout : attr->timeout);
2578	3270
2579	3271	if ((qp->ibqp.qp_type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt)
2580	3272	return modify_raw_packet_eth_prio(dev->mdev,
2581	3273	&qp->raw_packet_qp.sq,
2582		- sl & 0xf);
	3274	+ sl & 0xf, qp->ibqp.pd);
2583	3275
2584	3276	return 0;
2585	3277	}
..	..	@@ -2591,10 +3283,12 @@
2591	3283	MLX5_QP_OPTPAR_RAE \|
2592	3284	MLX5_QP_OPTPAR_RWE \|
2593	3285	MLX5_QP_OPTPAR_PKEY_INDEX \|
2594		- MLX5_QP_OPTPAR_PRI_PORT,
	3286	+ MLX5_QP_OPTPAR_PRI_PORT \|
	3287	+ MLX5_QP_OPTPAR_LAG_TX_AFF,
2595	3288	[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE \|
2596	3289	MLX5_QP_OPTPAR_PKEY_INDEX \|
2597		- MLX5_QP_OPTPAR_PRI_PORT,
	3290	+ MLX5_QP_OPTPAR_PRI_PORT \|
	3291	+ MLX5_QP_OPTPAR_LAG_TX_AFF,
2598	3292	[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX \|
2599	3293	MLX5_QP_OPTPAR_Q_KEY \|
2600	3294	MLX5_QP_OPTPAR_PRI_PORT,
..	..	@@ -2602,17 +3296,20 @@
2602	3296	MLX5_QP_OPTPAR_RAE \|
2603	3297	MLX5_QP_OPTPAR_RWE \|
2604	3298	MLX5_QP_OPTPAR_PKEY_INDEX \|
2605		- MLX5_QP_OPTPAR_PRI_PORT,
	3299	+ MLX5_QP_OPTPAR_PRI_PORT \|
	3300	+ MLX5_QP_OPTPAR_LAG_TX_AFF,
2606	3301	},
2607	3302	[MLX5_QP_STATE_RTR] = {
2608	3303	[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH \|
2609	3304	MLX5_QP_OPTPAR_RRE \|
2610	3305	MLX5_QP_OPTPAR_RAE \|
2611	3306	MLX5_QP_OPTPAR_RWE \|
2612		- MLX5_QP_OPTPAR_PKEY_INDEX,
	3307	+ MLX5_QP_OPTPAR_PKEY_INDEX \|
	3308	+ MLX5_QP_OPTPAR_LAG_TX_AFF,
2613	3309	[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH \|
2614	3310	MLX5_QP_OPTPAR_RWE \|
2615		- MLX5_QP_OPTPAR_PKEY_INDEX,
	3311	+ MLX5_QP_OPTPAR_PKEY_INDEX \|
	3312	+ MLX5_QP_OPTPAR_LAG_TX_AFF,
2616	3313	[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX \|
2617	3314	MLX5_QP_OPTPAR_Q_KEY,
2618	3315	[MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_PKEY_INDEX \|
..	..	@@ -2621,7 +3318,8 @@
2621	3318	MLX5_QP_OPTPAR_RRE \|
2622	3319	MLX5_QP_OPTPAR_RAE \|
2623	3320	MLX5_QP_OPTPAR_RWE \|
2624		- MLX5_QP_OPTPAR_PKEY_INDEX,
	3321	+ MLX5_QP_OPTPAR_PKEY_INDEX \|
	3322	+ MLX5_QP_OPTPAR_LAG_TX_AFF,
2625	3323	},
2626	3324	},
2627	3325	[MLX5_QP_STATE_RTR] = {
..	..	@@ -2748,9 +3446,9 @@
2748	3446	return result;
2749	3447	}
2750	3448
2751		-static int modify_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
2752		- struct mlx5_ib_rq *rq, int new_state,
2753		- const struct mlx5_modify_raw_qp_param *raw_qp_param)
	3449	+static int modify_raw_packet_qp_rq(
	3450	+ struct mlx5_ib_dev dev, struct mlx5_ib_rq rq, int new_state,
	3451	+ const struct mlx5_modify_raw_qp_param raw_qp_param, struct ib_pd pd)
2754	3452	{
2755	3453	void *in;
2756	3454	void *rqc;
..	..	@@ -2763,6 +3461,7 @@
2763	3461	return -ENOMEM;
2764	3462
2765	3463	MLX5_SET(modify_rq_in, in, rq_state, rq->state);
	3464	+ MLX5_SET(modify_rq_in, in, uid, to_mpd(pd)->uid);
2766	3465
2767	3466	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
2768	3467	MLX5_SET(rqc, rqc, state, new_state);
..	..	@@ -2773,11 +3472,12 @@
2773	3472	MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
2774	3473	MLX5_SET(rqc, rqc, counter_set_id, raw_qp_param->rq_q_ctr_id);
2775	3474	} else
2776		- pr_info_once("%s: RAW PACKET QP counters are not supported on current FW\n",
2777		- dev->ib_dev.name);
	3475	+ dev_info_once(
	3476	+ &dev->ib_dev.dev,
	3477	+ "RAW PACKET QP counters are not supported on current FW\n");
2778	3478	}
2779	3479
2780		- err = mlx5_core_modify_rq(dev->mdev, rq->base.mqp.qpn, in, inlen);
	3480	+ err = mlx5_core_modify_rq(dev->mdev, rq->base.mqp.qpn, in);
2781	3481	if (err)
2782	3482	goto out;
2783	3483
..	..	@@ -2788,10 +3488,9 @@
2788	3488	return err;
2789	3489	}
2790	3490
2791		-static int modify_raw_packet_qp_sq(struct mlx5_core_dev *dev,
2792		- struct mlx5_ib_sq *sq,
2793		- int new_state,
2794		- const struct mlx5_modify_raw_qp_param *raw_qp_param)
	3491	+static int modify_raw_packet_qp_sq(
	3492	+ struct mlx5_core_dev dev, struct mlx5_ib_sq sq, int new_state,
	3493	+ const struct mlx5_modify_raw_qp_param raw_qp_param, struct ib_pd pd)
2795	3494	{
2796	3495	struct mlx5_ib_qp *ibqp = sq->base.container_mibqp;
2797	3496	struct mlx5_rate_limit old_rl = ibqp->rl;
..	..	@@ -2808,6 +3507,7 @@
2808	3507	if (!in)
2809	3508	return -ENOMEM;
2810	3509
	3510	+ MLX5_SET(modify_sq_in, in, uid, to_mpd(pd)->uid);
2811	3511	MLX5_SET(modify_sq_in, in, sq_state, sq->state);
2812	3512
2813	3513	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
..	..	@@ -2840,7 +3540,7 @@
2840	3540	MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index);
2841	3541	}
2842	3542
2843		- err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in, inlen);
	3543	+ err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in);
2844	3544	if (err) {
2845	3545	/* Remove new rate from table if failed */
2846	3546	if (new_rate_added)
..	..	@@ -2880,7 +3580,7 @@
2880	3580	switch (raw_qp_param->operation) {
2881	3581	case MLX5_CMD_OP_RST2INIT_QP:
2882	3582	rq_state = MLX5_RQC_STATE_RDY;
2883		- sq_state = MLX5_SQC_STATE_RDY;
	3583	+ sq_state = MLX5_SQC_STATE_RST;
2884	3584	break;
2885	3585	case MLX5_CMD_OP_2ERR_QP:
2886	3586	rq_state = MLX5_RQC_STATE_ERR;
..	..	@@ -2892,13 +3592,11 @@
2892	3592	break;
2893	3593	case MLX5_CMD_OP_RTR2RTS_QP:
2894	3594	case MLX5_CMD_OP_RTS2RTS_QP:
2895		- if (raw_qp_param->set_mask ==
2896		- MLX5_RAW_QP_RATE_LIMIT) {
2897		- modify_rq = 0;
2898		- sq_state = sq->state;
2899		- } else {
2900		- return raw_qp_param->set_mask ? -EINVAL : 0;
2901		- }
	3595	+ if (raw_qp_param->set_mask & ~MLX5_RAW_QP_RATE_LIMIT)
	3596	+ return -EINVAL;
	3597	+
	3598	+ modify_rq = 0;
	3599	+ sq_state = MLX5_SQC_STATE_RDY;
2902	3600	break;
2903	3601	case MLX5_CMD_OP_INIT2INIT_QP:
2904	3602	case MLX5_CMD_OP_INIT2RTR_QP:
..	..	@@ -2912,60 +3610,144 @@
2912	3610	}
2913	3611
2914	3612	if (modify_rq) {
2915		- err = modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param);
	3613	+ err = modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param,
	3614	+ qp->ibqp.pd);
2916	3615	if (err)
2917	3616	return err;
2918	3617	}
2919	3618
2920	3619	if (modify_sq) {
	3620	+ struct mlx5_flow_handle *flow_rule;
	3621	+
2921	3622	if (tx_affinity) {
2922	3623	err = modify_raw_packet_tx_affinity(dev->mdev, sq,
2923		- tx_affinity);
	3624	+ tx_affinity,
	3625	+ qp->ibqp.pd);
2924	3626	if (err)
2925	3627	return err;
2926	3628	}
2927	3629
2928		- return modify_raw_packet_qp_sq(dev->mdev, sq, sq_state, raw_qp_param);
	3630	+ flow_rule = create_flow_rule_vport_sq(dev, sq,
	3631	+ raw_qp_param->port);
	3632	+ if (IS_ERR(flow_rule))
	3633	+ return PTR_ERR(flow_rule);
	3634	+
	3635	+ err = modify_raw_packet_qp_sq(dev->mdev, sq, sq_state,
	3636	+ raw_qp_param, qp->ibqp.pd);
	3637	+ if (err) {
	3638	+ if (flow_rule)
	3639	+ mlx5_del_flow_rules(flow_rule);
	3640	+ return err;
	3641	+ }
	3642	+
	3643	+ if (flow_rule) {
	3644	+ destroy_flow_rule_vport_sq(sq);
	3645	+ sq->flow_rule = flow_rule;
	3646	+ }
	3647	+
	3648	+ return err;
2929	3649	}
2930	3650
2931	3651	return 0;
2932	3652	}
2933	3653
2934		-static unsigned int get_tx_affinity(struct mlx5_ib_dev *dev,
2935		- struct mlx5_ib_pd *pd,
2936		- struct mlx5_ib_qp_base *qp_base,
2937		- u8 port_num)
	3654	+static unsigned int get_tx_affinity_rr(struct mlx5_ib_dev *dev,
	3655	+ struct ib_udata *udata)
2938	3656	{
2939		- struct mlx5_ib_ucontext *ucontext = NULL;
2940		- unsigned int tx_port_affinity;
	3657	+ struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
	3658	+ udata, struct mlx5_ib_ucontext, ibucontext);
	3659	+ u8 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
	3660	+ atomic_t *tx_port_affinity;
2941	3661
2942		- if (pd && pd->ibpd.uobject && pd->ibpd.uobject->context)
2943		- ucontext = to_mucontext(pd->ibpd.uobject->context);
	3662	+ if (ucontext)
	3663	+ tx_port_affinity = &ucontext->tx_port_affinity;
	3664	+ else
	3665	+ tx_port_affinity = &dev->port[port_num].roce.tx_port_affinity;
2944	3666
2945		- if (ucontext) {
2946		- tx_port_affinity = (unsigned int)atomic_add_return(
2947		- 1, &ucontext->tx_port_affinity) %
2948		- MLX5_MAX_PORTS +
2949		- 1;
	3667	+ return (unsigned int)atomic_add_return(1, tx_port_affinity) %
	3668	+ MLX5_MAX_PORTS + 1;
	3669	+}
	3670	+
	3671	+static bool qp_supports_affinity(struct mlx5_ib_qp *qp)
	3672	+{
	3673	+ if ((qp->type == IB_QPT_RC) \|\| (qp->type == IB_QPT_UD) \|\|
	3674	+ (qp->type == IB_QPT_UC) \|\| (qp->type == IB_QPT_RAW_PACKET) \|\|
	3675	+ (qp->type == IB_QPT_XRC_INI) \|\| (qp->type == IB_QPT_XRC_TGT) \|\|
	3676	+ (qp->type == MLX5_IB_QPT_DCI))
	3677	+ return true;
	3678	+ return false;
	3679	+}
	3680	+
	3681	+static unsigned int get_tx_affinity(struct ib_qp *qp,
	3682	+ const struct ib_qp_attr *attr,
	3683	+ int attr_mask, u8 init,
	3684	+ struct ib_udata *udata)
	3685	+{
	3686	+ struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
	3687	+ udata, struct mlx5_ib_ucontext, ibucontext);
	3688	+ struct mlx5_ib_dev *dev = to_mdev(qp->device);
	3689	+ struct mlx5_ib_qp *mqp = to_mqp(qp);
	3690	+ struct mlx5_ib_qp_base *qp_base;
	3691	+ unsigned int tx_affinity;
	3692	+
	3693	+ if (!(mlx5_ib_lag_should_assign_affinity(dev) &&
	3694	+ qp_supports_affinity(mqp)))
	3695	+ return 0;
	3696	+
	3697	+ if (mqp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
	3698	+ tx_affinity = mqp->gsi_lag_port;
	3699	+ else if (init)
	3700	+ tx_affinity = get_tx_affinity_rr(dev, udata);
	3701	+ else if ((attr_mask & IB_QP_AV) && attr->xmit_slave)
	3702	+ tx_affinity =
	3703	+ mlx5_lag_get_slave_port(dev->mdev, attr->xmit_slave);
	3704	+ else
	3705	+ return 0;
	3706	+
	3707	+ qp_base = &mqp->trans_qp.base;
	3708	+ if (ucontext)
2950	3709	mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x ucontext %p\n",
2951		- tx_port_affinity, qp_base->mqp.qpn, ucontext);
2952		- } else {
2953		- tx_port_affinity =
2954		- (unsigned int)atomic_add_return(
2955		- 1, &dev->roce[port_num].tx_port_affinity) %
2956		- MLX5_MAX_PORTS +
2957		- 1;
	3710	+ tx_affinity, qp_base->mqp.qpn, ucontext);
	3711	+ else
2958	3712	mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x\n",
2959		- tx_port_affinity, qp_base->mqp.qpn);
2960		- }
	3713	+ tx_affinity, qp_base->mqp.qpn);
	3714	+ return tx_affinity;
	3715	+}
2961	3716
2962		- return tx_port_affinity;
	3717	+static int __mlx5_ib_qp_set_counter(struct ib_qp *qp,
	3718	+ struct rdma_counter *counter)
	3719	+{
	3720	+ struct mlx5_ib_dev *dev = to_mdev(qp->device);
	3721	+ u32 in[MLX5_ST_SZ_DW(rts2rts_qp_in)] = {};
	3722	+ struct mlx5_ib_qp *mqp = to_mqp(qp);
	3723	+ struct mlx5_ib_qp_base *base;
	3724	+ u32 set_id;
	3725	+ u32 *qpc;
	3726	+
	3727	+ if (counter)
	3728	+ set_id = counter->id;
	3729	+ else
	3730	+ set_id = mlx5_ib_get_counters_id(dev, mqp->port - 1);
	3731	+
	3732	+ base = &mqp->trans_qp.base;
	3733	+ MLX5_SET(rts2rts_qp_in, in, opcode, MLX5_CMD_OP_RTS2RTS_QP);
	3734	+ MLX5_SET(rts2rts_qp_in, in, qpn, base->mqp.qpn);
	3735	+ MLX5_SET(rts2rts_qp_in, in, uid, base->mqp.uid);
	3736	+ MLX5_SET(rts2rts_qp_in, in, opt_param_mask,
	3737	+ MLX5_QP_OPTPAR_COUNTER_SET_ID);
	3738	+
	3739	+ qpc = MLX5_ADDR_OF(rts2rts_qp_in, in, qpc);
	3740	+ MLX5_SET(qpc, qpc, counter_set_id, set_id);
	3741	+ return mlx5_cmd_exec_in(dev->mdev, rts2rts_qp, in);
2963	3742	}
2964	3743
2965	3744	static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
2966	3745	const struct ib_qp_attr *attr, int attr_mask,
2967		- enum ib_qp_state cur_state, enum ib_qp_state new_state,
2968		- const struct mlx5_ib_modify_qp *ucmd)
	3746	+ enum ib_qp_state cur_state,
	3747	+ enum ib_qp_state new_state,
	3748	+ const struct mlx5_ib_modify_qp *ucmd,
	3749	+ struct mlx5_ib_modify_qp_resp *resp,
	3750	+ struct ib_udata *udata)
2969	3751	{
2970	3752	static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
2971	3753	[MLX5_QP_STATE_RST] = {
..	..	@@ -3008,66 +3790,60 @@
3008	3790	struct mlx5_ib_qp *qp = to_mqp(ibqp);
3009	3791	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
3010	3792	struct mlx5_ib_cq send_cq, recv_cq;
3011		- struct mlx5_qp_context *context;
3012	3793	struct mlx5_ib_pd *pd;
3013		- struct mlx5_ib_port *mibport = NULL;
3014	3794	enum mlx5_qp_state mlx5_cur, mlx5_new;
3015		- enum mlx5_qp_optpar optpar;
	3795	+ void qpc, pri_path, *alt_path;
	3796	+ enum mlx5_qp_optpar optpar = 0;
	3797	+ u32 set_id = 0;
3016	3798	int mlx5_st;
3017	3799	int err;
3018	3800	u16 op;
3019	3801	u8 tx_affinity = 0;
3020	3802
3021		- mlx5_st = to_mlx5_st(ibqp->qp_type == IB_QPT_DRIVER ?
3022		- qp->qp_sub_type : ibqp->qp_type);
	3803	+ mlx5_st = to_mlx5_st(qp->type);
3023	3804	if (mlx5_st < 0)
3024	3805	return -EINVAL;
3025	3806
3026		- context = kzalloc(sizeof(*context), GFP_KERNEL);
3027		- if (!context)
	3807	+ qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);
	3808	+ if (!qpc)
3028	3809	return -ENOMEM;
3029	3810
3030		- pd = get_pd(qp);
3031		- context->flags = cpu_to_be32(mlx5_st << 16);
	3811	+ pd = to_mpd(qp->ibqp.pd);
	3812	+ MLX5_SET(qpc, qpc, st, mlx5_st);
3032	3813
3033	3814	if (!(attr_mask & IB_QP_PATH_MIG_STATE)) {
3034		- context->flags \|= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11);
	3815	+ MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
3035	3816	} else {
3036	3817	switch (attr->path_mig_state) {
3037	3818	case IB_MIG_MIGRATED:
3038		- context->flags \|= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11);
	3819	+ MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
3039	3820	break;
3040	3821	case IB_MIG_REARM:
3041		- context->flags \|= cpu_to_be32(MLX5_QP_PM_REARM << 11);
	3822	+ MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_REARM);
3042	3823	break;
3043	3824	case IB_MIG_ARMED:
3044		- context->flags \|= cpu_to_be32(MLX5_QP_PM_ARMED << 11);
	3825	+ MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_ARMED);
3045	3826	break;
3046	3827	}
3047	3828	}
3048	3829
3049		- if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) {
3050		- if ((ibqp->qp_type == IB_QPT_RC) \|\|
3051		- (ibqp->qp_type == IB_QPT_UD &&
3052		- !(qp->flags & MLX5_IB_QP_SQPN_QP1)) \|\|
3053		- (ibqp->qp_type == IB_QPT_UC) \|\|
3054		- (ibqp->qp_type == IB_QPT_RAW_PACKET) \|\|
3055		- (ibqp->qp_type == IB_QPT_XRC_INI) \|\|
3056		- (ibqp->qp_type == IB_QPT_XRC_TGT)) {
3057		- if (mlx5_lag_is_active(dev->mdev)) {
3058		- u8 p = mlx5_core_native_port_num(dev->mdev);
3059		- tx_affinity = get_tx_affinity(dev, pd, base, p);
3060		- context->flags \|= cpu_to_be32(tx_affinity << 24);
3061		- }
3062		- }
3063		- }
	3830	+ tx_affinity = get_tx_affinity(ibqp, attr, attr_mask,
	3831	+ cur_state == IB_QPS_RESET &&
	3832	+ new_state == IB_QPS_INIT, udata);
	3833	+
	3834	+ MLX5_SET(qpc, qpc, lag_tx_port_affinity, tx_affinity);
	3835	+ if (tx_affinity && new_state == IB_QPS_RTR &&
	3836	+ MLX5_CAP_GEN(dev->mdev, init2_lag_tx_port_affinity))
	3837	+ optpar \|= MLX5_QP_OPTPAR_LAG_TX_AFF;
3064	3838
3065	3839	if (is_sqp(ibqp->qp_type)) {
3066		- context->mtu_msgmax = (IB_MTU_256 << 5) \| 8;
	3840	+ MLX5_SET(qpc, qpc, mtu, IB_MTU_256);
	3841	+ MLX5_SET(qpc, qpc, log_msg_max, 8);
3067	3842	} else if ((ibqp->qp_type == IB_QPT_UD &&
3068		- !(qp->flags & MLX5_IB_QP_UNDERLAY)) \|\|
	3843	+ !(qp->flags & IB_QP_CREATE_SOURCE_QPN)) \|\|
3069	3844	ibqp->qp_type == MLX5_IB_QPT_REG_UMR) {
3070		- context->mtu_msgmax = (IB_MTU_4096 << 5) \| 12;
	3845	+ MLX5_SET(qpc, qpc, mtu, IB_MTU_4096);
	3846	+ MLX5_SET(qpc, qpc, log_msg_max, 12);
3071	3847	} else if (attr_mask & IB_QP_PATH_MTU) {
3072	3848	if (attr->path_mtu < IB_MTU_256 \|\|
3073	3849	attr->path_mtu > IB_MTU_4096) {
..	..	@@ -3075,40 +3851,45 @@
3075	3851	err = -EINVAL;
3076	3852	goto out;
3077	3853	}
3078		- context->mtu_msgmax = (attr->path_mtu << 5) \|
3079		- (u8)MLX5_CAP_GEN(dev->mdev, log_max_msg);
	3854	+ MLX5_SET(qpc, qpc, mtu, attr->path_mtu);
	3855	+ MLX5_SET(qpc, qpc, log_msg_max,
	3856	+ MLX5_CAP_GEN(dev->mdev, log_max_msg));
3080	3857	}
3081	3858
3082	3859	if (attr_mask & IB_QP_DEST_QPN)
3083		- context->log_pg_sz_remote_qpn = cpu_to_be32(attr->dest_qp_num);
	3860	+ MLX5_SET(qpc, qpc, remote_qpn, attr->dest_qp_num);
	3861	+
	3862	+ pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
	3863	+ alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);
3084	3864
3085	3865	if (attr_mask & IB_QP_PKEY_INDEX)
3086		- context->pri_path.pkey_index = cpu_to_be16(attr->pkey_index);
	3866	+ MLX5_SET(ads, pri_path, pkey_index, attr->pkey_index);
3087	3867
3088	3868	/* todo implement counter_index functionality */
3089	3869
3090	3870	if (is_sqp(ibqp->qp_type))
3091		- context->pri_path.port = qp->port;
	3871	+ MLX5_SET(ads, pri_path, vhca_port_num, qp->port);
3092	3872
3093	3873	if (attr_mask & IB_QP_PORT)
3094		- context->pri_path.port = attr->port_num;
	3874	+ MLX5_SET(ads, pri_path, vhca_port_num, attr->port_num);
3095	3875
3096	3876	if (attr_mask & IB_QP_AV) {
3097		- err = mlx5_set_path(dev, qp, &attr->ah_attr, &context->pri_path,
3098		- attr_mask & IB_QP_PORT ? attr->port_num : qp->port,
	3877	+ err = mlx5_set_path(dev, qp, &attr->ah_attr, pri_path,
	3878	+ attr_mask & IB_QP_PORT ? attr->port_num :
	3879	+ qp->port,
3099	3880	attr_mask, 0, attr, false);
3100	3881	if (err)
3101	3882	goto out;
3102	3883	}
3103	3884
3104	3885	if (attr_mask & IB_QP_TIMEOUT)
3105		- context->pri_path.ackto_lt \|= attr->timeout << 3;
	3886	+ MLX5_SET(ads, pri_path, ack_timeout, attr->timeout);
3106	3887
3107	3888	if (attr_mask & IB_QP_ALT_PATH) {
3108		- err = mlx5_set_path(dev, qp, &attr->alt_ah_attr,
3109		- &context->alt_path,
	3889	+ err = mlx5_set_path(dev, qp, &attr->alt_ah_attr, alt_path,
3110	3890	attr->alt_port_num,
3111		- attr_mask \| IB_QP_PKEY_INDEX \| IB_QP_TIMEOUT,
	3891	+ attr_mask \| IB_QP_PKEY_INDEX \|
	3892	+ IB_QP_TIMEOUT,
3112	3893	0, attr, true);
3113	3894	if (err)
3114	3895	goto out;
..	..	@@ -3117,65 +3898,68 @@
3117	3898	get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
3118	3899	&send_cq, &recv_cq);
3119	3900
3120		- context->flags_pd = cpu_to_be32(pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn);
3121		- context->cqn_send = send_cq ? cpu_to_be32(send_cq->mcq.cqn) : 0;
3122		- context->cqn_recv = recv_cq ? cpu_to_be32(recv_cq->mcq.cqn) : 0;
3123		- context->params1 = cpu_to_be32(MLX5_IB_ACK_REQ_FREQ << 28);
	3901	+ MLX5_SET(qpc, qpc, pd, pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn);
	3902	+ if (send_cq)
	3903	+ MLX5_SET(qpc, qpc, cqn_snd, send_cq->mcq.cqn);
	3904	+ if (recv_cq)
	3905	+ MLX5_SET(qpc, qpc, cqn_rcv, recv_cq->mcq.cqn);
	3906	+
	3907	+ MLX5_SET(qpc, qpc, log_ack_req_freq, MLX5_IB_ACK_REQ_FREQ);
3124	3908
3125	3909	if (attr_mask & IB_QP_RNR_RETRY)
3126		- context->params1 \|= cpu_to_be32(attr->rnr_retry << 13);
	3910	+ MLX5_SET(qpc, qpc, rnr_retry, attr->rnr_retry);
3127	3911
3128	3912	if (attr_mask & IB_QP_RETRY_CNT)
3129		- context->params1 \|= cpu_to_be32(attr->retry_cnt << 16);
	3913	+ MLX5_SET(qpc, qpc, retry_count, attr->retry_cnt);
3130	3914
3131		- if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
3132		- if (attr->max_rd_atomic)
3133		- context->params1 \|=
3134		- cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
3135		- }
	3915	+ if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && attr->max_rd_atomic)
	3916	+ MLX5_SET(qpc, qpc, log_sra_max, ilog2(attr->max_rd_atomic));
3136	3917
3137	3918	if (attr_mask & IB_QP_SQ_PSN)
3138		- context->next_send_psn = cpu_to_be32(attr->sq_psn);
	3919	+ MLX5_SET(qpc, qpc, next_send_psn, attr->sq_psn);
3139	3920
3140		- if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
3141		- if (attr->max_dest_rd_atomic)
3142		- context->params2 \|=
3143		- cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
	3921	+ if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && attr->max_dest_rd_atomic)
	3922	+ MLX5_SET(qpc, qpc, log_rra_max,
	3923	+ ilog2(attr->max_dest_rd_atomic));
	3924	+
	3925	+ if (attr_mask & (IB_QP_ACCESS_FLAGS \| IB_QP_MAX_DEST_RD_ATOMIC)) {
	3926	+ err = set_qpc_atomic_flags(qp, attr, attr_mask, qpc);
	3927	+ if (err)
	3928	+ goto out;
3144	3929	}
3145	3930
3146		- if (attr_mask & (IB_QP_ACCESS_FLAGS \| IB_QP_MAX_DEST_RD_ATOMIC))
3147		- context->params2 \|= to_mlx5_access_flags(qp, attr, attr_mask);
3148		-
3149	3931	if (attr_mask & IB_QP_MIN_RNR_TIMER)
3150		- context->rnr_nextrecvpsn \|= cpu_to_be32(attr->min_rnr_timer << 24);
	3932	+ MLX5_SET(qpc, qpc, min_rnr_nak, attr->min_rnr_timer);
3151	3933
3152	3934	if (attr_mask & IB_QP_RQ_PSN)
3153		- context->rnr_nextrecvpsn \|= cpu_to_be32(attr->rq_psn);
	3935	+ MLX5_SET(qpc, qpc, next_rcv_psn, attr->rq_psn);
3154	3936
3155	3937	if (attr_mask & IB_QP_QKEY)
3156		- context->qkey = cpu_to_be32(attr->qkey);
	3938	+ MLX5_SET(qpc, qpc, q_key, attr->qkey);
3157	3939
3158	3940	if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
3159		- context->db_rec_addr = cpu_to_be64(qp->db.dma);
	3941	+ MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
3160	3942
3161	3943	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3162	3944	u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
3163	3945	qp->port) - 1;
3164	3946
3165	3947	/* Underlay port should be used - index 0 function per port */
3166		- if (qp->flags & MLX5_IB_QP_UNDERLAY)
	3948	+ if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
3167	3949	port_num = 0;
3168	3950
3169		- mibport = &dev->port[port_num];
3170		- context->qp_counter_set_usr_page \|=
3171		- cpu_to_be32((u32)(mibport->cnts.set_id) << 24);
	3951	+ if (ibqp->counter)
	3952	+ set_id = ibqp->counter->id;
	3953	+ else
	3954	+ set_id = mlx5_ib_get_counters_id(dev, port_num);
	3955	+ MLX5_SET(qpc, qpc, counter_set_id, set_id);
3172	3956	}
3173	3957
3174	3958	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
3175		- context->sq_crq_size \|= cpu_to_be16(1 << 4);
	3959	+ MLX5_SET(qpc, qpc, rlky, 1);
3176	3960
3177		- if (qp->flags & MLX5_IB_QP_SQPN_QP1)
3178		- context->deth_sqpn = cpu_to_be32(1);
	3961	+ if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
	3962	+ MLX5_SET(qpc, qpc, deth_sqpn, 1);
3179	3963
3180	3964	mlx5_cur = to_mlx5_state(cur_state);
3181	3965	mlx5_new = to_mlx5_state(new_state);
..	..	@@ -3187,18 +3971,21 @@
3187	3971	}
3188	3972
3189	3973	op = optab[mlx5_cur][mlx5_new];
3190		- optpar = ib_mask_to_mlx5_opt(attr_mask);
	3974	+ optpar \|= ib_mask_to_mlx5_opt(attr_mask);
3191	3975	optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
3192	3976
3193	3977	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET \|\|
3194		- qp->flags & MLX5_IB_QP_UNDERLAY) {
	3978	+ qp->flags & IB_QP_CREATE_SOURCE_QPN) {
3195	3979	struct mlx5_modify_raw_qp_param raw_qp_param = {};
3196	3980
3197	3981	raw_qp_param.operation = op;
3198	3982	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3199		- raw_qp_param.rq_q_ctr_id = mibport->cnts.set_id;
	3983	+ raw_qp_param.rq_q_ctr_id = set_id;
3200	3984	raw_qp_param.set_mask \|= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
3201	3985	}
	3986	+
	3987	+ if (attr_mask & IB_QP_PORT)
	3988	+ raw_qp_param.port = attr->port_num;
3202	3989
3203	3990	if (attr_mask & IB_QP_RATE_LIMIT) {
3204	3991	raw_qp_param.rl.rate = attr->rate_limit;
..	..	@@ -3230,8 +4017,15 @@
3230	4017
3231	4018	err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity);
3232	4019	} else {
3233		- err = mlx5_core_qp_modify(dev->mdev, op, optpar, context,
3234		- &base->mqp);
	4020	+ if (udata) {
	4021	+ /* For the kernel flows, the resp will stay zero */
	4022	+ resp->ece_options =
	4023	+ MLX5_CAP_GEN(dev->mdev, ece_support) ?
	4024	+ ucmd->ece_options : 0;
	4025	+ resp->response_length = sizeof(*resp);
	4026	+ }
	4027	+ err = mlx5_core_qp_modify(dev, op, optpar, qpc, &base->mqp,
	4028	+ &resp->ece_options);
3235	4029	}
3236	4030
3237	4031	if (err)
..	..	@@ -3264,13 +4058,21 @@
3264	4058	qp->sq.head = 0;
3265	4059	qp->sq.tail = 0;
3266	4060	qp->sq.cur_post = 0;
	4061	+ if (qp->sq.wqe_cnt)
	4062	+ qp->sq.cur_edge = get_sq_edge(&qp->sq, 0);
3267	4063	qp->sq.last_poll = 0;
3268	4064	qp->db.db[MLX5_RCV_DBR] = 0;
3269	4065	qp->db.db[MLX5_SND_DBR] = 0;
3270	4066	}
3271	4067
	4068	+ if ((new_state == IB_QPS_RTS) && qp->counter_pending) {
	4069	+ err = __mlx5_ib_qp_set_counter(ibqp, ibqp->counter);
	4070	+ if (!err)
	4071	+ qp->counter_pending = 0;
	4072	+ }
	4073	+
3272	4074	out:
3273		- kfree(context);
	4075	+ kfree(qpc);
3274	4076	return err;
3275	4077	}
3276	4078
..	..	@@ -3304,7 +4106,7 @@
3304	4106	return is_valid_mask(attr_mask, req, opt);
3305	4107	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
3306	4108	req \|= IB_QP_PATH_MTU;
3307		- opt = IB_QP_PKEY_INDEX;
	4109	+ opt = IB_QP_PKEY_INDEX \| IB_QP_AV;
3308	4110	return is_valid_mask(attr_mask, req, opt);
3309	4111	} else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
3310	4112	req \|= IB_QP_TIMEOUT \| IB_QP_RETRY_CNT \| IB_QP_RNR_RETRY \|
..	..	@@ -3328,14 +4130,15 @@
3328	4130	* Other transitions and attributes are illegal
3329	4131	*/
3330	4132	static int mlx5_ib_modify_dct(struct ib_qp ibqp, struct ib_qp_attr attr,
3331		- int attr_mask, struct ib_udata *udata)
	4133	+ int attr_mask, struct mlx5_ib_modify_qp *ucmd,
	4134	+ struct ib_udata *udata)
3332	4135	{
3333	4136	struct mlx5_ib_qp *qp = to_mqp(ibqp);
3334	4137	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
3335	4138	enum ib_qp_state cur_state, new_state;
3336		- int err = 0;
3337	4139	int required = IB_QP_STATE;
3338	4140	void *dctc;
	4141	+ int err;
3339	4142
3340	4143	if (!(attr_mask & IB_QP_STATE))
3341	4144	return -EINVAL;
..	..	@@ -3344,13 +4147,24 @@
3344	4147	new_state = attr->qp_state;
3345	4148
3346	4149	dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
	4150	+ if (MLX5_CAP_GEN(dev->mdev, ece_support) && ucmd->ece_options)
	4151	+ /*
	4152	+ * DCT doesn't initialize QP till modify command is executed,
	4153	+ * so we need to overwrite previously set ECE field if user
	4154	+ * provided any value except zero, which means not set/not
	4155	+ * valid.
	4156	+ */
	4157	+ MLX5_SET(dctc, dctc, ece, ucmd->ece_options);
	4158	+
3347	4159	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
	4160	+ u16 set_id;
	4161	+
3348	4162	required \|= IB_QP_ACCESS_FLAGS \| IB_QP_PKEY_INDEX \| IB_QP_PORT;
3349	4163	if (!is_valid_mask(attr_mask, required, 0))
3350	4164	return -EINVAL;
3351	4165
3352	4166	if (attr->port_num == 0 \|\|
3353		- attr->port_num > MLX5_CAP_GEN(dev->mdev, num_ports)) {
	4167	+ attr->port_num > dev->num_ports) {
3354	4168	mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
3355	4169	attr->port_num, dev->num_ports);
3356	4170	return -EINVAL;
..	..	@@ -3360,23 +4174,38 @@
3360	4174	if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
3361	4175	MLX5_SET(dctc, dctc, rwe, 1);
3362	4176	if (attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) {
3363		- if (!mlx5_ib_dc_atomic_is_supported(dev))
	4177	+ int atomic_mode;
	4178	+
	4179	+ atomic_mode = get_atomic_mode(dev, MLX5_IB_QPT_DCT);
	4180	+ if (atomic_mode < 0)
3364	4181	return -EOPNOTSUPP;
	4182	+
	4183	+ MLX5_SET(dctc, dctc, atomic_mode, atomic_mode);
3365	4184	MLX5_SET(dctc, dctc, rae, 1);
3366		- MLX5_SET(dctc, dctc, atomic_mode, MLX5_ATOMIC_MODE_DCT_CX);
3367	4185	}
3368	4186	MLX5_SET(dctc, dctc, pkey_index, attr->pkey_index);
3369		- MLX5_SET(dctc, dctc, port, attr->port_num);
3370		- MLX5_SET(dctc, dctc, counter_set_id, dev->port[attr->port_num - 1].cnts.set_id);
	4187	+ if (mlx5_lag_is_active(dev->mdev))
	4188	+ MLX5_SET(dctc, dctc, port,
	4189	+ get_tx_affinity_rr(dev, udata));
	4190	+ else
	4191	+ MLX5_SET(dctc, dctc, port, attr->port_num);
3371	4192
	4193	+ set_id = mlx5_ib_get_counters_id(dev, attr->port_num - 1);
	4194	+ MLX5_SET(dctc, dctc, counter_set_id, set_id);
3372	4195	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
3373	4196	struct mlx5_ib_modify_qp_resp resp = {};
3374		- u32 min_resp_len = offsetof(typeof(resp), dctn) +
3375		- sizeof(resp.dctn);
	4197	+ u32 out[MLX5_ST_SZ_DW(create_dct_out)] = {};
	4198	+ u32 min_resp_len = offsetofend(typeof(resp), dctn);
3376	4199
3377	4200	if (udata->outlen < min_resp_len)
3378	4201	return -EINVAL;
3379		- resp.response_length = min_resp_len;
	4202	+ /*
	4203	+ * If we don't have enough space for the ECE options,
	4204	+ * simply indicate it with resp.response_length.
	4205	+ */
	4206	+ resp.response_length = (udata->outlen < sizeof(resp)) ?
	4207	+ min_resp_len :
	4208	+ sizeof(resp);
3380	4209
3381	4210	required \|= IB_QP_MIN_RNR_TIMER \| IB_QP_AV \| IB_QP_PATH_MTU;
3382	4211	if (!is_valid_mask(attr_mask, required, 0))
..	..	@@ -3390,47 +4219,99 @@
3390	4219	if (attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
3391	4220	MLX5_SET(dctc, dctc, eth_prio, attr->ah_attr.sl & 0x7);
3392	4221
3393		- err = mlx5_core_create_dct(dev->mdev, &qp->dct.mdct, qp->dct.in,
3394		- MLX5_ST_SZ_BYTES(create_dct_in));
	4222	+ err = mlx5_core_create_dct(dev, &qp->dct.mdct, qp->dct.in,
	4223	+ MLX5_ST_SZ_BYTES(create_dct_in), out,
	4224	+ sizeof(out));
3395	4225	if (err)
3396	4226	return err;
3397	4227	resp.dctn = qp->dct.mdct.mqp.qpn;
	4228	+ if (MLX5_CAP_GEN(dev->mdev, ece_support))
	4229	+ resp.ece_options = MLX5_GET(create_dct_out, out, ece);
3398	4230	err = ib_copy_to_udata(udata, &resp, resp.response_length);
3399	4231	if (err) {
3400		- mlx5_core_destroy_dct(dev->mdev, &qp->dct.mdct);
	4232	+ mlx5_core_destroy_dct(dev, &qp->dct.mdct);
3401	4233	return err;
3402	4234	}
3403	4235	} else {
3404	4236	mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state);
3405	4237	return -EINVAL;
3406	4238	}
3407		- if (err)
3408		- qp->state = IB_QPS_ERR;
3409		- else
3410		- qp->state = new_state;
3411		- return err;
	4239	+
	4240	+ qp->state = new_state;
	4241	+ return 0;
	4242	+}
	4243	+
	4244	+static bool mlx5_ib_modify_qp_allowed(struct mlx5_ib_dev *dev,
	4245	+ struct mlx5_ib_qp *qp,
	4246	+ enum ib_qp_type qp_type)
	4247	+{
	4248	+ if (dev->profile != &raw_eth_profile)
	4249	+ return true;
	4250	+
	4251	+ if (qp_type == IB_QPT_RAW_PACKET \|\| qp_type == MLX5_IB_QPT_REG_UMR)
	4252	+ return true;
	4253	+
	4254	+ /* Internal QP used for wc testing, with NOPs in wq */
	4255	+ if (qp->flags & MLX5_IB_QP_CREATE_WC_TEST)
	4256	+ return true;
	4257	+
	4258	+ return false;
	4259	+}
	4260	+
	4261	+static int validate_rd_atomic(struct mlx5_ib_dev dev, struct ib_qp_attr attr,
	4262	+ int attr_mask, enum ib_qp_type qp_type)
	4263	+{
	4264	+ int log_max_ra_res;
	4265	+ int log_max_ra_req;
	4266	+
	4267	+ if (qp_type == MLX5_IB_QPT_DCI) {
	4268	+ log_max_ra_res = 1 << MLX5_CAP_GEN(dev->mdev,
	4269	+ log_max_ra_res_dc);
	4270	+ log_max_ra_req = 1 << MLX5_CAP_GEN(dev->mdev,
	4271	+ log_max_ra_req_dc);
	4272	+ } else {
	4273	+ log_max_ra_res = 1 << MLX5_CAP_GEN(dev->mdev,
	4274	+ log_max_ra_res_qp);
	4275	+ log_max_ra_req = 1 << MLX5_CAP_GEN(dev->mdev,
	4276	+ log_max_ra_req_qp);
	4277	+ }
	4278	+
	4279	+ if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
	4280	+ attr->max_rd_atomic > log_max_ra_res) {
	4281	+ mlx5_ib_dbg(dev, "invalid max_rd_atomic value %d\n",
	4282	+ attr->max_rd_atomic);
	4283	+ return false;
	4284	+ }
	4285	+
	4286	+ if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
	4287	+ attr->max_dest_rd_atomic > log_max_ra_req) {
	4288	+ mlx5_ib_dbg(dev, "invalid max_dest_rd_atomic value %d\n",
	4289	+ attr->max_dest_rd_atomic);
	4290	+ return false;
	4291	+ }
	4292	+ return true;
3412	4293	}
3413	4294
3414	4295	int mlx5_ib_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
3415	4296	int attr_mask, struct ib_udata *udata)
3416	4297	{
3417	4298	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
	4299	+ struct mlx5_ib_modify_qp_resp resp = {};
3418	4300	struct mlx5_ib_qp *qp = to_mqp(ibqp);
3419	4301	struct mlx5_ib_modify_qp ucmd = {};
3420	4302	enum ib_qp_type qp_type;
3421	4303	enum ib_qp_state cur_state, new_state;
3422		- size_t required_cmd_sz;
3423	4304	int err = -EINVAL;
3424	4305	int port;
3425		- enum rdma_link_layer ll = IB_LINK_LAYER_UNSPECIFIED;
	4306	+
	4307	+ if (!mlx5_ib_modify_qp_allowed(dev, qp, ibqp->qp_type))
	4308	+ return -EOPNOTSUPP;
3426	4309
3427	4310	if (ibqp->rwq_ind_tbl)
3428	4311	return -ENOSYS;
3429	4312
3430	4313	if (udata && udata->inlen) {
3431		- required_cmd_sz = offsetof(typeof(ucmd), reserved) +
3432		- sizeof(ucmd.reserved);
3433		- if (udata->inlen < required_cmd_sz)
	4314	+ if (udata->inlen < offsetofend(typeof(ucmd), ece_options))
3434	4315	return -EINVAL;
3435	4316
3436	4317	if (udata->inlen > sizeof(ucmd) &&
..	..	@@ -3443,23 +4324,20 @@
3443	4324	return -EFAULT;
3444	4325
3445	4326	if (ucmd.comp_mask \|\|
3446		- memchr_inv(&ucmd.reserved, 0, sizeof(ucmd.reserved)) \|\|
3447	4327	memchr_inv(&ucmd.burst_info.reserved, 0,
3448	4328	sizeof(ucmd.burst_info.reserved)))
3449	4329	return -EOPNOTSUPP;
	4330	+
3450	4331	}
3451	4332
3452	4333	if (unlikely(ibqp->qp_type == IB_QPT_GSI))
3453	4334	return mlx5_ib_gsi_modify_qp(ibqp, attr, attr_mask);
3454	4335
3455		- if (ibqp->qp_type == IB_QPT_DRIVER)
3456		- qp_type = qp->qp_sub_type;
3457		- else
3458		- qp_type = (unlikely(ibqp->qp_type == MLX5_IB_QPT_HW_GSI)) ?
3459		- IB_QPT_GSI : ibqp->qp_type;
	4336	+ qp_type = (unlikely(ibqp->qp_type == MLX5_IB_QPT_HW_GSI)) ? IB_QPT_GSI :
	4337	+ qp->type;
3460	4338
3461	4339	if (qp_type == MLX5_IB_QPT_DCT)
3462		- return mlx5_ib_modify_dct(ibqp, attr, attr_mask, udata);
	4340	+ return mlx5_ib_modify_dct(ibqp, attr, attr_mask, &ucmd, udata);
3463	4341
3464	4342	mutex_lock(&qp->mutex);
3465	4343
..	..	@@ -3468,10 +4346,9 @@
3468	4346
3469	4347	if (!(cur_state == new_state && cur_state == IB_QPS_RESET)) {
3470	4348	port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
3471		- ll = dev->ib_dev.get_link_layer(&dev->ib_dev, port);
3472	4349	}
3473	4350
3474		- if (qp->flags & MLX5_IB_QP_UNDERLAY) {
	4351	+ if (qp->flags & IB_QP_CREATE_SOURCE_QPN) {
3475	4352	if (attr_mask & ~(IB_QP_STATE \| IB_QP_CUR_STATE)) {
3476	4353	mlx5_ib_dbg(dev, "invalid attr_mask 0x%x when underlay QP is used\n",
3477	4354	attr_mask);
..	..	@@ -3479,7 +4356,8 @@
3479	4356	}
3480	4357	} else if (qp_type != MLX5_IB_QPT_REG_UMR &&
3481	4358	qp_type != MLX5_IB_QPT_DCI &&
3482		- !ib_modify_qp_is_ok(cur_state, new_state, qp_type, attr_mask, ll)) {
	4359	+ !ib_modify_qp_is_ok(cur_state, new_state, qp_type,
	4360	+ attr_mask)) {
3483	4361	mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
3484	4362	cur_state, new_state, ibqp->qp_type, attr_mask);
3485	4363	goto out;
..	..	@@ -3508,21 +4386,8 @@
3508	4386	}
3509	4387	}
3510	4388
3511		- if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
3512		- attr->max_rd_atomic >
3513		- (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_res_qp))) {
3514		- mlx5_ib_dbg(dev, "invalid max_rd_atomic value %d\n",
3515		- attr->max_rd_atomic);
	4389	+ if (!validate_rd_atomic(dev, attr, attr_mask, qp_type))
3516	4390	goto out;
3517		- }
3518		-
3519		- if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
3520		- attr->max_dest_rd_atomic >
3521		- (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_req_qp))) {
3522		- mlx5_ib_dbg(dev, "invalid max_dest_rd_atomic value %d\n",
3523		- attr->max_dest_rd_atomic);
3524		- goto out;
3525		- }
3526	4391
3527	4392	if (cur_state == new_state && cur_state == IB_QPS_RESET) {
3528	4393	err = 0;
..	..	@@ -3530,1305 +4395,17 @@
3530	4395	}
3531	4396
3532	4397	err = __mlx5_ib_modify_qp(ibqp, attr, attr_mask, cur_state,
3533		- new_state, &ucmd);
	4398	+ new_state, &ucmd, &resp, udata);
	4399	+
	4400	+ /* resp.response_length is set in ECE supported flows only */
	4401	+ if (!err && resp.response_length &&
	4402	+ udata->outlen >= resp.response_length)
	4403	+ /* Return -EFAULT to the user and expect him to destroy QP. */
	4404	+ err = ib_copy_to_udata(udata, &resp, resp.response_length);
3534	4405
3535	4406	out:
3536	4407	mutex_unlock(&qp->mutex);
3537	4408	return err;
3538		-}
3539		-
3540		-static int mlx5_wq_overflow(struct mlx5_ib_wq wq, int nreq, struct ib_cq ib_cq)
3541		-{
3542		- struct mlx5_ib_cq *cq;
3543		- unsigned cur;
3544		-
3545		- cur = wq->head - wq->tail;
3546		- if (likely(cur + nreq < wq->max_post))
3547		- return 0;
3548		-
3549		- cq = to_mcq(ib_cq);
3550		- spin_lock(&cq->lock);
3551		- cur = wq->head - wq->tail;
3552		- spin_unlock(&cq->lock);
3553		-
3554		- return cur + nreq >= wq->max_post;
3555		-}
3556		-
3557		-static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
3558		- u64 remote_addr, u32 rkey)
3559		-{
3560		- rseg->raddr = cpu_to_be64(remote_addr);
3561		- rseg->rkey = cpu_to_be32(rkey);
3562		- rseg->reserved = 0;
3563		-}
3564		-
3565		-static void set_eth_seg(struct mlx5_wqe_eth_seg eseg,
3566		- const struct ib_send_wr wr, void qend,
3567		- struct mlx5_ib_qp qp, int size)
3568		-{
3569		- void *seg = eseg;
3570		-
3571		- memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
3572		-
3573		- if (wr->send_flags & IB_SEND_IP_CSUM)
3574		- eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM \|
3575		- MLX5_ETH_WQE_L4_CSUM;
3576		-
3577		- seg += sizeof(struct mlx5_wqe_eth_seg);
3578		- *size += sizeof(struct mlx5_wqe_eth_seg) / 16;
3579		-
3580		- if (wr->opcode == IB_WR_LSO) {
3581		- struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
3582		- int size_of_inl_hdr_start = sizeof(eseg->inline_hdr.start);
3583		- u64 left, leftlen, copysz;
3584		- void *pdata = ud_wr->header;
3585		-
3586		- left = ud_wr->hlen;
3587		- eseg->mss = cpu_to_be16(ud_wr->mss);
3588		- eseg->inline_hdr.sz = cpu_to_be16(left);
3589		-
3590		- /*
3591		- * check if there is space till the end of queue, if yes,
3592		- * copy all in one shot, otherwise copy till the end of queue,
3593		- * rollback and than the copy the left
3594		- */
3595		- leftlen = qend - (void *)eseg->inline_hdr.start;
3596		- copysz = min_t(u64, leftlen, left);
3597		-
3598		- memcpy(seg - size_of_inl_hdr_start, pdata, copysz);
3599		-
3600		- if (likely(copysz > size_of_inl_hdr_start)) {
3601		- seg += ALIGN(copysz - size_of_inl_hdr_start, 16);
3602		- *size += ALIGN(copysz - size_of_inl_hdr_start, 16) / 16;
3603		- }
3604		-
3605		- if (unlikely(copysz < left)) { /* the last wqe in the queue */
3606		- seg = mlx5_get_send_wqe(qp, 0);
3607		- left -= copysz;
3608		- pdata += copysz;
3609		- memcpy(seg, pdata, left);
3610		- seg += ALIGN(left, 16);
3611		- *size += ALIGN(left, 16) / 16;
3612		- }
3613		- }
3614		-
3615		- return seg;
3616		-}
3617		-
3618		-static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
3619		- const struct ib_send_wr *wr)
3620		-{
3621		- memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
3622		- dseg->av.dqp_dct = cpu_to_be32(ud_wr(wr)->remote_qpn \| MLX5_EXTENDED_UD_AV);
3623		- dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
3624		-}
3625		-
3626		-static void set_data_ptr_seg(struct mlx5_wqe_data_seg dseg, struct ib_sge sg)
3627		-{
3628		- dseg->byte_count = cpu_to_be32(sg->length);
3629		- dseg->lkey = cpu_to_be32(sg->lkey);
3630		- dseg->addr = cpu_to_be64(sg->addr);
3631		-}
3632		-
3633		-static u64 get_xlt_octo(u64 bytes)
3634		-{
3635		- return ALIGN(bytes, MLX5_IB_UMR_XLT_ALIGNMENT) /
3636		- MLX5_IB_UMR_OCTOWORD;
3637		-}
3638		-
3639		-static __be64 frwr_mkey_mask(void)
3640		-{
3641		- u64 result;
3642		-
3643		- result = MLX5_MKEY_MASK_LEN \|
3644		- MLX5_MKEY_MASK_PAGE_SIZE \|
3645		- MLX5_MKEY_MASK_START_ADDR \|
3646		- MLX5_MKEY_MASK_EN_RINVAL \|
3647		- MLX5_MKEY_MASK_KEY \|
3648		- MLX5_MKEY_MASK_LR \|
3649		- MLX5_MKEY_MASK_LW \|
3650		- MLX5_MKEY_MASK_RR \|
3651		- MLX5_MKEY_MASK_RW \|
3652		- MLX5_MKEY_MASK_A \|
3653		- MLX5_MKEY_MASK_SMALL_FENCE \|
3654		- MLX5_MKEY_MASK_FREE;
3655		-
3656		- return cpu_to_be64(result);
3657		-}
3658		-
3659		-static __be64 sig_mkey_mask(void)
3660		-{
3661		- u64 result;
3662		-
3663		- result = MLX5_MKEY_MASK_LEN \|
3664		- MLX5_MKEY_MASK_PAGE_SIZE \|
3665		- MLX5_MKEY_MASK_START_ADDR \|
3666		- MLX5_MKEY_MASK_EN_SIGERR \|
3667		- MLX5_MKEY_MASK_EN_RINVAL \|
3668		- MLX5_MKEY_MASK_KEY \|
3669		- MLX5_MKEY_MASK_LR \|
3670		- MLX5_MKEY_MASK_LW \|
3671		- MLX5_MKEY_MASK_RR \|
3672		- MLX5_MKEY_MASK_RW \|
3673		- MLX5_MKEY_MASK_SMALL_FENCE \|
3674		- MLX5_MKEY_MASK_FREE \|
3675		- MLX5_MKEY_MASK_BSF_EN;
3676		-
3677		- return cpu_to_be64(result);
3678		-}
3679		-
3680		-static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
3681		- struct mlx5_ib_mr *mr, bool umr_inline)
3682		-{
3683		- int size = mr->ndescs * mr->desc_size;
3684		-
3685		- memset(umr, 0, sizeof(*umr));
3686		-
3687		- umr->flags = MLX5_UMR_CHECK_NOT_FREE;
3688		- if (umr_inline)
3689		- umr->flags \|= MLX5_UMR_INLINE;
3690		- umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
3691		- umr->mkey_mask = frwr_mkey_mask();
3692		-}
3693		-
3694		-static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
3695		-{
3696		- memset(umr, 0, sizeof(*umr));
3697		- umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
3698		- umr->flags = MLX5_UMR_INLINE;
3699		-}
3700		-
3701		-static __be64 get_umr_enable_mr_mask(void)
3702		-{
3703		- u64 result;
3704		-
3705		- result = MLX5_MKEY_MASK_KEY \|
3706		- MLX5_MKEY_MASK_FREE;
3707		-
3708		- return cpu_to_be64(result);
3709		-}
3710		-
3711		-static __be64 get_umr_disable_mr_mask(void)
3712		-{
3713		- u64 result;
3714		-
3715		- result = MLX5_MKEY_MASK_FREE;
3716		-
3717		- return cpu_to_be64(result);
3718		-}
3719		-
3720		-static __be64 get_umr_update_translation_mask(void)
3721		-{
3722		- u64 result;
3723		-
3724		- result = MLX5_MKEY_MASK_LEN \|
3725		- MLX5_MKEY_MASK_PAGE_SIZE \|
3726		- MLX5_MKEY_MASK_START_ADDR;
3727		-
3728		- return cpu_to_be64(result);
3729		-}
3730		-
3731		-static __be64 get_umr_update_access_mask(int atomic)
3732		-{
3733		- u64 result;
3734		-
3735		- result = MLX5_MKEY_MASK_LR \|
3736		- MLX5_MKEY_MASK_LW \|
3737		- MLX5_MKEY_MASK_RR \|
3738		- MLX5_MKEY_MASK_RW;
3739		-
3740		- if (atomic)
3741		- result \|= MLX5_MKEY_MASK_A;
3742		-
3743		- return cpu_to_be64(result);
3744		-}
3745		-
3746		-static __be64 get_umr_update_pd_mask(void)
3747		-{
3748		- u64 result;
3749		-
3750		- result = MLX5_MKEY_MASK_PD;
3751		-
3752		- return cpu_to_be64(result);
3753		-}
3754		-
3755		-static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)
3756		-{
3757		- if ((mask & MLX5_MKEY_MASK_PAGE_SIZE &&
3758		- MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) \|\|
3759		- (mask & MLX5_MKEY_MASK_A &&
3760		- MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)))
3761		- return -EPERM;
3762		- return 0;
3763		-}
3764		-
3765		-static int set_reg_umr_segment(struct mlx5_ib_dev *dev,
3766		- struct mlx5_wqe_umr_ctrl_seg *umr,
3767		- const struct ib_send_wr *wr, int atomic)
3768		-{
3769		- const struct mlx5_umr_wr *umrwr = umr_wr(wr);
3770		-
3771		- memset(umr, 0, sizeof(*umr));
3772		-
3773		- if (!umrwr->ignore_free_state) {
3774		- if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE)
3775		- /* fail if free */
3776		- umr->flags = MLX5_UMR_CHECK_FREE;
3777		- else
3778		- /* fail if not free */
3779		- umr->flags = MLX5_UMR_CHECK_NOT_FREE;
3780		- }
3781		-
3782		- umr->xlt_octowords = cpu_to_be16(get_xlt_octo(umrwr->xlt_size));
3783		- if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_XLT) {
3784		- u64 offset = get_xlt_octo(umrwr->offset);
3785		-
3786		- umr->xlt_offset = cpu_to_be16(offset & 0xffff);
3787		- umr->xlt_offset_47_16 = cpu_to_be32(offset >> 16);
3788		- umr->flags \|= MLX5_UMR_TRANSLATION_OFFSET_EN;
3789		- }
3790		- if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)
3791		- umr->mkey_mask \|= get_umr_update_translation_mask();
3792		- if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {
3793		- umr->mkey_mask \|= get_umr_update_access_mask(atomic);
3794		- umr->mkey_mask \|= get_umr_update_pd_mask();
3795		- }
3796		- if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR)
3797		- umr->mkey_mask \|= get_umr_enable_mr_mask();
3798		- if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
3799		- umr->mkey_mask \|= get_umr_disable_mr_mask();
3800		-
3801		- if (!wr->num_sge)
3802		- umr->flags \|= MLX5_UMR_INLINE;
3803		-
3804		- return umr_check_mkey_mask(dev, be64_to_cpu(umr->mkey_mask));
3805		-}
3806		-
3807		-static u8 get_umr_flags(int acc)
3808		-{
3809		- return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) \|
3810		- (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) \|
3811		- (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) \|
3812		- (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) \|
3813		- MLX5_PERM_LOCAL_READ \| MLX5_PERM_UMR_EN;
3814		-}
3815		-
3816		-static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
3817		- struct mlx5_ib_mr *mr,
3818		- u32 key, int access)
3819		-{
3820		- int ndescs = ALIGN(mr->ndescs, 8) >> 1;
3821		-
3822		- memset(seg, 0, sizeof(*seg));
3823		-
3824		- if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
3825		- seg->log2_page_size = ilog2(mr->ibmr.page_size);
3826		- else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
3827		- /* KLMs take twice the size of MTTs */
3828		- ndescs *= 2;
3829		-
3830		- seg->flags = get_umr_flags(access) \| mr->access_mode;
3831		- seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) \| 0xffffff00);
3832		- seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
3833		- seg->start_addr = cpu_to_be64(mr->ibmr.iova);
3834		- seg->len = cpu_to_be64(mr->ibmr.length);
3835		- seg->xlt_oct_size = cpu_to_be32(ndescs);
3836		-}
3837		-
3838		-static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
3839		-{
3840		- memset(seg, 0, sizeof(*seg));
3841		- seg->status = MLX5_MKEY_STATUS_FREE;
3842		-}
3843		-
3844		-static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg,
3845		- const struct ib_send_wr *wr)
3846		-{
3847		- const struct mlx5_umr_wr *umrwr = umr_wr(wr);
3848		-
3849		- memset(seg, 0, sizeof(*seg));
3850		- if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
3851		- seg->status = MLX5_MKEY_STATUS_FREE;
3852		-
3853		- seg->flags = convert_access(umrwr->access_flags);
3854		- if (umrwr->pd)
3855		- seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn);
3856		- if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION &&
3857		- !umrwr->length)
3858		- seg->flags_pd \|= cpu_to_be32(MLX5_MKEY_LEN64);
3859		-
3860		- seg->start_addr = cpu_to_be64(umrwr->virt_addr);
3861		- seg->len = cpu_to_be64(umrwr->length);
3862		- seg->log2_page_size = umrwr->page_shift;
3863		- seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 \|
3864		- mlx5_mkey_variant(umrwr->mkey));
3865		-}
3866		-
3867		-static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
3868		- struct mlx5_ib_mr *mr,
3869		- struct mlx5_ib_pd *pd)
3870		-{
3871		- int bcount = mr->desc_size * mr->ndescs;
3872		-
3873		- dseg->addr = cpu_to_be64(mr->desc_map);
3874		- dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
3875		- dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
3876		-}
3877		-
3878		-static void set_reg_umr_inline_seg(void seg, struct mlx5_ib_qp qp,
3879		- struct mlx5_ib_mr *mr, int mr_list_size)
3880		-{
3881		- void *qend = qp->sq.qend;
3882		- void *addr = mr->descs;
3883		- int copy;
3884		-
3885		- if (unlikely(seg + mr_list_size > qend)) {
3886		- copy = qend - seg;
3887		- memcpy(seg, addr, copy);
3888		- addr += copy;
3889		- mr_list_size -= copy;
3890		- seg = mlx5_get_send_wqe(qp, 0);
3891		- }
3892		- memcpy(seg, addr, mr_list_size);
3893		- seg += mr_list_size;
3894		-}
3895		-
3896		-static __be32 send_ieth(const struct ib_send_wr *wr)
3897		-{
3898		- switch (wr->opcode) {
3899		- case IB_WR_SEND_WITH_IMM:
3900		- case IB_WR_RDMA_WRITE_WITH_IMM:
3901		- return wr->ex.imm_data;
3902		-
3903		- case IB_WR_SEND_WITH_INV:
3904		- return cpu_to_be32(wr->ex.invalidate_rkey);
3905		-
3906		- default:
3907		- return 0;
3908		- }
3909		-}
3910		-
3911		-static u8 calc_sig(void *wqe, int size)
3912		-{
3913		- u8 *p = wqe;
3914		- u8 res = 0;
3915		- int i;
3916		-
3917		- for (i = 0; i < size; i++)
3918		- res ^= p[i];
3919		-
3920		- return ~res;
3921		-}
3922		-
3923		-static u8 wq_sig(void *wqe)
3924		-{
3925		- return calc_sig(wqe, (((u8 )wqe + 8) & 0x3f) << 4);
3926		-}
3927		-
3928		-static int set_data_inl_seg(struct mlx5_ib_qp qp, const struct ib_send_wr wr,
3929		- void wqe, int sz)
3930		-{
3931		- struct mlx5_wqe_inline_seg *seg;
3932		- void *qend = qp->sq.qend;
3933		- void *addr;
3934		- int inl = 0;
3935		- int copy;
3936		- int len;
3937		- int i;
3938		-
3939		- seg = wqe;
3940		- wqe += sizeof(*seg);
3941		- for (i = 0; i < wr->num_sge; i++) {
3942		- addr = (void *)(unsigned long)(wr->sg_list[i].addr);
3943		- len = wr->sg_list[i].length;
3944		- inl += len;
3945		-
3946		- if (unlikely(inl > qp->max_inline_data))
3947		- return -ENOMEM;
3948		-
3949		- if (unlikely(wqe + len > qend)) {
3950		- copy = qend - wqe;
3951		- memcpy(wqe, addr, copy);
3952		- addr += copy;
3953		- len -= copy;
3954		- wqe = mlx5_get_send_wqe(qp, 0);
3955		- }
3956		- memcpy(wqe, addr, len);
3957		- wqe += len;
3958		- }
3959		-
3960		- seg->byte_count = cpu_to_be32(inl \| MLX5_INLINE_SEG);
3961		-
3962		- *sz = ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
3963		-
3964		- return 0;
3965		-}
3966		-
3967		-static u16 prot_field_size(enum ib_signature_type type)
3968		-{
3969		- switch (type) {
3970		- case IB_SIG_TYPE_T10_DIF:
3971		- return MLX5_DIF_SIZE;
3972		- default:
3973		- return 0;
3974		- }
3975		-}
3976		-
3977		-static u8 bs_selector(int block_size)
3978		-{
3979		- switch (block_size) {
3980		- case 512: return 0x1;
3981		- case 520: return 0x2;
3982		- case 4096: return 0x3;
3983		- case 4160: return 0x4;
3984		- case 1073741824: return 0x5;
3985		- default: return 0;
3986		- }
3987		-}
3988		-
3989		-static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
3990		- struct mlx5_bsf_inl *inl)
3991		-{
3992		- /* Valid inline section and allow BSF refresh */
3993		- inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID \|
3994		- MLX5_BSF_REFRESH_DIF);
3995		- inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
3996		- inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
3997		- /* repeating block */
3998		- inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
3999		- inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
4000		- MLX5_DIF_CRC : MLX5_DIF_IPCS;
4001		-
4002		- if (domain->sig.dif.ref_remap)
4003		- inl->dif_inc_ref_guard_check \|= MLX5_BSF_INC_REFTAG;
4004		-
4005		- if (domain->sig.dif.app_escape) {
4006		- if (domain->sig.dif.ref_escape)
4007		- inl->dif_inc_ref_guard_check \|= MLX5_BSF_APPREF_ESCAPE;
4008		- else
4009		- inl->dif_inc_ref_guard_check \|= MLX5_BSF_APPTAG_ESCAPE;
4010		- }
4011		-
4012		- inl->dif_app_bitmask_check =
4013		- cpu_to_be16(domain->sig.dif.apptag_check_mask);
4014		-}
4015		-
4016		-static int mlx5_set_bsf(struct ib_mr *sig_mr,
4017		- struct ib_sig_attrs *sig_attrs,
4018		- struct mlx5_bsf *bsf, u32 data_size)
4019		-{
4020		- struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
4021		- struct mlx5_bsf_basic *basic = &bsf->basic;
4022		- struct ib_sig_domain *mem = &sig_attrs->mem;
4023		- struct ib_sig_domain *wire = &sig_attrs->wire;
4024		-
4025		- memset(bsf, 0, sizeof(*bsf));
4026		-
4027		- /* Basic + Extended + Inline */
4028		- basic->bsf_size_sbs = 1 << 7;
4029		- /* Input domain check byte mask */
4030		- basic->check_byte_mask = sig_attrs->check_mask;
4031		- basic->raw_data_size = cpu_to_be32(data_size);
4032		-
4033		- /* Memory domain */
4034		- switch (sig_attrs->mem.sig_type) {
4035		- case IB_SIG_TYPE_NONE:
4036		- break;
4037		- case IB_SIG_TYPE_T10_DIF:
4038		- basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
4039		- basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
4040		- mlx5_fill_inl_bsf(mem, &bsf->m_inl);
4041		- break;
4042		- default:
4043		- return -EINVAL;
4044		- }
4045		-
4046		- /* Wire domain */
4047		- switch (sig_attrs->wire.sig_type) {
4048		- case IB_SIG_TYPE_NONE:
4049		- break;
4050		- case IB_SIG_TYPE_T10_DIF:
4051		- if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
4052		- mem->sig_type == wire->sig_type) {
4053		- /* Same block structure */
4054		- basic->bsf_size_sbs \|= 1 << 4;
4055		- if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
4056		- basic->wire.copy_byte_mask \|= MLX5_CPY_GRD_MASK;
4057		- if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
4058		- basic->wire.copy_byte_mask \|= MLX5_CPY_APP_MASK;
4059		- if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
4060		- basic->wire.copy_byte_mask \|= MLX5_CPY_REF_MASK;
4061		- } else
4062		- basic->wire.bs_selector = bs_selector(wire->sig.dif.pi_interval);
4063		-
4064		- basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
4065		- mlx5_fill_inl_bsf(wire, &bsf->w_inl);
4066		- break;
4067		- default:
4068		- return -EINVAL;
4069		- }
4070		-
4071		- return 0;
4072		-}
4073		-
4074		-static int set_sig_data_segment(const struct ib_sig_handover_wr *wr,
4075		- struct mlx5_ib_qp qp, void seg, int size)
4076		-{
4077		- struct ib_sig_attrs *sig_attrs = wr->sig_attrs;
4078		- struct ib_mr *sig_mr = wr->sig_mr;
4079		- struct mlx5_bsf *bsf;
4080		- u32 data_len = wr->wr.sg_list->length;
4081		- u32 data_key = wr->wr.sg_list->lkey;
4082		- u64 data_va = wr->wr.sg_list->addr;
4083		- int ret;
4084		- int wqe_size;
4085		-
4086		- if (!wr->prot \|\|
4087		- (data_key == wr->prot->lkey &&
4088		- data_va == wr->prot->addr &&
4089		- data_len == wr->prot->length)) {
4090		- /**
4091		- * Source domain doesn't contain signature information
4092		- * or data and protection are interleaved in memory.
4093		- * So need construct:
4094		- * ------------------
4095		- * \| data_klm \|
4096		- * ------------------
4097		- * \| BSF \|
4098		- * ------------------
4099		- **/
4100		- struct mlx5_klm data_klm = seg;
4101		-
4102		- data_klm->bcount = cpu_to_be32(data_len);
4103		- data_klm->key = cpu_to_be32(data_key);
4104		- data_klm->va = cpu_to_be64(data_va);
4105		- wqe_size = ALIGN(sizeof(*data_klm), 64);
4106		- } else {
4107		- /**
4108		- * Source domain contains signature information
4109		- * So need construct a strided block format:
4110		- * ---------------------------
4111		- * \| stride_block_ctrl \|
4112		- * ---------------------------
4113		- * \| data_klm \|
4114		- * ---------------------------
4115		- * \| prot_klm \|
4116		- * ---------------------------
4117		- * \| BSF \|
4118		- * ---------------------------
4119		- **/
4120		- struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
4121		- struct mlx5_stride_block_entry *data_sentry;
4122		- struct mlx5_stride_block_entry *prot_sentry;
4123		- u32 prot_key = wr->prot->lkey;
4124		- u64 prot_va = wr->prot->addr;
4125		- u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
4126		- int prot_size;
4127		-
4128		- sblock_ctrl = *seg;
4129		- data_sentry = (void )sblock_ctrl + sizeof(sblock_ctrl);
4130		- prot_sentry = (void )data_sentry + sizeof(data_sentry);
4131		-
4132		- prot_size = prot_field_size(sig_attrs->mem.sig_type);
4133		- if (!prot_size) {
4134		- pr_err("Bad block size given: %u\n", block_size);
4135		- return -EINVAL;
4136		- }
4137		- sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
4138		- prot_size);
4139		- sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
4140		- sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
4141		- sblock_ctrl->num_entries = cpu_to_be16(2);
4142		-
4143		- data_sentry->bcount = cpu_to_be16(block_size);
4144		- data_sentry->key = cpu_to_be32(data_key);
4145		- data_sentry->va = cpu_to_be64(data_va);
4146		- data_sentry->stride = cpu_to_be16(block_size);
4147		-
4148		- prot_sentry->bcount = cpu_to_be16(prot_size);
4149		- prot_sentry->key = cpu_to_be32(prot_key);
4150		- prot_sentry->va = cpu_to_be64(prot_va);
4151		- prot_sentry->stride = cpu_to_be16(prot_size);
4152		-
4153		- wqe_size = ALIGN(sizeof(sblock_ctrl) + sizeof(data_sentry) +
4154		- sizeof(*prot_sentry), 64);
4155		- }
4156		-
4157		- *seg += wqe_size;
4158		- *size += wqe_size / 16;
4159		- if (unlikely((*seg == qp->sq.qend)))
4160		- *seg = mlx5_get_send_wqe(qp, 0);
4161		-
4162		- bsf = *seg;
4163		- ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
4164		- if (ret)
4165		- return -EINVAL;
4166		-
4167		- seg += sizeof(bsf);
4168		- size += sizeof(bsf) / 16;
4169		- if (unlikely((*seg == qp->sq.qend)))
4170		- *seg = mlx5_get_send_wqe(qp, 0);
4171		-
4172		- return 0;
4173		-}
4174		-
4175		-static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
4176		- const struct ib_sig_handover_wr *wr, u32 size,
4177		- u32 length, u32 pdn)
4178		-{
4179		- struct ib_mr *sig_mr = wr->sig_mr;
4180		- u32 sig_key = sig_mr->rkey;
4181		- u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
4182		-
4183		- memset(seg, 0, sizeof(*seg));
4184		-
4185		- seg->flags = get_umr_flags(wr->access_flags) \|
4186		- MLX5_MKC_ACCESS_MODE_KLMS;
4187		- seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) \| 0xffffff00);
4188		- seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL \| sigerr << 26 \|
4189		- MLX5_MKEY_BSF_EN \| pdn);
4190		- seg->len = cpu_to_be64(length);
4191		- seg->xlt_oct_size = cpu_to_be32(get_xlt_octo(size));
4192		- seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
4193		-}
4194		-
4195		-static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
4196		- u32 size)
4197		-{
4198		- memset(umr, 0, sizeof(*umr));
4199		-
4200		- umr->flags = MLX5_FLAGS_INLINE \| MLX5_FLAGS_CHECK_FREE;
4201		- umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
4202		- umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
4203		- umr->mkey_mask = sig_mkey_mask();
4204		-}
4205		-
4206		-
4207		-static int set_sig_umr_wr(const struct ib_send_wr *send_wr,
4208		- struct mlx5_ib_qp qp, void seg, int size)
4209		-{
4210		- const struct ib_sig_handover_wr *wr = sig_handover_wr(send_wr);
4211		- struct mlx5_ib_mr *sig_mr = to_mmr(wr->sig_mr);
4212		- u32 pdn = get_pd(qp)->pdn;
4213		- u32 xlt_size;
4214		- int region_len, ret;
4215		-
4216		- if (unlikely(wr->wr.num_sge != 1) \|\|
4217		- unlikely(wr->access_flags & IB_ACCESS_REMOTE_ATOMIC) \|\|
4218		- unlikely(!sig_mr->sig) \|\| unlikely(!qp->signature_en) \|\|
4219		- unlikely(!sig_mr->sig->sig_status_checked))
4220		- return -EINVAL;
4221		-
4222		- /* length of the protected region, data + protection */
4223		- region_len = wr->wr.sg_list->length;
4224		- if (wr->prot &&
4225		- (wr->prot->lkey != wr->wr.sg_list->lkey \|\|
4226		- wr->prot->addr != wr->wr.sg_list->addr \|\|
4227		- wr->prot->length != wr->wr.sg_list->length))
4228		- region_len += wr->prot->length;
4229		-
4230		- /**
4231		- * KLM octoword size - if protection was provided
4232		- * then we use strided block format (3 octowords),
4233		- * else we use single KLM (1 octoword)
4234		- **/
4235		- xlt_size = wr->prot ? 0x30 : sizeof(struct mlx5_klm);
4236		-
4237		- set_sig_umr_segment(*seg, xlt_size);
4238		- *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
4239		- *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
4240		- if (unlikely((*seg == qp->sq.qend)))
4241		- *seg = mlx5_get_send_wqe(qp, 0);
4242		-
4243		- set_sig_mkey_segment(*seg, wr, xlt_size, region_len, pdn);
4244		- *seg += sizeof(struct mlx5_mkey_seg);
4245		- *size += sizeof(struct mlx5_mkey_seg) / 16;
4246		- if (unlikely((*seg == qp->sq.qend)))
4247		- *seg = mlx5_get_send_wqe(qp, 0);
4248		-
4249		- ret = set_sig_data_segment(wr, qp, seg, size);
4250		- if (ret)
4251		- return ret;
4252		-
4253		- sig_mr->sig->sig_status_checked = false;
4254		- return 0;
4255		-}
4256		-
4257		-static int set_psv_wr(struct ib_sig_domain *domain,
4258		- u32 psv_idx, void *seg, int size)
4259		-{
4260		- struct mlx5_seg_set_psv psv_seg = seg;
4261		-
4262		- memset(psv_seg, 0, sizeof(*psv_seg));
4263		- psv_seg->psv_num = cpu_to_be32(psv_idx);
4264		- switch (domain->sig_type) {
4265		- case IB_SIG_TYPE_NONE:
4266		- break;
4267		- case IB_SIG_TYPE_T10_DIF:
4268		- psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 \|
4269		- domain->sig.dif.app_tag);
4270		- psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
4271		- break;
4272		- default:
4273		- pr_err("Bad signature type (%d) is given.\n",
4274		- domain->sig_type);
4275		- return -EINVAL;
4276		- }
4277		-
4278		- seg += sizeof(psv_seg);
4279		- size += sizeof(psv_seg) / 16;
4280		-
4281		- return 0;
4282		-}
4283		-
4284		-static int set_reg_wr(struct mlx5_ib_qp *qp,
4285		- const struct ib_reg_wr *wr,
4286		- void *seg, int size)
4287		-{
4288		- struct mlx5_ib_mr *mr = to_mmr(wr->mr);
4289		- struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
4290		- int mr_list_size = mr->ndescs * mr->desc_size;
4291		- bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;
4292		-
4293		- if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
4294		- mlx5_ib_warn(to_mdev(qp->ibqp.device),
4295		- "Invalid IB_SEND_INLINE send flag\n");
4296		- return -EINVAL;
4297		- }
4298		-
4299		- set_reg_umr_seg(*seg, mr, umr_inline);
4300		- *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
4301		- *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
4302		- if (unlikely((*seg == qp->sq.qend)))
4303		- *seg = mlx5_get_send_wqe(qp, 0);
4304		-
4305		- set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
4306		- *seg += sizeof(struct mlx5_mkey_seg);
4307		- *size += sizeof(struct mlx5_mkey_seg) / 16;
4308		- if (unlikely((*seg == qp->sq.qend)))
4309		- *seg = mlx5_get_send_wqe(qp, 0);
4310		-
4311		- if (umr_inline) {
4312		- set_reg_umr_inline_seg(*seg, qp, mr, mr_list_size);
4313		- *size += get_xlt_octo(mr_list_size);
4314		- } else {
4315		- set_reg_data_seg(*seg, mr, pd);
4316		- *seg += sizeof(struct mlx5_wqe_data_seg);
4317		- *size += (sizeof(struct mlx5_wqe_data_seg) / 16);
4318		- }
4319		- return 0;
4320		-}
4321		-
4322		-static void set_linv_wr(struct mlx5_ib_qp qp, void seg, int size)
4323		-{
4324		- set_linv_umr_seg(*seg);
4325		- *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
4326		- *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
4327		- if (unlikely((*seg == qp->sq.qend)))
4328		- *seg = mlx5_get_send_wqe(qp, 0);
4329		- set_linv_mkey_seg(*seg);
4330		- *seg += sizeof(struct mlx5_mkey_seg);
4331		- *size += sizeof(struct mlx5_mkey_seg) / 16;
4332		- if (unlikely((*seg == qp->sq.qend)))
4333		- *seg = mlx5_get_send_wqe(qp, 0);
4334		-}
4335		-
4336		-static void dump_wqe(struct mlx5_ib_qp *qp, int idx, int size_16)
4337		-{
4338		- __be32 *p = NULL;
4339		- int tidx = idx;
4340		- int i, j;
4341		-
4342		- pr_debug("dump wqe at %p\n", mlx5_get_send_wqe(qp, tidx));
4343		- for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
4344		- if ((i & 0xf) == 0) {
4345		- void *buf = mlx5_get_send_wqe(qp, tidx);
4346		- tidx = (tidx + 1) & (qp->sq.wqe_cnt - 1);
4347		- p = buf;
4348		- j = 0;
4349		- }
4350		- pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
4351		- be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
4352		- be32_to_cpu(p[j + 3]));
4353		- }
4354		-}
4355		-
4356		-static int __begin_wqe(struct mlx5_ib_qp qp, void *seg,
4357		- struct mlx5_wqe_ctrl_seg **ctrl,
4358		- const struct ib_send_wr wr, unsigned idx,
4359		- int *size, int nreq, bool send_signaled, bool solicited)
4360		-{
4361		- if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
4362		- return -ENOMEM;
4363		-
4364		- *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
4365		- seg = mlx5_get_send_wqe(qp, idx);
4366		- ctrl = seg;
4367		- (uint32_t )(*seg + 8) = 0;
4368		- (*ctrl)->imm = send_ieth(wr);
4369		- (*ctrl)->fm_ce_se = qp->sq_signal_bits \|
4370		- (send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) \|
4371		- (solicited ? MLX5_WQE_CTRL_SOLICITED : 0);
4372		-
4373		- seg += sizeof(*ctrl);
4374		- size = sizeof(*ctrl) / 16;
4375		-
4376		- return 0;
4377		-}
4378		-
4379		-static int begin_wqe(struct mlx5_ib_qp qp, void *seg,
4380		- struct mlx5_wqe_ctrl_seg **ctrl,
4381		- const struct ib_send_wr wr, unsigned idx,
4382		- int *size, int nreq)
4383		-{
4384		- return __begin_wqe(qp, seg, ctrl, wr, idx, size, nreq,
4385		- wr->send_flags & IB_SEND_SIGNALED,
4386		- wr->send_flags & IB_SEND_SOLICITED);
4387		-}
4388		-
4389		-static void finish_wqe(struct mlx5_ib_qp *qp,
4390		- struct mlx5_wqe_ctrl_seg *ctrl,
4391		- u8 size, unsigned idx, u64 wr_id,
4392		- int nreq, u8 fence, u32 mlx5_opcode)
4393		-{
4394		- u8 opmod = 0;
4395		-
4396		- ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) \|
4397		- mlx5_opcode \| ((u32)opmod << 24));
4398		- ctrl->qpn_ds = cpu_to_be32(size \| (qp->trans_qp.base.mqp.qpn << 8));
4399		- ctrl->fm_ce_se \|= fence;
4400		- if (unlikely(qp->wq_sig))
4401		- ctrl->signature = wq_sig(ctrl);
4402		-
4403		- qp->sq.wrid[idx] = wr_id;
4404		- qp->sq.w_list[idx].opcode = mlx5_opcode;
4405		- qp->sq.wqe_head[idx] = qp->sq.head + nreq;
4406		- qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
4407		- qp->sq.w_list[idx].next = qp->sq.cur_post;
4408		-}
4409		-
4410		-static int _mlx5_ib_post_send(struct ib_qp ibqp, const struct ib_send_wr wr,
4411		- const struct ib_send_wr **bad_wr, bool drain)
4412		-{
4413		- struct mlx5_wqe_ctrl_seg ctrl = NULL; / compiler warning */
4414		- struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4415		- struct mlx5_core_dev *mdev = dev->mdev;
4416		- struct mlx5_ib_qp *qp;
4417		- struct mlx5_ib_mr *mr;
4418		- struct mlx5_wqe_data_seg *dpseg;
4419		- struct mlx5_wqe_xrc_seg *xrc;
4420		- struct mlx5_bf *bf;
4421		- int uninitialized_var(size);
4422		- void *qend;
4423		- unsigned long flags;
4424		- unsigned idx;
4425		- int err = 0;
4426		- int num_sge;
4427		- void *seg;
4428		- int nreq;
4429		- int i;
4430		- u8 next_fence = 0;
4431		- u8 fence;
4432		-
4433		- if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
4434		- !drain)) {
4435		- *bad_wr = wr;
4436		- return -EIO;
4437		- }
4438		-
4439		- if (unlikely(ibqp->qp_type == IB_QPT_GSI))
4440		- return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
4441		-
4442		- qp = to_mqp(ibqp);
4443		- bf = &qp->bf;
4444		- qend = qp->sq.qend;
4445		-
4446		- spin_lock_irqsave(&qp->sq.lock, flags);
4447		-
4448		- for (nreq = 0; wr; nreq++, wr = wr->next) {
4449		- if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {
4450		- mlx5_ib_warn(dev, "\n");
4451		- err = -EINVAL;
4452		- *bad_wr = wr;
4453		- goto out;
4454		- }
4455		-
4456		- num_sge = wr->num_sge;
4457		- if (unlikely(num_sge > qp->sq.max_gs)) {
4458		- mlx5_ib_warn(dev, "\n");
4459		- err = -EINVAL;
4460		- *bad_wr = wr;
4461		- goto out;
4462		- }
4463		-
4464		- err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, nreq);
4465		- if (err) {
4466		- mlx5_ib_warn(dev, "\n");
4467		- err = -ENOMEM;
4468		- *bad_wr = wr;
4469		- goto out;
4470		- }
4471		-
4472		- if (wr->opcode == IB_WR_REG_MR) {
4473		- fence = dev->umr_fence;
4474		- next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
4475		- } else {
4476		- if (wr->send_flags & IB_SEND_FENCE) {
4477		- if (qp->next_fence)
4478		- fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
4479		- else
4480		- fence = MLX5_FENCE_MODE_FENCE;
4481		- } else {
4482		- fence = qp->next_fence;
4483		- }
4484		- }
4485		-
4486		- switch (ibqp->qp_type) {
4487		- case IB_QPT_XRC_INI:
4488		- xrc = seg;
4489		- seg += sizeof(*xrc);
4490		- size += sizeof(*xrc) / 16;
4491		- /* fall through */
4492		- case IB_QPT_RC:
4493		- switch (wr->opcode) {
4494		- case IB_WR_RDMA_READ:
4495		- case IB_WR_RDMA_WRITE:
4496		- case IB_WR_RDMA_WRITE_WITH_IMM:
4497		- set_raddr_seg(seg, rdma_wr(wr)->remote_addr,
4498		- rdma_wr(wr)->rkey);
4499		- seg += sizeof(struct mlx5_wqe_raddr_seg);
4500		- size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
4501		- break;
4502		-
4503		- case IB_WR_ATOMIC_CMP_AND_SWP:
4504		- case IB_WR_ATOMIC_FETCH_AND_ADD:
4505		- case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
4506		- mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");
4507		- err = -ENOSYS;
4508		- *bad_wr = wr;
4509		- goto out;
4510		-
4511		- case IB_WR_LOCAL_INV:
4512		- qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
4513		- ctrl->imm = cpu_to_be32(wr->ex.invalidate_rkey);
4514		- set_linv_wr(qp, &seg, &size);
4515		- num_sge = 0;
4516		- break;
4517		-
4518		- case IB_WR_REG_MR:
4519		- qp->sq.wr_data[idx] = IB_WR_REG_MR;
4520		- ctrl->imm = cpu_to_be32(reg_wr(wr)->key);
4521		- err = set_reg_wr(qp, reg_wr(wr), &seg, &size);
4522		- if (err) {
4523		- *bad_wr = wr;
4524		- goto out;
4525		- }
4526		- num_sge = 0;
4527		- break;
4528		-
4529		- case IB_WR_REG_SIG_MR:
4530		- qp->sq.wr_data[idx] = IB_WR_REG_SIG_MR;
4531		- mr = to_mmr(sig_handover_wr(wr)->sig_mr);
4532		-
4533		- ctrl->imm = cpu_to_be32(mr->ibmr.rkey);
4534		- err = set_sig_umr_wr(wr, qp, &seg, &size);
4535		- if (err) {
4536		- mlx5_ib_warn(dev, "\n");
4537		- *bad_wr = wr;
4538		- goto out;
4539		- }
4540		-
4541		- finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
4542		- fence, MLX5_OPCODE_UMR);
4543		- /*
4544		- * SET_PSV WQEs are not signaled and solicited
4545		- * on error
4546		- */
4547		- err = __begin_wqe(qp, &seg, &ctrl, wr, &idx,
4548		- &size, nreq, false, true);
4549		- if (err) {
4550		- mlx5_ib_warn(dev, "\n");
4551		- err = -ENOMEM;
4552		- *bad_wr = wr;
4553		- goto out;
4554		- }
4555		-
4556		- err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->mem,
4557		- mr->sig->psv_memory.psv_idx, &seg,
4558		- &size);
4559		- if (err) {
4560		- mlx5_ib_warn(dev, "\n");
4561		- *bad_wr = wr;
4562		- goto out;
4563		- }
4564		-
4565		- finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
4566		- fence, MLX5_OPCODE_SET_PSV);
4567		- err = __begin_wqe(qp, &seg, &ctrl, wr, &idx,
4568		- &size, nreq, false, true);
4569		- if (err) {
4570		- mlx5_ib_warn(dev, "\n");
4571		- err = -ENOMEM;
4572		- *bad_wr = wr;
4573		- goto out;
4574		- }
4575		-
4576		- err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->wire,
4577		- mr->sig->psv_wire.psv_idx, &seg,
4578		- &size);
4579		- if (err) {
4580		- mlx5_ib_warn(dev, "\n");
4581		- *bad_wr = wr;
4582		- goto out;
4583		- }
4584		-
4585		- finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
4586		- fence, MLX5_OPCODE_SET_PSV);
4587		- qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
4588		- num_sge = 0;
4589		- goto skip_psv;
4590		-
4591		- default:
4592		- break;
4593		- }
4594		- break;
4595		-
4596		- case IB_QPT_UC:
4597		- switch (wr->opcode) {
4598		- case IB_WR_RDMA_WRITE:
4599		- case IB_WR_RDMA_WRITE_WITH_IMM:
4600		- set_raddr_seg(seg, rdma_wr(wr)->remote_addr,
4601		- rdma_wr(wr)->rkey);
4602		- seg += sizeof(struct mlx5_wqe_raddr_seg);
4603		- size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
4604		- break;
4605		-
4606		- default:
4607		- break;
4608		- }
4609		- break;
4610		-
4611		- case IB_QPT_SMI:
4612		- if (unlikely(!mdev->port_caps[qp->port - 1].has_smi)) {
4613		- mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
4614		- err = -EPERM;
4615		- *bad_wr = wr;
4616		- goto out;
4617		- }
4618		- /* fall through */
4619		- case MLX5_IB_QPT_HW_GSI:
4620		- set_datagram_seg(seg, wr);
4621		- seg += sizeof(struct mlx5_wqe_datagram_seg);
4622		- size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
4623		- if (unlikely((seg == qend)))
4624		- seg = mlx5_get_send_wqe(qp, 0);
4625		- break;
4626		- case IB_QPT_UD:
4627		- set_datagram_seg(seg, wr);
4628		- seg += sizeof(struct mlx5_wqe_datagram_seg);
4629		- size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
4630		-
4631		- if (unlikely((seg == qend)))
4632		- seg = mlx5_get_send_wqe(qp, 0);
4633		-
4634		- /* handle qp that supports ud offload */
4635		- if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
4636		- struct mlx5_wqe_eth_pad *pad;
4637		-
4638		- pad = seg;
4639		- memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
4640		- seg += sizeof(struct mlx5_wqe_eth_pad);
4641		- size += sizeof(struct mlx5_wqe_eth_pad) / 16;
4642		-
4643		- seg = set_eth_seg(seg, wr, qend, qp, &size);
4644		-
4645		- if (unlikely((seg == qend)))
4646		- seg = mlx5_get_send_wqe(qp, 0);
4647		- }
4648		- break;
4649		- case MLX5_IB_QPT_REG_UMR:
4650		- if (wr->opcode != MLX5_IB_WR_UMR) {
4651		- err = -EINVAL;
4652		- mlx5_ib_warn(dev, "bad opcode\n");
4653		- goto out;
4654		- }
4655		- qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
4656		- ctrl->imm = cpu_to_be32(umr_wr(wr)->mkey);
4657		- err = set_reg_umr_segment(dev, seg, wr, !!(MLX5_CAP_GEN(mdev, atomic)));
4658		- if (unlikely(err))
4659		- goto out;
4660		- seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
4661		- size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
4662		- if (unlikely((seg == qend)))
4663		- seg = mlx5_get_send_wqe(qp, 0);
4664		- set_reg_mkey_segment(seg, wr);
4665		- seg += sizeof(struct mlx5_mkey_seg);
4666		- size += sizeof(struct mlx5_mkey_seg) / 16;
4667		- if (unlikely((seg == qend)))
4668		- seg = mlx5_get_send_wqe(qp, 0);
4669		- break;
4670		-
4671		- default:
4672		- break;
4673		- }
4674		-
4675		- if (wr->send_flags & IB_SEND_INLINE && num_sge) {
4676		- int uninitialized_var(sz);
4677		-
4678		- err = set_data_inl_seg(qp, wr, seg, &sz);
4679		- if (unlikely(err)) {
4680		- mlx5_ib_warn(dev, "\n");
4681		- *bad_wr = wr;
4682		- goto out;
4683		- }
4684		- size += sz;
4685		- } else {
4686		- dpseg = seg;
4687		- for (i = 0; i < num_sge; i++) {
4688		- if (unlikely(dpseg == qend)) {
4689		- seg = mlx5_get_send_wqe(qp, 0);
4690		- dpseg = seg;
4691		- }
4692		- if (likely(wr->sg_list[i].length)) {
4693		- set_data_ptr_seg(dpseg, wr->sg_list + i);
4694		- size += sizeof(struct mlx5_wqe_data_seg) / 16;
4695		- dpseg++;
4696		- }
4697		- }
4698		- }
4699		-
4700		- qp->next_fence = next_fence;
4701		- finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq, fence,
4702		- mlx5_ib_opcode[wr->opcode]);
4703		-skip_psv:
4704		- if (0)
4705		- dump_wqe(qp, idx, size);
4706		- }
4707		-
4708		-out:
4709		- if (likely(nreq)) {
4710		- qp->sq.head += nreq;
4711		-
4712		- /* Make sure that descriptors are written before
4713		- * updating doorbell record and ringing the doorbell
4714		- */
4715		- wmb();
4716		-
4717		- qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
4718		-
4719		- /* Make sure doorbell record is visible to the HCA before
4720		- * we hit doorbell */
4721		- wmb();
4722		-
4723		- /* currently we support only regular doorbells */
4724		- mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset, NULL);
4725		- /* Make sure doorbells don't leak out of SQ spinlock
4726		- * and reach the HCA out of order.
4727		- */
4728		- mmiowb();
4729		- bf->offset ^= bf->buf_size;
4730		- }
4731		-
4732		- spin_unlock_irqrestore(&qp->sq.lock, flags);
4733		-
4734		- return err;
4735		-}
4736		-
4737		-int mlx5_ib_post_send(struct ib_qp ibqp, const struct ib_send_wr wr,
4738		- const struct ib_send_wr **bad_wr)
4739		-{
4740		- return _mlx5_ib_post_send(ibqp, wr, bad_wr, false);
4741		-}
4742		-
4743		-static void set_sig_seg(struct mlx5_rwqe_sig *sig, int size)
4744		-{
4745		- sig->signature = calc_sig(sig, size);
4746		-}
4747		-
4748		-static int _mlx5_ib_post_recv(struct ib_qp ibqp, const struct ib_recv_wr wr,
4749		- const struct ib_recv_wr **bad_wr, bool drain)
4750		-{
4751		- struct mlx5_ib_qp *qp = to_mqp(ibqp);
4752		- struct mlx5_wqe_data_seg *scat;
4753		- struct mlx5_rwqe_sig *sig;
4754		- struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4755		- struct mlx5_core_dev *mdev = dev->mdev;
4756		- unsigned long flags;
4757		- int err = 0;
4758		- int nreq;
4759		- int ind;
4760		- int i;
4761		-
4762		- if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
4763		- !drain)) {
4764		- *bad_wr = wr;
4765		- return -EIO;
4766		- }
4767		-
4768		- if (unlikely(ibqp->qp_type == IB_QPT_GSI))
4769		- return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);
4770		-
4771		- spin_lock_irqsave(&qp->rq.lock, flags);
4772		-
4773		- ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
4774		-
4775		- for (nreq = 0; wr; nreq++, wr = wr->next) {
4776		- if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
4777		- err = -ENOMEM;
4778		- *bad_wr = wr;
4779		- goto out;
4780		- }
4781		-
4782		- if (unlikely(wr->num_sge > qp->rq.max_gs)) {
4783		- err = -EINVAL;
4784		- *bad_wr = wr;
4785		- goto out;
4786		- }
4787		-
4788		- scat = get_recv_wqe(qp, ind);
4789		- if (qp->wq_sig)
4790		- scat++;
4791		-
4792		- for (i = 0; i < wr->num_sge; i++)
4793		- set_data_ptr_seg(scat + i, wr->sg_list + i);
4794		-
4795		- if (i < qp->rq.max_gs) {
4796		- scat[i].byte_count = 0;
4797		- scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
4798		- scat[i].addr = 0;
4799		- }
4800		-
4801		- if (qp->wq_sig) {
4802		- sig = (struct mlx5_rwqe_sig *)scat;
4803		- set_sig_seg(sig, (qp->rq.max_gs + 1) << 2);
4804		- }
4805		-
4806		- qp->rq.wrid[ind] = wr->wr_id;
4807		-
4808		- ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
4809		- }
4810		-
4811		-out:
4812		- if (likely(nreq)) {
4813		- qp->rq.head += nreq;
4814		-
4815		- /* Make sure that descriptors are written before
4816		- * doorbell record.
4817		- */
4818		- wmb();
4819		-
4820		- *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
4821		- }
4822		-
4823		- spin_unlock_irqrestore(&qp->rq.lock, flags);
4824		-
4825		- return err;
4826		-}
4827		-
4828		-int mlx5_ib_post_recv(struct ib_qp ibqp, const struct ib_recv_wr wr,
4829		- const struct ib_recv_wr **bad_wr)
4830		-{
4831		- return _mlx5_ib_post_recv(ibqp, wr, bad_wr, false);
4832	4409	}
4833	4410
4834	4411	static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state)
..	..	@@ -4856,50 +4433,34 @@
4856	4433	}
4857	4434	}
4858	4435
4859		-static int to_ib_qp_access_flags(int mlx5_flags)
4860		-{
4861		- int ib_flags = 0;
4862		-
4863		- if (mlx5_flags & MLX5_QP_BIT_RRE)
4864		- ib_flags \|= IB_ACCESS_REMOTE_READ;
4865		- if (mlx5_flags & MLX5_QP_BIT_RWE)
4866		- ib_flags \|= IB_ACCESS_REMOTE_WRITE;
4867		- if (mlx5_flags & MLX5_QP_BIT_RAE)
4868		- ib_flags \|= IB_ACCESS_REMOTE_ATOMIC;
4869		-
4870		- return ib_flags;
4871		-}
4872		-
4873	4436	static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev,
4874		- struct rdma_ah_attr *ah_attr,
4875		- struct mlx5_qp_path *path)
	4437	+ struct rdma_ah_attr ah_attr, void path)
4876	4438	{
	4439	+ int port = MLX5_GET(ads, path, vhca_port_num);
	4440	+ int static_rate;
4877	4441
4878	4442	memset(ah_attr, 0, sizeof(*ah_attr));
4879	4443
4880		- if (!path->port \|\| path->port > ibdev->num_ports)
	4444	+ if (!port \|\| port > ibdev->num_ports)
4881	4445	return;
4882	4446
4883		- ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, path->port);
	4447	+ ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, port);
4884	4448
4885		- rdma_ah_set_port_num(ah_attr, path->port);
4886		- rdma_ah_set_sl(ah_attr, path->dci_cfi_prio_sl & 0xf);
	4449	+ rdma_ah_set_port_num(ah_attr, port);
	4450	+ rdma_ah_set_sl(ah_attr, MLX5_GET(ads, path, sl));
4887	4451
4888		- rdma_ah_set_dlid(ah_attr, be16_to_cpu(path->rlid));
4889		- rdma_ah_set_path_bits(ah_attr, path->grh_mlid & 0x7f);
4890		- rdma_ah_set_static_rate(ah_attr,
4891		- path->static_rate ? path->static_rate - 5 : 0);
	4452	+ rdma_ah_set_dlid(ah_attr, MLX5_GET(ads, path, rlid));
	4453	+ rdma_ah_set_path_bits(ah_attr, MLX5_GET(ads, path, mlid));
4892	4454
4893		- if (path->grh_mlid & (1 << 7) \|\|
	4455	+ static_rate = MLX5_GET(ads, path, stat_rate);
	4456	+ rdma_ah_set_static_rate(ah_attr, mlx5_to_ib_rate_map(static_rate));
	4457	+ if (MLX5_GET(ads, path, grh) \|\|
4894	4458	ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
4895		- u32 tc_fl = be32_to_cpu(path->tclass_flowlabel);
4896		-
4897		- rdma_ah_set_grh(ah_attr, NULL,
4898		- tc_fl & 0xfffff,
4899		- path->mgid_index,
4900		- path->hop_limit,
4901		- (tc_fl >> 20) & 0xff);
4902		- rdma_ah_set_dgid_raw(ah_attr, path->rgid);
	4459	+ rdma_ah_set_grh(ah_attr, NULL, MLX5_GET(ads, path, flow_label),
	4460	+ MLX5_GET(ads, path, src_addr_index),
	4461	+ MLX5_GET(ads, path, hop_limit),
	4462	+ MLX5_GET(ads, path, tclass));
	4463	+ rdma_ah_set_dgid_raw(ah_attr, MLX5_ADDR_OF(ads, path, rgid_rip));
4903	4464	}
4904	4465	}
4905	4466
..	..	@@ -4956,7 +4517,7 @@
4956	4517	[MLX5_SQ_STATE_NA] = IB_QPS_RESET,
4957	4518	},
4958	4519	[MLX5_RQC_STATE_RDY] = {
4959		- [MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD,
	4520	+ [MLX5_SQC_STATE_RST] = MLX5_QP_STATE,
4960	4521	[MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4961	4522	[MLX5_SQC_STATE_ERR] = IB_QPS_SQE,
4962	4523	[MLX5_SQ_STATE_NA] = MLX5_QP_STATE,
..	..	@@ -4968,7 +4529,7 @@
4968	4529	[MLX5_SQ_STATE_NA] = IB_QPS_ERR,
4969	4530	},
4970	4531	[MLX5_RQ_STATE_NA] = {
4971		- [MLX5_SQC_STATE_RST] = IB_QPS_RESET,
	4532	+ [MLX5_SQC_STATE_RST] = MLX5_QP_STATE,
4972	4533	[MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4973	4534	[MLX5_SQC_STATE_ERR] = MLX5_QP_STATE,
4974	4535	[MLX5_SQ_STATE_NA] = MLX5_QP_STATE_BAD,
..	..	@@ -5021,61 +4582,58 @@
5021	4582	struct ib_qp_attr *qp_attr)
5022	4583	{
5023	4584	int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
5024		- struct mlx5_qp_context *context;
5025		- int mlx5_state;
	4585	+ void qpc, pri_path, *alt_path;
5026	4586	u32 *outb;
5027		- int err = 0;
	4587	+ int err;
5028	4588
5029	4589	outb = kzalloc(outlen, GFP_KERNEL);
5030	4590	if (!outb)
5031	4591	return -ENOMEM;
5032	4592
5033		- err = mlx5_core_qp_query(dev->mdev, &qp->trans_qp.base.mqp, outb,
5034		- outlen);
	4593	+ err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen);
5035	4594	if (err)
5036	4595	goto out;
5037	4596
5038		- /* FIXME: use MLX5_GET rather than mlx5_qp_context manual struct */
5039		- context = (struct mlx5_qp_context *)MLX5_ADDR_OF(query_qp_out, outb, qpc);
	4597	+ qpc = MLX5_ADDR_OF(query_qp_out, outb, qpc);
5040	4598
5041		- mlx5_state = be32_to_cpu(context->flags) >> 28;
	4599	+ qp->state = to_ib_qp_state(MLX5_GET(qpc, qpc, state));
	4600	+ if (MLX5_GET(qpc, qpc, state) == MLX5_QP_STATE_SQ_DRAINING)
	4601	+ qp_attr->sq_draining = 1;
5042	4602
5043		- qp->state = to_ib_qp_state(mlx5_state);
5044		- qp_attr->path_mtu = context->mtu_msgmax >> 5;
5045		- qp_attr->path_mig_state =
5046		- to_ib_mig_state((be32_to_cpu(context->flags) >> 11) & 0x3);
5047		- qp_attr->qkey = be32_to_cpu(context->qkey);
5048		- qp_attr->rq_psn = be32_to_cpu(context->rnr_nextrecvpsn) & 0xffffff;
5049		- qp_attr->sq_psn = be32_to_cpu(context->next_send_psn) & 0xffffff;
5050		- qp_attr->dest_qp_num = be32_to_cpu(context->log_pg_sz_remote_qpn) & 0xffffff;
5051		- qp_attr->qp_access_flags =
5052		- to_ib_qp_access_flags(be32_to_cpu(context->params2));
	4603	+ qp_attr->path_mtu = MLX5_GET(qpc, qpc, mtu);
	4604	+ qp_attr->path_mig_state = to_ib_mig_state(MLX5_GET(qpc, qpc, pm_state));
	4605	+ qp_attr->qkey = MLX5_GET(qpc, qpc, q_key);
	4606	+ qp_attr->rq_psn = MLX5_GET(qpc, qpc, next_rcv_psn);
	4607	+ qp_attr->sq_psn = MLX5_GET(qpc, qpc, next_send_psn);
	4608	+ qp_attr->dest_qp_num = MLX5_GET(qpc, qpc, remote_qpn);
	4609	+
	4610	+ if (MLX5_GET(qpc, qpc, rre))
	4611	+ qp_attr->qp_access_flags \|= IB_ACCESS_REMOTE_READ;
	4612	+ if (MLX5_GET(qpc, qpc, rwe))
	4613	+ qp_attr->qp_access_flags \|= IB_ACCESS_REMOTE_WRITE;
	4614	+ if (MLX5_GET(qpc, qpc, rae))
	4615	+ qp_attr->qp_access_flags \|= IB_ACCESS_REMOTE_ATOMIC;
	4616	+
	4617	+ qp_attr->max_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_sra_max);
	4618	+ qp_attr->max_dest_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_rra_max);
	4619	+ qp_attr->min_rnr_timer = MLX5_GET(qpc, qpc, min_rnr_nak);
	4620	+ qp_attr->retry_cnt = MLX5_GET(qpc, qpc, retry_count);
	4621	+ qp_attr->rnr_retry = MLX5_GET(qpc, qpc, rnr_retry);
	4622	+
	4623	+ pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
	4624	+ alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);
5053	4625
5054	4626	if (qp->ibqp.qp_type == IB_QPT_RC \|\| qp->ibqp.qp_type == IB_QPT_UC) {
5055		- to_rdma_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path);
5056		- to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path);
5057		- qp_attr->alt_pkey_index =
5058		- be16_to_cpu(context->alt_path.pkey_index);
5059		- qp_attr->alt_port_num =
5060		- rdma_ah_get_port_num(&qp_attr->alt_ah_attr);
	4627	+ to_rdma_ah_attr(dev, &qp_attr->ah_attr, pri_path);
	4628	+ to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, alt_path);
	4629	+ qp_attr->alt_pkey_index = MLX5_GET(ads, alt_path, pkey_index);
	4630	+ qp_attr->alt_port_num = MLX5_GET(ads, alt_path, vhca_port_num);
5061	4631	}
5062	4632
5063		- qp_attr->pkey_index = be16_to_cpu(context->pri_path.pkey_index);
5064		- qp_attr->port_num = context->pri_path.port;
5065		-
5066		- /* qp_attr->en_sqd_async_notify is only applicable in modify qp */
5067		- qp_attr->sq_draining = mlx5_state == MLX5_QP_STATE_SQ_DRAINING;
5068		-
5069		- qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context->params1) >> 21) & 0x7);
5070		-
5071		- qp_attr->max_dest_rd_atomic =
5072		- 1 << ((be32_to_cpu(context->params2) >> 21) & 0x7);
5073		- qp_attr->min_rnr_timer =
5074		- (be32_to_cpu(context->rnr_nextrecvpsn) >> 24) & 0x1f;
5075		- qp_attr->timeout = context->pri_path.ackto_lt >> 3;
5076		- qp_attr->retry_cnt = (be32_to_cpu(context->params1) >> 16) & 0x7;
5077		- qp_attr->rnr_retry = (be32_to_cpu(context->params1) >> 13) & 0x7;
5078		- qp_attr->alt_timeout = context->alt_path.ackto_lt >> 3;
	4633	+ qp_attr->pkey_index = MLX5_GET(ads, pri_path, pkey_index);
	4634	+ qp_attr->port_num = MLX5_GET(ads, pri_path, vhca_port_num);
	4635	+ qp_attr->timeout = MLX5_GET(ads, pri_path, ack_timeout);
	4636	+ qp_attr->alt_timeout = MLX5_GET(ads, alt_path, ack_timeout);
5079	4637
5080	4638	out:
5081	4639	kfree(outb);
..	..	@@ -5109,7 +4667,7 @@
5109	4667	if (!out)
5110	4668	return -ENOMEM;
5111	4669
5112		- err = mlx5_core_dct_query(dev->mdev, dct, out, outlen);
	4670	+ err = mlx5_core_dct_query(dev, dct, out, outlen);
5113	4671	if (err)
5114	4672	goto out;
5115	4673
..	..	@@ -5166,14 +4724,14 @@
5166	4724	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
5167	4725	memset(qp_attr, 0, sizeof(*qp_attr));
5168	4726
5169		- if (unlikely(qp->qp_sub_type == MLX5_IB_QPT_DCT))
	4727	+ if (unlikely(qp->type == MLX5_IB_QPT_DCT))
5170	4728	return mlx5_ib_dct_query_qp(dev, qp, qp_attr,
5171	4729	qp_attr_mask, qp_init_attr);
5172	4730
5173	4731	mutex_lock(&qp->mutex);
5174	4732
5175	4733	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET \|\|
5176		- qp->flags & MLX5_IB_QP_UNDERLAY) {
	4734	+ qp->flags & IB_QP_CREATE_SOURCE_QPN) {
5177	4735	err = query_raw_packet_qp_state(dev, qp, &raw_packet_qp_state);
5178	4736	if (err)
5179	4737	goto out;
..	..	@@ -5207,18 +4765,7 @@
5207	4765
5208	4766	qp_init_attr->cap = qp_attr->cap;
5209	4767
5210		- qp_init_attr->create_flags = 0;
5211		- if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
5212		- qp_init_attr->create_flags \|= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
5213		-
5214		- if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
5215		- qp_init_attr->create_flags \|= IB_QP_CREATE_CROSS_CHANNEL;
5216		- if (qp->flags & MLX5_IB_QP_MANAGED_SEND)
5217		- qp_init_attr->create_flags \|= IB_QP_CREATE_MANAGED_SEND;
5218		- if (qp->flags & MLX5_IB_QP_MANAGED_RECV)
5219		- qp_init_attr->create_flags \|= IB_QP_CREATE_MANAGED_RECV;
5220		- if (qp->flags & MLX5_IB_QP_SQPN_QP1)
5221		- qp_init_attr->create_flags \|= mlx5_ib_create_qp_sqpn_qp1();
	4768	+ qp_init_attr->create_flags = qp->flags;
5222	4769
5223	4770	qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ?
5224	4771	IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
..	..	@@ -5228,42 +4775,23 @@
5228	4775	return err;
5229	4776	}
5230	4777
5231		-struct ib_xrcd mlx5_ib_alloc_xrcd(struct ib_device ibdev,
5232		- struct ib_ucontext *context,
5233		- struct ib_udata *udata)
	4778	+int mlx5_ib_alloc_xrcd(struct ib_xrcd ibxrcd, struct ib_udata udata)
5234	4779	{
5235		- struct mlx5_ib_dev *dev = to_mdev(ibdev);
5236		- struct mlx5_ib_xrcd *xrcd;
5237		- int err;
	4780	+ struct mlx5_ib_dev *dev = to_mdev(ibxrcd->device);
	4781	+ struct mlx5_ib_xrcd *xrcd = to_mxrcd(ibxrcd);
5238	4782
5239	4783	if (!MLX5_CAP_GEN(dev->mdev, xrc))
5240		- return ERR_PTR(-ENOSYS);
	4784	+ return -EOPNOTSUPP;
5241	4785
5242		- xrcd = kmalloc(sizeof(*xrcd), GFP_KERNEL);
5243		- if (!xrcd)
5244		- return ERR_PTR(-ENOMEM);
5245		-
5246		- err = mlx5_core_xrcd_alloc(dev->mdev, &xrcd->xrcdn);
5247		- if (err) {
5248		- kfree(xrcd);
5249		- return ERR_PTR(-ENOMEM);
5250		- }
5251		-
5252		- return &xrcd->ibxrcd;
	4786	+ return mlx5_cmd_xrcd_alloc(dev->mdev, &xrcd->xrcdn, 0);
5253	4787	}
5254	4788
5255		-int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd)
	4789	+int mlx5_ib_dealloc_xrcd(struct ib_xrcd xrcd, struct ib_udata udata)
5256	4790	{
5257	4791	struct mlx5_ib_dev *dev = to_mdev(xrcd->device);
5258	4792	u32 xrcdn = to_mxrcd(xrcd)->xrcdn;
5259		- int err;
5260	4793
5261		- err = mlx5_core_xrcd_dealloc(dev->mdev, xrcdn);
5262		- if (err)
5263		- mlx5_ib_warn(dev, "failed to dealloc xrcdn 0x%x\n", xrcdn);
5264		-
5265		- kfree(xrcd);
5266		- return 0;
	4794	+ return mlx5_cmd_xrcd_dealloc(dev->mdev, xrcdn, 0);
5267	4795	}
5268	4796
5269	4797	static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
..	..	@@ -5296,7 +4824,7 @@
5296	4824	if (dev->delay_drop.activate)
5297	4825	goto out;
5298	4826
5299		- err = mlx5_core_set_delay_drop(dev->mdev, dev->delay_drop.timeout);
	4827	+ err = mlx5_core_set_delay_drop(dev, dev->delay_drop.timeout);
5300	4828	if (err)
5301	4829	goto out;
5302	4830
..	..	@@ -5328,6 +4856,7 @@
5328	4856	if (!in)
5329	4857	return -ENOMEM;
5330	4858
	4859	+ MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
5331	4860	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
5332	4861	MLX5_SET(rqc, rqc, mem_rq_type,
5333	4862	MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
..	..	@@ -5350,12 +4879,21 @@
5350	4879	}
5351	4880	MLX5_SET(wq, wq, log_wq_stride, rwq->log_rq_stride);
5352	4881	if (rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ) {
	4882	+ /*
	4883	+ * In Firmware number of strides in each WQE is:
	4884	+ * "512 * 2^single_wqe_log_num_of_strides"
	4885	+ * Values 3 to 8 are accepted as 10 to 15, 9 to 18 are
	4886	+ * accepted as 0 to 9
	4887	+ */
	4888	+ static const u8 fw_map[] = { 10, 11, 12, 13, 14, 15, 0, 1,
	4889	+ 2, 3, 4, 5, 6, 7, 8, 9 };
5353	4890	MLX5_SET(wq, wq, two_byte_shift_en, rwq->two_byte_shift_en);
5354	4891	MLX5_SET(wq, wq, log_wqe_stride_size,
5355	4892	rwq->single_stride_log_num_of_bytes -
5356	4893	MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES);
5357		- MLX5_SET(wq, wq, log_wqe_num_of_strides, rwq->log_num_strides -
5358		- MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES);
	4894	+ MLX5_SET(wq, wq, log_wqe_num_of_strides,
	4895	+ fw_map[rwq->log_num_strides -
	4896	+ MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES]);
5359	4897	}
5360	4898	MLX5_SET(wq, wq, log_wq_sz, rwq->log_rq_size);
5361	4899	MLX5_SET(wq, wq, pd, to_mpd(pd)->pdn);
..	..	@@ -5392,13 +4930,13 @@
5392	4930	}
5393	4931	rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
5394	4932	mlx5_ib_populate_pas(dev, rwq->umem, rwq->page_shift, rq_pas0, 0);
5395		- err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rwq->core_qp);
	4933	+ err = mlx5_core_create_rq_tracked(dev, in, inlen, &rwq->core_qp);
5396	4934	if (!err && init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
5397	4935	err = set_delay_drop(dev);
5398	4936	if (err) {
5399	4937	mlx5_ib_warn(dev, "Failed to enable delay drop err=%d\n",
5400	4938	err);
5401		- mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
	4939	+ mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
5402	4940	} else {
5403	4941	rwq->create_flags \|= MLX5_IB_WQ_FLAGS_DELAY_DROP;
5404	4942	}
..	..	@@ -5430,6 +4968,19 @@
5430	4968	return 0;
5431	4969	}
5432	4970
	4971	+static bool log_of_strides_valid(struct mlx5_ib_dev *dev, u32 log_num_strides)
	4972	+{
	4973	+ if ((log_num_strides > MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES) \|\|
	4974	+ (log_num_strides < MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES))
	4975	+ return false;
	4976	+
	4977	+ if (!MLX5_CAP_GEN(dev->mdev, ext_stride_num_range) &&
	4978	+ (log_num_strides < MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES))
	4979	+ return false;
	4980	+
	4981	+ return true;
	4982	+}
	4983	+
5433	4984	static int prepare_user_rq(struct ib_pd *pd,
5434	4985	struct ib_wq_init_attr *init_attr,
5435	4986	struct ib_udata *udata,
..	..	@@ -5440,8 +4991,8 @@
5440	4991	int err;
5441	4992	size_t required_cmd_sz;
5442	4993
5443		- required_cmd_sz = offsetof(typeof(ucmd), single_stride_log_num_of_bytes)
5444		- + sizeof(ucmd.single_stride_log_num_of_bytes);
	4994	+ required_cmd_sz = offsetofend(struct mlx5_ib_create_wq,
	4995	+ single_stride_log_num_of_bytes);
5445	4996	if (udata->inlen < required_cmd_sz) {
5446	4997	mlx5_ib_dbg(dev, "invalid inlen\n");
5447	4998	return -EINVAL;
..	..	@@ -5477,14 +5028,16 @@
5477	5028	MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES);
5478	5029	return -EINVAL;
5479	5030	}
5480		- if ((ucmd.single_wqe_log_num_of_strides >
5481		- MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES) \|\|
5482		- (ucmd.single_wqe_log_num_of_strides <
5483		- MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES)) {
5484		- mlx5_ib_dbg(dev, "Invalid log num strides (%u. Range is %u - %u)\n",
5485		- ucmd.single_wqe_log_num_of_strides,
5486		- MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES,
5487		- MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES);
	5031	+ if (!log_of_strides_valid(dev,
	5032	+ ucmd.single_wqe_log_num_of_strides)) {
	5033	+ mlx5_ib_dbg(
	5034	+ dev,
	5035	+ "Invalid log num strides (%u. Range is %u - %u)\n",
	5036	+ ucmd.single_wqe_log_num_of_strides,
	5037	+ MLX5_CAP_GEN(dev->mdev, ext_stride_num_range) ?
	5038	+ MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES :
	5039	+ MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES,
	5040	+ MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES);
5488	5041	return -EINVAL;
5489	5042	}
5490	5043	rwq->single_stride_log_num_of_bytes =
..	..	@@ -5500,11 +5053,10 @@
5500	5053	return err;
5501	5054	}
5502	5055
5503		- err = create_user_rq(dev, pd, rwq, &ucmd);
	5056	+ err = create_user_rq(dev, pd, udata, rwq, &ucmd);
5504	5057	if (err) {
5505	5058	mlx5_ib_dbg(dev, "err %d\n", err);
5506		- if (err)
5507		- return err;
	5059	+ return err;
5508	5060	}
5509	5061
5510	5062	rwq->user_index = ucmd.user_index;
..	..	@@ -5524,7 +5076,7 @@
5524	5076	if (!udata)
5525	5077	return ERR_PTR(-ENOSYS);
5526	5078
5527		- min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
	5079	+ min_resp_len = offsetofend(struct mlx5_ib_create_wq_resp, reserved);
5528	5080	if (udata->outlen && udata->outlen < min_resp_len)
5529	5081	return ERR_PTR(-EINVAL);
5530	5082
..	..	@@ -5554,8 +5106,8 @@
5554	5106	rwq->ibwq.wq_num = rwq->core_qp.qpn;
5555	5107	rwq->ibwq.state = IB_WQS_RESET;
5556	5108	if (udata->outlen) {
5557		- resp.response_length = offsetof(typeof(resp), response_length) +
5558		- sizeof(resp.response_length);
	5109	+ resp.response_length = offsetofend(
	5110	+ struct mlx5_ib_create_wq_resp, response_length);
5559	5111	err = ib_copy_to_udata(udata, &resp, resp.response_length);
5560	5112	if (err)
5561	5113	goto err_copy;
..	..	@@ -5566,32 +5118,35 @@
5566	5118	return &rwq->ibwq;
5567	5119
5568	5120	err_copy:
5569		- mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
	5121	+ mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
5570	5122	err_user_rq:
5571		- destroy_user_rq(dev, pd, rwq);
	5123	+ destroy_user_rq(dev, pd, rwq, udata);
5572	5124	err:
5573	5125	kfree(rwq);
5574	5126	return ERR_PTR(err);
5575	5127	}
5576	5128
5577		-int mlx5_ib_destroy_wq(struct ib_wq *wq)
	5129	+int mlx5_ib_destroy_wq(struct ib_wq wq, struct ib_udata udata)
5578	5130	{
5579	5131	struct mlx5_ib_dev *dev = to_mdev(wq->device);
5580	5132	struct mlx5_ib_rwq *rwq = to_mrwq(wq);
	5133	+ int ret;
5581	5134
5582		- mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
5583		- destroy_user_rq(dev, wq->pd, rwq);
	5135	+ ret = mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
	5136	+ if (ret)
	5137	+ return ret;
	5138	+ destroy_user_rq(dev, wq->pd, rwq, udata);
5584	5139	kfree(rwq);
5585		-
5586	5140	return 0;
5587	5141	}
5588	5142
5589		-struct ib_rwq_ind_table mlx5_ib_create_rwq_ind_table(struct ib_device device,
5590		- struct ib_rwq_ind_table_init_attr *init_attr,
5591		- struct ib_udata *udata)
	5143	+int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
	5144	+ struct ib_rwq_ind_table_init_attr *init_attr,
	5145	+ struct ib_udata *udata)
5592	5146	{
5593		- struct mlx5_ib_dev *dev = to_mdev(device);
5594		- struct mlx5_ib_rwq_ind_table *rwq_ind_tbl;
	5147	+ struct mlx5_ib_rwq_ind_table *rwq_ind_tbl =
	5148	+ to_mrwq_ind_table(ib_rwq_ind_table);
	5149	+ struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_table->device);
5595	5150	int sz = 1 << init_attr->log_ind_tbl_size;
5596	5151	struct mlx5_ib_create_rwq_ind_tbl_resp resp = {};
5597	5152	size_t min_resp_len;
..	..	@@ -5604,30 +5159,25 @@
5604	5159	if (udata->inlen > 0 &&
5605	5160	!ib_is_udata_cleared(udata, 0,
5606	5161	udata->inlen))
5607		- return ERR_PTR(-EOPNOTSUPP);
	5162	+ return -EOPNOTSUPP;
5608	5163
5609	5164	if (init_attr->log_ind_tbl_size >
5610	5165	MLX5_CAP_GEN(dev->mdev, log_max_rqt_size)) {
5611	5166	mlx5_ib_dbg(dev, "log_ind_tbl_size = %d is bigger than supported = %d\n",
5612	5167	init_attr->log_ind_tbl_size,
5613	5168	MLX5_CAP_GEN(dev->mdev, log_max_rqt_size));
5614		- return ERR_PTR(-EINVAL);
	5169	+ return -EINVAL;
5615	5170	}
5616	5171
5617		- min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
	5172	+ min_resp_len =
	5173	+ offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp, reserved);
5618	5174	if (udata->outlen && udata->outlen < min_resp_len)
5619		- return ERR_PTR(-EINVAL);
5620		-
5621		- rwq_ind_tbl = kzalloc(sizeof(*rwq_ind_tbl), GFP_KERNEL);
5622		- if (!rwq_ind_tbl)
5623		- return ERR_PTR(-ENOMEM);
	5175	+ return -EINVAL;
5624	5176
5625	5177	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
5626	5178	in = kvzalloc(inlen, GFP_KERNEL);
5627		- if (!in) {
5628		- err = -ENOMEM;
5629		- goto err;
5630		- }
	5179	+ if (!in)
	5180	+ return -ENOMEM;
5631	5181
5632	5182	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
5633	5183
..	..	@@ -5637,28 +5187,29 @@
5637	5187	for (i = 0; i < sz; i++)
5638	5188	MLX5_SET(rqtc, rqtc, rq_num[i], init_attr->ind_tbl[i]->wq_num);
5639	5189
	5190	+ rwq_ind_tbl->uid = to_mpd(init_attr->ind_tbl[0]->pd)->uid;
	5191	+ MLX5_SET(create_rqt_in, in, uid, rwq_ind_tbl->uid);
	5192	+
5640	5193	err = mlx5_core_create_rqt(dev->mdev, in, inlen, &rwq_ind_tbl->rqtn);
5641	5194	kvfree(in);
5642		-
5643	5195	if (err)
5644		- goto err;
	5196	+ return err;
5645	5197
5646	5198	rwq_ind_tbl->ib_rwq_ind_tbl.ind_tbl_num = rwq_ind_tbl->rqtn;
5647	5199	if (udata->outlen) {
5648		- resp.response_length = offsetof(typeof(resp), response_length) +
5649		- sizeof(resp.response_length);
	5200	+ resp.response_length =
	5201	+ offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp,
	5202	+ response_length);
5650	5203	err = ib_copy_to_udata(udata, &resp, resp.response_length);
5651	5204	if (err)
5652	5205	goto err_copy;
5653	5206	}
5654	5207
5655		- return &rwq_ind_tbl->ib_rwq_ind_tbl;
	5208	+ return 0;
5656	5209
5657	5210	err_copy:
5658		- mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn);
5659		-err:
5660		- kfree(rwq_ind_tbl);
5661		- return ERR_PTR(err);
	5211	+ mlx5_cmd_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid);
	5212	+ return err;
5662	5213	}
5663	5214
5664	5215	int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
..	..	@@ -5666,10 +5217,7 @@
5666	5217	struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = to_mrwq_ind_table(ib_rwq_ind_tbl);
5667	5218	struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_tbl->device);
5668	5219
5669		- mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn);
5670		-
5671		- kfree(rwq_ind_tbl);
5672		- return 0;
	5220	+ return mlx5_cmd_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid);
5673	5221	}
5674	5222
5675	5223	int mlx5_ib_modify_wq(struct ib_wq wq, struct ib_wq_attr wq_attr,
..	..	@@ -5686,7 +5234,7 @@
5686	5234	void *rqc;
5687	5235	void *in;
5688	5236
5689		- required_cmd_sz = offsetof(typeof(ucmd), reserved) + sizeof(ucmd.reserved);
	5237	+ required_cmd_sz = offsetofend(struct mlx5_ib_modify_wq, reserved);
5690	5238	if (udata->inlen < required_cmd_sz)
5691	5239	return -EINVAL;
5692	5240
..	..	@@ -5708,15 +5256,14 @@
5708	5256
5709	5257	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
5710	5258
5711		- curr_wq_state = (wq_attr_mask & IB_WQ_CUR_STATE) ?
5712		- wq_attr->curr_wq_state : wq->state;
5713		- wq_state = (wq_attr_mask & IB_WQ_STATE) ?
5714		- wq_attr->wq_state : curr_wq_state;
	5259	+ curr_wq_state = wq_attr->curr_wq_state;
	5260	+ wq_state = wq_attr->wq_state;
5715	5261	if (curr_wq_state == IB_WQS_ERR)
5716	5262	curr_wq_state = MLX5_RQC_STATE_ERR;
5717	5263	if (wq_state == IB_WQS_ERR)
5718	5264	wq_state = MLX5_RQC_STATE_ERR;
5719	5265	MLX5_SET(modify_rq_in, in, rq_state, curr_wq_state);
	5266	+ MLX5_SET(modify_rq_in, in, uid, to_mpd(wq->pd)->uid);
5720	5267	MLX5_SET(rqc, rqc, state, wq_state);
5721	5268
5722	5269	if (wq_attr_mask & IB_WQ_FLAGS) {
..	..	@@ -5742,17 +5289,20 @@
5742	5289	}
5743	5290
5744	5291	if (curr_wq_state == IB_WQS_RESET && wq_state == IB_WQS_RDY) {
	5292	+ u16 set_id;
	5293	+
	5294	+ set_id = mlx5_ib_get_counters_id(dev, 0);
5745	5295	if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
5746	5296	MLX5_SET64(modify_rq_in, in, modify_bitmask,
5747	5297	MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
5748		- MLX5_SET(rqc, rqc, counter_set_id,
5749		- dev->port->cnts.set_id);
	5298	+ MLX5_SET(rqc, rqc, counter_set_id, set_id);
5750	5299	} else
5751		- pr_info_once("%s: Receive WQ counters are not supported on current FW\n",
5752		- dev->ib_dev.name);
	5300	+ dev_info_once(
	5301	+ &dev->ib_dev.dev,
	5302	+ "Receive WQ counters are not supported on current FW\n");
5753	5303	}
5754	5304
5755		- err = mlx5_core_modify_rq(dev->mdev, rwq->core_qp.qpn, in, inlen);
	5305	+ err = mlx5_core_modify_rq(dev->mdev, rwq->core_qp.qpn, in);
5756	5306	if (!err)
5757	5307	rwq->ibwq.state = (wq_state == MLX5_RQC_STATE_ERR) ? IB_WQS_ERR : wq_state;
5758	5308
..	..	@@ -5819,7 +5369,7 @@
5819	5369	/* Run the CQ handler - this makes sure that the drain WR will
5820	5370	* be processed if wasn't processed yet.
5821	5371	*/
5822		- mcq->mcq.comp(&mcq->mcq);
	5372	+ mcq->mcq.comp(&mcq->mcq, NULL);
5823	5373	}
5824	5374
5825	5375	wait_for_completion(&sdrain->done);
..	..	@@ -5851,7 +5401,7 @@
5851	5401	sdrain.cqe.done = mlx5_ib_drain_qp_done;
5852	5402	init_completion(&sdrain.done);
5853	5403
5854		- ret = _mlx5_ib_post_send(qp, &swr.wr, &bad_swr, true);
	5404	+ ret = mlx5_ib_post_send_drain(qp, &swr.wr, &bad_swr);
5855	5405	if (ret) {
5856	5406	WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
5857	5407	return;
..	..	@@ -5881,7 +5431,7 @@
5881	5431	rdrain.cqe.done = mlx5_ib_drain_qp_done;
5882	5432	init_completion(&rdrain.done);
5883	5433
5884		- ret = _mlx5_ib_post_recv(qp, &rwr, &bad_rwr, true);
	5434	+ ret = mlx5_ib_post_recv_drain(qp, &rwr, &bad_rwr);
5885	5435	if (ret) {
5886	5436	WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
5887	5437	return;
..	..	@@ -5889,3 +5439,40 @@
5889	5439
5890	5440	handle_drain_completion(cq, &rdrain, dev);
5891	5441	}
	5442	+
	5443	+/**
	5444	+ * Bind a qp to a counter. If @counter is NULL then bind the qp to
	5445	+ * the default counter
	5446	+ */
	5447	+int mlx5_ib_qp_set_counter(struct ib_qp qp, struct rdma_counter counter)
	5448	+{
	5449	+ struct mlx5_ib_dev *dev = to_mdev(qp->device);
	5450	+ struct mlx5_ib_qp *mqp = to_mqp(qp);
	5451	+ int err = 0;
	5452	+
	5453	+ mutex_lock(&mqp->mutex);
	5454	+ if (mqp->state == IB_QPS_RESET) {
	5455	+ qp->counter = counter;
	5456	+ goto out;
	5457	+ }
	5458	+
	5459	+ if (!MLX5_CAP_GEN(dev->mdev, rts2rts_qp_counters_set_id)) {
	5460	+ err = -EOPNOTSUPP;
	5461	+ goto out;
	5462	+ }
	5463	+
	5464	+ if (mqp->state == IB_QPS_RTS) {
	5465	+ err = __mlx5_ib_qp_set_counter(qp, counter);
	5466	+ if (!err)
	5467	+ qp->counter = counter;
	5468	+
	5469	+ goto out;
	5470	+ }
	5471	+
	5472	+ mqp->counter_pending = 1;
	5473	+ qp->counter = counter;
	5474	+
	5475	+out:
	5476	+ mutex_unlock(&mqp->mutex);
	5477	+ return err;
	5478	+}