forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/drivers/infiniband/hw/mlx5/srq.c
....@@ -1,50 +1,19 @@
1
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
12 /*
2
- * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3
- *
4
- * This software is available to you under a choice of one of two
5
- * licenses.  You may choose to be licensed under the terms of the GNU
6
- * General Public License (GPL) Version 2, available from the file
7
- * COPYING in the main directory of this source tree, or the
8
- * OpenIB.org BSD license below:
9
- *
10
- *     Redistribution and use in source and binary forms, with or
11
- *     without modification, are permitted provided that the following
12
- *     conditions are met:
13
- *
14
- *      - Redistributions of source code must retain the above
15
- *        copyright notice, this list of conditions and the following
16
- *        disclaimer.
17
- *
18
- *      - Redistributions in binary form must reproduce the above
19
- *        copyright notice, this list of conditions and the following
20
- *        disclaimer in the documentation and/or other materials
21
- *        provided with the distribution.
22
- *
23
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30
- * SOFTWARE.
3
+ * Copyright (c) 2013-2018, Mellanox Technologies inc.  All rights reserved.
314  */
325 
336 #include <linux/module.h>
347 #include <linux/mlx5/qp.h>
35
-#include <linux/mlx5/srq.h>
368 #include <linux/slab.h>
379 #include <rdma/ib_umem.h>
3810 #include <rdma/ib_user_verbs.h>
39
-
4011 #include "mlx5_ib.h"
41
-
42
-/* not supported currently */
43
-static int srq_signature;
12
+#include "srq.h"
4413 
4514 static void *get_wqe(struct mlx5_ib_srq *srq, int n)
4615 {
47
-	return mlx5_buf_offset(&srq->buf, n << srq->msrq.wqe_shift);
16
+	return mlx5_frag_buf_get_wqe(&srq->fbc, n);
4817 }
4918 
5019 static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, enum mlx5_event type)
....@@ -78,6 +47,8 @@
7847 {
7948 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
8049 	struct mlx5_ib_create_srq ucmd = {};
50
+	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
51
+		udata, struct mlx5_ib_ucontext, ibucontext);
8152 	size_t ucmdlen;
8253 	int err;
8354 	int npages;
....@@ -102,16 +73,14 @@
10273 		return -EINVAL;
10374 
10475 	if (in->type != IB_SRQT_BASIC) {
105
-		err = get_srq_user_index(to_mucontext(pd->uobject->context),
106
-					 &ucmd, udata->inlen, &uidx);
76
+		err = get_srq_user_index(ucontext, &ucmd, udata->inlen, &uidx);
10777 		if (err)
10878 			return err;
10979 	}
11080 
11181 	srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE);
11282 
113
-	srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, buf_size,
114
-				0, 0);
83
+	srq->umem = ib_umem_get(pd->device, ucmd.buf_addr, buf_size, 0);
11584 	if (IS_ERR(srq->umem)) {
11685 		mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size);
11786 		err = PTR_ERR(srq->umem);
....@@ -135,8 +104,7 @@
135104 
136105 	mlx5_ib_populate_pas(dev, srq->umem, page_shift, in->pas, 0);
137106 
138
-	err = mlx5_ib_db_map_user(to_mucontext(pd->uobject->context),
139
-				  ucmd.db_addr, &srq->db);
107
+	err = mlx5_ib_db_map_user(ucontext, udata, ucmd.db_addr, &srq->db);
140108 	if (err) {
141109 		mlx5_ib_dbg(dev, "map doorbell failed\n");
142110 		goto err_in;
....@@ -144,6 +112,7 @@
144112 
145113 	in->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
146114 	in->page_offset = offset;
115
+	in->uid = (in->type != IB_SRQT_XRC) ?  to_mpd(pd)->uid : 0;
147116 	if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
148117 	    in->type != IB_SRQT_BASIC)
149118 		in->user_index = uidx;
....@@ -172,11 +141,15 @@
172141 		return err;
173142 	}
174143 
175
-	if (mlx5_buf_alloc(dev->mdev, buf_size, &srq->buf)) {
144
+	if (mlx5_frag_buf_alloc_node(dev->mdev, buf_size, &srq->buf,
145
+				     dev->mdev->priv.numa_node)) {
176146 		mlx5_ib_dbg(dev, "buf alloc failed\n");
177147 		err = -ENOMEM;
178148 		goto err_db;
179149 	}
150
+
151
+	mlx5_init_fbc(srq->buf.frags, srq->msrq.wqe_shift, ilog2(srq->msrq.max),
152
+		      &srq->fbc);
180153 
181154 	srq->head    = 0;
182155 	srq->tail    = srq->msrq.max - 1;
....@@ -194,14 +167,14 @@
194167 		err = -ENOMEM;
195168 		goto err_buf;
196169 	}
197
-	mlx5_fill_page_array(&srq->buf, in->pas);
170
+	mlx5_fill_page_frag_array(&srq->buf, in->pas);
198171 
199172 	srq->wrid = kvmalloc_array(srq->msrq.max, sizeof(u64), GFP_KERNEL);
200173 	if (!srq->wrid) {
201174 		err = -ENOMEM;
202175 		goto err_in;
203176 	}
204
-	srq->wq_sig = !!srq_signature;
177
+	srq->wq_sig = 0;
205178 
206179 	in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT;
207180 	if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
....@@ -214,16 +187,22 @@
214187 	kvfree(in->pas);
215188 
216189 err_buf:
217
-	mlx5_buf_free(dev->mdev, &srq->buf);
190
+	mlx5_frag_buf_free(dev->mdev, &srq->buf);
218191 
219192 err_db:
220193 	mlx5_db_free(dev->mdev, &srq->db);
221194 	return err;
222195 }
223196 
224
-static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq)
197
+static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
198
+			     struct ib_udata *udata)
225199 {
226
-	mlx5_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db);
200
+	mlx5_ib_db_unmap_user(
201
+		rdma_udata_to_drv_context(
202
+			udata,
203
+			struct mlx5_ib_ucontext,
204
+			ibucontext),
205
+		&srq->db);
227206 	ib_umem_release(srq->umem);
228207 }
229208 
....@@ -231,20 +210,20 @@
231210 static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq)
232211 {
233212 	kvfree(srq->wrid);
234
-	mlx5_buf_free(dev->mdev, &srq->buf);
213
+	mlx5_frag_buf_free(dev->mdev, &srq->buf);
235214 	mlx5_db_free(dev->mdev, &srq->db);
236215 }
237216 
238
-struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
239
-				  struct ib_srq_init_attr *init_attr,
240
-				  struct ib_udata *udata)
217
+int mlx5_ib_create_srq(struct ib_srq *ib_srq,
218
+		       struct ib_srq_init_attr *init_attr,
219
+		       struct ib_udata *udata)
241220 {
242
-	struct mlx5_ib_dev *dev = to_mdev(pd->device);
243
-	struct mlx5_ib_srq *srq;
221
+	struct mlx5_ib_dev *dev = to_mdev(ib_srq->device);
222
+	struct mlx5_ib_srq *srq = to_msrq(ib_srq);
244223 	size_t desc_size;
245224 	size_t buf_size;
246225 	int err;
247
-	struct mlx5_srq_attr in = {0};
226
+	struct mlx5_srq_attr in = {};
248227 	__u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
249228 
250229 	/* Sanity check SRQ size before proceeding */
....@@ -252,12 +231,8 @@
252231 		mlx5_ib_dbg(dev, "max_wr %d, cap %d\n",
253232 			    init_attr->attr.max_wr,
254233 			    max_srq_wqes);
255
-		return ERR_PTR(-EINVAL);
234
+		return -EINVAL;
256235 	}
257
-
258
-	srq = kmalloc(sizeof(*srq), GFP_KERNEL);
259
-	if (!srq)
260
-		return ERR_PTR(-ENOMEM);
261236 
262237 	mutex_init(&srq->mutex);
263238 	spin_lock_init(&srq->lock);
....@@ -266,35 +241,32 @@
266241 
267242 	desc_size = sizeof(struct mlx5_wqe_srq_next_seg) +
268243 		    srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg);
269
-	if (desc_size == 0 || srq->msrq.max_gs > desc_size) {
270
-		err = -EINVAL;
271
-		goto err_srq;
272
-	}
244
+	if (desc_size == 0 || srq->msrq.max_gs > desc_size)
245
+		return -EINVAL;
246
+
273247 	desc_size = roundup_pow_of_two(desc_size);
274248 	desc_size = max_t(size_t, 32, desc_size);
275
-	if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg)) {
276
-		err = -EINVAL;
277
-		goto err_srq;
278
-	}
249
+	if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg))
250
+		return -EINVAL;
251
+
279252 	srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) /
280253 		sizeof(struct mlx5_wqe_data_seg);
281254 	srq->msrq.wqe_shift = ilog2(desc_size);
282255 	buf_size = srq->msrq.max * desc_size;
283
-	if (buf_size < desc_size) {
284
-		err = -EINVAL;
285
-		goto err_srq;
286
-	}
256
+	if (buf_size < desc_size)
257
+		return -EINVAL;
258
+
287259 	in.type = init_attr->srq_type;
288260 
289
-	if (pd->uobject)
290
-		err = create_srq_user(pd, srq, &in, udata, buf_size);
261
+	if (udata)
262
+		err = create_srq_user(ib_srq->pd, srq, &in, udata, buf_size);
291263 	else
292264 		err = create_srq_kernel(dev, srq, &in, buf_size);
293265 
294266 	if (err) {
295267 		mlx5_ib_warn(dev, "create srq %s failed, err %d\n",
296
-			     pd->uobject ? "user" : "kernel", err);
297
-		goto err_srq;
268
+			     udata ? "user" : "kernel", err);
269
+		return err;
298270 	}
299271 
300272 	in.log_size = ilog2(srq->msrq.max);
....@@ -302,10 +274,10 @@
302274 	if (srq->wq_sig)
303275 		in.flags |= MLX5_SRQ_FLAG_WQ_SIG;
304276 
305
-	if (init_attr->srq_type == IB_SRQT_XRC)
277
+	if (init_attr->srq_type == IB_SRQT_XRC && init_attr->ext.xrc.xrcd)
306278 		in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn;
307279 	else
308
-		in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn;
280
+		in.xrcd = dev->devr.xrcdn0;
309281 
310282 	if (init_attr->srq_type == IB_SRQT_TM) {
311283 		in.tm_log_list_size =
....@@ -324,9 +296,9 @@
324296 	else
325297 		in.cqn = to_mcq(dev->devr.c0)->mcq.cqn;
326298 
327
-	in.pd = to_mpd(pd)->pdn;
299
+	in.pd = to_mpd(ib_srq->pd)->pdn;
328300 	in.db_record = srq->db.dma;
329
-	err = mlx5_core_create_srq(dev->mdev, &srq->msrq, &in);
301
+	err = mlx5_cmd_create_srq(dev, &srq->msrq, &in);
330302 	kvfree(in.pas);
331303 	if (err) {
332304 		mlx5_ib_dbg(dev, "create SRQ failed, err %d\n", err);
....@@ -338,30 +310,33 @@
338310 	srq->msrq.event = mlx5_ib_srq_event;
339311 	srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
340312 
341
-	if (pd->uobject)
342
-		if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof(__u32))) {
313
+	if (udata) {
314
+		struct mlx5_ib_create_srq_resp resp = {
315
+			.srqn = srq->msrq.srqn,
316
+		};
317
+
318
+		if (ib_copy_to_udata(udata, &resp, min(udata->outlen,
319
+				     sizeof(resp)))) {
343320 			mlx5_ib_dbg(dev, "copy to user failed\n");
344321 			err = -EFAULT;
345322 			goto err_core;
346323 		}
324
+	}
347325 
348326 	init_attr->attr.max_wr = srq->msrq.max - 1;
349327 
350
-	return &srq->ibsrq;
328
+	return 0;
351329 
352330 err_core:
353
-	mlx5_core_destroy_srq(dev->mdev, &srq->msrq);
331
+	mlx5_cmd_destroy_srq(dev, &srq->msrq);
354332 
355333 err_usr_kern_srq:
356
-	if (pd->uobject)
357
-		destroy_srq_user(pd, srq);
334
+	if (udata)
335
+		destroy_srq_user(ib_srq->pd, srq, udata);
358336 	else
359337 		destroy_srq_kernel(dev, srq);
360338 
361
-err_srq:
362
-	kfree(srq);
363
-
364
-	return ERR_PTR(err);
339
+	return err;
365340 }
366341 
367342 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
....@@ -380,7 +355,7 @@
380355 			return -EINVAL;
381356 
382357 		mutex_lock(&srq->mutex);
383
-		ret = mlx5_core_arm_srq(dev->mdev, &srq->msrq, attr->srq_limit, 1);
358
+		ret = mlx5_cmd_arm_srq(dev, &srq->msrq, attr->srq_limit, 1);
384359 		mutex_unlock(&srq->mutex);
385360 
386361 		if (ret)
....@@ -401,7 +376,7 @@
401376 	if (!out)
402377 		return -ENOMEM;
403378 
404
-	ret = mlx5_core_query_srq(dev->mdev, &srq->msrq, out);
379
+	ret = mlx5_cmd_query_srq(dev, &srq->msrq, out);
405380 	if (ret)
406381 		goto out_box;
407382 
....@@ -414,21 +389,20 @@
414389 	return ret;
415390 }
416391 
417
-int mlx5_ib_destroy_srq(struct ib_srq *srq)
392
+int mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)
418393 {
419394 	struct mlx5_ib_dev *dev = to_mdev(srq->device);
420395 	struct mlx5_ib_srq *msrq = to_msrq(srq);
396
+	int ret;
421397 
422
-	mlx5_core_destroy_srq(dev->mdev, &msrq->msrq);
398
+	ret = mlx5_cmd_destroy_srq(dev, &msrq->msrq);
399
+	if (ret)
400
+		return ret;
423401 
424
-	if (srq->uobject) {
425
-		mlx5_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db);
426
-		ib_umem_release(msrq->umem);
427
-	} else {
402
+	if (udata)
403
+		destroy_srq_user(srq->pd, msrq, udata);
404
+	else
428405 		destroy_srq_kernel(dev, msrq);
429
-	}
430
-
431
-	kfree(srq);
432406 	return 0;
433407 }
434408