update kernel to 5.10.198

hc

2024-01-03 2f7c68cb55ecb7331f2381deb497c27155f32faf

 kernel/drivers/infiniband/hw/mlx5/mem.c
..
..
@@ -34,6 +34,7 @@
34
34
 #include <rdma/ib_umem.h>
35
35
 #include <rdma/ib_umem_odp.h>
36
36
 #include "mlx5_ib.h"
37
+#include <linux/jiffies.h>
37
38
 
38
39
 /* @umem: umem object to scan
39
40
  * @addr: ib virtual address requested by the user
..
..
@@ -55,27 +56,16 @@
55
56
 	int i = 0;
56
57
 	struct scatterlist *sg;
57
58
 	int entry;
58
-	unsigned long page_shift = umem->page_shift;
59
59
 
60
-	if (umem->odp_data) {
61
-		*ncont = ib_umem_page_count(umem);
62
-		*count = *ncont << (page_shift - PAGE_SHIFT);
63
-		*shift = page_shift;
64
-		if (order)
65
-			*order = ilog2(roundup_pow_of_two(*ncont));
66
-
67
-		return;
68
-	}
69
-
70
-	addr = addr >> page_shift;
60
+	addr = addr >> PAGE_SHIFT;
71
61
 	tmp = (unsigned long)addr;
72
62
 	m = find_first_bit(&tmp, BITS_PER_LONG);
73
63
 	if (max_page_shift)
74
-		m = min_t(unsigned long, max_page_shift - page_shift, m);
64
+		m = min_t(unsigned long, max_page_shift - PAGE_SHIFT, m);
75
65
 
76
66
 	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
77
-		len = sg_dma_len(sg) >> page_shift;
78
-		pfn = sg_dma_address(sg) >> page_shift;
67
+		len = sg_dma_len(sg) >> PAGE_SHIFT;
68
+		pfn = sg_dma_address(sg) >> PAGE_SHIFT;
79
69
 		if (base + p != pfn) {
80
70
 			/* If either the offset or the new
81
71
 			 * base are unaligned update m
..
..
@@ -107,23 +97,9 @@
107
97
 
108
98
 		*ncont = 0;
109
99
 	}
110
-	*shift = page_shift + m;
100
+	*shift = PAGE_SHIFT + m;
111
101
 	*count = i;
112
102
 }
113
-
114
-#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
115
-static u64 umem_dma_to_mtt(dma_addr_t umem_dma)
116
-{
117
-	u64 mtt_entry = umem_dma & ODP_DMA_ADDR_MASK;
118
-
119
-	if (umem_dma & ODP_READ_ALLOWED_BIT)
120
-		mtt_entry |= MLX5_IB_MTT_READ;
121
-	if (umem_dma & ODP_WRITE_ALLOWED_BIT)
122
-		mtt_entry |= MLX5_IB_MTT_WRITE;
123
-
124
-	return mtt_entry;
125
-}
126
-#endif
127
103
 
128
104
 /*
129
105
  * Populate the given array with bus addresses from the umem.
..
..
@@ -142,8 +118,7 @@
142
118
 			    int page_shift, size_t offset, size_t num_pages,
143
119
 			    __be64 *pas, int access_flags)
144
120
 {
145
-	unsigned long umem_page_shift = umem->page_shift;
146
-	int shift = page_shift - umem_page_shift;
121
+	int shift = page_shift - PAGE_SHIFT;
147
122
 	int mask = (1 << shift) - 1;
148
123
 	int i, k, idx;
149
124
 	u64 cur = 0;
..
..
@@ -151,25 +126,10 @@
151
126
 	int len;
152
127
 	struct scatterlist *sg;
153
128
 	int entry;
154
-#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
155
-	const bool odp = umem->odp_data != NULL;
156
-
157
-	if (odp) {
158
-		WARN_ON(shift != 0);
159
-		WARN_ON(access_flags != (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE));
160
-
161
-		for (i = 0; i < num_pages; ++i) {
162
-			dma_addr_t pa = umem->odp_data->dma_list[offset + i];
163
-
164
-			pas[i] = cpu_to_be64(umem_dma_to_mtt(pa));
165
-		}
166
-		return;
167
-	}
168
-#endif
169
129
 
170
130
 	i = 0;
171
131
 	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
172
-		len = sg_dma_len(sg) >> umem_page_shift;
132
+		len = sg_dma_len(sg) >> PAGE_SHIFT;
173
133
 		base = sg_dma_address(sg);
174
134
 
175
135
 		/* Skip elements below offset */
..
..
@@ -188,7 +148,7 @@
188
148
 
189
149
 		for (; k < len; k++) {
190
150
 			if (!(i & mask)) {
191
-				cur = base + (k << umem_page_shift);
151
+				cur = base + (k << PAGE_SHIFT);
192
152
 				cur |= access_flags;
193
153
 				idx = (i >> shift) - offset;
194
154
 
..
..
@@ -209,8 +169,8 @@
209
169
 			  int page_shift, __be64 *pas, int access_flags)
210
170
 {
211
171
 	return __mlx5_ib_populate_pas(dev, umem, page_shift, 0,
212
-				      ib_umem_num_pages(umem), pas,
213
-				      access_flags);
172
+				      ib_umem_num_dma_blocks(umem, PAGE_SIZE),
173
+				      pas, access_flags);
214
174
 }
215
175
 int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset)
216
176
 {
..
..
@@ -232,3 +192,201 @@
232
192
 	*offset = buf_off >> ilog2(off_size);
233
193
 	return 0;
234
194
 }
195
+
196
+#define WR_ID_BF 0xBF
197
+#define WR_ID_END 0xBAD
198
+#define TEST_WC_NUM_WQES 255
199
+#define TEST_WC_POLLING_MAX_TIME_JIFFIES msecs_to_jiffies(100)
200
+static int post_send_nop(struct mlx5_ib_dev *dev, struct ib_qp *ibqp, u64 wr_id,
201
+			 bool signaled)
202
+{
203
+	struct mlx5_ib_qp *qp = to_mqp(ibqp);
204
+	struct mlx5_wqe_ctrl_seg *ctrl;
205
+	struct mlx5_bf *bf = &qp->bf;
206
+	__be32 mmio_wqe[16] = {};
207
+	unsigned long flags;
208
+	unsigned int idx;
209
+	int i;
210
+
211
+	if (unlikely(dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR))
212
+		return -EIO;
213
+
214
+	spin_lock_irqsave(&qp->sq.lock, flags);
215
+
216
+	idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
217
+	ctrl = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx);
218
+
219
+	memset(ctrl, 0, sizeof(struct mlx5_wqe_ctrl_seg));
220
+	ctrl->fm_ce_se = signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0;
221
+	ctrl->opmod_idx_opcode =
222
+		cpu_to_be32(((u32)(qp->sq.cur_post) << 8) | MLX5_OPCODE_NOP);
223
+	ctrl->qpn_ds = cpu_to_be32((sizeof(struct mlx5_wqe_ctrl_seg) / 16) |
224
+				   (qp->trans_qp.base.mqp.qpn << 8));
225
+
226
+	qp->sq.wrid[idx] = wr_id;
227
+	qp->sq.w_list[idx].opcode = MLX5_OPCODE_NOP;
228
+	qp->sq.wqe_head[idx] = qp->sq.head + 1;
229
+	qp->sq.cur_post += DIV_ROUND_UP(sizeof(struct mlx5_wqe_ctrl_seg),
230
+					MLX5_SEND_WQE_BB);
231
+	qp->sq.w_list[idx].next = qp->sq.cur_post;
232
+	qp->sq.head++;
233
+
234
+	memcpy(mmio_wqe, ctrl, sizeof(*ctrl));
235
+	((struct mlx5_wqe_ctrl_seg *)&mmio_wqe)->fm_ce_se |=
236
+		MLX5_WQE_CTRL_CQ_UPDATE;
237
+
238
+	/* Make sure that descriptors are written before
239
+	 * updating doorbell record and ringing the doorbell
240
+	 */
241
+	wmb();
242
+
243
+	qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
244
+
245
+	/* Make sure doorbell record is visible to the HCA before
246
+	 * we hit doorbell
247
+	 */
248
+	wmb();
249
+	for (i = 0; i < 8; i++)
250
+		mlx5_write64(&mmio_wqe[i * 2],
251
+			     bf->bfreg->map + bf->offset + i * 8);
252
+
253
+	bf->offset ^= bf->buf_size;
254
+
255
+	spin_unlock_irqrestore(&qp->sq.lock, flags);
256
+
257
+	return 0;
258
+}
259
+
260
+static int test_wc_poll_cq_result(struct mlx5_ib_dev *dev, struct ib_cq *cq)
261
+{
262
+	int ret;
263
+	struct ib_wc wc = {};
264
+	unsigned long end = jiffies + TEST_WC_POLLING_MAX_TIME_JIFFIES;
265
+
266
+	do {
267
+		ret = ib_poll_cq(cq, 1, &wc);
268
+		if (ret < 0 || wc.status)
269
+			return ret < 0 ? ret : -EINVAL;
270
+		if (ret)
271
+			break;
272
+	} while (!time_after(jiffies, end));
273
+
274
+	if (!ret)
275
+		return -ETIMEDOUT;
276
+
277
+	if (wc.wr_id != WR_ID_BF)
278
+		ret = 0;
279
+
280
+	return ret;
281
+}
282
+
283
+static int test_wc_do_send(struct mlx5_ib_dev *dev, struct ib_qp *qp)
284
+{
285
+	int err, i;
286
+
287
+	for (i = 0; i < TEST_WC_NUM_WQES; i++) {
288
+		err = post_send_nop(dev, qp, WR_ID_BF, false);
289
+		if (err)
290
+			return err;
291
+	}
292
+
293
+	return post_send_nop(dev, qp, WR_ID_END, true);
294
+}
295
+
296
+int mlx5_ib_test_wc(struct mlx5_ib_dev *dev)
297
+{
298
+	struct ib_cq_init_attr cq_attr = { .cqe = TEST_WC_NUM_WQES + 1 };
299
+	int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
300
+	struct ib_qp_init_attr qp_init_attr = {
301
+		.cap = { .max_send_wr = TEST_WC_NUM_WQES },
302
+		.qp_type = IB_QPT_UD,
303
+		.sq_sig_type = IB_SIGNAL_REQ_WR,
304
+		.create_flags = MLX5_IB_QP_CREATE_WC_TEST,
305
+	};
306
+	struct ib_qp_attr qp_attr = { .port_num = 1 };
307
+	struct ib_device *ibdev = &dev->ib_dev;
308
+	struct ib_qp *qp;
309
+	struct ib_cq *cq;
310
+	struct ib_pd *pd;
311
+	int ret;
312
+
313
+	if (!MLX5_CAP_GEN(dev->mdev, bf))
314
+		return 0;
315
+
316
+	if (!dev->mdev->roce.roce_en &&
317
+	    port_type_cap == MLX5_CAP_PORT_TYPE_ETH) {
318
+		if (mlx5_core_is_pf(dev->mdev))
319
+			dev->wc_support = arch_can_pci_mmap_wc();
320
+		return 0;
321
+	}
322
+
323
+	ret = mlx5_alloc_bfreg(dev->mdev, &dev->wc_bfreg, true, false);
324
+	if (ret)
325
+		goto print_err;
326
+
327
+	if (!dev->wc_bfreg.wc)
328
+		goto out1;
329
+
330
+	pd = ib_alloc_pd(ibdev, 0);
331
+	if (IS_ERR(pd)) {
332
+		ret = PTR_ERR(pd);
333
+		goto out1;
334
+	}
335
+
336
+	cq = ib_create_cq(ibdev, NULL, NULL, NULL, &cq_attr);
337
+	if (IS_ERR(cq)) {
338
+		ret = PTR_ERR(cq);
339
+		goto out2;
340
+	}
341
+
342
+	qp_init_attr.recv_cq = cq;
343
+	qp_init_attr.send_cq = cq;
344
+	qp = ib_create_qp(pd, &qp_init_attr);
345
+	if (IS_ERR(qp)) {
346
+		ret = PTR_ERR(qp);
347
+		goto out3;
348
+	}
349
+
350
+	qp_attr.qp_state = IB_QPS_INIT;
351
+	ret = ib_modify_qp(qp, &qp_attr,
352
+			   IB_QP_STATE | IB_QP_PORT | IB_QP_PKEY_INDEX |
353
+				   IB_QP_QKEY);
354
+	if (ret)
355
+		goto out4;
356
+
357
+	qp_attr.qp_state = IB_QPS_RTR;
358
+	ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
359
+	if (ret)
360
+		goto out4;
361
+
362
+	qp_attr.qp_state = IB_QPS_RTS;
363
+	ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
364
+	if (ret)
365
+		goto out4;
366
+
367
+	ret = test_wc_do_send(dev, qp);
368
+	if (ret < 0)
369
+		goto out4;
370
+
371
+	ret = test_wc_poll_cq_result(dev, cq);
372
+	if (ret > 0) {
373
+		dev->wc_support = true;
374
+		ret = 0;
375
+	}
376
+
377
+out4:
378
+	ib_destroy_qp(qp);
379
+out3:
380
+	ib_destroy_cq(cq);
381
+out2:
382
+	ib_dealloc_pd(pd);
383
+out1:
384
+	mlx5_free_bfreg(dev->mdev, &dev->wc_bfreg);
385
+print_err:
386
+	if (ret)
387
+		mlx5_ib_err(
388
+			dev,
389
+			"Error %d while trying to test write-combining support\n",
390
+			ret);
391
+	return ret;
392
+}