rtl88x2CE_WiFi_linux add concurrent mode

hc

2024-09-20 cf4ce59b3b70238352c7f1729f0f7223214828ad

 kernel/net/smc/smc_wr.c
..
..
@@ -44,11 +44,28 @@
44
44
 	struct smc_link		*link;
45
45
 	u32			idx;
46
46
 	struct smc_wr_tx_pend_priv priv;
47
+	u8			compl_requested;
47
48
 };
48
49
 
49
50
 /******************************** send queue *********************************/
50
51
 
51
52
 /*------------------------------- completion --------------------------------*/
53
+
54
+/* returns true if at least one tx work request is pending on the given link */
55
+static inline bool smc_wr_is_tx_pend(struct smc_link *link)
56
+{
57
+	if (find_first_bit(link->wr_tx_mask, link->wr_tx_cnt) !=
58
+							link->wr_tx_cnt) {
59
+		return true;
60
+	}
61
+	return false;
62
+}
63
+
64
+/* wait till all pending tx work requests on the given link are completed */
65
+void smc_wr_tx_wait_no_pending_sends(struct smc_link *link)
66
+{
67
+	wait_event(link->wr_tx_wait, !smc_wr_is_tx_pend(link));
68
+}
52
69
 
53
70
 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
54
71
 {
..
..
@@ -66,7 +83,6 @@
66
83
 	struct smc_wr_tx_pend pnd_snd;
67
84
 	struct smc_link *link;
68
85
 	u32 pnd_snd_idx;
69
-	int i;
70
86
 
71
87
 	link = wc->qp->qp_context;
72
88
 
..
..
@@ -75,7 +91,7 @@
75
91
 			link->wr_reg_state = FAILED;
76
92
 		else
77
93
 			link->wr_reg_state = CONFIRMED;
78
-		wake_up(&link->wr_reg_wait);
94
+		smc_wr_wakeup_reg_wait(link);
79
95
 		return;
80
96
 	}
81
97
 
..
..
@@ -83,6 +99,8 @@
83
99
 	if (pnd_snd_idx == link->wr_tx_cnt)
84
100
 		return;
85
101
 	link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;
102
+	if (link->wr_tx_pends[pnd_snd_idx].compl_requested)
103
+		complete(&link->wr_tx_compl[pnd_snd_idx]);
86
104
 	memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd));
87
105
 	/* clear the full struct smc_wr_tx_pend including .priv */
88
106
 	memset(&link->wr_tx_pends[pnd_snd_idx], 0,
..
..
@@ -92,16 +110,8 @@
92
110
 	if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
93
111
 		return;
94
112
 	if (wc->status) {
95
-		for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
96
-			/* clear full struct smc_wr_tx_pend including .priv */
97
-			memset(&link->wr_tx_pends[i], 0,
98
-			       sizeof(link->wr_tx_pends[i]));
99
-			memset(&link->wr_tx_bufs[i], 0,
100
-			       sizeof(link->wr_tx_bufs[i]));
101
-			clear_bit(i, link->wr_tx_mask);
102
-		}
103
-		/* terminate connections of this link group abnormally */
104
-		smc_lgr_terminate(smc_get_lgr(link));
113
+		/* terminate link */
114
+		smcr_link_down_cond_sched(link);
105
115
 	}
106
116
 	if (pnd_snd.handler)
107
117
 		pnd_snd.handler(&pnd_snd.priv, link, wc->status);
..
..
@@ -146,6 +156,8 @@
146
156
 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
147
157
 {
148
158
 	*idx = link->wr_tx_cnt;
159
+	if (!smc_link_sendable(link))
160
+		return -ENOLINK;
149
161
 	for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
150
162
 		if (!test_and_set_bit(*idx, link->wr_tx_mask))
151
163
 			return 0;
..
..
@@ -160,6 +172,7 @@
160
172
  * @link:		Pointer to smc_link used to later send the message.
161
173
  * @handler:		Send completion handler function pointer.
162
174
  * @wr_buf:		Out value returns pointer to message buffer.
175
+ * @wr_rdma_buf:	Out value returns pointer to rdma work request.
163
176
  * @wr_pend_priv:	Out value returns pointer serving as handler context.
164
177
  *
165
178
  * Return: 0 on success, or -errno on error.
..
..
@@ -167,8 +180,10 @@
167
180
 int smc_wr_tx_get_free_slot(struct smc_link *link,
168
181
 			    smc_wr_tx_handler handler,
169
182
 			    struct smc_wr_buf **wr_buf,
183
+			    struct smc_rdma_wr **wr_rdma_buf,
170
184
 			    struct smc_wr_tx_pend_priv **wr_pend_priv)
171
185
 {
186
+	struct smc_link_group *lgr = smc_get_lgr(link);
172
187
 	struct smc_wr_tx_pend *wr_pend;
173
188
 	u32 idx = link->wr_tx_cnt;
174
189
 	struct ib_send_wr *wr_ib;
..
..
@@ -177,19 +192,20 @@
177
192
 
178
193
 	*wr_buf = NULL;
179
194
 	*wr_pend_priv = NULL;
180
-	if (in_softirq()) {
195
+	if (in_softirq() || lgr->terminating) {
181
196
 		rc = smc_wr_tx_get_free_slot_index(link, &idx);
182
197
 		if (rc)
183
198
 			return rc;
184
199
 	} else {
185
-		rc = wait_event_timeout(
200
+		rc = wait_event_interruptible_timeout(
186
201
 			link->wr_tx_wait,
187
-			link->state == SMC_LNK_INACTIVE ||
202
+			!smc_link_sendable(link) ||
203
+			lgr->terminating ||
188
204
 			(smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
189
205
 			SMC_WR_TX_WAIT_FREE_SLOT_TIME);
190
206
 		if (!rc) {
191
-			/* timeout - terminate connections */
192
-			smc_lgr_terminate(smc_get_lgr(link));
207
+			/* timeout - terminate link */
208
+			smcr_link_down_cond_sched(link);
193
209
 			return -EPIPE;
194
210
 		}
195
211
 		if (idx == link->wr_tx_cnt)
..
..
@@ -204,6 +220,8 @@
204
220
 	wr_ib = &link->wr_tx_ibs[idx];
205
221
 	wr_ib->wr_id = wr_id;
206
222
 	*wr_buf = &link->wr_tx_bufs[idx];
223
+	if (wr_rdma_buf)
224
+		*wr_rdma_buf = &link->wr_tx_rdmas[idx];
207
225
 	*wr_pend_priv = &wr_pend->priv;
208
226
 	return 0;
209
227
 }
..
..
@@ -218,11 +236,12 @@
218
236
 		u32 idx = pend->idx;
219
237
 
220
238
 		/* clear the full struct smc_wr_tx_pend including .priv */
221
-		memset(&link->wr_tx_pends[pend->idx], 0,
222
-		       sizeof(link->wr_tx_pends[pend->idx]));
223
-		memset(&link->wr_tx_bufs[pend->idx], 0,
224
-		       sizeof(link->wr_tx_bufs[pend->idx]));
239
+		memset(&link->wr_tx_pends[idx], 0,
240
+		       sizeof(link->wr_tx_pends[idx]));
241
+		memset(&link->wr_tx_bufs[idx], 0,
242
+		       sizeof(link->wr_tx_bufs[idx]));
225
243
 		test_and_clear_bit(idx, link->wr_tx_mask);
244
+		wake_up(&link->wr_tx_wait);
226
245
 		return 1;
227
246
 	}
228
247
 
..
..
@@ -243,8 +262,37 @@
243
262
 	rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL);
244
263
 	if (rc) {
245
264
 		smc_wr_tx_put_slot(link, priv);
246
-		smc_lgr_terminate(smc_get_lgr(link));
265
+		smcr_link_down_cond_sched(link);
247
266
 	}
267
+	return rc;
268
+}
269
+
270
+/* Send prepared WR slot via ib_post_send and wait for send completion
271
+ * notification.
272
+ * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
273
+ */
274
+int smc_wr_tx_send_wait(struct smc_link *link, struct smc_wr_tx_pend_priv *priv,
275
+			unsigned long timeout)
276
+{
277
+	struct smc_wr_tx_pend *pend;
278
+	u32 pnd_idx;
279
+	int rc;
280
+
281
+	pend = container_of(priv, struct smc_wr_tx_pend, priv);
282
+	pend->compl_requested = 1;
283
+	pnd_idx = pend->idx;
284
+	init_completion(&link->wr_tx_compl[pnd_idx]);
285
+
286
+	rc = smc_wr_tx_send(link, priv);
287
+	if (rc)
288
+		return rc;
289
+	/* wait for completion by smc_wr_tx_process_cqe() */
290
+	rc = wait_for_completion_interruptible_timeout(
291
+					&link->wr_tx_compl[pnd_idx], timeout);
292
+	if (rc <= 0)
293
+		rc = -ENODATA;
294
+	if (rc > 0)
295
+		rc = 0;
248
296
 	return rc;
249
297
 }
250
298
 
..
..
@@ -263,12 +311,15 @@
263
311
 	if (rc)
264
312
 		return rc;
265
313
 
314
+	atomic_inc(&link->wr_reg_refcnt);
266
315
 	rc = wait_event_interruptible_timeout(link->wr_reg_wait,
267
316
 					      (link->wr_reg_state != POSTED),
268
317
 					      SMC_WR_REG_MR_WAIT_TIME);
318
+	if (atomic_dec_and_test(&link->wr_reg_refcnt))
319
+		wake_up_all(&link->wr_reg_wait);
269
320
 	if (!rc) {
270
-		/* timeout - terminate connections */
271
-		smc_lgr_terminate(smc_get_lgr(link));
321
+		/* timeout - terminate link */
322
+		smcr_link_down_cond_sched(link);
272
323
 		return -EPIPE;
273
324
 	}
274
325
 	if (rc == -ERESTARTSYS)
..
..
@@ -285,25 +336,6 @@
285
336
 		break;
286
337
 	}
287
338
 	return rc;
288
-}
289
-
290
-void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type,
291
-			     smc_wr_tx_filter filter,
292
-			     smc_wr_tx_dismisser dismisser,
293
-			     unsigned long data)
294
-{
295
-	struct smc_wr_tx_pend_priv *tx_pend;
296
-	struct smc_wr_rx_hdr *wr_tx;
297
-	int i;
298
-
299
-	for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
300
-		wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i];
301
-		if (wr_tx->type != wr_tx_hdr_type)
302
-			continue;
303
-		tx_pend = &link->wr_tx_pends[i].priv;
304
-		if (filter(tx_pend, data))
305
-			dismisser(tx_pend);
306
-	}
307
339
 }
308
340
 
309
341
 /****************************** receive queue ********************************/
..
..
@@ -366,10 +398,7 @@
366
398
 			case IB_WC_RETRY_EXC_ERR:
367
399
 			case IB_WC_RNR_RETRY_EXC_ERR:
368
400
 			case IB_WC_WR_FLUSH_ERR:
369
-				/* terminate connections of this link group
370
-				 * abnormally
371
-				 */
372
-				smc_lgr_terminate(smc_get_lgr(link));
401
+				smcr_link_down_cond_sched(link);
373
402
 				break;
374
403
 			default:
375
404
 				smc_wr_rx_post(link); /* refill WR RX */
..
..
@@ -465,12 +494,26 @@
465
494
 			lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE;
466
495
 		lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE;
467
496
 		lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
497
+		lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[0].lkey =
498
+			lnk->roce_pd->local_dma_lkey;
499
+		lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[1].lkey =
500
+			lnk->roce_pd->local_dma_lkey;
501
+		lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[0].lkey =
502
+			lnk->roce_pd->local_dma_lkey;
503
+		lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[1].lkey =
504
+			lnk->roce_pd->local_dma_lkey;
468
505
 		lnk->wr_tx_ibs[i].next = NULL;
469
506
 		lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i];
470
507
 		lnk->wr_tx_ibs[i].num_sge = 1;
471
508
 		lnk->wr_tx_ibs[i].opcode = IB_WR_SEND;
472
509
 		lnk->wr_tx_ibs[i].send_flags =
473
510
 			IB_SEND_SIGNALED | IB_SEND_SOLICITED;
511
+		lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.opcode = IB_WR_RDMA_WRITE;
512
+		lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.opcode = IB_WR_RDMA_WRITE;
513
+		lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.sg_list =
514
+			lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge;
515
+		lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.sg_list =
516
+			lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge;
474
517
 	}
475
518
 	for (i = 0; i < lnk->wr_rx_cnt; i++) {
476
519
 		lnk->wr_rx_sges[i].addr =
..
..
@@ -492,12 +535,16 @@
492
535
 {
493
536
 	struct ib_device *ibdev;
494
537
 
495
-	memset(lnk->wr_tx_mask, 0,
496
-	       BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
497
-
498
538
 	if (!lnk->smcibdev)
499
539
 		return;
500
540
 	ibdev = lnk->smcibdev->ibdev;
541
+
542
+	smc_wr_wakeup_reg_wait(lnk);
543
+	smc_wr_wakeup_tx_wait(lnk);
544
+
545
+	smc_wr_tx_wait_no_pending_sends(lnk);
546
+	wait_event(lnk->wr_reg_wait, (!atomic_read(&lnk->wr_reg_refcnt)));
547
+	wait_event(lnk->wr_tx_wait, (!atomic_read(&lnk->wr_tx_refcnt)));
501
548
 
502
549
 	if (lnk->wr_rx_dma_addr) {
503
550
 		ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
..
..
@@ -515,14 +562,20 @@
515
562
 
516
563
 void smc_wr_free_link_mem(struct smc_link *lnk)
517
564
 {
565
+	kfree(lnk->wr_tx_compl);
566
+	lnk->wr_tx_compl = NULL;
518
567
 	kfree(lnk->wr_tx_pends);
519
568
 	lnk->wr_tx_pends = NULL;
520
569
 	kfree(lnk->wr_tx_mask);
521
570
 	lnk->wr_tx_mask = NULL;
522
571
 	kfree(lnk->wr_tx_sges);
523
572
 	lnk->wr_tx_sges = NULL;
573
+	kfree(lnk->wr_tx_rdma_sges);
574
+	lnk->wr_tx_rdma_sges = NULL;
524
575
 	kfree(lnk->wr_rx_sges);
525
576
 	lnk->wr_rx_sges = NULL;
577
+	kfree(lnk->wr_tx_rdmas);
578
+	lnk->wr_tx_rdmas = NULL;
526
579
 	kfree(lnk->wr_rx_ibs);
527
580
 	lnk->wr_rx_ibs = NULL;
528
581
 	kfree(lnk->wr_tx_ibs);
..
..
@@ -552,10 +605,20 @@
552
605
 				  GFP_KERNEL);
553
606
 	if (!link->wr_rx_ibs)
554
607
 		goto no_mem_wr_tx_ibs;
608
+	link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT,
609
+				    sizeof(link->wr_tx_rdmas[0]),
610
+				    GFP_KERNEL);
611
+	if (!link->wr_tx_rdmas)
612
+		goto no_mem_wr_rx_ibs;
613
+	link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT,
614
+					sizeof(link->wr_tx_rdma_sges[0]),
615
+					GFP_KERNEL);
616
+	if (!link->wr_tx_rdma_sges)
617
+		goto no_mem_wr_tx_rdmas;
555
618
 	link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
556
619
 				   GFP_KERNEL);
557
620
 	if (!link->wr_tx_sges)
558
-		goto no_mem_wr_rx_ibs;
621
+		goto no_mem_wr_tx_rdma_sges;
559
622
 	link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
560
623
 				   sizeof(link->wr_rx_sges[0]),
561
624
 				   GFP_KERNEL);
..
..
@@ -571,14 +634,25 @@
571
634
 				    GFP_KERNEL);
572
635
 	if (!link->wr_tx_pends)
573
636
 		goto no_mem_wr_tx_mask;
637
+	link->wr_tx_compl = kcalloc(SMC_WR_BUF_CNT,
638
+				    sizeof(link->wr_tx_compl[0]),
639
+				    GFP_KERNEL);
640
+	if (!link->wr_tx_compl)
641
+		goto no_mem_wr_tx_pends;
574
642
 	return 0;
575
643
 
644
+no_mem_wr_tx_pends:
645
+	kfree(link->wr_tx_pends);
576
646
 no_mem_wr_tx_mask:
577
647
 	kfree(link->wr_tx_mask);
578
648
 no_mem_wr_rx_sges:
579
649
 	kfree(link->wr_rx_sges);
580
650
 no_mem_wr_tx_sges:
581
651
 	kfree(link->wr_tx_sges);
652
+no_mem_wr_tx_rdma_sges:
653
+	kfree(link->wr_tx_rdma_sges);
654
+no_mem_wr_tx_rdmas:
655
+	kfree(link->wr_tx_rdmas);
582
656
 no_mem_wr_rx_ibs:
583
657
 	kfree(link->wr_rx_ibs);
584
658
 no_mem_wr_tx_ibs:
..
..
@@ -631,7 +705,9 @@
631
705
 	memset(lnk->wr_tx_mask, 0,
632
706
 	       BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
633
707
 	init_waitqueue_head(&lnk->wr_tx_wait);
708
+	atomic_set(&lnk->wr_tx_refcnt, 0);
634
709
 	init_waitqueue_head(&lnk->wr_reg_wait);
710
+	atomic_set(&lnk->wr_reg_refcnt, 0);
635
711
 	return rc;
636
712
 
637
713
 dma_unmap: