forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/drivers/infiniband/hw/hfi1/sdma.c
....@@ -232,11 +232,11 @@
232232 static void sdma_complete(struct kref *);
233233 static void sdma_finalput(struct sdma_state *);
234234 static void sdma_get(struct sdma_state *);
235
-static void sdma_hw_clean_up_task(unsigned long);
235
+static void sdma_hw_clean_up_task(struct tasklet_struct *);
236236 static void sdma_put(struct sdma_state *);
237237 static void sdma_set_state(struct sdma_engine *, enum sdma_states);
238238 static void sdma_start_hw_clean_up(struct sdma_engine *);
239
-static void sdma_sw_clean_up_task(unsigned long);
239
+static void sdma_sw_clean_up_task(struct tasklet_struct *);
240240 static void sdma_sendctrl(struct sdma_engine *, unsigned);
241241 static void init_sdma_regs(struct sdma_engine *, u32, uint);
242242 static void sdma_process_event(
....@@ -379,7 +379,7 @@
379379 	__sdma_txclean(sde->dd, tx);
380380 	if (complete)
381381 		(*complete)(tx, res);
382
-	if (wait && iowait_sdma_dec(wait))
382
+	if (iowait_sdma_dec(wait))
383383 		iowait_drain_wakeup(wait);
384384 }
385385 
....@@ -406,6 +406,7 @@
406406 	struct sdma_txreq *txp, *txp_next;
407407 	LIST_HEAD(flushlist);
408408 	unsigned long flags;
409
+	uint seq;
409410 
410411 	/* flush from head to tail */
411412 	sdma_flush_descq(sde);
....@@ -416,6 +417,22 @@
416417 	/* flush from flush list */
417418 	list_for_each_entry_safe(txp, txp_next, &flushlist, list)
418419 		complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED);
420
+	/* wakeup QPs orphaned on the dmawait list */
421
+	do {
422
+		struct iowait *w, *nw;
423
+
424
+		seq = read_seqbegin(&sde->waitlock);
425
+		if (!list_empty(&sde->dmawait)) {
426
+			write_seqlock(&sde->waitlock);
427
+			list_for_each_entry_safe(w, nw, &sde->dmawait, list) {
428
+				if (w->wakeup) {
429
+					w->wakeup(w, SDMA_AVAIL_REASON);
430
+					list_del_init(&w->list);
431
+				}
432
+			}
433
+			write_sequnlock(&sde->waitlock);
434
+		}
435
+	} while (read_seqretry(&sde->waitlock, seq));
419436 }
420437 
421438 /*
....@@ -528,9 +545,10 @@
528545 	schedule_work(&sde->err_halt_worker);
529546 }
530547 
531
-static void sdma_hw_clean_up_task(unsigned long opaque)
548
+static void sdma_hw_clean_up_task(struct tasklet_struct *t)
532549 {
533
-	struct sdma_engine *sde = (struct sdma_engine *)opaque;
550
+	struct sdma_engine *sde = from_tasklet(sde, t,
551
+					       sdma_hw_clean_up_task);
534552 	u64 statuscsr;
535553 
536554 	while (1) {
....@@ -587,9 +605,9 @@
587605 		sdma_desc_avail(sde, sdma_descq_freecnt(sde));
588606 }
589607 
590
-static void sdma_sw_clean_up_task(unsigned long opaque)
608
+static void sdma_sw_clean_up_task(struct tasklet_struct *t)
591609 {
592
-	struct sdma_engine *sde = (struct sdma_engine *)opaque;
610
+	struct sdma_engine *sde = from_tasklet(sde, t, sdma_sw_clean_up_task);
593611 	unsigned long flags;
594612 
595613 	spin_lock_irqsave(&sde->tail_lock, flags);
....@@ -816,7 +834,7 @@
816834 struct sdma_rht_map_elem {
817835 	u32 mask;
818836 	u8 ctr;
819
-	struct sdma_engine *sde[0];
837
+	struct sdma_engine *sde[];
820838 };
821839 
822840 struct sdma_rht_node {
....@@ -831,7 +849,7 @@
831849 	.nelem_hint = NR_CPUS_HINT,
832850 	.head_offset = offsetof(struct sdma_rht_node, node),
833851 	.key_offset = offsetof(struct sdma_rht_node, cpu_id),
834
-	.key_len = FIELD_SIZEOF(struct sdma_rht_node, cpu_id),
852
+	.key_len = sizeof_field(struct sdma_rht_node, cpu_id),
835853 	.max_size = NR_CPUS,
836854 	.min_size = 8,
837855 	.automatic_shrinking = true,
....@@ -853,20 +871,19 @@
853871 {
854872 	struct sdma_rht_node *rht_node;
855873 	struct sdma_engine *sde = NULL;
856
-	const struct cpumask *current_mask = &current->cpus_allowed;
857874 	unsigned long cpu_id;
858875 
859876 	/*
860877 	 * To ensure that always the same sdma engine(s) will be
861878 	 * selected make sure the process is pinned to this CPU only.
862879 	 */
863
-	if (cpumask_weight(current_mask) != 1)
880
+	if (current->nr_cpus_allowed != 1)
864881 		goto out;
865882 
866
-	cpu_id = smp_processor_id();
867883 	rcu_read_lock();
868
-	rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu_id,
869
-					  sdma_rht_params);
884
+	cpu_id = smp_processor_id();
885
+	rht_node = rhashtable_lookup(dd->sdma_rht, &cpu_id,
886
+				     sdma_rht_params);
870887 
871888 	if (rht_node && rht_node->map[vl]) {
872889 		struct sdma_rht_map_elem *map = rht_node->map[vl];
....@@ -1313,11 +1330,13 @@
13131330 		kvfree(sde->tx_ring);
13141331 		sde->tx_ring = NULL;
13151332 	}
1316
-	spin_lock_irq(&dd->sde_map_lock);
1317
-	sdma_map_free(rcu_access_pointer(dd->sdma_map));
1318
-	RCU_INIT_POINTER(dd->sdma_map, NULL);
1319
-	spin_unlock_irq(&dd->sde_map_lock);
1320
-	synchronize_rcu();
1333
+	if (rcu_access_pointer(dd->sdma_map)) {
1334
+		spin_lock_irq(&dd->sde_map_lock);
1335
+		sdma_map_free(rcu_access_pointer(dd->sdma_map));
1336
+		RCU_INIT_POINTER(dd->sdma_map, NULL);
1337
+		spin_unlock_irq(&dd->sde_map_lock);
1338
+		synchronize_rcu();
1339
+	}
13211340 	kfree(dd->per_sdma);
13221341 	dd->per_sdma = NULL;
13231342 
....@@ -1422,6 +1441,7 @@
14221441 		seqlock_init(&sde->head_lock);
14231442 		spin_lock_init(&sde->senddmactrl_lock);
14241443 		spin_lock_init(&sde->flushlist_lock);
1444
+		seqlock_init(&sde->waitlock);
14251445 		/* insure there is always a zero bit */
14261446 		sde->ahg_bits = 0xfffffffe00000000ULL;
14271447 
....@@ -1437,11 +1457,10 @@
14371457 		sde->tail_csr =
14381458 			get_kctxt_csr_addr(dd, this_idx, SD(TAIL));
14391459 
1440
-		tasklet_init(&sde->sdma_hw_clean_up_task, sdma_hw_clean_up_task,
1441
-			     (unsigned long)sde);
1442
-
1443
-		tasklet_init(&sde->sdma_sw_clean_up_task, sdma_sw_clean_up_task,
1444
-			     (unsigned long)sde);
1460
+		tasklet_setup(&sde->sdma_hw_clean_up_task,
1461
+			      sdma_hw_clean_up_task);
1462
+		tasklet_setup(&sde->sdma_sw_clean_up_task,
1463
+			      sdma_sw_clean_up_task);
14451464 		INIT_WORK(&sde->err_halt_worker, sdma_err_halt_wait);
14461465 		INIT_WORK(&sde->flush_worker, sdma_field_flush);
14471466 
....@@ -1450,12 +1469,9 @@
14501469 		timer_setup(&sde->err_progress_check_timer,
14511470 			    sdma_err_progress_check, 0);
14521471 
1453
-		sde->descq = dma_zalloc_coherent(
1454
-			&dd->pcidev->dev,
1455
-			descq_cnt * sizeof(u64[2]),
1456
-			&sde->descq_phys,
1457
-			GFP_KERNEL
1458
-		);
1472
+		sde->descq = dma_alloc_coherent(&dd->pcidev->dev,
1473
+						descq_cnt * sizeof(u64[2]),
1474
+						&sde->descq_phys, GFP_KERNEL);
14591475 		if (!sde->descq)
14601476 			goto bail;
14611477 		sde->tx_ring =
....@@ -1468,24 +1484,18 @@
14681484 
14691485 	dd->sdma_heads_size = L1_CACHE_BYTES * num_engines;
14701486 	/* Allocate memory for DMA of head registers to memory */
1471
-	dd->sdma_heads_dma = dma_zalloc_coherent(
1472
-		&dd->pcidev->dev,
1473
-		dd->sdma_heads_size,
1474
-		&dd->sdma_heads_phys,
1475
-		GFP_KERNEL
1476
-	);
1487
+	dd->sdma_heads_dma = dma_alloc_coherent(&dd->pcidev->dev,
1488
+						dd->sdma_heads_size,
1489
+						&dd->sdma_heads_phys,
1490
+						GFP_KERNEL);
14771491 	if (!dd->sdma_heads_dma) {
14781492 		dd_dev_err(dd, "failed to allocate SendDMA head memory\n");
14791493 		goto bail;
14801494 	}
14811495 
14821496 	/* Allocate memory for pad */
1483
-	dd->sdma_pad_dma = dma_zalloc_coherent(
1484
-		&dd->pcidev->dev,
1485
-		SDMA_PAD,
1486
-		&dd->sdma_pad_phys,
1487
-		GFP_KERNEL
1488
-	);
1497
+	dd->sdma_pad_dma = dma_alloc_coherent(&dd->pcidev->dev, SDMA_PAD,
1498
+					      &dd->sdma_pad_phys, GFP_KERNEL);
14891499 	if (!dd->sdma_pad_dma) {
14901500 		dd_dev_err(dd, "failed to allocate SendDMA pad memory\n");
14911501 		goto bail;
....@@ -1756,12 +1766,9 @@
17561766  */
17571767 static void sdma_desc_avail(struct sdma_engine *sde, uint avail)
17581768 {
1759
-	struct iowait *wait, *nw;
1769
+	struct iowait *wait, *nw, *twait;
17601770 	struct iowait *waits[SDMA_WAIT_BATCH_SIZE];
1761
-	uint i, n = 0, seq, max_idx = 0;
1762
-	struct sdma_txreq *stx;
1763
-	struct hfi1_ibdev *dev = &sde->dd->verbs_dev;
1764
-	u8 max_starved_cnt = 0;
1771
+	uint i, n = 0, seq, tidx = 0;
17651772 
17661773 #ifdef CONFIG_SDMA_VERBOSITY
17671774 	dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
....@@ -1770,49 +1777,50 @@
17701777 #endif
17711778 
17721779 	do {
1773
-		seq = read_seqbegin(&dev->iowait_lock);
1780
+		seq = read_seqbegin(&sde->waitlock);
17741781 		if (!list_empty(&sde->dmawait)) {
17751782 			/* at least one item */
1776
-			write_seqlock(&dev->iowait_lock);
1783
+			write_seqlock(&sde->waitlock);
17771784 			/* Harvest waiters wanting DMA descriptors */
17781785 			list_for_each_entry_safe(
17791786 					wait,
17801787 					nw,
17811788 					&sde->dmawait,
17821789 					list) {
1783
-				u16 num_desc = 0;
1790
+				u32 num_desc;
17841791 
17851792 				if (!wait->wakeup)
17861793 					continue;
17871794 				if (n == ARRAY_SIZE(waits))
17881795 					break;
1789
-				if (!list_empty(&wait->tx_head)) {
1790
-					stx = list_first_entry(
1791
-						&wait->tx_head,
1792
-						struct sdma_txreq,
1793
-						list);
1794
-					num_desc = stx->num_desc;
1795
-				}
1796
+				iowait_init_priority(wait);
1797
+				num_desc = iowait_get_all_desc(wait);
17961798 				if (num_desc > avail)
17971799 					break;
17981800 				avail -= num_desc;
1799
-				/* Find the most starved wait memeber */
1800
-				iowait_starve_find_max(wait, &max_starved_cnt,
1801
-						       n, &max_idx);
1801
+				/* Find the top-priority wait memeber */
1802
+				if (n) {
1803
+					twait = waits[tidx];
1804
+					tidx =
1805
+					    iowait_priority_update_top(wait,
1806
+								       twait,
1807
+								       n,
1808
+								       tidx);
1809
+				}
18021810 				list_del_init(&wait->list);
18031811 				waits[n++] = wait;
18041812 			}
1805
-			write_sequnlock(&dev->iowait_lock);
1813
+			write_sequnlock(&sde->waitlock);
18061814 			break;
18071815 		}
1808
-	} while (read_seqretry(&dev->iowait_lock, seq));
1816
+	} while (read_seqretry(&sde->waitlock, seq));
18091817 
1810
-	/* Schedule the most starved one first */
1818
+	/* Schedule the top-priority entry first */
18111819 	if (n)
1812
-		waits[max_idx]->wakeup(waits[max_idx], SDMA_AVAIL_REASON);
1820
+		waits[tidx]->wakeup(waits[tidx], SDMA_AVAIL_REASON);
18131821 
18141822 	for (i = 0; i < n; i++)
1815
-		if (i != max_idx)
1823
+		if (i != tidx)
18161824 			waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON);
18171825 }
18181826 
....@@ -2347,7 +2355,7 @@
23472355  */
23482356 static int sdma_check_progress(
23492357 	struct sdma_engine *sde,
2350
-	struct iowait *wait,
2358
+	struct iowait_work *wait,
23512359 	struct sdma_txreq *tx,
23522360 	bool pkts_sent)
23532361 {
....@@ -2357,12 +2365,12 @@
23572365 	if (tx->num_desc <= sde->desc_avail)
23582366 		return -EAGAIN;
23592367 	/* pulse the head_lock */
2360
-	if (wait && wait->sleep) {
2368
+	if (wait && iowait_ioww_to_iow(wait)->sleep) {
23612369 		unsigned seq;
23622370 
23632371 		seq = raw_seqcount_begin(
23642372 			(const seqcount_t *)&sde->head_lock.seqcount);
2365
-		ret = wait->sleep(sde, wait, tx, seq, pkts_sent);
2373
+		ret = wait->iow->sleep(sde, wait, tx, seq, pkts_sent);
23662374 		if (ret == -EAGAIN)
23672375 			sde->desc_avail = sdma_descq_freecnt(sde);
23682376 	} else {
....@@ -2374,7 +2382,7 @@
23742382 /**
23752383  * sdma_send_txreq() - submit a tx req to ring
23762384  * @sde: sdma engine to use
2377
- * @wait: wait structure to use when full (may be NULL)
2385
+ * @wait: SE wait structure to use when full (may be NULL)
23782386  * @tx: sdma_txreq to submit
23792387  * @pkts_sent: has any packet been sent yet?
23802388  *
....@@ -2387,7 +2395,7 @@
23872395  * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
23882396  */
23892397 int sdma_send_txreq(struct sdma_engine *sde,
2390
-		    struct iowait *wait,
2398
+		    struct iowait_work *wait,
23912399 		    struct sdma_txreq *tx,
23922400 		    bool pkts_sent)
23932401 {
....@@ -2398,7 +2406,7 @@
23982406 	/* user should have supplied entire packet */
23992407 	if (unlikely(tx->tlen))
24002408 		return -EINVAL;
2401
-	tx->wait = wait;
2409
+	tx->wait = iowait_ioww_to_iow(wait);
24022410 	spin_lock_irqsave(&sde->tail_lock, flags);
24032411 retry:
24042412 	if (unlikely(!__sdma_running(sde)))
....@@ -2407,14 +2415,14 @@
24072415 		goto nodesc;
24082416 	tail = submit_tx(sde, tx);
24092417 	if (wait)
2410
-		iowait_sdma_inc(wait);
2418
+		iowait_sdma_inc(iowait_ioww_to_iow(wait));
24112419 	sdma_update_tail(sde, tail);
24122420 unlock:
24132421 	spin_unlock_irqrestore(&sde->tail_lock, flags);
24142422 	return ret;
24152423 unlock_noconn:
24162424 	if (wait)
2417
-		iowait_sdma_inc(wait);
2425
+		iowait_sdma_inc(iowait_ioww_to_iow(wait));
24182426 	tx->next_descq_idx = 0;
24192427 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
24202428 	tx->sn = sde->tail_sn++;
....@@ -2423,10 +2431,7 @@
24232431 	spin_lock(&sde->flushlist_lock);
24242432 	list_add_tail(&tx->list, &sde->flushlist);
24252433 	spin_unlock(&sde->flushlist_lock);
2426
-	if (wait) {
2427
-		wait->tx_count++;
2428
-		wait->count += tx->num_desc;
2429
-	}
2434
+	iowait_inc_wait_count(wait, tx->num_desc);
24302435 	queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker);
24312436 	ret = -ECOMM;
24322437 	goto unlock;
....@@ -2443,9 +2448,9 @@
24432448 /**
24442449  * sdma_send_txlist() - submit a list of tx req to ring
24452450  * @sde: sdma engine to use
2446
- * @wait: wait structure to use when full (may be NULL)
2451
+ * @wait: SE wait structure to use when full (may be NULL)
24472452  * @tx_list: list of sdma_txreqs to submit
2448
- * @count: pointer to a u32 which, after return will contain the total number of
2453
+ * @count: pointer to a u16 which, after return will contain the total number of
24492454  *         sdma_txreqs removed from the tx_list. This will include sdma_txreqs
24502455  *         whose SDMA descriptors are submitted to the ring and the sdma_txreqs
24512456  *         which are added to SDMA engine flush list if the SDMA engine state is
....@@ -2468,8 +2473,8 @@
24682473  * -EINVAL - sdma_txreq incomplete, -EBUSY - no space in ring (wait == NULL)
24692474  * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
24702475  */
2471
-int sdma_send_txlist(struct sdma_engine *sde, struct iowait *wait,
2472
-		     struct list_head *tx_list, u32 *count_out)
2476
+int sdma_send_txlist(struct sdma_engine *sde, struct iowait_work *wait,
2477
+		     struct list_head *tx_list, u16 *count_out)
24732478 {
24742479 	struct sdma_txreq *tx, *tx_next;
24752480 	int ret = 0;
....@@ -2480,7 +2485,7 @@
24802485 	spin_lock_irqsave(&sde->tail_lock, flags);
24812486 retry:
24822487 	list_for_each_entry_safe(tx, tx_next, tx_list, list) {
2483
-		tx->wait = wait;
2488
+		tx->wait = iowait_ioww_to_iow(wait);
24842489 		if (unlikely(!__sdma_running(sde)))
24852490 			goto unlock_noconn;
24862491 		if (unlikely(tx->num_desc > sde->desc_avail))
....@@ -2501,8 +2506,9 @@
25012506 update_tail:
25022507 	total_count = submit_count + flush_count;
25032508 	if (wait) {
2504
-		iowait_sdma_add(wait, total_count);
2505
-		iowait_starve_clear(submit_count > 0, wait);
2509
+		iowait_sdma_add(iowait_ioww_to_iow(wait), total_count);
2510
+		iowait_starve_clear(submit_count > 0,
2511
+				    iowait_ioww_to_iow(wait));
25062512 	}
25072513 	if (tail != INVALID_TAIL)
25082514 		sdma_update_tail(sde, tail);
....@@ -2512,7 +2518,7 @@
25122518 unlock_noconn:
25132519 	spin_lock(&sde->flushlist_lock);
25142520 	list_for_each_entry_safe(tx, tx_next, tx_list, list) {
2515
-		tx->wait = wait;
2521
+		tx->wait = iowait_ioww_to_iow(wait);
25162522 		list_del_init(&tx->list);
25172523 		tx->next_descq_idx = 0;
25182524 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
....@@ -2521,10 +2527,7 @@
25212527 #endif
25222528 		list_add_tail(&tx->list, &sde->flushlist);
25232529 		flush_count++;
2524
-		if (wait) {
2525
-			wait->tx_count++;
2526
-			wait->count += tx->num_desc;
2527
-		}
2530
+		iowait_inc_wait_count(wait, tx->num_desc);
25282531 	}
25292532 	spin_unlock(&sde->flushlist_lock);
25302533 	queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker);
....@@ -2583,7 +2586,7 @@
25832586 			 * 7220, e.g.
25842587 			 */
25852588 			ss->go_s99_running = 1;
2586
-			/* fall through -- and start dma engine */
2589
+			fallthrough;	/* and start dma engine */
25872590 		case sdma_event_e10_go_hw_start:
25882591 			/* This reference means the state machine is started */
25892592 			sdma_get(&sde->state);
....@@ -2725,7 +2728,6 @@
27252728 		case sdma_event_e70_go_idle:
27262729 			break;
27272730 		case sdma_event_e85_link_down:
2728
-			/* fall through */
27292731 		case sdma_event_e80_hw_freeze:
27302732 			sdma_set_state(sde, sdma_state_s80_hw_freeze);
27312733 			atomic_dec(&sde->dd->sdma_unfreeze_count);
....@@ -3006,7 +3008,7 @@
30063008 		case sdma_event_e60_hw_halted:
30073009 			need_progress = 1;
30083010 			sdma_err_progress_check_schedule(sde);
3009
-			/* fall through */
3011
+			fallthrough;
30103012 		case sdma_event_e90_sw_halted:
30113013 			/*
30123014 			* SW initiated halt does not perform engines
....@@ -3020,7 +3022,7 @@
30203022 			break;
30213023 		case sdma_event_e85_link_down:
30223024 			ss->go_s99_running = 0;
3023
-			/* fall through */
3025
+			fallthrough;
30243026 		case sdma_event_e80_hw_freeze:
30253027 			sdma_set_state(sde, sdma_state_s80_hw_freeze);
30263028 			atomic_dec(&sde->dd->sdma_unfreeze_count);
....@@ -3250,7 +3252,7 @@
32503252 		tx->num_desc++;
32513253 		tx->descs[2].qw[0] = 0;
32523254 		tx->descs[2].qw[1] = 0;
3253
-		/* FALLTHROUGH */
3255
+		fallthrough;
32543256 	case SDMA_AHG_APPLY_UPDATE2:
32553257 		tx->num_desc++;
32563258 		tx->descs[1].qw[0] = 0;