forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/drivers/infiniband/hw/hfi1/sdma.c
....@@ -232,11 +232,11 @@
232232 static void sdma_complete(struct kref *);
233233 static void sdma_finalput(struct sdma_state *);
234234 static void sdma_get(struct sdma_state *);
235
-static void sdma_hw_clean_up_task(unsigned long);
235
+static void sdma_hw_clean_up_task(struct tasklet_struct *);
236236 static void sdma_put(struct sdma_state *);
237237 static void sdma_set_state(struct sdma_engine *, enum sdma_states);
238238 static void sdma_start_hw_clean_up(struct sdma_engine *);
239
-static void sdma_sw_clean_up_task(unsigned long);
239
+static void sdma_sw_clean_up_task(struct tasklet_struct *);
240240 static void sdma_sendctrl(struct sdma_engine *, unsigned);
241241 static void init_sdma_regs(struct sdma_engine *, u32, uint);
242242 static void sdma_process_event(
....@@ -379,7 +379,7 @@
379379 	__sdma_txclean(sde->dd, tx);
380380 	if (complete)
381381 		(*complete)(tx, res);
382
-	if (wait && iowait_sdma_dec(wait))
382
+	if (iowait_sdma_dec(wait))
383383 		iowait_drain_wakeup(wait);
384384 }
385385 
....@@ -406,6 +406,7 @@
406406 	struct sdma_txreq *txp, *txp_next;
407407 	LIST_HEAD(flushlist);
408408 	unsigned long flags;
409
+	uint seq;
409410 
410411 	/* flush from head to tail */
411412 	sdma_flush_descq(sde);
....@@ -416,6 +417,22 @@
416417 	/* flush from flush list */
417418 	list_for_each_entry_safe(txp, txp_next, &flushlist, list)
418419 		complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED);
420
+	/* wakeup QPs orphaned on the dmawait list */
421
+	do {
422
+		struct iowait *w, *nw;
423
+
424
+		seq = read_seqbegin(&sde->waitlock);
425
+		if (!list_empty(&sde->dmawait)) {
426
+			write_seqlock(&sde->waitlock);
427
+			list_for_each_entry_safe(w, nw, &sde->dmawait, list) {
428
+				if (w->wakeup) {
429
+					w->wakeup(w, SDMA_AVAIL_REASON);
430
+					list_del_init(&w->list);
431
+				}
432
+			}
433
+			write_sequnlock(&sde->waitlock);
434
+		}
435
+	} while (read_seqretry(&sde->waitlock, seq));
419436 }
420437 
421438 /*
....@@ -528,9 +545,10 @@
528545 	schedule_work(&sde->err_halt_worker);
529546 }
530547 
531
-static void sdma_hw_clean_up_task(unsigned long opaque)
548
+static void sdma_hw_clean_up_task(struct tasklet_struct *t)
532549 {
533
-	struct sdma_engine *sde = (struct sdma_engine *)opaque;
550
+	struct sdma_engine *sde = from_tasklet(sde, t,
551
+					       sdma_hw_clean_up_task);
534552 	u64 statuscsr;
535553 
536554 	while (1) {
....@@ -587,9 +605,9 @@
587605 		sdma_desc_avail(sde, sdma_descq_freecnt(sde));
588606 }
589607 
590
-static void sdma_sw_clean_up_task(unsigned long opaque)
608
+static void sdma_sw_clean_up_task(struct tasklet_struct *t)
591609 {
592
-	struct sdma_engine *sde = (struct sdma_engine *)opaque;
610
+	struct sdma_engine *sde = from_tasklet(sde, t, sdma_sw_clean_up_task);
593611 	unsigned long flags;
594612 
595613 	spin_lock_irqsave(&sde->tail_lock, flags);
....@@ -816,7 +834,7 @@
816834 struct sdma_rht_map_elem {
817835 	u32 mask;
818836 	u8 ctr;
819
-	struct sdma_engine *sde[0];
837
+	struct sdma_engine *sde[];
820838 };
821839 
822840 struct sdma_rht_node {
....@@ -831,7 +849,7 @@
831849 	.nelem_hint = NR_CPUS_HINT,
832850 	.head_offset = offsetof(struct sdma_rht_node, node),
833851 	.key_offset = offsetof(struct sdma_rht_node, cpu_id),
834
-	.key_len = FIELD_SIZEOF(struct sdma_rht_node, cpu_id),
852
+	.key_len = sizeof_field(struct sdma_rht_node, cpu_id),
835853 	.max_size = NR_CPUS,
836854 	.min_size = 8,
837855 	.automatic_shrinking = true,
....@@ -862,10 +880,10 @@
862880 	if (current->nr_cpus_allowed != 1)
863881 		goto out;
864882 
865
-	cpu_id = smp_processor_id();
866883 	rcu_read_lock();
867
-	rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu_id,
868
-					  sdma_rht_params);
884
+	cpu_id = smp_processor_id();
885
+	rht_node = rhashtable_lookup(dd->sdma_rht, &cpu_id,
886
+				     sdma_rht_params);
869887 
870888 	if (rht_node && rht_node->map[vl]) {
871889 		struct sdma_rht_map_elem *map = rht_node->map[vl];
....@@ -1312,11 +1330,13 @@
13121330 		kvfree(sde->tx_ring);
13131331 		sde->tx_ring = NULL;
13141332 	}
1315
-	spin_lock_irq(&dd->sde_map_lock);
1316
-	sdma_map_free(rcu_access_pointer(dd->sdma_map));
1317
-	RCU_INIT_POINTER(dd->sdma_map, NULL);
1318
-	spin_unlock_irq(&dd->sde_map_lock);
1319
-	synchronize_rcu();
1333
+	if (rcu_access_pointer(dd->sdma_map)) {
1334
+		spin_lock_irq(&dd->sde_map_lock);
1335
+		sdma_map_free(rcu_access_pointer(dd->sdma_map));
1336
+		RCU_INIT_POINTER(dd->sdma_map, NULL);
1337
+		spin_unlock_irq(&dd->sde_map_lock);
1338
+		synchronize_rcu();
1339
+	}
13201340 	kfree(dd->per_sdma);
13211341 	dd->per_sdma = NULL;
13221342 
....@@ -1421,6 +1441,7 @@
14211441 		seqlock_init(&sde->head_lock);
14221442 		spin_lock_init(&sde->senddmactrl_lock);
14231443 		spin_lock_init(&sde->flushlist_lock);
1444
+		seqlock_init(&sde->waitlock);
14241445 		/* insure there is always a zero bit */
14251446 		sde->ahg_bits = 0xfffffffe00000000ULL;
14261447 
....@@ -1436,11 +1457,10 @@
14361457 		sde->tail_csr =
14371458 			get_kctxt_csr_addr(dd, this_idx, SD(TAIL));
14381459 
1439
-		tasklet_init(&sde->sdma_hw_clean_up_task, sdma_hw_clean_up_task,
1440
-			     (unsigned long)sde);
1441
-
1442
-		tasklet_init(&sde->sdma_sw_clean_up_task, sdma_sw_clean_up_task,
1443
-			     (unsigned long)sde);
1460
+		tasklet_setup(&sde->sdma_hw_clean_up_task,
1461
+			      sdma_hw_clean_up_task);
1462
+		tasklet_setup(&sde->sdma_sw_clean_up_task,
1463
+			      sdma_sw_clean_up_task);
14441464 		INIT_WORK(&sde->err_halt_worker, sdma_err_halt_wait);
14451465 		INIT_WORK(&sde->flush_worker, sdma_field_flush);
14461466 
....@@ -1449,12 +1469,9 @@
14491469 		timer_setup(&sde->err_progress_check_timer,
14501470 			    sdma_err_progress_check, 0);
14511471 
1452
-		sde->descq = dma_zalloc_coherent(
1453
-			&dd->pcidev->dev,
1454
-			descq_cnt * sizeof(u64[2]),
1455
-			&sde->descq_phys,
1456
-			GFP_KERNEL
1457
-		);
1472
+		sde->descq = dma_alloc_coherent(&dd->pcidev->dev,
1473
+						descq_cnt * sizeof(u64[2]),
1474
+						&sde->descq_phys, GFP_KERNEL);
14581475 		if (!sde->descq)
14591476 			goto bail;
14601477 		sde->tx_ring =
....@@ -1467,24 +1484,18 @@
14671484 
14681485 	dd->sdma_heads_size = L1_CACHE_BYTES * num_engines;
14691486 	/* Allocate memory for DMA of head registers to memory */
1470
-	dd->sdma_heads_dma = dma_zalloc_coherent(
1471
-		&dd->pcidev->dev,
1472
-		dd->sdma_heads_size,
1473
-		&dd->sdma_heads_phys,
1474
-		GFP_KERNEL
1475
-	);
1487
+	dd->sdma_heads_dma = dma_alloc_coherent(&dd->pcidev->dev,
1488
+						dd->sdma_heads_size,
1489
+						&dd->sdma_heads_phys,
1490
+						GFP_KERNEL);
14761491 	if (!dd->sdma_heads_dma) {
14771492 		dd_dev_err(dd, "failed to allocate SendDMA head memory\n");
14781493 		goto bail;
14791494 	}
14801495 
14811496 	/* Allocate memory for pad */
1482
-	dd->sdma_pad_dma = dma_zalloc_coherent(
1483
-		&dd->pcidev->dev,
1484
-		SDMA_PAD,
1485
-		&dd->sdma_pad_phys,
1486
-		GFP_KERNEL
1487
-	);
1497
+	dd->sdma_pad_dma = dma_alloc_coherent(&dd->pcidev->dev, SDMA_PAD,
1498
+					      &dd->sdma_pad_phys, GFP_KERNEL);
14881499 	if (!dd->sdma_pad_dma) {
14891500 		dd_dev_err(dd, "failed to allocate SendDMA pad memory\n");
14901501 		goto bail;
....@@ -1755,12 +1766,9 @@
17551766  */
17561767 static void sdma_desc_avail(struct sdma_engine *sde, uint avail)
17571768 {
1758
-	struct iowait *wait, *nw;
1769
+	struct iowait *wait, *nw, *twait;
17591770 	struct iowait *waits[SDMA_WAIT_BATCH_SIZE];
1760
-	uint i, n = 0, seq, max_idx = 0;
1761
-	struct sdma_txreq *stx;
1762
-	struct hfi1_ibdev *dev = &sde->dd->verbs_dev;
1763
-	u8 max_starved_cnt = 0;
1771
+	uint i, n = 0, seq, tidx = 0;
17641772 
17651773 #ifdef CONFIG_SDMA_VERBOSITY
17661774 	dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
....@@ -1769,49 +1777,50 @@
17691777 #endif
17701778 
17711779 	do {
1772
-		seq = read_seqbegin(&dev->iowait_lock);
1780
+		seq = read_seqbegin(&sde->waitlock);
17731781 		if (!list_empty(&sde->dmawait)) {
17741782 			/* at least one item */
1775
-			write_seqlock(&dev->iowait_lock);
1783
+			write_seqlock(&sde->waitlock);
17761784 			/* Harvest waiters wanting DMA descriptors */
17771785 			list_for_each_entry_safe(
17781786 					wait,
17791787 					nw,
17801788 					&sde->dmawait,
17811789 					list) {
1782
-				u16 num_desc = 0;
1790
+				u32 num_desc;
17831791 
17841792 				if (!wait->wakeup)
17851793 					continue;
17861794 				if (n == ARRAY_SIZE(waits))
17871795 					break;
1788
-				if (!list_empty(&wait->tx_head)) {
1789
-					stx = list_first_entry(
1790
-						&wait->tx_head,
1791
-						struct sdma_txreq,
1792
-						list);
1793
-					num_desc = stx->num_desc;
1794
-				}
1796
+				iowait_init_priority(wait);
1797
+				num_desc = iowait_get_all_desc(wait);
17951798 				if (num_desc > avail)
17961799 					break;
17971800 				avail -= num_desc;
1798
-				/* Find the most starved wait memeber */
1799
-				iowait_starve_find_max(wait, &max_starved_cnt,
1800
-						       n, &max_idx);
1801
+				/* Find the top-priority wait memeber */
1802
+				if (n) {
1803
+					twait = waits[tidx];
1804
+					tidx =
1805
+					    iowait_priority_update_top(wait,
1806
+								       twait,
1807
+								       n,
1808
+								       tidx);
1809
+				}
18011810 				list_del_init(&wait->list);
18021811 				waits[n++] = wait;
18031812 			}
1804
-			write_sequnlock(&dev->iowait_lock);
1813
+			write_sequnlock(&sde->waitlock);
18051814 			break;
18061815 		}
1807
-	} while (read_seqretry(&dev->iowait_lock, seq));
1816
+	} while (read_seqretry(&sde->waitlock, seq));
18081817 
1809
-	/* Schedule the most starved one first */
1818
+	/* Schedule the top-priority entry first */
18101819 	if (n)
1811
-		waits[max_idx]->wakeup(waits[max_idx], SDMA_AVAIL_REASON);
1820
+		waits[tidx]->wakeup(waits[tidx], SDMA_AVAIL_REASON);
18121821 
18131822 	for (i = 0; i < n; i++)
1814
-		if (i != max_idx)
1823
+		if (i != tidx)
18151824 			waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON);
18161825 }
18171826 
....@@ -2346,7 +2355,7 @@
23462355  */
23472356 static int sdma_check_progress(
23482357 	struct sdma_engine *sde,
2349
-	struct iowait *wait,
2358
+	struct iowait_work *wait,
23502359 	struct sdma_txreq *tx,
23512360 	bool pkts_sent)
23522361 {
....@@ -2356,12 +2365,12 @@
23562365 	if (tx->num_desc <= sde->desc_avail)
23572366 		return -EAGAIN;
23582367 	/* pulse the head_lock */
2359
-	if (wait && wait->sleep) {
2368
+	if (wait && iowait_ioww_to_iow(wait)->sleep) {
23602369 		unsigned seq;
23612370 
23622371 		seq = raw_seqcount_begin(
23632372 			(const seqcount_t *)&sde->head_lock.seqcount);
2364
-		ret = wait->sleep(sde, wait, tx, seq, pkts_sent);
2373
+		ret = wait->iow->sleep(sde, wait, tx, seq, pkts_sent);
23652374 		if (ret == -EAGAIN)
23662375 			sde->desc_avail = sdma_descq_freecnt(sde);
23672376 	} else {
....@@ -2373,7 +2382,7 @@
23732382 /**
23742383  * sdma_send_txreq() - submit a tx req to ring
23752384  * @sde: sdma engine to use
2376
- * @wait: wait structure to use when full (may be NULL)
2385
+ * @wait: SE wait structure to use when full (may be NULL)
23772386  * @tx: sdma_txreq to submit
23782387  * @pkts_sent: has any packet been sent yet?
23792388  *
....@@ -2386,7 +2395,7 @@
23862395  * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
23872396  */
23882397 int sdma_send_txreq(struct sdma_engine *sde,
2389
-		    struct iowait *wait,
2398
+		    struct iowait_work *wait,
23902399 		    struct sdma_txreq *tx,
23912400 		    bool pkts_sent)
23922401 {
....@@ -2397,7 +2406,7 @@
23972406 	/* user should have supplied entire packet */
23982407 	if (unlikely(tx->tlen))
23992408 		return -EINVAL;
2400
-	tx->wait = wait;
2409
+	tx->wait = iowait_ioww_to_iow(wait);
24012410 	spin_lock_irqsave(&sde->tail_lock, flags);
24022411 retry:
24032412 	if (unlikely(!__sdma_running(sde)))
....@@ -2406,14 +2415,14 @@
24062415 		goto nodesc;
24072416 	tail = submit_tx(sde, tx);
24082417 	if (wait)
2409
-		iowait_sdma_inc(wait);
2418
+		iowait_sdma_inc(iowait_ioww_to_iow(wait));
24102419 	sdma_update_tail(sde, tail);
24112420 unlock:
24122421 	spin_unlock_irqrestore(&sde->tail_lock, flags);
24132422 	return ret;
24142423 unlock_noconn:
24152424 	if (wait)
2416
-		iowait_sdma_inc(wait);
2425
+		iowait_sdma_inc(iowait_ioww_to_iow(wait));
24172426 	tx->next_descq_idx = 0;
24182427 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
24192428 	tx->sn = sde->tail_sn++;
....@@ -2422,10 +2431,7 @@
24222431 	spin_lock(&sde->flushlist_lock);
24232432 	list_add_tail(&tx->list, &sde->flushlist);
24242433 	spin_unlock(&sde->flushlist_lock);
2425
-	if (wait) {
2426
-		wait->tx_count++;
2427
-		wait->count += tx->num_desc;
2428
-	}
2434
+	iowait_inc_wait_count(wait, tx->num_desc);
24292435 	queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker);
24302436 	ret = -ECOMM;
24312437 	goto unlock;
....@@ -2442,9 +2448,9 @@
24422448 /**
24432449  * sdma_send_txlist() - submit a list of tx req to ring
24442450  * @sde: sdma engine to use
2445
- * @wait: wait structure to use when full (may be NULL)
2451
+ * @wait: SE wait structure to use when full (may be NULL)
24462452  * @tx_list: list of sdma_txreqs to submit
2447
- * @count: pointer to a u32 which, after return will contain the total number of
2453
+ * @count: pointer to a u16 which, after return will contain the total number of
24482454  *         sdma_txreqs removed from the tx_list. This will include sdma_txreqs
24492455  *         whose SDMA descriptors are submitted to the ring and the sdma_txreqs
24502456  *         which are added to SDMA engine flush list if the SDMA engine state is
....@@ -2467,8 +2473,8 @@
24672473  * -EINVAL - sdma_txreq incomplete, -EBUSY - no space in ring (wait == NULL)
24682474  * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
24692475  */
2470
-int sdma_send_txlist(struct sdma_engine *sde, struct iowait *wait,
2471
-		     struct list_head *tx_list, u32 *count_out)
2476
+int sdma_send_txlist(struct sdma_engine *sde, struct iowait_work *wait,
2477
+		     struct list_head *tx_list, u16 *count_out)
24722478 {
24732479 	struct sdma_txreq *tx, *tx_next;
24742480 	int ret = 0;
....@@ -2479,7 +2485,7 @@
24792485 	spin_lock_irqsave(&sde->tail_lock, flags);
24802486 retry:
24812487 	list_for_each_entry_safe(tx, tx_next, tx_list, list) {
2482
-		tx->wait = wait;
2488
+		tx->wait = iowait_ioww_to_iow(wait);
24832489 		if (unlikely(!__sdma_running(sde)))
24842490 			goto unlock_noconn;
24852491 		if (unlikely(tx->num_desc > sde->desc_avail))
....@@ -2500,8 +2506,9 @@
25002506 update_tail:
25012507 	total_count = submit_count + flush_count;
25022508 	if (wait) {
2503
-		iowait_sdma_add(wait, total_count);
2504
-		iowait_starve_clear(submit_count > 0, wait);
2509
+		iowait_sdma_add(iowait_ioww_to_iow(wait), total_count);
2510
+		iowait_starve_clear(submit_count > 0,
2511
+				    iowait_ioww_to_iow(wait));
25052512 	}
25062513 	if (tail != INVALID_TAIL)
25072514 		sdma_update_tail(sde, tail);
....@@ -2511,7 +2518,7 @@
25112518 unlock_noconn:
25122519 	spin_lock(&sde->flushlist_lock);
25132520 	list_for_each_entry_safe(tx, tx_next, tx_list, list) {
2514
-		tx->wait = wait;
2521
+		tx->wait = iowait_ioww_to_iow(wait);
25152522 		list_del_init(&tx->list);
25162523 		tx->next_descq_idx = 0;
25172524 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
....@@ -2520,10 +2527,7 @@
25202527 #endif
25212528 		list_add_tail(&tx->list, &sde->flushlist);
25222529 		flush_count++;
2523
-		if (wait) {
2524
-			wait->tx_count++;
2525
-			wait->count += tx->num_desc;
2526
-		}
2530
+		iowait_inc_wait_count(wait, tx->num_desc);
25272531 	}
25282532 	spin_unlock(&sde->flushlist_lock);
25292533 	queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker);
....@@ -2582,7 +2586,7 @@
25822586 			 * 7220, e.g.
25832587 			 */
25842588 			ss->go_s99_running = 1;
2585
-			/* fall through -- and start dma engine */
2589
+			fallthrough;	/* and start dma engine */
25862590 		case sdma_event_e10_go_hw_start:
25872591 			/* This reference means the state machine is started */
25882592 			sdma_get(&sde->state);
....@@ -2724,7 +2728,6 @@
27242728 		case sdma_event_e70_go_idle:
27252729 			break;
27262730 		case sdma_event_e85_link_down:
2727
-			/* fall through */
27282731 		case sdma_event_e80_hw_freeze:
27292732 			sdma_set_state(sde, sdma_state_s80_hw_freeze);
27302733 			atomic_dec(&sde->dd->sdma_unfreeze_count);
....@@ -3005,7 +3008,7 @@
30053008 		case sdma_event_e60_hw_halted:
30063009 			need_progress = 1;
30073010 			sdma_err_progress_check_schedule(sde);
3008
-			/* fall through */
3011
+			fallthrough;
30093012 		case sdma_event_e90_sw_halted:
30103013 			/*
30113014 			* SW initiated halt does not perform engines
....@@ -3019,7 +3022,7 @@
30193022 			break;
30203023 		case sdma_event_e85_link_down:
30213024 			ss->go_s99_running = 0;
3022
-			/* fall through */
3025
+			fallthrough;
30233026 		case sdma_event_e80_hw_freeze:
30243027 			sdma_set_state(sde, sdma_state_s80_hw_freeze);
30253028 			atomic_dec(&sde->dd->sdma_unfreeze_count);
....@@ -3249,7 +3252,7 @@
32493252 		tx->num_desc++;
32503253 		tx->descs[2].qw[0] = 0;
32513254 		tx->descs[2].qw[1] = 0;
3252
-		/* FALLTHROUGH */
3255
+		fallthrough;
32533256 	case SDMA_AHG_APPLY_UPDATE2:
32543257 		tx->num_desc++;
32553258 		tx->descs[1].qw[0] = 0;