forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-01-03 2f7c68cb55ecb7331f2381deb497c27155f32faf 
 kernel/drivers/infiniband/hw/hfi1/sdma.c
....@@ -232,11 +232,11 @@
232232 static void sdma_complete(struct kref *);
233233 static void sdma_finalput(struct sdma_state *);
234234 static void sdma_get(struct sdma_state *);
235
-static void sdma_hw_clean_up_task(unsigned long);
235
+static void sdma_hw_clean_up_task(struct tasklet_struct *);
236236 static void sdma_put(struct sdma_state *);
237237 static void sdma_set_state(struct sdma_engine *, enum sdma_states);
238238 static void sdma_start_hw_clean_up(struct sdma_engine *);
239
-static void sdma_sw_clean_up_task(unsigned long);
239
+static void sdma_sw_clean_up_task(struct tasklet_struct *);
240240 static void sdma_sendctrl(struct sdma_engine *, unsigned);
241241 static void init_sdma_regs(struct sdma_engine *, u32, uint);
242242 static void sdma_process_event(
....@@ -379,7 +379,7 @@
379379 	__sdma_txclean(sde->dd, tx);
380380 	if (complete)
381381 		(*complete)(tx, res);
382
-	if (wait && iowait_sdma_dec(wait))
382
+	if (iowait_sdma_dec(wait))
383383 		iowait_drain_wakeup(wait);
384384 }
385385 
....@@ -406,6 +406,7 @@
406406 	struct sdma_txreq *txp, *txp_next;
407407 	LIST_HEAD(flushlist);
408408 	unsigned long flags;
409
+	uint seq;
409410 
410411 	/* flush from head to tail */
411412 	sdma_flush_descq(sde);
....@@ -416,6 +417,22 @@
416417 	/* flush from flush list */
417418 	list_for_each_entry_safe(txp, txp_next, &flushlist, list)
418419 		complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED);
420
+	/* wakeup QPs orphaned on the dmawait list */
421
+	do {
422
+		struct iowait *w, *nw;
423
+
424
+		seq = read_seqbegin(&sde->waitlock);
425
+		if (!list_empty(&sde->dmawait)) {
426
+			write_seqlock(&sde->waitlock);
427
+			list_for_each_entry_safe(w, nw, &sde->dmawait, list) {
428
+				if (w->wakeup) {
429
+					w->wakeup(w, SDMA_AVAIL_REASON);
430
+					list_del_init(&w->list);
431
+				}
432
+			}
433
+			write_sequnlock(&sde->waitlock);
434
+		}
435
+	} while (read_seqretry(&sde->waitlock, seq));
419436 }
420437 
421438 /*
....@@ -528,9 +545,10 @@
528545 	schedule_work(&sde->err_halt_worker);
529546 }
530547 
531
-static void sdma_hw_clean_up_task(unsigned long opaque)
548
+static void sdma_hw_clean_up_task(struct tasklet_struct *t)
532549 {
533
-	struct sdma_engine *sde = (struct sdma_engine *)opaque;
550
+	struct sdma_engine *sde = from_tasklet(sde, t,
551
+					       sdma_hw_clean_up_task);
534552 	u64 statuscsr;
535553 
536554 	while (1) {
....@@ -587,9 +605,9 @@
587605 		sdma_desc_avail(sde, sdma_descq_freecnt(sde));
588606 }
589607 
590
-static void sdma_sw_clean_up_task(unsigned long opaque)
608
+static void sdma_sw_clean_up_task(struct tasklet_struct *t)
591609 {
592
-	struct sdma_engine *sde = (struct sdma_engine *)opaque;
610
+	struct sdma_engine *sde = from_tasklet(sde, t, sdma_sw_clean_up_task);
593611 	unsigned long flags;
594612 
595613 	spin_lock_irqsave(&sde->tail_lock, flags);
....@@ -816,7 +834,7 @@
816834 struct sdma_rht_map_elem {
817835 	u32 mask;
818836 	u8 ctr;
819
-	struct sdma_engine *sde[0];
837
+	struct sdma_engine *sde[];
820838 };
821839 
822840 struct sdma_rht_node {
....@@ -831,7 +849,7 @@
831849 	.nelem_hint = NR_CPUS_HINT,
832850 	.head_offset = offsetof(struct sdma_rht_node, node),
833851 	.key_offset = offsetof(struct sdma_rht_node, cpu_id),
834
-	.key_len = FIELD_SIZEOF(struct sdma_rht_node, cpu_id),
852
+	.key_len = sizeof_field(struct sdma_rht_node, cpu_id),
835853 	.max_size = NR_CPUS,
836854 	.min_size = 8,
837855 	.automatic_shrinking = true,
....@@ -862,10 +880,10 @@
862880 	if (current->nr_cpus_allowed != 1)
863881 		goto out;
864882 
865
-	cpu_id = smp_processor_id();
866883 	rcu_read_lock();
867
-	rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu_id,
868
-					  sdma_rht_params);
884
+	cpu_id = smp_processor_id();
885
+	rht_node = rhashtable_lookup(dd->sdma_rht, &cpu_id,
886
+				     sdma_rht_params);
869887 
870888 	if (rht_node && rht_node->map[vl]) {
871889 		struct sdma_rht_map_elem *map = rht_node->map[vl];
....@@ -1312,11 +1330,13 @@
13121330 		kvfree(sde->tx_ring);
13131331 		sde->tx_ring = NULL;
13141332 	}
1315
-	spin_lock_irq(&dd->sde_map_lock);
1316
-	sdma_map_free(rcu_access_pointer(dd->sdma_map));
1317
-	RCU_INIT_POINTER(dd->sdma_map, NULL);
1318
-	spin_unlock_irq(&dd->sde_map_lock);
1319
-	synchronize_rcu();
1333
+	if (rcu_access_pointer(dd->sdma_map)) {
1334
+		spin_lock_irq(&dd->sde_map_lock);
1335
+		sdma_map_free(rcu_access_pointer(dd->sdma_map));
1336
+		RCU_INIT_POINTER(dd->sdma_map, NULL);
1337
+		spin_unlock_irq(&dd->sde_map_lock);
1338
+		synchronize_rcu();
1339
+	}
13201340 	kfree(dd->per_sdma);
13211341 	dd->per_sdma = NULL;
13221342 
....@@ -1421,6 +1441,7 @@
14211441 		seqlock_init(&sde->head_lock);
14221442 		spin_lock_init(&sde->senddmactrl_lock);
14231443 		spin_lock_init(&sde->flushlist_lock);
1444
+		seqlock_init(&sde->waitlock);
14241445 		/* insure there is always a zero bit */
14251446 		sde->ahg_bits = 0xfffffffe00000000ULL;
14261447 
....@@ -1436,11 +1457,10 @@
14361457 		sde->tail_csr =
14371458 			get_kctxt_csr_addr(dd, this_idx, SD(TAIL));
14381459 
1439
-		tasklet_init(&sde->sdma_hw_clean_up_task, sdma_hw_clean_up_task,
1440
-			     (unsigned long)sde);
1441
-
1442
-		tasklet_init(&sde->sdma_sw_clean_up_task, sdma_sw_clean_up_task,
1443
-			     (unsigned long)sde);
1460
+		tasklet_setup(&sde->sdma_hw_clean_up_task,
1461
+			      sdma_hw_clean_up_task);
1462
+		tasklet_setup(&sde->sdma_sw_clean_up_task,
1463
+			      sdma_sw_clean_up_task);
14441464 		INIT_WORK(&sde->err_halt_worker, sdma_err_halt_wait);
14451465 		INIT_WORK(&sde->flush_worker, sdma_field_flush);
14461466 
....@@ -1449,12 +1469,9 @@
14491469 		timer_setup(&sde->err_progress_check_timer,
14501470 			    sdma_err_progress_check, 0);
14511471 
1452
-		sde->descq = dma_zalloc_coherent(
1453
-			&dd->pcidev->dev,
1454
-			descq_cnt * sizeof(u64[2]),
1455
-			&sde->descq_phys,
1456
-			GFP_KERNEL
1457
-		);
1472
+		sde->descq = dma_alloc_coherent(&dd->pcidev->dev,
1473
+						descq_cnt * sizeof(u64[2]),
1474
+						&sde->descq_phys, GFP_KERNEL);
14581475 		if (!sde->descq)
14591476 			goto bail;
14601477 		sde->tx_ring =
....@@ -1467,24 +1484,18 @@
14671484 
14681485 	dd->sdma_heads_size = L1_CACHE_BYTES * num_engines;
14691486 	/* Allocate memory for DMA of head registers to memory */
1470
-	dd->sdma_heads_dma = dma_zalloc_coherent(
1471
-		&dd->pcidev->dev,
1472
-		dd->sdma_heads_size,
1473
-		&dd->sdma_heads_phys,
1474
-		GFP_KERNEL
1475
-	);
1487
+	dd->sdma_heads_dma = dma_alloc_coherent(&dd->pcidev->dev,
1488
+						dd->sdma_heads_size,
1489
+						&dd->sdma_heads_phys,
1490
+						GFP_KERNEL);
14761491 	if (!dd->sdma_heads_dma) {
14771492 		dd_dev_err(dd, "failed to allocate SendDMA head memory\n");
14781493 		goto bail;
14791494 	}
14801495 
14811496 	/* Allocate memory for pad */
1482
-	dd->sdma_pad_dma = dma_zalloc_coherent(
1483
-		&dd->pcidev->dev,
1484
-		SDMA_PAD,
1485
-		&dd->sdma_pad_phys,
1486
-		GFP_KERNEL
1487
-	);
1497
+	dd->sdma_pad_dma = dma_alloc_coherent(&dd->pcidev->dev, SDMA_PAD,
1498
+					      &dd->sdma_pad_phys, GFP_KERNEL);
14881499 	if (!dd->sdma_pad_dma) {
14891500 		dd_dev_err(dd, "failed to allocate SendDMA pad memory\n");
14901501 		goto bail;
....@@ -1626,20 +1637,18 @@
16261637 {
16271638 	switch (sdma_mapping_type(descp)) {
16281639 	case SDMA_MAP_SINGLE:
1629
-		dma_unmap_single(
1630
-			&dd->pcidev->dev,
1631
-			sdma_mapping_addr(descp),
1632
-			sdma_mapping_len(descp),
1633
-			DMA_TO_DEVICE);
1640
+		dma_unmap_single(&dd->pcidev->dev, sdma_mapping_addr(descp),
1641
+				 sdma_mapping_len(descp), DMA_TO_DEVICE);
16341642 		break;
16351643 	case SDMA_MAP_PAGE:
1636
-		dma_unmap_page(
1637
-			&dd->pcidev->dev,
1638
-			sdma_mapping_addr(descp),
1639
-			sdma_mapping_len(descp),
1640
-			DMA_TO_DEVICE);
1644
+		dma_unmap_page(&dd->pcidev->dev, sdma_mapping_addr(descp),
1645
+			       sdma_mapping_len(descp), DMA_TO_DEVICE);
16411646 		break;
16421647 	}
1648
+
1649
+	if (descp->pinning_ctx && descp->ctx_put)
1650
+		descp->ctx_put(descp->pinning_ctx);
1651
+	descp->pinning_ctx = NULL;
16431652 }
16441653 
16451654 /*
....@@ -1755,12 +1764,9 @@
17551764  */
17561765 static void sdma_desc_avail(struct sdma_engine *sde, uint avail)
17571766 {
1758
-	struct iowait *wait, *nw;
1767
+	struct iowait *wait, *nw, *twait;
17591768 	struct iowait *waits[SDMA_WAIT_BATCH_SIZE];
1760
-	uint i, n = 0, seq, max_idx = 0;
1761
-	struct sdma_txreq *stx;
1762
-	struct hfi1_ibdev *dev = &sde->dd->verbs_dev;
1763
-	u8 max_starved_cnt = 0;
1769
+	uint i, n = 0, seq, tidx = 0;
17641770 
17651771 #ifdef CONFIG_SDMA_VERBOSITY
17661772 	dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
....@@ -1769,49 +1775,50 @@
17691775 #endif
17701776 
17711777 	do {
1772
-		seq = read_seqbegin(&dev->iowait_lock);
1778
+		seq = read_seqbegin(&sde->waitlock);
17731779 		if (!list_empty(&sde->dmawait)) {
17741780 			/* at least one item */
1775
-			write_seqlock(&dev->iowait_lock);
1781
+			write_seqlock(&sde->waitlock);
17761782 			/* Harvest waiters wanting DMA descriptors */
17771783 			list_for_each_entry_safe(
17781784 					wait,
17791785 					nw,
17801786 					&sde->dmawait,
17811787 					list) {
1782
-				u16 num_desc = 0;
1788
+				u32 num_desc;
17831789 
17841790 				if (!wait->wakeup)
17851791 					continue;
17861792 				if (n == ARRAY_SIZE(waits))
17871793 					break;
1788
-				if (!list_empty(&wait->tx_head)) {
1789
-					stx = list_first_entry(
1790
-						&wait->tx_head,
1791
-						struct sdma_txreq,
1792
-						list);
1793
-					num_desc = stx->num_desc;
1794
-				}
1794
+				iowait_init_priority(wait);
1795
+				num_desc = iowait_get_all_desc(wait);
17951796 				if (num_desc > avail)
17961797 					break;
17971798 				avail -= num_desc;
1798
-				/* Find the most starved wait memeber */
1799
-				iowait_starve_find_max(wait, &max_starved_cnt,
1800
-						       n, &max_idx);
1799
+				/* Find the top-priority wait memeber */
1800
+				if (n) {
1801
+					twait = waits[tidx];
1802
+					tidx =
1803
+					    iowait_priority_update_top(wait,
1804
+								       twait,
1805
+								       n,
1806
+								       tidx);
1807
+				}
18011808 				list_del_init(&wait->list);
18021809 				waits[n++] = wait;
18031810 			}
1804
-			write_sequnlock(&dev->iowait_lock);
1811
+			write_sequnlock(&sde->waitlock);
18051812 			break;
18061813 		}
1807
-	} while (read_seqretry(&dev->iowait_lock, seq));
1814
+	} while (read_seqretry(&sde->waitlock, seq));
18081815 
1809
-	/* Schedule the most starved one first */
1816
+	/* Schedule the top-priority entry first */
18101817 	if (n)
1811
-		waits[max_idx]->wakeup(waits[max_idx], SDMA_AVAIL_REASON);
1818
+		waits[tidx]->wakeup(waits[tidx], SDMA_AVAIL_REASON);
18121819 
18131820 	for (i = 0; i < n; i++)
1814
-		if (i != max_idx)
1821
+		if (i != tidx)
18151822 			waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON);
18161823 }
18171824 
....@@ -2346,7 +2353,7 @@
23462353  */
23472354 static int sdma_check_progress(
23482355 	struct sdma_engine *sde,
2349
-	struct iowait *wait,
2356
+	struct iowait_work *wait,
23502357 	struct sdma_txreq *tx,
23512358 	bool pkts_sent)
23522359 {
....@@ -2356,12 +2363,12 @@
23562363 	if (tx->num_desc <= sde->desc_avail)
23572364 		return -EAGAIN;
23582365 	/* pulse the head_lock */
2359
-	if (wait && wait->sleep) {
2366
+	if (wait && iowait_ioww_to_iow(wait)->sleep) {
23602367 		unsigned seq;
23612368 
23622369 		seq = raw_seqcount_begin(
23632370 			(const seqcount_t *)&sde->head_lock.seqcount);
2364
-		ret = wait->sleep(sde, wait, tx, seq, pkts_sent);
2371
+		ret = wait->iow->sleep(sde, wait, tx, seq, pkts_sent);
23652372 		if (ret == -EAGAIN)
23662373 			sde->desc_avail = sdma_descq_freecnt(sde);
23672374 	} else {
....@@ -2373,7 +2380,7 @@
23732380 /**
23742381  * sdma_send_txreq() - submit a tx req to ring
23752382  * @sde: sdma engine to use
2376
- * @wait: wait structure to use when full (may be NULL)
2383
+ * @wait: SE wait structure to use when full (may be NULL)
23772384  * @tx: sdma_txreq to submit
23782385  * @pkts_sent: has any packet been sent yet?
23792386  *
....@@ -2386,7 +2393,7 @@
23862393  * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
23872394  */
23882395 int sdma_send_txreq(struct sdma_engine *sde,
2389
-		    struct iowait *wait,
2396
+		    struct iowait_work *wait,
23902397 		    struct sdma_txreq *tx,
23912398 		    bool pkts_sent)
23922399 {
....@@ -2397,7 +2404,7 @@
23972404 	/* user should have supplied entire packet */
23982405 	if (unlikely(tx->tlen))
23992406 		return -EINVAL;
2400
-	tx->wait = wait;
2407
+	tx->wait = iowait_ioww_to_iow(wait);
24012408 	spin_lock_irqsave(&sde->tail_lock, flags);
24022409 retry:
24032410 	if (unlikely(!__sdma_running(sde)))
....@@ -2406,14 +2413,14 @@
24062413 		goto nodesc;
24072414 	tail = submit_tx(sde, tx);
24082415 	if (wait)
2409
-		iowait_sdma_inc(wait);
2416
+		iowait_sdma_inc(iowait_ioww_to_iow(wait));
24102417 	sdma_update_tail(sde, tail);
24112418 unlock:
24122419 	spin_unlock_irqrestore(&sde->tail_lock, flags);
24132420 	return ret;
24142421 unlock_noconn:
24152422 	if (wait)
2416
-		iowait_sdma_inc(wait);
2423
+		iowait_sdma_inc(iowait_ioww_to_iow(wait));
24172424 	tx->next_descq_idx = 0;
24182425 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
24192426 	tx->sn = sde->tail_sn++;
....@@ -2422,10 +2429,7 @@
24222429 	spin_lock(&sde->flushlist_lock);
24232430 	list_add_tail(&tx->list, &sde->flushlist);
24242431 	spin_unlock(&sde->flushlist_lock);
2425
-	if (wait) {
2426
-		wait->tx_count++;
2427
-		wait->count += tx->num_desc;
2428
-	}
2432
+	iowait_inc_wait_count(wait, tx->num_desc);
24292433 	queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker);
24302434 	ret = -ECOMM;
24312435 	goto unlock;
....@@ -2442,9 +2446,9 @@
24422446 /**
24432447  * sdma_send_txlist() - submit a list of tx req to ring
24442448  * @sde: sdma engine to use
2445
- * @wait: wait structure to use when full (may be NULL)
2449
+ * @wait: SE wait structure to use when full (may be NULL)
24462450  * @tx_list: list of sdma_txreqs to submit
2447
- * @count: pointer to a u32 which, after return will contain the total number of
2451
+ * @count: pointer to a u16 which, after return will contain the total number of
24482452  *         sdma_txreqs removed from the tx_list. This will include sdma_txreqs
24492453  *         whose SDMA descriptors are submitted to the ring and the sdma_txreqs
24502454  *         which are added to SDMA engine flush list if the SDMA engine state is
....@@ -2467,8 +2471,8 @@
24672471  * -EINVAL - sdma_txreq incomplete, -EBUSY - no space in ring (wait == NULL)
24682472  * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
24692473  */
2470
-int sdma_send_txlist(struct sdma_engine *sde, struct iowait *wait,
2471
-		     struct list_head *tx_list, u32 *count_out)
2474
+int sdma_send_txlist(struct sdma_engine *sde, struct iowait_work *wait,
2475
+		     struct list_head *tx_list, u16 *count_out)
24722476 {
24732477 	struct sdma_txreq *tx, *tx_next;
24742478 	int ret = 0;
....@@ -2479,7 +2483,7 @@
24792483 	spin_lock_irqsave(&sde->tail_lock, flags);
24802484 retry:
24812485 	list_for_each_entry_safe(tx, tx_next, tx_list, list) {
2482
-		tx->wait = wait;
2486
+		tx->wait = iowait_ioww_to_iow(wait);
24832487 		if (unlikely(!__sdma_running(sde)))
24842488 			goto unlock_noconn;
24852489 		if (unlikely(tx->num_desc > sde->desc_avail))
....@@ -2500,8 +2504,9 @@
25002504 update_tail:
25012505 	total_count = submit_count + flush_count;
25022506 	if (wait) {
2503
-		iowait_sdma_add(wait, total_count);
2504
-		iowait_starve_clear(submit_count > 0, wait);
2507
+		iowait_sdma_add(iowait_ioww_to_iow(wait), total_count);
2508
+		iowait_starve_clear(submit_count > 0,
2509
+				    iowait_ioww_to_iow(wait));
25052510 	}
25062511 	if (tail != INVALID_TAIL)
25072512 		sdma_update_tail(sde, tail);
....@@ -2511,7 +2516,7 @@
25112516 unlock_noconn:
25122517 	spin_lock(&sde->flushlist_lock);
25132518 	list_for_each_entry_safe(tx, tx_next, tx_list, list) {
2514
-		tx->wait = wait;
2519
+		tx->wait = iowait_ioww_to_iow(wait);
25152520 		list_del_init(&tx->list);
25162521 		tx->next_descq_idx = 0;
25172522 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
....@@ -2520,10 +2525,7 @@
25202525 #endif
25212526 		list_add_tail(&tx->list, &sde->flushlist);
25222527 		flush_count++;
2523
-		if (wait) {
2524
-			wait->tx_count++;
2525
-			wait->count += tx->num_desc;
2526
-		}
2528
+		iowait_inc_wait_count(wait, tx->num_desc);
25272529 	}
25282530 	spin_unlock(&sde->flushlist_lock);
25292531 	queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker);
....@@ -2582,7 +2584,7 @@
25822584 			 * 7220, e.g.
25832585 			 */
25842586 			ss->go_s99_running = 1;
2585
-			/* fall through -- and start dma engine */
2587
+			fallthrough;	/* and start dma engine */
25862588 		case sdma_event_e10_go_hw_start:
25872589 			/* This reference means the state machine is started */
25882590 			sdma_get(&sde->state);
....@@ -2724,7 +2726,6 @@
27242726 		case sdma_event_e70_go_idle:
27252727 			break;
27262728 		case sdma_event_e85_link_down:
2727
-			/* fall through */
27282729 		case sdma_event_e80_hw_freeze:
27292730 			sdma_set_state(sde, sdma_state_s80_hw_freeze);
27302731 			atomic_dec(&sde->dd->sdma_unfreeze_count);
....@@ -3005,7 +3006,7 @@
30053006 		case sdma_event_e60_hw_halted:
30063007 			need_progress = 1;
30073008 			sdma_err_progress_check_schedule(sde);
3008
-			/* fall through */
3009
+			fallthrough;
30093010 		case sdma_event_e90_sw_halted:
30103011 			/*
30113012 			* SW initiated halt does not perform engines
....@@ -3019,7 +3020,7 @@
30193020 			break;
30203021 		case sdma_event_e85_link_down:
30213022 			ss->go_s99_running = 0;
3022
-			/* fall through */
3023
+			fallthrough;
30233024 		case sdma_event_e80_hw_freeze:
30243025 			sdma_set_state(sde, sdma_state_s80_hw_freeze);
30253026 			atomic_dec(&sde->dd->sdma_unfreeze_count);
....@@ -3168,7 +3169,7 @@
31683169 		/* Add descriptor for coalesce buffer */
31693170 		tx->desc_limit = MAX_DESC;
31703171 		return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, tx,
3171
-					 addr, tx->tlen);
3172
+					 addr, tx->tlen, NULL, NULL, NULL);
31723173 	}
31733174 
31743175 	return 1;
....@@ -3199,20 +3200,22 @@
31993200 {
32003201 	int rval = 0;
32013202 
3202
-	tx->num_desc++;
3203
-	if ((unlikely(tx->num_desc == tx->desc_limit))) {
3203
+	if ((unlikely(tx->num_desc + 1 == tx->desc_limit))) {
32043204 		rval = _extend_sdma_tx_descs(dd, tx);
32053205 		if (rval) {
32063206 			__sdma_txclean(dd, tx);
32073207 			return rval;
32083208 		}
32093209 	}
3210
+
32103211 	/* finish the one just added */
32113212 	make_tx_sdma_desc(
32123213 		tx,
32133214 		SDMA_MAP_NONE,
32143215 		dd->sdma_pad_phys,
3215
-		sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1)));
3216
+		sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1)),
3217
+		NULL, NULL, NULL);
3218
+	tx->num_desc++;
32163219 	_sdma_close_tx(dd, tx);
32173220 	return rval;
32183221 }
....@@ -3249,7 +3252,7 @@
32493252 		tx->num_desc++;
32503253 		tx->descs[2].qw[0] = 0;
32513254 		tx->descs[2].qw[1] = 0;
3252
-		/* FALLTHROUGH */
3255
+		fallthrough;
32533256 	case SDMA_AHG_APPLY_UPDATE2:
32543257 		tx->num_desc++;
32553258 		tx->descs[1].qw[0] = 0;