forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c
....@@ -1,16 +1,7 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Huawei HiNIC PCI Express Linux driver
34  * Copyright(c) 2017 Huawei Technologies Co., Ltd
4
- *
5
- * This program is free software; you can redistribute it and/or modify it
6
- * under the terms and conditions of the GNU General Public License,
7
- * version 2, as published by the Free Software Foundation.
8
- *
9
- * This program is distributed in the hope it will be useful, but WITHOUT
10
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12
- * for more details.
13
- *
145  */
156 
167 #include <linux/kernel.h>
....@@ -74,12 +65,6 @@
7465 			((void *)((cmdq_pages)->shadow_page_vaddr) \
7566 				+ (wq)->block_idx * CMDQ_BLOCK_SIZE)
7667 
77
-#define WQE_PAGE_OFF(wq, idx)   (((idx) & ((wq)->num_wqebbs_per_page - 1)) * \
78
-					(wq)->wqebb_size)
79
-
80
-#define WQE_PAGE_NUM(wq, idx)   (((idx) / ((wq)->num_wqebbs_per_page)) \
81
-					& ((wq)->num_q_pages - 1))
82
-
8368 #define WQ_PAGE_ADDR(wq, idx)           \
8469 			((wq)->shadow_block_vaddr[WQE_PAGE_NUM(wq, idx)])
8570 
....@@ -93,6 +78,17 @@
9378 		(((unsigned long)(wqe) - (unsigned long)(wq)->shadow_wqe) \
9479 			/ (wq)->max_wqe_size)
9580 
81
+static inline int WQE_PAGE_OFF(struct hinic_wq *wq, u16 idx)
82
+{
83
+	return (((idx) & ((wq)->num_wqebbs_per_page - 1))
84
+		<< (wq)->wqebb_size_shift);
85
+}
86
+
87
+static inline int WQE_PAGE_NUM(struct hinic_wq *wq, u16 idx)
88
+{
89
+	return (((idx) >> ((wq)->wqebbs_per_page_shift))
90
+		& ((wq)->num_q_pages - 1));
91
+}
9692 /**
9793  * queue_alloc_page - allocate page for Queue
9894  * @hwif: HW interface for allocating DMA
....@@ -109,8 +105,8 @@
109105 	struct pci_dev *pdev = hwif->pdev;
110106 	dma_addr_t dma_addr;
111107 
112
-	*vaddr = dma_zalloc_coherent(&pdev->dev, page_sz, &dma_addr,
113
-				     GFP_KERNEL);
108
+	*vaddr = dma_alloc_coherent(&pdev->dev, page_sz, &dma_addr,
109
+				    GFP_KERNEL);
114110 	if (!*vaddr) {
115111 		dev_err(&pdev->dev, "Failed to allocate dma for wqs page\n");
116112 		return -ENOMEM;
....@@ -196,20 +192,20 @@
196192 {
197193 	struct hinic_hwif *hwif = wqs->hwif;
198194 	struct pci_dev *pdev = hwif->pdev;
199
-	size_t size;
200195 
201
-	size = wqs->num_pages * sizeof(*wqs->page_paddr);
202
-	wqs->page_paddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
196
+	wqs->page_paddr = devm_kcalloc(&pdev->dev, wqs->num_pages,
197
+				       sizeof(*wqs->page_paddr), GFP_KERNEL);
203198 	if (!wqs->page_paddr)
204199 		return -ENOMEM;
205200 
206
-	size = wqs->num_pages * sizeof(*wqs->page_vaddr);
207
-	wqs->page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
201
+	wqs->page_vaddr = devm_kcalloc(&pdev->dev, wqs->num_pages,
202
+				       sizeof(*wqs->page_vaddr), GFP_KERNEL);
208203 	if (!wqs->page_vaddr)
209204 		goto err_page_vaddr;
210205 
211
-	size = wqs->num_pages * sizeof(*wqs->shadow_page_vaddr);
212
-	wqs->shadow_page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
206
+	wqs->shadow_page_vaddr = devm_kcalloc(&pdev->dev, wqs->num_pages,
207
+					      sizeof(*wqs->shadow_page_vaddr),
208
+					      GFP_KERNEL);
213209 	if (!wqs->shadow_page_vaddr)
214210 		goto err_page_shadow_vaddr;
215211 
....@@ -382,15 +378,14 @@
382378 {
383379 	struct hinic_hwif *hwif = wq->hwif;
384380 	struct pci_dev *pdev = hwif->pdev;
385
-	size_t size;
386381 
387
-	size = wq->num_q_pages * wq->max_wqe_size;
388
-	wq->shadow_wqe = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
382
+	wq->shadow_wqe = devm_kcalloc(&pdev->dev, wq->num_q_pages,
383
+				      wq->max_wqe_size, GFP_KERNEL);
389384 	if (!wq->shadow_wqe)
390385 		return -ENOMEM;
391386 
392
-	size = wq->num_q_pages * sizeof(wq->prod_idx);
393
-	wq->shadow_idx = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
387
+	wq->shadow_idx = devm_kcalloc(&pdev->dev, wq->num_q_pages,
388
+				      sizeof(*wq->shadow_idx), GFP_KERNEL);
394389 	if (!wq->shadow_idx)
395390 		goto err_shadow_idx;
396391 
....@@ -477,8 +472,8 @@
477472 		u64 *paddr = &wq->block_vaddr[i];
478473 		dma_addr_t dma_addr;
479474 
480
-		*vaddr = dma_zalloc_coherent(&pdev->dev, wq->wq_page_size,
481
-					     &dma_addr, GFP_KERNEL);
475
+		*vaddr = dma_alloc_coherent(&pdev->dev, wq->wq_page_size,
476
+					    &dma_addr, GFP_KERNEL);
482477 		if (!*vaddr) {
483478 			dev_err(&pdev->dev, "Failed to allocate wq page\n");
484479 			goto err_alloc_wq_pages;
....@@ -507,16 +502,17 @@
507502  * Return 0 - Success, negative - Failure
508503  **/
509504 int hinic_wq_allocate(struct hinic_wqs *wqs, struct hinic_wq *wq,
510
-		      u16 wqebb_size, u16 wq_page_size, u16 q_depth,
505
+		      u16 wqebb_size, u32 wq_page_size, u16 q_depth,
511506 		      u16 max_wqe_size)
512507 {
513508 	struct hinic_hwif *hwif = wqs->hwif;
514509 	struct pci_dev *pdev = hwif->pdev;
515510 	u16 num_wqebbs_per_page;
511
+	u16 wqebb_size_shift;
516512 	int err;
517513 
518
-	if (wqebb_size == 0) {
519
-		dev_err(&pdev->dev, "wqebb_size must be > 0\n");
514
+	if (!is_power_of_2(wqebb_size)) {
515
+		dev_err(&pdev->dev, "wqebb_size must be power of 2\n");
520516 		return -EINVAL;
521517 	}
522518 
....@@ -530,9 +526,11 @@
530526 		return -EINVAL;
531527 	}
532528 
533
-	num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) / wqebb_size;
529
+	wqebb_size_shift = ilog2(wqebb_size);
530
+	num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size)
531
+				>> wqebb_size_shift;
534532 
535
-	if (num_wqebbs_per_page & (num_wqebbs_per_page - 1)) {
533
+	if (!is_power_of_2(num_wqebbs_per_page)) {
536534 		dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n");
537535 		return -EINVAL;
538536 	}
....@@ -550,7 +548,8 @@
550548 	wq->q_depth = q_depth;
551549 	wq->max_wqe_size = max_wqe_size;
552550 	wq->num_wqebbs_per_page = num_wqebbs_per_page;
553
-
551
+	wq->wqebbs_per_page_shift = ilog2(num_wqebbs_per_page);
552
+	wq->wqebb_size_shift = wqebb_size_shift;
554553 	wq->block_vaddr = WQ_BASE_VADDR(wqs, wq);
555554 	wq->shadow_block_vaddr = WQ_BASE_ADDR(wqs, wq);
556555 	wq->block_paddr = WQ_BASE_PADDR(wqs, wq);
....@@ -600,15 +599,17 @@
600599  **/
601600 int hinic_wqs_cmdq_alloc(struct hinic_cmdq_pages *cmdq_pages,
602601 			 struct hinic_wq *wq, struct hinic_hwif *hwif,
603
-			 int cmdq_blocks, u16 wqebb_size, u16 wq_page_size,
602
+			 int cmdq_blocks, u16 wqebb_size, u32 wq_page_size,
604603 			 u16 q_depth, u16 max_wqe_size)
605604 {
606605 	struct pci_dev *pdev = hwif->pdev;
606
+	u16 num_wqebbs_per_page_shift;
607607 	u16 num_wqebbs_per_page;
608
+	u16 wqebb_size_shift;
608609 	int i, j, err = -ENOMEM;
609610 
610
-	if (wqebb_size == 0) {
611
-		dev_err(&pdev->dev, "wqebb_size must be > 0\n");
611
+	if (!is_power_of_2(wqebb_size)) {
612
+		dev_err(&pdev->dev, "wqebb_size must be power of 2\n");
612613 		return -EINVAL;
613614 	}
614615 
....@@ -622,9 +623,11 @@
622623 		return -EINVAL;
623624 	}
624625 
625
-	num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) / wqebb_size;
626
+	wqebb_size_shift = ilog2(wqebb_size);
627
+	num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size)
628
+				>> wqebb_size_shift;
626629 
627
-	if (num_wqebbs_per_page & (num_wqebbs_per_page - 1)) {
630
+	if (!is_power_of_2(num_wqebbs_per_page)) {
628631 		dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n");
629632 		return -EINVAL;
630633 	}
....@@ -636,6 +639,7 @@
636639 		dev_err(&pdev->dev, "Failed to allocate CMDQ page\n");
637640 		return err;
638641 	}
642
+	num_wqebbs_per_page_shift = ilog2(num_wqebbs_per_page);
639643 
640644 	for (i = 0; i < cmdq_blocks; i++) {
641645 		wq[i].hwif = hwif;
....@@ -647,7 +651,8 @@
647651 		wq[i].q_depth = q_depth;
648652 		wq[i].max_wqe_size = max_wqe_size;
649653 		wq[i].num_wqebbs_per_page = num_wqebbs_per_page;
650
-
654
+		wq[i].wqebbs_per_page_shift = num_wqebbs_per_page_shift;
655
+		wq[i].wqebb_size_shift = wqebb_size_shift;
651656 		wq[i].block_vaddr = CMDQ_BASE_VADDR(cmdq_pages, &wq[i]);
652657 		wq[i].shadow_block_vaddr = CMDQ_BASE_ADDR(cmdq_pages, &wq[i]);
653658 		wq[i].block_paddr = CMDQ_BASE_PADDR(cmdq_pages, &wq[i]);
....@@ -741,7 +746,7 @@
741746 
742747 	*prod_idx = MASKED_WQE_IDX(wq, atomic_read(&wq->prod_idx));
743748 
744
-	num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
749
+	num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) >> wq->wqebb_size_shift;
745750 
746751 	if (atomic_sub_return(num_wqebbs, &wq->delta) <= 0) {
747752 		atomic_add(num_wqebbs, &wq->delta);
....@@ -762,7 +767,10 @@
762767 
763768 	*prod_idx = curr_prod_idx;
764769 
765
-	if (curr_pg != end_pg) {
770
+	/* If we only have one page, still need to get shadown wqe when
771
+	 * wqe rolling-over page
772
+	 */
773
+	if (curr_pg != end_pg || end_prod_idx < *prod_idx) {
766774 		void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
767775 
768776 		copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *prod_idx);
....@@ -775,13 +783,28 @@
775783 }
776784 
777785 /**
786
+ * hinic_return_wqe - return the wqe when transmit failed
787
+ * @wq: wq to return wqe
788
+ * @wqe_size: wqe size
789
+ **/
790
+void hinic_return_wqe(struct hinic_wq *wq, unsigned int wqe_size)
791
+{
792
+	int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
793
+
794
+	atomic_sub(num_wqebbs, &wq->prod_idx);
795
+
796
+	atomic_add(num_wqebbs, &wq->delta);
797
+}
798
+
799
+/**
778800  * hinic_put_wqe - return the wqe place to use for a new wqe
779801  * @wq: wq to return wqe
780802  * @wqe_size: wqe size
781803  **/
782804 void hinic_put_wqe(struct hinic_wq *wq, unsigned int wqe_size)
783805 {
784
-	int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
806
+	int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size)
807
+			>> wq->wqebb_size_shift;
785808 
786809 	atomic_add(num_wqebbs, &wq->cons_idx);
787810 
....@@ -799,7 +822,8 @@
799822 struct hinic_hw_wqe *hinic_read_wqe(struct hinic_wq *wq, unsigned int wqe_size,
800823 				    u16 *cons_idx)
801824 {
802
-	int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
825
+	int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size)
826
+			>> wq->wqebb_size_shift;
803827 	u16 curr_cons_idx, end_cons_idx;
804828 	int curr_pg, end_pg;
805829 
....@@ -816,7 +840,10 @@
816840 
817841 	*cons_idx = curr_cons_idx;
818842 
819
-	if (curr_pg != end_pg) {
843
+	/* If we only have one page, still need to get shadown wqe when
844
+	 * wqe rolling-over page
845
+	 */
846
+	if (curr_pg != end_pg || end_cons_idx < curr_cons_idx) {
820847 		void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
821848 
822849 		copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *cons_idx);