forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/drivers/pci/controller/dwc/pcie-dw-ep-rockchip.c
....@@ -81,7 +81,15 @@
8181 #define PCIE_CLIENT_HOT_RESET_CTRL      0x180
8282 #define PCIE_CLIENT_LTSSM_STATUS	0x300
8383 #define PCIE_CLIENT_INTR_MASK		0x24
84
+#define PCIE_LTSSM_APP_DLY1_EN		BIT(0)
85
+#define PCIE_LTSSM_APP_DLY2_EN		BIT(1)
86
+#define PCIE_LTSSM_APP_DLY1_DONE	BIT(2)
87
+#define PCIE_LTSSM_APP_DLY2_DONE	BIT(3)
8488 #define PCIE_LTSSM_ENABLE_ENHANCE       BIT(4)
89
+#define PCIE_CLIENT_MSI_GEN_CON		0x38
90
+
91
+#define PCIe_CLIENT_MSI_OBJ_IRQ		0	/* rockchip ep object special irq */
92
+
8593 #define PCIE_ELBI_REG_NUM		0x2
8694 #define PCIE_ELBI_LOCAL_BASE		0x200e00
8795 
....@@ -99,6 +107,11 @@
99107 
100108 #define PCIE_DBI_SIZE			0x400000
101109 
110
+#define PCIE_EP_OBJ_INFO_DRV_VERSION	0x00000001
111
+
112
+#define PCIE_BAR_MAX_NUM		6
113
+#define PCIE_HOTRESET_TMOUT_US		10000
114
+
102115 struct rockchip_pcie {
103116 	struct dw_pcie			pci;
104117 	void __iomem			*apb_base;
....@@ -113,13 +126,17 @@
113126 	u32				num_ib_windows;
114127 	u32				num_ob_windows;
115128 	phys_addr_t			*outbound_addr;
116
-	u8				bar_to_atu[6];
117
-	dma_addr_t			ib_target_address;
118
-	u32				ib_target_size;
119
-	void				*ib_target_base;
129
+	u8				bar_to_atu[PCIE_BAR_MAX_NUM];
130
+	dma_addr_t			ib_target_address[PCIE_BAR_MAX_NUM];
131
+	u32				ib_target_size[PCIE_BAR_MAX_NUM];
132
+	void				*ib_target_base[PCIE_BAR_MAX_NUM];
120133 	struct dma_trx_obj		*dma_obj;
121134 	struct fasync_struct		*async;
122135 	phys_addr_t			dbi_base_physical;
136
+	struct pcie_ep_obj_info		*obj_info;
137
+	enum pcie_ep_mmap_resource	cur_mmap_res;
138
+	struct workqueue_struct		*hot_rst_wq;
139
+	struct work_struct		hot_rst_work;
123140 };
124141 
125142 struct rockchip_pcie_misc_dev {
....@@ -138,6 +155,12 @@
138155 };
139156 
140157 MODULE_DEVICE_TABLE(of, rockchip_pcie_ep_of_match);
158
+
159
+static void rockchip_pcie_devmode_update(struct rockchip_pcie *rockchip, int mode, int submode)
160
+{
161
+	rockchip->obj_info->devmode.mode = mode;
162
+	rockchip->obj_info->devmode.submode = submode;
163
+}
141164 
142165 static int rockchip_pcie_readl_apb(struct rockchip_pcie *rockchip, u32 reg)
143166 {
....@@ -183,9 +206,10 @@
183206 	struct device_node *np = dev->of_node;
184207 	void *addr;
185208 	struct resource *dbi_base;
186
-	struct resource *apb_base;
187209 	struct device_node *mem;
188210 	struct resource reg;
211
+	char name[8];
212
+	int i, idx;
189213 
190214 	dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM,
191215 						"pcie-dbi");
....@@ -197,17 +221,14 @@
197221 	rockchip->pci.dbi_base = devm_ioremap_resource(dev, dbi_base);
198222 	if (IS_ERR(rockchip->pci.dbi_base))
199223 		return PTR_ERR(rockchip->pci.dbi_base);
224
+	rockchip->pci.atu_base = rockchip->pci.dbi_base + DEFAULT_DBI_ATU_OFFSET;
200225 	rockchip->dbi_base_physical = dbi_base->start;
201226 
202
-	apb_base = platform_get_resource_byname(pdev, IORESOURCE_MEM,
203
-						"pcie-apb");
204
-	if (!apb_base) {
227
+	rockchip->apb_base = devm_platform_ioremap_resource_byname(pdev, "pcie-apb");
228
+	if (!rockchip->apb_base) {
205229 		dev_err(dev, "get pcie-apb failed\n");
206230 		return -ENODEV;
207231 	}
208
-	rockchip->apb_base = devm_ioremap_resource(dev, apb_base);
209
-	if (IS_ERR(rockchip->apb_base))
210
-		return PTR_ERR(rockchip->apb_base);
211232 
212233 	rockchip->rst_gpio = devm_gpiod_get_optional(dev, "reset",
213234 						     GPIOD_OUT_HIGH);
....@@ -255,22 +276,42 @@
255276 
256277 	rockchip->outbound_addr = addr;
257278 
258
-	mem = of_parse_phandle(np, "memory-region", 0);
259
-	if (!mem) {
260
-		dev_err(dev, "missing \"memory-region\" property\n");
261
-		return -ENODEV;
279
+	for (i = 0; i < PCIE_BAR_MAX_NUM; i++) {
280
+		snprintf(name, sizeof(name), "bar%d", i);
281
+		idx = of_property_match_string(np, "memory-region-names", name);
282
+		if (idx < 0)
283
+			continue;
284
+
285
+		mem = of_parse_phandle(np, "memory-region", idx);
286
+		if (!mem) {
287
+			dev_err(dev, "missing \"memory-region\" %s property\n", name);
288
+			return -ENODEV;
289
+		}
290
+
291
+		ret = of_address_to_resource(mem, 0, &reg);
292
+		if (ret < 0) {
293
+			dev_err(dev, "missing \"reg\" %s property\n", name);
294
+			return -ENODEV;
295
+		}
296
+
297
+		rockchip->ib_target_address[i] = reg.start;
298
+		rockchip->ib_target_size[i] = resource_size(&reg);
299
+		rockchip->ib_target_base[i] = rockchip_pcie_map_kernel(reg.start,
300
+							resource_size(&reg));
301
+		dev_info(dev, "%s: assigned [0x%llx-%llx]\n", name, rockchip->ib_target_address[i],
302
+			rockchip->ib_target_address[i] + rockchip->ib_target_size[i] - 1);
262303 	}
263304 
264
-	ret = of_address_to_resource(mem, 0, &reg);
265
-	if (ret < 0) {
266
-		dev_err(dev, "missing \"reg\" property\n");
305
+	if (rockchip->ib_target_size[0]) {
306
+		rockchip->obj_info = (struct pcie_ep_obj_info *)rockchip->ib_target_base[0];
307
+		memset_io(rockchip->obj_info, 0, sizeof(struct pcie_ep_obj_info));
308
+		rockchip->obj_info->magic = PCIE_EP_OBJ_INFO_MAGIC;
309
+		rockchip->obj_info->version = PCIE_EP_OBJ_INFO_DRV_VERSION;
310
+		rockchip_pcie_devmode_update(rockchip, RKEP_MODE_KERNEL, RKEP_SMODE_INIT);
311
+	} else {
312
+		dev_err(dev, "missing bar0 memory region\n");
267313 		return -ENODEV;
268314 	}
269
-
270
-	rockchip->ib_target_address = reg.start;
271
-	rockchip->ib_target_size = resource_size(&reg);
272
-	rockchip->ib_target_base = rockchip_pcie_map_kernel(reg.start,
273
-						resource_size(&reg));
274315 
275316 	return 0;
276317 }
....@@ -485,24 +526,27 @@
485526 
486527 	resbar_base = rockchip_pci_find_resbar_capability(rockchip);
487528 
488
-	/* Resize BAR0 4M 32bits, BAR2 64M 64bits-pref */
489
-	bar = 0;
529
+	/* Resize BAR0 4M 32bits, BAR2 64M 64bits-pref, BAR4 1MB 32bits */
530
+	bar = BAR_0;
490531 	dw_pcie_writel_dbi(pci, resbar_base + 0x4 + bar * 0x8, 0xfffff0);
491532 	dw_pcie_writel_dbi(pci, resbar_base + 0x8 + bar * 0x8, 0x2c0);
492
-	rockchip_pcie_ep_set_bar_flag(rockchip, BAR_0, PCI_BASE_ADDRESS_MEM_TYPE_32);
533
+	rockchip_pcie_ep_set_bar_flag(rockchip, bar, PCI_BASE_ADDRESS_MEM_TYPE_32);
493534 
494
-	bar = 2;
535
+	bar = BAR_2;
495536 	dw_pcie_writel_dbi(pci, resbar_base + 0x4 + bar * 0x8, 0xfffff0);
496537 	dw_pcie_writel_dbi(pci, resbar_base + 0x8 + bar * 0x8, 0x6c0);
497
-	rockchip_pcie_ep_set_bar_flag(rockchip, BAR_2,
538
+	rockchip_pcie_ep_set_bar_flag(rockchip, bar,
498539 				      PCI_BASE_ADDRESS_MEM_PREFETCH | PCI_BASE_ADDRESS_MEM_TYPE_64);
499540 
500
-	/* Disable BAR1 BAR4 BAR5*/
501
-	bar = 1;
541
+	bar = BAR_4;
542
+	dw_pcie_writel_dbi(pci, resbar_base + 0x4 + bar * 0x8, 0xfffff0);
543
+	dw_pcie_writel_dbi(pci, resbar_base + 0x8 + bar * 0x8, 0xc0);
544
+	rockchip_pcie_ep_set_bar_flag(rockchip, bar, PCI_BASE_ADDRESS_MEM_TYPE_32);
545
+
546
+	/* Disable BAR1 BAR5*/
547
+	bar = BAR_1;
502548 	dw_pcie_writel_dbi(pci, PCIE_TYPE0_HDR_DBI2_OFFSET + 0x10 + bar * 4, 0);
503
-	bar = 4;
504
-	dw_pcie_writel_dbi(pci, PCIE_TYPE0_HDR_DBI2_OFFSET + 0x10 + bar * 4, 0);
505
-	bar = 5;
549
+	bar = BAR_5;
506550 	dw_pcie_writel_dbi(pci, PCIE_TYPE0_HDR_DBI2_OFFSET + 0x10 + bar * 4, 0);
507551 }
508552 
....@@ -514,14 +558,13 @@
514558 	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, 0x0580);
515559 }
516560 
517
-static int rockchip_pcie_ep_set_bar(struct rockchip_pcie *rockchip)
561
+static int rockchip_pcie_ep_set_bar(struct rockchip_pcie *rockchip, enum pci_barno bar,
562
+				    dma_addr_t cpu_addr)
518563 {
519564 	int ret;
520565 	u32 free_win;
521566 	struct dw_pcie *pci = &rockchip->pci;
522
-	enum pci_barno bar;
523567 	enum dw_pcie_as_type as_type;
524
-	dma_addr_t cpu_addr;
525568 
526569 	free_win = find_first_zero_bit(rockchip->ib_window_map,
527570 				       rockchip->num_ib_windows);
....@@ -531,10 +574,8 @@
531574 	}
532575 
533576 	as_type = DW_PCIE_AS_MEM;
534
-	bar = BAR_0;
535
-	cpu_addr = rockchip->ib_target_address;
536577 
537
-	ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr, as_type);
578
+	ret = dw_pcie_prog_inbound_atu(pci, 0, free_win, bar, cpu_addr, as_type);
538579 	if (ret < 0) {
539580 		dev_err(pci->dev, "Failed to program IB window\n");
540581 		return ret;
....@@ -544,7 +585,6 @@
544585 	set_bit(free_win, rockchip->ib_window_map);
545586 
546587 	return 0;
547
-
548588 }
549589 
550590 static void rockchip_pcie_fast_link_setup(struct rockchip_pcie *rockchip)
....@@ -553,7 +593,8 @@
553593 
554594 	/* LTSSM EN ctrl mode */
555595 	val = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_HOT_RESET_CTRL);
556
-	val |= PCIE_LTSSM_ENABLE_ENHANCE | (PCIE_LTSSM_ENABLE_ENHANCE << 16);
596
+	val |= (PCIE_LTSSM_ENABLE_ENHANCE | PCIE_LTSSM_APP_DLY2_EN) |
597
+		((PCIE_LTSSM_ENABLE_ENHANCE | PCIE_LTSSM_APP_DLY2_EN) << 16);
557598 	rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL);
558599 }
559600 
....@@ -571,98 +612,117 @@
571612 static void rockchip_pcie_local_elbi_enable(struct rockchip_pcie *rockchip)
572613 {
573614 	int i;
574
-	u32 dlbi_reg;
615
+	u32 elbi_reg;
575616 	struct dw_pcie *pci = &rockchip->pci;
576617 
577618 	for (i = 0; i < PCIE_ELBI_REG_NUM; i++) {
578
-		dlbi_reg = PCIE_ELBI_LOCAL_BASE + PCIE_ELBI_LOCAL_ENABLE_OFF +
619
+		elbi_reg = PCIE_ELBI_LOCAL_BASE + PCIE_ELBI_LOCAL_ENABLE_OFF +
579620 			   i * 4;
580
-		dw_pcie_writel_dbi(pci, dlbi_reg, 0xffff0000);
621
+		dw_pcie_writel_dbi(pci, elbi_reg, 0xffff0000);
581622 	}
582623 }
583624 
584625 static void rockchip_pcie_elbi_clear(struct rockchip_pcie *rockchip)
585626 {
586627 	int i;
587
-	u32 dlbi_reg;
628
+	u32 elbi_reg;
588629 	struct dw_pcie *pci = &rockchip->pci;
589630 	u32 val;
590631 
591632 	for (i = 0; i < PCIE_ELBI_REG_NUM; i++) {
592
-		dlbi_reg = PCIE_ELBI_LOCAL_BASE + i * 4;
593
-		val = dw_pcie_readl_dbi(pci, dlbi_reg);
633
+		elbi_reg = PCIE_ELBI_LOCAL_BASE + i * 4;
634
+		val = dw_pcie_readl_dbi(pci, elbi_reg);
594635 		val <<= 16;
595
-		dw_pcie_writel_dbi(pci, dlbi_reg, val);
636
+		dw_pcie_writel_dbi(pci, elbi_reg, val);
596637 	}
638
+}
639
+
640
+static void rockchip_pcie_raise_msi_irq(struct rockchip_pcie *rockchip, u8 interrupt_num)
641
+{
642
+	rockchip_pcie_writel_apb(rockchip, BIT(interrupt_num), PCIE_CLIENT_MSI_GEN_CON);
597643 }
598644 
599645 static irqreturn_t rockchip_pcie_sys_irq_handler(int irq, void *arg)
600646 {
601647 	struct rockchip_pcie *rockchip = arg;
602648 	struct dw_pcie *pci = &rockchip->pci;
603
-	u32 dlbi_reg;
649
+	u32 elbi_reg;
604650 	u32 chn;
605
-	union int_status status;
651
+	union int_status wr_status, rd_status;
606652 	union int_clear clears;
607
-	u32 reg, val;
653
+	u32 reg, mask;
654
+	bool sigio = false;
608655 
609656 	/* ELBI helper, only check the valid bits, and discard the rest interrupts */
610
-	dlbi_reg = dw_pcie_readl_dbi(pci, PCIE_ELBI_LOCAL_BASE + PCIE_ELBI_APP_ELBI_INT_GEN0);
611
-	if (dlbi_reg & PCIE_ELBI_APP_ELBI_INT_GEN0_SIGIO) {
612
-		dev_dbg(rockchip->pci.dev, "SIGIO\n");
613
-		kill_fasync(&rockchip->async, SIGIO, POLL_IN);
657
+	elbi_reg = dw_pcie_readl_dbi(pci, PCIE_ELBI_LOCAL_BASE + PCIE_ELBI_APP_ELBI_INT_GEN0);
658
+	if (elbi_reg & PCIE_ELBI_APP_ELBI_INT_GEN0_SIGIO) {
659
+		sigio = true;
660
+		rockchip->obj_info->irq_type_ep = OBJ_IRQ_ELBI;
661
+		rockchip_pcie_elbi_clear(rockchip);
662
+		goto out;
614663 	}
615664 
616
-	rockchip_pcie_elbi_clear(rockchip);
617
-
618665 	/* DMA helper */
619
-	status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET +
620
-					   PCIE_DMA_WR_INT_STATUS);
666
+	mask = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_MASK);
667
+	wr_status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_STATUS) & (~mask);
668
+	mask = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_MASK);
669
+	rd_status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_STATUS) & (~mask);
670
+
621671 	for (chn = 0; chn < PCIE_DMA_CHANEL_MAX_NUM; chn++) {
622
-		if (status.donesta & BIT(chn)) {
623
-			clears.doneclr = 0x1 << chn;
672
+		if (wr_status.donesta & BIT(chn)) {
673
+			clears.doneclr = BIT(chn);
624674 			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
625675 					PCIE_DMA_WR_INT_CLEAR, clears.asdword);
626676 			if (rockchip->dma_obj && rockchip->dma_obj->cb)
627677 				rockchip->dma_obj->cb(rockchip->dma_obj, chn, DMA_TO_BUS);
628678 		}
629679 
630
-		if (status.abortsta & BIT(chn)) {
680
+		if (wr_status.abortsta & BIT(chn)) {
631681 			dev_err(pci->dev, "%s, abort\n", __func__);
632
-			clears.abortclr = 0x1 << chn;
682
+			clears.abortclr = BIT(chn);
633683 			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
634684 					PCIE_DMA_WR_INT_CLEAR, clears.asdword);
635685 		}
636686 	}
637687 
638
-	status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET +
639
-					   PCIE_DMA_RD_INT_STATUS);
640688 	for (chn = 0; chn < PCIE_DMA_CHANEL_MAX_NUM; chn++) {
641
-		if (status.donesta & BIT(chn)) {
642
-			clears.doneclr = 0x1 << chn;
689
+		if (rd_status.donesta & BIT(chn)) {
690
+			clears.doneclr = BIT(chn);
643691 			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
644692 					PCIE_DMA_RD_INT_CLEAR, clears.asdword);
645693 			if (rockchip->dma_obj && rockchip->dma_obj->cb)
646694 				rockchip->dma_obj->cb(rockchip->dma_obj, chn, DMA_FROM_BUS);
647695 		}
648696 
649
-		if (status.abortsta & BIT(chn)) {
697
+		if (rd_status.abortsta & BIT(chn)) {
650698 			dev_err(pci->dev, "%s, abort\n", __func__);
651
-			clears.abortclr = 0x1 << chn;
699
+			clears.abortclr = BIT(chn);
652700 			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
653701 					PCIE_DMA_RD_INT_CLEAR, clears.asdword);
654702 		}
655703 	}
656704 
657
-	reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
658
-	if (reg & BIT(2)) {
659
-		/* Setup command register */
660
-		val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
661
-		val &= 0xffff0000;
662
-		val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY |
663
-		       PCI_COMMAND_MASTER | PCI_COMMAND_SERR;
664
-		dw_pcie_writel_dbi(pci, PCI_COMMAND, val);
705
+	if (wr_status.asdword || rd_status.asdword) {
706
+		rockchip->obj_info->irq_type_rc = OBJ_IRQ_DMA;
707
+		rockchip->obj_info->dma_status_rc.wr |= wr_status.asdword;
708
+		rockchip->obj_info->dma_status_rc.rd |= rd_status.asdword;
709
+		rockchip_pcie_raise_msi_irq(rockchip, PCIe_CLIENT_MSI_OBJ_IRQ);
710
+
711
+		rockchip->obj_info->irq_type_ep = OBJ_IRQ_DMA;
712
+		rockchip->obj_info->dma_status_ep.wr |= wr_status.asdword;
713
+		rockchip->obj_info->dma_status_ep.rd |= rd_status.asdword;
714
+		sigio = true;
665715 	}
716
+
717
+out:
718
+	if (sigio) {
719
+		dev_dbg(rockchip->pci.dev, "SIGIO\n");
720
+		kill_fasync(&rockchip->async, SIGIO, POLL_IN);
721
+	}
722
+
723
+	reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
724
+	if (reg & BIT(2))
725
+		queue_work(rockchip->hot_rst_wq, &rockchip->hot_rst_work);
666726 
667727 	rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC);
668728 
....@@ -706,7 +766,7 @@
706766 	if (!rockchip_pcie_udma_enabled(rockchip))
707767 		return 0;
708768 
709
-	rockchip->dma_obj = pcie_dw_dmatest_register(pci, true);
769
+	rockchip->dma_obj = pcie_dw_dmatest_register(pci->dev, true);
710770 	if (IS_ERR(rockchip->dma_obj)) {
711771 		dev_err(rockchip->pci.dev, "failed to prepare dmatest\n");
712772 		return -EINVAL;
....@@ -812,6 +872,73 @@
812872 	table->start.chnl = table->chn;
813873 }
814874 
875
+static void rockchip_pcie_hot_rst_work(struct work_struct *work)
876
+{
877
+	struct rockchip_pcie *rockchip = container_of(work, struct rockchip_pcie, hot_rst_work);
878
+	u32 status;
879
+	int ret;
880
+
881
+	if (rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_HOT_RESET_CTRL) & PCIE_LTSSM_APP_DLY2_EN) {
882
+		ret = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_LTSSM_STATUS,
883
+			 status, ((status & 0x3F) == 0), 100, PCIE_HOTRESET_TMOUT_US);
884
+		if (ret)
885
+			dev_err(rockchip->pci.dev, "wait for detect quiet failed!\n");
886
+
887
+		rockchip_pcie_writel_apb(rockchip, (PCIE_LTSSM_APP_DLY2_DONE) | ((PCIE_LTSSM_APP_DLY2_DONE) << 16),
888
+					PCIE_CLIENT_HOT_RESET_CTRL);
889
+	}
890
+}
891
+
892
+static int rockchip_pcie_get_dma_status(struct dma_trx_obj *obj, u8 chn, enum dma_dir dir)
893
+{
894
+	struct rockchip_pcie *rockchip = dev_get_drvdata(obj->dev);
895
+	struct dw_pcie *pci = &rockchip->pci;
896
+	union int_status status;
897
+	union int_clear clears;
898
+	int ret = 0;
899
+
900
+	dev_dbg(pci->dev, "%s %x %x\n", __func__,
901
+		dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_STATUS),
902
+		dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_STATUS));
903
+
904
+	if (dir == DMA_TO_BUS) {
905
+		status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_STATUS);
906
+		if (status.donesta & BIT(chn)) {
907
+			clears.doneclr = BIT(chn);
908
+			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
909
+					   PCIE_DMA_WR_INT_CLEAR, clears.asdword);
910
+			ret = 1;
911
+		}
912
+
913
+		if (status.abortsta & BIT(chn)) {
914
+			dev_err(pci->dev, "%s, write abort\n", __func__);
915
+			clears.abortclr = BIT(chn);
916
+			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
917
+					   PCIE_DMA_WR_INT_CLEAR, clears.asdword);
918
+			ret = -1;
919
+		}
920
+	} else {
921
+		status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_STATUS);
922
+
923
+		if (status.donesta & BIT(chn)) {
924
+			clears.doneclr = BIT(chn);
925
+			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
926
+					   PCIE_DMA_RD_INT_CLEAR, clears.asdword);
927
+			ret = 1;
928
+		}
929
+
930
+		if (status.abortsta & BIT(chn)) {
931
+			dev_err(pci->dev, "%s, read abort %x\n", __func__, status.asdword);
932
+			clears.abortclr = BIT(chn);
933
+			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
934
+					   PCIE_DMA_RD_INT_CLEAR, clears.asdword);
935
+			ret = -1;
936
+		}
937
+	}
938
+
939
+	return ret;
940
+}
941
+
815942 static const struct dw_pcie_ops dw_pcie_ops = {
816943 	.start_link = rockchip_pcie_start_link,
817944 	.link_up = rockchip_pcie_link_up,
....@@ -847,6 +974,7 @@
847974 	struct pcie_ep_dma_cache_cfg cfg;
848975 	void __user *uarg = (void __user *)arg;
849976 	int i, ret;
977
+	enum pcie_ep_mmap_resource mmap_res;
850978 
851979 	switch (cmd) {
852980 	case PCIE_DMA_GET_ELBI_DATA:
....@@ -885,6 +1013,33 @@
8851013 		dw_pcie_writel_dbi(&rockchip->pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_MASK,
8861014 				   0xffffffff);
8871015 		break;
1016
+	case PCIE_DMA_RAISE_MSI_OBJ_IRQ_USER:
1017
+		rockchip->obj_info->irq_type_rc = OBJ_IRQ_USER;
1018
+		rockchip_pcie_raise_msi_irq(rockchip, PCIe_CLIENT_MSI_OBJ_IRQ);
1019
+		break;
1020
+	case PCIE_EP_GET_USER_INFO:
1021
+		msg.bar0_phys_addr = rockchip->ib_target_address[0];
1022
+
1023
+		ret = copy_to_user(uarg, &msg, sizeof(msg));
1024
+		if (ret) {
1025
+			dev_err(rockchip->pci.dev, "failed to get elbi data\n");
1026
+			return -EFAULT;
1027
+		}
1028
+		break;
1029
+	case PCIE_EP_SET_MMAP_RESOURCE:
1030
+		ret = copy_from_user(&mmap_res, uarg, sizeof(mmap_res));
1031
+		if (ret) {
1032
+			dev_err(rockchip->pci.dev, "failed to get copy from\n");
1033
+			return -EFAULT;
1034
+		}
1035
+
1036
+		if (mmap_res >= PCIE_EP_MMAP_RESOURCE_MAX) {
1037
+			dev_err(rockchip->pci.dev, "mmap index %d is out of number\n", mmap_res);
1038
+			return -EINVAL;
1039
+		}
1040
+
1041
+		rockchip->cur_mmap_res = mmap_res;
1042
+		break;
8881043 	default:
8891044 		break;
8901045 	}
....@@ -896,18 +1051,45 @@
8961051 	struct rockchip_pcie *rockchip = (struct rockchip_pcie *)file->private_data;
8971052 	size_t size = vma->vm_end - vma->vm_start;
8981053 	int err;
1054
+	unsigned long addr;
8991055 
900
-	if (size > PCIE_DBI_SIZE) {
901
-		dev_warn(rockchip->pci.dev, "mmap size is out of limitation\n");
1056
+	switch (rockchip->cur_mmap_res) {
1057
+	case PCIE_EP_MMAP_RESOURCE_DBI:
1058
+		if (size > PCIE_DBI_SIZE) {
1059
+			dev_warn(rockchip->pci.dev, "dbi mmap size is out of limitation\n");
1060
+			return -EINVAL;
1061
+		}
1062
+		addr = rockchip->dbi_base_physical;
1063
+		break;
1064
+	case PCIE_EP_MMAP_RESOURCE_BAR0:
1065
+		if (size > rockchip->ib_target_size[0]) {
1066
+			dev_warn(rockchip->pci.dev, "bar0 mmap size is out of limitation\n");
1067
+			return -EINVAL;
1068
+		}
1069
+		addr = rockchip->ib_target_address[0];
1070
+		break;
1071
+	case PCIE_EP_MMAP_RESOURCE_BAR2:
1072
+		if (size > rockchip->ib_target_size[2]) {
1073
+			dev_warn(rockchip->pci.dev, "bar2 mmap size is out of limitation\n");
1074
+			return -EINVAL;
1075
+		}
1076
+		addr = rockchip->ib_target_address[2];
1077
+		break;
1078
+	default:
1079
+		dev_err(rockchip->pci.dev, "cur mmap_res %d is unsurreport\n", rockchip->cur_mmap_res);
9021080 		return -EINVAL;
9031081 	}
9041082 
9051083 	vma->vm_flags |= VM_IO;
9061084 	vma->vm_flags |= (VM_DONTEXPAND | VM_DONTDUMP);
907
-	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1085
+
1086
+	if (rockchip->cur_mmap_res == PCIE_EP_MMAP_RESOURCE_BAR2)
1087
+		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1088
+	else
1089
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
9081090 
9091091 	err = remap_pfn_range(vma, vma->vm_start,
910
-			      __phys_to_pfn(rockchip->dbi_base_physical),
1092
+			      __phys_to_pfn(addr),
9111093 			      size, vma->vm_page_prot);
9121094 	if (err)
9131095 		return -EAGAIN;
....@@ -941,13 +1123,13 @@
9411123 
9421124 	ret = misc_register(&pcie_dev->dev);
9431125 	if (ret) {
944
-		pr_err("pcie: failed to register misc device.\n");
1126
+		dev_err(rockchip->pci.dev, "pcie: failed to register misc device.\n");
9451127 		return ret;
9461128 	}
9471129 
9481130 	pcie_dev->pcie = rockchip;
9491131 
950
-	pr_info("register misc device pcie-dev\n");
1132
+	dev_info(rockchip->pci.dev, "register misc device pcie_ep\n");
9511133 
9521134 	return 0;
9531135 }
....@@ -957,7 +1139,8 @@
9571139 	struct device *dev = &pdev->dev;
9581140 	struct rockchip_pcie *rockchip;
9591141 	int ret;
960
-	int retry;
1142
+	int retry, i;
1143
+	u32 reg;
9611144 
9621145 	rockchip = devm_kzalloc(dev, sizeof(*rockchip), GFP_KERNEL);
9631146 	if (!rockchip)
....@@ -993,8 +1176,10 @@
9931176 		goto disable_regulator;
9941177 
9951178 	if (dw_pcie_link_up(&rockchip->pci)) {
996
-		pr_info("%s, %d, already linkup\n", __func__, __LINE__);
1179
+		dev_info(dev, "already linkup\n");
9971180 		goto already_linkup;
1181
+	} else {
1182
+		dev_info(dev, "initial\n");
9981183 	}
9991184 
10001185 	ret = rockchip_pcie_phy_init(rockchip);
....@@ -1015,6 +1200,27 @@
10151200 	rockchip_pcie_fast_link_setup(rockchip);
10161201 
10171202 	rockchip_pcie_start_link(&rockchip->pci);
1203
+	rockchip_pcie_devmode_update(rockchip, RKEP_MODE_KERNEL, RKEP_SMODE_LNKRDY);
1204
+
1205
+	rockchip->hot_rst_wq = create_singlethread_workqueue("rkep_hot_rst_wq");
1206
+	if (!rockchip->hot_rst_wq) {
1207
+		dev_err(dev, "failed to create hot_rst workqueue\n");
1208
+		ret = -ENOMEM;
1209
+		goto deinit_phy;
1210
+	}
1211
+	INIT_WORK(&rockchip->hot_rst_work, rockchip_pcie_hot_rst_work);
1212
+
1213
+	reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
1214
+	if ((reg & BIT(2)) &&
1215
+	    (rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_HOT_RESET_CTRL) & PCIE_LTSSM_APP_DLY2_EN)) {
1216
+		rockchip_pcie_writel_apb(rockchip, PCIE_LTSSM_APP_DLY2_DONE | (PCIE_LTSSM_APP_DLY2_DONE << 16),
1217
+					 PCIE_CLIENT_HOT_RESET_CTRL);
1218
+		dev_info(dev, "hot reset ever\n");
1219
+	}
1220
+	rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC);
1221
+
1222
+	/* Enable client reset or link down interrupt */
1223
+	rockchip_pcie_writel_apb(rockchip, 0x40000, PCIE_CLIENT_INTR_MASK);
10181224 
10191225 	for (retry = 0; retry < 10000; retry++) {
10201226 		if (dw_pcie_link_up(&rockchip->pci)) {
....@@ -1042,8 +1248,11 @@
10421248 	}
10431249 
10441250 already_linkup:
1251
+	rockchip_pcie_devmode_update(rockchip, RKEP_MODE_KERNEL, RKEP_SMODE_LNKUP);
10451252 	rockchip->pci.iatu_unroll_enabled = rockchip_pcie_iatu_unroll_enabled(&rockchip->pci);
1046
-	rockchip_pcie_ep_set_bar(rockchip);
1253
+	for (i = 0; i < PCIE_BAR_MAX_NUM; i++)
1254
+		if (rockchip->ib_target_size[i])
1255
+			rockchip_pcie_ep_set_bar(rockchip, i, rockchip->ib_target_address[i]);
10471256 
10481257 	ret = rockchip_pcie_init_dma_trx(rockchip);
10491258 	if (ret) {
....@@ -1054,6 +1263,7 @@
10541263 	if (rockchip->dma_obj) {
10551264 		rockchip->dma_obj->start_dma_func = rockchip_pcie_start_dma_dwc;
10561265 		rockchip->dma_obj->config_dma_func = rockchip_pcie_config_dma_dwc;
1266
+		rockchip->dma_obj->get_dma_status = rockchip_pcie_get_dma_status;
10571267 	}
10581268 
10591269 	/* Enable client ELBI interrupt */
....@@ -1086,9 +1296,10 @@
10861296 		.of_match_table = rockchip_pcie_ep_of_match,
10871297 		.suppress_bind_attrs = true,
10881298 	},
1299
+	.probe = rockchip_pcie_ep_probe,
10891300 };
10901301 
1091
-module_platform_driver_probe(rk_plat_pcie_driver, rockchip_pcie_ep_probe);
1302
+module_platform_driver(rk_plat_pcie_driver);
10921303 
10931304 MODULE_AUTHOR("Simon Xue <xxm@rock-chips.com>");
10941305 MODULE_DESCRIPTION("RockChip PCIe Controller EP driver");