blame | history | patch | commit | commitdiff
hc
2023-11-06 e3e12f52b214121840b44c91de5b3e5af5d3eb84 
 kernel/drivers/pci/controller/dwc/pcie-dw-ep-rockchip.c
....@@ -10,6 +10,7 @@
1010 
1111 #include <linux/clk.h>
1212 #include <linux/gpio/consumer.h>
13
+#include <linux/iopoll.h>
1314 #include <linux/miscdevice.h>
1415 #include <linux/mfd/syscon.h>
1516 #include <linux/module.h>
....@@ -81,7 +82,15 @@
8182 #define PCIE_CLIENT_HOT_RESET_CTRL      0x180
8283 #define PCIE_CLIENT_LTSSM_STATUS	0x300
8384 #define PCIE_CLIENT_INTR_MASK		0x24
85
+#define PCIE_LTSSM_APP_DLY1_EN		BIT(0)
86
+#define PCIE_LTSSM_APP_DLY2_EN		BIT(1)
87
+#define PCIE_LTSSM_APP_DLY1_DONE	BIT(2)
88
+#define PCIE_LTSSM_APP_DLY2_DONE	BIT(3)
8489 #define PCIE_LTSSM_ENABLE_ENHANCE       BIT(4)
90
+#define PCIE_CLIENT_MSI_GEN_CON		0x38
91
+
92
+#define PCIe_CLIENT_MSI_OBJ_IRQ		0	/* rockchip ep object special irq */
93
+
8594 #define PCIE_ELBI_REG_NUM		0x2
8695 #define PCIE_ELBI_LOCAL_BASE		0x200e00
8796 
....@@ -99,6 +108,11 @@
99108 
100109 #define PCIE_DBI_SIZE			0x400000
101110 
111
+#define PCIE_EP_OBJ_INFO_DRV_VERSION	0x00000001
112
+
113
+#define PCIE_BAR_MAX_NUM		6
114
+#define PCIE_HOTRESET_TMOUT_US		10000
115
+
102116 struct rockchip_pcie {
103117 	struct dw_pcie			pci;
104118 	void __iomem			*apb_base;
....@@ -113,13 +127,17 @@
113127 	u32				num_ib_windows;
114128 	u32				num_ob_windows;
115129 	phys_addr_t			*outbound_addr;
116
-	u8				bar_to_atu[6];
117
-	dma_addr_t			ib_target_address;
118
-	u32				ib_target_size;
119
-	void				*ib_target_base;
130
+	u8				bar_to_atu[PCIE_BAR_MAX_NUM];
131
+	dma_addr_t			ib_target_address[PCIE_BAR_MAX_NUM];
132
+	u32				ib_target_size[PCIE_BAR_MAX_NUM];
133
+	void				*ib_target_base[PCIE_BAR_MAX_NUM];
120134 	struct dma_trx_obj		*dma_obj;
121135 	struct fasync_struct		*async;
122136 	phys_addr_t			dbi_base_physical;
137
+	struct pcie_ep_obj_info		*obj_info;
138
+	enum pcie_ep_mmap_resource	cur_mmap_res;
139
+	struct workqueue_struct		*hot_rst_wq;
140
+	struct work_struct		hot_rst_work;
123141 };
124142 
125143 struct rockchip_pcie_misc_dev {
....@@ -138,6 +156,12 @@
138156 };
139157 
140158 MODULE_DEVICE_TABLE(of, rockchip_pcie_ep_of_match);
159
+
160
+static void rockchip_pcie_devmode_update(struct rockchip_pcie *rockchip, int mode, int submode)
161
+{
162
+	rockchip->obj_info->devmode.mode = mode;
163
+	rockchip->obj_info->devmode.submode = submode;
164
+}
141165 
142166 static int rockchip_pcie_readl_apb(struct rockchip_pcie *rockchip, u32 reg)
143167 {
....@@ -186,6 +210,8 @@
186210 	struct resource *apb_base;
187211 	struct device_node *mem;
188212 	struct resource reg;
213
+	char name[8];
214
+	int i, idx;
189215 
190216 	dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM,
191217 						"pcie-dbi");
....@@ -255,22 +281,42 @@
255281 
256282 	rockchip->outbound_addr = addr;
257283 
258
-	mem = of_parse_phandle(np, "memory-region", 0);
259
-	if (!mem) {
260
-		dev_err(dev, "missing \"memory-region\" property\n");
261
-		return -ENODEV;
284
+	for (i = 0; i < PCIE_BAR_MAX_NUM; i++) {
285
+		snprintf(name, sizeof(name), "bar%d", i);
286
+		idx = of_property_match_string(np, "memory-region-names", name);
287
+		if (idx < 0)
288
+			continue;
289
+
290
+		mem = of_parse_phandle(np, "memory-region", idx);
291
+		if (!mem) {
292
+			dev_err(dev, "missing \"memory-region\" %s property\n", name);
293
+			return -ENODEV;
294
+		}
295
+
296
+		ret = of_address_to_resource(mem, 0, &reg);
297
+		if (ret < 0) {
298
+			dev_err(dev, "missing \"reg\" %s property\n", name);
299
+			return -ENODEV;
300
+		}
301
+
302
+		rockchip->ib_target_address[i] = reg.start;
303
+		rockchip->ib_target_size[i] = resource_size(&reg);
304
+		rockchip->ib_target_base[i] = rockchip_pcie_map_kernel(reg.start,
305
+							resource_size(&reg));
306
+		dev_info(dev, "%s: assigned [0x%llx-%llx]\n", name, rockchip->ib_target_address[i],
307
+			rockchip->ib_target_address[i] + rockchip->ib_target_size[i] - 1);
262308 	}
263309 
264
-	ret = of_address_to_resource(mem, 0, &reg);
265
-	if (ret < 0) {
266
-		dev_err(dev, "missing \"reg\" property\n");
310
+	if (rockchip->ib_target_size[0]) {
311
+		rockchip->obj_info = (struct pcie_ep_obj_info *)rockchip->ib_target_base[0];
312
+		memset_io(rockchip->obj_info, 0, sizeof(struct pcie_ep_obj_info));
313
+		rockchip->obj_info->magic = PCIE_EP_OBJ_INFO_MAGIC;
314
+		rockchip->obj_info->version = PCIE_EP_OBJ_INFO_DRV_VERSION;
315
+		rockchip_pcie_devmode_update(rockchip, RKEP_MODE_KERNEL, RKEP_SMODE_INIT);
316
+	} else {
317
+		dev_err(dev, "missing bar0 memory region\n");
267318 		return -ENODEV;
268319 	}
269
-
270
-	rockchip->ib_target_address = reg.start;
271
-	rockchip->ib_target_size = resource_size(&reg);
272
-	rockchip->ib_target_base = rockchip_pcie_map_kernel(reg.start,
273
-						resource_size(&reg));
274320 
275321 	return 0;
276322 }
....@@ -485,24 +531,27 @@
485531 
486532 	resbar_base = rockchip_pci_find_resbar_capability(rockchip);
487533 
488
-	/* Resize BAR0 4M 32bits, BAR2 64M 64bits-pref */
489
-	bar = 0;
534
+	/* Resize BAR0 4M 32bits, BAR2 64M 64bits-pref, BAR4 1MB 32bits */
535
+	bar = BAR_0;
490536 	dw_pcie_writel_dbi(pci, resbar_base + 0x4 + bar * 0x8, 0xfffff0);
491537 	dw_pcie_writel_dbi(pci, resbar_base + 0x8 + bar * 0x8, 0x2c0);
492
-	rockchip_pcie_ep_set_bar_flag(rockchip, BAR_0, PCI_BASE_ADDRESS_MEM_TYPE_32);
538
+	rockchip_pcie_ep_set_bar_flag(rockchip, bar, PCI_BASE_ADDRESS_MEM_TYPE_32);
493539 
494
-	bar = 2;
540
+	bar = BAR_2;
495541 	dw_pcie_writel_dbi(pci, resbar_base + 0x4 + bar * 0x8, 0xfffff0);
496542 	dw_pcie_writel_dbi(pci, resbar_base + 0x8 + bar * 0x8, 0x6c0);
497
-	rockchip_pcie_ep_set_bar_flag(rockchip, BAR_2,
543
+	rockchip_pcie_ep_set_bar_flag(rockchip, bar,
498544 				      PCI_BASE_ADDRESS_MEM_PREFETCH | PCI_BASE_ADDRESS_MEM_TYPE_64);
499545 
500
-	/* Disable BAR1 BAR4 BAR5*/
501
-	bar = 1;
546
+	bar = BAR_4;
547
+	dw_pcie_writel_dbi(pci, resbar_base + 0x4 + bar * 0x8, 0xfffff0);
548
+	dw_pcie_writel_dbi(pci, resbar_base + 0x8 + bar * 0x8, 0xc0);
549
+	rockchip_pcie_ep_set_bar_flag(rockchip, bar, PCI_BASE_ADDRESS_MEM_TYPE_32);
550
+
551
+	/* Disable BAR1 BAR5*/
552
+	bar = BAR_1;
502553 	dw_pcie_writel_dbi(pci, PCIE_TYPE0_HDR_DBI2_OFFSET + 0x10 + bar * 4, 0);
503
-	bar = 4;
504
-	dw_pcie_writel_dbi(pci, PCIE_TYPE0_HDR_DBI2_OFFSET + 0x10 + bar * 4, 0);
505
-	bar = 5;
554
+	bar = BAR_5;
506555 	dw_pcie_writel_dbi(pci, PCIE_TYPE0_HDR_DBI2_OFFSET + 0x10 + bar * 4, 0);
507556 }
508557 
....@@ -514,14 +563,13 @@
514563 	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, 0x0580);
515564 }
516565 
517
-static int rockchip_pcie_ep_set_bar(struct rockchip_pcie *rockchip)
566
+static int rockchip_pcie_ep_set_bar(struct rockchip_pcie *rockchip, enum pci_barno bar,
567
+				    dma_addr_t cpu_addr)
518568 {
519569 	int ret;
520570 	u32 free_win;
521571 	struct dw_pcie *pci = &rockchip->pci;
522
-	enum pci_barno bar;
523572 	enum dw_pcie_as_type as_type;
524
-	dma_addr_t cpu_addr;
525573 
526574 	free_win = find_first_zero_bit(rockchip->ib_window_map,
527575 				       rockchip->num_ib_windows);
....@@ -531,8 +579,6 @@
531579 	}
532580 
533581 	as_type = DW_PCIE_AS_MEM;
534
-	bar = BAR_0;
535
-	cpu_addr = rockchip->ib_target_address;
536582 
537583 	ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr, as_type);
538584 	if (ret < 0) {
....@@ -544,7 +590,6 @@
544590 	set_bit(free_win, rockchip->ib_window_map);
545591 
546592 	return 0;
547
-
548593 }
549594 
550595 static void rockchip_pcie_fast_link_setup(struct rockchip_pcie *rockchip)
....@@ -553,7 +598,8 @@
553598 
554599 	/* LTSSM EN ctrl mode */
555600 	val = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_HOT_RESET_CTRL);
556
-	val |= PCIE_LTSSM_ENABLE_ENHANCE | (PCIE_LTSSM_ENABLE_ENHANCE << 16);
601
+	val |= (PCIE_LTSSM_ENABLE_ENHANCE | PCIE_LTSSM_APP_DLY2_EN) |
602
+		((PCIE_LTSSM_ENABLE_ENHANCE | PCIE_LTSSM_APP_DLY2_EN) << 16);
557603 	rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL);
558604 }
559605 
....@@ -571,98 +617,117 @@
571617 static void rockchip_pcie_local_elbi_enable(struct rockchip_pcie *rockchip)
572618 {
573619 	int i;
574
-	u32 dlbi_reg;
620
+	u32 elbi_reg;
575621 	struct dw_pcie *pci = &rockchip->pci;
576622 
577623 	for (i = 0; i < PCIE_ELBI_REG_NUM; i++) {
578
-		dlbi_reg = PCIE_ELBI_LOCAL_BASE + PCIE_ELBI_LOCAL_ENABLE_OFF +
624
+		elbi_reg = PCIE_ELBI_LOCAL_BASE + PCIE_ELBI_LOCAL_ENABLE_OFF +
579625 			   i * 4;
580
-		dw_pcie_writel_dbi(pci, dlbi_reg, 0xffff0000);
626
+		dw_pcie_writel_dbi(pci, elbi_reg, 0xffff0000);
581627 	}
582628 }
583629 
584630 static void rockchip_pcie_elbi_clear(struct rockchip_pcie *rockchip)
585631 {
586632 	int i;
587
-	u32 dlbi_reg;
633
+	u32 elbi_reg;
588634 	struct dw_pcie *pci = &rockchip->pci;
589635 	u32 val;
590636 
591637 	for (i = 0; i < PCIE_ELBI_REG_NUM; i++) {
592
-		dlbi_reg = PCIE_ELBI_LOCAL_BASE + i * 4;
593
-		val = dw_pcie_readl_dbi(pci, dlbi_reg);
638
+		elbi_reg = PCIE_ELBI_LOCAL_BASE + i * 4;
639
+		val = dw_pcie_readl_dbi(pci, elbi_reg);
594640 		val <<= 16;
595
-		dw_pcie_writel_dbi(pci, dlbi_reg, val);
641
+		dw_pcie_writel_dbi(pci, elbi_reg, val);
596642 	}
643
+}
644
+
645
+static void rockchip_pcie_raise_msi_irq(struct rockchip_pcie *rockchip, u8 interrupt_num)
646
+{
647
+	rockchip_pcie_writel_apb(rockchip, BIT(interrupt_num), PCIE_CLIENT_MSI_GEN_CON);
597648 }
598649 
599650 static irqreturn_t rockchip_pcie_sys_irq_handler(int irq, void *arg)
600651 {
601652 	struct rockchip_pcie *rockchip = arg;
602653 	struct dw_pcie *pci = &rockchip->pci;
603
-	u32 dlbi_reg;
654
+	u32 elbi_reg;
604655 	u32 chn;
605
-	union int_status status;
656
+	union int_status wr_status, rd_status;
606657 	union int_clear clears;
607
-	u32 reg, val;
658
+	u32 reg, mask;
659
+	bool sigio = false;
608660 
609661 	/* ELBI helper, only check the valid bits, and discard the rest interrupts */
610
-	dlbi_reg = dw_pcie_readl_dbi(pci, PCIE_ELBI_LOCAL_BASE + PCIE_ELBI_APP_ELBI_INT_GEN0);
611
-	if (dlbi_reg & PCIE_ELBI_APP_ELBI_INT_GEN0_SIGIO) {
612
-		dev_dbg(rockchip->pci.dev, "SIGIO\n");
613
-		kill_fasync(&rockchip->async, SIGIO, POLL_IN);
662
+	elbi_reg = dw_pcie_readl_dbi(pci, PCIE_ELBI_LOCAL_BASE + PCIE_ELBI_APP_ELBI_INT_GEN0);
663
+	if (elbi_reg & PCIE_ELBI_APP_ELBI_INT_GEN0_SIGIO) {
664
+		sigio = true;
665
+		rockchip->obj_info->irq_type_ep = OBJ_IRQ_ELBI;
666
+		rockchip_pcie_elbi_clear(rockchip);
667
+		goto out;
614668 	}
615669 
616
-	rockchip_pcie_elbi_clear(rockchip);
617
-
618670 	/* DMA helper */
619
-	status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET +
620
-					   PCIE_DMA_WR_INT_STATUS);
671
+	mask = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_MASK);
672
+	wr_status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_STATUS) & (~mask);
673
+	mask = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_MASK);
674
+	rd_status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_STATUS) & (~mask);
675
+
621676 	for (chn = 0; chn < PCIE_DMA_CHANEL_MAX_NUM; chn++) {
622
-		if (status.donesta & BIT(chn)) {
623
-			clears.doneclr = 0x1 << chn;
677
+		if (wr_status.donesta & BIT(chn)) {
678
+			clears.doneclr = BIT(chn);
624679 			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
625680 					PCIE_DMA_WR_INT_CLEAR, clears.asdword);
626681 			if (rockchip->dma_obj && rockchip->dma_obj->cb)
627682 				rockchip->dma_obj->cb(rockchip->dma_obj, chn, DMA_TO_BUS);
628683 		}
629684 
630
-		if (status.abortsta & BIT(chn)) {
685
+		if (wr_status.abortsta & BIT(chn)) {
631686 			dev_err(pci->dev, "%s, abort\n", __func__);
632
-			clears.abortclr = 0x1 << chn;
687
+			clears.abortclr = BIT(chn);
633688 			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
634689 					PCIE_DMA_WR_INT_CLEAR, clears.asdword);
635690 		}
636691 	}
637692 
638
-	status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET +
639
-					   PCIE_DMA_RD_INT_STATUS);
640693 	for (chn = 0; chn < PCIE_DMA_CHANEL_MAX_NUM; chn++) {
641
-		if (status.donesta & BIT(chn)) {
642
-			clears.doneclr = 0x1 << chn;
694
+		if (rd_status.donesta & BIT(chn)) {
695
+			clears.doneclr = BIT(chn);
643696 			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
644697 					PCIE_DMA_RD_INT_CLEAR, clears.asdword);
645698 			if (rockchip->dma_obj && rockchip->dma_obj->cb)
646699 				rockchip->dma_obj->cb(rockchip->dma_obj, chn, DMA_FROM_BUS);
647700 		}
648701 
649
-		if (status.abortsta & BIT(chn)) {
702
+		if (rd_status.abortsta & BIT(chn)) {
650703 			dev_err(pci->dev, "%s, abort\n", __func__);
651
-			clears.abortclr = 0x1 << chn;
704
+			clears.abortclr = BIT(chn);
652705 			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
653706 					PCIE_DMA_RD_INT_CLEAR, clears.asdword);
654707 		}
655708 	}
656709 
657
-	reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
658
-	if (reg & BIT(2)) {
659
-		/* Setup command register */
660
-		val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
661
-		val &= 0xffff0000;
662
-		val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY |
663
-		       PCI_COMMAND_MASTER | PCI_COMMAND_SERR;
664
-		dw_pcie_writel_dbi(pci, PCI_COMMAND, val);
710
+	if (wr_status.asdword || rd_status.asdword) {
711
+		rockchip->obj_info->irq_type_rc = OBJ_IRQ_DMA;
712
+		rockchip->obj_info->dma_status_rc.wr |= wr_status.asdword;
713
+		rockchip->obj_info->dma_status_rc.rd |= rd_status.asdword;
714
+		rockchip_pcie_raise_msi_irq(rockchip, PCIe_CLIENT_MSI_OBJ_IRQ);
715
+
716
+		rockchip->obj_info->irq_type_ep = OBJ_IRQ_DMA;
717
+		rockchip->obj_info->dma_status_ep.wr |= wr_status.asdword;
718
+		rockchip->obj_info->dma_status_ep.rd |= rd_status.asdword;
719
+		sigio = true;
665720 	}
721
+
722
+out:
723
+	if (sigio) {
724
+		dev_dbg(rockchip->pci.dev, "SIGIO\n");
725
+		kill_fasync(&rockchip->async, SIGIO, POLL_IN);
726
+	}
727
+
728
+	reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
729
+	if (reg & BIT(2))
730
+		queue_work(rockchip->hot_rst_wq, &rockchip->hot_rst_work);
666731 
667732 	rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC);
668733 
....@@ -706,7 +771,7 @@
706771 	if (!rockchip_pcie_udma_enabled(rockchip))
707772 		return 0;
708773 
709
-	rockchip->dma_obj = pcie_dw_dmatest_register(pci, true);
774
+	rockchip->dma_obj = pcie_dw_dmatest_register(pci->dev, true);
710775 	if (IS_ERR(rockchip->dma_obj)) {
711776 		dev_err(rockchip->pci.dev, "failed to prepare dmatest\n");
712777 		return -EINVAL;
....@@ -812,6 +877,73 @@
812877 	table->start.chnl = table->chn;
813878 }
814879 
880
+static void rockchip_pcie_hot_rst_work(struct work_struct *work)
881
+{
882
+	struct rockchip_pcie *rockchip = container_of(work, struct rockchip_pcie, hot_rst_work);
883
+	u32 status;
884
+	int ret;
885
+
886
+	if (rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_HOT_RESET_CTRL) & PCIE_LTSSM_APP_DLY2_EN) {
887
+		ret = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_LTSSM_STATUS,
888
+			 status, ((status & 0x3F) == 0), 100, PCIE_HOTRESET_TMOUT_US);
889
+		if (ret)
890
+			dev_err(rockchip->pci.dev, "wait for detect quiet failed!\n");
891
+
892
+		rockchip_pcie_writel_apb(rockchip, (PCIE_LTSSM_APP_DLY2_DONE) | ((PCIE_LTSSM_APP_DLY2_DONE) << 16),
893
+					PCIE_CLIENT_HOT_RESET_CTRL);
894
+	}
895
+}
896
+
897
+static int rockchip_pcie_get_dma_status(struct dma_trx_obj *obj, u8 chn, enum dma_dir dir)
898
+{
899
+	struct rockchip_pcie *rockchip = dev_get_drvdata(obj->dev);
900
+	struct dw_pcie *pci = &rockchip->pci;
901
+	union int_status status;
902
+	union int_clear clears;
903
+	int ret = 0;
904
+
905
+	dev_dbg(pci->dev, "%s %x %x\n", __func__,
906
+		dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_STATUS),
907
+		dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_STATUS));
908
+
909
+	if (dir == DMA_TO_BUS) {
910
+		status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_STATUS);
911
+		if (status.donesta & BIT(chn)) {
912
+			clears.doneclr = BIT(chn);
913
+			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
914
+					   PCIE_DMA_WR_INT_CLEAR, clears.asdword);
915
+			ret = 1;
916
+		}
917
+
918
+		if (status.abortsta & BIT(chn)) {
919
+			dev_err(pci->dev, "%s, write abort\n", __func__);
920
+			clears.abortclr = BIT(chn);
921
+			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
922
+					   PCIE_DMA_WR_INT_CLEAR, clears.asdword);
923
+			ret = -1;
924
+		}
925
+	} else {
926
+		status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_STATUS);
927
+
928
+		if (status.donesta & BIT(chn)) {
929
+			clears.doneclr = BIT(chn);
930
+			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
931
+					   PCIE_DMA_RD_INT_CLEAR, clears.asdword);
932
+			ret = 1;
933
+		}
934
+
935
+		if (status.abortsta & BIT(chn)) {
936
+			dev_err(pci->dev, "%s, read abort %x\n", __func__, status.asdword);
937
+			clears.abortclr = BIT(chn);
938
+			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
939
+					   PCIE_DMA_RD_INT_CLEAR, clears.asdword);
940
+			ret = -1;
941
+		}
942
+	}
943
+
944
+	return ret;
945
+}
946
+
815947 static const struct dw_pcie_ops dw_pcie_ops = {
816948 	.start_link = rockchip_pcie_start_link,
817949 	.link_up = rockchip_pcie_link_up,
....@@ -847,6 +979,7 @@
847979 	struct pcie_ep_dma_cache_cfg cfg;
848980 	void __user *uarg = (void __user *)arg;
849981 	int i, ret;
982
+	enum pcie_ep_mmap_resource mmap_res;
850983 
851984 	switch (cmd) {
852985 	case PCIE_DMA_GET_ELBI_DATA:
....@@ -885,6 +1018,33 @@
8851018 		dw_pcie_writel_dbi(&rockchip->pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_MASK,
8861019 				   0xffffffff);
8871020 		break;
1021
+	case PCIE_DMA_RAISE_MSI_OBJ_IRQ_USER:
1022
+		rockchip->obj_info->irq_type_rc = OBJ_IRQ_USER;
1023
+		rockchip_pcie_raise_msi_irq(rockchip, PCIe_CLIENT_MSI_OBJ_IRQ);
1024
+		break;
1025
+	case PCIE_EP_GET_USER_INFO:
1026
+		msg.bar0_phys_addr = rockchip->ib_target_address[0];
1027
+
1028
+		ret = copy_to_user(uarg, &msg, sizeof(msg));
1029
+		if (ret) {
1030
+			dev_err(rockchip->pci.dev, "failed to get elbi data\n");
1031
+			return -EFAULT;
1032
+		}
1033
+		break;
1034
+	case PCIE_EP_SET_MMAP_RESOURCE:
1035
+		ret = copy_from_user(&mmap_res, uarg, sizeof(mmap_res));
1036
+		if (ret) {
1037
+			dev_err(rockchip->pci.dev, "failed to get copy from\n");
1038
+			return -EFAULT;
1039
+		}
1040
+
1041
+		if (mmap_res >= PCIE_EP_MMAP_RESOURCE_MAX) {
1042
+			dev_err(rockchip->pci.dev, "mmap index %d is out of number\n", mmap_res);
1043
+			return -EINVAL;
1044
+		}
1045
+
1046
+		rockchip->cur_mmap_res = mmap_res;
1047
+		break;
8881048 	default:
8891049 		break;
8901050 	}
....@@ -896,18 +1056,45 @@
8961056 	struct rockchip_pcie *rockchip = (struct rockchip_pcie *)file->private_data;
8971057 	size_t size = vma->vm_end - vma->vm_start;
8981058 	int err;
1059
+	unsigned long addr;
8991060 
900
-	if (size > PCIE_DBI_SIZE) {
901
-		dev_warn(rockchip->pci.dev, "mmap size is out of limitation\n");
1061
+	switch (rockchip->cur_mmap_res) {
1062
+	case PCIE_EP_MMAP_RESOURCE_DBI:
1063
+		if (size > PCIE_DBI_SIZE) {
1064
+			dev_warn(rockchip->pci.dev, "dbi mmap size is out of limitation\n");
1065
+			return -EINVAL;
1066
+		}
1067
+		addr = rockchip->dbi_base_physical;
1068
+		break;
1069
+	case PCIE_EP_MMAP_RESOURCE_BAR0:
1070
+		if (size > rockchip->ib_target_size[0]) {
1071
+			dev_warn(rockchip->pci.dev, "bar0 mmap size is out of limitation\n");
1072
+			return -EINVAL;
1073
+		}
1074
+		addr = rockchip->ib_target_address[0];
1075
+		break;
1076
+	case PCIE_EP_MMAP_RESOURCE_BAR2:
1077
+		if (size > rockchip->ib_target_size[2]) {
1078
+			dev_warn(rockchip->pci.dev, "bar2 mmap size is out of limitation\n");
1079
+			return -EINVAL;
1080
+		}
1081
+		addr = rockchip->ib_target_address[2];
1082
+		break;
1083
+	default:
1084
+		dev_err(rockchip->pci.dev, "cur mmap_res %d is unsurreport\n", rockchip->cur_mmap_res);
9021085 		return -EINVAL;
9031086 	}
9041087 
9051088 	vma->vm_flags |= VM_IO;
9061089 	vma->vm_flags |= (VM_DONTEXPAND | VM_DONTDUMP);
907
-	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1090
+
1091
+	if (rockchip->cur_mmap_res == PCIE_EP_MMAP_RESOURCE_BAR2)
1092
+		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1093
+	else
1094
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
9081095 
9091096 	err = remap_pfn_range(vma, vma->vm_start,
910
-			      __phys_to_pfn(rockchip->dbi_base_physical),
1097
+			      __phys_to_pfn(addr),
9111098 			      size, vma->vm_page_prot);
9121099 	if (err)
9131100 		return -EAGAIN;
....@@ -941,13 +1128,13 @@
9411128 
9421129 	ret = misc_register(&pcie_dev->dev);
9431130 	if (ret) {
944
-		pr_err("pcie: failed to register misc device.\n");
1131
+		dev_err(rockchip->pci.dev, "pcie: failed to register misc device.\n");
9451132 		return ret;
9461133 	}
9471134 
9481135 	pcie_dev->pcie = rockchip;
9491136 
950
-	pr_info("register misc device pcie-dev\n");
1137
+	dev_info(rockchip->pci.dev, "register misc device pcie_ep\n");
9511138 
9521139 	return 0;
9531140 }
....@@ -957,7 +1144,8 @@
9571144 	struct device *dev = &pdev->dev;
9581145 	struct rockchip_pcie *rockchip;
9591146 	int ret;
960
-	int retry;
1147
+	int retry, i;
1148
+	u32 reg;
9611149 
9621150 	rockchip = devm_kzalloc(dev, sizeof(*rockchip), GFP_KERNEL);
9631151 	if (!rockchip)
....@@ -993,8 +1181,10 @@
9931181 		goto disable_regulator;
9941182 
9951183 	if (dw_pcie_link_up(&rockchip->pci)) {
996
-		pr_info("%s, %d, already linkup\n", __func__, __LINE__);
1184
+		dev_info(dev, "already linkup\n");
9971185 		goto already_linkup;
1186
+	} else {
1187
+		dev_info(dev, "initial\n");
9981188 	}
9991189 
10001190 	ret = rockchip_pcie_phy_init(rockchip);
....@@ -1015,6 +1205,27 @@
10151205 	rockchip_pcie_fast_link_setup(rockchip);
10161206 
10171207 	rockchip_pcie_start_link(&rockchip->pci);
1208
+	rockchip_pcie_devmode_update(rockchip, RKEP_MODE_KERNEL, RKEP_SMODE_LNKRDY);
1209
+
1210
+	rockchip->hot_rst_wq = create_singlethread_workqueue("rkep_hot_rst_wq");
1211
+	if (!rockchip->hot_rst_wq) {
1212
+		dev_err(dev, "failed to create hot_rst workqueue\n");
1213
+		ret = -ENOMEM;
1214
+		goto deinit_phy;
1215
+	}
1216
+	INIT_WORK(&rockchip->hot_rst_work, rockchip_pcie_hot_rst_work);
1217
+
1218
+	reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
1219
+	if ((reg & BIT(2)) &&
1220
+	    (rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_HOT_RESET_CTRL) & PCIE_LTSSM_APP_DLY2_EN)) {
1221
+		rockchip_pcie_writel_apb(rockchip, PCIE_LTSSM_APP_DLY2_DONE | (PCIE_LTSSM_APP_DLY2_DONE << 16),
1222
+					 PCIE_CLIENT_HOT_RESET_CTRL);
1223
+		dev_info(dev, "hot reset ever\n");
1224
+	}
1225
+	rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC);
1226
+
1227
+	/* Enable client reset or link down interrupt */
1228
+	rockchip_pcie_writel_apb(rockchip, 0x40000, PCIE_CLIENT_INTR_MASK);
10181229 
10191230 	for (retry = 0; retry < 10000; retry++) {
10201231 		if (dw_pcie_link_up(&rockchip->pci)) {
....@@ -1042,8 +1253,11 @@
10421253 	}
10431254 
10441255 already_linkup:
1256
+	rockchip_pcie_devmode_update(rockchip, RKEP_MODE_KERNEL, RKEP_SMODE_LNKUP);
10451257 	rockchip->pci.iatu_unroll_enabled = rockchip_pcie_iatu_unroll_enabled(&rockchip->pci);
1046
-	rockchip_pcie_ep_set_bar(rockchip);
1258
+	for (i = 0; i < PCIE_BAR_MAX_NUM; i++)
1259
+		if (rockchip->ib_target_size[i])
1260
+			rockchip_pcie_ep_set_bar(rockchip, i, rockchip->ib_target_address[i]);
10471261 
10481262 	ret = rockchip_pcie_init_dma_trx(rockchip);
10491263 	if (ret) {
....@@ -1054,6 +1268,7 @@
10541268 	if (rockchip->dma_obj) {
10551269 		rockchip->dma_obj->start_dma_func = rockchip_pcie_start_dma_dwc;
10561270 		rockchip->dma_obj->config_dma_func = rockchip_pcie_config_dma_dwc;
1271
+		rockchip->dma_obj->get_dma_status = rockchip_pcie_get_dma_status;
10571272 	}
10581273 
10591274 	/* Enable client ELBI interrupt */
....@@ -1086,9 +1301,10 @@
10861301 		.of_match_table = rockchip_pcie_ep_of_match,
10871302 		.suppress_bind_attrs = true,
10881303 	},
1304
+	.probe = rockchip_pcie_ep_probe,
10891305 };
10901306 
1091
-module_platform_driver_probe(rk_plat_pcie_driver, rockchip_pcie_ep_probe);
1307
+module_platform_driver(rk_plat_pcie_driver);
10921308 
10931309 MODULE_AUTHOR("Simon Xue <xxm@rock-chips.com>");
10941310 MODULE_DESCRIPTION("RockChip PCIe Controller EP driver");