forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-10 ee930fffee469d076998274a2ca55e13dc1efb67 
 kernel/drivers/pci/controller/dwc/pcie-dw-ep-rockchip.c
....@@ -81,16 +81,26 @@
8181 #define PCIE_CLIENT_HOT_RESET_CTRL      0x180
8282 #define PCIE_CLIENT_LTSSM_STATUS	0x300
8383 #define PCIE_CLIENT_INTR_MASK		0x24
84
+#define PCIE_LTSSM_APP_DLY1_EN		BIT(0)
85
+#define PCIE_LTSSM_APP_DLY2_EN		BIT(1)
86
+#define PCIE_LTSSM_APP_DLY1_DONE	BIT(2)
87
+#define PCIE_LTSSM_APP_DLY2_DONE	BIT(3)
8488 #define PCIE_LTSSM_ENABLE_ENHANCE       BIT(4)
89
+#define PCIE_CLIENT_MSI_GEN_CON		0x38
90
+
91
+#define PCIe_CLIENT_MSI_OBJ_IRQ		0	/* rockchip ep object special irq */
92
+
8593 #define PCIE_ELBI_REG_NUM		0x2
8694 #define PCIE_ELBI_LOCAL_BASE		0x200e00
8795 
8896 #define PCIE_ELBI_APP_ELBI_INT_GEN0		0x0
89
-#define PCIE_ELBI_APP_ELBI_INT_GEN0_SIGIO	BIT(0)
97
+#define PCIE_ELBI_APP_ELBI_INT_GEN0_IRQ_USER	BIT(0)
9098 
9199 #define PCIE_ELBI_APP_ELBI_INT_GEN1		0x4
92100 
93101 #define PCIE_ELBI_LOCAL_ENABLE_OFF	0x8
102
+
103
+#define PCIE_ELBI_USER_DATA_OFF	0x10
94104 
95105 #define PCIE_DIRECT_SPEED_CHANGE	BIT(17)
96106 
....@@ -98,6 +108,11 @@
98108 #define PCIE_TYPE0_HDR_DBI2_OFFSET	0x100000
99109 
100110 #define PCIE_DBI_SIZE			0x400000
111
+
112
+#define PCIE_EP_OBJ_INFO_DRV_VERSION	0x00000001
113
+
114
+#define PCIE_BAR_MAX_NUM		6
115
+#define PCIE_HOTRESET_TMOUT_US		10000
101116 
102117 struct rockchip_pcie {
103118 	struct dw_pcie			pci;
....@@ -113,13 +128,19 @@
113128 	u32				num_ib_windows;
114129 	u32				num_ob_windows;
115130 	phys_addr_t			*outbound_addr;
116
-	u8				bar_to_atu[6];
117
-	dma_addr_t			ib_target_address;
118
-	u32				ib_target_size;
119
-	void				*ib_target_base;
131
+	u8				bar_to_atu[PCIE_BAR_MAX_NUM];
132
+	dma_addr_t			ib_target_address[PCIE_BAR_MAX_NUM];
133
+	u32				ib_target_size[PCIE_BAR_MAX_NUM];
134
+	void				*ib_target_base[PCIE_BAR_MAX_NUM];
120135 	struct dma_trx_obj		*dma_obj;
121
-	struct fasync_struct		*async;
122136 	phys_addr_t			dbi_base_physical;
137
+	struct pcie_ep_obj_info		*obj_info;
138
+	enum pcie_ep_mmap_resource	cur_mmap_res;
139
+	struct workqueue_struct		*hot_rst_wq;
140
+	struct work_struct		hot_rst_work;
141
+	struct mutex			file_mutex;
142
+	DECLARE_BITMAP(virtual_id_irq_bitmap, RKEP_EP_VIRTUAL_ID_MAX);
143
+	wait_queue_head_t wq_head;
123144 };
124145 
125146 struct rockchip_pcie_misc_dev {
....@@ -138,6 +159,12 @@
138159 };
139160 
140161 MODULE_DEVICE_TABLE(of, rockchip_pcie_ep_of_match);
162
+
163
+static void rockchip_pcie_devmode_update(struct rockchip_pcie *rockchip, int mode, int submode)
164
+{
165
+	rockchip->obj_info->devmode.mode = mode;
166
+	rockchip->obj_info->devmode.submode = submode;
167
+}
141168 
142169 static int rockchip_pcie_readl_apb(struct rockchip_pcie *rockchip, u32 reg)
143170 {
....@@ -183,9 +210,10 @@
183210 	struct device_node *np = dev->of_node;
184211 	void *addr;
185212 	struct resource *dbi_base;
186
-	struct resource *apb_base;
187213 	struct device_node *mem;
188214 	struct resource reg;
215
+	char name[8];
216
+	int i, idx;
189217 
190218 	dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM,
191219 						"pcie-dbi");
....@@ -197,17 +225,14 @@
197225 	rockchip->pci.dbi_base = devm_ioremap_resource(dev, dbi_base);
198226 	if (IS_ERR(rockchip->pci.dbi_base))
199227 		return PTR_ERR(rockchip->pci.dbi_base);
228
+	rockchip->pci.atu_base = rockchip->pci.dbi_base + DEFAULT_DBI_ATU_OFFSET;
200229 	rockchip->dbi_base_physical = dbi_base->start;
201230 
202
-	apb_base = platform_get_resource_byname(pdev, IORESOURCE_MEM,
203
-						"pcie-apb");
204
-	if (!apb_base) {
231
+	rockchip->apb_base = devm_platform_ioremap_resource_byname(pdev, "pcie-apb");
232
+	if (!rockchip->apb_base) {
205233 		dev_err(dev, "get pcie-apb failed\n");
206234 		return -ENODEV;
207235 	}
208
-	rockchip->apb_base = devm_ioremap_resource(dev, apb_base);
209
-	if (IS_ERR(rockchip->apb_base))
210
-		return PTR_ERR(rockchip->apb_base);
211236 
212237 	rockchip->rst_gpio = devm_gpiod_get_optional(dev, "reset",
213238 						     GPIOD_OUT_HIGH);
....@@ -255,22 +280,42 @@
255280 
256281 	rockchip->outbound_addr = addr;
257282 
258
-	mem = of_parse_phandle(np, "memory-region", 0);
259
-	if (!mem) {
260
-		dev_err(dev, "missing \"memory-region\" property\n");
261
-		return -ENODEV;
283
+	for (i = 0; i < PCIE_BAR_MAX_NUM; i++) {
284
+		snprintf(name, sizeof(name), "bar%d", i);
285
+		idx = of_property_match_string(np, "memory-region-names", name);
286
+		if (idx < 0)
287
+			continue;
288
+
289
+		mem = of_parse_phandle(np, "memory-region", idx);
290
+		if (!mem) {
291
+			dev_err(dev, "missing \"memory-region\" %s property\n", name);
292
+			return -ENODEV;
293
+		}
294
+
295
+		ret = of_address_to_resource(mem, 0, &reg);
296
+		if (ret < 0) {
297
+			dev_err(dev, "missing \"reg\" %s property\n", name);
298
+			return -ENODEV;
299
+		}
300
+
301
+		rockchip->ib_target_address[i] = reg.start;
302
+		rockchip->ib_target_size[i] = resource_size(&reg);
303
+		rockchip->ib_target_base[i] = rockchip_pcie_map_kernel(reg.start,
304
+							resource_size(&reg));
305
+		dev_info(dev, "%s: assigned [0x%llx-%llx]\n", name, rockchip->ib_target_address[i],
306
+			rockchip->ib_target_address[i] + rockchip->ib_target_size[i] - 1);
262307 	}
263308 
264
-	ret = of_address_to_resource(mem, 0, &reg);
265
-	if (ret < 0) {
266
-		dev_err(dev, "missing \"reg\" property\n");
309
+	if (rockchip->ib_target_size[0]) {
310
+		rockchip->obj_info = (struct pcie_ep_obj_info *)rockchip->ib_target_base[0];
311
+		memset_io(rockchip->obj_info, 0, sizeof(struct pcie_ep_obj_info));
312
+		rockchip->obj_info->magic = PCIE_EP_OBJ_INFO_MAGIC;
313
+		rockchip->obj_info->version = PCIE_EP_OBJ_INFO_DRV_VERSION;
314
+		rockchip_pcie_devmode_update(rockchip, RKEP_MODE_KERNEL, RKEP_SMODE_INIT);
315
+	} else {
316
+		dev_err(dev, "missing bar0 memory region\n");
267317 		return -ENODEV;
268318 	}
269
-
270
-	rockchip->ib_target_address = reg.start;
271
-	rockchip->ib_target_size = resource_size(&reg);
272
-	rockchip->ib_target_base = rockchip_pcie_map_kernel(reg.start,
273
-						resource_size(&reg));
274319 
275320 	return 0;
276321 }
....@@ -485,24 +530,27 @@
485530 
486531 	resbar_base = rockchip_pci_find_resbar_capability(rockchip);
487532 
488
-	/* Resize BAR0 4M 32bits, BAR2 64M 64bits-pref */
489
-	bar = 0;
490
-	dw_pcie_writel_dbi(pci, resbar_base + 0x4 + bar * 0x8, 0xfffff0);
533
+	/* Resize BAR0 4M 32bits, BAR2 64M 64bits-pref, BAR4 1MB 32bits */
534
+	bar = BAR_0;
535
+	dw_pcie_writel_dbi(pci, resbar_base + 0x4 + bar * 0x8, 0x40);
491536 	dw_pcie_writel_dbi(pci, resbar_base + 0x8 + bar * 0x8, 0x2c0);
492
-	rockchip_pcie_ep_set_bar_flag(rockchip, BAR_0, PCI_BASE_ADDRESS_MEM_TYPE_32);
537
+	rockchip_pcie_ep_set_bar_flag(rockchip, bar, PCI_BASE_ADDRESS_MEM_TYPE_32);
493538 
494
-	bar = 2;
495
-	dw_pcie_writel_dbi(pci, resbar_base + 0x4 + bar * 0x8, 0xfffff0);
539
+	bar = BAR_2;
540
+	dw_pcie_writel_dbi(pci, resbar_base + 0x4 + bar * 0x8, 0x400);
496541 	dw_pcie_writel_dbi(pci, resbar_base + 0x8 + bar * 0x8, 0x6c0);
497
-	rockchip_pcie_ep_set_bar_flag(rockchip, BAR_2,
542
+	rockchip_pcie_ep_set_bar_flag(rockchip, bar,
498543 				      PCI_BASE_ADDRESS_MEM_PREFETCH | PCI_BASE_ADDRESS_MEM_TYPE_64);
499544 
500
-	/* Disable BAR1 BAR4 BAR5*/
501
-	bar = 1;
545
+	bar = BAR_4;
546
+	dw_pcie_writel_dbi(pci, resbar_base + 0x4 + bar * 0x8, 0x10);
547
+	dw_pcie_writel_dbi(pci, resbar_base + 0x8 + bar * 0x8, 0xc0);
548
+	rockchip_pcie_ep_set_bar_flag(rockchip, bar, PCI_BASE_ADDRESS_MEM_TYPE_32);
549
+
550
+	/* Disable BAR1 BAR5*/
551
+	bar = BAR_1;
502552 	dw_pcie_writel_dbi(pci, PCIE_TYPE0_HDR_DBI2_OFFSET + 0x10 + bar * 4, 0);
503
-	bar = 4;
504
-	dw_pcie_writel_dbi(pci, PCIE_TYPE0_HDR_DBI2_OFFSET + 0x10 + bar * 4, 0);
505
-	bar = 5;
553
+	bar = BAR_5;
506554 	dw_pcie_writel_dbi(pci, PCIE_TYPE0_HDR_DBI2_OFFSET + 0x10 + bar * 4, 0);
507555 }
508556 
....@@ -514,14 +562,13 @@
514562 	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, 0x0580);
515563 }
516564 
517
-static int rockchip_pcie_ep_set_bar(struct rockchip_pcie *rockchip)
565
+static int rockchip_pcie_ep_set_bar(struct rockchip_pcie *rockchip, enum pci_barno bar,
566
+				    dma_addr_t cpu_addr)
518567 {
519568 	int ret;
520569 	u32 free_win;
521570 	struct dw_pcie *pci = &rockchip->pci;
522
-	enum pci_barno bar;
523571 	enum dw_pcie_as_type as_type;
524
-	dma_addr_t cpu_addr;
525572 
526573 	free_win = find_first_zero_bit(rockchip->ib_window_map,
527574 				       rockchip->num_ib_windows);
....@@ -531,10 +578,8 @@
531578 	}
532579 
533580 	as_type = DW_PCIE_AS_MEM;
534
-	bar = BAR_0;
535
-	cpu_addr = rockchip->ib_target_address;
536581 
537
-	ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr, as_type);
582
+	ret = dw_pcie_prog_inbound_atu(pci, 0, free_win, bar, cpu_addr, as_type);
538583 	if (ret < 0) {
539584 		dev_err(pci->dev, "Failed to program IB window\n");
540585 		return ret;
....@@ -544,7 +589,6 @@
544589 	set_bit(free_win, rockchip->ib_window_map);
545590 
546591 	return 0;
547
-
548592 }
549593 
550594 static void rockchip_pcie_fast_link_setup(struct rockchip_pcie *rockchip)
....@@ -553,7 +597,8 @@
553597 
554598 	/* LTSSM EN ctrl mode */
555599 	val = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_HOT_RESET_CTRL);
556
-	val |= PCIE_LTSSM_ENABLE_ENHANCE | (PCIE_LTSSM_ENABLE_ENHANCE << 16);
600
+	val |= (PCIE_LTSSM_ENABLE_ENHANCE | PCIE_LTSSM_APP_DLY2_EN) |
601
+		((PCIE_LTSSM_ENABLE_ENHANCE | PCIE_LTSSM_APP_DLY2_EN) << 16);
557602 	rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL);
558603 }
559604 
....@@ -571,98 +616,159 @@
571616 static void rockchip_pcie_local_elbi_enable(struct rockchip_pcie *rockchip)
572617 {
573618 	int i;
574
-	u32 dlbi_reg;
619
+	u32 elbi_reg;
575620 	struct dw_pcie *pci = &rockchip->pci;
576621 
577622 	for (i = 0; i < PCIE_ELBI_REG_NUM; i++) {
578
-		dlbi_reg = PCIE_ELBI_LOCAL_BASE + PCIE_ELBI_LOCAL_ENABLE_OFF +
623
+		elbi_reg = PCIE_ELBI_LOCAL_BASE + PCIE_ELBI_LOCAL_ENABLE_OFF +
579624 			   i * 4;
580
-		dw_pcie_writel_dbi(pci, dlbi_reg, 0xffff0000);
625
+		dw_pcie_writel_dbi(pci, elbi_reg, 0xffff0000);
581626 	}
582627 }
583628 
584629 static void rockchip_pcie_elbi_clear(struct rockchip_pcie *rockchip)
585630 {
586631 	int i;
587
-	u32 dlbi_reg;
632
+	u32 elbi_reg;
588633 	struct dw_pcie *pci = &rockchip->pci;
589634 	u32 val;
590635 
591636 	for (i = 0; i < PCIE_ELBI_REG_NUM; i++) {
592
-		dlbi_reg = PCIE_ELBI_LOCAL_BASE + i * 4;
593
-		val = dw_pcie_readl_dbi(pci, dlbi_reg);
637
+		elbi_reg = PCIE_ELBI_LOCAL_BASE + i * 4;
638
+		val = dw_pcie_readl_dbi(pci, elbi_reg);
594639 		val <<= 16;
595
-		dw_pcie_writel_dbi(pci, dlbi_reg, val);
640
+		dw_pcie_writel_dbi(pci, elbi_reg, val);
596641 	}
642
+}
643
+
644
+static void rockchip_pcie_raise_msi_irq(struct rockchip_pcie *rockchip, u8 interrupt_num)
645
+{
646
+	rockchip_pcie_writel_apb(rockchip, BIT(interrupt_num), PCIE_CLIENT_MSI_GEN_CON);
647
+}
648
+
649
+static int rockchip_pcie_raise_irq_user(struct rockchip_pcie *rockchip, u32 index)
650
+{
651
+	if (index >= RKEP_EP_VIRTUAL_ID_MAX) {
652
+		dev_err(rockchip->pci.dev, "raise irq_user, virtual id %d out of range\n", index);
653
+
654
+		return -EINVAL;
655
+	}
656
+
657
+	mutex_lock(&rockchip->file_mutex);
658
+	rockchip->obj_info->irq_type_rc = OBJ_IRQ_USER;
659
+	rockchip->obj_info->irq_user_data_rc = index;
660
+	rockchip_pcie_raise_msi_irq(rockchip, PCIe_CLIENT_MSI_OBJ_IRQ);
661
+	mutex_unlock(&rockchip->file_mutex);
662
+
663
+	return 0;
664
+}
665
+
666
+static int rockchip_pcie_poll_irq_user(struct rockchip_pcie *rockchip, struct pcie_ep_obj_poll_virtual_id_cfg *cfg)
667
+{
668
+	u32 index = cfg->virtual_id;
669
+
670
+	if (index >= RKEP_EP_VIRTUAL_ID_MAX) {
671
+		dev_err(rockchip->pci.dev, "poll irq_user, virtual id %d out of range\n", index);
672
+
673
+		return -EINVAL;
674
+	}
675
+
676
+	cfg->poll_status = NSIGPOLL;
677
+	if (cfg->sync)
678
+		wait_event_interruptible(rockchip->wq_head,
679
+					 test_bit(index, rockchip->virtual_id_irq_bitmap));
680
+	else
681
+		wait_event_interruptible_timeout(rockchip->wq_head,
682
+						 test_bit(index, rockchip->virtual_id_irq_bitmap),
683
+						 cfg->timeout_ms);
684
+	if (test_and_clear_bit(index, rockchip->virtual_id_irq_bitmap))
685
+		cfg->poll_status = POLL_IN;
686
+
687
+	dev_dbg(rockchip->pci.dev, "poll virtual id %d, ret=%d\n", index, cfg->poll_status);
688
+
689
+	return 0;
597690 }
598691 
599692 static irqreturn_t rockchip_pcie_sys_irq_handler(int irq, void *arg)
600693 {
601694 	struct rockchip_pcie *rockchip = arg;
602695 	struct dw_pcie *pci = &rockchip->pci;
603
-	u32 dlbi_reg;
696
+	u32 elbi_reg;
604697 	u32 chn;
605
-	union int_status status;
698
+	union int_status wr_status, rd_status;
606699 	union int_clear clears;
607
-	u32 reg, val;
700
+	u32 reg, mask;
608701 
609702 	/* ELBI helper, only check the valid bits, and discard the rest interrupts */
610
-	dlbi_reg = dw_pcie_readl_dbi(pci, PCIE_ELBI_LOCAL_BASE + PCIE_ELBI_APP_ELBI_INT_GEN0);
611
-	if (dlbi_reg & PCIE_ELBI_APP_ELBI_INT_GEN0_SIGIO) {
612
-		dev_dbg(rockchip->pci.dev, "SIGIO\n");
613
-		kill_fasync(&rockchip->async, SIGIO, POLL_IN);
703
+	elbi_reg = dw_pcie_readl_dbi(pci, PCIE_ELBI_LOCAL_BASE + PCIE_ELBI_APP_ELBI_INT_GEN0);
704
+	if (elbi_reg & PCIE_ELBI_APP_ELBI_INT_GEN0_IRQ_USER) {
705
+		rockchip_pcie_elbi_clear(rockchip);
706
+
707
+		if (rockchip->obj_info->irq_type_ep == OBJ_IRQ_USER) {
708
+			reg = rockchip->obj_info->irq_user_data_ep;
709
+			if (reg < RKEP_EP_VIRTUAL_ID_MAX) {
710
+				set_bit(reg, rockchip->virtual_id_irq_bitmap);
711
+				wake_up_interruptible(&rockchip->wq_head);
712
+			}
713
+		}
714
+		goto out;
614715 	}
615716 
616
-	rockchip_pcie_elbi_clear(rockchip);
617
-
618717 	/* DMA helper */
619
-	status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET +
620
-					   PCIE_DMA_WR_INT_STATUS);
718
+	mask = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_MASK);
719
+	wr_status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_STATUS) & (~mask);
720
+	mask = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_MASK);
721
+	rd_status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_STATUS) & (~mask);
722
+
621723 	for (chn = 0; chn < PCIE_DMA_CHANEL_MAX_NUM; chn++) {
622
-		if (status.donesta & BIT(chn)) {
623
-			clears.doneclr = 0x1 << chn;
724
+		if (wr_status.donesta & BIT(chn)) {
725
+			clears.doneclr = BIT(chn);
624726 			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
625727 					PCIE_DMA_WR_INT_CLEAR, clears.asdword);
626728 			if (rockchip->dma_obj && rockchip->dma_obj->cb)
627729 				rockchip->dma_obj->cb(rockchip->dma_obj, chn, DMA_TO_BUS);
628730 		}
629731 
630
-		if (status.abortsta & BIT(chn)) {
732
+		if (wr_status.abortsta & BIT(chn)) {
631733 			dev_err(pci->dev, "%s, abort\n", __func__);
632
-			clears.abortclr = 0x1 << chn;
734
+			clears.abortclr = BIT(chn);
633735 			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
634736 					PCIE_DMA_WR_INT_CLEAR, clears.asdword);
635737 		}
636738 	}
637739 
638
-	status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET +
639
-					   PCIE_DMA_RD_INT_STATUS);
640740 	for (chn = 0; chn < PCIE_DMA_CHANEL_MAX_NUM; chn++) {
641
-		if (status.donesta & BIT(chn)) {
642
-			clears.doneclr = 0x1 << chn;
741
+		if (rd_status.donesta & BIT(chn)) {
742
+			clears.doneclr = BIT(chn);
643743 			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
644744 					PCIE_DMA_RD_INT_CLEAR, clears.asdword);
645745 			if (rockchip->dma_obj && rockchip->dma_obj->cb)
646746 				rockchip->dma_obj->cb(rockchip->dma_obj, chn, DMA_FROM_BUS);
647747 		}
648748 
649
-		if (status.abortsta & BIT(chn)) {
749
+		if (rd_status.abortsta & BIT(chn)) {
650750 			dev_err(pci->dev, "%s, abort\n", __func__);
651
-			clears.abortclr = 0x1 << chn;
751
+			clears.abortclr = BIT(chn);
652752 			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
653753 					PCIE_DMA_RD_INT_CLEAR, clears.asdword);
654754 		}
655755 	}
656756 
657
-	reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
658
-	if (reg & BIT(2)) {
659
-		/* Setup command register */
660
-		val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
661
-		val &= 0xffff0000;
662
-		val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY |
663
-		       PCI_COMMAND_MASTER | PCI_COMMAND_SERR;
664
-		dw_pcie_writel_dbi(pci, PCI_COMMAND, val);
757
+	if (wr_status.asdword || rd_status.asdword) {
758
+		rockchip->obj_info->irq_type_rc = OBJ_IRQ_DMA;
759
+		rockchip->obj_info->dma_status_rc.wr |= wr_status.asdword;
760
+		rockchip->obj_info->dma_status_rc.rd |= rd_status.asdword;
761
+		rockchip_pcie_raise_msi_irq(rockchip, PCIe_CLIENT_MSI_OBJ_IRQ);
762
+
763
+		rockchip->obj_info->irq_type_ep = OBJ_IRQ_DMA;
764
+		rockchip->obj_info->dma_status_ep.wr |= wr_status.asdword;
765
+		rockchip->obj_info->dma_status_ep.rd |= rd_status.asdword;
665766 	}
767
+
768
+out:
769
+	reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
770
+	if (reg & BIT(2))
771
+		queue_work(rockchip->hot_rst_wq, &rockchip->hot_rst_work);
666772 
667773 	rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC);
668774 
....@@ -706,7 +812,7 @@
706812 	if (!rockchip_pcie_udma_enabled(rockchip))
707813 		return 0;
708814 
709
-	rockchip->dma_obj = pcie_dw_dmatest_register(pci, true);
815
+	rockchip->dma_obj = pcie_dw_dmatest_register(pci->dev, true);
710816 	if (IS_ERR(rockchip->dma_obj)) {
711817 		dev_err(rockchip->pci.dev, "failed to prepare dmatest\n");
712818 		return -EINVAL;
....@@ -812,17 +918,77 @@
812918 	table->start.chnl = table->chn;
813919 }
814920 
921
+static void rockchip_pcie_hot_rst_work(struct work_struct *work)
922
+{
923
+	struct rockchip_pcie *rockchip = container_of(work, struct rockchip_pcie, hot_rst_work);
924
+	u32 status;
925
+	int ret;
926
+
927
+	if (rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_HOT_RESET_CTRL) & PCIE_LTSSM_APP_DLY2_EN) {
928
+		ret = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_LTSSM_STATUS,
929
+			 status, ((status & 0x3F) == 0), 100, PCIE_HOTRESET_TMOUT_US);
930
+		if (ret)
931
+			dev_err(rockchip->pci.dev, "wait for detect quiet failed!\n");
932
+
933
+		rockchip_pcie_writel_apb(rockchip, (PCIE_LTSSM_APP_DLY2_DONE) | ((PCIE_LTSSM_APP_DLY2_DONE) << 16),
934
+					PCIE_CLIENT_HOT_RESET_CTRL);
935
+	}
936
+}
937
+
938
+static int rockchip_pcie_get_dma_status(struct dma_trx_obj *obj, u8 chn, enum dma_dir dir)
939
+{
940
+	struct rockchip_pcie *rockchip = dev_get_drvdata(obj->dev);
941
+	struct dw_pcie *pci = &rockchip->pci;
942
+	union int_status status;
943
+	union int_clear clears;
944
+	int ret = 0;
945
+
946
+	dev_dbg(pci->dev, "%s %x %x\n", __func__,
947
+		dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_STATUS),
948
+		dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_STATUS));
949
+
950
+	if (dir == DMA_TO_BUS) {
951
+		status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_WR_INT_STATUS);
952
+		if (status.donesta & BIT(chn)) {
953
+			clears.doneclr = BIT(chn);
954
+			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
955
+					   PCIE_DMA_WR_INT_CLEAR, clears.asdword);
956
+			ret = 1;
957
+		}
958
+
959
+		if (status.abortsta & BIT(chn)) {
960
+			dev_err(pci->dev, "%s, write abort\n", __func__);
961
+			clears.abortclr = BIT(chn);
962
+			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
963
+					   PCIE_DMA_WR_INT_CLEAR, clears.asdword);
964
+			ret = -1;
965
+		}
966
+	} else {
967
+		status.asdword = dw_pcie_readl_dbi(pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_STATUS);
968
+
969
+		if (status.donesta & BIT(chn)) {
970
+			clears.doneclr = BIT(chn);
971
+			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
972
+					   PCIE_DMA_RD_INT_CLEAR, clears.asdword);
973
+			ret = 1;
974
+		}
975
+
976
+		if (status.abortsta & BIT(chn)) {
977
+			dev_err(pci->dev, "%s, read abort %x\n", __func__, status.asdword);
978
+			clears.abortclr = BIT(chn);
979
+			dw_pcie_writel_dbi(pci, PCIE_DMA_OFFSET +
980
+					   PCIE_DMA_RD_INT_CLEAR, clears.asdword);
981
+			ret = -1;
982
+		}
983
+	}
984
+
985
+	return ret;
986
+}
987
+
815988 static const struct dw_pcie_ops dw_pcie_ops = {
816989 	.start_link = rockchip_pcie_start_link,
817990 	.link_up = rockchip_pcie_link_up,
818991 };
819
-
820
-static int pcie_ep_fasync(int fd, struct file *file, int mode)
821
-{
822
-	struct rockchip_pcie *rockchip = (struct rockchip_pcie *)file->private_data;
823
-
824
-	return fasync_helper(fd, file, mode, &rockchip->async);
825
-}
826992 
827993 static int pcie_ep_open(struct inode *inode, struct file *file)
828994 {
....@@ -835,34 +1001,16 @@
8351001 	return 0;
8361002 }
8371003 
838
-static int pcie_ep_release(struct inode *inode, struct file *file)
839
-{
840
-	return pcie_ep_fasync(-1, file, 0);
841
-}
842
-
8431004 static long pcie_ep_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
8441005 {
8451006 	struct rockchip_pcie *rockchip = (struct rockchip_pcie *)file->private_data;
846
-	struct pcie_ep_user_data msg;
8471007 	struct pcie_ep_dma_cache_cfg cfg;
8481008 	void __user *uarg = (void __user *)arg;
849
-	int i, ret;
1009
+	struct pcie_ep_obj_poll_virtual_id_cfg poll_cfg;
1010
+	enum pcie_ep_mmap_resource mmap_res;
1011
+	int ret, index;
8501012 
8511013 	switch (cmd) {
852
-	case PCIE_DMA_GET_ELBI_DATA:
853
-		for (i = 4; i <= 6; i++)
854
-			msg.elbi_app_user[i - 4] = dw_pcie_readl_dbi(&rockchip->pci,
855
-								     PCIE_ELBI_LOCAL_BASE + i * 4);
856
-		for (i = 8; i <= 15; i++)
857
-			msg.elbi_app_user[i - 5] = dw_pcie_readl_dbi(&rockchip->pci,
858
-								     PCIE_ELBI_LOCAL_BASE + i * 4);
859
-
860
-		ret = copy_to_user(uarg, &msg, sizeof(msg));
861
-		if (ret) {
862
-			dev_err(rockchip->pci.dev, "failed to get elbi data\n");
863
-			return -EFAULT;
864
-		}
865
-		break;
8661014 	case PCIE_DMA_CACHE_INVALIDE:
8671015 		ret = copy_from_user(&cfg, uarg, sizeof(cfg));
8681016 		if (ret) {
....@@ -885,6 +1033,51 @@
8851033 		dw_pcie_writel_dbi(&rockchip->pci, PCIE_DMA_OFFSET + PCIE_DMA_RD_INT_MASK,
8861034 				   0xffffffff);
8871035 		break;
1036
+	case PCIE_EP_RAISE_MSI:
1037
+		rockchip_pcie_raise_msi_irq(rockchip, PCIe_CLIENT_MSI_OBJ_IRQ);
1038
+		break;
1039
+	case PCIE_EP_SET_MMAP_RESOURCE:
1040
+		ret = copy_from_user(&mmap_res, uarg, sizeof(mmap_res));
1041
+		if (ret) {
1042
+			dev_err(rockchip->pci.dev, "failed to get copy from\n");
1043
+			return -EFAULT;
1044
+		}
1045
+
1046
+		if (mmap_res >= PCIE_EP_MMAP_RESOURCE_MAX) {
1047
+			dev_err(rockchip->pci.dev, "mmap index %d is out of number\n", mmap_res);
1048
+			return -EINVAL;
1049
+		}
1050
+
1051
+		rockchip->cur_mmap_res = mmap_res;
1052
+		break;
1053
+	case PCIE_EP_RAISE_IRQ_USER:
1054
+		ret = copy_from_user(&index, uarg, sizeof(index));
1055
+		if (ret) {
1056
+			dev_err(rockchip->pci.dev,
1057
+				"failed to get raise irq data copy from userspace\n");
1058
+			return -EFAULT;
1059
+		}
1060
+
1061
+		ret = rockchip_pcie_raise_irq_user(rockchip, index);
1062
+		if (ret < 0)
1063
+			return -EFAULT;
1064
+		break;
1065
+	case PCIE_EP_POLL_IRQ_USER:
1066
+		ret = copy_from_user(&poll_cfg, uarg, sizeof(poll_cfg));
1067
+		if (ret) {
1068
+			dev_err(rockchip->pci.dev,
1069
+				"failed to get poll irq data copy from userspace\n");
1070
+
1071
+			return -EFAULT;
1072
+		}
1073
+
1074
+		ret = rockchip_pcie_poll_irq_user(rockchip, &poll_cfg);
1075
+		if (ret < 0)
1076
+			return -EFAULT;
1077
+
1078
+		if (copy_to_user(uarg, &poll_cfg, sizeof(poll_cfg)))
1079
+			return -EFAULT;
1080
+		break;
8881081 	default:
8891082 		break;
8901083 	}
....@@ -896,18 +1089,45 @@
8961089 	struct rockchip_pcie *rockchip = (struct rockchip_pcie *)file->private_data;
8971090 	size_t size = vma->vm_end - vma->vm_start;
8981091 	int err;
1092
+	unsigned long addr;
8991093 
900
-	if (size > PCIE_DBI_SIZE) {
901
-		dev_warn(rockchip->pci.dev, "mmap size is out of limitation\n");
1094
+	switch (rockchip->cur_mmap_res) {
1095
+	case PCIE_EP_MMAP_RESOURCE_DBI:
1096
+		if (size > PCIE_DBI_SIZE) {
1097
+			dev_warn(rockchip->pci.dev, "dbi mmap size is out of limitation\n");
1098
+			return -EINVAL;
1099
+		}
1100
+		addr = rockchip->dbi_base_physical;
1101
+		break;
1102
+	case PCIE_EP_MMAP_RESOURCE_BAR0:
1103
+		if (size > rockchip->ib_target_size[0]) {
1104
+			dev_warn(rockchip->pci.dev, "bar0 mmap size is out of limitation\n");
1105
+			return -EINVAL;
1106
+		}
1107
+		addr = rockchip->ib_target_address[0];
1108
+		break;
1109
+	case PCIE_EP_MMAP_RESOURCE_BAR2:
1110
+		if (size > rockchip->ib_target_size[2]) {
1111
+			dev_warn(rockchip->pci.dev, "bar2 mmap size is out of limitation\n");
1112
+			return -EINVAL;
1113
+		}
1114
+		addr = rockchip->ib_target_address[2];
1115
+		break;
1116
+	default:
1117
+		dev_err(rockchip->pci.dev, "cur mmap_res %d is unsurreport\n", rockchip->cur_mmap_res);
9021118 		return -EINVAL;
9031119 	}
9041120 
9051121 	vma->vm_flags |= VM_IO;
9061122 	vma->vm_flags |= (VM_DONTEXPAND | VM_DONTDUMP);
907
-	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1123
+
1124
+	if (rockchip->cur_mmap_res == PCIE_EP_MMAP_RESOURCE_BAR2)
1125
+		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1126
+	else
1127
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
9081128 
9091129 	err = remap_pfn_range(vma, vma->vm_start,
910
-			      __phys_to_pfn(rockchip->dbi_base_physical),
1130
+			      __phys_to_pfn(addr),
9111131 			      size, vma->vm_page_prot);
9121132 	if (err)
9131133 		return -EAGAIN;
....@@ -918,9 +1138,7 @@
9181138 static const struct file_operations pcie_ep_ops = {
9191139 	.owner = THIS_MODULE,
9201140 	.open = pcie_ep_open,
921
-	.release = pcie_ep_release,
9221141 	.unlocked_ioctl = pcie_ep_ioctl,
923
-	.fasync = pcie_ep_fasync,
9241142 	.mmap = pcie_ep_mmap,
9251143 };
9261144 
....@@ -941,13 +1159,13 @@
9411159 
9421160 	ret = misc_register(&pcie_dev->dev);
9431161 	if (ret) {
944
-		pr_err("pcie: failed to register misc device.\n");
1162
+		dev_err(rockchip->pci.dev, "pcie: failed to register misc device.\n");
9451163 		return ret;
9461164 	}
9471165 
9481166 	pcie_dev->pcie = rockchip;
9491167 
950
-	pr_info("register misc device pcie-dev\n");
1168
+	dev_info(rockchip->pci.dev, "register misc device pcie_ep\n");
9511169 
9521170 	return 0;
9531171 }
....@@ -957,7 +1175,8 @@
9571175 	struct device *dev = &pdev->dev;
9581176 	struct rockchip_pcie *rockchip;
9591177 	int ret;
960
-	int retry;
1178
+	int retry, i;
1179
+	u32 reg;
9611180 
9621181 	rockchip = devm_kzalloc(dev, sizeof(*rockchip), GFP_KERNEL);
9631182 	if (!rockchip)
....@@ -993,8 +1212,10 @@
9931212 		goto disable_regulator;
9941213 
9951214 	if (dw_pcie_link_up(&rockchip->pci)) {
996
-		pr_info("%s, %d, already linkup\n", __func__, __LINE__);
1215
+		dev_info(dev, "already linkup\n");
9971216 		goto already_linkup;
1217
+	} else {
1218
+		dev_info(dev, "initial\n");
9981219 	}
9991220 
10001221 	ret = rockchip_pcie_phy_init(rockchip);
....@@ -1015,6 +1236,27 @@
10151236 	rockchip_pcie_fast_link_setup(rockchip);
10161237 
10171238 	rockchip_pcie_start_link(&rockchip->pci);
1239
+	rockchip_pcie_devmode_update(rockchip, RKEP_MODE_KERNEL, RKEP_SMODE_LNKRDY);
1240
+
1241
+	rockchip->hot_rst_wq = create_singlethread_workqueue("rkep_hot_rst_wq");
1242
+	if (!rockchip->hot_rst_wq) {
1243
+		dev_err(dev, "failed to create hot_rst workqueue\n");
1244
+		ret = -ENOMEM;
1245
+		goto deinit_phy;
1246
+	}
1247
+	INIT_WORK(&rockchip->hot_rst_work, rockchip_pcie_hot_rst_work);
1248
+
1249
+	reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
1250
+	if ((reg & BIT(2)) &&
1251
+	    (rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_HOT_RESET_CTRL) & PCIE_LTSSM_APP_DLY2_EN)) {
1252
+		rockchip_pcie_writel_apb(rockchip, PCIE_LTSSM_APP_DLY2_DONE | (PCIE_LTSSM_APP_DLY2_DONE << 16),
1253
+					 PCIE_CLIENT_HOT_RESET_CTRL);
1254
+		dev_info(dev, "hot reset ever\n");
1255
+	}
1256
+	rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC);
1257
+
1258
+	/* Enable client reset or link down interrupt */
1259
+	rockchip_pcie_writel_apb(rockchip, 0x40000, PCIE_CLIENT_INTR_MASK);
10181260 
10191261 	for (retry = 0; retry < 10000; retry++) {
10201262 		if (dw_pcie_link_up(&rockchip->pci)) {
....@@ -1042,8 +1284,11 @@
10421284 	}
10431285 
10441286 already_linkup:
1287
+	rockchip_pcie_devmode_update(rockchip, RKEP_MODE_KERNEL, RKEP_SMODE_LNKUP);
10451288 	rockchip->pci.iatu_unroll_enabled = rockchip_pcie_iatu_unroll_enabled(&rockchip->pci);
1046
-	rockchip_pcie_ep_set_bar(rockchip);
1289
+	for (i = 0; i < PCIE_BAR_MAX_NUM; i++)
1290
+		if (rockchip->ib_target_size[i])
1291
+			rockchip_pcie_ep_set_bar(rockchip, i, rockchip->ib_target_address[i]);
10471292 
10481293 	ret = rockchip_pcie_init_dma_trx(rockchip);
10491294 	if (ret) {
....@@ -1054,13 +1299,16 @@
10541299 	if (rockchip->dma_obj) {
10551300 		rockchip->dma_obj->start_dma_func = rockchip_pcie_start_dma_dwc;
10561301 		rockchip->dma_obj->config_dma_func = rockchip_pcie_config_dma_dwc;
1302
+		rockchip->dma_obj->get_dma_status = rockchip_pcie_get_dma_status;
10571303 	}
1304
+	mutex_init(&rockchip->file_mutex);
10581305 
10591306 	/* Enable client ELBI interrupt */
10601307 	rockchip_pcie_writel_apb(rockchip, 0x80000000, PCIE_CLIENT_INTR_MASK);
10611308 	/* Enable ELBI interrupt */
10621309 	rockchip_pcie_local_elbi_enable(rockchip);
10631310 
1311
+	init_waitqueue_head(&rockchip->wq_head);
10641312 	ret = rockchip_pcie_request_sys_irq(rockchip, pdev);
10651313 	if (ret)
10661314 		goto deinit_phy;
....@@ -1086,9 +1334,10 @@
10861334 		.of_match_table = rockchip_pcie_ep_of_match,
10871335 		.suppress_bind_attrs = true,
10881336 	},
1337
+	.probe = rockchip_pcie_ep_probe,
10891338 };
10901339 
1091
-module_platform_driver_probe(rk_plat_pcie_driver, rockchip_pcie_ep_probe);
1340
+module_platform_driver(rk_plat_pcie_driver);
10921341 
10931342 MODULE_AUTHOR("Simon Xue <xxm@rock-chips.com>");
10941343 MODULE_DESCRIPTION("RockChip PCIe Controller EP driver");