forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 95099d4622f8cb224d94e314c7a8e0df60b13f87 
 kernel/drivers/mmc/host/mtk-sd.c
....@@ -1,21 +1,14 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Copyright (c) 2014-2015 MediaTek Inc.
34  * Author: Chaotian.Jing <chaotian.jing@mediatek.com>
4
- *
5
- * This program is free software; you can redistribute it and/or modify
6
- * it under the terms of the GNU General Public License version 2 as
7
- * published by the Free Software Foundation.
8
- *
9
- * This program is distributed in the hope that it will be useful,
10
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
- * GNU General Public License for more details.
135  */
146 
157 #include <linux/module.h>
168 #include <linux/clk.h>
179 #include <linux/delay.h>
1810 #include <linux/dma-mapping.h>
11
+#include <linux/iopoll.h>
1912 #include <linux/ioport.h>
2013 #include <linux/irq.h>
2114 #include <linux/of_address.h>
....@@ -30,6 +23,7 @@
3023 #include <linux/slab.h>
3124 #include <linux/spinlock.h>
3225 #include <linux/interrupt.h>
26
+#include <linux/reset.h>
3327 
3428 #include <linux/mmc/card.h>
3529 #include <linux/mmc/core.h>
....@@ -38,6 +32,8 @@
3832 #include <linux/mmc/sd.h>
3933 #include <linux/mmc/sdio.h>
4034 #include <linux/mmc/slot-gpio.h>
35
+
36
+#include "cqhci.h"
4137 
4238 #define MAX_BD_NUM          1024
4339 
....@@ -87,6 +83,13 @@
8783 #define SDC_FIFO_CFG     0x228
8884 
8985 /*--------------------------------------------------------------------------*/
86
+/* Top Pad Register Offset                                                  */
87
+/*--------------------------------------------------------------------------*/
88
+#define EMMC_TOP_CONTROL	0x00
89
+#define EMMC_TOP_CMD		0x04
90
+#define EMMC50_PAD_DS_TUNE	0x0c
91
+
92
+/*--------------------------------------------------------------------------*/
9093 /* Register Mask                                                            */
9194 /*--------------------------------------------------------------------------*/
9295 
....@@ -129,6 +132,7 @@
129132 #define MSDC_PS_CDSTS           (0x1 << 1)	/* R  */
130133 #define MSDC_PS_CDDEBOUNCE      (0xf << 12)	/* RW */
131134 #define MSDC_PS_DAT             (0xff << 16)	/* R  */
135
+#define MSDC_PS_DATA1           (0x1 << 17)	/* R  */
132136 #define MSDC_PS_CMD             (0x1 << 24)	/* R  */
133137 #define MSDC_PS_WP              (0x1 << 31)	/* R  */
134138 
....@@ -152,6 +156,7 @@
152156 #define MSDC_INT_DMA_BDCSERR    (0x1 << 17)	/* W1C */
153157 #define MSDC_INT_DMA_GPDCSERR   (0x1 << 18)	/* W1C */
154158 #define MSDC_INT_DMA_PROTECT    (0x1 << 19)	/* W1C */
159
+#define MSDC_INT_CMDQ           (0x1 << 28)	/* W1C */
155160 
156161 /* MSDC_INTEN mask */
157162 #define MSDC_INTEN_MMCIRQ       (0x1 << 0)	/* RW */
....@@ -182,6 +187,7 @@
182187 /* SDC_CFG mask */
183188 #define SDC_CFG_SDIOINTWKUP     (0x1 << 0)	/* RW */
184189 #define SDC_CFG_INSWKUP         (0x1 << 1)	/* RW */
190
+#define SDC_CFG_WRDTOC          (0x1fff  << 2)  /* RW */
185191 #define SDC_CFG_BUSWIDTH        (0x3 << 16)	/* RW */
186192 #define SDC_CFG_SDIO            (0x1 << 19)	/* RW */
187193 #define SDC_CFG_SDIOIDE         (0x1 << 20)	/* RW */
....@@ -193,6 +199,7 @@
193199 #define SDC_STS_CMDBUSY         (0x1 << 1)	/* RW */
194200 #define SDC_STS_SWR_COMPL       (0x1 << 31)	/* RW */
195201 
202
+#define SDC_DAT1_IRQ_TRIGGER	(0x1 << 19)	/* RW */
196203 /* SDC_ADV_CFG0 mask */
197204 #define SDC_RX_ENHANCE_EN	(0x1 << 20)	/* RW */
198205 
....@@ -229,6 +236,7 @@
229236 #define MSDC_PATCH_BIT_DECRCTMO   (0x1 << 30)	/* RW */
230237 
231238 #define MSDC_PATCH_BIT1_CMDTA     (0x7 << 3)    /* RW */
239
+#define MSDC_PB1_BUSY_CHECK_SEL   (0x1 << 7)    /* RW */
232240 #define MSDC_PATCH_BIT1_STOP_DLY  (0xf << 8)    /* RW */
233241 
234242 #define MSDC_PATCH_BIT2_CFGRESP   (0x1 << 15)   /* RW */
....@@ -262,6 +270,23 @@
262270 #define SDC_FIFO_CFG_WRVALIDSEL   (0x1 << 24)  /* RW */
263271 #define SDC_FIFO_CFG_RDVALIDSEL   (0x1 << 25)  /* RW */
264272 
273
+/* EMMC_TOP_CONTROL mask */
274
+#define PAD_RXDLY_SEL           (0x1 << 0)      /* RW */
275
+#define DELAY_EN                (0x1 << 1)      /* RW */
276
+#define PAD_DAT_RD_RXDLY2       (0x1f << 2)     /* RW */
277
+#define PAD_DAT_RD_RXDLY        (0x1f << 7)     /* RW */
278
+#define PAD_DAT_RD_RXDLY2_SEL   (0x1 << 12)     /* RW */
279
+#define PAD_DAT_RD_RXDLY_SEL    (0x1 << 13)     /* RW */
280
+#define DATA_K_VALUE_SEL        (0x1 << 14)     /* RW */
281
+#define SDC_RX_ENH_EN           (0x1 << 15)     /* TW */
282
+
283
+/* EMMC_TOP_CMD mask */
284
+#define PAD_CMD_RXDLY2          (0x1f << 0)     /* RW */
285
+#define PAD_CMD_RXDLY           (0x1f << 5)     /* RW */
286
+#define PAD_CMD_RD_RXDLY2_SEL   (0x1 << 10)     /* RW */
287
+#define PAD_CMD_RD_RXDLY_SEL    (0x1 << 11)     /* RW */
288
+#define PAD_CMD_TX_DLY          (0x1f << 12)    /* RW */
289
+
265290 #define REQ_CMD_EIO  (0x1 << 0)
266291 #define REQ_CMD_TMO  (0x1 << 1)
267292 #define REQ_DAT_ERR  (0x1 << 2)
....@@ -276,6 +301,8 @@
276301 #define MTK_MMC_AUTOSUSPEND_DELAY	50
277302 #define CMD_TIMEOUT         (HZ/10 * 5)	/* 100ms x5 */
278303 #define DAT_TIMEOUT         (HZ    * 5)	/* 1000ms x5 */
304
+
305
+#define DEFAULT_DEBOUNCE	(8)	/* 8 cycles CD debounce */
279306 
280307 #define PAD_DELAY_MAX	32 /* PAD delay cells */
281308 /*--------------------------------------------------------------------------*/
....@@ -311,6 +338,7 @@
311338 	u32 ptr;
312339 	u32 bd_data_len;
313340 #define BDMA_DESC_BUFLEN	(0xffff) /* bit0 ~ bit15 */
341
+#define BDMA_DESC_BUFLEN_EXT	(0xffffff) /* bit0 ~ bit23 */
314342 };
315343 
316344 struct msdc_dma {
....@@ -334,10 +362,14 @@
334362 	u32 emmc50_cfg0;
335363 	u32 emmc50_cfg3;
336364 	u32 sdc_fifo_cfg;
365
+	u32 emmc_top_control;
366
+	u32 emmc_top_cmd;
367
+	u32 emmc50_pad_ds_tune;
337368 };
338369 
339370 struct mtk_mmc_compatible {
340371 	u8 clk_div_bits;
372
+	bool recheck_sdio_irq;
341373 	bool hs400_tune; /* only used for MT8173 */
342374 	u32 pad_tune_reg;
343375 	bool async_fifo;
....@@ -346,12 +378,15 @@
346378 	bool stop_clk_fix;
347379 	bool enhance_rx;
348380 	bool support_64g;
381
+	bool use_internal_cd;
349382 };
350383 
351384 struct msdc_tune_para {
352385 	u32 iocon;
353386 	u32 pad_tune;
354387 	u32 pad_cmd_tune;
388
+	u32 emmc_top_control;
389
+	u32 emmc_top_cmd;
355390 };
356391 
357392 struct msdc_delay_phase {
....@@ -363,7 +398,6 @@
363398 struct msdc_host {
364399 	struct device *dev;
365400 	const struct mtk_mmc_compatible *dev_comp;
366
-	struct mmc_host *mmc;	/* mmc structure */
367401 	int cmd_rsp;
368402 
369403 	spinlock_t lock;
....@@ -373,6 +407,7 @@
373407 	int error;
374408 
375409 	void __iomem *base;		/* host base address */
410
+	void __iomem *top_base;		/* host top register base address */
376411 
377412 	struct msdc_dma dma;	/* dma channel */
378413 	u64 dma_mask;
....@@ -385,9 +420,11 @@
385420 	struct pinctrl_state *pins_uhs;
386421 	struct delayed_work req_timeout;
387422 	int irq;		/* host interrupt */
423
+	struct reset_control *reset;
388424 
389425 	struct clk *src_clk;	/* msdc source clock */
390426 	struct clk *h_clk;      /* msdc h_clk */
427
+	struct clk *bus_clk;	/* bus clock which used to access register */
391428 	struct clk *src_clk_cg; /* msdc source clock control gate */
392429 	u32 mclk;		/* mmc subsystem clock frequency */
393430 	u32 src_clk_freq;	/* source clock frequency */
....@@ -400,13 +437,17 @@
400437 	bool hs400_cmd_resp_sel_rising;
401438 				 /* cmd response sample selection for HS400 */
402439 	bool hs400_mode;	/* current eMMC will run at hs400 mode */
440
+	bool internal_cd;	/* Use internal card-detect logic */
441
+	bool cqhci;		/* support eMMC hw cmdq */
403442 	struct msdc_save_para save_para; /* used when gate HCLK */
404443 	struct msdc_tune_para def_tune_para; /* default tune setting */
405444 	struct msdc_tune_para saved_tune_para; /* tune result of CMD21/CMD19 */
445
+	struct cqhci_host *cq_host;
406446 };
407447 
408448 static const struct mtk_mmc_compatible mt8135_compat = {
409449 	.clk_div_bits = 8,
450
+	.recheck_sdio_irq = true,
410451 	.hs400_tune = false,
411452 	.pad_tune_reg = MSDC_PAD_TUNE,
412453 	.async_fifo = false,
....@@ -419,6 +460,7 @@
419460 
420461 static const struct mtk_mmc_compatible mt8173_compat = {
421462 	.clk_div_bits = 8,
463
+	.recheck_sdio_irq = true,
422464 	.hs400_tune = true,
423465 	.pad_tune_reg = MSDC_PAD_TUNE,
424466 	.async_fifo = false,
....@@ -429,8 +471,22 @@
429471 	.support_64g = false,
430472 };
431473 
474
+static const struct mtk_mmc_compatible mt8183_compat = {
475
+	.clk_div_bits = 12,
476
+	.recheck_sdio_irq = false,
477
+	.hs400_tune = false,
478
+	.pad_tune_reg = MSDC_PAD_TUNE0,
479
+	.async_fifo = true,
480
+	.data_tune = true,
481
+	.busy_check = true,
482
+	.stop_clk_fix = true,
483
+	.enhance_rx = true,
484
+	.support_64g = true,
485
+};
486
+
432487 static const struct mtk_mmc_compatible mt2701_compat = {
433488 	.clk_div_bits = 12,
489
+	.recheck_sdio_irq = true,
434490 	.hs400_tune = false,
435491 	.pad_tune_reg = MSDC_PAD_TUNE0,
436492 	.async_fifo = true,
....@@ -443,6 +499,7 @@
443499 
444500 static const struct mtk_mmc_compatible mt2712_compat = {
445501 	.clk_div_bits = 12,
502
+	.recheck_sdio_irq = false,
446503 	.hs400_tune = false,
447504 	.pad_tune_reg = MSDC_PAD_TUNE0,
448505 	.async_fifo = true,
....@@ -455,6 +512,7 @@
455512 
456513 static const struct mtk_mmc_compatible mt7622_compat = {
457514 	.clk_div_bits = 12,
515
+	.recheck_sdio_irq = true,
458516 	.hs400_tune = false,
459517 	.pad_tune_reg = MSDC_PAD_TUNE0,
460518 	.async_fifo = true,
....@@ -465,12 +523,53 @@
465523 	.support_64g = false,
466524 };
467525 
526
+static const struct mtk_mmc_compatible mt8516_compat = {
527
+	.clk_div_bits = 12,
528
+	.recheck_sdio_irq = true,
529
+	.hs400_tune = false,
530
+	.pad_tune_reg = MSDC_PAD_TUNE0,
531
+	.async_fifo = true,
532
+	.data_tune = true,
533
+	.busy_check = true,
534
+	.stop_clk_fix = true,
535
+};
536
+
537
+static const struct mtk_mmc_compatible mt7620_compat = {
538
+	.clk_div_bits = 8,
539
+	.recheck_sdio_irq = true,
540
+	.hs400_tune = false,
541
+	.pad_tune_reg = MSDC_PAD_TUNE,
542
+	.async_fifo = false,
543
+	.data_tune = false,
544
+	.busy_check = false,
545
+	.stop_clk_fix = false,
546
+	.enhance_rx = false,
547
+	.use_internal_cd = true,
548
+};
549
+
550
+static const struct mtk_mmc_compatible mt6779_compat = {
551
+	.clk_div_bits = 12,
552
+	.recheck_sdio_irq = false,
553
+	.hs400_tune = false,
554
+	.pad_tune_reg = MSDC_PAD_TUNE0,
555
+	.async_fifo = true,
556
+	.data_tune = true,
557
+	.busy_check = true,
558
+	.stop_clk_fix = true,
559
+	.enhance_rx = true,
560
+	.support_64g = true,
561
+};
562
+
468563 static const struct of_device_id msdc_of_ids[] = {
469564 	{ .compatible = "mediatek,mt8135-mmc", .data = &mt8135_compat},
470565 	{ .compatible = "mediatek,mt8173-mmc", .data = &mt8173_compat},
566
+	{ .compatible = "mediatek,mt8183-mmc", .data = &mt8183_compat},
471567 	{ .compatible = "mediatek,mt2701-mmc", .data = &mt2701_compat},
472568 	{ .compatible = "mediatek,mt2712-mmc", .data = &mt2712_compat},
473569 	{ .compatible = "mediatek,mt7622-mmc", .data = &mt7622_compat},
570
+	{ .compatible = "mediatek,mt8516-mmc", .data = &mt8516_compat},
571
+	{ .compatible = "mediatek,mt7620-mmc", .data = &mt7620_compat},
572
+	{ .compatible = "mediatek,mt6779-mmc", .data = &mt6779_compat},
474573 	{}
475574 };
476575 MODULE_DEVICE_TABLE(of, msdc_of_ids);
....@@ -525,6 +624,7 @@
525624 
526625 static void msdc_cmd_next(struct msdc_host *host,
527626 		struct mmc_request *mrq, struct mmc_command *cmd);
627
+static void __msdc_enable_sdio_irq(struct msdc_host *host, int enb);
528628 
529629 static const u32 cmd_ints_mask = MSDC_INTEN_CMDRDY | MSDC_INTEN_RSPCRCERR |
530630 			MSDC_INTEN_CMDTMO | MSDC_INTEN_ACMDRDY |
....@@ -578,8 +678,14 @@
578678 			bd[j].bd_info |= (upper_32_bits(dma_address) & 0xf)
579679 					 << 28;
580680 		}
581
-		bd[j].bd_data_len &= ~BDMA_DESC_BUFLEN;
582
-		bd[j].bd_data_len |= (dma_len & BDMA_DESC_BUFLEN);
681
+
682
+		if (host->dev_comp->support_64g) {
683
+			bd[j].bd_data_len &= ~BDMA_DESC_BUFLEN_EXT;
684
+			bd[j].bd_data_len |= (dma_len & BDMA_DESC_BUFLEN_EXT);
685
+		} else {
686
+			bd[j].bd_data_len &= ~BDMA_DESC_BUFLEN;
687
+			bd[j].bd_data_len |= (dma_len & BDMA_DESC_BUFLEN);
688
+		}
583689 
584690 		if (j == data->sg_count - 1) /* the last bd */
585691 			bd[j].bd_info |= BDMA_DESC_EOL;
....@@ -627,21 +733,22 @@
627733 	}
628734 }
629735 
630
-/* clock control primitives */
631
-static void msdc_set_timeout(struct msdc_host *host, u32 ns, u32 clks)
736
+static u64 msdc_timeout_cal(struct msdc_host *host, u64 ns, u64 clks)
632737 {
633
-	u32 timeout, clk_ns;
738
+	struct mmc_host *mmc = mmc_from_priv(host);
739
+	u64 timeout, clk_ns;
634740 	u32 mode = 0;
635741 
636
-	host->timeout_ns = ns;
637
-	host->timeout_clks = clks;
638
-	if (host->mmc->actual_clock == 0) {
742
+	if (mmc->actual_clock == 0) {
639743 		timeout = 0;
640744 	} else {
641
-		clk_ns  = 1000000000UL / host->mmc->actual_clock;
642
-		timeout = (ns + clk_ns - 1) / clk_ns + clks;
745
+		clk_ns  = 1000000000ULL;
746
+		do_div(clk_ns, mmc->actual_clock);
747
+		timeout = ns + clk_ns - 1;
748
+		do_div(timeout, clk_ns);
749
+		timeout += clks;
643750 		/* in 1048576 sclk cycle unit */
644
-		timeout = (timeout + (0x1 << 20) - 1) >> 20;
751
+		timeout = DIV_ROUND_UP(timeout, (0x1 << 20));
645752 		if (host->dev_comp->clk_div_bits == 8)
646753 			sdr_get_field(host->base + MSDC_CFG,
647754 				      MSDC_CFG_CKMOD, &mode);
....@@ -651,21 +758,44 @@
651758 		/*DDR mode will double the clk cycles for data timeout */
652759 		timeout = mode >= 2 ? timeout * 2 : timeout;
653760 		timeout = timeout > 1 ? timeout - 1 : 0;
654
-		timeout = timeout > 255 ? 255 : timeout;
655761 	}
656
-	sdr_set_field(host->base + SDC_CFG, SDC_CFG_DTOC, timeout);
762
+	return timeout;
763
+}
764
+
765
+/* clock control primitives */
766
+static void msdc_set_timeout(struct msdc_host *host, u64 ns, u64 clks)
767
+{
768
+	u64 timeout;
769
+
770
+	host->timeout_ns = ns;
771
+	host->timeout_clks = clks;
772
+
773
+	timeout = msdc_timeout_cal(host, ns, clks);
774
+	sdr_set_field(host->base + SDC_CFG, SDC_CFG_DTOC,
775
+		      (u32)(timeout > 255 ? 255 : timeout));
776
+}
777
+
778
+static void msdc_set_busy_timeout(struct msdc_host *host, u64 ns, u64 clks)
779
+{
780
+	u64 timeout;
781
+
782
+	timeout = msdc_timeout_cal(host, ns, clks);
783
+	sdr_set_field(host->base + SDC_CFG, SDC_CFG_WRDTOC,
784
+		      (u32)(timeout > 8191 ? 8191 : timeout));
657785 }
658786 
659787 static void msdc_gate_clock(struct msdc_host *host)
660788 {
661789 	clk_disable_unprepare(host->src_clk_cg);
662790 	clk_disable_unprepare(host->src_clk);
791
+	clk_disable_unprepare(host->bus_clk);
663792 	clk_disable_unprepare(host->h_clk);
664793 }
665794 
666795 static void msdc_ungate_clock(struct msdc_host *host)
667796 {
668797 	clk_prepare_enable(host->h_clk);
798
+	clk_prepare_enable(host->bus_clk);
669799 	clk_prepare_enable(host->src_clk);
670800 	clk_prepare_enable(host->src_clk_cg);
671801 	while (!(readl(host->base + MSDC_CFG) & MSDC_CFG_CKSTB))
....@@ -674,6 +804,7 @@
674804 
675805 static void msdc_set_mclk(struct msdc_host *host, unsigned char timing, u32 hz)
676806 {
807
+	struct mmc_host *mmc = mmc_from_priv(host);
677808 	u32 mode;
678809 	u32 flags;
679810 	u32 div;
....@@ -683,7 +814,7 @@
683814 	if (!hz) {
684815 		dev_dbg(host->dev, "set mclk to 0\n");
685816 		host->mclk = 0;
686
-		host->mmc->actual_clock = 0;
817
+		mmc->actual_clock = 0;
687818 		sdr_clr_bits(host->base + MSDC_CFG, MSDC_CFG_CKPDN);
688819 		return;
689820 	}
....@@ -762,7 +893,7 @@
762893 	while (!(readl(host->base + MSDC_CFG) & MSDC_CFG_CKSTB))
763894 		cpu_relax();
764895 	sdr_set_bits(host->base + MSDC_CFG, MSDC_CFG_CKPDN);
765
-	host->mmc->actual_clock = sclk;
896
+	mmc->actual_clock = sclk;
766897 	host->mclk = hz;
767898 	host->timing = timing;
768899 	/* need because clk changed. */
....@@ -773,14 +904,30 @@
773904 	 * mmc_select_hs400() will drop to 50Mhz and High speed mode,
774905 	 * tune result of hs200/200Mhz is not suitable for 50Mhz
775906 	 */
776
-	if (host->mmc->actual_clock <= 52000000) {
907
+	if (mmc->actual_clock <= 52000000) {
777908 		writel(host->def_tune_para.iocon, host->base + MSDC_IOCON);
778
-		writel(host->def_tune_para.pad_tune, host->base + tune_reg);
909
+		if (host->top_base) {
910
+			writel(host->def_tune_para.emmc_top_control,
911
+			       host->top_base + EMMC_TOP_CONTROL);
912
+			writel(host->def_tune_para.emmc_top_cmd,
913
+			       host->top_base + EMMC_TOP_CMD);
914
+		} else {
915
+			writel(host->def_tune_para.pad_tune,
916
+			       host->base + tune_reg);
917
+		}
779918 	} else {
780919 		writel(host->saved_tune_para.iocon, host->base + MSDC_IOCON);
781
-		writel(host->saved_tune_para.pad_tune, host->base + tune_reg);
782920 		writel(host->saved_tune_para.pad_cmd_tune,
783921 		       host->base + PAD_CMD_TUNE);
922
+		if (host->top_base) {
923
+			writel(host->saved_tune_para.emmc_top_control,
924
+			       host->top_base + EMMC_TOP_CONTROL);
925
+			writel(host->saved_tune_para.emmc_top_cmd,
926
+			       host->top_base + EMMC_TOP_CMD);
927
+		} else {
928
+			writel(host->saved_tune_para.pad_tune,
929
+			       host->base + tune_reg);
930
+		}
784931 	}
785932 
786933 	if (timing == MMC_TIMING_MMC_HS400 &&
....@@ -788,7 +935,7 @@
788935 		sdr_set_field(host->base + tune_reg,
789936 			      MSDC_PAD_TUNE_CMDRRDLY,
790937 			      host->hs400_cmd_int_delay);
791
-	dev_dbg(host->dev, "sclk: %d, timing: %d\n", host->mmc->actual_clock,
938
+	dev_dbg(host->dev, "sclk: %d, timing: %d\n", mmc->actual_clock,
792939 		timing);
793940 }
794941 
....@@ -823,6 +970,7 @@
823970 static inline u32 msdc_cmd_prepare_raw_cmd(struct msdc_host *host,
824971 		struct mmc_request *mrq, struct mmc_command *cmd)
825972 {
973
+	struct mmc_host *mmc = mmc_from_priv(host);
826974 	/* rawcmd :
827975 	 * vol_swt << 30 | auto_cmd << 28 | blklen << 16 | go_irq << 15 |
828976 	 * stop << 14 | rw << 13 | dtype << 11 | rsptyp << 7 | brk << 6 | opcode
....@@ -849,7 +997,7 @@
849997 		struct mmc_data *data = cmd->data;
850998 
851999 		if (mmc_op_multi(opcode)) {
852
-			if (mmc_card_mmc(host->mmc->card) && mrq->sbc &&
1000
+			if (mmc_card_mmc(mmc->card) && mrq->sbc &&
8531001 			    !(mrq->sbc->arg & 0xFFFF0000))
8541002 				rawcmd |= 0x2 << 28; /* AutoCMD23 */
8551003 		}
....@@ -917,6 +1065,32 @@
9171065 	return cmd->error;
9181066 }
9191067 
1068
+/*
1069
+ * msdc_recheck_sdio_irq - recheck whether the SDIO irq is lost
1070
+ *
1071
+ * Host controller may lost interrupt in some special case.
1072
+ * Add SDIO irq recheck mechanism to make sure all interrupts
1073
+ * can be processed immediately
1074
+ */
1075
+static void msdc_recheck_sdio_irq(struct msdc_host *host)
1076
+{
1077
+	struct mmc_host *mmc = mmc_from_priv(host);
1078
+	u32 reg_int, reg_inten, reg_ps;
1079
+
1080
+	if (mmc->caps & MMC_CAP_SDIO_IRQ) {
1081
+		reg_inten = readl(host->base + MSDC_INTEN);
1082
+		if (reg_inten & MSDC_INTEN_SDIOIRQ) {
1083
+			reg_int = readl(host->base + MSDC_INT);
1084
+			reg_ps = readl(host->base + MSDC_PS);
1085
+			if (!(reg_int & MSDC_INT_SDIOIRQ ||
1086
+			      reg_ps & MSDC_PS_DATA1)) {
1087
+				__msdc_enable_sdio_irq(host, 0);
1088
+				sdio_signal_irq(mmc);
1089
+			}
1090
+		}
1091
+	}
1092
+}
1093
+
9201094 static void msdc_track_cmd_data(struct msdc_host *host,
9211095 				struct mmc_command *cmd, struct mmc_data *data)
9221096 {
....@@ -942,7 +1116,11 @@
9421116 	msdc_track_cmd_data(host, mrq->cmd, mrq->data);
9431117 	if (mrq->data)
9441118 		msdc_unprepare_data(host, mrq);
945
-	mmc_request_done(host->mmc, mrq);
1119
+	if (host->error)
1120
+		msdc_reset_hw(host);
1121
+	mmc_request_done(mmc_from_priv(host), mrq);
1122
+	if (host->dev_comp->recheck_sdio_irq)
1123
+		msdc_recheck_sdio_irq(host);
9461124 }
9471125 
9481126 /* returns true if command is fully handled; returns false otherwise */
....@@ -989,11 +1167,13 @@
9891167 	}
9901168 
9911169 	if (!sbc_error && !(events & MSDC_INT_CMDRDY)) {
992
-		if (cmd->opcode != MMC_SEND_TUNING_BLOCK &&
993
-		    cmd->opcode != MMC_SEND_TUNING_BLOCK_HS200)
1170
+		if (events & MSDC_INT_CMDTMO ||
1171
+		    (cmd->opcode != MMC_SEND_TUNING_BLOCK &&
1172
+		     cmd->opcode != MMC_SEND_TUNING_BLOCK_HS200))
9941173 			/*
9951174 			 * should not clear fifo/interrupt as the tune data
996
-			 * may have alreay come.
1175
+			 * may have alreay come when cmd19/cmd21 gets response
1176
+			 * CRC error.
9971177 			 */
9981178 			msdc_reset_hw(host);
9991179 		if (events & MSDC_INT_RSPCRCERR) {
....@@ -1054,6 +1234,7 @@
10541234 		struct mmc_request *mrq, struct mmc_command *cmd)
10551235 {
10561236 	u32 rawcmd;
1237
+	unsigned long flags;
10571238 
10581239 	WARN_ON(host->cmd);
10591240 	host->cmd = cmd;
....@@ -1071,7 +1252,10 @@
10711252 	cmd->error = 0;
10721253 	rawcmd = msdc_cmd_prepare_raw_cmd(host, mrq, cmd);
10731254 
1255
+	spin_lock_irqsave(&host->lock, flags);
10741256 	sdr_set_bits(host->base + MSDC_INTEN, cmd_ints_mask);
1257
+	spin_unlock_irqrestore(&host->lock, flags);
1258
+
10751259 	writel(cmd->arg, host->base + SDC_ARG);
10761260 	writel(rawcmd, host->base + SDC_CMD);
10771261 }
....@@ -1234,7 +1418,7 @@
12341418 static int msdc_ops_switch_volt(struct mmc_host *mmc, struct mmc_ios *ios)
12351419 {
12361420 	struct msdc_host *host = mmc_priv(mmc);
1237
-	int ret = 0;
1421
+	int ret;
12381422 
12391423 	if (!IS_ERR(mmc->supply.vqmmc)) {
12401424 		if (ios->signal_voltage != MMC_SIGNAL_VOLTAGE_330 &&
....@@ -1244,18 +1428,19 @@
12441428 		}
12451429 
12461430 		ret = mmc_regulator_set_vqmmc(mmc, ios);
1247
-		if (ret) {
1431
+		if (ret < 0) {
12481432 			dev_dbg(host->dev, "Regulator set error %d (%d)\n",
12491433 				ret, ios->signal_voltage);
1250
-		} else {
1251
-			/* Apply different pinctrl settings for different signal voltage */
1252
-			if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_180)
1253
-				pinctrl_select_state(host->pinctrl, host->pins_uhs);
1254
-			else
1255
-				pinctrl_select_state(host->pinctrl, host->pins_default);
1434
+			return ret;
12561435 		}
1436
+
1437
+		/* Apply different pinctrl settings for different signal voltage */
1438
+		if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_180)
1439
+			pinctrl_select_state(host->pinctrl, host->pins_uhs);
1440
+		else
1441
+			pinctrl_select_state(host->pinctrl, host->pins_default);
12571442 	}
1258
-	return ret;
1443
+	return 0;
12591444 }
12601445 
12611446 static int msdc_card_busy(struct mmc_host *mmc)
....@@ -1292,9 +1477,67 @@
12921477 	}
12931478 }
12941479 
1480
+static void __msdc_enable_sdio_irq(struct msdc_host *host, int enb)
1481
+{
1482
+	if (enb) {
1483
+		sdr_set_bits(host->base + MSDC_INTEN, MSDC_INTEN_SDIOIRQ);
1484
+		sdr_set_bits(host->base + SDC_CFG, SDC_CFG_SDIOIDE);
1485
+		if (host->dev_comp->recheck_sdio_irq)
1486
+			msdc_recheck_sdio_irq(host);
1487
+	} else {
1488
+		sdr_clr_bits(host->base + MSDC_INTEN, MSDC_INTEN_SDIOIRQ);
1489
+		sdr_clr_bits(host->base + SDC_CFG, SDC_CFG_SDIOIDE);
1490
+	}
1491
+}
1492
+
1493
+static void msdc_enable_sdio_irq(struct mmc_host *mmc, int enb)
1494
+{
1495
+	unsigned long flags;
1496
+	struct msdc_host *host = mmc_priv(mmc);
1497
+
1498
+	spin_lock_irqsave(&host->lock, flags);
1499
+	__msdc_enable_sdio_irq(host, enb);
1500
+	spin_unlock_irqrestore(&host->lock, flags);
1501
+
1502
+	if (enb)
1503
+		pm_runtime_get_noresume(host->dev);
1504
+	else
1505
+		pm_runtime_put_noidle(host->dev);
1506
+}
1507
+
1508
+static irqreturn_t msdc_cmdq_irq(struct msdc_host *host, u32 intsts)
1509
+{
1510
+	struct mmc_host *mmc = mmc_from_priv(host);
1511
+	int cmd_err = 0, dat_err = 0;
1512
+
1513
+	if (intsts & MSDC_INT_RSPCRCERR) {
1514
+		cmd_err = -EILSEQ;
1515
+		dev_err(host->dev, "%s: CMD CRC ERR", __func__);
1516
+	} else if (intsts & MSDC_INT_CMDTMO) {
1517
+		cmd_err = -ETIMEDOUT;
1518
+		dev_err(host->dev, "%s: CMD TIMEOUT ERR", __func__);
1519
+	}
1520
+
1521
+	if (intsts & MSDC_INT_DATCRCERR) {
1522
+		dat_err = -EILSEQ;
1523
+		dev_err(host->dev, "%s: DATA CRC ERR", __func__);
1524
+	} else if (intsts & MSDC_INT_DATTMO) {
1525
+		dat_err = -ETIMEDOUT;
1526
+		dev_err(host->dev, "%s: DATA TIMEOUT ERR", __func__);
1527
+	}
1528
+
1529
+	if (cmd_err || dat_err) {
1530
+		dev_err(host->dev, "cmd_err = %d, dat_err =%d, intsts = 0x%x",
1531
+			cmd_err, dat_err, intsts);
1532
+	}
1533
+
1534
+	return cqhci_irq(mmc, 0, cmd_err, dat_err);
1535
+}
1536
+
12951537 static irqreturn_t msdc_irq(int irq, void *dev_id)
12961538 {
12971539 	struct msdc_host *host = (struct msdc_host *) dev_id;
1540
+	struct mmc_host *mmc = mmc_from_priv(host);
12981541 
12991542 	while (true) {
13001543 		unsigned long flags;
....@@ -1306,6 +1549,8 @@
13061549 		spin_lock_irqsave(&host->lock, flags);
13071550 		events = readl(host->base + MSDC_INT);
13081551 		event_mask = readl(host->base + MSDC_INTEN);
1552
+		if ((events & event_mask) & MSDC_INT_SDIOIRQ)
1553
+			__msdc_enable_sdio_irq(host, 0);
13091554 		/* clear interrupts */
13101555 		writel(events & event_mask, host->base + MSDC_INT);
13111556 
....@@ -1314,8 +1559,25 @@
13141559 		data = host->data;
13151560 		spin_unlock_irqrestore(&host->lock, flags);
13161561 
1317
-		if (!(events & event_mask))
1562
+		if ((events & event_mask) & MSDC_INT_SDIOIRQ)
1563
+			sdio_signal_irq(mmc);
1564
+
1565
+		if ((events & event_mask) & MSDC_INT_CDSC) {
1566
+			if (host->internal_cd)
1567
+				mmc_detect_change(mmc, msecs_to_jiffies(20));
1568
+			events &= ~MSDC_INT_CDSC;
1569
+		}
1570
+
1571
+		if (!(events & (event_mask & ~MSDC_INT_SDIOIRQ)))
13181572 			break;
1573
+
1574
+		if ((mmc->caps2 & MMC_CAP2_CQE) &&
1575
+		    (events & MSDC_INT_CMDQ)) {
1576
+			msdc_cmdq_irq(host, events);
1577
+			/* clear interrupts */
1578
+			writel(events, host->base + MSDC_INT);
1579
+			return IRQ_HANDLED;
1580
+		}
13191581 
13201582 		if (!mrq) {
13211583 			dev_err(host->dev,
....@@ -1341,21 +1603,42 @@
13411603 	u32 val;
13421604 	u32 tune_reg = host->dev_comp->pad_tune_reg;
13431605 
1606
+	if (host->reset) {
1607
+		reset_control_assert(host->reset);
1608
+		usleep_range(10, 50);
1609
+		reset_control_deassert(host->reset);
1610
+	}
1611
+
13441612 	/* Configure to MMC/SD mode, clock free running */
13451613 	sdr_set_bits(host->base + MSDC_CFG, MSDC_CFG_MODE | MSDC_CFG_CKPDN);
13461614 
13471615 	/* Reset */
13481616 	msdc_reset_hw(host);
13491617 
1350
-	/* Disable card detection */
1351
-	sdr_clr_bits(host->base + MSDC_PS, MSDC_PS_CDEN);
1352
-
13531618 	/* Disable and clear all interrupts */
13541619 	writel(0, host->base + MSDC_INTEN);
13551620 	val = readl(host->base + MSDC_INT);
13561621 	writel(val, host->base + MSDC_INT);
13571622 
1358
-	writel(0, host->base + tune_reg);
1623
+	/* Configure card detection */
1624
+	if (host->internal_cd) {
1625
+		sdr_set_field(host->base + MSDC_PS, MSDC_PS_CDDEBOUNCE,
1626
+			      DEFAULT_DEBOUNCE);
1627
+		sdr_set_bits(host->base + MSDC_PS, MSDC_PS_CDEN);
1628
+		sdr_set_bits(host->base + MSDC_INTEN, MSDC_INTEN_CDSC);
1629
+		sdr_set_bits(host->base + SDC_CFG, SDC_CFG_INSWKUP);
1630
+	} else {
1631
+		sdr_clr_bits(host->base + SDC_CFG, SDC_CFG_INSWKUP);
1632
+		sdr_clr_bits(host->base + MSDC_PS, MSDC_PS_CDEN);
1633
+		sdr_clr_bits(host->base + MSDC_INTEN, MSDC_INTEN_CDSC);
1634
+	}
1635
+
1636
+	if (host->top_base) {
1637
+		writel(0, host->top_base + EMMC_TOP_CONTROL);
1638
+		writel(0, host->top_base + EMMC_TOP_CMD);
1639
+	} else {
1640
+		writel(0, host->base + tune_reg);
1641
+	}
13591642 	writel(0, host->base + MSDC_IOCON);
13601643 	sdr_set_field(host->base + MSDC_IOCON, MSDC_IOCON_DDLSEL, 0);
13611644 	writel(0x403c0046, host->base + MSDC_PATCH_BIT);
....@@ -1379,8 +1662,12 @@
13791662 		sdr_set_field(host->base + MSDC_PATCH_BIT2,
13801663 			      MSDC_PB2_RESPWAIT, 3);
13811664 		if (host->dev_comp->enhance_rx) {
1382
-			sdr_set_bits(host->base + SDC_ADV_CFG0,
1383
-				     SDC_RX_ENHANCE_EN);
1665
+			if (host->top_base)
1666
+				sdr_set_bits(host->top_base + EMMC_TOP_CONTROL,
1667
+					     SDC_RX_ENH_EN);
1668
+			else
1669
+				sdr_set_bits(host->base + SDC_ADV_CFG0,
1670
+					     SDC_RX_ENHANCE_EN);
13841671 		} else {
13851672 			sdr_set_field(host->base + MSDC_PATCH_BIT2,
13861673 				      MSDC_PB2_RESPSTSENSEL, 2);
....@@ -1398,11 +1685,26 @@
13981685 		sdr_set_bits(host->base + MSDC_PATCH_BIT2,
13991686 			     MSDC_PB2_SUPPORT_64G);
14001687 	if (host->dev_comp->data_tune) {
1401
-		sdr_set_bits(host->base + tune_reg,
1402
-			     MSDC_PAD_TUNE_RD_SEL | MSDC_PAD_TUNE_CMD_SEL);
1688
+		if (host->top_base) {
1689
+			sdr_set_bits(host->top_base + EMMC_TOP_CONTROL,
1690
+				     PAD_DAT_RD_RXDLY_SEL);
1691
+			sdr_clr_bits(host->top_base + EMMC_TOP_CONTROL,
1692
+				     DATA_K_VALUE_SEL);
1693
+			sdr_set_bits(host->top_base + EMMC_TOP_CMD,
1694
+				     PAD_CMD_RD_RXDLY_SEL);
1695
+		} else {
1696
+			sdr_set_bits(host->base + tune_reg,
1697
+				     MSDC_PAD_TUNE_RD_SEL |
1698
+				     MSDC_PAD_TUNE_CMD_SEL);
1699
+		}
14031700 	} else {
14041701 		/* choose clock tune */
1405
-		sdr_set_bits(host->base + tune_reg, MSDC_PAD_TUNE_RXDLYSEL);
1702
+		if (host->top_base)
1703
+			sdr_set_bits(host->top_base + EMMC_TOP_CONTROL,
1704
+				     PAD_RXDLY_SEL);
1705
+		else
1706
+			sdr_set_bits(host->base + tune_reg,
1707
+				     MSDC_PAD_TUNE_RXDLYSEL);
14061708 	}
14071709 
14081710 	/* Configure to enable SDIO mode.
....@@ -1410,22 +1712,41 @@
14101712 	 */
14111713 	sdr_set_bits(host->base + SDC_CFG, SDC_CFG_SDIO);
14121714 
1413
-	/* disable detect SDIO device interrupt function */
1715
+	/* Config SDIO device detect interrupt function */
14141716 	sdr_clr_bits(host->base + SDC_CFG, SDC_CFG_SDIOIDE);
1717
+	sdr_set_bits(host->base + SDC_ADV_CFG0, SDC_DAT1_IRQ_TRIGGER);
14151718 
14161719 	/* Configure to default data timeout */
14171720 	sdr_set_field(host->base + SDC_CFG, SDC_CFG_DTOC, 3);
14181721 
14191722 	host->def_tune_para.iocon = readl(host->base + MSDC_IOCON);
1420
-	host->def_tune_para.pad_tune = readl(host->base + tune_reg);
14211723 	host->saved_tune_para.iocon = readl(host->base + MSDC_IOCON);
1422
-	host->saved_tune_para.pad_tune = readl(host->base + tune_reg);
1724
+	if (host->top_base) {
1725
+		host->def_tune_para.emmc_top_control =
1726
+			readl(host->top_base + EMMC_TOP_CONTROL);
1727
+		host->def_tune_para.emmc_top_cmd =
1728
+			readl(host->top_base + EMMC_TOP_CMD);
1729
+		host->saved_tune_para.emmc_top_control =
1730
+			readl(host->top_base + EMMC_TOP_CONTROL);
1731
+		host->saved_tune_para.emmc_top_cmd =
1732
+			readl(host->top_base + EMMC_TOP_CMD);
1733
+	} else {
1734
+		host->def_tune_para.pad_tune = readl(host->base + tune_reg);
1735
+		host->saved_tune_para.pad_tune = readl(host->base + tune_reg);
1736
+	}
14231737 	dev_dbg(host->dev, "init hardware done!");
14241738 }
14251739 
14261740 static void msdc_deinit_hw(struct msdc_host *host)
14271741 {
14281742 	u32 val;
1743
+
1744
+	if (host->internal_cd) {
1745
+		/* Disabled card-detect */
1746
+		sdr_clr_bits(host->base + MSDC_PS, MSDC_PS_CDEN);
1747
+		sdr_clr_bits(host->base + SDC_CFG, SDC_CFG_INSWKUP);
1748
+	}
1749
+
14291750 	/* Disable and clear all interrupts */
14301751 	writel(0, host->base + MSDC_INTEN);
14311752 
....@@ -1567,6 +1888,30 @@
15671888 	return delay_phase;
15681889 }
15691890 
1891
+static inline void msdc_set_cmd_delay(struct msdc_host *host, u32 value)
1892
+{
1893
+	u32 tune_reg = host->dev_comp->pad_tune_reg;
1894
+
1895
+	if (host->top_base)
1896
+		sdr_set_field(host->top_base + EMMC_TOP_CMD, PAD_CMD_RXDLY,
1897
+			      value);
1898
+	else
1899
+		sdr_set_field(host->base + tune_reg, MSDC_PAD_TUNE_CMDRDLY,
1900
+			      value);
1901
+}
1902
+
1903
+static inline void msdc_set_data_delay(struct msdc_host *host, u32 value)
1904
+{
1905
+	u32 tune_reg = host->dev_comp->pad_tune_reg;
1906
+
1907
+	if (host->top_base)
1908
+		sdr_set_field(host->top_base + EMMC_TOP_CONTROL,
1909
+			      PAD_DAT_RD_RXDLY, value);
1910
+	else
1911
+		sdr_set_field(host->base + tune_reg, MSDC_PAD_TUNE_DATRRDLY,
1912
+			      value);
1913
+}
1914
+
15701915 static int msdc_tune_response(struct mmc_host *mmc, u32 opcode)
15711916 {
15721917 	struct msdc_host *host = mmc_priv(mmc);
....@@ -1587,8 +1932,7 @@
15871932 
15881933 	sdr_clr_bits(host->base + MSDC_IOCON, MSDC_IOCON_RSPL);
15891934 	for (i = 0 ; i < PAD_DELAY_MAX; i++) {
1590
-		sdr_set_field(host->base + tune_reg,
1591
-			      MSDC_PAD_TUNE_CMDRDLY, i);
1935
+		msdc_set_cmd_delay(host, i);
15921936 		/*
15931937 		 * Using the same parameters, it may sometimes pass the test,
15941938 		 * but sometimes it may fail. To make sure the parameters are
....@@ -1612,8 +1956,7 @@
16121956 
16131957 	sdr_set_bits(host->base + MSDC_IOCON, MSDC_IOCON_RSPL);
16141958 	for (i = 0; i < PAD_DELAY_MAX; i++) {
1615
-		sdr_set_field(host->base + tune_reg,
1616
-			      MSDC_PAD_TUNE_CMDRDLY, i);
1959
+		msdc_set_cmd_delay(host, i);
16171960 		/*
16181961 		 * Using the same parameters, it may sometimes pass the test,
16191962 		 * but sometimes it may fail. To make sure the parameters are
....@@ -1637,15 +1980,13 @@
16371980 		final_maxlen = final_fall_delay.maxlen;
16381981 	if (final_maxlen == final_rise_delay.maxlen) {
16391982 		sdr_clr_bits(host->base + MSDC_IOCON, MSDC_IOCON_RSPL);
1640
-		sdr_set_field(host->base + tune_reg, MSDC_PAD_TUNE_CMDRDLY,
1641
-			      final_rise_delay.final_phase);
16421983 		final_delay = final_rise_delay.final_phase;
16431984 	} else {
16441985 		sdr_set_bits(host->base + MSDC_IOCON, MSDC_IOCON_RSPL);
1645
-		sdr_set_field(host->base + tune_reg, MSDC_PAD_TUNE_CMDRDLY,
1646
-			      final_fall_delay.final_phase);
16471986 		final_delay = final_fall_delay.final_phase;
16481987 	}
1988
+	msdc_set_cmd_delay(host, final_delay);
1989
+
16491990 	if (host->dev_comp->async_fifo || host->hs200_cmd_int_delay)
16501991 		goto skip_internal;
16511992 
....@@ -1721,7 +2062,6 @@
17212062 	u32 rise_delay = 0, fall_delay = 0;
17222063 	struct msdc_delay_phase final_rise_delay, final_fall_delay = { 0,};
17232064 	u8 final_delay, final_maxlen;
1724
-	u32 tune_reg = host->dev_comp->pad_tune_reg;
17252065 	int i, ret;
17262066 
17272067 	sdr_set_field(host->base + MSDC_PATCH_BIT, MSDC_INT_DAT_LATCH_CK_SEL,
....@@ -1729,8 +2069,7 @@
17292069 	sdr_clr_bits(host->base + MSDC_IOCON, MSDC_IOCON_DSPL);
17302070 	sdr_clr_bits(host->base + MSDC_IOCON, MSDC_IOCON_W_DSPL);
17312071 	for (i = 0 ; i < PAD_DELAY_MAX; i++) {
1732
-		sdr_set_field(host->base + tune_reg,
1733
-			      MSDC_PAD_TUNE_DATRRDLY, i);
2072
+		msdc_set_data_delay(host, i);
17342073 		ret = mmc_send_tuning(mmc, opcode, NULL);
17352074 		if (!ret)
17362075 			rise_delay |= (1 << i);
....@@ -1744,8 +2083,7 @@
17442083 	sdr_set_bits(host->base + MSDC_IOCON, MSDC_IOCON_DSPL);
17452084 	sdr_set_bits(host->base + MSDC_IOCON, MSDC_IOCON_W_DSPL);
17462085 	for (i = 0; i < PAD_DELAY_MAX; i++) {
1747
-		sdr_set_field(host->base + tune_reg,
1748
-			      MSDC_PAD_TUNE_DATRRDLY, i);
2086
+		msdc_set_data_delay(host, i);
17492087 		ret = mmc_send_tuning(mmc, opcode, NULL);
17502088 		if (!ret)
17512089 			fall_delay |= (1 << i);
....@@ -1757,20 +2095,79 @@
17572095 	if (final_maxlen == final_rise_delay.maxlen) {
17582096 		sdr_clr_bits(host->base + MSDC_IOCON, MSDC_IOCON_DSPL);
17592097 		sdr_clr_bits(host->base + MSDC_IOCON, MSDC_IOCON_W_DSPL);
1760
-		sdr_set_field(host->base + tune_reg,
1761
-			      MSDC_PAD_TUNE_DATRRDLY,
1762
-			      final_rise_delay.final_phase);
17632098 		final_delay = final_rise_delay.final_phase;
17642099 	} else {
17652100 		sdr_set_bits(host->base + MSDC_IOCON, MSDC_IOCON_DSPL);
17662101 		sdr_set_bits(host->base + MSDC_IOCON, MSDC_IOCON_W_DSPL);
1767
-		sdr_set_field(host->base + tune_reg,
1768
-			      MSDC_PAD_TUNE_DATRRDLY,
1769
-			      final_fall_delay.final_phase);
2102
+		final_delay = final_fall_delay.final_phase;
2103
+	}
2104
+	msdc_set_data_delay(host, final_delay);
2105
+
2106
+	dev_dbg(host->dev, "Final data pad delay: %x\n", final_delay);
2107
+	return final_delay == 0xff ? -EIO : 0;
2108
+}
2109
+
2110
+/*
2111
+ * MSDC IP which supports data tune + async fifo can do CMD/DAT tune
2112
+ * together, which can save the tuning time.
2113
+ */
2114
+static int msdc_tune_together(struct mmc_host *mmc, u32 opcode)
2115
+{
2116
+	struct msdc_host *host = mmc_priv(mmc);
2117
+	u32 rise_delay = 0, fall_delay = 0;
2118
+	struct msdc_delay_phase final_rise_delay, final_fall_delay = { 0,};
2119
+	u8 final_delay, final_maxlen;
2120
+	int i, ret;
2121
+
2122
+	sdr_set_field(host->base + MSDC_PATCH_BIT, MSDC_INT_DAT_LATCH_CK_SEL,
2123
+		      host->latch_ck);
2124
+
2125
+	sdr_clr_bits(host->base + MSDC_IOCON, MSDC_IOCON_RSPL);
2126
+	sdr_clr_bits(host->base + MSDC_IOCON,
2127
+		     MSDC_IOCON_DSPL | MSDC_IOCON_W_DSPL);
2128
+	for (i = 0 ; i < PAD_DELAY_MAX; i++) {
2129
+		msdc_set_cmd_delay(host, i);
2130
+		msdc_set_data_delay(host, i);
2131
+		ret = mmc_send_tuning(mmc, opcode, NULL);
2132
+		if (!ret)
2133
+			rise_delay |= (1 << i);
2134
+	}
2135
+	final_rise_delay = get_best_delay(host, rise_delay);
2136
+	/* if rising edge has enough margin, then do not scan falling edge */
2137
+	if (final_rise_delay.maxlen >= 12 ||
2138
+	    (final_rise_delay.start == 0 && final_rise_delay.maxlen >= 4))
2139
+		goto skip_fall;
2140
+
2141
+	sdr_set_bits(host->base + MSDC_IOCON, MSDC_IOCON_RSPL);
2142
+	sdr_set_bits(host->base + MSDC_IOCON,
2143
+		     MSDC_IOCON_DSPL | MSDC_IOCON_W_DSPL);
2144
+	for (i = 0; i < PAD_DELAY_MAX; i++) {
2145
+		msdc_set_cmd_delay(host, i);
2146
+		msdc_set_data_delay(host, i);
2147
+		ret = mmc_send_tuning(mmc, opcode, NULL);
2148
+		if (!ret)
2149
+			fall_delay |= (1 << i);
2150
+	}
2151
+	final_fall_delay = get_best_delay(host, fall_delay);
2152
+
2153
+skip_fall:
2154
+	final_maxlen = max(final_rise_delay.maxlen, final_fall_delay.maxlen);
2155
+	if (final_maxlen == final_rise_delay.maxlen) {
2156
+		sdr_clr_bits(host->base + MSDC_IOCON, MSDC_IOCON_RSPL);
2157
+		sdr_clr_bits(host->base + MSDC_IOCON,
2158
+			     MSDC_IOCON_DSPL | MSDC_IOCON_W_DSPL);
2159
+		final_delay = final_rise_delay.final_phase;
2160
+	} else {
2161
+		sdr_set_bits(host->base + MSDC_IOCON, MSDC_IOCON_RSPL);
2162
+		sdr_set_bits(host->base + MSDC_IOCON,
2163
+			     MSDC_IOCON_DSPL | MSDC_IOCON_W_DSPL);
17702164 		final_delay = final_fall_delay.final_phase;
17712165 	}
17722166 
1773
-	dev_dbg(host->dev, "Final data pad delay: %x\n", final_delay);
2167
+	msdc_set_cmd_delay(host, final_delay);
2168
+	msdc_set_data_delay(host, final_delay);
2169
+
2170
+	dev_dbg(host->dev, "Final pad delay: %x\n", final_delay);
17742171 	return final_delay == 0xff ? -EIO : 0;
17752172 }
17762173 
....@@ -1780,6 +2177,15 @@
17802177 	int ret;
17812178 	u32 tune_reg = host->dev_comp->pad_tune_reg;
17822179 
2180
+	if (host->dev_comp->data_tune && host->dev_comp->async_fifo) {
2181
+		ret = msdc_tune_together(mmc, opcode);
2182
+		if (host->hs400_mode) {
2183
+			sdr_clr_bits(host->base + MSDC_IOCON,
2184
+				     MSDC_IOCON_DSPL | MSDC_IOCON_W_DSPL);
2185
+			msdc_set_data_delay(host, 0);
2186
+		}
2187
+		goto tune_done;
2188
+	}
17832189 	if (host->hs400_mode &&
17842190 	    host->dev_comp->hs400_tune)
17852191 		ret = hs400_tune_response(mmc, opcode);
....@@ -1795,9 +2201,16 @@
17952201 			dev_err(host->dev, "Tune data fail!\n");
17962202 	}
17972203 
2204
+tune_done:
17982205 	host->saved_tune_para.iocon = readl(host->base + MSDC_IOCON);
17992206 	host->saved_tune_para.pad_tune = readl(host->base + tune_reg);
18002207 	host->saved_tune_para.pad_cmd_tune = readl(host->base + PAD_CMD_TUNE);
2208
+	if (host->top_base) {
2209
+		host->saved_tune_para.emmc_top_control = readl(host->top_base +
2210
+				EMMC_TOP_CONTROL);
2211
+		host->saved_tune_para.emmc_top_cmd = readl(host->top_base +
2212
+				EMMC_TOP_CMD);
2213
+	}
18012214 	return ret;
18022215 }
18032216 
....@@ -1806,7 +2219,11 @@
18062219 	struct msdc_host *host = mmc_priv(mmc);
18072220 	host->hs400_mode = true;
18082221 
1809
-	writel(host->hs400_ds_delay, host->base + PAD_DS_TUNE);
2222
+	if (host->top_base)
2223
+		writel(host->hs400_ds_delay,
2224
+		       host->top_base + EMMC50_PAD_DS_TUNE);
2225
+	else
2226
+		writel(host->hs400_ds_delay, host->base + PAD_DS_TUNE);
18102227 	/* hs400 mode must set it to 0 */
18112228 	sdr_clr_bits(host->base + MSDC_PATCH_BIT2, MSDC_PATCH_BIT2_CFGCRCSTS);
18122229 	/* to improve read performance, set outstanding to 2 */
....@@ -1824,18 +2241,112 @@
18242241 	sdr_clr_bits(host->base + EMMC_IOCON, 1);
18252242 }
18262243 
2244
+static void msdc_ack_sdio_irq(struct mmc_host *mmc)
2245
+{
2246
+	unsigned long flags;
2247
+	struct msdc_host *host = mmc_priv(mmc);
2248
+
2249
+	spin_lock_irqsave(&host->lock, flags);
2250
+	__msdc_enable_sdio_irq(host, 1);
2251
+	spin_unlock_irqrestore(&host->lock, flags);
2252
+}
2253
+
2254
+static int msdc_get_cd(struct mmc_host *mmc)
2255
+{
2256
+	struct msdc_host *host = mmc_priv(mmc);
2257
+	int val;
2258
+
2259
+	if (mmc->caps & MMC_CAP_NONREMOVABLE)
2260
+		return 1;
2261
+
2262
+	if (!host->internal_cd)
2263
+		return mmc_gpio_get_cd(mmc);
2264
+
2265
+	val = readl(host->base + MSDC_PS) & MSDC_PS_CDSTS;
2266
+	if (mmc->caps2 & MMC_CAP2_CD_ACTIVE_HIGH)
2267
+		return !!val;
2268
+	else
2269
+		return !val;
2270
+}
2271
+
2272
+static void msdc_cqe_enable(struct mmc_host *mmc)
2273
+{
2274
+	struct msdc_host *host = mmc_priv(mmc);
2275
+
2276
+	/* enable cmdq irq */
2277
+	writel(MSDC_INT_CMDQ, host->base + MSDC_INTEN);
2278
+	/* enable busy check */
2279
+	sdr_set_bits(host->base + MSDC_PATCH_BIT1, MSDC_PB1_BUSY_CHECK_SEL);
2280
+	/* default write data / busy timeout 20s */
2281
+	msdc_set_busy_timeout(host, 20 * 1000000000ULL, 0);
2282
+	/* default read data timeout 1s */
2283
+	msdc_set_timeout(host, 1000000000ULL, 0);
2284
+}
2285
+
2286
+static void msdc_cqe_disable(struct mmc_host *mmc, bool recovery)
2287
+{
2288
+	struct msdc_host *host = mmc_priv(mmc);
2289
+	unsigned int val = 0;
2290
+
2291
+	/* disable cmdq irq */
2292
+	sdr_clr_bits(host->base + MSDC_INTEN, MSDC_INT_CMDQ);
2293
+	/* disable busy check */
2294
+	sdr_clr_bits(host->base + MSDC_PATCH_BIT1, MSDC_PB1_BUSY_CHECK_SEL);
2295
+
2296
+	val = readl(host->base + MSDC_INT);
2297
+	writel(val, host->base + MSDC_INT);
2298
+
2299
+	if (recovery) {
2300
+		sdr_set_field(host->base + MSDC_DMA_CTRL,
2301
+			      MSDC_DMA_CTRL_STOP, 1);
2302
+		if (WARN_ON(readl_poll_timeout(host->base + MSDC_DMA_CFG, val,
2303
+			!(val & MSDC_DMA_CFG_STS), 1, 3000)))
2304
+			return;
2305
+		msdc_reset_hw(host);
2306
+	}
2307
+}
2308
+
2309
+static void msdc_cqe_pre_enable(struct mmc_host *mmc)
2310
+{
2311
+	struct cqhci_host *cq_host = mmc->cqe_private;
2312
+	u32 reg;
2313
+
2314
+	reg = cqhci_readl(cq_host, CQHCI_CFG);
2315
+	reg |= CQHCI_ENABLE;
2316
+	cqhci_writel(cq_host, reg, CQHCI_CFG);
2317
+}
2318
+
2319
+static void msdc_cqe_post_disable(struct mmc_host *mmc)
2320
+{
2321
+	struct cqhci_host *cq_host = mmc->cqe_private;
2322
+	u32 reg;
2323
+
2324
+	reg = cqhci_readl(cq_host, CQHCI_CFG);
2325
+	reg &= ~CQHCI_ENABLE;
2326
+	cqhci_writel(cq_host, reg, CQHCI_CFG);
2327
+}
2328
+
18272329 static const struct mmc_host_ops mt_msdc_ops = {
18282330 	.post_req = msdc_post_req,
18292331 	.pre_req = msdc_pre_req,
18302332 	.request = msdc_ops_request,
18312333 	.set_ios = msdc_ops_set_ios,
18322334 	.get_ro = mmc_gpio_get_ro,
1833
-	.get_cd = mmc_gpio_get_cd,
2335
+	.get_cd = msdc_get_cd,
2336
+	.enable_sdio_irq = msdc_enable_sdio_irq,
2337
+	.ack_sdio_irq = msdc_ack_sdio_irq,
18342338 	.start_signal_voltage_switch = msdc_ops_switch_volt,
18352339 	.card_busy = msdc_card_busy,
18362340 	.execute_tuning = msdc_execute_tuning,
18372341 	.prepare_hs400_tuning = msdc_prepare_hs400_tuning,
18382342 	.hw_reset = msdc_hw_reset,
2343
+};
2344
+
2345
+static const struct cqhci_host_ops msdc_cmdq_ops = {
2346
+	.enable         = msdc_cqe_enable,
2347
+	.disable        = msdc_cqe_disable,
2348
+	.pre_enable = msdc_cqe_pre_enable,
2349
+	.post_disable = msdc_cqe_post_disable,
18392350 };
18402351 
18412352 static void msdc_of_property_parse(struct platform_device *pdev,
....@@ -1858,6 +2369,12 @@
18582369 		host->hs400_cmd_resp_sel_rising = true;
18592370 	else
18602371 		host->hs400_cmd_resp_sel_rising = false;
2372
+
2373
+	if (of_property_read_bool(pdev->dev.of_node,
2374
+				  "supports-cqe"))
2375
+		host->cqhci = true;
2376
+	else
2377
+		host->cqhci = false;
18612378 }
18622379 
18632380 static int msdc_drv_probe(struct platform_device *pdev)
....@@ -1882,11 +2399,17 @@
18822399 	if (ret)
18832400 		goto host_free;
18842401 
1885
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1886
-	host->base = devm_ioremap_resource(&pdev->dev, res);
2402
+	host->base = devm_platform_ioremap_resource(pdev, 0);
18872403 	if (IS_ERR(host->base)) {
18882404 		ret = PTR_ERR(host->base);
18892405 		goto host_free;
2406
+	}
2407
+
2408
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
2409
+	if (res) {
2410
+		host->top_base = devm_ioremap_resource(&pdev->dev, res);
2411
+		if (IS_ERR(host->top_base))
2412
+			host->top_base = NULL;
18902413 	}
18912414 
18922415 	ret = mmc_regulator_get_supply(mmc);
....@@ -1905,10 +2428,18 @@
19052428 		goto host_free;
19062429 	}
19072430 
2431
+	host->bus_clk = devm_clk_get(&pdev->dev, "bus_clk");
2432
+	if (IS_ERR(host->bus_clk))
2433
+		host->bus_clk = NULL;
19082434 	/*source clock control gate is optional clock*/
19092435 	host->src_clk_cg = devm_clk_get(&pdev->dev, "source_cg");
19102436 	if (IS_ERR(host->src_clk_cg))
19112437 		host->src_clk_cg = NULL;
2438
+
2439
+	host->reset = devm_reset_control_get_optional_exclusive(&pdev->dev,
2440
+								"hrst");
2441
+	if (IS_ERR(host->reset))
2442
+		return PTR_ERR(host->reset);
19122443 
19132444 	host->irq = platform_get_irq(pdev, 0);
19142445 	if (host->irq < 0) {
....@@ -1941,7 +2472,6 @@
19412472 
19422473 	host->dev = &pdev->dev;
19432474 	host->dev_comp = of_device_get_match_data(&pdev->dev);
1944
-	host->mmc = mmc;
19452475 	host->src_clk_freq = clk_get_rate(host->src_clk);
19462476 	/* Set host parameters to mmc */
19472477 	mmc->ops = &mt_msdc_ops;
....@@ -1950,10 +2480,28 @@
19502480 	else
19512481 		mmc->f_min = DIV_ROUND_UP(host->src_clk_freq, 4 * 4095);
19522482 
1953
-	mmc->caps |= MMC_CAP_ERASE | MMC_CAP_CMD23;
2483
+	if (!(mmc->caps & MMC_CAP_NONREMOVABLE) &&
2484
+	    !mmc_can_gpio_cd(mmc) &&
2485
+	    host->dev_comp->use_internal_cd) {
2486
+		/*
2487
+		 * Is removable but no GPIO declared, so
2488
+		 * use internal functionality.
2489
+		 */
2490
+		host->internal_cd = true;
2491
+	}
2492
+
2493
+	if (mmc->caps & MMC_CAP_SDIO_IRQ)
2494
+		mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
2495
+
2496
+	mmc->caps |= MMC_CAP_CMD23;
2497
+	if (host->cqhci)
2498
+		mmc->caps2 |= MMC_CAP2_CQE | MMC_CAP2_CQE_DCMD;
19542499 	/* MMC core transfer sizes tunable parameters */
19552500 	mmc->max_segs = MAX_BD_NUM;
1956
-	mmc->max_seg_size = BDMA_DESC_BUFLEN;
2501
+	if (host->dev_comp->support_64g)
2502
+		mmc->max_seg_size = BDMA_DESC_BUFLEN_EXT;
2503
+	else
2504
+		mmc->max_seg_size = BDMA_DESC_BUFLEN;
19572505 	mmc->max_blk_size = 2048;
19582506 	mmc->max_req_size = 512 * 1024;
19592507 	mmc->max_blk_count = mmc->max_req_size / 512;
....@@ -1982,8 +2530,28 @@
19822530 	msdc_ungate_clock(host);
19832531 	msdc_init_hw(host);
19842532 
2533
+	if (mmc->caps2 & MMC_CAP2_CQE) {
2534
+		host->cq_host = devm_kzalloc(mmc->parent,
2535
+					     sizeof(*host->cq_host),
2536
+					     GFP_KERNEL);
2537
+		if (!host->cq_host) {
2538
+			ret = -ENOMEM;
2539
+			goto host_free;
2540
+		}
2541
+		host->cq_host->caps |= CQHCI_TASK_DESC_SZ_128;
2542
+		host->cq_host->mmio = host->base + 0x800;
2543
+		host->cq_host->ops = &msdc_cmdq_ops;
2544
+		ret = cqhci_init(host->cq_host, mmc, true);
2545
+		if (ret)
2546
+			goto host_free;
2547
+		mmc->max_segs = 128;
2548
+		/* cqhci 16bit length */
2549
+		/* 0 size, means 65536 so we don't have to -1 here */
2550
+		mmc->max_seg_size = 64 * 1024;
2551
+	}
2552
+
19852553 	ret = devm_request_irq(&pdev->dev, host->irq, msdc_irq,
1986
-		IRQF_TRIGGER_LOW | IRQF_ONESHOT, pdev->name, host);
2554
+			       IRQF_TRIGGER_NONE, pdev->name, host);
19872555 	if (ret)
19882556 		goto release;
19892557 
....@@ -2029,7 +2597,7 @@
20292597 	pm_runtime_get_sync(host->dev);
20302598 
20312599 	platform_set_drvdata(pdev, NULL);
2032
-	mmc_remove_host(host->mmc);
2600
+	mmc_remove_host(mmc);
20332601 	msdc_deinit_hw(host);
20342602 	msdc_gate_clock(host);
20352603 
....@@ -2041,12 +2609,11 @@
20412609 	dma_free_coherent(&pdev->dev, MAX_BD_NUM * sizeof(struct mt_bdma_desc),
20422610 			host->dma.bd, host->dma.bd_addr);
20432611 
2044
-	mmc_free_host(host->mmc);
2612
+	mmc_free_host(mmc);
20452613 
20462614 	return 0;
20472615 }
20482616 
2049
-#ifdef CONFIG_PM
20502617 static void msdc_save_reg(struct msdc_host *host)
20512618 {
20522619 	u32 tune_reg = host->dev_comp->pad_tune_reg;
....@@ -2054,7 +2621,6 @@
20542621 	host->save_para.msdc_cfg = readl(host->base + MSDC_CFG);
20552622 	host->save_para.iocon = readl(host->base + MSDC_IOCON);
20562623 	host->save_para.sdc_cfg = readl(host->base + SDC_CFG);
2057
-	host->save_para.pad_tune = readl(host->base + tune_reg);
20582624 	host->save_para.patch_bit0 = readl(host->base + MSDC_PATCH_BIT);
20592625 	host->save_para.patch_bit1 = readl(host->base + MSDC_PATCH_BIT1);
20602626 	host->save_para.patch_bit2 = readl(host->base + MSDC_PATCH_BIT2);
....@@ -2063,16 +2629,26 @@
20632629 	host->save_para.emmc50_cfg0 = readl(host->base + EMMC50_CFG0);
20642630 	host->save_para.emmc50_cfg3 = readl(host->base + EMMC50_CFG3);
20652631 	host->save_para.sdc_fifo_cfg = readl(host->base + SDC_FIFO_CFG);
2632
+	if (host->top_base) {
2633
+		host->save_para.emmc_top_control =
2634
+			readl(host->top_base + EMMC_TOP_CONTROL);
2635
+		host->save_para.emmc_top_cmd =
2636
+			readl(host->top_base + EMMC_TOP_CMD);
2637
+		host->save_para.emmc50_pad_ds_tune =
2638
+			readl(host->top_base + EMMC50_PAD_DS_TUNE);
2639
+	} else {
2640
+		host->save_para.pad_tune = readl(host->base + tune_reg);
2641
+	}
20662642 }
20672643 
20682644 static void msdc_restore_reg(struct msdc_host *host)
20692645 {
2646
+	struct mmc_host *mmc = mmc_from_priv(host);
20702647 	u32 tune_reg = host->dev_comp->pad_tune_reg;
20712648 
20722649 	writel(host->save_para.msdc_cfg, host->base + MSDC_CFG);
20732650 	writel(host->save_para.iocon, host->base + MSDC_IOCON);
20742651 	writel(host->save_para.sdc_cfg, host->base + SDC_CFG);
2075
-	writel(host->save_para.pad_tune, host->base + tune_reg);
20762652 	writel(host->save_para.patch_bit0, host->base + MSDC_PATCH_BIT);
20772653 	writel(host->save_para.patch_bit1, host->base + MSDC_PATCH_BIT1);
20782654 	writel(host->save_para.patch_bit2, host->base + MSDC_PATCH_BIT2);
....@@ -2081,9 +2657,22 @@
20812657 	writel(host->save_para.emmc50_cfg0, host->base + EMMC50_CFG0);
20822658 	writel(host->save_para.emmc50_cfg3, host->base + EMMC50_CFG3);
20832659 	writel(host->save_para.sdc_fifo_cfg, host->base + SDC_FIFO_CFG);
2660
+	if (host->top_base) {
2661
+		writel(host->save_para.emmc_top_control,
2662
+		       host->top_base + EMMC_TOP_CONTROL);
2663
+		writel(host->save_para.emmc_top_cmd,
2664
+		       host->top_base + EMMC_TOP_CMD);
2665
+		writel(host->save_para.emmc50_pad_ds_tune,
2666
+		       host->top_base + EMMC50_PAD_DS_TUNE);
2667
+	} else {
2668
+		writel(host->save_para.pad_tune, host->base + tune_reg);
2669
+	}
2670
+
2671
+	if (sdio_irq_claimed(mmc))
2672
+		__msdc_enable_sdio_irq(host, 1);
20842673 }
20852674 
2086
-static int msdc_runtime_suspend(struct device *dev)
2675
+static int __maybe_unused msdc_runtime_suspend(struct device *dev)
20872676 {
20882677 	struct mmc_host *mmc = dev_get_drvdata(dev);
20892678 	struct msdc_host *host = mmc_priv(mmc);
....@@ -2093,7 +2682,7 @@
20932682 	return 0;
20942683 }
20952684 
2096
-static int msdc_runtime_resume(struct device *dev)
2685
+static int __maybe_unused msdc_runtime_resume(struct device *dev)
20972686 {
20982687 	struct mmc_host *mmc = dev_get_drvdata(dev);
20992688 	struct msdc_host *host = mmc_priv(mmc);
....@@ -2102,11 +2691,31 @@
21022691 	msdc_restore_reg(host);
21032692 	return 0;
21042693 }
2105
-#endif
2694
+
2695
+static int __maybe_unused msdc_suspend(struct device *dev)
2696
+{
2697
+	struct mmc_host *mmc = dev_get_drvdata(dev);
2698
+	int ret;
2699
+	u32 val;
2700
+
2701
+	if (mmc->caps2 & MMC_CAP2_CQE) {
2702
+		ret = cqhci_suspend(mmc);
2703
+		if (ret)
2704
+			return ret;
2705
+		val = readl(((struct msdc_host *)mmc_priv(mmc))->base + MSDC_INT);
2706
+		writel(val, ((struct msdc_host *)mmc_priv(mmc))->base + MSDC_INT);
2707
+	}
2708
+
2709
+	return pm_runtime_force_suspend(dev);
2710
+}
2711
+
2712
+static int __maybe_unused msdc_resume(struct device *dev)
2713
+{
2714
+	return pm_runtime_force_resume(dev);
2715
+}
21062716 
21072717 static const struct dev_pm_ops msdc_dev_pm_ops = {
2108
-	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
2109
-				pm_runtime_force_resume)
2718
+	SET_SYSTEM_SLEEP_PM_OPS(msdc_suspend, msdc_resume)
21102719 	SET_RUNTIME_PM_OPS(msdc_runtime_suspend, msdc_runtime_resume, NULL)
21112720 };
21122721