forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-10-09 244b2c5ca8b14627e4a17755e5922221e121c771 
 kernel/drivers/dma/tegra210-adma.c
....@@ -1,19 +1,8 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * ADMA driver for Nvidia's Tegra210 ADMA controller.
34  *
45  * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
5
- *
6
- * This program is free software; you can redistribute it and/or modify it
7
- * under the terms and conditions of the GNU General Public License,
8
- * version 2, as published by the Free Software Foundation.
9
- *
10
- * This program is distributed in the hope it will be useful, but WITHOUT
11
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13
- * more details.
14
- *
15
- * You should have received a copy of the GNU General Public License
16
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
176  */
187 
198 #include <linux/clk.h>
....@@ -30,35 +19,34 @@
3019 #define ADMA_CH_CMD					0x00
3120 #define ADMA_CH_STATUS					0x0c
3221 #define ADMA_CH_STATUS_XFER_EN				BIT(0)
22
+#define ADMA_CH_STATUS_XFER_PAUSED			BIT(1)
3323 
3424 #define ADMA_CH_INT_STATUS				0x10
3525 #define ADMA_CH_INT_STATUS_XFER_DONE			BIT(0)
3626 
3727 #define ADMA_CH_INT_CLEAR				0x1c
3828 #define ADMA_CH_CTRL					0x24
39
-#define ADMA_CH_CTRL_TX_REQ(val)			(((val) & 0xf) << 28)
40
-#define ADMA_CH_CTRL_TX_REQ_MAX				10
41
-#define ADMA_CH_CTRL_RX_REQ(val)			(((val) & 0xf) << 24)
42
-#define ADMA_CH_CTRL_RX_REQ_MAX				10
4329 #define ADMA_CH_CTRL_DIR(val)				(((val) & 0xf) << 12)
4430 #define ADMA_CH_CTRL_DIR_AHUB2MEM			2
4531 #define ADMA_CH_CTRL_DIR_MEM2AHUB			4
4632 #define ADMA_CH_CTRL_MODE_CONTINUOUS			(2 << 8)
4733 #define ADMA_CH_CTRL_FLOWCTRL_EN			BIT(1)
34
+#define ADMA_CH_CTRL_XFER_PAUSE_SHIFT			0
4835 
4936 #define ADMA_CH_CONFIG					0x28
5037 #define ADMA_CH_CONFIG_SRC_BUF(val)			(((val) & 0x7) << 28)
5138 #define ADMA_CH_CONFIG_TRG_BUF(val)			(((val) & 0x7) << 24)
52
-#define ADMA_CH_CONFIG_BURST_SIZE(val)			(((val) & 0x7) << 20)
53
-#define ADMA_CH_CONFIG_BURST_16				5
39
+#define ADMA_CH_CONFIG_BURST_SIZE_SHIFT			20
40
+#define ADMA_CH_CONFIG_MAX_BURST_SIZE                   16
5441 #define ADMA_CH_CONFIG_WEIGHT_FOR_WRR(val)		((val) & 0xf)
5542 #define ADMA_CH_CONFIG_MAX_BUFS				8
43
+#define TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(reqs)	(reqs << 4)
5644 
5745 #define ADMA_CH_FIFO_CTRL				0x2c
58
-#define ADMA_CH_FIFO_CTRL_OVRFW_THRES(val)		(((val) & 0xf) << 24)
59
-#define ADMA_CH_FIFO_CTRL_STARV_THRES(val)		(((val) & 0xf) << 16)
60
-#define ADMA_CH_FIFO_CTRL_TX_SIZE(val)			(((val) & 0xf) << 8)
61
-#define ADMA_CH_FIFO_CTRL_RX_SIZE(val)			((val) & 0xf)
46
+#define TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(val)		(((val) & 0xf) << 8)
47
+#define TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(val)		((val) & 0xf)
48
+#define TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(val)		(((val) & 0x1f) << 8)
49
+#define TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(val)		((val) & 0x1f)
6250 
6351 #define ADMA_CH_LOWER_SRC_ADDR				0x34
6452 #define ADMA_CH_LOWER_TRG_ADDR				0x3c
....@@ -68,25 +56,48 @@
6856 #define ADMA_CH_XFER_STATUS				0x54
6957 #define ADMA_CH_XFER_STATUS_COUNT_MASK			0xffff
7058 
71
-#define ADMA_GLOBAL_CMD					0xc00
72
-#define ADMA_GLOBAL_SOFT_RESET				0xc04
73
-#define ADMA_GLOBAL_INT_CLEAR				0xc20
74
-#define ADMA_GLOBAL_CTRL				0xc24
59
+#define ADMA_GLOBAL_CMD					0x00
60
+#define ADMA_GLOBAL_SOFT_RESET				0x04
7561 
76
-#define ADMA_CH_REG_OFFSET(a)				(a * 0x80)
62
+#define TEGRA_ADMA_BURST_COMPLETE_TIME			20
7763 
78
-#define ADMA_CH_FIFO_CTRL_DEFAULT	(ADMA_CH_FIFO_CTRL_OVRFW_THRES(1) | \
79
-					 ADMA_CH_FIFO_CTRL_STARV_THRES(1) | \
80
-					 ADMA_CH_FIFO_CTRL_TX_SIZE(3)     | \
81
-					 ADMA_CH_FIFO_CTRL_RX_SIZE(3))
64
+#define TEGRA210_FIFO_CTRL_DEFAULT (TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \
65
+				    TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(3))
66
+
67
+#define TEGRA186_FIFO_CTRL_DEFAULT (TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \
68
+				    TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(3))
69
+
70
+#define ADMA_CH_REG_FIELD_VAL(val, mask, shift)	(((val) & mask) << shift)
71
+
8272 struct tegra_adma;
8373 
8474 /*
8575  * struct tegra_adma_chip_data - Tegra chip specific data
76
+ * @global_reg_offset: Register offset of DMA global register.
77
+ * @global_int_clear: Register offset of DMA global interrupt clear.
78
+ * @ch_req_tx_shift: Register offset for AHUB transmit channel select.
79
+ * @ch_req_rx_shift: Register offset for AHUB receive channel select.
80
+ * @ch_base_offset: Register offset of DMA channel registers.
81
+ * @has_outstanding_reqs: If DMA channel can have outstanding requests.
82
+ * @ch_fifo_ctrl: Default value for channel FIFO CTRL register.
83
+ * @ch_req_mask: Mask for Tx or Rx channel select.
84
+ * @ch_req_max: Maximum number of Tx or Rx channels available.
85
+ * @ch_reg_size: Size of DMA channel register space.
8686  * @nr_channels: Number of DMA channels available.
8787  */
8888 struct tegra_adma_chip_data {
89
-	int nr_channels;
89
+	unsigned int (*adma_get_burst_config)(unsigned int burst_size);
90
+	unsigned int global_reg_offset;
91
+	unsigned int global_int_clear;
92
+	unsigned int ch_req_tx_shift;
93
+	unsigned int ch_req_rx_shift;
94
+	unsigned int ch_base_offset;
95
+	unsigned int ch_fifo_ctrl;
96
+	unsigned int ch_req_mask;
97
+	unsigned int ch_req_max;
98
+	unsigned int ch_reg_size;
99
+	unsigned int nr_channels;
100
+	bool has_outstanding_reqs;
90101 };
91102 
92103 /*
....@@ -150,18 +161,20 @@
150161 	/* Used to store global command register state when suspending */
151162 	unsigned int			global_cmd;
152163 
164
+	const struct tegra_adma_chip_data *cdata;
165
+
153166 	/* Last member of the structure */
154
-	struct tegra_adma_chan		channels[0];
167
+	struct tegra_adma_chan		channels[];
155168 };
156169 
157170 static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val)
158171 {
159
-	writel(val, tdma->base_addr + reg);
172
+	writel(val, tdma->base_addr + tdma->cdata->global_reg_offset + reg);
160173 }
161174 
162175 static inline u32 tdma_read(struct tegra_adma *tdma, u32 reg)
163176 {
164
-	return readl(tdma->base_addr + reg);
177
+	return readl(tdma->base_addr + tdma->cdata->global_reg_offset + reg);
165178 }
166179 
167180 static inline void tdma_ch_write(struct tegra_adma_chan *tdc, u32 reg, u32 val)
....@@ -211,14 +224,16 @@
211224 	int ret;
212225 
213226 	/* Clear any interrupts */
214
-	tdma_write(tdma, ADMA_GLOBAL_INT_CLEAR, 0x1);
227
+	tdma_write(tdma, tdma->cdata->ch_base_offset + tdma->cdata->global_int_clear, 0x1);
215228 
216229 	/* Assert soft reset */
217230 	tdma_write(tdma, ADMA_GLOBAL_SOFT_RESET, 0x1);
218231 
219232 	/* Wait for reset to clear */
220233 	ret = readx_poll_timeout(readl,
221
-				 tdma->base_addr + ADMA_GLOBAL_SOFT_RESET,
234
+				 tdma->base_addr +
235
+				 tdma->cdata->global_reg_offset +
236
+				 ADMA_GLOBAL_SOFT_RESET,
222237 				 status, status == 0, 20, 10000);
223238 	if (ret)
224239 		return ret;
....@@ -238,13 +253,13 @@
238253 	if (tdc->sreq_reserved)
239254 		return tdc->sreq_dir == direction ? 0 : -EINVAL;
240255 
256
+	if (sreq_index > tdma->cdata->ch_req_max) {
257
+		dev_err(tdma->dev, "invalid DMA request\n");
258
+		return -EINVAL;
259
+	}
260
+
241261 	switch (direction) {
242262 	case DMA_MEM_TO_DEV:
243
-		if (sreq_index > ADMA_CH_CTRL_TX_REQ_MAX) {
244
-			dev_err(tdma->dev, "invalid DMA request\n");
245
-			return -EINVAL;
246
-		}
247
-
248263 		if (test_and_set_bit(sreq_index, &tdma->tx_requests_reserved)) {
249264 			dev_err(tdma->dev, "DMA request reserved\n");
250265 			return -EINVAL;
....@@ -252,11 +267,6 @@
252267 		break;
253268 
254269 	case DMA_DEV_TO_MEM:
255
-		if (sreq_index > ADMA_CH_CTRL_RX_REQ_MAX) {
256
-			dev_err(tdma->dev, "invalid DMA request\n");
257
-			return -EINVAL;
258
-		}
259
-
260270 		if (test_and_set_bit(sreq_index, &tdma->rx_requests_reserved)) {
261271 			dev_err(tdma->dev, "DMA request reserved\n");
262272 			return -EINVAL;
....@@ -430,6 +440,51 @@
430440 	spin_unlock_irqrestore(&tdc->vc.lock, flags);
431441 }
432442 
443
+static bool tegra_adma_is_paused(struct tegra_adma_chan *tdc)
444
+{
445
+	u32 csts;
446
+
447
+	csts = tdma_ch_read(tdc, ADMA_CH_STATUS);
448
+	csts &= ADMA_CH_STATUS_XFER_PAUSED;
449
+
450
+	return csts ? true : false;
451
+}
452
+
453
+static int tegra_adma_pause(struct dma_chan *dc)
454
+{
455
+	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
456
+	struct tegra_adma_desc *desc = tdc->desc;
457
+	struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;
458
+	int dcnt = 10;
459
+
460
+	ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);
461
+	ch_regs->ctrl |= (1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT);
462
+	tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);
463
+
464
+	while (dcnt-- && !tegra_adma_is_paused(tdc))
465
+		udelay(TEGRA_ADMA_BURST_COMPLETE_TIME);
466
+
467
+	if (dcnt < 0) {
468
+		dev_err(tdc2dev(tdc), "unable to pause DMA channel\n");
469
+		return -EBUSY;
470
+	}
471
+
472
+	return 0;
473
+}
474
+
475
+static int tegra_adma_resume(struct dma_chan *dc)
476
+{
477
+	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
478
+	struct tegra_adma_desc *desc = tdc->desc;
479
+	struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;
480
+
481
+	ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);
482
+	ch_regs->ctrl &= ~(1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT);
483
+	tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);
484
+
485
+	return 0;
486
+}
487
+
433488 static int tegra_adma_terminate_all(struct dma_chan *dc)
434489 {
435490 	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
....@@ -483,12 +538,29 @@
483538 	return ret;
484539 }
485540 
541
+static unsigned int tegra210_adma_get_burst_config(unsigned int burst_size)
542
+{
543
+	if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE)
544
+		burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE;
545
+
546
+	return fls(burst_size) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT;
547
+}
548
+
549
+static unsigned int tegra186_adma_get_burst_config(unsigned int burst_size)
550
+{
551
+	if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE)
552
+		burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE;
553
+
554
+	return (burst_size - 1) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT;
555
+}
556
+
486557 static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc,
487558 				      struct tegra_adma_desc *desc,
488559 				      dma_addr_t buf_addr,
489560 				      enum dma_transfer_direction direction)
490561 {
491562 	struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;
563
+	const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata;
492564 	unsigned int burst_size, adma_dir;
493565 
494566 	if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS)
....@@ -497,17 +569,21 @@
497569 	switch (direction) {
498570 	case DMA_MEM_TO_DEV:
499571 		adma_dir = ADMA_CH_CTRL_DIR_MEM2AHUB;
500
-		burst_size = fls(tdc->sconfig.dst_maxburst);
572
+		burst_size = tdc->sconfig.dst_maxburst;
501573 		ch_regs->config = ADMA_CH_CONFIG_SRC_BUF(desc->num_periods - 1);
502
-		ch_regs->ctrl = ADMA_CH_CTRL_TX_REQ(tdc->sreq_index);
574
+		ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index,
575
+						      cdata->ch_req_mask,
576
+						      cdata->ch_req_tx_shift);
503577 		ch_regs->src_addr = buf_addr;
504578 		break;
505579 
506580 	case DMA_DEV_TO_MEM:
507581 		adma_dir = ADMA_CH_CTRL_DIR_AHUB2MEM;
508
-		burst_size = fls(tdc->sconfig.src_maxburst);
582
+		burst_size = tdc->sconfig.src_maxburst;
509583 		ch_regs->config = ADMA_CH_CONFIG_TRG_BUF(desc->num_periods - 1);
510
-		ch_regs->ctrl = ADMA_CH_CTRL_RX_REQ(tdc->sreq_index);
584
+		ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index,
585
+						      cdata->ch_req_mask,
586
+						      cdata->ch_req_rx_shift);
511587 		ch_regs->trg_addr = buf_addr;
512588 		break;
513589 
....@@ -516,15 +592,14 @@
516592 		return -EINVAL;
517593 	}
518594 
519
-	if (!burst_size || burst_size > ADMA_CH_CONFIG_BURST_16)
520
-		burst_size = ADMA_CH_CONFIG_BURST_16;
521
-
522595 	ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir) |
523596 			 ADMA_CH_CTRL_MODE_CONTINUOUS |
524597 			 ADMA_CH_CTRL_FLOWCTRL_EN;
525
-	ch_regs->config |= ADMA_CH_CONFIG_BURST_SIZE(burst_size);
598
+	ch_regs->config |= cdata->adma_get_burst_config(burst_size);
526599 	ch_regs->config |= ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1);
527
-	ch_regs->fifo_ctrl = ADMA_CH_FIFO_CTRL_DEFAULT;
600
+	if (cdata->has_outstanding_reqs)
601
+		ch_regs->config |= TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(8);
602
+	ch_regs->fifo_ctrl = cdata->ch_fifo_ctrl;
528603 	ch_regs->tc = desc->period_len & ADMA_CH_TC_COUNT_MASK;
529604 
530605 	return tegra_adma_request_alloc(tdc, direction);
....@@ -635,7 +710,7 @@
635710 	return chan;
636711 }
637712 
638
-static int tegra_adma_runtime_suspend(struct device *dev)
713
+static int __maybe_unused tegra_adma_runtime_suspend(struct device *dev)
639714 {
640715 	struct tegra_adma *tdma = dev_get_drvdata(dev);
641716 	struct tegra_adma_chan_regs *ch_reg;
....@@ -667,7 +742,7 @@
667742 	return 0;
668743 }
669744 
670
-static int tegra_adma_runtime_resume(struct device *dev)
745
+static int __maybe_unused tegra_adma_runtime_resume(struct device *dev)
671746 {
672747 	struct tegra_adma *tdma = dev_get_drvdata(dev);
673748 	struct tegra_adma_chan_regs *ch_reg;
....@@ -703,11 +778,38 @@
703778 }
704779 
705780 static const struct tegra_adma_chip_data tegra210_chip_data = {
706
-	.nr_channels = 22,
781
+	.adma_get_burst_config  = tegra210_adma_get_burst_config,
782
+	.global_reg_offset	= 0xc00,
783
+	.global_int_clear	= 0x20,
784
+	.ch_req_tx_shift	= 28,
785
+	.ch_req_rx_shift	= 24,
786
+	.ch_base_offset		= 0,
787
+	.has_outstanding_reqs	= false,
788
+	.ch_fifo_ctrl		= TEGRA210_FIFO_CTRL_DEFAULT,
789
+	.ch_req_mask		= 0xf,
790
+	.ch_req_max		= 10,
791
+	.ch_reg_size		= 0x80,
792
+	.nr_channels		= 22,
793
+};
794
+
795
+static const struct tegra_adma_chip_data tegra186_chip_data = {
796
+	.adma_get_burst_config  = tegra186_adma_get_burst_config,
797
+	.global_reg_offset	= 0,
798
+	.global_int_clear	= 0x402c,
799
+	.ch_req_tx_shift	= 27,
800
+	.ch_req_rx_shift	= 22,
801
+	.ch_base_offset		= 0x10000,
802
+	.has_outstanding_reqs	= true,
803
+	.ch_fifo_ctrl		= TEGRA186_FIFO_CTRL_DEFAULT,
804
+	.ch_req_mask		= 0x1f,
805
+	.ch_req_max		= 20,
806
+	.ch_reg_size		= 0x100,
807
+	.nr_channels		= 32,
707808 };
708809 
709810 static const struct of_device_id tegra_adma_of_match[] = {
710811 	{ .compatible = "nvidia,tegra210-adma", .data = &tegra210_chip_data },
812
+	{ .compatible = "nvidia,tegra186-adma", .data = &tegra186_chip_data },
711813 	{ },
712814 };
713815 MODULE_DEVICE_TABLE(of, tegra_adma_of_match);
....@@ -725,12 +827,14 @@
725827 		return -ENODEV;
726828 	}
727829 
728
-	tdma = devm_kzalloc(&pdev->dev, sizeof(*tdma) + cdata->nr_channels *
729
-			    sizeof(struct tegra_adma_chan), GFP_KERNEL);
830
+	tdma = devm_kzalloc(&pdev->dev,
831
+			    struct_size(tdma, channels, cdata->nr_channels),
832
+			    GFP_KERNEL);
730833 	if (!tdma)
731834 		return -ENOMEM;
732835 
733836 	tdma->dev = &pdev->dev;
837
+	tdma->cdata = cdata;
734838 	tdma->nr_channels = cdata->nr_channels;
735839 	platform_set_drvdata(pdev, tdma);
736840 
....@@ -749,7 +853,8 @@
749853 	for (i = 0; i < tdma->nr_channels; i++) {
750854 		struct tegra_adma_chan *tdc = &tdma->channels[i];
751855 
752
-		tdc->chan_addr = tdma->base_addr + ADMA_CH_REG_OFFSET(i);
856
+		tdc->chan_addr = tdma->base_addr + cdata->ch_base_offset
857
+				 + (cdata->ch_reg_size * i);
753858 
754859 		tdc->irq = of_irq_get(pdev->dev.of_node, i);
755860 		if (tdc->irq <= 0) {
....@@ -792,6 +897,8 @@
792897 	tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
793898 	tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
794899 	tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
900
+	tdma->dma_dev.device_pause = tegra_adma_pause;
901
+	tdma->dma_dev.device_resume = tegra_adma_resume;
795902 
796903 	ret = dma_async_device_register(&tdma->dma_dev);
797904 	if (ret < 0) {
....@@ -843,17 +950,11 @@
843950 	return 0;
844951 }
845952 
846
-#ifdef CONFIG_PM_SLEEP
847
-static int tegra_adma_pm_suspend(struct device *dev)
848
-{
849
-	return pm_runtime_suspended(dev) == false;
850
-}
851
-#endif
852
-
853953 static const struct dev_pm_ops tegra_adma_dev_pm_ops = {
854954 	SET_RUNTIME_PM_OPS(tegra_adma_runtime_suspend,
855955 			   tegra_adma_runtime_resume, NULL)
856
-	SET_SYSTEM_SLEEP_PM_OPS(tegra_adma_pm_suspend, NULL)
956
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
957
+				     pm_runtime_force_resume)
857958 };
858959 
859960 static struct platform_driver tegra_admac_driver = {