~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,16 +1,13 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* Copyright (c) 2012 Samsung Electronics Co., Ltd.
3	4	* http://www.samsung.com
4	5	*
5	6	* Copyright (C) 2010 Samsung Electronics Co. Ltd.
6	7	* Jaswinder Singh <jassi.brar@samsung.com>
7		- *
8		- * This program is free software; you can redistribute it and/or modify
9		- * it under the terms of the GNU General Public License as published by
10		- * the Free Software Foundation; either version 2 of the License, or
11		- * (at your option) any later version.
12	8	*/
13	9
	10	+#include <linux/debugfs.h>
14	11	#include <linux/kernel.h>
15	12	#include <linux/io.h>
16	13	#include <linux/init.h>
..	..	@@ -28,6 +25,7 @@
28	25	#include <linux/err.h>
29	26	#include <linux/pm_runtime.h>
30	27	#include <linux/bug.h>
	28	+#include <linux/reset.h>
31	29
32	30	#include "dmaengine.h"
33	31	#define PL330_MAX_CHAN 8
..	..	@@ -37,16 +35,6 @@
37	35
38	36	#define PL330_QUIRK_BROKEN_NO_FLUSHP BIT(0)
39	37	#define PL330_QUIRK_PERIPH_BURST BIT(1)
40		-
41		-#ifdef CONFIG_CPU_RV1126
42		-#undef writel
43		-#define writel(v, c) \
44		- do { \
45		- readl_relaxed(c); \
46		- __iowmb(); \
47		- writel_relaxed(v, c); \
48		- } while (0)
49		-#endif
50	38
51	39	enum pl330_cachectrl {
52	40	CCTRL0, /* Noncacheable and nonbufferable */
..	..	@@ -268,7 +256,7 @@
268	256	static unsigned cmd_line;
269	257	#define PL330_DBGCMD_DUMP(off, x...) do { \
270	258	printk("%x:", cmd_line); \
271		- printk(x); \
	259	+ printk(KERN_CONT x); \
272	260	cmd_line += off; \
273	261	} while (0)
274	262	#define PL330_DBGMC_START(addr) (cmd_line = addr)
..	..	@@ -298,7 +286,7 @@
298	286	u32 irq_ns;
299	287	};
300	288
301		-/**
	289	+/*
302	290	* Request Configuration.
303	291	* The PL330 core does not modify this and uses the last
304	292	* working configuration if the request doesn't provide any.
..	..	@@ -417,6 +405,12 @@
417	405	*/
418	406	BUSY,
419	407	/*
	408	+ * Pause was called while descriptor was BUSY. Due to hardware
	409	+ * limitations, only termination is possible for descriptors
	410	+ * that have been paused.
	411	+ */
	412	+ PAUSED,
	413	+ /*
420	414	* Sitting on the channel work_list but xfer done
421	415	* by PL330 core
422	416	*/
..	..	@@ -460,8 +454,7 @@
460	454	/* DMA-mapped view of the FIFO; may differ if an IOMMU is present */
461	455	dma_addr_t fifo_dma;
462	456	enum dma_data_direction dir;
463		- unsigned int src_interlace_size;
464		- unsigned int dst_interlace_size;
	457	+ struct dma_slave_config slave_config;
465	458
466	459	/* for runtime pm tracking */
467	460	bool active;
..	..	@@ -470,9 +463,6 @@
470	463	struct pl330_dmac {
471	464	/* DMA-Engine Device */
472	465	struct dma_device ddma;
473		-
474		- /* Holds info about sg limitations */
475		- struct device_dma_parameters dma_parms;
476	466
477	467	/* Pool of descriptors available for the DMAC's channels */
478	468	struct list_head desc_pool;
..	..	@@ -509,6 +499,9 @@
509	499	unsigned int num_peripherals;
510	500	struct dma_pl330_chan peripherals; / keep at end */
511	501	int quirks;
	502	+
	503	+ struct reset_control *rstc;
	504	+ struct reset_control *rstc_ocp;
512	505	};
513	506
514	507	static struct pl330_of_quirks {
..	..	@@ -554,15 +547,19 @@
554	547	/* For cyclic capability */
555	548	bool cyclic;
556	549	size_t num_periods;
557		- /* interlace size */
558		- unsigned int src_interlace_size;
559		- unsigned int dst_interlace_size;
	550	+
	551	+ /* interleaved size */
	552	+ struct data_chunk sgl;
560	553	};
561	554
562	555	struct _xfer_spec {
563	556	u32 ccr;
564	557	struct dma_pl330_desc *desc;
565	558	};
	559	+
	560	+static int pl330_config_write(struct dma_chan *chan,
	561	+ struct dma_slave_config *slave_config,
	562	+ enum dma_transfer_direction direction);
566	563
567	564	static inline bool _queue_full(struct pl330_thread *thrd)
568	565	{
..	..	@@ -1080,7 +1077,7 @@
1080	1077	return true;
1081	1078	}
1082	1079
1083		-static bool _start(struct pl330_thread *thrd)
	1080	+static bool pl330_start_thread(struct pl330_thread *thrd)
1084	1081	{
1085	1082	switch (_state(thrd)) {
1086	1083	case PL330_STATE_FAULT_COMPLETING:
..	..	@@ -1088,16 +1085,16 @@
1088	1085
1089	1086	if (_state(thrd) == PL330_STATE_KILLING)
1090	1087	UNTIL(thrd, PL330_STATE_STOPPED)
1091		- /* fall through */
	1088	+ fallthrough;
1092	1089
1093	1090	case PL330_STATE_FAULTING:
1094	1091	_stop(thrd);
1095		- /* fall through */
	1092	+ fallthrough;
1096	1093
1097	1094	case PL330_STATE_KILLING:
1098	1095	case PL330_STATE_COMPLETING:
1099	1096	UNTIL(thrd, PL330_STATE_STOPPED)
1100		- /* fall through */
	1097	+ fallthrough;
1101	1098
1102	1099	case PL330_STATE_STOPPED:
1103	1100	return _trigger(thrd);
..	..	@@ -1148,7 +1145,6 @@
1148	1145
1149	1146	switch (direction) {
1150	1147	case DMA_MEM_TO_MEM:
1151		- /* fall through */
1152	1148	case DMA_MEM_TO_DEV:
1153	1149	off += _emit_LD(dry_run, &buf[off], cond);
1154	1150	break;
..	..	@@ -1182,7 +1178,6 @@
1182	1178
1183	1179	switch (direction) {
1184	1180	case DMA_MEM_TO_MEM:
1185		- /* fall through */
1186	1181	case DMA_DEV_TO_MEM:
1187	1182	off += _emit_ST(dry_run, &buf[off], cond);
1188	1183	break;
..	..	@@ -1213,7 +1208,7 @@
1213	1208	const struct _xfer_spec *pxs, int cyc,
1214	1209	enum pl330_cond cond)
1215	1210	{
1216		- int off = 0;
	1211	+ int off = 0, i = 0, burstn = 1;
1217	1212
1218	1213	/*
1219	1214	* do FLUSHP at beginning to clear any stale dma requests before the
..	..	@@ -1221,22 +1216,31 @@
1221	1216	*/
1222	1217	if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
1223	1218	off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri);
	1219	+
	1220	+ if (pxs->desc->sgl.size) {
	1221	+ WARN_ON(BYTE_MOD_BURST_LEN(pxs->desc->sgl.size, pxs->ccr));
	1222	+ burstn = BYTE_TO_BURST(pxs->desc->sgl.size, pxs->ccr);
	1223	+ }
	1224	+
1224	1225	while (cyc--) {
1225		- off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri);
1226		- off += _emit_load(dry_run, &buf[off], cond, pxs->desc->rqtype,
1227		- pxs->desc->peri);
1228		- off += _emit_store(dry_run, &buf[off], cond, pxs->desc->rqtype,
1229		- pxs->desc->peri);
	1226	+ for (i = 0; i < burstn; i++) {
	1227	+ off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri);
	1228	+ off += _emit_load(dry_run, &buf[off], cond, pxs->desc->rqtype,
	1229	+ pxs->desc->peri);
	1230	+ off += _emit_store(dry_run, &buf[off], cond, pxs->desc->rqtype,
	1231	+ pxs->desc->peri);
	1232	+ }
	1233	+
1230	1234	switch (pxs->desc->rqtype) {
1231	1235	case DMA_DEV_TO_MEM:
1232		- if (pxs->desc->dst_interlace_size)
	1236	+ if (pxs->desc->sgl.dst_icg)
1233	1237	off += _emit_ADDH(dry_run, &buf[off], DST,
1234		- pxs->desc->dst_interlace_size);
	1238	+ pxs->desc->sgl.dst_icg);
1235	1239	break;
1236	1240	case DMA_MEM_TO_DEV:
1237		- if (pxs->desc->src_interlace_size)
	1241	+ if (pxs->desc->sgl.src_icg)
1238	1242	off += _emit_ADDH(dry_run, &buf[off], SRC,
1239		- pxs->desc->src_interlace_size);
	1243	+ pxs->desc->sgl.src_icg);
1240	1244	break;
1241	1245	default:
1242	1246	WARN_ON(1);
..	..	@@ -1258,7 +1262,6 @@
1258	1262
1259	1263	switch (pxs->desc->rqtype) {
1260	1264	case DMA_MEM_TO_DEV:
1261		- /* fall through */
1262	1265	case DMA_DEV_TO_MEM:
1263	1266	off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs, cyc,
1264	1267	cond);
..	..	@@ -1293,7 +1296,7 @@
1293	1296
1294	1297	switch (pxs->desc->rqtype) {
1295	1298	case DMA_MEM_TO_DEV:
1296		- /* fall through */
	1299	+ fallthrough;
1297	1300	case DMA_DEV_TO_MEM:
1298	1301	/*
1299	1302	* dregs_len = (total bytes - BURST_TO_BYTE(bursts, ccr)) /
..	..	@@ -1457,8 +1460,7 @@
1457	1460	off += _emit_LPEND(dry_run, &buf[off], &lpend);
1458	1461	}
1459	1462
1460		- if (!pxs->desc->src_interlace_size &&
1461		- !pxs->desc->dst_interlace_size) {
	1463	+ if (!pxs->desc->sgl.src_icg && !pxs->desc->sgl.dst_icg) {
1462	1464	num_dregs = BYTE_MOD_BURST_LEN(x->bytes, pxs->ccr);
1463	1465
1464	1466	if (num_dregs) {
..	..	@@ -1533,19 +1535,16 @@
1533	1535	BRST_SIZE(ccr);
1534	1536	int off = 0;
1535	1537
1536		- if (pxs->desc->rqtype == DMA_DEV_TO_MEM)
1537		- bursts = x->bytes / (BRST_SIZE(ccr) * BRST_LEN(ccr) +
1538		- pxs->desc->dst_interlace_size);
1539		- else if (pxs->desc->rqtype == DMA_MEM_TO_DEV)
1540		- bursts = x->bytes / (BRST_SIZE(ccr) * BRST_LEN(ccr) +
1541		- pxs->desc->src_interlace_size);
	1538	+ if (pxs->desc->sgl.size)
	1539	+ bursts = x->bytes / pxs->desc->sgl.size;
	1540	+
1542	1541	while (bursts) {
1543	1542	c = bursts;
1544	1543	off += _loop(pl330, dry_run, &buf[off], &c, pxs);
1545	1544	bursts -= c;
1546	1545	}
1547		- if (!pxs->desc->src_interlace_size &&
1548		- !pxs->desc->dst_interlace_size)
	1546	+
	1547	+ if (!pxs->desc->sgl.src_icg && !pxs->desc->sgl.dst_icg)
1549	1548	off += _dregs(pl330, dry_run, &buf[off], pxs, num_dregs);
1550	1549
1551	1550	return off;
..	..	@@ -1578,12 +1577,9 @@
1578	1577	unsigned long bursts = BYTE_TO_BURST(x->bytes, ccr);
1579	1578	int off = 0;
1580	1579
1581		- if (pxs->desc->rqtype == DMA_DEV_TO_MEM)
1582		- bursts = x->bytes / (BRST_SIZE(ccr) * BRST_LEN(ccr)
1583		- + pxs->desc->dst_interlace_size);
1584		- else if (pxs->desc->rqtype == DMA_MEM_TO_DEV)
1585		- bursts = x->bytes / (BRST_SIZE(ccr) * BRST_LEN(ccr)
1586		- + pxs->desc->src_interlace_size);
	1580	+ if (pxs->desc->sgl.size)
	1581	+ bursts = x->bytes / pxs->desc->sgl.size;
	1582	+
1587	1583	/* Setup Loop(s) */
1588	1584	off += _loop_cyclic(pl330, dry_run, &buf[off], bursts, pxs, ev);
1589	1585
..	..	@@ -1767,9 +1763,9 @@
1767	1763	tasklet_schedule(&pch->task);
1768	1764	}
1769	1765
1770		-static void pl330_dotask(unsigned long data)
	1766	+static void pl330_dotask(struct tasklet_struct *t)
1771	1767	{
1772		- struct pl330_dmac pl330 = (struct pl330_dmac ) data;
	1768	+ struct pl330_dmac *pl330 = from_tasklet(pl330, t, tasks);
1773	1769	unsigned long flags;
1774	1770	int i;
1775	1771
..	..	@@ -1898,7 +1894,7 @@
1898	1894	thrd->req[active].desc = NULL;
1899	1895	thrd->req_running = -1;
1900	1896	/* Get going again ASAP */
1901		- _start(thrd);
	1897	+ pl330_start_thread(thrd);
1902	1898	}
1903	1899
1904	1900	/* For now, just make a list of callbacks to be done */
..	..	@@ -2126,9 +2122,10 @@
2126	2122	if (ret) {
2127	2123	dev_err(pl330->ddma.dev, "%s:%d Can't to create channels for DMAC!\n",
2128	2124	__func__, __LINE__);
2129		- dma_free_coherent(pl330->ddma.dev,
	2125	+ dma_free_attrs(pl330->ddma.dev,
2130	2126	chans * pl330->mcbufsz,
2131		- pl330->mcode_cpu, pl330->mcode_bus);
	2127	+ pl330->mcode_cpu, pl330->mcode_bus,
	2128	+ DMA_ATTR_PRIVILEGED);
2132	2129	return ret;
2133	2130	}
2134	2131
..	..	@@ -2174,7 +2171,7 @@
2174	2171	return ret;
2175	2172	}
2176	2173
2177		- tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330);
	2174	+ tasklet_setup(&pl330->tasks, pl330_dotask);
2178	2175
2179	2176	pl330->state = INIT;
2180	2177
..	..	@@ -2207,9 +2204,9 @@
2207	2204	/* Free DMAC resources */
2208	2205	dmac_free_threads(pl330);
2209	2206
2210		- dma_free_coherent(pl330->ddma.dev,
	2207	+ dma_free_attrs(pl330->ddma.dev,
2211	2208	pl330->pcfg.num_chan * pl330->mcbufsz, pl330->mcode_cpu,
2212		- pl330->mcode_bus);
	2209	+ pl330->mcode_bus, DMA_ATTR_PRIVILEGED);
2213	2210	}
2214	2211
2215	2212	/* forward declaration */
..	..	@@ -2238,7 +2235,7 @@
2238	2235	list_for_each_entry(desc, &pch->work_list, node) {
2239	2236
2240	2237	/* If already submitted */
2241		- if (desc->status == BUSY)
	2238	+ if (desc->status == BUSY \|\| desc->status == PAUSED)
2242	2239	continue;
2243	2240
2244	2241	ret = pl330_submit_req(pch->thread, desc);
..	..	@@ -2257,9 +2254,9 @@
2257	2254	}
2258	2255	}
2259	2256
2260		-static void pl330_tasklet(unsigned long data)
	2257	+static void pl330_tasklet(struct tasklet_struct *t)
2261	2258	{
2262		- struct dma_pl330_chan pch = (struct dma_pl330_chan )data;
	2259	+ struct dma_pl330_chan *pch = from_tasklet(pch, t, task);
2263	2260	struct dma_pl330_desc desc, _dt;
2264	2261	unsigned long flags;
2265	2262	bool power_down = false;
..	..	@@ -2299,7 +2296,7 @@
2299	2296	} else {
2300	2297	/* Make sure the PL330 Channel thread is active */
2301	2298	spin_lock(&pch->thread->dmac->lock);
2302		- _start(pch->thread);
	2299	+ pl330_start_thread(pch->thread);
2303	2300	spin_unlock(&pch->thread->dmac->lock);
2304	2301	}
2305	2302
..	..	@@ -2367,7 +2364,7 @@
2367	2364	return -ENOMEM;
2368	2365	}
2369	2366
2370		- tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);
	2367	+ tasklet_setup(&pch->task, pl330_tasklet);
2371	2368
2372	2369	spin_unlock_irqrestore(&pl330->lock, flags);
2373	2370
..	..	@@ -2432,31 +2429,38 @@
2432	2429	return max_burst_len;
2433	2430	}
2434	2431
	2432	+static int pl330_config_write(struct dma_chan *chan,
	2433	+ struct dma_slave_config *slave_config,
	2434	+ enum dma_transfer_direction direction)
	2435	+{
	2436	+ struct dma_pl330_chan *pch = to_pchan(chan);
	2437	+
	2438	+ pl330_unprep_slave_fifo(pch);
	2439	+ if (direction == DMA_MEM_TO_DEV) {
	2440	+ if (slave_config->dst_addr)
	2441	+ pch->fifo_addr = slave_config->dst_addr;
	2442	+ if (slave_config->dst_addr_width)
	2443	+ pch->burst_sz = __ffs(slave_config->dst_addr_width);
	2444	+ pch->burst_len = fixup_burst_len(slave_config->dst_maxburst,
	2445	+ pch->dmac->quirks);
	2446	+ } else if (direction == DMA_DEV_TO_MEM) {
	2447	+ if (slave_config->src_addr)
	2448	+ pch->fifo_addr = slave_config->src_addr;
	2449	+ if (slave_config->src_addr_width)
	2450	+ pch->burst_sz = __ffs(slave_config->src_addr_width);
	2451	+ pch->burst_len = fixup_burst_len(slave_config->src_maxburst,
	2452	+ pch->dmac->quirks);
	2453	+ }
	2454	+
	2455	+ return 0;
	2456	+}
	2457	+
2435	2458	static int pl330_config(struct dma_chan *chan,
2436	2459	struct dma_slave_config *slave_config)
2437	2460	{
2438	2461	struct dma_pl330_chan *pch = to_pchan(chan);
2439	2462
2440		- pl330_unprep_slave_fifo(pch);
2441		- if (slave_config->direction == DMA_MEM_TO_DEV) {
2442		- if (slave_config->dst_addr)
2443		- pch->fifo_addr = slave_config->dst_addr;
2444		- if (slave_config->dst_addr_width)
2445		- pch->burst_sz = __ffs(slave_config->dst_addr_width);
2446		- if (slave_config->src_interlace_size)
2447		- pch->src_interlace_size = slave_config->src_interlace_size;
2448		- pch->burst_len = fixup_burst_len(slave_config->dst_maxburst,
2449		- pch->dmac->quirks);
2450		- } else if (slave_config->direction == DMA_DEV_TO_MEM) {
2451		- if (slave_config->src_addr)
2452		- pch->fifo_addr = slave_config->src_addr;
2453		- if (slave_config->src_addr_width)
2454		- pch->burst_sz = __ffs(slave_config->src_addr_width);
2455		- if (slave_config->dst_interlace_size)
2456		- pch->dst_interlace_size = slave_config->dst_interlace_size;
2457		- pch->burst_len = fixup_burst_len(slave_config->src_maxburst,
2458		- pch->dmac->quirks);
2459		- }
	2463	+ memcpy(&pch->slave_config, slave_config, sizeof(*slave_config));
2460	2464
2461	2465	return 0;
2462	2466	}
..	..	@@ -2467,7 +2471,6 @@
2467	2471	struct dma_pl330_desc *desc;
2468	2472	unsigned long flags;
2469	2473	struct pl330_dmac *pl330 = pch->dmac;
2470		- LIST_HEAD(list);
2471	2474	bool power_down = false;
2472	2475
2473	2476	pm_runtime_get_sync(pl330->ddma.dev);
..	..	@@ -2517,6 +2520,7 @@
2517	2520	{
2518	2521	struct dma_pl330_chan *pch = to_pchan(chan);
2519	2522	struct pl330_dmac *pl330 = pch->dmac;
	2523	+ struct dma_pl330_desc *desc;
2520	2524	unsigned long flags;
2521	2525
2522	2526	pm_runtime_get_sync(pl330->ddma.dev);
..	..	@@ -2526,6 +2530,10 @@
2526	2530	_stop(pch->thread);
2527	2531	spin_unlock(&pl330->lock);
2528	2532
	2533	+ list_for_each_entry(desc, &pch->work_list, node) {
	2534	+ if (desc->status == BUSY)
	2535	+ desc->status = PAUSED;
	2536	+ }
2529	2537	spin_unlock_irqrestore(&pch->lock, flags);
2530	2538	pm_runtime_mark_last_busy(pl330->ddma.dev);
2531	2539	pm_runtime_put_autosuspend(pl330->ddma.dev);
..	..	@@ -2615,7 +2623,7 @@
2615	2623	else if (running && desc == running)
2616	2624	transferred =
2617	2625	pl330_get_current_xferred_count(pch, desc);
2618		- else if (desc->status == BUSY)
	2626	+ else if (desc->status == BUSY \|\| desc->status == PAUSED)
2619	2627	/*
2620	2628	* Busy but not running means either just enqueued,
2621	2629	* or finished and not yet marked done
..	..	@@ -2631,6 +2639,9 @@
2631	2639	switch (desc->status) {
2632	2640	case DONE:
2633	2641	ret = DMA_COMPLETE;
	2642	+ break;
	2643	+ case PAUSED:
	2644	+ ret = DMA_PAUSED;
2634	2645	break;
2635	2646	case PREP:
2636	2647	case BUSY:
..	..	@@ -2672,7 +2683,7 @@
2672	2683	list_splice_tail_init(&pch->submitted_list, &pch->work_list);
2673	2684	spin_unlock_irqrestore(&pch->lock, flags);
2674	2685
2675		- pl330_tasklet((unsigned long)pch);
	2686	+ pl330_tasklet(&pch->task);
2676	2687	}
2677	2688
2678	2689	/*
..	..	@@ -2776,7 +2787,7 @@
2776	2787
2777	2788	/* If the DMAC pool is empty, alloc new */
2778	2789	if (!desc) {
2779		- DEFINE_SPINLOCK(lock);
	2790	+ static DEFINE_SPINLOCK(lock);
2780	2791	LIST_HEAD(pool);
2781	2792
2782	2793	if (!add_desc(&pool, &lock, GFP_ATOMIC, 1))
..	..	@@ -2796,6 +2807,10 @@
2796	2807
2797	2808	desc->cyclic = false;
2798	2809	desc->num_periods = 1;
	2810	+
	2811	+ desc->sgl.size = 0;
	2812	+ desc->sgl.src_icg = 0;
	2813	+ desc->sgl.dst_icg = 0;
2799	2814
2800	2815	dma_async_tx_descriptor_init(&desc->txd, &pch->chan);
2801	2816
..	..	@@ -2862,8 +2877,8 @@
2862	2877	{
2863	2878	struct dma_pl330_desc *desc = NULL;
2864	2879	struct dma_pl330_chan *pch = to_pchan(chan);
2865		- dma_addr_t dst;
2866		- dma_addr_t src;
	2880	+ dma_addr_t dst = 0;
	2881	+ dma_addr_t src = 0;
2867	2882
2868	2883	if (len % period_len != 0)
2869	2884	return NULL;
..	..	@@ -2873,6 +2888,8 @@
2873	2888	__func__, __LINE__);
2874	2889	return NULL;
2875	2890	}
	2891	+
	2892	+ pl330_config_write(chan, &pch->slave_config, direction);
2876	2893
2877	2894	if (!pl330_prep_slave_fifo(pch, direction))
2878	2895	return NULL;
..	..	@@ -2911,8 +2928,80 @@
2911	2928	desc->num_periods = len / period_len;
2912	2929	desc->txd.flags = flags;
2913	2930
2914		- desc->src_interlace_size = pch->src_interlace_size;
2915		- desc->dst_interlace_size = pch->dst_interlace_size;
	2931	+ return &desc->txd;
	2932	+}
	2933	+
	2934	+static struct dma_async_tx_descriptor *pl330_prep_interleaved_dma(
	2935	+ struct dma_chan chan, struct dma_interleaved_template xt,
	2936	+ unsigned long flags)
	2937	+{
	2938	+ struct dma_pl330_desc *desc = NULL;
	2939	+ struct dma_pl330_chan *pch = to_pchan(chan);
	2940	+ dma_addr_t dst = 0, src = 0;
	2941	+ size_t size, src_icg, dst_icg, period_bytes, buffer_bytes, full_buffer_bytes;
	2942	+ size_t nump = 0, numf = 0;
	2943	+
	2944	+ if (!xt->numf \|\| !xt->sgl[0].size \|\| xt->frame_size != 1)
	2945	+ return NULL;
	2946	+
	2947	+#ifdef CONFIG_NO_GKI
	2948	+ nump = xt->nump;
	2949	+#endif
	2950	+ numf = xt->numf;
	2951	+ size = xt->sgl[0].size;
	2952	+ period_bytes = size * nump;
	2953	+ buffer_bytes = size * numf;
	2954	+
	2955	+ if (flags & DMA_PREP_REPEAT && (!nump \|\| (numf % nump)))
	2956	+ return NULL;
	2957	+
	2958	+ src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
	2959	+ dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
	2960	+
	2961	+ pl330_config_write(chan, &pch->slave_config, xt->dir);
	2962	+
	2963	+ if (!pl330_prep_slave_fifo(pch, xt->dir))
	2964	+ return NULL;
	2965	+
	2966	+ desc = pl330_get_desc(pch);
	2967	+ if (!desc) {
	2968	+ dev_err(chan->device->dev, "Failed to get desc\n");
	2969	+ return NULL;
	2970	+ }
	2971	+
	2972	+ if (xt->dir == DMA_MEM_TO_DEV) {
	2973	+ desc->rqcfg.src_inc = 1;
	2974	+ desc->rqcfg.dst_inc = 0;
	2975	+ src = xt->src_start;
	2976	+ dst = pch->fifo_dma;
	2977	+ full_buffer_bytes = (size + src_icg) * numf;
	2978	+ } else {
	2979	+ desc->rqcfg.src_inc = 0;
	2980	+ desc->rqcfg.dst_inc = 1;
	2981	+ src = pch->fifo_dma;
	2982	+ dst = xt->dst_start;
	2983	+ full_buffer_bytes = (size + dst_icg) * numf;
	2984	+ }
	2985	+
	2986	+ desc->rqtype = xt->dir;
	2987	+ desc->rqcfg.brst_size = pch->burst_sz;
	2988	+ desc->rqcfg.brst_len = pch->burst_len;
	2989	+ desc->bytes_requested = full_buffer_bytes;
	2990	+ desc->sgl.size = size;
	2991	+ desc->sgl.src_icg = src_icg;
	2992	+ desc->sgl.dst_icg = dst_icg;
	2993	+ desc->txd.flags = flags;
	2994	+
	2995	+ if (flags & DMA_PREP_REPEAT) {
	2996	+ desc->cyclic = true;
	2997	+ desc->num_periods = numf / nump;
	2998	+ fill_px(&desc->px, dst, src, period_bytes);
	2999	+ } else {
	3000	+ fill_px(&desc->px, dst, src, buffer_bytes);
	3001	+ }
	3002	+
	3003	+ dev_dbg(chan->device->dev, "size: %zu, src_icg: %zu, dst_icg: %zu, nump: %zu, numf: %zu\n",
	3004	+ size, src_icg, dst_icg, nump, numf);
2916	3005
2917	3006	return &desc->txd;
2918	3007	}
..	..	@@ -3004,6 +3093,8 @@
3004	3093	if (unlikely(!pch \|\| !sgl \|\| !sg_len))
3005	3094	return NULL;
3006	3095
	3096	+ pl330_config_write(chan, &pch->slave_config, direction);
	3097	+
3007	3098	if (!pl330_prep_slave_fifo(pch, direction))
3008	3099	return NULL;
3009	3100
..	..	@@ -3044,8 +3135,6 @@
3044	3135	desc->rqcfg.brst_len = pch->burst_len;
3045	3136	desc->rqtype = direction;
3046	3137	desc->bytes_requested = sg_dma_len(sg);
3047		- desc->src_interlace_size = pch->src_interlace_size;
3048		- desc->dst_interlace_size = pch->dst_interlace_size;
3049	3138	}
3050	3139
3051	3140	/* Return the last desc in the chain */
..	..	@@ -3068,6 +3157,55 @@
3068	3157	BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) \| \
3069	3158	BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)
3070	3159
	3160	+#ifdef CONFIG_DEBUG_FS
	3161	+static int pl330_debugfs_show(struct seq_file s, void data)
	3162	+{
	3163	+ struct pl330_dmac *pl330 = s->private;
	3164	+ int chans, pchs, ch, pr;
	3165	+
	3166	+ chans = pl330->pcfg.num_chan;
	3167	+ pchs = pl330->num_peripherals;
	3168	+
	3169	+ seq_puts(s, "PL330 physical channels:\n");
	3170	+ seq_puts(s, "THREAD:\t\tCHANNEL:\n");
	3171	+ seq_puts(s, "--------\t-----\n");
	3172	+ for (ch = 0; ch < chans; ch++) {
	3173	+ struct pl330_thread *thrd = &pl330->channels[ch];
	3174	+ int found = -1;
	3175	+
	3176	+ for (pr = 0; pr < pchs; pr++) {
	3177	+ struct dma_pl330_chan *pch = &pl330->peripherals[pr];
	3178	+
	3179	+ if (!pch->thread \|\| thrd->id != pch->thread->id)
	3180	+ continue;
	3181	+
	3182	+ found = pr;
	3183	+ }
	3184	+
	3185	+ seq_printf(s, "%d\t\t", thrd->id);
	3186	+ if (found == -1)
	3187	+ seq_puts(s, "--\n");
	3188	+ else
	3189	+ seq_printf(s, "%d\n", found);
	3190	+ }
	3191	+
	3192	+ return 0;
	3193	+}
	3194	+
	3195	+DEFINE_SHOW_ATTRIBUTE(pl330_debugfs);
	3196	+
	3197	+static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
	3198	+{
	3199	+ debugfs_create_file(dev_name(pl330->ddma.dev),
	3200	+ S_IFREG \| 0444, NULL, pl330,
	3201	+ &pl330_debugfs_fops);
	3202	+}
	3203	+#else
	3204	+static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
	3205	+{
	3206	+}
	3207	+#endif
	3208	+
3071	3209	/*
3072	3210	* Runtime PM callbacks are provided by amba/bus.c driver.
3073	3211	*
..	..	@@ -3078,12 +3216,7 @@
3078	3216	{
3079	3217	struct amba_device *pcdev = to_amba_device(dev);
3080	3218
3081		- pm_runtime_disable(dev);
3082		-
3083		- if (!pm_runtime_status_suspended(dev)) {
3084		- /* amba did not disable the clock */
3085		- amba_pclk_disable(pcdev);
3086		- }
	3219	+ pm_runtime_force_suspend(dev);
3087	3220	amba_pclk_unprepare(pcdev);
3088	3221
3089	3222	return 0;
..	..	@@ -3098,15 +3231,14 @@
3098	3231	if (ret)
3099	3232	return ret;
3100	3233
3101		- if (!pm_runtime_status_suspended(dev))
3102		- ret = amba_pclk_enable(pcdev);
3103		-
3104		- pm_runtime_enable(dev);
	3234	+ pm_runtime_force_resume(dev);
3105	3235
3106	3236	return ret;
3107	3237	}
3108	3238
3109		-static SIMPLE_DEV_PM_OPS(pl330_pm, pl330_suspend, pl330_resume);
	3239	+static const struct dev_pm_ops pl330_pm = {
	3240	+ SET_LATE_SYSTEM_SLEEP_PM_OPS(pl330_suspend, pl330_resume)
	3241	+};
3110	3242
3111	3243	static int
3112	3244	pl330_probe(struct amba_device adev, const struct amba_id id)
..	..	@@ -3118,6 +3250,7 @@
3118	3250	struct resource *res;
3119	3251	int i, ret, irq;
3120	3252	int num_chan;
	3253	+ int val;
3121	3254	struct device_node *np = adev->dev.of_node;
3122	3255
3123	3256	ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));
..	..	@@ -3132,7 +3265,12 @@
3132	3265	pd = &pl330->ddma;
3133	3266	pd->dev = &adev->dev;
3134	3267
3135		- pl330->mcbufsz = 0;
	3268	+ if (!device_property_read_u32(&adev->dev, "arm,pl330-mcbufsz-bytes", &val)) {
	3269	+ if ((val > 0) && (val <= PAGE_SIZE))
	3270	+ pl330->mcbufsz = val;
	3271	+
	3272	+ dev_info(&adev->dev, "mcbufsz: %d bytes\n", pl330->mcbufsz);
	3273	+ }
3136	3274
3137	3275	/* get quirk */
3138	3276	for (i = 0; i < ARRAY_SIZE(of_quirks); i++)
..	..	@@ -3145,6 +3283,29 @@
3145	3283	return PTR_ERR(pl330->base);
3146	3284
3147	3285	amba_set_drvdata(adev, pl330);
	3286	+
	3287	+ pl330->rstc = devm_reset_control_get_optional(&adev->dev, "dma");
	3288	+ if (IS_ERR(pl330->rstc)) {
	3289	+ return dev_err_probe(&adev->dev, PTR_ERR(pl330->rstc), "Failed to get reset!\n");
	3290	+ } else {
	3291	+ ret = reset_control_deassert(pl330->rstc);
	3292	+ if (ret) {
	3293	+ dev_err(&adev->dev, "Couldn't deassert the device from reset!\n");
	3294	+ return ret;
	3295	+ }
	3296	+ }
	3297	+
	3298	+ pl330->rstc_ocp = devm_reset_control_get_optional(&adev->dev, "dma-ocp");
	3299	+ if (IS_ERR(pl330->rstc_ocp)) {
	3300	+ return dev_err_probe(&adev->dev, PTR_ERR(pl330->rstc_ocp),
	3301	+ "Failed to get OCP reset!\n");
	3302	+ } else {
	3303	+ ret = reset_control_deassert(pl330->rstc_ocp);
	3304	+ if (ret) {
	3305	+ dev_err(&adev->dev, "Couldn't deassert the device from OCP reset!\n");
	3306	+ return ret;
	3307	+ }
	3308	+ }
3148	3309
3149	3310	for (i = 0; i < AMBA_NR_IRQS; i++) {
3150	3311	irq = adev->irq[i];
..	..	@@ -3209,12 +3370,16 @@
3209	3370	dma_cap_set(DMA_SLAVE, pd->cap_mask);
3210	3371	dma_cap_set(DMA_CYCLIC, pd->cap_mask);
3211	3372	dma_cap_set(DMA_PRIVATE, pd->cap_mask);
	3373	+ dma_cap_set(DMA_INTERLEAVE, pd->cap_mask);
	3374	+ dma_cap_set(DMA_REPEAT, pd->cap_mask);
	3375	+ dma_cap_set(DMA_LOAD_EOT, pd->cap_mask);
3212	3376	}
3213	3377
3214	3378	pd->device_alloc_chan_resources = pl330_alloc_chan_resources;
3215	3379	pd->device_free_chan_resources = pl330_free_chan_resources;
3216	3380	pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy;
3217	3381	pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic;
	3382	+ pd->device_prep_interleaved_dma = pl330_prep_interleaved_dma;
3218	3383	pd->device_tx_status = pl330_tx_status;
3219	3384	pd->device_prep_slave_sg = pl330_prep_slave_sg;
3220	3385	pd->device_config = pl330_config;
..	..	@@ -3242,8 +3407,6 @@
3242	3407	}
3243	3408	}
3244	3409
3245		- adev->dev.dma_parms = &pl330->dma_parms;
3246		-
3247	3410	/*
3248	3411	* This is the limit for transfers with a buswidth of 1, larger
3249	3412	* buswidths will have larger limits.
..	..	@@ -3253,6 +3416,7 @@
3253	3416	dev_err(&adev->dev, "unable to set the seg size\n");
3254	3417
3255	3418
	3419	+ init_pl330_debugfs(pl330);
3256	3420	dev_info(&adev->dev,
3257	3421	"Loaded driver for PL330 DMAC-%x\n", adev->periphid);
3258	3422	dev_info(&adev->dev,
..	..	@@ -3284,10 +3448,15 @@
3284	3448	probe_err2:
3285	3449	pl330_del(pl330);
3286	3450
	3451	+ if (pl330->rstc_ocp)
	3452	+ reset_control_assert(pl330->rstc_ocp);
	3453	+
	3454	+ if (pl330->rstc)
	3455	+ reset_control_assert(pl330->rstc);
3287	3456	return ret;
3288	3457	}
3289	3458
3290		-static int pl330_remove(struct amba_device *adev)
	3459	+static void pl330_remove(struct amba_device *adev)
3291	3460	{
3292	3461	struct pl330_dmac *pl330 = amba_get_drvdata(adev);
3293	3462	struct dma_pl330_chan pch, _p;
..	..	@@ -3322,7 +3491,11 @@
3322	3491
3323	3492	pl330_del(pl330);
3324	3493
3325		- return 0;
	3494	+ if (pl330->rstc_ocp)
	3495	+ reset_control_assert(pl330->rstc_ocp);
	3496	+
	3497	+ if (pl330->rstc)
	3498	+ reset_control_assert(pl330->rstc);
3326	3499	}
3327	3500
3328	3501	static const struct amba_id pl330_ids[] = {