~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,19 +1,8 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* DMA driver for Nvidia's Tegra20 APB DMA controller.
3	4	*
4	5	* Copyright (c) 2012-2013, NVIDIA CORPORATION. All rights reserved.
5		- *
6		- * This program is free software; you can redistribute it and/or modify it
7		- * under the terms and conditions of the GNU General Public License,
8		- * version 2, as published by the Free Software Foundation.
9		- *
10		- * This program is distributed in the hope it will be useful, but WITHOUT
11		- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12		- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13		- * more details.
14		- *
15		- * You should have received a copy of the GNU General Public License
16		- * along with this program. If not, see <http://www.gnu.org/licenses/>.
17	6	*/
18	7
19	8	#include <linux/bitops.h>
..	..	@@ -35,8 +24,12 @@
35	24	#include <linux/pm_runtime.h>
36	25	#include <linux/reset.h>
37	26	#include <linux/slab.h>
	27	+#include <linux/wait.h>
38	28
39	29	#include "dmaengine.h"
	30	+
	31	+#define CREATE_TRACE_POINTS
	32	+#include <trace/events/tegra_apb_dma.h>
40	33
41	34	#define TEGRA_APBDMA_GENERAL 0x0
42	35	#define TEGRA_APBDMA_GENERAL_ENABLE BIT(31)
..	..	@@ -67,7 +60,7 @@
67	60	#define TEGRA_APBDMA_STATUS_COUNT_MASK 0xFFFC
68	61
69	62	#define TEGRA_APBDMA_CHAN_CSRE 0x00C
70		-#define TEGRA_APBDMA_CHAN_CSRE_PAUSE (1 << 31)
	63	+#define TEGRA_APBDMA_CHAN_CSRE_PAUSE BIT(31)
71	64
72	65	/* AHB memory address */
73	66	#define TEGRA_APBDMA_CHAN_AHBPTR 0x010
..	..	@@ -128,25 +121,25 @@
128	121	* @support_separate_wcount_reg: Support separate word count register.
129	122	*/
130	123	struct tegra_dma_chip_data {
131		- int nr_channels;
132		- int channel_reg_size;
133		- int max_dma_count;
	124	+ unsigned int nr_channels;
	125	+ unsigned int channel_reg_size;
	126	+ unsigned int max_dma_count;
134	127	bool support_channel_pause;
135	128	bool support_separate_wcount_reg;
136	129	};
137	130
138	131	/* DMA channel registers */
139	132	struct tegra_dma_channel_regs {
140		- unsigned long csr;
141		- unsigned long ahb_ptr;
142		- unsigned long apb_ptr;
143		- unsigned long ahb_seq;
144		- unsigned long apb_seq;
145		- unsigned long wcount;
	133	+ u32 csr;
	134	+ u32 ahb_ptr;
	135	+ u32 apb_ptr;
	136	+ u32 ahb_seq;
	137	+ u32 apb_seq;
	138	+ u32 wcount;
146	139	};
147	140
148	141	/*
149		- * tegra_dma_sg_req: Dma request details to configure hardware. This
	142	+ * tegra_dma_sg_req: DMA request details to configure hardware. This
150	143	* contains the details for one transfer to configure DMA hw.
151	144	* The client's request for data transfer can be broken into multiple
152	145	* sub-transfer as per requester details and hw support.
..	..	@@ -155,11 +148,12 @@
155	148	*/
156	149	struct tegra_dma_sg_req {
157	150	struct tegra_dma_channel_regs ch_regs;
158		- int req_len;
	151	+ unsigned int req_len;
159	152	bool configured;
160	153	bool last_sg;
161	154	struct list_head node;
162	155	struct tegra_dma_desc *dma_desc;
	156	+ unsigned int words_xferred;
163	157	};
164	158
165	159	/*
..	..	@@ -169,13 +163,13 @@
169	163	*/
170	164	struct tegra_dma_desc {
171	165	struct dma_async_tx_descriptor txd;
172		- int bytes_requested;
173		- int bytes_transferred;
	166	+ unsigned int bytes_requested;
	167	+ unsigned int bytes_transferred;
174	168	enum dma_status dma_status;
175	169	struct list_head node;
176	170	struct list_head tx_list;
177	171	struct list_head cb_node;
178		- int cb_count;
	172	+ unsigned int cb_count;
179	173	};
180	174
181	175	struct tegra_dma_channel;
..	..	@@ -186,10 +180,9 @@
186	180	/* tegra_dma_channel: Channel specific information */
187	181	struct tegra_dma_channel {
188	182	struct dma_chan dma_chan;
189		- char name[30];
	183	+ char name[12];
190	184	bool config_init;
191		- int id;
192		- int irq;
	185	+ unsigned int id;
193	186	void __iomem *chan_addr;
194	187	spinlock_t lock;
195	188	bool busy;
..	..	@@ -209,7 +202,9 @@
209	202	/* Channel-slave specific configuration */
210	203	unsigned int slave_id;
211	204	struct dma_slave_config dma_sconfig;
212		- struct tegra_dma_channel_regs channel_reg;
	205	+ struct tegra_dma_channel_regs channel_reg;
	206	+
	207	+ struct wait_queue_head wq;
213	208	};
214	209
215	210	/* tegra_dma: Tegra DMA specific information */
..	..	@@ -229,11 +224,8 @@
229	224	*/
230	225	u32 global_pause_count;
231	226
232		- /* Some register need to be cache before suspend */
233		- u32 reg_gen;
234		-
235	227	/* Last member of the structure */
236		- struct tegra_dma_channel channels[0];
	228	+ struct tegra_dma_channel channels[];
237	229	};
238	230
239	231	static inline void tdma_write(struct tegra_dma *tdma, u32 reg, u32 val)
..	..	@@ -247,7 +239,7 @@
247	239	}
248	240
249	241	static inline void tdc_write(struct tegra_dma_channel *tdc,
250		- u32 reg, u32 val)
	242	+ u32 reg, u32 val)
251	243	{
252	244	writel(val, tdc->chan_addr + reg);
253	245	}
..	..	@@ -262,8 +254,8 @@
262	254	return container_of(dc, struct tegra_dma_channel, dma_chan);
263	255	}
264	256
265		-static inline struct tegra_dma_desc *txd_to_tegra_dma_desc(
266		- struct dma_async_tx_descriptor *td)
	257	+static inline struct tegra_dma_desc *
	258	+txd_to_tegra_dma_desc(struct dma_async_tx_descriptor *td)
267	259	{
268	260	return container_of(td, struct tegra_dma_desc, txd);
269	261	}
..	..	@@ -274,12 +266,9 @@
274	266	}
275	267
276	268	static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *tx);
277		-static int tegra_dma_runtime_suspend(struct device *dev);
278		-static int tegra_dma_runtime_resume(struct device *dev);
279	269
280	270	/* Get DMA desc from free list, if not there then allocate it. */
281		-static struct tegra_dma_desc *tegra_dma_desc_get(
282		- struct tegra_dma_channel *tdc)
	271	+static struct tegra_dma_desc tegra_dma_desc_get(struct tegra_dma_channel tdc)
283	272	{
284	273	struct tegra_dma_desc *dma_desc;
285	274	unsigned long flags;
..	..	@@ -306,11 +295,12 @@
306	295	dma_async_tx_descriptor_init(&dma_desc->txd, &tdc->dma_chan);
307	296	dma_desc->txd.tx_submit = tegra_dma_tx_submit;
308	297	dma_desc->txd.flags = 0;
	298	+
309	299	return dma_desc;
310	300	}
311	301
312	302	static void tegra_dma_desc_put(struct tegra_dma_channel *tdc,
313		- struct tegra_dma_desc *dma_desc)
	303	+ struct tegra_dma_desc *dma_desc)
314	304	{
315	305	unsigned long flags;
316	306
..	..	@@ -321,29 +311,29 @@
321	311	spin_unlock_irqrestore(&tdc->lock, flags);
322	312	}
323	313
324		-static struct tegra_dma_sg_req *tegra_dma_sg_req_get(
325		- struct tegra_dma_channel *tdc)
	314	+static struct tegra_dma_sg_req *
	315	+tegra_dma_sg_req_get(struct tegra_dma_channel *tdc)
326	316	{
327		- struct tegra_dma_sg_req *sg_req = NULL;
	317	+ struct tegra_dma_sg_req *sg_req;
328	318	unsigned long flags;
329	319
330	320	spin_lock_irqsave(&tdc->lock, flags);
331	321	if (!list_empty(&tdc->free_sg_req)) {
332		- sg_req = list_first_entry(&tdc->free_sg_req,
333		- typeof(*sg_req), node);
	322	+ sg_req = list_first_entry(&tdc->free_sg_req, typeof(*sg_req),
	323	+ node);
334	324	list_del(&sg_req->node);
335	325	spin_unlock_irqrestore(&tdc->lock, flags);
336	326	return sg_req;
337	327	}
338	328	spin_unlock_irqrestore(&tdc->lock, flags);
339	329
340		- sg_req = kzalloc(sizeof(struct tegra_dma_sg_req), GFP_NOWAIT);
	330	+ sg_req = kzalloc(sizeof(*sg_req), GFP_NOWAIT);
341	331
342	332	return sg_req;
343	333	}
344	334
345	335	static int tegra_dma_slave_config(struct dma_chan *dc,
346		- struct dma_slave_config *sconfig)
	336	+ struct dma_slave_config *sconfig)
347	337	{
348	338	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
349	339
..	..	@@ -360,11 +350,12 @@
360	350	tdc->slave_id = sconfig->slave_id;
361	351	}
362	352	tdc->config_init = true;
	353	+
363	354	return 0;
364	355	}
365	356
366	357	static void tegra_dma_global_pause(struct tegra_dma_channel *tdc,
367		- bool wait_for_burst_complete)
	358	+ bool wait_for_burst_complete)
368	359	{
369	360	struct tegra_dma *tdma = tdc->tdma;
370	361
..	..	@@ -399,13 +390,13 @@
399	390	}
400	391
401	392	static void tegra_dma_pause(struct tegra_dma_channel *tdc,
402		- bool wait_for_burst_complete)
	393	+ bool wait_for_burst_complete)
403	394	{
404	395	struct tegra_dma *tdma = tdc->tdma;
405	396
406	397	if (tdma->chip_data->support_channel_pause) {
407	398	tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE,
408		- TEGRA_APBDMA_CHAN_CSRE_PAUSE);
	399	+ TEGRA_APBDMA_CHAN_CSRE_PAUSE);
409	400	if (wait_for_burst_complete)
410	401	udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME);
411	402	} else {
..	..	@@ -417,17 +408,15 @@
417	408	{
418	409	struct tegra_dma *tdma = tdc->tdma;
419	410
420		- if (tdma->chip_data->support_channel_pause) {
	411	+ if (tdma->chip_data->support_channel_pause)
421	412	tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 0);
422		- } else {
	413	+ else
423	414	tegra_dma_global_resume(tdc);
424		- }
425	415	}
426	416
427	417	static void tegra_dma_stop(struct tegra_dma_channel *tdc)
428	418	{
429		- u32 csr;
430		- u32 status;
	419	+ u32 csr, status;
431	420
432	421	/* Disable interrupts */
433	422	csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR);
..	..	@@ -448,7 +437,7 @@
448	437	}
449	438
450	439	static void tegra_dma_start(struct tegra_dma_channel *tdc,
451		- struct tegra_dma_sg_req *sg_req)
	440	+ struct tegra_dma_sg_req *sg_req)
452	441	{
453	442	struct tegra_dma_channel_regs *ch_regs = &sg_req->ch_regs;
454	443
..	..	@@ -462,11 +451,11 @@
462	451
463	452	/* Start DMA */
464	453	tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,
465		- ch_regs->csr \| TEGRA_APBDMA_CSR_ENB);
	454	+ ch_regs->csr \| TEGRA_APBDMA_CSR_ENB);
466	455	}
467	456
468	457	static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc,
469		- struct tegra_dma_sg_req *nsg_req)
	458	+ struct tegra_dma_sg_req *nsg_req)
470	459	{
471	460	unsigned long status;
472	461
..	..	@@ -500,10 +489,11 @@
500	489	tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, nsg_req->ch_regs.ahb_ptr);
501	490	if (tdc->tdma->chip_data->support_separate_wcount_reg)
502	491	tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT,
503		- nsg_req->ch_regs.wcount);
	492	+ nsg_req->ch_regs.wcount);
504	493	tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,
505		- nsg_req->ch_regs.csr \| TEGRA_APBDMA_CSR_ENB);
	494	+ nsg_req->ch_regs.csr \| TEGRA_APBDMA_CSR_ENB);
506	495	nsg_req->configured = true;
	496	+ nsg_req->words_xferred = 0;
507	497
508	498	tegra_dma_resume(tdc);
509	499	}
..	..	@@ -512,46 +502,41 @@
512	502	{
513	503	struct tegra_dma_sg_req *sg_req;
514	504
515		- if (list_empty(&tdc->pending_sg_req))
516		- return;
517		-
518		- sg_req = list_first_entry(&tdc->pending_sg_req,
519		- typeof(*sg_req), node);
	505	+ sg_req = list_first_entry(&tdc->pending_sg_req, typeof(*sg_req), node);
520	506	tegra_dma_start(tdc, sg_req);
521	507	sg_req->configured = true;
	508	+ sg_req->words_xferred = 0;
522	509	tdc->busy = true;
523	510	}
524	511
525	512	static void tdc_configure_next_head_desc(struct tegra_dma_channel *tdc)
526	513	{
527		- struct tegra_dma_sg_req *hsgreq;
528		- struct tegra_dma_sg_req *hnsgreq;
529		-
530		- if (list_empty(&tdc->pending_sg_req))
531		- return;
	514	+ struct tegra_dma_sg_req hsgreq, hnsgreq;
532	515
533	516	hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node);
534	517	if (!list_is_last(&hsgreq->node, &tdc->pending_sg_req)) {
535		- hnsgreq = list_first_entry(&hsgreq->node,
536		- typeof(*hnsgreq), node);
	518	+ hnsgreq = list_first_entry(&hsgreq->node, typeof(*hnsgreq),
	519	+ node);
537	520	tegra_dma_configure_for_next(tdc, hnsgreq);
538	521	}
539	522	}
540	523
541		-static inline int get_current_xferred_count(struct tegra_dma_channel *tdc,
542		- struct tegra_dma_sg_req *sg_req, unsigned long status)
	524	+static inline unsigned int
	525	+get_current_xferred_count(struct tegra_dma_channel *tdc,
	526	+ struct tegra_dma_sg_req *sg_req,
	527	+ unsigned long status)
543	528	{
544	529	return sg_req->req_len - (status & TEGRA_APBDMA_STATUS_COUNT_MASK) - 4;
545	530	}
546	531
547	532	static void tegra_dma_abort_all(struct tegra_dma_channel *tdc)
548	533	{
549		- struct tegra_dma_sg_req *sgreq;
550	534	struct tegra_dma_desc *dma_desc;
	535	+ struct tegra_dma_sg_req *sgreq;
551	536
552	537	while (!list_empty(&tdc->pending_sg_req)) {
553		- sgreq = list_first_entry(&tdc->pending_sg_req,
554		- typeof(*sgreq), node);
	538	+ sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq),
	539	+ node);
555	540	list_move_tail(&sgreq->node, &tdc->free_sg_req);
556	541	if (sgreq->last_sg) {
557	542	dma_desc = sgreq->dma_desc;
..	..	@@ -561,7 +546,7 @@
561	546	/* Add in cb list if it is not there. */
562	547	if (!dma_desc->cb_count)
563	548	list_add_tail(&dma_desc->cb_node,
564		- &tdc->cb_desc);
	549	+ &tdc->cb_desc);
565	550	dma_desc->cb_count++;
566	551	}
567	552	}
..	..	@@ -569,15 +554,9 @@
569	554	}
570	555
571	556	static bool handle_continuous_head_request(struct tegra_dma_channel *tdc,
572		- struct tegra_dma_sg_req *last_sg_req, bool to_terminate)
	557	+ bool to_terminate)
573	558	{
574		- struct tegra_dma_sg_req *hsgreq = NULL;
575		-
576		- if (list_empty(&tdc->pending_sg_req)) {
577		- dev_err(tdc2dev(tdc), "Dma is running without req\n");
578		- tegra_dma_stop(tdc);
579		- return false;
580		- }
	559	+ struct tegra_dma_sg_req *hsgreq;
581	560
582	561	/*
583	562	* Check that head req on list should be in flight.
..	..	@@ -587,7 +566,8 @@
587	566	hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node);
588	567	if (!hsgreq->configured) {
589	568	tegra_dma_stop(tdc);
590		- dev_err(tdc2dev(tdc), "Error in dma transfer, aborting dma\n");
	569	+ pm_runtime_put(tdc->tdma->dev);
	570	+ dev_err(tdc2dev(tdc), "DMA transfer underflow, aborting DMA\n");
591	571	tegra_dma_abort_all(tdc);
592	572	return false;
593	573	}
..	..	@@ -595,14 +575,15 @@
595	575	/* Configure next request */
596	576	if (!to_terminate)
597	577	tdc_configure_next_head_desc(tdc);
	578	+
598	579	return true;
599	580	}
600	581
601	582	static void handle_once_dma_done(struct tegra_dma_channel *tdc,
602		- bool to_terminate)
	583	+ bool to_terminate)
603	584	{
604		- struct tegra_dma_sg_req *sgreq;
605	585	struct tegra_dma_desc *dma_desc;
	586	+ struct tegra_dma_sg_req *sgreq;
606	587
607	588	tdc->busy = false;
608	589	sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node);
..	..	@@ -621,17 +602,22 @@
621	602	list_add_tail(&sgreq->node, &tdc->free_sg_req);
622	603
623	604	/* Do not start DMA if it is going to be terminate */
624		- if (to_terminate \|\| list_empty(&tdc->pending_sg_req))
	605	+ if (to_terminate)
625	606	return;
	607	+
	608	+ if (list_empty(&tdc->pending_sg_req)) {
	609	+ pm_runtime_put(tdc->tdma->dev);
	610	+ return;
	611	+ }
626	612
627	613	tdc_start_head_req(tdc);
628	614	}
629	615
630	616	static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc,
631		- bool to_terminate)
	617	+ bool to_terminate)
632	618	{
633		- struct tegra_dma_sg_req *sgreq;
634	619	struct tegra_dma_desc *dma_desc;
	620	+ struct tegra_dma_sg_req *sgreq;
635	621	bool st;
636	622
637	623	sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node);
..	..	@@ -646,32 +632,36 @@
646	632	list_add_tail(&dma_desc->cb_node, &tdc->cb_desc);
647	633	dma_desc->cb_count++;
648	634
	635	+ sgreq->words_xferred = 0;
	636	+
649	637	/* If not last req then put at end of pending list */
650	638	if (!list_is_last(&sgreq->node, &tdc->pending_sg_req)) {
651	639	list_move_tail(&sgreq->node, &tdc->pending_sg_req);
652	640	sgreq->configured = false;
653		- st = handle_continuous_head_request(tdc, sgreq, to_terminate);
	641	+ st = handle_continuous_head_request(tdc, to_terminate);
654	642	if (!st)
655	643	dma_desc->dma_status = DMA_ERROR;
656	644	}
657	645	}
658	646
659		-static void tegra_dma_tasklet(unsigned long data)
	647	+static void tegra_dma_tasklet(struct tasklet_struct *t)
660	648	{
661		- struct tegra_dma_channel tdc = (struct tegra_dma_channel )data;
	649	+ struct tegra_dma_channel *tdc = from_tasklet(tdc, t, tasklet);
662	650	struct dmaengine_desc_callback cb;
663	651	struct tegra_dma_desc *dma_desc;
	652	+ unsigned int cb_count;
664	653	unsigned long flags;
665		- int cb_count;
666	654
667	655	spin_lock_irqsave(&tdc->lock, flags);
668	656	while (!list_empty(&tdc->cb_desc)) {
669		- dma_desc = list_first_entry(&tdc->cb_desc,
670		- typeof(*dma_desc), cb_node);
	657	+ dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc),
	658	+ cb_node);
671	659	list_del(&dma_desc->cb_node);
672	660	dmaengine_desc_get_callback(&dma_desc->txd, &cb);
673	661	cb_count = dma_desc->cb_count;
674	662	dma_desc->cb_count = 0;
	663	+ trace_tegra_dma_complete_cb(&tdc->dma_chan, cb_count,
	664	+ cb.callback);
675	665	spin_unlock_irqrestore(&tdc->lock, flags);
676	666	while (cb_count--)
677	667	dmaengine_desc_callback_invoke(&cb, NULL);
..	..	@@ -683,23 +673,25 @@
683	673	static irqreturn_t tegra_dma_isr(int irq, void *dev_id)
684	674	{
685	675	struct tegra_dma_channel *tdc = dev_id;
686		- unsigned long status;
687		- unsigned long flags;
	676	+ u32 status;
688	677
689		- spin_lock_irqsave(&tdc->lock, flags);
	678	+ spin_lock(&tdc->lock);
690	679
	680	+ trace_tegra_dma_isr(&tdc->dma_chan, irq);
691	681	status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
692	682	if (status & TEGRA_APBDMA_STATUS_ISE_EOC) {
693	683	tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status);
694	684	tdc->isr_handler(tdc, false);
695	685	tasklet_schedule(&tdc->tasklet);
696		- spin_unlock_irqrestore(&tdc->lock, flags);
	686	+ wake_up_all(&tdc->wq);
	687	+ spin_unlock(&tdc->lock);
697	688	return IRQ_HANDLED;
698	689	}
699	690
700		- spin_unlock_irqrestore(&tdc->lock, flags);
701		- dev_info(tdc2dev(tdc),
702		- "Interrupt already served status 0x%08lx\n", status);
	691	+ spin_unlock(&tdc->lock);
	692	+ dev_info(tdc2dev(tdc), "Interrupt already served status 0x%08x\n",
	693	+ status);
	694	+
703	695	return IRQ_NONE;
704	696	}
705	697
..	..	@@ -715,6 +707,7 @@
715	707	cookie = dma_cookie_assign(&dma_desc->txd);
716	708	list_splice_tail_init(&dma_desc->tx_list, &tdc->pending_sg_req);
717	709	spin_unlock_irqrestore(&tdc->lock, flags);
	710	+
718	711	return cookie;
719	712	}
720	713
..	..	@@ -722,6 +715,7 @@
722	715	{
723	716	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
724	717	unsigned long flags;
	718	+ int err;
725	719
726	720	spin_lock_irqsave(&tdc->lock, flags);
727	721	if (list_empty(&tdc->pending_sg_req)) {
..	..	@@ -729,6 +723,12 @@
729	723	goto end;
730	724	}
731	725	if (!tdc->busy) {
	726	+ err = pm_runtime_resume_and_get(tdc->tdma->dev);
	727	+ if (err < 0) {
	728	+ dev_err(tdc2dev(tdc), "Failed to enable DMA\n");
	729	+ goto end;
	730	+ }
	731	+
732	732	tdc_start_head_req(tdc);
733	733
734	734	/* Continuous single mode: Configure next req */
..	..	@@ -748,11 +748,10 @@
748	748	static int tegra_dma_terminate_all(struct dma_chan *dc)
749	749	{
750	750	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
751		- struct tegra_dma_sg_req *sgreq;
752	751	struct tegra_dma_desc *dma_desc;
	752	+ struct tegra_dma_sg_req *sgreq;
753	753	unsigned long flags;
754		- unsigned long status;
755		- unsigned long wcount;
	754	+ u32 status, wcount;
756	755	bool was_busy;
757	756
758	757	spin_lock_irqsave(&tdc->lock, flags);
..	..	@@ -778,28 +777,127 @@
778	777	tegra_dma_stop(tdc);
779	778
780	779	if (!list_empty(&tdc->pending_sg_req) && was_busy) {
781		- sgreq = list_first_entry(&tdc->pending_sg_req,
782		- typeof(*sgreq), node);
	780	+ sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq),
	781	+ node);
783	782	sgreq->dma_desc->bytes_transferred +=
784	783	get_current_xferred_count(tdc, sgreq, wcount);
785	784	}
786	785	tegra_dma_resume(tdc);
787	786
	787	+ pm_runtime_put(tdc->tdma->dev);
	788	+ wake_up_all(&tdc->wq);
	789	+
788	790	skip_dma_stop:
789	791	tegra_dma_abort_all(tdc);
790	792
791	793	while (!list_empty(&tdc->cb_desc)) {
792		- dma_desc = list_first_entry(&tdc->cb_desc,
793		- typeof(*dma_desc), cb_node);
	794	+ dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc),
	795	+ cb_node);
794	796	list_del(&dma_desc->cb_node);
795	797	dma_desc->cb_count = 0;
796	798	}
797	799	spin_unlock_irqrestore(&tdc->lock, flags);
	800	+
798	801	return 0;
799	802	}
800	803
	804	+static bool tegra_dma_eoc_interrupt_deasserted(struct tegra_dma_channel *tdc)
	805	+{
	806	+ unsigned long flags;
	807	+ u32 status;
	808	+
	809	+ spin_lock_irqsave(&tdc->lock, flags);
	810	+ status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
	811	+ spin_unlock_irqrestore(&tdc->lock, flags);
	812	+
	813	+ return !(status & TEGRA_APBDMA_STATUS_ISE_EOC);
	814	+}
	815	+
	816	+static void tegra_dma_synchronize(struct dma_chan *dc)
	817	+{
	818	+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
	819	+ int err;
	820	+
	821	+ err = pm_runtime_resume_and_get(tdc->tdma->dev);
	822	+ if (err < 0) {
	823	+ dev_err(tdc2dev(tdc), "Failed to synchronize DMA: %d\n", err);
	824	+ return;
	825	+ }
	826	+
	827	+ /*
	828	+ * CPU, which handles interrupt, could be busy in
	829	+ * uninterruptible state, in this case sibling CPU
	830	+ * should wait until interrupt is handled.
	831	+ */
	832	+ wait_event(tdc->wq, tegra_dma_eoc_interrupt_deasserted(tdc));
	833	+
	834	+ tasklet_kill(&tdc->tasklet);
	835	+
	836	+ pm_runtime_put(tdc->tdma->dev);
	837	+}
	838	+
	839	+static unsigned int tegra_dma_sg_bytes_xferred(struct tegra_dma_channel *tdc,
	840	+ struct tegra_dma_sg_req *sg_req)
	841	+{
	842	+ u32 status, wcount = 0;
	843	+
	844	+ if (!list_is_first(&sg_req->node, &tdc->pending_sg_req))
	845	+ return 0;
	846	+
	847	+ if (tdc->tdma->chip_data->support_separate_wcount_reg)
	848	+ wcount = tdc_read(tdc, TEGRA_APBDMA_CHAN_WORD_TRANSFER);
	849	+
	850	+ status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
	851	+
	852	+ if (!tdc->tdma->chip_data->support_separate_wcount_reg)
	853	+ wcount = status;
	854	+
	855	+ if (status & TEGRA_APBDMA_STATUS_ISE_EOC)
	856	+ return sg_req->req_len;
	857	+
	858	+ wcount = get_current_xferred_count(tdc, sg_req, wcount);
	859	+
	860	+ if (!wcount) {
	861	+ /*
	862	+ * If wcount wasn't ever polled for this SG before, then
	863	+ * simply assume that transfer hasn't started yet.
	864	+ *
	865	+ * Otherwise it's the end of the transfer.
	866	+ *
	867	+ * The alternative would be to poll the status register
	868	+ * until EOC bit is set or wcount goes UP. That's so
	869	+ * because EOC bit is getting set only after the last
	870	+ * burst's completion and counter is less than the actual
	871	+ * transfer size by 4 bytes. The counter value wraps around
	872	+ * in a cyclic mode before EOC is set(!), so we can't easily
	873	+ * distinguish start of transfer from its end.
	874	+ */
	875	+ if (sg_req->words_xferred)
	876	+ wcount = sg_req->req_len - 4;
	877	+
	878	+ } else if (wcount < sg_req->words_xferred) {
	879	+ /*
	880	+ * This case will never happen for a non-cyclic transfer.
	881	+ *
	882	+ * For a cyclic transfer, although it is possible for the
	883	+ * next transfer to have already started (resetting the word
	884	+ * count), this case should still not happen because we should
	885	+ * have detected that the EOC bit is set and hence the transfer
	886	+ * was completed.
	887	+ */
	888	+ WARN_ON_ONCE(1);
	889	+
	890	+ wcount = sg_req->req_len - 4;
	891	+ } else {
	892	+ sg_req->words_xferred = wcount;
	893	+ }
	894	+
	895	+ return wcount;
	896	+}
	897	+
801	898	static enum dma_status tegra_dma_tx_status(struct dma_chan *dc,
802		- dma_cookie_t cookie, struct dma_tx_state *txstate)
	899	+ dma_cookie_t cookie,
	900	+ struct dma_tx_state *txstate)
803	901	{
804	902	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
805	903	struct tegra_dma_desc *dma_desc;
..	..	@@ -807,6 +905,7 @@
807	905	enum dma_status ret;
808	906	unsigned long flags;
809	907	unsigned int residual;
	908	+ unsigned int bytes = 0;
810	909
811	910	ret = dma_cookie_status(dc, cookie, txstate);
812	911	if (ret == DMA_COMPLETE)
..	..	@@ -826,6 +925,7 @@
826	925	list_for_each_entry(sg_req, &tdc->pending_sg_req, node) {
827	926	dma_desc = sg_req->dma_desc;
828	927	if (dma_desc->txd.cookie == cookie) {
	928	+ bytes = tegra_dma_sg_bytes_xferred(tdc, sg_req);
829	929	ret = dma_desc->dma_status;
830	930	goto found;
831	931	}
..	..	@@ -837,17 +937,19 @@
837	937	found:
838	938	if (dma_desc && txstate) {
839	939	residual = dma_desc->bytes_requested -
840		- (dma_desc->bytes_transferred %
	940	+ ((dma_desc->bytes_transferred + bytes) %
841	941	dma_desc->bytes_requested);
842	942	dma_set_residue(txstate, residual);
843	943	}
844	944
	945	+ trace_tegra_dma_tx_status(&tdc->dma_chan, cookie, txstate);
845	946	spin_unlock_irqrestore(&tdc->lock, flags);
	947	+
846	948	return ret;
847	949	}
848	950
849		-static inline int get_bus_width(struct tegra_dma_channel *tdc,
850		- enum dma_slave_buswidth slave_bw)
	951	+static inline unsigned int get_bus_width(struct tegra_dma_channel *tdc,
	952	+ enum dma_slave_buswidth slave_bw)
851	953	{
852	954	switch (slave_bw) {
853	955	case DMA_SLAVE_BUSWIDTH_1_BYTE:
..	..	@@ -860,16 +962,17 @@
860	962	return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64;
861	963	default:
862	964	dev_warn(tdc2dev(tdc),
863		- "slave bw is not supported, using 32bits\n");
	965	+ "slave bw is not supported, using 32bits\n");
864	966	return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32;
865	967	}
866	968	}
867	969
868		-static inline int get_burst_size(struct tegra_dma_channel *tdc,
869		- u32 burst_size, enum dma_slave_buswidth slave_bw, int len)
	970	+static inline unsigned int get_burst_size(struct tegra_dma_channel *tdc,
	971	+ u32 burst_size,
	972	+ enum dma_slave_buswidth slave_bw,
	973	+ u32 len)
870	974	{
871		- int burst_byte;
872		- int burst_ahb_width;
	975	+ unsigned int burst_byte, burst_ahb_width;
873	976
874	977	/*
875	978	* burst_size from client is in terms of the bus_width.
..	..	@@ -896,9 +999,12 @@
896	999	}
897	1000
898	1001	static int get_transfer_param(struct tegra_dma_channel *tdc,
899		- enum dma_transfer_direction direction, unsigned long *apb_addr,
900		- unsigned long apb_seq, unsigned long csr, unsigned int *burst_size,
901		- enum dma_slave_buswidth *slave_bw)
	1002	+ enum dma_transfer_direction direction,
	1003	+ u32 *apb_addr,
	1004	+ u32 *apb_seq,
	1005	+ u32 *csr,
	1006	+ unsigned int *burst_size,
	1007	+ enum dma_slave_buswidth *slave_bw)
902	1008	{
903	1009	switch (direction) {
904	1010	case DMA_MEM_TO_DEV:
..	..	@@ -918,14 +1024,16 @@
918	1024	return 0;
919	1025
920	1026	default:
921		- dev_err(tdc2dev(tdc), "Dma direction is not supported\n");
922		- return -EINVAL;
	1027	+ dev_err(tdc2dev(tdc), "DMA direction is not supported\n");
	1028	+ break;
923	1029	}
	1030	+
924	1031	return -EINVAL;
925	1032	}
926	1033
927	1034	static void tegra_dma_prep_wcount(struct tegra_dma_channel *tdc,
928		- struct tegra_dma_channel_regs *ch_regs, u32 len)
	1035	+ struct tegra_dma_channel_regs *ch_regs,
	1036	+ u32 len)
929	1037	{
930	1038	u32 len_field = (len - 4) & 0xFFFC;
931	1039
..	..	@@ -935,23 +1043,26 @@
935	1043	ch_regs->csr \|= len_field;
936	1044	}
937	1045
938		-static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg(
939		- struct dma_chan dc, struct scatterlist sgl, unsigned int sg_len,
940		- enum dma_transfer_direction direction, unsigned long flags,
941		- void *context)
	1046	+static struct dma_async_tx_descriptor *
	1047	+tegra_dma_prep_slave_sg(struct dma_chan *dc,
	1048	+ struct scatterlist *sgl,
	1049	+ unsigned int sg_len,
	1050	+ enum dma_transfer_direction direction,
	1051	+ unsigned long flags,
	1052	+ void *context)
942	1053	{
943	1054	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
944		- struct tegra_dma_desc *dma_desc;
945		- unsigned int i;
946		- struct scatterlist *sg;
947		- unsigned long csr, ahb_seq, apb_ptr, apb_seq;
948		- struct list_head req_list;
949		- struct tegra_dma_sg_req *sg_req = NULL;
950		- u32 burst_size;
	1055	+ struct tegra_dma_sg_req *sg_req = NULL;
	1056	+ u32 csr, ahb_seq, apb_ptr, apb_seq;
951	1057	enum dma_slave_buswidth slave_bw;
	1058	+ struct tegra_dma_desc *dma_desc;
	1059	+ struct list_head req_list;
	1060	+ struct scatterlist *sg;
	1061	+ unsigned int burst_size;
	1062	+ unsigned int i;
952	1063
953	1064	if (!tdc->config_init) {
954		- dev_err(tdc2dev(tdc), "dma channel is not configured\n");
	1065	+ dev_err(tdc2dev(tdc), "DMA channel is not configured\n");
955	1066	return NULL;
956	1067	}
957	1068	if (sg_len < 1) {
..	..	@@ -960,7 +1071,7 @@
960	1071	}
961	1072
962	1073	if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr,
963		- &burst_size, &slave_bw) < 0)
	1074	+ &burst_size, &slave_bw) < 0)
964	1075	return NULL;
965	1076
966	1077	INIT_LIST_HEAD(&req_list);
..	..	@@ -988,7 +1099,7 @@
988	1099
989	1100	dma_desc = tegra_dma_desc_get(tdc);
990	1101	if (!dma_desc) {
991		- dev_err(tdc2dev(tdc), "Dma descriptors not available\n");
	1102	+ dev_err(tdc2dev(tdc), "DMA descriptors not available\n");
992	1103	return NULL;
993	1104	}
994	1105	INIT_LIST_HEAD(&dma_desc->tx_list);
..	..	@@ -1006,16 +1117,16 @@
1006	1117	len = sg_dma_len(sg);
1007	1118
1008	1119	if ((len & 3) \|\| (mem & 3) \|\|
1009		- (len > tdc->tdma->chip_data->max_dma_count)) {
	1120	+ len > tdc->tdma->chip_data->max_dma_count) {
1010	1121	dev_err(tdc2dev(tdc),
1011		- "Dma length/memory address is not supported\n");
	1122	+ "DMA length/memory address is not supported\n");
1012	1123	tegra_dma_desc_put(tdc, dma_desc);
1013	1124	return NULL;
1014	1125	}
1015	1126
1016	1127	sg_req = tegra_dma_sg_req_get(tdc);
1017	1128	if (!sg_req) {
1018		- dev_err(tdc2dev(tdc), "Dma sg-req not available\n");
	1129	+ dev_err(tdc2dev(tdc), "DMA sg-req not available\n");
1019	1130	tegra_dma_desc_put(tdc, dma_desc);
1020	1131	return NULL;
1021	1132	}
..	..	@@ -1058,20 +1169,21 @@
1058	1169	return &dma_desc->txd;
1059	1170	}
1060	1171
1061		-static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic(
1062		- struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len,
1063		- size_t period_len, enum dma_transfer_direction direction,
1064		- unsigned long flags)
	1172	+static struct dma_async_tx_descriptor *
	1173	+tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr,
	1174	+ size_t buf_len,
	1175	+ size_t period_len,
	1176	+ enum dma_transfer_direction direction,
	1177	+ unsigned long flags)
1065	1178	{
1066	1179	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1067		- struct tegra_dma_desc *dma_desc = NULL;
1068	1180	struct tegra_dma_sg_req *sg_req = NULL;
1069		- unsigned long csr, ahb_seq, apb_ptr, apb_seq;
1070		- int len;
1071		- size_t remain_len;
1072		- dma_addr_t mem = buf_addr;
1073		- u32 burst_size;
	1181	+ u32 csr, ahb_seq, apb_ptr, apb_seq;
1074	1182	enum dma_slave_buswidth slave_bw;
	1183	+ struct tegra_dma_desc *dma_desc;
	1184	+ dma_addr_t mem = buf_addr;
	1185	+ unsigned int burst_size;
	1186	+ size_t len, remain_len;
1075	1187
1076	1188	if (!buf_len \|\| !period_len) {
1077	1189	dev_err(tdc2dev(tdc), "Invalid buffer/period len\n");
..	..	@@ -1090,7 +1202,7 @@
1090	1202	* terminating the DMA.
1091	1203	*/
1092	1204	if (tdc->busy) {
1093		- dev_err(tdc2dev(tdc), "Request not allowed when dma running\n");
	1205	+ dev_err(tdc2dev(tdc), "Request not allowed when DMA running\n");
1094	1206	return NULL;
1095	1207	}
1096	1208
..	..	@@ -1105,13 +1217,13 @@
1105	1217
1106	1218	len = period_len;
1107	1219	if ((len & 3) \|\| (buf_addr & 3) \|\|
1108		- (len > tdc->tdma->chip_data->max_dma_count)) {
	1220	+ len > tdc->tdma->chip_data->max_dma_count) {
1109	1221	dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n");
1110	1222	return NULL;
1111	1223	}
1112	1224
1113	1225	if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr,
1114		- &burst_size, &slave_bw) < 0)
	1226	+ &burst_size, &slave_bw) < 0)
1115	1227	return NULL;
1116	1228
1117	1229	ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB;
..	..	@@ -1151,7 +1263,7 @@
1151	1263	while (remain_len) {
1152	1264	sg_req = tegra_dma_sg_req_get(tdc);
1153	1265	if (!sg_req) {
1154		- dev_err(tdc2dev(tdc), "Dma sg-req not available\n");
	1266	+ dev_err(tdc2dev(tdc), "DMA sg-req not available\n");
1155	1267	tegra_dma_desc_put(tdc, dma_desc);
1156	1268	return NULL;
1157	1269	}
..	..	@@ -1197,15 +1309,8 @@
1197	1309	static int tegra_dma_alloc_chan_resources(struct dma_chan *dc)
1198	1310	{
1199	1311	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1200		- struct tegra_dma *tdma = tdc->tdma;
1201		- int ret;
1202	1312
1203	1313	dma_cookie_init(&tdc->dma_chan);
1204		- tdc->config_init = false;
1205		-
1206		- ret = pm_runtime_get_sync(tdma->dev);
1207		- if (ret < 0)
1208		- return ret;
1209	1314
1210	1315	return 0;
1211	1316	}
..	..	@@ -1213,12 +1318,10 @@
1213	1318	static void tegra_dma_free_chan_resources(struct dma_chan *dc)
1214	1319	{
1215	1320	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1216		- struct tegra_dma *tdma = tdc->tdma;
1217	1321	struct tegra_dma_desc *dma_desc;
1218	1322	struct tegra_dma_sg_req *sg_req;
1219	1323	struct list_head dma_desc_list;
1220	1324	struct list_head sg_req_list;
1221		- unsigned long flags;
1222	1325
1223	1326	INIT_LIST_HEAD(&dma_desc_list);
1224	1327	INIT_LIST_HEAD(&sg_req_list);
..	..	@@ -1226,19 +1329,18 @@
1226	1329	dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id);
1227	1330
1228	1331	tegra_dma_terminate_all(dc);
	1332	+ tasklet_kill(&tdc->tasklet);
1229	1333
1230		- spin_lock_irqsave(&tdc->lock, flags);
1231	1334	list_splice_init(&tdc->pending_sg_req, &sg_req_list);
1232	1335	list_splice_init(&tdc->free_sg_req, &sg_req_list);
1233	1336	list_splice_init(&tdc->free_dma_desc, &dma_desc_list);
1234	1337	INIT_LIST_HEAD(&tdc->cb_desc);
1235	1338	tdc->config_init = false;
1236	1339	tdc->isr_handler = NULL;
1237		- spin_unlock_irqrestore(&tdc->lock, flags);
1238	1340
1239	1341	while (!list_empty(&dma_desc_list)) {
1240		- dma_desc = list_first_entry(&dma_desc_list,
1241		- typeof(*dma_desc), node);
	1342	+ dma_desc = list_first_entry(&dma_desc_list, typeof(*dma_desc),
	1343	+ node);
1242	1344	list_del(&dma_desc->node);
1243	1345	kfree(dma_desc);
1244	1346	}
..	..	@@ -1248,7 +1350,6 @@
1248	1350	list_del(&sg_req->node);
1249	1351	kfree(sg_req);
1250	1352	}
1251		- pm_runtime_put(tdma->dev);
1252	1353
1253	1354	tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID;
1254	1355	}
..	..	@@ -1257,8 +1358,8 @@
1257	1358	struct of_dma *ofdma)
1258	1359	{
1259	1360	struct tegra_dma *tdma = ofdma->of_dma_data;
1260		- struct dma_chan *chan;
1261	1361	struct tegra_dma_channel *tdc;
	1362	+ struct dma_chan *chan;
1262	1363
1263	1364	if (dma_spec->args[0] > TEGRA_APBDMA_CSR_REQ_SEL_MASK) {
1264	1365	dev_err(tdma->dev, "Invalid slave id: %d\n", dma_spec->args[0]);
..	..	@@ -1311,22 +1412,48 @@
1311	1412	.support_separate_wcount_reg = true,
1312	1413	};
1313	1414
1314		-static int tegra_dma_probe(struct platform_device *pdev)
	1415	+static int tegra_dma_init_hw(struct tegra_dma *tdma)
1315	1416	{
1316		- struct resource *res;
1317		- struct tegra_dma *tdma;
1318		- int ret;
1319		- int i;
1320		- const struct tegra_dma_chip_data *cdata;
	1417	+ int err;
1321	1418
1322		- cdata = of_device_get_match_data(&pdev->dev);
1323		- if (!cdata) {
1324		- dev_err(&pdev->dev, "Error: No device match data found\n");
1325		- return -ENODEV;
	1419	+ err = reset_control_assert(tdma->rst);
	1420	+ if (err) {
	1421	+ dev_err(tdma->dev, "failed to assert reset: %d\n", err);
	1422	+ return err;
1326	1423	}
1327	1424
1328		- tdma = devm_kzalloc(&pdev->dev, sizeof(tdma) + cdata->nr_channels
1329		- sizeof(struct tegra_dma_channel), GFP_KERNEL);
	1425	+ err = clk_enable(tdma->dma_clk);
	1426	+ if (err) {
	1427	+ dev_err(tdma->dev, "failed to enable clk: %d\n", err);
	1428	+ return err;
	1429	+ }
	1430	+
	1431	+ /* reset DMA controller */
	1432	+ udelay(2);
	1433	+ reset_control_deassert(tdma->rst);
	1434	+
	1435	+ /* enable global DMA registers */
	1436	+ tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE);
	1437	+ tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0);
	1438	+ tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFF);
	1439	+
	1440	+ clk_disable(tdma->dma_clk);
	1441	+
	1442	+ return 0;
	1443	+}
	1444	+
	1445	+static int tegra_dma_probe(struct platform_device *pdev)
	1446	+{
	1447	+ const struct tegra_dma_chip_data *cdata;
	1448	+ struct tegra_dma *tdma;
	1449	+ unsigned int i;
	1450	+ size_t size;
	1451	+ int ret;
	1452	+
	1453	+ cdata = of_device_get_match_data(&pdev->dev);
	1454	+ size = struct_size(tdma, channels, cdata->nr_channels);
	1455	+
	1456	+ tdma = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
1330	1457	if (!tdma)
1331	1458	return -ENOMEM;
1332	1459
..	..	@@ -1334,8 +1461,7 @@
1334	1461	tdma->chip_data = cdata;
1335	1462	platform_set_drvdata(pdev, tdma);
1336	1463
1337		- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1338		- tdma->base_addr = devm_ioremap_resource(&pdev->dev, res);
	1464	+ tdma->base_addr = devm_platform_ioremap_resource(pdev, 0);
1339	1465	if (IS_ERR(tdma->base_addr))
1340	1466	return PTR_ERR(tdma->base_addr);
1341	1467
..	..	@@ -1353,64 +1479,53 @@
1353	1479
1354	1480	spin_lock_init(&tdma->global_lock);
1355	1481
1356		- pm_runtime_enable(&pdev->dev);
1357		- if (!pm_runtime_enabled(&pdev->dev))
1358		- ret = tegra_dma_runtime_resume(&pdev->dev);
1359		- else
1360		- ret = pm_runtime_get_sync(&pdev->dev);
1361		-
1362		- if (ret < 0) {
1363		- pm_runtime_disable(&pdev->dev);
	1482	+ ret = clk_prepare(tdma->dma_clk);
	1483	+ if (ret)
1364	1484	return ret;
1365		- }
1366	1485
1367		- /* Reset DMA controller */
1368		- reset_control_assert(tdma->rst);
1369		- udelay(2);
1370		- reset_control_deassert(tdma->rst);
	1486	+ ret = tegra_dma_init_hw(tdma);
	1487	+ if (ret)
	1488	+ goto err_clk_unprepare;
1371	1489
1372		- /* Enable global DMA registers */
1373		- tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE);
1374		- tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0);
1375		- tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul);
1376		-
1377		- pm_runtime_put(&pdev->dev);
	1490	+ pm_runtime_irq_safe(&pdev->dev);
	1491	+ pm_runtime_enable(&pdev->dev);
1378	1492
1379	1493	INIT_LIST_HEAD(&tdma->dma_dev.channels);
1380	1494	for (i = 0; i < cdata->nr_channels; i++) {
1381	1495	struct tegra_dma_channel *tdc = &tdma->channels[i];
	1496	+ int irq;
1382	1497
1383	1498	tdc->chan_addr = tdma->base_addr +
1384	1499	TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET +
1385	1500	(i * cdata->channel_reg_size);
1386	1501
1387		- res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
1388		- if (!res) {
1389		- ret = -EINVAL;
1390		- dev_err(&pdev->dev, "No irq resource for chan %d\n", i);
1391		- goto err_irq;
	1502	+ irq = platform_get_irq(pdev, i);
	1503	+ if (irq < 0) {
	1504	+ ret = irq;
	1505	+ goto err_pm_disable;
1392	1506	}
1393		- tdc->irq = res->start;
	1507	+
1394	1508	snprintf(tdc->name, sizeof(tdc->name), "apbdma.%d", i);
1395		- ret = request_irq(tdc->irq, tegra_dma_isr, 0, tdc->name, tdc);
	1509	+ ret = devm_request_irq(&pdev->dev, irq, tegra_dma_isr, 0,
	1510	+ tdc->name, tdc);
1396	1511	if (ret) {
1397	1512	dev_err(&pdev->dev,
1398	1513	"request_irq failed with err %d channel %d\n",
1399	1514	ret, i);
1400		- goto err_irq;
	1515	+ goto err_pm_disable;
1401	1516	}
1402	1517
1403	1518	tdc->dma_chan.device = &tdma->dma_dev;
1404	1519	dma_cookie_init(&tdc->dma_chan);
1405	1520	list_add_tail(&tdc->dma_chan.device_node,
1406		- &tdma->dma_dev.channels);
	1521	+ &tdma->dma_dev.channels);
1407	1522	tdc->tdma = tdma;
1408	1523	tdc->id = i;
1409	1524	tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID;
1410	1525
1411		- tasklet_init(&tdc->tasklet, tegra_dma_tasklet,
1412		- (unsigned long)tdc);
	1526	+ tasklet_setup(&tdc->tasklet, tegra_dma_tasklet);
1413	1527	spin_lock_init(&tdc->lock);
	1528	+ init_waitqueue_head(&tdc->wq);
1414	1529
1415	1530	INIT_LIST_HEAD(&tdc->pending_sg_req);
1416	1531	INIT_LIST_HEAD(&tdc->free_sg_req);
..	..	@@ -1439,14 +1554,10 @@
1439	1554	BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) \|
1440	1555	BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1441	1556	tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) \| BIT(DMA_MEM_TO_DEV);
1442		- /*
1443		- * XXX The hardware appears to support
1444		- * DMA_RESIDUE_GRANULARITY_BURST-level reporting, but it's
1445		- * only used by this driver during tegra_dma_terminate_all()
1446		- */
1447		- tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
	1557	+ tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1448	1558	tdma->dma_dev.device_config = tegra_dma_slave_config;
1449	1559	tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all;
	1560	+ tdma->dma_dev.device_synchronize = tegra_dma_synchronize;
1450	1561	tdma->dma_dev.device_tx_status = tegra_dma_tx_status;
1451	1562	tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending;
1452	1563
..	..	@@ -1454,7 +1565,7 @@
1454	1565	if (ret < 0) {
1455	1566	dev_err(&pdev->dev,
1456	1567	"Tegra20 APB DMA driver registration failed %d\n", ret);
1457		- goto err_irq;
	1568	+ goto err_pm_disable;
1458	1569	}
1459	1570
1460	1571	ret = of_dma_controller_register(pdev->dev.of_node,
..	..	@@ -1465,118 +1576,92 @@
1465	1576	goto err_unregister_dma_dev;
1466	1577	}
1467	1578
1468		- dev_info(&pdev->dev, "Tegra20 APB DMA driver register %d channels\n",
1469		- cdata->nr_channels);
	1579	+ dev_info(&pdev->dev, "Tegra20 APB DMA driver registered %u channels\n",
	1580	+ cdata->nr_channels);
	1581	+
1470	1582	return 0;
1471	1583
1472	1584	err_unregister_dma_dev:
1473	1585	dma_async_device_unregister(&tdma->dma_dev);
1474		-err_irq:
1475		- while (--i >= 0) {
1476		- struct tegra_dma_channel *tdc = &tdma->channels[i];
1477	1586
1478		- free_irq(tdc->irq, tdc);
1479		- tasklet_kill(&tdc->tasklet);
1480		- }
1481		-
	1587	+err_pm_disable:
1482	1588	pm_runtime_disable(&pdev->dev);
1483		- if (!pm_runtime_status_suspended(&pdev->dev))
1484		- tegra_dma_runtime_suspend(&pdev->dev);
	1589	+
	1590	+err_clk_unprepare:
	1591	+ clk_unprepare(tdma->dma_clk);
	1592	+
1485	1593	return ret;
1486	1594	}
1487	1595
1488	1596	static int tegra_dma_remove(struct platform_device *pdev)
1489	1597	{
1490	1598	struct tegra_dma *tdma = platform_get_drvdata(pdev);
1491		- int i;
1492		- struct tegra_dma_channel *tdc;
1493	1599
	1600	+ of_dma_controller_free(pdev->dev.of_node);
1494	1601	dma_async_device_unregister(&tdma->dma_dev);
1495		-
1496		- for (i = 0; i < tdma->chip_data->nr_channels; ++i) {
1497		- tdc = &tdma->channels[i];
1498		- free_irq(tdc->irq, tdc);
1499		- tasklet_kill(&tdc->tasklet);
1500		- }
1501		-
1502	1602	pm_runtime_disable(&pdev->dev);
1503		- if (!pm_runtime_status_suspended(&pdev->dev))
1504		- tegra_dma_runtime_suspend(&pdev->dev);
	1603	+ clk_unprepare(tdma->dma_clk);
1505	1604
1506	1605	return 0;
1507	1606	}
1508	1607
1509		-static int tegra_dma_runtime_suspend(struct device *dev)
	1608	+static int __maybe_unused tegra_dma_runtime_suspend(struct device *dev)
1510	1609	{
1511	1610	struct tegra_dma *tdma = dev_get_drvdata(dev);
1512		- int i;
1513	1611
1514		- tdma->reg_gen = tdma_read(tdma, TEGRA_APBDMA_GENERAL);
1515		- for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1516		- struct tegra_dma_channel *tdc = &tdma->channels[i];
1517		- struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg;
1518		-
1519		- /* Only save the state of DMA channels that are in use */
1520		- if (!tdc->config_init)
1521		- continue;
1522		-
1523		- ch_reg->csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR);
1524		- ch_reg->ahb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBPTR);
1525		- ch_reg->apb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBPTR);
1526		- ch_reg->ahb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBSEQ);
1527		- ch_reg->apb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBSEQ);
1528		- if (tdma->chip_data->support_separate_wcount_reg)
1529		- ch_reg->wcount = tdc_read(tdc,
1530		- TEGRA_APBDMA_CHAN_WCOUNT);
1531		- }
1532		-
1533		- clk_disable_unprepare(tdma->dma_clk);
	1612	+ clk_disable(tdma->dma_clk);
1534	1613
1535	1614	return 0;
1536	1615	}
1537	1616
1538		-static int tegra_dma_runtime_resume(struct device *dev)
	1617	+static int __maybe_unused tegra_dma_runtime_resume(struct device *dev)
1539	1618	{
1540	1619	struct tegra_dma *tdma = dev_get_drvdata(dev);
1541		- int i, ret;
1542	1620
1543		- ret = clk_prepare_enable(tdma->dma_clk);
1544		- if (ret < 0) {
1545		- dev_err(dev, "clk_enable failed: %d\n", ret);
1546		- return ret;
1547		- }
	1621	+ return clk_enable(tdma->dma_clk);
	1622	+}
1548	1623
1549		- tdma_write(tdma, TEGRA_APBDMA_GENERAL, tdma->reg_gen);
1550		- tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0);
1551		- tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul);
	1624	+static int __maybe_unused tegra_dma_dev_suspend(struct device *dev)
	1625	+{
	1626	+ struct tegra_dma *tdma = dev_get_drvdata(dev);
	1627	+ unsigned long flags;
	1628	+ unsigned int i;
	1629	+ bool busy;
1552	1630
1553	1631	for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1554	1632	struct tegra_dma_channel *tdc = &tdma->channels[i];
1555		- struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg;
1556	1633
1557		- /* Only restore the state of DMA channels that are in use */
1558		- if (!tdc->config_init)
1559		- continue;
	1634	+ tasklet_kill(&tdc->tasklet);
1560	1635
1561		- if (tdma->chip_data->support_separate_wcount_reg)
1562		- tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT,
1563		- ch_reg->wcount);
1564		- tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_reg->apb_seq);
1565		- tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_reg->apb_ptr);
1566		- tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_reg->ahb_seq);
1567		- tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_reg->ahb_ptr);
1568		- tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,
1569		- (ch_reg->csr & ~TEGRA_APBDMA_CSR_ENB));
	1636	+ spin_lock_irqsave(&tdc->lock, flags);
	1637	+ busy = tdc->busy;
	1638	+ spin_unlock_irqrestore(&tdc->lock, flags);
	1639	+
	1640	+ if (busy) {
	1641	+ dev_err(tdma->dev, "channel %u busy\n", i);
	1642	+ return -EBUSY;
	1643	+ }
1570	1644	}
1571	1645
1572		- return 0;
	1646	+ return pm_runtime_force_suspend(dev);
	1647	+}
	1648	+
	1649	+static int __maybe_unused tegra_dma_dev_resume(struct device *dev)
	1650	+{
	1651	+ struct tegra_dma *tdma = dev_get_drvdata(dev);
	1652	+ int err;
	1653	+
	1654	+ err = tegra_dma_init_hw(tdma);
	1655	+ if (err)
	1656	+ return err;
	1657	+
	1658	+ return pm_runtime_force_resume(dev);
1573	1659	}
1574	1660
1575	1661	static const struct dev_pm_ops tegra_dma_dev_pm_ops = {
1576	1662	SET_RUNTIME_PM_OPS(tegra_dma_runtime_suspend, tegra_dma_runtime_resume,
1577	1663	NULL)
1578		- SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
1579		- pm_runtime_force_resume)
	1664	+ SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_dev_suspend, tegra_dma_dev_resume)
1580	1665	};
1581	1666
1582	1667	static const struct of_device_id tegra_dma_of_match[] = {
..	..	@@ -1609,7 +1694,6 @@
1609	1694
1610	1695	module_platform_driver(tegra_dmac_driver);
1611	1696
1612		-MODULE_ALIAS("platform:tegra20-apbdma");
1613	1697	MODULE_DESCRIPTION("NVIDIA Tegra APB DMA Controller driver");
1614	1698	MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>");
1615	1699	MODULE_LICENSE("GPL v2");