forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/drivers/dma/xilinx/xilinx_dma.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-or-later
12 /*
23  * DMA driver for Xilinx Video DMA Engine
34  *
....@@ -25,10 +26,11 @@
2526  * Access (DMA) between a memory-mapped source address and a memory-mapped
2627  * destination address.
2728  *
28
- * This program is free software: you can redistribute it and/or modify
29
- * it under the terms of the GNU General Public License as published by
30
- * the Free Software Foundation, either version 2 of the License, or
31
- * (at your option) any later version.
29
+ * The AXI Multichannel Direct Memory Access (AXI MCDMA) core is a soft
30
+ * Xilinx IP that provides high-bandwidth direct memory access between
31
+ * memory and AXI4-Stream target peripherals. It provides scatter gather
32
+ * (SG) interface with multiple channels independent configuration support.
33
+ *
3234  */
3335 
3436 #include <linux/bitops.h>
....@@ -89,6 +91,7 @@
8991 #define XILINX_DMA_DMASR_DMA_DEC_ERR		BIT(6)
9092 #define XILINX_DMA_DMASR_DMA_SLAVE_ERR		BIT(5)
9193 #define XILINX_DMA_DMASR_DMA_INT_ERR		BIT(4)
94
+#define XILINX_DMA_DMASR_SG_MASK		BIT(3)
9295 #define XILINX_DMA_DMASR_IDLE			BIT(1)
9396 #define XILINX_DMA_DMASR_HALTED		BIT(0)
9497 #define XILINX_DMA_DMASR_DELAY_MASK		GENMASK(31, 24)
....@@ -122,7 +125,9 @@
122125 #define XILINX_VDMA_ENABLE_VERTICAL_FLIP	BIT(0)
123126 
124127 /* HW specific definitions */
125
-#define XILINX_DMA_MAX_CHANS_PER_DEVICE	0x20
128
+#define XILINX_MCDMA_MAX_CHANS_PER_DEVICE	0x20
129
+#define XILINX_DMA_MAX_CHANS_PER_DEVICE		0x2
130
+#define XILINX_CDMA_MAX_CHANS_PER_DEVICE	0x1
126131 
127132 #define XILINX_DMA_DMAXR_ALL_IRQ_MASK	\
128133 		(XILINX_DMA_DMASR_FRM_CNT_IRQ | \
....@@ -164,7 +169,9 @@
164169 #define XILINX_DMA_REG_BTT		0x28
165170 
166171 /* AXI DMA Specific Masks/Bit fields */
167
-#define XILINX_DMA_MAX_TRANS_LEN	GENMASK(22, 0)
172
+#define XILINX_DMA_MAX_TRANS_LEN_MIN	8
173
+#define XILINX_DMA_MAX_TRANS_LEN_MAX	23
174
+#define XILINX_DMA_V2_MAX_TRANS_LEN_MAX	26
168175 #define XILINX_DMA_CR_COALESCE_MAX	GENMASK(23, 16)
169176 #define XILINX_DMA_CR_CYCLIC_BD_EN_MASK	BIT(4)
170177 #define XILINX_DMA_CR_COALESCE_SHIFT	16
....@@ -174,24 +181,39 @@
174181 #define XILINX_DMA_NUM_DESCS		255
175182 #define XILINX_DMA_NUM_APP_WORDS	5
176183 
177
-/* Multi-Channel DMA Descriptor offsets*/
178
-#define XILINX_DMA_MCRX_CDESC(x)	(0x40 + (x-1) * 0x20)
179
-#define XILINX_DMA_MCRX_TDESC(x)	(0x48 + (x-1) * 0x20)
180
-
181
-/* Multi-Channel DMA Masks/Shifts */
182
-#define XILINX_DMA_BD_HSIZE_MASK	GENMASK(15, 0)
183
-#define XILINX_DMA_BD_STRIDE_MASK	GENMASK(15, 0)
184
-#define XILINX_DMA_BD_VSIZE_MASK	GENMASK(31, 19)
185
-#define XILINX_DMA_BD_TDEST_MASK	GENMASK(4, 0)
186
-#define XILINX_DMA_BD_STRIDE_SHIFT	0
187
-#define XILINX_DMA_BD_VSIZE_SHIFT	19
188
-
189184 /* AXI CDMA Specific Registers/Offsets */
190185 #define XILINX_CDMA_REG_SRCADDR		0x18
191186 #define XILINX_CDMA_REG_DSTADDR		0x20
192187 
193188 /* AXI CDMA Specific Masks */
194189 #define XILINX_CDMA_CR_SGMODE          BIT(3)
190
+
191
+#define xilinx_prep_dma_addr_t(addr)	\
192
+	((dma_addr_t)((u64)addr##_##msb << 32 | (addr)))
193
+
194
+/* AXI MCDMA Specific Registers/Offsets */
195
+#define XILINX_MCDMA_MM2S_CTRL_OFFSET		0x0000
196
+#define XILINX_MCDMA_S2MM_CTRL_OFFSET		0x0500
197
+#define XILINX_MCDMA_CHEN_OFFSET		0x0008
198
+#define XILINX_MCDMA_CH_ERR_OFFSET		0x0010
199
+#define XILINX_MCDMA_RXINT_SER_OFFSET		0x0020
200
+#define XILINX_MCDMA_TXINT_SER_OFFSET		0x0028
201
+#define XILINX_MCDMA_CHAN_CR_OFFSET(x)		(0x40 + (x) * 0x40)
202
+#define XILINX_MCDMA_CHAN_SR_OFFSET(x)		(0x44 + (x) * 0x40)
203
+#define XILINX_MCDMA_CHAN_CDESC_OFFSET(x)	(0x48 + (x) * 0x40)
204
+#define XILINX_MCDMA_CHAN_TDESC_OFFSET(x)	(0x50 + (x) * 0x40)
205
+
206
+/* AXI MCDMA Specific Masks/Shifts */
207
+#define XILINX_MCDMA_COALESCE_SHIFT		16
208
+#define XILINX_MCDMA_COALESCE_MAX		24
209
+#define XILINX_MCDMA_IRQ_ALL_MASK		GENMASK(7, 5)
210
+#define XILINX_MCDMA_COALESCE_MASK		GENMASK(23, 16)
211
+#define XILINX_MCDMA_CR_RUNSTOP_MASK		BIT(0)
212
+#define XILINX_MCDMA_IRQ_IOC_MASK		BIT(5)
213
+#define XILINX_MCDMA_IRQ_DELAY_MASK		BIT(6)
214
+#define XILINX_MCDMA_IRQ_ERR_MASK		BIT(7)
215
+#define XILINX_MCDMA_BD_EOP			BIT(30)
216
+#define XILINX_MCDMA_BD_SOP			BIT(31)
195217 
196218 /**
197219  * struct xilinx_vdma_desc_hw - Hardware Descriptor
....@@ -220,8 +242,8 @@
220242  * @next_desc_msb: MSB of Next Descriptor Pointer @0x04
221243  * @buf_addr: Buffer address @0x08
222244  * @buf_addr_msb: MSB of Buffer address @0x0C
223
- * @mcdma_control: Control field for mcdma @0x10
224
- * @vsize_stride: Vsize and Stride field for mcdma @0x14
245
+ * @reserved1: Reserved @0x10
246
+ * @reserved2: Reserved @0x14
225247  * @control: Control field @0x18
226248  * @status: Status field @0x1C
227249  * @app: APP Fields @0x20 - 0x30
....@@ -231,10 +253,34 @@
231253 	u32 next_desc_msb;
232254 	u32 buf_addr;
233255 	u32 buf_addr_msb;
234
-	u32 mcdma_control;
235
-	u32 vsize_stride;
256
+	u32 reserved1;
257
+	u32 reserved2;
236258 	u32 control;
237259 	u32 status;
260
+	u32 app[XILINX_DMA_NUM_APP_WORDS];
261
+} __aligned(64);
262
+
263
+/**
264
+ * struct xilinx_aximcdma_desc_hw - Hardware Descriptor for AXI MCDMA
265
+ * @next_desc: Next Descriptor Pointer @0x00
266
+ * @next_desc_msb: MSB of Next Descriptor Pointer @0x04
267
+ * @buf_addr: Buffer address @0x08
268
+ * @buf_addr_msb: MSB of Buffer address @0x0C
269
+ * @rsvd: Reserved field @0x10
270
+ * @control: Control Information field @0x14
271
+ * @status: Status field @0x18
272
+ * @sideband_status: Status of sideband signals @0x1C
273
+ * @app: APP Fields @0x20 - 0x30
274
+ */
275
+struct xilinx_aximcdma_desc_hw {
276
+	u32 next_desc;
277
+	u32 next_desc_msb;
278
+	u32 buf_addr;
279
+	u32 buf_addr_msb;
280
+	u32 rsvd;
281
+	u32 control;
282
+	u32 status;
283
+	u32 sideband_status;
238284 	u32 app[XILINX_DMA_NUM_APP_WORDS];
239285 } __aligned(64);
240286 
....@@ -285,6 +331,18 @@
285331 } __aligned(64);
286332 
287333 /**
334
+ * struct xilinx_aximcdma_tx_segment - Descriptor segment
335
+ * @hw: Hardware descriptor
336
+ * @node: Node in the descriptor segments list
337
+ * @phys: Physical address of segment
338
+ */
339
+struct xilinx_aximcdma_tx_segment {
340
+	struct xilinx_aximcdma_desc_hw hw;
341
+	struct list_head node;
342
+	dma_addr_t phys;
343
+} __aligned(64);
344
+
345
+/**
288346  * struct xilinx_cdma_tx_segment - Descriptor segment
289347  * @hw: Hardware descriptor
290348  * @node: Node in the descriptor segments list
....@@ -302,12 +360,16 @@
302360  * @segments: TX segments list
303361  * @node: Node in the channel descriptors list
304362  * @cyclic: Check for cyclic transfers.
363
+ * @err: Whether the descriptor has an error.
364
+ * @residue: Residue of the completed descriptor
305365  */
306366 struct xilinx_dma_tx_descriptor {
307367 	struct dma_async_tx_descriptor async_tx;
308368 	struct list_head segments;
309369 	struct list_head node;
310370 	bool cyclic;
371
+	bool err;
372
+	u32 residue;
311373 };
312374 
313375 /**
....@@ -339,8 +401,8 @@
339401  * @desc_pendingcount: Descriptor pending count
340402  * @ext_addr: Indicates 64 bit addressing is supported by dma channel
341403  * @desc_submitcount: Descriptor h/w submitted count
342
- * @residue: Residue for AXI DMA
343404  * @seg_v: Statically allocated segments base
405
+ * @seg_mv: Statically allocated segments base for MCDMA
344406  * @seg_p: Physical allocated segments base
345407  * @cyclic_seg_v: Statically allocated segment base for cyclic transfers
346408  * @cyclic_seg_p: Physical allocated segments base for cyclic dma
....@@ -377,8 +439,8 @@
377439 	u32 desc_pendingcount;
378440 	bool ext_addr;
379441 	u32 desc_submitcount;
380
-	u32 residue;
381442 	struct xilinx_axidma_tx_segment *seg_v;
443
+	struct xilinx_aximcdma_tx_segment *seg_mv;
382444 	dma_addr_t seg_p;
383445 	struct xilinx_axidma_tx_segment *cyclic_seg_v;
384446 	dma_addr_t cyclic_seg_p;
....@@ -394,12 +456,14 @@
394456  * @XDMA_TYPE_AXIDMA: Axi dma ip.
395457  * @XDMA_TYPE_CDMA: Axi cdma ip.
396458  * @XDMA_TYPE_VDMA: Axi vdma ip.
459
+ * @XDMA_TYPE_AXIMCDMA: Axi MCDMA ip.
397460  *
398461  */
399462 enum xdma_ip_type {
400463 	XDMA_TYPE_AXIDMA = 0,
401464 	XDMA_TYPE_CDMA,
402465 	XDMA_TYPE_VDMA,
466
+	XDMA_TYPE_AXIMCDMA
403467 };
404468 
405469 struct xilinx_dma_config {
....@@ -407,6 +471,8 @@
407471 	int (*clk_init)(struct platform_device *pdev, struct clk **axi_clk,
408472 			struct clk **tx_clk, struct clk **txs_clk,
409473 			struct clk **rx_clk, struct clk **rxs_clk);
474
+	irqreturn_t (*irq_handler)(int irq, void *data);
475
+	const int max_channels;
410476 };
411477 
412478 /**
....@@ -415,8 +481,6 @@
415481  * @dev: Device Structure
416482  * @common: DMA device structure
417483  * @chan: Driver specific DMA channel
418
- * @has_sg: Specifies whether Scatter-Gather is present or not
419
- * @mcdma: Specifies whether Multi-Channel is present or not
420484  * @flush_on_fsync: Flush on frame sync
421485  * @ext_addr: Indicates 64 bit addressing is supported by dma device
422486  * @pdev: Platform device structure pointer
....@@ -426,16 +490,15 @@
426490  * @txs_clk: DMA mm2s stream clock
427491  * @rx_clk: DMA s2mm clock
428492  * @rxs_clk: DMA s2mm stream clock
429
- * @nr_channels: Number of channels DMA device supports
430
- * @chan_id: DMA channel identifier
493
+ * @s2mm_chan_id: DMA s2mm channel identifier
494
+ * @mm2s_chan_id: DMA mm2s channel identifier
495
+ * @max_buffer_len: Max buffer length
431496  */
432497 struct xilinx_dma_device {
433498 	void __iomem *regs;
434499 	struct device *dev;
435500 	struct dma_device common;
436
-	struct xilinx_dma_chan *chan[XILINX_DMA_MAX_CHANS_PER_DEVICE];
437
-	bool has_sg;
438
-	bool mcdma;
501
+	struct xilinx_dma_chan *chan[XILINX_MCDMA_MAX_CHANS_PER_DEVICE];
439502 	u32 flush_on_fsync;
440503 	bool ext_addr;
441504 	struct platform_device  *pdev;
....@@ -445,8 +508,9 @@
445508 	struct clk *txs_clk;
446509 	struct clk *rx_clk;
447510 	struct clk *rxs_clk;
448
-	u32 nr_channels;
449
-	u32 chan_id;
511
+	u32 s2mm_chan_id;
512
+	u32 mm2s_chan_id;
513
+	u32 max_buffer_len;
450514 };
451515 
452516 /* Macros */
....@@ -547,6 +611,18 @@
547611 	}
548612 }
549613 
614
+static inline void xilinx_aximcdma_buf(struct xilinx_dma_chan *chan,
615
+				       struct xilinx_aximcdma_desc_hw *hw,
616
+				       dma_addr_t buf_addr, size_t sg_used)
617
+{
618
+	if (chan->ext_addr) {
619
+		hw->buf_addr = lower_32_bits(buf_addr + sg_used);
620
+		hw->buf_addr_msb = upper_32_bits(buf_addr + sg_used);
621
+	} else {
622
+		hw->buf_addr = buf_addr + sg_used;
623
+	}
624
+}
625
+
550626 /* -----------------------------------------------------------------------------
551627  * Descriptors and segments alloc and free
552628  */
....@@ -614,6 +690,33 @@
614690 	}
615691 	spin_unlock_irqrestore(&chan->lock, flags);
616692 
693
+	if (!segment)
694
+		dev_dbg(chan->dev, "Could not find free tx segment\n");
695
+
696
+	return segment;
697
+}
698
+
699
+/**
700
+ * xilinx_aximcdma_alloc_tx_segment - Allocate transaction segment
701
+ * @chan: Driver specific DMA channel
702
+ *
703
+ * Return: The allocated segment on success and NULL on failure.
704
+ */
705
+static struct xilinx_aximcdma_tx_segment *
706
+xilinx_aximcdma_alloc_tx_segment(struct xilinx_dma_chan *chan)
707
+{
708
+	struct xilinx_aximcdma_tx_segment *segment = NULL;
709
+	unsigned long flags;
710
+
711
+	spin_lock_irqsave(&chan->lock, flags);
712
+	if (!list_empty(&chan->free_seg_list)) {
713
+		segment = list_first_entry(&chan->free_seg_list,
714
+					   struct xilinx_aximcdma_tx_segment,
715
+					   node);
716
+		list_del(&segment->node);
717
+	}
718
+	spin_unlock_irqrestore(&chan->lock, flags);
719
+
617720 	return segment;
618721 }
619722 
....@@ -628,6 +731,17 @@
628731 	hw->next_desc_msb = next_desc_msb;
629732 }
630733 
734
+static void xilinx_mcdma_clean_hw_desc(struct xilinx_aximcdma_desc_hw *hw)
735
+{
736
+	u32 next_desc = hw->next_desc;
737
+	u32 next_desc_msb = hw->next_desc_msb;
738
+
739
+	memset(hw, 0, sizeof(struct xilinx_aximcdma_desc_hw));
740
+
741
+	hw->next_desc = next_desc;
742
+	hw->next_desc_msb = next_desc_msb;
743
+}
744
+
631745 /**
632746  * xilinx_dma_free_tx_segment - Free transaction segment
633747  * @chan: Driver specific DMA channel
....@@ -637,6 +751,20 @@
637751 				struct xilinx_axidma_tx_segment *segment)
638752 {
639753 	xilinx_dma_clean_hw_desc(&segment->hw);
754
+
755
+	list_add_tail(&segment->node, &chan->free_seg_list);
756
+}
757
+
758
+/**
759
+ * xilinx_mcdma_free_tx_segment - Free transaction segment
760
+ * @chan: Driver specific DMA channel
761
+ * @segment: DMA transaction segment
762
+ */
763
+static void xilinx_mcdma_free_tx_segment(struct xilinx_dma_chan *chan,
764
+					 struct xilinx_aximcdma_tx_segment *
765
+					 segment)
766
+{
767
+	xilinx_mcdma_clean_hw_desc(&segment->hw);
640768 
641769 	list_add_tail(&segment->node, &chan->free_seg_list);
642770 }
....@@ -695,6 +823,7 @@
695823 	struct xilinx_vdma_tx_segment *segment, *next;
696824 	struct xilinx_cdma_tx_segment *cdma_segment, *cdma_next;
697825 	struct xilinx_axidma_tx_segment *axidma_segment, *axidma_next;
826
+	struct xilinx_aximcdma_tx_segment *aximcdma_segment, *aximcdma_next;
698827 
699828 	if (!desc)
700829 		return;
....@@ -710,11 +839,17 @@
710839 			list_del(&cdma_segment->node);
711840 			xilinx_cdma_free_tx_segment(chan, cdma_segment);
712841 		}
713
-	} else {
842
+	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
714843 		list_for_each_entry_safe(axidma_segment, axidma_next,
715844 					 &desc->segments, node) {
716845 			list_del(&axidma_segment->node);
717846 			xilinx_dma_free_tx_segment(chan, axidma_segment);
847
+		}
848
+	} else {
849
+		list_for_each_entry_safe(aximcdma_segment, aximcdma_next,
850
+					 &desc->segments, node) {
851
+			list_del(&aximcdma_segment->node);
852
+			xilinx_mcdma_free_tx_segment(chan, aximcdma_segment);
718853 		}
719854 	}
720855 
....@@ -784,10 +919,73 @@
784919 				  chan->cyclic_seg_v, chan->cyclic_seg_p);
785920 	}
786921 
787
-	if (chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA) {
922
+	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {
923
+		spin_lock_irqsave(&chan->lock, flags);
924
+		INIT_LIST_HEAD(&chan->free_seg_list);
925
+		spin_unlock_irqrestore(&chan->lock, flags);
926
+
927
+		/* Free memory that is allocated for BD */
928
+		dma_free_coherent(chan->dev, sizeof(*chan->seg_mv) *
929
+				  XILINX_DMA_NUM_DESCS, chan->seg_mv,
930
+				  chan->seg_p);
931
+	}
932
+
933
+	if (chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA &&
934
+	    chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIMCDMA) {
788935 		dma_pool_destroy(chan->desc_pool);
789936 		chan->desc_pool = NULL;
790937 	}
938
+
939
+}
940
+
941
+/**
942
+ * xilinx_dma_get_residue - Compute residue for a given descriptor
943
+ * @chan: Driver specific dma channel
944
+ * @desc: dma transaction descriptor
945
+ *
946
+ * Return: The number of residue bytes for the descriptor.
947
+ */
948
+static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan,
949
+				  struct xilinx_dma_tx_descriptor *desc)
950
+{
951
+	struct xilinx_cdma_tx_segment *cdma_seg;
952
+	struct xilinx_axidma_tx_segment *axidma_seg;
953
+	struct xilinx_aximcdma_tx_segment *aximcdma_seg;
954
+	struct xilinx_cdma_desc_hw *cdma_hw;
955
+	struct xilinx_axidma_desc_hw *axidma_hw;
956
+	struct xilinx_aximcdma_desc_hw *aximcdma_hw;
957
+	struct list_head *entry;
958
+	u32 residue = 0;
959
+
960
+	list_for_each(entry, &desc->segments) {
961
+		if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {
962
+			cdma_seg = list_entry(entry,
963
+					      struct xilinx_cdma_tx_segment,
964
+					      node);
965
+			cdma_hw = &cdma_seg->hw;
966
+			residue += (cdma_hw->control - cdma_hw->status) &
967
+				   chan->xdev->max_buffer_len;
968
+		} else if (chan->xdev->dma_config->dmatype ==
969
+			   XDMA_TYPE_AXIDMA) {
970
+			axidma_seg = list_entry(entry,
971
+						struct xilinx_axidma_tx_segment,
972
+						node);
973
+			axidma_hw = &axidma_seg->hw;
974
+			residue += (axidma_hw->control - axidma_hw->status) &
975
+				   chan->xdev->max_buffer_len;
976
+		} else {
977
+			aximcdma_seg =
978
+				list_entry(entry,
979
+					   struct xilinx_aximcdma_tx_segment,
980
+					   node);
981
+			aximcdma_hw = &aximcdma_seg->hw;
982
+			residue +=
983
+				(aximcdma_hw->control - aximcdma_hw->status) &
984
+				chan->xdev->max_buffer_len;
985
+		}
986
+	}
987
+
988
+	return residue;
791989 }
792990 
793991 /**
....@@ -824,7 +1022,7 @@
8241022 	spin_lock_irqsave(&chan->lock, flags);
8251023 
8261024 	list_for_each_entry_safe(desc, next, &chan->done_list, node) {
827
-		struct dmaengine_desc_callback cb;
1025
+		struct dmaengine_result result;
8281026 
8291027 		if (desc->cyclic) {
8301028 			xilinx_dma_chan_handle_cyclic(chan, desc, &flags);
....@@ -834,13 +1032,21 @@
8341032 		/* Remove from the list of running transactions */
8351033 		list_del(&desc->node);
8361034 
837
-		/* Run the link descriptor callback function */
838
-		dmaengine_desc_get_callback(&desc->async_tx, &cb);
839
-		if (dmaengine_desc_callback_valid(&cb)) {
840
-			spin_unlock_irqrestore(&chan->lock, flags);
841
-			dmaengine_desc_callback_invoke(&cb, NULL);
842
-			spin_lock_irqsave(&chan->lock, flags);
1035
+		if (unlikely(desc->err)) {
1036
+			if (chan->direction == DMA_DEV_TO_MEM)
1037
+				result.result = DMA_TRANS_READ_FAILED;
1038
+			else
1039
+				result.result = DMA_TRANS_WRITE_FAILED;
1040
+		} else {
1041
+			result.result = DMA_TRANS_NOERROR;
8431042 		}
1043
+
1044
+		result.residue = desc->residue;
1045
+
1046
+		/* Run the link descriptor callback function */
1047
+		spin_unlock_irqrestore(&chan->lock, flags);
1048
+		dmaengine_desc_get_callback_invoke(&desc->async_tx, &result);
1049
+		spin_lock_irqsave(&chan->lock, flags);
8441050 
8451051 		/* Run any dependencies, then free the descriptor */
8461052 		dma_run_dependencies(&desc->async_tx);
....@@ -859,11 +1065,11 @@
8591065 
8601066 /**
8611067  * xilinx_dma_do_tasklet - Schedule completion tasklet
862
- * @data: Pointer to the Xilinx DMA channel structure
1068
+ * @t: Pointer to the Xilinx DMA channel structure
8631069  */
864
-static void xilinx_dma_do_tasklet(unsigned long data)
1070
+static void xilinx_dma_do_tasklet(struct tasklet_struct *t)
8651071 {
866
-	struct xilinx_dma_chan *chan = (struct xilinx_dma_chan *)data;
1072
+	struct xilinx_dma_chan *chan = from_tasklet(chan, t, tasklet);
8671073 
8681074 	xilinx_dma_chan_desc_cleanup(chan);
8691075 }
....@@ -889,16 +1095,34 @@
8891095 	 */
8901096 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
8911097 		/* Allocate the buffer descriptors. */
892
-		chan->seg_v = dma_zalloc_coherent(chan->dev,
893
-						  sizeof(*chan->seg_v) *
894
-						  XILINX_DMA_NUM_DESCS,
895
-						  &chan->seg_p, GFP_KERNEL);
1098
+		chan->seg_v = dma_alloc_coherent(chan->dev,
1099
+						 sizeof(*chan->seg_v) * XILINX_DMA_NUM_DESCS,
1100
+						 &chan->seg_p, GFP_KERNEL);
8961101 		if (!chan->seg_v) {
8971102 			dev_err(chan->dev,
8981103 				"unable to allocate channel %d descriptors\n",
8991104 				chan->id);
9001105 			return -ENOMEM;
9011106 		}
1107
+		/*
1108
+		 * For cyclic DMA mode we need to program the tail Descriptor
1109
+		 * register with a value which is not a part of the BD chain
1110
+		 * so allocating a desc segment during channel allocation for
1111
+		 * programming tail descriptor.
1112
+		 */
1113
+		chan->cyclic_seg_v = dma_alloc_coherent(chan->dev,
1114
+							sizeof(*chan->cyclic_seg_v),
1115
+							&chan->cyclic_seg_p,
1116
+							GFP_KERNEL);
1117
+		if (!chan->cyclic_seg_v) {
1118
+			dev_err(chan->dev,
1119
+				"unable to allocate desc segment for cyclic DMA\n");
1120
+			dma_free_coherent(chan->dev, sizeof(*chan->seg_v) *
1121
+				XILINX_DMA_NUM_DESCS, chan->seg_v,
1122
+				chan->seg_p);
1123
+			return -ENOMEM;
1124
+		}
1125
+		chan->cyclic_seg_v->phys = chan->cyclic_seg_p;
9021126 
9031127 		for (i = 0; i < XILINX_DMA_NUM_DESCS; i++) {
9041128 			chan->seg_v[i].hw.next_desc =
....@@ -910,6 +1134,30 @@
9101134 			chan->seg_v[i].phys = chan->seg_p +
9111135 				sizeof(*chan->seg_v) * i;
9121136 			list_add_tail(&chan->seg_v[i].node,
1137
+				      &chan->free_seg_list);
1138
+		}
1139
+	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {
1140
+		/* Allocate the buffer descriptors. */
1141
+		chan->seg_mv = dma_alloc_coherent(chan->dev,
1142
+						  sizeof(*chan->seg_mv) *
1143
+						  XILINX_DMA_NUM_DESCS,
1144
+						  &chan->seg_p, GFP_KERNEL);
1145
+		if (!chan->seg_mv) {
1146
+			dev_err(chan->dev,
1147
+				"unable to allocate channel %d descriptors\n",
1148
+				chan->id);
1149
+			return -ENOMEM;
1150
+		}
1151
+		for (i = 0; i < XILINX_DMA_NUM_DESCS; i++) {
1152
+			chan->seg_mv[i].hw.next_desc =
1153
+			lower_32_bits(chan->seg_p + sizeof(*chan->seg_mv) *
1154
+				((i + 1) % XILINX_DMA_NUM_DESCS));
1155
+			chan->seg_mv[i].hw.next_desc_msb =
1156
+			upper_32_bits(chan->seg_p + sizeof(*chan->seg_mv) *
1157
+				((i + 1) % XILINX_DMA_NUM_DESCS));
1158
+			chan->seg_mv[i].phys = chan->seg_p +
1159
+				sizeof(*chan->seg_mv) * i;
1160
+			list_add_tail(&chan->seg_mv[i].node,
9131161 				      &chan->free_seg_list);
9141162 		}
9151163 	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {
....@@ -927,29 +1175,12 @@
9271175 	}
9281176 
9291177 	if (!chan->desc_pool &&
930
-	    (chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA)) {
1178
+	    ((chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA) &&
1179
+		chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIMCDMA)) {
9311180 		dev_err(chan->dev,
9321181 			"unable to allocate channel %d descriptor pool\n",
9331182 			chan->id);
9341183 		return -ENOMEM;
935
-	}
936
-
937
-	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
938
-		/*
939
-		 * For cyclic DMA mode we need to program the tail Descriptor
940
-		 * register with a value which is not a part of the BD chain
941
-		 * so allocating a desc segment during channel allocation for
942
-		 * programming tail descriptor.
943
-		 */
944
-		chan->cyclic_seg_v = dma_zalloc_coherent(chan->dev,
945
-					sizeof(*chan->cyclic_seg_v),
946
-					&chan->cyclic_seg_p, GFP_KERNEL);
947
-		if (!chan->cyclic_seg_v) {
948
-			dev_err(chan->dev,
949
-				"unable to allocate desc segment for cyclic DMA\n");
950
-			return -ENOMEM;
951
-		}
952
-		chan->cyclic_seg_v->phys = chan->cyclic_seg_p;
9531184 	}
9541185 
9551186 	dma_cookie_init(dchan);
....@@ -970,6 +1201,34 @@
9701201 }
9711202 
9721203 /**
1204
+ * xilinx_dma_calc_copysize - Calculate the amount of data to copy
1205
+ * @chan: Driver specific DMA channel
1206
+ * @size: Total data that needs to be copied
1207
+ * @done: Amount of data that has been already copied
1208
+ *
1209
+ * Return: Amount of data that has to be copied
1210
+ */
1211
+static int xilinx_dma_calc_copysize(struct xilinx_dma_chan *chan,
1212
+				    int size, int done)
1213
+{
1214
+	size_t copy;
1215
+
1216
+	copy = min_t(size_t, size - done,
1217
+		     chan->xdev->max_buffer_len);
1218
+
1219
+	if ((copy + done < size) &&
1220
+	    chan->xdev->common.copy_align) {
1221
+		/*
1222
+		 * If this is not the last descriptor, make sure
1223
+		 * the next one will be properly aligned
1224
+		 */
1225
+		copy = rounddown(copy,
1226
+				 (1 << chan->xdev->common.copy_align));
1227
+	}
1228
+	return copy;
1229
+}
1230
+
1231
+/**
9731232  * xilinx_dma_tx_status - Get DMA transaction status
9741233  * @dchan: DMA channel
9751234  * @cookie: Transaction identifier
....@@ -983,8 +1242,6 @@
9831242 {
9841243 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
9851244 	struct xilinx_dma_tx_descriptor *desc;
986
-	struct xilinx_axidma_tx_segment *segment;
987
-	struct xilinx_axidma_desc_hw *hw;
9881245 	enum dma_status ret;
9891246 	unsigned long flags;
9901247 	u32 residue = 0;
....@@ -993,23 +1250,20 @@
9931250 	if (ret == DMA_COMPLETE || !txstate)
9941251 		return ret;
9951252 
996
-	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
997
-		spin_lock_irqsave(&chan->lock, flags);
998
-
1253
+	spin_lock_irqsave(&chan->lock, flags);
1254
+	if (!list_empty(&chan->active_list)) {
9991255 		desc = list_last_entry(&chan->active_list,
10001256 				       struct xilinx_dma_tx_descriptor, node);
1001
-		if (chan->has_sg) {
1002
-			list_for_each_entry(segment, &desc->segments, node) {
1003
-				hw = &segment->hw;
1004
-				residue += (hw->control - hw->status) &
1005
-					   XILINX_DMA_MAX_TRANS_LEN;
1006
-			}
1007
-		}
1008
-		spin_unlock_irqrestore(&chan->lock, flags);
1009
-
1010
-		chan->residue = residue;
1011
-		dma_set_residue(txstate, chan->residue);
1257
+		/*
1258
+		 * VDMA and simple mode do not support residue reporting, so the
1259
+		 * residue field will always be 0.
1260
+		 */
1261
+		if (chan->has_sg && chan->xdev->dma_config->dmatype != XDMA_TYPE_VDMA)
1262
+			residue = xilinx_dma_get_residue(chan, desc);
10121263 	}
1264
+	spin_unlock_irqrestore(&chan->lock, flags);
1265
+
1266
+	dma_set_residue(txstate, residue);
10131267 
10141268 	return ret;
10151269 }
....@@ -1078,9 +1332,10 @@
10781332 static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)
10791333 {
10801334 	struct xilinx_vdma_config *config = &chan->config;
1081
-	struct xilinx_dma_tx_descriptor *desc, *tail_desc;
1335
+	struct xilinx_dma_tx_descriptor *desc;
10821336 	u32 reg, j;
1083
-	struct xilinx_vdma_tx_segment *tail_segment;
1337
+	struct xilinx_vdma_tx_segment *segment, *last = NULL;
1338
+	int i = 0;
10841339 
10851340 	/* This function was invoked with lock held */
10861341 	if (chan->err)
....@@ -1094,19 +1349,6 @@
10941349 
10951350 	desc = list_first_entry(&chan->pending_list,
10961351 				struct xilinx_dma_tx_descriptor, node);
1097
-	tail_desc = list_last_entry(&chan->pending_list,
1098
-				    struct xilinx_dma_tx_descriptor, node);
1099
-
1100
-	tail_segment = list_last_entry(&tail_desc->segments,
1101
-				       struct xilinx_vdma_tx_segment, node);
1102
-
1103
-	/*
1104
-	 * If hardware is idle, then all descriptors on the running lists are
1105
-	 * done, start new transfers
1106
-	 */
1107
-	if (chan->has_sg)
1108
-		dma_ctrl_write(chan, XILINX_DMA_REG_CURDESC,
1109
-				desc->async_tx.phys);
11101352 
11111353 	/* Configure the hardware using info in the config structure */
11121354 	if (chan->has_vflip) {
....@@ -1124,15 +1366,11 @@
11241366 	else
11251367 		reg &= ~XILINX_DMA_DMACR_FRAMECNT_EN;
11261368 
1127
-	/*
1128
-	 * With SG, start with circular mode, so that BDs can be fetched.
1129
-	 * In direct register mode, if not parking, enable circular mode
1130
-	 */
1131
-	if (chan->has_sg || !config->park)
1132
-		reg |= XILINX_DMA_DMACR_CIRC_EN;
1133
-
1369
+	/* If not parking, enable circular mode */
11341370 	if (config->park)
11351371 		reg &= ~XILINX_DMA_DMACR_CIRC_EN;
1372
+	else
1373
+		reg |= XILINX_DMA_DMACR_CIRC_EN;
11361374 
11371375 	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);
11381376 
....@@ -1154,48 +1392,38 @@
11541392 		return;
11551393 
11561394 	/* Start the transfer */
1157
-	if (chan->has_sg) {
1158
-		dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC,
1159
-				tail_segment->phys);
1160
-		list_splice_tail_init(&chan->pending_list, &chan->active_list);
1161
-		chan->desc_pendingcount = 0;
1162
-	} else {
1163
-		struct xilinx_vdma_tx_segment *segment, *last = NULL;
1164
-		int i = 0;
1395
+	if (chan->desc_submitcount < chan->num_frms)
1396
+		i = chan->desc_submitcount;
11651397 
1166
-		if (chan->desc_submitcount < chan->num_frms)
1167
-			i = chan->desc_submitcount;
1168
-
1169
-		list_for_each_entry(segment, &desc->segments, node) {
1170
-			if (chan->ext_addr)
1171
-				vdma_desc_write_64(chan,
1172
-					XILINX_VDMA_REG_START_ADDRESS_64(i++),
1173
-					segment->hw.buf_addr,
1174
-					segment->hw.buf_addr_msb);
1175
-			else
1176
-				vdma_desc_write(chan,
1398
+	list_for_each_entry(segment, &desc->segments, node) {
1399
+		if (chan->ext_addr)
1400
+			vdma_desc_write_64(chan,
1401
+				   XILINX_VDMA_REG_START_ADDRESS_64(i++),
1402
+				   segment->hw.buf_addr,
1403
+				   segment->hw.buf_addr_msb);
1404
+		else
1405
+			vdma_desc_write(chan,
11771406 					XILINX_VDMA_REG_START_ADDRESS(i++),
11781407 					segment->hw.buf_addr);
11791408 
1180
-			last = segment;
1181
-		}
1182
-
1183
-		if (!last)
1184
-			return;
1185
-
1186
-		/* HW expects these parameters to be same for one transaction */
1187
-		vdma_desc_write(chan, XILINX_DMA_REG_HSIZE, last->hw.hsize);
1188
-		vdma_desc_write(chan, XILINX_DMA_REG_FRMDLY_STRIDE,
1189
-				last->hw.stride);
1190
-		vdma_desc_write(chan, XILINX_DMA_REG_VSIZE, last->hw.vsize);
1191
-
1192
-		chan->desc_submitcount++;
1193
-		chan->desc_pendingcount--;
1194
-		list_del(&desc->node);
1195
-		list_add_tail(&desc->node, &chan->active_list);
1196
-		if (chan->desc_submitcount == chan->num_frms)
1197
-			chan->desc_submitcount = 0;
1409
+		last = segment;
11981410 	}
1411
+
1412
+	if (!last)
1413
+		return;
1414
+
1415
+	/* HW expects these parameters to be same for one transaction */
1416
+	vdma_desc_write(chan, XILINX_DMA_REG_HSIZE, last->hw.hsize);
1417
+	vdma_desc_write(chan, XILINX_DMA_REG_FRMDLY_STRIDE,
1418
+			last->hw.stride);
1419
+	vdma_desc_write(chan, XILINX_DMA_REG_VSIZE, last->hw.vsize);
1420
+
1421
+	chan->desc_submitcount++;
1422
+	chan->desc_pendingcount--;
1423
+	list_del(&desc->node);
1424
+	list_add_tail(&desc->node, &chan->active_list);
1425
+	if (chan->desc_submitcount == chan->num_frms)
1426
+		chan->desc_submitcount = 0;
11991427 
12001428 	chan->idle = false;
12011429 }
....@@ -1257,12 +1485,14 @@
12571485 
12581486 		hw = &segment->hw;
12591487 
1260
-		xilinx_write(chan, XILINX_CDMA_REG_SRCADDR, hw->src_addr);
1261
-		xilinx_write(chan, XILINX_CDMA_REG_DSTADDR, hw->dest_addr);
1488
+		xilinx_write(chan, XILINX_CDMA_REG_SRCADDR,
1489
+			     xilinx_prep_dma_addr_t(hw->src_addr));
1490
+		xilinx_write(chan, XILINX_CDMA_REG_DSTADDR,
1491
+			     xilinx_prep_dma_addr_t(hw->dest_addr));
12621492 
12631493 		/* Start the transfer */
12641494 		dma_ctrl_write(chan, XILINX_DMA_REG_BTT,
1265
-				hw->control & XILINX_DMA_MAX_TRANS_LEN);
1495
+				hw->control & chan->xdev->max_buffer_len);
12661496 	}
12671497 
12681498 	list_splice_tail_init(&chan->pending_list, &chan->active_list);
....@@ -1305,25 +1535,9 @@
13051535 		dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);
13061536 	}
13071537 
1308
-	if (chan->has_sg && !chan->xdev->mcdma)
1538
+	if (chan->has_sg)
13091539 		xilinx_write(chan, XILINX_DMA_REG_CURDESC,
13101540 			     head_desc->async_tx.phys);
1311
-
1312
-	if (chan->has_sg && chan->xdev->mcdma) {
1313
-		if (chan->direction == DMA_MEM_TO_DEV) {
1314
-			dma_ctrl_write(chan, XILINX_DMA_REG_CURDESC,
1315
-				       head_desc->async_tx.phys);
1316
-		} else {
1317
-			if (!chan->tdest) {
1318
-				dma_ctrl_write(chan, XILINX_DMA_REG_CURDESC,
1319
-				       head_desc->async_tx.phys);
1320
-			} else {
1321
-				dma_ctrl_write(chan,
1322
-					XILINX_DMA_MCRX_CDESC(chan->tdest),
1323
-				       head_desc->async_tx.phys);
1324
-			}
1325
-		}
1326
-	}
13271541 
13281542 	xilinx_dma_start(chan);
13291543 
....@@ -1331,27 +1545,13 @@
13311545 		return;
13321546 
13331547 	/* Start the transfer */
1334
-	if (chan->has_sg && !chan->xdev->mcdma) {
1548
+	if (chan->has_sg) {
13351549 		if (chan->cyclic)
13361550 			xilinx_write(chan, XILINX_DMA_REG_TAILDESC,
13371551 				     chan->cyclic_seg_v->phys);
13381552 		else
13391553 			xilinx_write(chan, XILINX_DMA_REG_TAILDESC,
13401554 				     tail_segment->phys);
1341
-	} else if (chan->has_sg && chan->xdev->mcdma) {
1342
-		if (chan->direction == DMA_MEM_TO_DEV) {
1343
-			dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC,
1344
-			       tail_segment->phys);
1345
-		} else {
1346
-			if (!chan->tdest) {
1347
-				dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC,
1348
-					       tail_segment->phys);
1349
-			} else {
1350
-				dma_ctrl_write(chan,
1351
-					XILINX_DMA_MCRX_TDESC(chan->tdest),
1352
-					tail_segment->phys);
1353
-			}
1354
-		}
13551555 	} else {
13561556 		struct xilinx_axidma_tx_segment *segment;
13571557 		struct xilinx_axidma_desc_hw *hw;
....@@ -1361,12 +1561,83 @@
13611561 					   node);
13621562 		hw = &segment->hw;
13631563 
1364
-		xilinx_write(chan, XILINX_DMA_REG_SRCDSTADDR, hw->buf_addr);
1564
+		xilinx_write(chan, XILINX_DMA_REG_SRCDSTADDR,
1565
+			     xilinx_prep_dma_addr_t(hw->buf_addr));
13651566 
13661567 		/* Start the transfer */
13671568 		dma_ctrl_write(chan, XILINX_DMA_REG_BTT,
1368
-			       hw->control & XILINX_DMA_MAX_TRANS_LEN);
1569
+			       hw->control & chan->xdev->max_buffer_len);
13691570 	}
1571
+
1572
+	list_splice_tail_init(&chan->pending_list, &chan->active_list);
1573
+	chan->desc_pendingcount = 0;
1574
+	chan->idle = false;
1575
+}
1576
+
1577
+/**
1578
+ * xilinx_mcdma_start_transfer - Starts MCDMA transfer
1579
+ * @chan: Driver specific channel struct pointer
1580
+ */
1581
+static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan)
1582
+{
1583
+	struct xilinx_dma_tx_descriptor *head_desc, *tail_desc;
1584
+	struct xilinx_aximcdma_tx_segment *tail_segment;
1585
+	u32 reg;
1586
+
1587
+	/*
1588
+	 * lock has been held by calling functions, so we don't need it
1589
+	 * to take it here again.
1590
+	 */
1591
+
1592
+	if (chan->err)
1593
+		return;
1594
+
1595
+	if (!chan->idle)
1596
+		return;
1597
+
1598
+	if (list_empty(&chan->pending_list))
1599
+		return;
1600
+
1601
+	head_desc = list_first_entry(&chan->pending_list,
1602
+				     struct xilinx_dma_tx_descriptor, node);
1603
+	tail_desc = list_last_entry(&chan->pending_list,
1604
+				    struct xilinx_dma_tx_descriptor, node);
1605
+	tail_segment = list_last_entry(&tail_desc->segments,
1606
+				       struct xilinx_aximcdma_tx_segment, node);
1607
+
1608
+	reg = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest));
1609
+
1610
+	if (chan->desc_pendingcount <= XILINX_MCDMA_COALESCE_MAX) {
1611
+		reg &= ~XILINX_MCDMA_COALESCE_MASK;
1612
+		reg |= chan->desc_pendingcount <<
1613
+			XILINX_MCDMA_COALESCE_SHIFT;
1614
+	}
1615
+
1616
+	reg |= XILINX_MCDMA_IRQ_ALL_MASK;
1617
+	dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg);
1618
+
1619
+	/* Program current descriptor */
1620
+	xilinx_write(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET(chan->tdest),
1621
+		     head_desc->async_tx.phys);
1622
+
1623
+	/* Program channel enable register */
1624
+	reg = dma_ctrl_read(chan, XILINX_MCDMA_CHEN_OFFSET);
1625
+	reg |= BIT(chan->tdest);
1626
+	dma_ctrl_write(chan, XILINX_MCDMA_CHEN_OFFSET, reg);
1627
+
1628
+	/* Start the fetch of BDs for the channel */
1629
+	reg = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest));
1630
+	reg |= XILINX_MCDMA_CR_RUNSTOP_MASK;
1631
+	dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg);
1632
+
1633
+	xilinx_dma_start(chan);
1634
+
1635
+	if (chan->err)
1636
+		return;
1637
+
1638
+	/* Start the transfer */
1639
+	xilinx_write(chan, XILINX_MCDMA_CHAN_TDESC_OFFSET(chan->tdest),
1640
+		     tail_segment->phys);
13701641 
13711642 	list_splice_tail_init(&chan->pending_list, &chan->active_list);
13721643 	chan->desc_pendingcount = 0;
....@@ -1402,6 +1673,13 @@
14021673 		return;
14031674 
14041675 	list_for_each_entry_safe(desc, next, &chan->active_list, node) {
1676
+		if (chan->has_sg && chan->xdev->dma_config->dmatype !=
1677
+		    XDMA_TYPE_VDMA)
1678
+			desc->residue = xilinx_dma_get_residue(chan, desc);
1679
+		else
1680
+			desc->residue = 0;
1681
+		desc->err = chan->err;
1682
+
14051683 		list_del(&desc->node);
14061684 		if (!desc->cyclic)
14071685 			dma_cookie_complete(&desc->async_tx);
....@@ -1462,6 +1740,74 @@
14621740 		      XILINX_DMA_DMAXR_ALL_IRQ_MASK);
14631741 
14641742 	return 0;
1743
+}
1744
+
1745
+/**
1746
+ * xilinx_mcdma_irq_handler - MCDMA Interrupt handler
1747
+ * @irq: IRQ number
1748
+ * @data: Pointer to the Xilinx MCDMA channel structure
1749
+ *
1750
+ * Return: IRQ_HANDLED/IRQ_NONE
1751
+ */
1752
+static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data)
1753
+{
1754
+	struct xilinx_dma_chan *chan = data;
1755
+	u32 status, ser_offset, chan_sermask, chan_offset = 0, chan_id;
1756
+
1757
+	if (chan->direction == DMA_DEV_TO_MEM)
1758
+		ser_offset = XILINX_MCDMA_RXINT_SER_OFFSET;
1759
+	else
1760
+		ser_offset = XILINX_MCDMA_TXINT_SER_OFFSET;
1761
+
1762
+	/* Read the channel id raising the interrupt*/
1763
+	chan_sermask = dma_ctrl_read(chan, ser_offset);
1764
+	chan_id = ffs(chan_sermask);
1765
+
1766
+	if (!chan_id)
1767
+		return IRQ_NONE;
1768
+
1769
+	if (chan->direction == DMA_DEV_TO_MEM)
1770
+		chan_offset = chan->xdev->dma_config->max_channels / 2;
1771
+
1772
+	chan_offset = chan_offset + (chan_id - 1);
1773
+	chan = chan->xdev->chan[chan_offset];
1774
+	/* Read the status and ack the interrupts. */
1775
+	status = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_SR_OFFSET(chan->tdest));
1776
+	if (!(status & XILINX_MCDMA_IRQ_ALL_MASK))
1777
+		return IRQ_NONE;
1778
+
1779
+	dma_ctrl_write(chan, XILINX_MCDMA_CHAN_SR_OFFSET(chan->tdest),
1780
+		       status & XILINX_MCDMA_IRQ_ALL_MASK);
1781
+
1782
+	if (status & XILINX_MCDMA_IRQ_ERR_MASK) {
1783
+		dev_err(chan->dev, "Channel %p has errors %x cdr %x tdr %x\n",
1784
+			chan,
1785
+			dma_ctrl_read(chan, XILINX_MCDMA_CH_ERR_OFFSET),
1786
+			dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET
1787
+				      (chan->tdest)),
1788
+			dma_ctrl_read(chan, XILINX_MCDMA_CHAN_TDESC_OFFSET
1789
+				      (chan->tdest)));
1790
+		chan->err = true;
1791
+	}
1792
+
1793
+	if (status & XILINX_MCDMA_IRQ_DELAY_MASK) {
1794
+		/*
1795
+		 * Device takes too long to do the transfer when user requires
1796
+		 * responsiveness.
1797
+		 */
1798
+		dev_dbg(chan->dev, "Inter-packet latency too long\n");
1799
+	}
1800
+
1801
+	if (status & XILINX_MCDMA_IRQ_IOC_MASK) {
1802
+		spin_lock(&chan->lock);
1803
+		xilinx_dma_complete_descriptor(chan);
1804
+		chan->idle = true;
1805
+		chan->start_transfer(chan);
1806
+		spin_unlock(&chan->lock);
1807
+	}
1808
+
1809
+	tasklet_schedule(&chan->tasklet);
1810
+	return IRQ_HANDLED;
14651811 }
14661812 
14671813 /**
....@@ -1539,6 +1885,7 @@
15391885 	struct xilinx_vdma_tx_segment *tail_segment;
15401886 	struct xilinx_dma_tx_descriptor *tail_desc;
15411887 	struct xilinx_axidma_tx_segment *axidma_tail_segment;
1888
+	struct xilinx_aximcdma_tx_segment *aximcdma_tail_segment;
15421889 	struct xilinx_cdma_tx_segment *cdma_tail_segment;
15431890 
15441891 	if (list_empty(&chan->pending_list))
....@@ -1560,11 +1907,17 @@
15601907 						struct xilinx_cdma_tx_segment,
15611908 						node);
15621909 		cdma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys;
1563
-	} else {
1910
+	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
15641911 		axidma_tail_segment = list_last_entry(&tail_desc->segments,
15651912 					       struct xilinx_axidma_tx_segment,
15661913 					       node);
15671914 		axidma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys;
1915
+	} else {
1916
+		aximcdma_tail_segment =
1917
+			list_last_entry(&tail_desc->segments,
1918
+					struct xilinx_aximcdma_tx_segment,
1919
+					node);
1920
+		aximcdma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys;
15681921 	}
15691922 
15701923 	/*
....@@ -1729,7 +2082,7 @@
17292082 	struct xilinx_cdma_tx_segment *segment;
17302083 	struct xilinx_cdma_desc_hw *hw;
17312084 
1732
-	if (!len || len > XILINX_DMA_MAX_TRANS_LEN)
2085
+	if (!len || len > chan->xdev->max_buffer_len)
17332086 		return NULL;
17342087 
17352088 	desc = xilinx_dma_alloc_tx_descriptor(chan);
....@@ -1819,8 +2172,8 @@
18192172 			 * Calculate the maximum number of bytes to transfer,
18202173 			 * making sure it is less than the hw limit
18212174 			 */
1822
-			copy = min_t(size_t, sg_dma_len(sg) - sg_used,
1823
-				     XILINX_DMA_MAX_TRANS_LEN);
2175
+			copy = xilinx_dma_calc_copysize(chan, sg_dma_len(sg),
2176
+							sg_used);
18242177 			hw = &segment->hw;
18252178 
18262179 			/* Fill in the descriptor */
....@@ -1924,8 +2277,8 @@
19242277 			 * Calculate the maximum number of bytes to transfer,
19252278 			 * making sure it is less than the hw limit
19262279 			 */
1927
-			copy = min_t(size_t, period_len - sg_used,
1928
-				     XILINX_DMA_MAX_TRANS_LEN);
2280
+			copy = xilinx_dma_calc_copysize(chan, period_len,
2281
+							sg_used);
19292282 			hw = &segment->hw;
19302283 			xilinx_axidma_buf(chan, hw, buf_addr, sg_used,
19312284 					  period_len * i);
....@@ -1973,31 +2326,32 @@
19732326 }
19742327 
19752328 /**
1976
- * xilinx_dma_prep_interleaved - prepare a descriptor for a
1977
- *	DMA_SLAVE transaction
2329
+ * xilinx_mcdma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
19782330  * @dchan: DMA channel
1979
- * @xt: Interleaved template pointer
2331
+ * @sgl: scatterlist to transfer to/from
2332
+ * @sg_len: number of entries in @scatterlist
2333
+ * @direction: DMA direction
19802334  * @flags: transfer ack flags
2335
+ * @context: APP words of the descriptor
19812336  *
19822337  * Return: Async transaction descriptor on success and NULL on failure
19832338  */
19842339 static struct dma_async_tx_descriptor *
1985
-xilinx_dma_prep_interleaved(struct dma_chan *dchan,
1986
-				 struct dma_interleaved_template *xt,
1987
-				 unsigned long flags)
2340
+xilinx_mcdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
2341
+			   unsigned int sg_len,
2342
+			   enum dma_transfer_direction direction,
2343
+			   unsigned long flags, void *context)
19882344 {
19892345 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
19902346 	struct xilinx_dma_tx_descriptor *desc;
1991
-	struct xilinx_axidma_tx_segment *segment;
1992
-	struct xilinx_axidma_desc_hw *hw;
2347
+	struct xilinx_aximcdma_tx_segment *segment = NULL;
2348
+	u32 *app_w = (u32 *)context;
2349
+	struct scatterlist *sg;
2350
+	size_t copy;
2351
+	size_t sg_used;
2352
+	unsigned int i;
19932353 
1994
-	if (!is_slave_direction(xt->dir))
1995
-		return NULL;
1996
-
1997
-	if (!xt->numf || !xt->sgl[0].size)
1998
-		return NULL;
1999
-
2000
-	if (xt->frame_size != 1)
2354
+	if (!is_slave_direction(direction))
20012355 		return NULL;
20022356 
20032357 	/* Allocate a transaction descriptor. */
....@@ -2005,54 +2359,67 @@
20052359 	if (!desc)
20062360 		return NULL;
20072361 
2008
-	chan->direction = xt->dir;
20092362 	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
20102363 	desc->async_tx.tx_submit = xilinx_dma_tx_submit;
20112364 
2012
-	/* Get a free segment */
2013
-	segment = xilinx_axidma_alloc_tx_segment(chan);
2014
-	if (!segment)
2015
-		goto error;
2365
+	/* Build transactions using information in the scatter gather list */
2366
+	for_each_sg(sgl, sg, sg_len, i) {
2367
+		sg_used = 0;
20162368 
2017
-	hw = &segment->hw;
2369
+		/* Loop until the entire scatterlist entry is used */
2370
+		while (sg_used < sg_dma_len(sg)) {
2371
+			struct xilinx_aximcdma_desc_hw *hw;
20182372 
2019
-	/* Fill in the descriptor */
2020
-	if (xt->dir != DMA_MEM_TO_DEV)
2021
-		hw->buf_addr = xt->dst_start;
2022
-	else
2023
-		hw->buf_addr = xt->src_start;
2373
+			/* Get a free segment */
2374
+			segment = xilinx_aximcdma_alloc_tx_segment(chan);
2375
+			if (!segment)
2376
+				goto error;
20242377 
2025
-	hw->mcdma_control = chan->tdest & XILINX_DMA_BD_TDEST_MASK;
2026
-	hw->vsize_stride = (xt->numf << XILINX_DMA_BD_VSIZE_SHIFT) &
2027
-			    XILINX_DMA_BD_VSIZE_MASK;
2028
-	hw->vsize_stride |= (xt->sgl[0].icg + xt->sgl[0].size) &
2029
-			    XILINX_DMA_BD_STRIDE_MASK;
2030
-	hw->control = xt->sgl[0].size & XILINX_DMA_BD_HSIZE_MASK;
2378
+			/*
2379
+			 * Calculate the maximum number of bytes to transfer,
2380
+			 * making sure it is less than the hw limit
2381
+			 */
2382
+			copy = min_t(size_t, sg_dma_len(sg) - sg_used,
2383
+				     chan->xdev->max_buffer_len);
2384
+			hw = &segment->hw;
20312385 
2032
-	/*
2033
-	 * Insert the segment into the descriptor segments
2034
-	 * list.
2035
-	 */
2036
-	list_add_tail(&segment->node, &desc->segments);
2386
+			/* Fill in the descriptor */
2387
+			xilinx_aximcdma_buf(chan, hw, sg_dma_address(sg),
2388
+					    sg_used);
2389
+			hw->control = copy;
20372390 
2391
+			if (chan->direction == DMA_MEM_TO_DEV && app_w) {
2392
+				memcpy(hw->app, app_w, sizeof(u32) *
2393
+				       XILINX_DMA_NUM_APP_WORDS);
2394
+			}
2395
+
2396
+			sg_used += copy;
2397
+			/*
2398
+			 * Insert the segment into the descriptor segments
2399
+			 * list.
2400
+			 */
2401
+			list_add_tail(&segment->node, &desc->segments);
2402
+		}
2403
+	}
20382404 
20392405 	segment = list_first_entry(&desc->segments,
2040
-				   struct xilinx_axidma_tx_segment, node);
2406
+				   struct xilinx_aximcdma_tx_segment, node);
20412407 	desc->async_tx.phys = segment->phys;
20422408 
20432409 	/* For the last DMA_MEM_TO_DEV transfer, set EOP */
2044
-	if (xt->dir == DMA_MEM_TO_DEV) {
2045
-		segment->hw.control |= XILINX_DMA_BD_SOP;
2410
+	if (chan->direction == DMA_MEM_TO_DEV) {
2411
+		segment->hw.control |= XILINX_MCDMA_BD_SOP;
20462412 		segment = list_last_entry(&desc->segments,
2047
-					  struct xilinx_axidma_tx_segment,
2413
+					  struct xilinx_aximcdma_tx_segment,
20482414 					  node);
2049
-		segment->hw.control |= XILINX_DMA_BD_EOP;
2415
+		segment->hw.control |= XILINX_MCDMA_BD_EOP;
20502416 	}
20512417 
20522418 	return &desc->async_tx;
20532419 
20542420 error:
20552421 	xilinx_dma_free_tx_descriptor(chan, desc);
2422
+
20562423 	return NULL;
20572424 }
20582425 
....@@ -2068,16 +2435,17 @@
20682435 	u32 reg;
20692436 	int err;
20702437 
2071
-	if (chan->cyclic)
2072
-		xilinx_dma_chan_reset(chan);
2073
-
2074
-	err = chan->stop_transfer(chan);
2075
-	if (err) {
2076
-		dev_err(chan->dev, "Cannot stop channel %p: %x\n",
2077
-			chan, dma_ctrl_read(chan, XILINX_DMA_REG_DMASR));
2078
-		chan->err = true;
2438
+	if (!chan->cyclic) {
2439
+		err = chan->stop_transfer(chan);
2440
+		if (err) {
2441
+			dev_err(chan->dev, "Cannot stop channel %p: %x\n",
2442
+				chan, dma_ctrl_read(chan,
2443
+				XILINX_DMA_REG_DMASR));
2444
+			chan->err = true;
2445
+		}
20792446 	}
20802447 
2448
+	xilinx_dma_chan_reset(chan);
20812449 	/* Remove and free all of the descriptors in the lists */
20822450 	chan->terminating = true;
20832451 	xilinx_dma_free_descriptors(chan);
....@@ -2199,11 +2567,8 @@
21992567 	*tmp_clk = NULL;
22002568 
22012569 	*axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk");
2202
-	if (IS_ERR(*axi_clk)) {
2203
-		err = PTR_ERR(*axi_clk);
2204
-		dev_err(&pdev->dev, "failed to get axi_aclk (%d)\n", err);
2205
-		return err;
2206
-	}
2570
+	if (IS_ERR(*axi_clk))
2571
+		return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n");
22072572 
22082573 	*tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk");
22092574 	if (IS_ERR(*tx_clk))
....@@ -2264,18 +2629,12 @@
22642629 	*tmp2_clk = NULL;
22652630 
22662631 	*axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk");
2267
-	if (IS_ERR(*axi_clk)) {
2268
-		err = PTR_ERR(*axi_clk);
2269
-		dev_err(&pdev->dev, "failed to get axi_clk (%d)\n", err);
2270
-		return err;
2271
-	}
2632
+	if (IS_ERR(*axi_clk))
2633
+		return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n");
22722634 
22732635 	*dev_clk = devm_clk_get(&pdev->dev, "m_axi_aclk");
2274
-	if (IS_ERR(*dev_clk)) {
2275
-		err = PTR_ERR(*dev_clk);
2276
-		dev_err(&pdev->dev, "failed to get dev_clk (%d)\n", err);
2277
-		return err;
2278
-	}
2636
+	if (IS_ERR(*dev_clk))
2637
+		return dev_err_probe(&pdev->dev, PTR_ERR(*dev_clk), "failed to get dev_clk\n");
22792638 
22802639 	err = clk_prepare_enable(*axi_clk);
22812640 	if (err) {
....@@ -2304,11 +2663,8 @@
23042663 	int err;
23052664 
23062665 	*axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk");
2307
-	if (IS_ERR(*axi_clk)) {
2308
-		err = PTR_ERR(*axi_clk);
2309
-		dev_err(&pdev->dev, "failed to get axi_aclk (%d)\n", err);
2310
-		return err;
2311
-	}
2666
+	if (IS_ERR(*axi_clk))
2667
+		return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n");
23122668 
23132669 	*tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk");
23142670 	if (IS_ERR(*tx_clk))
....@@ -2328,7 +2684,8 @@
23282684 
23292685 	err = clk_prepare_enable(*axi_clk);
23302686 	if (err) {
2331
-		dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", err);
2687
+		dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n",
2688
+			err);
23322689 		return err;
23332690 	}
23342691 
....@@ -2386,12 +2743,11 @@
23862743  *
23872744  * @xdev: Driver specific device structure
23882745  * @node: Device node
2389
- * @chan_id: DMA Channel id
23902746  *
23912747  * Return: '0' on success and failure value on error
23922748  */
23932749 static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev,
2394
-				  struct device_node *node, int chan_id)
2750
+				  struct device_node *node)
23952751 {
23962752 	struct xilinx_dma_chan *chan;
23972753 	bool has_dre = false;
....@@ -2405,7 +2761,6 @@
24052761 
24062762 	chan->dev = xdev->dev;
24072763 	chan->xdev = xdev;
2408
-	chan->has_sg = xdev->has_sg;
24092764 	chan->desc_pendingcount = 0x0;
24102765 	chan->ext_addr = xdev->ext_addr;
24112766 	/* This variable ensures that descriptors are not
....@@ -2444,8 +2799,8 @@
24442799 	    of_device_is_compatible(node, "xlnx,axi-dma-mm2s-channel") ||
24452800 	    of_device_is_compatible(node, "xlnx,axi-cdma-channel")) {
24462801 		chan->direction = DMA_MEM_TO_DEV;
2447
-		chan->id = chan_id;
2448
-		chan->tdest = chan_id;
2802
+		chan->id = xdev->mm2s_chan_id++;
2803
+		chan->tdest = chan->id;
24492804 
24502805 		chan->ctrl_offset = XILINX_DMA_MM2S_CTRL_OFFSET;
24512806 		if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
....@@ -2461,8 +2816,8 @@
24612816 		   of_device_is_compatible(node,
24622817 					   "xlnx,axi-dma-s2mm-channel")) {
24632818 		chan->direction = DMA_DEV_TO_MEM;
2464
-		chan->id = chan_id;
2465
-		chan->tdest = chan_id - xdev->nr_channels;
2819
+		chan->id = xdev->s2mm_chan_id++;
2820
+		chan->tdest = chan->id - xdev->dma_config->max_channels / 2;
24662821 		chan->has_vflip = of_property_read_bool(node,
24672822 					"xlnx,enable-vert-flip");
24682823 		if (chan->has_vflip) {
....@@ -2471,7 +2826,11 @@
24712826 				XILINX_VDMA_ENABLE_VERTICAL_FLIP;
24722827 		}
24732828 
2474
-		chan->ctrl_offset = XILINX_DMA_S2MM_CTRL_OFFSET;
2829
+		if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA)
2830
+			chan->ctrl_offset = XILINX_MCDMA_S2MM_CTRL_OFFSET;
2831
+		else
2832
+			chan->ctrl_offset = XILINX_DMA_S2MM_CTRL_OFFSET;
2833
+
24752834 		if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
24762835 			chan->desc_offset = XILINX_VDMA_S2MM_DESC_OFFSET;
24772836 			chan->config.park = 1;
....@@ -2486,9 +2845,9 @@
24862845 	}
24872846 
24882847 	/* Request the interrupt */
2489
-	chan->irq = irq_of_parse_and_map(node, 0);
2490
-	err = request_irq(chan->irq, xilinx_dma_irq_handler, IRQF_SHARED,
2491
-			  "xilinx-dma-controller", chan);
2848
+	chan->irq = irq_of_parse_and_map(node, chan->tdest);
2849
+	err = request_irq(chan->irq, xdev->dma_config->irq_handler,
2850
+			  IRQF_SHARED, "xilinx-dma-controller", chan);
24922851 	if (err) {
24932852 		dev_err(xdev->dev, "unable to request IRQ %d\n", chan->irq);
24942853 		return err;
....@@ -2496,6 +2855,9 @@
24962855 
24972856 	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
24982857 		chan->start_transfer = xilinx_dma_start_transfer;
2858
+		chan->stop_transfer = xilinx_dma_stop_transfer;
2859
+	} else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {
2860
+		chan->start_transfer = xilinx_mcdma_start_transfer;
24992861 		chan->stop_transfer = xilinx_dma_stop_transfer;
25002862 	} else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {
25012863 		chan->start_transfer = xilinx_cdma_start_transfer;
....@@ -2505,9 +2867,18 @@
25052867 		chan->stop_transfer = xilinx_dma_stop_transfer;
25062868 	}
25072869 
2870
+	/* check if SG is enabled (only for AXIDMA, AXIMCDMA, and CDMA) */
2871
+	if (xdev->dma_config->dmatype != XDMA_TYPE_VDMA) {
2872
+		if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA ||
2873
+		    dma_ctrl_read(chan, XILINX_DMA_REG_DMASR) &
2874
+			    XILINX_DMA_DMASR_SG_MASK)
2875
+			chan->has_sg = true;
2876
+		dev_dbg(chan->dev, "ch %d: SG %s\n", chan->id,
2877
+			chan->has_sg ? "enabled" : "disabled");
2878
+	}
2879
+
25082880 	/* Initialize the tasklet */
2509
-	tasklet_init(&chan->tasklet, xilinx_dma_do_tasklet,
2510
-			(unsigned long)chan);
2881
+	tasklet_setup(&chan->tasklet, xilinx_dma_do_tasklet);
25112882 
25122883 	/*
25132884 	 * Initialize the DMA channel and add it to the DMA engine channels
....@@ -2545,13 +2916,11 @@
25452916 	u32 nr_channels = 1;
25462917 
25472918 	ret = of_property_read_u32(node, "dma-channels", &nr_channels);
2548
-	if ((ret < 0) && xdev->mcdma)
2919
+	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA && ret < 0)
25492920 		dev_warn(xdev->dev, "missing dma-channels property\n");
25502921 
25512922 	for (i = 0; i < nr_channels; i++)
2552
-		xilinx_dma_chan_probe(xdev, node, xdev->chan_id++);
2553
-
2554
-	xdev->nr_channels += nr_channels;
2923
+		xilinx_dma_chan_probe(xdev, node);
25552924 
25562925 	return 0;
25572926 }
....@@ -2569,7 +2938,7 @@
25692938 	struct xilinx_dma_device *xdev = ofdma->of_dma_data;
25702939 	int chan_id = dma_spec->args[0];
25712940 
2572
-	if (chan_id >= xdev->nr_channels || !xdev->chan[chan_id])
2941
+	if (chan_id >= xdev->dma_config->max_channels || !xdev->chan[chan_id])
25732942 		return NULL;
25742943 
25752944 	return dma_get_slave_channel(&xdev->chan[chan_id]->common);
....@@ -2578,22 +2947,35 @@
25782947 static const struct xilinx_dma_config axidma_config = {
25792948 	.dmatype = XDMA_TYPE_AXIDMA,
25802949 	.clk_init = axidma_clk_init,
2950
+	.irq_handler = xilinx_dma_irq_handler,
2951
+	.max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE,
25812952 };
25822953 
2954
+static const struct xilinx_dma_config aximcdma_config = {
2955
+	.dmatype = XDMA_TYPE_AXIMCDMA,
2956
+	.clk_init = axidma_clk_init,
2957
+	.irq_handler = xilinx_mcdma_irq_handler,
2958
+	.max_channels = XILINX_MCDMA_MAX_CHANS_PER_DEVICE,
2959
+};
25832960 static const struct xilinx_dma_config axicdma_config = {
25842961 	.dmatype = XDMA_TYPE_CDMA,
25852962 	.clk_init = axicdma_clk_init,
2963
+	.irq_handler = xilinx_dma_irq_handler,
2964
+	.max_channels = XILINX_CDMA_MAX_CHANS_PER_DEVICE,
25862965 };
25872966 
25882967 static const struct xilinx_dma_config axivdma_config = {
25892968 	.dmatype = XDMA_TYPE_VDMA,
25902969 	.clk_init = axivdma_clk_init,
2970
+	.irq_handler = xilinx_dma_irq_handler,
2971
+	.max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE,
25912972 };
25922973 
25932974 static const struct of_device_id xilinx_dma_of_ids[] = {
25942975 	{ .compatible = "xlnx,axi-dma-1.00.a", .data = &axidma_config },
25952976 	{ .compatible = "xlnx,axi-cdma-1.00.a", .data = &axicdma_config },
25962977 	{ .compatible = "xlnx,axi-vdma-1.00.a", .data = &axivdma_config },
2978
+	{ .compatible = "xlnx,axi-mcdma-1.00.a", .data = &aximcdma_config },
25972979 	{}
25982980 };
25992981 MODULE_DEVICE_TABLE(of, xilinx_dma_of_ids);
....@@ -2612,8 +2994,7 @@
26122994 	struct device_node *node = pdev->dev.of_node;
26132995 	struct xilinx_dma_device *xdev;
26142996 	struct device_node *child, *np = pdev->dev.of_node;
2615
-	struct resource *io;
2616
-	u32 num_frames, addr_width;
2997
+	u32 num_frames, addr_width, len_width;
26172998 	int i, err;
26182999 
26193000 	/* Allocate and initialize the DMA engine structure */
....@@ -2638,15 +3019,31 @@
26383019 		return err;
26393020 
26403021 	/* Request and map I/O memory */
2641
-	io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2642
-	xdev->regs = devm_ioremap_resource(&pdev->dev, io);
2643
-	if (IS_ERR(xdev->regs))
2644
-		return PTR_ERR(xdev->regs);
2645
-
3022
+	xdev->regs = devm_platform_ioremap_resource(pdev, 0);
3023
+	if (IS_ERR(xdev->regs)) {
3024
+		err = PTR_ERR(xdev->regs);
3025
+		goto disable_clks;
3026
+	}
26463027 	/* Retrieve the DMA engine properties from the device tree */
2647
-	xdev->has_sg = of_property_read_bool(node, "xlnx,include-sg");
2648
-	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA)
2649
-		xdev->mcdma = of_property_read_bool(node, "xlnx,mcdma");
3028
+	xdev->max_buffer_len = GENMASK(XILINX_DMA_MAX_TRANS_LEN_MAX - 1, 0);
3029
+	xdev->s2mm_chan_id = xdev->dma_config->max_channels / 2;
3030
+
3031
+	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA ||
3032
+	    xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {
3033
+		if (!of_property_read_u32(node, "xlnx,sg-length-width",
3034
+					  &len_width)) {
3035
+			if (len_width < XILINX_DMA_MAX_TRANS_LEN_MIN ||
3036
+			    len_width > XILINX_DMA_V2_MAX_TRANS_LEN_MAX) {
3037
+				dev_warn(xdev->dev,
3038
+					 "invalid xlnx,sg-length-width property value. Using default width\n");
3039
+			} else {
3040
+				if (len_width > XILINX_DMA_MAX_TRANS_LEN_MAX)
3041
+					dev_warn(xdev->dev, "Please ensure that IP supports buffer length > 23 bits\n");
3042
+				xdev->max_buffer_len =
3043
+					GENMASK(len_width - 1, 0);
3044
+			}
3045
+		}
3046
+	}
26503047 
26513048 	if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
26523049 		err = of_property_read_u32(node, "xlnx,num-fstores",
....@@ -2654,7 +3051,7 @@
26543051 		if (err < 0) {
26553052 			dev_err(xdev->dev,
26563053 				"missing xlnx,num-fstores property\n");
2657
-			return err;
3054
+			goto disable_clks;
26583055 		}
26593056 
26603057 		err = of_property_read_u32(node, "xlnx,flush-fsync",
....@@ -2674,7 +3071,11 @@
26743071 		xdev->ext_addr = false;
26753072 
26763073 	/* Set the dma mask bits */
2677
-	dma_set_mask_and_coherent(xdev->dev, DMA_BIT_MASK(addr_width));
3074
+	err = dma_set_mask_and_coherent(xdev->dev, DMA_BIT_MASK(addr_width));
3075
+	if (err < 0) {
3076
+		dev_err(xdev->dev, "DMA mask error %d\n", err);
3077
+		goto disable_clks;
3078
+	}
26783079 
26793080 	/* Initialize the DMA engine */
26803081 	xdev->common.dev = &pdev->dev;
....@@ -2697,14 +3098,17 @@
26973098 		xdev->common.device_prep_slave_sg = xilinx_dma_prep_slave_sg;
26983099 		xdev->common.device_prep_dma_cyclic =
26993100 					  xilinx_dma_prep_dma_cyclic;
2700
-		xdev->common.device_prep_interleaved_dma =
2701
-					xilinx_dma_prep_interleaved;
2702
-		/* Residue calculation is supported by only AXI DMA */
3101
+		/* Residue calculation is supported by only AXI DMA and CDMA */
27033102 		xdev->common.residue_granularity =
27043103 					  DMA_RESIDUE_GRANULARITY_SEGMENT;
27053104 	} else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {
27063105 		dma_cap_set(DMA_MEMCPY, xdev->common.cap_mask);
27073106 		xdev->common.device_prep_dma_memcpy = xilinx_cdma_prep_memcpy;
3107
+		/* Residue calculation is supported by only AXI DMA and CDMA */
3108
+		xdev->common.residue_granularity =
3109
+					  DMA_RESIDUE_GRANULARITY_SEGMENT;
3110
+	} else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {
3111
+		xdev->common.device_prep_slave_sg = xilinx_mcdma_prep_slave_sg;
27083112 	} else {
27093113 		xdev->common.device_prep_interleaved_dma =
27103114 				xilinx_vdma_dma_prep_interleaved;
....@@ -2715,12 +3119,14 @@
27153119 	/* Initialize the channels */
27163120 	for_each_child_of_node(node, child) {
27173121 		err = xilinx_dma_child_probe(xdev, child);
2718
-		if (err < 0)
2719
-			goto disable_clks;
3122
+		if (err < 0) {
3123
+			of_node_put(child);
3124
+			goto error;
3125
+		}
27203126 	}
27213127 
27223128 	if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
2723
-		for (i = 0; i < xdev->nr_channels; i++)
3129
+		for (i = 0; i < xdev->dma_config->max_channels; i++)
27243130 			if (xdev->chan[i])
27253131 				xdev->chan[i]->num_frms = num_frames;
27263132 	}
....@@ -2744,17 +3150,19 @@
27443150 		dev_info(&pdev->dev, "Xilinx AXI DMA Engine Driver Probed!!\n");
27453151 	else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA)
27463152 		dev_info(&pdev->dev, "Xilinx AXI CDMA Engine Driver Probed!!\n");
3153
+	else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA)
3154
+		dev_info(&pdev->dev, "Xilinx AXI MCDMA Engine Driver Probed!!\n");
27473155 	else
27483156 		dev_info(&pdev->dev, "Xilinx AXI VDMA Engine Driver Probed!!\n");
27493157 
27503158 	return 0;
27513159 
2752
-disable_clks:
2753
-	xdma_disable_allclks(xdev);
27543160 error:
2755
-	for (i = 0; i < xdev->nr_channels; i++)
3161
+	for (i = 0; i < xdev->dma_config->max_channels; i++)
27563162 		if (xdev->chan[i])
27573163 			xilinx_dma_chan_remove(xdev->chan[i]);
3164
+disable_clks:
3165
+	xdma_disable_allclks(xdev);
27583166 
27593167 	return err;
27603168 }
....@@ -2774,7 +3182,7 @@
27743182 
27753183 	dma_async_device_unregister(&xdev->common);
27763184 
2777
-	for (i = 0; i < xdev->nr_channels; i++)
3185
+	for (i = 0; i < xdev->dma_config->max_channels; i++)
27783186 		if (xdev->chan[i])
27793187 			xilinx_dma_chan_remove(xdev->chan[i]);
27803188