~hc/RK356X_SDK_RELEASE.git

..	..	@@ -3,13 +3,12 @@
3	3	* Copyright (C) 2012-2015 Spreadtrum Communications Inc.
4	4	*/
5	5
6		-#if defined(CONFIG_SERIAL_SPRD_CONSOLE) && defined(CONFIG_MAGIC_SYSRQ)
7		-#define SUPPORT_SYSRQ
8		-#endif
9		-
10	6	#include <linux/clk.h>
11	7	#include <linux/console.h>
12	8	#include <linux/delay.h>
	9	+#include <linux/dmaengine.h>
	10	+#include <linux/dma-mapping.h>
	11	+#include <linux/dma/sprd-dma.h>
13	12	#include <linux/io.h>
14	13	#include <linux/ioport.h>
15	14	#include <linux/kernel.h>
..	..	@@ -45,6 +44,8 @@
45	44
46	45	/* data number in TX and RX fifo */
47	46	#define SPRD_STS1 0x000C
	47	+#define SPRD_RX_FIFO_CNT_MASK GENMASK(7, 0)
	48	+#define SPRD_TX_FIFO_CNT_MASK GENMASK(15, 8)
48	49
49	50	/* interrupt enable register and its BITs */
50	51	#define SPRD_IEN 0x0010
..	..	@@ -66,67 +67,88 @@
66	67	#define SPRD_LCR_DATA_LEN6 0x4
67	68	#define SPRD_LCR_DATA_LEN7 0x8
68	69	#define SPRD_LCR_DATA_LEN8 0xc
69		-#define SPRD_LCR_PARITY (BIT(0) \| BIT(1))
	70	+#define SPRD_LCR_PARITY (BIT(0) \| BIT(1))
70	71	#define SPRD_LCR_PARITY_EN 0x2
71	72	#define SPRD_LCR_EVEN_PAR 0x0
72	73	#define SPRD_LCR_ODD_PAR 0x1
73	74
74	75	/* control register 1 */
75		-#define SPRD_CTL1 0x001C
	76	+#define SPRD_CTL1 0x001C
	77	+#define SPRD_DMA_EN BIT(15)
	78	+#define SPRD_LOOPBACK_EN BIT(14)
76	79	#define RX_HW_FLOW_CTL_THLD BIT(6)
77	80	#define RX_HW_FLOW_CTL_EN BIT(7)
78	81	#define TX_HW_FLOW_CTL_EN BIT(8)
79	82	#define RX_TOUT_THLD_DEF 0x3E00
80		-#define RX_HFC_THLD_DEF 0x40
	83	+#define RX_HFC_THLD_DEF 0x40
81	84
82	85	/* fifo threshold register */
83	86	#define SPRD_CTL2 0x0020
84		-#define THLD_TX_EMPTY 0x40
85		-#define THLD_RX_FULL 0x40
	87	+#define THLD_TX_EMPTY 0x40
	88	+#define THLD_TX_EMPTY_SHIFT 8
	89	+#define THLD_RX_FULL 0x40
	90	+#define THLD_RX_FULL_MASK GENMASK(6, 0)
86	91
87	92	/* config baud rate register */
88	93	#define SPRD_CLKD0 0x0024
	94	+#define SPRD_CLKD0_MASK GENMASK(15, 0)
89	95	#define SPRD_CLKD1 0x0028
	96	+#define SPRD_CLKD1_MASK GENMASK(20, 16)
	97	+#define SPRD_CLKD1_SHIFT 16
90	98
91	99	/* interrupt mask status register */
92		-#define SPRD_IMSR 0x002C
93		-#define SPRD_IMSR_RX_FIFO_FULL BIT(0)
	100	+#define SPRD_IMSR 0x002C
	101	+#define SPRD_IMSR_RX_FIFO_FULL BIT(0)
94	102	#define SPRD_IMSR_TX_FIFO_EMPTY BIT(1)
95		-#define SPRD_IMSR_BREAK_DETECT BIT(7)
96		-#define SPRD_IMSR_TIMEOUT BIT(13)
	103	+#define SPRD_IMSR_BREAK_DETECT BIT(7)
	104	+#define SPRD_IMSR_TIMEOUT BIT(13)
	105	+#define SPRD_DEFAULT_SOURCE_CLK 26000000
97	106
98		-struct reg_backup {
99		- u32 ien;
100		- u32 ctrl0;
101		- u32 ctrl1;
102		- u32 ctrl2;
103		- u32 clkd0;
104		- u32 clkd1;
105		- u32 dspwait;
	107	+#define SPRD_RX_DMA_STEP 1
	108	+#define SPRD_RX_FIFO_FULL 1
	109	+#define SPRD_TX_FIFO_FULL 0x20
	110	+#define SPRD_UART_RX_SIZE (UART_XMIT_SIZE / 4)
	111	+
	112	+struct sprd_uart_dma {
	113	+ struct dma_chan *chn;
	114	+ unsigned char *virt;
	115	+ dma_addr_t phys_addr;
	116	+ dma_cookie_t cookie;
	117	+ u32 trans_len;
	118	+ bool enable;
106	119	};
107	120
108	121	struct sprd_uart_port {
109	122	struct uart_port port;
110		- struct reg_backup reg_bak;
111	123	char name[16];
	124	+ struct clk *clk;
	125	+ struct sprd_uart_dma tx_dma;
	126	+ struct sprd_uart_dma rx_dma;
	127	+ dma_addr_t pos;
	128	+ unsigned char *rx_buf_tail;
112	129	};
113	130
114	131	static struct sprd_uart_port *sprd_port[UART_NR_MAX];
115	132	static int sprd_ports_num;
116	133
117		-static inline unsigned int serial_in(struct uart_port *port, int offset)
	134	+static int sprd_start_dma_rx(struct uart_port *port);
	135	+static int sprd_tx_dma_config(struct uart_port *port);
	136	+
	137	+static inline unsigned int serial_in(struct uart_port *port,
	138	+ unsigned int offset)
118	139	{
119	140	return readl_relaxed(port->membase + offset);
120	141	}
121	142
122		-static inline void serial_out(struct uart_port *port, int offset, int value)
	143	+static inline void serial_out(struct uart_port *port, unsigned int offset,
	144	+ int value)
123	145	{
124	146	writel_relaxed(value, port->membase + offset);
125	147	}
126	148
127	149	static unsigned int sprd_tx_empty(struct uart_port *port)
128	150	{
129		- if (serial_in(port, SPRD_STS1) & 0xff00)
	151	+ if (serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)
130	152	return 0;
131	153	else
132	154	return TIOCSER_TEMT;
..	..	@@ -139,37 +161,24 @@
139	161
140	162	static void sprd_set_mctrl(struct uart_port *port, unsigned int mctrl)
141	163	{
142		- /* nothing to do */
143		-}
	164	+ u32 val = serial_in(port, SPRD_CTL1);
144	165
145		-static void sprd_stop_tx(struct uart_port *port)
146		-{
147		- unsigned int ien, iclr;
	166	+ if (mctrl & TIOCM_LOOP)
	167	+ val \|= SPRD_LOOPBACK_EN;
	168	+ else
	169	+ val &= ~SPRD_LOOPBACK_EN;
148	170
149		- iclr = serial_in(port, SPRD_ICLR);
150		- ien = serial_in(port, SPRD_IEN);
151		-
152		- iclr \|= SPRD_IEN_TX_EMPTY;
153		- ien &= ~SPRD_IEN_TX_EMPTY;
154		-
155		- serial_out(port, SPRD_ICLR, iclr);
156		- serial_out(port, SPRD_IEN, ien);
157		-}
158		-
159		-static void sprd_start_tx(struct uart_port *port)
160		-{
161		- unsigned int ien;
162		-
163		- ien = serial_in(port, SPRD_IEN);
164		- if (!(ien & SPRD_IEN_TX_EMPTY)) {
165		- ien \|= SPRD_IEN_TX_EMPTY;
166		- serial_out(port, SPRD_IEN, ien);
167		- }
	171	+ serial_out(port, SPRD_CTL1, val);
168	172	}
169	173
170	174	static void sprd_stop_rx(struct uart_port *port)
171	175	{
	176	+ struct sprd_uart_port *sp =
	177	+ container_of(port, struct sprd_uart_port, port);
172	178	unsigned int ien, iclr;
	179	+
	180	+ if (sp->rx_dma.enable)
	181	+ dmaengine_terminate_all(sp->rx_dma.chn);
173	182
174	183	iclr = serial_in(port, SPRD_ICLR);
175	184	ien = serial_in(port, SPRD_IEN);
..	..	@@ -179,6 +188,370 @@
179	188
180	189	serial_out(port, SPRD_IEN, ien);
181	190	serial_out(port, SPRD_ICLR, iclr);
	191	+}
	192	+
	193	+static void sprd_uart_dma_enable(struct uart_port *port, bool enable)
	194	+{
	195	+ u32 val = serial_in(port, SPRD_CTL1);
	196	+
	197	+ if (enable)
	198	+ val \|= SPRD_DMA_EN;
	199	+ else
	200	+ val &= ~SPRD_DMA_EN;
	201	+
	202	+ serial_out(port, SPRD_CTL1, val);
	203	+}
	204	+
	205	+static void sprd_stop_tx_dma(struct uart_port *port)
	206	+{
	207	+ struct sprd_uart_port *sp =
	208	+ container_of(port, struct sprd_uart_port, port);
	209	+ struct circ_buf *xmit = &port->state->xmit;
	210	+ struct dma_tx_state state;
	211	+ u32 trans_len;
	212	+
	213	+ dmaengine_pause(sp->tx_dma.chn);
	214	+
	215	+ dmaengine_tx_status(sp->tx_dma.chn, sp->tx_dma.cookie, &state);
	216	+ if (state.residue) {
	217	+ trans_len = state.residue - sp->tx_dma.phys_addr;
	218	+ xmit->tail = (xmit->tail + trans_len) & (UART_XMIT_SIZE - 1);
	219	+ port->icount.tx += trans_len;
	220	+ dma_unmap_single(port->dev, sp->tx_dma.phys_addr,
	221	+ sp->tx_dma.trans_len, DMA_TO_DEVICE);
	222	+ }
	223	+
	224	+ dmaengine_terminate_all(sp->tx_dma.chn);
	225	+ sp->tx_dma.trans_len = 0;
	226	+}
	227	+
	228	+static int sprd_tx_buf_remap(struct uart_port *port)
	229	+{
	230	+ struct sprd_uart_port *sp =
	231	+ container_of(port, struct sprd_uart_port, port);
	232	+ struct circ_buf *xmit = &port->state->xmit;
	233	+
	234	+ sp->tx_dma.trans_len =
	235	+ CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
	236	+
	237	+ sp->tx_dma.phys_addr = dma_map_single(port->dev,
	238	+ (void *)&(xmit->buf[xmit->tail]),
	239	+ sp->tx_dma.trans_len,
	240	+ DMA_TO_DEVICE);
	241	+ return dma_mapping_error(port->dev, sp->tx_dma.phys_addr);
	242	+}
	243	+
	244	+static void sprd_complete_tx_dma(void *data)
	245	+{
	246	+ struct uart_port port = (struct uart_port )data;
	247	+ struct sprd_uart_port *sp =
	248	+ container_of(port, struct sprd_uart_port, port);
	249	+ struct circ_buf *xmit = &port->state->xmit;
	250	+ unsigned long flags;
	251	+
	252	+ spin_lock_irqsave(&port->lock, flags);
	253	+ dma_unmap_single(port->dev, sp->tx_dma.phys_addr,
	254	+ sp->tx_dma.trans_len, DMA_TO_DEVICE);
	255	+
	256	+ xmit->tail = (xmit->tail + sp->tx_dma.trans_len) & (UART_XMIT_SIZE - 1);
	257	+ port->icount.tx += sp->tx_dma.trans_len;
	258	+
	259	+ if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
	260	+ uart_write_wakeup(port);
	261	+
	262	+ if (uart_circ_empty(xmit) \|\| sprd_tx_buf_remap(port) \|\|
	263	+ sprd_tx_dma_config(port))
	264	+ sp->tx_dma.trans_len = 0;
	265	+
	266	+ spin_unlock_irqrestore(&port->lock, flags);
	267	+}
	268	+
	269	+static int sprd_uart_dma_submit(struct uart_port *port,
	270	+ struct sprd_uart_dma *ud, u32 trans_len,
	271	+ enum dma_transfer_direction direction,
	272	+ dma_async_tx_callback callback)
	273	+{
	274	+ struct dma_async_tx_descriptor *dma_des;
	275	+ unsigned long flags;
	276	+
	277	+ flags = SPRD_DMA_FLAGS(SPRD_DMA_CHN_MODE_NONE,
	278	+ SPRD_DMA_NO_TRG,
	279	+ SPRD_DMA_FRAG_REQ,
	280	+ SPRD_DMA_TRANS_INT);
	281	+
	282	+ dma_des = dmaengine_prep_slave_single(ud->chn, ud->phys_addr, trans_len,
	283	+ direction, flags);
	284	+ if (!dma_des)
	285	+ return -ENODEV;
	286	+
	287	+ dma_des->callback = callback;
	288	+ dma_des->callback_param = port;
	289	+
	290	+ ud->cookie = dmaengine_submit(dma_des);
	291	+ if (dma_submit_error(ud->cookie))
	292	+ return dma_submit_error(ud->cookie);
	293	+
	294	+ dma_async_issue_pending(ud->chn);
	295	+
	296	+ return 0;
	297	+}
	298	+
	299	+static int sprd_tx_dma_config(struct uart_port *port)
	300	+{
	301	+ struct sprd_uart_port *sp =
	302	+ container_of(port, struct sprd_uart_port, port);
	303	+ u32 burst = sp->tx_dma.trans_len > SPRD_TX_FIFO_FULL ?
	304	+ SPRD_TX_FIFO_FULL : sp->tx_dma.trans_len;
	305	+ int ret;
	306	+ struct dma_slave_config cfg = {
	307	+ .dst_addr = port->mapbase + SPRD_TXD,
	308	+ .src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
	309	+ .dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
	310	+ .src_maxburst = burst,
	311	+ };
	312	+
	313	+ ret = dmaengine_slave_config(sp->tx_dma.chn, &cfg);
	314	+ if (ret < 0)
	315	+ return ret;
	316	+
	317	+ return sprd_uart_dma_submit(port, &sp->tx_dma, sp->tx_dma.trans_len,
	318	+ DMA_MEM_TO_DEV, sprd_complete_tx_dma);
	319	+}
	320	+
	321	+static void sprd_start_tx_dma(struct uart_port *port)
	322	+{
	323	+ struct sprd_uart_port *sp =
	324	+ container_of(port, struct sprd_uart_port, port);
	325	+ struct circ_buf *xmit = &port->state->xmit;
	326	+
	327	+ if (port->x_char) {
	328	+ serial_out(port, SPRD_TXD, port->x_char);
	329	+ port->icount.tx++;
	330	+ port->x_char = 0;
	331	+ return;
	332	+ }
	333	+
	334	+ if (uart_circ_empty(xmit) \|\| uart_tx_stopped(port)) {
	335	+ sprd_stop_tx_dma(port);
	336	+ return;
	337	+ }
	338	+
	339	+ if (sp->tx_dma.trans_len)
	340	+ return;
	341	+
	342	+ if (sprd_tx_buf_remap(port) \|\| sprd_tx_dma_config(port))
	343	+ sp->tx_dma.trans_len = 0;
	344	+}
	345	+
	346	+static void sprd_rx_full_thld(struct uart_port *port, u32 thld)
	347	+{
	348	+ u32 val = serial_in(port, SPRD_CTL2);
	349	+
	350	+ val &= ~THLD_RX_FULL_MASK;
	351	+ val \|= thld & THLD_RX_FULL_MASK;
	352	+ serial_out(port, SPRD_CTL2, val);
	353	+}
	354	+
	355	+static int sprd_rx_alloc_buf(struct sprd_uart_port *sp)
	356	+{
	357	+ sp->rx_dma.virt = dma_alloc_coherent(sp->port.dev, SPRD_UART_RX_SIZE,
	358	+ &sp->rx_dma.phys_addr, GFP_KERNEL);
	359	+ if (!sp->rx_dma.virt)
	360	+ return -ENOMEM;
	361	+
	362	+ return 0;
	363	+}
	364	+
	365	+static void sprd_rx_free_buf(struct sprd_uart_port *sp)
	366	+{
	367	+ if (sp->rx_dma.virt)
	368	+ dma_free_coherent(sp->port.dev, SPRD_UART_RX_SIZE,
	369	+ sp->rx_dma.virt, sp->rx_dma.phys_addr);
	370	+ sp->rx_dma.virt = NULL;
	371	+}
	372	+
	373	+static int sprd_rx_dma_config(struct uart_port *port, u32 burst)
	374	+{
	375	+ struct sprd_uart_port *sp =
	376	+ container_of(port, struct sprd_uart_port, port);
	377	+ struct dma_slave_config cfg = {
	378	+ .src_addr = port->mapbase + SPRD_RXD,
	379	+ .src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
	380	+ .dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
	381	+ .src_maxburst = burst,
	382	+ };
	383	+
	384	+ return dmaengine_slave_config(sp->rx_dma.chn, &cfg);
	385	+}
	386	+
	387	+static void sprd_uart_dma_rx(struct uart_port *port)
	388	+{
	389	+ struct sprd_uart_port *sp =
	390	+ container_of(port, struct sprd_uart_port, port);
	391	+ struct tty_port *tty = &port->state->port;
	392	+
	393	+ port->icount.rx += sp->rx_dma.trans_len;
	394	+ tty_insert_flip_string(tty, sp->rx_buf_tail, sp->rx_dma.trans_len);
	395	+ tty_flip_buffer_push(tty);
	396	+}
	397	+
	398	+static void sprd_uart_dma_irq(struct uart_port *port)
	399	+{
	400	+ struct sprd_uart_port *sp =
	401	+ container_of(port, struct sprd_uart_port, port);
	402	+ struct dma_tx_state state;
	403	+ enum dma_status status;
	404	+
	405	+ status = dmaengine_tx_status(sp->rx_dma.chn,
	406	+ sp->rx_dma.cookie, &state);
	407	+ if (status == DMA_ERROR)
	408	+ sprd_stop_rx(port);
	409	+
	410	+ if (!state.residue && sp->pos == sp->rx_dma.phys_addr)
	411	+ return;
	412	+
	413	+ if (!state.residue) {
	414	+ sp->rx_dma.trans_len = SPRD_UART_RX_SIZE +
	415	+ sp->rx_dma.phys_addr - sp->pos;
	416	+ sp->pos = sp->rx_dma.phys_addr;
	417	+ } else {
	418	+ sp->rx_dma.trans_len = state.residue - sp->pos;
	419	+ sp->pos = state.residue;
	420	+ }
	421	+
	422	+ sprd_uart_dma_rx(port);
	423	+ sp->rx_buf_tail += sp->rx_dma.trans_len;
	424	+}
	425	+
	426	+static void sprd_complete_rx_dma(void *data)
	427	+{
	428	+ struct uart_port port = (struct uart_port )data;
	429	+ struct sprd_uart_port *sp =
	430	+ container_of(port, struct sprd_uart_port, port);
	431	+ struct dma_tx_state state;
	432	+ enum dma_status status;
	433	+ unsigned long flags;
	434	+
	435	+ spin_lock_irqsave(&port->lock, flags);
	436	+
	437	+ status = dmaengine_tx_status(sp->rx_dma.chn,
	438	+ sp->rx_dma.cookie, &state);
	439	+ if (status != DMA_COMPLETE) {
	440	+ sprd_stop_rx(port);
	441	+ spin_unlock_irqrestore(&port->lock, flags);
	442	+ return;
	443	+ }
	444	+
	445	+ if (sp->pos != sp->rx_dma.phys_addr) {
	446	+ sp->rx_dma.trans_len = SPRD_UART_RX_SIZE +
	447	+ sp->rx_dma.phys_addr - sp->pos;
	448	+ sprd_uart_dma_rx(port);
	449	+ sp->rx_buf_tail += sp->rx_dma.trans_len;
	450	+ }
	451	+
	452	+ if (sprd_start_dma_rx(port))
	453	+ sprd_stop_rx(port);
	454	+
	455	+ spin_unlock_irqrestore(&port->lock, flags);
	456	+}
	457	+
	458	+static int sprd_start_dma_rx(struct uart_port *port)
	459	+{
	460	+ struct sprd_uart_port *sp =
	461	+ container_of(port, struct sprd_uart_port, port);
	462	+ int ret;
	463	+
	464	+ if (!sp->rx_dma.enable)
	465	+ return 0;
	466	+
	467	+ sp->pos = sp->rx_dma.phys_addr;
	468	+ sp->rx_buf_tail = sp->rx_dma.virt;
	469	+ sprd_rx_full_thld(port, SPRD_RX_FIFO_FULL);
	470	+ ret = sprd_rx_dma_config(port, SPRD_RX_DMA_STEP);
	471	+ if (ret)
	472	+ return ret;
	473	+
	474	+ return sprd_uart_dma_submit(port, &sp->rx_dma, SPRD_UART_RX_SIZE,
	475	+ DMA_DEV_TO_MEM, sprd_complete_rx_dma);
	476	+}
	477	+
	478	+static void sprd_release_dma(struct uart_port *port)
	479	+{
	480	+ struct sprd_uart_port *sp =
	481	+ container_of(port, struct sprd_uart_port, port);
	482	+
	483	+ sprd_uart_dma_enable(port, false);
	484	+
	485	+ if (sp->rx_dma.enable)
	486	+ dma_release_channel(sp->rx_dma.chn);
	487	+
	488	+ if (sp->tx_dma.enable)
	489	+ dma_release_channel(sp->tx_dma.chn);
	490	+
	491	+ sp->tx_dma.enable = false;
	492	+ sp->rx_dma.enable = false;
	493	+}
	494	+
	495	+static void sprd_request_dma(struct uart_port *port)
	496	+{
	497	+ struct sprd_uart_port *sp =
	498	+ container_of(port, struct sprd_uart_port, port);
	499	+
	500	+ sp->tx_dma.enable = true;
	501	+ sp->rx_dma.enable = true;
	502	+
	503	+ sp->tx_dma.chn = dma_request_chan(port->dev, "tx");
	504	+ if (IS_ERR(sp->tx_dma.chn)) {
	505	+ dev_err(port->dev, "request TX DMA channel failed, ret = %ld\n",
	506	+ PTR_ERR(sp->tx_dma.chn));
	507	+ sp->tx_dma.enable = false;
	508	+ }
	509	+
	510	+ sp->rx_dma.chn = dma_request_chan(port->dev, "rx");
	511	+ if (IS_ERR(sp->rx_dma.chn)) {
	512	+ dev_err(port->dev, "request RX DMA channel failed, ret = %ld\n",
	513	+ PTR_ERR(sp->rx_dma.chn));
	514	+ sp->rx_dma.enable = false;
	515	+ }
	516	+}
	517	+
	518	+static void sprd_stop_tx(struct uart_port *port)
	519	+{
	520	+ struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
	521	+ port);
	522	+ unsigned int ien, iclr;
	523	+
	524	+ if (sp->tx_dma.enable) {
	525	+ sprd_stop_tx_dma(port);
	526	+ return;
	527	+ }
	528	+
	529	+ iclr = serial_in(port, SPRD_ICLR);
	530	+ ien = serial_in(port, SPRD_IEN);
	531	+
	532	+ iclr \|= SPRD_IEN_TX_EMPTY;
	533	+ ien &= ~SPRD_IEN_TX_EMPTY;
	534	+
	535	+ serial_out(port, SPRD_IEN, ien);
	536	+ serial_out(port, SPRD_ICLR, iclr);
	537	+}
	538	+
	539	+static void sprd_start_tx(struct uart_port *port)
	540	+{
	541	+ struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
	542	+ port);
	543	+ unsigned int ien;
	544	+
	545	+ if (sp->tx_dma.enable) {
	546	+ sprd_start_tx_dma(port);
	547	+ return;
	548	+ }
	549	+
	550	+ ien = serial_in(port, SPRD_IEN);
	551	+ if (!(ien & SPRD_IEN_TX_EMPTY)) {
	552	+ ien \|= SPRD_IEN_TX_EMPTY;
	553	+ serial_out(port, SPRD_IEN, ien);
	554	+ }
182	555	}
183	556
184	557	/* The Sprd serial does not support this function. */
..	..	@@ -221,17 +594,25 @@
221	594
222	595	static inline void sprd_rx(struct uart_port *port)
223	596	{
	597	+ struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
	598	+ port);
224	599	struct tty_port *tty = &port->state->port;
225	600	unsigned int ch, flag, lsr, max_count = SPRD_TIMEOUT;
226	601
227		- while ((serial_in(port, SPRD_STS1) & 0x00ff) && max_count--) {
	602	+ if (sp->rx_dma.enable) {
	603	+ sprd_uart_dma_irq(port);
	604	+ return;
	605	+ }
	606	+
	607	+ while ((serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK) &&
	608	+ max_count--) {
228	609	lsr = serial_in(port, SPRD_LSR);
229	610	ch = serial_in(port, SPRD_RXD);
230	611	flag = TTY_NORMAL;
231	612	port->icount.rx++;
232	613
233	614	if (lsr & (SPRD_LSR_BI \| SPRD_LSR_PE \|
234		- SPRD_LSR_FE \| SPRD_LSR_OE))
	615	+ SPRD_LSR_FE \| SPRD_LSR_OE))
235	616	if (handle_lsr_errors(port, &flag, &lsr))
236	617	continue;
237	618	if (uart_handle_sysrq_char(port, ch))
..	..	@@ -294,8 +675,11 @@
294	675	if (ims & SPRD_IMSR_TIMEOUT)
295	676	serial_out(port, SPRD_ICLR, SPRD_ICLR_TIMEOUT);
296	677
297		- if (ims & (SPRD_IMSR_RX_FIFO_FULL \|
298		- SPRD_IMSR_BREAK_DETECT \| SPRD_IMSR_TIMEOUT))
	678	+ if (ims & SPRD_IMSR_BREAK_DETECT)
	679	+ serial_out(port, SPRD_ICLR, SPRD_IMSR_BREAK_DETECT);
	680	+
	681	+ if (ims & (SPRD_IMSR_RX_FIFO_FULL \| SPRD_IMSR_BREAK_DETECT \|
	682	+ SPRD_IMSR_TIMEOUT))
299	683	sprd_rx(port);
300	684
301	685	if (ims & SPRD_IMSR_TX_FIFO_EMPTY)
..	..	@@ -306,6 +690,25 @@
306	690	return IRQ_HANDLED;
307	691	}
308	692
	693	+static void sprd_uart_dma_startup(struct uart_port *port,
	694	+ struct sprd_uart_port *sp)
	695	+{
	696	+ int ret;
	697	+
	698	+ sprd_request_dma(port);
	699	+ if (!(sp->rx_dma.enable \|\| sp->tx_dma.enable))
	700	+ return;
	701	+
	702	+ ret = sprd_start_dma_rx(port);
	703	+ if (ret) {
	704	+ sp->rx_dma.enable = false;
	705	+ dma_release_channel(sp->rx_dma.chn);
	706	+ dev_warn(port->dev, "fail to start RX dma mode\n");
	707	+ }
	708	+
	709	+ sprd_uart_dma_enable(port, true);
	710	+}
	711	+
309	712	static int sprd_startup(struct uart_port *port)
310	713	{
311	714	int ret = 0;
..	..	@@ -314,16 +717,17 @@
314	717	struct sprd_uart_port *sp;
315	718	unsigned long flags;
316	719
317		- serial_out(port, SPRD_CTL2, ((THLD_TX_EMPTY << 8) \| THLD_RX_FULL));
	720	+ serial_out(port, SPRD_CTL2,
	721	+ THLD_TX_EMPTY << THLD_TX_EMPTY_SHIFT \| THLD_RX_FULL);
318	722
319	723	/* clear rx fifo */
320	724	timeout = SPRD_TIMEOUT;
321		- while (timeout-- && serial_in(port, SPRD_STS1) & 0x00ff)
	725	+ while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK)
322	726	serial_in(port, SPRD_RXD);
323	727
324	728	/* clear tx fifo */
325	729	timeout = SPRD_TIMEOUT;
326		- while (timeout-- && serial_in(port, SPRD_STS1) & 0xff00)
	730	+ while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)
327	731	cpu_relax();
328	732
329	733	/* clear interrupt */
..	..	@@ -333,8 +737,11 @@
333	737	/* allocate irq */
334	738	sp = container_of(port, struct sprd_uart_port, port);
335	739	snprintf(sp->name, sizeof(sp->name), "sprd_serial%d", port->line);
	740	+
	741	+ sprd_uart_dma_startup(port, sp);
	742	+
336	743	ret = devm_request_irq(port->dev, port->irq, sprd_handle_irq,
337		- IRQF_SHARED, sp->name, port);
	744	+ IRQF_SHARED, sp->name, port);
338	745	if (ret) {
339	746	dev_err(port->dev, "fail to request serial irq %d, ret=%d\n",
340	747	port->irq, ret);
..	..	@@ -347,7 +754,9 @@
347	754	/* enable interrupt */
348	755	spin_lock_irqsave(&port->lock, flags);
349	756	ien = serial_in(port, SPRD_IEN);
350		- ien \|= SPRD_IEN_RX_FULL \| SPRD_IEN_BREAK_DETECT \| SPRD_IEN_TIMEOUT;
	757	+ ien \|= SPRD_IEN_BREAK_DETECT \| SPRD_IEN_TIMEOUT;
	758	+ if (!sp->rx_dma.enable)
	759	+ ien \|= SPRD_IEN_RX_FULL;
351	760	serial_out(port, SPRD_IEN, ien);
352	761	spin_unlock_irqrestore(&port->lock, flags);
353	762
..	..	@@ -356,14 +765,15 @@
356	765
357	766	static void sprd_shutdown(struct uart_port *port)
358	767	{
	768	+ sprd_release_dma(port);
359	769	serial_out(port, SPRD_IEN, 0);
360	770	serial_out(port, SPRD_ICLR, ~0);
361	771	devm_free_irq(port->dev, port->irq, port);
362	772	}
363	773
364	774	static void sprd_set_termios(struct uart_port *port,
365		- struct ktermios *termios,
366		- struct ktermios *old)
	775	+ struct ktermios *termios,
	776	+ struct ktermios *old)
367	777	{
368	778	unsigned int baud, quot;
369	779	unsigned int lcr = 0, fc;
..	..	@@ -372,7 +782,7 @@
372	782	/* ask the core to calculate the divisor for us */
373	783	baud = uart_get_baud_rate(port, termios, old, 0, SPRD_BAUD_IO_LIMIT);
374	784
375		- quot = (unsigned int)((port->uartclk + baud / 2) / baud);
	785	+ quot = port->uartclk / baud;
376	786
377	787	/* set data length */
378	788	switch (termios->c_cflag & CSIZE) {
..	..	@@ -444,10 +854,11 @@
444	854	}
445	855
446	856	/* clock divider bit0~bit15 */
447		- serial_out(port, SPRD_CLKD0, quot & 0xffff);
	857	+ serial_out(port, SPRD_CLKD0, quot & SPRD_CLKD0_MASK);
448	858
449	859	/* clock divider bit16~bit20 */
450		- serial_out(port, SPRD_CLKD1, (quot & 0x1f0000) >> 16);
	860	+ serial_out(port, SPRD_CLKD1,
	861	+ (quot & SPRD_CLKD1_MASK) >> SPRD_CLKD1_SHIFT);
451	862	serial_out(port, SPRD_LCR, lcr);
452	863	fc \|= RX_TOUT_THLD_DEF \| RX_HFC_THLD_DEF;
453	864	serial_out(port, SPRD_CTL1, fc);
..	..	@@ -480,8 +891,7 @@
480	891	port->type = PORT_SPRD;
481	892	}
482	893
483		-static int sprd_verify_port(struct uart_port *port,
484		- struct serial_struct *ser)
	894	+static int sprd_verify_port(struct uart_port port, struct serial_struct ser)
485	895	{
486	896	if (ser->type != PORT_SPRD)
487	897	return -EINVAL;
..	..	@@ -491,6 +901,50 @@
491	901	return -EINVAL;
492	902	return 0;
493	903	}
	904	+
	905	+static void sprd_pm(struct uart_port *port, unsigned int state,
	906	+ unsigned int oldstate)
	907	+{
	908	+ struct sprd_uart_port *sup =
	909	+ container_of(port, struct sprd_uart_port, port);
	910	+
	911	+ switch (state) {
	912	+ case UART_PM_STATE_ON:
	913	+ clk_prepare_enable(sup->clk);
	914	+ break;
	915	+ case UART_PM_STATE_OFF:
	916	+ clk_disable_unprepare(sup->clk);
	917	+ break;
	918	+ }
	919	+}
	920	+
	921	+#ifdef CONFIG_CONSOLE_POLL
	922	+static int sprd_poll_init(struct uart_port *port)
	923	+{
	924	+ if (port->state->pm_state != UART_PM_STATE_ON) {
	925	+ sprd_pm(port, UART_PM_STATE_ON, 0);
	926	+ port->state->pm_state = UART_PM_STATE_ON;
	927	+ }
	928	+
	929	+ return 0;
	930	+}
	931	+
	932	+static int sprd_poll_get_char(struct uart_port *port)
	933	+{
	934	+ while (!(serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK))
	935	+ cpu_relax();
	936	+
	937	+ return serial_in(port, SPRD_RXD);
	938	+}
	939	+
	940	+static void sprd_poll_put_char(struct uart_port *port, unsigned char ch)
	941	+{
	942	+ while (serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)
	943	+ cpu_relax();
	944	+
	945	+ serial_out(port, SPRD_TXD, ch);
	946	+}
	947	+#endif
494	948
495	949	static const struct uart_ops serial_sprd_ops = {
496	950	.tx_empty = sprd_tx_empty,
..	..	@@ -508,6 +962,12 @@
508	962	.request_port = sprd_request_port,
509	963	.config_port = sprd_config_port,
510	964	.verify_port = sprd_verify_port,
	965	+ .pm = sprd_pm,
	966	+#ifdef CONFIG_CONSOLE_POLL
	967	+ .poll_init = sprd_poll_init,
	968	+ .poll_get_char = sprd_poll_get_char,
	969	+ .poll_put_char = sprd_poll_put_char,
	970	+#endif
511	971	};
512	972
513	973	#ifdef CONFIG_SERIAL_SPRD_CONSOLE
..	..	@@ -521,7 +981,7 @@
521	981	if (--tmout == 0)
522	982	break;
523	983	udelay(1);
524		- } while (status & 0xff00);
	984	+ } while (status & SPRD_TX_FIFO_CNT_MASK);
525	985	}
526	986
527	987	static void sprd_console_putchar(struct uart_port *port, int ch)
..	..	@@ -531,7 +991,7 @@
531	991	}
532	992
533	993	static void sprd_console_write(struct console co, const char s,
534		- unsigned int count)
	994	+ unsigned int count)
535	995	{
536	996	struct uart_port *port = &sprd_port[co->index]->port;
537	997	int locked = 1;
..	..	@@ -553,9 +1013,9 @@
553	1013	spin_unlock_irqrestore(&port->lock, flags);
554	1014	}
555	1015
556		-static int __init sprd_console_setup(struct console co, char options)
	1016	+static int sprd_console_setup(struct console co, char options)
557	1017	{
558		- struct uart_port *port;
	1018	+ struct sprd_uart_port *sprd_uart_port;
559	1019	int baud = 115200;
560	1020	int bits = 8;
561	1021	int parity = 'n';
..	..	@@ -564,15 +1024,17 @@
564	1024	if (co->index >= UART_NR_MAX \|\| co->index < 0)
565	1025	co->index = 0;
566	1026
567		- port = &sprd_port[co->index]->port;
568		- if (port == NULL) {
	1027	+ sprd_uart_port = sprd_port[co->index];
	1028	+ if (!sprd_uart_port \|\| !sprd_uart_port->port.membase) {
569	1029	pr_info("serial port %d not yet initialized\n", co->index);
570	1030	return -ENODEV;
571	1031	}
	1032	+
572	1033	if (options)
573	1034	uart_parse_options(options, &baud, &parity, &bits, &flow);
574	1035
575		- return uart_set_options(port, co, baud, parity, bits, flow);
	1036	+ return uart_set_options(&sprd_uart_port->port, co, baud,
	1037	+ parity, bits, flow);
576	1038	}
577	1039
578	1040	static struct uart_driver sprd_uart_driver;
..	..	@@ -586,6 +1048,13 @@
586	1048	.data = &sprd_uart_driver,
587	1049	};
588	1050
	1051	+static int __init sprd_serial_console_init(void)
	1052	+{
	1053	+ register_console(&sprd_console);
	1054	+ return 0;
	1055	+}
	1056	+console_initcall(sprd_serial_console_init);
	1057	+
589	1058	#define SPRD_CONSOLE (&sprd_console)
590	1059
591	1060	/* Support for earlycon */
..	..	@@ -594,23 +1063,21 @@
594	1063	unsigned int timeout = SPRD_TIMEOUT;
595	1064
596	1065	while (timeout-- &&
597		- !(readl(port->membase + SPRD_LSR) & SPRD_LSR_TX_OVER))
	1066	+ !(readl(port->membase + SPRD_LSR) & SPRD_LSR_TX_OVER))
598	1067	cpu_relax();
599	1068
600	1069	writeb(c, port->membase + SPRD_TXD);
601	1070	}
602	1071
603		-static void sprd_early_write(struct console con, const char s,
604		- unsigned n)
	1072	+static void sprd_early_write(struct console con, const char s, unsigned int n)
605	1073	{
606	1074	struct earlycon_device *dev = con->data;
607	1075
608	1076	uart_console_write(&dev->port, s, n, sprd_putc);
609	1077	}
610	1078
611		-static int __init sprd_early_console_setup(
612		- struct earlycon_device *device,
613		- const char *opt)
	1079	+static int __init sprd_early_console_setup(struct earlycon_device *device,
	1080	+ const char *opt)
614	1081	{
615	1082	if (!device->port.membase)
616	1083	return -ENODEV;
..	..	@@ -635,29 +1102,6 @@
635	1102	.cons = SPRD_CONSOLE,
636	1103	};
637	1104
638		-static int sprd_probe_dt_alias(int index, struct device *dev)
639		-{
640		- struct device_node *np;
641		- int ret = index;
642		-
643		- if (!IS_ENABLED(CONFIG_OF))
644		- return ret;
645		-
646		- np = dev->of_node;
647		- if (!np)
648		- return ret;
649		-
650		- ret = of_alias_get_id(np, "serial");
651		- if (ret < 0)
652		- ret = index;
653		- else if (ret >= ARRAY_SIZE(sprd_port) \|\| sprd_port[ret] != NULL) {
654		- dev_warn(dev, "requested serial port %d not available.\n", ret);
655		- ret = index;
656		- }
657		-
658		- return ret;
659		-}
660		-
661	1105	static int sprd_remove(struct platform_device *dev)
662	1106	{
663	1107	struct sprd_uart_port *sup = platform_get_drvdata(dev);
..	..	@@ -665,6 +1109,7 @@
665	1109	if (sup) {
666	1110	uart_remove_one_port(&sprd_uart_driver, &sup->port);
667	1111	sprd_port[sup->port.line] = NULL;
	1112	+ sprd_rx_free_buf(sup);
668	1113	sprd_ports_num--;
669	1114	}
670	1115
..	..	@@ -674,30 +1119,79 @@
674	1119	return 0;
675	1120	}
676	1121
	1122	+static bool sprd_uart_is_console(struct uart_port *uport)
	1123	+{
	1124	+ struct console *cons = sprd_uart_driver.cons;
	1125	+
	1126	+ if ((cons && cons->index >= 0 && cons->index == uport->line) \|\|
	1127	+ of_console_check(uport->dev->of_node, SPRD_TTY_NAME, uport->line))
	1128	+ return true;
	1129	+
	1130	+ return false;
	1131	+}
	1132	+
	1133	+static int sprd_clk_init(struct uart_port *uport)
	1134	+{
	1135	+ struct clk clk_uart, clk_parent;
	1136	+ struct sprd_uart_port *u = container_of(uport, struct sprd_uart_port, port);
	1137	+
	1138	+ clk_uart = devm_clk_get(uport->dev, "uart");
	1139	+ if (IS_ERR(clk_uart)) {
	1140	+ dev_warn(uport->dev, "uart%d can't get uart clock\n",
	1141	+ uport->line);
	1142	+ clk_uart = NULL;
	1143	+ }
	1144	+
	1145	+ clk_parent = devm_clk_get(uport->dev, "source");
	1146	+ if (IS_ERR(clk_parent)) {
	1147	+ dev_warn(uport->dev, "uart%d can't get source clock\n",
	1148	+ uport->line);
	1149	+ clk_parent = NULL;
	1150	+ }
	1151	+
	1152	+ if (!clk_uart \|\| clk_set_parent(clk_uart, clk_parent))
	1153	+ uport->uartclk = SPRD_DEFAULT_SOURCE_CLK;
	1154	+ else
	1155	+ uport->uartclk = clk_get_rate(clk_uart);
	1156	+
	1157	+ u->clk = devm_clk_get(uport->dev, "enable");
	1158	+ if (IS_ERR(u->clk)) {
	1159	+ if (PTR_ERR(u->clk) == -EPROBE_DEFER)
	1160	+ return -EPROBE_DEFER;
	1161	+
	1162	+ dev_warn(uport->dev, "uart%d can't get enable clock\n",
	1163	+ uport->line);
	1164	+
	1165	+ /* To keep console alive even if the error occurred */
	1166	+ if (!sprd_uart_is_console(uport))
	1167	+ return PTR_ERR(u->clk);
	1168	+
	1169	+ u->clk = NULL;
	1170	+ }
	1171	+
	1172	+ return 0;
	1173	+}
	1174	+
677	1175	static int sprd_probe(struct platform_device *pdev)
678	1176	{
679	1177	struct resource *res;
680	1178	struct uart_port *up;
681		- struct clk *clk;
	1179	+ struct sprd_uart_port *sport;
682	1180	int irq;
683	1181	int index;
684	1182	int ret;
685	1183
686		- for (index = 0; index < ARRAY_SIZE(sprd_port); index++)
687		- if (sprd_port[index] == NULL)
688		- break;
	1184	+ index = of_alias_get_id(pdev->dev.of_node, "serial");
	1185	+ if (index < 0 \|\| index >= UART_NR_MAX) {
	1186	+ dev_err(&pdev->dev, "got a wrong serial alias id %d\n", index);
	1187	+ return -EINVAL;
	1188	+ }
689	1189
690		- if (index == ARRAY_SIZE(sprd_port))
691		- return -EBUSY;
692		-
693		- index = sprd_probe_dt_alias(index, &pdev->dev);
694		-
695		- sprd_port[index] = devm_kzalloc(&pdev->dev,
696		- sizeof(*sprd_port[index]), GFP_KERNEL);
697		- if (!sprd_port[index])
	1190	+ sport = devm_kzalloc(&pdev->dev, sizeof(*sport), GFP_KERNEL);
	1191	+ if (!sport)
698	1192	return -ENOMEM;
699	1193
700		- up = &sprd_port[index]->port;
	1194	+ up = &sport->port;
701	1195	up->dev = &pdev->dev;
702	1196	up->line = index;
703	1197	up->type = PORT_SPRD;
..	..	@@ -706,45 +1200,57 @@
706	1200	up->fifosize = SPRD_FIFO_SIZE;
707	1201	up->ops = &serial_sprd_ops;
708	1202	up->flags = UPF_BOOT_AUTOCONF;
	1203	+ up->has_sysrq = IS_ENABLED(CONFIG_SERIAL_SPRD_CONSOLE);
709	1204
710		- clk = devm_clk_get(&pdev->dev, NULL);
711		- if (!IS_ERR_OR_NULL(clk))
712		- up->uartclk = clk_get_rate(clk);
	1205	+ ret = sprd_clk_init(up);
	1206	+ if (ret)
	1207	+ return ret;
713	1208
714	1209	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
715		- if (!res) {
716		- dev_err(&pdev->dev, "not provide mem resource\n");
717		- return -ENODEV;
718		- }
719		- up->mapbase = res->start;
720	1210	up->membase = devm_ioremap_resource(&pdev->dev, res);
721	1211	if (IS_ERR(up->membase))
722	1212	return PTR_ERR(up->membase);
723	1213
	1214	+ up->mapbase = res->start;
	1215	+
724	1216	irq = platform_get_irq(pdev, 0);
725		- if (irq < 0) {
726		- dev_err(&pdev->dev, "not provide irq resource: %d\n", irq);
	1217	+ if (irq < 0)
727	1218	return irq;
728		- }
729	1219	up->irq = irq;
	1220	+
	1221	+ /*
	1222	+ * Allocate one dma buffer to prepare for receive transfer, in case
	1223	+ * memory allocation failure at runtime.
	1224	+ */
	1225	+ ret = sprd_rx_alloc_buf(sport);
	1226	+ if (ret)
	1227	+ return ret;
730	1228
731	1229	if (!sprd_ports_num) {
732	1230	ret = uart_register_driver(&sprd_uart_driver);
733	1231	if (ret < 0) {
734	1232	pr_err("Failed to register SPRD-UART driver\n");
735		- return ret;
	1233	+ goto free_rx_buf;
736	1234	}
737	1235	}
	1236	+
738	1237	sprd_ports_num++;
	1238	+ sprd_port[index] = sport;
739	1239
740	1240	ret = uart_add_one_port(&sprd_uart_driver, up);
741		- if (ret) {
742		- sprd_port[index] = NULL;
743		- sprd_remove(pdev);
744		- }
	1241	+ if (ret)
	1242	+ goto clean_port;
745	1243
746	1244	platform_set_drvdata(pdev, up);
747	1245
	1246	+ return 0;
	1247	+
	1248	+clean_port:
	1249	+ sprd_port[index] = NULL;
	1250	+ if (--sprd_ports_num == 0)
	1251	+ uart_unregister_driver(&sprd_uart_driver);
	1252	+free_rx_buf:
	1253	+ sprd_rx_free_buf(sport);
748	1254	return ret;
749	1255	}
750	1256