~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* Driver for Broadcom BCM2835 SPI Controllers
3	4	*
..	..	@@ -8,21 +9,11 @@
8	9	* This driver is inspired by:
9	10	* spi-ath79.c, Copyright (C) 2009-2011 Gabor Juhos <juhosg@openwrt.org>
10	11	* spi-atmel.c, Copyright (C) 2006 Atmel Corporation
11		- *
12		- * This program is free software; you can redistribute it and/or modify
13		- * it under the terms of the GNU General Public License as published by
14		- * the Free Software Foundation; either version 2 of the License, or
15		- * (at your option) any later version.
16		- *
17		- * This program is distributed in the hope that it will be useful,
18		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
19		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20		- * GNU General Public License for more details.
21	12	*/
22	13
23		-#include <asm/page.h>
24	14	#include <linux/clk.h>
25	15	#include <linux/completion.h>
	16	+#include <linux/debugfs.h>
26	17	#include <linux/delay.h>
27	18	#include <linux/dma-mapping.h>
28	19	#include <linux/dmaengine.h>
..	..	@@ -34,7 +25,9 @@
34	25	#include <linux/of.h>
35	26	#include <linux/of_address.h>
36	27	#include <linux/of_device.h>
37		-#include <linux/of_gpio.h>
	28	+#include <linux/gpio/consumer.h>
	29	+#include <linux/gpio/machine.h> /* FIXME: using chip internals */
	30	+#include <linux/gpio/driver.h> /* FIXME: using chip internals */
38	31	#include <linux/of_irq.h>
39	32	#include <linux/spi/spi.h>
40	33
..	..	@@ -72,31 +65,142 @@
72	65	#define BCM2835_SPI_CS_CS_10 0x00000002
73	66	#define BCM2835_SPI_CS_CS_01 0x00000001
74	67
75		-#define BCM2835_SPI_POLLING_LIMIT_US 30
76		-#define BCM2835_SPI_POLLING_JIFFIES 2
	68	+#define BCM2835_SPI_FIFO_SIZE 64
	69	+#define BCM2835_SPI_FIFO_SIZE_3_4 48
77	70	#define BCM2835_SPI_DMA_MIN_LENGTH 96
	71	+#define BCM2835_SPI_NUM_CS 24 /* raise as necessary */
78	72	#define BCM2835_SPI_MODE_BITS (SPI_CPOL \| SPI_CPHA \| SPI_CS_HIGH \
79	73	\| SPI_NO_CS \| SPI_3WIRE)
80	74
81	75	#define DRV_NAME "spi-bcm2835"
82	76
	77	+/* define polling limits */
	78	+static unsigned int polling_limit_us = 30;
	79	+module_param(polling_limit_us, uint, 0664);
	80	+MODULE_PARM_DESC(polling_limit_us,
	81	+ "time in us to run a transfer in polling mode\n");
	82	+
	83	+/**
	84	+ * struct bcm2835_spi - BCM2835 SPI controller
	85	+ * @regs: base address of register map
	86	+ * @clk: core clock, divided to calculate serial clock
	87	+ * @clk_hz: core clock cached speed
	88	+ * @irq: interrupt, signals TX FIFO empty or RX FIFO ¾ full
	89	+ * @tfr: SPI transfer currently processed
	90	+ * @ctlr: SPI controller reverse lookup
	91	+ * @tx_buf: pointer whence next transmitted byte is read
	92	+ * @rx_buf: pointer where next received byte is written
	93	+ * @tx_len: remaining bytes to transmit
	94	+ * @rx_len: remaining bytes to receive
	95	+ * @tx_prologue: bytes transmitted without DMA if first TX sglist entry's
	96	+ * length is not a multiple of 4 (to overcome hardware limitation)
	97	+ * @rx_prologue: bytes received without DMA if first RX sglist entry's
	98	+ * length is not a multiple of 4 (to overcome hardware limitation)
	99	+ * @tx_spillover: whether @tx_prologue spills over to second TX sglist entry
	100	+ * @prepare_cs: precalculated CS register value for ->prepare_message()
	101	+ * (uses slave-specific clock polarity and phase settings)
	102	+ * @debugfs_dir: the debugfs directory - neede to remove debugfs when
	103	+ * unloading the module
	104	+ * @count_transfer_polling: count of how often polling mode is used
	105	+ * @count_transfer_irq: count of how often interrupt mode is used
	106	+ * @count_transfer_irq_after_polling: count of how often we fall back to
	107	+ * interrupt mode after starting in polling mode.
	108	+ * These are counted as well in @count_transfer_polling and
	109	+ * @count_transfer_irq
	110	+ * @count_transfer_dma: count how often dma mode is used
	111	+ * @chip_select: SPI slave currently selected
	112	+ * (used by bcm2835_spi_dma_tx_done() to write @clear_rx_cs)
	113	+ * @tx_dma_active: whether a TX DMA descriptor is in progress
	114	+ * @rx_dma_active: whether a RX DMA descriptor is in progress
	115	+ * (used by bcm2835_spi_dma_tx_done() to handle a race)
	116	+ * @fill_tx_desc: preallocated TX DMA descriptor used for RX-only transfers
	117	+ * (cyclically copies from zero page to TX FIFO)
	118	+ * @fill_tx_addr: bus address of zero page
	119	+ * @clear_rx_desc: preallocated RX DMA descriptor used for TX-only transfers
	120	+ * (cyclically clears RX FIFO by writing @clear_rx_cs to CS register)
	121	+ * @clear_rx_addr: bus address of @clear_rx_cs
	122	+ * @clear_rx_cs: precalculated CS register value to clear RX FIFO
	123	+ * (uses slave-specific clock polarity and phase settings)
	124	+ */
83	125	struct bcm2835_spi {
84	126	void __iomem *regs;
85	127	struct clk *clk;
	128	+ unsigned long clk_hz;
86	129	int irq;
	130	+ struct spi_transfer *tfr;
	131	+ struct spi_controller *ctlr;
87	132	const u8 *tx_buf;
88	133	u8 *rx_buf;
89	134	int tx_len;
90	135	int rx_len;
91		- unsigned int dma_pending;
	136	+ int tx_prologue;
	137	+ int rx_prologue;
	138	+ unsigned int tx_spillover;
	139	+ u32 prepare_cs[BCM2835_SPI_NUM_CS];
	140	+
	141	+ struct dentry *debugfs_dir;
	142	+ u64 count_transfer_polling;
	143	+ u64 count_transfer_irq;
	144	+ u64 count_transfer_irq_after_polling;
	145	+ u64 count_transfer_dma;
	146	+
	147	+ u8 chip_select;
	148	+ unsigned int tx_dma_active;
	149	+ unsigned int rx_dma_active;
	150	+ struct dma_async_tx_descriptor *fill_tx_desc;
	151	+ dma_addr_t fill_tx_addr;
	152	+ struct dma_async_tx_descriptor *clear_rx_desc[BCM2835_SPI_NUM_CS];
	153	+ dma_addr_t clear_rx_addr;
	154	+ u32 clear_rx_cs[BCM2835_SPI_NUM_CS] ____cacheline_aligned;
92	155	};
93	156
94		-static inline u32 bcm2835_rd(struct bcm2835_spi *bs, unsigned reg)
	157	+#if defined(CONFIG_DEBUG_FS)
	158	+static void bcm2835_debugfs_create(struct bcm2835_spi *bs,
	159	+ const char *dname)
	160	+{
	161	+ char name[64];
	162	+ struct dentry *dir;
	163	+
	164	+ /* get full name */
	165	+ snprintf(name, sizeof(name), "spi-bcm2835-%s", dname);
	166	+
	167	+ /* the base directory */
	168	+ dir = debugfs_create_dir(name, NULL);
	169	+ bs->debugfs_dir = dir;
	170	+
	171	+ /* the counters */
	172	+ debugfs_create_u64("count_transfer_polling", 0444, dir,
	173	+ &bs->count_transfer_polling);
	174	+ debugfs_create_u64("count_transfer_irq", 0444, dir,
	175	+ &bs->count_transfer_irq);
	176	+ debugfs_create_u64("count_transfer_irq_after_polling", 0444, dir,
	177	+ &bs->count_transfer_irq_after_polling);
	178	+ debugfs_create_u64("count_transfer_dma", 0444, dir,
	179	+ &bs->count_transfer_dma);
	180	+}
	181	+
	182	+static void bcm2835_debugfs_remove(struct bcm2835_spi *bs)
	183	+{
	184	+ debugfs_remove_recursive(bs->debugfs_dir);
	185	+ bs->debugfs_dir = NULL;
	186	+}
	187	+#else
	188	+static void bcm2835_debugfs_create(struct bcm2835_spi *bs,
	189	+ const char *dname)
	190	+{
	191	+}
	192	+
	193	+static void bcm2835_debugfs_remove(struct bcm2835_spi *bs)
	194	+{
	195	+}
	196	+#endif /* CONFIG_DEBUG_FS */
	197	+
	198	+static inline u32 bcm2835_rd(struct bcm2835_spi *bs, unsigned int reg)
95	199	{
96	200	return readl(bs->regs + reg);
97	201	}
98	202
99		-static inline void bcm2835_wr(struct bcm2835_spi *bs, unsigned reg, u32 val)
	203	+static inline void bcm2835_wr(struct bcm2835_spi *bs, unsigned int reg, u32 val)
100	204	{
101	205	writel(val, bs->regs + reg);
102	206	}
..	..	@@ -126,9 +230,115 @@
126	230	}
127	231	}
128	232
129		-static void bcm2835_spi_reset_hw(struct spi_master *master)
	233	+/**
	234	+ * bcm2835_rd_fifo_count() - blindly read exactly @count bytes from RX FIFO
	235	+ * @bs: BCM2835 SPI controller
	236	+ * @count: bytes to read from RX FIFO
	237	+ *
	238	+ * The caller must ensure that @bs->rx_len is greater than or equal to @count,
	239	+ * that the RX FIFO contains at least @count bytes and that the DMA Enable flag
	240	+ * in the CS register is set (such that a read from the FIFO register receives
	241	+ * 32-bit instead of just 8-bit). Moreover @bs->rx_buf must not be %NULL.
	242	+ */
	243	+static inline void bcm2835_rd_fifo_count(struct bcm2835_spi *bs, int count)
130	244	{
131		- struct bcm2835_spi *bs = spi_master_get_devdata(master);
	245	+ u32 val;
	246	+ int len;
	247	+
	248	+ bs->rx_len -= count;
	249	+
	250	+ do {
	251	+ val = bcm2835_rd(bs, BCM2835_SPI_FIFO);
	252	+ len = min(count, 4);
	253	+ memcpy(bs->rx_buf, &val, len);
	254	+ bs->rx_buf += len;
	255	+ count -= 4;
	256	+ } while (count > 0);
	257	+}
	258	+
	259	+/**
	260	+ * bcm2835_wr_fifo_count() - blindly write exactly @count bytes to TX FIFO
	261	+ * @bs: BCM2835 SPI controller
	262	+ * @count: bytes to write to TX FIFO
	263	+ *
	264	+ * The caller must ensure that @bs->tx_len is greater than or equal to @count,
	265	+ * that the TX FIFO can accommodate @count bytes and that the DMA Enable flag
	266	+ * in the CS register is set (such that a write to the FIFO register transmits
	267	+ * 32-bit instead of just 8-bit).
	268	+ */
	269	+static inline void bcm2835_wr_fifo_count(struct bcm2835_spi *bs, int count)
	270	+{
	271	+ u32 val;
	272	+ int len;
	273	+
	274	+ bs->tx_len -= count;
	275	+
	276	+ do {
	277	+ if (bs->tx_buf) {
	278	+ len = min(count, 4);
	279	+ memcpy(&val, bs->tx_buf, len);
	280	+ bs->tx_buf += len;
	281	+ } else {
	282	+ val = 0;
	283	+ }
	284	+ bcm2835_wr(bs, BCM2835_SPI_FIFO, val);
	285	+ count -= 4;
	286	+ } while (count > 0);
	287	+}
	288	+
	289	+/**
	290	+ * bcm2835_wait_tx_fifo_empty() - busy-wait for TX FIFO to empty
	291	+ * @bs: BCM2835 SPI controller
	292	+ *
	293	+ * The caller must ensure that the RX FIFO can accommodate as many bytes
	294	+ * as have been written to the TX FIFO: Transmission is halted once the
	295	+ * RX FIFO is full, causing this function to spin forever.
	296	+ */
	297	+static inline void bcm2835_wait_tx_fifo_empty(struct bcm2835_spi *bs)
	298	+{
	299	+ while (!(bcm2835_rd(bs, BCM2835_SPI_CS) & BCM2835_SPI_CS_DONE))
	300	+ cpu_relax();
	301	+}
	302	+
	303	+/**
	304	+ * bcm2835_rd_fifo_blind() - blindly read up to @count bytes from RX FIFO
	305	+ * @bs: BCM2835 SPI controller
	306	+ * @count: bytes available for reading in RX FIFO
	307	+ */
	308	+static inline void bcm2835_rd_fifo_blind(struct bcm2835_spi *bs, int count)
	309	+{
	310	+ u8 val;
	311	+
	312	+ count = min(count, bs->rx_len);
	313	+ bs->rx_len -= count;
	314	+
	315	+ do {
	316	+ val = bcm2835_rd(bs, BCM2835_SPI_FIFO);
	317	+ if (bs->rx_buf)
	318	+ *bs->rx_buf++ = val;
	319	+ } while (--count);
	320	+}
	321	+
	322	+/**
	323	+ * bcm2835_wr_fifo_blind() - blindly write up to @count bytes to TX FIFO
	324	+ * @bs: BCM2835 SPI controller
	325	+ * @count: bytes available for writing in TX FIFO
	326	+ */
	327	+static inline void bcm2835_wr_fifo_blind(struct bcm2835_spi *bs, int count)
	328	+{
	329	+ u8 val;
	330	+
	331	+ count = min(count, bs->tx_len);
	332	+ bs->tx_len -= count;
	333	+
	334	+ do {
	335	+ val = bs->tx_buf ? *bs->tx_buf++ : 0;
	336	+ bcm2835_wr(bs, BCM2835_SPI_FIFO, val);
	337	+ } while (--count);
	338	+}
	339	+
	340	+static void bcm2835_spi_reset_hw(struct bcm2835_spi *bs)
	341	+{
132	342	u32 cs = bcm2835_rd(bs, BCM2835_SPI_CS);
133	343
134	344	/* Disable SPI interrupts and transfer */
..	..	@@ -136,6 +346,13 @@
136	346	BCM2835_SPI_CS_INTD \|
137	347	BCM2835_SPI_CS_DMAEN \|
138	348	BCM2835_SPI_CS_TA);
	349	+ /*
	350	+ * Transmission sometimes breaks unless the DONE bit is written at the
	351	+ * end of every transfer. The spec says it's a RO bit. Either the
	352	+ * spec is wrong and the bit is actually of type RW1C, or it's a
	353	+ * hardware erratum.
	354	+ */
	355	+ cs \|= BCM2835_SPI_CS_DONE;
139	356	/* and reset RX/TX FIFOS */
140	357	cs \|= BCM2835_SPI_CS_CLEAR_RX \| BCM2835_SPI_CS_CLEAR_TX;
141	358
..	..	@@ -147,8 +364,20 @@
147	364
148	365	static irqreturn_t bcm2835_spi_interrupt(int irq, void *dev_id)
149	366	{
150		- struct spi_master *master = dev_id;
151		- struct bcm2835_spi *bs = spi_master_get_devdata(master);
	367	+ struct bcm2835_spi *bs = dev_id;
	368	+ u32 cs = bcm2835_rd(bs, BCM2835_SPI_CS);
	369	+
	370	+ /*
	371	+ * An interrupt is signaled either if DONE is set (TX FIFO empty)
	372	+ * or if RXR is set (RX FIFO >= ¾ full).
	373	+ */
	374	+ if (cs & BCM2835_SPI_CS_RXF)
	375	+ bcm2835_rd_fifo_blind(bs, BCM2835_SPI_FIFO_SIZE);
	376	+ else if (cs & BCM2835_SPI_CS_RXR)
	377	+ bcm2835_rd_fifo_blind(bs, BCM2835_SPI_FIFO_SIZE_3_4);
	378	+
	379	+ if (bs->tx_len && cs & BCM2835_SPI_CS_DONE)
	380	+ bcm2835_wr_fifo_blind(bs, BCM2835_SPI_FIFO_SIZE);
152	381
153	382	/* Read as many bytes as possible from FIFO */
154	383	bcm2835_rd_fifo(bs);
..	..	@@ -157,43 +386,36 @@
157	386
158	387	if (!bs->rx_len) {
159	388	/* Transfer complete - reset SPI HW */
160		- bcm2835_spi_reset_hw(master);
	389	+ bcm2835_spi_reset_hw(bs);
161	390	/* wake up the framework */
162		- complete(&master->xfer_completion);
	391	+ complete(&bs->ctlr->xfer_completion);
163	392	}
164	393
165	394	return IRQ_HANDLED;
166	395	}
167	396
168		-static int bcm2835_spi_transfer_one_irq(struct spi_master *master,
	397	+static int bcm2835_spi_transfer_one_irq(struct spi_controller *ctlr,
169	398	struct spi_device *spi,
170	399	struct spi_transfer *tfr,
171		- u32 cs)
	400	+ u32 cs, bool fifo_empty)
172	401	{
173		- struct bcm2835_spi *bs = spi_master_get_devdata(master);
	402	+ struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
174	403
175		- /* fill in fifo if we have gpio-cs
176		- * note that there have been rare events where the native-CS
177		- * flapped for <1us which may change the behaviour
178		- * with gpio-cs this does not happen, so it is implemented
179		- * only for this case
180		- */
181		- if (gpio_is_valid(spi->cs_gpio)) {
182		- /* enable HW block, but without interrupts enabled
183		- * this would triggern an immediate interrupt
184		- */
185		- bcm2835_wr(bs, BCM2835_SPI_CS,
186		- cs \| BCM2835_SPI_CS_TA);
187		- /* fill in tx fifo as much as possible */
188		- bcm2835_wr_fifo(bs);
189		- }
	404	+ /* update usage statistics */
	405	+ bs->count_transfer_irq++;
190	406
191	407	/*
192		- * Enable the HW block. This will immediately trigger a DONE (TX
193		- * empty) interrupt, upon which we will fill the TX FIFO with the
194		- * first TX bytes. Pre-filling the TX FIFO here to avoid the
195		- * interrupt doesn't work:-(
	408	+ * Enable HW block, but with interrupts still disabled.
	409	+ * Otherwise the empty TX FIFO would immediately trigger an interrupt.
196	410	*/
	411	+ bcm2835_wr(bs, BCM2835_SPI_CS, cs \| BCM2835_SPI_CS_TA);
	412	+
	413	+ /* fill TX FIFO as much as possible */
	414	+ if (fifo_empty)
	415	+ bcm2835_wr_fifo_blind(bs, BCM2835_SPI_FIFO_SIZE);
	416	+ bcm2835_wr_fifo(bs);
	417	+
	418	+ /* enable interrupts */
197	419	cs \|= BCM2835_SPI_CS_INTR \| BCM2835_SPI_CS_INTD \| BCM2835_SPI_CS_TA;
198	420	bcm2835_wr(bs, BCM2835_SPI_CS, cs);
199	421
..	..	@@ -201,47 +423,243 @@
201	423	return 1;
202	424	}
203	425
204		-/*
205		- * DMA support
	426	+/**
	427	+ * bcm2835_spi_transfer_prologue() - transfer first few bytes without DMA
	428	+ * @ctlr: SPI master controller
	429	+ * @tfr: SPI transfer
	430	+ * @bs: BCM2835 SPI controller
	431	+ * @cs: CS register
206	432	*
207		- * this implementation has currently a few issues in so far as it does
208		- * not work arrount limitations of the HW.
	433	+ * A limitation in DMA mode is that the FIFO must be accessed in 4 byte chunks.
	434	+ * Only the final write access is permitted to transmit less than 4 bytes, the
	435	+ * SPI controller deduces its intended size from the DLEN register.
209	436	*
210		- * the main one being that DMA transfers are limited to 16 bit
211		- * (so 0 to 65535 bytes) by the SPI HW due to BCM2835_SPI_DLEN
	437	+ * If a TX or RX sglist contains multiple entries, one per page, and the first
	438	+ * entry starts in the middle of a page, that first entry's length may not be
	439	+ * a multiple of 4. Subsequent entries are fine because they span an entire
	440	+ * page, hence do have a length that's a multiple of 4.
212	441	*
213		- * also we currently assume that the scatter-gather fragments are
214		- * all multiple of 4 (except the last) - otherwise we would need
215		- * to reset the FIFO before subsequent transfers...
216		- * this also means that tx/rx transfers sg's need to be of equal size!
	442	+ * This cannot happen with kmalloc'ed buffers (which is what most clients use)
	443	+ * because they are contiguous in physical memory and therefore not split on
	444	+ * page boundaries by spi_map_buf(). But it can happen with vmalloc'ed
	445	+ * buffers.
217	446	*
218		- * there may be a few more border-cases we may need to address as well
219		- * but unfortunately this would mean splitting up the scatter-gather
220		- * list making it slightly unpractical...
	447	+ * The DMA engine is incapable of combining sglist entries into a continuous
	448	+ * stream of 4 byte chunks, it treats every entry separately: A TX entry is
	449	+ * rounded up a to a multiple of 4 bytes by transmitting surplus bytes, an RX
	450	+ * entry is rounded up by throwing away received bytes.
	451	+ *
	452	+ * Overcome this limitation by transferring the first few bytes without DMA:
	453	+ * E.g. if the first TX sglist entry's length is 23 and the first RX's is 42,
	454	+ * write 3 bytes to the TX FIFO but read only 2 bytes from the RX FIFO.
	455	+ * The residue of 1 byte in the RX FIFO is picked up by DMA. Together with
	456	+ * the rest of the first RX sglist entry it makes up a multiple of 4 bytes.
	457	+ *
	458	+ * Should the RX prologue be larger, say, 3 vis-à-vis a TX prologue of 1,
	459	+ * write 1 + 4 = 5 bytes to the TX FIFO and read 3 bytes from the RX FIFO.
	460	+ * Caution, the additional 4 bytes spill over to the second TX sglist entry
	461	+ * if the length of the first is exactly 1.
	462	+ *
	463	+ * At most 6 bytes are written and at most 3 bytes read. Do we know the
	464	+ * transfer has this many bytes? Yes, see BCM2835_SPI_DMA_MIN_LENGTH.
	465	+ *
	466	+ * The FIFO is normally accessed with 8-bit width by the CPU and 32-bit width
	467	+ * by the DMA engine. Toggling the DMA Enable flag in the CS register switches
	468	+ * the width but also garbles the FIFO's contents. The prologue must therefore
	469	+ * be transmitted in 32-bit width to ensure that the following DMA transfer can
	470	+ * pick up the residue in the RX FIFO in ungarbled form.
221	471	*/
222		-static void bcm2835_spi_dma_done(void *data)
	472	+static void bcm2835_spi_transfer_prologue(struct spi_controller *ctlr,
	473	+ struct spi_transfer *tfr,
	474	+ struct bcm2835_spi *bs,
	475	+ u32 cs)
223	476	{
224		- struct spi_master *master = data;
225		- struct bcm2835_spi *bs = spi_master_get_devdata(master);
	477	+ int tx_remaining;
226	478
227		- /* reset fifo and HW */
228		- bcm2835_spi_reset_hw(master);
	479	+ bs->tfr = tfr;
	480	+ bs->tx_prologue = 0;
	481	+ bs->rx_prologue = 0;
	482	+ bs->tx_spillover = false;
229	483
230		- /* and terminate tx-dma as we do not have an irq for it
	484	+ if (bs->tx_buf && !sg_is_last(&tfr->tx_sg.sgl[0]))
	485	+ bs->tx_prologue = sg_dma_len(&tfr->tx_sg.sgl[0]) & 3;
	486	+
	487	+ if (bs->rx_buf && !sg_is_last(&tfr->rx_sg.sgl[0])) {
	488	+ bs->rx_prologue = sg_dma_len(&tfr->rx_sg.sgl[0]) & 3;
	489	+
	490	+ if (bs->rx_prologue > bs->tx_prologue) {
	491	+ if (!bs->tx_buf \|\| sg_is_last(&tfr->tx_sg.sgl[0])) {
	492	+ bs->tx_prologue = bs->rx_prologue;
	493	+ } else {
	494	+ bs->tx_prologue += 4;
	495	+ bs->tx_spillover =
	496	+ !(sg_dma_len(&tfr->tx_sg.sgl[0]) & ~3);
	497	+ }
	498	+ }
	499	+ }
	500	+
	501	+ /* rx_prologue > 0 implies tx_prologue > 0, so check only the latter */
	502	+ if (!bs->tx_prologue)
	503	+ return;
	504	+
	505	+ /* Write and read RX prologue. Adjust first entry in RX sglist. */
	506	+ if (bs->rx_prologue) {
	507	+ bcm2835_wr(bs, BCM2835_SPI_DLEN, bs->rx_prologue);
	508	+ bcm2835_wr(bs, BCM2835_SPI_CS, cs \| BCM2835_SPI_CS_TA
	509	+ \| BCM2835_SPI_CS_DMAEN);
	510	+ bcm2835_wr_fifo_count(bs, bs->rx_prologue);
	511	+ bcm2835_wait_tx_fifo_empty(bs);
	512	+ bcm2835_rd_fifo_count(bs, bs->rx_prologue);
	513	+ bcm2835_wr(bs, BCM2835_SPI_CS, cs \| BCM2835_SPI_CS_CLEAR_RX
	514	+ \| BCM2835_SPI_CS_CLEAR_TX
	515	+ \| BCM2835_SPI_CS_DONE);
	516	+
	517	+ dma_sync_single_for_device(ctlr->dma_rx->device->dev,
	518	+ sg_dma_address(&tfr->rx_sg.sgl[0]),
	519	+ bs->rx_prologue, DMA_FROM_DEVICE);
	520	+
	521	+ sg_dma_address(&tfr->rx_sg.sgl[0]) += bs->rx_prologue;
	522	+ sg_dma_len(&tfr->rx_sg.sgl[0]) -= bs->rx_prologue;
	523	+ }
	524	+
	525	+ if (!bs->tx_buf)
	526	+ return;
	527	+
	528	+ /*
	529	+ * Write remaining TX prologue. Adjust first entry in TX sglist.
	530	+ * Also adjust second entry if prologue spills over to it.
	531	+ */
	532	+ tx_remaining = bs->tx_prologue - bs->rx_prologue;
	533	+ if (tx_remaining) {
	534	+ bcm2835_wr(bs, BCM2835_SPI_DLEN, tx_remaining);
	535	+ bcm2835_wr(bs, BCM2835_SPI_CS, cs \| BCM2835_SPI_CS_TA
	536	+ \| BCM2835_SPI_CS_DMAEN);
	537	+ bcm2835_wr_fifo_count(bs, tx_remaining);
	538	+ bcm2835_wait_tx_fifo_empty(bs);
	539	+ bcm2835_wr(bs, BCM2835_SPI_CS, cs \| BCM2835_SPI_CS_CLEAR_TX
	540	+ \| BCM2835_SPI_CS_DONE);
	541	+ }
	542	+
	543	+ if (likely(!bs->tx_spillover)) {
	544	+ sg_dma_address(&tfr->tx_sg.sgl[0]) += bs->tx_prologue;
	545	+ sg_dma_len(&tfr->tx_sg.sgl[0]) -= bs->tx_prologue;
	546	+ } else {
	547	+ sg_dma_len(&tfr->tx_sg.sgl[0]) = 0;
	548	+ sg_dma_address(&tfr->tx_sg.sgl[1]) += 4;
	549	+ sg_dma_len(&tfr->tx_sg.sgl[1]) -= 4;
	550	+ }
	551	+}
	552	+
	553	+/**
	554	+ * bcm2835_spi_undo_prologue() - reconstruct original sglist state
	555	+ * @bs: BCM2835 SPI controller
	556	+ *
	557	+ * Undo changes which were made to an SPI transfer's sglist when transmitting
	558	+ * the prologue. This is necessary to ensure the same memory ranges are
	559	+ * unmapped that were originally mapped.
	560	+ */
	561	+static void bcm2835_spi_undo_prologue(struct bcm2835_spi *bs)
	562	+{
	563	+ struct spi_transfer *tfr = bs->tfr;
	564	+
	565	+ if (!bs->tx_prologue)
	566	+ return;
	567	+
	568	+ if (bs->rx_prologue) {
	569	+ sg_dma_address(&tfr->rx_sg.sgl[0]) -= bs->rx_prologue;
	570	+ sg_dma_len(&tfr->rx_sg.sgl[0]) += bs->rx_prologue;
	571	+ }
	572	+
	573	+ if (!bs->tx_buf)
	574	+ goto out;
	575	+
	576	+ if (likely(!bs->tx_spillover)) {
	577	+ sg_dma_address(&tfr->tx_sg.sgl[0]) -= bs->tx_prologue;
	578	+ sg_dma_len(&tfr->tx_sg.sgl[0]) += bs->tx_prologue;
	579	+ } else {
	580	+ sg_dma_len(&tfr->tx_sg.sgl[0]) = bs->tx_prologue - 4;
	581	+ sg_dma_address(&tfr->tx_sg.sgl[1]) -= 4;
	582	+ sg_dma_len(&tfr->tx_sg.sgl[1]) += 4;
	583	+ }
	584	+out:
	585	+ bs->tx_prologue = 0;
	586	+}
	587	+
	588	+/**
	589	+ * bcm2835_spi_dma_rx_done() - callback for DMA RX channel
	590	+ * @data: SPI master controller
	591	+ *
	592	+ * Used for bidirectional and RX-only transfers.
	593	+ */
	594	+static void bcm2835_spi_dma_rx_done(void *data)
	595	+{
	596	+ struct spi_controller *ctlr = data;
	597	+ struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
	598	+
	599	+ /* terminate tx-dma as we do not have an irq for it
231	600	* because when the rx dma will terminate and this callback
232	601	* is called the tx-dma must have finished - can't get to this
233	602	* situation otherwise...
234	603	*/
235		- if (cmpxchg(&bs->dma_pending, true, false)) {
236		- dmaengine_terminate_all(master->dma_tx);
237		- }
	604	+ dmaengine_terminate_async(ctlr->dma_tx);
	605	+ bs->tx_dma_active = false;
	606	+ bs->rx_dma_active = false;
	607	+ bcm2835_spi_undo_prologue(bs);
	608	+
	609	+ /* reset fifo and HW */
	610	+ bcm2835_spi_reset_hw(bs);
238	611
239	612	/* and mark as completed */;
240		- complete(&master->xfer_completion);
	613	+ complete(&ctlr->xfer_completion);
241	614	}
242	615
243		-static int bcm2835_spi_prepare_sg(struct spi_master *master,
	616	+/**
	617	+ * bcm2835_spi_dma_tx_done() - callback for DMA TX channel
	618	+ * @data: SPI master controller
	619	+ *
	620	+ * Used for TX-only transfers.
	621	+ */
	622	+static void bcm2835_spi_dma_tx_done(void *data)
	623	+{
	624	+ struct spi_controller *ctlr = data;
	625	+ struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
	626	+
	627	+ /* busy-wait for TX FIFO to empty */
	628	+ while (!(bcm2835_rd(bs, BCM2835_SPI_CS) & BCM2835_SPI_CS_DONE))
	629	+ bcm2835_wr(bs, BCM2835_SPI_CS,
	630	+ bs->clear_rx_cs[bs->chip_select]);
	631	+
	632	+ bs->tx_dma_active = false;
	633	+ smp_wmb();
	634	+
	635	+ /*
	636	+ * In case of a very short transfer, RX DMA may not have been
	637	+ * issued yet. The onus is then on bcm2835_spi_transfer_one_dma()
	638	+ * to terminate it immediately after issuing.
	639	+ */
	640	+ if (cmpxchg(&bs->rx_dma_active, true, false))
	641	+ dmaengine_terminate_async(ctlr->dma_rx);
	642	+
	643	+ bcm2835_spi_undo_prologue(bs);
	644	+ bcm2835_spi_reset_hw(bs);
	645	+ complete(&ctlr->xfer_completion);
	646	+}
	647	+
	648	+/**
	649	+ * bcm2835_spi_prepare_sg() - prepare and submit DMA descriptor for sglist
	650	+ * @ctlr: SPI master controller
	651	+ * @spi: SPI slave
	652	+ * @tfr: SPI transfer
	653	+ * @bs: BCM2835 SPI controller
	654	+ * @is_tx: whether to submit DMA descriptor for TX or RX sglist
	655	+ *
	656	+ * Prepare and submit a DMA descriptor for the TX or RX sglist of @tfr.
	657	+ * Return 0 on success or a negative error number.
	658	+ */
	659	+static int bcm2835_spi_prepare_sg(struct spi_controller *ctlr,
	660	+ struct spi_device *spi,
244	661	struct spi_transfer *tfr,
	662	+ struct bcm2835_spi *bs,
245	663	bool is_tx)
246	664	{
247	665	struct dma_chan *chan;
..	..	@@ -255,14 +673,13 @@
255	673
256	674	if (is_tx) {
257	675	dir = DMA_MEM_TO_DEV;
258		- chan = master->dma_tx;
	676	+ chan = ctlr->dma_tx;
259	677	nents = tfr->tx_sg.nents;
260	678	sgl = tfr->tx_sg.sgl;
261		- flags = 0 /* no tx interrupt */;
262		-
	679	+ flags = tfr->rx_buf ? 0 : DMA_PREP_INTERRUPT;
263	680	} else {
264	681	dir = DMA_DEV_TO_MEM;
265		- chan = master->dma_rx;
	682	+ chan = ctlr->dma_rx;
266	683	nents = tfr->rx_sg.nents;
267	684	sgl = tfr->rx_sg.sgl;
268	685	flags = DMA_PREP_INTERRUPT;
..	..	@@ -272,10 +689,17 @@
272	689	if (!desc)
273	690	return -EINVAL;
274	691
275		- /* set callback for rx */
	692	+ /*
	693	+ * Completion is signaled by the RX channel for bidirectional and
	694	+ * RX-only transfers; else by the TX channel for TX-only transfers.
	695	+ */
276	696	if (!is_tx) {
277		- desc->callback = bcm2835_spi_dma_done;
278		- desc->callback_param = master;
	697	+ desc->callback = bcm2835_spi_dma_rx_done;
	698	+ desc->callback_param = ctlr;
	699	+ } else if (!tfr->rx_buf) {
	700	+ desc->callback = bcm2835_spi_dma_tx_done;
	701	+ desc->callback_param = ctlr;
	702	+ bs->chip_select = spi->chip_select;
279	703	}
280	704
281	705	/* submit it to DMA-engine */
..	..	@@ -284,205 +708,332 @@
284	708	return dma_submit_error(cookie);
285	709	}
286	710
287		-static inline int bcm2835_check_sg_length(struct sg_table *sgt)
288		-{
289		- int i;
290		- struct scatterlist *sgl;
291		-
292		- /* check that the sg entries are word-sized (except for last) */
293		- for_each_sg(sgt->sgl, sgl, (int)sgt->nents - 1, i) {
294		- if (sg_dma_len(sgl) % 4)
295		- return -EFAULT;
296		- }
297		-
298		- return 0;
299		-}
300		-
301		-static int bcm2835_spi_transfer_one_dma(struct spi_master *master,
	711	+/**
	712	+ * bcm2835_spi_transfer_one_dma() - perform SPI transfer using DMA engine
	713	+ * @ctlr: SPI master controller
	714	+ * @spi: SPI slave
	715	+ * @tfr: SPI transfer
	716	+ * @cs: CS register
	717	+ *
	718	+ * For bidirectional transfers (both tx_buf and rx_buf are non-%NULL), set up
	719	+ * the TX and RX DMA channel to copy between memory and FIFO register.
	720	+ *
	721	+ * For TX-only transfers (rx_buf is %NULL), copying the RX FIFO's contents to
	722	+ * memory is pointless. However not reading the RX FIFO isn't an option either
	723	+ * because transmission is halted once it's full. As a workaround, cyclically
	724	+ * clear the RX FIFO by setting the CLEAR_RX bit in the CS register.
	725	+ *
	726	+ * The CS register value is precalculated in bcm2835_spi_setup(). Normally
	727	+ * this is called only once, on slave registration. A DMA descriptor to write
	728	+ * this value is preallocated in bcm2835_dma_init(). All that's left to do
	729	+ * when performing a TX-only transfer is to submit this descriptor to the RX
	730	+ * DMA channel. Latency is thereby minimized. The descriptor does not
	731	+ * generate any interrupts while running. It must be terminated once the
	732	+ * TX DMA channel is done.
	733	+ *
	734	+ * Clearing the RX FIFO is paced by the DREQ signal. The signal is asserted
	735	+ * when the RX FIFO becomes half full, i.e. 32 bytes. (Tuneable with the DC
	736	+ * register.) Reading 32 bytes from the RX FIFO would normally require 8 bus
	737	+ * accesses, whereas clearing it requires only 1 bus access. So an 8-fold
	738	+ * reduction in bus traffic and thus energy consumption is achieved.
	739	+ *
	740	+ * For RX-only transfers (tx_buf is %NULL), fill the TX FIFO by cyclically
	741	+ * copying from the zero page. The DMA descriptor to do this is preallocated
	742	+ * in bcm2835_dma_init(). It must be terminated once the RX DMA channel is
	743	+ * done and can then be reused.
	744	+ *
	745	+ * The BCM2835 DMA driver autodetects when a transaction copies from the zero
	746	+ * page and utilizes the DMA controller's ability to synthesize zeroes instead
	747	+ * of copying them from memory. This reduces traffic on the memory bus. The
	748	+ * feature is not available on so-called "lite" channels, but normally TX DMA
	749	+ * is backed by a full-featured channel.
	750	+ *
	751	+ * Zero-filling the TX FIFO is paced by the DREQ signal. Unfortunately the
	752	+ * BCM2835 SPI controller continues to assert DREQ even after the DLEN register
	753	+ * has been counted down to zero (hardware erratum). Thus, when the transfer
	754	+ * has finished, the DMA engine zero-fills the TX FIFO until it is half full.
	755	+ * (Tuneable with the DC register.) So up to 9 gratuitous bus accesses are
	756	+ * performed at the end of an RX-only transfer.
	757	+ */
	758	+static int bcm2835_spi_transfer_one_dma(struct spi_controller *ctlr,
302	759	struct spi_device *spi,
303	760	struct spi_transfer *tfr,
304	761	u32 cs)
305	762	{
306		- struct bcm2835_spi *bs = spi_master_get_devdata(master);
	763	+ struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
	764	+ dma_cookie_t cookie;
307	765	int ret;
308	766
309		- /* check that the scatter gather segments are all a multiple of 4 */
310		- if (bcm2835_check_sg_length(&tfr->tx_sg) \|\|
311		- bcm2835_check_sg_length(&tfr->rx_sg)) {
312		- dev_warn_once(&spi->dev,
313		- "scatter gather segment length is not a multiple of 4 - falling back to interrupt mode\n");
314		- return bcm2835_spi_transfer_one_irq(master, spi, tfr, cs);
315		- }
	767	+ /* update usage statistics */
	768	+ bs->count_transfer_dma++;
	769	+
	770	+ /*
	771	+ * Transfer first few bytes without DMA if length of first TX or RX
	772	+ * sglist entry is not a multiple of 4 bytes (hardware limitation).
	773	+ */
	774	+ bcm2835_spi_transfer_prologue(ctlr, tfr, bs, cs);
316	775
317	776	/* setup tx-DMA */
318		- ret = bcm2835_spi_prepare_sg(master, tfr, true);
	777	+ if (bs->tx_buf) {
	778	+ ret = bcm2835_spi_prepare_sg(ctlr, spi, tfr, bs, true);
	779	+ } else {
	780	+ cookie = dmaengine_submit(bs->fill_tx_desc);
	781	+ ret = dma_submit_error(cookie);
	782	+ }
319	783	if (ret)
320		- return ret;
321		-
322		- /* start TX early */
323		- dma_async_issue_pending(master->dma_tx);
324		-
325		- /* mark as dma pending */
326		- bs->dma_pending = 1;
	784	+ goto err_reset_hw;
327	785
328	786	/* set the DMA length */
329		- bcm2835_wr(bs, BCM2835_SPI_DLEN, tfr->len);
	787	+ bcm2835_wr(bs, BCM2835_SPI_DLEN, bs->tx_len);
330	788
331	789	/* start the HW */
332	790	bcm2835_wr(bs, BCM2835_SPI_CS,
333	791	cs \| BCM2835_SPI_CS_TA \| BCM2835_SPI_CS_DMAEN);
334	792
	793	+ bs->tx_dma_active = true;
	794	+ smp_wmb();
	795	+
	796	+ /* start TX early */
	797	+ dma_async_issue_pending(ctlr->dma_tx);
	798	+
335	799	/* setup rx-DMA late - to run transfers while
336	800	* mapping of the rx buffers still takes place
337	801	* this saves 10us or more.
338	802	*/
339		- ret = bcm2835_spi_prepare_sg(master, tfr, false);
	803	+ if (bs->rx_buf) {
	804	+ ret = bcm2835_spi_prepare_sg(ctlr, spi, tfr, bs, false);
	805	+ } else {
	806	+ cookie = dmaengine_submit(bs->clear_rx_desc[spi->chip_select]);
	807	+ ret = dma_submit_error(cookie);
	808	+ }
340	809	if (ret) {
341	810	/* need to reset on errors */
342		- dmaengine_terminate_all(master->dma_tx);
343		- bs->dma_pending = false;
344		- bcm2835_spi_reset_hw(master);
345		- return ret;
	811	+ dmaengine_terminate_sync(ctlr->dma_tx);
	812	+ bs->tx_dma_active = false;
	813	+ goto err_reset_hw;
346	814	}
347	815
348	816	/* start rx dma late */
349		- dma_async_issue_pending(master->dma_rx);
	817	+ dma_async_issue_pending(ctlr->dma_rx);
	818	+ bs->rx_dma_active = true;
	819	+ smp_mb();
	820	+
	821	+ /*
	822	+ * In case of a very short TX-only transfer, bcm2835_spi_dma_tx_done()
	823	+ * may run before RX DMA is issued. Terminate RX DMA if so.
	824	+ */
	825	+ if (!bs->rx_buf && !bs->tx_dma_active &&
	826	+ cmpxchg(&bs->rx_dma_active, true, false)) {
	827	+ dmaengine_terminate_async(ctlr->dma_rx);
	828	+ bcm2835_spi_reset_hw(bs);
	829	+ }
350	830
351	831	/* wait for wakeup in framework */
352	832	return 1;
	833	+
	834	+err_reset_hw:
	835	+ bcm2835_spi_reset_hw(bs);
	836	+ bcm2835_spi_undo_prologue(bs);
	837	+ return ret;
353	838	}
354	839
355		-static bool bcm2835_spi_can_dma(struct spi_master *master,
	840	+static bool bcm2835_spi_can_dma(struct spi_controller *ctlr,
356	841	struct spi_device *spi,
357	842	struct spi_transfer *tfr)
358	843	{
359		- /* only run for gpio_cs */
360		- if (!gpio_is_valid(spi->cs_gpio))
361		- return false;
362		-
363	844	/* we start DMA efforts only on bigger transfers */
364	845	if (tfr->len < BCM2835_SPI_DMA_MIN_LENGTH)
365	846	return false;
366		-
367		- /* BCM2835_SPI_DLEN has defined a max transfer size as
368		- * 16 bit, so max is 65535
369		- * we can revisit this by using an alternative transfer
370		- * method - ideally this would get done without any more
371		- * interaction...
372		- */
373		- if (tfr->len > 65535) {
374		- dev_warn_once(&spi->dev,
375		- "transfer size of %d too big for dma-transfer\n",
376		- tfr->len);
377		- return false;
378		- }
379		-
380		- /* if we run rx/tx_buf with word aligned addresses then we are OK */
381		- if ((((size_t)tfr->rx_buf & 3) == 0) &&
382		- (((size_t)tfr->tx_buf & 3) == 0))
383		- return true;
384		-
385		- /* otherwise we only allow transfers within the same page
386		- * to avoid wasting time on dma_mapping when it is not practical
387		- */
388		- if (((size_t)tfr->tx_buf & (PAGE_SIZE - 1)) + tfr->len > PAGE_SIZE) {
389		- dev_warn_once(&spi->dev,
390		- "Unaligned spi tx-transfer bridging page\n");
391		- return false;
392		- }
393		- if (((size_t)tfr->rx_buf & (PAGE_SIZE - 1)) + tfr->len > PAGE_SIZE) {
394		- dev_warn_once(&spi->dev,
395		- "Unaligned spi rx-transfer bridging page\n");
396		- return false;
397		- }
398	847
399	848	/* return OK */
400	849	return true;
401	850	}
402	851
403		-static void bcm2835_dma_release(struct spi_master *master)
	852	+static void bcm2835_dma_release(struct spi_controller *ctlr,
	853	+ struct bcm2835_spi *bs)
404	854	{
405		- if (master->dma_tx) {
406		- dmaengine_terminate_all(master->dma_tx);
407		- dma_release_channel(master->dma_tx);
408		- master->dma_tx = NULL;
	855	+ int i;
	856	+
	857	+ if (ctlr->dma_tx) {
	858	+ dmaengine_terminate_sync(ctlr->dma_tx);
	859	+
	860	+ if (bs->fill_tx_desc)
	861	+ dmaengine_desc_free(bs->fill_tx_desc);
	862	+
	863	+ if (bs->fill_tx_addr)
	864	+ dma_unmap_page_attrs(ctlr->dma_tx->device->dev,
	865	+ bs->fill_tx_addr, sizeof(u32),
	866	+ DMA_TO_DEVICE,
	867	+ DMA_ATTR_SKIP_CPU_SYNC);
	868	+
	869	+ dma_release_channel(ctlr->dma_tx);
	870	+ ctlr->dma_tx = NULL;
409	871	}
410		- if (master->dma_rx) {
411		- dmaengine_terminate_all(master->dma_rx);
412		- dma_release_channel(master->dma_rx);
413		- master->dma_rx = NULL;
	872	+
	873	+ if (ctlr->dma_rx) {
	874	+ dmaengine_terminate_sync(ctlr->dma_rx);
	875	+
	876	+ for (i = 0; i < BCM2835_SPI_NUM_CS; i++)
	877	+ if (bs->clear_rx_desc[i])
	878	+ dmaengine_desc_free(bs->clear_rx_desc[i]);
	879	+
	880	+ if (bs->clear_rx_addr)
	881	+ dma_unmap_single(ctlr->dma_rx->device->dev,
	882	+ bs->clear_rx_addr,
	883	+ sizeof(bs->clear_rx_cs),
	884	+ DMA_TO_DEVICE);
	885	+
	886	+ dma_release_channel(ctlr->dma_rx);
	887	+ ctlr->dma_rx = NULL;
414	888	}
415	889	}
416	890
417		-static void bcm2835_dma_init(struct spi_master master, struct device dev)
	891	+static int bcm2835_dma_init(struct spi_controller ctlr, struct device dev,
	892	+ struct bcm2835_spi *bs)
418	893	{
419	894	struct dma_slave_config slave_config;
420	895	const __be32 *addr;
421	896	dma_addr_t dma_reg_base;
422		- int ret;
	897	+ int ret, i;
423	898
424	899	/* base address in dma-space */
425		- addr = of_get_address(master->dev.of_node, 0, NULL, NULL);
	900	+ addr = of_get_address(ctlr->dev.of_node, 0, NULL, NULL);
426	901	if (!addr) {
427	902	dev_err(dev, "could not get DMA-register address - not using dma mode\n");
428		- goto err;
	903	+ /* Fall back to interrupt mode */
	904	+ return 0;
429	905	}
430	906	dma_reg_base = be32_to_cpup(addr);
431	907
432	908	/* get tx/rx dma */
433		- master->dma_tx = dma_request_slave_channel(dev, "tx");
434		- if (!master->dma_tx) {
	909	+ ctlr->dma_tx = dma_request_chan(dev, "tx");
	910	+ if (IS_ERR(ctlr->dma_tx)) {
435	911	dev_err(dev, "no tx-dma configuration found - not using dma mode\n");
	912	+ ret = PTR_ERR(ctlr->dma_tx);
	913	+ ctlr->dma_tx = NULL;
436	914	goto err;
437	915	}
438		- master->dma_rx = dma_request_slave_channel(dev, "rx");
439		- if (!master->dma_rx) {
	916	+ ctlr->dma_rx = dma_request_chan(dev, "rx");
	917	+ if (IS_ERR(ctlr->dma_rx)) {
440	918	dev_err(dev, "no rx-dma configuration found - not using dma mode\n");
	919	+ ret = PTR_ERR(ctlr->dma_rx);
	920	+ ctlr->dma_rx = NULL;
441	921	goto err_release;
442	922	}
443	923
444		- /* configure DMAs */
445		- slave_config.direction = DMA_MEM_TO_DEV;
	924	+ /*
	925	+ * The TX DMA channel either copies a transfer's TX buffer to the FIFO
	926	+ * or, in case of an RX-only transfer, cyclically copies from the zero
	927	+ * page to the FIFO using a preallocated, reusable descriptor.
	928	+ */
446	929	slave_config.dst_addr = (u32)(dma_reg_base + BCM2835_SPI_FIFO);
447	930	slave_config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
448	931
449		- ret = dmaengine_slave_config(master->dma_tx, &slave_config);
	932	+ ret = dmaengine_slave_config(ctlr->dma_tx, &slave_config);
450	933	if (ret)
451	934	goto err_config;
452	935
453		- slave_config.direction = DMA_DEV_TO_MEM;
	936	+ bs->fill_tx_addr = dma_map_page_attrs(ctlr->dma_tx->device->dev,
	937	+ ZERO_PAGE(0), 0, sizeof(u32),
	938	+ DMA_TO_DEVICE,
	939	+ DMA_ATTR_SKIP_CPU_SYNC);
	940	+ if (dma_mapping_error(ctlr->dma_tx->device->dev, bs->fill_tx_addr)) {
	941	+ dev_err(dev, "cannot map zero page - not using DMA mode\n");
	942	+ bs->fill_tx_addr = 0;
	943	+ ret = -ENOMEM;
	944	+ goto err_release;
	945	+ }
	946	+
	947	+ bs->fill_tx_desc = dmaengine_prep_dma_cyclic(ctlr->dma_tx,
	948	+ bs->fill_tx_addr,
	949	+ sizeof(u32), 0,
	950	+ DMA_MEM_TO_DEV, 0);
	951	+ if (!bs->fill_tx_desc) {
	952	+ dev_err(dev, "cannot prepare fill_tx_desc - not using DMA mode\n");
	953	+ ret = -ENOMEM;
	954	+ goto err_release;
	955	+ }
	956	+
	957	+ ret = dmaengine_desc_set_reuse(bs->fill_tx_desc);
	958	+ if (ret) {
	959	+ dev_err(dev, "cannot reuse fill_tx_desc - not using DMA mode\n");
	960	+ goto err_release;
	961	+ }
	962	+
	963	+ /*
	964	+ * The RX DMA channel is used bidirectionally: It either reads the
	965	+ * RX FIFO or, in case of a TX-only transfer, cyclically writes a
	966	+ * precalculated value to the CS register to clear the RX FIFO.
	967	+ */
454	968	slave_config.src_addr = (u32)(dma_reg_base + BCM2835_SPI_FIFO);
455	969	slave_config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
	970	+ slave_config.dst_addr = (u32)(dma_reg_base + BCM2835_SPI_CS);
	971	+ slave_config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
456	972
457		- ret = dmaengine_slave_config(master->dma_rx, &slave_config);
	973	+ ret = dmaengine_slave_config(ctlr->dma_rx, &slave_config);
458	974	if (ret)
459	975	goto err_config;
460	976
461		- /* all went well, so set can_dma */
462		- master->can_dma = bcm2835_spi_can_dma;
463		- master->max_dma_len = 65535; /* limitation by BCM2835_SPI_DLEN */
464		- /* need to do TX AND RX DMA, so we need dummy buffers */
465		- master->flags = SPI_MASTER_MUST_RX \| SPI_MASTER_MUST_TX;
	977	+ bs->clear_rx_addr = dma_map_single(ctlr->dma_rx->device->dev,
	978	+ bs->clear_rx_cs,
	979	+ sizeof(bs->clear_rx_cs),
	980	+ DMA_TO_DEVICE);
	981	+ if (dma_mapping_error(ctlr->dma_rx->device->dev, bs->clear_rx_addr)) {
	982	+ dev_err(dev, "cannot map clear_rx_cs - not using DMA mode\n");
	983	+ bs->clear_rx_addr = 0;
	984	+ ret = -ENOMEM;
	985	+ goto err_release;
	986	+ }
466	987
467		- return;
	988	+ for (i = 0; i < BCM2835_SPI_NUM_CS; i++) {
	989	+ bs->clear_rx_desc[i] = dmaengine_prep_dma_cyclic(ctlr->dma_rx,
	990	+ bs->clear_rx_addr + i * sizeof(u32),
	991	+ sizeof(u32), 0,
	992	+ DMA_MEM_TO_DEV, 0);
	993	+ if (!bs->clear_rx_desc[i]) {
	994	+ dev_err(dev, "cannot prepare clear_rx_desc - not using DMA mode\n");
	995	+ ret = -ENOMEM;
	996	+ goto err_release;
	997	+ }
	998	+
	999	+ ret = dmaengine_desc_set_reuse(bs->clear_rx_desc[i]);
	1000	+ if (ret) {
	1001	+ dev_err(dev, "cannot reuse clear_rx_desc - not using DMA mode\n");
	1002	+ goto err_release;
	1003	+ }
	1004	+ }
	1005	+
	1006	+ /* all went well, so set can_dma */
	1007	+ ctlr->can_dma = bcm2835_spi_can_dma;
	1008	+
	1009	+ return 0;
468	1010
469	1011	err_config:
470	1012	dev_err(dev, "issue configuring dma: %d - not using DMA mode\n",
471	1013	ret);
472	1014	err_release:
473		- bcm2835_dma_release(master);
	1015	+ bcm2835_dma_release(ctlr, bs);
474	1016	err:
475		- return;
	1017	+ /*
	1018	+ * Only report error for deferred probing, otherwise fall back to
	1019	+ * interrupt mode
	1020	+ */
	1021	+ if (ret != -EPROBE_DEFER)
	1022	+ ret = 0;
	1023	+
	1024	+ return ret;
476	1025	}
477	1026
478		-static int bcm2835_spi_transfer_one_poll(struct spi_master *master,
	1027	+static int bcm2835_spi_transfer_one_poll(struct spi_controller *ctlr,
479	1028	struct spi_device *spi,
480	1029	struct spi_transfer *tfr,
481		- u32 cs,
482		- unsigned long long xfer_time_us)
	1030	+ u32 cs)
483	1031	{
484		- struct bcm2835_spi *bs = spi_master_get_devdata(master);
	1032	+ struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
485	1033	unsigned long timeout;
	1034	+
	1035	+ /* update usage statistics */
	1036	+ bs->count_transfer_polling++;
486	1037
487	1038	/* enable HW block without interrupts */
488	1039	bcm2835_wr(bs, BCM2835_SPI_CS, cs \| BCM2835_SPI_CS_TA);
..	..	@@ -491,10 +1042,10 @@
491	1042	* if we are interrupted here, then the data is
492	1043	* getting transferred by the HW while we are interrupted
493	1044	*/
494		- bcm2835_wr_fifo(bs);
	1045	+ bcm2835_wr_fifo_blind(bs, BCM2835_SPI_FIFO_SIZE);
495	1046
496		- /* set the timeout */
497		- timeout = jiffies + BCM2835_SPI_POLLING_JIFFIES;
	1047	+ /* set the timeout to at least 2 jiffies */
	1048	+ timeout = jiffies + 2 + HZ * polling_limit_us / 1000000;
498	1049
499	1050	/* loop until finished the transfer */
500	1051	while (bs->rx_len) {
..	..	@@ -513,36 +1064,38 @@
513	1064	jiffies - timeout,
514	1065	bs->tx_len, bs->rx_len);
515	1066	/* fall back to interrupt mode */
516		- return bcm2835_spi_transfer_one_irq(master, spi,
517		- tfr, cs);
	1067	+
	1068	+ /* update usage statistics */
	1069	+ bs->count_transfer_irq_after_polling++;
	1070	+
	1071	+ return bcm2835_spi_transfer_one_irq(ctlr, spi,
	1072	+ tfr, cs, false);
518	1073	}
519	1074	}
520	1075
521	1076	/* Transfer complete - reset SPI HW */
522		- bcm2835_spi_reset_hw(master);
	1077	+ bcm2835_spi_reset_hw(bs);
523	1078	/* and return without waiting for completion */
524	1079	return 0;
525	1080	}
526	1081
527		-static int bcm2835_spi_transfer_one(struct spi_master *master,
	1082	+static int bcm2835_spi_transfer_one(struct spi_controller *ctlr,
528	1083	struct spi_device *spi,
529	1084	struct spi_transfer *tfr)
530	1085	{
531		- struct bcm2835_spi *bs = spi_master_get_devdata(master);
532		- unsigned long spi_hz, clk_hz, cdiv;
533		- unsigned long spi_used_hz;
534		- unsigned long long xfer_time_us;
535		- u32 cs = bcm2835_rd(bs, BCM2835_SPI_CS);
	1086	+ struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
	1087	+ unsigned long spi_hz, cdiv;
	1088	+ unsigned long hz_per_byte, byte_limit;
	1089	+ u32 cs = bs->prepare_cs[spi->chip_select];
536	1090
537	1091	/* set clock */
538	1092	spi_hz = tfr->speed_hz;
539		- clk_hz = clk_get_rate(bs->clk);
540	1093
541		- if (spi_hz >= clk_hz / 2) {
	1094	+ if (spi_hz >= bs->clk_hz / 2) {
542	1095	cdiv = 2; /* clk_hz/2 is the fastest we can go */
543	1096	} else if (spi_hz) {
544	1097	/* CDIV must be a multiple of two */
545		- cdiv = DIV_ROUND_UP(clk_hz, spi_hz);
	1098	+ cdiv = DIV_ROUND_UP(bs->clk_hz, spi_hz);
546	1099	cdiv += (cdiv % 2);
547	1100
548	1101	if (cdiv >= 65536)
..	..	@@ -550,22 +1103,12 @@
550	1103	} else {
551	1104	cdiv = 0; /* 0 is the slowest we can go */
552	1105	}
553		- spi_used_hz = cdiv ? (clk_hz / cdiv) : (clk_hz / 65536);
	1106	+ tfr->effective_speed_hz = cdiv ? (bs->clk_hz / cdiv) : (bs->clk_hz / 65536);
554	1107	bcm2835_wr(bs, BCM2835_SPI_CLK, cdiv);
555	1108
556	1109	/* handle all the 3-wire mode */
557		- if (spi->mode & SPI_3WIRE && tfr->rx_buf &&
558		- tfr->rx_buf != master->dummy_rx)
	1110	+ if (spi->mode & SPI_3WIRE && tfr->rx_buf)
559	1111	cs \|= BCM2835_SPI_CS_REN;
560		- else
561		- cs &= ~BCM2835_SPI_CS_REN;
562		-
563		- /* for gpio_cs set dummy CS so that no HW-CS get changed
564		- * we can not run this in bcm2835_spi_set_cs, as it does
565		- * not get called for cs_gpio cases, so we need to do it here
566		- */
567		- if (gpio_is_valid(spi->cs_gpio) \|\| (spi->mode & SPI_NO_CS))
568		- cs \|= BCM2835_SPI_CS_CS_10 \| BCM2835_SPI_CS_CS_01;
569	1112
570	1113	/* set transmit buffers and length */
571	1114	bs->tx_buf = tfr->tx_buf;
..	..	@@ -573,109 +1116,76 @@
573	1116	bs->tx_len = tfr->len;
574	1117	bs->rx_len = tfr->len;
575	1118
576		- /* calculate the estimated time in us the transfer runs */
577		- xfer_time_us = (unsigned long long)tfr->len
578		- * 9 /* clocks/byte - SPI-HW waits 1 clock after each byte */
579		- * 1000000;
580		- do_div(xfer_time_us, spi_used_hz);
	1119	+ /* Calculate the estimated time in us the transfer runs. Note that
	1120	+ * there is 1 idle clocks cycles after each byte getting transferred
	1121	+ * so we have 9 cycles/byte. This is used to find the number of Hz
	1122	+ * per byte per polling limit. E.g., we can transfer 1 byte in 30 us
	1123	+ * per 300,000 Hz of bus clock.
	1124	+ */
	1125	+ hz_per_byte = polling_limit_us ? (9 * 1000000) / polling_limit_us : 0;
	1126	+ byte_limit = hz_per_byte ? tfr->effective_speed_hz / hz_per_byte : 1;
581	1127
582		- /* for short requests run polling*/
583		- if (xfer_time_us <= BCM2835_SPI_POLLING_LIMIT_US)
584		- return bcm2835_spi_transfer_one_poll(master, spi, tfr,
585		- cs, xfer_time_us);
	1128	+ /* run in polling mode for short transfers */
	1129	+ if (tfr->len < byte_limit)
	1130	+ return bcm2835_spi_transfer_one_poll(ctlr, spi, tfr, cs);
586	1131
587		- /* run in dma mode if conditions are right */
588		- if (master->can_dma && bcm2835_spi_can_dma(master, spi, tfr))
589		- return bcm2835_spi_transfer_one_dma(master, spi, tfr, cs);
	1132	+ /* run in dma mode if conditions are right
	1133	+ * Note that unlike poll or interrupt mode DMA mode does not have
	1134	+ * this 1 idle clock cycle pattern but runs the spi clock without gaps
	1135	+ */
	1136	+ if (ctlr->can_dma && bcm2835_spi_can_dma(ctlr, spi, tfr))
	1137	+ return bcm2835_spi_transfer_one_dma(ctlr, spi, tfr, cs);
590	1138
591	1139	/* run in interrupt-mode */
592		- return bcm2835_spi_transfer_one_irq(master, spi, tfr, cs);
	1140	+ return bcm2835_spi_transfer_one_irq(ctlr, spi, tfr, cs, true);
593	1141	}
594	1142
595		-static int bcm2835_spi_prepare_message(struct spi_master *master,
	1143	+static int bcm2835_spi_prepare_message(struct spi_controller *ctlr,
596	1144	struct spi_message *msg)
597	1145	{
598	1146	struct spi_device *spi = msg->spi;
599		- struct bcm2835_spi *bs = spi_master_get_devdata(master);
600		- u32 cs = bcm2835_rd(bs, BCM2835_SPI_CS);
	1147	+ struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
	1148	+ int ret;
601	1149
602		- cs &= ~(BCM2835_SPI_CS_CPOL \| BCM2835_SPI_CS_CPHA);
	1150	+ if (ctlr->can_dma) {
	1151	+ /*
	1152	+ * DMA transfers are limited to 16 bit (0 to 65535 bytes) by
	1153	+ * the SPI HW due to DLEN. Split up transfers (32-bit FIFO
	1154	+ * aligned) if the limit is exceeded.
	1155	+ */
	1156	+ ret = spi_split_transfers_maxsize(ctlr, msg, 65532,
	1157	+ GFP_KERNEL \| GFP_DMA);
	1158	+ if (ret)
	1159	+ return ret;
	1160	+ }
603	1161
604		- if (spi->mode & SPI_CPOL)
605		- cs \|= BCM2835_SPI_CS_CPOL;
606		- if (spi->mode & SPI_CPHA)
607		- cs \|= BCM2835_SPI_CS_CPHA;
608		-
609		- bcm2835_wr(bs, BCM2835_SPI_CS, cs);
	1162	+ /*
	1163	+ * Set up clock polarity before spi_transfer_one_message() asserts
	1164	+ * chip select to avoid a gratuitous clock signal edge.
	1165	+ */
	1166	+ bcm2835_wr(bs, BCM2835_SPI_CS, bs->prepare_cs[spi->chip_select]);
610	1167
611	1168	return 0;
612	1169	}
613	1170
614		-static void bcm2835_spi_handle_err(struct spi_master *master,
	1171	+static void bcm2835_spi_handle_err(struct spi_controller *ctlr,
615	1172	struct spi_message *msg)
616	1173	{
617		- struct bcm2835_spi *bs = spi_master_get_devdata(master);
	1174	+ struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
618	1175
619	1176	/* if an error occurred and we have an active dma, then terminate */
620		- if (cmpxchg(&bs->dma_pending, true, false)) {
621		- dmaengine_terminate_all(master->dma_tx);
622		- dmaengine_terminate_all(master->dma_rx);
	1177	+ if (ctlr->dma_tx) {
	1178	+ dmaengine_terminate_sync(ctlr->dma_tx);
	1179	+ bs->tx_dma_active = false;
623	1180	}
	1181	+ if (ctlr->dma_rx) {
	1182	+ dmaengine_terminate_sync(ctlr->dma_rx);
	1183	+ bs->rx_dma_active = false;
	1184	+ }
	1185	+ bcm2835_spi_undo_prologue(bs);
	1186	+
624	1187	/* and reset */
625		- bcm2835_spi_reset_hw(master);
626		-}
627		-
628		-static void bcm2835_spi_set_cs(struct spi_device *spi, bool gpio_level)
629		-{
630		- /*
631		- * we can assume that we are "native" as per spi_set_cs
632		- * calling us ONLY when cs_gpio is not set
633		- * we can also assume that we are CS < 3 as per bcm2835_spi_setup
634		- * we would not get called because of error handling there.
635		- * the level passed is the electrical level not enabled/disabled
636		- * so it has to get translated back to enable/disable
637		- * see spi_set_cs in spi.c for the implementation
638		- */
639		-
640		- struct spi_master *master = spi->master;
641		- struct bcm2835_spi *bs = spi_master_get_devdata(master);
642		- u32 cs = bcm2835_rd(bs, BCM2835_SPI_CS);
643		- bool enable;
644		-
645		- /* calculate the enable flag from the passed gpio_level */
646		- enable = (spi->mode & SPI_CS_HIGH) ? gpio_level : !gpio_level;
647		-
648		- /* set flags for "reverse" polarity in the registers */
649		- if (spi->mode & SPI_CS_HIGH) {
650		- /* set the correct CS-bits */
651		- cs \|= BCM2835_SPI_CS_CSPOL;
652		- cs \|= BCM2835_SPI_CS_CSPOL0 << spi->chip_select;
653		- } else {
654		- /* clean the CS-bits */
655		- cs &= ~BCM2835_SPI_CS_CSPOL;
656		- cs &= ~(BCM2835_SPI_CS_CSPOL0 << spi->chip_select);
657		- }
658		-
659		- /* select the correct chip_select depending on disabled/enabled */
660		- if (enable) {
661		- /* set cs correctly */
662		- if (spi->mode & SPI_NO_CS) {
663		- /* use the "undefined" chip-select */
664		- cs \|= BCM2835_SPI_CS_CS_10 \| BCM2835_SPI_CS_CS_01;
665		- } else {
666		- /* set the chip select */
667		- cs &= ~(BCM2835_SPI_CS_CS_10 \| BCM2835_SPI_CS_CS_01);
668		- cs \|= spi->chip_select;
669		- }
670		- } else {
671		- /* disable CSPOL which puts HW-CS into deselected state */
672		- cs &= ~BCM2835_SPI_CS_CSPOL;
673		- /* use the "undefined" chip-select as precaution */
674		- cs \|= BCM2835_SPI_CS_CS_10 \| BCM2835_SPI_CS_CS_01;
675		- }
676		-
677		- /* finally set the calculated flags in SPI_CS */
678		- bcm2835_wr(bs, BCM2835_SPI_CS, cs);
	1188	+ bcm2835_spi_reset_hw(bs);
679	1189	}
680	1190
681	1191	static int chip_match_name(struct gpio_chip chip, void data)
..	..	@@ -685,14 +1195,55 @@
685	1195
686	1196	static int bcm2835_spi_setup(struct spi_device *spi)
687	1197	{
688		- int err;
	1198	+ struct spi_controller *ctlr = spi->controller;
	1199	+ struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
689	1200	struct gpio_chip *chip;
	1201	+ u32 cs;
	1202	+
	1203	+ if (spi->chip_select >= BCM2835_SPI_NUM_CS) {
	1204	+ dev_err(&spi->dev, "only %d chip-selects supported\n",
	1205	+ BCM2835_SPI_NUM_CS - 1);
	1206	+ return -EINVAL;
	1207	+ }
	1208	+
	1209	+ /*
	1210	+ * Precalculate SPI slave's CS register value for ->prepare_message():
	1211	+ * The driver always uses software-controlled GPIO chip select, hence
	1212	+ * set the hardware-controlled native chip select to an invalid value
	1213	+ * to prevent it from interfering.
	1214	+ */
	1215	+ cs = BCM2835_SPI_CS_CS_10 \| BCM2835_SPI_CS_CS_01;
	1216	+ if (spi->mode & SPI_CPOL)
	1217	+ cs \|= BCM2835_SPI_CS_CPOL;
	1218	+ if (spi->mode & SPI_CPHA)
	1219	+ cs \|= BCM2835_SPI_CS_CPHA;
	1220	+ bs->prepare_cs[spi->chip_select] = cs;
	1221	+
	1222	+ /*
	1223	+ * Precalculate SPI slave's CS register value to clear RX FIFO
	1224	+ * in case of a TX-only DMA transfer.
	1225	+ */
	1226	+ if (ctlr->dma_rx) {
	1227	+ bs->clear_rx_cs[spi->chip_select] = cs \|
	1228	+ BCM2835_SPI_CS_TA \|
	1229	+ BCM2835_SPI_CS_DMAEN \|
	1230	+ BCM2835_SPI_CS_CLEAR_RX;
	1231	+ dma_sync_single_for_device(ctlr->dma_rx->device->dev,
	1232	+ bs->clear_rx_addr,
	1233	+ sizeof(bs->clear_rx_cs),
	1234	+ DMA_TO_DEVICE);
	1235	+ }
	1236	+
690	1237	/*
691	1238	* sanity checking the native-chipselects
692	1239	*/
693	1240	if (spi->mode & SPI_NO_CS)
694	1241	return 0;
695		- if (gpio_is_valid(spi->cs_gpio))
	1242	+ /*
	1243	+ * The SPI core has successfully requested the CS GPIO line from the
	1244	+ * device tree, so we are done.
	1245	+ */
	1246	+ if (spi->cs_gpiod)
696	1247	return 0;
697	1248	if (spi->chip_select > 1) {
698	1249	/* error in the case of native CS requested with CS > 1
..	..	@@ -703,113 +1254,120 @@
703	1254	"setup: only two native chip-selects are supported\n");
704	1255	return -EINVAL;
705	1256	}
706		- /* now translate native cs to GPIO */
	1257	+
	1258	+ /*
	1259	+ * Translate native CS to GPIO
	1260	+ *
	1261	+ * FIXME: poking around in the gpiolib internals like this is
	1262	+ * not very good practice. Find a way to locate the real problem
	1263	+ * and fix it. Why is the GPIO descriptor in spi->cs_gpiod
	1264	+ * sometimes not assigned correctly? Erroneous device trees?
	1265	+ */
707	1266
708	1267	/* get the gpio chip for the base */
709	1268	chip = gpiochip_find("pinctrl-bcm2835", chip_match_name);
710	1269	if (!chip)
711	1270	return 0;
712	1271
713		- /* and calculate the real CS */
714		- spi->cs_gpio = chip->base + 8 - spi->chip_select;
	1272	+ spi->cs_gpiod = gpiochip_request_own_desc(chip, 8 - spi->chip_select,
	1273	+ DRV_NAME,
	1274	+ GPIO_LOOKUP_FLAGS_DEFAULT,
	1275	+ GPIOD_OUT_LOW);
	1276	+ if (IS_ERR(spi->cs_gpiod))
	1277	+ return PTR_ERR(spi->cs_gpiod);
715	1278
716	1279	/* and set up the "mode" and level */
717		- dev_info(&spi->dev, "setting up native-CS%i as GPIO %i\n",
718		- spi->chip_select, spi->cs_gpio);
719		-
720		- /* set up GPIO as output and pull to the correct level */
721		- err = gpio_direction_output(spi->cs_gpio,
722		- (spi->mode & SPI_CS_HIGH) ? 0 : 1);
723		- if (err) {
724		- dev_err(&spi->dev,
725		- "could not set CS%i gpio %i as output: %i",
726		- spi->chip_select, spi->cs_gpio, err);
727		- return err;
728		- }
	1280	+ dev_info(&spi->dev, "setting up native-CS%i to use GPIO\n",
	1281	+ spi->chip_select);
729	1282
730	1283	return 0;
731	1284	}
732	1285
733	1286	static int bcm2835_spi_probe(struct platform_device *pdev)
734	1287	{
735		- struct spi_master *master;
	1288	+ struct spi_controller *ctlr;
736	1289	struct bcm2835_spi *bs;
737		- struct resource *res;
738	1290	int err;
739	1291
740		- master = devm_spi_alloc_master(&pdev->dev, sizeof(*bs));
741		- if (!master) {
742		- dev_err(&pdev->dev, "spi_alloc_master() failed\n");
	1292	+ ctlr = devm_spi_alloc_master(&pdev->dev, ALIGN(sizeof(*bs),
	1293	+ dma_get_cache_alignment()));
	1294	+ if (!ctlr)
743	1295	return -ENOMEM;
744		- }
745	1296
746		- platform_set_drvdata(pdev, master);
	1297	+ platform_set_drvdata(pdev, ctlr);
747	1298
748		- master->mode_bits = BCM2835_SPI_MODE_BITS;
749		- master->bits_per_word_mask = SPI_BPW_MASK(8);
750		- master->num_chipselect = 3;
751		- master->setup = bcm2835_spi_setup;
752		- master->set_cs = bcm2835_spi_set_cs;
753		- master->transfer_one = bcm2835_spi_transfer_one;
754		- master->handle_err = bcm2835_spi_handle_err;
755		- master->prepare_message = bcm2835_spi_prepare_message;
756		- master->dev.of_node = pdev->dev.of_node;
	1299	+ ctlr->use_gpio_descriptors = true;
	1300	+ ctlr->mode_bits = BCM2835_SPI_MODE_BITS;
	1301	+ ctlr->bits_per_word_mask = SPI_BPW_MASK(8);
	1302	+ ctlr->num_chipselect = 3;
	1303	+ ctlr->setup = bcm2835_spi_setup;
	1304	+ ctlr->transfer_one = bcm2835_spi_transfer_one;
	1305	+ ctlr->handle_err = bcm2835_spi_handle_err;
	1306	+ ctlr->prepare_message = bcm2835_spi_prepare_message;
	1307	+ ctlr->dev.of_node = pdev->dev.of_node;
757	1308
758		- bs = spi_master_get_devdata(master);
	1309	+ bs = spi_controller_get_devdata(ctlr);
	1310	+ bs->ctlr = ctlr;
759	1311
760		- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
761		- bs->regs = devm_ioremap_resource(&pdev->dev, res);
	1312	+ bs->regs = devm_platform_ioremap_resource(pdev, 0);
762	1313	if (IS_ERR(bs->regs))
763	1314	return PTR_ERR(bs->regs);
764	1315
765	1316	bs->clk = devm_clk_get(&pdev->dev, NULL);
766		- if (IS_ERR(bs->clk)) {
767		- err = PTR_ERR(bs->clk);
768		- dev_err(&pdev->dev, "could not get clk: %d\n", err);
769		- return err;
770		- }
	1317	+ if (IS_ERR(bs->clk))
	1318	+ return dev_err_probe(&pdev->dev, PTR_ERR(bs->clk),
	1319	+ "could not get clk\n");
771	1320
772	1321	bs->irq = platform_get_irq(pdev, 0);
773		- if (bs->irq <= 0) {
774		- dev_err(&pdev->dev, "could not get IRQ: %d\n", bs->irq);
	1322	+ if (bs->irq <= 0)
775	1323	return bs->irq ? bs->irq : -ENODEV;
776		- }
777	1324
778	1325	clk_prepare_enable(bs->clk);
	1326	+ bs->clk_hz = clk_get_rate(bs->clk);
779	1327
780		- bcm2835_dma_init(master, &pdev->dev);
	1328	+ err = bcm2835_dma_init(ctlr, &pdev->dev, bs);
	1329	+ if (err)
	1330	+ goto out_clk_disable;
781	1331
782	1332	/* initialise the hardware with the default polarities */
783	1333	bcm2835_wr(bs, BCM2835_SPI_CS,
784	1334	BCM2835_SPI_CS_CLEAR_RX \| BCM2835_SPI_CS_CLEAR_TX);
785	1335
786	1336	err = devm_request_irq(&pdev->dev, bs->irq, bcm2835_spi_interrupt, 0,
787		- dev_name(&pdev->dev), master);
	1337	+ dev_name(&pdev->dev), bs);
788	1338	if (err) {
789	1339	dev_err(&pdev->dev, "could not request IRQ: %d\n", err);
790	1340	goto out_dma_release;
791	1341	}
792	1342
793		- err = spi_register_master(master);
	1343	+ err = spi_register_controller(ctlr);
794	1344	if (err) {
795		- dev_err(&pdev->dev, "could not register SPI master: %d\n", err);
	1345	+ dev_err(&pdev->dev, "could not register SPI controller: %d\n",
	1346	+ err);
796	1347	goto out_dma_release;
797	1348	}
	1349	+
	1350	+ bcm2835_debugfs_create(bs, dev_name(&pdev->dev));
798	1351
799	1352	return 0;
800	1353
801	1354	out_dma_release:
802		- bcm2835_dma_release(master);
	1355	+ bcm2835_dma_release(ctlr, bs);
	1356	+out_clk_disable:
803	1357	clk_disable_unprepare(bs->clk);
804	1358	return err;
805	1359	}
806	1360
807	1361	static int bcm2835_spi_remove(struct platform_device *pdev)
808	1362	{
809		- struct spi_master *master = platform_get_drvdata(pdev);
810		- struct bcm2835_spi *bs = spi_master_get_devdata(master);
	1363	+ struct spi_controller *ctlr = platform_get_drvdata(pdev);
	1364	+ struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
811	1365
812		- spi_unregister_master(master);
	1366	+ bcm2835_debugfs_remove(bs);
	1367	+
	1368	+ spi_unregister_controller(ctlr);
	1369	+
	1370	+ bcm2835_dma_release(ctlr, bs);
813	1371
814	1372	/* Clear FIFOs, and disable the HW block */
815	1373	bcm2835_wr(bs, BCM2835_SPI_CS,
..	..	@@ -817,9 +1375,16 @@
817	1375
818	1376	clk_disable_unprepare(bs->clk);
819	1377
820		- bcm2835_dma_release(master);
821		-
822	1378	return 0;
	1379	+}
	1380	+
	1381	+static void bcm2835_spi_shutdown(struct platform_device *pdev)
	1382	+{
	1383	+ int ret;
	1384	+
	1385	+ ret = bcm2835_spi_remove(pdev);
	1386	+ if (ret)
	1387	+ dev_err(&pdev->dev, "failed to shutdown\n");
823	1388	}
824	1389
825	1390	static const struct of_device_id bcm2835_spi_match[] = {
..	..	@@ -835,9 +1400,10 @@
835	1400	},
836	1401	.probe = bcm2835_spi_probe,
837	1402	.remove = bcm2835_spi_remove,
	1403	+ .shutdown = bcm2835_spi_shutdown,
838	1404	};
839	1405	module_platform_driver(bcm2835_spi_driver);
840	1406
841	1407	MODULE_DESCRIPTION("SPI controller driver for Broadcom BCM2835");
842	1408	MODULE_AUTHOR("Chris Boot <bootc@bootc.net>");
843		-MODULE_LICENSE("GPL v2");
	1409	+MODULE_LICENSE("GPL");