~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,18 +1,9 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* Copyright © 2010-2015 Broadcom Corporation
3		- *
4		- * This program is free software; you can redistribute it and/or modify
5		- * it under the terms of the GNU General Public License version 2 as
6		- * published by the Free Software Foundation.
7		- *
8		- * This program is distributed in the hope that it will be useful,
9		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
10		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11		- * GNU General Public License for more details.
12	4	*/
13	5
14	6	#include <linux/clk.h>
15		-#include <linux/version.h>
16	7	#include <linux/module.h>
17	8	#include <linux/init.h>
18	9	#include <linux/delay.h>
..	..	@@ -34,6 +25,7 @@
34	25	#include <linux/of.h>
35	26	#include <linux/of_platform.h>
36	27	#include <linux/slab.h>
	28	+#include <linux/static_key.h>
37	29	#include <linux/list.h>
38	30	#include <linux/log2.h>
39	31
..	..	@@ -92,6 +84,12 @@
92	84	#define FLASH_DMA_ECC_ERROR (1 << 8)
93	85	#define FLASH_DMA_CORR_ERROR (1 << 9)
94	86
	87	+/* Bitfields for DMA_MODE */
	88	+#define FLASH_DMA_MODE_STOP_ON_ERROR BIT(1) /* stop in Uncorr ECC error */
	89	+#define FLASH_DMA_MODE_MODE BIT(0) /* link list */
	90	+#define FLASH_DMA_MODE_MASK (FLASH_DMA_MODE_STOP_ON_ERROR \| \
	91	+ FLASH_DMA_MODE_MODE)
	92	+
95	93	/* 512B flash cache in the NAND controller HW */
96	94	#define FC_SHIFT 9U
97	95	#define FC_BYTES 512U
..	..	@@ -104,6 +102,102 @@
104	102	#define NAND_CTRL_RDY (INTFC_CTLR_READY \| INTFC_FLASH_READY)
105	103	#define NAND_POLL_STATUS_TIMEOUT_MS 100
106	104
	105	+#define EDU_CMD_WRITE 0x00
	106	+#define EDU_CMD_READ 0x01
	107	+#define EDU_STATUS_ACTIVE BIT(0)
	108	+#define EDU_ERR_STATUS_ERRACK BIT(0)
	109	+#define EDU_DONE_MASK GENMASK(1, 0)
	110	+
	111	+#define EDU_CONFIG_MODE_NAND BIT(0)
	112	+#define EDU_CONFIG_SWAP_BYTE BIT(1)
	113	+#ifdef CONFIG_CPU_BIG_ENDIAN
	114	+#define EDU_CONFIG_SWAP_CFG EDU_CONFIG_SWAP_BYTE
	115	+#else
	116	+#define EDU_CONFIG_SWAP_CFG 0
	117	+#endif
	118	+
	119	+/* edu registers */
	120	+enum edu_reg {
	121	+ EDU_CONFIG = 0,
	122	+ EDU_DRAM_ADDR,
	123	+ EDU_EXT_ADDR,
	124	+ EDU_LENGTH,
	125	+ EDU_CMD,
	126	+ EDU_STOP,
	127	+ EDU_STATUS,
	128	+ EDU_DONE,
	129	+ EDU_ERR_STATUS,
	130	+};
	131	+
	132	+static const u16 edu_regs[] = {
	133	+ [EDU_CONFIG] = 0x00,
	134	+ [EDU_DRAM_ADDR] = 0x04,
	135	+ [EDU_EXT_ADDR] = 0x08,
	136	+ [EDU_LENGTH] = 0x0c,
	137	+ [EDU_CMD] = 0x10,
	138	+ [EDU_STOP] = 0x14,
	139	+ [EDU_STATUS] = 0x18,
	140	+ [EDU_DONE] = 0x1c,
	141	+ [EDU_ERR_STATUS] = 0x20,
	142	+};
	143	+
	144	+/* flash_dma registers */
	145	+enum flash_dma_reg {
	146	+ FLASH_DMA_REVISION = 0,
	147	+ FLASH_DMA_FIRST_DESC,
	148	+ FLASH_DMA_FIRST_DESC_EXT,
	149	+ FLASH_DMA_CTRL,
	150	+ FLASH_DMA_MODE,
	151	+ FLASH_DMA_STATUS,
	152	+ FLASH_DMA_INTERRUPT_DESC,
	153	+ FLASH_DMA_INTERRUPT_DESC_EXT,
	154	+ FLASH_DMA_ERROR_STATUS,
	155	+ FLASH_DMA_CURRENT_DESC,
	156	+ FLASH_DMA_CURRENT_DESC_EXT,
	157	+};
	158	+
	159	+/* flash_dma registers v0*/
	160	+static const u16 flash_dma_regs_v0[] = {
	161	+ [FLASH_DMA_REVISION] = 0x00,
	162	+ [FLASH_DMA_FIRST_DESC] = 0x04,
	163	+ [FLASH_DMA_CTRL] = 0x08,
	164	+ [FLASH_DMA_MODE] = 0x0c,
	165	+ [FLASH_DMA_STATUS] = 0x10,
	166	+ [FLASH_DMA_INTERRUPT_DESC] = 0x14,
	167	+ [FLASH_DMA_ERROR_STATUS] = 0x18,
	168	+ [FLASH_DMA_CURRENT_DESC] = 0x1c,
	169	+};
	170	+
	171	+/* flash_dma registers v1*/
	172	+static const u16 flash_dma_regs_v1[] = {
	173	+ [FLASH_DMA_REVISION] = 0x00,
	174	+ [FLASH_DMA_FIRST_DESC] = 0x04,
	175	+ [FLASH_DMA_FIRST_DESC_EXT] = 0x08,
	176	+ [FLASH_DMA_CTRL] = 0x0c,
	177	+ [FLASH_DMA_MODE] = 0x10,
	178	+ [FLASH_DMA_STATUS] = 0x14,
	179	+ [FLASH_DMA_INTERRUPT_DESC] = 0x18,
	180	+ [FLASH_DMA_INTERRUPT_DESC_EXT] = 0x1c,
	181	+ [FLASH_DMA_ERROR_STATUS] = 0x20,
	182	+ [FLASH_DMA_CURRENT_DESC] = 0x24,
	183	+ [FLASH_DMA_CURRENT_DESC_EXT] = 0x28,
	184	+};
	185	+
	186	+/* flash_dma registers v4 */
	187	+static const u16 flash_dma_regs_v4[] = {
	188	+ [FLASH_DMA_REVISION] = 0x00,
	189	+ [FLASH_DMA_FIRST_DESC] = 0x08,
	190	+ [FLASH_DMA_FIRST_DESC_EXT] = 0x0c,
	191	+ [FLASH_DMA_CTRL] = 0x10,
	192	+ [FLASH_DMA_MODE] = 0x14,
	193	+ [FLASH_DMA_STATUS] = 0x18,
	194	+ [FLASH_DMA_INTERRUPT_DESC] = 0x20,
	195	+ [FLASH_DMA_INTERRUPT_DESC_EXT] = 0x24,
	196	+ [FLASH_DMA_ERROR_STATUS] = 0x28,
	197	+ [FLASH_DMA_CURRENT_DESC] = 0x30,
	198	+ [FLASH_DMA_CURRENT_DESC_EXT] = 0x34,
	199	+};
	200	+
107	201	/* Controller feature flags */
108	202	enum {
109	203	BRCMNAND_HAS_1K_SECTORS = BIT(0),
..	..	@@ -111,6 +205,10 @@
111	205	BRCMNAND_HAS_CACHE_MODE = BIT(2),
112	206	BRCMNAND_HAS_WP = BIT(3),
113	207	};
	208	+
	209	+struct brcmnand_host;
	210	+
	211	+static DEFINE_STATIC_KEY_FALSE(brcmnand_soc_has_ops_key);
114	212
115	213	struct brcmnand_controller {
116	214	struct device *dev;
..	..	@@ -130,14 +228,31 @@
130	228
131	229	int cmd_pending;
132	230	bool dma_pending;
	231	+ bool edu_pending;
133	232	struct completion done;
134	233	struct completion dma_done;
	234	+ struct completion edu_done;
135	235
136	236	/* List of NAND hosts (one for each chip-select) */
137	237	struct list_head host_list;
138	238
	239	+ /* EDU info, per-transaction */
	240	+ const u16 *edu_offsets;
	241	+ void __iomem *edu_base;
	242	+ int edu_irq;
	243	+ int edu_count;
	244	+ u64 edu_dram_addr;
	245	+ u32 edu_ext_addr;
	246	+ u32 edu_cmd;
	247	+ u32 edu_config;
	248	+
	249	+ /* flash_dma reg */
	250	+ const u16 *flash_dma_offsets;
139	251	struct brcm_nand_dma_desc *dma_desc;
140	252	dma_addr_t dma_pa;
	253	+
	254	+ int (dma_trans)(struct brcmnand_host host, u64 addr, u32 *buf,
	255	+ u32 len, u8 dma_cmd);
141	256
142	257	/* in-memory cache of the FLASH_CACHE, used only for some commands */
143	258	u8 flash_cache[FC_BYTES];
..	..	@@ -151,7 +266,9 @@
151	266	const unsigned int *block_sizes;
152	267	unsigned int max_page_size;
153	268	const unsigned int *page_sizes;
	269	+ unsigned int page_size_shift;
154	270	unsigned int max_oob;
	271	+ u32 ecc_level_shift;
155	272	u32 features;
156	273
157	274	/* for low-power standby/resume only */
..	..	@@ -159,6 +276,8 @@
159	276	u32 nand_cs_nand_xor;
160	277	u32 corr_stat_threshold;
161	278	u32 flash_dma_mode;
	279	+ u32 flash_edu_mode;
	280	+ bool pio_poll_mode;
162	281	};
163	282
164	283	struct brcmnand_cfg {
..	..	@@ -223,8 +342,38 @@
223	342	BRCMNAND_FC_BASE,
224	343	};
225	344
226		-/* BRCMNAND v4.0 */
227		-static const u16 brcmnand_regs_v40[] = {
	345	+/* BRCMNAND v2.1-v2.2 */
	346	+static const u16 brcmnand_regs_v21[] = {
	347	+ [BRCMNAND_CMD_START] = 0x04,
	348	+ [BRCMNAND_CMD_EXT_ADDRESS] = 0x08,
	349	+ [BRCMNAND_CMD_ADDRESS] = 0x0c,
	350	+ [BRCMNAND_INTFC_STATUS] = 0x5c,
	351	+ [BRCMNAND_CS_SELECT] = 0x14,
	352	+ [BRCMNAND_CS_XOR] = 0x18,
	353	+ [BRCMNAND_LL_OP] = 0,
	354	+ [BRCMNAND_CS0_BASE] = 0x40,
	355	+ [BRCMNAND_CS1_BASE] = 0,
	356	+ [BRCMNAND_CORR_THRESHOLD] = 0,
	357	+ [BRCMNAND_CORR_THRESHOLD_EXT] = 0,
	358	+ [BRCMNAND_UNCORR_COUNT] = 0,
	359	+ [BRCMNAND_CORR_COUNT] = 0,
	360	+ [BRCMNAND_CORR_EXT_ADDR] = 0x60,
	361	+ [BRCMNAND_CORR_ADDR] = 0x64,
	362	+ [BRCMNAND_UNCORR_EXT_ADDR] = 0x68,
	363	+ [BRCMNAND_UNCORR_ADDR] = 0x6c,
	364	+ [BRCMNAND_SEMAPHORE] = 0x50,
	365	+ [BRCMNAND_ID] = 0x54,
	366	+ [BRCMNAND_ID_EXT] = 0,
	367	+ [BRCMNAND_LL_RDATA] = 0,
	368	+ [BRCMNAND_OOB_READ_BASE] = 0x20,
	369	+ [BRCMNAND_OOB_READ_10_BASE] = 0,
	370	+ [BRCMNAND_OOB_WRITE_BASE] = 0x30,
	371	+ [BRCMNAND_OOB_WRITE_10_BASE] = 0,
	372	+ [BRCMNAND_FC_BASE] = 0x200,
	373	+};
	374	+
	375	+/* BRCMNAND v3.3-v4.0 */
	376	+static const u16 brcmnand_regs_v33[] = {
228	377	[BRCMNAND_CMD_START] = 0x04,
229	378	[BRCMNAND_CMD_EXT_ADDRESS] = 0x08,
230	379	[BRCMNAND_CMD_ADDRESS] = 0x0c,
..	..	@@ -421,6 +570,9 @@
421	570	CFG_BUS_WIDTH = BIT(CFG_BUS_WIDTH_SHIFT),
422	571	CFG_DEVICE_SIZE_SHIFT = 24,
423	572
	573	+ /* Only for v2.1 */
	574	+ CFG_PAGE_SIZE_SHIFT_v2_1 = 30,
	575	+
424	576	/* Only for pre-v7.1 (with no CFG_EXT register) */
425	577	CFG_PAGE_SIZE_SHIFT = 20,
426	578	CFG_BLK_SIZE_SHIFT = 28,
..	..	@@ -441,27 +593,69 @@
441	593	INTFC_CTLR_READY = BIT(31),
442	594	};
443	595
	596	+/***********************************************************************
	597	+ * NAND ACC CONTROL bitfield
	598	+ *
	599	+ * Some bits have remained constant throughout hardware revision, while
	600	+ * others have shifted around.
	601	+ ***********************************************************************/
	602	+
	603	+/* Constant for all versions (where supported) */
	604	+enum {
	605	+ /* See BRCMNAND_HAS_CACHE_MODE */
	606	+ ACC_CONTROL_CACHE_MODE = BIT(22),
	607	+
	608	+ /* See BRCMNAND_HAS_PREFETCH */
	609	+ ACC_CONTROL_PREFETCH = BIT(23),
	610	+
	611	+ ACC_CONTROL_PAGE_HIT = BIT(24),
	612	+ ACC_CONTROL_WR_PREEMPT = BIT(25),
	613	+ ACC_CONTROL_PARTIAL_PAGE = BIT(26),
	614	+ ACC_CONTROL_RD_ERASED = BIT(27),
	615	+ ACC_CONTROL_FAST_PGM_RDIN = BIT(28),
	616	+ ACC_CONTROL_WR_ECC = BIT(30),
	617	+ ACC_CONTROL_RD_ECC = BIT(31),
	618	+};
	619	+
	620	+#define ACC_CONTROL_ECC_SHIFT 16
	621	+/* Only for v7.2 */
	622	+#define ACC_CONTROL_ECC_EXT_SHIFT 13
	623	+
	624	+static inline bool brcmnand_non_mmio_ops(struct brcmnand_controller *ctrl)
	625	+{
	626	+ return static_branch_unlikely(&brcmnand_soc_has_ops_key);
	627	+}
	628	+
444	629	static inline u32 nand_readreg(struct brcmnand_controller *ctrl, u32 offs)
445	630	{
	631	+ if (brcmnand_non_mmio_ops(ctrl))
	632	+ return brcmnand_soc_read(ctrl->soc, offs);
446	633	return brcmnand_readl(ctrl->nand_base + offs);
447	634	}
448	635
449	636	static inline void nand_writereg(struct brcmnand_controller *ctrl, u32 offs,
450	637	u32 val)
451	638	{
452		- brcmnand_writel(val, ctrl->nand_base + offs);
	639	+ if (brcmnand_non_mmio_ops(ctrl))
	640	+ brcmnand_soc_write(ctrl->soc, val, offs);
	641	+ else
	642	+ brcmnand_writel(val, ctrl->nand_base + offs);
453	643	}
454	644
455	645	static int brcmnand_revision_init(struct brcmnand_controller *ctrl)
456	646	{
457	647	static const unsigned int block_sizes_v6[] = { 8, 16, 128, 256, 512, 1024, 2048, 0 };
458	648	static const unsigned int block_sizes_v4[] = { 16, 128, 8, 512, 256, 1024, 2048, 0 };
459		- static const unsigned int page_sizes[] = { 512, 2048, 4096, 8192, 0 };
	649	+ static const unsigned int block_sizes_v2_2[] = { 16, 128, 8, 512, 256, 0 };
	650	+ static const unsigned int block_sizes_v2_1[] = { 16, 128, 8, 512, 0 };
	651	+ static const unsigned int page_sizes_v3_4[] = { 512, 2048, 4096, 8192, 0 };
	652	+ static const unsigned int page_sizes_v2_2[] = { 512, 2048, 4096, 0 };
	653	+ static const unsigned int page_sizes_v2_1[] = { 512, 2048, 0 };
460	654
461	655	ctrl->nand_version = nand_readreg(ctrl, 0) & 0xffff;
462	656
463		- /* Only support v4.0+? */
464		- if (ctrl->nand_version < 0x0400) {
	657	+ /* Only support v2.1+ */
	658	+ if (ctrl->nand_version < 0x0201) {
465	659	dev_err(ctrl->dev, "version %#x not supported\n",
466	660	ctrl->nand_version);
467	661	return -ENODEV;
..	..	@@ -470,14 +664,16 @@
470	664	/* Register offsets */
471	665	if (ctrl->nand_version >= 0x0702)
472	666	ctrl->reg_offsets = brcmnand_regs_v72;
473		- else if (ctrl->nand_version >= 0x0701)
	667	+ else if (ctrl->nand_version == 0x0701)
474	668	ctrl->reg_offsets = brcmnand_regs_v71;
475	669	else if (ctrl->nand_version >= 0x0600)
476	670	ctrl->reg_offsets = brcmnand_regs_v60;
477	671	else if (ctrl->nand_version >= 0x0500)
478	672	ctrl->reg_offsets = brcmnand_regs_v50;
479		- else if (ctrl->nand_version >= 0x0400)
480		- ctrl->reg_offsets = brcmnand_regs_v40;
	673	+ else if (ctrl->nand_version >= 0x0303)
	674	+ ctrl->reg_offsets = brcmnand_regs_v33;
	675	+ else if (ctrl->nand_version >= 0x0201)
	676	+ ctrl->reg_offsets = brcmnand_regs_v21;
481	677
482	678	/* Chip-select stride */
483	679	if (ctrl->nand_version >= 0x0701)
..	..	@@ -503,20 +699,38 @@
503	699	ctrl->max_page_size = 16 * 1024;
504	700	ctrl->max_block_size = 2 * 1024 * 1024;
505	701	} else {
506		- ctrl->page_sizes = page_sizes;
	702	+ if (ctrl->nand_version >= 0x0304)
	703	+ ctrl->page_sizes = page_sizes_v3_4;
	704	+ else if (ctrl->nand_version >= 0x0202)
	705	+ ctrl->page_sizes = page_sizes_v2_2;
	706	+ else
	707	+ ctrl->page_sizes = page_sizes_v2_1;
	708	+
	709	+ if (ctrl->nand_version >= 0x0202)
	710	+ ctrl->page_size_shift = CFG_PAGE_SIZE_SHIFT;
	711	+ else
	712	+ ctrl->page_size_shift = CFG_PAGE_SIZE_SHIFT_v2_1;
	713	+
507	714	if (ctrl->nand_version >= 0x0600)
508	715	ctrl->block_sizes = block_sizes_v6;
509		- else
	716	+ else if (ctrl->nand_version >= 0x0400)
510	717	ctrl->block_sizes = block_sizes_v4;
	718	+ else if (ctrl->nand_version >= 0x0202)
	719	+ ctrl->block_sizes = block_sizes_v2_2;
	720	+ else
	721	+ ctrl->block_sizes = block_sizes_v2_1;
511	722
512	723	if (ctrl->nand_version < 0x0400) {
513		- ctrl->max_page_size = 4096;
	724	+ if (ctrl->nand_version < 0x0202)
	725	+ ctrl->max_page_size = 2048;
	726	+ else
	727	+ ctrl->max_page_size = 4096;
514	728	ctrl->max_block_size = 512 * 1024;
515	729	}
516	730	}
517	731
518	732	/* Maximum spare area sector size (per 512B) */
519		- if (ctrl->nand_version >= 0x0702)
	733	+ if (ctrl->nand_version == 0x0702)
520	734	ctrl->max_oob = 128;
521	735	else if (ctrl->nand_version >= 0x0600)
522	736	ctrl->max_oob = 64;
..	..	@@ -544,7 +758,24 @@
544	758	else if (of_property_read_bool(ctrl->dev->of_node, "brcm,nand-has-wp"))
545	759	ctrl->features \|= BRCMNAND_HAS_WP;
546	760
	761	+ /* v7.2 has different ecc level shift in the acc register */
	762	+ if (ctrl->nand_version == 0x0702)
	763	+ ctrl->ecc_level_shift = ACC_CONTROL_ECC_EXT_SHIFT;
	764	+ else
	765	+ ctrl->ecc_level_shift = ACC_CONTROL_ECC_SHIFT;
	766	+
547	767	return 0;
	768	+}
	769	+
	770	+static void brcmnand_flash_dma_revision_init(struct brcmnand_controller *ctrl)
	771	+{
	772	+ /* flash_dma register offsets */
	773	+ if (ctrl->nand_version >= 0x0703)
	774	+ ctrl->flash_dma_offsets = flash_dma_regs_v4;
	775	+ else if (ctrl->nand_version == 0x0602)
	776	+ ctrl->flash_dma_offsets = flash_dma_regs_v0;
	777	+ else
	778	+ ctrl->flash_dma_offsets = flash_dma_regs_v1;
548	779	}
549	780
550	781	static inline u32 brcmnand_read_reg(struct brcmnand_controller *ctrl,
..	..	@@ -580,13 +811,34 @@
580	811
581	812	static inline u32 brcmnand_read_fc(struct brcmnand_controller *ctrl, int word)
582	813	{
	814	+ if (brcmnand_non_mmio_ops(ctrl))
	815	+ return brcmnand_soc_read(ctrl->soc, BRCMNAND_NON_MMIO_FC_ADDR);
583	816	return __raw_readl(ctrl->nand_fc + word * 4);
584	817	}
585	818
586	819	static inline void brcmnand_write_fc(struct brcmnand_controller *ctrl,
587	820	int word, u32 val)
588	821	{
589		- __raw_writel(val, ctrl->nand_fc + word * 4);
	822	+ if (brcmnand_non_mmio_ops(ctrl))
	823	+ brcmnand_soc_write(ctrl->soc, val, BRCMNAND_NON_MMIO_FC_ADDR);
	824	+ else
	825	+ __raw_writel(val, ctrl->nand_fc + word * 4);
	826	+}
	827	+
	828	+static inline void edu_writel(struct brcmnand_controller *ctrl,
	829	+ enum edu_reg reg, u32 val)
	830	+{
	831	+ u16 offs = ctrl->edu_offsets[reg];
	832	+
	833	+ brcmnand_writel(val, ctrl->edu_base + offs);
	834	+}
	835	+
	836	+static inline u32 edu_readl(struct brcmnand_controller *ctrl,
	837	+ enum edu_reg reg)
	838	+{
	839	+ u16 offs = ctrl->edu_offsets[reg];
	840	+
	841	+ return brcmnand_readl(ctrl->edu_base + offs);
590	842	}
591	843
592	844	static void brcmnand_clear_ecc_addr(struct brcmnand_controller *ctrl)
..	..	@@ -669,7 +921,10 @@
669	921	enum brcmnand_reg reg = BRCMNAND_CORR_THRESHOLD;
670	922	int cs = host->cs;
671	923
672		- if (ctrl->nand_version >= 0x0702)
	924	+ if (!ctrl->reg_offsets[reg])
	925	+ return;
	926	+
	927	+ if (ctrl->nand_version == 0x0702)
673	928	bits = 7;
674	929	else if (ctrl->nand_version >= 0x0600)
675	930	bits = 6;
..	..	@@ -697,52 +952,27 @@
697	952	return 0;
698	953	}
699	954
700		-/***********************************************************************
701		- * NAND ACC CONTROL bitfield
702		- *
703		- * Some bits have remained constant throughout hardware revision, while
704		- * others have shifted around.
705		- ***********************************************************************/
706		-
707		-/* Constant for all versions (where supported) */
708		-enum {
709		- /* See BRCMNAND_HAS_CACHE_MODE */
710		- ACC_CONTROL_CACHE_MODE = BIT(22),
711		-
712		- /* See BRCMNAND_HAS_PREFETCH */
713		- ACC_CONTROL_PREFETCH = BIT(23),
714		-
715		- ACC_CONTROL_PAGE_HIT = BIT(24),
716		- ACC_CONTROL_WR_PREEMPT = BIT(25),
717		- ACC_CONTROL_PARTIAL_PAGE = BIT(26),
718		- ACC_CONTROL_RD_ERASED = BIT(27),
719		- ACC_CONTROL_FAST_PGM_RDIN = BIT(28),
720		- ACC_CONTROL_WR_ECC = BIT(30),
721		- ACC_CONTROL_RD_ECC = BIT(31),
722		-};
723		-
724	955	static inline u32 brcmnand_spare_area_mask(struct brcmnand_controller *ctrl)
725	956	{
726		- if (ctrl->nand_version >= 0x0702)
	957	+ if (ctrl->nand_version == 0x0702)
727	958	return GENMASK(7, 0);
728	959	else if (ctrl->nand_version >= 0x0600)
729	960	return GENMASK(6, 0);
730		- else
	961	+ else if (ctrl->nand_version >= 0x0303)
731	962	return GENMASK(5, 0);
	963	+ else
	964	+ return GENMASK(4, 0);
732	965	}
733		-
734		-#define NAND_ACC_CONTROL_ECC_SHIFT 16
735		-#define NAND_ACC_CONTROL_ECC_EXT_SHIFT 13
736	966
737	967	static inline u32 brcmnand_ecc_level_mask(struct brcmnand_controller *ctrl)
738	968	{
739	969	u32 mask = (ctrl->nand_version >= 0x0600) ? 0x1f : 0x0f;
740	970
741		- mask <<= NAND_ACC_CONTROL_ECC_SHIFT;
	971	+ mask <<= ACC_CONTROL_ECC_SHIFT;
742	972
743	973	/* v7.2 includes additional ECC levels */
744		- if (ctrl->nand_version >= 0x0702)
745		- mask \|= 0x7 << NAND_ACC_CONTROL_ECC_EXT_SHIFT;
	974	+ if (ctrl->nand_version == 0x0702)
	975	+ mask \|= 0x7 << ACC_CONTROL_ECC_EXT_SHIFT;
746	976
747	977	return mask;
748	978	}
..	..	@@ -756,8 +986,8 @@
756	986
757	987	if (en) {
758	988	acc_control \|= ecc_flags; /* enable RD/WR ECC */
759		- acc_control \|= host->hwcfg.ecc_level
760		- << NAND_ACC_CONTROL_ECC_SHIFT;
	989	+ acc_control &= ~brcmnand_ecc_level_mask(ctrl);
	990	+ acc_control \|= host->hwcfg.ecc_level << ctrl->ecc_level_shift;
761	991	} else {
762	992	acc_control &= ~ecc_flags; /* disable RD/WR ECC */
763	993	acc_control &= ~brcmnand_ecc_level_mask(ctrl);
..	..	@@ -836,6 +1066,14 @@
836	1066	cpu_relax();
837	1067	} while (time_after(limit, jiffies));
838	1068
	1069	+ /*
	1070	+ * do a final check after time out in case the CPU was busy and the driver
	1071	+ * did not get enough time to perform the polling to avoid false alarms
	1072	+ */
	1073	+ val = brcmnand_read_reg(ctrl, BRCMNAND_INTFC_STATUS);
	1074	+ if ((val & mask) == expected_val)
	1075	+ return 0;
	1076	+
839	1077	dev_warn(ctrl->dev, "timeout on status poll (expected %x got %x)\n",
840	1078	expected_val, val & mask);
841	1079
..	..	@@ -853,23 +1091,33 @@
853	1091	* Flash DMA
854	1092	***********************************************************************/
855	1093
856		-enum flash_dma_reg {
857		- FLASH_DMA_REVISION = 0x00,
858		- FLASH_DMA_FIRST_DESC = 0x04,
859		- FLASH_DMA_FIRST_DESC_EXT = 0x08,
860		- FLASH_DMA_CTRL = 0x0c,
861		- FLASH_DMA_MODE = 0x10,
862		- FLASH_DMA_STATUS = 0x14,
863		- FLASH_DMA_INTERRUPT_DESC = 0x18,
864		- FLASH_DMA_INTERRUPT_DESC_EXT = 0x1c,
865		- FLASH_DMA_ERROR_STATUS = 0x20,
866		- FLASH_DMA_CURRENT_DESC = 0x24,
867		- FLASH_DMA_CURRENT_DESC_EXT = 0x28,
868		-};
869		-
870	1094	static inline bool has_flash_dma(struct brcmnand_controller *ctrl)
871	1095	{
872	1096	return ctrl->flash_dma_base;
	1097	+}
	1098	+
	1099	+static inline bool has_edu(struct brcmnand_controller *ctrl)
	1100	+{
	1101	+ return ctrl->edu_base;
	1102	+}
	1103	+
	1104	+static inline bool use_dma(struct brcmnand_controller *ctrl)
	1105	+{
	1106	+ return has_flash_dma(ctrl) \|\| has_edu(ctrl);
	1107	+}
	1108	+
	1109	+static inline void disable_ctrl_irqs(struct brcmnand_controller *ctrl)
	1110	+{
	1111	+ if (ctrl->pio_poll_mode)
	1112	+ return;
	1113	+
	1114	+ if (has_flash_dma(ctrl)) {
	1115	+ ctrl->flash_dma_base = NULL;
	1116	+ disable_irq(ctrl->dma_irq);
	1117	+ }
	1118	+
	1119	+ disable_irq(ctrl->irq);
	1120	+ ctrl->pio_poll_mode = true;
873	1121	}
874	1122
875	1123	static inline bool flash_dma_buf_ok(const void *buf)
..	..	@@ -878,14 +1126,19 @@
878	1126	likely(IS_ALIGNED((uintptr_t)buf, 4));
879	1127	}
880	1128
881		-static inline void flash_dma_writel(struct brcmnand_controller *ctrl, u8 offs,
882		- u32 val)
	1129	+static inline void flash_dma_writel(struct brcmnand_controller *ctrl,
	1130	+ enum flash_dma_reg dma_reg, u32 val)
883	1131	{
	1132	+ u16 offs = ctrl->flash_dma_offsets[dma_reg];
	1133	+
884	1134	brcmnand_writel(val, ctrl->flash_dma_base + offs);
885	1135	}
886	1136
887		-static inline u32 flash_dma_readl(struct brcmnand_controller *ctrl, u8 offs)
	1137	+static inline u32 flash_dma_readl(struct brcmnand_controller *ctrl,
	1138	+ enum flash_dma_reg dma_reg)
888	1139	{
	1140	+ u16 offs = ctrl->flash_dma_offsets[dma_reg];
	1141	+
889	1142	return brcmnand_readl(ctrl->flash_dma_base + offs);
890	1143	}
891	1144
..	..	@@ -944,32 +1197,29 @@
944	1197	struct brcmnand_cfg *cfg = &host->hwcfg;
945	1198	int sas = cfg->spare_area_size << cfg->sector_size_1k;
946	1199	int sectors = cfg->page_size / (512 << cfg->sector_size_1k);
	1200	+ u32 next;
947	1201
948		- if (section >= sectors * 2)
	1202	+ if (section > sectors)
949	1203	return -ERANGE;
950	1204
951		- oobregion->offset = (section / 2) * sas;
	1205	+ next = (section * sas);
	1206	+ if (section < sectors)
	1207	+ next += 6;
952	1208
953		- if (section & 1) {
954		- oobregion->offset += 9;
955		- oobregion->length = 7;
	1209	+ if (section) {
	1210	+ oobregion->offset = ((section - 1) * sas) + 9;
956	1211	} else {
957		- oobregion->length = 6;
958		-
959		- /* First sector of each page may have BBI */
960		- if (!section) {
961		- /*
962		- * Small-page NAND use byte 6 for BBI while large-page
963		- * NAND use bytes 0 and 1.
964		- */
965		- if (cfg->page_size > 512) {
966		- oobregion->offset += 2;
967		- oobregion->length -= 2;
968		- } else {
969		- oobregion->length--;
970		- }
	1212	+ if (cfg->page_size > 512) {
	1213	+ /* Large page NAND uses first 2 bytes for BBI */
	1214	+ oobregion->offset = 2;
	1215	+ } else {
	1216	+ /* Small page NAND uses last byte before ECC for BBI */
	1217	+ oobregion->offset = 0;
	1218	+ next--;
971	1219	}
972	1220	}
	1221	+
	1222	+ oobregion->length = next - oobregion->offset;
973	1223
974	1224	return 0;
975	1225	}
..	..	@@ -991,7 +1241,7 @@
991	1241	if (section >= sectors)
992	1242	return -ERANGE;
993	1243
994		- oobregion->offset = (section * (sas + 1)) - chip->ecc.bytes;
	1244	+ oobregion->offset = ((section + 1) * sas) - chip->ecc.bytes;
995	1245	oobregion->length = chip->ecc.bytes;
996	1246
997	1247	return 0;
..	..	@@ -1213,20 +1463,80 @@
1213	1463	const u8 *oob, int sas, int sector_1k)
1214	1464	{
1215	1465	int tbytes = sas << sector_1k;
1216		- int j;
	1466	+ int j, k = 0;
	1467	+ u32 last = 0xffffffff;
	1468	+ u8 plast = (u8 )&last;
1217	1469
1218	1470	/* Adjust OOB values for 1K sector size */
1219	1471	if (sector_1k && (i & 0x01))
1220	1472	tbytes = max(0, tbytes - (int)ctrl->max_oob);
1221	1473	tbytes = min_t(int, tbytes, ctrl->max_oob);
1222	1474
1223		- for (j = 0; j < tbytes; j += 4)
	1475	+ /*
	1476	+ * tbytes may not be multiple of words. Make sure we don't read out of
	1477	+ * the boundary and stop at last word.
	1478	+ */
	1479	+ for (j = 0; (j + 3) < tbytes; j += 4)
1224	1480	oob_reg_write(ctrl, j,
1225	1481	(oob[j + 0] << 24) \|
1226	1482	(oob[j + 1] << 16) \|
1227	1483	(oob[j + 2] << 8) \|
1228	1484	(oob[j + 3] << 0));
	1485	+
	1486	+ /* handle the remaing bytes */
	1487	+ while (j < tbytes)
	1488	+ plast[k++] = oob[j++];
	1489	+
	1490	+ if (tbytes & 0x3)
	1491	+ oob_reg_write(ctrl, (tbytes & ~0x3), (__force u32)cpu_to_be32(last));
	1492	+
1229	1493	return tbytes;
	1494	+}
	1495	+
	1496	+static void brcmnand_edu_init(struct brcmnand_controller *ctrl)
	1497	+{
	1498	+ /* initialize edu */
	1499	+ edu_writel(ctrl, EDU_ERR_STATUS, 0);
	1500	+ edu_readl(ctrl, EDU_ERR_STATUS);
	1501	+ edu_writel(ctrl, EDU_DONE, 0);
	1502	+ edu_writel(ctrl, EDU_DONE, 0);
	1503	+ edu_writel(ctrl, EDU_DONE, 0);
	1504	+ edu_writel(ctrl, EDU_DONE, 0);
	1505	+ edu_readl(ctrl, EDU_DONE);
	1506	+}
	1507	+
	1508	+/* edu irq */
	1509	+static irqreturn_t brcmnand_edu_irq(int irq, void *data)
	1510	+{
	1511	+ struct brcmnand_controller *ctrl = data;
	1512	+
	1513	+ if (ctrl->edu_count) {
	1514	+ ctrl->edu_count--;
	1515	+ while (!(edu_readl(ctrl, EDU_DONE) & EDU_DONE_MASK))
	1516	+ udelay(1);
	1517	+ edu_writel(ctrl, EDU_DONE, 0);
	1518	+ edu_readl(ctrl, EDU_DONE);
	1519	+ }
	1520	+
	1521	+ if (ctrl->edu_count) {
	1522	+ ctrl->edu_dram_addr += FC_BYTES;
	1523	+ ctrl->edu_ext_addr += FC_BYTES;
	1524	+
	1525	+ edu_writel(ctrl, EDU_DRAM_ADDR, (u32)ctrl->edu_dram_addr);
	1526	+ edu_readl(ctrl, EDU_DRAM_ADDR);
	1527	+ edu_writel(ctrl, EDU_EXT_ADDR, ctrl->edu_ext_addr);
	1528	+ edu_readl(ctrl, EDU_EXT_ADDR);
	1529	+
	1530	+ mb(); /* flush previous writes */
	1531	+ edu_writel(ctrl, EDU_CMD, ctrl->edu_cmd);
	1532	+ edu_readl(ctrl, EDU_CMD);
	1533	+
	1534	+ return IRQ_HANDLED;
	1535	+ }
	1536	+
	1537	+ complete(&ctrl->edu_done);
	1538	+
	1539	+ return IRQ_HANDLED;
1230	1540	}
1231	1541
1232	1542	static irqreturn_t brcmnand_ctlrdy_irq(int irq, void *data)
..	..	@@ -1236,6 +1546,16 @@
1236	1546	/* Discard all NAND_CTLRDY interrupts during DMA */
1237	1547	if (ctrl->dma_pending)
1238	1548	return IRQ_HANDLED;
	1549	+
	1550	+ /* check if you need to piggy back on the ctrlrdy irq */
	1551	+ if (ctrl->edu_pending) {
	1552	+ if (irq == ctrl->irq && ((int)ctrl->edu_irq >= 0))
	1553	+ /* Discard interrupts while using dedicated edu irq */
	1554	+ return IRQ_HANDLED;
	1555	+
	1556	+ /* no registered edu irq, call handler */
	1557	+ return brcmnand_edu_irq(irq, data);
	1558	+ }
1239	1559
1240	1560	complete(&ctrl->done);
1241	1561	return IRQ_HANDLED;
..	..	@@ -1271,7 +1591,17 @@
1271	1591
1272	1592	dev_dbg(ctrl->dev, "send native cmd %d addr 0x%llx\n", cmd, cmd_addr);
1273	1593
1274		- BUG_ON(ctrl->cmd_pending != 0);
	1594	+ /*
	1595	+ * If we came here through _panic_write and there is a pending
	1596	+ * command, try to wait for it. If it times out, rather than
	1597	+ * hitting BUG_ON, just return so we don't crash while crashing.
	1598	+ */
	1599	+ if (oops_in_progress) {
	1600	+ if (ctrl->cmd_pending &&
	1601	+ bcmnand_ctrl_poll_status(ctrl, NAND_CTRL_RDY, NAND_CTRL_RDY, 0))
	1602	+ return;
	1603	+ } else
	1604	+ BUG_ON(ctrl->cmd_pending != 0);
1275	1605	ctrl->cmd_pending = cmd;
1276	1606
1277	1607	ret = bcmnand_ctrl_poll_status(ctrl, NAND_CTRL_RDY, NAND_CTRL_RDY, 0);
..	..	@@ -1286,22 +1616,48 @@
1286	1616	* NAND MTD API: read/program/erase
1287	1617	***********************************************************************/
1288	1618
1289		-static void brcmnand_cmd_ctrl(struct mtd_info *mtd, int dat,
1290		- unsigned int ctrl)
	1619	+static void brcmnand_cmd_ctrl(struct nand_chip *chip, int dat,
	1620	+ unsigned int ctrl)
1291	1621	{
1292	1622	/* intentionally left blank */
1293	1623	}
1294	1624
1295		-static int brcmnand_waitfunc(struct mtd_info mtd, struct nand_chip this)
	1625	+static bool brcmstb_nand_wait_for_completion(struct nand_chip *chip)
1296	1626	{
1297		- struct nand_chip *chip = mtd_to_nand(mtd);
1298	1627	struct brcmnand_host *host = nand_get_controller_data(chip);
1299	1628	struct brcmnand_controller *ctrl = host->ctrl;
1300		- unsigned long timeo = msecs_to_jiffies(100);
	1629	+ struct mtd_info *mtd = nand_to_mtd(chip);
	1630	+ bool err = false;
	1631	+ int sts;
	1632	+
	1633	+ if (mtd->oops_panic_write) {
	1634	+ /* switch to interrupt polling and PIO mode */
	1635	+ disable_ctrl_irqs(ctrl);
	1636	+ sts = bcmnand_ctrl_poll_status(ctrl, NAND_CTRL_RDY,
	1637	+ NAND_CTRL_RDY, 0);
	1638	+ err = (sts < 0) ? true : false;
	1639	+ } else {
	1640	+ unsigned long timeo = msecs_to_jiffies(
	1641	+ NAND_POLL_STATUS_TIMEOUT_MS);
	1642	+ /* wait for completion interrupt */
	1643	+ sts = wait_for_completion_timeout(&ctrl->done, timeo);
	1644	+ err = (sts <= 0) ? true : false;
	1645	+ }
	1646	+
	1647	+ return err;
	1648	+}
	1649	+
	1650	+static int brcmnand_waitfunc(struct nand_chip *chip)
	1651	+{
	1652	+ struct brcmnand_host *host = nand_get_controller_data(chip);
	1653	+ struct brcmnand_controller *ctrl = host->ctrl;
	1654	+ bool err = false;
1301	1655
1302	1656	dev_dbg(ctrl->dev, "wait on native cmd %d\n", ctrl->cmd_pending);
1303		- if (ctrl->cmd_pending &&
1304		- wait_for_completion_timeout(&ctrl->done, timeo) <= 0) {
	1657	+ if (ctrl->cmd_pending)
	1658	+ err = brcmstb_nand_wait_for_completion(chip);
	1659	+
	1660	+ if (err) {
1305	1661	u32 cmd = brcmnand_read_reg(ctrl, BRCMNAND_CMD_START)
1306	1662	>> brcmnand_cmd_shift(ctrl);
1307	1663
..	..	@@ -1329,7 +1685,6 @@
1329	1685	enum brcmnand_llop_type type, u32 data,
1330	1686	bool last_op)
1331	1687	{
1332		- struct mtd_info *mtd = nand_to_mtd(&host->chip);
1333	1688	struct nand_chip *chip = &host->chip;
1334	1689	struct brcmnand_controller *ctrl = host->ctrl;
1335	1690	u32 tmp;
..	..	@@ -1362,13 +1717,13 @@
1362	1717	(void)brcmnand_read_reg(ctrl, BRCMNAND_LL_OP);
1363	1718
1364	1719	brcmnand_send_cmd(host, CMD_LOW_LEVEL_OP);
1365		- return brcmnand_waitfunc(mtd, chip);
	1720	+ return brcmnand_waitfunc(chip);
1366	1721	}
1367	1722
1368		-static void brcmnand_cmdfunc(struct mtd_info *mtd, unsigned command,
	1723	+static void brcmnand_cmdfunc(struct nand_chip *chip, unsigned command,
1369	1724	int column, int page_addr)
1370	1725	{
1371		- struct nand_chip *chip = mtd_to_nand(mtd);
	1726	+ struct mtd_info *mtd = nand_to_mtd(chip);
1372	1727	struct brcmnand_host *host = nand_get_controller_data(chip);
1373	1728	struct brcmnand_controller *ctrl = host->ctrl;
1374	1729	u64 addr = (u64)page_addr << chip->page_shift;
..	..	@@ -1433,7 +1788,7 @@
1433	1788
1434	1789	brcmnand_set_cmd_addr(mtd, addr);
1435	1790	brcmnand_send_cmd(host, native_cmd);
1436		- brcmnand_waitfunc(mtd, chip);
	1791	+ brcmnand_waitfunc(chip);
1437	1792
1438	1793	if (native_cmd == CMD_PARAMETER_READ \|\|
1439	1794	native_cmd == CMD_PARAMETER_CHANGE_COL) {
..	..	@@ -1467,9 +1822,8 @@
1467	1822	brcmnand_wp(mtd, 1);
1468	1823	}
1469	1824
1470		-static uint8_t brcmnand_read_byte(struct mtd_info *mtd)
	1825	+static uint8_t brcmnand_read_byte(struct nand_chip *chip)
1471	1826	{
1472		- struct nand_chip *chip = mtd_to_nand(mtd);
1473	1827	struct brcmnand_host *host = nand_get_controller_data(chip);
1474	1828	struct brcmnand_controller *ctrl = host->ctrl;
1475	1829	uint8_t ret = 0;
..	..	@@ -1524,19 +1878,18 @@
1524	1878	return ret;
1525	1879	}
1526	1880
1527		-static void brcmnand_read_buf(struct mtd_info mtd, uint8_t buf, int len)
	1881	+static void brcmnand_read_buf(struct nand_chip chip, uint8_t buf, int len)
1528	1882	{
1529	1883	int i;
1530	1884
1531	1885	for (i = 0; i < len; i++, buf++)
1532		- *buf = brcmnand_read_byte(mtd);
	1886	+ *buf = brcmnand_read_byte(chip);
1533	1887	}
1534	1888
1535		-static void brcmnand_write_buf(struct mtd_info mtd, const uint8_t buf,
1536		- int len)
	1889	+static void brcmnand_write_buf(struct nand_chip chip, const uint8_t buf,
	1890	+ int len)
1537	1891	{
1538	1892	int i;
1539		- struct nand_chip *chip = mtd_to_nand(mtd);
1540	1893	struct brcmnand_host *host = nand_get_controller_data(chip);
1541	1894
1542	1895	switch (host->last_cmd) {
..	..	@@ -1549,6 +1902,97 @@
1549	1902	BUG();
1550	1903	break;
1551	1904	}
	1905	+}
	1906	+
	1907	+/**
	1908	+ * Kick EDU engine
	1909	+ */
	1910	+static int brcmnand_edu_trans(struct brcmnand_host host, u64 addr, u32 buf,
	1911	+ u32 len, u8 cmd)
	1912	+{
	1913	+ struct brcmnand_controller *ctrl = host->ctrl;
	1914	+ unsigned long timeo = msecs_to_jiffies(200);
	1915	+ int ret = 0;
	1916	+ int dir = (cmd == CMD_PAGE_READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
	1917	+ u8 edu_cmd = (cmd == CMD_PAGE_READ ? EDU_CMD_READ : EDU_CMD_WRITE);
	1918	+ unsigned int trans = len >> FC_SHIFT;
	1919	+ dma_addr_t pa;
	1920	+
	1921	+ pa = dma_map_single(ctrl->dev, buf, len, dir);
	1922	+ if (dma_mapping_error(ctrl->dev, pa)) {
	1923	+ dev_err(ctrl->dev, "unable to map buffer for EDU DMA\n");
	1924	+ return -ENOMEM;
	1925	+ }
	1926	+
	1927	+ ctrl->edu_pending = true;
	1928	+ ctrl->edu_dram_addr = pa;
	1929	+ ctrl->edu_ext_addr = addr;
	1930	+ ctrl->edu_cmd = edu_cmd;
	1931	+ ctrl->edu_count = trans;
	1932	+
	1933	+ edu_writel(ctrl, EDU_DRAM_ADDR, (u32)ctrl->edu_dram_addr);
	1934	+ edu_readl(ctrl, EDU_DRAM_ADDR);
	1935	+ edu_writel(ctrl, EDU_EXT_ADDR, ctrl->edu_ext_addr);
	1936	+ edu_readl(ctrl, EDU_EXT_ADDR);
	1937	+ edu_writel(ctrl, EDU_LENGTH, FC_BYTES);
	1938	+ edu_readl(ctrl, EDU_LENGTH);
	1939	+
	1940	+ /* Start edu engine */
	1941	+ mb(); /* flush previous writes */
	1942	+ edu_writel(ctrl, EDU_CMD, ctrl->edu_cmd);
	1943	+ edu_readl(ctrl, EDU_CMD);
	1944	+
	1945	+ if (wait_for_completion_timeout(&ctrl->edu_done, timeo) <= 0) {
	1946	+ dev_err(ctrl->dev,
	1947	+ "timeout waiting for EDU; status %#x, error status %#x\n",
	1948	+ edu_readl(ctrl, EDU_STATUS),
	1949	+ edu_readl(ctrl, EDU_ERR_STATUS));
	1950	+ }
	1951	+
	1952	+ dma_unmap_single(ctrl->dev, pa, len, dir);
	1953	+
	1954	+ /* for program page check NAND status */
	1955	+ if (((brcmnand_read_reg(ctrl, BRCMNAND_INTFC_STATUS) &
	1956	+ INTFC_FLASH_STATUS) & NAND_STATUS_FAIL) &&
	1957	+ edu_cmd == EDU_CMD_WRITE) {
	1958	+ dev_info(ctrl->dev, "program failed at %llx\n",
	1959	+ (unsigned long long)addr);
	1960	+ ret = -EIO;
	1961	+ }
	1962	+
	1963	+ /* Make sure the EDU status is clean */
	1964	+ if (edu_readl(ctrl, EDU_STATUS) & EDU_STATUS_ACTIVE)
	1965	+ dev_warn(ctrl->dev, "EDU still active: %#x\n",
	1966	+ edu_readl(ctrl, EDU_STATUS));
	1967	+
	1968	+ if (unlikely(edu_readl(ctrl, EDU_ERR_STATUS) & EDU_ERR_STATUS_ERRACK)) {
	1969	+ dev_warn(ctrl->dev, "EDU RBUS error at addr %llx\n",
	1970	+ (unsigned long long)addr);
	1971	+ ret = -EIO;
	1972	+ }
	1973	+
	1974	+ ctrl->edu_pending = false;
	1975	+ brcmnand_edu_init(ctrl);
	1976	+ edu_writel(ctrl, EDU_STOP, 0); /* force stop */
	1977	+ edu_readl(ctrl, EDU_STOP);
	1978	+
	1979	+ if (!ret && edu_cmd == EDU_CMD_READ) {
	1980	+ u64 err_addr = 0;
	1981	+
	1982	+ /*
	1983	+ * check for ECC errors here, subpage ECC errors are
	1984	+ * retained in ECC error address register
	1985	+ */
	1986	+ err_addr = brcmnand_get_uncorrecc_addr(ctrl);
	1987	+ if (!err_addr) {
	1988	+ err_addr = brcmnand_get_correcc_addr(ctrl);
	1989	+ if (err_addr)
	1990	+ ret = -EUCLEAN;
	1991	+ } else
	1992	+ ret = -EBADMSG;
	1993	+ }
	1994	+
	1995	+ return ret;
1552	1996	}
1553	1997
1554	1998	/**
..	..	@@ -1594,8 +2038,11 @@
1594	2038
1595	2039	flash_dma_writel(ctrl, FLASH_DMA_FIRST_DESC, lower_32_bits(desc));
1596	2040	(void)flash_dma_readl(ctrl, FLASH_DMA_FIRST_DESC);
1597		- flash_dma_writel(ctrl, FLASH_DMA_FIRST_DESC_EXT, upper_32_bits(desc));
1598		- (void)flash_dma_readl(ctrl, FLASH_DMA_FIRST_DESC_EXT);
	2041	+ if (ctrl->nand_version > 0x0602) {
	2042	+ flash_dma_writel(ctrl, FLASH_DMA_FIRST_DESC_EXT,
	2043	+ upper_32_bits(desc));
	2044	+ (void)flash_dma_readl(ctrl, FLASH_DMA_FIRST_DESC_EXT);
	2045	+ }
1599	2046
1600	2047	/* Start FLASH_DMA engine */
1601	2048	ctrl->dma_pending = true;
..	..	@@ -1657,7 +2104,7 @@
1657	2104	brcmnand_set_cmd_addr(mtd, addr);
1658	2105	/* SPARE_AREA_READ does not use ECC, so just use PAGE_READ */
1659	2106	brcmnand_send_cmd(host, CMD_PAGE_READ);
1660		- brcmnand_waitfunc(mtd, chip);
	2107	+ brcmnand_waitfunc(chip);
1661	2108
1662	2109	if (likely(buf)) {
1663	2110	brcmnand_soc_data_bus_prepare(ctrl->soc, false);
..	..	@@ -1708,28 +2155,31 @@
1708	2155	static int brcmstb_nand_verify_erased_page(struct mtd_info *mtd,
1709	2156	struct nand_chip chip, void buf, u64 addr)
1710	2157	{
1711		- int i, sas;
1712		- void *oob = chip->oob_poi;
	2158	+ struct mtd_oob_region ecc;
	2159	+ int i;
1713	2160	int bitflips = 0;
1714	2161	int page = addr >> chip->page_shift;
1715	2162	int ret;
	2163	+ void *ecc_bytes;
	2164	+ void *ecc_chunk;
1716	2165
1717		- if (!buf) {
1718		- buf = chip->data_buf;
1719		- /* Invalidate page cache */
1720		- chip->pagebuf = -1;
1721		- }
1722		-
1723		- sas = mtd->oobsize / chip->ecc.steps;
	2166	+ if (!buf)
	2167	+ buf = nand_get_data_buf(chip);
1724	2168
1725	2169	/* read without ecc for verification */
1726		- ret = chip->ecc.read_page_raw(mtd, chip, buf, true, page);
	2170	+ ret = chip->ecc.read_page_raw(chip, buf, true, page);
1727	2171	if (ret)
1728	2172	return ret;
1729	2173
1730		- for (i = 0; i < chip->ecc.steps; i++, oob += sas) {
1731		- ret = nand_check_erased_ecc_chunk(buf, chip->ecc.size,
1732		- oob, sas, NULL, 0,
	2174	+ for (i = 0; i < chip->ecc.steps; i++) {
	2175	+ ecc_chunk = buf + chip->ecc.size * i;
	2176	+
	2177	+ mtd_ooblayout_ecc(mtd, i, &ecc);
	2178	+ ecc_bytes = chip->oob_poi + ecc.offset;
	2179	+
	2180	+ ret = nand_check_erased_ecc_chunk(ecc_chunk, chip->ecc.size,
	2181	+ ecc_bytes, ecc.length,
	2182	+ NULL, 0,
1733	2183	chip->ecc.strength);
1734	2184	if (ret < 0)
1735	2185	return ret;
..	..	@@ -1748,21 +2198,28 @@
1748	2198	u64 err_addr = 0;
1749	2199	int err;
1750	2200	bool retry = true;
	2201	+ bool edu_err = false;
1751	2202
1752	2203	dev_dbg(ctrl->dev, "read %llx -> %p\n", (unsigned long long)addr, buf);
1753	2204
1754	2205	try_dmaread:
1755	2206	brcmnand_clear_ecc_addr(ctrl);
1756	2207
1757		- if (has_flash_dma(ctrl) && !oob && flash_dma_buf_ok(buf)) {
1758		- err = brcmnand_dma_trans(host, addr, buf, trans * FC_BYTES,
1759		- CMD_PAGE_READ);
	2208	+ if (ctrl->dma_trans && !oob && flash_dma_buf_ok(buf)) {
	2209	+ err = ctrl->dma_trans(host, addr, buf,
	2210	+ trans * FC_BYTES,
	2211	+ CMD_PAGE_READ);
	2212	+
1760	2213	if (err) {
1761	2214	if (mtd_is_bitflip_or_eccerr(err))
1762	2215	err_addr = addr;
1763	2216	else
1764	2217	return -EIO;
1765	2218	}
	2219	+
	2220	+ if (has_edu(ctrl) && err_addr)
	2221	+ edu_err = true;
	2222	+
1766	2223	} else {
1767	2224	if (oob)
1768	2225	memset(oob, 0x99, mtd->oobsize);
..	..	@@ -1810,6 +2267,11 @@
1810	2267	if (mtd_is_bitflip(err)) {
1811	2268	unsigned int corrected = brcmnand_count_corrected(ctrl);
1812	2269
	2270	+ /* in case of EDU correctable error we read again using PIO */
	2271	+ if (edu_err)
	2272	+ err = brcmnand_read_by_pio(mtd, chip, addr, trans, buf,
	2273	+ oob, &err_addr);
	2274	+
1813	2275	dev_dbg(ctrl->dev, "corrected error at 0x%llx\n",
1814	2276	(unsigned long long)err_addr);
1815	2277	mtd->ecc_stats.corrected += corrected;
..	..	@@ -1820,9 +2282,10 @@
1820	2282	return 0;
1821	2283	}
1822	2284
1823		-static int brcmnand_read_page(struct mtd_info mtd, struct nand_chip chip,
1824		- uint8_t *buf, int oob_required, int page)
	2285	+static int brcmnand_read_page(struct nand_chip chip, uint8_t buf,
	2286	+ int oob_required, int page)
1825	2287	{
	2288	+ struct mtd_info *mtd = nand_to_mtd(chip);
1826	2289	struct brcmnand_host *host = nand_get_controller_data(chip);
1827	2290	u8 oob = oob_required ? (u8 )chip->oob_poi : NULL;
1828	2291
..	..	@@ -1832,10 +2295,11 @@
1832	2295	mtd->writesize >> FC_SHIFT, (u32 *)buf, oob);
1833	2296	}
1834	2297
1835		-static int brcmnand_read_page_raw(struct mtd_info mtd, struct nand_chip chip,
1836		- uint8_t *buf, int oob_required, int page)
	2298	+static int brcmnand_read_page_raw(struct nand_chip chip, uint8_t buf,
	2299	+ int oob_required, int page)
1837	2300	{
1838	2301	struct brcmnand_host *host = nand_get_controller_data(chip);
	2302	+ struct mtd_info *mtd = nand_to_mtd(chip);
1839	2303	u8 oob = oob_required ? (u8 )chip->oob_poi : NULL;
1840	2304	int ret;
1841	2305
..	..	@@ -1848,17 +2312,18 @@
1848	2312	return ret;
1849	2313	}
1850	2314
1851		-static int brcmnand_read_oob(struct mtd_info mtd, struct nand_chip chip,
1852		- int page)
	2315	+static int brcmnand_read_oob(struct nand_chip *chip, int page)
1853	2316	{
	2317	+ struct mtd_info *mtd = nand_to_mtd(chip);
	2318	+
1854	2319	return brcmnand_read(mtd, chip, (u64)page << chip->page_shift,
1855	2320	mtd->writesize >> FC_SHIFT,
1856	2321	NULL, (u8 *)chip->oob_poi);
1857	2322	}
1858	2323
1859		-static int brcmnand_read_oob_raw(struct mtd_info mtd, struct nand_chip chip,
1860		- int page)
	2324	+static int brcmnand_read_oob_raw(struct nand_chip *chip, int page)
1861	2325	{
	2326	+ struct mtd_info *mtd = nand_to_mtd(chip);
1862	2327	struct brcmnand_host *host = nand_get_controller_data(chip);
1863	2328
1864	2329	brcmnand_set_ecc_enabled(host, 0);
..	..	@@ -1889,10 +2354,12 @@
1889	2354	for (i = 0; i < ctrl->max_oob; i += 4)
1890	2355	oob_reg_write(ctrl, i, 0xffffffff);
1891	2356
1892		- if (has_flash_dma(ctrl) && !oob && flash_dma_buf_ok(buf)) {
1893		- if (brcmnand_dma_trans(host, addr, (u32 *)buf,
1894		- mtd->writesize, CMD_PROGRAM_PAGE))
	2357	+ if (use_dma(ctrl) && !oob && flash_dma_buf_ok(buf)) {
	2358	+ if (ctrl->dma_trans(host, addr, (u32 *)buf, mtd->writesize,
	2359	+ CMD_PROGRAM_PAGE))
	2360	+
1895	2361	ret = -EIO;
	2362	+
1896	2363	goto out;
1897	2364	}
1898	2365
..	..	@@ -1920,7 +2387,7 @@
1920	2387
1921	2388	/* we cannot use SPARE_AREA_PROGRAM when PARTIAL_PAGE_EN=0 */
1922	2389	brcmnand_send_cmd(host, CMD_PROGRAM_PAGE);
1923		- status = brcmnand_waitfunc(mtd, chip);
	2390	+ status = brcmnand_waitfunc(chip);
1924	2391
1925	2392	if (status & NAND_STATUS_FAIL) {
1926	2393	dev_info(ctrl->dev, "program failed at %llx\n",
..	..	@@ -1934,9 +2401,10 @@
1934	2401	return ret;
1935	2402	}
1936	2403
1937		-static int brcmnand_write_page(struct mtd_info mtd, struct nand_chip chip,
1938		- const uint8_t *buf, int oob_required, int page)
	2404	+static int brcmnand_write_page(struct nand_chip chip, const uint8_t buf,
	2405	+ int oob_required, int page)
1939	2406	{
	2407	+ struct mtd_info *mtd = nand_to_mtd(chip);
1940	2408	struct brcmnand_host *host = nand_get_controller_data(chip);
1941	2409	void *oob = oob_required ? chip->oob_poi : NULL;
1942	2410
..	..	@@ -1946,10 +2414,10 @@
1946	2414	return nand_prog_page_end_op(chip);
1947	2415	}
1948	2416
1949		-static int brcmnand_write_page_raw(struct mtd_info *mtd,
1950		- struct nand_chip chip, const uint8_t buf,
	2417	+static int brcmnand_write_page_raw(struct nand_chip chip, const uint8_t buf,
1951	2418	int oob_required, int page)
1952	2419	{
	2420	+ struct mtd_info *mtd = nand_to_mtd(chip);
1953	2421	struct brcmnand_host *host = nand_get_controller_data(chip);
1954	2422	void *oob = oob_required ? chip->oob_poi : NULL;
1955	2423
..	..	@@ -1961,16 +2429,16 @@
1961	2429	return nand_prog_page_end_op(chip);
1962	2430	}
1963	2431
1964		-static int brcmnand_write_oob(struct mtd_info mtd, struct nand_chip chip,
1965		- int page)
	2432	+static int brcmnand_write_oob(struct nand_chip *chip, int page)
1966	2433	{
1967		- return brcmnand_write(mtd, chip, (u64)page << chip->page_shift,
1968		- NULL, chip->oob_poi);
	2434	+ return brcmnand_write(nand_to_mtd(chip), chip,
	2435	+ (u64)page << chip->page_shift, NULL,
	2436	+ chip->oob_poi);
1969	2437	}
1970	2438
1971		-static int brcmnand_write_oob_raw(struct mtd_info mtd, struct nand_chip chip,
1972		- int page)
	2439	+static int brcmnand_write_oob_raw(struct nand_chip *chip, int page)
1973	2440	{
	2441	+ struct mtd_info *mtd = nand_to_mtd(chip);
1974	2442	struct brcmnand_host *host = nand_get_controller_data(chip);
1975	2443	int ret;
1976	2444
..	..	@@ -2059,7 +2527,7 @@
2059	2527	(!!(cfg->device_width == 16) << CFG_BUS_WIDTH_SHIFT) \|
2060	2528	(device_size << CFG_DEVICE_SIZE_SHIFT);
2061	2529	if (cfg_offs == cfg_ext_offs) {
2062		- tmp \|= (page_size << CFG_PAGE_SIZE_SHIFT) \|
	2530	+ tmp \|= (page_size << ctrl->page_size_shift) \|
2063	2531	(block_size << CFG_BLK_SIZE_SHIFT);
2064	2532	nand_writereg(ctrl, cfg_offs, tmp);
2065	2533	} else {
..	..	@@ -2071,9 +2539,11 @@
2071	2539
2072	2540	tmp = nand_readreg(ctrl, acc_control_offs);
2073	2541	tmp &= ~brcmnand_ecc_level_mask(ctrl);
2074		- tmp \|= cfg->ecc_level << NAND_ACC_CONTROL_ECC_SHIFT;
2075	2542	tmp &= ~brcmnand_spare_area_mask(ctrl);
2076		- tmp \|= cfg->spare_area_size;
	2543	+ if (ctrl->nand_version >= 0x0302) {
	2544	+ tmp \|= cfg->ecc_level << ctrl->ecc_level_shift;
	2545	+ tmp \|= cfg->spare_area_size;
	2546	+ }
2077	2547	nand_writereg(ctrl, acc_control_offs, tmp);
2078	2548
2079	2549	brcmnand_set_sector_size_1k(host, cfg->sector_size_1k);
..	..	@@ -2120,6 +2590,10 @@
2120	2590	{
2121	2591	struct mtd_info *mtd = nand_to_mtd(&host->chip);
2122	2592	struct nand_chip *chip = &host->chip;
	2593	+ const struct nand_ecc_props *requirements =
	2594	+ nanddev_get_ecc_requirements(&chip->base);
	2595	+ struct nand_memory_organization *memorg =
	2596	+ nanddev_get_memorg(&chip->base);
2123	2597	struct brcmnand_controller *ctrl = host->ctrl;
2124	2598	struct brcmnand_cfg *cfg = &host->hwcfg;
2125	2599	char msg[128];
..	..	@@ -2141,10 +2615,11 @@
2141	2615	if (cfg->spare_area_size > ctrl->max_oob)
2142	2616	cfg->spare_area_size = ctrl->max_oob;
2143	2617	/*
2144		- * Set oobsize to be consistent with controller's spare_area_size, as
2145		- * the rest is inaccessible.
	2618	+ * Set mtd and memorg oobsize to be consistent with controller's
	2619	+ * spare_area_size, as the rest is inaccessible.
2146	2620	*/
2147	2621	mtd->oobsize = cfg->spare_area_size * (mtd->writesize >> FC_SHIFT);
	2622	+ memorg->oobsize = mtd->oobsize;
2148	2623
2149	2624	cfg->device_size = mtd->size;
2150	2625	cfg->block_size = mtd->erasesize;
..	..	@@ -2153,31 +2628,42 @@
2153	2628	cfg->col_adr_bytes = 2;
2154	2629	cfg->blk_adr_bytes = get_blk_adr_bytes(mtd->size, mtd->writesize);
2155	2630
2156		- if (chip->ecc.mode != NAND_ECC_HW) {
	2631	+ if (chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST) {
2157	2632	dev_err(ctrl->dev, "only HW ECC supported; selected: %d\n",
2158		- chip->ecc.mode);
	2633	+ chip->ecc.engine_type);
2159	2634	return -EINVAL;
2160	2635	}
2161	2636
2162		- if (chip->ecc.algo == NAND_ECC_UNKNOWN) {
	2637	+ if (chip->ecc.algo == NAND_ECC_ALGO_UNKNOWN) {
2163	2638	if (chip->ecc.strength == 1 && chip->ecc.size == 512)
2164	2639	/* Default to Hamming for 1-bit ECC, if unspecified */
2165		- chip->ecc.algo = NAND_ECC_HAMMING;
	2640	+ chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
2166	2641	else
2167	2642	/* Otherwise, BCH */
2168		- chip->ecc.algo = NAND_ECC_BCH;
	2643	+ chip->ecc.algo = NAND_ECC_ALGO_BCH;
2169	2644	}
2170	2645
2171		- if (chip->ecc.algo == NAND_ECC_HAMMING && (chip->ecc.strength != 1 \|\|
2172		- chip->ecc.size != 512)) {
	2646	+ if (chip->ecc.algo == NAND_ECC_ALGO_HAMMING &&
	2647	+ (chip->ecc.strength != 1 \|\| chip->ecc.size != 512)) {
2173	2648	dev_err(ctrl->dev, "invalid Hamming params: %d bits per %d bytes\n",
2174	2649	chip->ecc.strength, chip->ecc.size);
2175	2650	return -EINVAL;
2176	2651	}
2177	2652
	2653	+ if (chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_NONE &&
	2654	+ (!chip->ecc.size \|\| !chip->ecc.strength)) {
	2655	+ if (requirements->step_size && requirements->strength) {
	2656	+ /* use detected ECC parameters */
	2657	+ chip->ecc.size = requirements->step_size;
	2658	+ chip->ecc.strength = requirements->strength;
	2659	+ dev_info(ctrl->dev, "Using ECC step-size %d, strength %d\n",
	2660	+ chip->ecc.size, chip->ecc.strength);
	2661	+ }
	2662	+ }
	2663	+
2178	2664	switch (chip->ecc.size) {
2179	2665	case 512:
2180		- if (chip->ecc.algo == NAND_ECC_HAMMING)
	2666	+ if (chip->ecc.algo == NAND_ECC_ALGO_HAMMING)
2181	2667	cfg->ecc_level = 15;
2182	2668	else
2183	2669	cfg->ecc_level = chip->ecc.strength;
..	..	@@ -2248,7 +2734,7 @@
2248	2734	* to/from, and have nand_base pass us a bounce buffer instead, as
2249	2735	* needed.
2250	2736	*/
2251		- chip->options \|= NAND_USE_BOUNCE_BUFFER;
	2737	+ chip->options \|= NAND_USES_DMA;
2252	2738
2253	2739	if (chip->bbt_options & NAND_BBT_USE_FLASH)
2254	2740	chip->bbt_options \|= NAND_BBT_NO_OOB;
..	..	@@ -2304,17 +2790,14 @@
2304	2790	mtd->owner = THIS_MODULE;
2305	2791	mtd->dev.parent = &pdev->dev;
2306	2792
2307		- chip->IO_ADDR_R = (void __iomem *)0xdeadbeef;
2308		- chip->IO_ADDR_W = (void __iomem *)0xdeadbeef;
	2793	+ chip->legacy.cmd_ctrl = brcmnand_cmd_ctrl;
	2794	+ chip->legacy.cmdfunc = brcmnand_cmdfunc;
	2795	+ chip->legacy.waitfunc = brcmnand_waitfunc;
	2796	+ chip->legacy.read_byte = brcmnand_read_byte;
	2797	+ chip->legacy.read_buf = brcmnand_read_buf;
	2798	+ chip->legacy.write_buf = brcmnand_write_buf;
2309	2799
2310		- chip->cmd_ctrl = brcmnand_cmd_ctrl;
2311		- chip->cmdfunc = brcmnand_cmdfunc;
2312		- chip->waitfunc = brcmnand_waitfunc;
2313		- chip->read_byte = brcmnand_read_byte;
2314		- chip->read_buf = brcmnand_read_buf;
2315		- chip->write_buf = brcmnand_write_buf;
2316		-
2317		- chip->ecc.mode = NAND_ECC_HW;
	2800	+ chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2318	2801	chip->ecc.read_page = brcmnand_read_page;
2319	2802	chip->ecc.write_page = brcmnand_write_page;
2320	2803	chip->ecc.read_page_raw = brcmnand_read_page_raw;
..	..	@@ -2392,6 +2875,8 @@
2392	2875
2393	2876	if (has_flash_dma(ctrl))
2394	2877	ctrl->flash_dma_mode = flash_dma_readl(ctrl, FLASH_DMA_MODE);
	2878	+ else if (has_edu(ctrl))
	2879	+ ctrl->edu_config = edu_readl(ctrl, EDU_CONFIG);
2395	2880
2396	2881	return 0;
2397	2882	}
..	..	@@ -2404,6 +2889,13 @@
2404	2889	if (has_flash_dma(ctrl)) {
2405	2890	flash_dma_writel(ctrl, FLASH_DMA_MODE, ctrl->flash_dma_mode);
2406	2891	flash_dma_writel(ctrl, FLASH_DMA_ERROR_STATUS, 0);
	2892	+ }
	2893	+
	2894	+ if (has_edu(ctrl)) {
	2895	+ ctrl->edu_config = edu_readl(ctrl, EDU_CONFIG);
	2896	+ edu_writel(ctrl, EDU_CONFIG, ctrl->edu_config);
	2897	+ edu_readl(ctrl, EDU_CONFIG);
	2898	+ brcmnand_edu_init(ctrl);
2407	2899	}
2408	2900
2409	2901	brcmnand_write_reg(ctrl, BRCMNAND_CS_SELECT, ctrl->nand_cs_nand_select);
..	..	@@ -2435,6 +2927,8 @@
2435	2927	EXPORT_SYMBOL_GPL(brcmnand_pm_ops);
2436	2928
2437	2929	static const struct of_device_id brcmnand_of_match[] = {
	2930	+ { .compatible = "brcm,brcmnand-v2.1" },
	2931	+ { .compatible = "brcm,brcmnand-v2.2" },
2438	2932	{ .compatible = "brcm,brcmnand-v4.0" },
2439	2933	{ .compatible = "brcm,brcmnand-v5.0" },
2440	2934	{ .compatible = "brcm,brcmnand-v6.0" },
..	..	@@ -2443,6 +2937,7 @@
2443	2937	{ .compatible = "brcm,brcmnand-v7.0" },
2444	2938	{ .compatible = "brcm,brcmnand-v7.1" },
2445	2939	{ .compatible = "brcm,brcmnand-v7.2" },
	2940	+ { .compatible = "brcm,brcmnand-v7.3" },
2446	2941	{},
2447	2942	};
2448	2943	MODULE_DEVICE_TABLE(of, brcmnand_of_match);
..	..	@@ -2450,6 +2945,49 @@
2450	2945	/***********************************************************************
2451	2946	* Platform driver setup (per controller)
2452	2947	***********************************************************************/
	2948	+static int brcmnand_edu_setup(struct platform_device *pdev)
	2949	+{
	2950	+ struct device *dev = &pdev->dev;
	2951	+ struct brcmnand_controller *ctrl = dev_get_drvdata(&pdev->dev);
	2952	+ struct resource *res;
	2953	+ int ret;
	2954	+
	2955	+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "flash-edu");
	2956	+ if (res) {
	2957	+ ctrl->edu_base = devm_ioremap_resource(dev, res);
	2958	+ if (IS_ERR(ctrl->edu_base))
	2959	+ return PTR_ERR(ctrl->edu_base);
	2960	+
	2961	+ ctrl->edu_offsets = edu_regs;
	2962	+
	2963	+ edu_writel(ctrl, EDU_CONFIG, EDU_CONFIG_MODE_NAND \|
	2964	+ EDU_CONFIG_SWAP_CFG);
	2965	+ edu_readl(ctrl, EDU_CONFIG);
	2966	+
	2967	+ /* initialize edu */
	2968	+ brcmnand_edu_init(ctrl);
	2969	+
	2970	+ ctrl->edu_irq = platform_get_irq_optional(pdev, 1);
	2971	+ if (ctrl->edu_irq < 0) {
	2972	+ dev_warn(dev,
	2973	+ "FLASH EDU enabled, using ctlrdy irq\n");
	2974	+ } else {
	2975	+ ret = devm_request_irq(dev, ctrl->edu_irq,
	2976	+ brcmnand_edu_irq, 0,
	2977	+ "brcmnand-edu", ctrl);
	2978	+ if (ret < 0) {
	2979	+ dev_err(ctrl->dev, "can't allocate IRQ %d: error %d\n",
	2980	+ ctrl->edu_irq, ret);
	2981	+ return ret;
	2982	+ }
	2983	+
	2984	+ dev_info(dev, "FLASH EDU enabled using irq %u\n",
	2985	+ ctrl->edu_irq);
	2986	+ }
	2987	+ }
	2988	+
	2989	+ return 0;
	2990	+}
2453	2991
2454	2992	int brcmnand_probe(struct platform_device pdev, struct brcmnand_soc soc)
2455	2993	{
..	..	@@ -2473,8 +3011,15 @@
2473	3011	dev_set_drvdata(dev, ctrl);
2474	3012	ctrl->dev = dev;
2475	3013
	3014	+ /* Enable the static key if the soc provides I/O operations indicating
	3015	+ * that a non-memory mapped IO access path must be used
	3016	+ */
	3017	+ if (brcmnand_soc_has_ops(ctrl->soc))
	3018	+ static_branch_enable(&brcmnand_soc_has_ops_key);
	3019	+
2476	3020	init_completion(&ctrl->done);
2477	3021	init_completion(&ctrl->dma_done);
	3022	+ init_completion(&ctrl->edu_done);
2478	3023	nand_controller_init(&ctrl->controller);
2479	3024	ctrl->controller.ops = &brcmnand_controller_ops;
2480	3025	INIT_LIST_HEAD(&ctrl->host_list);
..	..	@@ -2529,7 +3074,21 @@
2529	3074	goto err;
2530	3075	}
2531	3076
2532		- flash_dma_writel(ctrl, FLASH_DMA_MODE, 1); /* linked-list */
	3077	+ /* initialize the dma version */
	3078	+ brcmnand_flash_dma_revision_init(ctrl);
	3079	+
	3080	+ ret = -EIO;
	3081	+ if (ctrl->nand_version >= 0x0700)
	3082	+ ret = dma_set_mask_and_coherent(&pdev->dev,
	3083	+ DMA_BIT_MASK(40));
	3084	+ if (ret)
	3085	+ ret = dma_set_mask_and_coherent(&pdev->dev,
	3086	+ DMA_BIT_MASK(32));
	3087	+ if (ret)
	3088	+ goto err;
	3089	+
	3090	+ /* linked-list and stop on error */
	3091	+ flash_dma_writel(ctrl, FLASH_DMA_MODE, FLASH_DMA_MODE_MASK);
2533	3092	flash_dma_writel(ctrl, FLASH_DMA_ERROR_STATUS, 0);
2534	3093
2535	3094	/* Allocate descriptor(s) */
..	..	@@ -2558,6 +3117,16 @@
2558	3117	}
2559	3118
2560	3119	dev_info(dev, "enabling FLASH_DMA\n");
	3120	+ /* set flash dma transfer function to call */
	3121	+ ctrl->dma_trans = brcmnand_dma_trans;
	3122	+ } else {
	3123	+ ret = brcmnand_edu_setup(pdev);
	3124	+ if (ret < 0)
	3125	+ goto err;
	3126	+
	3127	+ if (has_edu(ctrl))
	3128	+ /* set edu transfer function to call */
	3129	+ ctrl->dma_trans = brcmnand_edu_trans;
2561	3130	}
2562	3131
2563	3132	/* Disable automatic device ID config, direct addressing */
..	..	@@ -2648,9 +3217,15 @@
2648	3217	{
2649	3218	struct brcmnand_controller *ctrl = dev_get_drvdata(&pdev->dev);
2650	3219	struct brcmnand_host *host;
	3220	+ struct nand_chip *chip;
	3221	+ int ret;
2651	3222
2652		- list_for_each_entry(host, &ctrl->host_list, node)
2653		- nand_release(&host->chip);
	3223	+ list_for_each_entry(host, &ctrl->host_list, node) {
	3224	+ chip = &host->chip;
	3225	+ ret = mtd_device_unregister(nand_to_mtd(chip));
	3226	+ WARN_ON(ret);
	3227	+ nand_cleanup(chip);
	3228	+ }
2654	3229
2655	3230	clk_disable_unprepare(ctrl->clk);
2656	3231