forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-10 cde9070d9970eef1f7ec2360586c802a16230ad8 
 kernel/arch/powerpc/kernel/head_8xx.S
....@@ -1,3 +1,4 @@
1
+/* SPDX-License-Identifier: GPL-2.0-or-later */
12 /*
23  *  PowerPC version
34  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
....@@ -11,31 +12,35 @@
1112  *
1213  *  This file contains low-level support and setup for PowerPC 8xx
1314  *  embedded processors, including trap and interrupt dispatch.
14
- *
15
- *  This program is free software; you can redistribute it and/or
16
- *  modify it under the terms of the GNU General Public License
17
- *  as published by the Free Software Foundation; either version
18
- *  2 of the License, or (at your option) any later version.
19
- *
2015  */
2116 
2217 #include <linux/init.h>
18
+#include <linux/magic.h>
19
+#include <linux/pgtable.h>
20
+#include <linux/sizes.h>
2321 #include <asm/processor.h>
2422 #include <asm/page.h>
2523 #include <asm/mmu.h>
2624 #include <asm/cache.h>
27
-#include <asm/pgtable.h>
2825 #include <asm/cputable.h>
2926 #include <asm/thread_info.h>
3027 #include <asm/ppc_asm.h>
3128 #include <asm/asm-offsets.h>
3229 #include <asm/ptrace.h>
3330 #include <asm/export.h>
31
+#include <asm/code-patching-asm.h>
3432 
33
+#include "head_32.h"
34
+
35
+.macro compare_to_kernel_boundary scratch, addr
3536 #if CONFIG_TASK_SIZE <= 0x80000000 && CONFIG_PAGE_OFFSET >= 0x80000000
3637 /* By simply checking Address >= 0x80000000, we know if its a kernel address */
37
-#define SIMPLE_KERNEL_ADDRESS		1
38
+	not.	\scratch, \addr
39
+#else
40
+	rlwinm	\scratch, \addr, 16, 0xfff8
41
+	cmpli	cr0, \scratch, PAGE_OFFSET@h
3842 #endif
43
+.endm
3944 
4045 /*
4146  * We need an ITLB miss handler for kernel addresses if:
....@@ -105,98 +110,22 @@
105110 	mtspr	SPRN_SRR0,r0
106111 	rfi				/* enables MMU */
107112 
108
-/*
109
- * Exception entry code.  This code runs with address translation
110
- * turned off, i.e. using physical addresses.
111
- * We assume sprg3 has the physical address of the current
112
- * task's thread_struct.
113
- */
114
-#define EXCEPTION_PROLOG	\
115
-	mtspr	SPRN_SPRG_SCRATCH0, r10;	\
116
-	mtspr	SPRN_SPRG_SCRATCH1, r11;	\
117
-	mfcr	r10;		\
118
-	EXCEPTION_PROLOG_1;	\
119
-	EXCEPTION_PROLOG_2
120113 
121
-#define EXCEPTION_PROLOG_1	\
122
-	mfspr	r11,SPRN_SRR1;		/* check whether user or kernel */ \
123
-	andi.	r11,r11,MSR_PR;	\
124
-	tophys(r11,r1);			/* use tophys(r1) if kernel */ \
125
-	beq	1f;		\
126
-	mfspr	r11,SPRN_SPRG_THREAD;	\
127
-	lwz	r11,THREAD_INFO-THREAD(r11);	\
128
-	addi	r11,r11,THREAD_SIZE;	\
129
-	tophys(r11,r11);	\
130
-1:	subi	r11,r11,INT_FRAME_SIZE	/* alloc exc. frame */
114
+#ifdef CONFIG_PERF_EVENTS
115
+	.align	4
131116 
117
+	.globl	itlb_miss_counter
118
+itlb_miss_counter:
119
+	.space	4
132120 
133
-#define EXCEPTION_PROLOG_2	\
134
-	stw	r10,_CCR(r11);		/* save registers */ \
135
-	stw	r12,GPR12(r11);	\
136
-	stw	r9,GPR9(r11);	\
137
-	mfspr	r10,SPRN_SPRG_SCRATCH0;	\
138
-	stw	r10,GPR10(r11);	\
139
-	mfspr	r12,SPRN_SPRG_SCRATCH1;	\
140
-	stw	r12,GPR11(r11);	\
141
-	mflr	r10;		\
142
-	stw	r10,_LINK(r11);	\
143
-	mfspr	r12,SPRN_SRR0;	\
144
-	mfspr	r9,SPRN_SRR1;	\
145
-	stw	r1,GPR1(r11);	\
146
-	stw	r1,0(r11);	\
147
-	tovirt(r1,r11);			/* set new kernel sp */	\
148
-	li	r10,MSR_KERNEL & ~(MSR_IR|MSR_DR); /* can take exceptions */ \
149
-	mtmsr	r10;		\
150
-	stw	r0,GPR0(r11);	\
151
-	SAVE_4GPRS(3, r11);	\
152
-	SAVE_2GPRS(7, r11)
121
+	.globl	dtlb_miss_counter
122
+dtlb_miss_counter:
123
+	.space	4
153124 
154
-/*
155
- * Note: code which follows this uses cr0.eq (set if from kernel),
156
- * r11, r12 (SRR0), and r9 (SRR1).
157
- *
158
- * Note2: once we have set r1 we are in a position to take exceptions
159
- * again, and we could thus set MSR:RI at that point.
160
- */
161
-
162
-/*
163
- * Exception vectors.
164
- */
165
-#define EXCEPTION(n, label, hdlr, xfer)		\
166
-	. = n;					\
167
-label:						\
168
-	EXCEPTION_PROLOG;			\
169
-	addi	r3,r1,STACK_FRAME_OVERHEAD;	\
170
-	xfer(n, hdlr)
171
-
172
-#define EXC_XFER_TEMPLATE(n, hdlr, trap, copyee, tfer, ret)	\
173
-	li	r10,trap;					\
174
-	stw	r10,_TRAP(r11);					\
175
-	li	r10,MSR_KERNEL;					\
176
-	copyee(r10, r9);					\
177
-	bl	tfer;						\
178
-i##n:								\
179
-	.long	hdlr;						\
180
-	.long	ret
181
-
182
-#define COPY_EE(d, s)		rlwimi d,s,0,16,16
183
-#define NOCOPY(d, s)
184
-
185
-#define EXC_XFER_STD(n, hdlr)		\
186
-	EXC_XFER_TEMPLATE(n, hdlr, n, NOCOPY, transfer_to_handler_full,	\
187
-			  ret_from_except_full)
188
-
189
-#define EXC_XFER_LITE(n, hdlr)		\
190
-	EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
191
-			  ret_from_except)
192
-
193
-#define EXC_XFER_EE(n, hdlr)		\
194
-	EXC_XFER_TEMPLATE(n, hdlr, n, COPY_EE, transfer_to_handler_full, \
195
-			  ret_from_except_full)
196
-
197
-#define EXC_XFER_EE_LITE(n, hdlr)	\
198
-	EXC_XFER_TEMPLATE(n, hdlr, n+1, COPY_EE, transfer_to_handler, \
199
-			  ret_from_except)
125
+	.globl	instruction_counter
126
+instruction_counter:
127
+	.space	4
128
+#endif
200129 
201130 /* System reset */
202131 	EXCEPTION(0x100, Reset, system_reset_exception, EXC_XFER_STD)
....@@ -204,27 +133,12 @@
204133 /* Machine check */
205134 	. = 0x200
206135 MachineCheck:
207
-	EXCEPTION_PROLOG
208
-	mfspr r4,SPRN_DAR
209
-	stw r4,_DAR(r11)
210
-	li r5,RPN_PATTERN
211
-	mtspr SPRN_DAR,r5	/* Tag DAR, to be used in DTLB Error */
212
-	mfspr r5,SPRN_DSISR
213
-	stw r5,_DSISR(r11)
136
+	EXCEPTION_PROLOG handle_dar_dsisr=1
137
+	save_dar_dsisr_on_stack r4, r5, r11
138
+	li	r6, RPN_PATTERN
139
+	mtspr	SPRN_DAR, r6	/* Tag DAR, to be used in DTLB Error */
214140 	addi r3,r1,STACK_FRAME_OVERHEAD
215141 	EXC_XFER_STD(0x200, machine_check_exception)
216
-
217
-/* Data access exception.
218
- * This is "never generated" by the MPC8xx.
219
- */
220
-	. = 0x300
221
-DataAccess:
222
-
223
-/* Instruction access exception.
224
- * This is "never generated" by the MPC8xx.
225
- */
226
-	. = 0x400
227
-InstructionAccess:
228142 
229143 /* External interrupt */
230144 	EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
....@@ -232,39 +146,31 @@
232146 /* Alignment exception */
233147 	. = 0x600
234148 Alignment:
235
-	EXCEPTION_PROLOG
236
-	mfspr	r4,SPRN_DAR
237
-	stw	r4,_DAR(r11)
238
-	li	r5,RPN_PATTERN
239
-	mtspr	SPRN_DAR,r5	/* Tag DAR, to be used in DTLB Error */
240
-	mfspr	r5,SPRN_DSISR
241
-	stw	r5,_DSISR(r11)
149
+	EXCEPTION_PROLOG handle_dar_dsisr=1
150
+	save_dar_dsisr_on_stack r4, r5, r11
151
+	li	r6, RPN_PATTERN
152
+	mtspr	SPRN_DAR, r6	/* Tag DAR, to be used in DTLB Error */
242153 	addi	r3,r1,STACK_FRAME_OVERHEAD
243
-	EXC_XFER_EE(0x600, alignment_exception)
154
+	b	.Lalignment_exception_ool
244155 
245156 /* Program check exception */
246157 	EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD)
247158 
248
-/* No FPU on MPC8xx.  This exception is not supposed to happen.
249
-*/
250
-	EXCEPTION(0x800, FPUnavailable, unknown_exception, EXC_XFER_STD)
251
-
252159 /* Decrementer */
253160 	EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
254161 
255
-	EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_EE)
256
-	EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_EE)
162
+	/* With VMAP_STACK there's not enough room for this at 0x600 */
163
+	. = 0xa00
164
+.Lalignment_exception_ool:
165
+	EXC_XFER_STD(0x600, alignment_exception)
257166 
258167 /* System call */
259168 	. = 0xc00
260169 SystemCall:
261
-	EXCEPTION_PROLOG
262
-	EXC_XFER_EE_LITE(0xc00, DoSyscall)
170
+	SYSCALL_ENTRY	0xc00
263171 
264172 /* Single step - not used on 601 */
265173 	EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD)
266
-	EXCEPTION(0xe00, Trap_0e, unknown_exception, EXC_XFER_EE)
267
-	EXCEPTION(0xf00, Trap_0f, unknown_exception, EXC_XFER_EE)
268174 
269175 /* On the MPC8xx, this is a software emulation interrupt.  It occurs
270176  * for all unimplemented and illegal instructions.
....@@ -274,7 +180,7 @@
274180 	. = 0x1100
275181 /*
276182  * For the MPC8xx, this is a software tablewalk to load the instruction
277
- * TLB.  The task switch loads the M_TW register with the pointer to the first
183
+ * TLB.  The task switch loads the M_TWB register with the pointer to the first
278184  * level table.
279185  * If we discover there is no second level table (value is zero) or if there
280186  * is an invalid pte, we load that into the TLB, which causes another fault
....@@ -284,205 +190,105 @@
284190  */
285191 
286192 #ifdef CONFIG_8xx_CPU15
287
-#define INVALIDATE_ADJACENT_PAGES_CPU15(tmp, addr)	\
288
-	addi	tmp, addr, PAGE_SIZE;	\
289
-	tlbie	tmp;			\
290
-	addi	tmp, addr, -PAGE_SIZE;	\
291
-	tlbie	tmp
193
+#define INVALIDATE_ADJACENT_PAGES_CPU15(addr)	\
194
+	addi	addr, addr, PAGE_SIZE;	\
195
+	tlbie	addr;			\
196
+	addi	addr, addr, -(PAGE_SIZE << 1);	\
197
+	tlbie	addr;			\
198
+	addi	addr, addr, PAGE_SIZE
292199 #else
293
-#define INVALIDATE_ADJACENT_PAGES_CPU15(tmp, addr)
200
+#define INVALIDATE_ADJACENT_PAGES_CPU15(addr)
294201 #endif
295202 
296203 InstructionTLBMiss:
297204 	mtspr	SPRN_SPRG_SCRATCH0, r10
298205 	mtspr	SPRN_SPRG_SCRATCH1, r11
299
-#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
300
-	mtspr	SPRN_SPRG_SCRATCH2, r12
301
-#endif
302206 
303207 	/* If we are faulting a kernel address, we have to use the
304208 	 * kernel page tables.
305209 	 */
306210 	mfspr	r10, SPRN_SRR0	/* Get effective address of fault */
307
-	INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10)
308
-	/* Only modules will cause ITLB Misses as we always
309
-	 * pin the first 8MB of kernel memory */
310
-#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
311
-	mfcr	r12
312
-#endif
211
+	INVALIDATE_ADJACENT_PAGES_CPU15(r10)
212
+	mtspr	SPRN_MD_EPN, r10
313213 #ifdef ITLB_MISS_KERNEL
314
-#if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
315
-	andis.	r11, r10, 0x8000	/* Address >= 0x80000000 */
316
-#else
317
-	rlwinm	r11, r10, 16, 0xfff8
318
-	cmpli	cr0, r11, PAGE_OFFSET@h
319
-#ifndef CONFIG_PIN_TLB_TEXT
320
-	/* It is assumed that kernel code fits into the first 8M page */
321
-_ENTRY(ITLBMiss_cmp)
322
-	cmpli	cr7, r11, (PAGE_OFFSET + 0x0800000)@h
214
+	mfcr	r11
215
+	compare_to_kernel_boundary r10, r10
323216 #endif
324
-#endif
325
-#endif
326
-	mfspr	r11, SPRN_M_TW	/* Get level 1 table */
217
+	mfspr	r10, SPRN_M_TWB	/* Get level 1 table */
327218 #ifdef ITLB_MISS_KERNEL
328
-#if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
329
-	beq+	3f
330
-#else
331219 	blt+	3f
332
-#endif
333
-#ifndef CONFIG_PIN_TLB_TEXT
334
-	blt	cr7, ITLBMissLinear
335
-#endif
336
-	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha
220
+	rlwinm	r10, r10, 0, 20, 31
221
+	oris	r10, r10, (swapper_pg_dir - PAGE_OFFSET)@ha
337222 3:
338
-#endif
339
-	/* Insert level 1 index */
340
-	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
341
-	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */
342
-
343
-	/* Extract level 2 index */
344
-	rlwinm	r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
345
-#ifdef CONFIG_HUGETLB_PAGE
346223 	mtcr	r11
347
-	bt-	28, 10f		/* bit 28 = Large page (8M) */
348
-	bt-	29, 20f		/* bit 29 = Large page (8M or 512k) */
349224 #endif
350
-	rlwimi	r10, r11, 0, 0, 32 - PAGE_SHIFT - 1	/* Add level 2 base */
225
+	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r10)	/* Get level 1 entry */
226
+	mtspr	SPRN_MD_TWC, r11
227
+	mfspr	r10, SPRN_MD_TWC
351228 	lwz	r10, 0(r10)	/* Get the pte */
352
-4:
353
-#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
354
-	mtcr	r12
355
-#endif
356
-	/* Load the MI_TWC with the attributes for this "segment." */
357
-	mtspr	SPRN_MI_TWC, r11	/* Set segment attributes */
358
-
359
-	rlwinm	r11, r10, 32-7, _PAGE_PRESENT
360
-	and	r11, r11, r10
361
-	rlwimi	r10, r11, 0, _PAGE_PRESENT
362
-	li	r11, RPN_PATTERN | 0x200
229
+	rlwimi	r11, r10, 0, _PAGE_GUARDED | _PAGE_ACCESSED
230
+	rlwimi	r11, r10, 32 - 9, _PMD_PAGE_512K
231
+	mtspr	SPRN_MI_TWC, r11
363232 	/* The Linux PTE won't go exactly into the MMU TLB.
364233 	 * Software indicator bits 20 and 23 must be clear.
365234 	 * Software indicator bits 22, 24, 25, 26, and 27 must be
366235 	 * set.  All other Linux PTE bits control the behavior
367236 	 * of the MMU.
368237 	 */
369
-	rlwimi	r11, r10, 4, 0x0400	/* Copy _PAGE_EXEC into bit 21 */
370
-	rlwimi	r10, r11, 0, 0x0ff0	/* Set 22, 24-27, clear 20,23 */
238
+	rlwinm	r10, r10, 0, ~0x0f00	/* Clear bits 20-23 */
239
+	rlwimi	r10, r10, 4, 0x0400	/* Copy _PAGE_EXEC into bit 21 */
240
+	ori	r10, r10, RPN_PATTERN | 0x200 /* Set 22 and 24-27 */
371241 	mtspr	SPRN_MI_RPN, r10	/* Update TLB entry */
372242 
373243 	/* Restore registers */
374
-_ENTRY(itlb_miss_exit_1)
375
-	mfspr	r10, SPRN_SPRG_SCRATCH0
244
+0:	mfspr	r10, SPRN_SPRG_SCRATCH0
376245 	mfspr	r11, SPRN_SPRG_SCRATCH1
377
-#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
378
-	mfspr	r12, SPRN_SPRG_SCRATCH2
379
-#endif
380246 	rfi
247
+	patch_site	0b, patch__itlbmiss_exit_1
248
+
381249 #ifdef CONFIG_PERF_EVENTS
382
-_ENTRY(itlb_miss_perf)
383
-	lis	r10, (itlb_miss_counter - PAGE_OFFSET)@ha
384
-	lwz	r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10)
385
-	addi	r11, r11, 1
386
-	stw	r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10)
387
-#endif
250
+	patch_site	0f, patch__itlbmiss_perf
251
+0:	lwz	r10, (itlb_miss_counter - PAGE_OFFSET)@l(0)
252
+	addi	r10, r10, 1
253
+	stw	r10, (itlb_miss_counter - PAGE_OFFSET)@l(0)
388254 	mfspr	r10, SPRN_SPRG_SCRATCH0
389255 	mfspr	r11, SPRN_SPRG_SCRATCH1
390
-#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
391
-	mfspr	r12, SPRN_SPRG_SCRATCH2
392
-#endif
393256 	rfi
394
-
395
-#ifdef CONFIG_HUGETLB_PAGE
396
-10:	/* 8M pages */
397
-#ifdef CONFIG_PPC_16K_PAGES
398
-	/* Extract level 2 index */
399
-	rlwinm	r10, r10, 32 - (PAGE_SHIFT_8M - PAGE_SHIFT), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29
400
-	/* Add level 2 base */
401
-	rlwimi	r10, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1
402
-#else
403
-	/* Level 2 base */
404
-	rlwinm	r10, r11, 0, ~HUGEPD_SHIFT_MASK
405
-#endif
406
-	lwz	r10, 0(r10)	/* Get the pte */
407
-	b	4b
408
-
409
-20:	/* 512k pages */
410
-	/* Extract level 2 index */
411
-	rlwinm	r10, r10, 32 - (PAGE_SHIFT_512K - PAGE_SHIFT), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29
412
-	/* Add level 2 base */
413
-	rlwimi	r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
414
-	lwz	r10, 0(r10)	/* Get the pte */
415
-	b	4b
416257 #endif
417258 
418259 	. = 0x1200
419260 DataStoreTLBMiss:
420
-	mtspr	SPRN_SPRG_SCRATCH0, r10
421
-	mtspr	SPRN_SPRG_SCRATCH1, r11
422
-	mtspr	SPRN_SPRG_SCRATCH2, r12
423
-	mfcr	r12
261
+	mtspr	SPRN_DAR, r10
262
+	mtspr	SPRN_M_TW, r11
263
+	mfcr	r11
424264 
425265 	/* If we are faulting a kernel address, we have to use the
426266 	 * kernel page tables.
427267 	 */
428268 	mfspr	r10, SPRN_MD_EPN
429
-	rlwinm	r11, r10, 16, 0xfff8
430
-	cmpli	cr0, r11, PAGE_OFFSET@h
431
-	mfspr	r11, SPRN_M_TW	/* Get level 1 table */
269
+	compare_to_kernel_boundary r10, r10
270
+	mfspr	r10, SPRN_M_TWB	/* Get level 1 table */
432271 	blt+	3f
433
-	rlwinm	r11, r10, 16, 0xfff8
434
-#ifndef CONFIG_PIN_TLB_IMMR
435
-	cmpli	cr0, r11, VIRT_IMMR_BASE@h
436
-#endif
437
-_ENTRY(DTLBMiss_cmp)
438
-	cmpli	cr7, r11, (PAGE_OFFSET + 0x1800000)@h
439
-#ifndef CONFIG_PIN_TLB_IMMR
440
-_ENTRY(DTLBMiss_jmp)
441
-	beq-	DTLBMissIMMR
442
-#endif
443
-	blt	cr7, DTLBMissLinear
444
-	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha
272
+	rlwinm	r10, r10, 0, 20, 31
273
+	oris	r10, r10, (swapper_pg_dir - PAGE_OFFSET)@ha
445274 3:
446
-
447
-	/* Insert level 1 index */
448
-	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
449
-	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */
450
-
451
-	/* We have a pte table, so load fetch the pte from the table.
452
-	 */
453
-	/* Extract level 2 index */
454
-	rlwinm	r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
455
-#ifdef CONFIG_HUGETLB_PAGE
456275 	mtcr	r11
457
-	bt-	28, 10f		/* bit 28 = Large page (8M) */
458
-	bt-	29, 20f		/* bit 29 = Large page (8M or 512k) */
459
-#endif
460
-	rlwimi	r10, r11, 0, 0, 32 - PAGE_SHIFT - 1	/* Add level 2 base */
461
-	lwz	r10, 0(r10)	/* Get the pte */
462
-4:
463
-	mtcr	r12
276
+	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r10)	/* Get level 1 entry */
464277 
465
-	/* Insert the Guarded flag into the TWC from the Linux PTE.
278
+	mtspr	SPRN_MD_TWC, r11
279
+	mfspr	r10, SPRN_MD_TWC
280
+	lwz	r10, 0(r10)	/* Get the pte */
281
+
282
+	/* Insert Guarded and Accessed flags into the TWC from the Linux PTE.
466283 	 * It is bit 27 of both the Linux PTE and the TWC (at least
467284 	 * I got that right :-).  It will be better when we can put
468285 	 * this into the Linux pgd/pmd and load it in the operation
469286 	 * above.
470287 	 */
471
-	rlwimi	r11, r10, 0, _PAGE_GUARDED
288
+	rlwimi	r11, r10, 0, _PAGE_GUARDED | _PAGE_ACCESSED
289
+	rlwimi	r11, r10, 32 - 9, _PMD_PAGE_512K
472290 	mtspr	SPRN_MD_TWC, r11
473291 
474
-	/* Both _PAGE_ACCESSED and _PAGE_PRESENT has to be set.
475
-	 * We also need to know if the insn is a load/store, so:
476
-	 * Clear _PAGE_PRESENT and load that which will
477
-	 * trap into DTLB Error with store bit set accordinly.
478
-	 */
479
-	/* PRESENT=0x1, ACCESSED=0x20
480
-	 * r11 = ((r10 & PRESENT) & ((r10 & ACCESSED) >> 5));
481
-	 * r10 = (r10 & ~PRESENT) | r11;
482
-	 */
483
-	rlwinm	r11, r10, 32-7, _PAGE_PRESENT
484
-	and	r11, r11, r10
485
-	rlwimi	r10, r11, 0, _PAGE_PRESENT
486292 	/* The Linux PTE won't go exactly into the MMU TLB.
487293 	 * Software indicator bits 24, 25, 26, and 27 must be
488294 	 * set.  All other Linux PTE bits control the behavior
....@@ -493,45 +299,22 @@
493299 	mtspr	SPRN_MD_RPN, r10	/* Update TLB entry */
494300 
495301 	/* Restore registers */
302
+
303
+0:	mfspr	r10, SPRN_DAR
496304 	mtspr	SPRN_DAR, r11	/* Tag DAR */
497
-_ENTRY(dtlb_miss_exit_1)
498
-	mfspr	r10, SPRN_SPRG_SCRATCH0
499
-	mfspr	r11, SPRN_SPRG_SCRATCH1
500
-	mfspr	r12, SPRN_SPRG_SCRATCH2
305
+	mfspr	r11, SPRN_M_TW
501306 	rfi
307
+	patch_site	0b, patch__dtlbmiss_exit_1
308
+
502309 #ifdef CONFIG_PERF_EVENTS
503
-_ENTRY(dtlb_miss_perf)
504
-	lis	r10, (dtlb_miss_counter - PAGE_OFFSET)@ha
505
-	lwz	r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10)
506
-	addi	r11, r11, 1
507
-	stw	r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10)
508
-#endif
509
-	mfspr	r10, SPRN_SPRG_SCRATCH0
510
-	mfspr	r11, SPRN_SPRG_SCRATCH1
511
-	mfspr	r12, SPRN_SPRG_SCRATCH2
310
+	patch_site	0f, patch__dtlbmiss_perf
311
+0:	lwz	r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0)
312
+	addi	r10, r10, 1
313
+	stw	r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0)
314
+	mfspr	r10, SPRN_DAR
315
+	mtspr	SPRN_DAR, r11	/* Tag DAR */
316
+	mfspr	r11, SPRN_M_TW
512317 	rfi
513
-
514
-#ifdef CONFIG_HUGETLB_PAGE
515
-10:	/* 8M pages */
516
-	/* Extract level 2 index */
517
-#ifdef CONFIG_PPC_16K_PAGES
518
-	rlwinm	r10, r10, 32 - (PAGE_SHIFT_8M - PAGE_SHIFT), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29
519
-	/* Add level 2 base */
520
-	rlwimi	r10, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1
521
-#else
522
-	/* Level 2 base */
523
-	rlwinm	r10, r11, 0, ~HUGEPD_SHIFT_MASK
524
-#endif
525
-	lwz	r10, 0(r10)	/* Get the pte */
526
-	b	4b
527
-
528
-20:	/* 512k pages */
529
-	/* Extract level 2 index */
530
-	rlwinm	r10, r10, 32 - (PAGE_SHIFT_512K - PAGE_SHIFT), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29
531
-	/* Add level 2 base */
532
-	rlwimi	r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
533
-	lwz	r10, 0(r10)	/* Get the pte */
534
-	b	4b
535318 #endif
536319 
537320 /* This is an instruction TLB error on the MPC8xx.  This could be due
....@@ -544,11 +327,12 @@
544327 	mr	r4,r12
545328 	andis.	r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
546329 	andis.	r10,r9,SRR1_ISI_NOPT@h
547
-	beq+	1f
330
+	beq+	.Litlbie
548331 	tlbie	r4
549
-itlbie:
550332 	/* 0x400 is InstructionAccess exception, needed by bad_page_fault() */
551
-1:	EXC_XFER_LITE(0x400, handle_page_fault)
333
+.Litlbie:
334
+	stw	r4, _DAR(r11)
335
+	EXC_XFER_LITE(0x400, handle_page_fault)
552336 
553337 /* This is the data TLB error on the MPC8xx.  This could be due to
554338  * many reasons, including a dirty update to a pte.  We bail out to
....@@ -556,58 +340,57 @@
556340  */
557341 	. = 0x1400
558342 DataTLBError:
559
-	mtspr	SPRN_SPRG_SCRATCH0, r10
560
-	mtspr	SPRN_SPRG_SCRATCH1, r11
561
-	mfcr	r10
562
-
343
+	EXCEPTION_PROLOG_0 handle_dar_dsisr=1
563344 	mfspr	r11, SPRN_DAR
564
-	cmpwi	cr0, r11, RPN_PATTERN
565
-	beq-	FixupDAR	/* must be a buggy dcbX, icbi insn. */
345
+	cmpwi	cr1, r11, RPN_PATTERN
346
+	beq-	cr1, FixupDAR	/* must be a buggy dcbX, icbi insn. */
566347 DARFixed:/* Return from dcbx instruction bug workaround */
348
+#ifdef CONFIG_VMAP_STACK
349
+	li	r11, RPN_PATTERN
350
+	mtspr	SPRN_DAR, r11	/* Tag DAR, to be used in DTLB Error */
351
+#endif
567352 	EXCEPTION_PROLOG_1
568
-	EXCEPTION_PROLOG_2
569
-	mfspr	r5,SPRN_DSISR
570
-	stw	r5,_DSISR(r11)
571
-	mfspr	r4,SPRN_DAR
353
+	EXCEPTION_PROLOG_2 handle_dar_dsisr=1
354
+	get_and_save_dar_dsisr_on_stack r4, r5, r11
572355 	andis.	r10,r5,DSISR_NOHPTE@h
573
-	beq+	1f
356
+	beq+	.Ldtlbie
574357 	tlbie	r4
575
-dtlbie:
576
-1:	li	r10,RPN_PATTERN
358
+.Ldtlbie:
359
+#ifndef CONFIG_VMAP_STACK
360
+	li	r10,RPN_PATTERN
577361 	mtspr	SPRN_DAR,r10	/* Tag DAR, to be used in DTLB Error */
362
+#endif
578363 	/* 0x300 is DataAccess exception, needed by bad_page_fault() */
579364 	EXC_XFER_LITE(0x300, handle_page_fault)
580365 
581
-	EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_EE)
582
-	EXCEPTION(0x1600, Trap_16, unknown_exception, EXC_XFER_EE)
583
-	EXCEPTION(0x1700, Trap_17, unknown_exception, EXC_XFER_EE)
584
-	EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_EE)
585
-	EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_EE)
586
-	EXCEPTION(0x1a00, Trap_1a, unknown_exception, EXC_XFER_EE)
587
-	EXCEPTION(0x1b00, Trap_1b, unknown_exception, EXC_XFER_EE)
366
+stack_overflow:
367
+	vmap_stack_overflow_exception
588368 
589369 /* On the MPC8xx, these next four traps are used for development
590370  * support of breakpoints and such.  Someday I will get around to
591371  * using them.
592372  */
593
-	. = 0x1c00
594
-DataBreakpoint:
595
-	mtspr	SPRN_SPRG_SCRATCH0, r10
596
-	mtspr	SPRN_SPRG_SCRATCH1, r11
597
-	mfcr	r10
598
-	mfspr	r11, SPRN_SRR0
599
-	cmplwi	cr0, r11, (dtlbie - PAGE_OFFSET)@l
600
-	cmplwi	cr7, r11, (itlbie - PAGE_OFFSET)@l
601
-	beq-	cr0, 11f
602
-	beq-	cr7, 11f
373
+do_databreakpoint:
603374 	EXCEPTION_PROLOG_1
604
-	EXCEPTION_PROLOG_2
375
+	EXCEPTION_PROLOG_2 handle_dar_dsisr=1
605376 	addi	r3,r1,STACK_FRAME_OVERHEAD
606377 	mfspr	r4,SPRN_BAR
607378 	stw	r4,_DAR(r11)
379
+#ifdef CONFIG_VMAP_STACK
380
+	lwz	r5,_DSISR(r11)
381
+#else
608382 	mfspr	r5,SPRN_DSISR
609
-	EXC_XFER_EE(0x1c00, do_break)
610
-11:
383
+#endif
384
+	EXC_XFER_STD(0x1c00, do_break)
385
+
386
+	. = 0x1c00
387
+DataBreakpoint:
388
+	EXCEPTION_PROLOG_0 handle_dar_dsisr=1
389
+	mfspr	r11, SPRN_SRR0
390
+	cmplwi	cr1, r11, (.Ldtlbie - PAGE_OFFSET)@l
391
+	cmplwi	cr7, r11, (.Litlbie - PAGE_OFFSET)@l
392
+	cror	4*cr1+eq, 4*cr1+eq, 4*cr7+eq
393
+	bne	cr1, do_databreakpoint
611394 	mtcr	r10
612395 	mfspr	r10, SPRN_SPRG_SCRATCH0
613396 	mfspr	r11, SPRN_SPRG_SCRATCH1
....@@ -617,115 +400,48 @@
617400 	. = 0x1d00
618401 InstructionBreakpoint:
619402 	mtspr	SPRN_SPRG_SCRATCH0, r10
620
-	mtspr	SPRN_SPRG_SCRATCH1, r11
621
-	lis	r10, (instruction_counter - PAGE_OFFSET)@ha
622
-	lwz	r11, (instruction_counter - PAGE_OFFSET)@l(r10)
623
-	addi	r11, r11, -1
624
-	stw	r11, (instruction_counter - PAGE_OFFSET)@l(r10)
403
+	lwz	r10, (instruction_counter - PAGE_OFFSET)@l(0)
404
+	addi	r10, r10, -1
405
+	stw	r10, (instruction_counter - PAGE_OFFSET)@l(0)
625406 	lis	r10, 0xffff
626407 	ori	r10, r10, 0x01
627408 	mtspr	SPRN_COUNTA, r10
628409 	mfspr	r10, SPRN_SPRG_SCRATCH0
629
-	mfspr	r11, SPRN_SPRG_SCRATCH1
630410 	rfi
631411 #else
632
-	EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_EE)
412
+	EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_STD)
633413 #endif
634
-	EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_EE)
635
-	EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_EE)
414
+	EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD)
415
+	EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD)
636416 
637417 	. = 0x2000
638
-
639
-/*
640
- * Bottom part of DataStoreTLBMiss handlers for IMMR area and linear RAM.
641
- * not enough space in the DataStoreTLBMiss area.
642
- */
643
-DTLBMissIMMR:
644
-	mtcr	r12
645
-	/* Set 512k byte guarded page and mark it valid */
646
-	li	r10, MD_PS512K | MD_GUARDED | MD_SVALID
647
-	mtspr	SPRN_MD_TWC, r10
648
-	mfspr	r10, SPRN_IMMR			/* Get current IMMR */
649
-	rlwinm	r10, r10, 0, 0xfff80000		/* Get 512 kbytes boundary */
650
-	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
651
-			  _PAGE_PRESENT | _PAGE_NO_CACHE
652
-	mtspr	SPRN_MD_RPN, r10	/* Update TLB entry */
653
-
654
-	li	r11, RPN_PATTERN
655
-	mtspr	SPRN_DAR, r11	/* Tag DAR */
656
-_ENTRY(dtlb_miss_exit_2)
657
-	mfspr	r10, SPRN_SPRG_SCRATCH0
658
-	mfspr	r11, SPRN_SPRG_SCRATCH1
659
-	mfspr	r12, SPRN_SPRG_SCRATCH2
660
-	rfi
661
-
662
-DTLBMissLinear:
663
-	mtcr	r12
664
-	/* Set 8M byte page and mark it valid */
665
-	li	r11, MD_PS8MEG | MD_SVALID
666
-	mtspr	SPRN_MD_TWC, r11
667
-	rlwinm	r10, r10, 0, 0x0f800000	/* 8xx supports max 256Mb RAM */
668
-	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
669
-			  _PAGE_PRESENT
670
-	mtspr	SPRN_MD_RPN, r10	/* Update TLB entry */
671
-
672
-	li	r11, RPN_PATTERN
673
-	mtspr	SPRN_DAR, r11	/* Tag DAR */
674
-_ENTRY(dtlb_miss_exit_3)
675
-	mfspr	r10, SPRN_SPRG_SCRATCH0
676
-	mfspr	r11, SPRN_SPRG_SCRATCH1
677
-	mfspr	r12, SPRN_SPRG_SCRATCH2
678
-	rfi
679
-
680
-#ifndef CONFIG_PIN_TLB_TEXT
681
-ITLBMissLinear:
682
-	mtcr	r12
683
-	/* Set 8M byte page and mark it valid */
684
-	li	r11, MI_PS8MEG | MI_SVALID
685
-	mtspr	SPRN_MI_TWC, r11
686
-	rlwinm	r10, r10, 0, 0x0f800000	/* 8xx supports max 256Mb RAM */
687
-	ori	r10, r10, 0xf0 | MI_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
688
-			  _PAGE_PRESENT
689
-	mtspr	SPRN_MI_RPN, r10	/* Update TLB entry */
690
-
691
-_ENTRY(itlb_miss_exit_2)
692
-	mfspr	r10, SPRN_SPRG_SCRATCH0
693
-	mfspr	r11, SPRN_SPRG_SCRATCH1
694
-	mfspr	r12, SPRN_SPRG_SCRATCH2
695
-	rfi
696
-#endif
697418 
698419 /* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions
699420  * by decoding the registers used by the dcbx instruction and adding them.
700421  * DAR is set to the calculated address.
701422  */
702
- /* define if you don't want to use self modifying code */
703
-#define NO_SELF_MODIFYING_CODE
704423 FixupDAR:/* Entry point for dcbx workaround. */
705
-	mtspr	SPRN_SPRG_SCRATCH2, r10
424
+	mtspr	SPRN_M_TW, r10
706425 	/* fetch instruction from memory. */
707426 	mfspr	r10, SPRN_SRR0
427
+	mtspr	SPRN_MD_EPN, r10
708428 	rlwinm	r11, r10, 16, 0xfff8
709
-	cmpli	cr0, r11, PAGE_OFFSET@h
710
-	mfspr	r11, SPRN_M_TW	/* Get level 1 table */
711
-	blt+	3f
712
-	rlwinm	r11, r10, 16, 0xfff8
713
-_ENTRY(FixupDAR_cmp)
714
-	cmpli	cr7, r11, (PAGE_OFFSET + 0x1800000)@h
429
+	cmpli	cr1, r11, PAGE_OFFSET@h
430
+	mfspr	r11, SPRN_M_TWB	/* Get level 1 table */
431
+	blt+	cr1, 3f
432
+
715433 	/* create physical page address from effective address */
716434 	tophys(r11, r10)
717
-	blt-	cr7, 201f
718
-	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha
719
-	/* Insert level 1 index */
720
-3:	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
435
+	mfspr	r11, SPRN_M_TWB	/* Get level 1 table */
436
+	rlwinm	r11, r11, 0, 20, 31
437
+	oris	r11, r11, (swapper_pg_dir - PAGE_OFFSET)@ha
438
+3:
721439 	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */
722
-	mtcr	r11
723
-	bt	28,200f		/* bit 28 = Large page (8M) */
724
-	bt	29,202f		/* bit 29 = Large page (8M or 512K) */
725
-	rlwinm	r11, r11,0,0,19	/* Extract page descriptor page address */
726
-	/* Insert level 2 index */
727
-	rlwimi	r11, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
440
+	mtspr	SPRN_MD_TWC, r11
441
+	mtcrf	0x01, r11
442
+	mfspr	r11, SPRN_MD_TWC
728443 	lwz	r11, 0(r11)	/* Get the pte */
444
+	bt	28,200f		/* bit 28 = Large page (8M) */
729445 	/* concat physical page address(r11) and page offset(r10) */
730446 	rlwimi	r11, r10, 0, 32 - PAGE_SHIFT, 31
731447 201:	lwz	r11,0(r11)
....@@ -734,65 +450,28 @@
734450  * no need to include them here */
735451 	xoris	r10, r11, 0x7c00	/* check if major OP code is 31 */
736452 	rlwinm	r10, r10, 0, 21, 5
737
-	cmpwi	cr0, r10, 2028	/* Is dcbz? */
738
-	beq+	142f
739
-	cmpwi	cr0, r10, 940	/* Is dcbi? */
740
-	beq+	142f
741
-	cmpwi	cr0, r10, 108	/* Is dcbst? */
742
-	beq+	144f		/* Fix up store bit! */
743
-	cmpwi	cr0, r10, 172	/* Is dcbf? */
744
-	beq+	142f
745
-	cmpwi	cr0, r10, 1964	/* Is icbi? */
746
-	beq+	142f
747
-141:	mfspr	r10,SPRN_SPRG_SCRATCH2
453
+	cmpwi	cr1, r10, 2028	/* Is dcbz? */
454
+	beq+	cr1, 142f
455
+	cmpwi	cr1, r10, 940	/* Is dcbi? */
456
+	beq+	cr1, 142f
457
+	cmpwi	cr1, r10, 108	/* Is dcbst? */
458
+	beq+	cr1, 144f		/* Fix up store bit! */
459
+	cmpwi	cr1, r10, 172	/* Is dcbf? */
460
+	beq+	cr1, 142f
461
+	cmpwi	cr1, r10, 1964	/* Is icbi? */
462
+	beq+	cr1, 142f
463
+141:	mfspr	r10,SPRN_M_TW
748464 	b	DARFixed	/* Nope, go back to normal TLB processing */
749465 
750
-	/* concat physical page address(r11) and page offset(r10) */
751466 200:
752
-#ifdef CONFIG_PPC_16K_PAGES
753
-	rlwinm	r11, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1
754
-	rlwimi	r11, r10, 32 - (PAGE_SHIFT_8M - 2), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29
755
-#else
756
-	rlwinm	r11, r10, 0, ~HUGEPD_SHIFT_MASK
757
-#endif
758
-	lwz	r11, 0(r11)	/* Get the pte */
759467 	/* concat physical page address(r11) and page offset(r10) */
760468 	rlwimi	r11, r10, 0, 32 - PAGE_SHIFT_8M, 31
761
-	b	201b
762
-
763
-202:
764
-	rlwinm	r11, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
765
-	rlwimi	r11, r10, 32 - (PAGE_SHIFT_512K - 2), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29
766
-	lwz	r11, 0(r11)	/* Get the pte */
767
-	/* concat physical page address(r11) and page offset(r10) */
768
-	rlwimi	r11, r10, 0, 32 - PAGE_SHIFT_512K, 31
769469 	b	201b
770470 
771471 144:	mfspr	r10, SPRN_DSISR
772472 	rlwinm	r10, r10,0,7,5	/* Clear store bit for buggy dcbst insn */
773473 	mtspr	SPRN_DSISR, r10
774474 142:	/* continue, it was a dcbx, dcbi instruction. */
775
-#ifndef NO_SELF_MODIFYING_CODE
776
-	andis.	r10,r11,0x1f	/* test if reg RA is r0 */
777
-	li	r10,modified_instr@l
778
-	dcbtst	r0,r10		/* touch for store */
779
-	rlwinm	r11,r11,0,0,20	/* Zero lower 10 bits */
780
-	oris	r11,r11,640	/* Transform instr. to a "add r10,RA,RB" */
781
-	ori	r11,r11,532
782
-	stw	r11,0(r10)	/* store add/and instruction */
783
-	dcbf	0,r10		/* flush new instr. to memory. */
784
-	icbi	0,r10		/* invalidate instr. cache line */
785
-	mfspr	r11, SPRN_SPRG_SCRATCH1	/* restore r11 */
786
-	mfspr	r10, SPRN_SPRG_SCRATCH0	/* restore r10 */
787
-	isync			/* Wait until new instr is loaded from memory */
788
-modified_instr:
789
-	.space	4		/* this is where the add instr. is stored */
790
-	bne+	143f
791
-	subf	r10,r0,r10	/* r10=r10-r0, only if reg RA is r0 */
792
-143:	mtdar	r10		/* store faulting EA in DAR */
793
-	mfspr	r10,SPRN_SPRG_SCRATCH2
794
-	b	DARFixed	/* Go back to normal TLB handling */
795
-#else
796475 	mfctr	r10
797476 	mtdar	r10			/* save ctr reg in DAR */
798477 	rlwinm	r10, r11, 24, 24, 28	/* offset into jump table for reg RB */
....@@ -834,8 +513,9 @@
834513 	add	r10, r10, r30	;b	151f
835514 	add	r10, r10, r31
836515 151:
837
-	rlwinm. r11,r11,19,24,28	/* offset into jump table for reg RA */
838
-	beq	152f			/* if reg RA is zero, don't add it */
516
+	rlwinm	r11,r11,19,24,28	/* offset into jump table for reg RA */
517
+	cmpwi	cr1, r11, 0
518
+	beq	cr1, 152f		/* if reg RA is zero, don't add it */
839519 	addi	r11, r11, 150b@l	/* add start of table */
840520 	mtctr	r11			/* load ctr with jump address */
841521 	rlwinm	r11,r11,0,16,10		/* make sure we don't execute this more than once */
....@@ -843,8 +523,15 @@
843523 152:
844524 	mfdar	r11
845525 	mtctr	r11			/* restore ctr reg from DAR */
526
+#ifdef CONFIG_VMAP_STACK
527
+	mfspr	r11, SPRN_SPRG_THREAD
528
+	stw	r10, DAR(r11)
529
+	mfspr	r10, SPRN_DSISR
530
+	stw	r10, DSISR(r11)
531
+#else
846532 	mtdar	r10			/* save fault EA to DAR */
847
-	mfspr	r10,SPRN_SPRG_SCRATCH2
533
+#endif
534
+	mfspr	r10,SPRN_M_TW
848535 	b	DARFixed		/* Go back to normal TLB handling */
849536 
850537 	/* special handling for r10,r11 since these are modified already */
....@@ -856,7 +543,6 @@
856543 	add	r10, r10, r11	/* add it */
857544 	mfctr	r11		/* restore r11 */
858545 	b	151b
859
-#endif
860546 
861547 /*
862548  * This is where the main kernel code starts.
....@@ -874,18 +560,24 @@
874560 	/* stack */
875561 	lis	r1,init_thread_union@ha
876562 	addi	r1,r1,init_thread_union@l
563
+	lis	r0, STACK_END_MAGIC@h
564
+	ori	r0, r0, STACK_END_MAGIC@l
565
+	stw	r0, 0(r1)
877566 	li	r0,0
878567 	stwu	r0,THREAD_SIZE-STACK_FRAME_OVERHEAD(r1)
879568 
880569 	lis	r6, swapper_pg_dir@ha
881570 	tophys(r6,r6)
882
-	mtspr	SPRN_M_TW, r6
571
+	mtspr	SPRN_M_TWB, r6
883572 
884573 	bl	early_init	/* We have to do this with MMU on */
885574 
886575 /*
887576  * Decide what sort of machine this is and initialize the MMU.
888577  */
578
+#ifdef CONFIG_KASAN
579
+	bl	kasan_early_init
580
+#endif
889581 	li	r3,0
890582 	mr	r4,r31
891583 	bl	machine_init
....@@ -910,6 +602,31 @@
910602 	rfi
911603 /* Load up the kernel context */
912604 2:
605
+#ifdef CONFIG_PIN_TLB_IMMR
606
+	lis	r0, MD_TWAM@h
607
+	oris	r0, r0, 0x1f00
608
+	mtspr	SPRN_MD_CTR, r0
609
+	LOAD_REG_IMMEDIATE(r0, VIRT_IMMR_BASE | MD_EVALID)
610
+	tlbie	r0
611
+	mtspr	SPRN_MD_EPN, r0
612
+	LOAD_REG_IMMEDIATE(r0, MD_SVALID | MD_PS512K | MD_GUARDED)
613
+	mtspr	SPRN_MD_TWC, r0
614
+	mfspr   r0, SPRN_IMMR
615
+	rlwinm	r0, r0, 0, 0xfff80000
616
+	ori	r0, r0, 0xf0 | _PAGE_DIRTY | _PAGE_SPS | _PAGE_SH | \
617
+			_PAGE_NO_CACHE | _PAGE_PRESENT
618
+	mtspr	SPRN_MD_RPN, r0
619
+	lis	r0, (MD_TWAM | MD_RSV4I)@h
620
+	mtspr	SPRN_MD_CTR, r0
621
+#endif
622
+#ifndef CONFIG_PIN_TLB_TEXT
623
+	li	r0, 0
624
+	mtspr	SPRN_MI_CTR, r0
625
+#endif
626
+#if !defined(CONFIG_PIN_TLB_DATA) && !defined(CONFIG_PIN_TLB_IMMR)
627
+	lis	r0, MD_TWAM@h
628
+	mtspr	SPRN_MD_CTR, r0
629
+#endif
913630 	tlbia			/* Clear all TLB entries */
914631 	sync			/* wait for tlbia/tlbie to finish */
915632 
....@@ -942,34 +659,10 @@
942659 initial_mmu:
943660 	li	r8, 0
944661 	mtspr	SPRN_MI_CTR, r8		/* remove PINNED ITLB entries */
945
-	lis	r10, MD_RESETVAL@h
946
-#ifndef CONFIG_8xx_COPYBACK
947
-	oris	r10, r10, MD_WTDEF@h
948
-#endif
662
+	lis	r10, MD_TWAM@h
949663 	mtspr	SPRN_MD_CTR, r10	/* remove PINNED DTLB entries */
950664 
951665 	tlbia			/* Invalidate all TLB entries */
952
-#ifdef CONFIG_PIN_TLB_TEXT
953
-	lis	r8, MI_RSV4I@h
954
-	ori	r8, r8, 0x1c00
955
-
956
-	mtspr	SPRN_MI_CTR, r8	/* Set instruction MMU control */
957
-#endif
958
-
959
-#ifdef CONFIG_PIN_TLB_DATA
960
-	oris	r10, r10, MD_RSV4I@h
961
-	mtspr	SPRN_MD_CTR, r10	/* Set data TLB control */
962
-#endif
963
-
964
-	/* Now map the lower 8 Meg into the ITLB. */
965
-	lis	r8, KERNELBASE@h	/* Create vaddr for TLB */
966
-	ori	r8, r8, MI_EVALID	/* Mark it valid */
967
-	mtspr	SPRN_MI_EPN, r8
968
-	li	r8, MI_PS8MEG /* Set 8M byte page */
969
-	ori	r8, r8, MI_SVALID	/* Make it valid */
970
-	mtspr	SPRN_MI_TWC, r8
971
-	li	r8, MI_BOOTINIT		/* Create RPN for address 0 */
972
-	mtspr	SPRN_MI_RPN, r8		/* Store TLB entry */
973666 
974667 	lis	r8, MI_APG_INIT@h	/* Set protection modes */
975668 	ori	r8, r8, MI_APG_INIT@l
....@@ -978,27 +671,32 @@
978671 	ori	r8, r8, MD_APG_INIT@l
979672 	mtspr	SPRN_MD_AP, r8
980673 
981
-	/* Map a 512k page for the IMMR to get the processor
982
-	 * internal registers (among other things).
983
-	 */
984
-#ifdef CONFIG_PIN_TLB_IMMR
985
-	oris	r10, r10, MD_RSV4I@h
986
-	ori	r10, r10, 0x1c00
987
-	mtspr	SPRN_MD_CTR, r10
674
+	/* Map the lower RAM (up to 32 Mbytes) into the ITLB and DTLB */
675
+	lis	r8, MI_RSV4I@h
676
+	ori	r8, r8, 0x1c00
677
+	oris	r12, r10, MD_RSV4I@h
678
+	ori	r12, r12, 0x1c00
679
+	li	r9, 4				/* up to 4 pages of 8M */
680
+	mtctr	r9
681
+	lis	r9, KERNELBASE@h		/* Create vaddr for TLB */
682
+	li	r10, MI_PS8MEG | _PMD_ACCESSED | MI_SVALID
683
+	li	r11, MI_BOOTINIT		/* Create RPN for address 0 */
684
+1:
685
+	mtspr	SPRN_MI_CTR, r8	/* Set instruction MMU control */
686
+	addi	r8, r8, 0x100
687
+	ori	r0, r9, MI_EVALID		/* Mark it valid */
688
+	mtspr	SPRN_MI_EPN, r0
689
+	mtspr	SPRN_MI_TWC, r10
690
+	mtspr	SPRN_MI_RPN, r11		/* Store TLB entry */
691
+	mtspr	SPRN_MD_CTR, r12
692
+	addi	r12, r12, 0x100
693
+	mtspr	SPRN_MD_EPN, r0
694
+	mtspr	SPRN_MD_TWC, r10
695
+	mtspr	SPRN_MD_RPN, r11
696
+	addis	r9, r9, 0x80
697
+	addis	r11, r11, 0x80
988698 
989
-	mfspr	r9, 638			/* Get current IMMR */
990
-	andis.	r9, r9, 0xfff8		/* Get 512 kbytes boundary */
991
-
992
-	lis	r8, VIRT_IMMR_BASE@h	/* Create vaddr for TLB */
993
-	ori	r8, r8, MD_EVALID	/* Mark it valid */
994
-	mtspr	SPRN_MD_EPN, r8
995
-	li	r8, MD_PS512K | MD_GUARDED	/* Set 512k byte page */
996
-	ori	r8, r8, MD_SVALID	/* Make it valid */
997
-	mtspr	SPRN_MD_TWC, r8
998
-	mr	r8, r9			/* Create paddr for TLB */
999
-	ori	r8, r8, MI_BOOTINIT|0x2 /* Inhibit cache -- Cort */
1000
-	mtspr	SPRN_MD_RPN, r8
1001
-#endif
699
+	bdnz	1b
1002700 
1003701 	/* Since the cache is enabled according to the information we
1004702 	 * just loaded into the TLB, invalidate and enable the caches here.
....@@ -1009,17 +707,7 @@
1009707 	mtspr	SPRN_DC_CST, r8
1010708 	lis	r8, IDC_ENABLE@h
1011709 	mtspr	SPRN_IC_CST, r8
1012
-#ifdef CONFIG_8xx_COPYBACK
1013710 	mtspr	SPRN_DC_CST, r8
1014
-#else
1015
-	/* For a debug option, I left this here to easily enable
1016
-	 * the write through cache mode
1017
-	 */
1018
-	lis	r8, DC_SFWT@h
1019
-	mtspr	SPRN_DC_CST, r8
1020
-	lis	r8, IDC_ENABLE@h
1021
-	mtspr	SPRN_DC_CST, r8
1022
-#endif
1023711 	/* Disable debug mode entry on breakpoints */
1024712 	mfspr	r8, SPRN_DER
1025713 #ifdef CONFIG_PERF_EVENTS
....@@ -1030,6 +718,109 @@
1030718 	mtspr	SPRN_DER, r8
1031719 	blr
1032720 
721
+#ifdef CONFIG_PIN_TLB
722
+_GLOBAL(mmu_pin_tlb)
723
+	lis	r9, (1f - PAGE_OFFSET)@h
724
+	ori	r9, r9, (1f - PAGE_OFFSET)@l
725
+	mfmsr	r10
726
+	mflr	r11
727
+	li	r12, MSR_KERNEL & ~(MSR_IR | MSR_DR | MSR_RI)
728
+	rlwinm	r0, r10, 0, ~MSR_RI
729
+	rlwinm	r0, r0, 0, ~MSR_EE
730
+	mtmsr	r0
731
+	isync
732
+	.align	4
733
+	mtspr	SPRN_SRR0, r9
734
+	mtspr	SPRN_SRR1, r12
735
+	rfi
736
+1:
737
+	li	r5, 0
738
+	lis	r6, MD_TWAM@h
739
+	mtspr	SPRN_MI_CTR, r5
740
+	mtspr	SPRN_MD_CTR, r6
741
+	tlbia
742
+
743
+#ifdef CONFIG_PIN_TLB_TEXT
744
+	LOAD_REG_IMMEDIATE(r5, 28 << 8)
745
+	LOAD_REG_IMMEDIATE(r6, PAGE_OFFSET)
746
+	LOAD_REG_IMMEDIATE(r7, MI_SVALID | MI_PS8MEG | _PMD_ACCESSED)
747
+	LOAD_REG_IMMEDIATE(r8, 0xf0 | _PAGE_RO | _PAGE_SPS | _PAGE_SH | _PAGE_PRESENT)
748
+	LOAD_REG_ADDR(r9, _sinittext)
749
+	li	r0, 4
750
+	mtctr	r0
751
+
752
+2:	ori	r0, r6, MI_EVALID
753
+	mtspr	SPRN_MI_CTR, r5
754
+	mtspr	SPRN_MI_EPN, r0
755
+	mtspr	SPRN_MI_TWC, r7
756
+	mtspr	SPRN_MI_RPN, r8
757
+	addi	r5, r5, 0x100
758
+	addis	r6, r6, SZ_8M@h
759
+	addis	r8, r8, SZ_8M@h
760
+	cmplw	r6, r9
761
+	bdnzt	lt, 2b
762
+	lis	r0, MI_RSV4I@h
763
+	mtspr	SPRN_MI_CTR, r0
764
+#endif
765
+	LOAD_REG_IMMEDIATE(r5, 28 << 8 | MD_TWAM)
766
+#ifdef CONFIG_PIN_TLB_DATA
767
+	LOAD_REG_IMMEDIATE(r6, PAGE_OFFSET)
768
+	LOAD_REG_IMMEDIATE(r7, MI_SVALID | MI_PS8MEG | _PMD_ACCESSED)
769
+	li	r8, 0
770
+#ifdef CONFIG_PIN_TLB_IMMR
771
+	li	r0, 3
772
+#else
773
+	li	r0, 4
774
+#endif
775
+	mtctr	r0
776
+	cmpwi	r4, 0
777
+	beq	4f
778
+	LOAD_REG_ADDR(r9, _sinittext)
779
+
780
+2:	ori	r0, r6, MD_EVALID
781
+	ori	r12, r8, 0xf0 | _PAGE_RO | _PAGE_SPS | _PAGE_SH | _PAGE_PRESENT
782
+	mtspr	SPRN_MD_CTR, r5
783
+	mtspr	SPRN_MD_EPN, r0
784
+	mtspr	SPRN_MD_TWC, r7
785
+	mtspr	SPRN_MD_RPN, r12
786
+	addi	r5, r5, 0x100
787
+	addis	r6, r6, SZ_8M@h
788
+	addis	r8, r8, SZ_8M@h
789
+	cmplw	r6, r9
790
+	bdnzt	lt, 2b
791
+4:
792
+2:	ori	r0, r6, MD_EVALID
793
+	ori	r12, r8, 0xf0 | _PAGE_DIRTY | _PAGE_SPS | _PAGE_SH | _PAGE_PRESENT
794
+	mtspr	SPRN_MD_CTR, r5
795
+	mtspr	SPRN_MD_EPN, r0
796
+	mtspr	SPRN_MD_TWC, r7
797
+	mtspr	SPRN_MD_RPN, r12
798
+	addi	r5, r5, 0x100
799
+	addis	r6, r6, SZ_8M@h
800
+	addis	r8, r8, SZ_8M@h
801
+	cmplw	r6, r3
802
+	bdnzt	lt, 2b
803
+#endif
804
+#ifdef CONFIG_PIN_TLB_IMMR
805
+	LOAD_REG_IMMEDIATE(r0, VIRT_IMMR_BASE | MD_EVALID)
806
+	LOAD_REG_IMMEDIATE(r7, MD_SVALID | MD_PS512K | MD_GUARDED | _PMD_ACCESSED)
807
+	mfspr   r8, SPRN_IMMR
808
+	rlwinm	r8, r8, 0, 0xfff80000
809
+	ori	r8, r8, 0xf0 | _PAGE_DIRTY | _PAGE_SPS | _PAGE_SH | \
810
+			_PAGE_NO_CACHE | _PAGE_PRESENT
811
+	mtspr	SPRN_MD_CTR, r5
812
+	mtspr	SPRN_MD_EPN, r0
813
+	mtspr	SPRN_MD_TWC, r7
814
+	mtspr	SPRN_MD_RPN, r8
815
+#endif
816
+#if defined(CONFIG_PIN_TLB_IMMR) || defined(CONFIG_PIN_TLB_DATA)
817
+	lis	r0, (MD_RSV4I | MD_TWAM)@h
818
+	mtspr	SPRN_MD_CTR, r0
819
+#endif
820
+	mtspr	SPRN_SRR1, r10
821
+	mtspr	SPRN_SRR0, r11
822
+	rfi
823
+#endif /* CONFIG_PIN_TLB */
1033824 
1034825 /*
1035826  * We put a few things here that have to be page-aligned.
....@@ -1052,19 +843,6 @@
1052843 /* Room for two PTE table poiners, usually the kernel and current user
1053844  * pointer to their respective root page table (pgdir).
1054845  */
846
+	.globl	abatron_pteptrs
1055847 abatron_pteptrs:
1056848 	.space	8
1057
-
1058
-#ifdef CONFIG_PERF_EVENTS
1059
-	.globl	itlb_miss_counter
1060
-itlb_miss_counter:
1061
-	.space	4
1062
-
1063
-	.globl	dtlb_miss_counter
1064
-dtlb_miss_counter:
1065
-	.space	4
1066
-
1067
-	.globl	instruction_counter
1068
-instruction_counter:
1069
-	.space	4
1070
-#endif