forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-10 37f49e37ab4cb5d0bc4c60eb5c6d4dd57db767bb 
 kernel/arch/powerpc/kernel/entry_64.S
....@@ -1,3 +1,4 @@
1
+/* SPDX-License-Identifier: GPL-2.0-or-later */
12 /*
23  *  PowerPC version 
34  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
....@@ -11,15 +12,11 @@
1112  *
1213  *  This file contains the system call entry code, context switch
1314  *  code, and exception/interrupt return code for PowerPC.
14
- *
15
- *  This program is free software; you can redistribute it and/or
16
- *  modify it under the terms of the GNU General Public License
17
- *  as published by the Free Software Foundation; either version
18
- *  2 of the License, or (at your option) any later version.
1915  */
2016 
2117 #include <linux/errno.h>
2218 #include <linux/err.h>
19
+#include <asm/cache.h>
2320 #include <asm/unistd.h>
2421 #include <asm/processor.h>
2522 #include <asm/page.h>
....@@ -46,6 +43,7 @@
4643 #include <asm/exception-64e.h>
4744 #endif
4845 #include <asm/feature-fixups.h>
46
+#include <asm/kup.h>
4947 
5048 /*
5149  * System calls.
....@@ -54,6 +52,11 @@
5452 SYS_CALL_TABLE:
5553 	.tc sys_call_table[TC],sys_call_table
5654 
55
+#ifdef CONFIG_COMPAT
56
+COMPAT_SYS_CALL_TABLE:
57
+	.tc compat_sys_call_table[TC],compat_sys_call_table
58
+#endif
59
+
5760 /* This value is used to mark exception frames on the stack. */
5861 exception_marker:
5962 	.tc	ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
....@@ -61,360 +64,301 @@
6164 	.section	".text"
6265 	.align 7
6366 
64
-	.globl system_call_common
65
-system_call_common:
67
+#ifdef CONFIG_PPC_BOOK3S
68
+.macro system_call_vectored name trapnr
69
+	.globl system_call_vectored_\name
70
+system_call_vectored_\name:
71
+_ASM_NOKPROBE_SYMBOL(system_call_vectored_\name)
6672 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
6773 BEGIN_FTR_SECTION
6874 	extrdi.	r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
6975 	bne	.Ltabort_syscall
7076 END_FTR_SECTION_IFSET(CPU_FTR_TM)
7177 #endif
72
-	andi.	r10,r12,MSR_PR
78
+	SCV_INTERRUPT_TO_KERNEL
7379 	mr	r10,r1
74
-	addi	r1,r1,-INT_FRAME_SIZE
75
-	beq-	1f
7680 	ld	r1,PACAKSAVE(r13)
77
-1:	std	r10,0(r1)
81
+	std	r10,0(r1)
7882 	std	r11,_NIP(r1)
7983 	std	r12,_MSR(r1)
8084 	std	r0,GPR0(r1)
8185 	std	r10,GPR1(r1)
82
-	beq	2f			/* if from kernel mode */
83
-#ifdef CONFIG_PPC_FSL_BOOK3E
84
-START_BTB_FLUSH_SECTION
85
-	BTB_FLUSH(r10)
86
-END_BTB_FLUSH_SECTION
87
-#endif
88
-	ACCOUNT_CPU_USER_ENTRY(r13, r10, r11)
89
-2:	std	r2,GPR2(r1)
86
+	std	r2,GPR2(r1)
87
+	ld	r2,PACATOC(r13)
88
+	mfcr	r12
89
+	li	r11,0
90
+	/* Can we avoid saving r3-r8 in common case? */
9091 	std	r3,GPR3(r1)
91
-	mfcr	r2
9292 	std	r4,GPR4(r1)
9393 	std	r5,GPR5(r1)
9494 	std	r6,GPR6(r1)
9595 	std	r7,GPR7(r1)
9696 	std	r8,GPR8(r1)
97
-	li	r11,0
97
+	/* Zero r9-r12, this should only be required when restoring all GPRs */
9898 	std	r11,GPR9(r1)
9999 	std	r11,GPR10(r1)
100100 	std	r11,GPR11(r1)
101101 	std	r11,GPR12(r1)
102
-	std	r11,_XER(r1)
103
-	std	r11,_CTR(r1)
104102 	std	r9,GPR13(r1)
105
-	mflr	r10
106
-	/*
107
-	 * This clears CR0.SO (bit 28), which is the error indication on
108
-	 * return from this system call.
109
-	 */
110
-	rldimi	r2,r11,28,(63-28)
111
-	li	r11,0xc01
112
-	std	r10,_LINK(r1)
103
+	SAVE_NVGPRS(r1)
104
+	std	r11,_XER(r1)
105
+	std	r11,_LINK(r1)
106
+	std	r11,_CTR(r1)
107
+
108
+	li	r11,\trapnr
113109 	std	r11,_TRAP(r1)
110
+	std	r12,_CCR(r1)
114111 	std	r3,ORIG_GPR3(r1)
115
-	std	r2,_CCR(r1)
116
-	ld	r2,PACATOC(r13)
117
-	addi	r9,r1,STACK_FRAME_OVERHEAD
112
+	addi	r10,r1,STACK_FRAME_OVERHEAD
118113 	ld	r11,exception_marker@toc(r2)
119
-	std	r11,-16(r9)		/* "regshere" marker */
120
-#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
121
-BEGIN_FW_FTR_SECTION
122
-	beq	33f
123
-	/* if from user, see if there are any DTL entries to process */
124
-	ld	r10,PACALPPACAPTR(r13)	/* get ptr to VPA */
125
-	ld	r11,PACA_DTL_RIDX(r13)	/* get log read index */
126
-	addi	r10,r10,LPPACA_DTLIDX
127
-	LDX_BE	r10,0,r10		/* get log write index */
128
-	cmpd	cr1,r11,r10
129
-	beq+	cr1,33f
130
-	bl	accumulate_stolen_time
131
-	REST_GPR(0,r1)
132
-	REST_4GPRS(3,r1)
133
-	REST_2GPRS(7,r1)
134
-	addi	r9,r1,STACK_FRAME_OVERHEAD
135
-33:
136
-END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
137
-#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
114
+	std	r11,-16(r10)		/* "regshere" marker */
115
+
116
+BEGIN_FTR_SECTION
117
+	HMT_MEDIUM
118
+END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
138119 
139120 	/*
140
-	 * A syscall should always be called with interrupts enabled
141
-	 * so we just unconditionally hard-enable here. When some kind
142
-	 * of irq tracing is used, we additionally check that condition
143
-	 * is correct
144
-	 */
145
-#if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
146
-	lbz	r10,PACAIRQSOFTMASK(r13)
147
-1:	tdnei	r10,IRQS_ENABLED
148
-	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
149
-#endif
150
-
151
-#ifdef CONFIG_PPC_BOOK3E
152
-	wrteei	1
153
-#else
154
-	li	r11,MSR_RI
155
-	ori	r11,r11,MSR_EE
156
-	mtmsrd	r11,1
157
-#endif /* CONFIG_PPC_BOOK3E */
158
-
159
-system_call:			/* label this so stack traces look sane */
160
-	/* We do need to set SOFTE in the stack frame or the return
161
-	 * from interrupt will be painful
162
-	 */
163
-	li	r10,IRQS_ENABLED
164
-	std	r10,SOFTE(r1)
165
-
166
-	CURRENT_THREAD_INFO(r11, r1)
167
-	ld	r10,TI_FLAGS(r11)
168
-	andi.	r11,r10,_TIF_SYSCALL_DOTRACE
169
-	bne	.Lsyscall_dotrace		/* does not return */
170
-	cmpldi	0,r0,NR_syscalls
171
-	bge-	.Lsyscall_enosys
172
-
173
-.Lsyscall:
174
-/*
175
- * Need to vector to 32 Bit or default sys_call_table here,
176
- * based on caller's run-mode / personality.
177
- */
178
-	ld	r11,SYS_CALL_TABLE@toc(2)
179
-	andis.	r10,r10,_TIF_32BIT@h
180
-	beq	15f
181
-	addi	r11,r11,8	/* use 32-bit syscall entries */
182
-	clrldi	r3,r3,32
183
-	clrldi	r4,r4,32
184
-	clrldi	r5,r5,32
185
-	clrldi	r6,r6,32
186
-	clrldi	r7,r7,32
187
-	clrldi	r8,r8,32
188
-15:
189
-	slwi	r0,r0,4
190
-
191
-	barrier_nospec_asm
192
-	/*
193
-	 * Prevent the load of the handler below (based on the user-passed
194
-	 * system call number) being speculatively executed until the test
195
-	 * against NR_syscalls and branch to .Lsyscall_enosys above has
196
-	 * committed.
121
+	 * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
122
+	 * would clobber syscall parameters. Also we always enter with IRQs
123
+	 * enabled and nothing pending. system_call_exception() will call
124
+	 * trace_hardirqs_off().
125
+	 *
126
+	 * scv enters with MSR[EE]=1, so don't set PACA_IRQ_HARD_DIS. The
127
+	 * entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED.
197128 	 */
198129 
199
-	ldx	r12,r11,r0	/* Fetch system call handler [ptr] */
200
-	mtctr   r12
201
-	bctrl			/* Call handler */
130
+	/* Calling convention has r9 = orig r0, r10 = regs */
131
+	mr	r9,r0
132
+	bl	system_call_exception
202133 
203
-.Lsyscall_exit:
204
-	std	r3,RESULT(r1)
134
+.Lsyscall_vectored_\name\()_exit:
135
+	addi    r4,r1,STACK_FRAME_OVERHEAD
136
+	li	r5,1 /* scv */
137
+	bl	syscall_exit_prepare
205138 
206
-#ifdef CONFIG_DEBUG_RSEQ
207
-	/* Check whether the syscall is issued inside a restartable sequence */
208
-	addi    r3,r1,STACK_FRAME_OVERHEAD
209
-	bl      rseq_syscall
210
-	ld	r3,RESULT(r1)
211
-#endif
139
+	ld	r2,_CCR(r1)
140
+	ld	r4,_NIP(r1)
141
+	ld	r5,_MSR(r1)
212142 
213
-	CURRENT_THREAD_INFO(r12, r1)
214
-
215
-	ld	r8,_MSR(r1)
216
-#ifdef CONFIG_PPC_BOOK3S
217
-	/* No MSR:RI on BookE */
218
-	andi.	r10,r8,MSR_RI
219
-	beq-	.Lunrecov_restore
220
-#endif
221
-
222
-/*
223
- * This is a few instructions into the actual syscall exit path (which actually
224
- * starts at .Lsyscall_exit) to cater to kprobe blacklisting and to reduce the
225
- * number of visible symbols for profiling purposes.
226
- *
227
- * We can probe from system_call until this point as MSR_RI is set. But once it
228
- * is cleared below, we won't be able to take a trap.
229
- *
230
- * This is blacklisted from kprobes further below with _ASM_NOKPROBE_SYMBOL().
231
- */
232
-system_call_exit:
233
-	/*
234
-	 * Disable interrupts so current_thread_info()->flags can't change,
235
-	 * and so that we don't get interrupted after loading SRR0/1.
236
-	 */
237
-#ifdef CONFIG_PPC_BOOK3E
238
-	wrteei	0
239
-#else
240
-	/*
241
-	 * For performance reasons we clear RI the same time that we
242
-	 * clear EE. We only need to clear RI just before we restore r13
243
-	 * below, but batching it with EE saves us one expensive mtmsrd call.
244
-	 * We have to be careful to restore RI if we branch anywhere from
245
-	 * here (eg syscall_exit_work).
246
-	 */
247
-	li	r11,0
248
-	mtmsrd	r11,1
249
-#endif /* CONFIG_PPC_BOOK3E */
250
-
251
-	ld	r9,TI_FLAGS(r12)
252
-	li	r11,-MAX_ERRNO
253
-	lis	r0,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)@h
254
-	ori	r0,r0,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)@l
255
-	and.	r0,r9,r0
256
-	bne-	.Lsyscall_exit_work
257
-
258
-	andi.	r0,r8,MSR_FP
259
-	beq 2f
260
-#ifdef CONFIG_ALTIVEC
261
-	andis.	r0,r8,MSR_VEC@h
262
-	bne	3f
263
-#endif
264
-2:	addi    r3,r1,STACK_FRAME_OVERHEAD
265
-#ifdef CONFIG_PPC_BOOK3S
266
-	li	r10,MSR_RI
267
-	mtmsrd	r10,1		/* Restore RI */
268
-#endif
269
-	bl	restore_math
270
-#ifdef CONFIG_PPC_BOOK3S
271
-	li	r11,0
272
-	mtmsrd	r11,1
273
-#endif
274
-	ld	r8,_MSR(r1)
275
-	ld	r3,RESULT(r1)
276
-	li	r11,-MAX_ERRNO
277
-
278
-3:	cmpld	r3,r11
279
-	ld	r5,_CCR(r1)
280
-	bge-	.Lsyscall_error
281
-.Lsyscall_error_cont:
282
-	ld	r7,_NIP(r1)
283143 BEGIN_FTR_SECTION
284144 	stdcx.	r0,0,r1			/* to clear the reservation */
285145 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
286
-	andi.	r6,r8,MSR_PR
287
-	ld	r4,_LINK(r1)
288
-
289
-	beq-	1f
290
-	ACCOUNT_CPU_USER_EXIT(r13, r11, r12)
291146 
292147 BEGIN_FTR_SECTION
293148 	HMT_MEDIUM_LOW
294149 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
295150 
296
-	ld	r13,GPR13(r1)	/* only restore r13 if returning to usermode */
297
-	ld	r2,GPR2(r1)
298
-	ld	r1,GPR1(r1)
151
+	cmpdi	r3,0
152
+	bne	.Lsyscall_vectored_\name\()_restore_regs
153
+
154
+	/* rfscv returns with LR->NIA and CTR->MSR */
299155 	mtlr	r4
300
-	mtcr	r5
301
-	mtspr	SPRN_SRR0,r7
302
-	mtspr	SPRN_SRR1,r8
303
-	RFI_TO_USER
304
-	b	.	/* prevent speculative execution */
156
+	mtctr	r5
305157 
306
-	/* exit to kernel */
307
-1:	ld	r2,GPR2(r1)
308
-	ld	r1,GPR1(r1)
309
-	mtlr	r4
310
-	mtcr	r5
311
-	mtspr	SPRN_SRR0,r7
312
-	mtspr	SPRN_SRR1,r8
313
-	RFI_TO_KERNEL
314
-	b	.	/* prevent speculative execution */
315
-
316
-.Lsyscall_error:
317
-	oris	r5,r5,0x1000	/* Set SO bit in CR */
318
-	neg	r3,r3
319
-	std	r5,_CCR(r1)
320
-	b	.Lsyscall_error_cont
321
-
322
-/* Traced system call support */
323
-.Lsyscall_dotrace:
324
-	bl	save_nvgprs
325
-	addi	r3,r1,STACK_FRAME_OVERHEAD
326
-	bl	do_syscall_trace_enter
327
-
328
-	/*
329
-	 * We use the return value of do_syscall_trace_enter() as the syscall
330
-	 * number. If the syscall was rejected for any reason do_syscall_trace_enter()
331
-	 * returns an invalid syscall number and the test below against
332
-	 * NR_syscalls will fail.
333
-	 */
334
-	mr	r0,r3
335
-
336
-	/* Restore argument registers just clobbered and/or possibly changed. */
337
-	ld	r3,GPR3(r1)
158
+	/* Could zero these as per ABI, but we may consider a stricter ABI
159
+	 * which preserves these if libc implementations can benefit, so
160
+	 * restore them for now until further measurement is done. */
161
+	ld	r0,GPR0(r1)
338162 	ld	r4,GPR4(r1)
339163 	ld	r5,GPR5(r1)
340164 	ld	r6,GPR6(r1)
341165 	ld	r7,GPR7(r1)
342166 	ld	r8,GPR8(r1)
167
+	/* Zero volatile regs that may contain sensitive kernel data */
168
+	li	r9,0
169
+	li	r10,0
170
+	li	r11,0
171
+	li	r12,0
172
+	mtspr	SPRN_XER,r0
343173 
344
-	/* Repopulate r9 and r10 for the syscall path */
345
-	addi	r9,r1,STACK_FRAME_OVERHEAD
346
-	CURRENT_THREAD_INFO(r10, r1)
347
-	ld	r10,TI_FLAGS(r10)
174
+	/*
175
+	 * We don't need to restore AMR on the way back to userspace for KUAP.
176
+	 * The value of AMR only matters while we're in the kernel.
177
+	 */
178
+	mtcr	r2
179
+	ld	r2,GPR2(r1)
180
+	ld	r3,GPR3(r1)
181
+	ld	r13,GPR13(r1)
182
+	ld	r1,GPR1(r1)
183
+	RFSCV_TO_USER
184
+	b	.	/* prevent speculative execution */
348185 
349
-	cmpldi	r0,NR_syscalls
350
-	blt+	.Lsyscall
186
+.Lsyscall_vectored_\name\()_restore_regs:
187
+	li	r3,0
188
+	mtmsrd	r3,1
189
+	mtspr	SPRN_SRR0,r4
190
+	mtspr	SPRN_SRR1,r5
351191 
352
-	/* Return code is already in r3 thanks to do_syscall_trace_enter() */
353
-	b	.Lsyscall_exit
192
+	ld	r3,_CTR(r1)
193
+	ld	r4,_LINK(r1)
194
+	ld	r5,_XER(r1)
354195 
355
-
356
-.Lsyscall_enosys:
357
-	li	r3,-ENOSYS
358
-	b	.Lsyscall_exit
359
-	
360
-.Lsyscall_exit_work:
361
-#ifdef CONFIG_PPC_BOOK3S
362
-	li	r10,MSR_RI
363
-	mtmsrd	r10,1		/* Restore RI */
364
-#endif
365
-	/* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
366
-	 If TIF_NOERROR is set, just save r3 as it is. */
367
-
368
-	andis.	r0,r9,_TIF_RESTOREALL@h
369
-	beq+	0f
370196 	REST_NVGPRS(r1)
371
-	b	2f
372
-0:	cmpld	r3,r11		/* r11 is -MAX_ERRNO */
373
-	blt+	1f
374
-	andis.	r0,r9,_TIF_NOERROR@h
375
-	bne-	1f
376
-	ld	r5,_CCR(r1)
377
-	neg	r3,r3
378
-	oris	r5,r5,0x1000	/* Set SO bit in CR */
379
-	std	r5,_CCR(r1)
380
-1:	std	r3,GPR3(r1)
381
-2:	andis.	r0,r9,(_TIF_PERSYSCALL_MASK)@h
382
-	beq	4f
197
+	ld	r0,GPR0(r1)
198
+	mtcr	r2
199
+	mtctr	r3
200
+	mtlr	r4
201
+	mtspr	SPRN_XER,r5
202
+	REST_10GPRS(2, r1)
203
+	REST_2GPRS(12, r1)
204
+	ld	r1,GPR1(r1)
205
+	RFI_TO_USER
206
+.endm
383207 
384
-	/* Clear per-syscall TIF flags if any are set.  */
208
+system_call_vectored common 0x3000
209
+/*
210
+ * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0
211
+ * which is tested by system_call_exception when r0 is -1 (as set by vector
212
+ * entry code).
213
+ */
214
+system_call_vectored sigill 0x7ff0
385215 
386
-	lis	r11,(_TIF_PERSYSCALL_MASK)@h
387
-	addi	r12,r12,TI_FLAGS
388
-3:	ldarx	r10,0,r12
389
-	andc	r10,r10,r11
390
-	stdcx.	r10,0,r12
391
-	bne-	3b
392
-	subi	r12,r12,TI_FLAGS
393216 
394
-4:	/* Anything else left to do? */
217
+/*
218
+ * Entered via kernel return set up by kernel/sstep.c, must match entry regs
219
+ */
220
+	.globl system_call_vectored_emulate
221
+system_call_vectored_emulate:
222
+_ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate)
223
+	li	r10,IRQS_ALL_DISABLED
224
+	stb	r10,PACAIRQSOFTMASK(r13)
225
+	b	system_call_vectored_common
226
+#endif
227
+
228
+	.balign IFETCH_ALIGN_BYTES
229
+	.globl system_call_common
230
+system_call_common:
231
+_ASM_NOKPROBE_SYMBOL(system_call_common)
232
+#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
395233 BEGIN_FTR_SECTION
396
-	lis	r3,INIT_PPR@highest	/* Set thread.ppr = 3 */
397
-	ld	r10,PACACURRENT(r13)
398
-	sldi	r3,r3,32	/* bits 11-13 are used for ppr */
399
-	std	r3,TASKTHREADPPR(r10)
234
+	extrdi.	r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
235
+	bne	.Ltabort_syscall
236
+END_FTR_SECTION_IFSET(CPU_FTR_TM)
237
+#endif
238
+	mr	r10,r1
239
+	ld	r1,PACAKSAVE(r13)
240
+	std	r10,0(r1)
241
+	std	r11,_NIP(r1)
242
+	std	r12,_MSR(r1)
243
+	std	r0,GPR0(r1)
244
+	std	r10,GPR1(r1)
245
+	std	r2,GPR2(r1)
246
+#ifdef CONFIG_PPC_FSL_BOOK3E
247
+START_BTB_FLUSH_SECTION
248
+	BTB_FLUSH(r10)
249
+END_BTB_FLUSH_SECTION
250
+#endif
251
+	ld	r2,PACATOC(r13)
252
+	mfcr	r12
253
+	li	r11,0
254
+	/* Can we avoid saving r3-r8 in common case? */
255
+	std	r3,GPR3(r1)
256
+	std	r4,GPR4(r1)
257
+	std	r5,GPR5(r1)
258
+	std	r6,GPR6(r1)
259
+	std	r7,GPR7(r1)
260
+	std	r8,GPR8(r1)
261
+	/* Zero r9-r12, this should only be required when restoring all GPRs */
262
+	std	r11,GPR9(r1)
263
+	std	r11,GPR10(r1)
264
+	std	r11,GPR11(r1)
265
+	std	r11,GPR12(r1)
266
+	std	r9,GPR13(r1)
267
+	SAVE_NVGPRS(r1)
268
+	std	r11,_XER(r1)
269
+	std	r11,_CTR(r1)
270
+	mflr	r10
271
+
272
+	/*
273
+	 * This clears CR0.SO (bit 28), which is the error indication on
274
+	 * return from this system call.
275
+	 */
276
+	rldimi	r12,r11,28,(63-28)
277
+	li	r11,0xc00
278
+	std	r10,_LINK(r1)
279
+	std	r11,_TRAP(r1)
280
+	std	r12,_CCR(r1)
281
+	std	r3,ORIG_GPR3(r1)
282
+	addi	r10,r1,STACK_FRAME_OVERHEAD
283
+	ld	r11,exception_marker@toc(r2)
284
+	std	r11,-16(r10)		/* "regshere" marker */
285
+
286
+	/*
287
+	 * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
288
+	 * would clobber syscall parameters. Also we always enter with IRQs
289
+	 * enabled and nothing pending. system_call_exception() will call
290
+	 * trace_hardirqs_off().
291
+	 */
292
+	li	r11,IRQS_ALL_DISABLED
293
+	li	r12,PACA_IRQ_HARD_DIS
294
+	stb	r11,PACAIRQSOFTMASK(r13)
295
+	stb	r12,PACAIRQHAPPENED(r13)
296
+
297
+	/* Calling convention has r9 = orig r0, r10 = regs */
298
+	mr	r9,r0
299
+	bl	system_call_exception
300
+
301
+.Lsyscall_exit:
302
+	addi    r4,r1,STACK_FRAME_OVERHEAD
303
+	li	r5,0 /* !scv */
304
+	bl	syscall_exit_prepare
305
+
306
+	ld	r2,_CCR(r1)
307
+	ld	r4,_NIP(r1)
308
+	ld	r5,_MSR(r1)
309
+	ld	r6,_LINK(r1)
310
+
311
+BEGIN_FTR_SECTION
312
+	stdcx.	r0,0,r1			/* to clear the reservation */
313
+END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
314
+
315
+	mtspr	SPRN_SRR0,r4
316
+	mtspr	SPRN_SRR1,r5
317
+	mtlr	r6
318
+
319
+	cmpdi	r3,0
320
+	bne	.Lsyscall_restore_regs
321
+	/* Zero volatile regs that may contain sensitive kernel data */
322
+	li	r0,0
323
+	li	r4,0
324
+	li	r5,0
325
+	li	r6,0
326
+	li	r7,0
327
+	li	r8,0
328
+	li	r9,0
329
+	li	r10,0
330
+	li	r11,0
331
+	li	r12,0
332
+	mtctr	r0
333
+	mtspr	SPRN_XER,r0
334
+.Lsyscall_restore_regs_cont:
335
+
336
+BEGIN_FTR_SECTION
337
+	HMT_MEDIUM_LOW
400338 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
401339 
402
-	andi.	r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
403
-	beq	ret_from_except_lite
340
+	/*
341
+	 * We don't need to restore AMR on the way back to userspace for KUAP.
342
+	 * The value of AMR only matters while we're in the kernel.
343
+	 */
344
+	mtcr	r2
345
+	ld	r2,GPR2(r1)
346
+	ld	r3,GPR3(r1)
347
+	ld	r13,GPR13(r1)
348
+	ld	r1,GPR1(r1)
349
+	RFI_TO_USER
350
+	b	.	/* prevent speculative execution */
404351 
405
-	/* Re-enable interrupts */
406
-#ifdef CONFIG_PPC_BOOK3E
407
-	wrteei	1
408
-#else
409
-	li	r10,MSR_RI
410
-	ori	r10,r10,MSR_EE
411
-	mtmsrd	r10,1
412
-#endif /* CONFIG_PPC_BOOK3E */
413
-
414
-	bl	save_nvgprs
415
-	addi	r3,r1,STACK_FRAME_OVERHEAD
416
-	bl	do_syscall_trace_leave
417
-	b	ret_from_except
352
+.Lsyscall_restore_regs:
353
+	ld	r3,_CTR(r1)
354
+	ld	r4,_XER(r1)
355
+	REST_NVGPRS(r1)
356
+	mtctr	r3
357
+	mtspr	SPRN_XER,r4
358
+	ld	r0,GPR0(r1)
359
+	REST_8GPRS(4, r1)
360
+	ld	r12,GPR12(r1)
361
+	b	.Lsyscall_restore_regs_cont
418362 
419363 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
420364 .Ltabort_syscall:
....@@ -442,9 +386,34 @@
442386 	RFI_TO_USER
443387 	b	.	/* prevent speculative execution */
444388 #endif
445
-_ASM_NOKPROBE_SYMBOL(system_call_common);
446
-_ASM_NOKPROBE_SYMBOL(system_call_exit);
447389 
390
+#ifdef CONFIG_PPC_BOOK3S
391
+_GLOBAL(ret_from_fork_scv)
392
+	bl	schedule_tail
393
+	REST_NVGPRS(r1)
394
+	li	r3,0	/* fork() return value */
395
+	b	.Lsyscall_vectored_common_exit
396
+#endif
397
+
398
+_GLOBAL(ret_from_fork)
399
+	bl	schedule_tail
400
+	REST_NVGPRS(r1)
401
+	li	r3,0	/* fork() return value */
402
+	b	.Lsyscall_exit
403
+
404
+_GLOBAL(ret_from_kernel_thread)
405
+	bl	schedule_tail
406
+	REST_NVGPRS(r1)
407
+	mtctr	r14
408
+	mr	r3,r15
409
+#ifdef PPC64_ELF_ABI_v2
410
+	mr	r12,r14
411
+#endif
412
+	bctrl
413
+	li	r3,0
414
+	b	.Lsyscall_exit
415
+
416
+#ifdef CONFIG_PPC_BOOK3E
448417 /* Save non-volatile GPRs, if not already saved. */
449418 _GLOBAL(save_nvgprs)
450419 	ld	r11,_TRAP(r1)
....@@ -455,73 +424,17 @@
455424 	std	r0,_TRAP(r1)
456425 	blr
457426 _ASM_NOKPROBE_SYMBOL(save_nvgprs);
458
-
459
-	
460
-/*
461
- * The sigsuspend and rt_sigsuspend system calls can call do_signal
462
- * and thus put the process into the stopped state where we might
463
- * want to examine its user state with ptrace.  Therefore we need
464
- * to save all the nonvolatile registers (r14 - r31) before calling
465
- * the C code.  Similarly, fork, vfork and clone need the full
466
- * register state on the stack so that it can be copied to the child.
467
- */
468
-
469
-_GLOBAL(ppc_fork)
470
-	bl	save_nvgprs
471
-	bl	sys_fork
472
-	b	.Lsyscall_exit
473
-
474
-_GLOBAL(ppc_vfork)
475
-	bl	save_nvgprs
476
-	bl	sys_vfork
477
-	b	.Lsyscall_exit
478
-
479
-_GLOBAL(ppc_clone)
480
-	bl	save_nvgprs
481
-	bl	sys_clone
482
-	b	.Lsyscall_exit
483
-
484
-_GLOBAL(ppc32_swapcontext)
485
-	bl	save_nvgprs
486
-	bl	compat_sys_swapcontext
487
-	b	.Lsyscall_exit
488
-
489
-_GLOBAL(ppc64_swapcontext)
490
-	bl	save_nvgprs
491
-	bl	sys_swapcontext
492
-	b	.Lsyscall_exit
493
-
494
-_GLOBAL(ppc_switch_endian)
495
-	bl	save_nvgprs
496
-	bl	sys_switch_endian
497
-	b	.Lsyscall_exit
498
-
499
-_GLOBAL(ret_from_fork)
500
-	bl	schedule_tail
501
-	REST_NVGPRS(r1)
502
-	li	r3,0
503
-	b	.Lsyscall_exit
504
-
505
-_GLOBAL(ret_from_kernel_thread)
506
-	bl	schedule_tail
507
-	REST_NVGPRS(r1)
508
-	mtlr	r14
509
-	mr	r3,r15
510
-#ifdef PPC64_ELF_ABI_v2
511
-	mr	r12,r14
512427 #endif
513
-	blrl
514
-	li	r3,0
515
-	b	.Lsyscall_exit
516428 
517429 #ifdef CONFIG_PPC_BOOK3S_64
518430 
519431 #define FLUSH_COUNT_CACHE	\
520432 1:	nop;			\
521
-	patch_site 1b, patch__call_flush_count_cache
522
-
523
-
524
-#define BCCTR_FLUSH	.long 0x4c400420
433
+	patch_site 1b, patch__call_flush_branch_caches1; \
434
+1:	nop;			\
435
+	patch_site 1b, patch__call_flush_branch_caches2; \
436
+1:	nop;			\
437
+	patch_site 1b, patch__call_flush_branch_caches3
525438 
526439 .macro nops number
527440 	.rept \number
....@@ -530,8 +443,8 @@
530443 .endm
531444 
532445 .balign 32
533
-.global flush_count_cache
534
-flush_count_cache:
446
+.global flush_branch_caches
447
+flush_branch_caches:
535448 	/* Save LR into r9 */
536449 	mflr	r9
537450 
....@@ -553,7 +466,7 @@
553466 	li	r9,0x7fff
554467 	mtctr	r9
555468 
556
-	BCCTR_FLUSH
469
+	PPC_BCCTR_FLUSH
557470 
558471 2:	nop
559472 	patch_site 2b patch__flush_count_cache_return
....@@ -562,7 +475,7 @@
562475 
563476 	.rept 278
564477 	.balign 32
565
-	BCCTR_FLUSH
478
+	PPC_BCCTR_FLUSH
566479 	nops	7
567480 	.endr
568481 
....@@ -576,7 +489,7 @@
576489  * state of one is saved on its kernel stack.  Then the state
577490  * of the other is restored from its kernel stack.  The memory
578491  * management hardware is updated to the second process's state.
579
- * Finally, we can return to the second process, via ret_from_except.
492
+ * Finally, we can return to the second process, via interrupt_return.
580493  * On entry, r3 points to the THREAD for the current task, r4
581494  * points to the THREAD for the new task.
582495  *
....@@ -595,14 +508,15 @@
595508 	std	r0,16(r1)
596509 	stdu	r1,-SWITCH_FRAME_SIZE(r1)
597510 	/* r3-r13 are caller saved -- Cort */
598
-	SAVE_8GPRS(14, r1)
599
-	SAVE_10GPRS(22, r1)
511
+	SAVE_NVGPRS(r1)
600512 	std	r0,_NIP(r1)	/* Return to switch caller */
601513 	mfcr	r23
602514 	std	r23,_CCR(r1)
603515 	std	r1,KSP(r3)	/* Set old stack pointer */
604516 
605
-	FLUSH_COUNT_CACHE
517
+	kuap_check_amr r9, r10
518
+
519
+	FLUSH_COUNT_CACHE	/* Clobbers r9, ctr */
606520 
607521 	/*
608522 	 * On SMP kernels, care must be taken because a task may be
....@@ -615,7 +529,7 @@
615529 	 * kernel/sched/core.c).
616530 	 *
617531 	 * Uncacheable stores in the case of involuntary preemption must
618
-	 * be taken care of. The smp_mb__before_spin_lock() in __schedule()
532
+	 * be taken care of. The smp_mb__after_spinlock() in __schedule()
619533 	 * is implemented as hwsync on powerpc, which orders MMIO too. So
620534 	 * long as there is an hwsync in the context switch path, it will
621535 	 * be executed on the source CPU after the task has performed
....@@ -637,6 +551,10 @@
637551 
638552 	addi	r6,r4,-THREAD	/* Convert THREAD to 'current' */
639553 	std	r6,PACACURRENT(r13)	/* Set new 'current' */
554
+#if defined(CONFIG_STACKPROTECTOR)
555
+	ld	r6, TASK_CANARY(r6)
556
+	std	r6, PACA_CANARY(r13)
557
+#endif
640558 
641559 	ld	r8,KSP(r4)	/* new stack pointer */
642560 #ifdef CONFIG_PPC_BOOK3S_64
....@@ -685,13 +603,15 @@
685603 
686604 	isync
687605 	slbie	r6
606
+BEGIN_FTR_SECTION
688607 	slbie	r6		/* Workaround POWER5 < DD2.1 issue */
608
+END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
689609 	slbmte	r7,r0
690610 	isync
691611 2:
692612 #endif /* CONFIG_PPC_BOOK3S_64 */
693613 
694
-	CURRENT_THREAD_INFO(r7, r8)  /* base of new stack */
614
+	clrrdi	r7, r8, THREAD_SHIFT	/* base of new stack */
695615 	/* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
696616 	   because we don't need to leave the 288-byte ABI gap at the
697617 	   top of the kernel stack. */
....@@ -712,8 +632,7 @@
712632 	mtcrf	0xFF,r6
713633 
714634 	/* r3-r13 are destroyed -- Cort */
715
-	REST_8GPRS(14, r1)
716
-	REST_10GPRS(22, r1)
635
+	REST_NVGPRS(r1)
717636 
718637 	/* convert old thread to its task_struct for return value */
719638 	addi	r3,r3,-THREAD
....@@ -722,417 +641,148 @@
722641 	addi	r1,r1,SWITCH_FRAME_SIZE
723642 	blr
724643 
725
-	.align	7
726
-_GLOBAL(ret_from_except)
727
-	ld	r11,_TRAP(r1)
728
-	andi.	r0,r11,1
729
-	bne	ret_from_except_lite
730
-	REST_NVGPRS(r1)
731
-
732
-_GLOBAL(ret_from_except_lite)
644
+#ifdef CONFIG_PPC_BOOK3S
733645 	/*
734
-	 * Disable interrupts so that current_thread_info()->flags
735
-	 * can't change between when we test it and when we return
736
-	 * from the interrupt.
646
+	 * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
647
+	 * touched, no exit work created, then this can be used.
737648 	 */
738
-#ifdef CONFIG_PPC_BOOK3E
739
-	wrteei	0
740
-#else
741
-	li	r10,MSR_RI
742
-	mtmsrd	r10,1		  /* Update machine state */
743
-#endif /* CONFIG_PPC_BOOK3E */
744
-
745
-	CURRENT_THREAD_INFO(r9, r1)
746
-	ld	r3,_MSR(r1)
747
-#ifdef CONFIG_PPC_BOOK3E
748
-	ld	r10,PACACURRENT(r13)
749
-#endif /* CONFIG_PPC_BOOK3E */
750
-	ld	r4,TI_FLAGS(r9)
751
-	andi.	r3,r3,MSR_PR
752
-	beq	resume_kernel
753
-#ifdef CONFIG_PPC_BOOK3E
754
-	lwz	r3,(THREAD+THREAD_DBCR0)(r10)
755
-#endif /* CONFIG_PPC_BOOK3E */
756
-
757
-	/* Check current_thread_info()->flags */
758
-	andi.	r0,r4,_TIF_USER_WORK_MASK
759
-	bne	1f
760
-#ifdef CONFIG_PPC_BOOK3E
761
-	/*
762
-	 * Check to see if the dbcr0 register is set up to debug.
763
-	 * Use the internal debug mode bit to do this.
764
-	 */
765
-	andis.	r0,r3,DBCR0_IDM@h
766
-	beq	restore
767
-	mfmsr	r0
768
-	rlwinm	r0,r0,0,~MSR_DE	/* Clear MSR.DE */
769
-	mtmsr	r0
770
-	mtspr	SPRN_DBCR0,r3
771
-	li	r10, -1
772
-	mtspr	SPRN_DBSR,r10
773
-	b	restore
774
-#else
649
+	.balign IFETCH_ALIGN_BYTES
650
+	.globl fast_interrupt_return
651
+fast_interrupt_return:
652
+_ASM_NOKPROBE_SYMBOL(fast_interrupt_return)
653
+	kuap_check_amr r3, r4
654
+	ld	r5,_MSR(r1)
655
+	andi.	r0,r5,MSR_PR
656
+	bne	.Lfast_user_interrupt_return
657
+	kuap_restore_amr r3, r4
658
+	andi.	r0,r5,MSR_RI
659
+	li	r3,0 /* 0 return value, no EMULATE_STACK_STORE */
660
+	bne+	.Lfast_kernel_interrupt_return
775661 	addi	r3,r1,STACK_FRAME_OVERHEAD
776
-	bl	restore_math
777
-	b	restore
778
-#endif
779
-1:	andi.	r0,r4,_TIF_NEED_RESCHED_MASK
780
-	beq	2f
781
-	bl	restore_interrupts
782
-	SCHEDULE_USER
783
-	b	ret_from_except_lite
784
-2:
785
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
786
-	andi.	r0,r4,_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM
787
-	bne	3f		/* only restore TM if nothing else to do */
662
+	bl	unrecoverable_exception
663
+	b	. /* should not get here */
664
+
665
+	.balign IFETCH_ALIGN_BYTES
666
+	.globl interrupt_return
667
+interrupt_return:
668
+_ASM_NOKPROBE_SYMBOL(interrupt_return)
669
+	ld	r4,_MSR(r1)
670
+	andi.	r0,r4,MSR_PR
671
+	beq	.Lkernel_interrupt_return
788672 	addi	r3,r1,STACK_FRAME_OVERHEAD
789
-	bl	restore_tm_state
790
-	b	restore
791
-3:
792
-#endif
793
-	bl	save_nvgprs
794
-	/*
795
-	 * Use a non volatile GPR to save and restore our thread_info flags
796
-	 * across the call to restore_interrupts.
797
-	 */
798
-	mr	r30,r4
799
-	bl	restore_interrupts
800
-	mr	r4,r30
801
-	addi	r3,r1,STACK_FRAME_OVERHEAD
802
-	bl	do_notify_resume
803
-	b	ret_from_except
673
+	bl	interrupt_exit_user_prepare
674
+	cmpdi	r3,0
675
+	bne-	.Lrestore_nvgprs
804676 
805
-resume_kernel:
806
-	/* check current_thread_info, _TIF_EMULATE_STACK_STORE */
807
-	andis.	r8,r4,_TIF_EMULATE_STACK_STORE@h
808
-	beq+	1f
677
+.Lfast_user_interrupt_return:
678
+	ld	r11,_NIP(r1)
679
+	ld	r12,_MSR(r1)
680
+BEGIN_FTR_SECTION
681
+	ld	r10,_PPR(r1)
682
+	mtspr	SPRN_PPR,r10
683
+END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
684
+	mtspr	SPRN_SRR0,r11
685
+	mtspr	SPRN_SRR1,r12
809686 
810
-	addi	r8,r1,INT_FRAME_SIZE	/* Get the kprobed function entry */
811
-
812
-	ld	r3,GPR1(r1)
813
-	subi	r3,r3,INT_FRAME_SIZE	/* dst: Allocate a trampoline exception frame */
814
-	mr	r4,r1			/* src:  current exception frame */
815
-	mr	r1,r3			/* Reroute the trampoline frame to r1 */
816
-
817
-	/* Copy from the original to the trampoline. */
818
-	li	r5,INT_FRAME_SIZE/8	/* size: INT_FRAME_SIZE */
819
-	li	r6,0			/* start offset: 0 */
820
-	mtctr	r5
821
-2:	ldx	r0,r6,r4
822
-	stdx	r0,r6,r3
823
-	addi	r6,r6,8
824
-	bdnz	2b
825
-
826
-	/* Do real store operation to complete stdu */
827
-	ld	r5,GPR1(r1)
828
-	std	r8,0(r5)
829
-
830
-	/* Clear _TIF_EMULATE_STACK_STORE flag */
831
-	lis	r11,_TIF_EMULATE_STACK_STORE@h
832
-	addi	r5,r9,TI_FLAGS
833
-0:	ldarx	r4,0,r5
834
-	andc	r4,r4,r11
835
-	stdcx.	r4,0,r5
836
-	bne-	0b
837
-1:
838
-
839
-#ifdef CONFIG_PREEMPT
840
-	/* Check if we need to preempt */
841
-	lwz	r8,TI_PREEMPT(r9)
842
-	cmpwi	0,r8,0		/* if non-zero, just restore regs and return */
843
-	bne	restore
844
-	andi.	r0,r4,_TIF_NEED_RESCHED
845
-	bne+	check_count
846
-
847
-	andi.	r0,r4,_TIF_NEED_RESCHED_LAZY
848
-	beq+	restore
849
-	lwz	r8,TI_PREEMPT_LAZY(r9)
850
-
851
-	/* Check that preempt_count() == 0 and interrupts are enabled */
852
-check_count:
853
-	cmpwi	cr0,r8,0
854
-	bne	restore
855
-	ld	r0,SOFTE(r1)
856
-	andi.	r0,r0,IRQS_DISABLED
857
-	bne	restore
858
-
859
-	/*
860
-	 * Here we are preempting the current task. We want to make
861
-	 * sure we are soft-disabled first and reconcile irq state.
862
-	 */
863
-	RECONCILE_IRQ_STATE(r3,r4)
864
-1:	bl	preempt_schedule_irq
865
-
866
-	/* Re-test flags and eventually loop */
867
-	CURRENT_THREAD_INFO(r9, r1)
868
-	ld	r4,TI_FLAGS(r9)
869
-	andi.	r0,r4,_TIF_NEED_RESCHED_MASK
870
-	bne	1b
871
-
872
-	/*
873
-	 * arch_local_irq_restore() from preempt_schedule_irq above may
874
-	 * enable hard interrupt but we really should disable interrupts
875
-	 * when we return from the interrupt, and so that we don't get
876
-	 * interrupted after loading SRR0/1.
877
-	 */
878
-#ifdef CONFIG_PPC_BOOK3E
879
-	wrteei	0
880
-#else
881
-	li	r10,MSR_RI
882
-	mtmsrd	r10,1		  /* Update machine state */
883
-#endif /* CONFIG_PPC_BOOK3E */
884
-#endif /* CONFIG_PREEMPT */
885
-
886
-	.globl	fast_exc_return_irq
887
-fast_exc_return_irq:
888
-restore:
889
-	/*
890
-	 * This is the main kernel exit path. First we check if we
891
-	 * are about to re-enable interrupts
892
-	 */
893
-	ld	r5,SOFTE(r1)
894
-	lbz	r6,PACAIRQSOFTMASK(r13)
895
-	andi.	r5,r5,IRQS_DISABLED
896
-	bne	.Lrestore_irq_off
897
-
898
-	/* We are enabling, were we already enabled ? Yes, just return */
899
-	andi.	r6,r6,IRQS_DISABLED
900
-	beq	cr0,.Ldo_restore
901
-
902
-	/*
903
-	 * We are about to soft-enable interrupts (we are hard disabled
904
-	 * at this point). We check if there's anything that needs to
905
-	 * be replayed first.
906
-	 */
907
-	lbz	r0,PACAIRQHAPPENED(r13)
908
-	cmpwi	cr0,r0,0
909
-	bne-	.Lrestore_check_irq_replay
910
-
911
-	/*
912
-	 * Get here when nothing happened while soft-disabled, just
913
-	 * soft-enable and move-on. We will hard-enable as a side
914
-	 * effect of rfi
915
-	 */
916
-.Lrestore_no_replay:
917
-	TRACE_ENABLE_INTS
918
-	li	r0,IRQS_ENABLED
919
-	stb	r0,PACAIRQSOFTMASK(r13);
920
-
921
-	/*
922
-	 * Final return path. BookE is handled in a different file
923
-	 */
924
-.Ldo_restore:
925
-#ifdef CONFIG_PPC_BOOK3E
926
-	b	exception_return_book3e
927
-#else
928
-	/*
929
-	 * Clear the reservation. If we know the CPU tracks the address of
930
-	 * the reservation then we can potentially save some cycles and use
931
-	 * a larx. On POWER6 and POWER7 this is significantly faster.
932
-	 */
933687 BEGIN_FTR_SECTION
934688 	stdcx.	r0,0,r1		/* to clear the reservation */
935689 FTR_SECTION_ELSE
936
-	ldarx	r4,0,r1
690
+	ldarx	r0,0,r1
937691 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
938692 
939
-	/*
940
-	 * Some code path such as load_up_fpu or altivec return directly
941
-	 * here. They run entirely hard disabled and do not alter the
942
-	 * interrupt state. They also don't use lwarx/stwcx. and thus
943
-	 * are known not to leave dangling reservations.
944
-	 */
945
-	.globl	fast_exception_return
946
-fast_exception_return:
947
-	ld	r3,_MSR(r1)
948
-	ld	r4,_CTR(r1)
949
-	ld	r0,_LINK(r1)
950
-	mtctr	r4
951
-	mtlr	r0
952
-	ld	r4,_XER(r1)
953
-	mtspr	SPRN_XER,r4
693
+	ld	r3,_CCR(r1)
694
+	ld	r4,_LINK(r1)
695
+	ld	r5,_CTR(r1)
696
+	ld	r6,_XER(r1)
697
+	li	r0,0
954698 
955
-	REST_8GPRS(5, r1)
956
-
957
-	andi.	r0,r3,MSR_RI
958
-	beq-	.Lunrecov_restore
959
-
960
-	/* Load PPR from thread struct before we clear MSR:RI */
961
-BEGIN_FTR_SECTION
962
-	ld	r2,PACACURRENT(r13)
963
-	ld	r2,TASKTHREADPPR(r2)
964
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
965
-
966
-	/*
967
-	 * Clear RI before restoring r13.  If we are returning to
968
-	 * userspace and we take an exception after restoring r13,
969
-	 * we end up corrupting the userspace r13 value.
970
-	 */
971
-	li	r4,0
972
-	mtmsrd	r4,1
973
-
974
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
975
-	/* TM debug */
976
-	std	r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
977
-#endif
978
-	/*
979
-	 * r13 is our per cpu area, only restore it if we are returning to
980
-	 * userspace the value stored in the stack frame may belong to
981
-	 * another CPU.
982
-	 */
983
-	andi.	r0,r3,MSR_PR
984
-	beq	1f
985
-BEGIN_FTR_SECTION
986
-	mtspr	SPRN_PPR,r2	/* Restore PPR */
987
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
988
-	ACCOUNT_CPU_USER_EXIT(r13, r2, r4)
699
+	REST_4GPRS(7, r1)
700
+	REST_2GPRS(11, r1)
989701 	REST_GPR(13, r1)
990702 
991
-	mtspr	SPRN_SRR1,r3
703
+	mtcr	r3
704
+	mtlr	r4
705
+	mtctr	r5
706
+	mtspr	SPRN_XER,r6
992707 
993
-	ld	r2,_CCR(r1)
994
-	mtcrf	0xFF,r2
995
-	ld	r2,_NIP(r1)
996
-	mtspr	SPRN_SRR0,r2
997
-
998
-	ld	r0,GPR0(r1)
999
-	ld	r2,GPR2(r1)
1000
-	ld	r3,GPR3(r1)
1001
-	ld	r4,GPR4(r1)
1002
-	ld	r1,GPR1(r1)
708
+	REST_4GPRS(2, r1)
709
+	REST_GPR(6, r1)
710
+	REST_GPR(0, r1)
711
+	REST_GPR(1, r1)
1003712 	RFI_TO_USER
1004713 	b	.	/* prevent speculative execution */
1005714 
1006
-1:	mtspr	SPRN_SRR1,r3
715
+.Lrestore_nvgprs:
716
+	REST_NVGPRS(r1)
717
+	b	.Lfast_user_interrupt_return
1007718 
1008
-	ld	r2,_CCR(r1)
1009
-	mtcrf	0xFF,r2
1010
-	ld	r2,_NIP(r1)
1011
-	mtspr	SPRN_SRR0,r2
719
+	.balign IFETCH_ALIGN_BYTES
720
+.Lkernel_interrupt_return:
721
+	addi	r3,r1,STACK_FRAME_OVERHEAD
722
+	bl	interrupt_exit_kernel_prepare
723
+
724
+.Lfast_kernel_interrupt_return:
725
+	cmpdi	cr1,r3,0
726
+	ld	r11,_NIP(r1)
727
+	ld	r12,_MSR(r1)
728
+	mtspr	SPRN_SRR0,r11
729
+	mtspr	SPRN_SRR1,r12
730
+
731
+BEGIN_FTR_SECTION
732
+	stdcx.	r0,0,r1		/* to clear the reservation */
733
+FTR_SECTION_ELSE
734
+	ldarx	r0,0,r1
735
+ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
736
+
737
+	ld	r3,_LINK(r1)
738
+	ld	r4,_CTR(r1)
739
+	ld	r5,_XER(r1)
740
+	ld	r6,_CCR(r1)
741
+	li	r0,0
742
+
743
+	REST_4GPRS(7, r1)
744
+	REST_2GPRS(11, r1)
745
+
746
+	mtlr	r3
747
+	mtctr	r4
748
+	mtspr	SPRN_XER,r5
1012749 
1013750 	/*
1014751 	 * Leaving a stale exception_marker on the stack can confuse
1015752 	 * the reliable stack unwinder later on. Clear it.
1016753 	 */
1017
-	li	r2,0
1018
-	std	r2,STACK_FRAME_OVERHEAD-16(r1)
754
+	std	r0,STACK_FRAME_OVERHEAD-16(r1)
1019755 
1020
-	ld	r0,GPR0(r1)
1021
-	ld	r2,GPR2(r1)
1022
-	ld	r3,GPR3(r1)
1023
-	ld	r4,GPR4(r1)
1024
-	ld	r1,GPR1(r1)
756
+	REST_4GPRS(2, r1)
757
+
758
+	bne-	cr1,1f /* emulate stack store */
759
+	mtcr	r6
760
+	REST_GPR(6, r1)
761
+	REST_GPR(0, r1)
762
+	REST_GPR(1, r1)
1025763 	RFI_TO_KERNEL
1026764 	b	.	/* prevent speculative execution */
1027765 
1028
-#endif /* CONFIG_PPC_BOOK3E */
1029
-
1030
-	/*
1031
-	 * We are returning to a context with interrupts soft disabled.
1032
-	 *
1033
-	 * However, we may also about to hard enable, so we need to
1034
-	 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
1035
-	 * or that bit can get out of sync and bad things will happen
766
+1:	/*
767
+	 * Emulate stack store with update. New r1 value was already calculated
768
+	 * and updated in our interrupt regs by emulate_loadstore, but we can't
769
+	 * store the previous value of r1 to the stack before re-loading our
770
+	 * registers from it, otherwise they could be clobbered.  Use
771
+	 * PACA_EXGEN as temporary storage to hold the store data, as
772
+	 * interrupts are disabled here so it won't be clobbered.
1036773 	 */
1037
-.Lrestore_irq_off:
1038
-	ld	r3,_MSR(r1)
1039
-	lbz	r7,PACAIRQHAPPENED(r13)
1040
-	andi.	r0,r3,MSR_EE
1041
-	beq	1f
1042
-	rlwinm	r7,r7,0,~PACA_IRQ_HARD_DIS
1043
-	stb	r7,PACAIRQHAPPENED(r13)
1044
-1:
1045
-#if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
1046
-	/* The interrupt should not have soft enabled. */
1047
-	lbz	r7,PACAIRQSOFTMASK(r13)
1048
-1:	tdeqi	r7,IRQS_ENABLED
1049
-	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1050
-#endif
1051
-	b	.Ldo_restore
774
+	mtcr	r6
775
+	std	r9,PACA_EXGEN+0(r13)
776
+	addi	r9,r1,INT_FRAME_SIZE /* get original r1 */
777
+	REST_GPR(6, r1)
778
+	REST_GPR(0, r1)
779
+	REST_GPR(1, r1)
780
+	std	r9,0(r1) /* perform store component of stdu */
781
+	ld	r9,PACA_EXGEN+0(r13)
1052782 
1053
-	/*
1054
-	 * Something did happen, check if a re-emit is needed
1055
-	 * (this also clears paca->irq_happened)
1056
-	 */
1057
-.Lrestore_check_irq_replay:
1058
-	/* XXX: We could implement a fast path here where we check
1059
-	 * for irq_happened being just 0x01, in which case we can
1060
-	 * clear it and return. That means that we would potentially
1061
-	 * miss a decrementer having wrapped all the way around.
1062
-	 *
1063
-	 * Still, this might be useful for things like hash_page
1064
-	 */
1065
-	bl	__check_irq_replay
1066
-	cmpwi	cr0,r3,0
1067
-	beq	.Lrestore_no_replay
1068
- 
1069
-	/*
1070
-	 * We need to re-emit an interrupt. We do so by re-using our
1071
-	 * existing exception frame. We first change the trap value,
1072
-	 * but we need to ensure we preserve the low nibble of it
1073
-	 */
1074
-	ld	r4,_TRAP(r1)
1075
-	clrldi	r4,r4,60
1076
-	or	r4,r4,r3
1077
-	std	r4,_TRAP(r1)
1078
-
1079
-	/*
1080
-	 * PACA_IRQ_HARD_DIS won't always be set here, so set it now
1081
-	 * to reconcile the IRQ state. Tracing is already accounted for.
1082
-	 */
1083
-	lbz	r4,PACAIRQHAPPENED(r13)
1084
-	ori	r4,r4,PACA_IRQ_HARD_DIS
1085
-	stb	r4,PACAIRQHAPPENED(r13)
1086
-
1087
-	/*
1088
-	 * Then find the right handler and call it. Interrupts are
1089
-	 * still soft-disabled and we keep them that way.
1090
-	*/
1091
-	cmpwi	cr0,r3,0x500
1092
-	bne	1f
1093
-	addi	r3,r1,STACK_FRAME_OVERHEAD;
1094
- 	bl	do_IRQ
1095
-	b	ret_from_except
1096
-1:	cmpwi	cr0,r3,0xf00
1097
-	bne	1f
1098
-	addi	r3,r1,STACK_FRAME_OVERHEAD;
1099
-	bl	performance_monitor_exception
1100
-	b	ret_from_except
1101
-1:	cmpwi	cr0,r3,0xe60
1102
-	bne	1f
1103
-	addi	r3,r1,STACK_FRAME_OVERHEAD;
1104
-	bl	handle_hmi_exception
1105
-	b	ret_from_except
1106
-1:	cmpwi	cr0,r3,0x900
1107
-	bne	1f
1108
-	addi	r3,r1,STACK_FRAME_OVERHEAD;
1109
-	bl	timer_interrupt
1110
-	b	ret_from_except
1111
-#ifdef CONFIG_PPC_DOORBELL
1112
-1:
1113
-#ifdef CONFIG_PPC_BOOK3E
1114
-	cmpwi	cr0,r3,0x280
1115
-#else
1116
-	cmpwi	cr0,r3,0xa00
1117
-#endif /* CONFIG_PPC_BOOK3E */
1118
-	bne	1f
1119
-	addi	r3,r1,STACK_FRAME_OVERHEAD;
1120
-	bl	doorbell_exception
1121
-#endif /* CONFIG_PPC_DOORBELL */
1122
-1:	b	ret_from_except /* What else to do here ? */
1123
- 
1124
-.Lunrecov_restore:
1125
-	addi	r3,r1,STACK_FRAME_OVERHEAD
1126
-	bl	unrecoverable_exception
1127
-	b	.Lunrecov_restore
1128
-
1129
-_ASM_NOKPROBE_SYMBOL(ret_from_except);
1130
-_ASM_NOKPROBE_SYMBOL(ret_from_except_lite);
1131
-_ASM_NOKPROBE_SYMBOL(resume_kernel);
1132
-_ASM_NOKPROBE_SYMBOL(fast_exc_return_irq);
1133
-_ASM_NOKPROBE_SYMBOL(restore);
1134
-_ASM_NOKPROBE_SYMBOL(fast_exception_return);
1135
-
783
+	RFI_TO_KERNEL
784
+	b	.	/* prevent speculative execution */
785
+#endif /* CONFIG_PPC_BOOK3S */
1136786 
1137787 #ifdef CONFIG_PPC_RTAS
1138788 /*
....@@ -1146,7 +796,7 @@
1146796 _GLOBAL(enter_rtas)
1147797 	mflr	r0
1148798 	std	r0,16(r1)
1149
-        stdu	r1,-RTAS_FRAME_SIZE(r1)	/* Save SP and create stack space. */
799
+        stdu	r1,-SWITCH_FRAME_SIZE(r1) /* Save SP and create stack space. */
1150800 
1151801 	/* Because RTAS is running in 32b mode, it clobbers the high order half
1152802 	 * of all registers that it saves.  We therefore save those registers
....@@ -1154,8 +804,7 @@
1154804    	 */
1155805 	SAVE_GPR(2, r1)			/* Save the TOC */
1156806 	SAVE_GPR(13, r1)		/* Save paca */
1157
-	SAVE_8GPRS(14, r1)		/* Save the non-volatiles */
1158
-	SAVE_10GPRS(22, r1)		/* ditto */
807
+	SAVE_NVGPRS(r1)			/* Save the non-volatiles */
1159808 
1160809 	mfcr	r4
1161810 	std	r4,_CCR(r1)
....@@ -1262,8 +911,7 @@
1262911 	/* relocation is on at this point */
1263912 	REST_GPR(2, r1)			/* Restore the TOC */
1264913 	REST_GPR(13, r1)		/* Restore paca */
1265
-	REST_8GPRS(14, r1)		/* Restore the non-volatiles */
1266
-	REST_10GPRS(22, r1)		/* ditto */
914
+	REST_NVGPRS(r1)			/* Restore the non-volatiles */
1267915 
1268916 	GET_PACA(r13)
1269917 
....@@ -1278,7 +926,7 @@
1278926 	ld	r8,_DSISR(r1)
1279927 	mtdsisr	r8
1280928 
1281
-        addi	r1,r1,RTAS_FRAME_SIZE	/* Unstack our frame */
929
+        addi	r1,r1,SWITCH_FRAME_SIZE	/* Unstack our frame */
1282930 	ld	r0,16(r1)		/* get return address */
1283931 
1284932 	mtlr    r0
....@@ -1289,7 +937,7 @@
1289937 _GLOBAL(enter_prom)
1290938 	mflr	r0
1291939 	std	r0,16(r1)
1292
-        stdu	r1,-PROM_FRAME_SIZE(r1)	/* Save SP and create stack space */
940
+        stdu	r1,-SWITCH_FRAME_SIZE(r1) /* Save SP and create stack space */
1293941 
1294942 	/* Because PROM is running in 32b mode, it clobbers the high order half
1295943 	 * of all registers that it saves.  We therefore save those registers
....@@ -1297,8 +945,7 @@
1297945    	 */
1298946 	SAVE_GPR(2, r1)
1299947 	SAVE_GPR(13, r1)
1300
-	SAVE_8GPRS(14, r1)
1301
-	SAVE_10GPRS(22, r1)
948
+	SAVE_NVGPRS(r1)
1302949 	mfcr	r10
1303950 	mfmsr	r11
1304951 	std	r10,_CCR(r1)
....@@ -1342,12 +989,11 @@
1342989 	/* Restore other registers */
1343990 	REST_GPR(2, r1)
1344991 	REST_GPR(13, r1)
1345
-	REST_8GPRS(14, r1)
1346
-	REST_10GPRS(22, r1)
992
+	REST_NVGPRS(r1)
1347993 	ld	r4,_CCR(r1)
1348994 	mtcr	r4
1349
-	
1350
-        addi	r1,r1,PROM_FRAME_SIZE
995
+
996
+        addi	r1,r1,SWITCH_FRAME_SIZE
1351997 	ld	r0,16(r1)
1352998 	mtlr    r0
1353999         blr