hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d
kernel/arch/powerpc/kernel/entry_64.S
....@@ -1,3 +1,4 @@
1
+/* SPDX-License-Identifier: GPL-2.0-or-later */
12 /*
23 * PowerPC version
34 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
....@@ -11,15 +12,11 @@
1112 *
1213 * This file contains the system call entry code, context switch
1314 * code, and exception/interrupt return code for PowerPC.
14
- *
15
- * This program is free software; you can redistribute it and/or
16
- * modify it under the terms of the GNU General Public License
17
- * as published by the Free Software Foundation; either version
18
- * 2 of the License, or (at your option) any later version.
1915 */
2016
2117 #include <linux/errno.h>
2218 #include <linux/err.h>
19
+#include <asm/cache.h>
2320 #include <asm/unistd.h>
2421 #include <asm/processor.h>
2522 #include <asm/page.h>
....@@ -46,6 +43,7 @@
4643 #include <asm/exception-64e.h>
4744 #endif
4845 #include <asm/feature-fixups.h>
46
+#include <asm/kup.h>
4947
5048 /*
5149 * System calls.
....@@ -54,6 +52,11 @@
5452 SYS_CALL_TABLE:
5553 .tc sys_call_table[TC],sys_call_table
5654
55
+#ifdef CONFIG_COMPAT
56
+COMPAT_SYS_CALL_TABLE:
57
+ .tc compat_sys_call_table[TC],compat_sys_call_table
58
+#endif
59
+
5760 /* This value is used to mark exception frames on the stack. */
5861 exception_marker:
5962 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
....@@ -61,360 +64,301 @@
6164 .section ".text"
6265 .align 7
6366
64
- .globl system_call_common
65
-system_call_common:
67
+#ifdef CONFIG_PPC_BOOK3S
68
+.macro system_call_vectored name trapnr
69
+ .globl system_call_vectored_\name
70
+system_call_vectored_\name:
71
+_ASM_NOKPROBE_SYMBOL(system_call_vectored_\name)
6672 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
6773 BEGIN_FTR_SECTION
6874 extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
6975 bne .Ltabort_syscall
7076 END_FTR_SECTION_IFSET(CPU_FTR_TM)
7177 #endif
72
- andi. r10,r12,MSR_PR
78
+ SCV_INTERRUPT_TO_KERNEL
7379 mr r10,r1
74
- addi r1,r1,-INT_FRAME_SIZE
75
- beq- 1f
7680 ld r1,PACAKSAVE(r13)
77
-1: std r10,0(r1)
81
+ std r10,0(r1)
7882 std r11,_NIP(r1)
7983 std r12,_MSR(r1)
8084 std r0,GPR0(r1)
8185 std r10,GPR1(r1)
82
- beq 2f /* if from kernel mode */
83
-#ifdef CONFIG_PPC_FSL_BOOK3E
84
-START_BTB_FLUSH_SECTION
85
- BTB_FLUSH(r10)
86
-END_BTB_FLUSH_SECTION
87
-#endif
88
- ACCOUNT_CPU_USER_ENTRY(r13, r10, r11)
89
-2: std r2,GPR2(r1)
86
+ std r2,GPR2(r1)
87
+ ld r2,PACATOC(r13)
88
+ mfcr r12
89
+ li r11,0
90
+ /* Can we avoid saving r3-r8 in common case? */
9091 std r3,GPR3(r1)
91
- mfcr r2
9292 std r4,GPR4(r1)
9393 std r5,GPR5(r1)
9494 std r6,GPR6(r1)
9595 std r7,GPR7(r1)
9696 std r8,GPR8(r1)
97
- li r11,0
97
+ /* Zero r9-r12, this should only be required when restoring all GPRs */
9898 std r11,GPR9(r1)
9999 std r11,GPR10(r1)
100100 std r11,GPR11(r1)
101101 std r11,GPR12(r1)
102
- std r11,_XER(r1)
103
- std r11,_CTR(r1)
104102 std r9,GPR13(r1)
105
- mflr r10
106
- /*
107
- * This clears CR0.SO (bit 28), which is the error indication on
108
- * return from this system call.
109
- */
110
- rldimi r2,r11,28,(63-28)
111
- li r11,0xc01
112
- std r10,_LINK(r1)
103
+ SAVE_NVGPRS(r1)
104
+ std r11,_XER(r1)
105
+ std r11,_LINK(r1)
106
+ std r11,_CTR(r1)
107
+
108
+ li r11,\trapnr
113109 std r11,_TRAP(r1)
110
+ std r12,_CCR(r1)
114111 std r3,ORIG_GPR3(r1)
115
- std r2,_CCR(r1)
116
- ld r2,PACATOC(r13)
117
- addi r9,r1,STACK_FRAME_OVERHEAD
112
+ addi r10,r1,STACK_FRAME_OVERHEAD
118113 ld r11,exception_marker@toc(r2)
119
- std r11,-16(r9) /* "regshere" marker */
120
-#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
121
-BEGIN_FW_FTR_SECTION
122
- beq 33f
123
- /* if from user, see if there are any DTL entries to process */
124
- ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
125
- ld r11,PACA_DTL_RIDX(r13) /* get log read index */
126
- addi r10,r10,LPPACA_DTLIDX
127
- LDX_BE r10,0,r10 /* get log write index */
128
- cmpd cr1,r11,r10
129
- beq+ cr1,33f
130
- bl accumulate_stolen_time
131
- REST_GPR(0,r1)
132
- REST_4GPRS(3,r1)
133
- REST_2GPRS(7,r1)
134
- addi r9,r1,STACK_FRAME_OVERHEAD
135
-33:
136
-END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
137
-#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
114
+ std r11,-16(r10) /* "regshere" marker */
115
+
116
+BEGIN_FTR_SECTION
117
+ HMT_MEDIUM
118
+END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
138119
139120 /*
140
- * A syscall should always be called with interrupts enabled
141
- * so we just unconditionally hard-enable here. When some kind
142
- * of irq tracing is used, we additionally check that condition
143
- * is correct
144
- */
145
-#if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
146
- lbz r10,PACAIRQSOFTMASK(r13)
147
-1: tdnei r10,IRQS_ENABLED
148
- EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
149
-#endif
150
-
151
-#ifdef CONFIG_PPC_BOOK3E
152
- wrteei 1
153
-#else
154
- li r11,MSR_RI
155
- ori r11,r11,MSR_EE
156
- mtmsrd r11,1
157
-#endif /* CONFIG_PPC_BOOK3E */
158
-
159
-system_call: /* label this so stack traces look sane */
160
- /* We do need to set SOFTE in the stack frame or the return
161
- * from interrupt will be painful
162
- */
163
- li r10,IRQS_ENABLED
164
- std r10,SOFTE(r1)
165
-
166
- CURRENT_THREAD_INFO(r11, r1)
167
- ld r10,TI_FLAGS(r11)
168
- andi. r11,r10,_TIF_SYSCALL_DOTRACE
169
- bne .Lsyscall_dotrace /* does not return */
170
- cmpldi 0,r0,NR_syscalls
171
- bge- .Lsyscall_enosys
172
-
173
-.Lsyscall:
174
-/*
175
- * Need to vector to 32 Bit or default sys_call_table here,
176
- * based on caller's run-mode / personality.
177
- */
178
- ld r11,SYS_CALL_TABLE@toc(2)
179
- andis. r10,r10,_TIF_32BIT@h
180
- beq 15f
181
- addi r11,r11,8 /* use 32-bit syscall entries */
182
- clrldi r3,r3,32
183
- clrldi r4,r4,32
184
- clrldi r5,r5,32
185
- clrldi r6,r6,32
186
- clrldi r7,r7,32
187
- clrldi r8,r8,32
188
-15:
189
- slwi r0,r0,4
190
-
191
- barrier_nospec_asm
192
- /*
193
- * Prevent the load of the handler below (based on the user-passed
194
- * system call number) being speculatively executed until the test
195
- * against NR_syscalls and branch to .Lsyscall_enosys above has
196
- * committed.
121
+ * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
122
+ * would clobber syscall parameters. Also we always enter with IRQs
123
+ * enabled and nothing pending. system_call_exception() will call
124
+ * trace_hardirqs_off().
125
+ *
126
+ * scv enters with MSR[EE]=1, so don't set PACA_IRQ_HARD_DIS. The
127
+ * entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED.
197128 */
198129
199
- ldx r12,r11,r0 /* Fetch system call handler [ptr] */
200
- mtctr r12
201
- bctrl /* Call handler */
130
+ /* Calling convention has r9 = orig r0, r10 = regs */
131
+ mr r9,r0
132
+ bl system_call_exception
202133
203
-.Lsyscall_exit:
204
- std r3,RESULT(r1)
134
+.Lsyscall_vectored_\name\()_exit:
135
+ addi r4,r1,STACK_FRAME_OVERHEAD
136
+ li r5,1 /* scv */
137
+ bl syscall_exit_prepare
205138
206
-#ifdef CONFIG_DEBUG_RSEQ
207
- /* Check whether the syscall is issued inside a restartable sequence */
208
- addi r3,r1,STACK_FRAME_OVERHEAD
209
- bl rseq_syscall
210
- ld r3,RESULT(r1)
211
-#endif
139
+ ld r2,_CCR(r1)
140
+ ld r4,_NIP(r1)
141
+ ld r5,_MSR(r1)
212142
213
- CURRENT_THREAD_INFO(r12, r1)
214
-
215
- ld r8,_MSR(r1)
216
-#ifdef CONFIG_PPC_BOOK3S
217
- /* No MSR:RI on BookE */
218
- andi. r10,r8,MSR_RI
219
- beq- .Lunrecov_restore
220
-#endif
221
-
222
-/*
223
- * This is a few instructions into the actual syscall exit path (which actually
224
- * starts at .Lsyscall_exit) to cater to kprobe blacklisting and to reduce the
225
- * number of visible symbols for profiling purposes.
226
- *
227
- * We can probe from system_call until this point as MSR_RI is set. But once it
228
- * is cleared below, we won't be able to take a trap.
229
- *
230
- * This is blacklisted from kprobes further below with _ASM_NOKPROBE_SYMBOL().
231
- */
232
-system_call_exit:
233
- /*
234
- * Disable interrupts so current_thread_info()->flags can't change,
235
- * and so that we don't get interrupted after loading SRR0/1.
236
- */
237
-#ifdef CONFIG_PPC_BOOK3E
238
- wrteei 0
239
-#else
240
- /*
241
- * For performance reasons we clear RI the same time that we
242
- * clear EE. We only need to clear RI just before we restore r13
243
- * below, but batching it with EE saves us one expensive mtmsrd call.
244
- * We have to be careful to restore RI if we branch anywhere from
245
- * here (eg syscall_exit_work).
246
- */
247
- li r11,0
248
- mtmsrd r11,1
249
-#endif /* CONFIG_PPC_BOOK3E */
250
-
251
- ld r9,TI_FLAGS(r12)
252
- li r11,-MAX_ERRNO
253
- lis r0,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)@h
254
- ori r0,r0,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)@l
255
- and. r0,r9,r0
256
- bne- .Lsyscall_exit_work
257
-
258
- andi. r0,r8,MSR_FP
259
- beq 2f
260
-#ifdef CONFIG_ALTIVEC
261
- andis. r0,r8,MSR_VEC@h
262
- bne 3f
263
-#endif
264
-2: addi r3,r1,STACK_FRAME_OVERHEAD
265
-#ifdef CONFIG_PPC_BOOK3S
266
- li r10,MSR_RI
267
- mtmsrd r10,1 /* Restore RI */
268
-#endif
269
- bl restore_math
270
-#ifdef CONFIG_PPC_BOOK3S
271
- li r11,0
272
- mtmsrd r11,1
273
-#endif
274
- ld r8,_MSR(r1)
275
- ld r3,RESULT(r1)
276
- li r11,-MAX_ERRNO
277
-
278
-3: cmpld r3,r11
279
- ld r5,_CCR(r1)
280
- bge- .Lsyscall_error
281
-.Lsyscall_error_cont:
282
- ld r7,_NIP(r1)
283143 BEGIN_FTR_SECTION
284144 stdcx. r0,0,r1 /* to clear the reservation */
285145 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
286
- andi. r6,r8,MSR_PR
287
- ld r4,_LINK(r1)
288
-
289
- beq- 1f
290
- ACCOUNT_CPU_USER_EXIT(r13, r11, r12)
291146
292147 BEGIN_FTR_SECTION
293148 HMT_MEDIUM_LOW
294149 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
295150
296
- ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
297
- ld r2,GPR2(r1)
298
- ld r1,GPR1(r1)
151
+ cmpdi r3,0
152
+ bne .Lsyscall_vectored_\name\()_restore_regs
153
+
154
+ /* rfscv returns with LR->NIA and CTR->MSR */
299155 mtlr r4
300
- mtcr r5
301
- mtspr SPRN_SRR0,r7
302
- mtspr SPRN_SRR1,r8
303
- RFI_TO_USER
304
- b . /* prevent speculative execution */
156
+ mtctr r5
305157
306
- /* exit to kernel */
307
-1: ld r2,GPR2(r1)
308
- ld r1,GPR1(r1)
309
- mtlr r4
310
- mtcr r5
311
- mtspr SPRN_SRR0,r7
312
- mtspr SPRN_SRR1,r8
313
- RFI_TO_KERNEL
314
- b . /* prevent speculative execution */
315
-
316
-.Lsyscall_error:
317
- oris r5,r5,0x1000 /* Set SO bit in CR */
318
- neg r3,r3
319
- std r5,_CCR(r1)
320
- b .Lsyscall_error_cont
321
-
322
-/* Traced system call support */
323
-.Lsyscall_dotrace:
324
- bl save_nvgprs
325
- addi r3,r1,STACK_FRAME_OVERHEAD
326
- bl do_syscall_trace_enter
327
-
328
- /*
329
- * We use the return value of do_syscall_trace_enter() as the syscall
330
- * number. If the syscall was rejected for any reason do_syscall_trace_enter()
331
- * returns an invalid syscall number and the test below against
332
- * NR_syscalls will fail.
333
- */
334
- mr r0,r3
335
-
336
- /* Restore argument registers just clobbered and/or possibly changed. */
337
- ld r3,GPR3(r1)
158
+ /* Could zero these as per ABI, but we may consider a stricter ABI
159
+ * which preserves these if libc implementations can benefit, so
160
+ * restore them for now until further measurement is done. */
161
+ ld r0,GPR0(r1)
338162 ld r4,GPR4(r1)
339163 ld r5,GPR5(r1)
340164 ld r6,GPR6(r1)
341165 ld r7,GPR7(r1)
342166 ld r8,GPR8(r1)
167
+ /* Zero volatile regs that may contain sensitive kernel data */
168
+ li r9,0
169
+ li r10,0
170
+ li r11,0
171
+ li r12,0
172
+ mtspr SPRN_XER,r0
343173
344
- /* Repopulate r9 and r10 for the syscall path */
345
- addi r9,r1,STACK_FRAME_OVERHEAD
346
- CURRENT_THREAD_INFO(r10, r1)
347
- ld r10,TI_FLAGS(r10)
174
+ /*
175
+ * We don't need to restore AMR on the way back to userspace for KUAP.
176
+ * The value of AMR only matters while we're in the kernel.
177
+ */
178
+ mtcr r2
179
+ ld r2,GPR2(r1)
180
+ ld r3,GPR3(r1)
181
+ ld r13,GPR13(r1)
182
+ ld r1,GPR1(r1)
183
+ RFSCV_TO_USER
184
+ b . /* prevent speculative execution */
348185
349
- cmpldi r0,NR_syscalls
350
- blt+ .Lsyscall
186
+.Lsyscall_vectored_\name\()_restore_regs:
187
+ li r3,0
188
+ mtmsrd r3,1
189
+ mtspr SPRN_SRR0,r4
190
+ mtspr SPRN_SRR1,r5
351191
352
- /* Return code is already in r3 thanks to do_syscall_trace_enter() */
353
- b .Lsyscall_exit
192
+ ld r3,_CTR(r1)
193
+ ld r4,_LINK(r1)
194
+ ld r5,_XER(r1)
354195
355
-
356
-.Lsyscall_enosys:
357
- li r3,-ENOSYS
358
- b .Lsyscall_exit
359
-
360
-.Lsyscall_exit_work:
361
-#ifdef CONFIG_PPC_BOOK3S
362
- li r10,MSR_RI
363
- mtmsrd r10,1 /* Restore RI */
364
-#endif
365
- /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
366
- If TIF_NOERROR is set, just save r3 as it is. */
367
-
368
- andis. r0,r9,_TIF_RESTOREALL@h
369
- beq+ 0f
370196 REST_NVGPRS(r1)
371
- b 2f
372
-0: cmpld r3,r11 /* r11 is -MAX_ERRNO */
373
- blt+ 1f
374
- andis. r0,r9,_TIF_NOERROR@h
375
- bne- 1f
376
- ld r5,_CCR(r1)
377
- neg r3,r3
378
- oris r5,r5,0x1000 /* Set SO bit in CR */
379
- std r5,_CCR(r1)
380
-1: std r3,GPR3(r1)
381
-2: andis. r0,r9,(_TIF_PERSYSCALL_MASK)@h
382
- beq 4f
197
+ ld r0,GPR0(r1)
198
+ mtcr r2
199
+ mtctr r3
200
+ mtlr r4
201
+ mtspr SPRN_XER,r5
202
+ REST_10GPRS(2, r1)
203
+ REST_2GPRS(12, r1)
204
+ ld r1,GPR1(r1)
205
+ RFI_TO_USER
206
+.endm
383207
384
- /* Clear per-syscall TIF flags if any are set. */
208
+system_call_vectored common 0x3000
209
+/*
210
+ * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0
211
+ * which is tested by system_call_exception when r0 is -1 (as set by vector
212
+ * entry code).
213
+ */
214
+system_call_vectored sigill 0x7ff0
385215
386
- lis r11,(_TIF_PERSYSCALL_MASK)@h
387
- addi r12,r12,TI_FLAGS
388
-3: ldarx r10,0,r12
389
- andc r10,r10,r11
390
- stdcx. r10,0,r12
391
- bne- 3b
392
- subi r12,r12,TI_FLAGS
393216
394
-4: /* Anything else left to do? */
217
+/*
218
+ * Entered via kernel return set up by kernel/sstep.c, must match entry regs
219
+ */
220
+ .globl system_call_vectored_emulate
221
+system_call_vectored_emulate:
222
+_ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate)
223
+ li r10,IRQS_ALL_DISABLED
224
+ stb r10,PACAIRQSOFTMASK(r13)
225
+ b system_call_vectored_common
226
+#endif
227
+
228
+ .balign IFETCH_ALIGN_BYTES
229
+ .globl system_call_common
230
+system_call_common:
231
+_ASM_NOKPROBE_SYMBOL(system_call_common)
232
+#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
395233 BEGIN_FTR_SECTION
396
- lis r3,INIT_PPR@highest /* Set thread.ppr = 3 */
397
- ld r10,PACACURRENT(r13)
398
- sldi r3,r3,32 /* bits 11-13 are used for ppr */
399
- std r3,TASKTHREADPPR(r10)
234
+ extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
235
+ bne .Ltabort_syscall
236
+END_FTR_SECTION_IFSET(CPU_FTR_TM)
237
+#endif
238
+ mr r10,r1
239
+ ld r1,PACAKSAVE(r13)
240
+ std r10,0(r1)
241
+ std r11,_NIP(r1)
242
+ std r12,_MSR(r1)
243
+ std r0,GPR0(r1)
244
+ std r10,GPR1(r1)
245
+ std r2,GPR2(r1)
246
+#ifdef CONFIG_PPC_FSL_BOOK3E
247
+START_BTB_FLUSH_SECTION
248
+ BTB_FLUSH(r10)
249
+END_BTB_FLUSH_SECTION
250
+#endif
251
+ ld r2,PACATOC(r13)
252
+ mfcr r12
253
+ li r11,0
254
+ /* Can we avoid saving r3-r8 in common case? */
255
+ std r3,GPR3(r1)
256
+ std r4,GPR4(r1)
257
+ std r5,GPR5(r1)
258
+ std r6,GPR6(r1)
259
+ std r7,GPR7(r1)
260
+ std r8,GPR8(r1)
261
+ /* Zero r9-r12, this should only be required when restoring all GPRs */
262
+ std r11,GPR9(r1)
263
+ std r11,GPR10(r1)
264
+ std r11,GPR11(r1)
265
+ std r11,GPR12(r1)
266
+ std r9,GPR13(r1)
267
+ SAVE_NVGPRS(r1)
268
+ std r11,_XER(r1)
269
+ std r11,_CTR(r1)
270
+ mflr r10
271
+
272
+ /*
273
+ * This clears CR0.SO (bit 28), which is the error indication on
274
+ * return from this system call.
275
+ */
276
+ rldimi r12,r11,28,(63-28)
277
+ li r11,0xc00
278
+ std r10,_LINK(r1)
279
+ std r11,_TRAP(r1)
280
+ std r12,_CCR(r1)
281
+ std r3,ORIG_GPR3(r1)
282
+ addi r10,r1,STACK_FRAME_OVERHEAD
283
+ ld r11,exception_marker@toc(r2)
284
+ std r11,-16(r10) /* "regshere" marker */
285
+
286
+ /*
287
+ * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
288
+ * would clobber syscall parameters. Also we always enter with IRQs
289
+ * enabled and nothing pending. system_call_exception() will call
290
+ * trace_hardirqs_off().
291
+ */
292
+ li r11,IRQS_ALL_DISABLED
293
+ li r12,PACA_IRQ_HARD_DIS
294
+ stb r11,PACAIRQSOFTMASK(r13)
295
+ stb r12,PACAIRQHAPPENED(r13)
296
+
297
+ /* Calling convention has r9 = orig r0, r10 = regs */
298
+ mr r9,r0
299
+ bl system_call_exception
300
+
301
+.Lsyscall_exit:
302
+ addi r4,r1,STACK_FRAME_OVERHEAD
303
+ li r5,0 /* !scv */
304
+ bl syscall_exit_prepare
305
+
306
+ ld r2,_CCR(r1)
307
+ ld r4,_NIP(r1)
308
+ ld r5,_MSR(r1)
309
+ ld r6,_LINK(r1)
310
+
311
+BEGIN_FTR_SECTION
312
+ stdcx. r0,0,r1 /* to clear the reservation */
313
+END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
314
+
315
+ mtspr SPRN_SRR0,r4
316
+ mtspr SPRN_SRR1,r5
317
+ mtlr r6
318
+
319
+ cmpdi r3,0
320
+ bne .Lsyscall_restore_regs
321
+ /* Zero volatile regs that may contain sensitive kernel data */
322
+ li r0,0
323
+ li r4,0
324
+ li r5,0
325
+ li r6,0
326
+ li r7,0
327
+ li r8,0
328
+ li r9,0
329
+ li r10,0
330
+ li r11,0
331
+ li r12,0
332
+ mtctr r0
333
+ mtspr SPRN_XER,r0
334
+.Lsyscall_restore_regs_cont:
335
+
336
+BEGIN_FTR_SECTION
337
+ HMT_MEDIUM_LOW
400338 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
401339
402
- andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
403
- beq ret_from_except_lite
340
+ /*
341
+ * We don't need to restore AMR on the way back to userspace for KUAP.
342
+ * The value of AMR only matters while we're in the kernel.
343
+ */
344
+ mtcr r2
345
+ ld r2,GPR2(r1)
346
+ ld r3,GPR3(r1)
347
+ ld r13,GPR13(r1)
348
+ ld r1,GPR1(r1)
349
+ RFI_TO_USER
350
+ b . /* prevent speculative execution */
404351
405
- /* Re-enable interrupts */
406
-#ifdef CONFIG_PPC_BOOK3E
407
- wrteei 1
408
-#else
409
- li r10,MSR_RI
410
- ori r10,r10,MSR_EE
411
- mtmsrd r10,1
412
-#endif /* CONFIG_PPC_BOOK3E */
413
-
414
- bl save_nvgprs
415
- addi r3,r1,STACK_FRAME_OVERHEAD
416
- bl do_syscall_trace_leave
417
- b ret_from_except
352
+.Lsyscall_restore_regs:
353
+ ld r3,_CTR(r1)
354
+ ld r4,_XER(r1)
355
+ REST_NVGPRS(r1)
356
+ mtctr r3
357
+ mtspr SPRN_XER,r4
358
+ ld r0,GPR0(r1)
359
+ REST_8GPRS(4, r1)
360
+ ld r12,GPR12(r1)
361
+ b .Lsyscall_restore_regs_cont
418362
419363 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
420364 .Ltabort_syscall:
....@@ -442,9 +386,34 @@
442386 RFI_TO_USER
443387 b . /* prevent speculative execution */
444388 #endif
445
-_ASM_NOKPROBE_SYMBOL(system_call_common);
446
-_ASM_NOKPROBE_SYMBOL(system_call_exit);
447389
390
+#ifdef CONFIG_PPC_BOOK3S
391
+_GLOBAL(ret_from_fork_scv)
392
+ bl schedule_tail
393
+ REST_NVGPRS(r1)
394
+ li r3,0 /* fork() return value */
395
+ b .Lsyscall_vectored_common_exit
396
+#endif
397
+
398
+_GLOBAL(ret_from_fork)
399
+ bl schedule_tail
400
+ REST_NVGPRS(r1)
401
+ li r3,0 /* fork() return value */
402
+ b .Lsyscall_exit
403
+
404
+_GLOBAL(ret_from_kernel_thread)
405
+ bl schedule_tail
406
+ REST_NVGPRS(r1)
407
+ mtctr r14
408
+ mr r3,r15
409
+#ifdef PPC64_ELF_ABI_v2
410
+ mr r12,r14
411
+#endif
412
+ bctrl
413
+ li r3,0
414
+ b .Lsyscall_exit
415
+
416
+#ifdef CONFIG_PPC_BOOK3E
448417 /* Save non-volatile GPRs, if not already saved. */
449418 _GLOBAL(save_nvgprs)
450419 ld r11,_TRAP(r1)
....@@ -455,73 +424,17 @@
455424 std r0,_TRAP(r1)
456425 blr
457426 _ASM_NOKPROBE_SYMBOL(save_nvgprs);
458
-
459
-
460
-/*
461
- * The sigsuspend and rt_sigsuspend system calls can call do_signal
462
- * and thus put the process into the stopped state where we might
463
- * want to examine its user state with ptrace. Therefore we need
464
- * to save all the nonvolatile registers (r14 - r31) before calling
465
- * the C code. Similarly, fork, vfork and clone need the full
466
- * register state on the stack so that it can be copied to the child.
467
- */
468
-
469
-_GLOBAL(ppc_fork)
470
- bl save_nvgprs
471
- bl sys_fork
472
- b .Lsyscall_exit
473
-
474
-_GLOBAL(ppc_vfork)
475
- bl save_nvgprs
476
- bl sys_vfork
477
- b .Lsyscall_exit
478
-
479
-_GLOBAL(ppc_clone)
480
- bl save_nvgprs
481
- bl sys_clone
482
- b .Lsyscall_exit
483
-
484
-_GLOBAL(ppc32_swapcontext)
485
- bl save_nvgprs
486
- bl compat_sys_swapcontext
487
- b .Lsyscall_exit
488
-
489
-_GLOBAL(ppc64_swapcontext)
490
- bl save_nvgprs
491
- bl sys_swapcontext
492
- b .Lsyscall_exit
493
-
494
-_GLOBAL(ppc_switch_endian)
495
- bl save_nvgprs
496
- bl sys_switch_endian
497
- b .Lsyscall_exit
498
-
499
-_GLOBAL(ret_from_fork)
500
- bl schedule_tail
501
- REST_NVGPRS(r1)
502
- li r3,0
503
- b .Lsyscall_exit
504
-
505
-_GLOBAL(ret_from_kernel_thread)
506
- bl schedule_tail
507
- REST_NVGPRS(r1)
508
- mtlr r14
509
- mr r3,r15
510
-#ifdef PPC64_ELF_ABI_v2
511
- mr r12,r14
512427 #endif
513
- blrl
514
- li r3,0
515
- b .Lsyscall_exit
516428
517429 #ifdef CONFIG_PPC_BOOK3S_64
518430
519431 #define FLUSH_COUNT_CACHE \
520432 1: nop; \
521
- patch_site 1b, patch__call_flush_count_cache
522
-
523
-
524
-#define BCCTR_FLUSH .long 0x4c400420
433
+ patch_site 1b, patch__call_flush_branch_caches1; \
434
+1: nop; \
435
+ patch_site 1b, patch__call_flush_branch_caches2; \
436
+1: nop; \
437
+ patch_site 1b, patch__call_flush_branch_caches3
525438
526439 .macro nops number
527440 .rept \number
....@@ -530,8 +443,8 @@
530443 .endm
531444
532445 .balign 32
533
-.global flush_count_cache
534
-flush_count_cache:
446
+.global flush_branch_caches
447
+flush_branch_caches:
535448 /* Save LR into r9 */
536449 mflr r9
537450
....@@ -553,7 +466,7 @@
553466 li r9,0x7fff
554467 mtctr r9
555468
556
- BCCTR_FLUSH
469
+ PPC_BCCTR_FLUSH
557470
558471 2: nop
559472 patch_site 2b patch__flush_count_cache_return
....@@ -562,7 +475,7 @@
562475
563476 .rept 278
564477 .balign 32
565
- BCCTR_FLUSH
478
+ PPC_BCCTR_FLUSH
566479 nops 7
567480 .endr
568481
....@@ -576,7 +489,7 @@
576489 * state of one is saved on its kernel stack. Then the state
577490 * of the other is restored from its kernel stack. The memory
578491 * management hardware is updated to the second process's state.
579
- * Finally, we can return to the second process, via ret_from_except.
492
+ * Finally, we can return to the second process, via interrupt_return.
580493 * On entry, r3 points to the THREAD for the current task, r4
581494 * points to the THREAD for the new task.
582495 *
....@@ -595,14 +508,15 @@
595508 std r0,16(r1)
596509 stdu r1,-SWITCH_FRAME_SIZE(r1)
597510 /* r3-r13 are caller saved -- Cort */
598
- SAVE_8GPRS(14, r1)
599
- SAVE_10GPRS(22, r1)
511
+ SAVE_NVGPRS(r1)
600512 std r0,_NIP(r1) /* Return to switch caller */
601513 mfcr r23
602514 std r23,_CCR(r1)
603515 std r1,KSP(r3) /* Set old stack pointer */
604516
605
- FLUSH_COUNT_CACHE
517
+ kuap_check_amr r9, r10
518
+
519
+ FLUSH_COUNT_CACHE /* Clobbers r9, ctr */
606520
607521 /*
608522 * On SMP kernels, care must be taken because a task may be
....@@ -615,7 +529,7 @@
615529 * kernel/sched/core.c).
616530 *
617531 * Uncacheable stores in the case of involuntary preemption must
618
- * be taken care of. The smp_mb__before_spin_lock() in __schedule()
532
+ * be taken care of. The smp_mb__after_spinlock() in __schedule()
619533 * is implemented as hwsync on powerpc, which orders MMIO too. So
620534 * long as there is an hwsync in the context switch path, it will
621535 * be executed on the source CPU after the task has performed
....@@ -637,6 +551,10 @@
637551
638552 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
639553 std r6,PACACURRENT(r13) /* Set new 'current' */
554
+#if defined(CONFIG_STACKPROTECTOR)
555
+ ld r6, TASK_CANARY(r6)
556
+ std r6, PACA_CANARY(r13)
557
+#endif
640558
641559 ld r8,KSP(r4) /* new stack pointer */
642560 #ifdef CONFIG_PPC_BOOK3S_64
....@@ -685,13 +603,15 @@
685603
686604 isync
687605 slbie r6
606
+BEGIN_FTR_SECTION
688607 slbie r6 /* Workaround POWER5 < DD2.1 issue */
608
+END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
689609 slbmte r7,r0
690610 isync
691611 2:
692612 #endif /* CONFIG_PPC_BOOK3S_64 */
693613
694
- CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
614
+ clrrdi r7, r8, THREAD_SHIFT /* base of new stack */
695615 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
696616 because we don't need to leave the 288-byte ABI gap at the
697617 top of the kernel stack. */
....@@ -712,8 +632,7 @@
712632 mtcrf 0xFF,r6
713633
714634 /* r3-r13 are destroyed -- Cort */
715
- REST_8GPRS(14, r1)
716
- REST_10GPRS(22, r1)
635
+ REST_NVGPRS(r1)
717636
718637 /* convert old thread to its task_struct for return value */
719638 addi r3,r3,-THREAD
....@@ -722,417 +641,148 @@
722641 addi r1,r1,SWITCH_FRAME_SIZE
723642 blr
724643
725
- .align 7
726
-_GLOBAL(ret_from_except)
727
- ld r11,_TRAP(r1)
728
- andi. r0,r11,1
729
- bne ret_from_except_lite
730
- REST_NVGPRS(r1)
731
-
732
-_GLOBAL(ret_from_except_lite)
644
+#ifdef CONFIG_PPC_BOOK3S
733645 /*
734
- * Disable interrupts so that current_thread_info()->flags
735
- * can't change between when we test it and when we return
736
- * from the interrupt.
646
+ * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
647
+ * touched, no exit work created, then this can be used.
737648 */
738
-#ifdef CONFIG_PPC_BOOK3E
739
- wrteei 0
740
-#else
741
- li r10,MSR_RI
742
- mtmsrd r10,1 /* Update machine state */
743
-#endif /* CONFIG_PPC_BOOK3E */
744
-
745
- CURRENT_THREAD_INFO(r9, r1)
746
- ld r3,_MSR(r1)
747
-#ifdef CONFIG_PPC_BOOK3E
748
- ld r10,PACACURRENT(r13)
749
-#endif /* CONFIG_PPC_BOOK3E */
750
- ld r4,TI_FLAGS(r9)
751
- andi. r3,r3,MSR_PR
752
- beq resume_kernel
753
-#ifdef CONFIG_PPC_BOOK3E
754
- lwz r3,(THREAD+THREAD_DBCR0)(r10)
755
-#endif /* CONFIG_PPC_BOOK3E */
756
-
757
- /* Check current_thread_info()->flags */
758
- andi. r0,r4,_TIF_USER_WORK_MASK
759
- bne 1f
760
-#ifdef CONFIG_PPC_BOOK3E
761
- /*
762
- * Check to see if the dbcr0 register is set up to debug.
763
- * Use the internal debug mode bit to do this.
764
- */
765
- andis. r0,r3,DBCR0_IDM@h
766
- beq restore
767
- mfmsr r0
768
- rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
769
- mtmsr r0
770
- mtspr SPRN_DBCR0,r3
771
- li r10, -1
772
- mtspr SPRN_DBSR,r10
773
- b restore
774
-#else
649
+ .balign IFETCH_ALIGN_BYTES
650
+ .globl fast_interrupt_return
651
+fast_interrupt_return:
652
+_ASM_NOKPROBE_SYMBOL(fast_interrupt_return)
653
+ kuap_check_amr r3, r4
654
+ ld r5,_MSR(r1)
655
+ andi. r0,r5,MSR_PR
656
+ bne .Lfast_user_interrupt_return
657
+ kuap_restore_amr r3, r4
658
+ andi. r0,r5,MSR_RI
659
+ li r3,0 /* 0 return value, no EMULATE_STACK_STORE */
660
+ bne+ .Lfast_kernel_interrupt_return
775661 addi r3,r1,STACK_FRAME_OVERHEAD
776
- bl restore_math
777
- b restore
778
-#endif
779
-1: andi. r0,r4,_TIF_NEED_RESCHED_MASK
780
- beq 2f
781
- bl restore_interrupts
782
- SCHEDULE_USER
783
- b ret_from_except_lite
784
-2:
785
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
786
- andi. r0,r4,_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM
787
- bne 3f /* only restore TM if nothing else to do */
662
+ bl unrecoverable_exception
663
+ b . /* should not get here */
664
+
665
+ .balign IFETCH_ALIGN_BYTES
666
+ .globl interrupt_return
667
+interrupt_return:
668
+_ASM_NOKPROBE_SYMBOL(interrupt_return)
669
+ ld r4,_MSR(r1)
670
+ andi. r0,r4,MSR_PR
671
+ beq .Lkernel_interrupt_return
788672 addi r3,r1,STACK_FRAME_OVERHEAD
789
- bl restore_tm_state
790
- b restore
791
-3:
792
-#endif
793
- bl save_nvgprs
794
- /*
795
- * Use a non volatile GPR to save and restore our thread_info flags
796
- * across the call to restore_interrupts.
797
- */
798
- mr r30,r4
799
- bl restore_interrupts
800
- mr r4,r30
801
- addi r3,r1,STACK_FRAME_OVERHEAD
802
- bl do_notify_resume
803
- b ret_from_except
673
+ bl interrupt_exit_user_prepare
674
+ cmpdi r3,0
675
+ bne- .Lrestore_nvgprs
804676
805
-resume_kernel:
806
- /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
807
- andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
808
- beq+ 1f
677
+.Lfast_user_interrupt_return:
678
+ ld r11,_NIP(r1)
679
+ ld r12,_MSR(r1)
680
+BEGIN_FTR_SECTION
681
+ ld r10,_PPR(r1)
682
+ mtspr SPRN_PPR,r10
683
+END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
684
+ mtspr SPRN_SRR0,r11
685
+ mtspr SPRN_SRR1,r12
809686
810
- addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
811
-
812
- ld r3,GPR1(r1)
813
- subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
814
- mr r4,r1 /* src: current exception frame */
815
- mr r1,r3 /* Reroute the trampoline frame to r1 */
816
-
817
- /* Copy from the original to the trampoline. */
818
- li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
819
- li r6,0 /* start offset: 0 */
820
- mtctr r5
821
-2: ldx r0,r6,r4
822
- stdx r0,r6,r3
823
- addi r6,r6,8
824
- bdnz 2b
825
-
826
- /* Do real store operation to complete stdu */
827
- ld r5,GPR1(r1)
828
- std r8,0(r5)
829
-
830
- /* Clear _TIF_EMULATE_STACK_STORE flag */
831
- lis r11,_TIF_EMULATE_STACK_STORE@h
832
- addi r5,r9,TI_FLAGS
833
-0: ldarx r4,0,r5
834
- andc r4,r4,r11
835
- stdcx. r4,0,r5
836
- bne- 0b
837
-1:
838
-
839
-#ifdef CONFIG_PREEMPT
840
- /* Check if we need to preempt */
841
- lwz r8,TI_PREEMPT(r9)
842
- cmpwi 0,r8,0 /* if non-zero, just restore regs and return */
843
- bne restore
844
- andi. r0,r4,_TIF_NEED_RESCHED
845
- bne+ check_count
846
-
847
- andi. r0,r4,_TIF_NEED_RESCHED_LAZY
848
- beq+ restore
849
- lwz r8,TI_PREEMPT_LAZY(r9)
850
-
851
- /* Check that preempt_count() == 0 and interrupts are enabled */
852
-check_count:
853
- cmpwi cr0,r8,0
854
- bne restore
855
- ld r0,SOFTE(r1)
856
- andi. r0,r0,IRQS_DISABLED
857
- bne restore
858
-
859
- /*
860
- * Here we are preempting the current task. We want to make
861
- * sure we are soft-disabled first and reconcile irq state.
862
- */
863
- RECONCILE_IRQ_STATE(r3,r4)
864
-1: bl preempt_schedule_irq
865
-
866
- /* Re-test flags and eventually loop */
867
- CURRENT_THREAD_INFO(r9, r1)
868
- ld r4,TI_FLAGS(r9)
869
- andi. r0,r4,_TIF_NEED_RESCHED_MASK
870
- bne 1b
871
-
872
- /*
873
- * arch_local_irq_restore() from preempt_schedule_irq above may
874
- * enable hard interrupt but we really should disable interrupts
875
- * when we return from the interrupt, and so that we don't get
876
- * interrupted after loading SRR0/1.
877
- */
878
-#ifdef CONFIG_PPC_BOOK3E
879
- wrteei 0
880
-#else
881
- li r10,MSR_RI
882
- mtmsrd r10,1 /* Update machine state */
883
-#endif /* CONFIG_PPC_BOOK3E */
884
-#endif /* CONFIG_PREEMPT */
885
-
886
- .globl fast_exc_return_irq
887
-fast_exc_return_irq:
888
-restore:
889
- /*
890
- * This is the main kernel exit path. First we check if we
891
- * are about to re-enable interrupts
892
- */
893
- ld r5,SOFTE(r1)
894
- lbz r6,PACAIRQSOFTMASK(r13)
895
- andi. r5,r5,IRQS_DISABLED
896
- bne .Lrestore_irq_off
897
-
898
- /* We are enabling, were we already enabled ? Yes, just return */
899
- andi. r6,r6,IRQS_DISABLED
900
- beq cr0,.Ldo_restore
901
-
902
- /*
903
- * We are about to soft-enable interrupts (we are hard disabled
904
- * at this point). We check if there's anything that needs to
905
- * be replayed first.
906
- */
907
- lbz r0,PACAIRQHAPPENED(r13)
908
- cmpwi cr0,r0,0
909
- bne- .Lrestore_check_irq_replay
910
-
911
- /*
912
- * Get here when nothing happened while soft-disabled, just
913
- * soft-enable and move-on. We will hard-enable as a side
914
- * effect of rfi
915
- */
916
-.Lrestore_no_replay:
917
- TRACE_ENABLE_INTS
918
- li r0,IRQS_ENABLED
919
- stb r0,PACAIRQSOFTMASK(r13);
920
-
921
- /*
922
- * Final return path. BookE is handled in a different file
923
- */
924
-.Ldo_restore:
925
-#ifdef CONFIG_PPC_BOOK3E
926
- b exception_return_book3e
927
-#else
928
- /*
929
- * Clear the reservation. If we know the CPU tracks the address of
930
- * the reservation then we can potentially save some cycles and use
931
- * a larx. On POWER6 and POWER7 this is significantly faster.
932
- */
933687 BEGIN_FTR_SECTION
934688 stdcx. r0,0,r1 /* to clear the reservation */
935689 FTR_SECTION_ELSE
936
- ldarx r4,0,r1
690
+ ldarx r0,0,r1
937691 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
938692
939
- /*
940
- * Some code path such as load_up_fpu or altivec return directly
941
- * here. They run entirely hard disabled and do not alter the
942
- * interrupt state. They also don't use lwarx/stwcx. and thus
943
- * are known not to leave dangling reservations.
944
- */
945
- .globl fast_exception_return
946
-fast_exception_return:
947
- ld r3,_MSR(r1)
948
- ld r4,_CTR(r1)
949
- ld r0,_LINK(r1)
950
- mtctr r4
951
- mtlr r0
952
- ld r4,_XER(r1)
953
- mtspr SPRN_XER,r4
693
+ ld r3,_CCR(r1)
694
+ ld r4,_LINK(r1)
695
+ ld r5,_CTR(r1)
696
+ ld r6,_XER(r1)
697
+ li r0,0
954698
955
- REST_8GPRS(5, r1)
956
-
957
- andi. r0,r3,MSR_RI
958
- beq- .Lunrecov_restore
959
-
960
- /* Load PPR from thread struct before we clear MSR:RI */
961
-BEGIN_FTR_SECTION
962
- ld r2,PACACURRENT(r13)
963
- ld r2,TASKTHREADPPR(r2)
964
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
965
-
966
- /*
967
- * Clear RI before restoring r13. If we are returning to
968
- * userspace and we take an exception after restoring r13,
969
- * we end up corrupting the userspace r13 value.
970
- */
971
- li r4,0
972
- mtmsrd r4,1
973
-
974
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
975
- /* TM debug */
976
- std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
977
-#endif
978
- /*
979
- * r13 is our per cpu area, only restore it if we are returning to
980
- * userspace the value stored in the stack frame may belong to
981
- * another CPU.
982
- */
983
- andi. r0,r3,MSR_PR
984
- beq 1f
985
-BEGIN_FTR_SECTION
986
- mtspr SPRN_PPR,r2 /* Restore PPR */
987
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
988
- ACCOUNT_CPU_USER_EXIT(r13, r2, r4)
699
+ REST_4GPRS(7, r1)
700
+ REST_2GPRS(11, r1)
989701 REST_GPR(13, r1)
990702
991
- mtspr SPRN_SRR1,r3
703
+ mtcr r3
704
+ mtlr r4
705
+ mtctr r5
706
+ mtspr SPRN_XER,r6
992707
993
- ld r2,_CCR(r1)
994
- mtcrf 0xFF,r2
995
- ld r2,_NIP(r1)
996
- mtspr SPRN_SRR0,r2
997
-
998
- ld r0,GPR0(r1)
999
- ld r2,GPR2(r1)
1000
- ld r3,GPR3(r1)
1001
- ld r4,GPR4(r1)
1002
- ld r1,GPR1(r1)
708
+ REST_4GPRS(2, r1)
709
+ REST_GPR(6, r1)
710
+ REST_GPR(0, r1)
711
+ REST_GPR(1, r1)
1003712 RFI_TO_USER
1004713 b . /* prevent speculative execution */
1005714
1006
-1: mtspr SPRN_SRR1,r3
715
+.Lrestore_nvgprs:
716
+ REST_NVGPRS(r1)
717
+ b .Lfast_user_interrupt_return
1007718
1008
- ld r2,_CCR(r1)
1009
- mtcrf 0xFF,r2
1010
- ld r2,_NIP(r1)
1011
- mtspr SPRN_SRR0,r2
719
+ .balign IFETCH_ALIGN_BYTES
720
+.Lkernel_interrupt_return:
721
+ addi r3,r1,STACK_FRAME_OVERHEAD
722
+ bl interrupt_exit_kernel_prepare
723
+
724
+.Lfast_kernel_interrupt_return:
725
+ cmpdi cr1,r3,0
726
+ ld r11,_NIP(r1)
727
+ ld r12,_MSR(r1)
728
+ mtspr SPRN_SRR0,r11
729
+ mtspr SPRN_SRR1,r12
730
+
731
+BEGIN_FTR_SECTION
732
+ stdcx. r0,0,r1 /* to clear the reservation */
733
+FTR_SECTION_ELSE
734
+ ldarx r0,0,r1
735
+ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
736
+
737
+ ld r3,_LINK(r1)
738
+ ld r4,_CTR(r1)
739
+ ld r5,_XER(r1)
740
+ ld r6,_CCR(r1)
741
+ li r0,0
742
+
743
+ REST_4GPRS(7, r1)
744
+ REST_2GPRS(11, r1)
745
+
746
+ mtlr r3
747
+ mtctr r4
748
+ mtspr SPRN_XER,r5
1012749
1013750 /*
1014751 * Leaving a stale exception_marker on the stack can confuse
1015752 * the reliable stack unwinder later on. Clear it.
1016753 */
1017
- li r2,0
1018
- std r2,STACK_FRAME_OVERHEAD-16(r1)
754
+ std r0,STACK_FRAME_OVERHEAD-16(r1)
1019755
1020
- ld r0,GPR0(r1)
1021
- ld r2,GPR2(r1)
1022
- ld r3,GPR3(r1)
1023
- ld r4,GPR4(r1)
1024
- ld r1,GPR1(r1)
756
+ REST_4GPRS(2, r1)
757
+
758
+ bne- cr1,1f /* emulate stack store */
759
+ mtcr r6
760
+ REST_GPR(6, r1)
761
+ REST_GPR(0, r1)
762
+ REST_GPR(1, r1)
1025763 RFI_TO_KERNEL
1026764 b . /* prevent speculative execution */
1027765
1028
-#endif /* CONFIG_PPC_BOOK3E */
1029
-
1030
- /*
1031
- * We are returning to a context with interrupts soft disabled.
1032
- *
1033
- * However, we may also about to hard enable, so we need to
1034
- * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
1035
- * or that bit can get out of sync and bad things will happen
766
+1: /*
767
+ * Emulate stack store with update. New r1 value was already calculated
768
+ * and updated in our interrupt regs by emulate_loadstore, but we can't
769
+ * store the previous value of r1 to the stack before re-loading our
770
+ * registers from it, otherwise they could be clobbered. Use
771
+ * PACA_EXGEN as temporary storage to hold the store data, as
772
+ * interrupts are disabled here so it won't be clobbered.
1036773 */
1037
-.Lrestore_irq_off:
1038
- ld r3,_MSR(r1)
1039
- lbz r7,PACAIRQHAPPENED(r13)
1040
- andi. r0,r3,MSR_EE
1041
- beq 1f
1042
- rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
1043
- stb r7,PACAIRQHAPPENED(r13)
1044
-1:
1045
-#if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
1046
- /* The interrupt should not have soft enabled. */
1047
- lbz r7,PACAIRQSOFTMASK(r13)
1048
-1: tdeqi r7,IRQS_ENABLED
1049
- EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1050
-#endif
1051
- b .Ldo_restore
774
+ mtcr r6
775
+ std r9,PACA_EXGEN+0(r13)
776
+ addi r9,r1,INT_FRAME_SIZE /* get original r1 */
777
+ REST_GPR(6, r1)
778
+ REST_GPR(0, r1)
779
+ REST_GPR(1, r1)
780
+ std r9,0(r1) /* perform store component of stdu */
781
+ ld r9,PACA_EXGEN+0(r13)
1052782
1053
- /*
1054
- * Something did happen, check if a re-emit is needed
1055
- * (this also clears paca->irq_happened)
1056
- */
1057
-.Lrestore_check_irq_replay:
1058
- /* XXX: We could implement a fast path here where we check
1059
- * for irq_happened being just 0x01, in which case we can
1060
- * clear it and return. That means that we would potentially
1061
- * miss a decrementer having wrapped all the way around.
1062
- *
1063
- * Still, this might be useful for things like hash_page
1064
- */
1065
- bl __check_irq_replay
1066
- cmpwi cr0,r3,0
1067
- beq .Lrestore_no_replay
1068
-
1069
- /*
1070
- * We need to re-emit an interrupt. We do so by re-using our
1071
- * existing exception frame. We first change the trap value,
1072
- * but we need to ensure we preserve the low nibble of it
1073
- */
1074
- ld r4,_TRAP(r1)
1075
- clrldi r4,r4,60
1076
- or r4,r4,r3
1077
- std r4,_TRAP(r1)
1078
-
1079
- /*
1080
- * PACA_IRQ_HARD_DIS won't always be set here, so set it now
1081
- * to reconcile the IRQ state. Tracing is already accounted for.
1082
- */
1083
- lbz r4,PACAIRQHAPPENED(r13)
1084
- ori r4,r4,PACA_IRQ_HARD_DIS
1085
- stb r4,PACAIRQHAPPENED(r13)
1086
-
1087
- /*
1088
- * Then find the right handler and call it. Interrupts are
1089
- * still soft-disabled and we keep them that way.
1090
- */
1091
- cmpwi cr0,r3,0x500
1092
- bne 1f
1093
- addi r3,r1,STACK_FRAME_OVERHEAD;
1094
- bl do_IRQ
1095
- b ret_from_except
1096
-1: cmpwi cr0,r3,0xf00
1097
- bne 1f
1098
- addi r3,r1,STACK_FRAME_OVERHEAD;
1099
- bl performance_monitor_exception
1100
- b ret_from_except
1101
-1: cmpwi cr0,r3,0xe60
1102
- bne 1f
1103
- addi r3,r1,STACK_FRAME_OVERHEAD;
1104
- bl handle_hmi_exception
1105
- b ret_from_except
1106
-1: cmpwi cr0,r3,0x900
1107
- bne 1f
1108
- addi r3,r1,STACK_FRAME_OVERHEAD;
1109
- bl timer_interrupt
1110
- b ret_from_except
1111
-#ifdef CONFIG_PPC_DOORBELL
1112
-1:
1113
-#ifdef CONFIG_PPC_BOOK3E
1114
- cmpwi cr0,r3,0x280
1115
-#else
1116
- cmpwi cr0,r3,0xa00
1117
-#endif /* CONFIG_PPC_BOOK3E */
1118
- bne 1f
1119
- addi r3,r1,STACK_FRAME_OVERHEAD;
1120
- bl doorbell_exception
1121
-#endif /* CONFIG_PPC_DOORBELL */
1122
-1: b ret_from_except /* What else to do here ? */
1123
-
1124
-.Lunrecov_restore:
1125
- addi r3,r1,STACK_FRAME_OVERHEAD
1126
- bl unrecoverable_exception
1127
- b .Lunrecov_restore
1128
-
1129
-_ASM_NOKPROBE_SYMBOL(ret_from_except);
1130
-_ASM_NOKPROBE_SYMBOL(ret_from_except_lite);
1131
-_ASM_NOKPROBE_SYMBOL(resume_kernel);
1132
-_ASM_NOKPROBE_SYMBOL(fast_exc_return_irq);
1133
-_ASM_NOKPROBE_SYMBOL(restore);
1134
-_ASM_NOKPROBE_SYMBOL(fast_exception_return);
1135
-
783
+ RFI_TO_KERNEL
784
+ b . /* prevent speculative execution */
785
+#endif /* CONFIG_PPC_BOOK3S */
1136786
1137787 #ifdef CONFIG_PPC_RTAS
1138788 /*
....@@ -1146,7 +796,7 @@
1146796 _GLOBAL(enter_rtas)
1147797 mflr r0
1148798 std r0,16(r1)
1149
- stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
799
+ stdu r1,-SWITCH_FRAME_SIZE(r1) /* Save SP and create stack space. */
1150800
1151801 /* Because RTAS is running in 32b mode, it clobbers the high order half
1152802 * of all registers that it saves. We therefore save those registers
....@@ -1154,8 +804,7 @@
1154804 */
1155805 SAVE_GPR(2, r1) /* Save the TOC */
1156806 SAVE_GPR(13, r1) /* Save paca */
1157
- SAVE_8GPRS(14, r1) /* Save the non-volatiles */
1158
- SAVE_10GPRS(22, r1) /* ditto */
807
+ SAVE_NVGPRS(r1) /* Save the non-volatiles */
1159808
1160809 mfcr r4
1161810 std r4,_CCR(r1)
....@@ -1262,8 +911,7 @@
1262911 /* relocation is on at this point */
1263912 REST_GPR(2, r1) /* Restore the TOC */
1264913 REST_GPR(13, r1) /* Restore paca */
1265
- REST_8GPRS(14, r1) /* Restore the non-volatiles */
1266
- REST_10GPRS(22, r1) /* ditto */
914
+ REST_NVGPRS(r1) /* Restore the non-volatiles */
1267915
1268916 GET_PACA(r13)
1269917
....@@ -1278,7 +926,7 @@
1278926 ld r8,_DSISR(r1)
1279927 mtdsisr r8
1280928
1281
- addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
929
+ addi r1,r1,SWITCH_FRAME_SIZE /* Unstack our frame */
1282930 ld r0,16(r1) /* get return address */
1283931
1284932 mtlr r0
....@@ -1289,7 +937,7 @@
1289937 _GLOBAL(enter_prom)
1290938 mflr r0
1291939 std r0,16(r1)
1292
- stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
940
+ stdu r1,-SWITCH_FRAME_SIZE(r1) /* Save SP and create stack space */
1293941
1294942 /* Because PROM is running in 32b mode, it clobbers the high order half
1295943 * of all registers that it saves. We therefore save those registers
....@@ -1297,8 +945,7 @@
1297945 */
1298946 SAVE_GPR(2, r1)
1299947 SAVE_GPR(13, r1)
1300
- SAVE_8GPRS(14, r1)
1301
- SAVE_10GPRS(22, r1)
948
+ SAVE_NVGPRS(r1)
1302949 mfcr r10
1303950 mfmsr r11
1304951 std r10,_CCR(r1)
....@@ -1342,12 +989,11 @@
1342989 /* Restore other registers */
1343990 REST_GPR(2, r1)
1344991 REST_GPR(13, r1)
1345
- REST_8GPRS(14, r1)
1346
- REST_10GPRS(22, r1)
992
+ REST_NVGPRS(r1)
1347993 ld r4,_CCR(r1)
1348994 mtcr r4
1349
-
1350
- addi r1,r1,PROM_FRAME_SIZE
995
+
996
+ addi r1,r1,SWITCH_FRAME_SIZE
1351997 ld r0,16(r1)
1352998 mtlr r0
1353999 blr