hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d
kernel/arch/powerpc/kernel/entry_64.S
....@@ -1,3 +1,4 @@
1
+/* SPDX-License-Identifier: GPL-2.0-or-later */
12 /*
23 * PowerPC version
34 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
....@@ -11,15 +12,11 @@
1112 *
1213 * This file contains the system call entry code, context switch
1314 * code, and exception/interrupt return code for PowerPC.
14
- *
15
- * This program is free software; you can redistribute it and/or
16
- * modify it under the terms of the GNU General Public License
17
- * as published by the Free Software Foundation; either version
18
- * 2 of the License, or (at your option) any later version.
1915 */
2016
2117 #include <linux/errno.h>
2218 #include <linux/err.h>
19
+#include <asm/cache.h>
2320 #include <asm/unistd.h>
2421 #include <asm/processor.h>
2522 #include <asm/page.h>
....@@ -46,6 +43,7 @@
4643 #include <asm/exception-64e.h>
4744 #endif
4845 #include <asm/feature-fixups.h>
46
+#include <asm/kup.h>
4947
5048 /*
5149 * System calls.
....@@ -54,6 +52,11 @@
5452 SYS_CALL_TABLE:
5553 .tc sys_call_table[TC],sys_call_table
5654
55
+#ifdef CONFIG_COMPAT
56
+COMPAT_SYS_CALL_TABLE:
57
+ .tc compat_sys_call_table[TC],compat_sys_call_table
58
+#endif
59
+
5760 /* This value is used to mark exception frames on the stack. */
5861 exception_marker:
5962 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
....@@ -61,358 +64,301 @@
6164 .section ".text"
6265 .align 7
6366
64
- .globl system_call_common
65
-system_call_common:
67
+#ifdef CONFIG_PPC_BOOK3S
68
+.macro system_call_vectored name trapnr
69
+ .globl system_call_vectored_\name
70
+system_call_vectored_\name:
71
+_ASM_NOKPROBE_SYMBOL(system_call_vectored_\name)
6672 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
6773 BEGIN_FTR_SECTION
6874 extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
6975 bne .Ltabort_syscall
7076 END_FTR_SECTION_IFSET(CPU_FTR_TM)
7177 #endif
72
- andi. r10,r12,MSR_PR
78
+ SCV_INTERRUPT_TO_KERNEL
7379 mr r10,r1
74
- addi r1,r1,-INT_FRAME_SIZE
75
- beq- 1f
7680 ld r1,PACAKSAVE(r13)
77
-1: std r10,0(r1)
81
+ std r10,0(r1)
7882 std r11,_NIP(r1)
7983 std r12,_MSR(r1)
8084 std r0,GPR0(r1)
8185 std r10,GPR1(r1)
82
- beq 2f /* if from kernel mode */
83
-#ifdef CONFIG_PPC_FSL_BOOK3E
84
-START_BTB_FLUSH_SECTION
85
- BTB_FLUSH(r10)
86
-END_BTB_FLUSH_SECTION
87
-#endif
88
- ACCOUNT_CPU_USER_ENTRY(r13, r10, r11)
89
-2: std r2,GPR2(r1)
86
+ std r2,GPR2(r1)
87
+ ld r2,PACATOC(r13)
88
+ mfcr r12
89
+ li r11,0
90
+ /* Can we avoid saving r3-r8 in common case? */
9091 std r3,GPR3(r1)
91
- mfcr r2
9292 std r4,GPR4(r1)
9393 std r5,GPR5(r1)
9494 std r6,GPR6(r1)
9595 std r7,GPR7(r1)
9696 std r8,GPR8(r1)
97
- li r11,0
97
+ /* Zero r9-r12, this should only be required when restoring all GPRs */
9898 std r11,GPR9(r1)
9999 std r11,GPR10(r1)
100100 std r11,GPR11(r1)
101101 std r11,GPR12(r1)
102
- std r11,_XER(r1)
103
- std r11,_CTR(r1)
104102 std r9,GPR13(r1)
105
- mflr r10
106
- /*
107
- * This clears CR0.SO (bit 28), which is the error indication on
108
- * return from this system call.
109
- */
110
- rldimi r2,r11,28,(63-28)
111
- li r11,0xc01
112
- std r10,_LINK(r1)
103
+ SAVE_NVGPRS(r1)
104
+ std r11,_XER(r1)
105
+ std r11,_LINK(r1)
106
+ std r11,_CTR(r1)
107
+
108
+ li r11,\trapnr
113109 std r11,_TRAP(r1)
110
+ std r12,_CCR(r1)
114111 std r3,ORIG_GPR3(r1)
115
- std r2,_CCR(r1)
116
- ld r2,PACATOC(r13)
117
- addi r9,r1,STACK_FRAME_OVERHEAD
112
+ addi r10,r1,STACK_FRAME_OVERHEAD
118113 ld r11,exception_marker@toc(r2)
119
- std r11,-16(r9) /* "regshere" marker */
120
-#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
121
-BEGIN_FW_FTR_SECTION
122
- beq 33f
123
- /* if from user, see if there are any DTL entries to process */
124
- ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
125
- ld r11,PACA_DTL_RIDX(r13) /* get log read index */
126
- addi r10,r10,LPPACA_DTLIDX
127
- LDX_BE r10,0,r10 /* get log write index */
128
- cmpd cr1,r11,r10
129
- beq+ cr1,33f
130
- bl accumulate_stolen_time
131
- REST_GPR(0,r1)
132
- REST_4GPRS(3,r1)
133
- REST_2GPRS(7,r1)
134
- addi r9,r1,STACK_FRAME_OVERHEAD
135
-33:
136
-END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
137
-#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
114
+ std r11,-16(r10) /* "regshere" marker */
115
+
116
+BEGIN_FTR_SECTION
117
+ HMT_MEDIUM
118
+END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
138119
139120 /*
140
- * A syscall should always be called with interrupts enabled
141
- * so we just unconditionally hard-enable here. When some kind
142
- * of irq tracing is used, we additionally check that condition
143
- * is correct
144
- */
145
-#if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
146
- lbz r10,PACAIRQSOFTMASK(r13)
147
-1: tdnei r10,IRQS_ENABLED
148
- EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
149
-#endif
150
-
151
-#ifdef CONFIG_PPC_BOOK3E
152
- wrteei 1
153
-#else
154
- li r11,MSR_RI
155
- ori r11,r11,MSR_EE
156
- mtmsrd r11,1
157
-#endif /* CONFIG_PPC_BOOK3E */
158
-
159
-system_call: /* label this so stack traces look sane */
160
- /* We do need to set SOFTE in the stack frame or the return
161
- * from interrupt will be painful
162
- */
163
- li r10,IRQS_ENABLED
164
- std r10,SOFTE(r1)
165
-
166
- CURRENT_THREAD_INFO(r11, r1)
167
- ld r10,TI_FLAGS(r11)
168
- andi. r11,r10,_TIF_SYSCALL_DOTRACE
169
- bne .Lsyscall_dotrace /* does not return */
170
- cmpldi 0,r0,NR_syscalls
171
- bge- .Lsyscall_enosys
172
-
173
-.Lsyscall:
174
-/*
175
- * Need to vector to 32 Bit or default sys_call_table here,
176
- * based on caller's run-mode / personality.
177
- */
178
- ld r11,SYS_CALL_TABLE@toc(2)
179
- andi. r10,r10,_TIF_32BIT
180
- beq 15f
181
- addi r11,r11,8 /* use 32-bit syscall entries */
182
- clrldi r3,r3,32
183
- clrldi r4,r4,32
184
- clrldi r5,r5,32
185
- clrldi r6,r6,32
186
- clrldi r7,r7,32
187
- clrldi r8,r8,32
188
-15:
189
- slwi r0,r0,4
190
-
191
- barrier_nospec_asm
192
- /*
193
- * Prevent the load of the handler below (based on the user-passed
194
- * system call number) being speculatively executed until the test
195
- * against NR_syscalls and branch to .Lsyscall_enosys above has
196
- * committed.
121
+ * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
122
+ * would clobber syscall parameters. Also we always enter with IRQs
123
+ * enabled and nothing pending. system_call_exception() will call
124
+ * trace_hardirqs_off().
125
+ *
126
+ * scv enters with MSR[EE]=1, so don't set PACA_IRQ_HARD_DIS. The
127
+ * entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED.
197128 */
198129
199
- ldx r12,r11,r0 /* Fetch system call handler [ptr] */
200
- mtctr r12
201
- bctrl /* Call handler */
130
+ /* Calling convention has r9 = orig r0, r10 = regs */
131
+ mr r9,r0
132
+ bl system_call_exception
202133
203
-.Lsyscall_exit:
204
- std r3,RESULT(r1)
134
+.Lsyscall_vectored_\name\()_exit:
135
+ addi r4,r1,STACK_FRAME_OVERHEAD
136
+ li r5,1 /* scv */
137
+ bl syscall_exit_prepare
205138
206
-#ifdef CONFIG_DEBUG_RSEQ
207
- /* Check whether the syscall is issued inside a restartable sequence */
208
- addi r3,r1,STACK_FRAME_OVERHEAD
209
- bl rseq_syscall
210
- ld r3,RESULT(r1)
211
-#endif
139
+ ld r2,_CCR(r1)
140
+ ld r4,_NIP(r1)
141
+ ld r5,_MSR(r1)
212142
213
- CURRENT_THREAD_INFO(r12, r1)
214
-
215
- ld r8,_MSR(r1)
216
-#ifdef CONFIG_PPC_BOOK3S
217
- /* No MSR:RI on BookE */
218
- andi. r10,r8,MSR_RI
219
- beq- .Lunrecov_restore
220
-#endif
221
-
222
-/*
223
- * This is a few instructions into the actual syscall exit path (which actually
224
- * starts at .Lsyscall_exit) to cater to kprobe blacklisting and to reduce the
225
- * number of visible symbols for profiling purposes.
226
- *
227
- * We can probe from system_call until this point as MSR_RI is set. But once it
228
- * is cleared below, we won't be able to take a trap.
229
- *
230
- * This is blacklisted from kprobes further below with _ASM_NOKPROBE_SYMBOL().
231
- */
232
-system_call_exit:
233
- /*
234
- * Disable interrupts so current_thread_info()->flags can't change,
235
- * and so that we don't get interrupted after loading SRR0/1.
236
- */
237
-#ifdef CONFIG_PPC_BOOK3E
238
- wrteei 0
239
-#else
240
- /*
241
- * For performance reasons we clear RI the same time that we
242
- * clear EE. We only need to clear RI just before we restore r13
243
- * below, but batching it with EE saves us one expensive mtmsrd call.
244
- * We have to be careful to restore RI if we branch anywhere from
245
- * here (eg syscall_exit_work).
246
- */
247
- li r11,0
248
- mtmsrd r11,1
249
-#endif /* CONFIG_PPC_BOOK3E */
250
-
251
- ld r9,TI_FLAGS(r12)
252
- li r11,-MAX_ERRNO
253
- andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
254
- bne- .Lsyscall_exit_work
255
-
256
- andi. r0,r8,MSR_FP
257
- beq 2f
258
-#ifdef CONFIG_ALTIVEC
259
- andis. r0,r8,MSR_VEC@h
260
- bne 3f
261
-#endif
262
-2: addi r3,r1,STACK_FRAME_OVERHEAD
263
-#ifdef CONFIG_PPC_BOOK3S
264
- li r10,MSR_RI
265
- mtmsrd r10,1 /* Restore RI */
266
-#endif
267
- bl restore_math
268
-#ifdef CONFIG_PPC_BOOK3S
269
- li r11,0
270
- mtmsrd r11,1
271
-#endif
272
- ld r8,_MSR(r1)
273
- ld r3,RESULT(r1)
274
- li r11,-MAX_ERRNO
275
-
276
-3: cmpld r3,r11
277
- ld r5,_CCR(r1)
278
- bge- .Lsyscall_error
279
-.Lsyscall_error_cont:
280
- ld r7,_NIP(r1)
281143 BEGIN_FTR_SECTION
282144 stdcx. r0,0,r1 /* to clear the reservation */
283145 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
284
- andi. r6,r8,MSR_PR
285
- ld r4,_LINK(r1)
286
-
287
- beq- 1f
288
- ACCOUNT_CPU_USER_EXIT(r13, r11, r12)
289146
290147 BEGIN_FTR_SECTION
291148 HMT_MEDIUM_LOW
292149 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
293150
294
- ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
295
- ld r2,GPR2(r1)
296
- ld r1,GPR1(r1)
151
+ cmpdi r3,0
152
+ bne .Lsyscall_vectored_\name\()_restore_regs
153
+
154
+ /* rfscv returns with LR->NIA and CTR->MSR */
297155 mtlr r4
298
- mtcr r5
299
- mtspr SPRN_SRR0,r7
300
- mtspr SPRN_SRR1,r8
301
- RFI_TO_USER
302
- b . /* prevent speculative execution */
156
+ mtctr r5
303157
304
- /* exit to kernel */
305
-1: ld r2,GPR2(r1)
306
- ld r1,GPR1(r1)
307
- mtlr r4
308
- mtcr r5
309
- mtspr SPRN_SRR0,r7
310
- mtspr SPRN_SRR1,r8
311
- RFI_TO_KERNEL
312
- b . /* prevent speculative execution */
313
-
314
-.Lsyscall_error:
315
- oris r5,r5,0x1000 /* Set SO bit in CR */
316
- neg r3,r3
317
- std r5,_CCR(r1)
318
- b .Lsyscall_error_cont
319
-
320
-/* Traced system call support */
321
-.Lsyscall_dotrace:
322
- bl save_nvgprs
323
- addi r3,r1,STACK_FRAME_OVERHEAD
324
- bl do_syscall_trace_enter
325
-
326
- /*
327
- * We use the return value of do_syscall_trace_enter() as the syscall
328
- * number. If the syscall was rejected for any reason do_syscall_trace_enter()
329
- * returns an invalid syscall number and the test below against
330
- * NR_syscalls will fail.
331
- */
332
- mr r0,r3
333
-
334
- /* Restore argument registers just clobbered and/or possibly changed. */
335
- ld r3,GPR3(r1)
158
+ /* Could zero these as per ABI, but we may consider a stricter ABI
159
+ * which preserves these if libc implementations can benefit, so
160
+ * restore them for now until further measurement is done. */
161
+ ld r0,GPR0(r1)
336162 ld r4,GPR4(r1)
337163 ld r5,GPR5(r1)
338164 ld r6,GPR6(r1)
339165 ld r7,GPR7(r1)
340166 ld r8,GPR8(r1)
167
+ /* Zero volatile regs that may contain sensitive kernel data */
168
+ li r9,0
169
+ li r10,0
170
+ li r11,0
171
+ li r12,0
172
+ mtspr SPRN_XER,r0
341173
342
- /* Repopulate r9 and r10 for the syscall path */
343
- addi r9,r1,STACK_FRAME_OVERHEAD
344
- CURRENT_THREAD_INFO(r10, r1)
345
- ld r10,TI_FLAGS(r10)
174
+ /*
175
+ * We don't need to restore AMR on the way back to userspace for KUAP.
176
+ * The value of AMR only matters while we're in the kernel.
177
+ */
178
+ mtcr r2
179
+ ld r2,GPR2(r1)
180
+ ld r3,GPR3(r1)
181
+ ld r13,GPR13(r1)
182
+ ld r1,GPR1(r1)
183
+ RFSCV_TO_USER
184
+ b . /* prevent speculative execution */
346185
347
- cmpldi r0,NR_syscalls
348
- blt+ .Lsyscall
186
+.Lsyscall_vectored_\name\()_restore_regs:
187
+ li r3,0
188
+ mtmsrd r3,1
189
+ mtspr SPRN_SRR0,r4
190
+ mtspr SPRN_SRR1,r5
349191
350
- /* Return code is already in r3 thanks to do_syscall_trace_enter() */
351
- b .Lsyscall_exit
192
+ ld r3,_CTR(r1)
193
+ ld r4,_LINK(r1)
194
+ ld r5,_XER(r1)
352195
353
-
354
-.Lsyscall_enosys:
355
- li r3,-ENOSYS
356
- b .Lsyscall_exit
357
-
358
-.Lsyscall_exit_work:
359
-#ifdef CONFIG_PPC_BOOK3S
360
- li r10,MSR_RI
361
- mtmsrd r10,1 /* Restore RI */
362
-#endif
363
- /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
364
- If TIF_NOERROR is set, just save r3 as it is. */
365
-
366
- andi. r0,r9,_TIF_RESTOREALL
367
- beq+ 0f
368196 REST_NVGPRS(r1)
369
- b 2f
370
-0: cmpld r3,r11 /* r11 is -MAX_ERRNO */
371
- blt+ 1f
372
- andi. r0,r9,_TIF_NOERROR
373
- bne- 1f
374
- ld r5,_CCR(r1)
375
- neg r3,r3
376
- oris r5,r5,0x1000 /* Set SO bit in CR */
377
- std r5,_CCR(r1)
378
-1: std r3,GPR3(r1)
379
-2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
380
- beq 4f
197
+ ld r0,GPR0(r1)
198
+ mtcr r2
199
+ mtctr r3
200
+ mtlr r4
201
+ mtspr SPRN_XER,r5
202
+ REST_10GPRS(2, r1)
203
+ REST_2GPRS(12, r1)
204
+ ld r1,GPR1(r1)
205
+ RFI_TO_USER
206
+.endm
381207
382
- /* Clear per-syscall TIF flags if any are set. */
208
+system_call_vectored common 0x3000
209
+/*
210
+ * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0
211
+ * which is tested by system_call_exception when r0 is -1 (as set by vector
212
+ * entry code).
213
+ */
214
+system_call_vectored sigill 0x7ff0
383215
384
- li r11,_TIF_PERSYSCALL_MASK
385
- addi r12,r12,TI_FLAGS
386
-3: ldarx r10,0,r12
387
- andc r10,r10,r11
388
- stdcx. r10,0,r12
389
- bne- 3b
390
- subi r12,r12,TI_FLAGS
391216
392
-4: /* Anything else left to do? */
217
+/*
218
+ * Entered via kernel return set up by kernel/sstep.c, must match entry regs
219
+ */
220
+ .globl system_call_vectored_emulate
221
+system_call_vectored_emulate:
222
+_ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate)
223
+ li r10,IRQS_ALL_DISABLED
224
+ stb r10,PACAIRQSOFTMASK(r13)
225
+ b system_call_vectored_common
226
+#endif
227
+
228
+ .balign IFETCH_ALIGN_BYTES
229
+ .globl system_call_common
230
+system_call_common:
231
+_ASM_NOKPROBE_SYMBOL(system_call_common)
232
+#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
393233 BEGIN_FTR_SECTION
394
- lis r3,INIT_PPR@highest /* Set thread.ppr = 3 */
395
- ld r10,PACACURRENT(r13)
396
- sldi r3,r3,32 /* bits 11-13 are used for ppr */
397
- std r3,TASKTHREADPPR(r10)
234
+ extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
235
+ bne .Ltabort_syscall
236
+END_FTR_SECTION_IFSET(CPU_FTR_TM)
237
+#endif
238
+ mr r10,r1
239
+ ld r1,PACAKSAVE(r13)
240
+ std r10,0(r1)
241
+ std r11,_NIP(r1)
242
+ std r12,_MSR(r1)
243
+ std r0,GPR0(r1)
244
+ std r10,GPR1(r1)
245
+ std r2,GPR2(r1)
246
+#ifdef CONFIG_PPC_FSL_BOOK3E
247
+START_BTB_FLUSH_SECTION
248
+ BTB_FLUSH(r10)
249
+END_BTB_FLUSH_SECTION
250
+#endif
251
+ ld r2,PACATOC(r13)
252
+ mfcr r12
253
+ li r11,0
254
+ /* Can we avoid saving r3-r8 in common case? */
255
+ std r3,GPR3(r1)
256
+ std r4,GPR4(r1)
257
+ std r5,GPR5(r1)
258
+ std r6,GPR6(r1)
259
+ std r7,GPR7(r1)
260
+ std r8,GPR8(r1)
261
+ /* Zero r9-r12, this should only be required when restoring all GPRs */
262
+ std r11,GPR9(r1)
263
+ std r11,GPR10(r1)
264
+ std r11,GPR11(r1)
265
+ std r11,GPR12(r1)
266
+ std r9,GPR13(r1)
267
+ SAVE_NVGPRS(r1)
268
+ std r11,_XER(r1)
269
+ std r11,_CTR(r1)
270
+ mflr r10
271
+
272
+ /*
273
+ * This clears CR0.SO (bit 28), which is the error indication on
274
+ * return from this system call.
275
+ */
276
+ rldimi r12,r11,28,(63-28)
277
+ li r11,0xc00
278
+ std r10,_LINK(r1)
279
+ std r11,_TRAP(r1)
280
+ std r12,_CCR(r1)
281
+ std r3,ORIG_GPR3(r1)
282
+ addi r10,r1,STACK_FRAME_OVERHEAD
283
+ ld r11,exception_marker@toc(r2)
284
+ std r11,-16(r10) /* "regshere" marker */
285
+
286
+ /*
287
+ * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
288
+ * would clobber syscall parameters. Also we always enter with IRQs
289
+ * enabled and nothing pending. system_call_exception() will call
290
+ * trace_hardirqs_off().
291
+ */
292
+ li r11,IRQS_ALL_DISABLED
293
+ li r12,PACA_IRQ_HARD_DIS
294
+ stb r11,PACAIRQSOFTMASK(r13)
295
+ stb r12,PACAIRQHAPPENED(r13)
296
+
297
+ /* Calling convention has r9 = orig r0, r10 = regs */
298
+ mr r9,r0
299
+ bl system_call_exception
300
+
301
+.Lsyscall_exit:
302
+ addi r4,r1,STACK_FRAME_OVERHEAD
303
+ li r5,0 /* !scv */
304
+ bl syscall_exit_prepare
305
+
306
+ ld r2,_CCR(r1)
307
+ ld r4,_NIP(r1)
308
+ ld r5,_MSR(r1)
309
+ ld r6,_LINK(r1)
310
+
311
+BEGIN_FTR_SECTION
312
+ stdcx. r0,0,r1 /* to clear the reservation */
313
+END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
314
+
315
+ mtspr SPRN_SRR0,r4
316
+ mtspr SPRN_SRR1,r5
317
+ mtlr r6
318
+
319
+ cmpdi r3,0
320
+ bne .Lsyscall_restore_regs
321
+ /* Zero volatile regs that may contain sensitive kernel data */
322
+ li r0,0
323
+ li r4,0
324
+ li r5,0
325
+ li r6,0
326
+ li r7,0
327
+ li r8,0
328
+ li r9,0
329
+ li r10,0
330
+ li r11,0
331
+ li r12,0
332
+ mtctr r0
333
+ mtspr SPRN_XER,r0
334
+.Lsyscall_restore_regs_cont:
335
+
336
+BEGIN_FTR_SECTION
337
+ HMT_MEDIUM_LOW
398338 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
399339
400
- andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
401
- beq ret_from_except_lite
340
+ /*
341
+ * We don't need to restore AMR on the way back to userspace for KUAP.
342
+ * The value of AMR only matters while we're in the kernel.
343
+ */
344
+ mtcr r2
345
+ ld r2,GPR2(r1)
346
+ ld r3,GPR3(r1)
347
+ ld r13,GPR13(r1)
348
+ ld r1,GPR1(r1)
349
+ RFI_TO_USER
350
+ b . /* prevent speculative execution */
402351
403
- /* Re-enable interrupts */
404
-#ifdef CONFIG_PPC_BOOK3E
405
- wrteei 1
406
-#else
407
- li r10,MSR_RI
408
- ori r10,r10,MSR_EE
409
- mtmsrd r10,1
410
-#endif /* CONFIG_PPC_BOOK3E */
411
-
412
- bl save_nvgprs
413
- addi r3,r1,STACK_FRAME_OVERHEAD
414
- bl do_syscall_trace_leave
415
- b ret_from_except
352
+.Lsyscall_restore_regs:
353
+ ld r3,_CTR(r1)
354
+ ld r4,_XER(r1)
355
+ REST_NVGPRS(r1)
356
+ mtctr r3
357
+ mtspr SPRN_XER,r4
358
+ ld r0,GPR0(r1)
359
+ REST_8GPRS(4, r1)
360
+ ld r12,GPR12(r1)
361
+ b .Lsyscall_restore_regs_cont
416362
417363 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
418364 .Ltabort_syscall:
....@@ -440,9 +386,34 @@
440386 RFI_TO_USER
441387 b . /* prevent speculative execution */
442388 #endif
443
-_ASM_NOKPROBE_SYMBOL(system_call_common);
444
-_ASM_NOKPROBE_SYMBOL(system_call_exit);
445389
390
+#ifdef CONFIG_PPC_BOOK3S
391
+_GLOBAL(ret_from_fork_scv)
392
+ bl schedule_tail
393
+ REST_NVGPRS(r1)
394
+ li r3,0 /* fork() return value */
395
+ b .Lsyscall_vectored_common_exit
396
+#endif
397
+
398
+_GLOBAL(ret_from_fork)
399
+ bl schedule_tail
400
+ REST_NVGPRS(r1)
401
+ li r3,0 /* fork() return value */
402
+ b .Lsyscall_exit
403
+
404
+_GLOBAL(ret_from_kernel_thread)
405
+ bl schedule_tail
406
+ REST_NVGPRS(r1)
407
+ mtctr r14
408
+ mr r3,r15
409
+#ifdef PPC64_ELF_ABI_v2
410
+ mr r12,r14
411
+#endif
412
+ bctrl
413
+ li r3,0
414
+ b .Lsyscall_exit
415
+
416
+#ifdef CONFIG_PPC_BOOK3E
446417 /* Save non-volatile GPRs, if not already saved. */
447418 _GLOBAL(save_nvgprs)
448419 ld r11,_TRAP(r1)
....@@ -453,73 +424,17 @@
453424 std r0,_TRAP(r1)
454425 blr
455426 _ASM_NOKPROBE_SYMBOL(save_nvgprs);
456
-
457
-
458
-/*
459
- * The sigsuspend and rt_sigsuspend system calls can call do_signal
460
- * and thus put the process into the stopped state where we might
461
- * want to examine its user state with ptrace. Therefore we need
462
- * to save all the nonvolatile registers (r14 - r31) before calling
463
- * the C code. Similarly, fork, vfork and clone need the full
464
- * register state on the stack so that it can be copied to the child.
465
- */
466
-
467
-_GLOBAL(ppc_fork)
468
- bl save_nvgprs
469
- bl sys_fork
470
- b .Lsyscall_exit
471
-
472
-_GLOBAL(ppc_vfork)
473
- bl save_nvgprs
474
- bl sys_vfork
475
- b .Lsyscall_exit
476
-
477
-_GLOBAL(ppc_clone)
478
- bl save_nvgprs
479
- bl sys_clone
480
- b .Lsyscall_exit
481
-
482
-_GLOBAL(ppc32_swapcontext)
483
- bl save_nvgprs
484
- bl compat_sys_swapcontext
485
- b .Lsyscall_exit
486
-
487
-_GLOBAL(ppc64_swapcontext)
488
- bl save_nvgprs
489
- bl sys_swapcontext
490
- b .Lsyscall_exit
491
-
492
-_GLOBAL(ppc_switch_endian)
493
- bl save_nvgprs
494
- bl sys_switch_endian
495
- b .Lsyscall_exit
496
-
497
-_GLOBAL(ret_from_fork)
498
- bl schedule_tail
499
- REST_NVGPRS(r1)
500
- li r3,0
501
- b .Lsyscall_exit
502
-
503
-_GLOBAL(ret_from_kernel_thread)
504
- bl schedule_tail
505
- REST_NVGPRS(r1)
506
- mtlr r14
507
- mr r3,r15
508
-#ifdef PPC64_ELF_ABI_v2
509
- mr r12,r14
510427 #endif
511
- blrl
512
- li r3,0
513
- b .Lsyscall_exit
514428
515429 #ifdef CONFIG_PPC_BOOK3S_64
516430
517431 #define FLUSH_COUNT_CACHE \
518432 1: nop; \
519
- patch_site 1b, patch__call_flush_count_cache
520
-
521
-
522
-#define BCCTR_FLUSH .long 0x4c400420
433
+ patch_site 1b, patch__call_flush_branch_caches1; \
434
+1: nop; \
435
+ patch_site 1b, patch__call_flush_branch_caches2; \
436
+1: nop; \
437
+ patch_site 1b, patch__call_flush_branch_caches3
523438
524439 .macro nops number
525440 .rept \number
....@@ -528,8 +443,8 @@
528443 .endm
529444
530445 .balign 32
531
-.global flush_count_cache
532
-flush_count_cache:
446
+.global flush_branch_caches
447
+flush_branch_caches:
533448 /* Save LR into r9 */
534449 mflr r9
535450
....@@ -551,7 +466,7 @@
551466 li r9,0x7fff
552467 mtctr r9
553468
554
- BCCTR_FLUSH
469
+ PPC_BCCTR_FLUSH
555470
556471 2: nop
557472 patch_site 2b patch__flush_count_cache_return
....@@ -560,7 +475,7 @@
560475
561476 .rept 278
562477 .balign 32
563
- BCCTR_FLUSH
478
+ PPC_BCCTR_FLUSH
564479 nops 7
565480 .endr
566481
....@@ -574,7 +489,7 @@
574489 * state of one is saved on its kernel stack. Then the state
575490 * of the other is restored from its kernel stack. The memory
576491 * management hardware is updated to the second process's state.
577
- * Finally, we can return to the second process, via ret_from_except.
492
+ * Finally, we can return to the second process, via interrupt_return.
578493 * On entry, r3 points to the THREAD for the current task, r4
579494 * points to the THREAD for the new task.
580495 *
....@@ -593,14 +508,15 @@
593508 std r0,16(r1)
594509 stdu r1,-SWITCH_FRAME_SIZE(r1)
595510 /* r3-r13 are caller saved -- Cort */
596
- SAVE_8GPRS(14, r1)
597
- SAVE_10GPRS(22, r1)
511
+ SAVE_NVGPRS(r1)
598512 std r0,_NIP(r1) /* Return to switch caller */
599513 mfcr r23
600514 std r23,_CCR(r1)
601515 std r1,KSP(r3) /* Set old stack pointer */
602516
603
- FLUSH_COUNT_CACHE
517
+ kuap_check_amr r9, r10
518
+
519
+ FLUSH_COUNT_CACHE /* Clobbers r9, ctr */
604520
605521 /*
606522 * On SMP kernels, care must be taken because a task may be
....@@ -613,7 +529,7 @@
613529 * kernel/sched/core.c).
614530 *
615531 * Uncacheable stores in the case of involuntary preemption must
616
- * be taken care of. The smp_mb__before_spin_lock() in __schedule()
532
+ * be taken care of. The smp_mb__after_spinlock() in __schedule()
617533 * is implemented as hwsync on powerpc, which orders MMIO too. So
618534 * long as there is an hwsync in the context switch path, it will
619535 * be executed on the source CPU after the task has performed
....@@ -635,6 +551,10 @@
635551
636552 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
637553 std r6,PACACURRENT(r13) /* Set new 'current' */
554
+#if defined(CONFIG_STACKPROTECTOR)
555
+ ld r6, TASK_CANARY(r6)
556
+ std r6, PACA_CANARY(r13)
557
+#endif
638558
639559 ld r8,KSP(r4) /* new stack pointer */
640560 #ifdef CONFIG_PPC_BOOK3S_64
....@@ -683,13 +603,15 @@
683603
684604 isync
685605 slbie r6
606
+BEGIN_FTR_SECTION
686607 slbie r6 /* Workaround POWER5 < DD2.1 issue */
608
+END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
687609 slbmte r7,r0
688610 isync
689611 2:
690612 #endif /* CONFIG_PPC_BOOK3S_64 */
691613
692
- CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
614
+ clrrdi r7, r8, THREAD_SHIFT /* base of new stack */
693615 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
694616 because we don't need to leave the 288-byte ABI gap at the
695617 top of the kernel stack. */
....@@ -710,8 +632,7 @@
710632 mtcrf 0xFF,r6
711633
712634 /* r3-r13 are destroyed -- Cort */
713
- REST_8GPRS(14, r1)
714
- REST_10GPRS(22, r1)
635
+ REST_NVGPRS(r1)
715636
716637 /* convert old thread to its task_struct for return value */
717638 addi r3,r3,-THREAD
....@@ -720,409 +641,148 @@
720641 addi r1,r1,SWITCH_FRAME_SIZE
721642 blr
722643
723
- .align 7
724
-_GLOBAL(ret_from_except)
725
- ld r11,_TRAP(r1)
726
- andi. r0,r11,1
727
- bne ret_from_except_lite
728
- REST_NVGPRS(r1)
729
-
730
-_GLOBAL(ret_from_except_lite)
644
+#ifdef CONFIG_PPC_BOOK3S
731645 /*
732
- * Disable interrupts so that current_thread_info()->flags
733
- * can't change between when we test it and when we return
734
- * from the interrupt.
646
+ * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
647
+ * touched, no exit work created, then this can be used.
735648 */
736
-#ifdef CONFIG_PPC_BOOK3E
737
- wrteei 0
738
-#else
739
- li r10,MSR_RI
740
- mtmsrd r10,1 /* Update machine state */
741
-#endif /* CONFIG_PPC_BOOK3E */
742
-
743
- CURRENT_THREAD_INFO(r9, r1)
744
- ld r3,_MSR(r1)
745
-#ifdef CONFIG_PPC_BOOK3E
746
- ld r10,PACACURRENT(r13)
747
-#endif /* CONFIG_PPC_BOOK3E */
748
- ld r4,TI_FLAGS(r9)
749
- andi. r3,r3,MSR_PR
750
- beq resume_kernel
751
-#ifdef CONFIG_PPC_BOOK3E
752
- lwz r3,(THREAD+THREAD_DBCR0)(r10)
753
-#endif /* CONFIG_PPC_BOOK3E */
754
-
755
- /* Check current_thread_info()->flags */
756
- andi. r0,r4,_TIF_USER_WORK_MASK
757
- bne 1f
758
-#ifdef CONFIG_PPC_BOOK3E
759
- /*
760
- * Check to see if the dbcr0 register is set up to debug.
761
- * Use the internal debug mode bit to do this.
762
- */
763
- andis. r0,r3,DBCR0_IDM@h
764
- beq restore
765
- mfmsr r0
766
- rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
767
- mtmsr r0
768
- mtspr SPRN_DBCR0,r3
769
- li r10, -1
770
- mtspr SPRN_DBSR,r10
771
- b restore
772
-#else
649
+ .balign IFETCH_ALIGN_BYTES
650
+ .globl fast_interrupt_return
651
+fast_interrupt_return:
652
+_ASM_NOKPROBE_SYMBOL(fast_interrupt_return)
653
+ kuap_check_amr r3, r4
654
+ ld r5,_MSR(r1)
655
+ andi. r0,r5,MSR_PR
656
+ bne .Lfast_user_interrupt_return
657
+ kuap_restore_amr r3, r4
658
+ andi. r0,r5,MSR_RI
659
+ li r3,0 /* 0 return value, no EMULATE_STACK_STORE */
660
+ bne+ .Lfast_kernel_interrupt_return
773661 addi r3,r1,STACK_FRAME_OVERHEAD
774
- bl restore_math
775
- b restore
776
-#endif
777
-1: andi. r0,r4,_TIF_NEED_RESCHED
778
- beq 2f
779
- bl restore_interrupts
780
- SCHEDULE_USER
781
- b ret_from_except_lite
782
-2:
783
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
784
- andi. r0,r4,_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM
785
- bne 3f /* only restore TM if nothing else to do */
662
+ bl unrecoverable_exception
663
+ b . /* should not get here */
664
+
665
+ .balign IFETCH_ALIGN_BYTES
666
+ .globl interrupt_return
667
+interrupt_return:
668
+_ASM_NOKPROBE_SYMBOL(interrupt_return)
669
+ ld r4,_MSR(r1)
670
+ andi. r0,r4,MSR_PR
671
+ beq .Lkernel_interrupt_return
786672 addi r3,r1,STACK_FRAME_OVERHEAD
787
- bl restore_tm_state
788
- b restore
789
-3:
790
-#endif
791
- bl save_nvgprs
792
- /*
793
- * Use a non volatile GPR to save and restore our thread_info flags
794
- * across the call to restore_interrupts.
795
- */
796
- mr r30,r4
797
- bl restore_interrupts
798
- mr r4,r30
799
- addi r3,r1,STACK_FRAME_OVERHEAD
800
- bl do_notify_resume
801
- b ret_from_except
673
+ bl interrupt_exit_user_prepare
674
+ cmpdi r3,0
675
+ bne- .Lrestore_nvgprs
802676
803
-resume_kernel:
804
- /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
805
- andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
806
- beq+ 1f
677
+.Lfast_user_interrupt_return:
678
+ ld r11,_NIP(r1)
679
+ ld r12,_MSR(r1)
680
+BEGIN_FTR_SECTION
681
+ ld r10,_PPR(r1)
682
+ mtspr SPRN_PPR,r10
683
+END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
684
+ mtspr SPRN_SRR0,r11
685
+ mtspr SPRN_SRR1,r12
807686
808
- addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
809
-
810
- ld r3,GPR1(r1)
811
- subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
812
- mr r4,r1 /* src: current exception frame */
813
- mr r1,r3 /* Reroute the trampoline frame to r1 */
814
-
815
- /* Copy from the original to the trampoline. */
816
- li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
817
- li r6,0 /* start offset: 0 */
818
- mtctr r5
819
-2: ldx r0,r6,r4
820
- stdx r0,r6,r3
821
- addi r6,r6,8
822
- bdnz 2b
823
-
824
- /* Do real store operation to complete stdu */
825
- ld r5,GPR1(r1)
826
- std r8,0(r5)
827
-
828
- /* Clear _TIF_EMULATE_STACK_STORE flag */
829
- lis r11,_TIF_EMULATE_STACK_STORE@h
830
- addi r5,r9,TI_FLAGS
831
-0: ldarx r4,0,r5
832
- andc r4,r4,r11
833
- stdcx. r4,0,r5
834
- bne- 0b
835
-1:
836
-
837
-#ifdef CONFIG_PREEMPT
838
- /* Check if we need to preempt */
839
- andi. r0,r4,_TIF_NEED_RESCHED
840
- beq+ restore
841
- /* Check that preempt_count() == 0 and interrupts are enabled */
842
- lwz r8,TI_PREEMPT(r9)
843
- cmpwi cr0,r8,0
844
- bne restore
845
- ld r0,SOFTE(r1)
846
- andi. r0,r0,IRQS_DISABLED
847
- bne restore
848
-
849
- /*
850
- * Here we are preempting the current task. We want to make
851
- * sure we are soft-disabled first and reconcile irq state.
852
- */
853
- RECONCILE_IRQ_STATE(r3,r4)
854
-1: bl preempt_schedule_irq
855
-
856
- /* Re-test flags and eventually loop */
857
- CURRENT_THREAD_INFO(r9, r1)
858
- ld r4,TI_FLAGS(r9)
859
- andi. r0,r4,_TIF_NEED_RESCHED
860
- bne 1b
861
-
862
- /*
863
- * arch_local_irq_restore() from preempt_schedule_irq above may
864
- * enable hard interrupt but we really should disable interrupts
865
- * when we return from the interrupt, and so that we don't get
866
- * interrupted after loading SRR0/1.
867
- */
868
-#ifdef CONFIG_PPC_BOOK3E
869
- wrteei 0
870
-#else
871
- li r10,MSR_RI
872
- mtmsrd r10,1 /* Update machine state */
873
-#endif /* CONFIG_PPC_BOOK3E */
874
-#endif /* CONFIG_PREEMPT */
875
-
876
- .globl fast_exc_return_irq
877
-fast_exc_return_irq:
878
-restore:
879
- /*
880
- * This is the main kernel exit path. First we check if we
881
- * are about to re-enable interrupts
882
- */
883
- ld r5,SOFTE(r1)
884
- lbz r6,PACAIRQSOFTMASK(r13)
885
- andi. r5,r5,IRQS_DISABLED
886
- bne .Lrestore_irq_off
887
-
888
- /* We are enabling, were we already enabled ? Yes, just return */
889
- andi. r6,r6,IRQS_DISABLED
890
- beq cr0,.Ldo_restore
891
-
892
- /*
893
- * We are about to soft-enable interrupts (we are hard disabled
894
- * at this point). We check if there's anything that needs to
895
- * be replayed first.
896
- */
897
- lbz r0,PACAIRQHAPPENED(r13)
898
- cmpwi cr0,r0,0
899
- bne- .Lrestore_check_irq_replay
900
-
901
- /*
902
- * Get here when nothing happened while soft-disabled, just
903
- * soft-enable and move-on. We will hard-enable as a side
904
- * effect of rfi
905
- */
906
-.Lrestore_no_replay:
907
- TRACE_ENABLE_INTS
908
- li r0,IRQS_ENABLED
909
- stb r0,PACAIRQSOFTMASK(r13);
910
-
911
- /*
912
- * Final return path. BookE is handled in a different file
913
- */
914
-.Ldo_restore:
915
-#ifdef CONFIG_PPC_BOOK3E
916
- b exception_return_book3e
917
-#else
918
- /*
919
- * Clear the reservation. If we know the CPU tracks the address of
920
- * the reservation then we can potentially save some cycles and use
921
- * a larx. On POWER6 and POWER7 this is significantly faster.
922
- */
923687 BEGIN_FTR_SECTION
924688 stdcx. r0,0,r1 /* to clear the reservation */
925689 FTR_SECTION_ELSE
926
- ldarx r4,0,r1
690
+ ldarx r0,0,r1
927691 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
928692
929
- /*
930
- * Some code path such as load_up_fpu or altivec return directly
931
- * here. They run entirely hard disabled and do not alter the
932
- * interrupt state. They also don't use lwarx/stwcx. and thus
933
- * are known not to leave dangling reservations.
934
- */
935
- .globl fast_exception_return
936
-fast_exception_return:
937
- ld r3,_MSR(r1)
938
- ld r4,_CTR(r1)
939
- ld r0,_LINK(r1)
940
- mtctr r4
941
- mtlr r0
942
- ld r4,_XER(r1)
943
- mtspr SPRN_XER,r4
693
+ ld r3,_CCR(r1)
694
+ ld r4,_LINK(r1)
695
+ ld r5,_CTR(r1)
696
+ ld r6,_XER(r1)
697
+ li r0,0
944698
945
- REST_8GPRS(5, r1)
946
-
947
- andi. r0,r3,MSR_RI
948
- beq- .Lunrecov_restore
949
-
950
- /* Load PPR from thread struct before we clear MSR:RI */
951
-BEGIN_FTR_SECTION
952
- ld r2,PACACURRENT(r13)
953
- ld r2,TASKTHREADPPR(r2)
954
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
955
-
956
- /*
957
- * Clear RI before restoring r13. If we are returning to
958
- * userspace and we take an exception after restoring r13,
959
- * we end up corrupting the userspace r13 value.
960
- */
961
- li r4,0
962
- mtmsrd r4,1
963
-
964
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
965
- /* TM debug */
966
- std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
967
-#endif
968
- /*
969
- * r13 is our per cpu area, only restore it if we are returning to
970
- * userspace the value stored in the stack frame may belong to
971
- * another CPU.
972
- */
973
- andi. r0,r3,MSR_PR
974
- beq 1f
975
-BEGIN_FTR_SECTION
976
- mtspr SPRN_PPR,r2 /* Restore PPR */
977
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
978
- ACCOUNT_CPU_USER_EXIT(r13, r2, r4)
699
+ REST_4GPRS(7, r1)
700
+ REST_2GPRS(11, r1)
979701 REST_GPR(13, r1)
980702
981
- mtspr SPRN_SRR1,r3
703
+ mtcr r3
704
+ mtlr r4
705
+ mtctr r5
706
+ mtspr SPRN_XER,r6
982707
983
- ld r2,_CCR(r1)
984
- mtcrf 0xFF,r2
985
- ld r2,_NIP(r1)
986
- mtspr SPRN_SRR0,r2
987
-
988
- ld r0,GPR0(r1)
989
- ld r2,GPR2(r1)
990
- ld r3,GPR3(r1)
991
- ld r4,GPR4(r1)
992
- ld r1,GPR1(r1)
708
+ REST_4GPRS(2, r1)
709
+ REST_GPR(6, r1)
710
+ REST_GPR(0, r1)
711
+ REST_GPR(1, r1)
993712 RFI_TO_USER
994713 b . /* prevent speculative execution */
995714
996
-1: mtspr SPRN_SRR1,r3
715
+.Lrestore_nvgprs:
716
+ REST_NVGPRS(r1)
717
+ b .Lfast_user_interrupt_return
997718
998
- ld r2,_CCR(r1)
999
- mtcrf 0xFF,r2
1000
- ld r2,_NIP(r1)
1001
- mtspr SPRN_SRR0,r2
719
+ .balign IFETCH_ALIGN_BYTES
720
+.Lkernel_interrupt_return:
721
+ addi r3,r1,STACK_FRAME_OVERHEAD
722
+ bl interrupt_exit_kernel_prepare
723
+
724
+.Lfast_kernel_interrupt_return:
725
+ cmpdi cr1,r3,0
726
+ ld r11,_NIP(r1)
727
+ ld r12,_MSR(r1)
728
+ mtspr SPRN_SRR0,r11
729
+ mtspr SPRN_SRR1,r12
730
+
731
+BEGIN_FTR_SECTION
732
+ stdcx. r0,0,r1 /* to clear the reservation */
733
+FTR_SECTION_ELSE
734
+ ldarx r0,0,r1
735
+ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
736
+
737
+ ld r3,_LINK(r1)
738
+ ld r4,_CTR(r1)
739
+ ld r5,_XER(r1)
740
+ ld r6,_CCR(r1)
741
+ li r0,0
742
+
743
+ REST_4GPRS(7, r1)
744
+ REST_2GPRS(11, r1)
745
+
746
+ mtlr r3
747
+ mtctr r4
748
+ mtspr SPRN_XER,r5
1002749
1003750 /*
1004751 * Leaving a stale exception_marker on the stack can confuse
1005752 * the reliable stack unwinder later on. Clear it.
1006753 */
1007
- li r2,0
1008
- std r2,STACK_FRAME_OVERHEAD-16(r1)
754
+ std r0,STACK_FRAME_OVERHEAD-16(r1)
1009755
1010
- ld r0,GPR0(r1)
1011
- ld r2,GPR2(r1)
1012
- ld r3,GPR3(r1)
1013
- ld r4,GPR4(r1)
1014
- ld r1,GPR1(r1)
756
+ REST_4GPRS(2, r1)
757
+
758
+ bne- cr1,1f /* emulate stack store */
759
+ mtcr r6
760
+ REST_GPR(6, r1)
761
+ REST_GPR(0, r1)
762
+ REST_GPR(1, r1)
1015763 RFI_TO_KERNEL
1016764 b . /* prevent speculative execution */
1017765
1018
-#endif /* CONFIG_PPC_BOOK3E */
1019
-
1020
- /*
1021
- * We are returning to a context with interrupts soft disabled.
1022
- *
1023
- * However, we may also about to hard enable, so we need to
1024
- * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
1025
- * or that bit can get out of sync and bad things will happen
766
+1: /*
767
+ * Emulate stack store with update. New r1 value was already calculated
768
+ * and updated in our interrupt regs by emulate_loadstore, but we can't
769
+ * store the previous value of r1 to the stack before re-loading our
770
+ * registers from it, otherwise they could be clobbered. Use
771
+ * PACA_EXGEN as temporary storage to hold the store data, as
772
+ * interrupts are disabled here so it won't be clobbered.
1026773 */
1027
-.Lrestore_irq_off:
1028
- ld r3,_MSR(r1)
1029
- lbz r7,PACAIRQHAPPENED(r13)
1030
- andi. r0,r3,MSR_EE
1031
- beq 1f
1032
- rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
1033
- stb r7,PACAIRQHAPPENED(r13)
1034
-1:
1035
-#if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
1036
- /* The interrupt should not have soft enabled. */
1037
- lbz r7,PACAIRQSOFTMASK(r13)
1038
-1: tdeqi r7,IRQS_ENABLED
1039
- EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1040
-#endif
1041
- b .Ldo_restore
774
+ mtcr r6
775
+ std r9,PACA_EXGEN+0(r13)
776
+ addi r9,r1,INT_FRAME_SIZE /* get original r1 */
777
+ REST_GPR(6, r1)
778
+ REST_GPR(0, r1)
779
+ REST_GPR(1, r1)
780
+ std r9,0(r1) /* perform store component of stdu */
781
+ ld r9,PACA_EXGEN+0(r13)
1042782
1043
- /*
1044
- * Something did happen, check if a re-emit is needed
1045
- * (this also clears paca->irq_happened)
1046
- */
1047
-.Lrestore_check_irq_replay:
1048
- /* XXX: We could implement a fast path here where we check
1049
- * for irq_happened being just 0x01, in which case we can
1050
- * clear it and return. That means that we would potentially
1051
- * miss a decrementer having wrapped all the way around.
1052
- *
1053
- * Still, this might be useful for things like hash_page
1054
- */
1055
- bl __check_irq_replay
1056
- cmpwi cr0,r3,0
1057
- beq .Lrestore_no_replay
1058
-
1059
- /*
1060
- * We need to re-emit an interrupt. We do so by re-using our
1061
- * existing exception frame. We first change the trap value,
1062
- * but we need to ensure we preserve the low nibble of it
1063
- */
1064
- ld r4,_TRAP(r1)
1065
- clrldi r4,r4,60
1066
- or r4,r4,r3
1067
- std r4,_TRAP(r1)
1068
-
1069
- /*
1070
- * PACA_IRQ_HARD_DIS won't always be set here, so set it now
1071
- * to reconcile the IRQ state. Tracing is already accounted for.
1072
- */
1073
- lbz r4,PACAIRQHAPPENED(r13)
1074
- ori r4,r4,PACA_IRQ_HARD_DIS
1075
- stb r4,PACAIRQHAPPENED(r13)
1076
-
1077
- /*
1078
- * Then find the right handler and call it. Interrupts are
1079
- * still soft-disabled and we keep them that way.
1080
- */
1081
- cmpwi cr0,r3,0x500
1082
- bne 1f
1083
- addi r3,r1,STACK_FRAME_OVERHEAD;
1084
- bl do_IRQ
1085
- b ret_from_except
1086
-1: cmpwi cr0,r3,0xf00
1087
- bne 1f
1088
- addi r3,r1,STACK_FRAME_OVERHEAD;
1089
- bl performance_monitor_exception
1090
- b ret_from_except
1091
-1: cmpwi cr0,r3,0xe60
1092
- bne 1f
1093
- addi r3,r1,STACK_FRAME_OVERHEAD;
1094
- bl handle_hmi_exception
1095
- b ret_from_except
1096
-1: cmpwi cr0,r3,0x900
1097
- bne 1f
1098
- addi r3,r1,STACK_FRAME_OVERHEAD;
1099
- bl timer_interrupt
1100
- b ret_from_except
1101
-#ifdef CONFIG_PPC_DOORBELL
1102
-1:
1103
-#ifdef CONFIG_PPC_BOOK3E
1104
- cmpwi cr0,r3,0x280
1105
-#else
1106
- cmpwi cr0,r3,0xa00
1107
-#endif /* CONFIG_PPC_BOOK3E */
1108
- bne 1f
1109
- addi r3,r1,STACK_FRAME_OVERHEAD;
1110
- bl doorbell_exception
1111
-#endif /* CONFIG_PPC_DOORBELL */
1112
-1: b ret_from_except /* What else to do here ? */
1113
-
1114
-.Lunrecov_restore:
1115
- addi r3,r1,STACK_FRAME_OVERHEAD
1116
- bl unrecoverable_exception
1117
- b .Lunrecov_restore
1118
-
1119
-_ASM_NOKPROBE_SYMBOL(ret_from_except);
1120
-_ASM_NOKPROBE_SYMBOL(ret_from_except_lite);
1121
-_ASM_NOKPROBE_SYMBOL(resume_kernel);
1122
-_ASM_NOKPROBE_SYMBOL(fast_exc_return_irq);
1123
-_ASM_NOKPROBE_SYMBOL(restore);
1124
-_ASM_NOKPROBE_SYMBOL(fast_exception_return);
1125
-
783
+ RFI_TO_KERNEL
784
+ b . /* prevent speculative execution */
785
+#endif /* CONFIG_PPC_BOOK3S */
1126786
1127787 #ifdef CONFIG_PPC_RTAS
1128788 /*
....@@ -1136,7 +796,7 @@
1136796 _GLOBAL(enter_rtas)
1137797 mflr r0
1138798 std r0,16(r1)
1139
- stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
799
+ stdu r1,-SWITCH_FRAME_SIZE(r1) /* Save SP and create stack space. */
1140800
1141801 /* Because RTAS is running in 32b mode, it clobbers the high order half
1142802 * of all registers that it saves. We therefore save those registers
....@@ -1144,8 +804,7 @@
1144804 */
1145805 SAVE_GPR(2, r1) /* Save the TOC */
1146806 SAVE_GPR(13, r1) /* Save paca */
1147
- SAVE_8GPRS(14, r1) /* Save the non-volatiles */
1148
- SAVE_10GPRS(22, r1) /* ditto */
807
+ SAVE_NVGPRS(r1) /* Save the non-volatiles */
1149808
1150809 mfcr r4
1151810 std r4,_CCR(r1)
....@@ -1252,8 +911,7 @@
1252911 /* relocation is on at this point */
1253912 REST_GPR(2, r1) /* Restore the TOC */
1254913 REST_GPR(13, r1) /* Restore paca */
1255
- REST_8GPRS(14, r1) /* Restore the non-volatiles */
1256
- REST_10GPRS(22, r1) /* ditto */
914
+ REST_NVGPRS(r1) /* Restore the non-volatiles */
1257915
1258916 GET_PACA(r13)
1259917
....@@ -1268,7 +926,7 @@
1268926 ld r8,_DSISR(r1)
1269927 mtdsisr r8
1270928
1271
- addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
929
+ addi r1,r1,SWITCH_FRAME_SIZE /* Unstack our frame */
1272930 ld r0,16(r1) /* get return address */
1273931
1274932 mtlr r0
....@@ -1279,7 +937,7 @@
1279937 _GLOBAL(enter_prom)
1280938 mflr r0
1281939 std r0,16(r1)
1282
- stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
940
+ stdu r1,-SWITCH_FRAME_SIZE(r1) /* Save SP and create stack space */
1283941
1284942 /* Because PROM is running in 32b mode, it clobbers the high order half
1285943 * of all registers that it saves. We therefore save those registers
....@@ -1287,8 +945,7 @@
1287945 */
1288946 SAVE_GPR(2, r1)
1289947 SAVE_GPR(13, r1)
1290
- SAVE_8GPRS(14, r1)
1291
- SAVE_10GPRS(22, r1)
948
+ SAVE_NVGPRS(r1)
1292949 mfcr r10
1293950 mfmsr r11
1294951 std r10,_CCR(r1)
....@@ -1332,12 +989,11 @@
1332989 /* Restore other registers */
1333990 REST_GPR(2, r1)
1334991 REST_GPR(13, r1)
1335
- REST_8GPRS(14, r1)
1336
- REST_10GPRS(22, r1)
992
+ REST_NVGPRS(r1)
1337993 ld r4,_CCR(r1)
1338994 mtcr r4
1339
-
1340
- addi r1,r1,PROM_FRAME_SIZE
995
+
996
+ addi r1,r1,SWITCH_FRAME_SIZE
1341997 ld r0,16(r1)
1342998 mtlr r0
1343999 blr