forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 072de836f53be56a70cecf70b43ae43b7ce17376 
 kernel/arch/arm64/kernel/entry.S
....@@ -1,21 +1,10 @@
1
+/* SPDX-License-Identifier: GPL-2.0-only */
12 /*
23  * Low-level exception handling code
34  *
45  * Copyright (C) 2012 ARM Ltd.
56  * Authors:	Catalin Marinas <catalin.marinas@arm.com>
67  *		Will Deacon <will.deacon@arm.com>
7
- *
8
- * This program is free software; you can redistribute it and/or modify
9
- * it under the terms of the GNU General Public License version 2 as
10
- * published by the Free Software Foundation.
11
- *
12
- * This program is distributed in the hope that it will be useful,
13
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
14
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15
- * GNU General Public License for more details.
16
- *
17
- * You should have received a copy of the GNU General Public License
18
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
198  */
209 
2110 #include <linux/arm-smccc.h>
....@@ -25,6 +14,8 @@
2514 #include <asm/alternative.h>
2615 #include <asm/assembler.h>
2716 #include <asm/asm-offsets.h>
17
+#include <asm/asm_pointer_auth.h>
18
+#include <asm/bug.h>
2819 #include <asm/cpufeature.h>
2920 #include <asm/errno.h>
3021 #include <asm/esr.h>
....@@ -33,23 +24,24 @@
3324 #include <asm/mmu.h>
3425 #include <asm/processor.h>
3526 #include <asm/ptrace.h>
27
+#include <asm/scs.h>
3628 #include <asm/thread_info.h>
3729 #include <asm/asm-uaccess.h>
3830 #include <asm/unistd.h>
3931 
4032 /*
41
- * Context tracking subsystem.  Used to instrument transitions
42
- * between user and kernel mode.
33
+ * Context tracking and irqflag tracing need to instrument transitions between
34
+ * user and kernel mode.
4335  */
44
-	.macro ct_user_exit
45
-#ifdef CONFIG_CONTEXT_TRACKING
46
-	bl	context_tracking_user_exit
36
+	.macro user_exit_irqoff
37
+#if defined(CONFIG_CONTEXT_TRACKING) || defined(CONFIG_TRACE_IRQFLAGS)
38
+	bl	enter_from_user_mode
4739 #endif
4840 	.endm
4941 
50
-	.macro ct_user_enter
51
-#ifdef CONFIG_CONTEXT_TRACKING
52
-	bl	context_tracking_user_enter
42
+	.macro user_enter_irqoff
43
+#if defined(CONFIG_CONTEXT_TRACKING) || defined(CONFIG_TRACE_IRQFLAGS)
44
+	bl	exit_to_user_mode
5345 #endif
5446 	.endm
5547 
....@@ -70,24 +62,28 @@
7062 
7163 	.macro kernel_ventry, el, label, regsize = 64
7264 	.align 7
73
-#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
74
-alternative_if ARM64_UNMAP_KERNEL_AT_EL0
65
+.Lventry_start\@:
7566 	.if	\el == 0
67
+	/*
68
+	 * This must be the first instruction of the EL0 vector entries. It is
69
+	 * skipped by the trampoline vectors, to trigger the cleanup.
70
+	 */
71
+	b	.Lskip_tramp_vectors_cleanup\@
7672 	.if	\regsize == 64
7773 	mrs	x30, tpidrro_el0
7874 	msr	tpidrro_el0, xzr
7975 	.else
8076 	mov	x30, xzr
8177 	.endif
78
+.Lskip_tramp_vectors_cleanup\@:
8279 	.endif
83
-alternative_else_nop_endif
84
-#endif
8580 
8681 	sub	sp, sp, #S_FRAME_SIZE
8782 #ifdef CONFIG_VMAP_STACK
8883 	/*
8984 	 * Test whether the SP has overflowed, without corrupting a GPR.
90
-	 * Task and IRQ stacks are aligned to (1 << THREAD_SHIFT).
85
+	 * Task and IRQ stacks are aligned so that SP & (1 << THREAD_SHIFT)
86
+	 * should always be zero.
9187 	 */
9288 	add	sp, sp, x0			// sp' = sp + x0
9389 	sub	x0, sp, x0			// x0' = sp' - x0 = (sp + x0) - x0 = sp
....@@ -127,19 +123,24 @@
127123 	mrs	x0, tpidrro_el0
128124 #endif
129125 	b	el\()\el\()_\label
126
+.org .Lventry_start\@ + 128	// Did we overflow the ventry slot?
130127 	.endm
131128 
132
-	.macro tramp_alias, dst, sym
129
+	.macro tramp_alias, dst, sym, tmp
133130 	mov_q	\dst, TRAMP_VALIAS
134
-	add	\dst, \dst, #(\sym - .entry.tramp.text)
131
+	adr_l	\tmp, \sym
132
+	add	\dst, \dst, \tmp
133
+	adr_l	\tmp, .entry.tramp.text
134
+	sub	\dst, \dst, \tmp
135135 	.endm
136136 
137
-	// This macro corrupts x0-x3. It is the caller's duty
138
-	// to save/restore them if required.
137
+	/*
138
+	 * This macro corrupts x0-x3. It is the caller's duty  to save/restore
139
+	 * them if required.
140
+	 */
139141 	.macro	apply_ssbd, state, tmp1, tmp2
140
-#ifdef CONFIG_ARM64_SSBD
141
-alternative_cb	arm64_enable_wa2_handling
142
-	b	.L__asm_ssbd_skip\@
142
+alternative_cb	spectre_v4_patch_fw_mitigation_enable
143
+	b	.L__asm_ssbd_skip\@		// Patched to NOP
143144 alternative_cb_end
144145 	ldr_this_cpu	\tmp2, arm64_ssbd_callback_required, \tmp1
145146 	cbz	\tmp2,	.L__asm_ssbd_skip\@
....@@ -147,10 +148,76 @@
147148 	tbnz	\tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@
148149 	mov	w0, #ARM_SMCCC_ARCH_WORKAROUND_2
149150 	mov	w1, #\state
150
-alternative_cb	arm64_update_smccc_conduit
151
+alternative_cb	smccc_patch_fw_mitigation_conduit
151152 	nop					// Patched to SMC/HVC #0
152153 alternative_cb_end
153154 .L__asm_ssbd_skip\@:
155
+	.endm
156
+
157
+	/* Check for MTE asynchronous tag check faults */
158
+	.macro check_mte_async_tcf, tmp, ti_flags, thread_sctlr
159
+#ifdef CONFIG_ARM64_MTE
160
+	.arch_extension lse
161
+alternative_if_not ARM64_MTE
162
+	b	1f
163
+alternative_else_nop_endif
164
+	/*
165
+	 * Asynchronous tag check faults are only possible in ASYNC (2) or
166
+	 * ASYM (3) modes. In each of these modes bit 1 of SCTLR_EL1.TCF0 is
167
+	 * set, so skip the check if it is unset.
168
+	 */
169
+	tbz	\thread_sctlr, #(SCTLR_EL1_TCF0_SHIFT + 1), 1f
170
+	mrs_s	\tmp, SYS_TFSRE0_EL1
171
+	tbz	\tmp, #SYS_TFSR_EL1_TF0_SHIFT, 1f
172
+	/* Asynchronous TCF occurred for TTBR0 access, set the TI flag */
173
+	mov	\tmp, #_TIF_MTE_ASYNC_FAULT
174
+	add	\ti_flags, tsk, #TSK_TI_FLAGS
175
+	stset	\tmp, [\ti_flags]
176
+1:
177
+#endif
178
+	.endm
179
+
180
+	/* Clear the MTE asynchronous tag check faults */
181
+	.macro clear_mte_async_tcf thread_sctlr
182
+#ifdef CONFIG_ARM64_MTE
183
+alternative_if ARM64_MTE
184
+	/* See comment in check_mte_async_tcf above. */
185
+	tbz	\thread_sctlr, #(SCTLR_EL1_TCF0_SHIFT + 1), 1f
186
+	dsb	ish
187
+	msr_s	SYS_TFSRE0_EL1, xzr
188
+1:
189
+alternative_else_nop_endif
190
+#endif
191
+	.endm
192
+
193
+	.macro mte_set_gcr, mte_ctrl, tmp
194
+#ifdef CONFIG_ARM64_MTE
195
+	ubfx	\tmp, \mte_ctrl, #MTE_CTRL_GCR_USER_EXCL_SHIFT, #16
196
+	orr	\tmp, \tmp, #SYS_GCR_EL1_RRND
197
+	msr_s	SYS_GCR_EL1, \tmp
198
+#endif
199
+	.endm
200
+
201
+	.macro mte_set_kernel_gcr, tmp, tmp2
202
+#ifdef CONFIG_KASAN_HW_TAGS
203
+alternative_cb	kasan_hw_tags_enable
204
+	b	1f
205
+alternative_cb_end
206
+	mov	\tmp, KERNEL_GCR_EL1
207
+	msr_s	SYS_GCR_EL1, \tmp
208
+1:
209
+#endif
210
+	.endm
211
+
212
+	.macro mte_set_user_gcr, tsk, tmp, tmp2
213
+#ifdef CONFIG_KASAN_HW_TAGS
214
+alternative_cb	kasan_hw_tags_enable
215
+	b	1f
216
+alternative_cb_end
217
+	ldr	\tmp, [\tsk, #THREAD_MTE_CTRL]
218
+
219
+	mte_set_gcr \tmp, \tmp2
220
+1:
154221 #endif
155222 	.endm
156223 
....@@ -177,19 +244,63 @@
177244 	.if	\el == 0
178245 	clear_gp_regs
179246 	mrs	x21, sp_el0
180
-	ldr_this_cpu	tsk, __entry_task, x20	// Ensure MDSCR_EL1.SS is clear,
181
-	ldr	x19, [tsk, #TSK_TI_FLAGS]	// since we can unmask debug
182
-	disable_step_tsk x19, x20		// exceptions when scheduling.
247
+	ldr_this_cpu	tsk, __entry_task, x20
248
+	msr	sp_el0, tsk
249
+
250
+	/*
251
+	 * Ensure MDSCR_EL1.SS is clear, since we can unmask debug exceptions
252
+	 * when scheduling.
253
+	 */
254
+	ldr	x19, [tsk, #TSK_TI_FLAGS]
255
+	disable_step_tsk x19, x20
256
+
257
+	/* Check for asynchronous tag check faults in user space */
258
+	ldr	x0, [tsk, THREAD_SCTLR_USER]
259
+	check_mte_async_tcf x22, x23, x0
260
+
261
+#ifdef CONFIG_ARM64_PTR_AUTH
262
+alternative_if ARM64_HAS_ADDRESS_AUTH
263
+	/*
264
+	 * Enable IA for in-kernel PAC if the task had it disabled. Although
265
+	 * this could be implemented with an unconditional MRS which would avoid
266
+	 * a load, this was measured to be slower on Cortex-A75 and Cortex-A76.
267
+	 *
268
+	 * Install the kernel IA key only if IA was enabled in the task. If IA
269
+	 * was disabled on kernel exit then we would have left the kernel IA
270
+	 * installed so there is no need to install it again.
271
+	 */
272
+	tbz	x0, SCTLR_ELx_ENIA_SHIFT, 1f
273
+	__ptrauth_keys_install_kernel_nosync tsk, x20, x22, x23
274
+	b	2f
275
+1:
276
+	mrs	x0, sctlr_el1
277
+	orr	x0, x0, SCTLR_ELx_ENIA
278
+	msr	sctlr_el1, x0
279
+2:
280
+alternative_else_nop_endif
281
+#endif
183282 
184283 	apply_ssbd 1, x22, x23
185284 
186
-#ifdef CONFIG_SHADOW_CALL_STACK
187
-	ldr	x18, [tsk, #TSK_TI_SCS]		// Restore shadow call stack
188
-	str	xzr, [tsk, #TSK_TI_SCS]		// Limit visibility of saved SCS
189
-#endif
285
+	mte_set_kernel_gcr x22, x23
286
+
287
+	/*
288
+	 * Any non-self-synchronizing system register updates required for
289
+	 * kernel entry should be placed before this point.
290
+	 */
291
+alternative_if ARM64_MTE
292
+	isb
293
+	b	1f
294
+alternative_else_nop_endif
295
+alternative_if ARM64_HAS_ADDRESS_AUTH
296
+	isb
297
+alternative_else_nop_endif
298
+1:
299
+
300
+	scs_load tsk, x20
190301 	.else
191302 	add	x21, sp, #S_FRAME_SIZE
192
-	get_thread_info tsk
303
+	get_current_task tsk
193304 	/* Save the task's original addr_limit and set USER_DS */
194305 	ldr	x20, [tsk, #TSK_TI_ADDR_LIMIT]
195306 	str	x20, [sp, #S_ORIG_ADDR_LIMIT]
....@@ -214,28 +325,9 @@
214325 	add	x29, sp, #S_STACKFRAME
215326 
216327 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
217
-	/*
218
-	 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from
219
-	 * EL0, there is no need to check the state of TTBR0_EL1 since
220
-	 * accesses are always enabled.
221
-	 * Note that the meaning of this bit differs from the ARMv8.1 PAN
222
-	 * feature as all TTBR0_EL1 accesses are disabled, not just those to
223
-	 * user mappings.
224
-	 */
225
-alternative_if ARM64_HAS_PAN
226
-	b	1f				// skip TTBR0 PAN
328
+alternative_if_not ARM64_HAS_PAN
329
+	bl	__swpan_entry_el\el
227330 alternative_else_nop_endif
228
-
229
-	.if	\el != 0
230
-	mrs	x21, ttbr0_el1
231
-	tst	x21, #TTBR_ASID_MASK		// Check for the reserved ASID
232
-	orr	x23, x23, #PSR_PAN_BIT		// Set the emulated PAN in the saved SPSR
233
-	b.eq	1f				// TTBR0 access already disabled
234
-	and	x23, x23, #~PSR_PAN_BIT		// Clear the emulated PAN in the saved SPSR
235
-	.endif
236
-
237
-	__uaccess_ttbr0_disable x21
238
-1:
239331 #endif
240332 
241333 	stp	x22, x23, [sp, #S_PC]
....@@ -246,16 +338,25 @@
246338 	str	w21, [sp, #S_SYSCALLNO]
247339 	.endif
248340 
249
-	/*
250
-	 * Set sp_el0 to current thread_info.
251
-	 */
252
-	.if	\el == 0
253
-	msr	sp_el0, tsk
254
-	.endif
341
+	/* Save pmr */
342
+alternative_if ARM64_HAS_IRQ_PRIO_MASKING
343
+	mrs_s	x20, SYS_ICC_PMR_EL1
344
+	str	x20, [sp, #S_PMR_SAVE]
345
+	mov	x20, #GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET
346
+	msr_s	SYS_ICC_PMR_EL1, x20
347
+alternative_else_nop_endif
348
+
349
+	/* Re-enable tag checking (TCO set on exception entry) */
350
+#ifdef CONFIG_ARM64_MTE
351
+alternative_if ARM64_MTE
352
+	SET_PSTATE_TCO(0)
353
+alternative_else_nop_endif
354
+#endif
255355 
256356 	/*
257357 	 * Registers that may be useful after this macro is invoked:
258358 	 *
359
+	 * x20 - ICC_PMR_EL1
259360 	 * x21 - aborted SP
260361 	 * x22 - aborted PC
261362 	 * x23 - aborted PSTATE
....@@ -273,46 +374,22 @@
273374 	/* No need to restore UAO, it will be restored from SPSR_EL1 */
274375 	.endif
275376 
276
-	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
277
-	.if	\el == 0
278
-	ct_user_enter
279
-	.endif
280
-
281
-#ifdef CONFIG_SHADOW_CALL_STACK
282
-	.if	\el == 0
283
-	str	x18, [tsk, #TSK_TI_SCS]		// Save shadow call stack
284
-	.endif
285
-#endif
286
-
287
-#ifdef CONFIG_ARM64_SW_TTBR0_PAN
288
-	/*
289
-	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
290
-	 * PAN bit checking.
291
-	 */
292
-alternative_if ARM64_HAS_PAN
293
-	b	2f				// skip TTBR0 PAN
377
+	/* Restore pmr */
378
+alternative_if ARM64_HAS_IRQ_PRIO_MASKING
379
+	ldr	x20, [sp, #S_PMR_SAVE]
380
+	msr_s	SYS_ICC_PMR_EL1, x20
381
+	mrs_s	x21, SYS_ICC_CTLR_EL1
382
+	tbz	x21, #6, .L__skip_pmr_sync\@	// Check for ICC_CTLR_EL1.PMHE
383
+	dsb	sy				// Ensure priority change is seen by redistributor
384
+.L__skip_pmr_sync\@:
294385 alternative_else_nop_endif
295386 
296
-	.if	\el != 0
297
-	tbnz	x22, #22, 1f			// Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
298
-	.endif
387
+	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
299388 
300
-	__uaccess_ttbr0_enable x0, x1
301
-
302
-	.if	\el == 0
303
-	/*
304
-	 * Enable errata workarounds only if returning to user. The only
305
-	 * workaround currently required for TTBR0_EL1 changes are for the
306
-	 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
307
-	 * corruption).
308
-	 */
309
-	bl	post_ttbr_update_workaround
310
-	.endif
311
-1:
312
-	.if	\el != 0
313
-	and	x22, x22, #~PSR_PAN_BIT		// ARMv8.0 CPUs do not understand this bit
314
-	.endif
315
-2:
389
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
390
+alternative_if_not ARM64_HAS_PAN
391
+	bl	__swpan_exit_el\el
392
+alternative_else_nop_endif
316393 #endif
317394 
318395 	.if	\el == 0
....@@ -332,6 +409,34 @@
332409 alternative_else_nop_endif
333410 #endif
334411 3:
412
+	scs_save tsk, x0
413
+
414
+	/* Ignore asynchronous tag check faults in the uaccess routines */
415
+	ldr	x0, [tsk, THREAD_SCTLR_USER]
416
+	clear_mte_async_tcf x0
417
+
418
+#ifdef CONFIG_ARM64_PTR_AUTH
419
+alternative_if ARM64_HAS_ADDRESS_AUTH
420
+	/*
421
+	 * IA was enabled for in-kernel PAC. Disable it now if needed, or
422
+	 * alternatively install the user's IA. All other per-task keys and
423
+	 * SCTLR bits were updated on task switch.
424
+	 *
425
+	 * No kernel C function calls after this.
426
+	 */
427
+	tbz	x0, SCTLR_ELx_ENIA_SHIFT, 1f
428
+	__ptrauth_keys_install_user tsk, x0, x1, x2
429
+	b	2f
430
+1:
431
+	mrs	x0, sctlr_el1
432
+	bic	x0, x0, SCTLR_ELx_ENIA
433
+	msr	sctlr_el1, x0
434
+2:
435
+alternative_else_nop_endif
436
+#endif
437
+
438
+	mte_set_user_gcr tsk, x0, x1
439
+
335440 	apply_ssbd 0, x0, x1
336441 	.endif
337442 
....@@ -352,34 +457,81 @@
352457 	ldp	x24, x25, [sp, #16 * 12]
353458 	ldp	x26, x27, [sp, #16 * 13]
354459 	ldp	x28, x29, [sp, #16 * 14]
355
-	ldr	lr, [sp, #S_LR]
356
-	add	sp, sp, #S_FRAME_SIZE		// restore sp
357
-	/*
358
-	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on eret context synchronization
359
-	 * when returning from IPI handler, and when returning to user-space.
360
-	 */
361460 
362461 	.if	\el == 0
363
-alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
462
+alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
463
+	ldr	lr, [sp, #S_LR]
464
+	add	sp, sp, #S_FRAME_SIZE		// restore sp
465
+	eret
466
+alternative_else_nop_endif
364467 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
365468 	bne	4f
366
-	msr	far_el1, x30
367
-	tramp_alias	x30, tramp_exit_native
469
+	msr	far_el1, x29
470
+	tramp_alias	x30, tramp_exit_native, x29
368471 	br	x30
369472 4:
370
-	tramp_alias	x30, tramp_exit_compat
473
+	tramp_alias	x30, tramp_exit_compat, x29
371474 	br	x30
372475 #endif
373476 	.else
477
+	ldr	lr, [sp, #S_LR]
478
+	add	sp, sp, #S_FRAME_SIZE		// restore sp
479
+
480
+	/* Ensure any device/NC reads complete */
481
+	alternative_insn nop, "dmb sy", ARM64_WORKAROUND_1508412
482
+
374483 	eret
375484 	.endif
376485 	sb
377486 	.endm
378487 
488
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
489
+	/*
490
+	 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from
491
+	 * EL0, there is no need to check the state of TTBR0_EL1 since
492
+	 * accesses are always enabled.
493
+	 * Note that the meaning of this bit differs from the ARMv8.1 PAN
494
+	 * feature as all TTBR0_EL1 accesses are disabled, not just those to
495
+	 * user mappings.
496
+	 */
497
+SYM_CODE_START_LOCAL(__swpan_entry_el1)
498
+	mrs	x21, ttbr0_el1
499
+	tst	x21, #TTBR_ASID_MASK		// Check for the reserved ASID
500
+	orr	x23, x23, #PSR_PAN_BIT		// Set the emulated PAN in the saved SPSR
501
+	b.eq	1f				// TTBR0 access already disabled
502
+	and	x23, x23, #~PSR_PAN_BIT		// Clear the emulated PAN in the saved SPSR
503
+SYM_INNER_LABEL(__swpan_entry_el0, SYM_L_LOCAL)
504
+	__uaccess_ttbr0_disable x21
505
+1:	ret
506
+SYM_CODE_END(__swpan_entry_el1)
507
+
508
+	/*
509
+	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
510
+	 * PAN bit checking.
511
+	 */
512
+SYM_CODE_START_LOCAL(__swpan_exit_el1)
513
+	tbnz	x22, #22, 1f			// Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
514
+	__uaccess_ttbr0_enable x0, x1
515
+1:	and	x22, x22, #~PSR_PAN_BIT		// ARMv8.0 CPUs do not understand this bit
516
+	ret
517
+SYM_CODE_END(__swpan_exit_el1)
518
+
519
+SYM_CODE_START_LOCAL(__swpan_exit_el0)
520
+	__uaccess_ttbr0_enable x0, x1
521
+	/*
522
+	 * Enable errata workarounds only if returning to user. The only
523
+	 * workaround currently required for TTBR0_EL1 changes are for the
524
+	 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
525
+	 * corruption).
526
+	 */
527
+	b	post_ttbr_update_workaround
528
+SYM_CODE_END(__swpan_exit_el0)
529
+#endif
530
+
379531 	.macro	irq_stack_entry
380532 	mov	x19, sp			// preserve the original sp
381533 #ifdef CONFIG_SHADOW_CALL_STACK
382
-	mov	x20, x18		// preserve the original shadow stack
534
+	mov	x24, scs_sp		// preserve the original shadow stack
383535 #endif
384536 
385537 	/*
....@@ -401,7 +553,7 @@
401553 
402554 #ifdef CONFIG_SHADOW_CALL_STACK
403555 	/* also switch to the irq shadow stack */
404
-	ldr_this_cpu x18, irq_shadow_call_stack_ptr, x26
556
+	ldr_this_cpu scs_sp, irq_shadow_call_stack_ptr, x26
405557 #endif
406558 
407559 9998:
....@@ -409,37 +561,88 @@
409561 
410562 	/*
411563 	 * The callee-saved regs (x19-x29) should be preserved between
412
-	 * irq_stack_entry and irq_stack_exit.
564
+	 * irq_stack_entry and irq_stack_exit, but note that kernel_entry
565
+	 * uses x20-x23 to store data for later use.
413566 	 */
414567 	.macro	irq_stack_exit
415568 	mov	sp, x19
416569 #ifdef CONFIG_SHADOW_CALL_STACK
417
-	mov	x18, x20
570
+	mov	scs_sp, x24
418571 #endif
419572 	.endm
420573 
421
-/*
422
- * These are the registers used in the syscall handler, and allow us to
423
- * have in theory up to 7 arguments to a function - x0 to x6.
424
- *
425
- * x7 is reserved for the system call number in 32-bit mode.
426
- */
427
-wsc_nr	.req	w25		// number of system calls
428
-xsc_nr	.req	x25		// number of system calls (zero-extended)
429
-wscno	.req	w26		// syscall number
430
-xscno	.req	x26		// syscall number (zero-extended)
431
-stbl	.req	x27		// syscall table pointer
574
+/* GPRs used by entry code */
432575 tsk	.req	x28		// current thread_info
433576 
434577 /*
435578  * Interrupt handling.
436579  */
437
-	.macro	irq_handler
438
-	ldr_l	x1, handle_arch_irq
580
+	.macro	irq_handler, handler:req
581
+	ldr_l	x1, \handler
439582 	mov	x0, sp
440583 	irq_stack_entry
441584 	blr	x1
442585 	irq_stack_exit
586
+	.endm
587
+
588
+#ifdef CONFIG_ARM64_PSEUDO_NMI
589
+	/*
590
+	 * Set res to 0 if irqs were unmasked in interrupted context.
591
+	 * Otherwise set res to non-0 value.
592
+	 */
593
+	.macro	test_irqs_unmasked res:req, pmr:req
594
+alternative_if ARM64_HAS_IRQ_PRIO_MASKING
595
+	sub	\res, \pmr, #GIC_PRIO_IRQON
596
+alternative_else
597
+	mov	\res, xzr
598
+alternative_endif
599
+	.endm
600
+#endif
601
+
602
+	.macro	gic_prio_kentry_setup, tmp:req
603
+#ifdef CONFIG_ARM64_PSEUDO_NMI
604
+	alternative_if ARM64_HAS_IRQ_PRIO_MASKING
605
+	mov	\tmp, #(GIC_PRIO_PSR_I_SET | GIC_PRIO_IRQON)
606
+	msr_s	SYS_ICC_PMR_EL1, \tmp
607
+	alternative_else_nop_endif
608
+#endif
609
+	.endm
610
+
611
+	.macro el1_interrupt_handler, handler:req
612
+	enable_da_f
613
+
614
+	mov	x0, sp
615
+	bl	enter_el1_irq_or_nmi
616
+
617
+	irq_handler	\handler
618
+
619
+#ifdef CONFIG_PREEMPTION
620
+	ldr	x24, [tsk, #TSK_TI_PREEMPT]	// get preempt count
621
+alternative_if ARM64_HAS_IRQ_PRIO_MASKING
622
+	/*
623
+	 * DA_F were cleared at start of handling. If anything is set in DAIF,
624
+	 * we come back from an NMI, so skip preemption
625
+	 */
626
+	mrs	x0, daif
627
+	orr	x24, x24, x0
628
+alternative_else_nop_endif
629
+	cbnz	x24, 1f				// preempt count != 0 || NMI return path
630
+	bl	arm64_preempt_schedule_irq	// irq en/disable is done inside
631
+1:
632
+#endif
633
+
634
+	mov	x0, sp
635
+	bl	exit_el1_irq_or_nmi
636
+	.endm
637
+
638
+	.macro el0_interrupt_handler, handler:req
639
+	user_exit_irqoff
640
+	enable_da_f
641
+
642
+	tbz	x22, #55, 1f
643
+	bl	do_el0_irq_bp_hardening
644
+1:
645
+	irq_handler	\handler
443646 	.endm
444647 
445648 	.text
....@@ -450,7 +653,7 @@
450653 	.pushsection ".entry.text", "ax"
451654 
452655 	.align	11
453
-ENTRY(vectors)
656
+SYM_CODE_START(vectors)
454657 	kernel_ventry	1, sync_invalid			// Synchronous EL1t
455658 	kernel_ventry	1, irq_invalid			// IRQ EL1t
456659 	kernel_ventry	1, fiq_invalid			// FIQ EL1t
....@@ -477,7 +680,7 @@
477680 	kernel_ventry	0, fiq_invalid, 32		// FIQ 32-bit EL0
478681 	kernel_ventry	0, error_invalid, 32		// Error 32-bit EL0
479682 #endif
480
-END(vectors)
683
+SYM_CODE_END(vectors)
481684 
482685 #ifdef CONFIG_VMAP_STACK
483686 	/*
....@@ -519,439 +722,161 @@
519722 	ASM_BUG()
520723 	.endm
521724 
522
-el0_sync_invalid:
725
+SYM_CODE_START_LOCAL(el0_sync_invalid)
523726 	inv_entry 0, BAD_SYNC
524
-ENDPROC(el0_sync_invalid)
727
+SYM_CODE_END(el0_sync_invalid)
525728 
526
-el0_irq_invalid:
729
+SYM_CODE_START_LOCAL(el0_irq_invalid)
527730 	inv_entry 0, BAD_IRQ
528
-ENDPROC(el0_irq_invalid)
731
+SYM_CODE_END(el0_irq_invalid)
529732 
530
-el0_fiq_invalid:
733
+SYM_CODE_START_LOCAL(el0_fiq_invalid)
531734 	inv_entry 0, BAD_FIQ
532
-ENDPROC(el0_fiq_invalid)
735
+SYM_CODE_END(el0_fiq_invalid)
533736 
534
-el0_error_invalid:
737
+SYM_CODE_START_LOCAL(el0_error_invalid)
535738 	inv_entry 0, BAD_ERROR
536
-ENDPROC(el0_error_invalid)
739
+SYM_CODE_END(el0_error_invalid)
537740 
538741 #ifdef CONFIG_COMPAT
539
-el0_fiq_invalid_compat:
742
+SYM_CODE_START_LOCAL(el0_fiq_invalid_compat)
540743 	inv_entry 0, BAD_FIQ, 32
541
-ENDPROC(el0_fiq_invalid_compat)
744
+SYM_CODE_END(el0_fiq_invalid_compat)
542745 #endif
543746 
544
-el1_sync_invalid:
747
+SYM_CODE_START_LOCAL(el1_sync_invalid)
545748 	inv_entry 1, BAD_SYNC
546
-ENDPROC(el1_sync_invalid)
749
+SYM_CODE_END(el1_sync_invalid)
547750 
548
-el1_irq_invalid:
751
+SYM_CODE_START_LOCAL(el1_irq_invalid)
549752 	inv_entry 1, BAD_IRQ
550
-ENDPROC(el1_irq_invalid)
753
+SYM_CODE_END(el1_irq_invalid)
551754 
552
-el1_fiq_invalid:
755
+SYM_CODE_START_LOCAL(el1_fiq_invalid)
553756 	inv_entry 1, BAD_FIQ
554
-ENDPROC(el1_fiq_invalid)
757
+SYM_CODE_END(el1_fiq_invalid)
555758 
556
-el1_error_invalid:
759
+SYM_CODE_START_LOCAL(el1_error_invalid)
557760 	inv_entry 1, BAD_ERROR
558
-ENDPROC(el1_error_invalid)
761
+SYM_CODE_END(el1_error_invalid)
559762 
560763 /*
561764  * EL1 mode handlers.
562765  */
563766 	.align	6
564
-el1_sync:
767
+SYM_CODE_START_LOCAL_NOALIGN(el1_sync)
565768 	kernel_entry 1
566
-	mrs	x1, esr_el1			// read the syndrome register
567
-	lsr	x24, x1, #ESR_ELx_EC_SHIFT	// exception class
568
-	cmp	x24, #ESR_ELx_EC_DABT_CUR	// data abort in EL1
569
-	b.eq	el1_da
570
-	cmp	x24, #ESR_ELx_EC_IABT_CUR	// instruction abort in EL1
571
-	b.eq	el1_ia
572
-	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
573
-	b.eq	el1_undef
574
-	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
575
-	b.eq	el1_sp_pc
576
-	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
577
-	b.eq	el1_sp_pc
578
-	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL1
579
-	b.eq	el1_undef
580
-	cmp	x24, #ESR_ELx_EC_BREAKPT_CUR	// debug exception in EL1
581
-	b.ge	el1_dbg
582
-	b	el1_inv
583
-
584
-el1_ia:
585
-	/*
586
-	 * Fall through to the Data abort case
587
-	 */
588
-el1_da:
589
-	/*
590
-	 * Data abort handling
591
-	 */
592
-	mrs	x3, far_el1
593
-	inherit_daif	pstate=x23, tmp=x2
594
-	untagged_addr x0, x3
595
-	mov	x2, sp				// struct pt_regs
596
-	bl	do_mem_abort
597
-
598
-	kernel_exit 1
599
-el1_sp_pc:
600
-	/*
601
-	 * Stack or PC alignment exception handling
602
-	 */
603
-	mrs	x0, far_el1
604
-	inherit_daif	pstate=x23, tmp=x2
605
-	mov	x2, sp
606
-	bl	do_sp_pc_abort
607
-	ASM_BUG()
608
-el1_undef:
609
-	/*
610
-	 * Undefined instruction
611
-	 */
612
-	inherit_daif	pstate=x23, tmp=x2
613769 	mov	x0, sp
614
-	bl	do_undefinstr
770
+	bl	el1_sync_handler
615771 	kernel_exit 1
616
-el1_dbg:
617
-	/*
618
-	 * Debug exception handling
619
-	 */
620
-	cmp	x24, #ESR_ELx_EC_BRK64		// if BRK64
621
-	cinc	x24, x24, eq			// set bit '0'
622
-	tbz	x24, #0, el1_inv		// EL1 only
623
-	mrs	x0, far_el1
624
-	mov	x2, sp				// struct pt_regs
625
-	bl	do_debug_exception
626
-	kernel_exit 1
627
-el1_inv:
628
-	// TODO: add support for undefined instructions in kernel mode
629
-	inherit_daif	pstate=x23, tmp=x2
630
-	mov	x0, sp
631
-	mov	x2, x1
632
-	mov	x1, #BAD_SYNC
633
-	bl	bad_mode
634
-	ASM_BUG()
635
-ENDPROC(el1_sync)
772
+SYM_CODE_END(el1_sync)
636773 
637774 	.align	6
638
-el1_irq:
775
+SYM_CODE_START_LOCAL_NOALIGN(el1_irq)
639776 	kernel_entry 1
640
-	enable_da_f
641
-#ifdef CONFIG_TRACE_IRQFLAGS
642
-	bl	trace_hardirqs_off
643
-#endif
644
-
645
-	irq_handler
646
-
647
-#ifdef CONFIG_PREEMPT
648
-	ldr	w24, [tsk, #TSK_TI_PREEMPT]	// get preempt count
649
-	cbnz	w24, 1f				// preempt count != 0
650
-	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get flags
651
-	tbz	x0, #TIF_NEED_RESCHED, 1f	// needs rescheduling?
652
-	bl	el1_preempt
653
-1:
654
-#endif
655
-#ifdef CONFIG_TRACE_IRQFLAGS
656
-	bl	trace_hardirqs_on
657
-#endif
777
+	el1_interrupt_handler handle_arch_irq
658778 	kernel_exit 1
659
-ENDPROC(el1_irq)
660
-
661
-#ifdef CONFIG_PREEMPT
662
-el1_preempt:
663
-	mov	x24, lr
664
-1:	bl	preempt_schedule_irq		// irq en/disable is done inside
665
-	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get new tasks TI_FLAGS
666
-	tbnz	x0, #TIF_NEED_RESCHED, 1b	// needs rescheduling?
667
-	ret	x24
668
-#endif
779
+SYM_CODE_END(el1_irq)
669780 
670781 /*
671782  * EL0 mode handlers.
672783  */
673784 	.align	6
674
-el0_sync:
785
+SYM_CODE_START_LOCAL_NOALIGN(el0_sync)
675786 	kernel_entry 0
676
-	mrs	x25, esr_el1			// read the syndrome register
677
-	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
678
-	cmp	x24, #ESR_ELx_EC_SVC64		// SVC in 64-bit state
679
-	b.eq	el0_svc
680
-	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
681
-	b.eq	el0_da
682
-	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
683
-	b.eq	el0_ia
684
-	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
685
-	b.eq	el0_fpsimd_acc
686
-	cmp	x24, #ESR_ELx_EC_SVE		// SVE access
687
-	b.eq	el0_sve_acc
688
-	cmp	x24, #ESR_ELx_EC_FP_EXC64	// FP/ASIMD exception
689
-	b.eq	el0_fpsimd_exc
690
-	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
691
-	b.eq	el0_sys
692
-	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
693
-	b.eq	el0_sp_pc
694
-	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
695
-	b.eq	el0_sp_pc
696
-	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
697
-	b.eq	el0_undef
698
-	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
699
-	b.ge	el0_dbg
700
-	b	el0_inv
787
+	mov	x0, sp
788
+	bl	el0_sync_handler
789
+	b	ret_to_user
790
+SYM_CODE_END(el0_sync)
701791 
702792 #ifdef CONFIG_COMPAT
703793 	.align	6
704
-el0_sync_compat:
794
+SYM_CODE_START_LOCAL_NOALIGN(el0_sync_compat)
705795 	kernel_entry 0, 32
706
-	mrs	x25, esr_el1			// read the syndrome register
707
-	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
708
-	cmp	x24, #ESR_ELx_EC_SVC32		// SVC in 32-bit state
709
-	b.eq	el0_svc_compat
710
-	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
711
-	b.eq	el0_da
712
-	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
713
-	b.eq	el0_ia
714
-	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
715
-	b.eq	el0_fpsimd_acc
716
-	cmp	x24, #ESR_ELx_EC_FP_EXC32	// FP/ASIMD exception
717
-	b.eq	el0_fpsimd_exc
718
-	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
719
-	b.eq	el0_sp_pc
720
-	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
721
-	b.eq	el0_undef
722
-	cmp	x24, #ESR_ELx_EC_CP15_32	// CP15 MRC/MCR trap
723
-	b.eq	el0_undef
724
-	cmp	x24, #ESR_ELx_EC_CP15_64	// CP15 MRRC/MCRR trap
725
-	b.eq	el0_undef
726
-	cmp	x24, #ESR_ELx_EC_CP14_MR	// CP14 MRC/MCR trap
727
-	b.eq	el0_undef
728
-	cmp	x24, #ESR_ELx_EC_CP14_LS	// CP14 LDC/STC trap
729
-	b.eq	el0_undef
730
-	cmp	x24, #ESR_ELx_EC_CP14_64	// CP14 MRRC/MCRR trap
731
-	b.eq	el0_undef
732
-	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
733
-	b.ge	el0_dbg
734
-	b	el0_inv
735
-el0_svc_compat:
736796 	mov	x0, sp
737
-	bl	el0_svc_compat_handler
797
+	bl	el0_sync_compat_handler
738798 	b	ret_to_user
799
+SYM_CODE_END(el0_sync_compat)
739800 
740801 	.align	6
741
-el0_irq_compat:
802
+SYM_CODE_START_LOCAL_NOALIGN(el0_irq_compat)
742803 	kernel_entry 0, 32
743804 	b	el0_irq_naked
805
+SYM_CODE_END(el0_irq_compat)
744806 
745
-el0_error_compat:
807
+SYM_CODE_START_LOCAL_NOALIGN(el0_error_compat)
746808 	kernel_entry 0, 32
747809 	b	el0_error_naked
810
+SYM_CODE_END(el0_error_compat)
748811 #endif
749
-
750
-el0_da:
751
-	/*
752
-	 * Data abort handling
753
-	 */
754
-	mrs	x26, far_el1
755
-	enable_daif
756
-	ct_user_exit
757
-	untagged_addr x0, x26
758
-	mov	x1, x25
759
-	mov	x2, sp
760
-	bl	do_mem_abort
761
-	b	ret_to_user
762
-el0_ia:
763
-	/*
764
-	 * Instruction abort handling
765
-	 */
766
-	mrs	x26, far_el1
767
-	enable_da_f
768
-#ifdef CONFIG_TRACE_IRQFLAGS
769
-	bl	trace_hardirqs_off
770
-#endif
771
-	ct_user_exit
772
-	mov	x0, x26
773
-	mov	x1, x25
774
-	mov	x2, sp
775
-	bl	do_el0_ia_bp_hardening
776
-	b	ret_to_user
777
-el0_fpsimd_acc:
778
-	/*
779
-	 * Floating Point or Advanced SIMD access
780
-	 */
781
-	enable_daif
782
-	ct_user_exit
783
-	mov	x0, x25
784
-	mov	x1, sp
785
-	bl	do_fpsimd_acc
786
-	b	ret_to_user
787
-el0_sve_acc:
788
-	/*
789
-	 * Scalable Vector Extension access
790
-	 */
791
-	enable_daif
792
-	ct_user_exit
793
-	mov	x0, x25
794
-	mov	x1, sp
795
-	bl	do_sve_acc
796
-	b	ret_to_user
797
-el0_fpsimd_exc:
798
-	/*
799
-	 * Floating Point, Advanced SIMD or SVE exception
800
-	 */
801
-	enable_daif
802
-	ct_user_exit
803
-	mov	x0, x25
804
-	mov	x1, sp
805
-	bl	do_fpsimd_exc
806
-	b	ret_to_user
807
-el0_sp_pc:
808
-	/*
809
-	 * Stack or PC alignment exception handling
810
-	 */
811
-	mrs	x26, far_el1
812
-	enable_da_f
813
-#ifdef CONFIG_TRACE_IRQFLAGS
814
-	bl	trace_hardirqs_off
815
-#endif
816
-	ct_user_exit
817
-	mov	x0, x26
818
-	mov	x1, x25
819
-	mov	x2, sp
820
-	bl	do_sp_pc_abort
821
-	b	ret_to_user
822
-el0_undef:
823
-	/*
824
-	 * Undefined instruction
825
-	 */
826
-	enable_daif
827
-	ct_user_exit
828
-	mov	x0, sp
829
-	bl	do_undefinstr
830
-	b	ret_to_user
831
-el0_sys:
832
-	/*
833
-	 * System instructions, for trapped cache maintenance instructions
834
-	 */
835
-	enable_daif
836
-	ct_user_exit
837
-	mov	x0, x25
838
-	mov	x1, sp
839
-	bl	do_sysinstr
840
-	b	ret_to_user
841
-el0_dbg:
842
-	/*
843
-	 * Debug exception handling
844
-	 */
845
-	tbnz	x24, #0, el0_inv		// EL0 only
846
-	mrs	x0, far_el1
847
-	mov	x1, x25
848
-	mov	x2, sp
849
-	bl	do_debug_exception
850
-	enable_da_f
851
-	ct_user_exit
852
-	b	ret_to_user
853
-el0_inv:
854
-	enable_daif
855
-	ct_user_exit
856
-	mov	x0, sp
857
-	mov	x1, #BAD_SYNC
858
-	mov	x2, x25
859
-	bl	bad_el0_sync
860
-	b	ret_to_user
861
-ENDPROC(el0_sync)
862812 
863813 	.align	6
864
-el0_irq:
814
+SYM_CODE_START_LOCAL_NOALIGN(el0_irq)
865815 	kernel_entry 0
866816 el0_irq_naked:
867
-	enable_da_f
868
-#ifdef CONFIG_TRACE_IRQFLAGS
869
-	bl	trace_hardirqs_off
870
-#endif
871
-
872
-	ct_user_exit
873
-#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
874
-	tbz	x22, #55, 1f
875
-	bl	do_el0_irq_bp_hardening
876
-1:
877
-#endif
878
-	irq_handler
879
-
880
-#ifdef CONFIG_TRACE_IRQFLAGS
881
-	bl	trace_hardirqs_on
882
-#endif
817
+	el0_interrupt_handler handle_arch_irq
883818 	b	ret_to_user
884
-ENDPROC(el0_irq)
819
+SYM_CODE_END(el0_irq)
885820 
886
-el1_error:
821
+SYM_CODE_START_LOCAL(el1_error)
887822 	kernel_entry 1
888823 	mrs	x1, esr_el1
889824 	enable_dbg
890825 	mov	x0, sp
891826 	bl	do_serror
892827 	kernel_exit 1
893
-ENDPROC(el1_error)
828
+SYM_CODE_END(el1_error)
894829 
895
-el0_error:
830
+SYM_CODE_START_LOCAL(el0_error)
896831 	kernel_entry 0
897832 el0_error_naked:
898
-	mrs	x1, esr_el1
833
+	mrs	x25, esr_el1
834
+	user_exit_irqoff
899835 	enable_dbg
900836 	mov	x0, sp
837
+	mov	x1, x25
901838 	bl	do_serror
902839 	enable_da_f
903
-	ct_user_exit
904840 	b	ret_to_user
905
-ENDPROC(el0_error)
841
+SYM_CODE_END(el0_error)
842
+
843
+/*
844
+ * "slow" syscall return path.
845
+ */
846
+SYM_CODE_START_LOCAL(ret_to_user)
847
+	disable_daif
848
+	gic_prio_kentry_setup tmp=x3
849
+#ifdef CONFIG_TRACE_IRQFLAGS
850
+	bl	trace_hardirqs_off
851
+#endif
852
+	ldr	x19, [tsk, #TSK_TI_FLAGS]
853
+	and	x2, x19, #_TIF_WORK_MASK
854
+	cbnz	x2, work_pending
855
+finish_ret_to_user:
856
+	user_enter_irqoff
857
+	enable_step_tsk x19, x2
858
+#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
859
+	bl	stackleak_erase
860
+#endif
861
+	kernel_exit 0
906862 
907863 /*
908864  * Ok, we need to do extra processing, enter the slow path.
909865  */
910866 work_pending:
911867 	mov	x0, sp				// 'regs'
868
+	mov	x1, x19
912869 	bl	do_notify_resume
913
-#ifdef CONFIG_TRACE_IRQFLAGS
914
-	bl	trace_hardirqs_on		// enabled while in userspace
915
-#endif
916
-	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for single-step
870
+	ldr	x19, [tsk, #TSK_TI_FLAGS]	// re-check for single-step
917871 	b	finish_ret_to_user
918
-/*
919
- * "slow" syscall return path.
920
- */
921
-ret_to_user:
922
-	disable_daif
923
-	ldr	x1, [tsk, #TSK_TI_FLAGS]
924
-	and	x2, x1, #_TIF_WORK_MASK
925
-	cbnz	x2, work_pending
926
-finish_ret_to_user:
927
-	enable_step_tsk x1, x2
928
-#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
929
-	bl	stackleak_erase
930
-#endif
931
-	kernel_exit 0
932
-ENDPROC(ret_to_user)
933
-
934
-/*
935
- * SVC handler.
936
- */
937
-	.align	6
938
-el0_svc:
939
-	mov	x0, sp
940
-	bl	el0_svc_handler
941
-	b	ret_to_user
942
-ENDPROC(el0_svc)
872
+SYM_CODE_END(ret_to_user)
943873 
944874 	.popsection				// .entry.text
945875 
946
-#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
947
-/*
948
- * Exception vectors trampoline.
949
- */
950
-	.pushsection ".entry.tramp.text", "ax"
951
-
876
+	// Move from tramp_pg_dir to swapper_pg_dir
952877 	.macro tramp_map_kernel, tmp
953878 	mrs	\tmp, ttbr1_el1
954
-	add	\tmp, \tmp, #(PAGE_SIZE + RESERVED_TTBR0_SIZE)
879
+	add	\tmp, \tmp, #(2 * PAGE_SIZE)
955880 	bic	\tmp, \tmp, #USER_ASID_FLAG
956881 	msr	ttbr1_el1, \tmp
957882 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
....@@ -968,9 +893,10 @@
968893 #endif /* CONFIG_QCOM_FALKOR_ERRATUM_1003 */
969894 	.endm
970895 
896
+	// Move from swapper_pg_dir to tramp_pg_dir
971897 	.macro tramp_unmap_kernel, tmp
972898 	mrs	\tmp, ttbr1_el1
973
-	sub	\tmp, \tmp, #(PAGE_SIZE + RESERVED_TTBR0_SIZE)
899
+	sub	\tmp, \tmp, #(2 * PAGE_SIZE)
974900 	orr	\tmp, \tmp, #USER_ASID_FLAG
975901 	msr	ttbr1_el1, \tmp
976902 	/*
....@@ -980,12 +906,47 @@
980906 	 */
981907 	.endm
982908 
983
-	.macro tramp_ventry, regsize = 64
909
+	.macro tramp_data_page	dst
910
+	adr_l	\dst, .entry.tramp.text
911
+	sub	\dst, \dst, PAGE_SIZE
912
+	.endm
913
+
914
+	.macro tramp_data_read_var	dst, var
915
+#ifdef CONFIG_RANDOMIZE_BASE
916
+	tramp_data_page		\dst
917
+	add	\dst, \dst, #:lo12:__entry_tramp_data_\var
918
+	ldr	\dst, [\dst]
919
+#else
920
+	ldr	\dst, =\var
921
+#endif
922
+	.endm
923
+
924
+#define BHB_MITIGATION_NONE	0
925
+#define BHB_MITIGATION_LOOP	1
926
+#define BHB_MITIGATION_FW	2
927
+#define BHB_MITIGATION_INSN	3
928
+
929
+	.macro tramp_ventry, vector_start, regsize, kpti, bhb
984930 	.align	7
985931 1:
986932 	.if	\regsize == 64
987933 	msr	tpidrro_el0, x30	// Restored in kernel_ventry
988934 	.endif
935
+
936
+	.if	\bhb == BHB_MITIGATION_LOOP
937
+	/*
938
+	 * This sequence must appear before the first indirect branch. i.e. the
939
+	 * ret out of tramp_ventry. It appears here because x30 is free.
940
+	 */
941
+	__mitigate_spectre_bhb_loop	x30
942
+	.endif // \bhb == BHB_MITIGATION_LOOP
943
+
944
+	.if	\bhb == BHB_MITIGATION_INSN
945
+	clearbhb
946
+	isb
947
+	.endif // \bhb == BHB_MITIGATION_INSN
948
+
949
+	.if	\kpti == 1
989950 	/*
990951 	 * Defend against branch aliasing attacks by pushing a dummy
991952 	 * entry onto the return stack and using a RET instruction to
....@@ -995,65 +956,140 @@
995956 	b	.
996957 2:
997958 	tramp_map_kernel	x30
998
-#ifdef CONFIG_RANDOMIZE_BASE
999
-	adr	x30, tramp_vectors + PAGE_SIZE
1000959 alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003
1001
-	ldr	x30, [x30]
1002
-#else
1003
-	ldr	x30, =vectors
1004
-#endif
1005
-	prfm	plil1strm, [x30, #(1b - tramp_vectors)]
960
+	tramp_data_read_var	x30, vectors
961
+alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM
962
+	prfm	plil1strm, [x30, #(1b - \vector_start)]
963
+alternative_else_nop_endif
964
+
1006965 	msr	vbar_el1, x30
1007
-	add	x30, x30, #(1b - tramp_vectors)
1008966 	isb
967
+	.else
968
+	ldr	x30, =vectors
969
+	.endif // \kpti == 1
970
+
971
+	.if	\bhb == BHB_MITIGATION_FW
972
+	/*
973
+	 * The firmware sequence must appear before the first indirect branch.
974
+	 * i.e. the ret out of tramp_ventry. But it also needs the stack to be
975
+	 * mapped to save/restore the registers the SMC clobbers.
976
+	 */
977
+	__mitigate_spectre_bhb_fw
978
+	.endif // \bhb == BHB_MITIGATION_FW
979
+
980
+	add	x30, x30, #(1b - \vector_start + 4)
1009981 	ret
982
+.org 1b + 128	// Did we overflow the ventry slot?
1010983 	.endm
1011984 
1012985 	.macro tramp_exit, regsize = 64
1013
-	adr	x30, tramp_vectors
986
+	tramp_data_read_var	x30, this_cpu_vector
987
+	this_cpu_offset x29
988
+	ldr	x30, [x30, x29]
989
+
1014990 	msr	vbar_el1, x30
1015
-	tramp_unmap_kernel	x30
991
+	ldr	lr, [sp, #S_LR]
992
+	tramp_unmap_kernel	x29
1016993 	.if	\regsize == 64
1017
-	mrs	x30, far_el1
994
+	mrs	x29, far_el1
1018995 	.endif
996
+	add	sp, sp, #S_FRAME_SIZE		// restore sp
1019997 	eret
1020998 	sb
1021999 	.endm
10221000 
1023
-	.align	11
1024
-ENTRY(tramp_vectors)
1001
+	.macro	generate_tramp_vector,	kpti, bhb
1002
+.Lvector_start\@:
10251003 	.space	0x400
10261004 
1027
-	tramp_ventry
1028
-	tramp_ventry
1029
-	tramp_ventry
1030
-	tramp_ventry
1005
+	.rept	4
1006
+	tramp_ventry	.Lvector_start\@, 64, \kpti, \bhb
1007
+	.endr
1008
+	.rept	4
1009
+	tramp_ventry	.Lvector_start\@, 32, \kpti, \bhb
1010
+	.endr
1011
+	.endm
10311012 
1032
-	tramp_ventry	32
1033
-	tramp_ventry	32
1034
-	tramp_ventry	32
1035
-	tramp_ventry	32
1036
-END(tramp_vectors)
1013
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
1014
+/*
1015
+ * Exception vectors trampoline.
1016
+ * The order must match __bp_harden_el1_vectors and the
1017
+ * arm64_bp_harden_el1_vectors enum.
1018
+ */
1019
+	.pushsection ".entry.tramp.text", "ax"
1020
+	.align	11
1021
+SYM_CODE_START_NOALIGN(tramp_vectors)
1022
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
1023
+	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_LOOP
1024
+	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_FW
1025
+	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_INSN
1026
+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
1027
+	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_NONE
1028
+SYM_CODE_END(tramp_vectors)
10371029 
1038
-ENTRY(tramp_exit_native)
1030
+SYM_CODE_START(tramp_exit_native)
10391031 	tramp_exit
1040
-END(tramp_exit_native)
1032
+SYM_CODE_END(tramp_exit_native)
10411033 
1042
-ENTRY(tramp_exit_compat)
1034
+SYM_CODE_START(tramp_exit_compat)
10431035 	tramp_exit	32
1044
-END(tramp_exit_compat)
1036
+SYM_CODE_END(tramp_exit_compat)
10451037 
10461038 	.ltorg
10471039 	.popsection				// .entry.tramp.text
10481040 #ifdef CONFIG_RANDOMIZE_BASE
10491041 	.pushsection ".rodata", "a"
10501042 	.align PAGE_SHIFT
1051
-	.globl	__entry_tramp_data_start
1052
-__entry_tramp_data_start:
1043
+SYM_DATA_START(__entry_tramp_data_start)
1044
+__entry_tramp_data_vectors:
10531045 	.quad	vectors
1046
+#ifdef CONFIG_ARM_SDE_INTERFACE
1047
+__entry_tramp_data___sdei_asm_handler:
1048
+	.quad	__sdei_asm_handler
1049
+#endif /* CONFIG_ARM_SDE_INTERFACE */
1050
+__entry_tramp_data_this_cpu_vector:
1051
+	.quad	this_cpu_vector
1052
+SYM_DATA_END(__entry_tramp_data_start)
10541053 	.popsection				// .rodata
10551054 #endif /* CONFIG_RANDOMIZE_BASE */
10561055 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
1056
+
1057
+/*
1058
+ * Exception vectors for spectre mitigations on entry from EL1 when
1059
+ * kpti is not in use.
1060
+ */
1061
+	.macro generate_el1_vector, bhb
1062
+.Lvector_start\@:
1063
+	kernel_ventry	1, sync_invalid			// Synchronous EL1t
1064
+	kernel_ventry	1, irq_invalid			// IRQ EL1t
1065
+	kernel_ventry	1, fiq_invalid			// FIQ EL1t
1066
+	kernel_ventry	1, error_invalid		// Error EL1t
1067
+
1068
+	kernel_ventry	1, sync				// Synchronous EL1h
1069
+	kernel_ventry	1, irq				// IRQ EL1h
1070
+	kernel_ventry	1, fiq_invalid			// FIQ EL1h
1071
+	kernel_ventry	1, error			// Error EL1h
1072
+
1073
+	.rept	4
1074
+	tramp_ventry	.Lvector_start\@, 64, 0, \bhb
1075
+	.endr
1076
+	.rept 4
1077
+	tramp_ventry	.Lvector_start\@, 32, 0, \bhb
1078
+	.endr
1079
+	.endm
1080
+
1081
+/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */
1082
+	.pushsection ".entry.text", "ax"
1083
+	.align	11
1084
+SYM_CODE_START(__bp_harden_el1_vectors)
1085
+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
1086
+	generate_el1_vector	bhb=BHB_MITIGATION_LOOP
1087
+	generate_el1_vector	bhb=BHB_MITIGATION_FW
1088
+	generate_el1_vector	bhb=BHB_MITIGATION_INSN
1089
+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
1090
+SYM_CODE_END(__bp_harden_el1_vectors)
1091
+	.popsection
1092
+
10571093 
10581094 /*
10591095  * Register switch for AArch64. The callee-saved registers need to be saved
....@@ -1063,7 +1099,7 @@
10631099  * Previous and next are guaranteed not to be the same.
10641100  *
10651101  */
1066
-ENTRY(cpu_switch_to)
1102
+SYM_FUNC_START(cpu_switch_to)
10671103 	mov	x10, #THREAD_CPU_CONTEXT
10681104 	add	x8, x0, x10
10691105 	mov	x9, sp
....@@ -1084,26 +1120,24 @@
10841120 	ldr	lr, [x8]
10851121 	mov	sp, x9
10861122 	msr	sp_el0, x1
1087
-#ifdef CONFIG_SHADOW_CALL_STACK
1088
-	str	x18, [x0, #TSK_TI_SCS]
1089
-	ldr	x18, [x1, #TSK_TI_SCS]
1090
-	str	xzr, [x1, #TSK_TI_SCS]		// limit visibility of saved SCS
1091
-#endif
1123
+	ptrauth_keys_install_kernel x1, x8, x9, x10
1124
+	scs_save x0, x8
1125
+	scs_load x1, x8
10921126 	ret
1093
-ENDPROC(cpu_switch_to)
1127
+SYM_FUNC_END(cpu_switch_to)
10941128 NOKPROBE(cpu_switch_to)
10951129 
10961130 /*
10971131  * This is how we return from a fork.
10981132  */
1099
-ENTRY(ret_from_fork)
1133
+SYM_CODE_START(ret_from_fork)
11001134 	bl	schedule_tail
11011135 	cbz	x19, 1f				// not a kernel thread
11021136 	mov	x0, x20
11031137 	blr	x19
1104
-1:	get_thread_info tsk
1138
+1:	get_current_task tsk
11051139 	b	ret_to_user
1106
-ENDPROC(ret_from_fork)
1140
+SYM_CODE_END(ret_from_fork)
11071141 NOKPROBE(ret_from_fork)
11081142 
11091143 #ifdef CONFIG_ARM_SDE_INTERFACE
....@@ -1132,7 +1166,7 @@
11321166  */
11331167 .ltorg
11341168 .pushsection ".entry.tramp.text", "ax"
1135
-ENTRY(__sdei_asm_entry_trampoline)
1169
+SYM_CODE_START(__sdei_asm_entry_trampoline)
11361170 	mrs	x4, ttbr1_el1
11371171 	tbz	x4, #USER_ASID_BIT, 1f
11381172 
....@@ -1146,15 +1180,9 @@
11461180 	 */
11471181 1:	str	x4, [x1, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)]
11481182 
1149
-#ifdef CONFIG_RANDOMIZE_BASE
1150
-	adr	x4, tramp_vectors + PAGE_SIZE
1151
-	add	x4, x4, #:lo12:__sdei_asm_trampoline_next_handler
1152
-	ldr	x4, [x4]
1153
-#else
1154
-	ldr	x4, =__sdei_asm_handler
1155
-#endif
1183
+	tramp_data_read_var     x4, __sdei_asm_handler
11561184 	br	x4
1157
-ENDPROC(__sdei_asm_entry_trampoline)
1185
+SYM_CODE_END(__sdei_asm_entry_trampoline)
11581186 NOKPROBE(__sdei_asm_entry_trampoline)
11591187 
11601188 /*
....@@ -1164,23 +1192,17 @@
11641192  * x2: exit_mode
11651193  * x4: struct sdei_registered_event argument from registration time.
11661194  */
1167
-ENTRY(__sdei_asm_exit_trampoline)
1195
+SYM_CODE_START(__sdei_asm_exit_trampoline)
11681196 	ldr	x4, [x4, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)]
11691197 	cbnz	x4, 1f
11701198 
11711199 	tramp_unmap_kernel	tmp=x4
11721200 
11731201 1:	sdei_handler_exit exit_mode=x2
1174
-ENDPROC(__sdei_asm_exit_trampoline)
1202
+SYM_CODE_END(__sdei_asm_exit_trampoline)
11751203 NOKPROBE(__sdei_asm_exit_trampoline)
11761204 	.ltorg
11771205 .popsection		// .entry.tramp.text
1178
-#ifdef CONFIG_RANDOMIZE_BASE
1179
-.pushsection ".rodata", "a"
1180
-__sdei_asm_trampoline_next_handler:
1181
-	.quad	__sdei_asm_handler
1182
-.popsection		// .rodata
1183
-#endif /* CONFIG_RANDOMIZE_BASE */
11841206 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
11851207 
11861208 /*
....@@ -1196,7 +1218,7 @@
11961218  * follow SMC-CC. We save (or retrieve) all the registers as the handler may
11971219  * want them.
11981220  */
1199
-ENTRY(__sdei_asm_handler)
1221
+SYM_CODE_START(__sdei_asm_handler)
12001222 	stp     x2, x3, [x1, #SDEI_EVENT_INTREGS + S_PC]
12011223 	stp     x4, x5, [x1, #SDEI_EVENT_INTREGS + 16 * 2]
12021224 	stp     x6, x7, [x1, #SDEI_EVENT_INTREGS + 16 * 3]
....@@ -1216,13 +1238,16 @@
12161238 
12171239 	mov	x19, x1
12181240 
1241
+#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK)
1242
+	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
1243
+#endif
1244
+
12191245 #ifdef CONFIG_VMAP_STACK
12201246 	/*
12211247 	 * entry.S may have been using sp as a scratch register, find whether
12221248 	 * this is a normal or critical event and switch to the appropriate
12231249 	 * stack for this CPU.
12241250 	 */
1225
-	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
12261251 	cbnz	w4, 1f
12271252 	ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
12281253 	b	2f
....@@ -1230,6 +1255,15 @@
12301255 2:	mov	x6, #SDEI_STACK_SIZE
12311256 	add	x5, x5, x6
12321257 	mov	sp, x5
1258
+#endif
1259
+
1260
+#ifdef CONFIG_SHADOW_CALL_STACK
1261
+	/* Use a separate shadow call stack for normal and critical events */
1262
+	cbnz	w4, 3f
1263
+	ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6
1264
+	b	4f
1265
+3:	ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6
1266
+4:
12331267 #endif
12341268 
12351269 	/*
....@@ -1276,9 +1310,9 @@
12761310 alternative_else_nop_endif
12771311 
12781312 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
1279
-	tramp_alias	dst=x5, sym=__sdei_asm_exit_trampoline
1313
+	tramp_alias	dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3
12801314 	br	x5
12811315 #endif
1282
-ENDPROC(__sdei_asm_handler)
1316
+SYM_CODE_END(__sdei_asm_handler)
12831317 NOKPROBE(__sdei_asm_handler)
12841318 #endif /* CONFIG_ARM_SDE_INTERFACE */