.. | .. |
---|
| 1 | +/* SPDX-License-Identifier: GPL-2.0-only */ |
---|
1 | 2 | /* |
---|
2 | 3 | * Low-level exception handling code |
---|
3 | 4 | * |
---|
4 | 5 | * Copyright (C) 2012 ARM Ltd. |
---|
5 | 6 | * Authors: Catalin Marinas <catalin.marinas@arm.com> |
---|
6 | 7 | * Will Deacon <will.deacon@arm.com> |
---|
7 | | - * |
---|
8 | | - * This program is free software; you can redistribute it and/or modify |
---|
9 | | - * it under the terms of the GNU General Public License version 2 as |
---|
10 | | - * published by the Free Software Foundation. |
---|
11 | | - * |
---|
12 | | - * This program is distributed in the hope that it will be useful, |
---|
13 | | - * but WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
14 | | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
---|
15 | | - * GNU General Public License for more details. |
---|
16 | | - * |
---|
17 | | - * You should have received a copy of the GNU General Public License |
---|
18 | | - * along with this program. If not, see <http://www.gnu.org/licenses/>. |
---|
19 | 8 | */ |
---|
20 | 9 | |
---|
21 | 10 | #include <linux/arm-smccc.h> |
---|
.. | .. |
---|
25 | 14 | #include <asm/alternative.h> |
---|
26 | 15 | #include <asm/assembler.h> |
---|
27 | 16 | #include <asm/asm-offsets.h> |
---|
| 17 | +#include <asm/asm_pointer_auth.h> |
---|
| 18 | +#include <asm/bug.h> |
---|
28 | 19 | #include <asm/cpufeature.h> |
---|
29 | 20 | #include <asm/errno.h> |
---|
30 | 21 | #include <asm/esr.h> |
---|
.. | .. |
---|
33 | 24 | #include <asm/mmu.h> |
---|
34 | 25 | #include <asm/processor.h> |
---|
35 | 26 | #include <asm/ptrace.h> |
---|
| 27 | +#include <asm/scs.h> |
---|
36 | 28 | #include <asm/thread_info.h> |
---|
37 | 29 | #include <asm/asm-uaccess.h> |
---|
38 | 30 | #include <asm/unistd.h> |
---|
39 | 31 | |
---|
40 | 32 | /* |
---|
41 | | - * Context tracking subsystem. Used to instrument transitions |
---|
42 | | - * between user and kernel mode. |
---|
| 33 | + * Context tracking and irqflag tracing need to instrument transitions between |
---|
| 34 | + * user and kernel mode. |
---|
43 | 35 | */ |
---|
44 | | - .macro ct_user_exit |
---|
45 | | -#ifdef CONFIG_CONTEXT_TRACKING |
---|
46 | | - bl context_tracking_user_exit |
---|
| 36 | + .macro user_exit_irqoff |
---|
| 37 | +#if defined(CONFIG_CONTEXT_TRACKING) || defined(CONFIG_TRACE_IRQFLAGS) |
---|
| 38 | + bl enter_from_user_mode |
---|
47 | 39 | #endif |
---|
48 | 40 | .endm |
---|
49 | 41 | |
---|
50 | | - .macro ct_user_enter |
---|
51 | | -#ifdef CONFIG_CONTEXT_TRACKING |
---|
52 | | - bl context_tracking_user_enter |
---|
| 42 | + .macro user_enter_irqoff |
---|
| 43 | +#if defined(CONFIG_CONTEXT_TRACKING) || defined(CONFIG_TRACE_IRQFLAGS) |
---|
| 44 | + bl exit_to_user_mode |
---|
53 | 45 | #endif |
---|
54 | 46 | .endm |
---|
55 | 47 | |
---|
.. | .. |
---|
70 | 62 | |
---|
71 | 63 | .macro kernel_ventry, el, label, regsize = 64 |
---|
72 | 64 | .align 7 |
---|
73 | | -#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
---|
74 | | -alternative_if ARM64_UNMAP_KERNEL_AT_EL0 |
---|
| 65 | +.Lventry_start\@: |
---|
75 | 66 | .if \el == 0 |
---|
| 67 | + /* |
---|
| 68 | + * This must be the first instruction of the EL0 vector entries. It is |
---|
| 69 | + * skipped by the trampoline vectors, to trigger the cleanup. |
---|
| 70 | + */ |
---|
| 71 | + b .Lskip_tramp_vectors_cleanup\@ |
---|
76 | 72 | .if \regsize == 64 |
---|
77 | 73 | mrs x30, tpidrro_el0 |
---|
78 | 74 | msr tpidrro_el0, xzr |
---|
79 | 75 | .else |
---|
80 | 76 | mov x30, xzr |
---|
81 | 77 | .endif |
---|
| 78 | +.Lskip_tramp_vectors_cleanup\@: |
---|
82 | 79 | .endif |
---|
83 | | -alternative_else_nop_endif |
---|
84 | | -#endif |
---|
85 | 80 | |
---|
86 | 81 | sub sp, sp, #S_FRAME_SIZE |
---|
87 | 82 | #ifdef CONFIG_VMAP_STACK |
---|
88 | 83 | /* |
---|
89 | 84 | * Test whether the SP has overflowed, without corrupting a GPR. |
---|
90 | | - * Task and IRQ stacks are aligned to (1 << THREAD_SHIFT). |
---|
| 85 | + * Task and IRQ stacks are aligned so that SP & (1 << THREAD_SHIFT) |
---|
| 86 | + * should always be zero. |
---|
91 | 87 | */ |
---|
92 | 88 | add sp, sp, x0 // sp' = sp + x0 |
---|
93 | 89 | sub x0, sp, x0 // x0' = sp' - x0 = (sp + x0) - x0 = sp |
---|
.. | .. |
---|
127 | 123 | mrs x0, tpidrro_el0 |
---|
128 | 124 | #endif |
---|
129 | 125 | b el\()\el\()_\label |
---|
| 126 | +.org .Lventry_start\@ + 128 // Did we overflow the ventry slot? |
---|
130 | 127 | .endm |
---|
131 | 128 | |
---|
132 | | - .macro tramp_alias, dst, sym |
---|
| 129 | + .macro tramp_alias, dst, sym, tmp |
---|
133 | 130 | mov_q \dst, TRAMP_VALIAS |
---|
134 | | - add \dst, \dst, #(\sym - .entry.tramp.text) |
---|
| 131 | + adr_l \tmp, \sym |
---|
| 132 | + add \dst, \dst, \tmp |
---|
| 133 | + adr_l \tmp, .entry.tramp.text |
---|
| 134 | + sub \dst, \dst, \tmp |
---|
135 | 135 | .endm |
---|
136 | 136 | |
---|
137 | | - // This macro corrupts x0-x3. It is the caller's duty |
---|
138 | | - // to save/restore them if required. |
---|
| 137 | + /* |
---|
| 138 | + * This macro corrupts x0-x3. It is the caller's duty to save/restore |
---|
| 139 | + * them if required. |
---|
| 140 | + */ |
---|
139 | 141 | .macro apply_ssbd, state, tmp1, tmp2 |
---|
140 | | -#ifdef CONFIG_ARM64_SSBD |
---|
141 | | -alternative_cb arm64_enable_wa2_handling |
---|
142 | | - b .L__asm_ssbd_skip\@ |
---|
| 142 | +alternative_cb spectre_v4_patch_fw_mitigation_enable |
---|
| 143 | + b .L__asm_ssbd_skip\@ // Patched to NOP |
---|
143 | 144 | alternative_cb_end |
---|
144 | 145 | ldr_this_cpu \tmp2, arm64_ssbd_callback_required, \tmp1 |
---|
145 | 146 | cbz \tmp2, .L__asm_ssbd_skip\@ |
---|
.. | .. |
---|
147 | 148 | tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@ |
---|
148 | 149 | mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2 |
---|
149 | 150 | mov w1, #\state |
---|
150 | | -alternative_cb arm64_update_smccc_conduit |
---|
| 151 | +alternative_cb smccc_patch_fw_mitigation_conduit |
---|
151 | 152 | nop // Patched to SMC/HVC #0 |
---|
152 | 153 | alternative_cb_end |
---|
153 | 154 | .L__asm_ssbd_skip\@: |
---|
| 155 | + .endm |
---|
| 156 | + |
---|
| 157 | + /* Check for MTE asynchronous tag check faults */ |
---|
| 158 | + .macro check_mte_async_tcf, tmp, ti_flags, thread_sctlr |
---|
| 159 | +#ifdef CONFIG_ARM64_MTE |
---|
| 160 | + .arch_extension lse |
---|
| 161 | +alternative_if_not ARM64_MTE |
---|
| 162 | + b 1f |
---|
| 163 | +alternative_else_nop_endif |
---|
| 164 | + /* |
---|
| 165 | + * Asynchronous tag check faults are only possible in ASYNC (2) or |
---|
| 166 | + * ASYM (3) modes. In each of these modes bit 1 of SCTLR_EL1.TCF0 is |
---|
| 167 | + * set, so skip the check if it is unset. |
---|
| 168 | + */ |
---|
| 169 | + tbz \thread_sctlr, #(SCTLR_EL1_TCF0_SHIFT + 1), 1f |
---|
| 170 | + mrs_s \tmp, SYS_TFSRE0_EL1 |
---|
| 171 | + tbz \tmp, #SYS_TFSR_EL1_TF0_SHIFT, 1f |
---|
| 172 | + /* Asynchronous TCF occurred for TTBR0 access, set the TI flag */ |
---|
| 173 | + mov \tmp, #_TIF_MTE_ASYNC_FAULT |
---|
| 174 | + add \ti_flags, tsk, #TSK_TI_FLAGS |
---|
| 175 | + stset \tmp, [\ti_flags] |
---|
| 176 | +1: |
---|
| 177 | +#endif |
---|
| 178 | + .endm |
---|
| 179 | + |
---|
| 180 | + /* Clear the MTE asynchronous tag check faults */ |
---|
| 181 | + .macro clear_mte_async_tcf thread_sctlr |
---|
| 182 | +#ifdef CONFIG_ARM64_MTE |
---|
| 183 | +alternative_if ARM64_MTE |
---|
| 184 | + /* See comment in check_mte_async_tcf above. */ |
---|
| 185 | + tbz \thread_sctlr, #(SCTLR_EL1_TCF0_SHIFT + 1), 1f |
---|
| 186 | + dsb ish |
---|
| 187 | + msr_s SYS_TFSRE0_EL1, xzr |
---|
| 188 | +1: |
---|
| 189 | +alternative_else_nop_endif |
---|
| 190 | +#endif |
---|
| 191 | + .endm |
---|
| 192 | + |
---|
| 193 | + .macro mte_set_gcr, mte_ctrl, tmp |
---|
| 194 | +#ifdef CONFIG_ARM64_MTE |
---|
| 195 | + ubfx \tmp, \mte_ctrl, #MTE_CTRL_GCR_USER_EXCL_SHIFT, #16 |
---|
| 196 | + orr \tmp, \tmp, #SYS_GCR_EL1_RRND |
---|
| 197 | + msr_s SYS_GCR_EL1, \tmp |
---|
| 198 | +#endif |
---|
| 199 | + .endm |
---|
| 200 | + |
---|
| 201 | + .macro mte_set_kernel_gcr, tmp, tmp2 |
---|
| 202 | +#ifdef CONFIG_KASAN_HW_TAGS |
---|
| 203 | +alternative_cb kasan_hw_tags_enable |
---|
| 204 | + b 1f |
---|
| 205 | +alternative_cb_end |
---|
| 206 | + mov \tmp, KERNEL_GCR_EL1 |
---|
| 207 | + msr_s SYS_GCR_EL1, \tmp |
---|
| 208 | +1: |
---|
| 209 | +#endif |
---|
| 210 | + .endm |
---|
| 211 | + |
---|
| 212 | + .macro mte_set_user_gcr, tsk, tmp, tmp2 |
---|
| 213 | +#ifdef CONFIG_KASAN_HW_TAGS |
---|
| 214 | +alternative_cb kasan_hw_tags_enable |
---|
| 215 | + b 1f |
---|
| 216 | +alternative_cb_end |
---|
| 217 | + ldr \tmp, [\tsk, #THREAD_MTE_CTRL] |
---|
| 218 | + |
---|
| 219 | + mte_set_gcr \tmp, \tmp2 |
---|
| 220 | +1: |
---|
154 | 221 | #endif |
---|
155 | 222 | .endm |
---|
156 | 223 | |
---|
.. | .. |
---|
177 | 244 | .if \el == 0 |
---|
178 | 245 | clear_gp_regs |
---|
179 | 246 | mrs x21, sp_el0 |
---|
180 | | - ldr_this_cpu tsk, __entry_task, x20 // Ensure MDSCR_EL1.SS is clear, |
---|
181 | | - ldr x19, [tsk, #TSK_TI_FLAGS] // since we can unmask debug |
---|
182 | | - disable_step_tsk x19, x20 // exceptions when scheduling. |
---|
| 247 | + ldr_this_cpu tsk, __entry_task, x20 |
---|
| 248 | + msr sp_el0, tsk |
---|
| 249 | + |
---|
| 250 | + /* |
---|
| 251 | + * Ensure MDSCR_EL1.SS is clear, since we can unmask debug exceptions |
---|
| 252 | + * when scheduling. |
---|
| 253 | + */ |
---|
| 254 | + ldr x19, [tsk, #TSK_TI_FLAGS] |
---|
| 255 | + disable_step_tsk x19, x20 |
---|
| 256 | + |
---|
| 257 | + /* Check for asynchronous tag check faults in user space */ |
---|
| 258 | + ldr x0, [tsk, THREAD_SCTLR_USER] |
---|
| 259 | + check_mte_async_tcf x22, x23, x0 |
---|
| 260 | + |
---|
| 261 | +#ifdef CONFIG_ARM64_PTR_AUTH |
---|
| 262 | +alternative_if ARM64_HAS_ADDRESS_AUTH |
---|
| 263 | + /* |
---|
| 264 | + * Enable IA for in-kernel PAC if the task had it disabled. Although |
---|
| 265 | + * this could be implemented with an unconditional MRS which would avoid |
---|
| 266 | + * a load, this was measured to be slower on Cortex-A75 and Cortex-A76. |
---|
| 267 | + * |
---|
| 268 | + * Install the kernel IA key only if IA was enabled in the task. If IA |
---|
| 269 | + * was disabled on kernel exit then we would have left the kernel IA |
---|
| 270 | + * installed so there is no need to install it again. |
---|
| 271 | + */ |
---|
| 272 | + tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f |
---|
| 273 | + __ptrauth_keys_install_kernel_nosync tsk, x20, x22, x23 |
---|
| 274 | + b 2f |
---|
| 275 | +1: |
---|
| 276 | + mrs x0, sctlr_el1 |
---|
| 277 | + orr x0, x0, SCTLR_ELx_ENIA |
---|
| 278 | + msr sctlr_el1, x0 |
---|
| 279 | +2: |
---|
| 280 | +alternative_else_nop_endif |
---|
| 281 | +#endif |
---|
183 | 282 | |
---|
184 | 283 | apply_ssbd 1, x22, x23 |
---|
185 | 284 | |
---|
186 | | -#ifdef CONFIG_SHADOW_CALL_STACK |
---|
187 | | - ldr x18, [tsk, #TSK_TI_SCS] // Restore shadow call stack |
---|
188 | | - str xzr, [tsk, #TSK_TI_SCS] // Limit visibility of saved SCS |
---|
189 | | -#endif |
---|
| 285 | + mte_set_kernel_gcr x22, x23 |
---|
| 286 | + |
---|
| 287 | + /* |
---|
| 288 | + * Any non-self-synchronizing system register updates required for |
---|
| 289 | + * kernel entry should be placed before this point. |
---|
| 290 | + */ |
---|
| 291 | +alternative_if ARM64_MTE |
---|
| 292 | + isb |
---|
| 293 | + b 1f |
---|
| 294 | +alternative_else_nop_endif |
---|
| 295 | +alternative_if ARM64_HAS_ADDRESS_AUTH |
---|
| 296 | + isb |
---|
| 297 | +alternative_else_nop_endif |
---|
| 298 | +1: |
---|
| 299 | + |
---|
| 300 | + scs_load tsk, x20 |
---|
190 | 301 | .else |
---|
191 | 302 | add x21, sp, #S_FRAME_SIZE |
---|
192 | | - get_thread_info tsk |
---|
| 303 | + get_current_task tsk |
---|
193 | 304 | /* Save the task's original addr_limit and set USER_DS */ |
---|
194 | 305 | ldr x20, [tsk, #TSK_TI_ADDR_LIMIT] |
---|
195 | 306 | str x20, [sp, #S_ORIG_ADDR_LIMIT] |
---|
.. | .. |
---|
214 | 325 | add x29, sp, #S_STACKFRAME |
---|
215 | 326 | |
---|
216 | 327 | #ifdef CONFIG_ARM64_SW_TTBR0_PAN |
---|
217 | | - /* |
---|
218 | | - * Set the TTBR0 PAN bit in SPSR. When the exception is taken from |
---|
219 | | - * EL0, there is no need to check the state of TTBR0_EL1 since |
---|
220 | | - * accesses are always enabled. |
---|
221 | | - * Note that the meaning of this bit differs from the ARMv8.1 PAN |
---|
222 | | - * feature as all TTBR0_EL1 accesses are disabled, not just those to |
---|
223 | | - * user mappings. |
---|
224 | | - */ |
---|
225 | | -alternative_if ARM64_HAS_PAN |
---|
226 | | - b 1f // skip TTBR0 PAN |
---|
| 328 | +alternative_if_not ARM64_HAS_PAN |
---|
| 329 | + bl __swpan_entry_el\el |
---|
227 | 330 | alternative_else_nop_endif |
---|
228 | | - |
---|
229 | | - .if \el != 0 |
---|
230 | | - mrs x21, ttbr0_el1 |
---|
231 | | - tst x21, #TTBR_ASID_MASK // Check for the reserved ASID |
---|
232 | | - orr x23, x23, #PSR_PAN_BIT // Set the emulated PAN in the saved SPSR |
---|
233 | | - b.eq 1f // TTBR0 access already disabled |
---|
234 | | - and x23, x23, #~PSR_PAN_BIT // Clear the emulated PAN in the saved SPSR |
---|
235 | | - .endif |
---|
236 | | - |
---|
237 | | - __uaccess_ttbr0_disable x21 |
---|
238 | | -1: |
---|
239 | 331 | #endif |
---|
240 | 332 | |
---|
241 | 333 | stp x22, x23, [sp, #S_PC] |
---|
.. | .. |
---|
246 | 338 | str w21, [sp, #S_SYSCALLNO] |
---|
247 | 339 | .endif |
---|
248 | 340 | |
---|
249 | | - /* |
---|
250 | | - * Set sp_el0 to current thread_info. |
---|
251 | | - */ |
---|
252 | | - .if \el == 0 |
---|
253 | | - msr sp_el0, tsk |
---|
254 | | - .endif |
---|
| 341 | + /* Save pmr */ |
---|
| 342 | +alternative_if ARM64_HAS_IRQ_PRIO_MASKING |
---|
| 343 | + mrs_s x20, SYS_ICC_PMR_EL1 |
---|
| 344 | + str x20, [sp, #S_PMR_SAVE] |
---|
| 345 | + mov x20, #GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET |
---|
| 346 | + msr_s SYS_ICC_PMR_EL1, x20 |
---|
| 347 | +alternative_else_nop_endif |
---|
| 348 | + |
---|
| 349 | + /* Re-enable tag checking (TCO set on exception entry) */ |
---|
| 350 | +#ifdef CONFIG_ARM64_MTE |
---|
| 351 | +alternative_if ARM64_MTE |
---|
| 352 | + SET_PSTATE_TCO(0) |
---|
| 353 | +alternative_else_nop_endif |
---|
| 354 | +#endif |
---|
255 | 355 | |
---|
256 | 356 | /* |
---|
257 | 357 | * Registers that may be useful after this macro is invoked: |
---|
258 | 358 | * |
---|
| 359 | + * x20 - ICC_PMR_EL1 |
---|
259 | 360 | * x21 - aborted SP |
---|
260 | 361 | * x22 - aborted PC |
---|
261 | 362 | * x23 - aborted PSTATE |
---|
.. | .. |
---|
273 | 374 | /* No need to restore UAO, it will be restored from SPSR_EL1 */ |
---|
274 | 375 | .endif |
---|
275 | 376 | |
---|
276 | | - ldp x21, x22, [sp, #S_PC] // load ELR, SPSR |
---|
277 | | - .if \el == 0 |
---|
278 | | - ct_user_enter |
---|
279 | | - .endif |
---|
280 | | - |
---|
281 | | -#ifdef CONFIG_SHADOW_CALL_STACK |
---|
282 | | - .if \el == 0 |
---|
283 | | - str x18, [tsk, #TSK_TI_SCS] // Save shadow call stack |
---|
284 | | - .endif |
---|
285 | | -#endif |
---|
286 | | - |
---|
287 | | -#ifdef CONFIG_ARM64_SW_TTBR0_PAN |
---|
288 | | - /* |
---|
289 | | - * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR |
---|
290 | | - * PAN bit checking. |
---|
291 | | - */ |
---|
292 | | -alternative_if ARM64_HAS_PAN |
---|
293 | | - b 2f // skip TTBR0 PAN |
---|
| 377 | + /* Restore pmr */ |
---|
| 378 | +alternative_if ARM64_HAS_IRQ_PRIO_MASKING |
---|
| 379 | + ldr x20, [sp, #S_PMR_SAVE] |
---|
| 380 | + msr_s SYS_ICC_PMR_EL1, x20 |
---|
| 381 | + mrs_s x21, SYS_ICC_CTLR_EL1 |
---|
| 382 | + tbz x21, #6, .L__skip_pmr_sync\@ // Check for ICC_CTLR_EL1.PMHE |
---|
| 383 | + dsb sy // Ensure priority change is seen by redistributor |
---|
| 384 | +.L__skip_pmr_sync\@: |
---|
294 | 385 | alternative_else_nop_endif |
---|
295 | 386 | |
---|
296 | | - .if \el != 0 |
---|
297 | | - tbnz x22, #22, 1f // Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set |
---|
298 | | - .endif |
---|
| 387 | + ldp x21, x22, [sp, #S_PC] // load ELR, SPSR |
---|
299 | 388 | |
---|
300 | | - __uaccess_ttbr0_enable x0, x1 |
---|
301 | | - |
---|
302 | | - .if \el == 0 |
---|
303 | | - /* |
---|
304 | | - * Enable errata workarounds only if returning to user. The only |
---|
305 | | - * workaround currently required for TTBR0_EL1 changes are for the |
---|
306 | | - * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache |
---|
307 | | - * corruption). |
---|
308 | | - */ |
---|
309 | | - bl post_ttbr_update_workaround |
---|
310 | | - .endif |
---|
311 | | -1: |
---|
312 | | - .if \el != 0 |
---|
313 | | - and x22, x22, #~PSR_PAN_BIT // ARMv8.0 CPUs do not understand this bit |
---|
314 | | - .endif |
---|
315 | | -2: |
---|
| 389 | +#ifdef CONFIG_ARM64_SW_TTBR0_PAN |
---|
| 390 | +alternative_if_not ARM64_HAS_PAN |
---|
| 391 | + bl __swpan_exit_el\el |
---|
| 392 | +alternative_else_nop_endif |
---|
316 | 393 | #endif |
---|
317 | 394 | |
---|
318 | 395 | .if \el == 0 |
---|
.. | .. |
---|
332 | 409 | alternative_else_nop_endif |
---|
333 | 410 | #endif |
---|
334 | 411 | 3: |
---|
| 412 | + scs_save tsk, x0 |
---|
| 413 | + |
---|
| 414 | + /* Ignore asynchronous tag check faults in the uaccess routines */ |
---|
| 415 | + ldr x0, [tsk, THREAD_SCTLR_USER] |
---|
| 416 | + clear_mte_async_tcf x0 |
---|
| 417 | + |
---|
| 418 | +#ifdef CONFIG_ARM64_PTR_AUTH |
---|
| 419 | +alternative_if ARM64_HAS_ADDRESS_AUTH |
---|
| 420 | + /* |
---|
| 421 | + * IA was enabled for in-kernel PAC. Disable it now if needed, or |
---|
| 422 | + * alternatively install the user's IA. All other per-task keys and |
---|
| 423 | + * SCTLR bits were updated on task switch. |
---|
| 424 | + * |
---|
| 425 | + * No kernel C function calls after this. |
---|
| 426 | + */ |
---|
| 427 | + tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f |
---|
| 428 | + __ptrauth_keys_install_user tsk, x0, x1, x2 |
---|
| 429 | + b 2f |
---|
| 430 | +1: |
---|
| 431 | + mrs x0, sctlr_el1 |
---|
| 432 | + bic x0, x0, SCTLR_ELx_ENIA |
---|
| 433 | + msr sctlr_el1, x0 |
---|
| 434 | +2: |
---|
| 435 | +alternative_else_nop_endif |
---|
| 436 | +#endif |
---|
| 437 | + |
---|
| 438 | + mte_set_user_gcr tsk, x0, x1 |
---|
| 439 | + |
---|
335 | 440 | apply_ssbd 0, x0, x1 |
---|
336 | 441 | .endif |
---|
337 | 442 | |
---|
.. | .. |
---|
352 | 457 | ldp x24, x25, [sp, #16 * 12] |
---|
353 | 458 | ldp x26, x27, [sp, #16 * 13] |
---|
354 | 459 | ldp x28, x29, [sp, #16 * 14] |
---|
355 | | - ldr lr, [sp, #S_LR] |
---|
356 | | - add sp, sp, #S_FRAME_SIZE // restore sp |
---|
357 | | - /* |
---|
358 | | - * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on eret context synchronization |
---|
359 | | - * when returning from IPI handler, and when returning to user-space. |
---|
360 | | - */ |
---|
361 | 460 | |
---|
362 | 461 | .if \el == 0 |
---|
363 | | -alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 |
---|
| 462 | +alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 |
---|
| 463 | + ldr lr, [sp, #S_LR] |
---|
| 464 | + add sp, sp, #S_FRAME_SIZE // restore sp |
---|
| 465 | + eret |
---|
| 466 | +alternative_else_nop_endif |
---|
364 | 467 | #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
---|
365 | 468 | bne 4f |
---|
366 | | - msr far_el1, x30 |
---|
367 | | - tramp_alias x30, tramp_exit_native |
---|
| 469 | + msr far_el1, x29 |
---|
| 470 | + tramp_alias x30, tramp_exit_native, x29 |
---|
368 | 471 | br x30 |
---|
369 | 472 | 4: |
---|
370 | | - tramp_alias x30, tramp_exit_compat |
---|
| 473 | + tramp_alias x30, tramp_exit_compat, x29 |
---|
371 | 474 | br x30 |
---|
372 | 475 | #endif |
---|
373 | 476 | .else |
---|
| 477 | + ldr lr, [sp, #S_LR] |
---|
| 478 | + add sp, sp, #S_FRAME_SIZE // restore sp |
---|
| 479 | + |
---|
| 480 | + /* Ensure any device/NC reads complete */ |
---|
| 481 | + alternative_insn nop, "dmb sy", ARM64_WORKAROUND_1508412 |
---|
| 482 | + |
---|
374 | 483 | eret |
---|
375 | 484 | .endif |
---|
376 | 485 | sb |
---|
377 | 486 | .endm |
---|
378 | 487 | |
---|
| 488 | +#ifdef CONFIG_ARM64_SW_TTBR0_PAN |
---|
| 489 | + /* |
---|
| 490 | + * Set the TTBR0 PAN bit in SPSR. When the exception is taken from |
---|
| 491 | + * EL0, there is no need to check the state of TTBR0_EL1 since |
---|
| 492 | + * accesses are always enabled. |
---|
| 493 | + * Note that the meaning of this bit differs from the ARMv8.1 PAN |
---|
| 494 | + * feature as all TTBR0_EL1 accesses are disabled, not just those to |
---|
| 495 | + * user mappings. |
---|
| 496 | + */ |
---|
| 497 | +SYM_CODE_START_LOCAL(__swpan_entry_el1) |
---|
| 498 | + mrs x21, ttbr0_el1 |
---|
| 499 | + tst x21, #TTBR_ASID_MASK // Check for the reserved ASID |
---|
| 500 | + orr x23, x23, #PSR_PAN_BIT // Set the emulated PAN in the saved SPSR |
---|
| 501 | + b.eq 1f // TTBR0 access already disabled |
---|
| 502 | + and x23, x23, #~PSR_PAN_BIT // Clear the emulated PAN in the saved SPSR |
---|
| 503 | +SYM_INNER_LABEL(__swpan_entry_el0, SYM_L_LOCAL) |
---|
| 504 | + __uaccess_ttbr0_disable x21 |
---|
| 505 | +1: ret |
---|
| 506 | +SYM_CODE_END(__swpan_entry_el1) |
---|
| 507 | + |
---|
| 508 | + /* |
---|
| 509 | + * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR |
---|
| 510 | + * PAN bit checking. |
---|
| 511 | + */ |
---|
| 512 | +SYM_CODE_START_LOCAL(__swpan_exit_el1) |
---|
| 513 | + tbnz x22, #22, 1f // Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set |
---|
| 514 | + __uaccess_ttbr0_enable x0, x1 |
---|
| 515 | +1: and x22, x22, #~PSR_PAN_BIT // ARMv8.0 CPUs do not understand this bit |
---|
| 516 | + ret |
---|
| 517 | +SYM_CODE_END(__swpan_exit_el1) |
---|
| 518 | + |
---|
| 519 | +SYM_CODE_START_LOCAL(__swpan_exit_el0) |
---|
| 520 | + __uaccess_ttbr0_enable x0, x1 |
---|
| 521 | + /* |
---|
| 522 | + * Enable errata workarounds only if returning to user. The only |
---|
| 523 | + * workaround currently required for TTBR0_EL1 changes are for the |
---|
| 524 | + * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache |
---|
| 525 | + * corruption). |
---|
| 526 | + */ |
---|
| 527 | + b post_ttbr_update_workaround |
---|
| 528 | +SYM_CODE_END(__swpan_exit_el0) |
---|
| 529 | +#endif |
---|
| 530 | + |
---|
379 | 531 | .macro irq_stack_entry |
---|
380 | 532 | mov x19, sp // preserve the original sp |
---|
381 | 533 | #ifdef CONFIG_SHADOW_CALL_STACK |
---|
382 | | - mov x20, x18 // preserve the original shadow stack |
---|
| 534 | + mov x24, scs_sp // preserve the original shadow stack |
---|
383 | 535 | #endif |
---|
384 | 536 | |
---|
385 | 537 | /* |
---|
.. | .. |
---|
401 | 553 | |
---|
402 | 554 | #ifdef CONFIG_SHADOW_CALL_STACK |
---|
403 | 555 | /* also switch to the irq shadow stack */ |
---|
404 | | - ldr_this_cpu x18, irq_shadow_call_stack_ptr, x26 |
---|
| 556 | + ldr_this_cpu scs_sp, irq_shadow_call_stack_ptr, x26 |
---|
405 | 557 | #endif |
---|
406 | 558 | |
---|
407 | 559 | 9998: |
---|
.. | .. |
---|
409 | 561 | |
---|
410 | 562 | /* |
---|
411 | 563 | * The callee-saved regs (x19-x29) should be preserved between |
---|
412 | | - * irq_stack_entry and irq_stack_exit. |
---|
| 564 | + * irq_stack_entry and irq_stack_exit, but note that kernel_entry |
---|
| 565 | + * uses x20-x23 to store data for later use. |
---|
413 | 566 | */ |
---|
414 | 567 | .macro irq_stack_exit |
---|
415 | 568 | mov sp, x19 |
---|
416 | 569 | #ifdef CONFIG_SHADOW_CALL_STACK |
---|
417 | | - mov x18, x20 |
---|
| 570 | + mov scs_sp, x24 |
---|
418 | 571 | #endif |
---|
419 | 572 | .endm |
---|
420 | 573 | |
---|
421 | | -/* |
---|
422 | | - * These are the registers used in the syscall handler, and allow us to |
---|
423 | | - * have in theory up to 7 arguments to a function - x0 to x6. |
---|
424 | | - * |
---|
425 | | - * x7 is reserved for the system call number in 32-bit mode. |
---|
426 | | - */ |
---|
427 | | -wsc_nr .req w25 // number of system calls |
---|
428 | | -xsc_nr .req x25 // number of system calls (zero-extended) |
---|
429 | | -wscno .req w26 // syscall number |
---|
430 | | -xscno .req x26 // syscall number (zero-extended) |
---|
431 | | -stbl .req x27 // syscall table pointer |
---|
| 574 | +/* GPRs used by entry code */ |
---|
432 | 575 | tsk .req x28 // current thread_info |
---|
433 | 576 | |
---|
434 | 577 | /* |
---|
435 | 578 | * Interrupt handling. |
---|
436 | 579 | */ |
---|
437 | | - .macro irq_handler |
---|
438 | | - ldr_l x1, handle_arch_irq |
---|
| 580 | + .macro irq_handler, handler:req |
---|
| 581 | + ldr_l x1, \handler |
---|
439 | 582 | mov x0, sp |
---|
440 | 583 | irq_stack_entry |
---|
441 | 584 | blr x1 |
---|
442 | 585 | irq_stack_exit |
---|
| 586 | + .endm |
---|
| 587 | + |
---|
| 588 | +#ifdef CONFIG_ARM64_PSEUDO_NMI |
---|
| 589 | + /* |
---|
| 590 | + * Set res to 0 if irqs were unmasked in interrupted context. |
---|
| 591 | + * Otherwise set res to non-0 value. |
---|
| 592 | + */ |
---|
| 593 | + .macro test_irqs_unmasked res:req, pmr:req |
---|
| 594 | +alternative_if ARM64_HAS_IRQ_PRIO_MASKING |
---|
| 595 | + sub \res, \pmr, #GIC_PRIO_IRQON |
---|
| 596 | +alternative_else |
---|
| 597 | + mov \res, xzr |
---|
| 598 | +alternative_endif |
---|
| 599 | + .endm |
---|
| 600 | +#endif |
---|
| 601 | + |
---|
| 602 | + .macro gic_prio_kentry_setup, tmp:req |
---|
| 603 | +#ifdef CONFIG_ARM64_PSEUDO_NMI |
---|
| 604 | + alternative_if ARM64_HAS_IRQ_PRIO_MASKING |
---|
| 605 | + mov \tmp, #(GIC_PRIO_PSR_I_SET | GIC_PRIO_IRQON) |
---|
| 606 | + msr_s SYS_ICC_PMR_EL1, \tmp |
---|
| 607 | + alternative_else_nop_endif |
---|
| 608 | +#endif |
---|
| 609 | + .endm |
---|
| 610 | + |
---|
| 611 | + .macro el1_interrupt_handler, handler:req |
---|
| 612 | + enable_da_f |
---|
| 613 | + |
---|
| 614 | + mov x0, sp |
---|
| 615 | + bl enter_el1_irq_or_nmi |
---|
| 616 | + |
---|
| 617 | + irq_handler \handler |
---|
| 618 | + |
---|
| 619 | +#ifdef CONFIG_PREEMPTION |
---|
| 620 | + ldr x24, [tsk, #TSK_TI_PREEMPT] // get preempt count |
---|
| 621 | +alternative_if ARM64_HAS_IRQ_PRIO_MASKING |
---|
| 622 | + /* |
---|
| 623 | + * DA_F were cleared at start of handling. If anything is set in DAIF, |
---|
| 624 | + * we come back from an NMI, so skip preemption |
---|
| 625 | + */ |
---|
| 626 | + mrs x0, daif |
---|
| 627 | + orr x24, x24, x0 |
---|
| 628 | +alternative_else_nop_endif |
---|
| 629 | + |
---|
| 630 | + cbz x24, 1f // (need_resched + count) == 0 |
---|
| 631 | + cbnz w24, 2f // count != 0 |
---|
| 632 | + |
---|
| 633 | + ldr w24, [tsk, #TSK_TI_PREEMPT_LAZY] // get preempt lazy count |
---|
| 634 | + cbnz w24, 2f // preempt lazy count != 0 |
---|
| 635 | + |
---|
| 636 | + ldr x0, [tsk, #TSK_TI_FLAGS] // get flags |
---|
| 637 | + tbz x0, #TIF_NEED_RESCHED_LAZY, 2f // needs rescheduling? |
---|
| 638 | +1: |
---|
| 639 | + bl arm64_preempt_schedule_irq // irq en/disable is done inside |
---|
| 640 | +2: |
---|
| 641 | +#endif |
---|
| 642 | + |
---|
| 643 | + mov x0, sp |
---|
| 644 | + bl exit_el1_irq_or_nmi |
---|
| 645 | + .endm |
---|
| 646 | + |
---|
| 647 | + .macro el0_interrupt_handler, handler:req |
---|
| 648 | + user_exit_irqoff |
---|
| 649 | + enable_da_f |
---|
| 650 | + |
---|
| 651 | + tbz x22, #55, 1f |
---|
| 652 | + bl do_el0_irq_bp_hardening |
---|
| 653 | +1: |
---|
| 654 | + irq_handler \handler |
---|
443 | 655 | .endm |
---|
444 | 656 | |
---|
445 | 657 | .text |
---|
.. | .. |
---|
450 | 662 | .pushsection ".entry.text", "ax" |
---|
451 | 663 | |
---|
452 | 664 | .align 11 |
---|
453 | | -ENTRY(vectors) |
---|
| 665 | +SYM_CODE_START(vectors) |
---|
454 | 666 | kernel_ventry 1, sync_invalid // Synchronous EL1t |
---|
455 | 667 | kernel_ventry 1, irq_invalid // IRQ EL1t |
---|
456 | 668 | kernel_ventry 1, fiq_invalid // FIQ EL1t |
---|
.. | .. |
---|
477 | 689 | kernel_ventry 0, fiq_invalid, 32 // FIQ 32-bit EL0 |
---|
478 | 690 | kernel_ventry 0, error_invalid, 32 // Error 32-bit EL0 |
---|
479 | 691 | #endif |
---|
480 | | -END(vectors) |
---|
| 692 | +SYM_CODE_END(vectors) |
---|
481 | 693 | |
---|
482 | 694 | #ifdef CONFIG_VMAP_STACK |
---|
483 | 695 | /* |
---|
.. | .. |
---|
519 | 731 | ASM_BUG() |
---|
520 | 732 | .endm |
---|
521 | 733 | |
---|
522 | | -el0_sync_invalid: |
---|
| 734 | +SYM_CODE_START_LOCAL(el0_sync_invalid) |
---|
523 | 735 | inv_entry 0, BAD_SYNC |
---|
524 | | -ENDPROC(el0_sync_invalid) |
---|
| 736 | +SYM_CODE_END(el0_sync_invalid) |
---|
525 | 737 | |
---|
526 | | -el0_irq_invalid: |
---|
| 738 | +SYM_CODE_START_LOCAL(el0_irq_invalid) |
---|
527 | 739 | inv_entry 0, BAD_IRQ |
---|
528 | | -ENDPROC(el0_irq_invalid) |
---|
| 740 | +SYM_CODE_END(el0_irq_invalid) |
---|
529 | 741 | |
---|
530 | | -el0_fiq_invalid: |
---|
| 742 | +SYM_CODE_START_LOCAL(el0_fiq_invalid) |
---|
531 | 743 | inv_entry 0, BAD_FIQ |
---|
532 | | -ENDPROC(el0_fiq_invalid) |
---|
| 744 | +SYM_CODE_END(el0_fiq_invalid) |
---|
533 | 745 | |
---|
534 | | -el0_error_invalid: |
---|
| 746 | +SYM_CODE_START_LOCAL(el0_error_invalid) |
---|
535 | 747 | inv_entry 0, BAD_ERROR |
---|
536 | | -ENDPROC(el0_error_invalid) |
---|
| 748 | +SYM_CODE_END(el0_error_invalid) |
---|
537 | 749 | |
---|
538 | 750 | #ifdef CONFIG_COMPAT |
---|
539 | | -el0_fiq_invalid_compat: |
---|
| 751 | +SYM_CODE_START_LOCAL(el0_fiq_invalid_compat) |
---|
540 | 752 | inv_entry 0, BAD_FIQ, 32 |
---|
541 | | -ENDPROC(el0_fiq_invalid_compat) |
---|
| 753 | +SYM_CODE_END(el0_fiq_invalid_compat) |
---|
542 | 754 | #endif |
---|
543 | 755 | |
---|
544 | | -el1_sync_invalid: |
---|
| 756 | +SYM_CODE_START_LOCAL(el1_sync_invalid) |
---|
545 | 757 | inv_entry 1, BAD_SYNC |
---|
546 | | -ENDPROC(el1_sync_invalid) |
---|
| 758 | +SYM_CODE_END(el1_sync_invalid) |
---|
547 | 759 | |
---|
548 | | -el1_irq_invalid: |
---|
| 760 | +SYM_CODE_START_LOCAL(el1_irq_invalid) |
---|
549 | 761 | inv_entry 1, BAD_IRQ |
---|
550 | | -ENDPROC(el1_irq_invalid) |
---|
| 762 | +SYM_CODE_END(el1_irq_invalid) |
---|
551 | 763 | |
---|
552 | | -el1_fiq_invalid: |
---|
| 764 | +SYM_CODE_START_LOCAL(el1_fiq_invalid) |
---|
553 | 765 | inv_entry 1, BAD_FIQ |
---|
554 | | -ENDPROC(el1_fiq_invalid) |
---|
| 766 | +SYM_CODE_END(el1_fiq_invalid) |
---|
555 | 767 | |
---|
556 | | -el1_error_invalid: |
---|
| 768 | +SYM_CODE_START_LOCAL(el1_error_invalid) |
---|
557 | 769 | inv_entry 1, BAD_ERROR |
---|
558 | | -ENDPROC(el1_error_invalid) |
---|
| 770 | +SYM_CODE_END(el1_error_invalid) |
---|
559 | 771 | |
---|
560 | 772 | /* |
---|
561 | 773 | * EL1 mode handlers. |
---|
562 | 774 | */ |
---|
563 | 775 | .align 6 |
---|
564 | | -el1_sync: |
---|
| 776 | +SYM_CODE_START_LOCAL_NOALIGN(el1_sync) |
---|
565 | 777 | kernel_entry 1 |
---|
566 | | - mrs x1, esr_el1 // read the syndrome register |
---|
567 | | - lsr x24, x1, #ESR_ELx_EC_SHIFT // exception class |
---|
568 | | - cmp x24, #ESR_ELx_EC_DABT_CUR // data abort in EL1 |
---|
569 | | - b.eq el1_da |
---|
570 | | - cmp x24, #ESR_ELx_EC_IABT_CUR // instruction abort in EL1 |
---|
571 | | - b.eq el1_ia |
---|
572 | | - cmp x24, #ESR_ELx_EC_SYS64 // configurable trap |
---|
573 | | - b.eq el1_undef |
---|
574 | | - cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception |
---|
575 | | - b.eq el1_sp_pc |
---|
576 | | - cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception |
---|
577 | | - b.eq el1_sp_pc |
---|
578 | | - cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL1 |
---|
579 | | - b.eq el1_undef |
---|
580 | | - cmp x24, #ESR_ELx_EC_BREAKPT_CUR // debug exception in EL1 |
---|
581 | | - b.ge el1_dbg |
---|
582 | | - b el1_inv |
---|
583 | | - |
---|
584 | | -el1_ia: |
---|
585 | | - /* |
---|
586 | | - * Fall through to the Data abort case |
---|
587 | | - */ |
---|
588 | | -el1_da: |
---|
589 | | - /* |
---|
590 | | - * Data abort handling |
---|
591 | | - */ |
---|
592 | | - mrs x3, far_el1 |
---|
593 | | - inherit_daif pstate=x23, tmp=x2 |
---|
594 | | - untagged_addr x0, x3 |
---|
595 | | - mov x2, sp // struct pt_regs |
---|
596 | | - bl do_mem_abort |
---|
597 | | - |
---|
598 | | - kernel_exit 1 |
---|
599 | | -el1_sp_pc: |
---|
600 | | - /* |
---|
601 | | - * Stack or PC alignment exception handling |
---|
602 | | - */ |
---|
603 | | - mrs x0, far_el1 |
---|
604 | | - inherit_daif pstate=x23, tmp=x2 |
---|
605 | | - mov x2, sp |
---|
606 | | - bl do_sp_pc_abort |
---|
607 | | - ASM_BUG() |
---|
608 | | -el1_undef: |
---|
609 | | - /* |
---|
610 | | - * Undefined instruction |
---|
611 | | - */ |
---|
612 | | - inherit_daif pstate=x23, tmp=x2 |
---|
613 | 778 | mov x0, sp |
---|
614 | | - bl do_undefinstr |
---|
| 779 | + bl el1_sync_handler |
---|
615 | 780 | kernel_exit 1 |
---|
616 | | -el1_dbg: |
---|
617 | | - /* |
---|
618 | | - * Debug exception handling |
---|
619 | | - */ |
---|
620 | | - cmp x24, #ESR_ELx_EC_BRK64 // if BRK64 |
---|
621 | | - cinc x24, x24, eq // set bit '0' |
---|
622 | | - tbz x24, #0, el1_inv // EL1 only |
---|
623 | | - mrs x0, far_el1 |
---|
624 | | - mov x2, sp // struct pt_regs |
---|
625 | | - bl do_debug_exception |
---|
626 | | - kernel_exit 1 |
---|
627 | | -el1_inv: |
---|
628 | | - // TODO: add support for undefined instructions in kernel mode |
---|
629 | | - inherit_daif pstate=x23, tmp=x2 |
---|
630 | | - mov x0, sp |
---|
631 | | - mov x2, x1 |
---|
632 | | - mov x1, #BAD_SYNC |
---|
633 | | - bl bad_mode |
---|
634 | | - ASM_BUG() |
---|
635 | | -ENDPROC(el1_sync) |
---|
| 781 | +SYM_CODE_END(el1_sync) |
---|
636 | 782 | |
---|
637 | 783 | .align 6 |
---|
638 | | -el1_irq: |
---|
| 784 | +SYM_CODE_START_LOCAL_NOALIGN(el1_irq) |
---|
639 | 785 | kernel_entry 1 |
---|
640 | | - enable_da_f |
---|
641 | | -#ifdef CONFIG_TRACE_IRQFLAGS |
---|
642 | | - bl trace_hardirqs_off |
---|
643 | | -#endif |
---|
644 | | - |
---|
645 | | - irq_handler |
---|
646 | | - |
---|
647 | | -#ifdef CONFIG_PREEMPT |
---|
648 | | - ldr w24, [tsk, #TSK_TI_PREEMPT] // get preempt count |
---|
649 | | - cbnz w24, 1f // preempt count != 0 |
---|
650 | | - ldr x0, [tsk, #TSK_TI_FLAGS] // get flags |
---|
651 | | - tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling? |
---|
652 | | - bl el1_preempt |
---|
653 | | -1: |
---|
654 | | -#endif |
---|
655 | | -#ifdef CONFIG_TRACE_IRQFLAGS |
---|
656 | | - bl trace_hardirqs_on |
---|
657 | | -#endif |
---|
| 786 | + el1_interrupt_handler handle_arch_irq |
---|
658 | 787 | kernel_exit 1 |
---|
659 | | -ENDPROC(el1_irq) |
---|
660 | | - |
---|
661 | | -#ifdef CONFIG_PREEMPT |
---|
662 | | -el1_preempt: |
---|
663 | | - mov x24, lr |
---|
664 | | -1: bl preempt_schedule_irq // irq en/disable is done inside |
---|
665 | | - ldr x0, [tsk, #TSK_TI_FLAGS] // get new tasks TI_FLAGS |
---|
666 | | - tbnz x0, #TIF_NEED_RESCHED, 1b // needs rescheduling? |
---|
667 | | - ret x24 |
---|
668 | | -#endif |
---|
| 788 | +SYM_CODE_END(el1_irq) |
---|
669 | 789 | |
---|
670 | 790 | /* |
---|
671 | 791 | * EL0 mode handlers. |
---|
672 | 792 | */ |
---|
673 | 793 | .align 6 |
---|
674 | | -el0_sync: |
---|
| 794 | +SYM_CODE_START_LOCAL_NOALIGN(el0_sync) |
---|
675 | 795 | kernel_entry 0 |
---|
676 | | - mrs x25, esr_el1 // read the syndrome register |
---|
677 | | - lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class |
---|
678 | | - cmp x24, #ESR_ELx_EC_SVC64 // SVC in 64-bit state |
---|
679 | | - b.eq el0_svc |
---|
680 | | - cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0 |
---|
681 | | - b.eq el0_da |
---|
682 | | - cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0 |
---|
683 | | - b.eq el0_ia |
---|
684 | | - cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access |
---|
685 | | - b.eq el0_fpsimd_acc |
---|
686 | | - cmp x24, #ESR_ELx_EC_SVE // SVE access |
---|
687 | | - b.eq el0_sve_acc |
---|
688 | | - cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception |
---|
689 | | - b.eq el0_fpsimd_exc |
---|
690 | | - cmp x24, #ESR_ELx_EC_SYS64 // configurable trap |
---|
691 | | - b.eq el0_sys |
---|
692 | | - cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception |
---|
693 | | - b.eq el0_sp_pc |
---|
694 | | - cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception |
---|
695 | | - b.eq el0_sp_pc |
---|
696 | | - cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0 |
---|
697 | | - b.eq el0_undef |
---|
698 | | - cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0 |
---|
699 | | - b.ge el0_dbg |
---|
700 | | - b el0_inv |
---|
| 796 | + mov x0, sp |
---|
| 797 | + bl el0_sync_handler |
---|
| 798 | + b ret_to_user |
---|
| 799 | +SYM_CODE_END(el0_sync) |
---|
701 | 800 | |
---|
702 | 801 | #ifdef CONFIG_COMPAT |
---|
703 | 802 | .align 6 |
---|
704 | | -el0_sync_compat: |
---|
| 803 | +SYM_CODE_START_LOCAL_NOALIGN(el0_sync_compat) |
---|
705 | 804 | kernel_entry 0, 32 |
---|
706 | | - mrs x25, esr_el1 // read the syndrome register |
---|
707 | | - lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class |
---|
708 | | - cmp x24, #ESR_ELx_EC_SVC32 // SVC in 32-bit state |
---|
709 | | - b.eq el0_svc_compat |
---|
710 | | - cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0 |
---|
711 | | - b.eq el0_da |
---|
712 | | - cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0 |
---|
713 | | - b.eq el0_ia |
---|
714 | | - cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access |
---|
715 | | - b.eq el0_fpsimd_acc |
---|
716 | | - cmp x24, #ESR_ELx_EC_FP_EXC32 // FP/ASIMD exception |
---|
717 | | - b.eq el0_fpsimd_exc |
---|
718 | | - cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception |
---|
719 | | - b.eq el0_sp_pc |
---|
720 | | - cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0 |
---|
721 | | - b.eq el0_undef |
---|
722 | | - cmp x24, #ESR_ELx_EC_CP15_32 // CP15 MRC/MCR trap |
---|
723 | | - b.eq el0_undef |
---|
724 | | - cmp x24, #ESR_ELx_EC_CP15_64 // CP15 MRRC/MCRR trap |
---|
725 | | - b.eq el0_undef |
---|
726 | | - cmp x24, #ESR_ELx_EC_CP14_MR // CP14 MRC/MCR trap |
---|
727 | | - b.eq el0_undef |
---|
728 | | - cmp x24, #ESR_ELx_EC_CP14_LS // CP14 LDC/STC trap |
---|
729 | | - b.eq el0_undef |
---|
730 | | - cmp x24, #ESR_ELx_EC_CP14_64 // CP14 MRRC/MCRR trap |
---|
731 | | - b.eq el0_undef |
---|
732 | | - cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0 |
---|
733 | | - b.ge el0_dbg |
---|
734 | | - b el0_inv |
---|
735 | | -el0_svc_compat: |
---|
736 | 805 | mov x0, sp |
---|
737 | | - bl el0_svc_compat_handler |
---|
| 806 | + bl el0_sync_compat_handler |
---|
738 | 807 | b ret_to_user |
---|
| 808 | +SYM_CODE_END(el0_sync_compat) |
---|
739 | 809 | |
---|
740 | 810 | .align 6 |
---|
741 | | -el0_irq_compat: |
---|
| 811 | +SYM_CODE_START_LOCAL_NOALIGN(el0_irq_compat) |
---|
742 | 812 | kernel_entry 0, 32 |
---|
743 | 813 | b el0_irq_naked |
---|
| 814 | +SYM_CODE_END(el0_irq_compat) |
---|
744 | 815 | |
---|
745 | | -el0_error_compat: |
---|
| 816 | +SYM_CODE_START_LOCAL_NOALIGN(el0_error_compat) |
---|
746 | 817 | kernel_entry 0, 32 |
---|
747 | 818 | b el0_error_naked |
---|
| 819 | +SYM_CODE_END(el0_error_compat) |
---|
748 | 820 | #endif |
---|
749 | | - |
---|
750 | | -el0_da: |
---|
751 | | - /* |
---|
752 | | - * Data abort handling |
---|
753 | | - */ |
---|
754 | | - mrs x26, far_el1 |
---|
755 | | - enable_daif |
---|
756 | | - ct_user_exit |
---|
757 | | - untagged_addr x0, x26 |
---|
758 | | - mov x1, x25 |
---|
759 | | - mov x2, sp |
---|
760 | | - bl do_mem_abort |
---|
761 | | - b ret_to_user |
---|
762 | | -el0_ia: |
---|
763 | | - /* |
---|
764 | | - * Instruction abort handling |
---|
765 | | - */ |
---|
766 | | - mrs x26, far_el1 |
---|
767 | | - enable_da_f |
---|
768 | | -#ifdef CONFIG_TRACE_IRQFLAGS |
---|
769 | | - bl trace_hardirqs_off |
---|
770 | | -#endif |
---|
771 | | - ct_user_exit |
---|
772 | | - mov x0, x26 |
---|
773 | | - mov x1, x25 |
---|
774 | | - mov x2, sp |
---|
775 | | - bl do_el0_ia_bp_hardening |
---|
776 | | - b ret_to_user |
---|
777 | | -el0_fpsimd_acc: |
---|
778 | | - /* |
---|
779 | | - * Floating Point or Advanced SIMD access |
---|
780 | | - */ |
---|
781 | | - enable_daif |
---|
782 | | - ct_user_exit |
---|
783 | | - mov x0, x25 |
---|
784 | | - mov x1, sp |
---|
785 | | - bl do_fpsimd_acc |
---|
786 | | - b ret_to_user |
---|
787 | | -el0_sve_acc: |
---|
788 | | - /* |
---|
789 | | - * Scalable Vector Extension access |
---|
790 | | - */ |
---|
791 | | - enable_daif |
---|
792 | | - ct_user_exit |
---|
793 | | - mov x0, x25 |
---|
794 | | - mov x1, sp |
---|
795 | | - bl do_sve_acc |
---|
796 | | - b ret_to_user |
---|
797 | | -el0_fpsimd_exc: |
---|
798 | | - /* |
---|
799 | | - * Floating Point, Advanced SIMD or SVE exception |
---|
800 | | - */ |
---|
801 | | - enable_daif |
---|
802 | | - ct_user_exit |
---|
803 | | - mov x0, x25 |
---|
804 | | - mov x1, sp |
---|
805 | | - bl do_fpsimd_exc |
---|
806 | | - b ret_to_user |
---|
807 | | -el0_sp_pc: |
---|
808 | | - /* |
---|
809 | | - * Stack or PC alignment exception handling |
---|
810 | | - */ |
---|
811 | | - mrs x26, far_el1 |
---|
812 | | - enable_da_f |
---|
813 | | -#ifdef CONFIG_TRACE_IRQFLAGS |
---|
814 | | - bl trace_hardirqs_off |
---|
815 | | -#endif |
---|
816 | | - ct_user_exit |
---|
817 | | - mov x0, x26 |
---|
818 | | - mov x1, x25 |
---|
819 | | - mov x2, sp |
---|
820 | | - bl do_sp_pc_abort |
---|
821 | | - b ret_to_user |
---|
822 | | -el0_undef: |
---|
823 | | - /* |
---|
824 | | - * Undefined instruction |
---|
825 | | - */ |
---|
826 | | - enable_daif |
---|
827 | | - ct_user_exit |
---|
828 | | - mov x0, sp |
---|
829 | | - bl do_undefinstr |
---|
830 | | - b ret_to_user |
---|
831 | | -el0_sys: |
---|
832 | | - /* |
---|
833 | | - * System instructions, for trapped cache maintenance instructions |
---|
834 | | - */ |
---|
835 | | - enable_daif |
---|
836 | | - ct_user_exit |
---|
837 | | - mov x0, x25 |
---|
838 | | - mov x1, sp |
---|
839 | | - bl do_sysinstr |
---|
840 | | - b ret_to_user |
---|
841 | | -el0_dbg: |
---|
842 | | - /* |
---|
843 | | - * Debug exception handling |
---|
844 | | - */ |
---|
845 | | - tbnz x24, #0, el0_inv // EL0 only |
---|
846 | | - mrs x0, far_el1 |
---|
847 | | - mov x1, x25 |
---|
848 | | - mov x2, sp |
---|
849 | | - bl do_debug_exception |
---|
850 | | - enable_da_f |
---|
851 | | - ct_user_exit |
---|
852 | | - b ret_to_user |
---|
853 | | -el0_inv: |
---|
854 | | - enable_daif |
---|
855 | | - ct_user_exit |
---|
856 | | - mov x0, sp |
---|
857 | | - mov x1, #BAD_SYNC |
---|
858 | | - mov x2, x25 |
---|
859 | | - bl bad_el0_sync |
---|
860 | | - b ret_to_user |
---|
861 | | -ENDPROC(el0_sync) |
---|
862 | 821 | |
---|
863 | 822 | .align 6 |
---|
864 | | -el0_irq: |
---|
| 823 | +SYM_CODE_START_LOCAL_NOALIGN(el0_irq) |
---|
865 | 824 | kernel_entry 0 |
---|
866 | 825 | el0_irq_naked: |
---|
867 | | - enable_da_f |
---|
868 | | -#ifdef CONFIG_TRACE_IRQFLAGS |
---|
869 | | - bl trace_hardirqs_off |
---|
870 | | -#endif |
---|
871 | | - |
---|
872 | | - ct_user_exit |
---|
873 | | -#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR |
---|
874 | | - tbz x22, #55, 1f |
---|
875 | | - bl do_el0_irq_bp_hardening |
---|
876 | | -1: |
---|
877 | | -#endif |
---|
878 | | - irq_handler |
---|
879 | | - |
---|
880 | | -#ifdef CONFIG_TRACE_IRQFLAGS |
---|
881 | | - bl trace_hardirqs_on |
---|
882 | | -#endif |
---|
| 826 | + el0_interrupt_handler handle_arch_irq |
---|
883 | 827 | b ret_to_user |
---|
884 | | -ENDPROC(el0_irq) |
---|
| 828 | +SYM_CODE_END(el0_irq) |
---|
885 | 829 | |
---|
886 | | -el1_error: |
---|
| 830 | +SYM_CODE_START_LOCAL(el1_error) |
---|
887 | 831 | kernel_entry 1 |
---|
888 | 832 | mrs x1, esr_el1 |
---|
889 | 833 | enable_dbg |
---|
890 | 834 | mov x0, sp |
---|
891 | 835 | bl do_serror |
---|
892 | 836 | kernel_exit 1 |
---|
893 | | -ENDPROC(el1_error) |
---|
| 837 | +SYM_CODE_END(el1_error) |
---|
894 | 838 | |
---|
895 | | -el0_error: |
---|
| 839 | +SYM_CODE_START_LOCAL(el0_error) |
---|
896 | 840 | kernel_entry 0 |
---|
897 | 841 | el0_error_naked: |
---|
898 | | - mrs x1, esr_el1 |
---|
| 842 | + mrs x25, esr_el1 |
---|
| 843 | + user_exit_irqoff |
---|
899 | 844 | enable_dbg |
---|
900 | 845 | mov x0, sp |
---|
| 846 | + mov x1, x25 |
---|
901 | 847 | bl do_serror |
---|
902 | 848 | enable_da_f |
---|
903 | | - ct_user_exit |
---|
904 | 849 | b ret_to_user |
---|
905 | | -ENDPROC(el0_error) |
---|
| 850 | +SYM_CODE_END(el0_error) |
---|
| 851 | + |
---|
| 852 | +/* |
---|
| 853 | + * "slow" syscall return path. |
---|
| 854 | + */ |
---|
| 855 | +SYM_CODE_START_LOCAL(ret_to_user) |
---|
| 856 | + disable_daif |
---|
| 857 | + gic_prio_kentry_setup tmp=x3 |
---|
| 858 | +#ifdef CONFIG_TRACE_IRQFLAGS |
---|
| 859 | + bl trace_hardirqs_off |
---|
| 860 | +#endif |
---|
| 861 | + ldr x19, [tsk, #TSK_TI_FLAGS] |
---|
| 862 | + and x2, x19, #_TIF_WORK_MASK |
---|
| 863 | + cbnz x2, work_pending |
---|
| 864 | +finish_ret_to_user: |
---|
| 865 | + user_enter_irqoff |
---|
| 866 | + enable_step_tsk x19, x2 |
---|
| 867 | +#ifdef CONFIG_GCC_PLUGIN_STACKLEAK |
---|
| 868 | + bl stackleak_erase |
---|
| 869 | +#endif |
---|
| 870 | + kernel_exit 0 |
---|
906 | 871 | |
---|
907 | 872 | /* |
---|
908 | 873 | * Ok, we need to do extra processing, enter the slow path. |
---|
909 | 874 | */ |
---|
910 | 875 | work_pending: |
---|
911 | 876 | mov x0, sp // 'regs' |
---|
| 877 | + mov x1, x19 |
---|
912 | 878 | bl do_notify_resume |
---|
913 | | -#ifdef CONFIG_TRACE_IRQFLAGS |
---|
914 | | - bl trace_hardirqs_on // enabled while in userspace |
---|
915 | | -#endif |
---|
916 | | - ldr x1, [tsk, #TSK_TI_FLAGS] // re-check for single-step |
---|
| 879 | + ldr x19, [tsk, #TSK_TI_FLAGS] // re-check for single-step |
---|
917 | 880 | b finish_ret_to_user |
---|
918 | | -/* |
---|
919 | | - * "slow" syscall return path. |
---|
920 | | - */ |
---|
921 | | -ret_to_user: |
---|
922 | | - disable_daif |
---|
923 | | - ldr x1, [tsk, #TSK_TI_FLAGS] |
---|
924 | | - and x2, x1, #_TIF_WORK_MASK |
---|
925 | | - cbnz x2, work_pending |
---|
926 | | -finish_ret_to_user: |
---|
927 | | - enable_step_tsk x1, x2 |
---|
928 | | -#ifdef CONFIG_GCC_PLUGIN_STACKLEAK |
---|
929 | | - bl stackleak_erase |
---|
930 | | -#endif |
---|
931 | | - kernel_exit 0 |
---|
932 | | -ENDPROC(ret_to_user) |
---|
933 | | - |
---|
934 | | -/* |
---|
935 | | - * SVC handler. |
---|
936 | | - */ |
---|
937 | | - .align 6 |
---|
938 | | -el0_svc: |
---|
939 | | - mov x0, sp |
---|
940 | | - bl el0_svc_handler |
---|
941 | | - b ret_to_user |
---|
942 | | -ENDPROC(el0_svc) |
---|
| 881 | +SYM_CODE_END(ret_to_user) |
---|
943 | 882 | |
---|
944 | 883 | .popsection // .entry.text |
---|
945 | 884 | |
---|
946 | | -#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
---|
947 | | -/* |
---|
948 | | - * Exception vectors trampoline. |
---|
949 | | - */ |
---|
950 | | - .pushsection ".entry.tramp.text", "ax" |
---|
951 | | - |
---|
| 885 | + // Move from tramp_pg_dir to swapper_pg_dir |
---|
952 | 886 | .macro tramp_map_kernel, tmp |
---|
953 | 887 | mrs \tmp, ttbr1_el1 |
---|
954 | | - add \tmp, \tmp, #(PAGE_SIZE + RESERVED_TTBR0_SIZE) |
---|
| 888 | + add \tmp, \tmp, #(2 * PAGE_SIZE) |
---|
955 | 889 | bic \tmp, \tmp, #USER_ASID_FLAG |
---|
956 | 890 | msr ttbr1_el1, \tmp |
---|
957 | 891 | #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003 |
---|
.. | .. |
---|
968 | 902 | #endif /* CONFIG_QCOM_FALKOR_ERRATUM_1003 */ |
---|
969 | 903 | .endm |
---|
970 | 904 | |
---|
| 905 | + // Move from swapper_pg_dir to tramp_pg_dir |
---|
971 | 906 | .macro tramp_unmap_kernel, tmp |
---|
972 | 907 | mrs \tmp, ttbr1_el1 |
---|
973 | | - sub \tmp, \tmp, #(PAGE_SIZE + RESERVED_TTBR0_SIZE) |
---|
| 908 | + sub \tmp, \tmp, #(2 * PAGE_SIZE) |
---|
974 | 909 | orr \tmp, \tmp, #USER_ASID_FLAG |
---|
975 | 910 | msr ttbr1_el1, \tmp |
---|
976 | 911 | /* |
---|
.. | .. |
---|
980 | 915 | */ |
---|
981 | 916 | .endm |
---|
982 | 917 | |
---|
983 | | - .macro tramp_ventry, regsize = 64 |
---|
| 918 | + .macro tramp_data_page dst |
---|
| 919 | + adr_l \dst, .entry.tramp.text |
---|
| 920 | + sub \dst, \dst, PAGE_SIZE |
---|
| 921 | + .endm |
---|
| 922 | + |
---|
| 923 | + .macro tramp_data_read_var dst, var |
---|
| 924 | +#ifdef CONFIG_RANDOMIZE_BASE |
---|
| 925 | + tramp_data_page \dst |
---|
| 926 | + add \dst, \dst, #:lo12:__entry_tramp_data_\var |
---|
| 927 | + ldr \dst, [\dst] |
---|
| 928 | +#else |
---|
| 929 | + ldr \dst, =\var |
---|
| 930 | +#endif |
---|
| 931 | + .endm |
---|
| 932 | + |
---|
| 933 | +#define BHB_MITIGATION_NONE 0 |
---|
| 934 | +#define BHB_MITIGATION_LOOP 1 |
---|
| 935 | +#define BHB_MITIGATION_FW 2 |
---|
| 936 | +#define BHB_MITIGATION_INSN 3 |
---|
| 937 | + |
---|
| 938 | + .macro tramp_ventry, vector_start, regsize, kpti, bhb |
---|
984 | 939 | .align 7 |
---|
985 | 940 | 1: |
---|
986 | 941 | .if \regsize == 64 |
---|
987 | 942 | msr tpidrro_el0, x30 // Restored in kernel_ventry |
---|
988 | 943 | .endif |
---|
| 944 | + |
---|
| 945 | + .if \bhb == BHB_MITIGATION_LOOP |
---|
| 946 | + /* |
---|
| 947 | + * This sequence must appear before the first indirect branch. i.e. the |
---|
| 948 | + * ret out of tramp_ventry. It appears here because x30 is free. |
---|
| 949 | + */ |
---|
| 950 | + __mitigate_spectre_bhb_loop x30 |
---|
| 951 | + .endif // \bhb == BHB_MITIGATION_LOOP |
---|
| 952 | + |
---|
| 953 | + .if \bhb == BHB_MITIGATION_INSN |
---|
| 954 | + clearbhb |
---|
| 955 | + isb |
---|
| 956 | + .endif // \bhb == BHB_MITIGATION_INSN |
---|
| 957 | + |
---|
| 958 | + .if \kpti == 1 |
---|
989 | 959 | /* |
---|
990 | 960 | * Defend against branch aliasing attacks by pushing a dummy |
---|
991 | 961 | * entry onto the return stack and using a RET instruction to |
---|
.. | .. |
---|
995 | 965 | b . |
---|
996 | 966 | 2: |
---|
997 | 967 | tramp_map_kernel x30 |
---|
998 | | -#ifdef CONFIG_RANDOMIZE_BASE |
---|
999 | | - adr x30, tramp_vectors + PAGE_SIZE |
---|
1000 | 968 | alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003 |
---|
1001 | | - ldr x30, [x30] |
---|
1002 | | -#else |
---|
1003 | | - ldr x30, =vectors |
---|
1004 | | -#endif |
---|
1005 | | - prfm plil1strm, [x30, #(1b - tramp_vectors)] |
---|
| 969 | + tramp_data_read_var x30, vectors |
---|
| 970 | +alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM |
---|
| 971 | + prfm plil1strm, [x30, #(1b - \vector_start)] |
---|
| 972 | +alternative_else_nop_endif |
---|
| 973 | + |
---|
1006 | 974 | msr vbar_el1, x30 |
---|
1007 | | - add x30, x30, #(1b - tramp_vectors) |
---|
1008 | 975 | isb |
---|
| 976 | + .else |
---|
| 977 | + ldr x30, =vectors |
---|
| 978 | + .endif // \kpti == 1 |
---|
| 979 | + |
---|
| 980 | + .if \bhb == BHB_MITIGATION_FW |
---|
| 981 | + /* |
---|
| 982 | + * The firmware sequence must appear before the first indirect branch. |
---|
| 983 | + * i.e. the ret out of tramp_ventry. But it also needs the stack to be |
---|
| 984 | + * mapped to save/restore the registers the SMC clobbers. |
---|
| 985 | + */ |
---|
| 986 | + __mitigate_spectre_bhb_fw |
---|
| 987 | + .endif // \bhb == BHB_MITIGATION_FW |
---|
| 988 | + |
---|
| 989 | + add x30, x30, #(1b - \vector_start + 4) |
---|
1009 | 990 | ret |
---|
| 991 | +.org 1b + 128 // Did we overflow the ventry slot? |
---|
1010 | 992 | .endm |
---|
1011 | 993 | |
---|
1012 | 994 | .macro tramp_exit, regsize = 64 |
---|
1013 | | - adr x30, tramp_vectors |
---|
| 995 | + tramp_data_read_var x30, this_cpu_vector |
---|
| 996 | + this_cpu_offset x29 |
---|
| 997 | + ldr x30, [x30, x29] |
---|
| 998 | + |
---|
1014 | 999 | msr vbar_el1, x30 |
---|
1015 | | - tramp_unmap_kernel x30 |
---|
| 1000 | + ldr lr, [sp, #S_LR] |
---|
| 1001 | + tramp_unmap_kernel x29 |
---|
1016 | 1002 | .if \regsize == 64 |
---|
1017 | | - mrs x30, far_el1 |
---|
| 1003 | + mrs x29, far_el1 |
---|
1018 | 1004 | .endif |
---|
| 1005 | + add sp, sp, #S_FRAME_SIZE // restore sp |
---|
1019 | 1006 | eret |
---|
1020 | 1007 | sb |
---|
1021 | 1008 | .endm |
---|
1022 | 1009 | |
---|
1023 | | - .align 11 |
---|
1024 | | -ENTRY(tramp_vectors) |
---|
| 1010 | + .macro generate_tramp_vector, kpti, bhb |
---|
| 1011 | +.Lvector_start\@: |
---|
1025 | 1012 | .space 0x400 |
---|
1026 | 1013 | |
---|
1027 | | - tramp_ventry |
---|
1028 | | - tramp_ventry |
---|
1029 | | - tramp_ventry |
---|
1030 | | - tramp_ventry |
---|
| 1014 | + .rept 4 |
---|
| 1015 | + tramp_ventry .Lvector_start\@, 64, \kpti, \bhb |
---|
| 1016 | + .endr |
---|
| 1017 | + .rept 4 |
---|
| 1018 | + tramp_ventry .Lvector_start\@, 32, \kpti, \bhb |
---|
| 1019 | + .endr |
---|
| 1020 | + .endm |
---|
1031 | 1021 | |
---|
1032 | | - tramp_ventry 32 |
---|
1033 | | - tramp_ventry 32 |
---|
1034 | | - tramp_ventry 32 |
---|
1035 | | - tramp_ventry 32 |
---|
1036 | | -END(tramp_vectors) |
---|
| 1022 | +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
---|
| 1023 | +/* |
---|
| 1024 | + * Exception vectors trampoline. |
---|
| 1025 | + * The order must match __bp_harden_el1_vectors and the |
---|
| 1026 | + * arm64_bp_harden_el1_vectors enum. |
---|
| 1027 | + */ |
---|
| 1028 | + .pushsection ".entry.tramp.text", "ax" |
---|
| 1029 | + .align 11 |
---|
| 1030 | +SYM_CODE_START_NOALIGN(tramp_vectors) |
---|
| 1031 | +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
---|
| 1032 | + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP |
---|
| 1033 | + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW |
---|
| 1034 | + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_INSN |
---|
| 1035 | +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
---|
| 1036 | + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE |
---|
| 1037 | +SYM_CODE_END(tramp_vectors) |
---|
1037 | 1038 | |
---|
1038 | | -ENTRY(tramp_exit_native) |
---|
| 1039 | +SYM_CODE_START(tramp_exit_native) |
---|
1039 | 1040 | tramp_exit |
---|
1040 | | -END(tramp_exit_native) |
---|
| 1041 | +SYM_CODE_END(tramp_exit_native) |
---|
1041 | 1042 | |
---|
1042 | | -ENTRY(tramp_exit_compat) |
---|
| 1043 | +SYM_CODE_START(tramp_exit_compat) |
---|
1043 | 1044 | tramp_exit 32 |
---|
1044 | | -END(tramp_exit_compat) |
---|
| 1045 | +SYM_CODE_END(tramp_exit_compat) |
---|
1045 | 1046 | |
---|
1046 | 1047 | .ltorg |
---|
1047 | 1048 | .popsection // .entry.tramp.text |
---|
1048 | 1049 | #ifdef CONFIG_RANDOMIZE_BASE |
---|
1049 | 1050 | .pushsection ".rodata", "a" |
---|
1050 | 1051 | .align PAGE_SHIFT |
---|
1051 | | - .globl __entry_tramp_data_start |
---|
1052 | | -__entry_tramp_data_start: |
---|
| 1052 | +SYM_DATA_START(__entry_tramp_data_start) |
---|
| 1053 | +__entry_tramp_data_vectors: |
---|
1053 | 1054 | .quad vectors |
---|
| 1055 | +#ifdef CONFIG_ARM_SDE_INTERFACE |
---|
| 1056 | +__entry_tramp_data___sdei_asm_handler: |
---|
| 1057 | + .quad __sdei_asm_handler |
---|
| 1058 | +#endif /* CONFIG_ARM_SDE_INTERFACE */ |
---|
| 1059 | +__entry_tramp_data_this_cpu_vector: |
---|
| 1060 | + .quad this_cpu_vector |
---|
| 1061 | +SYM_DATA_END(__entry_tramp_data_start) |
---|
1054 | 1062 | .popsection // .rodata |
---|
1055 | 1063 | #endif /* CONFIG_RANDOMIZE_BASE */ |
---|
1056 | 1064 | #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ |
---|
| 1065 | + |
---|
| 1066 | +/* |
---|
| 1067 | + * Exception vectors for spectre mitigations on entry from EL1 when |
---|
| 1068 | + * kpti is not in use. |
---|
| 1069 | + */ |
---|
| 1070 | + .macro generate_el1_vector, bhb |
---|
| 1071 | +.Lvector_start\@: |
---|
| 1072 | + kernel_ventry 1, sync_invalid // Synchronous EL1t |
---|
| 1073 | + kernel_ventry 1, irq_invalid // IRQ EL1t |
---|
| 1074 | + kernel_ventry 1, fiq_invalid // FIQ EL1t |
---|
| 1075 | + kernel_ventry 1, error_invalid // Error EL1t |
---|
| 1076 | + |
---|
| 1077 | + kernel_ventry 1, sync // Synchronous EL1h |
---|
| 1078 | + kernel_ventry 1, irq // IRQ EL1h |
---|
| 1079 | + kernel_ventry 1, fiq_invalid // FIQ EL1h |
---|
| 1080 | + kernel_ventry 1, error // Error EL1h |
---|
| 1081 | + |
---|
| 1082 | + .rept 4 |
---|
| 1083 | + tramp_ventry .Lvector_start\@, 64, 0, \bhb |
---|
| 1084 | + .endr |
---|
| 1085 | + .rept 4 |
---|
| 1086 | + tramp_ventry .Lvector_start\@, 32, 0, \bhb |
---|
| 1087 | + .endr |
---|
| 1088 | + .endm |
---|
| 1089 | + |
---|
| 1090 | +/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */ |
---|
| 1091 | + .pushsection ".entry.text", "ax" |
---|
| 1092 | + .align 11 |
---|
| 1093 | +SYM_CODE_START(__bp_harden_el1_vectors) |
---|
| 1094 | +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
---|
| 1095 | + generate_el1_vector bhb=BHB_MITIGATION_LOOP |
---|
| 1096 | + generate_el1_vector bhb=BHB_MITIGATION_FW |
---|
| 1097 | + generate_el1_vector bhb=BHB_MITIGATION_INSN |
---|
| 1098 | +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
---|
| 1099 | +SYM_CODE_END(__bp_harden_el1_vectors) |
---|
| 1100 | + .popsection |
---|
| 1101 | + |
---|
1057 | 1102 | |
---|
1058 | 1103 | /* |
---|
1059 | 1104 | * Register switch for AArch64. The callee-saved registers need to be saved |
---|
.. | .. |
---|
1063 | 1108 | * Previous and next are guaranteed not to be the same. |
---|
1064 | 1109 | * |
---|
1065 | 1110 | */ |
---|
1066 | | -ENTRY(cpu_switch_to) |
---|
| 1111 | +SYM_FUNC_START(cpu_switch_to) |
---|
1067 | 1112 | mov x10, #THREAD_CPU_CONTEXT |
---|
1068 | 1113 | add x8, x0, x10 |
---|
1069 | 1114 | mov x9, sp |
---|
.. | .. |
---|
1084 | 1129 | ldr lr, [x8] |
---|
1085 | 1130 | mov sp, x9 |
---|
1086 | 1131 | msr sp_el0, x1 |
---|
1087 | | -#ifdef CONFIG_SHADOW_CALL_STACK |
---|
1088 | | - str x18, [x0, #TSK_TI_SCS] |
---|
1089 | | - ldr x18, [x1, #TSK_TI_SCS] |
---|
1090 | | - str xzr, [x1, #TSK_TI_SCS] // limit visibility of saved SCS |
---|
1091 | | -#endif |
---|
| 1132 | + ptrauth_keys_install_kernel x1, x8, x9, x10 |
---|
| 1133 | + scs_save x0, x8 |
---|
| 1134 | + scs_load x1, x8 |
---|
1092 | 1135 | ret |
---|
1093 | | -ENDPROC(cpu_switch_to) |
---|
| 1136 | +SYM_FUNC_END(cpu_switch_to) |
---|
1094 | 1137 | NOKPROBE(cpu_switch_to) |
---|
1095 | 1138 | |
---|
1096 | 1139 | /* |
---|
1097 | 1140 | * This is how we return from a fork. |
---|
1098 | 1141 | */ |
---|
1099 | | -ENTRY(ret_from_fork) |
---|
| 1142 | +SYM_CODE_START(ret_from_fork) |
---|
1100 | 1143 | bl schedule_tail |
---|
1101 | 1144 | cbz x19, 1f // not a kernel thread |
---|
1102 | 1145 | mov x0, x20 |
---|
1103 | 1146 | blr x19 |
---|
1104 | | -1: get_thread_info tsk |
---|
| 1147 | +1: get_current_task tsk |
---|
1105 | 1148 | b ret_to_user |
---|
1106 | | -ENDPROC(ret_from_fork) |
---|
| 1149 | +SYM_CODE_END(ret_from_fork) |
---|
1107 | 1150 | NOKPROBE(ret_from_fork) |
---|
1108 | 1151 | |
---|
1109 | 1152 | #ifdef CONFIG_ARM_SDE_INTERFACE |
---|
.. | .. |
---|
1132 | 1175 | */ |
---|
1133 | 1176 | .ltorg |
---|
1134 | 1177 | .pushsection ".entry.tramp.text", "ax" |
---|
1135 | | -ENTRY(__sdei_asm_entry_trampoline) |
---|
| 1178 | +SYM_CODE_START(__sdei_asm_entry_trampoline) |
---|
1136 | 1179 | mrs x4, ttbr1_el1 |
---|
1137 | 1180 | tbz x4, #USER_ASID_BIT, 1f |
---|
1138 | 1181 | |
---|
.. | .. |
---|
1146 | 1189 | */ |
---|
1147 | 1190 | 1: str x4, [x1, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)] |
---|
1148 | 1191 | |
---|
1149 | | -#ifdef CONFIG_RANDOMIZE_BASE |
---|
1150 | | - adr x4, tramp_vectors + PAGE_SIZE |
---|
1151 | | - add x4, x4, #:lo12:__sdei_asm_trampoline_next_handler |
---|
1152 | | - ldr x4, [x4] |
---|
1153 | | -#else |
---|
1154 | | - ldr x4, =__sdei_asm_handler |
---|
1155 | | -#endif |
---|
| 1192 | + tramp_data_read_var x4, __sdei_asm_handler |
---|
1156 | 1193 | br x4 |
---|
1157 | | -ENDPROC(__sdei_asm_entry_trampoline) |
---|
| 1194 | +SYM_CODE_END(__sdei_asm_entry_trampoline) |
---|
1158 | 1195 | NOKPROBE(__sdei_asm_entry_trampoline) |
---|
1159 | 1196 | |
---|
1160 | 1197 | /* |
---|
.. | .. |
---|
1164 | 1201 | * x2: exit_mode |
---|
1165 | 1202 | * x4: struct sdei_registered_event argument from registration time. |
---|
1166 | 1203 | */ |
---|
1167 | | -ENTRY(__sdei_asm_exit_trampoline) |
---|
| 1204 | +SYM_CODE_START(__sdei_asm_exit_trampoline) |
---|
1168 | 1205 | ldr x4, [x4, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)] |
---|
1169 | 1206 | cbnz x4, 1f |
---|
1170 | 1207 | |
---|
1171 | 1208 | tramp_unmap_kernel tmp=x4 |
---|
1172 | 1209 | |
---|
1173 | 1210 | 1: sdei_handler_exit exit_mode=x2 |
---|
1174 | | -ENDPROC(__sdei_asm_exit_trampoline) |
---|
| 1211 | +SYM_CODE_END(__sdei_asm_exit_trampoline) |
---|
1175 | 1212 | NOKPROBE(__sdei_asm_exit_trampoline) |
---|
1176 | 1213 | .ltorg |
---|
1177 | 1214 | .popsection // .entry.tramp.text |
---|
1178 | | -#ifdef CONFIG_RANDOMIZE_BASE |
---|
1179 | | -.pushsection ".rodata", "a" |
---|
1180 | | -__sdei_asm_trampoline_next_handler: |
---|
1181 | | - .quad __sdei_asm_handler |
---|
1182 | | -.popsection // .rodata |
---|
1183 | | -#endif /* CONFIG_RANDOMIZE_BASE */ |
---|
1184 | 1215 | #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ |
---|
1185 | 1216 | |
---|
1186 | 1217 | /* |
---|
.. | .. |
---|
1196 | 1227 | * follow SMC-CC. We save (or retrieve) all the registers as the handler may |
---|
1197 | 1228 | * want them. |
---|
1198 | 1229 | */ |
---|
1199 | | -ENTRY(__sdei_asm_handler) |
---|
| 1230 | +SYM_CODE_START(__sdei_asm_handler) |
---|
1200 | 1231 | stp x2, x3, [x1, #SDEI_EVENT_INTREGS + S_PC] |
---|
1201 | 1232 | stp x4, x5, [x1, #SDEI_EVENT_INTREGS + 16 * 2] |
---|
1202 | 1233 | stp x6, x7, [x1, #SDEI_EVENT_INTREGS + 16 * 3] |
---|
.. | .. |
---|
1216 | 1247 | |
---|
1217 | 1248 | mov x19, x1 |
---|
1218 | 1249 | |
---|
| 1250 | +#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK) |
---|
| 1251 | + ldrb w4, [x19, #SDEI_EVENT_PRIORITY] |
---|
| 1252 | +#endif |
---|
| 1253 | + |
---|
1219 | 1254 | #ifdef CONFIG_VMAP_STACK |
---|
1220 | 1255 | /* |
---|
1221 | 1256 | * entry.S may have been using sp as a scratch register, find whether |
---|
1222 | 1257 | * this is a normal or critical event and switch to the appropriate |
---|
1223 | 1258 | * stack for this CPU. |
---|
1224 | 1259 | */ |
---|
1225 | | - ldrb w4, [x19, #SDEI_EVENT_PRIORITY] |
---|
1226 | 1260 | cbnz w4, 1f |
---|
1227 | 1261 | ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6 |
---|
1228 | 1262 | b 2f |
---|
.. | .. |
---|
1230 | 1264 | 2: mov x6, #SDEI_STACK_SIZE |
---|
1231 | 1265 | add x5, x5, x6 |
---|
1232 | 1266 | mov sp, x5 |
---|
| 1267 | +#endif |
---|
| 1268 | + |
---|
| 1269 | +#ifdef CONFIG_SHADOW_CALL_STACK |
---|
| 1270 | + /* Use a separate shadow call stack for normal and critical events */ |
---|
| 1271 | + cbnz w4, 3f |
---|
| 1272 | + ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6 |
---|
| 1273 | + b 4f |
---|
| 1274 | +3: ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6 |
---|
| 1275 | +4: |
---|
1233 | 1276 | #endif |
---|
1234 | 1277 | |
---|
1235 | 1278 | /* |
---|
.. | .. |
---|
1276 | 1319 | alternative_else_nop_endif |
---|
1277 | 1320 | |
---|
1278 | 1321 | #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
---|
1279 | | - tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline |
---|
| 1322 | + tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3 |
---|
1280 | 1323 | br x5 |
---|
1281 | 1324 | #endif |
---|
1282 | | -ENDPROC(__sdei_asm_handler) |
---|
| 1325 | +SYM_CODE_END(__sdei_asm_handler) |
---|
1283 | 1326 | NOKPROBE(__sdei_asm_handler) |
---|
1284 | 1327 | #endif /* CONFIG_ARM_SDE_INTERFACE */ |
---|