hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d
kernel/arch/arm/kernel/entry-armv.S
....@@ -1,13 +1,10 @@
1
+/* SPDX-License-Identifier: GPL-2.0-only */
12 /*
23 * linux/arch/arm/kernel/entry-armv.S
34 *
45 * Copyright (C) 1996,1997,1998 Russell King.
56 * ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk)
67 * nommu support by Hyok S. Choi (hyok.choi@samsung.com)
7
- *
8
- * This program is free software; you can redistribute it and/or modify
9
- * it under the terms of the GNU General Public License version 2 as
10
- * published by the Free Software Foundation.
118 *
129 * Low-level vector interface routines
1310 *
....@@ -207,13 +204,20 @@
207204 svc_entry
208205 irq_handler
209206
210
-#ifdef CONFIG_PREEMPT
207
+#ifdef CONFIG_PREEMPTION
211208 ldr r8, [tsk, #TI_PREEMPT] @ get preempt count
212
- ldr r0, [tsk, #TI_FLAGS] @ get flags
213209 teq r8, #0 @ if preempt count != 0
210
+ bne 1f @ return from exeption
211
+ ldr r0, [tsk, #TI_FLAGS] @ get flags
212
+ tst r0, #_TIF_NEED_RESCHED @ if NEED_RESCHED is set
213
+ blne svc_preempt @ preempt!
214
+
215
+ ldr r8, [tsk, #TI_PREEMPT_LAZY] @ get preempt lazy count
216
+ teq r8, #0 @ if preempt lazy count != 0
214217 movne r0, #0 @ force flags to 0
215
- tst r0, #_TIF_NEED_RESCHED
218
+ tst r0, #_TIF_NEED_RESCHED_LAZY
216219 blne svc_preempt
220
+1:
217221 #endif
218222
219223 svc_exit r5, irq = 1 @ return from exception
....@@ -222,14 +226,20 @@
222226
223227 .ltorg
224228
225
-#ifdef CONFIG_PREEMPT
229
+#ifdef CONFIG_PREEMPTION
226230 svc_preempt:
227231 mov r8, lr
228232 1: bl preempt_schedule_irq @ irq en/disable is done inside
229233 ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS
230234 tst r0, #_TIF_NEED_RESCHED
235
+ bne 1b
236
+ tst r0, #_TIF_NEED_RESCHED_LAZY
231237 reteq r8 @ go again
232
- b 1b
238
+ ldr r0, [tsk, #TI_PREEMPT_LAZY] @ get preempt lazy count
239
+ teq r0, #0 @ if preempt lazy count != 0
240
+ beq 1b
241
+ ret r8 @ go again
242
+
233243 #endif
234244
235245 __und_fault:
....@@ -255,31 +265,10 @@
255265 #else
256266 svc_entry
257267 #endif
258
- @
259
- @ call emulation code, which returns using r9 if it has emulated
260
- @ the instruction, or the more conventional lr if we are to treat
261
- @ this as a real undefined instruction
262
- @
263
- @ r0 - instruction
264
- @
265
-#ifndef CONFIG_THUMB2_KERNEL
266
- ldr r0, [r4, #-4]
267
-#else
268
- mov r1, #2
269
- ldrh r0, [r4, #-2] @ Thumb instruction at LR - 2
270
- cmp r0, #0xe800 @ 32-bit instruction if xx >= 0
271
- blo __und_svc_fault
272
- ldrh r9, [r4] @ bottom 16 bits
273
- add r4, r4, #2
274
- str r4, [sp, #S_PC]
275
- orr r0, r9, r0, lsl #16
276
-#endif
277
- badr r9, __und_svc_finish
278
- mov r2, r4
279
- bl call_fpe
280268
281269 mov r1, #4 @ PC correction to apply
282
-__und_svc_fault:
270
+ THUMB( tst r5, #PSR_T_BIT ) @ exception taken in Thumb mode?
271
+ THUMB( movne r1, #2 ) @ if so, fix up PC correction
283272 mov r0, sp @ struct pt_regs *regs
284273 bl __und_fault
285274
....@@ -627,7 +616,7 @@
627616 @ Test if we need to give access to iWMMXt coprocessors
628617 ldr r5, [r10, #TI_FLAGS]
629618 rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only
630
- movcss r7, r5, lsr #(TIF_USING_IWMMXT + 1)
619
+ movscs r7, r5, lsr #(TIF_USING_IWMMXT + 1)
631620 bcs iwmmxt_task_enable
632621 #endif
633622 ARM( add pc, pc, r8, lsr #6 )
....@@ -820,7 +809,7 @@
820809 * existing ones. This mechanism should be used only for things that are
821810 * really small and justified, and not be abused freely.
822811 *
823
- * See Documentation/arm/kernel_user_helpers.txt for formal definitions.
812
+ * See Documentation/arm/kernel_user_helpers.rst for formal definitions.
824813 */
825814 THUMB( .arm )
826815
....@@ -863,7 +852,7 @@
863852 smp_dmb arm
864853 1: ldrexd r0, r1, [r2] @ load current val
865854 eors r3, r0, r4 @ compare with oldval (1)
866
- eoreqs r3, r1, r5 @ compare with oldval (2)
855
+ eorseq r3, r1, r5 @ compare with oldval (2)
867856 strexdeq r3, r6, r7, [r2] @ store newval if eq
868857 teqeq r3, #1 @ success?
869858 beq 1b @ if no then retry
....@@ -887,8 +876,8 @@
887876 ldmia r1, {r6, lr} @ load new val
888877 1: ldmia r2, {r0, r1} @ load current val
889878 eors r3, r0, r4 @ compare with oldval (1)
890
- eoreqs r3, r1, r5 @ compare with oldval (2)
891
-2: stmeqia r2, {r6, lr} @ store newval if eq
879
+ eorseq r3, r1, r5 @ compare with oldval (2)
880
+2: stmiaeq r2, {r6, lr} @ store newval if eq
892881 rsbs r0, r3, #0 @ set return val and C flag
893882 ldmfd sp!, {r4, r5, r6, pc}
894883
....@@ -902,7 +891,7 @@
902891 mov r7, #0xffff0fff
903892 sub r7, r7, #(0xffff0fff - (0xffff0f60 + (1b - __kuser_cmpxchg64)))
904893 subs r8, r4, r7
905
- rsbcss r8, r8, #(2b - 1b)
894
+ rsbscs r8, r8, #(2b - 1b)
906895 strcs r7, [sp, #S_PC]
907896 #if __LINUX_ARM_ARCH__ < 6
908897 bcc kuser_cmpxchg32_fixup
....@@ -960,7 +949,7 @@
960949 mov r7, #0xffff0fff
961950 sub r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg)))
962951 subs r8, r4, r7
963
- rsbcss r8, r8, #(2b - 1b)
952
+ rsbscs r8, r8, #(2b - 1b)
964953 strcs r7, [sp, #S_PC]
965954 ret lr
966955 .previous
....@@ -1029,12 +1018,11 @@
10291018 sub lr, lr, #\correction
10301019 .endif
10311020
1032
- @
1033
- @ Save r0, lr_<exception> (parent PC) and spsr_<exception>
1034
- @ (parent CPSR)
1035
- @
1021
+ @ Save r0, lr_<exception> (parent PC)
10361022 stmia sp, {r0, lr} @ save r0, lr
1037
- mrs lr, spsr
1023
+
1024
+ @ Save spsr_<exception> (parent CPSR)
1025
+2: mrs lr, spsr
10381026 str lr, [sp, #8] @ save spsr
10391027
10401028 @
....@@ -1055,6 +1043,44 @@
10551043 movs pc, lr @ branch to handler in SVC mode
10561044 ENDPROC(vector_\name)
10571045
1046
+#ifdef CONFIG_HARDEN_BRANCH_HISTORY
1047
+ .subsection 1
1048
+ .align 5
1049
+vector_bhb_loop8_\name:
1050
+ .if \correction
1051
+ sub lr, lr, #\correction
1052
+ .endif
1053
+
1054
+ @ Save r0, lr_<exception> (parent PC)
1055
+ stmia sp, {r0, lr}
1056
+
1057
+ @ bhb workaround
1058
+ mov r0, #8
1059
+3: W(b) . + 4
1060
+ subs r0, r0, #1
1061
+ bne 3b
1062
+ dsb
1063
+ isb
1064
+ b 2b
1065
+ENDPROC(vector_bhb_loop8_\name)
1066
+
1067
+vector_bhb_bpiall_\name:
1068
+ .if \correction
1069
+ sub lr, lr, #\correction
1070
+ .endif
1071
+
1072
+ @ Save r0, lr_<exception> (parent PC)
1073
+ stmia sp, {r0, lr}
1074
+
1075
+ @ bhb workaround
1076
+ mcr p15, 0, r0, c7, c5, 6 @ BPIALL
1077
+ @ isb not needed due to "movs pc, lr" in the vector stub
1078
+ @ which gives a "context synchronisation".
1079
+ b 2b
1080
+ENDPROC(vector_bhb_bpiall_\name)
1081
+ .previous
1082
+#endif
1083
+
10581084 .align 2
10591085 @ handler addresses follow this label
10601086 1:
....@@ -1063,6 +1089,10 @@
10631089 .section .stubs, "ax", %progbits
10641090 @ This must be the first word
10651091 .word vector_swi
1092
+#ifdef CONFIG_HARDEN_BRANCH_HISTORY
1093
+ .word vector_bhb_loop8_swi
1094
+ .word vector_bhb_bpiall_swi
1095
+#endif
10661096
10671097 vector_rst:
10681098 ARM( swi SYS_ERROR0 )
....@@ -1177,8 +1207,10 @@
11771207 * FIQ "NMI" handler
11781208 *-----------------------------------------------------------------------------
11791209 * Handle a FIQ using the SVC stack allowing FIQ act like NMI on x86
1180
- * systems.
1210
+ * systems. This must be the last vector stub, so lets place it in its own
1211
+ * subsection.
11811212 */
1213
+ .subsection 2
11821214 vector_stub fiq, FIQ_MODE, 4
11831215
11841216 .long __fiq_usr @ 0 (USR_26 / USR_32)
....@@ -1211,6 +1243,30 @@
12111243 W(b) vector_irq
12121244 W(b) vector_fiq
12131245
1246
+#ifdef CONFIG_HARDEN_BRANCH_HISTORY
1247
+ .section .vectors.bhb.loop8, "ax", %progbits
1248
+.L__vectors_bhb_loop8_start:
1249
+ W(b) vector_rst
1250
+ W(b) vector_bhb_loop8_und
1251
+ W(ldr) pc, .L__vectors_bhb_loop8_start + 0x1004
1252
+ W(b) vector_bhb_loop8_pabt
1253
+ W(b) vector_bhb_loop8_dabt
1254
+ W(b) vector_addrexcptn
1255
+ W(b) vector_bhb_loop8_irq
1256
+ W(b) vector_bhb_loop8_fiq
1257
+
1258
+ .section .vectors.bhb.bpiall, "ax", %progbits
1259
+.L__vectors_bhb_bpiall_start:
1260
+ W(b) vector_rst
1261
+ W(b) vector_bhb_bpiall_und
1262
+ W(ldr) pc, .L__vectors_bhb_bpiall_start + 0x1008
1263
+ W(b) vector_bhb_bpiall_pabt
1264
+ W(b) vector_bhb_bpiall_dabt
1265
+ W(b) vector_addrexcptn
1266
+ W(b) vector_bhb_bpiall_irq
1267
+ W(b) vector_bhb_bpiall_fiq
1268
+#endif
1269
+
12141270 .data
12151271 .align 2
12161272