forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-10 23fa18eaa71266feff7ba8d83022d9e1cc83c65a 
 kernel/arch/arm64/mm/proc.S
....@@ -1,32 +1,24 @@
1
+/* SPDX-License-Identifier: GPL-2.0-only */
12 /*
23  * Based on arch/arm/mm/proc.S
34  *
45  * Copyright (C) 2001 Deep Blue Solutions Ltd.
56  * Copyright (C) 2012 ARM Ltd.
67  * Author: Catalin Marinas <catalin.marinas@arm.com>
7
- *
8
- * This program is free software; you can redistribute it and/or modify
9
- * it under the terms of the GNU General Public License version 2 as
10
- * published by the Free Software Foundation.
11
- *
12
- * This program is distributed in the hope that it will be useful,
13
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
14
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15
- * GNU General Public License for more details.
16
- *
17
- * You should have received a copy of the GNU General Public License
18
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
198  */
209 
2110 #include <linux/init.h>
2211 #include <linux/linkage.h>
12
+#include <linux/pgtable.h>
2313 #include <asm/assembler.h>
2414 #include <asm/asm-offsets.h>
15
+#include <asm/asm_pointer_auth.h>
2516 #include <asm/hwcap.h>
26
-#include <asm/pgtable.h>
2717 #include <asm/pgtable-hwdef.h>
2818 #include <asm/cpufeature.h>
2919 #include <asm/alternative.h>
20
+#include <asm/smp.h>
21
+#include <asm/sysreg.h>
3022 
3123 #ifdef CONFIG_ARM64_64K_PAGES
3224 #define TCR_TG_FLAGS	TCR_TG0_64K | TCR_TG1_64K
....@@ -48,54 +40,36 @@
4840 #define TCR_CACHE_FLAGS	TCR_IRGN_WBWA | TCR_ORGN_WBWA
4941 
5042 #ifdef CONFIG_KASAN_SW_TAGS
51
-#define TCR_KASAN_FLAGS TCR_TBI1
43
+#define TCR_KASAN_SW_FLAGS TCR_TBI1 | TCR_TBID1
5244 #else
53
-#define TCR_KASAN_FLAGS 0
45
+#define TCR_KASAN_SW_FLAGS 0
5446 #endif
5547 
56
-#define MAIR(attr, mt)	((attr) << ((mt) * 8))
48
+#ifdef CONFIG_KASAN_HW_TAGS
49
+#define TCR_MTE_FLAGS SYS_TCR_EL1_TCMA1 | TCR_TBI1 | TCR_TBID1
50
+#elif defined(CONFIG_ARM64_MTE)
51
+/*
52
+ * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on
53
+ * TBI being enabled at EL1.
54
+ */
55
+#define TCR_MTE_FLAGS TCR_TBI1 | TCR_TBID1
56
+#else
57
+#define TCR_MTE_FLAGS 0
58
+#endif
5759 
5860 /*
59
- *	cpu_cache_off()
60
- *
61
- *	Turn the CPU D-cache off.
61
+ * Default MAIR_EL1. MT_NORMAL_TAGGED is initially mapped as Normal memory and
62
+ * changed during mte_cpu_setup to Normal Tagged if the system supports MTE.
6263  */
63
-ENTRY(cpu_cache_off)
64
-	mrs	x0, sctlr_el1
65
-	bic	x0, x0, #1 << 2			// clear SCTLR.C
66
-	msr	sctlr_el1, x0
67
-	isb
68
-	ret
69
-ENDPROC(cpu_cache_off)
70
-
71
-/*
72
- *	cpu_reset(loc)
73
- *
74
- *	Perform a soft reset of the system.  Put the CPU into the same state
75
- *	as it would be if it had been reset, and branch to what would be the
76
- *	reset vector. It must be executed with the flat identity mapping.
77
- *
78
- *	- loc   - location to jump to for soft reset
79
- */
80
-	.align	5
81
-ENTRY(cpu_reset)
82
-	mrs	x1, sctlr_el1
83
-	bic	x1, x1, #1
84
-	msr	sctlr_el1, x1			// disable the MMU
85
-	isb
86
-	ret	x0
87
-ENDPROC(cpu_reset)
88
-
89
-/*
90
- *	cpu_do_idle()
91
- *
92
- *	Idle the processor (wait for interrupt).
93
- */
94
-ENTRY(cpu_do_idle)
95
-	dsb	sy				// WFI may enter a low-power mode
96
-	wfi
97
-	ret
98
-ENDPROC(cpu_do_idle)
64
+#define MAIR_EL1_SET							\
65
+	(MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) |	\
66
+	 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) |	\
67
+	 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_GRE, MT_DEVICE_GRE) |		\
68
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) |		\
69
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) |			\
70
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_WT, MT_NORMAL_WT) |		\
71
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED) |		\
72
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_iNC_oWB, MT_NORMAL_iNC_oWB))
9973 
10074 #ifdef CONFIG_CPU_PM
10175 /**
....@@ -105,7 +79,7 @@
10579  *
10680  * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
10781  */
108
-ENTRY(cpu_do_suspend)
82
+SYM_FUNC_START(cpu_do_suspend)
10983 	mrs	x2, tpidr_el0
11084 	mrs	x3, tpidrro_el0
11185 	mrs	x4, contextidr_el1
....@@ -134,7 +108,7 @@
134108 	 */
135109 	str	x18, [x0, #96]
136110 	ret
137
-ENDPROC(cpu_do_suspend)
111
+SYM_FUNC_END(cpu_do_suspend)
138112 
139113 /**
140114  * cpu_do_resume - restore CPU register context
....@@ -142,7 +116,7 @@
142116  * x0: Address of context pointer
143117  */
144118 	.pushsection ".idmap.text", "awx"
145
-ENTRY(cpu_do_resume)
119
+SYM_FUNC_START(cpu_do_resume)
146120 	ldp	x2, x3, [x0]
147121 	ldp	x4, x5, [x0, #16]
148122 	ldp	x6, x8, [x0, #32]
....@@ -191,44 +165,25 @@
191165 	ubfx	x11, x11, #1, #1
192166 	msr	oslar_el1, x11
193167 	reset_pmuserenr_el0 x0			// Disable PMU access from EL0
168
+	reset_amuserenr_el0 x0			// Disable AMU access from EL0
194169 
195170 alternative_if ARM64_HAS_RAS_EXTN
196171 	msr_s	SYS_DISR_EL1, xzr
197172 alternative_else_nop_endif
198173 
174
+	ptrauth_keys_install_kernel_nosync x14, x1, x2, x3
199175 	isb
200176 	ret
201
-ENDPROC(cpu_do_resume)
177
+SYM_FUNC_END(cpu_do_resume)
202178 	.popsection
203179 #endif
204
-
205
-/*
206
- *	cpu_do_switch_mm(pgd_phys, tsk)
207
- *
208
- *	Set the translation table base pointer to be pgd_phys.
209
- *
210
- *	- pgd_phys - physical address of new TTB
211
- */
212
-ENTRY(cpu_do_switch_mm)
213
-	mrs	x2, ttbr1_el1
214
-	mmid	x1, x1				// get mm->context.id
215
-	phys_to_ttbr x3, x0
216
-#ifdef CONFIG_ARM64_SW_TTBR0_PAN
217
-	bfi	x3, x1, #48, #16		// set the ASID field in TTBR0
218
-#endif
219
-	bfi	x2, x1, #48, #16		// set the ASID
220
-	msr	ttbr1_el1, x2			// in TTBR1 (since TCR.A1 is set)
221
-	isb
222
-	msr	ttbr0_el1, x3			// now update TTBR0
223
-	isb
224
-	b	post_ttbr_update_workaround	// Back to C code...
225
-ENDPROC(cpu_do_switch_mm)
226180 
227181 	.pushsection ".idmap.text", "awx"
228182 
229183 .macro	__idmap_cpu_set_reserved_ttbr1, tmp1, tmp2
230
-	adrp	\tmp1, empty_zero_page
184
+	adrp	\tmp1, reserved_pg_dir
231185 	phys_to_ttbr \tmp2, \tmp1
186
+	offset_ttbr1 \tmp2, \tmp1
232187 	msr	ttbr1_el1, \tmp2
233188 	isb
234189 	tlbi	vmalle1
....@@ -237,24 +192,24 @@
237192 .endm
238193 
239194 /*
240
- * void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd)
195
+ * void idmap_cpu_replace_ttbr1(phys_addr_t ttbr1)
241196  *
242197  * This is the low-level counterpart to cpu_replace_ttbr1, and should not be
243198  * called by anything else. It can only be executed from a TTBR0 mapping.
244199  */
245
-ENTRY(idmap_cpu_replace_ttbr1)
200
+SYM_FUNC_START(idmap_cpu_replace_ttbr1)
246201 	save_and_disable_daif flags=x2
247202 
248203 	__idmap_cpu_set_reserved_ttbr1 x1, x3
249204 
250
-	phys_to_ttbr x3, x0
251
-	msr	ttbr1_el1, x3
205
+	offset_ttbr1 x0, x3
206
+	msr	ttbr1_el1, x0
252207 	isb
253208 
254209 	restore_daif x2
255210 
256211 	ret
257
-ENDPROC(idmap_cpu_replace_ttbr1)
212
+SYM_FUNC_END(idmap_cpu_replace_ttbr1)
258213 	.popsection
259214 
260215 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
....@@ -282,7 +237,7 @@
282237  */
283238 __idmap_kpti_flag:
284239 	.long	1
285
-ENTRY(idmap_kpti_install_ng_mappings)
240
+SYM_FUNC_START(idmap_kpti_install_ng_mappings)
286241 	cpu		.req	w0
287242 	num_cpus	.req	w1
288243 	swapper_pa	.req	x2
....@@ -302,6 +257,7 @@
302257 	pte		.req	x16
303258 
304259 	mrs	swapper_ttb, ttbr1_el1
260
+	restore_ttbr1	swapper_ttb
305261 	adr	flag_ptr, __idmap_kpti_flag
306262 
307263 	cbnz	cpu, __idmap_kpti_secondary
....@@ -342,17 +298,7 @@
342298 	/* We're done: fire up the MMU again */
343299 	mrs	x17, sctlr_el1
344300 	orr	x17, x17, #SCTLR_ELx_M
345
-	msr	sctlr_el1, x17
346
-	isb
347
-
348
-	/*
349
-	 * Invalidate the local I-cache so that any instructions fetched
350
-	 * speculatively from the PoC are discarded, since they may have
351
-	 * been dynamically patched at the PoU.
352
-	 */
353
-	ic	iallu
354
-	dsb	nsh
355
-	isb
301
+	set_sctlr_el1	x17
356302 
357303 	/* Set the flag to zero to indicate that we're all done */
358304 	str	wzr, [flag_ptr]
....@@ -446,65 +392,63 @@
446392 	cbnz	w16, 1b
447393 
448394 	/* All done, act like nothing happened */
395
+	offset_ttbr1 swapper_ttb, x16
449396 	msr	ttbr1_el1, swapper_ttb
450397 	isb
451398 	ret
452399 
453400 	.unreq	swapper_ttb
454401 	.unreq	flag_ptr
455
-ENDPROC(idmap_kpti_install_ng_mappings)
402
+SYM_FUNC_END(idmap_kpti_install_ng_mappings)
456403 	.popsection
457404 #endif
458405 
459406 /*
460407  *	__cpu_setup
461408  *
462
- *	Initialise the processor for turning the MMU on.  Return in x0 the
463
- *	value of the SCTLR_EL1 register.
409
+ *	Initialise the processor for turning the MMU on.
410
+ *
411
+ * Output:
412
+ *	Return in x0 the value of the SCTLR_EL1 register.
464413  */
465414 	.pushsection ".idmap.text", "awx"
466
-ENTRY(__cpu_setup)
415
+SYM_FUNC_START(__cpu_setup)
467416 	tlbi	vmalle1				// Invalidate local TLB
468417 	dsb	nsh
469418 
470
-	mov	x0, #3 << 20
471
-	msr	cpacr_el1, x0			// Enable FP/ASIMD
472
-	mov	x0, #1 << 12			// Reset mdscr_el1 and disable
473
-	msr	mdscr_el1, x0			// access to the DCC from EL0
419
+	mov	x1, #3 << 20
420
+	msr	cpacr_el1, x1			// Enable FP/ASIMD
421
+	mov	x1, #1 << 12			// Reset mdscr_el1 and disable
422
+	msr	mdscr_el1, x1			// access to the DCC from EL0
474423 	isb					// Unmask debug exceptions now,
475424 	enable_dbg				// since this is per-cpu
476
-	reset_pmuserenr_el0 x0			// Disable PMU access from EL0
425
+	reset_pmuserenr_el0 x1			// Disable PMU access from EL0
426
+	reset_amuserenr_el0 x1			// Disable AMU access from EL0
427
+
477428 	/*
478
-	 * Memory region attributes for LPAE:
479
-	 *
480
-	 *   n = AttrIndx[2:0]
481
-	 *			n	MAIR
482
-	 *   DEVICE_nGnRnE	000	00000000
483
-	 *   DEVICE_nGnRE	001	00000100
484
-	 *   DEVICE_GRE		010	00001100
485
-	 *   NORMAL_NC		011	01000100
486
-	 *   NORMAL		100	11111111
487
-	 *   NORMAL_WT		101	10111011
429
+	 * Memory region attributes
488430 	 */
489
-	ldr	x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \
490
-		     MAIR(0x04, MT_DEVICE_nGnRE) | \
491
-		     MAIR(0x0c, MT_DEVICE_GRE) | \
492
-		     MAIR(0x44, MT_NORMAL_NC) | \
493
-		     MAIR(0xff, MT_NORMAL) | \
494
-		     MAIR(0xbb, MT_NORMAL_WT)
431
+	mov_q	x5, MAIR_EL1_SET
495432 	msr	mair_el1, x5
496433 	/*
497
-	 * Prepare SCTLR
434
+	 * Set/prepare TCR and TTBR. TCR_EL1.T1SZ gets further
435
+	 * adjusted if the kernel is compiled with 52bit VA support.
498436 	 */
499
-	mov_q	x0, SCTLR_EL1_SET
500
-	/*
501
-	 * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
502
-	 * both user and kernel.
503
-	 */
504
-	ldr	x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
437
+	mov_q	x10, TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
505438 			TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \
506
-			TCR_TBI0 | TCR_A1 | TCR_KASAN_FLAGS
507
-	tcr_set_idmap_t0sz	x10, x9
439
+			TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS
440
+
441
+	tcr_clear_errata_bits x10, x9, x5
442
+
443
+#ifdef CONFIG_ARM64_VA_BITS_52
444
+	ldr_l		x9, vabits_actual
445
+	sub		x9, xzr, x9
446
+	add		x9, x9, #64
447
+	tcr_set_t1sz	x10, x9
448
+#else
449
+	ldr_l		x9, idmap_t0sz
450
+#endif
451
+	tcr_set_t0sz	x10, x9
508452 
509453 	/*
510454 	 * Set the IPS bits in TCR_EL1.
....@@ -523,5 +467,9 @@
523467 1:
524468 #endif	/* CONFIG_ARM64_HW_AFDBM */
525469 	msr	tcr_el1, x10
470
+	/*
471
+	 * Prepare SCTLR
472
+	 */
473
+	mov_q	x0, INIT_SCTLR_EL1_MMU_ON
526474 	ret					// return to head.S
527
-ENDPROC(__cpu_setup)
475
+SYM_FUNC_END(__cpu_setup)