forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-09 b22da3d8526a935aa31e086e63f60ff3246cb61c 
 kernel/arch/arm64/mm/proc.S
....@@ -1,32 +1,24 @@
1
+/* SPDX-License-Identifier: GPL-2.0-only */
12 /*
23  * Based on arch/arm/mm/proc.S
34  *
45  * Copyright (C) 2001 Deep Blue Solutions Ltd.
56  * Copyright (C) 2012 ARM Ltd.
67  * Author: Catalin Marinas <catalin.marinas@arm.com>
7
- *
8
- * This program is free software; you can redistribute it and/or modify
9
- * it under the terms of the GNU General Public License version 2 as
10
- * published by the Free Software Foundation.
11
- *
12
- * This program is distributed in the hope that it will be useful,
13
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
14
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15
- * GNU General Public License for more details.
16
- *
17
- * You should have received a copy of the GNU General Public License
18
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
198  */
209 
2110 #include <linux/init.h>
2211 #include <linux/linkage.h>
12
+#include <linux/pgtable.h>
2313 #include <asm/assembler.h>
2414 #include <asm/asm-offsets.h>
15
+#include <asm/asm_pointer_auth.h>
2516 #include <asm/hwcap.h>
26
-#include <asm/pgtable.h>
2717 #include <asm/pgtable-hwdef.h>
2818 #include <asm/cpufeature.h>
2919 #include <asm/alternative.h>
20
+#include <asm/smp.h>
21
+#include <asm/sysreg.h>
3022 
3123 #ifdef CONFIG_ARM64_64K_PAGES
3224 #define TCR_TG_FLAGS	TCR_TG0_64K | TCR_TG1_64K
....@@ -48,54 +40,34 @@
4840 #define TCR_CACHE_FLAGS	TCR_IRGN_WBWA | TCR_ORGN_WBWA
4941 
5042 #ifdef CONFIG_KASAN_SW_TAGS
51
-#define TCR_KASAN_FLAGS TCR_TBI1
43
+#define TCR_KASAN_SW_FLAGS TCR_TBI1 | TCR_TBID1
5244 #else
53
-#define TCR_KASAN_FLAGS 0
45
+#define TCR_KASAN_SW_FLAGS 0
5446 #endif
5547 
56
-#define MAIR(attr, mt)	((attr) << ((mt) * 8))
48
+#ifdef CONFIG_KASAN_HW_TAGS
49
+#define TCR_MTE_FLAGS SYS_TCR_EL1_TCMA1 | TCR_TBI1 | TCR_TBID1
50
+#else
51
+/*
52
+ * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on
53
+ * TBI being enabled at EL1.
54
+ */
55
+#define TCR_MTE_FLAGS TCR_TBI1 | TCR_TBID1
56
+#endif
5757 
5858 /*
59
- *	cpu_cache_off()
60
- *
61
- *	Turn the CPU D-cache off.
59
+ * Default MAIR_EL1. MT_NORMAL_TAGGED is initially mapped as Normal memory and
60
+ * changed during __cpu_setup to Normal Tagged if the system supports MTE.
6261  */
63
-ENTRY(cpu_cache_off)
64
-	mrs	x0, sctlr_el1
65
-	bic	x0, x0, #1 << 2			// clear SCTLR.C
66
-	msr	sctlr_el1, x0
67
-	isb
68
-	ret
69
-ENDPROC(cpu_cache_off)
70
-
71
-/*
72
- *	cpu_reset(loc)
73
- *
74
- *	Perform a soft reset of the system.  Put the CPU into the same state
75
- *	as it would be if it had been reset, and branch to what would be the
76
- *	reset vector. It must be executed with the flat identity mapping.
77
- *
78
- *	- loc   - location to jump to for soft reset
79
- */
80
-	.align	5
81
-ENTRY(cpu_reset)
82
-	mrs	x1, sctlr_el1
83
-	bic	x1, x1, #1
84
-	msr	sctlr_el1, x1			// disable the MMU
85
-	isb
86
-	ret	x0
87
-ENDPROC(cpu_reset)
88
-
89
-/*
90
- *	cpu_do_idle()
91
- *
92
- *	Idle the processor (wait for interrupt).
93
- */
94
-ENTRY(cpu_do_idle)
95
-	dsb	sy				// WFI may enter a low-power mode
96
-	wfi
97
-	ret
98
-ENDPROC(cpu_do_idle)
62
+#define MAIR_EL1_SET							\
63
+	(MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) |	\
64
+	 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) |	\
65
+	 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_GRE, MT_DEVICE_GRE) |		\
66
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) |		\
67
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) |			\
68
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_WT, MT_NORMAL_WT) |		\
69
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED) |		\
70
+	 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_iNC_oWB, MT_NORMAL_iNC_oWB))
9971 
10072 #ifdef CONFIG_CPU_PM
10173 /**
....@@ -105,7 +77,7 @@
10577  *
10678  * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
10779  */
108
-ENTRY(cpu_do_suspend)
80
+SYM_FUNC_START(cpu_do_suspend)
10981 	mrs	x2, tpidr_el0
11082 	mrs	x3, tpidrro_el0
11183 	mrs	x4, contextidr_el1
....@@ -134,7 +106,7 @@
134106 	 */
135107 	str	x18, [x0, #96]
136108 	ret
137
-ENDPROC(cpu_do_suspend)
109
+SYM_FUNC_END(cpu_do_suspend)
138110 
139111 /**
140112  * cpu_do_resume - restore CPU register context
....@@ -142,7 +114,7 @@
142114  * x0: Address of context pointer
143115  */
144116 	.pushsection ".idmap.text", "awx"
145
-ENTRY(cpu_do_resume)
117
+SYM_FUNC_START(cpu_do_resume)
146118 	ldp	x2, x3, [x0]
147119 	ldp	x4, x5, [x0, #16]
148120 	ldp	x6, x8, [x0, #32]
....@@ -191,44 +163,25 @@
191163 	ubfx	x11, x11, #1, #1
192164 	msr	oslar_el1, x11
193165 	reset_pmuserenr_el0 x0			// Disable PMU access from EL0
166
+	reset_amuserenr_el0 x0			// Disable AMU access from EL0
194167 
195168 alternative_if ARM64_HAS_RAS_EXTN
196169 	msr_s	SYS_DISR_EL1, xzr
197170 alternative_else_nop_endif
198171 
172
+	ptrauth_keys_install_kernel_nosync x14, x1, x2, x3
199173 	isb
200174 	ret
201
-ENDPROC(cpu_do_resume)
175
+SYM_FUNC_END(cpu_do_resume)
202176 	.popsection
203177 #endif
204
-
205
-/*
206
- *	cpu_do_switch_mm(pgd_phys, tsk)
207
- *
208
- *	Set the translation table base pointer to be pgd_phys.
209
- *
210
- *	- pgd_phys - physical address of new TTB
211
- */
212
-ENTRY(cpu_do_switch_mm)
213
-	mrs	x2, ttbr1_el1
214
-	mmid	x1, x1				// get mm->context.id
215
-	phys_to_ttbr x3, x0
216
-#ifdef CONFIG_ARM64_SW_TTBR0_PAN
217
-	bfi	x3, x1, #48, #16		// set the ASID field in TTBR0
218
-#endif
219
-	bfi	x2, x1, #48, #16		// set the ASID
220
-	msr	ttbr1_el1, x2			// in TTBR1 (since TCR.A1 is set)
221
-	isb
222
-	msr	ttbr0_el1, x3			// now update TTBR0
223
-	isb
224
-	b	post_ttbr_update_workaround	// Back to C code...
225
-ENDPROC(cpu_do_switch_mm)
226178 
227179 	.pushsection ".idmap.text", "awx"
228180 
229181 .macro	__idmap_cpu_set_reserved_ttbr1, tmp1, tmp2
230
-	adrp	\tmp1, empty_zero_page
182
+	adrp	\tmp1, reserved_pg_dir
231183 	phys_to_ttbr \tmp2, \tmp1
184
+	offset_ttbr1 \tmp2, \tmp1
232185 	msr	ttbr1_el1, \tmp2
233186 	isb
234187 	tlbi	vmalle1
....@@ -237,24 +190,24 @@
237190 .endm
238191 
239192 /*
240
- * void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd)
193
+ * void idmap_cpu_replace_ttbr1(phys_addr_t ttbr1)
241194  *
242195  * This is the low-level counterpart to cpu_replace_ttbr1, and should not be
243196  * called by anything else. It can only be executed from a TTBR0 mapping.
244197  */
245
-ENTRY(idmap_cpu_replace_ttbr1)
198
+SYM_FUNC_START(idmap_cpu_replace_ttbr1)
246199 	save_and_disable_daif flags=x2
247200 
248201 	__idmap_cpu_set_reserved_ttbr1 x1, x3
249202 
250
-	phys_to_ttbr x3, x0
251
-	msr	ttbr1_el1, x3
203
+	offset_ttbr1 x0, x3
204
+	msr	ttbr1_el1, x0
252205 	isb
253206 
254207 	restore_daif x2
255208 
256209 	ret
257
-ENDPROC(idmap_cpu_replace_ttbr1)
210
+SYM_FUNC_END(idmap_cpu_replace_ttbr1)
258211 	.popsection
259212 
260213 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
....@@ -282,7 +235,7 @@
282235  */
283236 __idmap_kpti_flag:
284237 	.long	1
285
-ENTRY(idmap_kpti_install_ng_mappings)
238
+SYM_FUNC_START(idmap_kpti_install_ng_mappings)
286239 	cpu		.req	w0
287240 	num_cpus	.req	w1
288241 	swapper_pa	.req	x2
....@@ -302,6 +255,7 @@
302255 	pte		.req	x16
303256 
304257 	mrs	swapper_ttb, ttbr1_el1
258
+	restore_ttbr1	swapper_ttb
305259 	adr	flag_ptr, __idmap_kpti_flag
306260 
307261 	cbnz	cpu, __idmap_kpti_secondary
....@@ -342,17 +296,7 @@
342296 	/* We're done: fire up the MMU again */
343297 	mrs	x17, sctlr_el1
344298 	orr	x17, x17, #SCTLR_ELx_M
345
-	msr	sctlr_el1, x17
346
-	isb
347
-
348
-	/*
349
-	 * Invalidate the local I-cache so that any instructions fetched
350
-	 * speculatively from the PoC are discarded, since they may have
351
-	 * been dynamically patched at the PoU.
352
-	 */
353
-	ic	iallu
354
-	dsb	nsh
355
-	isb
299
+	set_sctlr_el1	x17
356300 
357301 	/* Set the flag to zero to indicate that we're all done */
358302 	str	wzr, [flag_ptr]
....@@ -446,65 +390,107 @@
446390 	cbnz	w16, 1b
447391 
448392 	/* All done, act like nothing happened */
393
+	offset_ttbr1 swapper_ttb, x16
449394 	msr	ttbr1_el1, swapper_ttb
450395 	isb
451396 	ret
452397 
453398 	.unreq	swapper_ttb
454399 	.unreq	flag_ptr
455
-ENDPROC(idmap_kpti_install_ng_mappings)
400
+SYM_FUNC_END(idmap_kpti_install_ng_mappings)
456401 	.popsection
457402 #endif
458403 
459404 /*
460405  *	__cpu_setup
461406  *
462
- *	Initialise the processor for turning the MMU on.  Return in x0 the
463
- *	value of the SCTLR_EL1 register.
407
+ *	Initialise the processor for turning the MMU on.
408
+ *
409
+ * Output:
410
+ *	Return in x0 the value of the SCTLR_EL1 register.
464411  */
465412 	.pushsection ".idmap.text", "awx"
466
-ENTRY(__cpu_setup)
413
+SYM_FUNC_START(__cpu_setup)
467414 	tlbi	vmalle1				// Invalidate local TLB
468415 	dsb	nsh
469416 
470
-	mov	x0, #3 << 20
471
-	msr	cpacr_el1, x0			// Enable FP/ASIMD
472
-	mov	x0, #1 << 12			// Reset mdscr_el1 and disable
473
-	msr	mdscr_el1, x0			// access to the DCC from EL0
417
+	mov	x1, #3 << 20
418
+	msr	cpacr_el1, x1			// Enable FP/ASIMD
419
+	mov	x1, #1 << 12			// Reset mdscr_el1 and disable
420
+	msr	mdscr_el1, x1			// access to the DCC from EL0
474421 	isb					// Unmask debug exceptions now,
475422 	enable_dbg				// since this is per-cpu
476
-	reset_pmuserenr_el0 x0			// Disable PMU access from EL0
423
+	reset_pmuserenr_el0 x1			// Disable PMU access from EL0
424
+	reset_amuserenr_el0 x1			// Disable AMU access from EL0
425
+
477426 	/*
478
-	 * Memory region attributes for LPAE:
479
-	 *
480
-	 *   n = AttrIndx[2:0]
481
-	 *			n	MAIR
482
-	 *   DEVICE_nGnRnE	000	00000000
483
-	 *   DEVICE_nGnRE	001	00000100
484
-	 *   DEVICE_GRE		010	00001100
485
-	 *   NORMAL_NC		011	01000100
486
-	 *   NORMAL		100	11111111
487
-	 *   NORMAL_WT		101	10111011
427
+	 * Memory region attributes
488428 	 */
489
-	ldr	x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \
490
-		     MAIR(0x04, MT_DEVICE_nGnRE) | \
491
-		     MAIR(0x0c, MT_DEVICE_GRE) | \
492
-		     MAIR(0x44, MT_NORMAL_NC) | \
493
-		     MAIR(0xff, MT_NORMAL) | \
494
-		     MAIR(0xbb, MT_NORMAL_WT)
429
+	mov_q	x5, MAIR_EL1_SET
430
+#ifdef CONFIG_ARM64_MTE
431
+	mte_tcr	.req	x20
432
+
433
+	mov	mte_tcr, #0
434
+
435
+	/*
436
+	 * Update MAIR_EL1, GCR_EL1 and TFSR*_EL1 if MTE is supported
437
+	 * (ID_AA64PFR1_EL1[11:8] > 1).
438
+	 */
439
+	mrs	x10, ID_AA64PFR1_EL1
440
+	ubfx	x10, x10, #ID_AA64PFR1_MTE_SHIFT, #4
441
+	cmp	x10, #ID_AA64PFR1_MTE
442
+	b.lt	1f
443
+
444
+	/* Normal Tagged memory type at the corresponding MAIR index */
445
+	mov	x10, #MAIR_ATTR_NORMAL_TAGGED
446
+	bfi	x5, x10, #(8 *  MT_NORMAL_TAGGED), #8
447
+
448
+	mov	x10, #KERNEL_GCR_EL1
449
+	msr_s	SYS_GCR_EL1, x10
450
+
451
+	/*
452
+	 * If GCR_EL1.RRND=1 is implemented the same way as RRND=0, then
453
+	 * RGSR_EL1.SEED must be non-zero for IRG to produce
454
+	 * pseudorandom numbers. As RGSR_EL1 is UNKNOWN out of reset, we
455
+	 * must initialize it.
456
+	 */
457
+	mrs	x10, CNTVCT_EL0
458
+	ands	x10, x10, #SYS_RGSR_EL1_SEED_MASK
459
+	csinc	x10, x10, xzr, ne
460
+	lsl	x10, x10, #SYS_RGSR_EL1_SEED_SHIFT
461
+	msr_s	SYS_RGSR_EL1, x10
462
+
463
+	/* clear any pending tag check faults in TFSR*_EL1 */
464
+	msr_s	SYS_TFSR_EL1, xzr
465
+	msr_s	SYS_TFSRE0_EL1, xzr
466
+
467
+	/* set the TCR_EL1 bits */
468
+	mov_q	mte_tcr, TCR_MTE_FLAGS
469
+1:
470
+#endif
495471 	msr	mair_el1, x5
496472 	/*
497
-	 * Prepare SCTLR
473
+	 * Set/prepare TCR and TTBR. TCR_EL1.T1SZ gets further
474
+	 * adjusted if the kernel is compiled with 52bit VA support.
498475 	 */
499
-	mov_q	x0, SCTLR_EL1_SET
500
-	/*
501
-	 * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
502
-	 * both user and kernel.
503
-	 */
504
-	ldr	x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
476
+	mov_q	x10, TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
505477 			TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \
506
-			TCR_TBI0 | TCR_A1 | TCR_KASAN_FLAGS
507
-	tcr_set_idmap_t0sz	x10, x9
478
+			TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS
479
+#ifdef CONFIG_ARM64_MTE
480
+	orr	x10, x10, mte_tcr
481
+	.unreq	mte_tcr
482
+#endif
483
+	tcr_clear_errata_bits x10, x9, x5
484
+
485
+#ifdef CONFIG_ARM64_VA_BITS_52
486
+	ldr_l		x9, vabits_actual
487
+	sub		x9, xzr, x9
488
+	add		x9, x9, #64
489
+	tcr_set_t1sz	x10, x9
490
+#else
491
+	ldr_l		x9, idmap_t0sz
492
+#endif
493
+	tcr_set_t0sz	x10, x9
508494 
509495 	/*
510496 	 * Set the IPS bits in TCR_EL1.
....@@ -523,5 +509,9 @@
523509 1:
524510 #endif	/* CONFIG_ARM64_HW_AFDBM */
525511 	msr	tcr_el1, x10
512
+	/*
513
+	 * Prepare SCTLR
514
+	 */
515
+	mov_q	x0, INIT_SCTLR_EL1_MMU_ON
526516 	ret					// return to head.S
527
-ENDPROC(__cpu_setup)
517
+SYM_FUNC_END(__cpu_setup)