forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 95099d4622f8cb224d94e314c7a8e0df60b13f87 
 kernel/arch/arm64/include/asm/kvm_mmu.h
....@@ -1,18 +1,7 @@
1
+/* SPDX-License-Identifier: GPL-2.0-only */
12 /*
23  * Copyright (C) 2012,2013 - ARM Ltd
34  * Author: Marc Zyngier <marc.zyngier@arm.com>
4
- *
5
- * This program is free software; you can redistribute it and/or modify
6
- * it under the terms of the GNU General Public License version 2 as
7
- * published by the Free Software Foundation.
8
- *
9
- * This program is distributed in the hope that it will be useful,
10
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
- * GNU General Public License for more details.
13
- *
14
- * You should have received a copy of the GNU General Public License
15
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
165  */
176 
187 #ifndef __ARM64_KVM_MMU_H__
....@@ -20,6 +9,7 @@
209 
2110 #include <asm/page.h>
2211 #include <asm/memory.h>
12
+#include <asm/mmu.h>
2313 #include <asm/cpufeature.h>
2414 
2515 /*
....@@ -54,16 +44,6 @@
5444  *	HYP_VA_MIN = 1 << (VA_BITS - 1)
5545  * HYP_VA_MAX = HYP_VA_MIN + (1 << (VA_BITS - 1)) - 1
5646  *
57
- * This of course assumes that the trampoline page exists within the
58
- * VA_BITS range. If it doesn't, then it means we're in the odd case
59
- * where the kernel idmap (as well as HYP) uses more levels than the
60
- * kernel runtime page tables (as seen when the kernel is configured
61
- * for 4k pages, 39bits VA, and yet memory lives just above that
62
- * limit, forcing the idmap to use 4 levels of page tables while the
63
- * kernel itself only uses 3). In this particular case, it doesn't
64
- * matter which side of VA_BITS we use, as we're guaranteed not to
65
- * conflict with anything.
66
- *
6747  * When using VHE, there are no separate hyp mappings and all KVM
6848  * functionality is already mapped as part of the main kernel
6949  * mappings, and none of this applies in that case.
....@@ -92,18 +72,56 @@
9272 alternative_cb_end
9373 .endm
9474 
75
+/*
76
+ * Convert a hypervisor VA to a PA
77
+ * reg: hypervisor address to be converted in place
78
+ * tmp: temporary register
79
+ */
80
+.macro hyp_pa reg, tmp
81
+	ldr_l	\tmp, hyp_physvirt_offset
82
+	add	\reg, \reg, \tmp
83
+.endm
84
+
85
+/*
86
+ * Convert a hypervisor VA to a kernel image address
87
+ * reg: hypervisor address to be converted in place
88
+ * tmp: temporary register
89
+ *
90
+ * The actual code generation takes place in kvm_get_kimage_voffset, and
91
+ * the instructions below are only there to reserve the space and
92
+ * perform the register allocation (kvm_get_kimage_voffset uses the
93
+ * specific registers encoded in the instructions).
94
+ */
95
+.macro hyp_kimg_va reg, tmp
96
+	/* Convert hyp VA -> PA. */
97
+	hyp_pa	\reg, \tmp
98
+
99
+	/* Load kimage_voffset. */
100
+alternative_cb kvm_get_kimage_voffset
101
+	movz	\tmp, #0
102
+	movk	\tmp, #0, lsl #16
103
+	movk	\tmp, #0, lsl #32
104
+	movk	\tmp, #0, lsl #48
105
+alternative_cb_end
106
+
107
+	/* Convert PA -> kimg VA. */
108
+	add	\reg, \reg, \tmp
109
+.endm
110
+
95111 #else
96112 
113
+#include <linux/pgtable.h>
97114 #include <asm/pgalloc.h>
98115 #include <asm/cache.h>
99116 #include <asm/cacheflush.h>
100117 #include <asm/mmu_context.h>
101
-#include <asm/pgtable.h>
102118 
103119 void kvm_update_va_mask(struct alt_instr *alt,
104120 			__le32 *origptr, __le32 *updptr, int nr_inst);
121
+void kvm_compute_layout(void);
122
+void kvm_apply_hyp_relocations(void);
105123 
106
-static inline unsigned long __kern_hyp_va(unsigned long v)
124
+static __always_inline unsigned long __kern_hyp_va(unsigned long v)
107125 {
108126 	asm volatile(ALTERNATIVE_CB("and %0, %0, #1\n"
109127 				    "ror %0, %0, #1\n"
....@@ -118,35 +136,19 @@
118136 #define kern_hyp_va(v) 	((typeof(v))(__kern_hyp_va((unsigned long)(v))))
119137 
120138 /*
121
- * Obtain the PC-relative address of a kernel symbol
122
- * s: symbol
123
- *
124
- * The goal of this macro is to return a symbol's address based on a
125
- * PC-relative computation, as opposed to a loading the VA from a
126
- * constant pool or something similar. This works well for HYP, as an
127
- * absolute VA is guaranteed to be wrong. Only use this if trying to
128
- * obtain the address of a symbol (i.e. not something you obtained by
129
- * following a pointer).
130
- */
131
-#define hyp_symbol_addr(s)						\
132
-	({								\
133
-		typeof(s) *addr;					\
134
-		asm("adrp	%0, %1\n"				\
135
-		    "add	%0, %0, :lo12:%1\n"			\
136
-		    : "=r" (addr) : "S" (&s));				\
137
-		addr;							\
138
-	})
139
-
140
-/*
141
- * We currently only support a 40bit IPA.
139
+ * We currently support using a VM-specified IPA size. For backward
140
+ * compatibility, the default IPA size is fixed to 40bits.
142141  */
143142 #define KVM_PHYS_SHIFT	(40)
144
-#define KVM_PHYS_SIZE	(1UL << KVM_PHYS_SHIFT)
145
-#define KVM_PHYS_MASK	(KVM_PHYS_SIZE - 1UL)
146143 
144
+#define kvm_phys_shift(kvm)		VTCR_EL2_IPA(kvm->arch.vtcr)
145
+#define kvm_phys_size(kvm)		(_AC(1, ULL) << kvm_phys_shift(kvm))
146
+#define kvm_phys_mask(kvm)		(kvm_phys_size(kvm) - _AC(1, ULL))
147
+
148
+#include <asm/kvm_pgtable.h>
147149 #include <asm/stage2_pgtable.h>
148150 
149
-int create_hyp_mappings(void *from, void *to, pgprot_t prot);
151
+int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot);
150152 int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size,
151153 			   void __iomem **kaddr,
152154 			   void __iomem **haddr);
....@@ -155,108 +157,24 @@
155157 void free_hyp_pgds(void);
156158 
157159 void stage2_unmap_vm(struct kvm *kvm);
158
-int kvm_alloc_stage2_pgd(struct kvm *kvm);
159
-void kvm_free_stage2_pgd(struct kvm *kvm);
160
+int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu);
161
+void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu);
160162 int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
161163 			  phys_addr_t pa, unsigned long size, bool writable);
162164 
163
-int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run);
164
-
165
-void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);
165
+int kvm_handle_guest_abort(struct kvm_vcpu *vcpu);
166166 
167167 phys_addr_t kvm_mmu_get_httbr(void);
168168 phys_addr_t kvm_get_idmap_vector(void);
169
-int kvm_mmu_init(void);
170
-void kvm_clear_hyp_idmap(void);
169
+int kvm_mmu_init(u32 *hyp_va_bits);
171170 
172
-#define kvm_mk_pmd(ptep)					\
173
-	__pmd(__phys_to_pmd_val(__pa(ptep)) | PMD_TYPE_TABLE)
174
-#define kvm_mk_pud(pmdp)					\
175
-	__pud(__phys_to_pud_val(__pa(pmdp)) | PMD_TYPE_TABLE)
176
-#define kvm_mk_pgd(pudp)					\
177
-	__pgd(__phys_to_pgd_val(__pa(pudp)) | PUD_TYPE_TABLE)
178
-
179
-static inline pte_t kvm_s2pte_mkwrite(pte_t pte)
171
+static inline void *__kvm_vector_slot2addr(void *base,
172
+					   enum arm64_hyp_spectre_vector slot)
180173 {
181
-	pte_val(pte) |= PTE_S2_RDWR;
182
-	return pte;
174
+	int idx = slot - (slot != HYP_VECTOR_DIRECT);
175
+
176
+	return base + (idx * SZ_2K);
183177 }
184
-
185
-static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd)
186
-{
187
-	pmd_val(pmd) |= PMD_S2_RDWR;
188
-	return pmd;
189
-}
190
-
191
-static inline pte_t kvm_s2pte_mkexec(pte_t pte)
192
-{
193
-	pte_val(pte) &= ~PTE_S2_XN;
194
-	return pte;
195
-}
196
-
197
-static inline pmd_t kvm_s2pmd_mkexec(pmd_t pmd)
198
-{
199
-	pmd_val(pmd) &= ~PMD_S2_XN;
200
-	return pmd;
201
-}
202
-
203
-static inline void kvm_set_s2pte_readonly(pte_t *ptep)
204
-{
205
-	pteval_t old_pteval, pteval;
206
-
207
-	pteval = READ_ONCE(pte_val(*ptep));
208
-	do {
209
-		old_pteval = pteval;
210
-		pteval &= ~PTE_S2_RDWR;
211
-		pteval |= PTE_S2_RDONLY;
212
-		pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
213
-	} while (pteval != old_pteval);
214
-}
215
-
216
-static inline bool kvm_s2pte_readonly(pte_t *ptep)
217
-{
218
-	return (READ_ONCE(pte_val(*ptep)) & PTE_S2_RDWR) == PTE_S2_RDONLY;
219
-}
220
-
221
-static inline bool kvm_s2pte_exec(pte_t *ptep)
222
-{
223
-	return !(READ_ONCE(pte_val(*ptep)) & PTE_S2_XN);
224
-}
225
-
226
-static inline void kvm_set_s2pmd_readonly(pmd_t *pmdp)
227
-{
228
-	kvm_set_s2pte_readonly((pte_t *)pmdp);
229
-}
230
-
231
-static inline bool kvm_s2pmd_readonly(pmd_t *pmdp)
232
-{
233
-	return kvm_s2pte_readonly((pte_t *)pmdp);
234
-}
235
-
236
-static inline bool kvm_s2pmd_exec(pmd_t *pmdp)
237
-{
238
-	return !(READ_ONCE(pmd_val(*pmdp)) & PMD_S2_XN);
239
-}
240
-
241
-static inline bool kvm_page_empty(void *ptr)
242
-{
243
-	struct page *ptr_page = virt_to_page(ptr);
244
-	return page_count(ptr_page) == 1;
245
-}
246
-
247
-#define hyp_pte_table_empty(ptep) kvm_page_empty(ptep)
248
-
249
-#ifdef __PAGETABLE_PMD_FOLDED
250
-#define hyp_pmd_table_empty(pmdp) (0)
251
-#else
252
-#define hyp_pmd_table_empty(pmdp) kvm_page_empty(pmdp)
253
-#endif
254
-
255
-#ifdef __PAGETABLE_PUD_FOLDED
256
-#define hyp_pud_table_empty(pudp) (0)
257
-#else
258
-#define hyp_pud_table_empty(pudp) kvm_page_empty(pudp)
259
-#endif
260178 
261179 struct kvm;
262180 
....@@ -298,84 +216,14 @@
298216 	}
299217 }
300218 
301
-static inline void __kvm_flush_dcache_pte(pte_t pte)
302
-{
303
-	if (!cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) {
304
-		struct page *page = pte_page(pte);
305
-		kvm_flush_dcache_to_poc(page_address(page), PAGE_SIZE);
306
-	}
307
-}
308
-
309
-static inline void __kvm_flush_dcache_pmd(pmd_t pmd)
310
-{
311
-	if (!cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) {
312
-		struct page *page = pmd_page(pmd);
313
-		kvm_flush_dcache_to_poc(page_address(page), PMD_SIZE);
314
-	}
315
-}
316
-
317
-static inline void __kvm_flush_dcache_pud(pud_t pud)
318
-{
319
-	if (!cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) {
320
-		struct page *page = pud_page(pud);
321
-		kvm_flush_dcache_to_poc(page_address(page), PUD_SIZE);
322
-	}
323
-}
324
-
325
-#define kvm_virt_to_phys(x)		__pa_symbol(x)
326
-
327219 void kvm_set_way_flush(struct kvm_vcpu *vcpu);
328220 void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled);
329
-
330
-static inline bool __kvm_cpu_uses_extended_idmap(void)
331
-{
332
-	return __cpu_uses_extended_idmap_level();
333
-}
334
-
335
-static inline unsigned long __kvm_idmap_ptrs_per_pgd(void)
336
-{
337
-	return idmap_ptrs_per_pgd;
338
-}
339
-
340
-/*
341
- * Can't use pgd_populate here, because the extended idmap adds an extra level
342
- * above CONFIG_PGTABLE_LEVELS (which is 2 or 3 if we're using the extended
343
- * idmap), and pgd_populate is only available if CONFIG_PGTABLE_LEVELS = 4.
344
- */
345
-static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
346
-				       pgd_t *hyp_pgd,
347
-				       pgd_t *merged_hyp_pgd,
348
-				       unsigned long hyp_idmap_start)
349
-{
350
-	int idmap_idx;
351
-	u64 pgd_addr;
352
-
353
-	/*
354
-	 * Use the first entry to access the HYP mappings. It is
355
-	 * guaranteed to be free, otherwise we wouldn't use an
356
-	 * extended idmap.
357
-	 */
358
-	VM_BUG_ON(pgd_val(merged_hyp_pgd[0]));
359
-	pgd_addr = __phys_to_pgd_val(__pa(hyp_pgd));
360
-	merged_hyp_pgd[0] = __pgd(pgd_addr | PMD_TYPE_TABLE);
361
-
362
-	/*
363
-	 * Create another extended level entry that points to the boot HYP map,
364
-	 * which contains an ID mapping of the HYP init code. We essentially
365
-	 * merge the boot and runtime HYP maps by doing so, but they don't
366
-	 * overlap anyway, so this is fine.
367
-	 */
368
-	idmap_idx = hyp_idmap_start >> VA_BITS;
369
-	VM_BUG_ON(pgd_val(merged_hyp_pgd[idmap_idx]));
370
-	pgd_addr = __phys_to_pgd_val(__pa(boot_hyp_pgd));
371
-	merged_hyp_pgd[idmap_idx] = __pgd(pgd_addr | PMD_TYPE_TABLE);
372
-}
373221 
374222 static inline unsigned int kvm_get_vmid_bits(void)
375223 {
376224 	int reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
377225 
378
-	return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
226
+	return get_vmid_bits(reg);
379227 }
380228 
381229 /*
....@@ -405,128 +253,44 @@
405253 	return ret;
406254 }
407255 
408
-#ifdef CONFIG_KVM_INDIRECT_VECTORS
409
-/*
410
- * EL2 vectors can be mapped and rerouted in a number of ways,
411
- * depending on the kernel configuration and CPU present:
412
- *
413
- * - If the CPU has the ARM64_HARDEN_BRANCH_PREDICTOR cap, the
414
- *   hardening sequence is placed in one of the vector slots, which is
415
- *   executed before jumping to the real vectors.
416
- *
417
- * - If the CPU has both the ARM64_HARDEN_EL2_VECTORS cap and the
418
- *   ARM64_HARDEN_BRANCH_PREDICTOR cap, the slot containing the
419
- *   hardening sequence is mapped next to the idmap page, and executed
420
- *   before jumping to the real vectors.
421
- *
422
- * - If the CPU only has the ARM64_HARDEN_EL2_VECTORS cap, then an
423
- *   empty slot is selected, mapped next to the idmap page, and
424
- *   executed before jumping to the real vectors.
425
- *
426
- * Note that ARM64_HARDEN_EL2_VECTORS is somewhat incompatible with
427
- * VHE, as we don't have hypervisor-specific mappings. If the system
428
- * is VHE and yet selects this capability, it will be ignored.
429
- */
430
-#include <asm/mmu.h>
431
-
432
-extern void *__kvm_bp_vect_base;
433
-extern int __kvm_harden_el2_vector_slot;
434
-
435
-static inline void *kvm_get_hyp_vector(void)
436
-{
437
-	struct bp_hardening_data *data = arm64_get_bp_hardening_data();
438
-	void *vect = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
439
-	int slot = -1;
440
-
441
-	if (cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR) && data->fn) {
442
-		vect = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs_start));
443
-		slot = data->hyp_vectors_slot;
444
-	}
445
-
446
-	if (this_cpu_has_cap(ARM64_HARDEN_EL2_VECTORS) && !has_vhe()) {
447
-		vect = __kvm_bp_vect_base;
448
-		if (slot == -1)
449
-			slot = __kvm_harden_el2_vector_slot;
450
-	}
451
-
452
-	if (slot != -1)
453
-		vect += slot * SZ_2K;
454
-
455
-	return vect;
456
-}
457
-
458
-/*  This is only called on a !VHE system */
459
-static inline int kvm_map_vectors(void)
460
-{
461
-	/*
462
-	 * HBP  = ARM64_HARDEN_BRANCH_PREDICTOR
463
-	 * HEL2 = ARM64_HARDEN_EL2_VECTORS
464
-	 *
465
-	 * !HBP + !HEL2 -> use direct vectors
466
-	 *  HBP + !HEL2 -> use hardened vectors in place
467
-	 * !HBP +  HEL2 -> allocate one vector slot and use exec mapping
468
-	 *  HBP +  HEL2 -> use hardened vertors and use exec mapping
469
-	 */
470
-	if (cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR)) {
471
-		__kvm_bp_vect_base = kvm_ksym_ref(__bp_harden_hyp_vecs_start);
472
-		__kvm_bp_vect_base = kern_hyp_va(__kvm_bp_vect_base);
473
-	}
474
-
475
-	if (cpus_have_const_cap(ARM64_HARDEN_EL2_VECTORS)) {
476
-		phys_addr_t vect_pa = __pa_symbol(__bp_harden_hyp_vecs_start);
477
-		unsigned long size = (__bp_harden_hyp_vecs_end -
478
-				      __bp_harden_hyp_vecs_start);
479
-
480
-		/*
481
-		 * Always allocate a spare vector slot, as we don't
482
-		 * know yet which CPUs have a BP hardening slot that
483
-		 * we can reuse.
484
-		 */
485
-		__kvm_harden_el2_vector_slot = atomic_inc_return(&arm64_el2_vector_last_slot);
486
-		BUG_ON(__kvm_harden_el2_vector_slot >= BP_HARDEN_EL2_SLOTS);
487
-		return create_hyp_exec_mappings(vect_pa, size,
488
-						&__kvm_bp_vect_base);
489
-	}
490
-
491
-	return 0;
492
-}
493
-#else
494
-static inline void *kvm_get_hyp_vector(void)
495
-{
496
-	return kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
497
-}
498
-
499
-static inline int kvm_map_vectors(void)
500
-{
501
-	return 0;
502
-}
503
-#endif
504
-
505
-#ifdef CONFIG_ARM64_SSBD
506
-DECLARE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
507
-
508
-static inline int hyp_map_aux_data(void)
509
-{
510
-	int cpu, err;
511
-
512
-	for_each_possible_cpu(cpu) {
513
-		u64 *ptr;
514
-
515
-		ptr = per_cpu_ptr(&arm64_ssbd_callback_required, cpu);
516
-		err = create_hyp_mappings(ptr, ptr + 1, PAGE_HYP);
517
-		if (err)
518
-			return err;
519
-	}
520
-	return 0;
521
-}
522
-#else
523
-static inline int hyp_map_aux_data(void)
524
-{
525
-	return 0;
526
-}
527
-#endif
528
-
529256 #define kvm_phys_to_vttbr(addr)		phys_to_ttbr(addr)
530257 
258
+static __always_inline u64 kvm_get_vttbr(struct kvm_s2_mmu *mmu)
259
+{
260
+	struct kvm_vmid *vmid = &mmu->vmid;
261
+	u64 vmid_field, baddr;
262
+	u64 cnp = system_supports_cnp() ? VTTBR_CNP_BIT : 0;
263
+
264
+	baddr = mmu->pgd_phys;
265
+	vmid_field = (u64)vmid->vmid << VTTBR_VMID_SHIFT;
266
+	return kvm_phys_to_vttbr(baddr) | vmid_field | cnp;
267
+}
268
+
269
+/*
270
+ * Must be called from hyp code running at EL2 with an updated VTTBR
271
+ * and interrupts disabled.
272
+ */
273
+static __always_inline void __load_stage2(struct kvm_s2_mmu *mmu, unsigned long vtcr)
274
+{
275
+	write_sysreg(vtcr, vtcr_el2);
276
+	write_sysreg(kvm_get_vttbr(mmu), vttbr_el2);
277
+
278
+	/*
279
+	 * ARM errata 1165522 and 1530923 require the actual execution of the
280
+	 * above before we can switch to the EL1/EL0 translation regime used by
281
+	 * the guest.
282
+	 */
283
+	asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));
284
+}
285
+
286
+static __always_inline void __load_guest_stage2(struct kvm_s2_mmu *mmu)
287
+{
288
+	__load_stage2(mmu, kern_hyp_va(mmu->arch)->vtcr);
289
+}
290
+
291
+static inline struct kvm *kvm_s2_mmu_to_kvm(struct kvm_s2_mmu *mmu)
292
+{
293
+	return container_of(mmu->arch, struct kvm, arch);
294
+}
531295 #endif /* __ASSEMBLY__ */
532296 #endif /* __ARM64_KVM_MMU_H__ */