forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-01-05 071106ecf68c401173c58808b1cf5f68cc50d390 
 kernel/arch/arm64/kvm/sys_regs.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Copyright (C) 2012,2013 - ARM Ltd
34  * Author: Marc Zyngier <marc.zyngier@arm.com>
....@@ -6,20 +7,9 @@
67  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
78  * Authors: Rusty Russell <rusty@rustcorp.com.au>
89  *          Christoffer Dall <c.dall@virtualopensystems.com>
9
- *
10
- * This program is free software; you can redistribute it and/or modify
11
- * it under the terms of the GNU General Public License, version 2, as
12
- * published by the Free Software Foundation.
13
- *
14
- * This program is distributed in the hope that it will be useful,
15
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
16
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
- * GNU General Public License for more details.
18
- *
19
- * You should have received a copy of the GNU General Public License
20
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
2110  */
2211 
12
+#include <linux/bitfield.h>
2313 #include <linux/bsearch.h>
2414 #include <linux/kvm_host.h>
2515 #include <linux/mm.h>
....@@ -31,9 +21,7 @@
3121 #include <asm/debug-monitors.h>
3222 #include <asm/esr.h>
3323 #include <asm/kvm_arm.h>
34
-#include <asm/kvm_coproc.h>
3524 #include <asm/kvm_emulate.h>
36
-#include <asm/kvm_host.h>
3725 #include <asm/kvm_hyp.h>
3826 #include <asm/kvm_mmu.h>
3927 #include <asm/perf_event.h>
....@@ -46,7 +34,7 @@
4634 #include "trace.h"
4735 
4836 /*
49
- * All of this file is extremly similar to the ARM coproc.c, but the
37
+ * All of this file is extremely similar to the ARM coproc.c, but the
5038  * types are different. My gut feeling is that it should be pretty
5139  * easy to merge, but that would be an ABI breakage -- again. VFP
5240  * would also need to be abstracted.
....@@ -55,6 +43,10 @@
5543  * that has to do with init and userspace access has to go via the
5644  * 64bit interface.
5745  */
46
+
47
+#define reg_to_encoding(x)						\
48
+	sys_reg((u32)(x)->Op0, (u32)(x)->Op1,				\
49
+		(u32)(x)->CRn, (u32)(x)->CRm, (u32)(x)->Op2)
5850 
5951 static bool read_from_write_only(struct kvm_vcpu *vcpu,
6052 				 struct sys_reg_params *params,
....@@ -76,90 +68,23 @@
7668 	return false;
7769 }
7870 
79
-u64 vcpu_read_sys_reg(struct kvm_vcpu *vcpu, int reg)
71
+u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
8072 {
81
-	if (!vcpu->arch.sysregs_loaded_on_cpu)
82
-		goto immediate_read;
73
+	u64 val = 0x8badf00d8badf00d;
8374 
84
-	/*
85
-	 * System registers listed in the switch are not saved on every
86
-	 * exit from the guest but are only saved on vcpu_put.
87
-	 *
88
-	 * Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
89
-	 * should never be listed below, because the guest cannot modify its
90
-	 * own MPIDR_EL1 and MPIDR_EL1 is accessed for VCPU A from VCPU B's
91
-	 * thread when emulating cross-VCPU communication.
92
-	 */
93
-	switch (reg) {
94
-	case CSSELR_EL1:	return read_sysreg_s(SYS_CSSELR_EL1);
95
-	case SCTLR_EL1:		return read_sysreg_s(sctlr_EL12);
96
-	case ACTLR_EL1:		return read_sysreg_s(SYS_ACTLR_EL1);
97
-	case CPACR_EL1:		return read_sysreg_s(cpacr_EL12);
98
-	case TTBR0_EL1:		return read_sysreg_s(ttbr0_EL12);
99
-	case TTBR1_EL1:		return read_sysreg_s(ttbr1_EL12);
100
-	case TCR_EL1:		return read_sysreg_s(tcr_EL12);
101
-	case ESR_EL1:		return read_sysreg_s(esr_EL12);
102
-	case AFSR0_EL1:		return read_sysreg_s(afsr0_EL12);
103
-	case AFSR1_EL1:		return read_sysreg_s(afsr1_EL12);
104
-	case FAR_EL1:		return read_sysreg_s(far_EL12);
105
-	case MAIR_EL1:		return read_sysreg_s(mair_EL12);
106
-	case VBAR_EL1:		return read_sysreg_s(vbar_EL12);
107
-	case CONTEXTIDR_EL1:	return read_sysreg_s(contextidr_EL12);
108
-	case TPIDR_EL0:		return read_sysreg_s(SYS_TPIDR_EL0);
109
-	case TPIDRRO_EL0:	return read_sysreg_s(SYS_TPIDRRO_EL0);
110
-	case TPIDR_EL1:		return read_sysreg_s(SYS_TPIDR_EL1);
111
-	case AMAIR_EL1:		return read_sysreg_s(amair_EL12);
112
-	case CNTKCTL_EL1:	return read_sysreg_s(cntkctl_EL12);
113
-	case PAR_EL1:		return read_sysreg_s(SYS_PAR_EL1);
114
-	case DACR32_EL2:	return read_sysreg_s(SYS_DACR32_EL2);
115
-	case IFSR32_EL2:	return read_sysreg_s(SYS_IFSR32_EL2);
116
-	case DBGVCR32_EL2:	return read_sysreg_s(SYS_DBGVCR32_EL2);
117
-	}
75
+	if (vcpu->arch.sysregs_loaded_on_cpu &&
76
+	    __vcpu_read_sys_reg_from_cpu(reg, &val))
77
+		return val;
11878 
119
-immediate_read:
12079 	return __vcpu_sys_reg(vcpu, reg);
12180 }
12281 
12382 void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
12483 {
125
-	if (!vcpu->arch.sysregs_loaded_on_cpu)
126
-		goto immediate_write;
84
+	if (vcpu->arch.sysregs_loaded_on_cpu &&
85
+	    __vcpu_write_sys_reg_to_cpu(val, reg))
86
+		return;
12787 
128
-	/*
129
-	 * System registers listed in the switch are not restored on every
130
-	 * entry to the guest but are only restored on vcpu_load.
131
-	 *
132
-	 * Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
133
-	 * should never be listed below, because the the MPIDR should only be
134
-	 * set once, before running the VCPU, and never changed later.
135
-	 */
136
-	switch (reg) {
137
-	case CSSELR_EL1:	write_sysreg_s(val, SYS_CSSELR_EL1);	return;
138
-	case SCTLR_EL1:		write_sysreg_s(val, sctlr_EL12);	return;
139
-	case ACTLR_EL1:		write_sysreg_s(val, SYS_ACTLR_EL1);	return;
140
-	case CPACR_EL1:		write_sysreg_s(val, cpacr_EL12);	return;
141
-	case TTBR0_EL1:		write_sysreg_s(val, ttbr0_EL12);	return;
142
-	case TTBR1_EL1:		write_sysreg_s(val, ttbr1_EL12);	return;
143
-	case TCR_EL1:		write_sysreg_s(val, tcr_EL12);		return;
144
-	case ESR_EL1:		write_sysreg_s(val, esr_EL12);		return;
145
-	case AFSR0_EL1:		write_sysreg_s(val, afsr0_EL12);	return;
146
-	case AFSR1_EL1:		write_sysreg_s(val, afsr1_EL12);	return;
147
-	case FAR_EL1:		write_sysreg_s(val, far_EL12);		return;
148
-	case MAIR_EL1:		write_sysreg_s(val, mair_EL12);		return;
149
-	case VBAR_EL1:		write_sysreg_s(val, vbar_EL12);		return;
150
-	case CONTEXTIDR_EL1:	write_sysreg_s(val, contextidr_EL12);	return;
151
-	case TPIDR_EL0:		write_sysreg_s(val, SYS_TPIDR_EL0);	return;
152
-	case TPIDRRO_EL0:	write_sysreg_s(val, SYS_TPIDRRO_EL0);	return;
153
-	case TPIDR_EL1:		write_sysreg_s(val, SYS_TPIDR_EL1);	return;
154
-	case AMAIR_EL1:		write_sysreg_s(val, amair_EL12);	return;
155
-	case CNTKCTL_EL1:	write_sysreg_s(val, cntkctl_EL12);	return;
156
-	case PAR_EL1:		write_sysreg_s(val, SYS_PAR_EL1);	return;
157
-	case DACR32_EL2:	write_sysreg_s(val, SYS_DACR32_EL2);	return;
158
-	case IFSR32_EL2:	write_sysreg_s(val, SYS_IFSR32_EL2);	return;
159
-	case DBGVCR32_EL2:	write_sysreg_s(val, SYS_DBGVCR32_EL2);	return;
160
-	}
161
-
162
-immediate_write:
16388 	 __vcpu_sys_reg(vcpu, reg) = val;
16489 }
16590 
....@@ -167,7 +92,7 @@
16792 static u32 cache_levels;
16893 
16994 /* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
170
-#define CSSELR_MAX 12
95
+#define CSSELR_MAX 14
17196 
17297 /* Which cache CCSIDR represents depends on CSSELR value. */
17398 static u32 get_ccsidr(u32 csselr)
....@@ -207,6 +132,24 @@
207132 	return true;
208133 }
209134 
135
+static void get_access_mask(const struct sys_reg_desc *r, u64 *mask, u64 *shift)
136
+{
137
+	switch (r->aarch32_map) {
138
+	case AA32_LO:
139
+		*mask = GENMASK_ULL(31, 0);
140
+		*shift = 0;
141
+		break;
142
+	case AA32_HI:
143
+		*mask = GENMASK_ULL(63, 32);
144
+		*shift = 32;
145
+		break;
146
+	default:
147
+		*mask = GENMASK_ULL(63, 0);
148
+		*shift = 0;
149
+		break;
150
+	}
151
+}
152
+
210153 /*
211154  * Generic accessor for VM registers. Only called as long as HCR_TVM
212155  * is set. If the guest enables the MMU, we stop trapping the VM
....@@ -217,28 +160,38 @@
217160 			  const struct sys_reg_desc *r)
218161 {
219162 	bool was_enabled = vcpu_has_cache_enabled(vcpu);
220
-	u64 val;
221
-	int reg = r->reg;
163
+	u64 val, mask, shift;
222164 
223165 	BUG_ON(!p->is_write);
224166 
225
-	/* See the 32bit mapping in kvm_host.h */
226
-	if (p->is_aarch32)
227
-		reg = r->reg / 2;
167
+	get_access_mask(r, &mask, &shift);
228168 
229
-	if (!p->is_aarch32 || !p->is_32bit) {
230
-		val = p->regval;
169
+	if (~mask) {
170
+		val = vcpu_read_sys_reg(vcpu, r->reg);
171
+		val &= ~mask;
231172 	} else {
232
-		val = vcpu_read_sys_reg(vcpu, reg);
233
-		if (r->reg % 2)
234
-			val = (p->regval << 32) | (u64)lower_32_bits(val);
235
-		else
236
-			val = ((u64)upper_32_bits(val) << 32) |
237
-				lower_32_bits(p->regval);
173
+		val = 0;
238174 	}
239
-	vcpu_write_sys_reg(vcpu, val, reg);
175
+
176
+	val |= (p->regval & (mask >> shift)) << shift;
177
+	vcpu_write_sys_reg(vcpu, val, r->reg);
240178 
241179 	kvm_toggle_cache(vcpu, was_enabled);
180
+	return true;
181
+}
182
+
183
+static bool access_actlr(struct kvm_vcpu *vcpu,
184
+			 struct sys_reg_params *p,
185
+			 const struct sys_reg_desc *r)
186
+{
187
+	u64 mask, shift;
188
+
189
+	if (p->is_write)
190
+		return ignore_write(vcpu, p);
191
+
192
+	get_access_mask(r, &mask, &shift);
193
+	p->regval = (vcpu_read_sys_reg(vcpu, r->reg) & mask) >> shift;
194
+
242195 	return true;
243196 }
244197 
....@@ -264,7 +217,7 @@
264217 	 * equivalent to ICC_SGI0R_EL1, as there is no "alternative" secure
265218 	 * group.
266219 	 */
267
-	if (p->is_aarch32) {
220
+	if (p->Op0 == 0) {		/* AArch32 */
268221 		switch (p->Op1) {
269222 		default:		/* Keep GCC quiet */
270223 		case 0:			/* ICC_SGI1R */
....@@ -275,7 +228,7 @@
275228 			g1 = false;
276229 			break;
277230 		}
278
-	} else {
231
+	} else {			/* AArch64 */
279232 		switch (p->Op2) {
280233 		default:		/* Keep GCC quiet */
281234 		case 5:			/* ICC_SGI1R_EL1 */
....@@ -314,12 +267,28 @@
314267 		return read_zero(vcpu, p);
315268 }
316269 
317
-static bool trap_undef(struct kvm_vcpu *vcpu,
318
-		       struct sys_reg_params *p,
319
-		       const struct sys_reg_desc *r)
270
+/*
271
+ * ARMv8.1 mandates at least a trivial LORegion implementation, where all the
272
+ * RW registers are RES0 (which we can implement as RAZ/WI). On an ARMv8.0
273
+ * system, these registers should UNDEF. LORID_EL1 being a RO register, we
274
+ * treat it separately.
275
+ */
276
+static bool trap_loregion(struct kvm_vcpu *vcpu,
277
+			  struct sys_reg_params *p,
278
+			  const struct sys_reg_desc *r)
320279 {
321
-	kvm_inject_undefined(vcpu);
322
-	return false;
280
+	u64 val = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
281
+	u32 sr = reg_to_encoding(r);
282
+
283
+	if (!(val & (0xfUL << ID_AA64MMFR1_LOR_SHIFT))) {
284
+		kvm_inject_undefined(vcpu);
285
+		return false;
286
+	}
287
+
288
+	if (p->is_write && sr == SYS_LORID_EL1)
289
+		return write_to_read_only(vcpu, p, r);
290
+
291
+	return trap_raz_wi(vcpu, p, r);
323292 }
324293 
325294 static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
....@@ -400,26 +369,30 @@
400369  */
401370 static void reg_to_dbg(struct kvm_vcpu *vcpu,
402371 		       struct sys_reg_params *p,
372
+		       const struct sys_reg_desc *rd,
403373 		       u64 *dbg_reg)
404374 {
405
-	u64 val = p->regval;
375
+	u64 mask, shift, val;
406376 
407
-	if (p->is_32bit) {
408
-		val &= 0xffffffffUL;
409
-		val |= ((*dbg_reg >> 32) << 32);
410
-	}
377
+	get_access_mask(rd, &mask, &shift);
411378 
379
+	val = *dbg_reg;
380
+	val &= ~mask;
381
+	val |= (p->regval & (mask >> shift)) << shift;
412382 	*dbg_reg = val;
383
+
413384 	vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
414385 }
415386 
416387 static void dbg_to_reg(struct kvm_vcpu *vcpu,
417388 		       struct sys_reg_params *p,
389
+		       const struct sys_reg_desc *rd,
418390 		       u64 *dbg_reg)
419391 {
420
-	p->regval = *dbg_reg;
421
-	if (p->is_32bit)
422
-		p->regval &= 0xffffffffUL;
392
+	u64 mask, shift;
393
+
394
+	get_access_mask(rd, &mask, &shift);
395
+	p->regval = (*dbg_reg & mask) >> shift;
423396 }
424397 
425398 static bool trap_bvr(struct kvm_vcpu *vcpu,
....@@ -429,9 +402,9 @@
429402 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
430403 
431404 	if (p->is_write)
432
-		reg_to_dbg(vcpu, p, dbg_reg);
405
+		reg_to_dbg(vcpu, p, rd, dbg_reg);
433406 	else
434
-		dbg_to_reg(vcpu, p, dbg_reg);
407
+		dbg_to_reg(vcpu, p, rd, dbg_reg);
435408 
436409 	trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
437410 
....@@ -471,9 +444,9 @@
471444 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
472445 
473446 	if (p->is_write)
474
-		reg_to_dbg(vcpu, p, dbg_reg);
447
+		reg_to_dbg(vcpu, p, rd, dbg_reg);
475448 	else
476
-		dbg_to_reg(vcpu, p, dbg_reg);
449
+		dbg_to_reg(vcpu, p, rd, dbg_reg);
477450 
478451 	trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
479452 
....@@ -514,9 +487,9 @@
514487 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
515488 
516489 	if (p->is_write)
517
-		reg_to_dbg(vcpu, p, dbg_reg);
490
+		reg_to_dbg(vcpu, p, rd, dbg_reg);
518491 	else
519
-		dbg_to_reg(vcpu, p, dbg_reg);
492
+		dbg_to_reg(vcpu, p, rd, dbg_reg);
520493 
521494 	trace_trap_reg(__func__, rd->CRm, p->is_write,
522495 		vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm]);
....@@ -557,9 +530,9 @@
557530 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
558531 
559532 	if (p->is_write)
560
-		reg_to_dbg(vcpu, p, dbg_reg);
533
+		reg_to_dbg(vcpu, p, rd, dbg_reg);
561534 	else
562
-		dbg_to_reg(vcpu, p, dbg_reg);
535
+		dbg_to_reg(vcpu, p, rd, dbg_reg);
563536 
564537 	trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
565538 
....@@ -598,6 +571,12 @@
598571 	vcpu_write_sys_reg(vcpu, amair, AMAIR_EL1);
599572 }
600573 
574
+static void reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
575
+{
576
+	u64 actlr = read_sysreg(actlr_el1);
577
+	vcpu_write_sys_reg(vcpu, actlr, ACTLR_EL1);
578
+}
579
+
601580 static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
602581 {
603582 	u64 mpidr;
....@@ -615,6 +594,15 @@
615594 	vcpu_write_sys_reg(vcpu, (1ULL << 31) | mpidr, MPIDR_EL1);
616595 }
617596 
597
+static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu,
598
+				   const struct sys_reg_desc *r)
599
+{
600
+	if (kvm_vcpu_has_pmu(vcpu))
601
+		return 0;
602
+
603
+	return REG_HIDDEN;
604
+}
605
+
618606 static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
619607 {
620608 	u64 pmcr, val;
....@@ -630,6 +618,8 @@
630618 	 */
631619 	val = ((pmcr & ~ARMV8_PMU_PMCR_MASK)
632620 	       | (ARMV8_PMU_PMCR_MASK & 0xdecafbad)) & (~ARMV8_PMU_PMCR_E);
621
+	if (!kvm_supports_32bit_el0())
622
+		val |= ARMV8_PMU_PMCR_LC;
633623 	__vcpu_sys_reg(vcpu, r->reg) = val;
634624 }
635625 
....@@ -669,9 +659,6 @@
669659 {
670660 	u64 val;
671661 
672
-	if (!kvm_arm_pmu_v3_ready(vcpu))
673
-		return trap_raz_wi(vcpu, p, r);
674
-
675662 	if (pmu_access_el0_disabled(vcpu))
676663 		return false;
677664 
....@@ -680,8 +667,11 @@
680667 		val = __vcpu_sys_reg(vcpu, PMCR_EL0);
681668 		val &= ~ARMV8_PMU_PMCR_MASK;
682669 		val |= p->regval & ARMV8_PMU_PMCR_MASK;
670
+		if (!kvm_supports_32bit_el0())
671
+			val |= ARMV8_PMU_PMCR_LC;
683672 		__vcpu_sys_reg(vcpu, PMCR_EL0) = val;
684673 		kvm_pmu_handle_pmcr(vcpu, val);
674
+		kvm_vcpu_pmu_restore_guest(vcpu);
685675 	} else {
686676 		/* PMCR.P & PMCR.C are RAZ */
687677 		val = __vcpu_sys_reg(vcpu, PMCR_EL0)
....@@ -695,9 +685,6 @@
695685 static bool access_pmselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
696686 			  const struct sys_reg_desc *r)
697687 {
698
-	if (!kvm_arm_pmu_v3_ready(vcpu))
699
-		return trap_raz_wi(vcpu, p, r);
700
-
701688 	if (pmu_access_event_counter_el0_disabled(vcpu))
702689 		return false;
703690 
....@@ -714,20 +701,18 @@
714701 static bool access_pmceid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
715702 			  const struct sys_reg_desc *r)
716703 {
717
-	u64 pmceid;
718
-
719
-	if (!kvm_arm_pmu_v3_ready(vcpu))
720
-		return trap_raz_wi(vcpu, p, r);
704
+	u64 pmceid, mask, shift;
721705 
722706 	BUG_ON(p->is_write);
723707 
724708 	if (pmu_access_el0_disabled(vcpu))
725709 		return false;
726710 
727
-	if (!(p->Op2 & 1))
728
-		pmceid = read_sysreg(pmceid0_el0);
729
-	else
730
-		pmceid = read_sysreg(pmceid1_el0);
711
+	get_access_mask(r, &mask, &shift);
712
+
713
+	pmceid = kvm_pmu_get_pmceid(vcpu, (p->Op2 & 1));
714
+	pmceid &= mask;
715
+	pmceid >>= shift;
731716 
732717 	p->regval = pmceid;
733718 
....@@ -752,10 +737,7 @@
752737 			      struct sys_reg_params *p,
753738 			      const struct sys_reg_desc *r)
754739 {
755
-	u64 idx;
756
-
757
-	if (!kvm_arm_pmu_v3_ready(vcpu))
758
-		return trap_raz_wi(vcpu, p, r);
740
+	u64 idx = ~0UL;
759741 
760742 	if (r->CRn == 9 && r->CRm == 13) {
761743 		if (r->Op2 == 2) {
....@@ -771,8 +753,6 @@
771753 				return false;
772754 
773755 			idx = ARMV8_PMU_CYCLE_IDX;
774
-		} else {
775
-			return false;
776756 		}
777757 	} else if (r->CRn == 0 && r->CRm == 9) {
778758 		/* PMCCNTR */
....@@ -786,9 +766,10 @@
786766 			return false;
787767 
788768 		idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
789
-	} else {
790
-		return false;
791769 	}
770
+
771
+	/* Catch any decoding mistake */
772
+	WARN_ON(idx == ~0UL);
792773 
793774 	if (!pmu_counter_idx_valid(vcpu, idx))
794775 		return false;
....@@ -809,9 +790,6 @@
809790 			       const struct sys_reg_desc *r)
810791 {
811792 	u64 idx, reg;
812
-
813
-	if (!kvm_arm_pmu_v3_ready(vcpu))
814
-		return trap_raz_wi(vcpu, p, r);
815793 
816794 	if (pmu_access_el0_disabled(vcpu))
817795 		return false;
....@@ -837,6 +815,7 @@
837815 	if (p->is_write) {
838816 		kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
839817 		__vcpu_sys_reg(vcpu, reg) = p->regval & ARMV8_PMU_EVTYPE_MASK;
818
+		kvm_vcpu_pmu_restore_guest(vcpu);
840819 	} else {
841820 		p->regval = __vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_MASK;
842821 	}
....@@ -849,9 +828,6 @@
849828 {
850829 	u64 val, mask;
851830 
852
-	if (!kvm_arm_pmu_v3_ready(vcpu))
853
-		return trap_raz_wi(vcpu, p, r);
854
-
855831 	if (pmu_access_el0_disabled(vcpu))
856832 		return false;
857833 
....@@ -861,11 +837,12 @@
861837 		if (r->Op2 & 0x1) {
862838 			/* accessing PMCNTENSET_EL0 */
863839 			__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) |= val;
864
-			kvm_pmu_enable_counter(vcpu, val);
840
+			kvm_pmu_enable_counter_mask(vcpu, val);
841
+			kvm_vcpu_pmu_restore_guest(vcpu);
865842 		} else {
866843 			/* accessing PMCNTENCLR_EL0 */
867844 			__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= ~val;
868
-			kvm_pmu_disable_counter(vcpu, val);
845
+			kvm_pmu_disable_counter_mask(vcpu, val);
869846 		}
870847 	} else {
871848 		p->regval = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask;
....@@ -879,13 +856,8 @@
879856 {
880857 	u64 mask = kvm_pmu_valid_counter_mask(vcpu);
881858 
882
-	if (!kvm_arm_pmu_v3_ready(vcpu))
883
-		return trap_raz_wi(vcpu, p, r);
884
-
885
-	if (!vcpu_mode_priv(vcpu)) {
886
-		kvm_inject_undefined(vcpu);
859
+	if (check_pmu_access_disabled(vcpu, 0))
887860 		return false;
888
-	}
889861 
890862 	if (p->is_write) {
891863 		u64 val = p->regval & mask;
....@@ -907,9 +879,6 @@
907879 			 const struct sys_reg_desc *r)
908880 {
909881 	u64 mask = kvm_pmu_valid_counter_mask(vcpu);
910
-
911
-	if (!kvm_arm_pmu_v3_ready(vcpu))
912
-		return trap_raz_wi(vcpu, p, r);
913882 
914883 	if (pmu_access_el0_disabled(vcpu))
915884 		return false;
....@@ -933,9 +902,6 @@
933902 {
934903 	u64 mask;
935904 
936
-	if (!kvm_arm_pmu_v3_ready(vcpu))
937
-		return trap_raz_wi(vcpu, p, r);
938
-
939905 	if (!p->is_write)
940906 		return read_from_write_only(vcpu, p, r);
941907 
....@@ -950,9 +916,6 @@
950916 static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
951917 			     const struct sys_reg_desc *r)
952918 {
953
-	if (!kvm_arm_pmu_v3_ready(vcpu))
954
-		return trap_raz_wi(vcpu, p, r);
955
-
956919 	if (p->is_write) {
957920 		if (!vcpu_mode_priv(vcpu)) {
958921 			kvm_inject_undefined(vcpu);
....@@ -980,78 +943,151 @@
980943 	{ SYS_DESC(SYS_DBGWCRn_EL1(n)),					\
981944 	  trap_wcr, reset_wcr, 0, 0,  get_wcr, set_wcr }
982945 
946
+#define PMU_SYS_REG(r)						\
947
+	SYS_DESC(r), .reset = reset_unknown, .visibility = pmu_visibility
948
+
983949 /* Macro to expand the PMEVCNTRn_EL0 register */
984950 #define PMU_PMEVCNTR_EL0(n)						\
985
-	{ SYS_DESC(SYS_PMEVCNTRn_EL0(n)),					\
986
-	  access_pmu_evcntr, reset_unknown, (PMEVCNTR0_EL0 + n), }
951
+	{ PMU_SYS_REG(SYS_PMEVCNTRn_EL0(n)),				\
952
+	  .access = access_pmu_evcntr, .reg = (PMEVCNTR0_EL0 + n), }
987953 
988954 /* Macro to expand the PMEVTYPERn_EL0 register */
989955 #define PMU_PMEVTYPER_EL0(n)						\
990
-	{ SYS_DESC(SYS_PMEVTYPERn_EL0(n)),					\
991
-	  access_pmu_evtyper, reset_unknown, (PMEVTYPER0_EL0 + n), }
956
+	{ PMU_SYS_REG(SYS_PMEVTYPERn_EL0(n)),				\
957
+	  .access = access_pmu_evtyper, .reg = (PMEVTYPER0_EL0 + n), }
992958 
993
-static bool access_cntp_tval(struct kvm_vcpu *vcpu,
994
-		struct sys_reg_params *p,
995
-		const struct sys_reg_desc *r)
959
+static bool undef_access(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
960
+			 const struct sys_reg_desc *r)
996961 {
997
-	u64 now = kvm_phys_timer_read();
998
-	u64 cval;
962
+	kvm_inject_undefined(vcpu);
999963 
1000
-	if (p->is_write) {
1001
-		kvm_arm_timer_set_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL,
1002
-				      p->regval + now);
1003
-	} else {
1004
-		cval = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL);
1005
-		p->regval = cval - now;
964
+	return false;
965
+}
966
+
967
+/* Macro to expand the AMU counter and type registers*/
968
+#define AMU_AMEVCNTR0_EL0(n) { SYS_DESC(SYS_AMEVCNTR0_EL0(n)), undef_access }
969
+#define AMU_AMEVTYPER0_EL0(n) { SYS_DESC(SYS_AMEVTYPER0_EL0(n)), undef_access }
970
+#define AMU_AMEVCNTR1_EL0(n) { SYS_DESC(SYS_AMEVCNTR1_EL0(n)), undef_access }
971
+#define AMU_AMEVTYPER1_EL0(n) { SYS_DESC(SYS_AMEVTYPER1_EL0(n)), undef_access }
972
+
973
+static unsigned int ptrauth_visibility(const struct kvm_vcpu *vcpu,
974
+			const struct sys_reg_desc *rd)
975
+{
976
+	return vcpu_has_ptrauth(vcpu) ? 0 : REG_HIDDEN;
977
+}
978
+
979
+/*
980
+ * If we land here on a PtrAuth access, that is because we didn't
981
+ * fixup the access on exit by allowing the PtrAuth sysregs. The only
982
+ * way this happens is when the guest does not have PtrAuth support
983
+ * enabled.
984
+ */
985
+#define __PTRAUTH_KEY(k)						\
986
+	{ SYS_DESC(SYS_## k), undef_access, reset_unknown, k,		\
987
+	.visibility = ptrauth_visibility}
988
+
989
+#define PTRAUTH_KEY(k)							\
990
+	__PTRAUTH_KEY(k ## KEYLO_EL1),					\
991
+	__PTRAUTH_KEY(k ## KEYHI_EL1)
992
+
993
+static bool access_arch_timer(struct kvm_vcpu *vcpu,
994
+			      struct sys_reg_params *p,
995
+			      const struct sys_reg_desc *r)
996
+{
997
+	enum kvm_arch_timers tmr;
998
+	enum kvm_arch_timer_regs treg;
999
+	u64 reg = reg_to_encoding(r);
1000
+
1001
+	switch (reg) {
1002
+	case SYS_CNTP_TVAL_EL0:
1003
+	case SYS_AARCH32_CNTP_TVAL:
1004
+		tmr = TIMER_PTIMER;
1005
+		treg = TIMER_REG_TVAL;
1006
+		break;
1007
+	case SYS_CNTP_CTL_EL0:
1008
+	case SYS_AARCH32_CNTP_CTL:
1009
+		tmr = TIMER_PTIMER;
1010
+		treg = TIMER_REG_CTL;
1011
+		break;
1012
+	case SYS_CNTP_CVAL_EL0:
1013
+	case SYS_AARCH32_CNTP_CVAL:
1014
+		tmr = TIMER_PTIMER;
1015
+		treg = TIMER_REG_CVAL;
1016
+		break;
1017
+	default:
1018
+		BUG();
10061019 	}
10071020 
1008
-	return true;
1009
-}
1010
-
1011
-static bool access_cntp_ctl(struct kvm_vcpu *vcpu,
1012
-		struct sys_reg_params *p,
1013
-		const struct sys_reg_desc *r)
1014
-{
10151021 	if (p->is_write)
1016
-		kvm_arm_timer_set_reg(vcpu, KVM_REG_ARM_PTIMER_CTL, p->regval);
1022
+		kvm_arm_timer_write_sysreg(vcpu, tmr, treg, p->regval);
10171023 	else
1018
-		p->regval = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_PTIMER_CTL);
1024
+		p->regval = kvm_arm_timer_read_sysreg(vcpu, tmr, treg);
10191025 
10201026 	return true;
10211027 }
10221028 
1023
-static bool access_cntp_cval(struct kvm_vcpu *vcpu,
1024
-		struct sys_reg_params *p,
1025
-		const struct sys_reg_desc *r)
1026
-{
1027
-	if (p->is_write)
1028
-		kvm_arm_timer_set_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL, p->regval);
1029
-	else
1030
-		p->regval = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_PTIMER_CVAL);
1031
-
1032
-	return true;
1033
-}
1029
+#define FEATURE(x)	(GENMASK_ULL(x##_SHIFT + 3, x##_SHIFT))
10341030 
10351031 /* Read a sanitised cpufeature ID register by sys_reg_desc */
1036
-static u64 read_id_reg(struct sys_reg_desc const *r, bool raz)
1032
+static u64 read_id_reg(const struct kvm_vcpu *vcpu,
1033
+		struct sys_reg_desc const *r, bool raz)
10371034 {
1038
-	u32 id = sys_reg((u32)r->Op0, (u32)r->Op1,
1039
-			 (u32)r->CRn, (u32)r->CRm, (u32)r->Op2);
1035
+	u32 id = reg_to_encoding(r);
10401036 	u64 val = raz ? 0 : read_sanitised_ftr_reg(id);
10411037 
1042
-	if (id == SYS_ID_AA64PFR0_EL1) {
1043
-		if (val & (0xfUL << ID_AA64PFR0_SVE_SHIFT))
1044
-			kvm_debug("SVE unsupported for guests, suppressing\n");
1045
-
1046
-		val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT);
1047
-	} else if (id == SYS_ID_AA64MMFR1_EL1) {
1048
-		if (val & (0xfUL << ID_AA64MMFR1_LOR_SHIFT))
1049
-			kvm_debug("LORegions unsupported for guests, suppressing\n");
1050
-
1051
-		val &= ~(0xfUL << ID_AA64MMFR1_LOR_SHIFT);
1038
+	switch (id) {
1039
+	case SYS_ID_AA64PFR0_EL1:
1040
+		if (!vcpu_has_sve(vcpu))
1041
+			val &= ~FEATURE(ID_AA64PFR0_SVE);
1042
+		val &= ~FEATURE(ID_AA64PFR0_AMU);
1043
+		val &= ~FEATURE(ID_AA64PFR0_CSV2);
1044
+		val |= FIELD_PREP(FEATURE(ID_AA64PFR0_CSV2), (u64)vcpu->kvm->arch.pfr0_csv2);
1045
+		val &= ~FEATURE(ID_AA64PFR0_CSV3);
1046
+		val |= FIELD_PREP(FEATURE(ID_AA64PFR0_CSV3), (u64)vcpu->kvm->arch.pfr0_csv3);
1047
+		break;
1048
+	case SYS_ID_AA64PFR1_EL1:
1049
+		val &= ~FEATURE(ID_AA64PFR1_MTE);
1050
+		break;
1051
+	case SYS_ID_AA64ISAR1_EL1:
1052
+		if (!vcpu_has_ptrauth(vcpu))
1053
+			val &= ~(FEATURE(ID_AA64ISAR1_APA) |
1054
+				 FEATURE(ID_AA64ISAR1_API) |
1055
+				 FEATURE(ID_AA64ISAR1_GPA) |
1056
+				 FEATURE(ID_AA64ISAR1_GPI));
1057
+		break;
1058
+	case SYS_ID_AA64DFR0_EL1:
1059
+		/* Limit debug to ARMv8.0 */
1060
+		val &= ~FEATURE(ID_AA64DFR0_DEBUGVER);
1061
+		val |= FIELD_PREP(FEATURE(ID_AA64DFR0_DEBUGVER), 6);
1062
+		/* Limit guests to PMUv3 for ARMv8.4 */
1063
+		val = cpuid_feature_cap_perfmon_field(val,
1064
+						      ID_AA64DFR0_PMUVER_SHIFT,
1065
+						      kvm_vcpu_has_pmu(vcpu) ? ID_AA64DFR0_PMUVER_8_4 : 0);
1066
+		break;
1067
+	case SYS_ID_DFR0_EL1:
1068
+		/* Limit guests to PMUv3 for ARMv8.4 */
1069
+		val = cpuid_feature_cap_perfmon_field(val,
1070
+						      ID_DFR0_PERFMON_SHIFT,
1071
+						      kvm_vcpu_has_pmu(vcpu) ? ID_DFR0_PERFMON_8_4 : 0);
1072
+		break;
10521073 	}
10531074 
10541075 	return val;
1076
+}
1077
+
1078
+static unsigned int id_visibility(const struct kvm_vcpu *vcpu,
1079
+				  const struct sys_reg_desc *r)
1080
+{
1081
+	u32 id = reg_to_encoding(r);
1082
+
1083
+	switch (id) {
1084
+	case SYS_ID_AA64ZFR0_EL1:
1085
+		if (!vcpu_has_sve(vcpu))
1086
+			return REG_RAZ;
1087
+		break;
1088
+	}
1089
+
1090
+	return 0;
10551091 }
10561092 
10571093 /* cpufeature ID register access trap handlers */
....@@ -1064,7 +1100,7 @@
10641100 	if (p->is_write)
10651101 		return write_to_read_only(vcpu, p, r);
10661102 
1067
-	p->regval = read_id_reg(r, raz);
1103
+	p->regval = read_id_reg(vcpu, r, raz);
10681104 	return true;
10691105 }
10701106 
....@@ -1072,7 +1108,9 @@
10721108 			  struct sys_reg_params *p,
10731109 			  const struct sys_reg_desc *r)
10741110 {
1075
-	return __access_id_reg(vcpu, p, r, false);
1111
+	bool raz = sysreg_visible_as_raz(vcpu, r);
1112
+
1113
+	return __access_id_reg(vcpu, p, r, raz);
10761114 }
10771115 
10781116 static bool access_raz_id_reg(struct kvm_vcpu *vcpu,
....@@ -1086,6 +1124,58 @@
10861124 static int reg_to_user(void __user *uaddr, const u64 *val, u64 id);
10871125 static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
10881126 
1127
+/* Visibility overrides for SVE-specific control registers */
1128
+static unsigned int sve_visibility(const struct kvm_vcpu *vcpu,
1129
+				   const struct sys_reg_desc *rd)
1130
+{
1131
+	if (vcpu_has_sve(vcpu))
1132
+		return 0;
1133
+
1134
+	return REG_HIDDEN;
1135
+}
1136
+
1137
+static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu,
1138
+			       const struct sys_reg_desc *rd,
1139
+			       const struct kvm_one_reg *reg, void __user *uaddr)
1140
+{
1141
+	const u64 id = sys_reg_to_index(rd);
1142
+	u8 csv2, csv3;
1143
+	int err;
1144
+	u64 val;
1145
+
1146
+	err = reg_from_user(&val, uaddr, id);
1147
+	if (err)
1148
+		return err;
1149
+
1150
+	/*
1151
+	 * Allow AA64PFR0_EL1.CSV2 to be set from userspace as long as
1152
+	 * it doesn't promise more than what is actually provided (the
1153
+	 * guest could otherwise be covered in ectoplasmic residue).
1154
+	 */
1155
+	csv2 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV2_SHIFT);
1156
+	if (csv2 > 1 ||
1157
+	    (csv2 && arm64_get_spectre_v2_state() != SPECTRE_UNAFFECTED))
1158
+		return -EINVAL;
1159
+
1160
+	/* Same thing for CSV3 */
1161
+	csv3 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV3_SHIFT);
1162
+	if (csv3 > 1 ||
1163
+	    (csv3 && arm64_get_meltdown_state() != SPECTRE_UNAFFECTED))
1164
+		return -EINVAL;
1165
+
1166
+	/* We can only differ with CSV[23], and anything else is an error */
1167
+	val ^= read_id_reg(vcpu, rd, false);
1168
+	val &= ~((0xFUL << ID_AA64PFR0_CSV2_SHIFT) |
1169
+		 (0xFUL << ID_AA64PFR0_CSV3_SHIFT));
1170
+	if (val)
1171
+		return -EINVAL;
1172
+
1173
+	vcpu->kvm->arch.pfr0_csv2 = csv2;
1174
+	vcpu->kvm->arch.pfr0_csv3 = csv3 ;
1175
+
1176
+	return 0;
1177
+}
1178
+
10891179 /*
10901180  * cpufeature ID register user accessors
10911181  *
....@@ -1093,16 +1183,18 @@
10931183  * are stored, and for set_id_reg() we don't allow the effective value
10941184  * to be changed.
10951185  */
1096
-static int __get_id_reg(const struct sys_reg_desc *rd, void __user *uaddr,
1186
+static int __get_id_reg(const struct kvm_vcpu *vcpu,
1187
+			const struct sys_reg_desc *rd, void __user *uaddr,
10971188 			bool raz)
10981189 {
10991190 	const u64 id = sys_reg_to_index(rd);
1100
-	const u64 val = read_id_reg(rd, raz);
1191
+	const u64 val = read_id_reg(vcpu, rd, raz);
11011192 
11021193 	return reg_to_user(uaddr, &val, id);
11031194 }
11041195 
1105
-static int __set_id_reg(const struct sys_reg_desc *rd, void __user *uaddr,
1196
+static int __set_id_reg(const struct kvm_vcpu *vcpu,
1197
+			const struct sys_reg_desc *rd, void __user *uaddr,
11061198 			bool raz)
11071199 {
11081200 	const u64 id = sys_reg_to_index(rd);
....@@ -1114,7 +1206,7 @@
11141206 		return err;
11151207 
11161208 	/* This is what we mean by invariant: you can't change it. */
1117
-	if (val != read_id_reg(rd, raz))
1209
+	if (val != read_id_reg(vcpu, rd, raz))
11181210 		return -EINVAL;
11191211 
11201212 	return 0;
....@@ -1123,25 +1215,89 @@
11231215 static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
11241216 		      const struct kvm_one_reg *reg, void __user *uaddr)
11251217 {
1126
-	return __get_id_reg(rd, uaddr, false);
1218
+	bool raz = sysreg_visible_as_raz(vcpu, rd);
1219
+
1220
+	return __get_id_reg(vcpu, rd, uaddr, raz);
11271221 }
11281222 
11291223 static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
11301224 		      const struct kvm_one_reg *reg, void __user *uaddr)
11311225 {
1132
-	return __set_id_reg(rd, uaddr, false);
1226
+	bool raz = sysreg_visible_as_raz(vcpu, rd);
1227
+
1228
+	return __set_id_reg(vcpu, rd, uaddr, raz);
11331229 }
11341230 
11351231 static int get_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
11361232 			  const struct kvm_one_reg *reg, void __user *uaddr)
11371233 {
1138
-	return __get_id_reg(rd, uaddr, true);
1234
+	return __get_id_reg(vcpu, rd, uaddr, true);
11391235 }
11401236 
11411237 static int set_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
11421238 			  const struct kvm_one_reg *reg, void __user *uaddr)
11431239 {
1144
-	return __set_id_reg(rd, uaddr, true);
1240
+	return __set_id_reg(vcpu, rd, uaddr, true);
1241
+}
1242
+
1243
+static bool access_ctr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1244
+		       const struct sys_reg_desc *r)
1245
+{
1246
+	if (p->is_write)
1247
+		return write_to_read_only(vcpu, p, r);
1248
+
1249
+	p->regval = read_sanitised_ftr_reg(SYS_CTR_EL0);
1250
+	return true;
1251
+}
1252
+
1253
+static bool access_clidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1254
+			 const struct sys_reg_desc *r)
1255
+{
1256
+	if (p->is_write)
1257
+		return write_to_read_only(vcpu, p, r);
1258
+
1259
+	p->regval = read_sysreg(clidr_el1);
1260
+	return true;
1261
+}
1262
+
1263
+static bool access_csselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1264
+			  const struct sys_reg_desc *r)
1265
+{
1266
+	int reg = r->reg;
1267
+
1268
+	if (p->is_write)
1269
+		vcpu_write_sys_reg(vcpu, p->regval, reg);
1270
+	else
1271
+		p->regval = vcpu_read_sys_reg(vcpu, reg);
1272
+	return true;
1273
+}
1274
+
1275
+static bool access_ccsidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1276
+			  const struct sys_reg_desc *r)
1277
+{
1278
+	u32 csselr;
1279
+
1280
+	if (p->is_write)
1281
+		return write_to_read_only(vcpu, p, r);
1282
+
1283
+	csselr = vcpu_read_sys_reg(vcpu, CSSELR_EL1);
1284
+	p->regval = get_ccsidr(csselr);
1285
+
1286
+	/*
1287
+	 * Guests should not be doing cache operations by set/way at all, and
1288
+	 * for this reason, we trap them and attempt to infer the intent, so
1289
+	 * that we can flush the entire guest's address space at the appropriate
1290
+	 * time.
1291
+	 * To prevent this trapping from causing performance problems, let's
1292
+	 * expose the geometry of all data and unified caches (which are
1293
+	 * guaranteed to be PIPT and thus non-aliasing) as 1 set and 1 way.
1294
+	 * [If guests should attempt to infer aliasing properties from the
1295
+	 * geometry (which is not permitted by the architecture), they would
1296
+	 * only do so for virtually indexed caches.]
1297
+	 */
1298
+	if (!(csselr & 1)) // data or unified cache
1299
+		p->regval &= ~GENMASK(27, 3);
1300
+	return true;
11451301 }
11461302 
11471303 /* sys_reg_desc initialiser for known cpufeature ID registers */
....@@ -1150,6 +1306,7 @@
11501306 	.access	= access_id_reg,		\
11511307 	.get_user = get_id_reg,			\
11521308 	.set_user = set_id_reg,			\
1309
+	.visibility = id_visibility,		\
11531310 }
11541311 
11551312 /*
....@@ -1253,25 +1410,26 @@
12531410 	ID_SANITISED(ID_ISAR4_EL1),
12541411 	ID_SANITISED(ID_ISAR5_EL1),
12551412 	ID_SANITISED(ID_MMFR4_EL1),
1256
-	ID_UNALLOCATED(2,7),
1413
+	ID_SANITISED(ID_ISAR6_EL1),
12571414 
12581415 	/* CRm=3 */
12591416 	ID_SANITISED(MVFR0_EL1),
12601417 	ID_SANITISED(MVFR1_EL1),
12611418 	ID_SANITISED(MVFR2_EL1),
12621419 	ID_UNALLOCATED(3,3),
1263
-	ID_UNALLOCATED(3,4),
1264
-	ID_UNALLOCATED(3,5),
1265
-	ID_UNALLOCATED(3,6),
1420
+	ID_SANITISED(ID_PFR2_EL1),
1421
+	ID_HIDDEN(ID_DFR1_EL1),
1422
+	ID_SANITISED(ID_MMFR5_EL1),
12661423 	ID_UNALLOCATED(3,7),
12671424 
12681425 	/* AArch64 ID registers */
12691426 	/* CRm=4 */
1270
-	ID_SANITISED(ID_AA64PFR0_EL1),
1427
+	{ SYS_DESC(SYS_ID_AA64PFR0_EL1), .access = access_id_reg,
1428
+	  .get_user = get_id_reg, .set_user = set_id_aa64pfr0_el1, },
12711429 	ID_SANITISED(ID_AA64PFR1_EL1),
12721430 	ID_UNALLOCATED(4,2),
12731431 	ID_UNALLOCATED(4,3),
1274
-	ID_UNALLOCATED(4,4),
1432
+	ID_SANITISED(ID_AA64ZFR0_EL1),
12751433 	ID_UNALLOCATED(4,5),
12761434 	ID_UNALLOCATED(4,6),
12771435 	ID_UNALLOCATED(4,7),
....@@ -1289,7 +1447,7 @@
12891447 	/* CRm=6 */
12901448 	ID_SANITISED(ID_AA64ISAR0_EL1),
12911449 	ID_SANITISED(ID_AA64ISAR1_EL1),
1292
-	ID_UNALLOCATED(6,2),
1450
+	ID_SANITISED(ID_AA64ISAR2_EL1),
12931451 	ID_UNALLOCATED(6,3),
12941452 	ID_UNALLOCATED(6,4),
12951453 	ID_UNALLOCATED(6,5),
....@@ -1307,10 +1465,23 @@
13071465 	ID_UNALLOCATED(7,7),
13081466 
13091467 	{ SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
1468
+	{ SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
13101469 	{ SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
1470
+
1471
+	{ SYS_DESC(SYS_RGSR_EL1), undef_access },
1472
+	{ SYS_DESC(SYS_GCR_EL1), undef_access },
1473
+
1474
+	{ SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility },
1475
+	{ SYS_DESC(SYS_TRFCR_EL1), undef_access },
13111476 	{ SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
13121477 	{ SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
13131478 	{ SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 },
1479
+
1480
+	PTRAUTH_KEY(APIA),
1481
+	PTRAUTH_KEY(APIB),
1482
+	PTRAUTH_KEY(APDA),
1483
+	PTRAUTH_KEY(APDB),
1484
+	PTRAUTH_KEY(APGA),
13141485 
13151486 	{ SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
13161487 	{ SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
....@@ -1325,20 +1496,26 @@
13251496 	{ SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
13261497 	{ SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
13271498 
1499
+	{ SYS_DESC(SYS_TFSR_EL1), undef_access },
1500
+	{ SYS_DESC(SYS_TFSRE0_EL1), undef_access },
1501
+
13281502 	{ SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
13291503 	{ SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
13301504 
1331
-	{ SYS_DESC(SYS_PMINTENSET_EL1), access_pminten, reset_unknown, PMINTENSET_EL1 },
1332
-	{ SYS_DESC(SYS_PMINTENCLR_EL1), access_pminten, NULL, PMINTENSET_EL1 },
1505
+	{ PMU_SYS_REG(SYS_PMINTENSET_EL1),
1506
+	  .access = access_pminten, .reg = PMINTENSET_EL1 },
1507
+	{ PMU_SYS_REG(SYS_PMINTENCLR_EL1),
1508
+	  .access = access_pminten, .reg = PMINTENSET_EL1 },
1509
+	{ SYS_DESC(SYS_PMMIR_EL1), trap_raz_wi },
13331510 
13341511 	{ SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
13351512 	{ SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
13361513 
1337
-	{ SYS_DESC(SYS_LORSA_EL1), trap_undef },
1338
-	{ SYS_DESC(SYS_LOREA_EL1), trap_undef },
1339
-	{ SYS_DESC(SYS_LORN_EL1), trap_undef },
1340
-	{ SYS_DESC(SYS_LORC_EL1), trap_undef },
1341
-	{ SYS_DESC(SYS_LORID_EL1), trap_undef },
1514
+	{ SYS_DESC(SYS_LORSA_EL1), trap_loregion },
1515
+	{ SYS_DESC(SYS_LOREA_EL1), trap_loregion },
1516
+	{ SYS_DESC(SYS_LORN_EL1), trap_loregion },
1517
+	{ SYS_DESC(SYS_LORC_EL1), trap_loregion },
1518
+	{ SYS_DESC(SYS_LORID_EL1), trap_loregion },
13421519 
13431520 	{ SYS_DESC(SYS_VBAR_EL1), NULL, reset_val, VBAR_EL1, 0 },
13441521 	{ SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 },
....@@ -1359,34 +1536,127 @@
13591536 	{ SYS_DESC(SYS_CONTEXTIDR_EL1), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
13601537 	{ SYS_DESC(SYS_TPIDR_EL1), NULL, reset_unknown, TPIDR_EL1 },
13611538 
1539
+	{ SYS_DESC(SYS_SCXTNUM_EL1), undef_access },
1540
+
13621541 	{ SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0},
13631542 
1364
-	{ SYS_DESC(SYS_CSSELR_EL1), NULL, reset_unknown, CSSELR_EL1 },
1543
+	{ SYS_DESC(SYS_CCSIDR_EL1), access_ccsidr },
1544
+	{ SYS_DESC(SYS_CLIDR_EL1), access_clidr },
1545
+	{ SYS_DESC(SYS_CSSELR_EL1), access_csselr, reset_unknown, CSSELR_EL1 },
1546
+	{ SYS_DESC(SYS_CTR_EL0), access_ctr },
13651547 
1366
-	{ SYS_DESC(SYS_PMCR_EL0), access_pmcr, reset_pmcr, PMCR_EL0 },
1367
-	{ SYS_DESC(SYS_PMCNTENSET_EL0), access_pmcnten, reset_unknown, PMCNTENSET_EL0 },
1368
-	{ SYS_DESC(SYS_PMCNTENCLR_EL0), access_pmcnten, NULL, PMCNTENSET_EL0 },
1369
-	{ SYS_DESC(SYS_PMOVSCLR_EL0), access_pmovs, NULL, PMOVSSET_EL0 },
1370
-	{ SYS_DESC(SYS_PMSWINC_EL0), access_pmswinc, reset_unknown, PMSWINC_EL0 },
1371
-	{ SYS_DESC(SYS_PMSELR_EL0), access_pmselr, reset_unknown, PMSELR_EL0 },
1372
-	{ SYS_DESC(SYS_PMCEID0_EL0), access_pmceid },
1373
-	{ SYS_DESC(SYS_PMCEID1_EL0), access_pmceid },
1374
-	{ SYS_DESC(SYS_PMCCNTR_EL0), access_pmu_evcntr, reset_unknown, PMCCNTR_EL0 },
1375
-	{ SYS_DESC(SYS_PMXEVTYPER_EL0), access_pmu_evtyper },
1376
-	{ SYS_DESC(SYS_PMXEVCNTR_EL0), access_pmu_evcntr },
1548
+	{ PMU_SYS_REG(SYS_PMCR_EL0), .access = access_pmcr,
1549
+	  .reset = reset_pmcr, .reg = PMCR_EL0 },
1550
+	{ PMU_SYS_REG(SYS_PMCNTENSET_EL0),
1551
+	  .access = access_pmcnten, .reg = PMCNTENSET_EL0 },
1552
+	{ PMU_SYS_REG(SYS_PMCNTENCLR_EL0),
1553
+	  .access = access_pmcnten, .reg = PMCNTENSET_EL0 },
1554
+	{ PMU_SYS_REG(SYS_PMOVSCLR_EL0),
1555
+	  .access = access_pmovs, .reg = PMOVSSET_EL0 },
1556
+	{ PMU_SYS_REG(SYS_PMSWINC_EL0),
1557
+	  .access = access_pmswinc, .reg = PMSWINC_EL0 },
1558
+	{ PMU_SYS_REG(SYS_PMSELR_EL0),
1559
+	  .access = access_pmselr, .reg = PMSELR_EL0 },
1560
+	{ PMU_SYS_REG(SYS_PMCEID0_EL0),
1561
+	  .access = access_pmceid, .reset = NULL },
1562
+	{ PMU_SYS_REG(SYS_PMCEID1_EL0),
1563
+	  .access = access_pmceid, .reset = NULL },
1564
+	{ PMU_SYS_REG(SYS_PMCCNTR_EL0),
1565
+	  .access = access_pmu_evcntr, .reg = PMCCNTR_EL0 },
1566
+	{ PMU_SYS_REG(SYS_PMXEVTYPER_EL0),
1567
+	  .access = access_pmu_evtyper, .reset = NULL },
1568
+	{ PMU_SYS_REG(SYS_PMXEVCNTR_EL0),
1569
+	  .access = access_pmu_evcntr, .reset = NULL },
13771570 	/*
13781571 	 * PMUSERENR_EL0 resets as unknown in 64bit mode while it resets as zero
13791572 	 * in 32bit mode. Here we choose to reset it as zero for consistency.
13801573 	 */
1381
-	{ SYS_DESC(SYS_PMUSERENR_EL0), access_pmuserenr, reset_val, PMUSERENR_EL0, 0 },
1382
-	{ SYS_DESC(SYS_PMOVSSET_EL0), access_pmovs, reset_unknown, PMOVSSET_EL0 },
1574
+	{ PMU_SYS_REG(SYS_PMUSERENR_EL0), .access = access_pmuserenr,
1575
+	  .reset = reset_val, .reg = PMUSERENR_EL0, .val = 0 },
1576
+	{ PMU_SYS_REG(SYS_PMOVSSET_EL0),
1577
+	  .access = access_pmovs, .reg = PMOVSSET_EL0 },
13831578 
13841579 	{ SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
13851580 	{ SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
13861581 
1387
-	{ SYS_DESC(SYS_CNTP_TVAL_EL0), access_cntp_tval },
1388
-	{ SYS_DESC(SYS_CNTP_CTL_EL0), access_cntp_ctl },
1389
-	{ SYS_DESC(SYS_CNTP_CVAL_EL0), access_cntp_cval },
1582
+	{ SYS_DESC(SYS_SCXTNUM_EL0), undef_access },
1583
+
1584
+	{ SYS_DESC(SYS_AMCR_EL0), undef_access },
1585
+	{ SYS_DESC(SYS_AMCFGR_EL0), undef_access },
1586
+	{ SYS_DESC(SYS_AMCGCR_EL0), undef_access },
1587
+	{ SYS_DESC(SYS_AMUSERENR_EL0), undef_access },
1588
+	{ SYS_DESC(SYS_AMCNTENCLR0_EL0), undef_access },
1589
+	{ SYS_DESC(SYS_AMCNTENSET0_EL0), undef_access },
1590
+	{ SYS_DESC(SYS_AMCNTENCLR1_EL0), undef_access },
1591
+	{ SYS_DESC(SYS_AMCNTENSET1_EL0), undef_access },
1592
+	AMU_AMEVCNTR0_EL0(0),
1593
+	AMU_AMEVCNTR0_EL0(1),
1594
+	AMU_AMEVCNTR0_EL0(2),
1595
+	AMU_AMEVCNTR0_EL0(3),
1596
+	AMU_AMEVCNTR0_EL0(4),
1597
+	AMU_AMEVCNTR0_EL0(5),
1598
+	AMU_AMEVCNTR0_EL0(6),
1599
+	AMU_AMEVCNTR0_EL0(7),
1600
+	AMU_AMEVCNTR0_EL0(8),
1601
+	AMU_AMEVCNTR0_EL0(9),
1602
+	AMU_AMEVCNTR0_EL0(10),
1603
+	AMU_AMEVCNTR0_EL0(11),
1604
+	AMU_AMEVCNTR0_EL0(12),
1605
+	AMU_AMEVCNTR0_EL0(13),
1606
+	AMU_AMEVCNTR0_EL0(14),
1607
+	AMU_AMEVCNTR0_EL0(15),
1608
+	AMU_AMEVTYPER0_EL0(0),
1609
+	AMU_AMEVTYPER0_EL0(1),
1610
+	AMU_AMEVTYPER0_EL0(2),
1611
+	AMU_AMEVTYPER0_EL0(3),
1612
+	AMU_AMEVTYPER0_EL0(4),
1613
+	AMU_AMEVTYPER0_EL0(5),
1614
+	AMU_AMEVTYPER0_EL0(6),
1615
+	AMU_AMEVTYPER0_EL0(7),
1616
+	AMU_AMEVTYPER0_EL0(8),
1617
+	AMU_AMEVTYPER0_EL0(9),
1618
+	AMU_AMEVTYPER0_EL0(10),
1619
+	AMU_AMEVTYPER0_EL0(11),
1620
+	AMU_AMEVTYPER0_EL0(12),
1621
+	AMU_AMEVTYPER0_EL0(13),
1622
+	AMU_AMEVTYPER0_EL0(14),
1623
+	AMU_AMEVTYPER0_EL0(15),
1624
+	AMU_AMEVCNTR1_EL0(0),
1625
+	AMU_AMEVCNTR1_EL0(1),
1626
+	AMU_AMEVCNTR1_EL0(2),
1627
+	AMU_AMEVCNTR1_EL0(3),
1628
+	AMU_AMEVCNTR1_EL0(4),
1629
+	AMU_AMEVCNTR1_EL0(5),
1630
+	AMU_AMEVCNTR1_EL0(6),
1631
+	AMU_AMEVCNTR1_EL0(7),
1632
+	AMU_AMEVCNTR1_EL0(8),
1633
+	AMU_AMEVCNTR1_EL0(9),
1634
+	AMU_AMEVCNTR1_EL0(10),
1635
+	AMU_AMEVCNTR1_EL0(11),
1636
+	AMU_AMEVCNTR1_EL0(12),
1637
+	AMU_AMEVCNTR1_EL0(13),
1638
+	AMU_AMEVCNTR1_EL0(14),
1639
+	AMU_AMEVCNTR1_EL0(15),
1640
+	AMU_AMEVTYPER1_EL0(0),
1641
+	AMU_AMEVTYPER1_EL0(1),
1642
+	AMU_AMEVTYPER1_EL0(2),
1643
+	AMU_AMEVTYPER1_EL0(3),
1644
+	AMU_AMEVTYPER1_EL0(4),
1645
+	AMU_AMEVTYPER1_EL0(5),
1646
+	AMU_AMEVTYPER1_EL0(6),
1647
+	AMU_AMEVTYPER1_EL0(7),
1648
+	AMU_AMEVTYPER1_EL0(8),
1649
+	AMU_AMEVTYPER1_EL0(9),
1650
+	AMU_AMEVTYPER1_EL0(10),
1651
+	AMU_AMEVTYPER1_EL0(11),
1652
+	AMU_AMEVTYPER1_EL0(12),
1653
+	AMU_AMEVTYPER1_EL0(13),
1654
+	AMU_AMEVTYPER1_EL0(14),
1655
+	AMU_AMEVTYPER1_EL0(15),
1656
+
1657
+	{ SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },
1658
+	{ SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer },
1659
+	{ SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer },
13901660 
13911661 	/* PMEVCNTRn_EL0 */
13921662 	PMU_PMEVCNTR_EL0(0),
....@@ -1456,14 +1726,15 @@
14561726 	 * PMCCFILTR_EL0 resets as unknown in 64bit mode while it resets as zero
14571727 	 * in 32bit mode. Here we choose to reset it as zero for consistency.
14581728 	 */
1459
-	{ SYS_DESC(SYS_PMCCFILTR_EL0), access_pmu_evtyper, reset_val, PMCCFILTR_EL0, 0 },
1729
+	{ PMU_SYS_REG(SYS_PMCCFILTR_EL0), .access = access_pmu_evtyper,
1730
+	  .reset = reset_val, .reg = PMCCFILTR_EL0, .val = 0 },
14601731 
14611732 	{ SYS_DESC(SYS_DACR32_EL2), NULL, reset_unknown, DACR32_EL2 },
14621733 	{ SYS_DESC(SYS_IFSR32_EL2), NULL, reset_unknown, IFSR32_EL2 },
14631734 	{ SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x700 },
14641735 };
14651736 
1466
-static bool trap_dbgidr(struct kvm_vcpu *vcpu,
1737
+static bool trap_dbgdidr(struct kvm_vcpu *vcpu,
14671738 			struct sys_reg_params *p,
14681739 			const struct sys_reg_desc *r)
14691740 {
....@@ -1477,71 +1748,32 @@
14771748 		p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
14781749 			     (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
14791750 			     (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)
1480
-			     | (6 << 16) | (el3 << 14) | (el3 << 12));
1751
+			     | (6 << 16) | (1 << 15) | (el3 << 14) | (el3 << 12));
14811752 		return true;
14821753 	}
14831754 }
14841755 
1485
-static bool trap_debug32(struct kvm_vcpu *vcpu,
1486
-			 struct sys_reg_params *p,
1487
-			 const struct sys_reg_desc *r)
1488
-{
1489
-	if (p->is_write) {
1490
-		vcpu_cp14(vcpu, r->reg) = p->regval;
1491
-		vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
1492
-	} else {
1493
-		p->regval = vcpu_cp14(vcpu, r->reg);
1494
-	}
1495
-
1496
-	return true;
1497
-}
1498
-
1499
-/* AArch32 debug register mappings
1756
+/*
1757
+ * AArch32 debug register mappings
15001758  *
15011759  * AArch32 DBGBVRn is mapped to DBGBVRn_EL1[31:0]
15021760  * AArch32 DBGBXVRn is mapped to DBGBVRn_EL1[63:32]
15031761  *
1504
- * All control registers and watchpoint value registers are mapped to
1505
- * the lower 32 bits of their AArch64 equivalents. We share the trap
1506
- * handlers with the above AArch64 code which checks what mode the
1507
- * system is in.
1762
+ * None of the other registers share their location, so treat them as
1763
+ * if they were 64bit.
15081764  */
1509
-
1510
-static bool trap_xvr(struct kvm_vcpu *vcpu,
1511
-		     struct sys_reg_params *p,
1512
-		     const struct sys_reg_desc *rd)
1513
-{
1514
-	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
1515
-
1516
-	if (p->is_write) {
1517
-		u64 val = *dbg_reg;
1518
-
1519
-		val &= 0xffffffffUL;
1520
-		val |= p->regval << 32;
1521
-		*dbg_reg = val;
1522
-
1523
-		vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
1524
-	} else {
1525
-		p->regval = *dbg_reg >> 32;
1526
-	}
1527
-
1528
-	trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
1529
-
1530
-	return true;
1531
-}
1532
-
1533
-#define DBG_BCR_BVR_WCR_WVR(n)						\
1534
-	/* DBGBVRn */							\
1535
-	{ Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, 	\
1536
-	/* DBGBCRn */							\
1537
-	{ Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n },	\
1538
-	/* DBGWVRn */							\
1539
-	{ Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n },	\
1540
-	/* DBGWCRn */							\
1765
+#define DBG_BCR_BVR_WCR_WVR(n)						      \
1766
+	/* DBGBVRn */							      \
1767
+	{ AA32(LO), Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, \
1768
+	/* DBGBCRn */							      \
1769
+	{ Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n },	      \
1770
+	/* DBGWVRn */							      \
1771
+	{ Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n },	      \
1772
+	/* DBGWCRn */							      \
15411773 	{ Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n }
15421774 
1543
-#define DBGBXVR(n)							\
1544
-	{ Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_xvr, NULL, n }
1775
+#define DBGBXVR(n)							      \
1776
+	{ AA32(HI), Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_bvr, NULL, n }
15451777 
15461778 /*
15471779  * Trapped cp14 registers. We generally ignore most of the external
....@@ -1549,8 +1781,8 @@
15491781  * guest. Revisit this one day, would this principle change.
15501782  */
15511783 static const struct sys_reg_desc cp14_regs[] = {
1552
-	/* DBGIDR */
1553
-	{ Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgidr },
1784
+	/* DBGDIDR */
1785
+	{ Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgdidr },
15541786 	/* DBGDTRRXext */
15551787 	{ Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
15561788 
....@@ -1559,9 +1791,9 @@
15591791 	{ Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
15601792 	DBG_BCR_BVR_WCR_WVR(1),
15611793 	/* DBGDCCINT */
1562
-	{ Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug32, NULL, cp14_DBGDCCINT },
1794
+	{ Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug_regs, NULL, MDCCINT_EL1 },
15631795 	/* DBGDSCRext */
1564
-	{ Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug32, NULL, cp14_DBGDSCRext },
1796
+	{ Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug_regs, NULL, MDSCR_EL1 },
15651797 	DBG_BCR_BVR_WCR_WVR(2),
15661798 	/* DBGDTR[RT]Xint */
15671799 	{ Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
....@@ -1576,7 +1808,7 @@
15761808 	{ Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
15771809 	DBG_BCR_BVR_WCR_WVR(6),
15781810 	/* DBGVCR */
1579
-	{ Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug32, NULL, cp14_DBGVCR },
1811
+	{ Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug_regs, NULL, DBGVCR32_EL2 },
15801812 	DBG_BCR_BVR_WCR_WVR(7),
15811813 	DBG_BCR_BVR_WCR_WVR(8),
15821814 	DBG_BCR_BVR_WCR_WVR(9),
....@@ -1661,18 +1893,30 @@
16611893  * register).
16621894  */
16631895 static const struct sys_reg_desc cp15_regs[] = {
1664
-	{ Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, c1_SCTLR },
1665
-	{ Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
1666
-	{ Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 },
1667
-	{ Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, c2_TTBCR },
1668
-	{ Op1( 0), CRn( 2), CRm( 0), Op2( 3), access_vm_reg, NULL, c2_TTBCR2 },
1669
-	{ Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, c3_DACR },
1670
-	{ Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, c5_DFSR },
1671
-	{ Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, c5_IFSR },
1672
-	{ Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, c5_ADFSR },
1673
-	{ Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, c5_AIFSR },
1674
-	{ Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, c6_DFAR },
1675
-	{ Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, c6_IFAR },
1896
+	{ Op1( 0), CRn( 0), CRm( 0), Op2( 1), access_ctr },
1897
+	{ Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, SCTLR_EL1 },
1898
+	/* ACTLR */
1899
+	{ AA32(LO), Op1( 0), CRn( 1), CRm( 0), Op2( 1), access_actlr, NULL, ACTLR_EL1 },
1900
+	/* ACTLR2 */
1901
+	{ AA32(HI), Op1( 0), CRn( 1), CRm( 0), Op2( 3), access_actlr, NULL, ACTLR_EL1 },
1902
+	{ Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
1903
+	{ Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, TTBR1_EL1 },
1904
+	/* TTBCR */
1905
+	{ AA32(LO), Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, TCR_EL1 },
1906
+	/* TTBCR2 */
1907
+	{ AA32(HI), Op1( 0), CRn( 2), CRm( 0), Op2( 3), access_vm_reg, NULL, TCR_EL1 },
1908
+	{ Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, DACR32_EL2 },
1909
+	/* DFSR */
1910
+	{ Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, ESR_EL1 },
1911
+	{ Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, IFSR32_EL2 },
1912
+	/* ADFSR */
1913
+	{ Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, AFSR0_EL1 },
1914
+	/* AIFSR */
1915
+	{ Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, AFSR1_EL1 },
1916
+	/* DFAR */
1917
+	{ AA32(LO), Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, FAR_EL1 },
1918
+	/* IFAR */
1919
+	{ AA32(HI), Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, FAR_EL1 },
16761920 
16771921 	/*
16781922 	 * DC{C,I,CI}SW operations:
....@@ -1688,8 +1932,8 @@
16881932 	{ Op1( 0), CRn( 9), CRm(12), Op2( 3), access_pmovs },
16891933 	{ Op1( 0), CRn( 9), CRm(12), Op2( 4), access_pmswinc },
16901934 	{ Op1( 0), CRn( 9), CRm(12), Op2( 5), access_pmselr },
1691
-	{ Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },
1692
-	{ Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },
1935
+	{ AA32(LO), Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },
1936
+	{ AA32(LO), Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },
16931937 	{ Op1( 0), CRn( 9), CRm(13), Op2( 0), access_pmu_evcntr },
16941938 	{ Op1( 0), CRn( 9), CRm(13), Op2( 1), access_pmu_evtyper },
16951939 	{ Op1( 0), CRn( 9), CRm(13), Op2( 2), access_pmu_evcntr },
....@@ -1697,21 +1941,28 @@
16971941 	{ Op1( 0), CRn( 9), CRm(14), Op2( 1), access_pminten },
16981942 	{ Op1( 0), CRn( 9), CRm(14), Op2( 2), access_pminten },
16991943 	{ Op1( 0), CRn( 9), CRm(14), Op2( 3), access_pmovs },
1944
+	{ AA32(HI), Op1( 0), CRn( 9), CRm(14), Op2( 4), access_pmceid },
1945
+	{ AA32(HI), Op1( 0), CRn( 9), CRm(14), Op2( 5), access_pmceid },
1946
+	/* PMMIR */
1947
+	{ Op1( 0), CRn( 9), CRm(14), Op2( 6), trap_raz_wi },
17001948 
1701
-	{ Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, c10_PRRR },
1702
-	{ Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, c10_NMRR },
1703
-	{ Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, c10_AMAIR0 },
1704
-	{ Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, c10_AMAIR1 },
1949
+	/* PRRR/MAIR0 */
1950
+	{ AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 },
1951
+	/* NMRR/MAIR1 */
1952
+	{ AA32(HI), Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, MAIR_EL1 },
1953
+	/* AMAIR0 */
1954
+	{ AA32(LO), Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, AMAIR_EL1 },
1955
+	/* AMAIR1 */
1956
+	{ AA32(HI), Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, AMAIR_EL1 },
17051957 
17061958 	/* ICC_SRE */
17071959 	{ Op1( 0), CRn(12), CRm(12), Op2( 5), access_gic_sre },
17081960 
1709
-	{ Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, c13_CID },
1961
+	{ Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, CONTEXTIDR_EL1 },
17101962 
1711
-	/* CNTP_TVAL */
1712
-	{ Op1( 0), CRn(14), CRm( 2), Op2( 0), access_cntp_tval },
1713
-	/* CNTP_CTL */
1714
-	{ Op1( 0), CRn(14), CRm( 2), Op2( 1), access_cntp_ctl },
1963
+	/* Arch Tmers */
1964
+	{ SYS_DESC(SYS_AARCH32_CNTP_TVAL), access_arch_timer },
1965
+	{ SYS_DESC(SYS_AARCH32_CNTP_CTL), access_arch_timer },
17151966 
17161967 	/* PMEVCNTRn */
17171968 	PMU_PMEVCNTR(0),
....@@ -1779,73 +2030,61 @@
17792030 	PMU_PMEVTYPER(30),
17802031 	/* PMCCFILTR */
17812032 	{ Op1(0), CRn(14), CRm(15), Op2(7), access_pmu_evtyper },
2033
+
2034
+	{ Op1(1), CRn( 0), CRm( 0), Op2(0), access_ccsidr },
2035
+	{ Op1(1), CRn( 0), CRm( 0), Op2(1), access_clidr },
2036
+	{ Op1(2), CRn( 0), CRm( 0), Op2(0), access_csselr, NULL, CSSELR_EL1 },
17822037 };
17832038 
17842039 static const struct sys_reg_desc cp15_64_regs[] = {
1785
-	{ Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
2040
+	{ Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
17862041 	{ Op1( 0), CRn( 0), CRm( 9), Op2( 0), access_pmu_evcntr },
17872042 	{ Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI1R */
1788
-	{ Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR1 },
2043
+	{ Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR1_EL1 },
17892044 	{ Op1( 1), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_ASGI1R */
17902045 	{ Op1( 2), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI0R */
1791
-	{ Op1( 2), CRn( 0), CRm(14), Op2( 0), access_cntp_cval },
2046
+	{ SYS_DESC(SYS_AARCH32_CNTP_CVAL),    access_arch_timer },
17922047 };
17932048 
1794
-/* Target specific emulation tables */
1795
-static struct kvm_sys_reg_target_table *target_tables[KVM_ARM_NUM_TARGETS];
1796
-
1797
-void kvm_register_target_sys_reg_table(unsigned int target,
1798
-				       struct kvm_sys_reg_target_table *table)
2049
+static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n,
2050
+			      bool is_32)
17992051 {
1800
-	target_tables[target] = table;
1801
-}
2052
+	unsigned int i;
18022053 
1803
-/* Get specific register table for this target. */
1804
-static const struct sys_reg_desc *get_target_table(unsigned target,
1805
-						   bool mode_is_64,
1806
-						   size_t *num)
1807
-{
1808
-	struct kvm_sys_reg_target_table *table;
2054
+	for (i = 0; i < n; i++) {
2055
+		if (!is_32 && table[i].reg && !table[i].reset) {
2056
+			kvm_err("sys_reg table %p entry %d has lacks reset\n",
2057
+				table, i);
2058
+			return 1;
2059
+		}
18092060 
1810
-	table = target_tables[target];
1811
-	if (mode_is_64) {
1812
-		*num = table->table64.num;
1813
-		return table->table64.table;
1814
-	} else {
1815
-		*num = table->table32.num;
1816
-		return table->table32.table;
2061
+		if (i && cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
2062
+			kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1);
2063
+			return 1;
2064
+		}
18172065 	}
1818
-}
18192066 
1820
-#define reg_to_match_value(x)						\
1821
-	({								\
1822
-		unsigned long val;					\
1823
-		val  = (x)->Op0 << 14;					\
1824
-		val |= (x)->Op1 << 11;					\
1825
-		val |= (x)->CRn << 7;					\
1826
-		val |= (x)->CRm << 3;					\
1827
-		val |= (x)->Op2;					\
1828
-		val;							\
1829
-	 })
2067
+	return 0;
2068
+}
18302069 
18312070 static int match_sys_reg(const void *key, const void *elt)
18322071 {
18332072 	const unsigned long pval = (unsigned long)key;
18342073 	const struct sys_reg_desc *r = elt;
18352074 
1836
-	return pval - reg_to_match_value(r);
2075
+	return pval - reg_to_encoding(r);
18372076 }
18382077 
18392078 static const struct sys_reg_desc *find_reg(const struct sys_reg_params *params,
18402079 					 const struct sys_reg_desc table[],
18412080 					 unsigned int num)
18422081 {
1843
-	unsigned long pval = reg_to_match_value(params);
2082
+	unsigned long pval = reg_to_encoding(params);
18442083 
18452084 	return bsearch((void *)pval, table, num, sizeof(table[0]), match_sys_reg);
18462085 }
18472086 
1848
-int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run)
2087
+int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu)
18492088 {
18502089 	kvm_inject_undefined(vcpu);
18512090 	return 1;
....@@ -1855,6 +2094,14 @@
18552094 			   struct sys_reg_params *params,
18562095 			   const struct sys_reg_desc *r)
18572096 {
2097
+	trace_kvm_sys_access(*vcpu_pc(vcpu), params, r);
2098
+
2099
+	/* Check for regs disabled by runtime config */
2100
+	if (sysreg_hidden(vcpu, r)) {
2101
+		kvm_inject_undefined(vcpu);
2102
+		return;
2103
+	}
2104
+
18582105 	/*
18592106 	 * Not having an accessor means that we have configured a trap
18602107 	 * that we don't know how to handle. This certainly qualifies
....@@ -1864,7 +2111,7 @@
18642111 
18652112 	/* Skip instruction if instructed so */
18662113 	if (likely(r->access(vcpu, params, r)))
1867
-		kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
2114
+		kvm_incr_pc(vcpu);
18682115 }
18692116 
18702117 /*
....@@ -1901,10 +2148,10 @@
19012148 static void unhandled_cp_access(struct kvm_vcpu *vcpu,
19022149 				struct sys_reg_params *params)
19032150 {
1904
-	u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu);
2151
+	u8 esr_ec = kvm_vcpu_trap_get_class(vcpu);
19052152 	int cp = -1;
19062153 
1907
-	switch(hsr_ec) {
2154
+	switch (esr_ec) {
19082155 	case ESR_ELx_EC_CP15_32:
19092156 	case ESR_ELx_EC_CP15_64:
19102157 		cp = 15;
....@@ -1917,9 +2164,9 @@
19172164 		WARN_ON(1);
19182165 	}
19192166 
1920
-	kvm_err("Unsupported guest CP%d access at: %08lx\n",
1921
-		cp, *vcpu_pc(vcpu));
1922
-	print_sys_reg_instr(params);
2167
+	print_sys_reg_msg(params,
2168
+			  "Unsupported guest CP%d access at: %08lx [%08lx]\n",
2169
+			  cp, *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
19232170 	kvm_inject_undefined(vcpu);
19242171 }
19252172 
....@@ -1930,22 +2177,18 @@
19302177  */
19312178 static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
19322179 			    const struct sys_reg_desc *global,
1933
-			    size_t nr_global,
1934
-			    const struct sys_reg_desc *target_specific,
1935
-			    size_t nr_specific)
2180
+			    size_t nr_global)
19362181 {
19372182 	struct sys_reg_params params;
1938
-	u32 hsr = kvm_vcpu_get_hsr(vcpu);
2183
+	u32 esr = kvm_vcpu_get_esr(vcpu);
19392184 	int Rt = kvm_vcpu_sys_get_rt(vcpu);
1940
-	int Rt2 = (hsr >> 10) & 0x1f;
2185
+	int Rt2 = (esr >> 10) & 0x1f;
19412186 
1942
-	params.is_aarch32 = true;
1943
-	params.is_32bit = false;
1944
-	params.CRm = (hsr >> 1) & 0xf;
1945
-	params.is_write = ((hsr & 1) == 0);
2187
+	params.CRm = (esr >> 1) & 0xf;
2188
+	params.is_write = ((esr & 1) == 0);
19462189 
19472190 	params.Op0 = 0;
1948
-	params.Op1 = (hsr >> 16) & 0xf;
2191
+	params.Op1 = (esr >> 16) & 0xf;
19492192 	params.Op2 = 0;
19502193 	params.CRn = 0;
19512194 
....@@ -1959,14 +2202,11 @@
19592202 	}
19602203 
19612204 	/*
1962
-	 * Try to emulate the coprocessor access using the target
1963
-	 * specific table first, and using the global table afterwards.
1964
-	 * If either of the tables contains a handler, handle the
2205
+	 * If the table contains a handler, handle the
19652206 	 * potential register operation in the case of a read and return
19662207 	 * with success.
19672208 	 */
1968
-	if (!emulate_cp(vcpu, &params, target_specific, nr_specific) ||
1969
-	    !emulate_cp(vcpu, &params, global, nr_global)) {
2209
+	if (!emulate_cp(vcpu, &params, global, nr_global)) {
19702210 		/* Split up the value between registers for the read side */
19712211 		if (!params.is_write) {
19722212 			vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
....@@ -1987,26 +2227,21 @@
19872227  */
19882228 static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
19892229 			    const struct sys_reg_desc *global,
1990
-			    size_t nr_global,
1991
-			    const struct sys_reg_desc *target_specific,
1992
-			    size_t nr_specific)
2230
+			    size_t nr_global)
19932231 {
19942232 	struct sys_reg_params params;
1995
-	u32 hsr = kvm_vcpu_get_hsr(vcpu);
2233
+	u32 esr = kvm_vcpu_get_esr(vcpu);
19962234 	int Rt  = kvm_vcpu_sys_get_rt(vcpu);
19972235 
1998
-	params.is_aarch32 = true;
1999
-	params.is_32bit = true;
2000
-	params.CRm = (hsr >> 1) & 0xf;
2236
+	params.CRm = (esr >> 1) & 0xf;
20012237 	params.regval = vcpu_get_reg(vcpu, Rt);
2002
-	params.is_write = ((hsr & 1) == 0);
2003
-	params.CRn = (hsr >> 10) & 0xf;
2238
+	params.is_write = ((esr & 1) == 0);
2239
+	params.CRn = (esr >> 10) & 0xf;
20042240 	params.Op0 = 0;
2005
-	params.Op1 = (hsr >> 14) & 0x7;
2006
-	params.Op2 = (hsr >> 17) & 0x7;
2241
+	params.Op1 = (esr >> 14) & 0x7;
2242
+	params.Op2 = (esr >> 17) & 0x7;
20072243 
2008
-	if (!emulate_cp(vcpu, &params, target_specific, nr_specific) ||
2009
-	    !emulate_cp(vcpu, &params, global, nr_global)) {
2244
+	if (!emulate_cp(vcpu, &params, global, nr_global)) {
20102245 		if (!params.is_write)
20112246 			vcpu_set_reg(vcpu, Rt, params.regval);
20122247 		return 1;
....@@ -2016,98 +2251,81 @@
20162251 	return 1;
20172252 }
20182253 
2019
-int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
2254
+int kvm_handle_cp15_64(struct kvm_vcpu *vcpu)
20202255 {
2021
-	const struct sys_reg_desc *target_specific;
2022
-	size_t num;
2023
-
2024
-	target_specific = get_target_table(vcpu->arch.target, false, &num);
2025
-	return kvm_handle_cp_64(vcpu,
2026
-				cp15_64_regs, ARRAY_SIZE(cp15_64_regs),
2027
-				target_specific, num);
2256
+	return kvm_handle_cp_64(vcpu, cp15_64_regs, ARRAY_SIZE(cp15_64_regs));
20282257 }
20292258 
2030
-int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
2259
+int kvm_handle_cp15_32(struct kvm_vcpu *vcpu)
20312260 {
2032
-	const struct sys_reg_desc *target_specific;
2033
-	size_t num;
2034
-
2035
-	target_specific = get_target_table(vcpu->arch.target, false, &num);
2036
-	return kvm_handle_cp_32(vcpu,
2037
-				cp15_regs, ARRAY_SIZE(cp15_regs),
2038
-				target_specific, num);
2261
+	return kvm_handle_cp_32(vcpu, cp15_regs, ARRAY_SIZE(cp15_regs));
20392262 }
20402263 
2041
-int kvm_handle_cp14_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
2264
+int kvm_handle_cp14_64(struct kvm_vcpu *vcpu)
20422265 {
2043
-	return kvm_handle_cp_64(vcpu,
2044
-				cp14_64_regs, ARRAY_SIZE(cp14_64_regs),
2045
-				NULL, 0);
2266
+	return kvm_handle_cp_64(vcpu, cp14_64_regs, ARRAY_SIZE(cp14_64_regs));
20462267 }
20472268 
2048
-int kvm_handle_cp14_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
2269
+int kvm_handle_cp14_32(struct kvm_vcpu *vcpu)
20492270 {
2050
-	return kvm_handle_cp_32(vcpu,
2051
-				cp14_regs, ARRAY_SIZE(cp14_regs),
2052
-				NULL, 0);
2271
+	return kvm_handle_cp_32(vcpu, cp14_regs, ARRAY_SIZE(cp14_regs));
2272
+}
2273
+
2274
+static bool is_imp_def_sys_reg(struct sys_reg_params *params)
2275
+{
2276
+	// See ARM DDI 0487E.a, section D12.3.2
2277
+	return params->Op0 == 3 && (params->CRn & 0b1011) == 0b1011;
20532278 }
20542279 
20552280 static int emulate_sys_reg(struct kvm_vcpu *vcpu,
20562281 			   struct sys_reg_params *params)
20572282 {
2058
-	size_t num;
2059
-	const struct sys_reg_desc *table, *r;
2283
+	const struct sys_reg_desc *r;
20602284 
2061
-	table = get_target_table(vcpu->arch.target, true, &num);
2062
-
2063
-	/* Search target-specific then generic table. */
2064
-	r = find_reg(params, table, num);
2065
-	if (!r)
2066
-		r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2285
+	r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
20672286 
20682287 	if (likely(r)) {
20692288 		perform_access(vcpu, params, r);
2289
+	} else if (is_imp_def_sys_reg(params)) {
2290
+		kvm_inject_undefined(vcpu);
20702291 	} else {
2071
-		kvm_err("Unsupported guest sys_reg access at: %lx\n",
2072
-			*vcpu_pc(vcpu));
2073
-		print_sys_reg_instr(params);
2292
+		print_sys_reg_msg(params,
2293
+				  "Unsupported guest sys_reg access at: %lx [%08lx]\n",
2294
+				  *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
20742295 		kvm_inject_undefined(vcpu);
20752296 	}
20762297 	return 1;
20772298 }
20782299 
2079
-static void reset_sys_reg_descs(struct kvm_vcpu *vcpu,
2080
-				const struct sys_reg_desc *table, size_t num,
2081
-				unsigned long *bmap)
2300
+/**
2301
+ * kvm_reset_sys_regs - sets system registers to reset value
2302
+ * @vcpu: The VCPU pointer
2303
+ *
2304
+ * This function finds the right table above and sets the registers on the
2305
+ * virtual CPU struct to their architecturally defined reset values.
2306
+ */
2307
+void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
20822308 {
20832309 	unsigned long i;
20842310 
2085
-	for (i = 0; i < num; i++)
2086
-		if (table[i].reset) {
2087
-			int reg = table[i].reg;
2088
-
2089
-			table[i].reset(vcpu, &table[i]);
2090
-			if (reg > 0 && reg < NR_SYS_REGS)
2091
-				set_bit(reg, bmap);
2092
-		}
2311
+	for (i = 0; i < ARRAY_SIZE(sys_reg_descs); i++)
2312
+		if (sys_reg_descs[i].reset)
2313
+			sys_reg_descs[i].reset(vcpu, &sys_reg_descs[i]);
20932314 }
20942315 
20952316 /**
20962317  * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
20972318  * @vcpu: The VCPU pointer
2098
- * @run:  The kvm_run struct
20992319  */
2100
-int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run)
2320
+int kvm_handle_sys_reg(struct kvm_vcpu *vcpu)
21012321 {
21022322 	struct sys_reg_params params;
2103
-	unsigned long esr = kvm_vcpu_get_hsr(vcpu);
2323
+	unsigned long esr = kvm_vcpu_get_esr(vcpu);
21042324 	int Rt = kvm_vcpu_sys_get_rt(vcpu);
21052325 	int ret;
21062326 
21072327 	trace_kvm_handle_sys_reg(esr);
21082328 
2109
-	params.is_aarch32 = false;
2110
-	params.is_32bit = false;
21112329 	params.Op0 = (esr >> 20) & 3;
21122330 	params.Op1 = (esr >> 14) & 0x7;
21132331 	params.CRn = (esr >> 10) & 0xf;
....@@ -2171,8 +2389,7 @@
21712389 static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu,
21722390 						    u64 id)
21732391 {
2174
-	size_t num;
2175
-	const struct sys_reg_desc *table, *r;
2392
+	const struct sys_reg_desc *r;
21762393 	struct sys_reg_params params;
21772394 
21782395 	/* We only do sys_reg for now. */
....@@ -2182,10 +2399,7 @@
21822399 	if (!index_to_params(id, &params))
21832400 		return NULL;
21842401 
2185
-	table = get_target_table(vcpu->arch.target, true, &num);
2186
-	r = find_reg(&params, table, num);
2187
-	if (!r)
2188
-		r = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2402
+	r = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
21892403 
21902404 	/* Not saved in the sys_reg array and not otherwise accessible? */
21912405 	if (r && !(r->reg || r->get_user))
....@@ -2210,10 +2424,14 @@
22102424 	}
22112425 
22122426 FUNCTION_INVARIANT(midr_el1)
2213
-FUNCTION_INVARIANT(ctr_el0)
22142427 FUNCTION_INVARIANT(revidr_el1)
22152428 FUNCTION_INVARIANT(clidr_el1)
22162429 FUNCTION_INVARIANT(aidr_el1)
2430
+
2431
+static void get_ctr_el0(struct kvm_vcpu *v, const struct sys_reg_desc *r)
2432
+{
2433
+	((struct sys_reg_desc *)r)->val = read_sanitised_ftr_reg(SYS_CTR_EL0);
2434
+}
22172435 
22182436 /* ->val is filled in by kvm_sys_reg_table_init() */
22192437 static struct sys_reg_desc invariant_sys_regs[] = {
....@@ -2371,6 +2589,10 @@
23712589 	if (!r)
23722590 		return get_invariant_sys_reg(reg->id, uaddr);
23732591 
2592
+	/* Check for regs disabled by runtime config */
2593
+	if (sysreg_hidden(vcpu, r))
2594
+		return -ENOENT;
2595
+
23742596 	if (r->get_user)
23752597 		return (r->get_user)(vcpu, r, reg, uaddr);
23762598 
....@@ -2391,6 +2613,10 @@
23912613 	r = index_to_sys_reg_desc(vcpu, reg->id);
23922614 	if (!r)
23932615 		return set_invariant_sys_reg(reg->id, uaddr);
2616
+
2617
+	/* Check for regs disabled by runtime config */
2618
+	if (sysreg_hidden(vcpu, r))
2619
+		return -ENOENT;
23942620 
23952621 	if (r->set_user)
23962622 		return (r->set_user)(vcpu, r, reg, uaddr);
....@@ -2448,7 +2674,8 @@
24482674 	return true;
24492675 }
24502676 
2451
-static int walk_one_sys_reg(const struct sys_reg_desc *rd,
2677
+static int walk_one_sys_reg(const struct kvm_vcpu *vcpu,
2678
+			    const struct sys_reg_desc *rd,
24522679 			    u64 __user **uind,
24532680 			    unsigned int *total)
24542681 {
....@@ -2457,6 +2684,9 @@
24572684 	 * and for which no custom user accessor is provided.
24582685 	 */
24592686 	if (!(rd->reg || rd->get_user))
2687
+		return 0;
2688
+
2689
+	if (sysreg_hidden(vcpu, rd))
24602690 		return 0;
24612691 
24622692 	if (!copy_reg_to_user(rd, uind))
....@@ -2469,35 +2699,17 @@
24692699 /* Assumed ordered tables, see kvm_sys_reg_table_init. */
24702700 static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
24712701 {
2472
-	const struct sys_reg_desc *i1, *i2, *end1, *end2;
2702
+	const struct sys_reg_desc *i2, *end2;
24732703 	unsigned int total = 0;
2474
-	size_t num;
24752704 	int err;
24762705 
2477
-	/* We check for duplicates here, to allow arch-specific overrides. */
2478
-	i1 = get_target_table(vcpu->arch.target, true, &num);
2479
-	end1 = i1 + num;
24802706 	i2 = sys_reg_descs;
24812707 	end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
24822708 
2483
-	BUG_ON(i1 == end1 || i2 == end2);
2484
-
2485
-	/* Walk carefully, as both tables may refer to the same register. */
2486
-	while (i1 || i2) {
2487
-		int cmp = cmp_sys_reg(i1, i2);
2488
-		/* target-specific overrides generic entry. */
2489
-		if (cmp <= 0)
2490
-			err = walk_one_sys_reg(i1, &uind, &total);
2491
-		else
2492
-			err = walk_one_sys_reg(i2, &uind, &total);
2493
-
2709
+	while (i2 != end2) {
2710
+		err = walk_one_sys_reg(vcpu, i2++, &uind, &total);
24942711 		if (err)
24952712 			return err;
2496
-
2497
-		if (cmp <= 0 && ++i1 == end1)
2498
-			i1 = NULL;
2499
-		if (cmp >= 0 && ++i2 == end2)
2500
-			i2 = NULL;
25012713 	}
25022714 	return total;
25032715 }
....@@ -2529,32 +2741,18 @@
25292741 	return write_demux_regids(uindices);
25302742 }
25312743 
2532
-static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n)
2533
-{
2534
-	unsigned int i;
2535
-
2536
-	for (i = 1; i < n; i++) {
2537
-		if (cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
2538
-			kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1);
2539
-			return 1;
2540
-		}
2541
-	}
2542
-
2543
-	return 0;
2544
-}
2545
-
25462744 void kvm_sys_reg_table_init(void)
25472745 {
25482746 	unsigned int i;
25492747 	struct sys_reg_desc clidr;
25502748 
25512749 	/* Make sure tables are unique and in order. */
2552
-	BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs)));
2553
-	BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs)));
2554
-	BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs)));
2555
-	BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs)));
2556
-	BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs)));
2557
-	BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs)));
2750
+	BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false));
2751
+	BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true));
2752
+	BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true));
2753
+	BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true));
2754
+	BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true));
2755
+	BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs), false));
25582756 
25592757 	/* We abuse the reset function to overwrite the table itself. */
25602758 	for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
....@@ -2577,30 +2775,4 @@
25772775 			break;
25782776 	/* Clear all higher bits. */
25792777 	cache_levels &= (1 << (i*3))-1;
2580
-}
2581
-
2582
-/**
2583
- * kvm_reset_sys_regs - sets system registers to reset value
2584
- * @vcpu: The VCPU pointer
2585
- *
2586
- * This function finds the right table above and sets the registers on the
2587
- * virtual CPU struct to their architecturally defined reset values.
2588
- */
2589
-void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
2590
-{
2591
-	size_t num;
2592
-	const struct sys_reg_desc *table;
2593
-	DECLARE_BITMAP(bmap, NR_SYS_REGS) = { 0, };
2594
-
2595
-	/* Generic chip reset first (so target could override). */
2596
-	reset_sys_reg_descs(vcpu, sys_reg_descs, ARRAY_SIZE(sys_reg_descs), bmap);
2597
-
2598
-	table = get_target_table(vcpu->arch.target, true, &num);
2599
-	reset_sys_reg_descs(vcpu, table, num, bmap);
2600
-
2601
-	for (num = 1; num < NR_SYS_REGS; num++) {
2602
-		if (WARN(!test_bit(num, bmap),
2603
-			 "Didn't reset __vcpu_sys_reg(%zi)\n", num))
2604
-			break;
2605
-	}
26062778 }