~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* Kernel-based Virtual Machine driver for Linux
3	4	* cpuid support routines
..	..	@@ -6,10 +7,6 @@
6	7	*
7	8	* Copyright 2011 Red Hat, Inc. and/or its affiliates.
8	9	* Copyright IBM Corporation, 2008
9		- *
10		- * This work is licensed under the terms of the GNU GPL, version 2. See
11		- * the COPYING file in the top-level directory.
12		- *
13	10	*/
14	11
15	12	#include <linux/kvm_host.h>
..	..	@@ -26,6 +23,13 @@
26	23	#include "mmu.h"
27	24	#include "trace.h"
28	25	#include "pmu.h"
	26	+
	27	+/*
	28	+ * Unlike "struct cpuinfo_x86.x86_capability", kvm_cpu_caps doesn't need to be
	29	+ * aligned to sizeof(unsigned long) because it's not accessed via bitops.
	30	+ */
	31	+u32 kvm_cpu_caps[NCAPINTS] __read_mostly;
	32	+EXPORT_SYMBOL_GPL(kvm_cpu_caps);
29	33
30	34	static u32 xstate_required_size(u64 xstate_bv, bool compacted)
31	35	{
..	..	@@ -48,86 +52,34 @@
48	52	return ret;
49	53	}
50	54
51		-bool kvm_mpx_supported(void)
	55	+#define F feature_bit
	56	+
	57	+static inline struct kvm_cpuid_entry2 *cpuid_entry2_find(
	58	+ struct kvm_cpuid_entry2 *entries, int nent, u32 function, u32 index)
52	59	{
53		- return ((host_xcr0 & (XFEATURE_MASK_BNDREGS \| XFEATURE_MASK_BNDCSR))
54		- && kvm_x86_ops->mpx_supported());
55		-}
56		-EXPORT_SYMBOL_GPL(kvm_mpx_supported);
	60	+ struct kvm_cpuid_entry2 *e;
	61	+ int i;
57	62
58		-u64 kvm_supported_xcr0(void)
59		-{
60		- u64 xcr0 = KVM_SUPPORTED_XCR0 & host_xcr0;
	63	+ for (i = 0; i < nent; i++) {
	64	+ e = &entries[i];
61	65
62		- if (!kvm_mpx_supported())
63		- xcr0 &= ~(XFEATURE_MASK_BNDREGS \| XFEATURE_MASK_BNDCSR);
	66	+ if (e->function == function && (e->index == index \|\|
	67	+ !(e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX)))
	68	+ return e;
	69	+ }
64	70
65		- return xcr0;
	71	+ return NULL;
66	72	}
67	73
68		-#define F(x) bit(X86_FEATURE_##x)
69		-
70		-/* For scattered features from cpufeatures.h; we currently expose none */
71		-#define KF(x) bit(KVM_CPUID_BIT_##x)
72		-
73		-int kvm_update_cpuid(struct kvm_vcpu *vcpu)
	74	+static int kvm_check_cpuid(struct kvm_cpuid_entry2 *entries, int nent)
74	75	{
75	76	struct kvm_cpuid_entry2 *best;
76		- struct kvm_lapic *apic = vcpu->arch.apic;
77		-
78		- best = kvm_find_cpuid_entry(vcpu, 1, 0);
79		- if (!best)
80		- return 0;
81		-
82		- /* Update OSXSAVE bit */
83		- if (boot_cpu_has(X86_FEATURE_XSAVE) && best->function == 0x1) {
84		- best->ecx &= ~F(OSXSAVE);
85		- if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE))
86		- best->ecx \|= F(OSXSAVE);
87		- }
88		-
89		- best->edx &= ~F(APIC);
90		- if (vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)
91		- best->edx \|= F(APIC);
92		-
93		- if (apic) {
94		- if (best->ecx & F(TSC_DEADLINE_TIMER))
95		- apic->lapic_timer.timer_mode_mask = 3 << 17;
96		- else
97		- apic->lapic_timer.timer_mode_mask = 1 << 17;
98		- }
99		-
100		- best = kvm_find_cpuid_entry(vcpu, 7, 0);
101		- if (best) {
102		- /* Update OSPKE bit */
103		- if (boot_cpu_has(X86_FEATURE_PKU) && best->function == 0x7) {
104		- best->ecx &= ~F(OSPKE);
105		- if (kvm_read_cr4_bits(vcpu, X86_CR4_PKE))
106		- best->ecx \|= F(OSPKE);
107		- }
108		- }
109		-
110		- best = kvm_find_cpuid_entry(vcpu, 0xD, 0);
111		- if (!best) {
112		- vcpu->arch.guest_supported_xcr0 = 0;
113		- vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
114		- } else {
115		- vcpu->arch.guest_supported_xcr0 =
116		- (best->eax \| ((u64)best->edx << 32)) &
117		- kvm_supported_xcr0();
118		- vcpu->arch.guest_xstate_size = best->ebx =
119		- xstate_required_size(vcpu->arch.xcr0, false);
120		- }
121		-
122		- best = kvm_find_cpuid_entry(vcpu, 0xD, 1);
123		- if (best && (best->eax & (F(XSAVES) \| F(XSAVEC))))
124		- best->ebx = xstate_required_size(vcpu->arch.xcr0, true);
125	77
126	78	/*
127	79	* The existing code assumes virtual address is 48-bit or 57-bit in the
128	80	* canonical address checks; exit if it is ever changed.
129	81	*/
130		- best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
	82	+ best = cpuid_entry2_find(entries, nent, 0x80000008, 0);
131	83	if (best) {
132	84	int vaddr_bits = (best->eax & 0xff00) >> 8;
133	85
..	..	@@ -135,25 +87,106 @@
135	87	return -EINVAL;
136	88	}
137	89
	90	+ return 0;
	91	+}
	92	+
	93	+void kvm_update_pv_runtime(struct kvm_vcpu *vcpu)
	94	+{
	95	+ struct kvm_cpuid_entry2 *best;
	96	+
	97	+ best = kvm_find_cpuid_entry(vcpu, KVM_CPUID_FEATURES, 0);
	98	+
	99	+ /*
	100	+ * save the feature bitmap to avoid cpuid lookup for every PV
	101	+ * operation
	102	+ */
	103	+ if (best)
	104	+ vcpu->arch.pv_cpuid.features = best->eax;
	105	+}
	106	+
	107	+void kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu)
	108	+{
	109	+ struct kvm_cpuid_entry2 *best;
	110	+
	111	+ best = kvm_find_cpuid_entry(vcpu, 1, 0);
	112	+ if (best) {
	113	+ /* Update OSXSAVE bit */
	114	+ if (boot_cpu_has(X86_FEATURE_XSAVE))
	115	+ cpuid_entry_change(best, X86_FEATURE_OSXSAVE,
	116	+ kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE));
	117	+
	118	+ cpuid_entry_change(best, X86_FEATURE_APIC,
	119	+ vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE);
	120	+ }
	121	+
	122	+ best = kvm_find_cpuid_entry(vcpu, 7, 0);
	123	+ if (best && boot_cpu_has(X86_FEATURE_PKU) && best->function == 0x7)
	124	+ cpuid_entry_change(best, X86_FEATURE_OSPKE,
	125	+ kvm_read_cr4_bits(vcpu, X86_CR4_PKE));
	126	+
	127	+ best = kvm_find_cpuid_entry(vcpu, 0xD, 0);
	128	+ if (best)
	129	+ best->ebx = xstate_required_size(vcpu->arch.xcr0, false);
	130	+
	131	+ best = kvm_find_cpuid_entry(vcpu, 0xD, 1);
	132	+ if (best && (cpuid_entry_has(best, X86_FEATURE_XSAVES) \|\|
	133	+ cpuid_entry_has(best, X86_FEATURE_XSAVEC)))
	134	+ best->ebx = xstate_required_size(vcpu->arch.xcr0, true);
	135	+
138	136	best = kvm_find_cpuid_entry(vcpu, KVM_CPUID_FEATURES, 0);
139	137	if (kvm_hlt_in_guest(vcpu->kvm) && best &&
140	138	(best->eax & (1 << KVM_FEATURE_PV_UNHALT)))
141	139	best->eax &= ~(1 << KVM_FEATURE_PV_UNHALT);
142	140
143		- /* Update physical-address width */
	141	+ if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT)) {
	142	+ best = kvm_find_cpuid_entry(vcpu, 0x1, 0);
	143	+ if (best)
	144	+ cpuid_entry_change(best, X86_FEATURE_MWAIT,
	145	+ vcpu->arch.ia32_misc_enable_msr &
	146	+ MSR_IA32_MISC_ENABLE_MWAIT);
	147	+ }
	148	+}
	149	+
	150	+static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
	151	+{
	152	+ struct kvm_lapic *apic = vcpu->arch.apic;
	153	+ struct kvm_cpuid_entry2 *best;
	154	+
	155	+ best = kvm_find_cpuid_entry(vcpu, 1, 0);
	156	+ if (best && apic) {
	157	+ if (cpuid_entry_has(best, X86_FEATURE_TSC_DEADLINE_TIMER))
	158	+ apic->lapic_timer.timer_mode_mask = 3 << 17;
	159	+ else
	160	+ apic->lapic_timer.timer_mode_mask = 1 << 17;
	161	+
	162	+ kvm_apic_set_version(vcpu);
	163	+ }
	164	+
	165	+ best = kvm_find_cpuid_entry(vcpu, 0xD, 0);
	166	+ if (!best)
	167	+ vcpu->arch.guest_supported_xcr0 = 0;
	168	+ else
	169	+ vcpu->arch.guest_supported_xcr0 =
	170	+ (best->eax \| ((u64)best->edx << 32)) & supported_xcr0;
	171	+
	172	+ kvm_update_pv_runtime(vcpu);
	173	+
144	174	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
145	175	kvm_mmu_reset_context(vcpu);
146	176
147	177	kvm_pmu_refresh(vcpu);
148		- return 0;
	178	+ vcpu->arch.cr4_guest_rsvd_bits =
	179	+ __cr4_reserved_bits(guest_cpuid_has, vcpu);
	180	+
	181	+ vcpu->arch.cr3_lm_rsvd_bits = rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
	182	+
	183	+ /* Invoke the vendor callback only after the above state is updated. */
	184	+ kvm_x86_ops.vcpu_after_set_cpuid(vcpu);
149	185	}
150	186
151	187	static int is_efer_nx(void)
152	188	{
153		- unsigned long long efer = 0;
154		-
155		- rdmsrl_safe(MSR_EFER, &efer);
156		- return efer & EFER_NX;
	189	+ return host_efer & EFER_NX;
157	190	}
158	191
159	192	static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
..	..	@@ -169,8 +202,8 @@
169	202	break;
170	203	}
171	204	}
172		- if (entry && (entry->edx & F(NX)) && !is_efer_nx()) {
173		- entry->edx &= ~F(NX);
	205	+ if (entry && cpuid_entry_has(entry, X86_FEATURE_NX) && !is_efer_nx()) {
	206	+ cpuid_entry_clear(entry, X86_FEATURE_NX);
174	207	printk(KERN_INFO "kvm: guest NX capability removed\n");
175	208	}
176	209	}
..	..	@@ -188,7 +221,6 @@
188	221	not_found:
189	222	return 36;
190	223	}
191		-EXPORT_SYMBOL_GPL(cpuid_query_maxphyaddr);
192	224
193	225	/* when an old userspace process fills a new kernel module */
194	226	int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
..	..	@@ -196,43 +228,53 @@
196	228	struct kvm_cpuid_entry __user *entries)
197	229	{
198	230	int r, i;
199		- struct kvm_cpuid_entry *cpuid_entries = NULL;
	231	+ struct kvm_cpuid_entry *e = NULL;
	232	+ struct kvm_cpuid_entry2 *e2 = NULL;
200	233
201		- r = -E2BIG;
202	234	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
203		- goto out;
204		- r = -ENOMEM;
	235	+ return -E2BIG;
	236	+
205	237	if (cpuid->nent) {
206		- cpuid_entries =
207		- vmalloc(array_size(sizeof(struct kvm_cpuid_entry),
208		- cpuid->nent));
209		- if (!cpuid_entries)
210		- goto out;
211		- r = -EFAULT;
212		- if (copy_from_user(cpuid_entries, entries,
213		- cpuid->nent * sizeof(struct kvm_cpuid_entry)))
214		- goto out;
	238	+ e = vmemdup_user(entries, array_size(sizeof(*e), cpuid->nent));
	239	+ if (IS_ERR(e))
	240	+ return PTR_ERR(e);
	241	+
	242	+ e2 = kvmalloc_array(cpuid->nent, sizeof(*e2), GFP_KERNEL_ACCOUNT);
	243	+ if (!e2) {
	244	+ r = -ENOMEM;
	245	+ goto out_free_cpuid;
	246	+ }
215	247	}
216	248	for (i = 0; i < cpuid->nent; i++) {
217		- vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function;
218		- vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax;
219		- vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx;
220		- vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx;
221		- vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx;
222		- vcpu->arch.cpuid_entries[i].index = 0;
223		- vcpu->arch.cpuid_entries[i].flags = 0;
224		- vcpu->arch.cpuid_entries[i].padding[0] = 0;
225		- vcpu->arch.cpuid_entries[i].padding[1] = 0;
226		- vcpu->arch.cpuid_entries[i].padding[2] = 0;
	249	+ e2[i].function = e[i].function;
	250	+ e2[i].eax = e[i].eax;
	251	+ e2[i].ebx = e[i].ebx;
	252	+ e2[i].ecx = e[i].ecx;
	253	+ e2[i].edx = e[i].edx;
	254	+ e2[i].index = 0;
	255	+ e2[i].flags = 0;
	256	+ e2[i].padding[0] = 0;
	257	+ e2[i].padding[1] = 0;
	258	+ e2[i].padding[2] = 0;
227	259	}
228		- vcpu->arch.cpuid_nent = cpuid->nent;
229		- cpuid_fix_nx_cap(vcpu);
230		- kvm_apic_set_version(vcpu);
231		- kvm_x86_ops->cpuid_update(vcpu);
232		- r = kvm_update_cpuid(vcpu);
233	260
234		-out:
235		- vfree(cpuid_entries);
	261	+ r = kvm_check_cpuid(e2, cpuid->nent);
	262	+ if (r) {
	263	+ kvfree(e2);
	264	+ goto out_free_cpuid;
	265	+ }
	266	+
	267	+ kvfree(vcpu->arch.cpuid_entries);
	268	+ vcpu->arch.cpuid_entries = e2;
	269	+ vcpu->arch.cpuid_nent = cpuid->nent;
	270	+
	271	+ cpuid_fix_nx_cap(vcpu);
	272	+ kvm_update_cpuid_runtime(vcpu);
	273	+ kvm_vcpu_after_set_cpuid(vcpu);
	274	+
	275	+out_free_cpuid:
	276	+ kvfree(e);
	277	+
236	278	return r;
237	279	}
238	280
..	..	@@ -240,21 +282,32 @@
240	282	struct kvm_cpuid2 *cpuid,
241	283	struct kvm_cpuid_entry2 __user *entries)
242	284	{
	285	+ struct kvm_cpuid_entry2 *e2 = NULL;
243	286	int r;
244	287
245		- r = -E2BIG;
246	288	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
247		- goto out;
248		- r = -EFAULT;
249		- if (copy_from_user(&vcpu->arch.cpuid_entries, entries,
250		- cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
251		- goto out;
	289	+ return -E2BIG;
	290	+
	291	+ if (cpuid->nent) {
	292	+ e2 = vmemdup_user(entries, array_size(sizeof(*e2), cpuid->nent));
	293	+ if (IS_ERR(e2))
	294	+ return PTR_ERR(e2);
	295	+ }
	296	+
	297	+ r = kvm_check_cpuid(e2, cpuid->nent);
	298	+ if (r) {
	299	+ kvfree(e2);
	300	+ return r;
	301	+ }
	302	+
	303	+ kvfree(vcpu->arch.cpuid_entries);
	304	+ vcpu->arch.cpuid_entries = e2;
252	305	vcpu->arch.cpuid_nent = cpuid->nent;
253		- kvm_apic_set_version(vcpu);
254		- kvm_x86_ops->cpuid_update(vcpu);
255		- r = kvm_update_cpuid(vcpu);
256		-out:
257		- return r;
	306	+
	307	+ kvm_update_cpuid_runtime(vcpu);
	308	+ kvm_vcpu_after_set_cpuid(vcpu);
	309	+
	310	+ return 0;
258	311	}
259	312
260	313	int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
..	..	@@ -267,7 +320,7 @@
267	320	if (cpuid->nent < vcpu->arch.cpuid_nent)
268	321	goto out;
269	322	r = -EFAULT;
270		- if (copy_to_user(entries, &vcpu->arch.cpuid_entries,
	323	+ if (copy_to_user(entries, vcpu->arch.cpuid_entries,
271	324	vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
272	325	goto out;
273	326	return 0;
..	..	@@ -277,70 +330,55 @@
277	330	return r;
278	331	}
279	332
280		-static void cpuid_mask(u32 *word, int wordnum)
	333	+static __always_inline void kvm_cpu_cap_mask(enum cpuid_leafs leaf, u32 mask)
281	334	{
282		- *word &= boot_cpu_data.x86_capability[wordnum];
	335	+ const struct cpuid_reg cpuid = x86_feature_cpuid(leaf * 32);
	336	+ struct kvm_cpuid_entry2 entry;
	337	+
	338	+ reverse_cpuid_check(leaf);
	339	+ kvm_cpu_caps[leaf] &= mask;
	340	+
	341	+ cpuid_count(cpuid.function, cpuid.index,
	342	+ &entry.eax, &entry.ebx, &entry.ecx, &entry.edx);
	343	+
	344	+ kvm_cpu_caps[leaf] &= *__cpuid_entry_get_reg(&entry, cpuid.reg);
283	345	}
284	346
285		-static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
286		- u32 index)
	347	+void kvm_set_cpu_caps(void)
287	348	{
288		- entry->function = function;
289		- entry->index = index;
290		- cpuid_count(entry->function, entry->index,
291		- &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
292		- entry->flags = 0;
293		-}
294		-
295		-static int __do_cpuid_ent_emulated(struct kvm_cpuid_entry2 *entry,
296		- u32 func, u32 index, int *nent, int maxnent)
297		-{
298		- switch (func) {
299		- case 0:
300		- entry->eax = 7;
301		- ++*nent;
302		- break;
303		- case 1:
304		- entry->ecx = F(MOVBE);
305		- ++*nent;
306		- break;
307		- case 7:
308		- entry->flags \|= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
309		- if (index == 0)
310		- entry->ecx = F(RDPID);
311		- ++*nent;
312		- default:
313		- break;
314		- }
315		-
316		- entry->function = func;
317		- entry->index = index;
318		-
319		- return 0;
320		-}
321		-
322		-static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
323		- u32 index, int *nent, int maxnent)
324		-{
325		- int r;
326		- unsigned f_nx = is_efer_nx() ? F(NX) : 0;
	349	+ unsigned int f_nx = is_efer_nx() ? F(NX) : 0;
327	350	#ifdef CONFIG_X86_64
328		- unsigned f_gbpages = (kvm_x86_ops->get_lpage_level() == PT_PDPE_LEVEL)
329		- ? F(GBPAGES) : 0;
330		- unsigned f_lm = F(LM);
	351	+ unsigned int f_gbpages = F(GBPAGES);
	352	+ unsigned int f_lm = F(LM);
331	353	#else
332		- unsigned f_gbpages = 0;
333		- unsigned f_lm = 0;
	354	+ unsigned int f_gbpages = 0;
	355	+ unsigned int f_lm = 0;
334	356	#endif
335		- unsigned f_rdtscp = kvm_x86_ops->rdtscp_supported() ? F(RDTSCP) : 0;
336		- unsigned f_invpcid = kvm_x86_ops->invpcid_supported() ? F(INVPCID) : 0;
337		- unsigned f_mpx = kvm_mpx_supported() ? F(MPX) : 0;
338		- unsigned f_xsaves = kvm_x86_ops->xsaves_supported() ? F(XSAVES) : 0;
339		- unsigned f_umip = kvm_x86_ops->umip_emulated() ? F(UMIP) : 0;
340		- unsigned f_la57 = 0;
341	357
342		- /* cpuid 1.edx */
343		- const u32 kvm_cpuid_1_edx_x86_features =
	358	+ BUILD_BUG_ON(sizeof(kvm_cpu_caps) >
	359	+ sizeof(boot_cpu_data.x86_capability));
	360	+
	361	+ memcpy(&kvm_cpu_caps, &boot_cpu_data.x86_capability,
	362	+ sizeof(kvm_cpu_caps));
	363	+
	364	+ kvm_cpu_cap_mask(CPUID_1_ECX,
	365	+ /*
	366	+ * NOTE: MONITOR (and MWAIT) are emulated as NOP, but not
	367	+ * advertised to guests via CPUID!
	368	+ */
	369	+ F(XMM3) \| F(PCLMULQDQ) \| 0 /* DTES64, MONITOR */ \|
	370	+ 0 /* DS-CPL, VMX, SMX, EST */ \|
	371	+ 0 /* TM2 / \| F(SSSE3) \| 0 / CNXT-ID / \| 0 / Reserved */ \|
	372	+ F(FMA) \| F(CX16) \| 0 /* xTPR Update */ \| F(PDCM) \|
	373	+ F(PCID) \| 0 /* Reserved, DCA */ \| F(XMM4_1) \|
	374	+ F(XMM4_2) \| F(X2APIC) \| F(MOVBE) \| F(POPCNT) \|
	375	+ 0 /* Reserved/ \| F(AES) \| F(XSAVE) \| 0 / OSXSAVE */ \| F(AVX) \|
	376	+ F(F16C) \| F(RDRAND)
	377	+ );
	378	+ /* KVM emulates x2apic in software irrespective of host support. */
	379	+ kvm_cpu_cap_set(X86_FEATURE_X2APIC);
	380	+
	381	+ kvm_cpu_cap_mask(CPUID_1_EDX,
344	382	F(FPU) \| F(VME) \| F(DE) \| F(PSE) \|
345	383	F(TSC) \| F(MSR) \| F(PAE) \| F(MCE) \|
346	384	F(CX8) \| F(APIC) \| 0 /* Reserved */ \| F(SEP) \|
..	..	@@ -348,189 +386,299 @@
348	386	F(PAT) \| F(PSE36) \| 0 /* PSN */ \| F(CLFLUSH) \|
349	387	0 /* Reserved, DS, ACPI */ \| F(MMX) \|
350	388	F(FXSR) \| F(XMM) \| F(XMM2) \| F(SELFSNOOP) \|
351		- 0 /* HTT, TM, Reserved, PBE */;
352		- /* cpuid 0x80000001.edx */
353		- const u32 kvm_cpuid_8000_0001_edx_x86_features =
	389	+ 0 /* HTT, TM, Reserved, PBE */
	390	+ );
	391	+
	392	+ kvm_cpu_cap_mask(CPUID_7_0_EBX,
	393	+ F(FSGSBASE) \| F(BMI1) \| F(HLE) \| F(AVX2) \| F(SMEP) \|
	394	+ F(BMI2) \| F(ERMS) \| 0 /INVPCID/ \| F(RTM) \| 0 /MPX/ \| F(RDSEED) \|
	395	+ F(ADX) \| F(SMAP) \| F(AVX512IFMA) \| F(AVX512F) \| F(AVX512PF) \|
	396	+ F(AVX512ER) \| F(AVX512CD) \| F(CLFLUSHOPT) \| F(CLWB) \| F(AVX512DQ) \|
	397	+ F(SHA_NI) \| F(AVX512BW) \| F(AVX512VL) \| 0 /INTEL_PT/
	398	+ );
	399	+
	400	+ kvm_cpu_cap_mask(CPUID_7_ECX,
	401	+ F(AVX512VBMI) \| F(LA57) \| F(PKU) \| 0 /OSPKE/ \| F(RDPID) \|
	402	+ F(AVX512_VPOPCNTDQ) \| F(UMIP) \| F(AVX512_VBMI2) \| F(GFNI) \|
	403	+ F(VAES) \| F(VPCLMULQDQ) \| F(AVX512_VNNI) \| F(AVX512_BITALG) \|
	404	+ F(CLDEMOTE) \| F(MOVDIRI) \| F(MOVDIR64B) \| 0 /WAITPKG/
	405	+ );
	406	+ /* Set LA57 based on hardware capability. */
	407	+ if (cpuid_ecx(7) & F(LA57))
	408	+ kvm_cpu_cap_set(X86_FEATURE_LA57);
	409	+
	410	+ /*
	411	+ * PKU not yet implemented for shadow paging and requires OSPKE
	412	+ * to be set on the host. Clear it if that is not the case
	413	+ */
	414	+ if (!tdp_enabled \|\| !boot_cpu_has(X86_FEATURE_OSPKE))
	415	+ kvm_cpu_cap_clear(X86_FEATURE_PKU);
	416	+
	417	+ kvm_cpu_cap_mask(CPUID_7_EDX,
	418	+ F(AVX512_4VNNIW) \| F(AVX512_4FMAPS) \| F(SPEC_CTRL) \|
	419	+ F(SPEC_CTRL_SSBD) \| F(ARCH_CAPABILITIES) \| F(INTEL_STIBP) \|
	420	+ F(MD_CLEAR) \| F(AVX512_VP2INTERSECT) \| F(FSRM) \|
	421	+ F(SERIALIZE) \| F(TSXLDTRK)
	422	+ );
	423	+
	424	+ /* TSC_ADJUST and ARCH_CAPABILITIES are emulated in software. */
	425	+ kvm_cpu_cap_set(X86_FEATURE_TSC_ADJUST);
	426	+ kvm_cpu_cap_set(X86_FEATURE_ARCH_CAPABILITIES);
	427	+
	428	+ if (boot_cpu_has(X86_FEATURE_IBPB) && boot_cpu_has(X86_FEATURE_IBRS))
	429	+ kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL);
	430	+ if (boot_cpu_has(X86_FEATURE_STIBP))
	431	+ kvm_cpu_cap_set(X86_FEATURE_INTEL_STIBP);
	432	+ if (boot_cpu_has(X86_FEATURE_AMD_SSBD))
	433	+ kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL_SSBD);
	434	+
	435	+ kvm_cpu_cap_mask(CPUID_7_1_EAX,
	436	+ F(AVX512_BF16)
	437	+ );
	438	+
	439	+ kvm_cpu_cap_mask(CPUID_D_1_EAX,
	440	+ F(XSAVEOPT) \| F(XSAVEC) \| F(XGETBV1) \| F(XSAVES)
	441	+ );
	442	+
	443	+ kvm_cpu_cap_mask(CPUID_8000_0001_ECX,
	444	+ F(LAHF_LM) \| F(CMP_LEGACY) \| 0 /SVM/ \| 0 /* ExtApicSpace */ \|
	445	+ F(CR8_LEGACY) \| F(ABM) \| F(SSE4A) \| F(MISALIGNSSE) \|
	446	+ F(3DNOWPREFETCH) \| F(OSVW) \| 0 /* IBS */ \| F(XOP) \|
	447	+ 0 /* SKINIT, WDT, LWP */ \| F(FMA4) \| F(TBM) \|
	448	+ F(TOPOEXT) \| F(PERFCTR_CORE)
	449	+ );
	450	+
	451	+ kvm_cpu_cap_mask(CPUID_8000_0001_EDX,
354	452	F(FPU) \| F(VME) \| F(DE) \| F(PSE) \|
355	453	F(TSC) \| F(MSR) \| F(PAE) \| F(MCE) \|
356	454	F(CX8) \| F(APIC) \| 0 /* Reserved */ \| F(SYSCALL) \|
357	455	F(MTRR) \| F(PGE) \| F(MCA) \| F(CMOV) \|
358	456	F(PAT) \| F(PSE36) \| 0 /* Reserved */ \|
359	457	f_nx \| 0 /* Reserved */ \| F(MMXEXT) \| F(MMX) \|
360		- F(FXSR) \| F(FXSR_OPT) \| f_gbpages \| f_rdtscp \|
361		- 0 /* Reserved */ \| f_lm \| F(3DNOWEXT) \| F(3DNOW);
362		- /* cpuid 1.ecx */
363		- const u32 kvm_cpuid_1_ecx_x86_features =
364		- /* NOTE: MONITOR (and MWAIT) are emulated as NOP,
365		- * but not advertised to guests via CPUID ! */
366		- F(XMM3) \| F(PCLMULQDQ) \| 0 /* DTES64, MONITOR */ \|
367		- 0 /* DS-CPL, VMX, SMX, EST */ \|
368		- 0 /* TM2 / \| F(SSSE3) \| 0 / CNXT-ID / \| 0 / Reserved */ \|
369		- F(FMA) \| F(CX16) \| 0 /* xTPR Update, PDCM */ \|
370		- F(PCID) \| 0 /* Reserved, DCA */ \| F(XMM4_1) \|
371		- F(XMM4_2) \| F(X2APIC) \| F(MOVBE) \| F(POPCNT) \|
372		- 0 /* Reserved/ \| F(AES) \| F(XSAVE) \| 0 / OSXSAVE */ \| F(AVX) \|
373		- F(F16C) \| F(RDRAND);
374		- /* cpuid 0x80000001.ecx */
375		- const u32 kvm_cpuid_8000_0001_ecx_x86_features =
376		- F(LAHF_LM) \| F(CMP_LEGACY) \| 0 /SVM/ \| 0 /* ExtApicSpace */ \|
377		- F(CR8_LEGACY) \| F(ABM) \| F(SSE4A) \| F(MISALIGNSSE) \|
378		- F(3DNOWPREFETCH) \| F(OSVW) \| 0 /* IBS */ \| F(XOP) \|
379		- 0 /* SKINIT, WDT, LWP */ \| F(FMA4) \| F(TBM) \|
380		- F(TOPOEXT) \| F(PERFCTR_CORE);
	458	+ F(FXSR) \| F(FXSR_OPT) \| f_gbpages \| F(RDTSCP) \|
	459	+ 0 /* Reserved */ \| f_lm \| F(3DNOWEXT) \| F(3DNOW)
	460	+ );
381	461
382		- /* cpuid 0x80000008.ebx */
383		- const u32 kvm_cpuid_8000_0008_ebx_x86_features =
384		- F(AMD_IBPB) \| F(AMD_IBRS) \| F(AMD_SSBD) \| F(VIRT_SSBD) \|
385		- F(AMD_SSB_NO) \| F(AMD_STIBP);
	462	+ if (!tdp_enabled && IS_ENABLED(CONFIG_X86_64))
	463	+ kvm_cpu_cap_set(X86_FEATURE_GBPAGES);
386	464
387		- /* cpuid 0xC0000001.edx */
388		- const u32 kvm_cpuid_C000_0001_edx_x86_features =
	465	+ kvm_cpu_cap_mask(CPUID_8000_0008_EBX,
	466	+ F(CLZERO) \| F(XSAVEERPTR) \|
	467	+ F(WBNOINVD) \| F(AMD_IBPB) \| F(AMD_IBRS) \| F(AMD_SSBD) \| F(VIRT_SSBD) \|
	468	+ F(AMD_SSB_NO) \| F(AMD_STIBP) \| F(AMD_STIBP_ALWAYS_ON)
	469	+ );
	470	+
	471	+ /*
	472	+ * AMD has separate bits for each SPEC_CTRL bit.
	473	+ * arch/x86/kernel/cpu/bugs.c is kind enough to
	474	+ * record that in cpufeatures so use them.
	475	+ */
	476	+ if (boot_cpu_has(X86_FEATURE_IBPB))
	477	+ kvm_cpu_cap_set(X86_FEATURE_AMD_IBPB);
	478	+ if (boot_cpu_has(X86_FEATURE_IBRS))
	479	+ kvm_cpu_cap_set(X86_FEATURE_AMD_IBRS);
	480	+ if (boot_cpu_has(X86_FEATURE_STIBP))
	481	+ kvm_cpu_cap_set(X86_FEATURE_AMD_STIBP);
	482	+ if (boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD))
	483	+ kvm_cpu_cap_set(X86_FEATURE_AMD_SSBD);
	484	+ if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
	485	+ kvm_cpu_cap_set(X86_FEATURE_AMD_SSB_NO);
	486	+ /*
	487	+ * The preference is to use SPEC CTRL MSR instead of the
	488	+ * VIRT_SPEC MSR.
	489	+ */
	490	+ if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
	491	+ !boot_cpu_has(X86_FEATURE_AMD_SSBD))
	492	+ kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
	493	+
	494	+ if (cpu_feature_enabled(X86_FEATURE_SRSO_NO))
	495	+ kvm_cpu_cap_set(X86_FEATURE_SRSO_NO);
	496	+
	497	+ /*
	498	+ * Hide all SVM features by default, SVM will set the cap bits for
	499	+ * features it emulates and/or exposes for L1.
	500	+ */
	501	+ kvm_cpu_cap_mask(CPUID_8000_000A_EDX, 0);
	502	+
	503	+ kvm_cpu_cap_mask(CPUID_C000_0001_EDX,
389	504	F(XSTORE) \| F(XSTORE_EN) \| F(XCRYPT) \| F(XCRYPT_EN) \|
390	505	F(ACE2) \| F(ACE2_EN) \| F(PHE) \| F(PHE_EN) \|
391		- F(PMM) \| F(PMM_EN);
	506	+ F(PMM) \| F(PMM_EN)
	507	+ );
	508	+}
	509	+EXPORT_SYMBOL_GPL(kvm_set_cpu_caps);
392	510
393		- /* cpuid 7.0.ebx */
394		- const u32 kvm_cpuid_7_0_ebx_x86_features =
395		- F(FSGSBASE) \| F(BMI1) \| F(HLE) \| F(AVX2) \| F(SMEP) \|
396		- F(BMI2) \| F(ERMS) \| f_invpcid \| F(RTM) \| f_mpx \| F(RDSEED) \|
397		- F(ADX) \| F(SMAP) \| F(AVX512IFMA) \| F(AVX512F) \| F(AVX512PF) \|
398		- F(AVX512ER) \| F(AVX512CD) \| F(CLFLUSHOPT) \| F(CLWB) \| F(AVX512DQ) \|
399		- F(SHA_NI) \| F(AVX512BW) \| F(AVX512VL);
	511	+struct kvm_cpuid_array {
	512	+ struct kvm_cpuid_entry2 *entries;
	513	+ int maxnent;
	514	+ int nent;
	515	+};
400	516
401		- /* cpuid 0xD.1.eax */
402		- const u32 kvm_cpuid_D_1_eax_x86_features =
403		- F(XSAVEOPT) \| F(XSAVEC) \| F(XGETBV1) \| f_xsaves;
	517	+static struct kvm_cpuid_entry2 get_next_cpuid(struct kvm_cpuid_array array)
	518	+{
	519	+ if (array->nent >= array->maxnent)
	520	+ return NULL;
404	521
405		- /* cpuid 7.0.ecx*/
406		- const u32 kvm_cpuid_7_0_ecx_x86_features =
407		- F(AVX512VBMI) \| F(LA57) \| F(PKU) \| 0 /OSPKE/ \|
408		- F(AVX512_VPOPCNTDQ) \| F(UMIP) \| F(AVX512_VBMI2) \| F(GFNI) \|
409		- F(VAES) \| F(VPCLMULQDQ) \| F(AVX512_VNNI) \| F(AVX512_BITALG) \|
410		- F(CLDEMOTE);
	522	+ return &array->entries[array->nent++];
	523	+}
411	524
412		- /* cpuid 7.0.edx*/
413		- const u32 kvm_cpuid_7_0_edx_x86_features =
414		- F(AVX512_4VNNIW) \| F(AVX512_4FMAPS) \| F(SPEC_CTRL) \|
415		- F(SPEC_CTRL_SSBD) \| F(ARCH_CAPABILITIES) \| F(INTEL_STIBP) \|
416		- F(MD_CLEAR);
	525	+static struct kvm_cpuid_entry2 do_host_cpuid(struct kvm_cpuid_array array,
	526	+ u32 function, u32 index)
	527	+{
	528	+ struct kvm_cpuid_entry2 *entry = get_next_cpuid(array);
	529	+
	530	+ if (!entry)
	531	+ return NULL;
	532	+
	533	+ entry->function = function;
	534	+ entry->index = index;
	535	+ entry->flags = 0;
	536	+
	537	+ cpuid_count(entry->function, entry->index,
	538	+ &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
	539	+
	540	+ switch (function) {
	541	+ case 4:
	542	+ case 7:
	543	+ case 0xb:
	544	+ case 0xd:
	545	+ case 0xf:
	546	+ case 0x10:
	547	+ case 0x12:
	548	+ case 0x14:
	549	+ case 0x17:
	550	+ case 0x18:
	551	+ case 0x1f:
	552	+ case 0x8000001d:
	553	+ entry->flags \|= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
	554	+ break;
	555	+ }
	556	+
	557	+ return entry;
	558	+}
	559	+
	560	+static int __do_cpuid_func_emulated(struct kvm_cpuid_array *array, u32 func)
	561	+{
	562	+ struct kvm_cpuid_entry2 *entry;
	563	+
	564	+ if (array->nent >= array->maxnent)
	565	+ return -E2BIG;
	566	+
	567	+ entry = &array->entries[array->nent];
	568	+ entry->function = func;
	569	+ entry->index = 0;
	570	+ entry->flags = 0;
	571	+
	572	+ switch (func) {
	573	+ case 0:
	574	+ entry->eax = 7;
	575	+ ++array->nent;
	576	+ break;
	577	+ case 1:
	578	+ entry->ecx = F(MOVBE);
	579	+ ++array->nent;
	580	+ break;
	581	+ case 7:
	582	+ entry->flags \|= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
	583	+ entry->eax = 0;
	584	+ if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP))
	585	+ entry->ecx = F(RDPID);
	586	+ ++array->nent;
	587	+ default:
	588	+ break;
	589	+ }
	590	+
	591	+ return 0;
	592	+}
	593	+
	594	+static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
	595	+{
	596	+ struct kvm_cpuid_entry2 *entry;
	597	+ int r, i, max_idx;
417	598
418	599	/* all calls to cpuid_count() should be made on the same cpu */
419	600	get_cpu();
420	601
421	602	r = -E2BIG;
422	603
423		- if (WARN_ON(*nent >= maxnent))
	604	+ entry = do_host_cpuid(array, function, 0);
	605	+ if (!entry)
424	606	goto out;
425		-
426		- do_cpuid_1_ent(entry, function, index);
427		- ++*nent;
428	607
429	608	switch (function) {
430	609	case 0:
431		- entry->eax = min(entry->eax, (u32)0xd);
	610	+ /* Limited to the highest leaf implemented in KVM. */
	611	+ entry->eax = min(entry->eax, 0x1fU);
432	612	break;
433	613	case 1:
434		- entry->edx &= kvm_cpuid_1_edx_x86_features;
435		- cpuid_mask(&entry->edx, CPUID_1_EDX);
436		- entry->ecx &= kvm_cpuid_1_ecx_x86_features;
437		- cpuid_mask(&entry->ecx, CPUID_1_ECX);
438		- /* we support x2apic emulation even if host does not support
439		- * it since we emulate x2apic in software */
440		- entry->ecx \|= F(X2APIC);
	614	+ cpuid_entry_override(entry, CPUID_1_EDX);
	615	+ cpuid_entry_override(entry, CPUID_1_ECX);
441	616	break;
442		- /* function 2 entries are STATEFUL. That is, repeated cpuid commands
443		- * may return different values. This forces us to get_cpu() before
444		- * issuing the first command, and also to emulate this annoying behavior
445		- * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */
446		- case 2: {
447		- int t, times = entry->eax & 0xff;
448		-
449		- entry->flags \|= KVM_CPUID_FLAG_STATEFUL_FUNC;
450		- entry->flags \|= KVM_CPUID_FLAG_STATE_READ_NEXT;
451		- for (t = 1; t < times; ++t) {
452		- if (*nent >= maxnent)
	617	+ case 2:
	618	+ /*
	619	+ * On ancient CPUs, function 2 entries are STATEFUL. That is,
	620	+ * CPUID(function=2, index=0) may return different results each
	621	+ * time, with the least-significant byte in EAX enumerating the
	622	+ * number of times software should do CPUID(2, 0).
	623	+ *
	624	+ * Modern CPUs, i.e. every CPU KVM has ever run on are less
	625	+ * idiotic. Intel's SDM states that EAX & 0xff "will always
	626	+ * return 01H. Software should ignore this value and not
	627	+ * interpret it as an informational descriptor", while AMD's
	628	+ * APM states that CPUID(2) is reserved.
	629	+ *
	630	+ * WARN if a frankenstein CPU that supports virtualization and
	631	+ * a stateful CPUID.0x2 is encountered.
	632	+ */
	633	+ WARN_ON_ONCE((entry->eax & 0xff) > 1);
	634	+ break;
	635	+ /* functions 4 and 0x8000001d have additional index. */
	636	+ case 4:
	637	+ case 0x8000001d:
	638	+ /*
	639	+ * Read entries until the cache type in the previous entry is
	640	+ * zero, i.e. indicates an invalid entry.
	641	+ */
	642	+ for (i = 1; entry->eax & 0x1f; ++i) {
	643	+ entry = do_host_cpuid(array, function, i);
	644	+ if (!entry)
453	645	goto out;
454		-
455		- do_cpuid_1_ent(&entry[t], function, 0);
456		- entry[t].flags \|= KVM_CPUID_FLAG_STATEFUL_FUNC;
457		- ++*nent;
458	646	}
459	647	break;
460		- }
461		- /* function 4 has additional index. */
462		- case 4: {
463		- int i, cache_type;
464		-
465		- entry->flags \|= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
466		- /* read more entries until cache_type is zero */
467		- for (i = 1; ; ++i) {
468		- if (*nent >= maxnent)
469		- goto out;
470		-
471		- cache_type = entry[i - 1].eax & 0x1f;
472		- if (!cache_type)
473		- break;
474		- do_cpuid_1_ent(&entry[i], function, i);
475		- entry[i].flags \|=
476		- KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
477		- ++*nent;
478		- }
479		- break;
480		- }
481	648	case 6: /* Thermal management */
482	649	entry->eax = 0x4; /* allow ARAT */
483	650	entry->ebx = 0;
484	651	entry->ecx = 0;
485	652	entry->edx = 0;
486	653	break;
487		- case 7: {
488		- entry->flags \|= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
489		- /* Mask ebx against host capability word 9 */
490		- if (index == 0) {
491		- entry->ebx &= kvm_cpuid_7_0_ebx_x86_features;
492		- cpuid_mask(&entry->ebx, CPUID_7_0_EBX);
493		- // TSC_ADJUST is emulated
494		- entry->ebx \|= F(TSC_ADJUST);
495		- entry->ecx &= kvm_cpuid_7_0_ecx_x86_features;
496		- f_la57 = entry->ecx & F(LA57);
497		- cpuid_mask(&entry->ecx, CPUID_7_ECX);
498		- /* Set LA57 based on hardware capability. */
499		- entry->ecx \|= f_la57;
500		- entry->ecx \|= f_umip;
501		- /* PKU is not yet implemented for shadow paging. */
502		- if (!tdp_enabled \|\| !boot_cpu_has(X86_FEATURE_OSPKE))
503		- entry->ecx &= ~F(PKU);
	654	+ /* function 7 has additional index. */
	655	+ case 7:
	656	+ entry->eax = min(entry->eax, 1u);
	657	+ cpuid_entry_override(entry, CPUID_7_0_EBX);
	658	+ cpuid_entry_override(entry, CPUID_7_ECX);
	659	+ cpuid_entry_override(entry, CPUID_7_EDX);
504	660
505		- entry->edx &= kvm_cpuid_7_0_edx_x86_features;
506		- cpuid_mask(&entry->edx, CPUID_7_EDX);
507		- if (boot_cpu_has(X86_FEATURE_IBPB) &&
508		- boot_cpu_has(X86_FEATURE_IBRS))
509		- entry->edx \|= F(SPEC_CTRL);
510		- if (boot_cpu_has(X86_FEATURE_STIBP))
511		- entry->edx \|= F(INTEL_STIBP);
512		- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) \|\|
513		- boot_cpu_has(X86_FEATURE_AMD_SSBD))
514		- entry->edx \|= F(SPEC_CTRL_SSBD);
515		- /*
516		- * We emulate ARCH_CAPABILITIES in software even
517		- * if the host doesn't support it.
518		- */
519		- entry->edx \|= F(ARCH_CAPABILITIES);
520		- } else {
	661	+ /* KVM only supports 0x7.0 and 0x7.1, capped above via min(). */
	662	+ if (entry->eax == 1) {
	663	+ entry = do_host_cpuid(array, function, 1);
	664	+ if (!entry)
	665	+ goto out;
	666	+
	667	+ cpuid_entry_override(entry, CPUID_7_1_EAX);
521	668	entry->ebx = 0;
522	669	entry->ecx = 0;
523	670	entry->edx = 0;
524	671	}
525		- entry->eax = 0;
526		- break;
527		- }
528		- case 9:
529	672	break;
530	673	case 0xa: { /* Architectural Performance Monitoring */
531	674	struct x86_pmu_capability cap;
532	675	union cpuid10_eax eax;
533	676	union cpuid10_edx edx;
	677	+
	678	+ if (!static_cpu_has(X86_FEATURE_ARCH_PERFMON)) {
	679	+ entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
	680	+ break;
	681	+ }
534	682
535	683	perf_get_x86_pmu_capability(&cap);
536	684
..	..	@@ -546,9 +694,12 @@
546	694	eax.split.bit_width = cap.bit_width_gp;
547	695	eax.split.mask_length = cap.events_mask_len;
548	696
549		- edx.split.num_counters_fixed = cap.num_counters_fixed;
	697	+ edx.split.num_counters_fixed = min(cap.num_counters_fixed, MAX_FIXED_COUNTERS);
550	698	edx.split.bit_width_fixed = cap.bit_width_fixed;
551		- edx.split.reserved = 0;
	699	+ if (cap.version)
	700	+ edx.split.anythread_deprecated = 1;
	701	+ edx.split.reserved1 = 0;
	702	+ edx.split.reserved2 = 0;
552	703
553	704	entry->eax = eax.full;
554	705	entry->ebx = cap.events_mask;
..	..	@@ -556,67 +707,77 @@
556	707	entry->edx = edx.full;
557	708	break;
558	709	}
559		- /* function 0xb has additional index. */
560		- case 0xb: {
561		- int i, level_type;
562		-
563		- entry->flags \|= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
564		- /* read more entries until level_type is zero */
565		- for (i = 1; ; ++i) {
566		- if (*nent >= maxnent)
567		- goto out;
568		-
569		- level_type = entry[i - 1].ecx & 0xff00;
570		- if (!level_type)
571		- break;
572		- do_cpuid_1_ent(&entry[i], function, i);
573		- entry[i].flags \|=
574		- KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
575		- ++*nent;
576		- }
	710	+ case 0x1f:
	711	+ case 0xb:
	712	+ /*
	713	+ * No topology; a valid topology is indicated by the presence
	714	+ * of subleaf 1.
	715	+ */
	716	+ entry->eax = entry->ebx = entry->ecx = 0;
577	717	break;
578		- }
579		- case 0xd: {
580		- int idx, i;
581		- u64 supported = kvm_supported_xcr0();
582		-
583		- entry->eax &= supported;
584		- entry->ebx = xstate_required_size(supported, false);
	718	+ case 0xd:
	719	+ entry->eax &= supported_xcr0;
	720	+ entry->ebx = xstate_required_size(supported_xcr0, false);
585	721	entry->ecx = entry->ebx;
586		- entry->edx &= supported >> 32;
587		- entry->flags \|= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
588		- if (!supported)
	722	+ entry->edx &= supported_xcr0 >> 32;
	723	+ if (!supported_xcr0)
589	724	break;
590	725
591		- for (idx = 1, i = 1; idx < 64; ++idx) {
592		- u64 mask = ((u64)1 << idx);
593		- if (*nent >= maxnent)
	726	+ entry = do_host_cpuid(array, function, 1);
	727	+ if (!entry)
	728	+ goto out;
	729	+
	730	+ cpuid_entry_override(entry, CPUID_D_1_EAX);
	731	+ if (entry->eax & (F(XSAVES)\|F(XSAVEC)))
	732	+ entry->ebx = xstate_required_size(supported_xcr0 \| supported_xss,
	733	+ true);
	734	+ else {
	735	+ WARN_ON_ONCE(supported_xss != 0);
	736	+ entry->ebx = 0;
	737	+ }
	738	+ entry->ecx &= supported_xss;
	739	+ entry->edx &= supported_xss >> 32;
	740	+
	741	+ for (i = 2; i < 64; ++i) {
	742	+ bool s_state;
	743	+ if (supported_xcr0 & BIT_ULL(i))
	744	+ s_state = false;
	745	+ else if (supported_xss & BIT_ULL(i))
	746	+ s_state = true;
	747	+ else
	748	+ continue;
	749	+
	750	+ entry = do_host_cpuid(array, function, i);
	751	+ if (!entry)
594	752	goto out;
595	753
596		- do_cpuid_1_ent(&entry[i], function, idx);
597		- if (idx == 1) {
598		- entry[i].eax &= kvm_cpuid_D_1_eax_x86_features;
599		- cpuid_mask(&entry[i].eax, CPUID_D_1_EAX);
600		- entry[i].ebx = 0;
601		- if (entry[i].eax & (F(XSAVES)\|F(XSAVEC)))
602		- entry[i].ebx =
603		- xstate_required_size(supported,
604		- true);
605		- } else {
606		- if (entry[i].eax == 0 \|\| !(supported & mask))
607		- continue;
608		- if (WARN_ON_ONCE(entry[i].ecx & 1))
609		- continue;
	754	+ /*
	755	+ * The supported check above should have filtered out
	756	+ * invalid sub-leafs. Only valid sub-leafs should
	757	+ * reach this point, and they should have a non-zero
	758	+ * save state size. Furthermore, check whether the
	759	+ * processor agrees with supported_xcr0/supported_xss
	760	+ * on whether this is an XCR0- or IA32_XSS-managed area.
	761	+ */
	762	+ if (WARN_ON_ONCE(!entry->eax \|\| (entry->ecx & 0x1) != s_state)) {
	763	+ --array->nent;
	764	+ continue;
610	765	}
611		- entry[i].ecx = 0;
612		- entry[i].edx = 0;
613		- entry[i].flags \|=
614		- KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
615		- ++*nent;
616		- ++i;
	766	+ entry->edx = 0;
617	767	}
618	768	break;
619		- }
	769	+ /* Intel PT */
	770	+ case 0x14:
	771	+ if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT)) {
	772	+ entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
	773	+ break;
	774	+ }
	775	+
	776	+ for (i = 1, max_idx = entry->eax; i <= max_idx; ++i) {
	777	+ if (!do_host_cpuid(array, function, i))
	778	+ goto out;
	779	+ }
	780	+ break;
620	781	case KVM_CPUID_SIGNATURE: {
621	782	static const char signature[12] = "KVMKVMKVM\0\0";
622	783	const u32 sigptr = (const u32 )signature;
..	..	@@ -636,7 +797,10 @@
636	797	(1 << KVM_FEATURE_PV_UNHALT) \|
637	798	(1 << KVM_FEATURE_PV_TLB_FLUSH) \|
638	799	(1 << KVM_FEATURE_ASYNC_PF_VMEXIT) \|
639		- (1 << KVM_FEATURE_PV_SEND_IPI);
	800	+ (1 << KVM_FEATURE_PV_SEND_IPI) \|
	801	+ (1 << KVM_FEATURE_POLL_CONTROL) \|
	802	+ (1 << KVM_FEATURE_PV_SCHED_YIELD) \|
	803	+ (1 << KVM_FEATURE_ASYNC_PF_INT);
640	804
641	805	if (sched_info_on())
642	806	entry->eax \|= (1 << KVM_FEATURE_STEAL_TIME);
..	..	@@ -649,10 +813,13 @@
649	813	entry->eax = min(entry->eax, 0x8000001f);
650	814	break;
651	815	case 0x80000001:
652		- entry->edx &= kvm_cpuid_8000_0001_edx_x86_features;
653		- cpuid_mask(&entry->edx, CPUID_8000_0001_EDX);
654		- entry->ecx &= kvm_cpuid_8000_0001_ecx_x86_features;
655		- cpuid_mask(&entry->ecx, CPUID_8000_0001_ECX);
	816	+ entry->ebx &= ~GENMASK(27, 16);
	817	+ cpuid_entry_override(entry, CPUID_8000_0001_EDX);
	818	+ cpuid_entry_override(entry, CPUID_8000_0001_ECX);
	819	+ break;
	820	+ case 0x80000006:
	821	+ /* Drop reserved bits, pass host L2 cache and TLB info. */
	822	+ entry->edx &= ~GENMASK(17, 16);
656	823	break;
657	824	case 0x80000007: /* Advanced power management */
658	825	/* invariant TSC is CPUID.80000007H:EDX[8] */
..	..	@@ -675,34 +842,38 @@
675	842	g_phys_as = phys_as;
676	843
677	844	entry->eax = g_phys_as \| (virt_as << 8);
	845	+ entry->ecx &= ~(GENMASK(31, 16) \| GENMASK(11, 8));
678	846	entry->edx = 0;
679		- /*
680		- * IBRS, IBPB and VIRT_SSBD aren't necessarily present in
681		- * hardware cpuid
682		- */
683		- if (boot_cpu_has(X86_FEATURE_AMD_IBPB))
684		- entry->ebx \|= F(AMD_IBPB);
685		- if (boot_cpu_has(X86_FEATURE_AMD_IBRS))
686		- entry->ebx \|= F(AMD_IBRS);
687		- if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
688		- entry->ebx \|= F(VIRT_SSBD);
689		- entry->ebx &= kvm_cpuid_8000_0008_ebx_x86_features;
690		- cpuid_mask(&entry->ebx, CPUID_8000_0008_EBX);
691		- /*
692		- * The preference is to use SPEC CTRL MSR instead of the
693		- * VIRT_SPEC MSR.
694		- */
695		- if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
696		- !boot_cpu_has(X86_FEATURE_AMD_SSBD))
697		- entry->ebx \|= F(VIRT_SSBD);
	847	+ cpuid_entry_override(entry, CPUID_8000_0008_EBX);
698	848	break;
699	849	}
	850	+ case 0x8000000A:
	851	+ if (!kvm_cpu_cap_has(X86_FEATURE_SVM)) {
	852	+ entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
	853	+ break;
	854	+ }
	855	+ entry->eax = 1; /* SVM revision 1 */
	856	+ entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
	857	+ ASID emulation to nested SVM */
	858	+ entry->ecx = 0; /* Reserved */
	859	+ cpuid_entry_override(entry, CPUID_8000_000A_EDX);
	860	+ break;
700	861	case 0x80000019:
701	862	entry->ecx = entry->edx = 0;
702	863	break;
703	864	case 0x8000001a:
	865	+ entry->eax &= GENMASK(2, 0);
	866	+ entry->ebx = entry->ecx = entry->edx = 0;
704	867	break;
705		- case 0x8000001d:
	868	+ case 0x8000001e:
	869	+ /* Do not return host topology information. */
	870	+ entry->eax = entry->ebx = entry->ecx = 0;
	871	+ entry->edx = 0; /* reserved */
	872	+ break;
	873	+ /* Support memory encryption cpuid if host supports it */
	874	+ case 0x8000001F:
	875	+ if (!boot_cpu_has(X86_FEATURE_SEV))
	876	+ entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
706	877	break;
707	878	/Add support for Centaur's CPUID instruction/
708	879	case 0xC0000000:
..	..	@@ -710,8 +881,7 @@
710	881	entry->eax = min(entry->eax, 0xC0000004);
711	882	break;
712	883	case 0xC0000001:
713		- entry->edx &= kvm_cpuid_C000_0001_edx_x86_features;
714		- cpuid_mask(&entry->edx, CPUID_C000_0001_EDX);
	884	+ cpuid_entry_override(entry, CPUID_C000_0001_EDX);
715	885	break;
716	886	case 3: /* Processor serial number */
717	887	case 5: /* MONITOR/MWAIT */
..	..	@@ -723,8 +893,6 @@
723	893	break;
724	894	}
725	895
726		- kvm_x86_ops->set_supported_cpuid(function, entry);
727		-
728	896	r = 0;
729	897
730	898	out:
..	..	@@ -733,30 +901,39 @@
733	901	return r;
734	902	}
735	903
736		-static int do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 func,
737		- u32 idx, int *nent, int maxnent, unsigned int type)
	904	+static int do_cpuid_func(struct kvm_cpuid_array *array, u32 func,
	905	+ unsigned int type)
738	906	{
739		- if (*nent >= maxnent)
740		- return -E2BIG;
741		-
742	907	if (type == KVM_GET_EMULATED_CPUID)
743		- return __do_cpuid_ent_emulated(entry, func, idx, nent, maxnent);
	908	+ return __do_cpuid_func_emulated(array, func);
744	909
745		- return __do_cpuid_ent(entry, func, idx, nent, maxnent);
	910	+ return __do_cpuid_func(array, func);
746	911	}
747	912
748		-#undef F
	913	+#define CENTAUR_CPUID_SIGNATURE 0xC0000000
749	914
750		-struct kvm_cpuid_param {
751		- u32 func;
752		- u32 idx;
753		- bool has_leaf_count;
754		- bool (qualifier)(const struct kvm_cpuid_param param);
755		-};
756		-
757		-static bool is_centaur_cpu(const struct kvm_cpuid_param *param)
	915	+static int get_cpuid_func(struct kvm_cpuid_array *array, u32 func,
	916	+ unsigned int type)
758	917	{
759		- return boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR;
	918	+ u32 limit;
	919	+ int r;
	920	+
	921	+ if (func == CENTAUR_CPUID_SIGNATURE &&
	922	+ boot_cpu_data.x86_vendor != X86_VENDOR_CENTAUR)
	923	+ return 0;
	924	+
	925	+ r = do_cpuid_func(array, func, type);
	926	+ if (r)
	927	+ return r;
	928	+
	929	+ limit = array->entries[array->nent - 1].eax;
	930	+ for (func = func + 1; func <= limit; ++func) {
	931	+ r = do_cpuid_func(array, func, type);
	932	+ if (r)
	933	+ break;
	934	+ }
	935	+
	936	+ return r;
760	937	}
761	938
762	939	static bool sanity_check_entries(struct kvm_cpuid_entry2 __user *entries,
..	..	@@ -790,172 +967,168 @@
790	967	struct kvm_cpuid_entry2 __user *entries,
791	968	unsigned int type)
792	969	{
793		- struct kvm_cpuid_entry2 *cpuid_entries;
794		- int limit, nent = 0, r = -E2BIG, i;
795		- u32 func;
796		- static const struct kvm_cpuid_param param[] = {
797		- { .func = 0, .has_leaf_count = true },
798		- { .func = 0x80000000, .has_leaf_count = true },
799		- { .func = 0xC0000000, .qualifier = is_centaur_cpu, .has_leaf_count = true },
800		- { .func = KVM_CPUID_SIGNATURE },
801		- { .func = KVM_CPUID_FEATURES },
	970	+ static const u32 funcs[] = {
	971	+ 0, 0x80000000, CENTAUR_CPUID_SIGNATURE, KVM_CPUID_SIGNATURE,
802	972	};
803	973
	974	+ struct kvm_cpuid_array array = {
	975	+ .nent = 0,
	976	+ };
	977	+ int r, i;
	978	+
804	979	if (cpuid->nent < 1)
805		- goto out;
	980	+ return -E2BIG;
806	981	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
807	982	cpuid->nent = KVM_MAX_CPUID_ENTRIES;
808	983
809	984	if (sanity_check_entries(entries, cpuid->nent, type))
810	985	return -EINVAL;
811	986
812		- r = -ENOMEM;
813		- cpuid_entries = vzalloc(array_size(sizeof(struct kvm_cpuid_entry2),
	987	+ array.entries = vzalloc(array_size(sizeof(struct kvm_cpuid_entry2),
814	988	cpuid->nent));
815		- if (!cpuid_entries)
816		- goto out;
	989	+ if (!array.entries)
	990	+ return -ENOMEM;
817	991
818		- r = 0;
819		- for (i = 0; i < ARRAY_SIZE(param); i++) {
820		- const struct kvm_cpuid_param *ent = &param[i];
	992	+ array.maxnent = cpuid->nent;
821	993
822		- if (ent->qualifier && !ent->qualifier(ent))
823		- continue;
824		-
825		- r = do_cpuid_ent(&cpuid_entries[nent], ent->func, ent->idx,
826		- &nent, cpuid->nent, type);
827		-
828		- if (r)
829		- goto out_free;
830		-
831		- if (!ent->has_leaf_count)
832		- continue;
833		-
834		- limit = cpuid_entries[nent - 1].eax;
835		- for (func = ent->func + 1; func <= limit && nent < cpuid->nent && r == 0; ++func)
836		- r = do_cpuid_ent(&cpuid_entries[nent], func, ent->idx,
837		- &nent, cpuid->nent, type);
838		-
	994	+ for (i = 0; i < ARRAY_SIZE(funcs); i++) {
	995	+ r = get_cpuid_func(&array, funcs[i], type);
839	996	if (r)
840	997	goto out_free;
841	998	}
	999	+ cpuid->nent = array.nent;
842	1000
843		- r = -EFAULT;
844		- if (copy_to_user(entries, cpuid_entries,
845		- nent * sizeof(struct kvm_cpuid_entry2)))
846		- goto out_free;
847		- cpuid->nent = nent;
848		- r = 0;
	1001	+ if (copy_to_user(entries, array.entries,
	1002	+ array.nent * sizeof(struct kvm_cpuid_entry2)))
	1003	+ r = -EFAULT;
849	1004
850	1005	out_free:
851		- vfree(cpuid_entries);
852		-out:
	1006	+ vfree(array.entries);
853	1007	return r;
854		-}
855		-
856		-static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i)
857		-{
858		- struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i];
859		- struct kvm_cpuid_entry2 *ej;
860		- int j = i;
861		- int nent = vcpu->arch.cpuid_nent;
862		-
863		- e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT;
864		- /* when no next entry is found, the current entry[i] is reselected */
865		- do {
866		- j = (j + 1) % nent;
867		- ej = &vcpu->arch.cpuid_entries[j];
868		- } while (ej->function != e->function);
869		-
870		- ej->flags \|= KVM_CPUID_FLAG_STATE_READ_NEXT;
871		-
872		- return j;
873		-}
874		-
875		-/* find an entry with matching function, matching index (if needed), and that
876		- * should be read next (if it's stateful) */
877		-static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e,
878		- u32 function, u32 index)
879		-{
880		- if (e->function != function)
881		- return 0;
882		- if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index)
883		- return 0;
884		- if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) &&
885		- !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT))
886		- return 0;
887		- return 1;
888	1008	}
889	1009
890	1010	struct kvm_cpuid_entry2 kvm_find_cpuid_entry(struct kvm_vcpu vcpu,
891	1011	u32 function, u32 index)
892	1012	{
893		- int i;
894		- struct kvm_cpuid_entry2 *best = NULL;
895		-
896		- for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
897		- struct kvm_cpuid_entry2 *e;
898		-
899		- e = &vcpu->arch.cpuid_entries[i];
900		- if (is_matching_cpuid_entry(e, function, index)) {
901		- if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC)
902		- move_to_next_stateful_cpuid_entry(vcpu, i);
903		- best = e;
904		- break;
905		- }
906		- }
907		- return best;
	1013	+ return cpuid_entry2_find(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent,
	1014	+ function, index);
908	1015	}
909	1016	EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry);
910	1017
911	1018	/*
912		- * If no match is found, check whether we exceed the vCPU's limit
913		- * and return the content of the highest valid _standard_ leaf instead.
914		- * This is to satisfy the CPUID specification.
	1019	+ * Intel CPUID semantics treats any query for an out-of-range leaf as if the
	1020	+ * highest basic leaf (i.e. CPUID.0H:EAX) were requested. AMD CPUID semantics
	1021	+ * returns all zeroes for any undefined leaf, whether or not the leaf is in
	1022	+ * range. Centaur/VIA follows Intel semantics.
	1023	+ *
	1024	+ * A leaf is considered out-of-range if its function is higher than the maximum
	1025	+ * supported leaf of its associated class or if its associated class does not
	1026	+ * exist.
	1027	+ *
	1028	+ * There are three primary classes to be considered, with their respective
	1029	+ * ranges described as "<base> - <top>[,<base2> - <top2>] inclusive. A primary
	1030	+ * class exists if a guest CPUID entry for its <base> leaf exists. For a given
	1031	+ * class, CPUID.<base>.EAX contains the max supported leaf for the class.
	1032	+ *
	1033	+ * - Basic: 0x00000000 - 0x3fffffff, 0x50000000 - 0x7fffffff
	1034	+ * - Hypervisor: 0x40000000 - 0x4fffffff
	1035	+ * - Extended: 0x80000000 - 0xbfffffff
	1036	+ * - Centaur: 0xc0000000 - 0xcfffffff
	1037	+ *
	1038	+ * The Hypervisor class is further subdivided into sub-classes that each act as
	1039	+ * their own indepdent class associated with a 0x100 byte range. E.g. if Qemu
	1040	+ * is advertising support for both HyperV and KVM, the resulting Hypervisor
	1041	+ * CPUID sub-classes are:
	1042	+ *
	1043	+ * - HyperV: 0x40000000 - 0x400000ff
	1044	+ * - KVM: 0x40000100 - 0x400001ff
915	1045	*/
916		-static struct kvm_cpuid_entry2* check_cpuid_limit(struct kvm_vcpu *vcpu,
917		- u32 function, u32 index)
	1046	+static struct kvm_cpuid_entry2 *
	1047	+get_out_of_range_cpuid_entry(struct kvm_vcpu vcpu, u32 fn_ptr, u32 index)
918	1048	{
919		- struct kvm_cpuid_entry2 *maxlevel;
	1049	+ struct kvm_cpuid_entry2 basic, class;
	1050	+ u32 function = *fn_ptr;
920	1051
921		- maxlevel = kvm_find_cpuid_entry(vcpu, function & 0x80000000, 0);
922		- if (!maxlevel \|\| maxlevel->eax >= function)
	1052	+ basic = kvm_find_cpuid_entry(vcpu, 0, 0);
	1053	+ if (!basic)
923	1054	return NULL;
924		- if (function & 0x80000000) {
925		- maxlevel = kvm_find_cpuid_entry(vcpu, 0, 0);
926		- if (!maxlevel)
927		- return NULL;
928		- }
929		- return kvm_find_cpuid_entry(vcpu, maxlevel->eax, index);
	1055	+
	1056	+ if (is_guest_vendor_amd(basic->ebx, basic->ecx, basic->edx) \|\|
	1057	+ is_guest_vendor_hygon(basic->ebx, basic->ecx, basic->edx))
	1058	+ return NULL;
	1059	+
	1060	+ if (function >= 0x40000000 && function <= 0x4fffffff)
	1061	+ class = kvm_find_cpuid_entry(vcpu, function & 0xffffff00, 0);
	1062	+ else if (function >= 0xc0000000)
	1063	+ class = kvm_find_cpuid_entry(vcpu, 0xc0000000, 0);
	1064	+ else
	1065	+ class = kvm_find_cpuid_entry(vcpu, function & 0x80000000, 0);
	1066	+
	1067	+ if (class && function <= class->eax)
	1068	+ return NULL;
	1069	+
	1070	+ /*
	1071	+ * Leaf specific adjustments are also applied when redirecting to the
	1072	+ * max basic entry, e.g. if the max basic leaf is 0xb but there is no
	1073	+ * entry for CPUID.0xb.index (see below), then the output value for EDX
	1074	+ * needs to be pulled from CPUID.0xb.1.
	1075	+ */
	1076	+ *fn_ptr = basic->eax;
	1077	+
	1078	+ /*
	1079	+ * The class does not exist or the requested function is out of range;
	1080	+ * the effective CPUID entry is the max basic leaf. Note, the index of
	1081	+ * the original requested leaf is observed!
	1082	+ */
	1083	+ return kvm_find_cpuid_entry(vcpu, basic->eax, index);
930	1084	}
931	1085
932	1086	bool kvm_cpuid(struct kvm_vcpu vcpu, u32 eax, u32 *ebx,
933		- u32 ecx, u32 edx, bool check_limit)
	1087	+ u32 ecx, u32 edx, bool exact_only)
934	1088	{
935		- u32 function = eax, index = ecx;
936		- struct kvm_cpuid_entry2 *best;
937		- bool entry_found = true;
	1089	+ u32 orig_function = eax, function = eax, index = *ecx;
	1090	+ struct kvm_cpuid_entry2 *entry;
	1091	+ bool exact, used_max_basic = false;
938	1092
939		- best = kvm_find_cpuid_entry(vcpu, function, index);
	1093	+ entry = kvm_find_cpuid_entry(vcpu, function, index);
	1094	+ exact = !!entry;
940	1095
941		- if (!best) {
942		- entry_found = false;
943		- if (!check_limit)
944		- goto out;
945		-
946		- best = check_cpuid_limit(vcpu, function, index);
	1096	+ if (!entry && !exact_only) {
	1097	+ entry = get_out_of_range_cpuid_entry(vcpu, &function, index);
	1098	+ used_max_basic = !!entry;
947	1099	}
948	1100
949		-out:
950		- if (best) {
951		- *eax = best->eax;
952		- *ebx = best->ebx;
953		- *ecx = best->ecx;
954		- *edx = best->edx;
955		- } else
	1101	+ if (entry) {
	1102	+ *eax = entry->eax;
	1103	+ *ebx = entry->ebx;
	1104	+ *ecx = entry->ecx;
	1105	+ *edx = entry->edx;
	1106	+ if (function == 7 && index == 0) {
	1107	+ u64 data;
	1108	+ if (!__kvm_get_msr(vcpu, MSR_IA32_TSX_CTRL, &data, true) &&
	1109	+ (data & TSX_CTRL_CPUID_CLEAR))
	1110	+ *ebx &= ~(F(RTM) \| F(HLE));
	1111	+ }
	1112	+ } else {
956	1113	eax = ebx = ecx = edx = 0;
957		- trace_kvm_cpuid(function, eax, ebx, ecx, edx, entry_found);
958		- return entry_found;
	1114	+ /*
	1115	+ * When leaf 0BH or 1FH is defined, CL is pass-through
	1116	+ * and EDX is always the x2APIC ID, even for undefined
	1117	+ * subleaves. Index 1 will exist iff the leaf is
	1118	+ * implemented, so we pass through CL iff leaf 1
	1119	+ * exists. EDX can be copied from any existing index.
	1120	+ */
	1121	+ if (function == 0xb \|\| function == 0x1f) {
	1122	+ entry = kvm_find_cpuid_entry(vcpu, function, 1);
	1123	+ if (entry) {
	1124	+ *ecx = index & 0xff;
	1125	+ *edx = entry->edx;
	1126	+ }
	1127	+ }
	1128	+ }
	1129	+ trace_kvm_cpuid(orig_function, index, eax, ebx, ecx, edx, exact,
	1130	+ used_max_basic);
	1131	+ return exact;
959	1132	}
960	1133	EXPORT_SYMBOL_GPL(kvm_cpuid);
961	1134
..	..	@@ -966,13 +1139,13 @@
966	1139	if (cpuid_fault_enabled(vcpu) && !kvm_require_cpl(vcpu, 0))
967	1140	return 1;
968	1141
969		- eax = kvm_register_read(vcpu, VCPU_REGS_RAX);
970		- ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
971		- kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, true);
972		- kvm_register_write(vcpu, VCPU_REGS_RAX, eax);
973		- kvm_register_write(vcpu, VCPU_REGS_RBX, ebx);
974		- kvm_register_write(vcpu, VCPU_REGS_RCX, ecx);
975		- kvm_register_write(vcpu, VCPU_REGS_RDX, edx);
	1142	+ eax = kvm_rax_read(vcpu);
	1143	+ ecx = kvm_rcx_read(vcpu);
	1144	+ kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, false);
	1145	+ kvm_rax_write(vcpu, eax);
	1146	+ kvm_rbx_write(vcpu, ebx);
	1147	+ kvm_rcx_write(vcpu, ecx);
	1148	+ kvm_rdx_write(vcpu, edx);
976	1149	return kvm_skip_emulated_instruction(vcpu);
977	1150	}
978	1151	EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);