移去rt

hc

2023-12-08 01573e231f18eb2d99162747186f59511f56b64d

 kernel/arch/x86/kvm/cpuid.c
..
..
@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
1
2
 /*
2
3
  * Kernel-based Virtual Machine driver for Linux
3
4
  * cpuid support routines
..
..
@@ -6,10 +7,6 @@
6
7
  *
7
8
  * Copyright 2011 Red Hat, Inc. and/or its affiliates.
8
9
  * Copyright IBM Corporation, 2008
9
- *
10
- * This work is licensed under the terms of the GNU GPL, version 2.  See
11
- * the COPYING file in the top-level directory.
12
- *
13
10
  */
14
11
 
15
12
 #include <linux/kvm_host.h>
..
..
@@ -26,6 +23,13 @@
26
23
 #include "mmu.h"
27
24
 #include "trace.h"
28
25
 #include "pmu.h"
26
+
27
+/*
28
+ * Unlike "struct cpuinfo_x86.x86_capability", kvm_cpu_caps doesn't need to be
29
+ * aligned to sizeof(unsigned long) because it's not accessed via bitops.
30
+ */
31
+u32 kvm_cpu_caps[NCAPINTS] __read_mostly;
32
+EXPORT_SYMBOL_GPL(kvm_cpu_caps);
29
33
 
30
34
 static u32 xstate_required_size(u64 xstate_bv, bool compacted)
31
35
 {
..
..
@@ -48,86 +52,34 @@
48
52
 	return ret;
49
53
 }
50
54
 
51
-bool kvm_mpx_supported(void)
55
+#define F feature_bit
56
+
57
+static inline struct kvm_cpuid_entry2 *cpuid_entry2_find(
58
+	struct kvm_cpuid_entry2 *entries, int nent, u32 function, u32 index)
52
59
 {
53
-	return ((host_xcr0 & (XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR))
54
-		 && kvm_x86_ops->mpx_supported());
55
-}
56
-EXPORT_SYMBOL_GPL(kvm_mpx_supported);
60
+	struct kvm_cpuid_entry2 *e;
61
+	int i;
57
62
 
58
-u64 kvm_supported_xcr0(void)
59
-{
60
-	u64 xcr0 = KVM_SUPPORTED_XCR0 & host_xcr0;
63
+	for (i = 0; i < nent; i++) {
64
+		e = &entries[i];
61
65
 
62
-	if (!kvm_mpx_supported())
63
-		xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR);
66
+		if (e->function == function && (e->index == index ||
67
+		    !(e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX)))
68
+			return e;
69
+	}
64
70
 
65
-	return xcr0;
71
+	return NULL;
66
72
 }
67
73
 
68
-#define F(x) bit(X86_FEATURE_##x)
69
-
70
-/* For scattered features from cpufeatures.h; we currently expose none */
71
-#define KF(x) bit(KVM_CPUID_BIT_##x)
72
-
73
-int kvm_update_cpuid(struct kvm_vcpu *vcpu)
74
+static int kvm_check_cpuid(struct kvm_cpuid_entry2 *entries, int nent)
74
75
 {
75
76
 	struct kvm_cpuid_entry2 *best;
76
-	struct kvm_lapic *apic = vcpu->arch.apic;
77
-
78
-	best = kvm_find_cpuid_entry(vcpu, 1, 0);
79
-	if (!best)
80
-		return 0;
81
-
82
-	/* Update OSXSAVE bit */
83
-	if (boot_cpu_has(X86_FEATURE_XSAVE) && best->function == 0x1) {
84
-		best->ecx &= ~F(OSXSAVE);
85
-		if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE))
86
-			best->ecx |= F(OSXSAVE);
87
-	}
88
-
89
-	best->edx &= ~F(APIC);
90
-	if (vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)
91
-		best->edx |= F(APIC);
92
-
93
-	if (apic) {
94
-		if (best->ecx & F(TSC_DEADLINE_TIMER))
95
-			apic->lapic_timer.timer_mode_mask = 3 << 17;
96
-		else
97
-			apic->lapic_timer.timer_mode_mask = 1 << 17;
98
-	}
99
-
100
-	best = kvm_find_cpuid_entry(vcpu, 7, 0);
101
-	if (best) {
102
-		/* Update OSPKE bit */
103
-		if (boot_cpu_has(X86_FEATURE_PKU) && best->function == 0x7) {
104
-			best->ecx &= ~F(OSPKE);
105
-			if (kvm_read_cr4_bits(vcpu, X86_CR4_PKE))
106
-				best->ecx |= F(OSPKE);
107
-		}
108
-	}
109
-
110
-	best = kvm_find_cpuid_entry(vcpu, 0xD, 0);
111
-	if (!best) {
112
-		vcpu->arch.guest_supported_xcr0 = 0;
113
-		vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
114
-	} else {
115
-		vcpu->arch.guest_supported_xcr0 =
116
-			(best->eax | ((u64)best->edx << 32)) &
117
-			kvm_supported_xcr0();
118
-		vcpu->arch.guest_xstate_size = best->ebx =
119
-			xstate_required_size(vcpu->arch.xcr0, false);
120
-	}
121
-
122
-	best = kvm_find_cpuid_entry(vcpu, 0xD, 1);
123
-	if (best && (best->eax & (F(XSAVES) | F(XSAVEC))))
124
-		best->ebx = xstate_required_size(vcpu->arch.xcr0, true);
125
77
 
126
78
 	/*
127
79
 	 * The existing code assumes virtual address is 48-bit or 57-bit in the
128
80
 	 * canonical address checks; exit if it is ever changed.
129
81
 	 */
130
-	best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
82
+	best = cpuid_entry2_find(entries, nent, 0x80000008, 0);
131
83
 	if (best) {
132
84
 		int vaddr_bits = (best->eax & 0xff00) >> 8;
133
85
 
..
..
@@ -135,25 +87,106 @@
135
87
 			return -EINVAL;
136
88
 	}
137
89
 
90
+	return 0;
91
+}
92
+
93
+void kvm_update_pv_runtime(struct kvm_vcpu *vcpu)
94
+{
95
+	struct kvm_cpuid_entry2 *best;
96
+
97
+	best = kvm_find_cpuid_entry(vcpu, KVM_CPUID_FEATURES, 0);
98
+
99
+	/*
100
+	 * save the feature bitmap to avoid cpuid lookup for every PV
101
+	 * operation
102
+	 */
103
+	if (best)
104
+		vcpu->arch.pv_cpuid.features = best->eax;
105
+}
106
+
107
+void kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu)
108
+{
109
+	struct kvm_cpuid_entry2 *best;
110
+
111
+	best = kvm_find_cpuid_entry(vcpu, 1, 0);
112
+	if (best) {
113
+		/* Update OSXSAVE bit */
114
+		if (boot_cpu_has(X86_FEATURE_XSAVE))
115
+			cpuid_entry_change(best, X86_FEATURE_OSXSAVE,
116
+				   kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE));
117
+
118
+		cpuid_entry_change(best, X86_FEATURE_APIC,
119
+			   vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE);
120
+	}
121
+
122
+	best = kvm_find_cpuid_entry(vcpu, 7, 0);
123
+	if (best && boot_cpu_has(X86_FEATURE_PKU) && best->function == 0x7)
124
+		cpuid_entry_change(best, X86_FEATURE_OSPKE,
125
+				   kvm_read_cr4_bits(vcpu, X86_CR4_PKE));
126
+
127
+	best = kvm_find_cpuid_entry(vcpu, 0xD, 0);
128
+	if (best)
129
+		best->ebx = xstate_required_size(vcpu->arch.xcr0, false);
130
+
131
+	best = kvm_find_cpuid_entry(vcpu, 0xD, 1);
132
+	if (best && (cpuid_entry_has(best, X86_FEATURE_XSAVES) ||
133
+		     cpuid_entry_has(best, X86_FEATURE_XSAVEC)))
134
+		best->ebx = xstate_required_size(vcpu->arch.xcr0, true);
135
+
138
136
 	best = kvm_find_cpuid_entry(vcpu, KVM_CPUID_FEATURES, 0);
139
137
 	if (kvm_hlt_in_guest(vcpu->kvm) && best &&
140
138
 		(best->eax & (1 << KVM_FEATURE_PV_UNHALT)))
141
139
 		best->eax &= ~(1 << KVM_FEATURE_PV_UNHALT);
142
140
 
143
-	/* Update physical-address width */
141
+	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT)) {
142
+		best = kvm_find_cpuid_entry(vcpu, 0x1, 0);
143
+		if (best)
144
+			cpuid_entry_change(best, X86_FEATURE_MWAIT,
145
+					   vcpu->arch.ia32_misc_enable_msr &
146
+					   MSR_IA32_MISC_ENABLE_MWAIT);
147
+	}
148
+}
149
+
150
+static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
151
+{
152
+	struct kvm_lapic *apic = vcpu->arch.apic;
153
+	struct kvm_cpuid_entry2 *best;
154
+
155
+	best = kvm_find_cpuid_entry(vcpu, 1, 0);
156
+	if (best && apic) {
157
+		if (cpuid_entry_has(best, X86_FEATURE_TSC_DEADLINE_TIMER))
158
+			apic->lapic_timer.timer_mode_mask = 3 << 17;
159
+		else
160
+			apic->lapic_timer.timer_mode_mask = 1 << 17;
161
+
162
+		kvm_apic_set_version(vcpu);
163
+	}
164
+
165
+	best = kvm_find_cpuid_entry(vcpu, 0xD, 0);
166
+	if (!best)
167
+		vcpu->arch.guest_supported_xcr0 = 0;
168
+	else
169
+		vcpu->arch.guest_supported_xcr0 =
170
+			(best->eax | ((u64)best->edx << 32)) & supported_xcr0;
171
+
172
+	kvm_update_pv_runtime(vcpu);
173
+
144
174
 	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
145
175
 	kvm_mmu_reset_context(vcpu);
146
176
 
147
177
 	kvm_pmu_refresh(vcpu);
148
-	return 0;
178
+	vcpu->arch.cr4_guest_rsvd_bits =
179
+	    __cr4_reserved_bits(guest_cpuid_has, vcpu);
180
+
181
+	vcpu->arch.cr3_lm_rsvd_bits = rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
182
+
183
+	/* Invoke the vendor callback only after the above state is updated. */
184
+	kvm_x86_ops.vcpu_after_set_cpuid(vcpu);
149
185
 }
150
186
 
151
187
 static int is_efer_nx(void)
152
188
 {
153
-	unsigned long long efer = 0;
154
-
155
-	rdmsrl_safe(MSR_EFER, &efer);
156
-	return efer & EFER_NX;
189
+	return host_efer & EFER_NX;
157
190
 }
158
191
 
159
192
 static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
..
..
@@ -169,8 +202,8 @@
169
202
 			break;
170
203
 		}
171
204
 	}
172
-	if (entry && (entry->edx & F(NX)) && !is_efer_nx()) {
173
-		entry->edx &= ~F(NX);
205
+	if (entry && cpuid_entry_has(entry, X86_FEATURE_NX) && !is_efer_nx()) {
206
+		cpuid_entry_clear(entry, X86_FEATURE_NX);
174
207
 		printk(KERN_INFO "kvm: guest NX capability removed\n");
175
208
 	}
176
209
 }
..
..
@@ -188,7 +221,6 @@
188
221
 not_found:
189
222
 	return 36;
190
223
 }
191
-EXPORT_SYMBOL_GPL(cpuid_query_maxphyaddr);
192
224
 
193
225
 /* when an old userspace process fills a new kernel module */
194
226
 int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
..
..
@@ -196,43 +228,53 @@
196
228
 			     struct kvm_cpuid_entry __user *entries)
197
229
 {
198
230
 	int r, i;
199
-	struct kvm_cpuid_entry *cpuid_entries = NULL;
231
+	struct kvm_cpuid_entry *e = NULL;
232
+	struct kvm_cpuid_entry2 *e2 = NULL;
200
233
 
201
-	r = -E2BIG;
202
234
 	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
203
-		goto out;
204
-	r = -ENOMEM;
235
+		return -E2BIG;
236
+
205
237
 	if (cpuid->nent) {
206
-		cpuid_entries =
207
-			vmalloc(array_size(sizeof(struct kvm_cpuid_entry),
208
-					   cpuid->nent));
209
-		if (!cpuid_entries)
210
-			goto out;
211
-		r = -EFAULT;
212
-		if (copy_from_user(cpuid_entries, entries,
213
-				   cpuid->nent * sizeof(struct kvm_cpuid_entry)))
214
-			goto out;
238
+		e = vmemdup_user(entries, array_size(sizeof(*e), cpuid->nent));
239
+		if (IS_ERR(e))
240
+			return PTR_ERR(e);
241
+
242
+		e2 = kvmalloc_array(cpuid->nent, sizeof(*e2), GFP_KERNEL_ACCOUNT);
243
+		if (!e2) {
244
+			r = -ENOMEM;
245
+			goto out_free_cpuid;
246
+		}
215
247
 	}
216
248
 	for (i = 0; i < cpuid->nent; i++) {
217
-		vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function;
218
-		vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax;
219
-		vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx;
220
-		vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx;
221
-		vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx;
222
-		vcpu->arch.cpuid_entries[i].index = 0;
223
-		vcpu->arch.cpuid_entries[i].flags = 0;
224
-		vcpu->arch.cpuid_entries[i].padding[0] = 0;
225
-		vcpu->arch.cpuid_entries[i].padding[1] = 0;
226
-		vcpu->arch.cpuid_entries[i].padding[2] = 0;
249
+		e2[i].function = e[i].function;
250
+		e2[i].eax = e[i].eax;
251
+		e2[i].ebx = e[i].ebx;
252
+		e2[i].ecx = e[i].ecx;
253
+		e2[i].edx = e[i].edx;
254
+		e2[i].index = 0;
255
+		e2[i].flags = 0;
256
+		e2[i].padding[0] = 0;
257
+		e2[i].padding[1] = 0;
258
+		e2[i].padding[2] = 0;
227
259
 	}
228
-	vcpu->arch.cpuid_nent = cpuid->nent;
229
-	cpuid_fix_nx_cap(vcpu);
230
-	kvm_apic_set_version(vcpu);
231
-	kvm_x86_ops->cpuid_update(vcpu);
232
-	r = kvm_update_cpuid(vcpu);
233
260
 
234
-out:
235
-	vfree(cpuid_entries);
261
+	r = kvm_check_cpuid(e2, cpuid->nent);
262
+	if (r) {
263
+		kvfree(e2);
264
+		goto out_free_cpuid;
265
+	}
266
+
267
+	kvfree(vcpu->arch.cpuid_entries);
268
+	vcpu->arch.cpuid_entries = e2;
269
+	vcpu->arch.cpuid_nent = cpuid->nent;
270
+
271
+	cpuid_fix_nx_cap(vcpu);
272
+	kvm_update_cpuid_runtime(vcpu);
273
+	kvm_vcpu_after_set_cpuid(vcpu);
274
+
275
+out_free_cpuid:
276
+	kvfree(e);
277
+
236
278
 	return r;
237
279
 }
238
280
 
..
..
@@ -240,21 +282,32 @@
240
282
 			      struct kvm_cpuid2 *cpuid,
241
283
 			      struct kvm_cpuid_entry2 __user *entries)
242
284
 {
285
+	struct kvm_cpuid_entry2 *e2 = NULL;
243
286
 	int r;
244
287
 
245
-	r = -E2BIG;
246
288
 	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
247
-		goto out;
248
-	r = -EFAULT;
249
-	if (copy_from_user(&vcpu->arch.cpuid_entries, entries,
250
-			   cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
251
-		goto out;
289
+		return -E2BIG;
290
+
291
+	if (cpuid->nent) {
292
+		e2 = vmemdup_user(entries, array_size(sizeof(*e2), cpuid->nent));
293
+		if (IS_ERR(e2))
294
+			return PTR_ERR(e2);
295
+	}
296
+
297
+	r = kvm_check_cpuid(e2, cpuid->nent);
298
+	if (r) {
299
+		kvfree(e2);
300
+		return r;
301
+	}
302
+
303
+	kvfree(vcpu->arch.cpuid_entries);
304
+	vcpu->arch.cpuid_entries = e2;
252
305
 	vcpu->arch.cpuid_nent = cpuid->nent;
253
-	kvm_apic_set_version(vcpu);
254
-	kvm_x86_ops->cpuid_update(vcpu);
255
-	r = kvm_update_cpuid(vcpu);
256
-out:
257
-	return r;
306
+
307
+	kvm_update_cpuid_runtime(vcpu);
308
+	kvm_vcpu_after_set_cpuid(vcpu);
309
+
310
+	return 0;
258
311
 }
259
312
 
260
313
 int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
..
..
@@ -267,7 +320,7 @@
267
320
 	if (cpuid->nent < vcpu->arch.cpuid_nent)
268
321
 		goto out;
269
322
 	r = -EFAULT;
270
-	if (copy_to_user(entries, &vcpu->arch.cpuid_entries,
323
+	if (copy_to_user(entries, vcpu->arch.cpuid_entries,
271
324
 			 vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
272
325
 		goto out;
273
326
 	return 0;
..
..
@@ -277,70 +330,55 @@
277
330
 	return r;
278
331
 }
279
332
 
280
-static void cpuid_mask(u32 *word, int wordnum)
333
+static __always_inline void kvm_cpu_cap_mask(enum cpuid_leafs leaf, u32 mask)
281
334
 {
282
-	*word &= boot_cpu_data.x86_capability[wordnum];
335
+	const struct cpuid_reg cpuid = x86_feature_cpuid(leaf * 32);
336
+	struct kvm_cpuid_entry2 entry;
337
+
338
+	reverse_cpuid_check(leaf);
339
+	kvm_cpu_caps[leaf] &= mask;
340
+
341
+	cpuid_count(cpuid.function, cpuid.index,
342
+		    &entry.eax, &entry.ebx, &entry.ecx, &entry.edx);
343
+
344
+	kvm_cpu_caps[leaf] &= *__cpuid_entry_get_reg(&entry, cpuid.reg);
283
345
 }
284
346
 
285
-static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
286
-			   u32 index)
347
+void kvm_set_cpu_caps(void)
287
348
 {
288
-	entry->function = function;
289
-	entry->index = index;
290
-	cpuid_count(entry->function, entry->index,
291
-		    &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
292
-	entry->flags = 0;
293
-}
294
-
295
-static int __do_cpuid_ent_emulated(struct kvm_cpuid_entry2 *entry,
296
-				   u32 func, u32 index, int *nent, int maxnent)
297
-{
298
-	switch (func) {
299
-	case 0:
300
-		entry->eax = 7;
301
-		++*nent;
302
-		break;
303
-	case 1:
304
-		entry->ecx = F(MOVBE);
305
-		++*nent;
306
-		break;
307
-	case 7:
308
-		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
309
-		if (index == 0)
310
-			entry->ecx = F(RDPID);
311
-		++*nent;
312
-	default:
313
-		break;
314
-	}
315
-
316
-	entry->function = func;
317
-	entry->index = index;
318
-
319
-	return 0;
320
-}
321
-
322
-static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
323
-				 u32 index, int *nent, int maxnent)
324
-{
325
-	int r;
326
-	unsigned f_nx = is_efer_nx() ? F(NX) : 0;
349
+	unsigned int f_nx = is_efer_nx() ? F(NX) : 0;
327
350
 #ifdef CONFIG_X86_64
328
-	unsigned f_gbpages = (kvm_x86_ops->get_lpage_level() == PT_PDPE_LEVEL)
329
-				? F(GBPAGES) : 0;
330
-	unsigned f_lm = F(LM);
351
+	unsigned int f_gbpages = F(GBPAGES);
352
+	unsigned int f_lm = F(LM);
331
353
 #else
332
-	unsigned f_gbpages = 0;
333
-	unsigned f_lm = 0;
354
+	unsigned int f_gbpages = 0;
355
+	unsigned int f_lm = 0;
334
356
 #endif
335
-	unsigned f_rdtscp = kvm_x86_ops->rdtscp_supported() ? F(RDTSCP) : 0;
336
-	unsigned f_invpcid = kvm_x86_ops->invpcid_supported() ? F(INVPCID) : 0;
337
-	unsigned f_mpx = kvm_mpx_supported() ? F(MPX) : 0;
338
-	unsigned f_xsaves = kvm_x86_ops->xsaves_supported() ? F(XSAVES) : 0;
339
-	unsigned f_umip = kvm_x86_ops->umip_emulated() ? F(UMIP) : 0;
340
-	unsigned f_la57 = 0;
341
357
 
342
-	/* cpuid 1.edx */
343
-	const u32 kvm_cpuid_1_edx_x86_features =
358
+	BUILD_BUG_ON(sizeof(kvm_cpu_caps) >
359
+		     sizeof(boot_cpu_data.x86_capability));
360
+
361
+	memcpy(&kvm_cpu_caps, &boot_cpu_data.x86_capability,
362
+	       sizeof(kvm_cpu_caps));
363
+
364
+	kvm_cpu_cap_mask(CPUID_1_ECX,
365
+		/*
366
+		 * NOTE: MONITOR (and MWAIT) are emulated as NOP, but *not*
367
+		 * advertised to guests via CPUID!
368
+		 */
369
+		F(XMM3) | F(PCLMULQDQ) | 0 /* DTES64, MONITOR */ |
370
+		0 /* DS-CPL, VMX, SMX, EST */ |
371
+		0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ |
372
+		F(FMA) | F(CX16) | 0 /* xTPR Update */ | F(PDCM) |
373
+		F(PCID) | 0 /* Reserved, DCA */ | F(XMM4_1) |
374
+		F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) |
375
+		0 /* Reserved*/ | F(AES) | F(XSAVE) | 0 /* OSXSAVE */ | F(AVX) |
376
+		F(F16C) | F(RDRAND)
377
+	);
378
+	/* KVM emulates x2apic in software irrespective of host support. */
379
+	kvm_cpu_cap_set(X86_FEATURE_X2APIC);
380
+
381
+	kvm_cpu_cap_mask(CPUID_1_EDX,
344
382
 		F(FPU) | F(VME) | F(DE) | F(PSE) |
345
383
 		F(TSC) | F(MSR) | F(PAE) | F(MCE) |
346
384
 		F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) |
..
..
@@ -348,189 +386,290 @@
348
386
 		F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLUSH) |
349
387
 		0 /* Reserved, DS, ACPI */ | F(MMX) |
350
388
 		F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) |
351
-		0 /* HTT, TM, Reserved, PBE */;
352
-	/* cpuid 0x80000001.edx */
353
-	const u32 kvm_cpuid_8000_0001_edx_x86_features =
389
+		0 /* HTT, TM, Reserved, PBE */
390
+	);
391
+
392
+	kvm_cpu_cap_mask(CPUID_7_0_EBX,
393
+		F(FSGSBASE) | F(BMI1) | F(HLE) | F(AVX2) | F(SMEP) |
394
+		F(BMI2) | F(ERMS) | 0 /*INVPCID*/ | F(RTM) | 0 /*MPX*/ | F(RDSEED) |
395
+		F(ADX) | F(SMAP) | F(AVX512IFMA) | F(AVX512F) | F(AVX512PF) |
396
+		F(AVX512ER) | F(AVX512CD) | F(CLFLUSHOPT) | F(CLWB) | F(AVX512DQ) |
397
+		F(SHA_NI) | F(AVX512BW) | F(AVX512VL) | 0 /*INTEL_PT*/
398
+	);
399
+
400
+	kvm_cpu_cap_mask(CPUID_7_ECX,
401
+		F(AVX512VBMI) | F(LA57) | F(PKU) | 0 /*OSPKE*/ | F(RDPID) |
402
+		F(AVX512_VPOPCNTDQ) | F(UMIP) | F(AVX512_VBMI2) | F(GFNI) |
403
+		F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG) |
404
+		F(CLDEMOTE) | F(MOVDIRI) | F(MOVDIR64B) | 0 /*WAITPKG*/
405
+	);
406
+	/* Set LA57 based on hardware capability. */
407
+	if (cpuid_ecx(7) & F(LA57))
408
+		kvm_cpu_cap_set(X86_FEATURE_LA57);
409
+
410
+	/*
411
+	 * PKU not yet implemented for shadow paging and requires OSPKE
412
+	 * to be set on the host. Clear it if that is not the case
413
+	 */
414
+	if (!tdp_enabled || !boot_cpu_has(X86_FEATURE_OSPKE))
415
+		kvm_cpu_cap_clear(X86_FEATURE_PKU);
416
+
417
+	kvm_cpu_cap_mask(CPUID_7_EDX,
418
+		F(AVX512_4VNNIW) | F(AVX512_4FMAPS) | F(SPEC_CTRL) |
419
+		F(SPEC_CTRL_SSBD) | F(ARCH_CAPABILITIES) | F(INTEL_STIBP) |
420
+		F(MD_CLEAR) | F(AVX512_VP2INTERSECT) | F(FSRM) |
421
+		F(SERIALIZE) | F(TSXLDTRK)
422
+	);
423
+
424
+	/* TSC_ADJUST and ARCH_CAPABILITIES are emulated in software. */
425
+	kvm_cpu_cap_set(X86_FEATURE_TSC_ADJUST);
426
+	kvm_cpu_cap_set(X86_FEATURE_ARCH_CAPABILITIES);
427
+
428
+	if (boot_cpu_has(X86_FEATURE_IBPB) && boot_cpu_has(X86_FEATURE_IBRS))
429
+		kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL);
430
+	if (boot_cpu_has(X86_FEATURE_STIBP))
431
+		kvm_cpu_cap_set(X86_FEATURE_INTEL_STIBP);
432
+	if (boot_cpu_has(X86_FEATURE_AMD_SSBD))
433
+		kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL_SSBD);
434
+
435
+	kvm_cpu_cap_mask(CPUID_7_1_EAX,
436
+		F(AVX512_BF16)
437
+	);
438
+
439
+	kvm_cpu_cap_mask(CPUID_D_1_EAX,
440
+		F(XSAVEOPT) | F(XSAVEC) | F(XGETBV1) | F(XSAVES)
441
+	);
442
+
443
+	kvm_cpu_cap_mask(CPUID_8000_0001_ECX,
444
+		F(LAHF_LM) | F(CMP_LEGACY) | 0 /*SVM*/ | 0 /* ExtApicSpace */ |
445
+		F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
446
+		F(3DNOWPREFETCH) | F(OSVW) | 0 /* IBS */ | F(XOP) |
447
+		0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM) |
448
+		F(TOPOEXT) | F(PERFCTR_CORE)
449
+	);
450
+
451
+	kvm_cpu_cap_mask(CPUID_8000_0001_EDX,
354
452
 		F(FPU) | F(VME) | F(DE) | F(PSE) |
355
453
 		F(TSC) | F(MSR) | F(PAE) | F(MCE) |
356
454
 		F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) |
357
455
 		F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
358
456
 		F(PAT) | F(PSE36) | 0 /* Reserved */ |
359
457
 		f_nx | 0 /* Reserved */ | F(MMXEXT) | F(MMX) |
360
-		F(FXSR) | F(FXSR_OPT) | f_gbpages | f_rdtscp |
361
-		0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW);
362
-	/* cpuid 1.ecx */
363
-	const u32 kvm_cpuid_1_ecx_x86_features =
364
-		/* NOTE: MONITOR (and MWAIT) are emulated as NOP,
365
-		 * but *not* advertised to guests via CPUID ! */
366
-		F(XMM3) | F(PCLMULQDQ) | 0 /* DTES64, MONITOR */ |
367
-		0 /* DS-CPL, VMX, SMX, EST */ |
368
-		0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ |
369
-		F(FMA) | F(CX16) | 0 /* xTPR Update, PDCM */ |
370
-		F(PCID) | 0 /* Reserved, DCA */ | F(XMM4_1) |
371
-		F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) |
372
-		0 /* Reserved*/ | F(AES) | F(XSAVE) | 0 /* OSXSAVE */ | F(AVX) |
373
-		F(F16C) | F(RDRAND);
374
-	/* cpuid 0x80000001.ecx */
375
-	const u32 kvm_cpuid_8000_0001_ecx_x86_features =
376
-		F(LAHF_LM) | F(CMP_LEGACY) | 0 /*SVM*/ | 0 /* ExtApicSpace */ |
377
-		F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
378
-		F(3DNOWPREFETCH) | F(OSVW) | 0 /* IBS */ | F(XOP) |
379
-		0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM) |
380
-		F(TOPOEXT) | F(PERFCTR_CORE);
458
+		F(FXSR) | F(FXSR_OPT) | f_gbpages | F(RDTSCP) |
459
+		0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW)
460
+	);
381
461
 
382
-	/* cpuid 0x80000008.ebx */
383
-	const u32 kvm_cpuid_8000_0008_ebx_x86_features =
384
-		F(AMD_IBPB) | F(AMD_IBRS) | F(AMD_SSBD) | F(VIRT_SSBD) |
385
-		F(AMD_SSB_NO) | F(AMD_STIBP);
462
+	if (!tdp_enabled && IS_ENABLED(CONFIG_X86_64))
463
+		kvm_cpu_cap_set(X86_FEATURE_GBPAGES);
386
464
 
387
-	/* cpuid 0xC0000001.edx */
388
-	const u32 kvm_cpuid_C000_0001_edx_x86_features =
465
+	kvm_cpu_cap_mask(CPUID_8000_0008_EBX,
466
+		F(CLZERO) | F(XSAVEERPTR) |
467
+		F(WBNOINVD) | F(AMD_IBPB) | F(AMD_IBRS) | F(AMD_SSBD) | F(VIRT_SSBD) |
468
+		F(AMD_SSB_NO) | F(AMD_STIBP) | F(AMD_STIBP_ALWAYS_ON)
469
+	);
470
+
471
+	/*
472
+	 * AMD has separate bits for each SPEC_CTRL bit.
473
+	 * arch/x86/kernel/cpu/bugs.c is kind enough to
474
+	 * record that in cpufeatures so use them.
475
+	 */
476
+	if (boot_cpu_has(X86_FEATURE_IBPB))
477
+		kvm_cpu_cap_set(X86_FEATURE_AMD_IBPB);
478
+	if (boot_cpu_has(X86_FEATURE_IBRS))
479
+		kvm_cpu_cap_set(X86_FEATURE_AMD_IBRS);
480
+	if (boot_cpu_has(X86_FEATURE_STIBP))
481
+		kvm_cpu_cap_set(X86_FEATURE_AMD_STIBP);
482
+	if (boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD))
483
+		kvm_cpu_cap_set(X86_FEATURE_AMD_SSBD);
484
+	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
485
+		kvm_cpu_cap_set(X86_FEATURE_AMD_SSB_NO);
486
+	/*
487
+	 * The preference is to use SPEC CTRL MSR instead of the
488
+	 * VIRT_SPEC MSR.
489
+	 */
490
+	if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
491
+	    !boot_cpu_has(X86_FEATURE_AMD_SSBD))
492
+		kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
493
+
494
+	/*
495
+	 * Hide all SVM features by default, SVM will set the cap bits for
496
+	 * features it emulates and/or exposes for L1.
497
+	 */
498
+	kvm_cpu_cap_mask(CPUID_8000_000A_EDX, 0);
499
+
500
+	kvm_cpu_cap_mask(CPUID_C000_0001_EDX,
389
501
 		F(XSTORE) | F(XSTORE_EN) | F(XCRYPT) | F(XCRYPT_EN) |
390
502
 		F(ACE2) | F(ACE2_EN) | F(PHE) | F(PHE_EN) |
391
-		F(PMM) | F(PMM_EN);
503
+		F(PMM) | F(PMM_EN)
504
+	);
505
+}
506
+EXPORT_SYMBOL_GPL(kvm_set_cpu_caps);
392
507
 
393
-	/* cpuid 7.0.ebx */
394
-	const u32 kvm_cpuid_7_0_ebx_x86_features =
395
-		F(FSGSBASE) | F(BMI1) | F(HLE) | F(AVX2) | F(SMEP) |
396
-		F(BMI2) | F(ERMS) | f_invpcid | F(RTM) | f_mpx | F(RDSEED) |
397
-		F(ADX) | F(SMAP) | F(AVX512IFMA) | F(AVX512F) | F(AVX512PF) |
398
-		F(AVX512ER) | F(AVX512CD) | F(CLFLUSHOPT) | F(CLWB) | F(AVX512DQ) |
399
-		F(SHA_NI) | F(AVX512BW) | F(AVX512VL);
508
+struct kvm_cpuid_array {
509
+	struct kvm_cpuid_entry2 *entries;
510
+	int maxnent;
511
+	int nent;
512
+};
400
513
 
401
-	/* cpuid 0xD.1.eax */
402
-	const u32 kvm_cpuid_D_1_eax_x86_features =
403
-		F(XSAVEOPT) | F(XSAVEC) | F(XGETBV1) | f_xsaves;
514
+static struct kvm_cpuid_entry2 *do_host_cpuid(struct kvm_cpuid_array *array,
515
+					      u32 function, u32 index)
516
+{
517
+	struct kvm_cpuid_entry2 *entry;
404
518
 
405
-	/* cpuid 7.0.ecx*/
406
-	const u32 kvm_cpuid_7_0_ecx_x86_features =
407
-		F(AVX512VBMI) | F(LA57) | F(PKU) | 0 /*OSPKE*/ |
408
-		F(AVX512_VPOPCNTDQ) | F(UMIP) | F(AVX512_VBMI2) | F(GFNI) |
409
-		F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG) |
410
-		F(CLDEMOTE);
519
+	if (array->nent >= array->maxnent)
520
+		return NULL;
411
521
 
412
-	/* cpuid 7.0.edx*/
413
-	const u32 kvm_cpuid_7_0_edx_x86_features =
414
-		F(AVX512_4VNNIW) | F(AVX512_4FMAPS) | F(SPEC_CTRL) |
415
-		F(SPEC_CTRL_SSBD) | F(ARCH_CAPABILITIES) | F(INTEL_STIBP) |
416
-		F(MD_CLEAR);
522
+	entry = &array->entries[array->nent++];
523
+
524
+	entry->function = function;
525
+	entry->index = index;
526
+	entry->flags = 0;
527
+
528
+	cpuid_count(entry->function, entry->index,
529
+		    &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
530
+
531
+	switch (function) {
532
+	case 4:
533
+	case 7:
534
+	case 0xb:
535
+	case 0xd:
536
+	case 0xf:
537
+	case 0x10:
538
+	case 0x12:
539
+	case 0x14:
540
+	case 0x17:
541
+	case 0x18:
542
+	case 0x1f:
543
+	case 0x8000001d:
544
+		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
545
+		break;
546
+	}
547
+
548
+	return entry;
549
+}
550
+
551
+static int __do_cpuid_func_emulated(struct kvm_cpuid_array *array, u32 func)
552
+{
553
+	struct kvm_cpuid_entry2 *entry;
554
+
555
+	if (array->nent >= array->maxnent)
556
+		return -E2BIG;
557
+
558
+	entry = &array->entries[array->nent];
559
+	entry->function = func;
560
+	entry->index = 0;
561
+	entry->flags = 0;
562
+
563
+	switch (func) {
564
+	case 0:
565
+		entry->eax = 7;
566
+		++array->nent;
567
+		break;
568
+	case 1:
569
+		entry->ecx = F(MOVBE);
570
+		++array->nent;
571
+		break;
572
+	case 7:
573
+		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
574
+		entry->eax = 0;
575
+		if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP))
576
+			entry->ecx = F(RDPID);
577
+		++array->nent;
578
+	default:
579
+		break;
580
+	}
581
+
582
+	return 0;
583
+}
584
+
585
+static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
586
+{
587
+	struct kvm_cpuid_entry2 *entry;
588
+	int r, i, max_idx;
417
589
 
418
590
 	/* all calls to cpuid_count() should be made on the same cpu */
419
591
 	get_cpu();
420
592
 
421
593
 	r = -E2BIG;
422
594
 
423
-	if (WARN_ON(*nent >= maxnent))
595
+	entry = do_host_cpuid(array, function, 0);
596
+	if (!entry)
424
597
 		goto out;
425
-
426
-	do_cpuid_1_ent(entry, function, index);
427
-	++*nent;
428
598
 
429
599
 	switch (function) {
430
600
 	case 0:
431
-		entry->eax = min(entry->eax, (u32)0xd);
601
+		/* Limited to the highest leaf implemented in KVM. */
602
+		entry->eax = min(entry->eax, 0x1fU);
432
603
 		break;
433
604
 	case 1:
434
-		entry->edx &= kvm_cpuid_1_edx_x86_features;
435
-		cpuid_mask(&entry->edx, CPUID_1_EDX);
436
-		entry->ecx &= kvm_cpuid_1_ecx_x86_features;
437
-		cpuid_mask(&entry->ecx, CPUID_1_ECX);
438
-		/* we support x2apic emulation even if host does not support
439
-		 * it since we emulate x2apic in software */
440
-		entry->ecx |= F(X2APIC);
605
+		cpuid_entry_override(entry, CPUID_1_EDX);
606
+		cpuid_entry_override(entry, CPUID_1_ECX);
441
607
 		break;
442
-	/* function 2 entries are STATEFUL. That is, repeated cpuid commands
443
-	 * may return different values. This forces us to get_cpu() before
444
-	 * issuing the first command, and also to emulate this annoying behavior
445
-	 * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */
446
-	case 2: {
447
-		int t, times = entry->eax & 0xff;
448
-
449
-		entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
450
-		entry->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
451
-		for (t = 1; t < times; ++t) {
452
-			if (*nent >= maxnent)
608
+	case 2:
609
+		/*
610
+		 * On ancient CPUs, function 2 entries are STATEFUL.  That is,
611
+		 * CPUID(function=2, index=0) may return different results each
612
+		 * time, with the least-significant byte in EAX enumerating the
613
+		 * number of times software should do CPUID(2, 0).
614
+		 *
615
+		 * Modern CPUs, i.e. every CPU KVM has *ever* run on are less
616
+		 * idiotic.  Intel's SDM states that EAX & 0xff "will always
617
+		 * return 01H. Software should ignore this value and not
618
+		 * interpret it as an informational descriptor", while AMD's
619
+		 * APM states that CPUID(2) is reserved.
620
+		 *
621
+		 * WARN if a frankenstein CPU that supports virtualization and
622
+		 * a stateful CPUID.0x2 is encountered.
623
+		 */
624
+		WARN_ON_ONCE((entry->eax & 0xff) > 1);
625
+		break;
626
+	/* functions 4 and 0x8000001d have additional index. */
627
+	case 4:
628
+	case 0x8000001d:
629
+		/*
630
+		 * Read entries until the cache type in the previous entry is
631
+		 * zero, i.e. indicates an invalid entry.
632
+		 */
633
+		for (i = 1; entry->eax & 0x1f; ++i) {
634
+			entry = do_host_cpuid(array, function, i);
635
+			if (!entry)
453
636
 				goto out;
454
-
455
-			do_cpuid_1_ent(&entry[t], function, 0);
456
-			entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
457
-			++*nent;
458
637
 		}
459
638
 		break;
460
-	}
461
-	/* function 4 has additional index. */
462
-	case 4: {
463
-		int i, cache_type;
464
-
465
-		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
466
-		/* read more entries until cache_type is zero */
467
-		for (i = 1; ; ++i) {
468
-			if (*nent >= maxnent)
469
-				goto out;
470
-
471
-			cache_type = entry[i - 1].eax & 0x1f;
472
-			if (!cache_type)
473
-				break;
474
-			do_cpuid_1_ent(&entry[i], function, i);
475
-			entry[i].flags |=
476
-			       KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
477
-			++*nent;
478
-		}
479
-		break;
480
-	}
481
639
 	case 6: /* Thermal management */
482
640
 		entry->eax = 0x4; /* allow ARAT */
483
641
 		entry->ebx = 0;
484
642
 		entry->ecx = 0;
485
643
 		entry->edx = 0;
486
644
 		break;
487
-	case 7: {
488
-		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
489
-		/* Mask ebx against host capability word 9 */
490
-		if (index == 0) {
491
-			entry->ebx &= kvm_cpuid_7_0_ebx_x86_features;
492
-			cpuid_mask(&entry->ebx, CPUID_7_0_EBX);
493
-			// TSC_ADJUST is emulated
494
-			entry->ebx |= F(TSC_ADJUST);
495
-			entry->ecx &= kvm_cpuid_7_0_ecx_x86_features;
496
-			f_la57 = entry->ecx & F(LA57);
497
-			cpuid_mask(&entry->ecx, CPUID_7_ECX);
498
-			/* Set LA57 based on hardware capability. */
499
-			entry->ecx |= f_la57;
500
-			entry->ecx |= f_umip;
501
-			/* PKU is not yet implemented for shadow paging. */
502
-			if (!tdp_enabled || !boot_cpu_has(X86_FEATURE_OSPKE))
503
-				entry->ecx &= ~F(PKU);
645
+	/* function 7 has additional index. */
646
+	case 7:
647
+		entry->eax = min(entry->eax, 1u);
648
+		cpuid_entry_override(entry, CPUID_7_0_EBX);
649
+		cpuid_entry_override(entry, CPUID_7_ECX);
650
+		cpuid_entry_override(entry, CPUID_7_EDX);
504
651
 
505
-			entry->edx &= kvm_cpuid_7_0_edx_x86_features;
506
-			cpuid_mask(&entry->edx, CPUID_7_EDX);
507
-			if (boot_cpu_has(X86_FEATURE_IBPB) &&
508
-			    boot_cpu_has(X86_FEATURE_IBRS))
509
-				entry->edx |= F(SPEC_CTRL);
510
-			if (boot_cpu_has(X86_FEATURE_STIBP))
511
-				entry->edx |= F(INTEL_STIBP);
512
-			if (boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
513
-			    boot_cpu_has(X86_FEATURE_AMD_SSBD))
514
-				entry->edx |= F(SPEC_CTRL_SSBD);
515
-			/*
516
-			 * We emulate ARCH_CAPABILITIES in software even
517
-			 * if the host doesn't support it.
518
-			 */
519
-			entry->edx |= F(ARCH_CAPABILITIES);
520
-		} else {
652
+		/* KVM only supports 0x7.0 and 0x7.1, capped above via min(). */
653
+		if (entry->eax == 1) {
654
+			entry = do_host_cpuid(array, function, 1);
655
+			if (!entry)
656
+				goto out;
657
+
658
+			cpuid_entry_override(entry, CPUID_7_1_EAX);
521
659
 			entry->ebx = 0;
522
660
 			entry->ecx = 0;
523
661
 			entry->edx = 0;
524
662
 		}
525
-		entry->eax = 0;
526
-		break;
527
-	}
528
-	case 9:
529
663
 		break;
530
664
 	case 0xa: { /* Architectural Performance Monitoring */
531
665
 		struct x86_pmu_capability cap;
532
666
 		union cpuid10_eax eax;
533
667
 		union cpuid10_edx edx;
668
+
669
+		if (!static_cpu_has(X86_FEATURE_ARCH_PERFMON)) {
670
+			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
671
+			break;
672
+		}
534
673
 
535
674
 		perf_get_x86_pmu_capability(&cap);
536
675
 
..
..
@@ -546,9 +685,12 @@
546
685
 		eax.split.bit_width = cap.bit_width_gp;
547
686
 		eax.split.mask_length = cap.events_mask_len;
548
687
 
549
-		edx.split.num_counters_fixed = cap.num_counters_fixed;
688
+		edx.split.num_counters_fixed = min(cap.num_counters_fixed, MAX_FIXED_COUNTERS);
550
689
 		edx.split.bit_width_fixed = cap.bit_width_fixed;
551
-		edx.split.reserved = 0;
690
+		if (cap.version)
691
+			edx.split.anythread_deprecated = 1;
692
+		edx.split.reserved1 = 0;
693
+		edx.split.reserved2 = 0;
552
694
 
553
695
 		entry->eax = eax.full;
554
696
 		entry->ebx = cap.events_mask;
..
..
@@ -556,67 +698,86 @@
556
698
 		entry->edx = edx.full;
557
699
 		break;
558
700
 	}
559
-	/* function 0xb has additional index. */
560
-	case 0xb: {
561
-		int i, level_type;
562
-
563
-		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
564
-		/* read more entries until level_type is zero */
565
-		for (i = 1; ; ++i) {
566
-			if (*nent >= maxnent)
701
+	/*
702
+	 * Per Intel's SDM, the 0x1f is a superset of 0xb,
703
+	 * thus they can be handled by common code.
704
+	 */
705
+	case 0x1f:
706
+	case 0xb:
707
+		/*
708
+		 * Populate entries until the level type (ECX[15:8]) of the
709
+		 * previous entry is zero.  Note, CPUID EAX.{0x1f,0xb}.0 is
710
+		 * the starting entry, filled by the primary do_host_cpuid().
711
+		 */
712
+		for (i = 1; entry->ecx & 0xff00; ++i) {
713
+			entry = do_host_cpuid(array, function, i);
714
+			if (!entry)
567
715
 				goto out;
568
-
569
-			level_type = entry[i - 1].ecx & 0xff00;
570
-			if (!level_type)
571
-				break;
572
-			do_cpuid_1_ent(&entry[i], function, i);
573
-			entry[i].flags |=
574
-			       KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
575
-			++*nent;
576
716
 		}
577
717
 		break;
578
-	}
579
-	case 0xd: {
580
-		int idx, i;
581
-		u64 supported = kvm_supported_xcr0();
582
-
583
-		entry->eax &= supported;
584
-		entry->ebx = xstate_required_size(supported, false);
718
+	case 0xd:
719
+		entry->eax &= supported_xcr0;
720
+		entry->ebx = xstate_required_size(supported_xcr0, false);
585
721
 		entry->ecx = entry->ebx;
586
-		entry->edx &= supported >> 32;
587
-		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
588
-		if (!supported)
722
+		entry->edx &= supported_xcr0 >> 32;
723
+		if (!supported_xcr0)
589
724
 			break;
590
725
 
591
-		for (idx = 1, i = 1; idx < 64; ++idx) {
592
-			u64 mask = ((u64)1 << idx);
593
-			if (*nent >= maxnent)
726
+		entry = do_host_cpuid(array, function, 1);
727
+		if (!entry)
728
+			goto out;
729
+
730
+		cpuid_entry_override(entry, CPUID_D_1_EAX);
731
+		if (entry->eax & (F(XSAVES)|F(XSAVEC)))
732
+			entry->ebx = xstate_required_size(supported_xcr0 | supported_xss,
733
+							  true);
734
+		else {
735
+			WARN_ON_ONCE(supported_xss != 0);
736
+			entry->ebx = 0;
737
+		}
738
+		entry->ecx &= supported_xss;
739
+		entry->edx &= supported_xss >> 32;
740
+
741
+		for (i = 2; i < 64; ++i) {
742
+			bool s_state;
743
+			if (supported_xcr0 & BIT_ULL(i))
744
+				s_state = false;
745
+			else if (supported_xss & BIT_ULL(i))
746
+				s_state = true;
747
+			else
748
+				continue;
749
+
750
+			entry = do_host_cpuid(array, function, i);
751
+			if (!entry)
594
752
 				goto out;
595
753
 
596
-			do_cpuid_1_ent(&entry[i], function, idx);
597
-			if (idx == 1) {
598
-				entry[i].eax &= kvm_cpuid_D_1_eax_x86_features;
599
-				cpuid_mask(&entry[i].eax, CPUID_D_1_EAX);
600
-				entry[i].ebx = 0;
601
-				if (entry[i].eax & (F(XSAVES)|F(XSAVEC)))
602
-					entry[i].ebx =
603
-						xstate_required_size(supported,
604
-								     true);
605
-			} else {
606
-				if (entry[i].eax == 0 || !(supported & mask))
607
-					continue;
608
-				if (WARN_ON_ONCE(entry[i].ecx & 1))
609
-					continue;
754
+			/*
755
+			 * The supported check above should have filtered out
756
+			 * invalid sub-leafs.  Only valid sub-leafs should
757
+			 * reach this point, and they should have a non-zero
758
+			 * save state size.  Furthermore, check whether the
759
+			 * processor agrees with supported_xcr0/supported_xss
760
+			 * on whether this is an XCR0- or IA32_XSS-managed area.
761
+			 */
762
+			if (WARN_ON_ONCE(!entry->eax || (entry->ecx & 0x1) != s_state)) {
763
+				--array->nent;
764
+				continue;
610
765
 			}
611
-			entry[i].ecx = 0;
612
-			entry[i].edx = 0;
613
-			entry[i].flags |=
614
-			       KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
615
-			++*nent;
616
-			++i;
766
+			entry->edx = 0;
617
767
 		}
618
768
 		break;
619
-	}
769
+	/* Intel PT */
770
+	case 0x14:
771
+		if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT)) {
772
+			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
773
+			break;
774
+		}
775
+
776
+		for (i = 1, max_idx = entry->eax; i <= max_idx; ++i) {
777
+			if (!do_host_cpuid(array, function, i))
778
+				goto out;
779
+		}
780
+		break;
620
781
 	case KVM_CPUID_SIGNATURE: {
621
782
 		static const char signature[12] = "KVMKVMKVM\0\0";
622
783
 		const u32 *sigptr = (const u32 *)signature;
..
..
@@ -636,7 +797,10 @@
636
797
 			     (1 << KVM_FEATURE_PV_UNHALT) |
637
798
 			     (1 << KVM_FEATURE_PV_TLB_FLUSH) |
638
799
 			     (1 << KVM_FEATURE_ASYNC_PF_VMEXIT) |
639
-			     (1 << KVM_FEATURE_PV_SEND_IPI);
800
+			     (1 << KVM_FEATURE_PV_SEND_IPI) |
801
+			     (1 << KVM_FEATURE_POLL_CONTROL) |
802
+			     (1 << KVM_FEATURE_PV_SCHED_YIELD) |
803
+			     (1 << KVM_FEATURE_ASYNC_PF_INT);
640
804
 
641
805
 		if (sched_info_on())
642
806
 			entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
..
..
@@ -649,10 +813,13 @@
649
813
 		entry->eax = min(entry->eax, 0x8000001f);
650
814
 		break;
651
815
 	case 0x80000001:
652
-		entry->edx &= kvm_cpuid_8000_0001_edx_x86_features;
653
-		cpuid_mask(&entry->edx, CPUID_8000_0001_EDX);
654
-		entry->ecx &= kvm_cpuid_8000_0001_ecx_x86_features;
655
-		cpuid_mask(&entry->ecx, CPUID_8000_0001_ECX);
816
+		entry->ebx &= ~GENMASK(27, 16);
817
+		cpuid_entry_override(entry, CPUID_8000_0001_EDX);
818
+		cpuid_entry_override(entry, CPUID_8000_0001_ECX);
819
+		break;
820
+	case 0x80000006:
821
+		/* Drop reserved bits, pass host L2 cache and TLB info. */
822
+		entry->edx &= ~GENMASK(17, 16);
656
823
 		break;
657
824
 	case 0x80000007: /* Advanced power management */
658
825
 		/* invariant TSC is CPUID.80000007H:EDX[8] */
..
..
@@ -675,34 +842,35 @@
675
842
 			g_phys_as = phys_as;
676
843
 
677
844
 		entry->eax = g_phys_as | (virt_as << 8);
845
+		entry->ecx &= ~(GENMASK(31, 16) | GENMASK(11, 8));
678
846
 		entry->edx = 0;
679
-		/*
680
-		 * IBRS, IBPB and VIRT_SSBD aren't necessarily present in
681
-		 * hardware cpuid
682
-		 */
683
-		if (boot_cpu_has(X86_FEATURE_AMD_IBPB))
684
-			entry->ebx |= F(AMD_IBPB);
685
-		if (boot_cpu_has(X86_FEATURE_AMD_IBRS))
686
-			entry->ebx |= F(AMD_IBRS);
687
-		if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
688
-			entry->ebx |= F(VIRT_SSBD);
689
-		entry->ebx &= kvm_cpuid_8000_0008_ebx_x86_features;
690
-		cpuid_mask(&entry->ebx, CPUID_8000_0008_EBX);
691
-		/*
692
-		 * The preference is to use SPEC CTRL MSR instead of the
693
-		 * VIRT_SPEC MSR.
694
-		 */
695
-		if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
696
-		    !boot_cpu_has(X86_FEATURE_AMD_SSBD))
697
-			entry->ebx |= F(VIRT_SSBD);
847
+		cpuid_entry_override(entry, CPUID_8000_0008_EBX);
698
848
 		break;
699
849
 	}
850
+	case 0x8000000A:
851
+		if (!kvm_cpu_cap_has(X86_FEATURE_SVM)) {
852
+			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
853
+			break;
854
+		}
855
+		entry->eax = 1; /* SVM revision 1 */
856
+		entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
857
+				   ASID emulation to nested SVM */
858
+		entry->ecx = 0; /* Reserved */
859
+		cpuid_entry_override(entry, CPUID_8000_000A_EDX);
860
+		break;
700
861
 	case 0x80000019:
701
862
 		entry->ecx = entry->edx = 0;
702
863
 		break;
703
864
 	case 0x8000001a:
865
+		entry->eax &= GENMASK(2, 0);
866
+		entry->ebx = entry->ecx = entry->edx = 0;
704
867
 		break;
705
-	case 0x8000001d:
868
+	case 0x8000001e:
869
+		break;
870
+	/* Support memory encryption cpuid if host supports it */
871
+	case 0x8000001F:
872
+		if (!boot_cpu_has(X86_FEATURE_SEV))
873
+			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
706
874
 		break;
707
875
 	/*Add support for Centaur's CPUID instruction*/
708
876
 	case 0xC0000000:
..
..
@@ -710,8 +878,7 @@
710
878
 		entry->eax = min(entry->eax, 0xC0000004);
711
879
 		break;
712
880
 	case 0xC0000001:
713
-		entry->edx &= kvm_cpuid_C000_0001_edx_x86_features;
714
-		cpuid_mask(&entry->edx, CPUID_C000_0001_EDX);
881
+		cpuid_entry_override(entry, CPUID_C000_0001_EDX);
715
882
 		break;
716
883
 	case 3: /* Processor serial number */
717
884
 	case 5: /* MONITOR/MWAIT */
..
..
@@ -723,8 +890,6 @@
723
890
 		break;
724
891
 	}
725
892
 
726
-	kvm_x86_ops->set_supported_cpuid(function, entry);
727
-
728
893
 	r = 0;
729
894
 
730
895
 out:
..
..
@@ -733,30 +898,39 @@
733
898
 	return r;
734
899
 }
735
900
 
736
-static int do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 func,
737
-			u32 idx, int *nent, int maxnent, unsigned int type)
901
+static int do_cpuid_func(struct kvm_cpuid_array *array, u32 func,
902
+			 unsigned int type)
738
903
 {
739
-	if (*nent >= maxnent)
740
-		return -E2BIG;
741
-
742
904
 	if (type == KVM_GET_EMULATED_CPUID)
743
-		return __do_cpuid_ent_emulated(entry, func, idx, nent, maxnent);
905
+		return __do_cpuid_func_emulated(array, func);
744
906
 
745
-	return __do_cpuid_ent(entry, func, idx, nent, maxnent);
907
+	return __do_cpuid_func(array, func);
746
908
 }
747
909
 
748
-#undef F
910
+#define CENTAUR_CPUID_SIGNATURE 0xC0000000
749
911
 
750
-struct kvm_cpuid_param {
751
-	u32 func;
752
-	u32 idx;
753
-	bool has_leaf_count;
754
-	bool (*qualifier)(const struct kvm_cpuid_param *param);
755
-};
756
-
757
-static bool is_centaur_cpu(const struct kvm_cpuid_param *param)
912
+static int get_cpuid_func(struct kvm_cpuid_array *array, u32 func,
913
+			  unsigned int type)
758
914
 {
759
-	return boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR;
915
+	u32 limit;
916
+	int r;
917
+
918
+	if (func == CENTAUR_CPUID_SIGNATURE &&
919
+	    boot_cpu_data.x86_vendor != X86_VENDOR_CENTAUR)
920
+		return 0;
921
+
922
+	r = do_cpuid_func(array, func, type);
923
+	if (r)
924
+		return r;
925
+
926
+	limit = array->entries[array->nent - 1].eax;
927
+	for (func = func + 1; func <= limit; ++func) {
928
+		r = do_cpuid_func(array, func, type);
929
+		if (r)
930
+			break;
931
+	}
932
+
933
+	return r;
760
934
 }
761
935
 
762
936
 static bool sanity_check_entries(struct kvm_cpuid_entry2 __user *entries,
..
..
@@ -790,172 +964,168 @@
790
964
 			    struct kvm_cpuid_entry2 __user *entries,
791
965
 			    unsigned int type)
792
966
 {
793
-	struct kvm_cpuid_entry2 *cpuid_entries;
794
-	int limit, nent = 0, r = -E2BIG, i;
795
-	u32 func;
796
-	static const struct kvm_cpuid_param param[] = {
797
-		{ .func = 0, .has_leaf_count = true },
798
-		{ .func = 0x80000000, .has_leaf_count = true },
799
-		{ .func = 0xC0000000, .qualifier = is_centaur_cpu, .has_leaf_count = true },
800
-		{ .func = KVM_CPUID_SIGNATURE },
801
-		{ .func = KVM_CPUID_FEATURES },
967
+	static const u32 funcs[] = {
968
+		0, 0x80000000, CENTAUR_CPUID_SIGNATURE, KVM_CPUID_SIGNATURE,
802
969
 	};
803
970
 
971
+	struct kvm_cpuid_array array = {
972
+		.nent = 0,
973
+	};
974
+	int r, i;
975
+
804
976
 	if (cpuid->nent < 1)
805
-		goto out;
977
+		return -E2BIG;
806
978
 	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
807
979
 		cpuid->nent = KVM_MAX_CPUID_ENTRIES;
808
980
 
809
981
 	if (sanity_check_entries(entries, cpuid->nent, type))
810
982
 		return -EINVAL;
811
983
 
812
-	r = -ENOMEM;
813
-	cpuid_entries = vzalloc(array_size(sizeof(struct kvm_cpuid_entry2),
984
+	array.entries = vzalloc(array_size(sizeof(struct kvm_cpuid_entry2),
814
985
 					   cpuid->nent));
815
-	if (!cpuid_entries)
816
-		goto out;
986
+	if (!array.entries)
987
+		return -ENOMEM;
817
988
 
818
-	r = 0;
819
-	for (i = 0; i < ARRAY_SIZE(param); i++) {
820
-		const struct kvm_cpuid_param *ent = &param[i];
989
+	array.maxnent = cpuid->nent;
821
990
 
822
-		if (ent->qualifier && !ent->qualifier(ent))
823
-			continue;
824
-
825
-		r = do_cpuid_ent(&cpuid_entries[nent], ent->func, ent->idx,
826
-				&nent, cpuid->nent, type);
827
-
828
-		if (r)
829
-			goto out_free;
830
-
831
-		if (!ent->has_leaf_count)
832
-			continue;
833
-
834
-		limit = cpuid_entries[nent - 1].eax;
835
-		for (func = ent->func + 1; func <= limit && nent < cpuid->nent && r == 0; ++func)
836
-			r = do_cpuid_ent(&cpuid_entries[nent], func, ent->idx,
837
-				     &nent, cpuid->nent, type);
838
-
991
+	for (i = 0; i < ARRAY_SIZE(funcs); i++) {
992
+		r = get_cpuid_func(&array, funcs[i], type);
839
993
 		if (r)
840
994
 			goto out_free;
841
995
 	}
996
+	cpuid->nent = array.nent;
842
997
 
843
-	r = -EFAULT;
844
-	if (copy_to_user(entries, cpuid_entries,
845
-			 nent * sizeof(struct kvm_cpuid_entry2)))
846
-		goto out_free;
847
-	cpuid->nent = nent;
848
-	r = 0;
998
+	if (copy_to_user(entries, array.entries,
999
+			 array.nent * sizeof(struct kvm_cpuid_entry2)))
1000
+		r = -EFAULT;
849
1001
 
850
1002
 out_free:
851
-	vfree(cpuid_entries);
852
-out:
1003
+	vfree(array.entries);
853
1004
 	return r;
854
-}
855
-
856
-static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i)
857
-{
858
-	struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i];
859
-	struct kvm_cpuid_entry2 *ej;
860
-	int j = i;
861
-	int nent = vcpu->arch.cpuid_nent;
862
-
863
-	e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT;
864
-	/* when no next entry is found, the current entry[i] is reselected */
865
-	do {
866
-		j = (j + 1) % nent;
867
-		ej = &vcpu->arch.cpuid_entries[j];
868
-	} while (ej->function != e->function);
869
-
870
-	ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
871
-
872
-	return j;
873
-}
874
-
875
-/* find an entry with matching function, matching index (if needed), and that
876
- * should be read next (if it's stateful) */
877
-static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e,
878
-	u32 function, u32 index)
879
-{
880
-	if (e->function != function)
881
-		return 0;
882
-	if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index)
883
-		return 0;
884
-	if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) &&
885
-	    !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT))
886
-		return 0;
887
-	return 1;
888
1005
 }
889
1006
 
890
1007
 struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
891
1008
 					      u32 function, u32 index)
892
1009
 {
893
-	int i;
894
-	struct kvm_cpuid_entry2 *best = NULL;
895
-
896
-	for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
897
-		struct kvm_cpuid_entry2 *e;
898
-
899
-		e = &vcpu->arch.cpuid_entries[i];
900
-		if (is_matching_cpuid_entry(e, function, index)) {
901
-			if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC)
902
-				move_to_next_stateful_cpuid_entry(vcpu, i);
903
-			best = e;
904
-			break;
905
-		}
906
-	}
907
-	return best;
1010
+	return cpuid_entry2_find(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent,
1011
+				 function, index);
908
1012
 }
909
1013
 EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry);
910
1014
 
911
1015
 /*
912
- * If no match is found, check whether we exceed the vCPU's limit
913
- * and return the content of the highest valid _standard_ leaf instead.
914
- * This is to satisfy the CPUID specification.
1016
+ * Intel CPUID semantics treats any query for an out-of-range leaf as if the
1017
+ * highest basic leaf (i.e. CPUID.0H:EAX) were requested.  AMD CPUID semantics
1018
+ * returns all zeroes for any undefined leaf, whether or not the leaf is in
1019
+ * range.  Centaur/VIA follows Intel semantics.
1020
+ *
1021
+ * A leaf is considered out-of-range if its function is higher than the maximum
1022
+ * supported leaf of its associated class or if its associated class does not
1023
+ * exist.
1024
+ *
1025
+ * There are three primary classes to be considered, with their respective
1026
+ * ranges described as "<base> - <top>[,<base2> - <top2>] inclusive.  A primary
1027
+ * class exists if a guest CPUID entry for its <base> leaf exists.  For a given
1028
+ * class, CPUID.<base>.EAX contains the max supported leaf for the class.
1029
+ *
1030
+ *  - Basic:      0x00000000 - 0x3fffffff, 0x50000000 - 0x7fffffff
1031
+ *  - Hypervisor: 0x40000000 - 0x4fffffff
1032
+ *  - Extended:   0x80000000 - 0xbfffffff
1033
+ *  - Centaur:    0xc0000000 - 0xcfffffff
1034
+ *
1035
+ * The Hypervisor class is further subdivided into sub-classes that each act as
1036
+ * their own indepdent class associated with a 0x100 byte range.  E.g. if Qemu
1037
+ * is advertising support for both HyperV and KVM, the resulting Hypervisor
1038
+ * CPUID sub-classes are:
1039
+ *
1040
+ *  - HyperV:     0x40000000 - 0x400000ff
1041
+ *  - KVM:        0x40000100 - 0x400001ff
915
1042
  */
916
-static struct kvm_cpuid_entry2* check_cpuid_limit(struct kvm_vcpu *vcpu,
917
-                                                  u32 function, u32 index)
1043
+static struct kvm_cpuid_entry2 *
1044
+get_out_of_range_cpuid_entry(struct kvm_vcpu *vcpu, u32 *fn_ptr, u32 index)
918
1045
 {
919
-	struct kvm_cpuid_entry2 *maxlevel;
1046
+	struct kvm_cpuid_entry2 *basic, *class;
1047
+	u32 function = *fn_ptr;
920
1048
 
921
-	maxlevel = kvm_find_cpuid_entry(vcpu, function & 0x80000000, 0);
922
-	if (!maxlevel || maxlevel->eax >= function)
1049
+	basic = kvm_find_cpuid_entry(vcpu, 0, 0);
1050
+	if (!basic)
923
1051
 		return NULL;
924
-	if (function & 0x80000000) {
925
-		maxlevel = kvm_find_cpuid_entry(vcpu, 0, 0);
926
-		if (!maxlevel)
927
-			return NULL;
928
-	}
929
-	return kvm_find_cpuid_entry(vcpu, maxlevel->eax, index);
1052
+
1053
+	if (is_guest_vendor_amd(basic->ebx, basic->ecx, basic->edx) ||
1054
+	    is_guest_vendor_hygon(basic->ebx, basic->ecx, basic->edx))
1055
+		return NULL;
1056
+
1057
+	if (function >= 0x40000000 && function <= 0x4fffffff)
1058
+		class = kvm_find_cpuid_entry(vcpu, function & 0xffffff00, 0);
1059
+	else if (function >= 0xc0000000)
1060
+		class = kvm_find_cpuid_entry(vcpu, 0xc0000000, 0);
1061
+	else
1062
+		class = kvm_find_cpuid_entry(vcpu, function & 0x80000000, 0);
1063
+
1064
+	if (class && function <= class->eax)
1065
+		return NULL;
1066
+
1067
+	/*
1068
+	 * Leaf specific adjustments are also applied when redirecting to the
1069
+	 * max basic entry, e.g. if the max basic leaf is 0xb but there is no
1070
+	 * entry for CPUID.0xb.index (see below), then the output value for EDX
1071
+	 * needs to be pulled from CPUID.0xb.1.
1072
+	 */
1073
+	*fn_ptr = basic->eax;
1074
+
1075
+	/*
1076
+	 * The class does not exist or the requested function is out of range;
1077
+	 * the effective CPUID entry is the max basic leaf.  Note, the index of
1078
+	 * the original requested leaf is observed!
1079
+	 */
1080
+	return kvm_find_cpuid_entry(vcpu, basic->eax, index);
930
1081
 }
931
1082
 
932
1083
 bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx,
933
-	       u32 *ecx, u32 *edx, bool check_limit)
1084
+	       u32 *ecx, u32 *edx, bool exact_only)
934
1085
 {
935
-	u32 function = *eax, index = *ecx;
936
-	struct kvm_cpuid_entry2 *best;
937
-	bool entry_found = true;
1086
+	u32 orig_function = *eax, function = *eax, index = *ecx;
1087
+	struct kvm_cpuid_entry2 *entry;
1088
+	bool exact, used_max_basic = false;
938
1089
 
939
-	best = kvm_find_cpuid_entry(vcpu, function, index);
1090
+	entry = kvm_find_cpuid_entry(vcpu, function, index);
1091
+	exact = !!entry;
940
1092
 
941
-	if (!best) {
942
-		entry_found = false;
943
-		if (!check_limit)
944
-			goto out;
945
-
946
-		best = check_cpuid_limit(vcpu, function, index);
1093
+	if (!entry && !exact_only) {
1094
+		entry = get_out_of_range_cpuid_entry(vcpu, &function, index);
1095
+		used_max_basic = !!entry;
947
1096
 	}
948
1097
 
949
-out:
950
-	if (best) {
951
-		*eax = best->eax;
952
-		*ebx = best->ebx;
953
-		*ecx = best->ecx;
954
-		*edx = best->edx;
955
-	} else
1098
+	if (entry) {
1099
+		*eax = entry->eax;
1100
+		*ebx = entry->ebx;
1101
+		*ecx = entry->ecx;
1102
+		*edx = entry->edx;
1103
+		if (function == 7 && index == 0) {
1104
+			u64 data;
1105
+		        if (!__kvm_get_msr(vcpu, MSR_IA32_TSX_CTRL, &data, true) &&
1106
+			    (data & TSX_CTRL_CPUID_CLEAR))
1107
+				*ebx &= ~(F(RTM) | F(HLE));
1108
+		}
1109
+	} else {
956
1110
 		*eax = *ebx = *ecx = *edx = 0;
957
-	trace_kvm_cpuid(function, *eax, *ebx, *ecx, *edx, entry_found);
958
-	return entry_found;
1111
+		/*
1112
+		 * When leaf 0BH or 1FH is defined, CL is pass-through
1113
+		 * and EDX is always the x2APIC ID, even for undefined
1114
+		 * subleaves. Index 1 will exist iff the leaf is
1115
+		 * implemented, so we pass through CL iff leaf 1
1116
+		 * exists. EDX can be copied from any existing index.
1117
+		 */
1118
+		if (function == 0xb || function == 0x1f) {
1119
+			entry = kvm_find_cpuid_entry(vcpu, function, 1);
1120
+			if (entry) {
1121
+				*ecx = index & 0xff;
1122
+				*edx = entry->edx;
1123
+			}
1124
+		}
1125
+	}
1126
+	trace_kvm_cpuid(orig_function, index, *eax, *ebx, *ecx, *edx, exact,
1127
+			used_max_basic);
1128
+	return exact;
959
1129
 }
960
1130
 EXPORT_SYMBOL_GPL(kvm_cpuid);
961
1131
 
..
..
@@ -966,13 +1136,13 @@
966
1136
 	if (cpuid_fault_enabled(vcpu) && !kvm_require_cpl(vcpu, 0))
967
1137
 		return 1;
968
1138
 
969
-	eax = kvm_register_read(vcpu, VCPU_REGS_RAX);
970
-	ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
971
-	kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, true);
972
-	kvm_register_write(vcpu, VCPU_REGS_RAX, eax);
973
-	kvm_register_write(vcpu, VCPU_REGS_RBX, ebx);
974
-	kvm_register_write(vcpu, VCPU_REGS_RCX, ecx);
975
-	kvm_register_write(vcpu, VCPU_REGS_RDX, edx);
1139
+	eax = kvm_rax_read(vcpu);
1140
+	ecx = kvm_rcx_read(vcpu);
1141
+	kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, false);
1142
+	kvm_rax_write(vcpu, eax);
1143
+	kvm_rbx_write(vcpu, ebx);
1144
+	kvm_rcx_write(vcpu, ecx);
1145
+	kvm_rdx_write(vcpu, edx);
976
1146
 	return kvm_skip_emulated_instruction(vcpu);
977
1147
 }
978
1148
 EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);