~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/******************************************************************************
2	3	* emulate.c
3	4	*
..	..	@@ -14,16 +15,14 @@
14	15	* Avi Kivity <avi@qumranet.com>
15	16	* Yaniv Kamay <yaniv@qumranet.com>
16	17	*
17		- * This work is licensed under the terms of the GNU GPL, version 2. See
18		- * the COPYING file in the top-level directory.
19		- *
20	18	* From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
21	19	*/
22	20
23	21	#include <linux/kvm_host.h>
24	22	#include "kvm_cache_regs.h"
25		-#include <asm/kvm_emulate.h>
	23	+#include "kvm_emulate.h"
26	24	#include <linux/stringify.h>
	25	+#include <asm/fpu/api.h>
27	26	#include <asm/debugreg.h>
28	27	#include <asm/nospec-branch.h>
29	28
..	..	@@ -189,28 +188,6 @@
189	188	#define X8(x...) X4(x), X4(x)
190	189	#define X16(x...) X8(x), X8(x)
191	190
192		-#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
193		-#define FASTOP_SIZE 8
194		-
195		-/*
196		- * fastop functions have a special calling convention:
197		- *
198		- * dst: rax (in/out)
199		- * src: rdx (in/out)
200		- * src2: rcx (in)
201		- * flags: rflags (in/out)
202		- * ex: rsi (in:fastop pointer, out:zero if exception)
203		- *
204		- * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
205		- * different operand sizes can be reached by calculation, rather than a jump
206		- * table (which would be bigger than the code).
207		- *
208		- * fastop functions are declared as taking a never-defined fastop parameter,
209		- * so they can't be called from C directly.
210		- */
211		-
212		-struct fastop;
213		-
214	191	struct opcode {
215	192	u64 flags : 56;
216	193	u64 intercept : 8;
..	..	@@ -312,31 +289,65 @@
312	289	#define ON64(x)
313	290	#endif
314	291
315		-static int fastop(struct x86_emulate_ctxt ctxt, void (fop)(struct fastop *));
	292	+/*
	293	+ * fastop functions have a special calling convention:
	294	+ *
	295	+ * dst: rax (in/out)
	296	+ * src: rdx (in/out)
	297	+ * src2: rcx (in)
	298	+ * flags: rflags (in/out)
	299	+ * ex: rsi (in:fastop pointer, out:zero if exception)
	300	+ *
	301	+ * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
	302	+ * different operand sizes can be reached by calculation, rather than a jump
	303	+ * table (which would be bigger than the code).
	304	+ *
	305	+ * The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for ENDBR
	306	+ * and 1 for the straight line speculation INT3, leaves 7 bytes for the
	307	+ * body of the function. Currently none is larger than 4.
	308	+ */
	309	+static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
316	310
317		-#define FOP_FUNC(name) \
	311	+#define FASTOP_SIZE 16
	312	+
	313	+#define __FOP_FUNC(name) \
318	314	".align " __stringify(FASTOP_SIZE) " \n\t" \
319	315	".type " name ", @function \n\t" \
320	316	name ":\n\t"
321	317
322		-#define FOP_RET "ret \n\t"
	318	+#define FOP_FUNC(name) \
	319	+ __FOP_FUNC(#name)
323	320
324		-#define FOP_START(op) \
	321	+#define __FOP_RET(name) \
	322	+ ASM_RET \
	323	+ ".size " name ", .-" name "\n\t"
	324	+
	325	+#define FOP_RET(name) \
	326	+ __FOP_RET(#name)
	327	+
	328	+#define __FOP_START(op, align) \
325	329	extern void em_##op(struct fastop *fake); \
326	330	asm(".pushsection .text, \"ax\" \n\t" \
327	331	".global em_" #op " \n\t" \
328		- FOP_FUNC("em_" #op)
	332	+ ".align " __stringify(align) " \n\t" \
	333	+ "em_" #op ":\n\t"
	334	+
	335	+#define FOP_START(op) __FOP_START(op, FASTOP_SIZE)
329	336
330	337	#define FOP_END \
331	338	".popsection")
332	339
	340	+#define __FOPNOP(name) \
	341	+ __FOP_FUNC(name) \
	342	+ __FOP_RET(name)
	343	+
333	344	#define FOPNOP() \
334		- FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
335		- FOP_RET
	345	+ __FOPNOP(__stringify(__UNIQUE_ID(nop)))
336	346
337	347	#define FOP1E(op, dst) \
338		- FOP_FUNC(#op "_" #dst) \
339		- "10: " #op " %" #dst " \n\t" FOP_RET
	348	+ __FOP_FUNC(#op "_" #dst) \
	349	+ "10: " #op " %" #dst " \n\t" \
	350	+ __FOP_RET(#op "_" #dst)
340	351
341	352	#define FOP1EEX(op, dst) \
342	353	FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
..	..	@@ -368,8 +379,9 @@
368	379	FOP_END
369	380
370	381	#define FOP2E(op, dst, src) \
371		- FOP_FUNC(#op "_" #dst "_" #src) \
372		- #op " %" #src ", %" #dst " \n\t" FOP_RET
	382	+ __FOP_FUNC(#op "_" #dst "_" #src) \
	383	+ #op " %" #src ", %" #dst " \n\t" \
	384	+ __FOP_RET(#op "_" #dst "_" #src)
373	385
374	386	#define FASTOP2(op) \
375	387	FOP_START(op) \
..	..	@@ -407,8 +419,9 @@
407	419	FOP_END
408	420
409	421	#define FOP3E(op, dst, src, src2) \
410		- FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
411		- #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
	422	+ __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
	423	+ #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
	424	+ __FOP_RET(#op "_" #dst "_" #src "_" #src2)
412	425
413	426	/* 3-operand, word-only, src2=cl */
414	427	#define FASTOP3WCL(op) \
..	..	@@ -420,19 +433,29 @@
420	433	FOP_END
421	434
422	435	/* Special case for SETcc - 1 instruction per cc */
	436	+
	437	+/*
	438	+ * Depending on .config the SETcc functions look like:
	439	+ *
	440	+ * SETcc %al [3 bytes]
	441	+ * RET \| JMP __x86_return_thunk [1,5 bytes; CONFIG_RETHUNK]
	442	+ * INT3 [1 byte; CONFIG_SLS]
	443	+ */
	444	+#define SETCC_ALIGN 16
	445	+
423	446	#define FOP_SETCC(op) \
424		- ".align 4 \n\t" \
	447	+ ".align " __stringify(SETCC_ALIGN) " \n\t" \
425	448	".type " #op ", @function \n\t" \
426	449	#op ": \n\t" \
427	450	#op " %al \n\t" \
428		- FOP_RET
	451	+ __FOP_RET(#op) \
	452	+ ".skip " __stringify(SETCC_ALIGN) " - (.-" #op "), 0xcc \n\t"
429	453
430	454	asm(".pushsection .fixup, \"ax\"\n"
431		- ".global kvm_fastop_exception \n"
432		- "kvm_fastop_exception: xor %esi, %esi; ret\n"
	455	+ "kvm_fastop_exception: xor %esi, %esi; " ASM_RET
433	456	".popsection");
434	457
435		-FOP_START(setcc)
	458	+__FOP_START(setcc, SETCC_ALIGN)
436	459	FOP_SETCC(seto)
437	460	FOP_SETCC(setno)
438	461	FOP_SETCC(setc)
..	..	@@ -451,7 +474,10 @@
451	474	FOP_SETCC(setnle)
452	475	FOP_END;
453	476
454		-FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
	477	+FOP_START(salc)
	478	+FOP_FUNC(salc)
	479	+"pushf; sbb %al, %al; popf \n\t"
	480	+FOP_RET(salc)
455	481	FOP_END;
456	482
457	483	/*
..	..	@@ -654,6 +680,17 @@
654	680	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
655	681	}
656	682
	683	+static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)
	684	+{
	685	+ return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;
	686	+}
	687	+
	688	+static inline bool emul_is_noncanonical_address(u64 la,
	689	+ struct x86_emulate_ctxt *ctxt)
	690	+{
	691	+ return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la;
	692	+}
	693	+
657	694	/*
658	695	* x86 defines three classes of vector instructions: explicitly
659	696	* aligned, explicitly unaligned, and the rest, which change behaviour
..	..	@@ -759,8 +796,7 @@
759	796	ctxt->mode, linear);
760	797	}
761	798
762		-static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
763		- enum x86emul_mode mode)
	799	+static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst)
764	800	{
765	801	ulong linear;
766	802	int rc;
..	..	@@ -770,41 +806,71 @@
770	806
771	807	if (ctxt->op_bytes != sizeof(unsigned long))
772	808	addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
773		- rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
	809	+ rc = __linearize(ctxt, addr, &max_size, 1, false, true, ctxt->mode, &linear);
774	810	if (rc == X86EMUL_CONTINUE)
775	811	ctxt->_eip = addr.ea;
776	812	return rc;
777	813	}
778	814
779		-static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
	815	+static inline int emulator_recalc_and_set_mode(struct x86_emulate_ctxt *ctxt)
780	816	{
781		- return assign_eip(ctxt, dst, ctxt->mode);
	817	+ u64 efer;
	818	+ struct desc_struct cs;
	819	+ u16 selector;
	820	+ u32 base3;
	821	+
	822	+ ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
	823	+
	824	+ if (!(ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PE)) {
	825	+ /* Real mode. cpu must not have long mode active */
	826	+ if (efer & EFER_LMA)
	827	+ return X86EMUL_UNHANDLEABLE;
	828	+ ctxt->mode = X86EMUL_MODE_REAL;
	829	+ return X86EMUL_CONTINUE;
	830	+ }
	831	+
	832	+ if (ctxt->eflags & X86_EFLAGS_VM) {
	833	+ /* Protected/VM86 mode. cpu must not have long mode active */
	834	+ if (efer & EFER_LMA)
	835	+ return X86EMUL_UNHANDLEABLE;
	836	+ ctxt->mode = X86EMUL_MODE_VM86;
	837	+ return X86EMUL_CONTINUE;
	838	+ }
	839	+
	840	+ if (!ctxt->ops->get_segment(ctxt, &selector, &cs, &base3, VCPU_SREG_CS))
	841	+ return X86EMUL_UNHANDLEABLE;
	842	+
	843	+ if (efer & EFER_LMA) {
	844	+ if (cs.l) {
	845	+ /* Proper long mode */
	846	+ ctxt->mode = X86EMUL_MODE_PROT64;
	847	+ } else if (cs.d) {
	848	+ /* 32 bit compatibility mode*/
	849	+ ctxt->mode = X86EMUL_MODE_PROT32;
	850	+ } else {
	851	+ ctxt->mode = X86EMUL_MODE_PROT16;
	852	+ }
	853	+ } else {
	854	+ /* Legacy 32 bit / 16 bit mode */
	855	+ ctxt->mode = cs.d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
	856	+ }
	857	+
	858	+ return X86EMUL_CONTINUE;
782	859	}
783	860
784		-static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
785		- const struct desc_struct *cs_desc)
	861	+static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
786	862	{
787		- enum x86emul_mode mode = ctxt->mode;
788		- int rc;
	863	+ return assign_eip(ctxt, dst);
	864	+}
789	865
790		-#ifdef CONFIG_X86_64
791		- if (ctxt->mode >= X86EMUL_MODE_PROT16) {
792		- if (cs_desc->l) {
793		- u64 efer = 0;
	866	+static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst)
	867	+{
	868	+ int rc = emulator_recalc_and_set_mode(ctxt);
794	869
795		- ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
796		- if (efer & EFER_LMA)
797		- mode = X86EMUL_MODE_PROT64;
798		- } else
799		- mode = X86EMUL_MODE_PROT32; /* temporary value */
800		- }
801		-#endif
802		- if (mode == X86EMUL_MODE_PROT16 \|\| mode == X86EMUL_MODE_PROT32)
803		- mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
804		- rc = assign_eip(ctxt, dst, mode);
805		- if (rc == X86EMUL_CONTINUE)
806		- ctxt->mode = mode;
807		- return rc;
	870	+ if (rc != X86EMUL_CONTINUE)
	871	+ return rc;
	872	+
	873	+ return assign_eip(ctxt, dst);
808	874	}
809	875
810	876	static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
..	..	@@ -1033,7 +1099,7 @@
1033	1099	static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1034	1100	{
1035	1101	u8 rc;
1036		- void (fop)(void) = (void )em_setcc + 4 * (condition & 0xf);
	1102	+ void (fop)(void) = (void )em_setcc + SETCC_ALIGN * (condition & 0xf);
1037	1103
1038	1104	flags = (flags & EFLAGS_MASK) \| X86_EFLAGS_IF;
1039	1105	asm("push %[flags]; popf; " CALL_NOSPEC
..	..	@@ -1059,8 +1125,23 @@
1059	1125	}
1060	1126	}
1061	1127
1062		-static void read_sse_reg(struct x86_emulate_ctxt ctxt, sse128_t data, int reg)
	1128	+static void emulator_get_fpu(void)
1063	1129	{
	1130	+ fpregs_lock();
	1131	+
	1132	+ fpregs_assert_state_consistent();
	1133	+ if (test_thread_flag(TIF_NEED_FPU_LOAD))
	1134	+ switch_fpu_return();
	1135	+}
	1136	+
	1137	+static void emulator_put_fpu(void)
	1138	+{
	1139	+ fpregs_unlock();
	1140	+}
	1141	+
	1142	+static void read_sse_reg(sse128_t *data, int reg)
	1143	+{
	1144	+ emulator_get_fpu();
1064	1145	switch (reg) {
1065	1146	case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1066	1147	case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
..	..	@@ -1082,11 +1163,12 @@
1082	1163	#endif
1083	1164	default: BUG();
1084	1165	}
	1166	+ emulator_put_fpu();
1085	1167	}
1086	1168
1087		-static void write_sse_reg(struct x86_emulate_ctxt ctxt, sse128_t data,
1088		- int reg)
	1169	+static void write_sse_reg(sse128_t *data, int reg)
1089	1170	{
	1171	+ emulator_get_fpu();
1090	1172	switch (reg) {
1091	1173	case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1092	1174	case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
..	..	@@ -1108,10 +1190,12 @@
1108	1190	#endif
1109	1191	default: BUG();
1110	1192	}
	1193	+ emulator_put_fpu();
1111	1194	}
1112	1195
1113		-static void read_mmx_reg(struct x86_emulate_ctxt ctxt, u64 data, int reg)
	1196	+static void read_mmx_reg(u64 *data, int reg)
1114	1197	{
	1198	+ emulator_get_fpu();
1115	1199	switch (reg) {
1116	1200	case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1117	1201	case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
..	..	@@ -1123,10 +1207,12 @@
1123	1207	case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1124	1208	default: BUG();
1125	1209	}
	1210	+ emulator_put_fpu();
1126	1211	}
1127	1212
1128		-static void write_mmx_reg(struct x86_emulate_ctxt ctxt, u64 data, int reg)
	1213	+static void write_mmx_reg(u64 *data, int reg)
1129	1214	{
	1215	+ emulator_get_fpu();
1130	1216	switch (reg) {
1131	1217	case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1132	1218	case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
..	..	@@ -1138,6 +1224,7 @@
1138	1224	case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1139	1225	default: BUG();
1140	1226	}
	1227	+ emulator_put_fpu();
1141	1228	}
1142	1229
1143	1230	static int em_fninit(struct x86_emulate_ctxt *ctxt)
..	..	@@ -1145,7 +1232,9 @@
1145	1232	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS \| X86_CR0_EM))
1146	1233	return emulate_nm(ctxt);
1147	1234
	1235	+ emulator_get_fpu();
1148	1236	asm volatile("fninit");
	1237	+ emulator_put_fpu();
1149	1238	return X86EMUL_CONTINUE;
1150	1239	}
1151	1240
..	..	@@ -1156,7 +1245,9 @@
1156	1245	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS \| X86_CR0_EM))
1157	1246	return emulate_nm(ctxt);
1158	1247
	1248	+ emulator_get_fpu();
1159	1249	asm volatile("fnstcw %0": "+m"(fcw));
	1250	+ emulator_put_fpu();
1160	1251
1161	1252	ctxt->dst.val = fcw;
1162	1253
..	..	@@ -1170,7 +1261,9 @@
1170	1261	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS \| X86_CR0_EM))
1171	1262	return emulate_nm(ctxt);
1172	1263
	1264	+ emulator_get_fpu();
1173	1265	asm volatile("fnstsw %0": "+m"(fsw));
	1266	+ emulator_put_fpu();
1174	1267
1175	1268	ctxt->dst.val = fsw;
1176	1269
..	..	@@ -1189,7 +1282,7 @@
1189	1282	op->type = OP_XMM;
1190	1283	op->bytes = 16;
1191	1284	op->addr.xmm = reg;
1192		- read_sse_reg(ctxt, &op->vec_val, reg);
	1285	+ read_sse_reg(&op->vec_val, reg);
1193	1286	return;
1194	1287	}
1195	1288	if (ctxt->d & Mmx) {
..	..	@@ -1240,7 +1333,7 @@
1240	1333	op->type = OP_XMM;
1241	1334	op->bytes = 16;
1242	1335	op->addr.xmm = ctxt->modrm_rm;
1243		- read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
	1336	+ read_sse_reg(&op->vec_val, ctxt->modrm_rm);
1244	1337	return rc;
1245	1338	}
1246	1339	if (ctxt->d & Mmx) {
..	..	@@ -1509,7 +1602,7 @@
1509	1602	return emulate_gp(ctxt, index << 3 \| 0x2);
1510	1603
1511	1604	addr = dt.address + index * 8;
1512		- return linear_read_system(ctxt, addr, desc, sizeof *desc);
	1605	+ return linear_read_system(ctxt, addr, desc, sizeof(*desc));
1513	1606	}
1514	1607
1515	1608	static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
..	..	@@ -1522,7 +1615,7 @@
1522	1615	struct desc_struct desc;
1523	1616	u16 sel;
1524	1617
1525		- memset (dt, 0, sizeof *dt);
	1618	+ memset(dt, 0, sizeof(*dt));
1526	1619	if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1527	1620	VCPU_SREG_LDTR))
1528	1621	return;
..	..	@@ -1586,7 +1679,7 @@
1586	1679	if (rc != X86EMUL_CONTINUE)
1587	1680	return rc;
1588	1681
1589		- return linear_write_system(ctxt, addr, desc, sizeof *desc);
	1682	+ return linear_write_system(ctxt, addr, desc, sizeof(*desc));
1590	1683	}
1591	1684
1592	1685	static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
..	..	@@ -1604,7 +1697,7 @@
1604	1697	u16 dummy;
1605	1698	u32 base3 = 0;
1606	1699
1607		- memset(&seg_desc, 0, sizeof seg_desc);
	1700	+ memset(&seg_desc, 0, sizeof(seg_desc));
1608	1701
1609	1702	if (ctxt->mode == X86EMUL_MODE_REAL) {
1610	1703	/* set real mode segment descriptor (keep limit etc. for
..	..	@@ -1669,11 +1762,6 @@
1669	1762	goto exception;
1670	1763	}
1671	1764
1672		- if (!seg_desc.p) {
1673		- err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1674		- goto exception;
1675		- }
1676		-
1677	1765	dpl = seg_desc.dpl;
1678	1766
1679	1767	switch (seg) {
..	..	@@ -1713,12 +1801,6 @@
1713	1801	case VCPU_SREG_TR:
1714	1802	if (seg_desc.s \|\| (seg_desc.type != 1 && seg_desc.type != 9))
1715	1803	goto exception;
1716		- old_desc = seg_desc;
1717		- seg_desc.type \|= 2; /* busy */
1718		- ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1719		- sizeof(seg_desc), &ctxt->exception);
1720		- if (ret != X86EMUL_CONTINUE)
1721		- return ret;
1722	1804	break;
1723	1805	case VCPU_SREG_LDTR:
1724	1806	if (seg_desc.s \|\| seg_desc.type != 2)
..	..	@@ -1737,6 +1819,11 @@
1737	1819	break;
1738	1820	}
1739	1821
	1822	+ if (!seg_desc.p) {
	1823	+ err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
	1824	+ goto exception;
	1825	+ }
	1826	+
1740	1827	if (seg_desc.s) {
1741	1828	/* mark segment as accessed */
1742	1829	if (!(seg_desc.type & 1)) {
..	..	@@ -1751,8 +1838,17 @@
1751	1838	if (ret != X86EMUL_CONTINUE)
1752	1839	return ret;
1753	1840	if (emul_is_noncanonical_address(get_desc_base(&seg_desc) \|
1754		- ((u64)base3 << 32), ctxt))
1755		- return emulate_gp(ctxt, 0);
	1841	+ ((u64)base3 << 32), ctxt))
	1842	+ return emulate_gp(ctxt, err_code);
	1843	+ }
	1844	+
	1845	+ if (seg == VCPU_SREG_TR) {
	1846	+ old_desc = seg_desc;
	1847	+ seg_desc.type \|= 2; /* busy */
	1848	+ ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
	1849	+ sizeof(seg_desc), &ctxt->exception);
	1850	+ if (ret != X86EMUL_CONTINUE)
	1851	+ return ret;
1756	1852	}
1757	1853	load:
1758	1854	ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
..	..	@@ -1817,10 +1913,10 @@
1817	1913	op->bytes * op->count);
1818	1914	break;
1819	1915	case OP_XMM:
1820		- write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
	1916	+ write_sse_reg(&op->vec_val, op->addr.xmm);
1821	1917	break;
1822	1918	case OP_MM:
1823		- write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
	1919	+ write_mmx_reg(&op->mm_val, op->addr.mm);
1824	1920	break;
1825	1921	case OP_NONE:
1826	1922	/* no writeback */
..	..	@@ -1972,7 +2068,7 @@
1972	2068	if (rc != X86EMUL_CONTINUE)
1973	2069	return rc;
1974	2070
1975		- if (ctxt->modrm_reg == VCPU_SREG_SS)
	2071	+ if (seg == VCPU_SREG_SS)
1976	2072	ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1977	2073	if (ctxt->op_bytes > 2)
1978	2074	rsp_increment(ctxt, ctxt->op_bytes - 2);
..	..	@@ -2189,7 +2285,7 @@
2189	2285	if (rc != X86EMUL_CONTINUE)
2190	2286	return rc;
2191	2287
2192		- rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
	2288	+ rc = assign_eip_far(ctxt, ctxt->src.val);
2193	2289	/* Error handling is not implemented. */
2194	2290	if (rc != X86EMUL_CONTINUE)
2195	2291	return X86EMUL_UNHANDLEABLE;
..	..	@@ -2270,7 +2366,7 @@
2270	2366	&new_desc);
2271	2367	if (rc != X86EMUL_CONTINUE)
2272	2368	return rc;
2273		- rc = assign_eip_far(ctxt, eip, &new_desc);
	2369	+ rc = assign_eip_far(ctxt, eip);
2274	2370	/* Error handling is not implemented. */
2275	2371	if (rc != X86EMUL_CONTINUE)
2276	2372	return X86EMUL_UNHANDLEABLE;
..	..	@@ -2332,26 +2428,11 @@
2332	2428	static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2333	2429	{
2334	2430	#ifdef CONFIG_X86_64
2335		- u32 eax, ebx, ecx, edx;
2336		-
2337		- eax = 0x80000001;
2338		- ecx = 0;
2339		- ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2340		- return edx & bit(X86_FEATURE_LM);
	2431	+ return ctxt->ops->guest_has_long_mode(ctxt);
2341	2432	#else
2342	2433	return false;
2343	2434	#endif
2344	2435	}
2345		-
2346		-#define GET_SMSTATE(type, smbase, offset) \
2347		- ({ \
2348		- type __val; \
2349		- int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2350		- sizeof(__val)); \
2351		- if (r != X86EMUL_CONTINUE) \
2352		- return X86EMUL_UNHANDLEABLE; \
2353		- __val; \
2354		- })
2355	2436
2356	2437	static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2357	2438	{
..	..	@@ -2365,28 +2446,30 @@
2365	2446	desc->type = (flags >> 8) & 15;
2366	2447	}
2367	2448
2368		-static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
	2449	+static int rsm_load_seg_32(struct x86_emulate_ctxt ctxt, const char smstate,
	2450	+ int n)
2369	2451	{
2370	2452	struct desc_struct desc;
2371	2453	int offset;
2372	2454	u16 selector;
2373	2455
2374		- selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
	2456	+ selector = GET_SMSTATE(u32, smstate, 0x7fa8 + n * 4);
2375	2457
2376	2458	if (n < 3)
2377	2459	offset = 0x7f84 + n * 12;
2378	2460	else
2379	2461	offset = 0x7f2c + (n - 3) * 12;
2380	2462
2381		- set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2382		- set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2383		- rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
	2463	+ set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
	2464	+ set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
	2465	+ rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, offset));
2384	2466	ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2385	2467	return X86EMUL_CONTINUE;
2386	2468	}
2387	2469
2388	2470	#ifdef CONFIG_X86_64
2389		-static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
	2471	+static int rsm_load_seg_64(struct x86_emulate_ctxt ctxt, const char smstate,
	2472	+ int n)
2390	2473	{
2391	2474	struct desc_struct desc;
2392	2475	int offset;
..	..	@@ -2395,11 +2478,11 @@
2395	2478
2396	2479	offset = 0x7e00 + n * 16;
2397	2480
2398		- selector = GET_SMSTATE(u16, smbase, offset);
2399		- rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2400		- set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2401		- set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2402		- base3 = GET_SMSTATE(u32, smbase, offset + 12);
	2481	+ selector = GET_SMSTATE(u16, smstate, offset);
	2482	+ rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smstate, offset + 2) << 8);
	2483	+ set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
	2484	+ set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
	2485	+ base3 = GET_SMSTATE(u32, smstate, offset + 12);
2403	2486
2404	2487	ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2405	2488	return X86EMUL_CONTINUE;
..	..	@@ -2451,7 +2534,8 @@
2451	2534	return X86EMUL_CONTINUE;
2452	2535	}
2453	2536
2454		-static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
	2537	+static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,
	2538	+ const char *smstate)
2455	2539	{
2456	2540	struct desc_struct desc;
2457	2541	struct desc_ptr dt;
..	..	@@ -2459,54 +2543,60 @@
2459	2543	u32 val, cr0, cr3, cr4;
2460	2544	int i;
2461	2545
2462		- cr0 = GET_SMSTATE(u32, smbase, 0x7ffc);
2463		- cr3 = GET_SMSTATE(u32, smbase, 0x7ff8);
2464		- ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7ff4) \| X86_EFLAGS_FIXED;
2465		- ctxt->_eip = GET_SMSTATE(u32, smbase, 0x7ff0);
	2546	+ cr0 = GET_SMSTATE(u32, smstate, 0x7ffc);
	2547	+ cr3 = GET_SMSTATE(u32, smstate, 0x7ff8);
	2548	+ ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7ff4) \| X86_EFLAGS_FIXED;
	2549	+ ctxt->_eip = GET_SMSTATE(u32, smstate, 0x7ff0);
2466	2550
2467	2551	for (i = 0; i < 8; i++)
2468		- reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i 4);
	2552	+ reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i 4);
2469	2553
2470		- val = GET_SMSTATE(u32, smbase, 0x7fcc);
2471		- ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) \| DR6_FIXED_1);
2472		- val = GET_SMSTATE(u32, smbase, 0x7fc8);
2473		- ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) \| DR7_FIXED_1);
	2554	+ val = GET_SMSTATE(u32, smstate, 0x7fcc);
2474	2555
2475		- selector = GET_SMSTATE(u32, smbase, 0x7fc4);
2476		- set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f64));
2477		- set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f60));
2478		- rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f5c));
	2556	+ if (ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) \| DR6_FIXED_1))
	2557	+ return X86EMUL_UNHANDLEABLE;
	2558	+
	2559	+ val = GET_SMSTATE(u32, smstate, 0x7fc8);
	2560	+
	2561	+ if (ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) \| DR7_FIXED_1))
	2562	+ return X86EMUL_UNHANDLEABLE;
	2563	+
	2564	+ selector = GET_SMSTATE(u32, smstate, 0x7fc4);
	2565	+ set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f64));
	2566	+ set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f60));
	2567	+ rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f5c));
2479	2568	ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2480	2569
2481		- selector = GET_SMSTATE(u32, smbase, 0x7fc0);
2482		- set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f80));
2483		- set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f7c));
2484		- rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f78));
	2570	+ selector = GET_SMSTATE(u32, smstate, 0x7fc0);
	2571	+ set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f80));
	2572	+ set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f7c));
	2573	+ rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f78));
2485	2574	ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2486	2575
2487		- dt.address = GET_SMSTATE(u32, smbase, 0x7f74);
2488		- dt.size = GET_SMSTATE(u32, smbase, 0x7f70);
	2576	+ dt.address = GET_SMSTATE(u32, smstate, 0x7f74);
	2577	+ dt.size = GET_SMSTATE(u32, smstate, 0x7f70);
2489	2578	ctxt->ops->set_gdt(ctxt, &dt);
2490	2579
2491		- dt.address = GET_SMSTATE(u32, smbase, 0x7f58);
2492		- dt.size = GET_SMSTATE(u32, smbase, 0x7f54);
	2580	+ dt.address = GET_SMSTATE(u32, smstate, 0x7f58);
	2581	+ dt.size = GET_SMSTATE(u32, smstate, 0x7f54);
2493	2582	ctxt->ops->set_idt(ctxt, &dt);
2494	2583
2495	2584	for (i = 0; i < 6; i++) {
2496		- int r = rsm_load_seg_32(ctxt, smbase, i);
	2585	+ int r = rsm_load_seg_32(ctxt, smstate, i);
2497	2586	if (r != X86EMUL_CONTINUE)
2498	2587	return r;
2499	2588	}
2500	2589
2501		- cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
	2590	+ cr4 = GET_SMSTATE(u32, smstate, 0x7f14);
2502	2591
2503		- ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
	2592	+ ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));
2504	2593
2505	2594	return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2506	2595	}
2507	2596
2508	2597	#ifdef CONFIG_X86_64
2509		-static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
	2598	+static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,
	2599	+ const char *smstate)
2510	2600	{
2511	2601	struct desc_struct desc;
2512	2602	struct desc_ptr dt;
..	..	@@ -2516,43 +2606,50 @@
2516	2606	int i, r;
2517	2607
2518	2608	for (i = 0; i < 16; i++)
2519		- reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i 8);
	2609	+ reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i 8);
2520	2610
2521		- ctxt->_eip = GET_SMSTATE(u64, smbase, 0x7f78);
2522		- ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) \| X86_EFLAGS_FIXED;
	2611	+ ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);
	2612	+ ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) \| X86_EFLAGS_FIXED;
2523	2613
2524		- val = GET_SMSTATE(u32, smbase, 0x7f68);
2525		- ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) \| DR6_FIXED_1);
2526		- val = GET_SMSTATE(u32, smbase, 0x7f60);
2527		- ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) \| DR7_FIXED_1);
	2614	+ val = GET_SMSTATE(u64, smstate, 0x7f68);
2528	2615
2529		- cr0 = GET_SMSTATE(u64, smbase, 0x7f58);
2530		- cr3 = GET_SMSTATE(u64, smbase, 0x7f50);
2531		- cr4 = GET_SMSTATE(u64, smbase, 0x7f48);
2532		- ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2533		- val = GET_SMSTATE(u64, smbase, 0x7ed0);
2534		- ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
	2616	+ if (ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) \| DR6_FIXED_1))
	2617	+ return X86EMUL_UNHANDLEABLE;
2535	2618
2536		- selector = GET_SMSTATE(u32, smbase, 0x7e90);
2537		- rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2538		- set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e94));
2539		- set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e98));
2540		- base3 = GET_SMSTATE(u32, smbase, 0x7e9c);
	2619	+ val = GET_SMSTATE(u64, smstate, 0x7f60);
	2620	+
	2621	+ if (ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) \| DR7_FIXED_1))
	2622	+ return X86EMUL_UNHANDLEABLE;
	2623	+
	2624	+ cr0 = GET_SMSTATE(u64, smstate, 0x7f58);
	2625	+ cr3 = GET_SMSTATE(u64, smstate, 0x7f50);
	2626	+ cr4 = GET_SMSTATE(u64, smstate, 0x7f48);
	2627	+ ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));
	2628	+ val = GET_SMSTATE(u64, smstate, 0x7ed0);
	2629	+
	2630	+ if (ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA))
	2631	+ return X86EMUL_UNHANDLEABLE;
	2632	+
	2633	+ selector = GET_SMSTATE(u32, smstate, 0x7e90);
	2634	+ rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e92) << 8);
	2635	+ set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e94));
	2636	+ set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e98));
	2637	+ base3 = GET_SMSTATE(u32, smstate, 0x7e9c);
2541	2638	ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2542	2639
2543		- dt.size = GET_SMSTATE(u32, smbase, 0x7e84);
2544		- dt.address = GET_SMSTATE(u64, smbase, 0x7e88);
	2640	+ dt.size = GET_SMSTATE(u32, smstate, 0x7e84);
	2641	+ dt.address = GET_SMSTATE(u64, smstate, 0x7e88);
2545	2642	ctxt->ops->set_idt(ctxt, &dt);
2546	2643
2547		- selector = GET_SMSTATE(u32, smbase, 0x7e70);
2548		- rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2549		- set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e74));
2550		- set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e78));
2551		- base3 = GET_SMSTATE(u32, smbase, 0x7e7c);
	2644	+ selector = GET_SMSTATE(u32, smstate, 0x7e70);
	2645	+ rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e72) << 8);
	2646	+ set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e74));
	2647	+ set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e78));
	2648	+ base3 = GET_SMSTATE(u32, smstate, 0x7e7c);
2552	2649	ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2553	2650
2554		- dt.size = GET_SMSTATE(u32, smbase, 0x7e64);
2555		- dt.address = GET_SMSTATE(u64, smbase, 0x7e68);
	2651	+ dt.size = GET_SMSTATE(u32, smstate, 0x7e64);
	2652	+ dt.address = GET_SMSTATE(u64, smstate, 0x7e68);
2556	2653	ctxt->ops->set_gdt(ctxt, &dt);
2557	2654
2558	2655	r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
..	..	@@ -2560,7 +2657,7 @@
2560	2657	return r;
2561	2658
2562	2659	for (i = 0; i < 6; i++) {
2563		- r = rsm_load_seg_64(ctxt, smbase, i);
	2660	+ r = rsm_load_seg_64(ctxt, smstate, i);
2564	2661	if (r != X86EMUL_CONTINUE)
2565	2662	return r;
2566	2663	}
..	..	@@ -2572,11 +2669,24 @@
2572	2669	static int em_rsm(struct x86_emulate_ctxt *ctxt)
2573	2670	{
2574	2671	unsigned long cr0, cr4, efer;
	2672	+ char buf[512];
2575	2673	u64 smbase;
2576	2674	int ret;
2577	2675
2578	2676	if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2579	2677	return emulate_ud(ctxt);
	2678	+
	2679	+ smbase = ctxt->ops->get_smbase(ctxt);
	2680	+
	2681	+ ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));
	2682	+ if (ret != X86EMUL_CONTINUE)
	2683	+ return X86EMUL_UNHANDLEABLE;
	2684	+
	2685	+ if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
	2686	+ ctxt->ops->set_nmi_mask(ctxt, false);
	2687	+
	2688	+ ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
	2689	+ ~(X86EMUL_SMM_INSIDE_NMI_MASK \| X86EMUL_SMM_MASK));
2580	2690
2581	2691	/*
2582	2692	* Get back to real mode, to prepare a safe state in which to load
..	..	@@ -2614,33 +2724,28 @@
2614	2724	ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2615	2725	}
2616	2726
2617		- smbase = ctxt->ops->get_smbase(ctxt);
2618		-
2619	2727	/*
2620	2728	* Give pre_leave_smm() a chance to make ISA-specific changes to the
2621	2729	* vCPU state (e.g. enter guest mode) before loading state from the SMM
2622	2730	* state-save area.
2623	2731	*/
2624		- if (ctxt->ops->pre_leave_smm(ctxt, smbase))
	2732	+ if (ctxt->ops->pre_leave_smm(ctxt, buf))
2625	2733	return X86EMUL_UNHANDLEABLE;
2626	2734
2627	2735	#ifdef CONFIG_X86_64
2628	2736	if (emulator_has_longmode(ctxt))
2629		- ret = rsm_load_state_64(ctxt, smbase + 0x8000);
	2737	+ ret = rsm_load_state_64(ctxt, buf);
2630	2738	else
2631	2739	#endif
2632		- ret = rsm_load_state_32(ctxt, smbase + 0x8000);
	2740	+ ret = rsm_load_state_32(ctxt, buf);
2633	2741
2634	2742	if (ret != X86EMUL_CONTINUE) {
2635	2743	/* FIXME: should triple fault */
2636	2744	return X86EMUL_UNHANDLEABLE;
2637	2745	}
2638	2746
2639		- if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2640		- ctxt->ops->set_nmi_mask(ctxt, false);
	2747	+ ctxt->ops->post_leave_smm(ctxt);
2641	2748
2642		- ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2643		- ~(X86EMUL_SMM_INSIDE_NMI_MASK \| X86EMUL_SMM_MASK));
2644	2749	return X86EMUL_CONTINUE;
2645	2750	}
2646	2751
..	..	@@ -2676,10 +2781,8 @@
2676	2781	u32 eax, ebx, ecx, edx;
2677	2782
2678	2783	eax = ecx = 0;
2679		- ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2680		- return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2681		- && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2682		- && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
	2784	+ ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
	2785	+ return is_guest_vendor_intel(ebx, ecx, edx);
2683	2786	}
2684	2787
2685	2788	static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
..	..	@@ -2696,33 +2799,24 @@
2696	2799
2697	2800	eax = 0x00000000;
2698	2801	ecx = 0x00000000;
2699		- ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
	2802	+ ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2700	2803	/*
2701		- * Intel ("GenuineIntel")
2702		- * remark: Intel CPUs only support "syscall" in 64bit
2703		- * longmode. Also an 64bit guest with a
2704		- * 32bit compat-app running will #UD !! While this
2705		- * behaviour can be fixed (by emulating) into AMD
2706		- * response - CPUs of AMD can't behave like Intel.
	2804	+ * remark: Intel CPUs only support "syscall" in 64bit longmode. Also a
	2805	+ * 64bit guest with a 32bit compat-app running will #UD !! While this
	2806	+ * behaviour can be fixed (by emulating) into AMD response - CPUs of
	2807	+ * AMD can't behave like Intel.
2707	2808	*/
2708		- if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2709		- ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2710		- edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
	2809	+ if (is_guest_vendor_intel(ebx, ecx, edx))
2711	2810	return false;
2712	2811
2713		- /* AMD ("AuthenticAMD") */
2714		- if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2715		- ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2716		- edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
	2812	+ if (is_guest_vendor_amd(ebx, ecx, edx) \|\|
	2813	+ is_guest_vendor_hygon(ebx, ecx, edx))
2717	2814	return true;
2718	2815
2719		- /* AMD ("AMDisbetter!") */
2720		- if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2721		- ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2722		- edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2723		- return true;
2724		-
2725		- /* default: (not Intel, not AMD), apply Intel's stricter rules... */
	2816	+ /*
	2817	+ * default: (not Intel, not AMD, not Hygon), apply Intel's
	2818	+ * stricter rules...
	2819	+ */
2726	2820	return false;
2727	2821	}
2728	2822
..	..	@@ -2743,11 +2837,10 @@
2743	2837	return emulate_ud(ctxt);
2744	2838
2745	2839	ops->get_msr(ctxt, MSR_EFER, &efer);
2746		- setup_syscalls_segments(ctxt, &cs, &ss);
2747		-
2748	2840	if (!(efer & EFER_SCE))
2749	2841	return emulate_ud(ctxt);
2750	2842
	2843	+ setup_syscalls_segments(ctxt, &cs, &ss);
2751	2844	ops->get_msr(ctxt, MSR_STAR, &msr_data);
2752	2845	msr_data >>= 32;
2753	2846	cs_sel = (u16)(msr_data & 0xfffc);
..	..	@@ -2811,12 +2904,11 @@
2811	2904	if (ctxt->mode == X86EMUL_MODE_PROT64)
2812	2905	return X86EMUL_UNHANDLEABLE;
2813	2906
2814		- setup_syscalls_segments(ctxt, &cs, &ss);
2815		-
2816	2907	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2817	2908	if ((msr_data & 0xfffc) == 0x0)
2818	2909	return emulate_gp(ctxt, 0);
2819	2910
	2911	+ setup_syscalls_segments(ctxt, &cs, &ss);
2820	2912	ctxt->eflags &= ~(X86_EFLAGS_VM \| X86_EFLAGS_IF);
2821	2913	cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2822	2914	ss_sel = cs_sel + 8;
..	..	@@ -2834,6 +2926,8 @@
2834	2926	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2835	2927	*reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2836	2928	(u32)msr_data;
	2929	+ if (efer & EFER_LMA)
	2930	+ ctxt->mode = X86EMUL_MODE_PROT64;
2837	2931
2838	2932	return X86EMUL_CONTINUE;
2839	2933	}
..	..	@@ -2892,6 +2986,7 @@
2892	2986	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2893	2987
2894	2988	ctxt->_eip = rdx;
	2989	+ ctxt->mode = usermode;
2895	2990	*reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2896	2991
2897	2992	return X86EMUL_CONTINUE;
..	..	@@ -2983,7 +3078,7 @@
2983	3078	case 0xa4: /* movsb */
2984	3079	case 0xa5: /* movsd/w */
2985	3080	*reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2986		- /* fall through */
	3081	+ fallthrough;
2987	3082	case 0xaa: /* stosb */
2988	3083	case 0xab: /* stosd/w */
2989	3084	*reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
..	..	@@ -3077,17 +3172,17 @@
3077	3172	int ret;
3078	3173	u32 new_tss_base = get_desc_base(new_desc);
3079	3174
3080		- ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
	3175	+ ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3081	3176	if (ret != X86EMUL_CONTINUE)
3082	3177	return ret;
3083	3178
3084	3179	save_state_to_tss16(ctxt, &tss_seg);
3085	3180
3086		- ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
	3181	+ ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3087	3182	if (ret != X86EMUL_CONTINUE)
3088	3183	return ret;
3089	3184
3090		- ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
	3185	+ ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3091	3186	if (ret != X86EMUL_CONTINUE)
3092	3187	return ret;
3093	3188
..	..	@@ -3096,7 +3191,7 @@
3096	3191
3097	3192	ret = linear_write_system(ctxt, new_tss_base,
3098	3193	&tss_seg.prev_task_link,
3099		- sizeof tss_seg.prev_task_link);
	3194	+ sizeof(tss_seg.prev_task_link));
3100	3195	if (ret != X86EMUL_CONTINUE)
3101	3196	return ret;
3102	3197	}
..	..	@@ -3218,7 +3313,7 @@
3218	3313	u32 eip_offset = offsetof(struct tss_segment_32, eip);
3219	3314	u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3220	3315
3221		- ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
	3316	+ ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3222	3317	if (ret != X86EMUL_CONTINUE)
3223	3318	return ret;
3224	3319
..	..	@@ -3230,7 +3325,7 @@
3230	3325	if (ret != X86EMUL_CONTINUE)
3231	3326	return ret;
3232	3327
3233		- ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
	3328	+ ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3234	3329	if (ret != X86EMUL_CONTINUE)
3235	3330	return ret;
3236	3331
..	..	@@ -3239,7 +3334,7 @@
3239	3334
3240	3335	ret = linear_write_system(ctxt, new_tss_base,
3241	3336	&tss_seg.prev_task_link,
3242		- sizeof tss_seg.prev_task_link);
	3337	+ sizeof(tss_seg.prev_task_link));
3243	3338	if (ret != X86EMUL_CONTINUE)
3244	3339	return ret;
3245	3340	}
..	..	@@ -3488,7 +3583,7 @@
3488	3583	if (rc != X86EMUL_CONTINUE)
3489	3584	return rc;
3490	3585
3491		- rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
	3586	+ rc = assign_eip_far(ctxt, ctxt->src.val);
3492	3587	if (rc != X86EMUL_CONTINUE)
3493	3588	goto fail;
3494	3589
..	..	@@ -3560,8 +3655,10 @@
3560	3655	{
3561	3656	u64 tsc_aux = 0;
3562	3657
3563		- if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
	3658	+ if (!ctxt->ops->guest_has_rdpid(ctxt))
3564	3659	return emulate_ud(ctxt);
	3660	+
	3661	+ ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux);
3565	3662	ctxt->dst.val = tsc_aux;
3566	3663	return X86EMUL_CONTINUE;
3567	3664	}
..	..	@@ -3593,18 +3690,11 @@
3593	3690	return X86EMUL_CONTINUE;
3594	3691	}
3595	3692
3596		-#define FFL(x) bit(X86_FEATURE_##x)
3597		-
3598	3693	static int em_movbe(struct x86_emulate_ctxt *ctxt)
3599	3694	{
3600		- u32 ebx, ecx, edx, eax = 1;
3601	3695	u16 tmp;
3602	3696
3603		- /*
3604		- * Check MOVBE is set in the guest-visible CPUID leaf.
3605		- */
3606		- ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3607		- if (!(ecx & FFL(MOVBE)))
	3697	+ if (!ctxt->ops->guest_has_movbe(ctxt))
3608	3698	return emulate_ud(ctxt);
3609	3699
3610	3700	switch (ctxt->op_bytes) {
..	..	@@ -3635,11 +3725,25 @@
3635	3725
3636	3726	static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3637	3727	{
3638		- if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
	3728	+ int cr_num = ctxt->modrm_reg;
	3729	+ int r;
	3730	+
	3731	+ if (ctxt->ops->set_cr(ctxt, cr_num, ctxt->src.val))
3639	3732	return emulate_gp(ctxt, 0);
3640	3733
3641	3734	/* Disable writeback. */
3642	3735	ctxt->dst.type = OP_NONE;
	3736	+
	3737	+ if (cr_num == 0) {
	3738	+ /*
	3739	+ * CR0 write might have updated CR0.PE and/or CR0.PG
	3740	+ * which can affect the cpu's execution mode.
	3741	+ */
	3742	+ r = emulator_recalc_and_set_mode(ctxt);
	3743	+ if (r != X86EMUL_CONTINUE)
	3744	+ return r;
	3745	+ }
	3746	+
3643	3747	return X86EMUL_CONTINUE;
3644	3748	}
3645	3749
..	..	@@ -3663,21 +3767,35 @@
3663	3767
3664	3768	static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3665	3769	{
	3770	+ u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3666	3771	u64 msr_data;
	3772	+ int r;
3667	3773
3668	3774	msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3669	3775	\| ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3670		- if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
	3776	+ r = ctxt->ops->set_msr(ctxt, msr_index, msr_data);
	3777	+
	3778	+ if (r == X86EMUL_IO_NEEDED)
	3779	+ return r;
	3780	+
	3781	+ if (r > 0)
3671	3782	return emulate_gp(ctxt, 0);
3672	3783
3673		- return X86EMUL_CONTINUE;
	3784	+ return r < 0 ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE;
3674	3785	}
3675	3786
3676	3787	static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3677	3788	{
	3789	+ u64 msr_index = reg_read(ctxt, VCPU_REGS_RCX);
3678	3790	u64 msr_data;
	3791	+ int r;
3679	3792
3680		- if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
	3793	+ r = ctxt->ops->get_msr(ctxt, msr_index, &msr_data);
	3794	+
	3795	+ if (r == X86EMUL_IO_NEEDED)
	3796	+ return r;
	3797	+
	3798	+ if (r)
3681	3799	return emulate_gp(ctxt, 0);
3682	3800
3683	3801	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
..	..	@@ -3945,7 +4063,7 @@
3945	4063
3946	4064	eax = reg_read(ctxt, VCPU_REGS_RAX);
3947	4065	ecx = reg_read(ctxt, VCPU_REGS_RCX);
3948		- ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
	4066	+ ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3949	4067	*reg_write(ctxt, VCPU_REGS_RAX) = eax;
3950	4068	*reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3951	4069	*reg_write(ctxt, VCPU_REGS_RCX) = ecx;
..	..	@@ -4008,10 +4126,7 @@
4008	4126
4009	4127	static int check_fxsr(struct x86_emulate_ctxt *ctxt)
4010	4128	{
4011		- u32 eax = 1, ebx, ecx = 0, edx;
4012		-
4013		- ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
4014		- if (!(edx & FFL(FXSR)))
	4129	+ if (!ctxt->ops->guest_has_fxsr(ctxt))
4015	4130	return emulate_ud(ctxt);
4016	4131
4017	4132	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS \| X86_CR0_EM))
..	..	@@ -4073,7 +4188,11 @@
4073	4188	if (rc != X86EMUL_CONTINUE)
4074	4189	return rc;
4075	4190
	4191	+ emulator_get_fpu();
	4192	+
4076	4193	rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
	4194	+
	4195	+ emulator_put_fpu();
4077	4196
4078	4197	if (rc != X86EMUL_CONTINUE)
4079	4198	return rc;
..	..	@@ -4117,6 +4236,8 @@
4117	4236	if (rc != X86EMUL_CONTINUE)
4118	4237	return rc;
4119	4238
	4239	+ emulator_get_fpu();
	4240	+
4120	4241	if (size < __fxstate_size(16)) {
4121	4242	rc = fxregs_fixup(&fx_state, size);
4122	4243	if (rc != X86EMUL_CONTINUE)
..	..	@@ -4132,7 +4253,23 @@
4132	4253	rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4133	4254
4134	4255	out:
	4256	+ emulator_put_fpu();
	4257	+
4135	4258	return rc;
	4259	+}
	4260	+
	4261	+static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
	4262	+{
	4263	+ u32 eax, ecx, edx;
	4264	+
	4265	+ eax = reg_read(ctxt, VCPU_REGS_RAX);
	4266	+ edx = reg_read(ctxt, VCPU_REGS_RDX);
	4267	+ ecx = reg_read(ctxt, VCPU_REGS_RCX);
	4268	+
	4269	+ if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) \| eax))
	4270	+ return emulate_gp(ctxt, 0);
	4271	+
	4272	+ return X86EMUL_CONTINUE;
4136	4273	}
4137	4274
4138	4275	static bool valid_cr(int nr)
..	..	@@ -4147,84 +4284,10 @@
4147	4284	}
4148	4285	}
4149	4286
4150		-static int check_cr_read(struct x86_emulate_ctxt *ctxt)
	4287	+static int check_cr_access(struct x86_emulate_ctxt *ctxt)
4151	4288	{
4152	4289	if (!valid_cr(ctxt->modrm_reg))
4153	4290	return emulate_ud(ctxt);
4154		-
4155		- return X86EMUL_CONTINUE;
4156		-}
4157		-
4158		-static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4159		-{
4160		- u64 new_val = ctxt->src.val64;
4161		- int cr = ctxt->modrm_reg;
4162		- u64 efer = 0;
4163		-
4164		- static u64 cr_reserved_bits[] = {
4165		- 0xffffffff00000000ULL,
4166		- 0, 0, 0, /* CR3 checked later */
4167		- CR4_RESERVED_BITS,
4168		- 0, 0, 0,
4169		- CR8_RESERVED_BITS,
4170		- };
4171		-
4172		- if (!valid_cr(cr))
4173		- return emulate_ud(ctxt);
4174		-
4175		- if (new_val & cr_reserved_bits[cr])
4176		- return emulate_gp(ctxt, 0);
4177		-
4178		- switch (cr) {
4179		- case 0: {
4180		- u64 cr4;
4181		- if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) \|\|
4182		- ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4183		- return emulate_gp(ctxt, 0);
4184		-
4185		- cr4 = ctxt->ops->get_cr(ctxt, 4);
4186		- ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4187		-
4188		- if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4189		- !(cr4 & X86_CR4_PAE))
4190		- return emulate_gp(ctxt, 0);
4191		-
4192		- break;
4193		- }
4194		- case 3: {
4195		- u64 rsvd = 0;
4196		-
4197		- ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4198		- if (efer & EFER_LMA) {
4199		- u64 maxphyaddr;
4200		- u32 eax, ebx, ecx, edx;
4201		-
4202		- eax = 0x80000008;
4203		- ecx = 0;
4204		- if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx,
4205		- &edx, false))
4206		- maxphyaddr = eax & 0xff;
4207		- else
4208		- maxphyaddr = 36;
4209		- rsvd = rsvd_bits(maxphyaddr, 63);
4210		- if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)
4211		- rsvd &= ~X86_CR3_PCID_NOFLUSH;
4212		- }
4213		-
4214		- if (new_val & rsvd)
4215		- return emulate_gp(ctxt, 0);
4216		-
4217		- break;
4218		- }
4219		- case 4: {
4220		- ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4221		-
4222		- if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4223		- return emulate_gp(ctxt, 0);
4224		-
4225		- break;
4226		- }
4227		- }
4228	4291
4229	4292	return X86EMUL_CONTINUE;
4230	4293	}
..	..	@@ -4255,7 +4318,7 @@
4255	4318	ulong dr6;
4256	4319
4257	4320	ctxt->ops->get_dr(ctxt, 6, &dr6);
4258		- dr6 &= ~15;
	4321	+ dr6 &= ~DR_TRAP_BITS;
4259	4322	dr6 \|= DR6_BD \| DR6_RTM;
4260	4323	ctxt->ops->set_dr(ctxt, 6, dr6);
4261	4324	return emulate_db(ctxt);
..	..	@@ -4388,6 +4451,12 @@
4388	4451	N, N, N, N, N, N,
4389	4452	};
4390	4453
	4454	+static const struct opcode group7_rm2[] = {
	4455	+ N,
	4456	+ II(ImplicitOps \| Priv, em_xsetbv, xsetbv),
	4457	+ N, N, N, N, N, N,
	4458	+};
	4459	+
4391	4460	static const struct opcode group7_rm3[] = {
4392	4461	DIP(SrcNone \| Prot \| Priv, vmrun, check_svme_pa),
4393	4462	II(SrcNone \| Prot \| EmulateOnUD, em_hypercall, vmmcall),
..	..	@@ -4477,7 +4546,8 @@
4477	4546	}, {
4478	4547	EXT(0, group7_rm0),
4479	4548	EXT(0, group7_rm1),
4480		- N, EXT(0, group7_rm3),
	4549	+ EXT(0, group7_rm2),
	4550	+ EXT(0, group7_rm3),
4481	4551	II(SrcNone \| DstMem \| Mov, em_smsw, smsw), N,
4482	4552	II(SrcMem16 \| Mov \| Priv, em_lmsw, lmsw),
4483	4553	EXT(0, group7_rm7),
..	..	@@ -4496,7 +4566,7 @@
4496	4566	* from the register case of group9.
4497	4567	*/
4498	4568	static const struct gprefix pfx_0f_c7_7 = {
4499		- N, N, N, II(DstMem \| ModRM \| Op3264 \| EmulateOnUD, em_rdpid, rdtscp),
	4569	+ N, N, N, II(DstMem \| ModRM \| Op3264 \| EmulateOnUD, em_rdpid, rdpid),
4500	4570	};
4501	4571
4502	4572
..	..	@@ -4754,13 +4824,17 @@
4754	4824	GP(ModRM \| DstReg \| SrcMem \| Mov \| Sse, &pfx_0f_10_0f_11),
4755	4825	GP(ModRM \| DstMem \| SrcReg \| Mov \| Sse, &pfx_0f_10_0f_11),
4756	4826	N, N, N, N, N, N,
4757		- D(ImplicitOps \| ModRM \| SrcMem \| NoAccess),
4758		- N, N, N, N, N, N, D(ImplicitOps \| ModRM \| SrcMem \| NoAccess),
	4827	+ D(ImplicitOps \| ModRM \| SrcMem \| NoAccess), /* 4 * prefetch + 4 * reserved NOP */
	4828	+ D(ImplicitOps \| ModRM \| SrcMem \| NoAccess), N, N,
	4829	+ D(ImplicitOps \| ModRM \| SrcMem \| NoAccess), /* 8 * reserved NOP */
	4830	+ D(ImplicitOps \| ModRM \| SrcMem \| NoAccess), /* 8 * reserved NOP */
	4831	+ D(ImplicitOps \| ModRM \| SrcMem \| NoAccess), /* 8 * reserved NOP */
	4832	+ D(ImplicitOps \| ModRM \| SrcMem \| NoAccess), /* NOP + 7 * reserved NOP */
4759	4833	/* 0x20 - 0x2F */
4760		- DIP(ModRM \| DstMem \| Priv \| Op3264 \| NoMod, cr_read, check_cr_read),
	4834	+ DIP(ModRM \| DstMem \| Priv \| Op3264 \| NoMod, cr_read, check_cr_access),
4761	4835	DIP(ModRM \| DstMem \| Priv \| Op3264 \| NoMod, dr_read, check_dr_read),
4762	4836	IIP(ModRM \| SrcMem \| Priv \| Op3264 \| NoMod, em_cr_write, cr_write,
4763		- check_cr_write),
	4837	+ check_cr_access),
4764	4838	IIP(ModRM \| SrcMem \| Priv \| Op3264 \| NoMod, em_dr_write, dr_write,
4765	4839	check_dr_write),
4766	4840	N, N, N, N,
..	..	@@ -5124,7 +5198,7 @@
5124	5198	else {
5125	5199	rc = __do_insn_fetch_bytes(ctxt, 1);
5126	5200	if (rc != X86EMUL_CONTINUE)
5127		- return rc;
	5201	+ goto done;
5128	5202	}
5129	5203
5130	5204	switch (mode) {
..	..	@@ -5426,7 +5500,9 @@
5426	5500	{
5427	5501	int rc;
5428	5502
	5503	+ emulator_get_fpu();
5429	5504	rc = asm_safe("fwait");
	5505	+ emulator_put_fpu();
5430	5506
5431	5507	if (unlikely(rc != X86EMUL_CONTINUE))
5432	5508	return emulate_exception(ctxt, MF_VECTOR, 0, false);
..	..	@@ -5434,14 +5510,13 @@
5434	5510	return X86EMUL_CONTINUE;
5435	5511	}
5436	5512
5437		-static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5438		- struct operand *op)
	5513	+static void fetch_possible_mmx_operand(struct operand *op)
5439	5514	{
5440	5515	if (op->type == OP_MM)
5441		- read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
	5516	+ read_mmx_reg(&op->mm_val, op->addr.mm);
5442	5517	}
5443	5518
5444		-static int fastop(struct x86_emulate_ctxt ctxt, void (fop)(struct fastop *))
	5519	+static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)
5445	5520	{
5446	5521	ulong flags = (ctxt->eflags & EFLAGS_MASK) \| X86_EFLAGS_IF;
5447	5522
..	..	@@ -5517,10 +5592,10 @@
5517	5592	* Now that we know the fpu is exception safe, we can fetch
5518	5593	* operands from it.
5519	5594	*/
5520		- fetch_possible_mmx_operand(ctxt, &ctxt->src);
5521		- fetch_possible_mmx_operand(ctxt, &ctxt->src2);
	5595	+ fetch_possible_mmx_operand(&ctxt->src);
	5596	+ fetch_possible_mmx_operand(&ctxt->src2);
5522	5597	if (!(ctxt->d & Mov))
5523		- fetch_possible_mmx_operand(ctxt, &ctxt->dst);
	5598	+ fetch_possible_mmx_operand(&ctxt->dst);
5524	5599	}
5525	5600
5526	5601	if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
..	..	@@ -5619,14 +5694,10 @@
5619	5694	ctxt->eflags &= ~X86_EFLAGS_RF;
5620	5695
5621	5696	if (ctxt->execute) {
5622		- if (ctxt->d & Fastop) {
5623		- void (fop)(struct fastop ) = (void *)ctxt->execute;
5624		- rc = fastop(ctxt, fop);
5625		- if (rc != X86EMUL_CONTINUE)
5626		- goto done;
5627		- goto writeback;
5628		- }
5629		- rc = ctxt->execute(ctxt);
	5697	+ if (ctxt->d & Fastop)
	5698	+ rc = fastop(ctxt, ctxt->fop);
	5699	+ else
	5700	+ rc = ctxt->execute(ctxt);
5630	5701	if (rc != X86EMUL_CONTINUE)
5631	5702	goto done;
5632	5703	goto writeback;
..	..	@@ -5755,6 +5826,8 @@
5755	5826	}
5756	5827
5757	5828	ctxt->eip = ctxt->_eip;
	5829	+ if (ctxt->mode != X86EMUL_MODE_PROT64)
	5830	+ ctxt->eip = (u32)ctxt->_eip;
5758	5831
5759	5832	done:
5760	5833	if (rc == X86EMUL_PROPAGATE_FAULT) {