| .. | .. | 
|---|
| 5 | 5 |  #include <linux/preempt.h> | 
|---|
| 6 | 6 |  #include <asm/processor.h> | 
|---|
| 7 | 7 |  #include <asm/cpufeature.h> | 
|---|
 | 8 | +#include <asm/special_insns.h>  | 
|---|
 | 9 | +  | 
|---|
 | 10 | +#ifdef CONFIG_X86_32  | 
|---|
 | 11 | +static inline void iret_to_self(void)  | 
|---|
 | 12 | +{  | 
|---|
 | 13 | +	asm volatile (  | 
|---|
 | 14 | +		"pushfl\n\t"  | 
|---|
 | 15 | +		"pushl %%cs\n\t"  | 
|---|
 | 16 | +		"pushl $1f\n\t"  | 
|---|
 | 17 | +		"iret\n\t"  | 
|---|
 | 18 | +		"1:"  | 
|---|
 | 19 | +		: ASM_CALL_CONSTRAINT : : "memory");  | 
|---|
 | 20 | +}  | 
|---|
 | 21 | +#else  | 
|---|
 | 22 | +static inline void iret_to_self(void)  | 
|---|
 | 23 | +{  | 
|---|
 | 24 | +	unsigned int tmp;  | 
|---|
 | 25 | +  | 
|---|
 | 26 | +	asm volatile (  | 
|---|
 | 27 | +		"mov %%ss, %0\n\t"  | 
|---|
 | 28 | +		"pushq %q0\n\t"  | 
|---|
 | 29 | +		"pushq %%rsp\n\t"  | 
|---|
 | 30 | +		"addq $8, (%%rsp)\n\t"  | 
|---|
 | 31 | +		"pushfq\n\t"  | 
|---|
 | 32 | +		"mov %%cs, %0\n\t"  | 
|---|
 | 33 | +		"pushq %q0\n\t"  | 
|---|
 | 34 | +		"pushq $1f\n\t"  | 
|---|
 | 35 | +		"iretq\n\t"  | 
|---|
 | 36 | +		"1:"  | 
|---|
 | 37 | +		: "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");  | 
|---|
 | 38 | +}  | 
|---|
 | 39 | +#endif /* CONFIG_X86_32 */  | 
|---|
 | 40 | +  | 
|---|
 | 41 | +/*  | 
|---|
 | 42 | + * This function forces the icache and prefetched instruction stream to  | 
|---|
 | 43 | + * catch up with reality in two very specific cases:  | 
|---|
 | 44 | + *  | 
|---|
 | 45 | + *  a) Text was modified using one virtual address and is about to be executed  | 
|---|
 | 46 | + *     from the same physical page at a different virtual address.  | 
|---|
 | 47 | + *  | 
|---|
 | 48 | + *  b) Text was modified on a different CPU, may subsequently be  | 
|---|
 | 49 | + *     executed on this CPU, and you want to make sure the new version  | 
|---|
 | 50 | + *     gets executed.  This generally means you're calling this in an IPI.  | 
|---|
 | 51 | + *  | 
|---|
 | 52 | + * If you're calling this for a different reason, you're probably doing  | 
|---|
 | 53 | + * it wrong.  | 
|---|
 | 54 | + *  | 
|---|
 | 55 | + * Like all of Linux's memory ordering operations, this is a  | 
|---|
 | 56 | + * compiler barrier as well.  | 
|---|
 | 57 | + */  | 
|---|
 | 58 | +static inline void sync_core(void)  | 
|---|
 | 59 | +{  | 
|---|
 | 60 | +	/*  | 
|---|
 | 61 | +	 * The SERIALIZE instruction is the most straightforward way to  | 
|---|
 | 62 | +	 * do this, but it is not universally available.  | 
|---|
 | 63 | +	 */  | 
|---|
 | 64 | +	if (static_cpu_has(X86_FEATURE_SERIALIZE)) {  | 
|---|
 | 65 | +		serialize();  | 
|---|
 | 66 | +		return;  | 
|---|
 | 67 | +	}  | 
|---|
 | 68 | +  | 
|---|
 | 69 | +	/*  | 
|---|
 | 70 | +	 * For all other processors, there are quite a few ways to do this.  | 
|---|
 | 71 | +	 * IRET-to-self is nice because it works on every CPU, at any CPL  | 
|---|
 | 72 | +	 * (so it's compatible with paravirtualization), and it never exits  | 
|---|
 | 73 | +	 * to a hypervisor.  The only downsides are that it's a bit slow  | 
|---|
 | 74 | +	 * (it seems to be a bit more than 2x slower than the fastest  | 
|---|
 | 75 | +	 * options) and that it unmasks NMIs.  The "push %cs" is needed,  | 
|---|
 | 76 | +	 * because in paravirtual environments __KERNEL_CS may not be a  | 
|---|
 | 77 | +	 * valid CS value when we do IRET directly.  | 
|---|
 | 78 | +	 *  | 
|---|
 | 79 | +	 * In case NMI unmasking or performance ever becomes a problem,  | 
|---|
 | 80 | +	 * the next best option appears to be MOV-to-CR2 and an  | 
|---|
 | 81 | +	 * unconditional jump.  That sequence also works on all CPUs,  | 
|---|
 | 82 | +	 * but it will fault at CPL3 (i.e. Xen PV).  | 
|---|
 | 83 | +	 *  | 
|---|
 | 84 | +	 * CPUID is the conventional way, but it's nasty: it doesn't  | 
|---|
 | 85 | +	 * exist on some 486-like CPUs, and it usually exits to a  | 
|---|
 | 86 | +	 * hypervisor.  | 
|---|
 | 87 | +	 */  | 
|---|
 | 88 | +	iret_to_self();  | 
|---|
 | 89 | +}  | 
|---|
| 8 | 90 |   | 
|---|
| 9 | 91 |  /* | 
|---|
| 10 | 92 |   * Ensure that a core serializing instruction is issued before returning | 
|---|