.. | .. |
---|
5 | 5 | #include <linux/preempt.h> |
---|
6 | 6 | #include <asm/processor.h> |
---|
7 | 7 | #include <asm/cpufeature.h> |
---|
| 8 | +#include <asm/special_insns.h> |
---|
| 9 | + |
---|
| 10 | +#ifdef CONFIG_X86_32 |
---|
| 11 | +static inline void iret_to_self(void) |
---|
| 12 | +{ |
---|
| 13 | + asm volatile ( |
---|
| 14 | + "pushfl\n\t" |
---|
| 15 | + "pushl %%cs\n\t" |
---|
| 16 | + "pushl $1f\n\t" |
---|
| 17 | + "iret\n\t" |
---|
| 18 | + "1:" |
---|
| 19 | + : ASM_CALL_CONSTRAINT : : "memory"); |
---|
| 20 | +} |
---|
| 21 | +#else |
---|
| 22 | +static inline void iret_to_self(void) |
---|
| 23 | +{ |
---|
| 24 | + unsigned int tmp; |
---|
| 25 | + |
---|
| 26 | + asm volatile ( |
---|
| 27 | + "mov %%ss, %0\n\t" |
---|
| 28 | + "pushq %q0\n\t" |
---|
| 29 | + "pushq %%rsp\n\t" |
---|
| 30 | + "addq $8, (%%rsp)\n\t" |
---|
| 31 | + "pushfq\n\t" |
---|
| 32 | + "mov %%cs, %0\n\t" |
---|
| 33 | + "pushq %q0\n\t" |
---|
| 34 | + "pushq $1f\n\t" |
---|
| 35 | + "iretq\n\t" |
---|
| 36 | + "1:" |
---|
| 37 | + : "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory"); |
---|
| 38 | +} |
---|
| 39 | +#endif /* CONFIG_X86_32 */ |
---|
| 40 | + |
---|
| 41 | +/* |
---|
| 42 | + * This function forces the icache and prefetched instruction stream to |
---|
| 43 | + * catch up with reality in two very specific cases: |
---|
| 44 | + * |
---|
| 45 | + * a) Text was modified using one virtual address and is about to be executed |
---|
| 46 | + * from the same physical page at a different virtual address. |
---|
| 47 | + * |
---|
| 48 | + * b) Text was modified on a different CPU, may subsequently be |
---|
| 49 | + * executed on this CPU, and you want to make sure the new version |
---|
| 50 | + * gets executed. This generally means you're calling this in an IPI. |
---|
| 51 | + * |
---|
| 52 | + * If you're calling this for a different reason, you're probably doing |
---|
| 53 | + * it wrong. |
---|
| 54 | + * |
---|
| 55 | + * Like all of Linux's memory ordering operations, this is a |
---|
| 56 | + * compiler barrier as well. |
---|
| 57 | + */ |
---|
| 58 | +static inline void sync_core(void) |
---|
| 59 | +{ |
---|
| 60 | + /* |
---|
| 61 | + * The SERIALIZE instruction is the most straightforward way to |
---|
| 62 | + * do this, but it is not universally available. |
---|
| 63 | + */ |
---|
| 64 | + if (static_cpu_has(X86_FEATURE_SERIALIZE)) { |
---|
| 65 | + serialize(); |
---|
| 66 | + return; |
---|
| 67 | + } |
---|
| 68 | + |
---|
| 69 | + /* |
---|
| 70 | + * For all other processors, there are quite a few ways to do this. |
---|
| 71 | + * IRET-to-self is nice because it works on every CPU, at any CPL |
---|
| 72 | + * (so it's compatible with paravirtualization), and it never exits |
---|
| 73 | + * to a hypervisor. The only downsides are that it's a bit slow |
---|
| 74 | + * (it seems to be a bit more than 2x slower than the fastest |
---|
| 75 | + * options) and that it unmasks NMIs. The "push %cs" is needed, |
---|
| 76 | + * because in paravirtual environments __KERNEL_CS may not be a |
---|
| 77 | + * valid CS value when we do IRET directly. |
---|
| 78 | + * |
---|
| 79 | + * In case NMI unmasking or performance ever becomes a problem, |
---|
| 80 | + * the next best option appears to be MOV-to-CR2 and an |
---|
| 81 | + * unconditional jump. That sequence also works on all CPUs, |
---|
| 82 | + * but it will fault at CPL3 (i.e. Xen PV). |
---|
| 83 | + * |
---|
| 84 | + * CPUID is the conventional way, but it's nasty: it doesn't |
---|
| 85 | + * exist on some 486-like CPUs, and it usually exits to a |
---|
| 86 | + * hypervisor. |
---|
| 87 | + */ |
---|
| 88 | + iret_to_self(); |
---|
| 89 | +} |
---|
8 | 90 | |
---|
9 | 91 | /* |
---|
10 | 92 | * Ensure that a core serializing instruction is issued before returning |
---|