| .. | .. |
|---|
| 9 | 9 | #include <asm/nospec-branch.h> |
|---|
| 10 | 10 | |
|---|
| 11 | 11 | /* Provide __cpuidle; we can't safely include <linux/cpu.h> */ |
|---|
| 12 | | -#define __cpuidle __attribute__((__section__(".cpuidle.text"))) |
|---|
| 12 | +#define __cpuidle __section(".cpuidle.text") |
|---|
| 13 | 13 | |
|---|
| 14 | 14 | /* |
|---|
| 15 | 15 | * Interrupt control: |
|---|
| .. | .. |
|---|
| 17 | 17 | |
|---|
| 18 | 18 | /* Declaration required for gcc < 4.9 to prevent -Werror=missing-prototypes */ |
|---|
| 19 | 19 | extern inline unsigned long native_save_fl(void); |
|---|
| 20 | | -extern inline unsigned long native_save_fl(void) |
|---|
| 20 | +extern __always_inline unsigned long native_save_fl(void) |
|---|
| 21 | 21 | { |
|---|
| 22 | 22 | unsigned long flags; |
|---|
| 23 | 23 | |
|---|
| .. | .. |
|---|
| 44 | 44 | :"memory", "cc"); |
|---|
| 45 | 45 | } |
|---|
| 46 | 46 | |
|---|
| 47 | | -static inline void native_irq_disable(void) |
|---|
| 47 | +static __always_inline void native_irq_disable(void) |
|---|
| 48 | 48 | { |
|---|
| 49 | 49 | asm volatile("cli": : :"memory"); |
|---|
| 50 | 50 | } |
|---|
| 51 | 51 | |
|---|
| 52 | | -static inline void native_irq_enable(void) |
|---|
| 52 | +static __always_inline void native_irq_enable(void) |
|---|
| 53 | 53 | { |
|---|
| 54 | 54 | asm volatile("sti": : :"memory"); |
|---|
| 55 | 55 | } |
|---|
| .. | .. |
|---|
| 68 | 68 | |
|---|
| 69 | 69 | #endif |
|---|
| 70 | 70 | |
|---|
| 71 | | -#ifdef CONFIG_PARAVIRT |
|---|
| 71 | +#ifdef CONFIG_PARAVIRT_XXL |
|---|
| 72 | 72 | #include <asm/paravirt.h> |
|---|
| 73 | 73 | #else |
|---|
| 74 | 74 | #ifndef __ASSEMBLY__ |
|---|
| 75 | 75 | #include <linux/types.h> |
|---|
| 76 | 76 | |
|---|
| 77 | | -static inline notrace unsigned long arch_local_save_flags(void) |
|---|
| 77 | +static __always_inline unsigned long arch_local_save_flags(void) |
|---|
| 78 | 78 | { |
|---|
| 79 | 79 | return native_save_fl(); |
|---|
| 80 | 80 | } |
|---|
| 81 | 81 | |
|---|
| 82 | | -static inline notrace void arch_local_irq_restore(unsigned long flags) |
|---|
| 82 | +static __always_inline void arch_local_irq_restore(unsigned long flags) |
|---|
| 83 | 83 | { |
|---|
| 84 | 84 | native_restore_fl(flags); |
|---|
| 85 | 85 | } |
|---|
| 86 | 86 | |
|---|
| 87 | | -static inline notrace void arch_local_irq_disable(void) |
|---|
| 87 | +static __always_inline void arch_local_irq_disable(void) |
|---|
| 88 | 88 | { |
|---|
| 89 | 89 | native_irq_disable(); |
|---|
| 90 | 90 | } |
|---|
| 91 | 91 | |
|---|
| 92 | | -static inline notrace void arch_local_irq_enable(void) |
|---|
| 92 | +static __always_inline void arch_local_irq_enable(void) |
|---|
| 93 | 93 | { |
|---|
| 94 | 94 | native_irq_enable(); |
|---|
| 95 | 95 | } |
|---|
| .. | .. |
|---|
| 115 | 115 | /* |
|---|
| 116 | 116 | * For spinlocks, etc: |
|---|
| 117 | 117 | */ |
|---|
| 118 | | -static inline notrace unsigned long arch_local_irq_save(void) |
|---|
| 118 | +static __always_inline unsigned long arch_local_irq_save(void) |
|---|
| 119 | 119 | { |
|---|
| 120 | 120 | unsigned long flags = arch_local_save_flags(); |
|---|
| 121 | 121 | arch_local_irq_disable(); |
|---|
| .. | .. |
|---|
| 127 | 127 | #define DISABLE_INTERRUPTS(x) cli |
|---|
| 128 | 128 | |
|---|
| 129 | 129 | #ifdef CONFIG_X86_64 |
|---|
| 130 | | -#define SWAPGS swapgs |
|---|
| 131 | | -/* |
|---|
| 132 | | - * Currently paravirt can't handle swapgs nicely when we |
|---|
| 133 | | - * don't have a stack we can rely on (such as a user space |
|---|
| 134 | | - * stack). So we either find a way around these or just fault |
|---|
| 135 | | - * and emulate if a guest tries to call swapgs directly. |
|---|
| 136 | | - * |
|---|
| 137 | | - * Either way, this is a good way to document that we don't |
|---|
| 138 | | - * have a reliable stack. x86_64 only. |
|---|
| 139 | | - */ |
|---|
| 140 | | -#define SWAPGS_UNSAFE_STACK swapgs |
|---|
| 141 | | - |
|---|
| 142 | | -#define PARAVIRT_ADJUST_EXCEPTION_FRAME /* */ |
|---|
| 130 | +#ifdef CONFIG_DEBUG_ENTRY |
|---|
| 131 | +#define SAVE_FLAGS(x) pushfq; popq %rax |
|---|
| 132 | +#endif |
|---|
| 143 | 133 | |
|---|
| 144 | 134 | #define INTERRUPT_RETURN jmp native_iret |
|---|
| 145 | 135 | #define USERGS_SYSRET64 \ |
|---|
| .. | .. |
|---|
| 149 | 139 | swapgs; \ |
|---|
| 150 | 140 | sysretl |
|---|
| 151 | 141 | |
|---|
| 152 | | -#ifdef CONFIG_DEBUG_ENTRY |
|---|
| 153 | | -#define SAVE_FLAGS(x) pushfq; popq %rax |
|---|
| 154 | | -#endif |
|---|
| 155 | 142 | #else |
|---|
| 156 | 143 | #define INTERRUPT_RETURN iret |
|---|
| 157 | | -#define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit |
|---|
| 158 | | -#define GET_CR0_INTO_EAX movl %cr0, %eax |
|---|
| 159 | 144 | #endif |
|---|
| 160 | 145 | |
|---|
| 161 | | - |
|---|
| 162 | 146 | #endif /* __ASSEMBLY__ */ |
|---|
| 163 | | -#endif /* CONFIG_PARAVIRT */ |
|---|
| 147 | +#endif /* CONFIG_PARAVIRT_XXL */ |
|---|
| 164 | 148 | |
|---|
| 165 | 149 | #ifndef __ASSEMBLY__ |
|---|
| 166 | | -static inline int arch_irqs_disabled_flags(unsigned long flags) |
|---|
| 150 | +static __always_inline int arch_irqs_disabled_flags(unsigned long flags) |
|---|
| 167 | 151 | { |
|---|
| 168 | 152 | return !(flags & X86_EFLAGS_IF); |
|---|
| 169 | 153 | } |
|---|
| 170 | 154 | |
|---|
| 171 | | -static inline int arch_irqs_disabled(void) |
|---|
| 155 | +static __always_inline int arch_irqs_disabled(void) |
|---|
| 172 | 156 | { |
|---|
| 173 | 157 | unsigned long flags = arch_local_save_flags(); |
|---|
| 174 | 158 | |
|---|
| 175 | 159 | return arch_irqs_disabled_flags(flags); |
|---|
| 176 | 160 | } |
|---|
| 161 | +#else |
|---|
| 162 | +#ifdef CONFIG_X86_64 |
|---|
| 163 | +#ifdef CONFIG_XEN_PV |
|---|
| 164 | +#define SWAPGS ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV |
|---|
| 165 | +#else |
|---|
| 166 | +#define SWAPGS swapgs |
|---|
| 167 | +#endif |
|---|
| 168 | +#endif |
|---|
| 177 | 169 | #endif /* !__ASSEMBLY__ */ |
|---|
| 178 | | - |
|---|
| 179 | | -#ifdef __ASSEMBLY__ |
|---|
| 180 | | -#ifdef CONFIG_TRACE_IRQFLAGS |
|---|
| 181 | | -# define TRACE_IRQS_ON call trace_hardirqs_on_thunk; |
|---|
| 182 | | -# define TRACE_IRQS_OFF call trace_hardirqs_off_thunk; |
|---|
| 183 | | -#else |
|---|
| 184 | | -# define TRACE_IRQS_ON |
|---|
| 185 | | -# define TRACE_IRQS_OFF |
|---|
| 186 | | -#endif |
|---|
| 187 | | -#ifdef CONFIG_DEBUG_LOCK_ALLOC |
|---|
| 188 | | -# ifdef CONFIG_X86_64 |
|---|
| 189 | | -# define LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk |
|---|
| 190 | | -# define LOCKDEP_SYS_EXIT_IRQ \ |
|---|
| 191 | | - TRACE_IRQS_ON; \ |
|---|
| 192 | | - sti; \ |
|---|
| 193 | | - call lockdep_sys_exit_thunk; \ |
|---|
| 194 | | - cli; \ |
|---|
| 195 | | - TRACE_IRQS_OFF; |
|---|
| 196 | | -# else |
|---|
| 197 | | -# define LOCKDEP_SYS_EXIT \ |
|---|
| 198 | | - pushl %eax; \ |
|---|
| 199 | | - pushl %ecx; \ |
|---|
| 200 | | - pushl %edx; \ |
|---|
| 201 | | - call lockdep_sys_exit; \ |
|---|
| 202 | | - popl %edx; \ |
|---|
| 203 | | - popl %ecx; \ |
|---|
| 204 | | - popl %eax; |
|---|
| 205 | | -# define LOCKDEP_SYS_EXIT_IRQ |
|---|
| 206 | | -# endif |
|---|
| 207 | | -#else |
|---|
| 208 | | -# define LOCKDEP_SYS_EXIT |
|---|
| 209 | | -# define LOCKDEP_SYS_EXIT_IRQ |
|---|
| 210 | | -#endif |
|---|
| 211 | | -#endif /* __ASSEMBLY__ */ |
|---|
| 212 | 170 | |
|---|
| 213 | 171 | #endif |
|---|