.. | .. |
---|
9 | 9 | #include <asm/nospec-branch.h> |
---|
10 | 10 | |
---|
11 | 11 | /* Provide __cpuidle; we can't safely include <linux/cpu.h> */ |
---|
12 | | -#define __cpuidle __attribute__((__section__(".cpuidle.text"))) |
---|
| 12 | +#define __cpuidle __section(".cpuidle.text") |
---|
13 | 13 | |
---|
14 | 14 | /* |
---|
15 | 15 | * Interrupt control: |
---|
.. | .. |
---|
17 | 17 | |
---|
18 | 18 | /* Declaration required for gcc < 4.9 to prevent -Werror=missing-prototypes */ |
---|
19 | 19 | extern inline unsigned long native_save_fl(void); |
---|
20 | | -extern inline unsigned long native_save_fl(void) |
---|
| 20 | +extern __always_inline unsigned long native_save_fl(void) |
---|
21 | 21 | { |
---|
22 | 22 | unsigned long flags; |
---|
23 | 23 | |
---|
.. | .. |
---|
44 | 44 | :"memory", "cc"); |
---|
45 | 45 | } |
---|
46 | 46 | |
---|
47 | | -static inline void native_irq_disable(void) |
---|
| 47 | +static __always_inline void native_irq_disable(void) |
---|
48 | 48 | { |
---|
49 | 49 | asm volatile("cli": : :"memory"); |
---|
50 | 50 | } |
---|
51 | 51 | |
---|
52 | | -static inline void native_irq_enable(void) |
---|
| 52 | +static __always_inline void native_irq_enable(void) |
---|
53 | 53 | { |
---|
54 | 54 | asm volatile("sti": : :"memory"); |
---|
55 | 55 | } |
---|
.. | .. |
---|
68 | 68 | |
---|
69 | 69 | #endif |
---|
70 | 70 | |
---|
71 | | -#ifdef CONFIG_PARAVIRT |
---|
| 71 | +#ifdef CONFIG_PARAVIRT_XXL |
---|
72 | 72 | #include <asm/paravirt.h> |
---|
73 | 73 | #else |
---|
74 | 74 | #ifndef __ASSEMBLY__ |
---|
75 | 75 | #include <linux/types.h> |
---|
76 | 76 | |
---|
77 | | -static inline notrace unsigned long arch_local_save_flags(void) |
---|
| 77 | +static __always_inline unsigned long arch_local_save_flags(void) |
---|
78 | 78 | { |
---|
79 | 79 | return native_save_fl(); |
---|
80 | 80 | } |
---|
81 | 81 | |
---|
82 | | -static inline notrace void arch_local_irq_restore(unsigned long flags) |
---|
| 82 | +static __always_inline void arch_local_irq_restore(unsigned long flags) |
---|
83 | 83 | { |
---|
84 | 84 | native_restore_fl(flags); |
---|
85 | 85 | } |
---|
86 | 86 | |
---|
87 | | -static inline notrace void arch_local_irq_disable(void) |
---|
| 87 | +static __always_inline void arch_local_irq_disable(void) |
---|
88 | 88 | { |
---|
89 | 89 | native_irq_disable(); |
---|
90 | 90 | } |
---|
91 | 91 | |
---|
92 | | -static inline notrace void arch_local_irq_enable(void) |
---|
| 92 | +static __always_inline void arch_local_irq_enable(void) |
---|
93 | 93 | { |
---|
94 | 94 | native_irq_enable(); |
---|
95 | 95 | } |
---|
.. | .. |
---|
115 | 115 | /* |
---|
116 | 116 | * For spinlocks, etc: |
---|
117 | 117 | */ |
---|
118 | | -static inline notrace unsigned long arch_local_irq_save(void) |
---|
| 118 | +static __always_inline unsigned long arch_local_irq_save(void) |
---|
119 | 119 | { |
---|
120 | 120 | unsigned long flags = arch_local_save_flags(); |
---|
121 | 121 | arch_local_irq_disable(); |
---|
.. | .. |
---|
127 | 127 | #define DISABLE_INTERRUPTS(x) cli |
---|
128 | 128 | |
---|
129 | 129 | #ifdef CONFIG_X86_64 |
---|
130 | | -#define SWAPGS swapgs |
---|
131 | | -/* |
---|
132 | | - * Currently paravirt can't handle swapgs nicely when we |
---|
133 | | - * don't have a stack we can rely on (such as a user space |
---|
134 | | - * stack). So we either find a way around these or just fault |
---|
135 | | - * and emulate if a guest tries to call swapgs directly. |
---|
136 | | - * |
---|
137 | | - * Either way, this is a good way to document that we don't |
---|
138 | | - * have a reliable stack. x86_64 only. |
---|
139 | | - */ |
---|
140 | | -#define SWAPGS_UNSAFE_STACK swapgs |
---|
141 | | - |
---|
142 | | -#define PARAVIRT_ADJUST_EXCEPTION_FRAME /* */ |
---|
| 130 | +#ifdef CONFIG_DEBUG_ENTRY |
---|
| 131 | +#define SAVE_FLAGS(x) pushfq; popq %rax |
---|
| 132 | +#endif |
---|
143 | 133 | |
---|
144 | 134 | #define INTERRUPT_RETURN jmp native_iret |
---|
145 | 135 | #define USERGS_SYSRET64 \ |
---|
.. | .. |
---|
149 | 139 | swapgs; \ |
---|
150 | 140 | sysretl |
---|
151 | 141 | |
---|
152 | | -#ifdef CONFIG_DEBUG_ENTRY |
---|
153 | | -#define SAVE_FLAGS(x) pushfq; popq %rax |
---|
154 | | -#endif |
---|
155 | 142 | #else |
---|
156 | 143 | #define INTERRUPT_RETURN iret |
---|
157 | | -#define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit |
---|
158 | | -#define GET_CR0_INTO_EAX movl %cr0, %eax |
---|
159 | 144 | #endif |
---|
160 | 145 | |
---|
161 | | - |
---|
162 | 146 | #endif /* __ASSEMBLY__ */ |
---|
163 | | -#endif /* CONFIG_PARAVIRT */ |
---|
| 147 | +#endif /* CONFIG_PARAVIRT_XXL */ |
---|
164 | 148 | |
---|
165 | 149 | #ifndef __ASSEMBLY__ |
---|
166 | | -static inline int arch_irqs_disabled_flags(unsigned long flags) |
---|
| 150 | +static __always_inline int arch_irqs_disabled_flags(unsigned long flags) |
---|
167 | 151 | { |
---|
168 | 152 | return !(flags & X86_EFLAGS_IF); |
---|
169 | 153 | } |
---|
170 | 154 | |
---|
171 | | -static inline int arch_irqs_disabled(void) |
---|
| 155 | +static __always_inline int arch_irqs_disabled(void) |
---|
172 | 156 | { |
---|
173 | 157 | unsigned long flags = arch_local_save_flags(); |
---|
174 | 158 | |
---|
175 | 159 | return arch_irqs_disabled_flags(flags); |
---|
176 | 160 | } |
---|
| 161 | +#else |
---|
| 162 | +#ifdef CONFIG_X86_64 |
---|
| 163 | +#ifdef CONFIG_XEN_PV |
---|
| 164 | +#define SWAPGS ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV |
---|
| 165 | +#else |
---|
| 166 | +#define SWAPGS swapgs |
---|
| 167 | +#endif |
---|
| 168 | +#endif |
---|
177 | 169 | #endif /* !__ASSEMBLY__ */ |
---|
178 | | - |
---|
179 | | -#ifdef __ASSEMBLY__ |
---|
180 | | -#ifdef CONFIG_TRACE_IRQFLAGS |
---|
181 | | -# define TRACE_IRQS_ON call trace_hardirqs_on_thunk; |
---|
182 | | -# define TRACE_IRQS_OFF call trace_hardirqs_off_thunk; |
---|
183 | | -#else |
---|
184 | | -# define TRACE_IRQS_ON |
---|
185 | | -# define TRACE_IRQS_OFF |
---|
186 | | -#endif |
---|
187 | | -#ifdef CONFIG_DEBUG_LOCK_ALLOC |
---|
188 | | -# ifdef CONFIG_X86_64 |
---|
189 | | -# define LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk |
---|
190 | | -# define LOCKDEP_SYS_EXIT_IRQ \ |
---|
191 | | - TRACE_IRQS_ON; \ |
---|
192 | | - sti; \ |
---|
193 | | - call lockdep_sys_exit_thunk; \ |
---|
194 | | - cli; \ |
---|
195 | | - TRACE_IRQS_OFF; |
---|
196 | | -# else |
---|
197 | | -# define LOCKDEP_SYS_EXIT \ |
---|
198 | | - pushl %eax; \ |
---|
199 | | - pushl %ecx; \ |
---|
200 | | - pushl %edx; \ |
---|
201 | | - call lockdep_sys_exit; \ |
---|
202 | | - popl %edx; \ |
---|
203 | | - popl %ecx; \ |
---|
204 | | - popl %eax; |
---|
205 | | -# define LOCKDEP_SYS_EXIT_IRQ |
---|
206 | | -# endif |
---|
207 | | -#else |
---|
208 | | -# define LOCKDEP_SYS_EXIT |
---|
209 | | -# define LOCKDEP_SYS_EXIT_IRQ |
---|
210 | | -#endif |
---|
211 | | -#endif /* __ASSEMBLY__ */ |
---|
212 | 170 | |
---|
213 | 171 | #endif |
---|