.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* |
---|
2 | 3 | * Common interrupt code for 32 and 64 bit |
---|
3 | 4 | */ |
---|
.. | .. |
---|
12 | 13 | #include <linux/export.h> |
---|
13 | 14 | #include <linux/irq.h> |
---|
14 | 15 | |
---|
| 16 | +#include <asm/irq_stack.h> |
---|
15 | 17 | #include <asm/apic.h> |
---|
16 | 18 | #include <asm/io_apic.h> |
---|
17 | 19 | #include <asm/irq.h> |
---|
18 | 20 | #include <asm/mce.h> |
---|
19 | 21 | #include <asm/hw_irq.h> |
---|
20 | 22 | #include <asm/desc.h> |
---|
| 23 | +#include <asm/traps.h> |
---|
21 | 24 | |
---|
22 | 25 | #define CREATE_TRACE_POINTS |
---|
23 | 26 | #include <asm/trace/irq_vectors.h> |
---|
24 | 27 | |
---|
25 | 28 | DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); |
---|
26 | 29 | EXPORT_PER_CPU_SYMBOL(irq_stat); |
---|
27 | | - |
---|
28 | | -DEFINE_PER_CPU(struct pt_regs *, irq_regs); |
---|
29 | | -EXPORT_PER_CPU_SYMBOL(irq_regs); |
---|
30 | 30 | |
---|
31 | 31 | atomic_t irq_err_count; |
---|
32 | 32 | |
---|
.. | .. |
---|
134 | 134 | seq_printf(p, "%10u ", per_cpu(mce_poll_count, j)); |
---|
135 | 135 | seq_puts(p, " Machine check polls\n"); |
---|
136 | 136 | #endif |
---|
137 | | -#if IS_ENABLED(CONFIG_HYPERV) || defined(CONFIG_XEN) |
---|
| 137 | +#ifdef CONFIG_X86_HV_CALLBACK_VECTOR |
---|
138 | 138 | if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) { |
---|
139 | 139 | seq_printf(p, "%*s: ", prec, "HYP"); |
---|
140 | 140 | for_each_online_cpu(j) |
---|
.. | .. |
---|
223 | 223 | return sum; |
---|
224 | 224 | } |
---|
225 | 225 | |
---|
| 226 | +static __always_inline void handle_irq(struct irq_desc *desc, |
---|
| 227 | + struct pt_regs *regs) |
---|
| 228 | +{ |
---|
| 229 | + if (IS_ENABLED(CONFIG_X86_64)) |
---|
| 230 | + run_irq_on_irqstack_cond(desc->handle_irq, desc, regs); |
---|
| 231 | + else |
---|
| 232 | + __handle_irq(desc, regs); |
---|
| 233 | +} |
---|
226 | 234 | |
---|
227 | 235 | /* |
---|
228 | | - * do_IRQ handles all normal device IRQ's (the special |
---|
229 | | - * SMP cross-CPU interrupts have their own specific |
---|
230 | | - * handlers). |
---|
| 236 | + * common_interrupt() handles all normal device IRQ's (the special SMP |
---|
| 237 | + * cross-CPU interrupts have their own entry points). |
---|
231 | 238 | */ |
---|
232 | | -__visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs) |
---|
| 239 | +DEFINE_IDTENTRY_IRQ(common_interrupt) |
---|
233 | 240 | { |
---|
234 | 241 | struct pt_regs *old_regs = set_irq_regs(regs); |
---|
235 | | - struct irq_desc * desc; |
---|
236 | | - /* high bit used in ret_from_ code */ |
---|
237 | | - unsigned vector = ~regs->orig_ax; |
---|
| 242 | + struct irq_desc *desc; |
---|
238 | 243 | |
---|
239 | | - entering_irq(); |
---|
240 | | - |
---|
241 | | - /* entering_irq() tells RCU that we're not quiescent. Check it. */ |
---|
| 244 | + /* entry code tells RCU that we're not quiescent. Check it. */ |
---|
242 | 245 | RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU"); |
---|
243 | 246 | |
---|
244 | 247 | desc = __this_cpu_read(vector_irq[vector]); |
---|
245 | | - |
---|
246 | | - if (!handle_irq(desc, regs)) { |
---|
| 248 | + if (likely(!IS_ERR_OR_NULL(desc))) { |
---|
| 249 | + handle_irq(desc, regs); |
---|
| 250 | + } else { |
---|
247 | 251 | ack_APIC_irq(); |
---|
248 | 252 | |
---|
249 | | - if (desc != VECTOR_RETRIGGERED && desc != VECTOR_SHUTDOWN) { |
---|
250 | | - pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n", |
---|
| 253 | + if (desc == VECTOR_UNUSED) { |
---|
| 254 | + pr_emerg_ratelimited("%s: %d.%u No irq handler for vector\n", |
---|
251 | 255 | __func__, smp_processor_id(), |
---|
252 | 256 | vector); |
---|
253 | 257 | } else { |
---|
.. | .. |
---|
255 | 259 | } |
---|
256 | 260 | } |
---|
257 | 261 | |
---|
258 | | - exiting_irq(); |
---|
259 | | - |
---|
260 | 262 | set_irq_regs(old_regs); |
---|
261 | | - return 1; |
---|
262 | 263 | } |
---|
263 | 264 | |
---|
264 | 265 | #ifdef CONFIG_X86_LOCAL_APIC |
---|
.. | .. |
---|
267 | 268 | /* |
---|
268 | 269 | * Handler for X86_PLATFORM_IPI_VECTOR. |
---|
269 | 270 | */ |
---|
270 | | -__visible void __irq_entry smp_x86_platform_ipi(struct pt_regs *regs) |
---|
| 271 | +DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi) |
---|
271 | 272 | { |
---|
272 | 273 | struct pt_regs *old_regs = set_irq_regs(regs); |
---|
273 | 274 | |
---|
274 | | - entering_ack_irq(); |
---|
| 275 | + ack_APIC_irq(); |
---|
275 | 276 | trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR); |
---|
276 | 277 | inc_irq_stat(x86_platform_ipis); |
---|
277 | 278 | if (x86_platform_ipi_callback) |
---|
278 | 279 | x86_platform_ipi_callback(); |
---|
279 | 280 | trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR); |
---|
280 | | - exiting_irq(); |
---|
281 | 281 | set_irq_regs(old_regs); |
---|
282 | 282 | } |
---|
283 | 283 | #endif |
---|
.. | .. |
---|
300 | 300 | /* |
---|
301 | 301 | * Handler for POSTED_INTERRUPT_VECTOR. |
---|
302 | 302 | */ |
---|
303 | | -__visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs) |
---|
| 303 | +DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi) |
---|
304 | 304 | { |
---|
305 | | - struct pt_regs *old_regs = set_irq_regs(regs); |
---|
306 | | - |
---|
307 | | - entering_ack_irq(); |
---|
| 305 | + ack_APIC_irq(); |
---|
308 | 306 | inc_irq_stat(kvm_posted_intr_ipis); |
---|
309 | | - exiting_irq(); |
---|
310 | | - set_irq_regs(old_regs); |
---|
311 | 307 | } |
---|
312 | 308 | |
---|
313 | 309 | /* |
---|
314 | 310 | * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR. |
---|
315 | 311 | */ |
---|
316 | | -__visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs) |
---|
| 312 | +DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi) |
---|
317 | 313 | { |
---|
318 | | - struct pt_regs *old_regs = set_irq_regs(regs); |
---|
319 | | - |
---|
320 | | - entering_ack_irq(); |
---|
| 314 | + ack_APIC_irq(); |
---|
321 | 315 | inc_irq_stat(kvm_posted_intr_wakeup_ipis); |
---|
322 | 316 | kvm_posted_intr_wakeup_handler(); |
---|
323 | | - exiting_irq(); |
---|
324 | | - set_irq_regs(old_regs); |
---|
325 | 317 | } |
---|
326 | 318 | |
---|
327 | 319 | /* |
---|
328 | 320 | * Handler for POSTED_INTERRUPT_NESTED_VECTOR. |
---|
329 | 321 | */ |
---|
330 | | -__visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs) |
---|
| 322 | +DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_nested_ipi) |
---|
331 | 323 | { |
---|
332 | | - struct pt_regs *old_regs = set_irq_regs(regs); |
---|
333 | | - |
---|
334 | | - entering_ack_irq(); |
---|
| 324 | + ack_APIC_irq(); |
---|
335 | 325 | inc_irq_stat(kvm_posted_intr_nested_ipis); |
---|
336 | | - exiting_irq(); |
---|
337 | | - set_irq_regs(old_regs); |
---|
338 | 326 | } |
---|
339 | 327 | #endif |
---|
340 | 328 | |
---|