| .. | .. |
|---|
| 125 | 125 | * tricks |
|---|
| 126 | 126 | */ |
|---|
| 127 | 127 | cpu_do_idle(); |
|---|
| 128 | + hard_cond_local_irq_enable(); |
|---|
| 128 | 129 | raw_local_irq_enable(); |
|---|
| 129 | 130 | } |
|---|
| 130 | 131 | |
|---|
| .. | .. |
|---|
| 824 | 825 | core_initcall(tagged_addr_init); |
|---|
| 825 | 826 | #endif /* CONFIG_ARM64_TAGGED_ADDR_ABI */ |
|---|
| 826 | 827 | |
|---|
| 828 | +#ifdef CONFIG_IRQ_PIPELINE |
|---|
| 829 | + |
|---|
| 830 | +/* |
|---|
| 831 | + * When pipelining interrupts, we have to reconcile the hardware and |
|---|
| 832 | + * the virtual states. Hard irqs are off on entry while the current |
|---|
| 833 | + * stage has to be unstalled: fix this up by stalling the in-band |
|---|
| 834 | + * stage on entry, unstalling on exit. |
|---|
| 835 | + */ |
|---|
| 836 | +static inline void arm64_preempt_irq_enter(void) |
|---|
| 837 | +{ |
|---|
| 838 | + WARN_ON_ONCE(irq_pipeline_debug() && test_inband_stall()); |
|---|
| 839 | + stall_inband(); |
|---|
| 840 | + trace_hardirqs_off(); |
|---|
| 841 | +} |
|---|
| 842 | + |
|---|
| 843 | +static inline void arm64_preempt_irq_exit(void) |
|---|
| 844 | +{ |
|---|
| 845 | + trace_hardirqs_on(); |
|---|
| 846 | + unstall_inband(); |
|---|
| 847 | +} |
|---|
| 848 | + |
|---|
| 849 | +#else |
|---|
| 850 | + |
|---|
| 851 | +static inline void arm64_preempt_irq_enter(void) |
|---|
| 852 | +{ } |
|---|
| 853 | + |
|---|
| 854 | +static inline void arm64_preempt_irq_exit(void) |
|---|
| 855 | +{ } |
|---|
| 856 | + |
|---|
| 857 | +#endif |
|---|
| 858 | + |
|---|
| 827 | 859 | asmlinkage void __sched arm64_preempt_schedule_irq(void) |
|---|
| 828 | 860 | { |
|---|
| 861 | + arm64_preempt_irq_enter(); |
|---|
| 862 | + |
|---|
| 829 | 863 | lockdep_assert_irqs_disabled(); |
|---|
| 830 | 864 | |
|---|
| 831 | 865 | /* |
|---|
| .. | .. |
|---|
| 838 | 872 | */ |
|---|
| 839 | 873 | if (system_capabilities_finalized()) |
|---|
| 840 | 874 | preempt_schedule_irq(); |
|---|
| 875 | + |
|---|
| 876 | + arm64_preempt_irq_exit(); |
|---|
| 841 | 877 | } |
|---|
| 842 | 878 | |
|---|
| 843 | 879 | #ifdef CONFIG_BINFMT_ELF |
|---|