.. | .. |
---|
125 | 125 | * tricks |
---|
126 | 126 | */ |
---|
127 | 127 | cpu_do_idle(); |
---|
| 128 | + hard_cond_local_irq_enable(); |
---|
128 | 129 | raw_local_irq_enable(); |
---|
129 | 130 | } |
---|
130 | 131 | |
---|
.. | .. |
---|
824 | 825 | core_initcall(tagged_addr_init); |
---|
825 | 826 | #endif /* CONFIG_ARM64_TAGGED_ADDR_ABI */ |
---|
826 | 827 | |
---|
| 828 | +#ifdef CONFIG_IRQ_PIPELINE |
---|
| 829 | + |
---|
| 830 | +/* |
---|
| 831 | + * When pipelining interrupts, we have to reconcile the hardware and |
---|
| 832 | + * the virtual states. Hard irqs are off on entry while the current |
---|
| 833 | + * stage has to be unstalled: fix this up by stalling the in-band |
---|
| 834 | + * stage on entry, unstalling on exit. |
---|
| 835 | + */ |
---|
| 836 | +static inline void arm64_preempt_irq_enter(void) |
---|
| 837 | +{ |
---|
| 838 | + WARN_ON_ONCE(irq_pipeline_debug() && test_inband_stall()); |
---|
| 839 | + stall_inband(); |
---|
| 840 | + trace_hardirqs_off(); |
---|
| 841 | +} |
---|
| 842 | + |
---|
| 843 | +static inline void arm64_preempt_irq_exit(void) |
---|
| 844 | +{ |
---|
| 845 | + trace_hardirqs_on(); |
---|
| 846 | + unstall_inband(); |
---|
| 847 | +} |
---|
| 848 | + |
---|
| 849 | +#else |
---|
| 850 | + |
---|
| 851 | +static inline void arm64_preempt_irq_enter(void) |
---|
| 852 | +{ } |
---|
| 853 | + |
---|
| 854 | +static inline void arm64_preempt_irq_exit(void) |
---|
| 855 | +{ } |
---|
| 856 | + |
---|
| 857 | +#endif |
---|
| 858 | + |
---|
827 | 859 | asmlinkage void __sched arm64_preempt_schedule_irq(void) |
---|
828 | 860 | { |
---|
| 861 | + arm64_preempt_irq_enter(); |
---|
| 862 | + |
---|
829 | 863 | lockdep_assert_irqs_disabled(); |
---|
830 | 864 | |
---|
831 | 865 | /* |
---|
.. | .. |
---|
838 | 872 | */ |
---|
839 | 873 | if (system_capabilities_finalized()) |
---|
840 | 874 | preempt_schedule_irq(); |
---|
| 875 | + |
---|
| 876 | + arm64_preempt_irq_exit(); |
---|
841 | 877 | } |
---|
842 | 878 | |
---|
843 | 879 | #ifdef CONFIG_BINFMT_ELF |
---|