| .. | .. |
|---|
| 118 | 118 | static inline void rcu_nocb_flush_deferred_wakeup(void) { } |
|---|
| 119 | 119 | #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ |
|---|
| 120 | 120 | |
|---|
| 121 | +#ifdef CONFIG_IRQ_PIPELINE |
|---|
| 122 | +void rcu_oob_prepare_lock(void); |
|---|
| 123 | +void rcu_oob_finish_lock(void); |
|---|
| 124 | +#else |
|---|
| 125 | +#define rcu_oob_prepare_lock() do { } while (0) |
|---|
| 126 | +#define rcu_oob_finish_lock() do { } while (0) |
|---|
| 127 | +#endif |
|---|
| 128 | + |
|---|
| 121 | 129 | /** |
|---|
| 122 | 130 | * RCU_NONIDLE - Indicate idle-loop code that needs RCU readers |
|---|
| 123 | 131 | * @a: Code that RCU needs to pay attention to. |
|---|
| .. | .. |
|---|
| 290 | 298 | |
|---|
| 291 | 299 | static inline int rcu_read_lock_sched_held(void) |
|---|
| 292 | 300 | { |
|---|
| 293 | | - return !preemptible(); |
|---|
| 301 | + return !running_inband() || !preemptible(); |
|---|
| 294 | 302 | } |
|---|
| 295 | 303 | |
|---|
| 296 | 304 | static inline int rcu_read_lock_any_held(void) |
|---|
| .. | .. |
|---|
| 646 | 654 | */ |
|---|
| 647 | 655 | static __always_inline void rcu_read_lock(void) |
|---|
| 648 | 656 | { |
|---|
| 657 | + rcu_oob_prepare_lock(); |
|---|
| 649 | 658 | __rcu_read_lock(); |
|---|
| 650 | 659 | __acquire(RCU); |
|---|
| 651 | 660 | rcu_lock_acquire(&rcu_lock_map); |
|---|
| .. | .. |
|---|
| 702 | 711 | "rcu_read_unlock() used illegally while idle"); |
|---|
| 703 | 712 | __release(RCU); |
|---|
| 704 | 713 | __rcu_read_unlock(); |
|---|
| 714 | + rcu_oob_finish_lock(); |
|---|
| 705 | 715 | rcu_lock_release(&rcu_lock_map); /* Keep acq info for rls diags. */ |
|---|
| 706 | 716 | } |
|---|
| 707 | 717 | |
|---|
| .. | .. |
|---|
| 755 | 765 | static inline void rcu_read_lock_sched(void) |
|---|
| 756 | 766 | { |
|---|
| 757 | 767 | preempt_disable(); |
|---|
| 768 | + rcu_oob_prepare_lock(); |
|---|
| 758 | 769 | __acquire(RCU_SCHED); |
|---|
| 759 | 770 | rcu_lock_acquire(&rcu_sched_lock_map); |
|---|
| 760 | 771 | RCU_LOCKDEP_WARN(!rcu_is_watching(), |
|---|
| .. | .. |
|---|
| 779 | 790 | "rcu_read_unlock_sched() used illegally while idle"); |
|---|
| 780 | 791 | rcu_lock_release(&rcu_sched_lock_map); |
|---|
| 781 | 792 | __release(RCU_SCHED); |
|---|
| 793 | + rcu_oob_finish_lock(); |
|---|
| 782 | 794 | preempt_enable(); |
|---|
| 783 | 795 | } |
|---|
| 784 | 796 | |
|---|