.. | .. |
---|
14 | 14 | |
---|
15 | 15 | #include <linux/typecheck.h> |
---|
16 | 16 | #include <asm/irqflags.h> |
---|
| 17 | +#include <asm/percpu.h> |
---|
17 | 18 | |
---|
18 | | -/* Currently trace_softirqs_on/off is used only by lockdep */ |
---|
| 19 | +/* Currently lockdep_softirqs_on/off is used only by lockdep */ |
---|
19 | 20 | #ifdef CONFIG_PROVE_LOCKING |
---|
20 | | - extern void trace_softirqs_on(unsigned long ip); |
---|
21 | | - extern void trace_softirqs_off(unsigned long ip); |
---|
| 21 | + extern void lockdep_softirqs_on(unsigned long ip); |
---|
| 22 | + extern void lockdep_softirqs_off(unsigned long ip); |
---|
| 23 | + extern void lockdep_hardirqs_on_prepare(unsigned long ip); |
---|
22 | 24 | extern void lockdep_hardirqs_on(unsigned long ip); |
---|
23 | 25 | extern void lockdep_hardirqs_off(unsigned long ip); |
---|
24 | 26 | #else |
---|
25 | | - static inline void trace_softirqs_on(unsigned long ip) { } |
---|
26 | | - static inline void trace_softirqs_off(unsigned long ip) { } |
---|
| 27 | + static inline void lockdep_softirqs_on(unsigned long ip) { } |
---|
| 28 | + static inline void lockdep_softirqs_off(unsigned long ip) { } |
---|
| 29 | + static inline void lockdep_hardirqs_on_prepare(unsigned long ip) { } |
---|
27 | 30 | static inline void lockdep_hardirqs_on(unsigned long ip) { } |
---|
28 | 31 | static inline void lockdep_hardirqs_off(unsigned long ip) { } |
---|
29 | 32 | #endif |
---|
30 | 33 | |
---|
31 | 34 | #ifdef CONFIG_TRACE_IRQFLAGS |
---|
32 | | - extern void trace_hardirqs_on(void); |
---|
33 | | - extern void trace_hardirqs_off(void); |
---|
34 | | -# define trace_hardirq_context(p) ((p)->hardirq_context) |
---|
35 | | -# define trace_softirq_context(p) ((p)->softirq_context) |
---|
36 | | -# define trace_hardirqs_enabled(p) ((p)->hardirqs_enabled) |
---|
37 | | -# define trace_softirqs_enabled(p) ((p)->softirqs_enabled) |
---|
38 | | -# define trace_hardirq_enter() \ |
---|
39 | | -do { \ |
---|
40 | | - current->hardirq_context++; \ |
---|
| 35 | + |
---|
| 36 | +/* Per-task IRQ trace events information. */ |
---|
| 37 | +struct irqtrace_events { |
---|
| 38 | + unsigned int irq_events; |
---|
| 39 | + unsigned long hardirq_enable_ip; |
---|
| 40 | + unsigned long hardirq_disable_ip; |
---|
| 41 | + unsigned int hardirq_enable_event; |
---|
| 42 | + unsigned int hardirq_disable_event; |
---|
| 43 | + unsigned long softirq_disable_ip; |
---|
| 44 | + unsigned long softirq_enable_ip; |
---|
| 45 | + unsigned int softirq_disable_event; |
---|
| 46 | + unsigned int softirq_enable_event; |
---|
| 47 | +}; |
---|
| 48 | + |
---|
| 49 | +DECLARE_PER_CPU(int, hardirqs_enabled); |
---|
| 50 | +DECLARE_PER_CPU(int, hardirq_context); |
---|
| 51 | + |
---|
| 52 | +extern void trace_hardirqs_on_prepare(void); |
---|
| 53 | +extern void trace_hardirqs_off_finish(void); |
---|
| 54 | +extern void trace_hardirqs_on(void); |
---|
| 55 | +extern void trace_hardirqs_off(void); |
---|
| 56 | + |
---|
| 57 | +# define lockdep_hardirq_context() (raw_cpu_read(hardirq_context)) |
---|
| 58 | +# define lockdep_softirq_context(p) ((p)->softirq_context) |
---|
| 59 | +# define lockdep_hardirqs_enabled() (this_cpu_read(hardirqs_enabled)) |
---|
| 60 | +# define lockdep_softirqs_enabled(p) ((p)->softirqs_enabled) |
---|
| 61 | +# define lockdep_hardirq_enter() \ |
---|
| 62 | +do { \ |
---|
| 63 | + if (__this_cpu_inc_return(hardirq_context) == 1)\ |
---|
| 64 | + current->hardirq_threaded = 0; \ |
---|
41 | 65 | } while (0) |
---|
42 | | -# define trace_hardirq_exit() \ |
---|
| 66 | +# define lockdep_hardirq_threaded() \ |
---|
43 | 67 | do { \ |
---|
44 | | - current->hardirq_context--; \ |
---|
| 68 | + current->hardirq_threaded = 1; \ |
---|
45 | 69 | } while (0) |
---|
| 70 | +# define lockdep_hardirq_exit() \ |
---|
| 71 | +do { \ |
---|
| 72 | + __this_cpu_dec(hardirq_context); \ |
---|
| 73 | +} while (0) |
---|
| 74 | + |
---|
| 75 | +# define lockdep_hrtimer_enter(__hrtimer) \ |
---|
| 76 | +({ \ |
---|
| 77 | + bool __expires_hardirq = true; \ |
---|
| 78 | + \ |
---|
| 79 | + if (!__hrtimer->is_hard) { \ |
---|
| 80 | + current->irq_config = 1; \ |
---|
| 81 | + __expires_hardirq = false; \ |
---|
| 82 | + } \ |
---|
| 83 | + __expires_hardirq; \ |
---|
| 84 | +}) |
---|
| 85 | + |
---|
| 86 | +# define lockdep_hrtimer_exit(__expires_hardirq) \ |
---|
| 87 | + do { \ |
---|
| 88 | + if (!__expires_hardirq) \ |
---|
| 89 | + current->irq_config = 0; \ |
---|
| 90 | + } while (0) |
---|
| 91 | + |
---|
| 92 | +# define lockdep_posixtimer_enter() \ |
---|
| 93 | + do { \ |
---|
| 94 | + current->irq_config = 1; \ |
---|
| 95 | + } while (0) |
---|
| 96 | + |
---|
| 97 | +# define lockdep_posixtimer_exit() \ |
---|
| 98 | + do { \ |
---|
| 99 | + current->irq_config = 0; \ |
---|
| 100 | + } while (0) |
---|
| 101 | + |
---|
| 102 | +# define lockdep_irq_work_enter(__work) \ |
---|
| 103 | + do { \ |
---|
| 104 | + if (!(atomic_read(&__work->flags) & IRQ_WORK_HARD_IRQ))\ |
---|
| 105 | + current->irq_config = 1; \ |
---|
| 106 | + } while (0) |
---|
| 107 | +# define lockdep_irq_work_exit(__work) \ |
---|
| 108 | + do { \ |
---|
| 109 | + if (!(atomic_read(&__work->flags) & IRQ_WORK_HARD_IRQ))\ |
---|
| 110 | + current->irq_config = 0; \ |
---|
| 111 | + } while (0) |
---|
| 112 | + |
---|
46 | 113 | #else |
---|
47 | | -# define trace_hardirqs_on() do { } while (0) |
---|
48 | | -# define trace_hardirqs_off() do { } while (0) |
---|
49 | | -# define trace_hardirq_context(p) 0 |
---|
50 | | -# define trace_softirq_context(p) 0 |
---|
51 | | -# define trace_hardirqs_enabled(p) 0 |
---|
52 | | -# define trace_softirqs_enabled(p) 0 |
---|
53 | | -# define trace_hardirq_enter() do { } while (0) |
---|
54 | | -# define trace_hardirq_exit() do { } while (0) |
---|
55 | | -# define lockdep_softirq_enter() do { } while (0) |
---|
56 | | -# define lockdep_softirq_exit() do { } while (0) |
---|
| 114 | +# define trace_hardirqs_on_prepare() do { } while (0) |
---|
| 115 | +# define trace_hardirqs_off_finish() do { } while (0) |
---|
| 116 | +# define trace_hardirqs_on() do { } while (0) |
---|
| 117 | +# define trace_hardirqs_off() do { } while (0) |
---|
| 118 | +# define lockdep_hardirq_context() 0 |
---|
| 119 | +# define lockdep_softirq_context(p) 0 |
---|
| 120 | +# define lockdep_hardirqs_enabled() 0 |
---|
| 121 | +# define lockdep_softirqs_enabled(p) 0 |
---|
| 122 | +# define lockdep_hardirq_enter() do { } while (0) |
---|
| 123 | +# define lockdep_hardirq_threaded() do { } while (0) |
---|
| 124 | +# define lockdep_hardirq_exit() do { } while (0) |
---|
| 125 | +# define lockdep_softirq_enter() do { } while (0) |
---|
| 126 | +# define lockdep_softirq_exit() do { } while (0) |
---|
| 127 | +# define lockdep_hrtimer_enter(__hrtimer) false |
---|
| 128 | +# define lockdep_hrtimer_exit(__context) do { } while (0) |
---|
| 129 | +# define lockdep_posixtimer_enter() do { } while (0) |
---|
| 130 | +# define lockdep_posixtimer_exit() do { } while (0) |
---|
| 131 | +# define lockdep_irq_work_enter(__work) do { } while (0) |
---|
| 132 | +# define lockdep_irq_work_exit(__work) do { } while (0) |
---|
57 | 133 | #endif |
---|
58 | 134 | |
---|
59 | | -#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_PREEMPT_RT_FULL) |
---|
| 135 | +#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_PREEMPT_RT) |
---|
60 | 136 | # define lockdep_softirq_enter() \ |
---|
61 | 137 | do { \ |
---|
62 | 138 | current->softirq_context++; \ |
---|
.. | .. |
---|
67 | 143 | } while (0) |
---|
68 | 144 | |
---|
69 | 145 | #else |
---|
70 | | -# define lockdep_softirq_enter() do { } while (0) |
---|
71 | | -# define lockdep_softirq_exit() do { } while (0) |
---|
| 146 | +# define lockdep_softirq_enter() do { } while (0) |
---|
| 147 | +# define lockdep_softirq_exit() do { } while (0) |
---|
72 | 148 | #endif |
---|
73 | 149 | |
---|
74 | 150 | #if defined(CONFIG_IRQSOFF_TRACER) || \ |
---|
.. | .. |
---|
113 | 189 | * if !TRACE_IRQFLAGS. |
---|
114 | 190 | */ |
---|
115 | 191 | #ifdef CONFIG_TRACE_IRQFLAGS |
---|
116 | | -#define local_irq_enable() \ |
---|
117 | | - do { trace_hardirqs_on(); raw_local_irq_enable(); } while (0) |
---|
118 | | -#define local_irq_disable() \ |
---|
119 | | - do { raw_local_irq_disable(); trace_hardirqs_off(); } while (0) |
---|
| 192 | + |
---|
| 193 | +#define local_irq_enable() \ |
---|
| 194 | + do { \ |
---|
| 195 | + trace_hardirqs_on(); \ |
---|
| 196 | + raw_local_irq_enable(); \ |
---|
| 197 | + } while (0) |
---|
| 198 | + |
---|
| 199 | +#define local_irq_disable() \ |
---|
| 200 | + do { \ |
---|
| 201 | + bool was_disabled = raw_irqs_disabled();\ |
---|
| 202 | + raw_local_irq_disable(); \ |
---|
| 203 | + if (!was_disabled) \ |
---|
| 204 | + trace_hardirqs_off(); \ |
---|
| 205 | + } while (0) |
---|
| 206 | + |
---|
120 | 207 | #define local_irq_save(flags) \ |
---|
121 | 208 | do { \ |
---|
122 | 209 | raw_local_irq_save(flags); \ |
---|
123 | | - trace_hardirqs_off(); \ |
---|
| 210 | + if (!raw_irqs_disabled_flags(flags)) \ |
---|
| 211 | + trace_hardirqs_off(); \ |
---|
124 | 212 | } while (0) |
---|
125 | | - |
---|
126 | 213 | |
---|
127 | 214 | #define local_irq_restore(flags) \ |
---|
128 | 215 | do { \ |
---|
129 | | - if (raw_irqs_disabled_flags(flags)) { \ |
---|
130 | | - raw_local_irq_restore(flags); \ |
---|
131 | | - trace_hardirqs_off(); \ |
---|
132 | | - } else { \ |
---|
| 216 | + if (!raw_irqs_disabled_flags(flags)) \ |
---|
133 | 217 | trace_hardirqs_on(); \ |
---|
134 | | - raw_local_irq_restore(flags); \ |
---|
135 | | - } \ |
---|
| 218 | + raw_local_irq_restore(flags); \ |
---|
136 | 219 | } while (0) |
---|
137 | 220 | |
---|
138 | 221 | #define safe_halt() \ |
---|
.. | .. |
---|
146 | 229 | |
---|
147 | 230 | #define local_irq_enable() do { raw_local_irq_enable(); } while (0) |
---|
148 | 231 | #define local_irq_disable() do { raw_local_irq_disable(); } while (0) |
---|
149 | | -#define local_irq_save(flags) \ |
---|
150 | | - do { \ |
---|
151 | | - raw_local_irq_save(flags); \ |
---|
152 | | - } while (0) |
---|
| 232 | +#define local_irq_save(flags) do { raw_local_irq_save(flags); } while (0) |
---|
153 | 233 | #define local_irq_restore(flags) do { raw_local_irq_restore(flags); } while (0) |
---|
154 | 234 | #define safe_halt() do { raw_safe_halt(); } while (0) |
---|
155 | 235 | |
---|