hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d
kernel/include/linux/irqflags.h
....@@ -14,49 +14,125 @@
1414
1515 #include <linux/typecheck.h>
1616 #include <asm/irqflags.h>
17
+#include <asm/percpu.h>
1718
18
-/* Currently trace_softirqs_on/off is used only by lockdep */
19
+/* Currently lockdep_softirqs_on/off is used only by lockdep */
1920 #ifdef CONFIG_PROVE_LOCKING
20
- extern void trace_softirqs_on(unsigned long ip);
21
- extern void trace_softirqs_off(unsigned long ip);
21
+ extern void lockdep_softirqs_on(unsigned long ip);
22
+ extern void lockdep_softirqs_off(unsigned long ip);
23
+ extern void lockdep_hardirqs_on_prepare(unsigned long ip);
2224 extern void lockdep_hardirqs_on(unsigned long ip);
2325 extern void lockdep_hardirqs_off(unsigned long ip);
2426 #else
25
- static inline void trace_softirqs_on(unsigned long ip) { }
26
- static inline void trace_softirqs_off(unsigned long ip) { }
27
+ static inline void lockdep_softirqs_on(unsigned long ip) { }
28
+ static inline void lockdep_softirqs_off(unsigned long ip) { }
29
+ static inline void lockdep_hardirqs_on_prepare(unsigned long ip) { }
2730 static inline void lockdep_hardirqs_on(unsigned long ip) { }
2831 static inline void lockdep_hardirqs_off(unsigned long ip) { }
2932 #endif
3033
3134 #ifdef CONFIG_TRACE_IRQFLAGS
32
- extern void trace_hardirqs_on(void);
33
- extern void trace_hardirqs_off(void);
34
-# define trace_hardirq_context(p) ((p)->hardirq_context)
35
-# define trace_softirq_context(p) ((p)->softirq_context)
36
-# define trace_hardirqs_enabled(p) ((p)->hardirqs_enabled)
37
-# define trace_softirqs_enabled(p) ((p)->softirqs_enabled)
38
-# define trace_hardirq_enter() \
39
-do { \
40
- current->hardirq_context++; \
35
+
36
+/* Per-task IRQ trace events information. */
37
+struct irqtrace_events {
38
+ unsigned int irq_events;
39
+ unsigned long hardirq_enable_ip;
40
+ unsigned long hardirq_disable_ip;
41
+ unsigned int hardirq_enable_event;
42
+ unsigned int hardirq_disable_event;
43
+ unsigned long softirq_disable_ip;
44
+ unsigned long softirq_enable_ip;
45
+ unsigned int softirq_disable_event;
46
+ unsigned int softirq_enable_event;
47
+};
48
+
49
+DECLARE_PER_CPU(int, hardirqs_enabled);
50
+DECLARE_PER_CPU(int, hardirq_context);
51
+
52
+extern void trace_hardirqs_on_prepare(void);
53
+extern void trace_hardirqs_off_finish(void);
54
+extern void trace_hardirqs_on(void);
55
+extern void trace_hardirqs_off(void);
56
+
57
+# define lockdep_hardirq_context() (raw_cpu_read(hardirq_context))
58
+# define lockdep_softirq_context(p) ((p)->softirq_context)
59
+# define lockdep_hardirqs_enabled() (this_cpu_read(hardirqs_enabled))
60
+# define lockdep_softirqs_enabled(p) ((p)->softirqs_enabled)
61
+# define lockdep_hardirq_enter() \
62
+do { \
63
+ if (__this_cpu_inc_return(hardirq_context) == 1)\
64
+ current->hardirq_threaded = 0; \
4165 } while (0)
42
-# define trace_hardirq_exit() \
66
+# define lockdep_hardirq_threaded() \
4367 do { \
44
- current->hardirq_context--; \
68
+ current->hardirq_threaded = 1; \
4569 } while (0)
70
+# define lockdep_hardirq_exit() \
71
+do { \
72
+ __this_cpu_dec(hardirq_context); \
73
+} while (0)
74
+
75
+# define lockdep_hrtimer_enter(__hrtimer) \
76
+({ \
77
+ bool __expires_hardirq = true; \
78
+ \
79
+ if (!__hrtimer->is_hard) { \
80
+ current->irq_config = 1; \
81
+ __expires_hardirq = false; \
82
+ } \
83
+ __expires_hardirq; \
84
+})
85
+
86
+# define lockdep_hrtimer_exit(__expires_hardirq) \
87
+ do { \
88
+ if (!__expires_hardirq) \
89
+ current->irq_config = 0; \
90
+ } while (0)
91
+
92
+# define lockdep_posixtimer_enter() \
93
+ do { \
94
+ current->irq_config = 1; \
95
+ } while (0)
96
+
97
+# define lockdep_posixtimer_exit() \
98
+ do { \
99
+ current->irq_config = 0; \
100
+ } while (0)
101
+
102
+# define lockdep_irq_work_enter(__work) \
103
+ do { \
104
+ if (!(atomic_read(&__work->flags) & IRQ_WORK_HARD_IRQ))\
105
+ current->irq_config = 1; \
106
+ } while (0)
107
+# define lockdep_irq_work_exit(__work) \
108
+ do { \
109
+ if (!(atomic_read(&__work->flags) & IRQ_WORK_HARD_IRQ))\
110
+ current->irq_config = 0; \
111
+ } while (0)
112
+
46113 #else
47
-# define trace_hardirqs_on() do { } while (0)
48
-# define trace_hardirqs_off() do { } while (0)
49
-# define trace_hardirq_context(p) 0
50
-# define trace_softirq_context(p) 0
51
-# define trace_hardirqs_enabled(p) 0
52
-# define trace_softirqs_enabled(p) 0
53
-# define trace_hardirq_enter() do { } while (0)
54
-# define trace_hardirq_exit() do { } while (0)
55
-# define lockdep_softirq_enter() do { } while (0)
56
-# define lockdep_softirq_exit() do { } while (0)
114
+# define trace_hardirqs_on_prepare() do { } while (0)
115
+# define trace_hardirqs_off_finish() do { } while (0)
116
+# define trace_hardirqs_on() do { } while (0)
117
+# define trace_hardirqs_off() do { } while (0)
118
+# define lockdep_hardirq_context() 0
119
+# define lockdep_softirq_context(p) 0
120
+# define lockdep_hardirqs_enabled() 0
121
+# define lockdep_softirqs_enabled(p) 0
122
+# define lockdep_hardirq_enter() do { } while (0)
123
+# define lockdep_hardirq_threaded() do { } while (0)
124
+# define lockdep_hardirq_exit() do { } while (0)
125
+# define lockdep_softirq_enter() do { } while (0)
126
+# define lockdep_softirq_exit() do { } while (0)
127
+# define lockdep_hrtimer_enter(__hrtimer) false
128
+# define lockdep_hrtimer_exit(__context) do { } while (0)
129
+# define lockdep_posixtimer_enter() do { } while (0)
130
+# define lockdep_posixtimer_exit() do { } while (0)
131
+# define lockdep_irq_work_enter(__work) do { } while (0)
132
+# define lockdep_irq_work_exit(__work) do { } while (0)
57133 #endif
58134
59
-#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_PREEMPT_RT_FULL)
135
+#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_PREEMPT_RT)
60136 # define lockdep_softirq_enter() \
61137 do { \
62138 current->softirq_context++; \
....@@ -67,8 +143,8 @@
67143 } while (0)
68144
69145 #else
70
-# define lockdep_softirq_enter() do { } while (0)
71
-# define lockdep_softirq_exit() do { } while (0)
146
+# define lockdep_softirq_enter() do { } while (0)
147
+# define lockdep_softirq_exit() do { } while (0)
72148 #endif
73149
74150 #if defined(CONFIG_IRQSOFF_TRACER) || \
....@@ -113,26 +189,33 @@
113189 * if !TRACE_IRQFLAGS.
114190 */
115191 #ifdef CONFIG_TRACE_IRQFLAGS
116
-#define local_irq_enable() \
117
- do { trace_hardirqs_on(); raw_local_irq_enable(); } while (0)
118
-#define local_irq_disable() \
119
- do { raw_local_irq_disable(); trace_hardirqs_off(); } while (0)
192
+
193
+#define local_irq_enable() \
194
+ do { \
195
+ trace_hardirqs_on(); \
196
+ raw_local_irq_enable(); \
197
+ } while (0)
198
+
199
+#define local_irq_disable() \
200
+ do { \
201
+ bool was_disabled = raw_irqs_disabled();\
202
+ raw_local_irq_disable(); \
203
+ if (!was_disabled) \
204
+ trace_hardirqs_off(); \
205
+ } while (0)
206
+
120207 #define local_irq_save(flags) \
121208 do { \
122209 raw_local_irq_save(flags); \
123
- trace_hardirqs_off(); \
210
+ if (!raw_irqs_disabled_flags(flags)) \
211
+ trace_hardirqs_off(); \
124212 } while (0)
125
-
126213
127214 #define local_irq_restore(flags) \
128215 do { \
129
- if (raw_irqs_disabled_flags(flags)) { \
130
- raw_local_irq_restore(flags); \
131
- trace_hardirqs_off(); \
132
- } else { \
216
+ if (!raw_irqs_disabled_flags(flags)) \
133217 trace_hardirqs_on(); \
134
- raw_local_irq_restore(flags); \
135
- } \
218
+ raw_local_irq_restore(flags); \
136219 } while (0)
137220
138221 #define safe_halt() \
....@@ -146,10 +229,7 @@
146229
147230 #define local_irq_enable() do { raw_local_irq_enable(); } while (0)
148231 #define local_irq_disable() do { raw_local_irq_disable(); } while (0)
149
-#define local_irq_save(flags) \
150
- do { \
151
- raw_local_irq_save(flags); \
152
- } while (0)
232
+#define local_irq_save(flags) do { raw_local_irq_save(flags); } while (0)
153233 #define local_irq_restore(flags) do { raw_local_irq_restore(flags); } while (0)
154234 #define safe_halt() do { raw_safe_halt(); } while (0)
155235