hc
2024-05-10 37f49e37ab4cb5d0bc4c60eb5c6d4dd57db767bb
kernel/include/linux/tracepoint.h
....@@ -1,3 +1,4 @@
1
+/* SPDX-License-Identifier: GPL-2.0-only */
12 #ifndef _LINUX_TRACEPOINT_H
23 #define _LINUX_TRACEPOINT_H
34
....@@ -9,9 +10,6 @@
910 * Copyright (C) 2008-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
1011 *
1112 * Heavily inspired from the Linux Kernel Markers.
12
- *
13
- * This file is released under the GPLv2.
14
- * See the file COPYING for more details.
1513 */
1614
1715 #include <linux/smp.h>
....@@ -21,6 +19,7 @@
2119 #include <linux/cpumask.h>
2220 #include <linux/rcupdate.h>
2321 #include <linux/tracepoint-defs.h>
22
+#include <linux/static_call.h>
2423
2524 struct module;
2625 struct tracepoint;
....@@ -92,7 +91,7 @@
9291 static inline void tracepoint_synchronize_unregister(void)
9392 {
9493 synchronize_srcu(&tracepoint_srcu);
95
- synchronize_sched();
94
+ synchronize_rcu();
9695 }
9796 #else
9897 static inline void tracepoint_synchronize_unregister(void)
....@@ -104,7 +103,9 @@
104103 extern void syscall_unregfunc(void);
105104 #endif /* CONFIG_HAVE_SYSCALL_TRACEPOINTS */
106105
106
+#ifndef PARAMS
107107 #define PARAMS(args...) args
108
+#endif
108109
109110 #define TRACE_DEFINE_ENUM(x)
110111 #define TRACE_DEFINE_SIZEOF(x)
....@@ -128,8 +129,7 @@
128129
129130 #define __TRACEPOINT_ENTRY(name) \
130131 static tracepoint_ptr_t __tracepoint_ptr_##name __used \
131
- __attribute__((section("__tracepoints_ptrs"))) = \
132
- &__tracepoint_##name
132
+ __section("__tracepoints_ptrs") = &__tracepoint_##name
133133 #endif
134134
135135 #endif /* _LINUX_TRACEPOINT_H */
....@@ -161,6 +161,12 @@
161161
162162 #ifdef TRACEPOINTS_ENABLED
163163
164
+#ifdef CONFIG_HAVE_STATIC_CALL
165
+#define __DO_TRACE_CALL(name) static_call(tp_func_##name)
166
+#else
167
+#define __DO_TRACE_CALL(name) __traceiter_##name
168
+#endif /* CONFIG_HAVE_STATIC_CALL */
169
+
164170 /*
165171 * it_func[0] is never NULL because there is at least one element in the array
166172 * when the array itself is non NULL.
....@@ -168,15 +174,13 @@
168174 * Note, the proto and args passed in includes "__data" as the first parameter.
169175 * The reason for this is to handle the "void" prototype. If a tracepoint
170176 * has a "void" prototype, then it is invalid to declare a function
171
- * as "(void *, void)". The DECLARE_TRACE_NOARGS() will pass in just
172
- * "void *data", where as the DECLARE_TRACE() will pass in "void *data, proto".
177
+ * as "(void *, void)".
173178 */
174
-#define __DO_TRACE(tp, proto, args, cond, rcuidle) \
179
+#define __DO_TRACE(name, proto, args, cond, rcuidle) \
175180 do { \
176181 struct tracepoint_func *it_func_ptr; \
177
- void *it_func; \
178
- void *__data; \
179182 int __maybe_unused __idx = 0; \
183
+ void *__data; \
180184 \
181185 if (!(cond)) \
182186 return; \
....@@ -196,14 +200,11 @@
196200 rcu_irq_enter_irqson(); \
197201 } \
198202 \
199
- it_func_ptr = rcu_dereference_raw((tp)->funcs); \
200
- \
203
+ it_func_ptr = \
204
+ rcu_dereference_raw((&__tracepoint_##name)->funcs); \
201205 if (it_func_ptr) { \
202
- do { \
203
- it_func = (it_func_ptr)->func; \
204
- __data = (it_func_ptr)->data; \
205
- ((void(*)(proto))(it_func))(args); \
206
- } while ((++it_func_ptr)->func); \
206
+ __data = (it_func_ptr)->data; \
207
+ __DO_TRACE_CALL(name)(args); \
207208 } \
208209 \
209210 if (rcuidle) { \
....@@ -219,7 +220,7 @@
219220 static inline void trace_##name##_rcuidle(proto) \
220221 { \
221222 if (static_key_false(&__tracepoint_##name.key)) \
222
- __DO_TRACE(&__tracepoint_##name, \
223
+ __DO_TRACE(name, \
223224 TP_PROTO(data_proto), \
224225 TP_ARGS(data_args), \
225226 TP_CONDITION(cond), 1); \
....@@ -233,26 +234,25 @@
233234 * not add unwanted padding between the beginning of the section and the
234235 * structure. Force alignment to the same alignment as the section start.
235236 *
236
- * When lockdep is enabled, we make sure to always do the RCU portions of
237
- * the tracepoint code, regardless of whether tracing is on. However,
238
- * don't check if the condition is false, due to interaction with idle
239
- * instrumentation. This lets us find RCU issues triggered with tracepoints
240
- * even when this tracepoint is off. This code has no purpose other than
241
- * poking RCU a bit.
237
+ * When lockdep is enabled, we make sure to always test if RCU is
238
+ * "watching" regardless if the tracepoint is enabled or not. Tracepoints
239
+ * require RCU to be active, and it should always warn at the tracepoint
240
+ * site if it is not watching, as it will need to be active when the
241
+ * tracepoint is enabled.
242242 */
243243 #define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \
244
+ extern int __traceiter_##name(data_proto); \
245
+ DECLARE_STATIC_CALL(tp_func_##name, __traceiter_##name); \
244246 extern struct tracepoint __tracepoint_##name; \
245
- static inline void trace_##name(proto) \
247
+ static inline void __nocfi trace_##name(proto) \
246248 { \
247249 if (static_key_false(&__tracepoint_##name.key)) \
248
- __DO_TRACE(&__tracepoint_##name, \
250
+ __DO_TRACE(name, \
249251 TP_PROTO(data_proto), \
250252 TP_ARGS(data_args), \
251253 TP_CONDITION(cond), 0); \
252254 if (IS_ENABLED(CONFIG_LOCKDEP) && (cond)) { \
253
- rcu_read_lock_sched_notrace(); \
254
- rcu_dereference_sched(__tracepoint_##name.funcs);\
255
- rcu_read_unlock_sched_notrace(); \
255
+ WARN_ON_ONCE(!rcu_is_watching()); \
256256 } \
257257 } \
258258 __DECLARE_TRACE_RCU(name, PARAMS(proto), PARAMS(args), \
....@@ -291,21 +291,52 @@
291291 * structures, so we create an array of pointers that will be used for iteration
292292 * on the tracepoints.
293293 */
294
-#define DEFINE_TRACE_FN(name, reg, unreg) \
295
- static const char __tpstrtab_##name[] \
296
- __attribute__((section("__tracepoints_strings"))) = #name; \
297
- struct tracepoint __tracepoint_##name \
298
- __attribute__((section("__tracepoints"), used)) = \
299
- { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\
300
- __TRACEPOINT_ENTRY(name);
294
+#define DEFINE_TRACE_FN(_name, _reg, _unreg, proto, args) \
295
+ static const char __tpstrtab_##_name[] \
296
+ __section("__tracepoints_strings") = #_name; \
297
+ extern struct static_call_key STATIC_CALL_KEY(tp_func_##_name); \
298
+ int __traceiter_##_name(void *__data, proto); \
299
+ struct tracepoint __tracepoint_##_name __used \
300
+ __section("__tracepoints") = { \
301
+ .name = __tpstrtab_##_name, \
302
+ .key = STATIC_KEY_INIT_FALSE, \
303
+ .static_call_key = &STATIC_CALL_KEY(tp_func_##_name), \
304
+ .static_call_tramp = STATIC_CALL_TRAMP_ADDR(tp_func_##_name), \
305
+ .iterator = &__traceiter_##_name, \
306
+ .regfunc = _reg, \
307
+ .unregfunc = _unreg, \
308
+ .funcs = NULL }; \
309
+ __TRACEPOINT_ENTRY(_name); \
310
+ int __nocfi __traceiter_##_name(void *__data, proto) \
311
+ { \
312
+ struct tracepoint_func *it_func_ptr; \
313
+ void *it_func; \
314
+ \
315
+ it_func_ptr = \
316
+ rcu_dereference_raw((&__tracepoint_##_name)->funcs); \
317
+ if (it_func_ptr) { \
318
+ do { \
319
+ it_func = (it_func_ptr)->func; \
320
+ __data = (it_func_ptr)->data; \
321
+ ((void(*)(void *, proto))(it_func))(__data, args); \
322
+ } while ((++it_func_ptr)->func); \
323
+ } \
324
+ return 0; \
325
+ } \
326
+ DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
301327
302
-#define DEFINE_TRACE(name) \
303
- DEFINE_TRACE_FN(name, NULL, NULL);
328
+#define DEFINE_TRACE(name, proto, args) \
329
+ DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
304330
305331 #define EXPORT_TRACEPOINT_SYMBOL_GPL(name) \
306
- EXPORT_SYMBOL_GPL(__tracepoint_##name)
332
+ EXPORT_SYMBOL_GPL(__tracepoint_##name); \
333
+ EXPORT_SYMBOL_GPL(__traceiter_##name); \
334
+ EXPORT_STATIC_CALL_GPL(tp_func_##name)
307335 #define EXPORT_TRACEPOINT_SYMBOL(name) \
308
- EXPORT_SYMBOL(__tracepoint_##name)
336
+ EXPORT_SYMBOL(__tracepoint_##name); \
337
+ EXPORT_SYMBOL(__traceiter_##name); \
338
+ EXPORT_STATIC_CALL(tp_func_##name)
339
+
309340
310341 #else /* !TRACEPOINTS_ENABLED */
311342 #define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \
....@@ -334,8 +365,8 @@
334365 return false; \
335366 }
336367
337
-#define DEFINE_TRACE_FN(name, reg, unreg)
338
-#define DEFINE_TRACE(name)
368
+#define DEFINE_TRACE_FN(name, reg, unreg, proto, args)
369
+#define DEFINE_TRACE(name, proto, args)
339370 #define EXPORT_TRACEPOINT_SYMBOL_GPL(name)
340371 #define EXPORT_TRACEPOINT_SYMBOL(name)
341372
....@@ -374,7 +405,7 @@
374405 static const char *___tp_str __tracepoint_string = str; \
375406 ___tp_str; \
376407 })
377
-#define __tracepoint_string __attribute__((section("__tracepoint_str"), used))
408
+#define __tracepoint_string __used __section("__tracepoint_str")
378409 #else
379410 /*
380411 * tracepoint_string() is used to save the string address for userspace
....@@ -384,25 +415,6 @@
384415 # define tracepoint_string(str) str
385416 # define __tracepoint_string
386417 #endif
387
-
388
-/*
389
- * The need for the DECLARE_TRACE_NOARGS() is to handle the prototype
390
- * (void). "void" is a special value in a function prototype and can
391
- * not be combined with other arguments. Since the DECLARE_TRACE()
392
- * macro adds a data element at the beginning of the prototype,
393
- * we need a way to differentiate "(void *data, proto)" from
394
- * "(void *data, void)". The second prototype is invalid.
395
- *
396
- * DECLARE_TRACE_NOARGS() passes "void" as the tracepoint prototype
397
- * and "void *__data" as the callback prototype.
398
- *
399
- * DECLARE_TRACE() passes "proto" as the tracepoint protoype and
400
- * "void *__data, proto" as the callback prototype.
401
- */
402
-#define DECLARE_TRACE_NOARGS(name) \
403
- __DECLARE_TRACE(name, void, , \
404
- cpu_online(raw_smp_processor_id()), \
405
- void *__data, __data)
406418
407419 #define DECLARE_TRACE(name, proto, args) \
408420 __DECLARE_TRACE(name, PARAMS(proto), PARAMS(args), \
....@@ -558,4 +570,19 @@
558570
559571 #define TRACE_EVENT_PERF_PERM(event, expr...)
560572
573
+#define DECLARE_EVENT_NOP(name, proto, args) \
574
+ static inline void trace_##name(proto) \
575
+ { } \
576
+ static inline bool trace_##name##_enabled(void) \
577
+ { \
578
+ return false; \
579
+ }
580
+
581
+#define TRACE_EVENT_NOP(name, proto, args, struct, assign, print) \
582
+ DECLARE_EVENT_NOP(name, PARAMS(proto), PARAMS(args))
583
+
584
+#define DECLARE_EVENT_CLASS_NOP(name, proto, args, tstruct, assign, print)
585
+#define DEFINE_EVENT_NOP(template, name, proto, args) \
586
+ DECLARE_EVENT_NOP(name, PARAMS(proto), PARAMS(args))
587
+
561588 #endif /* ifdef TRACE_EVENT (see note above) */