.. | .. |
---|
| 1 | +/* SPDX-License-Identifier: GPL-2.0-only */ |
---|
1 | 2 | #ifndef _LINUX_TRACEPOINT_H |
---|
2 | 3 | #define _LINUX_TRACEPOINT_H |
---|
3 | 4 | |
---|
.. | .. |
---|
9 | 10 | * Copyright (C) 2008-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
---|
10 | 11 | * |
---|
11 | 12 | * Heavily inspired from the Linux Kernel Markers. |
---|
12 | | - * |
---|
13 | | - * This file is released under the GPLv2. |
---|
14 | | - * See the file COPYING for more details. |
---|
15 | 13 | */ |
---|
16 | 14 | |
---|
17 | 15 | #include <linux/smp.h> |
---|
.. | .. |
---|
21 | 19 | #include <linux/cpumask.h> |
---|
22 | 20 | #include <linux/rcupdate.h> |
---|
23 | 21 | #include <linux/tracepoint-defs.h> |
---|
| 22 | +#include <linux/static_call.h> |
---|
24 | 23 | |
---|
25 | 24 | struct module; |
---|
26 | 25 | struct tracepoint; |
---|
.. | .. |
---|
92 | 91 | static inline void tracepoint_synchronize_unregister(void) |
---|
93 | 92 | { |
---|
94 | 93 | synchronize_srcu(&tracepoint_srcu); |
---|
95 | | - synchronize_sched(); |
---|
| 94 | + synchronize_rcu(); |
---|
96 | 95 | } |
---|
97 | 96 | #else |
---|
98 | 97 | static inline void tracepoint_synchronize_unregister(void) |
---|
.. | .. |
---|
104 | 103 | extern void syscall_unregfunc(void); |
---|
105 | 104 | #endif /* CONFIG_HAVE_SYSCALL_TRACEPOINTS */ |
---|
106 | 105 | |
---|
| 106 | +#ifndef PARAMS |
---|
107 | 107 | #define PARAMS(args...) args |
---|
| 108 | +#endif |
---|
108 | 109 | |
---|
109 | 110 | #define TRACE_DEFINE_ENUM(x) |
---|
110 | 111 | #define TRACE_DEFINE_SIZEOF(x) |
---|
.. | .. |
---|
128 | 129 | |
---|
129 | 130 | #define __TRACEPOINT_ENTRY(name) \ |
---|
130 | 131 | static tracepoint_ptr_t __tracepoint_ptr_##name __used \ |
---|
131 | | - __attribute__((section("__tracepoints_ptrs"))) = \ |
---|
132 | | - &__tracepoint_##name |
---|
| 132 | + __section("__tracepoints_ptrs") = &__tracepoint_##name |
---|
133 | 133 | #endif |
---|
134 | 134 | |
---|
135 | 135 | #endif /* _LINUX_TRACEPOINT_H */ |
---|
.. | .. |
---|
161 | 161 | |
---|
162 | 162 | #ifdef TRACEPOINTS_ENABLED |
---|
163 | 163 | |
---|
| 164 | +#ifdef CONFIG_HAVE_STATIC_CALL |
---|
| 165 | +#define __DO_TRACE_CALL(name) static_call(tp_func_##name) |
---|
| 166 | +#else |
---|
| 167 | +#define __DO_TRACE_CALL(name) __traceiter_##name |
---|
| 168 | +#endif /* CONFIG_HAVE_STATIC_CALL */ |
---|
| 169 | + |
---|
164 | 170 | /* |
---|
165 | 171 | * it_func[0] is never NULL because there is at least one element in the array |
---|
166 | 172 | * when the array itself is non NULL. |
---|
.. | .. |
---|
168 | 174 | * Note, the proto and args passed in includes "__data" as the first parameter. |
---|
169 | 175 | * The reason for this is to handle the "void" prototype. If a tracepoint |
---|
170 | 176 | * has a "void" prototype, then it is invalid to declare a function |
---|
171 | | - * as "(void *, void)". The DECLARE_TRACE_NOARGS() will pass in just |
---|
172 | | - * "void *data", where as the DECLARE_TRACE() will pass in "void *data, proto". |
---|
| 177 | + * as "(void *, void)". |
---|
173 | 178 | */ |
---|
174 | | -#define __DO_TRACE(tp, proto, args, cond, rcuidle) \ |
---|
| 179 | +#define __DO_TRACE(name, proto, args, cond, rcuidle) \ |
---|
175 | 180 | do { \ |
---|
176 | 181 | struct tracepoint_func *it_func_ptr; \ |
---|
177 | | - void *it_func; \ |
---|
178 | | - void *__data; \ |
---|
179 | 182 | int __maybe_unused __idx = 0; \ |
---|
| 183 | + void *__data; \ |
---|
180 | 184 | \ |
---|
181 | 185 | if (!(cond)) \ |
---|
182 | 186 | return; \ |
---|
.. | .. |
---|
196 | 200 | rcu_irq_enter_irqson(); \ |
---|
197 | 201 | } \ |
---|
198 | 202 | \ |
---|
199 | | - it_func_ptr = rcu_dereference_raw((tp)->funcs); \ |
---|
200 | | - \ |
---|
| 203 | + it_func_ptr = \ |
---|
| 204 | + rcu_dereference_raw((&__tracepoint_##name)->funcs); \ |
---|
201 | 205 | if (it_func_ptr) { \ |
---|
202 | | - do { \ |
---|
203 | | - it_func = (it_func_ptr)->func; \ |
---|
204 | | - __data = (it_func_ptr)->data; \ |
---|
205 | | - ((void(*)(proto))(it_func))(args); \ |
---|
206 | | - } while ((++it_func_ptr)->func); \ |
---|
| 206 | + __data = (it_func_ptr)->data; \ |
---|
| 207 | + __DO_TRACE_CALL(name)(args); \ |
---|
207 | 208 | } \ |
---|
208 | 209 | \ |
---|
209 | 210 | if (rcuidle) { \ |
---|
.. | .. |
---|
219 | 220 | static inline void trace_##name##_rcuidle(proto) \ |
---|
220 | 221 | { \ |
---|
221 | 222 | if (static_key_false(&__tracepoint_##name.key)) \ |
---|
222 | | - __DO_TRACE(&__tracepoint_##name, \ |
---|
| 223 | + __DO_TRACE(name, \ |
---|
223 | 224 | TP_PROTO(data_proto), \ |
---|
224 | 225 | TP_ARGS(data_args), \ |
---|
225 | 226 | TP_CONDITION(cond), 1); \ |
---|
.. | .. |
---|
233 | 234 | * not add unwanted padding between the beginning of the section and the |
---|
234 | 235 | * structure. Force alignment to the same alignment as the section start. |
---|
235 | 236 | * |
---|
236 | | - * When lockdep is enabled, we make sure to always do the RCU portions of |
---|
237 | | - * the tracepoint code, regardless of whether tracing is on. However, |
---|
238 | | - * don't check if the condition is false, due to interaction with idle |
---|
239 | | - * instrumentation. This lets us find RCU issues triggered with tracepoints |
---|
240 | | - * even when this tracepoint is off. This code has no purpose other than |
---|
241 | | - * poking RCU a bit. |
---|
| 237 | + * When lockdep is enabled, we make sure to always test if RCU is |
---|
| 238 | + * "watching" regardless if the tracepoint is enabled or not. Tracepoints |
---|
| 239 | + * require RCU to be active, and it should always warn at the tracepoint |
---|
| 240 | + * site if it is not watching, as it will need to be active when the |
---|
| 241 | + * tracepoint is enabled. |
---|
242 | 242 | */ |
---|
243 | 243 | #define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \ |
---|
| 244 | + extern int __traceiter_##name(data_proto); \ |
---|
| 245 | + DECLARE_STATIC_CALL(tp_func_##name, __traceiter_##name); \ |
---|
244 | 246 | extern struct tracepoint __tracepoint_##name; \ |
---|
245 | | - static inline void trace_##name(proto) \ |
---|
| 247 | + static inline void __nocfi trace_##name(proto) \ |
---|
246 | 248 | { \ |
---|
247 | 249 | if (static_key_false(&__tracepoint_##name.key)) \ |
---|
248 | | - __DO_TRACE(&__tracepoint_##name, \ |
---|
| 250 | + __DO_TRACE(name, \ |
---|
249 | 251 | TP_PROTO(data_proto), \ |
---|
250 | 252 | TP_ARGS(data_args), \ |
---|
251 | 253 | TP_CONDITION(cond), 0); \ |
---|
252 | 254 | if (IS_ENABLED(CONFIG_LOCKDEP) && (cond)) { \ |
---|
253 | | - rcu_read_lock_sched_notrace(); \ |
---|
254 | | - rcu_dereference_sched(__tracepoint_##name.funcs);\ |
---|
255 | | - rcu_read_unlock_sched_notrace(); \ |
---|
| 255 | + WARN_ON_ONCE(!rcu_is_watching()); \ |
---|
256 | 256 | } \ |
---|
257 | 257 | } \ |
---|
258 | 258 | __DECLARE_TRACE_RCU(name, PARAMS(proto), PARAMS(args), \ |
---|
.. | .. |
---|
291 | 291 | * structures, so we create an array of pointers that will be used for iteration |
---|
292 | 292 | * on the tracepoints. |
---|
293 | 293 | */ |
---|
294 | | -#define DEFINE_TRACE_FN(name, reg, unreg) \ |
---|
295 | | - static const char __tpstrtab_##name[] \ |
---|
296 | | - __attribute__((section("__tracepoints_strings"))) = #name; \ |
---|
297 | | - struct tracepoint __tracepoint_##name \ |
---|
298 | | - __attribute__((section("__tracepoints"), used)) = \ |
---|
299 | | - { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\ |
---|
300 | | - __TRACEPOINT_ENTRY(name); |
---|
| 294 | +#define DEFINE_TRACE_FN(_name, _reg, _unreg, proto, args) \ |
---|
| 295 | + static const char __tpstrtab_##_name[] \ |
---|
| 296 | + __section("__tracepoints_strings") = #_name; \ |
---|
| 297 | + extern struct static_call_key STATIC_CALL_KEY(tp_func_##_name); \ |
---|
| 298 | + int __traceiter_##_name(void *__data, proto); \ |
---|
| 299 | + struct tracepoint __tracepoint_##_name __used \ |
---|
| 300 | + __section("__tracepoints") = { \ |
---|
| 301 | + .name = __tpstrtab_##_name, \ |
---|
| 302 | + .key = STATIC_KEY_INIT_FALSE, \ |
---|
| 303 | + .static_call_key = &STATIC_CALL_KEY(tp_func_##_name), \ |
---|
| 304 | + .static_call_tramp = STATIC_CALL_TRAMP_ADDR(tp_func_##_name), \ |
---|
| 305 | + .iterator = &__traceiter_##_name, \ |
---|
| 306 | + .regfunc = _reg, \ |
---|
| 307 | + .unregfunc = _unreg, \ |
---|
| 308 | + .funcs = NULL }; \ |
---|
| 309 | + __TRACEPOINT_ENTRY(_name); \ |
---|
| 310 | + int __nocfi __traceiter_##_name(void *__data, proto) \ |
---|
| 311 | + { \ |
---|
| 312 | + struct tracepoint_func *it_func_ptr; \ |
---|
| 313 | + void *it_func; \ |
---|
| 314 | + \ |
---|
| 315 | + it_func_ptr = \ |
---|
| 316 | + rcu_dereference_raw((&__tracepoint_##_name)->funcs); \ |
---|
| 317 | + if (it_func_ptr) { \ |
---|
| 318 | + do { \ |
---|
| 319 | + it_func = (it_func_ptr)->func; \ |
---|
| 320 | + __data = (it_func_ptr)->data; \ |
---|
| 321 | + ((void(*)(void *, proto))(it_func))(__data, args); \ |
---|
| 322 | + } while ((++it_func_ptr)->func); \ |
---|
| 323 | + } \ |
---|
| 324 | + return 0; \ |
---|
| 325 | + } \ |
---|
| 326 | + DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name); |
---|
301 | 327 | |
---|
302 | | -#define DEFINE_TRACE(name) \ |
---|
303 | | - DEFINE_TRACE_FN(name, NULL, NULL); |
---|
| 328 | +#define DEFINE_TRACE(name, proto, args) \ |
---|
| 329 | + DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args)); |
---|
304 | 330 | |
---|
305 | 331 | #define EXPORT_TRACEPOINT_SYMBOL_GPL(name) \ |
---|
306 | | - EXPORT_SYMBOL_GPL(__tracepoint_##name) |
---|
| 332 | + EXPORT_SYMBOL_GPL(__tracepoint_##name); \ |
---|
| 333 | + EXPORT_SYMBOL_GPL(__traceiter_##name); \ |
---|
| 334 | + EXPORT_STATIC_CALL_GPL(tp_func_##name) |
---|
307 | 335 | #define EXPORT_TRACEPOINT_SYMBOL(name) \ |
---|
308 | | - EXPORT_SYMBOL(__tracepoint_##name) |
---|
| 336 | + EXPORT_SYMBOL(__tracepoint_##name); \ |
---|
| 337 | + EXPORT_SYMBOL(__traceiter_##name); \ |
---|
| 338 | + EXPORT_STATIC_CALL(tp_func_##name) |
---|
| 339 | + |
---|
309 | 340 | |
---|
310 | 341 | #else /* !TRACEPOINTS_ENABLED */ |
---|
311 | 342 | #define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \ |
---|
.. | .. |
---|
334 | 365 | return false; \ |
---|
335 | 366 | } |
---|
336 | 367 | |
---|
337 | | -#define DEFINE_TRACE_FN(name, reg, unreg) |
---|
338 | | -#define DEFINE_TRACE(name) |
---|
| 368 | +#define DEFINE_TRACE_FN(name, reg, unreg, proto, args) |
---|
| 369 | +#define DEFINE_TRACE(name, proto, args) |
---|
339 | 370 | #define EXPORT_TRACEPOINT_SYMBOL_GPL(name) |
---|
340 | 371 | #define EXPORT_TRACEPOINT_SYMBOL(name) |
---|
341 | 372 | |
---|
.. | .. |
---|
374 | 405 | static const char *___tp_str __tracepoint_string = str; \ |
---|
375 | 406 | ___tp_str; \ |
---|
376 | 407 | }) |
---|
377 | | -#define __tracepoint_string __attribute__((section("__tracepoint_str"), used)) |
---|
| 408 | +#define __tracepoint_string __used __section("__tracepoint_str") |
---|
378 | 409 | #else |
---|
379 | 410 | /* |
---|
380 | 411 | * tracepoint_string() is used to save the string address for userspace |
---|
.. | .. |
---|
384 | 415 | # define tracepoint_string(str) str |
---|
385 | 416 | # define __tracepoint_string |
---|
386 | 417 | #endif |
---|
387 | | - |
---|
388 | | -/* |
---|
389 | | - * The need for the DECLARE_TRACE_NOARGS() is to handle the prototype |
---|
390 | | - * (void). "void" is a special value in a function prototype and can |
---|
391 | | - * not be combined with other arguments. Since the DECLARE_TRACE() |
---|
392 | | - * macro adds a data element at the beginning of the prototype, |
---|
393 | | - * we need a way to differentiate "(void *data, proto)" from |
---|
394 | | - * "(void *data, void)". The second prototype is invalid. |
---|
395 | | - * |
---|
396 | | - * DECLARE_TRACE_NOARGS() passes "void" as the tracepoint prototype |
---|
397 | | - * and "void *__data" as the callback prototype. |
---|
398 | | - * |
---|
399 | | - * DECLARE_TRACE() passes "proto" as the tracepoint protoype and |
---|
400 | | - * "void *__data, proto" as the callback prototype. |
---|
401 | | - */ |
---|
402 | | -#define DECLARE_TRACE_NOARGS(name) \ |
---|
403 | | - __DECLARE_TRACE(name, void, , \ |
---|
404 | | - cpu_online(raw_smp_processor_id()), \ |
---|
405 | | - void *__data, __data) |
---|
406 | 418 | |
---|
407 | 419 | #define DECLARE_TRACE(name, proto, args) \ |
---|
408 | 420 | __DECLARE_TRACE(name, PARAMS(proto), PARAMS(args), \ |
---|
.. | .. |
---|
558 | 570 | |
---|
559 | 571 | #define TRACE_EVENT_PERF_PERM(event, expr...) |
---|
560 | 572 | |
---|
| 573 | +#define DECLARE_EVENT_NOP(name, proto, args) \ |
---|
| 574 | + static inline void trace_##name(proto) \ |
---|
| 575 | + { } \ |
---|
| 576 | + static inline bool trace_##name##_enabled(void) \ |
---|
| 577 | + { \ |
---|
| 578 | + return false; \ |
---|
| 579 | + } |
---|
| 580 | + |
---|
| 581 | +#define TRACE_EVENT_NOP(name, proto, args, struct, assign, print) \ |
---|
| 582 | + DECLARE_EVENT_NOP(name, PARAMS(proto), PARAMS(args)) |
---|
| 583 | + |
---|
| 584 | +#define DECLARE_EVENT_CLASS_NOP(name, proto, args, tstruct, assign, print) |
---|
| 585 | +#define DEFINE_EVENT_NOP(template, name, proto, args) \ |
---|
| 586 | + DECLARE_EVENT_NOP(name, PARAMS(proto), PARAMS(args)) |
---|
| 587 | + |
---|
561 | 588 | #endif /* ifdef TRACE_EVENT (see note above) */ |
---|