| .. | .. |
|---|
| 1 | +/* SPDX-License-Identifier: GPL-2.0-only */ |
|---|
| 1 | 2 | #ifndef _LINUX_TRACEPOINT_H |
|---|
| 2 | 3 | #define _LINUX_TRACEPOINT_H |
|---|
| 3 | 4 | |
|---|
| .. | .. |
|---|
| 9 | 10 | * Copyright (C) 2008-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
|---|
| 10 | 11 | * |
|---|
| 11 | 12 | * Heavily inspired from the Linux Kernel Markers. |
|---|
| 12 | | - * |
|---|
| 13 | | - * This file is released under the GPLv2. |
|---|
| 14 | | - * See the file COPYING for more details. |
|---|
| 15 | 13 | */ |
|---|
| 16 | 14 | |
|---|
| 17 | 15 | #include <linux/smp.h> |
|---|
| .. | .. |
|---|
| 21 | 19 | #include <linux/cpumask.h> |
|---|
| 22 | 20 | #include <linux/rcupdate.h> |
|---|
| 23 | 21 | #include <linux/tracepoint-defs.h> |
|---|
| 22 | +#include <linux/static_call.h> |
|---|
| 24 | 23 | |
|---|
| 25 | 24 | struct module; |
|---|
| 26 | 25 | struct tracepoint; |
|---|
| .. | .. |
|---|
| 92 | 91 | static inline void tracepoint_synchronize_unregister(void) |
|---|
| 93 | 92 | { |
|---|
| 94 | 93 | synchronize_srcu(&tracepoint_srcu); |
|---|
| 95 | | - synchronize_sched(); |
|---|
| 94 | + synchronize_rcu(); |
|---|
| 96 | 95 | } |
|---|
| 97 | 96 | #else |
|---|
| 98 | 97 | static inline void tracepoint_synchronize_unregister(void) |
|---|
| .. | .. |
|---|
| 104 | 103 | extern void syscall_unregfunc(void); |
|---|
| 105 | 104 | #endif /* CONFIG_HAVE_SYSCALL_TRACEPOINTS */ |
|---|
| 106 | 105 | |
|---|
| 106 | +#ifndef PARAMS |
|---|
| 107 | 107 | #define PARAMS(args...) args |
|---|
| 108 | +#endif |
|---|
| 108 | 109 | |
|---|
| 109 | 110 | #define TRACE_DEFINE_ENUM(x) |
|---|
| 110 | 111 | #define TRACE_DEFINE_SIZEOF(x) |
|---|
| .. | .. |
|---|
| 128 | 129 | |
|---|
| 129 | 130 | #define __TRACEPOINT_ENTRY(name) \ |
|---|
| 130 | 131 | static tracepoint_ptr_t __tracepoint_ptr_##name __used \ |
|---|
| 131 | | - __attribute__((section("__tracepoints_ptrs"))) = \ |
|---|
| 132 | | - &__tracepoint_##name |
|---|
| 132 | + __section("__tracepoints_ptrs") = &__tracepoint_##name |
|---|
| 133 | 133 | #endif |
|---|
| 134 | 134 | |
|---|
| 135 | 135 | #endif /* _LINUX_TRACEPOINT_H */ |
|---|
| .. | .. |
|---|
| 161 | 161 | |
|---|
| 162 | 162 | #ifdef TRACEPOINTS_ENABLED |
|---|
| 163 | 163 | |
|---|
| 164 | +#ifdef CONFIG_HAVE_STATIC_CALL |
|---|
| 165 | +#define __DO_TRACE_CALL(name) static_call(tp_func_##name) |
|---|
| 166 | +#else |
|---|
| 167 | +#define __DO_TRACE_CALL(name) __traceiter_##name |
|---|
| 168 | +#endif /* CONFIG_HAVE_STATIC_CALL */ |
|---|
| 169 | + |
|---|
| 164 | 170 | /* |
|---|
| 165 | 171 | * it_func[0] is never NULL because there is at least one element in the array |
|---|
| 166 | 172 | * when the array itself is non NULL. |
|---|
| .. | .. |
|---|
| 168 | 174 | * Note, the proto and args passed in includes "__data" as the first parameter. |
|---|
| 169 | 175 | * The reason for this is to handle the "void" prototype. If a tracepoint |
|---|
| 170 | 176 | * has a "void" prototype, then it is invalid to declare a function |
|---|
| 171 | | - * as "(void *, void)". The DECLARE_TRACE_NOARGS() will pass in just |
|---|
| 172 | | - * "void *data", where as the DECLARE_TRACE() will pass in "void *data, proto". |
|---|
| 177 | + * as "(void *, void)". |
|---|
| 173 | 178 | */ |
|---|
| 174 | | -#define __DO_TRACE(tp, proto, args, cond, rcuidle) \ |
|---|
| 179 | +#define __DO_TRACE(name, proto, args, cond, rcuidle) \ |
|---|
| 175 | 180 | do { \ |
|---|
| 176 | 181 | struct tracepoint_func *it_func_ptr; \ |
|---|
| 177 | | - void *it_func; \ |
|---|
| 178 | | - void *__data; \ |
|---|
| 179 | 182 | int __maybe_unused __idx = 0; \ |
|---|
| 183 | + void *__data; \ |
|---|
| 180 | 184 | \ |
|---|
| 181 | 185 | if (!(cond)) \ |
|---|
| 182 | 186 | return; \ |
|---|
| .. | .. |
|---|
| 196 | 200 | rcu_irq_enter_irqson(); \ |
|---|
| 197 | 201 | } \ |
|---|
| 198 | 202 | \ |
|---|
| 199 | | - it_func_ptr = rcu_dereference_raw((tp)->funcs); \ |
|---|
| 200 | | - \ |
|---|
| 203 | + it_func_ptr = \ |
|---|
| 204 | + rcu_dereference_raw((&__tracepoint_##name)->funcs); \ |
|---|
| 201 | 205 | if (it_func_ptr) { \ |
|---|
| 202 | | - do { \ |
|---|
| 203 | | - it_func = (it_func_ptr)->func; \ |
|---|
| 204 | | - __data = (it_func_ptr)->data; \ |
|---|
| 205 | | - ((void(*)(proto))(it_func))(args); \ |
|---|
| 206 | | - } while ((++it_func_ptr)->func); \ |
|---|
| 206 | + __data = (it_func_ptr)->data; \ |
|---|
| 207 | + __DO_TRACE_CALL(name)(args); \ |
|---|
| 207 | 208 | } \ |
|---|
| 208 | 209 | \ |
|---|
| 209 | 210 | if (rcuidle) { \ |
|---|
| .. | .. |
|---|
| 219 | 220 | static inline void trace_##name##_rcuidle(proto) \ |
|---|
| 220 | 221 | { \ |
|---|
| 221 | 222 | if (static_key_false(&__tracepoint_##name.key)) \ |
|---|
| 222 | | - __DO_TRACE(&__tracepoint_##name, \ |
|---|
| 223 | + __DO_TRACE(name, \ |
|---|
| 223 | 224 | TP_PROTO(data_proto), \ |
|---|
| 224 | 225 | TP_ARGS(data_args), \ |
|---|
| 225 | 226 | TP_CONDITION(cond), 1); \ |
|---|
| .. | .. |
|---|
| 233 | 234 | * not add unwanted padding between the beginning of the section and the |
|---|
| 234 | 235 | * structure. Force alignment to the same alignment as the section start. |
|---|
| 235 | 236 | * |
|---|
| 236 | | - * When lockdep is enabled, we make sure to always do the RCU portions of |
|---|
| 237 | | - * the tracepoint code, regardless of whether tracing is on. However, |
|---|
| 238 | | - * don't check if the condition is false, due to interaction with idle |
|---|
| 239 | | - * instrumentation. This lets us find RCU issues triggered with tracepoints |
|---|
| 240 | | - * even when this tracepoint is off. This code has no purpose other than |
|---|
| 241 | | - * poking RCU a bit. |
|---|
| 237 | + * When lockdep is enabled, we make sure to always test if RCU is |
|---|
| 238 | + * "watching" regardless if the tracepoint is enabled or not. Tracepoints |
|---|
| 239 | + * require RCU to be active, and it should always warn at the tracepoint |
|---|
| 240 | + * site if it is not watching, as it will need to be active when the |
|---|
| 241 | + * tracepoint is enabled. |
|---|
| 242 | 242 | */ |
|---|
| 243 | 243 | #define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \ |
|---|
| 244 | + extern int __traceiter_##name(data_proto); \ |
|---|
| 245 | + DECLARE_STATIC_CALL(tp_func_##name, __traceiter_##name); \ |
|---|
| 244 | 246 | extern struct tracepoint __tracepoint_##name; \ |
|---|
| 245 | | - static inline void trace_##name(proto) \ |
|---|
| 247 | + static inline void __nocfi trace_##name(proto) \ |
|---|
| 246 | 248 | { \ |
|---|
| 247 | 249 | if (static_key_false(&__tracepoint_##name.key)) \ |
|---|
| 248 | | - __DO_TRACE(&__tracepoint_##name, \ |
|---|
| 250 | + __DO_TRACE(name, \ |
|---|
| 249 | 251 | TP_PROTO(data_proto), \ |
|---|
| 250 | 252 | TP_ARGS(data_args), \ |
|---|
| 251 | 253 | TP_CONDITION(cond), 0); \ |
|---|
| 252 | 254 | if (IS_ENABLED(CONFIG_LOCKDEP) && (cond)) { \ |
|---|
| 253 | | - rcu_read_lock_sched_notrace(); \ |
|---|
| 254 | | - rcu_dereference_sched(__tracepoint_##name.funcs);\ |
|---|
| 255 | | - rcu_read_unlock_sched_notrace(); \ |
|---|
| 255 | + WARN_ON_ONCE(!rcu_is_watching()); \ |
|---|
| 256 | 256 | } \ |
|---|
| 257 | 257 | } \ |
|---|
| 258 | 258 | __DECLARE_TRACE_RCU(name, PARAMS(proto), PARAMS(args), \ |
|---|
| .. | .. |
|---|
| 291 | 291 | * structures, so we create an array of pointers that will be used for iteration |
|---|
| 292 | 292 | * on the tracepoints. |
|---|
| 293 | 293 | */ |
|---|
| 294 | | -#define DEFINE_TRACE_FN(name, reg, unreg) \ |
|---|
| 295 | | - static const char __tpstrtab_##name[] \ |
|---|
| 296 | | - __attribute__((section("__tracepoints_strings"))) = #name; \ |
|---|
| 297 | | - struct tracepoint __tracepoint_##name \ |
|---|
| 298 | | - __attribute__((section("__tracepoints"), used)) = \ |
|---|
| 299 | | - { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\ |
|---|
| 300 | | - __TRACEPOINT_ENTRY(name); |
|---|
| 294 | +#define DEFINE_TRACE_FN(_name, _reg, _unreg, proto, args) \ |
|---|
| 295 | + static const char __tpstrtab_##_name[] \ |
|---|
| 296 | + __section("__tracepoints_strings") = #_name; \ |
|---|
| 297 | + extern struct static_call_key STATIC_CALL_KEY(tp_func_##_name); \ |
|---|
| 298 | + int __traceiter_##_name(void *__data, proto); \ |
|---|
| 299 | + struct tracepoint __tracepoint_##_name __used \ |
|---|
| 300 | + __section("__tracepoints") = { \ |
|---|
| 301 | + .name = __tpstrtab_##_name, \ |
|---|
| 302 | + .key = STATIC_KEY_INIT_FALSE, \ |
|---|
| 303 | + .static_call_key = &STATIC_CALL_KEY(tp_func_##_name), \ |
|---|
| 304 | + .static_call_tramp = STATIC_CALL_TRAMP_ADDR(tp_func_##_name), \ |
|---|
| 305 | + .iterator = &__traceiter_##_name, \ |
|---|
| 306 | + .regfunc = _reg, \ |
|---|
| 307 | + .unregfunc = _unreg, \ |
|---|
| 308 | + .funcs = NULL }; \ |
|---|
| 309 | + __TRACEPOINT_ENTRY(_name); \ |
|---|
| 310 | + int __nocfi __traceiter_##_name(void *__data, proto) \ |
|---|
| 311 | + { \ |
|---|
| 312 | + struct tracepoint_func *it_func_ptr; \ |
|---|
| 313 | + void *it_func; \ |
|---|
| 314 | + \ |
|---|
| 315 | + it_func_ptr = \ |
|---|
| 316 | + rcu_dereference_raw((&__tracepoint_##_name)->funcs); \ |
|---|
| 317 | + if (it_func_ptr) { \ |
|---|
| 318 | + do { \ |
|---|
| 319 | + it_func = (it_func_ptr)->func; \ |
|---|
| 320 | + __data = (it_func_ptr)->data; \ |
|---|
| 321 | + ((void(*)(void *, proto))(it_func))(__data, args); \ |
|---|
| 322 | + } while ((++it_func_ptr)->func); \ |
|---|
| 323 | + } \ |
|---|
| 324 | + return 0; \ |
|---|
| 325 | + } \ |
|---|
| 326 | + DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name); |
|---|
| 301 | 327 | |
|---|
| 302 | | -#define DEFINE_TRACE(name) \ |
|---|
| 303 | | - DEFINE_TRACE_FN(name, NULL, NULL); |
|---|
| 328 | +#define DEFINE_TRACE(name, proto, args) \ |
|---|
| 329 | + DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args)); |
|---|
| 304 | 330 | |
|---|
| 305 | 331 | #define EXPORT_TRACEPOINT_SYMBOL_GPL(name) \ |
|---|
| 306 | | - EXPORT_SYMBOL_GPL(__tracepoint_##name) |
|---|
| 332 | + EXPORT_SYMBOL_GPL(__tracepoint_##name); \ |
|---|
| 333 | + EXPORT_SYMBOL_GPL(__traceiter_##name); \ |
|---|
| 334 | + EXPORT_STATIC_CALL_GPL(tp_func_##name) |
|---|
| 307 | 335 | #define EXPORT_TRACEPOINT_SYMBOL(name) \ |
|---|
| 308 | | - EXPORT_SYMBOL(__tracepoint_##name) |
|---|
| 336 | + EXPORT_SYMBOL(__tracepoint_##name); \ |
|---|
| 337 | + EXPORT_SYMBOL(__traceiter_##name); \ |
|---|
| 338 | + EXPORT_STATIC_CALL(tp_func_##name) |
|---|
| 339 | + |
|---|
| 309 | 340 | |
|---|
| 310 | 341 | #else /* !TRACEPOINTS_ENABLED */ |
|---|
| 311 | 342 | #define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \ |
|---|
| .. | .. |
|---|
| 334 | 365 | return false; \ |
|---|
| 335 | 366 | } |
|---|
| 336 | 367 | |
|---|
| 337 | | -#define DEFINE_TRACE_FN(name, reg, unreg) |
|---|
| 338 | | -#define DEFINE_TRACE(name) |
|---|
| 368 | +#define DEFINE_TRACE_FN(name, reg, unreg, proto, args) |
|---|
| 369 | +#define DEFINE_TRACE(name, proto, args) |
|---|
| 339 | 370 | #define EXPORT_TRACEPOINT_SYMBOL_GPL(name) |
|---|
| 340 | 371 | #define EXPORT_TRACEPOINT_SYMBOL(name) |
|---|
| 341 | 372 | |
|---|
| .. | .. |
|---|
| 374 | 405 | static const char *___tp_str __tracepoint_string = str; \ |
|---|
| 375 | 406 | ___tp_str; \ |
|---|
| 376 | 407 | }) |
|---|
| 377 | | -#define __tracepoint_string __attribute__((section("__tracepoint_str"), used)) |
|---|
| 408 | +#define __tracepoint_string __used __section("__tracepoint_str") |
|---|
| 378 | 409 | #else |
|---|
| 379 | 410 | /* |
|---|
| 380 | 411 | * tracepoint_string() is used to save the string address for userspace |
|---|
| .. | .. |
|---|
| 384 | 415 | # define tracepoint_string(str) str |
|---|
| 385 | 416 | # define __tracepoint_string |
|---|
| 386 | 417 | #endif |
|---|
| 387 | | - |
|---|
| 388 | | -/* |
|---|
| 389 | | - * The need for the DECLARE_TRACE_NOARGS() is to handle the prototype |
|---|
| 390 | | - * (void). "void" is a special value in a function prototype and can |
|---|
| 391 | | - * not be combined with other arguments. Since the DECLARE_TRACE() |
|---|
| 392 | | - * macro adds a data element at the beginning of the prototype, |
|---|
| 393 | | - * we need a way to differentiate "(void *data, proto)" from |
|---|
| 394 | | - * "(void *data, void)". The second prototype is invalid. |
|---|
| 395 | | - * |
|---|
| 396 | | - * DECLARE_TRACE_NOARGS() passes "void" as the tracepoint prototype |
|---|
| 397 | | - * and "void *__data" as the callback prototype. |
|---|
| 398 | | - * |
|---|
| 399 | | - * DECLARE_TRACE() passes "proto" as the tracepoint protoype and |
|---|
| 400 | | - * "void *__data, proto" as the callback prototype. |
|---|
| 401 | | - */ |
|---|
| 402 | | -#define DECLARE_TRACE_NOARGS(name) \ |
|---|
| 403 | | - __DECLARE_TRACE(name, void, , \ |
|---|
| 404 | | - cpu_online(raw_smp_processor_id()), \ |
|---|
| 405 | | - void *__data, __data) |
|---|
| 406 | 418 | |
|---|
| 407 | 419 | #define DECLARE_TRACE(name, proto, args) \ |
|---|
| 408 | 420 | __DECLARE_TRACE(name, PARAMS(proto), PARAMS(args), \ |
|---|
| .. | .. |
|---|
| 558 | 570 | |
|---|
| 559 | 571 | #define TRACE_EVENT_PERF_PERM(event, expr...) |
|---|
| 560 | 572 | |
|---|
| 573 | +#define DECLARE_EVENT_NOP(name, proto, args) \ |
|---|
| 574 | + static inline void trace_##name(proto) \ |
|---|
| 575 | + { } \ |
|---|
| 576 | + static inline bool trace_##name##_enabled(void) \ |
|---|
| 577 | + { \ |
|---|
| 578 | + return false; \ |
|---|
| 579 | + } |
|---|
| 580 | + |
|---|
| 581 | +#define TRACE_EVENT_NOP(name, proto, args, struct, assign, print) \ |
|---|
| 582 | + DECLARE_EVENT_NOP(name, PARAMS(proto), PARAMS(args)) |
|---|
| 583 | + |
|---|
| 584 | +#define DECLARE_EVENT_CLASS_NOP(name, proto, args, tstruct, assign, print) |
|---|
| 585 | +#define DEFINE_EVENT_NOP(template, name, proto, args) \ |
|---|
| 586 | + DECLARE_EVENT_NOP(name, PARAMS(proto), PARAMS(args)) |
|---|
| 587 | + |
|---|
| 561 | 588 | #endif /* ifdef TRACE_EVENT (see note above) */ |
|---|