.. | .. |
---|
7 | 7 | #include <linux/module.h> /* for MODULE_NAME_LEN via KSYM_SYMBOL_LEN */ |
---|
8 | 8 | #include <linux/ftrace.h> |
---|
9 | 9 | #include <linux/perf_event.h> |
---|
| 10 | +#include <linux/xarray.h> |
---|
10 | 11 | #include <asm/syscall.h> |
---|
11 | 12 | |
---|
12 | 13 | #include "trace_output.h" |
---|
.. | .. |
---|
30 | 31 | extern struct syscall_metadata *__start_syscalls_metadata[]; |
---|
31 | 32 | extern struct syscall_metadata *__stop_syscalls_metadata[]; |
---|
32 | 33 | |
---|
| 34 | +static DEFINE_XARRAY(syscalls_metadata_sparse); |
---|
33 | 35 | static struct syscall_metadata **syscalls_metadata; |
---|
34 | 36 | |
---|
35 | 37 | #ifndef ARCH_HAS_SYSCALL_MATCH_SYM_NAME |
---|
.. | .. |
---|
101 | 103 | |
---|
102 | 104 | static struct syscall_metadata *syscall_nr_to_meta(int nr) |
---|
103 | 105 | { |
---|
| 106 | + if (IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR)) |
---|
| 107 | + return xa_load(&syscalls_metadata_sparse, (unsigned long)nr); |
---|
| 108 | + |
---|
104 | 109 | if (!syscalls_metadata || nr >= NR_syscalls || nr < 0) |
---|
105 | 110 | return NULL; |
---|
106 | 111 | |
---|
.. | .. |
---|
198 | 203 | |
---|
199 | 204 | extern char *__bad_type_size(void); |
---|
200 | 205 | |
---|
201 | | -#define SYSCALL_FIELD(type, field, name) \ |
---|
202 | | - sizeof(type) != sizeof(trace.field) ? \ |
---|
203 | | - __bad_type_size() : \ |
---|
204 | | - #type, #name, offsetof(typeof(trace), field), \ |
---|
205 | | - sizeof(trace.field), is_signed_type(type) |
---|
| 206 | +#define SYSCALL_FIELD(_type, _name) { \ |
---|
| 207 | + .type = #_type, .name = #_name, \ |
---|
| 208 | + .size = sizeof(_type), .align = __alignof__(_type), \ |
---|
| 209 | + .is_signed = is_signed_type(_type), .filter_type = FILTER_OTHER } |
---|
206 | 210 | |
---|
207 | 211 | static int __init |
---|
208 | 212 | __set_enter_print_fmt(struct syscall_metadata *entry, char *buf, int len) |
---|
.. | .. |
---|
269 | 273 | { |
---|
270 | 274 | struct syscall_trace_enter trace; |
---|
271 | 275 | struct syscall_metadata *meta = call->data; |
---|
272 | | - int ret; |
---|
273 | | - int i; |
---|
274 | 276 | int offset = offsetof(typeof(trace), args); |
---|
275 | | - |
---|
276 | | - ret = trace_define_field(call, SYSCALL_FIELD(int, nr, __syscall_nr), |
---|
277 | | - FILTER_OTHER); |
---|
278 | | - if (ret) |
---|
279 | | - return ret; |
---|
| 277 | + int ret = 0; |
---|
| 278 | + int i; |
---|
280 | 279 | |
---|
281 | 280 | for (i = 0; i < meta->nb_args; i++) { |
---|
282 | 281 | ret = trace_define_field(call, meta->types[i], |
---|
283 | 282 | meta->args[i], offset, |
---|
284 | 283 | sizeof(unsigned long), 0, |
---|
285 | 284 | FILTER_OTHER); |
---|
| 285 | + if (ret) |
---|
| 286 | + break; |
---|
286 | 287 | offset += sizeof(unsigned long); |
---|
287 | 288 | } |
---|
288 | | - |
---|
289 | | - return ret; |
---|
290 | | -} |
---|
291 | | - |
---|
292 | | -static int __init syscall_exit_define_fields(struct trace_event_call *call) |
---|
293 | | -{ |
---|
294 | | - struct syscall_trace_exit trace; |
---|
295 | | - int ret; |
---|
296 | | - |
---|
297 | | - ret = trace_define_field(call, SYSCALL_FIELD(int, nr, __syscall_nr), |
---|
298 | | - FILTER_OTHER); |
---|
299 | | - if (ret) |
---|
300 | | - return ret; |
---|
301 | | - |
---|
302 | | - ret = trace_define_field(call, SYSCALL_FIELD(long, ret, ret), |
---|
303 | | - FILTER_OTHER); |
---|
304 | 289 | |
---|
305 | 290 | return ret; |
---|
306 | 291 | } |
---|
.. | .. |
---|
312 | 297 | struct syscall_trace_enter *entry; |
---|
313 | 298 | struct syscall_metadata *sys_data; |
---|
314 | 299 | struct ring_buffer_event *event; |
---|
315 | | - struct ring_buffer *buffer; |
---|
| 300 | + struct trace_buffer *buffer; |
---|
316 | 301 | unsigned long irq_flags; |
---|
| 302 | + unsigned long args[6]; |
---|
317 | 303 | int pc; |
---|
318 | 304 | int syscall_nr; |
---|
319 | 305 | int size; |
---|
.. | .. |
---|
339 | 325 | local_save_flags(irq_flags); |
---|
340 | 326 | pc = preempt_count(); |
---|
341 | 327 | |
---|
342 | | - buffer = tr->trace_buffer.buffer; |
---|
| 328 | + buffer = tr->array_buffer.buffer; |
---|
343 | 329 | event = trace_buffer_lock_reserve(buffer, |
---|
344 | 330 | sys_data->enter_event->event.type, size, irq_flags, pc); |
---|
345 | 331 | if (!event) |
---|
.. | .. |
---|
347 | 333 | |
---|
348 | 334 | entry = ring_buffer_event_data(event); |
---|
349 | 335 | entry->nr = syscall_nr; |
---|
350 | | - syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args); |
---|
| 336 | + syscall_get_arguments(current, regs, args); |
---|
| 337 | + memcpy(entry->args, args, sizeof(unsigned long) * sys_data->nb_args); |
---|
351 | 338 | |
---|
352 | 339 | event_trigger_unlock_commit(trace_file, buffer, event, entry, |
---|
353 | 340 | irq_flags, pc); |
---|
.. | .. |
---|
360 | 347 | struct syscall_trace_exit *entry; |
---|
361 | 348 | struct syscall_metadata *sys_data; |
---|
362 | 349 | struct ring_buffer_event *event; |
---|
363 | | - struct ring_buffer *buffer; |
---|
| 350 | + struct trace_buffer *buffer; |
---|
364 | 351 | unsigned long irq_flags; |
---|
365 | 352 | int pc; |
---|
366 | 353 | int syscall_nr; |
---|
.. | .. |
---|
384 | 371 | local_save_flags(irq_flags); |
---|
385 | 372 | pc = preempt_count(); |
---|
386 | 373 | |
---|
387 | | - buffer = tr->trace_buffer.buffer; |
---|
| 374 | + buffer = tr->array_buffer.buffer; |
---|
388 | 375 | event = trace_buffer_lock_reserve(buffer, |
---|
389 | 376 | sys_data->exit_event->event.type, sizeof(*entry), |
---|
390 | 377 | irq_flags, pc); |
---|
.. | .. |
---|
500 | 487 | return id; |
---|
501 | 488 | } |
---|
502 | 489 | |
---|
| 490 | +static struct trace_event_fields __refdata syscall_enter_fields_array[] = { |
---|
| 491 | + SYSCALL_FIELD(int, __syscall_nr), |
---|
| 492 | + { .type = TRACE_FUNCTION_TYPE, |
---|
| 493 | + .define_fields = syscall_enter_define_fields }, |
---|
| 494 | + {} |
---|
| 495 | +}; |
---|
| 496 | + |
---|
503 | 497 | struct trace_event_functions enter_syscall_print_funcs = { |
---|
504 | 498 | .trace = print_syscall_enter, |
---|
505 | 499 | }; |
---|
.. | .. |
---|
511 | 505 | struct trace_event_class __refdata event_class_syscall_enter = { |
---|
512 | 506 | .system = "syscalls", |
---|
513 | 507 | .reg = syscall_enter_register, |
---|
514 | | - .define_fields = syscall_enter_define_fields, |
---|
| 508 | + .fields_array = syscall_enter_fields_array, |
---|
515 | 509 | .get_fields = syscall_get_enter_fields, |
---|
516 | 510 | .raw_init = init_syscall_trace, |
---|
517 | 511 | }; |
---|
.. | .. |
---|
519 | 513 | struct trace_event_class __refdata event_class_syscall_exit = { |
---|
520 | 514 | .system = "syscalls", |
---|
521 | 515 | .reg = syscall_exit_register, |
---|
522 | | - .define_fields = syscall_exit_define_fields, |
---|
| 516 | + .fields_array = (struct trace_event_fields[]){ |
---|
| 517 | + SYSCALL_FIELD(int, __syscall_nr), |
---|
| 518 | + SYSCALL_FIELD(long, ret), |
---|
| 519 | + {} |
---|
| 520 | + }, |
---|
523 | 521 | .fields = LIST_HEAD_INIT(event_class_syscall_exit.fields), |
---|
524 | 522 | .raw_init = init_syscall_trace, |
---|
525 | 523 | }; |
---|
.. | .. |
---|
534 | 532 | struct syscall_metadata *meta; |
---|
535 | 533 | unsigned long addr; |
---|
536 | 534 | int i; |
---|
| 535 | + void *ret; |
---|
537 | 536 | |
---|
538 | | - syscalls_metadata = kcalloc(NR_syscalls, sizeof(*syscalls_metadata), |
---|
539 | | - GFP_KERNEL); |
---|
540 | | - if (!syscalls_metadata) { |
---|
541 | | - WARN_ON(1); |
---|
542 | | - return; |
---|
| 537 | + if (!IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR)) { |
---|
| 538 | + syscalls_metadata = kcalloc(NR_syscalls, |
---|
| 539 | + sizeof(*syscalls_metadata), |
---|
| 540 | + GFP_KERNEL); |
---|
| 541 | + if (!syscalls_metadata) { |
---|
| 542 | + WARN_ON(1); |
---|
| 543 | + return; |
---|
| 544 | + } |
---|
543 | 545 | } |
---|
544 | 546 | |
---|
545 | 547 | for (i = 0; i < NR_syscalls; i++) { |
---|
.. | .. |
---|
549 | 551 | continue; |
---|
550 | 552 | |
---|
551 | 553 | meta->syscall_nr = i; |
---|
552 | | - syscalls_metadata[i] = meta; |
---|
| 554 | + |
---|
| 555 | + if (!IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR)) { |
---|
| 556 | + syscalls_metadata[i] = meta; |
---|
| 557 | + } else { |
---|
| 558 | + ret = xa_store(&syscalls_metadata_sparse, i, meta, |
---|
| 559 | + GFP_KERNEL); |
---|
| 560 | + WARN(xa_is_err(ret), |
---|
| 561 | + "Syscall memory allocation failed\n"); |
---|
| 562 | + } |
---|
| 563 | + |
---|
553 | 564 | } |
---|
554 | 565 | } |
---|
555 | 566 | |
---|
.. | .. |
---|
583 | 594 | struct syscall_metadata *sys_data; |
---|
584 | 595 | struct syscall_trace_enter *rec; |
---|
585 | 596 | struct hlist_head *head; |
---|
| 597 | + unsigned long args[6]; |
---|
586 | 598 | bool valid_prog_array; |
---|
587 | 599 | int syscall_nr; |
---|
588 | 600 | int rctx; |
---|
.. | .. |
---|
613 | 625 | return; |
---|
614 | 626 | |
---|
615 | 627 | rec->nr = syscall_nr; |
---|
616 | | - syscall_get_arguments(current, regs, 0, sys_data->nb_args, |
---|
617 | | - (unsigned long *)&rec->args); |
---|
| 628 | + syscall_get_arguments(current, regs, args); |
---|
| 629 | + memcpy(&rec->args, args, sizeof(unsigned long) * sys_data->nb_args); |
---|
618 | 630 | |
---|
619 | 631 | if ((valid_prog_array && |
---|
620 | 632 | !perf_call_bpf_enter(sys_data->enter_event, regs, sys_data, rec)) || |
---|