forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-10 093a6c67005148ae32a5c9e4553491b9f5c2457b 
 kernel/include/linux/ftrace.h
....@@ -51,15 +51,13 @@
5151 
5252 struct module;
5353 struct ftrace_hash;
54
+struct ftrace_direct_func;
5455 
5556 #if defined(CONFIG_FUNCTION_TRACER) && defined(CONFIG_MODULES) && \
5657 	defined(CONFIG_DYNAMIC_FTRACE)
5758 const char *
5859 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
5960 		   unsigned long *off, char **modname, char *sym);
60
-int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
61
-			   char *type, char *name,
62
-			   char *module_name, int *exported);
6361 #else
6462 static inline const char *
6563 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
....@@ -67,6 +65,13 @@
6765 {
6866 	return NULL;
6967 }
68
+#endif
69
+
70
+#if defined(CONFIG_FUNCTION_TRACER) && defined(CONFIG_DYNAMIC_FTRACE)
71
+int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
72
+			   char *type, char *name,
73
+			   char *module_name, int *exported);
74
+#else
7075 static inline int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
7176 					 char *type, char *name,
7277 					 char *module_name, int *exported)
....@@ -75,14 +80,12 @@
7580 }
7681 #endif
7782 
78
-
7983 #ifdef CONFIG_FUNCTION_TRACER
8084 
8185 extern int ftrace_enabled;
8286 extern int
8387 ftrace_enable_sysctl(struct ctl_table *table, int write,
84
-		     void __user *buffer, size_t *lenp,
85
-		     loff_t *ppos);
88
+		     void *buffer, size_t *lenp, loff_t *ppos);
8689 
8790 struct ftrace_ops;
8891 
....@@ -142,24 +145,30 @@
142145  * PID     - Is affected by set_ftrace_pid (allows filtering on those pids)
143146  * RCU     - Set when the ops can only be called when RCU is watching.
144147  * TRACE_ARRAY - The ops->private points to a trace_array descriptor.
148
+ * PERMANENT - Set when the ops is permanent and should not be affected by
149
+ *             ftrace_enabled.
150
+ * DIRECT - Used by the direct ftrace_ops helper for direct functions
151
+ *            (internal ftrace only, should not be used by others)
145152  */
146153 enum {
147
-	FTRACE_OPS_FL_ENABLED			= 1 << 0,
148
-	FTRACE_OPS_FL_DYNAMIC			= 1 << 1,
149
-	FTRACE_OPS_FL_SAVE_REGS			= 1 << 2,
150
-	FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED	= 1 << 3,
151
-	FTRACE_OPS_FL_RECURSION_SAFE		= 1 << 4,
152
-	FTRACE_OPS_FL_STUB			= 1 << 5,
153
-	FTRACE_OPS_FL_INITIALIZED		= 1 << 6,
154
-	FTRACE_OPS_FL_DELETED			= 1 << 7,
155
-	FTRACE_OPS_FL_ADDING			= 1 << 8,
156
-	FTRACE_OPS_FL_REMOVING			= 1 << 9,
157
-	FTRACE_OPS_FL_MODIFYING			= 1 << 10,
158
-	FTRACE_OPS_FL_ALLOC_TRAMP		= 1 << 11,
159
-	FTRACE_OPS_FL_IPMODIFY			= 1 << 12,
160
-	FTRACE_OPS_FL_PID			= 1 << 13,
161
-	FTRACE_OPS_FL_RCU			= 1 << 14,
162
-	FTRACE_OPS_FL_TRACE_ARRAY		= 1 << 15,
154
+	FTRACE_OPS_FL_ENABLED			= BIT(0),
155
+	FTRACE_OPS_FL_DYNAMIC			= BIT(1),
156
+	FTRACE_OPS_FL_SAVE_REGS			= BIT(2),
157
+	FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED	= BIT(3),
158
+	FTRACE_OPS_FL_RECURSION_SAFE		= BIT(4),
159
+	FTRACE_OPS_FL_STUB			= BIT(5),
160
+	FTRACE_OPS_FL_INITIALIZED		= BIT(6),
161
+	FTRACE_OPS_FL_DELETED			= BIT(7),
162
+	FTRACE_OPS_FL_ADDING			= BIT(8),
163
+	FTRACE_OPS_FL_REMOVING			= BIT(9),
164
+	FTRACE_OPS_FL_MODIFYING			= BIT(10),
165
+	FTRACE_OPS_FL_ALLOC_TRAMP		= BIT(11),
166
+	FTRACE_OPS_FL_IPMODIFY			= BIT(12),
167
+	FTRACE_OPS_FL_PID			= BIT(13),
168
+	FTRACE_OPS_FL_RCU			= BIT(14),
169
+	FTRACE_OPS_FL_TRACE_ARRAY		= BIT(15),
170
+	FTRACE_OPS_FL_PERMANENT                 = BIT(16),
171
+	FTRACE_OPS_FL_DIRECT			= BIT(17),
163172 };
164173 
165174 #ifdef CONFIG_DYNAMIC_FTRACE
....@@ -200,8 +209,32 @@
200209 	struct ftrace_ops_hash		old_hash;
201210 	unsigned long			trampoline;
202211 	unsigned long			trampoline_size;
212
+	struct list_head		list;
203213 #endif
204214 };
215
+
216
+extern struct ftrace_ops __rcu *ftrace_ops_list;
217
+extern struct ftrace_ops ftrace_list_end;
218
+
219
+/*
220
+ * Traverse the ftrace_ops_list, invoking all entries.  The reason that we
221
+ * can use rcu_dereference_raw_check() is that elements removed from this list
222
+ * are simply leaked, so there is no need to interact with a grace-period
223
+ * mechanism.  The rcu_dereference_raw_check() calls are needed to handle
224
+ * concurrent insertions into the ftrace_ops_list.
225
+ *
226
+ * Silly Alpha and silly pointer-speculation compiler optimizations!
227
+ */
228
+#define do_for_each_ftrace_op(op, list)			\
229
+	op = rcu_dereference_raw_check(list);			\
230
+	do
231
+
232
+/*
233
+ * Optimized for just a single item in the list (as that is the normal case).
234
+ */
235
+#define while_for_each_ftrace_op(op)				\
236
+	while (likely(op = rcu_dereference_raw_check((op)->next)) &&	\
237
+	       unlikely((op) != &ftrace_list_end))
205238 
206239 /*
207240  * Type of the current tracing.
....@@ -224,16 +257,8 @@
224257 int register_ftrace_function(struct ftrace_ops *ops);
225258 int unregister_ftrace_function(struct ftrace_ops *ops);
226259 
227
-#ifdef CONFIG_CFI_CLANG
228
-/* Use a C stub with the correct type for CFI */
229
-static inline void ftrace_stub(unsigned long a0, unsigned long a1,
230
-			       struct ftrace_ops *op, struct pt_regs *regs)
231
-{
232
-}
233
-#else
234260 extern void ftrace_stub(unsigned long a0, unsigned long a1,
235261 			struct ftrace_ops *op, struct pt_regs *regs);
236
-#endif
237262 
238263 #else /* !CONFIG_FUNCTION_TRACER */
239264 /*
....@@ -247,23 +272,81 @@
247272 static inline void ftrace_free_mem(struct module *mod, void *start, void *end) { }
248273 #endif /* CONFIG_FUNCTION_TRACER */
249274 
275
+struct ftrace_func_entry {
276
+	struct hlist_node hlist;
277
+	unsigned long ip;
278
+	unsigned long direct; /* for direct lookup only */
279
+};
280
+
281
+struct dyn_ftrace;
282
+
283
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
284
+extern int ftrace_direct_func_count;
285
+int register_ftrace_direct(unsigned long ip, unsigned long addr);
286
+int unregister_ftrace_direct(unsigned long ip, unsigned long addr);
287
+int modify_ftrace_direct(unsigned long ip, unsigned long old_addr, unsigned long new_addr);
288
+struct ftrace_direct_func *ftrace_find_direct_func(unsigned long addr);
289
+int ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
290
+				struct dyn_ftrace *rec,
291
+				unsigned long old_addr,
292
+				unsigned long new_addr);
293
+unsigned long ftrace_find_rec_direct(unsigned long ip);
294
+#else
295
+# define ftrace_direct_func_count 0
296
+static inline int register_ftrace_direct(unsigned long ip, unsigned long addr)
297
+{
298
+	return -ENOTSUPP;
299
+}
300
+static inline int unregister_ftrace_direct(unsigned long ip, unsigned long addr)
301
+{
302
+	return -ENOTSUPP;
303
+}
304
+static inline int modify_ftrace_direct(unsigned long ip,
305
+				       unsigned long old_addr, unsigned long new_addr)
306
+{
307
+	return -ENOTSUPP;
308
+}
309
+static inline struct ftrace_direct_func *ftrace_find_direct_func(unsigned long addr)
310
+{
311
+	return NULL;
312
+}
313
+static inline int ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
314
+					      struct dyn_ftrace *rec,
315
+					      unsigned long old_addr,
316
+					      unsigned long new_addr)
317
+{
318
+	return -ENODEV;
319
+}
320
+static inline unsigned long ftrace_find_rec_direct(unsigned long ip)
321
+{
322
+	return 0;
323
+}
324
+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
325
+
326
+#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
327
+/*
328
+ * This must be implemented by the architecture.
329
+ * It is the way the ftrace direct_ops helper, when called
330
+ * via ftrace (because there's other callbacks besides the
331
+ * direct call), can inform the architecture's trampoline that this
332
+ * routine has a direct caller, and what the caller is.
333
+ *
334
+ * For example, in x86, it returns the direct caller
335
+ * callback function via the regs->orig_ax parameter.
336
+ * Then in the ftrace trampoline, if this is set, it makes
337
+ * the return from the trampoline jump to the direct caller
338
+ * instead of going back to the function it just traced.
339
+ */
340
+static inline void arch_ftrace_set_direct_caller(struct pt_regs *regs,
341
+						 unsigned long addr) { }
342
+#endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
343
+
250344 #ifdef CONFIG_STACK_TRACER
251345 
252
-#define STACK_TRACE_ENTRIES 500
253
-
254
-struct stack_trace;
255
-
256
-extern unsigned stack_trace_index[];
257
-extern struct stack_trace stack_trace_max;
258
-extern unsigned long stack_trace_max_size;
259
-extern arch_spinlock_t stack_trace_max_lock;
260
-
261346 extern int stack_tracer_enabled;
262
-void stack_trace_print(void);
263
-int
264
-stack_trace_sysctl(struct ctl_table *table, int write,
265
-		   void __user *buffer, size_t *lenp,
266
-		   loff_t *ppos);
347
+
348
+int stack_trace_sysctl(struct ctl_table *table, int write, void *buffer,
349
+		       size_t *lenp, loff_t *ppos);
267350 
268351 /* DO NOT MODIFY THIS VARIABLE DIRECTLY! */
269352 DECLARE_PER_CPU(int, disable_stack_tracer);
....@@ -309,8 +392,6 @@
309392 int ftrace_arch_code_modify_prepare(void);
310393 int ftrace_arch_code_modify_post_process(void);
311394 
312
-struct dyn_ftrace;
313
-
314395 enum ftrace_bug_type {
315396 	FTRACE_BUG_UNKNOWN,
316397 	FTRACE_BUG_INIT,
....@@ -348,9 +429,10 @@
348429  *  REGS_EN - the function is set up to save regs.
349430  *  IPMODIFY - the record allows for the IP address to be changed.
350431  *  DISABLED - the record is not ready to be touched yet
432
+ *  DIRECT   - there is a direct function to call
351433  *
352434  * When a new ftrace_ops is registered and wants a function to save
353
- * pt_regs, the rec->flag REGS is set. When the function has been
435
+ * pt_regs, the rec->flags REGS is set. When the function has been
354436  * set up to save regs, the REG_EN flag is set. Once a function
355437  * starts saving regs it will do so until all ftrace_ops are removed
356438  * from tracing that function.
....@@ -363,15 +445,14 @@
363445 	FTRACE_FL_TRAMP_EN	= (1UL << 27),
364446 	FTRACE_FL_IPMODIFY	= (1UL << 26),
365447 	FTRACE_FL_DISABLED	= (1UL << 25),
448
+	FTRACE_FL_DIRECT	= (1UL << 24),
449
+	FTRACE_FL_DIRECT_EN	= (1UL << 23),
366450 };
367451 
368
-#define FTRACE_REF_MAX_SHIFT	25
369
-#define FTRACE_FL_BITS		7
370
-#define FTRACE_FL_MASKED_BITS	((1UL << FTRACE_FL_BITS) - 1)
371
-#define FTRACE_FL_MASK		(FTRACE_FL_MASKED_BITS << FTRACE_REF_MAX_SHIFT)
452
+#define FTRACE_REF_MAX_SHIFT	23
372453 #define FTRACE_REF_MAX		((1UL << FTRACE_REF_MAX_SHIFT) - 1)
373454 
374
-#define ftrace_rec_count(rec)	((rec)->flags & ~FTRACE_FL_MASK)
455
+#define ftrace_rec_count(rec)	((rec)->flags & FTRACE_REF_MAX)
375456 
376457 struct dyn_ftrace {
377458 	unsigned long		ip; /* address of mcount call-site */
....@@ -397,6 +478,7 @@
397478 	FTRACE_UPDATE_TRACE_FUNC	= (1 << 2),
398479 	FTRACE_START_FUNC_RET		= (1 << 3),
399480 	FTRACE_STOP_FUNC_RET		= (1 << 4),
481
+	FTRACE_MAY_SLEEP		= (1 << 5),
400482 };
401483 
402484 /*
....@@ -428,6 +510,9 @@
428510 };
429511 
430512 void arch_ftrace_update_code(int command);
513
+void arch_ftrace_update_trampoline(struct ftrace_ops *ops);
514
+void *arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec);
515
+void arch_ftrace_trampoline_free(struct ftrace_ops *ops);
431516 
432517 struct ftrace_rec_iter;
433518 
....@@ -441,8 +526,8 @@
441526 	     iter = ftrace_rec_iter_next(iter))
442527 
443528 
444
-int ftrace_update_record(struct dyn_ftrace *rec, int enable);
445
-int ftrace_test_record(struct dyn_ftrace *rec, int enable);
529
+int ftrace_update_record(struct dyn_ftrace *rec, bool enable);
530
+int ftrace_test_record(struct dyn_ftrace *rec, bool enable);
446531 void ftrace_run_stop_machine(int command);
447532 unsigned long ftrace_location(unsigned long ip);
448533 unsigned long ftrace_location_range(unsigned long start, unsigned long end);
....@@ -513,7 +598,7 @@
513598 /**
514599  * ftrace_make_nop - convert code into nop
515600  * @mod: module structure if called by module load initialization
516
- * @rec: the mcount call site record
601
+ * @rec: the call site record (e.g. mcount/fentry)
517602  * @addr: the address that the call site should be calling
518603  *
519604  * This is a very sensitive operation and great care needs
....@@ -534,9 +619,38 @@
534619 extern int ftrace_make_nop(struct module *mod,
535620 			   struct dyn_ftrace *rec, unsigned long addr);
536621 
622
+
623
+/**
624
+ * ftrace_init_nop - initialize a nop call site
625
+ * @mod: module structure if called by module load initialization
626
+ * @rec: the call site record (e.g. mcount/fentry)
627
+ *
628
+ * This is a very sensitive operation and great care needs
629
+ * to be taken by the arch.  The operation should carefully
630
+ * read the location, check to see if what is read is indeed
631
+ * what we expect it to be, and then on success of the compare,
632
+ * it should write to the location.
633
+ *
634
+ * The code segment at @rec->ip should contain the contents created by
635
+ * the compiler
636
+ *
637
+ * Return must be:
638
+ *  0 on success
639
+ *  -EFAULT on error reading the location
640
+ *  -EINVAL on a failed compare of the contents
641
+ *  -EPERM  on error writing to the location
642
+ * Any other value will be considered a failure.
643
+ */
644
+#ifndef ftrace_init_nop
645
+static inline int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
646
+{
647
+	return ftrace_make_nop(mod, rec, MCOUNT_ADDR);
648
+}
649
+#endif
650
+
537651 /**
538652  * ftrace_make_call - convert a nop call site into a call to addr
539
- * @rec: the mcount call site record
653
+ * @rec: the call site record (e.g. mcount/fentry)
540654  * @addr: the address that the call site should call
541655  *
542656  * This is a very sensitive operation and great care needs
....@@ -559,7 +673,7 @@
559673 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
560674 /**
561675  * ftrace_modify_call - convert from one addr to another (no nop)
562
- * @rec: the mcount call site record
676
+ * @rec: the call site record (e.g. mcount/fentry)
563677  * @old_addr: the address expected to be currently called to
564678  * @addr: the address to change to
565679  *
....@@ -697,7 +811,7 @@
697811 #define CALLER_ADDR5 ((unsigned long)ftrace_return_address(5))
698812 #define CALLER_ADDR6 ((unsigned long)ftrace_return_address(6))
699813 
700
-static inline unsigned long get_lock_parent_ip(void)
814
+static __always_inline unsigned long get_lock_parent_ip(void)
701815 {
702816 	unsigned long addr = CALLER_ADDR0;
703817 
....@@ -723,6 +837,11 @@
723837 
724838 #ifdef CONFIG_FTRACE_MCOUNT_RECORD
725839 extern void ftrace_init(void);
840
+#ifdef CC_USING_PATCHABLE_FUNCTION_ENTRY
841
+#define FTRACE_CALLSITE_SECTION	"__patchable_function_entries"
842
+#else
843
+#define FTRACE_CALLSITE_SECTION	"__mcount_loc"
844
+#endif
726845 #else
727846 static inline void ftrace_init(void) { }
728847 #endif
....@@ -755,7 +874,14 @@
755874 typedef void (*trace_func_graph_ret_t)(struct ftrace_graph_ret *); /* return */
756875 typedef int (*trace_func_graph_ent_t)(struct ftrace_graph_ent *); /* entry */
757876 
877
+extern int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace);
878
+
758879 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
880
+
881
+struct fgraph_ops {
882
+	trace_func_graph_ent_t		entryfunc;
883
+	trace_func_graph_ret_t		retfunc;
884
+};
759885 
760886 /*
761887  * Stack of return addresses for functions
....@@ -788,6 +914,9 @@
788914 function_graph_enter(unsigned long ret, unsigned long func,
789915 		     unsigned long frame_pointer, unsigned long *retp);
790916 
917
+struct ftrace_ret_stack *
918
+ftrace_graph_get_ret_stack(struct task_struct *task, int idx);
919
+
791920 unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
792921 				    unsigned long ret, unsigned long *retp);
793922 
....@@ -798,11 +927,11 @@
798927  */
799928 #define __notrace_funcgraph		notrace
800929 
801
-#define FTRACE_NOTRACE_DEPTH 65536
802930 #define FTRACE_RETFUNC_DEPTH 50
803931 #define FTRACE_RETSTACK_ALLOC_SIZE 32
804
-extern int register_ftrace_graph(trace_func_graph_ret_t retfunc,
805
-				trace_func_graph_ent_t entryfunc);
932
+
933
+extern int register_ftrace_graph(struct fgraph_ops *ops);
934
+extern void unregister_ftrace_graph(struct fgraph_ops *ops);
806935 
807936 extern bool ftrace_graph_is_dead(void);
808937 extern void ftrace_graph_stop(void);
....@@ -811,16 +940,9 @@
811940 extern trace_func_graph_ret_t ftrace_graph_return;
812941 extern trace_func_graph_ent_t ftrace_graph_entry;
813942 
814
-extern void unregister_ftrace_graph(void);
815
-
816943 extern void ftrace_graph_init_task(struct task_struct *t);
817944 extern void ftrace_graph_exit_task(struct task_struct *t);
818945 extern void ftrace_graph_init_idle_task(struct task_struct *t, int cpu);
819
-
820
-static inline int task_curr_ret_stack(struct task_struct *t)
821
-{
822
-	return t->curr_ret_stack;
823
-}
824946 
825947 static inline void pause_graph_tracing(void)
826948 {
....@@ -839,17 +961,9 @@
839961 static inline void ftrace_graph_exit_task(struct task_struct *t) { }
840962 static inline void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) { }
841963 
842
-static inline int register_ftrace_graph(trace_func_graph_ret_t retfunc,
843
-			  trace_func_graph_ent_t entryfunc)
844
-{
845
-	return -1;
846
-}
847
-static inline void unregister_ftrace_graph(void) { }
848
-
849
-static inline int task_curr_ret_stack(struct task_struct *tsk)
850
-{
851
-	return -1;
852
-}
964
+/* Define as macros as fgraph_ops may not be defined */
965
+#define register_ftrace_graph(ops) ({ -1; })
966
+#define unregister_ftrace_graph(ops) do { } while (0)
853967 
854968 static inline unsigned long
855969 ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret,
....@@ -913,8 +1027,7 @@
9131027 extern int __disable_trace_on_warning;
9141028 
9151029 int tracepoint_printk_sysctl(struct ctl_table *table, int write,
916
-			     void __user *buffer, size_t *lenp,
917
-			     loff_t *ppos);
1030
+			     void *buffer, size_t *lenp, loff_t *ppos);
9181031 
9191032 #else /* CONFIG_TRACING */
9201033 static inline void  disable_trace_on_warning(void) { }