~hc/RK356X_SDK_RELEASE.git

..	..	@@ -7,27 +7,78 @@
7	7	*/
8	8	#define pr_fmt(fmt) "trace_kprobe: " fmt
9	9
	10	+#include <linux/security.h>
10	11	#include <linux/module.h>
11	12	#include <linux/uaccess.h>
12	13	#include <linux/rculist.h>
13	14	#include <linux/error-injection.h>
14	15
	16	+#include <asm/setup.h> /* for COMMAND_LINE_SIZE */
	17	+
	18	+#include "trace_dynevent.h"
15	19	#include "trace_kprobe_selftest.h"
16	20	#include "trace_probe.h"
	21	+#include "trace_probe_tmpl.h"
17	22
18	23	#define KPROBE_EVENT_SYSTEM "kprobes"
19	24	#define KRETPROBE_MAXACTIVE_MAX 4096
20	25
21		-/**
	26	+/* Kprobe early definition from command line */
	27	+static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata;
	28	+
	29	+static int __init set_kprobe_boot_events(char *str)
	30	+{
	31	+ strlcpy(kprobe_boot_events_buf, str, COMMAND_LINE_SIZE);
	32	+ disable_tracing_selftest("running kprobe events");
	33	+
	34	+ return 1;
	35	+}
	36	+__setup("kprobe_event=", set_kprobe_boot_events);
	37	+
	38	+static int trace_kprobe_create(int argc, const char **argv);
	39	+static int trace_kprobe_show(struct seq_file m, struct dyn_event ev);
	40	+static int trace_kprobe_release(struct dyn_event *ev);
	41	+static bool trace_kprobe_is_busy(struct dyn_event *ev);
	42	+static bool trace_kprobe_match(const char system, const char event,
	43	+ int argc, const char *argv, struct dyn_event ev);
	44	+
	45	+static struct dyn_event_operations trace_kprobe_ops = {
	46	+ .create = trace_kprobe_create,
	47	+ .show = trace_kprobe_show,
	48	+ .is_busy = trace_kprobe_is_busy,
	49	+ .free = trace_kprobe_release,
	50	+ .match = trace_kprobe_match,
	51	+};
	52	+
	53	+/*
22	54	* Kprobe event core functions
23	55	*/
24	56	struct trace_kprobe {
25		- struct list_head list;
	57	+ struct dyn_event devent;
26	58	struct kretprobe rp; /* Use rp.kp for kprobe use */
27	59	unsigned long __percpu *nhit;
28	60	const char symbol; / symbol name */
29	61	struct trace_probe tp;
30	62	};
	63	+
	64	+static bool is_trace_kprobe(struct dyn_event *ev)
	65	+{
	66	+ return ev->ops == &trace_kprobe_ops;
	67	+}
	68	+
	69	+static struct trace_kprobe to_trace_kprobe(struct dyn_event ev)
	70	+{
	71	+ return container_of(ev, struct trace_kprobe, devent);
	72	+}
	73	+
	74	+/**
	75	+ * for_each_trace_kprobe - iterate over the trace_kprobe list
	76	+ * @pos: the struct trace_kprobe * for each entry
	77	+ * @dpos: the struct dyn_event * to use as a loop cursor
	78	+ */
	79	+#define for_each_trace_kprobe(pos, dpos) \
	80	+ for_each_dyn_event(dpos) \
	81	+ if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos)))
31	82
32	83	#define SIZEOF_TRACE_KPROBE(n) \
33	84	(offsetof(struct trace_kprobe, tp.args) + \
..	..	@@ -56,9 +107,10 @@
56	107	static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
57	108	struct module *mod)
58	109	{
59		- int len = strlen(mod->name);
	110	+ int len = strlen(module_name(mod));
60	111	const char *name = trace_kprobe_symbol(tk);
61		- return strncmp(mod->name, name, len) == 0 && name[len] == ':';
	112	+
	113	+ return strncmp(module_name(mod), name, len) == 0 && name[len] == ':';
62	114	}
63	115
64	116	static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
..	..	@@ -80,6 +132,45 @@
80	132	return ret;
81	133	}
82	134
	135	+static bool trace_kprobe_is_busy(struct dyn_event *ev)
	136	+{
	137	+ struct trace_kprobe *tk = to_trace_kprobe(ev);
	138	+
	139	+ return trace_probe_is_enabled(&tk->tp);
	140	+}
	141	+
	142	+static bool trace_kprobe_match_command_head(struct trace_kprobe *tk,
	143	+ int argc, const char **argv)
	144	+{
	145	+ char buf[MAX_ARGSTR_LEN + 1];
	146	+
	147	+ if (!argc)
	148	+ return true;
	149	+
	150	+ if (!tk->symbol)
	151	+ snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr);
	152	+ else if (tk->rp.kp.offset)
	153	+ snprintf(buf, sizeof(buf), "%s+%u",
	154	+ trace_kprobe_symbol(tk), tk->rp.kp.offset);
	155	+ else
	156	+ snprintf(buf, sizeof(buf), "%s", trace_kprobe_symbol(tk));
	157	+ if (strcmp(buf, argv[0]))
	158	+ return false;
	159	+ argc--; argv++;
	160	+
	161	+ return trace_probe_match_command_args(&tk->tp, argc, argv);
	162	+}
	163	+
	164	+static bool trace_kprobe_match(const char system, const char event,
	165	+ int argc, const char *argv, struct dyn_event ev)
	166	+{
	167	+ struct trace_kprobe *tk = to_trace_kprobe(ev);
	168	+
	169	+ return strcmp(trace_probe_name(&tk->tp), event) == 0 &&
	170	+ (!system \|\| strcmp(trace_probe_group_name(&tk->tp), system) == 0) &&
	171	+ trace_kprobe_match_command_head(tk, argc, argv);
	172	+}
	173	+
83	174	static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
84	175	{
85	176	unsigned long nhit = 0;
..	..	@@ -89,6 +180,12 @@
89	180	nhit += *per_cpu_ptr(tk->nhit, cpu);
90	181
91	182	return nhit;
	183	+}
	184	+
	185	+static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk)
	186	+{
	187	+ return !(list_empty(&tk->rp.kp.list) &&
	188	+ hlist_unhashed(&tk->rp.kp.hlist));
92	189	}
93	190
94	191	/* Return 0 if it fails to find the symbol address */
..	..	@@ -108,209 +205,51 @@
108	205	return addr;
109	206	}
110	207
	208	+static nokprobe_inline struct trace_kprobe *
	209	+trace_kprobe_primary_from_call(struct trace_event_call *call)
	210	+{
	211	+ struct trace_probe *tp;
	212	+
	213	+ tp = trace_probe_primary_from_call(call);
	214	+ if (WARN_ON_ONCE(!tp))
	215	+ return NULL;
	216	+
	217	+ return container_of(tp, struct trace_kprobe, tp);
	218	+}
	219	+
111	220	bool trace_kprobe_on_func_entry(struct trace_event_call *call)
112	221	{
113		- struct trace_kprobe tk = (struct trace_kprobe )call->data;
	222	+ struct trace_kprobe *tk = trace_kprobe_primary_from_call(call);
114	223
115		- return (kprobe_on_func_entry(tk->rp.kp.addr,
	224	+ return tk ? (kprobe_on_func_entry(tk->rp.kp.addr,
116	225	tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name,
117		- tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0);
	226	+ tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0) : false;
118	227	}
119	228
120	229	bool trace_kprobe_error_injectable(struct trace_event_call *call)
121	230	{
122		- struct trace_kprobe tk = (struct trace_kprobe )call->data;
	231	+ struct trace_kprobe *tk = trace_kprobe_primary_from_call(call);
123	232
124		- return within_error_injection_list(trace_kprobe_address(tk));
	233	+ return tk ? within_error_injection_list(trace_kprobe_address(tk)) :
	234	+ false;
125	235	}
126	236
127	237	static int register_kprobe_event(struct trace_kprobe *tk);
128	238	static int unregister_kprobe_event(struct trace_kprobe *tk);
129	239
130		-static DEFINE_MUTEX(probe_lock);
131		-static LIST_HEAD(probe_list);
132		-
133	240	static int kprobe_dispatcher(struct kprobe kp, struct pt_regs regs);
134	241	static int kretprobe_dispatcher(struct kretprobe_instance *ri,
135	242	struct pt_regs *regs);
136	243
137		-/* Memory fetching by symbol */
138		-struct symbol_cache {
139		- char *symbol;
140		- long offset;
141		- unsigned long addr;
142		-};
143		-
144		-unsigned long update_symbol_cache(struct symbol_cache *sc)
	244	+static void free_trace_kprobe(struct trace_kprobe *tk)
145	245	{
146		- sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol);
147		-
148		- if (sc->addr)
149		- sc->addr += sc->offset;
150		-
151		- return sc->addr;
152		-}
153		-
154		-void free_symbol_cache(struct symbol_cache *sc)
155		-{
156		- kfree(sc->symbol);
157		- kfree(sc);
158		-}
159		-
160		-struct symbol_cache alloc_symbol_cache(const char sym, long offset)
161		-{
162		- struct symbol_cache *sc;
163		-
164		- if (!sym \|\| strlen(sym) == 0)
165		- return NULL;
166		-
167		- sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL);
168		- if (!sc)
169		- return NULL;
170		-
171		- sc->symbol = kstrdup(sym, GFP_KERNEL);
172		- if (!sc->symbol) {
173		- kfree(sc);
174		- return NULL;
175		- }
176		- sc->offset = offset;
177		- update_symbol_cache(sc);
178		-
179		- return sc;
180		-}
181		-
182		-/*
183		- * Kprobes-specific fetch functions
184		- */
185		-#define DEFINE_FETCH_stack(type) \
186		-static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
187		- void offset, void dest) \
188		-{ \
189		- (type )dest = (type)regs_get_kernel_stack_nth(regs, \
190		- (unsigned int)((unsigned long)offset)); \
191		-} \
192		-NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
193		-
194		-DEFINE_BASIC_FETCH_FUNCS(stack)
195		-/* No string on the stack entry */
196		-#define fetch_stack_string NULL
197		-#define fetch_stack_string_size NULL
198		-
199		-#define DEFINE_FETCH_memory(type) \
200		-static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
201		- void addr, void dest) \
202		-{ \
203		- type retval; \
204		- if (probe_kernel_address(addr, retval)) \
205		- (type )dest = 0; \
206		- else \
207		- (type )dest = retval; \
208		-} \
209		-NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
210		-
211		-DEFINE_BASIC_FETCH_FUNCS(memory)
212		-/*
213		- * Fetch a null-terminated string. Caller MUST set (u32 )dest with max
214		- * length and relative data location.
215		- */
216		-static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
217		- void addr, void dest)
218		-{
219		- int maxlen = get_rloc_len((u32 )dest);
220		- u8 *dst = get_rloc_data(dest);
221		- long ret;
222		-
223		- if (!maxlen)
224		- return;
225		-
226		- /*
227		- * Try to get string again, since the string can be changed while
228		- * probing.
229		- */
230		- ret = strncpy_from_unsafe(dst, addr, maxlen);
231		-
232		- if (ret < 0) { /* Failed to fetch string */
233		- dst[0] = '\0';
234		- (u32 )dest = make_data_rloc(0, get_rloc_offs((u32 )dest));
235		- } else {
236		- (u32 )dest = make_data_rloc(ret, get_rloc_offs((u32 )dest));
	246	+ if (tk) {
	247	+ trace_probe_cleanup(&tk->tp);
	248	+ kfree(tk->symbol);
	249	+ free_percpu(tk->nhit);
	250	+ kfree(tk);
237	251	}
238	252	}
239		-NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string));
240		-
241		-/* Return the length of string -- including null terminal byte */
242		-static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
243		- void addr, void dest)
244		-{
245		- mm_segment_t old_fs;
246		- int ret, len = 0;
247		- u8 c;
248		-
249		- old_fs = get_fs();
250		- set_fs(KERNEL_DS);
251		- pagefault_disable();
252		-
253		- do {
254		- ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
255		- len++;
256		- } while (c && ret == 0 && len < MAX_STRING_SIZE);
257		-
258		- pagefault_enable();
259		- set_fs(old_fs);
260		-
261		- if (ret < 0) /* Failed to check the length */
262		- (u32 )dest = 0;
263		- else
264		- (u32 )dest = len;
265		-}
266		-NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size));
267		-
268		-#define DEFINE_FETCH_symbol(type) \
269		-void FETCH_FUNC_NAME(symbol, type)(struct pt_regs regs, void data, void *dest)\
270		-{ \
271		- struct symbol_cache *sc = data; \
272		- if (sc->addr) \
273		- fetch_memory_##type(regs, (void *)sc->addr, dest); \
274		- else \
275		- (type )dest = 0; \
276		-} \
277		-NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
278		-
279		-DEFINE_BASIC_FETCH_FUNCS(symbol)
280		-DEFINE_FETCH_symbol(string)
281		-DEFINE_FETCH_symbol(string_size)
282		-
283		-/* kprobes don't support file_offset fetch methods */
284		-#define fetch_file_offset_u8 NULL
285		-#define fetch_file_offset_u16 NULL
286		-#define fetch_file_offset_u32 NULL
287		-#define fetch_file_offset_u64 NULL
288		-#define fetch_file_offset_string NULL
289		-#define fetch_file_offset_string_size NULL
290		-
291		-/* Fetch type information table */
292		-static const struct fetch_type kprobes_fetch_type_table[] = {
293		- /* Special types */
294		- [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
295		- sizeof(u32), 1, "__data_loc char[]"),
296		- [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
297		- string_size, sizeof(u32), 0, "u32"),
298		- /* Basic types */
299		- ASSIGN_FETCH_TYPE(u8, u8, 0),
300		- ASSIGN_FETCH_TYPE(u16, u16, 0),
301		- ASSIGN_FETCH_TYPE(u32, u32, 0),
302		- ASSIGN_FETCH_TYPE(u64, u64, 0),
303		- ASSIGN_FETCH_TYPE(s8, u8, 1),
304		- ASSIGN_FETCH_TYPE(s16, u16, 1),
305		- ASSIGN_FETCH_TYPE(s32, u32, 1),
306		- ASSIGN_FETCH_TYPE(s64, u64, 1),
307		- ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0),
308		- ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
309		- ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
310		- ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
311		-
312		- ASSIGN_FETCH_TYPE_END
313		-};
314	253
315	254	/*
316	255	* Allocate new trace_probe and initialize it (including kprobes).
..	..	@@ -349,59 +288,29 @@
349	288	tk->rp.kp.pre_handler = kprobe_dispatcher;
350	289
351	290	tk->rp.maxactive = maxactive;
	291	+ INIT_HLIST_NODE(&tk->rp.kp.hlist);
	292	+ INIT_LIST_HEAD(&tk->rp.kp.list);
352	293
353		- if (!event \|\| !is_good_name(event)) {
354		- ret = -EINVAL;
355		- goto error;
356		- }
357		-
358		- tk->tp.call.class = &tk->tp.class;
359		- tk->tp.call.name = kstrdup(event, GFP_KERNEL);
360		- if (!tk->tp.call.name)
	294	+ ret = trace_probe_init(&tk->tp, event, group, false);
	295	+ if (ret < 0)
361	296	goto error;
362	297
363		- if (!group \|\| !is_good_name(group)) {
364		- ret = -EINVAL;
365		- goto error;
366		- }
367		-
368		- tk->tp.class.system = kstrdup(group, GFP_KERNEL);
369		- if (!tk->tp.class.system)
370		- goto error;
371		-
372		- INIT_LIST_HEAD(&tk->list);
373		- INIT_LIST_HEAD(&tk->tp.files);
	298	+ dyn_event_init(&tk->devent, &trace_kprobe_ops);
374	299	return tk;
375	300	error:
376		- kfree(tk->tp.call.name);
377		- kfree(tk->symbol);
378		- free_percpu(tk->nhit);
379		- kfree(tk);
	301	+ free_trace_kprobe(tk);
380	302	return ERR_PTR(ret);
381		-}
382		-
383		-static void free_trace_kprobe(struct trace_kprobe *tk)
384		-{
385		- int i;
386		-
387		- for (i = 0; i < tk->tp.nr_args; i++)
388		- traceprobe_free_probe_arg(&tk->tp.args[i]);
389		-
390		- kfree(tk->tp.call.class->system);
391		- kfree(tk->tp.call.name);
392		- kfree(tk->symbol);
393		- free_percpu(tk->nhit);
394		- kfree(tk);
395	303	}
396	304
397	305	static struct trace_kprobe find_trace_kprobe(const char event,
398	306	const char *group)
399	307	{
	308	+ struct dyn_event *pos;
400	309	struct trace_kprobe *tk;
401	310
402		- list_for_each_entry(tk, &probe_list, list)
403		- if (strcmp(trace_event_name(&tk->tp.call), event) == 0 &&
404		- strcmp(tk->tp.call.class->system, group) == 0)
	311	+ for_each_trace_kprobe(tk, pos)
	312	+ if (strcmp(trace_probe_name(&tk->tp), event) == 0 &&
	313	+ strcmp(trace_probe_group_name(&tk->tp), group) == 0)
405	314	return tk;
406	315	return NULL;
407	316	}
..	..	@@ -410,7 +319,7 @@
410	319	{
411	320	int ret = 0;
412	321
413		- if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) {
	322	+ if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) {
414	323	if (trace_kprobe_is_return(tk))
415	324	ret = enable_kretprobe(&tk->rp);
416	325	else
..	..	@@ -420,41 +329,70 @@
420	329	return ret;
421	330	}
422	331
	332	+static void __disable_trace_kprobe(struct trace_probe *tp)
	333	+{
	334	+ struct trace_probe *pos;
	335	+ struct trace_kprobe *tk;
	336	+
	337	+ list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
	338	+ tk = container_of(pos, struct trace_kprobe, tp);
	339	+ if (!trace_kprobe_is_registered(tk))
	340	+ continue;
	341	+ if (trace_kprobe_is_return(tk))
	342	+ disable_kretprobe(&tk->rp);
	343	+ else
	344	+ disable_kprobe(&tk->rp.kp);
	345	+ }
	346	+}
	347	+
423	348	/*
424	349	* Enable trace_probe
425	350	* if the file is NULL, enable "perf" handler, or enable "trace" handler.
426	351	*/
427		-static int
428		-enable_trace_kprobe(struct trace_kprobe tk, struct trace_event_file file)
	352	+static int enable_trace_kprobe(struct trace_event_call *call,
	353	+ struct trace_event_file *file)
429	354	{
430		- struct event_file_link *link;
	355	+ struct trace_probe pos, tp;
	356	+ struct trace_kprobe *tk;
	357	+ bool enabled;
431	358	int ret = 0;
432	359
	360	+ tp = trace_probe_primary_from_call(call);
	361	+ if (WARN_ON_ONCE(!tp))
	362	+ return -ENODEV;
	363	+ enabled = trace_probe_is_enabled(tp);
	364	+
	365	+ /* This also changes "enabled" state */
433	366	if (file) {
434		- link = kmalloc(sizeof(*link), GFP_KERNEL);
435		- if (!link) {
436		- ret = -ENOMEM;
437		- goto out;
438		- }
	367	+ ret = trace_probe_add_file(tp, file);
	368	+ if (ret)
	369	+ return ret;
	370	+ } else
	371	+ trace_probe_set_flag(tp, TP_FLAG_PROFILE);
439	372
440		- link->file = file;
441		- list_add_tail_rcu(&link->list, &tk->tp.files);
	373	+ if (enabled)
	374	+ return 0;
442	375
443		- tk->tp.flags \|= TP_FLAG_TRACE;
444		- ret = __enable_trace_kprobe(tk);
445		- if (ret) {
446		- list_del_rcu(&link->list);
447		- kfree(link);
448		- tk->tp.flags &= ~TP_FLAG_TRACE;
449		- }
450		-
451		- } else {
452		- tk->tp.flags \|= TP_FLAG_PROFILE;
	376	+ list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
	377	+ tk = container_of(pos, struct trace_kprobe, tp);
	378	+ if (trace_kprobe_has_gone(tk))
	379	+ continue;
453	380	ret = __enable_trace_kprobe(tk);
454	381	if (ret)
455		- tk->tp.flags &= ~TP_FLAG_PROFILE;
	382	+ break;
	383	+ enabled = true;
456	384	}
457		- out:
	385	+
	386	+ if (ret) {
	387	+ /* Failed to enable one of them. Roll back all */
	388	+ if (enabled)
	389	+ __disable_trace_kprobe(tp);
	390	+ if (file)
	391	+ trace_probe_remove_file(tp, file);
	392	+ else
	393	+ trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
	394	+ }
	395	+
458	396	return ret;
459	397	}
460	398
..	..	@@ -462,59 +400,38 @@
462	400	* Disable trace_probe
463	401	* if the file is NULL, disable "perf" handler, or disable "trace" handler.
464	402	*/
465		-static int
466		-disable_trace_kprobe(struct trace_kprobe tk, struct trace_event_file file)
	403	+static int disable_trace_kprobe(struct trace_event_call *call,
	404	+ struct trace_event_file *file)
467	405	{
468		- struct event_file_link *link = NULL;
469		- int wait = 0;
470		- int ret = 0;
	406	+ struct trace_probe *tp;
	407	+
	408	+ tp = trace_probe_primary_from_call(call);
	409	+ if (WARN_ON_ONCE(!tp))
	410	+ return -ENODEV;
471	411
472	412	if (file) {
473		- link = find_event_file_link(&tk->tp, file);
474		- if (!link) {
475		- ret = -EINVAL;
	413	+ if (!trace_probe_get_file_link(tp, file))
	414	+ return -ENOENT;
	415	+ if (!trace_probe_has_single_file(tp))
476	416	goto out;
477		- }
478		-
479		- list_del_rcu(&link->list);
480		- wait = 1;
481		- if (!list_empty(&tk->tp.files))
482		- goto out;
483		-
484		- tk->tp.flags &= ~TP_FLAG_TRACE;
	417	+ trace_probe_clear_flag(tp, TP_FLAG_TRACE);
485	418	} else
486		- tk->tp.flags &= ~TP_FLAG_PROFILE;
	419	+ trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
487	420
488		- if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) {
489		- if (trace_kprobe_is_return(tk))
490		- disable_kretprobe(&tk->rp);
491		- else
492		- disable_kprobe(&tk->rp.kp);
493		- wait = 1;
494		- }
	421	+ if (!trace_probe_is_enabled(tp))
	422	+ __disable_trace_kprobe(tp);
495	423
496		- /*
497		- * if tk is not added to any list, it must be a local trace_kprobe
498		- * created with perf_event_open. We don't need to wait for these
499		- * trace_kprobes
500		- */
501		- if (list_empty(&tk->list))
502		- wait = 0;
503	424	out:
504		- if (wait) {
	425	+ if (file)
505	426	/*
506		- * Synchronize with kprobe_trace_func/kretprobe_trace_func
507		- * to ensure disabled (all running handlers are finished).
508		- * This is not only for kfree(), but also the caller,
509		- * trace_remove_event_call() supposes it for releasing
510		- * event_call related objects, which will be accessed in
511		- * the kprobe_trace_func/kretprobe_trace_func.
	427	+ * Synchronization is done in below function. For perf event,
	428	+ * file == NULL and perf_trace_event_unreg() calls
	429	+ * tracepoint_synchronize_unregister() to ensure synchronize
	430	+ * event. We don't need to care about it.
512	431	*/
513		- synchronize_sched();
514		- kfree(link); /* Ignored if link == NULL */
515		- }
	432	+ trace_probe_remove_file(tp, file);
516	433
517		- return ret;
	434	+ return 0;
518	435	}
519	436
520	437	#if defined(CONFIG_DYNAMIC_FTRACE) && \
..	..	@@ -566,7 +483,11 @@
566	483	{
567	484	int i, ret;
568	485
569		- if (trace_probe_is_registered(&tk->tp))
	486	+ ret = security_locked_down(LOCKDOWN_KPROBES);
	487	+ if (ret)
	488	+ return ret;
	489	+
	490	+ if (trace_kprobe_is_registered(tk))
570	491	return -EINVAL;
571	492
572	493	if (within_notrace_func(tk)) {
..	..	@@ -575,8 +496,11 @@
575	496	return -EINVAL;
576	497	}
577	498
578		- for (i = 0; i < tk->tp.nr_args; i++)
579		- traceprobe_update_arg(&tk->tp.args[i]);
	499	+ for (i = 0; i < tk->tp.nr_args; i++) {
	500	+ ret = traceprobe_update_arg(&tk->tp.args[i]);
	501	+ if (ret)
	502	+ return ret;
	503	+ }
580	504
581	505	/* Set/clear disabled flag according to tp->flag */
582	506	if (trace_probe_is_enabled(&tk->tp))
..	..	@@ -589,34 +513,32 @@
589	513	else
590	514	ret = register_kprobe(&tk->rp.kp);
591	515
592		- if (ret == 0) {
593		- tk->tp.flags \|= TP_FLAG_REGISTERED;
594		- } else if (ret == -EILSEQ) {
595		- pr_warn("Probing address(0x%p) is not an instruction boundary.\n",
596		- tk->rp.kp.addr);
597		- ret = -EINVAL;
598		- }
599	516	return ret;
600	517	}
601	518
602	519	/* Internal unregister function - just handle kprobes and flags /
603	520	static void __unregister_trace_kprobe(struct trace_kprobe *tk)
604	521	{
605		- if (trace_probe_is_registered(&tk->tp)) {
	522	+ if (trace_kprobe_is_registered(tk)) {
606	523	if (trace_kprobe_is_return(tk))
607	524	unregister_kretprobe(&tk->rp);
608	525	else
609	526	unregister_kprobe(&tk->rp.kp);
610		- tk->tp.flags &= ~TP_FLAG_REGISTERED;
611		- /* Cleanup kprobe for reuse */
	527	+ /* Cleanup kprobe for reuse and mark it unregistered */
	528	+ INIT_HLIST_NODE(&tk->rp.kp.hlist);
	529	+ INIT_LIST_HEAD(&tk->rp.kp.list);
612	530	if (tk->rp.kp.symbol_name)
613	531	tk->rp.kp.addr = NULL;
614	532	}
615	533	}
616	534
617		-/* Unregister a trace_probe and probe_event: call with locking probe_lock */
	535	+/* Unregister a trace_probe and probe_event */
618	536	static int unregister_trace_kprobe(struct trace_kprobe *tk)
619	537	{
	538	+ /* If other probes are on the event, just unregister kprobe */
	539	+ if (trace_probe_has_sibling(&tk->tp))
	540	+ goto unreg;
	541	+
620	542	/* Enabled event can not be unregistered */
621	543	if (trace_probe_is_enabled(&tk->tp))
622	544	return -EBUSY;
..	..	@@ -625,10 +547,80 @@
625	547	if (unregister_kprobe_event(tk))
626	548	return -EBUSY;
627	549
	550	+unreg:
628	551	__unregister_trace_kprobe(tk);
629		- list_del(&tk->list);
	552	+ dyn_event_remove(&tk->devent);
	553	+ trace_probe_unlink(&tk->tp);
630	554
631	555	return 0;
	556	+}
	557	+
	558	+static bool trace_kprobe_has_same_kprobe(struct trace_kprobe *orig,
	559	+ struct trace_kprobe *comp)
	560	+{
	561	+ struct trace_probe_event *tpe = orig->tp.event;
	562	+ struct trace_probe *pos;
	563	+ int i;
	564	+
	565	+ list_for_each_entry(pos, &tpe->probes, list) {
	566	+ orig = container_of(pos, struct trace_kprobe, tp);
	567	+ if (strcmp(trace_kprobe_symbol(orig),
	568	+ trace_kprobe_symbol(comp)) \|\|
	569	+ trace_kprobe_offset(orig) != trace_kprobe_offset(comp))
	570	+ continue;
	571	+
	572	+ /*
	573	+ * trace_probe_compare_arg_type() ensured that nr_args and
	574	+ * each argument name and type are same. Let's compare comm.
	575	+ */
	576	+ for (i = 0; i < orig->tp.nr_args; i++) {
	577	+ if (strcmp(orig->tp.args[i].comm,
	578	+ comp->tp.args[i].comm))
	579	+ break;
	580	+ }
	581	+
	582	+ if (i == orig->tp.nr_args)
	583	+ return true;
	584	+ }
	585	+
	586	+ return false;
	587	+}
	588	+
	589	+static int append_trace_kprobe(struct trace_kprobe tk, struct trace_kprobe to)
	590	+{
	591	+ int ret;
	592	+
	593	+ ret = trace_probe_compare_arg_type(&tk->tp, &to->tp);
	594	+ if (ret) {
	595	+ /* Note that argument starts index = 2 */
	596	+ trace_probe_log_set_index(ret + 1);
	597	+ trace_probe_log_err(0, DIFF_ARG_TYPE);
	598	+ return -EEXIST;
	599	+ }
	600	+ if (trace_kprobe_has_same_kprobe(to, tk)) {
	601	+ trace_probe_log_set_index(0);
	602	+ trace_probe_log_err(0, SAME_PROBE);
	603	+ return -EEXIST;
	604	+ }
	605	+
	606	+ /* Append to existing event */
	607	+ ret = trace_probe_append(&tk->tp, &to->tp);
	608	+ if (ret)
	609	+ return ret;
	610	+
	611	+ /* Register kprobe /
	612	+ ret = __register_trace_kprobe(tk);
	613	+ if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
	614	+ pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
	615	+ ret = 0;
	616	+ }
	617	+
	618	+ if (ret)
	619	+ trace_probe_unlink(&tk->tp);
	620	+ else
	621	+ dyn_event_add(&tk->devent);
	622	+
	623	+ return ret;
632	624	}
633	625
634	626	/* Register a trace_probe and probe_event */
..	..	@@ -637,22 +629,29 @@
637	629	struct trace_kprobe *old_tk;
638	630	int ret;
639	631
640		- mutex_lock(&probe_lock);
	632	+ mutex_lock(&event_mutex);
641	633
642		- /* Delete old (same name) event if exist */
643		- old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call),
644		- tk->tp.call.class->system);
	634	+ old_tk = find_trace_kprobe(trace_probe_name(&tk->tp),
	635	+ trace_probe_group_name(&tk->tp));
645	636	if (old_tk) {
646		- ret = unregister_trace_kprobe(old_tk);
647		- if (ret < 0)
648		- goto end;
649		- free_trace_kprobe(old_tk);
	637	+ if (trace_kprobe_is_return(tk) != trace_kprobe_is_return(old_tk)) {
	638	+ trace_probe_log_set_index(0);
	639	+ trace_probe_log_err(0, DIFF_PROBE_TYPE);
	640	+ ret = -EEXIST;
	641	+ } else {
	642	+ ret = append_trace_kprobe(tk, old_tk);
	643	+ }
	644	+ goto end;
650	645	}
651	646
652	647	/* Register new event */
653	648	ret = register_kprobe_event(tk);
654	649	if (ret) {
655		- pr_warn("Failed to register probe event(%d)\n", ret);
	650	+ if (ret == -EEXIST) {
	651	+ trace_probe_log_set_index(0);
	652	+ trace_probe_log_err(0, EVENT_EXIST);
	653	+ } else
	654	+ pr_warn("Failed to register probe event(%d)\n", ret);
656	655	goto end;
657	656	}
658	657
..	..	@@ -666,10 +665,10 @@
666	665	if (ret < 0)
667	666	unregister_kprobe_event(tk);
668	667	else
669		- list_add_tail(&tk->list, &probe_list);
	668	+ dyn_event_add(&tk->devent);
670	669
671	670	end:
672		- mutex_unlock(&probe_lock);
	671	+ mutex_unlock(&event_mutex);
673	672	return ret;
674	673	}
675	674
..	..	@@ -678,6 +677,7 @@
678	677	unsigned long val, void *data)
679	678	{
680	679	struct module *mod = data;
	680	+ struct dyn_event *pos;
681	681	struct trace_kprobe *tk;
682	682	int ret;
683	683
..	..	@@ -685,19 +685,19 @@
685	685	return NOTIFY_DONE;
686	686
687	687	/* Update probes on coming module */
688		- mutex_lock(&probe_lock);
689		- list_for_each_entry(tk, &probe_list, list) {
	688	+ mutex_lock(&event_mutex);
	689	+ for_each_trace_kprobe(tk, pos) {
690	690	if (trace_kprobe_within_module(tk, mod)) {
691	691	/* Don't need to check busy - this should have gone. */
692	692	__unregister_trace_kprobe(tk);
693	693	ret = __register_trace_kprobe(tk);
694	694	if (ret)
695	695	pr_warn("Failed to re-register probe %s on %s: %d\n",
696		- trace_event_name(&tk->tp.call),
697		- mod->name, ret);
	696	+ trace_probe_name(&tk->tp),
	697	+ module_name(mod), ret);
698	698	}
699	699	}
700		- mutex_unlock(&probe_lock);
	700	+ mutex_unlock(&event_mutex);
701	701
702	702	return NOTIFY_DONE;
703	703	}
..	..	@@ -715,7 +715,7 @@
715	715	*name = '_';
716	716	}
717	717
718		-static int create_trace_kprobe(int argc, char **argv)
	718	+static int trace_kprobe_create(int argc, const char *argv[])
719	719	{
720	720	/*
721	721	* Argument syntax:
..	..	@@ -723,6 +723,9 @@
723	723	* p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]\|KADDR [FETCHARGS]
724	724	* - Add kretprobe:
725	725	* r[MAXACTIVE][:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS]
	726	+ * Or
	727	+ * p:[GRP/]EVENT] [MOD:]KSYM[+0]%return [FETCHARGS]
	728	+ *
726	729	* Fetch args:
727	730	* $retval : fetch return value
728	731	* $stack : fetch stack address
..	..	@@ -738,115 +741,114 @@
738	741	* Type of args:
739	742	* FETCHARG:TYPE : use TYPE instead of unsigned long.
740	743	*/
741		- struct trace_kprobe *tk;
742		- int i, ret = 0;
743		- bool is_return = false, is_delete = false;
744		- char symbol = NULL, event = NULL, *group = NULL;
	744	+ struct trace_kprobe *tk = NULL;
	745	+ int i, len, ret = 0;
	746	+ bool is_return = false;
	747	+ char symbol = NULL, tmp = NULL;
	748	+ const char event = NULL, group = KPROBE_EVENT_SYSTEM;
745	749	int maxactive = 0;
746		- char *arg;
747	750	long offset = 0;
748	751	void *addr = NULL;
749	752	char buf[MAX_EVENT_NAME_LEN];
	753	+ unsigned int flags = TPARG_FL_KERNEL;
750	754
751		- /* argc must be >= 1 */
752		- if (argv[0][0] == 'p')
753		- is_return = false;
754		- else if (argv[0][0] == 'r')
	755	+ switch (argv[0][0]) {
	756	+ case 'r':
755	757	is_return = true;
756		- else if (argv[0][0] == '-')
757		- is_delete = true;
758		- else {
759		- pr_info("Probe definition must be started with 'p', 'r' or"
760		- " '-'.\n");
761		- return -EINVAL;
	758	+ break;
	759	+ case 'p':
	760	+ break;
	761	+ default:
	762	+ return -ECANCELED;
762	763	}
	764	+ if (argc < 2)
	765	+ return -ECANCELED;
	766	+
	767	+ trace_probe_log_init("trace_kprobe", argc, argv);
763	768
764	769	event = strchr(&argv[0][1], ':');
765		- if (event) {
766		- event[0] = '\0';
	770	+ if (event)
767	771	event++;
768		- }
769		- if (is_return && isdigit(argv[0][1])) {
770		- ret = kstrtouint(&argv[0][1], 0, &maxactive);
771		- if (ret) {
772		- pr_info("Failed to parse maxactive.\n");
773		- return ret;
	772	+
	773	+ if (isdigit(argv[0][1])) {
	774	+ if (!is_return) {
	775	+ trace_probe_log_err(1, MAXACT_NO_KPROBE);
	776	+ goto parse_error;
	777	+ }
	778	+ if (event)
	779	+ len = event - &argv[0][1] - 1;
	780	+ else
	781	+ len = strlen(&argv[0][1]);
	782	+ if (len > MAX_EVENT_NAME_LEN - 1) {
	783	+ trace_probe_log_err(1, BAD_MAXACT);
	784	+ goto parse_error;
	785	+ }
	786	+ memcpy(buf, &argv[0][1], len);
	787	+ buf[len] = '\0';
	788	+ ret = kstrtouint(buf, 0, &maxactive);
	789	+ if (ret \|\| !maxactive) {
	790	+ trace_probe_log_err(1, BAD_MAXACT);
	791	+ goto parse_error;
774	792	}
775	793	/* kretprobes instances are iterated over via a list. The
776	794	* maximum should stay reasonable.
777	795	*/
778	796	if (maxactive > KRETPROBE_MAXACTIVE_MAX) {
779		- pr_info("Maxactive is too big (%d > %d).\n",
780		- maxactive, KRETPROBE_MAXACTIVE_MAX);
781		- return -E2BIG;
	797	+ trace_probe_log_err(1, MAXACT_TOO_BIG);
	798	+ goto parse_error;
782	799	}
783		- }
784		-
785		- if (event) {
786		- if (strchr(event, '/')) {
787		- group = event;
788		- event = strchr(group, '/') + 1;
789		- event[-1] = '\0';
790		- if (strlen(group) == 0) {
791		- pr_info("Group name is not specified\n");
792		- return -EINVAL;
793		- }
794		- }
795		- if (strlen(event) == 0) {
796		- pr_info("Event name is not specified\n");
797		- return -EINVAL;
798		- }
799		- }
800		- if (!group)
801		- group = KPROBE_EVENT_SYSTEM;
802		-
803		- if (is_delete) {
804		- if (!event) {
805		- pr_info("Delete command needs an event name.\n");
806		- return -EINVAL;
807		- }
808		- mutex_lock(&probe_lock);
809		- tk = find_trace_kprobe(event, group);
810		- if (!tk) {
811		- mutex_unlock(&probe_lock);
812		- pr_info("Event %s/%s doesn't exist.\n", group, event);
813		- return -ENOENT;
814		- }
815		- /* delete an event */
816		- ret = unregister_trace_kprobe(tk);
817		- if (ret == 0)
818		- free_trace_kprobe(tk);
819		- mutex_unlock(&probe_lock);
820		- return ret;
821		- }
822		-
823		- if (argc < 2) {
824		- pr_info("Probe point is not specified.\n");
825		- return -EINVAL;
826	800	}
827	801
828	802	/* try to parse an address. if that fails, try to read the
829	803	* input as a symbol. */
830	804	if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) {
	805	+ trace_probe_log_set_index(1);
	806	+ /* Check whether uprobe event specified */
	807	+ if (strchr(argv[1], '/') && strchr(argv[1], ':')) {
	808	+ ret = -ECANCELED;
	809	+ goto error;
	810	+ }
831	811	/* a symbol specified */
832		- symbol = argv[1];
	812	+ symbol = kstrdup(argv[1], GFP_KERNEL);
	813	+ if (!symbol)
	814	+ return -ENOMEM;
	815	+
	816	+ tmp = strchr(symbol, '%');
	817	+ if (tmp) {
	818	+ if (!strcmp(tmp, "%return")) {
	819	+ *tmp = '\0';
	820	+ is_return = true;
	821	+ } else {
	822	+ trace_probe_log_err(tmp - symbol, BAD_ADDR_SUFFIX);
	823	+ goto parse_error;
	824	+ }
	825	+ }
	826	+
833	827	/* TODO: support .init module functions */
834	828	ret = traceprobe_split_symbol_offset(symbol, &offset);
835	829	if (ret \|\| offset < 0 \|\| offset > UINT_MAX) {
836		- pr_info("Failed to parse either an address or a symbol.\n");
837		- return ret;
	830	+ trace_probe_log_err(0, BAD_PROBE_ADDR);
	831	+ goto parse_error;
838	832	}
	833	+ if (is_return)
	834	+ flags \|= TPARG_FL_RETURN;
	835	+ ret = kprobe_on_func_entry(NULL, symbol, offset);
	836	+ if (ret == 0)
	837	+ flags \|= TPARG_FL_FENTRY;
839	838	/* Defer the ENOENT case until register kprobe */
840		- if (offset && is_return &&
841		- kprobe_on_func_entry(NULL, symbol, offset) == -EINVAL) {
842		- pr_info("Given offset is not valid for return probe.\n");
843		- return -EINVAL;
	839	+ if (ret == -EINVAL && is_return) {
	840	+ trace_probe_log_err(0, BAD_RETPROBE);
	841	+ goto parse_error;
844	842	}
845	843	}
846		- argc -= 2; argv += 2;
847	844
848		- /* setup a probe */
849		- if (!event) {
	845	+ trace_probe_log_set_index(0);
	846	+ if (event) {
	847	+ ret = traceprobe_parse_event_name(&event, &group, buf,
	848	+ event - argv[0]);
	849	+ if (ret)
	850	+ goto parse_error;
	851	+ } else {
850	852	/* Make a new event name */
851	853	if (symbol)
852	854	snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
..	..	@@ -857,129 +859,259 @@
857	859	sanitize_event_name(buf);
858	860	event = buf;
859	861	}
	862	+
	863	+ /* setup a probe */
860	864	tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive,
861		- argc, is_return);
	865	+ argc - 2, is_return);
862	866	if (IS_ERR(tk)) {
863		- pr_info("Failed to allocate trace_probe.(%d)\n",
864		- (int)PTR_ERR(tk));
865		- return PTR_ERR(tk);
	867	+ ret = PTR_ERR(tk);
	868	+ /* This must return -ENOMEM, else there is a bug */
	869	+ WARN_ON_ONCE(ret != -ENOMEM);
	870	+ goto out; /* We know tk is not allocated */
866	871	}
	872	+ argc -= 2; argv += 2;
867	873
868	874	/* parse arguments */
869		- ret = 0;
870	875	for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
871		- struct probe_arg *parg = &tk->tp.args[i];
872		-
873		- /* Increment count for freeing args in error case */
874		- tk->tp.nr_args++;
875		-
876		- /* Parse argument name */
877		- arg = strchr(argv[i], '=');
878		- if (arg) {
879		- *arg++ = '\0';
880		- parg->name = kstrdup(argv[i], GFP_KERNEL);
881		- } else {
882		- arg = argv[i];
883		- /* If argument name is omitted, set "argN" */
884		- snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
885		- parg->name = kstrdup(buf, GFP_KERNEL);
886		- }
887		-
888		- if (!parg->name) {
889		- pr_info("Failed to allocate argument[%d] name.\n", i);
	876	+ tmp = kstrdup(argv[i], GFP_KERNEL);
	877	+ if (!tmp) {
890	878	ret = -ENOMEM;
891	879	goto error;
892	880	}
893	881
894		- if (!is_good_name(parg->name)) {
895		- pr_info("Invalid argument[%d] name: %s\n",
896		- i, parg->name);
897		- ret = -EINVAL;
898		- goto error;
899		- }
900		-
901		- if (traceprobe_conflict_field_name(parg->name,
902		- tk->tp.args, i)) {
903		- pr_info("Argument[%d] name '%s' conflicts with "
904		- "another field.\n", i, argv[i]);
905		- ret = -EINVAL;
906		- goto error;
907		- }
908		-
909		- /* Parse fetch argument */
910		- ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,
911		- is_return, true,
912		- kprobes_fetch_type_table);
913		- if (ret) {
914		- pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
915		- goto error;
916		- }
	882	+ trace_probe_log_set_index(i + 2);
	883	+ ret = traceprobe_parse_probe_arg(&tk->tp, i, tmp, flags);
	884	+ kfree(tmp);
	885	+ if (ret)
	886	+ goto error; /* This can be -ENOMEM */
917	887	}
	888	+
	889	+ ret = traceprobe_set_print_fmt(&tk->tp, is_return);
	890	+ if (ret < 0)
	891	+ goto error;
918	892
919	893	ret = register_trace_kprobe(tk);
920		- if (ret)
	894	+ if (ret) {
	895	+ trace_probe_log_set_index(1);
	896	+ if (ret == -EILSEQ)
	897	+ trace_probe_log_err(0, BAD_INSN_BNDRY);
	898	+ else if (ret == -ENOENT)
	899	+ trace_probe_log_err(0, BAD_PROBE_ADDR);
	900	+ else if (ret != -ENOMEM && ret != -EEXIST)
	901	+ trace_probe_log_err(0, FAIL_REG_PROBE);
921	902	goto error;
922		- return 0;
923		-
924		-error:
925		- free_trace_kprobe(tk);
926		- return ret;
927		-}
928		-
929		-static int release_all_trace_kprobes(void)
930		-{
931		- struct trace_kprobe *tk;
932		- int ret = 0;
933		-
934		- mutex_lock(&probe_lock);
935		- /* Ensure no probe is in use. */
936		- list_for_each_entry(tk, &probe_list, list)
937		- if (trace_probe_is_enabled(&tk->tp)) {
938		- ret = -EBUSY;
939		- goto end;
940		- }
941		- /* TODO: Use batch unregistration */
942		- while (!list_empty(&probe_list)) {
943		- tk = list_entry(probe_list.next, struct trace_kprobe, list);
944		- ret = unregister_trace_kprobe(tk);
945		- if (ret)
946		- goto end;
947		- free_trace_kprobe(tk);
948	903	}
949	904
950		-end:
951		- mutex_unlock(&probe_lock);
	905	+out:
	906	+ trace_probe_log_clear();
	907	+ kfree(symbol);
	908	+ return ret;
	909	+
	910	+parse_error:
	911	+ ret = -EINVAL;
	912	+error:
	913	+ free_trace_kprobe(tk);
	914	+ goto out;
	915	+}
	916	+
	917	+static int create_or_delete_trace_kprobe(int argc, char **argv)
	918	+{
	919	+ int ret;
	920	+
	921	+ if (argv[0][0] == '-')
	922	+ return dyn_event_release(argc, argv, &trace_kprobe_ops);
	923	+
	924	+ ret = trace_kprobe_create(argc, (const char **)argv);
	925	+ return ret == -ECANCELED ? -EINVAL : ret;
	926	+}
	927	+
	928	+static int trace_kprobe_run_command(struct dynevent_cmd *cmd)
	929	+{
	930	+ return trace_run_command(cmd->seq.buffer, create_or_delete_trace_kprobe);
	931	+}
	932	+
	933	+/**
	934	+ * kprobe_event_cmd_init - Initialize a kprobe event command object
	935	+ * @cmd: A pointer to the dynevent_cmd struct representing the new event
	936	+ * @buf: A pointer to the buffer used to build the command
	937	+ * @maxlen: The length of the buffer passed in @buf
	938	+ *
	939	+ * Initialize a synthetic event command object. Use this before
	940	+ * calling any of the other kprobe_event functions.
	941	+ */
	942	+void kprobe_event_cmd_init(struct dynevent_cmd cmd, char buf, int maxlen)
	943	+{
	944	+ dynevent_cmd_init(cmd, buf, maxlen, DYNEVENT_TYPE_KPROBE,
	945	+ trace_kprobe_run_command);
	946	+}
	947	+EXPORT_SYMBOL_GPL(kprobe_event_cmd_init);
	948	+
	949	+/**
	950	+ * __kprobe_event_gen_cmd_start - Generate a kprobe event command from arg list
	951	+ * @cmd: A pointer to the dynevent_cmd struct representing the new event
	952	+ * @name: The name of the kprobe event
	953	+ * @loc: The location of the kprobe event
	954	+ * @kretprobe: Is this a return probe?
	955	+ * @args: Variable number of arg (pairs), one pair for each field
	956	+ *
	957	+ * NOTE: Users normally won't want to call this function directly, but
	958	+ * rather use the kprobe_event_gen_cmd_start() wrapper, which automatically
	959	+ * adds a NULL to the end of the arg list. If this function is used
	960	+ * directly, make sure the last arg in the variable arg list is NULL.
	961	+ *
	962	+ * Generate a kprobe event command to be executed by
	963	+ * kprobe_event_gen_cmd_end(). This function can be used to generate the
	964	+ * complete command or only the first part of it; in the latter case,
	965	+ * kprobe_event_add_fields() can be used to add more fields following this.
	966	+ *
	967	+ * Unlikely the synth_event_gen_cmd_start(), @loc must be specified. This
	968	+ * returns -EINVAL if @loc == NULL.
	969	+ *
	970	+ * Return: 0 if successful, error otherwise.
	971	+ */
	972	+int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd, bool kretprobe,
	973	+ const char name, const char loc, ...)
	974	+{
	975	+ char buf[MAX_EVENT_NAME_LEN];
	976	+ struct dynevent_arg arg;
	977	+ va_list args;
	978	+ int ret;
	979	+
	980	+ if (cmd->type != DYNEVENT_TYPE_KPROBE)
	981	+ return -EINVAL;
	982	+
	983	+ if (!loc)
	984	+ return -EINVAL;
	985	+
	986	+ if (kretprobe)
	987	+ snprintf(buf, MAX_EVENT_NAME_LEN, "r:kprobes/%s", name);
	988	+ else
	989	+ snprintf(buf, MAX_EVENT_NAME_LEN, "p:kprobes/%s", name);
	990	+
	991	+ ret = dynevent_str_add(cmd, buf);
	992	+ if (ret)
	993	+ return ret;
	994	+
	995	+ dynevent_arg_init(&arg, 0);
	996	+ arg.str = loc;
	997	+ ret = dynevent_arg_add(cmd, &arg, NULL);
	998	+ if (ret)
	999	+ return ret;
	1000	+
	1001	+ va_start(args, loc);
	1002	+ for (;;) {
	1003	+ const char *field;
	1004	+
	1005	+ field = va_arg(args, const char *);
	1006	+ if (!field)
	1007	+ break;
	1008	+
	1009	+ if (++cmd->n_fields > MAX_TRACE_ARGS) {
	1010	+ ret = -EINVAL;
	1011	+ break;
	1012	+ }
	1013	+
	1014	+ arg.str = field;
	1015	+ ret = dynevent_arg_add(cmd, &arg, NULL);
	1016	+ if (ret)
	1017	+ break;
	1018	+ }
	1019	+ va_end(args);
952	1020
953	1021	return ret;
954	1022	}
	1023	+EXPORT_SYMBOL_GPL(__kprobe_event_gen_cmd_start);
955	1024
956		-/* Probes listing interfaces */
957		-static void probes_seq_start(struct seq_file m, loff_t *pos)
	1025	+/**
	1026	+ * __kprobe_event_add_fields - Add probe fields to a kprobe command from arg list
	1027	+ * @cmd: A pointer to the dynevent_cmd struct representing the new event
	1028	+ * @args: Variable number of arg (pairs), one pair for each field
	1029	+ *
	1030	+ * NOTE: Users normally won't want to call this function directly, but
	1031	+ * rather use the kprobe_event_add_fields() wrapper, which
	1032	+ * automatically adds a NULL to the end of the arg list. If this
	1033	+ * function is used directly, make sure the last arg in the variable
	1034	+ * arg list is NULL.
	1035	+ *
	1036	+ * Add probe fields to an existing kprobe command using a variable
	1037	+ * list of args. Fields are added in the same order they're listed.
	1038	+ *
	1039	+ * Return: 0 if successful, error otherwise.
	1040	+ */
	1041	+int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...)
958	1042	{
959		- mutex_lock(&probe_lock);
960		- return seq_list_start(&probe_list, *pos);
	1043	+ struct dynevent_arg arg;
	1044	+ va_list args;
	1045	+ int ret = 0;
	1046	+
	1047	+ if (cmd->type != DYNEVENT_TYPE_KPROBE)
	1048	+ return -EINVAL;
	1049	+
	1050	+ dynevent_arg_init(&arg, 0);
	1051	+
	1052	+ va_start(args, cmd);
	1053	+ for (;;) {
	1054	+ const char *field;
	1055	+
	1056	+ field = va_arg(args, const char *);
	1057	+ if (!field)
	1058	+ break;
	1059	+
	1060	+ if (++cmd->n_fields > MAX_TRACE_ARGS) {
	1061	+ ret = -EINVAL;
	1062	+ break;
	1063	+ }
	1064	+
	1065	+ arg.str = field;
	1066	+ ret = dynevent_arg_add(cmd, &arg, NULL);
	1067	+ if (ret)
	1068	+ break;
	1069	+ }
	1070	+ va_end(args);
	1071	+
	1072	+ return ret;
	1073	+}
	1074	+EXPORT_SYMBOL_GPL(__kprobe_event_add_fields);
	1075	+
	1076	+/**
	1077	+ * kprobe_event_delete - Delete a kprobe event
	1078	+ * @name: The name of the kprobe event to delete
	1079	+ *
	1080	+ * Delete a kprobe event with the give @name from kernel code rather
	1081	+ * than directly from the command line.
	1082	+ *
	1083	+ * Return: 0 if successful, error otherwise.
	1084	+ */
	1085	+int kprobe_event_delete(const char *name)
	1086	+{
	1087	+ char buf[MAX_EVENT_NAME_LEN];
	1088	+
	1089	+ snprintf(buf, MAX_EVENT_NAME_LEN, "-:%s", name);
	1090	+
	1091	+ return trace_run_command(buf, create_or_delete_trace_kprobe);
	1092	+}
	1093	+EXPORT_SYMBOL_GPL(kprobe_event_delete);
	1094	+
	1095	+static int trace_kprobe_release(struct dyn_event *ev)
	1096	+{
	1097	+ struct trace_kprobe *tk = to_trace_kprobe(ev);
	1098	+ int ret = unregister_trace_kprobe(tk);
	1099	+
	1100	+ if (!ret)
	1101	+ free_trace_kprobe(tk);
	1102	+ return ret;
961	1103	}
962	1104
963		-static void probes_seq_next(struct seq_file m, void v, loff_t pos)
	1105	+static int trace_kprobe_show(struct seq_file m, struct dyn_event ev)
964	1106	{
965		- return seq_list_next(v, &probe_list, pos);
966		-}
967		-
968		-static void probes_seq_stop(struct seq_file m, void v)
969		-{
970		- mutex_unlock(&probe_lock);
971		-}
972		-
973		-static int probes_seq_show(struct seq_file m, void v)
974		-{
975		- struct trace_kprobe *tk = v;
	1107	+ struct trace_kprobe *tk = to_trace_kprobe(ev);
976	1108	int i;
977	1109
978	1110	seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p');
979	1111	if (trace_kprobe_is_return(tk) && tk->rp.maxactive)
980	1112	seq_printf(m, "%d", tk->rp.maxactive);
981		- seq_printf(m, ":%s/%s", tk->tp.call.class->system,
982		- trace_event_name(&tk->tp.call));
	1113	+ seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp),
	1114	+ trace_probe_name(&tk->tp));
983	1115
984	1116	if (!tk->symbol)
985	1117	seq_printf(m, " 0x%p", tk->rp.kp.addr);
..	..	@@ -996,10 +1128,20 @@
996	1128	return 0;
997	1129	}
998	1130
	1131	+static int probes_seq_show(struct seq_file m, void v)
	1132	+{
	1133	+ struct dyn_event *ev = v;
	1134	+
	1135	+ if (!is_trace_kprobe(ev))
	1136	+ return 0;
	1137	+
	1138	+ return trace_kprobe_show(m, ev);
	1139	+}
	1140	+
999	1141	static const struct seq_operations probes_seq_op = {
1000		- .start = probes_seq_start,
1001		- .next = probes_seq_next,
1002		- .stop = probes_seq_stop,
	1142	+ .start = dyn_event_seq_start,
	1143	+ .next = dyn_event_seq_next,
	1144	+ .stop = dyn_event_seq_stop,
1003	1145	.show = probes_seq_show
1004	1146	};
1005	1147
..	..	@@ -1007,8 +1149,12 @@
1007	1149	{
1008	1150	int ret;
1009	1151
	1152	+ ret = security_locked_down(LOCKDOWN_TRACEFS);
	1153	+ if (ret)
	1154	+ return ret;
	1155	+
1010	1156	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1011		- ret = release_all_trace_kprobes();
	1157	+ ret = dyn_events_release_all(&trace_kprobe_ops);
1012	1158	if (ret < 0)
1013	1159	return ret;
1014	1160	}
..	..	@@ -1020,7 +1166,7 @@
1020	1166	size_t count, loff_t *ppos)
1021	1167	{
1022	1168	return trace_parse_run_command(file, buffer, count, ppos,
1023		- create_trace_kprobe);
	1169	+ create_or_delete_trace_kprobe);
1024	1170	}
1025	1171
1026	1172	static const struct file_operations kprobe_events_ops = {
..	..	@@ -1035,25 +1181,39 @@
1035	1181	/* Probes profiling interfaces */
1036	1182	static int probes_profile_seq_show(struct seq_file m, void v)
1037	1183	{
1038		- struct trace_kprobe *tk = v;
	1184	+ struct dyn_event *ev = v;
	1185	+ struct trace_kprobe *tk;
	1186	+ unsigned long nmissed;
1039	1187
	1188	+ if (!is_trace_kprobe(ev))
	1189	+ return 0;
	1190	+
	1191	+ tk = to_trace_kprobe(ev);
	1192	+ nmissed = trace_kprobe_is_return(tk) ?
	1193	+ tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed;
1040	1194	seq_printf(m, " %-44s %15lu %15lu\n",
1041		- trace_event_name(&tk->tp.call),
	1195	+ trace_probe_name(&tk->tp),
1042	1196	trace_kprobe_nhit(tk),
1043		- tk->rp.kp.nmissed);
	1197	+ nmissed);
1044	1198
1045	1199	return 0;
1046	1200	}
1047	1201
1048	1202	static const struct seq_operations profile_seq_op = {
1049		- .start = probes_seq_start,
1050		- .next = probes_seq_next,
1051		- .stop = probes_seq_stop,
	1203	+ .start = dyn_event_seq_start,
	1204	+ .next = dyn_event_seq_next,
	1205	+ .stop = dyn_event_seq_stop,
1052	1206	.show = probes_profile_seq_show
1053	1207	};
1054	1208
1055	1209	static int profile_open(struct inode inode, struct file file)
1056	1210	{
	1211	+ int ret;
	1212	+
	1213	+ ret = security_locked_down(LOCKDOWN_TRACEFS);
	1214	+ if (ret)
	1215	+ return ret;
	1216	+
1057	1217	return seq_open(file, &profile_seq_op);
1058	1218	}
1059	1219
..	..	@@ -1065,41 +1225,193 @@
1065	1225	.release = seq_release,
1066	1226	};
1067	1227
	1228	+/* Kprobe specific fetch functions */
	1229	+
	1230	+/* Return the length of string -- including null terminal byte */
	1231	+static nokprobe_inline int
	1232	+fetch_store_strlen_user(unsigned long addr)
	1233	+{
	1234	+ const void __user uaddr = (__force const void __user )addr;
	1235	+
	1236	+ return strnlen_user_nofault(uaddr, MAX_STRING_SIZE);
	1237	+}
	1238	+
	1239	+/* Return the length of string -- including null terminal byte */
	1240	+static nokprobe_inline int
	1241	+fetch_store_strlen(unsigned long addr)
	1242	+{
	1243	+ int ret, len = 0;
	1244	+ u8 c;
	1245	+
	1246	+#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
	1247	+ if (addr < TASK_SIZE)
	1248	+ return fetch_store_strlen_user(addr);
	1249	+#endif
	1250	+
	1251	+ do {
	1252	+ ret = copy_from_kernel_nofault(&c, (u8 *)addr + len, 1);
	1253	+ len++;
	1254	+ } while (c && ret == 0 && len < MAX_STRING_SIZE);
	1255	+
	1256	+ return (ret < 0) ? ret : len;
	1257	+}
	1258	+
	1259	+/*
	1260	+ * Fetch a null-terminated string from user. Caller MUST set (u32 )buf
	1261	+ * with max length and relative data location.
	1262	+ */
	1263	+static nokprobe_inline int
	1264	+fetch_store_string_user(unsigned long addr, void dest, void base)
	1265	+{
	1266	+ const void __user uaddr = (__force const void __user )addr;
	1267	+ int maxlen = get_loc_len((u32 )dest);
	1268	+ void *__dest;
	1269	+ long ret;
	1270	+
	1271	+ if (unlikely(!maxlen))
	1272	+ return -ENOMEM;
	1273	+
	1274	+ __dest = get_loc_data(dest, base);
	1275	+
	1276	+ ret = strncpy_from_user_nofault(__dest, uaddr, maxlen);
	1277	+ if (ret >= 0)
	1278	+ (u32 )dest = make_data_loc(ret, __dest - base);
	1279	+
	1280	+ return ret;
	1281	+}
	1282	+
	1283	+/*
	1284	+ * Fetch a null-terminated string. Caller MUST set (u32 )buf with max
	1285	+ * length and relative data location.
	1286	+ */
	1287	+static nokprobe_inline int
	1288	+fetch_store_string(unsigned long addr, void dest, void base)
	1289	+{
	1290	+ int maxlen = get_loc_len((u32 )dest);
	1291	+ void *__dest;
	1292	+ long ret;
	1293	+
	1294	+#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
	1295	+ if ((unsigned long)addr < TASK_SIZE)
	1296	+ return fetch_store_string_user(addr, dest, base);
	1297	+#endif
	1298	+
	1299	+ if (unlikely(!maxlen))
	1300	+ return -ENOMEM;
	1301	+
	1302	+ __dest = get_loc_data(dest, base);
	1303	+
	1304	+ /*
	1305	+ * Try to get string again, since the string can be changed while
	1306	+ * probing.
	1307	+ */
	1308	+ ret = strncpy_from_kernel_nofault(__dest, (void *)addr, maxlen);
	1309	+ if (ret >= 0)
	1310	+ (u32 )dest = make_data_loc(ret, __dest - base);
	1311	+
	1312	+ return ret;
	1313	+}
	1314	+
	1315	+static nokprobe_inline int
	1316	+probe_mem_read_user(void dest, void src, size_t size)
	1317	+{
	1318	+ const void __user uaddr = (__force const void __user )src;
	1319	+
	1320	+ return copy_from_user_nofault(dest, uaddr, size);
	1321	+}
	1322	+
	1323	+static nokprobe_inline int
	1324	+probe_mem_read(void dest, void src, size_t size)
	1325	+{
	1326	+#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
	1327	+ if ((unsigned long)src < TASK_SIZE)
	1328	+ return probe_mem_read_user(dest, src, size);
	1329	+#endif
	1330	+ return copy_from_kernel_nofault(dest, src, size);
	1331	+}
	1332	+
	1333	+/* Note that we don't verify it, since the code does not come from user space */
	1334	+static int
	1335	+process_fetch_insn(struct fetch_insn code, struct pt_regs regs, void *dest,
	1336	+ void *base)
	1337	+{
	1338	+ unsigned long val;
	1339	+
	1340	+retry:
	1341	+ /* 1st stage: get value from context */
	1342	+ switch (code->op) {
	1343	+ case FETCH_OP_REG:
	1344	+ val = regs_get_register(regs, code->param);
	1345	+ break;
	1346	+ case FETCH_OP_STACK:
	1347	+ val = regs_get_kernel_stack_nth(regs, code->param);
	1348	+ break;
	1349	+ case FETCH_OP_STACKP:
	1350	+ val = kernel_stack_pointer(regs);
	1351	+ break;
	1352	+ case FETCH_OP_RETVAL:
	1353	+ val = regs_return_value(regs);
	1354	+ break;
	1355	+ case FETCH_OP_IMM:
	1356	+ val = code->immediate;
	1357	+ break;
	1358	+ case FETCH_OP_COMM:
	1359	+ val = (unsigned long)current->comm;
	1360	+ break;
	1361	+ case FETCH_OP_DATA:
	1362	+ val = (unsigned long)code->data;
	1363	+ break;
	1364	+#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
	1365	+ case FETCH_OP_ARG:
	1366	+ val = regs_get_kernel_argument(regs, code->param);
	1367	+ break;
	1368	+#endif
	1369	+ case FETCH_NOP_SYMBOL: /* Ignore a place holder */
	1370	+ code++;
	1371	+ goto retry;
	1372	+ default:
	1373	+ return -EILSEQ;
	1374	+ }
	1375	+ code++;
	1376	+
	1377	+ return process_fetch_insn_bottom(code, val, dest, base);
	1378	+}
	1379	+NOKPROBE_SYMBOL(process_fetch_insn)
	1380	+
1068	1381	/* Kprobe handler */
1069	1382	static nokprobe_inline void
1070	1383	__kprobe_trace_func(struct trace_kprobe tk, struct pt_regs regs,
1071	1384	struct trace_event_file *trace_file)
1072	1385	{
1073	1386	struct kprobe_trace_entry_head *entry;
1074		- struct ring_buffer_event *event;
1075		- struct ring_buffer *buffer;
1076		- int size, dsize, pc;
1077		- unsigned long irq_flags;
1078		- struct trace_event_call *call = &tk->tp.call;
	1387	+ struct trace_event_call *call = trace_probe_event_call(&tk->tp);
	1388	+ struct trace_event_buffer fbuffer;
	1389	+ int dsize;
1079	1390
1080	1391	WARN_ON(call != trace_file->event_call);
1081	1392
1082	1393	if (trace_trigger_soft_disabled(trace_file))
1083	1394	return;
1084	1395
1085		- local_save_flags(irq_flags);
1086		- pc = preempt_count();
	1396	+ fbuffer.trace_ctx = tracing_gen_ctx();
	1397	+ fbuffer.trace_file = trace_file;
1087	1398
1088	1399	dsize = __get_data_size(&tk->tp, regs);
1089		- size = sizeof(*entry) + tk->tp.size + dsize;
1090	1400
1091		- event = trace_event_buffer_lock_reserve(&buffer, trace_file,
1092		- call->event.type,
1093		- size, irq_flags, pc);
1094		- if (!event)
	1401	+ fbuffer.event =
	1402	+ trace_event_buffer_lock_reserve(&fbuffer.buffer, trace_file,
	1403	+ call->event.type,
	1404	+ sizeof(*entry) + tk->tp.size + dsize,
	1405	+ fbuffer.trace_ctx);
	1406	+ if (!fbuffer.event)
1095	1407	return;
1096	1408
1097		- entry = ring_buffer_event_data(event);
	1409	+ fbuffer.regs = regs;
	1410	+ entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
1098	1411	entry->ip = (unsigned long)tk->rp.kp.addr;
1099		- store_trace_args(sizeof(entry), &tk->tp, regs, (u8 )&entry[1], dsize);
	1412	+ store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1100	1413
1101		- event_trigger_unlock_commit_regs(trace_file, buffer, event,
1102		- entry, irq_flags, pc, regs);
	1414	+ trace_event_buffer_commit(&fbuffer);
1103	1415	}
1104	1416
1105	1417	static void
..	..	@@ -1107,7 +1419,7 @@
1107	1419	{
1108	1420	struct event_file_link *link;
1109	1421
1110		- list_for_each_entry_rcu(link, &tk->tp.files, list)
	1422	+ trace_probe_for_each_link_rcu(link, &tk->tp)
1111	1423	__kprobe_trace_func(tk, regs, link->file);
1112	1424	}
1113	1425	NOKPROBE_SYMBOL(kprobe_trace_func);
..	..	@@ -1119,36 +1431,34 @@
1119	1431	struct trace_event_file *trace_file)
1120	1432	{
1121	1433	struct kretprobe_trace_entry_head *entry;
1122		- struct ring_buffer_event *event;
1123		- struct ring_buffer *buffer;
1124		- int size, pc, dsize;
1125		- unsigned long irq_flags;
1126		- struct trace_event_call *call = &tk->tp.call;
	1434	+ struct trace_event_buffer fbuffer;
	1435	+ struct trace_event_call *call = trace_probe_event_call(&tk->tp);
	1436	+ int dsize;
1127	1437
1128	1438	WARN_ON(call != trace_file->event_call);
1129	1439
1130	1440	if (trace_trigger_soft_disabled(trace_file))
1131	1441	return;
1132	1442
1133		- local_save_flags(irq_flags);
1134		- pc = preempt_count();
	1443	+ fbuffer.trace_ctx = tracing_gen_ctx();
	1444	+ fbuffer.trace_file = trace_file;
1135	1445
1136	1446	dsize = __get_data_size(&tk->tp, regs);
1137		- size = sizeof(*entry) + tk->tp.size + dsize;
1138		-
1139		- event = trace_event_buffer_lock_reserve(&buffer, trace_file,
1140		- call->event.type,
1141		- size, irq_flags, pc);
1142		- if (!event)
	1447	+ fbuffer.event =
	1448	+ trace_event_buffer_lock_reserve(&fbuffer.buffer, trace_file,
	1449	+ call->event.type,
	1450	+ sizeof(*entry) + tk->tp.size + dsize,
	1451	+ fbuffer.trace_ctx);
	1452	+ if (!fbuffer.event)
1143	1453	return;
1144	1454
1145		- entry = ring_buffer_event_data(event);
	1455	+ fbuffer.regs = regs;
	1456	+ entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
1146	1457	entry->func = (unsigned long)tk->rp.kp.addr;
1147	1458	entry->ret_ip = (unsigned long)ri->ret_addr;
1148		- store_trace_args(sizeof(entry), &tk->tp, regs, (u8 )&entry[1], dsize);
	1459	+ store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1149	1460
1150		- event_trigger_unlock_commit_regs(trace_file, buffer, event,
1151		- entry, irq_flags, pc, regs);
	1461	+ trace_event_buffer_commit(&fbuffer);
1152	1462	}
1153	1463
1154	1464	static void
..	..	@@ -1157,7 +1467,7 @@
1157	1467	{
1158	1468	struct event_file_link *link;
1159	1469
1160		- list_for_each_entry_rcu(link, &tk->tp.files, list)
	1470	+ trace_probe_for_each_link_rcu(link, &tk->tp)
1161	1471	__kretprobe_trace_func(tk, ri, regs, link->file);
1162	1472	}
1163	1473	NOKPROBE_SYMBOL(kretprobe_trace_func);
..	..	@@ -1170,24 +1480,23 @@
1170	1480	struct kprobe_trace_entry_head *field;
1171	1481	struct trace_seq *s = &iter->seq;
1172	1482	struct trace_probe *tp;
1173		- u8 *data;
1174		- int i;
1175	1483
1176	1484	field = (struct kprobe_trace_entry_head *)iter->ent;
1177		- tp = container_of(event, struct trace_probe, call.event);
	1485	+ tp = trace_probe_primary_from_call(
	1486	+ container_of(event, struct trace_event_call, event));
	1487	+ if (WARN_ON_ONCE(!tp))
	1488	+ goto out;
1178	1489
1179		- trace_seq_printf(s, "%s: (", trace_event_name(&tp->call));
	1490	+ trace_seq_printf(s, "%s: (", trace_probe_name(tp));
1180	1491
1181	1492	if (!seq_print_ip_sym(s, field->ip, flags \| TRACE_ITER_SYM_OFFSET))
1182	1493	goto out;
1183	1494
1184	1495	trace_seq_putc(s, ')');
1185	1496
1186		- data = (u8 *)&field[1];
1187		- for (i = 0; i < tp->nr_args; i++)
1188		- if (!tp->args[i].type->print(s, tp->args[i].name,
1189		- data + tp->args[i].offset, field))
1190		- goto out;
	1497	+ if (print_probe_args(s, tp->args, tp->nr_args,
	1498	+ (u8 *)&field[1], field) < 0)
	1499	+ goto out;
1191	1500
1192	1501	trace_seq_putc(s, '\n');
1193	1502	out:
..	..	@@ -1201,13 +1510,14 @@
1201	1510	struct kretprobe_trace_entry_head *field;
1202	1511	struct trace_seq *s = &iter->seq;
1203	1512	struct trace_probe *tp;
1204		- u8 *data;
1205		- int i;
1206	1513
1207	1514	field = (struct kretprobe_trace_entry_head *)iter->ent;
1208		- tp = container_of(event, struct trace_probe, call.event);
	1515	+ tp = trace_probe_primary_from_call(
	1516	+ container_of(event, struct trace_event_call, event));
	1517	+ if (WARN_ON_ONCE(!tp))
	1518	+ goto out;
1209	1519
1210		- trace_seq_printf(s, "%s: (", trace_event_name(&tp->call));
	1520	+ trace_seq_printf(s, "%s: (", trace_probe_name(tp));
1211	1521
1212	1522	if (!seq_print_ip_sym(s, field->ret_ip, flags \| TRACE_ITER_SYM_OFFSET))
1213	1523	goto out;
..	..	@@ -1219,11 +1529,9 @@
1219	1529
1220	1530	trace_seq_putc(s, ')');
1221	1531
1222		- data = (u8 *)&field[1];
1223		- for (i = 0; i < tp->nr_args; i++)
1224		- if (!tp->args[i].type->print(s, tp->args[i].name,
1225		- data + tp->args[i].offset, field))
1226		- goto out;
	1532	+ if (print_probe_args(s, tp->args, tp->nr_args,
	1533	+ (u8 *)&field[1], field) < 0)
	1534	+ goto out;
1227	1535
1228	1536	trace_seq_putc(s, '\n');
1229	1537
..	..	@@ -1234,49 +1542,33 @@
1234	1542
1235	1543	static int kprobe_event_define_fields(struct trace_event_call *event_call)
1236	1544	{
1237		- int ret, i;
	1545	+ int ret;
1238	1546	struct kprobe_trace_entry_head field;
1239		- struct trace_kprobe tk = (struct trace_kprobe )event_call->data;
	1547	+ struct trace_probe *tp;
	1548	+
	1549	+ tp = trace_probe_primary_from_call(event_call);
	1550	+ if (WARN_ON_ONCE(!tp))
	1551	+ return -ENOENT;
1240	1552
1241	1553	DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
1242		- /* Set argument names as fields */
1243		- for (i = 0; i < tk->tp.nr_args; i++) {
1244		- struct probe_arg *parg = &tk->tp.args[i];
1245	1554
1246		- ret = trace_define_field(event_call, parg->type->fmttype,
1247		- parg->name,
1248		- sizeof(field) + parg->offset,
1249		- parg->type->size,
1250		- parg->type->is_signed,
1251		- FILTER_OTHER);
1252		- if (ret)
1253		- return ret;
1254		- }
1255		- return 0;
	1555	+ return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
1256	1556	}
1257	1557
1258	1558	static int kretprobe_event_define_fields(struct trace_event_call *event_call)
1259	1559	{
1260		- int ret, i;
	1560	+ int ret;
1261	1561	struct kretprobe_trace_entry_head field;
1262		- struct trace_kprobe tk = (struct trace_kprobe )event_call->data;
	1562	+ struct trace_probe *tp;
	1563	+
	1564	+ tp = trace_probe_primary_from_call(event_call);
	1565	+ if (WARN_ON_ONCE(!tp))
	1566	+ return -ENOENT;
1263	1567
1264	1568	DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1265	1569	DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
1266		- /* Set argument names as fields */
1267		- for (i = 0; i < tk->tp.nr_args; i++) {
1268		- struct probe_arg *parg = &tk->tp.args[i];
1269	1570
1270		- ret = trace_define_field(event_call, parg->type->fmttype,
1271		- parg->name,
1272		- sizeof(field) + parg->offset,
1273		- parg->type->size,
1274		- parg->type->is_signed,
1275		- FILTER_OTHER);
1276		- if (ret)
1277		- return ret;
1278		- }
1279		- return 0;
	1571	+ return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
1280	1572	}
1281	1573
1282	1574	#ifdef CONFIG_PERF_EVENTS
..	..	@@ -1285,7 +1577,7 @@
1285	1577	static int
1286	1578	kprobe_perf_func(struct trace_kprobe tk, struct pt_regs regs)
1287	1579	{
1288		- struct trace_event_call *call = &tk->tp.call;
	1580	+ struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1289	1581	struct kprobe_trace_entry_head *entry;
1290	1582	struct hlist_head *head;
1291	1583	int size, __size, dsize;
..	..	@@ -1323,7 +1615,7 @@
1323	1615
1324	1616	entry->ip = (unsigned long)tk->rp.kp.addr;
1325	1617	memset(&entry[1], 0, dsize);
1326		- store_trace_args(sizeof(entry), &tk->tp, regs, (u8 )&entry[1], dsize);
	1618	+ store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1327	1619	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1328	1620	head, NULL);
1329	1621	return 0;
..	..	@@ -1335,7 +1627,7 @@
1335	1627	kretprobe_perf_func(struct trace_kprobe tk, struct kretprobe_instance ri,
1336	1628	struct pt_regs *regs)
1337	1629	{
1338		- struct trace_event_call *call = &tk->tp.call;
	1630	+ struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1339	1631	struct kretprobe_trace_entry_head *entry;
1340	1632	struct hlist_head *head;
1341	1633	int size, __size, dsize;
..	..	@@ -1359,7 +1651,7 @@
1359	1651
1360	1652	entry->func = (unsigned long)tk->rp.kp.addr;
1361	1653	entry->ret_ip = (unsigned long)ri->ret_addr;
1362		- store_trace_args(sizeof(entry), &tk->tp, regs, (u8 )&entry[1], dsize);
	1654	+ store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1363	1655	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1364	1656	head, NULL);
1365	1657	}
..	..	@@ -1376,7 +1668,7 @@
1376	1668	if (perf_type_tracepoint)
1377	1669	tk = find_trace_kprobe(pevent, group);
1378	1670	else
1379		- tk = event->tp_event->data;
	1671	+ tk = trace_kprobe_primary_from_call(event->tp_event);
1380	1672	if (!tk)
1381	1673	return -EINVAL;
1382	1674
..	..	@@ -1404,20 +1696,19 @@
1404	1696	static int kprobe_register(struct trace_event_call *event,
1405	1697	enum trace_reg type, void *data)
1406	1698	{
1407		- struct trace_kprobe tk = (struct trace_kprobe )event->data;
1408	1699	struct trace_event_file *file = data;
1409	1700
1410	1701	switch (type) {
1411	1702	case TRACE_REG_REGISTER:
1412		- return enable_trace_kprobe(tk, file);
	1703	+ return enable_trace_kprobe(event, file);
1413	1704	case TRACE_REG_UNREGISTER:
1414		- return disable_trace_kprobe(tk, file);
	1705	+ return disable_trace_kprobe(event, file);
1415	1706
1416	1707	#ifdef CONFIG_PERF_EVENTS
1417	1708	case TRACE_REG_PERF_REGISTER:
1418		- return enable_trace_kprobe(tk, NULL);
	1709	+ return enable_trace_kprobe(event, NULL);
1419	1710	case TRACE_REG_PERF_UNREGISTER:
1420		- return disable_trace_kprobe(tk, NULL);
	1711	+ return disable_trace_kprobe(event, NULL);
1421	1712	case TRACE_REG_PERF_OPEN:
1422	1713	case TRACE_REG_PERF_CLOSE:
1423	1714	case TRACE_REG_PERF_ADD:
..	..	@@ -1435,10 +1726,10 @@
1435	1726
1436	1727	raw_cpu_inc(*tk->nhit);
1437	1728
1438		- if (tk->tp.flags & TP_FLAG_TRACE)
	1729	+ if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE))
1439	1730	kprobe_trace_func(tk, regs);
1440	1731	#ifdef CONFIG_PERF_EVENTS
1441		- if (tk->tp.flags & TP_FLAG_PROFILE)
	1732	+ if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE))
1442	1733	ret = kprobe_perf_func(tk, regs);
1443	1734	#endif
1444	1735	return ret;
..	..	@@ -1452,10 +1743,10 @@
1452	1743
1453	1744	raw_cpu_inc(*tk->nhit);
1454	1745
1455		- if (tk->tp.flags & TP_FLAG_TRACE)
	1746	+ if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE))
1456	1747	kretprobe_trace_func(tk, ri, regs);
1457	1748	#ifdef CONFIG_PERF_EVENTS
1458		- if (tk->tp.flags & TP_FLAG_PROFILE)
	1749	+ if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE))
1459	1750	kretprobe_perf_func(tk, ri, regs);
1460	1751	#endif
1461	1752	return 0; /* We don't tweek kernel, so just return 0 */
..	..	@@ -1470,56 +1761,44 @@
1470	1761	.trace = print_kprobe_event
1471	1762	};
1472	1763
1473		-static inline void init_trace_event_call(struct trace_kprobe *tk,
1474		- struct trace_event_call *call)
	1764	+static struct trace_event_fields kretprobe_fields_array[] = {
	1765	+ { .type = TRACE_FUNCTION_TYPE,
	1766	+ .define_fields = kretprobe_event_define_fields },
	1767	+ {}
	1768	+};
	1769	+
	1770	+static struct trace_event_fields kprobe_fields_array[] = {
	1771	+ { .type = TRACE_FUNCTION_TYPE,
	1772	+ .define_fields = kprobe_event_define_fields },
	1773	+ {}
	1774	+};
	1775	+
	1776	+static inline void init_trace_event_call(struct trace_kprobe *tk)
1475	1777	{
1476		- INIT_LIST_HEAD(&call->class->fields);
	1778	+ struct trace_event_call *call = trace_probe_event_call(&tk->tp);
	1779	+
1477	1780	if (trace_kprobe_is_return(tk)) {
1478	1781	call->event.funcs = &kretprobe_funcs;
1479		- call->class->define_fields = kretprobe_event_define_fields;
	1782	+ call->class->fields_array = kretprobe_fields_array;
1480	1783	} else {
1481	1784	call->event.funcs = &kprobe_funcs;
1482		- call->class->define_fields = kprobe_event_define_fields;
	1785	+ call->class->fields_array = kprobe_fields_array;
1483	1786	}
1484	1787
1485	1788	call->flags = TRACE_EVENT_FL_KPROBE;
1486	1789	call->class->reg = kprobe_register;
1487		- call->data = tk;
1488	1790	}
1489	1791
1490	1792	static int register_kprobe_event(struct trace_kprobe *tk)
1491	1793	{
1492		- struct trace_event_call *call = &tk->tp.call;
1493		- int ret = 0;
	1794	+ init_trace_event_call(tk);
1494	1795
1495		- init_trace_event_call(tk, call);
1496		-
1497		- if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0)
1498		- return -ENOMEM;
1499		- ret = register_trace_event(&call->event);
1500		- if (!ret) {
1501		- kfree(call->print_fmt);
1502		- return -ENODEV;
1503		- }
1504		- ret = trace_add_event_call(call);
1505		- if (ret) {
1506		- pr_info("Failed to register kprobe event: %s\n",
1507		- trace_event_name(call));
1508		- kfree(call->print_fmt);
1509		- unregister_trace_event(&call->event);
1510		- }
1511		- return ret;
	1796	+ return trace_probe_register_event_call(&tk->tp);
1512	1797	}
1513	1798
1514	1799	static int unregister_kprobe_event(struct trace_kprobe *tk)
1515	1800	{
1516		- int ret;
1517		-
1518		- /* tp->event is unregistered in trace_remove_event_call() */
1519		- ret = trace_remove_event_call(&tk->tp.call);
1520		- if (!ret)
1521		- kfree(tk->tp.call.print_fmt);
1522		- return ret;
	1801	+ return trace_probe_unregister_event_call(&tk->tp);
1523	1802	}
1524	1803
1525	1804	#ifdef CONFIG_PERF_EVENTS
..	..	@@ -1533,7 +1812,7 @@
1533	1812	char *event;
1534	1813
1535	1814	/*
1536		- * local trace_kprobes are not added to probe_list, so they are never
	1815	+ * local trace_kprobes are not added to dyn_event, so they are never
1537	1816	* searched in find_trace_kprobe(). Therefore, there is no concern of
1538	1817	* duplicated name here.
1539	1818	*/
..	..	@@ -1549,20 +1828,18 @@
1549	1828	return ERR_CAST(tk);
1550	1829	}
1551	1830
1552		- init_trace_event_call(tk, &tk->tp.call);
	1831	+ init_trace_event_call(tk);
1553	1832
1554		- if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) {
	1833	+ if (traceprobe_set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) {
1555	1834	ret = -ENOMEM;
1556	1835	goto error;
1557	1836	}
1558	1837
1559	1838	ret = __register_trace_kprobe(tk);
1560		- if (ret < 0) {
1561		- kfree(tk->tp.call.print_fmt);
	1839	+ if (ret < 0)
1562	1840	goto error;
1563		- }
1564	1841
1565		- return &tk->tp.call;
	1842	+ return trace_probe_event_call(&tk->tp);
1566	1843	error:
1567	1844	free_trace_kprobe(tk);
1568	1845	return ERR_PTR(ret);
..	..	@@ -1572,7 +1849,9 @@
1572	1849	{
1573	1850	struct trace_kprobe *tk;
1574	1851
1575		- tk = container_of(event_call, struct trace_kprobe, tp.call);
	1852	+ tk = trace_kprobe_primary_from_call(event_call);
	1853	+ if (unlikely(!tk))
	1854	+ return;
1576	1855
1577	1856	if (trace_probe_is_enabled(&tk->tp)) {
1578	1857	WARN_ON(1);
..	..	@@ -1581,25 +1860,78 @@
1581	1860
1582	1861	__unregister_trace_kprobe(tk);
1583	1862
1584		- kfree(tk->tp.call.print_fmt);
1585	1863	free_trace_kprobe(tk);
1586	1864	}
1587	1865	#endif /* CONFIG_PERF_EVENTS */
1588	1866
1589		-/* Make a tracefs interface for controlling probe points */
1590		-static __init int init_kprobe_trace(void)
	1867	+static __init void enable_boot_kprobe_events(void)
1591	1868	{
1592		- struct dentry *d_tracer;
1593		- struct dentry *entry;
	1869	+ struct trace_array *tr = top_trace_array();
	1870	+ struct trace_event_file *file;
	1871	+ struct trace_kprobe *tk;
	1872	+ struct dyn_event *pos;
	1873	+
	1874	+ mutex_lock(&event_mutex);
	1875	+ for_each_trace_kprobe(tk, pos) {
	1876	+ list_for_each_entry(file, &tr->events, list)
	1877	+ if (file->event_call == trace_probe_event_call(&tk->tp))
	1878	+ trace_event_enable_disable(file, 1, 0);
	1879	+ }
	1880	+ mutex_unlock(&event_mutex);
	1881	+}
	1882	+
	1883	+static __init void setup_boot_kprobe_events(void)
	1884	+{
	1885	+ char p, cmd = kprobe_boot_events_buf;
	1886	+ int ret;
	1887	+
	1888	+ strreplace(kprobe_boot_events_buf, ',', ' ');
	1889	+
	1890	+ while (cmd && *cmd != '\0') {
	1891	+ p = strchr(cmd, ';');
	1892	+ if (p)
	1893	+ *p++ = '\0';
	1894	+
	1895	+ ret = trace_run_command(cmd, create_or_delete_trace_kprobe);
	1896	+ if (ret)
	1897	+ pr_warn("Failed to add event(%d): %s\n", ret, cmd);
	1898	+
	1899	+ cmd = p;
	1900	+ }
	1901	+
	1902	+ enable_boot_kprobe_events();
	1903	+}
	1904	+
	1905	+/*
	1906	+ * Register dynevent at core_initcall. This allows kernel to setup kprobe
	1907	+ * events in postcore_initcall without tracefs.
	1908	+ */
	1909	+static __init int init_kprobe_trace_early(void)
	1910	+{
	1911	+ int ret;
	1912	+
	1913	+ ret = dyn_event_register(&trace_kprobe_ops);
	1914	+ if (ret)
	1915	+ return ret;
1594	1916
1595	1917	if (register_module_notifier(&trace_kprobe_module_nb))
1596	1918	return -EINVAL;
1597	1919
1598		- d_tracer = tracing_init_dentry();
1599		- if (IS_ERR(d_tracer))
	1920	+ return 0;
	1921	+}
	1922	+core_initcall(init_kprobe_trace_early);
	1923	+
	1924	+/* Make a tracefs interface for controlling probe points */
	1925	+static __init int init_kprobe_trace(void)
	1926	+{
	1927	+ int ret;
	1928	+ struct dentry *entry;
	1929	+
	1930	+ ret = tracing_init_dentry();
	1931	+ if (ret)
1600	1932	return 0;
1601	1933
1602		- entry = tracefs_create_file("kprobe_events", 0644, d_tracer,
	1934	+ entry = tracefs_create_file("kprobe_events", 0644, NULL,
1603	1935	NULL, &kprobe_events_ops);
1604	1936
1605	1937	/* Event list interface */
..	..	@@ -1607,11 +1939,14 @@
1607	1939	pr_warn("Could not create tracefs 'kprobe_events' entry\n");
1608	1940
1609	1941	/* Profile interface */
1610		- entry = tracefs_create_file("kprobe_profile", 0444, d_tracer,
	1942	+ entry = tracefs_create_file("kprobe_profile", 0444, NULL,
1611	1943	NULL, &kprobe_profile_ops);
1612	1944
1613	1945	if (!entry)
1614	1946	pr_warn("Could not create tracefs 'kprobe_profile' entry\n");
	1947	+
	1948	+ setup_boot_kprobe_events();
	1949	+
1615	1950	return 0;
1616	1951	}
1617	1952	fs_initcall(init_kprobe_trace);
..	..	@@ -1624,7 +1959,7 @@
1624	1959	struct trace_event_file *file;
1625	1960
1626	1961	list_for_each_entry(file, &tr->events, list)
1627		- if (file->event_call == &tk->tp.call)
	1962	+ if (file->event_call == trace_probe_event_call(&tk->tp))
1628	1963	return file;
1629	1964
1630	1965	return NULL;
..	..	@@ -1644,13 +1979,15 @@
1644	1979	if (tracing_is_disabled())
1645	1980	return -ENODEV;
1646	1981
	1982	+ if (tracing_selftest_disabled)
	1983	+ return 0;
	1984	+
1647	1985	target = kprobe_trace_selftest_target;
1648	1986
1649	1987	pr_info("Testing kprobe tracing: ");
1650	1988
1651		- ret = trace_run_command("p:testprobe kprobe_trace_selftest_target "
1652		- "$stack $stack0 +0($stack)",
1653		- create_trace_kprobe);
	1989	+ ret = trace_run_command("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)",
	1990	+ create_or_delete_trace_kprobe);
1654	1991	if (WARN_ON_ONCE(ret)) {
1655	1992	pr_warn("error on probing function entry.\n");
1656	1993	warn++;
..	..	@@ -1666,12 +2003,13 @@
1666	2003	pr_warn("error on getting probe file.\n");
1667	2004	warn++;
1668	2005	} else
1669		- enable_trace_kprobe(tk, file);
	2006	+ enable_trace_kprobe(
	2007	+ trace_probe_event_call(&tk->tp), file);
1670	2008	}
1671	2009	}
1672	2010
1673		- ret = trace_run_command("r:testprobe2 kprobe_trace_selftest_target "
1674		- "$retval", create_trace_kprobe);
	2011	+ ret = trace_run_command("r:testprobe2 kprobe_trace_selftest_target $retval",
	2012	+ create_or_delete_trace_kprobe);
1675	2013	if (WARN_ON_ONCE(ret)) {
1676	2014	pr_warn("error on probing function return.\n");
1677	2015	warn++;
..	..	@@ -1687,7 +2025,8 @@
1687	2025	pr_warn("error on getting probe file.\n");
1688	2026	warn++;
1689	2027	} else
1690		- enable_trace_kprobe(tk, file);
	2028	+ enable_trace_kprobe(
	2029	+ trace_probe_event_call(&tk->tp), file);
1691	2030	}
1692	2031	}
1693	2032
..	..	@@ -1720,7 +2059,8 @@
1720	2059	pr_warn("error on getting probe file.\n");
1721	2060	warn++;
1722	2061	} else
1723		- disable_trace_kprobe(tk, file);
	2062	+ disable_trace_kprobe(
	2063	+ trace_probe_event_call(&tk->tp), file);
1724	2064	}
1725	2065
1726	2066	tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
..	..	@@ -1738,23 +2078,28 @@
1738	2078	pr_warn("error on getting probe file.\n");
1739	2079	warn++;
1740	2080	} else
1741		- disable_trace_kprobe(tk, file);
	2081	+ disable_trace_kprobe(
	2082	+ trace_probe_event_call(&tk->tp), file);
1742	2083	}
1743	2084
1744		- ret = trace_run_command("-:testprobe", create_trace_kprobe);
	2085	+ ret = trace_run_command("-:testprobe", create_or_delete_trace_kprobe);
1745	2086	if (WARN_ON_ONCE(ret)) {
1746	2087	pr_warn("error on deleting a probe.\n");
1747	2088	warn++;
1748	2089	}
1749	2090
1750		- ret = trace_run_command("-:testprobe2", create_trace_kprobe);
	2091	+ ret = trace_run_command("-:testprobe2", create_or_delete_trace_kprobe);
1751	2092	if (WARN_ON_ONCE(ret)) {
1752	2093	pr_warn("error on deleting a probe.\n");
1753	2094	warn++;
1754	2095	}
1755	2096
1756	2097	end:
1757		- release_all_trace_kprobes();
	2098	+ ret = dyn_events_release_all(&trace_kprobe_ops);
	2099	+ if (WARN_ON_ONCE(ret)) {
	2100	+ pr_warn("error on cleaning up probes.\n");
	2101	+ warn++;
	2102	+ }
1758	2103	/*
1759	2104	* Wait for the optimizer work to finish. Otherwise it might fiddle
1760	2105	* with probes in already freed __init text.