~hc/RK356X_SDK_RELEASE.git

..	..	@@ -23,6 +23,7 @@
23	23	#include <linux/list.h>
24	24	#include <linux/module.h>
25	25	#include <linux/memory.h>
	26	+#include <linux/vmalloc.h>
26	27
27	28	#include <trace/syscall.h>
28	29
..	..	@@ -34,88 +35,48 @@
34	35
35	36	#ifdef CONFIG_DYNAMIC_FTRACE
36	37
	38	+static int ftrace_poke_late = 0;
	39	+
37	40	int ftrace_arch_code_modify_prepare(void)
38	41	__acquires(&text_mutex)
39	42	{
	43	+ /*
	44	+ * Need to grab text_mutex to prevent a race from module loading
	45	+ * and live kernel patching from changing the text permissions while
	46	+ * ftrace has it set to "read/write".
	47	+ */
40	48	mutex_lock(&text_mutex);
41		- set_kernel_text_rw();
42		- set_all_modules_text_rw();
	49	+ ftrace_poke_late = 1;
43	50	return 0;
44	51	}
45	52
46	53	int ftrace_arch_code_modify_post_process(void)
47	54	__releases(&text_mutex)
48	55	{
49		- set_all_modules_text_ro();
50		- set_kernel_text_ro();
	56	+ /*
	57	+ * ftrace_make_{call,nop}() may be called during
	58	+ * module load, and we need to finish the text_poke_queue()
	59	+ * that they do, here.
	60	+ */
	61	+ text_poke_finish();
	62	+ ftrace_poke_late = 0;
51	63	mutex_unlock(&text_mutex);
52	64	return 0;
53	65	}
54	66
55		-union ftrace_code_union {
56		- char code[MCOUNT_INSN_SIZE];
57		- struct {
58		- unsigned char op;
59		- int offset;
60		- } __attribute__((packed));
61		-};
62		-
63		-static int ftrace_calc_offset(long ip, long addr)
64		-{
65		- return (int)(addr - ip);
66		-}
67		-
68		-static unsigned char *
69		-ftrace_text_replace(unsigned char op, unsigned long ip, unsigned long addr)
70		-{
71		- static union ftrace_code_union calc;
72		-
73		- calc.op = op;
74		- calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
75		-
76		- return calc.code;
77		-}
78		-
79		-static unsigned char *
80		-ftrace_call_replace(unsigned long ip, unsigned long addr)
81		-{
82		- return ftrace_text_replace(0xe8, ip, addr);
83		-}
84		-
85		-static inline int
86		-within(unsigned long addr, unsigned long start, unsigned long end)
87		-{
88		- return addr >= start && addr < end;
89		-}
90		-
91		-static unsigned long text_ip_addr(unsigned long ip)
92		-{
93		- /*
94		- * On x86_64, kernel text mappings are mapped read-only, so we use
95		- * the kernel identity mapping instead of the kernel text mapping
96		- * to modify the kernel text.
97		- *
98		- * For 32bit kernels, these mappings are same and we can use
99		- * kernel identity mapping to modify code.
100		- */
101		- if (within(ip, (unsigned long)_text, (unsigned long)_etext))
102		- ip = (unsigned long)__va(__pa_symbol(ip));
103		-
104		- return ip;
105		-}
106		-
107		-static const unsigned char *ftrace_nop_replace(void)
	67	+static const char *ftrace_nop_replace(void)
108	68	{
109	69	return ideal_nops[NOP_ATOMIC5];
110	70	}
111	71
112		-static int
113		-ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
114		- unsigned const char *new_code)
	72	+static const char *ftrace_call_replace(unsigned long ip, unsigned long addr)
115	73	{
116		- unsigned char replaced[MCOUNT_INSN_SIZE];
	74	+ return text_gen_insn(CALL_INSN_OPCODE, (void )ip, (void )addr);
	75	+}
117	76
118		- ftrace_expected = old_code;
	77	+static int ftrace_verify_code(unsigned long ip, const char *old_code)
	78	+{
	79	+ char cur_code[MCOUNT_INSN_SIZE];
119	80
120	81	/*
121	82	* Note:
..	..	@@ -124,31 +85,47 @@
124	85	* Carefully read and modify the code with probe_kernel_*(), and make
125	86	* sure what we read is what we expected it to be before modifying it.
126	87	*/
127		-
128	88	/* read the text we want to modify */
129		- if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
	89	+ if (copy_from_kernel_nofault(cur_code, (void *)ip, MCOUNT_INSN_SIZE)) {
	90	+ WARN_ON(1);
130	91	return -EFAULT;
	92	+ }
131	93
132	94	/* Make sure it is what we expect it to be */
133		- if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
	95	+ if (memcmp(cur_code, old_code, MCOUNT_INSN_SIZE) != 0) {
	96	+ ftrace_expected = old_code;
	97	+ WARN_ON(1);
134	98	return -EINVAL;
135		-
136		- ip = text_ip_addr(ip);
137		-
138		- /* replace the text with the new text */
139		- if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
140		- return -EPERM;
141		-
142		- sync_core();
	99	+ }
143	100
144	101	return 0;
145	102	}
146	103
147		-int ftrace_make_nop(struct module *mod,
148		- struct dyn_ftrace *rec, unsigned long addr)
	104	+/*
	105	+ * Marked __ref because it calls text_poke_early() which is .init.text. That is
	106	+ * ok because that call will happen early, during boot, when .init sections are
	107	+ * still present.
	108	+ */
	109	+static int __ref
	110	+ftrace_modify_code_direct(unsigned long ip, const char *old_code,
	111	+ const char *new_code)
149	112	{
150		- unsigned const char new, old;
	113	+ int ret = ftrace_verify_code(ip, old_code);
	114	+ if (ret)
	115	+ return ret;
	116	+
	117	+ /* replace the text with the new text */
	118	+ if (ftrace_poke_late)
	119	+ text_poke_queue((void *)ip, new_code, MCOUNT_INSN_SIZE, NULL);
	120	+ else
	121	+ text_poke_early((void *)ip, new_code, MCOUNT_INSN_SIZE);
	122	+ return 0;
	123	+}
	124	+
	125	+int ftrace_make_nop(struct module mod, struct dyn_ftrace rec, unsigned long addr)
	126	+{
151	127	unsigned long ip = rec->ip;
	128	+ const char new, old;
152	129
153	130	old = ftrace_call_replace(ip, addr);
154	131	new = ftrace_nop_replace();
..	..	@@ -162,19 +139,20 @@
162	139	* just modify the code directly.
163	140	*/
164	141	if (addr == MCOUNT_ADDR)
165		- return ftrace_modify_code_direct(rec->ip, old, new);
	142	+ return ftrace_modify_code_direct(ip, old, new);
166	143
167		- ftrace_expected = NULL;
168		-
169		- /* Normal cases use add_brk_on_nop */
	144	+ /*
	145	+ * x86 overrides ftrace_replace_code -- this function will never be used
	146	+ * in this case.
	147	+ */
170	148	WARN_ONCE(1, "invalid use of ftrace_make_nop");
171	149	return -EINVAL;
172	150	}
173	151
174	152	int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
175	153	{
176		- unsigned const char new, old;
177	154	unsigned long ip = rec->ip;
	155	+ const char new, old;
178	156
179	157	old = ftrace_nop_replace();
180	158	new = ftrace_call_replace(ip, addr);
..	..	@@ -182,43 +160,6 @@
182	160	/* Should only be called when module is loaded */
183	161	return ftrace_modify_code_direct(rec->ip, old, new);
184	162	}
185		-
186		-/*
187		- * The modifying_ftrace_code is used to tell the breakpoint
188		- * handler to call ftrace_int3_handler(). If it fails to
189		- * call this handler for a breakpoint added by ftrace, then
190		- * the kernel may crash.
191		- *
192		- * As atomic_writes on x86 do not need a barrier, we do not
193		- * need to add smp_mb()s for this to work. It is also considered
194		- * that we can not read the modifying_ftrace_code before
195		- * executing the breakpoint. That would be quite remarkable if
196		- * it could do that. Here's the flow that is required:
197		- *
198		- * CPU-0 CPU-1
199		- *
200		- * atomic_inc(mfc);
201		- * write int3s
202		- * <trap-int3> // implicit (r)mb
203		- * if (atomic_read(mfc))
204		- * call ftrace_int3_handler()
205		- *
206		- * Then when we are finished:
207		- *
208		- * atomic_dec(mfc);
209		- *
210		- * If we hit a breakpoint that was not set by ftrace, it does not
211		- * matter if ftrace_int3_handler() is called or not. It will
212		- * simply be ignored. But it is crucial that a ftrace nop/caller
213		- * breakpoint is handled. No other user should ever place a
214		- * breakpoint on an ftrace nop/caller location. It must only
215		- * be done by this code.
216		- */
217		-atomic_t modifying_ftrace_code __read_mostly;
218		-
219		-static int
220		-ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
221		- unsigned const char *new_code);
222	163
223	164	/*
224	165	* Should never be called:
..	..	@@ -232,458 +173,84 @@
232	173	unsigned long addr)
233	174	{
234	175	WARN_ON(1);
235		- ftrace_expected = NULL;
236	176	return -EINVAL;
237		-}
238		-
239		-static unsigned long ftrace_update_func;
240		-static unsigned long ftrace_update_func_call;
241		-
242		-static int update_ftrace_func(unsigned long ip, void *new)
243		-{
244		- unsigned char old[MCOUNT_INSN_SIZE];
245		- int ret;
246		-
247		- memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);
248		-
249		- ftrace_update_func = ip;
250		- /* Make sure the breakpoints see the ftrace_update_func update */
251		- smp_wmb();
252		-
253		- /* See comment above by declaration of modifying_ftrace_code */
254		- atomic_inc(&modifying_ftrace_code);
255		-
256		- ret = ftrace_modify_code(ip, old, new);
257		-
258		- atomic_dec(&modifying_ftrace_code);
259		-
260		- return ret;
261	177	}
262	178
263	179	int ftrace_update_ftrace_func(ftrace_func_t func)
264	180	{
265		- unsigned long ip = (unsigned long)(&ftrace_call);
266		- unsigned char *new;
267		- int ret;
268		-
269		- ftrace_update_func_call = (unsigned long)func;
270		-
271		- new = ftrace_call_replace(ip, (unsigned long)func);
272		- ret = update_ftrace_func(ip, new);
273		-
274		- /* Also update the regs callback function */
275		- if (!ret) {
276		- ip = (unsigned long)(&ftrace_regs_call);
277		- new = ftrace_call_replace(ip, (unsigned long)func);
278		- ret = update_ftrace_func(ip, new);
279		- }
280		-
281		- return ret;
282		-}
283		-
284		-static int is_ftrace_caller(unsigned long ip)
285		-{
286		- if (ip == ftrace_update_func)
287		- return 1;
288		-
289		- return 0;
290		-}
291		-
292		-/*
293		- * A breakpoint was added to the code address we are about to
294		- * modify, and this is the handle that will just skip over it.
295		- * We are either changing a nop into a trace call, or a trace
296		- * call to a nop. While the change is taking place, we treat
297		- * it just like it was a nop.
298		- */
299		-int ftrace_int3_handler(struct pt_regs *regs)
300		-{
301	181	unsigned long ip;
	182	+ const char *new;
302	183
303		- if (WARN_ON_ONCE(!regs))
304		- return 0;
	184	+ ip = (unsigned long)(&ftrace_call);
	185	+ new = ftrace_call_replace(ip, (unsigned long)func);
	186	+ text_poke_bp((void *)ip, new, MCOUNT_INSN_SIZE, NULL);
305	187
306		- ip = regs->ip - INT3_INSN_SIZE;
307		-
308		-#ifdef CONFIG_X86_64
309		- if (ftrace_location(ip)) {
310		- int3_emulate_call(regs, (unsigned long)ftrace_regs_caller);
311		- return 1;
312		- } else if (is_ftrace_caller(ip)) {
313		- if (!ftrace_update_func_call) {
314		- int3_emulate_jmp(regs, ip + CALL_INSN_SIZE);
315		- return 1;
316		- }
317		- int3_emulate_call(regs, ftrace_update_func_call);
318		- return 1;
319		- }
320		-#else
321		- if (ftrace_location(ip) \|\| is_ftrace_caller(ip)) {
322		- int3_emulate_jmp(regs, ip + CALL_INSN_SIZE);
323		- return 1;
324		- }
325		-#endif
	188	+ ip = (unsigned long)(&ftrace_regs_call);
	189	+ new = ftrace_call_replace(ip, (unsigned long)func);
	190	+ text_poke_bp((void *)ip, new, MCOUNT_INSN_SIZE, NULL);
326	191
327	192	return 0;
328		-}
329		-
330		-static int ftrace_write(unsigned long ip, const char *val, int size)
331		-{
332		- ip = text_ip_addr(ip);
333		-
334		- if (probe_kernel_write((void *)ip, val, size))
335		- return -EPERM;
336		-
337		- return 0;
338		-}
339		-
340		-static int add_break(unsigned long ip, const char *old)
341		-{
342		- unsigned char replaced[MCOUNT_INSN_SIZE];
343		- unsigned char brk = BREAKPOINT_INSTRUCTION;
344		-
345		- if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
346		- return -EFAULT;
347		-
348		- ftrace_expected = old;
349		-
350		- /* Make sure it is what we expect it to be */
351		- if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
352		- return -EINVAL;
353		-
354		- return ftrace_write(ip, &brk, 1);
355		-}
356		-
357		-static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
358		-{
359		- unsigned const char *old;
360		- unsigned long ip = rec->ip;
361		-
362		- old = ftrace_call_replace(ip, addr);
363		-
364		- return add_break(rec->ip, old);
365		-}
366		-
367		-
368		-static int add_brk_on_nop(struct dyn_ftrace *rec)
369		-{
370		- unsigned const char *old;
371		-
372		- old = ftrace_nop_replace();
373		-
374		- return add_break(rec->ip, old);
375		-}
376		-
377		-static int add_breakpoints(struct dyn_ftrace *rec, int enable)
378		-{
379		- unsigned long ftrace_addr;
380		- int ret;
381		-
382		- ftrace_addr = ftrace_get_addr_curr(rec);
383		-
384		- ret = ftrace_test_record(rec, enable);
385		-
386		- switch (ret) {
387		- case FTRACE_UPDATE_IGNORE:
388		- return 0;
389		-
390		- case FTRACE_UPDATE_MAKE_CALL:
391		- /* converting nop to call */
392		- return add_brk_on_nop(rec);
393		-
394		- case FTRACE_UPDATE_MODIFY_CALL:
395		- case FTRACE_UPDATE_MAKE_NOP:
396		- /* converting a call to a nop */
397		- return add_brk_on_call(rec, ftrace_addr);
398		- }
399		- return 0;
400		-}
401		-
402		-/*
403		- * On error, we need to remove breakpoints. This needs to
404		- * be done caefully. If the address does not currently have a
405		- * breakpoint, we know we are done. Otherwise, we look at the
406		- * remaining 4 bytes of the instruction. If it matches a nop
407		- * we replace the breakpoint with the nop. Otherwise we replace
408		- * it with the call instruction.
409		- */
410		-static int remove_breakpoint(struct dyn_ftrace *rec)
411		-{
412		- unsigned char ins[MCOUNT_INSN_SIZE];
413		- unsigned char brk = BREAKPOINT_INSTRUCTION;
414		- const unsigned char *nop;
415		- unsigned long ftrace_addr;
416		- unsigned long ip = rec->ip;
417		-
418		- /* If we fail the read, just give up */
419		- if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
420		- return -EFAULT;
421		-
422		- /* If this does not have a breakpoint, we are done */
423		- if (ins[0] != brk)
424		- return 0;
425		-
426		- nop = ftrace_nop_replace();
427		-
428		- /*
429		- * If the last 4 bytes of the instruction do not match
430		- * a nop, then we assume that this is a call to ftrace_addr.
431		- */
432		- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
433		- /*
434		- * For extra paranoidism, we check if the breakpoint is on
435		- * a call that would actually jump to the ftrace_addr.
436		- * If not, don't touch the breakpoint, we make just create
437		- * a disaster.
438		- */
439		- ftrace_addr = ftrace_get_addr_new(rec);
440		- nop = ftrace_call_replace(ip, ftrace_addr);
441		-
442		- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
443		- goto update;
444		-
445		- /* Check both ftrace_addr and ftrace_old_addr */
446		- ftrace_addr = ftrace_get_addr_curr(rec);
447		- nop = ftrace_call_replace(ip, ftrace_addr);
448		-
449		- ftrace_expected = nop;
450		-
451		- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
452		- return -EINVAL;
453		- }
454		-
455		- update:
456		- return ftrace_write(ip, nop, 1);
457		-}
458		-
459		-static int add_update_code(unsigned long ip, unsigned const char *new)
460		-{
461		- /* skip breakpoint */
462		- ip++;
463		- new++;
464		- return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);
465		-}
466		-
467		-static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
468		-{
469		- unsigned long ip = rec->ip;
470		- unsigned const char *new;
471		-
472		- new = ftrace_call_replace(ip, addr);
473		- return add_update_code(ip, new);
474		-}
475		-
476		-static int add_update_nop(struct dyn_ftrace *rec)
477		-{
478		- unsigned long ip = rec->ip;
479		- unsigned const char *new;
480		-
481		- new = ftrace_nop_replace();
482		- return add_update_code(ip, new);
483		-}
484		-
485		-static int add_update(struct dyn_ftrace *rec, int enable)
486		-{
487		- unsigned long ftrace_addr;
488		- int ret;
489		-
490		- ret = ftrace_test_record(rec, enable);
491		-
492		- ftrace_addr = ftrace_get_addr_new(rec);
493		-
494		- switch (ret) {
495		- case FTRACE_UPDATE_IGNORE:
496		- return 0;
497		-
498		- case FTRACE_UPDATE_MODIFY_CALL:
499		- case FTRACE_UPDATE_MAKE_CALL:
500		- /* converting nop to call */
501		- return add_update_call(rec, ftrace_addr);
502		-
503		- case FTRACE_UPDATE_MAKE_NOP:
504		- /* converting a call to a nop */
505		- return add_update_nop(rec);
506		- }
507		-
508		- return 0;
509		-}
510		-
511		-static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
512		-{
513		- unsigned long ip = rec->ip;
514		- unsigned const char *new;
515		-
516		- new = ftrace_call_replace(ip, addr);
517		-
518		- return ftrace_write(ip, new, 1);
519		-}
520		-
521		-static int finish_update_nop(struct dyn_ftrace *rec)
522		-{
523		- unsigned long ip = rec->ip;
524		- unsigned const char *new;
525		-
526		- new = ftrace_nop_replace();
527		-
528		- return ftrace_write(ip, new, 1);
529		-}
530		-
531		-static int finish_update(struct dyn_ftrace *rec, int enable)
532		-{
533		- unsigned long ftrace_addr;
534		- int ret;
535		-
536		- ret = ftrace_update_record(rec, enable);
537		-
538		- ftrace_addr = ftrace_get_addr_new(rec);
539		-
540		- switch (ret) {
541		- case FTRACE_UPDATE_IGNORE:
542		- return 0;
543		-
544		- case FTRACE_UPDATE_MODIFY_CALL:
545		- case FTRACE_UPDATE_MAKE_CALL:
546		- /* converting nop to call */
547		- return finish_update_call(rec, ftrace_addr);
548		-
549		- case FTRACE_UPDATE_MAKE_NOP:
550		- /* converting a call to a nop */
551		- return finish_update_nop(rec);
552		- }
553		-
554		- return 0;
555		-}
556		-
557		-static void do_sync_core(void *data)
558		-{
559		- sync_core();
560		-}
561		-
562		-static void run_sync(void)
563		-{
564		- int enable_irqs;
565		-
566		- /* No need to sync if there's only one CPU */
567		- if (num_online_cpus() == 1)
568		- return;
569		-
570		- enable_irqs = irqs_disabled();
571		-
572		- /* We may be called with interrupts disabled (on bootup). */
573		- if (enable_irqs)
574		- local_irq_enable();
575		- on_each_cpu(do_sync_core, NULL, 1);
576		- if (enable_irqs)
577		- local_irq_disable();
578	193	}
579	194
580	195	void ftrace_replace_code(int enable)
581	196	{
582	197	struct ftrace_rec_iter *iter;
583	198	struct dyn_ftrace *rec;
584		- const char *report = "adding breakpoints";
585		- int count = 0;
	199	+ const char new, old;
586	200	int ret;
587	201
588	202	for_ftrace_rec_iter(iter) {
589	203	rec = ftrace_rec_iter_record(iter);
590	204
591		- ret = add_breakpoints(rec, enable);
592		- if (ret)
593		- goto remove_breakpoints;
594		- count++;
	205	+ switch (ftrace_test_record(rec, enable)) {
	206	+ case FTRACE_UPDATE_IGNORE:
	207	+ default:
	208	+ continue;
	209	+
	210	+ case FTRACE_UPDATE_MAKE_CALL:
	211	+ old = ftrace_nop_replace();
	212	+ break;
	213	+
	214	+ case FTRACE_UPDATE_MODIFY_CALL:
	215	+ case FTRACE_UPDATE_MAKE_NOP:
	216	+ old = ftrace_call_replace(rec->ip, ftrace_get_addr_curr(rec));
	217	+ break;
	218	+ }
	219	+
	220	+ ret = ftrace_verify_code(rec->ip, old);
	221	+ if (ret) {
	222	+ ftrace_bug(ret, rec);
	223	+ return;
	224	+ }
595	225	}
596		-
597		- run_sync();
598		-
599		- report = "updating code";
600		- count = 0;
601	226
602	227	for_ftrace_rec_iter(iter) {
603	228	rec = ftrace_rec_iter_record(iter);
604	229
605		- ret = add_update(rec, enable);
606		- if (ret)
607		- goto remove_breakpoints;
608		- count++;
	230	+ switch (ftrace_test_record(rec, enable)) {
	231	+ case FTRACE_UPDATE_IGNORE:
	232	+ default:
	233	+ continue;
	234	+
	235	+ case FTRACE_UPDATE_MAKE_CALL:
	236	+ case FTRACE_UPDATE_MODIFY_CALL:
	237	+ new = ftrace_call_replace(rec->ip, ftrace_get_addr_new(rec));
	238	+ break;
	239	+
	240	+ case FTRACE_UPDATE_MAKE_NOP:
	241	+ new = ftrace_nop_replace();
	242	+ break;
	243	+ }
	244	+
	245	+ text_poke_queue((void *)rec->ip, new, MCOUNT_INSN_SIZE, NULL);
	246	+ ftrace_update_record(rec, enable);
609	247	}
610		-
611		- run_sync();
612		-
613		- report = "removing breakpoints";
614		- count = 0;
615		-
616		- for_ftrace_rec_iter(iter) {
617		- rec = ftrace_rec_iter_record(iter);
618		-
619		- ret = finish_update(rec, enable);
620		- if (ret)
621		- goto remove_breakpoints;
622		- count++;
623		- }
624		-
625		- run_sync();
626		-
627		- return;
628		-
629		- remove_breakpoints:
630		- pr_warn("Failed on %s (%d):\n", report, count);
631		- ftrace_bug(ret, rec);
632		- for_ftrace_rec_iter(iter) {
633		- rec = ftrace_rec_iter_record(iter);
634		- /*
635		- * Breakpoints are handled only when this function is in
636		- * progress. The system could not work with them.
637		- */
638		- if (remove_breakpoint(rec))
639		- BUG();
640		- }
641		- run_sync();
642		-}
643		-
644		-static int
645		-ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
646		- unsigned const char *new_code)
647		-{
648		- int ret;
649		-
650		- ret = add_break(ip, old_code);
651		- if (ret)
652		- goto out;
653		-
654		- run_sync();
655		-
656		- ret = add_update_code(ip, new_code);
657		- if (ret)
658		- goto fail_update;
659		-
660		- run_sync();
661		-
662		- ret = ftrace_write(ip, new_code, 1);
663		- /*
664		- * The breakpoint is handled only when this function is in progress.
665		- * The system could not work if we could not remove it.
666		- */
667		- BUG_ON(ret);
668		- out:
669		- run_sync();
670		- return ret;
671		-
672		- fail_update:
673		- /* Also here the system could not work with the breakpoint */
674		- if (ftrace_write(ip, old_code, 1))
675		- BUG();
676		- goto out;
	248	+ text_poke_finish();
677	249	}
678	250
679	251	void arch_ftrace_update_code(int command)
680	252	{
681		- /* See comment above by declaration of modifying_ftrace_code */
682		- atomic_inc(&modifying_ftrace_code);
683		-
684	253	ftrace_modify_all_code(command);
685		-
686		- atomic_dec(&modifying_ftrace_code);
687	254	}
688	255
689	256	int __init ftrace_dyn_arch_init(void)
..	..	@@ -701,12 +268,8 @@
701	268	{
702	269	return module_alloc(size);
703	270	}
704		-static inline void tramp_free(void *tramp, int size)
	271	+static inline void tramp_free(void *tramp)
705	272	{
706		- int npages = PAGE_ALIGN(size) >> PAGE_SHIFT;
707		-
708		- set_memory_nx((unsigned long)tramp, npages);
709		- set_memory_rw((unsigned long)tramp, npages);
710	273	module_memfree(tramp);
711	274	}
712	275	#else
..	..	@@ -715,14 +278,16 @@
715	278	{
716	279	return NULL;
717	280	}
718		-static inline void tramp_free(void *tramp, int size) { }
	281	+static inline void tramp_free(void *tramp) { }
719	282	#endif
720	283
721	284	/* Defined as markers to the end of the ftrace default trampolines */
722	285	extern void ftrace_regs_caller_end(void);
723		-extern void ftrace_epilogue(void);
	286	+extern void ftrace_regs_caller_ret(void);
	287	+extern void ftrace_caller_end(void);
724	288	extern void ftrace_caller_op_ptr(void);
725	289	extern void ftrace_regs_caller_op_ptr(void);
	290	+extern void ftrace_regs_caller_jmp(void);
726	291
727	292	/* movq function_trace_op(%rip), %rdx */
728	293	/* 0x48 0x8b 0x15 <offset-to-ftrace_trace_op (4 bytes)> */
..	..	@@ -744,7 +309,7 @@
744	309	} __attribute__((packed));
745	310	};
746	311
747		-#define RET_SIZE 1
	312	+#define RET_SIZE (IS_ENABLED(CONFIG_RETPOLINE) ? 5 : 1 + IS_ENABLED(CONFIG_SLS))
748	313
749	314	static unsigned long
750	315	create_trampoline(struct ftrace_ops ops, unsigned int tramp_size)
..	..	@@ -752,15 +317,17 @@
752	317	unsigned long start_offset;
753	318	unsigned long end_offset;
754	319	unsigned long op_offset;
	320	+ unsigned long call_offset;
	321	+ unsigned long jmp_offset;
755	322	unsigned long offset;
756	323	unsigned long npages;
757	324	unsigned long size;
758		- unsigned long retq;
759	325	unsigned long *ptr;
760	326	void *trampoline;
761	327	void *ip;
762	328	/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */
763	329	unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };
	330	+ unsigned const char retq[] = { RET_INSN_OPCODE, INT3_INSN_OPCODE };
764	331	union ftrace_op_code_union op_ptr;
765	332	int ret;
766	333
..	..	@@ -768,10 +335,14 @@
768	335	start_offset = (unsigned long)ftrace_regs_caller;
769	336	end_offset = (unsigned long)ftrace_regs_caller_end;
770	337	op_offset = (unsigned long)ftrace_regs_caller_op_ptr;
	338	+ call_offset = (unsigned long)ftrace_regs_call;
	339	+ jmp_offset = (unsigned long)ftrace_regs_caller_jmp;
771	340	} else {
772	341	start_offset = (unsigned long)ftrace_caller;
773		- end_offset = (unsigned long)ftrace_epilogue;
	342	+ end_offset = (unsigned long)ftrace_caller_end;
774	343	op_offset = (unsigned long)ftrace_caller_op_ptr;
	344	+ call_offset = (unsigned long)ftrace_call;
	345	+ jmp_offset = 0;
775	346	}
776	347
777	348	size = end_offset - start_offset;
..	..	@@ -789,17 +360,28 @@
789	360	npages = DIV_ROUND_UP(*tramp_size, PAGE_SIZE);
790	361
791	362	/* Copy ftrace_caller onto the trampoline memory */
792		- ret = probe_kernel_read(trampoline, (void *)start_offset, size);
	363	+ ret = copy_from_kernel_nofault(trampoline, (void *)start_offset, size);
793	364	if (WARN_ON(ret < 0))
794	365	goto fail;
795	366
796	367	ip = trampoline + size;
797	368
798	369	/* The trampoline ends with ret(q) */
799		- retq = (unsigned long)ftrace_stub;
800		- ret = probe_kernel_read(ip, (void *)retq, RET_SIZE);
801		- if (WARN_ON(ret < 0))
802		- goto fail;
	370	+ if (cpu_feature_enabled(X86_FEATURE_RETHUNK))
	371	+ memcpy(ip, text_gen_insn(JMP32_INSN_OPCODE, ip, &__x86_return_thunk), JMP32_INSN_SIZE);
	372	+ else
	373	+ memcpy(ip, retq, sizeof(retq));
	374	+
	375	+ /* No need to test direct calls on created trampolines */
	376	+ if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
	377	+ /* NOP the jnz 1f; but make sure it's a 2 byte jnz */
	378	+ ip = trampoline + (jmp_offset - start_offset);
	379	+ if (WARN_ON((char )ip != 0x75))
	380	+ goto fail;
	381	+ ret = copy_from_kernel_nofault(ip, ideal_nops[2], 2);
	382	+ if (ret < 0)
	383	+ goto fail;
	384	+ }
803	385
804	386	/*
805	387	* The address of the ftrace_ops that is used for this trampoline
..	..	@@ -828,19 +410,53 @@
828	410	/* put in the new offset to the ftrace_ops */
829	411	memcpy(trampoline + op_offset, &op_ptr, OP_REF_SIZE);
830	412
	413	+ /* put in the call to the function */
	414	+ mutex_lock(&text_mutex);
	415	+ call_offset -= start_offset;
	416	+ memcpy(trampoline + call_offset,
	417	+ text_gen_insn(CALL_INSN_OPCODE,
	418	+ trampoline + call_offset,
	419	+ ftrace_ops_get_func(ops)), CALL_INSN_SIZE);
	420	+ mutex_unlock(&text_mutex);
	421	+
831	422	/* ALLOC_TRAMP flags lets us know we created it */
832	423	ops->flags \|= FTRACE_OPS_FL_ALLOC_TRAMP;
833	424
834		- /*
835		- * Module allocation needs to be completed by making the page
836		- * executable. The page is still writable, which is a security hazard,
837		- * but anyhow ftrace breaks W^X completely.
838		- */
	425	+ set_vm_flush_reset_perms(trampoline);
	426	+
	427	+ if (likely(system_state != SYSTEM_BOOTING))
	428	+ set_memory_ro((unsigned long)trampoline, npages);
839	429	set_memory_x((unsigned long)trampoline, npages);
840	430	return (unsigned long)trampoline;
841	431	fail:
842		- tramp_free(trampoline, *tramp_size);
	432	+ tramp_free(trampoline);
843	433	return 0;
	434	+}
	435	+
	436	+void set_ftrace_ops_ro(void)
	437	+{
	438	+ struct ftrace_ops *ops;
	439	+ unsigned long start_offset;
	440	+ unsigned long end_offset;
	441	+ unsigned long npages;
	442	+ unsigned long size;
	443	+
	444	+ do_for_each_ftrace_op(ops, ftrace_ops_list) {
	445	+ if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
	446	+ continue;
	447	+
	448	+ if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
	449	+ start_offset = (unsigned long)ftrace_regs_caller;
	450	+ end_offset = (unsigned long)ftrace_regs_caller_end;
	451	+ } else {
	452	+ start_offset = (unsigned long)ftrace_caller;
	453	+ end_offset = (unsigned long)ftrace_caller_end;
	454	+ }
	455	+ size = end_offset - start_offset;
	456	+ size = size + RET_SIZE + sizeof(void *);
	457	+ npages = DIV_ROUND_UP(size, PAGE_SIZE);
	458	+ set_memory_ro((unsigned long)ops->trampoline, npages);
	459	+ } while_for_each_ftrace_op(ops);
844	460	}
845	461
846	462	static unsigned long calc_trampoline_call_offset(bool save_regs)
..	..	@@ -862,62 +478,54 @@
862	478	void arch_ftrace_update_trampoline(struct ftrace_ops *ops)
863	479	{
864	480	ftrace_func_t func;
865		- unsigned char *new;
866	481	unsigned long offset;
867	482	unsigned long ip;
868	483	unsigned int size;
869		- int ret, npages;
	484	+ const char *new;
870	485
871		- if (ops->trampoline) {
872		- /*
873		- * The ftrace_ops caller may set up its own trampoline.
874		- * In such a case, this code must not modify it.
875		- */
876		- if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
877		- return;
878		- npages = PAGE_ALIGN(ops->trampoline_size) >> PAGE_SHIFT;
879		- set_memory_rw(ops->trampoline, npages);
880		- } else {
	486	+ if (!ops->trampoline) {
881	487	ops->trampoline = create_trampoline(ops, &size);
882	488	if (!ops->trampoline)
883	489	return;
884	490	ops->trampoline_size = size;
885		- npages = PAGE_ALIGN(size) >> PAGE_SHIFT;
	491	+ return;
886	492	}
	493	+
	494	+ /*
	495	+ * The ftrace_ops caller may set up its own trampoline.
	496	+ * In such a case, this code must not modify it.
	497	+ */
	498	+ if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
	499	+ return;
887	500
888	501	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);
889	502	ip = ops->trampoline + offset;
890		-
891	503	func = ftrace_ops_get_func(ops);
892	504
893		- ftrace_update_func_call = (unsigned long)func;
894		-
	505	+ mutex_lock(&text_mutex);
895	506	/* Do a safe modify in case the trampoline is executing */
896	507	new = ftrace_call_replace(ip, (unsigned long)func);
897		- ret = update_ftrace_func(ip, new);
898		- set_memory_ro(ops->trampoline, npages);
899		-
900		- /* The update should never fail */
901		- WARN_ON(ret);
	508	+ text_poke_bp((void *)ip, new, MCOUNT_INSN_SIZE, NULL);
	509	+ mutex_unlock(&text_mutex);
902	510	}
903	511
904	512	/* Return the address of the function the trampoline calls */
905	513	static void addr_from_call(void ptr)
906	514	{
907		- union ftrace_code_union calc;
	515	+ union text_poke_insn call;
908	516	int ret;
909	517
910		- ret = probe_kernel_read(&calc, ptr, MCOUNT_INSN_SIZE);
	518	+ ret = copy_from_kernel_nofault(&call, ptr, CALL_INSN_SIZE);
911	519	if (WARN_ON_ONCE(ret < 0))
912	520	return NULL;
913	521
914	522	/* Make sure this is a call */
915		- if (WARN_ON_ONCE(calc.op != 0xe8)) {
916		- pr_warn("Expected e8, got %x\n", calc.op);
	523	+ if (WARN_ON_ONCE(call.opcode != CALL_INSN_OPCODE)) {
	524	+ pr_warn("Expected E8, got %x\n", call.opcode);
917	525	return NULL;
918	526	}
919	527
920		- return ptr + MCOUNT_INSN_SIZE + calc.offset;
	528	+ return ptr + CALL_INSN_SIZE + call.disp;
921	529	}
922	530
923	531	void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
..	..	@@ -972,7 +580,7 @@
972	580	if (!ops \|\| !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
973	581	return;
974	582
975		- tramp_free((void *)ops->trampoline, ops->trampoline_size);
	583	+ tramp_free((void *)ops->trampoline);
976	584	ops->trampoline = 0;
977	585	}
978	586
..	..	@@ -984,19 +592,18 @@
984	592	#ifdef CONFIG_DYNAMIC_FTRACE
985	593	extern void ftrace_graph_call(void);
986	594
987		-static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
	595	+static const char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
988	596	{
989		- return ftrace_text_replace(0xe9, ip, addr);
	597	+ return text_gen_insn(JMP32_INSN_OPCODE, (void )ip, (void )addr);
990	598	}
991	599
992	600	static int ftrace_mod_jmp(unsigned long ip, void *func)
993	601	{
994		- unsigned char *new;
	602	+ const char *new;
995	603
996		- ftrace_update_func_call = 0UL;
997	604	new = ftrace_jmp_replace(ip, (unsigned long)func);
998		-
999		- return update_ftrace_func(ip, new);
	605	+ text_poke_bp((void *)ip, new, MCOUNT_INSN_SIZE, NULL);
	606	+ return 0;
1000	607	}
1001	608
1002	609	int ftrace_enable_ftrace_graph_caller(void)
..	..	@@ -1022,10 +629,9 @@
1022	629	void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
1023	630	unsigned long frame_pointer)
1024	631	{
	632	+ unsigned long return_hooker = (unsigned long)&return_to_handler;
1025	633	unsigned long old;
1026	634	int faulted;
1027		- unsigned long return_hooker = (unsigned long)
1028		- &return_to_handler;
1029	635
1030	636	/*
1031	637	* When resuming from suspend-to-ram, this function can be indirectly