forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-11 297b60346df8beafee954a0fd7c2d64f33f3b9bc 
 kernel/arch/x86/kernel/ftrace.c
....@@ -23,6 +23,7 @@
2323 #include <linux/list.h>
2424 #include <linux/module.h>
2525 #include <linux/memory.h>
26
+#include <linux/vmalloc.h>
2627 
2728 #include <trace/syscall.h>
2829 
....@@ -34,88 +35,48 @@
3435 
3536 #ifdef CONFIG_DYNAMIC_FTRACE
3637 
38
+static int ftrace_poke_late = 0;
39
+
3740 int ftrace_arch_code_modify_prepare(void)
3841     __acquires(&text_mutex)
3942 {
43
+	/*
44
+	 * Need to grab text_mutex to prevent a race from module loading
45
+	 * and live kernel patching from changing the text permissions while
46
+	 * ftrace has it set to "read/write".
47
+	 */
4048 	mutex_lock(&text_mutex);
41
-	set_kernel_text_rw();
42
-	set_all_modules_text_rw();
49
+	ftrace_poke_late = 1;
4350 	return 0;
4451 }
4552 
4653 int ftrace_arch_code_modify_post_process(void)
4754     __releases(&text_mutex)
4855 {
49
-	set_all_modules_text_ro();
50
-	set_kernel_text_ro();
56
+	/*
57
+	 * ftrace_make_{call,nop}() may be called during
58
+	 * module load, and we need to finish the text_poke_queue()
59
+	 * that they do, here.
60
+	 */
61
+	text_poke_finish();
62
+	ftrace_poke_late = 0;
5163 	mutex_unlock(&text_mutex);
5264 	return 0;
5365 }
5466 
55
-union ftrace_code_union {
56
-	char code[MCOUNT_INSN_SIZE];
57
-	struct {
58
-		unsigned char op;
59
-		int offset;
60
-	} __attribute__((packed));
61
-};
62
-
63
-static int ftrace_calc_offset(long ip, long addr)
64
-{
65
-	return (int)(addr - ip);
66
-}
67
-
68
-static unsigned char *
69
-ftrace_text_replace(unsigned char op, unsigned long ip, unsigned long addr)
70
-{
71
-	static union ftrace_code_union calc;
72
-
73
-	calc.op		= op;
74
-	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
75
-
76
-	return calc.code;
77
-}
78
-
79
-static unsigned char *
80
-ftrace_call_replace(unsigned long ip, unsigned long addr)
81
-{
82
-	return ftrace_text_replace(0xe8, ip, addr);
83
-}
84
-
85
-static inline int
86
-within(unsigned long addr, unsigned long start, unsigned long end)
87
-{
88
-	return addr >= start && addr < end;
89
-}
90
-
91
-static unsigned long text_ip_addr(unsigned long ip)
92
-{
93
-	/*
94
-	 * On x86_64, kernel text mappings are mapped read-only, so we use
95
-	 * the kernel identity mapping instead of the kernel text mapping
96
-	 * to modify the kernel text.
97
-	 *
98
-	 * For 32bit kernels, these mappings are same and we can use
99
-	 * kernel identity mapping to modify code.
100
-	 */
101
-	if (within(ip, (unsigned long)_text, (unsigned long)_etext))
102
-		ip = (unsigned long)__va(__pa_symbol(ip));
103
-
104
-	return ip;
105
-}
106
-
107
-static const unsigned char *ftrace_nop_replace(void)
67
+static const char *ftrace_nop_replace(void)
10868 {
10969 	return ideal_nops[NOP_ATOMIC5];
11070 }
11171 
112
-static int
113
-ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
114
-		   unsigned const char *new_code)
72
+static const char *ftrace_call_replace(unsigned long ip, unsigned long addr)
11573 {
116
-	unsigned char replaced[MCOUNT_INSN_SIZE];
74
+	return text_gen_insn(CALL_INSN_OPCODE, (void *)ip, (void *)addr);
75
+}
11776 
118
-	ftrace_expected = old_code;
77
+static int ftrace_verify_code(unsigned long ip, const char *old_code)
78
+{
79
+	char cur_code[MCOUNT_INSN_SIZE];
11980 
12081 	/*
12182 	 * Note:
....@@ -124,31 +85,47 @@
12485 	 * Carefully read and modify the code with probe_kernel_*(), and make
12586 	 * sure what we read is what we expected it to be before modifying it.
12687 	 */
127
-
12888 	/* read the text we want to modify */
129
-	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
89
+	if (copy_from_kernel_nofault(cur_code, (void *)ip, MCOUNT_INSN_SIZE)) {
90
+		WARN_ON(1);
13091 		return -EFAULT;
92
+	}
13193 
13294 	/* Make sure it is what we expect it to be */
133
-	if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
95
+	if (memcmp(cur_code, old_code, MCOUNT_INSN_SIZE) != 0) {
96
+		ftrace_expected = old_code;
97
+		WARN_ON(1);
13498 		return -EINVAL;
135
-
136
-	ip = text_ip_addr(ip);
137
-
138
-	/* replace the text with the new text */
139
-	if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
140
-		return -EPERM;
141
-
142
-	sync_core();
99
+	}
143100 
144101 	return 0;
145102 }
146103 
147
-int ftrace_make_nop(struct module *mod,
148
-		    struct dyn_ftrace *rec, unsigned long addr)
104
+/*
105
+ * Marked __ref because it calls text_poke_early() which is .init.text. That is
106
+ * ok because that call will happen early, during boot, when .init sections are
107
+ * still present.
108
+ */
109
+static int __ref
110
+ftrace_modify_code_direct(unsigned long ip, const char *old_code,
111
+			  const char *new_code)
149112 {
150
-	unsigned const char *new, *old;
113
+	int ret = ftrace_verify_code(ip, old_code);
114
+	if (ret)
115
+		return ret;
116
+
117
+	/* replace the text with the new text */
118
+	if (ftrace_poke_late)
119
+		text_poke_queue((void *)ip, new_code, MCOUNT_INSN_SIZE, NULL);
120
+	else
121
+		text_poke_early((void *)ip, new_code, MCOUNT_INSN_SIZE);
122
+	return 0;
123
+}
124
+
125
+int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr)
126
+{
151127 	unsigned long ip = rec->ip;
128
+	const char *new, *old;
152129 
153130 	old = ftrace_call_replace(ip, addr);
154131 	new = ftrace_nop_replace();
....@@ -162,19 +139,20 @@
162139 	 * just modify the code directly.
163140 	 */
164141 	if (addr == MCOUNT_ADDR)
165
-		return ftrace_modify_code_direct(rec->ip, old, new);
142
+		return ftrace_modify_code_direct(ip, old, new);
166143 
167
-	ftrace_expected = NULL;
168
-
169
-	/* Normal cases use add_brk_on_nop */
144
+	/*
145
+	 * x86 overrides ftrace_replace_code -- this function will never be used
146
+	 * in this case.
147
+	 */
170148 	WARN_ONCE(1, "invalid use of ftrace_make_nop");
171149 	return -EINVAL;
172150 }
173151 
174152 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
175153 {
176
-	unsigned const char *new, *old;
177154 	unsigned long ip = rec->ip;
155
+	const char *new, *old;
178156 
179157 	old = ftrace_nop_replace();
180158 	new = ftrace_call_replace(ip, addr);
....@@ -182,43 +160,6 @@
182160 	/* Should only be called when module is loaded */
183161 	return ftrace_modify_code_direct(rec->ip, old, new);
184162 }
185
-
186
-/*
187
- * The modifying_ftrace_code is used to tell the breakpoint
188
- * handler to call ftrace_int3_handler(). If it fails to
189
- * call this handler for a breakpoint added by ftrace, then
190
- * the kernel may crash.
191
- *
192
- * As atomic_writes on x86 do not need a barrier, we do not
193
- * need to add smp_mb()s for this to work. It is also considered
194
- * that we can not read the modifying_ftrace_code before
195
- * executing the breakpoint. That would be quite remarkable if
196
- * it could do that. Here's the flow that is required:
197
- *
198
- *   CPU-0                          CPU-1
199
- *
200
- * atomic_inc(mfc);
201
- * write int3s
202
- *				<trap-int3> // implicit (r)mb
203
- *				if (atomic_read(mfc))
204
- *					call ftrace_int3_handler()
205
- *
206
- * Then when we are finished:
207
- *
208
- * atomic_dec(mfc);
209
- *
210
- * If we hit a breakpoint that was not set by ftrace, it does not
211
- * matter if ftrace_int3_handler() is called or not. It will
212
- * simply be ignored. But it is crucial that a ftrace nop/caller
213
- * breakpoint is handled. No other user should ever place a
214
- * breakpoint on an ftrace nop/caller location. It must only
215
- * be done by this code.
216
- */
217
-atomic_t modifying_ftrace_code __read_mostly;
218
-
219
-static int
220
-ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
221
-		   unsigned const char *new_code);
222163 
223164 /*
224165  * Should never be called:
....@@ -232,458 +173,86 @@
232173 				 unsigned long addr)
233174 {
234175 	WARN_ON(1);
235
-	ftrace_expected = NULL;
236176 	return -EINVAL;
237
-}
238
-
239
-static unsigned long ftrace_update_func;
240
-static unsigned long ftrace_update_func_call;
241
-
242
-static int update_ftrace_func(unsigned long ip, void *new)
243
-{
244
-	unsigned char old[MCOUNT_INSN_SIZE];
245
-	int ret;
246
-
247
-	memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);
248
-
249
-	ftrace_update_func = ip;
250
-	/* Make sure the breakpoints see the ftrace_update_func update */
251
-	smp_wmb();
252
-
253
-	/* See comment above by declaration of modifying_ftrace_code */
254
-	atomic_inc(&modifying_ftrace_code);
255
-
256
-	ret = ftrace_modify_code(ip, old, new);
257
-
258
-	atomic_dec(&modifying_ftrace_code);
259
-
260
-	return ret;
261177 }
262178 
263179 int ftrace_update_ftrace_func(ftrace_func_t func)
264180 {
265
-	unsigned long ip = (unsigned long)(&ftrace_call);
266
-	unsigned char *new;
267
-	int ret;
268
-
269
-	ftrace_update_func_call = (unsigned long)func;
270
-
271
-	new = ftrace_call_replace(ip, (unsigned long)func);
272
-	ret = update_ftrace_func(ip, new);
273
-
274
-	/* Also update the regs callback function */
275
-	if (!ret) {
276
-		ip = (unsigned long)(&ftrace_regs_call);
277
-		new = ftrace_call_replace(ip, (unsigned long)func);
278
-		ret = update_ftrace_func(ip, new);
279
-	}
280
-
281
-	return ret;
282
-}
283
-
284
-static int is_ftrace_caller(unsigned long ip)
285
-{
286
-	if (ip == ftrace_update_func)
287
-		return 1;
288
-
289
-	return 0;
290
-}
291
-
292
-/*
293
- * A breakpoint was added to the code address we are about to
294
- * modify, and this is the handle that will just skip over it.
295
- * We are either changing a nop into a trace call, or a trace
296
- * call to a nop. While the change is taking place, we treat
297
- * it just like it was a nop.
298
- */
299
-int ftrace_int3_handler(struct pt_regs *regs)
300
-{
301181 	unsigned long ip;
182
+	const char *new;
302183 
303
-	if (WARN_ON_ONCE(!regs))
304
-		return 0;
184
+	ip = (unsigned long)(&ftrace_call);
185
+	new = ftrace_call_replace(ip, (unsigned long)func);
186
+	text_poke_bp((void *)ip, new, MCOUNT_INSN_SIZE, NULL);
305187 
306
-	ip = regs->ip - INT3_INSN_SIZE;
307
-
308
-#ifdef CONFIG_X86_64
309
-	if (ftrace_location(ip)) {
310
-		int3_emulate_call(regs, (unsigned long)ftrace_regs_caller);
311
-		return 1;
312
-	} else if (is_ftrace_caller(ip)) {
313
-		if (!ftrace_update_func_call) {
314
-			int3_emulate_jmp(regs, ip + CALL_INSN_SIZE);
315
-			return 1;
316
-		}
317
-		int3_emulate_call(regs, ftrace_update_func_call);
318
-		return 1;
319
-	}
320
-#else
321
-	if (ftrace_location(ip) || is_ftrace_caller(ip)) {
322
-		int3_emulate_jmp(regs, ip + CALL_INSN_SIZE);
323
-		return 1;
324
-	}
325
-#endif
188
+	ip = (unsigned long)(&ftrace_regs_call);
189
+	new = ftrace_call_replace(ip, (unsigned long)func);
190
+	text_poke_bp((void *)ip, new, MCOUNT_INSN_SIZE, NULL);
326191 
327192 	return 0;
328
-}
329
-
330
-static int ftrace_write(unsigned long ip, const char *val, int size)
331
-{
332
-	ip = text_ip_addr(ip);
333
-
334
-	if (probe_kernel_write((void *)ip, val, size))
335
-		return -EPERM;
336
-
337
-	return 0;
338
-}
339
-
340
-static int add_break(unsigned long ip, const char *old)
341
-{
342
-	unsigned char replaced[MCOUNT_INSN_SIZE];
343
-	unsigned char brk = BREAKPOINT_INSTRUCTION;
344
-
345
-	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
346
-		return -EFAULT;
347
-
348
-	ftrace_expected = old;
349
-
350
-	/* Make sure it is what we expect it to be */
351
-	if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
352
-		return -EINVAL;
353
-
354
-	return ftrace_write(ip, &brk, 1);
355
-}
356
-
357
-static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
358
-{
359
-	unsigned const char *old;
360
-	unsigned long ip = rec->ip;
361
-
362
-	old = ftrace_call_replace(ip, addr);
363
-
364
-	return add_break(rec->ip, old);
365
-}
366
-
367
-
368
-static int add_brk_on_nop(struct dyn_ftrace *rec)
369
-{
370
-	unsigned const char *old;
371
-
372
-	old = ftrace_nop_replace();
373
-
374
-	return add_break(rec->ip, old);
375
-}
376
-
377
-static int add_breakpoints(struct dyn_ftrace *rec, int enable)
378
-{
379
-	unsigned long ftrace_addr;
380
-	int ret;
381
-
382
-	ftrace_addr = ftrace_get_addr_curr(rec);
383
-
384
-	ret = ftrace_test_record(rec, enable);
385
-
386
-	switch (ret) {
387
-	case FTRACE_UPDATE_IGNORE:
388
-		return 0;
389
-
390
-	case FTRACE_UPDATE_MAKE_CALL:
391
-		/* converting nop to call */
392
-		return add_brk_on_nop(rec);
393
-
394
-	case FTRACE_UPDATE_MODIFY_CALL:
395
-	case FTRACE_UPDATE_MAKE_NOP:
396
-		/* converting a call to a nop */
397
-		return add_brk_on_call(rec, ftrace_addr);
398
-	}
399
-	return 0;
400
-}
401
-
402
-/*
403
- * On error, we need to remove breakpoints. This needs to
404
- * be done caefully. If the address does not currently have a
405
- * breakpoint, we know we are done. Otherwise, we look at the
406
- * remaining 4 bytes of the instruction. If it matches a nop
407
- * we replace the breakpoint with the nop. Otherwise we replace
408
- * it with the call instruction.
409
- */
410
-static int remove_breakpoint(struct dyn_ftrace *rec)
411
-{
412
-	unsigned char ins[MCOUNT_INSN_SIZE];
413
-	unsigned char brk = BREAKPOINT_INSTRUCTION;
414
-	const unsigned char *nop;
415
-	unsigned long ftrace_addr;
416
-	unsigned long ip = rec->ip;
417
-
418
-	/* If we fail the read, just give up */
419
-	if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
420
-		return -EFAULT;
421
-
422
-	/* If this does not have a breakpoint, we are done */
423
-	if (ins[0] != brk)
424
-		return 0;
425
-
426
-	nop = ftrace_nop_replace();
427
-
428
-	/*
429
-	 * If the last 4 bytes of the instruction do not match
430
-	 * a nop, then we assume that this is a call to ftrace_addr.
431
-	 */
432
-	if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
433
-		/*
434
-		 * For extra paranoidism, we check if the breakpoint is on
435
-		 * a call that would actually jump to the ftrace_addr.
436
-		 * If not, don't touch the breakpoint, we make just create
437
-		 * a disaster.
438
-		 */
439
-		ftrace_addr = ftrace_get_addr_new(rec);
440
-		nop = ftrace_call_replace(ip, ftrace_addr);
441
-
442
-		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
443
-			goto update;
444
-
445
-		/* Check both ftrace_addr and ftrace_old_addr */
446
-		ftrace_addr = ftrace_get_addr_curr(rec);
447
-		nop = ftrace_call_replace(ip, ftrace_addr);
448
-
449
-		ftrace_expected = nop;
450
-
451
-		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
452
-			return -EINVAL;
453
-	}
454
-
455
- update:
456
-	return ftrace_write(ip, nop, 1);
457
-}
458
-
459
-static int add_update_code(unsigned long ip, unsigned const char *new)
460
-{
461
-	/* skip breakpoint */
462
-	ip++;
463
-	new++;
464
-	return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);
465
-}
466
-
467
-static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
468
-{
469
-	unsigned long ip = rec->ip;
470
-	unsigned const char *new;
471
-
472
-	new = ftrace_call_replace(ip, addr);
473
-	return add_update_code(ip, new);
474
-}
475
-
476
-static int add_update_nop(struct dyn_ftrace *rec)
477
-{
478
-	unsigned long ip = rec->ip;
479
-	unsigned const char *new;
480
-
481
-	new = ftrace_nop_replace();
482
-	return add_update_code(ip, new);
483
-}
484
-
485
-static int add_update(struct dyn_ftrace *rec, int enable)
486
-{
487
-	unsigned long ftrace_addr;
488
-	int ret;
489
-
490
-	ret = ftrace_test_record(rec, enable);
491
-
492
-	ftrace_addr  = ftrace_get_addr_new(rec);
493
-
494
-	switch (ret) {
495
-	case FTRACE_UPDATE_IGNORE:
496
-		return 0;
497
-
498
-	case FTRACE_UPDATE_MODIFY_CALL:
499
-	case FTRACE_UPDATE_MAKE_CALL:
500
-		/* converting nop to call */
501
-		return add_update_call(rec, ftrace_addr);
502
-
503
-	case FTRACE_UPDATE_MAKE_NOP:
504
-		/* converting a call to a nop */
505
-		return add_update_nop(rec);
506
-	}
507
-
508
-	return 0;
509
-}
510
-
511
-static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
512
-{
513
-	unsigned long ip = rec->ip;
514
-	unsigned const char *new;
515
-
516
-	new = ftrace_call_replace(ip, addr);
517
-
518
-	return ftrace_write(ip, new, 1);
519
-}
520
-
521
-static int finish_update_nop(struct dyn_ftrace *rec)
522
-{
523
-	unsigned long ip = rec->ip;
524
-	unsigned const char *new;
525
-
526
-	new = ftrace_nop_replace();
527
-
528
-	return ftrace_write(ip, new, 1);
529
-}
530
-
531
-static int finish_update(struct dyn_ftrace *rec, int enable)
532
-{
533
-	unsigned long ftrace_addr;
534
-	int ret;
535
-
536
-	ret = ftrace_update_record(rec, enable);
537
-
538
-	ftrace_addr = ftrace_get_addr_new(rec);
539
-
540
-	switch (ret) {
541
-	case FTRACE_UPDATE_IGNORE:
542
-		return 0;
543
-
544
-	case FTRACE_UPDATE_MODIFY_CALL:
545
-	case FTRACE_UPDATE_MAKE_CALL:
546
-		/* converting nop to call */
547
-		return finish_update_call(rec, ftrace_addr);
548
-
549
-	case FTRACE_UPDATE_MAKE_NOP:
550
-		/* converting a call to a nop */
551
-		return finish_update_nop(rec);
552
-	}
553
-
554
-	return 0;
555
-}
556
-
557
-static void do_sync_core(void *data)
558
-{
559
-	sync_core();
560
-}
561
-
562
-static void run_sync(void)
563
-{
564
-	int enable_irqs;
565
-
566
-	/* No need to sync if there's only one CPU */
567
-	if (num_online_cpus() == 1)
568
-		return;
569
-
570
-	enable_irqs = irqs_disabled();
571
-
572
-	/* We may be called with interrupts disabled (on bootup). */
573
-	if (enable_irqs)
574
-		local_irq_enable();
575
-	on_each_cpu(do_sync_core, NULL, 1);
576
-	if (enable_irqs)
577
-		local_irq_disable();
578193 }
579194 
580195 void ftrace_replace_code(int enable)
581196 {
582197 	struct ftrace_rec_iter *iter;
583198 	struct dyn_ftrace *rec;
584
-	const char *report = "adding breakpoints";
585
-	int count = 0;
199
+	const char *new, *old;
586200 	int ret;
587201 
588202 	for_ftrace_rec_iter(iter) {
589203 		rec = ftrace_rec_iter_record(iter);
590204 
591
-		ret = add_breakpoints(rec, enable);
592
-		if (ret)
593
-			goto remove_breakpoints;
594
-		count++;
205
+		switch (ftrace_test_record(rec, enable)) {
206
+		case FTRACE_UPDATE_IGNORE:
207
+		default:
208
+			continue;
209
+
210
+		case FTRACE_UPDATE_MAKE_CALL:
211
+			old = ftrace_nop_replace();
212
+			break;
213
+
214
+		case FTRACE_UPDATE_MODIFY_CALL:
215
+		case FTRACE_UPDATE_MAKE_NOP:
216
+			old = ftrace_call_replace(rec->ip, ftrace_get_addr_curr(rec));
217
+			break;
218
+		}
219
+
220
+		ret = ftrace_verify_code(rec->ip, old);
221
+		if (ret) {
222
+			ftrace_expected = old;
223
+			ftrace_bug(ret, rec);
224
+			ftrace_expected = NULL;
225
+			return;
226
+		}
595227 	}
596
-
597
-	run_sync();
598
-
599
-	report = "updating code";
600
-	count = 0;
601228 
602229 	for_ftrace_rec_iter(iter) {
603230 		rec = ftrace_rec_iter_record(iter);
604231 
605
-		ret = add_update(rec, enable);
606
-		if (ret)
607
-			goto remove_breakpoints;
608
-		count++;
232
+		switch (ftrace_test_record(rec, enable)) {
233
+		case FTRACE_UPDATE_IGNORE:
234
+		default:
235
+			continue;
236
+
237
+		case FTRACE_UPDATE_MAKE_CALL:
238
+		case FTRACE_UPDATE_MODIFY_CALL:
239
+			new = ftrace_call_replace(rec->ip, ftrace_get_addr_new(rec));
240
+			break;
241
+
242
+		case FTRACE_UPDATE_MAKE_NOP:
243
+			new = ftrace_nop_replace();
244
+			break;
245
+		}
246
+
247
+		text_poke_queue((void *)rec->ip, new, MCOUNT_INSN_SIZE, NULL);
248
+		ftrace_update_record(rec, enable);
609249 	}
610
-
611
-	run_sync();
612
-
613
-	report = "removing breakpoints";
614
-	count = 0;
615
-
616
-	for_ftrace_rec_iter(iter) {
617
-		rec = ftrace_rec_iter_record(iter);
618
-
619
-		ret = finish_update(rec, enable);
620
-		if (ret)
621
-			goto remove_breakpoints;
622
-		count++;
623
-	}
624
-
625
-	run_sync();
626
-
627
-	return;
628
-
629
- remove_breakpoints:
630
-	pr_warn("Failed on %s (%d):\n", report, count);
631
-	ftrace_bug(ret, rec);
632
-	for_ftrace_rec_iter(iter) {
633
-		rec = ftrace_rec_iter_record(iter);
634
-		/*
635
-		 * Breakpoints are handled only when this function is in
636
-		 * progress. The system could not work with them.
637
-		 */
638
-		if (remove_breakpoint(rec))
639
-			BUG();
640
-	}
641
-	run_sync();
642
-}
643
-
644
-static int
645
-ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
646
-		   unsigned const char *new_code)
647
-{
648
-	int ret;
649
-
650
-	ret = add_break(ip, old_code);
651
-	if (ret)
652
-		goto out;
653
-
654
-	run_sync();
655
-
656
-	ret = add_update_code(ip, new_code);
657
-	if (ret)
658
-		goto fail_update;
659
-
660
-	run_sync();
661
-
662
-	ret = ftrace_write(ip, new_code, 1);
663
-	/*
664
-	 * The breakpoint is handled only when this function is in progress.
665
-	 * The system could not work if we could not remove it.
666
-	 */
667
-	BUG_ON(ret);
668
- out:
669
-	run_sync();
670
-	return ret;
671
-
672
- fail_update:
673
-	/* Also here the system could not work with the breakpoint */
674
-	if (ftrace_write(ip, old_code, 1))
675
-		BUG();
676
-	goto out;
250
+	text_poke_finish();
677251 }
678252 
679253 void arch_ftrace_update_code(int command)
680254 {
681
-	/* See comment above by declaration of modifying_ftrace_code */
682
-	atomic_inc(&modifying_ftrace_code);
683
-
684255 	ftrace_modify_all_code(command);
685
-
686
-	atomic_dec(&modifying_ftrace_code);
687256 }
688257 
689258 int __init ftrace_dyn_arch_init(void)
....@@ -701,12 +270,8 @@
701270 {
702271 	return module_alloc(size);
703272 }
704
-static inline void tramp_free(void *tramp, int size)
273
+static inline void tramp_free(void *tramp)
705274 {
706
-	int npages = PAGE_ALIGN(size) >> PAGE_SHIFT;
707
-
708
-	set_memory_nx((unsigned long)tramp, npages);
709
-	set_memory_rw((unsigned long)tramp, npages);
710275 	module_memfree(tramp);
711276 }
712277 #else
....@@ -715,14 +280,16 @@
715280 {
716281 	return NULL;
717282 }
718
-static inline void tramp_free(void *tramp, int size) { }
283
+static inline void tramp_free(void *tramp) { }
719284 #endif
720285 
721286 /* Defined as markers to the end of the ftrace default trampolines */
722287 extern void ftrace_regs_caller_end(void);
723
-extern void ftrace_epilogue(void);
288
+extern void ftrace_regs_caller_ret(void);
289
+extern void ftrace_caller_end(void);
724290 extern void ftrace_caller_op_ptr(void);
725291 extern void ftrace_regs_caller_op_ptr(void);
292
+extern void ftrace_regs_caller_jmp(void);
726293 
727294 /* movq function_trace_op(%rip), %rdx */
728295 /* 0x48 0x8b 0x15 <offset-to-ftrace_trace_op (4 bytes)> */
....@@ -744,7 +311,7 @@
744311 	} __attribute__((packed));
745312 };
746313 
747
-#define RET_SIZE		1
314
+#define RET_SIZE		(IS_ENABLED(CONFIG_RETPOLINE) ? 5 : 1 + IS_ENABLED(CONFIG_SLS))
748315 
749316 static unsigned long
750317 create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
....@@ -752,15 +319,17 @@
752319 	unsigned long start_offset;
753320 	unsigned long end_offset;
754321 	unsigned long op_offset;
322
+	unsigned long call_offset;
323
+	unsigned long jmp_offset;
755324 	unsigned long offset;
756325 	unsigned long npages;
757326 	unsigned long size;
758
-	unsigned long retq;
759327 	unsigned long *ptr;
760328 	void *trampoline;
761329 	void *ip;
762330 	/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */
763331 	unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };
332
+	unsigned const char retq[] = { RET_INSN_OPCODE, INT3_INSN_OPCODE };
764333 	union ftrace_op_code_union op_ptr;
765334 	int ret;
766335 
....@@ -768,10 +337,14 @@
768337 		start_offset = (unsigned long)ftrace_regs_caller;
769338 		end_offset = (unsigned long)ftrace_regs_caller_end;
770339 		op_offset = (unsigned long)ftrace_regs_caller_op_ptr;
340
+		call_offset = (unsigned long)ftrace_regs_call;
341
+		jmp_offset = (unsigned long)ftrace_regs_caller_jmp;
771342 	} else {
772343 		start_offset = (unsigned long)ftrace_caller;
773
-		end_offset = (unsigned long)ftrace_epilogue;
344
+		end_offset = (unsigned long)ftrace_caller_end;
774345 		op_offset = (unsigned long)ftrace_caller_op_ptr;
346
+		call_offset = (unsigned long)ftrace_call;
347
+		jmp_offset = 0;
775348 	}
776349 
777350 	size = end_offset - start_offset;
....@@ -789,17 +362,28 @@
789362 	npages = DIV_ROUND_UP(*tramp_size, PAGE_SIZE);
790363 
791364 	/* Copy ftrace_caller onto the trampoline memory */
792
-	ret = probe_kernel_read(trampoline, (void *)start_offset, size);
365
+	ret = copy_from_kernel_nofault(trampoline, (void *)start_offset, size);
793366 	if (WARN_ON(ret < 0))
794367 		goto fail;
795368 
796369 	ip = trampoline + size;
797370 
798371 	/* The trampoline ends with ret(q) */
799
-	retq = (unsigned long)ftrace_stub;
800
-	ret = probe_kernel_read(ip, (void *)retq, RET_SIZE);
801
-	if (WARN_ON(ret < 0))
802
-		goto fail;
372
+	if (cpu_feature_enabled(X86_FEATURE_RETHUNK))
373
+		memcpy(ip, text_gen_insn(JMP32_INSN_OPCODE, ip, &__x86_return_thunk), JMP32_INSN_SIZE);
374
+	else
375
+		memcpy(ip, retq, sizeof(retq));
376
+
377
+	/* No need to test direct calls on created trampolines */
378
+	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
379
+		/* NOP the jnz 1f; but make sure it's a 2 byte jnz */
380
+		ip = trampoline + (jmp_offset - start_offset);
381
+		if (WARN_ON(*(char *)ip != 0x75))
382
+			goto fail;
383
+		ret = copy_from_kernel_nofault(ip, ideal_nops[2], 2);
384
+		if (ret < 0)
385
+			goto fail;
386
+	}
803387 
804388 	/*
805389 	 * The address of the ftrace_ops that is used for this trampoline
....@@ -828,19 +412,53 @@
828412 	/* put in the new offset to the ftrace_ops */
829413 	memcpy(trampoline + op_offset, &op_ptr, OP_REF_SIZE);
830414 
415
+	/* put in the call to the function */
416
+	mutex_lock(&text_mutex);
417
+	call_offset -= start_offset;
418
+	memcpy(trampoline + call_offset,
419
+	       text_gen_insn(CALL_INSN_OPCODE,
420
+			     trampoline + call_offset,
421
+			     ftrace_ops_get_func(ops)), CALL_INSN_SIZE);
422
+	mutex_unlock(&text_mutex);
423
+
831424 	/* ALLOC_TRAMP flags lets us know we created it */
832425 	ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP;
833426 
834
-	/*
835
-	 * Module allocation needs to be completed by making the page
836
-	 * executable. The page is still writable, which is a security hazard,
837
-	 * but anyhow ftrace breaks W^X completely.
838
-	 */
427
+	set_vm_flush_reset_perms(trampoline);
428
+
429
+	if (likely(system_state != SYSTEM_BOOTING))
430
+		set_memory_ro((unsigned long)trampoline, npages);
839431 	set_memory_x((unsigned long)trampoline, npages);
840432 	return (unsigned long)trampoline;
841433 fail:
842
-	tramp_free(trampoline, *tramp_size);
434
+	tramp_free(trampoline);
843435 	return 0;
436
+}
437
+
438
+void set_ftrace_ops_ro(void)
439
+{
440
+	struct ftrace_ops *ops;
441
+	unsigned long start_offset;
442
+	unsigned long end_offset;
443
+	unsigned long npages;
444
+	unsigned long size;
445
+
446
+	do_for_each_ftrace_op(ops, ftrace_ops_list) {
447
+		if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
448
+			continue;
449
+
450
+		if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
451
+			start_offset = (unsigned long)ftrace_regs_caller;
452
+			end_offset = (unsigned long)ftrace_regs_caller_end;
453
+		} else {
454
+			start_offset = (unsigned long)ftrace_caller;
455
+			end_offset = (unsigned long)ftrace_caller_end;
456
+		}
457
+		size = end_offset - start_offset;
458
+		size = size + RET_SIZE + sizeof(void *);
459
+		npages = DIV_ROUND_UP(size, PAGE_SIZE);
460
+		set_memory_ro((unsigned long)ops->trampoline, npages);
461
+	} while_for_each_ftrace_op(ops);
844462 }
845463 
846464 static unsigned long calc_trampoline_call_offset(bool save_regs)
....@@ -862,62 +480,54 @@
862480 void arch_ftrace_update_trampoline(struct ftrace_ops *ops)
863481 {
864482 	ftrace_func_t func;
865
-	unsigned char *new;
866483 	unsigned long offset;
867484 	unsigned long ip;
868485 	unsigned int size;
869
-	int ret, npages;
486
+	const char *new;
870487 
871
-	if (ops->trampoline) {
872
-		/*
873
-		 * The ftrace_ops caller may set up its own trampoline.
874
-		 * In such a case, this code must not modify it.
875
-		 */
876
-		if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
877
-			return;
878
-		npages = PAGE_ALIGN(ops->trampoline_size) >> PAGE_SHIFT;
879
-		set_memory_rw(ops->trampoline, npages);
880
-	} else {
488
+	if (!ops->trampoline) {
881489 		ops->trampoline = create_trampoline(ops, &size);
882490 		if (!ops->trampoline)
883491 			return;
884492 		ops->trampoline_size = size;
885
-		npages = PAGE_ALIGN(size) >> PAGE_SHIFT;
493
+		return;
886494 	}
495
+
496
+	/*
497
+	 * The ftrace_ops caller may set up its own trampoline.
498
+	 * In such a case, this code must not modify it.
499
+	 */
500
+	if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
501
+		return;
887502 
888503 	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);
889504 	ip = ops->trampoline + offset;
890
-
891505 	func = ftrace_ops_get_func(ops);
892506 
893
-	ftrace_update_func_call = (unsigned long)func;
894
-
507
+	mutex_lock(&text_mutex);
895508 	/* Do a safe modify in case the trampoline is executing */
896509 	new = ftrace_call_replace(ip, (unsigned long)func);
897
-	ret = update_ftrace_func(ip, new);
898
-	set_memory_ro(ops->trampoline, npages);
899
-
900
-	/* The update should never fail */
901
-	WARN_ON(ret);
510
+	text_poke_bp((void *)ip, new, MCOUNT_INSN_SIZE, NULL);
511
+	mutex_unlock(&text_mutex);
902512 }
903513 
904514 /* Return the address of the function the trampoline calls */
905515 static void *addr_from_call(void *ptr)
906516 {
907
-	union ftrace_code_union calc;
517
+	union text_poke_insn call;
908518 	int ret;
909519 
910
-	ret = probe_kernel_read(&calc, ptr, MCOUNT_INSN_SIZE);
520
+	ret = copy_from_kernel_nofault(&call, ptr, CALL_INSN_SIZE);
911521 	if (WARN_ON_ONCE(ret < 0))
912522 		return NULL;
913523 
914524 	/* Make sure this is a call */
915
-	if (WARN_ON_ONCE(calc.op != 0xe8)) {
916
-		pr_warn("Expected e8, got %x\n", calc.op);
525
+	if (WARN_ON_ONCE(call.opcode != CALL_INSN_OPCODE)) {
526
+		pr_warn("Expected E8, got %x\n", call.opcode);
917527 		return NULL;
918528 	}
919529 
920
-	return ptr + MCOUNT_INSN_SIZE + calc.offset;
530
+	return ptr + CALL_INSN_SIZE + call.disp;
921531 }
922532 
923533 void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
....@@ -972,7 +582,7 @@
972582 	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
973583 		return;
974584 
975
-	tramp_free((void *)ops->trampoline, ops->trampoline_size);
585
+	tramp_free((void *)ops->trampoline);
976586 	ops->trampoline = 0;
977587 }
978588 
....@@ -984,19 +594,18 @@
984594 #ifdef CONFIG_DYNAMIC_FTRACE
985595 extern void ftrace_graph_call(void);
986596 
987
-static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
597
+static const char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
988598 {
989
-	return ftrace_text_replace(0xe9, ip, addr);
599
+	return text_gen_insn(JMP32_INSN_OPCODE, (void *)ip, (void *)addr);
990600 }
991601 
992602 static int ftrace_mod_jmp(unsigned long ip, void *func)
993603 {
994
-	unsigned char *new;
604
+	const char *new;
995605 
996
-	ftrace_update_func_call = 0UL;
997606 	new = ftrace_jmp_replace(ip, (unsigned long)func);
998
-
999
-	return update_ftrace_func(ip, new);
607
+	text_poke_bp((void *)ip, new, MCOUNT_INSN_SIZE, NULL);
608
+	return 0;
1000609 }
1001610 
1002611 int ftrace_enable_ftrace_graph_caller(void)
....@@ -1022,10 +631,9 @@
1022631 void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
1023632 			   unsigned long frame_pointer)
1024633 {
634
+	unsigned long return_hooker = (unsigned long)&return_to_handler;
1025635 	unsigned long old;
1026636 	int faulted;
1027
-	unsigned long return_hooker = (unsigned long)
1028
-				&return_to_handler;
1029637 
1030638 	/*
1031639 	 * When resuming from suspend-to-ram, this function can be indirectly