forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-10 23fa18eaa71266feff7ba8d83022d9e1cc83c65a 
 kernel/arch/powerpc/kernel/trace/ftrace.c
....@@ -27,26 +27,37 @@
2727 #include <asm/code-patching.h>
2828 #include <asm/ftrace.h>
2929 #include <asm/syscall.h>
30
+#include <asm/inst.h>
3031 
3132 
3233 #ifdef CONFIG_DYNAMIC_FTRACE
33
-static unsigned int
34
+
35
+/*
36
+ * We generally only have a single long_branch tramp and at most 2 or 3 plt
37
+ * tramps generated. But, we don't use the plt tramps currently. We also allot
38
+ * 2 tramps after .text and .init.text. So, we only end up with around 3 usable
39
+ * tramps in total. Set aside 8 just to be sure.
40
+ */
41
+#define	NUM_FTRACE_TRAMPS	8
42
+static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS];
43
+
44
+static struct ppc_inst
3445 ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
3546 {
36
-	unsigned int op;
47
+	struct ppc_inst op;
3748 
3849 	addr = ppc_function_entry((void *)addr);
3950 
4051 	/* if (link) set op to 'bl' else 'b' */
41
-	op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
52
+	create_branch(&op, (struct ppc_inst *)ip, addr, link ? 1 : 0);
4253 
4354 	return op;
4455 }
4556 
4657 static int
47
-ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
58
+ftrace_modify_code(unsigned long ip, struct ppc_inst old, struct ppc_inst new)
4859 {
49
-	unsigned int replaced;
60
+	struct ppc_inst replaced;
5061 
5162 	/*
5263 	 * Note:
....@@ -57,18 +68,18 @@
5768 	 */
5869 
5970 	/* read the text we want to modify */
60
-	if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
71
+	if (probe_kernel_read_inst(&replaced, (void *)ip))
6172 		return -EFAULT;
6273 
6374 	/* Make sure it is what we expect it to be */
64
-	if (replaced != old) {
65
-		pr_err("%p: replaced (%#x) != old (%#x)",
66
-		(void *)ip, replaced, old);
75
+	if (!ppc_inst_equal(replaced, old)) {
76
+		pr_err("%p: replaced (%s) != old (%s)",
77
+		(void *)ip, ppc_inst_as_str(replaced), ppc_inst_as_str(old));
6778 		return -EINVAL;
6879 	}
6980 
7081 	/* replace the text with the new text */
71
-	if (patch_instruction((unsigned int *)ip, new))
82
+	if (patch_instruction((struct ppc_inst *)ip, new))
7283 		return -EPERM;
7384 
7485 	return 0;
....@@ -79,24 +90,28 @@
7990  */
8091 static int test_24bit_addr(unsigned long ip, unsigned long addr)
8192 {
93
+	struct ppc_inst op;
8294 	addr = ppc_function_entry((void *)addr);
8395 
8496 	/* use the create_branch to verify that this offset can be branched */
85
-	return create_branch((unsigned int *)ip, addr, 0);
97
+	return create_branch(&op, (struct ppc_inst *)ip, addr, 0) == 0;
8698 }
8799 
88
-#ifdef CONFIG_MODULES
89
-
90
-static int is_bl_op(unsigned int op)
100
+static int is_bl_op(struct ppc_inst op)
91101 {
92
-	return (op & 0xfc000003) == 0x48000001;
102
+	return (ppc_inst_val(op) & 0xfc000003) == 0x48000001;
93103 }
94104 
95
-static unsigned long find_bl_target(unsigned long ip, unsigned int op)
105
+static int is_b_op(struct ppc_inst op)
96106 {
97
-	static int offset;
107
+	return (ppc_inst_val(op) & 0xfc000003) == 0x48000000;
108
+}
98109 
99
-	offset = (op & 0x03fffffc);
110
+static unsigned long find_bl_target(unsigned long ip, struct ppc_inst op)
111
+{
112
+	int offset;
113
+
114
+	offset = (ppc_inst_val(op) & 0x03fffffc);
100115 	/* make it signed */
101116 	if (offset & 0x02000000)
102117 		offset |= 0xfe000000;
....@@ -104,6 +119,7 @@
104119 	return ip + (long)offset;
105120 }
106121 
122
+#ifdef CONFIG_MODULES
107123 #ifdef CONFIG_PPC64
108124 static int
109125 __ftrace_make_nop(struct module *mod,
....@@ -111,17 +127,17 @@
111127 {
112128 	unsigned long entry, ptr, tramp;
113129 	unsigned long ip = rec->ip;
114
-	unsigned int op, pop;
130
+	struct ppc_inst op, pop;
115131 
116132 	/* read where this goes */
117
-	if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
133
+	if (probe_kernel_read_inst(&op, (void *)ip)) {
118134 		pr_err("Fetching opcode failed.\n");
119135 		return -EFAULT;
120136 	}
121137 
122138 	/* Make sure that that this is still a 24bit jump */
123139 	if (!is_bl_op(op)) {
124
-		pr_err("Not expected bl: opcode is %x\n", op);
140
+		pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
125141 		return -EINVAL;
126142 	}
127143 
....@@ -146,16 +162,18 @@
146162 
147163 #ifdef CONFIG_MPROFILE_KERNEL
148164 	/* When using -mkernel_profile there is no load to jump over */
149
-	pop = PPC_INST_NOP;
165
+	pop = ppc_inst(PPC_INST_NOP);
150166 
151
-	if (probe_kernel_read(&op, (void *)(ip - 4), 4)) {
167
+	if (probe_kernel_read_inst(&op, (void *)(ip - 4))) {
152168 		pr_err("Fetching instruction at %lx failed.\n", ip - 4);
153169 		return -EFAULT;
154170 	}
155171 
156172 	/* We expect either a mflr r0, or a std r0, LRSAVE(r1) */
157
-	if (op != PPC_INST_MFLR && op != PPC_INST_STD_LR) {
158
-		pr_err("Unexpected instruction %08x around bl _mcount\n", op);
173
+	if (!ppc_inst_equal(op, ppc_inst(PPC_INST_MFLR)) &&
174
+	    !ppc_inst_equal(op, ppc_inst(PPC_INST_STD_LR))) {
175
+		pr_err("Unexpected instruction %s around bl _mcount\n",
176
+		       ppc_inst_as_str(op));
159177 		return -EINVAL;
160178 	}
161179 #else
....@@ -173,24 +191,24 @@
173191 	 * Use a b +8 to jump over the load.
174192 	 */
175193 
176
-	pop = PPC_INST_BRANCH | 8;	/* b +8 */
194
+	pop = ppc_inst(PPC_INST_BRANCH | 8);	/* b +8 */
177195 
178196 	/*
179197 	 * Check what is in the next instruction. We can see ld r2,40(r1), but
180198 	 * on first pass after boot we will see mflr r0.
181199 	 */
182
-	if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) {
200
+	if (probe_kernel_read_inst(&op, (void *)(ip + 4))) {
183201 		pr_err("Fetching op failed.\n");
184202 		return -EFAULT;
185203 	}
186204 
187
-	if (op != PPC_INST_LD_TOC) {
188
-		pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC, op);
205
+	if (!ppc_inst_equal(op,  ppc_inst(PPC_INST_LD_TOC))) {
206
+		pr_err("Expected %08x found %s\n", PPC_INST_LD_TOC, ppc_inst_as_str(op));
189207 		return -EINVAL;
190208 	}
191209 #endif /* CONFIG_MPROFILE_KERNEL */
192210 
193
-	if (patch_instruction((unsigned int *)ip, pop)) {
211
+	if (patch_instruction((struct ppc_inst *)ip, pop)) {
194212 		pr_err("Patching NOP failed.\n");
195213 		return -EPERM;
196214 	}
....@@ -203,17 +221,17 @@
203221 __ftrace_make_nop(struct module *mod,
204222 		  struct dyn_ftrace *rec, unsigned long addr)
205223 {
206
-	unsigned int op;
224
+	struct ppc_inst op;
207225 	unsigned int jmp[4];
208226 	unsigned long ip = rec->ip;
209227 	unsigned long tramp;
210228 
211
-	if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
229
+	if (copy_from_kernel_nofault(&op, (void *)ip, MCOUNT_INSN_SIZE))
212230 		return -EFAULT;
213231 
214232 	/* Make sure that that this is still a 24bit jump */
215233 	if (!is_bl_op(op)) {
216
-		pr_err("Not expected bl: opcode is %x\n", op);
234
+		pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
217235 		return -EINVAL;
218236 	}
219237 
....@@ -231,7 +249,7 @@
231249 	pr_devel("ip:%lx jumps to %lx", ip, tramp);
232250 
233251 	/* Find where the trampoline jumps to */
234
-	if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
252
+	if (copy_from_kernel_nofault(jmp, (void *)tramp, sizeof(jmp))) {
235253 		pr_err("Failed to read %lx\n", tramp);
236254 		return -EFAULT;
237255 	}
....@@ -260,9 +278,9 @@
260278 		return -EINVAL;
261279 	}
262280 
263
-	op = PPC_INST_NOP;
281
+	op = ppc_inst(PPC_INST_NOP);
264282 
265
-	if (patch_instruction((unsigned int *)ip, op))
283
+	if (patch_instruction((struct ppc_inst *)ip, op))
266284 		return -EPERM;
267285 
268286 	return 0;
....@@ -270,11 +288,153 @@
270288 #endif /* PPC64 */
271289 #endif /* CONFIG_MODULES */
272290 
291
+static unsigned long find_ftrace_tramp(unsigned long ip)
292
+{
293
+	int i;
294
+	struct ppc_inst instr;
295
+
296
+	/*
297
+	 * We have the compiler generated long_branch tramps at the end
298
+	 * and we prefer those
299
+	 */
300
+	for (i = NUM_FTRACE_TRAMPS - 1; i >= 0; i--)
301
+		if (!ftrace_tramps[i])
302
+			continue;
303
+		else if (create_branch(&instr, (void *)ip,
304
+				       ftrace_tramps[i], 0) == 0)
305
+			return ftrace_tramps[i];
306
+
307
+	return 0;
308
+}
309
+
310
+static int add_ftrace_tramp(unsigned long tramp)
311
+{
312
+	int i;
313
+
314
+	for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
315
+		if (!ftrace_tramps[i]) {
316
+			ftrace_tramps[i] = tramp;
317
+			return 0;
318
+		}
319
+
320
+	return -1;
321
+}
322
+
323
+/*
324
+ * If this is a compiler generated long_branch trampoline (essentially, a
325
+ * trampoline that has a branch to _mcount()), we re-write the branch to
326
+ * instead go to ftrace_[regs_]caller() and note down the location of this
327
+ * trampoline.
328
+ */
329
+static int setup_mcount_compiler_tramp(unsigned long tramp)
330
+{
331
+	int i;
332
+	struct ppc_inst op;
333
+	unsigned long ptr;
334
+	struct ppc_inst instr;
335
+	static unsigned long ftrace_plt_tramps[NUM_FTRACE_TRAMPS];
336
+
337
+	/* Is this a known long jump tramp? */
338
+	for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
339
+		if (ftrace_tramps[i] == tramp)
340
+			return 0;
341
+
342
+	/* Is this a known plt tramp? */
343
+	for (i = 0; i < NUM_FTRACE_TRAMPS; i++)
344
+		if (!ftrace_plt_tramps[i])
345
+			break;
346
+		else if (ftrace_plt_tramps[i] == tramp)
347
+			return -1;
348
+
349
+	/* New trampoline -- read where this goes */
350
+	if (probe_kernel_read_inst(&op, (void *)tramp)) {
351
+		pr_debug("Fetching opcode failed.\n");
352
+		return -1;
353
+	}
354
+
355
+	/* Is this a 24 bit branch? */
356
+	if (!is_b_op(op)) {
357
+		pr_debug("Trampoline is not a long branch tramp.\n");
358
+		return -1;
359
+	}
360
+
361
+	/* lets find where the pointer goes */
362
+	ptr = find_bl_target(tramp, op);
363
+
364
+	if (ptr != ppc_global_function_entry((void *)_mcount)) {
365
+		pr_debug("Trampoline target %p is not _mcount\n", (void *)ptr);
366
+		return -1;
367
+	}
368
+
369
+	/* Let's re-write the tramp to go to ftrace_[regs_]caller */
370
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
371
+	ptr = ppc_global_function_entry((void *)ftrace_regs_caller);
372
+#else
373
+	ptr = ppc_global_function_entry((void *)ftrace_caller);
374
+#endif
375
+	if (create_branch(&instr, (void *)tramp, ptr, 0)) {
376
+		pr_debug("%ps is not reachable from existing mcount tramp\n",
377
+				(void *)ptr);
378
+		return -1;
379
+	}
380
+
381
+	if (patch_branch((struct ppc_inst *)tramp, ptr, 0)) {
382
+		pr_debug("REL24 out of range!\n");
383
+		return -1;
384
+	}
385
+
386
+	if (add_ftrace_tramp(tramp)) {
387
+		pr_debug("No tramp locations left\n");
388
+		return -1;
389
+	}
390
+
391
+	return 0;
392
+}
393
+
394
+static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr)
395
+{
396
+	unsigned long tramp, ip = rec->ip;
397
+	struct ppc_inst op;
398
+
399
+	/* Read where this goes */
400
+	if (probe_kernel_read_inst(&op, (void *)ip)) {
401
+		pr_err("Fetching opcode failed.\n");
402
+		return -EFAULT;
403
+	}
404
+
405
+	/* Make sure that that this is still a 24bit jump */
406
+	if (!is_bl_op(op)) {
407
+		pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
408
+		return -EINVAL;
409
+	}
410
+
411
+	/* Let's find where the pointer goes */
412
+	tramp = find_bl_target(ip, op);
413
+
414
+	pr_devel("ip:%lx jumps to %lx", ip, tramp);
415
+
416
+	if (setup_mcount_compiler_tramp(tramp)) {
417
+		/* Are other trampolines reachable? */
418
+		if (!find_ftrace_tramp(ip)) {
419
+			pr_err("No ftrace trampolines reachable from %ps\n",
420
+					(void *)ip);
421
+			return -EINVAL;
422
+		}
423
+	}
424
+
425
+	if (patch_instruction((struct ppc_inst *)ip, ppc_inst(PPC_INST_NOP))) {
426
+		pr_err("Patching NOP failed.\n");
427
+		return -EPERM;
428
+	}
429
+
430
+	return 0;
431
+}
432
+
273433 int ftrace_make_nop(struct module *mod,
274434 		    struct dyn_ftrace *rec, unsigned long addr)
275435 {
276436 	unsigned long ip = rec->ip;
277
-	unsigned int old, new;
437
+	struct ppc_inst old, new;
278438 
279439 	/*
280440 	 * If the calling address is more that 24 bits away,
....@@ -284,9 +444,10 @@
284444 	if (test_24bit_addr(ip, addr)) {
285445 		/* within range */
286446 		old = ftrace_call_replace(ip, addr, 1);
287
-		new = PPC_INST_NOP;
447
+		new = ppc_inst(PPC_INST_NOP);
288448 		return ftrace_modify_code(ip, old, new);
289
-	}
449
+	} else if (core_kernel_text(ip))
450
+		return __ftrace_make_nop_kernel(rec, addr);
290451 
291452 #ifdef CONFIG_MODULES
292453 	/*
....@@ -326,7 +487,7 @@
326487  */
327488 #ifndef CONFIG_MPROFILE_KERNEL
328489 static int
329
-expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
490
+expected_nop_sequence(void *ip, struct ppc_inst op0, struct ppc_inst op1)
330491 {
331492 	/*
332493 	 * We expect to see:
....@@ -337,16 +498,17 @@
337498 	 * The load offset is different depending on the ABI. For simplicity
338499 	 * just mask it out when doing the compare.
339500 	 */
340
-	if ((op0 != 0x48000008) || ((op1 & 0xffff0000) != 0xe8410000))
501
+	if (!ppc_inst_equal(op0, ppc_inst(0x48000008)) ||
502
+	    (ppc_inst_val(op1) & 0xffff0000) != 0xe8410000)
341503 		return 0;
342504 	return 1;
343505 }
344506 #else
345507 static int
346
-expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
508
+expected_nop_sequence(void *ip, struct ppc_inst op0, struct ppc_inst op1)
347509 {
348510 	/* look for patched "NOP" on ppc64 with -mprofile-kernel */
349
-	if (op0 != PPC_INST_NOP)
511
+	if (!ppc_inst_equal(op0, ppc_inst(PPC_INST_NOP)))
350512 		return 0;
351513 	return 1;
352514 }
....@@ -355,18 +517,22 @@
355517 static int
356518 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
357519 {
358
-	unsigned int op[2];
520
+	struct ppc_inst op[2];
521
+	struct ppc_inst instr;
359522 	void *ip = (void *)rec->ip;
360523 	unsigned long entry, ptr, tramp;
361524 	struct module *mod = rec->arch.mod;
362525 
363526 	/* read where this goes */
364
-	if (probe_kernel_read(op, ip, sizeof(op)))
527
+	if (probe_kernel_read_inst(op, ip))
528
+		return -EFAULT;
529
+
530
+	if (probe_kernel_read_inst(op + 1, ip + 4))
365531 		return -EFAULT;
366532 
367533 	if (!expected_nop_sequence(ip, op[0], op[1])) {
368
-		pr_err("Unexpected call sequence at %p: %x %x\n",
369
-		ip, op[0], op[1]);
534
+		pr_err("Unexpected call sequence at %p: %s %s\n",
535
+		ip, ppc_inst_as_str(op[0]), ppc_inst_as_str(op[1]));
370536 		return -EINVAL;
371537 	}
372538 
....@@ -402,7 +568,7 @@
402568 	}
403569 
404570 	/* Ensure branch is within 24 bits */
405
-	if (!create_branch(ip, tramp, BRANCH_SET_LINK)) {
571
+	if (create_branch(&instr, ip, tramp, BRANCH_SET_LINK)) {
406572 		pr_err("Branch out of range\n");
407573 		return -EINVAL;
408574 	}
....@@ -419,16 +585,17 @@
419585 static int
420586 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
421587 {
422
-	unsigned int op;
588
+	int err;
589
+	struct ppc_inst op;
423590 	unsigned long ip = rec->ip;
424591 
425592 	/* read where this goes */
426
-	if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
593
+	if (probe_kernel_read_inst(&op, (void *)ip))
427594 		return -EFAULT;
428595 
429596 	/* It should be pointing to a nop */
430
-	if (op != PPC_INST_NOP) {
431
-		pr_err("Expected NOP but have %x\n", op);
597
+	if (!ppc_inst_equal(op,  ppc_inst(PPC_INST_NOP))) {
598
+		pr_err("Expected NOP but have %s\n", ppc_inst_as_str(op));
432599 		return -EINVAL;
433600 	}
434601 
....@@ -439,16 +606,16 @@
439606 	}
440607 
441608 	/* create the branch to the trampoline */
442
-	op = create_branch((unsigned int *)ip,
443
-			   rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
444
-	if (!op) {
609
+	err = create_branch(&op, (struct ppc_inst *)ip,
610
+			    rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
611
+	if (err) {
445612 		pr_err("REL24 out of range!\n");
446613 		return -EINVAL;
447614 	}
448615 
449616 	pr_devel("write to %lx\n", rec->ip);
450617 
451
-	if (patch_instruction((unsigned int *)ip, op))
618
+	if (patch_instruction((struct ppc_inst *)ip, op))
452619 		return -EPERM;
453620 
454621 	return 0;
....@@ -456,10 +623,57 @@
456623 #endif /* CONFIG_PPC64 */
457624 #endif /* CONFIG_MODULES */
458625 
626
+static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr)
627
+{
628
+	struct ppc_inst op;
629
+	void *ip = (void *)rec->ip;
630
+	unsigned long tramp, entry, ptr;
631
+
632
+	/* Make sure we're being asked to patch branch to a known ftrace addr */
633
+	entry = ppc_global_function_entry((void *)ftrace_caller);
634
+	ptr = ppc_global_function_entry((void *)addr);
635
+
636
+	if (ptr != entry) {
637
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
638
+		entry = ppc_global_function_entry((void *)ftrace_regs_caller);
639
+		if (ptr != entry) {
640
+#endif
641
+			pr_err("Unknown ftrace addr to patch: %ps\n", (void *)ptr);
642
+			return -EINVAL;
643
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
644
+		}
645
+#endif
646
+	}
647
+
648
+	/* Make sure we have a nop */
649
+	if (probe_kernel_read_inst(&op, ip)) {
650
+		pr_err("Unable to read ftrace location %p\n", ip);
651
+		return -EFAULT;
652
+	}
653
+
654
+	if (!ppc_inst_equal(op, ppc_inst(PPC_INST_NOP))) {
655
+		pr_err("Unexpected call sequence at %p: %s\n", ip, ppc_inst_as_str(op));
656
+		return -EINVAL;
657
+	}
658
+
659
+	tramp = find_ftrace_tramp((unsigned long)ip);
660
+	if (!tramp) {
661
+		pr_err("No ftrace trampolines reachable from %ps\n", ip);
662
+		return -EINVAL;
663
+	}
664
+
665
+	if (patch_branch(ip, tramp, BRANCH_SET_LINK)) {
666
+		pr_err("Error patching branch to ftrace tramp!\n");
667
+		return -EINVAL;
668
+	}
669
+
670
+	return 0;
671
+}
672
+
459673 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
460674 {
461675 	unsigned long ip = rec->ip;
462
-	unsigned int old, new;
676
+	struct ppc_inst old, new;
463677 
464678 	/*
465679 	 * If the calling address is more that 24 bits away,
....@@ -468,10 +682,11 @@
468682 	 */
469683 	if (test_24bit_addr(ip, addr)) {
470684 		/* within range */
471
-		old = PPC_INST_NOP;
685
+		old = ppc_inst(PPC_INST_NOP);
472686 		new = ftrace_call_replace(ip, addr, 1);
473687 		return ftrace_modify_code(ip, old, new);
474
-	}
688
+	} else if (core_kernel_text(ip))
689
+		return __ftrace_make_call_kernel(rec, addr);
475690 
476691 #ifdef CONFIG_MODULES
477692 	/*
....@@ -497,7 +712,7 @@
497712 __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
498713 					unsigned long addr)
499714 {
500
-	unsigned int op;
715
+	struct ppc_inst op;
501716 	unsigned long ip = rec->ip;
502717 	unsigned long entry, ptr, tramp;
503718 	struct module *mod = rec->arch.mod;
....@@ -509,14 +724,14 @@
509724 	}
510725 
511726 	/* read where this goes */
512
-	if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
727
+	if (probe_kernel_read_inst(&op, (void *)ip)) {
513728 		pr_err("Fetching opcode failed.\n");
514729 		return -EFAULT;
515730 	}
516731 
517732 	/* Make sure that that this is still a 24bit jump */
518733 	if (!is_bl_op(op)) {
519
-		pr_err("Not expected bl: opcode is %x\n", op);
734
+		pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op));
520735 		return -EINVAL;
521736 	}
522737 
....@@ -545,7 +760,7 @@
545760 	/* The new target may be within range */
546761 	if (test_24bit_addr(ip, addr)) {
547762 		/* within range */
548
-		if (patch_branch((unsigned int *)ip, addr, BRANCH_SET_LINK)) {
763
+		if (patch_branch((struct ppc_inst *)ip, addr, BRANCH_SET_LINK)) {
549764 			pr_err("REL24 out of range!\n");
550765 			return -EINVAL;
551766 		}
....@@ -573,12 +788,12 @@
573788 	}
574789 
575790 	/* Ensure branch is within 24 bits */
576
-	if (!create_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) {
791
+	if (create_branch(&op, (struct ppc_inst *)ip, tramp, BRANCH_SET_LINK)) {
577792 		pr_err("Branch out of range\n");
578793 		return -EINVAL;
579794 	}
580795 
581
-	if (patch_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) {
796
+	if (patch_branch((struct ppc_inst *)ip, tramp, BRANCH_SET_LINK)) {
582797 		pr_err("REL24 out of range!\n");
583798 		return -EINVAL;
584799 	}
....@@ -591,7 +806,7 @@
591806 			unsigned long addr)
592807 {
593808 	unsigned long ip = rec->ip;
594
-	unsigned int old, new;
809
+	struct ppc_inst old, new;
595810 
596811 	/*
597812 	 * If the calling address is more that 24 bits away,
....@@ -603,6 +818,12 @@
603818 		old = ftrace_call_replace(ip, old_addr, 1);
604819 		new = ftrace_call_replace(ip, addr, 1);
605820 		return ftrace_modify_code(ip, old, new);
821
+	} else if (core_kernel_text(ip)) {
822
+		/*
823
+		 * We always patch out of range locations to go to the regs
824
+		 * variant, so there is nothing to do here
825
+		 */
826
+		return 0;
606827 	}
607828 
608829 #ifdef CONFIG_MODULES
....@@ -625,10 +846,10 @@
625846 int ftrace_update_ftrace_func(ftrace_func_t func)
626847 {
627848 	unsigned long ip = (unsigned long)(&ftrace_call);
628
-	unsigned int old, new;
849
+	struct ppc_inst old, new;
629850 	int ret;
630851 
631
-	old = *(unsigned int *)&ftrace_call;
852
+	old = ppc_inst_read((struct ppc_inst *)&ftrace_call);
632853 	new = ftrace_call_replace(ip, (unsigned long)func, 1);
633854 	ret = ftrace_modify_code(ip, old, new);
634855 
....@@ -636,7 +857,7 @@
636857 	/* Also update the regs callback function */
637858 	if (!ret) {
638859 		ip = (unsigned long)(&ftrace_regs_call);
639
-		old = *(unsigned int *)&ftrace_regs_call;
860
+		old = ppc_inst_read((struct ppc_inst *)&ftrace_regs_call);
640861 		new = ftrace_call_replace(ip, (unsigned long)func, 1);
641862 		ret = ftrace_modify_code(ip, old, new);
642863 	}
....@@ -654,10 +875,61 @@
654875 	ftrace_modify_all_code(command);
655876 }
656877 
878
+#ifdef CONFIG_PPC64
879
+#define PACATOC offsetof(struct paca_struct, kernel_toc)
880
+
881
+extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[];
882
+
883
+void ftrace_free_init_tramp(void)
884
+{
885
+	int i;
886
+
887
+	for (i = 0; i < NUM_FTRACE_TRAMPS && ftrace_tramps[i]; i++)
888
+		if (ftrace_tramps[i] == (unsigned long)ftrace_tramp_init) {
889
+			ftrace_tramps[i] = 0;
890
+			return;
891
+		}
892
+}
893
+
894
+int __init ftrace_dyn_arch_init(void)
895
+{
896
+	int i;
897
+	unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init };
898
+	u32 stub_insns[] = {
899
+		0xe98d0000 | PACATOC,	/* ld      r12,PACATOC(r13)	*/
900
+		0x3d8c0000,		/* addis   r12,r12,<high>	*/
901
+		0x398c0000,		/* addi    r12,r12,<low>	*/
902
+		0x7d8903a6,		/* mtctr   r12			*/
903
+		0x4e800420,		/* bctr				*/
904
+	};
905
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
906
+	unsigned long addr = ppc_global_function_entry((void *)ftrace_regs_caller);
907
+#else
908
+	unsigned long addr = ppc_global_function_entry((void *)ftrace_caller);
909
+#endif
910
+	long reladdr = addr - kernel_toc_addr();
911
+
912
+	if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
913
+		pr_err("Address of %ps out of range of kernel_toc.\n",
914
+				(void *)addr);
915
+		return -1;
916
+	}
917
+
918
+	for (i = 0; i < 2; i++) {
919
+		memcpy(tramp[i], stub_insns, sizeof(stub_insns));
920
+		tramp[i][1] |= PPC_HA(reladdr);
921
+		tramp[i][2] |= PPC_LO(reladdr);
922
+		add_ftrace_tramp((unsigned long)tramp[i]);
923
+	}
924
+
925
+	return 0;
926
+}
927
+#else
657928 int __init ftrace_dyn_arch_init(void)
658929 {
659930 	return 0;
660931 }
932
+#endif
661933 #endif /* CONFIG_DYNAMIC_FTRACE */
662934 
663935 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
....@@ -670,7 +942,7 @@
670942 	unsigned long ip = (unsigned long)(&ftrace_graph_call);
671943 	unsigned long addr = (unsigned long)(&ftrace_graph_caller);
672944 	unsigned long stub = (unsigned long)(&ftrace_graph_stub);
673
-	unsigned int old, new;
945
+	struct ppc_inst old, new;
674946 
675947 	old = ftrace_call_replace(ip, stub, 0);
676948 	new = ftrace_call_replace(ip, addr, 0);
....@@ -683,7 +955,7 @@
683955 	unsigned long ip = (unsigned long)(&ftrace_graph_call);
684956 	unsigned long addr = (unsigned long)(&ftrace_graph_caller);
685957 	unsigned long stub = (unsigned long)(&ftrace_graph_stub);
686
-	unsigned int old, new;
958
+	struct ppc_inst old, new;
687959 
688960 	old = ftrace_call_replace(ip, addr, 0);
689961 	new = ftrace_call_replace(ip, stub, 0);
....@@ -695,7 +967,8 @@
695967  * Hook the return address and push it in the stack of return addrs
696968  * in current thread info. Return the address we want to divert to.
697969  */
698
-unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip)
970
+unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip,
971
+						unsigned long sp)
699972 {
700973 	unsigned long return_hooker;
701974 
....@@ -707,19 +980,12 @@
707980 
708981 	return_hooker = ppc_function_entry(return_to_handler);
709982 
710
-	if (!function_graph_enter(parent, ip, 0, NULL))
983
+	if (!function_graph_enter(parent, ip, 0, (unsigned long *)sp))
711984 		parent = return_hooker;
712985 out:
713986 	return parent;
714987 }
715988 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
716
-
717
-#if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64)
718
-unsigned long __init arch_syscall_addr(int nr)
719
-{
720
-	return sys_call_table[nr*2];
721
-}
722
-#endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 */
723989 
724990 #ifdef PPC64_ELF_ABI_v1
725991 char *arch_ftrace_match_adjust(char *str, const char *search)