forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/arch/powerpc/kernel/module_64.c
....@@ -1,19 +1,7 @@
1
+// SPDX-License-Identifier: GPL-2.0-or-later
12 /*  Kernel module help for PPC64.
23     Copyright (C) 2001, 2003 Rusty Russell IBM Corporation.
34 
4
-    This program is free software; you can redistribute it and/or modify
5
-    it under the terms of the GNU General Public License as published by
6
-    the Free Software Foundation; either version 2 of the License, or
7
-    (at your option) any later version.
8
-
9
-    This program is distributed in the hope that it will be useful,
10
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
11
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
-    GNU General Public License for more details.
13
-
14
-    You should have received a copy of the GNU General Public License
15
-    along with this program; if not, write to the Free Software
16
-    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
175 */
186 
197 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
....@@ -32,6 +20,7 @@
3220 #include <linux/sort.h>
3321 #include <asm/setup.h>
3422 #include <asm/sections.h>
23
+#include <asm/inst.h>
3524 
3625 /* FIXME: We don't do .init separately.  To do this, we'd need to have
3726    a separate r2 value in the init and core section, and stub between
....@@ -133,57 +122,28 @@
133122  * the stub, but it's significantly shorter to put these values at the
134123  * end of the stub code, and patch the stub address (32-bits relative
135124  * to the TOC ptr, r2) into the stub.
125
+ *
126
+ * addis   r11,r2, <high>
127
+ * addi    r11,r11, <low>
128
+ * std     r2,R2_STACK_OFFSET(r1)
129
+ * ld      r12,32(r11)
130
+ * ld      r2,40(r11)
131
+ * mtctr   r12
132
+ * bctr
136133  */
137
-
138134 static u32 ppc64_stub_insns[] = {
139
-	0x3d620000,			/* addis   r11,r2, <high> */
140
-	0x396b0000,			/* addi    r11,r11, <low> */
135
+	PPC_INST_ADDIS | __PPC_RT(R11) | __PPC_RA(R2),
136
+	PPC_INST_ADDI | __PPC_RT(R11) | __PPC_RA(R11),
141137 	/* Save current r2 value in magic place on the stack. */
142
-	0xf8410000|R2_STACK_OFFSET,	/* std     r2,R2_STACK_OFFSET(r1) */
143
-	0xe98b0020,			/* ld      r12,32(r11) */
138
+	PPC_INST_STD | __PPC_RS(R2) | __PPC_RA(R1) | R2_STACK_OFFSET,
139
+	PPC_INST_LD | __PPC_RT(R12) | __PPC_RA(R11) | 32,
144140 #ifdef PPC64_ELF_ABI_v1
145141 	/* Set up new r2 from function descriptor */
146
-	0xe84b0028,			/* ld      r2,40(r11) */
142
+	PPC_INST_LD | __PPC_RT(R2) | __PPC_RA(R11) | 40,
147143 #endif
148
-	0x7d8903a6,			/* mtctr   r12 */
149
-	0x4e800420			/* bctr */
144
+	PPC_INST_MTCTR | __PPC_RS(R12),
145
+	PPC_INST_BCTR,
150146 };
151
-
152
-#ifdef CONFIG_DYNAMIC_FTRACE
153
-int module_trampoline_target(struct module *mod, unsigned long addr,
154
-			     unsigned long *target)
155
-{
156
-	struct ppc64_stub_entry *stub;
157
-	func_desc_t funcdata;
158
-	u32 magic;
159
-
160
-	if (!within_module_core(addr, mod)) {
161
-		pr_err("%s: stub %lx not in module %s\n", __func__, addr, mod->name);
162
-		return -EFAULT;
163
-	}
164
-
165
-	stub = (struct ppc64_stub_entry *)addr;
166
-
167
-	if (probe_kernel_read(&magic, &stub->magic, sizeof(magic))) {
168
-		pr_err("%s: fault reading magic for stub %lx for %s\n", __func__, addr, mod->name);
169
-		return -EFAULT;
170
-	}
171
-
172
-	if (magic != STUB_MAGIC) {
173
-		pr_err("%s: bad magic for stub %lx for %s\n", __func__, addr, mod->name);
174
-		return -EFAULT;
175
-	}
176
-
177
-	if (probe_kernel_read(&funcdata, &stub->funcdata, sizeof(funcdata))) {
178
-		pr_err("%s: fault reading funcdata for stub %lx for %s\n", __func__, addr, mod->name);
179
-                return -EFAULT;
180
-	}
181
-
182
-	*target = stub_func_addr(funcdata);
183
-
184
-	return 0;
185
-}
186
-#endif
187147 
188148 /* Count how many different 24-bit relocations (different symbol,
189149    different addend) */
....@@ -231,21 +191,6 @@
231191 		return 0;
232192 }
233193 
234
-static void relaswap(void *_x, void *_y, int size)
235
-{
236
-	uint64_t *x, *y, tmp;
237
-	int i;
238
-
239
-	y = (uint64_t *)_x;
240
-	x = (uint64_t *)_y;
241
-
242
-	for (i = 0; i < sizeof(Elf64_Rela) / sizeof(uint64_t); i++) {
243
-		tmp = x[i];
244
-		x[i] = y[i];
245
-		y[i] = tmp;
246
-	}
247
-}
248
-
249194 /* Get size of potential trampolines required. */
250195 static unsigned long get_stubs_size(const Elf64_Ehdr *hdr,
251196 				    const Elf64_Shdr *sechdrs)
....@@ -269,7 +214,7 @@
269214 			 */
270215 			sort((void *)sechdrs[i].sh_addr,
271216 			     sechdrs[i].sh_size / sizeof(Elf64_Rela),
272
-			     sizeof(Elf64_Rela), relacmp, relaswap);
217
+			     sizeof(Elf64_Rela), relacmp, NULL);
273218 
274219 			relocs += count_relocs((void *)sechdrs[i].sh_addr,
275220 					       sechdrs[i].sh_size
....@@ -389,6 +334,92 @@
389334 	return 0;
390335 }
391336 
337
+#ifdef CONFIG_MPROFILE_KERNEL
338
+
339
+#define PACATOC offsetof(struct paca_struct, kernel_toc)
340
+
341
+/*
342
+ * ld      r12,PACATOC(r13)
343
+ * addis   r12,r12,<high>
344
+ * addi    r12,r12,<low>
345
+ * mtctr   r12
346
+ * bctr
347
+ */
348
+static u32 stub_insns[] = {
349
+	PPC_INST_LD | __PPC_RT(R12) | __PPC_RA(R13) | PACATOC,
350
+	PPC_INST_ADDIS | __PPC_RT(R12) | __PPC_RA(R12),
351
+	PPC_INST_ADDI | __PPC_RT(R12) | __PPC_RA(R12),
352
+	PPC_INST_MTCTR | __PPC_RS(R12),
353
+	PPC_INST_BCTR,
354
+};
355
+
356
+/*
357
+ * For mprofile-kernel we use a special stub for ftrace_caller() because we
358
+ * can't rely on r2 containing this module's TOC when we enter the stub.
359
+ *
360
+ * That can happen if the function calling us didn't need to use the toc. In
361
+ * that case it won't have setup r2, and the r2 value will be either the
362
+ * kernel's toc, or possibly another modules toc.
363
+ *
364
+ * To deal with that this stub uses the kernel toc, which is always accessible
365
+ * via the paca (in r13). The target (ftrace_caller()) is responsible for
366
+ * saving and restoring the toc before returning.
367
+ */
368
+static inline int create_ftrace_stub(struct ppc64_stub_entry *entry,
369
+					unsigned long addr,
370
+					struct module *me)
371
+{
372
+	long reladdr;
373
+
374
+	memcpy(entry->jump, stub_insns, sizeof(stub_insns));
375
+
376
+	/* Stub uses address relative to kernel toc (from the paca) */
377
+	reladdr = addr - kernel_toc_addr();
378
+	if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
379
+		pr_err("%s: Address of %ps out of range of kernel_toc.\n",
380
+							me->name, (void *)addr);
381
+		return 0;
382
+	}
383
+
384
+	entry->jump[1] |= PPC_HA(reladdr);
385
+	entry->jump[2] |= PPC_LO(reladdr);
386
+
387
+	/* Eventhough we don't use funcdata in the stub, it's needed elsewhere. */
388
+	entry->funcdata = func_desc(addr);
389
+	entry->magic = STUB_MAGIC;
390
+
391
+	return 1;
392
+}
393
+
394
+static bool is_mprofile_ftrace_call(const char *name)
395
+{
396
+	if (!strcmp("_mcount", name))
397
+		return true;
398
+#ifdef CONFIG_DYNAMIC_FTRACE
399
+	if (!strcmp("ftrace_caller", name))
400
+		return true;
401
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
402
+	if (!strcmp("ftrace_regs_caller", name))
403
+		return true;
404
+#endif
405
+#endif
406
+
407
+	return false;
408
+}
409
+#else
410
+static inline int create_ftrace_stub(struct ppc64_stub_entry *entry,
411
+					unsigned long addr,
412
+					struct module *me)
413
+{
414
+	return 0;
415
+}
416
+
417
+static bool is_mprofile_ftrace_call(const char *name)
418
+{
419
+	return false;
420
+}
421
+#endif
422
+
392423 /*
393424  * r2 is the TOC pointer: it actually points 0x8000 into the TOC (this gives the
394425  * value maximum span in an instruction which uses a signed offset). Round down
....@@ -400,20 +431,17 @@
400431 	return (sechdrs[me->arch.toc_section].sh_addr & ~0xfful) + 0x8000;
401432 }
402433 
403
-/* Both low and high 16 bits are added as SIGNED additions, so if low
404
-   16 bits has high bit set, high 16 bits must be adjusted.  These
405
-   macros do that (stolen from binutils). */
406
-#define PPC_LO(v) ((v) & 0xffff)
407
-#define PPC_HI(v) (((v) >> 16) & 0xffff)
408
-#define PPC_HA(v) PPC_HI ((v) + 0x8000)
409
-
410434 /* Patch stub to reference function and correct r2 value. */
411435 static inline int create_stub(const Elf64_Shdr *sechdrs,
412436 			      struct ppc64_stub_entry *entry,
413437 			      unsigned long addr,
414
-			      struct module *me)
438
+			      struct module *me,
439
+			      const char *name)
415440 {
416441 	long reladdr;
442
+
443
+	if (is_mprofile_ftrace_call(name))
444
+		return create_ftrace_stub(entry, addr, me);
417445 
418446 	memcpy(entry->jump, ppc64_stub_insns, sizeof(ppc64_stub_insns));
419447 
....@@ -438,7 +466,8 @@
438466    stub to set up the TOC ptr (r2) for the function. */
439467 static unsigned long stub_for_addr(const Elf64_Shdr *sechdrs,
440468 				   unsigned long addr,
441
-				   struct module *me)
469
+				   struct module *me,
470
+				   const char *name)
442471 {
443472 	struct ppc64_stub_entry *stubs;
444473 	unsigned int i, num_stubs;
....@@ -455,54 +484,11 @@
455484 			return (unsigned long)&stubs[i];
456485 	}
457486 
458
-	if (!create_stub(sechdrs, &stubs[i], addr, me))
487
+	if (!create_stub(sechdrs, &stubs[i], addr, me, name))
459488 		return 0;
460489 
461490 	return (unsigned long)&stubs[i];
462491 }
463
-
464
-#ifdef CONFIG_MPROFILE_KERNEL
465
-static bool is_mprofile_mcount_callsite(const char *name, u32 *instruction)
466
-{
467
-	if (strcmp("_mcount", name))
468
-		return false;
469
-
470
-	/*
471
-	 * Check if this is one of the -mprofile-kernel sequences.
472
-	 */
473
-	if (instruction[-1] == PPC_INST_STD_LR &&
474
-	    instruction[-2] == PPC_INST_MFLR)
475
-		return true;
476
-
477
-	if (instruction[-1] == PPC_INST_MFLR)
478
-		return true;
479
-
480
-	return false;
481
-}
482
-
483
-/*
484
- * In case of _mcount calls, do not save the current callee's TOC (in r2) into
485
- * the original caller's stack frame. If we did we would clobber the saved TOC
486
- * value of the original caller.
487
- */
488
-static void squash_toc_save_inst(const char *name, unsigned long addr)
489
-{
490
-	struct ppc64_stub_entry *stub = (struct ppc64_stub_entry *)addr;
491
-
492
-	/* Only for calls to _mcount */
493
-	if (strcmp("_mcount", name) != 0)
494
-		return;
495
-
496
-	stub->jump[2] = PPC_INST_NOP;
497
-}
498
-#else
499
-static void squash_toc_save_inst(const char *name, unsigned long addr) { }
500
-
501
-static bool is_mprofile_mcount_callsite(const char *name, u32 *instruction)
502
-{
503
-	return false;
504
-}
505
-#endif
506492 
507493 /* We expect a noop next: if it is, replace it with instruction to
508494    restore r2. */
....@@ -510,7 +496,7 @@
510496 {
511497 	u32 *prev_insn = instruction - 1;
512498 
513
-	if (is_mprofile_mcount_callsite(name, prev_insn))
499
+	if (is_mprofile_ftrace_call(name))
514500 		return 1;
515501 
516502 	/*
....@@ -518,7 +504,7 @@
518504 	 * "link" branches and they don't return, so they don't need the r2
519505 	 * restore afterwards.
520506 	 */
521
-	if (!instr_is_relative_link_branch(*prev_insn))
507
+	if (!instr_is_relative_link_branch(ppc_inst(*prev_insn)))
522508 		return 1;
523509 
524510 	if (*instruction != PPC_INST_NOP) {
....@@ -648,14 +634,13 @@
648634 			if (sym->st_shndx == SHN_UNDEF ||
649635 			    sym->st_shndx == SHN_LIVEPATCH) {
650636 				/* External: go via stub */
651
-				value = stub_for_addr(sechdrs, value, me);
637
+				value = stub_for_addr(sechdrs, value, me,
638
+						strtab + sym->st_name);
652639 				if (!value)
653640 					return -ENOENT;
654641 				if (!restore_r2(strtab + sym->st_name,
655642 							(u32 *)location + 1, me))
656643 					return -ENOEXEC;
657
-
658
-				squash_toc_save_inst(strtab + sym->st_name, value);
659644 			} else
660645 				value += local_entry_offset(sym);
661646 
....@@ -711,18 +696,21 @@
711696 		         *	ld r2, ...(r12)
712697 			 *	add r2, r2, r12
713698 			 */
714
-			if ((((uint32_t *)location)[0] & ~0xfffc)
715
-			    != 0xe84c0000)
699
+			if ((((uint32_t *)location)[0] & ~0xfffc) !=
700
+			    (PPC_INST_LD | __PPC_RT(R2) | __PPC_RA(R12)))
716701 				break;
717
-			if (((uint32_t *)location)[1] != 0x7c426214)
702
+			if (((uint32_t *)location)[1] !=
703
+			    (PPC_INST_ADD | __PPC_RT(R2) | __PPC_RA(R2) | __PPC_RB(R12)))
718704 				break;
719705 			/*
720706 			 * If found, replace it with:
721707 			 *	addis r2, r12, (.TOC.-func)@ha
722708 			 *	addi  r2,  r2, (.TOC.-func)@l
723709 			 */
724
-			((uint32_t *)location)[0] = 0x3c4c0000 + PPC_HA(value);
725
-			((uint32_t *)location)[1] = 0x38420000 + PPC_LO(value);
710
+			((uint32_t *)location)[0] = PPC_INST_ADDIS | __PPC_RT(R2) |
711
+						    __PPC_RA(R12) | PPC_HA(value);
712
+			((uint32_t *)location)[1] = PPC_INST_ADDI | __PPC_RT(R2) |
713
+						    __PPC_RA(R2) | PPC_LO(value);
726714 			break;
727715 
728716 		case R_PPC64_REL16_HA:
....@@ -754,82 +742,53 @@
754742 }
755743 
756744 #ifdef CONFIG_DYNAMIC_FTRACE
757
-
758
-#ifdef CONFIG_MPROFILE_KERNEL
759
-
760
-#define PACATOC offsetof(struct paca_struct, kernel_toc)
761
-
762
-/*
763
- * For mprofile-kernel we use a special stub for ftrace_caller() because we
764
- * can't rely on r2 containing this module's TOC when we enter the stub.
765
- *
766
- * That can happen if the function calling us didn't need to use the toc. In
767
- * that case it won't have setup r2, and the r2 value will be either the
768
- * kernel's toc, or possibly another modules toc.
769
- *
770
- * To deal with that this stub uses the kernel toc, which is always accessible
771
- * via the paca (in r13). The target (ftrace_caller()) is responsible for
772
- * saving and restoring the toc before returning.
773
- */
774
-static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs,
775
-				struct module *me, unsigned long addr)
745
+int module_trampoline_target(struct module *mod, unsigned long addr,
746
+			     unsigned long *target)
776747 {
777
-	struct ppc64_stub_entry *entry;
778
-	unsigned int i, num_stubs;
779
-	static u32 stub_insns[] = {
780
-		0xe98d0000 | PACATOC, 	/* ld      r12,PACATOC(r13)	*/
781
-		0x3d8c0000,		/* addis   r12,r12,<high>	*/
782
-		0x398c0000, 		/* addi    r12,r12,<low>	*/
783
-		0x7d8903a6, 		/* mtctr   r12			*/
784
-		0x4e800420, 		/* bctr				*/
785
-	};
786
-	long reladdr;
748
+	struct ppc64_stub_entry *stub;
749
+	func_desc_t funcdata;
750
+	u32 magic;
787751 
788
-	num_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*entry);
789
-
790
-	/* Find the next available stub entry */
791
-	entry = (void *)sechdrs[me->arch.stubs_section].sh_addr;
792
-	for (i = 0; i < num_stubs && stub_func_addr(entry->funcdata); i++, entry++);
793
-
794
-	if (i >= num_stubs) {
795
-		pr_err("%s: Unable to find a free slot for ftrace stub.\n", me->name);
796
-		return 0;
752
+	if (!within_module_core(addr, mod)) {
753
+		pr_err("%s: stub %lx not in module %s\n", __func__, addr, mod->name);
754
+		return -EFAULT;
797755 	}
798756 
799
-	memcpy(entry->jump, stub_insns, sizeof(stub_insns));
757
+	stub = (struct ppc64_stub_entry *)addr;
800758 
801
-	/* Stub uses address relative to kernel toc (from the paca) */
802
-	reladdr = addr - kernel_toc_addr();
803
-	if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
804
-		pr_err("%s: Address of %ps out of range of kernel_toc.\n",
805
-							me->name, (void *)addr);
806
-		return 0;
759
+	if (copy_from_kernel_nofault(&magic, &stub->magic,
760
+			sizeof(magic))) {
761
+		pr_err("%s: fault reading magic for stub %lx for %s\n", __func__, addr, mod->name);
762
+		return -EFAULT;
807763 	}
808764 
809
-	entry->jump[1] |= PPC_HA(reladdr);
810
-	entry->jump[2] |= PPC_LO(reladdr);
765
+	if (magic != STUB_MAGIC) {
766
+		pr_err("%s: bad magic for stub %lx for %s\n", __func__, addr, mod->name);
767
+		return -EFAULT;
768
+	}
811769 
812
-	/* Eventhough we don't use funcdata in the stub, it's needed elsewhere. */
813
-	entry->funcdata = func_desc(addr);
814
-	entry->magic = STUB_MAGIC;
770
+	if (copy_from_kernel_nofault(&funcdata, &stub->funcdata,
771
+			sizeof(funcdata))) {
772
+		pr_err("%s: fault reading funcdata for stub %lx for %s\n", __func__, addr, mod->name);
773
+                return -EFAULT;
774
+	}
815775 
816
-	return (unsigned long)entry;
776
+	*target = stub_func_addr(funcdata);
777
+
778
+	return 0;
817779 }
818
-#else
819
-static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs,
820
-				struct module *me, unsigned long addr)
821
-{
822
-	return stub_for_addr(sechdrs, addr, me);
823
-}
824
-#endif
825780 
826781 int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs)
827782 {
828
-	mod->arch.tramp = create_ftrace_stub(sechdrs, mod,
829
-					(unsigned long)ftrace_caller);
783
+	mod->arch.tramp = stub_for_addr(sechdrs,
784
+					(unsigned long)ftrace_caller,
785
+					mod,
786
+					"ftrace_caller");
830787 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
831
-	mod->arch.tramp_regs = create_ftrace_stub(sechdrs, mod,
832
-					(unsigned long)ftrace_regs_caller);
788
+	mod->arch.tramp_regs = stub_for_addr(sechdrs,
789
+					(unsigned long)ftrace_regs_caller,
790
+					mod,
791
+					"ftrace_regs_caller");
833792 	if (!mod->arch.tramp_regs)
834793 		return -ENOENT;
835794 #endif