forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/arch/arm64/kernel/module-plts.c
....@@ -1,41 +1,101 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Copyright (C) 2014-2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
3
- *
4
- * This program is free software; you can redistribute it and/or modify
5
- * it under the terms of the GNU General Public License version 2 as
6
- * published by the Free Software Foundation.
74  */
85 
96 #include <linux/elf.h>
7
+#include <linux/ftrace.h>
108 #include <linux/kernel.h>
119 #include <linux/module.h>
10
+#include <linux/moduleloader.h>
11
+#include <linux/slab.h>
1212 #include <linux/sort.h>
13
+
14
+static struct plt_entry __get_adrp_add_pair(u64 dst, u64 pc,
15
+					    enum aarch64_insn_register reg)
16
+{
17
+	u32 adrp, add;
18
+
19
+	adrp = aarch64_insn_gen_adr(pc, dst, reg, AARCH64_INSN_ADR_TYPE_ADRP);
20
+	add = aarch64_insn_gen_add_sub_imm(reg, reg, dst % SZ_4K,
21
+					   AARCH64_INSN_VARIANT_64BIT,
22
+					   AARCH64_INSN_ADSB_ADD);
23
+
24
+	return (struct plt_entry){ cpu_to_le32(adrp), cpu_to_le32(add) };
25
+}
26
+
27
+struct plt_entry get_plt_entry(u64 dst, void *pc)
28
+{
29
+	struct plt_entry plt;
30
+	static u32 br;
31
+
32
+	if (!br)
33
+		br = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_16,
34
+						 AARCH64_INSN_BRANCH_NOLINK);
35
+
36
+	plt = __get_adrp_add_pair(dst, (u64)pc, AARCH64_INSN_REG_16);
37
+	plt.br = cpu_to_le32(br);
38
+
39
+	return plt;
40
+}
41
+
42
+bool plt_entries_equal(const struct plt_entry *a, const struct plt_entry *b)
43
+{
44
+	u64 p, q;
45
+
46
+	/*
47
+	 * Check whether both entries refer to the same target:
48
+	 * do the cheapest checks first.
49
+	 * If the 'add' or 'br' opcodes are different, then the target
50
+	 * cannot be the same.
51
+	 */
52
+	if (a->add != b->add || a->br != b->br)
53
+		return false;
54
+
55
+	p = ALIGN_DOWN((u64)a, SZ_4K);
56
+	q = ALIGN_DOWN((u64)b, SZ_4K);
57
+
58
+	/*
59
+	 * If the 'adrp' opcodes are the same then we just need to check
60
+	 * that they refer to the same 4k region.
61
+	 */
62
+	if (a->adrp == b->adrp && p == q)
63
+		return true;
64
+
65
+	return (p + aarch64_insn_adrp_get_offset(le32_to_cpu(a->adrp))) ==
66
+	       (q + aarch64_insn_adrp_get_offset(le32_to_cpu(b->adrp)));
67
+}
1368 
1469 static bool in_init(const struct module *mod, void *loc)
1570 {
1671 	return (u64)loc - (u64)mod->init_layout.base < mod->init_layout.size;
1772 }
1873 
19
-u64 module_emit_plt_entry(struct module *mod, void *loc, const Elf64_Rela *rela,
74
+u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs,
75
+			  void *loc, const Elf64_Rela *rela,
2076 			  Elf64_Sym *sym)
2177 {
2278 	struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
2379 							  &mod->arch.init;
24
-	struct plt_entry *plt = (struct plt_entry *)pltsec->plt->sh_addr;
80
+	struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr;
2581 	int i = pltsec->plt_num_entries;
82
+	int j = i - 1;
2683 	u64 val = sym->st_value + rela->r_addend;
2784 
28
-	plt[i] = get_plt_entry(val);
85
+	if (is_forbidden_offset_for_adrp(&plt[i].adrp))
86
+		i++;
87
+
88
+	plt[i] = get_plt_entry(val, &plt[i]);
2989 
3090 	/*
3191 	 * Check if the entry we just created is a duplicate. Given that the
3292 	 * relocations are sorted, this will be the last entry we allocated.
3393 	 * (if one exists).
3494 	 */
35
-	if (i > 0 && plt_entries_equal(plt + i, plt + i - 1))
36
-		return (u64)&plt[i - 1];
95
+	if (j >= 0 && plt_entries_equal(plt + i, plt + j))
96
+		return (u64)&plt[j];
3797 
38
-	pltsec->plt_num_entries++;
98
+	pltsec->plt_num_entries += i - j;
3999 	if (WARN_ON(pltsec->plt_num_entries > pltsec->plt_max_entries))
40100 		return 0;
41101 
....@@ -43,41 +103,31 @@
43103 }
44104 
45105 #ifdef CONFIG_ARM64_ERRATUM_843419
46
-u64 module_emit_veneer_for_adrp(struct module *mod, void *loc, u64 val)
106
+u64 module_emit_veneer_for_adrp(struct module *mod, Elf64_Shdr *sechdrs,
107
+				void *loc, u64 val)
47108 {
48109 	struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
49110 							  &mod->arch.init;
50
-	struct plt_entry *plt = (struct plt_entry *)pltsec->plt->sh_addr;
111
+	struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr;
51112 	int i = pltsec->plt_num_entries++;
52
-	u32 mov0, mov1, mov2, br;
113
+	u32 br;
53114 	int rd;
54115 
55116 	if (WARN_ON(pltsec->plt_num_entries > pltsec->plt_max_entries))
56117 		return 0;
57118 
119
+	if (is_forbidden_offset_for_adrp(&plt[i].adrp))
120
+		i = pltsec->plt_num_entries++;
121
+
58122 	/* get the destination register of the ADRP instruction */
59123 	rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD,
60124 					  le32_to_cpup((__le32 *)loc));
61125 
62
-	/* generate the veneer instructions */
63
-	mov0 = aarch64_insn_gen_movewide(rd, (u16)~val, 0,
64
-					 AARCH64_INSN_VARIANT_64BIT,
65
-					 AARCH64_INSN_MOVEWIDE_INVERSE);
66
-	mov1 = aarch64_insn_gen_movewide(rd, (u16)(val >> 16), 16,
67
-					 AARCH64_INSN_VARIANT_64BIT,
68
-					 AARCH64_INSN_MOVEWIDE_KEEP);
69
-	mov2 = aarch64_insn_gen_movewide(rd, (u16)(val >> 32), 32,
70
-					 AARCH64_INSN_VARIANT_64BIT,
71
-					 AARCH64_INSN_MOVEWIDE_KEEP);
72126 	br = aarch64_insn_gen_branch_imm((u64)&plt[i].br, (u64)loc + 4,
73127 					 AARCH64_INSN_BRANCH_NOLINK);
74128 
75
-	plt[i] = (struct plt_entry){
76
-			cpu_to_le32(mov0),
77
-			cpu_to_le32(mov1),
78
-			cpu_to_le32(mov2),
79
-			cpu_to_le32(br)
80
-		};
129
+	plt[i] = __get_adrp_add_pair(val, (u64)&plt[i], rd);
130
+	plt[i].br = cpu_to_le32(br);
81131 
82132 	return (u64)&plt[i];
83133 }
....@@ -193,16 +243,60 @@
193243 			break;
194244 		}
195245 	}
246
+
247
+	if (IS_ENABLED(CONFIG_ARM64_ERRATUM_843419) &&
248
+	    cpus_have_const_cap(ARM64_WORKAROUND_843419))
249
+		/*
250
+		 * Add some slack so we can skip PLT slots that may trigger
251
+		 * the erratum due to the placement of the ADRP instruction.
252
+		 */
253
+		ret += DIV_ROUND_UP(ret, (SZ_4K / sizeof(struct plt_entry)));
254
+
196255 	return ret;
256
+}
257
+
258
+static bool branch_rela_needs_plt(Elf64_Sym *syms, Elf64_Rela *rela,
259
+				  Elf64_Word dstidx)
260
+{
261
+
262
+	Elf64_Sym *s = syms + ELF64_R_SYM(rela->r_info);
263
+
264
+	if (s->st_shndx == dstidx)
265
+		return false;
266
+
267
+	return ELF64_R_TYPE(rela->r_info) == R_AARCH64_JUMP26 ||
268
+	       ELF64_R_TYPE(rela->r_info) == R_AARCH64_CALL26;
269
+}
270
+
271
+/* Group branch PLT relas at the front end of the array. */
272
+static int partition_branch_plt_relas(Elf64_Sym *syms, Elf64_Rela *rela,
273
+				      int numrels, Elf64_Word dstidx)
274
+{
275
+	int i = 0, j = numrels - 1;
276
+
277
+	if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE))
278
+		return 0;
279
+
280
+	while (i < j) {
281
+		if (branch_rela_needs_plt(syms, &rela[i], dstidx))
282
+			i++;
283
+		else if (branch_rela_needs_plt(syms, &rela[j], dstidx))
284
+			swap(rela[i], rela[j]);
285
+		else
286
+			j--;
287
+	}
288
+
289
+	return i;
197290 }
198291 
199292 int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
200293 			      char *secstrings, struct module *mod)
201294 {
295
+	bool copy_rela_for_fips140 = false;
202296 	unsigned long core_plts = 0;
203297 	unsigned long init_plts = 0;
204298 	Elf64_Sym *syms = NULL;
205
-	Elf_Shdr *tramp = NULL;
299
+	Elf_Shdr *pltsec, *tramp = NULL;
206300 	int i;
207301 
208302 	/*
....@@ -211,18 +305,17 @@
211305 	 */
212306 	for (i = 0; i < ehdr->e_shnum; i++) {
213307 		if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt"))
214
-			mod->arch.core.plt = sechdrs + i;
308
+			mod->arch.core.plt_shndx = i;
215309 		else if (!strcmp(secstrings + sechdrs[i].sh_name, ".init.plt"))
216
-			mod->arch.init.plt = sechdrs + i;
217
-		else if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE) &&
218
-			 !strcmp(secstrings + sechdrs[i].sh_name,
310
+			mod->arch.init.plt_shndx = i;
311
+		else if (!strcmp(secstrings + sechdrs[i].sh_name,
219312 				 ".text.ftrace_trampoline"))
220313 			tramp = sechdrs + i;
221314 		else if (sechdrs[i].sh_type == SHT_SYMTAB)
222315 			syms = (Elf64_Sym *)sechdrs[i].sh_addr;
223316 	}
224317 
225
-	if (!mod->arch.core.plt || !mod->arch.init.plt) {
318
+	if (!mod->arch.core.plt_shndx || !mod->arch.init.plt_shndx) {
226319 		pr_err("%s: module PLT section(s) missing\n", mod->name);
227320 		return -ENOEXEC;
228321 	}
....@@ -231,22 +324,60 @@
231324 		return -ENOEXEC;
232325 	}
233326 
327
+	if (IS_ENABLED(CONFIG_CRYPTO_FIPS140) &&
328
+	    !strcmp(mod->name, "fips140"))
329
+		copy_rela_for_fips140 = true;
330
+
234331 	for (i = 0; i < ehdr->e_shnum; i++) {
235332 		Elf64_Rela *rels = (void *)ehdr + sechdrs[i].sh_offset;
236
-		int numrels = sechdrs[i].sh_size / sizeof(Elf64_Rela);
333
+		int nents, numrels = sechdrs[i].sh_size / sizeof(Elf64_Rela);
237334 		Elf64_Shdr *dstsec = sechdrs + sechdrs[i].sh_info;
238335 
239336 		if (sechdrs[i].sh_type != SHT_RELA)
240337 			continue;
241338 
339
+#ifdef CONFIG_CRYPTO_FIPS140
340
+		if (copy_rela_for_fips140 &&
341
+		    !strcmp(secstrings + dstsec->sh_name, ".rodata")) {
342
+			void *p = kmemdup(rels, numrels * sizeof(Elf64_Rela),
343
+					  GFP_KERNEL);
344
+			if (!p) {
345
+				pr_err("fips140: failed to allocate .rodata RELA buffer\n");
346
+				return -ENOMEM;
347
+			}
348
+			mod->arch.rodata_relocations = p;
349
+			mod->arch.num_rodata_relocations = numrels;
350
+		}
351
+#endif
352
+
242353 		/* ignore relocations that operate on non-exec sections */
243354 		if (!(dstsec->sh_flags & SHF_EXECINSTR))
244355 			continue;
245356 
246
-		/* sort by type, symbol index and addend */
247
-		sort(rels, numrels, sizeof(Elf64_Rela), cmp_rela, NULL);
357
+#ifdef CONFIG_CRYPTO_FIPS140
358
+		if (copy_rela_for_fips140 &&
359
+		    !strcmp(secstrings + dstsec->sh_name, ".text")) {
360
+			void *p = kmemdup(rels, numrels * sizeof(Elf64_Rela),
361
+					  GFP_KERNEL);
362
+			if (!p) {
363
+				pr_err("fips140: failed to allocate .text RELA buffer\n");
364
+				return -ENOMEM;
365
+			}
366
+			mod->arch.text_relocations = p;
367
+			mod->arch.num_text_relocations = numrels;
368
+		}
369
+#endif
248370 
249
-		if (strncmp(secstrings + dstsec->sh_name, ".init", 5) != 0)
371
+		/*
372
+		 * sort branch relocations requiring a PLT by type, symbol index
373
+		 * and addend
374
+		 */
375
+		nents = partition_branch_plt_relas(syms, rels, numrels,
376
+						   sechdrs[i].sh_info);
377
+		if (nents)
378
+			sort(rels, nents, sizeof(Elf64_Rela), cmp_rela, NULL);
379
+
380
+		if (!module_init_layout_section(secstrings + dstsec->sh_name))
250381 			core_plts += count_plts(syms, rels, numrels,
251382 						sechdrs[i].sh_info, dstsec);
252383 		else
....@@ -254,17 +385,19 @@
254385 						sechdrs[i].sh_info, dstsec);
255386 	}
256387 
257
-	mod->arch.core.plt->sh_type = SHT_NOBITS;
258
-	mod->arch.core.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
259
-	mod->arch.core.plt->sh_addralign = L1_CACHE_BYTES;
260
-	mod->arch.core.plt->sh_size = (core_plts  + 1) * sizeof(struct plt_entry);
388
+	pltsec = sechdrs + mod->arch.core.plt_shndx;
389
+	pltsec->sh_type = SHT_NOBITS;
390
+	pltsec->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
391
+	pltsec->sh_addralign = L1_CACHE_BYTES;
392
+	pltsec->sh_size = (core_plts  + 1) * sizeof(struct plt_entry);
261393 	mod->arch.core.plt_num_entries = 0;
262394 	mod->arch.core.plt_max_entries = core_plts;
263395 
264
-	mod->arch.init.plt->sh_type = SHT_NOBITS;
265
-	mod->arch.init.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
266
-	mod->arch.init.plt->sh_addralign = L1_CACHE_BYTES;
267
-	mod->arch.init.plt->sh_size = (init_plts + 1) * sizeof(struct plt_entry);
396
+	pltsec = sechdrs + mod->arch.init.plt_shndx;
397
+	pltsec->sh_type = SHT_NOBITS;
398
+	pltsec->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
399
+	pltsec->sh_addralign = L1_CACHE_BYTES;
400
+	pltsec->sh_size = (init_plts + 1) * sizeof(struct plt_entry);
268401 	mod->arch.init.plt_num_entries = 0;
269402 	mod->arch.init.plt_max_entries = init_plts;
270403 
....@@ -272,7 +405,7 @@
272405 		tramp->sh_type = SHT_NOBITS;
273406 		tramp->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
274407 		tramp->sh_addralign = __alignof__(struct plt_entry);
275
-		tramp->sh_size = sizeof(struct plt_entry);
408
+		tramp->sh_size = NR_FTRACE_PLTS * sizeof(struct plt_entry);
276409 	}
277410 
278411 	return 0;