forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/arch/arm64/kernel/module-plts.c
....@@ -1,41 +1,100 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Copyright (C) 2014-2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
3
- *
4
- * This program is free software; you can redistribute it and/or modify
5
- * it under the terms of the GNU General Public License version 2 as
6
- * published by the Free Software Foundation.
74  */
85 
96 #include <linux/elf.h>
7
+#include <linux/ftrace.h>
108 #include <linux/kernel.h>
119 #include <linux/module.h>
10
+#include <linux/slab.h>
1211 #include <linux/sort.h>
12
+
13
+static struct plt_entry __get_adrp_add_pair(u64 dst, u64 pc,
14
+					    enum aarch64_insn_register reg)
15
+{
16
+	u32 adrp, add;
17
+
18
+	adrp = aarch64_insn_gen_adr(pc, dst, reg, AARCH64_INSN_ADR_TYPE_ADRP);
19
+	add = aarch64_insn_gen_add_sub_imm(reg, reg, dst % SZ_4K,
20
+					   AARCH64_INSN_VARIANT_64BIT,
21
+					   AARCH64_INSN_ADSB_ADD);
22
+
23
+	return (struct plt_entry){ cpu_to_le32(adrp), cpu_to_le32(add) };
24
+}
25
+
26
+struct plt_entry get_plt_entry(u64 dst, void *pc)
27
+{
28
+	struct plt_entry plt;
29
+	static u32 br;
30
+
31
+	if (!br)
32
+		br = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_16,
33
+						 AARCH64_INSN_BRANCH_NOLINK);
34
+
35
+	plt = __get_adrp_add_pair(dst, (u64)pc, AARCH64_INSN_REG_16);
36
+	plt.br = cpu_to_le32(br);
37
+
38
+	return plt;
39
+}
40
+
41
+bool plt_entries_equal(const struct plt_entry *a, const struct plt_entry *b)
42
+{
43
+	u64 p, q;
44
+
45
+	/*
46
+	 * Check whether both entries refer to the same target:
47
+	 * do the cheapest checks first.
48
+	 * If the 'add' or 'br' opcodes are different, then the target
49
+	 * cannot be the same.
50
+	 */
51
+	if (a->add != b->add || a->br != b->br)
52
+		return false;
53
+
54
+	p = ALIGN_DOWN((u64)a, SZ_4K);
55
+	q = ALIGN_DOWN((u64)b, SZ_4K);
56
+
57
+	/*
58
+	 * If the 'adrp' opcodes are the same then we just need to check
59
+	 * that they refer to the same 4k region.
60
+	 */
61
+	if (a->adrp == b->adrp && p == q)
62
+		return true;
63
+
64
+	return (p + aarch64_insn_adrp_get_offset(le32_to_cpu(a->adrp))) ==
65
+	       (q + aarch64_insn_adrp_get_offset(le32_to_cpu(b->adrp)));
66
+}
1367 
1468 static bool in_init(const struct module *mod, void *loc)
1569 {
1670 	return (u64)loc - (u64)mod->init_layout.base < mod->init_layout.size;
1771 }
1872 
19
-u64 module_emit_plt_entry(struct module *mod, void *loc, const Elf64_Rela *rela,
73
+u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs,
74
+			  void *loc, const Elf64_Rela *rela,
2075 			  Elf64_Sym *sym)
2176 {
2277 	struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
2378 							  &mod->arch.init;
24
-	struct plt_entry *plt = (struct plt_entry *)pltsec->plt->sh_addr;
79
+	struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr;
2580 	int i = pltsec->plt_num_entries;
81
+	int j = i - 1;
2682 	u64 val = sym->st_value + rela->r_addend;
2783 
28
-	plt[i] = get_plt_entry(val);
84
+	if (is_forbidden_offset_for_adrp(&plt[i].adrp))
85
+		i++;
86
+
87
+	plt[i] = get_plt_entry(val, &plt[i]);
2988 
3089 	/*
3190 	 * Check if the entry we just created is a duplicate. Given that the
3291 	 * relocations are sorted, this will be the last entry we allocated.
3392 	 * (if one exists).
3493 	 */
35
-	if (i > 0 && plt_entries_equal(plt + i, plt + i - 1))
36
-		return (u64)&plt[i - 1];
94
+	if (j >= 0 && plt_entries_equal(plt + i, plt + j))
95
+		return (u64)&plt[j];
3796 
38
-	pltsec->plt_num_entries++;
97
+	pltsec->plt_num_entries += i - j;
3998 	if (WARN_ON(pltsec->plt_num_entries > pltsec->plt_max_entries))
4099 		return 0;
41100 
....@@ -43,41 +102,31 @@
43102 }
44103 
45104 #ifdef CONFIG_ARM64_ERRATUM_843419
46
-u64 module_emit_veneer_for_adrp(struct module *mod, void *loc, u64 val)
105
+u64 module_emit_veneer_for_adrp(struct module *mod, Elf64_Shdr *sechdrs,
106
+				void *loc, u64 val)
47107 {
48108 	struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
49109 							  &mod->arch.init;
50
-	struct plt_entry *plt = (struct plt_entry *)pltsec->plt->sh_addr;
110
+	struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr;
51111 	int i = pltsec->plt_num_entries++;
52
-	u32 mov0, mov1, mov2, br;
112
+	u32 br;
53113 	int rd;
54114 
55115 	if (WARN_ON(pltsec->plt_num_entries > pltsec->plt_max_entries))
56116 		return 0;
57117 
118
+	if (is_forbidden_offset_for_adrp(&plt[i].adrp))
119
+		i = pltsec->plt_num_entries++;
120
+
58121 	/* get the destination register of the ADRP instruction */
59122 	rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD,
60123 					  le32_to_cpup((__le32 *)loc));
61124 
62
-	/* generate the veneer instructions */
63
-	mov0 = aarch64_insn_gen_movewide(rd, (u16)~val, 0,
64
-					 AARCH64_INSN_VARIANT_64BIT,
65
-					 AARCH64_INSN_MOVEWIDE_INVERSE);
66
-	mov1 = aarch64_insn_gen_movewide(rd, (u16)(val >> 16), 16,
67
-					 AARCH64_INSN_VARIANT_64BIT,
68
-					 AARCH64_INSN_MOVEWIDE_KEEP);
69
-	mov2 = aarch64_insn_gen_movewide(rd, (u16)(val >> 32), 32,
70
-					 AARCH64_INSN_VARIANT_64BIT,
71
-					 AARCH64_INSN_MOVEWIDE_KEEP);
72125 	br = aarch64_insn_gen_branch_imm((u64)&plt[i].br, (u64)loc + 4,
73126 					 AARCH64_INSN_BRANCH_NOLINK);
74127 
75
-	plt[i] = (struct plt_entry){
76
-			cpu_to_le32(mov0),
77
-			cpu_to_le32(mov1),
78
-			cpu_to_le32(mov2),
79
-			cpu_to_le32(br)
80
-		};
128
+	plt[i] = __get_adrp_add_pair(val, (u64)&plt[i], rd);
129
+	plt[i].br = cpu_to_le32(br);
81130 
82131 	return (u64)&plt[i];
83132 }
....@@ -193,16 +242,60 @@
193242 			break;
194243 		}
195244 	}
245
+
246
+	if (IS_ENABLED(CONFIG_ARM64_ERRATUM_843419) &&
247
+	    cpus_have_const_cap(ARM64_WORKAROUND_843419))
248
+		/*
249
+		 * Add some slack so we can skip PLT slots that may trigger
250
+		 * the erratum due to the placement of the ADRP instruction.
251
+		 */
252
+		ret += DIV_ROUND_UP(ret, (SZ_4K / sizeof(struct plt_entry)));
253
+
196254 	return ret;
255
+}
256
+
257
+static bool branch_rela_needs_plt(Elf64_Sym *syms, Elf64_Rela *rela,
258
+				  Elf64_Word dstidx)
259
+{
260
+
261
+	Elf64_Sym *s = syms + ELF64_R_SYM(rela->r_info);
262
+
263
+	if (s->st_shndx == dstidx)
264
+		return false;
265
+
266
+	return ELF64_R_TYPE(rela->r_info) == R_AARCH64_JUMP26 ||
267
+	       ELF64_R_TYPE(rela->r_info) == R_AARCH64_CALL26;
268
+}
269
+
270
+/* Group branch PLT relas at the front end of the array. */
271
+static int partition_branch_plt_relas(Elf64_Sym *syms, Elf64_Rela *rela,
272
+				      int numrels, Elf64_Word dstidx)
273
+{
274
+	int i = 0, j = numrels - 1;
275
+
276
+	if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE))
277
+		return 0;
278
+
279
+	while (i < j) {
280
+		if (branch_rela_needs_plt(syms, &rela[i], dstidx))
281
+			i++;
282
+		else if (branch_rela_needs_plt(syms, &rela[j], dstidx))
283
+			swap(rela[i], rela[j]);
284
+		else
285
+			j--;
286
+	}
287
+
288
+	return i;
197289 }
198290 
199291 int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
200292 			      char *secstrings, struct module *mod)
201293 {
294
+	bool copy_rela_for_fips140 = false;
202295 	unsigned long core_plts = 0;
203296 	unsigned long init_plts = 0;
204297 	Elf64_Sym *syms = NULL;
205
-	Elf_Shdr *tramp = NULL;
298
+	Elf_Shdr *pltsec, *tramp = NULL;
206299 	int i;
207300 
208301 	/*
....@@ -211,18 +304,17 @@
211304 	 */
212305 	for (i = 0; i < ehdr->e_shnum; i++) {
213306 		if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt"))
214
-			mod->arch.core.plt = sechdrs + i;
307
+			mod->arch.core.plt_shndx = i;
215308 		else if (!strcmp(secstrings + sechdrs[i].sh_name, ".init.plt"))
216
-			mod->arch.init.plt = sechdrs + i;
217
-		else if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE) &&
218
-			 !strcmp(secstrings + sechdrs[i].sh_name,
309
+			mod->arch.init.plt_shndx = i;
310
+		else if (!strcmp(secstrings + sechdrs[i].sh_name,
219311 				 ".text.ftrace_trampoline"))
220312 			tramp = sechdrs + i;
221313 		else if (sechdrs[i].sh_type == SHT_SYMTAB)
222314 			syms = (Elf64_Sym *)sechdrs[i].sh_addr;
223315 	}
224316 
225
-	if (!mod->arch.core.plt || !mod->arch.init.plt) {
317
+	if (!mod->arch.core.plt_shndx || !mod->arch.init.plt_shndx) {
226318 		pr_err("%s: module PLT section(s) missing\n", mod->name);
227319 		return -ENOEXEC;
228320 	}
....@@ -231,22 +323,60 @@
231323 		return -ENOEXEC;
232324 	}
233325 
326
+	if (IS_ENABLED(CONFIG_CRYPTO_FIPS140) &&
327
+	    !strcmp(mod->name, "fips140"))
328
+		copy_rela_for_fips140 = true;
329
+
234330 	for (i = 0; i < ehdr->e_shnum; i++) {
235331 		Elf64_Rela *rels = (void *)ehdr + sechdrs[i].sh_offset;
236
-		int numrels = sechdrs[i].sh_size / sizeof(Elf64_Rela);
332
+		int nents, numrels = sechdrs[i].sh_size / sizeof(Elf64_Rela);
237333 		Elf64_Shdr *dstsec = sechdrs + sechdrs[i].sh_info;
238334 
239335 		if (sechdrs[i].sh_type != SHT_RELA)
240336 			continue;
241337 
338
+#ifdef CONFIG_CRYPTO_FIPS140
339
+		if (copy_rela_for_fips140 &&
340
+		    !strcmp(secstrings + dstsec->sh_name, ".rodata")) {
341
+			void *p = kmemdup(rels, numrels * sizeof(Elf64_Rela),
342
+					  GFP_KERNEL);
343
+			if (!p) {
344
+				pr_err("fips140: failed to allocate .rodata RELA buffer\n");
345
+				return -ENOMEM;
346
+			}
347
+			mod->arch.rodata_relocations = p;
348
+			mod->arch.num_rodata_relocations = numrels;
349
+		}
350
+#endif
351
+
242352 		/* ignore relocations that operate on non-exec sections */
243353 		if (!(dstsec->sh_flags & SHF_EXECINSTR))
244354 			continue;
245355 
246
-		/* sort by type, symbol index and addend */
247
-		sort(rels, numrels, sizeof(Elf64_Rela), cmp_rela, NULL);
356
+#ifdef CONFIG_CRYPTO_FIPS140
357
+		if (copy_rela_for_fips140 &&
358
+		    !strcmp(secstrings + dstsec->sh_name, ".text")) {
359
+			void *p = kmemdup(rels, numrels * sizeof(Elf64_Rela),
360
+					  GFP_KERNEL);
361
+			if (!p) {
362
+				pr_err("fips140: failed to allocate .text RELA buffer\n");
363
+				return -ENOMEM;
364
+			}
365
+			mod->arch.text_relocations = p;
366
+			mod->arch.num_text_relocations = numrels;
367
+		}
368
+#endif
248369 
249
-		if (strncmp(secstrings + dstsec->sh_name, ".init", 5) != 0)
370
+		/*
371
+		 * sort branch relocations requiring a PLT by type, symbol index
372
+		 * and addend
373
+		 */
374
+		nents = partition_branch_plt_relas(syms, rels, numrels,
375
+						   sechdrs[i].sh_info);
376
+		if (nents)
377
+			sort(rels, nents, sizeof(Elf64_Rela), cmp_rela, NULL);
378
+
379
+		if (!str_has_prefix(secstrings + dstsec->sh_name, ".init"))
250380 			core_plts += count_plts(syms, rels, numrels,
251381 						sechdrs[i].sh_info, dstsec);
252382 		else
....@@ -254,17 +384,19 @@
254384 						sechdrs[i].sh_info, dstsec);
255385 	}
256386 
257
-	mod->arch.core.plt->sh_type = SHT_NOBITS;
258
-	mod->arch.core.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
259
-	mod->arch.core.plt->sh_addralign = L1_CACHE_BYTES;
260
-	mod->arch.core.plt->sh_size = (core_plts  + 1) * sizeof(struct plt_entry);
387
+	pltsec = sechdrs + mod->arch.core.plt_shndx;
388
+	pltsec->sh_type = SHT_NOBITS;
389
+	pltsec->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
390
+	pltsec->sh_addralign = L1_CACHE_BYTES;
391
+	pltsec->sh_size = (core_plts  + 1) * sizeof(struct plt_entry);
261392 	mod->arch.core.plt_num_entries = 0;
262393 	mod->arch.core.plt_max_entries = core_plts;
263394 
264
-	mod->arch.init.plt->sh_type = SHT_NOBITS;
265
-	mod->arch.init.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
266
-	mod->arch.init.plt->sh_addralign = L1_CACHE_BYTES;
267
-	mod->arch.init.plt->sh_size = (init_plts + 1) * sizeof(struct plt_entry);
395
+	pltsec = sechdrs + mod->arch.init.plt_shndx;
396
+	pltsec->sh_type = SHT_NOBITS;
397
+	pltsec->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
398
+	pltsec->sh_addralign = L1_CACHE_BYTES;
399
+	pltsec->sh_size = (init_plts + 1) * sizeof(struct plt_entry);
268400 	mod->arch.init.plt_num_entries = 0;
269401 	mod->arch.init.plt_max_entries = init_plts;
270402 
....@@ -272,7 +404,7 @@
272404 		tramp->sh_type = SHT_NOBITS;
273405 		tramp->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
274406 		tramp->sh_addralign = __alignof__(struct plt_entry);
275
-		tramp->sh_size = sizeof(struct plt_entry);
407
+		tramp->sh_size = NR_FTRACE_PLTS * sizeof(struct plt_entry);
276408 	}
277409 
278410 	return 0;