add ax88772_rst

hc

2024-05-10 9999e48639b3cecb08ffb37358bcba3b48161b29

 kernel/arch/arm64/kernel/module-plts.c
..
..
@@ -1,41 +1,101 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
1
2
 /*
2
3
  * Copyright (C) 2014-2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
3
- *
4
- * This program is free software; you can redistribute it and/or modify
5
- * it under the terms of the GNU General Public License version 2 as
6
- * published by the Free Software Foundation.
7
4
  */
8
5
 
9
6
 #include <linux/elf.h>
7
+#include <linux/ftrace.h>
10
8
 #include <linux/kernel.h>
11
9
 #include <linux/module.h>
10
+#include <linux/moduleloader.h>
11
+#include <linux/slab.h>
12
12
 #include <linux/sort.h>
13
+
14
+static struct plt_entry __get_adrp_add_pair(u64 dst, u64 pc,
15
+					    enum aarch64_insn_register reg)
16
+{
17
+	u32 adrp, add;
18
+
19
+	adrp = aarch64_insn_gen_adr(pc, dst, reg, AARCH64_INSN_ADR_TYPE_ADRP);
20
+	add = aarch64_insn_gen_add_sub_imm(reg, reg, dst % SZ_4K,
21
+					   AARCH64_INSN_VARIANT_64BIT,
22
+					   AARCH64_INSN_ADSB_ADD);
23
+
24
+	return (struct plt_entry){ cpu_to_le32(adrp), cpu_to_le32(add) };
25
+}
26
+
27
+struct plt_entry get_plt_entry(u64 dst, void *pc)
28
+{
29
+	struct plt_entry plt;
30
+	static u32 br;
31
+
32
+	if (!br)
33
+		br = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_16,
34
+						 AARCH64_INSN_BRANCH_NOLINK);
35
+
36
+	plt = __get_adrp_add_pair(dst, (u64)pc, AARCH64_INSN_REG_16);
37
+	plt.br = cpu_to_le32(br);
38
+
39
+	return plt;
40
+}
41
+
42
+bool plt_entries_equal(const struct plt_entry *a, const struct plt_entry *b)
43
+{
44
+	u64 p, q;
45
+
46
+	/*
47
+	 * Check whether both entries refer to the same target:
48
+	 * do the cheapest checks first.
49
+	 * If the 'add' or 'br' opcodes are different, then the target
50
+	 * cannot be the same.
51
+	 */
52
+	if (a->add != b->add || a->br != b->br)
53
+		return false;
54
+
55
+	p = ALIGN_DOWN((u64)a, SZ_4K);
56
+	q = ALIGN_DOWN((u64)b, SZ_4K);
57
+
58
+	/*
59
+	 * If the 'adrp' opcodes are the same then we just need to check
60
+	 * that they refer to the same 4k region.
61
+	 */
62
+	if (a->adrp == b->adrp && p == q)
63
+		return true;
64
+
65
+	return (p + aarch64_insn_adrp_get_offset(le32_to_cpu(a->adrp))) ==
66
+	       (q + aarch64_insn_adrp_get_offset(le32_to_cpu(b->adrp)));
67
+}
13
68
 
14
69
 static bool in_init(const struct module *mod, void *loc)
15
70
 {
16
71
 	return (u64)loc - (u64)mod->init_layout.base < mod->init_layout.size;
17
72
 }
18
73
 
19
-u64 module_emit_plt_entry(struct module *mod, void *loc, const Elf64_Rela *rela,
74
+u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs,
75
+			  void *loc, const Elf64_Rela *rela,
20
76
 			  Elf64_Sym *sym)
21
77
 {
22
78
 	struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
23
79
 							  &mod->arch.init;
24
-	struct plt_entry *plt = (struct plt_entry *)pltsec->plt->sh_addr;
80
+	struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr;
25
81
 	int i = pltsec->plt_num_entries;
82
+	int j = i - 1;
26
83
 	u64 val = sym->st_value + rela->r_addend;
27
84
 
28
-	plt[i] = get_plt_entry(val);
85
+	if (is_forbidden_offset_for_adrp(&plt[i].adrp))
86
+		i++;
87
+
88
+	plt[i] = get_plt_entry(val, &plt[i]);
29
89
 
30
90
 	/*
31
91
 	 * Check if the entry we just created is a duplicate. Given that the
32
92
 	 * relocations are sorted, this will be the last entry we allocated.
33
93
 	 * (if one exists).
34
94
 	 */
35
-	if (i > 0 && plt_entries_equal(plt + i, plt + i - 1))
36
-		return (u64)&plt[i - 1];
95
+	if (j >= 0 && plt_entries_equal(plt + i, plt + j))
96
+		return (u64)&plt[j];
37
97
 
38
-	pltsec->plt_num_entries++;
98
+	pltsec->plt_num_entries += i - j;
39
99
 	if (WARN_ON(pltsec->plt_num_entries > pltsec->plt_max_entries))
40
100
 		return 0;
41
101
 
..
..
@@ -43,41 +103,31 @@
43
103
 }
44
104
 
45
105
 #ifdef CONFIG_ARM64_ERRATUM_843419
46
-u64 module_emit_veneer_for_adrp(struct module *mod, void *loc, u64 val)
106
+u64 module_emit_veneer_for_adrp(struct module *mod, Elf64_Shdr *sechdrs,
107
+				void *loc, u64 val)
47
108
 {
48
109
 	struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
49
110
 							  &mod->arch.init;
50
-	struct plt_entry *plt = (struct plt_entry *)pltsec->plt->sh_addr;
111
+	struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr;
51
112
 	int i = pltsec->plt_num_entries++;
52
-	u32 mov0, mov1, mov2, br;
113
+	u32 br;
53
114
 	int rd;
54
115
 
55
116
 	if (WARN_ON(pltsec->plt_num_entries > pltsec->plt_max_entries))
56
117
 		return 0;
57
118
 
119
+	if (is_forbidden_offset_for_adrp(&plt[i].adrp))
120
+		i = pltsec->plt_num_entries++;
121
+
58
122
 	/* get the destination register of the ADRP instruction */
59
123
 	rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD,
60
124
 					  le32_to_cpup((__le32 *)loc));
61
125
 
62
-	/* generate the veneer instructions */
63
-	mov0 = aarch64_insn_gen_movewide(rd, (u16)~val, 0,
64
-					 AARCH64_INSN_VARIANT_64BIT,
65
-					 AARCH64_INSN_MOVEWIDE_INVERSE);
66
-	mov1 = aarch64_insn_gen_movewide(rd, (u16)(val >> 16), 16,
67
-					 AARCH64_INSN_VARIANT_64BIT,
68
-					 AARCH64_INSN_MOVEWIDE_KEEP);
69
-	mov2 = aarch64_insn_gen_movewide(rd, (u16)(val >> 32), 32,
70
-					 AARCH64_INSN_VARIANT_64BIT,
71
-					 AARCH64_INSN_MOVEWIDE_KEEP);
72
126
 	br = aarch64_insn_gen_branch_imm((u64)&plt[i].br, (u64)loc + 4,
73
127
 					 AARCH64_INSN_BRANCH_NOLINK);
74
128
 
75
-	plt[i] = (struct plt_entry){
76
-			cpu_to_le32(mov0),
77
-			cpu_to_le32(mov1),
78
-			cpu_to_le32(mov2),
79
-			cpu_to_le32(br)
80
-		};
129
+	plt[i] = __get_adrp_add_pair(val, (u64)&plt[i], rd);
130
+	plt[i].br = cpu_to_le32(br);
81
131
 
82
132
 	return (u64)&plt[i];
83
133
 }
..
..
@@ -193,16 +243,60 @@
193
243
 			break;
194
244
 		}
195
245
 	}
246
+
247
+	if (IS_ENABLED(CONFIG_ARM64_ERRATUM_843419) &&
248
+	    cpus_have_const_cap(ARM64_WORKAROUND_843419))
249
+		/*
250
+		 * Add some slack so we can skip PLT slots that may trigger
251
+		 * the erratum due to the placement of the ADRP instruction.
252
+		 */
253
+		ret += DIV_ROUND_UP(ret, (SZ_4K / sizeof(struct plt_entry)));
254
+
196
255
 	return ret;
256
+}
257
+
258
+static bool branch_rela_needs_plt(Elf64_Sym *syms, Elf64_Rela *rela,
259
+				  Elf64_Word dstidx)
260
+{
261
+
262
+	Elf64_Sym *s = syms + ELF64_R_SYM(rela->r_info);
263
+
264
+	if (s->st_shndx == dstidx)
265
+		return false;
266
+
267
+	return ELF64_R_TYPE(rela->r_info) == R_AARCH64_JUMP26 ||
268
+	       ELF64_R_TYPE(rela->r_info) == R_AARCH64_CALL26;
269
+}
270
+
271
+/* Group branch PLT relas at the front end of the array. */
272
+static int partition_branch_plt_relas(Elf64_Sym *syms, Elf64_Rela *rela,
273
+				      int numrels, Elf64_Word dstidx)
274
+{
275
+	int i = 0, j = numrels - 1;
276
+
277
+	if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE))
278
+		return 0;
279
+
280
+	while (i < j) {
281
+		if (branch_rela_needs_plt(syms, &rela[i], dstidx))
282
+			i++;
283
+		else if (branch_rela_needs_plt(syms, &rela[j], dstidx))
284
+			swap(rela[i], rela[j]);
285
+		else
286
+			j--;
287
+	}
288
+
289
+	return i;
197
290
 }
198
291
 
199
292
 int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
200
293
 			      char *secstrings, struct module *mod)
201
294
 {
295
+	bool copy_rela_for_fips140 = false;
202
296
 	unsigned long core_plts = 0;
203
297
 	unsigned long init_plts = 0;
204
298
 	Elf64_Sym *syms = NULL;
205
-	Elf_Shdr *tramp = NULL;
299
+	Elf_Shdr *pltsec, *tramp = NULL;
206
300
 	int i;
207
301
 
208
302
 	/*
..
..
@@ -211,18 +305,17 @@
211
305
 	 */
212
306
 	for (i = 0; i < ehdr->e_shnum; i++) {
213
307
 		if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt"))
214
-			mod->arch.core.plt = sechdrs + i;
308
+			mod->arch.core.plt_shndx = i;
215
309
 		else if (!strcmp(secstrings + sechdrs[i].sh_name, ".init.plt"))
216
-			mod->arch.init.plt = sechdrs + i;
217
-		else if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE) &&
218
-			 !strcmp(secstrings + sechdrs[i].sh_name,
310
+			mod->arch.init.plt_shndx = i;
311
+		else if (!strcmp(secstrings + sechdrs[i].sh_name,
219
312
 				 ".text.ftrace_trampoline"))
220
313
 			tramp = sechdrs + i;
221
314
 		else if (sechdrs[i].sh_type == SHT_SYMTAB)
222
315
 			syms = (Elf64_Sym *)sechdrs[i].sh_addr;
223
316
 	}
224
317
 
225
-	if (!mod->arch.core.plt || !mod->arch.init.plt) {
318
+	if (!mod->arch.core.plt_shndx || !mod->arch.init.plt_shndx) {
226
319
 		pr_err("%s: module PLT section(s) missing\n", mod->name);
227
320
 		return -ENOEXEC;
228
321
 	}
..
..
@@ -231,22 +324,60 @@
231
324
 		return -ENOEXEC;
232
325
 	}
233
326
 
327
+	if (IS_ENABLED(CONFIG_CRYPTO_FIPS140) &&
328
+	    !strcmp(mod->name, "fips140"))
329
+		copy_rela_for_fips140 = true;
330
+
234
331
 	for (i = 0; i < ehdr->e_shnum; i++) {
235
332
 		Elf64_Rela *rels = (void *)ehdr + sechdrs[i].sh_offset;
236
-		int numrels = sechdrs[i].sh_size / sizeof(Elf64_Rela);
333
+		int nents, numrels = sechdrs[i].sh_size / sizeof(Elf64_Rela);
237
334
 		Elf64_Shdr *dstsec = sechdrs + sechdrs[i].sh_info;
238
335
 
239
336
 		if (sechdrs[i].sh_type != SHT_RELA)
240
337
 			continue;
241
338
 
339
+#ifdef CONFIG_CRYPTO_FIPS140
340
+		if (copy_rela_for_fips140 &&
341
+		    !strcmp(secstrings + dstsec->sh_name, ".rodata")) {
342
+			void *p = kmemdup(rels, numrels * sizeof(Elf64_Rela),
343
+					  GFP_KERNEL);
344
+			if (!p) {
345
+				pr_err("fips140: failed to allocate .rodata RELA buffer\n");
346
+				return -ENOMEM;
347
+			}
348
+			mod->arch.rodata_relocations = p;
349
+			mod->arch.num_rodata_relocations = numrels;
350
+		}
351
+#endif
352
+
242
353
 		/* ignore relocations that operate on non-exec sections */
243
354
 		if (!(dstsec->sh_flags & SHF_EXECINSTR))
244
355
 			continue;
245
356
 
246
-		/* sort by type, symbol index and addend */
247
-		sort(rels, numrels, sizeof(Elf64_Rela), cmp_rela, NULL);
357
+#ifdef CONFIG_CRYPTO_FIPS140
358
+		if (copy_rela_for_fips140 &&
359
+		    !strcmp(secstrings + dstsec->sh_name, ".text")) {
360
+			void *p = kmemdup(rels, numrels * sizeof(Elf64_Rela),
361
+					  GFP_KERNEL);
362
+			if (!p) {
363
+				pr_err("fips140: failed to allocate .text RELA buffer\n");
364
+				return -ENOMEM;
365
+			}
366
+			mod->arch.text_relocations = p;
367
+			mod->arch.num_text_relocations = numrels;
368
+		}
369
+#endif
248
370
 
249
-		if (strncmp(secstrings + dstsec->sh_name, ".init", 5) != 0)
371
+		/*
372
+		 * sort branch relocations requiring a PLT by type, symbol index
373
+		 * and addend
374
+		 */
375
+		nents = partition_branch_plt_relas(syms, rels, numrels,
376
+						   sechdrs[i].sh_info);
377
+		if (nents)
378
+			sort(rels, nents, sizeof(Elf64_Rela), cmp_rela, NULL);
379
+
380
+		if (!module_init_layout_section(secstrings + dstsec->sh_name))
250
381
 			core_plts += count_plts(syms, rels, numrels,
251
382
 						sechdrs[i].sh_info, dstsec);
252
383
 		else
..
..
@@ -254,17 +385,19 @@
254
385
 						sechdrs[i].sh_info, dstsec);
255
386
 	}
256
387
 
257
-	mod->arch.core.plt->sh_type = SHT_NOBITS;
258
-	mod->arch.core.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
259
-	mod->arch.core.plt->sh_addralign = L1_CACHE_BYTES;
260
-	mod->arch.core.plt->sh_size = (core_plts  + 1) * sizeof(struct plt_entry);
388
+	pltsec = sechdrs + mod->arch.core.plt_shndx;
389
+	pltsec->sh_type = SHT_NOBITS;
390
+	pltsec->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
391
+	pltsec->sh_addralign = L1_CACHE_BYTES;
392
+	pltsec->sh_size = (core_plts  + 1) * sizeof(struct plt_entry);
261
393
 	mod->arch.core.plt_num_entries = 0;
262
394
 	mod->arch.core.plt_max_entries = core_plts;
263
395
 
264
-	mod->arch.init.plt->sh_type = SHT_NOBITS;
265
-	mod->arch.init.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
266
-	mod->arch.init.plt->sh_addralign = L1_CACHE_BYTES;
267
-	mod->arch.init.plt->sh_size = (init_plts + 1) * sizeof(struct plt_entry);
396
+	pltsec = sechdrs + mod->arch.init.plt_shndx;
397
+	pltsec->sh_type = SHT_NOBITS;
398
+	pltsec->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
399
+	pltsec->sh_addralign = L1_CACHE_BYTES;
400
+	pltsec->sh_size = (init_plts + 1) * sizeof(struct plt_entry);
268
401
 	mod->arch.init.plt_num_entries = 0;
269
402
 	mod->arch.init.plt_max_entries = init_plts;
270
403
 
..
..
@@ -272,7 +405,7 @@
272
405
 		tramp->sh_type = SHT_NOBITS;
273
406
 		tramp->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
274
407
 		tramp->sh_addralign = __alignof__(struct plt_entry);
275
-		tramp->sh_size = sizeof(struct plt_entry);
408
+		tramp->sh_size = NR_FTRACE_PLTS * sizeof(struct plt_entry);
276
409
 	}
277
410
 
278
411
 	return 0;