.. | .. |
---|
16 | 16 | #include <linux/fs.h> |
---|
17 | 17 | #include <linux/string.h> |
---|
18 | 18 | #include <linux/kernel.h> |
---|
| 19 | +#include <linux/kasan.h> |
---|
19 | 20 | #include <linux/moduleloader.h> |
---|
20 | 21 | #include <linux/bug.h> |
---|
| 22 | +#include <linux/memory.h> |
---|
21 | 23 | #include <asm/alternative.h> |
---|
22 | 24 | #include <asm/nospec-branch.h> |
---|
23 | 25 | #include <asm/facility.h> |
---|
.. | .. |
---|
28 | 30 | #define DEBUGP(fmt , ...) |
---|
29 | 31 | #endif |
---|
30 | 32 | |
---|
31 | | -#define PLT_ENTRY_SIZE 20 |
---|
| 33 | +#define PLT_ENTRY_SIZE 22 |
---|
32 | 34 | |
---|
33 | 35 | void *module_alloc(unsigned long size) |
---|
34 | 36 | { |
---|
| 37 | + void *p; |
---|
| 38 | + |
---|
35 | 39 | if (PAGE_ALIGN(size) > MODULES_LEN) |
---|
36 | 40 | return NULL; |
---|
37 | | - return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, |
---|
38 | | - GFP_KERNEL, PAGE_KERNEL_EXEC, |
---|
39 | | - 0, NUMA_NO_NODE, |
---|
40 | | - __builtin_return_address(0)); |
---|
| 41 | + p = __vmalloc_node_range(size, MODULE_ALIGN, MODULES_VADDR, MODULES_END, |
---|
| 42 | + GFP_KERNEL, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, |
---|
| 43 | + __builtin_return_address(0)); |
---|
| 44 | + if (p && (kasan_module_alloc(p, size) < 0)) { |
---|
| 45 | + vfree(p); |
---|
| 46 | + return NULL; |
---|
| 47 | + } |
---|
| 48 | + return p; |
---|
41 | 49 | } |
---|
42 | 50 | |
---|
43 | 51 | void module_arch_freeing_init(struct module *mod) |
---|
.. | .. |
---|
167 | 175 | } |
---|
168 | 176 | |
---|
169 | 177 | static int apply_rela_bits(Elf_Addr loc, Elf_Addr val, |
---|
170 | | - int sign, int bits, int shift) |
---|
| 178 | + int sign, int bits, int shift, |
---|
| 179 | + void *(*write)(void *dest, const void *src, size_t len)) |
---|
171 | 180 | { |
---|
172 | 181 | unsigned long umax; |
---|
173 | 182 | long min, max; |
---|
| 183 | + void *dest = (void *)loc; |
---|
174 | 184 | |
---|
175 | 185 | if (val & ((1UL << shift) - 1)) |
---|
176 | 186 | return -ENOEXEC; |
---|
.. | .. |
---|
187 | 197 | return -ENOEXEC; |
---|
188 | 198 | } |
---|
189 | 199 | |
---|
190 | | - if (bits == 8) |
---|
191 | | - *(unsigned char *) loc = val; |
---|
192 | | - else if (bits == 12) |
---|
193 | | - *(unsigned short *) loc = (val & 0xfff) | |
---|
| 200 | + if (bits == 8) { |
---|
| 201 | + unsigned char tmp = val; |
---|
| 202 | + write(dest, &tmp, 1); |
---|
| 203 | + } else if (bits == 12) { |
---|
| 204 | + unsigned short tmp = (val & 0xfff) | |
---|
194 | 205 | (*(unsigned short *) loc & 0xf000); |
---|
195 | | - else if (bits == 16) |
---|
196 | | - *(unsigned short *) loc = val; |
---|
197 | | - else if (bits == 20) |
---|
198 | | - *(unsigned int *) loc = (val & 0xfff) << 16 | |
---|
199 | | - (val & 0xff000) >> 4 | |
---|
200 | | - (*(unsigned int *) loc & 0xf00000ff); |
---|
201 | | - else if (bits == 32) |
---|
202 | | - *(unsigned int *) loc = val; |
---|
203 | | - else if (bits == 64) |
---|
204 | | - *(unsigned long *) loc = val; |
---|
| 206 | + write(dest, &tmp, 2); |
---|
| 207 | + } else if (bits == 16) { |
---|
| 208 | + unsigned short tmp = val; |
---|
| 209 | + write(dest, &tmp, 2); |
---|
| 210 | + } else if (bits == 20) { |
---|
| 211 | + unsigned int tmp = (val & 0xfff) << 16 | |
---|
| 212 | + (val & 0xff000) >> 4 | (*(unsigned int *) loc & 0xf00000ff); |
---|
| 213 | + write(dest, &tmp, 4); |
---|
| 214 | + } else if (bits == 32) { |
---|
| 215 | + unsigned int tmp = val; |
---|
| 216 | + write(dest, &tmp, 4); |
---|
| 217 | + } else if (bits == 64) { |
---|
| 218 | + unsigned long tmp = val; |
---|
| 219 | + write(dest, &tmp, 8); |
---|
| 220 | + } |
---|
205 | 221 | return 0; |
---|
206 | 222 | } |
---|
207 | 223 | |
---|
208 | 224 | static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab, |
---|
209 | | - const char *strtab, struct module *me) |
---|
| 225 | + const char *strtab, struct module *me, |
---|
| 226 | + void *(*write)(void *dest, const void *src, size_t len)) |
---|
210 | 227 | { |
---|
211 | 228 | struct mod_arch_syminfo *info; |
---|
212 | 229 | Elf_Addr loc, val; |
---|
.. | .. |
---|
234 | 251 | case R_390_64: /* Direct 64 bit. */ |
---|
235 | 252 | val += rela->r_addend; |
---|
236 | 253 | if (r_type == R_390_8) |
---|
237 | | - rc = apply_rela_bits(loc, val, 0, 8, 0); |
---|
| 254 | + rc = apply_rela_bits(loc, val, 0, 8, 0, write); |
---|
238 | 255 | else if (r_type == R_390_12) |
---|
239 | | - rc = apply_rela_bits(loc, val, 0, 12, 0); |
---|
| 256 | + rc = apply_rela_bits(loc, val, 0, 12, 0, write); |
---|
240 | 257 | else if (r_type == R_390_16) |
---|
241 | | - rc = apply_rela_bits(loc, val, 0, 16, 0); |
---|
| 258 | + rc = apply_rela_bits(loc, val, 0, 16, 0, write); |
---|
242 | 259 | else if (r_type == R_390_20) |
---|
243 | | - rc = apply_rela_bits(loc, val, 1, 20, 0); |
---|
| 260 | + rc = apply_rela_bits(loc, val, 1, 20, 0, write); |
---|
244 | 261 | else if (r_type == R_390_32) |
---|
245 | | - rc = apply_rela_bits(loc, val, 0, 32, 0); |
---|
| 262 | + rc = apply_rela_bits(loc, val, 0, 32, 0, write); |
---|
246 | 263 | else if (r_type == R_390_64) |
---|
247 | | - rc = apply_rela_bits(loc, val, 0, 64, 0); |
---|
| 264 | + rc = apply_rela_bits(loc, val, 0, 64, 0, write); |
---|
248 | 265 | break; |
---|
249 | 266 | case R_390_PC16: /* PC relative 16 bit. */ |
---|
250 | 267 | case R_390_PC16DBL: /* PC relative 16 bit shifted by 1. */ |
---|
.. | .. |
---|
253 | 270 | case R_390_PC64: /* PC relative 64 bit. */ |
---|
254 | 271 | val += rela->r_addend - loc; |
---|
255 | 272 | if (r_type == R_390_PC16) |
---|
256 | | - rc = apply_rela_bits(loc, val, 1, 16, 0); |
---|
| 273 | + rc = apply_rela_bits(loc, val, 1, 16, 0, write); |
---|
257 | 274 | else if (r_type == R_390_PC16DBL) |
---|
258 | | - rc = apply_rela_bits(loc, val, 1, 16, 1); |
---|
| 275 | + rc = apply_rela_bits(loc, val, 1, 16, 1, write); |
---|
259 | 276 | else if (r_type == R_390_PC32DBL) |
---|
260 | | - rc = apply_rela_bits(loc, val, 1, 32, 1); |
---|
| 277 | + rc = apply_rela_bits(loc, val, 1, 32, 1, write); |
---|
261 | 278 | else if (r_type == R_390_PC32) |
---|
262 | | - rc = apply_rela_bits(loc, val, 1, 32, 0); |
---|
| 279 | + rc = apply_rela_bits(loc, val, 1, 32, 0, write); |
---|
263 | 280 | else if (r_type == R_390_PC64) |
---|
264 | | - rc = apply_rela_bits(loc, val, 1, 64, 0); |
---|
| 281 | + rc = apply_rela_bits(loc, val, 1, 64, 0, write); |
---|
265 | 282 | break; |
---|
266 | 283 | case R_390_GOT12: /* 12 bit GOT offset. */ |
---|
267 | 284 | case R_390_GOT16: /* 16 bit GOT offset. */ |
---|
.. | .. |
---|
276 | 293 | case R_390_GOTPLT64: /* 64 bit offset to jump slot. */ |
---|
277 | 294 | case R_390_GOTPLTENT: /* 32 bit rel. offset to jump slot >> 1. */ |
---|
278 | 295 | if (info->got_initialized == 0) { |
---|
279 | | - Elf_Addr *gotent; |
---|
| 296 | + Elf_Addr *gotent = me->core_layout.base + |
---|
| 297 | + me->arch.got_offset + |
---|
| 298 | + info->got_offset; |
---|
280 | 299 | |
---|
281 | | - gotent = me->core_layout.base + me->arch.got_offset + |
---|
282 | | - info->got_offset; |
---|
283 | | - *gotent = val; |
---|
| 300 | + write(gotent, &val, sizeof(*gotent)); |
---|
284 | 301 | info->got_initialized = 1; |
---|
285 | 302 | } |
---|
286 | 303 | val = info->got_offset + rela->r_addend; |
---|
287 | 304 | if (r_type == R_390_GOT12 || |
---|
288 | 305 | r_type == R_390_GOTPLT12) |
---|
289 | | - rc = apply_rela_bits(loc, val, 0, 12, 0); |
---|
| 306 | + rc = apply_rela_bits(loc, val, 0, 12, 0, write); |
---|
290 | 307 | else if (r_type == R_390_GOT16 || |
---|
291 | 308 | r_type == R_390_GOTPLT16) |
---|
292 | | - rc = apply_rela_bits(loc, val, 0, 16, 0); |
---|
| 309 | + rc = apply_rela_bits(loc, val, 0, 16, 0, write); |
---|
293 | 310 | else if (r_type == R_390_GOT20 || |
---|
294 | 311 | r_type == R_390_GOTPLT20) |
---|
295 | | - rc = apply_rela_bits(loc, val, 1, 20, 0); |
---|
| 312 | + rc = apply_rela_bits(loc, val, 1, 20, 0, write); |
---|
296 | 313 | else if (r_type == R_390_GOT32 || |
---|
297 | 314 | r_type == R_390_GOTPLT32) |
---|
298 | | - rc = apply_rela_bits(loc, val, 0, 32, 0); |
---|
| 315 | + rc = apply_rela_bits(loc, val, 0, 32, 0, write); |
---|
299 | 316 | else if (r_type == R_390_GOT64 || |
---|
300 | 317 | r_type == R_390_GOTPLT64) |
---|
301 | | - rc = apply_rela_bits(loc, val, 0, 64, 0); |
---|
| 318 | + rc = apply_rela_bits(loc, val, 0, 64, 0, write); |
---|
302 | 319 | else if (r_type == R_390_GOTENT || |
---|
303 | 320 | r_type == R_390_GOTPLTENT) { |
---|
304 | 321 | val += (Elf_Addr) me->core_layout.base - loc; |
---|
305 | | - rc = apply_rela_bits(loc, val, 1, 32, 1); |
---|
| 322 | + rc = apply_rela_bits(loc, val, 1, 32, 1, write); |
---|
306 | 323 | } |
---|
307 | 324 | break; |
---|
308 | 325 | case R_390_PLT16DBL: /* 16 bit PC rel. PLT shifted by 1. */ |
---|
.. | .. |
---|
313 | 330 | case R_390_PLTOFF32: /* 32 bit offset from GOT to PLT. */ |
---|
314 | 331 | case R_390_PLTOFF64: /* 16 bit offset from GOT to PLT. */ |
---|
315 | 332 | if (info->plt_initialized == 0) { |
---|
316 | | - unsigned int *ip; |
---|
317 | | - ip = me->core_layout.base + me->arch.plt_offset + |
---|
318 | | - info->plt_offset; |
---|
319 | | - ip[0] = 0x0d10e310; /* basr 1,0 */ |
---|
320 | | - ip[1] = 0x100a0004; /* lg 1,10(1) */ |
---|
| 333 | + unsigned char insn[PLT_ENTRY_SIZE]; |
---|
| 334 | + char *plt_base; |
---|
| 335 | + char *ip; |
---|
| 336 | + |
---|
| 337 | + plt_base = me->core_layout.base + me->arch.plt_offset; |
---|
| 338 | + ip = plt_base + info->plt_offset; |
---|
| 339 | + *(int *)insn = 0x0d10e310; /* basr 1,0 */ |
---|
| 340 | + *(int *)&insn[4] = 0x100c0004; /* lg 1,12(1) */ |
---|
321 | 341 | if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_disable) { |
---|
322 | | - unsigned int *ij; |
---|
323 | | - ij = me->core_layout.base + |
---|
324 | | - me->arch.plt_offset + |
---|
325 | | - me->arch.plt_size - PLT_ENTRY_SIZE; |
---|
326 | | - ip[2] = 0xa7f40000 + /* j __jump_r1 */ |
---|
327 | | - (unsigned int)(u16) |
---|
328 | | - (((unsigned long) ij - 8 - |
---|
329 | | - (unsigned long) ip) / 2); |
---|
| 342 | + char *jump_r1; |
---|
| 343 | + |
---|
| 344 | + jump_r1 = plt_base + me->arch.plt_size - |
---|
| 345 | + PLT_ENTRY_SIZE; |
---|
| 346 | + /* brcl 0xf,__jump_r1 */ |
---|
| 347 | + *(short *)&insn[8] = 0xc0f4; |
---|
| 348 | + *(int *)&insn[10] = (jump_r1 - (ip + 8)) / 2; |
---|
330 | 349 | } else { |
---|
331 | | - ip[2] = 0x07f10000; /* br %r1 */ |
---|
| 350 | + *(int *)&insn[8] = 0x07f10000; /* br %r1 */ |
---|
332 | 351 | } |
---|
333 | | - ip[3] = (unsigned int) (val >> 32); |
---|
334 | | - ip[4] = (unsigned int) val; |
---|
| 352 | + *(long *)&insn[14] = val; |
---|
| 353 | + |
---|
| 354 | + write(ip, insn, sizeof(insn)); |
---|
335 | 355 | info->plt_initialized = 1; |
---|
336 | 356 | } |
---|
337 | 357 | if (r_type == R_390_PLTOFF16 || |
---|
.. | .. |
---|
350 | 370 | val += rela->r_addend - loc; |
---|
351 | 371 | } |
---|
352 | 372 | if (r_type == R_390_PLT16DBL) |
---|
353 | | - rc = apply_rela_bits(loc, val, 1, 16, 1); |
---|
| 373 | + rc = apply_rela_bits(loc, val, 1, 16, 1, write); |
---|
354 | 374 | else if (r_type == R_390_PLTOFF16) |
---|
355 | | - rc = apply_rela_bits(loc, val, 0, 16, 0); |
---|
| 375 | + rc = apply_rela_bits(loc, val, 0, 16, 0, write); |
---|
356 | 376 | else if (r_type == R_390_PLT32DBL) |
---|
357 | | - rc = apply_rela_bits(loc, val, 1, 32, 1); |
---|
| 377 | + rc = apply_rela_bits(loc, val, 1, 32, 1, write); |
---|
358 | 378 | else if (r_type == R_390_PLT32 || |
---|
359 | 379 | r_type == R_390_PLTOFF32) |
---|
360 | | - rc = apply_rela_bits(loc, val, 0, 32, 0); |
---|
| 380 | + rc = apply_rela_bits(loc, val, 0, 32, 0, write); |
---|
361 | 381 | else if (r_type == R_390_PLT64 || |
---|
362 | 382 | r_type == R_390_PLTOFF64) |
---|
363 | | - rc = apply_rela_bits(loc, val, 0, 64, 0); |
---|
| 383 | + rc = apply_rela_bits(loc, val, 0, 64, 0, write); |
---|
364 | 384 | break; |
---|
365 | 385 | case R_390_GOTOFF16: /* 16 bit offset to GOT. */ |
---|
366 | 386 | case R_390_GOTOFF32: /* 32 bit offset to GOT. */ |
---|
.. | .. |
---|
368 | 388 | val = val + rela->r_addend - |
---|
369 | 389 | ((Elf_Addr) me->core_layout.base + me->arch.got_offset); |
---|
370 | 390 | if (r_type == R_390_GOTOFF16) |
---|
371 | | - rc = apply_rela_bits(loc, val, 0, 16, 0); |
---|
| 391 | + rc = apply_rela_bits(loc, val, 0, 16, 0, write); |
---|
372 | 392 | else if (r_type == R_390_GOTOFF32) |
---|
373 | | - rc = apply_rela_bits(loc, val, 0, 32, 0); |
---|
| 393 | + rc = apply_rela_bits(loc, val, 0, 32, 0, write); |
---|
374 | 394 | else if (r_type == R_390_GOTOFF64) |
---|
375 | | - rc = apply_rela_bits(loc, val, 0, 64, 0); |
---|
| 395 | + rc = apply_rela_bits(loc, val, 0, 64, 0, write); |
---|
376 | 396 | break; |
---|
377 | 397 | case R_390_GOTPC: /* 32 bit PC relative offset to GOT. */ |
---|
378 | 398 | case R_390_GOTPCDBL: /* 32 bit PC rel. off. to GOT shifted by 1. */ |
---|
379 | 399 | val = (Elf_Addr) me->core_layout.base + me->arch.got_offset + |
---|
380 | 400 | rela->r_addend - loc; |
---|
381 | 401 | if (r_type == R_390_GOTPC) |
---|
382 | | - rc = apply_rela_bits(loc, val, 1, 32, 0); |
---|
| 402 | + rc = apply_rela_bits(loc, val, 1, 32, 0, write); |
---|
383 | 403 | else if (r_type == R_390_GOTPCDBL) |
---|
384 | | - rc = apply_rela_bits(loc, val, 1, 32, 1); |
---|
| 404 | + rc = apply_rela_bits(loc, val, 1, 32, 1, write); |
---|
385 | 405 | break; |
---|
386 | 406 | case R_390_COPY: |
---|
387 | 407 | case R_390_GLOB_DAT: /* Create GOT entry. */ |
---|
.. | .. |
---|
405 | 425 | return 0; |
---|
406 | 426 | } |
---|
407 | 427 | |
---|
408 | | -int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, |
---|
| 428 | +static int __apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, |
---|
409 | 429 | unsigned int symindex, unsigned int relsec, |
---|
410 | | - struct module *me) |
---|
| 430 | + struct module *me, |
---|
| 431 | + void *(*write)(void *dest, const void *src, size_t len)) |
---|
411 | 432 | { |
---|
412 | 433 | Elf_Addr base; |
---|
413 | 434 | Elf_Sym *symtab; |
---|
.. | .. |
---|
423 | 444 | n = sechdrs[relsec].sh_size / sizeof(Elf_Rela); |
---|
424 | 445 | |
---|
425 | 446 | for (i = 0; i < n; i++, rela++) { |
---|
426 | | - rc = apply_rela(rela, base, symtab, strtab, me); |
---|
| 447 | + rc = apply_rela(rela, base, symtab, strtab, me, write); |
---|
427 | 448 | if (rc) |
---|
428 | 449 | return rc; |
---|
429 | 450 | } |
---|
430 | 451 | return 0; |
---|
| 452 | +} |
---|
| 453 | + |
---|
| 454 | +int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, |
---|
| 455 | + unsigned int symindex, unsigned int relsec, |
---|
| 456 | + struct module *me) |
---|
| 457 | +{ |
---|
| 458 | + bool early = me->state == MODULE_STATE_UNFORMED; |
---|
| 459 | + void *(*write)(void *, const void *, size_t) = memcpy; |
---|
| 460 | + |
---|
| 461 | + if (!early) |
---|
| 462 | + write = s390_kernel_write; |
---|
| 463 | + |
---|
| 464 | + return __apply_relocate_add(sechdrs, strtab, symindex, relsec, me, |
---|
| 465 | + write); |
---|
431 | 466 | } |
---|
432 | 467 | |
---|
433 | 468 | int module_finalize(const Elf_Ehdr *hdr, |
---|
.. | .. |
---|
465 | 500 | apply_alternatives(aseg, aseg + s->sh_size); |
---|
466 | 501 | |
---|
467 | 502 | if (IS_ENABLED(CONFIG_EXPOLINE) && |
---|
468 | | - (!strncmp(".s390_indirect", secname, 14))) |
---|
| 503 | + (str_has_prefix(secname, ".s390_indirect"))) |
---|
469 | 504 | nospec_revert(aseg, aseg + s->sh_size); |
---|
470 | 505 | |
---|
471 | 506 | if (IS_ENABLED(CONFIG_EXPOLINE) && |
---|
472 | | - (!strncmp(".s390_return", secname, 12))) |
---|
| 507 | + (str_has_prefix(secname, ".s390_return"))) |
---|
473 | 508 | nospec_revert(aseg, aseg + s->sh_size); |
---|
474 | 509 | } |
---|
475 | 510 | |
---|