| .. | .. |
|---|
| 16 | 16 | #include <linux/fs.h> |
|---|
| 17 | 17 | #include <linux/string.h> |
|---|
| 18 | 18 | #include <linux/kernel.h> |
|---|
| 19 | +#include <linux/kasan.h> |
|---|
| 19 | 20 | #include <linux/moduleloader.h> |
|---|
| 20 | 21 | #include <linux/bug.h> |
|---|
| 22 | +#include <linux/memory.h> |
|---|
| 21 | 23 | #include <asm/alternative.h> |
|---|
| 22 | 24 | #include <asm/nospec-branch.h> |
|---|
| 23 | 25 | #include <asm/facility.h> |
|---|
| .. | .. |
|---|
| 28 | 30 | #define DEBUGP(fmt , ...) |
|---|
| 29 | 31 | #endif |
|---|
| 30 | 32 | |
|---|
| 31 | | -#define PLT_ENTRY_SIZE 20 |
|---|
| 33 | +#define PLT_ENTRY_SIZE 22 |
|---|
| 32 | 34 | |
|---|
| 33 | 35 | void *module_alloc(unsigned long size) |
|---|
| 34 | 36 | { |
|---|
| 37 | + void *p; |
|---|
| 38 | + |
|---|
| 35 | 39 | if (PAGE_ALIGN(size) > MODULES_LEN) |
|---|
| 36 | 40 | return NULL; |
|---|
| 37 | | - return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, |
|---|
| 38 | | - GFP_KERNEL, PAGE_KERNEL_EXEC, |
|---|
| 39 | | - 0, NUMA_NO_NODE, |
|---|
| 40 | | - __builtin_return_address(0)); |
|---|
| 41 | + p = __vmalloc_node_range(size, MODULE_ALIGN, MODULES_VADDR, MODULES_END, |
|---|
| 42 | + GFP_KERNEL, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, |
|---|
| 43 | + __builtin_return_address(0)); |
|---|
| 44 | + if (p && (kasan_module_alloc(p, size) < 0)) { |
|---|
| 45 | + vfree(p); |
|---|
| 46 | + return NULL; |
|---|
| 47 | + } |
|---|
| 48 | + return p; |
|---|
| 41 | 49 | } |
|---|
| 42 | 50 | |
|---|
| 43 | 51 | void module_arch_freeing_init(struct module *mod) |
|---|
| .. | .. |
|---|
| 167 | 175 | } |
|---|
| 168 | 176 | |
|---|
| 169 | 177 | static int apply_rela_bits(Elf_Addr loc, Elf_Addr val, |
|---|
| 170 | | - int sign, int bits, int shift) |
|---|
| 178 | + int sign, int bits, int shift, |
|---|
| 179 | + void *(*write)(void *dest, const void *src, size_t len)) |
|---|
| 171 | 180 | { |
|---|
| 172 | 181 | unsigned long umax; |
|---|
| 173 | 182 | long min, max; |
|---|
| 183 | + void *dest = (void *)loc; |
|---|
| 174 | 184 | |
|---|
| 175 | 185 | if (val & ((1UL << shift) - 1)) |
|---|
| 176 | 186 | return -ENOEXEC; |
|---|
| .. | .. |
|---|
| 187 | 197 | return -ENOEXEC; |
|---|
| 188 | 198 | } |
|---|
| 189 | 199 | |
|---|
| 190 | | - if (bits == 8) |
|---|
| 191 | | - *(unsigned char *) loc = val; |
|---|
| 192 | | - else if (bits == 12) |
|---|
| 193 | | - *(unsigned short *) loc = (val & 0xfff) | |
|---|
| 200 | + if (bits == 8) { |
|---|
| 201 | + unsigned char tmp = val; |
|---|
| 202 | + write(dest, &tmp, 1); |
|---|
| 203 | + } else if (bits == 12) { |
|---|
| 204 | + unsigned short tmp = (val & 0xfff) | |
|---|
| 194 | 205 | (*(unsigned short *) loc & 0xf000); |
|---|
| 195 | | - else if (bits == 16) |
|---|
| 196 | | - *(unsigned short *) loc = val; |
|---|
| 197 | | - else if (bits == 20) |
|---|
| 198 | | - *(unsigned int *) loc = (val & 0xfff) << 16 | |
|---|
| 199 | | - (val & 0xff000) >> 4 | |
|---|
| 200 | | - (*(unsigned int *) loc & 0xf00000ff); |
|---|
| 201 | | - else if (bits == 32) |
|---|
| 202 | | - *(unsigned int *) loc = val; |
|---|
| 203 | | - else if (bits == 64) |
|---|
| 204 | | - *(unsigned long *) loc = val; |
|---|
| 206 | + write(dest, &tmp, 2); |
|---|
| 207 | + } else if (bits == 16) { |
|---|
| 208 | + unsigned short tmp = val; |
|---|
| 209 | + write(dest, &tmp, 2); |
|---|
| 210 | + } else if (bits == 20) { |
|---|
| 211 | + unsigned int tmp = (val & 0xfff) << 16 | |
|---|
| 212 | + (val & 0xff000) >> 4 | (*(unsigned int *) loc & 0xf00000ff); |
|---|
| 213 | + write(dest, &tmp, 4); |
|---|
| 214 | + } else if (bits == 32) { |
|---|
| 215 | + unsigned int tmp = val; |
|---|
| 216 | + write(dest, &tmp, 4); |
|---|
| 217 | + } else if (bits == 64) { |
|---|
| 218 | + unsigned long tmp = val; |
|---|
| 219 | + write(dest, &tmp, 8); |
|---|
| 220 | + } |
|---|
| 205 | 221 | return 0; |
|---|
| 206 | 222 | } |
|---|
| 207 | 223 | |
|---|
| 208 | 224 | static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab, |
|---|
| 209 | | - const char *strtab, struct module *me) |
|---|
| 225 | + const char *strtab, struct module *me, |
|---|
| 226 | + void *(*write)(void *dest, const void *src, size_t len)) |
|---|
| 210 | 227 | { |
|---|
| 211 | 228 | struct mod_arch_syminfo *info; |
|---|
| 212 | 229 | Elf_Addr loc, val; |
|---|
| .. | .. |
|---|
| 234 | 251 | case R_390_64: /* Direct 64 bit. */ |
|---|
| 235 | 252 | val += rela->r_addend; |
|---|
| 236 | 253 | if (r_type == R_390_8) |
|---|
| 237 | | - rc = apply_rela_bits(loc, val, 0, 8, 0); |
|---|
| 254 | + rc = apply_rela_bits(loc, val, 0, 8, 0, write); |
|---|
| 238 | 255 | else if (r_type == R_390_12) |
|---|
| 239 | | - rc = apply_rela_bits(loc, val, 0, 12, 0); |
|---|
| 256 | + rc = apply_rela_bits(loc, val, 0, 12, 0, write); |
|---|
| 240 | 257 | else if (r_type == R_390_16) |
|---|
| 241 | | - rc = apply_rela_bits(loc, val, 0, 16, 0); |
|---|
| 258 | + rc = apply_rela_bits(loc, val, 0, 16, 0, write); |
|---|
| 242 | 259 | else if (r_type == R_390_20) |
|---|
| 243 | | - rc = apply_rela_bits(loc, val, 1, 20, 0); |
|---|
| 260 | + rc = apply_rela_bits(loc, val, 1, 20, 0, write); |
|---|
| 244 | 261 | else if (r_type == R_390_32) |
|---|
| 245 | | - rc = apply_rela_bits(loc, val, 0, 32, 0); |
|---|
| 262 | + rc = apply_rela_bits(loc, val, 0, 32, 0, write); |
|---|
| 246 | 263 | else if (r_type == R_390_64) |
|---|
| 247 | | - rc = apply_rela_bits(loc, val, 0, 64, 0); |
|---|
| 264 | + rc = apply_rela_bits(loc, val, 0, 64, 0, write); |
|---|
| 248 | 265 | break; |
|---|
| 249 | 266 | case R_390_PC16: /* PC relative 16 bit. */ |
|---|
| 250 | 267 | case R_390_PC16DBL: /* PC relative 16 bit shifted by 1. */ |
|---|
| .. | .. |
|---|
| 253 | 270 | case R_390_PC64: /* PC relative 64 bit. */ |
|---|
| 254 | 271 | val += rela->r_addend - loc; |
|---|
| 255 | 272 | if (r_type == R_390_PC16) |
|---|
| 256 | | - rc = apply_rela_bits(loc, val, 1, 16, 0); |
|---|
| 273 | + rc = apply_rela_bits(loc, val, 1, 16, 0, write); |
|---|
| 257 | 274 | else if (r_type == R_390_PC16DBL) |
|---|
| 258 | | - rc = apply_rela_bits(loc, val, 1, 16, 1); |
|---|
| 275 | + rc = apply_rela_bits(loc, val, 1, 16, 1, write); |
|---|
| 259 | 276 | else if (r_type == R_390_PC32DBL) |
|---|
| 260 | | - rc = apply_rela_bits(loc, val, 1, 32, 1); |
|---|
| 277 | + rc = apply_rela_bits(loc, val, 1, 32, 1, write); |
|---|
| 261 | 278 | else if (r_type == R_390_PC32) |
|---|
| 262 | | - rc = apply_rela_bits(loc, val, 1, 32, 0); |
|---|
| 279 | + rc = apply_rela_bits(loc, val, 1, 32, 0, write); |
|---|
| 263 | 280 | else if (r_type == R_390_PC64) |
|---|
| 264 | | - rc = apply_rela_bits(loc, val, 1, 64, 0); |
|---|
| 281 | + rc = apply_rela_bits(loc, val, 1, 64, 0, write); |
|---|
| 265 | 282 | break; |
|---|
| 266 | 283 | case R_390_GOT12: /* 12 bit GOT offset. */ |
|---|
| 267 | 284 | case R_390_GOT16: /* 16 bit GOT offset. */ |
|---|
| .. | .. |
|---|
| 276 | 293 | case R_390_GOTPLT64: /* 64 bit offset to jump slot. */ |
|---|
| 277 | 294 | case R_390_GOTPLTENT: /* 32 bit rel. offset to jump slot >> 1. */ |
|---|
| 278 | 295 | if (info->got_initialized == 0) { |
|---|
| 279 | | - Elf_Addr *gotent; |
|---|
| 296 | + Elf_Addr *gotent = me->core_layout.base + |
|---|
| 297 | + me->arch.got_offset + |
|---|
| 298 | + info->got_offset; |
|---|
| 280 | 299 | |
|---|
| 281 | | - gotent = me->core_layout.base + me->arch.got_offset + |
|---|
| 282 | | - info->got_offset; |
|---|
| 283 | | - *gotent = val; |
|---|
| 300 | + write(gotent, &val, sizeof(*gotent)); |
|---|
| 284 | 301 | info->got_initialized = 1; |
|---|
| 285 | 302 | } |
|---|
| 286 | 303 | val = info->got_offset + rela->r_addend; |
|---|
| 287 | 304 | if (r_type == R_390_GOT12 || |
|---|
| 288 | 305 | r_type == R_390_GOTPLT12) |
|---|
| 289 | | - rc = apply_rela_bits(loc, val, 0, 12, 0); |
|---|
| 306 | + rc = apply_rela_bits(loc, val, 0, 12, 0, write); |
|---|
| 290 | 307 | else if (r_type == R_390_GOT16 || |
|---|
| 291 | 308 | r_type == R_390_GOTPLT16) |
|---|
| 292 | | - rc = apply_rela_bits(loc, val, 0, 16, 0); |
|---|
| 309 | + rc = apply_rela_bits(loc, val, 0, 16, 0, write); |
|---|
| 293 | 310 | else if (r_type == R_390_GOT20 || |
|---|
| 294 | 311 | r_type == R_390_GOTPLT20) |
|---|
| 295 | | - rc = apply_rela_bits(loc, val, 1, 20, 0); |
|---|
| 312 | + rc = apply_rela_bits(loc, val, 1, 20, 0, write); |
|---|
| 296 | 313 | else if (r_type == R_390_GOT32 || |
|---|
| 297 | 314 | r_type == R_390_GOTPLT32) |
|---|
| 298 | | - rc = apply_rela_bits(loc, val, 0, 32, 0); |
|---|
| 315 | + rc = apply_rela_bits(loc, val, 0, 32, 0, write); |
|---|
| 299 | 316 | else if (r_type == R_390_GOT64 || |
|---|
| 300 | 317 | r_type == R_390_GOTPLT64) |
|---|
| 301 | | - rc = apply_rela_bits(loc, val, 0, 64, 0); |
|---|
| 318 | + rc = apply_rela_bits(loc, val, 0, 64, 0, write); |
|---|
| 302 | 319 | else if (r_type == R_390_GOTENT || |
|---|
| 303 | 320 | r_type == R_390_GOTPLTENT) { |
|---|
| 304 | 321 | val += (Elf_Addr) me->core_layout.base - loc; |
|---|
| 305 | | - rc = apply_rela_bits(loc, val, 1, 32, 1); |
|---|
| 322 | + rc = apply_rela_bits(loc, val, 1, 32, 1, write); |
|---|
| 306 | 323 | } |
|---|
| 307 | 324 | break; |
|---|
| 308 | 325 | case R_390_PLT16DBL: /* 16 bit PC rel. PLT shifted by 1. */ |
|---|
| .. | .. |
|---|
| 313 | 330 | case R_390_PLTOFF32: /* 32 bit offset from GOT to PLT. */ |
|---|
| 314 | 331 | case R_390_PLTOFF64: /* 16 bit offset from GOT to PLT. */ |
|---|
| 315 | 332 | if (info->plt_initialized == 0) { |
|---|
| 316 | | - unsigned int *ip; |
|---|
| 317 | | - ip = me->core_layout.base + me->arch.plt_offset + |
|---|
| 318 | | - info->plt_offset; |
|---|
| 319 | | - ip[0] = 0x0d10e310; /* basr 1,0 */ |
|---|
| 320 | | - ip[1] = 0x100a0004; /* lg 1,10(1) */ |
|---|
| 333 | + unsigned char insn[PLT_ENTRY_SIZE]; |
|---|
| 334 | + char *plt_base; |
|---|
| 335 | + char *ip; |
|---|
| 336 | + |
|---|
| 337 | + plt_base = me->core_layout.base + me->arch.plt_offset; |
|---|
| 338 | + ip = plt_base + info->plt_offset; |
|---|
| 339 | + *(int *)insn = 0x0d10e310; /* basr 1,0 */ |
|---|
| 340 | + *(int *)&insn[4] = 0x100c0004; /* lg 1,12(1) */ |
|---|
| 321 | 341 | if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_disable) { |
|---|
| 322 | | - unsigned int *ij; |
|---|
| 323 | | - ij = me->core_layout.base + |
|---|
| 324 | | - me->arch.plt_offset + |
|---|
| 325 | | - me->arch.plt_size - PLT_ENTRY_SIZE; |
|---|
| 326 | | - ip[2] = 0xa7f40000 + /* j __jump_r1 */ |
|---|
| 327 | | - (unsigned int)(u16) |
|---|
| 328 | | - (((unsigned long) ij - 8 - |
|---|
| 329 | | - (unsigned long) ip) / 2); |
|---|
| 342 | + char *jump_r1; |
|---|
| 343 | + |
|---|
| 344 | + jump_r1 = plt_base + me->arch.plt_size - |
|---|
| 345 | + PLT_ENTRY_SIZE; |
|---|
| 346 | + /* brcl 0xf,__jump_r1 */ |
|---|
| 347 | + *(short *)&insn[8] = 0xc0f4; |
|---|
| 348 | + *(int *)&insn[10] = (jump_r1 - (ip + 8)) / 2; |
|---|
| 330 | 349 | } else { |
|---|
| 331 | | - ip[2] = 0x07f10000; /* br %r1 */ |
|---|
| 350 | + *(int *)&insn[8] = 0x07f10000; /* br %r1 */ |
|---|
| 332 | 351 | } |
|---|
| 333 | | - ip[3] = (unsigned int) (val >> 32); |
|---|
| 334 | | - ip[4] = (unsigned int) val; |
|---|
| 352 | + *(long *)&insn[14] = val; |
|---|
| 353 | + |
|---|
| 354 | + write(ip, insn, sizeof(insn)); |
|---|
| 335 | 355 | info->plt_initialized = 1; |
|---|
| 336 | 356 | } |
|---|
| 337 | 357 | if (r_type == R_390_PLTOFF16 || |
|---|
| .. | .. |
|---|
| 350 | 370 | val += rela->r_addend - loc; |
|---|
| 351 | 371 | } |
|---|
| 352 | 372 | if (r_type == R_390_PLT16DBL) |
|---|
| 353 | | - rc = apply_rela_bits(loc, val, 1, 16, 1); |
|---|
| 373 | + rc = apply_rela_bits(loc, val, 1, 16, 1, write); |
|---|
| 354 | 374 | else if (r_type == R_390_PLTOFF16) |
|---|
| 355 | | - rc = apply_rela_bits(loc, val, 0, 16, 0); |
|---|
| 375 | + rc = apply_rela_bits(loc, val, 0, 16, 0, write); |
|---|
| 356 | 376 | else if (r_type == R_390_PLT32DBL) |
|---|
| 357 | | - rc = apply_rela_bits(loc, val, 1, 32, 1); |
|---|
| 377 | + rc = apply_rela_bits(loc, val, 1, 32, 1, write); |
|---|
| 358 | 378 | else if (r_type == R_390_PLT32 || |
|---|
| 359 | 379 | r_type == R_390_PLTOFF32) |
|---|
| 360 | | - rc = apply_rela_bits(loc, val, 0, 32, 0); |
|---|
| 380 | + rc = apply_rela_bits(loc, val, 0, 32, 0, write); |
|---|
| 361 | 381 | else if (r_type == R_390_PLT64 || |
|---|
| 362 | 382 | r_type == R_390_PLTOFF64) |
|---|
| 363 | | - rc = apply_rela_bits(loc, val, 0, 64, 0); |
|---|
| 383 | + rc = apply_rela_bits(loc, val, 0, 64, 0, write); |
|---|
| 364 | 384 | break; |
|---|
| 365 | 385 | case R_390_GOTOFF16: /* 16 bit offset to GOT. */ |
|---|
| 366 | 386 | case R_390_GOTOFF32: /* 32 bit offset to GOT. */ |
|---|
| .. | .. |
|---|
| 368 | 388 | val = val + rela->r_addend - |
|---|
| 369 | 389 | ((Elf_Addr) me->core_layout.base + me->arch.got_offset); |
|---|
| 370 | 390 | if (r_type == R_390_GOTOFF16) |
|---|
| 371 | | - rc = apply_rela_bits(loc, val, 0, 16, 0); |
|---|
| 391 | + rc = apply_rela_bits(loc, val, 0, 16, 0, write); |
|---|
| 372 | 392 | else if (r_type == R_390_GOTOFF32) |
|---|
| 373 | | - rc = apply_rela_bits(loc, val, 0, 32, 0); |
|---|
| 393 | + rc = apply_rela_bits(loc, val, 0, 32, 0, write); |
|---|
| 374 | 394 | else if (r_type == R_390_GOTOFF64) |
|---|
| 375 | | - rc = apply_rela_bits(loc, val, 0, 64, 0); |
|---|
| 395 | + rc = apply_rela_bits(loc, val, 0, 64, 0, write); |
|---|
| 376 | 396 | break; |
|---|
| 377 | 397 | case R_390_GOTPC: /* 32 bit PC relative offset to GOT. */ |
|---|
| 378 | 398 | case R_390_GOTPCDBL: /* 32 bit PC rel. off. to GOT shifted by 1. */ |
|---|
| 379 | 399 | val = (Elf_Addr) me->core_layout.base + me->arch.got_offset + |
|---|
| 380 | 400 | rela->r_addend - loc; |
|---|
| 381 | 401 | if (r_type == R_390_GOTPC) |
|---|
| 382 | | - rc = apply_rela_bits(loc, val, 1, 32, 0); |
|---|
| 402 | + rc = apply_rela_bits(loc, val, 1, 32, 0, write); |
|---|
| 383 | 403 | else if (r_type == R_390_GOTPCDBL) |
|---|
| 384 | | - rc = apply_rela_bits(loc, val, 1, 32, 1); |
|---|
| 404 | + rc = apply_rela_bits(loc, val, 1, 32, 1, write); |
|---|
| 385 | 405 | break; |
|---|
| 386 | 406 | case R_390_COPY: |
|---|
| 387 | 407 | case R_390_GLOB_DAT: /* Create GOT entry. */ |
|---|
| .. | .. |
|---|
| 405 | 425 | return 0; |
|---|
| 406 | 426 | } |
|---|
| 407 | 427 | |
|---|
| 408 | | -int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, |
|---|
| 428 | +static int __apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, |
|---|
| 409 | 429 | unsigned int symindex, unsigned int relsec, |
|---|
| 410 | | - struct module *me) |
|---|
| 430 | + struct module *me, |
|---|
| 431 | + void *(*write)(void *dest, const void *src, size_t len)) |
|---|
| 411 | 432 | { |
|---|
| 412 | 433 | Elf_Addr base; |
|---|
| 413 | 434 | Elf_Sym *symtab; |
|---|
| .. | .. |
|---|
| 423 | 444 | n = sechdrs[relsec].sh_size / sizeof(Elf_Rela); |
|---|
| 424 | 445 | |
|---|
| 425 | 446 | for (i = 0; i < n; i++, rela++) { |
|---|
| 426 | | - rc = apply_rela(rela, base, symtab, strtab, me); |
|---|
| 447 | + rc = apply_rela(rela, base, symtab, strtab, me, write); |
|---|
| 427 | 448 | if (rc) |
|---|
| 428 | 449 | return rc; |
|---|
| 429 | 450 | } |
|---|
| 430 | 451 | return 0; |
|---|
| 452 | +} |
|---|
| 453 | + |
|---|
| 454 | +int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, |
|---|
| 455 | + unsigned int symindex, unsigned int relsec, |
|---|
| 456 | + struct module *me) |
|---|
| 457 | +{ |
|---|
| 458 | + bool early = me->state == MODULE_STATE_UNFORMED; |
|---|
| 459 | + void *(*write)(void *, const void *, size_t) = memcpy; |
|---|
| 460 | + |
|---|
| 461 | + if (!early) |
|---|
| 462 | + write = s390_kernel_write; |
|---|
| 463 | + |
|---|
| 464 | + return __apply_relocate_add(sechdrs, strtab, symindex, relsec, me, |
|---|
| 465 | + write); |
|---|
| 431 | 466 | } |
|---|
| 432 | 467 | |
|---|
| 433 | 468 | int module_finalize(const Elf_Ehdr *hdr, |
|---|
| .. | .. |
|---|
| 465 | 500 | apply_alternatives(aseg, aseg + s->sh_size); |
|---|
| 466 | 501 | |
|---|
| 467 | 502 | if (IS_ENABLED(CONFIG_EXPOLINE) && |
|---|
| 468 | | - (!strncmp(".s390_indirect", secname, 14))) |
|---|
| 503 | + (str_has_prefix(secname, ".s390_indirect"))) |
|---|
| 469 | 504 | nospec_revert(aseg, aseg + s->sh_size); |
|---|
| 470 | 505 | |
|---|
| 471 | 506 | if (IS_ENABLED(CONFIG_EXPOLINE) && |
|---|
| 472 | | - (!strncmp(".s390_return", secname, 12))) |
|---|
| 507 | + (str_has_prefix(secname, ".s390_return"))) |
|---|
| 473 | 508 | nospec_revert(aseg, aseg + s->sh_size); |
|---|
| 474 | 509 | } |
|---|
| 475 | 510 | |
|---|