.. | .. |
---|
27 | 27 | #include <linux/uaccess.h> |
---|
28 | 28 | |
---|
29 | 29 | #include <asm/page.h> |
---|
30 | | -#include <asm/pgtable.h> |
---|
31 | 30 | #include <asm/openprom.h> |
---|
32 | 31 | #include <asm/oplib.h> |
---|
33 | 32 | #include <asm/asi.h> |
---|
.. | .. |
---|
37 | 36 | #include <asm/setup.h> |
---|
38 | 37 | |
---|
39 | 38 | int show_unhandled_signals = 1; |
---|
40 | | - |
---|
41 | | -static inline __kprobes int notify_page_fault(struct pt_regs *regs) |
---|
42 | | -{ |
---|
43 | | - int ret = 0; |
---|
44 | | - |
---|
45 | | - /* kprobe_running() needs smp_processor_id() */ |
---|
46 | | - if (kprobes_built_in() && !user_mode(regs)) { |
---|
47 | | - preempt_disable(); |
---|
48 | | - if (kprobe_running() && kprobe_fault_handler(regs, 0)) |
---|
49 | | - ret = 1; |
---|
50 | | - preempt_enable(); |
---|
51 | | - } |
---|
52 | | - return ret; |
---|
53 | | -} |
---|
54 | 39 | |
---|
55 | 40 | static void __kprobes unhandled_fault(unsigned long address, |
---|
56 | 41 | struct task_struct *tsk, |
---|
.. | .. |
---|
85 | 70 | } |
---|
86 | 71 | |
---|
87 | 72 | /* |
---|
88 | | - * We now make sure that mmap_sem is held in all paths that call |
---|
| 73 | + * We now make sure that mmap_lock is held in all paths that call |
---|
89 | 74 | * this. Additionally, to prevent kswapd from ripping ptes from |
---|
90 | 75 | * under us, raise interrupts around the time that we look at the |
---|
91 | 76 | * pte, kswapd will have to wait to get his smp ipi response from |
---|
.. | .. |
---|
94 | 79 | static unsigned int get_user_insn(unsigned long tpc) |
---|
95 | 80 | { |
---|
96 | 81 | pgd_t *pgdp = pgd_offset(current->mm, tpc); |
---|
| 82 | + p4d_t *p4dp; |
---|
97 | 83 | pud_t *pudp; |
---|
98 | 84 | pmd_t *pmdp; |
---|
99 | 85 | pte_t *ptep, pte; |
---|
.. | .. |
---|
102 | 88 | |
---|
103 | 89 | if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) |
---|
104 | 90 | goto out; |
---|
105 | | - pudp = pud_offset(pgdp, tpc); |
---|
| 91 | + p4dp = p4d_offset(pgdp, tpc); |
---|
| 92 | + if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp))) |
---|
| 93 | + goto out; |
---|
| 94 | + pudp = pud_offset(p4dp, tpc); |
---|
106 | 95 | if (pud_none(*pudp) || unlikely(pud_bad(*pudp))) |
---|
107 | 96 | goto out; |
---|
108 | 97 | |
---|
.. | .. |
---|
187 | 176 | if (unlikely(show_unhandled_signals)) |
---|
188 | 177 | show_signal_msg(regs, sig, code, addr, current); |
---|
189 | 178 | |
---|
190 | | - force_sig_fault(sig, code, (void __user *) addr, 0, current); |
---|
| 179 | + force_sig_fault(sig, code, (void __user *) addr, 0); |
---|
191 | 180 | } |
---|
192 | 181 | |
---|
193 | 182 | static unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn) |
---|
.. | .. |
---|
281 | 270 | int si_code, fault_code; |
---|
282 | 271 | vm_fault_t fault; |
---|
283 | 272 | unsigned long address, mm_rss; |
---|
284 | | - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; |
---|
| 273 | + unsigned int flags = FAULT_FLAG_DEFAULT; |
---|
285 | 274 | |
---|
286 | 275 | fault_code = get_thread_fault_code(); |
---|
287 | 276 | |
---|
288 | | - if (notify_page_fault(regs)) |
---|
| 277 | + if (kprobe_page_fault(regs, 0)) |
---|
289 | 278 | goto exit_exception; |
---|
290 | 279 | |
---|
291 | 280 | si_code = SEGV_MAPERR; |
---|
.. | .. |
---|
329 | 318 | |
---|
330 | 319 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
---|
331 | 320 | |
---|
332 | | - if (!down_read_trylock(&mm->mmap_sem)) { |
---|
| 321 | + if (!mmap_read_trylock(mm)) { |
---|
333 | 322 | if ((regs->tstate & TSTATE_PRIV) && |
---|
334 | 323 | !search_exception_tables(regs->tpc)) { |
---|
335 | 324 | insn = get_fault_insn(regs, insn); |
---|
.. | .. |
---|
337 | 326 | } |
---|
338 | 327 | |
---|
339 | 328 | retry: |
---|
340 | | - down_read(&mm->mmap_sem); |
---|
| 329 | + mmap_read_lock(mm); |
---|
341 | 330 | } |
---|
342 | 331 | |
---|
343 | 332 | if (fault_code & FAULT_CODE_BAD_RA) |
---|
.. | .. |
---|
433 | 422 | goto bad_area; |
---|
434 | 423 | } |
---|
435 | 424 | |
---|
436 | | - fault = handle_mm_fault(vma, address, flags); |
---|
| 425 | + fault = handle_mm_fault(vma, address, flags, regs); |
---|
437 | 426 | |
---|
438 | | - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) |
---|
| 427 | + if (fault_signal_pending(fault, regs)) |
---|
439 | 428 | goto exit_exception; |
---|
440 | 429 | |
---|
441 | 430 | if (unlikely(fault & VM_FAULT_ERROR)) { |
---|
.. | .. |
---|
449 | 438 | } |
---|
450 | 439 | |
---|
451 | 440 | if (flags & FAULT_FLAG_ALLOW_RETRY) { |
---|
452 | | - if (fault & VM_FAULT_MAJOR) { |
---|
453 | | - current->maj_flt++; |
---|
454 | | - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, |
---|
455 | | - 1, regs, address); |
---|
456 | | - } else { |
---|
457 | | - current->min_flt++; |
---|
458 | | - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, |
---|
459 | | - 1, regs, address); |
---|
460 | | - } |
---|
461 | 441 | if (fault & VM_FAULT_RETRY) { |
---|
462 | | - flags &= ~FAULT_FLAG_ALLOW_RETRY; |
---|
463 | 442 | flags |= FAULT_FLAG_TRIED; |
---|
464 | 443 | |
---|
465 | | - /* No need to up_read(&mm->mmap_sem) as we would |
---|
| 444 | + /* No need to mmap_read_unlock(mm) as we would |
---|
466 | 445 | * have already released it in __lock_page_or_retry |
---|
467 | 446 | * in mm/filemap.c. |
---|
468 | 447 | */ |
---|
.. | .. |
---|
470 | 449 | goto retry; |
---|
471 | 450 | } |
---|
472 | 451 | } |
---|
473 | | - up_read(&mm->mmap_sem); |
---|
| 452 | + mmap_read_unlock(mm); |
---|
474 | 453 | |
---|
475 | 454 | mm_rss = get_mm_rss(mm); |
---|
476 | 455 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) |
---|
.. | .. |
---|
501 | 480 | */ |
---|
502 | 481 | bad_area: |
---|
503 | 482 | insn = get_fault_insn(regs, insn); |
---|
504 | | - up_read(&mm->mmap_sem); |
---|
| 483 | + mmap_read_unlock(mm); |
---|
505 | 484 | |
---|
506 | 485 | handle_kernel_fault: |
---|
507 | 486 | do_kernel_fault(regs, si_code, fault_code, insn, address); |
---|
.. | .. |
---|
513 | 492 | */ |
---|
514 | 493 | out_of_memory: |
---|
515 | 494 | insn = get_fault_insn(regs, insn); |
---|
516 | | - up_read(&mm->mmap_sem); |
---|
| 495 | + mmap_read_unlock(mm); |
---|
517 | 496 | if (!(regs->tstate & TSTATE_PRIV)) { |
---|
518 | 497 | pagefault_out_of_memory(); |
---|
519 | 498 | goto exit_exception; |
---|
.. | .. |
---|
526 | 505 | |
---|
527 | 506 | do_sigbus: |
---|
528 | 507 | insn = get_fault_insn(regs, insn); |
---|
529 | | - up_read(&mm->mmap_sem); |
---|
| 508 | + mmap_read_unlock(mm); |
---|
530 | 509 | |
---|
531 | 510 | /* |
---|
532 | 511 | * Send a sigbus, regardless of whether we were in kernel |
---|