| .. | .. |
|---|
| 12 | 12 | #include <linux/interrupt.h> |
|---|
| 13 | 13 | #include <linux/module.h> |
|---|
| 14 | 14 | #include <linux/uaccess.h> |
|---|
| 15 | +#include <linux/perf_event.h> |
|---|
| 15 | 16 | |
|---|
| 16 | 17 | #include <asm/setup.h> |
|---|
| 17 | 18 | #include <asm/traps.h> |
|---|
| 18 | | -#include <asm/pgalloc.h> |
|---|
| 19 | 19 | |
|---|
| 20 | 20 | extern void die_if_kernel(char *, struct pt_regs *, long); |
|---|
| 21 | 21 | |
|---|
| .. | .. |
|---|
| 30 | 30 | pr_debug("send_fault_sig: %p,%d,%d\n", addr, signo, si_code); |
|---|
| 31 | 31 | |
|---|
| 32 | 32 | if (user_mode(regs)) { |
|---|
| 33 | | - force_sig_fault(signo, si_code, addr, current); |
|---|
| 33 | + force_sig_fault(signo, si_code, addr); |
|---|
| 34 | 34 | } else { |
|---|
| 35 | 35 | if (fixup_exception(regs)) |
|---|
| 36 | 36 | return -1; |
|---|
| 37 | 37 | |
|---|
| 38 | 38 | //if (signo == SIGBUS) |
|---|
| 39 | | - // force_sig_fault(si_signo, si_code, addr, current); |
|---|
| 39 | + // force_sig_fault(si_signo, si_code, addr); |
|---|
| 40 | 40 | |
|---|
| 41 | 41 | /* |
|---|
| 42 | 42 | * Oops. The kernel tried to access some bad page. We'll have to |
|---|
| .. | .. |
|---|
| 48 | 48 | pr_alert("Unable to handle kernel access"); |
|---|
| 49 | 49 | pr_cont(" at virtual address %p\n", addr); |
|---|
| 50 | 50 | die_if_kernel("Oops", regs, 0 /*error_code*/); |
|---|
| 51 | | - do_exit(SIGKILL); |
|---|
| 51 | + make_task_dead(SIGKILL); |
|---|
| 52 | 52 | } |
|---|
| 53 | 53 | |
|---|
| 54 | 54 | return 1; |
|---|
| .. | .. |
|---|
| 71 | 71 | struct mm_struct *mm = current->mm; |
|---|
| 72 | 72 | struct vm_area_struct * vma; |
|---|
| 73 | 73 | vm_fault_t fault; |
|---|
| 74 | | - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; |
|---|
| 74 | + unsigned int flags = FAULT_FLAG_DEFAULT; |
|---|
| 75 | 75 | |
|---|
| 76 | 76 | pr_debug("do page fault:\nregs->sr=%#x, regs->pc=%#lx, address=%#lx, %ld, %p\n", |
|---|
| 77 | 77 | regs->sr, regs->pc, address, error_code, mm ? mm->pgd : NULL); |
|---|
| .. | .. |
|---|
| 85 | 85 | |
|---|
| 86 | 86 | if (user_mode(regs)) |
|---|
| 87 | 87 | flags |= FAULT_FLAG_USER; |
|---|
| 88 | + |
|---|
| 89 | + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
|---|
| 88 | 90 | retry: |
|---|
| 89 | | - down_read(&mm->mmap_sem); |
|---|
| 91 | + mmap_read_lock(mm); |
|---|
| 90 | 92 | |
|---|
| 91 | 93 | vma = find_vma(mm, address); |
|---|
| 92 | 94 | if (!vma) |
|---|
| .. | .. |
|---|
| 116 | 118 | pr_debug("do_page_fault: good_area\n"); |
|---|
| 117 | 119 | switch (error_code & 3) { |
|---|
| 118 | 120 | default: /* 3: write, present */ |
|---|
| 119 | | - /* fall through */ |
|---|
| 121 | + fallthrough; |
|---|
| 120 | 122 | case 2: /* write, not present */ |
|---|
| 121 | 123 | if (!(vma->vm_flags & VM_WRITE)) |
|---|
| 122 | 124 | goto acc_err; |
|---|
| .. | .. |
|---|
| 125 | 127 | case 1: /* read, present */ |
|---|
| 126 | 128 | goto acc_err; |
|---|
| 127 | 129 | case 0: /* read, not present */ |
|---|
| 128 | | - if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) |
|---|
| 130 | + if (unlikely(!vma_is_accessible(vma))) |
|---|
| 129 | 131 | goto acc_err; |
|---|
| 130 | 132 | } |
|---|
| 131 | 133 | |
|---|
| .. | .. |
|---|
| 135 | 137 | * the fault. |
|---|
| 136 | 138 | */ |
|---|
| 137 | 139 | |
|---|
| 138 | | - fault = handle_mm_fault(vma, address, flags); |
|---|
| 140 | + fault = handle_mm_fault(vma, address, flags, regs); |
|---|
| 139 | 141 | pr_debug("handle_mm_fault returns %x\n", fault); |
|---|
| 140 | 142 | |
|---|
| 141 | | - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) |
|---|
| 143 | + if (fault_signal_pending(fault, regs)) |
|---|
| 142 | 144 | return 0; |
|---|
| 143 | 145 | |
|---|
| 144 | 146 | if (unlikely(fault & VM_FAULT_ERROR)) { |
|---|
| .. | .. |
|---|
| 151 | 153 | BUG(); |
|---|
| 152 | 154 | } |
|---|
| 153 | 155 | |
|---|
| 154 | | - /* |
|---|
| 155 | | - * Major/minor page fault accounting is only done on the |
|---|
| 156 | | - * initial attempt. If we go through a retry, it is extremely |
|---|
| 157 | | - * likely that the page will be found in page cache at that point. |
|---|
| 158 | | - */ |
|---|
| 159 | 156 | if (flags & FAULT_FLAG_ALLOW_RETRY) { |
|---|
| 160 | | - if (fault & VM_FAULT_MAJOR) |
|---|
| 161 | | - current->maj_flt++; |
|---|
| 162 | | - else |
|---|
| 163 | | - current->min_flt++; |
|---|
| 164 | 157 | if (fault & VM_FAULT_RETRY) { |
|---|
| 165 | | - /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk |
|---|
| 166 | | - * of starvation. */ |
|---|
| 167 | | - flags &= ~FAULT_FLAG_ALLOW_RETRY; |
|---|
| 168 | 158 | flags |= FAULT_FLAG_TRIED; |
|---|
| 169 | 159 | |
|---|
| 170 | 160 | /* |
|---|
| 171 | | - * No need to up_read(&mm->mmap_sem) as we would |
|---|
| 161 | + * No need to mmap_read_unlock(mm) as we would |
|---|
| 172 | 162 | * have already released it in __lock_page_or_retry |
|---|
| 173 | 163 | * in mm/filemap.c. |
|---|
| 174 | 164 | */ |
|---|
| .. | .. |
|---|
| 177 | 167 | } |
|---|
| 178 | 168 | } |
|---|
| 179 | 169 | |
|---|
| 180 | | - up_read(&mm->mmap_sem); |
|---|
| 170 | + mmap_read_unlock(mm); |
|---|
| 181 | 171 | return 0; |
|---|
| 182 | 172 | |
|---|
| 183 | 173 | /* |
|---|
| .. | .. |
|---|
| 185 | 175 | * us unable to handle the page fault gracefully. |
|---|
| 186 | 176 | */ |
|---|
| 187 | 177 | out_of_memory: |
|---|
| 188 | | - up_read(&mm->mmap_sem); |
|---|
| 178 | + mmap_read_unlock(mm); |
|---|
| 189 | 179 | if (!user_mode(regs)) |
|---|
| 190 | 180 | goto no_context; |
|---|
| 191 | 181 | pagefault_out_of_memory(); |
|---|
| .. | .. |
|---|
| 214 | 204 | current->thread.faddr = address; |
|---|
| 215 | 205 | |
|---|
| 216 | 206 | send_sig: |
|---|
| 217 | | - up_read(&mm->mmap_sem); |
|---|
| 207 | + mmap_read_unlock(mm); |
|---|
| 218 | 208 | return send_fault_sig(regs); |
|---|
| 219 | 209 | } |
|---|