| .. | .. |
|---|
| 14 | 14 | #include <linux/kdebug.h> |
|---|
| 15 | 15 | #include <linux/prefetch.h> |
|---|
| 16 | 16 | #include <linux/uaccess.h> |
|---|
| 17 | +#include <linux/perf_event.h> |
|---|
| 17 | 18 | |
|---|
| 18 | | -#include <asm/pgtable.h> |
|---|
| 19 | 19 | #include <asm/processor.h> |
|---|
| 20 | 20 | #include <asm/exception.h> |
|---|
| 21 | 21 | |
|---|
| 22 | 22 | extern int die(char *, struct pt_regs *, long); |
|---|
| 23 | | - |
|---|
| 24 | | -#ifdef CONFIG_KPROBES |
|---|
| 25 | | -static inline int notify_page_fault(struct pt_regs *regs, int trap) |
|---|
| 26 | | -{ |
|---|
| 27 | | - int ret = 0; |
|---|
| 28 | | - |
|---|
| 29 | | - if (!user_mode(regs)) { |
|---|
| 30 | | - /* kprobe_running() needs smp_processor_id() */ |
|---|
| 31 | | - preempt_disable(); |
|---|
| 32 | | - if (kprobe_running() && kprobe_fault_handler(regs, trap)) |
|---|
| 33 | | - ret = 1; |
|---|
| 34 | | - preempt_enable(); |
|---|
| 35 | | - } |
|---|
| 36 | | - |
|---|
| 37 | | - return ret; |
|---|
| 38 | | -} |
|---|
| 39 | | -#else |
|---|
| 40 | | -static inline int notify_page_fault(struct pt_regs *regs, int trap) |
|---|
| 41 | | -{ |
|---|
| 42 | | - return 0; |
|---|
| 43 | | -} |
|---|
| 44 | | -#endif |
|---|
| 45 | 23 | |
|---|
| 46 | 24 | /* |
|---|
| 47 | 25 | * Return TRUE if ADDRESS points at a page in the kernel's mapped segment |
|---|
| .. | .. |
|---|
| 51 | 29 | mapped_kernel_page_is_present (unsigned long address) |
|---|
| 52 | 30 | { |
|---|
| 53 | 31 | pgd_t *pgd; |
|---|
| 32 | + p4d_t *p4d; |
|---|
| 54 | 33 | pud_t *pud; |
|---|
| 55 | 34 | pmd_t *pmd; |
|---|
| 56 | 35 | pte_t *ptep, pte; |
|---|
| .. | .. |
|---|
| 59 | 38 | if (pgd_none(*pgd) || pgd_bad(*pgd)) |
|---|
| 60 | 39 | return 0; |
|---|
| 61 | 40 | |
|---|
| 62 | | - pud = pud_offset(pgd, address); |
|---|
| 41 | + p4d = p4d_offset(pgd, address); |
|---|
| 42 | + if (p4d_none(*p4d) || p4d_bad(*p4d)) |
|---|
| 43 | + return 0; |
|---|
| 44 | + |
|---|
| 45 | + pud = pud_offset(p4d, address); |
|---|
| 63 | 46 | if (pud_none(*pud) || pud_bad(*pud)) |
|---|
| 64 | 47 | return 0; |
|---|
| 65 | 48 | |
|---|
| .. | .. |
|---|
| 87 | 70 | struct mm_struct *mm = current->mm; |
|---|
| 88 | 71 | unsigned long mask; |
|---|
| 89 | 72 | vm_fault_t fault; |
|---|
| 90 | | - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; |
|---|
| 73 | + unsigned int flags = FAULT_FLAG_DEFAULT; |
|---|
| 91 | 74 | |
|---|
| 92 | 75 | mask = ((((isr >> IA64_ISR_X_BIT) & 1UL) << VM_EXEC_BIT) |
|---|
| 93 | 76 | | (((isr >> IA64_ISR_W_BIT) & 1UL) << VM_WRITE_BIT)); |
|---|
| 94 | 77 | |
|---|
| 95 | | - /* mmap_sem is performance critical.... */ |
|---|
| 96 | | - prefetchw(&mm->mmap_sem); |
|---|
| 78 | + /* mmap_lock is performance critical.... */ |
|---|
| 79 | + prefetchw(&mm->mmap_lock); |
|---|
| 97 | 80 | |
|---|
| 98 | 81 | /* |
|---|
| 99 | 82 | * If we're in an interrupt or have no user context, we must not take the fault.. |
|---|
| .. | .. |
|---|
| 104 | 87 | #ifdef CONFIG_VIRTUAL_MEM_MAP |
|---|
| 105 | 88 | /* |
|---|
| 106 | 89 | * If fault is in region 5 and we are in the kernel, we may already |
|---|
| 107 | | - * have the mmap_sem (pfn_valid macro is called during mmap). There |
|---|
| 90 | + * have the mmap_lock (pfn_valid macro is called during mmap). There |
|---|
| 108 | 91 | * is no vma for region 5 addr's anyway, so skip getting the semaphore |
|---|
| 109 | 92 | * and go directly to the exception handling code. |
|---|
| 110 | 93 | */ |
|---|
| .. | .. |
|---|
| 116 | 99 | /* |
|---|
| 117 | 100 | * This is to handle the kprobes on user space access instructions |
|---|
| 118 | 101 | */ |
|---|
| 119 | | - if (notify_page_fault(regs, TRAP_BRKPT)) |
|---|
| 102 | + if (kprobe_page_fault(regs, TRAP_BRKPT)) |
|---|
| 120 | 103 | return; |
|---|
| 121 | 104 | |
|---|
| 122 | 105 | if (user_mode(regs)) |
|---|
| 123 | 106 | flags |= FAULT_FLAG_USER; |
|---|
| 124 | 107 | if (mask & VM_WRITE) |
|---|
| 125 | 108 | flags |= FAULT_FLAG_WRITE; |
|---|
| 109 | + |
|---|
| 110 | + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
|---|
| 126 | 111 | retry: |
|---|
| 127 | | - down_read(&mm->mmap_sem); |
|---|
| 112 | + mmap_read_lock(mm); |
|---|
| 128 | 113 | |
|---|
| 129 | 114 | vma = find_vma_prev(mm, address, &prev_vma); |
|---|
| 130 | 115 | if (!vma && !prev_vma ) |
|---|
| .. | .. |
|---|
| 161 | 146 | * sure we exit gracefully rather than endlessly redo the |
|---|
| 162 | 147 | * fault. |
|---|
| 163 | 148 | */ |
|---|
| 164 | | - fault = handle_mm_fault(vma, address, flags); |
|---|
| 149 | + fault = handle_mm_fault(vma, address, flags, regs); |
|---|
| 165 | 150 | |
|---|
| 166 | | - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) |
|---|
| 151 | + if (fault_signal_pending(fault, regs)) |
|---|
| 167 | 152 | return; |
|---|
| 168 | 153 | |
|---|
| 169 | 154 | if (unlikely(fault & VM_FAULT_ERROR)) { |
|---|
| .. | .. |
|---|
| 184 | 169 | } |
|---|
| 185 | 170 | |
|---|
| 186 | 171 | if (flags & FAULT_FLAG_ALLOW_RETRY) { |
|---|
| 187 | | - if (fault & VM_FAULT_MAJOR) |
|---|
| 188 | | - current->maj_flt++; |
|---|
| 189 | | - else |
|---|
| 190 | | - current->min_flt++; |
|---|
| 191 | 172 | if (fault & VM_FAULT_RETRY) { |
|---|
| 192 | | - flags &= ~FAULT_FLAG_ALLOW_RETRY; |
|---|
| 193 | 173 | flags |= FAULT_FLAG_TRIED; |
|---|
| 194 | 174 | |
|---|
| 195 | | - /* No need to up_read(&mm->mmap_sem) as we would |
|---|
| 175 | + /* No need to mmap_read_unlock(mm) as we would |
|---|
| 196 | 176 | * have already released it in __lock_page_or_retry |
|---|
| 197 | 177 | * in mm/filemap.c. |
|---|
| 198 | 178 | */ |
|---|
| .. | .. |
|---|
| 201 | 181 | } |
|---|
| 202 | 182 | } |
|---|
| 203 | 183 | |
|---|
| 204 | | - up_read(&mm->mmap_sem); |
|---|
| 184 | + mmap_read_unlock(mm); |
|---|
| 205 | 185 | return; |
|---|
| 206 | 186 | |
|---|
| 207 | 187 | check_expansion: |
|---|
| .. | .. |
|---|
| 232 | 212 | goto good_area; |
|---|
| 233 | 213 | |
|---|
| 234 | 214 | bad_area: |
|---|
| 235 | | - up_read(&mm->mmap_sem); |
|---|
| 215 | + mmap_read_unlock(mm); |
|---|
| 236 | 216 | #ifdef CONFIG_VIRTUAL_MEM_MAP |
|---|
| 237 | 217 | bad_area_no_up: |
|---|
| 238 | 218 | #endif |
|---|
| .. | .. |
|---|
| 248 | 228 | return; |
|---|
| 249 | 229 | } |
|---|
| 250 | 230 | if (user_mode(regs)) { |
|---|
| 251 | | - struct siginfo si; |
|---|
| 252 | | - |
|---|
| 253 | | - clear_siginfo(&si); |
|---|
| 254 | | - si.si_signo = signal; |
|---|
| 255 | | - si.si_errno = 0; |
|---|
| 256 | | - si.si_code = code; |
|---|
| 257 | | - si.si_addr = (void __user *) address; |
|---|
| 258 | | - si.si_isr = isr; |
|---|
| 259 | | - si.si_flags = __ISR_VALID; |
|---|
| 260 | | - force_sig_info(signal, &si, current); |
|---|
| 231 | + force_sig_fault(signal, code, (void __user *) address, |
|---|
| 232 | + 0, __ISR_VALID, isr); |
|---|
| 261 | 233 | return; |
|---|
| 262 | 234 | } |
|---|
| 263 | 235 | |
|---|
| .. | .. |
|---|
| 302 | 274 | regs = NULL; |
|---|
| 303 | 275 | bust_spinlocks(0); |
|---|
| 304 | 276 | if (regs) |
|---|
| 305 | | - do_exit(SIGKILL); |
|---|
| 277 | + make_task_dead(SIGKILL); |
|---|
| 306 | 278 | return; |
|---|
| 307 | 279 | |
|---|
| 308 | 280 | out_of_memory: |
|---|
| 309 | | - up_read(&mm->mmap_sem); |
|---|
| 281 | + mmap_read_unlock(mm); |
|---|
| 310 | 282 | if (!user_mode(regs)) |
|---|
| 311 | 283 | goto no_context; |
|---|
| 312 | 284 | pagefault_out_of_memory(); |
|---|