| .. | .. |
|---|
| 28 | 28 | #include <linux/mman.h> |
|---|
| 29 | 29 | #include <linux/mm.h> |
|---|
| 30 | 30 | #include <linux/interrupt.h> |
|---|
| 31 | +#include <linux/perf_event.h> |
|---|
| 31 | 32 | |
|---|
| 32 | 33 | #include <asm/page.h> |
|---|
| 33 | | -#include <asm/pgtable.h> |
|---|
| 34 | 34 | #include <asm/mmu.h> |
|---|
| 35 | 35 | #include <linux/mmu_context.h> |
|---|
| 36 | 36 | #include <linux/uaccess.h> |
|---|
| .. | .. |
|---|
| 91 | 91 | int code = SEGV_MAPERR; |
|---|
| 92 | 92 | int is_write = error_code & ESR_S; |
|---|
| 93 | 93 | vm_fault_t fault; |
|---|
| 94 | | - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; |
|---|
| 94 | + unsigned int flags = FAULT_FLAG_DEFAULT; |
|---|
| 95 | 95 | |
|---|
| 96 | 96 | regs->ear = address; |
|---|
| 97 | 97 | regs->esr = error_code; |
|---|
| .. | .. |
|---|
| 122 | 122 | if (user_mode(regs)) |
|---|
| 123 | 123 | flags |= FAULT_FLAG_USER; |
|---|
| 124 | 124 | |
|---|
| 125 | + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
|---|
| 126 | + |
|---|
| 125 | 127 | /* When running in the kernel we expect faults to occur only to |
|---|
| 126 | 128 | * addresses in user space. All other faults represent errors in the |
|---|
| 127 | 129 | * kernel and should generate an OOPS. Unfortunately, in the case of an |
|---|
| 128 | | - * erroneous fault occurring in a code path which already holds mmap_sem |
|---|
| 130 | + * erroneous fault occurring in a code path which already holds mmap_lock |
|---|
| 129 | 131 | * we will deadlock attempting to validate the fault against the |
|---|
| 130 | 132 | * address space. Luckily the kernel only validly references user |
|---|
| 131 | 133 | * space from well defined areas of code, which are listed in the |
|---|
| .. | .. |
|---|
| 137 | 139 | * source. If this is invalid we can skip the address space check, |
|---|
| 138 | 140 | * thus avoiding the deadlock. |
|---|
| 139 | 141 | */ |
|---|
| 140 | | - if (unlikely(!down_read_trylock(&mm->mmap_sem))) { |
|---|
| 142 | + if (unlikely(!mmap_read_trylock(mm))) { |
|---|
| 141 | 143 | if (kernel_mode(regs) && !search_exception_tables(regs->pc)) |
|---|
| 142 | 144 | goto bad_area_nosemaphore; |
|---|
| 143 | 145 | |
|---|
| 144 | 146 | retry: |
|---|
| 145 | | - down_read(&mm->mmap_sem); |
|---|
| 147 | + mmap_read_lock(mm); |
|---|
| 146 | 148 | } |
|---|
| 147 | 149 | |
|---|
| 148 | 150 | vma = find_vma(mm, address); |
|---|
| .. | .. |
|---|
| 215 | 217 | * make sure we exit gracefully rather than endlessly redo |
|---|
| 216 | 218 | * the fault. |
|---|
| 217 | 219 | */ |
|---|
| 218 | | - fault = handle_mm_fault(vma, address, flags); |
|---|
| 220 | + fault = handle_mm_fault(vma, address, flags, regs); |
|---|
| 219 | 221 | |
|---|
| 220 | | - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) |
|---|
| 222 | + if (fault_signal_pending(fault, regs)) |
|---|
| 221 | 223 | return; |
|---|
| 222 | 224 | |
|---|
| 223 | 225 | if (unlikely(fault & VM_FAULT_ERROR)) { |
|---|
| .. | .. |
|---|
| 231 | 233 | } |
|---|
| 232 | 234 | |
|---|
| 233 | 235 | if (flags & FAULT_FLAG_ALLOW_RETRY) { |
|---|
| 234 | | - if (unlikely(fault & VM_FAULT_MAJOR)) |
|---|
| 235 | | - current->maj_flt++; |
|---|
| 236 | | - else |
|---|
| 237 | | - current->min_flt++; |
|---|
| 238 | 236 | if (fault & VM_FAULT_RETRY) { |
|---|
| 239 | | - flags &= ~FAULT_FLAG_ALLOW_RETRY; |
|---|
| 240 | 237 | flags |= FAULT_FLAG_TRIED; |
|---|
| 241 | 238 | |
|---|
| 242 | 239 | /* |
|---|
| 243 | | - * No need to up_read(&mm->mmap_sem) as we would |
|---|
| 240 | + * No need to mmap_read_unlock(mm) as we would |
|---|
| 244 | 241 | * have already released it in __lock_page_or_retry |
|---|
| 245 | 242 | * in mm/filemap.c. |
|---|
| 246 | 243 | */ |
|---|
| .. | .. |
|---|
| 249 | 246 | } |
|---|
| 250 | 247 | } |
|---|
| 251 | 248 | |
|---|
| 252 | | - up_read(&mm->mmap_sem); |
|---|
| 249 | + mmap_read_unlock(mm); |
|---|
| 253 | 250 | |
|---|
| 254 | 251 | /* |
|---|
| 255 | 252 | * keep track of tlb+htab misses that are good addrs but |
|---|
| .. | .. |
|---|
| 260 | 257 | return; |
|---|
| 261 | 258 | |
|---|
| 262 | 259 | bad_area: |
|---|
| 263 | | - up_read(&mm->mmap_sem); |
|---|
| 260 | + mmap_read_unlock(mm); |
|---|
| 264 | 261 | |
|---|
| 265 | 262 | bad_area_nosemaphore: |
|---|
| 266 | 263 | pte_errors++; |
|---|
| .. | .. |
|---|
| 279 | 276 | * us unable to handle the page fault gracefully. |
|---|
| 280 | 277 | */ |
|---|
| 281 | 278 | out_of_memory: |
|---|
| 282 | | - up_read(&mm->mmap_sem); |
|---|
| 279 | + mmap_read_unlock(mm); |
|---|
| 283 | 280 | if (!user_mode(regs)) |
|---|
| 284 | 281 | bad_page_fault(regs, address, SIGKILL); |
|---|
| 285 | 282 | else |
|---|
| .. | .. |
|---|
| 287 | 284 | return; |
|---|
| 288 | 285 | |
|---|
| 289 | 286 | do_sigbus: |
|---|
| 290 | | - up_read(&mm->mmap_sem); |
|---|
| 287 | + mmap_read_unlock(mm); |
|---|
| 291 | 288 | if (user_mode(regs)) { |
|---|
| 292 | | - force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address, current); |
|---|
| 289 | + force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); |
|---|
| 293 | 290 | return; |
|---|
| 294 | 291 | } |
|---|
| 295 | 292 | bad_page_fault(regs, address, SIGBUS); |
|---|