.. | .. |
---|
28 | 28 | #include <linux/mman.h> |
---|
29 | 29 | #include <linux/mm.h> |
---|
30 | 30 | #include <linux/interrupt.h> |
---|
| 31 | +#include <linux/perf_event.h> |
---|
31 | 32 | |
---|
32 | 33 | #include <asm/page.h> |
---|
33 | | -#include <asm/pgtable.h> |
---|
34 | 34 | #include <asm/mmu.h> |
---|
35 | 35 | #include <linux/mmu_context.h> |
---|
36 | 36 | #include <linux/uaccess.h> |
---|
.. | .. |
---|
91 | 91 | int code = SEGV_MAPERR; |
---|
92 | 92 | int is_write = error_code & ESR_S; |
---|
93 | 93 | vm_fault_t fault; |
---|
94 | | - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; |
---|
| 94 | + unsigned int flags = FAULT_FLAG_DEFAULT; |
---|
95 | 95 | |
---|
96 | 96 | regs->ear = address; |
---|
97 | 97 | regs->esr = error_code; |
---|
.. | .. |
---|
122 | 122 | if (user_mode(regs)) |
---|
123 | 123 | flags |= FAULT_FLAG_USER; |
---|
124 | 124 | |
---|
| 125 | + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
---|
| 126 | + |
---|
125 | 127 | /* When running in the kernel we expect faults to occur only to |
---|
126 | 128 | * addresses in user space. All other faults represent errors in the |
---|
127 | 129 | * kernel and should generate an OOPS. Unfortunately, in the case of an |
---|
128 | | - * erroneous fault occurring in a code path which already holds mmap_sem |
---|
| 130 | + * erroneous fault occurring in a code path which already holds mmap_lock |
---|
129 | 131 | * we will deadlock attempting to validate the fault against the |
---|
130 | 132 | * address space. Luckily the kernel only validly references user |
---|
131 | 133 | * space from well defined areas of code, which are listed in the |
---|
.. | .. |
---|
137 | 139 | * source. If this is invalid we can skip the address space check, |
---|
138 | 140 | * thus avoiding the deadlock. |
---|
139 | 141 | */ |
---|
140 | | - if (unlikely(!down_read_trylock(&mm->mmap_sem))) { |
---|
| 142 | + if (unlikely(!mmap_read_trylock(mm))) { |
---|
141 | 143 | if (kernel_mode(regs) && !search_exception_tables(regs->pc)) |
---|
142 | 144 | goto bad_area_nosemaphore; |
---|
143 | 145 | |
---|
144 | 146 | retry: |
---|
145 | | - down_read(&mm->mmap_sem); |
---|
| 147 | + mmap_read_lock(mm); |
---|
146 | 148 | } |
---|
147 | 149 | |
---|
148 | 150 | vma = find_vma(mm, address); |
---|
.. | .. |
---|
215 | 217 | * make sure we exit gracefully rather than endlessly redo |
---|
216 | 218 | * the fault. |
---|
217 | 219 | */ |
---|
218 | | - fault = handle_mm_fault(vma, address, flags); |
---|
| 220 | + fault = handle_mm_fault(vma, address, flags, regs); |
---|
219 | 221 | |
---|
220 | | - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) |
---|
| 222 | + if (fault_signal_pending(fault, regs)) |
---|
221 | 223 | return; |
---|
222 | 224 | |
---|
223 | 225 | if (unlikely(fault & VM_FAULT_ERROR)) { |
---|
.. | .. |
---|
231 | 233 | } |
---|
232 | 234 | |
---|
233 | 235 | if (flags & FAULT_FLAG_ALLOW_RETRY) { |
---|
234 | | - if (unlikely(fault & VM_FAULT_MAJOR)) |
---|
235 | | - current->maj_flt++; |
---|
236 | | - else |
---|
237 | | - current->min_flt++; |
---|
238 | 236 | if (fault & VM_FAULT_RETRY) { |
---|
239 | | - flags &= ~FAULT_FLAG_ALLOW_RETRY; |
---|
240 | 237 | flags |= FAULT_FLAG_TRIED; |
---|
241 | 238 | |
---|
242 | 239 | /* |
---|
243 | | - * No need to up_read(&mm->mmap_sem) as we would |
---|
| 240 | + * No need to mmap_read_unlock(mm) as we would |
---|
244 | 241 | * have already released it in __lock_page_or_retry |
---|
245 | 242 | * in mm/filemap.c. |
---|
246 | 243 | */ |
---|
.. | .. |
---|
249 | 246 | } |
---|
250 | 247 | } |
---|
251 | 248 | |
---|
252 | | - up_read(&mm->mmap_sem); |
---|
| 249 | + mmap_read_unlock(mm); |
---|
253 | 250 | |
---|
254 | 251 | /* |
---|
255 | 252 | * keep track of tlb+htab misses that are good addrs but |
---|
.. | .. |
---|
260 | 257 | return; |
---|
261 | 258 | |
---|
262 | 259 | bad_area: |
---|
263 | | - up_read(&mm->mmap_sem); |
---|
| 260 | + mmap_read_unlock(mm); |
---|
264 | 261 | |
---|
265 | 262 | bad_area_nosemaphore: |
---|
266 | 263 | pte_errors++; |
---|
.. | .. |
---|
279 | 276 | * us unable to handle the page fault gracefully. |
---|
280 | 277 | */ |
---|
281 | 278 | out_of_memory: |
---|
282 | | - up_read(&mm->mmap_sem); |
---|
| 279 | + mmap_read_unlock(mm); |
---|
283 | 280 | if (!user_mode(regs)) |
---|
284 | 281 | bad_page_fault(regs, address, SIGKILL); |
---|
285 | 282 | else |
---|
.. | .. |
---|
287 | 284 | return; |
---|
288 | 285 | |
---|
289 | 286 | do_sigbus: |
---|
290 | | - up_read(&mm->mmap_sem); |
---|
| 287 | + mmap_read_unlock(mm); |
---|
291 | 288 | if (user_mode(regs)) { |
---|
292 | | - force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address, current); |
---|
| 289 | + force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); |
---|
293 | 290 | return; |
---|
294 | 291 | } |
---|
295 | 292 | bad_page_fault(regs, address, SIGBUS); |
---|