.. | .. |
---|
14 | 14 | #include <linux/kdebug.h> |
---|
15 | 15 | #include <linux/prefetch.h> |
---|
16 | 16 | #include <linux/uaccess.h> |
---|
| 17 | +#include <linux/perf_event.h> |
---|
17 | 18 | |
---|
18 | | -#include <asm/pgtable.h> |
---|
19 | 19 | #include <asm/processor.h> |
---|
20 | 20 | #include <asm/exception.h> |
---|
21 | 21 | |
---|
22 | 22 | extern int die(char *, struct pt_regs *, long); |
---|
23 | | - |
---|
24 | | -#ifdef CONFIG_KPROBES |
---|
25 | | -static inline int notify_page_fault(struct pt_regs *regs, int trap) |
---|
26 | | -{ |
---|
27 | | - int ret = 0; |
---|
28 | | - |
---|
29 | | - if (!user_mode(regs)) { |
---|
30 | | - /* kprobe_running() needs smp_processor_id() */ |
---|
31 | | - preempt_disable(); |
---|
32 | | - if (kprobe_running() && kprobe_fault_handler(regs, trap)) |
---|
33 | | - ret = 1; |
---|
34 | | - preempt_enable(); |
---|
35 | | - } |
---|
36 | | - |
---|
37 | | - return ret; |
---|
38 | | -} |
---|
39 | | -#else |
---|
40 | | -static inline int notify_page_fault(struct pt_regs *regs, int trap) |
---|
41 | | -{ |
---|
42 | | - return 0; |
---|
43 | | -} |
---|
44 | | -#endif |
---|
45 | 23 | |
---|
46 | 24 | /* |
---|
47 | 25 | * Return TRUE if ADDRESS points at a page in the kernel's mapped segment |
---|
.. | .. |
---|
51 | 29 | mapped_kernel_page_is_present (unsigned long address) |
---|
52 | 30 | { |
---|
53 | 31 | pgd_t *pgd; |
---|
| 32 | + p4d_t *p4d; |
---|
54 | 33 | pud_t *pud; |
---|
55 | 34 | pmd_t *pmd; |
---|
56 | 35 | pte_t *ptep, pte; |
---|
.. | .. |
---|
59 | 38 | if (pgd_none(*pgd) || pgd_bad(*pgd)) |
---|
60 | 39 | return 0; |
---|
61 | 40 | |
---|
62 | | - pud = pud_offset(pgd, address); |
---|
| 41 | + p4d = p4d_offset(pgd, address); |
---|
| 42 | + if (p4d_none(*p4d) || p4d_bad(*p4d)) |
---|
| 43 | + return 0; |
---|
| 44 | + |
---|
| 45 | + pud = pud_offset(p4d, address); |
---|
63 | 46 | if (pud_none(*pud) || pud_bad(*pud)) |
---|
64 | 47 | return 0; |
---|
65 | 48 | |
---|
.. | .. |
---|
87 | 70 | struct mm_struct *mm = current->mm; |
---|
88 | 71 | unsigned long mask; |
---|
89 | 72 | vm_fault_t fault; |
---|
90 | | - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; |
---|
| 73 | + unsigned int flags = FAULT_FLAG_DEFAULT; |
---|
91 | 74 | |
---|
92 | 75 | mask = ((((isr >> IA64_ISR_X_BIT) & 1UL) << VM_EXEC_BIT) |
---|
93 | 76 | | (((isr >> IA64_ISR_W_BIT) & 1UL) << VM_WRITE_BIT)); |
---|
94 | 77 | |
---|
95 | | - /* mmap_sem is performance critical.... */ |
---|
96 | | - prefetchw(&mm->mmap_sem); |
---|
| 78 | + /* mmap_lock is performance critical.... */ |
---|
| 79 | + prefetchw(&mm->mmap_lock); |
---|
97 | 80 | |
---|
98 | 81 | /* |
---|
99 | 82 | * If we're in an interrupt or have no user context, we must not take the fault.. |
---|
.. | .. |
---|
104 | 87 | #ifdef CONFIG_VIRTUAL_MEM_MAP |
---|
105 | 88 | /* |
---|
106 | 89 | * If fault is in region 5 and we are in the kernel, we may already |
---|
107 | | - * have the mmap_sem (pfn_valid macro is called during mmap). There |
---|
| 90 | + * have the mmap_lock (pfn_valid macro is called during mmap). There |
---|
108 | 91 | * is no vma for region 5 addr's anyway, so skip getting the semaphore |
---|
109 | 92 | * and go directly to the exception handling code. |
---|
110 | 93 | */ |
---|
.. | .. |
---|
116 | 99 | /* |
---|
117 | 100 | * This is to handle the kprobes on user space access instructions |
---|
118 | 101 | */ |
---|
119 | | - if (notify_page_fault(regs, TRAP_BRKPT)) |
---|
| 102 | + if (kprobe_page_fault(regs, TRAP_BRKPT)) |
---|
120 | 103 | return; |
---|
121 | 104 | |
---|
122 | 105 | if (user_mode(regs)) |
---|
123 | 106 | flags |= FAULT_FLAG_USER; |
---|
124 | 107 | if (mask & VM_WRITE) |
---|
125 | 108 | flags |= FAULT_FLAG_WRITE; |
---|
| 109 | + |
---|
| 110 | + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
---|
126 | 111 | retry: |
---|
127 | | - down_read(&mm->mmap_sem); |
---|
| 112 | + mmap_read_lock(mm); |
---|
128 | 113 | |
---|
129 | 114 | vma = find_vma_prev(mm, address, &prev_vma); |
---|
130 | 115 | if (!vma && !prev_vma ) |
---|
.. | .. |
---|
161 | 146 | * sure we exit gracefully rather than endlessly redo the |
---|
162 | 147 | * fault. |
---|
163 | 148 | */ |
---|
164 | | - fault = handle_mm_fault(vma, address, flags); |
---|
| 149 | + fault = handle_mm_fault(vma, address, flags, regs); |
---|
165 | 150 | |
---|
166 | | - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) |
---|
| 151 | + if (fault_signal_pending(fault, regs)) |
---|
167 | 152 | return; |
---|
168 | 153 | |
---|
169 | 154 | if (unlikely(fault & VM_FAULT_ERROR)) { |
---|
.. | .. |
---|
184 | 169 | } |
---|
185 | 170 | |
---|
186 | 171 | if (flags & FAULT_FLAG_ALLOW_RETRY) { |
---|
187 | | - if (fault & VM_FAULT_MAJOR) |
---|
188 | | - current->maj_flt++; |
---|
189 | | - else |
---|
190 | | - current->min_flt++; |
---|
191 | 172 | if (fault & VM_FAULT_RETRY) { |
---|
192 | | - flags &= ~FAULT_FLAG_ALLOW_RETRY; |
---|
193 | 173 | flags |= FAULT_FLAG_TRIED; |
---|
194 | 174 | |
---|
195 | | - /* No need to up_read(&mm->mmap_sem) as we would |
---|
| 175 | + /* No need to mmap_read_unlock(mm) as we would |
---|
196 | 176 | * have already released it in __lock_page_or_retry |
---|
197 | 177 | * in mm/filemap.c. |
---|
198 | 178 | */ |
---|
.. | .. |
---|
201 | 181 | } |
---|
202 | 182 | } |
---|
203 | 183 | |
---|
204 | | - up_read(&mm->mmap_sem); |
---|
| 184 | + mmap_read_unlock(mm); |
---|
205 | 185 | return; |
---|
206 | 186 | |
---|
207 | 187 | check_expansion: |
---|
.. | .. |
---|
232 | 212 | goto good_area; |
---|
233 | 213 | |
---|
234 | 214 | bad_area: |
---|
235 | | - up_read(&mm->mmap_sem); |
---|
| 215 | + mmap_read_unlock(mm); |
---|
236 | 216 | #ifdef CONFIG_VIRTUAL_MEM_MAP |
---|
237 | 217 | bad_area_no_up: |
---|
238 | 218 | #endif |
---|
.. | .. |
---|
248 | 228 | return; |
---|
249 | 229 | } |
---|
250 | 230 | if (user_mode(regs)) { |
---|
251 | | - struct siginfo si; |
---|
252 | | - |
---|
253 | | - clear_siginfo(&si); |
---|
254 | | - si.si_signo = signal; |
---|
255 | | - si.si_errno = 0; |
---|
256 | | - si.si_code = code; |
---|
257 | | - si.si_addr = (void __user *) address; |
---|
258 | | - si.si_isr = isr; |
---|
259 | | - si.si_flags = __ISR_VALID; |
---|
260 | | - force_sig_info(signal, &si, current); |
---|
| 231 | + force_sig_fault(signal, code, (void __user *) address, |
---|
| 232 | + 0, __ISR_VALID, isr); |
---|
261 | 233 | return; |
---|
262 | 234 | } |
---|
263 | 235 | |
---|
.. | .. |
---|
302 | 274 | regs = NULL; |
---|
303 | 275 | bust_spinlocks(0); |
---|
304 | 276 | if (regs) |
---|
305 | | - do_exit(SIGKILL); |
---|
| 277 | + make_task_dead(SIGKILL); |
---|
306 | 278 | return; |
---|
307 | 279 | |
---|
308 | 280 | out_of_memory: |
---|
309 | | - up_read(&mm->mmap_sem); |
---|
| 281 | + mmap_read_unlock(mm); |
---|
310 | 282 | if (!user_mode(regs)) |
---|
311 | 283 | goto no_context; |
---|
312 | 284 | pagefault_out_of_memory(); |
---|