.. | .. |
---|
20 | 20 | #include <asm/mmu_context.h> |
---|
21 | 21 | #include <asm/cacheflush.h> |
---|
22 | 22 | #include <asm/hardirq.h> |
---|
23 | | -#include <asm/pgalloc.h> |
---|
24 | 23 | |
---|
25 | 24 | DEFINE_PER_CPU(unsigned long, asid_cache) = ASID_USER_FIRST; |
---|
26 | 25 | void bad_page_fault(struct pt_regs*, unsigned long, int); |
---|
.. | .. |
---|
43 | 42 | |
---|
44 | 43 | int is_write, is_exec; |
---|
45 | 44 | vm_fault_t fault; |
---|
46 | | - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; |
---|
| 45 | + unsigned int flags = FAULT_FLAG_DEFAULT; |
---|
47 | 46 | |
---|
48 | 47 | code = SEGV_MAPERR; |
---|
49 | 48 | |
---|
.. | .. |
---|
73 | 72 | |
---|
74 | 73 | if (user_mode(regs)) |
---|
75 | 74 | flags |= FAULT_FLAG_USER; |
---|
| 75 | + |
---|
| 76 | + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
---|
| 77 | + |
---|
76 | 78 | retry: |
---|
77 | | - down_read(&mm->mmap_sem); |
---|
| 79 | + mmap_read_lock(mm); |
---|
78 | 80 | vma = find_vma(mm, address); |
---|
79 | 81 | |
---|
80 | 82 | if (!vma) |
---|
.. | .. |
---|
108 | 110 | * make sure we exit gracefully rather than endlessly redo |
---|
109 | 111 | * the fault. |
---|
110 | 112 | */ |
---|
111 | | - fault = handle_mm_fault(vma, address, flags); |
---|
| 113 | + fault = handle_mm_fault(vma, address, flags, regs); |
---|
112 | 114 | |
---|
113 | | - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) |
---|
| 115 | + if (fault_signal_pending(fault, regs)) { |
---|
| 116 | + if (!user_mode(regs)) |
---|
| 117 | + goto bad_page_fault; |
---|
114 | 118 | return; |
---|
| 119 | + } |
---|
115 | 120 | |
---|
116 | 121 | if (unlikely(fault & VM_FAULT_ERROR)) { |
---|
117 | 122 | if (fault & VM_FAULT_OOM) |
---|
.. | .. |
---|
123 | 128 | BUG(); |
---|
124 | 129 | } |
---|
125 | 130 | if (flags & FAULT_FLAG_ALLOW_RETRY) { |
---|
126 | | - if (fault & VM_FAULT_MAJOR) |
---|
127 | | - current->maj_flt++; |
---|
128 | | - else |
---|
129 | | - current->min_flt++; |
---|
130 | 131 | if (fault & VM_FAULT_RETRY) { |
---|
131 | | - flags &= ~FAULT_FLAG_ALLOW_RETRY; |
---|
132 | 132 | flags |= FAULT_FLAG_TRIED; |
---|
133 | 133 | |
---|
134 | | - /* No need to up_read(&mm->mmap_sem) as we would |
---|
| 134 | + /* No need to mmap_read_unlock(mm) as we would |
---|
135 | 135 | * have already released it in __lock_page_or_retry |
---|
136 | 136 | * in mm/filemap.c. |
---|
137 | 137 | */ |
---|
.. | .. |
---|
140 | 140 | } |
---|
141 | 141 | } |
---|
142 | 142 | |
---|
143 | | - up_read(&mm->mmap_sem); |
---|
144 | | - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
---|
145 | | - if (flags & VM_FAULT_MAJOR) |
---|
146 | | - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); |
---|
147 | | - else |
---|
148 | | - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); |
---|
149 | | - |
---|
| 143 | + mmap_read_unlock(mm); |
---|
150 | 144 | return; |
---|
151 | 145 | |
---|
152 | 146 | /* Something tried to access memory that isn't in our memory map.. |
---|
153 | 147 | * Fix it, but check if it's kernel or user first.. |
---|
154 | 148 | */ |
---|
155 | 149 | bad_area: |
---|
156 | | - up_read(&mm->mmap_sem); |
---|
| 150 | + mmap_read_unlock(mm); |
---|
157 | 151 | if (user_mode(regs)) { |
---|
158 | 152 | current->thread.bad_vaddr = address; |
---|
159 | 153 | current->thread.error_code = is_write; |
---|
160 | | - force_sig_fault(SIGSEGV, code, (void *) address, current); |
---|
| 154 | + force_sig_fault(SIGSEGV, code, (void *) address); |
---|
161 | 155 | return; |
---|
162 | 156 | } |
---|
163 | 157 | bad_page_fault(regs, address, SIGSEGV); |
---|
.. | .. |
---|
168 | 162 | * us unable to handle the page fault gracefully. |
---|
169 | 163 | */ |
---|
170 | 164 | out_of_memory: |
---|
171 | | - up_read(&mm->mmap_sem); |
---|
| 165 | + mmap_read_unlock(mm); |
---|
172 | 166 | if (!user_mode(regs)) |
---|
173 | 167 | bad_page_fault(regs, address, SIGKILL); |
---|
174 | 168 | else |
---|
.. | .. |
---|
176 | 170 | return; |
---|
177 | 171 | |
---|
178 | 172 | do_sigbus: |
---|
179 | | - up_read(&mm->mmap_sem); |
---|
| 173 | + mmap_read_unlock(mm); |
---|
180 | 174 | |
---|
181 | 175 | /* Send a sigbus, regardless of whether we were in kernel |
---|
182 | 176 | * or user mode. |
---|
183 | 177 | */ |
---|
184 | 178 | current->thread.bad_vaddr = address; |
---|
185 | | - force_sig_fault(SIGBUS, BUS_ADRERR, (void *) address, current); |
---|
| 179 | + force_sig_fault(SIGBUS, BUS_ADRERR, (void *) address); |
---|
186 | 180 | |
---|
187 | 181 | /* Kernel mode? Handle exceptions or die */ |
---|
188 | 182 | if (!user_mode(regs)) |
---|
.. | .. |
---|
197 | 191 | struct mm_struct *act_mm = current->active_mm; |
---|
198 | 192 | int index = pgd_index(address); |
---|
199 | 193 | pgd_t *pgd, *pgd_k; |
---|
| 194 | + p4d_t *p4d, *p4d_k; |
---|
| 195 | + pud_t *pud, *pud_k; |
---|
200 | 196 | pmd_t *pmd, *pmd_k; |
---|
201 | 197 | pte_t *pte_k; |
---|
202 | 198 | |
---|
.. | .. |
---|
211 | 207 | |
---|
212 | 208 | pgd_val(*pgd) = pgd_val(*pgd_k); |
---|
213 | 209 | |
---|
214 | | - pmd = pmd_offset(pgd, address); |
---|
215 | | - pmd_k = pmd_offset(pgd_k, address); |
---|
| 210 | + p4d = p4d_offset(pgd, address); |
---|
| 211 | + p4d_k = p4d_offset(pgd_k, address); |
---|
| 212 | + if (!p4d_present(*p4d) || !p4d_present(*p4d_k)) |
---|
| 213 | + goto bad_page_fault; |
---|
| 214 | + |
---|
| 215 | + pud = pud_offset(p4d, address); |
---|
| 216 | + pud_k = pud_offset(p4d_k, address); |
---|
| 217 | + if (!pud_present(*pud) || !pud_present(*pud_k)) |
---|
| 218 | + goto bad_page_fault; |
---|
| 219 | + |
---|
| 220 | + pmd = pmd_offset(pud, address); |
---|
| 221 | + pmd_k = pmd_offset(pud_k, address); |
---|
216 | 222 | if (!pmd_present(*pmd) || !pmd_present(*pmd_k)) |
---|
217 | 223 | goto bad_page_fault; |
---|
218 | 224 | |
---|