.. | .. |
---|
44 | 44 | const int field = sizeof(unsigned long) * 2; |
---|
45 | 45 | int si_code; |
---|
46 | 46 | vm_fault_t fault; |
---|
47 | | - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; |
---|
| 47 | + unsigned int flags = FAULT_FLAG_DEFAULT; |
---|
48 | 48 | |
---|
49 | 49 | static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10); |
---|
50 | 50 | |
---|
.. | .. |
---|
96 | 96 | |
---|
97 | 97 | if (user_mode(regs)) |
---|
98 | 98 | flags |= FAULT_FLAG_USER; |
---|
| 99 | + |
---|
| 100 | + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
---|
99 | 101 | retry: |
---|
100 | | - down_read(&mm->mmap_sem); |
---|
| 102 | + mmap_read_lock(mm); |
---|
101 | 103 | vma = find_vma(mm, address); |
---|
102 | 104 | if (!vma) |
---|
103 | 105 | goto bad_area; |
---|
.. | .. |
---|
142 | 144 | goto bad_area; |
---|
143 | 145 | } |
---|
144 | 146 | } else { |
---|
145 | | - if (!(vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC))) |
---|
| 147 | + if (unlikely(!vma_is_accessible(vma))) |
---|
146 | 148 | goto bad_area; |
---|
147 | 149 | } |
---|
148 | 150 | } |
---|
.. | .. |
---|
152 | 154 | * make sure we exit gracefully rather than endlessly redo |
---|
153 | 155 | * the fault. |
---|
154 | 156 | */ |
---|
155 | | - fault = handle_mm_fault(vma, address, flags); |
---|
| 157 | + fault = handle_mm_fault(vma, address, flags, regs); |
---|
156 | 158 | |
---|
157 | | - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) |
---|
| 159 | + if (fault_signal_pending(fault, regs)) |
---|
158 | 160 | return; |
---|
159 | 161 | |
---|
160 | | - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
---|
161 | 162 | if (unlikely(fault & VM_FAULT_ERROR)) { |
---|
162 | 163 | if (fault & VM_FAULT_OOM) |
---|
163 | 164 | goto out_of_memory; |
---|
.. | .. |
---|
168 | 169 | BUG(); |
---|
169 | 170 | } |
---|
170 | 171 | if (flags & FAULT_FLAG_ALLOW_RETRY) { |
---|
171 | | - if (fault & VM_FAULT_MAJOR) { |
---|
172 | | - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, |
---|
173 | | - regs, address); |
---|
174 | | - tsk->maj_flt++; |
---|
175 | | - } else { |
---|
176 | | - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, |
---|
177 | | - regs, address); |
---|
178 | | - tsk->min_flt++; |
---|
179 | | - } |
---|
180 | 172 | if (fault & VM_FAULT_RETRY) { |
---|
181 | | - flags &= ~FAULT_FLAG_ALLOW_RETRY; |
---|
182 | 173 | flags |= FAULT_FLAG_TRIED; |
---|
183 | 174 | |
---|
184 | 175 | /* |
---|
185 | | - * No need to up_read(&mm->mmap_sem) as we would |
---|
| 176 | + * No need to mmap_read_unlock(mm) as we would |
---|
186 | 177 | * have already released it in __lock_page_or_retry |
---|
187 | 178 | * in mm/filemap.c. |
---|
188 | 179 | */ |
---|
.. | .. |
---|
191 | 182 | } |
---|
192 | 183 | } |
---|
193 | 184 | |
---|
194 | | - up_read(&mm->mmap_sem); |
---|
| 185 | + mmap_read_unlock(mm); |
---|
195 | 186 | return; |
---|
196 | 187 | |
---|
197 | 188 | /* |
---|
.. | .. |
---|
199 | 190 | * Fix it, but check if it's kernel or user first.. |
---|
200 | 191 | */ |
---|
201 | 192 | bad_area: |
---|
202 | | - up_read(&mm->mmap_sem); |
---|
| 193 | + mmap_read_unlock(mm); |
---|
203 | 194 | |
---|
204 | 195 | bad_area_nosemaphore: |
---|
205 | 196 | /* User mode accesses just cause a SIGSEGV */ |
---|
.. | .. |
---|
223 | 214 | pr_cont("\n"); |
---|
224 | 215 | } |
---|
225 | 216 | current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f; |
---|
226 | | - force_sig_fault(SIGSEGV, si_code, (void __user *)address, tsk); |
---|
| 217 | + force_sig_fault(SIGSEGV, si_code, (void __user *)address); |
---|
227 | 218 | return; |
---|
228 | 219 | } |
---|
229 | 220 | |
---|
.. | .. |
---|
251 | 242 | * We ran out of memory, call the OOM killer, and return the userspace |
---|
252 | 243 | * (which will retry the fault, or kill us if we got oom-killed). |
---|
253 | 244 | */ |
---|
254 | | - up_read(&mm->mmap_sem); |
---|
| 245 | + mmap_read_unlock(mm); |
---|
255 | 246 | if (!user_mode(regs)) |
---|
256 | 247 | goto no_context; |
---|
257 | 248 | pagefault_out_of_memory(); |
---|
258 | 249 | return; |
---|
259 | 250 | |
---|
260 | 251 | do_sigbus: |
---|
261 | | - up_read(&mm->mmap_sem); |
---|
| 252 | + mmap_read_unlock(mm); |
---|
262 | 253 | |
---|
263 | 254 | /* Kernel mode? Handle exceptions or die */ |
---|
264 | 255 | if (!user_mode(regs)) |
---|
.. | .. |
---|
279 | 270 | #endif |
---|
280 | 271 | current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f; |
---|
281 | 272 | tsk->thread.cp0_badvaddr = address; |
---|
282 | | - force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address, tsk); |
---|
| 273 | + force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); |
---|
283 | 274 | |
---|
284 | 275 | return; |
---|
285 | 276 | #ifndef CONFIG_64BIT |
---|
.. | .. |
---|
292 | 283 | * Do _not_ use "tsk" here. We might be inside |
---|
293 | 284 | * an interrupt in the middle of a task switch.. |
---|
294 | 285 | */ |
---|
295 | | - int offset = __pgd_offset(address); |
---|
| 286 | + int offset = pgd_index(address); |
---|
296 | 287 | pgd_t *pgd, *pgd_k; |
---|
| 288 | + p4d_t *p4d, *p4d_k; |
---|
297 | 289 | pud_t *pud, *pud_k; |
---|
298 | 290 | pmd_t *pmd, *pmd_k; |
---|
299 | 291 | pte_t *pte_k; |
---|
.. | .. |
---|
305 | 297 | goto no_context; |
---|
306 | 298 | set_pgd(pgd, *pgd_k); |
---|
307 | 299 | |
---|
308 | | - pud = pud_offset(pgd, address); |
---|
309 | | - pud_k = pud_offset(pgd_k, address); |
---|
| 300 | + p4d = p4d_offset(pgd, address); |
---|
| 301 | + p4d_k = p4d_offset(pgd_k, address); |
---|
| 302 | + if (!p4d_present(*p4d_k)) |
---|
| 303 | + goto no_context; |
---|
| 304 | + |
---|
| 305 | + pud = pud_offset(p4d, address); |
---|
| 306 | + pud_k = pud_offset(p4d_k, address); |
---|
310 | 307 | if (!pud_present(*pud_k)) |
---|
311 | 308 | goto no_context; |
---|
312 | 309 | |
---|