| .. | .. |
|---|
| 20 | 20 | #include <asm/mmu_context.h> |
|---|
| 21 | 21 | #include <asm/cacheflush.h> |
|---|
| 22 | 22 | #include <asm/hardirq.h> |
|---|
| 23 | | -#include <asm/pgalloc.h> |
|---|
| 24 | 23 | |
|---|
| 25 | 24 | DEFINE_PER_CPU(unsigned long, asid_cache) = ASID_USER_FIRST; |
|---|
| 26 | 25 | void bad_page_fault(struct pt_regs*, unsigned long, int); |
|---|
| .. | .. |
|---|
| 43 | 42 | |
|---|
| 44 | 43 | int is_write, is_exec; |
|---|
| 45 | 44 | vm_fault_t fault; |
|---|
| 46 | | - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; |
|---|
| 45 | + unsigned int flags = FAULT_FLAG_DEFAULT; |
|---|
| 47 | 46 | |
|---|
| 48 | 47 | code = SEGV_MAPERR; |
|---|
| 49 | 48 | |
|---|
| .. | .. |
|---|
| 73 | 72 | |
|---|
| 74 | 73 | if (user_mode(regs)) |
|---|
| 75 | 74 | flags |= FAULT_FLAG_USER; |
|---|
| 75 | + |
|---|
| 76 | + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
|---|
| 77 | + |
|---|
| 76 | 78 | retry: |
|---|
| 77 | | - down_read(&mm->mmap_sem); |
|---|
| 79 | + mmap_read_lock(mm); |
|---|
| 78 | 80 | vma = find_vma(mm, address); |
|---|
| 79 | 81 | |
|---|
| 80 | 82 | if (!vma) |
|---|
| .. | .. |
|---|
| 108 | 110 | * make sure we exit gracefully rather than endlessly redo |
|---|
| 109 | 111 | * the fault. |
|---|
| 110 | 112 | */ |
|---|
| 111 | | - fault = handle_mm_fault(vma, address, flags); |
|---|
| 113 | + fault = handle_mm_fault(vma, address, flags, regs); |
|---|
| 112 | 114 | |
|---|
| 113 | | - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) |
|---|
| 115 | + if (fault_signal_pending(fault, regs)) { |
|---|
| 116 | + if (!user_mode(regs)) |
|---|
| 117 | + goto bad_page_fault; |
|---|
| 114 | 118 | return; |
|---|
| 119 | + } |
|---|
| 115 | 120 | |
|---|
| 116 | 121 | if (unlikely(fault & VM_FAULT_ERROR)) { |
|---|
| 117 | 122 | if (fault & VM_FAULT_OOM) |
|---|
| .. | .. |
|---|
| 123 | 128 | BUG(); |
|---|
| 124 | 129 | } |
|---|
| 125 | 130 | if (flags & FAULT_FLAG_ALLOW_RETRY) { |
|---|
| 126 | | - if (fault & VM_FAULT_MAJOR) |
|---|
| 127 | | - current->maj_flt++; |
|---|
| 128 | | - else |
|---|
| 129 | | - current->min_flt++; |
|---|
| 130 | 131 | if (fault & VM_FAULT_RETRY) { |
|---|
| 131 | | - flags &= ~FAULT_FLAG_ALLOW_RETRY; |
|---|
| 132 | 132 | flags |= FAULT_FLAG_TRIED; |
|---|
| 133 | 133 | |
|---|
| 134 | | - /* No need to up_read(&mm->mmap_sem) as we would |
|---|
| 134 | + /* No need to mmap_read_unlock(mm) as we would |
|---|
| 135 | 135 | * have already released it in __lock_page_or_retry |
|---|
| 136 | 136 | * in mm/filemap.c. |
|---|
| 137 | 137 | */ |
|---|
| .. | .. |
|---|
| 140 | 140 | } |
|---|
| 141 | 141 | } |
|---|
| 142 | 142 | |
|---|
| 143 | | - up_read(&mm->mmap_sem); |
|---|
| 144 | | - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
|---|
| 145 | | - if (flags & VM_FAULT_MAJOR) |
|---|
| 146 | | - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); |
|---|
| 147 | | - else |
|---|
| 148 | | - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); |
|---|
| 149 | | - |
|---|
| 143 | + mmap_read_unlock(mm); |
|---|
| 150 | 144 | return; |
|---|
| 151 | 145 | |
|---|
| 152 | 146 | /* Something tried to access memory that isn't in our memory map.. |
|---|
| 153 | 147 | * Fix it, but check if it's kernel or user first.. |
|---|
| 154 | 148 | */ |
|---|
| 155 | 149 | bad_area: |
|---|
| 156 | | - up_read(&mm->mmap_sem); |
|---|
| 150 | + mmap_read_unlock(mm); |
|---|
| 157 | 151 | if (user_mode(regs)) { |
|---|
| 158 | 152 | current->thread.bad_vaddr = address; |
|---|
| 159 | 153 | current->thread.error_code = is_write; |
|---|
| 160 | | - force_sig_fault(SIGSEGV, code, (void *) address, current); |
|---|
| 154 | + force_sig_fault(SIGSEGV, code, (void *) address); |
|---|
| 161 | 155 | return; |
|---|
| 162 | 156 | } |
|---|
| 163 | 157 | bad_page_fault(regs, address, SIGSEGV); |
|---|
| .. | .. |
|---|
| 168 | 162 | * us unable to handle the page fault gracefully. |
|---|
| 169 | 163 | */ |
|---|
| 170 | 164 | out_of_memory: |
|---|
| 171 | | - up_read(&mm->mmap_sem); |
|---|
| 165 | + mmap_read_unlock(mm); |
|---|
| 172 | 166 | if (!user_mode(regs)) |
|---|
| 173 | 167 | bad_page_fault(regs, address, SIGKILL); |
|---|
| 174 | 168 | else |
|---|
| .. | .. |
|---|
| 176 | 170 | return; |
|---|
| 177 | 171 | |
|---|
| 178 | 172 | do_sigbus: |
|---|
| 179 | | - up_read(&mm->mmap_sem); |
|---|
| 173 | + mmap_read_unlock(mm); |
|---|
| 180 | 174 | |
|---|
| 181 | 175 | /* Send a sigbus, regardless of whether we were in kernel |
|---|
| 182 | 176 | * or user mode. |
|---|
| 183 | 177 | */ |
|---|
| 184 | 178 | current->thread.bad_vaddr = address; |
|---|
| 185 | | - force_sig_fault(SIGBUS, BUS_ADRERR, (void *) address, current); |
|---|
| 179 | + force_sig_fault(SIGBUS, BUS_ADRERR, (void *) address); |
|---|
| 186 | 180 | |
|---|
| 187 | 181 | /* Kernel mode? Handle exceptions or die */ |
|---|
| 188 | 182 | if (!user_mode(regs)) |
|---|
| .. | .. |
|---|
| 197 | 191 | struct mm_struct *act_mm = current->active_mm; |
|---|
| 198 | 192 | int index = pgd_index(address); |
|---|
| 199 | 193 | pgd_t *pgd, *pgd_k; |
|---|
| 194 | + p4d_t *p4d, *p4d_k; |
|---|
| 195 | + pud_t *pud, *pud_k; |
|---|
| 200 | 196 | pmd_t *pmd, *pmd_k; |
|---|
| 201 | 197 | pte_t *pte_k; |
|---|
| 202 | 198 | |
|---|
| .. | .. |
|---|
| 211 | 207 | |
|---|
| 212 | 208 | pgd_val(*pgd) = pgd_val(*pgd_k); |
|---|
| 213 | 209 | |
|---|
| 214 | | - pmd = pmd_offset(pgd, address); |
|---|
| 215 | | - pmd_k = pmd_offset(pgd_k, address); |
|---|
| 210 | + p4d = p4d_offset(pgd, address); |
|---|
| 211 | + p4d_k = p4d_offset(pgd_k, address); |
|---|
| 212 | + if (!p4d_present(*p4d) || !p4d_present(*p4d_k)) |
|---|
| 213 | + goto bad_page_fault; |
|---|
| 214 | + |
|---|
| 215 | + pud = pud_offset(p4d, address); |
|---|
| 216 | + pud_k = pud_offset(p4d_k, address); |
|---|
| 217 | + if (!pud_present(*pud) || !pud_present(*pud_k)) |
|---|
| 218 | + goto bad_page_fault; |
|---|
| 219 | + |
|---|
| 220 | + pmd = pmd_offset(pud, address); |
|---|
| 221 | + pmd_k = pmd_offset(pud_k, address); |
|---|
| 216 | 222 | if (!pmd_present(*pmd) || !pmd_present(*pmd_k)) |
|---|
| 217 | 223 | goto bad_page_fault; |
|---|
| 218 | 224 | |
|---|