write in 30M

hc

2024-02-20 ea08eeccae9297f7aabd2ef7f0c2517ac4549acc

 kernel/arch/arm/mm/fault.c
..
..
@@ -1,12 +1,9 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
1
2
 /*
2
3
  *  linux/arch/arm/mm/fault.c
3
4
  *
4
5
  *  Copyright (C) 1995  Linus Torvalds
5
6
  *  Modifications for ARM processor (c) 1995-2004 Russell King
6
- *
7
- * This program is free software; you can redistribute it and/or modify
8
- * it under the terms of the GNU General Public License version 2 as
9
- * published by the Free Software Foundation.
10
7
  */
11
8
 #include <linux/extable.h>
12
9
 #include <linux/signal.h>
..
..
@@ -21,7 +18,6 @@
21
18
 #include <linux/highmem.h>
22
19
 #include <linux/perf_event.h>
23
20
 
24
-#include <asm/pgtable.h>
25
21
 #include <asm/system_misc.h>
26
22
 #include <asm/system_info.h>
27
23
 #include <asm/tlbflush.h>
..
..
@@ -30,58 +26,37 @@
30
26
 
31
27
 #ifdef CONFIG_MMU
32
28
 
33
-#ifdef CONFIG_KPROBES
34
-static inline int notify_page_fault(struct pt_regs *regs, unsigned int fsr)
35
-{
36
-	int ret = 0;
37
-
38
-	if (!user_mode(regs)) {
39
-		/* kprobe_running() needs smp_processor_id() */
40
-		preempt_disable();
41
-		if (kprobe_running() && kprobe_fault_handler(regs, fsr))
42
-			ret = 1;
43
-		preempt_enable();
44
-	}
45
-
46
-	return ret;
47
-}
48
-#else
49
-static inline int notify_page_fault(struct pt_regs *regs, unsigned int fsr)
50
-{
51
-	return 0;
52
-}
53
-#endif
54
-
55
29
 /*
56
30
  * This is useful to dump out the page tables associated with
57
31
  * 'addr' in mm 'mm'.
58
32
  */
59
-void show_pte(struct mm_struct *mm, unsigned long addr)
33
+void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
60
34
 {
61
35
 	pgd_t *pgd;
62
36
 
63
37
 	if (!mm)
64
38
 		mm = &init_mm;
65
39
 
66
-	pr_alert("pgd = %p\n", mm->pgd);
40
+	printk("%spgd = %p\n", lvl, mm->pgd);
67
41
 	pgd = pgd_offset(mm, addr);
68
-	pr_alert("[%08lx] *pgd=%08llx",
69
-			addr, (long long)pgd_val(*pgd));
42
+	printk("%s[%08lx] *pgd=%08llx", lvl, addr, (long long)pgd_val(*pgd));
70
43
 
71
44
 	do {
45
+		p4d_t *p4d;
72
46
 		pud_t *pud;
73
47
 		pmd_t *pmd;
74
48
 		pte_t *pte;
75
49
 
76
-		if (pgd_none(*pgd))
50
+		p4d = p4d_offset(pgd, addr);
51
+		if (p4d_none(*p4d))
77
52
 			break;
78
53
 
79
-		if (pgd_bad(*pgd)) {
54
+		if (p4d_bad(*p4d)) {
80
55
 			pr_cont("(bad)");
81
56
 			break;
82
57
 		}
83
58
 
84
-		pud = pud_offset(pgd, addr);
59
+		pud = pud_offset(p4d, addr);
85
60
 		if (PTRS_PER_PUD != 1)
86
61
 			pr_cont(", *pud=%08llx", (long long)pud_val(*pud));
87
62
 
..
..
@@ -121,7 +96,7 @@
121
96
 	pr_cont("\n");
122
97
 }
123
98
 #else					/* CONFIG_MMU */
124
-void show_pte(struct mm_struct *mm, unsigned long addr)
99
+void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
125
100
 { }
126
101
 #endif					/* CONFIG_MMU */
127
102
 
..
..
@@ -142,14 +117,15 @@
142
117
 	 * No handler, we'll have to terminate things with extreme prejudice.
143
118
 	 */
144
119
 	bust_spinlocks(1);
120
+	pr_alert("8<--- cut here ---\n");
145
121
 	pr_alert("Unable to handle kernel %s at virtual address %08lx\n",
146
122
 		 (addr < PAGE_SIZE) ? "NULL pointer dereference" :
147
123
 		 "paging request", addr);
148
124
 
149
-	show_pte(mm, addr);
125
+	show_pte(KERN_ALERT, mm, addr);
150
126
 	die("Oops", regs, fsr);
151
127
 	bust_spinlocks(0);
152
-	do_exit(SIGKILL);
128
+	make_task_dead(SIGKILL);
153
129
 }
154
130
 
155
131
 /*
..
..
@@ -157,35 +133,35 @@
157
133
  * User mode accesses just cause a SIGSEGV
158
134
  */
159
135
 static void
160
-__do_user_fault(struct task_struct *tsk, unsigned long addr,
161
-		unsigned int fsr, unsigned int sig, int code,
162
-		struct pt_regs *regs)
136
+__do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig,
137
+		int code, struct pt_regs *regs)
163
138
 {
164
-	struct siginfo si;
139
+	struct task_struct *tsk = current;
165
140
 
166
141
 	if (addr > TASK_SIZE)
167
142
 		harden_branch_predictor();
168
143
 
169
-	clear_siginfo(&si);
170
-
171
144
 #ifdef CONFIG_DEBUG_USER
172
145
 	if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) ||
173
146
 	    ((user_debug & UDBG_BUS)  && (sig == SIGBUS))) {
174
-		printk(KERN_DEBUG "%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n",
147
+		pr_err("8<--- cut here ---\n");
148
+		pr_err("%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n",
175
149
 		       tsk->comm, sig, addr, fsr);
176
-		show_pte(tsk->mm, addr);
150
+		show_pte(KERN_ERR, tsk->mm, addr);
177
151
 		show_regs(regs);
178
152
 	}
153
+#endif
154
+#ifndef CONFIG_KUSER_HELPERS
155
+	if ((sig == SIGSEGV) && ((addr & PAGE_MASK) == 0xffff0000))
156
+		printk_ratelimited(KERN_DEBUG
157
+				   "%s: CONFIG_KUSER_HELPERS disabled at 0x%08lx\n",
158
+				   tsk->comm, addr);
179
159
 #endif
180
160
 
181
161
 	tsk->thread.address = addr;
182
162
 	tsk->thread.error_code = fsr;
183
163
 	tsk->thread.trap_no = 14;
184
-	si.si_signo = sig;
185
-	si.si_errno = 0;
186
-	si.si_code = code;
187
-	si.si_addr = (void __user *)addr;
188
-	force_sig_info(sig, &si, tsk);
164
+	force_sig_fault(sig, code, (void __user *)addr);
189
165
 }
190
166
 
191
167
 void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
..
..
@@ -198,7 +174,7 @@
198
174
 	 * have no context to handle this fault with.
199
175
 	 */
200
176
 	if (user_mode(regs))
201
-		__do_user_fault(tsk, addr, fsr, SIGSEGV, SEGV_MAPERR, regs);
177
+		__do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs);
202
178
 	else
203
179
 		__do_kernel_fault(mm, addr, fsr, regs);
204
180
 }
..
..
@@ -214,7 +190,7 @@
214
190
  */
215
191
 static inline bool access_error(unsigned int fsr, struct vm_area_struct *vma)
216
192
 {
217
-	unsigned int mask = VM_READ | VM_WRITE | VM_EXEC;
193
+	unsigned int mask = VM_ACCESS_FLAGS;
218
194
 
219
195
 	if ((fsr & FSR_WRITE) && !(fsr & FSR_CM))
220
196
 		mask = VM_WRITE;
..
..
@@ -226,7 +202,8 @@
226
202
 
227
203
 static vm_fault_t __kprobes
228
204
 __do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
229
-		unsigned int flags, struct task_struct *tsk)
205
+		unsigned int flags, struct task_struct *tsk,
206
+		struct pt_regs *regs)
230
207
 {
231
208
 	struct vm_area_struct *vma;
232
209
 	vm_fault_t fault;
..
..
@@ -248,7 +225,7 @@
248
225
 		goto out;
249
226
 	}
250
227
 
251
-	return handle_mm_fault(vma, addr & PAGE_MASK, flags);
228
+	return handle_mm_fault(vma, addr & PAGE_MASK, flags, regs);
252
229
 
253
230
 check_stack:
254
231
 	/* Don't allow expansion below FIRST_USER_ADDRESS */
..
..
@@ -266,9 +243,9 @@
266
243
 	struct mm_struct *mm;
267
244
 	int sig, code;
268
245
 	vm_fault_t fault;
269
-	unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
246
+	unsigned int flags = FAULT_FLAG_DEFAULT;
270
247
 
271
-	if (notify_page_fault(regs, fsr))
248
+	if (kprobe_page_fault(regs, fsr))
272
249
 		return 0;
273
250
 
274
251
 	tsk = current;
..
..
@@ -290,16 +267,18 @@
290
267
 	if ((fsr & FSR_WRITE) && !(fsr & FSR_CM))
291
268
 		flags |= FAULT_FLAG_WRITE;
292
269
 
270
+	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
271
+
293
272
 	/*
294
273
 	 * As per x86, we may deadlock here.  However, since the kernel only
295
274
 	 * validly references user space from well defined areas of the code,
296
275
 	 * we can bug out early if this is from code which shouldn't.
297
276
 	 */
298
-	if (!down_read_trylock(&mm->mmap_sem)) {
277
+	if (!mmap_read_trylock(mm)) {
299
278
 		if (!user_mode(regs) && !search_exception_tables(regs->ARM_pc))
300
279
 			goto no_context;
301
280
 retry:
302
-		down_read(&mm->mmap_sem);
281
+		mmap_read_lock(mm);
303
282
 	} else {
304
283
 		/*
305
284
 		 * The above down_read_trylock() might have succeeded in
..
..
@@ -314,45 +293,26 @@
314
293
 #endif
315
294
 	}
316
295
 
317
-	fault = __do_page_fault(mm, addr, fsr, flags, tsk);
296
+	fault = __do_page_fault(mm, addr, fsr, flags, tsk, regs);
318
297
 
319
298
 	/* If we need to retry but a fatal signal is pending, handle the
320
-	 * signal first. We do not need to release the mmap_sem because
299
+	 * signal first. We do not need to release the mmap_lock because
321
300
 	 * it would already be released in __lock_page_or_retry in
322
301
 	 * mm/filemap.c. */
323
-	if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
302
+	if (fault_signal_pending(fault, regs)) {
324
303
 		if (!user_mode(regs))
325
304
 			goto no_context;
326
305
 		return 0;
327
306
 	}
328
307
 
329
-	/*
330
-	 * Major/minor page fault accounting is only done on the
331
-	 * initial attempt. If we go through a retry, it is extremely
332
-	 * likely that the page will be found in page cache at that point.
333
-	 */
334
-
335
-	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
336
308
 	if (!(fault & VM_FAULT_ERROR) && flags & FAULT_FLAG_ALLOW_RETRY) {
337
-		if (fault & VM_FAULT_MAJOR) {
338
-			tsk->maj_flt++;
339
-			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
340
-					regs, addr);
341
-		} else {
342
-			tsk->min_flt++;
343
-			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
344
-					regs, addr);
345
-		}
346
309
 		if (fault & VM_FAULT_RETRY) {
347
-			/* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
348
-			* of starvation. */
349
-			flags &= ~FAULT_FLAG_ALLOW_RETRY;
350
310
 			flags |= FAULT_FLAG_TRIED;
351
311
 			goto retry;
352
312
 		}
353
313
 	}
354
314
 
355
-	up_read(&mm->mmap_sem);
315
+	mmap_read_unlock(mm);
356
316
 
357
317
 	/*
358
318
 	 * Handle the "normal" case first - VM_FAULT_MAJOR
..
..
@@ -394,7 +354,7 @@
394
354
 			SEGV_ACCERR : SEGV_MAPERR;
395
355
 	}
396
356
 
397
-	__do_user_fault(tsk, addr, fsr, sig, code, regs);
357
+	__do_user_fault(addr, fsr, sig, code, regs);
398
358
 	return 0;
399
359
 
400
360
 no_context:
..
..
@@ -433,6 +393,7 @@
433
393
 {
434
394
 	unsigned int index;
435
395
 	pgd_t *pgd, *pgd_k;
396
+	p4d_t *p4d, *p4d_k;
436
397
 	pud_t *pud, *pud_k;
437
398
 	pmd_t *pmd, *pmd_k;
438
399
 
..
..
@@ -447,13 +408,16 @@
447
408
 	pgd = cpu_get_pgd() + index;
448
409
 	pgd_k = init_mm.pgd + index;
449
410
 
450
-	if (pgd_none(*pgd_k))
451
-		goto bad_area;
452
-	if (!pgd_present(*pgd))
453
-		set_pgd(pgd, *pgd_k);
411
+	p4d = p4d_offset(pgd, addr);
412
+	p4d_k = p4d_offset(pgd_k, addr);
454
413
 
455
-	pud = pud_offset(pgd, addr);
456
-	pud_k = pud_offset(pgd_k, addr);
414
+	if (p4d_none(*p4d_k))
415
+		goto bad_area;
416
+	if (!p4d_present(*p4d))
417
+		set_p4d(p4d, *p4d_k);
418
+
419
+	pud = pud_offset(p4d, addr);
420
+	pud_k = pud_offset(p4d_k, addr);
457
421
 
458
422
 	if (pud_none(*pud_k))
459
423
 		goto bad_area;
..
..
@@ -554,21 +518,17 @@
554
518
 do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
555
519
 {
556
520
 	const struct fsr_info *inf = fsr_info + fsr_fs(fsr);
557
-	struct siginfo info;
558
521
 
559
522
 	if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs))
560
523
 		return;
561
524
 
525
+	pr_alert("8<--- cut here ---\n");
562
526
 	pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n",
563
527
 		inf->name, fsr, addr);
564
-	show_pte(current->mm, addr);
528
+	show_pte(KERN_ALERT, current->mm, addr);
565
529
 
566
-	clear_siginfo(&info);
567
-	info.si_signo = inf->sig;
568
-	info.si_errno = 0;
569
-	info.si_code  = inf->code;
570
-	info.si_addr  = (void __user *)addr;
571
-	arm_notify_die("", regs, &info, fsr, 0);
530
+	arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,
531
+		       fsr, 0);
572
532
 }
573
533
 
574
534
 void __init
..
..
@@ -588,7 +548,6 @@
588
548
 do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
589
549
 {
590
550
 	const struct fsr_info *inf = ifsr_info + fsr_fs(ifsr);
591
-	struct siginfo info;
592
551
 
593
552
 	if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs))
594
553
 		return;
..
..
@@ -596,12 +555,8 @@
596
555
 	pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n",
597
556
 		inf->name, ifsr, addr);
598
557
 
599
-	clear_siginfo(&info);
600
-	info.si_signo = inf->sig;
601
-	info.si_errno = 0;
602
-	info.si_code  = inf->code;
603
-	info.si_addr  = (void __user *)addr;
604
-	arm_notify_die("", regs, &info, ifsr, 0);
558
+	arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,
559
+		       ifsr, 0);
605
560
 }
606
561
 
607
562
 /*