blame | history | patch | commit | commitdiff
hc
2024-11-01 2f529f9b558ca1c1bd74be7437a84e4711743404 
 kernel/arch/x86/kernel/traps.c
....@@ -74,14 +74,22 @@
7474 
7575 static inline void cond_local_irq_enable(struct pt_regs *regs)
7676 {
77
-	if (regs->flags & X86_EFLAGS_IF)
78
-		local_irq_enable();
77
+	if (regs->flags & X86_EFLAGS_IF) {
78
+		if (running_inband())
79
+			local_irq_enable_full();
80
+		else
81
+			hard_local_irq_enable();
82
+	}
7983 }
8084 
8185 static inline void cond_local_irq_disable(struct pt_regs *regs)
8286 {
83
-	if (regs->flags & X86_EFLAGS_IF)
84
-		local_irq_disable();
87
+	if (regs->flags & X86_EFLAGS_IF) {
88
+		if (running_inband())
89
+			local_irq_disable_full();
90
+		else
91
+			hard_local_irq_disable();
92
+	}
8593 }
8694 
8795 __always_inline int is_valid_bugaddr(unsigned long addr)
....@@ -148,6 +156,39 @@
148156 	}
149157 }
150158 
159
+static __always_inline
160
+bool mark_trap_entry(int trapnr, struct pt_regs *regs)
161
+{
162
+	oob_trap_notify(trapnr, regs);
163
+
164
+	if (likely(running_inband())) {
165
+		hard_cond_local_irq_enable();
166
+		return true;
167
+	}
168
+
169
+	return false;
170
+}
171
+
172
+static __always_inline
173
+void mark_trap_exit(int trapnr, struct pt_regs *regs)
174
+{
175
+	oob_trap_unwind(trapnr, regs);
176
+	hard_cond_local_irq_disable();
177
+}
178
+
179
+static __always_inline
180
+bool mark_trap_entry_raw(int trapnr, struct pt_regs *regs)
181
+{
182
+	oob_trap_notify(trapnr, regs);
183
+	return running_inband();
184
+}
185
+
186
+static __always_inline
187
+void mark_trap_exit_raw(int trapnr, struct pt_regs *regs)
188
+{
189
+	oob_trap_unwind(trapnr, regs);
190
+}
191
+
151192 static void
152193 do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
153194 	long error_code, int sicode, void __user *addr)
....@@ -171,12 +212,17 @@
171212 {
172213 	RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
173214 
215
+	if (!mark_trap_entry(trapnr, regs))
216
+		return;
217
+
174218 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=
175219 			NOTIFY_STOP) {
176220 		cond_local_irq_enable(regs);
177221 		do_trap(trapnr, signr, str, regs, error_code, sicode, addr);
178222 		cond_local_irq_disable(regs);
179223 	}
224
+
225
+	mark_trap_exit(trapnr, regs);
180226 }
181227 
182228 /*
....@@ -230,14 +276,22 @@
230276 	 * Since we're emulating a CALL with exceptions, restore the interrupt
231277 	 * state to what it was at the exception site.
232278 	 */
233
-	if (regs->flags & X86_EFLAGS_IF)
234
-		raw_local_irq_enable();
279
+	if (regs->flags & X86_EFLAGS_IF) {
280
+		if (running_oob())
281
+			hard_local_irq_enable();
282
+		else
283
+			local_irq_enable_full();
284
+	}
235285 	if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) {
236286 		regs->ip += LEN_UD2;
237287 		handled = true;
238288 	}
239
-	if (regs->flags & X86_EFLAGS_IF)
240
-		raw_local_irq_disable();
289
+	if (regs->flags & X86_EFLAGS_IF) {
290
+		if (running_oob())
291
+			hard_local_irq_disable();
292
+		else
293
+			local_irq_disable_full();
294
+	}
241295 	instrumentation_end();
242296 
243297 	return handled;
....@@ -251,15 +305,26 @@
251305 	 * We use UD2 as a short encoding for 'CALL __WARN', as such
252306 	 * handle it before exception entry to avoid recursive WARN
253307 	 * in case exception entry is the one triggering WARNs.
308
+	 *
309
+	 * dovetail: handle_bug() may run oob, so we do not downgrade
310
+	 * in-band upon a failed __WARN assertion since it might have
311
+	 * tripped in a section of code which would not be happy to
312
+	 * switch stage. However, anything else should be notified to
313
+	 * the core, because the kernel execution might be about to
314
+	 * stop, so we'd need to switch in-band to get any output
315
+	 * before this happens.
254316 	 */
255317 	if (!user_mode(regs) && handle_bug(regs))
256318 		return;
257319 
258
-	state = irqentry_enter(regs);
259
-	instrumentation_begin();
260
-	handle_invalid_op(regs);
261
-	instrumentation_end();
262
-	irqentry_exit(regs, state);
320
+	if (mark_trap_entry_raw(X86_TRAP_UD, regs)) {
321
+		state = irqentry_enter(regs);
322
+		instrumentation_begin();
323
+		handle_invalid_op(regs);
324
+		instrumentation_end();
325
+		irqentry_exit(regs, state);
326
+		mark_trap_exit_raw(X86_TRAP_UD, regs);
327
+	}
263328 }
264329 
265330 DEFINE_IDTENTRY(exc_coproc_segment_overrun)
....@@ -290,8 +355,11 @@
290355 {
291356 	char *str = "alignment check";
292357 
293
-	if (notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_AC, SIGBUS) == NOTIFY_STOP)
358
+	if (!mark_trap_entry(X86_TRAP_AC, regs))
294359 		return;
360
+
361
+	if (notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_AC, SIGBUS) == NOTIFY_STOP)
362
+		goto mark_exit;
295363 
296364 	if (!user_mode(regs))
297365 		die("Split lock detected\n", regs, error_code);
....@@ -306,6 +374,9 @@
306374 
307375 out:
308376 	local_irq_disable();
377
+
378
+mark_exit:
379
+	mark_trap_exit(X86_TRAP_AC, regs);
309380 }
310381 
311382 #ifdef CONFIG_VMAP_STACK
....@@ -341,6 +412,9 @@
341412  *
342413  * The 32bit #DF shim provides CR2 already as an argument. On 64bit it needs
343414  * to be read before doing anything else.
415
+ *
416
+ * Dovetail: do not even ask the companion core to try restoring the
417
+ * in-band stage on double-fault, this would be a lost cause.
344418  */
345419 DEFINE_IDTENTRY_DF(exc_double_fault)
346420 {
....@@ -465,9 +539,12 @@
465539 
466540 DEFINE_IDTENTRY(exc_bounds)
467541 {
542
+	if (!mark_trap_entry(X86_TRAP_BR, regs))
543
+		return;
544
+
468545 	if (notify_die(DIE_TRAP, "bounds", regs, 0,
469546 			X86_TRAP_BR, SIGSEGV) == NOTIFY_STOP)
470
-		return;
547
+		goto out;
471548 	cond_local_irq_enable(regs);
472549 
473550 	if (!user_mode(regs))
....@@ -476,6 +553,8 @@
476553 	do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, 0, 0, NULL);
477554 
478555 	cond_local_irq_disable(regs);
556
+out:
557
+	mark_trap_exit(X86_TRAP_BR, regs);
479558 }
480559 
481560 enum kernel_gp_hint {
....@@ -570,9 +649,9 @@
570649 	}
571650 
572651 	if (v8086_mode(regs)) {
573
-		local_irq_enable();
652
+		local_irq_enable_full();
574653 		handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
575
-		local_irq_disable();
654
+		local_irq_disable_full();
576655 		return;
577656 	}
578657 
....@@ -585,9 +664,12 @@
585664 		tsk->thread.error_code = error_code;
586665 		tsk->thread.trap_nr = X86_TRAP_GP;
587666 
667
+		if (!mark_trap_entry(X86_TRAP_GP, regs))
668
+			goto exit;
669
+
588670 		show_signal(tsk, SIGSEGV, "", desc, regs, error_code);
589671 		force_sig(SIGSEGV);
590
-		goto exit;
672
+		goto mark_exit;
591673 	}
592674 
593675 	if (fixup_exception(regs, X86_TRAP_GP, error_code, 0))
....@@ -605,9 +687,12 @@
605687 	    kprobe_fault_handler(regs, X86_TRAP_GP))
606688 		goto exit;
607689 
690
+	if (!mark_trap_entry(X86_TRAP_GP, regs))
691
+		goto exit;
692
+
608693 	ret = notify_die(DIE_GPF, desc, regs, error_code, X86_TRAP_GP, SIGSEGV);
609694 	if (ret == NOTIFY_STOP)
610
-		goto exit;
695
+		goto mark_exit;
611696 
612697 	if (error_code)
613698 		snprintf(desc, sizeof(desc), "segment-related " GPFSTR);
....@@ -629,6 +714,8 @@
629714 
630715 	die_addr(desc, regs, error_code, gp_addr);
631716 
717
+mark_exit:
718
+	mark_trap_exit(X86_TRAP_GP, regs);
632719 exit:
633720 	cond_local_irq_disable(regs);
634721 }
....@@ -673,6 +760,9 @@
673760 	if (poke_int3_handler(regs))
674761 		return;
675762 
763
+	if (!mark_trap_entry_raw(X86_TRAP_BP, regs))
764
+		return;
765
+
676766 	/*
677767 	 * irqentry_enter_from_user_mode() uses static_branch_{,un}likely()
678768 	 * and therefore can trigger INT3, hence poke_int3_handler() must
....@@ -695,6 +785,8 @@
695785 		instrumentation_end();
696786 		irqentry_nmi_exit(regs, irq_state);
697787 	}
788
+
789
+	mark_trap_exit_raw(X86_TRAP_BP, regs);
698790 }
699791 
700792 #ifdef CONFIG_X86_64
....@@ -999,7 +1091,7 @@
9991091 		goto out;
10001092 
10011093 	/* It's safe to allow irq's after DR6 has been saved */
1002
-	local_irq_enable();
1094
+	local_irq_enable_full();
10031095 
10041096 	if (v8086_mode(regs)) {
10051097 		handle_vm86_trap((struct kernel_vm86_regs *)regs, 0, X86_TRAP_DB);
....@@ -1012,7 +1104,7 @@
10121104 		send_sigtrap(regs, 0, get_si_code(dr6));
10131105 
10141106 out_irq:
1015
-	local_irq_disable();
1107
+	local_irq_disable_full();
10161108 out:
10171109 	instrumentation_end();
10181110 	irqentry_exit_to_user_mode(regs);
....@@ -1022,13 +1114,19 @@
10221114 /* IST stack entry */
10231115 DEFINE_IDTENTRY_DEBUG(exc_debug)
10241116 {
1025
-	exc_debug_kernel(regs, debug_read_clear_dr6());
1117
+	if (mark_trap_entry_raw(X86_TRAP_DB, regs)) {
1118
+		exc_debug_kernel(regs, debug_read_clear_dr6());
1119
+		mark_trap_exit_raw(X86_TRAP_DB, regs);
1120
+	}
10261121 }
10271122 
10281123 /* User entry, runs on regular task stack */
10291124 DEFINE_IDTENTRY_DEBUG_USER(exc_debug)
10301125 {
1031
-	exc_debug_user(regs, debug_read_clear_dr6());
1126
+	if (mark_trap_entry_raw(X86_TRAP_DB, regs)) {
1127
+		exc_debug_user(regs, debug_read_clear_dr6());
1128
+		mark_trap_exit_raw(X86_TRAP_DB, regs);
1129
+	}
10321130 }
10331131 #else
10341132 /* 32 bit does not have separate entry points. */
....@@ -1062,13 +1160,16 @@
10621160 		if (fixup_exception(regs, trapnr, 0, 0))
10631161 			goto exit;
10641162 
1163
+		if (!mark_trap_entry(trapnr, regs))
1164
+			goto exit;
1165
+
10651166 		task->thread.error_code = 0;
10661167 		task->thread.trap_nr = trapnr;
10671168 
10681169 		if (notify_die(DIE_TRAP, str, regs, 0, trapnr,
10691170 			       SIGFPE) != NOTIFY_STOP)
10701171 			die(str, regs, 0);
1071
-		goto exit;
1172
+		goto mark_exit;
10721173 	}
10731174 
10741175 	/*
....@@ -1084,8 +1185,13 @@
10841185 	if (!si_code)
10851186 		goto exit;
10861187 
1188
+	if (!mark_trap_entry(trapnr, regs))
1189
+		goto exit;
1190
+
10871191 	force_sig_fault(SIGFPE, si_code,
10881192 			(void __user *)uprobe_get_trap_addr(regs));
1193
+mark_exit:
1194
+	mark_trap_exit(trapnr, regs);
10891195 exit:
10901196 	cond_local_irq_disable(regs);
10911197 }
....@@ -1158,7 +1264,10 @@
11581264 		 * to kill the task than getting stuck in a never-ending
11591265 		 * loop of #NM faults.
11601266 		 */
1161
-		die("unexpected #NM exception", regs, 0);
1267
+		if (mark_trap_entry(X86_TRAP_NM, regs)) {
1268
+			die("unexpected #NM exception", regs, 0);
1269
+			mark_trap_exit(X86_TRAP_NM, regs);
1270
+		}
11621271 	}
11631272 }
11641273