forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/arch/x86/kernel/nmi.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  *  Copyright (C) 1991, 1992  Linus Torvalds
34  *  Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
....@@ -21,13 +22,10 @@
2122 #include <linux/ratelimit.h>
2223 #include <linux/slab.h>
2324 #include <linux/export.h>
25
+#include <linux/atomic.h>
2426 #include <linux/sched/clock.h>
2527 
26
-#if defined(CONFIG_EDAC)
27
-#include <linux/edac.h>
28
-#endif
29
-
30
-#include <linux/atomic.h>
28
+#include <asm/cpu_entry_area.h>
3129 #include <asm/traps.h>
3230 #include <asm/mach_traps.h>
3331 #include <asm/nmi.h>
....@@ -35,6 +33,7 @@
3533 #include <asm/reboot.h>
3634 #include <asm/cache.h>
3735 #include <asm/nospec-branch.h>
36
+#include <asm/sev-es.h>
3837 
3938 #define CREATE_TRACE_POINTS
4039 #include <trace/events/nmi.h>
....@@ -304,7 +303,7 @@
304303 static DEFINE_PER_CPU(bool, swallow_nmi);
305304 static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
306305 
307
-static void default_do_nmi(struct pt_regs *regs)
306
+static noinstr void default_do_nmi(struct pt_regs *regs)
308307 {
309308 	unsigned char reason = 0;
310309 	int handled;
....@@ -330,6 +329,8 @@
330329 
331330 	__this_cpu_write(last_nmi_rip, regs->ip);
332331 
332
+	instrumentation_begin();
333
+
333334 	handled = nmi_handle(NMI_LOCAL, regs);
334335 	__this_cpu_add(nmi_stats.normal, handled);
335336 	if (handled) {
....@@ -343,7 +344,7 @@
343344 		 */
344345 		if (handled > 1)
345346 			__this_cpu_write(swallow_nmi, true);
346
-		return;
347
+		goto out;
347348 	}
348349 
349350 	/*
....@@ -375,7 +376,7 @@
375376 #endif
376377 		__this_cpu_add(nmi_stats.external, 1);
377378 		raw_spin_unlock(&nmi_reason_lock);
378
-		return;
379
+		goto out;
379380 	}
380381 	raw_spin_unlock(&nmi_reason_lock);
381382 
....@@ -400,9 +401,9 @@
400401 	 * a 'real' unknown NMI.  For example, while processing
401402 	 * a perf NMI another perf NMI comes in along with a
402403 	 * 'real' unknown NMI.  These two NMIs get combined into
403
-	 * one (as descibed above).  When the next NMI gets
404
+	 * one (as described above).  When the next NMI gets
404405 	 * processed, it will be flagged by perf as handled, but
405
-	 * noone will know that there was a 'real' unknown NMI sent
406
+	 * no one will know that there was a 'real' unknown NMI sent
406407 	 * also.  As a result it gets swallowed.  Or if the first
407408 	 * perf NMI returns two events handled then the second
408409 	 * NMI will get eaten by the logic below, again losing a
....@@ -413,8 +414,10 @@
413414 		__this_cpu_add(nmi_stats.swallow, 1);
414415 	else
415416 		unknown_nmi_error(reason, regs);
417
+
418
+out:
419
+	instrumentation_end();
416420 }
417
-NOKPROBE_SYMBOL(default_do_nmi);
418421 
419422 /*
420423  * NMIs can page fault or hit breakpoints which will cause it to lose
....@@ -468,28 +471,21 @@
468471 };
469472 static DEFINE_PER_CPU(enum nmi_states, nmi_state);
470473 static DEFINE_PER_CPU(unsigned long, nmi_cr2);
474
+static DEFINE_PER_CPU(unsigned long, nmi_dr7);
471475 
472
-#ifdef CONFIG_X86_64
473
-/*
474
- * In x86_64, we need to handle breakpoint -> NMI -> breakpoint.  Without
475
- * some care, the inner breakpoint will clobber the outer breakpoint's
476
- * stack.
477
- *
478
- * If a breakpoint is being processed, and the debug stack is being
479
- * used, if an NMI comes in and also hits a breakpoint, the stack
480
- * pointer will be set to the same fixed address as the breakpoint that
481
- * was interrupted, causing that stack to be corrupted. To handle this
482
- * case, check if the stack that was interrupted is the debug stack, and
483
- * if so, change the IDT so that new breakpoints will use the current
484
- * stack and not switch to the fixed address. On return of the NMI,
485
- * switch back to the original IDT.
486
- */
487
-static DEFINE_PER_CPU(int, update_debug_stack);
488
-#endif
489
-
490
-dotraplinkage notrace void
491
-do_nmi(struct pt_regs *regs, long error_code)
476
+DEFINE_IDTENTRY_RAW(exc_nmi)
492477 {
478
+	irqentry_state_t irq_state;
479
+
480
+	/*
481
+	 * Re-enable NMIs right here when running as an SEV-ES guest. This might
482
+	 * cause nested NMIs, but those can be handled safely.
483
+	 */
484
+	sev_es_nmi_complete();
485
+
486
+	if (IS_ENABLED(CONFIG_SMP) && arch_cpu_is_offline(smp_processor_id()))
487
+		return;
488
+
493489 	if (this_cpu_read(nmi_state) != NMI_NOT_RUNNING) {
494490 		this_cpu_write(nmi_state, NMI_LATCHED);
495491 		return;
....@@ -498,34 +494,26 @@
498494 	this_cpu_write(nmi_cr2, read_cr2());
499495 nmi_restart:
500496 
501
-#ifdef CONFIG_X86_64
502497 	/*
503
-	 * If we interrupted a breakpoint, it is possible that
504
-	 * the nmi handler will have breakpoints too. We need to
505
-	 * change the IDT such that breakpoints that happen here
506
-	 * continue to use the NMI stack.
498
+	 * Needs to happen before DR7 is accessed, because the hypervisor can
499
+	 * intercept DR7 reads/writes, turning those into #VC exceptions.
507500 	 */
508
-	if (unlikely(is_debug_stack(regs->sp))) {
509
-		debug_stack_set_zero();
510
-		this_cpu_write(update_debug_stack, 1);
511
-	}
512
-#endif
501
+	sev_es_ist_enter(regs);
513502 
514
-	nmi_enter();
503
+	this_cpu_write(nmi_dr7, local_db_save());
504
+
505
+	irq_state = irqentry_nmi_enter(regs);
515506 
516507 	inc_irq_stat(__nmi_count);
517508 
518509 	if (!ignore_nmis)
519510 		default_do_nmi(regs);
520511 
521
-	nmi_exit();
512
+	irqentry_nmi_exit(regs, irq_state);
522513 
523
-#ifdef CONFIG_X86_64
524
-	if (unlikely(this_cpu_read(update_debug_stack))) {
525
-		debug_stack_reset();
526
-		this_cpu_write(update_debug_stack, 0);
527
-	}
528
-#endif
514
+	local_db_restore(this_cpu_read(nmi_dr7));
515
+
516
+	sev_es_ist_exit();
529517 
530518 	if (unlikely(this_cpu_read(nmi_cr2) != read_cr2()))
531519 		write_cr2(this_cpu_read(nmi_cr2));
....@@ -535,7 +523,16 @@
535523 	if (user_mode(regs))
536524 		mds_user_clear_cpu_buffers();
537525 }
538
-NOKPROBE_SYMBOL(do_nmi);
526
+
527
+#if defined(CONFIG_X86_64) && IS_ENABLED(CONFIG_KVM_INTEL)
528
+DEFINE_IDTENTRY_RAW(exc_nmi_noist)
529
+{
530
+	exc_nmi(regs);
531
+}
532
+#endif
533
+#if IS_MODULE(CONFIG_KVM_INTEL)
534
+EXPORT_SYMBOL_GPL(asm_exc_nmi_noist);
535
+#endif
539536 
540537 void stop_nmi(void)
541538 {