blame | history | patch | commit | commitdiff
hc
2024-11-01 2f529f9b558ca1c1bd74be7437a84e4711743404 
 kernel/arch/x86/kernel/apic/apic.c
....@@ -31,6 +31,7 @@
3131 #include <linux/i8253.h>
3232 #include <linux/dmar.h>
3333 #include <linux/init.h>
34
+#include <linux/irq.h>
3435 #include <linux/cpu.h>
3536 #include <linux/dmi.h>
3637 #include <linux/smp.h>
....@@ -272,10 +273,10 @@
272273 {
273274 	unsigned long flags;
274275 
275
-	local_irq_save(flags);
276
+	flags = hard_local_irq_save();
276277 	apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id));
277278 	apic_write(APIC_ICR, low);
278
-	local_irq_restore(flags);
279
+	hard_local_irq_restore(flags);
279280 }
280281 
281282 u64 native_apic_icr_read(void)
....@@ -331,6 +332,9 @@
331332 static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
332333 {
333334 	unsigned int lvtt_value, tmp_value;
335
+	unsigned long flags;
336
+
337
+	flags = hard_cond_local_irq_save();
334338 
335339 	lvtt_value = LOCAL_TIMER_VECTOR;
336340 	if (!oneshot)
....@@ -353,6 +357,8 @@
353357 		 * According to Intel, MFENCE can do the serialization here.
354358 		 */
355359 		asm volatile("mfence" : : : "memory");
360
+		hard_cond_local_irq_restore(flags);
361
+		printk_once(KERN_DEBUG "TSC deadline timer enabled\n");
356362 		return;
357363 	}
358364 
....@@ -366,6 +372,8 @@
366372 
367373 	if (!oneshot)
368374 		apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
375
+
376
+	hard_cond_local_irq_restore(flags);
369377 }
370378 
371379 /*
....@@ -471,28 +479,34 @@
471479 static int lapic_next_deadline(unsigned long delta,
472480 			       struct clock_event_device *evt)
473481 {
482
+	unsigned long flags;
474483 	u64 tsc;
475484 
476485 	/* This MSR is special and need a special fence: */
477486 	weak_wrmsr_fence();
478487 
488
+	flags = hard_local_irq_save();
479489 	tsc = rdtsc();
480490 	wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
491
+	hard_local_irq_restore(flags);
481492 	return 0;
482493 }
483494 
484495 static int lapic_timer_shutdown(struct clock_event_device *evt)
485496 {
497
+	unsigned long flags;
486498 	unsigned int v;
487499 
488500 	/* Lapic used as dummy for broadcast ? */
489501 	if (evt->features & CLOCK_EVT_FEAT_DUMMY)
490502 		return 0;
491503 
504
+	flags = hard_local_irq_save();
492505 	v = apic_read(APIC_LVTT);
493506 	v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
494507 	apic_write(APIC_LVTT, v);
495508 	apic_write(APIC_TMICT, 0);
509
+	hard_local_irq_restore(flags);
496510 	return 0;
497511 }
498512 
....@@ -527,6 +541,32 @@
527541 #endif
528542 }
529543 
544
+static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
545
+
546
+#ifdef CONFIG_IRQ_PIPELINE
547
+
548
+#define LAPIC_TIMER_IRQ  apicm_vector_irq(LOCAL_TIMER_VECTOR)
549
+
550
+static irqreturn_t lapic_oob_handler(int irq, void *dev_id)
551
+{
552
+	struct clock_event_device *evt = this_cpu_ptr(&lapic_events);
553
+
554
+	trace_local_timer_entry(LOCAL_TIMER_VECTOR);
555
+	clockevents_handle_event(evt);
556
+	trace_local_timer_exit(LOCAL_TIMER_VECTOR);
557
+
558
+	return IRQ_HANDLED;
559
+}
560
+
561
+static struct irqaction lapic_oob_action = {
562
+	.handler = lapic_oob_handler,
563
+	.name = "Out-of-band LAPIC timer interrupt",
564
+	.flags = IRQF_TIMER | IRQF_PERCPU,
565
+};
566
+
567
+#else
568
+#define LAPIC_TIMER_IRQ  -1
569
+#endif
530570 
531571 /*
532572  * The local apic timer can be used for any function which is CPU local.
....@@ -534,8 +574,8 @@
534574 static struct clock_event_device lapic_clockevent = {
535575 	.name				= "lapic",
536576 	.features			= CLOCK_EVT_FEAT_PERIODIC |
537
-					  CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP
538
-					  | CLOCK_EVT_FEAT_DUMMY,
577
+					  CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP  |
578
+					  CLOCK_EVT_FEAT_PIPELINE | CLOCK_EVT_FEAT_DUMMY,
539579 	.shift				= 32,
540580 	.set_state_shutdown		= lapic_timer_shutdown,
541581 	.set_state_periodic		= lapic_timer_set_periodic,
....@@ -544,9 +584,8 @@
544584 	.set_next_event			= lapic_next_event,
545585 	.broadcast			= lapic_timer_broadcast,
546586 	.rating				= 100,
547
-	.irq				= -1,
587
+	.irq				= LAPIC_TIMER_IRQ,
548588 };
549
-static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
550589 
551590 static const struct x86_cpu_id deadline_match[] __initconst = {
552591 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x2, 0x2), 0x3a), /* EP */
....@@ -1042,6 +1081,9 @@
10421081 	/* Setup the lapic or request the broadcast */
10431082 	setup_APIC_timer();
10441083 	amd_e400_c1e_apic_setup();
1084
+#ifdef CONFIG_IRQ_PIPELINE
1085
+	setup_percpu_irq(LAPIC_TIMER_IRQ, &lapic_oob_action);
1086
+#endif
10451087 }
10461088 
10471089 void setup_secondary_APIC_clock(void)
....@@ -1092,7 +1134,8 @@
10921134  * [ if a single-CPU system runs an SMP kernel then we call the local
10931135  *   interrupt as well. Thus we cannot inline the local irq ... ]
10941136  */
1095
-DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt)
1137
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(LOCAL_TIMER_VECTOR,
1138
+				 sysvec_apic_timer_interrupt)
10961139 {
10971140 	struct pt_regs *old_regs = set_irq_regs(regs);
10981141 
....@@ -1513,7 +1556,7 @@
15131556 		 * per set bit.
15141557 		 */
15151558 		for_each_set_bit(bit, isr->map, APIC_IR_BITS)
1516
-			ack_APIC_irq();
1559
+			__ack_APIC_irq();
15171560 		return true;
15181561 	}
15191562 
....@@ -2131,7 +2174,7 @@
21312174  *
21322175  * Also called from sysvec_spurious_apic_interrupt().
21332176  */
2134
-DEFINE_IDTENTRY_IRQ(spurious_interrupt)
2177
+DEFINE_IDTENTRY_IRQ_PIPELINED(spurious_interrupt)
21352178 {
21362179 	u32 v;
21372180 
....@@ -2157,7 +2200,7 @@
21572200 	if (v & (1 << (vector & 0x1f))) {
21582201 		pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n",
21592202 			vector, smp_processor_id());
2160
-		ack_APIC_irq();
2203
+		__ack_APIC_irq();
21612204 	} else {
21622205 		pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n",
21632206 			vector, smp_processor_id());
....@@ -2166,13 +2209,18 @@
21662209 	trace_spurious_apic_exit(vector);
21672210 }
21682211 
2169
-DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt)
2212
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(SPURIOUS_APIC_VECTOR,
2213
+				 sysvec_spurious_apic_interrupt)
21702214 {
21712215 	__spurious_interrupt(regs, SPURIOUS_APIC_VECTOR);
21722216 }
21732217 
21742218 /*
21752219  * This interrupt should never happen with our APIC/SMP architecture
2220
+ *
2221
+ * irq_pipeline: same as spurious_interrupt, would run directly out of
2222
+ * the IDT, no deferral via the interrupt log which means that only
2223
+ * the hardware IRQ state is considered for masking.
21762224  */
21772225 DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)
21782226 {