blame | history | patch | commit | commitdiff
hc
2024-11-01 2f529f9b558ca1c1bd74be7437a84e4711743404 
 kernel/kernel/irq/chip.c
....@@ -15,6 +15,7 @@
1515 #include <linux/kernel_stat.h>
1616 #include <linux/irqdomain.h>
1717 #include <linux/wakeup_reason.h>
18
+#include <linux/irq_pipeline.h>
1819 
1920 #include <trace/events/irq.h>
2021 
....@@ -49,6 +50,10 @@
4950 
5051 	if (!chip)
5152 		chip = &no_irq_chip;
53
+	else
54
+		WARN_ONCE(irqs_pipelined() &&
55
+			  (chip->flags & IRQCHIP_PIPELINE_SAFE) == 0,
56
+			  "irqchip %s is not pipeline-safe!", chip->name);
5257 
5358 	desc->irq_data.chip = chip;
5459 	irq_put_desc_unlock(desc, flags);
....@@ -155,14 +160,6 @@
155160 	return 0;
156161 }
157162 EXPORT_SYMBOL(irq_set_chip_data);
158
-
159
-struct irq_data *irq_get_irq_data(unsigned int irq)
160
-{
161
-	struct irq_desc *desc = irq_to_desc(irq);
162
-
163
-	return desc ? &desc->irq_data : NULL;
164
-}
165
-EXPORT_SYMBOL_GPL(irq_get_irq_data);
166163 
167164 static void irq_state_clr_disabled(struct irq_desc *desc)
168165 {
....@@ -386,7 +383,8 @@
386383  */
387384 void irq_disable(struct irq_desc *desc)
388385 {
389
-	__irq_disable(desc, irq_settings_disable_unlazy(desc));
386
+	__irq_disable(desc,
387
+	      irq_settings_disable_unlazy(desc) || irqs_pipelined());
390388 }
391389 
392390 void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
....@@ -532,8 +530,22 @@
532530 	 * If the interrupt is an armed wakeup source, mark it pending
533531 	 * and suspended, disable it and notify the pm core about the
534532 	 * event.
533
+	 *
534
+	 * When pipelining, the logic is as follows:
535
+	 *
536
+	 * - from a pipeline entry context, we might have preempted
537
+	 * the oob stage, or irqs might be [virtually] off, so we may
538
+	 * not run the in-band PM code. Just make sure any wakeup
539
+	 * interrupt is detected later on when the flow handler
540
+	 * re-runs from the in-band stage.
541
+	 *
542
+	 * - from the in-band context, run the PM wakeup check.
535543 	 */
536
-	if (irq_pm_check_wakeup(desc))
544
+	if (irqs_pipelined()) {
545
+		WARN_ON_ONCE(irq_pipeline_debug() && !in_pipeline());
546
+		if (irqd_is_wakeup_armed(&desc->irq_data))
547
+			return true;
548
+	} else if (irq_pm_check_wakeup(desc))
537549 		return false;
538550 
539551 	/*
....@@ -557,8 +569,13 @@
557569 {
558570 	raw_spin_lock(&desc->lock);
559571 
560
-	if (!irq_may_run(desc))
572
+	if (start_irq_flow() && !irq_may_run(desc))
561573 		goto out_unlock;
574
+
575
+	if (on_pipeline_entry()) {
576
+		handle_oob_irq(desc);
577
+		goto out_unlock;
578
+	}
562579 
563580 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
564581 
....@@ -594,8 +611,13 @@
594611 
595612 	raw_spin_lock(&desc->lock);
596613 
597
-	if (!irq_may_run(desc))
614
+	if (start_irq_flow() && !irq_may_run(desc))
598615 		goto out_unlock;
616
+
617
+	if (on_pipeline_entry()) {
618
+		handle_oob_irq(desc);
619
+		goto out_unlock;
620
+	}
599621 
600622 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
601623 
....@@ -617,6 +639,20 @@
617639 	raw_spin_unlock(&desc->lock);
618640 }
619641 EXPORT_SYMBOL_GPL(handle_untracked_irq);
642
+
643
+static inline void cond_eoi_irq(struct irq_desc *desc)
644
+{
645
+	struct irq_chip *chip = desc->irq_data.chip;
646
+
647
+	if (!(chip->flags & IRQCHIP_EOI_THREADED))
648
+		chip->irq_eoi(&desc->irq_data);
649
+}
650
+
651
+static inline void mask_cond_eoi_irq(struct irq_desc *desc)
652
+{
653
+	mask_irq(desc);
654
+	cond_eoi_irq(desc);
655
+}
620656 
621657 /*
622658  * Called unconditionally from handle_level_irq() and only for oneshot
....@@ -648,10 +684,19 @@
648684 void handle_level_irq(struct irq_desc *desc)
649685 {
650686 	raw_spin_lock(&desc->lock);
651
-	mask_ack_irq(desc);
652687 
653
-	if (!irq_may_run(desc))
688
+	if (start_irq_flow()) {
689
+		mask_ack_irq(desc);
690
+
691
+		if (!irq_may_run(desc))
692
+			goto out_unlock;
693
+	}
694
+
695
+	if (on_pipeline_entry()) {
696
+		if (handle_oob_irq(desc))
697
+			goto out_unmask;
654698 		goto out_unlock;
699
+	}
655700 
656701 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
657702 
....@@ -666,7 +711,7 @@
666711 
667712 	kstat_incr_irqs_this_cpu(desc);
668713 	handle_irq_event(desc);
669
-
714
+out_unmask:
670715 	cond_unmask_irq(desc);
671716 
672717 out_unlock:
....@@ -677,7 +722,10 @@
677722 static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
678723 {
679724 	if (!(desc->istate & IRQS_ONESHOT)) {
680
-		chip->irq_eoi(&desc->irq_data);
725
+		if (!irqs_pipelined())
726
+			chip->irq_eoi(&desc->irq_data);
727
+		else if (!irqd_irq_disabled(&desc->irq_data))
728
+			unmask_irq(desc);
681729 		return;
682730 	}
683731 	/*
....@@ -688,9 +736,11 @@
688736 	 */
689737 	if (!irqd_irq_disabled(&desc->irq_data) &&
690738 	    irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot) {
691
-		chip->irq_eoi(&desc->irq_data);
739
+		if (!irqs_pipelined())
740
+			chip->irq_eoi(&desc->irq_data);
692741 		unmask_irq(desc);
693
-	} else if (!(chip->flags & IRQCHIP_EOI_THREADED)) {
742
+	} else if (!irqs_pipelined() &&
743
+		   !(chip->flags & IRQCHIP_EOI_THREADED)) {
694744 		chip->irq_eoi(&desc->irq_data);
695745 	}
696746 }
....@@ -710,8 +760,16 @@
710760 
711761 	raw_spin_lock(&desc->lock);
712762 
713
-	if (!irq_may_run(desc))
763
+	if (start_irq_flow() && !irq_may_run(desc))
714764 		goto out;
765
+
766
+	if (on_pipeline_entry()) {
767
+		if (handle_oob_irq(desc))
768
+			chip->irq_eoi(&desc->irq_data);
769
+		else
770
+			mask_cond_eoi_irq(desc);
771
+		goto out_unlock;
772
+	}
715773 
716774 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
717775 
....@@ -726,13 +784,13 @@
726784 	}
727785 
728786 	kstat_incr_irqs_this_cpu(desc);
729
-	if (desc->istate & IRQS_ONESHOT)
787
+	if (!irqs_pipelined() && (desc->istate & IRQS_ONESHOT))
730788 		mask_irq(desc);
731789 
732790 	handle_irq_event(desc);
733791 
734792 	cond_unmask_eoi_irq(desc, chip);
735
-
793
+out_unlock:
736794 	raw_spin_unlock(&desc->lock);
737795 	return;
738796 out:
....@@ -792,30 +850,42 @@
792850  */
793851 void handle_edge_irq(struct irq_desc *desc)
794852 {
853
+	struct irq_chip *chip = irq_desc_get_chip(desc);
854
+
795855 	raw_spin_lock(&desc->lock);
796856 
797
-	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
857
+	if (start_irq_flow()) {
858
+		desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
798859 
799
-	if (!irq_may_run(desc)) {
800
-		desc->istate |= IRQS_PENDING;
801
-		mask_ack_irq(desc);
802
-		goto out_unlock;
860
+		if (!irq_may_run(desc)) {
861
+			desc->istate |= IRQS_PENDING;
862
+			mask_ack_irq(desc);
863
+			goto out_unlock;
864
+		}
865
+
866
+		/*
867
+		 * If its disabled or no action available then mask it
868
+		 * and get out of here.
869
+		 */
870
+		if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
871
+			desc->istate |= IRQS_PENDING;
872
+			mask_ack_irq(desc);
873
+			goto out_unlock;
874
+		}
803875 	}
804876 
805
-	/*
806
-	 * If its disabled or no action available then mask it and get
807
-	 * out of here.
808
-	 */
809
-	if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
810
-		desc->istate |= IRQS_PENDING;
811
-		mask_ack_irq(desc);
877
+	if (on_pipeline_entry()) {
878
+		chip->irq_ack(&desc->irq_data);
879
+		desc->istate |= IRQS_EDGE;
880
+		handle_oob_irq(desc);
812881 		goto out_unlock;
813882 	}
814883 
815884 	kstat_incr_irqs_this_cpu(desc);
816885 
817886 	/* Start handling the irq */
818
-	desc->irq_data.chip->irq_ack(&desc->irq_data);
887
+	if (!irqs_pipelined())
888
+		chip->irq_ack(&desc->irq_data);
819889 
820890 	do {
821891 		if (unlikely(!desc->action)) {
....@@ -840,6 +910,8 @@
840910 		 !irqd_irq_disabled(&desc->irq_data));
841911 
842912 out_unlock:
913
+	if (on_pipeline_entry())
914
+		desc->istate &= ~IRQS_EDGE;
843915 	raw_spin_unlock(&desc->lock);
844916 }
845917 EXPORT_SYMBOL(handle_edge_irq);
....@@ -858,11 +930,20 @@
858930 
859931 	raw_spin_lock(&desc->lock);
860932 
861
-	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
933
+	if (start_irq_flow()) {
934
+		desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
862935 
863
-	if (!irq_may_run(desc)) {
864
-		desc->istate |= IRQS_PENDING;
865
-		goto out_eoi;
936
+		if (!irq_may_run(desc)) {
937
+			desc->istate |= IRQS_PENDING;
938
+			goto out_eoi;
939
+		}
940
+	}
941
+
942
+	if (on_pipeline_entry()) {
943
+		desc->istate |= IRQS_EDGE;
944
+		if (handle_oob_irq(desc))
945
+			goto out_eoi;
946
+		goto out;
866947 	}
867948 
868949 	/*
....@@ -887,6 +968,9 @@
887968 
888969 out_eoi:
889970 	chip->irq_eoi(&desc->irq_data);
971
+out:
972
+	if (on_pipeline_entry())
973
+		desc->istate &= ~IRQS_EDGE;
890974 	raw_spin_unlock(&desc->lock);
891975 }
892976 #endif
....@@ -900,6 +984,18 @@
900984 void handle_percpu_irq(struct irq_desc *desc)
901985 {
902986 	struct irq_chip *chip = irq_desc_get_chip(desc);
987
+	bool handled;
988
+
989
+	if (on_pipeline_entry()) {
990
+		if (chip->irq_ack)
991
+			chip->irq_ack(&desc->irq_data);
992
+		handled = handle_oob_irq(desc);
993
+		if (chip->irq_eoi)
994
+			chip->irq_eoi(&desc->irq_data);
995
+		if (!handled && chip->irq_mask)
996
+			chip->irq_mask(&desc->irq_data);
997
+		return;
998
+	}
903999 
9041000 	/*
9051001 	 * PER CPU interrupts are not serialized. Do not touch
....@@ -907,13 +1003,17 @@
9071003 	 */
9081004 	__kstat_incr_irqs_this_cpu(desc);
9091005 
910
-	if (chip->irq_ack)
911
-		chip->irq_ack(&desc->irq_data);
912
-
913
-	handle_irq_event_percpu(desc);
914
-
915
-	if (chip->irq_eoi)
916
-		chip->irq_eoi(&desc->irq_data);
1006
+	if (irqs_pipelined()) {
1007
+		handle_irq_event_percpu(desc);
1008
+		if (chip->irq_unmask)
1009
+			chip->irq_unmask(&desc->irq_data);
1010
+	} else {
1011
+		if (chip->irq_ack)
1012
+			chip->irq_ack(&desc->irq_data);
1013
+		handle_irq_event_percpu(desc);
1014
+		if (chip->irq_eoi)
1015
+			chip->irq_eoi(&desc->irq_data);
1016
+	}
9171017 }
9181018 
9191019 /**
....@@ -933,6 +1033,18 @@
9331033 	struct irqaction *action = desc->action;
9341034 	unsigned int irq = irq_desc_get_irq(desc);
9351035 	irqreturn_t res;
1036
+	bool handled;
1037
+
1038
+	if (on_pipeline_entry()) {
1039
+		if (chip->irq_ack)
1040
+			chip->irq_ack(&desc->irq_data);
1041
+		handled = handle_oob_irq(desc);
1042
+		if (chip->irq_eoi)
1043
+			chip->irq_eoi(&desc->irq_data);
1044
+		if (!handled && chip->irq_mask)
1045
+			chip->irq_mask(&desc->irq_data);
1046
+		return;
1047
+	}
9361048 
9371049 	/*
9381050 	 * PER CPU interrupts are not serialized. Do not touch
....@@ -940,7 +1052,7 @@
9401052 	 */
9411053 	__kstat_incr_irqs_this_cpu(desc);
9421054 
943
-	if (chip->irq_ack)
1055
+	if (!irqs_pipelined() && chip->irq_ack)
9441056 		chip->irq_ack(&desc->irq_data);
9451057 
9461058 	if (likely(action)) {
....@@ -958,8 +1070,11 @@
9581070 			    enabled ? " and unmasked" : "", irq, cpu);
9591071 	}
9601072 
961
-	if (chip->irq_eoi)
962
-		chip->irq_eoi(&desc->irq_data);
1073
+	if (irqs_pipelined()) {
1074
+		if (chip->irq_unmask)
1075
+			chip->irq_unmask(&desc->irq_data);
1076
+	} else if (chip->irq_eoi)
1077
+			chip->irq_eoi(&desc->irq_data);
9631078 }
9641079 
9651080 /**
....@@ -979,10 +1094,21 @@
9791094 	unsigned int irq = irq_desc_get_irq(desc);
9801095 	irqreturn_t res;
9811096 
982
-	__kstat_incr_irqs_this_cpu(desc);
983
-
9841097 	if (chip->irq_eoi)
9851098 		chip->irq_eoi(&desc->irq_data);
1099
+
1100
+	if (on_pipeline_entry()) {
1101
+		handle_oob_irq(desc);
1102
+		return;
1103
+	}
1104
+
1105
+	/* Trap spurious IPIs if pipelined. */
1106
+	if (irqs_pipelined() && !action) {
1107
+		print_irq_desc(irq, desc);
1108
+		return;
1109
+	}
1110
+
1111
+	__kstat_incr_irqs_this_cpu(desc);
9861112 
9871113 	trace_irq_handler_entry(irq, action);
9881114 	res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
....@@ -1076,6 +1202,7 @@
10761202 			desc->handle_irq = handle;
10771203 		}
10781204 
1205
+		irq_settings_set_chained(desc);
10791206 		irq_settings_set_noprobe(desc);
10801207 		irq_settings_set_norequest(desc);
10811208 		irq_settings_set_nothread(desc);
....@@ -1251,8 +1378,17 @@
12511378 
12521379 	raw_spin_lock(&desc->lock);
12531380 
1254
-	if (!irq_may_run(desc))
1381
+	if (start_irq_flow() && !irq_may_run(desc))
12551382 		goto out;
1383
+
1384
+	if (on_pipeline_entry()) {
1385
+		chip->irq_ack(&desc->irq_data);
1386
+		if (handle_oob_irq(desc))
1387
+			chip->irq_eoi(&desc->irq_data);
1388
+		else
1389
+			mask_cond_eoi_irq(desc);
1390
+		goto out_unlock;
1391
+	}
12561392 
12571393 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
12581394 
....@@ -1267,11 +1403,13 @@
12671403 	}
12681404 
12691405 	kstat_incr_irqs_this_cpu(desc);
1270
-	if (desc->istate & IRQS_ONESHOT)
1271
-		mask_irq(desc);
1406
+	if (!irqs_pipelined()) {
1407
+		if (desc->istate & IRQS_ONESHOT)
1408
+			mask_irq(desc);
12721409 
1273
-	/* Start handling the irq */
1274
-	desc->irq_data.chip->irq_ack(&desc->irq_data);
1410
+		/* Start handling the irq */
1411
+		chip->irq_ack(&desc->irq_data);
1412
+	}
12751413 
12761414 	handle_irq_event(desc);
12771415 
....@@ -1282,6 +1420,7 @@
12821420 out:
12831421 	if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
12841422 		chip->irq_eoi(&desc->irq_data);
1423
+out_unlock:
12851424 	raw_spin_unlock(&desc->lock);
12861425 }
12871426 EXPORT_SYMBOL_GPL(handle_fasteoi_ack_irq);
....@@ -1301,10 +1440,21 @@
13011440 	struct irq_chip *chip = desc->irq_data.chip;
13021441 
13031442 	raw_spin_lock(&desc->lock);
1304
-	mask_ack_irq(desc);
13051443 
1306
-	if (!irq_may_run(desc))
1307
-		goto out;
1444
+	if (start_irq_flow()) {
1445
+		mask_ack_irq(desc);
1446
+
1447
+		if (!irq_may_run(desc))
1448
+			goto out;
1449
+	}
1450
+
1451
+	if (on_pipeline_entry()) {
1452
+		if (handle_oob_irq(desc))
1453
+			chip->irq_eoi(&desc->irq_data);
1454
+		else
1455
+			cond_eoi_irq(desc);
1456
+		goto out_unlock;
1457
+	}
13081458 
13091459 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
13101460 
....@@ -1319,7 +1469,7 @@
13191469 	}
13201470 
13211471 	kstat_incr_irqs_this_cpu(desc);
1322
-	if (desc->istate & IRQS_ONESHOT)
1472
+	if (!irqs_pipelined() && (desc->istate & IRQS_ONESHOT))
13231473 		mask_irq(desc);
13241474 
13251475 	handle_irq_event(desc);
....@@ -1331,6 +1481,7 @@
13311481 out:
13321482 	if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
13331483 		chip->irq_eoi(&desc->irq_data);
1484
+out_unlock:
13341485 	raw_spin_unlock(&desc->lock);
13351486 }
13361487 EXPORT_SYMBOL_GPL(handle_fasteoi_mask_irq);