forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/kernel/irq/manage.c
....@@ -18,13 +18,13 @@
1818 #include <linux/sched.h>
1919 #include <linux/sched/rt.h>
2020 #include <linux/sched/task.h>
21
+#include <linux/sched/isolation.h>
2122 #include <uapi/linux/sched/types.h>
2223 #include <linux/task_work.h>
2324 
2425 #include "internals.h"
2526 
26
-#ifdef CONFIG_IRQ_FORCED_THREADING
27
-# ifndef CONFIG_PREEMPT_RT_BASE
27
+#if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)
2828 __read_mostly bool force_irqthreads;
2929 EXPORT_SYMBOL_GPL(force_irqthreads);
3030 
....@@ -34,7 +34,6 @@
3434 	return 0;
3535 }
3636 early_param("threadirqs", setup_forced_irqthreads);
37
-# endif
3837 #endif
3938 
4039 static void __synchronize_hardirq(struct irq_desc *desc, bool sync_chip)
....@@ -224,16 +223,70 @@
224223 {
225224 	struct irq_desc *desc = irq_data_to_desc(data);
226225 	struct irq_chip *chip = irq_data_get_irq_chip(data);
226
+	const struct cpumask  *prog_mask;
227227 	int ret;
228
+
229
+	static DEFINE_RAW_SPINLOCK(tmp_mask_lock);
230
+	static struct cpumask tmp_mask;
228231 
229232 	if (!chip || !chip->irq_set_affinity)
230233 		return -EINVAL;
231234 
232
-	ret = chip->irq_set_affinity(data, mask, force);
235
+	raw_spin_lock(&tmp_mask_lock);
236
+	/*
237
+	 * If this is a managed interrupt and housekeeping is enabled on
238
+	 * it check whether the requested affinity mask intersects with
239
+	 * a housekeeping CPU. If so, then remove the isolated CPUs from
240
+	 * the mask and just keep the housekeeping CPU(s). This prevents
241
+	 * the affinity setter from routing the interrupt to an isolated
242
+	 * CPU to avoid that I/O submitted from a housekeeping CPU causes
243
+	 * interrupts on an isolated one.
244
+	 *
245
+	 * If the masks do not intersect or include online CPU(s) then
246
+	 * keep the requested mask. The isolated target CPUs are only
247
+	 * receiving interrupts when the I/O operation was submitted
248
+	 * directly from them.
249
+	 *
250
+	 * If all housekeeping CPUs in the affinity mask are offline, the
251
+	 * interrupt will be migrated by the CPU hotplug code once a
252
+	 * housekeeping CPU which belongs to the affinity mask comes
253
+	 * online.
254
+	 */
255
+	if (irqd_affinity_is_managed(data) &&
256
+	    housekeeping_enabled(HK_FLAG_MANAGED_IRQ)) {
257
+		const struct cpumask *hk_mask;
258
+
259
+		hk_mask = housekeeping_cpumask(HK_FLAG_MANAGED_IRQ);
260
+
261
+		cpumask_and(&tmp_mask, mask, hk_mask);
262
+		if (!cpumask_intersects(&tmp_mask, cpu_online_mask))
263
+			prog_mask = mask;
264
+		else
265
+			prog_mask = &tmp_mask;
266
+	} else {
267
+		prog_mask = mask;
268
+	}
269
+
270
+	/*
271
+	 * Make sure we only provide online CPUs to the irqchip,
272
+	 * unless we are being asked to force the affinity (in which
273
+	 * case we do as we are told).
274
+	 */
275
+	cpumask_and(&tmp_mask, prog_mask, cpu_online_mask);
276
+	if (!force && !cpumask_empty(&tmp_mask))
277
+		ret = chip->irq_set_affinity(data, &tmp_mask, force);
278
+	else if (force)
279
+		ret = chip->irq_set_affinity(data, mask, force);
280
+	else
281
+		ret = -EINVAL;
282
+
283
+	raw_spin_unlock(&tmp_mask_lock);
284
+
233285 	switch (ret) {
234286 	case IRQ_SET_MASK_OK:
235287 	case IRQ_SET_MASK_OK_DONE:
236288 		cpumask_copy(desc->irq_common_data.affinity, mask);
289
+		fallthrough;
237290 	case IRQ_SET_MASK_OK_NOCOPY:
238291 		irq_validate_effective_affinity(data);
239292 		irq_set_thread_affinity(desc);
....@@ -242,6 +295,7 @@
242295 
243296 	return ret;
244297 }
298
+EXPORT_SYMBOL_GPL(irq_do_set_affinity);
245299 
246300 #ifdef CONFIG_GENERIC_PENDING_IRQ
247301 static inline int irq_set_affinity_pending(struct irq_data *data,
....@@ -347,7 +401,6 @@
347401 	raw_spin_unlock_irqrestore(&desc->lock, flags);
348402 	return ret;
349403 }
350
-EXPORT_SYMBOL_GPL(__irq_set_affinity);
351404 
352405 int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
353406 {
....@@ -411,7 +464,7 @@
411464 	/* The release function is promised process context */
412465 	might_sleep();
413466 
414
-	if (!desc)
467
+	if (!desc || desc->istate & IRQS_NMI)
415468 		return -EINVAL;
416469 
417470 	/* Complete initialisation of *notify */
....@@ -614,6 +667,21 @@
614667 }
615668 EXPORT_SYMBOL_GPL(disable_hardirq);
616669 
670
+/**
671
+ *	disable_nmi_nosync - disable an nmi without waiting
672
+ *	@irq: Interrupt to disable
673
+ *
674
+ *	Disable the selected interrupt line. Disables and enables are
675
+ *	nested.
676
+ *	The interrupt to disable must have been requested through request_nmi.
677
+ *	Unlike disable_nmi(), this function does not ensure existing
678
+ *	instances of the IRQ handler have completed before returning.
679
+ */
680
+void disable_nmi_nosync(unsigned int irq)
681
+{
682
+	disable_irq_nosync(irq);
683
+}
684
+
617685 void __enable_irq(struct irq_desc *desc)
618686 {
619687 	switch (desc->depth) {
....@@ -670,6 +738,20 @@
670738 }
671739 EXPORT_SYMBOL(enable_irq);
672740 
741
+/**
742
+ *	enable_nmi - enable handling of an nmi
743
+ *	@irq: Interrupt to enable
744
+ *
745
+ *	The interrupt to enable must have been requested through request_nmi.
746
+ *	Undoes the effect of one call to disable_nmi(). If this
747
+ *	matches the last disable, processing of interrupts on this
748
+ *	IRQ line is re-enabled.
749
+ */
750
+void enable_nmi(unsigned int irq)
751
+{
752
+	enable_irq(irq);
753
+}
754
+
673755 static int set_irq_wake_real(unsigned int irq, unsigned int on)
674756 {
675757 	struct irq_desc *desc = irq_to_desc(irq);
....@@ -695,6 +777,13 @@
695777  *
696778  *	Wakeup mode lets this IRQ wake the system from sleep
697779  *	states like "suspend to RAM".
780
+ *
781
+ *	Note: irq enable/disable state is completely orthogonal
782
+ *	to the enable/disable state of irq wake. An irq can be
783
+ *	disabled with disable_irq() and still wake the system as
784
+ *	long as the irq has wake enabled. If this does not hold,
785
+ *	then the underlying irq chip and the related driver need
786
+ *	to be investigated.
698787  */
699788 int irq_set_irq_wake(unsigned int irq, unsigned int on)
700789 {
....@@ -704,6 +793,12 @@
704793 
705794 	if (!desc)
706795 		return -EINVAL;
796
+
797
+	/* Don't use NMIs as wake up interrupts please */
798
+	if (desc->istate & IRQS_NMI) {
799
+		ret = -EINVAL;
800
+		goto out_unlock;
801
+	}
707802 
708803 	/* wakeup-capable irqs can be shared between drivers that
709804 	 * don't need to have the same sleep mode behaviors.
....@@ -727,6 +822,8 @@
727822 				irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
728823 		}
729824 	}
825
+
826
+out_unlock:
730827 	irq_put_desc_busunlock(desc, flags);
731828 	return ret;
732829 }
....@@ -787,6 +884,7 @@
787884 	case IRQ_SET_MASK_OK_DONE:
788885 		irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
789886 		irqd_set(&desc->irq_data, flags);
887
+		fallthrough;
790888 
791889 	case IRQ_SET_MASK_OK_NOCOPY:
792890 		flags = irqd_get_trigger_type(&desc->irq_data);
....@@ -801,7 +899,7 @@
801899 		ret = 0;
802900 		break;
803901 	default:
804
-		pr_err("Setting trigger mode %lu for irq %u failed (%pF)\n",
902
+		pr_err("Setting trigger mode %lu for irq %u failed (%pS)\n",
805903 		       flags, irq_desc_get_irq(desc), chip->irq_set_type);
806904 	}
807905 	if (unmask)
....@@ -979,7 +1077,7 @@
9791077 #endif
9801078 
9811079 /*
982
- * Interrupts which are not explicitely requested as threaded
1080
+ * Interrupts which are not explicitly requested as threaded
9831081  * interrupts rely on the implicit bh/preempt disable of the hard irq
9841082  * context. So we need to disable bh here to avoid deadlocks and other
9851083  * side effects.
....@@ -990,14 +1088,14 @@
9901088 	irqreturn_t ret;
9911089 
9921090 	local_bh_disable();
993
-	if (!IS_ENABLED(CONFIG_PREEMPT_RT_BASE))
1091
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
9941092 		local_irq_disable();
9951093 	ret = action->thread_fn(action->irq, action->dev_id);
9961094 	if (ret == IRQ_HANDLED)
9971095 		atomic_inc(&desc->threads_handled);
9981096 
9991097 	irq_finalize_oneshot(desc, action);
1000
-	if (!IS_ENABLED(CONFIG_PREEMPT_RT_BASE))
1098
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
10011099 		local_irq_enable();
10021100 	local_bh_enable();
10031101 	return ret;
....@@ -1067,6 +1165,31 @@
10671165 }
10681166 
10691167 /*
1168
+ * Internal function to notify that a interrupt thread is ready.
1169
+ */
1170
+static void irq_thread_set_ready(struct irq_desc *desc,
1171
+				 struct irqaction *action)
1172
+{
1173
+	set_bit(IRQTF_READY, &action->thread_flags);
1174
+	wake_up(&desc->wait_for_threads);
1175
+}
1176
+
1177
+/*
1178
+ * Internal function to wake up a interrupt thread and wait until it is
1179
+ * ready.
1180
+ */
1181
+static void wake_up_and_wait_for_irq_thread_ready(struct irq_desc *desc,
1182
+						  struct irqaction *action)
1183
+{
1184
+	if (!action || !action->thread)
1185
+		return;
1186
+
1187
+	wake_up_process(action->thread);
1188
+	wait_event(desc->wait_for_threads,
1189
+		   test_bit(IRQTF_READY, &action->thread_flags));
1190
+}
1191
+
1192
+/*
10701193  * Interrupt handler thread
10711194  */
10721195 static int irq_thread(void *data)
....@@ -1077,6 +1200,10 @@
10771200 	irqreturn_t (*handler_fn)(struct irq_desc *desc,
10781201 			struct irqaction *action);
10791202 
1203
+	irq_thread_set_ready(desc, action);
1204
+
1205
+	sched_set_fifo(current);
1206
+
10801207 	if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
10811208 					&action->thread_flags))
10821209 		handler_fn = irq_forced_thread_fn;
....@@ -1084,7 +1211,7 @@
10841211 		handler_fn = irq_thread_fn;
10851212 
10861213 	init_task_work(&on_exit_work, irq_thread_dtor);
1087
-	task_work_add(current, &on_exit_work, false);
1214
+	task_work_add(current, &on_exit_work, TWA_NONE);
10881215 
10891216 	irq_thread_check_affinity(desc, action);
10901217 
....@@ -1097,12 +1224,6 @@
10971224 		if (action_ret == IRQ_WAKE_THREAD)
10981225 			irq_wake_secondary(desc, action);
10991226 
1100
-#ifdef CONFIG_PREEMPT_RT_FULL
1101
-		migrate_disable();
1102
-		add_interrupt_randomness(action->irq, 0,
1103
-				 desc->random_ip ^ (unsigned long) action);
1104
-		migrate_enable();
1105
-#endif
11061227 		wake_threads_waitq(desc);
11071228 	}
11081229 
....@@ -1199,13 +1320,43 @@
11991320 		c->irq_release_resources(d);
12001321 }
12011322 
1323
+static bool irq_supports_nmi(struct irq_desc *desc)
1324
+{
1325
+	struct irq_data *d = irq_desc_get_irq_data(desc);
1326
+
1327
+#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1328
+	/* Only IRQs directly managed by the root irqchip can be set as NMI */
1329
+	if (d->parent_data)
1330
+		return false;
1331
+#endif
1332
+	/* Don't support NMIs for chips behind a slow bus */
1333
+	if (d->chip->irq_bus_lock || d->chip->irq_bus_sync_unlock)
1334
+		return false;
1335
+
1336
+	return d->chip->flags & IRQCHIP_SUPPORTS_NMI;
1337
+}
1338
+
1339
+static int irq_nmi_setup(struct irq_desc *desc)
1340
+{
1341
+	struct irq_data *d = irq_desc_get_irq_data(desc);
1342
+	struct irq_chip *c = d->chip;
1343
+
1344
+	return c->irq_nmi_setup ? c->irq_nmi_setup(d) : -EINVAL;
1345
+}
1346
+
1347
+static void irq_nmi_teardown(struct irq_desc *desc)
1348
+{
1349
+	struct irq_data *d = irq_desc_get_irq_data(desc);
1350
+	struct irq_chip *c = d->chip;
1351
+
1352
+	if (c->irq_nmi_teardown)
1353
+		c->irq_nmi_teardown(d);
1354
+}
1355
+
12021356 static int
12031357 setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary)
12041358 {
12051359 	struct task_struct *t;
1206
-	struct sched_param param = {
1207
-		.sched_priority = MAX_USER_RT_PRIO/2,
1208
-	};
12091360 
12101361 	if (!secondary) {
12111362 		t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
....@@ -1213,21 +1364,17 @@
12131364 	} else {
12141365 		t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq,
12151366 				   new->name);
1216
-		param.sched_priority -= 1;
12171367 	}
12181368 
12191369 	if (IS_ERR(t))
12201370 		return PTR_ERR(t);
1221
-
1222
-	sched_setscheduler_nocheck(t, SCHED_FIFO, &param);
12231371 
12241372 	/*
12251373 	 * We keep the reference to the task struct even if
12261374 	 * the thread dies to avoid that the interrupt code
12271375 	 * references an already freed task_struct.
12281376 	 */
1229
-	get_task_struct(t);
1230
-	new->thread = t;
1377
+	new->thread = get_task_struct(t);
12311378 	/*
12321379 	 * Tell the thread to set its affinity. This is
12331380 	 * important for shared interrupt handlers as we do
....@@ -1373,8 +1520,16 @@
13731520 		 * fields must have IRQF_SHARED set and the bits which
13741521 		 * set the trigger type must match. Also all must
13751522 		 * agree on ONESHOT.
1523
+		 * Interrupt lines used for NMIs cannot be shared.
13761524 		 */
13771525 		unsigned int oldtype;
1526
+
1527
+		if (desc->istate & IRQS_NMI) {
1528
+			pr_err("Invalid attempt to share NMI for %s (irq %d) on irqchip %s.\n",
1529
+				new->name, irq, desc->irq_data.chip->name);
1530
+			ret = -EINVAL;
1531
+			goto out_unlock;
1532
+		}
13781533 
13791534 		/*
13801535 		 * If nobody did set the configuration before, inherit
....@@ -1464,15 +1619,13 @@
14641619 		 * has. The type flags are unreliable as the
14651620 		 * underlying chip implementation can override them.
14661621 		 */
1467
-		pr_err("Threaded irq requested with handler=NULL and !ONESHOT for irq %d\n",
1468
-		       irq);
1622
+		pr_err("Threaded irq requested with handler=NULL and !ONESHOT for %s (irq %d)\n",
1623
+		       new->name, irq);
14691624 		ret = -EINVAL;
14701625 		goto out_unlock;
14711626 	}
14721627 
14731628 	if (!shared) {
1474
-		init_waitqueue_head(&desc->wait_for_threads);
1475
-
14761629 		/* Setup the type (level, edge polarity) if configured: */
14771630 		if (new->flags & IRQF_TRIGGER_MASK) {
14781631 			ret = __irq_set_trigger(desc,
....@@ -1514,9 +1667,6 @@
15141667 			irq_settings_set_no_balancing(desc);
15151668 			irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
15161669 		}
1517
-
1518
-		if (new->flags & IRQF_NO_SOFTIRQ_CALL)
1519
-			irq_settings_set_no_softirq_call(desc);
15201670 
15211671 		if (irq_settings_can_autoenable(desc)) {
15221672 			irq_startup(desc, IRQ_RESEND, IRQ_START_COND);
....@@ -1565,14 +1715,8 @@
15651715 
15661716 	irq_setup_timings(desc, new);
15671717 
1568
-	/*
1569
-	 * Strictly no need to wake it up, but hung_task complains
1570
-	 * when no hard interrupt wakes the thread up.
1571
-	 */
1572
-	if (new->thread)
1573
-		wake_up_process(new->thread);
1574
-	if (new->secondary)
1575
-		wake_up_process(new->secondary->thread);
1718
+	wake_up_and_wait_for_irq_thread_ready(desc, new);
1719
+	wake_up_and_wait_for_irq_thread_ready(desc, new->secondary);
15761720 
15771721 	register_irq_proc(irq, desc);
15781722 	new->dir = NULL;
....@@ -1617,34 +1761,6 @@
16171761 	module_put(desc->owner);
16181762 	return ret;
16191763 }
1620
-
1621
-/**
1622
- *	setup_irq - setup an interrupt
1623
- *	@irq: Interrupt line to setup
1624
- *	@act: irqaction for the interrupt
1625
- *
1626
- * Used to statically setup interrupts in the early boot process.
1627
- */
1628
-int setup_irq(unsigned int irq, struct irqaction *act)
1629
-{
1630
-	int retval;
1631
-	struct irq_desc *desc = irq_to_desc(irq);
1632
-
1633
-	if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1634
-		return -EINVAL;
1635
-
1636
-	retval = irq_chip_pm_get(&desc->irq_data);
1637
-	if (retval < 0)
1638
-		return retval;
1639
-
1640
-	retval = __setup_irq(irq, desc, act);
1641
-
1642
-	if (retval)
1643
-		irq_chip_pm_put(&desc->irq_data);
1644
-
1645
-	return retval;
1646
-}
1647
-EXPORT_SYMBOL_GPL(setup_irq);
16481764 
16491765 /*
16501766  * Internal function to unregister an irqaction - used to free
....@@ -1787,22 +1903,6 @@
17871903 }
17881904 
17891905 /**
1790
- *	remove_irq - free an interrupt
1791
- *	@irq: Interrupt line to free
1792
- *	@act: irqaction for the interrupt
1793
- *
1794
- * Used to remove interrupts statically setup by the early boot process.
1795
- */
1796
-void remove_irq(unsigned int irq, struct irqaction *act)
1797
-{
1798
-	struct irq_desc *desc = irq_to_desc(irq);
1799
-
1800
-	if (desc && !WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1801
-		__free_irq(desc, act->dev_id);
1802
-}
1803
-EXPORT_SYMBOL_GPL(remove_irq);
1804
-
1805
-/**
18061906  *	free_irq - free an interrupt allocated with request_irq
18071907  *	@irq: Interrupt line to free
18081908  *	@dev_id: Device identity to free
....@@ -1842,6 +1942,59 @@
18421942 	return devname;
18431943 }
18441944 EXPORT_SYMBOL(free_irq);
1945
+
1946
+/* This function must be called with desc->lock held */
1947
+static const void *__cleanup_nmi(unsigned int irq, struct irq_desc *desc)
1948
+{
1949
+	const char *devname = NULL;
1950
+
1951
+	desc->istate &= ~IRQS_NMI;
1952
+
1953
+	if (!WARN_ON(desc->action == NULL)) {
1954
+		irq_pm_remove_action(desc, desc->action);
1955
+		devname = desc->action->name;
1956
+		unregister_handler_proc(irq, desc->action);
1957
+
1958
+		kfree(desc->action);
1959
+		desc->action = NULL;
1960
+	}
1961
+
1962
+	irq_settings_clr_disable_unlazy(desc);
1963
+	irq_shutdown_and_deactivate(desc);
1964
+
1965
+	irq_release_resources(desc);
1966
+
1967
+	irq_chip_pm_put(&desc->irq_data);
1968
+	module_put(desc->owner);
1969
+
1970
+	return devname;
1971
+}
1972
+
1973
+const void *free_nmi(unsigned int irq, void *dev_id)
1974
+{
1975
+	struct irq_desc *desc = irq_to_desc(irq);
1976
+	unsigned long flags;
1977
+	const void *devname;
1978
+
1979
+	if (!desc || WARN_ON(!(desc->istate & IRQS_NMI)))
1980
+		return NULL;
1981
+
1982
+	if (WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1983
+		return NULL;
1984
+
1985
+	/* NMI still enabled */
1986
+	if (WARN_ON(desc->depth == 0))
1987
+		disable_nmi_nosync(irq);
1988
+
1989
+	raw_spin_lock_irqsave(&desc->lock, flags);
1990
+
1991
+	irq_nmi_teardown(desc);
1992
+	devname = __cleanup_nmi(irq, desc);
1993
+
1994
+	raw_spin_unlock_irqrestore(&desc->lock, flags);
1995
+
1996
+	return devname;
1997
+}
18451998 
18461999 /**
18472000  *	request_threaded_irq - allocate an interrupt line
....@@ -2012,6 +2165,101 @@
20122165 }
20132166 EXPORT_SYMBOL_GPL(request_any_context_irq);
20142167 
2168
+/**
2169
+ *	request_nmi - allocate an interrupt line for NMI delivery
2170
+ *	@irq: Interrupt line to allocate
2171
+ *	@handler: Function to be called when the IRQ occurs.
2172
+ *		  Threaded handler for threaded interrupts.
2173
+ *	@irqflags: Interrupt type flags
2174
+ *	@name: An ascii name for the claiming device
2175
+ *	@dev_id: A cookie passed back to the handler function
2176
+ *
2177
+ *	This call allocates interrupt resources and enables the
2178
+ *	interrupt line and IRQ handling. It sets up the IRQ line
2179
+ *	to be handled as an NMI.
2180
+ *
2181
+ *	An interrupt line delivering NMIs cannot be shared and IRQ handling
2182
+ *	cannot be threaded.
2183
+ *
2184
+ *	Interrupt lines requested for NMI delivering must produce per cpu
2185
+ *	interrupts and have auto enabling setting disabled.
2186
+ *
2187
+ *	Dev_id must be globally unique. Normally the address of the
2188
+ *	device data structure is used as the cookie. Since the handler
2189
+ *	receives this value it makes sense to use it.
2190
+ *
2191
+ *	If the interrupt line cannot be used to deliver NMIs, function
2192
+ *	will fail and return a negative value.
2193
+ */
2194
+int request_nmi(unsigned int irq, irq_handler_t handler,
2195
+		unsigned long irqflags, const char *name, void *dev_id)
2196
+{
2197
+	struct irqaction *action;
2198
+	struct irq_desc *desc;
2199
+	unsigned long flags;
2200
+	int retval;
2201
+
2202
+	if (irq == IRQ_NOTCONNECTED)
2203
+		return -ENOTCONN;
2204
+
2205
+	/* NMI cannot be shared, used for Polling */
2206
+	if (irqflags & (IRQF_SHARED | IRQF_COND_SUSPEND | IRQF_IRQPOLL))
2207
+		return -EINVAL;
2208
+
2209
+	if (!(irqflags & IRQF_PERCPU))
2210
+		return -EINVAL;
2211
+
2212
+	if (!handler)
2213
+		return -EINVAL;
2214
+
2215
+	desc = irq_to_desc(irq);
2216
+
2217
+	if (!desc || irq_settings_can_autoenable(desc) ||
2218
+	    !irq_settings_can_request(desc) ||
2219
+	    WARN_ON(irq_settings_is_per_cpu_devid(desc)) ||
2220
+	    !irq_supports_nmi(desc))
2221
+		return -EINVAL;
2222
+
2223
+	action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2224
+	if (!action)
2225
+		return -ENOMEM;
2226
+
2227
+	action->handler = handler;
2228
+	action->flags = irqflags | IRQF_NO_THREAD | IRQF_NOBALANCING;
2229
+	action->name = name;
2230
+	action->dev_id = dev_id;
2231
+
2232
+	retval = irq_chip_pm_get(&desc->irq_data);
2233
+	if (retval < 0)
2234
+		goto err_out;
2235
+
2236
+	retval = __setup_irq(irq, desc, action);
2237
+	if (retval)
2238
+		goto err_irq_setup;
2239
+
2240
+	raw_spin_lock_irqsave(&desc->lock, flags);
2241
+
2242
+	/* Setup NMI state */
2243
+	desc->istate |= IRQS_NMI;
2244
+	retval = irq_nmi_setup(desc);
2245
+	if (retval) {
2246
+		__cleanup_nmi(irq, desc);
2247
+		raw_spin_unlock_irqrestore(&desc->lock, flags);
2248
+		return -EINVAL;
2249
+	}
2250
+
2251
+	raw_spin_unlock_irqrestore(&desc->lock, flags);
2252
+
2253
+	return 0;
2254
+
2255
+err_irq_setup:
2256
+	irq_chip_pm_put(&desc->irq_data);
2257
+err_out:
2258
+	kfree(action);
2259
+
2260
+	return retval;
2261
+}
2262
+
20152263 void enable_percpu_irq(unsigned int irq, unsigned int type)
20162264 {
20172265 	unsigned int cpu = smp_processor_id();
....@@ -2045,6 +2293,11 @@
20452293 	irq_put_desc_unlock(desc, flags);
20462294 }
20472295 EXPORT_SYMBOL_GPL(enable_percpu_irq);
2296
+
2297
+void enable_percpu_nmi(unsigned int irq, unsigned int type)
2298
+{
2299
+	enable_percpu_irq(irq, type);
2300
+}
20482301 
20492302 /**
20502303  * irq_percpu_is_enabled - Check whether the per cpu irq is enabled
....@@ -2085,6 +2338,11 @@
20852338 }
20862339 EXPORT_SYMBOL_GPL(disable_percpu_irq);
20872340 
2341
+void disable_percpu_nmi(unsigned int irq)
2342
+{
2343
+	disable_percpu_irq(irq);
2344
+}
2345
+
20882346 /*
20892347  * Internal function to unregister a percpu irqaction.
20902348  */
....@@ -2115,6 +2373,8 @@
21152373 
21162374 	/* Found it - now remove it from the list of entries: */
21172375 	desc->action = NULL;
2376
+
2377
+	desc->istate &= ~IRQS_NMI;
21182378 
21192379 	raw_spin_unlock_irqrestore(&desc->lock, flags);
21202380 
....@@ -2168,6 +2428,19 @@
21682428 	chip_bus_sync_unlock(desc);
21692429 }
21702430 EXPORT_SYMBOL_GPL(free_percpu_irq);
2431
+
2432
+void free_percpu_nmi(unsigned int irq, void __percpu *dev_id)
2433
+{
2434
+	struct irq_desc *desc = irq_to_desc(irq);
2435
+
2436
+	if (!desc || !irq_settings_is_per_cpu_devid(desc))
2437
+		return;
2438
+
2439
+	if (WARN_ON(!(desc->istate & IRQS_NMI)))
2440
+		return;
2441
+
2442
+	kfree(__free_percpu_irq(irq, dev_id));
2443
+}
21712444 
21722445 /**
21732446  *	setup_percpu_irq - setup a per-cpu interrupt
....@@ -2258,6 +2531,158 @@
22582531 }
22592532 EXPORT_SYMBOL_GPL(__request_percpu_irq);
22602533 
2534
+/**
2535
+ *	request_percpu_nmi - allocate a percpu interrupt line for NMI delivery
2536
+ *	@irq: Interrupt line to allocate
2537
+ *	@handler: Function to be called when the IRQ occurs.
2538
+ *	@name: An ascii name for the claiming device
2539
+ *	@dev_id: A percpu cookie passed back to the handler function
2540
+ *
2541
+ *	This call allocates interrupt resources for a per CPU NMI. Per CPU NMIs
2542
+ *	have to be setup on each CPU by calling prepare_percpu_nmi() before
2543
+ *	being enabled on the same CPU by using enable_percpu_nmi().
2544
+ *
2545
+ *	Dev_id must be globally unique. It is a per-cpu variable, and
2546
+ *	the handler gets called with the interrupted CPU's instance of
2547
+ *	that variable.
2548
+ *
2549
+ *	Interrupt lines requested for NMI delivering should have auto enabling
2550
+ *	setting disabled.
2551
+ *
2552
+ *	If the interrupt line cannot be used to deliver NMIs, function
2553
+ *	will fail returning a negative value.
2554
+ */
2555
+int request_percpu_nmi(unsigned int irq, irq_handler_t handler,
2556
+		       const char *name, void __percpu *dev_id)
2557
+{
2558
+	struct irqaction *action;
2559
+	struct irq_desc *desc;
2560
+	unsigned long flags;
2561
+	int retval;
2562
+
2563
+	if (!handler)
2564
+		return -EINVAL;
2565
+
2566
+	desc = irq_to_desc(irq);
2567
+
2568
+	if (!desc || !irq_settings_can_request(desc) ||
2569
+	    !irq_settings_is_per_cpu_devid(desc) ||
2570
+	    irq_settings_can_autoenable(desc) ||
2571
+	    !irq_supports_nmi(desc))
2572
+		return -EINVAL;
2573
+
2574
+	/* The line cannot already be NMI */
2575
+	if (desc->istate & IRQS_NMI)
2576
+		return -EINVAL;
2577
+
2578
+	action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2579
+	if (!action)
2580
+		return -ENOMEM;
2581
+
2582
+	action->handler = handler;
2583
+	action->flags = IRQF_PERCPU | IRQF_NO_SUSPEND | IRQF_NO_THREAD
2584
+		| IRQF_NOBALANCING;
2585
+	action->name = name;
2586
+	action->percpu_dev_id = dev_id;
2587
+
2588
+	retval = irq_chip_pm_get(&desc->irq_data);
2589
+	if (retval < 0)
2590
+		goto err_out;
2591
+
2592
+	retval = __setup_irq(irq, desc, action);
2593
+	if (retval)
2594
+		goto err_irq_setup;
2595
+
2596
+	raw_spin_lock_irqsave(&desc->lock, flags);
2597
+	desc->istate |= IRQS_NMI;
2598
+	raw_spin_unlock_irqrestore(&desc->lock, flags);
2599
+
2600
+	return 0;
2601
+
2602
+err_irq_setup:
2603
+	irq_chip_pm_put(&desc->irq_data);
2604
+err_out:
2605
+	kfree(action);
2606
+
2607
+	return retval;
2608
+}
2609
+
2610
+/**
2611
+ *	prepare_percpu_nmi - performs CPU local setup for NMI delivery
2612
+ *	@irq: Interrupt line to prepare for NMI delivery
2613
+ *
2614
+ *	This call prepares an interrupt line to deliver NMI on the current CPU,
2615
+ *	before that interrupt line gets enabled with enable_percpu_nmi().
2616
+ *
2617
+ *	As a CPU local operation, this should be called from non-preemptible
2618
+ *	context.
2619
+ *
2620
+ *	If the interrupt line cannot be used to deliver NMIs, function
2621
+ *	will fail returning a negative value.
2622
+ */
2623
+int prepare_percpu_nmi(unsigned int irq)
2624
+{
2625
+	unsigned long flags;
2626
+	struct irq_desc *desc;
2627
+	int ret = 0;
2628
+
2629
+	WARN_ON(preemptible());
2630
+
2631
+	desc = irq_get_desc_lock(irq, &flags,
2632
+				 IRQ_GET_DESC_CHECK_PERCPU);
2633
+	if (!desc)
2634
+		return -EINVAL;
2635
+
2636
+	if (WARN(!(desc->istate & IRQS_NMI),
2637
+		 KERN_ERR "prepare_percpu_nmi called for a non-NMI interrupt: irq %u\n",
2638
+		 irq)) {
2639
+		ret = -EINVAL;
2640
+		goto out;
2641
+	}
2642
+
2643
+	ret = irq_nmi_setup(desc);
2644
+	if (ret) {
2645
+		pr_err("Failed to setup NMI delivery: irq %u\n", irq);
2646
+		goto out;
2647
+	}
2648
+
2649
+out:
2650
+	irq_put_desc_unlock(desc, flags);
2651
+	return ret;
2652
+}
2653
+
2654
+/**
2655
+ *	teardown_percpu_nmi - undoes NMI setup of IRQ line
2656
+ *	@irq: Interrupt line from which CPU local NMI configuration should be
2657
+ *	      removed
2658
+ *
2659
+ *	This call undoes the setup done by prepare_percpu_nmi().
2660
+ *
2661
+ *	IRQ line should not be enabled for the current CPU.
2662
+ *
2663
+ *	As a CPU local operation, this should be called from non-preemptible
2664
+ *	context.
2665
+ */
2666
+void teardown_percpu_nmi(unsigned int irq)
2667
+{
2668
+	unsigned long flags;
2669
+	struct irq_desc *desc;
2670
+
2671
+	WARN_ON(preemptible());
2672
+
2673
+	desc = irq_get_desc_lock(irq, &flags,
2674
+				 IRQ_GET_DESC_CHECK_PERCPU);
2675
+	if (!desc)
2676
+		return;
2677
+
2678
+	if (WARN_ON(!(desc->istate & IRQS_NMI)))
2679
+		goto out;
2680
+
2681
+	irq_nmi_teardown(desc);
2682
+out:
2683
+	irq_put_desc_unlock(desc, flags);
2684
+}
2685
+
22612686 int __irq_get_irqchip_state(struct irq_data *data, enum irqchip_irq_state which,
22622687 			    bool *state)
22632688 {
....@@ -2266,6 +2691,8 @@
22662691 
22672692 	do {
22682693 		chip = irq_data_get_irq_chip(data);
2694
+		if (WARN_ON_ONCE(!chip))
2695
+			return -ENODEV;
22692696 		if (chip->irq_get_irqchip_state)
22702697 			break;
22712698 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
....@@ -2343,6 +2770,10 @@
23432770 
23442771 	do {
23452772 		chip = irq_data_get_irq_chip(data);
2773
+		if (WARN_ON_ONCE(!chip)) {
2774
+			err = -ENODEV;
2775
+			goto out_unlock;
2776
+		}
23462777 		if (chip->irq_set_irqchip_state)
23472778 			break;
23482779 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
....@@ -2355,6 +2786,7 @@
23552786 	if (data)
23562787 		err = chip->irq_set_irqchip_state(data, which, val);
23572788 
2789
+out_unlock:
23582790 	irq_put_desc_busunlock(desc, flags);
23592791 	return err;
23602792 }