forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-11-07 f45e756958099c35d6afb746df1d40a1c6302cfc 
 kernel/kernel/softirq.c
....@@ -21,11 +21,14 @@
2121 #include <linux/freezer.h>
2222 #include <linux/kthread.h>
2323 #include <linux/rcupdate.h>
24
+#include <linux/delay.h>
2425 #include <linux/ftrace.h>
2526 #include <linux/smp.h>
2627 #include <linux/smpboot.h>
2728 #include <linux/tick.h>
29
+#include <linux/locallock.h>
2830 #include <linux/irq.h>
31
+#include <linux/sched/types.h>
2932 
3033 #define CREATE_TRACE_POINTS
3134 #include <trace/events/irq.h>
....@@ -56,11 +59,135 @@
5659 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
5760 
5861 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
62
+#ifdef CONFIG_PREEMPT_RT_FULL
63
+#define TIMER_SOFTIRQS ((1 << TIMER_SOFTIRQ) | (1 << HRTIMER_SOFTIRQ))
64
+DEFINE_PER_CPU(struct task_struct *, ktimer_softirqd);
65
+#endif
5966 
6067 const char * const softirq_to_name[NR_SOFTIRQS] = {
6168 	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
6269 	"TASKLET", "SCHED", "HRTIMER", "RCU"
6370 };
71
+
72
+#ifdef CONFIG_NO_HZ_COMMON
73
+# ifdef CONFIG_PREEMPT_RT_FULL
74
+
75
+struct softirq_runner {
76
+	struct task_struct *runner[NR_SOFTIRQS];
77
+};
78
+
79
+static DEFINE_PER_CPU(struct softirq_runner, softirq_runners);
80
+
81
+static inline void softirq_set_runner(unsigned int sirq)
82
+{
83
+	struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
84
+
85
+	sr->runner[sirq] = current;
86
+}
87
+
88
+static inline void softirq_clr_runner(unsigned int sirq)
89
+{
90
+	struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
91
+
92
+	sr->runner[sirq] = NULL;
93
+}
94
+
95
+static bool softirq_check_runner_tsk(struct task_struct *tsk,
96
+				     unsigned int *pending)
97
+{
98
+	bool ret = false;
99
+
100
+	if (!tsk)
101
+		return ret;
102
+
103
+	/*
104
+	 * The wakeup code in rtmutex.c wakes up the task
105
+	 * _before_ it sets pi_blocked_on to NULL under
106
+	 * tsk->pi_lock. So we need to check for both: state
107
+	 * and pi_blocked_on.
108
+	 * The test against UNINTERRUPTIBLE + ->sleeping_lock is in case the
109
+	 * task does cpu_chill().
110
+	 */
111
+	raw_spin_lock(&tsk->pi_lock);
112
+	if (tsk->pi_blocked_on || tsk->state == TASK_RUNNING ||
113
+	    (tsk->state == TASK_UNINTERRUPTIBLE && tsk->sleeping_lock)) {
114
+		/* Clear all bits pending in that task */
115
+		*pending &= ~(tsk->softirqs_raised);
116
+		ret = true;
117
+	}
118
+	raw_spin_unlock(&tsk->pi_lock);
119
+
120
+	return ret;
121
+}
122
+
123
+/*
124
+ * On preempt-rt a softirq running context might be blocked on a
125
+ * lock. There might be no other runnable task on this CPU because the
126
+ * lock owner runs on some other CPU. So we have to go into idle with
127
+ * the pending bit set. Therefor we need to check this otherwise we
128
+ * warn about false positives which confuses users and defeats the
129
+ * whole purpose of this test.
130
+ *
131
+ * This code is called with interrupts disabled.
132
+ */
133
+void softirq_check_pending_idle(void)
134
+{
135
+	struct task_struct *tsk;
136
+	static int rate_limit;
137
+	struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
138
+	u32 warnpending;
139
+	int i;
140
+
141
+	if (rate_limit >= 10)
142
+		return;
143
+
144
+	warnpending = local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK;
145
+	if (!warnpending)
146
+		return;
147
+	for (i = 0; i < NR_SOFTIRQS; i++) {
148
+		tsk = sr->runner[i];
149
+
150
+		if (softirq_check_runner_tsk(tsk, &warnpending))
151
+			warnpending &= ~(1 << i);
152
+	}
153
+
154
+	if (warnpending) {
155
+		tsk = __this_cpu_read(ksoftirqd);
156
+		softirq_check_runner_tsk(tsk, &warnpending);
157
+	}
158
+
159
+	if (warnpending) {
160
+		tsk = __this_cpu_read(ktimer_softirqd);
161
+		softirq_check_runner_tsk(tsk, &warnpending);
162
+	}
163
+
164
+	if (warnpending) {
165
+		printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
166
+		       warnpending);
167
+		rate_limit++;
168
+	}
169
+}
170
+# else
171
+/*
172
+ * On !PREEMPT_RT we just printk rate limited:
173
+ */
174
+void softirq_check_pending_idle(void)
175
+{
176
+	static int rate_limit;
177
+
178
+	if (rate_limit < 10 &&
179
+			(local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
180
+		printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
181
+		       local_softirq_pending());
182
+		rate_limit++;
183
+	}
184
+}
185
+# endif
186
+
187
+#else /* !CONFIG_NO_HZ_COMMON */
188
+static inline void softirq_set_runner(unsigned int sirq) { }
189
+static inline void softirq_clr_runner(unsigned int sirq) { }
190
+#endif
64191 
65192 /*
66193  * we cannot loop indefinitely here to avoid userspace starvation,
....@@ -77,6 +204,38 @@
77204 		wake_up_process(tsk);
78205 }
79206 
207
+#ifdef CONFIG_PREEMPT_RT_FULL
208
+static void wakeup_timer_softirqd(void)
209
+{
210
+	/* Interrupts are disabled: no need to stop preemption */
211
+	struct task_struct *tsk = __this_cpu_read(ktimer_softirqd);
212
+
213
+	if (tsk && tsk->state != TASK_RUNNING)
214
+		wake_up_process(tsk);
215
+}
216
+#endif
217
+
218
+static void handle_softirq(unsigned int vec_nr)
219
+{
220
+	struct softirq_action *h = softirq_vec + vec_nr;
221
+	int prev_count;
222
+
223
+	prev_count = preempt_count();
224
+
225
+	kstat_incr_softirqs_this_cpu(vec_nr);
226
+
227
+	trace_softirq_entry(vec_nr);
228
+	h->action(h);
229
+	trace_softirq_exit(vec_nr);
230
+	if (unlikely(prev_count != preempt_count())) {
231
+		pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
232
+		       vec_nr, softirq_to_name[vec_nr], h->action,
233
+		       prev_count, preempt_count());
234
+		preempt_count_set(prev_count);
235
+	}
236
+}
237
+
238
+#ifndef CONFIG_PREEMPT_RT_FULL
80239 /*
81240  * If ksoftirqd is scheduled, we do not want to process pending softirqs
82241  * right now. Let ksoftirqd handle this at its own rate, to get fairness,
....@@ -90,6 +249,47 @@
90249 	if (pending & SOFTIRQ_NOW_MASK)
91250 		return false;
92251 	return tsk && (tsk->state == TASK_RUNNING);
252
+}
253
+
254
+static inline int ksoftirqd_softirq_pending(void)
255
+{
256
+	return local_softirq_pending();
257
+}
258
+
259
+static void handle_pending_softirqs(u32 pending)
260
+{
261
+	struct softirq_action *h = softirq_vec;
262
+	int softirq_bit;
263
+
264
+	local_irq_enable();
265
+
266
+	h = softirq_vec;
267
+
268
+	while ((softirq_bit = ffs(pending))) {
269
+		unsigned int vec_nr;
270
+
271
+		h += softirq_bit - 1;
272
+		vec_nr = h - softirq_vec;
273
+		handle_softirq(vec_nr);
274
+
275
+		h++;
276
+		pending >>= softirq_bit;
277
+	}
278
+
279
+	rcu_bh_qs();
280
+	local_irq_disable();
281
+}
282
+
283
+static void run_ksoftirqd(unsigned int cpu)
284
+{
285
+	local_irq_disable();
286
+	if (ksoftirqd_softirq_pending()) {
287
+		__do_softirq();
288
+		local_irq_enable();
289
+		cond_resched();
290
+		return;
291
+	}
292
+	local_irq_enable();
93293 }
94294 
95295 /*
....@@ -251,10 +451,8 @@
251451 	unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
252452 	unsigned long old_flags = current->flags;
253453 	int max_restart = MAX_SOFTIRQ_RESTART;
254
-	struct softirq_action *h;
255454 	bool in_hardirq;
256455 	__u32 pending;
257
-	int softirq_bit;
258456 
259457 	/*
260458 	 * Mask out PF_MEMALLOC s current task context is borrowed for the
....@@ -273,36 +471,7 @@
273471 	/* Reset the pending bitmask before enabling irqs */
274472 	set_softirq_pending(0);
275473 
276
-	local_irq_enable();
277
-
278
-	h = softirq_vec;
279
-
280
-	while ((softirq_bit = ffs(pending))) {
281
-		unsigned int vec_nr;
282
-		int prev_count;
283
-
284
-		h += softirq_bit - 1;
285
-
286
-		vec_nr = h - softirq_vec;
287
-		prev_count = preempt_count();
288
-
289
-		kstat_incr_softirqs_this_cpu(vec_nr);
290
-
291
-		trace_softirq_entry(vec_nr);
292
-		h->action(h);
293
-		trace_softirq_exit(vec_nr);
294
-		if (unlikely(prev_count != preempt_count())) {
295
-			pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
296
-			       vec_nr, softirq_to_name[vec_nr], h->action,
297
-			       prev_count, preempt_count());
298
-			preempt_count_set(prev_count);
299
-		}
300
-		h++;
301
-		pending >>= softirq_bit;
302
-	}
303
-
304
-	rcu_bh_qs();
305
-	local_irq_disable();
474
+	handle_pending_softirqs(pending);
306475 
307476 	pending = local_softirq_pending();
308477 	if (pending) {
....@@ -339,6 +508,309 @@
339508 }
340509 
341510 /*
511
+ * This function must run with irqs disabled!
512
+ */
513
+void raise_softirq_irqoff(unsigned int nr)
514
+{
515
+	__raise_softirq_irqoff(nr);
516
+
517
+	/*
518
+	 * If we're in an interrupt or softirq, we're done
519
+	 * (this also catches softirq-disabled code). We will
520
+	 * actually run the softirq once we return from
521
+	 * the irq or softirq.
522
+	 *
523
+	 * Otherwise we wake up ksoftirqd to make sure we
524
+	 * schedule the softirq soon.
525
+	 */
526
+	if (!in_interrupt())
527
+		wakeup_softirqd();
528
+}
529
+
530
+void __raise_softirq_irqoff(unsigned int nr)
531
+{
532
+	trace_softirq_raise(nr);
533
+	or_softirq_pending(1UL << nr);
534
+}
535
+
536
+static inline void local_bh_disable_nort(void) { local_bh_disable(); }
537
+static inline void _local_bh_enable_nort(void) { _local_bh_enable(); }
538
+static void ksoftirqd_set_sched_params(unsigned int cpu) { }
539
+
540
+#else /* !PREEMPT_RT_FULL */
541
+
542
+/*
543
+ * On RT we serialize softirq execution with a cpu local lock per softirq
544
+ */
545
+static DEFINE_PER_CPU(struct local_irq_lock [NR_SOFTIRQS], local_softirq_locks);
546
+
547
+void __init softirq_early_init(void)
548
+{
549
+	int i;
550
+
551
+	for (i = 0; i < NR_SOFTIRQS; i++)
552
+		local_irq_lock_init(local_softirq_locks[i]);
553
+}
554
+
555
+static void lock_softirq(int which)
556
+{
557
+	local_lock(local_softirq_locks[which]);
558
+}
559
+
560
+static void unlock_softirq(int which)
561
+{
562
+	local_unlock(local_softirq_locks[which]);
563
+}
564
+
565
+static void do_single_softirq(int which)
566
+{
567
+	unsigned long old_flags = current->flags;
568
+
569
+	current->flags &= ~PF_MEMALLOC;
570
+	vtime_account_irq_enter(current);
571
+	current->flags |= PF_IN_SOFTIRQ;
572
+	lockdep_softirq_enter();
573
+	local_irq_enable();
574
+	handle_softirq(which);
575
+	local_irq_disable();
576
+	lockdep_softirq_exit();
577
+	current->flags &= ~PF_IN_SOFTIRQ;
578
+	vtime_account_irq_enter(current);
579
+	current_restore_flags(old_flags, PF_MEMALLOC);
580
+}
581
+
582
+/*
583
+ * Called with interrupts disabled. Process softirqs which were raised
584
+ * in current context (or on behalf of ksoftirqd).
585
+ */
586
+static void do_current_softirqs(void)
587
+{
588
+	while (current->softirqs_raised) {
589
+		int i = __ffs(current->softirqs_raised);
590
+		unsigned int pending, mask = (1U << i);
591
+
592
+		current->softirqs_raised &= ~mask;
593
+		local_irq_enable();
594
+
595
+		/*
596
+		 * If the lock is contended, we boost the owner to
597
+		 * process the softirq or leave the critical section
598
+		 * now.
599
+		 */
600
+		lock_softirq(i);
601
+		local_irq_disable();
602
+		softirq_set_runner(i);
603
+		/*
604
+		 * Check with the local_softirq_pending() bits,
605
+		 * whether we need to process this still or if someone
606
+		 * else took care of it.
607
+		 */
608
+		pending = local_softirq_pending();
609
+		if (pending & mask) {
610
+			set_softirq_pending(pending & ~mask);
611
+			do_single_softirq(i);
612
+		}
613
+		softirq_clr_runner(i);
614
+		WARN_ON(current->softirq_nestcnt != 1);
615
+		local_irq_enable();
616
+		unlock_softirq(i);
617
+		local_irq_disable();
618
+	}
619
+}
620
+
621
+void __local_bh_disable(void)
622
+{
623
+	if (++current->softirq_nestcnt == 1)
624
+		migrate_disable();
625
+}
626
+EXPORT_SYMBOL(__local_bh_disable);
627
+
628
+void __local_bh_enable(void)
629
+{
630
+	if (WARN_ON(current->softirq_nestcnt == 0))
631
+		return;
632
+
633
+	local_irq_disable();
634
+	if (current->softirq_nestcnt == 1 && current->softirqs_raised)
635
+		do_current_softirqs();
636
+	local_irq_enable();
637
+
638
+	if (--current->softirq_nestcnt == 0)
639
+		migrate_enable();
640
+}
641
+EXPORT_SYMBOL(__local_bh_enable);
642
+
643
+void _local_bh_enable(void)
644
+{
645
+	if (WARN_ON(current->softirq_nestcnt == 0))
646
+		return;
647
+	if (--current->softirq_nestcnt == 0)
648
+		migrate_enable();
649
+}
650
+EXPORT_SYMBOL(_local_bh_enable);
651
+
652
+int in_serving_softirq(void)
653
+{
654
+	return current->flags & PF_IN_SOFTIRQ;
655
+}
656
+EXPORT_SYMBOL(in_serving_softirq);
657
+
658
+/* Called with preemption disabled */
659
+static void run_ksoftirqd(unsigned int cpu)
660
+{
661
+	local_irq_disable();
662
+	current->softirq_nestcnt++;
663
+
664
+	do_current_softirqs();
665
+	current->softirq_nestcnt--;
666
+	local_irq_enable();
667
+	cond_resched();
668
+}
669
+
670
+/*
671
+ * Called from netif_rx_ni(). Preemption enabled, but migration
672
+ * disabled. So the cpu can't go away under us.
673
+ */
674
+void thread_do_softirq(void)
675
+{
676
+	if (!in_serving_softirq() && current->softirqs_raised) {
677
+		current->softirq_nestcnt++;
678
+		do_current_softirqs();
679
+		current->softirq_nestcnt--;
680
+	}
681
+}
682
+
683
+static void do_raise_softirq_irqoff(unsigned int nr)
684
+{
685
+	unsigned int mask;
686
+
687
+	mask = 1UL << nr;
688
+
689
+	trace_softirq_raise(nr);
690
+	or_softirq_pending(mask);
691
+
692
+	/*
693
+	 * If we are not in a hard interrupt and inside a bh disabled
694
+	 * region, we simply raise the flag on current. local_bh_enable()
695
+	 * will make sure that the softirq is executed. Otherwise we
696
+	 * delegate it to ksoftirqd.
697
+	 */
698
+	if (!in_irq() && current->softirq_nestcnt)
699
+		current->softirqs_raised |= mask;
700
+	else if (!__this_cpu_read(ksoftirqd) || !__this_cpu_read(ktimer_softirqd))
701
+		return;
702
+
703
+	if (mask & TIMER_SOFTIRQS)
704
+		__this_cpu_read(ktimer_softirqd)->softirqs_raised |= mask;
705
+	else
706
+		__this_cpu_read(ksoftirqd)->softirqs_raised |= mask;
707
+}
708
+
709
+static void wakeup_proper_softirq(unsigned int nr)
710
+{
711
+	if ((1UL << nr) & TIMER_SOFTIRQS)
712
+		wakeup_timer_softirqd();
713
+	else
714
+		wakeup_softirqd();
715
+}
716
+
717
+void __raise_softirq_irqoff(unsigned int nr)
718
+{
719
+	do_raise_softirq_irqoff(nr);
720
+	if (!in_irq() && !current->softirq_nestcnt)
721
+		wakeup_proper_softirq(nr);
722
+}
723
+
724
+/*
725
+ * Same as __raise_softirq_irqoff() but will process them in ksoftirqd
726
+ */
727
+void __raise_softirq_irqoff_ksoft(unsigned int nr)
728
+{
729
+	unsigned int mask;
730
+
731
+	if (WARN_ON_ONCE(!__this_cpu_read(ksoftirqd) ||
732
+			 !__this_cpu_read(ktimer_softirqd)))
733
+		return;
734
+	mask = 1UL << nr;
735
+
736
+	trace_softirq_raise(nr);
737
+	or_softirq_pending(mask);
738
+	if (mask & TIMER_SOFTIRQS)
739
+		__this_cpu_read(ktimer_softirqd)->softirqs_raised |= mask;
740
+	else
741
+		__this_cpu_read(ksoftirqd)->softirqs_raised |= mask;
742
+	wakeup_proper_softirq(nr);
743
+}
744
+
745
+/*
746
+ * This function must run with irqs disabled!
747
+ */
748
+void raise_softirq_irqoff(unsigned int nr)
749
+{
750
+	do_raise_softirq_irqoff(nr);
751
+
752
+	/*
753
+	 * If we're in an hard interrupt we let irq return code deal
754
+	 * with the wakeup of ksoftirqd.
755
+	 */
756
+	if (in_irq())
757
+		return;
758
+	/*
759
+	 * If we are in thread context but outside of a bh disabled
760
+	 * region, we need to wake ksoftirqd as well.
761
+	 *
762
+	 * CHECKME: Some of the places which do that could be wrapped
763
+	 * into local_bh_disable/enable pairs. Though it's unclear
764
+	 * whether this is worth the effort. To find those places just
765
+	 * raise a WARN() if the condition is met.
766
+	 */
767
+	if (!current->softirq_nestcnt)
768
+		wakeup_proper_softirq(nr);
769
+}
770
+
771
+static inline int ksoftirqd_softirq_pending(void)
772
+{
773
+	return current->softirqs_raised;
774
+}
775
+
776
+static inline void local_bh_disable_nort(void) { }
777
+static inline void _local_bh_enable_nort(void) { }
778
+
779
+static inline void ksoftirqd_set_sched_params(unsigned int cpu)
780
+{
781
+	/* Take over all but timer pending softirqs when starting */
782
+	local_irq_disable();
783
+	current->softirqs_raised = local_softirq_pending() & ~TIMER_SOFTIRQS;
784
+	local_irq_enable();
785
+}
786
+
787
+static inline void ktimer_softirqd_set_sched_params(unsigned int cpu)
788
+{
789
+	struct sched_param param = { .sched_priority = 1 };
790
+
791
+	sched_setscheduler(current, SCHED_FIFO, &param);
792
+
793
+	/* Take over timer pending softirqs when starting */
794
+	local_irq_disable();
795
+	current->softirqs_raised = local_softirq_pending() & TIMER_SOFTIRQS;
796
+	local_irq_enable();
797
+}
798
+
799
+static inline void ktimer_softirqd_clr_sched_params(unsigned int cpu,
800
+						    bool online)
801
+{
802
+	struct sched_param param = { .sched_priority = 0 };
803
+
804
+	sched_setscheduler(current, SCHED_NORMAL, &param);
805
+}
806
+
807
+static int ktimer_softirqd_should_run(unsigned int cpu)
808
+{
809
+	return current->softirqs_raised;
810
+}
811
+
812
+#endif /* PREEMPT_RT_FULL */
813
+/*
342814  * Enter an interrupt context.
343815  */
344816 void irq_enter(void)
....@@ -349,9 +821,9 @@
349821 		 * Prevent raise_softirq from needlessly waking up ksoftirqd
350822 		 * here, as softirq will be serviced on return from interrupt.
351823 		 */
352
-		local_bh_disable();
824
+		local_bh_disable_nort();
353825 		tick_irq_enter();
354
-		_local_bh_enable();
826
+		_local_bh_enable_nort();
355827 	}
356828 
357829 	__irq_enter();
....@@ -359,6 +831,7 @@
359831 
360832 static inline void invoke_softirq(void)
361833 {
834
+#ifndef CONFIG_PREEMPT_RT_FULL
362835 	if (ksoftirqd_running(local_softirq_pending()))
363836 		return;
364837 
....@@ -381,6 +854,18 @@
381854 	} else {
382855 		wakeup_softirqd();
383856 	}
857
+#else /* PREEMPT_RT_FULL */
858
+	unsigned long flags;
859
+
860
+	local_irq_save(flags);
861
+	if (__this_cpu_read(ksoftirqd) &&
862
+			__this_cpu_read(ksoftirqd)->softirqs_raised)
863
+		wakeup_softirqd();
864
+	if (__this_cpu_read(ktimer_softirqd) &&
865
+			__this_cpu_read(ktimer_softirqd)->softirqs_raised)
866
+		wakeup_timer_softirqd();
867
+	local_irq_restore(flags);
868
+#endif
384869 }
385870 
386871 static inline void tick_irq_exit(void)
....@@ -416,26 +901,6 @@
416901 	trace_hardirq_exit(); /* must be last! */
417902 }
418903 
419
-/*
420
- * This function must run with irqs disabled!
421
- */
422
-inline void raise_softirq_irqoff(unsigned int nr)
423
-{
424
-	__raise_softirq_irqoff(nr);
425
-
426
-	/*
427
-	 * If we're in an interrupt or softirq, we're done
428
-	 * (this also catches softirq-disabled code). We will
429
-	 * actually run the softirq once we return from
430
-	 * the irq or softirq.
431
-	 *
432
-	 * Otherwise we wake up ksoftirqd to make sure we
433
-	 * schedule the softirq soon.
434
-	 */
435
-	if (!in_interrupt())
436
-		wakeup_softirqd();
437
-}
438
-
439904 void raise_softirq(unsigned int nr)
440905 {
441906 	unsigned long flags;
....@@ -443,12 +908,6 @@
443908 	local_irq_save(flags);
444909 	raise_softirq_irqoff(nr);
445910 	local_irq_restore(flags);
446
-}
447
-
448
-void __raise_softirq_irqoff(unsigned int nr)
449
-{
450
-	trace_softirq_raise(nr);
451
-	or_softirq_pending(1UL << nr);
452911 }
453912 
454913 void open_softirq(int nr, void (*action)(struct softirq_action *))
....@@ -475,11 +934,44 @@
475934 	unsigned long flags;
476935 
477936 	local_irq_save(flags);
937
+	if (!tasklet_trylock(t)) {
938
+		local_irq_restore(flags);
939
+		return;
940
+	}
941
+
478942 	head = this_cpu_ptr(headp);
479
-	t->next = NULL;
480
-	*head->tail = t;
481
-	head->tail = &(t->next);
482
-	raise_softirq_irqoff(softirq_nr);
943
+again:
944
+	/* We may have been preempted before tasklet_trylock
945
+	 * and __tasklet_action may have already run.
946
+	 * So double check the sched bit while the takslet
947
+	 * is locked before adding it to the list.
948
+	 */
949
+	if (test_bit(TASKLET_STATE_SCHED, &t->state)) {
950
+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
951
+		if (test_and_set_bit(TASKLET_STATE_CHAINED, &t->state)) {
952
+			tasklet_unlock(t);
953
+			return;
954
+		}
955
+#endif
956
+		t->next = NULL;
957
+		*head->tail = t;
958
+		head->tail = &(t->next);
959
+		raise_softirq_irqoff(softirq_nr);
960
+		tasklet_unlock(t);
961
+	} else {
962
+		/* This is subtle. If we hit the corner case above
963
+		 * It is possible that we get preempted right here,
964
+		 * and another task has successfully called
965
+		 * tasklet_schedule(), then this function, and
966
+		 * failed on the trylock. Thus we must be sure
967
+		 * before releasing the tasklet lock, that the
968
+		 * SCHED_BIT is clear. Otherwise the tasklet
969
+		 * may get its SCHED_BIT set, but not added to the
970
+		 * list
971
+		 */
972
+		if (!tasklet_tryunlock(t))
973
+			goto again;
974
+	}
483975 	local_irq_restore(flags);
484976 }
485977 
....@@ -497,11 +989,21 @@
497989 }
498990 EXPORT_SYMBOL(__tasklet_hi_schedule);
499991 
992
+void tasklet_enable(struct tasklet_struct *t)
993
+{
994
+	if (!atomic_dec_and_test(&t->count))
995
+		return;
996
+	if (test_and_clear_bit(TASKLET_STATE_PENDING, &t->state))
997
+		tasklet_schedule(t);
998
+}
999
+EXPORT_SYMBOL(tasklet_enable);
1000
+
5001001 static void tasklet_action_common(struct softirq_action *a,
5011002 				  struct tasklet_head *tl_head,
5021003 				  unsigned int softirq_nr)
5031004 {
5041005 	struct tasklet_struct *list;
1006
+	int loops = 1000000;
5051007 
5061008 	local_irq_disable();
5071009 	list = tl_head->head;
....@@ -513,25 +1015,60 @@
5131015 		struct tasklet_struct *t = list;
5141016 
5151017 		list = list->next;
516
-
517
-		if (tasklet_trylock(t)) {
518
-			if (!atomic_read(&t->count)) {
519
-				if (!test_and_clear_bit(TASKLET_STATE_SCHED,
520
-							&t->state))
521
-					BUG();
522
-				t->func(t->data);
523
-				tasklet_unlock(t);
524
-				continue;
525
-			}
526
-			tasklet_unlock(t);
1018
+		/*
1019
+		 * Should always succeed - after a tasklist got on the
1020
+		 * list (after getting the SCHED bit set from 0 to 1),
1021
+		 * nothing but the tasklet softirq it got queued to can
1022
+		 * lock it:
1023
+		 */
1024
+		if (!tasklet_trylock(t)) {
1025
+			WARN_ON(1);
1026
+			continue;
5271027 		}
5281028 
529
-		local_irq_disable();
5301029 		t->next = NULL;
531
-		*tl_head->tail = t;
532
-		tl_head->tail = &t->next;
533
-		__raise_softirq_irqoff(softirq_nr);
534
-		local_irq_enable();
1030
+
1031
+		if (unlikely(atomic_read(&t->count))) {
1032
+out_disabled:
1033
+			/* implicit unlock: */
1034
+			wmb();
1035
+			t->state = TASKLET_STATEF_PENDING;
1036
+			continue;
1037
+		}
1038
+		/*
1039
+		 * After this point on the tasklet might be rescheduled
1040
+		 * on another CPU, but it can only be added to another
1041
+		 * CPU's tasklet list if we unlock the tasklet (which we
1042
+		 * dont do yet).
1043
+		 */
1044
+		if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
1045
+			WARN_ON(1);
1046
+again:
1047
+		t->func(t->data);
1048
+
1049
+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
1050
+		while (cmpxchg(&t->state, TASKLET_STATEF_RC, 0) != TASKLET_STATEF_RC) {
1051
+#else
1052
+		while (!tasklet_tryunlock(t)) {
1053
+#endif
1054
+			/*
1055
+			 * If it got disabled meanwhile, bail out:
1056
+			 */
1057
+			if (atomic_read(&t->count))
1058
+				goto out_disabled;
1059
+			/*
1060
+			 * If it got scheduled meanwhile, re-execute
1061
+			 * the tasklet function:
1062
+			 */
1063
+			if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
1064
+				goto again;
1065
+			if (!--loops) {
1066
+				printk("hm, tasklet state: %08lx\n", t->state);
1067
+				WARN_ON(1);
1068
+				tasklet_unlock(t);
1069
+				break;
1070
+			}
1071
+		}
5351072 	}
5361073 }
5371074 
....@@ -563,7 +1100,7 @@
5631100 
5641101 	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
5651102 		do {
566
-			yield();
1103
+			msleep(1);
5671104 		} while (test_bit(TASKLET_STATE_SCHED, &t->state));
5681105 	}
5691106 	tasklet_unlock_wait(t);
....@@ -637,25 +1174,26 @@
6371174 	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
6381175 }
6391176 
1177
+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
1178
+void tasklet_unlock_wait(struct tasklet_struct *t)
1179
+{
1180
+	while (test_bit(TASKLET_STATE_RUN, &(t)->state)) {
1181
+		/*
1182
+		 * Hack for now to avoid this busy-loop:
1183
+		 */
1184
+#ifdef CONFIG_PREEMPT_RT_FULL
1185
+		msleep(1);
1186
+#else
1187
+		barrier();
1188
+#endif
1189
+	}
1190
+}
1191
+EXPORT_SYMBOL(tasklet_unlock_wait);
1192
+#endif
1193
+
6401194 static int ksoftirqd_should_run(unsigned int cpu)
6411195 {
642
-	return local_softirq_pending();
643
-}
644
-
645
-static void run_ksoftirqd(unsigned int cpu)
646
-{
647
-	local_irq_disable();
648
-	if (local_softirq_pending()) {
649
-		/*
650
-		 * We can safely run softirq on inline stack, as we are not deep
651
-		 * in the task stack here.
652
-		 */
653
-		__do_softirq();
654
-		local_irq_enable();
655
-		cond_resched();
656
-		return;
657
-	}
658
-	local_irq_enable();
1196
+	return ksoftirqd_softirq_pending();
6591197 }
6601198 
6611199 #ifdef CONFIG_HOTPLUG_CPU
....@@ -722,17 +1260,31 @@
7221260 
7231261 static struct smp_hotplug_thread softirq_threads = {
7241262 	.store			= &ksoftirqd,
1263
+	.setup			= ksoftirqd_set_sched_params,
7251264 	.thread_should_run	= ksoftirqd_should_run,
7261265 	.thread_fn		= run_ksoftirqd,
7271266 	.thread_comm		= "ksoftirqd/%u",
7281267 };
1268
+
1269
+#ifdef CONFIG_PREEMPT_RT_FULL
1270
+static struct smp_hotplug_thread softirq_timer_threads = {
1271
+	.store			= &ktimer_softirqd,
1272
+	.setup			= ktimer_softirqd_set_sched_params,
1273
+	.cleanup		= ktimer_softirqd_clr_sched_params,
1274
+	.thread_should_run	= ktimer_softirqd_should_run,
1275
+	.thread_fn		= run_ksoftirqd,
1276
+	.thread_comm		= "ktimersoftd/%u",
1277
+};
1278
+#endif
7291279 
7301280 static __init int spawn_ksoftirqd(void)
7311281 {
7321282 	cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
7331283 				  takeover_tasklets);
7341284 	BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
735
-
1285
+#ifdef CONFIG_PREEMPT_RT_FULL
1286
+	BUG_ON(smpboot_register_percpu_thread(&softirq_timer_threads));
1287
+#endif
7361288 	return 0;
7371289 }
7381290 early_initcall(spawn_ksoftirqd);