forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/kernel/softirq.c
....@@ -1,9 +1,8 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  *	linux/kernel/softirq.c
34  *
45  *	Copyright (C) 1992 Linus Torvalds
5
- *
6
- *	Distribute under GPLv2.
76  *
87  *	Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
98  */
....@@ -21,17 +20,17 @@
2120 #include <linux/freezer.h>
2221 #include <linux/kthread.h>
2322 #include <linux/rcupdate.h>
24
-#include <linux/delay.h>
2523 #include <linux/ftrace.h>
2624 #include <linux/smp.h>
2725 #include <linux/smpboot.h>
2826 #include <linux/tick.h>
29
-#include <linux/locallock.h>
3027 #include <linux/irq.h>
31
-#include <linux/sched/types.h>
3228 
3329 #define CREATE_TRACE_POINTS
3430 #include <trace/events/irq.h>
31
+
32
+EXPORT_TRACEPOINT_SYMBOL_GPL(irq_handler_entry);
33
+EXPORT_TRACEPOINT_SYMBOL_GPL(irq_handler_exit);
3534 
3635 /*
3736    - No shared variables, all the data are CPU local.
....@@ -59,135 +58,19 @@
5958 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
6059 
6160 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
62
-#ifdef CONFIG_PREEMPT_RT_FULL
63
-#define TIMER_SOFTIRQS ((1 << TIMER_SOFTIRQ) | (1 << HRTIMER_SOFTIRQ))
64
-DEFINE_PER_CPU(struct task_struct *, ktimer_softirqd);
65
-#endif
61
+EXPORT_PER_CPU_SYMBOL_GPL(ksoftirqd);
62
+
63
+/*
64
+ * active_softirqs -- per cpu, a mask of softirqs that are being handled,
65
+ * with the expectation that approximate answers are acceptable and therefore
66
+ * no synchronization.
67
+ */
68
+DEFINE_PER_CPU(__u32, active_softirqs);
6669 
6770 const char * const softirq_to_name[NR_SOFTIRQS] = {
6871 	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
6972 	"TASKLET", "SCHED", "HRTIMER", "RCU"
7073 };
71
-
72
-#ifdef CONFIG_NO_HZ_COMMON
73
-# ifdef CONFIG_PREEMPT_RT_FULL
74
-
75
-struct softirq_runner {
76
-	struct task_struct *runner[NR_SOFTIRQS];
77
-};
78
-
79
-static DEFINE_PER_CPU(struct softirq_runner, softirq_runners);
80
-
81
-static inline void softirq_set_runner(unsigned int sirq)
82
-{
83
-	struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
84
-
85
-	sr->runner[sirq] = current;
86
-}
87
-
88
-static inline void softirq_clr_runner(unsigned int sirq)
89
-{
90
-	struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
91
-
92
-	sr->runner[sirq] = NULL;
93
-}
94
-
95
-static bool softirq_check_runner_tsk(struct task_struct *tsk,
96
-				     unsigned int *pending)
97
-{
98
-	bool ret = false;
99
-
100
-	if (!tsk)
101
-		return ret;
102
-
103
-	/*
104
-	 * The wakeup code in rtmutex.c wakes up the task
105
-	 * _before_ it sets pi_blocked_on to NULL under
106
-	 * tsk->pi_lock. So we need to check for both: state
107
-	 * and pi_blocked_on.
108
-	 * The test against UNINTERRUPTIBLE + ->sleeping_lock is in case the
109
-	 * task does cpu_chill().
110
-	 */
111
-	raw_spin_lock(&tsk->pi_lock);
112
-	if (tsk->pi_blocked_on || tsk->state == TASK_RUNNING ||
113
-	    (tsk->state == TASK_UNINTERRUPTIBLE && tsk->sleeping_lock)) {
114
-		/* Clear all bits pending in that task */
115
-		*pending &= ~(tsk->softirqs_raised);
116
-		ret = true;
117
-	}
118
-	raw_spin_unlock(&tsk->pi_lock);
119
-
120
-	return ret;
121
-}
122
-
123
-/*
124
- * On preempt-rt a softirq running context might be blocked on a
125
- * lock. There might be no other runnable task on this CPU because the
126
- * lock owner runs on some other CPU. So we have to go into idle with
127
- * the pending bit set. Therefor we need to check this otherwise we
128
- * warn about false positives which confuses users and defeats the
129
- * whole purpose of this test.
130
- *
131
- * This code is called with interrupts disabled.
132
- */
133
-void softirq_check_pending_idle(void)
134
-{
135
-	struct task_struct *tsk;
136
-	static int rate_limit;
137
-	struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
138
-	u32 warnpending;
139
-	int i;
140
-
141
-	if (rate_limit >= 10)
142
-		return;
143
-
144
-	warnpending = local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK;
145
-	if (!warnpending)
146
-		return;
147
-	for (i = 0; i < NR_SOFTIRQS; i++) {
148
-		tsk = sr->runner[i];
149
-
150
-		if (softirq_check_runner_tsk(tsk, &warnpending))
151
-			warnpending &= ~(1 << i);
152
-	}
153
-
154
-	if (warnpending) {
155
-		tsk = __this_cpu_read(ksoftirqd);
156
-		softirq_check_runner_tsk(tsk, &warnpending);
157
-	}
158
-
159
-	if (warnpending) {
160
-		tsk = __this_cpu_read(ktimer_softirqd);
161
-		softirq_check_runner_tsk(tsk, &warnpending);
162
-	}
163
-
164
-	if (warnpending) {
165
-		printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
166
-		       warnpending);
167
-		rate_limit++;
168
-	}
169
-}
170
-# else
171
-/*
172
- * On !PREEMPT_RT we just printk rate limited:
173
- */
174
-void softirq_check_pending_idle(void)
175
-{
176
-	static int rate_limit;
177
-
178
-	if (rate_limit < 10 &&
179
-			(local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
180
-		printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
181
-		       local_softirq_pending());
182
-		rate_limit++;
183
-	}
184
-}
185
-# endif
186
-
187
-#else /* !CONFIG_NO_HZ_COMMON */
188
-static inline void softirq_set_runner(unsigned int sirq) { }
189
-static inline void softirq_clr_runner(unsigned int sirq) { }
190
-#endif
19174 
19275 /*
19376  * we cannot loop indefinitely here to avoid userspace starvation,
....@@ -202,94 +85,6 @@
20285 
20386 	if (tsk && tsk->state != TASK_RUNNING)
20487 		wake_up_process(tsk);
205
-}
206
-
207
-#ifdef CONFIG_PREEMPT_RT_FULL
208
-static void wakeup_timer_softirqd(void)
209
-{
210
-	/* Interrupts are disabled: no need to stop preemption */
211
-	struct task_struct *tsk = __this_cpu_read(ktimer_softirqd);
212
-
213
-	if (tsk && tsk->state != TASK_RUNNING)
214
-		wake_up_process(tsk);
215
-}
216
-#endif
217
-
218
-static void handle_softirq(unsigned int vec_nr)
219
-{
220
-	struct softirq_action *h = softirq_vec + vec_nr;
221
-	int prev_count;
222
-
223
-	prev_count = preempt_count();
224
-
225
-	kstat_incr_softirqs_this_cpu(vec_nr);
226
-
227
-	trace_softirq_entry(vec_nr);
228
-	h->action(h);
229
-	trace_softirq_exit(vec_nr);
230
-	if (unlikely(prev_count != preempt_count())) {
231
-		pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
232
-		       vec_nr, softirq_to_name[vec_nr], h->action,
233
-		       prev_count, preempt_count());
234
-		preempt_count_set(prev_count);
235
-	}
236
-}
237
-
238
-#ifndef CONFIG_PREEMPT_RT_FULL
239
-/*
240
- * If ksoftirqd is scheduled, we do not want to process pending softirqs
241
- * right now. Let ksoftirqd handle this at its own rate, to get fairness,
242
- * unless we're doing some of the synchronous softirqs.
243
- */
244
-#define SOFTIRQ_NOW_MASK ((1 << HI_SOFTIRQ) | (1 << TASKLET_SOFTIRQ))
245
-static bool ksoftirqd_running(unsigned long pending)
246
-{
247
-	struct task_struct *tsk = __this_cpu_read(ksoftirqd);
248
-
249
-	if (pending & SOFTIRQ_NOW_MASK)
250
-		return false;
251
-	return tsk && (tsk->state == TASK_RUNNING);
252
-}
253
-
254
-static inline int ksoftirqd_softirq_pending(void)
255
-{
256
-	return local_softirq_pending();
257
-}
258
-
259
-static void handle_pending_softirqs(u32 pending)
260
-{
261
-	struct softirq_action *h = softirq_vec;
262
-	int softirq_bit;
263
-
264
-	local_irq_enable();
265
-
266
-	h = softirq_vec;
267
-
268
-	while ((softirq_bit = ffs(pending))) {
269
-		unsigned int vec_nr;
270
-
271
-		h += softirq_bit - 1;
272
-		vec_nr = h - softirq_vec;
273
-		handle_softirq(vec_nr);
274
-
275
-		h++;
276
-		pending >>= softirq_bit;
277
-	}
278
-
279
-	rcu_bh_qs();
280
-	local_irq_disable();
281
-}
282
-
283
-static void run_ksoftirqd(unsigned int cpu)
284
-{
285
-	local_irq_disable();
286
-	if (ksoftirqd_softirq_pending()) {
287
-		__do_softirq();
288
-		local_irq_enable();
289
-		cond_resched();
290
-		return;
291
-	}
292
-	local_irq_enable();
29388 }
29489 
29590 /*
....@@ -307,6 +102,12 @@
307102  * where hardirqs are disabled legitimately:
308103  */
309104 #ifdef CONFIG_TRACE_IRQFLAGS
105
+
106
+DEFINE_PER_CPU(int, hardirqs_enabled);
107
+DEFINE_PER_CPU(int, hardirq_context);
108
+EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
109
+EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
110
+
310111 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
311112 {
312113 	unsigned long flags;
....@@ -326,7 +127,7 @@
326127 	 * Were softirqs turned off above:
327128 	 */
328129 	if (softirq_count() == (cnt & SOFTIRQ_MASK))
329
-		trace_softirqs_off(ip);
130
+		lockdep_softirqs_off(ip);
330131 	raw_local_irq_restore(flags);
331132 
332133 	if (preempt_count() == cnt) {
....@@ -347,7 +148,7 @@
347148 		trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
348149 
349150 	if (softirq_count() == (cnt & SOFTIRQ_MASK))
350
-		trace_softirqs_on(_RET_IP_);
151
+		lockdep_softirqs_on(_RET_IP_);
351152 
352153 	__preempt_count_sub(cnt);
353154 }
....@@ -374,7 +175,7 @@
374175 	 * Are softirqs going to be turned on now:
375176 	 */
376177 	if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
377
-		trace_softirqs_on(ip);
178
+		lockdep_softirqs_on(ip);
378179 	/*
379180 	 * Keep preemption disabled until we are done with
380181 	 * softirq processing:
....@@ -424,9 +225,9 @@
424225 {
425226 	bool in_hardirq = false;
426227 
427
-	if (trace_hardirq_context(current)) {
228
+	if (lockdep_hardirq_context()) {
428229 		in_hardirq = true;
429
-		trace_hardirq_exit();
230
+		lockdep_hardirq_exit();
430231 	}
431232 
432233 	lockdep_softirq_enter();
....@@ -439,49 +240,102 @@
439240 	lockdep_softirq_exit();
440241 
441242 	if (in_hardirq)
442
-		trace_hardirq_enter();
243
+		lockdep_hardirq_enter();
443244 }
444245 #else
445246 static inline bool lockdep_softirq_start(void) { return false; }
446247 static inline void lockdep_softirq_end(bool in_hardirq) { }
447248 #endif
448249 
250
+#define softirq_deferred_for_rt(pending)		\
251
+({							\
252
+	__u32 deferred = 0;				\
253
+	if (cpupri_check_rt()) {			\
254
+		deferred = pending & LONG_SOFTIRQ_MASK; \
255
+		pending &= ~LONG_SOFTIRQ_MASK;		\
256
+	}						\
257
+	deferred;					\
258
+})
259
+
449260 asmlinkage __visible void __softirq_entry __do_softirq(void)
450261 {
451262 	unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
452263 	unsigned long old_flags = current->flags;
453264 	int max_restart = MAX_SOFTIRQ_RESTART;
265
+	struct softirq_action *h;
454266 	bool in_hardirq;
267
+	__u32 deferred;
455268 	__u32 pending;
269
+	int softirq_bit;
456270 
457271 	/*
458
-	 * Mask out PF_MEMALLOC s current task context is borrowed for the
459
-	 * softirq. A softirq handled such as network RX might set PF_MEMALLOC
460
-	 * again if the socket is related to swap
272
+	 * Mask out PF_MEMALLOC as the current task context is borrowed for the
273
+	 * softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
274
+	 * again if the socket is related to swapping.
461275 	 */
462276 	current->flags &= ~PF_MEMALLOC;
463277 
464278 	pending = local_softirq_pending();
279
+	deferred = softirq_deferred_for_rt(pending);
465280 	account_irq_enter_time(current);
466
-
467281 	__local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
468282 	in_hardirq = lockdep_softirq_start();
469283 
470284 restart:
471285 	/* Reset the pending bitmask before enabling irqs */
472
-	set_softirq_pending(0);
286
+	set_softirq_pending(deferred);
287
+	__this_cpu_write(active_softirqs, pending);
473288 
474
-	handle_pending_softirqs(pending);
289
+	local_irq_enable();
290
+
291
+	h = softirq_vec;
292
+
293
+	while ((softirq_bit = ffs(pending))) {
294
+		unsigned int vec_nr;
295
+		int prev_count;
296
+
297
+		h += softirq_bit - 1;
298
+
299
+		vec_nr = h - softirq_vec;
300
+		prev_count = preempt_count();
301
+
302
+		kstat_incr_softirqs_this_cpu(vec_nr);
303
+
304
+		trace_softirq_entry(vec_nr);
305
+		h->action(h);
306
+		trace_softirq_exit(vec_nr);
307
+		if (unlikely(prev_count != preempt_count())) {
308
+			pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
309
+			       vec_nr, softirq_to_name[vec_nr], h->action,
310
+			       prev_count, preempt_count());
311
+			preempt_count_set(prev_count);
312
+		}
313
+		h++;
314
+		pending >>= softirq_bit;
315
+	}
316
+
317
+	__this_cpu_write(active_softirqs, 0);
318
+	if (__this_cpu_read(ksoftirqd) == current)
319
+		rcu_softirq_qs();
320
+	local_irq_disable();
475321 
476322 	pending = local_softirq_pending();
323
+	deferred = softirq_deferred_for_rt(pending);
324
+
477325 	if (pending) {
478326 		if (time_before(jiffies, end) && !need_resched() &&
479327 		    --max_restart)
480328 			goto restart;
481329 
330
+#ifndef CONFIG_RT_SOFTINT_OPTIMIZATION
482331 		wakeup_softirqd();
332
+#endif
483333 	}
484334 
335
+#ifdef CONFIG_RT_SOFTINT_OPTIMIZATION
336
+	if (pending | deferred)
337
+		wakeup_softirqd();
338
+#endif
485339 	lockdep_softirq_end(in_hardirq);
486340 	account_irq_exit_time(current);
487341 	__local_bh_enable(SOFTIRQ_OFFSET);
....@@ -501,340 +355,40 @@
501355 
502356 	pending = local_softirq_pending();
503357 
504
-	if (pending && !ksoftirqd_running(pending))
358
+	if (pending)
505359 		do_softirq_own_stack();
506360 
507361 	local_irq_restore(flags);
508362 }
509363 
510
-/*
511
- * This function must run with irqs disabled!
364
+/**
365
+ * irq_enter_rcu - Enter an interrupt context with RCU watching
512366  */
513
-void raise_softirq_irqoff(unsigned int nr)
367
+void irq_enter_rcu(void)
514368 {
515
-	__raise_softirq_irqoff(nr);
516
-
517
-	/*
518
-	 * If we're in an interrupt or softirq, we're done
519
-	 * (this also catches softirq-disabled code). We will
520
-	 * actually run the softirq once we return from
521
-	 * the irq or softirq.
522
-	 *
523
-	 * Otherwise we wake up ksoftirqd to make sure we
524
-	 * schedule the softirq soon.
525
-	 */
526
-	if (!in_interrupt())
527
-		wakeup_softirqd();
528
-}
529
-
530
-void __raise_softirq_irqoff(unsigned int nr)
531
-{
532
-	trace_softirq_raise(nr);
533
-	or_softirq_pending(1UL << nr);
534
-}
535
-
536
-static inline void local_bh_disable_nort(void) { local_bh_disable(); }
537
-static inline void _local_bh_enable_nort(void) { _local_bh_enable(); }
538
-static void ksoftirqd_set_sched_params(unsigned int cpu) { }
539
-
540
-#else /* !PREEMPT_RT_FULL */
541
-
542
-/*
543
- * On RT we serialize softirq execution with a cpu local lock per softirq
544
- */
545
-static DEFINE_PER_CPU(struct local_irq_lock [NR_SOFTIRQS], local_softirq_locks);
546
-
547
-void __init softirq_early_init(void)
548
-{
549
-	int i;
550
-
551
-	for (i = 0; i < NR_SOFTIRQS; i++)
552
-		local_irq_lock_init(local_softirq_locks[i]);
553
-}
554
-
555
-static void lock_softirq(int which)
556
-{
557
-	local_lock(local_softirq_locks[which]);
558
-}
559
-
560
-static void unlock_softirq(int which)
561
-{
562
-	local_unlock(local_softirq_locks[which]);
563
-}
564
-
565
-static void do_single_softirq(int which)
566
-{
567
-	unsigned long old_flags = current->flags;
568
-
569
-	current->flags &= ~PF_MEMALLOC;
570
-	vtime_account_irq_enter(current);
571
-	current->flags |= PF_IN_SOFTIRQ;
572
-	lockdep_softirq_enter();
573
-	local_irq_enable();
574
-	handle_softirq(which);
575
-	local_irq_disable();
576
-	lockdep_softirq_exit();
577
-	current->flags &= ~PF_IN_SOFTIRQ;
578
-	vtime_account_irq_enter(current);
579
-	current_restore_flags(old_flags, PF_MEMALLOC);
580
-}
581
-
582
-/*
583
- * Called with interrupts disabled. Process softirqs which were raised
584
- * in current context (or on behalf of ksoftirqd).
585
- */
586
-static void do_current_softirqs(void)
587
-{
588
-	while (current->softirqs_raised) {
589
-		int i = __ffs(current->softirqs_raised);
590
-		unsigned int pending, mask = (1U << i);
591
-
592
-		current->softirqs_raised &= ~mask;
593
-		local_irq_enable();
594
-
595
-		/*
596
-		 * If the lock is contended, we boost the owner to
597
-		 * process the softirq or leave the critical section
598
-		 * now.
599
-		 */
600
-		lock_softirq(i);
601
-		local_irq_disable();
602
-		softirq_set_runner(i);
603
-		/*
604
-		 * Check with the local_softirq_pending() bits,
605
-		 * whether we need to process this still or if someone
606
-		 * else took care of it.
607
-		 */
608
-		pending = local_softirq_pending();
609
-		if (pending & mask) {
610
-			set_softirq_pending(pending & ~mask);
611
-			do_single_softirq(i);
612
-		}
613
-		softirq_clr_runner(i);
614
-		WARN_ON(current->softirq_nestcnt != 1);
615
-		local_irq_enable();
616
-		unlock_softirq(i);
617
-		local_irq_disable();
618
-	}
619
-}
620
-
621
-void __local_bh_disable(void)
622
-{
623
-	if (++current->softirq_nestcnt == 1)
624
-		migrate_disable();
625
-}
626
-EXPORT_SYMBOL(__local_bh_disable);
627
-
628
-void __local_bh_enable(void)
629
-{
630
-	if (WARN_ON(current->softirq_nestcnt == 0))
631
-		return;
632
-
633
-	local_irq_disable();
634
-	if (current->softirq_nestcnt == 1 && current->softirqs_raised)
635
-		do_current_softirqs();
636
-	local_irq_enable();
637
-
638
-	if (--current->softirq_nestcnt == 0)
639
-		migrate_enable();
640
-}
641
-EXPORT_SYMBOL(__local_bh_enable);
642
-
643
-void _local_bh_enable(void)
644
-{
645
-	if (WARN_ON(current->softirq_nestcnt == 0))
646
-		return;
647
-	if (--current->softirq_nestcnt == 0)
648
-		migrate_enable();
649
-}
650
-EXPORT_SYMBOL(_local_bh_enable);
651
-
652
-int in_serving_softirq(void)
653
-{
654
-	return current->flags & PF_IN_SOFTIRQ;
655
-}
656
-EXPORT_SYMBOL(in_serving_softirq);
657
-
658
-/* Called with preemption disabled */
659
-static void run_ksoftirqd(unsigned int cpu)
660
-{
661
-	local_irq_disable();
662
-	current->softirq_nestcnt++;
663
-
664
-	do_current_softirqs();
665
-	current->softirq_nestcnt--;
666
-	local_irq_enable();
667
-	cond_resched();
668
-}
669
-
670
-/*
671
- * Called from netif_rx_ni(). Preemption enabled, but migration
672
- * disabled. So the cpu can't go away under us.
673
- */
674
-void thread_do_softirq(void)
675
-{
676
-	if (!in_serving_softirq() && current->softirqs_raised) {
677
-		current->softirq_nestcnt++;
678
-		do_current_softirqs();
679
-		current->softirq_nestcnt--;
680
-	}
681
-}
682
-
683
-static void do_raise_softirq_irqoff(unsigned int nr)
684
-{
685
-	unsigned int mask;
686
-
687
-	mask = 1UL << nr;
688
-
689
-	trace_softirq_raise(nr);
690
-	or_softirq_pending(mask);
691
-
692
-	/*
693
-	 * If we are not in a hard interrupt and inside a bh disabled
694
-	 * region, we simply raise the flag on current. local_bh_enable()
695
-	 * will make sure that the softirq is executed. Otherwise we
696
-	 * delegate it to ksoftirqd.
697
-	 */
698
-	if (!in_irq() && current->softirq_nestcnt)
699
-		current->softirqs_raised |= mask;
700
-	else if (!__this_cpu_read(ksoftirqd) || !__this_cpu_read(ktimer_softirqd))
701
-		return;
702
-
703
-	if (mask & TIMER_SOFTIRQS)
704
-		__this_cpu_read(ktimer_softirqd)->softirqs_raised |= mask;
705
-	else
706
-		__this_cpu_read(ksoftirqd)->softirqs_raised |= mask;
707
-}
708
-
709
-static void wakeup_proper_softirq(unsigned int nr)
710
-{
711
-	if ((1UL << nr) & TIMER_SOFTIRQS)
712
-		wakeup_timer_softirqd();
713
-	else
714
-		wakeup_softirqd();
715
-}
716
-
717
-void __raise_softirq_irqoff(unsigned int nr)
718
-{
719
-	do_raise_softirq_irqoff(nr);
720
-	if (!in_irq() && !current->softirq_nestcnt)
721
-		wakeup_proper_softirq(nr);
722
-}
723
-
724
-/*
725
- * Same as __raise_softirq_irqoff() but will process them in ksoftirqd
726
- */
727
-void __raise_softirq_irqoff_ksoft(unsigned int nr)
728
-{
729
-	unsigned int mask;
730
-
731
-	if (WARN_ON_ONCE(!__this_cpu_read(ksoftirqd) ||
732
-			 !__this_cpu_read(ktimer_softirqd)))
733
-		return;
734
-	mask = 1UL << nr;
735
-
736
-	trace_softirq_raise(nr);
737
-	or_softirq_pending(mask);
738
-	if (mask & TIMER_SOFTIRQS)
739
-		__this_cpu_read(ktimer_softirqd)->softirqs_raised |= mask;
740
-	else
741
-		__this_cpu_read(ksoftirqd)->softirqs_raised |= mask;
742
-	wakeup_proper_softirq(nr);
743
-}
744
-
745
-/*
746
- * This function must run with irqs disabled!
747
- */
748
-void raise_softirq_irqoff(unsigned int nr)
749
-{
750
-	do_raise_softirq_irqoff(nr);
751
-
752
-	/*
753
-	 * If we're in an hard interrupt we let irq return code deal
754
-	 * with the wakeup of ksoftirqd.
755
-	 */
756
-	if (in_irq())
757
-		return;
758
-	/*
759
-	 * If we are in thread context but outside of a bh disabled
760
-	 * region, we need to wake ksoftirqd as well.
761
-	 *
762
-	 * CHECKME: Some of the places which do that could be wrapped
763
-	 * into local_bh_disable/enable pairs. Though it's unclear
764
-	 * whether this is worth the effort. To find those places just
765
-	 * raise a WARN() if the condition is met.
766
-	 */
767
-	if (!current->softirq_nestcnt)
768
-		wakeup_proper_softirq(nr);
769
-}
770
-
771
-static inline int ksoftirqd_softirq_pending(void)
772
-{
773
-	return current->softirqs_raised;
774
-}
775
-
776
-static inline void local_bh_disable_nort(void) { }
777
-static inline void _local_bh_enable_nort(void) { }
778
-
779
-static inline void ksoftirqd_set_sched_params(unsigned int cpu)
780
-{
781
-	/* Take over all but timer pending softirqs when starting */
782
-	local_irq_disable();
783
-	current->softirqs_raised = local_softirq_pending() & ~TIMER_SOFTIRQS;
784
-	local_irq_enable();
785
-}
786
-
787
-static inline void ktimer_softirqd_set_sched_params(unsigned int cpu)
788
-{
789
-	struct sched_param param = { .sched_priority = 1 };
790
-
791
-	sched_setscheduler(current, SCHED_FIFO, &param);
792
-
793
-	/* Take over timer pending softirqs when starting */
794
-	local_irq_disable();
795
-	current->softirqs_raised = local_softirq_pending() & TIMER_SOFTIRQS;
796
-	local_irq_enable();
797
-}
798
-
799
-static inline void ktimer_softirqd_clr_sched_params(unsigned int cpu,
800
-						    bool online)
801
-{
802
-	struct sched_param param = { .sched_priority = 0 };
803
-
804
-	sched_setscheduler(current, SCHED_NORMAL, &param);
805
-}
806
-
807
-static int ktimer_softirqd_should_run(unsigned int cpu)
808
-{
809
-	return current->softirqs_raised;
810
-}
811
-
812
-#endif /* PREEMPT_RT_FULL */
813
-/*
814
- * Enter an interrupt context.
815
- */
816
-void irq_enter(void)
817
-{
818
-	rcu_irq_enter();
819369 	if (is_idle_task(current) && !in_interrupt()) {
820370 		/*
821371 		 * Prevent raise_softirq from needlessly waking up ksoftirqd
822372 		 * here, as softirq will be serviced on return from interrupt.
823373 		 */
824
-		local_bh_disable_nort();
374
+		local_bh_disable();
825375 		tick_irq_enter();
826
-		_local_bh_enable_nort();
376
+		_local_bh_enable();
827377 	}
828
-
829378 	__irq_enter();
379
+}
380
+
381
+/**
382
+ * irq_enter - Enter an interrupt context including RCU update
383
+ */
384
+void irq_enter(void)
385
+{
386
+	rcu_irq_enter();
387
+	irq_enter_rcu();
830388 }
831389 
832390 static inline void invoke_softirq(void)
833391 {
834
-#ifndef CONFIG_PREEMPT_RT_FULL
835
-	if (ksoftirqd_running(local_softirq_pending()))
836
-		return;
837
-
838392 	if (!force_irqthreads) {
839393 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
840394 		/*
....@@ -854,18 +408,6 @@
854408 	} else {
855409 		wakeup_softirqd();
856410 	}
857
-#else /* PREEMPT_RT_FULL */
858
-	unsigned long flags;
859
-
860
-	local_irq_save(flags);
861
-	if (__this_cpu_read(ksoftirqd) &&
862
-			__this_cpu_read(ksoftirqd)->softirqs_raised)
863
-		wakeup_softirqd();
864
-	if (__this_cpu_read(ktimer_softirqd) &&
865
-			__this_cpu_read(ktimer_softirqd)->softirqs_raised)
866
-		wakeup_timer_softirqd();
867
-	local_irq_restore(flags);
868
-#endif
869411 }
870412 
871413 static inline void tick_irq_exit(void)
....@@ -881,10 +423,7 @@
881423 #endif
882424 }
883425 
884
-/*
885
- * Exit an interrupt context. Process softirqs if needed and possible:
886
- */
887
-void irq_exit(void)
426
+static inline void __irq_exit_rcu(void)
888427 {
889428 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
890429 	local_irq_disable();
....@@ -897,8 +436,51 @@
897436 		invoke_softirq();
898437 
899438 	tick_irq_exit();
439
+}
440
+
441
+/**
442
+ * irq_exit_rcu() - Exit an interrupt context without updating RCU
443
+ *
444
+ * Also processes softirqs if needed and possible.
445
+ */
446
+void irq_exit_rcu(void)
447
+{
448
+	__irq_exit_rcu();
449
+	 /* must be last! */
450
+	lockdep_hardirq_exit();
451
+}
452
+
453
+/**
454
+ * irq_exit - Exit an interrupt context, update RCU and lockdep
455
+ *
456
+ * Also processes softirqs if needed and possible.
457
+ */
458
+void irq_exit(void)
459
+{
460
+	__irq_exit_rcu();
900461 	rcu_irq_exit();
901
-	trace_hardirq_exit(); /* must be last! */
462
+	 /* must be last! */
463
+	lockdep_hardirq_exit();
464
+}
465
+
466
+/*
467
+ * This function must run with irqs disabled!
468
+ */
469
+inline void raise_softirq_irqoff(unsigned int nr)
470
+{
471
+	__raise_softirq_irqoff(nr);
472
+
473
+	/*
474
+	 * If we're in an interrupt or softirq, we're done
475
+	 * (this also catches softirq-disabled code). We will
476
+	 * actually run the softirq once we return from
477
+	 * the irq or softirq.
478
+	 *
479
+	 * Otherwise we wake up ksoftirqd to make sure we
480
+	 * schedule the softirq soon.
481
+	 */
482
+	if (!in_interrupt())
483
+		wakeup_softirqd();
902484 }
903485 
904486 void raise_softirq(unsigned int nr)
....@@ -908,6 +490,13 @@
908490 	local_irq_save(flags);
909491 	raise_softirq_irqoff(nr);
910492 	local_irq_restore(flags);
493
+}
494
+
495
+void __raise_softirq_irqoff(unsigned int nr)
496
+{
497
+	lockdep_assert_irqs_disabled();
498
+	trace_softirq_raise(nr);
499
+	or_softirq_pending(1UL << nr);
911500 }
912501 
913502 void open_softirq(int nr, void (*action)(struct softirq_action *))
....@@ -934,44 +523,11 @@
934523 	unsigned long flags;
935524 
936525 	local_irq_save(flags);
937
-	if (!tasklet_trylock(t)) {
938
-		local_irq_restore(flags);
939
-		return;
940
-	}
941
-
942526 	head = this_cpu_ptr(headp);
943
-again:
944
-	/* We may have been preempted before tasklet_trylock
945
-	 * and __tasklet_action may have already run.
946
-	 * So double check the sched bit while the takslet
947
-	 * is locked before adding it to the list.
948
-	 */
949
-	if (test_bit(TASKLET_STATE_SCHED, &t->state)) {
950
-#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
951
-		if (test_and_set_bit(TASKLET_STATE_CHAINED, &t->state)) {
952
-			tasklet_unlock(t);
953
-			return;
954
-		}
955
-#endif
956
-		t->next = NULL;
957
-		*head->tail = t;
958
-		head->tail = &(t->next);
959
-		raise_softirq_irqoff(softirq_nr);
960
-		tasklet_unlock(t);
961
-	} else {
962
-		/* This is subtle. If we hit the corner case above
963
-		 * It is possible that we get preempted right here,
964
-		 * and another task has successfully called
965
-		 * tasklet_schedule(), then this function, and
966
-		 * failed on the trylock. Thus we must be sure
967
-		 * before releasing the tasklet lock, that the
968
-		 * SCHED_BIT is clear. Otherwise the tasklet
969
-		 * may get its SCHED_BIT set, but not added to the
970
-		 * list
971
-		 */
972
-		if (!tasklet_tryunlock(t))
973
-			goto again;
974
-	}
527
+	t->next = NULL;
528
+	*head->tail = t;
529
+	head->tail = &(t->next);
530
+	raise_softirq_irqoff(softirq_nr);
975531 	local_irq_restore(flags);
976532 }
977533 
....@@ -989,21 +545,11 @@
989545 }
990546 EXPORT_SYMBOL(__tasklet_hi_schedule);
991547 
992
-void tasklet_enable(struct tasklet_struct *t)
993
-{
994
-	if (!atomic_dec_and_test(&t->count))
995
-		return;
996
-	if (test_and_clear_bit(TASKLET_STATE_PENDING, &t->state))
997
-		tasklet_schedule(t);
998
-}
999
-EXPORT_SYMBOL(tasklet_enable);
1000
-
1001548 static void tasklet_action_common(struct softirq_action *a,
1002549 				  struct tasklet_head *tl_head,
1003550 				  unsigned int softirq_nr)
1004551 {
1005552 	struct tasklet_struct *list;
1006
-	int loops = 1000000;
1007553 
1008554 	local_irq_disable();
1009555 	list = tl_head->head;
....@@ -1015,60 +561,33 @@
1015561 		struct tasklet_struct *t = list;
1016562 
1017563 		list = list->next;
1018
-		/*
1019
-		 * Should always succeed - after a tasklist got on the
1020
-		 * list (after getting the SCHED bit set from 0 to 1),
1021
-		 * nothing but the tasklet softirq it got queued to can
1022
-		 * lock it:
1023
-		 */
1024
-		if (!tasklet_trylock(t)) {
1025
-			WARN_ON(1);
1026
-			continue;
1027
-		}
1028564 
1029
-		t->next = NULL;
1030
-
1031
-		if (unlikely(atomic_read(&t->count))) {
1032
-out_disabled:
1033
-			/* implicit unlock: */
1034
-			wmb();
1035
-			t->state = TASKLET_STATEF_PENDING;
1036
-			continue;
1037
-		}
1038
-		/*
1039
-		 * After this point on the tasklet might be rescheduled
1040
-		 * on another CPU, but it can only be added to another
1041
-		 * CPU's tasklet list if we unlock the tasklet (which we
1042
-		 * dont do yet).
1043
-		 */
1044
-		if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
1045
-			WARN_ON(1);
1046
-again:
1047
-		t->func(t->data);
1048
-
1049
-#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
1050
-		while (cmpxchg(&t->state, TASKLET_STATEF_RC, 0) != TASKLET_STATEF_RC) {
1051
-#else
1052
-		while (!tasklet_tryunlock(t)) {
1053
-#endif
1054
-			/*
1055
-			 * If it got disabled meanwhile, bail out:
1056
-			 */
1057
-			if (atomic_read(&t->count))
1058
-				goto out_disabled;
1059
-			/*
1060
-			 * If it got scheduled meanwhile, re-execute
1061
-			 * the tasklet function:
1062
-			 */
1063
-			if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
1064
-				goto again;
1065
-			if (!--loops) {
1066
-				printk("hm, tasklet state: %08lx\n", t->state);
1067
-				WARN_ON(1);
565
+		if (tasklet_trylock(t)) {
566
+			if (!atomic_read(&t->count)) {
567
+				if (!test_and_clear_bit(TASKLET_STATE_SCHED,
568
+							&t->state))
569
+					BUG();
570
+				if (t->use_callback) {
571
+					trace_tasklet_entry(t->callback);
572
+					t->callback(t);
573
+					trace_tasklet_exit(t->callback);
574
+				} else {
575
+					trace_tasklet_entry(t->func);
576
+					t->func(t->data);
577
+					trace_tasklet_exit(t->func);
578
+				}
1068579 				tasklet_unlock(t);
1069
-				break;
580
+				continue;
1070581 			}
582
+			tasklet_unlock(t);
1071583 		}
584
+
585
+		local_irq_disable();
586
+		t->next = NULL;
587
+		*tl_head->tail = t;
588
+		tl_head->tail = &t->next;
589
+		__raise_softirq_irqoff(softirq_nr);
590
+		local_irq_enable();
1072591 	}
1073592 }
1074593 
....@@ -1082,6 +601,18 @@
1082601 	tasklet_action_common(a, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ);
1083602 }
1084603 
604
+void tasklet_setup(struct tasklet_struct *t,
605
+		   void (*callback)(struct tasklet_struct *))
606
+{
607
+	t->next = NULL;
608
+	t->state = 0;
609
+	atomic_set(&t->count, 0);
610
+	t->callback = callback;
611
+	t->use_callback = true;
612
+	t->data = 0;
613
+}
614
+EXPORT_SYMBOL(tasklet_setup);
615
+
1085616 void tasklet_init(struct tasklet_struct *t,
1086617 		  void (*func)(unsigned long), unsigned long data)
1087618 {
....@@ -1089,6 +620,7 @@
1089620 	t->state = 0;
1090621 	atomic_set(&t->count, 0);
1091622 	t->func = func;
623
+	t->use_callback = false;
1092624 	t->data = data;
1093625 }
1094626 EXPORT_SYMBOL(tasklet_init);
....@@ -1100,64 +632,13 @@
1100632 
1101633 	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
1102634 		do {
1103
-			msleep(1);
635
+			yield();
1104636 		} while (test_bit(TASKLET_STATE_SCHED, &t->state));
1105637 	}
1106638 	tasklet_unlock_wait(t);
1107639 	clear_bit(TASKLET_STATE_SCHED, &t->state);
1108640 }
1109641 EXPORT_SYMBOL(tasklet_kill);
1110
-
1111
-/*
1112
- * tasklet_hrtimer
1113
- */
1114
-
1115
-/*
1116
- * The trampoline is called when the hrtimer expires. It schedules a tasklet
1117
- * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
1118
- * hrtimer callback, but from softirq context.
1119
- */
1120
-static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
1121
-{
1122
-	struct tasklet_hrtimer *ttimer =
1123
-		container_of(timer, struct tasklet_hrtimer, timer);
1124
-
1125
-	tasklet_hi_schedule(&ttimer->tasklet);
1126
-	return HRTIMER_NORESTART;
1127
-}
1128
-
1129
-/*
1130
- * Helper function which calls the hrtimer callback from
1131
- * tasklet/softirq context
1132
- */
1133
-static void __tasklet_hrtimer_trampoline(unsigned long data)
1134
-{
1135
-	struct tasklet_hrtimer *ttimer = (void *)data;
1136
-	enum hrtimer_restart restart;
1137
-
1138
-	restart = ttimer->function(&ttimer->timer);
1139
-	if (restart != HRTIMER_NORESTART)
1140
-		hrtimer_restart(&ttimer->timer);
1141
-}
1142
-
1143
-/**
1144
- * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
1145
- * @ttimer:	 tasklet_hrtimer which is initialized
1146
- * @function:	 hrtimer callback function which gets called from softirq context
1147
- * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
1148
- * @mode:	 hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
1149
- */
1150
-void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
1151
-			  enum hrtimer_restart (*function)(struct hrtimer *),
1152
-			  clockid_t which_clock, enum hrtimer_mode mode)
1153
-{
1154
-	hrtimer_init(&ttimer->timer, which_clock, mode);
1155
-	ttimer->timer.function = __hrtimer_tasklet_trampoline;
1156
-	tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
1157
-		     (unsigned long)ttimer);
1158
-	ttimer->function = function;
1159
-}
1160
-EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
1161642 
1162643 void __init softirq_init(void)
1163644 {
....@@ -1174,26 +655,25 @@
1174655 	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
1175656 }
1176657 
1177
-#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
1178
-void tasklet_unlock_wait(struct tasklet_struct *t)
1179
-{
1180
-	while (test_bit(TASKLET_STATE_RUN, &(t)->state)) {
1181
-		/*
1182
-		 * Hack for now to avoid this busy-loop:
1183
-		 */
1184
-#ifdef CONFIG_PREEMPT_RT_FULL
1185
-		msleep(1);
1186
-#else
1187
-		barrier();
1188
-#endif
1189
-	}
1190
-}
1191
-EXPORT_SYMBOL(tasklet_unlock_wait);
1192
-#endif
1193
-
1194658 static int ksoftirqd_should_run(unsigned int cpu)
1195659 {
1196
-	return ksoftirqd_softirq_pending();
660
+	return local_softirq_pending();
661
+}
662
+
663
+static void run_ksoftirqd(unsigned int cpu)
664
+{
665
+	local_irq_disable();
666
+	if (local_softirq_pending()) {
667
+		/*
668
+		 * We can safely run softirq on inline stack, as we are not deep
669
+		 * in the task stack here.
670
+		 */
671
+		__do_softirq();
672
+		local_irq_enable();
673
+		cond_resched();
674
+		return;
675
+	}
676
+	local_irq_enable();
1197677 }
1198678 
1199679 #ifdef CONFIG_HOTPLUG_CPU
....@@ -1237,7 +717,7 @@
1237717 	/* Find end, append list for that CPU. */
1238718 	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
1239719 		*__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
1240
-		this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
720
+		__this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
1241721 		per_cpu(tasklet_vec, cpu).head = NULL;
1242722 		per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
1243723 	}
....@@ -1260,31 +740,17 @@
1260740 
1261741 static struct smp_hotplug_thread softirq_threads = {
1262742 	.store			= &ksoftirqd,
1263
-	.setup			= ksoftirqd_set_sched_params,
1264743 	.thread_should_run	= ksoftirqd_should_run,
1265744 	.thread_fn		= run_ksoftirqd,
1266745 	.thread_comm		= "ksoftirqd/%u",
1267746 };
1268
-
1269
-#ifdef CONFIG_PREEMPT_RT_FULL
1270
-static struct smp_hotplug_thread softirq_timer_threads = {
1271
-	.store			= &ktimer_softirqd,
1272
-	.setup			= ktimer_softirqd_set_sched_params,
1273
-	.cleanup		= ktimer_softirqd_clr_sched_params,
1274
-	.thread_should_run	= ktimer_softirqd_should_run,
1275
-	.thread_fn		= run_ksoftirqd,
1276
-	.thread_comm		= "ktimersoftd/%u",
1277
-};
1278
-#endif
1279747 
1280748 static __init int spawn_ksoftirqd(void)
1281749 {
1282750 	cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
1283751 				  takeover_tasklets);
1284752 	BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
1285
-#ifdef CONFIG_PREEMPT_RT_FULL
1286
-	BUG_ON(smpboot_register_percpu_thread(&softirq_timer_threads));
1287
-#endif
753
+
1288754 	return 0;
1289755 }
1290756 early_initcall(spawn_ksoftirqd);