forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-14 982b8cc116118b3463d3f332581945625722acd8 
 kernel/kernel/softirq.c
....@@ -1,9 +1,8 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  *	linux/kernel/softirq.c
34  *
45  *	Copyright (C) 1992 Linus Torvalds
5
- *
6
- *	Distribute under GPLv2.
76  *
87  *	Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
98  */
....@@ -29,6 +28,9 @@
2928 
3029 #define CREATE_TRACE_POINTS
3130 #include <trace/events/irq.h>
31
+
32
+EXPORT_TRACEPOINT_SYMBOL_GPL(irq_handler_entry);
33
+EXPORT_TRACEPOINT_SYMBOL_GPL(irq_handler_exit);
3234 
3335 /*
3436    - No shared variables, all the data are CPU local.
....@@ -56,6 +58,14 @@
5658 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
5759 
5860 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
61
+EXPORT_PER_CPU_SYMBOL_GPL(ksoftirqd);
62
+
63
+/*
64
+ * active_softirqs -- per cpu, a mask of softirqs that are being handled,
65
+ * with the expectation that approximate answers are acceptable and therefore
66
+ * no synchronization.
67
+ */
68
+DEFINE_PER_CPU(__u32, active_softirqs);
5969 
6070 const char * const softirq_to_name[NR_SOFTIRQS] = {
6171 	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
....@@ -78,21 +88,6 @@
7888 }
7989 
8090 /*
81
- * If ksoftirqd is scheduled, we do not want to process pending softirqs
82
- * right now. Let ksoftirqd handle this at its own rate, to get fairness,
83
- * unless we're doing some of the synchronous softirqs.
84
- */
85
-#define SOFTIRQ_NOW_MASK ((1 << HI_SOFTIRQ) | (1 << TASKLET_SOFTIRQ))
86
-static bool ksoftirqd_running(unsigned long pending)
87
-{
88
-	struct task_struct *tsk = __this_cpu_read(ksoftirqd);
89
-
90
-	if (pending & SOFTIRQ_NOW_MASK)
91
-		return false;
92
-	return tsk && (tsk->state == TASK_RUNNING);
93
-}
94
-
95
-/*
9691  * preempt_count and SOFTIRQ_OFFSET usage:
9792  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
9893  *   softirq processing.
....@@ -107,6 +102,12 @@
107102  * where hardirqs are disabled legitimately:
108103  */
109104 #ifdef CONFIG_TRACE_IRQFLAGS
105
+
106
+DEFINE_PER_CPU(int, hardirqs_enabled);
107
+DEFINE_PER_CPU(int, hardirq_context);
108
+EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
109
+EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
110
+
110111 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
111112 {
112113 	unsigned long flags;
....@@ -126,7 +127,7 @@
126127 	 * Were softirqs turned off above:
127128 	 */
128129 	if (softirq_count() == (cnt & SOFTIRQ_MASK))
129
-		trace_softirqs_off(ip);
130
+		lockdep_softirqs_off(ip);
130131 	raw_local_irq_restore(flags);
131132 
132133 	if (preempt_count() == cnt) {
....@@ -147,7 +148,7 @@
147148 		trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
148149 
149150 	if (softirq_count() == (cnt & SOFTIRQ_MASK))
150
-		trace_softirqs_on(_RET_IP_);
151
+		lockdep_softirqs_on(_RET_IP_);
151152 
152153 	__preempt_count_sub(cnt);
153154 }
....@@ -174,7 +175,7 @@
174175 	 * Are softirqs going to be turned on now:
175176 	 */
176177 	if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
177
-		trace_softirqs_on(ip);
178
+		lockdep_softirqs_on(ip);
178179 	/*
179180 	 * Keep preemption disabled until we are done with
180181 	 * softirq processing:
....@@ -224,9 +225,9 @@
224225 {
225226 	bool in_hardirq = false;
226227 
227
-	if (trace_hardirq_context(current)) {
228
+	if (lockdep_hardirq_context()) {
228229 		in_hardirq = true;
229
-		trace_hardirq_exit();
230
+		lockdep_hardirq_exit();
230231 	}
231232 
232233 	lockdep_softirq_enter();
....@@ -239,12 +240,22 @@
239240 	lockdep_softirq_exit();
240241 
241242 	if (in_hardirq)
242
-		trace_hardirq_enter();
243
+		lockdep_hardirq_enter();
243244 }
244245 #else
245246 static inline bool lockdep_softirq_start(void) { return false; }
246247 static inline void lockdep_softirq_end(bool in_hardirq) { }
247248 #endif
249
+
250
+#define softirq_deferred_for_rt(pending)		\
251
+({							\
252
+	__u32 deferred = 0;				\
253
+	if (cpupri_check_rt()) {			\
254
+		deferred = pending & LONG_SOFTIRQ_MASK; \
255
+		pending &= ~LONG_SOFTIRQ_MASK;		\
256
+	}						\
257
+	deferred;					\
258
+})
248259 
249260 asmlinkage __visible void __softirq_entry __do_softirq(void)
250261 {
....@@ -253,25 +264,27 @@
253264 	int max_restart = MAX_SOFTIRQ_RESTART;
254265 	struct softirq_action *h;
255266 	bool in_hardirq;
267
+	__u32 deferred;
256268 	__u32 pending;
257269 	int softirq_bit;
258270 
259271 	/*
260
-	 * Mask out PF_MEMALLOC s current task context is borrowed for the
261
-	 * softirq. A softirq handled such as network RX might set PF_MEMALLOC
262
-	 * again if the socket is related to swap
272
+	 * Mask out PF_MEMALLOC as the current task context is borrowed for the
273
+	 * softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
274
+	 * again if the socket is related to swapping.
263275 	 */
264276 	current->flags &= ~PF_MEMALLOC;
265277 
266278 	pending = local_softirq_pending();
279
+	deferred = softirq_deferred_for_rt(pending);
267280 	account_irq_enter_time(current);
268
-
269281 	__local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
270282 	in_hardirq = lockdep_softirq_start();
271283 
272284 restart:
273285 	/* Reset the pending bitmask before enabling irqs */
274
-	set_softirq_pending(0);
286
+	set_softirq_pending(deferred);
287
+	__this_cpu_write(active_softirqs, pending);
275288 
276289 	local_irq_enable();
277290 
....@@ -301,18 +314,28 @@
301314 		pending >>= softirq_bit;
302315 	}
303316 
304
-	rcu_bh_qs();
317
+	__this_cpu_write(active_softirqs, 0);
318
+	if (__this_cpu_read(ksoftirqd) == current)
319
+		rcu_softirq_qs();
305320 	local_irq_disable();
306321 
307322 	pending = local_softirq_pending();
323
+	deferred = softirq_deferred_for_rt(pending);
324
+
308325 	if (pending) {
309326 		if (time_before(jiffies, end) && !need_resched() &&
310327 		    --max_restart)
311328 			goto restart;
312329 
330
+#ifndef CONFIG_RT_SOFTINT_OPTIMIZATION
313331 		wakeup_softirqd();
332
+#endif
314333 	}
315334 
335
+#ifdef CONFIG_RT_SOFTINT_OPTIMIZATION
336
+	if (pending | deferred)
337
+		wakeup_softirqd();
338
+#endif
316339 	lockdep_softirq_end(in_hardirq);
317340 	account_irq_exit_time(current);
318341 	__local_bh_enable(SOFTIRQ_OFFSET);
....@@ -332,18 +355,17 @@
332355 
333356 	pending = local_softirq_pending();
334357 
335
-	if (pending && !ksoftirqd_running(pending))
358
+	if (pending)
336359 		do_softirq_own_stack();
337360 
338361 	local_irq_restore(flags);
339362 }
340363 
341
-/*
342
- * Enter an interrupt context.
364
+/**
365
+ * irq_enter_rcu - Enter an interrupt context with RCU watching
343366  */
344
-void irq_enter(void)
367
+void irq_enter_rcu(void)
345368 {
346
-	rcu_irq_enter();
347369 	if (is_idle_task(current) && !in_interrupt()) {
348370 		/*
349371 		 * Prevent raise_softirq from needlessly waking up ksoftirqd
....@@ -353,15 +375,20 @@
353375 		tick_irq_enter();
354376 		_local_bh_enable();
355377 	}
356
-
357378 	__irq_enter();
379
+}
380
+
381
+/**
382
+ * irq_enter - Enter an interrupt context including RCU update
383
+ */
384
+void irq_enter(void)
385
+{
386
+	rcu_irq_enter();
387
+	irq_enter_rcu();
358388 }
359389 
360390 static inline void invoke_softirq(void)
361391 {
362
-	if (ksoftirqd_running(local_softirq_pending()))
363
-		return;
364
-
365392 	if (!force_irqthreads) {
366393 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
367394 		/*
....@@ -396,10 +423,7 @@
396423 #endif
397424 }
398425 
399
-/*
400
- * Exit an interrupt context. Process softirqs if needed and possible:
401
- */
402
-void irq_exit(void)
426
+static inline void __irq_exit_rcu(void)
403427 {
404428 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
405429 	local_irq_disable();
....@@ -412,8 +436,31 @@
412436 		invoke_softirq();
413437 
414438 	tick_irq_exit();
439
+}
440
+
441
+/**
442
+ * irq_exit_rcu() - Exit an interrupt context without updating RCU
443
+ *
444
+ * Also processes softirqs if needed and possible.
445
+ */
446
+void irq_exit_rcu(void)
447
+{
448
+	__irq_exit_rcu();
449
+	 /* must be last! */
450
+	lockdep_hardirq_exit();
451
+}
452
+
453
+/**
454
+ * irq_exit - Exit an interrupt context, update RCU and lockdep
455
+ *
456
+ * Also processes softirqs if needed and possible.
457
+ */
458
+void irq_exit(void)
459
+{
460
+	__irq_exit_rcu();
415461 	rcu_irq_exit();
416
-	trace_hardirq_exit(); /* must be last! */
462
+	 /* must be last! */
463
+	lockdep_hardirq_exit();
417464 }
418465 
419466 /*
....@@ -447,6 +494,7 @@
447494 
448495 void __raise_softirq_irqoff(unsigned int nr)
449496 {
497
+	lockdep_assert_irqs_disabled();
450498 	trace_softirq_raise(nr);
451499 	or_softirq_pending(1UL << nr);
452500 }
....@@ -519,7 +567,15 @@
519567 				if (!test_and_clear_bit(TASKLET_STATE_SCHED,
520568 							&t->state))
521569 					BUG();
522
-				t->func(t->data);
570
+				if (t->use_callback) {
571
+					trace_tasklet_entry(t->callback);
572
+					t->callback(t);
573
+					trace_tasklet_exit(t->callback);
574
+				} else {
575
+					trace_tasklet_entry(t->func);
576
+					t->func(t->data);
577
+					trace_tasklet_exit(t->func);
578
+				}
523579 				tasklet_unlock(t);
524580 				continue;
525581 			}
....@@ -545,6 +601,18 @@
545601 	tasklet_action_common(a, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ);
546602 }
547603 
604
+void tasklet_setup(struct tasklet_struct *t,
605
+		   void (*callback)(struct tasklet_struct *))
606
+{
607
+	t->next = NULL;
608
+	t->state = 0;
609
+	atomic_set(&t->count, 0);
610
+	t->callback = callback;
611
+	t->use_callback = true;
612
+	t->data = 0;
613
+}
614
+EXPORT_SYMBOL(tasklet_setup);
615
+
548616 void tasklet_init(struct tasklet_struct *t,
549617 		  void (*func)(unsigned long), unsigned long data)
550618 {
....@@ -552,6 +620,7 @@
552620 	t->state = 0;
553621 	atomic_set(&t->count, 0);
554622 	t->func = func;
623
+	t->use_callback = false;
555624 	t->data = data;
556625 }
557626 EXPORT_SYMBOL(tasklet_init);
....@@ -570,57 +639,6 @@
570639 	clear_bit(TASKLET_STATE_SCHED, &t->state);
571640 }
572641 EXPORT_SYMBOL(tasklet_kill);
573
-
574
-/*
575
- * tasklet_hrtimer
576
- */
577
-
578
-/*
579
- * The trampoline is called when the hrtimer expires. It schedules a tasklet
580
- * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
581
- * hrtimer callback, but from softirq context.
582
- */
583
-static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
584
-{
585
-	struct tasklet_hrtimer *ttimer =
586
-		container_of(timer, struct tasklet_hrtimer, timer);
587
-
588
-	tasklet_hi_schedule(&ttimer->tasklet);
589
-	return HRTIMER_NORESTART;
590
-}
591
-
592
-/*
593
- * Helper function which calls the hrtimer callback from
594
- * tasklet/softirq context
595
- */
596
-static void __tasklet_hrtimer_trampoline(unsigned long data)
597
-{
598
-	struct tasklet_hrtimer *ttimer = (void *)data;
599
-	enum hrtimer_restart restart;
600
-
601
-	restart = ttimer->function(&ttimer->timer);
602
-	if (restart != HRTIMER_NORESTART)
603
-		hrtimer_restart(&ttimer->timer);
604
-}
605
-
606
-/**
607
- * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
608
- * @ttimer:	 tasklet_hrtimer which is initialized
609
- * @function:	 hrtimer callback function which gets called from softirq context
610
- * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
611
- * @mode:	 hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
612
- */
613
-void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
614
-			  enum hrtimer_restart (*function)(struct hrtimer *),
615
-			  clockid_t which_clock, enum hrtimer_mode mode)
616
-{
617
-	hrtimer_init(&ttimer->timer, which_clock, mode);
618
-	ttimer->timer.function = __hrtimer_tasklet_trampoline;
619
-	tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
620
-		     (unsigned long)ttimer);
621
-	ttimer->function = function;
622
-}
623
-EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
624642 
625643 void __init softirq_init(void)
626644 {
....@@ -699,7 +717,7 @@
699717 	/* Find end, append list for that CPU. */
700718 	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
701719 		*__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
702
-		this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
720
+		__this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
703721 		per_cpu(tasklet_vec, cpu).head = NULL;
704722 		per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
705723 	}