blame | history | patch | commit | commitdiff
hc
2023-11-06 e3e12f52b214121840b44c91de5b3e5af5d3eb84 
 kernel/kernel/rcu/tree.c
....@@ -61,6 +61,13 @@
6161 #include <linux/trace_events.h>
6262 #include <linux/suspend.h>
6363 #include <linux/ftrace.h>
64
+#include <linux/delay.h>
65
+#include <linux/gfp.h>
66
+#include <linux/oom.h>
67
+#include <linux/smpboot.h>
68
+#include <linux/jiffies.h>
69
+#include <linux/sched/isolation.h>
70
+#include "../time/tick-internal.h"
6471 
6572 #include "tree.h"
6673 #include "rcu.h"
....@@ -245,6 +252,19 @@
245252 			   this_cpu_ptr(&rcu_sched_data), true);
246253 }
247254 
255
+#ifdef CONFIG_PREEMPT_RT_FULL
256
+static void rcu_preempt_qs(void);
257
+
258
+void rcu_bh_qs(void)
259
+{
260
+	unsigned long flags;
261
+
262
+	/* Callers to this function, rcu_preempt_qs(), must disable irqs. */
263
+	local_irq_save(flags);
264
+	rcu_preempt_qs();
265
+	local_irq_restore(flags);
266
+}
267
+#else
248268 void rcu_bh_qs(void)
249269 {
250270 	RCU_LOCKDEP_WARN(preemptible(), "rcu_bh_qs() invoked with preemption enabled!!!");
....@@ -255,6 +275,7 @@
255275 		__this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);
256276 	}
257277 }
278
+#endif
258279 
259280 /*
260281  * Steal a bit from the bottom of ->dynticks for idle entry/exit
....@@ -569,6 +590,7 @@
569590 }
570591 EXPORT_SYMBOL_GPL(rcu_sched_get_gp_seq);
571592 
593
+#ifndef CONFIG_PREEMPT_RT_FULL
572594 /*
573595  * Return the number of RCU-bh GPs completed thus far for debug & stats.
574596  */
....@@ -577,6 +599,7 @@
577599 	return READ_ONCE(rcu_bh_state.gp_seq);
578600 }
579601 EXPORT_SYMBOL_GPL(rcu_bh_get_gp_seq);
602
+#endif
580603 
581604 /*
582605  * Return the number of RCU expedited batches completed thus far for
....@@ -600,6 +623,7 @@
600623 }
601624 EXPORT_SYMBOL_GPL(rcu_exp_batches_completed_sched);
602625 
626
+#ifndef CONFIG_PREEMPT_RT_FULL
603627 /*
604628  * Force a quiescent state.
605629  */
....@@ -617,6 +641,13 @@
617641 	force_quiescent_state(&rcu_bh_state);
618642 }
619643 EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state);
644
+
645
+#else
646
+void rcu_force_quiescent_state(void)
647
+{
648
+}
649
+EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
650
+#endif
620651 
621652 /*
622653  * Force a quiescent state for RCU-sched.
....@@ -675,9 +706,11 @@
675706 	case RCU_FLAVOR:
676707 		rsp = rcu_state_p;
677708 		break;
709
+#ifndef CONFIG_PREEMPT_RT_FULL
678710 	case RCU_BH_FLAVOR:
679711 		rsp = &rcu_bh_state;
680712 		break;
713
+#endif
681714 	case RCU_SCHED_FLAVOR:
682715 		rsp = &rcu_sched_state;
683716 		break;
....@@ -1264,6 +1297,7 @@
12641297 		    !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
12651298 		    (rnp->ffmask & rdp->grpmask)) {
12661299 			init_irq_work(&rdp->rcu_iw, rcu_iw_handler);
1300
+			rdp->rcu_iw.flags = IRQ_WORK_HARD_IRQ;
12671301 			rdp->rcu_iw_pending = true;
12681302 			rdp->rcu_iw_gp_seq = rnp->gp_seq;
12691303 			irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
....@@ -2873,18 +2907,17 @@
28732907 /*
28742908  * Do RCU core processing for the current CPU.
28752909  */
2876
-static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused)
2910
+static __latent_entropy void rcu_process_callbacks(void)
28772911 {
28782912 	struct rcu_state *rsp;
28792913 
28802914 	if (cpu_is_offline(smp_processor_id()))
28812915 		return;
2882
-	trace_rcu_utilization(TPS("Start RCU core"));
28832916 	for_each_rcu_flavor(rsp)
28842917 		__rcu_process_callbacks(rsp);
2885
-	trace_rcu_utilization(TPS("End RCU core"));
28862918 }
28872919 
2920
+static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
28882921 /*
28892922  * Schedule RCU callback invocation.  If the specified type of RCU
28902923  * does not support RCU priority boosting, just do a direct call,
....@@ -2896,18 +2929,105 @@
28962929 {
28972930 	if (unlikely(!READ_ONCE(rcu_scheduler_fully_active)))
28982931 		return;
2899
-	if (likely(!rsp->boost)) {
2900
-		rcu_do_batch(rsp, rdp);
2901
-		return;
2902
-	}
2903
-	invoke_rcu_callbacks_kthread();
2932
+	rcu_do_batch(rsp, rdp);
29042933 }
29052934 
2935
+static void rcu_wake_cond(struct task_struct *t, int status)
2936
+{
2937
+	/*
2938
+	 * If the thread is yielding, only wake it when this
2939
+	 * is invoked from idle
2940
+	 */
2941
+	if (t && (status != RCU_KTHREAD_YIELDING || is_idle_task(current)))
2942
+		wake_up_process(t);
2943
+}
2944
+
2945
+/*
2946
+ * Wake up this CPU's rcuc kthread to do RCU core processing.
2947
+ */
29062948 static void invoke_rcu_core(void)
29072949 {
2908
-	if (cpu_online(smp_processor_id()))
2909
-		raise_softirq(RCU_SOFTIRQ);
2950
+	unsigned long flags;
2951
+	struct task_struct *t;
2952
+
2953
+	if (!cpu_online(smp_processor_id()))
2954
+		return;
2955
+	local_irq_save(flags);
2956
+	__this_cpu_write(rcu_cpu_has_work, 1);
2957
+	t = __this_cpu_read(rcu_cpu_kthread_task);
2958
+	if (t != NULL && current != t)
2959
+		rcu_wake_cond(t, __this_cpu_read(rcu_cpu_kthread_status));
2960
+	local_irq_restore(flags);
29102961 }
2962
+
2963
+static void rcu_cpu_kthread_park(unsigned int cpu)
2964
+{
2965
+	per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
2966
+}
2967
+
2968
+static int rcu_cpu_kthread_should_run(unsigned int cpu)
2969
+{
2970
+	return __this_cpu_read(rcu_cpu_has_work);
2971
+}
2972
+
2973
+/*
2974
+ * Per-CPU kernel thread that invokes RCU callbacks.  This replaces the
2975
+ * RCU softirq used in flavors and configurations of RCU that do not
2976
+ * support RCU priority boosting.
2977
+ */
2978
+static void rcu_cpu_kthread(unsigned int cpu)
2979
+{
2980
+	unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
2981
+	char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
2982
+	int spincnt;
2983
+
2984
+	for (spincnt = 0; spincnt < 10; spincnt++) {
2985
+		trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
2986
+		local_bh_disable();
2987
+		*statusp = RCU_KTHREAD_RUNNING;
2988
+		this_cpu_inc(rcu_cpu_kthread_loops);
2989
+		local_irq_disable();
2990
+		work = *workp;
2991
+		*workp = 0;
2992
+		local_irq_enable();
2993
+		if (work)
2994
+			rcu_process_callbacks();
2995
+		local_bh_enable();
2996
+		if (*workp == 0) {
2997
+			trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
2998
+			*statusp = RCU_KTHREAD_WAITING;
2999
+			return;
3000
+		}
3001
+	}
3002
+	*statusp = RCU_KTHREAD_YIELDING;
3003
+	trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
3004
+	schedule_timeout_interruptible(2);
3005
+	trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
3006
+	*statusp = RCU_KTHREAD_WAITING;
3007
+}
3008
+
3009
+static struct smp_hotplug_thread rcu_cpu_thread_spec = {
3010
+	.store			= &rcu_cpu_kthread_task,
3011
+	.thread_should_run	= rcu_cpu_kthread_should_run,
3012
+	.thread_fn		= rcu_cpu_kthread,
3013
+	.thread_comm		= "rcuc/%u",
3014
+	.setup			= rcu_cpu_kthread_setup,
3015
+	.park			= rcu_cpu_kthread_park,
3016
+};
3017
+
3018
+/*
3019
+ * Spawn per-CPU RCU core processing kthreads.
3020
+ */
3021
+static int __init rcu_spawn_core_kthreads(void)
3022
+{
3023
+	int cpu;
3024
+
3025
+	for_each_possible_cpu(cpu)
3026
+		per_cpu(rcu_cpu_has_work, cpu) = 0;
3027
+	BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
3028
+	return 0;
3029
+}
3030
+early_initcall(rcu_spawn_core_kthreads);
29113031 
29123032 /*
29133033  * Handle any core-RCU processing required by a call_rcu() invocation.
....@@ -3060,6 +3180,7 @@
30603180 }
30613181 EXPORT_SYMBOL_GPL(call_rcu_sched);
30623182 
3183
+#ifndef CONFIG_PREEMPT_RT_FULL
30633184 /**
30643185  * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
30653186  * @head: structure to be used for queueing the RCU updates.
....@@ -3087,6 +3208,7 @@
30873208 	__call_rcu(head, func, &rcu_bh_state, -1, 0);
30883209 }
30893210 EXPORT_SYMBOL_GPL(call_rcu_bh);
3211
+#endif
30903212 
30913213 /*
30923214  * Queue an RCU callback for lazy invocation after a grace period.
....@@ -3172,6 +3294,7 @@
31723294 }
31733295 EXPORT_SYMBOL_GPL(synchronize_sched);
31743296 
3297
+#ifndef CONFIG_PREEMPT_RT_FULL
31753298 /**
31763299  * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed.
31773300  *
....@@ -3198,6 +3321,7 @@
31983321 		wait_rcu_gp(call_rcu_bh);
31993322 }
32003323 EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
3324
+#endif
32013325 
32023326 /**
32033327  * get_state_synchronize_rcu - Snapshot current RCU state
....@@ -3505,6 +3629,7 @@
35053629 	mutex_unlock(&rsp->barrier_mutex);
35063630 }
35073631 
3632
+#ifndef CONFIG_PREEMPT_RT_FULL
35083633 /**
35093634  * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete.
35103635  */
....@@ -3513,6 +3638,7 @@
35133638 	_rcu_barrier(&rcu_bh_state);
35143639 }
35153640 EXPORT_SYMBOL_GPL(rcu_barrier_bh);
3641
+#endif
35163642 
35173643 /**
35183644  * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks.
....@@ -3662,8 +3788,6 @@
36623788 		rnp->ffmask |= rdp->grpmask;
36633789 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
36643790 	}
3665
-	if (IS_ENABLED(CONFIG_TREE_SRCU))
3666
-		srcu_online_cpu(cpu);
36673791 	if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
36683792 		return 0; /* Too early in boot for scheduler work. */
36693793 	sync_sched_exp_online_cleanup(cpu);
....@@ -3691,8 +3815,6 @@
36913815 	}
36923816 
36933817 	rcutree_affinity_setting(cpu, cpu);
3694
-	if (IS_ENABLED(CONFIG_TREE_SRCU))
3695
-		srcu_offline_cpu(cpu);
36963818 	return 0;
36973819 }
36983820 
....@@ -4160,12 +4282,13 @@
41604282 
41614283 	rcu_bootup_announce();
41624284 	rcu_init_geometry();
4285
+#ifndef CONFIG_PREEMPT_RT_FULL
41634286 	rcu_init_one(&rcu_bh_state);
4287
+#endif
41644288 	rcu_init_one(&rcu_sched_state);
41654289 	if (dump_tree)
41664290 		rcu_dump_rcu_node_tree(&rcu_sched_state);
41674291 	__rcu_init_preempt();
4168
-	open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
41694292 
41704293 	/*
41714294 	 * We don't need protection against CPU-hotplug here because