~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,27 +1,14 @@
	1	+// SPDX-License-Identifier: GPL-2.0+
1	2	/*
2		- * Read-Copy Update mechanism for mutual exclusion
3		- *
4		- * This program is free software; you can redistribute it and/or modify
5		- * it under the terms of the GNU General Public License as published by
6		- * the Free Software Foundation; either version 2 of the License, or
7		- * (at your option) any later version.
8		- *
9		- * This program is distributed in the hope that it will be useful,
10		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12		- * GNU General Public License for more details.
13		- *
14		- * You should have received a copy of the GNU General Public License
15		- * along with this program; if not, you can access it online at
16		- * http://www.gnu.org/licenses/gpl-2.0.html.
	3	+ * Read-Copy Update mechanism for mutual exclusion (tree-based version)
17	4	*
18	5	* Copyright IBM Corporation, 2008
19	6	*
20	7	* Authors: Dipankar Sarma <dipankar@in.ibm.com>
21	8	* Manfred Spraul <manfred@colorfullife.com>
22		- * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version
	9	+ * Paul E. McKenney <paulmck@linux.ibm.com>
23	10	*
24		- * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
	11	+ * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
25	12	* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
26	13	*
27	14	* For detailed explanation of Read-Copy Update mechanism see -
..	..	@@ -56,11 +43,24 @@
56	43	#include <uapi/linux/sched/types.h>
57	44	#include <linux/prefetch.h>
58	45	#include <linux/delay.h>
59		-#include <linux/stop_machine.h>
60	46	#include <linux/random.h>
61	47	#include <linux/trace_events.h>
62	48	#include <linux/suspend.h>
63	49	#include <linux/ftrace.h>
	50	+#include <linux/tick.h>
	51	+#include <linux/sysrq.h>
	52	+#include <linux/kprobes.h>
	53	+#include <linux/gfp.h>
	54	+#include <linux/oom.h>
	55	+#include <linux/smpboot.h>
	56	+#include <linux/jiffies.h>
	57	+#include <linux/slab.h>
	58	+#include <linux/sched/isolation.h>
	59	+#include <linux/sched/clock.h>
	60	+#include <linux/vmalloc.h>
	61	+#include <linux/mm.h>
	62	+#include <linux/kasan.h>
	63	+#include "../time/tick-internal.h"
64	64
65	65	#include "tree.h"
66	66	#include "rcu.h"
..	..	@@ -73,49 +73,37 @@
73	73	/* Data structures. */
74	74
75	75	/*
76		- * In order to export the rcu_state name to the tracing tools, it
77		- * needs to be added in the __tracepoint_string section.
78		- * This requires defining a separate variable tp_<sname>_varname
79		- * that points to the string being used, and this will allow
80		- * the tracing userspace tools to be able to decipher the string
81		- * address to the matching string.
	76	+ * Steal a bit from the bottom of ->dynticks for idle entry/exit
	77	+ * control. Initially this is for TLB flushing.
82	78	*/
83		-#ifdef CONFIG_TRACING
84		-# define DEFINE_RCU_TPS(sname) \
85		-static char sname##_varname[] = #sname; \
86		-static const char *tp_##sname##_varname __used __tracepoint_string = sname##_varname;
87		-# define RCU_STATE_NAME(sname) sname##_varname
88		-#else
89		-# define DEFINE_RCU_TPS(sname)
90		-# define RCU_STATE_NAME(sname) __stringify(sname)
91		-#endif
	79	+#define RCU_DYNTICK_CTRL_MASK 0x1
	80	+#define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1)
92	81
93		-#define RCU_STATE_INITIALIZER(sname, sabbr, cr) \
94		-DEFINE_RCU_TPS(sname) \
95		-static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data); \
96		-struct rcu_state sname##_state = { \
97		- .level = { &sname##_state.node[0] }, \
98		- .rda = &sname##_data, \
99		- .call = cr, \
100		- .gp_state = RCU_GP_IDLE, \
101		- .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT, \
102		- .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \
103		- .name = RCU_STATE_NAME(sname), \
104		- .abbr = sabbr, \
105		- .exp_mutex = __MUTEX_INITIALIZER(sname##_state.exp_mutex), \
106		- .exp_wake_mutex = __MUTEX_INITIALIZER(sname##_state.exp_wake_mutex), \
107		- .ofl_lock = __SPIN_LOCK_UNLOCKED(sname##_state.ofl_lock), \
108		-}
109		-
110		-RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
111		-RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh);
112		-
113		-static struct rcu_state *const rcu_state_p;
114		-LIST_HEAD(rcu_struct_flavors);
	82	+static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
	83	+ .dynticks_nesting = 1,
	84	+ .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
	85	+ .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),
	86	+};
	87	+static struct rcu_state rcu_state = {
	88	+ .level = { &rcu_state.node[0] },
	89	+ .gp_state = RCU_GP_IDLE,
	90	+ .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT,
	91	+ .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
	92	+ .name = RCU_NAME,
	93	+ .abbr = RCU_ABBR,
	94	+ .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
	95	+ .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
	96	+ .ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock),
	97	+};
115	98
116	99	/* Dump rcu_node combining tree at boot to verify correct setup. */
117	100	static bool dump_tree;
118	101	module_param(dump_tree, bool, 0444);
	102	+/* By default, use RCU_SOFTIRQ instead of rcuc kthreads. */
	103	+static bool use_softirq = !IS_ENABLED(CONFIG_PREEMPT_RT);
	104	+#ifndef CONFIG_PREEMPT_RT
	105	+module_param(use_softirq, bool, 0444);
	106	+#endif
119	107	/* Control rcu_node-tree auto-balancing at boot time. */
120	108	static bool rcu_fanout_exact;
121	109	module_param(rcu_fanout_exact, bool, 0444);
..	..	@@ -126,9 +114,6 @@
126	114	/* Number of rcu_nodes at specified level. */
127	115	int num_rcu_lvl[] = NUM_RCU_LVL_INIT;
128	116	int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */
129		-/* panic() on RCU Stall sysctl. */
130		-int sysctl_panic_on_rcu_stall __read_mostly = CONFIG_BOOTPARAM_RCU_STALL_PANIC_VALUE;
131		-ATOMIC_NOTIFIER_HEAD(rcu_stall_notifier_list);
132	117
133	118	/*
134	119	* The rcu_scheduler_active variable is initialized to the value
..	..	@@ -159,21 +144,19 @@
159	144	*/
160	145	static int rcu_scheduler_fully_active __read_mostly;
161	146
162		-static void
163		-rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
164		- struct rcu_node *rnp, unsigned long gps, unsigned long flags);
	147	+static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
	148	+ unsigned long gps, unsigned long flags);
165	149	static void rcu_init_new_rnp(struct rcu_node *rnp_leaf);
166	150	static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf);
167	151	static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
168	152	static void invoke_rcu_core(void);
169		-static void invoke_rcu_callbacks(struct rcu_state rsp, struct rcu_data rdp);
170		-static void rcu_report_exp_rdp(struct rcu_state *rsp,
171		- struct rcu_data *rdp, bool wake);
	153	+static void rcu_report_exp_rdp(struct rcu_data *rdp);
172	154	static void sync_sched_exp_online_cleanup(int cpu);
	155	+static void check_cb_ovld_locked(struct rcu_data rdp, struct rcu_node rnp);
173	156
174	157	/* rcuc/rcub kthread realtime priority */
175	158	static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
176		-module_param(kthread_prio, int, 0644);
	159	+module_param(kthread_prio, int, 0444);
177	160
178	161	/* Delay in jiffies for grace-period initialization delays, debug only. */
179	162
..	..	@@ -184,7 +167,22 @@
184	167	static int gp_cleanup_delay;
185	168	module_param(gp_cleanup_delay, int, 0444);
186	169
187		-/* Retreive RCU kthreads priority for rcutorture */
	170	+// Add delay to rcu_read_unlock() for strict grace periods.
	171	+static int rcu_unlock_delay;
	172	+#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
	173	+module_param(rcu_unlock_delay, int, 0444);
	174	+#endif
	175	+
	176	+/*
	177	+ * This rcu parameter is runtime-read-only. It reflects
	178	+ * a minimum allowed number of objects which can be cached
	179	+ * per-CPU. Object size is equal to one page. This value
	180	+ * can be changed at boot time.
	181	+ */
	182	+static int rcu_min_cached_objs = 5;
	183	+module_param(rcu_min_cached_objs, int, 0444);
	184	+
	185	+/* Retrieve RCU kthreads priority for rcutorture */
188	186	int rcu_get_gp_kthreads_prio(void)
189	187	{
190	188	return kthread_prio;
..	..	@@ -208,7 +206,7 @@
208	206	* held, but the bit corresponding to the current CPU will be stable
209	207	* in most contexts.
210	208	*/
211		-unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp)
	209	+static unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp)
212	210	{
213	211	return READ_ONCE(rnp->qsmaskinitnext);
214	212	}
..	..	@@ -218,67 +216,39 @@
218	216	* permit this function to be invoked without holding the root rcu_node
219	217	* structure's ->lock, but of course results can be subject to change.
220	218	*/
221		-static int rcu_gp_in_progress(struct rcu_state *rsp)
	219	+static int rcu_gp_in_progress(void)
222	220	{
223		- return rcu_seq_state(rcu_seq_current(&rsp->gp_seq));
	221	+ return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));
224	222	}
225	223
226	224	/*
227		- * Note a quiescent state. Because we do not need to know
228		- * how many quiescent states passed, just if there was at least
229		- * one since the start of the grace period, this just sets a flag.
230		- * The caller must have disabled preemption.
	225	+ * Return the number of callbacks queued on the specified CPU.
	226	+ * Handles both the nocbs and normal cases.
231	227	*/
232		-void rcu_sched_qs(void)
	228	+static long rcu_get_n_cbs_cpu(int cpu)
233	229	{
234		- RCU_LOCKDEP_WARN(preemptible(), "rcu_sched_qs() invoked with preemption enabled!!!");
235		- if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.s))
236		- return;
237		- trace_rcu_grace_period(TPS("rcu_sched"),
238		- __this_cpu_read(rcu_sched_data.gp_seq),
239		- TPS("cpuqs"));
240		- __this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false);
241		- if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
242		- return;
243		- __this_cpu_write(rcu_sched_data.cpu_no_qs.b.exp, false);
244		- rcu_report_exp_rdp(&rcu_sched_state,
245		- this_cpu_ptr(&rcu_sched_data), true);
	230	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
	231	+
	232	+ if (rcu_segcblist_is_enabled(&rdp->cblist))
	233	+ return rcu_segcblist_n_cbs(&rdp->cblist);
	234	+ return 0;
246	235	}
247	236
248		-void rcu_bh_qs(void)
	237	+void rcu_softirq_qs(void)
249	238	{
250		- RCU_LOCKDEP_WARN(preemptible(), "rcu_bh_qs() invoked with preemption enabled!!!");
251		- if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) {
252		- trace_rcu_grace_period(TPS("rcu_bh"),
253		- __this_cpu_read(rcu_bh_data.gp_seq),
254		- TPS("cpuqs"));
255		- __this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);
256		- }
	239	+ rcu_qs();
	240	+ rcu_preempt_deferred_qs(current);
257	241	}
258		-
259		-/*
260		- * Steal a bit from the bottom of ->dynticks for idle entry/exit
261		- * control. Initially this is for TLB flushing.
262		- */
263		-#define RCU_DYNTICK_CTRL_MASK 0x1
264		-#define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1)
265		-#ifndef rcu_eqs_special_exit
266		-#define rcu_eqs_special_exit() do { } while (0)
267		-#endif
268		-
269		-static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
270		- .dynticks_nesting = 1,
271		- .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
272		- .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),
273		-};
274	242
275	243	/*
276	244	* Record entry into an extended quiescent state. This is only to be
277		- * called when not already in an extended quiescent state.
	245	+ * called when not already in an extended quiescent state, that is,
	246	+ * RCU is watching prior to the call to this function and is no longer
	247	+ * watching upon return.
278	248	*/
279		-static void rcu_dynticks_eqs_enter(void)
	249	+static noinstr void rcu_dynticks_eqs_enter(void)
280	250	{
281		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	251	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
282	252	int seq;
283	253
284	254	/*
..	..	@@ -286,8 +256,9 @@
286	256	* critical sections, and we also must force ordering with the
287	257	* next idle sojourn.
288	258	*/
289		- seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
290		- /* Better be in an extended quiescent state! */
	259	+ rcu_dynticks_task_trace_enter(); // Before ->dynticks update!
	260	+ seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
	261	+ // RCU is no longer watching. Better be in extended quiescent state!
291	262	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
292	263	(seq & RCU_DYNTICK_CTRL_CTR));
293	264	/* Better not have special action (TLB flush) pending! */
..	..	@@ -297,11 +268,12 @@
297	268
298	269	/*
299	270	* Record exit from an extended quiescent state. This is only to be
300		- * called from an extended quiescent state.
	271	+ * called from an extended quiescent state, that is, RCU is not watching
	272	+ * prior to the call to this function and is watching upon return.
301	273	*/
302		-static void rcu_dynticks_eqs_exit(void)
	274	+static noinstr void rcu_dynticks_eqs_exit(void)
303	275	{
304		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	276	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
305	277	int seq;
306	278
307	279	/*
..	..	@@ -309,14 +281,14 @@
309	281	* and we also must force ordering with the next RCU read-side
310	282	* critical section.
311	283	*/
312		- seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
	284	+ seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
	285	+ // RCU is now watching. Better not be in an extended quiescent state!
	286	+ rcu_dynticks_task_trace_exit(); // After ->dynticks update!
313	287	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
314	288	!(seq & RCU_DYNTICK_CTRL_CTR));
315	289	if (seq & RCU_DYNTICK_CTRL_MASK) {
316		- atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdtp->dynticks);
	290	+ arch_atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks);
317	291	smp_mb__after_atomic(); /* _exit after clearing mask. */
318		- /* Prefer duplicate flushes to losing a flush. */
319		- rcu_eqs_special_exit();
320	292	}
321	293	}
322	294
..	..	@@ -332,11 +304,11 @@
332	304	*/
333	305	static void rcu_dynticks_eqs_online(void)
334	306	{
335		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	307	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
336	308
337		- if (atomic_read(&rdtp->dynticks) & RCU_DYNTICK_CTRL_CTR)
	309	+ if (atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR)
338	310	return;
339		- atomic_add(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
	311	+ atomic_add(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
340	312	}
341	313
342	314	/*
..	..	@@ -344,20 +316,20 @@
344	316	*
345	317	* No ordering, as we are sampling CPU-local information.
346	318	*/
347		-bool rcu_dynticks_curr_cpu_in_eqs(void)
	319	+static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)
348	320	{
349		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	321	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
350	322
351		- return !(atomic_read(&rdtp->dynticks) & RCU_DYNTICK_CTRL_CTR);
	323	+ return !(arch_atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR);
352	324	}
353	325
354	326	/*
355	327	* Snapshot the ->dynticks counter with full ordering so as to allow
356	328	* stable comparison of this counter with past and future snapshots.
357	329	*/
358		-int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
	330	+static int rcu_dynticks_snap(struct rcu_data *rdp)
359	331	{
360		- int snap = atomic_add_return(0, &rdtp->dynticks);
	332	+ int snap = atomic_add_return(0, &rdp->dynticks);
361	333
362	334	return snap & ~RCU_DYNTICK_CTRL_MASK;
363	335	}
..	..	@@ -372,13 +344,35 @@
372	344	}
373	345
374	346	/*
375		- * Return true if the CPU corresponding to the specified rcu_dynticks
	347	+ * Return true if the CPU corresponding to the specified rcu_data
376	348	* structure has spent some time in an extended quiescent state since
377	349	* rcu_dynticks_snap() returned the specified snapshot.
378	350	*/
379		-static bool rcu_dynticks_in_eqs_since(struct rcu_dynticks *rdtp, int snap)
	351	+static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap)
380	352	{
381		- return snap != rcu_dynticks_snap(rdtp);
	353	+ return snap != rcu_dynticks_snap(rdp);
	354	+}
	355	+
	356	+/*
	357	+ * Return true if the referenced integer is zero while the specified
	358	+ * CPU remains within a single extended quiescent state.
	359	+ */
	360	+bool rcu_dynticks_zero_in_eqs(int cpu, int *vp)
	361	+{
	362	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
	363	+ int snap;
	364	+
	365	+ // If not quiescent, force back to earlier extended quiescent state.
	366	+ snap = atomic_read(&rdp->dynticks) & ~(RCU_DYNTICK_CTRL_MASK \|
	367	+ RCU_DYNTICK_CTRL_CTR);
	368	+
	369	+ smp_rmb(); // Order ->dynticks and *vp reads.
	370	+ if (READ_ONCE(*vp))
	371	+ return false; // Non-zero, so report failure;
	372	+ smp_rmb(); // Order *vp read and ->dynticks re-read.
	373	+
	374	+ // If still in the same extended quiescent state, we are good!
	375	+ return snap == (atomic_read(&rdp->dynticks) & ~RCU_DYNTICK_CTRL_MASK);
382	376	}
383	377
384	378	/*
..	..	@@ -392,14 +386,17 @@
392	386	{
393	387	int old;
394	388	int new;
395		- struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
	389	+ int new_old;
	390	+ struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
396	391
	392	+ new_old = atomic_read(&rdp->dynticks);
397	393	do {
398		- old = atomic_read(&rdtp->dynticks);
	394	+ old = new_old;
399	395	if (old & RCU_DYNTICK_CTRL_CTR)
400	396	return false;
401	397	new = old \| RCU_DYNTICK_CTRL_MASK;
402		- } while (atomic_cmpxchg(&rdtp->dynticks, old, new) != old);
	398	+ new_old = atomic_cmpxchg(&rdp->dynticks, old, new);
	399	+ } while (new_old != old);
403	400	return true;
404	401	}
405	402
..	..	@@ -414,105 +411,128 @@
414	411	*
415	412	* The caller must have disabled interrupts and must not be idle.
416	413	*/
417		-static void rcu_momentary_dyntick_idle(void)
	414	+notrace void rcu_momentary_dyntick_idle(void)
418	415	{
419		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
420	416	int special;
421	417
422		- raw_cpu_write(rcu_dynticks.rcu_need_heavy_qs, false);
423		- special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
	418	+ raw_cpu_write(rcu_data.rcu_need_heavy_qs, false);
	419	+ special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR,
	420	+ &this_cpu_ptr(&rcu_data)->dynticks);
424	421	/* It is illegal to call this from idle state. */
425	422	WARN_ON_ONCE(!(special & RCU_DYNTICK_CTRL_CTR));
	423	+ rcu_preempt_deferred_qs(current);
426	424	}
	425	+EXPORT_SYMBOL_GPL(rcu_momentary_dyntick_idle);
427	426
428		-/*
429		- * Note a context switch. This is a quiescent state for RCU-sched,
430		- * and requires special handling for preemptible RCU.
431		- * The caller must have disabled interrupts.
432		- */
433		-void rcu_note_context_switch(bool preempt)
434		-{
435		- barrier(); /* Avoid RCU read-side critical sections leaking down. */
436		- trace_rcu_utilization(TPS("Start context switch"));
437		- rcu_sched_qs();
438		- rcu_preempt_note_context_switch(preempt);
439		- /* Load rcu_urgent_qs before other flags. */
440		- if (!smp_load_acquire(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs)))
441		- goto out;
442		- this_cpu_write(rcu_dynticks.rcu_urgent_qs, false);
443		- if (unlikely(raw_cpu_read(rcu_dynticks.rcu_need_heavy_qs)))
444		- rcu_momentary_dyntick_idle();
445		- this_cpu_inc(rcu_dynticks.rcu_qs_ctr);
446		- if (!preempt)
447		- rcu_tasks_qs(current);
448		-out:
449		- trace_rcu_utilization(TPS("End context switch"));
450		- barrier(); /* Avoid RCU read-side critical sections leaking up. */
451		-}
452		-EXPORT_SYMBOL_GPL(rcu_note_context_switch);
453		-
454		-/*
455		- * Register a quiescent state for all RCU flavors. If there is an
456		- * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
457		- * dyntick-idle quiescent state visible to other CPUs (but only for those
458		- * RCU flavors in desperate need of a quiescent state, which will normally
459		- * be none of them). Either way, do a lightweight quiescent state for
460		- * all RCU flavors.
	427	+/**
	428	+ * rcu_is_cpu_rrupt_from_idle - see if 'interrupted' from idle
461	429	*
462		- * The barrier() calls are redundant in the common case when this is
463		- * called externally, but just in case this is called from within this
464		- * file.
	430	+ * If the current CPU is idle and running at a first-level (not nested)
	431	+ * interrupt, or directly, from idle, return true.
465	432	*
	433	+ * The caller must have at least disabled IRQs.
466	434	*/
467		-void rcu_all_qs(void)
	435	+static int rcu_is_cpu_rrupt_from_idle(void)
468	436	{
469		- unsigned long flags;
	437	+ long nesting;
470	438
471		- if (!raw_cpu_read(rcu_dynticks.rcu_urgent_qs))
472		- return;
473		- preempt_disable();
474		- /* Load rcu_urgent_qs before other flags. */
475		- if (!smp_load_acquire(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs))) {
476		- preempt_enable();
477		- return;
478		- }
479		- this_cpu_write(rcu_dynticks.rcu_urgent_qs, false);
480		- barrier(); /* Avoid RCU read-side critical sections leaking down. */
481		- if (unlikely(raw_cpu_read(rcu_dynticks.rcu_need_heavy_qs))) {
482		- local_irq_save(flags);
483		- rcu_momentary_dyntick_idle();
484		- local_irq_restore(flags);
485		- }
486		- if (unlikely(raw_cpu_read(rcu_sched_data.cpu_no_qs.b.exp)))
487		- rcu_sched_qs();
488		- this_cpu_inc(rcu_dynticks.rcu_qs_ctr);
489		- barrier(); /* Avoid RCU read-side critical sections leaking up. */
490		- preempt_enable();
	439	+ /*
	440	+ * Usually called from the tick; but also used from smp_function_call()
	441	+ * for expedited grace periods. This latter can result in running from
	442	+ * the idle task, instead of an actual IPI.
	443	+ */
	444	+ lockdep_assert_irqs_disabled();
	445	+
	446	+ /* Check for counter underflows */
	447	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) < 0,
	448	+ "RCU dynticks_nesting counter underflow!");
	449	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0,
	450	+ "RCU dynticks_nmi_nesting counter underflow/zero!");
	451	+
	452	+ /* Are we at first interrupt nesting level? */
	453	+ nesting = __this_cpu_read(rcu_data.dynticks_nmi_nesting);
	454	+ if (nesting > 1)
	455	+ return false;
	456	+
	457	+ /*
	458	+ * If we're not in an interrupt, we must be in the idle task!
	459	+ */
	460	+ WARN_ON_ONCE(!nesting && !is_idle_task(current));
	461	+
	462	+ /* Does CPU appear to be idle from an RCU standpoint? */
	463	+ return __this_cpu_read(rcu_data.dynticks_nesting) == 0;
491	464	}
492		-EXPORT_SYMBOL_GPL(rcu_all_qs);
493	465
494		-#define DEFAULT_RCU_BLIMIT 10 /* Maximum callbacks per rcu_do_batch. */
	466	+#define DEFAULT_RCU_BLIMIT (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 1000 : 10)
	467	+ // Maximum callbacks per rcu_do_batch ...
	468	+#define DEFAULT_MAX_RCU_BLIMIT 10000 // ... even during callback flood.
495	469	static long blimit = DEFAULT_RCU_BLIMIT;
496		-#define DEFAULT_RCU_QHIMARK 10000 /* If this many pending, ignore blimit. */
	470	+#define DEFAULT_RCU_QHIMARK 10000 // If this many pending, ignore blimit.
497	471	static long qhimark = DEFAULT_RCU_QHIMARK;
498		-#define DEFAULT_RCU_QLOMARK 100 /* Once only this many pending, use blimit. */
	472	+#define DEFAULT_RCU_QLOMARK 100 // Once only this many pending, use blimit.
499	473	static long qlowmark = DEFAULT_RCU_QLOMARK;
	474	+#define DEFAULT_RCU_QOVLD_MULT 2
	475	+#define DEFAULT_RCU_QOVLD (DEFAULT_RCU_QOVLD_MULT * DEFAULT_RCU_QHIMARK)
	476	+static long qovld = DEFAULT_RCU_QOVLD; // If this many pending, hammer QS.
	477	+static long qovld_calc = -1; // No pre-initialization lock acquisitions!
500	478
501	479	module_param(blimit, long, 0444);
502	480	module_param(qhimark, long, 0444);
503	481	module_param(qlowmark, long, 0444);
	482	+module_param(qovld, long, 0444);
504	483
505		-static ulong jiffies_till_first_fqs = ULONG_MAX;
	484	+static ulong jiffies_till_first_fqs = IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 0 : ULONG_MAX;
506	485	static ulong jiffies_till_next_fqs = ULONG_MAX;
507	486	static bool rcu_kick_kthreads;
	487	+static int rcu_divisor = 7;
	488	+module_param(rcu_divisor, int, 0644);
	489	+
	490	+/* Force an exit from rcu_do_batch() after 3 milliseconds. */
	491	+static long rcu_resched_ns = 3 * NSEC_PER_MSEC;
	492	+module_param(rcu_resched_ns, long, 0644);
	493	+
	494	+/*
	495	+ * How long the grace period must be before we start recruiting
	496	+ * quiescent-state help from rcu_note_context_switch().
	497	+ */
	498	+static ulong jiffies_till_sched_qs = ULONG_MAX;
	499	+module_param(jiffies_till_sched_qs, ulong, 0444);
	500	+static ulong jiffies_to_sched_qs; /* See adjust_jiffies_till_sched_qs(). */
	501	+module_param(jiffies_to_sched_qs, ulong, 0444); /* Display only! */
	502	+
	503	+/*
	504	+ * Make sure that we give the grace-period kthread time to detect any
	505	+ * idle CPUs before taking active measures to force quiescent states.
	506	+ * However, don't go below 100 milliseconds, adjusted upwards for really
	507	+ * large systems.
	508	+ */
	509	+static void adjust_jiffies_till_sched_qs(void)
	510	+{
	511	+ unsigned long j;
	512	+
	513	+ /* If jiffies_till_sched_qs was specified, respect the request. */
	514	+ if (jiffies_till_sched_qs != ULONG_MAX) {
	515	+ WRITE_ONCE(jiffies_to_sched_qs, jiffies_till_sched_qs);
	516	+ return;
	517	+ }
	518	+ /* Otherwise, set to third fqs scan, but bound below on large system. */
	519	+ j = READ_ONCE(jiffies_till_first_fqs) +
	520	+ 2 * READ_ONCE(jiffies_till_next_fqs);
	521	+ if (j < HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV)
	522	+ j = HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
	523	+ pr_info("RCU calculated value of scheduler-enlistment delay is %ld jiffies.\n", j);
	524	+ WRITE_ONCE(jiffies_to_sched_qs, j);
	525	+}
508	526
509	527	static int param_set_first_fqs_jiffies(const char val, const struct kernel_param kp)
510	528	{
511	529	ulong j;
512	530	int ret = kstrtoul(val, 0, &j);
513	531
514		- if (!ret)
	532	+ if (!ret) {
515	533	WRITE_ONCE((ulong )kp->arg, (j > HZ) ? HZ : j);
	534	+ adjust_jiffies_till_sched_qs();
	535	+ }
516	536	return ret;
517	537	}
518	538
..	..	@@ -521,8 +541,10 @@
521	541	ulong j;
522	542	int ret = kstrtoul(val, 0, &j);
523	543
524		- if (!ret)
	544	+ if (!ret) {
525	545	WRITE_ONCE((ulong )kp->arg, (j > HZ) ? HZ : (j ?: 1));
	546	+ adjust_jiffies_till_sched_qs();
	547	+ }
526	548	return ret;
527	549	}
528	550
..	..	@@ -540,43 +562,17 @@
540	562	module_param_cb(jiffies_till_next_fqs, &next_fqs_jiffies_ops, &jiffies_till_next_fqs, 0644);
541	563	module_param(rcu_kick_kthreads, bool, 0644);
542	564
543		-/*
544		- * How long the grace period must be before we start recruiting
545		- * quiescent-state help from rcu_note_context_switch().
546		- */
547		-static ulong jiffies_till_sched_qs = HZ / 10;
548		-module_param(jiffies_till_sched_qs, ulong, 0444);
549		-
550		-static void force_qs_rnp(struct rcu_state rsp, int (f)(struct rcu_data *rsp));
551		-static void force_quiescent_state(struct rcu_state *rsp);
552		-static int rcu_pending(void);
	565	+static void force_qs_rnp(int (f)(struct rcu_data rdp));
	566	+static int rcu_pending(int user);
553	567
554	568	/*
555	569	* Return the number of RCU GPs completed thus far for debug & stats.
556	570	*/
557	571	unsigned long rcu_get_gp_seq(void)
558	572	{
559		- return READ_ONCE(rcu_state_p->gp_seq);
	573	+ return READ_ONCE(rcu_state.gp_seq);
560	574	}
561	575	EXPORT_SYMBOL_GPL(rcu_get_gp_seq);
562		-
563		-/*
564		- * Return the number of RCU-sched GPs completed thus far for debug & stats.
565		- */
566		-unsigned long rcu_sched_get_gp_seq(void)
567		-{
568		- return READ_ONCE(rcu_sched_state.gp_seq);
569		-}
570		-EXPORT_SYMBOL_GPL(rcu_sched_get_gp_seq);
571		-
572		-/*
573		- * Return the number of RCU-bh GPs completed thus far for debug & stats.
574		- */
575		-unsigned long rcu_bh_get_gp_seq(void)
576		-{
577		- return READ_ONCE(rcu_bh_state.gp_seq);
578		-}
579		-EXPORT_SYMBOL_GPL(rcu_bh_get_gp_seq);
580	576
581	577	/*
582	578	* Return the number of RCU expedited batches completed thus far for
..	..	@@ -586,82 +582,17 @@
586	582	*/
587	583	unsigned long rcu_exp_batches_completed(void)
588	584	{
589		- return rcu_state_p->expedited_sequence;
	585	+ return rcu_state.expedited_sequence;
590	586	}
591	587	EXPORT_SYMBOL_GPL(rcu_exp_batches_completed);
592	588
593	589	/*
594		- * Return the number of RCU-sched expedited batches completed thus far
595		- * for debug & stats. Similar to rcu_exp_batches_completed().
	590	+ * Return the root node of the rcu_state structure.
596	591	*/
597		-unsigned long rcu_exp_batches_completed_sched(void)
	592	+static struct rcu_node *rcu_get_root(void)
598	593	{
599		- return rcu_sched_state.expedited_sequence;
	594	+ return &rcu_state.node[0];
600	595	}
601		-EXPORT_SYMBOL_GPL(rcu_exp_batches_completed_sched);
602		-
603		-/*
604		- * Force a quiescent state.
605		- */
606		-void rcu_force_quiescent_state(void)
607		-{
608		- force_quiescent_state(rcu_state_p);
609		-}
610		-EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
611		-
612		-/*
613		- * Force a quiescent state for RCU BH.
614		- */
615		-void rcu_bh_force_quiescent_state(void)
616		-{
617		- force_quiescent_state(&rcu_bh_state);
618		-}
619		-EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state);
620		-
621		-/*
622		- * Force a quiescent state for RCU-sched.
623		- */
624		-void rcu_sched_force_quiescent_state(void)
625		-{
626		- force_quiescent_state(&rcu_sched_state);
627		-}
628		-EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state);
629		-
630		-/*
631		- * Show the state of the grace-period kthreads.
632		- */
633		-void show_rcu_gp_kthreads(void)
634		-{
635		- int cpu;
636		- struct rcu_data *rdp;
637		- struct rcu_node *rnp;
638		- struct rcu_state *rsp;
639		-
640		- for_each_rcu_flavor(rsp) {
641		- pr_info("%s: wait state: %d ->state: %#lx\n",
642		- rsp->name, rsp->gp_state, rsp->gp_kthread->state);
643		- rcu_for_each_node_breadth_first(rsp, rnp) {
644		- if (ULONG_CMP_GE(rsp->gp_seq, rnp->gp_seq_needed))
645		- continue;
646		- pr_info("\trcu_node %d:%d ->gp_seq %lu ->gp_seq_needed %lu\n",
647		- rnp->grplo, rnp->grphi, rnp->gp_seq,
648		- rnp->gp_seq_needed);
649		- if (!rcu_is_leaf_node(rnp))
650		- continue;
651		- for_each_leaf_node_possible_cpu(rnp, cpu) {
652		- rdp = per_cpu_ptr(rsp->rda, cpu);
653		- if (rdp->gpwrap \|\|
654		- ULONG_CMP_GE(rsp->gp_seq,
655		- rdp->gp_seq_needed))
656		- continue;
657		- pr_info("\tcpu %d ->gp_seq_needed %lu\n",
658		- cpu, rdp->gp_seq_needed);
659		- }
660		- }
661		- /* sched_show_task(rsp->gp_kthread); */
662		- }
663		-}
664		-EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads);
665	596
666	597	/*
667	598	* Send along grace-period-related data for rcutorture diagnostics.
..	..	@@ -669,35 +600,16 @@
669	600	void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags,
670	601	unsigned long *gp_seq)
671	602	{
672		- struct rcu_state *rsp = NULL;
673		-
674	603	switch (test_type) {
675	604	case RCU_FLAVOR:
676		- rsp = rcu_state_p;
677		- break;
678		- case RCU_BH_FLAVOR:
679		- rsp = &rcu_bh_state;
680		- break;
681		- case RCU_SCHED_FLAVOR:
682		- rsp = &rcu_sched_state;
	605	+ *flags = READ_ONCE(rcu_state.gp_flags);
	606	+ *gp_seq = rcu_seq_current(&rcu_state.gp_seq);
683	607	break;
684	608	default:
685	609	break;
686	610	}
687		- if (rsp == NULL)
688		- return;
689		- *flags = READ_ONCE(rsp->gp_flags);
690		- *gp_seq = rcu_seq_current(&rsp->gp_seq);
691	611	}
692	612	EXPORT_SYMBOL_GPL(rcutorture_get_gp_data);
693		-
694		-/*
695		- * Return the root node of the specified rcu_state structure.
696		- */
697		-static struct rcu_node rcu_get_root(struct rcu_state rsp)
698		-{
699		- return &rsp->node[0];
700		-}
701	613
702	614	/*
703	615	* Enter an RCU extended quiescent state, which can be either the
..	..	@@ -707,31 +619,36 @@
707	619	* the possibility of usermode upcalls having messed up our count
708	620	* of interrupt nesting level during the prior busy period.
709	621	*/
710		-static void rcu_eqs_enter(bool user)
	622	+static noinstr void rcu_eqs_enter(bool user)
711	623	{
712		- struct rcu_state *rsp;
713		- struct rcu_data *rdp;
714		- struct rcu_dynticks *rdtp;
	624	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
715	625
716		- rdtp = this_cpu_ptr(&rcu_dynticks);
717		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0);
	626	+ WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE);
	627	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, 0);
718	628	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
719		- rdtp->dynticks_nesting == 0);
720		- if (rdtp->dynticks_nesting != 1) {
721		- rdtp->dynticks_nesting--;
	629	+ rdp->dynticks_nesting == 0);
	630	+ if (rdp->dynticks_nesting != 1) {
	631	+ // RCU will still be watching, so just do accounting and leave.
	632	+ rdp->dynticks_nesting--;
722	633	return;
723	634	}
724	635
725	636	lockdep_assert_irqs_disabled();
726		- trace_rcu_dyntick(TPS("Start"), rdtp->dynticks_nesting, 0, rdtp->dynticks);
	637	+ instrumentation_begin();
	638	+ trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks));
727	639	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
728		- for_each_rcu_flavor(rsp) {
729		- rdp = this_cpu_ptr(rsp->rda);
730		- do_nocb_deferred_wakeup(rdp);
731		- }
	640	+ rdp = this_cpu_ptr(&rcu_data);
732	641	rcu_prepare_for_idle();
733		- WRITE_ONCE(rdtp->dynticks_nesting, 0); /* Avoid irq-access tearing. */
	642	+ rcu_preempt_deferred_qs(current);
	643	+
	644	+ // instrumentation for the noinstr rcu_dynticks_eqs_enter()
	645	+ instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
	646	+
	647	+ instrumentation_end();
	648	+ WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */
	649	+ // RCU is watching here ...
734	650	rcu_dynticks_eqs_enter();
	651	+ // ... but is no longer watching here.
735	652	rcu_dynticks_task_enter();
736	653	}
737	654
..	..	@@ -751,6 +668,7 @@
751	668	lockdep_assert_irqs_disabled();
752	669	rcu_eqs_enter(false);
753	670	}
	671	+EXPORT_SYMBOL_GPL(rcu_idle_enter);
754	672
755	673	#ifdef CONFIG_NO_HZ_FULL
756	674	/**
..	..	@@ -764,9 +682,16 @@
764	682	* If you add or remove a call to rcu_user_enter(), be sure to test with
765	683	* CONFIG_RCU_EQS_DEBUG=y.
766	684	*/
767		-void rcu_user_enter(void)
	685	+noinstr void rcu_user_enter(void)
768	686	{
	687	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
	688	+
769	689	lockdep_assert_irqs_disabled();
	690	+
	691	+ instrumentation_begin();
	692	+ do_nocb_deferred_wakeup(rdp);
	693	+ instrumentation_end();
	694	+
770	695	rcu_eqs_enter(true);
771	696	}
772	697	#endif /* CONFIG_NO_HZ_FULL */
..	..	@@ -775,40 +700,56 @@
775	700	* rcu_nmi_exit - inform RCU of exit from NMI context
776	701	*
777	702	* If we are returning from the outermost NMI handler that interrupted an
778		- * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
	703	+ * RCU-idle period, update rdp->dynticks and rdp->dynticks_nmi_nesting
779	704	* to let the RCU grace-period handling know that the CPU is back to
780	705	* being RCU-idle.
781	706	*
782	707	* If you add or remove a call to rcu_nmi_exit(), be sure to test
783	708	* with CONFIG_RCU_EQS_DEBUG=y.
784	709	*/
785		-void rcu_nmi_exit(void)
	710	+noinstr void rcu_nmi_exit(void)
786	711	{
787		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	712	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
788	713
	714	+ instrumentation_begin();
789	715	/*
790	716	* Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
791	717	* (We are exiting an NMI handler, so RCU better be paying attention
792	718	* to us!)
793	719	*/
794		- WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);
	720	+ WARN_ON_ONCE(rdp->dynticks_nmi_nesting <= 0);
795	721	WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
796	722
797	723	/*
798	724	* If the nesting level is not 1, the CPU wasn't RCU-idle, so
799	725	* leave it in non-RCU-idle state.
800	726	*/
801		- if (rdtp->dynticks_nmi_nesting != 1) {
802		- trace_rcu_dyntick(TPS("--="), rdtp->dynticks_nmi_nesting, rdtp->dynticks_nmi_nesting - 2, rdtp->dynticks);
803		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* No store tearing. */
804		- rdtp->dynticks_nmi_nesting - 2);
	727	+ if (rdp->dynticks_nmi_nesting != 1) {
	728	+ trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2,
	729	+ atomic_read(&rdp->dynticks));
	730	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */
	731	+ rdp->dynticks_nmi_nesting - 2);
	732	+ instrumentation_end();
805	733	return;
806	734	}
807	735
808	736	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
809		- trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, rdtp->dynticks);
810		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
	737	+ trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks));
	738	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
	739	+
	740	+ if (!in_nmi())
	741	+ rcu_prepare_for_idle();
	742	+
	743	+ // instrumentation for the noinstr rcu_dynticks_eqs_enter()
	744	+ instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
	745	+ instrumentation_end();
	746	+
	747	+ // RCU is watching here ...
811	748	rcu_dynticks_eqs_enter();
	749	+ // ... but is no longer watching here.
	750	+
	751	+ if (!in_nmi())
	752	+ rcu_dynticks_task_enter();
812	753	}
813	754
814	755	/**
..	..	@@ -830,17 +771,51 @@
830	771	* If you add or remove a call to rcu_irq_exit(), be sure to test with
831	772	* CONFIG_RCU_EQS_DEBUG=y.
832	773	*/
833		-void rcu_irq_exit(void)
	774	+void noinstr rcu_irq_exit(void)
834	775	{
835		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
836		-
837	776	lockdep_assert_irqs_disabled();
838		- if (rdtp->dynticks_nmi_nesting == 1)
839		- rcu_prepare_for_idle();
840	777	rcu_nmi_exit();
841		- if (rdtp->dynticks_nmi_nesting == 0)
842		- rcu_dynticks_task_enter();
843	778	}
	779	+
	780	+/**
	781	+ * rcu_irq_exit_preempt - Inform RCU that current CPU is exiting irq
	782	+ * towards in kernel preemption
	783	+ *
	784	+ * Same as rcu_irq_exit() but has a sanity check that scheduling is safe
	785	+ * from RCU point of view. Invoked from return from interrupt before kernel
	786	+ * preemption.
	787	+ */
	788	+void rcu_irq_exit_preempt(void)
	789	+{
	790	+ lockdep_assert_irqs_disabled();
	791	+ rcu_nmi_exit();
	792	+
	793	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0,
	794	+ "RCU dynticks_nesting counter underflow/zero!");
	795	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) !=
	796	+ DYNTICK_IRQ_NONIDLE,
	797	+ "Bad RCU dynticks_nmi_nesting counter\n");
	798	+ RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
	799	+ "RCU in extended quiescent state!");
	800	+}
	801	+
	802	+#ifdef CONFIG_PROVE_RCU
	803	+/**
	804	+ * rcu_irq_exit_check_preempt - Validate that scheduling is possible
	805	+ */
	806	+void rcu_irq_exit_check_preempt(void)
	807	+{
	808	+ lockdep_assert_irqs_disabled();
	809	+
	810	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0,
	811	+ "RCU dynticks_nesting counter underflow/zero!");
	812	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) !=
	813	+ DYNTICK_IRQ_NONIDLE,
	814	+ "Bad RCU dynticks_nmi_nesting counter\n");
	815	+ RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
	816	+ "RCU in extended quiescent state!");
	817	+}
	818	+#endif /* #ifdef CONFIG_PROVE_RCU */
844	819
845	820	/*
846	821	* Wrapper for rcu_irq_exit() where interrupts are enabled.
..	..	@@ -865,26 +840,36 @@
865	840	* allow for the possibility of usermode upcalls messing up our count of
866	841	* interrupt nesting level during the busy period that is just now starting.
867	842	*/
868		-static void rcu_eqs_exit(bool user)
	843	+static void noinstr rcu_eqs_exit(bool user)
869	844	{
870		- struct rcu_dynticks *rdtp;
	845	+ struct rcu_data *rdp;
871	846	long oldval;
872	847
873	848	lockdep_assert_irqs_disabled();
874		- rdtp = this_cpu_ptr(&rcu_dynticks);
875		- oldval = rdtp->dynticks_nesting;
	849	+ rdp = this_cpu_ptr(&rcu_data);
	850	+ oldval = rdp->dynticks_nesting;
876	851	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
877	852	if (oldval) {
878		- rdtp->dynticks_nesting++;
	853	+ // RCU was already watching, so just do accounting and leave.
	854	+ rdp->dynticks_nesting++;
879	855	return;
880	856	}
881	857	rcu_dynticks_task_exit();
	858	+ // RCU is not watching here ...
882	859	rcu_dynticks_eqs_exit();
	860	+ // ... but is watching here.
	861	+ instrumentation_begin();
	862	+
	863	+ // instrumentation for the noinstr rcu_dynticks_eqs_exit()
	864	+ instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
	865	+
883	866	rcu_cleanup_after_idle();
884		- trace_rcu_dyntick(TPS("End"), rdtp->dynticks_nesting, 1, rdtp->dynticks);
	867	+ trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks));
885	868	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
886		- WRITE_ONCE(rdtp->dynticks_nesting, 1);
887		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
	869	+ WRITE_ONCE(rdp->dynticks_nesting, 1);
	870	+ WARN_ON_ONCE(rdp->dynticks_nmi_nesting);
	871	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
	872	+ instrumentation_end();
888	873	}
889	874
890	875	/**
..	..	@@ -904,6 +889,7 @@
904	889	rcu_eqs_exit(false);
905	890	local_irq_restore(flags);
906	891	}
	892	+EXPORT_SYMBOL_GPL(rcu_idle_exit);
907	893
908	894	#ifdef CONFIG_NO_HZ_FULL
909	895	/**
..	..	@@ -915,17 +901,78 @@
915	901	* If you add or remove a call to rcu_user_exit(), be sure to test with
916	902	* CONFIG_RCU_EQS_DEBUG=y.
917	903	*/
918		-void rcu_user_exit(void)
	904	+void noinstr rcu_user_exit(void)
919	905	{
920	906	rcu_eqs_exit(1);
	907	+}
	908	+
	909	+/**
	910	+ * __rcu_irq_enter_check_tick - Enable scheduler tick on CPU if RCU needs it.
	911	+ *
	912	+ * The scheduler tick is not normally enabled when CPUs enter the kernel
	913	+ * from nohz_full userspace execution. After all, nohz_full userspace
	914	+ * execution is an RCU quiescent state and the time executing in the kernel
	915	+ * is quite short. Except of course when it isn't. And it is not hard to
	916	+ * cause a large system to spend tens of seconds or even minutes looping
	917	+ * in the kernel, which can cause a number of problems, include RCU CPU
	918	+ * stall warnings.
	919	+ *
	920	+ * Therefore, if a nohz_full CPU fails to report a quiescent state
	921	+ * in a timely manner, the RCU grace-period kthread sets that CPU's
	922	+ * ->rcu_urgent_qs flag with the expectation that the next interrupt or
	923	+ * exception will invoke this function, which will turn on the scheduler
	924	+ * tick, which will enable RCU to detect that CPU's quiescent states,
	925	+ * for example, due to cond_resched() calls in CONFIG_PREEMPT=n kernels.
	926	+ * The tick will be disabled once a quiescent state is reported for
	927	+ * this CPU.
	928	+ *
	929	+ * Of course, in carefully tuned systems, there might never be an
	930	+ * interrupt or exception. In that case, the RCU grace-period kthread
	931	+ * will eventually cause one to happen. However, in less carefully
	932	+ * controlled environments, this function allows RCU to get what it
	933	+ * needs without creating otherwise useless interruptions.
	934	+ */
	935	+void __rcu_irq_enter_check_tick(void)
	936	+{
	937	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
	938	+
	939	+ // If we're here from NMI there's nothing to do.
	940	+ if (in_nmi())
	941	+ return;
	942	+
	943	+ RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
	944	+ "Illegal rcu_irq_enter_check_tick() from extended quiescent state");
	945	+
	946	+ if (!tick_nohz_full_cpu(rdp->cpu) \|\|
	947	+ !READ_ONCE(rdp->rcu_urgent_qs) \|\|
	948	+ READ_ONCE(rdp->rcu_forced_tick)) {
	949	+ // RCU doesn't need nohz_full help from this CPU, or it is
	950	+ // already getting that help.
	951	+ return;
	952	+ }
	953	+
	954	+ // We get here only when not in an extended quiescent state and
	955	+ // from interrupts (as opposed to NMIs). Therefore, (1) RCU is
	956	+ // already watching and (2) The fact that we are in an interrupt
	957	+ // handler and that the rcu_node lock is an irq-disabled lock
	958	+ // prevents self-deadlock. So we can safely recheck under the lock.
	959	+ // Note that the nohz_full state currently cannot change.
	960	+ raw_spin_lock_rcu_node(rdp->mynode);
	961	+ if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {
	962	+ // A nohz_full CPU is in the kernel and RCU needs a
	963	+ // quiescent state. Turn on the tick!
	964	+ WRITE_ONCE(rdp->rcu_forced_tick, true);
	965	+ tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
	966	+ }
	967	+ raw_spin_unlock_rcu_node(rdp->mynode);
921	968	}
922	969	#endif /* CONFIG_NO_HZ_FULL */
923	970
924	971	/**
925	972	* rcu_nmi_enter - inform RCU of entry to NMI context
926	973	*
927		- * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
928		- * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
	974	+ * If the CPU was idle from RCU's viewpoint, update rdp->dynticks and
	975	+ * rdp->dynticks_nmi_nesting to let the RCU grace-period handling know
929	976	* that the CPU is active. This implementation permits nested NMIs, as
930	977	* long as the nesting level does not overflow an int. (You will probably
931	978	* run out of stack space first.)
..	..	@@ -933,13 +980,13 @@
933	980	* If you add or remove a call to rcu_nmi_enter(), be sure to test
934	981	* with CONFIG_RCU_EQS_DEBUG=y.
935	982	*/
936		-void rcu_nmi_enter(void)
	983	+noinstr void rcu_nmi_enter(void)
937	984	{
938		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
939	985	long incby = 2;
	986	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
940	987
941	988	/* Complain about underflow. */
942		- WARN_ON_ONCE(rdtp->dynticks_nmi_nesting < 0);
	989	+ WARN_ON_ONCE(rdp->dynticks_nmi_nesting < 0);
943	990
944	991	/*
945	992	* If idle from RCU viewpoint, atomically increment ->dynticks
..	..	@@ -950,14 +997,40 @@
950	997	* period (observation due to Andy Lutomirski).
951	998	*/
952	999	if (rcu_dynticks_curr_cpu_in_eqs()) {
	1000	+
	1001	+ if (!in_nmi())
	1002	+ rcu_dynticks_task_exit();
	1003	+
	1004	+ // RCU is not watching here ...
953	1005	rcu_dynticks_eqs_exit();
	1006	+ // ... but is watching here.
	1007	+
	1008	+ if (!in_nmi()) {
	1009	+ instrumentation_begin();
	1010	+ rcu_cleanup_after_idle();
	1011	+ instrumentation_end();
	1012	+ }
	1013	+
	1014	+ instrumentation_begin();
	1015	+ // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs()
	1016	+ instrument_atomic_read(&rdp->dynticks, sizeof(rdp->dynticks));
	1017	+ // instrumentation for the noinstr rcu_dynticks_eqs_exit()
	1018	+ instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
	1019	+
954	1020	incby = 1;
	1021	+ } else if (!in_nmi()) {
	1022	+ instrumentation_begin();
	1023	+ rcu_irq_enter_check_tick();
	1024	+ } else {
	1025	+ instrumentation_begin();
955	1026	}
	1027	+
956	1028	trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
957		- rdtp->dynticks_nmi_nesting,
958		- rdtp->dynticks_nmi_nesting + incby, rdtp->dynticks);
959		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* Prevent store tearing. */
960		- rdtp->dynticks_nmi_nesting + incby);
	1029	+ rdp->dynticks_nmi_nesting,
	1030	+ rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks));
	1031	+ instrumentation_end();
	1032	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, /* Prevent store tearing. */
	1033	+ rdp->dynticks_nmi_nesting + incby);
961	1034	barrier();
962	1035	}
963	1036
..	..	@@ -983,16 +1056,10 @@
983	1056	* If you add or remove a call to rcu_irq_enter(), be sure to test with
984	1057	* CONFIG_RCU_EQS_DEBUG=y.
985	1058	*/
986		-void rcu_irq_enter(void)
	1059	+noinstr void rcu_irq_enter(void)
987	1060	{
988		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
989		-
990	1061	lockdep_assert_irqs_disabled();
991		- if (rdtp->dynticks_nmi_nesting == 0)
992		- rcu_dynticks_task_exit();
993	1062	rcu_nmi_enter();
994		- if (rdtp->dynticks_nmi_nesting == 1)
995		- rcu_cleanup_after_idle();
996	1063	}
997	1064
998	1065	/*
..	..	@@ -1010,15 +1077,34 @@
1010	1077	local_irq_restore(flags);
1011	1078	}
1012	1079
	1080	+/*
	1081	+ * If any sort of urgency was applied to the current CPU (for example,
	1082	+ * the scheduler-clock interrupt was enabled on a nohz_full CPU) in order
	1083	+ * to get to a quiescent state, disable it.
	1084	+ */
	1085	+static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp)
	1086	+{
	1087	+ raw_lockdep_assert_held_rcu_node(rdp->mynode);
	1088	+ WRITE_ONCE(rdp->rcu_urgent_qs, false);
	1089	+ WRITE_ONCE(rdp->rcu_need_heavy_qs, false);
	1090	+ if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) {
	1091	+ tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
	1092	+ WRITE_ONCE(rdp->rcu_forced_tick, false);
	1093	+ }
	1094	+}
	1095	+
1013	1096	/**
1014		- * rcu_is_watching - see if RCU thinks that the current CPU is idle
	1097	+ * rcu_is_watching - see if RCU thinks that the current CPU is not idle
1015	1098	*
1016	1099	* Return true if RCU is watching the running CPU, which means that this
1017	1100	* CPU can safely enter RCU read-side critical sections. In other words,
1018		- * if the current CPU is in its idle loop and is neither in an interrupt
1019		- * or NMI handler, return true.
	1101	+ * if the current CPU is not in its idle loop or is in an interrupt or
	1102	+ * NMI handler, return true.
	1103	+ *
	1104	+ * Make notrace because it can be called by the internal functions of
	1105	+ * ftrace, and making this notrace removes unnecessary recursion calls.
1020	1106	*/
1021		-bool notrace rcu_is_watching(void)
	1107	+notrace bool rcu_is_watching(void)
1022	1108	{
1023	1109	bool ret;
1024	1110
..	..	@@ -1044,7 +1130,7 @@
1044	1130	cpu = task_cpu(t);
1045	1131	if (!task_curr(t))
1046	1132	return; /* This task is not running on that CPU. */
1047		- smp_store_release(per_cpu_ptr(&rcu_dynticks.rcu_urgent_qs, cpu), true);
	1133	+ smp_store_release(per_cpu_ptr(&rcu_data.rcu_urgent_qs, cpu), true);
1048	1134	}
1049	1135
1050	1136	#if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
..	..	@@ -1055,11 +1141,7 @@
1055	1141	* Disable preemption to avoid false positives that could otherwise
1056	1142	* happen due to the current CPU number being sampled, this task being
1057	1143	* preempted, its old CPU being taken offline, resuming on some other CPU,
1058		- * then determining that its old CPU is now offline. Because there are
1059		- * multiple flavors of RCU, and because this function can be called in the
1060		- * midst of updating the flavors while a given CPU coming online or going
1061		- * offline, it is necessary to check all flavors. If any of the flavors
1062		- * believe that given CPU is online, it is considered to be online.
	1144	+ * then determining that its old CPU is now offline.
1063	1145	*
1064	1146	* Disable checking if in an NMI handler because we cannot safely
1065	1147	* report errors from NMI handlers anyway. In addition, it is OK to use
..	..	@@ -1070,38 +1152,21 @@
1070	1152	{
1071	1153	struct rcu_data *rdp;
1072	1154	struct rcu_node *rnp;
1073		- struct rcu_state *rsp;
	1155	+ bool ret = false;
1074	1156
1075	1157	if (in_nmi() \|\| !rcu_scheduler_fully_active)
1076	1158	return true;
1077		- preempt_disable();
1078		- for_each_rcu_flavor(rsp) {
1079		- rdp = this_cpu_ptr(rsp->rda);
1080		- rnp = rdp->mynode;
1081		- if (rdp->grpmask & rcu_rnp_online_cpus(rnp)) {
1082		- preempt_enable();
1083		- return true;
1084		- }
1085		- }
1086		- preempt_enable();
1087		- return false;
	1159	+ preempt_disable_notrace();
	1160	+ rdp = this_cpu_ptr(&rcu_data);
	1161	+ rnp = rdp->mynode;
	1162	+ if (rdp->grpmask & rcu_rnp_online_cpus(rnp))
	1163	+ ret = true;
	1164	+ preempt_enable_notrace();
	1165	+ return ret;
1088	1166	}
1089	1167	EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);
1090	1168
1091	1169	#endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */
1092		-
1093		-/**
1094		- * rcu_is_cpu_rrupt_from_idle - see if idle or immediately interrupted from idle
1095		- *
1096		- * If the current CPU is idle or running at a first-level (not nested)
1097		- * interrupt from idle, return true. The caller must have at least
1098		- * disabled preemption.
1099		- */
1100		-static int rcu_is_cpu_rrupt_from_idle(void)
1101		-{
1102		- return __this_cpu_read(rcu_dynticks.dynticks_nesting) <= 0 &&
1103		- __this_cpu_read(rcu_dynticks.dynticks_nmi_nesting) <= 1;
1104		-}
1105	1170
1106	1171	/*
1107	1172	* We are reporting a quiescent state on behalf of some other CPU, so
..	..	@@ -1127,34 +1192,13 @@
1127	1192	*/
1128	1193	static int dyntick_save_progress_counter(struct rcu_data *rdp)
1129	1194	{
1130		- rdp->dynticks_snap = rcu_dynticks_snap(rdp->dynticks);
	1195	+ rdp->dynticks_snap = rcu_dynticks_snap(rdp);
1131	1196	if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) {
1132		- trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("dti"));
	1197	+ trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
1133	1198	rcu_gpnum_ovf(rdp->mynode, rdp);
1134	1199	return 1;
1135	1200	}
1136	1201	return 0;
1137		-}
1138		-
1139		-/*
1140		- * Handler for the irq_work request posted when a grace period has
1141		- * gone on for too long, but not yet long enough for an RCU CPU
1142		- * stall warning. Set state appropriately, but just complain if
1143		- * there is unexpected state on entry.
1144		- */
1145		-static void rcu_iw_handler(struct irq_work *iwp)
1146		-{
1147		- struct rcu_data *rdp;
1148		- struct rcu_node *rnp;
1149		-
1150		- rdp = container_of(iwp, struct rcu_data, rcu_iw);
1151		- rnp = rdp->mynode;
1152		- raw_spin_lock_rcu_node(rnp);
1153		- if (!WARN_ON_ONCE(!rdp->rcu_iw_pending)) {
1154		- rdp->rcu_iw_gp_seq = rnp->gp_seq;
1155		- rdp->rcu_iw_pending = false;
1156		- }
1157		- raw_spin_unlock_rcu_node(rnp);
1158	1202	}
1159	1203
1160	1204	/*
..	..	@@ -1178,39 +1222,34 @@
1178	1222	* read-side critical section that started before the beginning
1179	1223	* of the current RCU grace period.
1180	1224	*/
1181		- if (rcu_dynticks_in_eqs_since(rdp->dynticks, rdp->dynticks_snap)) {
1182		- trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("dti"));
1183		- rdp->dynticks_fqs++;
	1225	+ if (rcu_dynticks_in_eqs_since(rdp, rdp->dynticks_snap)) {
	1226	+ trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
1184	1227	rcu_gpnum_ovf(rnp, rdp);
1185	1228	return 1;
1186	1229	}
1187	1230
1188	1231	/*
1189		- * Has this CPU encountered a cond_resched() since the beginning
1190		- * of the grace period? For this to be the case, the CPU has to
1191		- * have noticed the current grace period. This might not be the
1192		- * case for nohz_full CPUs looping in the kernel.
	1232	+ * Complain if a CPU that is considered to be offline from RCU's
	1233	+ * perspective has not yet reported a quiescent state. After all,
	1234	+ * the offline CPU should have reported a quiescent state during
	1235	+ * the CPU-offline process, or, failing that, by rcu_gp_init()
	1236	+ * if it ran concurrently with either the CPU going offline or the
	1237	+ * last task on a leaf rcu_node structure exiting its RCU read-side
	1238	+ * critical section while all CPUs corresponding to that structure
	1239	+ * are offline. This added warning detects bugs in any of these
	1240	+ * code paths.
	1241	+ *
	1242	+ * The rcu_node structure's ->lock is held here, which excludes
	1243	+ * the relevant portions the CPU-hotplug code, the grace-period
	1244	+ * initialization code, and the rcu_read_unlock() code paths.
	1245	+ *
	1246	+ * For more detail, please refer to the "Hotplug CPU" section
	1247	+ * of RCU's Requirements documentation.
1193	1248	*/
1194		- jtsq = jiffies_till_sched_qs;
1195		- ruqp = per_cpu_ptr(&rcu_dynticks.rcu_urgent_qs, rdp->cpu);
1196		- if (time_after(jiffies, rdp->rsp->gp_start + jtsq) &&
1197		- READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_dynticks.rcu_qs_ctr, rdp->cpu) &&
1198		- rcu_seq_current(&rdp->gp_seq) == rnp->gp_seq && !rdp->gpwrap) {
1199		- trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("rqc"));
1200		- rcu_gpnum_ovf(rnp, rdp);
1201		- return 1;
1202		- } else if (time_after(jiffies, rdp->rsp->gp_start + jtsq)) {
1203		- /* Load rcu_qs_ctr before store to rcu_urgent_qs. */
1204		- smp_store_release(ruqp, true);
1205		- }
1206		-
1207		- /* If waiting too long on an offline CPU, complain. */
1208		- if (!(rdp->grpmask & rcu_rnp_online_cpus(rnp)) &&
1209		- time_after(jiffies, rdp->rsp->gp_start + HZ)) {
	1249	+ if (WARN_ON_ONCE(!(rdp->grpmask & rcu_rnp_online_cpus(rnp)))) {
1210	1250	bool onl;
1211	1251	struct rcu_node *rnp1;
1212	1252
1213		- WARN_ON(1); /* Offline CPUs are supposed to report QS! */
1214	1253	pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
1215	1254	__func__, rnp->grplo, rnp->grphi, rnp->level,
1216	1255	(long)rnp->gp_seq, (long)rnp->completedqs);
..	..	@@ -1227,43 +1266,63 @@
1227	1266
1228	1267	/*
1229	1268	* A CPU running for an extended time within the kernel can
1230		- * delay RCU grace periods. When the CPU is in NO_HZ_FULL mode,
1231		- * even context-switching back and forth between a pair of
1232		- * in-kernel CPU-bound tasks cannot advance grace periods.
1233		- * So if the grace period is old enough, make the CPU pay attention.
1234		- * Note that the unsynchronized assignments to the per-CPU
1235		- * rcu_need_heavy_qs variable are safe. Yes, setting of
1236		- * bits can be lost, but they will be set again on the next
1237		- * force-quiescent-state pass. So lost bit sets do not result
1238		- * in incorrect behavior, merely in a grace period lasting
1239		- * a few jiffies longer than it might otherwise. Because
1240		- * there are at most four threads involved, and because the
1241		- * updates are only once every few jiffies, the probability of
1242		- * lossage (and thus of slight grace-period extension) is
1243		- * quite low.
	1269	+ * delay RCU grace periods: (1) At age jiffies_to_sched_qs,
	1270	+ * set .rcu_urgent_qs, (2) At age 2*jiffies_to_sched_qs, set
	1271	+ * both .rcu_need_heavy_qs and .rcu_urgent_qs. Note that the
	1272	+ * unsynchronized assignments to the per-CPU rcu_need_heavy_qs
	1273	+ * variable are safe because the assignments are repeated if this
	1274	+ * CPU failed to pass through a quiescent state. This code
	1275	+ * also checks .jiffies_resched in case jiffies_to_sched_qs
	1276	+ * is set way high.
1244	1277	*/
1245		- rnhqp = &per_cpu(rcu_dynticks.rcu_need_heavy_qs, rdp->cpu);
	1278	+ jtsq = READ_ONCE(jiffies_to_sched_qs);
	1279	+ ruqp = per_cpu_ptr(&rcu_data.rcu_urgent_qs, rdp->cpu);
	1280	+ rnhqp = &per_cpu(rcu_data.rcu_need_heavy_qs, rdp->cpu);
1246	1281	if (!READ_ONCE(*rnhqp) &&
1247		- (time_after(jiffies, rdp->rsp->gp_start + jtsq) \|\|
1248		- time_after(jiffies, rdp->rsp->jiffies_resched))) {
	1282	+ (time_after(jiffies, rcu_state.gp_start + jtsq * 2) \|\|
	1283	+ time_after(jiffies, rcu_state.jiffies_resched) \|\|
	1284	+ rcu_state.cbovld)) {
1249	1285	WRITE_ONCE(*rnhqp, true);
1250	1286	/* Store rcu_need_heavy_qs before rcu_urgent_qs. */
1251	1287	smp_store_release(ruqp, true);
1252		- rdp->rsp->jiffies_resched += jtsq; /* Re-enable beating. */
	1288	+ } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) {
	1289	+ WRITE_ONCE(*ruqp, true);
1253	1290	}
1254	1291
1255	1292	/*
1256		- * If more than halfway to RCU CPU stall-warning time, do a
1257		- * resched_cpu() to try to loosen things up a bit. Also check to
1258		- * see if the CPU is getting hammered with interrupts, but only
1259		- * once per grace period, just to keep the IPIs down to a dull roar.
	1293	+ * NO_HZ_FULL CPUs can run in-kernel without rcu_sched_clock_irq!
	1294	+ * The above code handles this, but only for straight cond_resched().
	1295	+ * And some in-kernel loops check need_resched() before calling
	1296	+ * cond_resched(), which defeats the above code for CPUs that are
	1297	+ * running in-kernel with scheduling-clock interrupts disabled.
	1298	+ * So hit them over the head with the resched_cpu() hammer!
1260	1299	*/
1261		- if (jiffies - rdp->rsp->gp_start > rcu_jiffies_till_stall_check() / 2) {
	1300	+ if (tick_nohz_full_cpu(rdp->cpu) &&
	1301	+ (time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) \|\|
	1302	+ rcu_state.cbovld)) {
	1303	+ WRITE_ONCE(*ruqp, true);
1262	1304	resched_cpu(rdp->cpu);
	1305	+ WRITE_ONCE(rdp->last_fqs_resched, jiffies);
	1306	+ }
	1307	+
	1308	+ /*
	1309	+ * If more than halfway to RCU CPU stall-warning time, invoke
	1310	+ * resched_cpu() more frequently to try to loosen things up a bit.
	1311	+ * Also check to see if the CPU is getting hammered with interrupts,
	1312	+ * but only once per grace period, just to keep the IPIs down to
	1313	+ * a dull roar.
	1314	+ */
	1315	+ if (time_after(jiffies, rcu_state.jiffies_resched)) {
	1316	+ if (time_after(jiffies,
	1317	+ READ_ONCE(rdp->last_fqs_resched) + jtsq)) {
	1318	+ resched_cpu(rdp->cpu);
	1319	+ WRITE_ONCE(rdp->last_fqs_resched, jiffies);
	1320	+ }
1263	1321	if (IS_ENABLED(CONFIG_IRQ_WORK) &&
1264	1322	!rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
1265	1323	(rnp->ffmask & rdp->grpmask)) {
1266	1324	init_irq_work(&rdp->rcu_iw, rcu_iw_handler);
	1325	+ atomic_set(&rdp->rcu_iw.flags, IRQ_WORK_HARD_IRQ);
1267	1326	rdp->rcu_iw_pending = true;
1268	1327	rdp->rcu_iw_gp_seq = rnp->gp_seq;
1269	1328	irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
..	..	@@ -1273,317 +1332,13 @@
1273	1332	return 0;
1274	1333	}
1275	1334
1276		-static void record_gp_stall_check_time(struct rcu_state *rsp)
1277		-{
1278		- unsigned long j = jiffies;
1279		- unsigned long j1;
1280		-
1281		- rsp->gp_start = j;
1282		- j1 = rcu_jiffies_till_stall_check();
1283		- /* Record ->gp_start before ->jiffies_stall. */
1284		- smp_store_release(&rsp->jiffies_stall, j + j1); /* ^^^ */
1285		- rsp->jiffies_resched = j + j1 / 2;
1286		- rsp->n_force_qs_gpstart = READ_ONCE(rsp->n_force_qs);
1287		-}
1288		-
1289		-/*
1290		- * Convert a ->gp_state value to a character string.
1291		- */
1292		-static const char *gp_state_getname(short gs)
1293		-{
1294		- if (gs < 0 \|\| gs >= ARRAY_SIZE(gp_state_names))
1295		- return "???";
1296		- return gp_state_names[gs];
1297		-}
1298		-
1299		-/*
1300		- * Complain about starvation of grace-period kthread.
1301		- */
1302		-static void rcu_check_gp_kthread_starvation(struct rcu_state *rsp)
1303		-{
1304		- unsigned long gpa;
1305		- unsigned long j;
1306		-
1307		- j = jiffies;
1308		- gpa = READ_ONCE(rsp->gp_activity);
1309		- if (j - gpa > 2 * HZ) {
1310		- pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#lx ->cpu=%d\n",
1311		- rsp->name, j - gpa,
1312		- (long)rcu_seq_current(&rsp->gp_seq),
1313		- rsp->gp_flags,
1314		- gp_state_getname(rsp->gp_state), rsp->gp_state,
1315		- rsp->gp_kthread ? rsp->gp_kthread->state : ~0,
1316		- rsp->gp_kthread ? task_cpu(rsp->gp_kthread) : -1);
1317		- if (rsp->gp_kthread) {
1318		- pr_err("RCU grace-period kthread stack dump:\n");
1319		- sched_show_task(rsp->gp_kthread);
1320		- wake_up_process(rsp->gp_kthread);
1321		- }
1322		- }
1323		-}
1324		-
1325		-/*
1326		- * Dump stacks of all tasks running on stalled CPUs. First try using
1327		- * NMIs, but fall back to manual remote stack tracing on architectures
1328		- * that don't support NMI-based stack dumps. The NMI-triggered stack
1329		- * traces are more accurate because they are printed by the target CPU.
1330		- */
1331		-static void rcu_dump_cpu_stacks(struct rcu_state *rsp)
1332		-{
1333		- int cpu;
1334		- unsigned long flags;
1335		- struct rcu_node *rnp;
1336		-
1337		- rcu_for_each_leaf_node(rsp, rnp) {
1338		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
1339		- for_each_leaf_node_possible_cpu(rnp, cpu)
1340		- if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu))
1341		- if (!trigger_single_cpu_backtrace(cpu))
1342		- dump_cpu_task(cpu);
1343		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1344		- }
1345		-}
1346		-
1347		-/*
1348		- * If too much time has passed in the current grace period, and if
1349		- * so configured, go kick the relevant kthreads.
1350		- */
1351		-static void rcu_stall_kick_kthreads(struct rcu_state *rsp)
1352		-{
1353		- unsigned long j;
1354		-
1355		- if (!rcu_kick_kthreads)
1356		- return;
1357		- j = READ_ONCE(rsp->jiffies_kick_kthreads);
1358		- if (time_after(jiffies, j) && rsp->gp_kthread &&
1359		- (rcu_gp_in_progress(rsp) \|\| READ_ONCE(rsp->gp_flags))) {
1360		- WARN_ONCE(1, "Kicking %s grace-period kthread\n", rsp->name);
1361		- rcu_ftrace_dump(DUMP_ALL);
1362		- wake_up_process(rsp->gp_kthread);
1363		- WRITE_ONCE(rsp->jiffies_kick_kthreads, j + HZ);
1364		- }
1365		-}
1366		-
1367		-static void panic_on_rcu_stall(void)
1368		-{
1369		- if (sysctl_panic_on_rcu_stall)
1370		- panic("RCU Stall\n");
1371		-}
1372		-
1373		-static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gp_seq)
1374		-{
1375		- int cpu;
1376		- unsigned long flags;
1377		- unsigned long gpa;
1378		- unsigned long j;
1379		- int ndetected = 0;
1380		- struct rcu_node *rnp = rcu_get_root(rsp);
1381		- long totqlen = 0;
1382		-
1383		- /* Kick and suppress, if so configured. */
1384		- rcu_stall_kick_kthreads(rsp);
1385		- if (rcu_cpu_stall_suppress)
1386		- return;
1387		-
1388		- /*
1389		- * OK, time to rat on our buddy...
1390		- * See Documentation/RCU/stallwarn.txt for info on how to debug
1391		- * RCU CPU stall warnings.
1392		- */
1393		- pr_err("INFO: %s detected stalls on CPUs/tasks:", rsp->name);
1394		- print_cpu_stall_info_begin();
1395		- rcu_for_each_leaf_node(rsp, rnp) {
1396		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
1397		- ndetected += rcu_print_task_stall(rnp);
1398		- if (rnp->qsmask != 0) {
1399		- for_each_leaf_node_possible_cpu(rnp, cpu)
1400		- if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
1401		- print_cpu_stall_info(rsp, cpu);
1402		- ndetected++;
1403		- }
1404		- }
1405		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1406		- }
1407		-
1408		- print_cpu_stall_info_end();
1409		- for_each_possible_cpu(cpu)
1410		- totqlen += rcu_segcblist_n_cbs(&per_cpu_ptr(rsp->rda,
1411		- cpu)->cblist);
1412		- pr_cont("(detected by %d, t=%ld jiffies, g=%ld, q=%lu)\n",
1413		- smp_processor_id(), (long)(jiffies - rsp->gp_start),
1414		- (long)rcu_seq_current(&rsp->gp_seq), totqlen);
1415		- if (ndetected) {
1416		- rcu_dump_cpu_stacks(rsp);
1417		-
1418		- /* Complain about tasks blocking the grace period. */
1419		- rcu_print_detail_task_stall(rsp);
1420		- } else {
1421		- if (rcu_seq_current(&rsp->gp_seq) != gp_seq) {
1422		- pr_err("INFO: Stall ended before state dump start\n");
1423		- } else {
1424		- j = jiffies;
1425		- gpa = READ_ONCE(rsp->gp_activity);
1426		- pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n",
1427		- rsp->name, j - gpa, j, gpa,
1428		- jiffies_till_next_fqs,
1429		- rcu_get_root(rsp)->qsmask);
1430		- /* In this case, the current CPU might be at fault. */
1431		- sched_show_task(current);
1432		- }
1433		- }
1434		- /* Rewrite if needed in case of slow consoles. */
1435		- if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall)))
1436		- WRITE_ONCE(rsp->jiffies_stall,
1437		- jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
1438		-
1439		- rcu_check_gp_kthread_starvation(rsp);
1440		-
1441		- atomic_notifier_call_chain(&rcu_stall_notifier_list, 0, NULL);
1442		-
1443		- panic_on_rcu_stall();
1444		-
1445		- force_quiescent_state(rsp); /* Kick them all. */
1446		-}
1447		-
1448		-static void print_cpu_stall(struct rcu_state *rsp)
1449		-{
1450		- int cpu;
1451		- unsigned long flags;
1452		- struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
1453		- struct rcu_node *rnp = rcu_get_root(rsp);
1454		- long totqlen = 0;
1455		-
1456		- /* Kick and suppress, if so configured. */
1457		- rcu_stall_kick_kthreads(rsp);
1458		- if (rcu_cpu_stall_suppress)
1459		- return;
1460		-
1461		- /*
1462		- * OK, time to rat on ourselves...
1463		- * See Documentation/RCU/stallwarn.txt for info on how to debug
1464		- * RCU CPU stall warnings.
1465		- */
1466		- pr_err("INFO: %s self-detected stall on CPU", rsp->name);
1467		- print_cpu_stall_info_begin();
1468		- raw_spin_lock_irqsave_rcu_node(rdp->mynode, flags);
1469		- print_cpu_stall_info(rsp, smp_processor_id());
1470		- raw_spin_unlock_irqrestore_rcu_node(rdp->mynode, flags);
1471		- print_cpu_stall_info_end();
1472		- for_each_possible_cpu(cpu)
1473		- totqlen += rcu_segcblist_n_cbs(&per_cpu_ptr(rsp->rda,
1474		- cpu)->cblist);
1475		- pr_cont(" (t=%lu jiffies g=%ld q=%lu)\n",
1476		- jiffies - rsp->gp_start,
1477		- (long)rcu_seq_current(&rsp->gp_seq), totqlen);
1478		-
1479		- rcu_check_gp_kthread_starvation(rsp);
1480		-
1481		- rcu_dump_cpu_stacks(rsp);
1482		-
1483		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
1484		- /* Rewrite if needed in case of slow consoles. */
1485		- if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall)))
1486		- WRITE_ONCE(rsp->jiffies_stall,
1487		- jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
1488		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1489		-
1490		- panic_on_rcu_stall();
1491		-
1492		- /*
1493		- * Attempt to revive the RCU machinery by forcing a context switch.
1494		- *
1495		- * A context switch would normally allow the RCU state machine to make
1496		- * progress and it could be we're stuck in kernel space without context
1497		- * switches for an entirely unreasonable amount of time.
1498		- */
1499		- resched_cpu(smp_processor_id());
1500		-}
1501		-
1502		-static void check_cpu_stall(struct rcu_state rsp, struct rcu_data rdp)
1503		-{
1504		- unsigned long gs1;
1505		- unsigned long gs2;
1506		- unsigned long gps;
1507		- unsigned long j;
1508		- unsigned long jn;
1509		- unsigned long js;
1510		- struct rcu_node *rnp;
1511		-
1512		- if ((rcu_cpu_stall_suppress && !rcu_kick_kthreads) \|\|
1513		- !rcu_gp_in_progress(rsp))
1514		- return;
1515		- rcu_stall_kick_kthreads(rsp);
1516		- j = jiffies;
1517		-
1518		- /*
1519		- * Lots of memory barriers to reject false positives.
1520		- *
1521		- * The idea is to pick up rsp->gp_seq, then rsp->jiffies_stall,
1522		- * then rsp->gp_start, and finally another copy of rsp->gp_seq.
1523		- * These values are updated in the opposite order with memory
1524		- * barriers (or equivalent) during grace-period initialization
1525		- * and cleanup. Now, a false positive can occur if we get an new
1526		- * value of rsp->gp_start and a old value of rsp->jiffies_stall.
1527		- * But given the memory barriers, the only way that this can happen
1528		- * is if one grace period ends and another starts between these
1529		- * two fetches. This is detected by comparing the second fetch
1530		- * of rsp->gp_seq with the previous fetch from rsp->gp_seq.
1531		- *
1532		- * Given this check, comparisons of jiffies, rsp->jiffies_stall,
1533		- * and rsp->gp_start suffice to forestall false positives.
1534		- */
1535		- gs1 = READ_ONCE(rsp->gp_seq);
1536		- smp_rmb(); /* Pick up ->gp_seq first... */
1537		- js = READ_ONCE(rsp->jiffies_stall);
1538		- smp_rmb(); /* ...then ->jiffies_stall before the rest... */
1539		- gps = READ_ONCE(rsp->gp_start);
1540		- smp_rmb(); /* ...and finally ->gp_start before ->gp_seq again. */
1541		- gs2 = READ_ONCE(rsp->gp_seq);
1542		- if (gs1 != gs2 \|\|
1543		- ULONG_CMP_LT(j, js) \|\|
1544		- ULONG_CMP_GE(gps, js))
1545		- return; /* No stall or GP completed since entering function. */
1546		- rnp = rdp->mynode;
1547		- jn = jiffies + 3 * rcu_jiffies_till_stall_check() + 3;
1548		- if (rcu_gp_in_progress(rsp) &&
1549		- (READ_ONCE(rnp->qsmask) & rdp->grpmask) &&
1550		- cmpxchg(&rsp->jiffies_stall, js, jn) == js) {
1551		-
1552		- /* We haven't checked in, so go dump stack. */
1553		- print_cpu_stall(rsp);
1554		-
1555		- } else if (rcu_gp_in_progress(rsp) &&
1556		- ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY) &&
1557		- cmpxchg(&rsp->jiffies_stall, js, jn) == js) {
1558		-
1559		- /* They had a few time units to dump stack, so complain. */
1560		- print_other_cpu_stall(rsp, gs2);
1561		- }
1562		-}
1563		-
1564		-/**
1565		- * rcu_cpu_stall_reset - prevent further stall warnings in current grace period
1566		- *
1567		- * Set the stall-warning timeout way off into the future, thus preventing
1568		- * any RCU CPU stall-warning messages from appearing in the current set of
1569		- * RCU grace periods.
1570		- *
1571		- * The caller must disable hard irqs.
1572		- */
1573		-void rcu_cpu_stall_reset(void)
1574		-{
1575		- struct rcu_state *rsp;
1576		-
1577		- for_each_rcu_flavor(rsp)
1578		- WRITE_ONCE(rsp->jiffies_stall, jiffies + ULONG_MAX / 2);
1579		-}
1580		-
1581	1335	/* Trace-event wrapper function for trace_rcu_future_grace_period. */
1582	1336	static void trace_rcu_this_gp(struct rcu_node rnp, struct rcu_data rdp,
1583	1337	unsigned long gp_seq_req, const char *s)
1584	1338	{
1585		- trace_rcu_future_grace_period(rdp->rsp->name, rnp->gp_seq, gp_seq_req,
1586		- rnp->level, rnp->grplo, rnp->grphi, s);
	1339	+ trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
	1340	+ gp_seq_req, rnp->level,
	1341	+ rnp->grplo, rnp->grphi, s);
1587	1342	}
1588	1343
1589	1344	/*
..	..	@@ -1606,7 +1361,6 @@
1606	1361	unsigned long gp_seq_req)
1607	1362	{
1608	1363	bool ret = false;
1609		- struct rcu_state *rsp = rdp->rsp;
1610	1364	struct rcu_node *rnp;
1611	1365
1612	1366	/*
..	..	@@ -1631,7 +1385,7 @@
1631	1385	TPS("Prestarted"));
1632	1386	goto unlock_out;
1633	1387	}
1634		- rnp->gp_seq_needed = gp_seq_req;
	1388	+ WRITE_ONCE(rnp->gp_seq_needed, gp_seq_req);
1635	1389	if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {
1636	1390	/*
1637	1391	* We just marked the leaf or internal node, and a
..	..	@@ -1650,24 +1404,24 @@
1650	1404	}
1651	1405
1652	1406	/* If GP already in progress, just leave, otherwise start one. */
1653		- if (rcu_gp_in_progress(rsp)) {
	1407	+ if (rcu_gp_in_progress()) {
1654	1408	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedleafroot"));
1655	1409	goto unlock_out;
1656	1410	}
1657	1411	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot"));
1658		- WRITE_ONCE(rsp->gp_flags, rsp->gp_flags \| RCU_GP_FLAG_INIT);
1659		- rsp->gp_req_activity = jiffies;
1660		- if (!rsp->gp_kthread) {
	1412	+ WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags \| RCU_GP_FLAG_INIT);
	1413	+ WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
	1414	+ if (!READ_ONCE(rcu_state.gp_kthread)) {
1661	1415	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread"));
1662	1416	goto unlock_out;
1663	1417	}
1664		- trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq), TPS("newreq"));
	1418	+ trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq"));
1665	1419	ret = true; /* Caller must wake GP kthread. */
1666	1420	unlock_out:
1667	1421	/* Push furthest requested GP to leaf node and rcu_data structure. */
1668	1422	if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) {
1669		- rnp_start->gp_seq_needed = rnp->gp_seq_needed;
1670		- rdp->gp_seq_needed = rnp->gp_seq_needed;
	1423	+ WRITE_ONCE(rnp_start->gp_seq_needed, rnp->gp_seq_needed);
	1424	+ WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
1671	1425	}
1672	1426	if (rnp != rnp_start)
1673	1427	raw_spin_unlock_rcu_node(rnp);
..	..	@@ -1678,10 +1432,10 @@
1678	1432	* Clean up any old requests for the just-ended grace period. Also return
1679	1433	* whether any additional grace periods have been requested.
1680	1434	*/
1681		-static bool rcu_future_gp_cleanup(struct rcu_state rsp, struct rcu_node rnp)
	1435	+static bool rcu_future_gp_cleanup(struct rcu_node *rnp)
1682	1436	{
1683	1437	bool needmore;
1684		- struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
	1438	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1685	1439
1686	1440	needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed);
1687	1441	if (!needmore)
..	..	@@ -1692,12 +1446,13 @@
1692	1446	}
1693	1447
1694	1448	/*
1695		- * Awaken the grace-period kthread. Don't do a self-awaken (unless in
1696		- * an interrupt or softirq handler), and don't bother awakening when there
1697		- * is nothing for the grace-period kthread to do (as in several CPUs raced
1698		- * to awaken, and we lost), and finally don't try to awaken a kthread that
1699		- * has not yet been created. If all those checks are passed, track some
1700		- * debug information and awaken.
	1449	+ * Awaken the grace-period kthread. Don't do a self-awaken (unless in an
	1450	+ * interrupt or softirq handler, in which case we just might immediately
	1451	+ * sleep upon return, resulting in a grace-period hang), and don't bother
	1452	+ * awakening when there is nothing for the grace-period kthread to do
	1453	+ * (as in several CPUs raced to awaken, we lost), and finally don't try
	1454	+ * to awaken a kthread that has not yet been created. If all those checks
	1455	+ * are passed, track some debug information and awaken.
1701	1456	*
1702	1457	* So why do the self-wakeup when in an interrupt or softirq handler
1703	1458	* in the grace-period kthread's context? Because the kthread might have
..	..	@@ -1705,14 +1460,16 @@
1705	1460	* pre-sleep check of the awaken condition. In this case, a wakeup really
1706	1461	* is required, and is therefore supplied.
1707	1462	*/
1708		-static void rcu_gp_kthread_wake(struct rcu_state *rsp)
	1463	+static void rcu_gp_kthread_wake(void)
1709	1464	{
1710		- if ((current == rsp->gp_kthread &&
1711		- !in_interrupt() && !in_serving_softirq()) \|\|
1712		- !READ_ONCE(rsp->gp_flags) \|\|
1713		- !rsp->gp_kthread)
	1465	+ struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
	1466	+
	1467	+ if ((current == t && !in_irq() && !in_serving_softirq()) \|\|
	1468	+ !READ_ONCE(rcu_state.gp_flags) \|\| !t)
1714	1469	return;
1715		- swake_up_one(&rsp->gp_wq);
	1470	+ WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
	1471	+ WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
	1472	+ swake_up_one(&rcu_state.gp_wq);
1716	1473	}
1717	1474
1718	1475	/*
..	..	@@ -1727,12 +1484,12 @@
1727	1484	*
1728	1485	* The caller must hold rnp->lock with interrupts disabled.
1729	1486	*/
1730		-static bool rcu_accelerate_cbs(struct rcu_state rsp, struct rcu_node rnp,
1731		- struct rcu_data *rdp)
	1487	+static bool rcu_accelerate_cbs(struct rcu_node rnp, struct rcu_data rdp)
1732	1488	{
1733	1489	unsigned long gp_seq_req;
1734	1490	bool ret = false;
1735	1491
	1492	+ rcu_lockdep_assert_cblist_protected(rdp);
1736	1493	raw_lockdep_assert_held_rcu_node(rnp);
1737	1494
1738	1495	/* If no pending (not yet ready to invoke) callbacks, nothing to do. */
..	..	@@ -1749,15 +1506,16 @@
1749	1506	* accelerating callback invocation to an earlier grace-period
1750	1507	* number.
1751	1508	*/
1752		- gp_seq_req = rcu_seq_snap(&rsp->gp_seq);
	1509	+ gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);
1753	1510	if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req))
1754	1511	ret = rcu_start_this_gp(rnp, rdp, gp_seq_req);
1755	1512
1756	1513	/* Trace depending on how much we were able to accelerate. */
1757	1514	if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL))
1758		- trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccWaitCB"));
	1515	+ trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB"));
1759	1516	else
1760		- trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccReadyCB"));
	1517	+ trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB"));
	1518	+
1761	1519	return ret;
1762	1520	}
1763	1521
..	..	@@ -1768,25 +1526,24 @@
1768	1526	* that a new grace-period request be made, invokes rcu_accelerate_cbs()
1769	1527	* while holding the leaf rcu_node structure's ->lock.
1770	1528	*/
1771		-static void rcu_accelerate_cbs_unlocked(struct rcu_state *rsp,
1772		- struct rcu_node *rnp,
	1529	+static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp,
1773	1530	struct rcu_data *rdp)
1774	1531	{
1775	1532	unsigned long c;
1776	1533	bool needwake;
1777	1534
1778		- lockdep_assert_irqs_disabled();
1779		- c = rcu_seq_snap(&rsp->gp_seq);
1780		- if (!rdp->gpwrap && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
	1535	+ rcu_lockdep_assert_cblist_protected(rdp);
	1536	+ c = rcu_seq_snap(&rcu_state.gp_seq);
	1537	+ if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
1781	1538	/* Old request still live, so mark recent callbacks. */
1782	1539	(void)rcu_segcblist_accelerate(&rdp->cblist, c);
1783	1540	return;
1784	1541	}
1785	1542	raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
1786		- needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
	1543	+ needwake = rcu_accelerate_cbs(rnp, rdp);
1787	1544	raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
1788	1545	if (needwake)
1789		- rcu_gp_kthread_wake(rsp);
	1546	+ rcu_gp_kthread_wake();
1790	1547	}
1791	1548
1792	1549	/*
..	..	@@ -1799,9 +1556,9 @@
1799	1556	*
1800	1557	* The caller must hold rnp->lock with interrupts disabled.
1801	1558	*/
1802		-static bool rcu_advance_cbs(struct rcu_state rsp, struct rcu_node rnp,
1803		- struct rcu_data *rdp)
	1559	+static bool rcu_advance_cbs(struct rcu_node rnp, struct rcu_data rdp)
1804	1560	{
	1561	+ rcu_lockdep_assert_cblist_protected(rdp);
1805	1562	raw_lockdep_assert_held_rcu_node(rnp);
1806	1563
1807	1564	/* If no pending (not yet ready to invoke) callbacks, nothing to do. */
..	..	@@ -1815,7 +1572,36 @@
1815	1572	rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq);
1816	1573
1817	1574	/* Classify any remaining callbacks. */
1818		- return rcu_accelerate_cbs(rsp, rnp, rdp);
	1575	+ return rcu_accelerate_cbs(rnp, rdp);
	1576	+}
	1577	+
	1578	+/*
	1579	+ * Move and classify callbacks, but only if doing so won't require
	1580	+ * that the RCU grace-period kthread be awakened.
	1581	+ */
	1582	+static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp,
	1583	+ struct rcu_data *rdp)
	1584	+{
	1585	+ rcu_lockdep_assert_cblist_protected(rdp);
	1586	+ if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) \|\| !raw_spin_trylock_rcu_node(rnp))
	1587	+ return;
	1588	+ // The grace period cannot end while we hold the rcu_node lock.
	1589	+ if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))
	1590	+ WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp));
	1591	+ raw_spin_unlock_rcu_node(rnp);
	1592	+}
	1593	+
	1594	+/*
	1595	+ * In CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels, attempt to generate a
	1596	+ * quiescent state. This is intended to be invoked when the CPU notices
	1597	+ * a new grace period.
	1598	+ */
	1599	+static void rcu_strict_gp_check_qs(void)
	1600	+{
	1601	+ if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD)) {
	1602	+ rcu_read_lock();
	1603	+ rcu_read_unlock();
	1604	+ }
1819	1605	}
1820	1606
1821	1607	/*
..	..	@@ -1824,11 +1610,12 @@
1824	1610	* structure corresponding to the current CPU, and must have irqs disabled.
1825	1611	* Returns true if the grace-period kthread needs to be awakened.
1826	1612	*/
1827		-static bool __note_gp_changes(struct rcu_state rsp, struct rcu_node rnp,
1828		- struct rcu_data *rdp)
	1613	+static bool __note_gp_changes(struct rcu_node rnp, struct rcu_data rdp)
1829	1614	{
1830		- bool ret;
1831		- bool need_gp;
	1615	+ bool ret = false;
	1616	+ bool need_qs;
	1617	+ const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	1618	+ rcu_segcblist_is_offloaded(&rdp->cblist);
1832	1619
1833	1620	raw_lockdep_assert_held_rcu_node(rnp);
1834	1621
..	..	@@ -1838,10 +1625,15 @@
1838	1625	/* Handle the ends of any preceding grace periods first. */
1839	1626	if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) \|\|
1840	1627	unlikely(READ_ONCE(rdp->gpwrap))) {
1841		- ret = rcu_advance_cbs(rsp, rnp, rdp); /* Advance callbacks. */
1842		- trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuend"));
	1628	+ if (!offloaded)
	1629	+ ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */
	1630	+ rdp->core_needs_qs = false;
	1631	+ trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));
1843	1632	} else {
1844		- ret = rcu_accelerate_cbs(rsp, rnp, rdp); /* Recent callbacks. */
	1633	+ if (!offloaded)
	1634	+ ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */
	1635	+ if (rdp->core_needs_qs)
	1636	+ rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask);
1845	1637	}
1846	1638
1847	1639	/* Now handle the beginnings of any new-to-this-CPU grace periods. */
..	..	@@ -1852,22 +1644,21 @@
1852	1644	* set up to detect a quiescent state, otherwise don't
1853	1645	* go looking for one.
1854	1646	*/
1855		- trace_rcu_grace_period(rsp->name, rnp->gp_seq, TPS("cpustart"));
1856		- need_gp = !!(rnp->qsmask & rdp->grpmask);
1857		- rdp->cpu_no_qs.b.norm = need_gp;
1858		- rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
1859		- rdp->core_needs_qs = need_gp;
	1647	+ trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));
	1648	+ need_qs = !!(rnp->qsmask & rdp->grpmask);
	1649	+ rdp->cpu_no_qs.b.norm = need_qs;
	1650	+ rdp->core_needs_qs = need_qs;
1860	1651	zero_cpu_stall_ticks(rdp);
1861	1652	}
1862	1653	rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */
1863		- if (ULONG_CMP_GE(rnp->gp_seq_needed, rdp->gp_seq_needed) \|\| rdp->gpwrap)
1864		- rdp->gp_seq_needed = rnp->gp_seq_needed;
	1654	+ if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) \|\| rdp->gpwrap)
	1655	+ WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
1865	1656	WRITE_ONCE(rdp->gpwrap, false);
1866	1657	rcu_gpnum_ovf(rnp, rdp);
1867	1658	return ret;
1868	1659	}
1869	1660
1870		-static void note_gp_changes(struct rcu_state rsp, struct rcu_data rdp)
	1661	+static void note_gp_changes(struct rcu_data *rdp)
1871	1662	{
1872	1663	unsigned long flags;
1873	1664	bool needwake;
..	..	@@ -1881,41 +1672,76 @@
1881	1672	local_irq_restore(flags);
1882	1673	return;
1883	1674	}
1884		- needwake = __note_gp_changes(rsp, rnp, rdp);
	1675	+ needwake = __note_gp_changes(rnp, rdp);
1885	1676	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
	1677	+ rcu_strict_gp_check_qs();
1886	1678	if (needwake)
1887		- rcu_gp_kthread_wake(rsp);
	1679	+ rcu_gp_kthread_wake();
1888	1680	}
1889	1681
1890		-static void rcu_gp_slow(struct rcu_state *rsp, int delay)
	1682	+static void rcu_gp_slow(int delay)
1891	1683	{
1892	1684	if (delay > 0 &&
1893		- !(rcu_seq_ctr(rsp->gp_seq) %
	1685	+ !(rcu_seq_ctr(rcu_state.gp_seq) %
1894	1686	(rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))
1895		- schedule_timeout_uninterruptible(delay);
	1687	+ schedule_timeout_idle(delay);
	1688	+}
	1689	+
	1690	+static unsigned long sleep_duration;
	1691	+
	1692	+/* Allow rcutorture to stall the grace-period kthread. */
	1693	+void rcu_gp_set_torture_wait(int duration)
	1694	+{
	1695	+ if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST) && duration > 0)
	1696	+ WRITE_ONCE(sleep_duration, duration);
	1697	+}
	1698	+EXPORT_SYMBOL_GPL(rcu_gp_set_torture_wait);
	1699	+
	1700	+/* Actually implement the aforementioned wait. */
	1701	+static void rcu_gp_torture_wait(void)
	1702	+{
	1703	+ unsigned long duration;
	1704	+
	1705	+ if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST))
	1706	+ return;
	1707	+ duration = xchg(&sleep_duration, 0UL);
	1708	+ if (duration > 0) {
	1709	+ pr_alert("%s: Waiting %lu jiffies\n", __func__, duration);
	1710	+ schedule_timeout_idle(duration);
	1711	+ pr_alert("%s: Wait complete\n", __func__);
	1712	+ }
	1713	+}
	1714	+
	1715	+/*
	1716	+ * Handler for on_each_cpu() to invoke the target CPU's RCU core
	1717	+ * processing.
	1718	+ */
	1719	+static void rcu_strict_gp_boundary(void *unused)
	1720	+{
	1721	+ invoke_rcu_core();
1896	1722	}
1897	1723
1898	1724	/*
1899	1725	* Initialize a new grace period. Return false if no grace period required.
1900	1726	*/
1901		-static bool rcu_gp_init(struct rcu_state *rsp)
	1727	+static bool rcu_gp_init(void)
1902	1728	{
1903	1729	unsigned long flags;
1904	1730	unsigned long oldmask;
1905	1731	unsigned long mask;
1906	1732	struct rcu_data *rdp;
1907		- struct rcu_node *rnp = rcu_get_root(rsp);
	1733	+ struct rcu_node *rnp = rcu_get_root();
1908	1734
1909		- WRITE_ONCE(rsp->gp_activity, jiffies);
	1735	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
1910	1736	raw_spin_lock_irq_rcu_node(rnp);
1911		- if (!READ_ONCE(rsp->gp_flags)) {
	1737	+ if (!READ_ONCE(rcu_state.gp_flags)) {
1912	1738	/* Spurious wakeup, tell caller to go back to sleep. */
1913	1739	raw_spin_unlock_irq_rcu_node(rnp);
1914	1740	return false;
1915	1741	}
1916		- WRITE_ONCE(rsp->gp_flags, 0); /* Clear all flags: New grace period. */
	1742	+ WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */
1917	1743
1918		- if (WARN_ON_ONCE(rcu_gp_in_progress(rsp))) {
	1744	+ if (WARN_ON_ONCE(rcu_gp_in_progress())) {
1919	1745	/*
1920	1746	* Grace period already in progress, don't start another.
1921	1747	* Not supposed to be able to happen.
..	..	@@ -1925,27 +1751,31 @@
1925	1751	}
1926	1752
1927	1753	/* Advance to a new grace period and initialize state. */
1928		- record_gp_stall_check_time(rsp);
	1754	+ record_gp_stall_check_time();
1929	1755	/* Record GP times before starting GP, hence rcu_seq_start(). */
1930		- rcu_seq_start(&rsp->gp_seq);
1931		- trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("start"));
	1756	+ rcu_seq_start(&rcu_state.gp_seq);
	1757	+ ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
	1758	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));
1932	1759	raw_spin_unlock_irq_rcu_node(rnp);
1933	1760
1934	1761	/*
1935		- * Apply per-leaf buffered online and offline operations to the
1936		- * rcu_node tree. Note that this new grace period need not wait
1937		- * for subsequent online CPUs, and that quiescent-state forcing
1938		- * will handle subsequent offline CPUs.
	1762	+ * Apply per-leaf buffered online and offline operations to
	1763	+ * the rcu_node tree. Note that this new grace period need not
	1764	+ * wait for subsequent online CPUs, and that RCU hooks in the CPU
	1765	+ * offlining path, when combined with checks in this function,
	1766	+ * will handle CPUs that are currently going offline or that will
	1767	+ * go offline later. Please also refer to "Hotplug CPU" section
	1768	+ * of RCU's Requirements documentation.
1939	1769	*/
1940		- rsp->gp_state = RCU_GP_ONOFF;
1941		- rcu_for_each_leaf_node(rsp, rnp) {
1942		- spin_lock(&rsp->ofl_lock);
	1770	+ rcu_state.gp_state = RCU_GP_ONOFF;
	1771	+ rcu_for_each_leaf_node(rnp) {
	1772	+ raw_spin_lock(&rcu_state.ofl_lock);
1943	1773	raw_spin_lock_irq_rcu_node(rnp);
1944	1774	if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
1945	1775	!rnp->wait_blkd_tasks) {
1946	1776	/* Nothing to do on this leaf rcu_node structure. */
1947	1777	raw_spin_unlock_irq_rcu_node(rnp);
1948		- spin_unlock(&rsp->ofl_lock);
	1778	+ raw_spin_unlock(&rcu_state.ofl_lock);
1949	1779	continue;
1950	1780	}
1951	1781
..	..	@@ -1981,46 +1811,50 @@
1981	1811	}
1982	1812
1983	1813	raw_spin_unlock_irq_rcu_node(rnp);
1984		- spin_unlock(&rsp->ofl_lock);
	1814	+ raw_spin_unlock(&rcu_state.ofl_lock);
1985	1815	}
1986		- rcu_gp_slow(rsp, gp_preinit_delay); /* Races with CPU hotplug. */
	1816	+ rcu_gp_slow(gp_preinit_delay); /* Races with CPU hotplug. */
1987	1817
1988	1818	/*
1989	1819	* Set the quiescent-state-needed bits in all the rcu_node
1990		- * structures for all currently online CPUs in breadth-first order,
1991		- * starting from the root rcu_node structure, relying on the layout
1992		- * of the tree within the rsp->node[] array. Note that other CPUs
1993		- * will access only the leaves of the hierarchy, thus seeing that no
1994		- * grace period is in progress, at least until the corresponding
1995		- * leaf node has been initialized.
	1820	+ * structures for all currently online CPUs in breadth-first
	1821	+ * order, starting from the root rcu_node structure, relying on the
	1822	+ * layout of the tree within the rcu_state.node[] array. Note that
	1823	+ * other CPUs will access only the leaves of the hierarchy, thus
	1824	+ * seeing that no grace period is in progress, at least until the
	1825	+ * corresponding leaf node has been initialized.
1996	1826	*
1997	1827	* The grace period cannot complete until the initialization
1998	1828	* process finishes, because this kthread handles both.
1999	1829	*/
2000		- rsp->gp_state = RCU_GP_INIT;
2001		- rcu_for_each_node_breadth_first(rsp, rnp) {
2002		- rcu_gp_slow(rsp, gp_init_delay);
	1830	+ rcu_state.gp_state = RCU_GP_INIT;
	1831	+ rcu_for_each_node_breadth_first(rnp) {
	1832	+ rcu_gp_slow(gp_init_delay);
2003	1833	raw_spin_lock_irqsave_rcu_node(rnp, flags);
2004		- rdp = this_cpu_ptr(rsp->rda);
2005		- rcu_preempt_check_blocked_tasks(rsp, rnp);
	1834	+ rdp = this_cpu_ptr(&rcu_data);
	1835	+ rcu_preempt_check_blocked_tasks(rnp);
2006	1836	rnp->qsmask = rnp->qsmaskinit;
2007		- WRITE_ONCE(rnp->gp_seq, rsp->gp_seq);
	1837	+ WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);
2008	1838	if (rnp == rdp->mynode)
2009		- (void)__note_gp_changes(rsp, rnp, rdp);
	1839	+ (void)__note_gp_changes(rnp, rdp);
2010	1840	rcu_preempt_boost_start_gp(rnp);
2011		- trace_rcu_grace_period_init(rsp->name, rnp->gp_seq,
	1841	+ trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,
2012	1842	rnp->level, rnp->grplo,
2013	1843	rnp->grphi, rnp->qsmask);
2014	1844	/* Quiescent states for tasks on any now-offline CPUs. */
2015	1845	mask = rnp->qsmask & ~rnp->qsmaskinitnext;
2016	1846	rnp->rcu_gp_init_mask = mask;
2017	1847	if ((mask \|\| rnp->wait_blkd_tasks) && rcu_is_leaf_node(rnp))
2018		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
	1848	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2019	1849	else
2020	1850	raw_spin_unlock_irq_rcu_node(rnp);
2021	1851	cond_resched_tasks_rcu_qs();
2022		- WRITE_ONCE(rsp->gp_activity, jiffies);
	1852	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
2023	1853	}
	1854	+
	1855	+ // If strict, make all CPUs aware of new grace period.
	1856	+ if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
	1857	+ on_each_cpu(rcu_strict_gp_boundary, NULL, 0);
2024	1858
2025	1859	return true;
2026	1860	}
..	..	@@ -2029,16 +1863,20 @@
2029	1863	* Helper function for swait_event_idle_exclusive() wakeup at force-quiescent-state
2030	1864	* time.
2031	1865	*/
2032		-static bool rcu_gp_fqs_check_wake(struct rcu_state rsp, int gfp)
	1866	+static bool rcu_gp_fqs_check_wake(int *gfp)
2033	1867	{
2034		- struct rcu_node *rnp = rcu_get_root(rsp);
	1868	+ struct rcu_node *rnp = rcu_get_root();
2035	1869
2036		- /* Someone like call_rcu() requested a force-quiescent-state scan. */
2037		- *gfp = READ_ONCE(rsp->gp_flags);
	1870	+ // If under overload conditions, force an immediate FQS scan.
	1871	+ if (*gfp & RCU_GP_FLAG_OVLD)
	1872	+ return true;
	1873	+
	1874	+ // Someone like call_rcu() requested a force-quiescent-state scan.
	1875	+ *gfp = READ_ONCE(rcu_state.gp_flags);
2038	1876	if (*gfp & RCU_GP_FLAG_FQS)
2039	1877	return true;
2040	1878
2041		- /* The current grace period has completed. */
	1879	+ // The current grace period has completed.
2042	1880	if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp))
2043	1881	return true;
2044	1882
..	..	@@ -2048,45 +1886,117 @@
2048	1886	/*
2049	1887	* Do one round of quiescent-state forcing.
2050	1888	*/
2051		-static void rcu_gp_fqs(struct rcu_state *rsp, bool first_time)
	1889	+static void rcu_gp_fqs(bool first_time)
2052	1890	{
2053		- struct rcu_node *rnp = rcu_get_root(rsp);
	1891	+ struct rcu_node *rnp = rcu_get_root();
2054	1892
2055		- WRITE_ONCE(rsp->gp_activity, jiffies);
2056		- rsp->n_force_qs++;
	1893	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	1894	+ WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1);
2057	1895	if (first_time) {
2058	1896	/* Collect dyntick-idle snapshots. */
2059		- force_qs_rnp(rsp, dyntick_save_progress_counter);
	1897	+ force_qs_rnp(dyntick_save_progress_counter);
2060	1898	} else {
2061	1899	/* Handle dyntick-idle and offline CPUs. */
2062		- force_qs_rnp(rsp, rcu_implicit_dynticks_qs);
	1900	+ force_qs_rnp(rcu_implicit_dynticks_qs);
2063	1901	}
2064	1902	/* Clear flag to prevent immediate re-entry. */
2065		- if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
	1903	+ if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
2066	1904	raw_spin_lock_irq_rcu_node(rnp);
2067		- WRITE_ONCE(rsp->gp_flags,
2068		- READ_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS);
	1905	+ WRITE_ONCE(rcu_state.gp_flags,
	1906	+ READ_ONCE(rcu_state.gp_flags) & ~RCU_GP_FLAG_FQS);
2069	1907	raw_spin_unlock_irq_rcu_node(rnp);
	1908	+ }
	1909	+}
	1910	+
	1911	+/*
	1912	+ * Loop doing repeated quiescent-state forcing until the grace period ends.
	1913	+ */
	1914	+static void rcu_gp_fqs_loop(void)
	1915	+{
	1916	+ bool first_gp_fqs;
	1917	+ int gf = 0;
	1918	+ unsigned long j;
	1919	+ int ret;
	1920	+ struct rcu_node *rnp = rcu_get_root();
	1921	+
	1922	+ first_gp_fqs = true;
	1923	+ j = READ_ONCE(jiffies_till_first_fqs);
	1924	+ if (rcu_state.cbovld)
	1925	+ gf = RCU_GP_FLAG_OVLD;
	1926	+ ret = 0;
	1927	+ for (;;) {
	1928	+ if (!ret) {
	1929	+ rcu_state.jiffies_force_qs = jiffies + j;
	1930	+ WRITE_ONCE(rcu_state.jiffies_kick_kthreads,
	1931	+ jiffies + (j ? 3 * j : 2));
	1932	+ }
	1933	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
	1934	+ TPS("fqswait"));
	1935	+ rcu_state.gp_state = RCU_GP_WAIT_FQS;
	1936	+ ret = swait_event_idle_timeout_exclusive(
	1937	+ rcu_state.gp_wq, rcu_gp_fqs_check_wake(&gf), j);
	1938	+ rcu_gp_torture_wait();
	1939	+ rcu_state.gp_state = RCU_GP_DOING_FQS;
	1940	+ /* Locking provides needed memory barriers. */
	1941	+ /* If grace period done, leave loop. */
	1942	+ if (!READ_ONCE(rnp->qsmask) &&
	1943	+ !rcu_preempt_blocked_readers_cgp(rnp))
	1944	+ break;
	1945	+ /* If time for quiescent-state forcing, do it. */
	1946	+ if (!time_after(rcu_state.jiffies_force_qs, jiffies) \|\|
	1947	+ (gf & (RCU_GP_FLAG_FQS \| RCU_GP_FLAG_OVLD))) {
	1948	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
	1949	+ TPS("fqsstart"));
	1950	+ rcu_gp_fqs(first_gp_fqs);
	1951	+ gf = 0;
	1952	+ if (first_gp_fqs) {
	1953	+ first_gp_fqs = false;
	1954	+ gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0;
	1955	+ }
	1956	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
	1957	+ TPS("fqsend"));
	1958	+ cond_resched_tasks_rcu_qs();
	1959	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	1960	+ ret = 0; /* Force full wait till next FQS. */
	1961	+ j = READ_ONCE(jiffies_till_next_fqs);
	1962	+ } else {
	1963	+ /* Deal with stray signal. */
	1964	+ cond_resched_tasks_rcu_qs();
	1965	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	1966	+ WARN_ON(signal_pending(current));
	1967	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
	1968	+ TPS("fqswaitsig"));
	1969	+ ret = 1; /* Keep old FQS timing. */
	1970	+ j = jiffies;
	1971	+ if (time_after(jiffies, rcu_state.jiffies_force_qs))
	1972	+ j = 1;
	1973	+ else
	1974	+ j = rcu_state.jiffies_force_qs - j;
	1975	+ gf = 0;
	1976	+ }
2070	1977	}
2071	1978	}
2072	1979
2073	1980	/*
2074	1981	* Clean up after the old grace period.
2075	1982	*/
2076		-static void rcu_gp_cleanup(struct rcu_state *rsp)
	1983	+static void rcu_gp_cleanup(void)
2077	1984	{
2078		- unsigned long gp_duration;
	1985	+ int cpu;
2079	1986	bool needgp = false;
	1987	+ unsigned long gp_duration;
2080	1988	unsigned long new_gp_seq;
	1989	+ bool offloaded;
2081	1990	struct rcu_data *rdp;
2082		- struct rcu_node *rnp = rcu_get_root(rsp);
	1991	+ struct rcu_node *rnp = rcu_get_root();
2083	1992	struct swait_queue_head *sq;
2084	1993
2085		- WRITE_ONCE(rsp->gp_activity, jiffies);
	1994	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
2086	1995	raw_spin_lock_irq_rcu_node(rnp);
2087		- gp_duration = jiffies - rsp->gp_start;
2088		- if (gp_duration > rsp->gp_max)
2089		- rsp->gp_max = gp_duration;
	1996	+ rcu_state.gp_end = jiffies;
	1997	+ gp_duration = rcu_state.gp_end - rcu_state.gp_start;
	1998	+ if (gp_duration > rcu_state.gp_max)
	1999	+ rcu_state.gp_max = gp_duration;
2090	2000
2091	2001	/*
2092	2002	* We know the grace period is complete, but to everyone else
..	..	@@ -2107,165 +2017,123 @@
2107	2017	* the rcu_node structures before the beginning of the next grace
2108	2018	* period is recorded in any of the rcu_node structures.
2109	2019	*/
2110		- new_gp_seq = rsp->gp_seq;
	2020	+ new_gp_seq = rcu_state.gp_seq;
2111	2021	rcu_seq_end(&new_gp_seq);
2112		- rcu_for_each_node_breadth_first(rsp, rnp) {
	2022	+ rcu_for_each_node_breadth_first(rnp) {
2113	2023	raw_spin_lock_irq_rcu_node(rnp);
2114	2024	if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))
2115		- dump_blkd_tasks(rsp, rnp, 10);
	2025	+ dump_blkd_tasks(rnp, 10);
2116	2026	WARN_ON_ONCE(rnp->qsmask);
2117	2027	WRITE_ONCE(rnp->gp_seq, new_gp_seq);
2118		- rdp = this_cpu_ptr(rsp->rda);
	2028	+ rdp = this_cpu_ptr(&rcu_data);
2119	2029	if (rnp == rdp->mynode)
2120		- needgp = __note_gp_changes(rsp, rnp, rdp) \|\| needgp;
	2030	+ needgp = __note_gp_changes(rnp, rdp) \|\| needgp;
2121	2031	/* smp_mb() provided by prior unlock-lock pair. */
2122		- needgp = rcu_future_gp_cleanup(rsp, rnp) \|\| needgp;
	2032	+ needgp = rcu_future_gp_cleanup(rnp) \|\| needgp;
	2033	+ // Reset overload indication for CPUs no longer overloaded
	2034	+ if (rcu_is_leaf_node(rnp))
	2035	+ for_each_leaf_node_cpu_mask(rnp, cpu, rnp->cbovldmask) {
	2036	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	2037	+ check_cb_ovld_locked(rdp, rnp);
	2038	+ }
2123	2039	sq = rcu_nocb_gp_get(rnp);
2124	2040	raw_spin_unlock_irq_rcu_node(rnp);
2125	2041	rcu_nocb_gp_cleanup(sq);
2126	2042	cond_resched_tasks_rcu_qs();
2127		- WRITE_ONCE(rsp->gp_activity, jiffies);
2128		- rcu_gp_slow(rsp, gp_cleanup_delay);
	2043	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	2044	+ rcu_gp_slow(gp_cleanup_delay);
2129	2045	}
2130		- rnp = rcu_get_root(rsp);
2131		- raw_spin_lock_irq_rcu_node(rnp); /* GP before rsp->gp_seq update. */
	2046	+ rnp = rcu_get_root();
	2047	+ raw_spin_lock_irq_rcu_node(rnp); /* GP before ->gp_seq update. */
2132	2048
2133		- /* Declare grace period done. */
2134		- rcu_seq_end(&rsp->gp_seq);
2135		- trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("end"));
2136		- rsp->gp_state = RCU_GP_IDLE;
	2049	+ /* Declare grace period done, trace first to use old GP number. */
	2050	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));
	2051	+ rcu_seq_end(&rcu_state.gp_seq);
	2052	+ ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
	2053	+ rcu_state.gp_state = RCU_GP_IDLE;
2137	2054	/* Check for GP requests since above loop. */
2138		- rdp = this_cpu_ptr(rsp->rda);
	2055	+ rdp = this_cpu_ptr(&rcu_data);
2139	2056	if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) {
2140	2057	trace_rcu_this_gp(rnp, rdp, rnp->gp_seq_needed,
2141	2058	TPS("CleanupMore"));
2142	2059	needgp = true;
2143	2060	}
2144	2061	/* Advance CBs to reduce false positives below. */
2145		- if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) {
2146		- WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT);
2147		- rsp->gp_req_activity = jiffies;
2148		- trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq),
	2062	+ offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2063	+ rcu_segcblist_is_offloaded(&rdp->cblist);
	2064	+ if ((offloaded \|\| !rcu_accelerate_cbs(rnp, rdp)) && needgp) {
	2065	+ WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);
	2066	+ WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
	2067	+ trace_rcu_grace_period(rcu_state.name,
	2068	+ rcu_state.gp_seq,
2149	2069	TPS("newreq"));
2150	2070	} else {
2151		- WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT);
	2071	+ WRITE_ONCE(rcu_state.gp_flags,
	2072	+ rcu_state.gp_flags & RCU_GP_FLAG_INIT);
2152	2073	}
2153	2074	raw_spin_unlock_irq_rcu_node(rnp);
	2075	+
	2076	+ // If strict, make all CPUs aware of the end of the old grace period.
	2077	+ if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
	2078	+ on_each_cpu(rcu_strict_gp_boundary, NULL, 0);
2154	2079	}
2155	2080
2156	2081	/*
2157	2082	* Body of kthread that handles grace periods.
2158	2083	*/
2159		-static int __noreturn rcu_gp_kthread(void *arg)
	2084	+static int __noreturn rcu_gp_kthread(void *unused)
2160	2085	{
2161		- bool first_gp_fqs;
2162		- int gf;
2163		- unsigned long j;
2164		- int ret;
2165		- struct rcu_state *rsp = arg;
2166		- struct rcu_node *rnp = rcu_get_root(rsp);
2167		-
2168	2086	rcu_bind_gp_kthread();
2169	2087	for (;;) {
2170	2088
2171	2089	/* Handle grace-period start. */
2172	2090	for (;;) {
2173		- trace_rcu_grace_period(rsp->name,
2174		- READ_ONCE(rsp->gp_seq),
	2091	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
2175	2092	TPS("reqwait"));
2176		- rsp->gp_state = RCU_GP_WAIT_GPS;
2177		- swait_event_idle_exclusive(rsp->gp_wq, READ_ONCE(rsp->gp_flags) &
2178		- RCU_GP_FLAG_INIT);
2179		- rsp->gp_state = RCU_GP_DONE_GPS;
	2093	+ rcu_state.gp_state = RCU_GP_WAIT_GPS;
	2094	+ swait_event_idle_exclusive(rcu_state.gp_wq,
	2095	+ READ_ONCE(rcu_state.gp_flags) &
	2096	+ RCU_GP_FLAG_INIT);
	2097	+ rcu_gp_torture_wait();
	2098	+ rcu_state.gp_state = RCU_GP_DONE_GPS;
2180	2099	/* Locking provides needed memory barrier. */
2181		- if (rcu_gp_init(rsp))
	2100	+ if (rcu_gp_init())
2182	2101	break;
2183	2102	cond_resched_tasks_rcu_qs();
2184		- WRITE_ONCE(rsp->gp_activity, jiffies);
	2103	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
2185	2104	WARN_ON(signal_pending(current));
2186		- trace_rcu_grace_period(rsp->name,
2187		- READ_ONCE(rsp->gp_seq),
	2105	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
2188	2106	TPS("reqwaitsig"));
2189	2107	}
2190	2108
2191	2109	/* Handle quiescent-state forcing. */
2192		- first_gp_fqs = true;
2193		- j = jiffies_till_first_fqs;
2194		- ret = 0;
2195		- for (;;) {
2196		- if (!ret) {
2197		- rsp->jiffies_force_qs = jiffies + j;
2198		- WRITE_ONCE(rsp->jiffies_kick_kthreads,
2199		- jiffies + 3 * j);
2200		- }
2201		- trace_rcu_grace_period(rsp->name,
2202		- READ_ONCE(rsp->gp_seq),
2203		- TPS("fqswait"));
2204		- rsp->gp_state = RCU_GP_WAIT_FQS;
2205		- ret = swait_event_idle_timeout_exclusive(rsp->gp_wq,
2206		- rcu_gp_fqs_check_wake(rsp, &gf), j);
2207		- rsp->gp_state = RCU_GP_DOING_FQS;
2208		- /* Locking provides needed memory barriers. */
2209		- /* If grace period done, leave loop. */
2210		- if (!READ_ONCE(rnp->qsmask) &&
2211		- !rcu_preempt_blocked_readers_cgp(rnp))
2212		- break;
2213		- /* If time for quiescent-state forcing, do it. */
2214		- if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) \|\|
2215		- (gf & RCU_GP_FLAG_FQS)) {
2216		- trace_rcu_grace_period(rsp->name,
2217		- READ_ONCE(rsp->gp_seq),
2218		- TPS("fqsstart"));
2219		- rcu_gp_fqs(rsp, first_gp_fqs);
2220		- first_gp_fqs = false;
2221		- trace_rcu_grace_period(rsp->name,
2222		- READ_ONCE(rsp->gp_seq),
2223		- TPS("fqsend"));
2224		- cond_resched_tasks_rcu_qs();
2225		- WRITE_ONCE(rsp->gp_activity, jiffies);
2226		- ret = 0; /* Force full wait till next FQS. */
2227		- j = jiffies_till_next_fqs;
2228		- } else {
2229		- /* Deal with stray signal. */
2230		- cond_resched_tasks_rcu_qs();
2231		- WRITE_ONCE(rsp->gp_activity, jiffies);
2232		- WARN_ON(signal_pending(current));
2233		- trace_rcu_grace_period(rsp->name,
2234		- READ_ONCE(rsp->gp_seq),
2235		- TPS("fqswaitsig"));
2236		- ret = 1; /* Keep old FQS timing. */
2237		- j = jiffies;
2238		- if (time_after(jiffies, rsp->jiffies_force_qs))
2239		- j = 1;
2240		- else
2241		- j = rsp->jiffies_force_qs - j;
2242		- }
2243		- }
	2110	+ rcu_gp_fqs_loop();
2244	2111
2245	2112	/* Handle grace-period end. */
2246		- rsp->gp_state = RCU_GP_CLEANUP;
2247		- rcu_gp_cleanup(rsp);
2248		- rsp->gp_state = RCU_GP_CLEANED;
	2113	+ rcu_state.gp_state = RCU_GP_CLEANUP;
	2114	+ rcu_gp_cleanup();
	2115	+ rcu_state.gp_state = RCU_GP_CLEANED;
2249	2116	}
2250	2117	}
2251	2118
2252	2119	/*
2253		- * Report a full set of quiescent states to the specified rcu_state data
2254		- * structure. Invoke rcu_gp_kthread_wake() to awaken the grace-period
2255		- * kthread if another grace period is required. Whether we wake
2256		- * the grace-period kthread or it awakens itself for the next round
2257		- * of quiescent-state forcing, that kthread will clean up after the
2258		- * just-completed grace period. Note that the caller must hold rnp->lock,
2259		- * which is released before return.
	2120	+ * Report a full set of quiescent states to the rcu_state data structure.
	2121	+ * Invoke rcu_gp_kthread_wake() to awaken the grace-period kthread if
	2122	+ * another grace period is required. Whether we wake the grace-period
	2123	+ * kthread or it awakens itself for the next round of quiescent-state
	2124	+ * forcing, that kthread will clean up after the just-completed grace
	2125	+ * period. Note that the caller must hold rnp->lock, which is released
	2126	+ * before return.
2260	2127	*/
2261		-static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
2262		- __releases(rcu_get_root(rsp)->lock)
	2128	+static void rcu_report_qs_rsp(unsigned long flags)
	2129	+ __releases(rcu_get_root()->lock)
2263	2130	{
2264		- raw_lockdep_assert_held_rcu_node(rcu_get_root(rsp));
2265		- WARN_ON_ONCE(!rcu_gp_in_progress(rsp));
2266		- WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) \| RCU_GP_FLAG_FQS);
2267		- raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(rsp), flags);
2268		- rcu_gp_kthread_wake(rsp);
	2131	+ raw_lockdep_assert_held_rcu_node(rcu_get_root());
	2132	+ WARN_ON_ONCE(!rcu_gp_in_progress());
	2133	+ WRITE_ONCE(rcu_state.gp_flags,
	2134	+ READ_ONCE(rcu_state.gp_flags) \| RCU_GP_FLAG_FQS);
	2135	+ raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(), flags);
	2136	+ rcu_gp_kthread_wake();
2269	2137	}
2270	2138
2271	2139	/*
..	..	@@ -2282,9 +2150,8 @@
2282	2150	* disabled. This allows propagating quiescent state due to resumed tasks
2283	2151	* during grace-period initialization.
2284	2152	*/
2285		-static void
2286		-rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
2287		- struct rcu_node *rnp, unsigned long gps, unsigned long flags)
	2153	+static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
	2154	+ unsigned long gps, unsigned long flags)
2288	2155	__releases(rnp->lock)
2289	2156	{
2290	2157	unsigned long oldmask = 0;
..	..	@@ -2306,8 +2173,8 @@
2306	2173	WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */
2307	2174	WARN_ON_ONCE(!rcu_is_leaf_node(rnp) &&
2308	2175	rcu_preempt_blocked_readers_cgp(rnp));
2309		- rnp->qsmask &= ~mask;
2310		- trace_rcu_quiescent_state_report(rsp->name, rnp->gp_seq,
	2176	+ WRITE_ONCE(rnp->qsmask, rnp->qsmask & ~mask);
	2177	+ trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,
2311	2178	mask, rnp->qsmask, rnp->level,
2312	2179	rnp->grplo, rnp->grphi,
2313	2180	!!rnp->gp_tasks);
..	..	@@ -2329,7 +2196,7 @@
2329	2196	rnp_c = rnp;
2330	2197	rnp = rnp->parent;
2331	2198	raw_spin_lock_irqsave_rcu_node(rnp, flags);
2332		- oldmask = rnp_c->qsmask;
	2199	+ oldmask = READ_ONCE(rnp_c->qsmask);
2333	2200	}
2334	2201
2335	2202	/*
..	..	@@ -2337,19 +2204,18 @@
2337	2204	* state for this grace period. Invoke rcu_report_qs_rsp()
2338	2205	* to clean up and start the next grace period if one is needed.
2339	2206	*/
2340		- rcu_report_qs_rsp(rsp, flags); /* releases rnp->lock. */
	2207	+ rcu_report_qs_rsp(flags); /* releases rnp->lock. */
2341	2208	}
2342	2209
2343	2210	/*
2344	2211	* Record a quiescent state for all tasks that were previously queued
2345	2212	* on the specified rcu_node structure and that were blocking the current
2346		- * RCU grace period. The caller must hold the specified rnp->lock with
	2213	+ * RCU grace period. The caller must hold the corresponding rnp->lock with
2347	2214	* irqs disabled, and this lock is released upon return, but irqs remain
2348	2215	* disabled.
2349	2216	*/
2350	2217	static void __maybe_unused
2351		-rcu_report_unblock_qs_rnp(struct rcu_state *rsp,
2352		- struct rcu_node *rnp, unsigned long flags)
	2218	+rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
2353	2219	__releases(rnp->lock)
2354	2220	{
2355	2221	unsigned long gps;
..	..	@@ -2357,8 +2223,7 @@
2357	2223	struct rcu_node *rnp_p;
2358	2224
2359	2225	raw_lockdep_assert_held_rcu_node(rnp);
2360		- if (WARN_ON_ONCE(rcu_state_p == &rcu_sched_state) \|\|
2361		- WARN_ON_ONCE(rsp != rcu_state_p) \|\|
	2226	+ if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT_RCU)) \|\|
2362	2227	WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)) \|\|
2363	2228	rnp->qsmask != 0) {
2364	2229	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
..	..	@@ -2372,7 +2237,7 @@
2372	2237	* Only one rcu_node structure in the tree, so don't
2373	2238	* try to report up to its nonexistent parent!
2374	2239	*/
2375		- rcu_report_qs_rsp(rsp, flags);
	2240	+ rcu_report_qs_rsp(flags);
2376	2241	return;
2377	2242	}
2378	2243
..	..	@@ -2381,7 +2246,7 @@
2381	2246	mask = rnp->grpmask;
2382	2247	raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
2383	2248	raw_spin_lock_rcu_node(rnp_p); /* irqs already disabled. */
2384		- rcu_report_qs_rnp(mask, rsp, rnp_p, gps, flags);
	2249	+ rcu_report_qs_rnp(mask, rnp_p, gps, flags);
2385	2250	}
2386	2251
2387	2252	/*
..	..	@@ -2389,13 +2254,16 @@
2389	2254	* structure. This must be called from the specified CPU.
2390	2255	*/
2391	2256	static void
2392		-rcu_report_qs_rdp(int cpu, struct rcu_state rsp, struct rcu_data rdp)
	2257	+rcu_report_qs_rdp(struct rcu_data *rdp)
2393	2258	{
2394	2259	unsigned long flags;
2395	2260	unsigned long mask;
2396		- bool needwake;
	2261	+ bool needwake = false;
	2262	+ const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2263	+ rcu_segcblist_is_offloaded(&rdp->cblist);
2397	2264	struct rcu_node *rnp;
2398	2265
	2266	+ WARN_ON_ONCE(rdp->cpu != smp_processor_id());
2399	2267	rnp = rdp->mynode;
2400	2268	raw_spin_lock_irqsave_rcu_node(rnp, flags);
2401	2269	if (rdp->cpu_no_qs.b.norm \|\| rdp->gp_seq != rnp->gp_seq \|\|
..	..	@@ -2408,26 +2276,26 @@
2408	2276	* within the current grace period.
2409	2277	*/
2410	2278	rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */
2411		- rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
2412	2279	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2413	2280	return;
2414	2281	}
2415	2282	mask = rdp->grpmask;
	2283	+ rdp->core_needs_qs = false;
2416	2284	if ((rnp->qsmask & mask) == 0) {
2417	2285	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2418	2286	} else {
2419		- rdp->core_needs_qs = false;
2420		-
2421	2287	/*
2422	2288	* This GP can't end until cpu checks in, so all of our
2423	2289	* callbacks can be processed during the next GP.
2424	2290	*/
2425		- needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
	2291	+ if (!offloaded)
	2292	+ needwake = rcu_accelerate_cbs(rnp, rdp);
2426	2293
2427		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
	2294	+ rcu_disable_urgency_upon_qs(rdp);
	2295	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2428	2296	/* ^^^ Released rnp->lock */
2429	2297	if (needwake)
2430		- rcu_gp_kthread_wake(rsp);
	2298	+ rcu_gp_kthread_wake();
2431	2299	}
2432	2300	}
2433	2301
..	..	@@ -2438,10 +2306,10 @@
2438	2306	* quiescent state for this grace period, and record that fact if so.
2439	2307	*/
2440	2308	static void
2441		-rcu_check_quiescent_state(struct rcu_state rsp, struct rcu_data rdp)
	2309	+rcu_check_quiescent_state(struct rcu_data *rdp)
2442	2310	{
2443	2311	/* Check for grace-period ends and beginnings. */
2444		- note_gp_changes(rsp, rdp);
	2312	+ note_gp_changes(rdp);
2445	2313
2446	2314	/*
2447	2315	* Does this CPU still need to do its part for current grace period?
..	..	@@ -2461,24 +2329,26 @@
2461	2329	* Tell RCU we are done (but rcu_report_qs_rdp() will be the
2462	2330	* judge of that).
2463	2331	*/
2464		- rcu_report_qs_rdp(rdp->cpu, rsp, rdp);
	2332	+ rcu_report_qs_rdp(rdp);
2465	2333	}
2466	2334
2467	2335	/*
2468		- * Trace the fact that this CPU is going offline.
	2336	+ * Near the end of the offline process. Trace the fact that this CPU
	2337	+ * is going offline.
2469	2338	*/
2470		-static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
	2339	+int rcutree_dying_cpu(unsigned int cpu)
2471	2340	{
2472		- RCU_TRACE(bool blkd;)
2473		- RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda);)
2474		- RCU_TRACE(struct rcu_node *rnp = rdp->mynode;)
	2341	+ bool blkd;
	2342	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
	2343	+ struct rcu_node *rnp = rdp->mynode;
2475	2344
2476	2345	if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
2477		- return;
	2346	+ return 0;
2478	2347
2479		- RCU_TRACE(blkd = !!(rnp->qsmask & rdp->grpmask);)
2480		- trace_rcu_grace_period(rsp->name, rnp->gp_seq,
	2348	+ blkd = !!(rnp->qsmask & rdp->grpmask);
	2349	+ trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
2481	2350	blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));
	2351	+ return 0;
2482	2352	}
2483	2353
2484	2354	/*
..	..	@@ -2532,35 +2402,44 @@
2532	2402	* There can only be one CPU hotplug operation at a time, so no need for
2533	2403	* explicit locking.
2534	2404	*/
2535		-static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
	2405	+int rcutree_dead_cpu(unsigned int cpu)
2536	2406	{
2537		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
	2407	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
2538	2408	struct rcu_node rnp = rdp->mynode; / Outgoing CPU's rdp & rnp. */
2539	2409
2540	2410	if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
2541		- return;
	2411	+ return 0;
2542	2412
2543	2413	/* Adjust any no-longer-needed kthreads. */
2544	2414	rcu_boost_kthread_setaffinity(rnp, -1);
	2415	+ /* Do any needed no-CB deferred wakeups from this CPU. */
	2416	+ do_nocb_deferred_wakeup(per_cpu_ptr(&rcu_data, cpu));
	2417	+
	2418	+ // Stop-machine done, so allow nohz_full to disable tick.
	2419	+ tick_dep_clear(TICK_DEP_BIT_RCU);
	2420	+ return 0;
2545	2421	}
2546	2422
2547	2423	/*
2548	2424	* Invoke any RCU callbacks that have made it to the end of their grace
2549	2425	* period. Thottle as specified by rdp->blimit.
2550	2426	*/
2551		-static void rcu_do_batch(struct rcu_state rsp, struct rcu_data rdp)
	2427	+static void rcu_do_batch(struct rcu_data *rdp)
2552	2428	{
	2429	+ int div;
2553	2430	unsigned long flags;
	2431	+ const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2432	+ rcu_segcblist_is_offloaded(&rdp->cblist);
2554	2433	struct rcu_head *rhp;
2555	2434	struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
2556	2435	long bl, count;
	2436	+ long pending, tlimit = 0;
2557	2437
2558	2438	/* If no callbacks are ready, just return. */
2559	2439	if (!rcu_segcblist_ready_cbs(&rdp->cblist)) {
2560		- trace_rcu_batch_start(rsp->name,
2561		- rcu_segcblist_n_lazy_cbs(&rdp->cblist),
	2440	+ trace_rcu_batch_start(rcu_state.name,
2562	2441	rcu_segcblist_n_cbs(&rdp->cblist), 0);
2563		- trace_rcu_batch_end(rsp->name, 0,
	2442	+ trace_rcu_batch_end(rcu_state.name, 0,
2564	2443	!rcu_segcblist_empty(&rdp->cblist),
2565	2444	need_resched(), is_idle_task(current),
2566	2445	rcu_is_callbacks_kthread());
..	..	@@ -2573,32 +2452,76 @@
2573	2452	* callback counts, as rcu_barrier() needs to be conservative.
2574	2453	*/
2575	2454	local_irq_save(flags);
	2455	+ rcu_nocb_lock(rdp);
2576	2456	WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
2577		- bl = rdp->blimit;
2578		- trace_rcu_batch_start(rsp->name, rcu_segcblist_n_lazy_cbs(&rdp->cblist),
	2457	+ pending = rcu_segcblist_n_cbs(&rdp->cblist);
	2458	+ div = READ_ONCE(rcu_divisor);
	2459	+ div = div < 0 ? 7 : div > sizeof(long) * 8 - 2 ? sizeof(long) * 8 - 2 : div;
	2460	+ bl = max(rdp->blimit, pending >> div);
	2461	+ if (in_serving_softirq() && unlikely(bl > 100)) {
	2462	+ long rrn = READ_ONCE(rcu_resched_ns);
	2463	+
	2464	+ rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;
	2465	+ tlimit = local_clock() + rrn;
	2466	+ }
	2467	+ trace_rcu_batch_start(rcu_state.name,
2579	2468	rcu_segcblist_n_cbs(&rdp->cblist), bl);
2580	2469	rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl);
2581		- local_irq_restore(flags);
	2470	+ if (offloaded)
	2471	+ rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
	2472	+ rcu_nocb_unlock_irqrestore(rdp, flags);
2582	2473
2583	2474	/* Invoke callbacks. */
	2475	+ tick_dep_set_task(current, TICK_DEP_BIT_RCU);
2584	2476	rhp = rcu_cblist_dequeue(&rcl);
2585	2477	for (; rhp; rhp = rcu_cblist_dequeue(&rcl)) {
	2478	+ rcu_callback_t f;
	2479	+
2586	2480	debug_rcu_head_unqueue(rhp);
2587		- if (__rcu_reclaim(rsp->name, rhp))
2588		- rcu_cblist_dequeued_lazy(&rcl);
	2481	+
	2482	+ rcu_lock_acquire(&rcu_callback_map);
	2483	+ trace_rcu_invoke_callback(rcu_state.name, rhp);
	2484	+
	2485	+ f = rhp->func;
	2486	+ WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
	2487	+ f(rhp);
	2488	+
	2489	+ rcu_lock_release(&rcu_callback_map);
	2490	+
2589	2491	/*
2590	2492	* Stop only if limit reached and CPU has something to do.
2591	2493	* Note: The rcl structure counts down from zero.
2592	2494	*/
2593		- if (-rcl.len >= bl &&
2594		- (need_resched() \|\|
2595		- (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
2596		- break;
	2495	+ if (in_serving_softirq()) {
	2496	+ if (-rcl.len >= bl && (need_resched() \|\|
	2497	+ (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
	2498	+ break;
	2499	+
	2500	+ /*
	2501	+ * Make sure we don't spend too much time here and deprive other
	2502	+ * softirq vectors of CPU cycles.
	2503	+ */
	2504	+ if (unlikely(tlimit)) {
	2505	+ /* only call local_clock() every 32 callbacks */
	2506	+ if (likely((-rcl.len & 31) \|\| local_clock() < tlimit))
	2507	+ continue;
	2508	+ /* Exceeded the time limit, so leave. */
	2509	+ break;
	2510	+ }
	2511	+ } else {
	2512	+ local_bh_enable();
	2513	+ lockdep_assert_irqs_enabled();
	2514	+ cond_resched_tasks_rcu_qs();
	2515	+ lockdep_assert_irqs_enabled();
	2516	+ local_bh_disable();
	2517	+ }
2597	2518	}
2598	2519
2599	2520	local_irq_save(flags);
	2521	+ rcu_nocb_lock(rdp);
2600	2522	count = -rcl.len;
2601		- trace_rcu_batch_end(rsp->name, count, !!rcl.head, need_resched(),
	2523	+ rdp->n_cbs_invoked += count;
	2524	+ trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),
2602	2525	is_idle_task(current), rcu_is_callbacks_kthread());
2603	2526
2604	2527	/* Update counts and requeue any remaining callbacks. */
..	..	@@ -2608,13 +2531,13 @@
2608	2531
2609	2532	/* Reinstate batch limit if we have worked down the excess. */
2610	2533	count = rcu_segcblist_n_cbs(&rdp->cblist);
2611		- if (rdp->blimit == LONG_MAX && count <= qlowmark)
	2534	+ if (rdp->blimit >= DEFAULT_MAX_RCU_BLIMIT && count <= qlowmark)
2612	2535	rdp->blimit = blimit;
2613	2536
2614	2537	/* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */
2615	2538	if (count == 0 && rdp->qlen_last_fqs_check != 0) {
2616	2539	rdp->qlen_last_fqs_check = 0;
2617		- rdp->n_force_qs_snap = rsp->n_force_qs;
	2540	+ rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
2618	2541	} else if (count < rdp->qlen_last_fqs_check - qhimark)
2619	2542	rdp->qlen_last_fqs_check = count;
2620	2543
..	..	@@ -2622,94 +2545,72 @@
2622	2545	* The following usually indicates a double call_rcu(). To track
2623	2546	* this down, try building with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.
2624	2547	*/
2625		- WARN_ON_ONCE(rcu_segcblist_empty(&rdp->cblist) != (count == 0));
	2548	+ WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist));
	2549	+ WARN_ON_ONCE(!IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2550	+ count != 0 && rcu_segcblist_empty(&rdp->cblist));
2626	2551
2627		- local_irq_restore(flags);
	2552	+ rcu_nocb_unlock_irqrestore(rdp, flags);
2628	2553
2629	2554	/* Re-invoke RCU core processing if there are callbacks remaining. */
2630		- if (rcu_segcblist_ready_cbs(&rdp->cblist))
	2555	+ if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist))
2631	2556	invoke_rcu_core();
	2557	+ tick_dep_clear_task(current, TICK_DEP_BIT_RCU);
2632	2558	}
2633	2559
2634	2560	/*
2635		- * Check to see if this CPU is in a non-context-switch quiescent state
2636		- * (user mode or idle loop for rcu, non-softirq execution for rcu_bh).
2637		- * Also schedule RCU core processing.
2638		- *
2639		- * This function must be called from hardirq context. It is normally
2640		- * invoked from the scheduling-clock interrupt.
	2561	+ * This function is invoked from each scheduling-clock interrupt,
	2562	+ * and checks to see if this CPU is in a non-context-switch quiescent
	2563	+ * state, for example, user mode or idle loop. It also schedules RCU
	2564	+ * core processing. If the current grace period has gone on too long,
	2565	+ * it will ask the scheduler to manufacture a context switch for the sole
	2566	+ * purpose of providing a providing the needed quiescent state.
2641	2567	*/
2642		-void rcu_check_callbacks(int user)
	2568	+void rcu_sched_clock_irq(int user)
2643	2569	{
2644	2570	trace_rcu_utilization(TPS("Start scheduler-tick"));
2645		- increment_cpu_stall_ticks();
2646		- if (user \|\| rcu_is_cpu_rrupt_from_idle()) {
2647		-
2648		- /*
2649		- * Get here if this CPU took its interrupt from user
2650		- * mode or from the idle loop, and if this is not a
2651		- * nested interrupt. In this case, the CPU is in
2652		- * a quiescent state, so note it.
2653		- *
2654		- * No memory barrier is required here because both
2655		- * rcu_sched_qs() and rcu_bh_qs() reference only CPU-local
2656		- * variables that other CPUs neither access nor modify,
2657		- * at least not while the corresponding CPU is online.
2658		- */
2659		-
2660		- rcu_sched_qs();
2661		- rcu_bh_qs();
2662		- rcu_note_voluntary_context_switch(current);
2663		-
2664		- } else if (!in_softirq()) {
2665		-
2666		- /*
2667		- * Get here if this CPU did not take its interrupt from
2668		- * softirq, in other words, if it is not interrupting
2669		- * a rcu_bh read-side critical section. This is an _bh
2670		- * critical section, so note it.
2671		- */
2672		-
2673		- rcu_bh_qs();
2674		- }
2675		- rcu_preempt_check_callbacks();
	2571	+ lockdep_assert_irqs_disabled();
	2572	+ raw_cpu_inc(rcu_data.ticks_this_gp);
2676	2573	/* The load-acquire pairs with the store-release setting to true. */
2677		- if (smp_load_acquire(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs))) {
	2574	+ if (smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) {
2678	2575	/* Idle and userspace execution already are quiescent states. */
2679	2576	if (!rcu_is_cpu_rrupt_from_idle() && !user) {
2680	2577	set_tsk_need_resched(current);
2681	2578	set_preempt_need_resched();
2682	2579	}
2683		- __this_cpu_write(rcu_dynticks.rcu_urgent_qs, false);
	2580	+ __this_cpu_write(rcu_data.rcu_urgent_qs, false);
2684	2581	}
2685		- if (rcu_pending())
	2582	+ rcu_flavor_sched_clock_irq(user);
	2583	+ if (rcu_pending(user))
2686	2584	invoke_rcu_core();
	2585	+ lockdep_assert_irqs_disabled();
2687	2586
2688	2587	trace_rcu_utilization(TPS("End scheduler-tick"));
2689	2588	}
2690	2589
2691	2590	/*
2692		- * Scan the leaf rcu_node structures, processing dyntick state for any that
2693		- * have not yet encountered a quiescent state, using the function specified.
2694		- * Also initiate boosting for any threads blocked on the root rcu_node.
2695		- *
2696		- * The caller must have suppressed start of new grace periods.
	2591	+ * Scan the leaf rcu_node structures. For each structure on which all
	2592	+ * CPUs have reported a quiescent state and on which there are tasks
	2593	+ * blocking the current grace period, initiate RCU priority boosting.
	2594	+ * Otherwise, invoke the specified function to check dyntick state for
	2595	+ * each CPU that has not yet reported a quiescent state.
2697	2596	*/
2698		-static void force_qs_rnp(struct rcu_state rsp, int (f)(struct rcu_data *rsp))
	2597	+static void force_qs_rnp(int (f)(struct rcu_data rdp))
2699	2598	{
2700	2599	int cpu;
2701	2600	unsigned long flags;
2702	2601	unsigned long mask;
	2602	+ struct rcu_data *rdp;
2703	2603	struct rcu_node *rnp;
2704	2604
2705		- rcu_for_each_leaf_node(rsp, rnp) {
	2605	+ rcu_state.cbovld = rcu_state.cbovldnext;
	2606	+ rcu_state.cbovldnext = false;
	2607	+ rcu_for_each_leaf_node(rnp) {
2706	2608	cond_resched_tasks_rcu_qs();
2707	2609	mask = 0;
2708	2610	raw_spin_lock_irqsave_rcu_node(rnp, flags);
	2611	+ rcu_state.cbovldnext \|= !!rnp->cbovldmask;
2709	2612	if (rnp->qsmask == 0) {
2710		- if (rcu_state_p == &rcu_sched_state \|\|
2711		- rsp != rcu_state_p \|\|
2712		- rcu_preempt_blocked_readers_cgp(rnp)) {
	2613	+ if (rcu_preempt_blocked_readers_cgp(rnp)) {
2713	2614	/*
2714	2615	* No point in scanning bits because they
2715	2616	* are all zero. But we might need to
..	..	@@ -2722,16 +2623,16 @@
2722	2623	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2723	2624	continue;
2724	2625	}
2725		- for_each_leaf_node_possible_cpu(rnp, cpu) {
2726		- unsigned long bit = leaf_node_cpu_bit(rnp, cpu);
2727		- if ((rnp->qsmask & bit) != 0) {
2728		- if (f(per_cpu_ptr(rsp->rda, cpu)))
2729		- mask \|= bit;
	2626	+ for_each_leaf_node_cpu_mask(rnp, cpu, rnp->qsmask) {
	2627	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	2628	+ if (f(rdp)) {
	2629	+ mask \|= rdp->grpmask;
	2630	+ rcu_disable_urgency_upon_qs(rdp);
2730	2631	}
2731	2632	}
2732	2633	if (mask != 0) {
2733	2634	/* Idle/offline CPUs, report (releases rnp->lock). */
2734		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
	2635	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2735	2636	} else {
2736	2637	/* Nothing to do here, so just drop the lock. */
2737	2638	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
..	..	@@ -2743,7 +2644,7 @@
2743	2644	* Force quiescent states on reluctant CPUs, and also detect which
2744	2645	* CPUs are in dyntick-idle mode.
2745	2646	*/
2746		-static void force_quiescent_state(struct rcu_state *rsp)
	2647	+void rcu_force_quiescent_state(void)
2747	2648	{
2748	2649	unsigned long flags;
2749	2650	bool ret;
..	..	@@ -2751,169 +2652,206 @@
2751	2652	struct rcu_node *rnp_old = NULL;
2752	2653
2753	2654	/* Funnel through hierarchy to reduce memory contention. */
2754		- rnp = __this_cpu_read(rsp->rda->mynode);
	2655	+ rnp = __this_cpu_read(rcu_data.mynode);
2755	2656	for (; rnp != NULL; rnp = rnp->parent) {
2756		- ret = (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) \|\|
2757		- !raw_spin_trylock(&rnp->fqslock);
	2657	+ ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) \|\|
	2658	+ !raw_spin_trylock(&rnp->fqslock);
2758	2659	if (rnp_old != NULL)
2759	2660	raw_spin_unlock(&rnp_old->fqslock);
2760	2661	if (ret)
2761	2662	return;
2762	2663	rnp_old = rnp;
2763	2664	}
2764		- /* rnp_old == rcu_get_root(rsp), rnp == NULL. */
	2665	+ /* rnp_old == rcu_get_root(), rnp == NULL. */
2765	2666
2766	2667	/* Reached the root of the rcu_node tree, acquire lock. */
2767	2668	raw_spin_lock_irqsave_rcu_node(rnp_old, flags);
2768	2669	raw_spin_unlock(&rnp_old->fqslock);
2769		- if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
	2670	+ if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
2770	2671	raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
2771	2672	return; /* Someone beat us to it. */
2772	2673	}
2773		- WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) \| RCU_GP_FLAG_FQS);
	2674	+ WRITE_ONCE(rcu_state.gp_flags,
	2675	+ READ_ONCE(rcu_state.gp_flags) \| RCU_GP_FLAG_FQS);
2774	2676	raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
2775		- rcu_gp_kthread_wake(rsp);
	2677	+ rcu_gp_kthread_wake();
	2678	+}
	2679	+EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
	2680	+
	2681	+// Workqueue handler for an RCU reader for kernels enforcing struct RCU
	2682	+// grace periods.
	2683	+static void strict_work_handler(struct work_struct *work)
	2684	+{
	2685	+ rcu_read_lock();
	2686	+ rcu_read_unlock();
2776	2687	}
2777	2688
2778		-/*
2779		- * This function checks for grace-period requests that fail to motivate
2780		- * RCU to come out of its idle mode.
2781		- */
2782		-static void
2783		-rcu_check_gp_start_stall(struct rcu_state rsp, struct rcu_node rnp,
2784		- struct rcu_data *rdp)
2785		-{
2786		- const unsigned long gpssdelay = rcu_jiffies_till_stall_check() * HZ;
2787		- unsigned long flags;
2788		- unsigned long j;
2789		- struct rcu_node *rnp_root = rcu_get_root(rsp);
2790		- static atomic_t warned = ATOMIC_INIT(0);
2791		-
2792		- if (!IS_ENABLED(CONFIG_PROVE_RCU) \|\| rcu_gp_in_progress(rsp) \|\|
2793		- ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed))
2794		- return;
2795		- j = jiffies; /* Expensive access, and in common case don't get here. */
2796		- if (time_before(j, READ_ONCE(rsp->gp_req_activity) + gpssdelay) \|\|
2797		- time_before(j, READ_ONCE(rsp->gp_activity) + gpssdelay) \|\|
2798		- atomic_read(&warned))
2799		- return;
2800		-
2801		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
2802		- j = jiffies;
2803		- if (rcu_gp_in_progress(rsp) \|\|
2804		- ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) \|\|
2805		- time_before(j, READ_ONCE(rsp->gp_req_activity) + gpssdelay) \|\|
2806		- time_before(j, READ_ONCE(rsp->gp_activity) + gpssdelay) \|\|
2807		- atomic_read(&warned)) {
2808		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2809		- return;
2810		- }
2811		- /* Hold onto the leaf lock to make others see warned==1. */
2812		-
2813		- if (rnp_root != rnp)
2814		- raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
2815		- j = jiffies;
2816		- if (rcu_gp_in_progress(rsp) \|\|
2817		- ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) \|\|
2818		- time_before(j, rsp->gp_req_activity + gpssdelay) \|\|
2819		- time_before(j, rsp->gp_activity + gpssdelay) \|\|
2820		- atomic_xchg(&warned, 1)) {
2821		- raw_spin_unlock_rcu_node(rnp_root); /* irqs remain disabled. */
2822		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2823		- return;
2824		- }
2825		- pr_alert("%s: g%ld->%ld gar:%lu ga:%lu f%#x gs:%d %s->state:%#lx\n",
2826		- __func__, (long)READ_ONCE(rsp->gp_seq),
2827		- (long)READ_ONCE(rnp_root->gp_seq_needed),
2828		- j - rsp->gp_req_activity, j - rsp->gp_activity,
2829		- rsp->gp_flags, rsp->gp_state, rsp->name,
2830		- rsp->gp_kthread ? rsp->gp_kthread->state : 0x1ffffL);
2831		- WARN_ON(1);
2832		- if (rnp_root != rnp)
2833		- raw_spin_unlock_rcu_node(rnp_root);
2834		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2835		-}
2836		-
2837		-/*
2838		- * This does the RCU core processing work for the specified rcu_state
2839		- * and rcu_data structures. This may be called only from the CPU to
2840		- * whom the rdp belongs.
2841		- */
2842		-static void
2843		-__rcu_process_callbacks(struct rcu_state *rsp)
	2689	+/* Perform RCU core processing work for the current CPU. */
	2690	+static __latent_entropy void rcu_core(void)
2844	2691	{
2845	2692	unsigned long flags;
2846		- struct rcu_data *rdp = raw_cpu_ptr(rsp->rda);
	2693	+ struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
2847	2694	struct rcu_node *rnp = rdp->mynode;
2848		-
2849		- WARN_ON_ONCE(!rdp->beenonline);
2850		-
2851		- /* Update RCU state based on any recent quiescent states. */
2852		- rcu_check_quiescent_state(rsp, rdp);
2853		-
2854		- /* No grace period and unregistered callbacks? */
2855		- if (!rcu_gp_in_progress(rsp) &&
2856		- rcu_segcblist_is_enabled(&rdp->cblist)) {
2857		- local_irq_save(flags);
2858		- if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
2859		- rcu_accelerate_cbs_unlocked(rsp, rnp, rdp);
2860		- local_irq_restore(flags);
2861		- }
2862		-
2863		- rcu_check_gp_start_stall(rsp, rnp, rdp);
2864		-
2865		- /* If there are callbacks ready, invoke them. */
2866		- if (rcu_segcblist_ready_cbs(&rdp->cblist))
2867		- invoke_rcu_callbacks(rsp, rdp);
2868		-
2869		- /* Do any needed deferred wakeups of rcuo kthreads. */
2870		- do_nocb_deferred_wakeup(rdp);
2871		-}
2872		-
2873		-/*
2874		- * Do RCU core processing for the current CPU.
2875		- */
2876		-static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused)
2877		-{
2878		- struct rcu_state *rsp;
	2695	+ const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2696	+ rcu_segcblist_is_offloaded(&rdp->cblist);
2879	2697
2880	2698	if (cpu_is_offline(smp_processor_id()))
2881	2699	return;
2882	2700	trace_rcu_utilization(TPS("Start RCU core"));
2883		- for_each_rcu_flavor(rsp)
2884		- __rcu_process_callbacks(rsp);
	2701	+ WARN_ON_ONCE(!rdp->beenonline);
	2702	+
	2703	+ /* Report any deferred quiescent states if preemption enabled. */
	2704	+ if (!(preempt_count() & PREEMPT_MASK)) {
	2705	+ rcu_preempt_deferred_qs(current);
	2706	+ } else if (rcu_preempt_need_deferred_qs(current)) {
	2707	+ set_tsk_need_resched(current);
	2708	+ set_preempt_need_resched();
	2709	+ }
	2710	+
	2711	+ /* Update RCU state based on any recent quiescent states. */
	2712	+ rcu_check_quiescent_state(rdp);
	2713	+
	2714	+ /* No grace period and unregistered callbacks? */
	2715	+ if (!rcu_gp_in_progress() &&
	2716	+ rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) {
	2717	+ local_irq_save(flags);
	2718	+ if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
	2719	+ rcu_accelerate_cbs_unlocked(rnp, rdp);
	2720	+ local_irq_restore(flags);
	2721	+ }
	2722	+
	2723	+ rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check());
	2724	+
	2725	+ /* If there are callbacks ready, invoke them. */
	2726	+ if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) &&
	2727	+ likely(READ_ONCE(rcu_scheduler_fully_active)))
	2728	+ rcu_do_batch(rdp);
	2729	+
	2730	+ /* Do any needed deferred wakeups of rcuo kthreads. */
	2731	+ do_nocb_deferred_wakeup(rdp);
2885	2732	trace_rcu_utilization(TPS("End RCU core"));
	2733	+
	2734	+ // If strict GPs, schedule an RCU reader in a clean environment.
	2735	+ if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
	2736	+ queue_work_on(rdp->cpu, rcu_gp_wq, &rdp->strict_work);
	2737	+}
	2738	+
	2739	+static void rcu_core_si(struct softirq_action *h)
	2740	+{
	2741	+ rcu_core();
	2742	+}
	2743	+
	2744	+static void rcu_wake_cond(struct task_struct *t, int status)
	2745	+{
	2746	+ /*
	2747	+ * If the thread is yielding, only wake it when this
	2748	+ * is invoked from idle
	2749	+ */
	2750	+ if (t && (status != RCU_KTHREAD_YIELDING \|\| is_idle_task(current)))
	2751	+ wake_up_process(t);
	2752	+}
	2753	+
	2754	+static void invoke_rcu_core_kthread(void)
	2755	+{
	2756	+ struct task_struct *t;
	2757	+ unsigned long flags;
	2758	+
	2759	+ local_irq_save(flags);
	2760	+ __this_cpu_write(rcu_data.rcu_cpu_has_work, 1);
	2761	+ t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task);
	2762	+ if (t != NULL && t != current)
	2763	+ rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status));
	2764	+ local_irq_restore(flags);
2886	2765	}
2887	2766
2888	2767	/*
2889		- * Schedule RCU callback invocation. If the specified type of RCU
2890		- * does not support RCU priority boosting, just do a direct call,
2891		- * otherwise wake up the per-CPU kernel kthread. Note that because we
2892		- * are running on the current CPU with softirqs disabled, the
2893		- * rcu_cpu_kthread_task cannot disappear out from under us.
	2768	+ * Wake up this CPU's rcuc kthread to do RCU core processing.
2894	2769	*/
2895		-static void invoke_rcu_callbacks(struct rcu_state rsp, struct rcu_data rdp)
2896		-{
2897		- if (unlikely(!READ_ONCE(rcu_scheduler_fully_active)))
2898		- return;
2899		- if (likely(!rsp->boost)) {
2900		- rcu_do_batch(rsp, rdp);
2901		- return;
2902		- }
2903		- invoke_rcu_callbacks_kthread();
2904		-}
2905		-
2906	2770	static void invoke_rcu_core(void)
2907	2771	{
2908		- if (cpu_online(smp_processor_id()))
	2772	+ if (!cpu_online(smp_processor_id()))
	2773	+ return;
	2774	+ if (use_softirq)
2909	2775	raise_softirq(RCU_SOFTIRQ);
	2776	+ else
	2777	+ invoke_rcu_core_kthread();
	2778	+}
	2779	+
	2780	+static void rcu_cpu_kthread_park(unsigned int cpu)
	2781	+{
	2782	+ per_cpu(rcu_data.rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
	2783	+}
	2784	+
	2785	+static int rcu_cpu_kthread_should_run(unsigned int cpu)
	2786	+{
	2787	+ return __this_cpu_read(rcu_data.rcu_cpu_has_work);
	2788	+}
	2789	+
	2790	+/*
	2791	+ * Per-CPU kernel thread that invokes RCU callbacks. This replaces
	2792	+ * the RCU softirq used in configurations of RCU that do not support RCU
	2793	+ * priority boosting.
	2794	+ */
	2795	+static void rcu_cpu_kthread(unsigned int cpu)
	2796	+{
	2797	+ unsigned int *statusp = this_cpu_ptr(&rcu_data.rcu_cpu_kthread_status);
	2798	+ char work, *workp = this_cpu_ptr(&rcu_data.rcu_cpu_has_work);
	2799	+ int spincnt;
	2800	+
	2801	+ trace_rcu_utilization(TPS("Start CPU kthread@rcu_run"));
	2802	+ for (spincnt = 0; spincnt < 10; spincnt++) {
	2803	+ local_bh_disable();
	2804	+ *statusp = RCU_KTHREAD_RUNNING;
	2805	+ local_irq_disable();
	2806	+ work = *workp;
	2807	+ *workp = 0;
	2808	+ local_irq_enable();
	2809	+ if (work)
	2810	+ rcu_core();
	2811	+ local_bh_enable();
	2812	+ if (*workp == 0) {
	2813	+ trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
	2814	+ *statusp = RCU_KTHREAD_WAITING;
	2815	+ return;
	2816	+ }
	2817	+ }
	2818	+ *statusp = RCU_KTHREAD_YIELDING;
	2819	+ trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
	2820	+ schedule_timeout_idle(2);
	2821	+ trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
	2822	+ *statusp = RCU_KTHREAD_WAITING;
	2823	+}
	2824	+
	2825	+static struct smp_hotplug_thread rcu_cpu_thread_spec = {
	2826	+ .store = &rcu_data.rcu_cpu_kthread_task,
	2827	+ .thread_should_run = rcu_cpu_kthread_should_run,
	2828	+ .thread_fn = rcu_cpu_kthread,
	2829	+ .thread_comm = "rcuc/%u",
	2830	+ .setup = rcu_cpu_kthread_setup,
	2831	+ .park = rcu_cpu_kthread_park,
	2832	+};
	2833	+
	2834	+/*
	2835	+ * Spawn per-CPU RCU core processing kthreads.
	2836	+ */
	2837	+static int __init rcu_spawn_core_kthreads(void)
	2838	+{
	2839	+ int cpu;
	2840	+
	2841	+ for_each_possible_cpu(cpu)
	2842	+ per_cpu(rcu_data.rcu_cpu_has_work, cpu) = 0;
	2843	+ if (!IS_ENABLED(CONFIG_RCU_BOOST) && use_softirq)
	2844	+ return 0;
	2845	+ WARN_ONCE(smpboot_register_percpu_thread(&rcu_cpu_thread_spec),
	2846	+ "%s: Could not start rcuc kthread, OOM is now expected behavior\n", __func__);
	2847	+ return 0;
2910	2848	}
2911	2849
2912	2850	/*
2913	2851	* Handle any core-RCU processing required by a call_rcu() invocation.
2914	2852	*/
2915		-static void __call_rcu_core(struct rcu_state rsp, struct rcu_data rdp,
2916		- struct rcu_head *head, unsigned long flags)
	2853	+static void __call_rcu_core(struct rcu_data rdp, struct rcu_head head,
	2854	+ unsigned long flags)
2917	2855	{
2918	2856	/*
2919	2857	* If called from an extended quiescent state, invoke the RCU
..	..	@@ -2928,27 +2866,27 @@
2928	2866
2929	2867	/*
2930	2868	* Force the grace period if too many callbacks or too long waiting.
2931		- * Enforce hysteresis, and don't invoke force_quiescent_state()
	2869	+ * Enforce hysteresis, and don't invoke rcu_force_quiescent_state()
2932	2870	* if some other CPU has recently done so. Also, don't bother
2933		- * invoking force_quiescent_state() if the newly enqueued callback
	2871	+ * invoking rcu_force_quiescent_state() if the newly enqueued callback
2934	2872	* is the only one waiting for a grace period to complete.
2935	2873	*/
2936	2874	if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) >
2937	2875	rdp->qlen_last_fqs_check + qhimark)) {
2938	2876
2939	2877	/* Are we ignoring a completed grace period? */
2940		- note_gp_changes(rsp, rdp);
	2878	+ note_gp_changes(rdp);
2941	2879
2942	2880	/* Start a new grace period if one not already started. */
2943		- if (!rcu_gp_in_progress(rsp)) {
2944		- rcu_accelerate_cbs_unlocked(rsp, rdp->mynode, rdp);
	2881	+ if (!rcu_gp_in_progress()) {
	2882	+ rcu_accelerate_cbs_unlocked(rdp->mynode, rdp);
2945	2883	} else {
2946	2884	/* Give the grace period a kick. */
2947		- rdp->blimit = LONG_MAX;
2948		- if (rsp->n_force_qs == rdp->n_force_qs_snap &&
	2885	+ rdp->blimit = DEFAULT_MAX_RCU_BLIMIT;
	2886	+ if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap &&
2949	2887	rcu_segcblist_first_pend_cb(&rdp->cblist) != head)
2950		- force_quiescent_state(rsp);
2951		- rdp->n_force_qs_snap = rsp->n_force_qs;
	2888	+ rcu_force_quiescent_state();
	2889	+ rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
2952	2890	rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
2953	2891	}
2954	2892	}
..	..	@@ -2962,17 +2900,54 @@
2962	2900	}
2963	2901
2964	2902	/*
2965		- * Helper function for call_rcu() and friends. The cpu argument will
2966		- * normally be -1, indicating "currently running CPU". It may specify
2967		- * a CPU only if that CPU is a no-CBs CPU. Currently, only _rcu_barrier()
2968		- * is expected to specify a CPU.
	2903	+ * Check and if necessary update the leaf rcu_node structure's
	2904	+ * ->cbovldmask bit corresponding to the current CPU based on that CPU's
	2905	+ * number of queued RCU callbacks. The caller must hold the leaf rcu_node
	2906	+ * structure's ->lock.
2969	2907	*/
	2908	+static void check_cb_ovld_locked(struct rcu_data rdp, struct rcu_node rnp)
	2909	+{
	2910	+ raw_lockdep_assert_held_rcu_node(rnp);
	2911	+ if (qovld_calc <= 0)
	2912	+ return; // Early boot and wildcard value set.
	2913	+ if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc)
	2914	+ WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask \| rdp->grpmask);
	2915	+ else
	2916	+ WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask & ~rdp->grpmask);
	2917	+}
	2918	+
	2919	+/*
	2920	+ * Check and if necessary update the leaf rcu_node structure's
	2921	+ * ->cbovldmask bit corresponding to the current CPU based on that CPU's
	2922	+ * number of queued RCU callbacks. No locks need be held, but the
	2923	+ * caller must have disabled interrupts.
	2924	+ *
	2925	+ * Note that this function ignores the possibility that there are a lot
	2926	+ * of callbacks all of which have already seen the end of their respective
	2927	+ * grace periods. This omission is due to the need for no-CBs CPUs to
	2928	+ * be holding ->nocb_lock to do this check, which is too heavy for a
	2929	+ * common-case operation.
	2930	+ */
	2931	+static void check_cb_ovld(struct rcu_data *rdp)
	2932	+{
	2933	+ struct rcu_node *const rnp = rdp->mynode;
	2934	+
	2935	+ if (qovld_calc <= 0 \|\|
	2936	+ ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) ==
	2937	+ !!(READ_ONCE(rnp->cbovldmask) & rdp->grpmask)))
	2938	+ return; // Early boot wildcard value or already set correctly.
	2939	+ raw_spin_lock_rcu_node(rnp);
	2940	+ check_cb_ovld_locked(rdp, rnp);
	2941	+ raw_spin_unlock_rcu_node(rnp);
	2942	+}
	2943	+
	2944	+/* Helper function for call_rcu() and friends. */
2970	2945	static void
2971		-__call_rcu(struct rcu_head *head, rcu_callback_t func,
2972		- struct rcu_state *rsp, int cpu, bool lazy)
	2946	+__call_rcu(struct rcu_head *head, rcu_callback_t func)
2973	2947	{
2974	2948	unsigned long flags;
2975	2949	struct rcu_data *rdp;
	2950	+ bool was_alldone;
2976	2951
2977	2952	/* Misaligned rcu_head! */
2978	2953	WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1));
..	..	@@ -2983,7 +2958,7 @@
2983	2958	* Use rcu:rcu_callback trace event to find the previous
2984	2959	* time callback was passed to __call_rcu().
2985	2960	*/
2986		- WARN_ONCE(1, "__call_rcu(): Double-freed CB %p->%pF()!!!\n",
	2961	+ WARN_ONCE(1, "__call_rcu(): Double-freed CB %p->%pS()!!!\n",
2987	2962	head, head->func);
2988	2963	WRITE_ONCE(head->func, rcu_leak_callback);
2989	2964	return;
..	..	@@ -2991,130 +2966,671 @@
2991	2966	head->func = func;
2992	2967	head->next = NULL;
2993	2968	local_irq_save(flags);
2994		- rdp = this_cpu_ptr(rsp->rda);
	2969	+ kasan_record_aux_stack(head);
	2970	+ rdp = this_cpu_ptr(&rcu_data);
2995	2971
2996	2972	/* Add the callback to our list. */
2997		- if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist)) \|\| cpu != -1) {
2998		- int offline;
2999		-
3000		- if (cpu != -1)
3001		- rdp = per_cpu_ptr(rsp->rda, cpu);
3002		- if (likely(rdp->mynode)) {
3003		- /* Post-boot, so this should be for a no-CBs CPU. */
3004		- offline = !__call_rcu_nocb(rdp, head, lazy, flags);
3005		- WARN_ON_ONCE(offline);
3006		- /* Offline CPU, _call_rcu() illegal, leak callback. */
3007		- local_irq_restore(flags);
3008		- return;
3009		- }
3010		- /*
3011		- * Very early boot, before rcu_init(). Initialize if needed
3012		- * and then drop through to queue the callback.
3013		- */
3014		- BUG_ON(cpu != -1);
	2973	+ if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) {
	2974	+ // This can trigger due to call_rcu() from offline CPU:
	2975	+ WARN_ON_ONCE(rcu_scheduler_active != RCU_SCHEDULER_INACTIVE);
3015	2976	WARN_ON_ONCE(!rcu_is_watching());
	2977	+ // Very early boot, before rcu_init(). Initialize if needed
	2978	+ // and then drop through to queue the callback.
3016	2979	if (rcu_segcblist_empty(&rdp->cblist))
3017	2980	rcu_segcblist_init(&rdp->cblist);
3018	2981	}
3019		- rcu_segcblist_enqueue(&rdp->cblist, head, lazy);
3020		- if (!lazy)
3021		- rcu_idle_count_callbacks_posted();
3022	2982
3023		- if (__is_kfree_rcu_offset((unsigned long)func))
3024		- trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func,
3025		- rcu_segcblist_n_lazy_cbs(&rdp->cblist),
	2983	+ check_cb_ovld(rdp);
	2984	+ if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags))
	2985	+ return; // Enqueued onto ->nocb_bypass, so just leave.
	2986	+ // If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock.
	2987	+ rcu_segcblist_enqueue(&rdp->cblist, head);
	2988	+ if (__is_kvfree_rcu_offset((unsigned long)func))
	2989	+ trace_rcu_kvfree_callback(rcu_state.name, head,
	2990	+ (unsigned long)func,
3026	2991	rcu_segcblist_n_cbs(&rdp->cblist));
3027	2992	else
3028		- trace_rcu_callback(rsp->name, head,
3029		- rcu_segcblist_n_lazy_cbs(&rdp->cblist),
	2993	+ trace_rcu_callback(rcu_state.name, head,
3030	2994	rcu_segcblist_n_cbs(&rdp->cblist));
3031	2995
3032	2996	/* Go handle any RCU core processing required. */
3033		- __call_rcu_core(rsp, rdp, head, flags);
	2997	+ if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2998	+ unlikely(rcu_segcblist_is_offloaded(&rdp->cblist))) {
	2999	+ __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */
	3000	+ } else {
	3001	+ __call_rcu_core(rdp, head, flags);
	3002	+ local_irq_restore(flags);
	3003	+ }
	3004	+}
	3005	+
	3006	+/**
	3007	+ * call_rcu() - Queue an RCU callback for invocation after a grace period.
	3008	+ * @head: structure to be used for queueing the RCU updates.
	3009	+ * @func: actual callback function to be invoked after the grace period
	3010	+ *
	3011	+ * The callback function will be invoked some time after a full grace
	3012	+ * period elapses, in other words after all pre-existing RCU read-side
	3013	+ * critical sections have completed. However, the callback function
	3014	+ * might well execute concurrently with RCU read-side critical sections
	3015	+ * that started after call_rcu() was invoked. RCU read-side critical
	3016	+ * sections are delimited by rcu_read_lock() and rcu_read_unlock(), and
	3017	+ * may be nested. In addition, regions of code across which interrupts,
	3018	+ * preemption, or softirqs have been disabled also serve as RCU read-side
	3019	+ * critical sections. This includes hardware interrupt handlers, softirq
	3020	+ * handlers, and NMI handlers.
	3021	+ *
	3022	+ * Note that all CPUs must agree that the grace period extended beyond
	3023	+ * all pre-existing RCU read-side critical section. On systems with more
	3024	+ * than one CPU, this means that when "func()" is invoked, each CPU is
	3025	+ * guaranteed to have executed a full memory barrier since the end of its
	3026	+ * last RCU read-side critical section whose beginning preceded the call
	3027	+ * to call_rcu(). It also means that each CPU executing an RCU read-side
	3028	+ * critical section that continues beyond the start of "func()" must have
	3029	+ * executed a memory barrier after the call_rcu() but before the beginning
	3030	+ * of that RCU read-side critical section. Note that these guarantees
	3031	+ * include CPUs that are offline, idle, or executing in user mode, as
	3032	+ * well as CPUs that are executing in the kernel.
	3033	+ *
	3034	+ * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
	3035	+ * resulting RCU callback function "func()", then both CPU A and CPU B are
	3036	+ * guaranteed to execute a full memory barrier during the time interval
	3037	+ * between the call to call_rcu() and the invocation of "func()" -- even
	3038	+ * if CPU A and CPU B are the same CPU (but again only if the system has
	3039	+ * more than one CPU).
	3040	+ */
	3041	+void call_rcu(struct rcu_head *head, rcu_callback_t func)
	3042	+{
	3043	+ __call_rcu(head, func);
	3044	+}
	3045	+EXPORT_SYMBOL_GPL(call_rcu);
	3046	+
	3047	+
	3048	+/* Maximum number of jiffies to wait before draining a batch. */
	3049	+#define KFREE_DRAIN_JIFFIES (HZ / 50)
	3050	+#define KFREE_N_BATCHES 2
	3051	+#define FREE_N_CHANNELS 2
	3052	+
	3053	+/**
	3054	+ * struct kvfree_rcu_bulk_data - single block to store kvfree_rcu() pointers
	3055	+ * @nr_records: Number of active pointers in the array
	3056	+ * @next: Next bulk object in the block chain
	3057	+ * @records: Array of the kvfree_rcu() pointers
	3058	+ */
	3059	+struct kvfree_rcu_bulk_data {
	3060	+ unsigned long nr_records;
	3061	+ struct kvfree_rcu_bulk_data *next;
	3062	+ void *records[];
	3063	+};
	3064	+
	3065	+/*
	3066	+ * This macro defines how many entries the "records" array
	3067	+ * will contain. It is based on the fact that the size of
	3068	+ * kvfree_rcu_bulk_data structure becomes exactly one page.
	3069	+ */
	3070	+#define KVFREE_BULK_MAX_ENTR \
	3071	+ ((PAGE_SIZE - sizeof(struct kvfree_rcu_bulk_data)) / sizeof(void *))
	3072	+
	3073	+/**
	3074	+ * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests
	3075	+ * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period
	3076	+ * @head_free: List of kfree_rcu() objects waiting for a grace period
	3077	+ * @bkvhead_free: Bulk-List of kvfree_rcu() objects waiting for a grace period
	3078	+ * @krcp: Pointer to @kfree_rcu_cpu structure
	3079	+ */
	3080	+
	3081	+struct kfree_rcu_cpu_work {
	3082	+ struct rcu_work rcu_work;
	3083	+ struct rcu_head *head_free;
	3084	+ struct kvfree_rcu_bulk_data *bkvhead_free[FREE_N_CHANNELS];
	3085	+ struct kfree_rcu_cpu *krcp;
	3086	+};
	3087	+
	3088	+/**
	3089	+ * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period
	3090	+ * @head: List of kfree_rcu() objects not yet waiting for a grace period
	3091	+ * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period
	3092	+ * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period
	3093	+ * @lock: Synchronize access to this structure
	3094	+ * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES
	3095	+ * @monitor_todo: Tracks whether a @monitor_work delayed work is pending
	3096	+ * @initialized: The @rcu_work fields have been initialized
	3097	+ * @count: Number of objects for which GP not started
	3098	+ * @bkvcache:
	3099	+ * A simple cache list that contains objects for reuse purpose.
	3100	+ * In order to save some per-cpu space the list is singular.
	3101	+ * Even though it is lockless an access has to be protected by the
	3102	+ * per-cpu lock.
	3103	+ * @page_cache_work: A work to refill the cache when it is empty
	3104	+ * @work_in_progress: Indicates that page_cache_work is running
	3105	+ * @hrtimer: A hrtimer for scheduling a page_cache_work
	3106	+ * @nr_bkv_objs: number of allocated objects at @bkvcache.
	3107	+ *
	3108	+ * This is a per-CPU structure. The reason that it is not included in
	3109	+ * the rcu_data structure is to permit this code to be extracted from
	3110	+ * the RCU files. Such extraction could allow further optimization of
	3111	+ * the interactions with the slab allocators.
	3112	+ */
	3113	+struct kfree_rcu_cpu {
	3114	+ struct rcu_head *head;
	3115	+ struct kvfree_rcu_bulk_data *bkvhead[FREE_N_CHANNELS];
	3116	+ struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES];
	3117	+ raw_spinlock_t lock;
	3118	+ struct delayed_work monitor_work;
	3119	+ bool monitor_todo;
	3120	+ bool initialized;
	3121	+ int count;
	3122	+
	3123	+ struct work_struct page_cache_work;
	3124	+ atomic_t work_in_progress;
	3125	+ struct hrtimer hrtimer;
	3126	+
	3127	+ struct llist_head bkvcache;
	3128	+ int nr_bkv_objs;
	3129	+};
	3130	+
	3131	+static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc) = {
	3132	+ .lock = __RAW_SPIN_LOCK_UNLOCKED(krc.lock),
	3133	+};
	3134	+
	3135	+static __always_inline void
	3136	+debug_rcu_bhead_unqueue(struct kvfree_rcu_bulk_data *bhead)
	3137	+{
	3138	+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
	3139	+ int i;
	3140	+
	3141	+ for (i = 0; i < bhead->nr_records; i++)
	3142	+ debug_rcu_head_unqueue((struct rcu_head *)(bhead->records[i]));
	3143	+#endif
	3144	+}
	3145	+
	3146	+static inline struct kfree_rcu_cpu *
	3147	+krc_this_cpu_lock(unsigned long *flags)
	3148	+{
	3149	+ struct kfree_rcu_cpu *krcp;
	3150	+
	3151	+ local_irq_save(*flags); // For safely calling this_cpu_ptr().
	3152	+ krcp = this_cpu_ptr(&krc);
	3153	+ raw_spin_lock(&krcp->lock);
	3154	+
	3155	+ return krcp;
	3156	+}
	3157	+
	3158	+static inline void
	3159	+krc_this_cpu_unlock(struct kfree_rcu_cpu *krcp, unsigned long flags)
	3160	+{
	3161	+ raw_spin_unlock(&krcp->lock);
3034	3162	local_irq_restore(flags);
3035	3163	}
3036	3164
3037		-/**
3038		- * call_rcu_sched() - Queue an RCU for invocation after sched grace period.
3039		- * @head: structure to be used for queueing the RCU updates.
3040		- * @func: actual callback function to be invoked after the grace period
3041		- *
3042		- * The callback function will be invoked some time after a full grace
3043		- * period elapses, in other words after all currently executing RCU
3044		- * read-side critical sections have completed. call_rcu_sched() assumes
3045		- * that the read-side critical sections end on enabling of preemption
3046		- * or on voluntary preemption.
3047		- * RCU read-side critical sections are delimited by:
3048		- *
3049		- * - rcu_read_lock_sched() and rcu_read_unlock_sched(), OR
3050		- * - anything that disables preemption.
3051		- *
3052		- * These may be nested.
3053		- *
3054		- * See the description of call_rcu() for more detailed information on
3055		- * memory ordering guarantees.
3056		- */
3057		-void call_rcu_sched(struct rcu_head *head, rcu_callback_t func)
	3165	+static inline struct kvfree_rcu_bulk_data *
	3166	+get_cached_bnode(struct kfree_rcu_cpu *krcp)
3058	3167	{
3059		- __call_rcu(head, func, &rcu_sched_state, -1, 0);
3060		-}
3061		-EXPORT_SYMBOL_GPL(call_rcu_sched);
	3168	+ if (!krcp->nr_bkv_objs)
	3169	+ return NULL;
3062	3170
3063		-/**
3064		- * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
3065		- * @head: structure to be used for queueing the RCU updates.
3066		- * @func: actual callback function to be invoked after the grace period
3067		- *
3068		- * The callback function will be invoked some time after a full grace
3069		- * period elapses, in other words after all currently executing RCU
3070		- * read-side critical sections have completed. call_rcu_bh() assumes
3071		- * that the read-side critical sections end on completion of a softirq
3072		- * handler. This means that read-side critical sections in process
3073		- * context must not be interrupted by softirqs. This interface is to be
3074		- * used when most of the read-side critical sections are in softirq context.
3075		- * RCU read-side critical sections are delimited by:
3076		- *
3077		- * - rcu_read_lock() and rcu_read_unlock(), if in interrupt context, OR
3078		- * - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
3079		- *
3080		- * These may be nested.
3081		- *
3082		- * See the description of call_rcu() for more detailed information on
3083		- * memory ordering guarantees.
3084		- */
3085		-void call_rcu_bh(struct rcu_head *head, rcu_callback_t func)
3086		-{
3087		- __call_rcu(head, func, &rcu_bh_state, -1, 0);
	3171	+ krcp->nr_bkv_objs--;
	3172	+ return (struct kvfree_rcu_bulk_data *)
	3173	+ llist_del_first(&krcp->bkvcache);
3088	3174	}
3089		-EXPORT_SYMBOL_GPL(call_rcu_bh);
	3175	+
	3176	+static inline bool
	3177	+put_cached_bnode(struct kfree_rcu_cpu *krcp,
	3178	+ struct kvfree_rcu_bulk_data *bnode)
	3179	+{
	3180	+ // Check the limit.
	3181	+ if (krcp->nr_bkv_objs >= rcu_min_cached_objs)
	3182	+ return false;
	3183	+
	3184	+ llist_add((struct llist_node *) bnode, &krcp->bkvcache);
	3185	+ krcp->nr_bkv_objs++;
	3186	+ return true;
	3187	+
	3188	+}
3090	3189
3091	3190	/*
3092		- * Queue an RCU callback for lazy invocation after a grace period.
3093		- * This will likely be later named something like "call_rcu_lazy()",
3094		- * but this change will require some way of tagging the lazy RCU
3095		- * callbacks in the list of pending callbacks. Until then, this
3096		- * function may only be called from __kfree_rcu().
	3191	+ * This function is invoked in workqueue context after a grace period.
	3192	+ * It frees all the objects queued on ->bhead_free or ->head_free.
3097	3193	*/
3098		-void kfree_call_rcu(struct rcu_head *head,
3099		- rcu_callback_t func)
	3194	+static void kfree_rcu_work(struct work_struct *work)
3100	3195	{
3101		- __call_rcu(head, func, rcu_state_p, -1, 1);
	3196	+ unsigned long flags;
	3197	+ struct kvfree_rcu_bulk_data bkvhead[FREE_N_CHANNELS], bnext;
	3198	+ struct rcu_head head, next;
	3199	+ struct kfree_rcu_cpu *krcp;
	3200	+ struct kfree_rcu_cpu_work *krwp;
	3201	+ int i, j;
	3202	+
	3203	+ krwp = container_of(to_rcu_work(work),
	3204	+ struct kfree_rcu_cpu_work, rcu_work);
	3205	+ krcp = krwp->krcp;
	3206	+
	3207	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3208	+ // Channels 1 and 2.
	3209	+ for (i = 0; i < FREE_N_CHANNELS; i++) {
	3210	+ bkvhead[i] = krwp->bkvhead_free[i];
	3211	+ krwp->bkvhead_free[i] = NULL;
	3212	+ }
	3213	+
	3214	+ // Channel 3.
	3215	+ head = krwp->head_free;
	3216	+ krwp->head_free = NULL;
	3217	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3218	+
	3219	+ // Handle two first channels.
	3220	+ for (i = 0; i < FREE_N_CHANNELS; i++) {
	3221	+ for (; bkvhead[i]; bkvhead[i] = bnext) {
	3222	+ bnext = bkvhead[i]->next;
	3223	+ debug_rcu_bhead_unqueue(bkvhead[i]);
	3224	+
	3225	+ rcu_lock_acquire(&rcu_callback_map);
	3226	+ if (i == 0) { // kmalloc() / kfree().
	3227	+ trace_rcu_invoke_kfree_bulk_callback(
	3228	+ rcu_state.name, bkvhead[i]->nr_records,
	3229	+ bkvhead[i]->records);
	3230	+
	3231	+ kfree_bulk(bkvhead[i]->nr_records,
	3232	+ bkvhead[i]->records);
	3233	+ } else { // vmalloc() / vfree().
	3234	+ for (j = 0; j < bkvhead[i]->nr_records; j++) {
	3235	+ trace_rcu_invoke_kvfree_callback(
	3236	+ rcu_state.name,
	3237	+ bkvhead[i]->records[j], 0);
	3238	+
	3239	+ vfree(bkvhead[i]->records[j]);
	3240	+ }
	3241	+ }
	3242	+ rcu_lock_release(&rcu_callback_map);
	3243	+
	3244	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3245	+ if (put_cached_bnode(krcp, bkvhead[i]))
	3246	+ bkvhead[i] = NULL;
	3247	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3248	+
	3249	+ if (bkvhead[i])
	3250	+ free_page((unsigned long) bkvhead[i]);
	3251	+
	3252	+ cond_resched_tasks_rcu_qs();
	3253	+ }
	3254	+ }
	3255	+
	3256	+ /*
	3257	+ * Emergency case only. It can happen under low memory
	3258	+ * condition when an allocation gets failed, so the "bulk"
	3259	+ * path can not be temporary maintained.
	3260	+ */
	3261	+ for (; head; head = next) {
	3262	+ unsigned long offset = (unsigned long)head->func;
	3263	+ void ptr = (void )head - offset;
	3264	+
	3265	+ next = head->next;
	3266	+ debug_rcu_head_unqueue((struct rcu_head *)ptr);
	3267	+ rcu_lock_acquire(&rcu_callback_map);
	3268	+ trace_rcu_invoke_kvfree_callback(rcu_state.name, head, offset);
	3269	+
	3270	+ if (!WARN_ON_ONCE(!__is_kvfree_rcu_offset(offset)))
	3271	+ kvfree(ptr);
	3272	+
	3273	+ rcu_lock_release(&rcu_callback_map);
	3274	+ cond_resched_tasks_rcu_qs();
	3275	+ }
3102	3276	}
3103		-EXPORT_SYMBOL_GPL(kfree_call_rcu);
3104	3277
3105	3278	/*
3106		- * Because a context switch is a grace period for RCU-sched and RCU-bh,
3107		- * any blocking grace-period wait automatically implies a grace period
3108		- * if there is only one CPU online at any point time during execution
3109		- * of either synchronize_sched() or synchronize_rcu_bh(). It is OK to
	3279	+ * Schedule the kfree batch RCU work to run in workqueue context after a GP.
	3280	+ *
	3281	+ * This function is invoked by kfree_rcu_monitor() when the KFREE_DRAIN_JIFFIES
	3282	+ * timeout has been reached.
	3283	+ */
	3284	+static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp)
	3285	+{
	3286	+ struct kfree_rcu_cpu_work *krwp;
	3287	+ bool repeat = false;
	3288	+ int i, j;
	3289	+
	3290	+ lockdep_assert_held(&krcp->lock);
	3291	+
	3292	+ for (i = 0; i < KFREE_N_BATCHES; i++) {
	3293	+ krwp = &(krcp->krw_arr[i]);
	3294	+
	3295	+ /*
	3296	+ * Try to detach bkvhead or head and attach it over any
	3297	+ * available corresponding free channel. It can be that
	3298	+ * a previous RCU batch is in progress, it means that
	3299	+ * immediately to queue another one is not possible so
	3300	+ * return false to tell caller to retry.
	3301	+ */
	3302	+ if ((krcp->bkvhead[0] && !krwp->bkvhead_free[0]) \|\|
	3303	+ (krcp->bkvhead[1] && !krwp->bkvhead_free[1]) \|\|
	3304	+ (krcp->head && !krwp->head_free)) {
	3305	+ // Channel 1 corresponds to SLAB ptrs.
	3306	+ // Channel 2 corresponds to vmalloc ptrs.
	3307	+ for (j = 0; j < FREE_N_CHANNELS; j++) {
	3308	+ if (!krwp->bkvhead_free[j]) {
	3309	+ krwp->bkvhead_free[j] = krcp->bkvhead[j];
	3310	+ krcp->bkvhead[j] = NULL;
	3311	+ }
	3312	+ }
	3313	+
	3314	+ // Channel 3 corresponds to emergency path.
	3315	+ if (!krwp->head_free) {
	3316	+ krwp->head_free = krcp->head;
	3317	+ krcp->head = NULL;
	3318	+ }
	3319	+
	3320	+ WRITE_ONCE(krcp->count, 0);
	3321	+
	3322	+ /*
	3323	+ * One work is per one batch, so there are three
	3324	+ * "free channels", the batch can handle. It can
	3325	+ * be that the work is in the pending state when
	3326	+ * channels have been detached following by each
	3327	+ * other.
	3328	+ */
	3329	+ queue_rcu_work(system_wq, &krwp->rcu_work);
	3330	+ }
	3331	+
	3332	+ // Repeat if any "free" corresponding channel is still busy.
	3333	+ if (krcp->bkvhead[0] \|\| krcp->bkvhead[1] \|\| krcp->head)
	3334	+ repeat = true;
	3335	+ }
	3336	+
	3337	+ return !repeat;
	3338	+}
	3339	+
	3340	+static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu *krcp,
	3341	+ unsigned long flags)
	3342	+{
	3343	+ // Attempt to start a new batch.
	3344	+ krcp->monitor_todo = false;
	3345	+ if (queue_kfree_rcu_work(krcp)) {
	3346	+ // Success! Our job is done here.
	3347	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3348	+ return;
	3349	+ }
	3350	+
	3351	+ // Previous RCU batch still in progress, try again later.
	3352	+ krcp->monitor_todo = true;
	3353	+ schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
	3354	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3355	+}
	3356	+
	3357	+/*
	3358	+ * This function is invoked after the KFREE_DRAIN_JIFFIES timeout.
	3359	+ * It invokes kfree_rcu_drain_unlock() to attempt to start another batch.
	3360	+ */
	3361	+static void kfree_rcu_monitor(struct work_struct *work)
	3362	+{
	3363	+ unsigned long flags;
	3364	+ struct kfree_rcu_cpu *krcp = container_of(work, struct kfree_rcu_cpu,
	3365	+ monitor_work.work);
	3366	+
	3367	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3368	+ if (krcp->monitor_todo)
	3369	+ kfree_rcu_drain_unlock(krcp, flags);
	3370	+ else
	3371	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3372	+}
	3373	+
	3374	+static enum hrtimer_restart
	3375	+schedule_page_work_fn(struct hrtimer *t)
	3376	+{
	3377	+ struct kfree_rcu_cpu *krcp =
	3378	+ container_of(t, struct kfree_rcu_cpu, hrtimer);
	3379	+
	3380	+ queue_work(system_highpri_wq, &krcp->page_cache_work);
	3381	+ return HRTIMER_NORESTART;
	3382	+}
	3383	+
	3384	+static void fill_page_cache_func(struct work_struct *work)
	3385	+{
	3386	+ struct kvfree_rcu_bulk_data *bnode;
	3387	+ struct kfree_rcu_cpu *krcp =
	3388	+ container_of(work, struct kfree_rcu_cpu,
	3389	+ page_cache_work);
	3390	+ unsigned long flags;
	3391	+ bool pushed;
	3392	+ int i;
	3393	+
	3394	+ for (i = 0; i < rcu_min_cached_objs; i++) {
	3395	+ bnode = (struct kvfree_rcu_bulk_data *)
	3396	+ __get_free_page(GFP_KERNEL \| __GFP_NORETRY \| __GFP_NOMEMALLOC \| __GFP_NOWARN);
	3397	+
	3398	+ if (!bnode)
	3399	+ break;
	3400	+
	3401	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3402	+ pushed = put_cached_bnode(krcp, bnode);
	3403	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3404	+
	3405	+ if (!pushed) {
	3406	+ free_page((unsigned long) bnode);
	3407	+ break;
	3408	+ }
	3409	+ }
	3410	+
	3411	+ atomic_set(&krcp->work_in_progress, 0);
	3412	+}
	3413	+
	3414	+static void
	3415	+run_page_cache_worker(struct kfree_rcu_cpu *krcp)
	3416	+{
	3417	+ if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
	3418	+ !atomic_xchg(&krcp->work_in_progress, 1)) {
	3419	+ hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC,
	3420	+ HRTIMER_MODE_REL);
	3421	+ krcp->hrtimer.function = schedule_page_work_fn;
	3422	+ hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
	3423	+ }
	3424	+}
	3425	+
	3426	+static inline bool
	3427	+kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu krcp, void ptr)
	3428	+{
	3429	+ struct kvfree_rcu_bulk_data *bnode;
	3430	+ int idx;
	3431	+
	3432	+ if (unlikely(!krcp->initialized))
	3433	+ return false;
	3434	+
	3435	+ lockdep_assert_held(&krcp->lock);
	3436	+ idx = !!is_vmalloc_addr(ptr);
	3437	+
	3438	+ /* Check if a new block is required. */
	3439	+ if (!krcp->bkvhead[idx] \|\|
	3440	+ krcp->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
	3441	+ bnode = get_cached_bnode(krcp);
	3442	+ /* Switch to emergency path. */
	3443	+ if (!bnode)
	3444	+ return false;
	3445	+
	3446	+ /* Initialize the new block. */
	3447	+ bnode->nr_records = 0;
	3448	+ bnode->next = krcp->bkvhead[idx];
	3449	+
	3450	+ /* Attach it to the head. */
	3451	+ krcp->bkvhead[idx] = bnode;
	3452	+ }
	3453	+
	3454	+ /* Finally insert. */
	3455	+ krcp->bkvhead[idx]->records
	3456	+ [krcp->bkvhead[idx]->nr_records++] = ptr;
	3457	+
	3458	+ return true;
	3459	+}
	3460	+
	3461	+/*
	3462	+ * Queue a request for lazy invocation of appropriate free routine after a
	3463	+ * grace period. Please note there are three paths are maintained, two are the
	3464	+ * main ones that use array of pointers interface and third one is emergency
	3465	+ * one, that is used only when the main path can not be maintained temporary,
	3466	+ * due to memory pressure.
	3467	+ *
	3468	+ * Each kvfree_call_rcu() request is added to a batch. The batch will be drained
	3469	+ * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch will
	3470	+ * be free'd in workqueue context. This allows us to: batch requests together to
	3471	+ * reduce the number of grace periods during heavy kfree_rcu()/kvfree_rcu() load.
	3472	+ */
	3473	+void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
	3474	+{
	3475	+ unsigned long flags;
	3476	+ struct kfree_rcu_cpu *krcp;
	3477	+ bool success;
	3478	+ void *ptr;
	3479	+
	3480	+ if (head) {
	3481	+ ptr = (void *) head - (unsigned long) func;
	3482	+ } else {
	3483	+ /*
	3484	+ * Please note there is a limitation for the head-less
	3485	+ * variant, that is why there is a clear rule for such
	3486	+ * objects: it can be used from might_sleep() context
	3487	+ * only. For other places please embed an rcu_head to
	3488	+ * your data.
	3489	+ */
	3490	+ might_sleep();
	3491	+ ptr = (unsigned long *) func;
	3492	+ }
	3493	+
	3494	+ krcp = krc_this_cpu_lock(&flags);
	3495	+
	3496	+ // Queue the object but don't yet schedule the batch.
	3497	+ if (debug_rcu_head_queue(ptr)) {
	3498	+ // Probable double kfree_rcu(), just leak.
	3499	+ WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n",
	3500	+ __func__, head);
	3501	+
	3502	+ // Mark as success and leave.
	3503	+ success = true;
	3504	+ goto unlock_return;
	3505	+ }
	3506	+
	3507	+ success = kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr);
	3508	+ if (!success) {
	3509	+ run_page_cache_worker(krcp);
	3510	+
	3511	+ if (head == NULL)
	3512	+ // Inline if kvfree_rcu(one_arg) call.
	3513	+ goto unlock_return;
	3514	+
	3515	+ head->func = func;
	3516	+ head->next = krcp->head;
	3517	+ krcp->head = head;
	3518	+ success = true;
	3519	+ }
	3520	+
	3521	+ WRITE_ONCE(krcp->count, krcp->count + 1);
	3522	+
	3523	+ // Set timer to drain after KFREE_DRAIN_JIFFIES.
	3524	+ if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
	3525	+ !krcp->monitor_todo) {
	3526	+ krcp->monitor_todo = true;
	3527	+ schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
	3528	+ }
	3529	+
	3530	+unlock_return:
	3531	+ krc_this_cpu_unlock(krcp, flags);
	3532	+
	3533	+ /*
	3534	+ * Inline kvfree() after synchronize_rcu(). We can do
	3535	+ * it from might_sleep() context only, so the current
	3536	+ * CPU can pass the QS state.
	3537	+ */
	3538	+ if (!success) {
	3539	+ debug_rcu_head_unqueue((struct rcu_head *) ptr);
	3540	+ synchronize_rcu();
	3541	+ kvfree(ptr);
	3542	+ }
	3543	+}
	3544	+EXPORT_SYMBOL_GPL(kvfree_call_rcu);
	3545	+
	3546	+static unsigned long
	3547	+kfree_rcu_shrink_count(struct shrinker shrink, struct shrink_control sc)
	3548	+{
	3549	+ int cpu;
	3550	+ unsigned long count = 0;
	3551	+
	3552	+ /* Snapshot count of all CPUs */
	3553	+ for_each_possible_cpu(cpu) {
	3554	+ struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
	3555	+
	3556	+ count += READ_ONCE(krcp->count);
	3557	+ }
	3558	+
	3559	+ return count;
	3560	+}
	3561	+
	3562	+static unsigned long
	3563	+kfree_rcu_shrink_scan(struct shrinker shrink, struct shrink_control sc)
	3564	+{
	3565	+ int cpu, freed = 0;
	3566	+ unsigned long flags;
	3567	+
	3568	+ for_each_possible_cpu(cpu) {
	3569	+ int count;
	3570	+ struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
	3571	+
	3572	+ count = krcp->count;
	3573	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3574	+ if (krcp->monitor_todo)
	3575	+ kfree_rcu_drain_unlock(krcp, flags);
	3576	+ else
	3577	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3578	+
	3579	+ sc->nr_to_scan -= count;
	3580	+ freed += count;
	3581	+
	3582	+ if (sc->nr_to_scan <= 0)
	3583	+ break;
	3584	+ }
	3585	+
	3586	+ return freed == 0 ? SHRINK_STOP : freed;
	3587	+}
	3588	+
	3589	+static struct shrinker kfree_rcu_shrinker = {
	3590	+ .count_objects = kfree_rcu_shrink_count,
	3591	+ .scan_objects = kfree_rcu_shrink_scan,
	3592	+ .batch = 0,
	3593	+ .seeks = DEFAULT_SEEKS,
	3594	+};
	3595	+
	3596	+void __init kfree_rcu_scheduler_running(void)
	3597	+{
	3598	+ int cpu;
	3599	+ unsigned long flags;
	3600	+
	3601	+ for_each_possible_cpu(cpu) {
	3602	+ struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
	3603	+
	3604	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3605	+ if (!krcp->head \|\| krcp->monitor_todo) {
	3606	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3607	+ continue;
	3608	+ }
	3609	+ krcp->monitor_todo = true;
	3610	+ schedule_delayed_work_on(cpu, &krcp->monitor_work,
	3611	+ KFREE_DRAIN_JIFFIES);
	3612	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3613	+ }
	3614	+}
	3615	+
	3616	+/*
	3617	+ * During early boot, any blocking grace-period wait automatically
	3618	+ * implies a grace period. Later on, this is never the case for PREEMPTION.
	3619	+ *
	3620	+ * Howevr, because a context switch is a grace period for !PREEMPTION, any
	3621	+ * blocking grace-period wait automatically implies a grace period if
	3622	+ * there is only one CPU online at any point time during execution of
	3623	+ * either synchronize_rcu() or synchronize_rcu_expedited(). It is OK to
3110	3624	* occasionally incorrectly indicate that there are multiple CPUs online
3111		- * when there was in fact only one the whole time, as this just adds
3112		- * some overhead: RCU still operates correctly.
	3625	+ * when there was in fact only one the whole time, as this just adds some
	3626	+ * overhead: RCU still operates correctly.
3113	3627	*/
3114	3628	static int rcu_blocking_is_gp(void)
3115	3629	{
3116	3630	int ret;
3117	3631
	3632	+ if (IS_ENABLED(CONFIG_PREEMPTION))
	3633	+ return rcu_scheduler_active == RCU_SCHEDULER_INACTIVE;
3118	3634	might_sleep(); /* Check for RCU read-side critical section. */
3119	3635	preempt_disable();
3120	3636	ret = num_online_cpus() <= 1;
..	..	@@ -3123,81 +3639,52 @@
3123	3639	}
3124	3640
3125	3641	/**
3126		- * synchronize_sched - wait until an rcu-sched grace period has elapsed.
	3642	+ * synchronize_rcu - wait until a grace period has elapsed.
3127	3643	*
3128		- * Control will return to the caller some time after a full rcu-sched
3129		- * grace period has elapsed, in other words after all currently executing
3130		- * rcu-sched read-side critical sections have completed. These read-side
3131		- * critical sections are delimited by rcu_read_lock_sched() and
3132		- * rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(),
3133		- * local_irq_disable(), and so on may be used in place of
3134		- * rcu_read_lock_sched().
3135		- *
3136		- * This means that all preempt_disable code sequences, including NMI and
3137		- * non-threaded hardware-interrupt handlers, in progress on entry will
3138		- * have completed before this primitive returns. However, this does not
3139		- * guarantee that softirq handlers will have completed, since in some
3140		- * kernels, these handlers can run in process context, and can block.
	3644	+ * Control will return to the caller some time after a full grace
	3645	+ * period has elapsed, in other words after all currently executing RCU
	3646	+ * read-side critical sections have completed. Note, however, that
	3647	+ * upon return from synchronize_rcu(), the caller might well be executing
	3648	+ * concurrently with new RCU read-side critical sections that began while
	3649	+ * synchronize_rcu() was waiting. RCU read-side critical sections are
	3650	+ * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
	3651	+ * In addition, regions of code across which interrupts, preemption, or
	3652	+ * softirqs have been disabled also serve as RCU read-side critical
	3653	+ * sections. This includes hardware interrupt handlers, softirq handlers,
	3654	+ * and NMI handlers.
3141	3655	*
3142	3656	* Note that this guarantee implies further memory-ordering guarantees.
3143		- * On systems with more than one CPU, when synchronize_sched() returns,
3144		- * each CPU is guaranteed to have executed a full memory barrier since the
3145		- * end of its last RCU-sched read-side critical section whose beginning
3146		- * preceded the call to synchronize_sched(). In addition, each CPU having
	3657	+ * On systems with more than one CPU, when synchronize_rcu() returns,
	3658	+ * each CPU is guaranteed to have executed a full memory barrier since
	3659	+ * the end of its last RCU read-side critical section whose beginning
	3660	+ * preceded the call to synchronize_rcu(). In addition, each CPU having
3147	3661	* an RCU read-side critical section that extends beyond the return from
3148		- * synchronize_sched() is guaranteed to have executed a full memory barrier
3149		- * after the beginning of synchronize_sched() and before the beginning of
	3662	+ * synchronize_rcu() is guaranteed to have executed a full memory barrier
	3663	+ * after the beginning of synchronize_rcu() and before the beginning of
3150	3664	* that RCU read-side critical section. Note that these guarantees include
3151	3665	* CPUs that are offline, idle, or executing in user mode, as well as CPUs
3152	3666	* that are executing in the kernel.
3153	3667	*
3154		- * Furthermore, if CPU A invoked synchronize_sched(), which returned
	3668	+ * Furthermore, if CPU A invoked synchronize_rcu(), which returned
3155	3669	* to its caller on CPU B, then both CPU A and CPU B are guaranteed
3156	3670	* to have executed a full memory barrier during the execution of
3157		- * synchronize_sched() -- even if CPU A and CPU B are the same CPU (but
	3671	+ * synchronize_rcu() -- even if CPU A and CPU B are the same CPU (but
3158	3672	* again only if the system has more than one CPU).
3159	3673	*/
3160		-void synchronize_sched(void)
	3674	+void synchronize_rcu(void)
3161	3675	{
3162	3676	RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) \|\|
3163	3677	lock_is_held(&rcu_lock_map) \|\|
3164	3678	lock_is_held(&rcu_sched_lock_map),
3165		- "Illegal synchronize_sched() in RCU-sched read-side critical section");
	3679	+ "Illegal synchronize_rcu() in RCU read-side critical section");
3166	3680	if (rcu_blocking_is_gp())
3167	3681	return;
3168	3682	if (rcu_gp_is_expedited())
3169		- synchronize_sched_expedited();
	3683	+ synchronize_rcu_expedited();
3170	3684	else
3171		- wait_rcu_gp(call_rcu_sched);
	3685	+ wait_rcu_gp(call_rcu);
3172	3686	}
3173		-EXPORT_SYMBOL_GPL(synchronize_sched);
3174		-
3175		-/**
3176		- * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed.
3177		- *
3178		- * Control will return to the caller some time after a full rcu_bh grace
3179		- * period has elapsed, in other words after all currently executing rcu_bh
3180		- * read-side critical sections have completed. RCU read-side critical
3181		- * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(),
3182		- * and may be nested.
3183		- *
3184		- * See the description of synchronize_sched() for more detailed information
3185		- * on memory ordering guarantees.
3186		- */
3187		-void synchronize_rcu_bh(void)
3188		-{
3189		- RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) \|\|
3190		- lock_is_held(&rcu_lock_map) \|\|
3191		- lock_is_held(&rcu_sched_lock_map),
3192		- "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
3193		- if (rcu_blocking_is_gp())
3194		- return;
3195		- if (rcu_gp_is_expedited())
3196		- synchronize_rcu_bh_expedited();
3197		- else
3198		- wait_rcu_gp(call_rcu_bh);
3199		-}
3200		-EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
	3687	+EXPORT_SYMBOL_GPL(synchronize_rcu);
3201	3688
3202	3689	/**
3203	3690	* get_state_synchronize_rcu - Snapshot current RCU state
..	..	@@ -3213,7 +3700,7 @@
3213	3700	* before the load from ->gp_seq.
3214	3701	*/
3215	3702	smp_mb(); /* ^^^ */
3216		- return rcu_seq_snap(&rcu_state_p->gp_seq);
	3703	+ return rcu_seq_snap(&rcu_state.gp_seq);
3217	3704	}
3218	3705	EXPORT_SYMBOL_GPL(get_state_synchronize_rcu);
3219	3706
..	..	@@ -3233,74 +3720,42 @@
3233	3720	*/
3234	3721	void cond_synchronize_rcu(unsigned long oldstate)
3235	3722	{
3236		- if (!rcu_seq_done(&rcu_state_p->gp_seq, oldstate))
	3723	+ if (!rcu_seq_done(&rcu_state.gp_seq, oldstate))
3237	3724	synchronize_rcu();
3238	3725	else
3239	3726	smp_mb(); /* Ensure GP ends before subsequent accesses. */
3240	3727	}
3241	3728	EXPORT_SYMBOL_GPL(cond_synchronize_rcu);
3242	3729
3243		-/**
3244		- * get_state_synchronize_sched - Snapshot current RCU-sched state
3245		- *
3246		- * Returns a cookie that is used by a later call to cond_synchronize_sched()
3247		- * to determine whether or not a full grace period has elapsed in the
3248		- * meantime.
3249		- */
3250		-unsigned long get_state_synchronize_sched(void)
3251		-{
3252		- /*
3253		- * Any prior manipulation of RCU-protected data must happen
3254		- * before the load from ->gp_seq.
3255		- */
3256		- smp_mb(); /* ^^^ */
3257		- return rcu_seq_snap(&rcu_sched_state.gp_seq);
3258		-}
3259		-EXPORT_SYMBOL_GPL(get_state_synchronize_sched);
3260		-
3261		-/**
3262		- * cond_synchronize_sched - Conditionally wait for an RCU-sched grace period
3263		- *
3264		- * @oldstate: return value from earlier call to get_state_synchronize_sched()
3265		- *
3266		- * If a full RCU-sched grace period has elapsed since the earlier call to
3267		- * get_state_synchronize_sched(), just return. Otherwise, invoke
3268		- * synchronize_sched() to wait for a full grace period.
3269		- *
3270		- * Yes, this function does not take counter wrap into account. But
3271		- * counter wrap is harmless. If the counter wraps, we have waited for
3272		- * more than 2 billion grace periods (and way more on a 64-bit system!),
3273		- * so waiting for one additional grace period should be just fine.
3274		- */
3275		-void cond_synchronize_sched(unsigned long oldstate)
3276		-{
3277		- if (!rcu_seq_done(&rcu_sched_state.gp_seq, oldstate))
3278		- synchronize_sched();
3279		- else
3280		- smp_mb(); /* Ensure GP ends before subsequent accesses. */
3281		-}
3282		-EXPORT_SYMBOL_GPL(cond_synchronize_sched);
3283		-
3284	3730	/*
3285		- * Check to see if there is any immediate RCU-related work to be done
3286		- * by the current CPU, for the specified type of RCU, returning 1 if so.
3287		- * The checks are in order of increasing expense: checks that can be
3288		- * carried out against CPU-local state are performed first. However,
3289		- * we must check for CPU stalls first, else we might not get a chance.
	3731	+ * Check to see if there is any immediate RCU-related work to be done by
	3732	+ * the current CPU, returning 1 if so and zero otherwise. The checks are
	3733	+ * in order of increasing expense: checks that can be carried out against
	3734	+ * CPU-local state are performed first. However, we must check for CPU
	3735	+ * stalls first, else we might not get a chance.
3290	3736	*/
3291		-static int __rcu_pending(struct rcu_state rsp, struct rcu_data rdp)
	3737	+static int rcu_pending(int user)
3292	3738	{
	3739	+ bool gp_in_progress;
	3740	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
3293	3741	struct rcu_node *rnp = rdp->mynode;
3294	3742
3295		- /* Check for CPU stalls, if enabled. */
3296		- check_cpu_stall(rsp, rdp);
	3743	+ lockdep_assert_irqs_disabled();
3297	3744
3298		- /* Is this CPU a NO_HZ_FULL CPU that should ignore RCU? */
3299		- if (rcu_nohz_full_cpu(rsp))
	3745	+ /* Check for CPU stalls, if enabled. */
	3746	+ check_cpu_stall(rdp);
	3747	+
	3748	+ /* Does this CPU need a deferred NOCB wakeup? */
	3749	+ if (rcu_nocb_need_deferred_wakeup(rdp))
	3750	+ return 1;
	3751	+
	3752	+ /* Is this a nohz_full CPU in userspace or idle? (Ignore RCU if so.) */
	3753	+ if ((user \|\| rcu_is_cpu_rrupt_from_idle()) && rcu_nohz_full_cpu())
3300	3754	return 0;
3301	3755
3302	3756	/* Is the RCU core waiting for a quiescent state from this CPU? */
3303		- if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm)
	3757	+ gp_in_progress = rcu_gp_in_progress();
	3758	+ if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm && gp_in_progress)
3304	3759	return 1;
3305	3760
3306	3761	/* Does this CPU have callbacks ready to invoke? */
..	..	@@ -3308,8 +3763,9 @@
3308	3763	return 1;
3309	3764
3310	3765	/* Has RCU gone idle with this CPU needing another grace period? */
3311		- if (!rcu_gp_in_progress(rsp) &&
3312		- rcu_segcblist_is_enabled(&rdp->cblist) &&
	3766	+ if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) &&
	3767	+ (!IS_ENABLED(CONFIG_RCU_NOCB_CPU) \|\|
	3768	+ !rcu_segcblist_is_offloaded(&rdp->cblist)) &&
3313	3769	!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
3314	3770	return 1;
3315	3771
..	..	@@ -3318,141 +3774,106 @@
3318	3774	unlikely(READ_ONCE(rdp->gpwrap))) /* outside lock */
3319	3775	return 1;
3320	3776
3321		- /* Does this CPU need a deferred NOCB wakeup? */
3322		- if (rcu_nocb_need_deferred_wakeup(rdp))
3323		- return 1;
3324		-
3325	3777	/* nothing to do */
3326	3778	return 0;
3327	3779	}
3328	3780
3329	3781	/*
3330		- * Check to see if there is any immediate RCU-related work to be done
3331		- * by the current CPU, returning 1 if so. This function is part of the
3332		- * RCU implementation; it is -not- an exported member of the RCU API.
3333		- */
3334		-static int rcu_pending(void)
3335		-{
3336		- struct rcu_state *rsp;
3337		-
3338		- for_each_rcu_flavor(rsp)
3339		- if (__rcu_pending(rsp, this_cpu_ptr(rsp->rda)))
3340		- return 1;
3341		- return 0;
3342		-}
3343		-
3344		-/*
3345		- * Return true if the specified CPU has any callback. If all_lazy is
3346		- * non-NULL, store an indication of whether all callbacks are lazy.
3347		- * (If there are no callbacks, all of them are deemed to be lazy.)
3348		- */
3349		-static bool rcu_cpu_has_callbacks(bool *all_lazy)
3350		-{
3351		- bool al = true;
3352		- bool hc = false;
3353		- struct rcu_data *rdp;
3354		- struct rcu_state *rsp;
3355		-
3356		- for_each_rcu_flavor(rsp) {
3357		- rdp = this_cpu_ptr(rsp->rda);
3358		- if (rcu_segcblist_empty(&rdp->cblist))
3359		- continue;
3360		- hc = true;
3361		- if (rcu_segcblist_n_nonlazy_cbs(&rdp->cblist) \|\| !all_lazy) {
3362		- al = false;
3363		- break;
3364		- }
3365		- }
3366		- if (all_lazy)
3367		- *all_lazy = al;
3368		- return hc;
3369		-}
3370		-
3371		-/*
3372		- * Helper function for _rcu_barrier() tracing. If tracing is disabled,
	3782	+ * Helper function for rcu_barrier() tracing. If tracing is disabled,
3373	3783	* the compiler is expected to optimize this away.
3374	3784	*/
3375		-static void _rcu_barrier_trace(struct rcu_state rsp, const char s,
3376		- int cpu, unsigned long done)
	3785	+static void rcu_barrier_trace(const char *s, int cpu, unsigned long done)
3377	3786	{
3378		- trace_rcu_barrier(rsp->name, s, cpu,
3379		- atomic_read(&rsp->barrier_cpu_count), done);
	3787	+ trace_rcu_barrier(rcu_state.name, s, cpu,
	3788	+ atomic_read(&rcu_state.barrier_cpu_count), done);
3380	3789	}
3381	3790
3382	3791	/*
3383		- * RCU callback function for _rcu_barrier(). If we are last, wake
3384		- * up the task executing _rcu_barrier().
	3792	+ * RCU callback function for rcu_barrier(). If we are last, wake
	3793	+ * up the task executing rcu_barrier().
	3794	+ *
	3795	+ * Note that the value of rcu_state.barrier_sequence must be captured
	3796	+ * before the atomic_dec_and_test(). Otherwise, if this CPU is not last,
	3797	+ * other CPUs might count the value down to zero before this CPU gets
	3798	+ * around to invoking rcu_barrier_trace(), which might result in bogus
	3799	+ * data from the next instance of rcu_barrier().
3385	3800	*/
3386	3801	static void rcu_barrier_callback(struct rcu_head *rhp)
3387	3802	{
3388		- struct rcu_data *rdp = container_of(rhp, struct rcu_data, barrier_head);
3389		- struct rcu_state *rsp = rdp->rsp;
	3803	+ unsigned long __maybe_unused s = rcu_state.barrier_sequence;
3390	3804
3391		- if (atomic_dec_and_test(&rsp->barrier_cpu_count)) {
3392		- _rcu_barrier_trace(rsp, TPS("LastCB"), -1,
3393		- rsp->barrier_sequence);
3394		- complete(&rsp->barrier_completion);
	3805	+ if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {
	3806	+ rcu_barrier_trace(TPS("LastCB"), -1, s);
	3807	+ complete(&rcu_state.barrier_completion);
3395	3808	} else {
3396		- _rcu_barrier_trace(rsp, TPS("CB"), -1, rsp->barrier_sequence);
	3809	+ rcu_barrier_trace(TPS("CB"), -1, s);
3397	3810	}
3398	3811	}
3399	3812
3400	3813	/*
3401	3814	* Called with preemption disabled, and from cross-cpu IRQ context.
3402	3815	*/
3403		-static void rcu_barrier_func(void *type)
	3816	+static void rcu_barrier_func(void *cpu_in)
3404	3817	{
3405		- struct rcu_state *rsp = type;
3406		- struct rcu_data *rdp = raw_cpu_ptr(rsp->rda);
	3818	+ uintptr_t cpu = (uintptr_t)cpu_in;
	3819	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3407	3820
3408		- _rcu_barrier_trace(rsp, TPS("IRQ"), -1, rsp->barrier_sequence);
	3821	+ rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);
3409	3822	rdp->barrier_head.func = rcu_barrier_callback;
3410	3823	debug_rcu_head_queue(&rdp->barrier_head);
3411		- if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head, 0)) {
3412		- atomic_inc(&rsp->barrier_cpu_count);
	3824	+ rcu_nocb_lock(rdp);
	3825	+ WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));
	3826	+ if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) {
	3827	+ atomic_inc(&rcu_state.barrier_cpu_count);
3413	3828	} else {
3414	3829	debug_rcu_head_unqueue(&rdp->barrier_head);
3415		- _rcu_barrier_trace(rsp, TPS("IRQNQ"), -1,
3416		- rsp->barrier_sequence);
	3830	+ rcu_barrier_trace(TPS("IRQNQ"), -1,
	3831	+ rcu_state.barrier_sequence);
3417	3832	}
	3833	+ rcu_nocb_unlock(rdp);
3418	3834	}
3419	3835
3420		-/*
3421		- * Orchestrate the specified type of RCU barrier, waiting for all
3422		- * RCU callbacks of the specified type to complete.
	3836	+/**
	3837	+ * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
	3838	+ *
	3839	+ * Note that this primitive does not necessarily wait for an RCU grace period
	3840	+ * to complete. For example, if there are no RCU callbacks queued anywhere
	3841	+ * in the system, then rcu_barrier() is within its rights to return
	3842	+ * immediately, without waiting for anything, much less an RCU grace period.
3423	3843	*/
3424		-static void _rcu_barrier(struct rcu_state *rsp)
	3844	+void rcu_barrier(void)
3425	3845	{
3426		- int cpu;
	3846	+ uintptr_t cpu;
3427	3847	struct rcu_data *rdp;
3428		- unsigned long s = rcu_seq_snap(&rsp->barrier_sequence);
	3848	+ unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);
3429	3849
3430		- _rcu_barrier_trace(rsp, TPS("Begin"), -1, s);
	3850	+ rcu_barrier_trace(TPS("Begin"), -1, s);
3431	3851
3432	3852	/* Take mutex to serialize concurrent rcu_barrier() requests. */
3433		- mutex_lock(&rsp->barrier_mutex);
	3853	+ mutex_lock(&rcu_state.barrier_mutex);
3434	3854
3435	3855	/* Did someone else do our work for us? */
3436		- if (rcu_seq_done(&rsp->barrier_sequence, s)) {
3437		- _rcu_barrier_trace(rsp, TPS("EarlyExit"), -1,
3438		- rsp->barrier_sequence);
	3856	+ if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {
	3857	+ rcu_barrier_trace(TPS("EarlyExit"), -1,
	3858	+ rcu_state.barrier_sequence);
3439	3859	smp_mb(); /* caller's subsequent code after above check. */
3440		- mutex_unlock(&rsp->barrier_mutex);
	3860	+ mutex_unlock(&rcu_state.barrier_mutex);
3441	3861	return;
3442	3862	}
3443	3863
3444	3864	/* Mark the start of the barrier operation. */
3445		- rcu_seq_start(&rsp->barrier_sequence);
3446		- _rcu_barrier_trace(rsp, TPS("Inc1"), -1, rsp->barrier_sequence);
	3865	+ rcu_seq_start(&rcu_state.barrier_sequence);
	3866	+ rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);
3447	3867
3448	3868	/*
3449		- * Initialize the count to one rather than to zero in order to
3450		- * avoid a too-soon return to zero in case of a short grace period
3451		- * (or preemption of this task). Exclude CPU-hotplug operations
3452		- * to ensure that no offline CPU has callbacks queued.
	3869	+ * Initialize the count to two rather than to zero in order
	3870	+ * to avoid a too-soon return to zero in case of an immediate
	3871	+ * invocation of the just-enqueued callback (or preemption of
	3872	+ * this task). Exclude CPU-hotplug operations to ensure that no
	3873	+ * offline non-offloaded CPU has callbacks queued.
3453	3874	*/
3454		- init_completion(&rsp->barrier_completion);
3455		- atomic_set(&rsp->barrier_cpu_count, 1);
	3875	+ init_completion(&rcu_state.barrier_completion);
	3876	+ atomic_set(&rcu_state.barrier_cpu_count, 2);
3456	3877	get_online_cpus();
3457	3878
3458	3879	/*
..	..	@@ -3461,28 +3882,27 @@
3461	3882	* corresponding CPU's preceding callbacks have been invoked.
3462	3883	*/
3463	3884	for_each_possible_cpu(cpu) {
3464		- if (!cpu_online(cpu) && !rcu_is_nocb_cpu(cpu))
	3885	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	3886	+ if (cpu_is_offline(cpu) &&
	3887	+ !rcu_segcblist_is_offloaded(&rdp->cblist))
3465	3888	continue;
3466		- rdp = per_cpu_ptr(rsp->rda, cpu);
3467		- if (rcu_is_nocb_cpu(cpu)) {
3468		- if (!rcu_nocb_cpu_needs_barrier(rsp, cpu)) {
3469		- _rcu_barrier_trace(rsp, TPS("OfflineNoCB"), cpu,
3470		- rsp->barrier_sequence);
3471		- } else {
3472		- _rcu_barrier_trace(rsp, TPS("OnlineNoCB"), cpu,
3473		- rsp->barrier_sequence);
3474		- smp_mb__before_atomic();
3475		- atomic_inc(&rsp->barrier_cpu_count);
3476		- __call_rcu(&rdp->barrier_head,
3477		- rcu_barrier_callback, rsp, cpu, 0);
3478		- }
3479		- } else if (rcu_segcblist_n_cbs(&rdp->cblist)) {
3480		- _rcu_barrier_trace(rsp, TPS("OnlineQ"), cpu,
3481		- rsp->barrier_sequence);
3482		- smp_call_function_single(cpu, rcu_barrier_func, rsp, 1);
	3889	+ if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) {
	3890	+ rcu_barrier_trace(TPS("OnlineQ"), cpu,
	3891	+ rcu_state.barrier_sequence);
	3892	+ smp_call_function_single(cpu, rcu_barrier_func, (void *)cpu, 1);
	3893	+ } else if (rcu_segcblist_n_cbs(&rdp->cblist) &&
	3894	+ cpu_is_offline(cpu)) {
	3895	+ rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu,
	3896	+ rcu_state.barrier_sequence);
	3897	+ local_irq_disable();
	3898	+ rcu_barrier_func((void *)cpu);
	3899	+ local_irq_enable();
	3900	+ } else if (cpu_is_offline(cpu)) {
	3901	+ rcu_barrier_trace(TPS("OfflineNoCBNoQ"), cpu,
	3902	+ rcu_state.barrier_sequence);
3483	3903	} else {
3484		- _rcu_barrier_trace(rsp, TPS("OnlineNQ"), cpu,
3485		- rsp->barrier_sequence);
	3904	+ rcu_barrier_trace(TPS("OnlineNQ"), cpu,
	3905	+ rcu_state.barrier_sequence);
3486	3906	}
3487	3907	}
3488	3908	put_online_cpus();
..	..	@@ -3491,37 +3911,20 @@
3491	3911	* Now that we have an rcu_barrier_callback() callback on each
3492	3912	* CPU, and thus each counted, remove the initial count.
3493	3913	*/
3494		- if (atomic_dec_and_test(&rsp->barrier_cpu_count))
3495		- complete(&rsp->barrier_completion);
	3914	+ if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count))
	3915	+ complete(&rcu_state.barrier_completion);
3496	3916
3497	3917	/* Wait for all rcu_barrier_callback() callbacks to be invoked. */
3498		- wait_for_completion(&rsp->barrier_completion);
	3918	+ wait_for_completion(&rcu_state.barrier_completion);
3499	3919
3500	3920	/* Mark the end of the barrier operation. */
3501		- _rcu_barrier_trace(rsp, TPS("Inc2"), -1, rsp->barrier_sequence);
3502		- rcu_seq_end(&rsp->barrier_sequence);
	3921	+ rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence);
	3922	+ rcu_seq_end(&rcu_state.barrier_sequence);
3503	3923
3504	3924	/* Other rcu_barrier() invocations can now safely proceed. */
3505		- mutex_unlock(&rsp->barrier_mutex);
	3925	+ mutex_unlock(&rcu_state.barrier_mutex);
3506	3926	}
3507		-
3508		-/**
3509		- * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete.
3510		- */
3511		-void rcu_barrier_bh(void)
3512		-{
3513		- _rcu_barrier(&rcu_bh_state);
3514		-}
3515		-EXPORT_SYMBOL_GPL(rcu_barrier_bh);
3516		-
3517		-/**
3518		- * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks.
3519		- */
3520		-void rcu_barrier_sched(void)
3521		-{
3522		- _rcu_barrier(&rcu_sched_state);
3523		-}
3524		-EXPORT_SYMBOL_GPL(rcu_barrier_sched);
	3927	+EXPORT_SYMBOL_GPL(rcu_barrier);
3525	3928
3526	3929	/*
3527	3930	* Propagate ->qsinitmask bits up the rcu_node tree to account for the
..	..	@@ -3555,46 +3958,48 @@
3555	3958	* Do boot-time initialization of a CPU's per-CPU RCU data.
3556	3959	*/
3557	3960	static void __init
3558		-rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
	3961	+rcu_boot_init_percpu_data(int cpu)
3559	3962	{
3560		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
	3963	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3561	3964
3562	3965	/* Set up local state, ensuring consistent view of global state. */
3563	3966	rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);
3564		- rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
3565		- WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != 1);
3566		- WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp->dynticks)));
3567		- rdp->rcu_ofl_gp_seq = rsp->gp_seq;
	3967	+ INIT_WORK(&rdp->strict_work, strict_work_handler);
	3968	+ WARN_ON_ONCE(rdp->dynticks_nesting != 1);
	3969	+ WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)));
	3970	+ rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;
3568	3971	rdp->rcu_ofl_gp_flags = RCU_GP_CLEANED;
3569		- rdp->rcu_onl_gp_seq = rsp->gp_seq;
	3972	+ rdp->rcu_onl_gp_seq = rcu_state.gp_seq;
3570	3973	rdp->rcu_onl_gp_flags = RCU_GP_CLEANED;
3571	3974	rdp->cpu = cpu;
3572		- rdp->rsp = rsp;
3573	3975	rcu_boot_init_nocb_percpu_data(rdp);
3574	3976	}
3575	3977
3576	3978	/*
3577		- * Initialize a CPU's per-CPU RCU data. Note that only one online or
	3979	+ * Invoked early in the CPU-online process, when pretty much all services
	3980	+ * are available. The incoming CPU is not present.
	3981	+ *
	3982	+ * Initializes a CPU's per-CPU RCU data. Note that only one online or
3578	3983	* offline event can be happening at a given time. Note also that we can
3579	3984	* accept some slop in the rsp->gp_seq access due to the fact that this
3580		- * CPU cannot possibly have any RCU callbacks in flight yet.
	3985	+ * CPU cannot possibly have any non-offloaded RCU callbacks in flight yet.
	3986	+ * And any offloaded callbacks are being numbered elsewhere.
3581	3987	*/
3582		-static void
3583		-rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
	3988	+int rcutree_prepare_cpu(unsigned int cpu)
3584	3989	{
3585	3990	unsigned long flags;
3586		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
3587		- struct rcu_node *rnp = rcu_get_root(rsp);
	3991	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
	3992	+ struct rcu_node *rnp = rcu_get_root();
3588	3993
3589	3994	/* Set up local state, ensuring consistent view of global state. */
3590	3995	raw_spin_lock_irqsave_rcu_node(rnp, flags);
3591	3996	rdp->qlen_last_fqs_check = 0;
3592		- rdp->n_force_qs_snap = rsp->n_force_qs;
	3997	+ rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
3593	3998	rdp->blimit = blimit;
3594	3999	if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */
3595		- !init_nocb_callback_list(rdp))
	4000	+ !rcu_segcblist_is_offloaded(&rdp->cblist))
3596	4001	rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */
3597		- rdp->dynticks->dynticks_nesting = 1; /* CPU not up, no tearing. */
	4002	+ rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */
3598	4003	rcu_dynticks_eqs_online();
3599	4004	raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
3600	4005
..	..	@@ -3606,30 +4011,16 @@
3606	4011	rnp = rdp->mynode;
3607	4012	raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
3608	4013	rdp->beenonline = true; /* We have now been online. */
3609		- rdp->gp_seq = rnp->gp_seq;
3610		- rdp->gp_seq_needed = rnp->gp_seq;
	4014	+ rdp->gp_seq = READ_ONCE(rnp->gp_seq);
	4015	+ rdp->gp_seq_needed = rdp->gp_seq;
3611	4016	rdp->cpu_no_qs.b.norm = true;
3612		- rdp->rcu_qs_ctr_snap = per_cpu(rcu_dynticks.rcu_qs_ctr, cpu);
3613	4017	rdp->core_needs_qs = false;
3614	4018	rdp->rcu_iw_pending = false;
3615		- rdp->rcu_iw_gp_seq = rnp->gp_seq - 1;
3616		- trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuonl"));
	4019	+ rdp->rcu_iw_gp_seq = rdp->gp_seq - 1;
	4020	+ trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
3617	4021	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3618		-}
3619		-
3620		-/*
3621		- * Invoked early in the CPU-online process, when pretty much all
3622		- * services are available. The incoming CPU is not present.
3623		- */
3624		-int rcutree_prepare_cpu(unsigned int cpu)
3625		-{
3626		- struct rcu_state *rsp;
3627		-
3628		- for_each_rcu_flavor(rsp)
3629		- rcu_init_percpu_data(cpu, rsp);
3630		-
3631	4022	rcu_prepare_kthreads(cpu);
3632		- rcu_spawn_all_nocb_kthreads(cpu);
	4023	+ rcu_spawn_cpu_nocb_kthread(cpu);
3633	4024
3634	4025	return 0;
3635	4026	}
..	..	@@ -3639,7 +4030,7 @@
3639	4030	*/
3640	4031	static void rcutree_affinity_setting(unsigned int cpu, int outgoing)
3641	4032	{
3642		- struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
	4033	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3643	4034
3644	4035	rcu_boost_kthread_setaffinity(rdp->mynode, outgoing);
3645	4036	}
..	..	@@ -3653,21 +4044,19 @@
3653	4044	unsigned long flags;
3654	4045	struct rcu_data *rdp;
3655	4046	struct rcu_node *rnp;
3656		- struct rcu_state *rsp;
3657	4047
3658		- for_each_rcu_flavor(rsp) {
3659		- rdp = per_cpu_ptr(rsp->rda, cpu);
3660		- rnp = rdp->mynode;
3661		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
3662		- rnp->ffmask \|= rdp->grpmask;
3663		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3664		- }
3665		- if (IS_ENABLED(CONFIG_TREE_SRCU))
3666		- srcu_online_cpu(cpu);
	4048	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	4049	+ rnp = rdp->mynode;
	4050	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4051	+ rnp->ffmask \|= rdp->grpmask;
	4052	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3667	4053	if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
3668	4054	return 0; /* Too early in boot for scheduler work. */
3669	4055	sync_sched_exp_online_cleanup(cpu);
3670	4056	rcutree_affinity_setting(cpu, -1);
	4057	+
	4058	+ // Stop-machine done, so allow nohz_full to disable tick.
	4059	+ tick_dep_clear(TICK_DEP_BIT_RCU);
3671	4060	return 0;
3672	4061	}
3673	4062
..	..	@@ -3680,49 +4069,19 @@
3680	4069	unsigned long flags;
3681	4070	struct rcu_data *rdp;
3682	4071	struct rcu_node *rnp;
3683		- struct rcu_state *rsp;
3684	4072
3685		- for_each_rcu_flavor(rsp) {
3686		- rdp = per_cpu_ptr(rsp->rda, cpu);
3687		- rnp = rdp->mynode;
3688		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
3689		- rnp->ffmask &= ~rdp->grpmask;
3690		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3691		- }
	4073	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	4074	+ rnp = rdp->mynode;
	4075	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4076	+ rnp->ffmask &= ~rdp->grpmask;
	4077	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3692	4078
3693	4079	rcutree_affinity_setting(cpu, cpu);
3694		- if (IS_ENABLED(CONFIG_TREE_SRCU))
3695		- srcu_offline_cpu(cpu);
	4080	+
	4081	+ // nohz_full CPUs need the tick for stop-machine to work quickly
	4082	+ tick_dep_set(TICK_DEP_BIT_RCU);
3696	4083	return 0;
3697	4084	}
3698		-
3699		-/*
3700		- * Near the end of the offline process. We do only tracing here.
3701		- */
3702		-int rcutree_dying_cpu(unsigned int cpu)
3703		-{
3704		- struct rcu_state *rsp;
3705		-
3706		- for_each_rcu_flavor(rsp)
3707		- rcu_cleanup_dying_cpu(rsp);
3708		- return 0;
3709		-}
3710		-
3711		-/*
3712		- * The outgoing CPU is gone and we are running elsewhere.
3713		- */
3714		-int rcutree_dead_cpu(unsigned int cpu)
3715		-{
3716		- struct rcu_state *rsp;
3717		-
3718		- for_each_rcu_flavor(rsp) {
3719		- rcu_cleanup_dead_cpu(cpu, rsp);
3720		- do_nocb_deferred_wakeup(per_cpu_ptr(rsp->rda, cpu));
3721		- }
3722		- return 0;
3723		-}
3724		-
3725		-static DEFINE_PER_CPU(int, rcu_cpu_started);
3726	4085
3727	4086	/*
3728	4087	* Mark the specified CPU as being online so that subsequent grace periods
..	..	@@ -3739,74 +4098,40 @@
3739	4098	{
3740	4099	unsigned long flags;
3741	4100	unsigned long mask;
3742		- int nbits;
3743		- unsigned long oldmask;
3744	4101	struct rcu_data *rdp;
3745	4102	struct rcu_node *rnp;
3746		- struct rcu_state *rsp;
	4103	+ bool newcpu;
3747	4104
3748		- if (per_cpu(rcu_cpu_started, cpu))
	4105	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	4106	+ if (rdp->cpu_started)
3749	4107	return;
	4108	+ rdp->cpu_started = true;
3750	4109
3751		- per_cpu(rcu_cpu_started, cpu) = 1;
3752		-
3753		- for_each_rcu_flavor(rsp) {
3754		- rdp = per_cpu_ptr(rsp->rda, cpu);
3755		- rnp = rdp->mynode;
3756		- mask = rdp->grpmask;
3757		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
3758		- rnp->qsmaskinitnext \|= mask;
3759		- oldmask = rnp->expmaskinitnext;
3760		- rnp->expmaskinitnext \|= mask;
3761		- oldmask ^= rnp->expmaskinitnext;
3762		- nbits = bitmap_weight(&oldmask, BITS_PER_LONG);
3763		- /* Allow lockless access for expedited grace periods. */
3764		- smp_store_release(&rsp->ncpus, rsp->ncpus + nbits); /* ^^^ */
3765		- rcu_gpnum_ovf(rnp, rdp); /* Offline-induced counter wrap? */
3766		- rdp->rcu_onl_gp_seq = READ_ONCE(rsp->gp_seq);
3767		- rdp->rcu_onl_gp_flags = READ_ONCE(rsp->gp_flags);
3768		- if (rnp->qsmask & mask) { /* RCU waiting on incoming CPU? */
3769		- /* Report QS -after- changing ->qsmaskinitnext! */
3770		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
3771		- } else {
3772		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3773		- }
	4110	+ rnp = rdp->mynode;
	4111	+ mask = rdp->grpmask;
	4112	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4113	+ WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext \| mask);
	4114	+ newcpu = !(rnp->expmaskinitnext & mask);
	4115	+ rnp->expmaskinitnext \|= mask;
	4116	+ /* Allow lockless access for expedited grace periods. */
	4117	+ smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + newcpu); /* ^^^ */
	4118	+ ASSERT_EXCLUSIVE_WRITER(rcu_state.ncpus);
	4119	+ rcu_gpnum_ovf(rnp, rdp); /* Offline-induced counter wrap? */
	4120	+ rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);
	4121	+ rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags);
	4122	+ if (rnp->qsmask & mask) { /* RCU waiting on incoming CPU? */
	4123	+ rcu_disable_urgency_upon_qs(rdp);
	4124	+ /* Report QS -after- changing ->qsmaskinitnext! */
	4125	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
	4126	+ } else {
	4127	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3774	4128	}
3775	4129	smp_mb(); /* Ensure RCU read-side usage follows above initialization. */
3776	4130	}
3777	4131
3778		-#ifdef CONFIG_HOTPLUG_CPU
3779		-/*
3780		- * The CPU is exiting the idle loop into the arch_cpu_idle_dead()
3781		- * function. We now remove it from the rcu_node tree's ->qsmaskinitnext
3782		- * bit masks.
3783		- */
3784		-static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp)
3785		-{
3786		- unsigned long flags;
3787		- unsigned long mask;
3788		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
3789		- struct rcu_node rnp = rdp->mynode; / Outgoing CPU's rdp & rnp. */
3790		-
3791		- /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
3792		- mask = rdp->grpmask;
3793		- spin_lock(&rsp->ofl_lock);
3794		- raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
3795		- rdp->rcu_ofl_gp_seq = READ_ONCE(rsp->gp_seq);
3796		- rdp->rcu_ofl_gp_flags = READ_ONCE(rsp->gp_flags);
3797		- if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */
3798		- /* Report quiescent state -before- changing ->qsmaskinitnext! */
3799		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
3800		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
3801		- }
3802		- rnp->qsmaskinitnext &= ~mask;
3803		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3804		- spin_unlock(&rsp->ofl_lock);
3805		-}
3806		-
3807	4132	/*
3808	4133	* The outgoing function has no further need of RCU, so remove it from
3809		- * the list of CPUs that RCU must track.
	4134	+ * the rcu_node tree's ->qsmaskinitnext bit masks.
3810	4135	*
3811	4136	* Note that this function is special in that it is invoked directly
3812	4137	* from the outgoing CPU rather than from the cpuhp_step mechanism.
..	..	@@ -3814,65 +4139,82 @@
3814	4139	*/
3815	4140	void rcu_report_dead(unsigned int cpu)
3816	4141	{
3817		- struct rcu_state *rsp;
	4142	+ unsigned long flags;
	4143	+ unsigned long mask;
	4144	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
	4145	+ struct rcu_node rnp = rdp->mynode; / Outgoing CPU's rdp & rnp. */
3818	4146
3819		- /* QS for any half-done expedited RCU-sched GP. */
	4147	+ /* QS for any half-done expedited grace period. */
3820	4148	preempt_disable();
3821		- rcu_report_exp_rdp(&rcu_sched_state,
3822		- this_cpu_ptr(rcu_sched_state.rda), true);
	4149	+ rcu_report_exp_rdp(this_cpu_ptr(&rcu_data));
3823	4150	preempt_enable();
3824		- for_each_rcu_flavor(rsp)
3825		- rcu_cleanup_dying_idle_cpu(cpu, rsp);
	4151	+ rcu_preempt_deferred_qs(current);
3826	4152
3827		- per_cpu(rcu_cpu_started, cpu) = 0;
	4153	+ /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
	4154	+ mask = rdp->grpmask;
	4155	+ raw_spin_lock(&rcu_state.ofl_lock);
	4156	+ raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
	4157	+ rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);
	4158	+ rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags);
	4159	+ if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */
	4160	+ /* Report quiescent state -before- changing ->qsmaskinitnext! */
	4161	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
	4162	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4163	+ }
	4164	+ WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext & ~mask);
	4165	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
	4166	+ raw_spin_unlock(&rcu_state.ofl_lock);
	4167	+
	4168	+ rdp->cpu_started = false;
3828	4169	}
3829	4170
3830		-/* Migrate the dead CPU's callbacks to the current CPU. */
3831		-static void rcu_migrate_callbacks(int cpu, struct rcu_state *rsp)
	4171	+#ifdef CONFIG_HOTPLUG_CPU
	4172	+/*
	4173	+ * The outgoing CPU has just passed through the dying-idle state, and we
	4174	+ * are being invoked from the CPU that was IPIed to continue the offline
	4175	+ * operation. Migrate the outgoing CPU's callbacks to the current CPU.
	4176	+ */
	4177	+void rcutree_migrate_callbacks(int cpu)
3832	4178	{
3833	4179	unsigned long flags;
3834	4180	struct rcu_data *my_rdp;
3835		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
3836		- struct rcu_node *rnp_root = rcu_get_root(rdp->rsp);
	4181	+ struct rcu_node *my_rnp;
	4182	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3837	4183	bool needwake;
3838	4184
3839		- if (rcu_is_nocb_cpu(cpu) \|\| rcu_segcblist_empty(&rdp->cblist))
	4185	+ if (rcu_segcblist_is_offloaded(&rdp->cblist) \|\|
	4186	+ rcu_segcblist_empty(&rdp->cblist))
3840	4187	return; /* No callbacks to migrate. */
3841	4188
3842	4189	local_irq_save(flags);
3843		- my_rdp = this_cpu_ptr(rsp->rda);
3844		- if (rcu_nocb_adopt_orphan_cbs(my_rdp, rdp, flags)) {
3845		- local_irq_restore(flags);
3846		- return;
3847		- }
3848		- raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
	4190	+ my_rdp = this_cpu_ptr(&rcu_data);
	4191	+ my_rnp = my_rdp->mynode;
	4192	+ rcu_nocb_lock(my_rdp); /* irqs already disabled. */
	4193	+ WARN_ON_ONCE(!rcu_nocb_flush_bypass(my_rdp, NULL, jiffies));
	4194	+ raw_spin_lock_rcu_node(my_rnp); /* irqs already disabled. */
3849	4195	/* Leverage recent GPs and set GP for new callbacks. */
3850		- needwake = rcu_advance_cbs(rsp, rnp_root, rdp) \|\|
3851		- rcu_advance_cbs(rsp, rnp_root, my_rdp);
	4196	+ needwake = rcu_advance_cbs(my_rnp, rdp) \|\|
	4197	+ rcu_advance_cbs(my_rnp, my_rdp);
3852	4198	rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist);
	4199	+ needwake = needwake \|\| rcu_advance_cbs(my_rnp, my_rdp);
	4200	+ rcu_segcblist_disable(&rdp->cblist);
3853	4201	WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) !=
3854	4202	!rcu_segcblist_n_cbs(&my_rdp->cblist));
3855		- raw_spin_unlock_irqrestore_rcu_node(rnp_root, flags);
	4203	+ if (rcu_segcblist_is_offloaded(&my_rdp->cblist)) {
	4204	+ raw_spin_unlock_rcu_node(my_rnp); /* irqs remain disabled. */
	4205	+ __call_rcu_nocb_wake(my_rdp, true, flags);
	4206	+ } else {
	4207	+ rcu_nocb_unlock(my_rdp); /* irqs remain disabled. */
	4208	+ raw_spin_unlock_irqrestore_rcu_node(my_rnp, flags);
	4209	+ }
3856	4210	if (needwake)
3857		- rcu_gp_kthread_wake(rsp);
	4211	+ rcu_gp_kthread_wake();
	4212	+ lockdep_assert_irqs_enabled();
3858	4213	WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 \|\|
3859	4214	!rcu_segcblist_empty(&rdp->cblist),
3860	4215	"rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n",
3861	4216	cpu, rcu_segcblist_n_cbs(&rdp->cblist),
3862	4217	rcu_segcblist_first_cb(&rdp->cblist));
3863		-}
3864		-
3865		-/*
3866		- * The outgoing CPU has just passed through the dying-idle state,
3867		- * and we are being invoked from the CPU that was IPIed to continue the
3868		- * offline operation. We need to migrate the outgoing CPU's callbacks.
3869		- */
3870		-void rcutree_migrate_callbacks(int cpu)
3871		-{
3872		- struct rcu_state *rsp;
3873		-
3874		- for_each_rcu_flavor(rsp)
3875		- rcu_migrate_callbacks(cpu, rsp);
3876	4218	}
3877	4219	#endif
3878	4220
..	..	@@ -3886,13 +4228,11 @@
3886	4228	switch (action) {
3887	4229	case PM_HIBERNATION_PREPARE:
3888	4230	case PM_SUSPEND_PREPARE:
3889		- if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
3890		- rcu_expedite_gp();
	4231	+ rcu_expedite_gp();
3891	4232	break;
3892	4233	case PM_POST_HIBERNATION:
3893	4234	case PM_POST_SUSPEND:
3894		- if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
3895		- rcu_unexpedite_gp();
	4235	+ rcu_unexpedite_gp();
3896	4236	break;
3897	4237	default:
3898	4238	break;
..	..	@@ -3901,14 +4241,13 @@
3901	4241	}
3902	4242
3903	4243	/*
3904		- * Spawn the kthreads that handle each RCU flavor's grace periods.
	4244	+ * Spawn the kthreads that handle RCU's grace periods.
3905	4245	*/
3906	4246	static int __init rcu_spawn_gp_kthread(void)
3907	4247	{
3908	4248	unsigned long flags;
3909	4249	int kthread_prio_in = kthread_prio;
3910	4250	struct rcu_node *rnp;
3911		- struct rcu_state *rsp;
3912	4251	struct sched_param sp;
3913	4252	struct task_struct *t;
3914	4253
..	..	@@ -3928,21 +4267,24 @@
3928	4267	kthread_prio, kthread_prio_in);
3929	4268
3930	4269	rcu_scheduler_fully_active = 1;
3931		- for_each_rcu_flavor(rsp) {
3932		- t = kthread_create(rcu_gp_kthread, rsp, "%s", rsp->name);
3933		- BUG_ON(IS_ERR(t));
3934		- rnp = rcu_get_root(rsp);
3935		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
3936		- rsp->gp_kthread = t;
3937		- if (kthread_prio) {
3938		- sp.sched_priority = kthread_prio;
3939		- sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
3940		- }
3941		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3942		- wake_up_process(t);
	4270	+ t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name);
	4271	+ if (WARN_ONCE(IS_ERR(t), "%s: Could not start grace-period kthread, OOM is now expected behavior\n", __func__))
	4272	+ return 0;
	4273	+ if (kthread_prio) {
	4274	+ sp.sched_priority = kthread_prio;
	4275	+ sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
3943	4276	}
	4277	+ rnp = rcu_get_root();
	4278	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4279	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	4280	+ WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
	4281	+ // Reset .gp_activity and .gp_req_activity before setting .gp_kthread.
	4282	+ smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */
	4283	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
	4284	+ wake_up_process(t);
3944	4285	rcu_spawn_nocb_kthreads();
3945	4286	rcu_spawn_boost_kthreads();
	4287	+ rcu_spawn_core_kthreads();
3946	4288	return 0;
3947	4289	}
3948	4290	early_initcall(rcu_spawn_gp_kthread);
..	..	@@ -3967,9 +4309,9 @@
3967	4309	}
3968	4310
3969	4311	/*
3970		- * Helper function for rcu_init() that initializes one rcu_state structure.
	4312	+ * Helper function for rcu_init() that initializes the rcu_state structure.
3971	4313	*/
3972		-static void __init rcu_init_one(struct rcu_state *rsp)
	4314	+static void __init rcu_init_one(void)
3973	4315	{
3974	4316	static const char * const buf[] = RCU_NODE_NAME_INIT;
3975	4317	static const char * const fqs[] = RCU_FQS_NAME_INIT;
..	..	@@ -3991,14 +4333,15 @@
3991	4333	/* Initialize the level-tracking arrays. */
3992	4334
3993	4335	for (i = 1; i < rcu_num_lvls; i++)
3994		- rsp->level[i] = rsp->level[i - 1] + num_rcu_lvl[i - 1];
	4336	+ rcu_state.level[i] =
	4337	+ rcu_state.level[i - 1] + num_rcu_lvl[i - 1];
3995	4338	rcu_init_levelspread(levelspread, num_rcu_lvl);
3996	4339
3997	4340	/* Initialize the elements themselves, starting from the leaves. */
3998	4341
3999	4342	for (i = rcu_num_lvls - 1; i >= 0; i--) {
4000	4343	cpustride *= levelspread[i];
4001		- rnp = rsp->level[i];
	4344	+ rnp = rcu_state.level[i];
4002	4345	for (j = 0; j < num_rcu_lvl[i]; j++, rnp++) {
4003	4346	raw_spin_lock_init(&ACCESS_PRIVATE(rnp, lock));
4004	4347	lockdep_set_class_and_name(&ACCESS_PRIVATE(rnp, lock),
..	..	@@ -4006,9 +4349,9 @@
4006	4349	raw_spin_lock_init(&rnp->fqslock);
4007	4350	lockdep_set_class_and_name(&rnp->fqslock,
4008	4351	&rcu_fqs_class[i], fqs[i]);
4009		- rnp->gp_seq = rsp->gp_seq;
4010		- rnp->gp_seq_needed = rsp->gp_seq;
4011		- rnp->completedqs = rsp->gp_seq;
	4352	+ rnp->gp_seq = rcu_state.gp_seq;
	4353	+ rnp->gp_seq_needed = rcu_state.gp_seq;
	4354	+ rnp->completedqs = rcu_state.gp_seq;
4012	4355	rnp->qsmask = 0;
4013	4356	rnp->qsmaskinit = 0;
4014	4357	rnp->grplo = j * cpustride;
..	..	@@ -4021,8 +4364,8 @@
4021	4364	rnp->parent = NULL;
4022	4365	} else {
4023	4366	rnp->grpnum = j % levelspread[i - 1];
4024		- rnp->grpmask = 1UL << rnp->grpnum;
4025		- rnp->parent = rsp->level[i - 1] +
	4367	+ rnp->grpmask = BIT(rnp->grpnum);
	4368	+ rnp->parent = rcu_state.level[i - 1] +
4026	4369	j / levelspread[i - 1];
4027	4370	}
4028	4371	rnp->level = i;
..	..	@@ -4036,16 +4379,15 @@
4036	4379	}
4037	4380	}
4038	4381
4039		- init_swait_queue_head(&rsp->gp_wq);
4040		- init_swait_queue_head(&rsp->expedited_wq);
4041		- rnp = rcu_first_leaf_node(rsp);
	4382	+ init_swait_queue_head(&rcu_state.gp_wq);
	4383	+ init_swait_queue_head(&rcu_state.expedited_wq);
	4384	+ rnp = rcu_first_leaf_node();
4042	4385	for_each_possible_cpu(i) {
4043	4386	while (i > rnp->grphi)
4044	4387	rnp++;
4045		- per_cpu_ptr(rsp->rda, i)->mynode = rnp;
4046		- rcu_boot_init_percpu_data(i, rsp);
	4388	+ per_cpu_ptr(&rcu_data, i)->mynode = rnp;
	4389	+ rcu_boot_init_percpu_data(i);
4047	4390	}
4048		- list_add(&rsp->flavors, &rcu_struct_flavors);
4049	4391	}
4050	4392
4051	4393	/*
..	..	@@ -4053,11 +4395,25 @@
4053	4395	* replace the definitions in tree.h because those are needed to size
4054	4396	* the ->node array in the rcu_state structure.
4055	4397	*/
4056		-static void __init rcu_init_geometry(void)
	4398	+void rcu_init_geometry(void)
4057	4399	{
4058	4400	ulong d;
4059	4401	int i;
	4402	+ static unsigned long old_nr_cpu_ids;
4060	4403	int rcu_capacity[RCU_NUM_LVLS];
	4404	+ static bool initialized;
	4405	+
	4406	+ if (initialized) {
	4407	+ /*
	4408	+ * Warn if setup_nr_cpu_ids() had not yet been invoked,
	4409	+ * unless nr_cpus_ids == NR_CPUS, in which case who cares?
	4410	+ */
	4411	+ WARN_ON_ONCE(old_nr_cpu_ids != nr_cpu_ids);
	4412	+ return;
	4413	+ }
	4414	+
	4415	+ old_nr_cpu_ids = nr_cpu_ids;
	4416	+ initialized = true;
4061	4417
4062	4418	/*
4063	4419	* Initialize any unspecified boot parameters.
..	..	@@ -4071,6 +4427,7 @@
4071	4427	jiffies_till_first_fqs = d;
4072	4428	if (jiffies_till_next_fqs == ULONG_MAX)
4073	4429	jiffies_till_next_fqs = d;
	4430	+ adjust_jiffies_till_sched_qs();
4074	4431
4075	4432	/* If the compile-time values are accurate, just leave. */
4076	4433	if (rcu_fanout_leaf == RCU_FANOUT_LEAF &&
..	..	@@ -4129,16 +4486,16 @@
4129	4486
4130	4487	/*
4131	4488	* Dump out the structure of the rcu_node combining tree associated
4132		- * with the rcu_state structure referenced by rsp.
	4489	+ * with the rcu_state structure.
4133	4490	*/
4134		-static void __init rcu_dump_rcu_node_tree(struct rcu_state *rsp)
	4491	+static void __init rcu_dump_rcu_node_tree(void)
4135	4492	{
4136	4493	int level = 0;
4137	4494	struct rcu_node *rnp;
4138	4495
4139	4496	pr_info("rcu_node tree layout dump\n");
4140	4497	pr_info(" ");
4141		- rcu_for_each_node_breadth_first(rsp, rnp) {
	4498	+ rcu_for_each_node_breadth_first(rnp) {
4142	4499	if (rnp->level != level) {
4143	4500	pr_cont("\n");
4144	4501	pr_info(" ");
..	..	@@ -4152,20 +4509,41 @@
4152	4509	struct workqueue_struct *rcu_gp_wq;
4153	4510	struct workqueue_struct *rcu_par_gp_wq;
4154	4511
	4512	+static void __init kfree_rcu_batch_init(void)
	4513	+{
	4514	+ int cpu;
	4515	+ int i;
	4516	+
	4517	+ for_each_possible_cpu(cpu) {
	4518	+ struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
	4519	+
	4520	+ for (i = 0; i < KFREE_N_BATCHES; i++) {
	4521	+ INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
	4522	+ krcp->krw_arr[i].krcp = krcp;
	4523	+ }
	4524	+
	4525	+ INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
	4526	+ INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
	4527	+ krcp->initialized = true;
	4528	+ }
	4529	+ if (register_shrinker(&kfree_rcu_shrinker))
	4530	+ pr_err("Failed to register kfree_rcu() shrinker!\n");
	4531	+}
	4532	+
4155	4533	void __init rcu_init(void)
4156	4534	{
4157	4535	int cpu;
4158	4536
4159	4537	rcu_early_boot_tests();
4160	4538
	4539	+ kfree_rcu_batch_init();
4161	4540	rcu_bootup_announce();
4162	4541	rcu_init_geometry();
4163		- rcu_init_one(&rcu_bh_state);
4164		- rcu_init_one(&rcu_sched_state);
	4542	+ rcu_init_one();
4165	4543	if (dump_tree)
4166		- rcu_dump_rcu_node_tree(&rcu_sched_state);
4167		- __rcu_init_preempt();
4168		- open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
	4544	+ rcu_dump_rcu_node_tree();
	4545	+ if (use_softirq)
	4546	+ open_softirq(RCU_SOFTIRQ, rcu_core_si);
4169	4547
4170	4548	/*
4171	4549	* We don't need protection against CPU-hotplug here because
..	..	@@ -4184,7 +4562,16 @@
4184	4562	WARN_ON(!rcu_gp_wq);
4185	4563	rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);
4186	4564	WARN_ON(!rcu_par_gp_wq);
	4565	+ srcu_init();
	4566	+
	4567	+ /* Fill in default value for rcutree.qovld boot parameter. */
	4568	+ /* -After- the rcu_node ->lock fields are initialized! */
	4569	+ if (qovld < 0)
	4570	+ qovld_calc = DEFAULT_RCU_QOVLD_MULT * qhimark;
	4571	+ else
	4572	+ qovld_calc = qovld;
4187	4573	}
4188	4574
	4575	+#include "tree_stall.h"
4189	4576	#include "tree_exp.h"
4190	4577	#include "tree_plugin.h"