~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,27 +1,14 @@
	1	+// SPDX-License-Identifier: GPL-2.0+
1	2	/*
2		- * Read-Copy Update mechanism for mutual exclusion
3		- *
4		- * This program is free software; you can redistribute it and/or modify
5		- * it under the terms of the GNU General Public License as published by
6		- * the Free Software Foundation; either version 2 of the License, or
7		- * (at your option) any later version.
8		- *
9		- * This program is distributed in the hope that it will be useful,
10		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12		- * GNU General Public License for more details.
13		- *
14		- * You should have received a copy of the GNU General Public License
15		- * along with this program; if not, you can access it online at
16		- * http://www.gnu.org/licenses/gpl-2.0.html.
	3	+ * Read-Copy Update mechanism for mutual exclusion (tree-based version)
17	4	*
18	5	* Copyright IBM Corporation, 2008
19	6	*
20	7	* Authors: Dipankar Sarma <dipankar@in.ibm.com>
21	8	* Manfred Spraul <manfred@colorfullife.com>
22		- * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version
	9	+ * Paul E. McKenney <paulmck@linux.ibm.com>
23	10	*
24		- * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
	11	+ * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
25	12	* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
26	13	*
27	14	* For detailed explanation of Read-Copy Update mechanism see -
..	..	@@ -56,17 +43,23 @@
56	43	#include <uapi/linux/sched/types.h>
57	44	#include <linux/prefetch.h>
58	45	#include <linux/delay.h>
59		-#include <linux/stop_machine.h>
60	46	#include <linux/random.h>
61	47	#include <linux/trace_events.h>
62	48	#include <linux/suspend.h>
63	49	#include <linux/ftrace.h>
64		-#include <linux/delay.h>
	50	+#include <linux/tick.h>
	51	+#include <linux/sysrq.h>
	52	+#include <linux/kprobes.h>
65	53	#include <linux/gfp.h>
66	54	#include <linux/oom.h>
67	55	#include <linux/smpboot.h>
68	56	#include <linux/jiffies.h>
	57	+#include <linux/slab.h>
69	58	#include <linux/sched/isolation.h>
	59	+#include <linux/sched/clock.h>
	60	+#include <linux/vmalloc.h>
	61	+#include <linux/mm.h>
	62	+#include <linux/kasan.h>
70	63	#include "../time/tick-internal.h"
71	64
72	65	#include "tree.h"
..	..	@@ -80,49 +73,35 @@
80	73	/* Data structures. */
81	74
82	75	/*
83		- * In order to export the rcu_state name to the tracing tools, it
84		- * needs to be added in the __tracepoint_string section.
85		- * This requires defining a separate variable tp_<sname>_varname
86		- * that points to the string being used, and this will allow
87		- * the tracing userspace tools to be able to decipher the string
88		- * address to the matching string.
	76	+ * Steal a bit from the bottom of ->dynticks for idle entry/exit
	77	+ * control. Initially this is for TLB flushing.
89	78	*/
90		-#ifdef CONFIG_TRACING
91		-# define DEFINE_RCU_TPS(sname) \
92		-static char sname##_varname[] = #sname; \
93		-static const char *tp_##sname##_varname __used __tracepoint_string = sname##_varname;
94		-# define RCU_STATE_NAME(sname) sname##_varname
95		-#else
96		-# define DEFINE_RCU_TPS(sname)
97		-# define RCU_STATE_NAME(sname) __stringify(sname)
98		-#endif
	79	+#define RCU_DYNTICK_CTRL_MASK 0x1
	80	+#define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1)
99	81
100		-#define RCU_STATE_INITIALIZER(sname, sabbr, cr) \
101		-DEFINE_RCU_TPS(sname) \
102		-static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data); \
103		-struct rcu_state sname##_state = { \
104		- .level = { &sname##_state.node[0] }, \
105		- .rda = &sname##_data, \
106		- .call = cr, \
107		- .gp_state = RCU_GP_IDLE, \
108		- .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT, \
109		- .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \
110		- .name = RCU_STATE_NAME(sname), \
111		- .abbr = sabbr, \
112		- .exp_mutex = __MUTEX_INITIALIZER(sname##_state.exp_mutex), \
113		- .exp_wake_mutex = __MUTEX_INITIALIZER(sname##_state.exp_wake_mutex), \
114		- .ofl_lock = __SPIN_LOCK_UNLOCKED(sname##_state.ofl_lock), \
115		-}
116		-
117		-RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
118		-RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh);
119		-
120		-static struct rcu_state *const rcu_state_p;
121		-LIST_HEAD(rcu_struct_flavors);
	82	+static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
	83	+ .dynticks_nesting = 1,
	84	+ .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
	85	+ .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),
	86	+};
	87	+static struct rcu_state rcu_state = {
	88	+ .level = { &rcu_state.node[0] },
	89	+ .gp_state = RCU_GP_IDLE,
	90	+ .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT,
	91	+ .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
	92	+ .name = RCU_NAME,
	93	+ .abbr = RCU_ABBR,
	94	+ .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
	95	+ .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
	96	+ .ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock),
	97	+};
122	98
123	99	/* Dump rcu_node combining tree at boot to verify correct setup. */
124	100	static bool dump_tree;
125	101	module_param(dump_tree, bool, 0444);
	102	+/* By default, use RCU_SOFTIRQ instead of rcuc kthreads. */
	103	+static bool use_softirq = true;
	104	+module_param(use_softirq, bool, 0444);
126	105	/* Control rcu_node-tree auto-balancing at boot time. */
127	106	static bool rcu_fanout_exact;
128	107	module_param(rcu_fanout_exact, bool, 0444);
..	..	@@ -133,9 +112,6 @@
133	112	/* Number of rcu_nodes at specified level. */
134	113	int num_rcu_lvl[] = NUM_RCU_LVL_INIT;
135	114	int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */
136		-/* panic() on RCU Stall sysctl. */
137		-int sysctl_panic_on_rcu_stall __read_mostly = CONFIG_BOOTPARAM_RCU_STALL_PANIC_VALUE;
138		-ATOMIC_NOTIFIER_HEAD(rcu_stall_notifier_list);
139	115
140	116	/*
141	117	* The rcu_scheduler_active variable is initialized to the value
..	..	@@ -166,21 +142,19 @@
166	142	*/
167	143	static int rcu_scheduler_fully_active __read_mostly;
168	144
169		-static void
170		-rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
171		- struct rcu_node *rnp, unsigned long gps, unsigned long flags);
	145	+static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
	146	+ unsigned long gps, unsigned long flags);
172	147	static void rcu_init_new_rnp(struct rcu_node *rnp_leaf);
173	148	static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf);
174	149	static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
175	150	static void invoke_rcu_core(void);
176		-static void invoke_rcu_callbacks(struct rcu_state rsp, struct rcu_data rdp);
177		-static void rcu_report_exp_rdp(struct rcu_state *rsp,
178		- struct rcu_data *rdp, bool wake);
	151	+static void rcu_report_exp_rdp(struct rcu_data *rdp);
179	152	static void sync_sched_exp_online_cleanup(int cpu);
	153	+static void check_cb_ovld_locked(struct rcu_data rdp, struct rcu_node rnp);
180	154
181	155	/* rcuc/rcub kthread realtime priority */
182	156	static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
183		-module_param(kthread_prio, int, 0644);
	157	+module_param(kthread_prio, int, 0444);
184	158
185	159	/* Delay in jiffies for grace-period initialization delays, debug only. */
186	160
..	..	@@ -191,7 +165,22 @@
191	165	static int gp_cleanup_delay;
192	166	module_param(gp_cleanup_delay, int, 0444);
193	167
194		-/* Retreive RCU kthreads priority for rcutorture */
	168	+// Add delay to rcu_read_unlock() for strict grace periods.
	169	+static int rcu_unlock_delay;
	170	+#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
	171	+module_param(rcu_unlock_delay, int, 0444);
	172	+#endif
	173	+
	174	+/*
	175	+ * This rcu parameter is runtime-read-only. It reflects
	176	+ * a minimum allowed number of objects which can be cached
	177	+ * per-CPU. Object size is equal to one page. This value
	178	+ * can be changed at boot time.
	179	+ */
	180	+static int rcu_min_cached_objs = 5;
	181	+module_param(rcu_min_cached_objs, int, 0444);
	182	+
	183	+/* Retrieve RCU kthreads priority for rcutorture */
195	184	int rcu_get_gp_kthreads_prio(void)
196	185	{
197	186	return kthread_prio;
..	..	@@ -215,7 +204,7 @@
215	204	* held, but the bit corresponding to the current CPU will be stable
216	205	* in most contexts.
217	206	*/
218		-unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp)
	207	+static unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp)
219	208	{
220	209	return READ_ONCE(rnp->qsmaskinitnext);
221	210	}
..	..	@@ -225,81 +214,39 @@
225	214	* permit this function to be invoked without holding the root rcu_node
226	215	* structure's ->lock, but of course results can be subject to change.
227	216	*/
228		-static int rcu_gp_in_progress(struct rcu_state *rsp)
	217	+static int rcu_gp_in_progress(void)
229	218	{
230		- return rcu_seq_state(rcu_seq_current(&rsp->gp_seq));
	219	+ return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));
231	220	}
232	221
233	222	/*
234		- * Note a quiescent state. Because we do not need to know
235		- * how many quiescent states passed, just if there was at least
236		- * one since the start of the grace period, this just sets a flag.
237		- * The caller must have disabled preemption.
	223	+ * Return the number of callbacks queued on the specified CPU.
	224	+ * Handles both the nocbs and normal cases.
238	225	*/
239		-void rcu_sched_qs(void)
	226	+static long rcu_get_n_cbs_cpu(int cpu)
240	227	{
241		- RCU_LOCKDEP_WARN(preemptible(), "rcu_sched_qs() invoked with preemption enabled!!!");
242		- if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.s))
243		- return;
244		- trace_rcu_grace_period(TPS("rcu_sched"),
245		- __this_cpu_read(rcu_sched_data.gp_seq),
246		- TPS("cpuqs"));
247		- __this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false);
248		- if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
249		- return;
250		- __this_cpu_write(rcu_sched_data.cpu_no_qs.b.exp, false);
251		- rcu_report_exp_rdp(&rcu_sched_state,
252		- this_cpu_ptr(&rcu_sched_data), true);
	228	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
	229	+
	230	+ if (rcu_segcblist_is_enabled(&rdp->cblist))
	231	+ return rcu_segcblist_n_cbs(&rdp->cblist);
	232	+ return 0;
253	233	}
254	234
255		-#ifdef CONFIG_PREEMPT_RT_FULL
256		-static void rcu_preempt_qs(void);
257		-
258		-void rcu_bh_qs(void)
	235	+void rcu_softirq_qs(void)
259	236	{
260		- unsigned long flags;
261		-
262		- /* Callers to this function, rcu_preempt_qs(), must disable irqs. */
263		- local_irq_save(flags);
264		- rcu_preempt_qs();
265		- local_irq_restore(flags);
	237	+ rcu_qs();
	238	+ rcu_preempt_deferred_qs(current);
266	239	}
267		-#else
268		-void rcu_bh_qs(void)
269		-{
270		- RCU_LOCKDEP_WARN(preemptible(), "rcu_bh_qs() invoked with preemption enabled!!!");
271		- if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) {
272		- trace_rcu_grace_period(TPS("rcu_bh"),
273		- __this_cpu_read(rcu_bh_data.gp_seq),
274		- TPS("cpuqs"));
275		- __this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);
276		- }
277		-}
278		-#endif
279		-
280		-/*
281		- * Steal a bit from the bottom of ->dynticks for idle entry/exit
282		- * control. Initially this is for TLB flushing.
283		- */
284		-#define RCU_DYNTICK_CTRL_MASK 0x1
285		-#define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1)
286		-#ifndef rcu_eqs_special_exit
287		-#define rcu_eqs_special_exit() do { } while (0)
288		-#endif
289		-
290		-static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
291		- .dynticks_nesting = 1,
292		- .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
293		- .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),
294		-};
295	240
296	241	/*
297	242	* Record entry into an extended quiescent state. This is only to be
298		- * called when not already in an extended quiescent state.
	243	+ * called when not already in an extended quiescent state, that is,
	244	+ * RCU is watching prior to the call to this function and is no longer
	245	+ * watching upon return.
299	246	*/
300		-static void rcu_dynticks_eqs_enter(void)
	247	+static noinstr void rcu_dynticks_eqs_enter(void)
301	248	{
302		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	249	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
303	250	int seq;
304	251
305	252	/*
..	..	@@ -307,8 +254,9 @@
307	254	* critical sections, and we also must force ordering with the
308	255	* next idle sojourn.
309	256	*/
310		- seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
311		- /* Better be in an extended quiescent state! */
	257	+ rcu_dynticks_task_trace_enter(); // Before ->dynticks update!
	258	+ seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
	259	+ // RCU is no longer watching. Better be in extended quiescent state!
312	260	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
313	261	(seq & RCU_DYNTICK_CTRL_CTR));
314	262	/* Better not have special action (TLB flush) pending! */
..	..	@@ -318,11 +266,12 @@
318	266
319	267	/*
320	268	* Record exit from an extended quiescent state. This is only to be
321		- * called from an extended quiescent state.
	269	+ * called from an extended quiescent state, that is, RCU is not watching
	270	+ * prior to the call to this function and is watching upon return.
322	271	*/
323		-static void rcu_dynticks_eqs_exit(void)
	272	+static noinstr void rcu_dynticks_eqs_exit(void)
324	273	{
325		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	274	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
326	275	int seq;
327	276
328	277	/*
..	..	@@ -330,14 +279,14 @@
330	279	* and we also must force ordering with the next RCU read-side
331	280	* critical section.
332	281	*/
333		- seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
	282	+ seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
	283	+ // RCU is now watching. Better not be in an extended quiescent state!
	284	+ rcu_dynticks_task_trace_exit(); // After ->dynticks update!
334	285	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
335	286	!(seq & RCU_DYNTICK_CTRL_CTR));
336	287	if (seq & RCU_DYNTICK_CTRL_MASK) {
337		- atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdtp->dynticks);
	288	+ arch_atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks);
338	289	smp_mb__after_atomic(); /* _exit after clearing mask. */
339		- /* Prefer duplicate flushes to losing a flush. */
340		- rcu_eqs_special_exit();
341	290	}
342	291	}
343	292
..	..	@@ -353,11 +302,11 @@
353	302	*/
354	303	static void rcu_dynticks_eqs_online(void)
355	304	{
356		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	305	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
357	306
358		- if (atomic_read(&rdtp->dynticks) & RCU_DYNTICK_CTRL_CTR)
	307	+ if (atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR)
359	308	return;
360		- atomic_add(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
	309	+ atomic_add(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
361	310	}
362	311
363	312	/*
..	..	@@ -365,20 +314,20 @@
365	314	*
366	315	* No ordering, as we are sampling CPU-local information.
367	316	*/
368		-bool rcu_dynticks_curr_cpu_in_eqs(void)
	317	+static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)
369	318	{
370		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	319	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
371	320
372		- return !(atomic_read(&rdtp->dynticks) & RCU_DYNTICK_CTRL_CTR);
	321	+ return !(arch_atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR);
373	322	}
374	323
375	324	/*
376	325	* Snapshot the ->dynticks counter with full ordering so as to allow
377	326	* stable comparison of this counter with past and future snapshots.
378	327	*/
379		-int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
	328	+static int rcu_dynticks_snap(struct rcu_data *rdp)
380	329	{
381		- int snap = atomic_add_return(0, &rdtp->dynticks);
	330	+ int snap = atomic_add_return(0, &rdp->dynticks);
382	331
383	332	return snap & ~RCU_DYNTICK_CTRL_MASK;
384	333	}
..	..	@@ -393,13 +342,35 @@
393	342	}
394	343
395	344	/*
396		- * Return true if the CPU corresponding to the specified rcu_dynticks
	345	+ * Return true if the CPU corresponding to the specified rcu_data
397	346	* structure has spent some time in an extended quiescent state since
398	347	* rcu_dynticks_snap() returned the specified snapshot.
399	348	*/
400		-static bool rcu_dynticks_in_eqs_since(struct rcu_dynticks *rdtp, int snap)
	349	+static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap)
401	350	{
402		- return snap != rcu_dynticks_snap(rdtp);
	351	+ return snap != rcu_dynticks_snap(rdp);
	352	+}
	353	+
	354	+/*
	355	+ * Return true if the referenced integer is zero while the specified
	356	+ * CPU remains within a single extended quiescent state.
	357	+ */
	358	+bool rcu_dynticks_zero_in_eqs(int cpu, int *vp)
	359	+{
	360	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
	361	+ int snap;
	362	+
	363	+ // If not quiescent, force back to earlier extended quiescent state.
	364	+ snap = atomic_read(&rdp->dynticks) & ~(RCU_DYNTICK_CTRL_MASK \|
	365	+ RCU_DYNTICK_CTRL_CTR);
	366	+
	367	+ smp_rmb(); // Order ->dynticks and *vp reads.
	368	+ if (READ_ONCE(*vp))
	369	+ return false; // Non-zero, so report failure;
	370	+ smp_rmb(); // Order *vp read and ->dynticks re-read.
	371	+
	372	+ // If still in the same extended quiescent state, we are good!
	373	+ return snap == (atomic_read(&rdp->dynticks) & ~RCU_DYNTICK_CTRL_MASK);
403	374	}
404	375
405	376	/*
..	..	@@ -413,14 +384,17 @@
413	384	{
414	385	int old;
415	386	int new;
416		- struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
	387	+ int new_old;
	388	+ struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
417	389
	390	+ new_old = atomic_read(&rdp->dynticks);
418	391	do {
419		- old = atomic_read(&rdtp->dynticks);
	392	+ old = new_old;
420	393	if (old & RCU_DYNTICK_CTRL_CTR)
421	394	return false;
422	395	new = old \| RCU_DYNTICK_CTRL_MASK;
423		- } while (atomic_cmpxchg(&rdtp->dynticks, old, new) != old);
	396	+ new_old = atomic_cmpxchg(&rdp->dynticks, old, new);
	397	+ } while (new_old != old);
424	398	return true;
425	399	}
426	400
..	..	@@ -435,105 +409,128 @@
435	409	*
436	410	* The caller must have disabled interrupts and must not be idle.
437	411	*/
438		-static void rcu_momentary_dyntick_idle(void)
	412	+notrace void rcu_momentary_dyntick_idle(void)
439	413	{
440		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
441	414	int special;
442	415
443		- raw_cpu_write(rcu_dynticks.rcu_need_heavy_qs, false);
444		- special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
	416	+ raw_cpu_write(rcu_data.rcu_need_heavy_qs, false);
	417	+ special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR,
	418	+ &this_cpu_ptr(&rcu_data)->dynticks);
445	419	/* It is illegal to call this from idle state. */
446	420	WARN_ON_ONCE(!(special & RCU_DYNTICK_CTRL_CTR));
	421	+ rcu_preempt_deferred_qs(current);
447	422	}
	423	+EXPORT_SYMBOL_GPL(rcu_momentary_dyntick_idle);
448	424
449		-/*
450		- * Note a context switch. This is a quiescent state for RCU-sched,
451		- * and requires special handling for preemptible RCU.
452		- * The caller must have disabled interrupts.
453		- */
454		-void rcu_note_context_switch(bool preempt)
455		-{
456		- barrier(); /* Avoid RCU read-side critical sections leaking down. */
457		- trace_rcu_utilization(TPS("Start context switch"));
458		- rcu_sched_qs();
459		- rcu_preempt_note_context_switch(preempt);
460		- /* Load rcu_urgent_qs before other flags. */
461		- if (!smp_load_acquire(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs)))
462		- goto out;
463		- this_cpu_write(rcu_dynticks.rcu_urgent_qs, false);
464		- if (unlikely(raw_cpu_read(rcu_dynticks.rcu_need_heavy_qs)))
465		- rcu_momentary_dyntick_idle();
466		- this_cpu_inc(rcu_dynticks.rcu_qs_ctr);
467		- if (!preempt)
468		- rcu_tasks_qs(current);
469		-out:
470		- trace_rcu_utilization(TPS("End context switch"));
471		- barrier(); /* Avoid RCU read-side critical sections leaking up. */
472		-}
473		-EXPORT_SYMBOL_GPL(rcu_note_context_switch);
474		-
475		-/*
476		- * Register a quiescent state for all RCU flavors. If there is an
477		- * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
478		- * dyntick-idle quiescent state visible to other CPUs (but only for those
479		- * RCU flavors in desperate need of a quiescent state, which will normally
480		- * be none of them). Either way, do a lightweight quiescent state for
481		- * all RCU flavors.
	425	+/**
	426	+ * rcu_is_cpu_rrupt_from_idle - see if 'interrupted' from idle
482	427	*
483		- * The barrier() calls are redundant in the common case when this is
484		- * called externally, but just in case this is called from within this
485		- * file.
	428	+ * If the current CPU is idle and running at a first-level (not nested)
	429	+ * interrupt, or directly, from idle, return true.
486	430	*
	431	+ * The caller must have at least disabled IRQs.
487	432	*/
488		-void rcu_all_qs(void)
	433	+static int rcu_is_cpu_rrupt_from_idle(void)
489	434	{
490		- unsigned long flags;
	435	+ long nesting;
491	436
492		- if (!raw_cpu_read(rcu_dynticks.rcu_urgent_qs))
493		- return;
494		- preempt_disable();
495		- /* Load rcu_urgent_qs before other flags. */
496		- if (!smp_load_acquire(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs))) {
497		- preempt_enable();
498		- return;
499		- }
500		- this_cpu_write(rcu_dynticks.rcu_urgent_qs, false);
501		- barrier(); /* Avoid RCU read-side critical sections leaking down. */
502		- if (unlikely(raw_cpu_read(rcu_dynticks.rcu_need_heavy_qs))) {
503		- local_irq_save(flags);
504		- rcu_momentary_dyntick_idle();
505		- local_irq_restore(flags);
506		- }
507		- if (unlikely(raw_cpu_read(rcu_sched_data.cpu_no_qs.b.exp)))
508		- rcu_sched_qs();
509		- this_cpu_inc(rcu_dynticks.rcu_qs_ctr);
510		- barrier(); /* Avoid RCU read-side critical sections leaking up. */
511		- preempt_enable();
	437	+ /*
	438	+ * Usually called from the tick; but also used from smp_function_call()
	439	+ * for expedited grace periods. This latter can result in running from
	440	+ * the idle task, instead of an actual IPI.
	441	+ */
	442	+ lockdep_assert_irqs_disabled();
	443	+
	444	+ /* Check for counter underflows */
	445	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) < 0,
	446	+ "RCU dynticks_nesting counter underflow!");
	447	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0,
	448	+ "RCU dynticks_nmi_nesting counter underflow/zero!");
	449	+
	450	+ /* Are we at first interrupt nesting level? */
	451	+ nesting = __this_cpu_read(rcu_data.dynticks_nmi_nesting);
	452	+ if (nesting > 1)
	453	+ return false;
	454	+
	455	+ /*
	456	+ * If we're not in an interrupt, we must be in the idle task!
	457	+ */
	458	+ WARN_ON_ONCE(!nesting && !is_idle_task(current));
	459	+
	460	+ /* Does CPU appear to be idle from an RCU standpoint? */
	461	+ return __this_cpu_read(rcu_data.dynticks_nesting) == 0;
512	462	}
513		-EXPORT_SYMBOL_GPL(rcu_all_qs);
514	463
515		-#define DEFAULT_RCU_BLIMIT 10 /* Maximum callbacks per rcu_do_batch. */
	464	+#define DEFAULT_RCU_BLIMIT (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 1000 : 10)
	465	+ // Maximum callbacks per rcu_do_batch ...
	466	+#define DEFAULT_MAX_RCU_BLIMIT 10000 // ... even during callback flood.
516	467	static long blimit = DEFAULT_RCU_BLIMIT;
517		-#define DEFAULT_RCU_QHIMARK 10000 /* If this many pending, ignore blimit. */
	468	+#define DEFAULT_RCU_QHIMARK 10000 // If this many pending, ignore blimit.
518	469	static long qhimark = DEFAULT_RCU_QHIMARK;
519		-#define DEFAULT_RCU_QLOMARK 100 /* Once only this many pending, use blimit. */
	470	+#define DEFAULT_RCU_QLOMARK 100 // Once only this many pending, use blimit.
520	471	static long qlowmark = DEFAULT_RCU_QLOMARK;
	472	+#define DEFAULT_RCU_QOVLD_MULT 2
	473	+#define DEFAULT_RCU_QOVLD (DEFAULT_RCU_QOVLD_MULT * DEFAULT_RCU_QHIMARK)
	474	+static long qovld = DEFAULT_RCU_QOVLD; // If this many pending, hammer QS.
	475	+static long qovld_calc = -1; // No pre-initialization lock acquisitions!
521	476
522	477	module_param(blimit, long, 0444);
523	478	module_param(qhimark, long, 0444);
524	479	module_param(qlowmark, long, 0444);
	480	+module_param(qovld, long, 0444);
525	481
526		-static ulong jiffies_till_first_fqs = ULONG_MAX;
	482	+static ulong jiffies_till_first_fqs = IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 0 : ULONG_MAX;
527	483	static ulong jiffies_till_next_fqs = ULONG_MAX;
528	484	static bool rcu_kick_kthreads;
	485	+static int rcu_divisor = 7;
	486	+module_param(rcu_divisor, int, 0644);
	487	+
	488	+/* Force an exit from rcu_do_batch() after 3 milliseconds. */
	489	+static long rcu_resched_ns = 3 * NSEC_PER_MSEC;
	490	+module_param(rcu_resched_ns, long, 0644);
	491	+
	492	+/*
	493	+ * How long the grace period must be before we start recruiting
	494	+ * quiescent-state help from rcu_note_context_switch().
	495	+ */
	496	+static ulong jiffies_till_sched_qs = ULONG_MAX;
	497	+module_param(jiffies_till_sched_qs, ulong, 0444);
	498	+static ulong jiffies_to_sched_qs; /* See adjust_jiffies_till_sched_qs(). */
	499	+module_param(jiffies_to_sched_qs, ulong, 0444); /* Display only! */
	500	+
	501	+/*
	502	+ * Make sure that we give the grace-period kthread time to detect any
	503	+ * idle CPUs before taking active measures to force quiescent states.
	504	+ * However, don't go below 100 milliseconds, adjusted upwards for really
	505	+ * large systems.
	506	+ */
	507	+static void adjust_jiffies_till_sched_qs(void)
	508	+{
	509	+ unsigned long j;
	510	+
	511	+ /* If jiffies_till_sched_qs was specified, respect the request. */
	512	+ if (jiffies_till_sched_qs != ULONG_MAX) {
	513	+ WRITE_ONCE(jiffies_to_sched_qs, jiffies_till_sched_qs);
	514	+ return;
	515	+ }
	516	+ /* Otherwise, set to third fqs scan, but bound below on large system. */
	517	+ j = READ_ONCE(jiffies_till_first_fqs) +
	518	+ 2 * READ_ONCE(jiffies_till_next_fqs);
	519	+ if (j < HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV)
	520	+ j = HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
	521	+ pr_info("RCU calculated value of scheduler-enlistment delay is %ld jiffies.\n", j);
	522	+ WRITE_ONCE(jiffies_to_sched_qs, j);
	523	+}
529	524
530	525	static int param_set_first_fqs_jiffies(const char val, const struct kernel_param kp)
531	526	{
532	527	ulong j;
533	528	int ret = kstrtoul(val, 0, &j);
534	529
535		- if (!ret)
	530	+ if (!ret) {
536	531	WRITE_ONCE((ulong )kp->arg, (j > HZ) ? HZ : j);
	532	+ adjust_jiffies_till_sched_qs();
	533	+ }
537	534	return ret;
538	535	}
539	536
..	..	@@ -542,8 +539,10 @@
542	539	ulong j;
543	540	int ret = kstrtoul(val, 0, &j);
544	541
545		- if (!ret)
	542	+ if (!ret) {
546	543	WRITE_ONCE((ulong )kp->arg, (j > HZ) ? HZ : (j ?: 1));
	544	+ adjust_jiffies_till_sched_qs();
	545	+ }
547	546	return ret;
548	547	}
549	548
..	..	@@ -561,45 +560,17 @@
561	560	module_param_cb(jiffies_till_next_fqs, &next_fqs_jiffies_ops, &jiffies_till_next_fqs, 0644);
562	561	module_param(rcu_kick_kthreads, bool, 0644);
563	562
564		-/*
565		- * How long the grace period must be before we start recruiting
566		- * quiescent-state help from rcu_note_context_switch().
567		- */
568		-static ulong jiffies_till_sched_qs = HZ / 10;
569		-module_param(jiffies_till_sched_qs, ulong, 0444);
570		-
571		-static void force_qs_rnp(struct rcu_state rsp, int (f)(struct rcu_data *rsp));
572		-static void force_quiescent_state(struct rcu_state *rsp);
573		-static int rcu_pending(void);
	563	+static void force_qs_rnp(int (f)(struct rcu_data rdp));
	564	+static int rcu_pending(int user);
574	565
575	566	/*
576	567	* Return the number of RCU GPs completed thus far for debug & stats.
577	568	*/
578	569	unsigned long rcu_get_gp_seq(void)
579	570	{
580		- return READ_ONCE(rcu_state_p->gp_seq);
	571	+ return READ_ONCE(rcu_state.gp_seq);
581	572	}
582	573	EXPORT_SYMBOL_GPL(rcu_get_gp_seq);
583		-
584		-/*
585		- * Return the number of RCU-sched GPs completed thus far for debug & stats.
586		- */
587		-unsigned long rcu_sched_get_gp_seq(void)
588		-{
589		- return READ_ONCE(rcu_sched_state.gp_seq);
590		-}
591		-EXPORT_SYMBOL_GPL(rcu_sched_get_gp_seq);
592		-
593		-#ifndef CONFIG_PREEMPT_RT_FULL
594		-/*
595		- * Return the number of RCU-bh GPs completed thus far for debug & stats.
596		- */
597		-unsigned long rcu_bh_get_gp_seq(void)
598		-{
599		- return READ_ONCE(rcu_bh_state.gp_seq);
600		-}
601		-EXPORT_SYMBOL_GPL(rcu_bh_get_gp_seq);
602		-#endif
603	574
604	575	/*
605	576	* Return the number of RCU expedited batches completed thus far for
..	..	@@ -609,90 +580,17 @@
609	580	*/
610	581	unsigned long rcu_exp_batches_completed(void)
611	582	{
612		- return rcu_state_p->expedited_sequence;
	583	+ return rcu_state.expedited_sequence;
613	584	}
614	585	EXPORT_SYMBOL_GPL(rcu_exp_batches_completed);
615	586
616	587	/*
617		- * Return the number of RCU-sched expedited batches completed thus far
618		- * for debug & stats. Similar to rcu_exp_batches_completed().
	588	+ * Return the root node of the rcu_state structure.
619	589	*/
620		-unsigned long rcu_exp_batches_completed_sched(void)
	590	+static struct rcu_node *rcu_get_root(void)
621	591	{
622		- return rcu_sched_state.expedited_sequence;
	592	+ return &rcu_state.node[0];
623	593	}
624		-EXPORT_SYMBOL_GPL(rcu_exp_batches_completed_sched);
625		-
626		-#ifndef CONFIG_PREEMPT_RT_FULL
627		-/*
628		- * Force a quiescent state.
629		- */
630		-void rcu_force_quiescent_state(void)
631		-{
632		- force_quiescent_state(rcu_state_p);
633		-}
634		-EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
635		-
636		-/*
637		- * Force a quiescent state for RCU BH.
638		- */
639		-void rcu_bh_force_quiescent_state(void)
640		-{
641		- force_quiescent_state(&rcu_bh_state);
642		-}
643		-EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state);
644		-
645		-#else
646		-void rcu_force_quiescent_state(void)
647		-{
648		-}
649		-EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
650		-#endif
651		-
652		-/*
653		- * Force a quiescent state for RCU-sched.
654		- */
655		-void rcu_sched_force_quiescent_state(void)
656		-{
657		- force_quiescent_state(&rcu_sched_state);
658		-}
659		-EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state);
660		-
661		-/*
662		- * Show the state of the grace-period kthreads.
663		- */
664		-void show_rcu_gp_kthreads(void)
665		-{
666		- int cpu;
667		- struct rcu_data *rdp;
668		- struct rcu_node *rnp;
669		- struct rcu_state *rsp;
670		-
671		- for_each_rcu_flavor(rsp) {
672		- pr_info("%s: wait state: %d ->state: %#lx\n",
673		- rsp->name, rsp->gp_state, rsp->gp_kthread->state);
674		- rcu_for_each_node_breadth_first(rsp, rnp) {
675		- if (ULONG_CMP_GE(rsp->gp_seq, rnp->gp_seq_needed))
676		- continue;
677		- pr_info("\trcu_node %d:%d ->gp_seq %lu ->gp_seq_needed %lu\n",
678		- rnp->grplo, rnp->grphi, rnp->gp_seq,
679		- rnp->gp_seq_needed);
680		- if (!rcu_is_leaf_node(rnp))
681		- continue;
682		- for_each_leaf_node_possible_cpu(rnp, cpu) {
683		- rdp = per_cpu_ptr(rsp->rda, cpu);
684		- if (rdp->gpwrap \|\|
685		- ULONG_CMP_GE(rsp->gp_seq,
686		- rdp->gp_seq_needed))
687		- continue;
688		- pr_info("\tcpu %d ->gp_seq_needed %lu\n",
689		- cpu, rdp->gp_seq_needed);
690		- }
691		- }
692		- /* sched_show_task(rsp->gp_kthread); */
693		- }
694		-}
695		-EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads);
696	594
697	595	/*
698	596	* Send along grace-period-related data for rcutorture diagnostics.
..	..	@@ -700,37 +598,16 @@
700	598	void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags,
701	599	unsigned long *gp_seq)
702	600	{
703		- struct rcu_state *rsp = NULL;
704		-
705	601	switch (test_type) {
706	602	case RCU_FLAVOR:
707		- rsp = rcu_state_p;
708		- break;
709		-#ifndef CONFIG_PREEMPT_RT_FULL
710		- case RCU_BH_FLAVOR:
711		- rsp = &rcu_bh_state;
712		- break;
713		-#endif
714		- case RCU_SCHED_FLAVOR:
715		- rsp = &rcu_sched_state;
	603	+ *flags = READ_ONCE(rcu_state.gp_flags);
	604	+ *gp_seq = rcu_seq_current(&rcu_state.gp_seq);
716	605	break;
717	606	default:
718	607	break;
719	608	}
720		- if (rsp == NULL)
721		- return;
722		- *flags = READ_ONCE(rsp->gp_flags);
723		- *gp_seq = rcu_seq_current(&rsp->gp_seq);
724	609	}
725	610	EXPORT_SYMBOL_GPL(rcutorture_get_gp_data);
726		-
727		-/*
728		- * Return the root node of the specified rcu_state structure.
729		- */
730		-static struct rcu_node rcu_get_root(struct rcu_state rsp)
731		-{
732		- return &rsp->node[0];
733		-}
734	611
735	612	/*
736	613	* Enter an RCU extended quiescent state, which can be either the
..	..	@@ -740,31 +617,36 @@
740	617	* the possibility of usermode upcalls having messed up our count
741	618	* of interrupt nesting level during the prior busy period.
742	619	*/
743		-static void rcu_eqs_enter(bool user)
	620	+static noinstr void rcu_eqs_enter(bool user)
744	621	{
745		- struct rcu_state *rsp;
746		- struct rcu_data *rdp;
747		- struct rcu_dynticks *rdtp;
	622	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
748	623
749		- rdtp = this_cpu_ptr(&rcu_dynticks);
750		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0);
	624	+ WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE);
	625	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, 0);
751	626	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
752		- rdtp->dynticks_nesting == 0);
753		- if (rdtp->dynticks_nesting != 1) {
754		- rdtp->dynticks_nesting--;
	627	+ rdp->dynticks_nesting == 0);
	628	+ if (rdp->dynticks_nesting != 1) {
	629	+ // RCU will still be watching, so just do accounting and leave.
	630	+ rdp->dynticks_nesting--;
755	631	return;
756	632	}
757	633
758	634	lockdep_assert_irqs_disabled();
759		- trace_rcu_dyntick(TPS("Start"), rdtp->dynticks_nesting, 0, rdtp->dynticks);
	635	+ instrumentation_begin();
	636	+ trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks));
760	637	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
761		- for_each_rcu_flavor(rsp) {
762		- rdp = this_cpu_ptr(rsp->rda);
763		- do_nocb_deferred_wakeup(rdp);
764		- }
	638	+ rdp = this_cpu_ptr(&rcu_data);
765	639	rcu_prepare_for_idle();
766		- WRITE_ONCE(rdtp->dynticks_nesting, 0); /* Avoid irq-access tearing. */
	640	+ rcu_preempt_deferred_qs(current);
	641	+
	642	+ // instrumentation for the noinstr rcu_dynticks_eqs_enter()
	643	+ instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
	644	+
	645	+ instrumentation_end();
	646	+ WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */
	647	+ // RCU is watching here ...
767	648	rcu_dynticks_eqs_enter();
	649	+ // ... but is no longer watching here.
768	650	rcu_dynticks_task_enter();
769	651	}
770	652
..	..	@@ -784,6 +666,7 @@
784	666	lockdep_assert_irqs_disabled();
785	667	rcu_eqs_enter(false);
786	668	}
	669	+EXPORT_SYMBOL_GPL(rcu_idle_enter);
787	670
788	671	#ifdef CONFIG_NO_HZ_FULL
789	672	/**
..	..	@@ -797,9 +680,16 @@
797	680	* If you add or remove a call to rcu_user_enter(), be sure to test with
798	681	* CONFIG_RCU_EQS_DEBUG=y.
799	682	*/
800		-void rcu_user_enter(void)
	683	+noinstr void rcu_user_enter(void)
801	684	{
	685	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
	686	+
802	687	lockdep_assert_irqs_disabled();
	688	+
	689	+ instrumentation_begin();
	690	+ do_nocb_deferred_wakeup(rdp);
	691	+ instrumentation_end();
	692	+
803	693	rcu_eqs_enter(true);
804	694	}
805	695	#endif /* CONFIG_NO_HZ_FULL */
..	..	@@ -808,40 +698,56 @@
808	698	* rcu_nmi_exit - inform RCU of exit from NMI context
809	699	*
810	700	* If we are returning from the outermost NMI handler that interrupted an
811		- * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
	701	+ * RCU-idle period, update rdp->dynticks and rdp->dynticks_nmi_nesting
812	702	* to let the RCU grace-period handling know that the CPU is back to
813	703	* being RCU-idle.
814	704	*
815	705	* If you add or remove a call to rcu_nmi_exit(), be sure to test
816	706	* with CONFIG_RCU_EQS_DEBUG=y.
817	707	*/
818		-void rcu_nmi_exit(void)
	708	+noinstr void rcu_nmi_exit(void)
819	709	{
820		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	710	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
821	711
	712	+ instrumentation_begin();
822	713	/*
823	714	* Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
824	715	* (We are exiting an NMI handler, so RCU better be paying attention
825	716	* to us!)
826	717	*/
827		- WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);
	718	+ WARN_ON_ONCE(rdp->dynticks_nmi_nesting <= 0);
828	719	WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
829	720
830	721	/*
831	722	* If the nesting level is not 1, the CPU wasn't RCU-idle, so
832	723	* leave it in non-RCU-idle state.
833	724	*/
834		- if (rdtp->dynticks_nmi_nesting != 1) {
835		- trace_rcu_dyntick(TPS("--="), rdtp->dynticks_nmi_nesting, rdtp->dynticks_nmi_nesting - 2, rdtp->dynticks);
836		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* No store tearing. */
837		- rdtp->dynticks_nmi_nesting - 2);
	725	+ if (rdp->dynticks_nmi_nesting != 1) {
	726	+ trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2,
	727	+ atomic_read(&rdp->dynticks));
	728	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */
	729	+ rdp->dynticks_nmi_nesting - 2);
	730	+ instrumentation_end();
838	731	return;
839	732	}
840	733
841	734	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
842		- trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, rdtp->dynticks);
843		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
	735	+ trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks));
	736	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
	737	+
	738	+ if (!in_nmi())
	739	+ rcu_prepare_for_idle();
	740	+
	741	+ // instrumentation for the noinstr rcu_dynticks_eqs_enter()
	742	+ instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
	743	+ instrumentation_end();
	744	+
	745	+ // RCU is watching here ...
844	746	rcu_dynticks_eqs_enter();
	747	+ // ... but is no longer watching here.
	748	+
	749	+ if (!in_nmi())
	750	+ rcu_dynticks_task_enter();
845	751	}
846	752
847	753	/**
..	..	@@ -863,17 +769,51 @@
863	769	* If you add or remove a call to rcu_irq_exit(), be sure to test with
864	770	* CONFIG_RCU_EQS_DEBUG=y.
865	771	*/
866		-void rcu_irq_exit(void)
	772	+void noinstr rcu_irq_exit(void)
867	773	{
868		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
869		-
870	774	lockdep_assert_irqs_disabled();
871		- if (rdtp->dynticks_nmi_nesting == 1)
872		- rcu_prepare_for_idle();
873	775	rcu_nmi_exit();
874		- if (rdtp->dynticks_nmi_nesting == 0)
875		- rcu_dynticks_task_enter();
876	776	}
	777	+
	778	+/**
	779	+ * rcu_irq_exit_preempt - Inform RCU that current CPU is exiting irq
	780	+ * towards in kernel preemption
	781	+ *
	782	+ * Same as rcu_irq_exit() but has a sanity check that scheduling is safe
	783	+ * from RCU point of view. Invoked from return from interrupt before kernel
	784	+ * preemption.
	785	+ */
	786	+void rcu_irq_exit_preempt(void)
	787	+{
	788	+ lockdep_assert_irqs_disabled();
	789	+ rcu_nmi_exit();
	790	+
	791	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0,
	792	+ "RCU dynticks_nesting counter underflow/zero!");
	793	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) !=
	794	+ DYNTICK_IRQ_NONIDLE,
	795	+ "Bad RCU dynticks_nmi_nesting counter\n");
	796	+ RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
	797	+ "RCU in extended quiescent state!");
	798	+}
	799	+
	800	+#ifdef CONFIG_PROVE_RCU
	801	+/**
	802	+ * rcu_irq_exit_check_preempt - Validate that scheduling is possible
	803	+ */
	804	+void rcu_irq_exit_check_preempt(void)
	805	+{
	806	+ lockdep_assert_irqs_disabled();
	807	+
	808	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0,
	809	+ "RCU dynticks_nesting counter underflow/zero!");
	810	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) !=
	811	+ DYNTICK_IRQ_NONIDLE,
	812	+ "Bad RCU dynticks_nmi_nesting counter\n");
	813	+ RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
	814	+ "RCU in extended quiescent state!");
	815	+}
	816	+#endif /* #ifdef CONFIG_PROVE_RCU */
877	817
878	818	/*
879	819	* Wrapper for rcu_irq_exit() where interrupts are enabled.
..	..	@@ -898,26 +838,36 @@
898	838	* allow for the possibility of usermode upcalls messing up our count of
899	839	* interrupt nesting level during the busy period that is just now starting.
900	840	*/
901		-static void rcu_eqs_exit(bool user)
	841	+static void noinstr rcu_eqs_exit(bool user)
902	842	{
903		- struct rcu_dynticks *rdtp;
	843	+ struct rcu_data *rdp;
904	844	long oldval;
905	845
906	846	lockdep_assert_irqs_disabled();
907		- rdtp = this_cpu_ptr(&rcu_dynticks);
908		- oldval = rdtp->dynticks_nesting;
	847	+ rdp = this_cpu_ptr(&rcu_data);
	848	+ oldval = rdp->dynticks_nesting;
909	849	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
910	850	if (oldval) {
911		- rdtp->dynticks_nesting++;
	851	+ // RCU was already watching, so just do accounting and leave.
	852	+ rdp->dynticks_nesting++;
912	853	return;
913	854	}
914	855	rcu_dynticks_task_exit();
	856	+ // RCU is not watching here ...
915	857	rcu_dynticks_eqs_exit();
	858	+ // ... but is watching here.
	859	+ instrumentation_begin();
	860	+
	861	+ // instrumentation for the noinstr rcu_dynticks_eqs_exit()
	862	+ instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
	863	+
916	864	rcu_cleanup_after_idle();
917		- trace_rcu_dyntick(TPS("End"), rdtp->dynticks_nesting, 1, rdtp->dynticks);
	865	+ trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks));
918	866	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
919		- WRITE_ONCE(rdtp->dynticks_nesting, 1);
920		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
	867	+ WRITE_ONCE(rdp->dynticks_nesting, 1);
	868	+ WARN_ON_ONCE(rdp->dynticks_nmi_nesting);
	869	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
	870	+ instrumentation_end();
921	871	}
922	872
923	873	/**
..	..	@@ -937,6 +887,7 @@
937	887	rcu_eqs_exit(false);
938	888	local_irq_restore(flags);
939	889	}
	890	+EXPORT_SYMBOL_GPL(rcu_idle_exit);
940	891
941	892	#ifdef CONFIG_NO_HZ_FULL
942	893	/**
..	..	@@ -948,17 +899,78 @@
948	899	* If you add or remove a call to rcu_user_exit(), be sure to test with
949	900	* CONFIG_RCU_EQS_DEBUG=y.
950	901	*/
951		-void rcu_user_exit(void)
	902	+void noinstr rcu_user_exit(void)
952	903	{
953	904	rcu_eqs_exit(1);
	905	+}
	906	+
	907	+/**
	908	+ * __rcu_irq_enter_check_tick - Enable scheduler tick on CPU if RCU needs it.
	909	+ *
	910	+ * The scheduler tick is not normally enabled when CPUs enter the kernel
	911	+ * from nohz_full userspace execution. After all, nohz_full userspace
	912	+ * execution is an RCU quiescent state and the time executing in the kernel
	913	+ * is quite short. Except of course when it isn't. And it is not hard to
	914	+ * cause a large system to spend tens of seconds or even minutes looping
	915	+ * in the kernel, which can cause a number of problems, include RCU CPU
	916	+ * stall warnings.
	917	+ *
	918	+ * Therefore, if a nohz_full CPU fails to report a quiescent state
	919	+ * in a timely manner, the RCU grace-period kthread sets that CPU's
	920	+ * ->rcu_urgent_qs flag with the expectation that the next interrupt or
	921	+ * exception will invoke this function, which will turn on the scheduler
	922	+ * tick, which will enable RCU to detect that CPU's quiescent states,
	923	+ * for example, due to cond_resched() calls in CONFIG_PREEMPT=n kernels.
	924	+ * The tick will be disabled once a quiescent state is reported for
	925	+ * this CPU.
	926	+ *
	927	+ * Of course, in carefully tuned systems, there might never be an
	928	+ * interrupt or exception. In that case, the RCU grace-period kthread
	929	+ * will eventually cause one to happen. However, in less carefully
	930	+ * controlled environments, this function allows RCU to get what it
	931	+ * needs without creating otherwise useless interruptions.
	932	+ */
	933	+void __rcu_irq_enter_check_tick(void)
	934	+{
	935	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
	936	+
	937	+ // If we're here from NMI there's nothing to do.
	938	+ if (in_nmi())
	939	+ return;
	940	+
	941	+ RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
	942	+ "Illegal rcu_irq_enter_check_tick() from extended quiescent state");
	943	+
	944	+ if (!tick_nohz_full_cpu(rdp->cpu) \|\|
	945	+ !READ_ONCE(rdp->rcu_urgent_qs) \|\|
	946	+ READ_ONCE(rdp->rcu_forced_tick)) {
	947	+ // RCU doesn't need nohz_full help from this CPU, or it is
	948	+ // already getting that help.
	949	+ return;
	950	+ }
	951	+
	952	+ // We get here only when not in an extended quiescent state and
	953	+ // from interrupts (as opposed to NMIs). Therefore, (1) RCU is
	954	+ // already watching and (2) The fact that we are in an interrupt
	955	+ // handler and that the rcu_node lock is an irq-disabled lock
	956	+ // prevents self-deadlock. So we can safely recheck under the lock.
	957	+ // Note that the nohz_full state currently cannot change.
	958	+ raw_spin_lock_rcu_node(rdp->mynode);
	959	+ if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {
	960	+ // A nohz_full CPU is in the kernel and RCU needs a
	961	+ // quiescent state. Turn on the tick!
	962	+ WRITE_ONCE(rdp->rcu_forced_tick, true);
	963	+ tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
	964	+ }
	965	+ raw_spin_unlock_rcu_node(rdp->mynode);
954	966	}
955	967	#endif /* CONFIG_NO_HZ_FULL */
956	968
957	969	/**
958	970	* rcu_nmi_enter - inform RCU of entry to NMI context
959	971	*
960		- * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
961		- * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
	972	+ * If the CPU was idle from RCU's viewpoint, update rdp->dynticks and
	973	+ * rdp->dynticks_nmi_nesting to let the RCU grace-period handling know
962	974	* that the CPU is active. This implementation permits nested NMIs, as
963	975	* long as the nesting level does not overflow an int. (You will probably
964	976	* run out of stack space first.)
..	..	@@ -966,13 +978,13 @@
966	978	* If you add or remove a call to rcu_nmi_enter(), be sure to test
967	979	* with CONFIG_RCU_EQS_DEBUG=y.
968	980	*/
969		-void rcu_nmi_enter(void)
	981	+noinstr void rcu_nmi_enter(void)
970	982	{
971		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
972	983	long incby = 2;
	984	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
973	985
974	986	/* Complain about underflow. */
975		- WARN_ON_ONCE(rdtp->dynticks_nmi_nesting < 0);
	987	+ WARN_ON_ONCE(rdp->dynticks_nmi_nesting < 0);
976	988
977	989	/*
978	990	* If idle from RCU viewpoint, atomically increment ->dynticks
..	..	@@ -983,14 +995,40 @@
983	995	* period (observation due to Andy Lutomirski).
984	996	*/
985	997	if (rcu_dynticks_curr_cpu_in_eqs()) {
	998	+
	999	+ if (!in_nmi())
	1000	+ rcu_dynticks_task_exit();
	1001	+
	1002	+ // RCU is not watching here ...
986	1003	rcu_dynticks_eqs_exit();
	1004	+ // ... but is watching here.
	1005	+
	1006	+ if (!in_nmi()) {
	1007	+ instrumentation_begin();
	1008	+ rcu_cleanup_after_idle();
	1009	+ instrumentation_end();
	1010	+ }
	1011	+
	1012	+ instrumentation_begin();
	1013	+ // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs()
	1014	+ instrument_atomic_read(&rdp->dynticks, sizeof(rdp->dynticks));
	1015	+ // instrumentation for the noinstr rcu_dynticks_eqs_exit()
	1016	+ instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
	1017	+
987	1018	incby = 1;
	1019	+ } else if (!in_nmi()) {
	1020	+ instrumentation_begin();
	1021	+ rcu_irq_enter_check_tick();
	1022	+ } else {
	1023	+ instrumentation_begin();
988	1024	}
	1025	+
989	1026	trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
990		- rdtp->dynticks_nmi_nesting,
991		- rdtp->dynticks_nmi_nesting + incby, rdtp->dynticks);
992		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* Prevent store tearing. */
993		- rdtp->dynticks_nmi_nesting + incby);
	1027	+ rdp->dynticks_nmi_nesting,
	1028	+ rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks));
	1029	+ instrumentation_end();
	1030	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, /* Prevent store tearing. */
	1031	+ rdp->dynticks_nmi_nesting + incby);
994	1032	barrier();
995	1033	}
996	1034
..	..	@@ -1016,16 +1054,10 @@
1016	1054	* If you add or remove a call to rcu_irq_enter(), be sure to test with
1017	1055	* CONFIG_RCU_EQS_DEBUG=y.
1018	1056	*/
1019		-void rcu_irq_enter(void)
	1057	+noinstr void rcu_irq_enter(void)
1020	1058	{
1021		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1022		-
1023	1059	lockdep_assert_irqs_disabled();
1024		- if (rdtp->dynticks_nmi_nesting == 0)
1025		- rcu_dynticks_task_exit();
1026	1060	rcu_nmi_enter();
1027		- if (rdtp->dynticks_nmi_nesting == 1)
1028		- rcu_cleanup_after_idle();
1029	1061	}
1030	1062
1031	1063	/*
..	..	@@ -1043,15 +1075,34 @@
1043	1075	local_irq_restore(flags);
1044	1076	}
1045	1077
	1078	+/*
	1079	+ * If any sort of urgency was applied to the current CPU (for example,
	1080	+ * the scheduler-clock interrupt was enabled on a nohz_full CPU) in order
	1081	+ * to get to a quiescent state, disable it.
	1082	+ */
	1083	+static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp)
	1084	+{
	1085	+ raw_lockdep_assert_held_rcu_node(rdp->mynode);
	1086	+ WRITE_ONCE(rdp->rcu_urgent_qs, false);
	1087	+ WRITE_ONCE(rdp->rcu_need_heavy_qs, false);
	1088	+ if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) {
	1089	+ tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
	1090	+ WRITE_ONCE(rdp->rcu_forced_tick, false);
	1091	+ }
	1092	+}
	1093	+
1046	1094	/**
1047		- * rcu_is_watching - see if RCU thinks that the current CPU is idle
	1095	+ * rcu_is_watching - see if RCU thinks that the current CPU is not idle
1048	1096	*
1049	1097	* Return true if RCU is watching the running CPU, which means that this
1050	1098	* CPU can safely enter RCU read-side critical sections. In other words,
1051		- * if the current CPU is in its idle loop and is neither in an interrupt
1052		- * or NMI handler, return true.
	1099	+ * if the current CPU is not in its idle loop or is in an interrupt or
	1100	+ * NMI handler, return true.
	1101	+ *
	1102	+ * Make notrace because it can be called by the internal functions of
	1103	+ * ftrace, and making this notrace removes unnecessary recursion calls.
1053	1104	*/
1054		-bool notrace rcu_is_watching(void)
	1105	+notrace bool rcu_is_watching(void)
1055	1106	{
1056	1107	bool ret;
1057	1108
..	..	@@ -1077,7 +1128,7 @@
1077	1128	cpu = task_cpu(t);
1078	1129	if (!task_curr(t))
1079	1130	return; /* This task is not running on that CPU. */
1080		- smp_store_release(per_cpu_ptr(&rcu_dynticks.rcu_urgent_qs, cpu), true);
	1131	+ smp_store_release(per_cpu_ptr(&rcu_data.rcu_urgent_qs, cpu), true);
1081	1132	}
1082	1133
1083	1134	#if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
..	..	@@ -1088,11 +1139,7 @@
1088	1139	* Disable preemption to avoid false positives that could otherwise
1089	1140	* happen due to the current CPU number being sampled, this task being
1090	1141	* preempted, its old CPU being taken offline, resuming on some other CPU,
1091		- * then determining that its old CPU is now offline. Because there are
1092		- * multiple flavors of RCU, and because this function can be called in the
1093		- * midst of updating the flavors while a given CPU coming online or going
1094		- * offline, it is necessary to check all flavors. If any of the flavors
1095		- * believe that given CPU is online, it is considered to be online.
	1142	+ * then determining that its old CPU is now offline.
1096	1143	*
1097	1144	* Disable checking if in an NMI handler because we cannot safely
1098	1145	* report errors from NMI handlers anyway. In addition, it is OK to use
..	..	@@ -1103,38 +1150,21 @@
1103	1150	{
1104	1151	struct rcu_data *rdp;
1105	1152	struct rcu_node *rnp;
1106		- struct rcu_state *rsp;
	1153	+ bool ret = false;
1107	1154
1108	1155	if (in_nmi() \|\| !rcu_scheduler_fully_active)
1109	1156	return true;
1110		- preempt_disable();
1111		- for_each_rcu_flavor(rsp) {
1112		- rdp = this_cpu_ptr(rsp->rda);
1113		- rnp = rdp->mynode;
1114		- if (rdp->grpmask & rcu_rnp_online_cpus(rnp)) {
1115		- preempt_enable();
1116		- return true;
1117		- }
1118		- }
1119		- preempt_enable();
1120		- return false;
	1157	+ preempt_disable_notrace();
	1158	+ rdp = this_cpu_ptr(&rcu_data);
	1159	+ rnp = rdp->mynode;
	1160	+ if (rdp->grpmask & rcu_rnp_online_cpus(rnp))
	1161	+ ret = true;
	1162	+ preempt_enable_notrace();
	1163	+ return ret;
1121	1164	}
1122	1165	EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);
1123	1166
1124	1167	#endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */
1125		-
1126		-/**
1127		- * rcu_is_cpu_rrupt_from_idle - see if idle or immediately interrupted from idle
1128		- *
1129		- * If the current CPU is idle or running at a first-level (not nested)
1130		- * interrupt from idle, return true. The caller must have at least
1131		- * disabled preemption.
1132		- */
1133		-static int rcu_is_cpu_rrupt_from_idle(void)
1134		-{
1135		- return __this_cpu_read(rcu_dynticks.dynticks_nesting) <= 0 &&
1136		- __this_cpu_read(rcu_dynticks.dynticks_nmi_nesting) <= 1;
1137		-}
1138	1168
1139	1169	/*
1140	1170	* We are reporting a quiescent state on behalf of some other CPU, so
..	..	@@ -1160,34 +1190,13 @@
1160	1190	*/
1161	1191	static int dyntick_save_progress_counter(struct rcu_data *rdp)
1162	1192	{
1163		- rdp->dynticks_snap = rcu_dynticks_snap(rdp->dynticks);
	1193	+ rdp->dynticks_snap = rcu_dynticks_snap(rdp);
1164	1194	if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) {
1165		- trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("dti"));
	1195	+ trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
1166	1196	rcu_gpnum_ovf(rdp->mynode, rdp);
1167	1197	return 1;
1168	1198	}
1169	1199	return 0;
1170		-}
1171		-
1172		-/*
1173		- * Handler for the irq_work request posted when a grace period has
1174		- * gone on for too long, but not yet long enough for an RCU CPU
1175		- * stall warning. Set state appropriately, but just complain if
1176		- * there is unexpected state on entry.
1177		- */
1178		-static void rcu_iw_handler(struct irq_work *iwp)
1179		-{
1180		- struct rcu_data *rdp;
1181		- struct rcu_node *rnp;
1182		-
1183		- rdp = container_of(iwp, struct rcu_data, rcu_iw);
1184		- rnp = rdp->mynode;
1185		- raw_spin_lock_rcu_node(rnp);
1186		- if (!WARN_ON_ONCE(!rdp->rcu_iw_pending)) {
1187		- rdp->rcu_iw_gp_seq = rnp->gp_seq;
1188		- rdp->rcu_iw_pending = false;
1189		- }
1190		- raw_spin_unlock_rcu_node(rnp);
1191	1200	}
1192	1201
1193	1202	/*
..	..	@@ -1211,39 +1220,34 @@
1211	1220	* read-side critical section that started before the beginning
1212	1221	* of the current RCU grace period.
1213	1222	*/
1214		- if (rcu_dynticks_in_eqs_since(rdp->dynticks, rdp->dynticks_snap)) {
1215		- trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("dti"));
1216		- rdp->dynticks_fqs++;
	1223	+ if (rcu_dynticks_in_eqs_since(rdp, rdp->dynticks_snap)) {
	1224	+ trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
1217	1225	rcu_gpnum_ovf(rnp, rdp);
1218	1226	return 1;
1219	1227	}
1220	1228
1221	1229	/*
1222		- * Has this CPU encountered a cond_resched() since the beginning
1223		- * of the grace period? For this to be the case, the CPU has to
1224		- * have noticed the current grace period. This might not be the
1225		- * case for nohz_full CPUs looping in the kernel.
	1230	+ * Complain if a CPU that is considered to be offline from RCU's
	1231	+ * perspective has not yet reported a quiescent state. After all,
	1232	+ * the offline CPU should have reported a quiescent state during
	1233	+ * the CPU-offline process, or, failing that, by rcu_gp_init()
	1234	+ * if it ran concurrently with either the CPU going offline or the
	1235	+ * last task on a leaf rcu_node structure exiting its RCU read-side
	1236	+ * critical section while all CPUs corresponding to that structure
	1237	+ * are offline. This added warning detects bugs in any of these
	1238	+ * code paths.
	1239	+ *
	1240	+ * The rcu_node structure's ->lock is held here, which excludes
	1241	+ * the relevant portions the CPU-hotplug code, the grace-period
	1242	+ * initialization code, and the rcu_read_unlock() code paths.
	1243	+ *
	1244	+ * For more detail, please refer to the "Hotplug CPU" section
	1245	+ * of RCU's Requirements documentation.
1226	1246	*/
1227		- jtsq = jiffies_till_sched_qs;
1228		- ruqp = per_cpu_ptr(&rcu_dynticks.rcu_urgent_qs, rdp->cpu);
1229		- if (time_after(jiffies, rdp->rsp->gp_start + jtsq) &&
1230		- READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_dynticks.rcu_qs_ctr, rdp->cpu) &&
1231		- rcu_seq_current(&rdp->gp_seq) == rnp->gp_seq && !rdp->gpwrap) {
1232		- trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("rqc"));
1233		- rcu_gpnum_ovf(rnp, rdp);
1234		- return 1;
1235		- } else if (time_after(jiffies, rdp->rsp->gp_start + jtsq)) {
1236		- /* Load rcu_qs_ctr before store to rcu_urgent_qs. */
1237		- smp_store_release(ruqp, true);
1238		- }
1239		-
1240		- /* If waiting too long on an offline CPU, complain. */
1241		- if (!(rdp->grpmask & rcu_rnp_online_cpus(rnp)) &&
1242		- time_after(jiffies, rdp->rsp->gp_start + HZ)) {
	1247	+ if (WARN_ON_ONCE(!(rdp->grpmask & rcu_rnp_online_cpus(rnp)))) {
1243	1248	bool onl;
1244	1249	struct rcu_node *rnp1;
1245	1250
1246		- WARN_ON(1); /* Offline CPUs are supposed to report QS! */
1247	1251	pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
1248	1252	__func__, rnp->grplo, rnp->grphi, rnp->level,
1249	1253	(long)rnp->gp_seq, (long)rnp->completedqs);
..	..	@@ -1260,44 +1264,63 @@
1260	1264
1261	1265	/*
1262	1266	* A CPU running for an extended time within the kernel can
1263		- * delay RCU grace periods. When the CPU is in NO_HZ_FULL mode,
1264		- * even context-switching back and forth between a pair of
1265		- * in-kernel CPU-bound tasks cannot advance grace periods.
1266		- * So if the grace period is old enough, make the CPU pay attention.
1267		- * Note that the unsynchronized assignments to the per-CPU
1268		- * rcu_need_heavy_qs variable are safe. Yes, setting of
1269		- * bits can be lost, but they will be set again on the next
1270		- * force-quiescent-state pass. So lost bit sets do not result
1271		- * in incorrect behavior, merely in a grace period lasting
1272		- * a few jiffies longer than it might otherwise. Because
1273		- * there are at most four threads involved, and because the
1274		- * updates are only once every few jiffies, the probability of
1275		- * lossage (and thus of slight grace-period extension) is
1276		- * quite low.
	1267	+ * delay RCU grace periods: (1) At age jiffies_to_sched_qs,
	1268	+ * set .rcu_urgent_qs, (2) At age 2*jiffies_to_sched_qs, set
	1269	+ * both .rcu_need_heavy_qs and .rcu_urgent_qs. Note that the
	1270	+ * unsynchronized assignments to the per-CPU rcu_need_heavy_qs
	1271	+ * variable are safe because the assignments are repeated if this
	1272	+ * CPU failed to pass through a quiescent state. This code
	1273	+ * also checks .jiffies_resched in case jiffies_to_sched_qs
	1274	+ * is set way high.
1277	1275	*/
1278		- rnhqp = &per_cpu(rcu_dynticks.rcu_need_heavy_qs, rdp->cpu);
	1276	+ jtsq = READ_ONCE(jiffies_to_sched_qs);
	1277	+ ruqp = per_cpu_ptr(&rcu_data.rcu_urgent_qs, rdp->cpu);
	1278	+ rnhqp = &per_cpu(rcu_data.rcu_need_heavy_qs, rdp->cpu);
1279	1279	if (!READ_ONCE(*rnhqp) &&
1280		- (time_after(jiffies, rdp->rsp->gp_start + jtsq) \|\|
1281		- time_after(jiffies, rdp->rsp->jiffies_resched))) {
	1280	+ (time_after(jiffies, rcu_state.gp_start + jtsq * 2) \|\|
	1281	+ time_after(jiffies, rcu_state.jiffies_resched) \|\|
	1282	+ rcu_state.cbovld)) {
1282	1283	WRITE_ONCE(*rnhqp, true);
1283	1284	/* Store rcu_need_heavy_qs before rcu_urgent_qs. */
1284	1285	smp_store_release(ruqp, true);
1285		- rdp->rsp->jiffies_resched += jtsq; /* Re-enable beating. */
	1286	+ } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) {
	1287	+ WRITE_ONCE(*ruqp, true);
1286	1288	}
1287	1289
1288	1290	/*
1289		- * If more than halfway to RCU CPU stall-warning time, do a
1290		- * resched_cpu() to try to loosen things up a bit. Also check to
1291		- * see if the CPU is getting hammered with interrupts, but only
1292		- * once per grace period, just to keep the IPIs down to a dull roar.
	1291	+ * NO_HZ_FULL CPUs can run in-kernel without rcu_sched_clock_irq!
	1292	+ * The above code handles this, but only for straight cond_resched().
	1293	+ * And some in-kernel loops check need_resched() before calling
	1294	+ * cond_resched(), which defeats the above code for CPUs that are
	1295	+ * running in-kernel with scheduling-clock interrupts disabled.
	1296	+ * So hit them over the head with the resched_cpu() hammer!
1293	1297	*/
1294		- if (jiffies - rdp->rsp->gp_start > rcu_jiffies_till_stall_check() / 2) {
	1298	+ if (tick_nohz_full_cpu(rdp->cpu) &&
	1299	+ (time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) \|\|
	1300	+ rcu_state.cbovld)) {
	1301	+ WRITE_ONCE(*ruqp, true);
1295	1302	resched_cpu(rdp->cpu);
	1303	+ WRITE_ONCE(rdp->last_fqs_resched, jiffies);
	1304	+ }
	1305	+
	1306	+ /*
	1307	+ * If more than halfway to RCU CPU stall-warning time, invoke
	1308	+ * resched_cpu() more frequently to try to loosen things up a bit.
	1309	+ * Also check to see if the CPU is getting hammered with interrupts,
	1310	+ * but only once per grace period, just to keep the IPIs down to
	1311	+ * a dull roar.
	1312	+ */
	1313	+ if (time_after(jiffies, rcu_state.jiffies_resched)) {
	1314	+ if (time_after(jiffies,
	1315	+ READ_ONCE(rdp->last_fqs_resched) + jtsq)) {
	1316	+ resched_cpu(rdp->cpu);
	1317	+ WRITE_ONCE(rdp->last_fqs_resched, jiffies);
	1318	+ }
1296	1319	if (IS_ENABLED(CONFIG_IRQ_WORK) &&
1297	1320	!rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
1298	1321	(rnp->ffmask & rdp->grpmask)) {
1299	1322	init_irq_work(&rdp->rcu_iw, rcu_iw_handler);
1300		- rdp->rcu_iw.flags = IRQ_WORK_HARD_IRQ;
	1323	+ atomic_set(&rdp->rcu_iw.flags, IRQ_WORK_HARD_IRQ);
1301	1324	rdp->rcu_iw_pending = true;
1302	1325	rdp->rcu_iw_gp_seq = rnp->gp_seq;
1303	1326	irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
..	..	@@ -1307,317 +1330,13 @@
1307	1330	return 0;
1308	1331	}
1309	1332
1310		-static void record_gp_stall_check_time(struct rcu_state *rsp)
1311		-{
1312		- unsigned long j = jiffies;
1313		- unsigned long j1;
1314		-
1315		- rsp->gp_start = j;
1316		- j1 = rcu_jiffies_till_stall_check();
1317		- /* Record ->gp_start before ->jiffies_stall. */
1318		- smp_store_release(&rsp->jiffies_stall, j + j1); /* ^^^ */
1319		- rsp->jiffies_resched = j + j1 / 2;
1320		- rsp->n_force_qs_gpstart = READ_ONCE(rsp->n_force_qs);
1321		-}
1322		-
1323		-/*
1324		- * Convert a ->gp_state value to a character string.
1325		- */
1326		-static const char *gp_state_getname(short gs)
1327		-{
1328		- if (gs < 0 \|\| gs >= ARRAY_SIZE(gp_state_names))
1329		- return "???";
1330		- return gp_state_names[gs];
1331		-}
1332		-
1333		-/*
1334		- * Complain about starvation of grace-period kthread.
1335		- */
1336		-static void rcu_check_gp_kthread_starvation(struct rcu_state *rsp)
1337		-{
1338		- unsigned long gpa;
1339		- unsigned long j;
1340		-
1341		- j = jiffies;
1342		- gpa = READ_ONCE(rsp->gp_activity);
1343		- if (j - gpa > 2 * HZ) {
1344		- pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#lx ->cpu=%d\n",
1345		- rsp->name, j - gpa,
1346		- (long)rcu_seq_current(&rsp->gp_seq),
1347		- rsp->gp_flags,
1348		- gp_state_getname(rsp->gp_state), rsp->gp_state,
1349		- rsp->gp_kthread ? rsp->gp_kthread->state : ~0,
1350		- rsp->gp_kthread ? task_cpu(rsp->gp_kthread) : -1);
1351		- if (rsp->gp_kthread) {
1352		- pr_err("RCU grace-period kthread stack dump:\n");
1353		- sched_show_task(rsp->gp_kthread);
1354		- wake_up_process(rsp->gp_kthread);
1355		- }
1356		- }
1357		-}
1358		-
1359		-/*
1360		- * Dump stacks of all tasks running on stalled CPUs. First try using
1361		- * NMIs, but fall back to manual remote stack tracing on architectures
1362		- * that don't support NMI-based stack dumps. The NMI-triggered stack
1363		- * traces are more accurate because they are printed by the target CPU.
1364		- */
1365		-static void rcu_dump_cpu_stacks(struct rcu_state *rsp)
1366		-{
1367		- int cpu;
1368		- unsigned long flags;
1369		- struct rcu_node *rnp;
1370		-
1371		- rcu_for_each_leaf_node(rsp, rnp) {
1372		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
1373		- for_each_leaf_node_possible_cpu(rnp, cpu)
1374		- if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu))
1375		- if (!trigger_single_cpu_backtrace(cpu))
1376		- dump_cpu_task(cpu);
1377		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1378		- }
1379		-}
1380		-
1381		-/*
1382		- * If too much time has passed in the current grace period, and if
1383		- * so configured, go kick the relevant kthreads.
1384		- */
1385		-static void rcu_stall_kick_kthreads(struct rcu_state *rsp)
1386		-{
1387		- unsigned long j;
1388		-
1389		- if (!rcu_kick_kthreads)
1390		- return;
1391		- j = READ_ONCE(rsp->jiffies_kick_kthreads);
1392		- if (time_after(jiffies, j) && rsp->gp_kthread &&
1393		- (rcu_gp_in_progress(rsp) \|\| READ_ONCE(rsp->gp_flags))) {
1394		- WARN_ONCE(1, "Kicking %s grace-period kthread\n", rsp->name);
1395		- rcu_ftrace_dump(DUMP_ALL);
1396		- wake_up_process(rsp->gp_kthread);
1397		- WRITE_ONCE(rsp->jiffies_kick_kthreads, j + HZ);
1398		- }
1399		-}
1400		-
1401		-static void panic_on_rcu_stall(void)
1402		-{
1403		- if (sysctl_panic_on_rcu_stall)
1404		- panic("RCU Stall\n");
1405		-}
1406		-
1407		-static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gp_seq)
1408		-{
1409		- int cpu;
1410		- unsigned long flags;
1411		- unsigned long gpa;
1412		- unsigned long j;
1413		- int ndetected = 0;
1414		- struct rcu_node *rnp = rcu_get_root(rsp);
1415		- long totqlen = 0;
1416		-
1417		- /* Kick and suppress, if so configured. */
1418		- rcu_stall_kick_kthreads(rsp);
1419		- if (rcu_cpu_stall_suppress)
1420		- return;
1421		-
1422		- /*
1423		- * OK, time to rat on our buddy...
1424		- * See Documentation/RCU/stallwarn.txt for info on how to debug
1425		- * RCU CPU stall warnings.
1426		- */
1427		- pr_err("INFO: %s detected stalls on CPUs/tasks:", rsp->name);
1428		- print_cpu_stall_info_begin();
1429		- rcu_for_each_leaf_node(rsp, rnp) {
1430		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
1431		- ndetected += rcu_print_task_stall(rnp);
1432		- if (rnp->qsmask != 0) {
1433		- for_each_leaf_node_possible_cpu(rnp, cpu)
1434		- if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
1435		- print_cpu_stall_info(rsp, cpu);
1436		- ndetected++;
1437		- }
1438		- }
1439		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1440		- }
1441		-
1442		- print_cpu_stall_info_end();
1443		- for_each_possible_cpu(cpu)
1444		- totqlen += rcu_segcblist_n_cbs(&per_cpu_ptr(rsp->rda,
1445		- cpu)->cblist);
1446		- pr_cont("(detected by %d, t=%ld jiffies, g=%ld, q=%lu)\n",
1447		- smp_processor_id(), (long)(jiffies - rsp->gp_start),
1448		- (long)rcu_seq_current(&rsp->gp_seq), totqlen);
1449		- if (ndetected) {
1450		- rcu_dump_cpu_stacks(rsp);
1451		-
1452		- /* Complain about tasks blocking the grace period. */
1453		- rcu_print_detail_task_stall(rsp);
1454		- } else {
1455		- if (rcu_seq_current(&rsp->gp_seq) != gp_seq) {
1456		- pr_err("INFO: Stall ended before state dump start\n");
1457		- } else {
1458		- j = jiffies;
1459		- gpa = READ_ONCE(rsp->gp_activity);
1460		- pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n",
1461		- rsp->name, j - gpa, j, gpa,
1462		- jiffies_till_next_fqs,
1463		- rcu_get_root(rsp)->qsmask);
1464		- /* In this case, the current CPU might be at fault. */
1465		- sched_show_task(current);
1466		- }
1467		- }
1468		- /* Rewrite if needed in case of slow consoles. */
1469		- if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall)))
1470		- WRITE_ONCE(rsp->jiffies_stall,
1471		- jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
1472		-
1473		- rcu_check_gp_kthread_starvation(rsp);
1474		-
1475		- atomic_notifier_call_chain(&rcu_stall_notifier_list, 0, NULL);
1476		-
1477		- panic_on_rcu_stall();
1478		-
1479		- force_quiescent_state(rsp); /* Kick them all. */
1480		-}
1481		-
1482		-static void print_cpu_stall(struct rcu_state *rsp)
1483		-{
1484		- int cpu;
1485		- unsigned long flags;
1486		- struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
1487		- struct rcu_node *rnp = rcu_get_root(rsp);
1488		- long totqlen = 0;
1489		-
1490		- /* Kick and suppress, if so configured. */
1491		- rcu_stall_kick_kthreads(rsp);
1492		- if (rcu_cpu_stall_suppress)
1493		- return;
1494		-
1495		- /*
1496		- * OK, time to rat on ourselves...
1497		- * See Documentation/RCU/stallwarn.txt for info on how to debug
1498		- * RCU CPU stall warnings.
1499		- */
1500		- pr_err("INFO: %s self-detected stall on CPU", rsp->name);
1501		- print_cpu_stall_info_begin();
1502		- raw_spin_lock_irqsave_rcu_node(rdp->mynode, flags);
1503		- print_cpu_stall_info(rsp, smp_processor_id());
1504		- raw_spin_unlock_irqrestore_rcu_node(rdp->mynode, flags);
1505		- print_cpu_stall_info_end();
1506		- for_each_possible_cpu(cpu)
1507		- totqlen += rcu_segcblist_n_cbs(&per_cpu_ptr(rsp->rda,
1508		- cpu)->cblist);
1509		- pr_cont(" (t=%lu jiffies g=%ld q=%lu)\n",
1510		- jiffies - rsp->gp_start,
1511		- (long)rcu_seq_current(&rsp->gp_seq), totqlen);
1512		-
1513		- rcu_check_gp_kthread_starvation(rsp);
1514		-
1515		- rcu_dump_cpu_stacks(rsp);
1516		-
1517		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
1518		- /* Rewrite if needed in case of slow consoles. */
1519		- if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall)))
1520		- WRITE_ONCE(rsp->jiffies_stall,
1521		- jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
1522		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1523		-
1524		- panic_on_rcu_stall();
1525		-
1526		- /*
1527		- * Attempt to revive the RCU machinery by forcing a context switch.
1528		- *
1529		- * A context switch would normally allow the RCU state machine to make
1530		- * progress and it could be we're stuck in kernel space without context
1531		- * switches for an entirely unreasonable amount of time.
1532		- */
1533		- resched_cpu(smp_processor_id());
1534		-}
1535		-
1536		-static void check_cpu_stall(struct rcu_state rsp, struct rcu_data rdp)
1537		-{
1538		- unsigned long gs1;
1539		- unsigned long gs2;
1540		- unsigned long gps;
1541		- unsigned long j;
1542		- unsigned long jn;
1543		- unsigned long js;
1544		- struct rcu_node *rnp;
1545		-
1546		- if ((rcu_cpu_stall_suppress && !rcu_kick_kthreads) \|\|
1547		- !rcu_gp_in_progress(rsp))
1548		- return;
1549		- rcu_stall_kick_kthreads(rsp);
1550		- j = jiffies;
1551		-
1552		- /*
1553		- * Lots of memory barriers to reject false positives.
1554		- *
1555		- * The idea is to pick up rsp->gp_seq, then rsp->jiffies_stall,
1556		- * then rsp->gp_start, and finally another copy of rsp->gp_seq.
1557		- * These values are updated in the opposite order with memory
1558		- * barriers (or equivalent) during grace-period initialization
1559		- * and cleanup. Now, a false positive can occur if we get an new
1560		- * value of rsp->gp_start and a old value of rsp->jiffies_stall.
1561		- * But given the memory barriers, the only way that this can happen
1562		- * is if one grace period ends and another starts between these
1563		- * two fetches. This is detected by comparing the second fetch
1564		- * of rsp->gp_seq with the previous fetch from rsp->gp_seq.
1565		- *
1566		- * Given this check, comparisons of jiffies, rsp->jiffies_stall,
1567		- * and rsp->gp_start suffice to forestall false positives.
1568		- */
1569		- gs1 = READ_ONCE(rsp->gp_seq);
1570		- smp_rmb(); /* Pick up ->gp_seq first... */
1571		- js = READ_ONCE(rsp->jiffies_stall);
1572		- smp_rmb(); /* ...then ->jiffies_stall before the rest... */
1573		- gps = READ_ONCE(rsp->gp_start);
1574		- smp_rmb(); /* ...and finally ->gp_start before ->gp_seq again. */
1575		- gs2 = READ_ONCE(rsp->gp_seq);
1576		- if (gs1 != gs2 \|\|
1577		- ULONG_CMP_LT(j, js) \|\|
1578		- ULONG_CMP_GE(gps, js))
1579		- return; /* No stall or GP completed since entering function. */
1580		- rnp = rdp->mynode;
1581		- jn = jiffies + 3 * rcu_jiffies_till_stall_check() + 3;
1582		- if (rcu_gp_in_progress(rsp) &&
1583		- (READ_ONCE(rnp->qsmask) & rdp->grpmask) &&
1584		- cmpxchg(&rsp->jiffies_stall, js, jn) == js) {
1585		-
1586		- /* We haven't checked in, so go dump stack. */
1587		- print_cpu_stall(rsp);
1588		-
1589		- } else if (rcu_gp_in_progress(rsp) &&
1590		- ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY) &&
1591		- cmpxchg(&rsp->jiffies_stall, js, jn) == js) {
1592		-
1593		- /* They had a few time units to dump stack, so complain. */
1594		- print_other_cpu_stall(rsp, gs2);
1595		- }
1596		-}
1597		-
1598		-/**
1599		- * rcu_cpu_stall_reset - prevent further stall warnings in current grace period
1600		- *
1601		- * Set the stall-warning timeout way off into the future, thus preventing
1602		- * any RCU CPU stall-warning messages from appearing in the current set of
1603		- * RCU grace periods.
1604		- *
1605		- * The caller must disable hard irqs.
1606		- */
1607		-void rcu_cpu_stall_reset(void)
1608		-{
1609		- struct rcu_state *rsp;
1610		-
1611		- for_each_rcu_flavor(rsp)
1612		- WRITE_ONCE(rsp->jiffies_stall, jiffies + ULONG_MAX / 2);
1613		-}
1614		-
1615	1333	/* Trace-event wrapper function for trace_rcu_future_grace_period. */
1616	1334	static void trace_rcu_this_gp(struct rcu_node rnp, struct rcu_data rdp,
1617	1335	unsigned long gp_seq_req, const char *s)
1618	1336	{
1619		- trace_rcu_future_grace_period(rdp->rsp->name, rnp->gp_seq, gp_seq_req,
1620		- rnp->level, rnp->grplo, rnp->grphi, s);
	1337	+ trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
	1338	+ gp_seq_req, rnp->level,
	1339	+ rnp->grplo, rnp->grphi, s);
1621	1340	}
1622	1341
1623	1342	/*
..	..	@@ -1640,7 +1359,6 @@
1640	1359	unsigned long gp_seq_req)
1641	1360	{
1642	1361	bool ret = false;
1643		- struct rcu_state *rsp = rdp->rsp;
1644	1362	struct rcu_node *rnp;
1645	1363
1646	1364	/*
..	..	@@ -1665,7 +1383,7 @@
1665	1383	TPS("Prestarted"));
1666	1384	goto unlock_out;
1667	1385	}
1668		- rnp->gp_seq_needed = gp_seq_req;
	1386	+ WRITE_ONCE(rnp->gp_seq_needed, gp_seq_req);
1669	1387	if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {
1670	1388	/*
1671	1389	* We just marked the leaf or internal node, and a
..	..	@@ -1684,24 +1402,24 @@
1684	1402	}
1685	1403
1686	1404	/* If GP already in progress, just leave, otherwise start one. */
1687		- if (rcu_gp_in_progress(rsp)) {
	1405	+ if (rcu_gp_in_progress()) {
1688	1406	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedleafroot"));
1689	1407	goto unlock_out;
1690	1408	}
1691	1409	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot"));
1692		- WRITE_ONCE(rsp->gp_flags, rsp->gp_flags \| RCU_GP_FLAG_INIT);
1693		- rsp->gp_req_activity = jiffies;
1694		- if (!rsp->gp_kthread) {
	1410	+ WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags \| RCU_GP_FLAG_INIT);
	1411	+ WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
	1412	+ if (!READ_ONCE(rcu_state.gp_kthread)) {
1695	1413	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread"));
1696	1414	goto unlock_out;
1697	1415	}
1698		- trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq), TPS("newreq"));
	1416	+ trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq"));
1699	1417	ret = true; /* Caller must wake GP kthread. */
1700	1418	unlock_out:
1701	1419	/* Push furthest requested GP to leaf node and rcu_data structure. */
1702	1420	if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) {
1703		- rnp_start->gp_seq_needed = rnp->gp_seq_needed;
1704		- rdp->gp_seq_needed = rnp->gp_seq_needed;
	1421	+ WRITE_ONCE(rnp_start->gp_seq_needed, rnp->gp_seq_needed);
	1422	+ WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
1705	1423	}
1706	1424	if (rnp != rnp_start)
1707	1425	raw_spin_unlock_rcu_node(rnp);
..	..	@@ -1712,10 +1430,10 @@
1712	1430	* Clean up any old requests for the just-ended grace period. Also return
1713	1431	* whether any additional grace periods have been requested.
1714	1432	*/
1715		-static bool rcu_future_gp_cleanup(struct rcu_state rsp, struct rcu_node rnp)
	1433	+static bool rcu_future_gp_cleanup(struct rcu_node *rnp)
1716	1434	{
1717	1435	bool needmore;
1718		- struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
	1436	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1719	1437
1720	1438	needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed);
1721	1439	if (!needmore)
..	..	@@ -1726,12 +1444,13 @@
1726	1444	}
1727	1445
1728	1446	/*
1729		- * Awaken the grace-period kthread. Don't do a self-awaken (unless in
1730		- * an interrupt or softirq handler), and don't bother awakening when there
1731		- * is nothing for the grace-period kthread to do (as in several CPUs raced
1732		- * to awaken, and we lost), and finally don't try to awaken a kthread that
1733		- * has not yet been created. If all those checks are passed, track some
1734		- * debug information and awaken.
	1447	+ * Awaken the grace-period kthread. Don't do a self-awaken (unless in an
	1448	+ * interrupt or softirq handler, in which case we just might immediately
	1449	+ * sleep upon return, resulting in a grace-period hang), and don't bother
	1450	+ * awakening when there is nothing for the grace-period kthread to do
	1451	+ * (as in several CPUs raced to awaken, we lost), and finally don't try
	1452	+ * to awaken a kthread that has not yet been created. If all those checks
	1453	+ * are passed, track some debug information and awaken.
1735	1454	*
1736	1455	* So why do the self-wakeup when in an interrupt or softirq handler
1737	1456	* in the grace-period kthread's context? Because the kthread might have
..	..	@@ -1739,14 +1458,16 @@
1739	1458	* pre-sleep check of the awaken condition. In this case, a wakeup really
1740	1459	* is required, and is therefore supplied.
1741	1460	*/
1742		-static void rcu_gp_kthread_wake(struct rcu_state *rsp)
	1461	+static void rcu_gp_kthread_wake(void)
1743	1462	{
1744		- if ((current == rsp->gp_kthread &&
1745		- !in_interrupt() && !in_serving_softirq()) \|\|
1746		- !READ_ONCE(rsp->gp_flags) \|\|
1747		- !rsp->gp_kthread)
	1463	+ struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
	1464	+
	1465	+ if ((current == t && !in_irq() && !in_serving_softirq()) \|\|
	1466	+ !READ_ONCE(rcu_state.gp_flags) \|\| !t)
1748	1467	return;
1749		- swake_up_one(&rsp->gp_wq);
	1468	+ WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
	1469	+ WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
	1470	+ swake_up_one(&rcu_state.gp_wq);
1750	1471	}
1751	1472
1752	1473	/*
..	..	@@ -1761,12 +1482,12 @@
1761	1482	*
1762	1483	* The caller must hold rnp->lock with interrupts disabled.
1763	1484	*/
1764		-static bool rcu_accelerate_cbs(struct rcu_state rsp, struct rcu_node rnp,
1765		- struct rcu_data *rdp)
	1485	+static bool rcu_accelerate_cbs(struct rcu_node rnp, struct rcu_data rdp)
1766	1486	{
1767	1487	unsigned long gp_seq_req;
1768	1488	bool ret = false;
1769	1489
	1490	+ rcu_lockdep_assert_cblist_protected(rdp);
1770	1491	raw_lockdep_assert_held_rcu_node(rnp);
1771	1492
1772	1493	/* If no pending (not yet ready to invoke) callbacks, nothing to do. */
..	..	@@ -1783,15 +1504,16 @@
1783	1504	* accelerating callback invocation to an earlier grace-period
1784	1505	* number.
1785	1506	*/
1786		- gp_seq_req = rcu_seq_snap(&rsp->gp_seq);
	1507	+ gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);
1787	1508	if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req))
1788	1509	ret = rcu_start_this_gp(rnp, rdp, gp_seq_req);
1789	1510
1790	1511	/* Trace depending on how much we were able to accelerate. */
1791	1512	if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL))
1792		- trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccWaitCB"));
	1513	+ trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB"));
1793	1514	else
1794		- trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccReadyCB"));
	1515	+ trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB"));
	1516	+
1795	1517	return ret;
1796	1518	}
1797	1519
..	..	@@ -1802,25 +1524,24 @@
1802	1524	* that a new grace-period request be made, invokes rcu_accelerate_cbs()
1803	1525	* while holding the leaf rcu_node structure's ->lock.
1804	1526	*/
1805		-static void rcu_accelerate_cbs_unlocked(struct rcu_state *rsp,
1806		- struct rcu_node *rnp,
	1527	+static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp,
1807	1528	struct rcu_data *rdp)
1808	1529	{
1809	1530	unsigned long c;
1810	1531	bool needwake;
1811	1532
1812		- lockdep_assert_irqs_disabled();
1813		- c = rcu_seq_snap(&rsp->gp_seq);
1814		- if (!rdp->gpwrap && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
	1533	+ rcu_lockdep_assert_cblist_protected(rdp);
	1534	+ c = rcu_seq_snap(&rcu_state.gp_seq);
	1535	+ if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
1815	1536	/* Old request still live, so mark recent callbacks. */
1816	1537	(void)rcu_segcblist_accelerate(&rdp->cblist, c);
1817	1538	return;
1818	1539	}
1819	1540	raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
1820		- needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
	1541	+ needwake = rcu_accelerate_cbs(rnp, rdp);
1821	1542	raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
1822	1543	if (needwake)
1823		- rcu_gp_kthread_wake(rsp);
	1544	+ rcu_gp_kthread_wake();
1824	1545	}
1825	1546
1826	1547	/*
..	..	@@ -1833,9 +1554,9 @@
1833	1554	*
1834	1555	* The caller must hold rnp->lock with interrupts disabled.
1835	1556	*/
1836		-static bool rcu_advance_cbs(struct rcu_state rsp, struct rcu_node rnp,
1837		- struct rcu_data *rdp)
	1557	+static bool rcu_advance_cbs(struct rcu_node rnp, struct rcu_data rdp)
1838	1558	{
	1559	+ rcu_lockdep_assert_cblist_protected(rdp);
1839	1560	raw_lockdep_assert_held_rcu_node(rnp);
1840	1561
1841	1562	/* If no pending (not yet ready to invoke) callbacks, nothing to do. */
..	..	@@ -1849,7 +1570,36 @@
1849	1570	rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq);
1850	1571
1851	1572	/* Classify any remaining callbacks. */
1852		- return rcu_accelerate_cbs(rsp, rnp, rdp);
	1573	+ return rcu_accelerate_cbs(rnp, rdp);
	1574	+}
	1575	+
	1576	+/*
	1577	+ * Move and classify callbacks, but only if doing so won't require
	1578	+ * that the RCU grace-period kthread be awakened.
	1579	+ */
	1580	+static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp,
	1581	+ struct rcu_data *rdp)
	1582	+{
	1583	+ rcu_lockdep_assert_cblist_protected(rdp);
	1584	+ if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) \|\| !raw_spin_trylock_rcu_node(rnp))
	1585	+ return;
	1586	+ // The grace period cannot end while we hold the rcu_node lock.
	1587	+ if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))
	1588	+ WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp));
	1589	+ raw_spin_unlock_rcu_node(rnp);
	1590	+}
	1591	+
	1592	+/*
	1593	+ * In CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels, attempt to generate a
	1594	+ * quiescent state. This is intended to be invoked when the CPU notices
	1595	+ * a new grace period.
	1596	+ */
	1597	+static void rcu_strict_gp_check_qs(void)
	1598	+{
	1599	+ if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD)) {
	1600	+ rcu_read_lock();
	1601	+ rcu_read_unlock();
	1602	+ }
1853	1603	}
1854	1604
1855	1605	/*
..	..	@@ -1858,11 +1608,12 @@
1858	1608	* structure corresponding to the current CPU, and must have irqs disabled.
1859	1609	* Returns true if the grace-period kthread needs to be awakened.
1860	1610	*/
1861		-static bool __note_gp_changes(struct rcu_state rsp, struct rcu_node rnp,
1862		- struct rcu_data *rdp)
	1611	+static bool __note_gp_changes(struct rcu_node rnp, struct rcu_data rdp)
1863	1612	{
1864		- bool ret;
1865		- bool need_gp;
	1613	+ bool ret = false;
	1614	+ bool need_qs;
	1615	+ const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	1616	+ rcu_segcblist_is_offloaded(&rdp->cblist);
1866	1617
1867	1618	raw_lockdep_assert_held_rcu_node(rnp);
1868	1619
..	..	@@ -1872,10 +1623,15 @@
1872	1623	/* Handle the ends of any preceding grace periods first. */
1873	1624	if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) \|\|
1874	1625	unlikely(READ_ONCE(rdp->gpwrap))) {
1875		- ret = rcu_advance_cbs(rsp, rnp, rdp); /* Advance callbacks. */
1876		- trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuend"));
	1626	+ if (!offloaded)
	1627	+ ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */
	1628	+ rdp->core_needs_qs = false;
	1629	+ trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));
1877	1630	} else {
1878		- ret = rcu_accelerate_cbs(rsp, rnp, rdp); /* Recent callbacks. */
	1631	+ if (!offloaded)
	1632	+ ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */
	1633	+ if (rdp->core_needs_qs)
	1634	+ rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask);
1879	1635	}
1880	1636
1881	1637	/* Now handle the beginnings of any new-to-this-CPU grace periods. */
..	..	@@ -1886,22 +1642,21 @@
1886	1642	* set up to detect a quiescent state, otherwise don't
1887	1643	* go looking for one.
1888	1644	*/
1889		- trace_rcu_grace_period(rsp->name, rnp->gp_seq, TPS("cpustart"));
1890		- need_gp = !!(rnp->qsmask & rdp->grpmask);
1891		- rdp->cpu_no_qs.b.norm = need_gp;
1892		- rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
1893		- rdp->core_needs_qs = need_gp;
	1645	+ trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));
	1646	+ need_qs = !!(rnp->qsmask & rdp->grpmask);
	1647	+ rdp->cpu_no_qs.b.norm = need_qs;
	1648	+ rdp->core_needs_qs = need_qs;
1894	1649	zero_cpu_stall_ticks(rdp);
1895	1650	}
1896	1651	rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */
1897		- if (ULONG_CMP_GE(rnp->gp_seq_needed, rdp->gp_seq_needed) \|\| rdp->gpwrap)
1898		- rdp->gp_seq_needed = rnp->gp_seq_needed;
	1652	+ if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) \|\| rdp->gpwrap)
	1653	+ WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
1899	1654	WRITE_ONCE(rdp->gpwrap, false);
1900	1655	rcu_gpnum_ovf(rnp, rdp);
1901	1656	return ret;
1902	1657	}
1903	1658
1904		-static void note_gp_changes(struct rcu_state rsp, struct rcu_data rdp)
	1659	+static void note_gp_changes(struct rcu_data *rdp)
1905	1660	{
1906	1661	unsigned long flags;
1907	1662	bool needwake;
..	..	@@ -1915,41 +1670,76 @@
1915	1670	local_irq_restore(flags);
1916	1671	return;
1917	1672	}
1918		- needwake = __note_gp_changes(rsp, rnp, rdp);
	1673	+ needwake = __note_gp_changes(rnp, rdp);
1919	1674	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
	1675	+ rcu_strict_gp_check_qs();
1920	1676	if (needwake)
1921		- rcu_gp_kthread_wake(rsp);
	1677	+ rcu_gp_kthread_wake();
1922	1678	}
1923	1679
1924		-static void rcu_gp_slow(struct rcu_state *rsp, int delay)
	1680	+static void rcu_gp_slow(int delay)
1925	1681	{
1926	1682	if (delay > 0 &&
1927		- !(rcu_seq_ctr(rsp->gp_seq) %
	1683	+ !(rcu_seq_ctr(rcu_state.gp_seq) %
1928	1684	(rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))
1929		- schedule_timeout_uninterruptible(delay);
	1685	+ schedule_timeout_idle(delay);
	1686	+}
	1687	+
	1688	+static unsigned long sleep_duration;
	1689	+
	1690	+/* Allow rcutorture to stall the grace-period kthread. */
	1691	+void rcu_gp_set_torture_wait(int duration)
	1692	+{
	1693	+ if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST) && duration > 0)
	1694	+ WRITE_ONCE(sleep_duration, duration);
	1695	+}
	1696	+EXPORT_SYMBOL_GPL(rcu_gp_set_torture_wait);
	1697	+
	1698	+/* Actually implement the aforementioned wait. */
	1699	+static void rcu_gp_torture_wait(void)
	1700	+{
	1701	+ unsigned long duration;
	1702	+
	1703	+ if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST))
	1704	+ return;
	1705	+ duration = xchg(&sleep_duration, 0UL);
	1706	+ if (duration > 0) {
	1707	+ pr_alert("%s: Waiting %lu jiffies\n", __func__, duration);
	1708	+ schedule_timeout_idle(duration);
	1709	+ pr_alert("%s: Wait complete\n", __func__);
	1710	+ }
	1711	+}
	1712	+
	1713	+/*
	1714	+ * Handler for on_each_cpu() to invoke the target CPU's RCU core
	1715	+ * processing.
	1716	+ */
	1717	+static void rcu_strict_gp_boundary(void *unused)
	1718	+{
	1719	+ invoke_rcu_core();
1930	1720	}
1931	1721
1932	1722	/*
1933	1723	* Initialize a new grace period. Return false if no grace period required.
1934	1724	*/
1935		-static bool rcu_gp_init(struct rcu_state *rsp)
	1725	+static bool rcu_gp_init(void)
1936	1726	{
1937	1727	unsigned long flags;
1938	1728	unsigned long oldmask;
1939	1729	unsigned long mask;
1940	1730	struct rcu_data *rdp;
1941		- struct rcu_node *rnp = rcu_get_root(rsp);
	1731	+ struct rcu_node *rnp = rcu_get_root();
1942	1732
1943		- WRITE_ONCE(rsp->gp_activity, jiffies);
	1733	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
1944	1734	raw_spin_lock_irq_rcu_node(rnp);
1945		- if (!READ_ONCE(rsp->gp_flags)) {
	1735	+ if (!READ_ONCE(rcu_state.gp_flags)) {
1946	1736	/* Spurious wakeup, tell caller to go back to sleep. */
1947	1737	raw_spin_unlock_irq_rcu_node(rnp);
1948	1738	return false;
1949	1739	}
1950		- WRITE_ONCE(rsp->gp_flags, 0); /* Clear all flags: New grace period. */
	1740	+ WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */
1951	1741
1952		- if (WARN_ON_ONCE(rcu_gp_in_progress(rsp))) {
	1742	+ if (WARN_ON_ONCE(rcu_gp_in_progress())) {
1953	1743	/*
1954	1744	* Grace period already in progress, don't start another.
1955	1745	* Not supposed to be able to happen.
..	..	@@ -1959,27 +1749,31 @@
1959	1749	}
1960	1750
1961	1751	/* Advance to a new grace period and initialize state. */
1962		- record_gp_stall_check_time(rsp);
	1752	+ record_gp_stall_check_time();
1963	1753	/* Record GP times before starting GP, hence rcu_seq_start(). */
1964		- rcu_seq_start(&rsp->gp_seq);
1965		- trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("start"));
	1754	+ rcu_seq_start(&rcu_state.gp_seq);
	1755	+ ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
	1756	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));
1966	1757	raw_spin_unlock_irq_rcu_node(rnp);
1967	1758
1968	1759	/*
1969		- * Apply per-leaf buffered online and offline operations to the
1970		- * rcu_node tree. Note that this new grace period need not wait
1971		- * for subsequent online CPUs, and that quiescent-state forcing
1972		- * will handle subsequent offline CPUs.
	1760	+ * Apply per-leaf buffered online and offline operations to
	1761	+ * the rcu_node tree. Note that this new grace period need not
	1762	+ * wait for subsequent online CPUs, and that RCU hooks in the CPU
	1763	+ * offlining path, when combined with checks in this function,
	1764	+ * will handle CPUs that are currently going offline or that will
	1765	+ * go offline later. Please also refer to "Hotplug CPU" section
	1766	+ * of RCU's Requirements documentation.
1973	1767	*/
1974		- rsp->gp_state = RCU_GP_ONOFF;
1975		- rcu_for_each_leaf_node(rsp, rnp) {
1976		- spin_lock(&rsp->ofl_lock);
	1768	+ rcu_state.gp_state = RCU_GP_ONOFF;
	1769	+ rcu_for_each_leaf_node(rnp) {
	1770	+ raw_spin_lock(&rcu_state.ofl_lock);
1977	1771	raw_spin_lock_irq_rcu_node(rnp);
1978	1772	if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
1979	1773	!rnp->wait_blkd_tasks) {
1980	1774	/* Nothing to do on this leaf rcu_node structure. */
1981	1775	raw_spin_unlock_irq_rcu_node(rnp);
1982		- spin_unlock(&rsp->ofl_lock);
	1776	+ raw_spin_unlock(&rcu_state.ofl_lock);
1983	1777	continue;
1984	1778	}
1985	1779
..	..	@@ -2015,46 +1809,50 @@
2015	1809	}
2016	1810
2017	1811	raw_spin_unlock_irq_rcu_node(rnp);
2018		- spin_unlock(&rsp->ofl_lock);
	1812	+ raw_spin_unlock(&rcu_state.ofl_lock);
2019	1813	}
2020		- rcu_gp_slow(rsp, gp_preinit_delay); /* Races with CPU hotplug. */
	1814	+ rcu_gp_slow(gp_preinit_delay); /* Races with CPU hotplug. */
2021	1815
2022	1816	/*
2023	1817	* Set the quiescent-state-needed bits in all the rcu_node
2024		- * structures for all currently online CPUs in breadth-first order,
2025		- * starting from the root rcu_node structure, relying on the layout
2026		- * of the tree within the rsp->node[] array. Note that other CPUs
2027		- * will access only the leaves of the hierarchy, thus seeing that no
2028		- * grace period is in progress, at least until the corresponding
2029		- * leaf node has been initialized.
	1818	+ * structures for all currently online CPUs in breadth-first
	1819	+ * order, starting from the root rcu_node structure, relying on the
	1820	+ * layout of the tree within the rcu_state.node[] array. Note that
	1821	+ * other CPUs will access only the leaves of the hierarchy, thus
	1822	+ * seeing that no grace period is in progress, at least until the
	1823	+ * corresponding leaf node has been initialized.
2030	1824	*
2031	1825	* The grace period cannot complete until the initialization
2032	1826	* process finishes, because this kthread handles both.
2033	1827	*/
2034		- rsp->gp_state = RCU_GP_INIT;
2035		- rcu_for_each_node_breadth_first(rsp, rnp) {
2036		- rcu_gp_slow(rsp, gp_init_delay);
	1828	+ rcu_state.gp_state = RCU_GP_INIT;
	1829	+ rcu_for_each_node_breadth_first(rnp) {
	1830	+ rcu_gp_slow(gp_init_delay);
2037	1831	raw_spin_lock_irqsave_rcu_node(rnp, flags);
2038		- rdp = this_cpu_ptr(rsp->rda);
2039		- rcu_preempt_check_blocked_tasks(rsp, rnp);
	1832	+ rdp = this_cpu_ptr(&rcu_data);
	1833	+ rcu_preempt_check_blocked_tasks(rnp);
2040	1834	rnp->qsmask = rnp->qsmaskinit;
2041		- WRITE_ONCE(rnp->gp_seq, rsp->gp_seq);
	1835	+ WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);
2042	1836	if (rnp == rdp->mynode)
2043		- (void)__note_gp_changes(rsp, rnp, rdp);
	1837	+ (void)__note_gp_changes(rnp, rdp);
2044	1838	rcu_preempt_boost_start_gp(rnp);
2045		- trace_rcu_grace_period_init(rsp->name, rnp->gp_seq,
	1839	+ trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,
2046	1840	rnp->level, rnp->grplo,
2047	1841	rnp->grphi, rnp->qsmask);
2048	1842	/* Quiescent states for tasks on any now-offline CPUs. */
2049	1843	mask = rnp->qsmask & ~rnp->qsmaskinitnext;
2050	1844	rnp->rcu_gp_init_mask = mask;
2051	1845	if ((mask \|\| rnp->wait_blkd_tasks) && rcu_is_leaf_node(rnp))
2052		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
	1846	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2053	1847	else
2054	1848	raw_spin_unlock_irq_rcu_node(rnp);
2055	1849	cond_resched_tasks_rcu_qs();
2056		- WRITE_ONCE(rsp->gp_activity, jiffies);
	1850	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
2057	1851	}
	1852	+
	1853	+ // If strict, make all CPUs aware of new grace period.
	1854	+ if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
	1855	+ on_each_cpu(rcu_strict_gp_boundary, NULL, 0);
2058	1856
2059	1857	return true;
2060	1858	}
..	..	@@ -2063,16 +1861,20 @@
2063	1861	* Helper function for swait_event_idle_exclusive() wakeup at force-quiescent-state
2064	1862	* time.
2065	1863	*/
2066		-static bool rcu_gp_fqs_check_wake(struct rcu_state rsp, int gfp)
	1864	+static bool rcu_gp_fqs_check_wake(int *gfp)
2067	1865	{
2068		- struct rcu_node *rnp = rcu_get_root(rsp);
	1866	+ struct rcu_node *rnp = rcu_get_root();
2069	1867
2070		- /* Someone like call_rcu() requested a force-quiescent-state scan. */
2071		- *gfp = READ_ONCE(rsp->gp_flags);
	1868	+ // If under overload conditions, force an immediate FQS scan.
	1869	+ if (*gfp & RCU_GP_FLAG_OVLD)
	1870	+ return true;
	1871	+
	1872	+ // Someone like call_rcu() requested a force-quiescent-state scan.
	1873	+ *gfp = READ_ONCE(rcu_state.gp_flags);
2072	1874	if (*gfp & RCU_GP_FLAG_FQS)
2073	1875	return true;
2074	1876
2075		- /* The current grace period has completed. */
	1877	+ // The current grace period has completed.
2076	1878	if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp))
2077	1879	return true;
2078	1880
..	..	@@ -2082,45 +1884,117 @@
2082	1884	/*
2083	1885	* Do one round of quiescent-state forcing.
2084	1886	*/
2085		-static void rcu_gp_fqs(struct rcu_state *rsp, bool first_time)
	1887	+static void rcu_gp_fqs(bool first_time)
2086	1888	{
2087		- struct rcu_node *rnp = rcu_get_root(rsp);
	1889	+ struct rcu_node *rnp = rcu_get_root();
2088	1890
2089		- WRITE_ONCE(rsp->gp_activity, jiffies);
2090		- rsp->n_force_qs++;
	1891	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	1892	+ WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1);
2091	1893	if (first_time) {
2092	1894	/* Collect dyntick-idle snapshots. */
2093		- force_qs_rnp(rsp, dyntick_save_progress_counter);
	1895	+ force_qs_rnp(dyntick_save_progress_counter);
2094	1896	} else {
2095	1897	/* Handle dyntick-idle and offline CPUs. */
2096		- force_qs_rnp(rsp, rcu_implicit_dynticks_qs);
	1898	+ force_qs_rnp(rcu_implicit_dynticks_qs);
2097	1899	}
2098	1900	/* Clear flag to prevent immediate re-entry. */
2099		- if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
	1901	+ if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
2100	1902	raw_spin_lock_irq_rcu_node(rnp);
2101		- WRITE_ONCE(rsp->gp_flags,
2102		- READ_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS);
	1903	+ WRITE_ONCE(rcu_state.gp_flags,
	1904	+ READ_ONCE(rcu_state.gp_flags) & ~RCU_GP_FLAG_FQS);
2103	1905	raw_spin_unlock_irq_rcu_node(rnp);
	1906	+ }
	1907	+}
	1908	+
	1909	+/*
	1910	+ * Loop doing repeated quiescent-state forcing until the grace period ends.
	1911	+ */
	1912	+static void rcu_gp_fqs_loop(void)
	1913	+{
	1914	+ bool first_gp_fqs;
	1915	+ int gf = 0;
	1916	+ unsigned long j;
	1917	+ int ret;
	1918	+ struct rcu_node *rnp = rcu_get_root();
	1919	+
	1920	+ first_gp_fqs = true;
	1921	+ j = READ_ONCE(jiffies_till_first_fqs);
	1922	+ if (rcu_state.cbovld)
	1923	+ gf = RCU_GP_FLAG_OVLD;
	1924	+ ret = 0;
	1925	+ for (;;) {
	1926	+ if (!ret) {
	1927	+ rcu_state.jiffies_force_qs = jiffies + j;
	1928	+ WRITE_ONCE(rcu_state.jiffies_kick_kthreads,
	1929	+ jiffies + (j ? 3 * j : 2));
	1930	+ }
	1931	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
	1932	+ TPS("fqswait"));
	1933	+ rcu_state.gp_state = RCU_GP_WAIT_FQS;
	1934	+ ret = swait_event_idle_timeout_exclusive(
	1935	+ rcu_state.gp_wq, rcu_gp_fqs_check_wake(&gf), j);
	1936	+ rcu_gp_torture_wait();
	1937	+ rcu_state.gp_state = RCU_GP_DOING_FQS;
	1938	+ /* Locking provides needed memory barriers. */
	1939	+ /* If grace period done, leave loop. */
	1940	+ if (!READ_ONCE(rnp->qsmask) &&
	1941	+ !rcu_preempt_blocked_readers_cgp(rnp))
	1942	+ break;
	1943	+ /* If time for quiescent-state forcing, do it. */
	1944	+ if (!time_after(rcu_state.jiffies_force_qs, jiffies) \|\|
	1945	+ (gf & (RCU_GP_FLAG_FQS \| RCU_GP_FLAG_OVLD))) {
	1946	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
	1947	+ TPS("fqsstart"));
	1948	+ rcu_gp_fqs(first_gp_fqs);
	1949	+ gf = 0;
	1950	+ if (first_gp_fqs) {
	1951	+ first_gp_fqs = false;
	1952	+ gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0;
	1953	+ }
	1954	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
	1955	+ TPS("fqsend"));
	1956	+ cond_resched_tasks_rcu_qs();
	1957	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	1958	+ ret = 0; /* Force full wait till next FQS. */
	1959	+ j = READ_ONCE(jiffies_till_next_fqs);
	1960	+ } else {
	1961	+ /* Deal with stray signal. */
	1962	+ cond_resched_tasks_rcu_qs();
	1963	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	1964	+ WARN_ON(signal_pending(current));
	1965	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
	1966	+ TPS("fqswaitsig"));
	1967	+ ret = 1; /* Keep old FQS timing. */
	1968	+ j = jiffies;
	1969	+ if (time_after(jiffies, rcu_state.jiffies_force_qs))
	1970	+ j = 1;
	1971	+ else
	1972	+ j = rcu_state.jiffies_force_qs - j;
	1973	+ gf = 0;
	1974	+ }
2104	1975	}
2105	1976	}
2106	1977
2107	1978	/*
2108	1979	* Clean up after the old grace period.
2109	1980	*/
2110		-static void rcu_gp_cleanup(struct rcu_state *rsp)
	1981	+static void rcu_gp_cleanup(void)
2111	1982	{
2112		- unsigned long gp_duration;
	1983	+ int cpu;
2113	1984	bool needgp = false;
	1985	+ unsigned long gp_duration;
2114	1986	unsigned long new_gp_seq;
	1987	+ bool offloaded;
2115	1988	struct rcu_data *rdp;
2116		- struct rcu_node *rnp = rcu_get_root(rsp);
	1989	+ struct rcu_node *rnp = rcu_get_root();
2117	1990	struct swait_queue_head *sq;
2118	1991
2119		- WRITE_ONCE(rsp->gp_activity, jiffies);
	1992	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
2120	1993	raw_spin_lock_irq_rcu_node(rnp);
2121		- gp_duration = jiffies - rsp->gp_start;
2122		- if (gp_duration > rsp->gp_max)
2123		- rsp->gp_max = gp_duration;
	1994	+ rcu_state.gp_end = jiffies;
	1995	+ gp_duration = rcu_state.gp_end - rcu_state.gp_start;
	1996	+ if (gp_duration > rcu_state.gp_max)
	1997	+ rcu_state.gp_max = gp_duration;
2124	1998
2125	1999	/*
2126	2000	* We know the grace period is complete, but to everyone else
..	..	@@ -2141,165 +2015,123 @@
2141	2015	* the rcu_node structures before the beginning of the next grace
2142	2016	* period is recorded in any of the rcu_node structures.
2143	2017	*/
2144		- new_gp_seq = rsp->gp_seq;
	2018	+ new_gp_seq = rcu_state.gp_seq;
2145	2019	rcu_seq_end(&new_gp_seq);
2146		- rcu_for_each_node_breadth_first(rsp, rnp) {
	2020	+ rcu_for_each_node_breadth_first(rnp) {
2147	2021	raw_spin_lock_irq_rcu_node(rnp);
2148	2022	if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))
2149		- dump_blkd_tasks(rsp, rnp, 10);
	2023	+ dump_blkd_tasks(rnp, 10);
2150	2024	WARN_ON_ONCE(rnp->qsmask);
2151	2025	WRITE_ONCE(rnp->gp_seq, new_gp_seq);
2152		- rdp = this_cpu_ptr(rsp->rda);
	2026	+ rdp = this_cpu_ptr(&rcu_data);
2153	2027	if (rnp == rdp->mynode)
2154		- needgp = __note_gp_changes(rsp, rnp, rdp) \|\| needgp;
	2028	+ needgp = __note_gp_changes(rnp, rdp) \|\| needgp;
2155	2029	/* smp_mb() provided by prior unlock-lock pair. */
2156		- needgp = rcu_future_gp_cleanup(rsp, rnp) \|\| needgp;
	2030	+ needgp = rcu_future_gp_cleanup(rnp) \|\| needgp;
	2031	+ // Reset overload indication for CPUs no longer overloaded
	2032	+ if (rcu_is_leaf_node(rnp))
	2033	+ for_each_leaf_node_cpu_mask(rnp, cpu, rnp->cbovldmask) {
	2034	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	2035	+ check_cb_ovld_locked(rdp, rnp);
	2036	+ }
2157	2037	sq = rcu_nocb_gp_get(rnp);
2158	2038	raw_spin_unlock_irq_rcu_node(rnp);
2159	2039	rcu_nocb_gp_cleanup(sq);
2160	2040	cond_resched_tasks_rcu_qs();
2161		- WRITE_ONCE(rsp->gp_activity, jiffies);
2162		- rcu_gp_slow(rsp, gp_cleanup_delay);
	2041	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	2042	+ rcu_gp_slow(gp_cleanup_delay);
2163	2043	}
2164		- rnp = rcu_get_root(rsp);
2165		- raw_spin_lock_irq_rcu_node(rnp); /* GP before rsp->gp_seq update. */
	2044	+ rnp = rcu_get_root();
	2045	+ raw_spin_lock_irq_rcu_node(rnp); /* GP before ->gp_seq update. */
2166	2046
2167		- /* Declare grace period done. */
2168		- rcu_seq_end(&rsp->gp_seq);
2169		- trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("end"));
2170		- rsp->gp_state = RCU_GP_IDLE;
	2047	+ /* Declare grace period done, trace first to use old GP number. */
	2048	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));
	2049	+ rcu_seq_end(&rcu_state.gp_seq);
	2050	+ ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
	2051	+ rcu_state.gp_state = RCU_GP_IDLE;
2171	2052	/* Check for GP requests since above loop. */
2172		- rdp = this_cpu_ptr(rsp->rda);
	2053	+ rdp = this_cpu_ptr(&rcu_data);
2173	2054	if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) {
2174	2055	trace_rcu_this_gp(rnp, rdp, rnp->gp_seq_needed,
2175	2056	TPS("CleanupMore"));
2176	2057	needgp = true;
2177	2058	}
2178	2059	/* Advance CBs to reduce false positives below. */
2179		- if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) {
2180		- WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT);
2181		- rsp->gp_req_activity = jiffies;
2182		- trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq),
	2060	+ offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2061	+ rcu_segcblist_is_offloaded(&rdp->cblist);
	2062	+ if ((offloaded \|\| !rcu_accelerate_cbs(rnp, rdp)) && needgp) {
	2063	+ WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);
	2064	+ WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
	2065	+ trace_rcu_grace_period(rcu_state.name,
	2066	+ rcu_state.gp_seq,
2183	2067	TPS("newreq"));
2184	2068	} else {
2185		- WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT);
	2069	+ WRITE_ONCE(rcu_state.gp_flags,
	2070	+ rcu_state.gp_flags & RCU_GP_FLAG_INIT);
2186	2071	}
2187	2072	raw_spin_unlock_irq_rcu_node(rnp);
	2073	+
	2074	+ // If strict, make all CPUs aware of the end of the old grace period.
	2075	+ if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
	2076	+ on_each_cpu(rcu_strict_gp_boundary, NULL, 0);
2188	2077	}
2189	2078
2190	2079	/*
2191	2080	* Body of kthread that handles grace periods.
2192	2081	*/
2193		-static int __noreturn rcu_gp_kthread(void *arg)
	2082	+static int __noreturn rcu_gp_kthread(void *unused)
2194	2083	{
2195		- bool first_gp_fqs;
2196		- int gf;
2197		- unsigned long j;
2198		- int ret;
2199		- struct rcu_state *rsp = arg;
2200		- struct rcu_node *rnp = rcu_get_root(rsp);
2201		-
2202	2084	rcu_bind_gp_kthread();
2203	2085	for (;;) {
2204	2086
2205	2087	/* Handle grace-period start. */
2206	2088	for (;;) {
2207		- trace_rcu_grace_period(rsp->name,
2208		- READ_ONCE(rsp->gp_seq),
	2089	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
2209	2090	TPS("reqwait"));
2210		- rsp->gp_state = RCU_GP_WAIT_GPS;
2211		- swait_event_idle_exclusive(rsp->gp_wq, READ_ONCE(rsp->gp_flags) &
2212		- RCU_GP_FLAG_INIT);
2213		- rsp->gp_state = RCU_GP_DONE_GPS;
	2091	+ rcu_state.gp_state = RCU_GP_WAIT_GPS;
	2092	+ swait_event_idle_exclusive(rcu_state.gp_wq,
	2093	+ READ_ONCE(rcu_state.gp_flags) &
	2094	+ RCU_GP_FLAG_INIT);
	2095	+ rcu_gp_torture_wait();
	2096	+ rcu_state.gp_state = RCU_GP_DONE_GPS;
2214	2097	/* Locking provides needed memory barrier. */
2215		- if (rcu_gp_init(rsp))
	2098	+ if (rcu_gp_init())
2216	2099	break;
2217	2100	cond_resched_tasks_rcu_qs();
2218		- WRITE_ONCE(rsp->gp_activity, jiffies);
	2101	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
2219	2102	WARN_ON(signal_pending(current));
2220		- trace_rcu_grace_period(rsp->name,
2221		- READ_ONCE(rsp->gp_seq),
	2103	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
2222	2104	TPS("reqwaitsig"));
2223	2105	}
2224	2106
2225	2107	/* Handle quiescent-state forcing. */
2226		- first_gp_fqs = true;
2227		- j = jiffies_till_first_fqs;
2228		- ret = 0;
2229		- for (;;) {
2230		- if (!ret) {
2231		- rsp->jiffies_force_qs = jiffies + j;
2232		- WRITE_ONCE(rsp->jiffies_kick_kthreads,
2233		- jiffies + 3 * j);
2234		- }
2235		- trace_rcu_grace_period(rsp->name,
2236		- READ_ONCE(rsp->gp_seq),
2237		- TPS("fqswait"));
2238		- rsp->gp_state = RCU_GP_WAIT_FQS;
2239		- ret = swait_event_idle_timeout_exclusive(rsp->gp_wq,
2240		- rcu_gp_fqs_check_wake(rsp, &gf), j);
2241		- rsp->gp_state = RCU_GP_DOING_FQS;
2242		- /* Locking provides needed memory barriers. */
2243		- /* If grace period done, leave loop. */
2244		- if (!READ_ONCE(rnp->qsmask) &&
2245		- !rcu_preempt_blocked_readers_cgp(rnp))
2246		- break;
2247		- /* If time for quiescent-state forcing, do it. */
2248		- if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) \|\|
2249		- (gf & RCU_GP_FLAG_FQS)) {
2250		- trace_rcu_grace_period(rsp->name,
2251		- READ_ONCE(rsp->gp_seq),
2252		- TPS("fqsstart"));
2253		- rcu_gp_fqs(rsp, first_gp_fqs);
2254		- first_gp_fqs = false;
2255		- trace_rcu_grace_period(rsp->name,
2256		- READ_ONCE(rsp->gp_seq),
2257		- TPS("fqsend"));
2258		- cond_resched_tasks_rcu_qs();
2259		- WRITE_ONCE(rsp->gp_activity, jiffies);
2260		- ret = 0; /* Force full wait till next FQS. */
2261		- j = jiffies_till_next_fqs;
2262		- } else {
2263		- /* Deal with stray signal. */
2264		- cond_resched_tasks_rcu_qs();
2265		- WRITE_ONCE(rsp->gp_activity, jiffies);
2266		- WARN_ON(signal_pending(current));
2267		- trace_rcu_grace_period(rsp->name,
2268		- READ_ONCE(rsp->gp_seq),
2269		- TPS("fqswaitsig"));
2270		- ret = 1; /* Keep old FQS timing. */
2271		- j = jiffies;
2272		- if (time_after(jiffies, rsp->jiffies_force_qs))
2273		- j = 1;
2274		- else
2275		- j = rsp->jiffies_force_qs - j;
2276		- }
2277		- }
	2108	+ rcu_gp_fqs_loop();
2278	2109
2279	2110	/* Handle grace-period end. */
2280		- rsp->gp_state = RCU_GP_CLEANUP;
2281		- rcu_gp_cleanup(rsp);
2282		- rsp->gp_state = RCU_GP_CLEANED;
	2111	+ rcu_state.gp_state = RCU_GP_CLEANUP;
	2112	+ rcu_gp_cleanup();
	2113	+ rcu_state.gp_state = RCU_GP_CLEANED;
2283	2114	}
2284	2115	}
2285	2116
2286	2117	/*
2287		- * Report a full set of quiescent states to the specified rcu_state data
2288		- * structure. Invoke rcu_gp_kthread_wake() to awaken the grace-period
2289		- * kthread if another grace period is required. Whether we wake
2290		- * the grace-period kthread or it awakens itself for the next round
2291		- * of quiescent-state forcing, that kthread will clean up after the
2292		- * just-completed grace period. Note that the caller must hold rnp->lock,
2293		- * which is released before return.
	2118	+ * Report a full set of quiescent states to the rcu_state data structure.
	2119	+ * Invoke rcu_gp_kthread_wake() to awaken the grace-period kthread if
	2120	+ * another grace period is required. Whether we wake the grace-period
	2121	+ * kthread or it awakens itself for the next round of quiescent-state
	2122	+ * forcing, that kthread will clean up after the just-completed grace
	2123	+ * period. Note that the caller must hold rnp->lock, which is released
	2124	+ * before return.
2294	2125	*/
2295		-static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
2296		- __releases(rcu_get_root(rsp)->lock)
	2126	+static void rcu_report_qs_rsp(unsigned long flags)
	2127	+ __releases(rcu_get_root()->lock)
2297	2128	{
2298		- raw_lockdep_assert_held_rcu_node(rcu_get_root(rsp));
2299		- WARN_ON_ONCE(!rcu_gp_in_progress(rsp));
2300		- WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) \| RCU_GP_FLAG_FQS);
2301		- raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(rsp), flags);
2302		- rcu_gp_kthread_wake(rsp);
	2129	+ raw_lockdep_assert_held_rcu_node(rcu_get_root());
	2130	+ WARN_ON_ONCE(!rcu_gp_in_progress());
	2131	+ WRITE_ONCE(rcu_state.gp_flags,
	2132	+ READ_ONCE(rcu_state.gp_flags) \| RCU_GP_FLAG_FQS);
	2133	+ raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(), flags);
	2134	+ rcu_gp_kthread_wake();
2303	2135	}
2304	2136
2305	2137	/*
..	..	@@ -2316,9 +2148,8 @@
2316	2148	* disabled. This allows propagating quiescent state due to resumed tasks
2317	2149	* during grace-period initialization.
2318	2150	*/
2319		-static void
2320		-rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
2321		- struct rcu_node *rnp, unsigned long gps, unsigned long flags)
	2151	+static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
	2152	+ unsigned long gps, unsigned long flags)
2322	2153	__releases(rnp->lock)
2323	2154	{
2324	2155	unsigned long oldmask = 0;
..	..	@@ -2340,8 +2171,8 @@
2340	2171	WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */
2341	2172	WARN_ON_ONCE(!rcu_is_leaf_node(rnp) &&
2342	2173	rcu_preempt_blocked_readers_cgp(rnp));
2343		- rnp->qsmask &= ~mask;
2344		- trace_rcu_quiescent_state_report(rsp->name, rnp->gp_seq,
	2174	+ WRITE_ONCE(rnp->qsmask, rnp->qsmask & ~mask);
	2175	+ trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,
2345	2176	mask, rnp->qsmask, rnp->level,
2346	2177	rnp->grplo, rnp->grphi,
2347	2178	!!rnp->gp_tasks);
..	..	@@ -2363,7 +2194,7 @@
2363	2194	rnp_c = rnp;
2364	2195	rnp = rnp->parent;
2365	2196	raw_spin_lock_irqsave_rcu_node(rnp, flags);
2366		- oldmask = rnp_c->qsmask;
	2197	+ oldmask = READ_ONCE(rnp_c->qsmask);
2367	2198	}
2368	2199
2369	2200	/*
..	..	@@ -2371,19 +2202,18 @@
2371	2202	* state for this grace period. Invoke rcu_report_qs_rsp()
2372	2203	* to clean up and start the next grace period if one is needed.
2373	2204	*/
2374		- rcu_report_qs_rsp(rsp, flags); /* releases rnp->lock. */
	2205	+ rcu_report_qs_rsp(flags); /* releases rnp->lock. */
2375	2206	}
2376	2207
2377	2208	/*
2378	2209	* Record a quiescent state for all tasks that were previously queued
2379	2210	* on the specified rcu_node structure and that were blocking the current
2380		- * RCU grace period. The caller must hold the specified rnp->lock with
	2211	+ * RCU grace period. The caller must hold the corresponding rnp->lock with
2381	2212	* irqs disabled, and this lock is released upon return, but irqs remain
2382	2213	* disabled.
2383	2214	*/
2384	2215	static void __maybe_unused
2385		-rcu_report_unblock_qs_rnp(struct rcu_state *rsp,
2386		- struct rcu_node *rnp, unsigned long flags)
	2216	+rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
2387	2217	__releases(rnp->lock)
2388	2218	{
2389	2219	unsigned long gps;
..	..	@@ -2391,8 +2221,7 @@
2391	2221	struct rcu_node *rnp_p;
2392	2222
2393	2223	raw_lockdep_assert_held_rcu_node(rnp);
2394		- if (WARN_ON_ONCE(rcu_state_p == &rcu_sched_state) \|\|
2395		- WARN_ON_ONCE(rsp != rcu_state_p) \|\|
	2224	+ if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT_RCU)) \|\|
2396	2225	WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)) \|\|
2397	2226	rnp->qsmask != 0) {
2398	2227	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
..	..	@@ -2406,7 +2235,7 @@
2406	2235	* Only one rcu_node structure in the tree, so don't
2407	2236	* try to report up to its nonexistent parent!
2408	2237	*/
2409		- rcu_report_qs_rsp(rsp, flags);
	2238	+ rcu_report_qs_rsp(flags);
2410	2239	return;
2411	2240	}
2412	2241
..	..	@@ -2415,7 +2244,7 @@
2415	2244	mask = rnp->grpmask;
2416	2245	raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
2417	2246	raw_spin_lock_rcu_node(rnp_p); /* irqs already disabled. */
2418		- rcu_report_qs_rnp(mask, rsp, rnp_p, gps, flags);
	2247	+ rcu_report_qs_rnp(mask, rnp_p, gps, flags);
2419	2248	}
2420	2249
2421	2250	/*
..	..	@@ -2423,13 +2252,16 @@
2423	2252	* structure. This must be called from the specified CPU.
2424	2253	*/
2425	2254	static void
2426		-rcu_report_qs_rdp(int cpu, struct rcu_state rsp, struct rcu_data rdp)
	2255	+rcu_report_qs_rdp(struct rcu_data *rdp)
2427	2256	{
2428	2257	unsigned long flags;
2429	2258	unsigned long mask;
2430		- bool needwake;
	2259	+ bool needwake = false;
	2260	+ const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2261	+ rcu_segcblist_is_offloaded(&rdp->cblist);
2431	2262	struct rcu_node *rnp;
2432	2263
	2264	+ WARN_ON_ONCE(rdp->cpu != smp_processor_id());
2433	2265	rnp = rdp->mynode;
2434	2266	raw_spin_lock_irqsave_rcu_node(rnp, flags);
2435	2267	if (rdp->cpu_no_qs.b.norm \|\| rdp->gp_seq != rnp->gp_seq \|\|
..	..	@@ -2442,26 +2274,26 @@
2442	2274	* within the current grace period.
2443	2275	*/
2444	2276	rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */
2445		- rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
2446	2277	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2447	2278	return;
2448	2279	}
2449	2280	mask = rdp->grpmask;
	2281	+ rdp->core_needs_qs = false;
2450	2282	if ((rnp->qsmask & mask) == 0) {
2451	2283	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2452	2284	} else {
2453		- rdp->core_needs_qs = false;
2454		-
2455	2285	/*
2456	2286	* This GP can't end until cpu checks in, so all of our
2457	2287	* callbacks can be processed during the next GP.
2458	2288	*/
2459		- needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
	2289	+ if (!offloaded)
	2290	+ needwake = rcu_accelerate_cbs(rnp, rdp);
2460	2291
2461		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
	2292	+ rcu_disable_urgency_upon_qs(rdp);
	2293	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2462	2294	/* ^^^ Released rnp->lock */
2463	2295	if (needwake)
2464		- rcu_gp_kthread_wake(rsp);
	2296	+ rcu_gp_kthread_wake();
2465	2297	}
2466	2298	}
2467	2299
..	..	@@ -2472,10 +2304,10 @@
2472	2304	* quiescent state for this grace period, and record that fact if so.
2473	2305	*/
2474	2306	static void
2475		-rcu_check_quiescent_state(struct rcu_state rsp, struct rcu_data rdp)
	2307	+rcu_check_quiescent_state(struct rcu_data *rdp)
2476	2308	{
2477	2309	/* Check for grace-period ends and beginnings. */
2478		- note_gp_changes(rsp, rdp);
	2310	+ note_gp_changes(rdp);
2479	2311
2480	2312	/*
2481	2313	* Does this CPU still need to do its part for current grace period?
..	..	@@ -2495,24 +2327,26 @@
2495	2327	* Tell RCU we are done (but rcu_report_qs_rdp() will be the
2496	2328	* judge of that).
2497	2329	*/
2498		- rcu_report_qs_rdp(rdp->cpu, rsp, rdp);
	2330	+ rcu_report_qs_rdp(rdp);
2499	2331	}
2500	2332
2501	2333	/*
2502		- * Trace the fact that this CPU is going offline.
	2334	+ * Near the end of the offline process. Trace the fact that this CPU
	2335	+ * is going offline.
2503	2336	*/
2504		-static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
	2337	+int rcutree_dying_cpu(unsigned int cpu)
2505	2338	{
2506		- RCU_TRACE(bool blkd;)
2507		- RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda);)
2508		- RCU_TRACE(struct rcu_node *rnp = rdp->mynode;)
	2339	+ bool blkd;
	2340	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
	2341	+ struct rcu_node *rnp = rdp->mynode;
2509	2342
2510	2343	if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
2511		- return;
	2344	+ return 0;
2512	2345
2513		- RCU_TRACE(blkd = !!(rnp->qsmask & rdp->grpmask);)
2514		- trace_rcu_grace_period(rsp->name, rnp->gp_seq,
	2346	+ blkd = !!(rnp->qsmask & rdp->grpmask);
	2347	+ trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
2515	2348	blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));
	2349	+ return 0;
2516	2350	}
2517	2351
2518	2352	/*
..	..	@@ -2566,35 +2400,44 @@
2566	2400	* There can only be one CPU hotplug operation at a time, so no need for
2567	2401	* explicit locking.
2568	2402	*/
2569		-static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
	2403	+int rcutree_dead_cpu(unsigned int cpu)
2570	2404	{
2571		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
	2405	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
2572	2406	struct rcu_node rnp = rdp->mynode; / Outgoing CPU's rdp & rnp. */
2573	2407
2574	2408	if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
2575		- return;
	2409	+ return 0;
2576	2410
2577	2411	/* Adjust any no-longer-needed kthreads. */
2578	2412	rcu_boost_kthread_setaffinity(rnp, -1);
	2413	+ /* Do any needed no-CB deferred wakeups from this CPU. */
	2414	+ do_nocb_deferred_wakeup(per_cpu_ptr(&rcu_data, cpu));
	2415	+
	2416	+ // Stop-machine done, so allow nohz_full to disable tick.
	2417	+ tick_dep_clear(TICK_DEP_BIT_RCU);
	2418	+ return 0;
2579	2419	}
2580	2420
2581	2421	/*
2582	2422	* Invoke any RCU callbacks that have made it to the end of their grace
2583	2423	* period. Thottle as specified by rdp->blimit.
2584	2424	*/
2585		-static void rcu_do_batch(struct rcu_state rsp, struct rcu_data rdp)
	2425	+static void rcu_do_batch(struct rcu_data *rdp)
2586	2426	{
	2427	+ int div;
2587	2428	unsigned long flags;
	2429	+ const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2430	+ rcu_segcblist_is_offloaded(&rdp->cblist);
2588	2431	struct rcu_head *rhp;
2589	2432	struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
2590	2433	long bl, count;
	2434	+ long pending, tlimit = 0;
2591	2435
2592	2436	/* If no callbacks are ready, just return. */
2593	2437	if (!rcu_segcblist_ready_cbs(&rdp->cblist)) {
2594		- trace_rcu_batch_start(rsp->name,
2595		- rcu_segcblist_n_lazy_cbs(&rdp->cblist),
	2438	+ trace_rcu_batch_start(rcu_state.name,
2596	2439	rcu_segcblist_n_cbs(&rdp->cblist), 0);
2597		- trace_rcu_batch_end(rsp->name, 0,
	2440	+ trace_rcu_batch_end(rcu_state.name, 0,
2598	2441	!rcu_segcblist_empty(&rdp->cblist),
2599	2442	need_resched(), is_idle_task(current),
2600	2443	rcu_is_callbacks_kthread());
..	..	@@ -2607,32 +2450,76 @@
2607	2450	* callback counts, as rcu_barrier() needs to be conservative.
2608	2451	*/
2609	2452	local_irq_save(flags);
	2453	+ rcu_nocb_lock(rdp);
2610	2454	WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
2611		- bl = rdp->blimit;
2612		- trace_rcu_batch_start(rsp->name, rcu_segcblist_n_lazy_cbs(&rdp->cblist),
	2455	+ pending = rcu_segcblist_n_cbs(&rdp->cblist);
	2456	+ div = READ_ONCE(rcu_divisor);
	2457	+ div = div < 0 ? 7 : div > sizeof(long) * 8 - 2 ? sizeof(long) * 8 - 2 : div;
	2458	+ bl = max(rdp->blimit, pending >> div);
	2459	+ if (in_serving_softirq() && unlikely(bl > 100)) {
	2460	+ long rrn = READ_ONCE(rcu_resched_ns);
	2461	+
	2462	+ rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;
	2463	+ tlimit = local_clock() + rrn;
	2464	+ }
	2465	+ trace_rcu_batch_start(rcu_state.name,
2613	2466	rcu_segcblist_n_cbs(&rdp->cblist), bl);
2614	2467	rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl);
2615		- local_irq_restore(flags);
	2468	+ if (offloaded)
	2469	+ rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
	2470	+ rcu_nocb_unlock_irqrestore(rdp, flags);
2616	2471
2617	2472	/* Invoke callbacks. */
	2473	+ tick_dep_set_task(current, TICK_DEP_BIT_RCU);
2618	2474	rhp = rcu_cblist_dequeue(&rcl);
2619	2475	for (; rhp; rhp = rcu_cblist_dequeue(&rcl)) {
	2476	+ rcu_callback_t f;
	2477	+
2620	2478	debug_rcu_head_unqueue(rhp);
2621		- if (__rcu_reclaim(rsp->name, rhp))
2622		- rcu_cblist_dequeued_lazy(&rcl);
	2479	+
	2480	+ rcu_lock_acquire(&rcu_callback_map);
	2481	+ trace_rcu_invoke_callback(rcu_state.name, rhp);
	2482	+
	2483	+ f = rhp->func;
	2484	+ WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
	2485	+ f(rhp);
	2486	+
	2487	+ rcu_lock_release(&rcu_callback_map);
	2488	+
2623	2489	/*
2624	2490	* Stop only if limit reached and CPU has something to do.
2625	2491	* Note: The rcl structure counts down from zero.
2626	2492	*/
2627		- if (-rcl.len >= bl &&
2628		- (need_resched() \|\|
2629		- (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
2630		- break;
	2493	+ if (in_serving_softirq()) {
	2494	+ if (-rcl.len >= bl && (need_resched() \|\|
	2495	+ (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
	2496	+ break;
	2497	+
	2498	+ /*
	2499	+ * Make sure we don't spend too much time here and deprive other
	2500	+ * softirq vectors of CPU cycles.
	2501	+ */
	2502	+ if (unlikely(tlimit)) {
	2503	+ /* only call local_clock() every 32 callbacks */
	2504	+ if (likely((-rcl.len & 31) \|\| local_clock() < tlimit))
	2505	+ continue;
	2506	+ /* Exceeded the time limit, so leave. */
	2507	+ break;
	2508	+ }
	2509	+ } else {
	2510	+ local_bh_enable();
	2511	+ lockdep_assert_irqs_enabled();
	2512	+ cond_resched_tasks_rcu_qs();
	2513	+ lockdep_assert_irqs_enabled();
	2514	+ local_bh_disable();
	2515	+ }
2631	2516	}
2632	2517
2633	2518	local_irq_save(flags);
	2519	+ rcu_nocb_lock(rdp);
2634	2520	count = -rcl.len;
2635		- trace_rcu_batch_end(rsp->name, count, !!rcl.head, need_resched(),
	2521	+ rdp->n_cbs_invoked += count;
	2522	+ trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),
2636	2523	is_idle_task(current), rcu_is_callbacks_kthread());
2637	2524
2638	2525	/* Update counts and requeue any remaining callbacks. */
..	..	@@ -2642,13 +2529,13 @@
2642	2529
2643	2530	/* Reinstate batch limit if we have worked down the excess. */
2644	2531	count = rcu_segcblist_n_cbs(&rdp->cblist);
2645		- if (rdp->blimit == LONG_MAX && count <= qlowmark)
	2532	+ if (rdp->blimit >= DEFAULT_MAX_RCU_BLIMIT && count <= qlowmark)
2646	2533	rdp->blimit = blimit;
2647	2534
2648	2535	/* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */
2649	2536	if (count == 0 && rdp->qlen_last_fqs_check != 0) {
2650	2537	rdp->qlen_last_fqs_check = 0;
2651		- rdp->n_force_qs_snap = rsp->n_force_qs;
	2538	+ rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
2652	2539	} else if (count < rdp->qlen_last_fqs_check - qhimark)
2653	2540	rdp->qlen_last_fqs_check = count;
2654	2541
..	..	@@ -2656,94 +2543,72 @@
2656	2543	* The following usually indicates a double call_rcu(). To track
2657	2544	* this down, try building with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.
2658	2545	*/
2659		- WARN_ON_ONCE(rcu_segcblist_empty(&rdp->cblist) != (count == 0));
	2546	+ WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist));
	2547	+ WARN_ON_ONCE(!IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2548	+ count != 0 && rcu_segcblist_empty(&rdp->cblist));
2660	2549
2661		- local_irq_restore(flags);
	2550	+ rcu_nocb_unlock_irqrestore(rdp, flags);
2662	2551
2663	2552	/* Re-invoke RCU core processing if there are callbacks remaining. */
2664		- if (rcu_segcblist_ready_cbs(&rdp->cblist))
	2553	+ if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist))
2665	2554	invoke_rcu_core();
	2555	+ tick_dep_clear_task(current, TICK_DEP_BIT_RCU);
2666	2556	}
2667	2557
2668	2558	/*
2669		- * Check to see if this CPU is in a non-context-switch quiescent state
2670		- * (user mode or idle loop for rcu, non-softirq execution for rcu_bh).
2671		- * Also schedule RCU core processing.
2672		- *
2673		- * This function must be called from hardirq context. It is normally
2674		- * invoked from the scheduling-clock interrupt.
	2559	+ * This function is invoked from each scheduling-clock interrupt,
	2560	+ * and checks to see if this CPU is in a non-context-switch quiescent
	2561	+ * state, for example, user mode or idle loop. It also schedules RCU
	2562	+ * core processing. If the current grace period has gone on too long,
	2563	+ * it will ask the scheduler to manufacture a context switch for the sole
	2564	+ * purpose of providing a providing the needed quiescent state.
2675	2565	*/
2676		-void rcu_check_callbacks(int user)
	2566	+void rcu_sched_clock_irq(int user)
2677	2567	{
2678	2568	trace_rcu_utilization(TPS("Start scheduler-tick"));
2679		- increment_cpu_stall_ticks();
2680		- if (user \|\| rcu_is_cpu_rrupt_from_idle()) {
2681		-
2682		- /*
2683		- * Get here if this CPU took its interrupt from user
2684		- * mode or from the idle loop, and if this is not a
2685		- * nested interrupt. In this case, the CPU is in
2686		- * a quiescent state, so note it.
2687		- *
2688		- * No memory barrier is required here because both
2689		- * rcu_sched_qs() and rcu_bh_qs() reference only CPU-local
2690		- * variables that other CPUs neither access nor modify,
2691		- * at least not while the corresponding CPU is online.
2692		- */
2693		-
2694		- rcu_sched_qs();
2695		- rcu_bh_qs();
2696		- rcu_note_voluntary_context_switch(current);
2697		-
2698		- } else if (!in_softirq()) {
2699		-
2700		- /*
2701		- * Get here if this CPU did not take its interrupt from
2702		- * softirq, in other words, if it is not interrupting
2703		- * a rcu_bh read-side critical section. This is an _bh
2704		- * critical section, so note it.
2705		- */
2706		-
2707		- rcu_bh_qs();
2708		- }
2709		- rcu_preempt_check_callbacks();
	2569	+ lockdep_assert_irqs_disabled();
	2570	+ raw_cpu_inc(rcu_data.ticks_this_gp);
2710	2571	/* The load-acquire pairs with the store-release setting to true. */
2711		- if (smp_load_acquire(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs))) {
	2572	+ if (smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) {
2712	2573	/* Idle and userspace execution already are quiescent states. */
2713	2574	if (!rcu_is_cpu_rrupt_from_idle() && !user) {
2714	2575	set_tsk_need_resched(current);
2715	2576	set_preempt_need_resched();
2716	2577	}
2717		- __this_cpu_write(rcu_dynticks.rcu_urgent_qs, false);
	2578	+ __this_cpu_write(rcu_data.rcu_urgent_qs, false);
2718	2579	}
2719		- if (rcu_pending())
	2580	+ rcu_flavor_sched_clock_irq(user);
	2581	+ if (rcu_pending(user))
2720	2582	invoke_rcu_core();
	2583	+ lockdep_assert_irqs_disabled();
2721	2584
2722	2585	trace_rcu_utilization(TPS("End scheduler-tick"));
2723	2586	}
2724	2587
2725	2588	/*
2726		- * Scan the leaf rcu_node structures, processing dyntick state for any that
2727		- * have not yet encountered a quiescent state, using the function specified.
2728		- * Also initiate boosting for any threads blocked on the root rcu_node.
2729		- *
2730		- * The caller must have suppressed start of new grace periods.
	2589	+ * Scan the leaf rcu_node structures. For each structure on which all
	2590	+ * CPUs have reported a quiescent state and on which there are tasks
	2591	+ * blocking the current grace period, initiate RCU priority boosting.
	2592	+ * Otherwise, invoke the specified function to check dyntick state for
	2593	+ * each CPU that has not yet reported a quiescent state.
2731	2594	*/
2732		-static void force_qs_rnp(struct rcu_state rsp, int (f)(struct rcu_data *rsp))
	2595	+static void force_qs_rnp(int (f)(struct rcu_data rdp))
2733	2596	{
2734	2597	int cpu;
2735	2598	unsigned long flags;
2736	2599	unsigned long mask;
	2600	+ struct rcu_data *rdp;
2737	2601	struct rcu_node *rnp;
2738	2602
2739		- rcu_for_each_leaf_node(rsp, rnp) {
	2603	+ rcu_state.cbovld = rcu_state.cbovldnext;
	2604	+ rcu_state.cbovldnext = false;
	2605	+ rcu_for_each_leaf_node(rnp) {
2740	2606	cond_resched_tasks_rcu_qs();
2741	2607	mask = 0;
2742	2608	raw_spin_lock_irqsave_rcu_node(rnp, flags);
	2609	+ rcu_state.cbovldnext \|= !!rnp->cbovldmask;
2743	2610	if (rnp->qsmask == 0) {
2744		- if (rcu_state_p == &rcu_sched_state \|\|
2745		- rsp != rcu_state_p \|\|
2746		- rcu_preempt_blocked_readers_cgp(rnp)) {
	2611	+ if (rcu_preempt_blocked_readers_cgp(rnp)) {
2747	2612	/*
2748	2613	* No point in scanning bits because they
2749	2614	* are all zero. But we might need to
..	..	@@ -2756,16 +2621,16 @@
2756	2621	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2757	2622	continue;
2758	2623	}
2759		- for_each_leaf_node_possible_cpu(rnp, cpu) {
2760		- unsigned long bit = leaf_node_cpu_bit(rnp, cpu);
2761		- if ((rnp->qsmask & bit) != 0) {
2762		- if (f(per_cpu_ptr(rsp->rda, cpu)))
2763		- mask \|= bit;
	2624	+ for_each_leaf_node_cpu_mask(rnp, cpu, rnp->qsmask) {
	2625	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	2626	+ if (f(rdp)) {
	2627	+ mask \|= rdp->grpmask;
	2628	+ rcu_disable_urgency_upon_qs(rdp);
2764	2629	}
2765	2630	}
2766	2631	if (mask != 0) {
2767	2632	/* Idle/offline CPUs, report (releases rnp->lock). */
2768		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
	2633	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2769	2634	} else {
2770	2635	/* Nothing to do here, so just drop the lock. */
2771	2636	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
..	..	@@ -2777,7 +2642,7 @@
2777	2642	* Force quiescent states on reluctant CPUs, and also detect which
2778	2643	* CPUs are in dyntick-idle mode.
2779	2644	*/
2780		-static void force_quiescent_state(struct rcu_state *rsp)
	2645	+void rcu_force_quiescent_state(void)
2781	2646	{
2782	2647	unsigned long flags;
2783	2648	bool ret;
..	..	@@ -2785,151 +2650,93 @@
2785	2650	struct rcu_node *rnp_old = NULL;
2786	2651
2787	2652	/* Funnel through hierarchy to reduce memory contention. */
2788		- rnp = __this_cpu_read(rsp->rda->mynode);
	2653	+ rnp = __this_cpu_read(rcu_data.mynode);
2789	2654	for (; rnp != NULL; rnp = rnp->parent) {
2790		- ret = (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) \|\|
2791		- !raw_spin_trylock(&rnp->fqslock);
	2655	+ ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) \|\|
	2656	+ !raw_spin_trylock(&rnp->fqslock);
2792	2657	if (rnp_old != NULL)
2793	2658	raw_spin_unlock(&rnp_old->fqslock);
2794	2659	if (ret)
2795	2660	return;
2796	2661	rnp_old = rnp;
2797	2662	}
2798		- /* rnp_old == rcu_get_root(rsp), rnp == NULL. */
	2663	+ /* rnp_old == rcu_get_root(), rnp == NULL. */
2799	2664
2800	2665	/* Reached the root of the rcu_node tree, acquire lock. */
2801	2666	raw_spin_lock_irqsave_rcu_node(rnp_old, flags);
2802	2667	raw_spin_unlock(&rnp_old->fqslock);
2803		- if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
	2668	+ if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
2804	2669	raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
2805	2670	return; /* Someone beat us to it. */
2806	2671	}
2807		- WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) \| RCU_GP_FLAG_FQS);
	2672	+ WRITE_ONCE(rcu_state.gp_flags,
	2673	+ READ_ONCE(rcu_state.gp_flags) \| RCU_GP_FLAG_FQS);
2808	2674	raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
2809		- rcu_gp_kthread_wake(rsp);
	2675	+ rcu_gp_kthread_wake();
	2676	+}
	2677	+EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
	2678	+
	2679	+// Workqueue handler for an RCU reader for kernels enforcing struct RCU
	2680	+// grace periods.
	2681	+static void strict_work_handler(struct work_struct *work)
	2682	+{
	2683	+ rcu_read_lock();
	2684	+ rcu_read_unlock();
2810	2685	}
2811	2686
2812		-/*
2813		- * This function checks for grace-period requests that fail to motivate
2814		- * RCU to come out of its idle mode.
2815		- */
2816		-static void
2817		-rcu_check_gp_start_stall(struct rcu_state rsp, struct rcu_node rnp,
2818		- struct rcu_data *rdp)
2819		-{
2820		- const unsigned long gpssdelay = rcu_jiffies_till_stall_check() * HZ;
2821		- unsigned long flags;
2822		- unsigned long j;
2823		- struct rcu_node *rnp_root = rcu_get_root(rsp);
2824		- static atomic_t warned = ATOMIC_INIT(0);
2825		-
2826		- if (!IS_ENABLED(CONFIG_PROVE_RCU) \|\| rcu_gp_in_progress(rsp) \|\|
2827		- ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed))
2828		- return;
2829		- j = jiffies; /* Expensive access, and in common case don't get here. */
2830		- if (time_before(j, READ_ONCE(rsp->gp_req_activity) + gpssdelay) \|\|
2831		- time_before(j, READ_ONCE(rsp->gp_activity) + gpssdelay) \|\|
2832		- atomic_read(&warned))
2833		- return;
2834		-
2835		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
2836		- j = jiffies;
2837		- if (rcu_gp_in_progress(rsp) \|\|
2838		- ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) \|\|
2839		- time_before(j, READ_ONCE(rsp->gp_req_activity) + gpssdelay) \|\|
2840		- time_before(j, READ_ONCE(rsp->gp_activity) + gpssdelay) \|\|
2841		- atomic_read(&warned)) {
2842		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2843		- return;
2844		- }
2845		- /* Hold onto the leaf lock to make others see warned==1. */
2846		-
2847		- if (rnp_root != rnp)
2848		- raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
2849		- j = jiffies;
2850		- if (rcu_gp_in_progress(rsp) \|\|
2851		- ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) \|\|
2852		- time_before(j, rsp->gp_req_activity + gpssdelay) \|\|
2853		- time_before(j, rsp->gp_activity + gpssdelay) \|\|
2854		- atomic_xchg(&warned, 1)) {
2855		- raw_spin_unlock_rcu_node(rnp_root); /* irqs remain disabled. */
2856		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2857		- return;
2858		- }
2859		- pr_alert("%s: g%ld->%ld gar:%lu ga:%lu f%#x gs:%d %s->state:%#lx\n",
2860		- __func__, (long)READ_ONCE(rsp->gp_seq),
2861		- (long)READ_ONCE(rnp_root->gp_seq_needed),
2862		- j - rsp->gp_req_activity, j - rsp->gp_activity,
2863		- rsp->gp_flags, rsp->gp_state, rsp->name,
2864		- rsp->gp_kthread ? rsp->gp_kthread->state : 0x1ffffL);
2865		- WARN_ON(1);
2866		- if (rnp_root != rnp)
2867		- raw_spin_unlock_rcu_node(rnp_root);
2868		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2869		-}
2870		-
2871		-/*
2872		- * This does the RCU core processing work for the specified rcu_state
2873		- * and rcu_data structures. This may be called only from the CPU to
2874		- * whom the rdp belongs.
2875		- */
2876		-static void
2877		-__rcu_process_callbacks(struct rcu_state *rsp)
	2687	+/* Perform RCU core processing work for the current CPU. */
	2688	+static __latent_entropy void rcu_core(void)
2878	2689	{
2879	2690	unsigned long flags;
2880		- struct rcu_data *rdp = raw_cpu_ptr(rsp->rda);
	2691	+ struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
2881	2692	struct rcu_node *rnp = rdp->mynode;
2882		-
2883		- WARN_ON_ONCE(!rdp->beenonline);
2884		-
2885		- /* Update RCU state based on any recent quiescent states. */
2886		- rcu_check_quiescent_state(rsp, rdp);
2887		-
2888		- /* No grace period and unregistered callbacks? */
2889		- if (!rcu_gp_in_progress(rsp) &&
2890		- rcu_segcblist_is_enabled(&rdp->cblist)) {
2891		- local_irq_save(flags);
2892		- if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
2893		- rcu_accelerate_cbs_unlocked(rsp, rnp, rdp);
2894		- local_irq_restore(flags);
2895		- }
2896		-
2897		- rcu_check_gp_start_stall(rsp, rnp, rdp);
2898		-
2899		- /* If there are callbacks ready, invoke them. */
2900		- if (rcu_segcblist_ready_cbs(&rdp->cblist))
2901		- invoke_rcu_callbacks(rsp, rdp);
2902		-
2903		- /* Do any needed deferred wakeups of rcuo kthreads. */
2904		- do_nocb_deferred_wakeup(rdp);
2905		-}
2906		-
2907		-/*
2908		- * Do RCU core processing for the current CPU.
2909		- */
2910		-static __latent_entropy void rcu_process_callbacks(void)
2911		-{
2912		- struct rcu_state *rsp;
	2693	+ const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2694	+ rcu_segcblist_is_offloaded(&rdp->cblist);
2913	2695
2914	2696	if (cpu_is_offline(smp_processor_id()))
2915	2697	return;
2916		- for_each_rcu_flavor(rsp)
2917		- __rcu_process_callbacks(rsp);
	2698	+ trace_rcu_utilization(TPS("Start RCU core"));
	2699	+ WARN_ON_ONCE(!rdp->beenonline);
	2700	+
	2701	+ /* Report any deferred quiescent states if preemption enabled. */
	2702	+ if (!(preempt_count() & PREEMPT_MASK)) {
	2703	+ rcu_preempt_deferred_qs(current);
	2704	+ } else if (rcu_preempt_need_deferred_qs(current)) {
	2705	+ set_tsk_need_resched(current);
	2706	+ set_preempt_need_resched();
	2707	+ }
	2708	+
	2709	+ /* Update RCU state based on any recent quiescent states. */
	2710	+ rcu_check_quiescent_state(rdp);
	2711	+
	2712	+ /* No grace period and unregistered callbacks? */
	2713	+ if (!rcu_gp_in_progress() &&
	2714	+ rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) {
	2715	+ local_irq_save(flags);
	2716	+ if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
	2717	+ rcu_accelerate_cbs_unlocked(rnp, rdp);
	2718	+ local_irq_restore(flags);
	2719	+ }
	2720	+
	2721	+ rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check());
	2722	+
	2723	+ /* If there are callbacks ready, invoke them. */
	2724	+ if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) &&
	2725	+ likely(READ_ONCE(rcu_scheduler_fully_active)))
	2726	+ rcu_do_batch(rdp);
	2727	+
	2728	+ /* Do any needed deferred wakeups of rcuo kthreads. */
	2729	+ do_nocb_deferred_wakeup(rdp);
	2730	+ trace_rcu_utilization(TPS("End RCU core"));
	2731	+
	2732	+ // If strict GPs, schedule an RCU reader in a clean environment.
	2733	+ if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
	2734	+ queue_work_on(rdp->cpu, rcu_gp_wq, &rdp->strict_work);
2918	2735	}
2919	2736
2920		-static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
2921		-/*
2922		- * Schedule RCU callback invocation. If the specified type of RCU
2923		- * does not support RCU priority boosting, just do a direct call,
2924		- * otherwise wake up the per-CPU kernel kthread. Note that because we
2925		- * are running on the current CPU with softirqs disabled, the
2926		- * rcu_cpu_kthread_task cannot disappear out from under us.
2927		- */
2928		-static void invoke_rcu_callbacks(struct rcu_state rsp, struct rcu_data rdp)
	2737	+static void rcu_core_si(struct softirq_action *h)
2929	2738	{
2930		- if (unlikely(!READ_ONCE(rcu_scheduler_fully_active)))
2931		- return;
2932		- rcu_do_batch(rsp, rdp);
	2739	+ rcu_core();
2933	2740	}
2934	2741
2935	2742	static void rcu_wake_cond(struct task_struct *t, int status)
..	..	@@ -2942,56 +2749,63 @@
2942	2749	wake_up_process(t);
2943	2750	}
2944	2751
	2752	+static void invoke_rcu_core_kthread(void)
	2753	+{
	2754	+ struct task_struct *t;
	2755	+ unsigned long flags;
	2756	+
	2757	+ local_irq_save(flags);
	2758	+ __this_cpu_write(rcu_data.rcu_cpu_has_work, 1);
	2759	+ t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task);
	2760	+ if (t != NULL && t != current)
	2761	+ rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status));
	2762	+ local_irq_restore(flags);
	2763	+}
	2764	+
2945	2765	/*
2946	2766	* Wake up this CPU's rcuc kthread to do RCU core processing.
2947	2767	*/
2948	2768	static void invoke_rcu_core(void)
2949	2769	{
2950		- unsigned long flags;
2951		- struct task_struct *t;
2952		-
2953	2770	if (!cpu_online(smp_processor_id()))
2954	2771	return;
2955		- local_irq_save(flags);
2956		- __this_cpu_write(rcu_cpu_has_work, 1);
2957		- t = __this_cpu_read(rcu_cpu_kthread_task);
2958		- if (t != NULL && current != t)
2959		- rcu_wake_cond(t, __this_cpu_read(rcu_cpu_kthread_status));
2960		- local_irq_restore(flags);
	2772	+ if (use_softirq)
	2773	+ raise_softirq(RCU_SOFTIRQ);
	2774	+ else
	2775	+ invoke_rcu_core_kthread();
2961	2776	}
2962	2777
2963	2778	static void rcu_cpu_kthread_park(unsigned int cpu)
2964	2779	{
2965		- per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
	2780	+ per_cpu(rcu_data.rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
2966	2781	}
2967	2782
2968	2783	static int rcu_cpu_kthread_should_run(unsigned int cpu)
2969	2784	{
2970		- return __this_cpu_read(rcu_cpu_has_work);
	2785	+ return __this_cpu_read(rcu_data.rcu_cpu_has_work);
2971	2786	}
2972	2787
2973	2788	/*
2974		- * Per-CPU kernel thread that invokes RCU callbacks. This replaces the
2975		- * RCU softirq used in flavors and configurations of RCU that do not
2976		- * support RCU priority boosting.
	2789	+ * Per-CPU kernel thread that invokes RCU callbacks. This replaces
	2790	+ * the RCU softirq used in configurations of RCU that do not support RCU
	2791	+ * priority boosting.
2977	2792	*/
2978	2793	static void rcu_cpu_kthread(unsigned int cpu)
2979	2794	{
2980		- unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
2981		- char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
	2795	+ unsigned int *statusp = this_cpu_ptr(&rcu_data.rcu_cpu_kthread_status);
	2796	+ char work, *workp = this_cpu_ptr(&rcu_data.rcu_cpu_has_work);
2982	2797	int spincnt;
2983	2798
	2799	+ trace_rcu_utilization(TPS("Start CPU kthread@rcu_run"));
2984	2800	for (spincnt = 0; spincnt < 10; spincnt++) {
2985		- trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
2986	2801	local_bh_disable();
2987	2802	*statusp = RCU_KTHREAD_RUNNING;
2988		- this_cpu_inc(rcu_cpu_kthread_loops);
2989	2803	local_irq_disable();
2990	2804	work = *workp;
2991	2805	*workp = 0;
2992	2806	local_irq_enable();
2993	2807	if (work)
2994		- rcu_process_callbacks();
	2808	+ rcu_core();
2995	2809	local_bh_enable();
2996	2810	if (*workp == 0) {
2997	2811	trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
..	..	@@ -3001,13 +2815,13 @@
3001	2815	}
3002	2816	*statusp = RCU_KTHREAD_YIELDING;
3003	2817	trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
3004		- schedule_timeout_interruptible(2);
	2818	+ schedule_timeout_idle(2);
3005	2819	trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
3006	2820	*statusp = RCU_KTHREAD_WAITING;
3007	2821	}
3008	2822
3009	2823	static struct smp_hotplug_thread rcu_cpu_thread_spec = {
3010		- .store = &rcu_cpu_kthread_task,
	2824	+ .store = &rcu_data.rcu_cpu_kthread_task,
3011	2825	.thread_should_run = rcu_cpu_kthread_should_run,
3012	2826	.thread_fn = rcu_cpu_kthread,
3013	2827	.thread_comm = "rcuc/%u",
..	..	@@ -3023,17 +2837,19 @@
3023	2837	int cpu;
3024	2838
3025	2839	for_each_possible_cpu(cpu)
3026		- per_cpu(rcu_cpu_has_work, cpu) = 0;
3027		- BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
	2840	+ per_cpu(rcu_data.rcu_cpu_has_work, cpu) = 0;
	2841	+ if (!IS_ENABLED(CONFIG_RCU_BOOST) && use_softirq)
	2842	+ return 0;
	2843	+ WARN_ONCE(smpboot_register_percpu_thread(&rcu_cpu_thread_spec),
	2844	+ "%s: Could not start rcuc kthread, OOM is now expected behavior\n", __func__);
3028	2845	return 0;
3029	2846	}
3030		-early_initcall(rcu_spawn_core_kthreads);
3031	2847
3032	2848	/*
3033	2849	* Handle any core-RCU processing required by a call_rcu() invocation.
3034	2850	*/
3035		-static void __call_rcu_core(struct rcu_state rsp, struct rcu_data rdp,
3036		- struct rcu_head *head, unsigned long flags)
	2851	+static void __call_rcu_core(struct rcu_data rdp, struct rcu_head head,
	2852	+ unsigned long flags)
3037	2853	{
3038	2854	/*
3039	2855	* If called from an extended quiescent state, invoke the RCU
..	..	@@ -3048,27 +2864,27 @@
3048	2864
3049	2865	/*
3050	2866	* Force the grace period if too many callbacks or too long waiting.
3051		- * Enforce hysteresis, and don't invoke force_quiescent_state()
	2867	+ * Enforce hysteresis, and don't invoke rcu_force_quiescent_state()
3052	2868	* if some other CPU has recently done so. Also, don't bother
3053		- * invoking force_quiescent_state() if the newly enqueued callback
	2869	+ * invoking rcu_force_quiescent_state() if the newly enqueued callback
3054	2870	* is the only one waiting for a grace period to complete.
3055	2871	*/
3056	2872	if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) >
3057	2873	rdp->qlen_last_fqs_check + qhimark)) {
3058	2874
3059	2875	/* Are we ignoring a completed grace period? */
3060		- note_gp_changes(rsp, rdp);
	2876	+ note_gp_changes(rdp);
3061	2877
3062	2878	/* Start a new grace period if one not already started. */
3063		- if (!rcu_gp_in_progress(rsp)) {
3064		- rcu_accelerate_cbs_unlocked(rsp, rdp->mynode, rdp);
	2879	+ if (!rcu_gp_in_progress()) {
	2880	+ rcu_accelerate_cbs_unlocked(rdp->mynode, rdp);
3065	2881	} else {
3066	2882	/* Give the grace period a kick. */
3067		- rdp->blimit = LONG_MAX;
3068		- if (rsp->n_force_qs == rdp->n_force_qs_snap &&
	2883	+ rdp->blimit = DEFAULT_MAX_RCU_BLIMIT;
	2884	+ if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap &&
3069	2885	rcu_segcblist_first_pend_cb(&rdp->cblist) != head)
3070		- force_quiescent_state(rsp);
3071		- rdp->n_force_qs_snap = rsp->n_force_qs;
	2886	+ rcu_force_quiescent_state();
	2887	+ rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
3072	2888	rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
3073	2889	}
3074	2890	}
..	..	@@ -3082,17 +2898,54 @@
3082	2898	}
3083	2899
3084	2900	/*
3085		- * Helper function for call_rcu() and friends. The cpu argument will
3086		- * normally be -1, indicating "currently running CPU". It may specify
3087		- * a CPU only if that CPU is a no-CBs CPU. Currently, only _rcu_barrier()
3088		- * is expected to specify a CPU.
	2901	+ * Check and if necessary update the leaf rcu_node structure's
	2902	+ * ->cbovldmask bit corresponding to the current CPU based on that CPU's
	2903	+ * number of queued RCU callbacks. The caller must hold the leaf rcu_node
	2904	+ * structure's ->lock.
3089	2905	*/
	2906	+static void check_cb_ovld_locked(struct rcu_data rdp, struct rcu_node rnp)
	2907	+{
	2908	+ raw_lockdep_assert_held_rcu_node(rnp);
	2909	+ if (qovld_calc <= 0)
	2910	+ return; // Early boot and wildcard value set.
	2911	+ if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc)
	2912	+ WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask \| rdp->grpmask);
	2913	+ else
	2914	+ WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask & ~rdp->grpmask);
	2915	+}
	2916	+
	2917	+/*
	2918	+ * Check and if necessary update the leaf rcu_node structure's
	2919	+ * ->cbovldmask bit corresponding to the current CPU based on that CPU's
	2920	+ * number of queued RCU callbacks. No locks need be held, but the
	2921	+ * caller must have disabled interrupts.
	2922	+ *
	2923	+ * Note that this function ignores the possibility that there are a lot
	2924	+ * of callbacks all of which have already seen the end of their respective
	2925	+ * grace periods. This omission is due to the need for no-CBs CPUs to
	2926	+ * be holding ->nocb_lock to do this check, which is too heavy for a
	2927	+ * common-case operation.
	2928	+ */
	2929	+static void check_cb_ovld(struct rcu_data *rdp)
	2930	+{
	2931	+ struct rcu_node *const rnp = rdp->mynode;
	2932	+
	2933	+ if (qovld_calc <= 0 \|\|
	2934	+ ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) ==
	2935	+ !!(READ_ONCE(rnp->cbovldmask) & rdp->grpmask)))
	2936	+ return; // Early boot wildcard value or already set correctly.
	2937	+ raw_spin_lock_rcu_node(rnp);
	2938	+ check_cb_ovld_locked(rdp, rnp);
	2939	+ raw_spin_unlock_rcu_node(rnp);
	2940	+}
	2941	+
	2942	+/* Helper function for call_rcu() and friends. */
3090	2943	static void
3091		-__call_rcu(struct rcu_head *head, rcu_callback_t func,
3092		- struct rcu_state *rsp, int cpu, bool lazy)
	2944	+__call_rcu(struct rcu_head *head, rcu_callback_t func)
3093	2945	{
3094	2946	unsigned long flags;
3095	2947	struct rcu_data *rdp;
	2948	+ bool was_alldone;
3096	2949
3097	2950	/* Misaligned rcu_head! */
3098	2951	WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1));
..	..	@@ -3103,7 +2956,7 @@
3103	2956	* Use rcu:rcu_callback trace event to find the previous
3104	2957	* time callback was passed to __call_rcu().
3105	2958	*/
3106		- WARN_ONCE(1, "__call_rcu(): Double-freed CB %p->%pF()!!!\n",
	2959	+ WARN_ONCE(1, "__call_rcu(): Double-freed CB %p->%pS()!!!\n",
3107	2960	head, head->func);
3108	2961	WRITE_ONCE(head->func, rcu_leak_callback);
3109	2962	return;
..	..	@@ -3111,132 +2964,671 @@
3111	2964	head->func = func;
3112	2965	head->next = NULL;
3113	2966	local_irq_save(flags);
3114		- rdp = this_cpu_ptr(rsp->rda);
	2967	+ kasan_record_aux_stack(head);
	2968	+ rdp = this_cpu_ptr(&rcu_data);
3115	2969
3116	2970	/* Add the callback to our list. */
3117		- if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist)) \|\| cpu != -1) {
3118		- int offline;
3119		-
3120		- if (cpu != -1)
3121		- rdp = per_cpu_ptr(rsp->rda, cpu);
3122		- if (likely(rdp->mynode)) {
3123		- /* Post-boot, so this should be for a no-CBs CPU. */
3124		- offline = !__call_rcu_nocb(rdp, head, lazy, flags);
3125		- WARN_ON_ONCE(offline);
3126		- /* Offline CPU, _call_rcu() illegal, leak callback. */
3127		- local_irq_restore(flags);
3128		- return;
3129		- }
3130		- /*
3131		- * Very early boot, before rcu_init(). Initialize if needed
3132		- * and then drop through to queue the callback.
3133		- */
3134		- BUG_ON(cpu != -1);
	2971	+ if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) {
	2972	+ // This can trigger due to call_rcu() from offline CPU:
	2973	+ WARN_ON_ONCE(rcu_scheduler_active != RCU_SCHEDULER_INACTIVE);
3135	2974	WARN_ON_ONCE(!rcu_is_watching());
	2975	+ // Very early boot, before rcu_init(). Initialize if needed
	2976	+ // and then drop through to queue the callback.
3136	2977	if (rcu_segcblist_empty(&rdp->cblist))
3137	2978	rcu_segcblist_init(&rdp->cblist);
3138	2979	}
3139		- rcu_segcblist_enqueue(&rdp->cblist, head, lazy);
3140		- if (!lazy)
3141		- rcu_idle_count_callbacks_posted();
3142	2980
3143		- if (__is_kfree_rcu_offset((unsigned long)func))
3144		- trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func,
3145		- rcu_segcblist_n_lazy_cbs(&rdp->cblist),
	2981	+ check_cb_ovld(rdp);
	2982	+ if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags))
	2983	+ return; // Enqueued onto ->nocb_bypass, so just leave.
	2984	+ // If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock.
	2985	+ rcu_segcblist_enqueue(&rdp->cblist, head);
	2986	+ if (__is_kvfree_rcu_offset((unsigned long)func))
	2987	+ trace_rcu_kvfree_callback(rcu_state.name, head,
	2988	+ (unsigned long)func,
3146	2989	rcu_segcblist_n_cbs(&rdp->cblist));
3147	2990	else
3148		- trace_rcu_callback(rsp->name, head,
3149		- rcu_segcblist_n_lazy_cbs(&rdp->cblist),
	2991	+ trace_rcu_callback(rcu_state.name, head,
3150	2992	rcu_segcblist_n_cbs(&rdp->cblist));
3151	2993
3152	2994	/* Go handle any RCU core processing required. */
3153		- __call_rcu_core(rsp, rdp, head, flags);
	2995	+ if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2996	+ unlikely(rcu_segcblist_is_offloaded(&rdp->cblist))) {
	2997	+ __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */
	2998	+ } else {
	2999	+ __call_rcu_core(rdp, head, flags);
	3000	+ local_irq_restore(flags);
	3001	+ }
	3002	+}
	3003	+
	3004	+/**
	3005	+ * call_rcu() - Queue an RCU callback for invocation after a grace period.
	3006	+ * @head: structure to be used for queueing the RCU updates.
	3007	+ * @func: actual callback function to be invoked after the grace period
	3008	+ *
	3009	+ * The callback function will be invoked some time after a full grace
	3010	+ * period elapses, in other words after all pre-existing RCU read-side
	3011	+ * critical sections have completed. However, the callback function
	3012	+ * might well execute concurrently with RCU read-side critical sections
	3013	+ * that started after call_rcu() was invoked. RCU read-side critical
	3014	+ * sections are delimited by rcu_read_lock() and rcu_read_unlock(), and
	3015	+ * may be nested. In addition, regions of code across which interrupts,
	3016	+ * preemption, or softirqs have been disabled also serve as RCU read-side
	3017	+ * critical sections. This includes hardware interrupt handlers, softirq
	3018	+ * handlers, and NMI handlers.
	3019	+ *
	3020	+ * Note that all CPUs must agree that the grace period extended beyond
	3021	+ * all pre-existing RCU read-side critical section. On systems with more
	3022	+ * than one CPU, this means that when "func()" is invoked, each CPU is
	3023	+ * guaranteed to have executed a full memory barrier since the end of its
	3024	+ * last RCU read-side critical section whose beginning preceded the call
	3025	+ * to call_rcu(). It also means that each CPU executing an RCU read-side
	3026	+ * critical section that continues beyond the start of "func()" must have
	3027	+ * executed a memory barrier after the call_rcu() but before the beginning
	3028	+ * of that RCU read-side critical section. Note that these guarantees
	3029	+ * include CPUs that are offline, idle, or executing in user mode, as
	3030	+ * well as CPUs that are executing in the kernel.
	3031	+ *
	3032	+ * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
	3033	+ * resulting RCU callback function "func()", then both CPU A and CPU B are
	3034	+ * guaranteed to execute a full memory barrier during the time interval
	3035	+ * between the call to call_rcu() and the invocation of "func()" -- even
	3036	+ * if CPU A and CPU B are the same CPU (but again only if the system has
	3037	+ * more than one CPU).
	3038	+ */
	3039	+void call_rcu(struct rcu_head *head, rcu_callback_t func)
	3040	+{
	3041	+ __call_rcu(head, func);
	3042	+}
	3043	+EXPORT_SYMBOL_GPL(call_rcu);
	3044	+
	3045	+
	3046	+/* Maximum number of jiffies to wait before draining a batch. */
	3047	+#define KFREE_DRAIN_JIFFIES (HZ / 50)
	3048	+#define KFREE_N_BATCHES 2
	3049	+#define FREE_N_CHANNELS 2
	3050	+
	3051	+/**
	3052	+ * struct kvfree_rcu_bulk_data - single block to store kvfree_rcu() pointers
	3053	+ * @nr_records: Number of active pointers in the array
	3054	+ * @next: Next bulk object in the block chain
	3055	+ * @records: Array of the kvfree_rcu() pointers
	3056	+ */
	3057	+struct kvfree_rcu_bulk_data {
	3058	+ unsigned long nr_records;
	3059	+ struct kvfree_rcu_bulk_data *next;
	3060	+ void *records[];
	3061	+};
	3062	+
	3063	+/*
	3064	+ * This macro defines how many entries the "records" array
	3065	+ * will contain. It is based on the fact that the size of
	3066	+ * kvfree_rcu_bulk_data structure becomes exactly one page.
	3067	+ */
	3068	+#define KVFREE_BULK_MAX_ENTR \
	3069	+ ((PAGE_SIZE - sizeof(struct kvfree_rcu_bulk_data)) / sizeof(void *))
	3070	+
	3071	+/**
	3072	+ * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests
	3073	+ * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period
	3074	+ * @head_free: List of kfree_rcu() objects waiting for a grace period
	3075	+ * @bkvhead_free: Bulk-List of kvfree_rcu() objects waiting for a grace period
	3076	+ * @krcp: Pointer to @kfree_rcu_cpu structure
	3077	+ */
	3078	+
	3079	+struct kfree_rcu_cpu_work {
	3080	+ struct rcu_work rcu_work;
	3081	+ struct rcu_head *head_free;
	3082	+ struct kvfree_rcu_bulk_data *bkvhead_free[FREE_N_CHANNELS];
	3083	+ struct kfree_rcu_cpu *krcp;
	3084	+};
	3085	+
	3086	+/**
	3087	+ * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period
	3088	+ * @head: List of kfree_rcu() objects not yet waiting for a grace period
	3089	+ * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period
	3090	+ * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period
	3091	+ * @lock: Synchronize access to this structure
	3092	+ * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES
	3093	+ * @monitor_todo: Tracks whether a @monitor_work delayed work is pending
	3094	+ * @initialized: The @rcu_work fields have been initialized
	3095	+ * @count: Number of objects for which GP not started
	3096	+ * @bkvcache:
	3097	+ * A simple cache list that contains objects for reuse purpose.
	3098	+ * In order to save some per-cpu space the list is singular.
	3099	+ * Even though it is lockless an access has to be protected by the
	3100	+ * per-cpu lock.
	3101	+ * @page_cache_work: A work to refill the cache when it is empty
	3102	+ * @work_in_progress: Indicates that page_cache_work is running
	3103	+ * @hrtimer: A hrtimer for scheduling a page_cache_work
	3104	+ * @nr_bkv_objs: number of allocated objects at @bkvcache.
	3105	+ *
	3106	+ * This is a per-CPU structure. The reason that it is not included in
	3107	+ * the rcu_data structure is to permit this code to be extracted from
	3108	+ * the RCU files. Such extraction could allow further optimization of
	3109	+ * the interactions with the slab allocators.
	3110	+ */
	3111	+struct kfree_rcu_cpu {
	3112	+ struct rcu_head *head;
	3113	+ struct kvfree_rcu_bulk_data *bkvhead[FREE_N_CHANNELS];
	3114	+ struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES];
	3115	+ raw_spinlock_t lock;
	3116	+ struct delayed_work monitor_work;
	3117	+ bool monitor_todo;
	3118	+ bool initialized;
	3119	+ int count;
	3120	+
	3121	+ struct work_struct page_cache_work;
	3122	+ atomic_t work_in_progress;
	3123	+ struct hrtimer hrtimer;
	3124	+
	3125	+ struct llist_head bkvcache;
	3126	+ int nr_bkv_objs;
	3127	+};
	3128	+
	3129	+static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc) = {
	3130	+ .lock = __RAW_SPIN_LOCK_UNLOCKED(krc.lock),
	3131	+};
	3132	+
	3133	+static __always_inline void
	3134	+debug_rcu_bhead_unqueue(struct kvfree_rcu_bulk_data *bhead)
	3135	+{
	3136	+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
	3137	+ int i;
	3138	+
	3139	+ for (i = 0; i < bhead->nr_records; i++)
	3140	+ debug_rcu_head_unqueue((struct rcu_head *)(bhead->records[i]));
	3141	+#endif
	3142	+}
	3143	+
	3144	+static inline struct kfree_rcu_cpu *
	3145	+krc_this_cpu_lock(unsigned long *flags)
	3146	+{
	3147	+ struct kfree_rcu_cpu *krcp;
	3148	+
	3149	+ local_irq_save(*flags); // For safely calling this_cpu_ptr().
	3150	+ krcp = this_cpu_ptr(&krc);
	3151	+ raw_spin_lock(&krcp->lock);
	3152	+
	3153	+ return krcp;
	3154	+}
	3155	+
	3156	+static inline void
	3157	+krc_this_cpu_unlock(struct kfree_rcu_cpu *krcp, unsigned long flags)
	3158	+{
	3159	+ raw_spin_unlock(&krcp->lock);
3154	3160	local_irq_restore(flags);
3155	3161	}
3156	3162
3157		-/**
3158		- * call_rcu_sched() - Queue an RCU for invocation after sched grace period.
3159		- * @head: structure to be used for queueing the RCU updates.
3160		- * @func: actual callback function to be invoked after the grace period
3161		- *
3162		- * The callback function will be invoked some time after a full grace
3163		- * period elapses, in other words after all currently executing RCU
3164		- * read-side critical sections have completed. call_rcu_sched() assumes
3165		- * that the read-side critical sections end on enabling of preemption
3166		- * or on voluntary preemption.
3167		- * RCU read-side critical sections are delimited by:
3168		- *
3169		- * - rcu_read_lock_sched() and rcu_read_unlock_sched(), OR
3170		- * - anything that disables preemption.
3171		- *
3172		- * These may be nested.
3173		- *
3174		- * See the description of call_rcu() for more detailed information on
3175		- * memory ordering guarantees.
3176		- */
3177		-void call_rcu_sched(struct rcu_head *head, rcu_callback_t func)
	3163	+static inline struct kvfree_rcu_bulk_data *
	3164	+get_cached_bnode(struct kfree_rcu_cpu *krcp)
3178	3165	{
3179		- __call_rcu(head, func, &rcu_sched_state, -1, 0);
3180		-}
3181		-EXPORT_SYMBOL_GPL(call_rcu_sched);
	3166	+ if (!krcp->nr_bkv_objs)
	3167	+ return NULL;
3182	3168
3183		-#ifndef CONFIG_PREEMPT_RT_FULL
3184		-/**
3185		- * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
3186		- * @head: structure to be used for queueing the RCU updates.
3187		- * @func: actual callback function to be invoked after the grace period
3188		- *
3189		- * The callback function will be invoked some time after a full grace
3190		- * period elapses, in other words after all currently executing RCU
3191		- * read-side critical sections have completed. call_rcu_bh() assumes
3192		- * that the read-side critical sections end on completion of a softirq
3193		- * handler. This means that read-side critical sections in process
3194		- * context must not be interrupted by softirqs. This interface is to be
3195		- * used when most of the read-side critical sections are in softirq context.
3196		- * RCU read-side critical sections are delimited by:
3197		- *
3198		- * - rcu_read_lock() and rcu_read_unlock(), if in interrupt context, OR
3199		- * - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
3200		- *
3201		- * These may be nested.
3202		- *
3203		- * See the description of call_rcu() for more detailed information on
3204		- * memory ordering guarantees.
3205		- */
3206		-void call_rcu_bh(struct rcu_head *head, rcu_callback_t func)
3207		-{
3208		- __call_rcu(head, func, &rcu_bh_state, -1, 0);
	3169	+ krcp->nr_bkv_objs--;
	3170	+ return (struct kvfree_rcu_bulk_data *)
	3171	+ llist_del_first(&krcp->bkvcache);
3209	3172	}
3210		-EXPORT_SYMBOL_GPL(call_rcu_bh);
3211		-#endif
	3173	+
	3174	+static inline bool
	3175	+put_cached_bnode(struct kfree_rcu_cpu *krcp,
	3176	+ struct kvfree_rcu_bulk_data *bnode)
	3177	+{
	3178	+ // Check the limit.
	3179	+ if (krcp->nr_bkv_objs >= rcu_min_cached_objs)
	3180	+ return false;
	3181	+
	3182	+ llist_add((struct llist_node *) bnode, &krcp->bkvcache);
	3183	+ krcp->nr_bkv_objs++;
	3184	+ return true;
	3185	+
	3186	+}
3212	3187
3213	3188	/*
3214		- * Queue an RCU callback for lazy invocation after a grace period.
3215		- * This will likely be later named something like "call_rcu_lazy()",
3216		- * but this change will require some way of tagging the lazy RCU
3217		- * callbacks in the list of pending callbacks. Until then, this
3218		- * function may only be called from __kfree_rcu().
	3189	+ * This function is invoked in workqueue context after a grace period.
	3190	+ * It frees all the objects queued on ->bhead_free or ->head_free.
3219	3191	*/
3220		-void kfree_call_rcu(struct rcu_head *head,
3221		- rcu_callback_t func)
	3192	+static void kfree_rcu_work(struct work_struct *work)
3222	3193	{
3223		- __call_rcu(head, func, rcu_state_p, -1, 1);
	3194	+ unsigned long flags;
	3195	+ struct kvfree_rcu_bulk_data bkvhead[FREE_N_CHANNELS], bnext;
	3196	+ struct rcu_head head, next;
	3197	+ struct kfree_rcu_cpu *krcp;
	3198	+ struct kfree_rcu_cpu_work *krwp;
	3199	+ int i, j;
	3200	+
	3201	+ krwp = container_of(to_rcu_work(work),
	3202	+ struct kfree_rcu_cpu_work, rcu_work);
	3203	+ krcp = krwp->krcp;
	3204	+
	3205	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3206	+ // Channels 1 and 2.
	3207	+ for (i = 0; i < FREE_N_CHANNELS; i++) {
	3208	+ bkvhead[i] = krwp->bkvhead_free[i];
	3209	+ krwp->bkvhead_free[i] = NULL;
	3210	+ }
	3211	+
	3212	+ // Channel 3.
	3213	+ head = krwp->head_free;
	3214	+ krwp->head_free = NULL;
	3215	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3216	+
	3217	+ // Handle two first channels.
	3218	+ for (i = 0; i < FREE_N_CHANNELS; i++) {
	3219	+ for (; bkvhead[i]; bkvhead[i] = bnext) {
	3220	+ bnext = bkvhead[i]->next;
	3221	+ debug_rcu_bhead_unqueue(bkvhead[i]);
	3222	+
	3223	+ rcu_lock_acquire(&rcu_callback_map);
	3224	+ if (i == 0) { // kmalloc() / kfree().
	3225	+ trace_rcu_invoke_kfree_bulk_callback(
	3226	+ rcu_state.name, bkvhead[i]->nr_records,
	3227	+ bkvhead[i]->records);
	3228	+
	3229	+ kfree_bulk(bkvhead[i]->nr_records,
	3230	+ bkvhead[i]->records);
	3231	+ } else { // vmalloc() / vfree().
	3232	+ for (j = 0; j < bkvhead[i]->nr_records; j++) {
	3233	+ trace_rcu_invoke_kvfree_callback(
	3234	+ rcu_state.name,
	3235	+ bkvhead[i]->records[j], 0);
	3236	+
	3237	+ vfree(bkvhead[i]->records[j]);
	3238	+ }
	3239	+ }
	3240	+ rcu_lock_release(&rcu_callback_map);
	3241	+
	3242	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3243	+ if (put_cached_bnode(krcp, bkvhead[i]))
	3244	+ bkvhead[i] = NULL;
	3245	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3246	+
	3247	+ if (bkvhead[i])
	3248	+ free_page((unsigned long) bkvhead[i]);
	3249	+
	3250	+ cond_resched_tasks_rcu_qs();
	3251	+ }
	3252	+ }
	3253	+
	3254	+ /*
	3255	+ * Emergency case only. It can happen under low memory
	3256	+ * condition when an allocation gets failed, so the "bulk"
	3257	+ * path can not be temporary maintained.
	3258	+ */
	3259	+ for (; head; head = next) {
	3260	+ unsigned long offset = (unsigned long)head->func;
	3261	+ void ptr = (void )head - offset;
	3262	+
	3263	+ next = head->next;
	3264	+ debug_rcu_head_unqueue((struct rcu_head *)ptr);
	3265	+ rcu_lock_acquire(&rcu_callback_map);
	3266	+ trace_rcu_invoke_kvfree_callback(rcu_state.name, head, offset);
	3267	+
	3268	+ if (!WARN_ON_ONCE(!__is_kvfree_rcu_offset(offset)))
	3269	+ kvfree(ptr);
	3270	+
	3271	+ rcu_lock_release(&rcu_callback_map);
	3272	+ cond_resched_tasks_rcu_qs();
	3273	+ }
3224	3274	}
3225		-EXPORT_SYMBOL_GPL(kfree_call_rcu);
3226	3275
3227	3276	/*
3228		- * Because a context switch is a grace period for RCU-sched and RCU-bh,
3229		- * any blocking grace-period wait automatically implies a grace period
3230		- * if there is only one CPU online at any point time during execution
3231		- * of either synchronize_sched() or synchronize_rcu_bh(). It is OK to
	3277	+ * Schedule the kfree batch RCU work to run in workqueue context after a GP.
	3278	+ *
	3279	+ * This function is invoked by kfree_rcu_monitor() when the KFREE_DRAIN_JIFFIES
	3280	+ * timeout has been reached.
	3281	+ */
	3282	+static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp)
	3283	+{
	3284	+ struct kfree_rcu_cpu_work *krwp;
	3285	+ bool repeat = false;
	3286	+ int i, j;
	3287	+
	3288	+ lockdep_assert_held(&krcp->lock);
	3289	+
	3290	+ for (i = 0; i < KFREE_N_BATCHES; i++) {
	3291	+ krwp = &(krcp->krw_arr[i]);
	3292	+
	3293	+ /*
	3294	+ * Try to detach bkvhead or head and attach it over any
	3295	+ * available corresponding free channel. It can be that
	3296	+ * a previous RCU batch is in progress, it means that
	3297	+ * immediately to queue another one is not possible so
	3298	+ * return false to tell caller to retry.
	3299	+ */
	3300	+ if ((krcp->bkvhead[0] && !krwp->bkvhead_free[0]) \|\|
	3301	+ (krcp->bkvhead[1] && !krwp->bkvhead_free[1]) \|\|
	3302	+ (krcp->head && !krwp->head_free)) {
	3303	+ // Channel 1 corresponds to SLAB ptrs.
	3304	+ // Channel 2 corresponds to vmalloc ptrs.
	3305	+ for (j = 0; j < FREE_N_CHANNELS; j++) {
	3306	+ if (!krwp->bkvhead_free[j]) {
	3307	+ krwp->bkvhead_free[j] = krcp->bkvhead[j];
	3308	+ krcp->bkvhead[j] = NULL;
	3309	+ }
	3310	+ }
	3311	+
	3312	+ // Channel 3 corresponds to emergency path.
	3313	+ if (!krwp->head_free) {
	3314	+ krwp->head_free = krcp->head;
	3315	+ krcp->head = NULL;
	3316	+ }
	3317	+
	3318	+ WRITE_ONCE(krcp->count, 0);
	3319	+
	3320	+ /*
	3321	+ * One work is per one batch, so there are three
	3322	+ * "free channels", the batch can handle. It can
	3323	+ * be that the work is in the pending state when
	3324	+ * channels have been detached following by each
	3325	+ * other.
	3326	+ */
	3327	+ queue_rcu_work(system_wq, &krwp->rcu_work);
	3328	+ }
	3329	+
	3330	+ // Repeat if any "free" corresponding channel is still busy.
	3331	+ if (krcp->bkvhead[0] \|\| krcp->bkvhead[1] \|\| krcp->head)
	3332	+ repeat = true;
	3333	+ }
	3334	+
	3335	+ return !repeat;
	3336	+}
	3337	+
	3338	+static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu *krcp,
	3339	+ unsigned long flags)
	3340	+{
	3341	+ // Attempt to start a new batch.
	3342	+ krcp->monitor_todo = false;
	3343	+ if (queue_kfree_rcu_work(krcp)) {
	3344	+ // Success! Our job is done here.
	3345	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3346	+ return;
	3347	+ }
	3348	+
	3349	+ // Previous RCU batch still in progress, try again later.
	3350	+ krcp->monitor_todo = true;
	3351	+ schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
	3352	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3353	+}
	3354	+
	3355	+/*
	3356	+ * This function is invoked after the KFREE_DRAIN_JIFFIES timeout.
	3357	+ * It invokes kfree_rcu_drain_unlock() to attempt to start another batch.
	3358	+ */
	3359	+static void kfree_rcu_monitor(struct work_struct *work)
	3360	+{
	3361	+ unsigned long flags;
	3362	+ struct kfree_rcu_cpu *krcp = container_of(work, struct kfree_rcu_cpu,
	3363	+ monitor_work.work);
	3364	+
	3365	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3366	+ if (krcp->monitor_todo)
	3367	+ kfree_rcu_drain_unlock(krcp, flags);
	3368	+ else
	3369	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3370	+}
	3371	+
	3372	+static enum hrtimer_restart
	3373	+schedule_page_work_fn(struct hrtimer *t)
	3374	+{
	3375	+ struct kfree_rcu_cpu *krcp =
	3376	+ container_of(t, struct kfree_rcu_cpu, hrtimer);
	3377	+
	3378	+ queue_work(system_highpri_wq, &krcp->page_cache_work);
	3379	+ return HRTIMER_NORESTART;
	3380	+}
	3381	+
	3382	+static void fill_page_cache_func(struct work_struct *work)
	3383	+{
	3384	+ struct kvfree_rcu_bulk_data *bnode;
	3385	+ struct kfree_rcu_cpu *krcp =
	3386	+ container_of(work, struct kfree_rcu_cpu,
	3387	+ page_cache_work);
	3388	+ unsigned long flags;
	3389	+ bool pushed;
	3390	+ int i;
	3391	+
	3392	+ for (i = 0; i < rcu_min_cached_objs; i++) {
	3393	+ bnode = (struct kvfree_rcu_bulk_data *)
	3394	+ __get_free_page(GFP_KERNEL \| __GFP_NORETRY \| __GFP_NOMEMALLOC \| __GFP_NOWARN);
	3395	+
	3396	+ if (!bnode)
	3397	+ break;
	3398	+
	3399	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3400	+ pushed = put_cached_bnode(krcp, bnode);
	3401	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3402	+
	3403	+ if (!pushed) {
	3404	+ free_page((unsigned long) bnode);
	3405	+ break;
	3406	+ }
	3407	+ }
	3408	+
	3409	+ atomic_set(&krcp->work_in_progress, 0);
	3410	+}
	3411	+
	3412	+static void
	3413	+run_page_cache_worker(struct kfree_rcu_cpu *krcp)
	3414	+{
	3415	+ if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
	3416	+ !atomic_xchg(&krcp->work_in_progress, 1)) {
	3417	+ hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC,
	3418	+ HRTIMER_MODE_REL);
	3419	+ krcp->hrtimer.function = schedule_page_work_fn;
	3420	+ hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
	3421	+ }
	3422	+}
	3423	+
	3424	+static inline bool
	3425	+kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu krcp, void ptr)
	3426	+{
	3427	+ struct kvfree_rcu_bulk_data *bnode;
	3428	+ int idx;
	3429	+
	3430	+ if (unlikely(!krcp->initialized))
	3431	+ return false;
	3432	+
	3433	+ lockdep_assert_held(&krcp->lock);
	3434	+ idx = !!is_vmalloc_addr(ptr);
	3435	+
	3436	+ /* Check if a new block is required. */
	3437	+ if (!krcp->bkvhead[idx] \|\|
	3438	+ krcp->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
	3439	+ bnode = get_cached_bnode(krcp);
	3440	+ /* Switch to emergency path. */
	3441	+ if (!bnode)
	3442	+ return false;
	3443	+
	3444	+ /* Initialize the new block. */
	3445	+ bnode->nr_records = 0;
	3446	+ bnode->next = krcp->bkvhead[idx];
	3447	+
	3448	+ /* Attach it to the head. */
	3449	+ krcp->bkvhead[idx] = bnode;
	3450	+ }
	3451	+
	3452	+ /* Finally insert. */
	3453	+ krcp->bkvhead[idx]->records
	3454	+ [krcp->bkvhead[idx]->nr_records++] = ptr;
	3455	+
	3456	+ return true;
	3457	+}
	3458	+
	3459	+/*
	3460	+ * Queue a request for lazy invocation of appropriate free routine after a
	3461	+ * grace period. Please note there are three paths are maintained, two are the
	3462	+ * main ones that use array of pointers interface and third one is emergency
	3463	+ * one, that is used only when the main path can not be maintained temporary,
	3464	+ * due to memory pressure.
	3465	+ *
	3466	+ * Each kvfree_call_rcu() request is added to a batch. The batch will be drained
	3467	+ * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch will
	3468	+ * be free'd in workqueue context. This allows us to: batch requests together to
	3469	+ * reduce the number of grace periods during heavy kfree_rcu()/kvfree_rcu() load.
	3470	+ */
	3471	+void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
	3472	+{
	3473	+ unsigned long flags;
	3474	+ struct kfree_rcu_cpu *krcp;
	3475	+ bool success;
	3476	+ void *ptr;
	3477	+
	3478	+ if (head) {
	3479	+ ptr = (void *) head - (unsigned long) func;
	3480	+ } else {
	3481	+ /*
	3482	+ * Please note there is a limitation for the head-less
	3483	+ * variant, that is why there is a clear rule for such
	3484	+ * objects: it can be used from might_sleep() context
	3485	+ * only. For other places please embed an rcu_head to
	3486	+ * your data.
	3487	+ */
	3488	+ might_sleep();
	3489	+ ptr = (unsigned long *) func;
	3490	+ }
	3491	+
	3492	+ krcp = krc_this_cpu_lock(&flags);
	3493	+
	3494	+ // Queue the object but don't yet schedule the batch.
	3495	+ if (debug_rcu_head_queue(ptr)) {
	3496	+ // Probable double kfree_rcu(), just leak.
	3497	+ WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n",
	3498	+ __func__, head);
	3499	+
	3500	+ // Mark as success and leave.
	3501	+ success = true;
	3502	+ goto unlock_return;
	3503	+ }
	3504	+
	3505	+ success = kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr);
	3506	+ if (!success) {
	3507	+ run_page_cache_worker(krcp);
	3508	+
	3509	+ if (head == NULL)
	3510	+ // Inline if kvfree_rcu(one_arg) call.
	3511	+ goto unlock_return;
	3512	+
	3513	+ head->func = func;
	3514	+ head->next = krcp->head;
	3515	+ krcp->head = head;
	3516	+ success = true;
	3517	+ }
	3518	+
	3519	+ WRITE_ONCE(krcp->count, krcp->count + 1);
	3520	+
	3521	+ // Set timer to drain after KFREE_DRAIN_JIFFIES.
	3522	+ if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
	3523	+ !krcp->monitor_todo) {
	3524	+ krcp->monitor_todo = true;
	3525	+ schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
	3526	+ }
	3527	+
	3528	+unlock_return:
	3529	+ krc_this_cpu_unlock(krcp, flags);
	3530	+
	3531	+ /*
	3532	+ * Inline kvfree() after synchronize_rcu(). We can do
	3533	+ * it from might_sleep() context only, so the current
	3534	+ * CPU can pass the QS state.
	3535	+ */
	3536	+ if (!success) {
	3537	+ debug_rcu_head_unqueue((struct rcu_head *) ptr);
	3538	+ synchronize_rcu();
	3539	+ kvfree(ptr);
	3540	+ }
	3541	+}
	3542	+EXPORT_SYMBOL_GPL(kvfree_call_rcu);
	3543	+
	3544	+static unsigned long
	3545	+kfree_rcu_shrink_count(struct shrinker shrink, struct shrink_control sc)
	3546	+{
	3547	+ int cpu;
	3548	+ unsigned long count = 0;
	3549	+
	3550	+ /* Snapshot count of all CPUs */
	3551	+ for_each_possible_cpu(cpu) {
	3552	+ struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
	3553	+
	3554	+ count += READ_ONCE(krcp->count);
	3555	+ }
	3556	+
	3557	+ return count;
	3558	+}
	3559	+
	3560	+static unsigned long
	3561	+kfree_rcu_shrink_scan(struct shrinker shrink, struct shrink_control sc)
	3562	+{
	3563	+ int cpu, freed = 0;
	3564	+ unsigned long flags;
	3565	+
	3566	+ for_each_possible_cpu(cpu) {
	3567	+ int count;
	3568	+ struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
	3569	+
	3570	+ count = krcp->count;
	3571	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3572	+ if (krcp->monitor_todo)
	3573	+ kfree_rcu_drain_unlock(krcp, flags);
	3574	+ else
	3575	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3576	+
	3577	+ sc->nr_to_scan -= count;
	3578	+ freed += count;
	3579	+
	3580	+ if (sc->nr_to_scan <= 0)
	3581	+ break;
	3582	+ }
	3583	+
	3584	+ return freed == 0 ? SHRINK_STOP : freed;
	3585	+}
	3586	+
	3587	+static struct shrinker kfree_rcu_shrinker = {
	3588	+ .count_objects = kfree_rcu_shrink_count,
	3589	+ .scan_objects = kfree_rcu_shrink_scan,
	3590	+ .batch = 0,
	3591	+ .seeks = DEFAULT_SEEKS,
	3592	+};
	3593	+
	3594	+void __init kfree_rcu_scheduler_running(void)
	3595	+{
	3596	+ int cpu;
	3597	+ unsigned long flags;
	3598	+
	3599	+ for_each_possible_cpu(cpu) {
	3600	+ struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
	3601	+
	3602	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3603	+ if (!krcp->head \|\| krcp->monitor_todo) {
	3604	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3605	+ continue;
	3606	+ }
	3607	+ krcp->monitor_todo = true;
	3608	+ schedule_delayed_work_on(cpu, &krcp->monitor_work,
	3609	+ KFREE_DRAIN_JIFFIES);
	3610	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3611	+ }
	3612	+}
	3613	+
	3614	+/*
	3615	+ * During early boot, any blocking grace-period wait automatically
	3616	+ * implies a grace period. Later on, this is never the case for PREEMPTION.
	3617	+ *
	3618	+ * Howevr, because a context switch is a grace period for !PREEMPTION, any
	3619	+ * blocking grace-period wait automatically implies a grace period if
	3620	+ * there is only one CPU online at any point time during execution of
	3621	+ * either synchronize_rcu() or synchronize_rcu_expedited(). It is OK to
3232	3622	* occasionally incorrectly indicate that there are multiple CPUs online
3233		- * when there was in fact only one the whole time, as this just adds
3234		- * some overhead: RCU still operates correctly.
	3623	+ * when there was in fact only one the whole time, as this just adds some
	3624	+ * overhead: RCU still operates correctly.
3235	3625	*/
3236	3626	static int rcu_blocking_is_gp(void)
3237	3627	{
3238	3628	int ret;
3239	3629
	3630	+ if (IS_ENABLED(CONFIG_PREEMPTION))
	3631	+ return rcu_scheduler_active == RCU_SCHEDULER_INACTIVE;
3240	3632	might_sleep(); /* Check for RCU read-side critical section. */
3241	3633	preempt_disable();
3242	3634	ret = num_online_cpus() <= 1;
..	..	@@ -3245,83 +3637,52 @@
3245	3637	}
3246	3638
3247	3639	/**
3248		- * synchronize_sched - wait until an rcu-sched grace period has elapsed.
	3640	+ * synchronize_rcu - wait until a grace period has elapsed.
3249	3641	*
3250		- * Control will return to the caller some time after a full rcu-sched
3251		- * grace period has elapsed, in other words after all currently executing
3252		- * rcu-sched read-side critical sections have completed. These read-side
3253		- * critical sections are delimited by rcu_read_lock_sched() and
3254		- * rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(),
3255		- * local_irq_disable(), and so on may be used in place of
3256		- * rcu_read_lock_sched().
3257		- *
3258		- * This means that all preempt_disable code sequences, including NMI and
3259		- * non-threaded hardware-interrupt handlers, in progress on entry will
3260		- * have completed before this primitive returns. However, this does not
3261		- * guarantee that softirq handlers will have completed, since in some
3262		- * kernels, these handlers can run in process context, and can block.
	3642	+ * Control will return to the caller some time after a full grace
	3643	+ * period has elapsed, in other words after all currently executing RCU
	3644	+ * read-side critical sections have completed. Note, however, that
	3645	+ * upon return from synchronize_rcu(), the caller might well be executing
	3646	+ * concurrently with new RCU read-side critical sections that began while
	3647	+ * synchronize_rcu() was waiting. RCU read-side critical sections are
	3648	+ * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
	3649	+ * In addition, regions of code across which interrupts, preemption, or
	3650	+ * softirqs have been disabled also serve as RCU read-side critical
	3651	+ * sections. This includes hardware interrupt handlers, softirq handlers,
	3652	+ * and NMI handlers.
3263	3653	*
3264	3654	* Note that this guarantee implies further memory-ordering guarantees.
3265		- * On systems with more than one CPU, when synchronize_sched() returns,
3266		- * each CPU is guaranteed to have executed a full memory barrier since the
3267		- * end of its last RCU-sched read-side critical section whose beginning
3268		- * preceded the call to synchronize_sched(). In addition, each CPU having
	3655	+ * On systems with more than one CPU, when synchronize_rcu() returns,
	3656	+ * each CPU is guaranteed to have executed a full memory barrier since
	3657	+ * the end of its last RCU read-side critical section whose beginning
	3658	+ * preceded the call to synchronize_rcu(). In addition, each CPU having
3269	3659	* an RCU read-side critical section that extends beyond the return from
3270		- * synchronize_sched() is guaranteed to have executed a full memory barrier
3271		- * after the beginning of synchronize_sched() and before the beginning of
	3660	+ * synchronize_rcu() is guaranteed to have executed a full memory barrier
	3661	+ * after the beginning of synchronize_rcu() and before the beginning of
3272	3662	* that RCU read-side critical section. Note that these guarantees include
3273	3663	* CPUs that are offline, idle, or executing in user mode, as well as CPUs
3274	3664	* that are executing in the kernel.
3275	3665	*
3276		- * Furthermore, if CPU A invoked synchronize_sched(), which returned
	3666	+ * Furthermore, if CPU A invoked synchronize_rcu(), which returned
3277	3667	* to its caller on CPU B, then both CPU A and CPU B are guaranteed
3278	3668	* to have executed a full memory barrier during the execution of
3279		- * synchronize_sched() -- even if CPU A and CPU B are the same CPU (but
	3669	+ * synchronize_rcu() -- even if CPU A and CPU B are the same CPU (but
3280	3670	* again only if the system has more than one CPU).
3281	3671	*/
3282		-void synchronize_sched(void)
	3672	+void synchronize_rcu(void)
3283	3673	{
3284	3674	RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) \|\|
3285	3675	lock_is_held(&rcu_lock_map) \|\|
3286	3676	lock_is_held(&rcu_sched_lock_map),
3287		- "Illegal synchronize_sched() in RCU-sched read-side critical section");
	3677	+ "Illegal synchronize_rcu() in RCU read-side critical section");
3288	3678	if (rcu_blocking_is_gp())
3289	3679	return;
3290	3680	if (rcu_gp_is_expedited())
3291		- synchronize_sched_expedited();
	3681	+ synchronize_rcu_expedited();
3292	3682	else
3293		- wait_rcu_gp(call_rcu_sched);
	3683	+ wait_rcu_gp(call_rcu);
3294	3684	}
3295		-EXPORT_SYMBOL_GPL(synchronize_sched);
3296		-
3297		-#ifndef CONFIG_PREEMPT_RT_FULL
3298		-/**
3299		- * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed.
3300		- *
3301		- * Control will return to the caller some time after a full rcu_bh grace
3302		- * period has elapsed, in other words after all currently executing rcu_bh
3303		- * read-side critical sections have completed. RCU read-side critical
3304		- * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(),
3305		- * and may be nested.
3306		- *
3307		- * See the description of synchronize_sched() for more detailed information
3308		- * on memory ordering guarantees.
3309		- */
3310		-void synchronize_rcu_bh(void)
3311		-{
3312		- RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) \|\|
3313		- lock_is_held(&rcu_lock_map) \|\|
3314		- lock_is_held(&rcu_sched_lock_map),
3315		- "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
3316		- if (rcu_blocking_is_gp())
3317		- return;
3318		- if (rcu_gp_is_expedited())
3319		- synchronize_rcu_bh_expedited();
3320		- else
3321		- wait_rcu_gp(call_rcu_bh);
3322		-}
3323		-EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
3324		-#endif
	3685	+EXPORT_SYMBOL_GPL(synchronize_rcu);
3325	3686
3326	3687	/**
3327	3688	* get_state_synchronize_rcu - Snapshot current RCU state
..	..	@@ -3337,7 +3698,7 @@
3337	3698	* before the load from ->gp_seq.
3338	3699	*/
3339	3700	smp_mb(); /* ^^^ */
3340		- return rcu_seq_snap(&rcu_state_p->gp_seq);
	3701	+ return rcu_seq_snap(&rcu_state.gp_seq);
3341	3702	}
3342	3703	EXPORT_SYMBOL_GPL(get_state_synchronize_rcu);
3343	3704
..	..	@@ -3357,74 +3718,42 @@
3357	3718	*/
3358	3719	void cond_synchronize_rcu(unsigned long oldstate)
3359	3720	{
3360		- if (!rcu_seq_done(&rcu_state_p->gp_seq, oldstate))
	3721	+ if (!rcu_seq_done(&rcu_state.gp_seq, oldstate))
3361	3722	synchronize_rcu();
3362	3723	else
3363	3724	smp_mb(); /* Ensure GP ends before subsequent accesses. */
3364	3725	}
3365	3726	EXPORT_SYMBOL_GPL(cond_synchronize_rcu);
3366	3727
3367		-/**
3368		- * get_state_synchronize_sched - Snapshot current RCU-sched state
3369		- *
3370		- * Returns a cookie that is used by a later call to cond_synchronize_sched()
3371		- * to determine whether or not a full grace period has elapsed in the
3372		- * meantime.
3373		- */
3374		-unsigned long get_state_synchronize_sched(void)
3375		-{
3376		- /*
3377		- * Any prior manipulation of RCU-protected data must happen
3378		- * before the load from ->gp_seq.
3379		- */
3380		- smp_mb(); /* ^^^ */
3381		- return rcu_seq_snap(&rcu_sched_state.gp_seq);
3382		-}
3383		-EXPORT_SYMBOL_GPL(get_state_synchronize_sched);
3384		-
3385		-/**
3386		- * cond_synchronize_sched - Conditionally wait for an RCU-sched grace period
3387		- *
3388		- * @oldstate: return value from earlier call to get_state_synchronize_sched()
3389		- *
3390		- * If a full RCU-sched grace period has elapsed since the earlier call to
3391		- * get_state_synchronize_sched(), just return. Otherwise, invoke
3392		- * synchronize_sched() to wait for a full grace period.
3393		- *
3394		- * Yes, this function does not take counter wrap into account. But
3395		- * counter wrap is harmless. If the counter wraps, we have waited for
3396		- * more than 2 billion grace periods (and way more on a 64-bit system!),
3397		- * so waiting for one additional grace period should be just fine.
3398		- */
3399		-void cond_synchronize_sched(unsigned long oldstate)
3400		-{
3401		- if (!rcu_seq_done(&rcu_sched_state.gp_seq, oldstate))
3402		- synchronize_sched();
3403		- else
3404		- smp_mb(); /* Ensure GP ends before subsequent accesses. */
3405		-}
3406		-EXPORT_SYMBOL_GPL(cond_synchronize_sched);
3407		-
3408	3728	/*
3409		- * Check to see if there is any immediate RCU-related work to be done
3410		- * by the current CPU, for the specified type of RCU, returning 1 if so.
3411		- * The checks are in order of increasing expense: checks that can be
3412		- * carried out against CPU-local state are performed first. However,
3413		- * we must check for CPU stalls first, else we might not get a chance.
	3729	+ * Check to see if there is any immediate RCU-related work to be done by
	3730	+ * the current CPU, returning 1 if so and zero otherwise. The checks are
	3731	+ * in order of increasing expense: checks that can be carried out against
	3732	+ * CPU-local state are performed first. However, we must check for CPU
	3733	+ * stalls first, else we might not get a chance.
3414	3734	*/
3415		-static int __rcu_pending(struct rcu_state rsp, struct rcu_data rdp)
	3735	+static int rcu_pending(int user)
3416	3736	{
	3737	+ bool gp_in_progress;
	3738	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
3417	3739	struct rcu_node *rnp = rdp->mynode;
3418	3740
3419		- /* Check for CPU stalls, if enabled. */
3420		- check_cpu_stall(rsp, rdp);
	3741	+ lockdep_assert_irqs_disabled();
3421	3742
3422		- /* Is this CPU a NO_HZ_FULL CPU that should ignore RCU? */
3423		- if (rcu_nohz_full_cpu(rsp))
	3743	+ /* Check for CPU stalls, if enabled. */
	3744	+ check_cpu_stall(rdp);
	3745	+
	3746	+ /* Does this CPU need a deferred NOCB wakeup? */
	3747	+ if (rcu_nocb_need_deferred_wakeup(rdp))
	3748	+ return 1;
	3749	+
	3750	+ /* Is this a nohz_full CPU in userspace or idle? (Ignore RCU if so.) */
	3751	+ if ((user \|\| rcu_is_cpu_rrupt_from_idle()) && rcu_nohz_full_cpu())
3424	3752	return 0;
3425	3753
3426	3754	/* Is the RCU core waiting for a quiescent state from this CPU? */
3427		- if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm)
	3755	+ gp_in_progress = rcu_gp_in_progress();
	3756	+ if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm && gp_in_progress)
3428	3757	return 1;
3429	3758
3430	3759	/* Does this CPU have callbacks ready to invoke? */
..	..	@@ -3432,8 +3761,9 @@
3432	3761	return 1;
3433	3762
3434	3763	/* Has RCU gone idle with this CPU needing another grace period? */
3435		- if (!rcu_gp_in_progress(rsp) &&
3436		- rcu_segcblist_is_enabled(&rdp->cblist) &&
	3764	+ if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) &&
	3765	+ (!IS_ENABLED(CONFIG_RCU_NOCB_CPU) \|\|
	3766	+ !rcu_segcblist_is_offloaded(&rdp->cblist)) &&
3437	3767	!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
3438	3768	return 1;
3439	3769
..	..	@@ -3442,141 +3772,106 @@
3442	3772	unlikely(READ_ONCE(rdp->gpwrap))) /* outside lock */
3443	3773	return 1;
3444	3774
3445		- /* Does this CPU need a deferred NOCB wakeup? */
3446		- if (rcu_nocb_need_deferred_wakeup(rdp))
3447		- return 1;
3448		-
3449	3775	/* nothing to do */
3450	3776	return 0;
3451	3777	}
3452	3778
3453	3779	/*
3454		- * Check to see if there is any immediate RCU-related work to be done
3455		- * by the current CPU, returning 1 if so. This function is part of the
3456		- * RCU implementation; it is -not- an exported member of the RCU API.
3457		- */
3458		-static int rcu_pending(void)
3459		-{
3460		- struct rcu_state *rsp;
3461		-
3462		- for_each_rcu_flavor(rsp)
3463		- if (__rcu_pending(rsp, this_cpu_ptr(rsp->rda)))
3464		- return 1;
3465		- return 0;
3466		-}
3467		-
3468		-/*
3469		- * Return true if the specified CPU has any callback. If all_lazy is
3470		- * non-NULL, store an indication of whether all callbacks are lazy.
3471		- * (If there are no callbacks, all of them are deemed to be lazy.)
3472		- */
3473		-static bool rcu_cpu_has_callbacks(bool *all_lazy)
3474		-{
3475		- bool al = true;
3476		- bool hc = false;
3477		- struct rcu_data *rdp;
3478		- struct rcu_state *rsp;
3479		-
3480		- for_each_rcu_flavor(rsp) {
3481		- rdp = this_cpu_ptr(rsp->rda);
3482		- if (rcu_segcblist_empty(&rdp->cblist))
3483		- continue;
3484		- hc = true;
3485		- if (rcu_segcblist_n_nonlazy_cbs(&rdp->cblist) \|\| !all_lazy) {
3486		- al = false;
3487		- break;
3488		- }
3489		- }
3490		- if (all_lazy)
3491		- *all_lazy = al;
3492		- return hc;
3493		-}
3494		-
3495		-/*
3496		- * Helper function for _rcu_barrier() tracing. If tracing is disabled,
	3780	+ * Helper function for rcu_barrier() tracing. If tracing is disabled,
3497	3781	* the compiler is expected to optimize this away.
3498	3782	*/
3499		-static void _rcu_barrier_trace(struct rcu_state rsp, const char s,
3500		- int cpu, unsigned long done)
	3783	+static void rcu_barrier_trace(const char *s, int cpu, unsigned long done)
3501	3784	{
3502		- trace_rcu_barrier(rsp->name, s, cpu,
3503		- atomic_read(&rsp->barrier_cpu_count), done);
	3785	+ trace_rcu_barrier(rcu_state.name, s, cpu,
	3786	+ atomic_read(&rcu_state.barrier_cpu_count), done);
3504	3787	}
3505	3788
3506	3789	/*
3507		- * RCU callback function for _rcu_barrier(). If we are last, wake
3508		- * up the task executing _rcu_barrier().
	3790	+ * RCU callback function for rcu_barrier(). If we are last, wake
	3791	+ * up the task executing rcu_barrier().
	3792	+ *
	3793	+ * Note that the value of rcu_state.barrier_sequence must be captured
	3794	+ * before the atomic_dec_and_test(). Otherwise, if this CPU is not last,
	3795	+ * other CPUs might count the value down to zero before this CPU gets
	3796	+ * around to invoking rcu_barrier_trace(), which might result in bogus
	3797	+ * data from the next instance of rcu_barrier().
3509	3798	*/
3510	3799	static void rcu_barrier_callback(struct rcu_head *rhp)
3511	3800	{
3512		- struct rcu_data *rdp = container_of(rhp, struct rcu_data, barrier_head);
3513		- struct rcu_state *rsp = rdp->rsp;
	3801	+ unsigned long __maybe_unused s = rcu_state.barrier_sequence;
3514	3802
3515		- if (atomic_dec_and_test(&rsp->barrier_cpu_count)) {
3516		- _rcu_barrier_trace(rsp, TPS("LastCB"), -1,
3517		- rsp->barrier_sequence);
3518		- complete(&rsp->barrier_completion);
	3803	+ if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {
	3804	+ rcu_barrier_trace(TPS("LastCB"), -1, s);
	3805	+ complete(&rcu_state.barrier_completion);
3519	3806	} else {
3520		- _rcu_barrier_trace(rsp, TPS("CB"), -1, rsp->barrier_sequence);
	3807	+ rcu_barrier_trace(TPS("CB"), -1, s);
3521	3808	}
3522	3809	}
3523	3810
3524	3811	/*
3525	3812	* Called with preemption disabled, and from cross-cpu IRQ context.
3526	3813	*/
3527		-static void rcu_barrier_func(void *type)
	3814	+static void rcu_barrier_func(void *cpu_in)
3528	3815	{
3529		- struct rcu_state *rsp = type;
3530		- struct rcu_data *rdp = raw_cpu_ptr(rsp->rda);
	3816	+ uintptr_t cpu = (uintptr_t)cpu_in;
	3817	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3531	3818
3532		- _rcu_barrier_trace(rsp, TPS("IRQ"), -1, rsp->barrier_sequence);
	3819	+ rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);
3533	3820	rdp->barrier_head.func = rcu_barrier_callback;
3534	3821	debug_rcu_head_queue(&rdp->barrier_head);
3535		- if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head, 0)) {
3536		- atomic_inc(&rsp->barrier_cpu_count);
	3822	+ rcu_nocb_lock(rdp);
	3823	+ WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));
	3824	+ if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) {
	3825	+ atomic_inc(&rcu_state.barrier_cpu_count);
3537	3826	} else {
3538	3827	debug_rcu_head_unqueue(&rdp->barrier_head);
3539		- _rcu_barrier_trace(rsp, TPS("IRQNQ"), -1,
3540		- rsp->barrier_sequence);
	3828	+ rcu_barrier_trace(TPS("IRQNQ"), -1,
	3829	+ rcu_state.barrier_sequence);
3541	3830	}
	3831	+ rcu_nocb_unlock(rdp);
3542	3832	}
3543	3833
3544		-/*
3545		- * Orchestrate the specified type of RCU barrier, waiting for all
3546		- * RCU callbacks of the specified type to complete.
	3834	+/**
	3835	+ * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
	3836	+ *
	3837	+ * Note that this primitive does not necessarily wait for an RCU grace period
	3838	+ * to complete. For example, if there are no RCU callbacks queued anywhere
	3839	+ * in the system, then rcu_barrier() is within its rights to return
	3840	+ * immediately, without waiting for anything, much less an RCU grace period.
3547	3841	*/
3548		-static void _rcu_barrier(struct rcu_state *rsp)
	3842	+void rcu_barrier(void)
3549	3843	{
3550		- int cpu;
	3844	+ uintptr_t cpu;
3551	3845	struct rcu_data *rdp;
3552		- unsigned long s = rcu_seq_snap(&rsp->barrier_sequence);
	3846	+ unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);
3553	3847
3554		- _rcu_barrier_trace(rsp, TPS("Begin"), -1, s);
	3848	+ rcu_barrier_trace(TPS("Begin"), -1, s);
3555	3849
3556	3850	/* Take mutex to serialize concurrent rcu_barrier() requests. */
3557		- mutex_lock(&rsp->barrier_mutex);
	3851	+ mutex_lock(&rcu_state.barrier_mutex);
3558	3852
3559	3853	/* Did someone else do our work for us? */
3560		- if (rcu_seq_done(&rsp->barrier_sequence, s)) {
3561		- _rcu_barrier_trace(rsp, TPS("EarlyExit"), -1,
3562		- rsp->barrier_sequence);
	3854	+ if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {
	3855	+ rcu_barrier_trace(TPS("EarlyExit"), -1,
	3856	+ rcu_state.barrier_sequence);
3563	3857	smp_mb(); /* caller's subsequent code after above check. */
3564		- mutex_unlock(&rsp->barrier_mutex);
	3858	+ mutex_unlock(&rcu_state.barrier_mutex);
3565	3859	return;
3566	3860	}
3567	3861
3568	3862	/* Mark the start of the barrier operation. */
3569		- rcu_seq_start(&rsp->barrier_sequence);
3570		- _rcu_barrier_trace(rsp, TPS("Inc1"), -1, rsp->barrier_sequence);
	3863	+ rcu_seq_start(&rcu_state.barrier_sequence);
	3864	+ rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);
3571	3865
3572	3866	/*
3573		- * Initialize the count to one rather than to zero in order to
3574		- * avoid a too-soon return to zero in case of a short grace period
3575		- * (or preemption of this task). Exclude CPU-hotplug operations
3576		- * to ensure that no offline CPU has callbacks queued.
	3867	+ * Initialize the count to two rather than to zero in order
	3868	+ * to avoid a too-soon return to zero in case of an immediate
	3869	+ * invocation of the just-enqueued callback (or preemption of
	3870	+ * this task). Exclude CPU-hotplug operations to ensure that no
	3871	+ * offline non-offloaded CPU has callbacks queued.
3577	3872	*/
3578		- init_completion(&rsp->barrier_completion);
3579		- atomic_set(&rsp->barrier_cpu_count, 1);
	3873	+ init_completion(&rcu_state.barrier_completion);
	3874	+ atomic_set(&rcu_state.barrier_cpu_count, 2);
3580	3875	get_online_cpus();
3581	3876
3582	3877	/*
..	..	@@ -3585,28 +3880,27 @@
3585	3880	* corresponding CPU's preceding callbacks have been invoked.
3586	3881	*/
3587	3882	for_each_possible_cpu(cpu) {
3588		- if (!cpu_online(cpu) && !rcu_is_nocb_cpu(cpu))
	3883	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	3884	+ if (cpu_is_offline(cpu) &&
	3885	+ !rcu_segcblist_is_offloaded(&rdp->cblist))
3589	3886	continue;
3590		- rdp = per_cpu_ptr(rsp->rda, cpu);
3591		- if (rcu_is_nocb_cpu(cpu)) {
3592		- if (!rcu_nocb_cpu_needs_barrier(rsp, cpu)) {
3593		- _rcu_barrier_trace(rsp, TPS("OfflineNoCB"), cpu,
3594		- rsp->barrier_sequence);
3595		- } else {
3596		- _rcu_barrier_trace(rsp, TPS("OnlineNoCB"), cpu,
3597		- rsp->barrier_sequence);
3598		- smp_mb__before_atomic();
3599		- atomic_inc(&rsp->barrier_cpu_count);
3600		- __call_rcu(&rdp->barrier_head,
3601		- rcu_barrier_callback, rsp, cpu, 0);
3602		- }
3603		- } else if (rcu_segcblist_n_cbs(&rdp->cblist)) {
3604		- _rcu_barrier_trace(rsp, TPS("OnlineQ"), cpu,
3605		- rsp->barrier_sequence);
3606		- smp_call_function_single(cpu, rcu_barrier_func, rsp, 1);
	3887	+ if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) {
	3888	+ rcu_barrier_trace(TPS("OnlineQ"), cpu,
	3889	+ rcu_state.barrier_sequence);
	3890	+ smp_call_function_single(cpu, rcu_barrier_func, (void *)cpu, 1);
	3891	+ } else if (rcu_segcblist_n_cbs(&rdp->cblist) &&
	3892	+ cpu_is_offline(cpu)) {
	3893	+ rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu,
	3894	+ rcu_state.barrier_sequence);
	3895	+ local_irq_disable();
	3896	+ rcu_barrier_func((void *)cpu);
	3897	+ local_irq_enable();
	3898	+ } else if (cpu_is_offline(cpu)) {
	3899	+ rcu_barrier_trace(TPS("OfflineNoCBNoQ"), cpu,
	3900	+ rcu_state.barrier_sequence);
3607	3901	} else {
3608		- _rcu_barrier_trace(rsp, TPS("OnlineNQ"), cpu,
3609		- rsp->barrier_sequence);
	3902	+ rcu_barrier_trace(TPS("OnlineNQ"), cpu,
	3903	+ rcu_state.barrier_sequence);
3610	3904	}
3611	3905	}
3612	3906	put_online_cpus();
..	..	@@ -3615,39 +3909,20 @@
3615	3909	* Now that we have an rcu_barrier_callback() callback on each
3616	3910	* CPU, and thus each counted, remove the initial count.
3617	3911	*/
3618		- if (atomic_dec_and_test(&rsp->barrier_cpu_count))
3619		- complete(&rsp->barrier_completion);
	3912	+ if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count))
	3913	+ complete(&rcu_state.barrier_completion);
3620	3914
3621	3915	/* Wait for all rcu_barrier_callback() callbacks to be invoked. */
3622		- wait_for_completion(&rsp->barrier_completion);
	3916	+ wait_for_completion(&rcu_state.barrier_completion);
3623	3917
3624	3918	/* Mark the end of the barrier operation. */
3625		- _rcu_barrier_trace(rsp, TPS("Inc2"), -1, rsp->barrier_sequence);
3626		- rcu_seq_end(&rsp->barrier_sequence);
	3919	+ rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence);
	3920	+ rcu_seq_end(&rcu_state.barrier_sequence);
3627	3921
3628	3922	/* Other rcu_barrier() invocations can now safely proceed. */
3629		- mutex_unlock(&rsp->barrier_mutex);
	3923	+ mutex_unlock(&rcu_state.barrier_mutex);
3630	3924	}
3631		-
3632		-#ifndef CONFIG_PREEMPT_RT_FULL
3633		-/**
3634		- * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete.
3635		- */
3636		-void rcu_barrier_bh(void)
3637		-{
3638		- _rcu_barrier(&rcu_bh_state);
3639		-}
3640		-EXPORT_SYMBOL_GPL(rcu_barrier_bh);
3641		-#endif
3642		-
3643		-/**
3644		- * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks.
3645		- */
3646		-void rcu_barrier_sched(void)
3647		-{
3648		- _rcu_barrier(&rcu_sched_state);
3649		-}
3650		-EXPORT_SYMBOL_GPL(rcu_barrier_sched);
	3925	+EXPORT_SYMBOL_GPL(rcu_barrier);
3651	3926
3652	3927	/*
3653	3928	* Propagate ->qsinitmask bits up the rcu_node tree to account for the
..	..	@@ -3681,46 +3956,48 @@
3681	3956	* Do boot-time initialization of a CPU's per-CPU RCU data.
3682	3957	*/
3683	3958	static void __init
3684		-rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
	3959	+rcu_boot_init_percpu_data(int cpu)
3685	3960	{
3686		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
	3961	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3687	3962
3688	3963	/* Set up local state, ensuring consistent view of global state. */
3689	3964	rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);
3690		- rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
3691		- WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != 1);
3692		- WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp->dynticks)));
3693		- rdp->rcu_ofl_gp_seq = rsp->gp_seq;
	3965	+ INIT_WORK(&rdp->strict_work, strict_work_handler);
	3966	+ WARN_ON_ONCE(rdp->dynticks_nesting != 1);
	3967	+ WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)));
	3968	+ rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;
3694	3969	rdp->rcu_ofl_gp_flags = RCU_GP_CLEANED;
3695		- rdp->rcu_onl_gp_seq = rsp->gp_seq;
	3970	+ rdp->rcu_onl_gp_seq = rcu_state.gp_seq;
3696	3971	rdp->rcu_onl_gp_flags = RCU_GP_CLEANED;
3697	3972	rdp->cpu = cpu;
3698		- rdp->rsp = rsp;
3699	3973	rcu_boot_init_nocb_percpu_data(rdp);
3700	3974	}
3701	3975
3702	3976	/*
3703		- * Initialize a CPU's per-CPU RCU data. Note that only one online or
	3977	+ * Invoked early in the CPU-online process, when pretty much all services
	3978	+ * are available. The incoming CPU is not present.
	3979	+ *
	3980	+ * Initializes a CPU's per-CPU RCU data. Note that only one online or
3704	3981	* offline event can be happening at a given time. Note also that we can
3705	3982	* accept some slop in the rsp->gp_seq access due to the fact that this
3706		- * CPU cannot possibly have any RCU callbacks in flight yet.
	3983	+ * CPU cannot possibly have any non-offloaded RCU callbacks in flight yet.
	3984	+ * And any offloaded callbacks are being numbered elsewhere.
3707	3985	*/
3708		-static void
3709		-rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
	3986	+int rcutree_prepare_cpu(unsigned int cpu)
3710	3987	{
3711	3988	unsigned long flags;
3712		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
3713		- struct rcu_node *rnp = rcu_get_root(rsp);
	3989	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
	3990	+ struct rcu_node *rnp = rcu_get_root();
3714	3991
3715	3992	/* Set up local state, ensuring consistent view of global state. */
3716	3993	raw_spin_lock_irqsave_rcu_node(rnp, flags);
3717	3994	rdp->qlen_last_fqs_check = 0;
3718		- rdp->n_force_qs_snap = rsp->n_force_qs;
	3995	+ rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
3719	3996	rdp->blimit = blimit;
3720	3997	if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */
3721		- !init_nocb_callback_list(rdp))
	3998	+ !rcu_segcblist_is_offloaded(&rdp->cblist))
3722	3999	rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */
3723		- rdp->dynticks->dynticks_nesting = 1; /* CPU not up, no tearing. */
	4000	+ rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */
3724	4001	rcu_dynticks_eqs_online();
3725	4002	raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
3726	4003
..	..	@@ -3732,30 +4009,16 @@
3732	4009	rnp = rdp->mynode;
3733	4010	raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
3734	4011	rdp->beenonline = true; /* We have now been online. */
3735		- rdp->gp_seq = rnp->gp_seq;
3736		- rdp->gp_seq_needed = rnp->gp_seq;
	4012	+ rdp->gp_seq = READ_ONCE(rnp->gp_seq);
	4013	+ rdp->gp_seq_needed = rdp->gp_seq;
3737	4014	rdp->cpu_no_qs.b.norm = true;
3738		- rdp->rcu_qs_ctr_snap = per_cpu(rcu_dynticks.rcu_qs_ctr, cpu);
3739	4015	rdp->core_needs_qs = false;
3740	4016	rdp->rcu_iw_pending = false;
3741		- rdp->rcu_iw_gp_seq = rnp->gp_seq - 1;
3742		- trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuonl"));
	4017	+ rdp->rcu_iw_gp_seq = rdp->gp_seq - 1;
	4018	+ trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
3743	4019	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3744		-}
3745		-
3746		-/*
3747		- * Invoked early in the CPU-online process, when pretty much all
3748		- * services are available. The incoming CPU is not present.
3749		- */
3750		-int rcutree_prepare_cpu(unsigned int cpu)
3751		-{
3752		- struct rcu_state *rsp;
3753		-
3754		- for_each_rcu_flavor(rsp)
3755		- rcu_init_percpu_data(cpu, rsp);
3756		-
3757	4020	rcu_prepare_kthreads(cpu);
3758		- rcu_spawn_all_nocb_kthreads(cpu);
	4021	+ rcu_spawn_cpu_nocb_kthread(cpu);
3759	4022
3760	4023	return 0;
3761	4024	}
..	..	@@ -3765,7 +4028,7 @@
3765	4028	*/
3766	4029	static void rcutree_affinity_setting(unsigned int cpu, int outgoing)
3767	4030	{
3768		- struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
	4031	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3769	4032
3770	4033	rcu_boost_kthread_setaffinity(rdp->mynode, outgoing);
3771	4034	}
..	..	@@ -3779,19 +4042,19 @@
3779	4042	unsigned long flags;
3780	4043	struct rcu_data *rdp;
3781	4044	struct rcu_node *rnp;
3782		- struct rcu_state *rsp;
3783	4045
3784		- for_each_rcu_flavor(rsp) {
3785		- rdp = per_cpu_ptr(rsp->rda, cpu);
3786		- rnp = rdp->mynode;
3787		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
3788		- rnp->ffmask \|= rdp->grpmask;
3789		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3790		- }
	4046	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	4047	+ rnp = rdp->mynode;
	4048	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4049	+ rnp->ffmask \|= rdp->grpmask;
	4050	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3791	4051	if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
3792	4052	return 0; /* Too early in boot for scheduler work. */
3793	4053	sync_sched_exp_online_cleanup(cpu);
3794	4054	rcutree_affinity_setting(cpu, -1);
	4055	+
	4056	+ // Stop-machine done, so allow nohz_full to disable tick.
	4057	+ tick_dep_clear(TICK_DEP_BIT_RCU);
3795	4058	return 0;
3796	4059	}
3797	4060
..	..	@@ -3804,47 +4067,19 @@
3804	4067	unsigned long flags;
3805	4068	struct rcu_data *rdp;
3806	4069	struct rcu_node *rnp;
3807		- struct rcu_state *rsp;
3808	4070
3809		- for_each_rcu_flavor(rsp) {
3810		- rdp = per_cpu_ptr(rsp->rda, cpu);
3811		- rnp = rdp->mynode;
3812		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
3813		- rnp->ffmask &= ~rdp->grpmask;
3814		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3815		- }
	4071	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	4072	+ rnp = rdp->mynode;
	4073	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4074	+ rnp->ffmask &= ~rdp->grpmask;
	4075	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3816	4076
3817	4077	rcutree_affinity_setting(cpu, cpu);
	4078	+
	4079	+ // nohz_full CPUs need the tick for stop-machine to work quickly
	4080	+ tick_dep_set(TICK_DEP_BIT_RCU);
3818	4081	return 0;
3819	4082	}
3820		-
3821		-/*
3822		- * Near the end of the offline process. We do only tracing here.
3823		- */
3824		-int rcutree_dying_cpu(unsigned int cpu)
3825		-{
3826		- struct rcu_state *rsp;
3827		-
3828		- for_each_rcu_flavor(rsp)
3829		- rcu_cleanup_dying_cpu(rsp);
3830		- return 0;
3831		-}
3832		-
3833		-/*
3834		- * The outgoing CPU is gone and we are running elsewhere.
3835		- */
3836		-int rcutree_dead_cpu(unsigned int cpu)
3837		-{
3838		- struct rcu_state *rsp;
3839		-
3840		- for_each_rcu_flavor(rsp) {
3841		- rcu_cleanup_dead_cpu(cpu, rsp);
3842		- do_nocb_deferred_wakeup(per_cpu_ptr(rsp->rda, cpu));
3843		- }
3844		- return 0;
3845		-}
3846		-
3847		-static DEFINE_PER_CPU(int, rcu_cpu_started);
3848	4083
3849	4084	/*
3850	4085	* Mark the specified CPU as being online so that subsequent grace periods
..	..	@@ -3861,74 +4096,40 @@
3861	4096	{
3862	4097	unsigned long flags;
3863	4098	unsigned long mask;
3864		- int nbits;
3865		- unsigned long oldmask;
3866	4099	struct rcu_data *rdp;
3867	4100	struct rcu_node *rnp;
3868		- struct rcu_state *rsp;
	4101	+ bool newcpu;
3869	4102
3870		- if (per_cpu(rcu_cpu_started, cpu))
	4103	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	4104	+ if (rdp->cpu_started)
3871	4105	return;
	4106	+ rdp->cpu_started = true;
3872	4107
3873		- per_cpu(rcu_cpu_started, cpu) = 1;
3874		-
3875		- for_each_rcu_flavor(rsp) {
3876		- rdp = per_cpu_ptr(rsp->rda, cpu);
3877		- rnp = rdp->mynode;
3878		- mask = rdp->grpmask;
3879		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
3880		- rnp->qsmaskinitnext \|= mask;
3881		- oldmask = rnp->expmaskinitnext;
3882		- rnp->expmaskinitnext \|= mask;
3883		- oldmask ^= rnp->expmaskinitnext;
3884		- nbits = bitmap_weight(&oldmask, BITS_PER_LONG);
3885		- /* Allow lockless access for expedited grace periods. */
3886		- smp_store_release(&rsp->ncpus, rsp->ncpus + nbits); /* ^^^ */
3887		- rcu_gpnum_ovf(rnp, rdp); /* Offline-induced counter wrap? */
3888		- rdp->rcu_onl_gp_seq = READ_ONCE(rsp->gp_seq);
3889		- rdp->rcu_onl_gp_flags = READ_ONCE(rsp->gp_flags);
3890		- if (rnp->qsmask & mask) { /* RCU waiting on incoming CPU? */
3891		- /* Report QS -after- changing ->qsmaskinitnext! */
3892		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
3893		- } else {
3894		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3895		- }
	4108	+ rnp = rdp->mynode;
	4109	+ mask = rdp->grpmask;
	4110	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4111	+ WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext \| mask);
	4112	+ newcpu = !(rnp->expmaskinitnext & mask);
	4113	+ rnp->expmaskinitnext \|= mask;
	4114	+ /* Allow lockless access for expedited grace periods. */
	4115	+ smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + newcpu); /* ^^^ */
	4116	+ ASSERT_EXCLUSIVE_WRITER(rcu_state.ncpus);
	4117	+ rcu_gpnum_ovf(rnp, rdp); /* Offline-induced counter wrap? */
	4118	+ rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);
	4119	+ rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags);
	4120	+ if (rnp->qsmask & mask) { /* RCU waiting on incoming CPU? */
	4121	+ rcu_disable_urgency_upon_qs(rdp);
	4122	+ /* Report QS -after- changing ->qsmaskinitnext! */
	4123	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
	4124	+ } else {
	4125	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3896	4126	}
3897	4127	smp_mb(); /* Ensure RCU read-side usage follows above initialization. */
3898	4128	}
3899	4129
3900		-#ifdef CONFIG_HOTPLUG_CPU
3901		-/*
3902		- * The CPU is exiting the idle loop into the arch_cpu_idle_dead()
3903		- * function. We now remove it from the rcu_node tree's ->qsmaskinitnext
3904		- * bit masks.
3905		- */
3906		-static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp)
3907		-{
3908		- unsigned long flags;
3909		- unsigned long mask;
3910		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
3911		- struct rcu_node rnp = rdp->mynode; / Outgoing CPU's rdp & rnp. */
3912		-
3913		- /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
3914		- mask = rdp->grpmask;
3915		- spin_lock(&rsp->ofl_lock);
3916		- raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
3917		- rdp->rcu_ofl_gp_seq = READ_ONCE(rsp->gp_seq);
3918		- rdp->rcu_ofl_gp_flags = READ_ONCE(rsp->gp_flags);
3919		- if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */
3920		- /* Report quiescent state -before- changing ->qsmaskinitnext! */
3921		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
3922		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
3923		- }
3924		- rnp->qsmaskinitnext &= ~mask;
3925		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3926		- spin_unlock(&rsp->ofl_lock);
3927		-}
3928		-
3929	4130	/*
3930	4131	* The outgoing function has no further need of RCU, so remove it from
3931		- * the list of CPUs that RCU must track.
	4132	+ * the rcu_node tree's ->qsmaskinitnext bit masks.
3932	4133	*
3933	4134	* Note that this function is special in that it is invoked directly
3934	4135	* from the outgoing CPU rather than from the cpuhp_step mechanism.
..	..	@@ -3936,65 +4137,82 @@
3936	4137	*/
3937	4138	void rcu_report_dead(unsigned int cpu)
3938	4139	{
3939		- struct rcu_state *rsp;
	4140	+ unsigned long flags;
	4141	+ unsigned long mask;
	4142	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
	4143	+ struct rcu_node rnp = rdp->mynode; / Outgoing CPU's rdp & rnp. */
3940	4144
3941		- /* QS for any half-done expedited RCU-sched GP. */
	4145	+ /* QS for any half-done expedited grace period. */
3942	4146	preempt_disable();
3943		- rcu_report_exp_rdp(&rcu_sched_state,
3944		- this_cpu_ptr(rcu_sched_state.rda), true);
	4147	+ rcu_report_exp_rdp(this_cpu_ptr(&rcu_data));
3945	4148	preempt_enable();
3946		- for_each_rcu_flavor(rsp)
3947		- rcu_cleanup_dying_idle_cpu(cpu, rsp);
	4149	+ rcu_preempt_deferred_qs(current);
3948	4150
3949		- per_cpu(rcu_cpu_started, cpu) = 0;
	4151	+ /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
	4152	+ mask = rdp->grpmask;
	4153	+ raw_spin_lock(&rcu_state.ofl_lock);
	4154	+ raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
	4155	+ rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);
	4156	+ rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags);
	4157	+ if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */
	4158	+ /* Report quiescent state -before- changing ->qsmaskinitnext! */
	4159	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
	4160	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4161	+ }
	4162	+ WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext & ~mask);
	4163	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
	4164	+ raw_spin_unlock(&rcu_state.ofl_lock);
	4165	+
	4166	+ rdp->cpu_started = false;
3950	4167	}
3951	4168
3952		-/* Migrate the dead CPU's callbacks to the current CPU. */
3953		-static void rcu_migrate_callbacks(int cpu, struct rcu_state *rsp)
	4169	+#ifdef CONFIG_HOTPLUG_CPU
	4170	+/*
	4171	+ * The outgoing CPU has just passed through the dying-idle state, and we
	4172	+ * are being invoked from the CPU that was IPIed to continue the offline
	4173	+ * operation. Migrate the outgoing CPU's callbacks to the current CPU.
	4174	+ */
	4175	+void rcutree_migrate_callbacks(int cpu)
3954	4176	{
3955	4177	unsigned long flags;
3956	4178	struct rcu_data *my_rdp;
3957		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
3958		- struct rcu_node *rnp_root = rcu_get_root(rdp->rsp);
	4179	+ struct rcu_node *my_rnp;
	4180	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3959	4181	bool needwake;
3960	4182
3961		- if (rcu_is_nocb_cpu(cpu) \|\| rcu_segcblist_empty(&rdp->cblist))
	4183	+ if (rcu_segcblist_is_offloaded(&rdp->cblist) \|\|
	4184	+ rcu_segcblist_empty(&rdp->cblist))
3962	4185	return; /* No callbacks to migrate. */
3963	4186
3964	4187	local_irq_save(flags);
3965		- my_rdp = this_cpu_ptr(rsp->rda);
3966		- if (rcu_nocb_adopt_orphan_cbs(my_rdp, rdp, flags)) {
3967		- local_irq_restore(flags);
3968		- return;
3969		- }
3970		- raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
	4188	+ my_rdp = this_cpu_ptr(&rcu_data);
	4189	+ my_rnp = my_rdp->mynode;
	4190	+ rcu_nocb_lock(my_rdp); /* irqs already disabled. */
	4191	+ WARN_ON_ONCE(!rcu_nocb_flush_bypass(my_rdp, NULL, jiffies));
	4192	+ raw_spin_lock_rcu_node(my_rnp); /* irqs already disabled. */
3971	4193	/* Leverage recent GPs and set GP for new callbacks. */
3972		- needwake = rcu_advance_cbs(rsp, rnp_root, rdp) \|\|
3973		- rcu_advance_cbs(rsp, rnp_root, my_rdp);
	4194	+ needwake = rcu_advance_cbs(my_rnp, rdp) \|\|
	4195	+ rcu_advance_cbs(my_rnp, my_rdp);
3974	4196	rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist);
	4197	+ needwake = needwake \|\| rcu_advance_cbs(my_rnp, my_rdp);
	4198	+ rcu_segcblist_disable(&rdp->cblist);
3975	4199	WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) !=
3976	4200	!rcu_segcblist_n_cbs(&my_rdp->cblist));
3977		- raw_spin_unlock_irqrestore_rcu_node(rnp_root, flags);
	4201	+ if (rcu_segcblist_is_offloaded(&my_rdp->cblist)) {
	4202	+ raw_spin_unlock_rcu_node(my_rnp); /* irqs remain disabled. */
	4203	+ __call_rcu_nocb_wake(my_rdp, true, flags);
	4204	+ } else {
	4205	+ rcu_nocb_unlock(my_rdp); /* irqs remain disabled. */
	4206	+ raw_spin_unlock_irqrestore_rcu_node(my_rnp, flags);
	4207	+ }
3978	4208	if (needwake)
3979		- rcu_gp_kthread_wake(rsp);
	4209	+ rcu_gp_kthread_wake();
	4210	+ lockdep_assert_irqs_enabled();
3980	4211	WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 \|\|
3981	4212	!rcu_segcblist_empty(&rdp->cblist),
3982	4213	"rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n",
3983	4214	cpu, rcu_segcblist_n_cbs(&rdp->cblist),
3984	4215	rcu_segcblist_first_cb(&rdp->cblist));
3985		-}
3986		-
3987		-/*
3988		- * The outgoing CPU has just passed through the dying-idle state,
3989		- * and we are being invoked from the CPU that was IPIed to continue the
3990		- * offline operation. We need to migrate the outgoing CPU's callbacks.
3991		- */
3992		-void rcutree_migrate_callbacks(int cpu)
3993		-{
3994		- struct rcu_state *rsp;
3995		-
3996		- for_each_rcu_flavor(rsp)
3997		- rcu_migrate_callbacks(cpu, rsp);
3998	4216	}
3999	4217	#endif
4000	4218
..	..	@@ -4008,13 +4226,11 @@
4008	4226	switch (action) {
4009	4227	case PM_HIBERNATION_PREPARE:
4010	4228	case PM_SUSPEND_PREPARE:
4011		- if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
4012		- rcu_expedite_gp();
	4229	+ rcu_expedite_gp();
4013	4230	break;
4014	4231	case PM_POST_HIBERNATION:
4015	4232	case PM_POST_SUSPEND:
4016		- if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
4017		- rcu_unexpedite_gp();
	4233	+ rcu_unexpedite_gp();
4018	4234	break;
4019	4235	default:
4020	4236	break;
..	..	@@ -4023,14 +4239,13 @@
4023	4239	}
4024	4240
4025	4241	/*
4026		- * Spawn the kthreads that handle each RCU flavor's grace periods.
	4242	+ * Spawn the kthreads that handle RCU's grace periods.
4027	4243	*/
4028	4244	static int __init rcu_spawn_gp_kthread(void)
4029	4245	{
4030	4246	unsigned long flags;
4031	4247	int kthread_prio_in = kthread_prio;
4032	4248	struct rcu_node *rnp;
4033		- struct rcu_state *rsp;
4034	4249	struct sched_param sp;
4035	4250	struct task_struct *t;
4036	4251
..	..	@@ -4050,21 +4265,24 @@
4050	4265	kthread_prio, kthread_prio_in);
4051	4266
4052	4267	rcu_scheduler_fully_active = 1;
4053		- for_each_rcu_flavor(rsp) {
4054		- t = kthread_create(rcu_gp_kthread, rsp, "%s", rsp->name);
4055		- BUG_ON(IS_ERR(t));
4056		- rnp = rcu_get_root(rsp);
4057		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
4058		- rsp->gp_kthread = t;
4059		- if (kthread_prio) {
4060		- sp.sched_priority = kthread_prio;
4061		- sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
4062		- }
4063		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
4064		- wake_up_process(t);
	4268	+ t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name);
	4269	+ if (WARN_ONCE(IS_ERR(t), "%s: Could not start grace-period kthread, OOM is now expected behavior\n", __func__))
	4270	+ return 0;
	4271	+ if (kthread_prio) {
	4272	+ sp.sched_priority = kthread_prio;
	4273	+ sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
4065	4274	}
	4275	+ rnp = rcu_get_root();
	4276	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4277	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	4278	+ WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
	4279	+ // Reset .gp_activity and .gp_req_activity before setting .gp_kthread.
	4280	+ smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */
	4281	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
	4282	+ wake_up_process(t);
4066	4283	rcu_spawn_nocb_kthreads();
4067	4284	rcu_spawn_boost_kthreads();
	4285	+ rcu_spawn_core_kthreads();
4068	4286	return 0;
4069	4287	}
4070	4288	early_initcall(rcu_spawn_gp_kthread);
..	..	@@ -4089,9 +4307,9 @@
4089	4307	}
4090	4308
4091	4309	/*
4092		- * Helper function for rcu_init() that initializes one rcu_state structure.
	4310	+ * Helper function for rcu_init() that initializes the rcu_state structure.
4093	4311	*/
4094		-static void __init rcu_init_one(struct rcu_state *rsp)
	4312	+static void __init rcu_init_one(void)
4095	4313	{
4096	4314	static const char * const buf[] = RCU_NODE_NAME_INIT;
4097	4315	static const char * const fqs[] = RCU_FQS_NAME_INIT;
..	..	@@ -4113,14 +4331,15 @@
4113	4331	/* Initialize the level-tracking arrays. */
4114	4332
4115	4333	for (i = 1; i < rcu_num_lvls; i++)
4116		- rsp->level[i] = rsp->level[i - 1] + num_rcu_lvl[i - 1];
	4334	+ rcu_state.level[i] =
	4335	+ rcu_state.level[i - 1] + num_rcu_lvl[i - 1];
4117	4336	rcu_init_levelspread(levelspread, num_rcu_lvl);
4118	4337
4119	4338	/* Initialize the elements themselves, starting from the leaves. */
4120	4339
4121	4340	for (i = rcu_num_lvls - 1; i >= 0; i--) {
4122	4341	cpustride *= levelspread[i];
4123		- rnp = rsp->level[i];
	4342	+ rnp = rcu_state.level[i];
4124	4343	for (j = 0; j < num_rcu_lvl[i]; j++, rnp++) {
4125	4344	raw_spin_lock_init(&ACCESS_PRIVATE(rnp, lock));
4126	4345	lockdep_set_class_and_name(&ACCESS_PRIVATE(rnp, lock),
..	..	@@ -4128,9 +4347,9 @@
4128	4347	raw_spin_lock_init(&rnp->fqslock);
4129	4348	lockdep_set_class_and_name(&rnp->fqslock,
4130	4349	&rcu_fqs_class[i], fqs[i]);
4131		- rnp->gp_seq = rsp->gp_seq;
4132		- rnp->gp_seq_needed = rsp->gp_seq;
4133		- rnp->completedqs = rsp->gp_seq;
	4350	+ rnp->gp_seq = rcu_state.gp_seq;
	4351	+ rnp->gp_seq_needed = rcu_state.gp_seq;
	4352	+ rnp->completedqs = rcu_state.gp_seq;
4134	4353	rnp->qsmask = 0;
4135	4354	rnp->qsmaskinit = 0;
4136	4355	rnp->grplo = j * cpustride;
..	..	@@ -4143,8 +4362,8 @@
4143	4362	rnp->parent = NULL;
4144	4363	} else {
4145	4364	rnp->grpnum = j % levelspread[i - 1];
4146		- rnp->grpmask = 1UL << rnp->grpnum;
4147		- rnp->parent = rsp->level[i - 1] +
	4365	+ rnp->grpmask = BIT(rnp->grpnum);
	4366	+ rnp->parent = rcu_state.level[i - 1] +
4148	4367	j / levelspread[i - 1];
4149	4368	}
4150	4369	rnp->level = i;
..	..	@@ -4158,16 +4377,15 @@
4158	4377	}
4159	4378	}
4160	4379
4161		- init_swait_queue_head(&rsp->gp_wq);
4162		- init_swait_queue_head(&rsp->expedited_wq);
4163		- rnp = rcu_first_leaf_node(rsp);
	4380	+ init_swait_queue_head(&rcu_state.gp_wq);
	4381	+ init_swait_queue_head(&rcu_state.expedited_wq);
	4382	+ rnp = rcu_first_leaf_node();
4164	4383	for_each_possible_cpu(i) {
4165	4384	while (i > rnp->grphi)
4166	4385	rnp++;
4167		- per_cpu_ptr(rsp->rda, i)->mynode = rnp;
4168		- rcu_boot_init_percpu_data(i, rsp);
	4386	+ per_cpu_ptr(&rcu_data, i)->mynode = rnp;
	4387	+ rcu_boot_init_percpu_data(i);
4169	4388	}
4170		- list_add(&rsp->flavors, &rcu_struct_flavors);
4171	4389	}
4172	4390
4173	4391	/*
..	..	@@ -4175,11 +4393,25 @@
4175	4393	* replace the definitions in tree.h because those are needed to size
4176	4394	* the ->node array in the rcu_state structure.
4177	4395	*/
4178		-static void __init rcu_init_geometry(void)
	4396	+void rcu_init_geometry(void)
4179	4397	{
4180	4398	ulong d;
4181	4399	int i;
	4400	+ static unsigned long old_nr_cpu_ids;
4182	4401	int rcu_capacity[RCU_NUM_LVLS];
	4402	+ static bool initialized;
	4403	+
	4404	+ if (initialized) {
	4405	+ /*
	4406	+ * Warn if setup_nr_cpu_ids() had not yet been invoked,
	4407	+ * unless nr_cpus_ids == NR_CPUS, in which case who cares?
	4408	+ */
	4409	+ WARN_ON_ONCE(old_nr_cpu_ids != nr_cpu_ids);
	4410	+ return;
	4411	+ }
	4412	+
	4413	+ old_nr_cpu_ids = nr_cpu_ids;
	4414	+ initialized = true;
4183	4415
4184	4416	/*
4185	4417	* Initialize any unspecified boot parameters.
..	..	@@ -4193,6 +4425,7 @@
4193	4425	jiffies_till_first_fqs = d;
4194	4426	if (jiffies_till_next_fqs == ULONG_MAX)
4195	4427	jiffies_till_next_fqs = d;
	4428	+ adjust_jiffies_till_sched_qs();
4196	4429
4197	4430	/* If the compile-time values are accurate, just leave. */
4198	4431	if (rcu_fanout_leaf == RCU_FANOUT_LEAF &&
..	..	@@ -4251,16 +4484,16 @@
4251	4484
4252	4485	/*
4253	4486	* Dump out the structure of the rcu_node combining tree associated
4254		- * with the rcu_state structure referenced by rsp.
	4487	+ * with the rcu_state structure.
4255	4488	*/
4256		-static void __init rcu_dump_rcu_node_tree(struct rcu_state *rsp)
	4489	+static void __init rcu_dump_rcu_node_tree(void)
4257	4490	{
4258	4491	int level = 0;
4259	4492	struct rcu_node *rnp;
4260	4493
4261	4494	pr_info("rcu_node tree layout dump\n");
4262	4495	pr_info(" ");
4263		- rcu_for_each_node_breadth_first(rsp, rnp) {
	4496	+ rcu_for_each_node_breadth_first(rnp) {
4264	4497	if (rnp->level != level) {
4265	4498	pr_cont("\n");
4266	4499	pr_info(" ");
..	..	@@ -4274,21 +4507,41 @@
4274	4507	struct workqueue_struct *rcu_gp_wq;
4275	4508	struct workqueue_struct *rcu_par_gp_wq;
4276	4509
	4510	+static void __init kfree_rcu_batch_init(void)
	4511	+{
	4512	+ int cpu;
	4513	+ int i;
	4514	+
	4515	+ for_each_possible_cpu(cpu) {
	4516	+ struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
	4517	+
	4518	+ for (i = 0; i < KFREE_N_BATCHES; i++) {
	4519	+ INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
	4520	+ krcp->krw_arr[i].krcp = krcp;
	4521	+ }
	4522	+
	4523	+ INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
	4524	+ INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
	4525	+ krcp->initialized = true;
	4526	+ }
	4527	+ if (register_shrinker(&kfree_rcu_shrinker))
	4528	+ pr_err("Failed to register kfree_rcu() shrinker!\n");
	4529	+}
	4530	+
4277	4531	void __init rcu_init(void)
4278	4532	{
4279	4533	int cpu;
4280	4534
4281	4535	rcu_early_boot_tests();
4282	4536
	4537	+ kfree_rcu_batch_init();
4283	4538	rcu_bootup_announce();
4284	4539	rcu_init_geometry();
4285		-#ifndef CONFIG_PREEMPT_RT_FULL
4286		- rcu_init_one(&rcu_bh_state);
4287		-#endif
4288		- rcu_init_one(&rcu_sched_state);
	4540	+ rcu_init_one();
4289	4541	if (dump_tree)
4290		- rcu_dump_rcu_node_tree(&rcu_sched_state);
4291		- __rcu_init_preempt();
	4542	+ rcu_dump_rcu_node_tree();
	4543	+ if (use_softirq)
	4544	+ open_softirq(RCU_SOFTIRQ, rcu_core_si);
4292	4545
4293	4546	/*
4294	4547	* We don't need protection against CPU-hotplug here because
..	..	@@ -4307,7 +4560,16 @@
4307	4560	WARN_ON(!rcu_gp_wq);
4308	4561	rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);
4309	4562	WARN_ON(!rcu_par_gp_wq);
	4563	+ srcu_init();
	4564	+
	4565	+ /* Fill in default value for rcutree.qovld boot parameter. */
	4566	+ /* -After- the rcu_node ->lock fields are initialized! */
	4567	+ if (qovld < 0)
	4568	+ qovld_calc = DEFAULT_RCU_QOVLD_MULT * qhimark;
	4569	+ else
	4570	+ qovld_calc = qovld;
4310	4571	}
4311	4572
	4573	+#include "tree_stall.h"
4312	4574	#include "tree_exp.h"
4313	4575	#include "tree_plugin.h"