~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,27 +1,14 @@
	1	+// SPDX-License-Identifier: GPL-2.0+
1	2	/*
2		- * Read-Copy Update mechanism for mutual exclusion
3		- *
4		- * This program is free software; you can redistribute it and/or modify
5		- * it under the terms of the GNU General Public License as published by
6		- * the Free Software Foundation; either version 2 of the License, or
7		- * (at your option) any later version.
8		- *
9		- * This program is distributed in the hope that it will be useful,
10		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12		- * GNU General Public License for more details.
13		- *
14		- * You should have received a copy of the GNU General Public License
15		- * along with this program; if not, you can access it online at
16		- * http://www.gnu.org/licenses/gpl-2.0.html.
	3	+ * Read-Copy Update mechanism for mutual exclusion (tree-based version)
17	4	*
18	5	* Copyright IBM Corporation, 2008
19	6	*
20	7	* Authors: Dipankar Sarma <dipankar@in.ibm.com>
21	8	* Manfred Spraul <manfred@colorfullife.com>
22		- * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version
	9	+ * Paul E. McKenney <paulmck@linux.ibm.com>
23	10	*
24		- * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
	11	+ * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
25	12	* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
26	13	*
27	14	* For detailed explanation of Read-Copy Update mechanism see -
..	..	@@ -56,17 +43,23 @@
56	43	#include <uapi/linux/sched/types.h>
57	44	#include <linux/prefetch.h>
58	45	#include <linux/delay.h>
59		-#include <linux/stop_machine.h>
60	46	#include <linux/random.h>
61	47	#include <linux/trace_events.h>
62	48	#include <linux/suspend.h>
63	49	#include <linux/ftrace.h>
64		-#include <linux/delay.h>
	50	+#include <linux/tick.h>
	51	+#include <linux/sysrq.h>
	52	+#include <linux/kprobes.h>
65	53	#include <linux/gfp.h>
66	54	#include <linux/oom.h>
67	55	#include <linux/smpboot.h>
68	56	#include <linux/jiffies.h>
	57	+#include <linux/slab.h>
69	58	#include <linux/sched/isolation.h>
	59	+#include <linux/sched/clock.h>
	60	+#include <linux/vmalloc.h>
	61	+#include <linux/mm.h>
	62	+#include <linux/kasan.h>
70	63	#include "../time/tick-internal.h"
71	64
72	65	#include "tree.h"
..	..	@@ -80,49 +73,35 @@
80	73	/* Data structures. */
81	74
82	75	/*
83		- * In order to export the rcu_state name to the tracing tools, it
84		- * needs to be added in the __tracepoint_string section.
85		- * This requires defining a separate variable tp_<sname>_varname
86		- * that points to the string being used, and this will allow
87		- * the tracing userspace tools to be able to decipher the string
88		- * address to the matching string.
	76	+ * Steal a bit from the bottom of ->dynticks for idle entry/exit
	77	+ * control. Initially this is for TLB flushing.
89	78	*/
90		-#ifdef CONFIG_TRACING
91		-# define DEFINE_RCU_TPS(sname) \
92		-static char sname##_varname[] = #sname; \
93		-static const char *tp_##sname##_varname __used __tracepoint_string = sname##_varname;
94		-# define RCU_STATE_NAME(sname) sname##_varname
95		-#else
96		-# define DEFINE_RCU_TPS(sname)
97		-# define RCU_STATE_NAME(sname) __stringify(sname)
98		-#endif
	79	+#define RCU_DYNTICK_CTRL_MASK 0x1
	80	+#define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1)
99	81
100		-#define RCU_STATE_INITIALIZER(sname, sabbr, cr) \
101		-DEFINE_RCU_TPS(sname) \
102		-static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data); \
103		-struct rcu_state sname##_state = { \
104		- .level = { &sname##_state.node[0] }, \
105		- .rda = &sname##_data, \
106		- .call = cr, \
107		- .gp_state = RCU_GP_IDLE, \
108		- .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT, \
109		- .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \
110		- .name = RCU_STATE_NAME(sname), \
111		- .abbr = sabbr, \
112		- .exp_mutex = __MUTEX_INITIALIZER(sname##_state.exp_mutex), \
113		- .exp_wake_mutex = __MUTEX_INITIALIZER(sname##_state.exp_wake_mutex), \
114		- .ofl_lock = __SPIN_LOCK_UNLOCKED(sname##_state.ofl_lock), \
115		-}
116		-
117		-RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
118		-RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh);
119		-
120		-static struct rcu_state *const rcu_state_p;
121		-LIST_HEAD(rcu_struct_flavors);
	82	+static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
	83	+ .dynticks_nesting = 1,
	84	+ .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
	85	+ .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),
	86	+};
	87	+static struct rcu_state rcu_state = {
	88	+ .level = { &rcu_state.node[0] },
	89	+ .gp_state = RCU_GP_IDLE,
	90	+ .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT,
	91	+ .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
	92	+ .name = RCU_NAME,
	93	+ .abbr = RCU_ABBR,
	94	+ .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
	95	+ .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
	96	+ .ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock),
	97	+};
122	98
123	99	/* Dump rcu_node combining tree at boot to verify correct setup. */
124	100	static bool dump_tree;
125	101	module_param(dump_tree, bool, 0444);
	102	+/* By default, use RCU_SOFTIRQ instead of rcuc kthreads. */
	103	+static bool use_softirq = true;
	104	+module_param(use_softirq, bool, 0444);
126	105	/* Control rcu_node-tree auto-balancing at boot time. */
127	106	static bool rcu_fanout_exact;
128	107	module_param(rcu_fanout_exact, bool, 0444);
..	..	@@ -133,9 +112,6 @@
133	112	/* Number of rcu_nodes at specified level. */
134	113	int num_rcu_lvl[] = NUM_RCU_LVL_INIT;
135	114	int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */
136		-/* panic() on RCU Stall sysctl. */
137		-int sysctl_panic_on_rcu_stall __read_mostly = CONFIG_BOOTPARAM_RCU_STALL_PANIC_VALUE;
138		-ATOMIC_NOTIFIER_HEAD(rcu_stall_notifier_list);
139	115
140	116	/*
141	117	* The rcu_scheduler_active variable is initialized to the value
..	..	@@ -166,21 +142,19 @@
166	142	*/
167	143	static int rcu_scheduler_fully_active __read_mostly;
168	144
169		-static void
170		-rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
171		- struct rcu_node *rnp, unsigned long gps, unsigned long flags);
	145	+static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
	146	+ unsigned long gps, unsigned long flags);
172	147	static void rcu_init_new_rnp(struct rcu_node *rnp_leaf);
173	148	static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf);
174	149	static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
175	150	static void invoke_rcu_core(void);
176		-static void invoke_rcu_callbacks(struct rcu_state rsp, struct rcu_data rdp);
177		-static void rcu_report_exp_rdp(struct rcu_state *rsp,
178		- struct rcu_data *rdp, bool wake);
	151	+static void rcu_report_exp_rdp(struct rcu_data *rdp);
179	152	static void sync_sched_exp_online_cleanup(int cpu);
	153	+static void check_cb_ovld_locked(struct rcu_data rdp, struct rcu_node rnp);
180	154
181	155	/* rcuc/rcub kthread realtime priority */
182	156	static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
183		-module_param(kthread_prio, int, 0644);
	157	+module_param(kthread_prio, int, 0444);
184	158
185	159	/* Delay in jiffies for grace-period initialization delays, debug only. */
186	160
..	..	@@ -191,7 +165,22 @@
191	165	static int gp_cleanup_delay;
192	166	module_param(gp_cleanup_delay, int, 0444);
193	167
194		-/* Retreive RCU kthreads priority for rcutorture */
	168	+// Add delay to rcu_read_unlock() for strict grace periods.
	169	+static int rcu_unlock_delay;
	170	+#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
	171	+module_param(rcu_unlock_delay, int, 0444);
	172	+#endif
	173	+
	174	+/*
	175	+ * This rcu parameter is runtime-read-only. It reflects
	176	+ * a minimum allowed number of objects which can be cached
	177	+ * per-CPU. Object size is equal to one page. This value
	178	+ * can be changed at boot time.
	179	+ */
	180	+static int rcu_min_cached_objs = 5;
	181	+module_param(rcu_min_cached_objs, int, 0444);
	182	+
	183	+/* Retrieve RCU kthreads priority for rcutorture */
195	184	int rcu_get_gp_kthreads_prio(void)
196	185	{
197	186	return kthread_prio;
..	..	@@ -215,7 +204,7 @@
215	204	* held, but the bit corresponding to the current CPU will be stable
216	205	* in most contexts.
217	206	*/
218		-unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp)
	207	+static unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp)
219	208	{
220	209	return READ_ONCE(rnp->qsmaskinitnext);
221	210	}
..	..	@@ -225,81 +214,39 @@
225	214	* permit this function to be invoked without holding the root rcu_node
226	215	* structure's ->lock, but of course results can be subject to change.
227	216	*/
228		-static int rcu_gp_in_progress(struct rcu_state *rsp)
	217	+static int rcu_gp_in_progress(void)
229	218	{
230		- return rcu_seq_state(rcu_seq_current(&rsp->gp_seq));
	219	+ return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));
231	220	}
232	221
233	222	/*
234		- * Note a quiescent state. Because we do not need to know
235		- * how many quiescent states passed, just if there was at least
236		- * one since the start of the grace period, this just sets a flag.
237		- * The caller must have disabled preemption.
	223	+ * Return the number of callbacks queued on the specified CPU.
	224	+ * Handles both the nocbs and normal cases.
238	225	*/
239		-void rcu_sched_qs(void)
	226	+static long rcu_get_n_cbs_cpu(int cpu)
240	227	{
241		- RCU_LOCKDEP_WARN(preemptible(), "rcu_sched_qs() invoked with preemption enabled!!!");
242		- if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.s))
243		- return;
244		- trace_rcu_grace_period(TPS("rcu_sched"),
245		- __this_cpu_read(rcu_sched_data.gp_seq),
246		- TPS("cpuqs"));
247		- __this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false);
248		- if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
249		- return;
250		- __this_cpu_write(rcu_sched_data.cpu_no_qs.b.exp, false);
251		- rcu_report_exp_rdp(&rcu_sched_state,
252		- this_cpu_ptr(&rcu_sched_data), true);
	228	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
	229	+
	230	+ if (rcu_segcblist_is_enabled(&rdp->cblist))
	231	+ return rcu_segcblist_n_cbs(&rdp->cblist);
	232	+ return 0;
253	233	}
254	234
255		-#ifdef CONFIG_PREEMPT_RT_FULL
256		-static void rcu_preempt_qs(void);
257		-
258		-void rcu_bh_qs(void)
	235	+void rcu_softirq_qs(void)
259	236	{
260		- unsigned long flags;
261		-
262		- /* Callers to this function, rcu_preempt_qs(), must disable irqs. */
263		- local_irq_save(flags);
264		- rcu_preempt_qs();
265		- local_irq_restore(flags);
	237	+ rcu_qs();
	238	+ rcu_preempt_deferred_qs(current);
266	239	}
267		-#else
268		-void rcu_bh_qs(void)
269		-{
270		- RCU_LOCKDEP_WARN(preemptible(), "rcu_bh_qs() invoked with preemption enabled!!!");
271		- if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) {
272		- trace_rcu_grace_period(TPS("rcu_bh"),
273		- __this_cpu_read(rcu_bh_data.gp_seq),
274		- TPS("cpuqs"));
275		- __this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);
276		- }
277		-}
278		-#endif
279		-
280		-/*
281		- * Steal a bit from the bottom of ->dynticks for idle entry/exit
282		- * control. Initially this is for TLB flushing.
283		- */
284		-#define RCU_DYNTICK_CTRL_MASK 0x1
285		-#define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1)
286		-#ifndef rcu_eqs_special_exit
287		-#define rcu_eqs_special_exit() do { } while (0)
288		-#endif
289		-
290		-static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
291		- .dynticks_nesting = 1,
292		- .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
293		- .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),
294		-};
295	240
296	241	/*
297	242	* Record entry into an extended quiescent state. This is only to be
298		- * called when not already in an extended quiescent state.
	243	+ * called when not already in an extended quiescent state, that is,
	244	+ * RCU is watching prior to the call to this function and is no longer
	245	+ * watching upon return.
299	246	*/
300		-static void rcu_dynticks_eqs_enter(void)
	247	+static noinstr void rcu_dynticks_eqs_enter(void)
301	248	{
302		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	249	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
303	250	int seq;
304	251
305	252	/*
..	..	@@ -307,8 +254,9 @@
307	254	* critical sections, and we also must force ordering with the
308	255	* next idle sojourn.
309	256	*/
310		- seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
311		- /* Better be in an extended quiescent state! */
	257	+ rcu_dynticks_task_trace_enter(); // Before ->dynticks update!
	258	+ seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
	259	+ // RCU is no longer watching. Better be in extended quiescent state!
312	260	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
313	261	(seq & RCU_DYNTICK_CTRL_CTR));
314	262	/* Better not have special action (TLB flush) pending! */
..	..	@@ -318,11 +266,12 @@
318	266
319	267	/*
320	268	* Record exit from an extended quiescent state. This is only to be
321		- * called from an extended quiescent state.
	269	+ * called from an extended quiescent state, that is, RCU is not watching
	270	+ * prior to the call to this function and is watching upon return.
322	271	*/
323		-static void rcu_dynticks_eqs_exit(void)
	272	+static noinstr void rcu_dynticks_eqs_exit(void)
324	273	{
325		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	274	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
326	275	int seq;
327	276
328	277	/*
..	..	@@ -330,14 +279,14 @@
330	279	* and we also must force ordering with the next RCU read-side
331	280	* critical section.
332	281	*/
333		- seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
	282	+ seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
	283	+ // RCU is now watching. Better not be in an extended quiescent state!
	284	+ rcu_dynticks_task_trace_exit(); // After ->dynticks update!
334	285	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
335	286	!(seq & RCU_DYNTICK_CTRL_CTR));
336	287	if (seq & RCU_DYNTICK_CTRL_MASK) {
337		- atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdtp->dynticks);
	288	+ arch_atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks);
338	289	smp_mb__after_atomic(); /* _exit after clearing mask. */
339		- /* Prefer duplicate flushes to losing a flush. */
340		- rcu_eqs_special_exit();
341	290	}
342	291	}
343	292
..	..	@@ -353,11 +302,11 @@
353	302	*/
354	303	static void rcu_dynticks_eqs_online(void)
355	304	{
356		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	305	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
357	306
358		- if (atomic_read(&rdtp->dynticks) & RCU_DYNTICK_CTRL_CTR)
	307	+ if (atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR)
359	308	return;
360		- atomic_add(RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
	309	+ atomic_add(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
361	310	}
362	311
363	312	/*
..	..	@@ -365,20 +314,20 @@
365	314	*
366	315	* No ordering, as we are sampling CPU-local information.
367	316	*/
368		-bool rcu_dynticks_curr_cpu_in_eqs(void)
	317	+static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)
369	318	{
370		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	319	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
371	320
372		- return !(atomic_read(&rdtp->dynticks) & RCU_DYNTICK_CTRL_CTR);
	321	+ return !(arch_atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR);
373	322	}
374	323
375	324	/*
376	325	* Snapshot the ->dynticks counter with full ordering so as to allow
377	326	* stable comparison of this counter with past and future snapshots.
378	327	*/
379		-int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
	328	+static int rcu_dynticks_snap(struct rcu_data *rdp)
380	329	{
381		- int snap = atomic_add_return(0, &rdtp->dynticks);
	330	+ int snap = atomic_add_return(0, &rdp->dynticks);
382	331
383	332	return snap & ~RCU_DYNTICK_CTRL_MASK;
384	333	}
..	..	@@ -393,13 +342,35 @@
393	342	}
394	343
395	344	/*
396		- * Return true if the CPU corresponding to the specified rcu_dynticks
	345	+ * Return true if the CPU corresponding to the specified rcu_data
397	346	* structure has spent some time in an extended quiescent state since
398	347	* rcu_dynticks_snap() returned the specified snapshot.
399	348	*/
400		-static bool rcu_dynticks_in_eqs_since(struct rcu_dynticks *rdtp, int snap)
	349	+static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap)
401	350	{
402		- return snap != rcu_dynticks_snap(rdtp);
	351	+ return snap != rcu_dynticks_snap(rdp);
	352	+}
	353	+
	354	+/*
	355	+ * Return true if the referenced integer is zero while the specified
	356	+ * CPU remains within a single extended quiescent state.
	357	+ */
	358	+bool rcu_dynticks_zero_in_eqs(int cpu, int *vp)
	359	+{
	360	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
	361	+ int snap;
	362	+
	363	+ // If not quiescent, force back to earlier extended quiescent state.
	364	+ snap = atomic_read(&rdp->dynticks) & ~(RCU_DYNTICK_CTRL_MASK \|
	365	+ RCU_DYNTICK_CTRL_CTR);
	366	+
	367	+ smp_rmb(); // Order ->dynticks and *vp reads.
	368	+ if (READ_ONCE(*vp))
	369	+ return false; // Non-zero, so report failure;
	370	+ smp_rmb(); // Order *vp read and ->dynticks re-read.
	371	+
	372	+ // If still in the same extended quiescent state, we are good!
	373	+ return snap == (atomic_read(&rdp->dynticks) & ~RCU_DYNTICK_CTRL_MASK);
403	374	}
404	375
405	376	/*
..	..	@@ -413,14 +384,17 @@
413	384	{
414	385	int old;
415	386	int new;
416		- struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
	387	+ int new_old;
	388	+ struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
417	389
	390	+ new_old = atomic_read(&rdp->dynticks);
418	391	do {
419		- old = atomic_read(&rdtp->dynticks);
	392	+ old = new_old;
420	393	if (old & RCU_DYNTICK_CTRL_CTR)
421	394	return false;
422	395	new = old \| RCU_DYNTICK_CTRL_MASK;
423		- } while (atomic_cmpxchg(&rdtp->dynticks, old, new) != old);
	396	+ new_old = atomic_cmpxchg(&rdp->dynticks, old, new);
	397	+ } while (new_old != old);
424	398	return true;
425	399	}
426	400
..	..	@@ -435,105 +409,128 @@
435	409	*
436	410	* The caller must have disabled interrupts and must not be idle.
437	411	*/
438		-static void rcu_momentary_dyntick_idle(void)
	412	+notrace void rcu_momentary_dyntick_idle(void)
439	413	{
440		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
441	414	int special;
442	415
443		- raw_cpu_write(rcu_dynticks.rcu_need_heavy_qs, false);
444		- special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR, &rdtp->dynticks);
	416	+ raw_cpu_write(rcu_data.rcu_need_heavy_qs, false);
	417	+ special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR,
	418	+ &this_cpu_ptr(&rcu_data)->dynticks);
445	419	/* It is illegal to call this from idle state. */
446	420	WARN_ON_ONCE(!(special & RCU_DYNTICK_CTRL_CTR));
	421	+ rcu_preempt_deferred_qs(current);
447	422	}
	423	+EXPORT_SYMBOL_GPL(rcu_momentary_dyntick_idle);
448	424
449		-/*
450		- * Note a context switch. This is a quiescent state for RCU-sched,
451		- * and requires special handling for preemptible RCU.
452		- * The caller must have disabled interrupts.
453		- */
454		-void rcu_note_context_switch(bool preempt)
455		-{
456		- barrier(); /* Avoid RCU read-side critical sections leaking down. */
457		- trace_rcu_utilization(TPS("Start context switch"));
458		- rcu_sched_qs();
459		- rcu_preempt_note_context_switch(preempt);
460		- /* Load rcu_urgent_qs before other flags. */
461		- if (!smp_load_acquire(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs)))
462		- goto out;
463		- this_cpu_write(rcu_dynticks.rcu_urgent_qs, false);
464		- if (unlikely(raw_cpu_read(rcu_dynticks.rcu_need_heavy_qs)))
465		- rcu_momentary_dyntick_idle();
466		- this_cpu_inc(rcu_dynticks.rcu_qs_ctr);
467		- if (!preempt)
468		- rcu_tasks_qs(current);
469		-out:
470		- trace_rcu_utilization(TPS("End context switch"));
471		- barrier(); /* Avoid RCU read-side critical sections leaking up. */
472		-}
473		-EXPORT_SYMBOL_GPL(rcu_note_context_switch);
474		-
475		-/*
476		- * Register a quiescent state for all RCU flavors. If there is an
477		- * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
478		- * dyntick-idle quiescent state visible to other CPUs (but only for those
479		- * RCU flavors in desperate need of a quiescent state, which will normally
480		- * be none of them). Either way, do a lightweight quiescent state for
481		- * all RCU flavors.
	425	+/**
	426	+ * rcu_is_cpu_rrupt_from_idle - see if 'interrupted' from idle
482	427	*
483		- * The barrier() calls are redundant in the common case when this is
484		- * called externally, but just in case this is called from within this
485		- * file.
	428	+ * If the current CPU is idle and running at a first-level (not nested)
	429	+ * interrupt, or directly, from idle, return true.
486	430	*
	431	+ * The caller must have at least disabled IRQs.
487	432	*/
488		-void rcu_all_qs(void)
	433	+static int rcu_is_cpu_rrupt_from_idle(void)
489	434	{
490		- unsigned long flags;
	435	+ long nesting;
491	436
492		- if (!raw_cpu_read(rcu_dynticks.rcu_urgent_qs))
493		- return;
494		- preempt_disable();
495		- /* Load rcu_urgent_qs before other flags. */
496		- if (!smp_load_acquire(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs))) {
497		- preempt_enable();
498		- return;
499		- }
500		- this_cpu_write(rcu_dynticks.rcu_urgent_qs, false);
501		- barrier(); /* Avoid RCU read-side critical sections leaking down. */
502		- if (unlikely(raw_cpu_read(rcu_dynticks.rcu_need_heavy_qs))) {
503		- local_irq_save(flags);
504		- rcu_momentary_dyntick_idle();
505		- local_irq_restore(flags);
506		- }
507		- if (unlikely(raw_cpu_read(rcu_sched_data.cpu_no_qs.b.exp)))
508		- rcu_sched_qs();
509		- this_cpu_inc(rcu_dynticks.rcu_qs_ctr);
510		- barrier(); /* Avoid RCU read-side critical sections leaking up. */
511		- preempt_enable();
	437	+ /*
	438	+ * Usually called from the tick; but also used from smp_function_call()
	439	+ * for expedited grace periods. This latter can result in running from
	440	+ * the idle task, instead of an actual IPI.
	441	+ */
	442	+ lockdep_assert_irqs_disabled();
	443	+
	444	+ /* Check for counter underflows */
	445	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) < 0,
	446	+ "RCU dynticks_nesting counter underflow!");
	447	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0,
	448	+ "RCU dynticks_nmi_nesting counter underflow/zero!");
	449	+
	450	+ /* Are we at first interrupt nesting level? */
	451	+ nesting = __this_cpu_read(rcu_data.dynticks_nmi_nesting);
	452	+ if (nesting > 1)
	453	+ return false;
	454	+
	455	+ /*
	456	+ * If we're not in an interrupt, we must be in the idle task!
	457	+ */
	458	+ WARN_ON_ONCE(!nesting && !is_idle_task(current));
	459	+
	460	+ /* Does CPU appear to be idle from an RCU standpoint? */
	461	+ return __this_cpu_read(rcu_data.dynticks_nesting) == 0;
512	462	}
513		-EXPORT_SYMBOL_GPL(rcu_all_qs);
514	463
515		-#define DEFAULT_RCU_BLIMIT 10 /* Maximum callbacks per rcu_do_batch. */
	464	+#define DEFAULT_RCU_BLIMIT (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 1000 : 10)
	465	+ // Maximum callbacks per rcu_do_batch ...
	466	+#define DEFAULT_MAX_RCU_BLIMIT 10000 // ... even during callback flood.
516	467	static long blimit = DEFAULT_RCU_BLIMIT;
517		-#define DEFAULT_RCU_QHIMARK 10000 /* If this many pending, ignore blimit. */
	468	+#define DEFAULT_RCU_QHIMARK 10000 // If this many pending, ignore blimit.
518	469	static long qhimark = DEFAULT_RCU_QHIMARK;
519		-#define DEFAULT_RCU_QLOMARK 100 /* Once only this many pending, use blimit. */
	470	+#define DEFAULT_RCU_QLOMARK 100 // Once only this many pending, use blimit.
520	471	static long qlowmark = DEFAULT_RCU_QLOMARK;
	472	+#define DEFAULT_RCU_QOVLD_MULT 2
	473	+#define DEFAULT_RCU_QOVLD (DEFAULT_RCU_QOVLD_MULT * DEFAULT_RCU_QHIMARK)
	474	+static long qovld = DEFAULT_RCU_QOVLD; // If this many pending, hammer QS.
	475	+static long qovld_calc = -1; // No pre-initialization lock acquisitions!
521	476
522	477	module_param(blimit, long, 0444);
523	478	module_param(qhimark, long, 0444);
524	479	module_param(qlowmark, long, 0444);
	480	+module_param(qovld, long, 0444);
525	481
526		-static ulong jiffies_till_first_fqs = ULONG_MAX;
	482	+static ulong jiffies_till_first_fqs = IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 0 : ULONG_MAX;
527	483	static ulong jiffies_till_next_fqs = ULONG_MAX;
528	484	static bool rcu_kick_kthreads;
	485	+static int rcu_divisor = 7;
	486	+module_param(rcu_divisor, int, 0644);
	487	+
	488	+/* Force an exit from rcu_do_batch() after 3 milliseconds. */
	489	+static long rcu_resched_ns = 3 * NSEC_PER_MSEC;
	490	+module_param(rcu_resched_ns, long, 0644);
	491	+
	492	+/*
	493	+ * How long the grace period must be before we start recruiting
	494	+ * quiescent-state help from rcu_note_context_switch().
	495	+ */
	496	+static ulong jiffies_till_sched_qs = ULONG_MAX;
	497	+module_param(jiffies_till_sched_qs, ulong, 0444);
	498	+static ulong jiffies_to_sched_qs; /* See adjust_jiffies_till_sched_qs(). */
	499	+module_param(jiffies_to_sched_qs, ulong, 0444); /* Display only! */
	500	+
	501	+/*
	502	+ * Make sure that we give the grace-period kthread time to detect any
	503	+ * idle CPUs before taking active measures to force quiescent states.
	504	+ * However, don't go below 100 milliseconds, adjusted upwards for really
	505	+ * large systems.
	506	+ */
	507	+static void adjust_jiffies_till_sched_qs(void)
	508	+{
	509	+ unsigned long j;
	510	+
	511	+ /* If jiffies_till_sched_qs was specified, respect the request. */
	512	+ if (jiffies_till_sched_qs != ULONG_MAX) {
	513	+ WRITE_ONCE(jiffies_to_sched_qs, jiffies_till_sched_qs);
	514	+ return;
	515	+ }
	516	+ /* Otherwise, set to third fqs scan, but bound below on large system. */
	517	+ j = READ_ONCE(jiffies_till_first_fqs) +
	518	+ 2 * READ_ONCE(jiffies_till_next_fqs);
	519	+ if (j < HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV)
	520	+ j = HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
	521	+ pr_info("RCU calculated value of scheduler-enlistment delay is %ld jiffies.\n", j);
	522	+ WRITE_ONCE(jiffies_to_sched_qs, j);
	523	+}
529	524
530	525	static int param_set_first_fqs_jiffies(const char val, const struct kernel_param kp)
531	526	{
532	527	ulong j;
533	528	int ret = kstrtoul(val, 0, &j);
534	529
535		- if (!ret)
	530	+ if (!ret) {
536	531	WRITE_ONCE((ulong )kp->arg, (j > HZ) ? HZ : j);
	532	+ adjust_jiffies_till_sched_qs();
	533	+ }
537	534	return ret;
538	535	}
539	536
..	..	@@ -542,8 +539,10 @@
542	539	ulong j;
543	540	int ret = kstrtoul(val, 0, &j);
544	541
545		- if (!ret)
	542	+ if (!ret) {
546	543	WRITE_ONCE((ulong )kp->arg, (j > HZ) ? HZ : (j ?: 1));
	544	+ adjust_jiffies_till_sched_qs();
	545	+ }
547	546	return ret;
548	547	}
549	548
..	..	@@ -561,45 +560,17 @@
561	560	module_param_cb(jiffies_till_next_fqs, &next_fqs_jiffies_ops, &jiffies_till_next_fqs, 0644);
562	561	module_param(rcu_kick_kthreads, bool, 0644);
563	562
564		-/*
565		- * How long the grace period must be before we start recruiting
566		- * quiescent-state help from rcu_note_context_switch().
567		- */
568		-static ulong jiffies_till_sched_qs = HZ / 10;
569		-module_param(jiffies_till_sched_qs, ulong, 0444);
570		-
571		-static void force_qs_rnp(struct rcu_state rsp, int (f)(struct rcu_data *rsp));
572		-static void force_quiescent_state(struct rcu_state *rsp);
573		-static int rcu_pending(void);
	563	+static void force_qs_rnp(int (f)(struct rcu_data rdp));
	564	+static int rcu_pending(int user);
574	565
575	566	/*
576	567	* Return the number of RCU GPs completed thus far for debug & stats.
577	568	*/
578	569	unsigned long rcu_get_gp_seq(void)
579	570	{
580		- return READ_ONCE(rcu_state_p->gp_seq);
	571	+ return READ_ONCE(rcu_state.gp_seq);
581	572	}
582	573	EXPORT_SYMBOL_GPL(rcu_get_gp_seq);
583		-
584		-/*
585		- * Return the number of RCU-sched GPs completed thus far for debug & stats.
586		- */
587		-unsigned long rcu_sched_get_gp_seq(void)
588		-{
589		- return READ_ONCE(rcu_sched_state.gp_seq);
590		-}
591		-EXPORT_SYMBOL_GPL(rcu_sched_get_gp_seq);
592		-
593		-#ifndef CONFIG_PREEMPT_RT_FULL
594		-/*
595		- * Return the number of RCU-bh GPs completed thus far for debug & stats.
596		- */
597		-unsigned long rcu_bh_get_gp_seq(void)
598		-{
599		- return READ_ONCE(rcu_bh_state.gp_seq);
600		-}
601		-EXPORT_SYMBOL_GPL(rcu_bh_get_gp_seq);
602		-#endif
603	574
604	575	/*
605	576	* Return the number of RCU expedited batches completed thus far for
..	..	@@ -609,90 +580,17 @@
609	580	*/
610	581	unsigned long rcu_exp_batches_completed(void)
611	582	{
612		- return rcu_state_p->expedited_sequence;
	583	+ return rcu_state.expedited_sequence;
613	584	}
614	585	EXPORT_SYMBOL_GPL(rcu_exp_batches_completed);
615	586
616	587	/*
617		- * Return the number of RCU-sched expedited batches completed thus far
618		- * for debug & stats. Similar to rcu_exp_batches_completed().
	588	+ * Return the root node of the rcu_state structure.
619	589	*/
620		-unsigned long rcu_exp_batches_completed_sched(void)
	590	+static struct rcu_node *rcu_get_root(void)
621	591	{
622		- return rcu_sched_state.expedited_sequence;
	592	+ return &rcu_state.node[0];
623	593	}
624		-EXPORT_SYMBOL_GPL(rcu_exp_batches_completed_sched);
625		-
626		-#ifndef CONFIG_PREEMPT_RT_FULL
627		-/*
628		- * Force a quiescent state.
629		- */
630		-void rcu_force_quiescent_state(void)
631		-{
632		- force_quiescent_state(rcu_state_p);
633		-}
634		-EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
635		-
636		-/*
637		- * Force a quiescent state for RCU BH.
638		- */
639		-void rcu_bh_force_quiescent_state(void)
640		-{
641		- force_quiescent_state(&rcu_bh_state);
642		-}
643		-EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state);
644		-
645		-#else
646		-void rcu_force_quiescent_state(void)
647		-{
648		-}
649		-EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
650		-#endif
651		-
652		-/*
653		- * Force a quiescent state for RCU-sched.
654		- */
655		-void rcu_sched_force_quiescent_state(void)
656		-{
657		- force_quiescent_state(&rcu_sched_state);
658		-}
659		-EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state);
660		-
661		-/*
662		- * Show the state of the grace-period kthreads.
663		- */
664		-void show_rcu_gp_kthreads(void)
665		-{
666		- int cpu;
667		- struct rcu_data *rdp;
668		- struct rcu_node *rnp;
669		- struct rcu_state *rsp;
670		-
671		- for_each_rcu_flavor(rsp) {
672		- pr_info("%s: wait state: %d ->state: %#lx\n",
673		- rsp->name, rsp->gp_state, rsp->gp_kthread->state);
674		- rcu_for_each_node_breadth_first(rsp, rnp) {
675		- if (ULONG_CMP_GE(rsp->gp_seq, rnp->gp_seq_needed))
676		- continue;
677		- pr_info("\trcu_node %d:%d ->gp_seq %lu ->gp_seq_needed %lu\n",
678		- rnp->grplo, rnp->grphi, rnp->gp_seq,
679		- rnp->gp_seq_needed);
680		- if (!rcu_is_leaf_node(rnp))
681		- continue;
682		- for_each_leaf_node_possible_cpu(rnp, cpu) {
683		- rdp = per_cpu_ptr(rsp->rda, cpu);
684		- if (rdp->gpwrap \|\|
685		- ULONG_CMP_GE(rsp->gp_seq,
686		- rdp->gp_seq_needed))
687		- continue;
688		- pr_info("\tcpu %d ->gp_seq_needed %lu\n",
689		- cpu, rdp->gp_seq_needed);
690		- }
691		- }
692		- /* sched_show_task(rsp->gp_kthread); */
693		- }
694		-}
695		-EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads);
696	594
697	595	/*
698	596	* Send along grace-period-related data for rcutorture diagnostics.
..	..	@@ -700,37 +598,16 @@
700	598	void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags,
701	599	unsigned long *gp_seq)
702	600	{
703		- struct rcu_state *rsp = NULL;
704		-
705	601	switch (test_type) {
706	602	case RCU_FLAVOR:
707		- rsp = rcu_state_p;
708		- break;
709		-#ifndef CONFIG_PREEMPT_RT_FULL
710		- case RCU_BH_FLAVOR:
711		- rsp = &rcu_bh_state;
712		- break;
713		-#endif
714		- case RCU_SCHED_FLAVOR:
715		- rsp = &rcu_sched_state;
	603	+ *flags = READ_ONCE(rcu_state.gp_flags);
	604	+ *gp_seq = rcu_seq_current(&rcu_state.gp_seq);
716	605	break;
717	606	default:
718	607	break;
719	608	}
720		- if (rsp == NULL)
721		- return;
722		- *flags = READ_ONCE(rsp->gp_flags);
723		- *gp_seq = rcu_seq_current(&rsp->gp_seq);
724	609	}
725	610	EXPORT_SYMBOL_GPL(rcutorture_get_gp_data);
726		-
727		-/*
728		- * Return the root node of the specified rcu_state structure.
729		- */
730		-static struct rcu_node rcu_get_root(struct rcu_state rsp)
731		-{
732		- return &rsp->node[0];
733		-}
734	611
735	612	/*
736	613	* Enter an RCU extended quiescent state, which can be either the
..	..	@@ -740,31 +617,36 @@
740	617	* the possibility of usermode upcalls having messed up our count
741	618	* of interrupt nesting level during the prior busy period.
742	619	*/
743		-static void rcu_eqs_enter(bool user)
	620	+static noinstr void rcu_eqs_enter(bool user)
744	621	{
745		- struct rcu_state *rsp;
746		- struct rcu_data *rdp;
747		- struct rcu_dynticks *rdtp;
	622	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
748	623
749		- rdtp = this_cpu_ptr(&rcu_dynticks);
750		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0);
	624	+ WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE);
	625	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, 0);
751	626	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
752		- rdtp->dynticks_nesting == 0);
753		- if (rdtp->dynticks_nesting != 1) {
754		- rdtp->dynticks_nesting--;
	627	+ rdp->dynticks_nesting == 0);
	628	+ if (rdp->dynticks_nesting != 1) {
	629	+ // RCU will still be watching, so just do accounting and leave.
	630	+ rdp->dynticks_nesting--;
755	631	return;
756	632	}
757	633
758	634	lockdep_assert_irqs_disabled();
759		- trace_rcu_dyntick(TPS("Start"), rdtp->dynticks_nesting, 0, rdtp->dynticks);
	635	+ instrumentation_begin();
	636	+ trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks));
760	637	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
761		- for_each_rcu_flavor(rsp) {
762		- rdp = this_cpu_ptr(rsp->rda);
763		- do_nocb_deferred_wakeup(rdp);
764		- }
	638	+ rdp = this_cpu_ptr(&rcu_data);
765	639	rcu_prepare_for_idle();
766		- WRITE_ONCE(rdtp->dynticks_nesting, 0); /* Avoid irq-access tearing. */
	640	+ rcu_preempt_deferred_qs(current);
	641	+
	642	+ // instrumentation for the noinstr rcu_dynticks_eqs_enter()
	643	+ instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
	644	+
	645	+ instrumentation_end();
	646	+ WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */
	647	+ // RCU is watching here ...
767	648	rcu_dynticks_eqs_enter();
	649	+ // ... but is no longer watching here.
768	650	rcu_dynticks_task_enter();
769	651	}
770	652
..	..	@@ -784,6 +666,7 @@
784	666	lockdep_assert_irqs_disabled();
785	667	rcu_eqs_enter(false);
786	668	}
	669	+EXPORT_SYMBOL_GPL(rcu_idle_enter);
787	670
788	671	#ifdef CONFIG_NO_HZ_FULL
789	672	/**
..	..	@@ -797,9 +680,16 @@
797	680	* If you add or remove a call to rcu_user_enter(), be sure to test with
798	681	* CONFIG_RCU_EQS_DEBUG=y.
799	682	*/
800		-void rcu_user_enter(void)
	683	+noinstr void rcu_user_enter(void)
801	684	{
	685	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
	686	+
802	687	lockdep_assert_irqs_disabled();
	688	+
	689	+ instrumentation_begin();
	690	+ do_nocb_deferred_wakeup(rdp);
	691	+ instrumentation_end();
	692	+
803	693	rcu_eqs_enter(true);
804	694	}
805	695	#endif /* CONFIG_NO_HZ_FULL */
..	..	@@ -808,40 +698,56 @@
808	698	* rcu_nmi_exit - inform RCU of exit from NMI context
809	699	*
810	700	* If we are returning from the outermost NMI handler that interrupted an
811		- * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
	701	+ * RCU-idle period, update rdp->dynticks and rdp->dynticks_nmi_nesting
812	702	* to let the RCU grace-period handling know that the CPU is back to
813	703	* being RCU-idle.
814	704	*
815	705	* If you add or remove a call to rcu_nmi_exit(), be sure to test
816	706	* with CONFIG_RCU_EQS_DEBUG=y.
817	707	*/
818		-void rcu_nmi_exit(void)
	708	+noinstr void rcu_nmi_exit(void)
819	709	{
820		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
	710	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
821	711
	712	+ instrumentation_begin();
822	713	/*
823	714	* Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
824	715	* (We are exiting an NMI handler, so RCU better be paying attention
825	716	* to us!)
826	717	*/
827		- WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);
	718	+ WARN_ON_ONCE(rdp->dynticks_nmi_nesting <= 0);
828	719	WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
829	720
830	721	/*
831	722	* If the nesting level is not 1, the CPU wasn't RCU-idle, so
832	723	* leave it in non-RCU-idle state.
833	724	*/
834		- if (rdtp->dynticks_nmi_nesting != 1) {
835		- trace_rcu_dyntick(TPS("--="), rdtp->dynticks_nmi_nesting, rdtp->dynticks_nmi_nesting - 2, rdtp->dynticks);
836		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* No store tearing. */
837		- rdtp->dynticks_nmi_nesting - 2);
	725	+ if (rdp->dynticks_nmi_nesting != 1) {
	726	+ trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2,
	727	+ atomic_read(&rdp->dynticks));
	728	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */
	729	+ rdp->dynticks_nmi_nesting - 2);
	730	+ instrumentation_end();
838	731	return;
839	732	}
840	733
841	734	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
842		- trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, rdtp->dynticks);
843		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
	735	+ trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks));
	736	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
	737	+
	738	+ if (!in_nmi())
	739	+ rcu_prepare_for_idle();
	740	+
	741	+ // instrumentation for the noinstr rcu_dynticks_eqs_enter()
	742	+ instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
	743	+ instrumentation_end();
	744	+
	745	+ // RCU is watching here ...
844	746	rcu_dynticks_eqs_enter();
	747	+ // ... but is no longer watching here.
	748	+
	749	+ if (!in_nmi())
	750	+ rcu_dynticks_task_enter();
845	751	}
846	752
847	753	/**
..	..	@@ -863,17 +769,51 @@
863	769	* If you add or remove a call to rcu_irq_exit(), be sure to test with
864	770	* CONFIG_RCU_EQS_DEBUG=y.
865	771	*/
866		-void rcu_irq_exit(void)
	772	+void noinstr rcu_irq_exit(void)
867	773	{
868		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
869		-
870	774	lockdep_assert_irqs_disabled();
871		- if (rdtp->dynticks_nmi_nesting == 1)
872		- rcu_prepare_for_idle();
873	775	rcu_nmi_exit();
874		- if (rdtp->dynticks_nmi_nesting == 0)
875		- rcu_dynticks_task_enter();
876	776	}
	777	+
	778	+/**
	779	+ * rcu_irq_exit_preempt - Inform RCU that current CPU is exiting irq
	780	+ * towards in kernel preemption
	781	+ *
	782	+ * Same as rcu_irq_exit() but has a sanity check that scheduling is safe
	783	+ * from RCU point of view. Invoked from return from interrupt before kernel
	784	+ * preemption.
	785	+ */
	786	+void rcu_irq_exit_preempt(void)
	787	+{
	788	+ lockdep_assert_irqs_disabled();
	789	+ rcu_nmi_exit();
	790	+
	791	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0,
	792	+ "RCU dynticks_nesting counter underflow/zero!");
	793	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) !=
	794	+ DYNTICK_IRQ_NONIDLE,
	795	+ "Bad RCU dynticks_nmi_nesting counter\n");
	796	+ RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
	797	+ "RCU in extended quiescent state!");
	798	+}
	799	+
	800	+#ifdef CONFIG_PROVE_RCU
	801	+/**
	802	+ * rcu_irq_exit_check_preempt - Validate that scheduling is possible
	803	+ */
	804	+void rcu_irq_exit_check_preempt(void)
	805	+{
	806	+ lockdep_assert_irqs_disabled();
	807	+
	808	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0,
	809	+ "RCU dynticks_nesting counter underflow/zero!");
	810	+ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) !=
	811	+ DYNTICK_IRQ_NONIDLE,
	812	+ "Bad RCU dynticks_nmi_nesting counter\n");
	813	+ RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
	814	+ "RCU in extended quiescent state!");
	815	+}
	816	+#endif /* #ifdef CONFIG_PROVE_RCU */
877	817
878	818	/*
879	819	* Wrapper for rcu_irq_exit() where interrupts are enabled.
..	..	@@ -898,26 +838,36 @@
898	838	* allow for the possibility of usermode upcalls messing up our count of
899	839	* interrupt nesting level during the busy period that is just now starting.
900	840	*/
901		-static void rcu_eqs_exit(bool user)
	841	+static void noinstr rcu_eqs_exit(bool user)
902	842	{
903		- struct rcu_dynticks *rdtp;
	843	+ struct rcu_data *rdp;
904	844	long oldval;
905	845
906	846	lockdep_assert_irqs_disabled();
907		- rdtp = this_cpu_ptr(&rcu_dynticks);
908		- oldval = rdtp->dynticks_nesting;
	847	+ rdp = this_cpu_ptr(&rcu_data);
	848	+ oldval = rdp->dynticks_nesting;
909	849	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
910	850	if (oldval) {
911		- rdtp->dynticks_nesting++;
	851	+ // RCU was already watching, so just do accounting and leave.
	852	+ rdp->dynticks_nesting++;
912	853	return;
913	854	}
914	855	rcu_dynticks_task_exit();
	856	+ // RCU is not watching here ...
915	857	rcu_dynticks_eqs_exit();
	858	+ // ... but is watching here.
	859	+ instrumentation_begin();
	860	+
	861	+ // instrumentation for the noinstr rcu_dynticks_eqs_exit()
	862	+ instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
	863	+
916	864	rcu_cleanup_after_idle();
917		- trace_rcu_dyntick(TPS("End"), rdtp->dynticks_nesting, 1, rdtp->dynticks);
	865	+ trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks));
918	866	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
919		- WRITE_ONCE(rdtp->dynticks_nesting, 1);
920		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
	867	+ WRITE_ONCE(rdp->dynticks_nesting, 1);
	868	+ WARN_ON_ONCE(rdp->dynticks_nmi_nesting);
	869	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
	870	+ instrumentation_end();
921	871	}
922	872
923	873	/**
..	..	@@ -937,6 +887,7 @@
937	887	rcu_eqs_exit(false);
938	888	local_irq_restore(flags);
939	889	}
	890	+EXPORT_SYMBOL_GPL(rcu_idle_exit);
940	891
941	892	#ifdef CONFIG_NO_HZ_FULL
942	893	/**
..	..	@@ -948,17 +899,79 @@
948	899	* If you add or remove a call to rcu_user_exit(), be sure to test with
949	900	* CONFIG_RCU_EQS_DEBUG=y.
950	901	*/
951		-void rcu_user_exit(void)
	902	+void noinstr rcu_user_exit(void)
952	903	{
953	904	rcu_eqs_exit(1);
954	905	}
	906	+
	907	+/**
	908	+ * __rcu_irq_enter_check_tick - Enable scheduler tick on CPU if RCU needs it.
	909	+ *
	910	+ * The scheduler tick is not normally enabled when CPUs enter the kernel
	911	+ * from nohz_full userspace execution. After all, nohz_full userspace
	912	+ * execution is an RCU quiescent state and the time executing in the kernel
	913	+ * is quite short. Except of course when it isn't. And it is not hard to
	914	+ * cause a large system to spend tens of seconds or even minutes looping
	915	+ * in the kernel, which can cause a number of problems, include RCU CPU
	916	+ * stall warnings.
	917	+ *
	918	+ * Therefore, if a nohz_full CPU fails to report a quiescent state
	919	+ * in a timely manner, the RCU grace-period kthread sets that CPU's
	920	+ * ->rcu_urgent_qs flag with the expectation that the next interrupt or
	921	+ * exception will invoke this function, which will turn on the scheduler
	922	+ * tick, which will enable RCU to detect that CPU's quiescent states,
	923	+ * for example, due to cond_resched() calls in CONFIG_PREEMPT=n kernels.
	924	+ * The tick will be disabled once a quiescent state is reported for
	925	+ * this CPU.
	926	+ *
	927	+ * Of course, in carefully tuned systems, there might never be an
	928	+ * interrupt or exception. In that case, the RCU grace-period kthread
	929	+ * will eventually cause one to happen. However, in less carefully
	930	+ * controlled environments, this function allows RCU to get what it
	931	+ * needs without creating otherwise useless interruptions.
	932	+ */
	933	+void __rcu_irq_enter_check_tick(void)
	934	+{
	935	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
	936	+
	937	+ // If we're here from NMI there's nothing to do.
	938	+ if (in_nmi())
	939	+ return;
	940	+
	941	+ RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
	942	+ "Illegal rcu_irq_enter_check_tick() from extended quiescent state");
	943	+
	944	+ if (!tick_nohz_full_cpu(rdp->cpu) \|\|
	945	+ !READ_ONCE(rdp->rcu_urgent_qs) \|\|
	946	+ READ_ONCE(rdp->rcu_forced_tick)) {
	947	+ // RCU doesn't need nohz_full help from this CPU, or it is
	948	+ // already getting that help.
	949	+ return;
	950	+ }
	951	+
	952	+ // We get here only when not in an extended quiescent state and
	953	+ // from interrupts (as opposed to NMIs). Therefore, (1) RCU is
	954	+ // already watching and (2) The fact that we are in an interrupt
	955	+ // handler and that the rcu_node lock is an irq-disabled lock
	956	+ // prevents self-deadlock. So we can safely recheck under the lock.
	957	+ // Note that the nohz_full state currently cannot change.
	958	+ raw_spin_lock_rcu_node(rdp->mynode);
	959	+ if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {
	960	+ // A nohz_full CPU is in the kernel and RCU needs a
	961	+ // quiescent state. Turn on the tick!
	962	+ WRITE_ONCE(rdp->rcu_forced_tick, true);
	963	+ tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
	964	+ }
	965	+ raw_spin_unlock_rcu_node(rdp->mynode);
	966	+}
	967	+NOKPROBE_SYMBOL(__rcu_irq_enter_check_tick);
955	968	#endif /* CONFIG_NO_HZ_FULL */
956	969
957	970	/**
958	971	* rcu_nmi_enter - inform RCU of entry to NMI context
959	972	*
960		- * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
961		- * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
	973	+ * If the CPU was idle from RCU's viewpoint, update rdp->dynticks and
	974	+ * rdp->dynticks_nmi_nesting to let the RCU grace-period handling know
962	975	* that the CPU is active. This implementation permits nested NMIs, as
963	976	* long as the nesting level does not overflow an int. (You will probably
964	977	* run out of stack space first.)
..	..	@@ -966,13 +979,13 @@
966	979	* If you add or remove a call to rcu_nmi_enter(), be sure to test
967	980	* with CONFIG_RCU_EQS_DEBUG=y.
968	981	*/
969		-void rcu_nmi_enter(void)
	982	+noinstr void rcu_nmi_enter(void)
970	983	{
971		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
972	984	long incby = 2;
	985	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
973	986
974	987	/* Complain about underflow. */
975		- WARN_ON_ONCE(rdtp->dynticks_nmi_nesting < 0);
	988	+ WARN_ON_ONCE(rdp->dynticks_nmi_nesting < 0);
976	989
977	990	/*
978	991	* If idle from RCU viewpoint, atomically increment ->dynticks
..	..	@@ -983,14 +996,40 @@
983	996	* period (observation due to Andy Lutomirski).
984	997	*/
985	998	if (rcu_dynticks_curr_cpu_in_eqs()) {
	999	+
	1000	+ if (!in_nmi())
	1001	+ rcu_dynticks_task_exit();
	1002	+
	1003	+ // RCU is not watching here ...
986	1004	rcu_dynticks_eqs_exit();
	1005	+ // ... but is watching here.
	1006	+
	1007	+ if (!in_nmi()) {
	1008	+ instrumentation_begin();
	1009	+ rcu_cleanup_after_idle();
	1010	+ instrumentation_end();
	1011	+ }
	1012	+
	1013	+ instrumentation_begin();
	1014	+ // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs()
	1015	+ instrument_atomic_read(&rdp->dynticks, sizeof(rdp->dynticks));
	1016	+ // instrumentation for the noinstr rcu_dynticks_eqs_exit()
	1017	+ instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
	1018	+
987	1019	incby = 1;
	1020	+ } else if (!in_nmi()) {
	1021	+ instrumentation_begin();
	1022	+ rcu_irq_enter_check_tick();
	1023	+ } else {
	1024	+ instrumentation_begin();
988	1025	}
	1026	+
989	1027	trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
990		- rdtp->dynticks_nmi_nesting,
991		- rdtp->dynticks_nmi_nesting + incby, rdtp->dynticks);
992		- WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* Prevent store tearing. */
993		- rdtp->dynticks_nmi_nesting + incby);
	1028	+ rdp->dynticks_nmi_nesting,
	1029	+ rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks));
	1030	+ instrumentation_end();
	1031	+ WRITE_ONCE(rdp->dynticks_nmi_nesting, /* Prevent store tearing. */
	1032	+ rdp->dynticks_nmi_nesting + incby);
994	1033	barrier();
995	1034	}
996	1035
..	..	@@ -1016,16 +1055,10 @@
1016	1055	* If you add or remove a call to rcu_irq_enter(), be sure to test with
1017	1056	* CONFIG_RCU_EQS_DEBUG=y.
1018	1057	*/
1019		-void rcu_irq_enter(void)
	1058	+noinstr void rcu_irq_enter(void)
1020	1059	{
1021		- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1022		-
1023	1060	lockdep_assert_irqs_disabled();
1024		- if (rdtp->dynticks_nmi_nesting == 0)
1025		- rcu_dynticks_task_exit();
1026	1061	rcu_nmi_enter();
1027		- if (rdtp->dynticks_nmi_nesting == 1)
1028		- rcu_cleanup_after_idle();
1029	1062	}
1030	1063
1031	1064	/*
..	..	@@ -1043,15 +1076,34 @@
1043	1076	local_irq_restore(flags);
1044	1077	}
1045	1078
	1079	+/*
	1080	+ * If any sort of urgency was applied to the current CPU (for example,
	1081	+ * the scheduler-clock interrupt was enabled on a nohz_full CPU) in order
	1082	+ * to get to a quiescent state, disable it.
	1083	+ */
	1084	+static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp)
	1085	+{
	1086	+ raw_lockdep_assert_held_rcu_node(rdp->mynode);
	1087	+ WRITE_ONCE(rdp->rcu_urgent_qs, false);
	1088	+ WRITE_ONCE(rdp->rcu_need_heavy_qs, false);
	1089	+ if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) {
	1090	+ tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
	1091	+ WRITE_ONCE(rdp->rcu_forced_tick, false);
	1092	+ }
	1093	+}
	1094	+
1046	1095	/**
1047		- * rcu_is_watching - see if RCU thinks that the current CPU is idle
	1096	+ * rcu_is_watching - see if RCU thinks that the current CPU is not idle
1048	1097	*
1049	1098	* Return true if RCU is watching the running CPU, which means that this
1050	1099	* CPU can safely enter RCU read-side critical sections. In other words,
1051		- * if the current CPU is in its idle loop and is neither in an interrupt
1052		- * or NMI handler, return true.
	1100	+ * if the current CPU is not in its idle loop or is in an interrupt or
	1101	+ * NMI handler, return true.
	1102	+ *
	1103	+ * Make notrace because it can be called by the internal functions of
	1104	+ * ftrace, and making this notrace removes unnecessary recursion calls.
1053	1105	*/
1054		-bool notrace rcu_is_watching(void)
	1106	+notrace bool rcu_is_watching(void)
1055	1107	{
1056	1108	bool ret;
1057	1109
..	..	@@ -1077,7 +1129,7 @@
1077	1129	cpu = task_cpu(t);
1078	1130	if (!task_curr(t))
1079	1131	return; /* This task is not running on that CPU. */
1080		- smp_store_release(per_cpu_ptr(&rcu_dynticks.rcu_urgent_qs, cpu), true);
	1132	+ smp_store_release(per_cpu_ptr(&rcu_data.rcu_urgent_qs, cpu), true);
1081	1133	}
1082	1134
1083	1135	#if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
..	..	@@ -1088,11 +1140,7 @@
1088	1140	* Disable preemption to avoid false positives that could otherwise
1089	1141	* happen due to the current CPU number being sampled, this task being
1090	1142	* preempted, its old CPU being taken offline, resuming on some other CPU,
1091		- * then determining that its old CPU is now offline. Because there are
1092		- * multiple flavors of RCU, and because this function can be called in the
1093		- * midst of updating the flavors while a given CPU coming online or going
1094		- * offline, it is necessary to check all flavors. If any of the flavors
1095		- * believe that given CPU is online, it is considered to be online.
	1143	+ * then determining that its old CPU is now offline.
1096	1144	*
1097	1145	* Disable checking if in an NMI handler because we cannot safely
1098	1146	* report errors from NMI handlers anyway. In addition, it is OK to use
..	..	@@ -1103,38 +1151,21 @@
1103	1151	{
1104	1152	struct rcu_data *rdp;
1105	1153	struct rcu_node *rnp;
1106		- struct rcu_state *rsp;
	1154	+ bool ret = false;
1107	1155
1108	1156	if (in_nmi() \|\| !rcu_scheduler_fully_active)
1109	1157	return true;
1110		- preempt_disable();
1111		- for_each_rcu_flavor(rsp) {
1112		- rdp = this_cpu_ptr(rsp->rda);
1113		- rnp = rdp->mynode;
1114		- if (rdp->grpmask & rcu_rnp_online_cpus(rnp)) {
1115		- preempt_enable();
1116		- return true;
1117		- }
1118		- }
1119		- preempt_enable();
1120		- return false;
	1158	+ preempt_disable_notrace();
	1159	+ rdp = this_cpu_ptr(&rcu_data);
	1160	+ rnp = rdp->mynode;
	1161	+ if (rdp->grpmask & rcu_rnp_online_cpus(rnp) \|\| READ_ONCE(rnp->ofl_seq) & 0x1)
	1162	+ ret = true;
	1163	+ preempt_enable_notrace();
	1164	+ return ret;
1121	1165	}
1122	1166	EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);
1123	1167
1124	1168	#endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */
1125		-
1126		-/**
1127		- * rcu_is_cpu_rrupt_from_idle - see if idle or immediately interrupted from idle
1128		- *
1129		- * If the current CPU is idle or running at a first-level (not nested)
1130		- * interrupt from idle, return true. The caller must have at least
1131		- * disabled preemption.
1132		- */
1133		-static int rcu_is_cpu_rrupt_from_idle(void)
1134		-{
1135		- return __this_cpu_read(rcu_dynticks.dynticks_nesting) <= 0 &&
1136		- __this_cpu_read(rcu_dynticks.dynticks_nmi_nesting) <= 1;
1137		-}
1138	1169
1139	1170	/*
1140	1171	* We are reporting a quiescent state on behalf of some other CPU, so
..	..	@@ -1160,34 +1191,13 @@
1160	1191	*/
1161	1192	static int dyntick_save_progress_counter(struct rcu_data *rdp)
1162	1193	{
1163		- rdp->dynticks_snap = rcu_dynticks_snap(rdp->dynticks);
	1194	+ rdp->dynticks_snap = rcu_dynticks_snap(rdp);
1164	1195	if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) {
1165		- trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("dti"));
	1196	+ trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
1166	1197	rcu_gpnum_ovf(rdp->mynode, rdp);
1167	1198	return 1;
1168	1199	}
1169	1200	return 0;
1170		-}
1171		-
1172		-/*
1173		- * Handler for the irq_work request posted when a grace period has
1174		- * gone on for too long, but not yet long enough for an RCU CPU
1175		- * stall warning. Set state appropriately, but just complain if
1176		- * there is unexpected state on entry.
1177		- */
1178		-static void rcu_iw_handler(struct irq_work *iwp)
1179		-{
1180		- struct rcu_data *rdp;
1181		- struct rcu_node *rnp;
1182		-
1183		- rdp = container_of(iwp, struct rcu_data, rcu_iw);
1184		- rnp = rdp->mynode;
1185		- raw_spin_lock_rcu_node(rnp);
1186		- if (!WARN_ON_ONCE(!rdp->rcu_iw_pending)) {
1187		- rdp->rcu_iw_gp_seq = rnp->gp_seq;
1188		- rdp->rcu_iw_pending = false;
1189		- }
1190		- raw_spin_unlock_rcu_node(rnp);
1191	1201	}
1192	1202
1193	1203	/*
..	..	@@ -1211,39 +1221,34 @@
1211	1221	* read-side critical section that started before the beginning
1212	1222	* of the current RCU grace period.
1213	1223	*/
1214		- if (rcu_dynticks_in_eqs_since(rdp->dynticks, rdp->dynticks_snap)) {
1215		- trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("dti"));
1216		- rdp->dynticks_fqs++;
	1224	+ if (rcu_dynticks_in_eqs_since(rdp, rdp->dynticks_snap)) {
	1225	+ trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
1217	1226	rcu_gpnum_ovf(rnp, rdp);
1218	1227	return 1;
1219	1228	}
1220	1229
1221	1230	/*
1222		- * Has this CPU encountered a cond_resched() since the beginning
1223		- * of the grace period? For this to be the case, the CPU has to
1224		- * have noticed the current grace period. This might not be the
1225		- * case for nohz_full CPUs looping in the kernel.
	1231	+ * Complain if a CPU that is considered to be offline from RCU's
	1232	+ * perspective has not yet reported a quiescent state. After all,
	1233	+ * the offline CPU should have reported a quiescent state during
	1234	+ * the CPU-offline process, or, failing that, by rcu_gp_init()
	1235	+ * if it ran concurrently with either the CPU going offline or the
	1236	+ * last task on a leaf rcu_node structure exiting its RCU read-side
	1237	+ * critical section while all CPUs corresponding to that structure
	1238	+ * are offline. This added warning detects bugs in any of these
	1239	+ * code paths.
	1240	+ *
	1241	+ * The rcu_node structure's ->lock is held here, which excludes
	1242	+ * the relevant portions the CPU-hotplug code, the grace-period
	1243	+ * initialization code, and the rcu_read_unlock() code paths.
	1244	+ *
	1245	+ * For more detail, please refer to the "Hotplug CPU" section
	1246	+ * of RCU's Requirements documentation.
1226	1247	*/
1227		- jtsq = jiffies_till_sched_qs;
1228		- ruqp = per_cpu_ptr(&rcu_dynticks.rcu_urgent_qs, rdp->cpu);
1229		- if (time_after(jiffies, rdp->rsp->gp_start + jtsq) &&
1230		- READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_dynticks.rcu_qs_ctr, rdp->cpu) &&
1231		- rcu_seq_current(&rdp->gp_seq) == rnp->gp_seq && !rdp->gpwrap) {
1232		- trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("rqc"));
1233		- rcu_gpnum_ovf(rnp, rdp);
1234		- return 1;
1235		- } else if (time_after(jiffies, rdp->rsp->gp_start + jtsq)) {
1236		- /* Load rcu_qs_ctr before store to rcu_urgent_qs. */
1237		- smp_store_release(ruqp, true);
1238		- }
1239		-
1240		- /* If waiting too long on an offline CPU, complain. */
1241		- if (!(rdp->grpmask & rcu_rnp_online_cpus(rnp)) &&
1242		- time_after(jiffies, rdp->rsp->gp_start + HZ)) {
	1248	+ if (WARN_ON_ONCE(!(rdp->grpmask & rcu_rnp_online_cpus(rnp)))) {
1243	1249	bool onl;
1244	1250	struct rcu_node *rnp1;
1245	1251
1246		- WARN_ON(1); /* Offline CPUs are supposed to report QS! */
1247	1252	pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
1248	1253	__func__, rnp->grplo, rnp->grphi, rnp->level,
1249	1254	(long)rnp->gp_seq, (long)rnp->completedqs);
..	..	@@ -1260,44 +1265,63 @@
1260	1265
1261	1266	/*
1262	1267	* A CPU running for an extended time within the kernel can
1263		- * delay RCU grace periods. When the CPU is in NO_HZ_FULL mode,
1264		- * even context-switching back and forth between a pair of
1265		- * in-kernel CPU-bound tasks cannot advance grace periods.
1266		- * So if the grace period is old enough, make the CPU pay attention.
1267		- * Note that the unsynchronized assignments to the per-CPU
1268		- * rcu_need_heavy_qs variable are safe. Yes, setting of
1269		- * bits can be lost, but they will be set again on the next
1270		- * force-quiescent-state pass. So lost bit sets do not result
1271		- * in incorrect behavior, merely in a grace period lasting
1272		- * a few jiffies longer than it might otherwise. Because
1273		- * there are at most four threads involved, and because the
1274		- * updates are only once every few jiffies, the probability of
1275		- * lossage (and thus of slight grace-period extension) is
1276		- * quite low.
	1268	+ * delay RCU grace periods: (1) At age jiffies_to_sched_qs,
	1269	+ * set .rcu_urgent_qs, (2) At age 2*jiffies_to_sched_qs, set
	1270	+ * both .rcu_need_heavy_qs and .rcu_urgent_qs. Note that the
	1271	+ * unsynchronized assignments to the per-CPU rcu_need_heavy_qs
	1272	+ * variable are safe because the assignments are repeated if this
	1273	+ * CPU failed to pass through a quiescent state. This code
	1274	+ * also checks .jiffies_resched in case jiffies_to_sched_qs
	1275	+ * is set way high.
1277	1276	*/
1278		- rnhqp = &per_cpu(rcu_dynticks.rcu_need_heavy_qs, rdp->cpu);
	1277	+ jtsq = READ_ONCE(jiffies_to_sched_qs);
	1278	+ ruqp = per_cpu_ptr(&rcu_data.rcu_urgent_qs, rdp->cpu);
	1279	+ rnhqp = &per_cpu(rcu_data.rcu_need_heavy_qs, rdp->cpu);
1279	1280	if (!READ_ONCE(*rnhqp) &&
1280		- (time_after(jiffies, rdp->rsp->gp_start + jtsq) \|\|
1281		- time_after(jiffies, rdp->rsp->jiffies_resched))) {
	1281	+ (time_after(jiffies, rcu_state.gp_start + jtsq * 2) \|\|
	1282	+ time_after(jiffies, rcu_state.jiffies_resched) \|\|
	1283	+ rcu_state.cbovld)) {
1282	1284	WRITE_ONCE(*rnhqp, true);
1283	1285	/* Store rcu_need_heavy_qs before rcu_urgent_qs. */
1284	1286	smp_store_release(ruqp, true);
1285		- rdp->rsp->jiffies_resched += jtsq; /* Re-enable beating. */
	1287	+ } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) {
	1288	+ WRITE_ONCE(*ruqp, true);
1286	1289	}
1287	1290
1288	1291	/*
1289		- * If more than halfway to RCU CPU stall-warning time, do a
1290		- * resched_cpu() to try to loosen things up a bit. Also check to
1291		- * see if the CPU is getting hammered with interrupts, but only
1292		- * once per grace period, just to keep the IPIs down to a dull roar.
	1292	+ * NO_HZ_FULL CPUs can run in-kernel without rcu_sched_clock_irq!
	1293	+ * The above code handles this, but only for straight cond_resched().
	1294	+ * And some in-kernel loops check need_resched() before calling
	1295	+ * cond_resched(), which defeats the above code for CPUs that are
	1296	+ * running in-kernel with scheduling-clock interrupts disabled.
	1297	+ * So hit them over the head with the resched_cpu() hammer!
1293	1298	*/
1294		- if (jiffies - rdp->rsp->gp_start > rcu_jiffies_till_stall_check() / 2) {
	1299	+ if (tick_nohz_full_cpu(rdp->cpu) &&
	1300	+ (time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) \|\|
	1301	+ rcu_state.cbovld)) {
	1302	+ WRITE_ONCE(*ruqp, true);
1295	1303	resched_cpu(rdp->cpu);
	1304	+ WRITE_ONCE(rdp->last_fqs_resched, jiffies);
	1305	+ }
	1306	+
	1307	+ /*
	1308	+ * If more than halfway to RCU CPU stall-warning time, invoke
	1309	+ * resched_cpu() more frequently to try to loosen things up a bit.
	1310	+ * Also check to see if the CPU is getting hammered with interrupts,
	1311	+ * but only once per grace period, just to keep the IPIs down to
	1312	+ * a dull roar.
	1313	+ */
	1314	+ if (time_after(jiffies, rcu_state.jiffies_resched)) {
	1315	+ if (time_after(jiffies,
	1316	+ READ_ONCE(rdp->last_fqs_resched) + jtsq)) {
	1317	+ resched_cpu(rdp->cpu);
	1318	+ WRITE_ONCE(rdp->last_fqs_resched, jiffies);
	1319	+ }
1296	1320	if (IS_ENABLED(CONFIG_IRQ_WORK) &&
1297	1321	!rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
1298	1322	(rnp->ffmask & rdp->grpmask)) {
1299	1323	init_irq_work(&rdp->rcu_iw, rcu_iw_handler);
1300		- rdp->rcu_iw.flags = IRQ_WORK_HARD_IRQ;
	1324	+ atomic_set(&rdp->rcu_iw.flags, IRQ_WORK_HARD_IRQ);
1301	1325	rdp->rcu_iw_pending = true;
1302	1326	rdp->rcu_iw_gp_seq = rnp->gp_seq;
1303	1327	irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
..	..	@@ -1307,317 +1331,13 @@
1307	1331	return 0;
1308	1332	}
1309	1333
1310		-static void record_gp_stall_check_time(struct rcu_state *rsp)
1311		-{
1312		- unsigned long j = jiffies;
1313		- unsigned long j1;
1314		-
1315		- rsp->gp_start = j;
1316		- j1 = rcu_jiffies_till_stall_check();
1317		- /* Record ->gp_start before ->jiffies_stall. */
1318		- smp_store_release(&rsp->jiffies_stall, j + j1); /* ^^^ */
1319		- rsp->jiffies_resched = j + j1 / 2;
1320		- rsp->n_force_qs_gpstart = READ_ONCE(rsp->n_force_qs);
1321		-}
1322		-
1323		-/*
1324		- * Convert a ->gp_state value to a character string.
1325		- */
1326		-static const char *gp_state_getname(short gs)
1327		-{
1328		- if (gs < 0 \|\| gs >= ARRAY_SIZE(gp_state_names))
1329		- return "???";
1330		- return gp_state_names[gs];
1331		-}
1332		-
1333		-/*
1334		- * Complain about starvation of grace-period kthread.
1335		- */
1336		-static void rcu_check_gp_kthread_starvation(struct rcu_state *rsp)
1337		-{
1338		- unsigned long gpa;
1339		- unsigned long j;
1340		-
1341		- j = jiffies;
1342		- gpa = READ_ONCE(rsp->gp_activity);
1343		- if (j - gpa > 2 * HZ) {
1344		- pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#lx ->cpu=%d\n",
1345		- rsp->name, j - gpa,
1346		- (long)rcu_seq_current(&rsp->gp_seq),
1347		- rsp->gp_flags,
1348		- gp_state_getname(rsp->gp_state), rsp->gp_state,
1349		- rsp->gp_kthread ? rsp->gp_kthread->state : ~0,
1350		- rsp->gp_kthread ? task_cpu(rsp->gp_kthread) : -1);
1351		- if (rsp->gp_kthread) {
1352		- pr_err("RCU grace-period kthread stack dump:\n");
1353		- sched_show_task(rsp->gp_kthread);
1354		- wake_up_process(rsp->gp_kthread);
1355		- }
1356		- }
1357		-}
1358		-
1359		-/*
1360		- * Dump stacks of all tasks running on stalled CPUs. First try using
1361		- * NMIs, but fall back to manual remote stack tracing on architectures
1362		- * that don't support NMI-based stack dumps. The NMI-triggered stack
1363		- * traces are more accurate because they are printed by the target CPU.
1364		- */
1365		-static void rcu_dump_cpu_stacks(struct rcu_state *rsp)
1366		-{
1367		- int cpu;
1368		- unsigned long flags;
1369		- struct rcu_node *rnp;
1370		-
1371		- rcu_for_each_leaf_node(rsp, rnp) {
1372		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
1373		- for_each_leaf_node_possible_cpu(rnp, cpu)
1374		- if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu))
1375		- if (!trigger_single_cpu_backtrace(cpu))
1376		- dump_cpu_task(cpu);
1377		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1378		- }
1379		-}
1380		-
1381		-/*
1382		- * If too much time has passed in the current grace period, and if
1383		- * so configured, go kick the relevant kthreads.
1384		- */
1385		-static void rcu_stall_kick_kthreads(struct rcu_state *rsp)
1386		-{
1387		- unsigned long j;
1388		-
1389		- if (!rcu_kick_kthreads)
1390		- return;
1391		- j = READ_ONCE(rsp->jiffies_kick_kthreads);
1392		- if (time_after(jiffies, j) && rsp->gp_kthread &&
1393		- (rcu_gp_in_progress(rsp) \|\| READ_ONCE(rsp->gp_flags))) {
1394		- WARN_ONCE(1, "Kicking %s grace-period kthread\n", rsp->name);
1395		- rcu_ftrace_dump(DUMP_ALL);
1396		- wake_up_process(rsp->gp_kthread);
1397		- WRITE_ONCE(rsp->jiffies_kick_kthreads, j + HZ);
1398		- }
1399		-}
1400		-
1401		-static void panic_on_rcu_stall(void)
1402		-{
1403		- if (sysctl_panic_on_rcu_stall)
1404		- panic("RCU Stall\n");
1405		-}
1406		-
1407		-static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gp_seq)
1408		-{
1409		- int cpu;
1410		- unsigned long flags;
1411		- unsigned long gpa;
1412		- unsigned long j;
1413		- int ndetected = 0;
1414		- struct rcu_node *rnp = rcu_get_root(rsp);
1415		- long totqlen = 0;
1416		-
1417		- /* Kick and suppress, if so configured. */
1418		- rcu_stall_kick_kthreads(rsp);
1419		- if (rcu_cpu_stall_suppress)
1420		- return;
1421		-
1422		- /*
1423		- * OK, time to rat on our buddy...
1424		- * See Documentation/RCU/stallwarn.txt for info on how to debug
1425		- * RCU CPU stall warnings.
1426		- */
1427		- pr_err("INFO: %s detected stalls on CPUs/tasks:", rsp->name);
1428		- print_cpu_stall_info_begin();
1429		- rcu_for_each_leaf_node(rsp, rnp) {
1430		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
1431		- ndetected += rcu_print_task_stall(rnp);
1432		- if (rnp->qsmask != 0) {
1433		- for_each_leaf_node_possible_cpu(rnp, cpu)
1434		- if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
1435		- print_cpu_stall_info(rsp, cpu);
1436		- ndetected++;
1437		- }
1438		- }
1439		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1440		- }
1441		-
1442		- print_cpu_stall_info_end();
1443		- for_each_possible_cpu(cpu)
1444		- totqlen += rcu_segcblist_n_cbs(&per_cpu_ptr(rsp->rda,
1445		- cpu)->cblist);
1446		- pr_cont("(detected by %d, t=%ld jiffies, g=%ld, q=%lu)\n",
1447		- smp_processor_id(), (long)(jiffies - rsp->gp_start),
1448		- (long)rcu_seq_current(&rsp->gp_seq), totqlen);
1449		- if (ndetected) {
1450		- rcu_dump_cpu_stacks(rsp);
1451		-
1452		- /* Complain about tasks blocking the grace period. */
1453		- rcu_print_detail_task_stall(rsp);
1454		- } else {
1455		- if (rcu_seq_current(&rsp->gp_seq) != gp_seq) {
1456		- pr_err("INFO: Stall ended before state dump start\n");
1457		- } else {
1458		- j = jiffies;
1459		- gpa = READ_ONCE(rsp->gp_activity);
1460		- pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n",
1461		- rsp->name, j - gpa, j, gpa,
1462		- jiffies_till_next_fqs,
1463		- rcu_get_root(rsp)->qsmask);
1464		- /* In this case, the current CPU might be at fault. */
1465		- sched_show_task(current);
1466		- }
1467		- }
1468		- /* Rewrite if needed in case of slow consoles. */
1469		- if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall)))
1470		- WRITE_ONCE(rsp->jiffies_stall,
1471		- jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
1472		-
1473		- rcu_check_gp_kthread_starvation(rsp);
1474		-
1475		- atomic_notifier_call_chain(&rcu_stall_notifier_list, 0, NULL);
1476		-
1477		- panic_on_rcu_stall();
1478		-
1479		- force_quiescent_state(rsp); /* Kick them all. */
1480		-}
1481		-
1482		-static void print_cpu_stall(struct rcu_state *rsp)
1483		-{
1484		- int cpu;
1485		- unsigned long flags;
1486		- struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
1487		- struct rcu_node *rnp = rcu_get_root(rsp);
1488		- long totqlen = 0;
1489		-
1490		- /* Kick and suppress, if so configured. */
1491		- rcu_stall_kick_kthreads(rsp);
1492		- if (rcu_cpu_stall_suppress)
1493		- return;
1494		-
1495		- /*
1496		- * OK, time to rat on ourselves...
1497		- * See Documentation/RCU/stallwarn.txt for info on how to debug
1498		- * RCU CPU stall warnings.
1499		- */
1500		- pr_err("INFO: %s self-detected stall on CPU", rsp->name);
1501		- print_cpu_stall_info_begin();
1502		- raw_spin_lock_irqsave_rcu_node(rdp->mynode, flags);
1503		- print_cpu_stall_info(rsp, smp_processor_id());
1504		- raw_spin_unlock_irqrestore_rcu_node(rdp->mynode, flags);
1505		- print_cpu_stall_info_end();
1506		- for_each_possible_cpu(cpu)
1507		- totqlen += rcu_segcblist_n_cbs(&per_cpu_ptr(rsp->rda,
1508		- cpu)->cblist);
1509		- pr_cont(" (t=%lu jiffies g=%ld q=%lu)\n",
1510		- jiffies - rsp->gp_start,
1511		- (long)rcu_seq_current(&rsp->gp_seq), totqlen);
1512		-
1513		- rcu_check_gp_kthread_starvation(rsp);
1514		-
1515		- rcu_dump_cpu_stacks(rsp);
1516		-
1517		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
1518		- /* Rewrite if needed in case of slow consoles. */
1519		- if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall)))
1520		- WRITE_ONCE(rsp->jiffies_stall,
1521		- jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
1522		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1523		-
1524		- panic_on_rcu_stall();
1525		-
1526		- /*
1527		- * Attempt to revive the RCU machinery by forcing a context switch.
1528		- *
1529		- * A context switch would normally allow the RCU state machine to make
1530		- * progress and it could be we're stuck in kernel space without context
1531		- * switches for an entirely unreasonable amount of time.
1532		- */
1533		- resched_cpu(smp_processor_id());
1534		-}
1535		-
1536		-static void check_cpu_stall(struct rcu_state rsp, struct rcu_data rdp)
1537		-{
1538		- unsigned long gs1;
1539		- unsigned long gs2;
1540		- unsigned long gps;
1541		- unsigned long j;
1542		- unsigned long jn;
1543		- unsigned long js;
1544		- struct rcu_node *rnp;
1545		-
1546		- if ((rcu_cpu_stall_suppress && !rcu_kick_kthreads) \|\|
1547		- !rcu_gp_in_progress(rsp))
1548		- return;
1549		- rcu_stall_kick_kthreads(rsp);
1550		- j = jiffies;
1551		-
1552		- /*
1553		- * Lots of memory barriers to reject false positives.
1554		- *
1555		- * The idea is to pick up rsp->gp_seq, then rsp->jiffies_stall,
1556		- * then rsp->gp_start, and finally another copy of rsp->gp_seq.
1557		- * These values are updated in the opposite order with memory
1558		- * barriers (or equivalent) during grace-period initialization
1559		- * and cleanup. Now, a false positive can occur if we get an new
1560		- * value of rsp->gp_start and a old value of rsp->jiffies_stall.
1561		- * But given the memory barriers, the only way that this can happen
1562		- * is if one grace period ends and another starts between these
1563		- * two fetches. This is detected by comparing the second fetch
1564		- * of rsp->gp_seq with the previous fetch from rsp->gp_seq.
1565		- *
1566		- * Given this check, comparisons of jiffies, rsp->jiffies_stall,
1567		- * and rsp->gp_start suffice to forestall false positives.
1568		- */
1569		- gs1 = READ_ONCE(rsp->gp_seq);
1570		- smp_rmb(); /* Pick up ->gp_seq first... */
1571		- js = READ_ONCE(rsp->jiffies_stall);
1572		- smp_rmb(); /* ...then ->jiffies_stall before the rest... */
1573		- gps = READ_ONCE(rsp->gp_start);
1574		- smp_rmb(); /* ...and finally ->gp_start before ->gp_seq again. */
1575		- gs2 = READ_ONCE(rsp->gp_seq);
1576		- if (gs1 != gs2 \|\|
1577		- ULONG_CMP_LT(j, js) \|\|
1578		- ULONG_CMP_GE(gps, js))
1579		- return; /* No stall or GP completed since entering function. */
1580		- rnp = rdp->mynode;
1581		- jn = jiffies + 3 * rcu_jiffies_till_stall_check() + 3;
1582		- if (rcu_gp_in_progress(rsp) &&
1583		- (READ_ONCE(rnp->qsmask) & rdp->grpmask) &&
1584		- cmpxchg(&rsp->jiffies_stall, js, jn) == js) {
1585		-
1586		- /* We haven't checked in, so go dump stack. */
1587		- print_cpu_stall(rsp);
1588		-
1589		- } else if (rcu_gp_in_progress(rsp) &&
1590		- ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY) &&
1591		- cmpxchg(&rsp->jiffies_stall, js, jn) == js) {
1592		-
1593		- /* They had a few time units to dump stack, so complain. */
1594		- print_other_cpu_stall(rsp, gs2);
1595		- }
1596		-}
1597		-
1598		-/**
1599		- * rcu_cpu_stall_reset - prevent further stall warnings in current grace period
1600		- *
1601		- * Set the stall-warning timeout way off into the future, thus preventing
1602		- * any RCU CPU stall-warning messages from appearing in the current set of
1603		- * RCU grace periods.
1604		- *
1605		- * The caller must disable hard irqs.
1606		- */
1607		-void rcu_cpu_stall_reset(void)
1608		-{
1609		- struct rcu_state *rsp;
1610		-
1611		- for_each_rcu_flavor(rsp)
1612		- WRITE_ONCE(rsp->jiffies_stall, jiffies + ULONG_MAX / 2);
1613		-}
1614		-
1615	1334	/* Trace-event wrapper function for trace_rcu_future_grace_period. */
1616	1335	static void trace_rcu_this_gp(struct rcu_node rnp, struct rcu_data rdp,
1617	1336	unsigned long gp_seq_req, const char *s)
1618	1337	{
1619		- trace_rcu_future_grace_period(rdp->rsp->name, rnp->gp_seq, gp_seq_req,
1620		- rnp->level, rnp->grplo, rnp->grphi, s);
	1338	+ trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
	1339	+ gp_seq_req, rnp->level,
	1340	+ rnp->grplo, rnp->grphi, s);
1621	1341	}
1622	1342
1623	1343	/*
..	..	@@ -1640,7 +1360,6 @@
1640	1360	unsigned long gp_seq_req)
1641	1361	{
1642	1362	bool ret = false;
1643		- struct rcu_state *rsp = rdp->rsp;
1644	1363	struct rcu_node *rnp;
1645	1364
1646	1365	/*
..	..	@@ -1665,7 +1384,7 @@
1665	1384	TPS("Prestarted"));
1666	1385	goto unlock_out;
1667	1386	}
1668		- rnp->gp_seq_needed = gp_seq_req;
	1387	+ WRITE_ONCE(rnp->gp_seq_needed, gp_seq_req);
1669	1388	if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {
1670	1389	/*
1671	1390	* We just marked the leaf or internal node, and a
..	..	@@ -1684,24 +1403,24 @@
1684	1403	}
1685	1404
1686	1405	/* If GP already in progress, just leave, otherwise start one. */
1687		- if (rcu_gp_in_progress(rsp)) {
	1406	+ if (rcu_gp_in_progress()) {
1688	1407	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedleafroot"));
1689	1408	goto unlock_out;
1690	1409	}
1691	1410	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot"));
1692		- WRITE_ONCE(rsp->gp_flags, rsp->gp_flags \| RCU_GP_FLAG_INIT);
1693		- rsp->gp_req_activity = jiffies;
1694		- if (!rsp->gp_kthread) {
	1411	+ WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags \| RCU_GP_FLAG_INIT);
	1412	+ WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
	1413	+ if (!READ_ONCE(rcu_state.gp_kthread)) {
1695	1414	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread"));
1696	1415	goto unlock_out;
1697	1416	}
1698		- trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq), TPS("newreq"));
	1417	+ trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq"));
1699	1418	ret = true; /* Caller must wake GP kthread. */
1700	1419	unlock_out:
1701	1420	/* Push furthest requested GP to leaf node and rcu_data structure. */
1702	1421	if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) {
1703		- rnp_start->gp_seq_needed = rnp->gp_seq_needed;
1704		- rdp->gp_seq_needed = rnp->gp_seq_needed;
	1422	+ WRITE_ONCE(rnp_start->gp_seq_needed, rnp->gp_seq_needed);
	1423	+ WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
1705	1424	}
1706	1425	if (rnp != rnp_start)
1707	1426	raw_spin_unlock_rcu_node(rnp);
..	..	@@ -1712,10 +1431,10 @@
1712	1431	* Clean up any old requests for the just-ended grace period. Also return
1713	1432	* whether any additional grace periods have been requested.
1714	1433	*/
1715		-static bool rcu_future_gp_cleanup(struct rcu_state rsp, struct rcu_node rnp)
	1434	+static bool rcu_future_gp_cleanup(struct rcu_node *rnp)
1716	1435	{
1717	1436	bool needmore;
1718		- struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
	1437	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
1719	1438
1720	1439	needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed);
1721	1440	if (!needmore)
..	..	@@ -1726,12 +1445,13 @@
1726	1445	}
1727	1446
1728	1447	/*
1729		- * Awaken the grace-period kthread. Don't do a self-awaken (unless in
1730		- * an interrupt or softirq handler), and don't bother awakening when there
1731		- * is nothing for the grace-period kthread to do (as in several CPUs raced
1732		- * to awaken, and we lost), and finally don't try to awaken a kthread that
1733		- * has not yet been created. If all those checks are passed, track some
1734		- * debug information and awaken.
	1448	+ * Awaken the grace-period kthread. Don't do a self-awaken (unless in an
	1449	+ * interrupt or softirq handler, in which case we just might immediately
	1450	+ * sleep upon return, resulting in a grace-period hang), and don't bother
	1451	+ * awakening when there is nothing for the grace-period kthread to do
	1452	+ * (as in several CPUs raced to awaken, we lost), and finally don't try
	1453	+ * to awaken a kthread that has not yet been created. If all those checks
	1454	+ * are passed, track some debug information and awaken.
1735	1455	*
1736	1456	* So why do the self-wakeup when in an interrupt or softirq handler
1737	1457	* in the grace-period kthread's context? Because the kthread might have
..	..	@@ -1739,14 +1459,16 @@
1739	1459	* pre-sleep check of the awaken condition. In this case, a wakeup really
1740	1460	* is required, and is therefore supplied.
1741	1461	*/
1742		-static void rcu_gp_kthread_wake(struct rcu_state *rsp)
	1462	+static void rcu_gp_kthread_wake(void)
1743	1463	{
1744		- if ((current == rsp->gp_kthread &&
1745		- !in_interrupt() && !in_serving_softirq()) \|\|
1746		- !READ_ONCE(rsp->gp_flags) \|\|
1747		- !rsp->gp_kthread)
	1464	+ struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
	1465	+
	1466	+ if ((current == t && !in_irq() && !in_serving_softirq()) \|\|
	1467	+ !READ_ONCE(rcu_state.gp_flags) \|\| !t)
1748	1468	return;
1749		- swake_up_one(&rsp->gp_wq);
	1469	+ WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
	1470	+ WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
	1471	+ swake_up_one(&rcu_state.gp_wq);
1750	1472	}
1751	1473
1752	1474	/*
..	..	@@ -1761,12 +1483,12 @@
1761	1483	*
1762	1484	* The caller must hold rnp->lock with interrupts disabled.
1763	1485	*/
1764		-static bool rcu_accelerate_cbs(struct rcu_state rsp, struct rcu_node rnp,
1765		- struct rcu_data *rdp)
	1486	+static bool rcu_accelerate_cbs(struct rcu_node rnp, struct rcu_data rdp)
1766	1487	{
1767	1488	unsigned long gp_seq_req;
1768	1489	bool ret = false;
1769	1490
	1491	+ rcu_lockdep_assert_cblist_protected(rdp);
1770	1492	raw_lockdep_assert_held_rcu_node(rnp);
1771	1493
1772	1494	/* If no pending (not yet ready to invoke) callbacks, nothing to do. */
..	..	@@ -1783,15 +1505,16 @@
1783	1505	* accelerating callback invocation to an earlier grace-period
1784	1506	* number.
1785	1507	*/
1786		- gp_seq_req = rcu_seq_snap(&rsp->gp_seq);
	1508	+ gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);
1787	1509	if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req))
1788	1510	ret = rcu_start_this_gp(rnp, rdp, gp_seq_req);
1789	1511
1790	1512	/* Trace depending on how much we were able to accelerate. */
1791	1513	if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL))
1792		- trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccWaitCB"));
	1514	+ trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB"));
1793	1515	else
1794		- trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccReadyCB"));
	1516	+ trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB"));
	1517	+
1795	1518	return ret;
1796	1519	}
1797	1520
..	..	@@ -1802,25 +1525,24 @@
1802	1525	* that a new grace-period request be made, invokes rcu_accelerate_cbs()
1803	1526	* while holding the leaf rcu_node structure's ->lock.
1804	1527	*/
1805		-static void rcu_accelerate_cbs_unlocked(struct rcu_state *rsp,
1806		- struct rcu_node *rnp,
	1528	+static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp,
1807	1529	struct rcu_data *rdp)
1808	1530	{
1809	1531	unsigned long c;
1810	1532	bool needwake;
1811	1533
1812		- lockdep_assert_irqs_disabled();
1813		- c = rcu_seq_snap(&rsp->gp_seq);
1814		- if (!rdp->gpwrap && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
	1534	+ rcu_lockdep_assert_cblist_protected(rdp);
	1535	+ c = rcu_seq_snap(&rcu_state.gp_seq);
	1536	+ if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
1815	1537	/* Old request still live, so mark recent callbacks. */
1816	1538	(void)rcu_segcblist_accelerate(&rdp->cblist, c);
1817	1539	return;
1818	1540	}
1819	1541	raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
1820		- needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
	1542	+ needwake = rcu_accelerate_cbs(rnp, rdp);
1821	1543	raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
1822	1544	if (needwake)
1823		- rcu_gp_kthread_wake(rsp);
	1545	+ rcu_gp_kthread_wake();
1824	1546	}
1825	1547
1826	1548	/*
..	..	@@ -1833,9 +1555,9 @@
1833	1555	*
1834	1556	* The caller must hold rnp->lock with interrupts disabled.
1835	1557	*/
1836		-static bool rcu_advance_cbs(struct rcu_state rsp, struct rcu_node rnp,
1837		- struct rcu_data *rdp)
	1558	+static bool rcu_advance_cbs(struct rcu_node rnp, struct rcu_data rdp)
1838	1559	{
	1560	+ rcu_lockdep_assert_cblist_protected(rdp);
1839	1561	raw_lockdep_assert_held_rcu_node(rnp);
1840	1562
1841	1563	/* If no pending (not yet ready to invoke) callbacks, nothing to do. */
..	..	@@ -1849,7 +1571,36 @@
1849	1571	rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq);
1850	1572
1851	1573	/* Classify any remaining callbacks. */
1852		- return rcu_accelerate_cbs(rsp, rnp, rdp);
	1574	+ return rcu_accelerate_cbs(rnp, rdp);
	1575	+}
	1576	+
	1577	+/*
	1578	+ * Move and classify callbacks, but only if doing so won't require
	1579	+ * that the RCU grace-period kthread be awakened.
	1580	+ */
	1581	+static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp,
	1582	+ struct rcu_data *rdp)
	1583	+{
	1584	+ rcu_lockdep_assert_cblist_protected(rdp);
	1585	+ if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) \|\| !raw_spin_trylock_rcu_node(rnp))
	1586	+ return;
	1587	+ // The grace period cannot end while we hold the rcu_node lock.
	1588	+ if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))
	1589	+ WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp));
	1590	+ raw_spin_unlock_rcu_node(rnp);
	1591	+}
	1592	+
	1593	+/*
	1594	+ * In CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels, attempt to generate a
	1595	+ * quiescent state. This is intended to be invoked when the CPU notices
	1596	+ * a new grace period.
	1597	+ */
	1598	+static void rcu_strict_gp_check_qs(void)
	1599	+{
	1600	+ if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD)) {
	1601	+ rcu_read_lock();
	1602	+ rcu_read_unlock();
	1603	+ }
1853	1604	}
1854	1605
1855	1606	/*
..	..	@@ -1858,11 +1609,12 @@
1858	1609	* structure corresponding to the current CPU, and must have irqs disabled.
1859	1610	* Returns true if the grace-period kthread needs to be awakened.
1860	1611	*/
1861		-static bool __note_gp_changes(struct rcu_state rsp, struct rcu_node rnp,
1862		- struct rcu_data *rdp)
	1612	+static bool __note_gp_changes(struct rcu_node rnp, struct rcu_data rdp)
1863	1613	{
1864		- bool ret;
1865		- bool need_gp;
	1614	+ bool ret = false;
	1615	+ bool need_qs;
	1616	+ const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	1617	+ rcu_segcblist_is_offloaded(&rdp->cblist);
1866	1618
1867	1619	raw_lockdep_assert_held_rcu_node(rnp);
1868	1620
..	..	@@ -1872,10 +1624,15 @@
1872	1624	/* Handle the ends of any preceding grace periods first. */
1873	1625	if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) \|\|
1874	1626	unlikely(READ_ONCE(rdp->gpwrap))) {
1875		- ret = rcu_advance_cbs(rsp, rnp, rdp); /* Advance callbacks. */
1876		- trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuend"));
	1627	+ if (!offloaded)
	1628	+ ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */
	1629	+ rdp->core_needs_qs = false;
	1630	+ trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));
1877	1631	} else {
1878		- ret = rcu_accelerate_cbs(rsp, rnp, rdp); /* Recent callbacks. */
	1632	+ if (!offloaded)
	1633	+ ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */
	1634	+ if (rdp->core_needs_qs)
	1635	+ rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask);
1879	1636	}
1880	1637
1881	1638	/* Now handle the beginnings of any new-to-this-CPU grace periods. */
..	..	@@ -1886,22 +1643,21 @@
1886	1643	* set up to detect a quiescent state, otherwise don't
1887	1644	* go looking for one.
1888	1645	*/
1889		- trace_rcu_grace_period(rsp->name, rnp->gp_seq, TPS("cpustart"));
1890		- need_gp = !!(rnp->qsmask & rdp->grpmask);
1891		- rdp->cpu_no_qs.b.norm = need_gp;
1892		- rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
1893		- rdp->core_needs_qs = need_gp;
	1646	+ trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));
	1647	+ need_qs = !!(rnp->qsmask & rdp->grpmask);
	1648	+ rdp->cpu_no_qs.b.norm = need_qs;
	1649	+ rdp->core_needs_qs = need_qs;
1894	1650	zero_cpu_stall_ticks(rdp);
1895	1651	}
1896	1652	rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */
1897		- if (ULONG_CMP_GE(rnp->gp_seq_needed, rdp->gp_seq_needed) \|\| rdp->gpwrap)
1898		- rdp->gp_seq_needed = rnp->gp_seq_needed;
	1653	+ if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) \|\| rdp->gpwrap)
	1654	+ WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
1899	1655	WRITE_ONCE(rdp->gpwrap, false);
1900	1656	rcu_gpnum_ovf(rnp, rdp);
1901	1657	return ret;
1902	1658	}
1903	1659
1904		-static void note_gp_changes(struct rcu_state rsp, struct rcu_data rdp)
	1660	+static void note_gp_changes(struct rcu_data *rdp)
1905	1661	{
1906	1662	unsigned long flags;
1907	1663	bool needwake;
..	..	@@ -1915,41 +1671,77 @@
1915	1671	local_irq_restore(flags);
1916	1672	return;
1917	1673	}
1918		- needwake = __note_gp_changes(rsp, rnp, rdp);
	1674	+ needwake = __note_gp_changes(rnp, rdp);
1919	1675	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
	1676	+ rcu_strict_gp_check_qs();
1920	1677	if (needwake)
1921		- rcu_gp_kthread_wake(rsp);
	1678	+ rcu_gp_kthread_wake();
1922	1679	}
1923	1680
1924		-static void rcu_gp_slow(struct rcu_state *rsp, int delay)
	1681	+static void rcu_gp_slow(int delay)
1925	1682	{
1926	1683	if (delay > 0 &&
1927		- !(rcu_seq_ctr(rsp->gp_seq) %
	1684	+ !(rcu_seq_ctr(rcu_state.gp_seq) %
1928	1685	(rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))
1929		- schedule_timeout_uninterruptible(delay);
	1686	+ schedule_timeout_idle(delay);
	1687	+}
	1688	+
	1689	+static unsigned long sleep_duration;
	1690	+
	1691	+/* Allow rcutorture to stall the grace-period kthread. */
	1692	+void rcu_gp_set_torture_wait(int duration)
	1693	+{
	1694	+ if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST) && duration > 0)
	1695	+ WRITE_ONCE(sleep_duration, duration);
	1696	+}
	1697	+EXPORT_SYMBOL_GPL(rcu_gp_set_torture_wait);
	1698	+
	1699	+/* Actually implement the aforementioned wait. */
	1700	+static void rcu_gp_torture_wait(void)
	1701	+{
	1702	+ unsigned long duration;
	1703	+
	1704	+ if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST))
	1705	+ return;
	1706	+ duration = xchg(&sleep_duration, 0UL);
	1707	+ if (duration > 0) {
	1708	+ pr_alert("%s: Waiting %lu jiffies\n", __func__, duration);
	1709	+ schedule_timeout_idle(duration);
	1710	+ pr_alert("%s: Wait complete\n", __func__);
	1711	+ }
	1712	+}
	1713	+
	1714	+/*
	1715	+ * Handler for on_each_cpu() to invoke the target CPU's RCU core
	1716	+ * processing.
	1717	+ */
	1718	+static void rcu_strict_gp_boundary(void *unused)
	1719	+{
	1720	+ invoke_rcu_core();
1930	1721	}
1931	1722
1932	1723	/*
1933	1724	* Initialize a new grace period. Return false if no grace period required.
1934	1725	*/
1935		-static bool rcu_gp_init(struct rcu_state *rsp)
	1726	+static bool rcu_gp_init(void)
1936	1727	{
	1728	+ unsigned long firstseq;
1937	1729	unsigned long flags;
1938	1730	unsigned long oldmask;
1939	1731	unsigned long mask;
1940	1732	struct rcu_data *rdp;
1941		- struct rcu_node *rnp = rcu_get_root(rsp);
	1733	+ struct rcu_node *rnp = rcu_get_root();
1942	1734
1943		- WRITE_ONCE(rsp->gp_activity, jiffies);
	1735	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
1944	1736	raw_spin_lock_irq_rcu_node(rnp);
1945		- if (!READ_ONCE(rsp->gp_flags)) {
	1737	+ if (!READ_ONCE(rcu_state.gp_flags)) {
1946	1738	/* Spurious wakeup, tell caller to go back to sleep. */
1947	1739	raw_spin_unlock_irq_rcu_node(rnp);
1948	1740	return false;
1949	1741	}
1950		- WRITE_ONCE(rsp->gp_flags, 0); /* Clear all flags: New grace period. */
	1742	+ WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */
1951	1743
1952		- if (WARN_ON_ONCE(rcu_gp_in_progress(rsp))) {
	1744	+ if (WARN_ON_ONCE(rcu_gp_in_progress())) {
1953	1745	/*
1954	1746	* Grace period already in progress, don't start another.
1955	1747	* Not supposed to be able to happen.
..	..	@@ -1959,27 +1751,37 @@
1959	1751	}
1960	1752
1961	1753	/* Advance to a new grace period and initialize state. */
1962		- record_gp_stall_check_time(rsp);
	1754	+ record_gp_stall_check_time();
1963	1755	/* Record GP times before starting GP, hence rcu_seq_start(). */
1964		- rcu_seq_start(&rsp->gp_seq);
1965		- trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("start"));
	1756	+ rcu_seq_start(&rcu_state.gp_seq);
	1757	+ ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
	1758	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));
1966	1759	raw_spin_unlock_irq_rcu_node(rnp);
1967	1760
1968	1761	/*
1969		- * Apply per-leaf buffered online and offline operations to the
1970		- * rcu_node tree. Note that this new grace period need not wait
1971		- * for subsequent online CPUs, and that quiescent-state forcing
1972		- * will handle subsequent offline CPUs.
	1762	+ * Apply per-leaf buffered online and offline operations to
	1763	+ * the rcu_node tree. Note that this new grace period need not
	1764	+ * wait for subsequent online CPUs, and that RCU hooks in the CPU
	1765	+ * offlining path, when combined with checks in this function,
	1766	+ * will handle CPUs that are currently going offline or that will
	1767	+ * go offline later. Please also refer to "Hotplug CPU" section
	1768	+ * of RCU's Requirements documentation.
1973	1769	*/
1974		- rsp->gp_state = RCU_GP_ONOFF;
1975		- rcu_for_each_leaf_node(rsp, rnp) {
1976		- spin_lock(&rsp->ofl_lock);
	1770	+ rcu_state.gp_state = RCU_GP_ONOFF;
	1771	+ rcu_for_each_leaf_node(rnp) {
	1772	+ smp_mb(); // Pair with barriers used when updating ->ofl_seq to odd values.
	1773	+ firstseq = READ_ONCE(rnp->ofl_seq);
	1774	+ if (firstseq & 0x1)
	1775	+ while (firstseq == READ_ONCE(rnp->ofl_seq))
	1776	+ schedule_timeout_idle(1); // Can't wake unless RCU is watching.
	1777	+ smp_mb(); // Pair with barriers used when updating ->ofl_seq to even values.
	1778	+ raw_spin_lock(&rcu_state.ofl_lock);
1977	1779	raw_spin_lock_irq_rcu_node(rnp);
1978	1780	if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
1979	1781	!rnp->wait_blkd_tasks) {
1980	1782	/* Nothing to do on this leaf rcu_node structure. */
1981	1783	raw_spin_unlock_irq_rcu_node(rnp);
1982		- spin_unlock(&rsp->ofl_lock);
	1784	+ raw_spin_unlock(&rcu_state.ofl_lock);
1983	1785	continue;
1984	1786	}
1985	1787
..	..	@@ -2015,46 +1817,50 @@
2015	1817	}
2016	1818
2017	1819	raw_spin_unlock_irq_rcu_node(rnp);
2018		- spin_unlock(&rsp->ofl_lock);
	1820	+ raw_spin_unlock(&rcu_state.ofl_lock);
2019	1821	}
2020		- rcu_gp_slow(rsp, gp_preinit_delay); /* Races with CPU hotplug. */
	1822	+ rcu_gp_slow(gp_preinit_delay); /* Races with CPU hotplug. */
2021	1823
2022	1824	/*
2023	1825	* Set the quiescent-state-needed bits in all the rcu_node
2024		- * structures for all currently online CPUs in breadth-first order,
2025		- * starting from the root rcu_node structure, relying on the layout
2026		- * of the tree within the rsp->node[] array. Note that other CPUs
2027		- * will access only the leaves of the hierarchy, thus seeing that no
2028		- * grace period is in progress, at least until the corresponding
2029		- * leaf node has been initialized.
	1826	+ * structures for all currently online CPUs in breadth-first
	1827	+ * order, starting from the root rcu_node structure, relying on the
	1828	+ * layout of the tree within the rcu_state.node[] array. Note that
	1829	+ * other CPUs will access only the leaves of the hierarchy, thus
	1830	+ * seeing that no grace period is in progress, at least until the
	1831	+ * corresponding leaf node has been initialized.
2030	1832	*
2031	1833	* The grace period cannot complete until the initialization
2032	1834	* process finishes, because this kthread handles both.
2033	1835	*/
2034		- rsp->gp_state = RCU_GP_INIT;
2035		- rcu_for_each_node_breadth_first(rsp, rnp) {
2036		- rcu_gp_slow(rsp, gp_init_delay);
	1836	+ rcu_state.gp_state = RCU_GP_INIT;
	1837	+ rcu_for_each_node_breadth_first(rnp) {
	1838	+ rcu_gp_slow(gp_init_delay);
2037	1839	raw_spin_lock_irqsave_rcu_node(rnp, flags);
2038		- rdp = this_cpu_ptr(rsp->rda);
2039		- rcu_preempt_check_blocked_tasks(rsp, rnp);
	1840	+ rdp = this_cpu_ptr(&rcu_data);
	1841	+ rcu_preempt_check_blocked_tasks(rnp);
2040	1842	rnp->qsmask = rnp->qsmaskinit;
2041		- WRITE_ONCE(rnp->gp_seq, rsp->gp_seq);
	1843	+ WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);
2042	1844	if (rnp == rdp->mynode)
2043		- (void)__note_gp_changes(rsp, rnp, rdp);
	1845	+ (void)__note_gp_changes(rnp, rdp);
2044	1846	rcu_preempt_boost_start_gp(rnp);
2045		- trace_rcu_grace_period_init(rsp->name, rnp->gp_seq,
	1847	+ trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,
2046	1848	rnp->level, rnp->grplo,
2047	1849	rnp->grphi, rnp->qsmask);
2048	1850	/* Quiescent states for tasks on any now-offline CPUs. */
2049	1851	mask = rnp->qsmask & ~rnp->qsmaskinitnext;
2050	1852	rnp->rcu_gp_init_mask = mask;
2051	1853	if ((mask \|\| rnp->wait_blkd_tasks) && rcu_is_leaf_node(rnp))
2052		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
	1854	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2053	1855	else
2054	1856	raw_spin_unlock_irq_rcu_node(rnp);
2055	1857	cond_resched_tasks_rcu_qs();
2056		- WRITE_ONCE(rsp->gp_activity, jiffies);
	1858	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
2057	1859	}
	1860	+
	1861	+ // If strict, make all CPUs aware of new grace period.
	1862	+ if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
	1863	+ on_each_cpu(rcu_strict_gp_boundary, NULL, 0);
2058	1864
2059	1865	return true;
2060	1866	}
..	..	@@ -2063,16 +1869,20 @@
2063	1869	* Helper function for swait_event_idle_exclusive() wakeup at force-quiescent-state
2064	1870	* time.
2065	1871	*/
2066		-static bool rcu_gp_fqs_check_wake(struct rcu_state rsp, int gfp)
	1872	+static bool rcu_gp_fqs_check_wake(int *gfp)
2067	1873	{
2068		- struct rcu_node *rnp = rcu_get_root(rsp);
	1874	+ struct rcu_node *rnp = rcu_get_root();
2069	1875
2070		- /* Someone like call_rcu() requested a force-quiescent-state scan. */
2071		- *gfp = READ_ONCE(rsp->gp_flags);
	1876	+ // If under overload conditions, force an immediate FQS scan.
	1877	+ if (*gfp & RCU_GP_FLAG_OVLD)
	1878	+ return true;
	1879	+
	1880	+ // Someone like call_rcu() requested a force-quiescent-state scan.
	1881	+ *gfp = READ_ONCE(rcu_state.gp_flags);
2072	1882	if (*gfp & RCU_GP_FLAG_FQS)
2073	1883	return true;
2074	1884
2075		- /* The current grace period has completed. */
	1885	+ // The current grace period has completed.
2076	1886	if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp))
2077	1887	return true;
2078	1888
..	..	@@ -2082,45 +1892,117 @@
2082	1892	/*
2083	1893	* Do one round of quiescent-state forcing.
2084	1894	*/
2085		-static void rcu_gp_fqs(struct rcu_state *rsp, bool first_time)
	1895	+static void rcu_gp_fqs(bool first_time)
2086	1896	{
2087		- struct rcu_node *rnp = rcu_get_root(rsp);
	1897	+ struct rcu_node *rnp = rcu_get_root();
2088	1898
2089		- WRITE_ONCE(rsp->gp_activity, jiffies);
2090		- rsp->n_force_qs++;
	1899	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	1900	+ WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1);
2091	1901	if (first_time) {
2092	1902	/* Collect dyntick-idle snapshots. */
2093		- force_qs_rnp(rsp, dyntick_save_progress_counter);
	1903	+ force_qs_rnp(dyntick_save_progress_counter);
2094	1904	} else {
2095	1905	/* Handle dyntick-idle and offline CPUs. */
2096		- force_qs_rnp(rsp, rcu_implicit_dynticks_qs);
	1906	+ force_qs_rnp(rcu_implicit_dynticks_qs);
2097	1907	}
2098	1908	/* Clear flag to prevent immediate re-entry. */
2099		- if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
	1909	+ if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
2100	1910	raw_spin_lock_irq_rcu_node(rnp);
2101		- WRITE_ONCE(rsp->gp_flags,
2102		- READ_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS);
	1911	+ WRITE_ONCE(rcu_state.gp_flags,
	1912	+ READ_ONCE(rcu_state.gp_flags) & ~RCU_GP_FLAG_FQS);
2103	1913	raw_spin_unlock_irq_rcu_node(rnp);
	1914	+ }
	1915	+}
	1916	+
	1917	+/*
	1918	+ * Loop doing repeated quiescent-state forcing until the grace period ends.
	1919	+ */
	1920	+static void rcu_gp_fqs_loop(void)
	1921	+{
	1922	+ bool first_gp_fqs;
	1923	+ int gf = 0;
	1924	+ unsigned long j;
	1925	+ int ret;
	1926	+ struct rcu_node *rnp = rcu_get_root();
	1927	+
	1928	+ first_gp_fqs = true;
	1929	+ j = READ_ONCE(jiffies_till_first_fqs);
	1930	+ if (rcu_state.cbovld)
	1931	+ gf = RCU_GP_FLAG_OVLD;
	1932	+ ret = 0;
	1933	+ for (;;) {
	1934	+ if (!ret) {
	1935	+ rcu_state.jiffies_force_qs = jiffies + j;
	1936	+ WRITE_ONCE(rcu_state.jiffies_kick_kthreads,
	1937	+ jiffies + (j ? 3 * j : 2));
	1938	+ }
	1939	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
	1940	+ TPS("fqswait"));
	1941	+ rcu_state.gp_state = RCU_GP_WAIT_FQS;
	1942	+ ret = swait_event_idle_timeout_exclusive(
	1943	+ rcu_state.gp_wq, rcu_gp_fqs_check_wake(&gf), j);
	1944	+ rcu_gp_torture_wait();
	1945	+ rcu_state.gp_state = RCU_GP_DOING_FQS;
	1946	+ /* Locking provides needed memory barriers. */
	1947	+ /* If grace period done, leave loop. */
	1948	+ if (!READ_ONCE(rnp->qsmask) &&
	1949	+ !rcu_preempt_blocked_readers_cgp(rnp))
	1950	+ break;
	1951	+ /* If time for quiescent-state forcing, do it. */
	1952	+ if (!time_after(rcu_state.jiffies_force_qs, jiffies) \|\|
	1953	+ (gf & (RCU_GP_FLAG_FQS \| RCU_GP_FLAG_OVLD))) {
	1954	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
	1955	+ TPS("fqsstart"));
	1956	+ rcu_gp_fqs(first_gp_fqs);
	1957	+ gf = 0;
	1958	+ if (first_gp_fqs) {
	1959	+ first_gp_fqs = false;
	1960	+ gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0;
	1961	+ }
	1962	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
	1963	+ TPS("fqsend"));
	1964	+ cond_resched_tasks_rcu_qs();
	1965	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	1966	+ ret = 0; /* Force full wait till next FQS. */
	1967	+ j = READ_ONCE(jiffies_till_next_fqs);
	1968	+ } else {
	1969	+ /* Deal with stray signal. */
	1970	+ cond_resched_tasks_rcu_qs();
	1971	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	1972	+ WARN_ON(signal_pending(current));
	1973	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
	1974	+ TPS("fqswaitsig"));
	1975	+ ret = 1; /* Keep old FQS timing. */
	1976	+ j = jiffies;
	1977	+ if (time_after(jiffies, rcu_state.jiffies_force_qs))
	1978	+ j = 1;
	1979	+ else
	1980	+ j = rcu_state.jiffies_force_qs - j;
	1981	+ gf = 0;
	1982	+ }
2104	1983	}
2105	1984	}
2106	1985
2107	1986	/*
2108	1987	* Clean up after the old grace period.
2109	1988	*/
2110		-static void rcu_gp_cleanup(struct rcu_state *rsp)
	1989	+static void rcu_gp_cleanup(void)
2111	1990	{
2112		- unsigned long gp_duration;
	1991	+ int cpu;
2113	1992	bool needgp = false;
	1993	+ unsigned long gp_duration;
2114	1994	unsigned long new_gp_seq;
	1995	+ bool offloaded;
2115	1996	struct rcu_data *rdp;
2116		- struct rcu_node *rnp = rcu_get_root(rsp);
	1997	+ struct rcu_node *rnp = rcu_get_root();
2117	1998	struct swait_queue_head *sq;
2118	1999
2119		- WRITE_ONCE(rsp->gp_activity, jiffies);
	2000	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
2120	2001	raw_spin_lock_irq_rcu_node(rnp);
2121		- gp_duration = jiffies - rsp->gp_start;
2122		- if (gp_duration > rsp->gp_max)
2123		- rsp->gp_max = gp_duration;
	2002	+ rcu_state.gp_end = jiffies;
	2003	+ gp_duration = rcu_state.gp_end - rcu_state.gp_start;
	2004	+ if (gp_duration > rcu_state.gp_max)
	2005	+ rcu_state.gp_max = gp_duration;
2124	2006
2125	2007	/*
2126	2008	* We know the grace period is complete, but to everyone else
..	..	@@ -2141,165 +2023,123 @@
2141	2023	* the rcu_node structures before the beginning of the next grace
2142	2024	* period is recorded in any of the rcu_node structures.
2143	2025	*/
2144		- new_gp_seq = rsp->gp_seq;
	2026	+ new_gp_seq = rcu_state.gp_seq;
2145	2027	rcu_seq_end(&new_gp_seq);
2146		- rcu_for_each_node_breadth_first(rsp, rnp) {
	2028	+ rcu_for_each_node_breadth_first(rnp) {
2147	2029	raw_spin_lock_irq_rcu_node(rnp);
2148	2030	if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))
2149		- dump_blkd_tasks(rsp, rnp, 10);
	2031	+ dump_blkd_tasks(rnp, 10);
2150	2032	WARN_ON_ONCE(rnp->qsmask);
2151	2033	WRITE_ONCE(rnp->gp_seq, new_gp_seq);
2152		- rdp = this_cpu_ptr(rsp->rda);
	2034	+ rdp = this_cpu_ptr(&rcu_data);
2153	2035	if (rnp == rdp->mynode)
2154		- needgp = __note_gp_changes(rsp, rnp, rdp) \|\| needgp;
	2036	+ needgp = __note_gp_changes(rnp, rdp) \|\| needgp;
2155	2037	/* smp_mb() provided by prior unlock-lock pair. */
2156		- needgp = rcu_future_gp_cleanup(rsp, rnp) \|\| needgp;
	2038	+ needgp = rcu_future_gp_cleanup(rnp) \|\| needgp;
	2039	+ // Reset overload indication for CPUs no longer overloaded
	2040	+ if (rcu_is_leaf_node(rnp))
	2041	+ for_each_leaf_node_cpu_mask(rnp, cpu, rnp->cbovldmask) {
	2042	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	2043	+ check_cb_ovld_locked(rdp, rnp);
	2044	+ }
2157	2045	sq = rcu_nocb_gp_get(rnp);
2158	2046	raw_spin_unlock_irq_rcu_node(rnp);
2159	2047	rcu_nocb_gp_cleanup(sq);
2160	2048	cond_resched_tasks_rcu_qs();
2161		- WRITE_ONCE(rsp->gp_activity, jiffies);
2162		- rcu_gp_slow(rsp, gp_cleanup_delay);
	2049	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	2050	+ rcu_gp_slow(gp_cleanup_delay);
2163	2051	}
2164		- rnp = rcu_get_root(rsp);
2165		- raw_spin_lock_irq_rcu_node(rnp); /* GP before rsp->gp_seq update. */
	2052	+ rnp = rcu_get_root();
	2053	+ raw_spin_lock_irq_rcu_node(rnp); /* GP before ->gp_seq update. */
2166	2054
2167		- /* Declare grace period done. */
2168		- rcu_seq_end(&rsp->gp_seq);
2169		- trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("end"));
2170		- rsp->gp_state = RCU_GP_IDLE;
	2055	+ /* Declare grace period done, trace first to use old GP number. */
	2056	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));
	2057	+ rcu_seq_end(&rcu_state.gp_seq);
	2058	+ ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
	2059	+ rcu_state.gp_state = RCU_GP_IDLE;
2171	2060	/* Check for GP requests since above loop. */
2172		- rdp = this_cpu_ptr(rsp->rda);
	2061	+ rdp = this_cpu_ptr(&rcu_data);
2173	2062	if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) {
2174	2063	trace_rcu_this_gp(rnp, rdp, rnp->gp_seq_needed,
2175	2064	TPS("CleanupMore"));
2176	2065	needgp = true;
2177	2066	}
2178	2067	/* Advance CBs to reduce false positives below. */
2179		- if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) {
2180		- WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT);
2181		- rsp->gp_req_activity = jiffies;
2182		- trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq),
	2068	+ offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2069	+ rcu_segcblist_is_offloaded(&rdp->cblist);
	2070	+ if ((offloaded \|\| !rcu_accelerate_cbs(rnp, rdp)) && needgp) {
	2071	+ WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);
	2072	+ WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
	2073	+ trace_rcu_grace_period(rcu_state.name,
	2074	+ rcu_state.gp_seq,
2183	2075	TPS("newreq"));
2184	2076	} else {
2185		- WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT);
	2077	+ WRITE_ONCE(rcu_state.gp_flags,
	2078	+ rcu_state.gp_flags & RCU_GP_FLAG_INIT);
2186	2079	}
2187	2080	raw_spin_unlock_irq_rcu_node(rnp);
	2081	+
	2082	+ // If strict, make all CPUs aware of the end of the old grace period.
	2083	+ if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
	2084	+ on_each_cpu(rcu_strict_gp_boundary, NULL, 0);
2188	2085	}
2189	2086
2190	2087	/*
2191	2088	* Body of kthread that handles grace periods.
2192	2089	*/
2193		-static int __noreturn rcu_gp_kthread(void *arg)
	2090	+static int __noreturn rcu_gp_kthread(void *unused)
2194	2091	{
2195		- bool first_gp_fqs;
2196		- int gf;
2197		- unsigned long j;
2198		- int ret;
2199		- struct rcu_state *rsp = arg;
2200		- struct rcu_node *rnp = rcu_get_root(rsp);
2201		-
2202	2092	rcu_bind_gp_kthread();
2203	2093	for (;;) {
2204	2094
2205	2095	/* Handle grace-period start. */
2206	2096	for (;;) {
2207		- trace_rcu_grace_period(rsp->name,
2208		- READ_ONCE(rsp->gp_seq),
	2097	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
2209	2098	TPS("reqwait"));
2210		- rsp->gp_state = RCU_GP_WAIT_GPS;
2211		- swait_event_idle_exclusive(rsp->gp_wq, READ_ONCE(rsp->gp_flags) &
2212		- RCU_GP_FLAG_INIT);
2213		- rsp->gp_state = RCU_GP_DONE_GPS;
	2099	+ rcu_state.gp_state = RCU_GP_WAIT_GPS;
	2100	+ swait_event_idle_exclusive(rcu_state.gp_wq,
	2101	+ READ_ONCE(rcu_state.gp_flags) &
	2102	+ RCU_GP_FLAG_INIT);
	2103	+ rcu_gp_torture_wait();
	2104	+ rcu_state.gp_state = RCU_GP_DONE_GPS;
2214	2105	/* Locking provides needed memory barrier. */
2215		- if (rcu_gp_init(rsp))
	2106	+ if (rcu_gp_init())
2216	2107	break;
2217	2108	cond_resched_tasks_rcu_qs();
2218		- WRITE_ONCE(rsp->gp_activity, jiffies);
	2109	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
2219	2110	WARN_ON(signal_pending(current));
2220		- trace_rcu_grace_period(rsp->name,
2221		- READ_ONCE(rsp->gp_seq),
	2111	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
2222	2112	TPS("reqwaitsig"));
2223	2113	}
2224	2114
2225	2115	/* Handle quiescent-state forcing. */
2226		- first_gp_fqs = true;
2227		- j = jiffies_till_first_fqs;
2228		- ret = 0;
2229		- for (;;) {
2230		- if (!ret) {
2231		- rsp->jiffies_force_qs = jiffies + j;
2232		- WRITE_ONCE(rsp->jiffies_kick_kthreads,
2233		- jiffies + 3 * j);
2234		- }
2235		- trace_rcu_grace_period(rsp->name,
2236		- READ_ONCE(rsp->gp_seq),
2237		- TPS("fqswait"));
2238		- rsp->gp_state = RCU_GP_WAIT_FQS;
2239		- ret = swait_event_idle_timeout_exclusive(rsp->gp_wq,
2240		- rcu_gp_fqs_check_wake(rsp, &gf), j);
2241		- rsp->gp_state = RCU_GP_DOING_FQS;
2242		- /* Locking provides needed memory barriers. */
2243		- /* If grace period done, leave loop. */
2244		- if (!READ_ONCE(rnp->qsmask) &&
2245		- !rcu_preempt_blocked_readers_cgp(rnp))
2246		- break;
2247		- /* If time for quiescent-state forcing, do it. */
2248		- if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) \|\|
2249		- (gf & RCU_GP_FLAG_FQS)) {
2250		- trace_rcu_grace_period(rsp->name,
2251		- READ_ONCE(rsp->gp_seq),
2252		- TPS("fqsstart"));
2253		- rcu_gp_fqs(rsp, first_gp_fqs);
2254		- first_gp_fqs = false;
2255		- trace_rcu_grace_period(rsp->name,
2256		- READ_ONCE(rsp->gp_seq),
2257		- TPS("fqsend"));
2258		- cond_resched_tasks_rcu_qs();
2259		- WRITE_ONCE(rsp->gp_activity, jiffies);
2260		- ret = 0; /* Force full wait till next FQS. */
2261		- j = jiffies_till_next_fqs;
2262		- } else {
2263		- /* Deal with stray signal. */
2264		- cond_resched_tasks_rcu_qs();
2265		- WRITE_ONCE(rsp->gp_activity, jiffies);
2266		- WARN_ON(signal_pending(current));
2267		- trace_rcu_grace_period(rsp->name,
2268		- READ_ONCE(rsp->gp_seq),
2269		- TPS("fqswaitsig"));
2270		- ret = 1; /* Keep old FQS timing. */
2271		- j = jiffies;
2272		- if (time_after(jiffies, rsp->jiffies_force_qs))
2273		- j = 1;
2274		- else
2275		- j = rsp->jiffies_force_qs - j;
2276		- }
2277		- }
	2116	+ rcu_gp_fqs_loop();
2278	2117
2279	2118	/* Handle grace-period end. */
2280		- rsp->gp_state = RCU_GP_CLEANUP;
2281		- rcu_gp_cleanup(rsp);
2282		- rsp->gp_state = RCU_GP_CLEANED;
	2119	+ rcu_state.gp_state = RCU_GP_CLEANUP;
	2120	+ rcu_gp_cleanup();
	2121	+ rcu_state.gp_state = RCU_GP_CLEANED;
2283	2122	}
2284	2123	}
2285	2124
2286	2125	/*
2287		- * Report a full set of quiescent states to the specified rcu_state data
2288		- * structure. Invoke rcu_gp_kthread_wake() to awaken the grace-period
2289		- * kthread if another grace period is required. Whether we wake
2290		- * the grace-period kthread or it awakens itself for the next round
2291		- * of quiescent-state forcing, that kthread will clean up after the
2292		- * just-completed grace period. Note that the caller must hold rnp->lock,
2293		- * which is released before return.
	2126	+ * Report a full set of quiescent states to the rcu_state data structure.
	2127	+ * Invoke rcu_gp_kthread_wake() to awaken the grace-period kthread if
	2128	+ * another grace period is required. Whether we wake the grace-period
	2129	+ * kthread or it awakens itself for the next round of quiescent-state
	2130	+ * forcing, that kthread will clean up after the just-completed grace
	2131	+ * period. Note that the caller must hold rnp->lock, which is released
	2132	+ * before return.
2294	2133	*/
2295		-static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
2296		- __releases(rcu_get_root(rsp)->lock)
	2134	+static void rcu_report_qs_rsp(unsigned long flags)
	2135	+ __releases(rcu_get_root()->lock)
2297	2136	{
2298		- raw_lockdep_assert_held_rcu_node(rcu_get_root(rsp));
2299		- WARN_ON_ONCE(!rcu_gp_in_progress(rsp));
2300		- WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) \| RCU_GP_FLAG_FQS);
2301		- raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(rsp), flags);
2302		- rcu_gp_kthread_wake(rsp);
	2137	+ raw_lockdep_assert_held_rcu_node(rcu_get_root());
	2138	+ WARN_ON_ONCE(!rcu_gp_in_progress());
	2139	+ WRITE_ONCE(rcu_state.gp_flags,
	2140	+ READ_ONCE(rcu_state.gp_flags) \| RCU_GP_FLAG_FQS);
	2141	+ raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(), flags);
	2142	+ rcu_gp_kthread_wake();
2303	2143	}
2304	2144
2305	2145	/*
..	..	@@ -2316,9 +2156,8 @@
2316	2156	* disabled. This allows propagating quiescent state due to resumed tasks
2317	2157	* during grace-period initialization.
2318	2158	*/
2319		-static void
2320		-rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
2321		- struct rcu_node *rnp, unsigned long gps, unsigned long flags)
	2159	+static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
	2160	+ unsigned long gps, unsigned long flags)
2322	2161	__releases(rnp->lock)
2323	2162	{
2324	2163	unsigned long oldmask = 0;
..	..	@@ -2340,8 +2179,8 @@
2340	2179	WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */
2341	2180	WARN_ON_ONCE(!rcu_is_leaf_node(rnp) &&
2342	2181	rcu_preempt_blocked_readers_cgp(rnp));
2343		- rnp->qsmask &= ~mask;
2344		- trace_rcu_quiescent_state_report(rsp->name, rnp->gp_seq,
	2182	+ WRITE_ONCE(rnp->qsmask, rnp->qsmask & ~mask);
	2183	+ trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,
2345	2184	mask, rnp->qsmask, rnp->level,
2346	2185	rnp->grplo, rnp->grphi,
2347	2186	!!rnp->gp_tasks);
..	..	@@ -2363,7 +2202,7 @@
2363	2202	rnp_c = rnp;
2364	2203	rnp = rnp->parent;
2365	2204	raw_spin_lock_irqsave_rcu_node(rnp, flags);
2366		- oldmask = rnp_c->qsmask;
	2205	+ oldmask = READ_ONCE(rnp_c->qsmask);
2367	2206	}
2368	2207
2369	2208	/*
..	..	@@ -2371,19 +2210,18 @@
2371	2210	* state for this grace period. Invoke rcu_report_qs_rsp()
2372	2211	* to clean up and start the next grace period if one is needed.
2373	2212	*/
2374		- rcu_report_qs_rsp(rsp, flags); /* releases rnp->lock. */
	2213	+ rcu_report_qs_rsp(flags); /* releases rnp->lock. */
2375	2214	}
2376	2215
2377	2216	/*
2378	2217	* Record a quiescent state for all tasks that were previously queued
2379	2218	* on the specified rcu_node structure and that were blocking the current
2380		- * RCU grace period. The caller must hold the specified rnp->lock with
	2219	+ * RCU grace period. The caller must hold the corresponding rnp->lock with
2381	2220	* irqs disabled, and this lock is released upon return, but irqs remain
2382	2221	* disabled.
2383	2222	*/
2384	2223	static void __maybe_unused
2385		-rcu_report_unblock_qs_rnp(struct rcu_state *rsp,
2386		- struct rcu_node *rnp, unsigned long flags)
	2224	+rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
2387	2225	__releases(rnp->lock)
2388	2226	{
2389	2227	unsigned long gps;
..	..	@@ -2391,8 +2229,7 @@
2391	2229	struct rcu_node *rnp_p;
2392	2230
2393	2231	raw_lockdep_assert_held_rcu_node(rnp);
2394		- if (WARN_ON_ONCE(rcu_state_p == &rcu_sched_state) \|\|
2395		- WARN_ON_ONCE(rsp != rcu_state_p) \|\|
	2232	+ if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT_RCU)) \|\|
2396	2233	WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)) \|\|
2397	2234	rnp->qsmask != 0) {
2398	2235	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
..	..	@@ -2406,7 +2243,7 @@
2406	2243	* Only one rcu_node structure in the tree, so don't
2407	2244	* try to report up to its nonexistent parent!
2408	2245	*/
2409		- rcu_report_qs_rsp(rsp, flags);
	2246	+ rcu_report_qs_rsp(flags);
2410	2247	return;
2411	2248	}
2412	2249
..	..	@@ -2415,7 +2252,7 @@
2415	2252	mask = rnp->grpmask;
2416	2253	raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
2417	2254	raw_spin_lock_rcu_node(rnp_p); /* irqs already disabled. */
2418		- rcu_report_qs_rnp(mask, rsp, rnp_p, gps, flags);
	2255	+ rcu_report_qs_rnp(mask, rnp_p, gps, flags);
2419	2256	}
2420	2257
2421	2258	/*
..	..	@@ -2423,13 +2260,16 @@
2423	2260	* structure. This must be called from the specified CPU.
2424	2261	*/
2425	2262	static void
2426		-rcu_report_qs_rdp(int cpu, struct rcu_state rsp, struct rcu_data rdp)
	2263	+rcu_report_qs_rdp(struct rcu_data *rdp)
2427	2264	{
2428	2265	unsigned long flags;
2429	2266	unsigned long mask;
2430		- bool needwake;
	2267	+ bool needwake = false;
	2268	+ const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2269	+ rcu_segcblist_is_offloaded(&rdp->cblist);
2431	2270	struct rcu_node *rnp;
2432	2271
	2272	+ WARN_ON_ONCE(rdp->cpu != smp_processor_id());
2433	2273	rnp = rdp->mynode;
2434	2274	raw_spin_lock_irqsave_rcu_node(rnp, flags);
2435	2275	if (rdp->cpu_no_qs.b.norm \|\| rdp->gp_seq != rnp->gp_seq \|\|
..	..	@@ -2442,26 +2282,26 @@
2442	2282	* within the current grace period.
2443	2283	*/
2444	2284	rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */
2445		- rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
2446	2285	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2447	2286	return;
2448	2287	}
2449	2288	mask = rdp->grpmask;
	2289	+ rdp->core_needs_qs = false;
2450	2290	if ((rnp->qsmask & mask) == 0) {
2451	2291	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2452	2292	} else {
2453		- rdp->core_needs_qs = false;
2454		-
2455	2293	/*
2456	2294	* This GP can't end until cpu checks in, so all of our
2457	2295	* callbacks can be processed during the next GP.
2458	2296	*/
2459		- needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
	2297	+ if (!offloaded)
	2298	+ needwake = rcu_accelerate_cbs(rnp, rdp);
2460	2299
2461		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
	2300	+ rcu_disable_urgency_upon_qs(rdp);
	2301	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2462	2302	/* ^^^ Released rnp->lock */
2463	2303	if (needwake)
2464		- rcu_gp_kthread_wake(rsp);
	2304	+ rcu_gp_kthread_wake();
2465	2305	}
2466	2306	}
2467	2307
..	..	@@ -2472,10 +2312,10 @@
2472	2312	* quiescent state for this grace period, and record that fact if so.
2473	2313	*/
2474	2314	static void
2475		-rcu_check_quiescent_state(struct rcu_state rsp, struct rcu_data rdp)
	2315	+rcu_check_quiescent_state(struct rcu_data *rdp)
2476	2316	{
2477	2317	/* Check for grace-period ends and beginnings. */
2478		- note_gp_changes(rsp, rdp);
	2318	+ note_gp_changes(rdp);
2479	2319
2480	2320	/*
2481	2321	* Does this CPU still need to do its part for current grace period?
..	..	@@ -2495,24 +2335,26 @@
2495	2335	* Tell RCU we are done (but rcu_report_qs_rdp() will be the
2496	2336	* judge of that).
2497	2337	*/
2498		- rcu_report_qs_rdp(rdp->cpu, rsp, rdp);
	2338	+ rcu_report_qs_rdp(rdp);
2499	2339	}
2500	2340
2501	2341	/*
2502		- * Trace the fact that this CPU is going offline.
	2342	+ * Near the end of the offline process. Trace the fact that this CPU
	2343	+ * is going offline.
2503	2344	*/
2504		-static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
	2345	+int rcutree_dying_cpu(unsigned int cpu)
2505	2346	{
2506		- RCU_TRACE(bool blkd;)
2507		- RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda);)
2508		- RCU_TRACE(struct rcu_node *rnp = rdp->mynode;)
	2347	+ bool blkd;
	2348	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
	2349	+ struct rcu_node *rnp = rdp->mynode;
2509	2350
2510	2351	if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
2511		- return;
	2352	+ return 0;
2512	2353
2513		- RCU_TRACE(blkd = !!(rnp->qsmask & rdp->grpmask);)
2514		- trace_rcu_grace_period(rsp->name, rnp->gp_seq,
	2354	+ blkd = !!(rnp->qsmask & rdp->grpmask);
	2355	+ trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
2515	2356	blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));
	2357	+ return 0;
2516	2358	}
2517	2359
2518	2360	/*
..	..	@@ -2566,35 +2408,44 @@
2566	2408	* There can only be one CPU hotplug operation at a time, so no need for
2567	2409	* explicit locking.
2568	2410	*/
2569		-static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
	2411	+int rcutree_dead_cpu(unsigned int cpu)
2570	2412	{
2571		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
	2413	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
2572	2414	struct rcu_node rnp = rdp->mynode; / Outgoing CPU's rdp & rnp. */
2573	2415
2574	2416	if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
2575		- return;
	2417	+ return 0;
2576	2418
2577	2419	/* Adjust any no-longer-needed kthreads. */
2578	2420	rcu_boost_kthread_setaffinity(rnp, -1);
	2421	+ /* Do any needed no-CB deferred wakeups from this CPU. */
	2422	+ do_nocb_deferred_wakeup(per_cpu_ptr(&rcu_data, cpu));
	2423	+
	2424	+ // Stop-machine done, so allow nohz_full to disable tick.
	2425	+ tick_dep_clear(TICK_DEP_BIT_RCU);
	2426	+ return 0;
2579	2427	}
2580	2428
2581	2429	/*
2582	2430	* Invoke any RCU callbacks that have made it to the end of their grace
2583	2431	* period. Thottle as specified by rdp->blimit.
2584	2432	*/
2585		-static void rcu_do_batch(struct rcu_state rsp, struct rcu_data rdp)
	2433	+static void rcu_do_batch(struct rcu_data *rdp)
2586	2434	{
	2435	+ int div;
2587	2436	unsigned long flags;
	2437	+ const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2438	+ rcu_segcblist_is_offloaded(&rdp->cblist);
2588	2439	struct rcu_head *rhp;
2589	2440	struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
2590	2441	long bl, count;
	2442	+ long pending, tlimit = 0;
2591	2443
2592	2444	/* If no callbacks are ready, just return. */
2593	2445	if (!rcu_segcblist_ready_cbs(&rdp->cblist)) {
2594		- trace_rcu_batch_start(rsp->name,
2595		- rcu_segcblist_n_lazy_cbs(&rdp->cblist),
	2446	+ trace_rcu_batch_start(rcu_state.name,
2596	2447	rcu_segcblist_n_cbs(&rdp->cblist), 0);
2597		- trace_rcu_batch_end(rsp->name, 0,
	2448	+ trace_rcu_batch_end(rcu_state.name, 0,
2598	2449	!rcu_segcblist_empty(&rdp->cblist),
2599	2450	need_resched(), is_idle_task(current),
2600	2451	rcu_is_callbacks_kthread());
..	..	@@ -2607,32 +2458,76 @@
2607	2458	* callback counts, as rcu_barrier() needs to be conservative.
2608	2459	*/
2609	2460	local_irq_save(flags);
	2461	+ rcu_nocb_lock(rdp);
2610	2462	WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
2611		- bl = rdp->blimit;
2612		- trace_rcu_batch_start(rsp->name, rcu_segcblist_n_lazy_cbs(&rdp->cblist),
	2463	+ pending = rcu_segcblist_n_cbs(&rdp->cblist);
	2464	+ div = READ_ONCE(rcu_divisor);
	2465	+ div = div < 0 ? 7 : div > sizeof(long) * 8 - 2 ? sizeof(long) * 8 - 2 : div;
	2466	+ bl = max(rdp->blimit, pending >> div);
	2467	+ if (in_serving_softirq() && unlikely(bl > 100)) {
	2468	+ long rrn = READ_ONCE(rcu_resched_ns);
	2469	+
	2470	+ rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;
	2471	+ tlimit = local_clock() + rrn;
	2472	+ }
	2473	+ trace_rcu_batch_start(rcu_state.name,
2613	2474	rcu_segcblist_n_cbs(&rdp->cblist), bl);
2614	2475	rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl);
2615		- local_irq_restore(flags);
	2476	+ if (offloaded)
	2477	+ rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
	2478	+ rcu_nocb_unlock_irqrestore(rdp, flags);
2616	2479
2617	2480	/* Invoke callbacks. */
	2481	+ tick_dep_set_task(current, TICK_DEP_BIT_RCU);
2618	2482	rhp = rcu_cblist_dequeue(&rcl);
2619	2483	for (; rhp; rhp = rcu_cblist_dequeue(&rcl)) {
	2484	+ rcu_callback_t f;
	2485	+
2620	2486	debug_rcu_head_unqueue(rhp);
2621		- if (__rcu_reclaim(rsp->name, rhp))
2622		- rcu_cblist_dequeued_lazy(&rcl);
	2487	+
	2488	+ rcu_lock_acquire(&rcu_callback_map);
	2489	+ trace_rcu_invoke_callback(rcu_state.name, rhp);
	2490	+
	2491	+ f = rhp->func;
	2492	+ WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
	2493	+ f(rhp);
	2494	+
	2495	+ rcu_lock_release(&rcu_callback_map);
	2496	+
2623	2497	/*
2624	2498	* Stop only if limit reached and CPU has something to do.
2625	2499	* Note: The rcl structure counts down from zero.
2626	2500	*/
2627		- if (-rcl.len >= bl &&
2628		- (need_resched() \|\|
2629		- (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
2630		- break;
	2501	+ if (in_serving_softirq()) {
	2502	+ if (-rcl.len >= bl && (need_resched() \|\|
	2503	+ (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
	2504	+ break;
	2505	+
	2506	+ /*
	2507	+ * Make sure we don't spend too much time here and deprive other
	2508	+ * softirq vectors of CPU cycles.
	2509	+ */
	2510	+ if (unlikely(tlimit)) {
	2511	+ /* only call local_clock() every 32 callbacks */
	2512	+ if (likely((-rcl.len & 31) \|\| local_clock() < tlimit))
	2513	+ continue;
	2514	+ /* Exceeded the time limit, so leave. */
	2515	+ break;
	2516	+ }
	2517	+ } else {
	2518	+ local_bh_enable();
	2519	+ lockdep_assert_irqs_enabled();
	2520	+ cond_resched_tasks_rcu_qs();
	2521	+ lockdep_assert_irqs_enabled();
	2522	+ local_bh_disable();
	2523	+ }
2631	2524	}
2632	2525
2633	2526	local_irq_save(flags);
	2527	+ rcu_nocb_lock(rdp);
2634	2528	count = -rcl.len;
2635		- trace_rcu_batch_end(rsp->name, count, !!rcl.head, need_resched(),
	2529	+ rdp->n_cbs_invoked += count;
	2530	+ trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),
2636	2531	is_idle_task(current), rcu_is_callbacks_kthread());
2637	2532
2638	2533	/* Update counts and requeue any remaining callbacks. */
..	..	@@ -2642,13 +2537,13 @@
2642	2537
2643	2538	/* Reinstate batch limit if we have worked down the excess. */
2644	2539	count = rcu_segcblist_n_cbs(&rdp->cblist);
2645		- if (rdp->blimit == LONG_MAX && count <= qlowmark)
	2540	+ if (rdp->blimit >= DEFAULT_MAX_RCU_BLIMIT && count <= qlowmark)
2646	2541	rdp->blimit = blimit;
2647	2542
2648	2543	/* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */
2649	2544	if (count == 0 && rdp->qlen_last_fqs_check != 0) {
2650	2545	rdp->qlen_last_fqs_check = 0;
2651		- rdp->n_force_qs_snap = rsp->n_force_qs;
	2546	+ rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
2652	2547	} else if (count < rdp->qlen_last_fqs_check - qhimark)
2653	2548	rdp->qlen_last_fqs_check = count;
2654	2549
..	..	@@ -2656,94 +2551,72 @@
2656	2551	* The following usually indicates a double call_rcu(). To track
2657	2552	* this down, try building with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.
2658	2553	*/
2659		- WARN_ON_ONCE(rcu_segcblist_empty(&rdp->cblist) != (count == 0));
	2554	+ WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist));
	2555	+ WARN_ON_ONCE(!IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2556	+ count != 0 && rcu_segcblist_empty(&rdp->cblist));
2660	2557
2661		- local_irq_restore(flags);
	2558	+ rcu_nocb_unlock_irqrestore(rdp, flags);
2662	2559
2663	2560	/* Re-invoke RCU core processing if there are callbacks remaining. */
2664		- if (rcu_segcblist_ready_cbs(&rdp->cblist))
	2561	+ if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist))
2665	2562	invoke_rcu_core();
	2563	+ tick_dep_clear_task(current, TICK_DEP_BIT_RCU);
2666	2564	}
2667	2565
2668	2566	/*
2669		- * Check to see if this CPU is in a non-context-switch quiescent state
2670		- * (user mode or idle loop for rcu, non-softirq execution for rcu_bh).
2671		- * Also schedule RCU core processing.
2672		- *
2673		- * This function must be called from hardirq context. It is normally
2674		- * invoked from the scheduling-clock interrupt.
	2567	+ * This function is invoked from each scheduling-clock interrupt,
	2568	+ * and checks to see if this CPU is in a non-context-switch quiescent
	2569	+ * state, for example, user mode or idle loop. It also schedules RCU
	2570	+ * core processing. If the current grace period has gone on too long,
	2571	+ * it will ask the scheduler to manufacture a context switch for the sole
	2572	+ * purpose of providing a providing the needed quiescent state.
2675	2573	*/
2676		-void rcu_check_callbacks(int user)
	2574	+void rcu_sched_clock_irq(int user)
2677	2575	{
2678	2576	trace_rcu_utilization(TPS("Start scheduler-tick"));
2679		- increment_cpu_stall_ticks();
2680		- if (user \|\| rcu_is_cpu_rrupt_from_idle()) {
2681		-
2682		- /*
2683		- * Get here if this CPU took its interrupt from user
2684		- * mode or from the idle loop, and if this is not a
2685		- * nested interrupt. In this case, the CPU is in
2686		- * a quiescent state, so note it.
2687		- *
2688		- * No memory barrier is required here because both
2689		- * rcu_sched_qs() and rcu_bh_qs() reference only CPU-local
2690		- * variables that other CPUs neither access nor modify,
2691		- * at least not while the corresponding CPU is online.
2692		- */
2693		-
2694		- rcu_sched_qs();
2695		- rcu_bh_qs();
2696		- rcu_note_voluntary_context_switch(current);
2697		-
2698		- } else if (!in_softirq()) {
2699		-
2700		- /*
2701		- * Get here if this CPU did not take its interrupt from
2702		- * softirq, in other words, if it is not interrupting
2703		- * a rcu_bh read-side critical section. This is an _bh
2704		- * critical section, so note it.
2705		- */
2706		-
2707		- rcu_bh_qs();
2708		- }
2709		- rcu_preempt_check_callbacks();
	2577	+ lockdep_assert_irqs_disabled();
	2578	+ raw_cpu_inc(rcu_data.ticks_this_gp);
2710	2579	/* The load-acquire pairs with the store-release setting to true. */
2711		- if (smp_load_acquire(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs))) {
	2580	+ if (smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) {
2712	2581	/* Idle and userspace execution already are quiescent states. */
2713	2582	if (!rcu_is_cpu_rrupt_from_idle() && !user) {
2714	2583	set_tsk_need_resched(current);
2715	2584	set_preempt_need_resched();
2716	2585	}
2717		- __this_cpu_write(rcu_dynticks.rcu_urgent_qs, false);
	2586	+ __this_cpu_write(rcu_data.rcu_urgent_qs, false);
2718	2587	}
2719		- if (rcu_pending())
	2588	+ rcu_flavor_sched_clock_irq(user);
	2589	+ if (rcu_pending(user))
2720	2590	invoke_rcu_core();
	2591	+ lockdep_assert_irqs_disabled();
2721	2592
2722	2593	trace_rcu_utilization(TPS("End scheduler-tick"));
2723	2594	}
2724	2595
2725	2596	/*
2726		- * Scan the leaf rcu_node structures, processing dyntick state for any that
2727		- * have not yet encountered a quiescent state, using the function specified.
2728		- * Also initiate boosting for any threads blocked on the root rcu_node.
2729		- *
2730		- * The caller must have suppressed start of new grace periods.
	2597	+ * Scan the leaf rcu_node structures. For each structure on which all
	2598	+ * CPUs have reported a quiescent state and on which there are tasks
	2599	+ * blocking the current grace period, initiate RCU priority boosting.
	2600	+ * Otherwise, invoke the specified function to check dyntick state for
	2601	+ * each CPU that has not yet reported a quiescent state.
2731	2602	*/
2732		-static void force_qs_rnp(struct rcu_state rsp, int (f)(struct rcu_data *rsp))
	2603	+static void force_qs_rnp(int (f)(struct rcu_data rdp))
2733	2604	{
2734	2605	int cpu;
2735	2606	unsigned long flags;
2736	2607	unsigned long mask;
	2608	+ struct rcu_data *rdp;
2737	2609	struct rcu_node *rnp;
2738	2610
2739		- rcu_for_each_leaf_node(rsp, rnp) {
	2611	+ rcu_state.cbovld = rcu_state.cbovldnext;
	2612	+ rcu_state.cbovldnext = false;
	2613	+ rcu_for_each_leaf_node(rnp) {
2740	2614	cond_resched_tasks_rcu_qs();
2741	2615	mask = 0;
2742	2616	raw_spin_lock_irqsave_rcu_node(rnp, flags);
	2617	+ rcu_state.cbovldnext \|= !!rnp->cbovldmask;
2743	2618	if (rnp->qsmask == 0) {
2744		- if (rcu_state_p == &rcu_sched_state \|\|
2745		- rsp != rcu_state_p \|\|
2746		- rcu_preempt_blocked_readers_cgp(rnp)) {
	2619	+ if (rcu_preempt_blocked_readers_cgp(rnp)) {
2747	2620	/*
2748	2621	* No point in scanning bits because they
2749	2622	* are all zero. But we might need to
..	..	@@ -2756,16 +2629,16 @@
2756	2629	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2757	2630	continue;
2758	2631	}
2759		- for_each_leaf_node_possible_cpu(rnp, cpu) {
2760		- unsigned long bit = leaf_node_cpu_bit(rnp, cpu);
2761		- if ((rnp->qsmask & bit) != 0) {
2762		- if (f(per_cpu_ptr(rsp->rda, cpu)))
2763		- mask \|= bit;
	2632	+ for_each_leaf_node_cpu_mask(rnp, cpu, rnp->qsmask) {
	2633	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	2634	+ if (f(rdp)) {
	2635	+ mask \|= rdp->grpmask;
	2636	+ rcu_disable_urgency_upon_qs(rdp);
2764	2637	}
2765	2638	}
2766	2639	if (mask != 0) {
2767	2640	/* Idle/offline CPUs, report (releases rnp->lock). */
2768		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
	2641	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
2769	2642	} else {
2770	2643	/* Nothing to do here, so just drop the lock. */
2771	2644	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
..	..	@@ -2777,7 +2650,7 @@
2777	2650	* Force quiescent states on reluctant CPUs, and also detect which
2778	2651	* CPUs are in dyntick-idle mode.
2779	2652	*/
2780		-static void force_quiescent_state(struct rcu_state *rsp)
	2653	+void rcu_force_quiescent_state(void)
2781	2654	{
2782	2655	unsigned long flags;
2783	2656	bool ret;
..	..	@@ -2785,151 +2658,93 @@
2785	2658	struct rcu_node *rnp_old = NULL;
2786	2659
2787	2660	/* Funnel through hierarchy to reduce memory contention. */
2788		- rnp = __this_cpu_read(rsp->rda->mynode);
	2661	+ rnp = raw_cpu_read(rcu_data.mynode);
2789	2662	for (; rnp != NULL; rnp = rnp->parent) {
2790		- ret = (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) \|\|
2791		- !raw_spin_trylock(&rnp->fqslock);
	2663	+ ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) \|\|
	2664	+ !raw_spin_trylock(&rnp->fqslock);
2792	2665	if (rnp_old != NULL)
2793	2666	raw_spin_unlock(&rnp_old->fqslock);
2794	2667	if (ret)
2795	2668	return;
2796	2669	rnp_old = rnp;
2797	2670	}
2798		- /* rnp_old == rcu_get_root(rsp), rnp == NULL. */
	2671	+ /* rnp_old == rcu_get_root(), rnp == NULL. */
2799	2672
2800	2673	/* Reached the root of the rcu_node tree, acquire lock. */
2801	2674	raw_spin_lock_irqsave_rcu_node(rnp_old, flags);
2802	2675	raw_spin_unlock(&rnp_old->fqslock);
2803		- if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
	2676	+ if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
2804	2677	raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
2805	2678	return; /* Someone beat us to it. */
2806	2679	}
2807		- WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) \| RCU_GP_FLAG_FQS);
	2680	+ WRITE_ONCE(rcu_state.gp_flags,
	2681	+ READ_ONCE(rcu_state.gp_flags) \| RCU_GP_FLAG_FQS);
2808	2682	raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
2809		- rcu_gp_kthread_wake(rsp);
	2683	+ rcu_gp_kthread_wake();
	2684	+}
	2685	+EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
	2686	+
	2687	+// Workqueue handler for an RCU reader for kernels enforcing struct RCU
	2688	+// grace periods.
	2689	+static void strict_work_handler(struct work_struct *work)
	2690	+{
	2691	+ rcu_read_lock();
	2692	+ rcu_read_unlock();
2810	2693	}
2811	2694
2812		-/*
2813		- * This function checks for grace-period requests that fail to motivate
2814		- * RCU to come out of its idle mode.
2815		- */
2816		-static void
2817		-rcu_check_gp_start_stall(struct rcu_state rsp, struct rcu_node rnp,
2818		- struct rcu_data *rdp)
2819		-{
2820		- const unsigned long gpssdelay = rcu_jiffies_till_stall_check() * HZ;
2821		- unsigned long flags;
2822		- unsigned long j;
2823		- struct rcu_node *rnp_root = rcu_get_root(rsp);
2824		- static atomic_t warned = ATOMIC_INIT(0);
2825		-
2826		- if (!IS_ENABLED(CONFIG_PROVE_RCU) \|\| rcu_gp_in_progress(rsp) \|\|
2827		- ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed))
2828		- return;
2829		- j = jiffies; /* Expensive access, and in common case don't get here. */
2830		- if (time_before(j, READ_ONCE(rsp->gp_req_activity) + gpssdelay) \|\|
2831		- time_before(j, READ_ONCE(rsp->gp_activity) + gpssdelay) \|\|
2832		- atomic_read(&warned))
2833		- return;
2834		-
2835		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
2836		- j = jiffies;
2837		- if (rcu_gp_in_progress(rsp) \|\|
2838		- ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) \|\|
2839		- time_before(j, READ_ONCE(rsp->gp_req_activity) + gpssdelay) \|\|
2840		- time_before(j, READ_ONCE(rsp->gp_activity) + gpssdelay) \|\|
2841		- atomic_read(&warned)) {
2842		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2843		- return;
2844		- }
2845		- /* Hold onto the leaf lock to make others see warned==1. */
2846		-
2847		- if (rnp_root != rnp)
2848		- raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
2849		- j = jiffies;
2850		- if (rcu_gp_in_progress(rsp) \|\|
2851		- ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) \|\|
2852		- time_before(j, rsp->gp_req_activity + gpssdelay) \|\|
2853		- time_before(j, rsp->gp_activity + gpssdelay) \|\|
2854		- atomic_xchg(&warned, 1)) {
2855		- raw_spin_unlock_rcu_node(rnp_root); /* irqs remain disabled. */
2856		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2857		- return;
2858		- }
2859		- pr_alert("%s: g%ld->%ld gar:%lu ga:%lu f%#x gs:%d %s->state:%#lx\n",
2860		- __func__, (long)READ_ONCE(rsp->gp_seq),
2861		- (long)READ_ONCE(rnp_root->gp_seq_needed),
2862		- j - rsp->gp_req_activity, j - rsp->gp_activity,
2863		- rsp->gp_flags, rsp->gp_state, rsp->name,
2864		- rsp->gp_kthread ? rsp->gp_kthread->state : 0x1ffffL);
2865		- WARN_ON(1);
2866		- if (rnp_root != rnp)
2867		- raw_spin_unlock_rcu_node(rnp_root);
2868		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
2869		-}
2870		-
2871		-/*
2872		- * This does the RCU core processing work for the specified rcu_state
2873		- * and rcu_data structures. This may be called only from the CPU to
2874		- * whom the rdp belongs.
2875		- */
2876		-static void
2877		-__rcu_process_callbacks(struct rcu_state *rsp)
	2695	+/* Perform RCU core processing work for the current CPU. */
	2696	+static __latent_entropy void rcu_core(void)
2878	2697	{
2879	2698	unsigned long flags;
2880		- struct rcu_data *rdp = raw_cpu_ptr(rsp->rda);
	2699	+ struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
2881	2700	struct rcu_node *rnp = rdp->mynode;
2882		-
2883		- WARN_ON_ONCE(!rdp->beenonline);
2884		-
2885		- /* Update RCU state based on any recent quiescent states. */
2886		- rcu_check_quiescent_state(rsp, rdp);
2887		-
2888		- /* No grace period and unregistered callbacks? */
2889		- if (!rcu_gp_in_progress(rsp) &&
2890		- rcu_segcblist_is_enabled(&rdp->cblist)) {
2891		- local_irq_save(flags);
2892		- if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
2893		- rcu_accelerate_cbs_unlocked(rsp, rnp, rdp);
2894		- local_irq_restore(flags);
2895		- }
2896		-
2897		- rcu_check_gp_start_stall(rsp, rnp, rdp);
2898		-
2899		- /* If there are callbacks ready, invoke them. */
2900		- if (rcu_segcblist_ready_cbs(&rdp->cblist))
2901		- invoke_rcu_callbacks(rsp, rdp);
2902		-
2903		- /* Do any needed deferred wakeups of rcuo kthreads. */
2904		- do_nocb_deferred_wakeup(rdp);
2905		-}
2906		-
2907		-/*
2908		- * Do RCU core processing for the current CPU.
2909		- */
2910		-static __latent_entropy void rcu_process_callbacks(void)
2911		-{
2912		- struct rcu_state *rsp;
	2701	+ const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	2702	+ rcu_segcblist_is_offloaded(&rdp->cblist);
2913	2703
2914	2704	if (cpu_is_offline(smp_processor_id()))
2915	2705	return;
2916		- for_each_rcu_flavor(rsp)
2917		- __rcu_process_callbacks(rsp);
	2706	+ trace_rcu_utilization(TPS("Start RCU core"));
	2707	+ WARN_ON_ONCE(!rdp->beenonline);
	2708	+
	2709	+ /* Report any deferred quiescent states if preemption enabled. */
	2710	+ if (!(preempt_count() & PREEMPT_MASK)) {
	2711	+ rcu_preempt_deferred_qs(current);
	2712	+ } else if (rcu_preempt_need_deferred_qs(current)) {
	2713	+ set_tsk_need_resched(current);
	2714	+ set_preempt_need_resched();
	2715	+ }
	2716	+
	2717	+ /* Update RCU state based on any recent quiescent states. */
	2718	+ rcu_check_quiescent_state(rdp);
	2719	+
	2720	+ /* No grace period and unregistered callbacks? */
	2721	+ if (!rcu_gp_in_progress() &&
	2722	+ rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) {
	2723	+ local_irq_save(flags);
	2724	+ if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
	2725	+ rcu_accelerate_cbs_unlocked(rnp, rdp);
	2726	+ local_irq_restore(flags);
	2727	+ }
	2728	+
	2729	+ rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check());
	2730	+
	2731	+ /* If there are callbacks ready, invoke them. */
	2732	+ if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) &&
	2733	+ likely(READ_ONCE(rcu_scheduler_fully_active)))
	2734	+ rcu_do_batch(rdp);
	2735	+
	2736	+ /* Do any needed deferred wakeups of rcuo kthreads. */
	2737	+ do_nocb_deferred_wakeup(rdp);
	2738	+ trace_rcu_utilization(TPS("End RCU core"));
	2739	+
	2740	+ // If strict GPs, schedule an RCU reader in a clean environment.
	2741	+ if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
	2742	+ queue_work_on(rdp->cpu, rcu_gp_wq, &rdp->strict_work);
2918	2743	}
2919	2744
2920		-static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
2921		-/*
2922		- * Schedule RCU callback invocation. If the specified type of RCU
2923		- * does not support RCU priority boosting, just do a direct call,
2924		- * otherwise wake up the per-CPU kernel kthread. Note that because we
2925		- * are running on the current CPU with softirqs disabled, the
2926		- * rcu_cpu_kthread_task cannot disappear out from under us.
2927		- */
2928		-static void invoke_rcu_callbacks(struct rcu_state rsp, struct rcu_data rdp)
	2745	+static void rcu_core_si(struct softirq_action *h)
2929	2746	{
2930		- if (unlikely(!READ_ONCE(rcu_scheduler_fully_active)))
2931		- return;
2932		- rcu_do_batch(rsp, rdp);
	2747	+ rcu_core();
2933	2748	}
2934	2749
2935	2750	static void rcu_wake_cond(struct task_struct *t, int status)
..	..	@@ -2942,56 +2757,63 @@
2942	2757	wake_up_process(t);
2943	2758	}
2944	2759
	2760	+static void invoke_rcu_core_kthread(void)
	2761	+{
	2762	+ struct task_struct *t;
	2763	+ unsigned long flags;
	2764	+
	2765	+ local_irq_save(flags);
	2766	+ __this_cpu_write(rcu_data.rcu_cpu_has_work, 1);
	2767	+ t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task);
	2768	+ if (t != NULL && t != current)
	2769	+ rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status));
	2770	+ local_irq_restore(flags);
	2771	+}
	2772	+
2945	2773	/*
2946	2774	* Wake up this CPU's rcuc kthread to do RCU core processing.
2947	2775	*/
2948	2776	static void invoke_rcu_core(void)
2949	2777	{
2950		- unsigned long flags;
2951		- struct task_struct *t;
2952		-
2953	2778	if (!cpu_online(smp_processor_id()))
2954	2779	return;
2955		- local_irq_save(flags);
2956		- __this_cpu_write(rcu_cpu_has_work, 1);
2957		- t = __this_cpu_read(rcu_cpu_kthread_task);
2958		- if (t != NULL && current != t)
2959		- rcu_wake_cond(t, __this_cpu_read(rcu_cpu_kthread_status));
2960		- local_irq_restore(flags);
	2780	+ if (use_softirq)
	2781	+ raise_softirq(RCU_SOFTIRQ);
	2782	+ else
	2783	+ invoke_rcu_core_kthread();
2961	2784	}
2962	2785
2963	2786	static void rcu_cpu_kthread_park(unsigned int cpu)
2964	2787	{
2965		- per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
	2788	+ per_cpu(rcu_data.rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
2966	2789	}
2967	2790
2968	2791	static int rcu_cpu_kthread_should_run(unsigned int cpu)
2969	2792	{
2970		- return __this_cpu_read(rcu_cpu_has_work);
	2793	+ return __this_cpu_read(rcu_data.rcu_cpu_has_work);
2971	2794	}
2972	2795
2973	2796	/*
2974		- * Per-CPU kernel thread that invokes RCU callbacks. This replaces the
2975		- * RCU softirq used in flavors and configurations of RCU that do not
2976		- * support RCU priority boosting.
	2797	+ * Per-CPU kernel thread that invokes RCU callbacks. This replaces
	2798	+ * the RCU softirq used in configurations of RCU that do not support RCU
	2799	+ * priority boosting.
2977	2800	*/
2978	2801	static void rcu_cpu_kthread(unsigned int cpu)
2979	2802	{
2980		- unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status);
2981		- char work, *workp = this_cpu_ptr(&rcu_cpu_has_work);
	2803	+ unsigned int *statusp = this_cpu_ptr(&rcu_data.rcu_cpu_kthread_status);
	2804	+ char work, *workp = this_cpu_ptr(&rcu_data.rcu_cpu_has_work);
2982	2805	int spincnt;
2983	2806
	2807	+ trace_rcu_utilization(TPS("Start CPU kthread@rcu_run"));
2984	2808	for (spincnt = 0; spincnt < 10; spincnt++) {
2985		- trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
2986	2809	local_bh_disable();
2987	2810	*statusp = RCU_KTHREAD_RUNNING;
2988		- this_cpu_inc(rcu_cpu_kthread_loops);
2989	2811	local_irq_disable();
2990	2812	work = *workp;
2991	2813	*workp = 0;
2992	2814	local_irq_enable();
2993	2815	if (work)
2994		- rcu_process_callbacks();
	2816	+ rcu_core();
2995	2817	local_bh_enable();
2996	2818	if (*workp == 0) {
2997	2819	trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
..	..	@@ -3001,13 +2823,13 @@
3001	2823	}
3002	2824	*statusp = RCU_KTHREAD_YIELDING;
3003	2825	trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
3004		- schedule_timeout_interruptible(2);
	2826	+ schedule_timeout_idle(2);
3005	2827	trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
3006	2828	*statusp = RCU_KTHREAD_WAITING;
3007	2829	}
3008	2830
3009	2831	static struct smp_hotplug_thread rcu_cpu_thread_spec = {
3010		- .store = &rcu_cpu_kthread_task,
	2832	+ .store = &rcu_data.rcu_cpu_kthread_task,
3011	2833	.thread_should_run = rcu_cpu_kthread_should_run,
3012	2834	.thread_fn = rcu_cpu_kthread,
3013	2835	.thread_comm = "rcuc/%u",
..	..	@@ -3023,17 +2845,19 @@
3023	2845	int cpu;
3024	2846
3025	2847	for_each_possible_cpu(cpu)
3026		- per_cpu(rcu_cpu_has_work, cpu) = 0;
3027		- BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
	2848	+ per_cpu(rcu_data.rcu_cpu_has_work, cpu) = 0;
	2849	+ if (!IS_ENABLED(CONFIG_RCU_BOOST) && use_softirq)
	2850	+ return 0;
	2851	+ WARN_ONCE(smpboot_register_percpu_thread(&rcu_cpu_thread_spec),
	2852	+ "%s: Could not start rcuc kthread, OOM is now expected behavior\n", __func__);
3028	2853	return 0;
3029	2854	}
3030		-early_initcall(rcu_spawn_core_kthreads);
3031	2855
3032	2856	/*
3033	2857	* Handle any core-RCU processing required by a call_rcu() invocation.
3034	2858	*/
3035		-static void __call_rcu_core(struct rcu_state rsp, struct rcu_data rdp,
3036		- struct rcu_head *head, unsigned long flags)
	2859	+static void __call_rcu_core(struct rcu_data rdp, struct rcu_head head,
	2860	+ unsigned long flags)
3037	2861	{
3038	2862	/*
3039	2863	* If called from an extended quiescent state, invoke the RCU
..	..	@@ -3048,27 +2872,27 @@
3048	2872
3049	2873	/*
3050	2874	* Force the grace period if too many callbacks or too long waiting.
3051		- * Enforce hysteresis, and don't invoke force_quiescent_state()
	2875	+ * Enforce hysteresis, and don't invoke rcu_force_quiescent_state()
3052	2876	* if some other CPU has recently done so. Also, don't bother
3053		- * invoking force_quiescent_state() if the newly enqueued callback
	2877	+ * invoking rcu_force_quiescent_state() if the newly enqueued callback
3054	2878	* is the only one waiting for a grace period to complete.
3055	2879	*/
3056	2880	if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) >
3057	2881	rdp->qlen_last_fqs_check + qhimark)) {
3058	2882
3059	2883	/* Are we ignoring a completed grace period? */
3060		- note_gp_changes(rsp, rdp);
	2884	+ note_gp_changes(rdp);
3061	2885
3062	2886	/* Start a new grace period if one not already started. */
3063		- if (!rcu_gp_in_progress(rsp)) {
3064		- rcu_accelerate_cbs_unlocked(rsp, rdp->mynode, rdp);
	2887	+ if (!rcu_gp_in_progress()) {
	2888	+ rcu_accelerate_cbs_unlocked(rdp->mynode, rdp);
3065	2889	} else {
3066	2890	/* Give the grace period a kick. */
3067		- rdp->blimit = LONG_MAX;
3068		- if (rsp->n_force_qs == rdp->n_force_qs_snap &&
	2891	+ rdp->blimit = DEFAULT_MAX_RCU_BLIMIT;
	2892	+ if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap &&
3069	2893	rcu_segcblist_first_pend_cb(&rdp->cblist) != head)
3070		- force_quiescent_state(rsp);
3071		- rdp->n_force_qs_snap = rsp->n_force_qs;
	2894	+ rcu_force_quiescent_state();
	2895	+ rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
3072	2896	rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
3073	2897	}
3074	2898	}
..	..	@@ -3082,17 +2906,54 @@
3082	2906	}
3083	2907
3084	2908	/*
3085		- * Helper function for call_rcu() and friends. The cpu argument will
3086		- * normally be -1, indicating "currently running CPU". It may specify
3087		- * a CPU only if that CPU is a no-CBs CPU. Currently, only _rcu_barrier()
3088		- * is expected to specify a CPU.
	2909	+ * Check and if necessary update the leaf rcu_node structure's
	2910	+ * ->cbovldmask bit corresponding to the current CPU based on that CPU's
	2911	+ * number of queued RCU callbacks. The caller must hold the leaf rcu_node
	2912	+ * structure's ->lock.
3089	2913	*/
	2914	+static void check_cb_ovld_locked(struct rcu_data rdp, struct rcu_node rnp)
	2915	+{
	2916	+ raw_lockdep_assert_held_rcu_node(rnp);
	2917	+ if (qovld_calc <= 0)
	2918	+ return; // Early boot and wildcard value set.
	2919	+ if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc)
	2920	+ WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask \| rdp->grpmask);
	2921	+ else
	2922	+ WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask & ~rdp->grpmask);
	2923	+}
	2924	+
	2925	+/*
	2926	+ * Check and if necessary update the leaf rcu_node structure's
	2927	+ * ->cbovldmask bit corresponding to the current CPU based on that CPU's
	2928	+ * number of queued RCU callbacks. No locks need be held, but the
	2929	+ * caller must have disabled interrupts.
	2930	+ *
	2931	+ * Note that this function ignores the possibility that there are a lot
	2932	+ * of callbacks all of which have already seen the end of their respective
	2933	+ * grace periods. This omission is due to the need for no-CBs CPUs to
	2934	+ * be holding ->nocb_lock to do this check, which is too heavy for a
	2935	+ * common-case operation.
	2936	+ */
	2937	+static void check_cb_ovld(struct rcu_data *rdp)
	2938	+{
	2939	+ struct rcu_node *const rnp = rdp->mynode;
	2940	+
	2941	+ if (qovld_calc <= 0 \|\|
	2942	+ ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) ==
	2943	+ !!(READ_ONCE(rnp->cbovldmask) & rdp->grpmask)))
	2944	+ return; // Early boot wildcard value or already set correctly.
	2945	+ raw_spin_lock_rcu_node(rnp);
	2946	+ check_cb_ovld_locked(rdp, rnp);
	2947	+ raw_spin_unlock_rcu_node(rnp);
	2948	+}
	2949	+
	2950	+/* Helper function for call_rcu() and friends. */
3090	2951	static void
3091		-__call_rcu(struct rcu_head *head, rcu_callback_t func,
3092		- struct rcu_state *rsp, int cpu, bool lazy)
	2952	+__call_rcu(struct rcu_head *head, rcu_callback_t func)
3093	2953	{
3094	2954	unsigned long flags;
3095	2955	struct rcu_data *rdp;
	2956	+ bool was_alldone;
3096	2957
3097	2958	/* Misaligned rcu_head! */
3098	2959	WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1));
..	..	@@ -3103,7 +2964,7 @@
3103	2964	* Use rcu:rcu_callback trace event to find the previous
3104	2965	* time callback was passed to __call_rcu().
3105	2966	*/
3106		- WARN_ONCE(1, "__call_rcu(): Double-freed CB %p->%pF()!!!\n",
	2967	+ WARN_ONCE(1, "__call_rcu(): Double-freed CB %p->%pS()!!!\n",
3107	2968	head, head->func);
3108	2969	WRITE_ONCE(head->func, rcu_leak_callback);
3109	2970	return;
..	..	@@ -3111,132 +2972,692 @@
3111	2972	head->func = func;
3112	2973	head->next = NULL;
3113	2974	local_irq_save(flags);
3114		- rdp = this_cpu_ptr(rsp->rda);
	2975	+ kasan_record_aux_stack(head);
	2976	+ rdp = this_cpu_ptr(&rcu_data);
3115	2977
3116	2978	/* Add the callback to our list. */
3117		- if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist)) \|\| cpu != -1) {
3118		- int offline;
3119		-
3120		- if (cpu != -1)
3121		- rdp = per_cpu_ptr(rsp->rda, cpu);
3122		- if (likely(rdp->mynode)) {
3123		- /* Post-boot, so this should be for a no-CBs CPU. */
3124		- offline = !__call_rcu_nocb(rdp, head, lazy, flags);
3125		- WARN_ON_ONCE(offline);
3126		- /* Offline CPU, _call_rcu() illegal, leak callback. */
3127		- local_irq_restore(flags);
3128		- return;
3129		- }
3130		- /*
3131		- * Very early boot, before rcu_init(). Initialize if needed
3132		- * and then drop through to queue the callback.
3133		- */
3134		- BUG_ON(cpu != -1);
	2979	+ if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) {
	2980	+ // This can trigger due to call_rcu() from offline CPU:
	2981	+ WARN_ON_ONCE(rcu_scheduler_active != RCU_SCHEDULER_INACTIVE);
3135	2982	WARN_ON_ONCE(!rcu_is_watching());
	2983	+ // Very early boot, before rcu_init(). Initialize if needed
	2984	+ // and then drop through to queue the callback.
3136	2985	if (rcu_segcblist_empty(&rdp->cblist))
3137	2986	rcu_segcblist_init(&rdp->cblist);
3138	2987	}
3139		- rcu_segcblist_enqueue(&rdp->cblist, head, lazy);
3140		- if (!lazy)
3141		- rcu_idle_count_callbacks_posted();
3142	2988
3143		- if (__is_kfree_rcu_offset((unsigned long)func))
3144		- trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func,
3145		- rcu_segcblist_n_lazy_cbs(&rdp->cblist),
	2989	+ check_cb_ovld(rdp);
	2990	+ if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags))
	2991	+ return; // Enqueued onto ->nocb_bypass, so just leave.
	2992	+ // If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock.
	2993	+ rcu_segcblist_enqueue(&rdp->cblist, head);
	2994	+ if (__is_kvfree_rcu_offset((unsigned long)func))
	2995	+ trace_rcu_kvfree_callback(rcu_state.name, head,
	2996	+ (unsigned long)func,
3146	2997	rcu_segcblist_n_cbs(&rdp->cblist));
3147	2998	else
3148		- trace_rcu_callback(rsp->name, head,
3149		- rcu_segcblist_n_lazy_cbs(&rdp->cblist),
	2999	+ trace_rcu_callback(rcu_state.name, head,
3150	3000	rcu_segcblist_n_cbs(&rdp->cblist));
3151	3001
3152	3002	/* Go handle any RCU core processing required. */
3153		- __call_rcu_core(rsp, rdp, head, flags);
	3003	+ if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
	3004	+ unlikely(rcu_segcblist_is_offloaded(&rdp->cblist))) {
	3005	+ __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */
	3006	+ } else {
	3007	+ __call_rcu_core(rdp, head, flags);
	3008	+ local_irq_restore(flags);
	3009	+ }
	3010	+}
	3011	+
	3012	+/**
	3013	+ * call_rcu() - Queue an RCU callback for invocation after a grace period.
	3014	+ * @head: structure to be used for queueing the RCU updates.
	3015	+ * @func: actual callback function to be invoked after the grace period
	3016	+ *
	3017	+ * The callback function will be invoked some time after a full grace
	3018	+ * period elapses, in other words after all pre-existing RCU read-side
	3019	+ * critical sections have completed. However, the callback function
	3020	+ * might well execute concurrently with RCU read-side critical sections
	3021	+ * that started after call_rcu() was invoked. RCU read-side critical
	3022	+ * sections are delimited by rcu_read_lock() and rcu_read_unlock(), and
	3023	+ * may be nested. In addition, regions of code across which interrupts,
	3024	+ * preemption, or softirqs have been disabled also serve as RCU read-side
	3025	+ * critical sections. This includes hardware interrupt handlers, softirq
	3026	+ * handlers, and NMI handlers.
	3027	+ *
	3028	+ * Note that all CPUs must agree that the grace period extended beyond
	3029	+ * all pre-existing RCU read-side critical section. On systems with more
	3030	+ * than one CPU, this means that when "func()" is invoked, each CPU is
	3031	+ * guaranteed to have executed a full memory barrier since the end of its
	3032	+ * last RCU read-side critical section whose beginning preceded the call
	3033	+ * to call_rcu(). It also means that each CPU executing an RCU read-side
	3034	+ * critical section that continues beyond the start of "func()" must have
	3035	+ * executed a memory barrier after the call_rcu() but before the beginning
	3036	+ * of that RCU read-side critical section. Note that these guarantees
	3037	+ * include CPUs that are offline, idle, or executing in user mode, as
	3038	+ * well as CPUs that are executing in the kernel.
	3039	+ *
	3040	+ * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
	3041	+ * resulting RCU callback function "func()", then both CPU A and CPU B are
	3042	+ * guaranteed to execute a full memory barrier during the time interval
	3043	+ * between the call to call_rcu() and the invocation of "func()" -- even
	3044	+ * if CPU A and CPU B are the same CPU (but again only if the system has
	3045	+ * more than one CPU).
	3046	+ */
	3047	+void call_rcu(struct rcu_head *head, rcu_callback_t func)
	3048	+{
	3049	+ __call_rcu(head, func);
	3050	+}
	3051	+EXPORT_SYMBOL_GPL(call_rcu);
	3052	+
	3053	+
	3054	+/* Maximum number of jiffies to wait before draining a batch. */
	3055	+#define KFREE_DRAIN_JIFFIES (HZ / 50)
	3056	+#define KFREE_N_BATCHES 2
	3057	+#define FREE_N_CHANNELS 2
	3058	+
	3059	+/**
	3060	+ * struct kvfree_rcu_bulk_data - single block to store kvfree_rcu() pointers
	3061	+ * @nr_records: Number of active pointers in the array
	3062	+ * @next: Next bulk object in the block chain
	3063	+ * @records: Array of the kvfree_rcu() pointers
	3064	+ */
	3065	+struct kvfree_rcu_bulk_data {
	3066	+ unsigned long nr_records;
	3067	+ struct kvfree_rcu_bulk_data *next;
	3068	+ void *records[];
	3069	+};
	3070	+
	3071	+/*
	3072	+ * This macro defines how many entries the "records" array
	3073	+ * will contain. It is based on the fact that the size of
	3074	+ * kvfree_rcu_bulk_data structure becomes exactly one page.
	3075	+ */
	3076	+#define KVFREE_BULK_MAX_ENTR \
	3077	+ ((PAGE_SIZE - sizeof(struct kvfree_rcu_bulk_data)) / sizeof(void *))
	3078	+
	3079	+/**
	3080	+ * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests
	3081	+ * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period
	3082	+ * @head_free: List of kfree_rcu() objects waiting for a grace period
	3083	+ * @bkvhead_free: Bulk-List of kvfree_rcu() objects waiting for a grace period
	3084	+ * @krcp: Pointer to @kfree_rcu_cpu structure
	3085	+ */
	3086	+
	3087	+struct kfree_rcu_cpu_work {
	3088	+ struct rcu_work rcu_work;
	3089	+ struct rcu_head *head_free;
	3090	+ struct kvfree_rcu_bulk_data *bkvhead_free[FREE_N_CHANNELS];
	3091	+ struct kfree_rcu_cpu *krcp;
	3092	+};
	3093	+
	3094	+/**
	3095	+ * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period
	3096	+ * @head: List of kfree_rcu() objects not yet waiting for a grace period
	3097	+ * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period
	3098	+ * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period
	3099	+ * @lock: Synchronize access to this structure
	3100	+ * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES
	3101	+ * @monitor_todo: Tracks whether a @monitor_work delayed work is pending
	3102	+ * @initialized: The @rcu_work fields have been initialized
	3103	+ * @count: Number of objects for which GP not started
	3104	+ * @bkvcache:
	3105	+ * A simple cache list that contains objects for reuse purpose.
	3106	+ * In order to save some per-cpu space the list is singular.
	3107	+ * Even though it is lockless an access has to be protected by the
	3108	+ * per-cpu lock.
	3109	+ * @page_cache_work: A work to refill the cache when it is empty
	3110	+ * @work_in_progress: Indicates that page_cache_work is running
	3111	+ * @hrtimer: A hrtimer for scheduling a page_cache_work
	3112	+ * @nr_bkv_objs: number of allocated objects at @bkvcache.
	3113	+ *
	3114	+ * This is a per-CPU structure. The reason that it is not included in
	3115	+ * the rcu_data structure is to permit this code to be extracted from
	3116	+ * the RCU files. Such extraction could allow further optimization of
	3117	+ * the interactions with the slab allocators.
	3118	+ */
	3119	+struct kfree_rcu_cpu {
	3120	+ struct rcu_head *head;
	3121	+ struct kvfree_rcu_bulk_data *bkvhead[FREE_N_CHANNELS];
	3122	+ struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES];
	3123	+ raw_spinlock_t lock;
	3124	+ struct delayed_work monitor_work;
	3125	+ bool monitor_todo;
	3126	+ bool initialized;
	3127	+ int count;
	3128	+
	3129	+ struct work_struct page_cache_work;
	3130	+ atomic_t work_in_progress;
	3131	+ struct hrtimer hrtimer;
	3132	+
	3133	+ struct llist_head bkvcache;
	3134	+ int nr_bkv_objs;
	3135	+};
	3136	+
	3137	+static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc) = {
	3138	+ .lock = __RAW_SPIN_LOCK_UNLOCKED(krc.lock),
	3139	+};
	3140	+
	3141	+static __always_inline void
	3142	+debug_rcu_bhead_unqueue(struct kvfree_rcu_bulk_data *bhead)
	3143	+{
	3144	+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
	3145	+ int i;
	3146	+
	3147	+ for (i = 0; i < bhead->nr_records; i++)
	3148	+ debug_rcu_head_unqueue((struct rcu_head *)(bhead->records[i]));
	3149	+#endif
	3150	+}
	3151	+
	3152	+static inline struct kfree_rcu_cpu *
	3153	+krc_this_cpu_lock(unsigned long *flags)
	3154	+{
	3155	+ struct kfree_rcu_cpu *krcp;
	3156	+
	3157	+ local_irq_save(*flags); // For safely calling this_cpu_ptr().
	3158	+ krcp = this_cpu_ptr(&krc);
	3159	+ raw_spin_lock(&krcp->lock);
	3160	+
	3161	+ return krcp;
	3162	+}
	3163	+
	3164	+static inline void
	3165	+krc_this_cpu_unlock(struct kfree_rcu_cpu *krcp, unsigned long flags)
	3166	+{
	3167	+ raw_spin_unlock(&krcp->lock);
3154	3168	local_irq_restore(flags);
3155	3169	}
3156	3170
3157		-/**
3158		- * call_rcu_sched() - Queue an RCU for invocation after sched grace period.
3159		- * @head: structure to be used for queueing the RCU updates.
3160		- * @func: actual callback function to be invoked after the grace period
3161		- *
3162		- * The callback function will be invoked some time after a full grace
3163		- * period elapses, in other words after all currently executing RCU
3164		- * read-side critical sections have completed. call_rcu_sched() assumes
3165		- * that the read-side critical sections end on enabling of preemption
3166		- * or on voluntary preemption.
3167		- * RCU read-side critical sections are delimited by:
3168		- *
3169		- * - rcu_read_lock_sched() and rcu_read_unlock_sched(), OR
3170		- * - anything that disables preemption.
3171		- *
3172		- * These may be nested.
3173		- *
3174		- * See the description of call_rcu() for more detailed information on
3175		- * memory ordering guarantees.
3176		- */
3177		-void call_rcu_sched(struct rcu_head *head, rcu_callback_t func)
	3171	+static inline struct kvfree_rcu_bulk_data *
	3172	+get_cached_bnode(struct kfree_rcu_cpu *krcp)
3178	3173	{
3179		- __call_rcu(head, func, &rcu_sched_state, -1, 0);
3180		-}
3181		-EXPORT_SYMBOL_GPL(call_rcu_sched);
	3174	+ if (!krcp->nr_bkv_objs)
	3175	+ return NULL;
3182	3176
3183		-#ifndef CONFIG_PREEMPT_RT_FULL
3184		-/**
3185		- * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
3186		- * @head: structure to be used for queueing the RCU updates.
3187		- * @func: actual callback function to be invoked after the grace period
3188		- *
3189		- * The callback function will be invoked some time after a full grace
3190		- * period elapses, in other words after all currently executing RCU
3191		- * read-side critical sections have completed. call_rcu_bh() assumes
3192		- * that the read-side critical sections end on completion of a softirq
3193		- * handler. This means that read-side critical sections in process
3194		- * context must not be interrupted by softirqs. This interface is to be
3195		- * used when most of the read-side critical sections are in softirq context.
3196		- * RCU read-side critical sections are delimited by:
3197		- *
3198		- * - rcu_read_lock() and rcu_read_unlock(), if in interrupt context, OR
3199		- * - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
3200		- *
3201		- * These may be nested.
3202		- *
3203		- * See the description of call_rcu() for more detailed information on
3204		- * memory ordering guarantees.
3205		- */
3206		-void call_rcu_bh(struct rcu_head *head, rcu_callback_t func)
3207		-{
3208		- __call_rcu(head, func, &rcu_bh_state, -1, 0);
	3177	+ krcp->nr_bkv_objs--;
	3178	+ return (struct kvfree_rcu_bulk_data *)
	3179	+ llist_del_first(&krcp->bkvcache);
3209	3180	}
3210		-EXPORT_SYMBOL_GPL(call_rcu_bh);
3211		-#endif
	3181	+
	3182	+static inline bool
	3183	+put_cached_bnode(struct kfree_rcu_cpu *krcp,
	3184	+ struct kvfree_rcu_bulk_data *bnode)
	3185	+{
	3186	+ // Check the limit.
	3187	+ if (krcp->nr_bkv_objs >= rcu_min_cached_objs)
	3188	+ return false;
	3189	+
	3190	+ llist_add((struct llist_node *) bnode, &krcp->bkvcache);
	3191	+ krcp->nr_bkv_objs++;
	3192	+ return true;
	3193	+
	3194	+}
3212	3195
3213	3196	/*
3214		- * Queue an RCU callback for lazy invocation after a grace period.
3215		- * This will likely be later named something like "call_rcu_lazy()",
3216		- * but this change will require some way of tagging the lazy RCU
3217		- * callbacks in the list of pending callbacks. Until then, this
3218		- * function may only be called from __kfree_rcu().
	3197	+ * This function is invoked in workqueue context after a grace period.
	3198	+ * It frees all the objects queued on ->bhead_free or ->head_free.
3219	3199	*/
3220		-void kfree_call_rcu(struct rcu_head *head,
3221		- rcu_callback_t func)
	3200	+static void kfree_rcu_work(struct work_struct *work)
3222	3201	{
3223		- __call_rcu(head, func, rcu_state_p, -1, 1);
	3202	+ unsigned long flags;
	3203	+ struct kvfree_rcu_bulk_data bkvhead[FREE_N_CHANNELS], bnext;
	3204	+ struct rcu_head head, next;
	3205	+ struct kfree_rcu_cpu *krcp;
	3206	+ struct kfree_rcu_cpu_work *krwp;
	3207	+ int i, j;
	3208	+
	3209	+ krwp = container_of(to_rcu_work(work),
	3210	+ struct kfree_rcu_cpu_work, rcu_work);
	3211	+ krcp = krwp->krcp;
	3212	+
	3213	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3214	+ // Channels 1 and 2.
	3215	+ for (i = 0; i < FREE_N_CHANNELS; i++) {
	3216	+ bkvhead[i] = krwp->bkvhead_free[i];
	3217	+ krwp->bkvhead_free[i] = NULL;
	3218	+ }
	3219	+
	3220	+ // Channel 3.
	3221	+ head = krwp->head_free;
	3222	+ krwp->head_free = NULL;
	3223	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3224	+
	3225	+ // Handle two first channels.
	3226	+ for (i = 0; i < FREE_N_CHANNELS; i++) {
	3227	+ for (; bkvhead[i]; bkvhead[i] = bnext) {
	3228	+ bnext = bkvhead[i]->next;
	3229	+ debug_rcu_bhead_unqueue(bkvhead[i]);
	3230	+
	3231	+ rcu_lock_acquire(&rcu_callback_map);
	3232	+ if (i == 0) { // kmalloc() / kfree().
	3233	+ trace_rcu_invoke_kfree_bulk_callback(
	3234	+ rcu_state.name, bkvhead[i]->nr_records,
	3235	+ bkvhead[i]->records);
	3236	+
	3237	+ kfree_bulk(bkvhead[i]->nr_records,
	3238	+ bkvhead[i]->records);
	3239	+ } else { // vmalloc() / vfree().
	3240	+ for (j = 0; j < bkvhead[i]->nr_records; j++) {
	3241	+ trace_rcu_invoke_kvfree_callback(
	3242	+ rcu_state.name,
	3243	+ bkvhead[i]->records[j], 0);
	3244	+
	3245	+ vfree(bkvhead[i]->records[j]);
	3246	+ }
	3247	+ }
	3248	+ rcu_lock_release(&rcu_callback_map);
	3249	+
	3250	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3251	+ if (put_cached_bnode(krcp, bkvhead[i]))
	3252	+ bkvhead[i] = NULL;
	3253	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3254	+
	3255	+ if (bkvhead[i])
	3256	+ free_page((unsigned long) bkvhead[i]);
	3257	+
	3258	+ cond_resched_tasks_rcu_qs();
	3259	+ }
	3260	+ }
	3261	+
	3262	+ /*
	3263	+ * Emergency case only. It can happen under low memory
	3264	+ * condition when an allocation gets failed, so the "bulk"
	3265	+ * path can not be temporary maintained.
	3266	+ */
	3267	+ for (; head; head = next) {
	3268	+ unsigned long offset = (unsigned long)head->func;
	3269	+ void ptr = (void )head - offset;
	3270	+
	3271	+ next = head->next;
	3272	+ debug_rcu_head_unqueue((struct rcu_head *)ptr);
	3273	+ rcu_lock_acquire(&rcu_callback_map);
	3274	+ trace_rcu_invoke_kvfree_callback(rcu_state.name, head, offset);
	3275	+
	3276	+ if (!WARN_ON_ONCE(!__is_kvfree_rcu_offset(offset)))
	3277	+ kvfree(ptr);
	3278	+
	3279	+ rcu_lock_release(&rcu_callback_map);
	3280	+ cond_resched_tasks_rcu_qs();
	3281	+ }
3224	3282	}
3225		-EXPORT_SYMBOL_GPL(kfree_call_rcu);
	3283	+
	3284	+static bool
	3285	+need_offload_krc(struct kfree_rcu_cpu *krcp)
	3286	+{
	3287	+ int i;
	3288	+
	3289	+ for (i = 0; i < FREE_N_CHANNELS; i++)
	3290	+ if (krcp->bkvhead[i])
	3291	+ return true;
	3292	+
	3293	+ return !!krcp->head;
	3294	+}
	3295	+
	3296	+static bool
	3297	+need_wait_for_krwp_work(struct kfree_rcu_cpu_work *krwp)
	3298	+{
	3299	+ int i;
	3300	+
	3301	+ for (i = 0; i < FREE_N_CHANNELS; i++)
	3302	+ if (krwp->bkvhead_free[i])
	3303	+ return true;
	3304	+
	3305	+ return !!krwp->head_free;
	3306	+}
3226	3307
3227	3308	/*
3228		- * Because a context switch is a grace period for RCU-sched and RCU-bh,
3229		- * any blocking grace-period wait automatically implies a grace period
3230		- * if there is only one CPU online at any point time during execution
3231		- * of either synchronize_sched() or synchronize_rcu_bh(). It is OK to
	3309	+ * Schedule the kfree batch RCU work to run in workqueue context after a GP.
	3310	+ *
	3311	+ * This function is invoked by kfree_rcu_monitor() when the KFREE_DRAIN_JIFFIES
	3312	+ * timeout has been reached.
	3313	+ */
	3314	+static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp)
	3315	+{
	3316	+ struct kfree_rcu_cpu_work *krwp;
	3317	+ bool repeat = false;
	3318	+ int i, j;
	3319	+
	3320	+ lockdep_assert_held(&krcp->lock);
	3321	+
	3322	+ for (i = 0; i < KFREE_N_BATCHES; i++) {
	3323	+ krwp = &(krcp->krw_arr[i]);
	3324	+
	3325	+ // Try to detach bulk_head or head and attach it, only when
	3326	+ // all channels are free. Any channel is not free means at krwp
	3327	+ // there is on-going rcu work to handle krwp's free business.
	3328	+ if (need_wait_for_krwp_work(krwp))
	3329	+ continue;
	3330	+
	3331	+ if (need_offload_krc(krcp)) {
	3332	+ // Channel 1 corresponds to SLAB ptrs.
	3333	+ // Channel 2 corresponds to vmalloc ptrs.
	3334	+ for (j = 0; j < FREE_N_CHANNELS; j++) {
	3335	+ if (!krwp->bkvhead_free[j]) {
	3336	+ krwp->bkvhead_free[j] = krcp->bkvhead[j];
	3337	+ krcp->bkvhead[j] = NULL;
	3338	+ }
	3339	+ }
	3340	+
	3341	+ // Channel 3 corresponds to emergency path.
	3342	+ if (!krwp->head_free) {
	3343	+ krwp->head_free = krcp->head;
	3344	+ krcp->head = NULL;
	3345	+ }
	3346	+
	3347	+ WRITE_ONCE(krcp->count, 0);
	3348	+
	3349	+ /*
	3350	+ * One work is per one batch, so there are three
	3351	+ * "free channels", the batch can handle. It can
	3352	+ * be that the work is in the pending state when
	3353	+ * channels have been detached following by each
	3354	+ * other.
	3355	+ */
	3356	+ queue_rcu_work(system_wq, &krwp->rcu_work);
	3357	+ }
	3358	+ }
	3359	+
	3360	+ // Repeat if any "free" corresponding channel is still busy.
	3361	+ if (need_offload_krc(krcp))
	3362	+ repeat = true;
	3363	+
	3364	+ return !repeat;
	3365	+}
	3366	+
	3367	+static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu *krcp,
	3368	+ unsigned long flags)
	3369	+{
	3370	+ // Attempt to start a new batch.
	3371	+ krcp->monitor_todo = false;
	3372	+ if (queue_kfree_rcu_work(krcp)) {
	3373	+ // Success! Our job is done here.
	3374	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3375	+ return;
	3376	+ }
	3377	+
	3378	+ // Previous RCU batch still in progress, try again later.
	3379	+ krcp->monitor_todo = true;
	3380	+ schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
	3381	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3382	+}
	3383	+
	3384	+/*
	3385	+ * This function is invoked after the KFREE_DRAIN_JIFFIES timeout.
	3386	+ * It invokes kfree_rcu_drain_unlock() to attempt to start another batch.
	3387	+ */
	3388	+static void kfree_rcu_monitor(struct work_struct *work)
	3389	+{
	3390	+ unsigned long flags;
	3391	+ struct kfree_rcu_cpu *krcp = container_of(work, struct kfree_rcu_cpu,
	3392	+ monitor_work.work);
	3393	+
	3394	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3395	+ if (krcp->monitor_todo)
	3396	+ kfree_rcu_drain_unlock(krcp, flags);
	3397	+ else
	3398	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3399	+}
	3400	+
	3401	+static enum hrtimer_restart
	3402	+schedule_page_work_fn(struct hrtimer *t)
	3403	+{
	3404	+ struct kfree_rcu_cpu *krcp =
	3405	+ container_of(t, struct kfree_rcu_cpu, hrtimer);
	3406	+
	3407	+ queue_work(system_highpri_wq, &krcp->page_cache_work);
	3408	+ return HRTIMER_NORESTART;
	3409	+}
	3410	+
	3411	+static void fill_page_cache_func(struct work_struct *work)
	3412	+{
	3413	+ struct kvfree_rcu_bulk_data *bnode;
	3414	+ struct kfree_rcu_cpu *krcp =
	3415	+ container_of(work, struct kfree_rcu_cpu,
	3416	+ page_cache_work);
	3417	+ unsigned long flags;
	3418	+ bool pushed;
	3419	+ int i;
	3420	+
	3421	+ for (i = 0; i < rcu_min_cached_objs; i++) {
	3422	+ bnode = (struct kvfree_rcu_bulk_data *)
	3423	+ __get_free_page(GFP_KERNEL \| __GFP_NORETRY \| __GFP_NOMEMALLOC \| __GFP_NOWARN);
	3424	+
	3425	+ if (!bnode)
	3426	+ break;
	3427	+
	3428	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3429	+ pushed = put_cached_bnode(krcp, bnode);
	3430	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3431	+
	3432	+ if (!pushed) {
	3433	+ free_page((unsigned long) bnode);
	3434	+ break;
	3435	+ }
	3436	+ }
	3437	+
	3438	+ atomic_set(&krcp->work_in_progress, 0);
	3439	+}
	3440	+
	3441	+static void
	3442	+run_page_cache_worker(struct kfree_rcu_cpu *krcp)
	3443	+{
	3444	+ if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
	3445	+ !atomic_xchg(&krcp->work_in_progress, 1)) {
	3446	+ hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC,
	3447	+ HRTIMER_MODE_REL);
	3448	+ krcp->hrtimer.function = schedule_page_work_fn;
	3449	+ hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
	3450	+ }
	3451	+}
	3452	+
	3453	+static inline bool
	3454	+kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu krcp, void ptr)
	3455	+{
	3456	+ struct kvfree_rcu_bulk_data *bnode;
	3457	+ int idx;
	3458	+
	3459	+ if (unlikely(!krcp->initialized))
	3460	+ return false;
	3461	+
	3462	+ lockdep_assert_held(&krcp->lock);
	3463	+ idx = !!is_vmalloc_addr(ptr);
	3464	+
	3465	+ /* Check if a new block is required. */
	3466	+ if (!krcp->bkvhead[idx] \|\|
	3467	+ krcp->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
	3468	+ bnode = get_cached_bnode(krcp);
	3469	+ /* Switch to emergency path. */
	3470	+ if (!bnode)
	3471	+ return false;
	3472	+
	3473	+ /* Initialize the new block. */
	3474	+ bnode->nr_records = 0;
	3475	+ bnode->next = krcp->bkvhead[idx];
	3476	+
	3477	+ /* Attach it to the head. */
	3478	+ krcp->bkvhead[idx] = bnode;
	3479	+ }
	3480	+
	3481	+ /* Finally insert. */
	3482	+ krcp->bkvhead[idx]->records
	3483	+ [krcp->bkvhead[idx]->nr_records++] = ptr;
	3484	+
	3485	+ return true;
	3486	+}
	3487	+
	3488	+/*
	3489	+ * Queue a request for lazy invocation of appropriate free routine after a
	3490	+ * grace period. Please note there are three paths are maintained, two are the
	3491	+ * main ones that use array of pointers interface and third one is emergency
	3492	+ * one, that is used only when the main path can not be maintained temporary,
	3493	+ * due to memory pressure.
	3494	+ *
	3495	+ * Each kvfree_call_rcu() request is added to a batch. The batch will be drained
	3496	+ * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch will
	3497	+ * be free'd in workqueue context. This allows us to: batch requests together to
	3498	+ * reduce the number of grace periods during heavy kfree_rcu()/kvfree_rcu() load.
	3499	+ */
	3500	+void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
	3501	+{
	3502	+ unsigned long flags;
	3503	+ struct kfree_rcu_cpu *krcp;
	3504	+ bool success;
	3505	+ void *ptr;
	3506	+
	3507	+ if (head) {
	3508	+ ptr = (void *) head - (unsigned long) func;
	3509	+ } else {
	3510	+ /*
	3511	+ * Please note there is a limitation for the head-less
	3512	+ * variant, that is why there is a clear rule for such
	3513	+ * objects: it can be used from might_sleep() context
	3514	+ * only. For other places please embed an rcu_head to
	3515	+ * your data.
	3516	+ */
	3517	+ might_sleep();
	3518	+ ptr = (unsigned long *) func;
	3519	+ }
	3520	+
	3521	+ krcp = krc_this_cpu_lock(&flags);
	3522	+
	3523	+ // Queue the object but don't yet schedule the batch.
	3524	+ if (debug_rcu_head_queue(ptr)) {
	3525	+ // Probable double kfree_rcu(), just leak.
	3526	+ WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n",
	3527	+ __func__, head);
	3528	+
	3529	+ // Mark as success and leave.
	3530	+ success = true;
	3531	+ goto unlock_return;
	3532	+ }
	3533	+
	3534	+ success = kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr);
	3535	+ if (!success) {
	3536	+ run_page_cache_worker(krcp);
	3537	+
	3538	+ if (head == NULL)
	3539	+ // Inline if kvfree_rcu(one_arg) call.
	3540	+ goto unlock_return;
	3541	+
	3542	+ head->func = func;
	3543	+ head->next = krcp->head;
	3544	+ krcp->head = head;
	3545	+ success = true;
	3546	+ }
	3547	+
	3548	+ WRITE_ONCE(krcp->count, krcp->count + 1);
	3549	+
	3550	+ // Set timer to drain after KFREE_DRAIN_JIFFIES.
	3551	+ if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
	3552	+ !krcp->monitor_todo) {
	3553	+ krcp->monitor_todo = true;
	3554	+ schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
	3555	+ }
	3556	+
	3557	+unlock_return:
	3558	+ krc_this_cpu_unlock(krcp, flags);
	3559	+
	3560	+ /*
	3561	+ * Inline kvfree() after synchronize_rcu(). We can do
	3562	+ * it from might_sleep() context only, so the current
	3563	+ * CPU can pass the QS state.
	3564	+ */
	3565	+ if (!success) {
	3566	+ debug_rcu_head_unqueue((struct rcu_head *) ptr);
	3567	+ synchronize_rcu();
	3568	+ kvfree(ptr);
	3569	+ }
	3570	+}
	3571	+EXPORT_SYMBOL_GPL(kvfree_call_rcu);
	3572	+
	3573	+static unsigned long
	3574	+kfree_rcu_shrink_count(struct shrinker shrink, struct shrink_control sc)
	3575	+{
	3576	+ int cpu;
	3577	+ unsigned long count = 0;
	3578	+
	3579	+ /* Snapshot count of all CPUs */
	3580	+ for_each_possible_cpu(cpu) {
	3581	+ struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
	3582	+
	3583	+ count += READ_ONCE(krcp->count);
	3584	+ }
	3585	+
	3586	+ return count;
	3587	+}
	3588	+
	3589	+static unsigned long
	3590	+kfree_rcu_shrink_scan(struct shrinker shrink, struct shrink_control sc)
	3591	+{
	3592	+ int cpu, freed = 0;
	3593	+ unsigned long flags;
	3594	+
	3595	+ for_each_possible_cpu(cpu) {
	3596	+ int count;
	3597	+ struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
	3598	+
	3599	+ count = krcp->count;
	3600	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3601	+ if (krcp->monitor_todo)
	3602	+ kfree_rcu_drain_unlock(krcp, flags);
	3603	+ else
	3604	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3605	+
	3606	+ sc->nr_to_scan -= count;
	3607	+ freed += count;
	3608	+
	3609	+ if (sc->nr_to_scan <= 0)
	3610	+ break;
	3611	+ }
	3612	+
	3613	+ return freed == 0 ? SHRINK_STOP : freed;
	3614	+}
	3615	+
	3616	+static struct shrinker kfree_rcu_shrinker = {
	3617	+ .count_objects = kfree_rcu_shrink_count,
	3618	+ .scan_objects = kfree_rcu_shrink_scan,
	3619	+ .batch = 0,
	3620	+ .seeks = DEFAULT_SEEKS,
	3621	+};
	3622	+
	3623	+void __init kfree_rcu_scheduler_running(void)
	3624	+{
	3625	+ int cpu;
	3626	+ unsigned long flags;
	3627	+
	3628	+ for_each_possible_cpu(cpu) {
	3629	+ struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
	3630	+
	3631	+ raw_spin_lock_irqsave(&krcp->lock, flags);
	3632	+ if (!krcp->head \|\| krcp->monitor_todo) {
	3633	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3634	+ continue;
	3635	+ }
	3636	+ krcp->monitor_todo = true;
	3637	+ schedule_delayed_work_on(cpu, &krcp->monitor_work,
	3638	+ KFREE_DRAIN_JIFFIES);
	3639	+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
	3640	+ }
	3641	+}
	3642	+
	3643	+/*
	3644	+ * During early boot, any blocking grace-period wait automatically
	3645	+ * implies a grace period. Later on, this is never the case for PREEMPTION.
	3646	+ *
	3647	+ * Howevr, because a context switch is a grace period for !PREEMPTION, any
	3648	+ * blocking grace-period wait automatically implies a grace period if
	3649	+ * there is only one CPU online at any point time during execution of
	3650	+ * either synchronize_rcu() or synchronize_rcu_expedited(). It is OK to
3232	3651	* occasionally incorrectly indicate that there are multiple CPUs online
3233		- * when there was in fact only one the whole time, as this just adds
3234		- * some overhead: RCU still operates correctly.
	3652	+ * when there was in fact only one the whole time, as this just adds some
	3653	+ * overhead: RCU still operates correctly.
3235	3654	*/
3236	3655	static int rcu_blocking_is_gp(void)
3237	3656	{
3238	3657	int ret;
3239	3658
	3659	+ if (IS_ENABLED(CONFIG_PREEMPTION))
	3660	+ return rcu_scheduler_active == RCU_SCHEDULER_INACTIVE;
3240	3661	might_sleep(); /* Check for RCU read-side critical section. */
3241	3662	preempt_disable();
3242	3663	ret = num_online_cpus() <= 1;
..	..	@@ -3245,83 +3666,52 @@
3245	3666	}
3246	3667
3247	3668	/**
3248		- * synchronize_sched - wait until an rcu-sched grace period has elapsed.
	3669	+ * synchronize_rcu - wait until a grace period has elapsed.
3249	3670	*
3250		- * Control will return to the caller some time after a full rcu-sched
3251		- * grace period has elapsed, in other words after all currently executing
3252		- * rcu-sched read-side critical sections have completed. These read-side
3253		- * critical sections are delimited by rcu_read_lock_sched() and
3254		- * rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(),
3255		- * local_irq_disable(), and so on may be used in place of
3256		- * rcu_read_lock_sched().
3257		- *
3258		- * This means that all preempt_disable code sequences, including NMI and
3259		- * non-threaded hardware-interrupt handlers, in progress on entry will
3260		- * have completed before this primitive returns. However, this does not
3261		- * guarantee that softirq handlers will have completed, since in some
3262		- * kernels, these handlers can run in process context, and can block.
	3671	+ * Control will return to the caller some time after a full grace
	3672	+ * period has elapsed, in other words after all currently executing RCU
	3673	+ * read-side critical sections have completed. Note, however, that
	3674	+ * upon return from synchronize_rcu(), the caller might well be executing
	3675	+ * concurrently with new RCU read-side critical sections that began while
	3676	+ * synchronize_rcu() was waiting. RCU read-side critical sections are
	3677	+ * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
	3678	+ * In addition, regions of code across which interrupts, preemption, or
	3679	+ * softirqs have been disabled also serve as RCU read-side critical
	3680	+ * sections. This includes hardware interrupt handlers, softirq handlers,
	3681	+ * and NMI handlers.
3263	3682	*
3264	3683	* Note that this guarantee implies further memory-ordering guarantees.
3265		- * On systems with more than one CPU, when synchronize_sched() returns,
3266		- * each CPU is guaranteed to have executed a full memory barrier since the
3267		- * end of its last RCU-sched read-side critical section whose beginning
3268		- * preceded the call to synchronize_sched(). In addition, each CPU having
	3684	+ * On systems with more than one CPU, when synchronize_rcu() returns,
	3685	+ * each CPU is guaranteed to have executed a full memory barrier since
	3686	+ * the end of its last RCU read-side critical section whose beginning
	3687	+ * preceded the call to synchronize_rcu(). In addition, each CPU having
3269	3688	* an RCU read-side critical section that extends beyond the return from
3270		- * synchronize_sched() is guaranteed to have executed a full memory barrier
3271		- * after the beginning of synchronize_sched() and before the beginning of
	3689	+ * synchronize_rcu() is guaranteed to have executed a full memory barrier
	3690	+ * after the beginning of synchronize_rcu() and before the beginning of
3272	3691	* that RCU read-side critical section. Note that these guarantees include
3273	3692	* CPUs that are offline, idle, or executing in user mode, as well as CPUs
3274	3693	* that are executing in the kernel.
3275	3694	*
3276		- * Furthermore, if CPU A invoked synchronize_sched(), which returned
	3695	+ * Furthermore, if CPU A invoked synchronize_rcu(), which returned
3277	3696	* to its caller on CPU B, then both CPU A and CPU B are guaranteed
3278	3697	* to have executed a full memory barrier during the execution of
3279		- * synchronize_sched() -- even if CPU A and CPU B are the same CPU (but
	3698	+ * synchronize_rcu() -- even if CPU A and CPU B are the same CPU (but
3280	3699	* again only if the system has more than one CPU).
3281	3700	*/
3282		-void synchronize_sched(void)
	3701	+void synchronize_rcu(void)
3283	3702	{
3284	3703	RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) \|\|
3285	3704	lock_is_held(&rcu_lock_map) \|\|
3286	3705	lock_is_held(&rcu_sched_lock_map),
3287		- "Illegal synchronize_sched() in RCU-sched read-side critical section");
	3706	+ "Illegal synchronize_rcu() in RCU read-side critical section");
3288	3707	if (rcu_blocking_is_gp())
3289	3708	return;
3290	3709	if (rcu_gp_is_expedited())
3291		- synchronize_sched_expedited();
	3710	+ synchronize_rcu_expedited();
3292	3711	else
3293		- wait_rcu_gp(call_rcu_sched);
	3712	+ wait_rcu_gp(call_rcu);
3294	3713	}
3295		-EXPORT_SYMBOL_GPL(synchronize_sched);
3296		-
3297		-#ifndef CONFIG_PREEMPT_RT_FULL
3298		-/**
3299		- * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed.
3300		- *
3301		- * Control will return to the caller some time after a full rcu_bh grace
3302		- * period has elapsed, in other words after all currently executing rcu_bh
3303		- * read-side critical sections have completed. RCU read-side critical
3304		- * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(),
3305		- * and may be nested.
3306		- *
3307		- * See the description of synchronize_sched() for more detailed information
3308		- * on memory ordering guarantees.
3309		- */
3310		-void synchronize_rcu_bh(void)
3311		-{
3312		- RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) \|\|
3313		- lock_is_held(&rcu_lock_map) \|\|
3314		- lock_is_held(&rcu_sched_lock_map),
3315		- "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
3316		- if (rcu_blocking_is_gp())
3317		- return;
3318		- if (rcu_gp_is_expedited())
3319		- synchronize_rcu_bh_expedited();
3320		- else
3321		- wait_rcu_gp(call_rcu_bh);
3322		-}
3323		-EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
3324		-#endif
	3714	+EXPORT_SYMBOL_GPL(synchronize_rcu);
3325	3715
3326	3716	/**
3327	3717	* get_state_synchronize_rcu - Snapshot current RCU state
..	..	@@ -3337,7 +3727,7 @@
3337	3727	* before the load from ->gp_seq.
3338	3728	*/
3339	3729	smp_mb(); /* ^^^ */
3340		- return rcu_seq_snap(&rcu_state_p->gp_seq);
	3730	+ return rcu_seq_snap(&rcu_state.gp_seq);
3341	3731	}
3342	3732	EXPORT_SYMBOL_GPL(get_state_synchronize_rcu);
3343	3733
..	..	@@ -3357,74 +3747,42 @@
3357	3747	*/
3358	3748	void cond_synchronize_rcu(unsigned long oldstate)
3359	3749	{
3360		- if (!rcu_seq_done(&rcu_state_p->gp_seq, oldstate))
	3750	+ if (!rcu_seq_done(&rcu_state.gp_seq, oldstate))
3361	3751	synchronize_rcu();
3362	3752	else
3363	3753	smp_mb(); /* Ensure GP ends before subsequent accesses. */
3364	3754	}
3365	3755	EXPORT_SYMBOL_GPL(cond_synchronize_rcu);
3366	3756
3367		-/**
3368		- * get_state_synchronize_sched - Snapshot current RCU-sched state
3369		- *
3370		- * Returns a cookie that is used by a later call to cond_synchronize_sched()
3371		- * to determine whether or not a full grace period has elapsed in the
3372		- * meantime.
3373		- */
3374		-unsigned long get_state_synchronize_sched(void)
3375		-{
3376		- /*
3377		- * Any prior manipulation of RCU-protected data must happen
3378		- * before the load from ->gp_seq.
3379		- */
3380		- smp_mb(); /* ^^^ */
3381		- return rcu_seq_snap(&rcu_sched_state.gp_seq);
3382		-}
3383		-EXPORT_SYMBOL_GPL(get_state_synchronize_sched);
3384		-
3385		-/**
3386		- * cond_synchronize_sched - Conditionally wait for an RCU-sched grace period
3387		- *
3388		- * @oldstate: return value from earlier call to get_state_synchronize_sched()
3389		- *
3390		- * If a full RCU-sched grace period has elapsed since the earlier call to
3391		- * get_state_synchronize_sched(), just return. Otherwise, invoke
3392		- * synchronize_sched() to wait for a full grace period.
3393		- *
3394		- * Yes, this function does not take counter wrap into account. But
3395		- * counter wrap is harmless. If the counter wraps, we have waited for
3396		- * more than 2 billion grace periods (and way more on a 64-bit system!),
3397		- * so waiting for one additional grace period should be just fine.
3398		- */
3399		-void cond_synchronize_sched(unsigned long oldstate)
3400		-{
3401		- if (!rcu_seq_done(&rcu_sched_state.gp_seq, oldstate))
3402		- synchronize_sched();
3403		- else
3404		- smp_mb(); /* Ensure GP ends before subsequent accesses. */
3405		-}
3406		-EXPORT_SYMBOL_GPL(cond_synchronize_sched);
3407		-
3408	3757	/*
3409		- * Check to see if there is any immediate RCU-related work to be done
3410		- * by the current CPU, for the specified type of RCU, returning 1 if so.
3411		- * The checks are in order of increasing expense: checks that can be
3412		- * carried out against CPU-local state are performed first. However,
3413		- * we must check for CPU stalls first, else we might not get a chance.
	3758	+ * Check to see if there is any immediate RCU-related work to be done by
	3759	+ * the current CPU, returning 1 if so and zero otherwise. The checks are
	3760	+ * in order of increasing expense: checks that can be carried out against
	3761	+ * CPU-local state are performed first. However, we must check for CPU
	3762	+ * stalls first, else we might not get a chance.
3414	3763	*/
3415		-static int __rcu_pending(struct rcu_state rsp, struct rcu_data rdp)
	3764	+static int rcu_pending(int user)
3416	3765	{
	3766	+ bool gp_in_progress;
	3767	+ struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
3417	3768	struct rcu_node *rnp = rdp->mynode;
3418	3769
3419		- /* Check for CPU stalls, if enabled. */
3420		- check_cpu_stall(rsp, rdp);
	3770	+ lockdep_assert_irqs_disabled();
3421	3771
3422		- /* Is this CPU a NO_HZ_FULL CPU that should ignore RCU? */
3423		- if (rcu_nohz_full_cpu(rsp))
	3772	+ /* Check for CPU stalls, if enabled. */
	3773	+ check_cpu_stall(rdp);
	3774	+
	3775	+ /* Does this CPU need a deferred NOCB wakeup? */
	3776	+ if (rcu_nocb_need_deferred_wakeup(rdp))
	3777	+ return 1;
	3778	+
	3779	+ /* Is this a nohz_full CPU in userspace or idle? (Ignore RCU if so.) */
	3780	+ if ((user \|\| rcu_is_cpu_rrupt_from_idle()) && rcu_nohz_full_cpu())
3424	3781	return 0;
3425	3782
3426	3783	/* Is the RCU core waiting for a quiescent state from this CPU? */
3427		- if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm)
	3784	+ gp_in_progress = rcu_gp_in_progress();
	3785	+ if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm && gp_in_progress)
3428	3786	return 1;
3429	3787
3430	3788	/* Does this CPU have callbacks ready to invoke? */
..	..	@@ -3432,8 +3790,9 @@
3432	3790	return 1;
3433	3791
3434	3792	/* Has RCU gone idle with this CPU needing another grace period? */
3435		- if (!rcu_gp_in_progress(rsp) &&
3436		- rcu_segcblist_is_enabled(&rdp->cblist) &&
	3793	+ if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) &&
	3794	+ (!IS_ENABLED(CONFIG_RCU_NOCB_CPU) \|\|
	3795	+ !rcu_segcblist_is_offloaded(&rdp->cblist)) &&
3437	3796	!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
3438	3797	return 1;
3439	3798
..	..	@@ -3442,141 +3801,106 @@
3442	3801	unlikely(READ_ONCE(rdp->gpwrap))) /* outside lock */
3443	3802	return 1;
3444	3803
3445		- /* Does this CPU need a deferred NOCB wakeup? */
3446		- if (rcu_nocb_need_deferred_wakeup(rdp))
3447		- return 1;
3448		-
3449	3804	/* nothing to do */
3450	3805	return 0;
3451	3806	}
3452	3807
3453	3808	/*
3454		- * Check to see if there is any immediate RCU-related work to be done
3455		- * by the current CPU, returning 1 if so. This function is part of the
3456		- * RCU implementation; it is -not- an exported member of the RCU API.
3457		- */
3458		-static int rcu_pending(void)
3459		-{
3460		- struct rcu_state *rsp;
3461		-
3462		- for_each_rcu_flavor(rsp)
3463		- if (__rcu_pending(rsp, this_cpu_ptr(rsp->rda)))
3464		- return 1;
3465		- return 0;
3466		-}
3467		-
3468		-/*
3469		- * Return true if the specified CPU has any callback. If all_lazy is
3470		- * non-NULL, store an indication of whether all callbacks are lazy.
3471		- * (If there are no callbacks, all of them are deemed to be lazy.)
3472		- */
3473		-static bool rcu_cpu_has_callbacks(bool *all_lazy)
3474		-{
3475		- bool al = true;
3476		- bool hc = false;
3477		- struct rcu_data *rdp;
3478		- struct rcu_state *rsp;
3479		-
3480		- for_each_rcu_flavor(rsp) {
3481		- rdp = this_cpu_ptr(rsp->rda);
3482		- if (rcu_segcblist_empty(&rdp->cblist))
3483		- continue;
3484		- hc = true;
3485		- if (rcu_segcblist_n_nonlazy_cbs(&rdp->cblist) \|\| !all_lazy) {
3486		- al = false;
3487		- break;
3488		- }
3489		- }
3490		- if (all_lazy)
3491		- *all_lazy = al;
3492		- return hc;
3493		-}
3494		-
3495		-/*
3496		- * Helper function for _rcu_barrier() tracing. If tracing is disabled,
	3809	+ * Helper function for rcu_barrier() tracing. If tracing is disabled,
3497	3810	* the compiler is expected to optimize this away.
3498	3811	*/
3499		-static void _rcu_barrier_trace(struct rcu_state rsp, const char s,
3500		- int cpu, unsigned long done)
	3812	+static void rcu_barrier_trace(const char *s, int cpu, unsigned long done)
3501	3813	{
3502		- trace_rcu_barrier(rsp->name, s, cpu,
3503		- atomic_read(&rsp->barrier_cpu_count), done);
	3814	+ trace_rcu_barrier(rcu_state.name, s, cpu,
	3815	+ atomic_read(&rcu_state.barrier_cpu_count), done);
3504	3816	}
3505	3817
3506	3818	/*
3507		- * RCU callback function for _rcu_barrier(). If we are last, wake
3508		- * up the task executing _rcu_barrier().
	3819	+ * RCU callback function for rcu_barrier(). If we are last, wake
	3820	+ * up the task executing rcu_barrier().
	3821	+ *
	3822	+ * Note that the value of rcu_state.barrier_sequence must be captured
	3823	+ * before the atomic_dec_and_test(). Otherwise, if this CPU is not last,
	3824	+ * other CPUs might count the value down to zero before this CPU gets
	3825	+ * around to invoking rcu_barrier_trace(), which might result in bogus
	3826	+ * data from the next instance of rcu_barrier().
3509	3827	*/
3510	3828	static void rcu_barrier_callback(struct rcu_head *rhp)
3511	3829	{
3512		- struct rcu_data *rdp = container_of(rhp, struct rcu_data, barrier_head);
3513		- struct rcu_state *rsp = rdp->rsp;
	3830	+ unsigned long __maybe_unused s = rcu_state.barrier_sequence;
3514	3831
3515		- if (atomic_dec_and_test(&rsp->barrier_cpu_count)) {
3516		- _rcu_barrier_trace(rsp, TPS("LastCB"), -1,
3517		- rsp->barrier_sequence);
3518		- complete(&rsp->barrier_completion);
	3832	+ if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {
	3833	+ rcu_barrier_trace(TPS("LastCB"), -1, s);
	3834	+ complete(&rcu_state.barrier_completion);
3519	3835	} else {
3520		- _rcu_barrier_trace(rsp, TPS("CB"), -1, rsp->barrier_sequence);
	3836	+ rcu_barrier_trace(TPS("CB"), -1, s);
3521	3837	}
3522	3838	}
3523	3839
3524	3840	/*
3525	3841	* Called with preemption disabled, and from cross-cpu IRQ context.
3526	3842	*/
3527		-static void rcu_barrier_func(void *type)
	3843	+static void rcu_barrier_func(void *cpu_in)
3528	3844	{
3529		- struct rcu_state *rsp = type;
3530		- struct rcu_data *rdp = raw_cpu_ptr(rsp->rda);
	3845	+ uintptr_t cpu = (uintptr_t)cpu_in;
	3846	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3531	3847
3532		- _rcu_barrier_trace(rsp, TPS("IRQ"), -1, rsp->barrier_sequence);
	3848	+ rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);
3533	3849	rdp->barrier_head.func = rcu_barrier_callback;
3534	3850	debug_rcu_head_queue(&rdp->barrier_head);
3535		- if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head, 0)) {
3536		- atomic_inc(&rsp->barrier_cpu_count);
	3851	+ rcu_nocb_lock(rdp);
	3852	+ WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));
	3853	+ if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) {
	3854	+ atomic_inc(&rcu_state.barrier_cpu_count);
3537	3855	} else {
3538	3856	debug_rcu_head_unqueue(&rdp->barrier_head);
3539		- _rcu_barrier_trace(rsp, TPS("IRQNQ"), -1,
3540		- rsp->barrier_sequence);
	3857	+ rcu_barrier_trace(TPS("IRQNQ"), -1,
	3858	+ rcu_state.barrier_sequence);
3541	3859	}
	3860	+ rcu_nocb_unlock(rdp);
3542	3861	}
3543	3862
3544		-/*
3545		- * Orchestrate the specified type of RCU barrier, waiting for all
3546		- * RCU callbacks of the specified type to complete.
	3863	+/**
	3864	+ * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
	3865	+ *
	3866	+ * Note that this primitive does not necessarily wait for an RCU grace period
	3867	+ * to complete. For example, if there are no RCU callbacks queued anywhere
	3868	+ * in the system, then rcu_barrier() is within its rights to return
	3869	+ * immediately, without waiting for anything, much less an RCU grace period.
3547	3870	*/
3548		-static void _rcu_barrier(struct rcu_state *rsp)
	3871	+void rcu_barrier(void)
3549	3872	{
3550		- int cpu;
	3873	+ uintptr_t cpu;
3551	3874	struct rcu_data *rdp;
3552		- unsigned long s = rcu_seq_snap(&rsp->barrier_sequence);
	3875	+ unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);
3553	3876
3554		- _rcu_barrier_trace(rsp, TPS("Begin"), -1, s);
	3877	+ rcu_barrier_trace(TPS("Begin"), -1, s);
3555	3878
3556	3879	/* Take mutex to serialize concurrent rcu_barrier() requests. */
3557		- mutex_lock(&rsp->barrier_mutex);
	3880	+ mutex_lock(&rcu_state.barrier_mutex);
3558	3881
3559	3882	/* Did someone else do our work for us? */
3560		- if (rcu_seq_done(&rsp->barrier_sequence, s)) {
3561		- _rcu_barrier_trace(rsp, TPS("EarlyExit"), -1,
3562		- rsp->barrier_sequence);
	3883	+ if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {
	3884	+ rcu_barrier_trace(TPS("EarlyExit"), -1,
	3885	+ rcu_state.barrier_sequence);
3563	3886	smp_mb(); /* caller's subsequent code after above check. */
3564		- mutex_unlock(&rsp->barrier_mutex);
	3887	+ mutex_unlock(&rcu_state.barrier_mutex);
3565	3888	return;
3566	3889	}
3567	3890
3568	3891	/* Mark the start of the barrier operation. */
3569		- rcu_seq_start(&rsp->barrier_sequence);
3570		- _rcu_barrier_trace(rsp, TPS("Inc1"), -1, rsp->barrier_sequence);
	3892	+ rcu_seq_start(&rcu_state.barrier_sequence);
	3893	+ rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);
3571	3894
3572	3895	/*
3573		- * Initialize the count to one rather than to zero in order to
3574		- * avoid a too-soon return to zero in case of a short grace period
3575		- * (or preemption of this task). Exclude CPU-hotplug operations
3576		- * to ensure that no offline CPU has callbacks queued.
	3896	+ * Initialize the count to two rather than to zero in order
	3897	+ * to avoid a too-soon return to zero in case of an immediate
	3898	+ * invocation of the just-enqueued callback (or preemption of
	3899	+ * this task). Exclude CPU-hotplug operations to ensure that no
	3900	+ * offline non-offloaded CPU has callbacks queued.
3577	3901	*/
3578		- init_completion(&rsp->barrier_completion);
3579		- atomic_set(&rsp->barrier_cpu_count, 1);
	3902	+ init_completion(&rcu_state.barrier_completion);
	3903	+ atomic_set(&rcu_state.barrier_cpu_count, 2);
3580	3904	get_online_cpus();
3581	3905
3582	3906	/*
..	..	@@ -3585,28 +3909,27 @@
3585	3909	* corresponding CPU's preceding callbacks have been invoked.
3586	3910	*/
3587	3911	for_each_possible_cpu(cpu) {
3588		- if (!cpu_online(cpu) && !rcu_is_nocb_cpu(cpu))
	3912	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	3913	+ if (cpu_is_offline(cpu) &&
	3914	+ !rcu_segcblist_is_offloaded(&rdp->cblist))
3589	3915	continue;
3590		- rdp = per_cpu_ptr(rsp->rda, cpu);
3591		- if (rcu_is_nocb_cpu(cpu)) {
3592		- if (!rcu_nocb_cpu_needs_barrier(rsp, cpu)) {
3593		- _rcu_barrier_trace(rsp, TPS("OfflineNoCB"), cpu,
3594		- rsp->barrier_sequence);
3595		- } else {
3596		- _rcu_barrier_trace(rsp, TPS("OnlineNoCB"), cpu,
3597		- rsp->barrier_sequence);
3598		- smp_mb__before_atomic();
3599		- atomic_inc(&rsp->barrier_cpu_count);
3600		- __call_rcu(&rdp->barrier_head,
3601		- rcu_barrier_callback, rsp, cpu, 0);
3602		- }
3603		- } else if (rcu_segcblist_n_cbs(&rdp->cblist)) {
3604		- _rcu_barrier_trace(rsp, TPS("OnlineQ"), cpu,
3605		- rsp->barrier_sequence);
3606		- smp_call_function_single(cpu, rcu_barrier_func, rsp, 1);
	3916	+ if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) {
	3917	+ rcu_barrier_trace(TPS("OnlineQ"), cpu,
	3918	+ rcu_state.barrier_sequence);
	3919	+ smp_call_function_single(cpu, rcu_barrier_func, (void *)cpu, 1);
	3920	+ } else if (rcu_segcblist_n_cbs(&rdp->cblist) &&
	3921	+ cpu_is_offline(cpu)) {
	3922	+ rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu,
	3923	+ rcu_state.barrier_sequence);
	3924	+ local_irq_disable();
	3925	+ rcu_barrier_func((void *)cpu);
	3926	+ local_irq_enable();
	3927	+ } else if (cpu_is_offline(cpu)) {
	3928	+ rcu_barrier_trace(TPS("OfflineNoCBNoQ"), cpu,
	3929	+ rcu_state.barrier_sequence);
3607	3930	} else {
3608		- _rcu_barrier_trace(rsp, TPS("OnlineNQ"), cpu,
3609		- rsp->barrier_sequence);
	3931	+ rcu_barrier_trace(TPS("OnlineNQ"), cpu,
	3932	+ rcu_state.barrier_sequence);
3610	3933	}
3611	3934	}
3612	3935	put_online_cpus();
..	..	@@ -3615,39 +3938,20 @@
3615	3938	* Now that we have an rcu_barrier_callback() callback on each
3616	3939	* CPU, and thus each counted, remove the initial count.
3617	3940	*/
3618		- if (atomic_dec_and_test(&rsp->barrier_cpu_count))
3619		- complete(&rsp->barrier_completion);
	3941	+ if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count))
	3942	+ complete(&rcu_state.barrier_completion);
3620	3943
3621	3944	/* Wait for all rcu_barrier_callback() callbacks to be invoked. */
3622		- wait_for_completion(&rsp->barrier_completion);
	3945	+ wait_for_completion(&rcu_state.barrier_completion);
3623	3946
3624	3947	/* Mark the end of the barrier operation. */
3625		- _rcu_barrier_trace(rsp, TPS("Inc2"), -1, rsp->barrier_sequence);
3626		- rcu_seq_end(&rsp->barrier_sequence);
	3948	+ rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence);
	3949	+ rcu_seq_end(&rcu_state.barrier_sequence);
3627	3950
3628	3951	/* Other rcu_barrier() invocations can now safely proceed. */
3629		- mutex_unlock(&rsp->barrier_mutex);
	3952	+ mutex_unlock(&rcu_state.barrier_mutex);
3630	3953	}
3631		-
3632		-#ifndef CONFIG_PREEMPT_RT_FULL
3633		-/**
3634		- * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete.
3635		- */
3636		-void rcu_barrier_bh(void)
3637		-{
3638		- _rcu_barrier(&rcu_bh_state);
3639		-}
3640		-EXPORT_SYMBOL_GPL(rcu_barrier_bh);
3641		-#endif
3642		-
3643		-/**
3644		- * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks.
3645		- */
3646		-void rcu_barrier_sched(void)
3647		-{
3648		- _rcu_barrier(&rcu_sched_state);
3649		-}
3650		-EXPORT_SYMBOL_GPL(rcu_barrier_sched);
	3954	+EXPORT_SYMBOL_GPL(rcu_barrier);
3651	3955
3652	3956	/*
3653	3957	* Propagate ->qsinitmask bits up the rcu_node tree to account for the
..	..	@@ -3681,46 +3985,48 @@
3681	3985	* Do boot-time initialization of a CPU's per-CPU RCU data.
3682	3986	*/
3683	3987	static void __init
3684		-rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
	3988	+rcu_boot_init_percpu_data(int cpu)
3685	3989	{
3686		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
	3990	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3687	3991
3688	3992	/* Set up local state, ensuring consistent view of global state. */
3689	3993	rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);
3690		- rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
3691		- WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != 1);
3692		- WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp->dynticks)));
3693		- rdp->rcu_ofl_gp_seq = rsp->gp_seq;
	3994	+ INIT_WORK(&rdp->strict_work, strict_work_handler);
	3995	+ WARN_ON_ONCE(rdp->dynticks_nesting != 1);
	3996	+ WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)));
	3997	+ rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;
3694	3998	rdp->rcu_ofl_gp_flags = RCU_GP_CLEANED;
3695		- rdp->rcu_onl_gp_seq = rsp->gp_seq;
	3999	+ rdp->rcu_onl_gp_seq = rcu_state.gp_seq;
3696	4000	rdp->rcu_onl_gp_flags = RCU_GP_CLEANED;
3697	4001	rdp->cpu = cpu;
3698		- rdp->rsp = rsp;
3699	4002	rcu_boot_init_nocb_percpu_data(rdp);
3700	4003	}
3701	4004
3702	4005	/*
3703		- * Initialize a CPU's per-CPU RCU data. Note that only one online or
	4006	+ * Invoked early in the CPU-online process, when pretty much all services
	4007	+ * are available. The incoming CPU is not present.
	4008	+ *
	4009	+ * Initializes a CPU's per-CPU RCU data. Note that only one online or
3704	4010	* offline event can be happening at a given time. Note also that we can
3705	4011	* accept some slop in the rsp->gp_seq access due to the fact that this
3706		- * CPU cannot possibly have any RCU callbacks in flight yet.
	4012	+ * CPU cannot possibly have any non-offloaded RCU callbacks in flight yet.
	4013	+ * And any offloaded callbacks are being numbered elsewhere.
3707	4014	*/
3708		-static void
3709		-rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
	4015	+int rcutree_prepare_cpu(unsigned int cpu)
3710	4016	{
3711	4017	unsigned long flags;
3712		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
3713		- struct rcu_node *rnp = rcu_get_root(rsp);
	4018	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
	4019	+ struct rcu_node *rnp = rcu_get_root();
3714	4020
3715	4021	/* Set up local state, ensuring consistent view of global state. */
3716	4022	raw_spin_lock_irqsave_rcu_node(rnp, flags);
3717	4023	rdp->qlen_last_fqs_check = 0;
3718		- rdp->n_force_qs_snap = rsp->n_force_qs;
	4024	+ rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
3719	4025	rdp->blimit = blimit;
3720	4026	if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */
3721		- !init_nocb_callback_list(rdp))
	4027	+ !rcu_segcblist_is_offloaded(&rdp->cblist))
3722	4028	rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */
3723		- rdp->dynticks->dynticks_nesting = 1; /* CPU not up, no tearing. */
	4029	+ rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */
3724	4030	rcu_dynticks_eqs_online();
3725	4031	raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
3726	4032
..	..	@@ -3732,30 +4038,16 @@
3732	4038	rnp = rdp->mynode;
3733	4039	raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
3734	4040	rdp->beenonline = true; /* We have now been online. */
3735		- rdp->gp_seq = rnp->gp_seq;
3736		- rdp->gp_seq_needed = rnp->gp_seq;
	4041	+ rdp->gp_seq = READ_ONCE(rnp->gp_seq);
	4042	+ rdp->gp_seq_needed = rdp->gp_seq;
3737	4043	rdp->cpu_no_qs.b.norm = true;
3738		- rdp->rcu_qs_ctr_snap = per_cpu(rcu_dynticks.rcu_qs_ctr, cpu);
3739	4044	rdp->core_needs_qs = false;
3740	4045	rdp->rcu_iw_pending = false;
3741		- rdp->rcu_iw_gp_seq = rnp->gp_seq - 1;
3742		- trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuonl"));
	4046	+ rdp->rcu_iw_gp_seq = rdp->gp_seq - 1;
	4047	+ trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
3743	4048	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3744		-}
3745		-
3746		-/*
3747		- * Invoked early in the CPU-online process, when pretty much all
3748		- * services are available. The incoming CPU is not present.
3749		- */
3750		-int rcutree_prepare_cpu(unsigned int cpu)
3751		-{
3752		- struct rcu_state *rsp;
3753		-
3754		- for_each_rcu_flavor(rsp)
3755		- rcu_init_percpu_data(cpu, rsp);
3756		-
3757	4049	rcu_prepare_kthreads(cpu);
3758		- rcu_spawn_all_nocb_kthreads(cpu);
	4050	+ rcu_spawn_cpu_nocb_kthread(cpu);
3759	4051
3760	4052	return 0;
3761	4053	}
..	..	@@ -3765,7 +4057,7 @@
3765	4057	*/
3766	4058	static void rcutree_affinity_setting(unsigned int cpu, int outgoing)
3767	4059	{
3768		- struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
	4060	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3769	4061
3770	4062	rcu_boost_kthread_setaffinity(rdp->mynode, outgoing);
3771	4063	}
..	..	@@ -3779,19 +4071,19 @@
3779	4071	unsigned long flags;
3780	4072	struct rcu_data *rdp;
3781	4073	struct rcu_node *rnp;
3782		- struct rcu_state *rsp;
3783	4074
3784		- for_each_rcu_flavor(rsp) {
3785		- rdp = per_cpu_ptr(rsp->rda, cpu);
3786		- rnp = rdp->mynode;
3787		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
3788		- rnp->ffmask \|= rdp->grpmask;
3789		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3790		- }
	4075	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	4076	+ rnp = rdp->mynode;
	4077	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4078	+ rnp->ffmask \|= rdp->grpmask;
	4079	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3791	4080	if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
3792	4081	return 0; /* Too early in boot for scheduler work. */
3793	4082	sync_sched_exp_online_cleanup(cpu);
3794	4083	rcutree_affinity_setting(cpu, -1);
	4084	+
	4085	+ // Stop-machine done, so allow nohz_full to disable tick.
	4086	+ tick_dep_clear(TICK_DEP_BIT_RCU);
3795	4087	return 0;
3796	4088	}
3797	4089
..	..	@@ -3804,47 +4096,19 @@
3804	4096	unsigned long flags;
3805	4097	struct rcu_data *rdp;
3806	4098	struct rcu_node *rnp;
3807		- struct rcu_state *rsp;
3808	4099
3809		- for_each_rcu_flavor(rsp) {
3810		- rdp = per_cpu_ptr(rsp->rda, cpu);
3811		- rnp = rdp->mynode;
3812		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
3813		- rnp->ffmask &= ~rdp->grpmask;
3814		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3815		- }
	4100	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	4101	+ rnp = rdp->mynode;
	4102	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4103	+ rnp->ffmask &= ~rdp->grpmask;
	4104	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3816	4105
3817	4106	rcutree_affinity_setting(cpu, cpu);
	4107	+
	4108	+ // nohz_full CPUs need the tick for stop-machine to work quickly
	4109	+ tick_dep_set(TICK_DEP_BIT_RCU);
3818	4110	return 0;
3819	4111	}
3820		-
3821		-/*
3822		- * Near the end of the offline process. We do only tracing here.
3823		- */
3824		-int rcutree_dying_cpu(unsigned int cpu)
3825		-{
3826		- struct rcu_state *rsp;
3827		-
3828		- for_each_rcu_flavor(rsp)
3829		- rcu_cleanup_dying_cpu(rsp);
3830		- return 0;
3831		-}
3832		-
3833		-/*
3834		- * The outgoing CPU is gone and we are running elsewhere.
3835		- */
3836		-int rcutree_dead_cpu(unsigned int cpu)
3837		-{
3838		- struct rcu_state *rsp;
3839		-
3840		- for_each_rcu_flavor(rsp) {
3841		- rcu_cleanup_dead_cpu(cpu, rsp);
3842		- do_nocb_deferred_wakeup(per_cpu_ptr(rsp->rda, cpu));
3843		- }
3844		- return 0;
3845		-}
3846		-
3847		-static DEFINE_PER_CPU(int, rcu_cpu_started);
3848	4112
3849	4113	/*
3850	4114	* Mark the specified CPU as being online so that subsequent grace periods
..	..	@@ -3861,74 +4125,46 @@
3861	4125	{
3862	4126	unsigned long flags;
3863	4127	unsigned long mask;
3864		- int nbits;
3865		- unsigned long oldmask;
3866	4128	struct rcu_data *rdp;
3867	4129	struct rcu_node *rnp;
3868		- struct rcu_state *rsp;
	4130	+ bool newcpu;
3869	4131
3870		- if (per_cpu(rcu_cpu_started, cpu))
	4132	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	4133	+ if (rdp->cpu_started)
3871	4134	return;
	4135	+ rdp->cpu_started = true;
3872	4136
3873		- per_cpu(rcu_cpu_started, cpu) = 1;
3874		-
3875		- for_each_rcu_flavor(rsp) {
3876		- rdp = per_cpu_ptr(rsp->rda, cpu);
3877		- rnp = rdp->mynode;
3878		- mask = rdp->grpmask;
3879		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
3880		- rnp->qsmaskinitnext \|= mask;
3881		- oldmask = rnp->expmaskinitnext;
3882		- rnp->expmaskinitnext \|= mask;
3883		- oldmask ^= rnp->expmaskinitnext;
3884		- nbits = bitmap_weight(&oldmask, BITS_PER_LONG);
3885		- /* Allow lockless access for expedited grace periods. */
3886		- smp_store_release(&rsp->ncpus, rsp->ncpus + nbits); /* ^^^ */
3887		- rcu_gpnum_ovf(rnp, rdp); /* Offline-induced counter wrap? */
3888		- rdp->rcu_onl_gp_seq = READ_ONCE(rsp->gp_seq);
3889		- rdp->rcu_onl_gp_flags = READ_ONCE(rsp->gp_flags);
3890		- if (rnp->qsmask & mask) { /* RCU waiting on incoming CPU? */
3891		- /* Report QS -after- changing ->qsmaskinitnext! */
3892		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
3893		- } else {
3894		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3895		- }
3896		- }
3897		- smp_mb(); /* Ensure RCU read-side usage follows above initialization. */
3898		-}
3899		-
3900		-#ifdef CONFIG_HOTPLUG_CPU
3901		-/*
3902		- * The CPU is exiting the idle loop into the arch_cpu_idle_dead()
3903		- * function. We now remove it from the rcu_node tree's ->qsmaskinitnext
3904		- * bit masks.
3905		- */
3906		-static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp)
3907		-{
3908		- unsigned long flags;
3909		- unsigned long mask;
3910		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
3911		- struct rcu_node rnp = rdp->mynode; / Outgoing CPU's rdp & rnp. */
3912		-
3913		- /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
	4137	+ rnp = rdp->mynode;
3914	4138	mask = rdp->grpmask;
3915		- spin_lock(&rsp->ofl_lock);
3916		- raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
3917		- rdp->rcu_ofl_gp_seq = READ_ONCE(rsp->gp_seq);
3918		- rdp->rcu_ofl_gp_flags = READ_ONCE(rsp->gp_flags);
3919		- if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */
3920		- /* Report quiescent state -before- changing ->qsmaskinitnext! */
3921		- rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags);
3922		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4139	+ WRITE_ONCE(rnp->ofl_seq, rnp->ofl_seq + 1);
	4140	+ WARN_ON_ONCE(!(rnp->ofl_seq & 0x1));
	4141	+ smp_mb(); // Pair with rcu_gp_cleanup()'s ->ofl_seq barrier().
	4142	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4143	+ WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext \| mask);
	4144	+ newcpu = !(rnp->expmaskinitnext & mask);
	4145	+ rnp->expmaskinitnext \|= mask;
	4146	+ /* Allow lockless access for expedited grace periods. */
	4147	+ smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + newcpu); /* ^^^ */
	4148	+ ASSERT_EXCLUSIVE_WRITER(rcu_state.ncpus);
	4149	+ rcu_gpnum_ovf(rnp, rdp); /* Offline-induced counter wrap? */
	4150	+ rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);
	4151	+ rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags);
	4152	+ if (rnp->qsmask & mask) { /* RCU waiting on incoming CPU? */
	4153	+ rcu_disable_urgency_upon_qs(rdp);
	4154	+ /* Report QS -after- changing ->qsmaskinitnext! */
	4155	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
	4156	+ } else {
	4157	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3923	4158	}
3924		- rnp->qsmaskinitnext &= ~mask;
3925		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3926		- spin_unlock(&rsp->ofl_lock);
	4159	+ smp_mb(); // Pair with rcu_gp_cleanup()'s ->ofl_seq barrier().
	4160	+ WRITE_ONCE(rnp->ofl_seq, rnp->ofl_seq + 1);
	4161	+ WARN_ON_ONCE(rnp->ofl_seq & 0x1);
	4162	+ smp_mb(); /* Ensure RCU read-side usage follows above initialization. */
3927	4163	}
3928	4164
3929	4165	/*
3930	4166	* The outgoing function has no further need of RCU, so remove it from
3931		- * the list of CPUs that RCU must track.
	4167	+ * the rcu_node tree's ->qsmaskinitnext bit masks.
3932	4168	*
3933	4169	* Note that this function is special in that it is invoked directly
3934	4170	* from the outgoing CPU rather than from the cpuhp_step mechanism.
..	..	@@ -3936,65 +4172,88 @@
3936	4172	*/
3937	4173	void rcu_report_dead(unsigned int cpu)
3938	4174	{
3939		- struct rcu_state *rsp;
	4175	+ unsigned long flags;
	4176	+ unsigned long mask;
	4177	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
	4178	+ struct rcu_node rnp = rdp->mynode; / Outgoing CPU's rdp & rnp. */
3940	4179
3941		- /* QS for any half-done expedited RCU-sched GP. */
	4180	+ /* QS for any half-done expedited grace period. */
3942	4181	preempt_disable();
3943		- rcu_report_exp_rdp(&rcu_sched_state,
3944		- this_cpu_ptr(rcu_sched_state.rda), true);
	4182	+ rcu_report_exp_rdp(this_cpu_ptr(&rcu_data));
3945	4183	preempt_enable();
3946		- for_each_rcu_flavor(rsp)
3947		- rcu_cleanup_dying_idle_cpu(cpu, rsp);
	4184	+ rcu_preempt_deferred_qs(current);
3948	4185
3949		- per_cpu(rcu_cpu_started, cpu) = 0;
	4186	+ /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
	4187	+ mask = rdp->grpmask;
	4188	+ WRITE_ONCE(rnp->ofl_seq, rnp->ofl_seq + 1);
	4189	+ WARN_ON_ONCE(!(rnp->ofl_seq & 0x1));
	4190	+ smp_mb(); // Pair with rcu_gp_cleanup()'s ->ofl_seq barrier().
	4191	+ raw_spin_lock(&rcu_state.ofl_lock);
	4192	+ raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
	4193	+ rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);
	4194	+ rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags);
	4195	+ if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */
	4196	+ /* Report quiescent state -before- changing ->qsmaskinitnext! */
	4197	+ rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
	4198	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4199	+ }
	4200	+ WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext & ~mask);
	4201	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
	4202	+ raw_spin_unlock(&rcu_state.ofl_lock);
	4203	+ smp_mb(); // Pair with rcu_gp_cleanup()'s ->ofl_seq barrier().
	4204	+ WRITE_ONCE(rnp->ofl_seq, rnp->ofl_seq + 1);
	4205	+ WARN_ON_ONCE(rnp->ofl_seq & 0x1);
	4206	+
	4207	+ rdp->cpu_started = false;
3950	4208	}
3951	4209
3952		-/* Migrate the dead CPU's callbacks to the current CPU. */
3953		-static void rcu_migrate_callbacks(int cpu, struct rcu_state *rsp)
	4210	+#ifdef CONFIG_HOTPLUG_CPU
	4211	+/*
	4212	+ * The outgoing CPU has just passed through the dying-idle state, and we
	4213	+ * are being invoked from the CPU that was IPIed to continue the offline
	4214	+ * operation. Migrate the outgoing CPU's callbacks to the current CPU.
	4215	+ */
	4216	+void rcutree_migrate_callbacks(int cpu)
3954	4217	{
3955	4218	unsigned long flags;
3956	4219	struct rcu_data *my_rdp;
3957		- struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
3958		- struct rcu_node *rnp_root = rcu_get_root(rdp->rsp);
	4220	+ struct rcu_node *my_rnp;
	4221	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3959	4222	bool needwake;
3960	4223
3961		- if (rcu_is_nocb_cpu(cpu) \|\| rcu_segcblist_empty(&rdp->cblist))
	4224	+ if (rcu_segcblist_is_offloaded(&rdp->cblist) \|\|
	4225	+ rcu_segcblist_empty(&rdp->cblist))
3962	4226	return; /* No callbacks to migrate. */
3963	4227
3964	4228	local_irq_save(flags);
3965		- my_rdp = this_cpu_ptr(rsp->rda);
3966		- if (rcu_nocb_adopt_orphan_cbs(my_rdp, rdp, flags)) {
3967		- local_irq_restore(flags);
3968		- return;
3969		- }
3970		- raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
	4229	+ my_rdp = this_cpu_ptr(&rcu_data);
	4230	+ my_rnp = my_rdp->mynode;
	4231	+ rcu_nocb_lock(my_rdp); /* irqs already disabled. */
	4232	+ WARN_ON_ONCE(!rcu_nocb_flush_bypass(my_rdp, NULL, jiffies));
	4233	+ raw_spin_lock_rcu_node(my_rnp); /* irqs already disabled. */
3971	4234	/* Leverage recent GPs and set GP for new callbacks. */
3972		- needwake = rcu_advance_cbs(rsp, rnp_root, rdp) \|\|
3973		- rcu_advance_cbs(rsp, rnp_root, my_rdp);
	4235	+ needwake = rcu_advance_cbs(my_rnp, rdp) \|\|
	4236	+ rcu_advance_cbs(my_rnp, my_rdp);
3974	4237	rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist);
	4238	+ needwake = needwake \|\| rcu_advance_cbs(my_rnp, my_rdp);
	4239	+ rcu_segcblist_disable(&rdp->cblist);
3975	4240	WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) !=
3976	4241	!rcu_segcblist_n_cbs(&my_rdp->cblist));
3977		- raw_spin_unlock_irqrestore_rcu_node(rnp_root, flags);
	4242	+ if (rcu_segcblist_is_offloaded(&my_rdp->cblist)) {
	4243	+ raw_spin_unlock_rcu_node(my_rnp); /* irqs remain disabled. */
	4244	+ __call_rcu_nocb_wake(my_rdp, true, flags);
	4245	+ } else {
	4246	+ rcu_nocb_unlock(my_rdp); /* irqs remain disabled. */
	4247	+ raw_spin_unlock_irqrestore_rcu_node(my_rnp, flags);
	4248	+ }
3978	4249	if (needwake)
3979		- rcu_gp_kthread_wake(rsp);
	4250	+ rcu_gp_kthread_wake();
	4251	+ lockdep_assert_irqs_enabled();
3980	4252	WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 \|\|
3981	4253	!rcu_segcblist_empty(&rdp->cblist),
3982	4254	"rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n",
3983	4255	cpu, rcu_segcblist_n_cbs(&rdp->cblist),
3984	4256	rcu_segcblist_first_cb(&rdp->cblist));
3985		-}
3986		-
3987		-/*
3988		- * The outgoing CPU has just passed through the dying-idle state,
3989		- * and we are being invoked from the CPU that was IPIed to continue the
3990		- * offline operation. We need to migrate the outgoing CPU's callbacks.
3991		- */
3992		-void rcutree_migrate_callbacks(int cpu)
3993		-{
3994		- struct rcu_state *rsp;
3995		-
3996		- for_each_rcu_flavor(rsp)
3997		- rcu_migrate_callbacks(cpu, rsp);
3998	4257	}
3999	4258	#endif
4000	4259
..	..	@@ -4008,13 +4267,11 @@
4008	4267	switch (action) {
4009	4268	case PM_HIBERNATION_PREPARE:
4010	4269	case PM_SUSPEND_PREPARE:
4011		- if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
4012		- rcu_expedite_gp();
	4270	+ rcu_expedite_gp();
4013	4271	break;
4014	4272	case PM_POST_HIBERNATION:
4015	4273	case PM_POST_SUSPEND:
4016		- if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
4017		- rcu_unexpedite_gp();
	4274	+ rcu_unexpedite_gp();
4018	4275	break;
4019	4276	default:
4020	4277	break;
..	..	@@ -4023,14 +4280,13 @@
4023	4280	}
4024	4281
4025	4282	/*
4026		- * Spawn the kthreads that handle each RCU flavor's grace periods.
	4283	+ * Spawn the kthreads that handle RCU's grace periods.
4027	4284	*/
4028	4285	static int __init rcu_spawn_gp_kthread(void)
4029	4286	{
4030	4287	unsigned long flags;
4031	4288	int kthread_prio_in = kthread_prio;
4032	4289	struct rcu_node *rnp;
4033		- struct rcu_state *rsp;
4034	4290	struct sched_param sp;
4035	4291	struct task_struct *t;
4036	4292
..	..	@@ -4050,21 +4306,24 @@
4050	4306	kthread_prio, kthread_prio_in);
4051	4307
4052	4308	rcu_scheduler_fully_active = 1;
4053		- for_each_rcu_flavor(rsp) {
4054		- t = kthread_create(rcu_gp_kthread, rsp, "%s", rsp->name);
4055		- BUG_ON(IS_ERR(t));
4056		- rnp = rcu_get_root(rsp);
4057		- raw_spin_lock_irqsave_rcu_node(rnp, flags);
4058		- rsp->gp_kthread = t;
4059		- if (kthread_prio) {
4060		- sp.sched_priority = kthread_prio;
4061		- sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
4062		- }
4063		- raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
4064		- wake_up_process(t);
	4309	+ t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name);
	4310	+ if (WARN_ONCE(IS_ERR(t), "%s: Could not start grace-period kthread, OOM is now expected behavior\n", __func__))
	4311	+ return 0;
	4312	+ if (kthread_prio) {
	4313	+ sp.sched_priority = kthread_prio;
	4314	+ sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
4065	4315	}
	4316	+ rnp = rcu_get_root();
	4317	+ raw_spin_lock_irqsave_rcu_node(rnp, flags);
	4318	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	4319	+ WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
	4320	+ // Reset .gp_activity and .gp_req_activity before setting .gp_kthread.
	4321	+ smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */
	4322	+ raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
	4323	+ wake_up_process(t);
4066	4324	rcu_spawn_nocb_kthreads();
4067	4325	rcu_spawn_boost_kthreads();
	4326	+ rcu_spawn_core_kthreads();
4068	4327	return 0;
4069	4328	}
4070	4329	early_initcall(rcu_spawn_gp_kthread);
..	..	@@ -4089,9 +4348,9 @@
4089	4348	}
4090	4349
4091	4350	/*
4092		- * Helper function for rcu_init() that initializes one rcu_state structure.
	4351	+ * Helper function for rcu_init() that initializes the rcu_state structure.
4093	4352	*/
4094		-static void __init rcu_init_one(struct rcu_state *rsp)
	4353	+static void __init rcu_init_one(void)
4095	4354	{
4096	4355	static const char * const buf[] = RCU_NODE_NAME_INIT;
4097	4356	static const char * const fqs[] = RCU_FQS_NAME_INIT;
..	..	@@ -4113,14 +4372,15 @@
4113	4372	/* Initialize the level-tracking arrays. */
4114	4373
4115	4374	for (i = 1; i < rcu_num_lvls; i++)
4116		- rsp->level[i] = rsp->level[i - 1] + num_rcu_lvl[i - 1];
	4375	+ rcu_state.level[i] =
	4376	+ rcu_state.level[i - 1] + num_rcu_lvl[i - 1];
4117	4377	rcu_init_levelspread(levelspread, num_rcu_lvl);
4118	4378
4119	4379	/* Initialize the elements themselves, starting from the leaves. */
4120	4380
4121	4381	for (i = rcu_num_lvls - 1; i >= 0; i--) {
4122	4382	cpustride *= levelspread[i];
4123		- rnp = rsp->level[i];
	4383	+ rnp = rcu_state.level[i];
4124	4384	for (j = 0; j < num_rcu_lvl[i]; j++, rnp++) {
4125	4385	raw_spin_lock_init(&ACCESS_PRIVATE(rnp, lock));
4126	4386	lockdep_set_class_and_name(&ACCESS_PRIVATE(rnp, lock),
..	..	@@ -4128,9 +4388,9 @@
4128	4388	raw_spin_lock_init(&rnp->fqslock);
4129	4389	lockdep_set_class_and_name(&rnp->fqslock,
4130	4390	&rcu_fqs_class[i], fqs[i]);
4131		- rnp->gp_seq = rsp->gp_seq;
4132		- rnp->gp_seq_needed = rsp->gp_seq;
4133		- rnp->completedqs = rsp->gp_seq;
	4391	+ rnp->gp_seq = rcu_state.gp_seq;
	4392	+ rnp->gp_seq_needed = rcu_state.gp_seq;
	4393	+ rnp->completedqs = rcu_state.gp_seq;
4134	4394	rnp->qsmask = 0;
4135	4395	rnp->qsmaskinit = 0;
4136	4396	rnp->grplo = j * cpustride;
..	..	@@ -4143,8 +4403,8 @@
4143	4403	rnp->parent = NULL;
4144	4404	} else {
4145	4405	rnp->grpnum = j % levelspread[i - 1];
4146		- rnp->grpmask = 1UL << rnp->grpnum;
4147		- rnp->parent = rsp->level[i - 1] +
	4406	+ rnp->grpmask = BIT(rnp->grpnum);
	4407	+ rnp->parent = rcu_state.level[i - 1] +
4148	4408	j / levelspread[i - 1];
4149	4409	}
4150	4410	rnp->level = i;
..	..	@@ -4158,16 +4418,15 @@
4158	4418	}
4159	4419	}
4160	4420
4161		- init_swait_queue_head(&rsp->gp_wq);
4162		- init_swait_queue_head(&rsp->expedited_wq);
4163		- rnp = rcu_first_leaf_node(rsp);
	4421	+ init_swait_queue_head(&rcu_state.gp_wq);
	4422	+ init_swait_queue_head(&rcu_state.expedited_wq);
	4423	+ rnp = rcu_first_leaf_node();
4164	4424	for_each_possible_cpu(i) {
4165	4425	while (i > rnp->grphi)
4166	4426	rnp++;
4167		- per_cpu_ptr(rsp->rda, i)->mynode = rnp;
4168		- rcu_boot_init_percpu_data(i, rsp);
	4427	+ per_cpu_ptr(&rcu_data, i)->mynode = rnp;
	4428	+ rcu_boot_init_percpu_data(i);
4169	4429	}
4170		- list_add(&rsp->flavors, &rcu_struct_flavors);
4171	4430	}
4172	4431
4173	4432	/*
..	..	@@ -4175,11 +4434,25 @@
4175	4434	* replace the definitions in tree.h because those are needed to size
4176	4435	* the ->node array in the rcu_state structure.
4177	4436	*/
4178		-static void __init rcu_init_geometry(void)
	4437	+void rcu_init_geometry(void)
4179	4438	{
4180	4439	ulong d;
4181	4440	int i;
	4441	+ static unsigned long old_nr_cpu_ids;
4182	4442	int rcu_capacity[RCU_NUM_LVLS];
	4443	+ static bool initialized;
	4444	+
	4445	+ if (initialized) {
	4446	+ /*
	4447	+ * Warn if setup_nr_cpu_ids() had not yet been invoked,
	4448	+ * unless nr_cpus_ids == NR_CPUS, in which case who cares?
	4449	+ */
	4450	+ WARN_ON_ONCE(old_nr_cpu_ids != nr_cpu_ids);
	4451	+ return;
	4452	+ }
	4453	+
	4454	+ old_nr_cpu_ids = nr_cpu_ids;
	4455	+ initialized = true;
4183	4456
4184	4457	/*
4185	4458	* Initialize any unspecified boot parameters.
..	..	@@ -4193,6 +4466,7 @@
4193	4466	jiffies_till_first_fqs = d;
4194	4467	if (jiffies_till_next_fqs == ULONG_MAX)
4195	4468	jiffies_till_next_fqs = d;
	4469	+ adjust_jiffies_till_sched_qs();
4196	4470
4197	4471	/* If the compile-time values are accurate, just leave. */
4198	4472	if (rcu_fanout_leaf == RCU_FANOUT_LEAF &&
..	..	@@ -4251,16 +4525,16 @@
4251	4525
4252	4526	/*
4253	4527	* Dump out the structure of the rcu_node combining tree associated
4254		- * with the rcu_state structure referenced by rsp.
	4528	+ * with the rcu_state structure.
4255	4529	*/
4256		-static void __init rcu_dump_rcu_node_tree(struct rcu_state *rsp)
	4530	+static void __init rcu_dump_rcu_node_tree(void)
4257	4531	{
4258	4532	int level = 0;
4259	4533	struct rcu_node *rnp;
4260	4534
4261	4535	pr_info("rcu_node tree layout dump\n");
4262	4536	pr_info(" ");
4263		- rcu_for_each_node_breadth_first(rsp, rnp) {
	4537	+ rcu_for_each_node_breadth_first(rnp) {
4264	4538	if (rnp->level != level) {
4265	4539	pr_cont("\n");
4266	4540	pr_info(" ");
..	..	@@ -4274,21 +4548,41 @@
4274	4548	struct workqueue_struct *rcu_gp_wq;
4275	4549	struct workqueue_struct *rcu_par_gp_wq;
4276	4550
	4551	+static void __init kfree_rcu_batch_init(void)
	4552	+{
	4553	+ int cpu;
	4554	+ int i;
	4555	+
	4556	+ for_each_possible_cpu(cpu) {
	4557	+ struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
	4558	+
	4559	+ for (i = 0; i < KFREE_N_BATCHES; i++) {
	4560	+ INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
	4561	+ krcp->krw_arr[i].krcp = krcp;
	4562	+ }
	4563	+
	4564	+ INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
	4565	+ INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
	4566	+ krcp->initialized = true;
	4567	+ }
	4568	+ if (register_shrinker(&kfree_rcu_shrinker))
	4569	+ pr_err("Failed to register kfree_rcu() shrinker!\n");
	4570	+}
	4571	+
4277	4572	void __init rcu_init(void)
4278	4573	{
4279	4574	int cpu;
4280	4575
4281	4576	rcu_early_boot_tests();
4282	4577
	4578	+ kfree_rcu_batch_init();
4283	4579	rcu_bootup_announce();
4284	4580	rcu_init_geometry();
4285		-#ifndef CONFIG_PREEMPT_RT_FULL
4286		- rcu_init_one(&rcu_bh_state);
4287		-#endif
4288		- rcu_init_one(&rcu_sched_state);
	4581	+ rcu_init_one();
4289	4582	if (dump_tree)
4290		- rcu_dump_rcu_node_tree(&rcu_sched_state);
4291		- __rcu_init_preempt();
	4583	+ rcu_dump_rcu_node_tree();
	4584	+ if (use_softirq)
	4585	+ open_softirq(RCU_SOFTIRQ, rcu_core_si);
4292	4586
4293	4587	/*
4294	4588	* We don't need protection against CPU-hotplug here because
..	..	@@ -4307,7 +4601,16 @@
4307	4601	WARN_ON(!rcu_gp_wq);
4308	4602	rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);
4309	4603	WARN_ON(!rcu_par_gp_wq);
	4604	+ srcu_init();
	4605	+
	4606	+ /* Fill in default value for rcutree.qovld boot parameter. */
	4607	+ /* -After- the rcu_node ->lock fields are initialized! */
	4608	+ if (qovld < 0)
	4609	+ qovld_calc = DEFAULT_RCU_QOVLD_MULT * qhimark;
	4610	+ else
	4611	+ qovld_calc = qovld;
4310	4612	}
4311	4613
	4614	+#include "tree_stall.h"
4312	4615	#include "tree_exp.h"
4313	4616	#include "tree_plugin.h"