forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/kernel/rcu/srcutiny.c
....@@ -1,24 +1,11 @@
1
+// SPDX-License-Identifier: GPL-2.0+
12 /*
23  * Sleepable Read-Copy Update mechanism for mutual exclusion,
34  *	tiny version for non-preemptible single-CPU use.
45  *
5
- * This program is free software; you can redistribute it and/or modify
6
- * it under the terms of the GNU General Public License as published by
7
- * the Free Software Foundation; either version 2 of the License, or
8
- * (at your option) any later version.
9
- *
10
- * This program is distributed in the hope that it will be useful,
11
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
12
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13
- * GNU General Public License for more details.
14
- *
15
- * You should have received a copy of the GNU General Public License
16
- * along with this program; if not, you can access it online at
17
- * http://www.gnu.org/licenses/gpl-2.0.html.
18
- *
196  * Copyright (C) IBM Corporation, 2017
207  *
21
- * Author: Paul McKenney <paulmck@us.ibm.com>
8
+ * Author: Paul McKenney <paulmck@linux.ibm.com>
229  */
2310 
2411 #include <linux/export.h>
....@@ -34,30 +21,34 @@
3421 #include "rcu.h"
3522 
3623 int rcu_scheduler_active __read_mostly;
24
+static LIST_HEAD(srcu_boot_list);
25
+static bool srcu_init_done;
3726 
38
-static int init_srcu_struct_fields(struct srcu_struct *sp)
27
+static int init_srcu_struct_fields(struct srcu_struct *ssp)
3928 {
40
-	sp->srcu_lock_nesting[0] = 0;
41
-	sp->srcu_lock_nesting[1] = 0;
42
-	init_swait_queue_head(&sp->srcu_wq);
43
-	sp->srcu_cb_head = NULL;
44
-	sp->srcu_cb_tail = &sp->srcu_cb_head;
45
-	sp->srcu_gp_running = false;
46
-	sp->srcu_gp_waiting = false;
47
-	sp->srcu_idx = 0;
48
-	INIT_WORK(&sp->srcu_work, srcu_drive_gp);
29
+	ssp->srcu_lock_nesting[0] = 0;
30
+	ssp->srcu_lock_nesting[1] = 0;
31
+	init_swait_queue_head(&ssp->srcu_wq);
32
+	ssp->srcu_cb_head = NULL;
33
+	ssp->srcu_cb_tail = &ssp->srcu_cb_head;
34
+	ssp->srcu_gp_running = false;
35
+	ssp->srcu_gp_waiting = false;
36
+	ssp->srcu_idx = 0;
37
+	ssp->srcu_idx_max = 0;
38
+	INIT_WORK(&ssp->srcu_work, srcu_drive_gp);
39
+	INIT_LIST_HEAD(&ssp->srcu_work.entry);
4940 	return 0;
5041 }
5142 
5243 #ifdef CONFIG_DEBUG_LOCK_ALLOC
5344 
54
-int __init_srcu_struct(struct srcu_struct *sp, const char *name,
45
+int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
5546 		       struct lock_class_key *key)
5647 {
5748 	/* Don't re-initialize a lock while it is held. */
58
-	debug_check_no_locks_freed((void *)sp, sizeof(*sp));
59
-	lockdep_init_map(&sp->dep_map, name, key, 0);
60
-	return init_srcu_struct_fields(sp);
49
+	debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));
50
+	lockdep_init_map(&ssp->dep_map, name, key, 0);
51
+	return init_srcu_struct_fields(ssp);
6152 }
6253 EXPORT_SYMBOL_GPL(__init_srcu_struct);
6354 
....@@ -65,15 +56,15 @@
6556 
6657 /*
6758  * init_srcu_struct - initialize a sleep-RCU structure
68
- * @sp: structure to initialize.
59
+ * @ssp: structure to initialize.
6960  *
7061  * Must invoke this on a given srcu_struct before passing that srcu_struct
7162  * to any other function.  Each srcu_struct represents a separate domain
7263  * of SRCU protection.
7364  */
74
-int init_srcu_struct(struct srcu_struct *sp)
65
+int init_srcu_struct(struct srcu_struct *ssp)
7566 {
76
-	return init_srcu_struct_fields(sp);
67
+	return init_srcu_struct_fields(ssp);
7768 }
7869 EXPORT_SYMBOL_GPL(init_srcu_struct);
7970 
....@@ -81,42 +72,41 @@
8172 
8273 /*
8374  * cleanup_srcu_struct - deconstruct a sleep-RCU structure
84
- * @sp: structure to clean up.
75
+ * @ssp: structure to clean up.
8576  *
8677  * Must invoke this after you are finished using a given srcu_struct that
8778  * was initialized via init_srcu_struct(), else you leak memory.
8879  */
89
-void _cleanup_srcu_struct(struct srcu_struct *sp, bool quiesced)
80
+void cleanup_srcu_struct(struct srcu_struct *ssp)
9081 {
91
-	WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]);
92
-	if (quiesced)
93
-		WARN_ON(work_pending(&sp->srcu_work));
94
-	else
95
-		flush_work(&sp->srcu_work);
96
-	WARN_ON(sp->srcu_gp_running);
97
-	WARN_ON(sp->srcu_gp_waiting);
98
-	WARN_ON(sp->srcu_cb_head);
99
-	WARN_ON(&sp->srcu_cb_head != sp->srcu_cb_tail);
82
+	WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]);
83
+	flush_work(&ssp->srcu_work);
84
+	WARN_ON(ssp->srcu_gp_running);
85
+	WARN_ON(ssp->srcu_gp_waiting);
86
+	WARN_ON(ssp->srcu_cb_head);
87
+	WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail);
88
+	WARN_ON(ssp->srcu_idx != ssp->srcu_idx_max);
89
+	WARN_ON(ssp->srcu_idx & 0x1);
10090 }
101
-EXPORT_SYMBOL_GPL(_cleanup_srcu_struct);
91
+EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
10292 
10393 /*
10494  * Removes the count for the old reader from the appropriate element of
10595  * the srcu_struct.
10696  */
107
-void __srcu_read_unlock(struct srcu_struct *sp, int idx)
97
+void __srcu_read_unlock(struct srcu_struct *ssp, int idx)
10898 {
109
-	int newval = sp->srcu_lock_nesting[idx] - 1;
99
+	int newval = ssp->srcu_lock_nesting[idx] - 1;
110100 
111
-	WRITE_ONCE(sp->srcu_lock_nesting[idx], newval);
112
-	if (!newval && READ_ONCE(sp->srcu_gp_waiting))
113
-		swake_up_one(&sp->srcu_wq);
101
+	WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval);
102
+	if (!newval && READ_ONCE(ssp->srcu_gp_waiting))
103
+		swake_up_one(&ssp->srcu_wq);
114104 }
115105 EXPORT_SYMBOL_GPL(__srcu_read_unlock);
116106 
117107 /*
118108  * Workqueue handler to drive one grace period and invoke any callbacks
119
- * that become ready as a result.  Single-CPU and !PREEMPT operation
109
+ * that become ready as a result.  Single-CPU and !PREEMPTION operation
120110  * means that we get away with murder on synchronization.  ;-)
121111  */
122112 void srcu_drive_gp(struct work_struct *wp)
....@@ -124,24 +114,25 @@
124114 	int idx;
125115 	struct rcu_head *lh;
126116 	struct rcu_head *rhp;
127
-	struct srcu_struct *sp;
117
+	struct srcu_struct *ssp;
128118 
129
-	sp = container_of(wp, struct srcu_struct, srcu_work);
130
-	if (sp->srcu_gp_running || !READ_ONCE(sp->srcu_cb_head))
119
+	ssp = container_of(wp, struct srcu_struct, srcu_work);
120
+	if (ssp->srcu_gp_running || USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
131121 		return; /* Already running or nothing to do. */
132122 
133123 	/* Remove recently arrived callbacks and wait for readers. */
134
-	WRITE_ONCE(sp->srcu_gp_running, true);
124
+	WRITE_ONCE(ssp->srcu_gp_running, true);
135125 	local_irq_disable();
136
-	lh = sp->srcu_cb_head;
137
-	sp->srcu_cb_head = NULL;
138
-	sp->srcu_cb_tail = &sp->srcu_cb_head;
126
+	lh = ssp->srcu_cb_head;
127
+	ssp->srcu_cb_head = NULL;
128
+	ssp->srcu_cb_tail = &ssp->srcu_cb_head;
139129 	local_irq_enable();
140
-	idx = sp->srcu_idx;
141
-	WRITE_ONCE(sp->srcu_idx, !sp->srcu_idx);
142
-	WRITE_ONCE(sp->srcu_gp_waiting, true);  /* srcu_read_unlock() wakes! */
143
-	swait_event_exclusive(sp->srcu_wq, !READ_ONCE(sp->srcu_lock_nesting[idx]));
144
-	WRITE_ONCE(sp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */
130
+	idx = (ssp->srcu_idx & 0x2) / 2;
131
+	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);
132
+	WRITE_ONCE(ssp->srcu_gp_waiting, true);  /* srcu_read_unlock() wakes! */
133
+	swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx]));
134
+	WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */
135
+	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);
145136 
146137 	/* Invoke the callbacks we removed above. */
147138 	while (lh) {
....@@ -158,17 +149,33 @@
158149 	 * at interrupt level, but the ->srcu_gp_running checks will
159150 	 * straighten that out.
160151 	 */
161
-	WRITE_ONCE(sp->srcu_gp_running, false);
162
-	if (READ_ONCE(sp->srcu_cb_head))
163
-		schedule_work(&sp->srcu_work);
152
+	WRITE_ONCE(ssp->srcu_gp_running, false);
153
+	if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
154
+		schedule_work(&ssp->srcu_work);
164155 }
165156 EXPORT_SYMBOL_GPL(srcu_drive_gp);
157
+
158
+static void srcu_gp_start_if_needed(struct srcu_struct *ssp)
159
+{
160
+	unsigned short cookie;
161
+
162
+	cookie = get_state_synchronize_srcu(ssp);
163
+	if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))
164
+		return;
165
+	WRITE_ONCE(ssp->srcu_idx_max, cookie);
166
+	if (!READ_ONCE(ssp->srcu_gp_running)) {
167
+		if (likely(srcu_init_done))
168
+			schedule_work(&ssp->srcu_work);
169
+		else if (list_empty(&ssp->srcu_work.entry))
170
+			list_add(&ssp->srcu_work.entry, &srcu_boot_list);
171
+	}
172
+}
166173 
167174 /*
168175  * Enqueue an SRCU callback on the specified srcu_struct structure,
169176  * initiating grace-period processing if it is not already running.
170177  */
171
-void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp,
178
+void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
172179 	       rcu_callback_t func)
173180 {
174181 	unsigned long flags;
....@@ -176,31 +183,90 @@
176183 	rhp->func = func;
177184 	rhp->next = NULL;
178185 	local_irq_save(flags);
179
-	*sp->srcu_cb_tail = rhp;
180
-	sp->srcu_cb_tail = &rhp->next;
186
+	*ssp->srcu_cb_tail = rhp;
187
+	ssp->srcu_cb_tail = &rhp->next;
181188 	local_irq_restore(flags);
182
-	if (!READ_ONCE(sp->srcu_gp_running))
183
-		schedule_work(&sp->srcu_work);
189
+	srcu_gp_start_if_needed(ssp);
184190 }
185191 EXPORT_SYMBOL_GPL(call_srcu);
186192 
187193 /*
188194  * synchronize_srcu - wait for prior SRCU read-side critical-section completion
189195  */
190
-void synchronize_srcu(struct srcu_struct *sp)
196
+void synchronize_srcu(struct srcu_struct *ssp)
191197 {
192198 	struct rcu_synchronize rs;
193199 
194200 	init_rcu_head_on_stack(&rs.head);
195201 	init_completion(&rs.completion);
196
-	call_srcu(sp, &rs.head, wakeme_after_rcu);
202
+	call_srcu(ssp, &rs.head, wakeme_after_rcu);
197203 	wait_for_completion(&rs.completion);
198204 	destroy_rcu_head_on_stack(&rs.head);
199205 }
200206 EXPORT_SYMBOL_GPL(synchronize_srcu);
201207 
208
+/*
209
+ * get_state_synchronize_srcu - Provide an end-of-grace-period cookie
210
+ */
211
+unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)
212
+{
213
+	unsigned long ret;
214
+
215
+	barrier();
216
+	ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1;
217
+	barrier();
218
+	return ret & USHRT_MAX;
219
+}
220
+EXPORT_SYMBOL_GPL(get_state_synchronize_srcu);
221
+
222
+/*
223
+ * start_poll_synchronize_srcu - Provide cookie and start grace period
224
+ *
225
+ * The difference between this and get_state_synchronize_srcu() is that
226
+ * this function ensures that the poll_state_synchronize_srcu() will
227
+ * eventually return the value true.
228
+ */
229
+unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)
230
+{
231
+	unsigned long ret = get_state_synchronize_srcu(ssp);
232
+
233
+	srcu_gp_start_if_needed(ssp);
234
+	return ret;
235
+}
236
+EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu);
237
+
238
+/*
239
+ * poll_state_synchronize_srcu - Has cookie's grace period ended?
240
+ */
241
+bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)
242
+{
243
+	bool ret = USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx), cookie);
244
+
245
+	barrier();
246
+	return ret;
247
+}
248
+EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu);
249
+
202250 /* Lockdep diagnostics.  */
203251 void __init rcu_scheduler_starting(void)
204252 {
205253 	rcu_scheduler_active = RCU_SCHEDULER_RUNNING;
206254 }
255
+
256
+/*
257
+ * Queue work for srcu_struct structures with early boot callbacks.
258
+ * The work won't actually execute until the workqueue initialization
259
+ * phase that takes place after the scheduler starts.
260
+ */
261
+void __init srcu_init(void)
262
+{
263
+	struct srcu_struct *ssp;
264
+
265
+	srcu_init_done = true;
266
+	while (!list_empty(&srcu_boot_list)) {
267
+		ssp = list_first_entry(&srcu_boot_list,
268
+				      struct srcu_struct, srcu_work.entry);
269
+		list_del_init(&ssp->srcu_work.entry);
270
+		schedule_work(&ssp->srcu_work);
271
+	}
272
+}