forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/kernel/rcu/rcu_segcblist.c
....@@ -1,23 +1,10 @@
1
+// SPDX-License-Identifier: GPL-2.0+
12 /*
23  * RCU segmented callback lists, function definitions
34  *
4
- * This program is free software; you can redistribute it and/or modify
5
- * it under the terms of the GNU General Public License as published by
6
- * the Free Software Foundation; either version 2 of the License, or
7
- * (at your option) any later version.
8
- *
9
- * This program is distributed in the hope that it will be useful,
10
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
- * GNU General Public License for more details.
13
- *
14
- * You should have received a copy of the GNU General Public License
15
- * along with this program; if not, you can access it online at
16
- * http://www.gnu.org/licenses/gpl-2.0.html.
17
- *
185  * Copyright IBM Corporation, 2017
196  *
20
- * Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
7
+ * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
218  */
229 
2310 #include <linux/types.h>
....@@ -33,15 +20,49 @@
3320 	rclp->head = NULL;
3421 	rclp->tail = &rclp->head;
3522 	rclp->len = 0;
36
-	rclp->len_lazy = 0;
23
+}
24
+
25
+/*
26
+ * Enqueue an rcu_head structure onto the specified callback list.
27
+ */
28
+void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp)
29
+{
30
+	*rclp->tail = rhp;
31
+	rclp->tail = &rhp->next;
32
+	WRITE_ONCE(rclp->len, rclp->len + 1);
33
+}
34
+
35
+/*
36
+ * Flush the second rcu_cblist structure onto the first one, obliterating
37
+ * any contents of the first.  If rhp is non-NULL, enqueue it as the sole
38
+ * element of the second rcu_cblist structure, but ensuring that the second
39
+ * rcu_cblist structure, if initially non-empty, always appears non-empty
40
+ * throughout the process.  If rdp is NULL, the second rcu_cblist structure
41
+ * is instead initialized to empty.
42
+ */
43
+void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
44
+			      struct rcu_cblist *srclp,
45
+			      struct rcu_head *rhp)
46
+{
47
+	drclp->head = srclp->head;
48
+	if (drclp->head)
49
+		drclp->tail = srclp->tail;
50
+	else
51
+		drclp->tail = &drclp->head;
52
+	drclp->len = srclp->len;
53
+	if (!rhp) {
54
+		rcu_cblist_init(srclp);
55
+	} else {
56
+		rhp->next = NULL;
57
+		srclp->head = rhp;
58
+		srclp->tail = &rhp->next;
59
+		WRITE_ONCE(srclp->len, 1);
60
+	}
3761 }
3862 
3963 /*
4064  * Dequeue the oldest rcu_head structure from the specified callback
41
- * list.  This function assumes that the callback is non-lazy, but
42
- * the caller can later invoke rcu_cblist_dequeued_lazy() if it
43
- * finds otherwise (and if it cares about laziness).  This allows
44
- * different users to have different ways of determining laziness.
65
+ * list.
4566  */
4667 struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp)
4768 {
....@@ -57,6 +78,67 @@
5778 	return rhp;
5879 }
5980 
81
+/* Set the length of an rcu_segcblist structure. */
82
+static void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v)
83
+{
84
+#ifdef CONFIG_RCU_NOCB_CPU
85
+	atomic_long_set(&rsclp->len, v);
86
+#else
87
+	WRITE_ONCE(rsclp->len, v);
88
+#endif
89
+}
90
+
91
+/*
92
+ * Increase the numeric length of an rcu_segcblist structure by the
93
+ * specified amount, which can be negative.  This can cause the ->len
94
+ * field to disagree with the actual number of callbacks on the structure.
95
+ * This increase is fully ordered with respect to the callers accesses
96
+ * both before and after.
97
+ */
98
+static void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v)
99
+{
100
+#ifdef CONFIG_RCU_NOCB_CPU
101
+	smp_mb__before_atomic(); /* Up to the caller! */
102
+	atomic_long_add(v, &rsclp->len);
103
+	smp_mb__after_atomic(); /* Up to the caller! */
104
+#else
105
+	smp_mb(); /* Up to the caller! */
106
+	WRITE_ONCE(rsclp->len, rsclp->len + v);
107
+	smp_mb(); /* Up to the caller! */
108
+#endif
109
+}
110
+
111
+/*
112
+ * Increase the numeric length of an rcu_segcblist structure by one.
113
+ * This can cause the ->len field to disagree with the actual number of
114
+ * callbacks on the structure.  This increase is fully ordered with respect
115
+ * to the callers accesses both before and after.
116
+ */
117
+void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp)
118
+{
119
+	rcu_segcblist_add_len(rsclp, 1);
120
+}
121
+
122
+/*
123
+ * Exchange the numeric length of the specified rcu_segcblist structure
124
+ * with the specified value.  This can cause the ->len field to disagree
125
+ * with the actual number of callbacks on the structure.  This exchange is
126
+ * fully ordered with respect to the callers accesses both before and after.
127
+ */
128
+static long rcu_segcblist_xchg_len(struct rcu_segcblist *rsclp, long v)
129
+{
130
+#ifdef CONFIG_RCU_NOCB_CPU
131
+	return atomic_long_xchg(&rsclp->len, v);
132
+#else
133
+	long ret = rsclp->len;
134
+
135
+	smp_mb(); /* Up to the caller! */
136
+	WRITE_ONCE(rsclp->len, v);
137
+	smp_mb(); /* Up to the caller! */
138
+	return ret;
139
+#endif
140
+}
141
+
60142 /*
61143  * Initialize an rcu_segcblist structure.
62144  */
....@@ -69,8 +151,8 @@
69151 	rsclp->head = NULL;
70152 	for (i = 0; i < RCU_CBLIST_NSEGS; i++)
71153 		rsclp->tails[i] = &rsclp->head;
72
-	rsclp->len = 0;
73
-	rsclp->len_lazy = 0;
154
+	rcu_segcblist_set_len(rsclp, 0);
155
+	rsclp->enabled = 1;
74156 }
75157 
76158 /*
....@@ -81,8 +163,16 @@
81163 {
82164 	WARN_ON_ONCE(!rcu_segcblist_empty(rsclp));
83165 	WARN_ON_ONCE(rcu_segcblist_n_cbs(rsclp));
84
-	WARN_ON_ONCE(rcu_segcblist_n_lazy_cbs(rsclp));
85
-	rsclp->tails[RCU_NEXT_TAIL] = NULL;
166
+	rsclp->enabled = 0;
167
+}
168
+
169
+/*
170
+ * Mark the specified rcu_segcblist structure as offloaded.  This
171
+ * structure must be empty.
172
+ */
173
+void rcu_segcblist_offload(struct rcu_segcblist *rsclp)
174
+{
175
+	rsclp->offloaded = 1;
86176 }
87177 
88178 /*
....@@ -92,7 +182,7 @@
92182 bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp)
93183 {
94184 	return rcu_segcblist_is_enabled(rsclp) &&
95
-	       &rsclp->head != rsclp->tails[RCU_DONE_TAIL];
185
+	       &rsclp->head != READ_ONCE(rsclp->tails[RCU_DONE_TAIL]);
96186 }
97187 
98188 /*
....@@ -131,6 +221,18 @@
131221 }
132222 
133223 /*
224
+ * Return false if there are no CBs awaiting grace periods, otherwise,
225
+ * return true and store the nearest waited-upon grace period into *lp.
226
+ */
227
+bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp)
228
+{
229
+	if (!rcu_segcblist_pend_cbs(rsclp))
230
+		return false;
231
+	*lp = rsclp->gp_seq[RCU_WAIT_TAIL];
232
+	return true;
233
+}
234
+
235
+/*
134236  * Enqueue the specified callback onto the specified rcu_segcblist
135237  * structure, updating accounting as needed.  Note that the ->len
136238  * field may be accessed locklessly, hence the WRITE_ONCE().
....@@ -140,15 +242,13 @@
140242  * absolutely not OK for it to ever miss posting a callback.
141243  */
142244 void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
143
-			   struct rcu_head *rhp, bool lazy)
245
+			   struct rcu_head *rhp)
144246 {
145
-	WRITE_ONCE(rsclp->len, rsclp->len + 1); /* ->len sampled locklessly. */
146
-	if (lazy)
147
-		rsclp->len_lazy++;
247
+	rcu_segcblist_inc_len(rsclp);
148248 	smp_mb(); /* Ensure counts are updated before callback is enqueued. */
149249 	rhp->next = NULL;
150
-	*rsclp->tails[RCU_NEXT_TAIL] = rhp;
151
-	rsclp->tails[RCU_NEXT_TAIL] = &rhp->next;
250
+	WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rhp);
251
+	WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], &rhp->next);
152252 }
153253 
154254 /*
....@@ -162,23 +262,21 @@
162262  * period.  You have been warned.
163263  */
164264 bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
165
-			   struct rcu_head *rhp, bool lazy)
265
+			   struct rcu_head *rhp)
166266 {
167267 	int i;
168268 
169269 	if (rcu_segcblist_n_cbs(rsclp) == 0)
170270 		return false;
171
-	WRITE_ONCE(rsclp->len, rsclp->len + 1);
172
-	if (lazy)
173
-		rsclp->len_lazy++;
271
+	rcu_segcblist_inc_len(rsclp);
174272 	smp_mb(); /* Ensure counts are updated before callback is entrained. */
175273 	rhp->next = NULL;
176274 	for (i = RCU_NEXT_TAIL; i > RCU_DONE_TAIL; i--)
177275 		if (rsclp->tails[i] != rsclp->tails[i - 1])
178276 			break;
179
-	*rsclp->tails[i] = rhp;
277
+	WRITE_ONCE(*rsclp->tails[i], rhp);
180278 	for (; i <= RCU_NEXT_TAIL; i++)
181
-		rsclp->tails[i] = &rhp->next;
279
+		WRITE_ONCE(rsclp->tails[i], &rhp->next);
182280 	return true;
183281 }
184282 
....@@ -194,10 +292,7 @@
194292 void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
195293 					       struct rcu_cblist *rclp)
196294 {
197
-	rclp->len_lazy += rsclp->len_lazy;
198
-	rclp->len += rsclp->len;
199
-	rsclp->len_lazy = 0;
200
-	WRITE_ONCE(rsclp->len, 0); /* ->len sampled locklessly. */
295
+	rclp->len = rcu_segcblist_xchg_len(rsclp, 0);
201296 }
202297 
203298 /*
....@@ -213,12 +308,12 @@
213308 	if (!rcu_segcblist_ready_cbs(rsclp))
214309 		return; /* Nothing to do. */
215310 	*rclp->tail = rsclp->head;
216
-	rsclp->head = *rsclp->tails[RCU_DONE_TAIL];
217
-	*rsclp->tails[RCU_DONE_TAIL] = NULL;
311
+	WRITE_ONCE(rsclp->head, *rsclp->tails[RCU_DONE_TAIL]);
312
+	WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);
218313 	rclp->tail = rsclp->tails[RCU_DONE_TAIL];
219314 	for (i = RCU_CBLIST_NSEGS - 1; i >= RCU_DONE_TAIL; i--)
220315 		if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL])
221
-			rsclp->tails[i] = &rsclp->head;
316
+			WRITE_ONCE(rsclp->tails[i], &rsclp->head);
222317 }
223318 
224319 /*
....@@ -237,9 +332,9 @@
237332 		return; /* Nothing to do. */
238333 	*rclp->tail = *rsclp->tails[RCU_DONE_TAIL];
239334 	rclp->tail = rsclp->tails[RCU_NEXT_TAIL];
240
-	*rsclp->tails[RCU_DONE_TAIL] = NULL;
335
+	WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);
241336 	for (i = RCU_DONE_TAIL + 1; i < RCU_CBLIST_NSEGS; i++)
242
-		rsclp->tails[i] = rsclp->tails[RCU_DONE_TAIL];
337
+		WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_DONE_TAIL]);
243338 }
244339 
245340 /*
....@@ -249,10 +344,7 @@
249344 void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
250345 				struct rcu_cblist *rclp)
251346 {
252
-	rsclp->len_lazy += rclp->len_lazy;
253
-	/* ->len sampled locklessly. */
254
-	WRITE_ONCE(rsclp->len, rsclp->len + rclp->len);
255
-	rclp->len_lazy = 0;
347
+	rcu_segcblist_add_len(rsclp, rclp->len);
256348 	rclp->len = 0;
257349 }
258350 
....@@ -268,10 +360,10 @@
268360 	if (!rclp->head)
269361 		return; /* No callbacks to move. */
270362 	*rclp->tail = rsclp->head;
271
-	rsclp->head = rclp->head;
363
+	WRITE_ONCE(rsclp->head, rclp->head);
272364 	for (i = RCU_DONE_TAIL; i < RCU_CBLIST_NSEGS; i++)
273365 		if (&rsclp->head == rsclp->tails[i])
274
-			rsclp->tails[i] = rclp->tail;
366
+			WRITE_ONCE(rsclp->tails[i], rclp->tail);
275367 		else
276368 			break;
277369 	rclp->head = NULL;
....@@ -287,10 +379,8 @@
287379 {
288380 	if (!rclp->head)
289381 		return; /* Nothing to do. */
290
-	*rsclp->tails[RCU_NEXT_TAIL] = rclp->head;
291
-	rsclp->tails[RCU_NEXT_TAIL] = rclp->tail;
292
-	rclp->head = NULL;
293
-	rclp->tail = &rclp->head;
382
+	WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head);
383
+	WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail);
294384 }
295385 
296386 /*
....@@ -312,7 +402,7 @@
312402 	for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) {
313403 		if (ULONG_CMP_LT(seq, rsclp->gp_seq[i]))
314404 			break;
315
-		rsclp->tails[RCU_DONE_TAIL] = rsclp->tails[i];
405
+		WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]);
316406 	}
317407 
318408 	/* If no callbacks moved, nothing more need be done. */
....@@ -321,7 +411,7 @@
321411 
322412 	/* Clean up tail pointers that might have been misordered above. */
323413 	for (j = RCU_WAIT_TAIL; j < i; j++)
324
-		rsclp->tails[j] = rsclp->tails[RCU_DONE_TAIL];
414
+		WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]);
325415 
326416 	/*
327417 	 * Callbacks moved, so clean up the misordered ->tails[] pointers
....@@ -332,7 +422,7 @@
332422 	for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) {
333423 		if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL])
334424 			break;  /* No more callbacks. */
335
-		rsclp->tails[j] = rsclp->tails[i];
425
+		WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]);
336426 		rsclp->gp_seq[j] = rsclp->gp_seq[i];
337427 	}
338428 }
....@@ -385,8 +475,16 @@
385475 	 * Also advance to the oldest segment of callbacks whose
386476 	 * ->gp_seq[] completion is at or after that passed in via "seq",
387477 	 * skipping any empty segments.
478
+	 *
479
+	 * Note that segment "i" (and any lower-numbered segments
480
+	 * containing older callbacks) will be unaffected, and their
481
+	 * grace-period numbers remain unchanged.  For example, if i ==
482
+	 * WAIT_TAIL, then neither WAIT_TAIL nor DONE_TAIL will be touched.
483
+	 * Instead, the CBs in NEXT_TAIL will be merged with those in
484
+	 * NEXT_READY_TAIL and the grace-period number of NEXT_READY_TAIL
485
+	 * would be updated.  NEXT_TAIL would then be empty.
388486 	 */
389
-	if (++i >= RCU_NEXT_TAIL)
487
+	if (rcu_segcblist_restempty(rsclp, i) || ++i >= RCU_NEXT_TAIL)
390488 		return false;
391489 
392490 	/*
....@@ -397,7 +495,7 @@
397495 	 * structure other than in the RCU_NEXT_TAIL segment.
398496 	 */
399497 	for (; i < RCU_NEXT_TAIL; i++) {
400
-		rsclp->tails[i] = rsclp->tails[RCU_NEXT_TAIL];
498
+		WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]);
401499 		rsclp->gp_seq[i] = seq;
402500 	}
403501 	return true;