kernel_5.10 no rt

hc

2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f

 kernel/kernel/rcu/rcu_segcblist.h
..
..
@@ -1,37 +1,25 @@
1
+/* SPDX-License-Identifier: GPL-2.0+ */
1
2
 /*
2
3
  * RCU segmented callback lists, internal-to-rcu header file
3
4
  *
4
- * This program is free software; you can redistribute it and/or modify
5
- * it under the terms of the GNU General Public License as published by
6
- * the Free Software Foundation; either version 2 of the License, or
7
- * (at your option) any later version.
8
- *
9
- * This program is distributed in the hope that it will be useful,
10
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
- * GNU General Public License for more details.
13
- *
14
- * You should have received a copy of the GNU General Public License
15
- * along with this program; if not, you can access it online at
16
- * http://www.gnu.org/licenses/gpl-2.0.html.
17
- *
18
5
  * Copyright IBM Corporation, 2017
19
6
  *
20
- * Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
7
+ * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
21
8
  */
22
9
 
23
10
 #include <linux/rcu_segcblist.h>
24
11
 
25
-/*
26
- * Account for the fact that a previously dequeued callback turned out
27
- * to be marked as lazy.
28
- */
29
-static inline void rcu_cblist_dequeued_lazy(struct rcu_cblist *rclp)
12
+/* Return number of callbacks in the specified callback list. */
13
+static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)
30
14
 {
31
-	rclp->len_lazy--;
15
+	return READ_ONCE(rclp->len);
32
16
 }
33
17
 
34
18
 void rcu_cblist_init(struct rcu_cblist *rclp);
19
+void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp);
20
+void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
21
+			      struct rcu_cblist *srclp,
22
+			      struct rcu_head *rhp);
35
23
 struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp);
36
24
 
37
25
 /*
..
..
@@ -49,34 +37,32 @@
49
37
  */
50
38
 static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
51
39
 {
52
-	return !rsclp->head;
40
+	return !READ_ONCE(rsclp->head);
53
41
 }
54
42
 
55
43
 /* Return number of callbacks in segmented callback list. */
56
44
 static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
57
45
 {
46
+#ifdef CONFIG_RCU_NOCB_CPU
47
+	return atomic_long_read(&rsclp->len);
48
+#else
58
49
 	return READ_ONCE(rsclp->len);
59
-}
60
-
61
-/* Return number of lazy callbacks in segmented callback list. */
62
-static inline long rcu_segcblist_n_lazy_cbs(struct rcu_segcblist *rsclp)
63
-{
64
-	return rsclp->len_lazy;
65
-}
66
-
67
-/* Return number of lazy callbacks in segmented callback list. */
68
-static inline long rcu_segcblist_n_nonlazy_cbs(struct rcu_segcblist *rsclp)
69
-{
70
-	return rsclp->len - rsclp->len_lazy;
50
+#endif
71
51
 }
72
52
 
73
53
 /*
74
54
  * Is the specified rcu_segcblist enabled, for example, not corresponding
75
- * to an offline or callback-offloaded CPU?
55
+ * to an offline CPU?
76
56
  */
77
57
 static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
78
58
 {
79
-	return !!rsclp->tails[RCU_NEXT_TAIL];
59
+	return rsclp->enabled;
60
+}
61
+
62
+/* Is the specified rcu_segcblist offloaded?  */
63
+static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)
64
+{
65
+	return rsclp->offloaded;
80
66
 }
81
67
 
82
68
 /*
..
..
@@ -86,40 +72,22 @@
86
72
  */
87
73
 static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
88
74
 {
89
-	return !*rsclp->tails[seg];
75
+	return !READ_ONCE(*READ_ONCE(rsclp->tails[seg]));
90
76
 }
91
77
 
92
-/*
93
- * Interim function to return rcu_segcblist head pointer.  Longer term, the
94
- * rcu_segcblist will be used more pervasively, removing the need for this
95
- * function.
96
- */
97
-static inline struct rcu_head *rcu_segcblist_head(struct rcu_segcblist *rsclp)
98
-{
99
-	return rsclp->head;
100
-}
101
-
102
-/*
103
- * Interim function to return rcu_segcblist head pointer.  Longer term, the
104
- * rcu_segcblist will be used more pervasively, removing the need for this
105
- * function.
106
- */
107
-static inline struct rcu_head **rcu_segcblist_tail(struct rcu_segcblist *rsclp)
108
-{
109
-	WARN_ON_ONCE(rcu_segcblist_empty(rsclp));
110
-	return rsclp->tails[RCU_NEXT_TAIL];
111
-}
112
-
78
+void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp);
113
79
 void rcu_segcblist_init(struct rcu_segcblist *rsclp);
114
80
 void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
81
+void rcu_segcblist_offload(struct rcu_segcblist *rsclp);
115
82
 bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);
116
83
 bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp);
117
84
 struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp);
118
85
 struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp);
86
+bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp);
119
87
 void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
120
-			   struct rcu_head *rhp, bool lazy);
88
+			   struct rcu_head *rhp);
121
89
 bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
122
-			   struct rcu_head *rhp, bool lazy);
90
+			   struct rcu_head *rhp);
123
91
 void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
124
92
 				 struct rcu_cblist *rclp);
125
93
 void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,