forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-10 37f49e37ab4cb5d0bc4c60eb5c6d4dd57db767bb 
 kernel/lib/percpu-refcount.c
....@@ -1,8 +1,10 @@
1
-#define pr_fmt(fmt) "%s: " fmt "\n", __func__
1
+// SPDX-License-Identifier: GPL-2.0-only
2
+#define pr_fmt(fmt) "%s: " fmt, __func__
23 
34 #include <linux/kernel.h>
45 #include <linux/sched.h>
56 #include <linux/wait.h>
7
+#include <linux/slab.h>
68 #include <linux/percpu-refcount.h>
79 
810 /*
....@@ -49,9 +51,10 @@
4951  * @flags: PERCPU_REF_INIT_* flags
5052  * @gfp: allocation mask to use
5153  *
52
- * Initializes @ref.  If @flags is zero, @ref starts in percpu mode with a
53
- * refcount of 1; analagous to atomic_long_set(ref, 1).  See the
54
- * definitions of PERCPU_REF_INIT_* flags for flag behaviors.
54
+ * Initializes @ref.  @ref starts out in percpu mode with a refcount of 1 unless
55
+ * @flags contains PERCPU_REF_INIT_ATOMIC or PERCPU_REF_INIT_DEAD.  These flags
56
+ * change the start state to atomic with the latter setting the initial refcount
57
+ * to 0.  See the definitions of PERCPU_REF_INIT_* flags for flag behaviors.
5558  *
5659  * Note that @release must not sleep - it may potentially be called from RCU
5760  * callback context by percpu_ref_kill().
....@@ -62,31 +65,56 @@
6265 	size_t align = max_t(size_t, 1 << __PERCPU_REF_FLAG_BITS,
6366 			     __alignof__(unsigned long));
6467 	unsigned long start_count = 0;
68
+	struct percpu_ref_data *data;
6569 
6670 	ref->percpu_count_ptr = (unsigned long)
6771 		__alloc_percpu_gfp(sizeof(unsigned long), align, gfp);
6872 	if (!ref->percpu_count_ptr)
6973 		return -ENOMEM;
7074 
71
-	ref->force_atomic = flags & PERCPU_REF_INIT_ATOMIC;
75
+	data = kzalloc(sizeof(*ref->data), gfp);
76
+	if (!data) {
77
+		free_percpu((void __percpu *)ref->percpu_count_ptr);
78
+		ref->percpu_count_ptr = 0;
79
+		return -ENOMEM;
80
+	}
7281 
73
-	if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD))
82
+	data->force_atomic = flags & PERCPU_REF_INIT_ATOMIC;
83
+	data->allow_reinit = flags & PERCPU_REF_ALLOW_REINIT;
84
+
85
+	if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD)) {
7486 		ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC;
75
-	else
87
+		data->allow_reinit = true;
88
+	} else {
7689 		start_count += PERCPU_COUNT_BIAS;
90
+	}
7791 
7892 	if (flags & PERCPU_REF_INIT_DEAD)
7993 		ref->percpu_count_ptr |= __PERCPU_REF_DEAD;
8094 	else
8195 		start_count++;
8296 
83
-	atomic_long_set(&ref->count, start_count);
97
+	atomic_long_set(&data->count, start_count);
8498 
85
-	ref->release = release;
86
-	ref->confirm_switch = NULL;
99
+	data->release = release;
100
+	data->confirm_switch = NULL;
101
+	data->ref = ref;
102
+	ref->data = data;
87103 	return 0;
88104 }
89105 EXPORT_SYMBOL_GPL(percpu_ref_init);
106
+
107
+static void __percpu_ref_exit(struct percpu_ref *ref)
108
+{
109
+	unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
110
+
111
+	if (percpu_count) {
112
+		/* non-NULL confirm_switch indicates switching in progress */
113
+		WARN_ON_ONCE(ref->data && ref->data->confirm_switch);
114
+		free_percpu(percpu_count);
115
+		ref->percpu_count_ptr = __PERCPU_REF_ATOMIC_DEAD;
116
+	}
117
+}
90118 
91119 /**
92120  * percpu_ref_exit - undo percpu_ref_init()
....@@ -100,24 +128,36 @@
100128  */
101129 void percpu_ref_exit(struct percpu_ref *ref)
102130 {
103
-	unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
131
+	struct percpu_ref_data *data = ref->data;
132
+	unsigned long flags;
104133 
105
-	if (percpu_count) {
106
-		/* non-NULL confirm_switch indicates switching in progress */
107
-		WARN_ON_ONCE(ref->confirm_switch);
108
-		free_percpu(percpu_count);
109
-		ref->percpu_count_ptr = __PERCPU_REF_ATOMIC_DEAD;
110
-	}
134
+	__percpu_ref_exit(ref);
135
+
136
+	if (!data)
137
+		return;
138
+
139
+	spin_lock_irqsave(&percpu_ref_switch_lock, flags);
140
+	ref->percpu_count_ptr |= atomic_long_read(&ref->data->count) <<
141
+		__PERCPU_REF_FLAG_BITS;
142
+	ref->data = NULL;
143
+	spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
144
+
145
+	kfree(data);
111146 }
112147 EXPORT_SYMBOL_GPL(percpu_ref_exit);
113148 
114149 static void percpu_ref_call_confirm_rcu(struct rcu_head *rcu)
115150 {
116
-	struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu);
151
+	struct percpu_ref_data *data = container_of(rcu,
152
+			struct percpu_ref_data, rcu);
153
+	struct percpu_ref *ref = data->ref;
117154 
118
-	ref->confirm_switch(ref);
119
-	ref->confirm_switch = NULL;
155
+	data->confirm_switch(ref);
156
+	data->confirm_switch = NULL;
120157 	wake_up_all(&percpu_ref_switch_waitq);
158
+
159
+	if (!data->allow_reinit)
160
+		__percpu_ref_exit(ref);
121161 
122162 	/* drop ref from percpu_ref_switch_to_atomic() */
123163 	percpu_ref_put(ref);
....@@ -125,7 +165,9 @@
125165 
126166 static void percpu_ref_switch_to_atomic_rcu(struct rcu_head *rcu)
127167 {
128
-	struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu);
168
+	struct percpu_ref_data *data = container_of(rcu,
169
+			struct percpu_ref_data, rcu);
170
+	struct percpu_ref *ref = data->ref;
129171 	unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
130172 	unsigned long count = 0;
131173 	int cpu;
....@@ -133,8 +175,8 @@
133175 	for_each_possible_cpu(cpu)
134176 		count += *per_cpu_ptr(percpu_count, cpu);
135177 
136
-	pr_debug("global %ld percpu %ld",
137
-		 atomic_long_read(&ref->count), (long)count);
178
+	pr_debug("global %lu percpu %lu\n",
179
+		 atomic_long_read(&data->count), count);
138180 
139181 	/*
140182 	 * It's crucial that we sum the percpu counters _before_ adding the sum
....@@ -148,11 +190,11 @@
148190 	 * reaching 0 before we add the percpu counts. But doing it at the same
149191 	 * time is equivalent and saves us atomic operations:
150192 	 */
151
-	atomic_long_add((long)count - PERCPU_COUNT_BIAS, &ref->count);
193
+	atomic_long_add((long)count - PERCPU_COUNT_BIAS, &data->count);
152194 
153
-	WARN_ONCE(atomic_long_read(&ref->count) <= 0,
154
-		  "percpu ref (%pf) <= 0 (%ld) after switching to atomic",
155
-		  ref->release, atomic_long_read(&ref->count));
195
+	WARN_ONCE(atomic_long_read(&data->count) <= 0,
196
+		  "percpu ref (%ps) <= 0 (%ld) after switching to atomic",
197
+		  data->release, atomic_long_read(&data->count));
156198 
157199 	/* @ref is viewed as dead on all CPUs, send out switch confirmation */
158200 	percpu_ref_call_confirm_rcu(rcu);
....@@ -178,10 +220,11 @@
178220 	 * Non-NULL ->confirm_switch is used to indicate that switching is
179221 	 * in progress.  Use noop one if unspecified.
180222 	 */
181
-	ref->confirm_switch = confirm_switch ?: percpu_ref_noop_confirm_switch;
223
+	ref->data->confirm_switch = confirm_switch ?:
224
+		percpu_ref_noop_confirm_switch;
182225 
183226 	percpu_ref_get(ref);	/* put after confirmation */
184
-	call_rcu_sched(&ref->rcu, percpu_ref_switch_to_atomic_rcu);
227
+	call_rcu(&ref->data->rcu, percpu_ref_switch_to_atomic_rcu);
185228 }
186229 
187230 static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref)
....@@ -194,7 +237,10 @@
194237 	if (!(ref->percpu_count_ptr & __PERCPU_REF_ATOMIC))
195238 		return;
196239 
197
-	atomic_long_add(PERCPU_COUNT_BIAS, &ref->count);
240
+	if (WARN_ON_ONCE(!ref->data->allow_reinit))
241
+		return;
242
+
243
+	atomic_long_add(PERCPU_COUNT_BIAS, &ref->data->count);
198244 
199245 	/*
200246 	 * Restore per-cpu operation.  smp_store_release() is paired
....@@ -212,6 +258,8 @@
212258 static void __percpu_ref_switch_mode(struct percpu_ref *ref,
213259 				     percpu_ref_func_t *confirm_switch)
214260 {
261
+	struct percpu_ref_data *data = ref->data;
262
+
215263 	lockdep_assert_held(&percpu_ref_switch_lock);
216264 
217265 	/*
....@@ -219,10 +267,10 @@
219267 	 * its completion.  If the caller ensures that ATOMIC switching
220268 	 * isn't in progress, this function can be called from any context.
221269 	 */
222
-	wait_event_lock_irq(percpu_ref_switch_waitq, !ref->confirm_switch,
270
+	wait_event_lock_irq(percpu_ref_switch_waitq, !data->confirm_switch,
223271 			    percpu_ref_switch_lock);
224272 
225
-	if (ref->force_atomic || (ref->percpu_count_ptr & __PERCPU_REF_DEAD))
273
+	if (data->force_atomic || (ref->percpu_count_ptr & __PERCPU_REF_DEAD))
226274 		__percpu_ref_switch_to_atomic(ref, confirm_switch);
227275 	else
228276 		__percpu_ref_switch_to_percpu(ref);
....@@ -255,7 +303,7 @@
255303 
256304 	spin_lock_irqsave(&percpu_ref_switch_lock, flags);
257305 
258
-	ref->force_atomic = true;
306
+	ref->data->force_atomic = true;
259307 	__percpu_ref_switch_mode(ref, confirm_switch);
260308 
261309 	spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
....@@ -273,7 +321,7 @@
273321 void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref)
274322 {
275323 	percpu_ref_switch_to_atomic(ref, NULL);
276
-	wait_event(percpu_ref_switch_waitq, !ref->confirm_switch);
324
+	wait_event(percpu_ref_switch_waitq, !ref->data->confirm_switch);
277325 }
278326 EXPORT_SYMBOL_GPL(percpu_ref_switch_to_atomic_sync);
279327 
....@@ -301,7 +349,7 @@
301349 
302350 	spin_lock_irqsave(&percpu_ref_switch_lock, flags);
303351 
304
-	ref->force_atomic = false;
352
+	ref->data->force_atomic = false;
305353 	__percpu_ref_switch_mode(ref, NULL);
306354 
307355 	spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
....@@ -333,7 +381,8 @@
333381 	spin_lock_irqsave(&percpu_ref_switch_lock, flags);
334382 
335383 	WARN_ONCE(ref->percpu_count_ptr & __PERCPU_REF_DEAD,
336
-		  "%s called more than once on %pf!", __func__, ref->release);
384
+		  "%s called more than once on %ps!", __func__,
385
+		  ref->data->release);
337386 
338387 	ref->percpu_count_ptr |= __PERCPU_REF_DEAD;
339388 	__percpu_ref_switch_mode(ref, confirm_kill);
....@@ -342,6 +391,34 @@
342391 	spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
343392 }
344393 EXPORT_SYMBOL_GPL(percpu_ref_kill_and_confirm);
394
+
395
+/**
396
+ * percpu_ref_is_zero - test whether a percpu refcount reached zero
397
+ * @ref: percpu_ref to test
398
+ *
399
+ * Returns %true if @ref reached zero.
400
+ *
401
+ * This function is safe to call as long as @ref is between init and exit.
402
+ */
403
+bool percpu_ref_is_zero(struct percpu_ref *ref)
404
+{
405
+	unsigned long __percpu *percpu_count;
406
+	unsigned long count, flags;
407
+
408
+	if (__ref_is_percpu(ref, &percpu_count))
409
+		return false;
410
+
411
+	/* protect us from being destroyed */
412
+	spin_lock_irqsave(&percpu_ref_switch_lock, flags);
413
+	if (ref->data)
414
+		count = atomic_long_read(&ref->data->count);
415
+	else
416
+		count = ref->percpu_count_ptr >> __PERCPU_REF_FLAG_BITS;
417
+	spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
418
+
419
+	return count == 0;
420
+}
421
+EXPORT_SYMBOL_GPL(percpu_ref_is_zero);
345422 
346423 /**
347424  * percpu_ref_reinit - re-initialize a percpu refcount
....@@ -356,11 +433,35 @@
356433  */
357434 void percpu_ref_reinit(struct percpu_ref *ref)
358435 {
436
+	WARN_ON_ONCE(!percpu_ref_is_zero(ref));
437
+
438
+	percpu_ref_resurrect(ref);
439
+}
440
+EXPORT_SYMBOL_GPL(percpu_ref_reinit);
441
+
442
+/**
443
+ * percpu_ref_resurrect - modify a percpu refcount from dead to live
444
+ * @ref: perpcu_ref to resurrect
445
+ *
446
+ * Modify @ref so that it's in the same state as before percpu_ref_kill() was
447
+ * called. @ref must be dead but must not yet have exited.
448
+ *
449
+ * If @ref->release() frees @ref then the caller is responsible for
450
+ * guaranteeing that @ref->release() does not get called while this
451
+ * function is in progress.
452
+ *
453
+ * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while
454
+ * this function is in progress.
455
+ */
456
+void percpu_ref_resurrect(struct percpu_ref *ref)
457
+{
458
+	unsigned long __percpu *percpu_count;
359459 	unsigned long flags;
360460 
361461 	spin_lock_irqsave(&percpu_ref_switch_lock, flags);
362462 
363
-	WARN_ON_ONCE(!percpu_ref_is_zero(ref));
463
+	WARN_ON_ONCE(!(ref->percpu_count_ptr & __PERCPU_REF_DEAD));
464
+	WARN_ON_ONCE(__ref_is_percpu(ref, &percpu_count));
364465 
365466 	ref->percpu_count_ptr &= ~__PERCPU_REF_DEAD;
366467 	percpu_ref_get(ref);
....@@ -368,4 +469,4 @@
368469 
369470 	spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
370471 }
371
-EXPORT_SYMBOL_GPL(percpu_ref_reinit);
472
+EXPORT_SYMBOL_GPL(percpu_ref_resurrect);