移去rt

hc

2023-12-08 01573e231f18eb2d99162747186f59511f56b64d

 kernel/kernel/stop_machine.c
..
..
@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-or-later
1
2
 /*
2
3
  * kernel/stop_machine.c
3
4
  *
..
..
@@ -5,9 +6,8 @@
5
6
  * Copyright (C) 2008, 2005	Rusty Russell rusty@rustcorp.com.au
6
7
  * Copyright (C) 2010		SUSE Linux Products GmbH
7
8
  * Copyright (C) 2010		Tejun Heo <tj@kernel.org>
8
- *
9
- * This file is released under the GPLv2 and any later version.
10
9
  */
10
+#include <linux/compiler.h>
11
11
 #include <linux/completion.h>
12
12
 #include <linux/cpu.h>
13
13
 #include <linux/init.h>
..
..
@@ -22,16 +22,7 @@
22
22
 #include <linux/atomic.h>
23
23
 #include <linux/nmi.h>
24
24
 #include <linux/sched/wake_q.h>
25
-
26
-/*
27
- * Structure to determine completion condition and record errors.  May
28
- * be shared by works on different cpus.
29
- */
30
-struct cpu_stop_done {
31
-	atomic_t		nr_todo;	/* nr left to execute */
32
-	int			ret;		/* collected return value */
33
-	struct completion	completion;	/* fired if nr_todo reaches 0 */
34
-};
25
+#include <linux/slab.h>
35
26
 
36
27
 /* the actual stopper, one per every possible cpu, enabled on online cpus */
37
28
 struct cpu_stopper {
..
..
@@ -42,10 +33,26 @@
42
33
 	struct list_head	works;		/* list of pending works */
43
34
 
44
35
 	struct cpu_stop_work	stop_work;	/* for stop_cpus */
36
+	unsigned long		caller;
37
+	cpu_stop_fn_t		fn;
45
38
 };
46
39
 
47
40
 static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
48
41
 static bool stop_machine_initialized = false;
42
+
43
+void print_stop_info(const char *log_lvl, struct task_struct *task)
44
+{
45
+	/*
46
+	 * If @task is a stopper task, it cannot migrate and task_cpu() is
47
+	 * stable.
48
+	 */
49
+	struct cpu_stopper *stopper = per_cpu_ptr(&cpu_stopper, task_cpu(task));
50
+
51
+	if (task != stopper->thread)
52
+		return;
53
+
54
+	printk("%sStopper: %pS <- %pS\n", log_lvl, stopper->fn, (void *)stopper->caller);
55
+}
49
56
 
50
57
 /* static data for stop_cpus */
51
58
 static DEFINE_MUTEX(stop_cpus_mutex);
..
..
@@ -123,7 +130,7 @@
123
130
 int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
124
131
 {
125
132
 	struct cpu_stop_done done;
126
-	struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done };
133
+	struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done, .caller = _RET_IP_ };
127
134
 
128
135
 	cpu_stop_init_done(&done, 1);
129
136
 	if (!cpu_stop_queue_work(cpu, &work))
..
..
@@ -168,7 +175,7 @@
168
175
 	/* Reset ack counter. */
169
176
 	atomic_set(&msdata->thread_ack, msdata->num_threads);
170
177
 	smp_wmb();
171
-	msdata->state = newstate;
178
+	WRITE_ONCE(msdata->state, newstate);
172
179
 }
173
180
 
174
181
 /* Last one to ack a state moves to the next state. */
..
..
@@ -178,12 +185,18 @@
178
185
 		set_state(msdata, msdata->state + 1);
179
186
 }
180
187
 
188
+notrace void __weak stop_machine_yield(const struct cpumask *cpumask)
189
+{
190
+	cpu_relax();
191
+}
192
+
181
193
 /* This is the cpu_stop function which stops the CPU. */
182
194
 static int multi_cpu_stop(void *data)
183
195
 {
184
196
 	struct multi_stop_data *msdata = data;
185
-	enum multi_stop_state curstate = MULTI_STOP_NONE;
197
+	enum multi_stop_state newstate, curstate = MULTI_STOP_NONE;
186
198
 	int cpu = smp_processor_id(), err = 0;
199
+	const struct cpumask *cpumask;
187
200
 	unsigned long flags;
188
201
 	bool is_active;
189
202
 
..
..
@@ -193,17 +206,21 @@
193
206
 	 */
194
207
 	local_save_flags(flags);
195
208
 
196
-	if (!msdata->active_cpus)
197
-		is_active = cpu == cpumask_first(cpu_online_mask);
198
-	else
199
-		is_active = cpumask_test_cpu(cpu, msdata->active_cpus);
209
+	if (!msdata->active_cpus) {
210
+		cpumask = cpu_online_mask;
211
+		is_active = cpu == cpumask_first(cpumask);
212
+	} else {
213
+		cpumask = msdata->active_cpus;
214
+		is_active = cpumask_test_cpu(cpu, cpumask);
215
+	}
200
216
 
201
217
 	/* Simple state machine */
202
218
 	do {
203
219
 		/* Chill out and ensure we re-read multi_stop_state. */
204
-		cpu_relax_yield();
205
-		if (msdata->state != curstate) {
206
-			curstate = msdata->state;
220
+		stop_machine_yield(cpumask);
221
+		newstate = READ_ONCE(msdata->state);
222
+		if (newstate != curstate) {
223
+			curstate = newstate;
207
224
 			switch (curstate) {
208
225
 			case MULTI_STOP_DISABLE_IRQ:
209
226
 				local_irq_disable();
..
..
@@ -225,6 +242,7 @@
225
242
 			 */
226
243
 			touch_nmi_watchdog();
227
244
 		}
245
+		rcu_momentary_dyntick_idle();
228
246
 	} while (curstate != MULTI_STOP_EXIT);
229
247
 
230
248
 	local_irq_restore(flags);
..
..
@@ -320,7 +338,8 @@
320
338
 	work1 = work2 = (struct cpu_stop_work){
321
339
 		.fn = multi_cpu_stop,
322
340
 		.arg = &msdata,
323
-		.done = &done
341
+		.done = &done,
342
+		.caller = _RET_IP_,
324
343
 	};
325
344
 
326
345
 	cpu_stop_init_done(&done, 2);
..
..
@@ -356,8 +375,57 @@
356
375
 bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
357
376
 			struct cpu_stop_work *work_buf)
358
377
 {
359
-	*work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, };
378
+	*work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, .caller = _RET_IP_, };
360
379
 	return cpu_stop_queue_work(cpu, work_buf);
380
+}
381
+EXPORT_SYMBOL_GPL(stop_one_cpu_nowait);
382
+
383
+/**
384
+ * stop_one_cpu_async - stop a cpu and wait for completion in a separated
385
+ *			function: stop_wait_work()
386
+ * @cpu: cpu to stop
387
+ * @fn: function to execute
388
+ * @arg: argument to @fn
389
+ * @work_buf: pointer to cpu_stop_work structure
390
+ *
391
+ * CONTEXT:
392
+ * Might sleep.
393
+ *
394
+ * RETURNS:
395
+ * 0 if cpu_stop_work was queued successfully and @fn will be called.
396
+ * ENOENT if @fn(@arg) was not executed because @cpu was offline.
397
+ */
398
+int stop_one_cpu_async(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
399
+		       struct cpu_stop_work *work_buf,
400
+		       struct cpu_stop_done *done)
401
+{
402
+	cpu_stop_init_done(done, 1);
403
+
404
+	work_buf->done = done;
405
+	work_buf->fn = fn;
406
+	work_buf->arg = arg;
407
+
408
+	if (cpu_stop_queue_work(cpu, work_buf))
409
+		return 0;
410
+
411
+	work_buf->done = NULL;
412
+
413
+	return -ENOENT;
414
+}
415
+
416
+/**
417
+ * cpu_stop_work_wait - wait for a stop initiated by stop_one_cpu_async().
418
+ * @work_buf: pointer to cpu_stop_work structure
419
+ *
420
+ * CONTEXT:
421
+ * Might sleep.
422
+ */
423
+void cpu_stop_work_wait(struct cpu_stop_work *work_buf)
424
+{
425
+	struct cpu_stop_done *done = work_buf->done;
426
+
427
+	wait_for_completion(&done->completion);
428
+	work_buf->done = NULL;
361
429
 }
362
430
 
363
431
 static bool queue_stop_cpus_work(const struct cpumask *cpumask,
..
..
@@ -375,6 +443,7 @@
375
443
 	 */
376
444
 	preempt_disable();
377
445
 	stop_cpus_in_progress = true;
446
+	barrier();
378
447
 	for_each_cpu(cpu, cpumask) {
379
448
 		work = &per_cpu(cpu_stopper.stop_work, cpu);
380
449
 		work->fn = fn;
..
..
@@ -383,6 +452,7 @@
383
452
 		if (cpu_stop_queue_work(cpu, work))
384
453
 			queued = true;
385
454
 	}
455
+	barrier();
386
456
 	stop_cpus_in_progress = false;
387
457
 	preempt_enable();
388
458
 
..
..
@@ -429,42 +499,12 @@
429
499
  * @cpumask were offline; otherwise, 0 if all executions of @fn
430
500
  * returned 0, any non zero return value if any returned non zero.
431
501
  */
432
-int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
502
+static int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
433
503
 {
434
504
 	int ret;
435
505
 
436
506
 	/* static works are used, process one request at a time */
437
507
 	mutex_lock(&stop_cpus_mutex);
438
-	ret = __stop_cpus(cpumask, fn, arg);
439
-	mutex_unlock(&stop_cpus_mutex);
440
-	return ret;
441
-}
442
-
443
-/**
444
- * try_stop_cpus - try to stop multiple cpus
445
- * @cpumask: cpus to stop
446
- * @fn: function to execute
447
- * @arg: argument to @fn
448
- *
449
- * Identical to stop_cpus() except that it fails with -EAGAIN if
450
- * someone else is already using the facility.
451
- *
452
- * CONTEXT:
453
- * Might sleep.
454
- *
455
- * RETURNS:
456
- * -EAGAIN if someone else is already stopping cpus, -ENOENT if
457
- * @fn(@arg) was not executed at all because all cpus in @cpumask were
458
- * offline; otherwise, 0 if all executions of @fn returned 0, any non
459
- * zero return value if any returned non zero.
460
- */
461
-int try_stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
462
-{
463
-	int ret;
464
-
465
-	/* static works are used, process one request at a time */
466
-	if (!mutex_trylock(&stop_cpus_mutex))
467
-		return -EAGAIN;
468
508
 	ret = __stop_cpus(cpumask, fn, arg);
469
509
 	mutex_unlock(&stop_cpus_mutex);
470
510
 	return ret;
..
..
@@ -504,6 +544,8 @@
504
544
 		int ret;
505
545
 
506
546
 		/* cpu stop callbacks must not sleep, make in_atomic() == T */
547
+		stopper->caller = work->caller;
548
+		stopper->fn = fn;
507
549
 		preempt_count_inc();
508
550
 		ret = fn(arg);
509
551
 		if (done) {
..
..
@@ -512,8 +554,10 @@
512
554
 			cpu_stop_signal_done(done);
513
555
 		}
514
556
 		preempt_count_dec();
557
+		stopper->fn = NULL;
558
+		stopper->caller = 0;
515
559
 		WARN_ONCE(preempt_count(),
516
-			  "cpu_stop: %pf(%p) leaked preempt count\n", fn, arg);
560
+			  "cpu_stop: %ps(%p) leaked preempt count\n", fn, arg);
517
561
 		goto repeat;
518
562
 	}
519
563
 }