forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/kernel/kthread.c
....@@ -1,12 +1,17 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /* Kernel thread helper functions.
23  *   Copyright (C) 2004 IBM Corporation, Rusty Russell.
4
+ *   Copyright (C) 2009 Red Hat, Inc.
35  *
46  * Creation is done via kthreadd, so that we get a clean environment
57  * even if we're invoked from userspace (think modprobe, hotplug cpu,
68  * etc.).
79  */
810 #include <uapi/linux/sched/types.h>
11
+#include <linux/mm.h>
12
+#include <linux/mmu_context.h>
913 #include <linux/sched.h>
14
+#include <linux/sched/mm.h>
1015 #include <linux/sched/task.h>
1116 #include <linux/kthread.h>
1217 #include <linux/completion.h>
....@@ -21,7 +26,10 @@
2126 #include <linux/freezer.h>
2227 #include <linux/ptrace.h>
2328 #include <linux/uaccess.h>
29
+#include <linux/numa.h>
30
+#include <linux/sched/isolation.h>
2431 #include <trace/events/sched.h>
32
+
2533 
2634 static DEFINE_SPINLOCK(kthread_create_lock);
2735 static LIST_HEAD(kthread_create_list);
....@@ -44,7 +52,9 @@
4452 struct kthread {
4553 	unsigned long flags;
4654 	unsigned int cpu;
55
+	int (*threadfn)(void *);
4756 	void *data;
57
+	mm_segment_t oldfs;
4858 	struct completion parked;
4959 	struct completion exited;
5060 #ifdef CONFIG_BLK_CGROUP
....@@ -72,6 +82,25 @@
7282 {
7383 	WARN_ON(!(k->flags & PF_KTHREAD));
7484 	return (__force void *)k->set_child_tid;
85
+}
86
+
87
+/*
88
+ * Variant of to_kthread() that doesn't assume @p is a kthread.
89
+ *
90
+ * Per construction; when:
91
+ *
92
+ *   (p->flags & PF_KTHREAD) && p->set_child_tid
93
+ *
94
+ * the task is both a kthread and struct kthread is persistent. However
95
+ * PF_KTHREAD on it's own is not, kernel_thread() can exec() (See umh.c and
96
+ * begin_new_exec()).
97
+ */
98
+static inline struct kthread *__to_kthread(struct task_struct *p)
99
+{
100
+	void *kthread = (__force void *)p->set_child_tid;
101
+	if (kthread && !(p->flags & PF_KTHREAD))
102
+		kthread = NULL;
103
+	return kthread;
75104 }
76105 
77106 void free_kthread_struct(struct task_struct *k)
....@@ -102,6 +131,12 @@
102131 }
103132 EXPORT_SYMBOL(kthread_should_stop);
104133 
134
+bool __kthread_should_park(struct task_struct *k)
135
+{
136
+	return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(k)->flags);
137
+}
138
+EXPORT_SYMBOL_GPL(__kthread_should_park);
139
+
105140 /**
106141  * kthread_should_park - should this kthread park now?
107142  *
....@@ -115,7 +150,7 @@
115150  */
116151 bool kthread_should_park(void)
117152 {
118
-	return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(current)->flags);
153
+	return __kthread_should_park(current);
119154 }
120155 EXPORT_SYMBOL_GPL(kthread_should_park);
121156 
....@@ -145,6 +180,21 @@
145180 EXPORT_SYMBOL_GPL(kthread_freezable_should_stop);
146181 
147182 /**
183
+ * kthread_func - return the function specified on kthread creation
184
+ * @task: kthread task in question
185
+ *
186
+ * Returns NULL if the task is not a kthread.
187
+ */
188
+void *kthread_func(struct task_struct *task)
189
+{
190
+	struct kthread *kthread = __to_kthread(task);
191
+	if (kthread)
192
+		return kthread->threadfn;
193
+	return NULL;
194
+}
195
+EXPORT_SYMBOL_GPL(kthread_func);
196
+
197
+/**
148198  * kthread_data - return data value specified on kthread creation
149199  * @task: kthread task in question
150200  *
....@@ -156,6 +206,7 @@
156206 {
157207 	return to_kthread(task)->data;
158208 }
209
+EXPORT_SYMBOL_GPL(kthread_data);
159210 
160211 /**
161212  * kthread_probe_data - speculative version of kthread_data()
....@@ -168,10 +219,11 @@
168219  */
169220 void *kthread_probe_data(struct task_struct *task)
170221 {
171
-	struct kthread *kthread = to_kthread(task);
222
+	struct kthread *kthread = __to_kthread(task);
172223 	void *data = NULL;
173224 
174
-	probe_kernel_read(&data, &kthread->data, sizeof(data));
225
+	if (kthread)
226
+		copy_from_kernel_nofault(&data, &kthread->data, sizeof(data));
175227 	return data;
176228 }
177229 
....@@ -236,6 +288,7 @@
236288 		do_exit(-ENOMEM);
237289 	}
238290 
291
+	self->threadfn = threadfn;
239292 	self->data = data;
240293 	init_completion(&self->exited);
241294 	init_completion(&self->parked);
....@@ -352,7 +405,8 @@
352405 		 * The kernel thread should not inherit these properties.
353406 		 */
354407 		sched_setscheduler_nocheck(task, SCHED_NORMAL, &param);
355
-		set_cpus_allowed_ptr(task, cpu_all_mask);
408
+		set_cpus_allowed_ptr(task,
409
+				     housekeeping_cpumask(HK_FLAG_KTHREAD));
356410 	}
357411 	kfree(create);
358412 	return task;
....@@ -422,6 +476,7 @@
422476 {
423477 	__kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE);
424478 }
479
+EXPORT_SYMBOL_GPL(kthread_bind_mask);
425480 
426481 /**
427482  * kthread_bind - bind a just-created kthread to a cpu.
....@@ -447,7 +502,6 @@
447502  *	     to "name.*%u". Code fills in cpu number.
448503  *
449504  * Description: This helper function creates and names a kernel thread
450
- * The thread will be woken and put into park mode.
451505  */
452506 struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data),
453507 					  void *data, unsigned int cpu,
....@@ -482,9 +536,9 @@
482536 	set_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
483537 }
484538 
485
-bool kthread_is_per_cpu(struct task_struct *k)
539
+bool kthread_is_per_cpu(struct task_struct *p)
486540 {
487
-	struct kthread *kthread = to_kthread(k);
541
+	struct kthread *kthread = __to_kthread(p);
488542 	if (!kthread)
489543 		return false;
490544 
....@@ -602,7 +656,7 @@
602656 	/* Setup a clean context for our children to inherit. */
603657 	set_task_comm(tsk, "kthreadd");
604658 	ignore_signals(tsk);
605
-	set_cpus_allowed_ptr(tsk, cpu_all_mask);
659
+	set_cpus_allowed_ptr(tsk, housekeeping_cpumask(HK_FLAG_KTHREAD));
606660 	set_mems_allowed(node_states[N_MEMORY]);
607661 
608662 	current->flags |= PF_NOFREEZE;
....@@ -638,7 +692,7 @@
638692 				struct lock_class_key *key)
639693 {
640694 	memset(worker, 0, sizeof(struct kthread_worker));
641
-	spin_lock_init(&worker->lock);
695
+	raw_spin_lock_init(&worker->lock);
642696 	lockdep_set_class_and_name(&worker->lock, key, name);
643697 	INIT_LIST_HEAD(&worker->work_list);
644698 	INIT_LIST_HEAD(&worker->delayed_work_list);
....@@ -680,21 +734,21 @@
680734 
681735 	if (kthread_should_stop()) {
682736 		__set_current_state(TASK_RUNNING);
683
-		spin_lock_irq(&worker->lock);
737
+		raw_spin_lock_irq(&worker->lock);
684738 		worker->task = NULL;
685
-		spin_unlock_irq(&worker->lock);
739
+		raw_spin_unlock_irq(&worker->lock);
686740 		return 0;
687741 	}
688742 
689743 	work = NULL;
690
-	spin_lock_irq(&worker->lock);
744
+	raw_spin_lock_irq(&worker->lock);
691745 	if (!list_empty(&worker->work_list)) {
692746 		work = list_first_entry(&worker->work_list,
693747 					struct kthread_work, node);
694748 		list_del_init(&work->node);
695749 	}
696750 	worker->current_work = work;
697
-	spin_unlock_irq(&worker->lock);
751
+	raw_spin_unlock_irq(&worker->lock);
698752 
699753 	if (work) {
700754 		__set_current_state(TASK_RUNNING);
....@@ -714,7 +768,7 @@
714768 {
715769 	struct kthread_worker *worker;
716770 	struct task_struct *task;
717
-	int node = -1;
771
+	int node = NUMA_NO_NODE;
718772 
719773 	worker = kzalloc(sizeof(*worker), GFP_KERNEL);
720774 	if (!worker)
....@@ -768,7 +822,7 @@
768822 
769823 /**
770824  * kthread_create_worker_on_cpu - create a kthread worker and bind it
771
- *	it to a given CPU and the associated NUMA node.
825
+ *	to a given CPU and the associated NUMA node.
772826  * @cpu: CPU number
773827  * @flags: flags modifying the default behavior of the worker
774828  * @namefmt: printf-style name for the kthread worker (task).
....@@ -851,12 +905,12 @@
851905 	bool ret = false;
852906 	unsigned long flags;
853907 
854
-	spin_lock_irqsave(&worker->lock, flags);
908
+	raw_spin_lock_irqsave(&worker->lock, flags);
855909 	if (!queuing_blocked(worker, work)) {
856910 		kthread_insert_work(worker, work, &worker->work_list);
857911 		ret = true;
858912 	}
859
-	spin_unlock_irqrestore(&worker->lock, flags);
913
+	raw_spin_unlock_irqrestore(&worker->lock, flags);
860914 	return ret;
861915 }
862916 EXPORT_SYMBOL_GPL(kthread_queue_work);
....@@ -874,6 +928,7 @@
874928 	struct kthread_delayed_work *dwork = from_timer(dwork, t, timer);
875929 	struct kthread_work *work = &dwork->work;
876930 	struct kthread_worker *worker = work->worker;
931
+	unsigned long flags;
877932 
878933 	/*
879934 	 * This might happen when a pending work is reinitialized.
....@@ -882,7 +937,7 @@
882937 	if (WARN_ON_ONCE(!worker))
883938 		return;
884939 
885
-	spin_lock(&worker->lock);
940
+	raw_spin_lock_irqsave(&worker->lock, flags);
886941 	/* Work must not be used with >1 worker, see kthread_queue_work(). */
887942 	WARN_ON_ONCE(work->worker != worker);
888943 
....@@ -892,18 +947,24 @@
892947 	if (!work->canceling)
893948 		kthread_insert_work(worker, work, &worker->work_list);
894949 
895
-	spin_unlock(&worker->lock);
950
+	raw_spin_unlock_irqrestore(&worker->lock, flags);
896951 }
897952 EXPORT_SYMBOL(kthread_delayed_work_timer_fn);
898953 
899
-void __kthread_queue_delayed_work(struct kthread_worker *worker,
900
-				  struct kthread_delayed_work *dwork,
901
-				  unsigned long delay)
954
+static void __kthread_queue_delayed_work(struct kthread_worker *worker,
955
+					 struct kthread_delayed_work *dwork,
956
+					 unsigned long delay)
902957 {
903958 	struct timer_list *timer = &dwork->timer;
904959 	struct kthread_work *work = &dwork->work;
905960 
906
-	WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn);
961
+	/*
962
+	 * With CFI, timer->function can point to a jump table entry in a module,
963
+	 * which fails the comparison. Disable the warning if CFI and modules are
964
+	 * both enabled.
965
+	 */
966
+	if (!IS_ENABLED(CONFIG_CFI_CLANG) || !IS_ENABLED(CONFIG_MODULES))
967
+		WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn);
907968 
908969 	/*
909970 	 * If @delay is 0, queue @dwork->work immediately.  This is for
....@@ -948,14 +1009,14 @@
9481009 	unsigned long flags;
9491010 	bool ret = false;
9501011 
951
-	spin_lock_irqsave(&worker->lock, flags);
1012
+	raw_spin_lock_irqsave(&worker->lock, flags);
9521013 
9531014 	if (!queuing_blocked(worker, work)) {
9541015 		__kthread_queue_delayed_work(worker, dwork, delay);
9551016 		ret = true;
9561017 	}
9571018 
958
-	spin_unlock_irqrestore(&worker->lock, flags);
1019
+	raw_spin_unlock_irqrestore(&worker->lock, flags);
9591020 	return ret;
9601021 }
9611022 EXPORT_SYMBOL_GPL(kthread_queue_delayed_work);
....@@ -991,7 +1052,7 @@
9911052 	if (!worker)
9921053 		return;
9931054 
994
-	spin_lock_irq(&worker->lock);
1055
+	raw_spin_lock_irq(&worker->lock);
9951056 	/* Work must not be used with >1 worker, see kthread_queue_work(). */
9961057 	WARN_ON_ONCE(work->worker != worker);
9971058 
....@@ -1003,7 +1064,7 @@
10031064 	else
10041065 		noop = true;
10051066 
1006
-	spin_unlock_irq(&worker->lock);
1067
+	raw_spin_unlock_irq(&worker->lock);
10071068 
10081069 	if (!noop)
10091070 		wait_for_completion(&fwork.done);
....@@ -1031,9 +1092,9 @@
10311092 	 * any queuing is blocked by setting the canceling counter.
10321093 	 */
10331094 	work->canceling++;
1034
-	spin_unlock_irqrestore(&worker->lock, *flags);
1095
+	raw_spin_unlock_irqrestore(&worker->lock, *flags);
10351096 	del_timer_sync(&dwork->timer);
1036
-	spin_lock_irqsave(&worker->lock, *flags);
1097
+	raw_spin_lock_irqsave(&worker->lock, *flags);
10371098 	work->canceling--;
10381099 }
10391100 
....@@ -1074,14 +1135,14 @@
10741135  * modify @dwork's timer so that it expires after @delay. If @delay is zero,
10751136  * @work is guaranteed to be queued immediately.
10761137  *
1077
- * Return: %true if @dwork was pending and its timer was modified,
1078
- * %false otherwise.
1138
+ * Return: %false if @dwork was idle and queued, %true otherwise.
10791139  *
10801140  * A special case is when the work is being canceled in parallel.
10811141  * It might be caused either by the real kthread_cancel_delayed_work_sync()
10821142  * or yet another kthread_mod_delayed_work() call. We let the other command
1083
- * win and return %false here. The caller is supposed to synchronize these
1084
- * operations a reasonable way.
1143
+ * win and return %true here. The return value can be used for reference
1144
+ * counting and the number of queued works stays the same. Anyway, the caller
1145
+ * is supposed to synchronize these operations a reasonable way.
10851146  *
10861147  * This function is safe to call from any context including IRQ handler.
10871148  * See __kthread_cancel_work() and kthread_delayed_work_timer_fn()
....@@ -1093,13 +1154,15 @@
10931154 {
10941155 	struct kthread_work *work = &dwork->work;
10951156 	unsigned long flags;
1096
-	int ret = false;
1157
+	int ret;
10971158 
1098
-	spin_lock_irqsave(&worker->lock, flags);
1159
+	raw_spin_lock_irqsave(&worker->lock, flags);
10991160 
11001161 	/* Do not bother with canceling when never queued. */
1101
-	if (!work->worker)
1162
+	if (!work->worker) {
1163
+		ret = false;
11021164 		goto fast_queue;
1165
+	}
11031166 
11041167 	/* Work must not be used with >1 worker, see kthread_queue_work() */
11051168 	WARN_ON_ONCE(work->worker != worker);
....@@ -1117,14 +1180,29 @@
11171180 	 * be used for reference counting.
11181181 	 */
11191182 	kthread_cancel_delayed_work_timer(work, &flags);
1120
-	if (work->canceling)
1183
+	if (work->canceling) {
1184
+		/* The number of works in the queue does not change. */
1185
+		ret = true;
11211186 		goto out;
1187
+	}
11221188 	ret = __kthread_cancel_work(work);
1189
+
1190
+	/*
1191
+	 * Canceling could run in parallel from kthread_cancel_delayed_work_sync
1192
+	 * and change work's canceling count as the spinlock is released and regain
1193
+	 * in __kthread_cancel_work so we need to check the count again. Otherwise,
1194
+	 * we might incorrectly queue the dwork and further cause
1195
+	 * cancel_delayed_work_sync thread waiting for flush dwork endlessly.
1196
+	 */
1197
+	if (work->canceling) {
1198
+		ret = false;
1199
+		goto out;
1200
+	}
11231201 
11241202 fast_queue:
11251203 	__kthread_queue_delayed_work(worker, dwork, delay);
11261204 out:
1127
-	spin_unlock_irqrestore(&worker->lock, flags);
1205
+	raw_spin_unlock_irqrestore(&worker->lock, flags);
11281206 	return ret;
11291207 }
11301208 EXPORT_SYMBOL_GPL(kthread_mod_delayed_work);
....@@ -1138,7 +1216,7 @@
11381216 	if (!worker)
11391217 		goto out;
11401218 
1141
-	spin_lock_irqsave(&worker->lock, flags);
1219
+	raw_spin_lock_irqsave(&worker->lock, flags);
11421220 	/* Work must not be used with >1 worker, see kthread_queue_work(). */
11431221 	WARN_ON_ONCE(work->worker != worker);
11441222 
....@@ -1155,13 +1233,13 @@
11551233 	 * In the meantime, block any queuing by setting the canceling counter.
11561234 	 */
11571235 	work->canceling++;
1158
-	spin_unlock_irqrestore(&worker->lock, flags);
1236
+	raw_spin_unlock_irqrestore(&worker->lock, flags);
11591237 	kthread_flush_work(work);
1160
-	spin_lock_irqsave(&worker->lock, flags);
1238
+	raw_spin_lock_irqsave(&worker->lock, flags);
11611239 	work->canceling--;
11621240 
11631241 out_fast:
1164
-	spin_unlock_irqrestore(&worker->lock, flags);
1242
+	raw_spin_unlock_irqrestore(&worker->lock, flags);
11651243 out:
11661244 	return ret;
11671245 }
....@@ -1245,6 +1323,65 @@
12451323 }
12461324 EXPORT_SYMBOL(kthread_destroy_worker);
12471325 
1326
+/**
1327
+ * kthread_use_mm - make the calling kthread operate on an address space
1328
+ * @mm: address space to operate on
1329
+ */
1330
+void kthread_use_mm(struct mm_struct *mm)
1331
+{
1332
+	struct mm_struct *active_mm;
1333
+	struct task_struct *tsk = current;
1334
+
1335
+	WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
1336
+	WARN_ON_ONCE(tsk->mm);
1337
+
1338
+	task_lock(tsk);
1339
+	/* Hold off tlb flush IPIs while switching mm's */
1340
+	local_irq_disable();
1341
+	active_mm = tsk->active_mm;
1342
+	if (active_mm != mm) {
1343
+		mmgrab(mm);
1344
+		tsk->active_mm = mm;
1345
+	}
1346
+	tsk->mm = mm;
1347
+	switch_mm_irqs_off(active_mm, mm, tsk);
1348
+	local_irq_enable();
1349
+	task_unlock(tsk);
1350
+#ifdef finish_arch_post_lock_switch
1351
+	finish_arch_post_lock_switch();
1352
+#endif
1353
+
1354
+	if (active_mm != mm)
1355
+		mmdrop(active_mm);
1356
+
1357
+	to_kthread(tsk)->oldfs = force_uaccess_begin();
1358
+}
1359
+EXPORT_SYMBOL_GPL(kthread_use_mm);
1360
+
1361
+/**
1362
+ * kthread_unuse_mm - reverse the effect of kthread_use_mm()
1363
+ * @mm: address space to operate on
1364
+ */
1365
+void kthread_unuse_mm(struct mm_struct *mm)
1366
+{
1367
+	struct task_struct *tsk = current;
1368
+
1369
+	WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
1370
+	WARN_ON_ONCE(!tsk->mm);
1371
+
1372
+	force_uaccess_end(to_kthread(tsk)->oldfs);
1373
+
1374
+	task_lock(tsk);
1375
+	sync_mm_rss(mm);
1376
+	local_irq_disable();
1377
+	tsk->mm = NULL;
1378
+	/* active_mm is still 'mm' */
1379
+	enter_lazy_tlb(mm, tsk);
1380
+	local_irq_enable();
1381
+	task_unlock(tsk);
1382
+}
1383
+EXPORT_SYMBOL_GPL(kthread_unuse_mm);
1384
+
12481385 #ifdef CONFIG_BLK_CGROUP
12491386 /**
12501387  * kthread_associate_blkcg - associate blkcg to current kthread