~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* kernel/workqueue.c - generic async execution with shared worker pool
3	4	*
..	..	@@ -50,8 +51,13 @@
50	51	#include <linux/sched/isolation.h>
51	52	#include <linux/nmi.h>
52	53	#include <linux/kvm_para.h>
	54	+#include <uapi/linux/sched/types.h>
53	55
54	56	#include "workqueue_internal.h"
	57	+
	58	+#include <trace/hooks/wqlockup.h>
	59	+/* events/workqueue.h uses default TRACE_INCLUDE_PATH */
	60	+#undef TRACE_INCLUDE_PATH
55	61
56	62	enum {
57	63	/*
..	..	@@ -128,16 +134,16 @@
128	134	*
129	135	* PL: wq_pool_mutex protected.
130	136	*
131		- * PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads.
	137	+ * PR: wq_pool_mutex protected for writes. RCU protected for reads.
132	138	*
133	139	* PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
134	140	*
135	141	* PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
136		- * sched-RCU for reads.
	142	+ * RCU for reads.
137	143	*
138	144	* WQ: wq->mutex protected.
139	145	*
140		- * WR: wq->mutex protected for writes. Sched-RCU protected for reads.
	146	+ * WR: wq->mutex protected for writes. RCU protected for reads.
141	147	*
142	148	* MD: wq_mayday_lock protected.
143	149	*/
..	..	@@ -145,7 +151,7 @@
145	151	/* struct worker is defined in workqueue_internal.h */
146	152
147	153	struct worker_pool {
148		- spinlock_t lock; /* the pool lock */
	154	+ raw_spinlock_t lock; /* the pool lock */
149	155	int cpu; /* I: the associated cpu */
150	156	int node; /* I: the associated node ID */
151	157	int id; /* I: pool ID */
..	..	@@ -184,7 +190,7 @@
184	190	atomic_t nr_running ____cacheline_aligned_in_smp;
185	191
186	192	/*
187		- * Destruction of pool is sched-RCU protected to allow dereferences
	193	+ * Destruction of pool is RCU protected to allow dereferences
188	194	* from get_work_pool().
189	195	*/
190	196	struct rcu_head rcu;
..	..	@@ -213,7 +219,7 @@
213	219	/*
214	220	* Release of unbound pwq is punted to system_wq. See put_pwq()
215	221	* and pwq_unbound_release_workfn() for details. pool_workqueue
216		- * itself is also sched-RCU protected so that the first pwq can be
	222	+ * itself is also RCU protected so that the first pwq can be
217	223	* determined without grabbing wq->mutex.
218	224	*/
219	225	struct work_struct unbound_release_work;
..	..	@@ -248,7 +254,7 @@
248	254	struct list_head flusher_overflow; /* WQ: flush overflow list */
249	255
250	256	struct list_head maydays; /* MD: pwqs requesting rescue */
251		- struct worker rescuer; / I: rescue worker */
	257	+ struct worker rescuer; / MD: rescue worker */
252	258
253	259	int nr_drainers; /* WQ: drain in progress */
254	260	int saved_max_active; /* WQ: saved pwq max_active */
..	..	@@ -260,13 +266,15 @@
260	266	struct wq_device wq_dev; / I: for sysfs interface */
261	267	#endif
262	268	#ifdef CONFIG_LOCKDEP
	269	+ char *lock_name;
	270	+ struct lock_class_key key;
263	271	struct lockdep_map lockdep_map;
264	272	#endif
265	273	char name[WQ_NAME_LEN]; /* I: workqueue name */
266	274
267	275	/*
268		- * Destruction of workqueue_struct is sched-RCU protected to allow
269		- * walking the workqueues list without grabbing wq_pool_mutex.
	276	+ * Destruction of workqueue_struct is RCU protected to allow walking
	277	+ * the workqueues list without grabbing wq_pool_mutex.
270	278	* This is used to dump all workqueues from sysrq.
271	279	*/
272	280	struct rcu_head rcu;
..	..	@@ -298,8 +306,9 @@
298	306
299	307	static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
300	308	static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
301		-static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
302		-static DECLARE_WAIT_QUEUE_HEAD(wq_manager_wait); /* wait for manager to go away */
	309	+static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
	310	+/* wait for manager to go away */
	311	+static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);
303	312
304	313	static LIST_HEAD(workqueues); /* PR: list of all workqueues */
305	314	static bool workqueue_freezing; /* PL: have wqs started freezing? */
..	..	@@ -353,25 +362,24 @@
353	362
354	363	static int worker_thread(void *__worker);
355	364	static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
	365	+static void show_pwq(struct pool_workqueue *pwq);
356	366
357	367	#define CREATE_TRACE_POINTS
358	368	#include <trace/events/workqueue.h>
359	369
360		-#define assert_rcu_or_pool_mutex() \
361		- RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
362		- !lockdep_is_held(&wq_pool_mutex), \
363		- "sched RCU or wq_pool_mutex should be held")
	370	+EXPORT_TRACEPOINT_SYMBOL_GPL(workqueue_execute_start);
	371	+EXPORT_TRACEPOINT_SYMBOL_GPL(workqueue_execute_end);
364	372
365		-#define assert_rcu_or_wq_mutex(wq) \
366		- RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
367		- !lockdep_is_held(&wq->mutex), \
368		- "sched RCU or wq->mutex should be held")
	373	+#define assert_rcu_or_pool_mutex() \
	374	+ RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
	375	+ !lockdep_is_held(&wq_pool_mutex), \
	376	+ "RCU or wq_pool_mutex should be held")
369	377
370	378	#define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
371		- RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
	379	+ RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
372	380	!lockdep_is_held(&wq->mutex) && \
373	381	!lockdep_is_held(&wq_pool_mutex), \
374		- "sched RCU, wq->mutex or wq_pool_mutex should be held")
	382	+ "RCU, wq->mutex or wq_pool_mutex should be held")
375	383
376	384	#define for_each_cpu_worker_pool(pool, cpu) \
377	385	for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
..	..	@@ -383,7 +391,7 @@
383	391	* @pool: iteration cursor
384	392	* @pi: integer used for iteration
385	393	*
386		- * This must be called either with wq_pool_mutex held or sched RCU read
	394	+ * This must be called either with wq_pool_mutex held or RCU read
387	395	* locked. If the pool needs to be used beyond the locking in effect, the
388	396	* caller is responsible for guaranteeing that the pool stays online.
389	397	*
..	..	@@ -415,7 +423,7 @@
415	423	* @pwq: iteration cursor
416	424	* @wq: the target workqueue
417	425	*
418		- * This must be called either with wq->mutex held or sched RCU read locked.
	426	+ * This must be called either with wq->mutex held or RCU read locked.
419	427	* If the pwq needs to be used beyond the locking in effect, the caller is
420	428	* responsible for guaranteeing that the pwq stays online.
421	429	*
..	..	@@ -423,13 +431,12 @@
423	431	* ignored.
424	432	*/
425	433	#define for_each_pwq(pwq, wq) \
426		- list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
427		- if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
428		- else
	434	+ list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
	435	+ lockdep_is_held(&(wq->mutex)))
429	436
430	437	#ifdef CONFIG_DEBUG_OBJECTS_WORK
431	438
432		-static struct debug_obj_descr work_debug_descr;
	439	+static const struct debug_obj_descr work_debug_descr;
433	440
434	441	static void work_debug_hint(void addr)
435	442	{
..	..	@@ -479,7 +486,7 @@
479	486	}
480	487	}
481	488
482		-static struct debug_obj_descr work_debug_descr = {
	489	+static const struct debug_obj_descr work_debug_descr = {
483	490	.name = "work_struct",
484	491	.debug_hint = work_debug_hint,
485	492	.is_static_object = work_is_static_object,
..	..	@@ -551,7 +558,7 @@
551	558	* @wq: the target workqueue
552	559	* @node: the node ID
553	560	*
554		- * This must be called with any of wq_pool_mutex, wq->mutex or sched RCU
	561	+ * This must be called with any of wq_pool_mutex, wq->mutex or RCU
555	562	* read locked.
556	563	* If the pwq needs to be used beyond the locking in effect, the caller is
557	564	* responsible for guaranteeing that the pwq stays online.
..	..	@@ -647,7 +654,7 @@
647	654	* The following mb guarantees that previous clear of a PENDING bit
648	655	* will not be reordered with any speculative LOADS or STORES from
649	656	* work->current_func, which is executed afterwards. This possible
650		- * reordering can lead to a missed execution on attempt to qeueue
	657	+ * reordering can lead to a missed execution on attempt to queue
651	658	* the same @work. E.g. consider this case:
652	659	*
653	660	* CPU#0 CPU#1
..	..	@@ -695,8 +702,8 @@
695	702	* @work: the work item of interest
696	703	*
697	704	* Pools are created and destroyed under wq_pool_mutex, and allows read
698		- * access under sched-RCU read lock. As such, this function should be
699		- * called under wq_pool_mutex or with preemption disabled.
	705	+ * access under RCU read lock. As such, this function should be
	706	+ * called under wq_pool_mutex or inside of a rcu_read_lock() region.
700	707	*
701	708	* All fields of the returned pool are accessible as long as the above
702	709	* mentioned locking is in effect. If the returned pool needs to be used
..	..	@@ -829,7 +836,7 @@
829	836	* Wake up the first idle worker of @pool.
830	837	*
831	838	* CONTEXT:
832		- * spin_lock_irq(pool->lock).
	839	+ * raw_spin_lock_irq(pool->lock).
833	840	*/
834	841	static void wake_up_worker(struct worker_pool *pool)
835	842	{
..	..	@@ -840,43 +847,42 @@
840	847	}
841	848
842	849	/**
843		- * wq_worker_waking_up - a worker is waking up
	850	+ * wq_worker_running - a worker is running again
844	851	* @task: task waking up
845		- * @cpu: CPU @task is waking up to
846	852	*
847		- * This function is called during try_to_wake_up() when a worker is
848		- * being awoken.
849		- *
850		- * CONTEXT:
851		- * spin_lock_irq(rq->lock)
	853	+ * This function is called when a worker returns from schedule()
852	854	*/
853		-void wq_worker_waking_up(struct task_struct *task, int cpu)
	855	+void wq_worker_running(struct task_struct *task)
854	856	{
855	857	struct worker *worker = kthread_data(task);
856	858
857		- if (!(worker->flags & WORKER_NOT_RUNNING)) {
858		- WARN_ON_ONCE(worker->pool->cpu != cpu);
	859	+ if (!worker->sleeping)
	860	+ return;
	861	+
	862	+ /*
	863	+ * If preempted by unbind_workers() between the WORKER_NOT_RUNNING check
	864	+ * and the nr_running increment below, we may ruin the nr_running reset
	865	+ * and leave with an unexpected pool->nr_running == 1 on the newly unbound
	866	+ * pool. Protect against such race.
	867	+ */
	868	+ preempt_disable();
	869	+ if (!(worker->flags & WORKER_NOT_RUNNING))
859	870	atomic_inc(&worker->pool->nr_running);
860		- }
	871	+ preempt_enable();
	872	+ worker->sleeping = 0;
861	873	}
862	874
863	875	/**
864	876	* wq_worker_sleeping - a worker is going to sleep
865	877	* @task: task going to sleep
866	878	*
867		- * This function is called during schedule() when a busy worker is
868		- * going to sleep. Worker on the same cpu can be woken up by
869		- * returning pointer to its task.
870		- *
871		- * CONTEXT:
872		- * spin_lock_irq(rq->lock)
873		- *
874		- * Return:
875		- * Worker task on @cpu to wake up, %NULL if none.
	879	+ * This function is called from schedule() when a busy worker is
	880	+ * going to sleep. Preemption needs to be disabled to protect ->sleeping
	881	+ * assignment.
876	882	*/
877		-struct task_struct wq_worker_sleeping(struct task_struct task)
	883	+void wq_worker_sleeping(struct task_struct *task)
878	884	{
879		- struct worker worker = kthread_data(task), to_wakeup = NULL;
	885	+ struct worker next, worker = kthread_data(task);
880	886	struct worker_pool *pool;
881	887
882	888	/*
..	..	@@ -885,13 +891,16 @@
885	891	* checking NOT_RUNNING.
886	892	*/
887	893	if (worker->flags & WORKER_NOT_RUNNING)
888		- return NULL;
	894	+ return;
889	895
890	896	pool = worker->pool;
891	897
892		- /* this can only happen on the local cpu */
893		- if (WARN_ON_ONCE(pool->cpu != raw_smp_processor_id()))
894		- return NULL;
	898	+ /* Return if preempted before wq_worker_running() was reached */
	899	+ if (worker->sleeping)
	900	+ return;
	901	+
	902	+ worker->sleeping = 1;
	903	+ raw_spin_lock_irq(&pool->lock);
895	904
896	905	/*
897	906	* The counterpart of the following dec_and_test, implied mb,
..	..	@@ -905,19 +914,33 @@
905	914	* lock is safe.
906	915	*/
907	916	if (atomic_dec_and_test(&pool->nr_running) &&
908		- !list_empty(&pool->worklist))
909		- to_wakeup = first_idle_worker(pool);
910		- return to_wakeup ? to_wakeup->task : NULL;
	917	+ !list_empty(&pool->worklist)) {
	918	+ next = first_idle_worker(pool);
	919	+ if (next)
	920	+ wake_up_process(next->task);
	921	+ }
	922	+ raw_spin_unlock_irq(&pool->lock);
911	923	}
912	924
913	925	/**
914	926	* wq_worker_last_func - retrieve worker's last work function
	927	+ * @task: Task to retrieve last work function of.
915	928	*
916	929	* Determine the last function a worker executed. This is called from
917	930	* the scheduler to get a worker's last known identity.
918	931	*
919	932	* CONTEXT:
920		- * spin_lock_irq(rq->lock)
	933	+ * raw_spin_lock_irq(rq->lock)
	934	+ *
	935	+ * This function is called during schedule() when a kworker is going
	936	+ * to sleep. It's used by psi to identify aggregation workers during
	937	+ * dequeuing, to allow periodic aggregation to shut-off when that
	938	+ * worker is the last task in the system or cgroup to go to sleep.
	939	+ *
	940	+ * As this function doesn't involve any workqueue-related locking, it
	941	+ * only returns stable values when called from inside the scheduler's
	942	+ * queuing and dequeuing paths, when @task, which must be a kworker,
	943	+ * is guaranteed to not be processing any works.
921	944	*
922	945	* Return:
923	946	* The last work function %current executed as a worker, NULL if it
..	..	@@ -938,7 +961,7 @@
938	961	* Set @flags in @worker->flags and adjust nr_running accordingly.
939	962	*
940	963	* CONTEXT:
941		- * spin_lock_irq(pool->lock)
	964	+ * raw_spin_lock_irq(pool->lock)
942	965	*/
943	966	static inline void worker_set_flags(struct worker *worker, unsigned int flags)
944	967	{
..	..	@@ -963,7 +986,7 @@
963	986	* Clear @flags in @worker->flags and adjust nr_running accordingly.
964	987	*
965	988	* CONTEXT:
966		- * spin_lock_irq(pool->lock)
	989	+ * raw_spin_lock_irq(pool->lock)
967	990	*/
968	991	static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
969	992	{
..	..	@@ -1011,7 +1034,7 @@
1011	1034	* actually occurs, it should be easy to locate the culprit work function.
1012	1035	*
1013	1036	* CONTEXT:
1014		- * spin_lock_irq(pool->lock).
	1037	+ * raw_spin_lock_irq(pool->lock).
1015	1038	*
1016	1039	* Return:
1017	1040	* Pointer to worker which is executing @work if found, %NULL
..	..	@@ -1046,7 +1069,7 @@
1046	1069	* nested inside outer list_for_each_entry_safe().
1047	1070	*
1048	1071	* CONTEXT:
1049		- * spin_lock_irq(pool->lock).
	1072	+ * raw_spin_lock_irq(pool->lock).
1050	1073	*/
1051	1074	static void move_linked_works(struct work_struct work, struct list_head head,
1052	1075	struct work_struct **nextp)
..	..	@@ -1121,12 +1144,12 @@
1121	1144	{
1122	1145	if (pwq) {
1123	1146	/*
1124		- * As both pwqs and pools are sched-RCU protected, the
	1147	+ * As both pwqs and pools are RCU protected, the
1125	1148	* following lock operations are safe.
1126	1149	*/
1127		- spin_lock_irq(&pwq->pool->lock);
	1150	+ raw_spin_lock_irq(&pwq->pool->lock);
1128	1151	put_pwq(pwq);
1129		- spin_unlock_irq(&pwq->pool->lock);
	1152	+ raw_spin_unlock_irq(&pwq->pool->lock);
1130	1153	}
1131	1154	}
1132	1155
..	..	@@ -1159,7 +1182,7 @@
1159	1182	* decrement nr_in_flight of its pwq and handle workqueue flushing.
1160	1183	*
1161	1184	* CONTEXT:
1162		- * spin_lock_irq(pool->lock).
	1185	+ * raw_spin_lock_irq(pool->lock).
1163	1186	*/
1164	1187	static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
1165	1188	{
..	..	@@ -1207,11 +1230,14 @@
1207	1230	* stable state - idle, on timer or on worklist.
1208	1231	*
1209	1232	* Return:
	1233	+ *
	1234	+ * ======== ================================================================
1210	1235	* 1 if @work was pending and we successfully stole PENDING
1211	1236	* 0 if @work was idle and we claimed PENDING
1212	1237	* -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
1213	1238	* -ENOENT if someone else is canceling @work, this state may persist
1214	1239	* for arbitrarily long
	1240	+ * ======== ================================================================
1215	1241	*
1216	1242	* Note:
1217	1243	* On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
..	..	@@ -1249,6 +1275,7 @@
1249	1275	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1250	1276	return 0;
1251	1277
	1278	+ rcu_read_lock();
1252	1279	/*
1253	1280	* The queueing is in progress, or it is already queued. Try to
1254	1281	* steal it from ->worklist without clearing WORK_STRUCT_PENDING.
..	..	@@ -1257,7 +1284,7 @@
1257	1284	if (!pool)
1258	1285	goto fail;
1259	1286
1260		- spin_lock(&pool->lock);
	1287	+ raw_spin_lock(&pool->lock);
1261	1288	/*
1262	1289	* work->data is guaranteed to point to pwq only while the work
1263	1290	* item is queued on pwq->wq, and both updating work->data to point
..	..	@@ -1286,11 +1313,13 @@
1286	1313	/* work->data points to pwq iff queued, point to pool */
1287	1314	set_work_pool_and_keep_pending(work, pool->id);
1288	1315
1289		- spin_unlock(&pool->lock);
	1316	+ raw_spin_unlock(&pool->lock);
	1317	+ rcu_read_unlock();
1290	1318	return 1;
1291	1319	}
1292		- spin_unlock(&pool->lock);
	1320	+ raw_spin_unlock(&pool->lock);
1293	1321	fail:
	1322	+ rcu_read_unlock();
1294	1323	local_irq_restore(*flags);
1295	1324	if (work_is_canceling(work))
1296	1325	return -ENOENT;
..	..	@@ -1309,12 +1338,15 @@
1309	1338	* work_struct flags.
1310	1339	*
1311	1340	* CONTEXT:
1312		- * spin_lock_irq(pool->lock).
	1341	+ * raw_spin_lock_irq(pool->lock).
1313	1342	*/
1314	1343	static void insert_work(struct pool_workqueue pwq, struct work_struct work,
1315	1344	struct list_head *head, unsigned int extra_flags)
1316	1345	{
1317	1346	struct worker_pool *pool = pwq->pool;
	1347	+
	1348	+ /* record the work call stack in order to print it in KASAN reports */
	1349	+ kasan_record_aux_stack(work);
1318	1350
1319	1351	/* we own @work, set data and link */
1320	1352	set_work_pwq(work, pwq, extra_flags);
..	..	@@ -1342,7 +1374,7 @@
1342	1374
1343	1375	worker = current_wq_worker();
1344	1376	/*
1345		- * Return %true iff I'm a worker execuing a work item on @wq. If
	1377	+ * Return %true iff I'm a worker executing a work item on @wq. If
1346	1378	* I'm @worker, it's safe to dereference it without locking.
1347	1379	*/
1348	1380	return worker && worker->current_pwq->wq == wq;
..	..	@@ -1403,6 +1435,7 @@
1403	1435	if (unlikely(wq->flags & __WQ_DRAINING) &&
1404	1436	WARN_ON_ONCE(!is_chained_work(wq)))
1405	1437	return;
	1438	+ rcu_read_lock();
1406	1439	retry:
1407	1440	/* pwq which will be used unless @work is executing elsewhere */
1408	1441	if (wq->flags & WQ_UNBOUND) {
..	..	@@ -1424,7 +1457,7 @@
1424	1457	if (last_pool && last_pool != pwq->pool) {
1425	1458	struct worker *worker;
1426	1459
1427		- spin_lock(&last_pool->lock);
	1460	+ raw_spin_lock(&last_pool->lock);
1428	1461
1429	1462	worker = find_worker_executing_work(last_pool, work);
1430	1463
..	..	@@ -1432,11 +1465,11 @@
1432	1465	pwq = worker->current_pwq;
1433	1466	} else {
1434	1467	/* meh... not running there, queue here */
1435		- spin_unlock(&last_pool->lock);
1436		- spin_lock(&pwq->pool->lock);
	1468	+ raw_spin_unlock(&last_pool->lock);
	1469	+ raw_spin_lock(&pwq->pool->lock);
1437	1470	}
1438	1471	} else {
1439		- spin_lock(&pwq->pool->lock);
	1472	+ raw_spin_lock(&pwq->pool->lock);
1440	1473	}
1441	1474
1442	1475	/*
..	..	@@ -1449,7 +1482,7 @@
1449	1482	*/
1450	1483	if (unlikely(!pwq->refcnt)) {
1451	1484	if (wq->flags & WQ_UNBOUND) {
1452		- spin_unlock(&pwq->pool->lock);
	1485	+ raw_spin_unlock(&pwq->pool->lock);
1453	1486	cpu_relax();
1454	1487	goto retry;
1455	1488	}
..	..	@@ -1461,10 +1494,8 @@
1461	1494	/* pwq determined, queue */
1462	1495	trace_workqueue_queue_work(req_cpu, pwq, work);
1463	1496
1464		- if (WARN_ON(!list_empty(&work->entry))) {
1465		- spin_unlock(&pwq->pool->lock);
1466		- return;
1467		- }
	1497	+ if (WARN_ON(!list_empty(&work->entry)))
	1498	+ goto out;
1468	1499
1469	1500	pwq->nr_in_flight[pwq->work_color]++;
1470	1501	work_flags = work_color_to_flags(pwq->work_color);
..	..	@@ -1483,7 +1514,9 @@
1483	1514	debug_work_activate(work);
1484	1515	insert_work(pwq, work, worklist, work_flags);
1485	1516
1486		- spin_unlock(&pwq->pool->lock);
	1517	+out:
	1518	+ raw_spin_unlock(&pwq->pool->lock);
	1519	+ rcu_read_unlock();
1487	1520	}
1488	1521
1489	1522	/**
..	..	@@ -1515,6 +1548,90 @@
1515	1548	}
1516	1549	EXPORT_SYMBOL(queue_work_on);
1517	1550
	1551	+/**
	1552	+ * workqueue_select_cpu_near - Select a CPU based on NUMA node
	1553	+ * @node: NUMA node ID that we want to select a CPU from
	1554	+ *
	1555	+ * This function will attempt to find a "random" cpu available on a given
	1556	+ * node. If there are no CPUs available on the given node it will return
	1557	+ * WORK_CPU_UNBOUND indicating that we should just schedule to any
	1558	+ * available CPU if we need to schedule this work.
	1559	+ */
	1560	+static int workqueue_select_cpu_near(int node)
	1561	+{
	1562	+ int cpu;
	1563	+
	1564	+ /* No point in doing this if NUMA isn't enabled for workqueues */
	1565	+ if (!wq_numa_enabled)
	1566	+ return WORK_CPU_UNBOUND;
	1567	+
	1568	+ /* Delay binding to CPU if node is not valid or online */
	1569	+ if (node < 0 \|\| node >= MAX_NUMNODES \|\| !node_online(node))
	1570	+ return WORK_CPU_UNBOUND;
	1571	+
	1572	+ /* Use local node/cpu if we are already there */
	1573	+ cpu = raw_smp_processor_id();
	1574	+ if (node == cpu_to_node(cpu))
	1575	+ return cpu;
	1576	+
	1577	+ /* Use "random" otherwise know as "first" online CPU of node */
	1578	+ cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
	1579	+
	1580	+ /* If CPU is valid return that, otherwise just defer */
	1581	+ return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;
	1582	+}
	1583	+
	1584	+/**
	1585	+ * queue_work_node - queue work on a "random" cpu for a given NUMA node
	1586	+ * @node: NUMA node that we are targeting the work for
	1587	+ * @wq: workqueue to use
	1588	+ * @work: work to queue
	1589	+ *
	1590	+ * We queue the work to a "random" CPU within a given NUMA node. The basic
	1591	+ * idea here is to provide a way to somehow associate work with a given
	1592	+ * NUMA node.
	1593	+ *
	1594	+ * This function will only make a best effort attempt at getting this onto
	1595	+ * the right NUMA node. If no node is requested or the requested node is
	1596	+ * offline then we just fall back to standard queue_work behavior.
	1597	+ *
	1598	+ * Currently the "random" CPU ends up being the first available CPU in the
	1599	+ * intersection of cpu_online_mask and the cpumask of the node, unless we
	1600	+ * are running on the node. In that case we just use the current CPU.
	1601	+ *
	1602	+ * Return: %false if @work was already on a queue, %true otherwise.
	1603	+ */
	1604	+bool queue_work_node(int node, struct workqueue_struct *wq,
	1605	+ struct work_struct *work)
	1606	+{
	1607	+ unsigned long flags;
	1608	+ bool ret = false;
	1609	+
	1610	+ /*
	1611	+ * This current implementation is specific to unbound workqueues.
	1612	+ * Specifically we only return the first available CPU for a given
	1613	+ * node instead of cycling through individual CPUs within the node.
	1614	+ *
	1615	+ * If this is used with a per-cpu workqueue then the logic in
	1616	+ * workqueue_select_cpu_near would need to be updated to allow for
	1617	+ * some round robin type logic.
	1618	+ */
	1619	+ WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));
	1620	+
	1621	+ local_irq_save(flags);
	1622	+
	1623	+ if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
	1624	+ int cpu = workqueue_select_cpu_near(node);
	1625	+
	1626	+ __queue_work(cpu, wq, work);
	1627	+ ret = true;
	1628	+ }
	1629	+
	1630	+ local_irq_restore(flags);
	1631	+ return ret;
	1632	+}
	1633	+EXPORT_SYMBOL_GPL(queue_work_node);
	1634	+
1518	1635	void delayed_work_timer_fn(struct timer_list *t)
1519	1636	{
1520	1637	struct delayed_work *dwork = from_timer(dwork, t, timer);
..	..	@@ -1531,9 +1648,14 @@
1531	1648	struct work_struct *work = &dwork->work;
1532	1649
1533	1650	WARN_ON_ONCE(!wq);
1534		-#ifndef CONFIG_CFI_CLANG
1535		- WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
1536		-#endif
	1651	+ /*
	1652	+ * With CFI, timer->function can point to a jump table entry in a module,
	1653	+ * which fails the comparison. Disable the warning if CFI and modules are
	1654	+ * both enabled.
	1655	+ */
	1656	+ if (!IS_ENABLED(CONFIG_CFI_CLANG) \|\| !IS_ENABLED(CONFIG_MODULES))
	1657	+ WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
	1658	+
1537	1659	WARN_ON_ONCE(timer_pending(timer));
1538	1660	WARN_ON_ONCE(!list_empty(&work->entry));
1539	1661
..	..	@@ -1644,7 +1766,7 @@
1644	1766	*
1645	1767	* Return: %false if @rwork was already pending, %true otherwise. Note
1646	1768	* that a full RCU grace period is guaranteed only after a %true return.
1647		- * While @rwork is guarnateed to be executed after a %false return, the
	1769	+ * While @rwork is guaranteed to be executed after a %false return, the
1648	1770	* execution may happen before a full RCU grace period has passed.
1649	1771	*/
1650	1772	bool queue_rcu_work(struct workqueue_struct wq, struct rcu_work rwork)
..	..	@@ -1669,7 +1791,7 @@
1669	1791	* necessary.
1670	1792	*
1671	1793	* LOCKING:
1672		- * spin_lock_irq(pool->lock).
	1794	+ * raw_spin_lock_irq(pool->lock).
1673	1795	*/
1674	1796	static void worker_enter_idle(struct worker *worker)
1675	1797	{
..	..	@@ -1709,7 +1831,7 @@
1709	1831	* @worker is leaving idle state. Update stats.
1710	1832	*
1711	1833	* LOCKING:
1712		- * spin_lock_irq(pool->lock).
	1834	+ * raw_spin_lock_irq(pool->lock).
1713	1835	*/
1714	1836	static void worker_leave_idle(struct worker *worker)
1715	1837	{
..	..	@@ -1838,17 +1960,26 @@
1838	1960	goto fail;
1839	1961
1840	1962	set_user_nice(worker->task, pool->attrs->nice);
	1963	+ if (IS_ENABLED(CONFIG_ROCKCHIP_OPTIMIZE_RT_PRIO)) {
	1964	+ struct sched_param param;
	1965	+
	1966	+ if (pool->attrs->nice == 0)
	1967	+ param.sched_priority = MAX_RT_PRIO / 2 - 4;
	1968	+ else
	1969	+ param.sched_priority = MAX_RT_PRIO / 2 - 2;
	1970	+ sched_setscheduler_nocheck(worker->task, SCHED_RR, &param);
	1971	+ }
1841	1972	kthread_bind_mask(worker->task, pool->attrs->cpumask);
1842	1973
1843	1974	/* successful, attach the worker to the pool */
1844	1975	worker_attach_to_pool(worker, pool);
1845	1976
1846	1977	/* start the newly created worker */
1847		- spin_lock_irq(&pool->lock);
	1978	+ raw_spin_lock_irq(&pool->lock);
1848	1979	worker->pool->nr_workers++;
1849	1980	worker_enter_idle(worker);
1850	1981	wake_up_process(worker->task);
1851		- spin_unlock_irq(&pool->lock);
	1982	+ raw_spin_unlock_irq(&pool->lock);
1852	1983
1853	1984	return worker;
1854	1985
..	..	@@ -1867,7 +1998,7 @@
1867	1998	* be idle.
1868	1999	*
1869	2000	* CONTEXT:
1870		- * spin_lock_irq(pool->lock).
	2001	+ * raw_spin_lock_irq(pool->lock).
1871	2002	*/
1872	2003	static void destroy_worker(struct worker *worker)
1873	2004	{
..	..	@@ -1893,7 +2024,7 @@
1893	2024	{
1894	2025	struct worker_pool *pool = from_timer(pool, t, idle_timer);
1895	2026
1896		- spin_lock_irq(&pool->lock);
	2027	+ raw_spin_lock_irq(&pool->lock);
1897	2028
1898	2029	while (too_many_workers(pool)) {
1899	2030	struct worker *worker;
..	..	@@ -1911,7 +2042,7 @@
1911	2042	destroy_worker(worker);
1912	2043	}
1913	2044
1914		- spin_unlock_irq(&pool->lock);
	2045	+ raw_spin_unlock_irq(&pool->lock);
1915	2046	}
1916	2047
1917	2048	static void send_mayday(struct work_struct *work)
..	..	@@ -1942,8 +2073,8 @@
1942	2073	struct worker_pool *pool = from_timer(pool, t, mayday_timer);
1943	2074	struct work_struct *work;
1944	2075
1945		- spin_lock_irq(&pool->lock);
1946		- spin_lock(&wq_mayday_lock); /* for wq->maydays */
	2076	+ raw_spin_lock_irq(&pool->lock);
	2077	+ raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */
1947	2078
1948	2079	if (need_to_create_worker(pool)) {
1949	2080	/*
..	..	@@ -1956,8 +2087,8 @@
1956	2087	send_mayday(work);
1957	2088	}
1958	2089
1959		- spin_unlock(&wq_mayday_lock);
1960		- spin_unlock_irq(&pool->lock);
	2090	+ raw_spin_unlock(&wq_mayday_lock);
	2091	+ raw_spin_unlock_irq(&pool->lock);
1961	2092
1962	2093	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
1963	2094	}
..	..	@@ -1976,7 +2107,7 @@
1976	2107	* may_start_working() %true.
1977	2108	*
1978	2109	* LOCKING:
1979		- * spin_lock_irq(pool->lock) which may be released and regrabbed
	2110	+ * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
1980	2111	* multiple times. Does GFP_KERNEL allocations. Called only from
1981	2112	* manager.
1982	2113	*/
..	..	@@ -1985,7 +2116,7 @@
1985	2116	__acquires(&pool->lock)
1986	2117	{
1987	2118	restart:
1988		- spin_unlock_irq(&pool->lock);
	2119	+ raw_spin_unlock_irq(&pool->lock);
1989	2120
1990	2121	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
1991	2122	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
..	..	@@ -2001,7 +2132,7 @@
2001	2132	}
2002	2133
2003	2134	del_timer_sync(&pool->mayday_timer);
2004		- spin_lock_irq(&pool->lock);
	2135	+ raw_spin_lock_irq(&pool->lock);
2005	2136	/*
2006	2137	* This is necessary even after a new worker was just successfully
2007	2138	* created as @pool->lock was dropped and the new worker might have
..	..	@@ -2024,7 +2155,7 @@
2024	2155	* and may_start_working() is true.
2025	2156	*
2026	2157	* CONTEXT:
2027		- * spin_lock_irq(pool->lock) which may be released and regrabbed
	2158	+ * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2028	2159	* multiple times. Does GFP_KERNEL allocations.
2029	2160	*
2030	2161	* Return:
..	..	@@ -2047,7 +2178,7 @@
2047	2178
2048	2179	pool->manager = NULL;
2049	2180	pool->flags &= ~POOL_MANAGER_ACTIVE;
2050		- wake_up(&wq_manager_wait);
	2181	+ rcuwait_wake_up(&manager_wait);
2051	2182	return true;
2052	2183	}
2053	2184
..	..	@@ -2063,7 +2194,7 @@
2063	2194	* call this function to process a work.
2064	2195	*
2065	2196	* CONTEXT:
2066		- * spin_lock_irq(pool->lock) which is released and regrabbed.
	2197	+ * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
2067	2198	*/
2068	2199	static void process_one_work(struct worker worker, struct work_struct work)
2069	2200	__releases(&pool->lock)
..	..	@@ -2145,7 +2276,7 @@
2145	2276	*/
2146	2277	set_work_pool_and_clear_pending(work, pool->id);
2147	2278
2148		- spin_unlock_irq(&pool->lock);
	2279	+ raw_spin_unlock_irq(&pool->lock);
2149	2280
2150	2281	lock_map_acquire(&pwq->wq->lockdep_map);
2151	2282	lock_map_acquire(&lockdep_map);
..	..	@@ -2177,13 +2308,13 @@
2177	2308	* While we must be careful to not use "work" after this, the trace
2178	2309	* point will only record its address.
2179	2310	*/
2180		- trace_workqueue_execute_end(work);
	2311	+ trace_workqueue_execute_end(work, worker->current_func);
2181	2312	lock_map_release(&lockdep_map);
2182	2313	lock_map_release(&pwq->wq->lockdep_map);
2183	2314
2184	2315	if (unlikely(in_atomic() \|\| lockdep_depth(current) > 0)) {
2185	2316	pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2186		- " last function: %pf\n",
	2317	+ " last function: %ps\n",
2187	2318	current->comm, preempt_count(), task_pid_nr(current),
2188	2319	worker->current_func);
2189	2320	debug_show_held_locks(current);
..	..	@@ -2191,7 +2322,7 @@
2191	2322	}
2192	2323
2193	2324	/*
2194		- * The following prevents a kworker from hogging CPU on !PREEMPT
	2325	+ * The following prevents a kworker from hogging CPU on !PREEMPTION
2195	2326	* kernels, where a requeueing work item waiting for something to
2196	2327	* happen could deadlock with stop_machine as such work item could
2197	2328	* indefinitely requeue itself while all other CPUs are trapped in
..	..	@@ -2200,7 +2331,7 @@
2200	2331	*/
2201	2332	cond_resched();
2202	2333
2203		- spin_lock_irq(&pool->lock);
	2334	+ raw_spin_lock_irq(&pool->lock);
2204	2335
2205	2336	/* clear cpu intensive status */
2206	2337	if (unlikely(cpu_intensive))
..	..	@@ -2226,7 +2357,7 @@
2226	2357	* fetches a work from the top and executes it.
2227	2358	*
2228	2359	* CONTEXT:
2229		- * spin_lock_irq(pool->lock) which may be released and regrabbed
	2360	+ * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2230	2361	* multiple times.
2231	2362	*/
2232	2363	static void process_scheduled_works(struct worker *worker)
..	..	@@ -2268,11 +2399,11 @@
2268	2399	/* tell the scheduler that this is a workqueue worker */
2269	2400	set_pf_worker(true);
2270	2401	woke_up:
2271		- spin_lock_irq(&pool->lock);
	2402	+ raw_spin_lock_irq(&pool->lock);
2272	2403
2273	2404	/* am I supposed to die? */
2274	2405	if (unlikely(worker->flags & WORKER_DIE)) {
2275		- spin_unlock_irq(&pool->lock);
	2406	+ raw_spin_unlock_irq(&pool->lock);
2276	2407	WARN_ON_ONCE(!list_empty(&worker->entry));
2277	2408	set_pf_worker(false);
2278	2409
..	..	@@ -2338,7 +2469,7 @@
2338	2469	*/
2339	2470	worker_enter_idle(worker);
2340	2471	__set_current_state(TASK_IDLE);
2341		- spin_unlock_irq(&pool->lock);
	2472	+ raw_spin_unlock_irq(&pool->lock);
2342	2473	schedule();
2343	2474	goto woke_up;
2344	2475	}
..	..	@@ -2392,7 +2523,7 @@
2392	2523	should_stop = kthread_should_stop();
2393	2524
2394	2525	/* see whether any pwq is asking for help */
2395		- spin_lock_irq(&wq_mayday_lock);
	2526	+ raw_spin_lock_irq(&wq_mayday_lock);
2396	2527
2397	2528	while (!list_empty(&wq->maydays)) {
2398	2529	struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
..	..	@@ -2404,11 +2535,11 @@
2404	2535	__set_current_state(TASK_RUNNING);
2405	2536	list_del_init(&pwq->mayday_node);
2406	2537
2407		- spin_unlock_irq(&wq_mayday_lock);
	2538	+ raw_spin_unlock_irq(&wq_mayday_lock);
2408	2539
2409	2540	worker_attach_to_pool(rescuer, pool);
2410	2541
2411		- spin_lock_irq(&pool->lock);
	2542	+ raw_spin_lock_irq(&pool->lock);
2412	2543
2413	2544	/*
2414	2545	* Slurp in all works issued via this workqueue and
..	..	@@ -2436,8 +2567,8 @@
2436	2567	* being used to relieve memory pressure, don't
2437	2568	* incur MAYDAY_INTERVAL delay inbetween.
2438	2569	*/
2439		- if (need_to_create_worker(pool)) {
2440		- spin_lock(&wq_mayday_lock);
	2570	+ if (pwq->nr_active && need_to_create_worker(pool)) {
	2571	+ raw_spin_lock(&wq_mayday_lock);
2441	2572	/*
2442	2573	* Queue iff we aren't racing destruction
2443	2574	* and somebody else hasn't queued it already.
..	..	@@ -2446,7 +2577,7 @@
2446	2577	get_pwq(pwq);
2447	2578	list_add_tail(&pwq->mayday_node, &wq->maydays);
2448	2579	}
2449		- spin_unlock(&wq_mayday_lock);
	2580	+ raw_spin_unlock(&wq_mayday_lock);
2450	2581	}
2451	2582	}
2452	2583
..	..	@@ -2464,14 +2595,14 @@
2464	2595	if (need_more_worker(pool))
2465	2596	wake_up_worker(pool);
2466	2597
2467		- spin_unlock_irq(&pool->lock);
	2598	+ raw_spin_unlock_irq(&pool->lock);
2468	2599
2469	2600	worker_detach_from_pool(rescuer);
2470	2601
2471		- spin_lock_irq(&wq_mayday_lock);
	2602	+ raw_spin_lock_irq(&wq_mayday_lock);
2472	2603	}
2473	2604
2474		- spin_unlock_irq(&wq_mayday_lock);
	2605	+ raw_spin_unlock_irq(&wq_mayday_lock);
2475	2606
2476	2607	if (should_stop) {
2477	2608	__set_current_state(TASK_RUNNING);
..	..	@@ -2508,11 +2639,11 @@
2508	2639	worker = current_wq_worker();
2509	2640
2510	2641	WARN_ONCE(current->flags & PF_MEMALLOC,
2511		- "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%pf",
	2642	+ "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
2512	2643	current->pid, current->comm, target_wq->name, target_func);
2513	2644	WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
2514	2645	(WQ_MEM_RECLAIM \| __WQ_LEGACY)) == WQ_MEM_RECLAIM),
2515		- "workqueue: WQ_MEM_RECLAIM %s:%pf is flushing !WQ_MEM_RECLAIM %s:%pf",
	2646	+ "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
2516	2647	worker->current_pwq->wq->name, worker->current_func,
2517	2648	target_wq->name, target_func);
2518	2649	}
..	..	@@ -2551,7 +2682,7 @@
2551	2682	* underneath us, so we can't reliably determine pwq from @target.
2552	2683	*
2553	2684	* CONTEXT:
2554		- * spin_lock_irq(pool->lock).
	2685	+ * raw_spin_lock_irq(pool->lock).
2555	2686	*/
2556	2687	static void insert_wq_barrier(struct pool_workqueue *pwq,
2557	2688	struct wq_barrier *barr,
..	..	@@ -2638,7 +2769,7 @@
2638	2769	for_each_pwq(pwq, wq) {
2639	2770	struct worker_pool *pool = pwq->pool;
2640	2771
2641		- spin_lock_irq(&pool->lock);
	2772	+ raw_spin_lock_irq(&pool->lock);
2642	2773
2643	2774	if (flush_color >= 0) {
2644	2775	WARN_ON_ONCE(pwq->flush_color != -1);
..	..	@@ -2655,7 +2786,7 @@
2655	2786	pwq->work_color = work_color;
2656	2787	}
2657	2788
2658		- spin_unlock_irq(&pool->lock);
	2789	+ raw_spin_unlock_irq(&pool->lock);
2659	2790	}
2660	2791
2661	2792	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
..	..	@@ -2743,7 +2874,7 @@
2743	2874	* First flushers are responsible for cascading flushes and
2744	2875	* handling overflow. Non-first flushers can simply return.
2745	2876	*/
2746		- if (wq->first_flusher != &this_flusher)
	2877	+ if (READ_ONCE(wq->first_flusher) != &this_flusher)
2747	2878	return;
2748	2879
2749	2880	mutex_lock(&wq->mutex);
..	..	@@ -2752,7 +2883,7 @@
2752	2883	if (wq->first_flusher != &this_flusher)
2753	2884	goto out_unlock;
2754	2885
2755		- wq->first_flusher = NULL;
	2886	+ WRITE_ONCE(wq->first_flusher, NULL);
2756	2887
2757	2888	WARN_ON_ONCE(!list_empty(&this_flusher.list));
2758	2889	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
..	..	@@ -2855,9 +2986,9 @@
2855	2986	for_each_pwq(pwq, wq) {
2856	2987	bool drained;
2857	2988
2858		- spin_lock_irq(&pwq->pool->lock);
	2989	+ raw_spin_lock_irq(&pwq->pool->lock);
2859	2990	drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
2860		- spin_unlock_irq(&pwq->pool->lock);
	2991	+ raw_spin_unlock_irq(&pwq->pool->lock);
2861	2992
2862	2993	if (drained)
2863	2994	continue;
..	..	@@ -2886,14 +3017,14 @@
2886	3017
2887	3018	might_sleep();
2888	3019
2889		- local_irq_disable();
	3020	+ rcu_read_lock();
2890	3021	pool = get_work_pool(work);
2891	3022	if (!pool) {
2892		- local_irq_enable();
	3023	+ rcu_read_unlock();
2893	3024	return false;
2894	3025	}
2895	3026
2896		- spin_lock(&pool->lock);
	3027	+ raw_spin_lock_irq(&pool->lock);
2897	3028	/* see the comment in try_to_grab_pending() with the same code */
2898	3029	pwq = get_work_pwq(work);
2899	3030	if (pwq) {
..	..	@@ -2909,7 +3040,7 @@
2909	3040	check_flush_dependency(pwq->wq, work);
2910	3041
2911	3042	insert_wq_barrier(pwq, barr, work, worker);
2912		- spin_unlock_irq(&pool->lock);
	3043	+ raw_spin_unlock_irq(&pool->lock);
2913	3044
2914	3045	/*
2915	3046	* Force a lock recursion deadlock when using flush_work() inside a
..	..	@@ -2925,10 +3056,11 @@
2925	3056	lock_map_acquire(&pwq->wq->lockdep_map);
2926	3057	lock_map_release(&pwq->wq->lockdep_map);
2927	3058	}
2928		-
	3059	+ rcu_read_unlock();
2929	3060	return true;
2930	3061	already_gone:
2931		- spin_unlock_irq(&pool->lock);
	3062	+ raw_spin_unlock_irq(&pool->lock);
	3063	+ rcu_read_unlock();
2932	3064	return false;
2933	3065	}
2934	3066
..	..	@@ -2942,10 +3074,8 @@
2942	3074	if (WARN_ON(!work->func))
2943	3075	return false;
2944	3076
2945		- if (!from_cancel) {
2946		- lock_map_acquire(&work->lockdep_map);
2947		- lock_map_release(&work->lockdep_map);
2948		- }
	3077	+ lock_map_acquire(&work->lockdep_map);
	3078	+ lock_map_release(&work->lockdep_map);
2949	3079
2950	3080	if (start_flush_work(work, &barr, from_cancel)) {
2951	3081	wait_for_completion(&barr.done);
..	..	@@ -3250,21 +3380,20 @@
3250	3380
3251	3381	/**
3252	3382	* alloc_workqueue_attrs - allocate a workqueue_attrs
3253		- * @gfp_mask: allocation mask to use
3254	3383	*
3255	3384	* Allocate a new workqueue_attrs, initialize with default settings and
3256	3385	* return it.
3257	3386	*
3258	3387	* Return: The allocated new workqueue_attr on success. %NULL on failure.
3259	3388	*/
3260		-struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
	3389	+struct workqueue_attrs *alloc_workqueue_attrs(void)
3261	3390	{
3262	3391	struct workqueue_attrs *attrs;
3263	3392
3264		- attrs = kzalloc(sizeof(*attrs), gfp_mask);
	3393	+ attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
3265	3394	if (!attrs)
3266	3395	goto fail;
3267		- if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
	3396	+ if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))
3268	3397	goto fail;
3269	3398
3270	3399	cpumask_copy(attrs->cpumask, cpu_possible_mask);
..	..	@@ -3321,7 +3450,7 @@
3321	3450	*/
3322	3451	static int init_worker_pool(struct worker_pool *pool)
3323	3452	{
3324		- spin_lock_init(&pool->lock);
	3453	+ raw_spin_lock_init(&pool->lock);
3325	3454	pool->id = -1;
3326	3455	pool->cpu = -1;
3327	3456	pool->node = NUMA_NO_NODE;
..	..	@@ -3342,23 +3471,62 @@
3342	3471	pool->refcnt = 1;
3343	3472
3344	3473	/* shouldn't fail above this point */
3345		- pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
	3474	+ pool->attrs = alloc_workqueue_attrs();
3346	3475	if (!pool->attrs)
3347	3476	return -ENOMEM;
3348	3477	return 0;
3349	3478	}
	3479	+
	3480	+#ifdef CONFIG_LOCKDEP
	3481	+static void wq_init_lockdep(struct workqueue_struct *wq)
	3482	+{
	3483	+ char *lock_name;
	3484	+
	3485	+ lockdep_register_key(&wq->key);
	3486	+ lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
	3487	+ if (!lock_name)
	3488	+ lock_name = wq->name;
	3489	+
	3490	+ wq->lock_name = lock_name;
	3491	+ lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0);
	3492	+}
	3493	+
	3494	+static void wq_unregister_lockdep(struct workqueue_struct *wq)
	3495	+{
	3496	+ lockdep_unregister_key(&wq->key);
	3497	+}
	3498	+
	3499	+static void wq_free_lockdep(struct workqueue_struct *wq)
	3500	+{
	3501	+ if (wq->lock_name != wq->name)
	3502	+ kfree(wq->lock_name);
	3503	+}
	3504	+#else
	3505	+static void wq_init_lockdep(struct workqueue_struct *wq)
	3506	+{
	3507	+}
	3508	+
	3509	+static void wq_unregister_lockdep(struct workqueue_struct *wq)
	3510	+{
	3511	+}
	3512	+
	3513	+static void wq_free_lockdep(struct workqueue_struct *wq)
	3514	+{
	3515	+}
	3516	+#endif
3350	3517
3351	3518	static void rcu_free_wq(struct rcu_head *rcu)
3352	3519	{
3353	3520	struct workqueue_struct *wq =
3354	3521	container_of(rcu, struct workqueue_struct, rcu);
3355	3522
	3523	+ wq_free_lockdep(wq);
	3524	+
3356	3525	if (!(wq->flags & WQ_UNBOUND))
3357	3526	free_percpu(wq->cpu_pwqs);
3358	3527	else
3359	3528	free_workqueue_attrs(wq->unbound_attrs);
3360	3529
3361		- kfree(wq->rescuer);
3362	3530	kfree(wq);
3363	3531	}
3364	3532
..	..	@@ -3371,11 +3539,23 @@
3371	3539	kfree(pool);
3372	3540	}
3373	3541
	3542	+/* This returns with the lock held on success (pool manager is inactive). */
	3543	+static bool wq_manager_inactive(struct worker_pool *pool)
	3544	+{
	3545	+ raw_spin_lock_irq(&pool->lock);
	3546	+
	3547	+ if (pool->flags & POOL_MANAGER_ACTIVE) {
	3548	+ raw_spin_unlock_irq(&pool->lock);
	3549	+ return false;
	3550	+ }
	3551	+ return true;
	3552	+}
	3553	+
3374	3554	/**
3375	3555	* put_unbound_pool - put a worker_pool
3376	3556	* @pool: worker_pool to put
3377	3557	*
3378		- * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU
	3558	+ * Put @pool. If its refcnt reaches zero, it gets destroyed in RCU
3379	3559	* safe manner. get_unbound_pool() calls this function on its failure path
3380	3560	* and this function should be able to release pools which went through,
3381	3561	* successfully or not, init_worker_pool().
..	..	@@ -3406,16 +3586,17 @@
3406	3586	* Become the manager and destroy all workers. This prevents
3407	3587	* @pool's workers from blocking on attach_mutex. We're the last
3408	3588	* manager and @pool gets freed with the flag set.
	3589	+ * Because of how wq_manager_inactive() works, we will hold the
	3590	+ * spinlock after a successful wait.
3409	3591	*/
3410		- spin_lock_irq(&pool->lock);
3411		- wait_event_lock_irq(wq_manager_wait,
3412		- !(pool->flags & POOL_MANAGER_ACTIVE), pool->lock);
	3592	+ rcuwait_wait_event(&manager_wait, wq_manager_inactive(pool),
	3593	+ TASK_UNINTERRUPTIBLE);
3413	3594	pool->flags \|= POOL_MANAGER_ACTIVE;
3414	3595
3415	3596	while ((worker = first_idle_worker(pool)))
3416	3597	destroy_worker(worker);
3417	3598	WARN_ON(pool->nr_workers \|\| pool->nr_idle);
3418		- spin_unlock_irq(&pool->lock);
	3599	+ raw_spin_unlock_irq(&pool->lock);
3419	3600
3420	3601	mutex_lock(&wq_pool_attach_mutex);
3421	3602	if (!list_empty(&pool->workers))
..	..	@@ -3429,8 +3610,8 @@
3429	3610	del_timer_sync(&pool->idle_timer);
3430	3611	del_timer_sync(&pool->mayday_timer);
3431	3612
3432		- /* sched-RCU protected to allow dereferences from get_work_pool() */
3433		- call_rcu_sched(&pool->rcu, rcu_free_pool);
	3613	+ /* RCU protected to allow dereferences from get_work_pool() */
	3614	+ call_rcu(&pool->rcu, rcu_free_pool);
3434	3615	}
3435	3616
3436	3617	/**
..	..	@@ -3543,14 +3724,16 @@
3543	3724	put_unbound_pool(pool);
3544	3725	mutex_unlock(&wq_pool_mutex);
3545	3726
3546		- call_rcu_sched(&pwq->rcu, rcu_free_pwq);
	3727	+ call_rcu(&pwq->rcu, rcu_free_pwq);
3547	3728
3548	3729	/*
3549	3730	* If we're the last pwq going away, @wq is already dead and no one
3550	3731	* is gonna access it anymore. Schedule RCU free.
3551	3732	*/
3552		- if (is_last)
3553		- call_rcu_sched(&wq->rcu, rcu_free_wq);
	3733	+ if (is_last) {
	3734	+ wq_unregister_lockdep(wq);
	3735	+ call_rcu(&wq->rcu, rcu_free_wq);
	3736	+ }
3554	3737	}
3555	3738
3556	3739	/**
..	..	@@ -3575,7 +3758,7 @@
3575	3758	return;
3576	3759
3577	3760	/* this function can be called during early boot w/ irq disabled */
3578		- spin_lock_irqsave(&pwq->pool->lock, flags);
	3761	+ raw_spin_lock_irqsave(&pwq->pool->lock, flags);
3579	3762
3580	3763	/*
3581	3764	* During [un]freezing, the caller is responsible for ensuring that
..	..	@@ -3605,7 +3788,7 @@
3605	3788	pwq->max_active = 0;
3606	3789	}
3607	3790
3608		- spin_unlock_irqrestore(&pwq->pool->lock, flags);
	3791	+ raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
3609	3792	}
3610	3793
3611	3794	/* initialize newly alloced @pwq which is associated with @wq and @pool */
..	..	@@ -3778,8 +3961,8 @@
3778	3961
3779	3962	ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_node_ids), GFP_KERNEL);
3780	3963
3781		- new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3782		- tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);
	3964	+ new_attrs = alloc_workqueue_attrs();
	3965	+ tmp_attrs = alloc_workqueue_attrs();
3783	3966	if (!ctx \|\| !new_attrs \|\| !tmp_attrs)
3784	3967	goto out_free;
3785	3968
..	..	@@ -3913,6 +4096,8 @@
3913	4096	*
3914	4097	* Performs GFP_KERNEL allocations.
3915	4098	*
	4099	+ * Assumes caller has CPU hotplug read exclusion, i.e. get_online_cpus().
	4100	+ *
3916	4101	* Return: 0 on success and -errno on failure.
3917	4102	*/
3918	4103	int apply_workqueue_attrs(struct workqueue_struct *wq,
..	..	@@ -3920,13 +4105,14 @@
3920	4105	{
3921	4106	int ret;
3922	4107
3923		- apply_wqattrs_lock();
	4108	+ lockdep_assert_cpus_held();
	4109	+
	4110	+ mutex_lock(&wq_pool_mutex);
3924	4111	ret = apply_workqueue_attrs_locked(wq, attrs);
3925		- apply_wqattrs_unlock();
	4112	+ mutex_unlock(&wq_pool_mutex);
3926	4113
3927	4114	return ret;
3928	4115	}
3929		-EXPORT_SYMBOL_GPL(apply_workqueue_attrs);
3930	4116
3931	4117	/**
3932	4118	* wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
..	..	@@ -4004,9 +4190,9 @@
4004	4190
4005	4191	use_dfl_pwq:
4006	4192	mutex_lock(&wq->mutex);
4007		- spin_lock_irq(&wq->dfl_pwq->pool->lock);
	4193	+ raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
4008	4194	get_pwq(wq->dfl_pwq);
4009		- spin_unlock_irq(&wq->dfl_pwq->pool->lock);
	4195	+ raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4010	4196	old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4011	4197	out_unlock:
4012	4198	mutex_unlock(&wq->mutex);
..	..	@@ -4036,16 +4222,21 @@
4036	4222	mutex_unlock(&wq->mutex);
4037	4223	}
4038	4224	return 0;
4039		- } else if (wq->flags & __WQ_ORDERED) {
	4225	+ }
	4226	+
	4227	+ get_online_cpus();
	4228	+ if (wq->flags & __WQ_ORDERED) {
4040	4229	ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
4041	4230	/* there should only be single pwq for ordering guarantee */
4042	4231	WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node \|\|
4043	4232	wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
4044	4233	"ordering guarantee broken for workqueue %s\n", wq->name);
4045		- return ret;
4046	4234	} else {
4047		- return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
	4235	+ ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
4048	4236	}
	4237	+ put_online_cpus();
	4238	+
	4239	+ return ret;
4049	4240	}
4050	4241
4051	4242	static int wq_clamp_max_active(int max_active, unsigned int flags,
..	..	@@ -4078,8 +4269,8 @@
4078	4269
4079	4270	rescuer->rescue_wq = wq;
4080	4271	rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name);
4081		- ret = PTR_ERR_OR_ZERO(rescuer->task);
4082		- if (ret) {
	4272	+ if (IS_ERR(rescuer->task)) {
	4273	+ ret = PTR_ERR(rescuer->task);
4083	4274	kfree(rescuer);
4084	4275	return ret;
4085	4276	}
..	..	@@ -4091,11 +4282,10 @@
4091	4282	return 0;
4092	4283	}
4093	4284
4094		-struct workqueue_struct __alloc_workqueue_key(const char fmt,
4095		- unsigned int flags,
4096		- int max_active,
4097		- struct lock_class_key *key,
4098		- const char *lock_name, ...)
	4285	+__printf(1, 4)
	4286	+struct workqueue_struct alloc_workqueue(const char fmt,
	4287	+ unsigned int flags,
	4288	+ int max_active, ...)
4099	4289	{
4100	4290	size_t tbl_size = 0;
4101	4291	va_list args;
..	..	@@ -4125,12 +4315,12 @@
4125	4315	return NULL;
4126	4316
4127	4317	if (flags & WQ_UNBOUND) {
4128		- wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);
	4318	+ wq->unbound_attrs = alloc_workqueue_attrs();
4129	4319	if (!wq->unbound_attrs)
4130	4320	goto err_free_wq;
4131	4321	}
4132	4322
4133		- va_start(args, lock_name);
	4323	+ va_start(args, max_active);
4134	4324	vsnprintf(wq->name, sizeof(wq->name), fmt, args);
4135	4325	va_end(args);
4136	4326
..	..	@@ -4147,11 +4337,11 @@
4147	4337	INIT_LIST_HEAD(&wq->flusher_overflow);
4148	4338	INIT_LIST_HEAD(&wq->maydays);
4149	4339
4150		- lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
	4340	+ wq_init_lockdep(wq);
4151	4341	INIT_LIST_HEAD(&wq->list);
4152	4342
4153	4343	if (alloc_and_link_pwqs(wq) < 0)
4154		- goto err_free_wq;
	4344	+ goto err_unreg_lockdep;
4155	4345
4156	4346	if (wq_online && init_rescuer(wq) < 0)
4157	4347	goto err_destroy;
..	..	@@ -4177,6 +4367,9 @@
4177	4367
4178	4368	return wq;
4179	4369
	4370	+err_unreg_lockdep:
	4371	+ wq_unregister_lockdep(wq);
	4372	+ wq_free_lockdep(wq);
4180	4373	err_free_wq:
4181	4374	free_workqueue_attrs(wq->unbound_attrs);
4182	4375	kfree(wq);
..	..	@@ -4185,7 +4378,23 @@
4185	4378	destroy_workqueue(wq);
4186	4379	return NULL;
4187	4380	}
4188		-EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
	4381	+EXPORT_SYMBOL_GPL(alloc_workqueue);
	4382	+
	4383	+static bool pwq_busy(struct pool_workqueue *pwq)
	4384	+{
	4385	+ int i;
	4386	+
	4387	+ for (i = 0; i < WORK_NR_COLORS; i++)
	4388	+ if (pwq->nr_in_flight[i])
	4389	+ return true;
	4390	+
	4391	+ if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))
	4392	+ return true;
	4393	+ if (pwq->nr_active \|\| !list_empty(&pwq->delayed_works))
	4394	+ return true;
	4395	+
	4396	+ return false;
	4397	+}
4189	4398
4190	4399	/**
4191	4400	* destroy_workqueue - safely terminate a workqueue
..	..	@@ -4212,35 +4421,34 @@
4212	4421	struct worker *rescuer = wq->rescuer;
4213	4422
4214	4423	/* this prevents new queueing */
4215		- spin_lock_irq(&wq_mayday_lock);
	4424	+ raw_spin_lock_irq(&wq_mayday_lock);
4216	4425	wq->rescuer = NULL;
4217		- spin_unlock_irq(&wq_mayday_lock);
	4426	+ raw_spin_unlock_irq(&wq_mayday_lock);
4218	4427
4219	4428	/* rescuer will empty maydays list before exiting */
4220	4429	kthread_stop(rescuer->task);
4221	4430	kfree(rescuer);
4222	4431	}
4223	4432
4224		- /* sanity checks */
	4433	+ /*
	4434	+ * Sanity checks - grab all the locks so that we wait for all
	4435	+ * in-flight operations which may do put_pwq().
	4436	+ */
	4437	+ mutex_lock(&wq_pool_mutex);
4225	4438	mutex_lock(&wq->mutex);
4226	4439	for_each_pwq(pwq, wq) {
4227		- int i;
4228		-
4229		- for (i = 0; i < WORK_NR_COLORS; i++) {
4230		- if (WARN_ON(pwq->nr_in_flight[i])) {
4231		- mutex_unlock(&wq->mutex);
4232		- show_workqueue_state();
4233		- return;
4234		- }
4235		- }
4236		-
4237		- if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) \|\|
4238		- WARN_ON(pwq->nr_active) \|\|
4239		- WARN_ON(!list_empty(&pwq->delayed_works))) {
	4440	+ raw_spin_lock_irq(&pwq->pool->lock);
	4441	+ if (WARN_ON(pwq_busy(pwq))) {
	4442	+ pr_warn("%s: %s has the following busy pwq\n",
	4443	+ __func__, wq->name);
	4444	+ show_pwq(pwq);
	4445	+ raw_spin_unlock_irq(&pwq->pool->lock);
4240	4446	mutex_unlock(&wq->mutex);
	4447	+ mutex_unlock(&wq_pool_mutex);
4241	4448	show_workqueue_state();
4242	4449	return;
4243	4450	}
	4451	+ raw_spin_unlock_irq(&pwq->pool->lock);
4244	4452	}
4245	4453	mutex_unlock(&wq->mutex);
4246	4454
..	..	@@ -4248,16 +4456,16 @@
4248	4456	* wq list is used to freeze wq, remove from list after
4249	4457	* flushing is complete in case freeze races us.
4250	4458	*/
4251		- mutex_lock(&wq_pool_mutex);
4252	4459	list_del_rcu(&wq->list);
4253	4460	mutex_unlock(&wq_pool_mutex);
4254	4461
4255	4462	if (!(wq->flags & WQ_UNBOUND)) {
	4463	+ wq_unregister_lockdep(wq);
4256	4464	/*
4257	4465	* The base ref is never dropped on per-cpu pwqs. Directly
4258	4466	* schedule RCU free.
4259	4467	*/
4260		- call_rcu_sched(&wq->rcu, rcu_free_wq);
	4468	+ call_rcu(&wq->rcu, rcu_free_wq);
4261	4469	} else {
4262	4470	/*
4263	4471	* We're the sole accessor of @wq at this point. Directly
..	..	@@ -4367,7 +4575,8 @@
4367	4575	struct pool_workqueue *pwq;
4368	4576	bool ret;
4369	4577
4370		- rcu_read_lock_sched();
	4578	+ rcu_read_lock();
	4579	+ preempt_disable();
4371	4580
4372	4581	if (cpu == WORK_CPU_UNBOUND)
4373	4582	cpu = smp_processor_id();
..	..	@@ -4378,7 +4587,8 @@
4378	4587	pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4379	4588
4380	4589	ret = !list_empty(&pwq->delayed_works);
4381		- rcu_read_unlock_sched();
	4590	+ preempt_enable();
	4591	+ rcu_read_unlock();
4382	4592
4383	4593	return ret;
4384	4594	}
..	..	@@ -4404,15 +4614,15 @@
4404	4614	if (work_pending(work))
4405	4615	ret \|= WORK_BUSY_PENDING;
4406	4616
4407		- local_irq_save(flags);
	4617	+ rcu_read_lock();
4408	4618	pool = get_work_pool(work);
4409	4619	if (pool) {
4410		- spin_lock(&pool->lock);
	4620	+ raw_spin_lock_irqsave(&pool->lock, flags);
4411	4621	if (find_worker_executing_work(pool, work))
4412	4622	ret \|= WORK_BUSY_RUNNING;
4413		- spin_unlock(&pool->lock);
	4623	+ raw_spin_unlock_irqrestore(&pool->lock, flags);
4414	4624	}
4415		- local_irq_restore(flags);
	4625	+ rcu_read_unlock();
4416	4626
4417	4627	return ret;
4418	4628	}
..	..	@@ -4476,14 +4686,14 @@
4476	4686	* Carefully copy the associated workqueue's workfn, name and desc.
4477	4687	* Keep the original last '\0' in case the original is garbage.
4478	4688	*/
4479		- probe_kernel_read(&fn, &worker->current_func, sizeof(fn));
4480		- probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));
4481		- probe_kernel_read(&wq, &pwq->wq, sizeof(wq));
4482		- probe_kernel_read(name, wq->name, sizeof(name) - 1);
4483		- probe_kernel_read(desc, worker->desc, sizeof(desc) - 1);
	4689	+ copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn));
	4690	+ copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
	4691	+ copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
	4692	+ copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
	4693	+ copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1);
4484	4694
4485	4695	if (fn \|\| name[0] \|\| desc[0]) {
4486		- printk("%sWorkqueue: %s %pf", log_lvl, name, fn);
	4696	+ printk("%sWorkqueue: %s %ps", log_lvl, name, fn);
4487	4697	if (strcmp(name, desc))
4488	4698	pr_cont(" (%s)", desc);
4489	4699	pr_cont("\n");
..	..	@@ -4508,7 +4718,7 @@
4508	4718	pr_cont("%s BAR(%d)", comma ? "," : "",
4509	4719	task_pid_nr(barr->task));
4510	4720	} else {
4511		- pr_cont("%s %pf", comma ? "," : "", work->func);
	4721	+ pr_cont("%s %ps", comma ? "," : "", work->func);
4512	4722	}
4513	4723	}
4514	4724
..	..	@@ -4541,9 +4751,9 @@
4541	4751	if (worker->current_pwq != pwq)
4542	4752	continue;
4543	4753
4544		- pr_cont("%s %d%s:%pf", comma ? "," : "",
	4754	+ pr_cont("%s %d%s:%ps", comma ? "," : "",
4545	4755	task_pid_nr(worker->task),
4546		- worker == pwq->wq->rescuer ? "(RESCUER)" : "",
	4756	+ worker->rescue_wq ? "(RESCUER)" : "",
4547	4757	worker->current_func);
4548	4758	list_for_each_entry(work, &worker->scheduled, entry)
4549	4759	pr_cont_work(false, work);
..	..	@@ -4597,7 +4807,7 @@
4597	4807	unsigned long flags;
4598	4808	int pi;
4599	4809
4600		- rcu_read_lock_sched();
	4810	+ rcu_read_lock();
4601	4811
4602	4812	pr_info("Showing busy workqueues and worker pools:\n");
4603	4813
..	..	@@ -4617,10 +4827,10 @@
4617	4827	pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
4618	4828
4619	4829	for_each_pwq(pwq, wq) {
4620		- spin_lock_irqsave(&pwq->pool->lock, flags);
	4830	+ raw_spin_lock_irqsave(&pwq->pool->lock, flags);
4621	4831	if (pwq->nr_active \|\| !list_empty(&pwq->delayed_works))
4622	4832	show_pwq(pwq);
4623		- spin_unlock_irqrestore(&pwq->pool->lock, flags);
	4833	+ raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
4624	4834	/*
4625	4835	* We could be printing a lot from atomic context, e.g.
4626	4836	* sysrq-t -> show_workqueue_state(). Avoid triggering
..	..	@@ -4634,7 +4844,7 @@
4634	4844	struct worker *worker;
4635	4845	bool first = true;
4636	4846
4637		- spin_lock_irqsave(&pool->lock, flags);
	4847	+ raw_spin_lock_irqsave(&pool->lock, flags);
4638	4848	if (pool->nr_workers == pool->nr_idle)
4639	4849	goto next_pool;
4640	4850
..	..	@@ -4653,7 +4863,7 @@
4653	4863	}
4654	4864	pr_cont("\n");
4655	4865	next_pool:
4656		- spin_unlock_irqrestore(&pool->lock, flags);
	4866	+ raw_spin_unlock_irqrestore(&pool->lock, flags);
4657	4867	/*
4658	4868	* We could be printing a lot from atomic context, e.g.
4659	4869	* sysrq-t -> show_workqueue_state(). Avoid triggering
..	..	@@ -4662,7 +4872,7 @@
4662	4872	touch_nmi_watchdog();
4663	4873	}
4664	4874
4665		- rcu_read_unlock_sched();
	4875	+ rcu_read_unlock();
4666	4876	}
4667	4877
4668	4878	/* used to show worker information through /proc/PID/{comm,stat,status} */
..	..	@@ -4683,7 +4893,7 @@
4683	4893	struct worker_pool *pool = worker->pool;
4684	4894
4685	4895	if (pool) {
4686		- spin_lock_irq(&pool->lock);
	4896	+ raw_spin_lock_irq(&pool->lock);
4687	4897	/*
4688	4898	* ->desc tracks information (wq name or
4689	4899	* set_worker_desc()) for the latest execution. If
..	..	@@ -4697,12 +4907,13 @@
4697	4907	scnprintf(buf + off, size - off, "-%s",
4698	4908	worker->desc);
4699	4909	}
4700		- spin_unlock_irq(&pool->lock);
	4910	+ raw_spin_unlock_irq(&pool->lock);
4701	4911	}
4702	4912	}
4703	4913
4704	4914	mutex_unlock(&wq_pool_attach_mutex);
4705	4915	}
	4916	+EXPORT_SYMBOL_GPL(wq_worker_comm);
4706	4917
4707	4918	#ifdef CONFIG_SMP
4708	4919
..	..	@@ -4728,7 +4939,7 @@
4728	4939
4729	4940	for_each_cpu_worker_pool(pool, cpu) {
4730	4941	mutex_lock(&wq_pool_attach_mutex);
4731		- spin_lock_irq(&pool->lock);
	4942	+ raw_spin_lock_irq(&pool->lock);
4732	4943
4733	4944	/*
4734	4945	* We've blocked all attach/detach operations. Make all workers
..	..	@@ -4742,7 +4953,7 @@
4742	4953
4743	4954	pool->flags \|= POOL_DISASSOCIATED;
4744	4955
4745		- spin_unlock_irq(&pool->lock);
	4956	+ raw_spin_unlock_irq(&pool->lock);
4746	4957	mutex_unlock(&wq_pool_attach_mutex);
4747	4958
4748	4959	/*
..	..	@@ -4768,9 +4979,9 @@
4768	4979	* worker blocking could lead to lengthy stalls. Kick off
4769	4980	* unbound chain execution of currently pending work items.
4770	4981	*/
4771		- spin_lock_irq(&pool->lock);
	4982	+ raw_spin_lock_irq(&pool->lock);
4772	4983	wake_up_worker(pool);
4773		- spin_unlock_irq(&pool->lock);
	4984	+ raw_spin_unlock_irq(&pool->lock);
4774	4985	}
4775	4986	}
4776	4987
..	..	@@ -4797,7 +5008,7 @@
4797	5008	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4798	5009	pool->attrs->cpumask) < 0);
4799	5010
4800		- spin_lock_irq(&pool->lock);
	5011	+ raw_spin_lock_irq(&pool->lock);
4801	5012
4802	5013	pool->flags &= ~POOL_DISASSOCIATED;
4803	5014
..	..	@@ -4826,7 +5037,7 @@
4826	5037	*
4827	5038	* WRITE_ONCE() is necessary because @worker->flags may be
4828	5039	* tested without holding any lock in
4829		- * wq_worker_waking_up(). Without it, NOT_RUNNING test may
	5040	+ * wq_worker_running(). Without it, NOT_RUNNING test may
4830	5041	* fail incorrectly leading to premature concurrency
4831	5042	* management operations.
4832	5043	*/
..	..	@@ -4836,7 +5047,7 @@
4836	5047	WRITE_ONCE(worker->flags, worker_flags);
4837	5048	}
4838	5049
4839		- spin_unlock_irq(&pool->lock);
	5050	+ raw_spin_unlock_irq(&pool->lock);
4840	5051	}
4841	5052
4842	5053	/**
..	..	@@ -5049,16 +5260,16 @@
5049	5260	* nr_active is monotonically decreasing. It's safe
5050	5261	* to peek without lock.
5051	5262	*/
5052		- rcu_read_lock_sched();
	5263	+ rcu_read_lock();
5053	5264	for_each_pwq(pwq, wq) {
5054	5265	WARN_ON_ONCE(pwq->nr_active < 0);
5055	5266	if (pwq->nr_active) {
5056	5267	busy = true;
5057		- rcu_read_unlock_sched();
	5268	+ rcu_read_unlock();
5058	5269	goto out_unlock;
5059	5270	}
5060	5271	}
5061		- rcu_read_unlock_sched();
	5272	+ rcu_read_unlock();
5062	5273	}
5063	5274	out_unlock:
5064	5275	mutex_unlock(&wq_pool_mutex);
..	..	@@ -5260,7 +5471,8 @@
5260	5471	const char *delim = "";
5261	5472	int node, written = 0;
5262	5473
5263		- rcu_read_lock_sched();
	5474	+ get_online_cpus();
	5475	+ rcu_read_lock();
5264	5476	for_each_node(node) {
5265	5477	written += scnprintf(buf + written, PAGE_SIZE - written,
5266	5478	"%s%d:%d", delim, node,
..	..	@@ -5268,7 +5480,8 @@
5268	5480	delim = " ";
5269	5481	}
5270	5482	written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
5271		- rcu_read_unlock_sched();
	5483	+ rcu_read_unlock();
	5484	+ put_online_cpus();
5272	5485
5273	5486	return written;
5274	5487	}
..	..	@@ -5293,7 +5506,7 @@
5293	5506
5294	5507	lockdep_assert_held(&wq_pool_mutex);
5295	5508
5296		- attrs = alloc_workqueue_attrs(GFP_KERNEL);
	5509	+ attrs = alloc_workqueue_attrs();
5297	5510	if (!attrs)
5298	5511	return NULL;
5299	5512
..	..	@@ -5639,6 +5852,7 @@
5639	5852	pr_cont_pool_info(pool);
5640	5853	pr_cont(" stuck for %us!\n",
5641	5854	jiffies_to_msecs(now - pool_ts) / 1000);
	5855	+ trace_android_vh_wq_lockup_pool(pool->cpu, pool_ts);
5642	5856	}
5643	5857	}
5644	5858
..	..	@@ -5722,7 +5936,14 @@
5722	5936	return;
5723	5937	}
5724	5938
5725		- wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL);
	5939	+ for_each_possible_cpu(cpu) {
	5940	+ if (WARN_ON(cpu_to_node(cpu) == NUMA_NO_NODE)) {
	5941	+ pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
	5942	+ return;
	5943	+ }
	5944	+ }
	5945	+
	5946	+ wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs();
5726	5947	BUG_ON(!wq_update_unbound_numa_attrs_buf);
5727	5948
5728	5949	/*
..	..	@@ -5739,11 +5960,6 @@
5739	5960
5740	5961	for_each_possible_cpu(cpu) {
5741	5962	node = cpu_to_node(cpu);
5742		- if (WARN_ON(node == NUMA_NO_NODE)) {
5743		- pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
5744		- /* happens iff arch is bonkers, let's just proceed */
5745		- return;
5746		- }
5747	5963	cpumask_set_cpu(cpu, tbl[node]);
5748	5964	}
5749	5965
..	..	@@ -5761,13 +5977,13 @@
5761	5977	* items. Actual work item execution starts only after kthreads can be
5762	5978	* created and scheduled right before early initcalls.
5763	5979	*/
5764		-int __init workqueue_init_early(void)
	5980	+void __init workqueue_init_early(void)
5765	5981	{
5766	5982	int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
5767	5983	int hk_flags = HK_FLAG_DOMAIN \| HK_FLAG_WQ;
5768	5984	int i, cpu;
5769	5985
5770		- WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
	5986	+ BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
5771	5987
5772	5988	BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
5773	5989	cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(hk_flags));
..	..	@@ -5797,7 +6013,7 @@
5797	6013	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
5798	6014	struct workqueue_attrs *attrs;
5799	6015
5800		- BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
	6016	+ BUG_ON(!(attrs = alloc_workqueue_attrs()));
5801	6017	attrs->nice = std_nice[i];
5802	6018	unbound_std_wq_attrs[i] = attrs;
5803	6019
..	..	@@ -5806,7 +6022,7 @@
5806	6022	* guaranteed by max_active which is enforced by pwqs.
5807	6023	* Turn off NUMA so that dfl_pwq is used for all nodes.
5808	6024	*/
5809		- BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
	6025	+ BUG_ON(!(attrs = alloc_workqueue_attrs()));
5810	6026	attrs->nice = std_nice[i];
5811	6027	attrs->no_numa = true;
5812	6028	ordered_wq_attrs[i] = attrs;
..	..	@@ -5828,8 +6044,6 @@
5828	6044	!system_unbound_wq \|\| !system_freezable_wq \|\|
5829	6045	!system_power_efficient_wq \|\|
5830	6046	!system_freezable_power_efficient_wq);
5831		-
5832		- return 0;
5833	6047	}
5834	6048
5835	6049	/**
..	..	@@ -5841,7 +6055,7 @@
5841	6055	* are no kworkers executing the work items yet. Populate the worker pools
5842	6056	* with the initial workers and enable future kworker creations.
5843	6057	*/
5844		-int __init workqueue_init(void)
	6058	+void __init workqueue_init(void)
5845	6059	{
5846	6060	struct workqueue_struct *wq;
5847	6061	struct worker_pool *pool;
..	..	@@ -5888,6 +6102,4 @@
5888	6102
5889	6103	wq_online = true;
5890	6104	wq_watchdog_init();
5891		-
5892		- return 0;
5893	6105	}