~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* kernel/workqueue.c - generic async execution with shared worker pool
3	4	*
..	..	@@ -50,8 +51,13 @@
50	51	#include <linux/sched/isolation.h>
51	52	#include <linux/nmi.h>
52	53	#include <linux/kvm_para.h>
	54	+#include <uapi/linux/sched/types.h>
53	55
54	56	#include "workqueue_internal.h"
	57	+
	58	+#include <trace/hooks/wqlockup.h>
	59	+/* events/workqueue.h uses default TRACE_INCLUDE_PATH */
	60	+#undef TRACE_INCLUDE_PATH
55	61
56	62	enum {
57	63	/*
..	..	@@ -128,16 +134,16 @@
128	134	*
129	135	* PL: wq_pool_mutex protected.
130	136	*
131		- * PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads.
	137	+ * PR: wq_pool_mutex protected for writes. RCU protected for reads.
132	138	*
133	139	* PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
134	140	*
135	141	* PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
136		- * sched-RCU for reads.
	142	+ * RCU for reads.
137	143	*
138	144	* WQ: wq->mutex protected.
139	145	*
140		- * WR: wq->mutex protected for writes. Sched-RCU protected for reads.
	146	+ * WR: wq->mutex protected for writes. RCU protected for reads.
141	147	*
142	148	* MD: wq_mayday_lock protected.
143	149	*/
..	..	@@ -145,7 +151,7 @@
145	151	/* struct worker is defined in workqueue_internal.h */
146	152
147	153	struct worker_pool {
148		- spinlock_t lock; /* the pool lock */
	154	+ raw_spinlock_t lock; /* the pool lock */
149	155	int cpu; /* I: the associated cpu */
150	156	int node; /* I: the associated node ID */
151	157	int id; /* I: pool ID */
..	..	@@ -184,7 +190,7 @@
184	190	atomic_t nr_running ____cacheline_aligned_in_smp;
185	191
186	192	/*
187		- * Destruction of pool is sched-RCU protected to allow dereferences
	193	+ * Destruction of pool is RCU protected to allow dereferences
188	194	* from get_work_pool().
189	195	*/
190	196	struct rcu_head rcu;
..	..	@@ -213,7 +219,7 @@
213	219	/*
214	220	* Release of unbound pwq is punted to system_wq. See put_pwq()
215	221	* and pwq_unbound_release_workfn() for details. pool_workqueue
216		- * itself is also sched-RCU protected so that the first pwq can be
	222	+ * itself is also RCU protected so that the first pwq can be
217	223	* determined without grabbing wq->mutex.
218	224	*/
219	225	struct work_struct unbound_release_work;
..	..	@@ -248,7 +254,7 @@
248	254	struct list_head flusher_overflow; /* WQ: flush overflow list */
249	255
250	256	struct list_head maydays; /* MD: pwqs requesting rescue */
251		- struct worker rescuer; / I: rescue worker */
	257	+ struct worker rescuer; / MD: rescue worker */
252	258
253	259	int nr_drainers; /* WQ: drain in progress */
254	260	int saved_max_active; /* WQ: saved pwq max_active */
..	..	@@ -260,13 +266,15 @@
260	266	struct wq_device wq_dev; / I: for sysfs interface */
261	267	#endif
262	268	#ifdef CONFIG_LOCKDEP
	269	+ char *lock_name;
	270	+ struct lock_class_key key;
263	271	struct lockdep_map lockdep_map;
264	272	#endif
265	273	char name[WQ_NAME_LEN]; /* I: workqueue name */
266	274
267	275	/*
268		- * Destruction of workqueue_struct is sched-RCU protected to allow
269		- * walking the workqueues list without grabbing wq_pool_mutex.
	276	+ * Destruction of workqueue_struct is RCU protected to allow walking
	277	+ * the workqueues list without grabbing wq_pool_mutex.
270	278	* This is used to dump all workqueues from sysrq.
271	279	*/
272	280	struct rcu_head rcu;
..	..	@@ -298,8 +306,9 @@
298	306
299	307	static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
300	308	static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
301		-static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
302		-static DECLARE_WAIT_QUEUE_HEAD(wq_manager_wait); /* wait for manager to go away */
	309	+static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
	310	+/* wait for manager to go away */
	311	+static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);
303	312
304	313	static LIST_HEAD(workqueues); /* PR: list of all workqueues */
305	314	static bool workqueue_freezing; /* PL: have wqs started freezing? */
..	..	@@ -353,25 +362,24 @@
353	362
354	363	static int worker_thread(void *__worker);
355	364	static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
	365	+static void show_pwq(struct pool_workqueue *pwq);
356	366
357	367	#define CREATE_TRACE_POINTS
358	368	#include <trace/events/workqueue.h>
359	369
360		-#define assert_rcu_or_pool_mutex() \
361		- RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
362		- !lockdep_is_held(&wq_pool_mutex), \
363		- "sched RCU or wq_pool_mutex should be held")
	370	+EXPORT_TRACEPOINT_SYMBOL_GPL(workqueue_execute_start);
	371	+EXPORT_TRACEPOINT_SYMBOL_GPL(workqueue_execute_end);
364	372
365		-#define assert_rcu_or_wq_mutex(wq) \
366		- RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
367		- !lockdep_is_held(&wq->mutex), \
368		- "sched RCU or wq->mutex should be held")
	373	+#define assert_rcu_or_pool_mutex() \
	374	+ RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
	375	+ !lockdep_is_held(&wq_pool_mutex), \
	376	+ "RCU or wq_pool_mutex should be held")
369	377
370	378	#define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
371		- RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
	379	+ RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
372	380	!lockdep_is_held(&wq->mutex) && \
373	381	!lockdep_is_held(&wq_pool_mutex), \
374		- "sched RCU, wq->mutex or wq_pool_mutex should be held")
	382	+ "RCU, wq->mutex or wq_pool_mutex should be held")
375	383
376	384	#define for_each_cpu_worker_pool(pool, cpu) \
377	385	for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
..	..	@@ -383,7 +391,7 @@
383	391	* @pool: iteration cursor
384	392	* @pi: integer used for iteration
385	393	*
386		- * This must be called either with wq_pool_mutex held or sched RCU read
	394	+ * This must be called either with wq_pool_mutex held or RCU read
387	395	* locked. If the pool needs to be used beyond the locking in effect, the
388	396	* caller is responsible for guaranteeing that the pool stays online.
389	397	*
..	..	@@ -415,7 +423,7 @@
415	423	* @pwq: iteration cursor
416	424	* @wq: the target workqueue
417	425	*
418		- * This must be called either with wq->mutex held or sched RCU read locked.
	426	+ * This must be called either with wq->mutex held or RCU read locked.
419	427	* If the pwq needs to be used beyond the locking in effect, the caller is
420	428	* responsible for guaranteeing that the pwq stays online.
421	429	*
..	..	@@ -423,13 +431,12 @@
423	431	* ignored.
424	432	*/
425	433	#define for_each_pwq(pwq, wq) \
426		- list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
427		- if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
428		- else
	434	+ list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
	435	+ lockdep_is_held(&(wq->mutex)))
429	436
430	437	#ifdef CONFIG_DEBUG_OBJECTS_WORK
431	438
432		-static struct debug_obj_descr work_debug_descr;
	439	+static const struct debug_obj_descr work_debug_descr;
433	440
434	441	static void work_debug_hint(void addr)
435	442	{
..	..	@@ -479,7 +486,7 @@
479	486	}
480	487	}
481	488
482		-static struct debug_obj_descr work_debug_descr = {
	489	+static const struct debug_obj_descr work_debug_descr = {
483	490	.name = "work_struct",
484	491	.debug_hint = work_debug_hint,
485	492	.is_static_object = work_is_static_object,
..	..	@@ -551,7 +558,7 @@
551	558	* @wq: the target workqueue
552	559	* @node: the node ID
553	560	*
554		- * This must be called with any of wq_pool_mutex, wq->mutex or sched RCU
	561	+ * This must be called with any of wq_pool_mutex, wq->mutex or RCU
555	562	* read locked.
556	563	* If the pwq needs to be used beyond the locking in effect, the caller is
557	564	* responsible for guaranteeing that the pwq stays online.
..	..	@@ -647,7 +654,7 @@
647	654	* The following mb guarantees that previous clear of a PENDING bit
648	655	* will not be reordered with any speculative LOADS or STORES from
649	656	* work->current_func, which is executed afterwards. This possible
650		- * reordering can lead to a missed execution on attempt to qeueue
	657	+ * reordering can lead to a missed execution on attempt to queue
651	658	* the same @work. E.g. consider this case:
652	659	*
653	660	* CPU#0 CPU#1
..	..	@@ -680,12 +687,17 @@
680	687	set_work_data(work, WORK_STRUCT_NO_POOL, 0);
681	688	}
682	689
	690	+static inline struct pool_workqueue *work_struct_pwq(unsigned long data)
	691	+{
	692	+ return (struct pool_workqueue *)(data & WORK_STRUCT_WQ_DATA_MASK);
	693	+}
	694	+
683	695	static struct pool_workqueue get_work_pwq(struct work_struct work)
684	696	{
685	697	unsigned long data = atomic_long_read(&work->data);
686	698
687	699	if (data & WORK_STRUCT_PWQ)
688		- return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
	700	+ return work_struct_pwq(data);
689	701	else
690	702	return NULL;
691	703	}
..	..	@@ -695,8 +707,8 @@
695	707	* @work: the work item of interest
696	708	*
697	709	* Pools are created and destroyed under wq_pool_mutex, and allows read
698		- * access under sched-RCU read lock. As such, this function should be
699		- * called under wq_pool_mutex or with preemption disabled.
	710	+ * access under RCU read lock. As such, this function should be
	711	+ * called under wq_pool_mutex or inside of a rcu_read_lock() region.
700	712	*
701	713	* All fields of the returned pool are accessible as long as the above
702	714	* mentioned locking is in effect. If the returned pool needs to be used
..	..	@@ -713,8 +725,7 @@
713	725	assert_rcu_or_pool_mutex();
714	726
715	727	if (data & WORK_STRUCT_PWQ)
716		- return ((struct pool_workqueue *)
717		- (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
	728	+ return work_struct_pwq(data)->pool;
718	729
719	730	pool_id = data >> WORK_OFFQ_POOL_SHIFT;
720	731	if (pool_id == WORK_OFFQ_POOL_NONE)
..	..	@@ -735,8 +746,7 @@
735	746	unsigned long data = atomic_long_read(&work->data);
736	747
737	748	if (data & WORK_STRUCT_PWQ)
738		- return ((struct pool_workqueue *)
739		- (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
	749	+ return work_struct_pwq(data)->pool->id;
740	750
741	751	return data >> WORK_OFFQ_POOL_SHIFT;
742	752	}
..	..	@@ -829,7 +839,7 @@
829	839	* Wake up the first idle worker of @pool.
830	840	*
831	841	* CONTEXT:
832		- * spin_lock_irq(pool->lock).
	842	+ * raw_spin_lock_irq(pool->lock).
833	843	*/
834	844	static void wake_up_worker(struct worker_pool *pool)
835	845	{
..	..	@@ -840,43 +850,42 @@
840	850	}
841	851
842	852	/**
843		- * wq_worker_waking_up - a worker is waking up
	853	+ * wq_worker_running - a worker is running again
844	854	* @task: task waking up
845		- * @cpu: CPU @task is waking up to
846	855	*
847		- * This function is called during try_to_wake_up() when a worker is
848		- * being awoken.
849		- *
850		- * CONTEXT:
851		- * spin_lock_irq(rq->lock)
	856	+ * This function is called when a worker returns from schedule()
852	857	*/
853		-void wq_worker_waking_up(struct task_struct *task, int cpu)
	858	+void wq_worker_running(struct task_struct *task)
854	859	{
855	860	struct worker *worker = kthread_data(task);
856	861
857		- if (!(worker->flags & WORKER_NOT_RUNNING)) {
858		- WARN_ON_ONCE(worker->pool->cpu != cpu);
	862	+ if (!worker->sleeping)
	863	+ return;
	864	+
	865	+ /*
	866	+ * If preempted by unbind_workers() between the WORKER_NOT_RUNNING check
	867	+ * and the nr_running increment below, we may ruin the nr_running reset
	868	+ * and leave with an unexpected pool->nr_running == 1 on the newly unbound
	869	+ * pool. Protect against such race.
	870	+ */
	871	+ preempt_disable();
	872	+ if (!(worker->flags & WORKER_NOT_RUNNING))
859	873	atomic_inc(&worker->pool->nr_running);
860		- }
	874	+ preempt_enable();
	875	+ worker->sleeping = 0;
861	876	}
862	877
863	878	/**
864	879	* wq_worker_sleeping - a worker is going to sleep
865	880	* @task: task going to sleep
866	881	*
867		- * This function is called during schedule() when a busy worker is
868		- * going to sleep. Worker on the same cpu can be woken up by
869		- * returning pointer to its task.
870		- *
871		- * CONTEXT:
872		- * spin_lock_irq(rq->lock)
873		- *
874		- * Return:
875		- * Worker task on @cpu to wake up, %NULL if none.
	882	+ * This function is called from schedule() when a busy worker is
	883	+ * going to sleep. Preemption needs to be disabled to protect ->sleeping
	884	+ * assignment.
876	885	*/
877		-struct task_struct wq_worker_sleeping(struct task_struct task)
	886	+void wq_worker_sleeping(struct task_struct *task)
878	887	{
879		- struct worker worker = kthread_data(task), to_wakeup = NULL;
	888	+ struct worker next, worker = kthread_data(task);
880	889	struct worker_pool *pool;
881	890
882	891	/*
..	..	@@ -885,13 +894,16 @@
885	894	* checking NOT_RUNNING.
886	895	*/
887	896	if (worker->flags & WORKER_NOT_RUNNING)
888		- return NULL;
	897	+ return;
889	898
890	899	pool = worker->pool;
891	900
892		- /* this can only happen on the local cpu */
893		- if (WARN_ON_ONCE(pool->cpu != raw_smp_processor_id()))
894		- return NULL;
	901	+ /* Return if preempted before wq_worker_running() was reached */
	902	+ if (worker->sleeping)
	903	+ return;
	904	+
	905	+ worker->sleeping = 1;
	906	+ raw_spin_lock_irq(&pool->lock);
895	907
896	908	/*
897	909	* The counterpart of the following dec_and_test, implied mb,
..	..	@@ -905,19 +917,33 @@
905	917	* lock is safe.
906	918	*/
907	919	if (atomic_dec_and_test(&pool->nr_running) &&
908		- !list_empty(&pool->worklist))
909		- to_wakeup = first_idle_worker(pool);
910		- return to_wakeup ? to_wakeup->task : NULL;
	920	+ !list_empty(&pool->worklist)) {
	921	+ next = first_idle_worker(pool);
	922	+ if (next)
	923	+ wake_up_process(next->task);
	924	+ }
	925	+ raw_spin_unlock_irq(&pool->lock);
911	926	}
912	927
913	928	/**
914	929	* wq_worker_last_func - retrieve worker's last work function
	930	+ * @task: Task to retrieve last work function of.
915	931	*
916	932	* Determine the last function a worker executed. This is called from
917	933	* the scheduler to get a worker's last known identity.
918	934	*
919	935	* CONTEXT:
920		- * spin_lock_irq(rq->lock)
	936	+ * raw_spin_lock_irq(rq->lock)
	937	+ *
	938	+ * This function is called during schedule() when a kworker is going
	939	+ * to sleep. It's used by psi to identify aggregation workers during
	940	+ * dequeuing, to allow periodic aggregation to shut-off when that
	941	+ * worker is the last task in the system or cgroup to go to sleep.
	942	+ *
	943	+ * As this function doesn't involve any workqueue-related locking, it
	944	+ * only returns stable values when called from inside the scheduler's
	945	+ * queuing and dequeuing paths, when @task, which must be a kworker,
	946	+ * is guaranteed to not be processing any works.
921	947	*
922	948	* Return:
923	949	* The last work function %current executed as a worker, NULL if it
..	..	@@ -938,7 +964,7 @@
938	964	* Set @flags in @worker->flags and adjust nr_running accordingly.
939	965	*
940	966	* CONTEXT:
941		- * spin_lock_irq(pool->lock)
	967	+ * raw_spin_lock_irq(pool->lock)
942	968	*/
943	969	static inline void worker_set_flags(struct worker *worker, unsigned int flags)
944	970	{
..	..	@@ -963,7 +989,7 @@
963	989	* Clear @flags in @worker->flags and adjust nr_running accordingly.
964	990	*
965	991	* CONTEXT:
966		- * spin_lock_irq(pool->lock)
	992	+ * raw_spin_lock_irq(pool->lock)
967	993	*/
968	994	static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
969	995	{
..	..	@@ -1011,7 +1037,7 @@
1011	1037	* actually occurs, it should be easy to locate the culprit work function.
1012	1038	*
1013	1039	* CONTEXT:
1014		- * spin_lock_irq(pool->lock).
	1040	+ * raw_spin_lock_irq(pool->lock).
1015	1041	*
1016	1042	* Return:
1017	1043	* Pointer to worker which is executing @work if found, %NULL
..	..	@@ -1046,7 +1072,7 @@
1046	1072	* nested inside outer list_for_each_entry_safe().
1047	1073	*
1048	1074	* CONTEXT:
1049		- * spin_lock_irq(pool->lock).
	1075	+ * raw_spin_lock_irq(pool->lock).
1050	1076	*/
1051	1077	static void move_linked_works(struct work_struct work, struct list_head head,
1052	1078	struct work_struct **nextp)
..	..	@@ -1121,12 +1147,12 @@
1121	1147	{
1122	1148	if (pwq) {
1123	1149	/*
1124		- * As both pwqs and pools are sched-RCU protected, the
	1150	+ * As both pwqs and pools are RCU protected, the
1125	1151	* following lock operations are safe.
1126	1152	*/
1127		- spin_lock_irq(&pwq->pool->lock);
	1153	+ raw_spin_lock_irq(&pwq->pool->lock);
1128	1154	put_pwq(pwq);
1129		- spin_unlock_irq(&pwq->pool->lock);
	1155	+ raw_spin_unlock_irq(&pwq->pool->lock);
1130	1156	}
1131	1157	}
1132	1158
..	..	@@ -1159,7 +1185,7 @@
1159	1185	* decrement nr_in_flight of its pwq and handle workqueue flushing.
1160	1186	*
1161	1187	* CONTEXT:
1162		- * spin_lock_irq(pool->lock).
	1188	+ * raw_spin_lock_irq(pool->lock).
1163	1189	*/
1164	1190	static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
1165	1191	{
..	..	@@ -1207,11 +1233,14 @@
1207	1233	* stable state - idle, on timer or on worklist.
1208	1234	*
1209	1235	* Return:
	1236	+ *
	1237	+ * ======== ================================================================
1210	1238	* 1 if @work was pending and we successfully stole PENDING
1211	1239	* 0 if @work was idle and we claimed PENDING
1212	1240	* -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
1213	1241	* -ENOENT if someone else is canceling @work, this state may persist
1214	1242	* for arbitrarily long
	1243	+ * ======== ================================================================
1215	1244	*
1216	1245	* Note:
1217	1246	* On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
..	..	@@ -1249,6 +1278,7 @@
1249	1278	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1250	1279	return 0;
1251	1280
	1281	+ rcu_read_lock();
1252	1282	/*
1253	1283	* The queueing is in progress, or it is already queued. Try to
1254	1284	* steal it from ->worklist without clearing WORK_STRUCT_PENDING.
..	..	@@ -1257,7 +1287,7 @@
1257	1287	if (!pool)
1258	1288	goto fail;
1259	1289
1260		- spin_lock(&pool->lock);
	1290	+ raw_spin_lock(&pool->lock);
1261	1291	/*
1262	1292	* work->data is guaranteed to point to pwq only while the work
1263	1293	* item is queued on pwq->wq, and both updating work->data to point
..	..	@@ -1286,11 +1316,13 @@
1286	1316	/* work->data points to pwq iff queued, point to pool */
1287	1317	set_work_pool_and_keep_pending(work, pool->id);
1288	1318
1289		- spin_unlock(&pool->lock);
	1319	+ raw_spin_unlock(&pool->lock);
	1320	+ rcu_read_unlock();
1290	1321	return 1;
1291	1322	}
1292		- spin_unlock(&pool->lock);
	1323	+ raw_spin_unlock(&pool->lock);
1293	1324	fail:
	1325	+ rcu_read_unlock();
1294	1326	local_irq_restore(*flags);
1295	1327	if (work_is_canceling(work))
1296	1328	return -ENOENT;
..	..	@@ -1309,12 +1341,15 @@
1309	1341	* work_struct flags.
1310	1342	*
1311	1343	* CONTEXT:
1312		- * spin_lock_irq(pool->lock).
	1344	+ * raw_spin_lock_irq(pool->lock).
1313	1345	*/
1314	1346	static void insert_work(struct pool_workqueue pwq, struct work_struct work,
1315	1347	struct list_head *head, unsigned int extra_flags)
1316	1348	{
1317	1349	struct worker_pool *pool = pwq->pool;
	1350	+
	1351	+ /* record the work call stack in order to print it in KASAN reports */
	1352	+ kasan_record_aux_stack(work);
1318	1353
1319	1354	/* we own @work, set data and link */
1320	1355	set_work_pwq(work, pwq, extra_flags);
..	..	@@ -1342,7 +1377,7 @@
1342	1377
1343	1378	worker = current_wq_worker();
1344	1379	/*
1345		- * Return %true iff I'm a worker execuing a work item on @wq. If
	1380	+ * Return %true iff I'm a worker executing a work item on @wq. If
1346	1381	* I'm @worker, it's safe to dereference it without locking.
1347	1382	*/
1348	1383	return worker && worker->current_pwq->wq == wq;
..	..	@@ -1403,6 +1438,7 @@
1403	1438	if (unlikely(wq->flags & __WQ_DRAINING) &&
1404	1439	WARN_ON_ONCE(!is_chained_work(wq)))
1405	1440	return;
	1441	+ rcu_read_lock();
1406	1442	retry:
1407	1443	/* pwq which will be used unless @work is executing elsewhere */
1408	1444	if (wq->flags & WQ_UNBOUND) {
..	..	@@ -1424,7 +1460,7 @@
1424	1460	if (last_pool && last_pool != pwq->pool) {
1425	1461	struct worker *worker;
1426	1462
1427		- spin_lock(&last_pool->lock);
	1463	+ raw_spin_lock(&last_pool->lock);
1428	1464
1429	1465	worker = find_worker_executing_work(last_pool, work);
1430	1466
..	..	@@ -1432,11 +1468,11 @@
1432	1468	pwq = worker->current_pwq;
1433	1469	} else {
1434	1470	/* meh... not running there, queue here */
1435		- spin_unlock(&last_pool->lock);
1436		- spin_lock(&pwq->pool->lock);
	1471	+ raw_spin_unlock(&last_pool->lock);
	1472	+ raw_spin_lock(&pwq->pool->lock);
1437	1473	}
1438	1474	} else {
1439		- spin_lock(&pwq->pool->lock);
	1475	+ raw_spin_lock(&pwq->pool->lock);
1440	1476	}
1441	1477
1442	1478	/*
..	..	@@ -1449,7 +1485,7 @@
1449	1485	*/
1450	1486	if (unlikely(!pwq->refcnt)) {
1451	1487	if (wq->flags & WQ_UNBOUND) {
1452		- spin_unlock(&pwq->pool->lock);
	1488	+ raw_spin_unlock(&pwq->pool->lock);
1453	1489	cpu_relax();
1454	1490	goto retry;
1455	1491	}
..	..	@@ -1461,10 +1497,8 @@
1461	1497	/* pwq determined, queue */
1462	1498	trace_workqueue_queue_work(req_cpu, pwq, work);
1463	1499
1464		- if (WARN_ON(!list_empty(&work->entry))) {
1465		- spin_unlock(&pwq->pool->lock);
1466		- return;
1467		- }
	1500	+ if (WARN_ON(!list_empty(&work->entry)))
	1501	+ goto out;
1468	1502
1469	1503	pwq->nr_in_flight[pwq->work_color]++;
1470	1504	work_flags = work_color_to_flags(pwq->work_color);
..	..	@@ -1483,7 +1517,9 @@
1483	1517	debug_work_activate(work);
1484	1518	insert_work(pwq, work, worklist, work_flags);
1485	1519
1486		- spin_unlock(&pwq->pool->lock);
	1520	+out:
	1521	+ raw_spin_unlock(&pwq->pool->lock);
	1522	+ rcu_read_unlock();
1487	1523	}
1488	1524
1489	1525	/**
..	..	@@ -1515,6 +1551,90 @@
1515	1551	}
1516	1552	EXPORT_SYMBOL(queue_work_on);
1517	1553
	1554	+/**
	1555	+ * workqueue_select_cpu_near - Select a CPU based on NUMA node
	1556	+ * @node: NUMA node ID that we want to select a CPU from
	1557	+ *
	1558	+ * This function will attempt to find a "random" cpu available on a given
	1559	+ * node. If there are no CPUs available on the given node it will return
	1560	+ * WORK_CPU_UNBOUND indicating that we should just schedule to any
	1561	+ * available CPU if we need to schedule this work.
	1562	+ */
	1563	+static int workqueue_select_cpu_near(int node)
	1564	+{
	1565	+ int cpu;
	1566	+
	1567	+ /* No point in doing this if NUMA isn't enabled for workqueues */
	1568	+ if (!wq_numa_enabled)
	1569	+ return WORK_CPU_UNBOUND;
	1570	+
	1571	+ /* Delay binding to CPU if node is not valid or online */
	1572	+ if (node < 0 \|\| node >= MAX_NUMNODES \|\| !node_online(node))
	1573	+ return WORK_CPU_UNBOUND;
	1574	+
	1575	+ /* Use local node/cpu if we are already there */
	1576	+ cpu = raw_smp_processor_id();
	1577	+ if (node == cpu_to_node(cpu))
	1578	+ return cpu;
	1579	+
	1580	+ /* Use "random" otherwise know as "first" online CPU of node */
	1581	+ cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
	1582	+
	1583	+ /* If CPU is valid return that, otherwise just defer */
	1584	+ return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;
	1585	+}
	1586	+
	1587	+/**
	1588	+ * queue_work_node - queue work on a "random" cpu for a given NUMA node
	1589	+ * @node: NUMA node that we are targeting the work for
	1590	+ * @wq: workqueue to use
	1591	+ * @work: work to queue
	1592	+ *
	1593	+ * We queue the work to a "random" CPU within a given NUMA node. The basic
	1594	+ * idea here is to provide a way to somehow associate work with a given
	1595	+ * NUMA node.
	1596	+ *
	1597	+ * This function will only make a best effort attempt at getting this onto
	1598	+ * the right NUMA node. If no node is requested or the requested node is
	1599	+ * offline then we just fall back to standard queue_work behavior.
	1600	+ *
	1601	+ * Currently the "random" CPU ends up being the first available CPU in the
	1602	+ * intersection of cpu_online_mask and the cpumask of the node, unless we
	1603	+ * are running on the node. In that case we just use the current CPU.
	1604	+ *
	1605	+ * Return: %false if @work was already on a queue, %true otherwise.
	1606	+ */
	1607	+bool queue_work_node(int node, struct workqueue_struct *wq,
	1608	+ struct work_struct *work)
	1609	+{
	1610	+ unsigned long flags;
	1611	+ bool ret = false;
	1612	+
	1613	+ /*
	1614	+ * This current implementation is specific to unbound workqueues.
	1615	+ * Specifically we only return the first available CPU for a given
	1616	+ * node instead of cycling through individual CPUs within the node.
	1617	+ *
	1618	+ * If this is used with a per-cpu workqueue then the logic in
	1619	+ * workqueue_select_cpu_near would need to be updated to allow for
	1620	+ * some round robin type logic.
	1621	+ */
	1622	+ WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));
	1623	+
	1624	+ local_irq_save(flags);
	1625	+
	1626	+ if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
	1627	+ int cpu = workqueue_select_cpu_near(node);
	1628	+
	1629	+ __queue_work(cpu, wq, work);
	1630	+ ret = true;
	1631	+ }
	1632	+
	1633	+ local_irq_restore(flags);
	1634	+ return ret;
	1635	+}
	1636	+EXPORT_SYMBOL_GPL(queue_work_node);
	1637	+
1518	1638	void delayed_work_timer_fn(struct timer_list *t)
1519	1639	{
1520	1640	struct delayed_work *dwork = from_timer(dwork, t, timer);
..	..	@@ -1531,9 +1651,14 @@
1531	1651	struct work_struct *work = &dwork->work;
1532	1652
1533	1653	WARN_ON_ONCE(!wq);
1534		-#ifndef CONFIG_CFI_CLANG
1535		- WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
1536		-#endif
	1654	+ /*
	1655	+ * With CFI, timer->function can point to a jump table entry in a module,
	1656	+ * which fails the comparison. Disable the warning if CFI and modules are
	1657	+ * both enabled.
	1658	+ */
	1659	+ if (!IS_ENABLED(CONFIG_CFI_CLANG) \|\| !IS_ENABLED(CONFIG_MODULES))
	1660	+ WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
	1661	+
1537	1662	WARN_ON_ONCE(timer_pending(timer));
1538	1663	WARN_ON_ONCE(!list_empty(&work->entry));
1539	1664
..	..	@@ -1644,7 +1769,7 @@
1644	1769	*
1645	1770	* Return: %false if @rwork was already pending, %true otherwise. Note
1646	1771	* that a full RCU grace period is guaranteed only after a %true return.
1647		- * While @rwork is guarnateed to be executed after a %false return, the
	1772	+ * While @rwork is guaranteed to be executed after a %false return, the
1648	1773	* execution may happen before a full RCU grace period has passed.
1649	1774	*/
1650	1775	bool queue_rcu_work(struct workqueue_struct wq, struct rcu_work rwork)
..	..	@@ -1669,7 +1794,7 @@
1669	1794	* necessary.
1670	1795	*
1671	1796	* LOCKING:
1672		- * spin_lock_irq(pool->lock).
	1797	+ * raw_spin_lock_irq(pool->lock).
1673	1798	*/
1674	1799	static void worker_enter_idle(struct worker *worker)
1675	1800	{
..	..	@@ -1709,7 +1834,7 @@
1709	1834	* @worker is leaving idle state. Update stats.
1710	1835	*
1711	1836	* LOCKING:
1712		- * spin_lock_irq(pool->lock).
	1837	+ * raw_spin_lock_irq(pool->lock).
1713	1838	*/
1714	1839	static void worker_leave_idle(struct worker *worker)
1715	1840	{
..	..	@@ -1838,17 +1963,26 @@
1838	1963	goto fail;
1839	1964
1840	1965	set_user_nice(worker->task, pool->attrs->nice);
	1966	+ if (IS_ENABLED(CONFIG_ROCKCHIP_OPTIMIZE_RT_PRIO)) {
	1967	+ struct sched_param param;
	1968	+
	1969	+ if (pool->attrs->nice == 0)
	1970	+ param.sched_priority = MAX_RT_PRIO / 2 - 4;
	1971	+ else
	1972	+ param.sched_priority = MAX_RT_PRIO / 2 - 2;
	1973	+ sched_setscheduler_nocheck(worker->task, SCHED_RR, &param);
	1974	+ }
1841	1975	kthread_bind_mask(worker->task, pool->attrs->cpumask);
1842	1976
1843	1977	/* successful, attach the worker to the pool */
1844	1978	worker_attach_to_pool(worker, pool);
1845	1979
1846	1980	/* start the newly created worker */
1847		- spin_lock_irq(&pool->lock);
	1981	+ raw_spin_lock_irq(&pool->lock);
1848	1982	worker->pool->nr_workers++;
1849	1983	worker_enter_idle(worker);
1850	1984	wake_up_process(worker->task);
1851		- spin_unlock_irq(&pool->lock);
	1985	+ raw_spin_unlock_irq(&pool->lock);
1852	1986
1853	1987	return worker;
1854	1988
..	..	@@ -1867,7 +2001,7 @@
1867	2001	* be idle.
1868	2002	*
1869	2003	* CONTEXT:
1870		- * spin_lock_irq(pool->lock).
	2004	+ * raw_spin_lock_irq(pool->lock).
1871	2005	*/
1872	2006	static void destroy_worker(struct worker *worker)
1873	2007	{
..	..	@@ -1893,7 +2027,7 @@
1893	2027	{
1894	2028	struct worker_pool *pool = from_timer(pool, t, idle_timer);
1895	2029
1896		- spin_lock_irq(&pool->lock);
	2030	+ raw_spin_lock_irq(&pool->lock);
1897	2031
1898	2032	while (too_many_workers(pool)) {
1899	2033	struct worker *worker;
..	..	@@ -1911,7 +2045,7 @@
1911	2045	destroy_worker(worker);
1912	2046	}
1913	2047
1914		- spin_unlock_irq(&pool->lock);
	2048	+ raw_spin_unlock_irq(&pool->lock);
1915	2049	}
1916	2050
1917	2051	static void send_mayday(struct work_struct *work)
..	..	@@ -1942,8 +2076,8 @@
1942	2076	struct worker_pool *pool = from_timer(pool, t, mayday_timer);
1943	2077	struct work_struct *work;
1944	2078
1945		- spin_lock_irq(&pool->lock);
1946		- spin_lock(&wq_mayday_lock); /* for wq->maydays */
	2079	+ raw_spin_lock_irq(&pool->lock);
	2080	+ raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */
1947	2081
1948	2082	if (need_to_create_worker(pool)) {
1949	2083	/*
..	..	@@ -1956,8 +2090,8 @@
1956	2090	send_mayday(work);
1957	2091	}
1958	2092
1959		- spin_unlock(&wq_mayday_lock);
1960		- spin_unlock_irq(&pool->lock);
	2093	+ raw_spin_unlock(&wq_mayday_lock);
	2094	+ raw_spin_unlock_irq(&pool->lock);
1961	2095
1962	2096	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
1963	2097	}
..	..	@@ -1976,7 +2110,7 @@
1976	2110	* may_start_working() %true.
1977	2111	*
1978	2112	* LOCKING:
1979		- * spin_lock_irq(pool->lock) which may be released and regrabbed
	2113	+ * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
1980	2114	* multiple times. Does GFP_KERNEL allocations. Called only from
1981	2115	* manager.
1982	2116	*/
..	..	@@ -1985,7 +2119,7 @@
1985	2119	__acquires(&pool->lock)
1986	2120	{
1987	2121	restart:
1988		- spin_unlock_irq(&pool->lock);
	2122	+ raw_spin_unlock_irq(&pool->lock);
1989	2123
1990	2124	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
1991	2125	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
..	..	@@ -2001,7 +2135,7 @@
2001	2135	}
2002	2136
2003	2137	del_timer_sync(&pool->mayday_timer);
2004		- spin_lock_irq(&pool->lock);
	2138	+ raw_spin_lock_irq(&pool->lock);
2005	2139	/*
2006	2140	* This is necessary even after a new worker was just successfully
2007	2141	* created as @pool->lock was dropped and the new worker might have
..	..	@@ -2024,7 +2158,7 @@
2024	2158	* and may_start_working() is true.
2025	2159	*
2026	2160	* CONTEXT:
2027		- * spin_lock_irq(pool->lock) which may be released and regrabbed
	2161	+ * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2028	2162	* multiple times. Does GFP_KERNEL allocations.
2029	2163	*
2030	2164	* Return:
..	..	@@ -2047,7 +2181,7 @@
2047	2181
2048	2182	pool->manager = NULL;
2049	2183	pool->flags &= ~POOL_MANAGER_ACTIVE;
2050		- wake_up(&wq_manager_wait);
	2184	+ rcuwait_wake_up(&manager_wait);
2051	2185	return true;
2052	2186	}
2053	2187
..	..	@@ -2063,7 +2197,7 @@
2063	2197	* call this function to process a work.
2064	2198	*
2065	2199	* CONTEXT:
2066		- * spin_lock_irq(pool->lock) which is released and regrabbed.
	2200	+ * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
2067	2201	*/
2068	2202	static void process_one_work(struct worker worker, struct work_struct work)
2069	2203	__releases(&pool->lock)
..	..	@@ -2145,7 +2279,7 @@
2145	2279	*/
2146	2280	set_work_pool_and_clear_pending(work, pool->id);
2147	2281
2148		- spin_unlock_irq(&pool->lock);
	2282	+ raw_spin_unlock_irq(&pool->lock);
2149	2283
2150	2284	lock_map_acquire(&pwq->wq->lockdep_map);
2151	2285	lock_map_acquire(&lockdep_map);
..	..	@@ -2177,13 +2311,13 @@
2177	2311	* While we must be careful to not use "work" after this, the trace
2178	2312	* point will only record its address.
2179	2313	*/
2180		- trace_workqueue_execute_end(work);
	2314	+ trace_workqueue_execute_end(work, worker->current_func);
2181	2315	lock_map_release(&lockdep_map);
2182	2316	lock_map_release(&pwq->wq->lockdep_map);
2183	2317
2184	2318	if (unlikely(in_atomic() \|\| lockdep_depth(current) > 0)) {
2185	2319	pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2186		- " last function: %pf\n",
	2320	+ " last function: %ps\n",
2187	2321	current->comm, preempt_count(), task_pid_nr(current),
2188	2322	worker->current_func);
2189	2323	debug_show_held_locks(current);
..	..	@@ -2191,7 +2325,7 @@
2191	2325	}
2192	2326
2193	2327	/*
2194		- * The following prevents a kworker from hogging CPU on !PREEMPT
	2328	+ * The following prevents a kworker from hogging CPU on !PREEMPTION
2195	2329	* kernels, where a requeueing work item waiting for something to
2196	2330	* happen could deadlock with stop_machine as such work item could
2197	2331	* indefinitely requeue itself while all other CPUs are trapped in
..	..	@@ -2200,7 +2334,7 @@
2200	2334	*/
2201	2335	cond_resched();
2202	2336
2203		- spin_lock_irq(&pool->lock);
	2337	+ raw_spin_lock_irq(&pool->lock);
2204	2338
2205	2339	/* clear cpu intensive status */
2206	2340	if (unlikely(cpu_intensive))
..	..	@@ -2226,7 +2360,7 @@
2226	2360	* fetches a work from the top and executes it.
2227	2361	*
2228	2362	* CONTEXT:
2229		- * spin_lock_irq(pool->lock) which may be released and regrabbed
	2363	+ * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2230	2364	* multiple times.
2231	2365	*/
2232	2366	static void process_scheduled_works(struct worker *worker)
..	..	@@ -2268,11 +2402,11 @@
2268	2402	/* tell the scheduler that this is a workqueue worker */
2269	2403	set_pf_worker(true);
2270	2404	woke_up:
2271		- spin_lock_irq(&pool->lock);
	2405	+ raw_spin_lock_irq(&pool->lock);
2272	2406
2273	2407	/* am I supposed to die? */
2274	2408	if (unlikely(worker->flags & WORKER_DIE)) {
2275		- spin_unlock_irq(&pool->lock);
	2409	+ raw_spin_unlock_irq(&pool->lock);
2276	2410	WARN_ON_ONCE(!list_empty(&worker->entry));
2277	2411	set_pf_worker(false);
2278	2412
..	..	@@ -2338,7 +2472,7 @@
2338	2472	*/
2339	2473	worker_enter_idle(worker);
2340	2474	__set_current_state(TASK_IDLE);
2341		- spin_unlock_irq(&pool->lock);
	2475	+ raw_spin_unlock_irq(&pool->lock);
2342	2476	schedule();
2343	2477	goto woke_up;
2344	2478	}
..	..	@@ -2392,7 +2526,7 @@
2392	2526	should_stop = kthread_should_stop();
2393	2527
2394	2528	/* see whether any pwq is asking for help */
2395		- spin_lock_irq(&wq_mayday_lock);
	2529	+ raw_spin_lock_irq(&wq_mayday_lock);
2396	2530
2397	2531	while (!list_empty(&wq->maydays)) {
2398	2532	struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
..	..	@@ -2404,11 +2538,11 @@
2404	2538	__set_current_state(TASK_RUNNING);
2405	2539	list_del_init(&pwq->mayday_node);
2406	2540
2407		- spin_unlock_irq(&wq_mayday_lock);
	2541	+ raw_spin_unlock_irq(&wq_mayday_lock);
2408	2542
2409	2543	worker_attach_to_pool(rescuer, pool);
2410	2544
2411		- spin_lock_irq(&pool->lock);
	2545	+ raw_spin_lock_irq(&pool->lock);
2412	2546
2413	2547	/*
2414	2548	* Slurp in all works issued via this workqueue and
..	..	@@ -2436,8 +2570,8 @@
2436	2570	* being used to relieve memory pressure, don't
2437	2571	* incur MAYDAY_INTERVAL delay inbetween.
2438	2572	*/
2439		- if (need_to_create_worker(pool)) {
2440		- spin_lock(&wq_mayday_lock);
	2573	+ if (pwq->nr_active && need_to_create_worker(pool)) {
	2574	+ raw_spin_lock(&wq_mayday_lock);
2441	2575	/*
2442	2576	* Queue iff we aren't racing destruction
2443	2577	* and somebody else hasn't queued it already.
..	..	@@ -2446,7 +2580,7 @@
2446	2580	get_pwq(pwq);
2447	2581	list_add_tail(&pwq->mayday_node, &wq->maydays);
2448	2582	}
2449		- spin_unlock(&wq_mayday_lock);
	2583	+ raw_spin_unlock(&wq_mayday_lock);
2450	2584	}
2451	2585	}
2452	2586
..	..	@@ -2464,14 +2598,14 @@
2464	2598	if (need_more_worker(pool))
2465	2599	wake_up_worker(pool);
2466	2600
2467		- spin_unlock_irq(&pool->lock);
	2601	+ raw_spin_unlock_irq(&pool->lock);
2468	2602
2469	2603	worker_detach_from_pool(rescuer);
2470	2604
2471		- spin_lock_irq(&wq_mayday_lock);
	2605	+ raw_spin_lock_irq(&wq_mayday_lock);
2472	2606	}
2473	2607
2474		- spin_unlock_irq(&wq_mayday_lock);
	2608	+ raw_spin_unlock_irq(&wq_mayday_lock);
2475	2609
2476	2610	if (should_stop) {
2477	2611	__set_current_state(TASK_RUNNING);
..	..	@@ -2508,11 +2642,11 @@
2508	2642	worker = current_wq_worker();
2509	2643
2510	2644	WARN_ONCE(current->flags & PF_MEMALLOC,
2511		- "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%pf",
	2645	+ "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
2512	2646	current->pid, current->comm, target_wq->name, target_func);
2513	2647	WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
2514	2648	(WQ_MEM_RECLAIM \| __WQ_LEGACY)) == WQ_MEM_RECLAIM),
2515		- "workqueue: WQ_MEM_RECLAIM %s:%pf is flushing !WQ_MEM_RECLAIM %s:%pf",
	2649	+ "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
2516	2650	worker->current_pwq->wq->name, worker->current_func,
2517	2651	target_wq->name, target_func);
2518	2652	}
..	..	@@ -2551,7 +2685,7 @@
2551	2685	* underneath us, so we can't reliably determine pwq from @target.
2552	2686	*
2553	2687	* CONTEXT:
2554		- * spin_lock_irq(pool->lock).
	2688	+ * raw_spin_lock_irq(pool->lock).
2555	2689	*/
2556	2690	static void insert_wq_barrier(struct pool_workqueue *pwq,
2557	2691	struct wq_barrier *barr,
..	..	@@ -2638,7 +2772,7 @@
2638	2772	for_each_pwq(pwq, wq) {
2639	2773	struct worker_pool *pool = pwq->pool;
2640	2774
2641		- spin_lock_irq(&pool->lock);
	2775	+ raw_spin_lock_irq(&pool->lock);
2642	2776
2643	2777	if (flush_color >= 0) {
2644	2778	WARN_ON_ONCE(pwq->flush_color != -1);
..	..	@@ -2655,7 +2789,7 @@
2655	2789	pwq->work_color = work_color;
2656	2790	}
2657	2791
2658		- spin_unlock_irq(&pool->lock);
	2792	+ raw_spin_unlock_irq(&pool->lock);
2659	2793	}
2660	2794
2661	2795	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
..	..	@@ -2743,7 +2877,7 @@
2743	2877	* First flushers are responsible for cascading flushes and
2744	2878	* handling overflow. Non-first flushers can simply return.
2745	2879	*/
2746		- if (wq->first_flusher != &this_flusher)
	2880	+ if (READ_ONCE(wq->first_flusher) != &this_flusher)
2747	2881	return;
2748	2882
2749	2883	mutex_lock(&wq->mutex);
..	..	@@ -2752,7 +2886,7 @@
2752	2886	if (wq->first_flusher != &this_flusher)
2753	2887	goto out_unlock;
2754	2888
2755		- wq->first_flusher = NULL;
	2889	+ WRITE_ONCE(wq->first_flusher, NULL);
2756	2890
2757	2891	WARN_ON_ONCE(!list_empty(&this_flusher.list));
2758	2892	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
..	..	@@ -2855,9 +2989,9 @@
2855	2989	for_each_pwq(pwq, wq) {
2856	2990	bool drained;
2857	2991
2858		- spin_lock_irq(&pwq->pool->lock);
	2992	+ raw_spin_lock_irq(&pwq->pool->lock);
2859	2993	drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
2860		- spin_unlock_irq(&pwq->pool->lock);
	2994	+ raw_spin_unlock_irq(&pwq->pool->lock);
2861	2995
2862	2996	if (drained)
2863	2997	continue;
..	..	@@ -2886,14 +3020,14 @@
2886	3020
2887	3021	might_sleep();
2888	3022
2889		- local_irq_disable();
	3023	+ rcu_read_lock();
2890	3024	pool = get_work_pool(work);
2891	3025	if (!pool) {
2892		- local_irq_enable();
	3026	+ rcu_read_unlock();
2893	3027	return false;
2894	3028	}
2895	3029
2896		- spin_lock(&pool->lock);
	3030	+ raw_spin_lock_irq(&pool->lock);
2897	3031	/* see the comment in try_to_grab_pending() with the same code */
2898	3032	pwq = get_work_pwq(work);
2899	3033	if (pwq) {
..	..	@@ -2909,7 +3043,7 @@
2909	3043	check_flush_dependency(pwq->wq, work);
2910	3044
2911	3045	insert_wq_barrier(pwq, barr, work, worker);
2912		- spin_unlock_irq(&pool->lock);
	3046	+ raw_spin_unlock_irq(&pool->lock);
2913	3047
2914	3048	/*
2915	3049	* Force a lock recursion deadlock when using flush_work() inside a
..	..	@@ -2925,10 +3059,11 @@
2925	3059	lock_map_acquire(&pwq->wq->lockdep_map);
2926	3060	lock_map_release(&pwq->wq->lockdep_map);
2927	3061	}
2928		-
	3062	+ rcu_read_unlock();
2929	3063	return true;
2930	3064	already_gone:
2931		- spin_unlock_irq(&pool->lock);
	3065	+ raw_spin_unlock_irq(&pool->lock);
	3066	+ rcu_read_unlock();
2932	3067	return false;
2933	3068	}
2934	3069
..	..	@@ -2942,10 +3077,8 @@
2942	3077	if (WARN_ON(!work->func))
2943	3078	return false;
2944	3079
2945		- if (!from_cancel) {
2946		- lock_map_acquire(&work->lockdep_map);
2947		- lock_map_release(&work->lockdep_map);
2948		- }
	3080	+ lock_map_acquire(&work->lockdep_map);
	3081	+ lock_map_release(&work->lockdep_map);
2949	3082
2950	3083	if (start_flush_work(work, &barr, from_cancel)) {
2951	3084	wait_for_completion(&barr.done);
..	..	@@ -3250,21 +3383,20 @@
3250	3383
3251	3384	/**
3252	3385	* alloc_workqueue_attrs - allocate a workqueue_attrs
3253		- * @gfp_mask: allocation mask to use
3254	3386	*
3255	3387	* Allocate a new workqueue_attrs, initialize with default settings and
3256	3388	* return it.
3257	3389	*
3258	3390	* Return: The allocated new workqueue_attr on success. %NULL on failure.
3259	3391	*/
3260		-struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
	3392	+struct workqueue_attrs *alloc_workqueue_attrs(void)
3261	3393	{
3262	3394	struct workqueue_attrs *attrs;
3263	3395
3264		- attrs = kzalloc(sizeof(*attrs), gfp_mask);
	3396	+ attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
3265	3397	if (!attrs)
3266	3398	goto fail;
3267		- if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
	3399	+ if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))
3268	3400	goto fail;
3269	3401
3270	3402	cpumask_copy(attrs->cpumask, cpu_possible_mask);
..	..	@@ -3321,7 +3453,7 @@
3321	3453	*/
3322	3454	static int init_worker_pool(struct worker_pool *pool)
3323	3455	{
3324		- spin_lock_init(&pool->lock);
	3456	+ raw_spin_lock_init(&pool->lock);
3325	3457	pool->id = -1;
3326	3458	pool->cpu = -1;
3327	3459	pool->node = NUMA_NO_NODE;
..	..	@@ -3342,23 +3474,62 @@
3342	3474	pool->refcnt = 1;
3343	3475
3344	3476	/* shouldn't fail above this point */
3345		- pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
	3477	+ pool->attrs = alloc_workqueue_attrs();
3346	3478	if (!pool->attrs)
3347	3479	return -ENOMEM;
3348	3480	return 0;
3349	3481	}
	3482	+
	3483	+#ifdef CONFIG_LOCKDEP
	3484	+static void wq_init_lockdep(struct workqueue_struct *wq)
	3485	+{
	3486	+ char *lock_name;
	3487	+
	3488	+ lockdep_register_key(&wq->key);
	3489	+ lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
	3490	+ if (!lock_name)
	3491	+ lock_name = wq->name;
	3492	+
	3493	+ wq->lock_name = lock_name;
	3494	+ lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0);
	3495	+}
	3496	+
	3497	+static void wq_unregister_lockdep(struct workqueue_struct *wq)
	3498	+{
	3499	+ lockdep_unregister_key(&wq->key);
	3500	+}
	3501	+
	3502	+static void wq_free_lockdep(struct workqueue_struct *wq)
	3503	+{
	3504	+ if (wq->lock_name != wq->name)
	3505	+ kfree(wq->lock_name);
	3506	+}
	3507	+#else
	3508	+static void wq_init_lockdep(struct workqueue_struct *wq)
	3509	+{
	3510	+}
	3511	+
	3512	+static void wq_unregister_lockdep(struct workqueue_struct *wq)
	3513	+{
	3514	+}
	3515	+
	3516	+static void wq_free_lockdep(struct workqueue_struct *wq)
	3517	+{
	3518	+}
	3519	+#endif
3350	3520
3351	3521	static void rcu_free_wq(struct rcu_head *rcu)
3352	3522	{
3353	3523	struct workqueue_struct *wq =
3354	3524	container_of(rcu, struct workqueue_struct, rcu);
3355	3525
	3526	+ wq_free_lockdep(wq);
	3527	+
3356	3528	if (!(wq->flags & WQ_UNBOUND))
3357	3529	free_percpu(wq->cpu_pwqs);
3358	3530	else
3359	3531	free_workqueue_attrs(wq->unbound_attrs);
3360	3532
3361		- kfree(wq->rescuer);
3362	3533	kfree(wq);
3363	3534	}
3364	3535
..	..	@@ -3371,11 +3542,23 @@
3371	3542	kfree(pool);
3372	3543	}
3373	3544
	3545	+/* This returns with the lock held on success (pool manager is inactive). */
	3546	+static bool wq_manager_inactive(struct worker_pool *pool)
	3547	+{
	3548	+ raw_spin_lock_irq(&pool->lock);
	3549	+
	3550	+ if (pool->flags & POOL_MANAGER_ACTIVE) {
	3551	+ raw_spin_unlock_irq(&pool->lock);
	3552	+ return false;
	3553	+ }
	3554	+ return true;
	3555	+}
	3556	+
3374	3557	/**
3375	3558	* put_unbound_pool - put a worker_pool
3376	3559	* @pool: worker_pool to put
3377	3560	*
3378		- * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU
	3561	+ * Put @pool. If its refcnt reaches zero, it gets destroyed in RCU
3379	3562	* safe manner. get_unbound_pool() calls this function on its failure path
3380	3563	* and this function should be able to release pools which went through,
3381	3564	* successfully or not, init_worker_pool().
..	..	@@ -3406,16 +3589,17 @@
3406	3589	* Become the manager and destroy all workers. This prevents
3407	3590	* @pool's workers from blocking on attach_mutex. We're the last
3408	3591	* manager and @pool gets freed with the flag set.
	3592	+ * Because of how wq_manager_inactive() works, we will hold the
	3593	+ * spinlock after a successful wait.
3409	3594	*/
3410		- spin_lock_irq(&pool->lock);
3411		- wait_event_lock_irq(wq_manager_wait,
3412		- !(pool->flags & POOL_MANAGER_ACTIVE), pool->lock);
	3595	+ rcuwait_wait_event(&manager_wait, wq_manager_inactive(pool),
	3596	+ TASK_UNINTERRUPTIBLE);
3413	3597	pool->flags \|= POOL_MANAGER_ACTIVE;
3414	3598
3415	3599	while ((worker = first_idle_worker(pool)))
3416	3600	destroy_worker(worker);
3417	3601	WARN_ON(pool->nr_workers \|\| pool->nr_idle);
3418		- spin_unlock_irq(&pool->lock);
	3602	+ raw_spin_unlock_irq(&pool->lock);
3419	3603
3420	3604	mutex_lock(&wq_pool_attach_mutex);
3421	3605	if (!list_empty(&pool->workers))
..	..	@@ -3429,8 +3613,8 @@
3429	3613	del_timer_sync(&pool->idle_timer);
3430	3614	del_timer_sync(&pool->mayday_timer);
3431	3615
3432		- /* sched-RCU protected to allow dereferences from get_work_pool() */
3433		- call_rcu_sched(&pool->rcu, rcu_free_pool);
	3616	+ /* RCU protected to allow dereferences from get_work_pool() */
	3617	+ call_rcu(&pool->rcu, rcu_free_pool);
3434	3618	}
3435	3619
3436	3620	/**
..	..	@@ -3543,14 +3727,16 @@
3543	3727	put_unbound_pool(pool);
3544	3728	mutex_unlock(&wq_pool_mutex);
3545	3729
3546		- call_rcu_sched(&pwq->rcu, rcu_free_pwq);
	3730	+ call_rcu(&pwq->rcu, rcu_free_pwq);
3547	3731
3548	3732	/*
3549	3733	* If we're the last pwq going away, @wq is already dead and no one
3550	3734	* is gonna access it anymore. Schedule RCU free.
3551	3735	*/
3552		- if (is_last)
3553		- call_rcu_sched(&wq->rcu, rcu_free_wq);
	3736	+ if (is_last) {
	3737	+ wq_unregister_lockdep(wq);
	3738	+ call_rcu(&wq->rcu, rcu_free_wq);
	3739	+ }
3554	3740	}
3555	3741
3556	3742	/**
..	..	@@ -3575,7 +3761,7 @@
3575	3761	return;
3576	3762
3577	3763	/* this function can be called during early boot w/ irq disabled */
3578		- spin_lock_irqsave(&pwq->pool->lock, flags);
	3764	+ raw_spin_lock_irqsave(&pwq->pool->lock, flags);
3579	3765
3580	3766	/*
3581	3767	* During [un]freezing, the caller is responsible for ensuring that
..	..	@@ -3605,7 +3791,7 @@
3605	3791	pwq->max_active = 0;
3606	3792	}
3607	3793
3608		- spin_unlock_irqrestore(&pwq->pool->lock, flags);
	3794	+ raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
3609	3795	}
3610	3796
3611	3797	/* initialize newly alloced @pwq which is associated with @wq and @pool */
..	..	@@ -3778,8 +3964,8 @@
3778	3964
3779	3965	ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_node_ids), GFP_KERNEL);
3780	3966
3781		- new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3782		- tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);
	3967	+ new_attrs = alloc_workqueue_attrs();
	3968	+ tmp_attrs = alloc_workqueue_attrs();
3783	3969	if (!ctx \|\| !new_attrs \|\| !tmp_attrs)
3784	3970	goto out_free;
3785	3971
..	..	@@ -3913,6 +4099,8 @@
3913	4099	*
3914	4100	* Performs GFP_KERNEL allocations.
3915	4101	*
	4102	+ * Assumes caller has CPU hotplug read exclusion, i.e. get_online_cpus().
	4103	+ *
3916	4104	* Return: 0 on success and -errno on failure.
3917	4105	*/
3918	4106	int apply_workqueue_attrs(struct workqueue_struct *wq,
..	..	@@ -3920,13 +4108,14 @@
3920	4108	{
3921	4109	int ret;
3922	4110
3923		- apply_wqattrs_lock();
	4111	+ lockdep_assert_cpus_held();
	4112	+
	4113	+ mutex_lock(&wq_pool_mutex);
3924	4114	ret = apply_workqueue_attrs_locked(wq, attrs);
3925		- apply_wqattrs_unlock();
	4115	+ mutex_unlock(&wq_pool_mutex);
3926	4116
3927	4117	return ret;
3928	4118	}
3929		-EXPORT_SYMBOL_GPL(apply_workqueue_attrs);
3930	4119
3931	4120	/**
3932	4121	* wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
..	..	@@ -4004,9 +4193,9 @@
4004	4193
4005	4194	use_dfl_pwq:
4006	4195	mutex_lock(&wq->mutex);
4007		- spin_lock_irq(&wq->dfl_pwq->pool->lock);
	4196	+ raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
4008	4197	get_pwq(wq->dfl_pwq);
4009		- spin_unlock_irq(&wq->dfl_pwq->pool->lock);
	4198	+ raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4010	4199	old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4011	4200	out_unlock:
4012	4201	mutex_unlock(&wq->mutex);
..	..	@@ -4036,16 +4225,21 @@
4036	4225	mutex_unlock(&wq->mutex);
4037	4226	}
4038	4227	return 0;
4039		- } else if (wq->flags & __WQ_ORDERED) {
	4228	+ }
	4229	+
	4230	+ get_online_cpus();
	4231	+ if (wq->flags & __WQ_ORDERED) {
4040	4232	ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
4041	4233	/* there should only be single pwq for ordering guarantee */
4042	4234	WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node \|\|
4043	4235	wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
4044	4236	"ordering guarantee broken for workqueue %s\n", wq->name);
4045		- return ret;
4046	4237	} else {
4047		- return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
	4238	+ ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
4048	4239	}
	4240	+ put_online_cpus();
	4241	+
	4242	+ return ret;
4049	4243	}
4050	4244
4051	4245	static int wq_clamp_max_active(int max_active, unsigned int flags,
..	..	@@ -4078,8 +4272,8 @@
4078	4272
4079	4273	rescuer->rescue_wq = wq;
4080	4274	rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name);
4081		- ret = PTR_ERR_OR_ZERO(rescuer->task);
4082		- if (ret) {
	4275	+ if (IS_ERR(rescuer->task)) {
	4276	+ ret = PTR_ERR(rescuer->task);
4083	4277	kfree(rescuer);
4084	4278	return ret;
4085	4279	}
..	..	@@ -4091,11 +4285,10 @@
4091	4285	return 0;
4092	4286	}
4093	4287
4094		-struct workqueue_struct __alloc_workqueue_key(const char fmt,
4095		- unsigned int flags,
4096		- int max_active,
4097		- struct lock_class_key *key,
4098		- const char *lock_name, ...)
	4288	+__printf(1, 4)
	4289	+struct workqueue_struct alloc_workqueue(const char fmt,
	4290	+ unsigned int flags,
	4291	+ int max_active, ...)
4099	4292	{
4100	4293	size_t tbl_size = 0;
4101	4294	va_list args;
..	..	@@ -4125,12 +4318,12 @@
4125	4318	return NULL;
4126	4319
4127	4320	if (flags & WQ_UNBOUND) {
4128		- wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);
	4321	+ wq->unbound_attrs = alloc_workqueue_attrs();
4129	4322	if (!wq->unbound_attrs)
4130	4323	goto err_free_wq;
4131	4324	}
4132	4325
4133		- va_start(args, lock_name);
	4326	+ va_start(args, max_active);
4134	4327	vsnprintf(wq->name, sizeof(wq->name), fmt, args);
4135	4328	va_end(args);
4136	4329
..	..	@@ -4147,11 +4340,11 @@
4147	4340	INIT_LIST_HEAD(&wq->flusher_overflow);
4148	4341	INIT_LIST_HEAD(&wq->maydays);
4149	4342
4150		- lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
	4343	+ wq_init_lockdep(wq);
4151	4344	INIT_LIST_HEAD(&wq->list);
4152	4345
4153	4346	if (alloc_and_link_pwqs(wq) < 0)
4154		- goto err_free_wq;
	4347	+ goto err_unreg_lockdep;
4155	4348
4156	4349	if (wq_online && init_rescuer(wq) < 0)
4157	4350	goto err_destroy;
..	..	@@ -4177,6 +4370,9 @@
4177	4370
4178	4371	return wq;
4179	4372
	4373	+err_unreg_lockdep:
	4374	+ wq_unregister_lockdep(wq);
	4375	+ wq_free_lockdep(wq);
4180	4376	err_free_wq:
4181	4377	free_workqueue_attrs(wq->unbound_attrs);
4182	4378	kfree(wq);
..	..	@@ -4185,7 +4381,23 @@
4185	4381	destroy_workqueue(wq);
4186	4382	return NULL;
4187	4383	}
4188		-EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
	4384	+EXPORT_SYMBOL_GPL(alloc_workqueue);
	4385	+
	4386	+static bool pwq_busy(struct pool_workqueue *pwq)
	4387	+{
	4388	+ int i;
	4389	+
	4390	+ for (i = 0; i < WORK_NR_COLORS; i++)
	4391	+ if (pwq->nr_in_flight[i])
	4392	+ return true;
	4393	+
	4394	+ if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))
	4395	+ return true;
	4396	+ if (pwq->nr_active \|\| !list_empty(&pwq->delayed_works))
	4397	+ return true;
	4398	+
	4399	+ return false;
	4400	+}
4189	4401
4190	4402	/**
4191	4403	* destroy_workqueue - safely terminate a workqueue
..	..	@@ -4212,35 +4424,34 @@
4212	4424	struct worker *rescuer = wq->rescuer;
4213	4425
4214	4426	/* this prevents new queueing */
4215		- spin_lock_irq(&wq_mayday_lock);
	4427	+ raw_spin_lock_irq(&wq_mayday_lock);
4216	4428	wq->rescuer = NULL;
4217		- spin_unlock_irq(&wq_mayday_lock);
	4429	+ raw_spin_unlock_irq(&wq_mayday_lock);
4218	4430
4219	4431	/* rescuer will empty maydays list before exiting */
4220	4432	kthread_stop(rescuer->task);
4221	4433	kfree(rescuer);
4222	4434	}
4223	4435
4224		- /* sanity checks */
	4436	+ /*
	4437	+ * Sanity checks - grab all the locks so that we wait for all
	4438	+ * in-flight operations which may do put_pwq().
	4439	+ */
	4440	+ mutex_lock(&wq_pool_mutex);
4225	4441	mutex_lock(&wq->mutex);
4226	4442	for_each_pwq(pwq, wq) {
4227		- int i;
4228		-
4229		- for (i = 0; i < WORK_NR_COLORS; i++) {
4230		- if (WARN_ON(pwq->nr_in_flight[i])) {
4231		- mutex_unlock(&wq->mutex);
4232		- show_workqueue_state();
4233		- return;
4234		- }
4235		- }
4236		-
4237		- if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) \|\|
4238		- WARN_ON(pwq->nr_active) \|\|
4239		- WARN_ON(!list_empty(&pwq->delayed_works))) {
	4443	+ raw_spin_lock_irq(&pwq->pool->lock);
	4444	+ if (WARN_ON(pwq_busy(pwq))) {
	4445	+ pr_warn("%s: %s has the following busy pwq\n",
	4446	+ __func__, wq->name);
	4447	+ show_pwq(pwq);
	4448	+ raw_spin_unlock_irq(&pwq->pool->lock);
4240	4449	mutex_unlock(&wq->mutex);
	4450	+ mutex_unlock(&wq_pool_mutex);
4241	4451	show_workqueue_state();
4242	4452	return;
4243	4453	}
	4454	+ raw_spin_unlock_irq(&pwq->pool->lock);
4244	4455	}
4245	4456	mutex_unlock(&wq->mutex);
4246	4457
..	..	@@ -4248,16 +4459,16 @@
4248	4459	* wq list is used to freeze wq, remove from list after
4249	4460	* flushing is complete in case freeze races us.
4250	4461	*/
4251		- mutex_lock(&wq_pool_mutex);
4252	4462	list_del_rcu(&wq->list);
4253	4463	mutex_unlock(&wq_pool_mutex);
4254	4464
4255	4465	if (!(wq->flags & WQ_UNBOUND)) {
	4466	+ wq_unregister_lockdep(wq);
4256	4467	/*
4257	4468	* The base ref is never dropped on per-cpu pwqs. Directly
4258	4469	* schedule RCU free.
4259	4470	*/
4260		- call_rcu_sched(&wq->rcu, rcu_free_wq);
	4471	+ call_rcu(&wq->rcu, rcu_free_wq);
4261	4472	} else {
4262	4473	/*
4263	4474	* We're the sole accessor of @wq at this point. Directly
..	..	@@ -4367,7 +4578,8 @@
4367	4578	struct pool_workqueue *pwq;
4368	4579	bool ret;
4369	4580
4370		- rcu_read_lock_sched();
	4581	+ rcu_read_lock();
	4582	+ preempt_disable();
4371	4583
4372	4584	if (cpu == WORK_CPU_UNBOUND)
4373	4585	cpu = smp_processor_id();
..	..	@@ -4378,7 +4590,8 @@
4378	4590	pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4379	4591
4380	4592	ret = !list_empty(&pwq->delayed_works);
4381		- rcu_read_unlock_sched();
	4593	+ preempt_enable();
	4594	+ rcu_read_unlock();
4382	4595
4383	4596	return ret;
4384	4597	}
..	..	@@ -4404,15 +4617,15 @@
4404	4617	if (work_pending(work))
4405	4618	ret \|= WORK_BUSY_PENDING;
4406	4619
4407		- local_irq_save(flags);
	4620	+ rcu_read_lock();
4408	4621	pool = get_work_pool(work);
4409	4622	if (pool) {
4410		- spin_lock(&pool->lock);
	4623	+ raw_spin_lock_irqsave(&pool->lock, flags);
4411	4624	if (find_worker_executing_work(pool, work))
4412	4625	ret \|= WORK_BUSY_RUNNING;
4413		- spin_unlock(&pool->lock);
	4626	+ raw_spin_unlock_irqrestore(&pool->lock, flags);
4414	4627	}
4415		- local_irq_restore(flags);
	4628	+ rcu_read_unlock();
4416	4629
4417	4630	return ret;
4418	4631	}
..	..	@@ -4476,14 +4689,14 @@
4476	4689	* Carefully copy the associated workqueue's workfn, name and desc.
4477	4690	* Keep the original last '\0' in case the original is garbage.
4478	4691	*/
4479		- probe_kernel_read(&fn, &worker->current_func, sizeof(fn));
4480		- probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));
4481		- probe_kernel_read(&wq, &pwq->wq, sizeof(wq));
4482		- probe_kernel_read(name, wq->name, sizeof(name) - 1);
4483		- probe_kernel_read(desc, worker->desc, sizeof(desc) - 1);
	4692	+ copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn));
	4693	+ copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
	4694	+ copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
	4695	+ copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
	4696	+ copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1);
4484	4697
4485	4698	if (fn \|\| name[0] \|\| desc[0]) {
4486		- printk("%sWorkqueue: %s %pf", log_lvl, name, fn);
	4699	+ printk("%sWorkqueue: %s %ps", log_lvl, name, fn);
4487	4700	if (strcmp(name, desc))
4488	4701	pr_cont(" (%s)", desc);
4489	4702	pr_cont("\n");
..	..	@@ -4508,7 +4721,7 @@
4508	4721	pr_cont("%s BAR(%d)", comma ? "," : "",
4509	4722	task_pid_nr(barr->task));
4510	4723	} else {
4511		- pr_cont("%s %pf", comma ? "," : "", work->func);
	4724	+ pr_cont("%s %ps", comma ? "," : "", work->func);
4512	4725	}
4513	4726	}
4514	4727
..	..	@@ -4541,9 +4754,9 @@
4541	4754	if (worker->current_pwq != pwq)
4542	4755	continue;
4543	4756
4544		- pr_cont("%s %d%s:%pf", comma ? "," : "",
	4757	+ pr_cont("%s %d%s:%ps", comma ? "," : "",
4545	4758	task_pid_nr(worker->task),
4546		- worker == pwq->wq->rescuer ? "(RESCUER)" : "",
	4759	+ worker->rescue_wq ? "(RESCUER)" : "",
4547	4760	worker->current_func);
4548	4761	list_for_each_entry(work, &worker->scheduled, entry)
4549	4762	pr_cont_work(false, work);
..	..	@@ -4597,7 +4810,7 @@
4597	4810	unsigned long flags;
4598	4811	int pi;
4599	4812
4600		- rcu_read_lock_sched();
	4813	+ rcu_read_lock();
4601	4814
4602	4815	pr_info("Showing busy workqueues and worker pools:\n");
4603	4816
..	..	@@ -4617,10 +4830,10 @@
4617	4830	pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
4618	4831
4619	4832	for_each_pwq(pwq, wq) {
4620		- spin_lock_irqsave(&pwq->pool->lock, flags);
	4833	+ raw_spin_lock_irqsave(&pwq->pool->lock, flags);
4621	4834	if (pwq->nr_active \|\| !list_empty(&pwq->delayed_works))
4622	4835	show_pwq(pwq);
4623		- spin_unlock_irqrestore(&pwq->pool->lock, flags);
	4836	+ raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
4624	4837	/*
4625	4838	* We could be printing a lot from atomic context, e.g.
4626	4839	* sysrq-t -> show_workqueue_state(). Avoid triggering
..	..	@@ -4634,7 +4847,7 @@
4634	4847	struct worker *worker;
4635	4848	bool first = true;
4636	4849
4637		- spin_lock_irqsave(&pool->lock, flags);
	4850	+ raw_spin_lock_irqsave(&pool->lock, flags);
4638	4851	if (pool->nr_workers == pool->nr_idle)
4639	4852	goto next_pool;
4640	4853
..	..	@@ -4653,7 +4866,7 @@
4653	4866	}
4654	4867	pr_cont("\n");
4655	4868	next_pool:
4656		- spin_unlock_irqrestore(&pool->lock, flags);
	4869	+ raw_spin_unlock_irqrestore(&pool->lock, flags);
4657	4870	/*
4658	4871	* We could be printing a lot from atomic context, e.g.
4659	4872	* sysrq-t -> show_workqueue_state(). Avoid triggering
..	..	@@ -4662,7 +4875,7 @@
4662	4875	touch_nmi_watchdog();
4663	4876	}
4664	4877
4665		- rcu_read_unlock_sched();
	4878	+ rcu_read_unlock();
4666	4879	}
4667	4880
4668	4881	/* used to show worker information through /proc/PID/{comm,stat,status} */
..	..	@@ -4683,7 +4896,7 @@
4683	4896	struct worker_pool *pool = worker->pool;
4684	4897
4685	4898	if (pool) {
4686		- spin_lock_irq(&pool->lock);
	4899	+ raw_spin_lock_irq(&pool->lock);
4687	4900	/*
4688	4901	* ->desc tracks information (wq name or
4689	4902	* set_worker_desc()) for the latest execution. If
..	..	@@ -4697,12 +4910,13 @@
4697	4910	scnprintf(buf + off, size - off, "-%s",
4698	4911	worker->desc);
4699	4912	}
4700		- spin_unlock_irq(&pool->lock);
	4913	+ raw_spin_unlock_irq(&pool->lock);
4701	4914	}
4702	4915	}
4703	4916
4704	4917	mutex_unlock(&wq_pool_attach_mutex);
4705	4918	}
	4919	+EXPORT_SYMBOL_GPL(wq_worker_comm);
4706	4920
4707	4921	#ifdef CONFIG_SMP
4708	4922
..	..	@@ -4728,7 +4942,7 @@
4728	4942
4729	4943	for_each_cpu_worker_pool(pool, cpu) {
4730	4944	mutex_lock(&wq_pool_attach_mutex);
4731		- spin_lock_irq(&pool->lock);
	4945	+ raw_spin_lock_irq(&pool->lock);
4732	4946
4733	4947	/*
4734	4948	* We've blocked all attach/detach operations. Make all workers
..	..	@@ -4742,7 +4956,7 @@
4742	4956
4743	4957	pool->flags \|= POOL_DISASSOCIATED;
4744	4958
4745		- spin_unlock_irq(&pool->lock);
	4959	+ raw_spin_unlock_irq(&pool->lock);
4746	4960	mutex_unlock(&wq_pool_attach_mutex);
4747	4961
4748	4962	/*
..	..	@@ -4768,9 +4982,9 @@
4768	4982	* worker blocking could lead to lengthy stalls. Kick off
4769	4983	* unbound chain execution of currently pending work items.
4770	4984	*/
4771		- spin_lock_irq(&pool->lock);
	4985	+ raw_spin_lock_irq(&pool->lock);
4772	4986	wake_up_worker(pool);
4773		- spin_unlock_irq(&pool->lock);
	4987	+ raw_spin_unlock_irq(&pool->lock);
4774	4988	}
4775	4989	}
4776	4990
..	..	@@ -4797,7 +5011,7 @@
4797	5011	WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4798	5012	pool->attrs->cpumask) < 0);
4799	5013
4800		- spin_lock_irq(&pool->lock);
	5014	+ raw_spin_lock_irq(&pool->lock);
4801	5015
4802	5016	pool->flags &= ~POOL_DISASSOCIATED;
4803	5017
..	..	@@ -4826,7 +5040,7 @@
4826	5040	*
4827	5041	* WRITE_ONCE() is necessary because @worker->flags may be
4828	5042	* tested without holding any lock in
4829		- * wq_worker_waking_up(). Without it, NOT_RUNNING test may
	5043	+ * wq_worker_running(). Without it, NOT_RUNNING test may
4830	5044	* fail incorrectly leading to premature concurrency
4831	5045	* management operations.
4832	5046	*/
..	..	@@ -4836,7 +5050,7 @@
4836	5050	WRITE_ONCE(worker->flags, worker_flags);
4837	5051	}
4838	5052
4839		- spin_unlock_irq(&pool->lock);
	5053	+ raw_spin_unlock_irq(&pool->lock);
4840	5054	}
4841	5055
4842	5056	/**
..	..	@@ -5049,16 +5263,16 @@
5049	5263	* nr_active is monotonically decreasing. It's safe
5050	5264	* to peek without lock.
5051	5265	*/
5052		- rcu_read_lock_sched();
	5266	+ rcu_read_lock();
5053	5267	for_each_pwq(pwq, wq) {
5054	5268	WARN_ON_ONCE(pwq->nr_active < 0);
5055	5269	if (pwq->nr_active) {
5056	5270	busy = true;
5057		- rcu_read_unlock_sched();
	5271	+ rcu_read_unlock();
5058	5272	goto out_unlock;
5059	5273	}
5060	5274	}
5061		- rcu_read_unlock_sched();
	5275	+ rcu_read_unlock();
5062	5276	}
5063	5277	out_unlock:
5064	5278	mutex_unlock(&wq_pool_mutex);
..	..	@@ -5260,7 +5474,8 @@
5260	5474	const char *delim = "";
5261	5475	int node, written = 0;
5262	5476
5263		- rcu_read_lock_sched();
	5477	+ get_online_cpus();
	5478	+ rcu_read_lock();
5264	5479	for_each_node(node) {
5265	5480	written += scnprintf(buf + written, PAGE_SIZE - written,
5266	5481	"%s%d:%d", delim, node,
..	..	@@ -5268,7 +5483,8 @@
5268	5483	delim = " ";
5269	5484	}
5270	5485	written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
5271		- rcu_read_unlock_sched();
	5486	+ rcu_read_unlock();
	5487	+ put_online_cpus();
5272	5488
5273	5489	return written;
5274	5490	}
..	..	@@ -5293,7 +5509,7 @@
5293	5509
5294	5510	lockdep_assert_held(&wq_pool_mutex);
5295	5511
5296		- attrs = alloc_workqueue_attrs(GFP_KERNEL);
	5512	+ attrs = alloc_workqueue_attrs();
5297	5513	if (!attrs)
5298	5514	return NULL;
5299	5515
..	..	@@ -5639,6 +5855,7 @@
5639	5855	pr_cont_pool_info(pool);
5640	5856	pr_cont(" stuck for %us!\n",
5641	5857	jiffies_to_msecs(now - pool_ts) / 1000);
	5858	+ trace_android_vh_wq_lockup_pool(pool->cpu, pool_ts);
5642	5859	}
5643	5860	}
5644	5861
..	..	@@ -5722,7 +5939,14 @@
5722	5939	return;
5723	5940	}
5724	5941
5725		- wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL);
	5942	+ for_each_possible_cpu(cpu) {
	5943	+ if (WARN_ON(cpu_to_node(cpu) == NUMA_NO_NODE)) {
	5944	+ pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
	5945	+ return;
	5946	+ }
	5947	+ }
	5948	+
	5949	+ wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs();
5726	5950	BUG_ON(!wq_update_unbound_numa_attrs_buf);
5727	5951
5728	5952	/*
..	..	@@ -5739,11 +5963,6 @@
5739	5963
5740	5964	for_each_possible_cpu(cpu) {
5741	5965	node = cpu_to_node(cpu);
5742		- if (WARN_ON(node == NUMA_NO_NODE)) {
5743		- pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
5744		- /* happens iff arch is bonkers, let's just proceed */
5745		- return;
5746		- }
5747	5966	cpumask_set_cpu(cpu, tbl[node]);
5748	5967	}
5749	5968
..	..	@@ -5761,13 +5980,13 @@
5761	5980	* items. Actual work item execution starts only after kthreads can be
5762	5981	* created and scheduled right before early initcalls.
5763	5982	*/
5764		-int __init workqueue_init_early(void)
	5983	+void __init workqueue_init_early(void)
5765	5984	{
5766	5985	int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
5767	5986	int hk_flags = HK_FLAG_DOMAIN \| HK_FLAG_WQ;
5768	5987	int i, cpu;
5769	5988
5770		- WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
	5989	+ BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
5771	5990
5772	5991	BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
5773	5992	cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(hk_flags));
..	..	@@ -5797,7 +6016,7 @@
5797	6016	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
5798	6017	struct workqueue_attrs *attrs;
5799	6018
5800		- BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
	6019	+ BUG_ON(!(attrs = alloc_workqueue_attrs()));
5801	6020	attrs->nice = std_nice[i];
5802	6021	unbound_std_wq_attrs[i] = attrs;
5803	6022
..	..	@@ -5806,7 +6025,7 @@
5806	6025	* guaranteed by max_active which is enforced by pwqs.
5807	6026	* Turn off NUMA so that dfl_pwq is used for all nodes.
5808	6027	*/
5809		- BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
	6028	+ BUG_ON(!(attrs = alloc_workqueue_attrs()));
5810	6029	attrs->nice = std_nice[i];
5811	6030	attrs->no_numa = true;
5812	6031	ordered_wq_attrs[i] = attrs;
..	..	@@ -5828,8 +6047,6 @@
5828	6047	!system_unbound_wq \|\| !system_freezable_wq \|\|
5829	6048	!system_power_efficient_wq \|\|
5830	6049	!system_freezable_power_efficient_wq);
5831		-
5832		- return 0;
5833	6050	}
5834	6051
5835	6052	/**
..	..	@@ -5841,7 +6058,7 @@
5841	6058	* are no kworkers executing the work items yet. Populate the worker pools
5842	6059	* with the initial workers and enable future kworker creations.
5843	6060	*/
5844		-int __init workqueue_init(void)
	6061	+void __init workqueue_init(void)
5845	6062	{
5846	6063	struct workqueue_struct *wq;
5847	6064	struct worker_pool *pool;
..	..	@@ -5888,6 +6105,4 @@
5888	6105
5889	6106	wq_online = true;
5890	6107	wq_watchdog_init();
5891		-
5892		- return 0;
5893	6108	}