~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* Generic helpers for smp ipi calls
3	4	*
..	..	@@ -13,19 +14,22 @@
13	14	#include <linux/export.h>
14	15	#include <linux/percpu.h>
15	16	#include <linux/init.h>
	17	+#include <linux/interrupt.h>
16	18	#include <linux/gfp.h>
17	19	#include <linux/smp.h>
18	20	#include <linux/cpu.h>
19	21	#include <linux/sched.h>
20	22	#include <linux/sched/idle.h>
21	23	#include <linux/hypervisor.h>
	24	+#include <linux/sched/clock.h>
	25	+#include <linux/nmi.h>
	26	+#include <linux/sched/debug.h>
	27	+#include <linux/suspend.h>
22	28
23	29	#include "smpboot.h"
	30	+#include "sched/smp.h"
24	31
25		-enum {
26		- CSD_FLAG_LOCK = 0x01,
27		- CSD_FLAG_SYNCHRONOUS = 0x02,
28		-};
	32	+#define CSD_TYPE(_csd) ((_csd)->flags & CSD_FLAG_TYPE_MASK)
29	33
30	34	struct call_function_data {
31	35	call_single_data_t __percpu *csd;
..	..	@@ -33,7 +37,7 @@
33	37	cpumask_var_t cpumask_ipi;
34	38	};
35	39
36		-static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);
	40	+static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data);
37	41
38	42	static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
39	43
..	..	@@ -83,6 +87,7 @@
83	87	* still pending.
84	88	*/
85	89	flush_smp_call_function_queue(false);
	90	+ irq_work_run();
86	91	return 0;
87	92	}
88	93
..	..	@@ -96,6 +101,103 @@
96	101	smpcfd_prepare_cpu(smp_processor_id());
97	102	}
98	103
	104	+#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
	105	+
	106	+static DEFINE_PER_CPU(call_single_data_t *, cur_csd);
	107	+static DEFINE_PER_CPU(smp_call_func_t, cur_csd_func);
	108	+static DEFINE_PER_CPU(void *, cur_csd_info);
	109	+
	110	+#define CSD_LOCK_TIMEOUT (5ULL * NSEC_PER_SEC)
	111	+static atomic_t csd_bug_count = ATOMIC_INIT(0);
	112	+
	113	+/* Record current CSD work for current CPU, NULL to erase. */
	114	+static void csd_lock_record(struct __call_single_data *csd)
	115	+{
	116	+ if (!csd) {
	117	+ smp_mb(); /* NULL cur_csd after unlock. */
	118	+ __this_cpu_write(cur_csd, NULL);
	119	+ return;
	120	+ }
	121	+ __this_cpu_write(cur_csd_func, csd->func);
	122	+ __this_cpu_write(cur_csd_info, csd->info);
	123	+ smp_wmb(); /* func and info before csd. */
	124	+ __this_cpu_write(cur_csd, csd);
	125	+ smp_mb(); /* Update cur_csd before function call. */
	126	+ /* Or before unlock, as the case may be. */
	127	+}
	128	+
	129	+static __always_inline int csd_lock_wait_getcpu(struct __call_single_data *csd)
	130	+{
	131	+ unsigned int csd_type;
	132	+
	133	+ csd_type = CSD_TYPE(csd);
	134	+ if (csd_type == CSD_TYPE_ASYNC \|\| csd_type == CSD_TYPE_SYNC)
	135	+ return csd->dst; /* Other CSD_TYPE_ values might not have ->dst. */
	136	+ return -1;
	137	+}
	138	+
	139	+/*
	140	+ * Complain if too much time spent waiting. Note that only
	141	+ * the CSD_TYPE_SYNC/ASYNC types provide the destination CPU,
	142	+ * so waiting on other types gets much less information.
	143	+ */
	144	+static __always_inline bool csd_lock_wait_toolong(struct __call_single_data csd, u64 ts0, u64 ts1, int *bug_id)
	145	+{
	146	+ int cpu = -1;
	147	+ int cpux;
	148	+ bool firsttime;
	149	+ u64 ts2, ts_delta;
	150	+ call_single_data_t *cpu_cur_csd;
	151	+ unsigned int flags = READ_ONCE(csd->flags);
	152	+
	153	+ if (!(flags & CSD_FLAG_LOCK)) {
	154	+ if (!unlikely(*bug_id))
	155	+ return true;
	156	+ cpu = csd_lock_wait_getcpu(csd);
	157	+ pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock.\n",
	158	+ *bug_id, raw_smp_processor_id(), cpu);
	159	+ return true;
	160	+ }
	161	+
	162	+ ts2 = sched_clock();
	163	+ ts_delta = ts2 - *ts1;
	164	+ if (likely(ts_delta <= CSD_LOCK_TIMEOUT))
	165	+ return false;
	166	+
	167	+ firsttime = !*bug_id;
	168	+ if (firsttime)
	169	+ *bug_id = atomic_inc_return(&csd_bug_count);
	170	+ cpu = csd_lock_wait_getcpu(csd);
	171	+ if (WARN_ONCE(cpu < 0 \|\| cpu >= nr_cpu_ids, "%s: cpu = %d\n", __func__, cpu))
	172	+ cpux = 0;
	173	+ else
	174	+ cpux = cpu;
	175	+ cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* Before func and info. */
	176	+ pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %llu ns for CPU#%02d %pS(%ps).\n",
	177	+ firsttime ? "Detected" : "Continued", *bug_id, raw_smp_processor_id(), ts2 - ts0,
	178	+ cpu, csd->func, csd->info);
	179	+ if (cpu_cur_csd && csd != cpu_cur_csd) {
	180	+ pr_alert("\tcsd: CSD lock (#%d) handling prior %pS(%ps) request.\n",
	181	+ *bug_id, READ_ONCE(per_cpu(cur_csd_func, cpux)),
	182	+ READ_ONCE(per_cpu(cur_csd_info, cpux)));
	183	+ } else {
	184	+ pr_alert("\tcsd: CSD lock (#%d) %s.\n",
	185	+ *bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request");
	186	+ }
	187	+ if (cpu >= 0) {
	188	+ if (!trigger_single_cpu_backtrace(cpu))
	189	+ dump_cpu_task(cpu);
	190	+ if (!cpu_cur_csd) {
	191	+ pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu);
	192	+ arch_send_call_function_single_ipi(cpu);
	193	+ }
	194	+ }
	195	+ dump_stack();
	196	+ *ts1 = ts2;
	197	+
	198	+ return false;
	199	+}
	200	+
99	201	/*
100	202	* csd_lock/csd_unlock used to serialize access to per-cpu csd resources
101	203	*
..	..	@@ -105,8 +207,28 @@
105	207	*/
106	208	static __always_inline void csd_lock_wait(struct __call_single_data *csd)
107	209	{
	210	+ int bug_id = 0;
	211	+ u64 ts0, ts1;
	212	+
	213	+ ts1 = ts0 = sched_clock();
	214	+ for (;;) {
	215	+ if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id))
	216	+ break;
	217	+ cpu_relax();
	218	+ }
	219	+ smp_acquire__after_ctrl_dep();
	220	+}
	221	+
	222	+#else
	223	+static void csd_lock_record(struct __call_single_data *csd)
	224	+{
	225	+}
	226	+
	227	+static __always_inline void csd_lock_wait(struct __call_single_data *csd)
	228	+{
108	229	smp_cond_load_acquire(&csd->flags, !(VAL & CSD_FLAG_LOCK));
109	230	}
	231	+#endif
110	232
111	233	static __always_inline void csd_lock(struct __call_single_data *csd)
112	234	{
..	..	@@ -133,37 +255,8 @@
133	255
134	256	static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
135	257
136		-/*
137		- * Insert a previously allocated call_single_data_t element
138		- * for execution on the given CPU. data must already have
139		- * ->func, ->info, and ->flags set.
140		- */
141		-static int generic_exec_single(int cpu, struct __call_single_data *csd,
142		- smp_call_func_t func, void *info)
	258	+void __smp_call_single_queue(int cpu, struct llist_node *node)
143	259	{
144		- if (cpu == smp_processor_id()) {
145		- unsigned long flags;
146		-
147		- /*
148		- * We can unlock early even for the synchronous on-stack case,
149		- * since we're doing this from the same CPU..
150		- */
151		- csd_unlock(csd);
152		- local_irq_save(flags);
153		- func(info);
154		- local_irq_restore(flags);
155		- return 0;
156		- }
157		-
158		-
159		- if ((unsigned)cpu >= nr_cpu_ids \|\| !cpu_online(cpu)) {
160		- csd_unlock(csd);
161		- return -ENXIO;
162		- }
163		-
164		- csd->func = func;
165		- csd->info = info;
166		-
167	260	/*
168	261	* The list addition should be visible before sending the IPI
169	262	* handler locks the list to pull the entry off it because of
..	..	@@ -175,8 +268,41 @@
175	268	* locking and barrier primitives. Generic code isn't really
176	269	* equipped to do the right thing...
177	270	*/
178		- if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
179		- arch_send_call_function_single_ipi(cpu);
	271	+ if (llist_add(node, &per_cpu(call_single_queue, cpu)))
	272	+ send_call_function_single_ipi(cpu);
	273	+}
	274	+
	275	+/*
	276	+ * Insert a previously allocated call_single_data_t element
	277	+ * for execution on the given CPU. data must already have
	278	+ * ->func, ->info, and ->flags set.
	279	+ */
	280	+static int generic_exec_single(int cpu, struct __call_single_data *csd)
	281	+{
	282	+ if (cpu == smp_processor_id()) {
	283	+ smp_call_func_t func = csd->func;
	284	+ void *info = csd->info;
	285	+ unsigned long flags;
	286	+
	287	+ /*
	288	+ * We can unlock early even for the synchronous on-stack case,
	289	+ * since we're doing this from the same CPU..
	290	+ */
	291	+ csd_lock_record(csd);
	292	+ csd_unlock(csd);
	293	+ local_irq_save(flags);
	294	+ func(info);
	295	+ csd_lock_record(NULL);
	296	+ local_irq_restore(flags);
	297	+ return 0;
	298	+ }
	299	+
	300	+ if ((unsigned)cpu >= nr_cpu_ids \|\| !cpu_online(cpu)) {
	301	+ csd_unlock(csd);
	302	+ return -ENXIO;
	303	+ }
	304	+
	305	+ __smp_call_single_queue(cpu, &csd->llist);
180	306
181	307	return 0;
182	308	}
..	..	@@ -208,9 +334,9 @@
208	334	*/
209	335	static void flush_smp_call_function_queue(bool warn_cpu_offline)
210	336	{
211		- struct llist_head *head;
212		- struct llist_node *entry;
213	337	call_single_data_t csd, csd_next;
	338	+ struct llist_node entry, prev;
	339	+ struct llist_head *head;
214	340	static bool warned;
215	341
216	342	lockdep_assert_irqs_disabled();
..	..	@@ -221,7 +347,7 @@
221	347
222	348	/* There shouldn't be any pending callbacks on an offline CPU. */
223	349	if (unlikely(warn_cpu_offline && !cpu_online(smp_processor_id()) &&
224		- !warned && !llist_empty(head))) {
	350	+ !warned && entry != NULL)) {
225	351	warned = true;
226	352	WARN(1, "IPI on offline CPU %d\n", smp_processor_id());
227	353
..	..	@@ -229,32 +355,106 @@
229	355	* We don't have to use the _safe() variant here
230	356	* because we are not invoking the IPI handlers yet.
231	357	*/
232		- llist_for_each_entry(csd, entry, llist)
233		- pr_warn("IPI callback %pS sent to offline CPU\n",
234		- csd->func);
235		- }
	358	+ llist_for_each_entry(csd, entry, llist) {
	359	+ switch (CSD_TYPE(csd)) {
	360	+ case CSD_TYPE_ASYNC:
	361	+ case CSD_TYPE_SYNC:
	362	+ case CSD_TYPE_IRQ_WORK:
	363	+ pr_warn("IPI callback %pS sent to offline CPU\n",
	364	+ csd->func);
	365	+ break;
236	366
237		- llist_for_each_entry_safe(csd, csd_next, entry, llist) {
238		- smp_call_func_t func = csd->func;
239		- void *info = csd->info;
	367	+ case CSD_TYPE_TTWU:
	368	+ pr_warn("IPI task-wakeup sent to offline CPU\n");
	369	+ break;
240	370
241		- /* Do we wait until after callback? */
242		- if (csd->flags & CSD_FLAG_SYNCHRONOUS) {
243		- func(info);
244		- csd_unlock(csd);
245		- } else {
246		- csd_unlock(csd);
247		- func(info);
	371	+ default:
	372	+ pr_warn("IPI callback, unknown type %d, sent to offline CPU\n",
	373	+ CSD_TYPE(csd));
	374	+ break;
	375	+ }
248	376	}
249	377	}
250	378
251	379	/*
252		- * Handle irq works queued remotely by irq_work_queue_on().
253		- * Smp functions above are typically synchronous so they
254		- * better run first since some other CPUs may be busy waiting
255		- * for them.
	380	+ * First; run all SYNC callbacks, people are waiting for us.
256	381	*/
257		- irq_work_run();
	382	+ prev = NULL;
	383	+ llist_for_each_entry_safe(csd, csd_next, entry, llist) {
	384	+ /* Do we wait until after callback? */
	385	+ if (CSD_TYPE(csd) == CSD_TYPE_SYNC) {
	386	+ smp_call_func_t func = csd->func;
	387	+ void *info = csd->info;
	388	+
	389	+ if (prev) {
	390	+ prev->next = &csd_next->llist;
	391	+ } else {
	392	+ entry = &csd_next->llist;
	393	+ }
	394	+
	395	+ csd_lock_record(csd);
	396	+ func(info);
	397	+ csd_unlock(csd);
	398	+ csd_lock_record(NULL);
	399	+ } else {
	400	+ prev = &csd->llist;
	401	+ }
	402	+ }
	403	+
	404	+ if (!entry)
	405	+ return;
	406	+
	407	+ /*
	408	+ * Second; run all !SYNC callbacks.
	409	+ */
	410	+ prev = NULL;
	411	+ llist_for_each_entry_safe(csd, csd_next, entry, llist) {
	412	+ int type = CSD_TYPE(csd);
	413	+
	414	+ if (type != CSD_TYPE_TTWU) {
	415	+ if (prev) {
	416	+ prev->next = &csd_next->llist;
	417	+ } else {
	418	+ entry = &csd_next->llist;
	419	+ }
	420	+
	421	+ if (type == CSD_TYPE_ASYNC) {
	422	+ smp_call_func_t func = csd->func;
	423	+ void *info = csd->info;
	424	+
	425	+ csd_lock_record(csd);
	426	+ csd_unlock(csd);
	427	+ func(info);
	428	+ csd_lock_record(NULL);
	429	+ } else if (type == CSD_TYPE_IRQ_WORK) {
	430	+ irq_work_single(csd);
	431	+ }
	432	+
	433	+ } else {
	434	+ prev = &csd->llist;
	435	+ }
	436	+ }
	437	+
	438	+ /*
	439	+ * Third; only CSD_TYPE_TTWU is left, issue those.
	440	+ */
	441	+ if (entry)
	442	+ sched_ttwu_pending(entry);
	443	+}
	444	+
	445	+void flush_smp_call_function_from_idle(void)
	446	+{
	447	+ unsigned long flags;
	448	+
	449	+ if (llist_empty(this_cpu_ptr(&call_single_queue)))
	450	+ return;
	451	+
	452	+ local_irq_save(flags);
	453	+ flush_smp_call_function_queue(true);
	454	+ if (local_softirq_pending())
	455	+ do_softirq();
	456	+
	457	+ local_irq_restore(flags);
258	458	}
259	459
260	460	/*
..	..	@@ -270,7 +470,7 @@
270	470	{
271	471	call_single_data_t *csd;
272	472	call_single_data_t csd_stack = {
273		- .flags = CSD_FLAG_LOCK \| CSD_FLAG_SYNCHRONOUS,
	473	+ .flags = CSD_FLAG_LOCK \| CSD_TYPE_SYNC,
274	474	};
275	475	int this_cpu;
276	476	int err;
..	..	@@ -290,13 +490,28 @@
290	490	WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
291	491	&& !oops_in_progress);
292	492
	493	+ /*
	494	+ * When @wait we can deadlock when we interrupt between llist_add() and
	495	+ * arch_send_call_function_ipi*(); when !@wait we can deadlock due to
	496	+ * csd_lock() on because the interrupt context uses the same csd
	497	+ * storage.
	498	+ */
	499	+ WARN_ON_ONCE(!in_task());
	500	+
293	501	csd = &csd_stack;
294	502	if (!wait) {
295	503	csd = this_cpu_ptr(&csd_data);
296	504	csd_lock(csd);
297	505	}
298	506
299		- err = generic_exec_single(cpu, csd, func, info);
	507	+ csd->func = func;
	508	+ csd->info = info;
	509	+#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
	510	+ csd->src = smp_processor_id();
	511	+ csd->dst = cpu;
	512	+#endif
	513	+
	514	+ err = generic_exec_single(cpu, csd);
300	515
301	516	if (wait)
302	517	csd_lock_wait(csd);
..	..	@@ -320,6 +535,11 @@
320	535	* (ie: embedded in an object) and is responsible for synchronizing it
321	536	* such that the IPIs performed on the @csd are strictly serialized.
322	537	*
	538	+ * If the function is called with one csd which has not yet been
	539	+ * processed by previous call to smp_call_function_single_async(), the
	540	+ * function will return immediately with -EBUSY showing that the csd
	541	+ * object is still in progress.
	542	+ *
323	543	* NOTE: Be careful, there is unfortunately no current debugging facility to
324	544	* validate the correctness of this serialization.
325	545	*/
..	..	@@ -329,14 +549,17 @@
329	549
330	550	preempt_disable();
331	551
332		- /* We could deadlock if we have to wait here with interrupts disabled! */
333		- if (WARN_ON_ONCE(csd->flags & CSD_FLAG_LOCK))
334		- csd_lock_wait(csd);
	552	+ if (csd->flags & CSD_FLAG_LOCK) {
	553	+ err = -EBUSY;
	554	+ goto out;
	555	+ }
335	556
336	557	csd->flags = CSD_FLAG_LOCK;
337	558	smp_wmb();
338	559
339		- err = generic_exec_single(cpu, csd, csd->func, csd->info);
	560	+ err = generic_exec_single(cpu, csd);
	561	+
	562	+out:
340	563	preempt_enable();
341	564
342	565	return err;
..	..	@@ -386,22 +609,9 @@
386	609	}
387	610	EXPORT_SYMBOL_GPL(smp_call_function_any);
388	611
389		-/**
390		- * smp_call_function_many(): Run a function on a set of other CPUs.
391		- * @mask: The set of cpus to run on (only runs on online subset).
392		- * @func: The function to run. This must be fast and non-blocking.
393		- * @info: An arbitrary pointer to pass to the function.
394		- * @wait: If true, wait (atomically) until function has completed
395		- * on other CPUs.
396		- *
397		- * If @wait is true, then returns once @func has returned.
398		- *
399		- * You must not call this function with disabled interrupts or from a
400		- * hardware interrupt handler or from a bottom half handler. Preemption
401		- * must be disabled when calling this function.
402		- */
403		-void smp_call_function_many(const struct cpumask *mask,
404		- smp_call_func_t func, void *info, bool wait)
	612	+static void smp_call_function_many_cond(const struct cpumask *mask,
	613	+ smp_call_func_t func, void *info,
	614	+ bool wait, smp_cond_func_t cond_func)
405	615	{
406	616	struct call_function_data *cfd;
407	617	int cpu, next_cpu, this_cpu = smp_processor_id();
..	..	@@ -414,6 +624,14 @@
414	624	*/
415	625	WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
416	626	&& !oops_in_progress && !early_boot_irqs_disabled);
	627	+
	628	+ /*
	629	+ * When @wait we can deadlock when we interrupt between llist_add() and
	630	+ * arch_send_call_function_ipi*(); when !@wait we can deadlock due to
	631	+ * csd_lock() on because the interrupt context uses the same csd
	632	+ * storage.
	633	+ */
	634	+ WARN_ON_ONCE(!in_task());
417	635
418	636	/* Try to fastpath. So, what's a CPU they want? Ignoring this one. */
419	637	cpu = cpumask_first_and(mask, cpu_online_mask);
..	..	@@ -431,7 +649,8 @@
431	649
432	650	/* Fastpath: do that cpu by itself. */
433	651	if (next_cpu >= nr_cpu_ids) {
434		- smp_call_function_single(cpu, func, info, wait);
	652	+ if (!cond_func \|\| cond_func(cpu, info))
	653	+ smp_call_function_single(cpu, func, info, wait);
435	654	return;
436	655	}
437	656
..	..	@@ -448,11 +667,18 @@
448	667	for_each_cpu(cpu, cfd->cpumask) {
449	668	call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu);
450	669
	670	+ if (cond_func && !cond_func(cpu, info))
	671	+ continue;
	672	+
451	673	csd_lock(csd);
452	674	if (wait)
453		- csd->flags \|= CSD_FLAG_SYNCHRONOUS;
	675	+ csd->flags \|= CSD_TYPE_SYNC;
454	676	csd->func = func;
455	677	csd->info = info;
	678	+#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
	679	+ csd->src = smp_processor_id();
	680	+ csd->dst = cpu;
	681	+#endif
456	682	if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
457	683	__cpumask_set_cpu(cpu, cfd->cpumask_ipi);
458	684	}
..	..	@@ -468,6 +694,26 @@
468	694	csd_lock_wait(csd);
469	695	}
470	696	}
	697	+}
	698	+
	699	+/**
	700	+ * smp_call_function_many(): Run a function on a set of other CPUs.
	701	+ * @mask: The set of cpus to run on (only runs on online subset).
	702	+ * @func: The function to run. This must be fast and non-blocking.
	703	+ * @info: An arbitrary pointer to pass to the function.
	704	+ * @wait: If true, wait (atomically) until function has completed
	705	+ * on other CPUs.
	706	+ *
	707	+ * If @wait is true, then returns once @func has returned.
	708	+ *
	709	+ * You must not call this function with disabled interrupts or from a
	710	+ * hardware interrupt handler or from a bottom half handler. Preemption
	711	+ * must be disabled when calling this function.
	712	+ */
	713	+void smp_call_function_many(const struct cpumask *mask,
	714	+ smp_call_func_t func, void *info, bool wait)
	715	+{
	716	+ smp_call_function_many_cond(mask, func, info, wait, NULL);
471	717	}
472	718	EXPORT_SYMBOL(smp_call_function_many);
473	719
..	..	@@ -486,13 +732,11 @@
486	732	* You must not call this function with disabled interrupts or from a
487	733	* hardware interrupt handler or from a bottom half handler.
488	734	*/
489		-int smp_call_function(smp_call_func_t func, void *info, int wait)
	735	+void smp_call_function(smp_call_func_t func, void *info, int wait)
490	736	{
491	737	preempt_disable();
492	738	smp_call_function_many(cpu_online_mask, func, info, wait);
493	739	preempt_enable();
494		-
495		- return 0;
496	740	}
497	741	EXPORT_SYMBOL(smp_call_function);
498	742
..	..	@@ -529,8 +773,7 @@
529	773	{
530	774	int nr_cpus;
531	775
532		- get_option(&str, &nr_cpus);
533		- if (nr_cpus > 0 && nr_cpus < nr_cpu_ids)
	776	+ if (get_option(&str, &nr_cpus) && nr_cpus > 0 && nr_cpus < nr_cpu_ids)
534	777	nr_cpu_ids = nr_cpus;
535	778
536	779	return 0;
..	..	@@ -563,20 +806,13 @@
563	806	void __init smp_init(void)
564	807	{
565	808	int num_nodes, num_cpus;
566		- unsigned int cpu;
567	809
568	810	idle_threads_init();
569	811	cpuhp_threads_init();
570	812
571	813	pr_info("Bringing up secondary CPUs ...\n");
572	814
573		- /* FIXME: This should be done in userspace --RR */
574		- for_each_present_cpu(cpu) {
575		- if (num_online_cpus() >= setup_max_cpus)
576		- break;
577		- if (!cpu_online(cpu))
578		- cpu_up(cpu);
579		- }
	815	+ bringup_nonboot_cpus(setup_max_cpus);
580	816
581	817	num_nodes = num_online_nodes();
582	818	num_cpus = num_online_cpus();
..	..	@@ -593,18 +829,16 @@
593	829	* early_boot_irqs_disabled is set. Use local_irq_save/restore() instead
594	830	* of local_irq_disable/enable().
595	831	*/
596		-int on_each_cpu(void (func) (void info), void *info, int wait)
	832	+void on_each_cpu(smp_call_func_t func, void *info, int wait)
597	833	{
598	834	unsigned long flags;
599		- int ret = 0;
600	835
601	836	preempt_disable();
602		- ret = smp_call_function(func, info, wait);
	837	+ smp_call_function(func, info, wait);
603	838	local_irq_save(flags);
604	839	func(info);
605	840	local_irq_restore(flags);
606	841	preempt_enable();
607		- return ret;
608	842	}
609	843	EXPORT_SYMBOL(on_each_cpu);
610	844
..	..	@@ -646,7 +880,7 @@
646	880	* for all the required CPUs to finish. This may include the local
647	881	* processor.
648	882	* @cond_func: A callback function that is passed a cpu id and
649		- * the the info parameter. The function is called
	883	+ * the info parameter. The function is called
650	884	* with preemption disabled. The function should
651	885	* return a blooean value indicating whether to IPI
652	886	* the specified CPU.
..	..	@@ -655,11 +889,6 @@
655	889	* @info: An arbitrary pointer to pass to both functions.
656	890	* @wait: If true, wait (atomically) until function has
657	891	* completed on other CPUs.
658		- * @gfp_flags: GFP flags to use when allocating the cpumask
659		- * used internally by the function.
660		- *
661		- * The function might sleep if the GFP flags indicates a non
662		- * atomic allocation is allowed.
663	892	*
664	893	* Preemption is disabled to protect against CPUs going offline but not online.
665	894	* CPUs going online during the call will not be seen or sent an IPI.
..	..	@@ -667,37 +896,27 @@
667	896	* You must not call this function with disabled interrupts or
668	897	* from a hardware interrupt handler or from a bottom half handler.
669	898	*/
670		-void on_each_cpu_cond(bool (cond_func)(int cpu, void info),
671		- smp_call_func_t func, void *info, bool wait,
672		- gfp_t gfp_flags)
	899	+void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func,
	900	+ void info, bool wait, const struct cpumask mask)
673	901	{
674		- cpumask_var_t cpus;
675		- int cpu, ret;
	902	+ int cpu = get_cpu();
676	903
677		- might_sleep_if(gfpflags_allow_blocking(gfp_flags));
	904	+ smp_call_function_many_cond(mask, func, info, wait, cond_func);
	905	+ if (cpumask_test_cpu(cpu, mask) && cond_func(cpu, info)) {
	906	+ unsigned long flags;
678	907
679		- if (likely(zalloc_cpumask_var(&cpus, (gfp_flags\|__GFP_NOWARN)))) {
680		- preempt_disable();
681		- for_each_online_cpu(cpu)
682		- if (cond_func(cpu, info))
683		- cpumask_set_cpu(cpu, cpus);
684		- on_each_cpu_mask(cpus, func, info, wait);
685		- preempt_enable();
686		- free_cpumask_var(cpus);
687		- } else {
688		- /*
689		- * No free cpumask, bother. No matter, we'll
690		- * just have to IPI them one by one.
691		- */
692		- preempt_disable();
693		- for_each_online_cpu(cpu)
694		- if (cond_func(cpu, info)) {
695		- ret = smp_call_function_single(cpu, func,
696		- info, wait);
697		- WARN_ON_ONCE(ret);
698		- }
699		- preempt_enable();
	908	+ local_irq_save(flags);
	909	+ func(info);
	910	+ local_irq_restore(flags);
700	911	}
	912	+ put_cpu();
	913	+}
	914	+EXPORT_SYMBOL(on_each_cpu_cond_mask);
	915	+
	916	+void on_each_cpu_cond(smp_cond_func_t cond_func, smp_call_func_t func,
	917	+ void *info, bool wait)
	918	+{
	919	+ on_each_cpu_cond_mask(cond_func, func, info, wait, cpu_online_mask);
701	920	}
702	921	EXPORT_SYMBOL(on_each_cpu_cond);
703	922
..	..	@@ -739,13 +958,37 @@
739	958	if (cpu == smp_processor_id())
740	959	continue;
741	960
742		- wake_up_if_idle(cpu);
	961	+#if IS_ENABLED(CONFIG_SUSPEND)
	962	+ if (s2idle_state == S2IDLE_STATE_ENTER \|\| cpu_active(cpu))
	963	+#endif
	964	+ wake_up_if_idle(cpu);
743	965	}
744	966	preempt_enable();
745	967	}
746	968	EXPORT_SYMBOL_GPL(wake_up_all_idle_cpus);
747	969
748	970	/**
	971	+ * wake_up_all_online_idle_cpus - break all online cpus out of idle
	972	+ * wake_up_all_online_idle_cpus try to break all online cpus which is in idle
	973	+ * state even including idle polling cpus, for non-idle cpus, we will do nothing
	974	+ * for them.
	975	+ */
	976	+void wake_up_all_online_idle_cpus(void)
	977	+{
	978	+ int cpu;
	979	+
	980	+ preempt_disable();
	981	+ for_each_online_cpu(cpu) {
	982	+ if (cpu == smp_processor_id())
	983	+ continue;
	984	+
	985	+ wake_up_if_idle(cpu);
	986	+ }
	987	+ preempt_enable();
	988	+}
	989	+EXPORT_SYMBOL_GPL(wake_up_all_online_idle_cpus);
	990	+
	991	+/**
749	992	* smp_call_on_cpu - Call a function on a specific cpu
750	993	*
751	994	* Used to call a function on a specific cpu and wait for it to return.