forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-01-05 071106ecf68c401173c58808b1cf5f68cc50d390 
 kernel/kernel/sched/topology.c
....@@ -4,11 +4,16 @@
44  */
55 #include "sched.h"
66 
7
+#include <trace/hooks/sched.h>
8
+
79 DEFINE_MUTEX(sched_domains_mutex);
10
+#ifdef CONFIG_LOCKDEP
11
+EXPORT_SYMBOL_GPL(sched_domains_mutex);
12
+#endif
813 
914 /* Protected by sched_domains_mutex: */
10
-cpumask_var_t sched_domains_tmpmask;
11
-cpumask_var_t sched_domains_tmpmask2;
15
+static cpumask_var_t sched_domains_tmpmask;
16
+static cpumask_var_t sched_domains_tmpmask2;
1217 
1318 #ifdef CONFIG_SCHED_DEBUG
1419 
....@@ -25,22 +30,22 @@
2530 	return sched_debug_enabled;
2631 }
2732 
33
+#define SD_FLAG(_name, mflags) [__##_name] = { .meta_flags = mflags, .name = #_name },
34
+const struct sd_flag_debug sd_flag_debug[] = {
35
+#include <linux/sched/sd_flags.h>
36
+};
37
+#undef SD_FLAG
38
+
2839 static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
2940 				  struct cpumask *groupmask)
3041 {
3142 	struct sched_group *group = sd->groups;
43
+	unsigned long flags = sd->flags;
44
+	unsigned int idx;
3245 
3346 	cpumask_clear(groupmask);
3447 
3548 	printk(KERN_DEBUG "%*s domain-%d: ", level, "", level);
36
-
37
-	if (!(sd->flags & SD_LOAD_BALANCE)) {
38
-		printk("does not load-balance\n");
39
-		if (sd->parent)
40
-			printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain has parent");
41
-		return -1;
42
-	}
43
-
4449 	printk(KERN_CONT "span=%*pbl level=%s\n",
4550 	       cpumask_pr_args(sched_domain_span(sd)), sd->name);
4651 
....@@ -49,6 +54,21 @@
4954 	}
5055 	if (group && !cpumask_test_cpu(cpu, sched_group_span(group))) {
5156 		printk(KERN_ERR "ERROR: domain->groups does not contain CPU%d\n", cpu);
57
+	}
58
+
59
+	for_each_set_bit(idx, &flags, __SD_FLAG_CNT) {
60
+		unsigned int flag = BIT(idx);
61
+		unsigned int meta_flags = sd_flag_debug[idx].meta_flags;
62
+
63
+		if ((meta_flags & SDF_SHARED_CHILD) && sd->child &&
64
+		    !(sd->child->flags & flag))
65
+			printk(KERN_ERR "ERROR: flag %s set here but not in child\n",
66
+			       sd_flag_debug[idx].name);
67
+
68
+		if ((meta_flags & SDF_SHARED_PARENT) && sd->parent &&
69
+		    !(sd->parent->flags & flag))
70
+			printk(KERN_ERR "ERROR: flag %s set here but not in parent\n",
71
+			       sd_flag_debug[idx].name);
5272 	}
5373 
5474 	printk(KERN_DEBUG "%*s groups:", level + 1, "");
....@@ -145,23 +165,22 @@
145165 }
146166 #endif /* CONFIG_SCHED_DEBUG */
147167 
168
+/* Generate a mask of SD flags with the SDF_NEEDS_GROUPS metaflag */
169
+#define SD_FLAG(name, mflags) (name * !!((mflags) & SDF_NEEDS_GROUPS)) |
170
+static const unsigned int SD_DEGENERATE_GROUPS_MASK =
171
+#include <linux/sched/sd_flags.h>
172
+0;
173
+#undef SD_FLAG
174
+
148175 static int sd_degenerate(struct sched_domain *sd)
149176 {
150177 	if (cpumask_weight(sched_domain_span(sd)) == 1)
151178 		return 1;
152179 
153180 	/* Following flags need at least 2 groups */
154
-	if (sd->flags & (SD_LOAD_BALANCE |
155
-			 SD_BALANCE_NEWIDLE |
156
-			 SD_BALANCE_FORK |
157
-			 SD_BALANCE_EXEC |
158
-			 SD_SHARE_CPUCAPACITY |
159
-			 SD_ASYM_CPUCAPACITY |
160
-			 SD_SHARE_PKG_RESOURCES |
161
-			 SD_SHARE_POWERDOMAIN)) {
162
-		if (sd->groups != sd->groups->next)
163
-			return 0;
164
-	}
181
+	if ((sd->flags & SD_DEGENERATE_GROUPS_MASK) &&
182
+	    (sd->groups != sd->groups->next))
183
+		return 0;
165184 
166185 	/* Following flags don't use groups */
167186 	if (sd->flags & (SD_WAKE_AFFINE))
....@@ -182,36 +201,24 @@
182201 		return 0;
183202 
184203 	/* Flags needing groups don't count if only 1 group in parent */
185
-	if (parent->groups == parent->groups->next) {
186
-		pflags &= ~(SD_LOAD_BALANCE |
187
-				SD_BALANCE_NEWIDLE |
188
-				SD_BALANCE_FORK |
189
-				SD_BALANCE_EXEC |
190
-				SD_ASYM_CPUCAPACITY |
191
-				SD_SHARE_CPUCAPACITY |
192
-				SD_SHARE_PKG_RESOURCES |
193
-				SD_PREFER_SIBLING |
194
-				SD_SHARE_POWERDOMAIN);
195
-		if (nr_node_ids == 1)
196
-			pflags &= ~SD_SERIALIZE;
197
-	}
204
+	if (parent->groups == parent->groups->next)
205
+		pflags &= ~SD_DEGENERATE_GROUPS_MASK;
206
+
198207 	if (~cflags & pflags)
199208 		return 0;
200209 
201210 	return 1;
202211 }
203212 
204
-DEFINE_STATIC_KEY_FALSE(sched_energy_present);
205
-
206
-#ifdef CONFIG_ENERGY_MODEL
207213 #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
214
+DEFINE_STATIC_KEY_FALSE(sched_energy_present);
208215 unsigned int sysctl_sched_energy_aware = 1;
209216 DEFINE_MUTEX(sched_energy_mutex);
210217 bool sched_energy_update;
211218 
212219 #ifdef CONFIG_PROC_SYSCTL
213220 int sched_energy_aware_handler(struct ctl_table *table, int write,
214
-			 void __user *buffer, size_t *lenp, loff_t *ppos)
221
+		void *buffer, size_t *lenp, loff_t *ppos)
215222 {
216223 	int ret, state;
217224 
....@@ -233,7 +240,6 @@
233240 	return ret;
234241 }
235242 #endif
236
-#endif /* defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) */
237243 
238244 static void free_pd(struct perf_domain *pd)
239245 {
....@@ -285,10 +291,10 @@
285291 	printk(KERN_DEBUG "root_domain %*pbl:", cpumask_pr_args(cpu_map));
286292 
287293 	while (pd) {
288
-		printk(KERN_CONT " pd%d:{ cpus=%*pbl nr_cstate=%d }",
294
+		printk(KERN_CONT " pd%d:{ cpus=%*pbl nr_pstate=%d }",
289295 				cpumask_first(perf_domain_span(pd)),
290296 				cpumask_pr_args(perf_domain_span(pd)),
291
-				em_pd_nr_cap_states(pd->em_pd));
297
+				em_pd_nr_perf_states(pd->em_pd));
292298 		pd = pd->next;
293299 	}
294300 
....@@ -320,44 +326,55 @@
320326  * EAS can be used on a root domain if it meets all the following conditions:
321327  *    1. an Energy Model (EM) is available;
322328  *    2. the SD_ASYM_CPUCAPACITY flag is set in the sched_domain hierarchy.
323
- *    3. the EM complexity is low enough to keep scheduling overheads low;
329
+ *    3. no SMT is detected.
330
+ *    4. the EM complexity is low enough to keep scheduling overheads low;
324331  *
325332  * The complexity of the Energy Model is defined as:
326333  *
327
- *              C = nr_pd * (nr_cpus + nr_cs)
334
+ *              C = nr_pd * (nr_cpus + nr_ps)
328335  *
329336  * with parameters defined as:
330337  *  - nr_pd:    the number of performance domains
331338  *  - nr_cpus:  the number of CPUs
332
- *  - nr_cs:    the sum of the number of capacity states of all performance
339
+ *  - nr_ps:    the sum of the number of performance states of all performance
333340  *              domains (for example, on a system with 2 performance domains,
334
- *              with 10 capacity states each, nr_cs = 2 * 10 = 20).
341
+ *              with 10 performance states each, nr_ps = 2 * 10 = 20).
335342  *
336343  * It is generally not a good idea to use such a model in the wake-up path on
337344  * very complex platforms because of the associated scheduling overheads. The
338345  * arbitrary constraint below prevents that. It makes EAS usable up to 16 CPUs
339
- * with per-CPU DVFS and less than 8 capacity states each, for example.
346
+ * with per-CPU DVFS and less than 8 performance states each, for example.
340347  */
341348 #define EM_MAX_COMPLEXITY 2048
342349 
343350 static bool build_perf_domains(const struct cpumask *cpu_map)
344351 {
345
-	int i, nr_pd = 0, nr_cs = 0, nr_cpus = cpumask_weight(cpu_map);
352
+	int i, nr_pd = 0, nr_ps = 0, nr_cpus = cpumask_weight(cpu_map);
346353 	struct perf_domain *pd = NULL, *tmp;
347354 	int cpu = cpumask_first(cpu_map);
348355 	struct root_domain *rd = cpu_rq(cpu)->rd;
356
+	bool eas_check = false;
349357 
350
-#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
351358 	if (!sysctl_sched_energy_aware)
352359 		goto free;
353
-#endif
354360 
355
-	/* EAS is enabled for asymmetric CPU capacity topologies. */
356
-	if (!per_cpu(sd_asym_cpucapacity, cpu)) {
361
+	/*
362
+	 * EAS is enabled for asymmetric CPU capacity topologies.
363
+	 * Allow vendor to override if desired.
364
+	 */
365
+	trace_android_rvh_build_perf_domains(&eas_check);
366
+	if (!per_cpu(sd_asym_cpucapacity, cpu) && !eas_check) {
357367 		if (sched_debug()) {
358368 			pr_info("rd %*pbl: CPUs do not have asymmetric capacities\n",
359369 					cpumask_pr_args(cpu_map));
360370 		}
371
+		goto free;
372
+	}
373
+
374
+	/* EAS definitely does *not* handle SMT */
375
+	if (sched_smt_active()) {
376
+		pr_warn("rd %*pbl: Disabling EAS, SMT is not supported\n",
377
+			cpumask_pr_args(cpu_map));
361378 		goto free;
362379 	}
363380 
....@@ -374,15 +391,15 @@
374391 		pd = tmp;
375392 
376393 		/*
377
-		 * Count performance domains and capacity states for the
394
+		 * Count performance domains and performance states for the
378395 		 * complexity check.
379396 		 */
380397 		nr_pd++;
381
-		nr_cs += em_pd_nr_cap_states(pd->em_pd);
398
+		nr_ps += em_pd_nr_perf_states(pd->em_pd);
382399 	}
383400 
384401 	/* Bail out if the Energy Model complexity is too high. */
385
-	if (nr_pd * (nr_cs + nr_cpus) > EM_MAX_COMPLEXITY) {
402
+	if (nr_pd * (nr_ps + nr_cpus) > EM_MAX_COMPLEXITY) {
386403 		WARN(1, "rd %*pbl: Failed to start EAS, EM complexity is too high\n",
387404 						cpumask_pr_args(cpu_map));
388405 		goto free;
....@@ -409,7 +426,7 @@
409426 }
410427 #else
411428 static void free_pd(struct perf_domain *pd) { }
412
-#endif /* CONFIG_ENERGY_MODEL */
429
+#endif /* CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL*/
413430 
414431 static void free_rootdomain(struct rcu_head *rcu)
415432 {
....@@ -459,7 +476,7 @@
459476 	raw_spin_unlock_irqrestore(&rq->lock, flags);
460477 
461478 	if (old_rd)
462
-		call_rcu_sched(&old_rd->rcu, free_rootdomain);
479
+		call_rcu(&old_rd->rcu, free_rootdomain);
463480 }
464481 
465482 void sched_get_rd(struct root_domain *rd)
....@@ -472,7 +489,7 @@
472489 	if (!atomic_dec_and_test(&rd->refcount))
473490 		return;
474491 
475
-	call_rcu_sched(&rd->rcu, free_rootdomain);
492
+	call_rcu(&rd->rcu, free_rootdomain);
476493 }
477494 
478495 static int init_rootdomain(struct root_domain *rd)
....@@ -498,9 +515,6 @@
498515 
499516 	if (cpupri_init(&rd->cpupri) != 0)
500517 		goto free_cpudl;
501
-
502
-	init_max_cpu_capacity(&rd->max_cpu_capacity);
503
-
504518 	return 0;
505519 
506520 free_cpudl:
....@@ -606,13 +620,13 @@
606620  * the cpumask of the domain), this allows us to quickly tell if
607621  * two CPUs are in the same cache domain, see cpus_share_cache().
608622  */
609
-DEFINE_PER_CPU(struct sched_domain *, sd_llc);
623
+DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc);
610624 DEFINE_PER_CPU(int, sd_llc_size);
611625 DEFINE_PER_CPU(int, sd_llc_id);
612
-DEFINE_PER_CPU(struct sched_domain_shared *, sd_llc_shared);
613
-DEFINE_PER_CPU(struct sched_domain *, sd_numa);
614
-DEFINE_PER_CPU(struct sched_domain *, sd_asym_packing);
615
-DEFINE_PER_CPU(struct sched_domain *, sd_asym_cpucapacity);
626
+DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
627
+DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa);
628
+DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
629
+DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
616630 DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity);
617631 
618632 static void update_top_cache_domain(int cpu)
....@@ -1050,6 +1064,7 @@
10501064 	struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
10511065 	struct sched_domain *child = sd->child;
10521066 	struct sched_group *sg;
1067
+	bool already_visited;
10531068 
10541069 	if (child)
10551070 		cpu = cpumask_first(sched_domain_span(child));
....@@ -1057,9 +1072,14 @@
10571072 	sg = *per_cpu_ptr(sdd->sg, cpu);
10581073 	sg->sgc = *per_cpu_ptr(sdd->sgc, cpu);
10591074 
1060
-	/* For claim_allocations: */
1061
-	atomic_inc(&sg->ref);
1062
-	atomic_inc(&sg->sgc->ref);
1075
+	/* Increase refcounts for claim_allocations: */
1076
+	already_visited = atomic_inc_return(&sg->ref) > 1;
1077
+	/* sgc visits should follow a similar trend as sg */
1078
+	WARN_ON(already_visited != (atomic_inc_return(&sg->sgc->ref) > 1));
1079
+
1080
+	/* If we have already visited that group, it's already initialized. */
1081
+	if (already_visited)
1082
+		return sg;
10631083 
10641084 	if (child) {
10651085 		cpumask_copy(sched_group_span(sg), sched_domain_span(child));
....@@ -1078,8 +1098,8 @@
10781098 
10791099 /*
10801100  * build_sched_groups will build a circular linked list of the groups
1081
- * covered by the given span, and will set each group's ->cpumask correctly,
1082
- * and ->cpu_capacity to 0.
1101
+ * covered by the given span, will set each group's ->cpumask correctly,
1102
+ * and will initialize their ->sgc.
10831103  *
10841104  * Assumes the sched_domain tree is fully constructed
10851105  */
....@@ -1186,16 +1206,13 @@
11861206 	if (!attr || attr->relax_domain_level < 0) {
11871207 		if (default_relax_domain_level < 0)
11881208 			return;
1189
-		else
1190
-			request = default_relax_domain_level;
1209
+		request = default_relax_domain_level;
11911210 	} else
11921211 		request = attr->relax_domain_level;
1193
-	if (request < sd->level) {
1212
+
1213
+	if (sd->level > request) {
11941214 		/* Turn off idle balance on this domain: */
11951215 		sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
1196
-	} else {
1197
-		/* Turn on idle balance on this domain: */
1198
-		sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
11991216 	}
12001217 }
12011218 
....@@ -1209,13 +1226,13 @@
12091226 	case sa_rootdomain:
12101227 		if (!atomic_read(&d->rd->refcount))
12111228 			free_rootdomain(&d->rd->rcu);
1212
-		/* Fall through */
1229
+		fallthrough;
12131230 	case sa_sd:
12141231 		free_percpu(d->sd);
1215
-		/* Fall through */
1232
+		fallthrough;
12161233 	case sa_sd_storage:
12171234 		__sdt_free(cpu_map);
1218
-		/* Fall through */
1235
+		fallthrough;
12191236 	case sa_none:
12201237 		break;
12211238 	}
....@@ -1269,6 +1286,7 @@
12691286 int				sched_max_numa_distance;
12701287 static int			*sched_domains_numa_distance;
12711288 static struct cpumask		***sched_domains_numa_masks;
1289
+int __read_mostly		node_reclaim_distance = RECLAIM_DISTANCE;
12721290 #endif
12731291 
12741292 /*
....@@ -1281,7 +1299,6 @@
12811299  *   SD_SHARE_CPUCAPACITY   - describes SMT topologies
12821300  *   SD_SHARE_PKG_RESOURCES - describes shared caches
12831301  *   SD_NUMA                - describes NUMA topologies
1284
- *   SD_SHARE_POWERDOMAIN   - describes shared power domain
12851302  *
12861303  * Odd one out, which beside describing the topology has a quirk also
12871304  * prescribes the desired behaviour that goes along with it:
....@@ -1292,8 +1309,7 @@
12921309 	(SD_SHARE_CPUCAPACITY	|	\
12931310 	 SD_SHARE_PKG_RESOURCES |	\
12941311 	 SD_NUMA		|	\
1295
-	 SD_ASYM_PACKING	|	\
1296
-	 SD_SHARE_POWERDOMAIN)
1312
+	 SD_ASYM_PACKING)
12971313 
12981314 static struct sched_domain *
12991315 sd_init(struct sched_domain_topology_level *tl,
....@@ -1325,18 +1341,12 @@
13251341 	*sd = (struct sched_domain){
13261342 		.min_interval		= sd_weight,
13271343 		.max_interval		= 2*sd_weight,
1328
-		.busy_factor		= 32,
1329
-		.imbalance_pct		= 125,
1344
+		.busy_factor		= 16,
1345
+		.imbalance_pct		= 117,
13301346 
13311347 		.cache_nice_tries	= 0,
1332
-		.busy_idx		= 0,
1333
-		.idle_idx		= 0,
1334
-		.newidle_idx		= 0,
1335
-		.wake_idx		= 0,
1336
-		.forkexec_idx		= 0,
13371348 
1338
-		.flags			= 1*SD_LOAD_BALANCE
1339
-					| 1*SD_BALANCE_NEWIDLE
1349
+		.flags			= 1*SD_BALANCE_NEWIDLE
13401350 					| 1*SD_BALANCE_EXEC
13411351 					| 1*SD_BALANCE_FORK
13421352 					| 0*SD_BALANCE_WAKE
....@@ -1351,7 +1361,6 @@
13511361 
13521362 		.last_balance		= jiffies,
13531363 		.balance_interval	= sd_weight,
1354
-		.smt_gain		= 0,
13551364 		.max_newidle_lb_cost	= 0,
13561365 		.next_decay_max_lb_cost	= jiffies,
13571366 		.child			= child,
....@@ -1367,37 +1376,24 @@
13671376 	 * Convert topological properties into behaviour.
13681377 	 */
13691378 
1370
-	if (sd->flags & SD_ASYM_CPUCAPACITY) {
1371
-		struct sched_domain *t = sd;
1372
-
1373
-		/*
1374
-		 * Don't attempt to spread across CPUs of different capacities.
1375
-		 */
1376
-		if (sd->child)
1377
-			sd->child->flags &= ~SD_PREFER_SIBLING;
1378
-
1379
-		for_each_lower_domain(t)
1380
-			t->flags |= SD_BALANCE_WAKE;
1381
-	}
1379
+	/* Don't attempt to spread across CPUs of different capacities. */
1380
+	if ((sd->flags & SD_ASYM_CPUCAPACITY) && sd->child)
1381
+		sd->child->flags &= ~SD_PREFER_SIBLING;
13821382 
13831383 	if (sd->flags & SD_SHARE_CPUCAPACITY) {
13841384 		sd->imbalance_pct = 110;
1385
-		sd->smt_gain = 1178; /* ~15% */
13861385 
13871386 	} else if (sd->flags & SD_SHARE_PKG_RESOURCES) {
13881387 		sd->imbalance_pct = 117;
13891388 		sd->cache_nice_tries = 1;
1390
-		sd->busy_idx = 2;
13911389 
13921390 #ifdef CONFIG_NUMA
13931391 	} else if (sd->flags & SD_NUMA) {
13941392 		sd->cache_nice_tries = 2;
1395
-		sd->busy_idx = 3;
1396
-		sd->idle_idx = 2;
13971393 
13981394 		sd->flags &= ~SD_PREFER_SIBLING;
13991395 		sd->flags |= SD_SERIALIZE;
1400
-		if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) {
1396
+		if (sched_domains_numa_distance[tl->numa_level] > node_reclaim_distance) {
14011397 			sd->flags &= ~(SD_BALANCE_EXEC |
14021398 				       SD_BALANCE_FORK |
14031399 				       SD_WAKE_AFFINE);
....@@ -1406,8 +1402,6 @@
14061402 #endif
14071403 	} else {
14081404 		sd->cache_nice_tries = 1;
1409
-		sd->busy_idx = 2;
1410
-		sd->idle_idx = 1;
14111405 	}
14121406 
14131407 	/*
....@@ -1548,66 +1542,58 @@
15481542 	}
15491543 }
15501544 
1545
+
1546
+#define NR_DISTANCE_VALUES (1 << DISTANCE_BITS)
1547
+
15511548 void sched_init_numa(void)
15521549 {
1553
-	int next_distance, curr_distance = node_distance(0, 0);
15541550 	struct sched_domain_topology_level *tl;
1555
-	int level = 0;
1556
-	int i, j, k;
1557
-
1558
-	sched_domains_numa_distance = kzalloc(sizeof(int) * (nr_node_ids + 1), GFP_KERNEL);
1559
-	if (!sched_domains_numa_distance)
1560
-		return;
1561
-
1562
-	/* Includes NUMA identity node at level 0. */
1563
-	sched_domains_numa_distance[level++] = curr_distance;
1564
-	sched_domains_numa_levels = level;
1551
+	unsigned long *distance_map;
1552
+	int nr_levels = 0;
1553
+	int i, j;
15651554 
15661555 	/*
15671556 	 * O(nr_nodes^2) deduplicating selection sort -- in order to find the
15681557 	 * unique distances in the node_distance() table.
1569
-	 *
1570
-	 * Assumes node_distance(0,j) includes all distances in
1571
-	 * node_distance(i,j) in order to avoid cubic time.
15721558 	 */
1573
-	next_distance = curr_distance;
1559
+	distance_map = bitmap_alloc(NR_DISTANCE_VALUES, GFP_KERNEL);
1560
+	if (!distance_map)
1561
+		return;
1562
+
1563
+	bitmap_zero(distance_map, NR_DISTANCE_VALUES);
15741564 	for (i = 0; i < nr_node_ids; i++) {
15751565 		for (j = 0; j < nr_node_ids; j++) {
1576
-			for (k = 0; k < nr_node_ids; k++) {
1577
-				int distance = node_distance(i, k);
1566
+			int distance = node_distance(i, j);
15781567 
1579
-				if (distance > curr_distance &&
1580
-				    (distance < next_distance ||
1581
-				     next_distance == curr_distance))
1582
-					next_distance = distance;
1583
-
1584
-				/*
1585
-				 * While not a strong assumption it would be nice to know
1586
-				 * about cases where if node A is connected to B, B is not
1587
-				 * equally connected to A.
1588
-				 */
1589
-				if (sched_debug() && node_distance(k, i) != distance)
1590
-					sched_numa_warn("Node-distance not symmetric");
1591
-
1592
-				if (sched_debug() && i && !find_numa_distance(distance))
1593
-					sched_numa_warn("Node-0 not representative");
1568
+			if (distance < LOCAL_DISTANCE || distance >= NR_DISTANCE_VALUES) {
1569
+				sched_numa_warn("Invalid distance value range");
1570
+				return;
15941571 			}
1595
-			if (next_distance != curr_distance) {
1596
-				sched_domains_numa_distance[level++] = next_distance;
1597
-				sched_domains_numa_levels = level;
1598
-				curr_distance = next_distance;
1599
-			} else break;
1600
-		}
16011572 
1602
-		/*
1603
-		 * In case of sched_debug() we verify the above assumption.
1604
-		 */
1605
-		if (!sched_debug())
1606
-			break;
1573
+			bitmap_set(distance_map, distance, 1);
1574
+		}
1575
+	}
1576
+	/*
1577
+	 * We can now figure out how many unique distance values there are and
1578
+	 * allocate memory accordingly.
1579
+	 */
1580
+	nr_levels = bitmap_weight(distance_map, NR_DISTANCE_VALUES);
1581
+
1582
+	sched_domains_numa_distance = kcalloc(nr_levels, sizeof(int), GFP_KERNEL);
1583
+	if (!sched_domains_numa_distance) {
1584
+		bitmap_free(distance_map);
1585
+		return;
16071586 	}
16081587 
1588
+	for (i = 0, j = 0; i < nr_levels; i++, j++) {
1589
+		j = find_next_bit(distance_map, NR_DISTANCE_VALUES, j);
1590
+		sched_domains_numa_distance[i] = j;
1591
+	}
1592
+
1593
+	bitmap_free(distance_map);
1594
+
16091595 	/*
1610
-	 * 'level' contains the number of unique distances
1596
+	 * 'nr_levels' contains the number of unique distances
16111597 	 *
16121598 	 * The sched_domains_numa_distance[] array includes the actual distance
16131599 	 * numbers.
....@@ -1616,15 +1602,15 @@
16161602 	/*
16171603 	 * Here, we should temporarily reset sched_domains_numa_levels to 0.
16181604 	 * If it fails to allocate memory for array sched_domains_numa_masks[][],
1619
-	 * the array will contain less then 'level' members. This could be
1605
+	 * the array will contain less then 'nr_levels' members. This could be
16201606 	 * dangerous when we use it to iterate array sched_domains_numa_masks[][]
16211607 	 * in other functions.
16221608 	 *
1623
-	 * We reset it to 'level' at the end of this function.
1609
+	 * We reset it to 'nr_levels' at the end of this function.
16241610 	 */
16251611 	sched_domains_numa_levels = 0;
16261612 
1627
-	sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL);
1613
+	sched_domains_numa_masks = kzalloc(sizeof(void *) * nr_levels, GFP_KERNEL);
16281614 	if (!sched_domains_numa_masks)
16291615 		return;
16301616 
....@@ -1632,7 +1618,7 @@
16321618 	 * Now for each level, construct a mask per node which contains all
16331619 	 * CPUs of nodes that are that many hops away from us.
16341620 	 */
1635
-	for (i = 0; i < level; i++) {
1621
+	for (i = 0; i < nr_levels; i++) {
16361622 		sched_domains_numa_masks[i] =
16371623 			kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL);
16381624 		if (!sched_domains_numa_masks[i])
....@@ -1640,12 +1626,17 @@
16401626 
16411627 		for (j = 0; j < nr_node_ids; j++) {
16421628 			struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL);
1629
+			int k;
1630
+
16431631 			if (!mask)
16441632 				return;
16451633 
16461634 			sched_domains_numa_masks[i][j] = mask;
16471635 
16481636 			for_each_node(k) {
1637
+				if (sched_debug() && (node_distance(j, k) != node_distance(k, j)))
1638
+					sched_numa_warn("Node-distance not symmetric");
1639
+
16491640 				if (node_distance(j, k) > sched_domains_numa_distance[i])
16501641 					continue;
16511642 
....@@ -1657,7 +1648,7 @@
16571648 	/* Compute default topology size */
16581649 	for (i = 0; sched_domain_topology[i].mask; i++);
16591650 
1660
-	tl = kzalloc((i + level + 1) *
1651
+	tl = kzalloc((i + nr_levels + 1) *
16611652 			sizeof(struct sched_domain_topology_level), GFP_KERNEL);
16621653 	if (!tl)
16631654 		return;
....@@ -1680,7 +1671,7 @@
16801671 	/*
16811672 	 * .. and append 'j' levels of NUMA goodness.
16821673 	 */
1683
-	for (j = 1; j < level; i++, j++) {
1674
+	for (j = 1; j < nr_levels; i++, j++) {
16841675 		tl[i] = (struct sched_domain_topology_level){
16851676 			.mask = sd_numa_mask,
16861677 			.sd_flags = cpu_numa_flags,
....@@ -1692,8 +1683,8 @@
16921683 
16931684 	sched_domain_topology = tl;
16941685 
1695
-	sched_domains_numa_levels = level;
1696
-	sched_max_numa_distance = sched_domains_numa_distance[level - 1];
1686
+	sched_domains_numa_levels = nr_levels;
1687
+	sched_max_numa_distance = sched_domains_numa_distance[nr_levels - 1];
16971688 
16981689 	init_numa_topology_type();
16991690 }
....@@ -1719,6 +1710,26 @@
17191710 		for (j = 0; j < nr_node_ids; j++)
17201711 			cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
17211712 	}
1713
+}
1714
+
1715
+/*
1716
+ * sched_numa_find_closest() - given the NUMA topology, find the cpu
1717
+ *                             closest to @cpu from @cpumask.
1718
+ * cpumask: cpumask to find a cpu from
1719
+ * cpu: cpu to be close to
1720
+ *
1721
+ * returns: cpu, or nr_cpu_ids when nothing found.
1722
+ */
1723
+int sched_numa_find_closest(const struct cpumask *cpus, int cpu)
1724
+{
1725
+	int i, j = cpu_to_node(cpu);
1726
+
1727
+	for (i = 0; i < sched_domains_numa_levels; i++) {
1728
+		cpu = cpumask_any_and(cpus, sched_domains_numa_masks[i][j]);
1729
+		if (cpu < nr_cpu_ids)
1730
+			return cpu;
1731
+	}
1732
+	return nr_cpu_ids;
17221733 }
17231734 
17241735 #endif /* CONFIG_NUMA */
....@@ -1859,6 +1870,42 @@
18591870 }
18601871 
18611872 /*
1873
+ * Ensure topology masks are sane, i.e. there are no conflicts (overlaps) for
1874
+ * any two given CPUs at this (non-NUMA) topology level.
1875
+ */
1876
+static bool topology_span_sane(struct sched_domain_topology_level *tl,
1877
+			      const struct cpumask *cpu_map, int cpu)
1878
+{
1879
+	int i;
1880
+
1881
+	/* NUMA levels are allowed to overlap */
1882
+	if (tl->flags & SDTL_OVERLAP)
1883
+		return true;
1884
+
1885
+	/*
1886
+	 * Non-NUMA levels cannot partially overlap - they must be either
1887
+	 * completely equal or completely disjoint. Otherwise we can end up
1888
+	 * breaking the sched_group lists - i.e. a later get_group() pass
1889
+	 * breaks the linking done for an earlier span.
1890
+	 */
1891
+	for_each_cpu(i, cpu_map) {
1892
+		if (i == cpu)
1893
+			continue;
1894
+		/*
1895
+		 * We should 'and' all those masks with 'cpu_map' to exactly
1896
+		 * match the topology we're about to build, but that can only
1897
+		 * remove CPUs, which only lessens our ability to detect
1898
+		 * overlaps
1899
+		 */
1900
+		if (!cpumask_equal(tl->mask(cpu), tl->mask(i)) &&
1901
+		    cpumask_intersects(tl->mask(cpu), tl->mask(i)))
1902
+			return false;
1903
+	}
1904
+
1905
+	return true;
1906
+}
1907
+
1908
+/*
18621909  * Find the sched_domain_topology_level where all CPU capacities are visible
18631910  * for all CPUs.
18641911  */
....@@ -1871,10 +1918,10 @@
18711918 	unsigned long cap;
18721919 
18731920 	/* Is there any asymmetry? */
1874
-	cap = arch_scale_cpu_capacity(NULL, cpumask_first(cpu_map));
1921
+	cap = arch_scale_cpu_capacity(cpumask_first(cpu_map));
18751922 
18761923 	for_each_cpu(i, cpu_map) {
1877
-		if (arch_scale_cpu_capacity(NULL, i) != cap) {
1924
+		if (arch_scale_cpu_capacity(i) != cap) {
18781925 			asym = true;
18791926 			break;
18801927 		}
....@@ -1889,7 +1936,7 @@
18891936 	 * to everyone.
18901937 	 */
18911938 	for_each_cpu(i, cpu_map) {
1892
-		unsigned long max_capacity = arch_scale_cpu_capacity(NULL, i);
1939
+		unsigned long max_capacity = arch_scale_cpu_capacity(i);
18931940 		int tl_id = 0;
18941941 
18951942 		for_each_sd_topology(tl) {
....@@ -1899,7 +1946,7 @@
18991946 			for_each_cpu_and(j, tl->mask(i), cpu_map) {
19001947 				unsigned long capacity;
19011948 
1902
-				capacity = arch_scale_cpu_capacity(NULL, j);
1949
+				capacity = arch_scale_cpu_capacity(j);
19031950 
19041951 				if (capacity <= max_capacity)
19051952 					continue;
....@@ -1924,12 +1971,16 @@
19241971 static int
19251972 build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr)
19261973 {
1927
-	enum s_alloc alloc_state;
1974
+	enum s_alloc alloc_state = sa_none;
19281975 	struct sched_domain *sd;
19291976 	struct s_data d;
1977
+	struct rq *rq = NULL;
19301978 	int i, ret = -ENOMEM;
19311979 	struct sched_domain_topology_level *tl_asym;
19321980 	bool has_asym = false;
1981
+
1982
+	if (WARN_ON(cpumask_empty(cpu_map)))
1983
+		goto error;
19331984 
19341985 	alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
19351986 	if (alloc_state != sa_rootdomain)
....@@ -1940,15 +1991,17 @@
19401991 	/* Set up domains for CPUs specified by the cpu_map: */
19411992 	for_each_cpu(i, cpu_map) {
19421993 		struct sched_domain_topology_level *tl;
1994
+		int dflags = 0;
19431995 
19441996 		sd = NULL;
19451997 		for_each_sd_topology(tl) {
1946
-			int dflags = 0;
1947
-
19481998 			if (tl == tl_asym) {
19491999 				dflags |= SD_ASYM_CPUCAPACITY;
19502000 				has_asym = true;
19512001 			}
2002
+
2003
+			if (WARN_ON(!topology_span_sane(tl, cpu_map, i)))
2004
+				goto error;
19522005 
19532006 			sd = build_sched_domain(tl, cpu_map, attr, sd, dflags, i);
19542007 
....@@ -1989,13 +2042,25 @@
19892042 	/* Attach the domains */
19902043 	rcu_read_lock();
19912044 	for_each_cpu(i, cpu_map) {
2045
+		rq = cpu_rq(i);
19922046 		sd = *per_cpu_ptr(d.sd, i);
2047
+
2048
+		/* Use READ_ONCE()/WRITE_ONCE() to avoid load/store tearing: */
2049
+		if (rq->cpu_capacity_orig > READ_ONCE(d.rd->max_cpu_capacity))
2050
+			WRITE_ONCE(d.rd->max_cpu_capacity, rq->cpu_capacity_orig);
2051
+
19932052 		cpu_attach_domain(sd, d.rd, i);
19942053 	}
19952054 	rcu_read_unlock();
19962055 
19972056 	if (has_asym)
19982057 		static_branch_inc_cpuslocked(&sched_asym_cpucapacity);
2058
+
2059
+	if (rq && sched_debug_enabled) {
2060
+		pr_info("root domain span: %*pbl (max cpu_capacity = %lu)\n",
2061
+			cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity);
2062
+	}
2063
+	trace_android_vh_build_sched_domains(has_asym);
19992064 
20002065 	ret = 0;
20012066 error:
....@@ -2056,9 +2121,8 @@
20562121 }
20572122 
20582123 /*
2059
- * Set up scheduler domains and groups. Callers must hold the hotplug lock.
2060
- * For now this just excludes isolated CPUs, but could be used to
2061
- * exclude other special cases in the future.
2124
+ * Set up scheduler domains and groups.  For now this just excludes isolated
2125
+ * CPUs, but could be used to exclude other special cases in the future.
20622126  */
20632127 int sched_init_domains(const struct cpumask *cpu_map)
20642128 {
....@@ -2139,16 +2203,16 @@
21392203  * ndoms_new == 0 is a special case for destroying existing domains,
21402204  * and it will not create the default domain.
21412205  *
2142
- * Call with hotplug lock held
2206
+ * Call with hotplug lock and sched_domains_mutex held
21432207  */
2144
-void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
2145
-			     struct sched_domain_attr *dattr_new)
2208
+void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new[],
2209
+				    struct sched_domain_attr *dattr_new)
21462210 {
21472211 	bool __maybe_unused has_eas = false;
21482212 	int i, j, n;
21492213 	int new_topology;
21502214 
2151
-	mutex_lock(&sched_domains_mutex);
2215
+	lockdep_assert_held(&sched_domains_mutex);
21522216 
21532217 	/* Always unregister in case we don't destroy any domains: */
21542218 	unregister_sched_domain_sysctl();
....@@ -2173,8 +2237,19 @@
21732237 	for (i = 0; i < ndoms_cur; i++) {
21742238 		for (j = 0; j < n && !new_topology; j++) {
21752239 			if (cpumask_equal(doms_cur[i], doms_new[j]) &&
2176
-			    dattrs_equal(dattr_cur, i, dattr_new, j))
2240
+			    dattrs_equal(dattr_cur, i, dattr_new, j)) {
2241
+				struct root_domain *rd;
2242
+
2243
+				/*
2244
+				 * This domain won't be destroyed and as such
2245
+				 * its dl_bw->total_bw needs to be cleared.  It
2246
+				 * will be recomputed in function
2247
+				 * update_tasks_root_domain().
2248
+				 */
2249
+				rd = cpu_rq(cpumask_any(doms_cur[i]))->rd;
2250
+				dl_clear_root_domain(rd);
21772251 				goto match1;
2252
+			}
21782253 		}
21792254 		/* No match - a current sched domain not in new doms_new[] */
21802255 		detach_destroy_domains(doms_cur[i]);
....@@ -2203,10 +2278,10 @@
22032278 		;
22042279 	}
22052280 
2206
-#ifdef CONFIG_ENERGY_MODEL
2281
+#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
22072282 	/* Build perf. domains: */
22082283 	for (i = 0; i < ndoms_new; i++) {
2209
-		for (j = 0; j < n; j++) {
2284
+		for (j = 0; j < n && !sched_energy_update; j++) {
22102285 			if (cpumask_equal(doms_new[i], doms_cur[j]) &&
22112286 			    cpu_rq(cpumask_first(doms_cur[j]))->rd->pd) {
22122287 				has_eas = true;
....@@ -2231,6 +2306,15 @@
22312306 	ndoms_cur = ndoms_new;
22322307 
22332308 	register_sched_domain_sysctl();
2309
+}
22342310 
2311
+/*
2312
+ * Call with hotplug lock held
2313
+ */
2314
+void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
2315
+			     struct sched_domain_attr *dattr_new)
2316
+{
2317
+	mutex_lock(&sched_domains_mutex);
2318
+	partition_sched_domains_locked(ndoms_new, doms_new, dattr_new);
22352319 	mutex_unlock(&sched_domains_mutex);
22362320 }