forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 072de836f53be56a70cecf70b43ae43b7ce17376 
 kernel/drivers/cpuidle/governors/menu.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * menu.c - the menu idle governor
34  *
....@@ -5,9 +6,6 @@
56  * Copyright (C) 2009 Intel Corporation
67  * Author:
78  *        Arjan van de Ven <arjan@linux.intel.com>
8
- *
9
- * This code is licenced under the GPL version 2 as described
10
- * in the COPYING file that acompanies the Linux Kernel.
119  */
1210 
1311 #include <linux/kernel.h>
....@@ -21,22 +19,12 @@
2119 #include <linux/sched/stat.h>
2220 #include <linux/math64.h>
2321 
24
-/*
25
- * Please note when changing the tuning values:
26
- * If (MAX_INTERESTING-1) * RESOLUTION > UINT_MAX, the result of
27
- * a scaling operation multiplication may overflow on 32 bit platforms.
28
- * In that case, #define RESOLUTION as ULL to get 64 bit result:
29
- * #define RESOLUTION 1024ULL
30
- *
31
- * The default values do not overflow.
32
- */
3322 #define BUCKETS 12
3423 #define INTERVAL_SHIFT 3
3524 #define INTERVALS (1UL << INTERVAL_SHIFT)
3625 #define RESOLUTION 1024
3726 #define DECAY 8
38
-#define MAX_INTERESTING 50000
39
-
27
+#define MAX_INTERESTING (50000 * NSEC_PER_USEC)
4028 
4129 /*
4230  * Concepts and ideas behind the menu governor
....@@ -119,24 +107,17 @@
119107  */
120108 
121109 struct menu_device {
122
-	int		last_state_idx;
123110 	int             needs_update;
124111 	int             tick_wakeup;
125112 
126
-	unsigned int	next_timer_us;
127
-	unsigned int	predicted_us;
113
+	u64		next_timer_ns;
128114 	unsigned int	bucket;
129115 	unsigned int	correction_factor[BUCKETS];
130116 	unsigned int	intervals[INTERVALS];
131117 	int		interval_ptr;
132118 };
133119 
134
-static inline int get_loadavg(unsigned long load)
135
-{
136
-	return LOAD_INT(load) * 10 + LOAD_FRAC(load) / 10;
137
-}
138
-
139
-static inline int which_bucket(unsigned int duration, unsigned long nr_iowaiters)
120
+static inline int which_bucket(u64 duration_ns, unsigned long nr_iowaiters)
140121 {
141122 	int bucket = 0;
142123 
....@@ -149,15 +130,15 @@
149130 	if (nr_iowaiters)
150131 		bucket = BUCKETS/2;
151132 
152
-	if (duration < 10)
133
+	if (duration_ns < 10ULL * NSEC_PER_USEC)
153134 		return bucket;
154
-	if (duration < 100)
135
+	if (duration_ns < 100ULL * NSEC_PER_USEC)
155136 		return bucket + 1;
156
-	if (duration < 1000)
137
+	if (duration_ns < 1000ULL * NSEC_PER_USEC)
157138 		return bucket + 2;
158
-	if (duration < 10000)
139
+	if (duration_ns < 10000ULL * NSEC_PER_USEC)
159140 		return bucket + 3;
160
-	if (duration < 100000)
141
+	if (duration_ns < 100000ULL * NSEC_PER_USEC)
161142 		return bucket + 4;
162143 	return bucket + 5;
163144 }
....@@ -169,18 +150,10 @@
169150  * to be, the higher this multiplier, and thus the higher
170151  * the barrier to go to an expensive C state.
171152  */
172
-static inline int performance_multiplier(unsigned long nr_iowaiters, unsigned long load)
153
+static inline int performance_multiplier(unsigned long nr_iowaiters)
173154 {
174
-	int mult = 1;
175
-
176
-	/* for higher loadavg, we are more reluctant */
177
-
178
-	mult += 2 * get_loadavg(load);
179
-
180
-	/* for IO wait tasks (per cpu!) we add 5x each */
181
-	mult += 10 * nr_iowaiters;
182
-
183
-	return mult;
155
+	/* for IO wait tasks (per cpu!) we add 10x each */
156
+	return 1 + 10 * nr_iowaiters;
184157 }
185158 
186159 static DEFINE_PER_CPU(struct menu_device, menu_devices);
....@@ -193,17 +166,19 @@
193166  * of points is below a threshold. If it is... then use the
194167  * average of these 8 points as the estimated value.
195168  */
196
-static unsigned int get_typical_interval(struct menu_device *data)
169
+static unsigned int get_typical_interval(struct menu_device *data,
170
+					 unsigned int predicted_us)
197171 {
198172 	int i, divisor;
199
-	unsigned int max, thresh, avg;
173
+	unsigned int min, max, thresh, avg;
200174 	uint64_t sum, variance;
201175 
202
-	thresh = UINT_MAX; /* Discard outliers above this value */
176
+	thresh = INT_MAX; /* Discard outliers above this value */
203177 
204178 again:
205179 
206180 	/* First calculate the average of past intervals */
181
+	min = UINT_MAX;
207182 	max = 0;
208183 	sum = 0;
209184 	divisor = 0;
....@@ -214,8 +189,19 @@
214189 			divisor++;
215190 			if (value > max)
216191 				max = value;
192
+
193
+			if (value < min)
194
+				min = value;
217195 		}
218196 	}
197
+
198
+	/*
199
+	 * If the result of the computation is going to be discarded anyway,
200
+	 * avoid the computation altogether.
201
+	 */
202
+	if (min >= predicted_us)
203
+		return UINT_MAX;
204
+
219205 	if (divisor == INTERVALS)
220206 		avg = sum >> INTERVAL_SHIFT;
221207 	else
....@@ -280,64 +266,47 @@
280266 		       bool *stop_tick)
281267 {
282268 	struct menu_device *data = this_cpu_ptr(&menu_devices);
283
-	int latency_req = cpuidle_governor_latency_req(dev->cpu);
284
-	int i;
285
-	int first_idx;
286
-	int idx;
287
-	unsigned int interactivity_req;
288
-	unsigned int expected_interval;
289
-	unsigned long nr_iowaiters, cpu_load;
269
+	s64 latency_req = cpuidle_governor_latency_req(dev->cpu);
270
+	unsigned int predicted_us;
271
+	u64 predicted_ns;
272
+	u64 interactivity_req;
273
+	unsigned long nr_iowaiters;
290274 	ktime_t delta_next;
275
+	int i, idx;
291276 
292277 	if (data->needs_update) {
293278 		menu_update(drv, dev);
294279 		data->needs_update = 0;
295280 	}
296281 
297
-	/* Special case when user has set very strict latency requirement */
298
-	if (unlikely(latency_req == 0)) {
299
-		*stop_tick = false;
282
+	/* determine the expected residency time, round up */
283
+	data->next_timer_ns = tick_nohz_get_sleep_length(&delta_next);
284
+
285
+	nr_iowaiters = nr_iowait_cpu(dev->cpu);
286
+	data->bucket = which_bucket(data->next_timer_ns, nr_iowaiters);
287
+
288
+	if (unlikely(drv->state_count <= 1 || latency_req == 0) ||
289
+	    ((data->next_timer_ns < drv->states[1].target_residency_ns ||
290
+	      latency_req < drv->states[1].exit_latency_ns) &&
291
+	     !dev->states_usage[0].disable)) {
292
+		/*
293
+		 * In this case state[0] will be used no matter what, so return
294
+		 * it right away and keep the tick running if state[0] is a
295
+		 * polling one.
296
+		 */
297
+		*stop_tick = !(drv->states[0].flags & CPUIDLE_FLAG_POLLING);
300298 		return 0;
301299 	}
302300 
303
-	/* determine the expected residency time, round up */
304
-	data->next_timer_us = ktime_to_us(tick_nohz_get_sleep_length(&delta_next));
305
-
306
-	get_iowait_load(&nr_iowaiters, &cpu_load);
307
-	data->bucket = which_bucket(data->next_timer_us, nr_iowaiters);
308
-
309
-	/*
310
-	 * Force the result of multiplication to be 64 bits even if both
311
-	 * operands are 32 bits.
312
-	 * Make sure to round up for half microseconds.
313
-	 */
314
-	data->predicted_us = DIV_ROUND_CLOSEST_ULL((uint64_t)data->next_timer_us *
315
-					 data->correction_factor[data->bucket],
316
-					 RESOLUTION * DECAY);
317
-
318
-	expected_interval = get_typical_interval(data);
319
-	expected_interval = min(expected_interval, data->next_timer_us);
320
-
321
-	first_idx = 0;
322
-	if (drv->states[0].flags & CPUIDLE_FLAG_POLLING) {
323
-		struct cpuidle_state *s = &drv->states[1];
324
-		unsigned int polling_threshold;
325
-
326
-		/*
327
-		 * Default to a physical idle state, not to busy polling, unless
328
-		 * a timer is going to trigger really really soon.
329
-		 */
330
-		polling_threshold = max_t(unsigned int, 20, s->target_residency);
331
-		if (data->next_timer_us > polling_threshold &&
332
-		    latency_req > s->exit_latency && !s->disabled &&
333
-		    !dev->states_usage[1].disable)
334
-			first_idx = 1;
335
-	}
336
-
337
-	/*
338
-	 * Use the lowest expected idle interval to pick the idle state.
339
-	 */
340
-	data->predicted_us = min(data->predicted_us, expected_interval);
301
+	/* Round up the result for half microseconds. */
302
+	predicted_us = div_u64(data->next_timer_ns *
303
+			       data->correction_factor[data->bucket] +
304
+			       (RESOLUTION * DECAY * NSEC_PER_USEC) / 2,
305
+			       RESOLUTION * DECAY * NSEC_PER_USEC);
306
+	/* Use the lowest expected idle interval to pick the idle state. */
307
+	predicted_ns = (u64)min(predicted_us,
308
+				get_typical_interval(data, predicted_us)) *
309
+				NSEC_PER_USEC;
341310 
342311 	if (tick_nohz_tick_stopped()) {
343312 		/*
....@@ -348,34 +317,46 @@
348317 		 * the known time till the closest timer event for the idle
349318 		 * state selection.
350319 		 */
351
-		if (data->predicted_us < TICK_USEC)
352
-			data->predicted_us = ktime_to_us(delta_next);
320
+		if (predicted_ns < TICK_NSEC)
321
+			predicted_ns = delta_next;
353322 	} else {
354323 		/*
355324 		 * Use the performance multiplier and the user-configurable
356325 		 * latency_req to determine the maximum exit latency.
357326 		 */
358
-		interactivity_req = data->predicted_us / performance_multiplier(nr_iowaiters, cpu_load);
327
+		interactivity_req = div64_u64(predicted_ns,
328
+					      performance_multiplier(nr_iowaiters));
359329 		if (latency_req > interactivity_req)
360330 			latency_req = interactivity_req;
361331 	}
362332 
363
-	expected_interval = data->predicted_us;
364333 	/*
365334 	 * Find the idle state with the lowest power while satisfying
366335 	 * our constraints.
367336 	 */
368337 	idx = -1;
369
-	for (i = first_idx; i < drv->state_count; i++) {
338
+	for (i = 0; i < drv->state_count; i++) {
370339 		struct cpuidle_state *s = &drv->states[i];
371
-		struct cpuidle_state_usage *su = &dev->states_usage[i];
372340 
373
-		if (s->disabled || su->disable)
341
+		if (dev->states_usage[i].disable)
374342 			continue;
343
+
375344 		if (idx == -1)
376345 			idx = i; /* first enabled state */
377
-		if (s->target_residency > data->predicted_us) {
378
-			if (data->predicted_us < TICK_USEC)
346
+
347
+		if (s->target_residency_ns > predicted_ns) {
348
+			/*
349
+			 * Use a physical idle state, not busy polling, unless
350
+			 * a timer is going to trigger soon enough.
351
+			 */
352
+			if ((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) &&
353
+			    s->exit_latency_ns <= latency_req &&
354
+			    s->target_residency_ns <= data->next_timer_ns) {
355
+				predicted_ns = s->target_residency_ns;
356
+				idx = i;
357
+				break;
358
+			}
359
+			if (predicted_ns < TICK_NSEC)
379360 				break;
380361 
381362 			if (!tick_nohz_tick_stopped()) {
....@@ -385,7 +366,7 @@
385366 				 * tick in that case and let the governor run
386367 				 * again in the next iteration of the loop.
387368 				 */
388
-				expected_interval = drv->states[idx].target_residency;
369
+				predicted_ns = drv->states[idx].target_residency_ns;
389370 				break;
390371 			}
391372 
....@@ -395,22 +376,15 @@
395376 			 * closest timer event, select this one to avoid getting
396377 			 * stuck in the shallow one for too long.
397378 			 */
398
-			if (drv->states[idx].target_residency < TICK_USEC &&
399
-			    s->target_residency <= ktime_to_us(delta_next))
379
+			if (drv->states[idx].target_residency_ns < TICK_NSEC &&
380
+			    s->target_residency_ns <= delta_next)
400381 				idx = i;
401382 
402
-			goto out;
383
+			return idx;
403384 		}
404
-		if (s->exit_latency > latency_req) {
405
-			/*
406
-			 * If we break out of the loop for latency reasons, use
407
-			 * the target residency of the selected state as the
408
-			 * expected idle duration so that the tick is retained
409
-			 * as long as that target residency is low enough.
410
-			 */
411
-			expected_interval = drv->states[idx].target_residency;
385
+		if (s->exit_latency_ns > latency_req)
412386 			break;
413
-		}
387
+
414388 		idx = i;
415389 	}
416390 
....@@ -422,12 +396,10 @@
422396 	 * expected idle duration is shorter than the tick period length.
423397 	 */
424398 	if (((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) ||
425
-	     expected_interval < TICK_USEC) && !tick_nohz_tick_stopped()) {
426
-		unsigned int delta_next_us = ktime_to_us(delta_next);
427
-
399
+	     predicted_ns < TICK_NSEC) && !tick_nohz_tick_stopped()) {
428400 		*stop_tick = false;
429401 
430
-		if (idx > 0 && drv->states[idx].target_residency > delta_next_us) {
402
+		if (idx > 0 && drv->states[idx].target_residency_ns > delta_next) {
431403 			/*
432404 			 * The tick is not going to be stopped and the target
433405 			 * residency of the state to be returned is not within
....@@ -435,21 +407,17 @@
435407 			 * tick, so try to correct that.
436408 			 */
437409 			for (i = idx - 1; i >= 0; i--) {
438
-				if (drv->states[i].disabled ||
439
-				    dev->states_usage[i].disable)
410
+				if (dev->states_usage[i].disable)
440411 					continue;
441412 
442413 				idx = i;
443
-				if (drv->states[i].target_residency <= delta_next_us)
414
+				if (drv->states[i].target_residency_ns <= delta_next)
444415 					break;
445416 			}
446417 		}
447418 	}
448419 
449
-out:
450
-	data->last_state_idx = idx;
451
-
452
-	return data->last_state_idx;
420
+	return idx;
453421 }
454422 
455423 /**
....@@ -464,7 +432,7 @@
464432 {
465433 	struct menu_device *data = this_cpu_ptr(&menu_devices);
466434 
467
-	data->last_state_idx = index;
435
+	dev->last_state_idx = index;
468436 	data->needs_update = 1;
469437 	data->tick_wakeup = tick_nohz_idle_got_tick();
470438 }
....@@ -477,9 +445,9 @@
477445 static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
478446 {
479447 	struct menu_device *data = this_cpu_ptr(&menu_devices);
480
-	int last_idx = data->last_state_idx;
448
+	int last_idx = dev->last_state_idx;
481449 	struct cpuidle_state *target = &drv->states[last_idx];
482
-	unsigned int measured_us;
450
+	u64 measured_ns;
483451 	unsigned int new_factor;
484452 
485453 	/*
....@@ -497,7 +465,7 @@
497465 	 * assume the state was never reached and the exit latency is 0.
498466 	 */
499467 
500
-	if (data->tick_wakeup && data->next_timer_us > TICK_USEC) {
468
+	if (data->tick_wakeup && data->next_timer_ns > TICK_NSEC) {
501469 		/*
502470 		 * The nohz code said that there wouldn't be any events within
503471 		 * the tick boundary (if the tick was stopped), but the idle
....@@ -507,7 +475,7 @@
507475 		 * have been idle long (but not forever) to help the idle
508476 		 * duration predictor do a better job next time.
509477 		 */
510
-		measured_us = 9 * MAX_INTERESTING / 10;
478
+		measured_ns = 9 * MAX_INTERESTING / 10;
511479 	} else if ((drv->states[last_idx].flags & CPUIDLE_FLAG_POLLING) &&
512480 		   dev->poll_time_limit) {
513481 		/*
....@@ -517,28 +485,29 @@
517485 		 * the CPU might have been woken up from idle by the next timer.
518486 		 * Assume that to be the case.
519487 		 */
520
-		measured_us = data->next_timer_us;
488
+		measured_ns = data->next_timer_ns;
521489 	} else {
522490 		/* measured value */
523
-		measured_us = cpuidle_get_last_residency(dev);
491
+		measured_ns = dev->last_residency_ns;
524492 
525493 		/* Deduct exit latency */
526
-		if (measured_us > 2 * target->exit_latency)
527
-			measured_us -= target->exit_latency;
494
+		if (measured_ns > 2 * target->exit_latency_ns)
495
+			measured_ns -= target->exit_latency_ns;
528496 		else
529
-			measured_us /= 2;
497
+			measured_ns /= 2;
530498 	}
531499 
532500 	/* Make sure our coefficients do not exceed unity */
533
-	if (measured_us > data->next_timer_us)
534
-		measured_us = data->next_timer_us;
501
+	if (measured_ns > data->next_timer_ns)
502
+		measured_ns = data->next_timer_ns;
535503 
536504 	/* Update our correction ratio */
537505 	new_factor = data->correction_factor[data->bucket];
538506 	new_factor -= new_factor / DECAY;
539507 
540
-	if (data->next_timer_us > 0 && measured_us < MAX_INTERESTING)
541
-		new_factor += RESOLUTION * measured_us / data->next_timer_us;
508
+	if (data->next_timer_ns > 0 && measured_ns < MAX_INTERESTING)
509
+		new_factor += div64_u64(RESOLUTION * measured_ns,
510
+					data->next_timer_ns);
542511 	else
543512 		/*
544513 		 * we were idle so long that we count it as a perfect
....@@ -558,7 +527,7 @@
558527 	data->correction_factor[data->bucket] = new_factor;
559528 
560529 	/* update the repeating-pattern data */
561
-	data->intervals[data->interval_ptr++] = measured_us;
530
+	data->intervals[data->interval_ptr++] = ktime_to_us(measured_ns);
562531 	if (data->interval_ptr >= INTERVALS)
563532 		data->interval_ptr = 0;
564533 }