.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0 |
---|
1 | 2 | /* |
---|
2 | 3 | * drivers/base/power/domain_governor.c - Governors for device PM domains. |
---|
3 | 4 | * |
---|
4 | 5 | * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp. |
---|
5 | | - * |
---|
6 | | - * This file is released under the GPLv2. |
---|
7 | 6 | */ |
---|
8 | | - |
---|
9 | 7 | #include <linux/kernel.h> |
---|
10 | 8 | #include <linux/pm_domain.h> |
---|
11 | 9 | #include <linux/pm_qos.h> |
---|
12 | 10 | #include <linux/hrtimer.h> |
---|
| 11 | +#include <linux/cpuidle.h> |
---|
| 12 | +#include <linux/cpumask.h> |
---|
| 13 | +#include <linux/ktime.h> |
---|
| 14 | + |
---|
| 15 | +#include <trace/hooks/pm_domain.h> |
---|
13 | 16 | |
---|
14 | 17 | static int dev_update_qos_constraint(struct device *dev, void *data) |
---|
15 | 18 | { |
---|
.. | .. |
---|
32 | 35 | * take its current PM QoS constraint (that's the only thing |
---|
33 | 36 | * known at this point anyway). |
---|
34 | 37 | */ |
---|
35 | | - constraint_ns = dev_pm_qos_read_value(dev); |
---|
| 38 | + constraint_ns = dev_pm_qos_read_value(dev, DEV_PM_QOS_RESUME_LATENCY); |
---|
36 | 39 | constraint_ns *= NSEC_PER_USEC; |
---|
37 | 40 | } |
---|
38 | 41 | |
---|
.. | .. |
---|
65 | 68 | td->constraint_changed = false; |
---|
66 | 69 | td->cached_suspend_ok = false; |
---|
67 | 70 | td->effective_constraint_ns = 0; |
---|
68 | | - constraint_ns = __dev_pm_qos_read_value(dev); |
---|
| 71 | + constraint_ns = __dev_pm_qos_resume_latency(dev); |
---|
69 | 72 | |
---|
70 | 73 | spin_unlock_irqrestore(&dev->power.lock, flags); |
---|
71 | 74 | |
---|
.. | .. |
---|
116 | 119 | return td->cached_suspend_ok; |
---|
117 | 120 | } |
---|
118 | 121 | |
---|
| 122 | +static void update_domain_next_wakeup(struct generic_pm_domain *genpd, ktime_t now) |
---|
| 123 | +{ |
---|
| 124 | + ktime_t domain_wakeup = KTIME_MAX; |
---|
| 125 | + ktime_t next_wakeup; |
---|
| 126 | + struct pm_domain_data *pdd; |
---|
| 127 | + struct gpd_link *link; |
---|
| 128 | + |
---|
| 129 | + if (!(genpd->flags & GENPD_FLAG_MIN_RESIDENCY)) |
---|
| 130 | + return; |
---|
| 131 | + |
---|
| 132 | + /* |
---|
| 133 | + * Devices that have a predictable wakeup pattern, may specify |
---|
| 134 | + * their next wakeup. Let's find the next wakeup from all the |
---|
| 135 | + * devices attached to this domain and from all the sub-domains. |
---|
| 136 | + * It is possible that component's a next wakeup may have become |
---|
| 137 | + * stale when we read that here. We will ignore to ensure the domain |
---|
| 138 | + * is able to enter its optimal idle state. |
---|
| 139 | + */ |
---|
| 140 | + list_for_each_entry(pdd, &genpd->dev_list, list_node) { |
---|
| 141 | + next_wakeup = to_gpd_data(pdd)->next_wakeup; |
---|
| 142 | + if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now)) |
---|
| 143 | + if (ktime_before(next_wakeup, domain_wakeup)) |
---|
| 144 | + domain_wakeup = next_wakeup; |
---|
| 145 | + } |
---|
| 146 | + |
---|
| 147 | + list_for_each_entry(link, &genpd->parent_links, parent_node) { |
---|
| 148 | + next_wakeup = link->child->next_wakeup; |
---|
| 149 | + if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now)) |
---|
| 150 | + if (ktime_before(next_wakeup, domain_wakeup)) |
---|
| 151 | + domain_wakeup = next_wakeup; |
---|
| 152 | + } |
---|
| 153 | + |
---|
| 154 | + genpd->next_wakeup = domain_wakeup; |
---|
| 155 | +} |
---|
| 156 | + |
---|
| 157 | +static bool next_wakeup_allows_state(struct generic_pm_domain *genpd, |
---|
| 158 | + unsigned int state, ktime_t now) |
---|
| 159 | +{ |
---|
| 160 | + ktime_t domain_wakeup = genpd->next_wakeup; |
---|
| 161 | + s64 idle_time_ns, min_sleep_ns; |
---|
| 162 | + |
---|
| 163 | + min_sleep_ns = genpd->states[state].power_off_latency_ns + |
---|
| 164 | + genpd->states[state].residency_ns; |
---|
| 165 | + |
---|
| 166 | + idle_time_ns = ktime_to_ns(ktime_sub(domain_wakeup, now)); |
---|
| 167 | + |
---|
| 168 | + return idle_time_ns >= min_sleep_ns; |
---|
| 169 | +} |
---|
| 170 | + |
---|
119 | 171 | static bool __default_power_down_ok(struct dev_pm_domain *pd, |
---|
120 | 172 | unsigned int state) |
---|
121 | 173 | { |
---|
.. | .. |
---|
124 | 176 | struct pm_domain_data *pdd; |
---|
125 | 177 | s64 min_off_time_ns; |
---|
126 | 178 | s64 off_on_time_ns; |
---|
| 179 | + bool allow = true; |
---|
| 180 | + |
---|
| 181 | + trace_android_vh_allow_domain_state(genpd, state, &allow); |
---|
| 182 | + if (!allow) |
---|
| 183 | + return false; |
---|
127 | 184 | |
---|
128 | 185 | off_on_time_ns = genpd->states[state].power_off_latency_ns + |
---|
129 | 186 | genpd->states[state].power_on_latency_ns; |
---|
130 | | - |
---|
131 | 187 | |
---|
132 | 188 | min_off_time_ns = -1; |
---|
133 | 189 | /* |
---|
.. | .. |
---|
135 | 191 | * |
---|
136 | 192 | * All subdomains have been powered off already at this point. |
---|
137 | 193 | */ |
---|
138 | | - list_for_each_entry(link, &genpd->master_links, master_node) { |
---|
139 | | - struct generic_pm_domain *sd = link->slave; |
---|
| 194 | + list_for_each_entry(link, &genpd->parent_links, parent_node) { |
---|
| 195 | + struct generic_pm_domain *sd = link->child; |
---|
140 | 196 | s64 sd_max_off_ns = sd->max_off_time_ns; |
---|
141 | 197 | |
---|
142 | 198 | if (sd_max_off_ns < 0) |
---|
.. | .. |
---|
201 | 257 | } |
---|
202 | 258 | |
---|
203 | 259 | /** |
---|
204 | | - * default_power_down_ok - Default generic PM domain power off governor routine. |
---|
| 260 | + * _default_power_down_ok - Default generic PM domain power off governor routine. |
---|
205 | 261 | * @pd: PM domain to check. |
---|
206 | 262 | * |
---|
207 | 263 | * This routine must be executed under the PM domain's lock. |
---|
208 | 264 | */ |
---|
209 | | -static bool default_power_down_ok(struct dev_pm_domain *pd) |
---|
| 265 | +static bool _default_power_down_ok(struct dev_pm_domain *pd, ktime_t now) |
---|
210 | 266 | { |
---|
211 | 267 | struct generic_pm_domain *genpd = pd_to_genpd(pd); |
---|
| 268 | + int state_idx = genpd->state_count - 1; |
---|
212 | 269 | struct gpd_link *link; |
---|
213 | 270 | |
---|
214 | | - if (!genpd->max_off_time_changed) |
---|
| 271 | + /* |
---|
| 272 | + * Find the next wakeup from devices that can determine their own wakeup |
---|
| 273 | + * to find when the domain would wakeup and do it for every device down |
---|
| 274 | + * the hierarchy. It is not worth while to sleep if the state's residency |
---|
| 275 | + * cannot be met. |
---|
| 276 | + */ |
---|
| 277 | + update_domain_next_wakeup(genpd, now); |
---|
| 278 | + if ((genpd->flags & GENPD_FLAG_MIN_RESIDENCY) && (genpd->next_wakeup != KTIME_MAX)) { |
---|
| 279 | + /* Let's find out the deepest domain idle state, the devices prefer */ |
---|
| 280 | + while (state_idx >= 0) { |
---|
| 281 | + if (next_wakeup_allows_state(genpd, state_idx, now)) { |
---|
| 282 | + genpd->max_off_time_changed = true; |
---|
| 283 | + break; |
---|
| 284 | + } |
---|
| 285 | + state_idx--; |
---|
| 286 | + } |
---|
| 287 | + |
---|
| 288 | + if (state_idx < 0) { |
---|
| 289 | + state_idx = 0; |
---|
| 290 | + genpd->cached_power_down_ok = false; |
---|
| 291 | + goto done; |
---|
| 292 | + } |
---|
| 293 | + } |
---|
| 294 | + |
---|
| 295 | + if (!genpd->max_off_time_changed) { |
---|
| 296 | + genpd->state_idx = genpd->cached_power_down_state_idx; |
---|
215 | 297 | return genpd->cached_power_down_ok; |
---|
| 298 | + } |
---|
216 | 299 | |
---|
217 | 300 | /* |
---|
218 | | - * We have to invalidate the cached results for the masters, so |
---|
| 301 | + * We have to invalidate the cached results for the parents, so |
---|
219 | 302 | * use the observation that default_power_down_ok() is not |
---|
220 | | - * going to be called for any master until this instance |
---|
| 303 | + * going to be called for any parent until this instance |
---|
221 | 304 | * returns. |
---|
222 | 305 | */ |
---|
223 | | - list_for_each_entry(link, &genpd->slave_links, slave_node) |
---|
224 | | - link->master->max_off_time_changed = true; |
---|
| 306 | + list_for_each_entry(link, &genpd->child_links, child_node) |
---|
| 307 | + link->parent->max_off_time_changed = true; |
---|
225 | 308 | |
---|
226 | 309 | genpd->max_off_time_ns = -1; |
---|
227 | 310 | genpd->max_off_time_changed = false; |
---|
228 | 311 | genpd->cached_power_down_ok = true; |
---|
229 | | - genpd->state_idx = genpd->state_count - 1; |
---|
230 | 312 | |
---|
231 | | - /* Find a state to power down to, starting from the deepest. */ |
---|
232 | | - while (!__default_power_down_ok(pd, genpd->state_idx)) { |
---|
233 | | - if (genpd->state_idx == 0) { |
---|
| 313 | + /* |
---|
| 314 | + * Find a state to power down to, starting from the state |
---|
| 315 | + * determined by the next wakeup. |
---|
| 316 | + */ |
---|
| 317 | + while (!__default_power_down_ok(pd, state_idx)) { |
---|
| 318 | + if (state_idx == 0) { |
---|
234 | 319 | genpd->cached_power_down_ok = false; |
---|
235 | 320 | break; |
---|
236 | 321 | } |
---|
237 | | - genpd->state_idx--; |
---|
| 322 | + state_idx--; |
---|
238 | 323 | } |
---|
239 | 324 | |
---|
| 325 | +done: |
---|
| 326 | + genpd->state_idx = state_idx; |
---|
| 327 | + genpd->cached_power_down_state_idx = genpd->state_idx; |
---|
240 | 328 | return genpd->cached_power_down_ok; |
---|
| 329 | +} |
---|
| 330 | + |
---|
| 331 | +static bool default_power_down_ok(struct dev_pm_domain *pd) |
---|
| 332 | +{ |
---|
| 333 | + return _default_power_down_ok(pd, ktime_get()); |
---|
241 | 334 | } |
---|
242 | 335 | |
---|
243 | 336 | static bool always_on_power_down_ok(struct dev_pm_domain *domain) |
---|
.. | .. |
---|
245 | 338 | return false; |
---|
246 | 339 | } |
---|
247 | 340 | |
---|
| 341 | +#ifdef CONFIG_CPU_IDLE |
---|
| 342 | +static bool cpu_power_down_ok(struct dev_pm_domain *pd) |
---|
| 343 | +{ |
---|
| 344 | + struct generic_pm_domain *genpd = pd_to_genpd(pd); |
---|
| 345 | + struct cpuidle_device *dev; |
---|
| 346 | + ktime_t domain_wakeup, next_hrtimer; |
---|
| 347 | + ktime_t now = ktime_get(); |
---|
| 348 | + s64 idle_duration_ns; |
---|
| 349 | + int cpu, i; |
---|
| 350 | + |
---|
| 351 | + /* Validate dev PM QoS constraints. */ |
---|
| 352 | + if (!_default_power_down_ok(pd, now)) |
---|
| 353 | + return false; |
---|
| 354 | + |
---|
| 355 | + if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN)) |
---|
| 356 | + return true; |
---|
| 357 | + |
---|
| 358 | + /* |
---|
| 359 | + * Find the next wakeup for any of the online CPUs within the PM domain |
---|
| 360 | + * and its subdomains. Note, we only need the genpd->cpus, as it already |
---|
| 361 | + * contains a mask of all CPUs from subdomains. |
---|
| 362 | + */ |
---|
| 363 | + domain_wakeup = ktime_set(KTIME_SEC_MAX, 0); |
---|
| 364 | + for_each_cpu_and(cpu, genpd->cpus, cpu_online_mask) { |
---|
| 365 | + dev = per_cpu(cpuidle_devices, cpu); |
---|
| 366 | + if (dev) { |
---|
| 367 | + next_hrtimer = READ_ONCE(dev->next_hrtimer); |
---|
| 368 | + if (ktime_before(next_hrtimer, domain_wakeup)) |
---|
| 369 | + domain_wakeup = next_hrtimer; |
---|
| 370 | + } |
---|
| 371 | + } |
---|
| 372 | + |
---|
| 373 | + /* The minimum idle duration is from now - until the next wakeup. */ |
---|
| 374 | + idle_duration_ns = ktime_to_ns(ktime_sub(domain_wakeup, now)); |
---|
| 375 | + if (idle_duration_ns <= 0) |
---|
| 376 | + return false; |
---|
| 377 | + |
---|
| 378 | + /* |
---|
| 379 | + * Find the deepest idle state that has its residency value satisfied |
---|
| 380 | + * and by also taking into account the power off latency for the state. |
---|
| 381 | + * Start at the state picked by the dev PM QoS constraint validation. |
---|
| 382 | + */ |
---|
| 383 | + i = genpd->state_idx; |
---|
| 384 | + do { |
---|
| 385 | + if (idle_duration_ns >= (genpd->states[i].residency_ns + |
---|
| 386 | + genpd->states[i].power_off_latency_ns)) { |
---|
| 387 | + genpd->state_idx = i; |
---|
| 388 | + return true; |
---|
| 389 | + } |
---|
| 390 | + } while (--i >= 0); |
---|
| 391 | + |
---|
| 392 | + return false; |
---|
| 393 | +} |
---|
| 394 | + |
---|
| 395 | +struct dev_power_governor pm_domain_cpu_gov = { |
---|
| 396 | + .suspend_ok = default_suspend_ok, |
---|
| 397 | + .power_down_ok = cpu_power_down_ok, |
---|
| 398 | +}; |
---|
| 399 | +#endif |
---|
| 400 | + |
---|
248 | 401 | struct dev_power_governor simple_qos_governor = { |
---|
249 | 402 | .suspend_ok = default_suspend_ok, |
---|
250 | 403 | .power_down_ok = default_power_down_ok, |
---|