.. | .. |
---|
27 | 27 | struct { |
---|
28 | 28 | u32 promotion_count; |
---|
29 | 29 | u32 demotion_count; |
---|
30 | | - u32 promotion_time; |
---|
31 | | - u32 demotion_time; |
---|
| 30 | + u64 promotion_time_ns; |
---|
| 31 | + u64 demotion_time_ns; |
---|
32 | 32 | } threshold; |
---|
33 | 33 | struct { |
---|
34 | 34 | int promotion_count; |
---|
.. | .. |
---|
38 | 38 | |
---|
39 | 39 | struct ladder_device { |
---|
40 | 40 | struct ladder_device_state states[CPUIDLE_STATE_MAX]; |
---|
41 | | - int last_state_idx; |
---|
42 | 41 | }; |
---|
43 | 42 | |
---|
44 | 43 | static DEFINE_PER_CPU(struct ladder_device, ladder_devices); |
---|
.. | .. |
---|
49 | 48 | * @old_idx: the current state index |
---|
50 | 49 | * @new_idx: the new target state index |
---|
51 | 50 | */ |
---|
52 | | -static inline void ladder_do_selection(struct ladder_device *ldev, |
---|
| 51 | +static inline void ladder_do_selection(struct cpuidle_device *dev, |
---|
| 52 | + struct ladder_device *ldev, |
---|
53 | 53 | int old_idx, int new_idx) |
---|
54 | 54 | { |
---|
55 | 55 | ldev->states[old_idx].stats.promotion_count = 0; |
---|
56 | 56 | ldev->states[old_idx].stats.demotion_count = 0; |
---|
57 | | - ldev->last_state_idx = new_idx; |
---|
| 57 | + dev->last_state_idx = new_idx; |
---|
58 | 58 | } |
---|
59 | 59 | |
---|
60 | 60 | /** |
---|
.. | .. |
---|
68 | 68 | { |
---|
69 | 69 | struct ladder_device *ldev = this_cpu_ptr(&ladder_devices); |
---|
70 | 70 | struct ladder_device_state *last_state; |
---|
71 | | - int last_residency, last_idx = ldev->last_state_idx; |
---|
| 71 | + int last_idx = dev->last_state_idx; |
---|
72 | 72 | int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0; |
---|
73 | | - int latency_req = cpuidle_governor_latency_req(dev->cpu); |
---|
| 73 | + s64 latency_req = cpuidle_governor_latency_req(dev->cpu); |
---|
| 74 | + s64 last_residency; |
---|
74 | 75 | |
---|
75 | 76 | /* Special case when user has set very strict latency requirement */ |
---|
76 | 77 | if (unlikely(latency_req == 0)) { |
---|
77 | | - ladder_do_selection(ldev, last_idx, 0); |
---|
| 78 | + ladder_do_selection(dev, ldev, last_idx, 0); |
---|
78 | 79 | return 0; |
---|
79 | 80 | } |
---|
80 | 81 | |
---|
81 | 82 | last_state = &ldev->states[last_idx]; |
---|
82 | 83 | |
---|
83 | | - last_residency = cpuidle_get_last_residency(dev) - drv->states[last_idx].exit_latency; |
---|
| 84 | + last_residency = dev->last_residency_ns - drv->states[last_idx].exit_latency_ns; |
---|
84 | 85 | |
---|
85 | 86 | /* consider promotion */ |
---|
86 | 87 | if (last_idx < drv->state_count - 1 && |
---|
87 | | - !drv->states[last_idx + 1].disabled && |
---|
88 | 88 | !dev->states_usage[last_idx + 1].disable && |
---|
89 | | - last_residency > last_state->threshold.promotion_time && |
---|
90 | | - drv->states[last_idx + 1].exit_latency <= latency_req) { |
---|
| 89 | + last_residency > last_state->threshold.promotion_time_ns && |
---|
| 90 | + drv->states[last_idx + 1].exit_latency_ns <= latency_req) { |
---|
91 | 91 | last_state->stats.promotion_count++; |
---|
92 | 92 | last_state->stats.demotion_count = 0; |
---|
93 | 93 | if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) { |
---|
94 | | - ladder_do_selection(ldev, last_idx, last_idx + 1); |
---|
| 94 | + ladder_do_selection(dev, ldev, last_idx, last_idx + 1); |
---|
95 | 95 | return last_idx + 1; |
---|
96 | 96 | } |
---|
97 | 97 | } |
---|
98 | 98 | |
---|
99 | 99 | /* consider demotion */ |
---|
100 | 100 | if (last_idx > first_idx && |
---|
101 | | - (drv->states[last_idx].disabled || |
---|
102 | | - dev->states_usage[last_idx].disable || |
---|
103 | | - drv->states[last_idx].exit_latency > latency_req)) { |
---|
| 101 | + (dev->states_usage[last_idx].disable || |
---|
| 102 | + drv->states[last_idx].exit_latency_ns > latency_req)) { |
---|
104 | 103 | int i; |
---|
105 | 104 | |
---|
106 | 105 | for (i = last_idx - 1; i > first_idx; i--) { |
---|
107 | | - if (drv->states[i].exit_latency <= latency_req) |
---|
| 106 | + if (drv->states[i].exit_latency_ns <= latency_req) |
---|
108 | 107 | break; |
---|
109 | 108 | } |
---|
110 | | - ladder_do_selection(ldev, last_idx, i); |
---|
| 109 | + ladder_do_selection(dev, ldev, last_idx, i); |
---|
111 | 110 | return i; |
---|
112 | 111 | } |
---|
113 | 112 | |
---|
114 | 113 | if (last_idx > first_idx && |
---|
115 | | - last_residency < last_state->threshold.demotion_time) { |
---|
| 114 | + last_residency < last_state->threshold.demotion_time_ns) { |
---|
116 | 115 | last_state->stats.demotion_count++; |
---|
117 | 116 | last_state->stats.promotion_count = 0; |
---|
118 | 117 | if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) { |
---|
119 | | - ladder_do_selection(ldev, last_idx, last_idx - 1); |
---|
| 118 | + ladder_do_selection(dev, ldev, last_idx, last_idx - 1); |
---|
120 | 119 | return last_idx - 1; |
---|
121 | 120 | } |
---|
122 | 121 | } |
---|
.. | .. |
---|
139 | 138 | struct ladder_device_state *lstate; |
---|
140 | 139 | struct cpuidle_state *state; |
---|
141 | 140 | |
---|
142 | | - ldev->last_state_idx = first_idx; |
---|
| 141 | + dev->last_state_idx = first_idx; |
---|
143 | 142 | |
---|
144 | 143 | for (i = first_idx; i < drv->state_count; i++) { |
---|
145 | 144 | state = &drv->states[i]; |
---|
.. | .. |
---|
152 | 151 | lstate->threshold.demotion_count = DEMOTION_COUNT; |
---|
153 | 152 | |
---|
154 | 153 | if (i < drv->state_count - 1) |
---|
155 | | - lstate->threshold.promotion_time = state->exit_latency; |
---|
| 154 | + lstate->threshold.promotion_time_ns = state->exit_latency_ns; |
---|
156 | 155 | if (i > first_idx) |
---|
157 | | - lstate->threshold.demotion_time = state->exit_latency; |
---|
| 156 | + lstate->threshold.demotion_time_ns = state->exit_latency_ns; |
---|
158 | 157 | } |
---|
159 | 158 | |
---|
160 | 159 | return 0; |
---|
.. | .. |
---|
167 | 166 | */ |
---|
168 | 167 | static void ladder_reflect(struct cpuidle_device *dev, int index) |
---|
169 | 168 | { |
---|
170 | | - struct ladder_device *ldev = this_cpu_ptr(&ladder_devices); |
---|
171 | 169 | if (index > 0) |
---|
172 | | - ldev->last_state_idx = index; |
---|
| 170 | + dev->last_state_idx = index; |
---|
173 | 171 | } |
---|
174 | 172 | |
---|
175 | 173 | static struct cpuidle_governor ladder_governor = { |
---|