.. | .. |
---|
27 | 27 | # include <asm/smp.h> |
---|
28 | 28 | #endif |
---|
29 | 29 | |
---|
| 30 | +static void delay_loop(u64 __loops); |
---|
| 31 | + |
---|
| 32 | +/* |
---|
| 33 | + * Calibration and selection of the delay mechanism happens only once |
---|
| 34 | + * during boot. |
---|
| 35 | + */ |
---|
| 36 | +static void (*delay_fn)(u64) __ro_after_init = delay_loop; |
---|
| 37 | +static void (*delay_halt_fn)(u64 start, u64 cycles) __ro_after_init; |
---|
| 38 | + |
---|
30 | 39 | /* simple loop based delay: */ |
---|
31 | | -static void delay_loop(unsigned long loops) |
---|
| 40 | +static void delay_loop(u64 __loops) |
---|
32 | 41 | { |
---|
| 42 | + unsigned long loops = (unsigned long)__loops; |
---|
| 43 | + |
---|
33 | 44 | asm volatile( |
---|
34 | 45 | " test %0,%0 \n" |
---|
35 | 46 | " jz 3f \n" |
---|
.. | .. |
---|
43 | 54 | " jnz 2b \n" |
---|
44 | 55 | "3: dec %0 \n" |
---|
45 | 56 | |
---|
46 | | - : /* we don't need output */ |
---|
47 | | - :"a" (loops) |
---|
| 57 | + : "+a" (loops) |
---|
| 58 | + : |
---|
48 | 59 | ); |
---|
49 | 60 | } |
---|
50 | 61 | |
---|
51 | 62 | /* TSC based delay: */ |
---|
52 | | -static void delay_tsc(unsigned long __loops) |
---|
| 63 | +static void delay_tsc(u64 cycles) |
---|
53 | 64 | { |
---|
54 | | - u64 bclock, now, loops = __loops; |
---|
| 65 | + u64 bclock, now; |
---|
55 | 66 | int cpu; |
---|
56 | 67 | |
---|
57 | 68 | preempt_disable(); |
---|
.. | .. |
---|
59 | 70 | bclock = rdtsc_ordered(); |
---|
60 | 71 | for (;;) { |
---|
61 | 72 | now = rdtsc_ordered(); |
---|
62 | | - if ((now - bclock) >= loops) |
---|
| 73 | + if ((now - bclock) >= cycles) |
---|
63 | 74 | break; |
---|
64 | 75 | |
---|
65 | 76 | /* Allow RT tasks to run */ |
---|
.. | .. |
---|
77 | 88 | * counter for this CPU. |
---|
78 | 89 | */ |
---|
79 | 90 | if (unlikely(cpu != smp_processor_id())) { |
---|
80 | | - loops -= (now - bclock); |
---|
| 91 | + cycles -= (now - bclock); |
---|
81 | 92 | cpu = smp_processor_id(); |
---|
82 | 93 | bclock = rdtsc_ordered(); |
---|
83 | 94 | } |
---|
.. | .. |
---|
86 | 97 | } |
---|
87 | 98 | |
---|
88 | 99 | /* |
---|
89 | | - * On some AMD platforms, MWAITX has a configurable 32-bit timer, that |
---|
90 | | - * counts with TSC frequency. The input value is the loop of the |
---|
91 | | - * counter, it will exit when the timer expires. |
---|
| 100 | + * On Intel the TPAUSE instruction waits until any of: |
---|
| 101 | + * 1) the TSC counter exceeds the value provided in EDX:EAX |
---|
| 102 | + * 2) global timeout in IA32_UMWAIT_CONTROL is exceeded |
---|
| 103 | + * 3) an external interrupt occurs |
---|
92 | 104 | */ |
---|
93 | | -static void delay_mwaitx(unsigned long __loops) |
---|
| 105 | +static void delay_halt_tpause(u64 start, u64 cycles) |
---|
94 | 106 | { |
---|
95 | | - u64 start, end, delay, loops = __loops; |
---|
| 107 | + u64 until = start + cycles; |
---|
| 108 | + u32 eax, edx; |
---|
| 109 | + |
---|
| 110 | + eax = lower_32_bits(until); |
---|
| 111 | + edx = upper_32_bits(until); |
---|
| 112 | + |
---|
| 113 | + /* |
---|
| 114 | + * Hard code the deeper (C0.2) sleep state because exit latency is |
---|
| 115 | + * small compared to the "microseconds" that usleep() will delay. |
---|
| 116 | + */ |
---|
| 117 | + __tpause(TPAUSE_C02_STATE, edx, eax); |
---|
| 118 | +} |
---|
| 119 | + |
---|
| 120 | +/* |
---|
| 121 | + * On some AMD platforms, MWAITX has a configurable 32-bit timer, that |
---|
| 122 | + * counts with TSC frequency. The input value is the number of TSC cycles |
---|
| 123 | + * to wait. MWAITX will also exit when the timer expires. |
---|
| 124 | + */ |
---|
| 125 | +static void delay_halt_mwaitx(u64 unused, u64 cycles) |
---|
| 126 | +{ |
---|
| 127 | + u64 delay; |
---|
| 128 | + |
---|
| 129 | + delay = min_t(u64, MWAITX_MAX_WAIT_CYCLES, cycles); |
---|
| 130 | + /* |
---|
| 131 | + * Use cpu_tss_rw as a cacheline-aligned, seldomly accessed per-cpu |
---|
| 132 | + * variable as the monitor target. |
---|
| 133 | + */ |
---|
| 134 | + __monitorx(raw_cpu_ptr(&cpu_tss_rw), 0, 0); |
---|
| 135 | + |
---|
| 136 | + /* |
---|
| 137 | + * AMD, like Intel, supports the EAX hint and EAX=0xf means, do not |
---|
| 138 | + * enter any deep C-state and we use it here in delay() to minimize |
---|
| 139 | + * wakeup latency. |
---|
| 140 | + */ |
---|
| 141 | + __mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE); |
---|
| 142 | +} |
---|
| 143 | + |
---|
| 144 | +/* |
---|
| 145 | + * Call a vendor specific function to delay for a given amount of time. Because |
---|
| 146 | + * these functions may return earlier than requested, check for actual elapsed |
---|
| 147 | + * time and call again until done. |
---|
| 148 | + */ |
---|
| 149 | +static void delay_halt(u64 __cycles) |
---|
| 150 | +{ |
---|
| 151 | + u64 start, end, cycles = __cycles; |
---|
96 | 152 | |
---|
97 | 153 | /* |
---|
98 | 154 | * Timer value of 0 causes MWAITX to wait indefinitely, unless there |
---|
99 | 155 | * is a store on the memory monitored by MONITORX. |
---|
100 | 156 | */ |
---|
101 | | - if (loops == 0) |
---|
| 157 | + if (!cycles) |
---|
102 | 158 | return; |
---|
103 | 159 | |
---|
104 | 160 | start = rdtsc_ordered(); |
---|
105 | 161 | |
---|
106 | 162 | for (;;) { |
---|
107 | | - delay = min_t(u64, MWAITX_MAX_LOOPS, loops); |
---|
108 | | - |
---|
109 | | - /* |
---|
110 | | - * Use cpu_tss_rw as a cacheline-aligned, seldomly |
---|
111 | | - * accessed per-cpu variable as the monitor target. |
---|
112 | | - */ |
---|
113 | | - __monitorx(raw_cpu_ptr(&cpu_tss_rw), 0, 0); |
---|
114 | | - |
---|
115 | | - /* |
---|
116 | | - * AMD, like Intel's MWAIT version, supports the EAX hint and |
---|
117 | | - * EAX=0xf0 means, do not enter any deep C-state and we use it |
---|
118 | | - * here in delay() to minimize wakeup latency. |
---|
119 | | - */ |
---|
120 | | - __mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE); |
---|
121 | | - |
---|
| 163 | + delay_halt_fn(start, cycles); |
---|
122 | 164 | end = rdtsc_ordered(); |
---|
123 | 165 | |
---|
124 | | - if (loops <= end - start) |
---|
| 166 | + if (cycles <= end - start) |
---|
125 | 167 | break; |
---|
126 | 168 | |
---|
127 | | - loops -= end - start; |
---|
128 | | - |
---|
| 169 | + cycles -= end - start; |
---|
129 | 170 | start = end; |
---|
130 | 171 | } |
---|
131 | 172 | } |
---|
132 | 173 | |
---|
133 | | -/* |
---|
134 | | - * Since we calibrate only once at boot, this |
---|
135 | | - * function should be set once at boot and not changed |
---|
136 | | - */ |
---|
137 | | -static void (*delay_fn)(unsigned long) = delay_loop; |
---|
138 | | - |
---|
139 | | -void use_tsc_delay(void) |
---|
| 174 | +void __init use_tsc_delay(void) |
---|
140 | 175 | { |
---|
141 | 176 | if (delay_fn == delay_loop) |
---|
142 | 177 | delay_fn = delay_tsc; |
---|
143 | 178 | } |
---|
144 | 179 | |
---|
| 180 | +void __init use_tpause_delay(void) |
---|
| 181 | +{ |
---|
| 182 | + delay_halt_fn = delay_halt_tpause; |
---|
| 183 | + delay_fn = delay_halt; |
---|
| 184 | +} |
---|
| 185 | + |
---|
145 | 186 | void use_mwaitx_delay(void) |
---|
146 | 187 | { |
---|
147 | | - delay_fn = delay_mwaitx; |
---|
| 188 | + delay_halt_fn = delay_halt_mwaitx; |
---|
| 189 | + delay_fn = delay_halt; |
---|
148 | 190 | } |
---|
149 | 191 | |
---|
150 | 192 | int read_current_timer(unsigned long *timer_val) |
---|
.. | .. |
---|
162 | 204 | } |
---|
163 | 205 | EXPORT_SYMBOL(__delay); |
---|
164 | 206 | |
---|
165 | | -void __const_udelay(unsigned long xloops) |
---|
| 207 | +noinline void __const_udelay(unsigned long xloops) |
---|
166 | 208 | { |
---|
167 | 209 | unsigned long lpj = this_cpu_read(cpu_info.loops_per_jiffy) ? : loops_per_jiffy; |
---|
168 | 210 | int d0; |
---|