hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da
kernel/arch/x86/lib/delay.c
....@@ -27,9 +27,20 @@
2727 # include <asm/smp.h>
2828 #endif
2929
30
+static void delay_loop(u64 __loops);
31
+
32
+/*
33
+ * Calibration and selection of the delay mechanism happens only once
34
+ * during boot.
35
+ */
36
+static void (*delay_fn)(u64) __ro_after_init = delay_loop;
37
+static void (*delay_halt_fn)(u64 start, u64 cycles) __ro_after_init;
38
+
3039 /* simple loop based delay: */
31
-static void delay_loop(unsigned long loops)
40
+static void delay_loop(u64 __loops)
3241 {
42
+ unsigned long loops = (unsigned long)__loops;
43
+
3344 asm volatile(
3445 " test %0,%0 \n"
3546 " jz 3f \n"
....@@ -43,15 +54,15 @@
4354 " jnz 2b \n"
4455 "3: dec %0 \n"
4556
46
- : /* we don't need output */
47
- :"a" (loops)
57
+ : "+a" (loops)
58
+ :
4859 );
4960 }
5061
5162 /* TSC based delay: */
52
-static void delay_tsc(unsigned long __loops)
63
+static void delay_tsc(u64 cycles)
5364 {
54
- u64 bclock, now, loops = __loops;
65
+ u64 bclock, now;
5566 int cpu;
5667
5768 preempt_disable();
....@@ -59,7 +70,7 @@
5970 bclock = rdtsc_ordered();
6071 for (;;) {
6172 now = rdtsc_ordered();
62
- if ((now - bclock) >= loops)
73
+ if ((now - bclock) >= cycles)
6374 break;
6475
6576 /* Allow RT tasks to run */
....@@ -77,7 +88,7 @@
7788 * counter for this CPU.
7889 */
7990 if (unlikely(cpu != smp_processor_id())) {
80
- loops -= (now - bclock);
91
+ cycles -= (now - bclock);
8192 cpu = smp_processor_id();
8293 bclock = rdtsc_ordered();
8394 }
....@@ -86,65 +97,96 @@
8697 }
8798
8899 /*
89
- * On some AMD platforms, MWAITX has a configurable 32-bit timer, that
90
- * counts with TSC frequency. The input value is the loop of the
91
- * counter, it will exit when the timer expires.
100
+ * On Intel the TPAUSE instruction waits until any of:
101
+ * 1) the TSC counter exceeds the value provided in EDX:EAX
102
+ * 2) global timeout in IA32_UMWAIT_CONTROL is exceeded
103
+ * 3) an external interrupt occurs
92104 */
93
-static void delay_mwaitx(unsigned long __loops)
105
+static void delay_halt_tpause(u64 start, u64 cycles)
94106 {
95
- u64 start, end, delay, loops = __loops;
107
+ u64 until = start + cycles;
108
+ u32 eax, edx;
109
+
110
+ eax = lower_32_bits(until);
111
+ edx = upper_32_bits(until);
112
+
113
+ /*
114
+ * Hard code the deeper (C0.2) sleep state because exit latency is
115
+ * small compared to the "microseconds" that usleep() will delay.
116
+ */
117
+ __tpause(TPAUSE_C02_STATE, edx, eax);
118
+}
119
+
120
+/*
121
+ * On some AMD platforms, MWAITX has a configurable 32-bit timer, that
122
+ * counts with TSC frequency. The input value is the number of TSC cycles
123
+ * to wait. MWAITX will also exit when the timer expires.
124
+ */
125
+static void delay_halt_mwaitx(u64 unused, u64 cycles)
126
+{
127
+ u64 delay;
128
+
129
+ delay = min_t(u64, MWAITX_MAX_WAIT_CYCLES, cycles);
130
+ /*
131
+ * Use cpu_tss_rw as a cacheline-aligned, seldomly accessed per-cpu
132
+ * variable as the monitor target.
133
+ */
134
+ __monitorx(raw_cpu_ptr(&cpu_tss_rw), 0, 0);
135
+
136
+ /*
137
+ * AMD, like Intel, supports the EAX hint and EAX=0xf means, do not
138
+ * enter any deep C-state and we use it here in delay() to minimize
139
+ * wakeup latency.
140
+ */
141
+ __mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE);
142
+}
143
+
144
+/*
145
+ * Call a vendor specific function to delay for a given amount of time. Because
146
+ * these functions may return earlier than requested, check for actual elapsed
147
+ * time and call again until done.
148
+ */
149
+static void delay_halt(u64 __cycles)
150
+{
151
+ u64 start, end, cycles = __cycles;
96152
97153 /*
98154 * Timer value of 0 causes MWAITX to wait indefinitely, unless there
99155 * is a store on the memory monitored by MONITORX.
100156 */
101
- if (loops == 0)
157
+ if (!cycles)
102158 return;
103159
104160 start = rdtsc_ordered();
105161
106162 for (;;) {
107
- delay = min_t(u64, MWAITX_MAX_LOOPS, loops);
108
-
109
- /*
110
- * Use cpu_tss_rw as a cacheline-aligned, seldomly
111
- * accessed per-cpu variable as the monitor target.
112
- */
113
- __monitorx(raw_cpu_ptr(&cpu_tss_rw), 0, 0);
114
-
115
- /*
116
- * AMD, like Intel's MWAIT version, supports the EAX hint and
117
- * EAX=0xf0 means, do not enter any deep C-state and we use it
118
- * here in delay() to minimize wakeup latency.
119
- */
120
- __mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE);
121
-
163
+ delay_halt_fn(start, cycles);
122164 end = rdtsc_ordered();
123165
124
- if (loops <= end - start)
166
+ if (cycles <= end - start)
125167 break;
126168
127
- loops -= end - start;
128
-
169
+ cycles -= end - start;
129170 start = end;
130171 }
131172 }
132173
133
-/*
134
- * Since we calibrate only once at boot, this
135
- * function should be set once at boot and not changed
136
- */
137
-static void (*delay_fn)(unsigned long) = delay_loop;
138
-
139
-void use_tsc_delay(void)
174
+void __init use_tsc_delay(void)
140175 {
141176 if (delay_fn == delay_loop)
142177 delay_fn = delay_tsc;
143178 }
144179
180
+void __init use_tpause_delay(void)
181
+{
182
+ delay_halt_fn = delay_halt_tpause;
183
+ delay_fn = delay_halt;
184
+}
185
+
145186 void use_mwaitx_delay(void)
146187 {
147
- delay_fn = delay_mwaitx;
188
+ delay_halt_fn = delay_halt_mwaitx;
189
+ delay_fn = delay_halt;
148190 }
149191
150192 int read_current_timer(unsigned long *timer_val)
....@@ -162,7 +204,7 @@
162204 }
163205 EXPORT_SYMBOL(__delay);
164206
165
-void __const_udelay(unsigned long xloops)
207
+noinline void __const_udelay(unsigned long xloops)
166208 {
167209 unsigned long lpj = this_cpu_read(cpu_info.loops_per_jiffy) ? : loops_per_jiffy;
168210 int d0;