.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* |
---|
2 | 3 | * (C) 2010,2011 Thomas Renninger <trenn@suse.de>, Novell Inc. |
---|
3 | | - * |
---|
4 | | - * Licensed under the terms of the GNU GPL License version 2. |
---|
5 | 4 | */ |
---|
6 | 5 | |
---|
7 | 6 | #if defined(__i386__) || defined(__x86_64__) |
---|
.. | .. |
---|
19 | 18 | |
---|
20 | 19 | #define MSR_APERF 0xE8 |
---|
21 | 20 | #define MSR_MPERF 0xE7 |
---|
| 21 | + |
---|
| 22 | +#define RDPRU ".byte 0x0f, 0x01, 0xfd" |
---|
| 23 | +#define RDPRU_ECX_MPERF 0 |
---|
| 24 | +#define RDPRU_ECX_APERF 1 |
---|
22 | 25 | |
---|
23 | 26 | #define MSR_TSC 0x10 |
---|
24 | 27 | |
---|
.. | .. |
---|
67 | 70 | */ |
---|
68 | 71 | static unsigned long max_frequency; |
---|
69 | 72 | |
---|
70 | | -static unsigned long long tsc_at_measure_start; |
---|
71 | | -static unsigned long long tsc_at_measure_end; |
---|
| 73 | +static unsigned long long *tsc_at_measure_start; |
---|
| 74 | +static unsigned long long *tsc_at_measure_end; |
---|
72 | 75 | static unsigned long long *mperf_previous_count; |
---|
73 | 76 | static unsigned long long *aperf_previous_count; |
---|
74 | 77 | static unsigned long long *mperf_current_count; |
---|
.. | .. |
---|
87 | 90 | return ret; |
---|
88 | 91 | } |
---|
89 | 92 | |
---|
90 | | -static int mperf_init_stats(unsigned int cpu) |
---|
| 93 | +static int get_aperf_mperf(int cpu, unsigned long long *aval, |
---|
| 94 | + unsigned long long *mval) |
---|
91 | 95 | { |
---|
92 | | - unsigned long long val; |
---|
| 96 | + unsigned long low_a, high_a; |
---|
| 97 | + unsigned long low_m, high_m; |
---|
93 | 98 | int ret; |
---|
94 | 99 | |
---|
95 | | - ret = read_msr(cpu, MSR_APERF, &val); |
---|
96 | | - aperf_previous_count[cpu] = val; |
---|
97 | | - ret |= read_msr(cpu, MSR_MPERF, &val); |
---|
98 | | - mperf_previous_count[cpu] = val; |
---|
| 100 | + /* |
---|
| 101 | + * Running on the cpu from which we read the registers will |
---|
| 102 | + * prevent APERF/MPERF from going out of sync because of IPI |
---|
| 103 | + * latency introduced by read_msr()s. |
---|
| 104 | + */ |
---|
| 105 | + if (mperf_monitor.flags.per_cpu_schedule) { |
---|
| 106 | + if (bind_cpu(cpu)) |
---|
| 107 | + return 1; |
---|
| 108 | + } |
---|
| 109 | + |
---|
| 110 | + if (cpupower_cpu_info.caps & CPUPOWER_CAP_AMD_RDPRU) { |
---|
| 111 | + asm volatile(RDPRU |
---|
| 112 | + : "=a" (low_a), "=d" (high_a) |
---|
| 113 | + : "c" (RDPRU_ECX_APERF)); |
---|
| 114 | + asm volatile(RDPRU |
---|
| 115 | + : "=a" (low_m), "=d" (high_m) |
---|
| 116 | + : "c" (RDPRU_ECX_MPERF)); |
---|
| 117 | + |
---|
| 118 | + *aval = ((low_a) | (high_a) << 32); |
---|
| 119 | + *mval = ((low_m) | (high_m) << 32); |
---|
| 120 | + |
---|
| 121 | + return 0; |
---|
| 122 | + } |
---|
| 123 | + |
---|
| 124 | + ret = read_msr(cpu, MSR_APERF, aval); |
---|
| 125 | + ret |= read_msr(cpu, MSR_MPERF, mval); |
---|
| 126 | + |
---|
| 127 | + return ret; |
---|
| 128 | +} |
---|
| 129 | + |
---|
| 130 | +static int mperf_init_stats(unsigned int cpu) |
---|
| 131 | +{ |
---|
| 132 | + unsigned long long aval, mval; |
---|
| 133 | + int ret; |
---|
| 134 | + |
---|
| 135 | + ret = get_aperf_mperf(cpu, &aval, &mval); |
---|
| 136 | + aperf_previous_count[cpu] = aval; |
---|
| 137 | + mperf_previous_count[cpu] = mval; |
---|
99 | 138 | is_valid[cpu] = !ret; |
---|
100 | 139 | |
---|
101 | 140 | return 0; |
---|
.. | .. |
---|
103 | 142 | |
---|
104 | 143 | static int mperf_measure_stats(unsigned int cpu) |
---|
105 | 144 | { |
---|
106 | | - unsigned long long val; |
---|
| 145 | + unsigned long long aval, mval; |
---|
107 | 146 | int ret; |
---|
108 | 147 | |
---|
109 | | - ret = read_msr(cpu, MSR_APERF, &val); |
---|
110 | | - aperf_current_count[cpu] = val; |
---|
111 | | - ret |= read_msr(cpu, MSR_MPERF, &val); |
---|
112 | | - mperf_current_count[cpu] = val; |
---|
| 148 | + ret = get_aperf_mperf(cpu, &aval, &mval); |
---|
| 149 | + aperf_current_count[cpu] = aval; |
---|
| 150 | + mperf_current_count[cpu] = mval; |
---|
113 | 151 | is_valid[cpu] = !ret; |
---|
114 | 152 | |
---|
115 | 153 | return 0; |
---|
.. | .. |
---|
131 | 169 | aperf_diff = aperf_current_count[cpu] - aperf_previous_count[cpu]; |
---|
132 | 170 | |
---|
133 | 171 | if (max_freq_mode == MAX_FREQ_TSC_REF) { |
---|
134 | | - tsc_diff = tsc_at_measure_end - tsc_at_measure_start; |
---|
| 172 | + tsc_diff = tsc_at_measure_end[cpu] - tsc_at_measure_start[cpu]; |
---|
135 | 173 | *percent = 100.0 * mperf_diff / tsc_diff; |
---|
136 | 174 | dprint("%s: TSC Ref - mperf_diff: %llu, tsc_diff: %llu\n", |
---|
137 | 175 | mperf_cstates[id].name, mperf_diff, tsc_diff); |
---|
.. | .. |
---|
168 | 206 | |
---|
169 | 207 | if (max_freq_mode == MAX_FREQ_TSC_REF) { |
---|
170 | 208 | /* Calculate max_freq from TSC count */ |
---|
171 | | - tsc_diff = tsc_at_measure_end - tsc_at_measure_start; |
---|
| 209 | + tsc_diff = tsc_at_measure_end[cpu] - tsc_at_measure_start[cpu]; |
---|
172 | 210 | time_diff = timespec_diff_us(time_start, time_end); |
---|
173 | 211 | max_frequency = tsc_diff / time_diff; |
---|
174 | 212 | } |
---|
.. | .. |
---|
187 | 225 | static int mperf_start(void) |
---|
188 | 226 | { |
---|
189 | 227 | int cpu; |
---|
190 | | - unsigned long long dbg; |
---|
191 | 228 | |
---|
192 | 229 | clock_gettime(CLOCK_REALTIME, &time_start); |
---|
193 | | - mperf_get_tsc(&tsc_at_measure_start); |
---|
194 | 230 | |
---|
195 | | - for (cpu = 0; cpu < cpu_count; cpu++) |
---|
| 231 | + for (cpu = 0; cpu < cpu_count; cpu++) { |
---|
| 232 | + mperf_get_tsc(&tsc_at_measure_start[cpu]); |
---|
196 | 233 | mperf_init_stats(cpu); |
---|
| 234 | + } |
---|
197 | 235 | |
---|
198 | | - mperf_get_tsc(&dbg); |
---|
199 | | - dprint("TSC diff: %llu\n", dbg - tsc_at_measure_start); |
---|
200 | 236 | return 0; |
---|
201 | 237 | } |
---|
202 | 238 | |
---|
203 | 239 | static int mperf_stop(void) |
---|
204 | 240 | { |
---|
205 | | - unsigned long long dbg; |
---|
206 | 241 | int cpu; |
---|
207 | 242 | |
---|
208 | | - for (cpu = 0; cpu < cpu_count; cpu++) |
---|
| 243 | + for (cpu = 0; cpu < cpu_count; cpu++) { |
---|
209 | 244 | mperf_measure_stats(cpu); |
---|
| 245 | + mperf_get_tsc(&tsc_at_measure_end[cpu]); |
---|
| 246 | + } |
---|
210 | 247 | |
---|
211 | | - mperf_get_tsc(&tsc_at_measure_end); |
---|
212 | 248 | clock_gettime(CLOCK_REALTIME, &time_end); |
---|
213 | | - |
---|
214 | | - mperf_get_tsc(&dbg); |
---|
215 | | - dprint("TSC diff: %llu\n", dbg - tsc_at_measure_end); |
---|
216 | | - |
---|
217 | 249 | return 0; |
---|
218 | 250 | } |
---|
219 | 251 | |
---|
.. | .. |
---|
241 | 273 | if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_INV_TSC)) |
---|
242 | 274 | goto use_sysfs; |
---|
243 | 275 | |
---|
244 | | - if (cpupower_cpu_info.vendor == X86_VENDOR_AMD) { |
---|
| 276 | + if (cpupower_cpu_info.vendor == X86_VENDOR_AMD || |
---|
| 277 | + cpupower_cpu_info.vendor == X86_VENDOR_HYGON) { |
---|
245 | 278 | /* MSR_AMD_HWCR tells us whether TSC runs at P0/mperf |
---|
246 | 279 | * freq. |
---|
247 | 280 | * A test whether hwcr is accessable/available would be: |
---|
.. | .. |
---|
305 | 338 | if (init_maxfreq_mode()) |
---|
306 | 339 | return NULL; |
---|
307 | 340 | |
---|
| 341 | + if (cpupower_cpu_info.vendor == X86_VENDOR_AMD) |
---|
| 342 | + mperf_monitor.flags.per_cpu_schedule = 1; |
---|
| 343 | + |
---|
308 | 344 | /* Free this at program termination */ |
---|
309 | 345 | is_valid = calloc(cpu_count, sizeof(int)); |
---|
310 | 346 | mperf_previous_count = calloc(cpu_count, sizeof(unsigned long long)); |
---|
311 | 347 | aperf_previous_count = calloc(cpu_count, sizeof(unsigned long long)); |
---|
312 | 348 | mperf_current_count = calloc(cpu_count, sizeof(unsigned long long)); |
---|
313 | 349 | aperf_current_count = calloc(cpu_count, sizeof(unsigned long long)); |
---|
314 | | - |
---|
| 350 | + tsc_at_measure_start = calloc(cpu_count, sizeof(unsigned long long)); |
---|
| 351 | + tsc_at_measure_end = calloc(cpu_count, sizeof(unsigned long long)); |
---|
315 | 352 | mperf_monitor.name_len = strlen(mperf_monitor.name); |
---|
316 | 353 | return &mperf_monitor; |
---|
317 | 354 | } |
---|
.. | .. |
---|
322 | 359 | free(aperf_previous_count); |
---|
323 | 360 | free(mperf_current_count); |
---|
324 | 361 | free(aperf_current_count); |
---|
| 362 | + free(tsc_at_measure_start); |
---|
| 363 | + free(tsc_at_measure_end); |
---|
325 | 364 | free(is_valid); |
---|
326 | 365 | } |
---|
327 | 366 | |
---|
.. | .. |
---|
333 | 372 | .stop = mperf_stop, |
---|
334 | 373 | .do_register = mperf_register, |
---|
335 | 374 | .unregister = mperf_unregister, |
---|
336 | | - .needs_root = 1, |
---|
| 375 | + .flags.needs_root = 1, |
---|
337 | 376 | .overflow_s = 922000000 /* 922337203 seconds TSC overflow |
---|
338 | 377 | at 20GHz */ |
---|
339 | 378 | }; |
---|