| .. | .. | 
|---|
| 60 | 60 |  #define EAX_EDX_RET(val, low, high)	"=A" (val) | 
|---|
| 61 | 61 |  #endif | 
|---|
| 62 | 62 |   | 
|---|
| 63 |  | -#ifdef CONFIG_TRACEPOINTS  | 
|---|
| 64 | 63 |  /* | 
|---|
| 65 | 64 |   * Be very careful with includes. This header is prone to include loops. | 
|---|
| 66 | 65 |   */ | 
|---|
| 67 | 66 |  #include <asm/atomic.h> | 
|---|
| 68 | 67 |  #include <linux/tracepoint-defs.h> | 
|---|
| 69 | 68 |   | 
|---|
| 70 |  | -extern struct tracepoint __tracepoint_read_msr;  | 
|---|
| 71 |  | -extern struct tracepoint __tracepoint_write_msr;  | 
|---|
| 72 |  | -extern struct tracepoint __tracepoint_rdpmc;  | 
|---|
| 73 |  | -#define msr_tracepoint_active(t) static_key_false(&(t).key)  | 
|---|
 | 69 | +#ifdef CONFIG_TRACEPOINTS  | 
|---|
 | 70 | +DECLARE_TRACEPOINT(read_msr);  | 
|---|
 | 71 | +DECLARE_TRACEPOINT(write_msr);  | 
|---|
 | 72 | +DECLARE_TRACEPOINT(rdpmc);  | 
|---|
| 74 | 73 |  extern void do_trace_write_msr(unsigned int msr, u64 val, int failed); | 
|---|
| 75 | 74 |  extern void do_trace_read_msr(unsigned int msr, u64 val, int failed); | 
|---|
| 76 | 75 |  extern void do_trace_rdpmc(unsigned int msr, u64 val, int failed); | 
|---|
| 77 | 76 |  #else | 
|---|
| 78 |  | -#define msr_tracepoint_active(t) false  | 
|---|
| 79 | 77 |  static inline void do_trace_write_msr(unsigned int msr, u64 val, int failed) {} | 
|---|
| 80 | 78 |  static inline void do_trace_read_msr(unsigned int msr, u64 val, int failed) {} | 
|---|
| 81 | 79 |  static inline void do_trace_rdpmc(unsigned int msr, u64 val, int failed) {} | 
|---|
| .. | .. | 
|---|
| 128 | 126 |   | 
|---|
| 129 | 127 |  	val = __rdmsr(msr); | 
|---|
| 130 | 128 |   | 
|---|
| 131 |  | -	if (msr_tracepoint_active(__tracepoint_read_msr))  | 
|---|
 | 129 | +	if (tracepoint_enabled(read_msr))  | 
|---|
| 132 | 130 |  		do_trace_read_msr(msr, val, 0); | 
|---|
| 133 | 131 |   | 
|---|
| 134 | 132 |  	return val; | 
|---|
| .. | .. | 
|---|
| 150 | 148 |  		     _ASM_EXTABLE(2b, 3b) | 
|---|
| 151 | 149 |  		     : [err] "=r" (*err), EAX_EDX_RET(val, low, high) | 
|---|
| 152 | 150 |  		     : "c" (msr), [fault] "i" (-EIO)); | 
|---|
| 153 |  | -	if (msr_tracepoint_active(__tracepoint_read_msr))  | 
|---|
 | 151 | +	if (tracepoint_enabled(read_msr))  | 
|---|
| 154 | 152 |  		do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err); | 
|---|
| 155 | 153 |  	return EAX_EDX_VAL(val, low, high); | 
|---|
| 156 | 154 |  } | 
|---|
| .. | .. | 
|---|
| 161 | 159 |  { | 
|---|
| 162 | 160 |  	__wrmsr(msr, low, high); | 
|---|
| 163 | 161 |   | 
|---|
| 164 |  | -	if (msr_tracepoint_active(__tracepoint_write_msr))  | 
|---|
 | 162 | +	if (tracepoint_enabled(write_msr))  | 
|---|
| 165 | 163 |  		do_trace_write_msr(msr, ((u64)high << 32 | low), 0); | 
|---|
| 166 | 164 |  } | 
|---|
| 167 | 165 |   | 
|---|
| .. | .. | 
|---|
| 181 | 179 |  		     : "c" (msr), "0" (low), "d" (high), | 
|---|
| 182 | 180 |  		       [fault] "i" (-EIO) | 
|---|
| 183 | 181 |  		     : "memory"); | 
|---|
| 184 |  | -	if (msr_tracepoint_active(__tracepoint_write_msr))  | 
|---|
 | 182 | +	if (tracepoint_enabled(write_msr))  | 
|---|
| 185 | 183 |  		do_trace_write_msr(msr, ((u64)high << 32 | low), err); | 
|---|
| 186 | 184 |  	return err; | 
|---|
| 187 | 185 |  } | 
|---|
| .. | .. | 
|---|
| 217 | 215 |   */ | 
|---|
| 218 | 216 |  static __always_inline unsigned long long rdtsc_ordered(void) | 
|---|
| 219 | 217 |  { | 
|---|
 | 218 | +	DECLARE_ARGS(val, low, high);  | 
|---|
 | 219 | +  | 
|---|
| 220 | 220 |  	/* | 
|---|
| 221 | 221 |  	 * The RDTSC instruction is not ordered relative to memory | 
|---|
| 222 | 222 |  	 * access.  The Intel SDM and the AMD APM are both vague on this | 
|---|
| .. | .. | 
|---|
| 227 | 227 |  	 * ordering guarantees as reading from a global memory location | 
|---|
| 228 | 228 |  	 * that some other imaginary CPU is updating continuously with a | 
|---|
| 229 | 229 |  	 * time stamp. | 
|---|
 | 230 | +	 *  | 
|---|
 | 231 | +	 * Thus, use the preferred barrier on the respective CPU, aiming for  | 
|---|
 | 232 | +	 * RDTSCP as the default.  | 
|---|
| 230 | 233 |  	 */ | 
|---|
| 231 |  | -	barrier_nospec();  | 
|---|
| 232 |  | -	return rdtsc();  | 
|---|
 | 234 | +	asm volatile(ALTERNATIVE_2("rdtsc",  | 
|---|
 | 235 | +				   "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,  | 
|---|
 | 236 | +				   "rdtscp", X86_FEATURE_RDTSCP)  | 
|---|
 | 237 | +			: EAX_EDX_RET(val, low, high)  | 
|---|
 | 238 | +			/* RDTSCP clobbers ECX with MSR_TSC_AUX. */  | 
|---|
 | 239 | +			:: "ecx");  | 
|---|
 | 240 | +  | 
|---|
 | 241 | +	return EAX_EDX_VAL(val, low, high);  | 
|---|
| 233 | 242 |  } | 
|---|
| 234 | 243 |   | 
|---|
| 235 | 244 |  static inline unsigned long long native_read_pmc(int counter) | 
|---|
| .. | .. | 
|---|
| 237 | 246 |  	DECLARE_ARGS(val, low, high); | 
|---|
| 238 | 247 |   | 
|---|
| 239 | 248 |  	asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter)); | 
|---|
| 240 |  | -	if (msr_tracepoint_active(__tracepoint_rdpmc))  | 
|---|
 | 249 | +	if (tracepoint_enabled(rdpmc))  | 
|---|
| 241 | 250 |  		do_trace_rdpmc(counter, EAX_EDX_VAL(val, low, high), 0); | 
|---|
| 242 | 251 |  	return EAX_EDX_VAL(val, low, high); | 
|---|
| 243 | 252 |  } | 
|---|
| 244 | 253 |   | 
|---|
| 245 |  | -#ifdef CONFIG_PARAVIRT  | 
|---|
 | 254 | +#ifdef CONFIG_PARAVIRT_XXL  | 
|---|
| 246 | 255 |  #include <asm/paravirt.h> | 
|---|
| 247 | 256 |  #else | 
|---|
| 248 | 257 |  #include <linux/errno.h> | 
|---|
| .. | .. | 
|---|
| 305 | 314 |   | 
|---|
| 306 | 315 |  #define rdpmcl(counter, val) ((val) = native_read_pmc(counter)) | 
|---|
| 307 | 316 |   | 
|---|
| 308 |  | -#endif	/* !CONFIG_PARAVIRT */  | 
|---|
 | 317 | +#endif	/* !CONFIG_PARAVIRT_XXL */  | 
|---|
| 309 | 318 |   | 
|---|
| 310 | 319 |  /* | 
|---|
| 311 | 320 |   * 64-bit version of wrmsr_safe(): | 
|---|