| .. | .. | 
|---|
| 8 | 8 |   | 
|---|
| 9 | 9 |  DECLARE_PER_CPU(unsigned long, cpu_dr7); | 
|---|
| 10 | 10 |   | 
|---|
| 11 |  | -#ifndef CONFIG_PARAVIRT  | 
|---|
 | 11 | +#ifndef CONFIG_PARAVIRT_XXL  | 
|---|
| 12 | 12 |  /* | 
|---|
| 13 | 13 |   * These special macros can be used to get or set a debugging register | 
|---|
| 14 | 14 |   */ | 
|---|
| .. | .. | 
|---|
| 18 | 18 |  	native_set_debugreg(register, value) | 
|---|
| 19 | 19 |  #endif | 
|---|
| 20 | 20 |   | 
|---|
| 21 |  | -static inline unsigned long native_get_debugreg(int regno)  | 
|---|
 | 21 | +static __always_inline unsigned long native_get_debugreg(int regno)  | 
|---|
| 22 | 22 |  { | 
|---|
| 23 | 23 |  	unsigned long val = 0;	/* Damn you, gcc! */ | 
|---|
| 24 | 24 |   | 
|---|
| .. | .. | 
|---|
| 39 | 39 |  		asm("mov %%db6, %0" :"=r" (val)); | 
|---|
| 40 | 40 |  		break; | 
|---|
| 41 | 41 |  	case 7: | 
|---|
| 42 |  | -		asm("mov %%db7, %0" :"=r" (val));  | 
|---|
 | 42 | +		/*  | 
|---|
 | 43 | +		 * Apply __FORCE_ORDER to DR7 reads to forbid re-ordering them  | 
|---|
 | 44 | +		 * with other code.  | 
|---|
 | 45 | +		 *  | 
|---|
 | 46 | +		 * This is needed because a DR7 access can cause a #VC exception  | 
|---|
 | 47 | +		 * when running under SEV-ES. Taking a #VC exception is not a  | 
|---|
 | 48 | +		 * safe thing to do just anywhere in the entry code and  | 
|---|
 | 49 | +		 * re-ordering might place the access into an unsafe location.  | 
|---|
 | 50 | +		 *  | 
|---|
 | 51 | +		 * This happened in the NMI handler, where the DR7 read was  | 
|---|
 | 52 | +		 * re-ordered to happen before the call to sev_es_ist_enter(),  | 
|---|
 | 53 | +		 * causing stack recursion.  | 
|---|
 | 54 | +		 */  | 
|---|
 | 55 | +		asm volatile("mov %%db7, %0" : "=r" (val) : __FORCE_ORDER);  | 
|---|
| 43 | 56 |  		break; | 
|---|
| 44 | 57 |  	default: | 
|---|
| 45 | 58 |  		BUG(); | 
|---|
| .. | .. | 
|---|
| 47 | 60 |  	return val; | 
|---|
| 48 | 61 |  } | 
|---|
| 49 | 62 |   | 
|---|
| 50 |  | -static inline void native_set_debugreg(int regno, unsigned long value)  | 
|---|
 | 63 | +static __always_inline void native_set_debugreg(int regno, unsigned long value)  | 
|---|
| 51 | 64 |  { | 
|---|
| 52 | 65 |  	switch (regno) { | 
|---|
| 53 | 66 |  	case 0: | 
|---|
| .. | .. | 
|---|
| 66 | 79 |  		asm("mov %0, %%db6"	::"r" (value)); | 
|---|
| 67 | 80 |  		break; | 
|---|
| 68 | 81 |  	case 7: | 
|---|
| 69 |  | -		asm("mov %0, %%db7"	::"r" (value));  | 
|---|
 | 82 | +		/*  | 
|---|
 | 83 | +		 * Apply __FORCE_ORDER to DR7 writes to forbid re-ordering them  | 
|---|
 | 84 | +		 * with other code.  | 
|---|
 | 85 | +		 *  | 
|---|
 | 86 | +		 * While is didn't happen with a DR7 write (see the DR7 read  | 
|---|
 | 87 | +		 * comment above which explains where it happened), add the  | 
|---|
 | 88 | +		 * __FORCE_ORDER here too to avoid similar problems in the  | 
|---|
 | 89 | +		 * future.  | 
|---|
 | 90 | +		 */  | 
|---|
 | 91 | +		asm volatile("mov %0, %%db7"	::"r" (value), __FORCE_ORDER);  | 
|---|
| 70 | 92 |  		break; | 
|---|
| 71 | 93 |  	default: | 
|---|
| 72 | 94 |  		BUG(); | 
|---|
| .. | .. | 
|---|
| 85 | 107 |  	set_debugreg(0UL, 3); | 
|---|
| 86 | 108 |  } | 
|---|
| 87 | 109 |   | 
|---|
| 88 |  | -static inline int hw_breakpoint_active(void)  | 
|---|
 | 110 | +static __always_inline bool hw_breakpoint_active(void)  | 
|---|
| 89 | 111 |  { | 
|---|
| 90 | 112 |  	return __this_cpu_read(cpu_dr7) & DR_GLOBAL_ENABLE_MASK; | 
|---|
| 91 | 113 |  } | 
|---|
| 92 | 114 |   | 
|---|
| 93 |  | -extern void aout_dump_debugregs(struct user *dump);  | 
|---|
| 94 |  | -  | 
|---|
| 95 | 115 |  extern void hw_breakpoint_restore(void); | 
|---|
| 96 | 116 |   | 
|---|
| 97 |  | -#ifdef CONFIG_X86_64  | 
|---|
| 98 |  | -DECLARE_PER_CPU(int, debug_stack_usage);  | 
|---|
| 99 |  | -static inline void debug_stack_usage_inc(void)  | 
|---|
 | 117 | +static __always_inline unsigned long local_db_save(void)  | 
|---|
| 100 | 118 |  { | 
|---|
| 101 |  | -	__this_cpu_inc(debug_stack_usage);  | 
|---|
 | 119 | +	unsigned long dr7;  | 
|---|
 | 120 | +  | 
|---|
 | 121 | +	if (static_cpu_has(X86_FEATURE_HYPERVISOR) && !hw_breakpoint_active())  | 
|---|
 | 122 | +		return 0;  | 
|---|
 | 123 | +  | 
|---|
 | 124 | +	get_debugreg(dr7, 7);  | 
|---|
 | 125 | +	dr7 &= ~0x400; /* architecturally set bit */  | 
|---|
 | 126 | +	if (dr7)  | 
|---|
 | 127 | +		set_debugreg(0, 7);  | 
|---|
 | 128 | +	/*  | 
|---|
 | 129 | +	 * Ensure the compiler doesn't lower the above statements into  | 
|---|
 | 130 | +	 * the critical section; disabling breakpoints late would not  | 
|---|
 | 131 | +	 * be good.  | 
|---|
 | 132 | +	 */  | 
|---|
 | 133 | +	barrier();  | 
|---|
 | 134 | +  | 
|---|
 | 135 | +	return dr7;  | 
|---|
| 102 | 136 |  } | 
|---|
| 103 |  | -static inline void debug_stack_usage_dec(void)  | 
|---|
 | 137 | +  | 
|---|
 | 138 | +static __always_inline void local_db_restore(unsigned long dr7)  | 
|---|
| 104 | 139 |  { | 
|---|
| 105 |  | -	__this_cpu_dec(debug_stack_usage);  | 
|---|
 | 140 | +	/*  | 
|---|
 | 141 | +	 * Ensure the compiler doesn't raise this statement into  | 
|---|
 | 142 | +	 * the critical section; enabling breakpoints early would  | 
|---|
 | 143 | +	 * not be good.  | 
|---|
 | 144 | +	 */  | 
|---|
 | 145 | +	barrier();  | 
|---|
 | 146 | +	if (dr7)  | 
|---|
 | 147 | +		set_debugreg(dr7, 7);  | 
|---|
| 106 | 148 |  } | 
|---|
| 107 |  | -int is_debug_stack(unsigned long addr);  | 
|---|
| 108 |  | -void debug_stack_set_zero(void);  | 
|---|
| 109 |  | -void debug_stack_reset(void);  | 
|---|
| 110 |  | -#else /* !X86_64 */  | 
|---|
| 111 |  | -static inline int is_debug_stack(unsigned long addr) { return 0; }  | 
|---|
| 112 |  | -static inline void debug_stack_set_zero(void) { }  | 
|---|
| 113 |  | -static inline void debug_stack_reset(void) { }  | 
|---|
| 114 |  | -static inline void debug_stack_usage_inc(void) { }  | 
|---|
| 115 |  | -static inline void debug_stack_usage_dec(void) { }  | 
|---|
| 116 |  | -#endif /* X86_64 */  | 
|---|
| 117 | 149 |   | 
|---|
| 118 | 150 |  #ifdef CONFIG_CPU_SUP_AMD | 
|---|
| 119 | 151 |  extern void set_dr_addr_mask(unsigned long mask, int dr); | 
|---|