| .. | .. |
|---|
| 5 | 5 | #include <linux/sched.h> |
|---|
| 6 | 6 | #include <linux/vtime.h> |
|---|
| 7 | 7 | #include <linux/context_tracking_state.h> |
|---|
| 8 | +#include <linux/instrumentation.h> |
|---|
| 9 | + |
|---|
| 8 | 10 | #include <asm/ptrace.h> |
|---|
| 9 | 11 | |
|---|
| 10 | 12 | |
|---|
| .. | .. |
|---|
| 22 | 24 | |
|---|
| 23 | 25 | static inline void user_enter(void) |
|---|
| 24 | 26 | { |
|---|
| 25 | | - if (context_tracking_is_enabled()) |
|---|
| 27 | + if (context_tracking_enabled()) |
|---|
| 26 | 28 | context_tracking_enter(CONTEXT_USER); |
|---|
| 27 | 29 | |
|---|
| 28 | 30 | } |
|---|
| 29 | 31 | static inline void user_exit(void) |
|---|
| 30 | 32 | { |
|---|
| 31 | | - if (context_tracking_is_enabled()) |
|---|
| 33 | + if (context_tracking_enabled()) |
|---|
| 32 | 34 | context_tracking_exit(CONTEXT_USER); |
|---|
| 33 | 35 | } |
|---|
| 34 | 36 | |
|---|
| 35 | 37 | /* Called with interrupts disabled. */ |
|---|
| 36 | | -static inline void user_enter_irqoff(void) |
|---|
| 38 | +static __always_inline void user_enter_irqoff(void) |
|---|
| 37 | 39 | { |
|---|
| 38 | | - if (context_tracking_is_enabled()) |
|---|
| 40 | + if (context_tracking_enabled()) |
|---|
| 39 | 41 | __context_tracking_enter(CONTEXT_USER); |
|---|
| 40 | 42 | |
|---|
| 41 | 43 | } |
|---|
| 42 | | -static inline void user_exit_irqoff(void) |
|---|
| 44 | +static __always_inline void user_exit_irqoff(void) |
|---|
| 43 | 45 | { |
|---|
| 44 | | - if (context_tracking_is_enabled()) |
|---|
| 46 | + if (context_tracking_enabled()) |
|---|
| 45 | 47 | __context_tracking_exit(CONTEXT_USER); |
|---|
| 46 | 48 | } |
|---|
| 47 | 49 | |
|---|
| .. | .. |
|---|
| 49 | 51 | { |
|---|
| 50 | 52 | enum ctx_state prev_ctx; |
|---|
| 51 | 53 | |
|---|
| 52 | | - if (!context_tracking_is_enabled()) |
|---|
| 54 | + if (!context_tracking_enabled()) |
|---|
| 53 | 55 | return 0; |
|---|
| 54 | 56 | |
|---|
| 55 | 57 | prev_ctx = this_cpu_read(context_tracking.state); |
|---|
| .. | .. |
|---|
| 61 | 63 | |
|---|
| 62 | 64 | static inline void exception_exit(enum ctx_state prev_ctx) |
|---|
| 63 | 65 | { |
|---|
| 64 | | - if (context_tracking_is_enabled()) { |
|---|
| 66 | + if (context_tracking_enabled()) { |
|---|
| 65 | 67 | if (prev_ctx != CONTEXT_KERNEL) |
|---|
| 66 | 68 | context_tracking_enter(prev_ctx); |
|---|
| 67 | 69 | } |
|---|
| .. | .. |
|---|
| 75 | 77 | * is enabled. If context tracking is disabled, returns |
|---|
| 76 | 78 | * CONTEXT_DISABLED. This should be used primarily for debugging. |
|---|
| 77 | 79 | */ |
|---|
| 78 | | -static inline enum ctx_state ct_state(void) |
|---|
| 80 | +static __always_inline enum ctx_state ct_state(void) |
|---|
| 79 | 81 | { |
|---|
| 80 | | - return context_tracking_is_enabled() ? |
|---|
| 82 | + return context_tracking_enabled() ? |
|---|
| 81 | 83 | this_cpu_read(context_tracking.state) : CONTEXT_DISABLED; |
|---|
| 82 | 84 | } |
|---|
| 83 | 85 | #else |
|---|
| .. | .. |
|---|
| 90 | 92 | static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; } |
|---|
| 91 | 93 | #endif /* !CONFIG_CONTEXT_TRACKING */ |
|---|
| 92 | 94 | |
|---|
| 93 | | -#define CT_WARN_ON(cond) WARN_ON(context_tracking_is_enabled() && (cond)) |
|---|
| 95 | +#define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond)) |
|---|
| 94 | 96 | |
|---|
| 95 | 97 | #ifdef CONFIG_CONTEXT_TRACKING_FORCE |
|---|
| 96 | 98 | extern void context_tracking_init(void); |
|---|
| .. | .. |
|---|
| 101 | 103 | |
|---|
| 102 | 104 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN |
|---|
| 103 | 105 | /* must be called with irqs disabled */ |
|---|
| 104 | | -static inline void guest_enter_irqoff(void) |
|---|
| 106 | +static __always_inline void guest_enter_irqoff(void) |
|---|
| 105 | 107 | { |
|---|
| 106 | | - if (vtime_accounting_cpu_enabled()) |
|---|
| 108 | + instrumentation_begin(); |
|---|
| 109 | + if (vtime_accounting_enabled_this_cpu()) |
|---|
| 107 | 110 | vtime_guest_enter(current); |
|---|
| 108 | 111 | else |
|---|
| 109 | 112 | current->flags |= PF_VCPU; |
|---|
| 113 | + instrumentation_end(); |
|---|
| 110 | 114 | |
|---|
| 111 | | - if (context_tracking_is_enabled()) |
|---|
| 115 | + if (context_tracking_enabled()) |
|---|
| 112 | 116 | __context_tracking_enter(CONTEXT_GUEST); |
|---|
| 113 | 117 | |
|---|
| 114 | 118 | /* KVM does not hold any references to rcu protected data when it |
|---|
| .. | .. |
|---|
| 118 | 122 | * one time slice). Lets treat guest mode as quiescent state, just like |
|---|
| 119 | 123 | * we do with user-mode execution. |
|---|
| 120 | 124 | */ |
|---|
| 121 | | - if (!context_tracking_cpu_is_enabled()) |
|---|
| 125 | + if (!context_tracking_enabled_this_cpu()) { |
|---|
| 126 | + instrumentation_begin(); |
|---|
| 122 | 127 | rcu_virt_note_context_switch(smp_processor_id()); |
|---|
| 128 | + instrumentation_end(); |
|---|
| 129 | + } |
|---|
| 123 | 130 | } |
|---|
| 124 | 131 | |
|---|
| 125 | | -static inline void guest_exit_irqoff(void) |
|---|
| 132 | +static __always_inline void context_tracking_guest_exit(void) |
|---|
| 126 | 133 | { |
|---|
| 127 | | - if (context_tracking_is_enabled()) |
|---|
| 134 | + if (context_tracking_enabled()) |
|---|
| 128 | 135 | __context_tracking_exit(CONTEXT_GUEST); |
|---|
| 136 | +} |
|---|
| 129 | 137 | |
|---|
| 130 | | - if (vtime_accounting_cpu_enabled()) |
|---|
| 138 | +static __always_inline void vtime_account_guest_exit(void) |
|---|
| 139 | +{ |
|---|
| 140 | + if (vtime_accounting_enabled_this_cpu()) |
|---|
| 131 | 141 | vtime_guest_exit(current); |
|---|
| 132 | 142 | else |
|---|
| 133 | 143 | current->flags &= ~PF_VCPU; |
|---|
| 134 | 144 | } |
|---|
| 135 | 145 | |
|---|
| 146 | +static __always_inline void guest_exit_irqoff(void) |
|---|
| 147 | +{ |
|---|
| 148 | + context_tracking_guest_exit(); |
|---|
| 149 | + |
|---|
| 150 | + instrumentation_begin(); |
|---|
| 151 | + vtime_account_guest_exit(); |
|---|
| 152 | + instrumentation_end(); |
|---|
| 153 | +} |
|---|
| 154 | + |
|---|
| 136 | 155 | #else |
|---|
| 137 | | -static inline void guest_enter_irqoff(void) |
|---|
| 156 | +static __always_inline void guest_enter_irqoff(void) |
|---|
| 138 | 157 | { |
|---|
| 139 | 158 | /* |
|---|
| 140 | 159 | * This is running in ioctl context so its safe |
|---|
| 141 | 160 | * to assume that it's the stime pending cputime |
|---|
| 142 | 161 | * to flush. |
|---|
| 143 | 162 | */ |
|---|
| 144 | | - vtime_account_system(current); |
|---|
| 163 | + instrumentation_begin(); |
|---|
| 164 | + vtime_account_kernel(current); |
|---|
| 145 | 165 | current->flags |= PF_VCPU; |
|---|
| 146 | 166 | rcu_virt_note_context_switch(smp_processor_id()); |
|---|
| 167 | + instrumentation_end(); |
|---|
| 147 | 168 | } |
|---|
| 148 | 169 | |
|---|
| 149 | | -static inline void guest_exit_irqoff(void) |
|---|
| 170 | +static __always_inline void context_tracking_guest_exit(void) { } |
|---|
| 171 | + |
|---|
| 172 | +static __always_inline void vtime_account_guest_exit(void) |
|---|
| 150 | 173 | { |
|---|
| 151 | | - /* Flush the guest cputime we spent on the guest */ |
|---|
| 152 | | - vtime_account_system(current); |
|---|
| 174 | + vtime_account_kernel(current); |
|---|
| 153 | 175 | current->flags &= ~PF_VCPU; |
|---|
| 154 | 176 | } |
|---|
| 155 | | -#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */ |
|---|
| 156 | 177 | |
|---|
| 157 | | -static inline void guest_enter(void) |
|---|
| 178 | +static __always_inline void guest_exit_irqoff(void) |
|---|
| 158 | 179 | { |
|---|
| 159 | | - unsigned long flags; |
|---|
| 160 | | - |
|---|
| 161 | | - local_irq_save(flags); |
|---|
| 162 | | - guest_enter_irqoff(); |
|---|
| 163 | | - local_irq_restore(flags); |
|---|
| 180 | + instrumentation_begin(); |
|---|
| 181 | + /* Flush the guest cputime we spent on the guest */ |
|---|
| 182 | + vtime_account_guest_exit(); |
|---|
| 183 | + instrumentation_end(); |
|---|
| 164 | 184 | } |
|---|
| 185 | +#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */ |
|---|
| 165 | 186 | |
|---|
| 166 | 187 | static inline void guest_exit(void) |
|---|
| 167 | 188 | { |
|---|