| .. | .. |
|---|
| 73 | 73 | #endif |
|---|
| 74 | 74 | } |
|---|
| 75 | 75 | |
|---|
| 76 | | -static inline int is_kprobe_on_ftrace(struct ftrace_insn *insn) |
|---|
| 77 | | -{ |
|---|
| 78 | | -#ifdef CONFIG_KPROBES |
|---|
| 79 | | - if (insn->opc == BREAKPOINT_INSTRUCTION) |
|---|
| 80 | | - return 1; |
|---|
| 81 | | -#endif |
|---|
| 82 | | - return 0; |
|---|
| 83 | | -} |
|---|
| 84 | | - |
|---|
| 85 | | -static inline void ftrace_generate_kprobe_nop_insn(struct ftrace_insn *insn) |
|---|
| 86 | | -{ |
|---|
| 87 | | -#ifdef CONFIG_KPROBES |
|---|
| 88 | | - insn->opc = BREAKPOINT_INSTRUCTION; |
|---|
| 89 | | - insn->disp = KPROBE_ON_FTRACE_NOP; |
|---|
| 90 | | -#endif |
|---|
| 91 | | -} |
|---|
| 92 | | - |
|---|
| 93 | | -static inline void ftrace_generate_kprobe_call_insn(struct ftrace_insn *insn) |
|---|
| 94 | | -{ |
|---|
| 95 | | -#ifdef CONFIG_KPROBES |
|---|
| 96 | | - insn->opc = BREAKPOINT_INSTRUCTION; |
|---|
| 97 | | - insn->disp = KPROBE_ON_FTRACE_CALL; |
|---|
| 98 | | -#endif |
|---|
| 99 | | -} |
|---|
| 100 | | - |
|---|
| 101 | 76 | int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, |
|---|
| 102 | 77 | unsigned long addr) |
|---|
| 103 | 78 | { |
|---|
| .. | .. |
|---|
| 109 | 84 | { |
|---|
| 110 | 85 | struct ftrace_insn orig, new, old; |
|---|
| 111 | 86 | |
|---|
| 112 | | - if (probe_kernel_read(&old, (void *) rec->ip, sizeof(old))) |
|---|
| 87 | + if (copy_from_kernel_nofault(&old, (void *) rec->ip, sizeof(old))) |
|---|
| 113 | 88 | return -EFAULT; |
|---|
| 114 | 89 | if (addr == MCOUNT_ADDR) { |
|---|
| 115 | 90 | /* Initial code replacement */ |
|---|
| 116 | 91 | ftrace_generate_orig_insn(&orig); |
|---|
| 117 | 92 | ftrace_generate_nop_insn(&new); |
|---|
| 118 | | - } else if (is_kprobe_on_ftrace(&old)) { |
|---|
| 119 | | - /* |
|---|
| 120 | | - * If we find a breakpoint instruction, a kprobe has been |
|---|
| 121 | | - * placed at the beginning of the function. We write the |
|---|
| 122 | | - * constant KPROBE_ON_FTRACE_NOP into the remaining four |
|---|
| 123 | | - * bytes of the original instruction so that the kprobes |
|---|
| 124 | | - * handler can execute a nop, if it reaches this breakpoint. |
|---|
| 125 | | - */ |
|---|
| 126 | | - ftrace_generate_kprobe_call_insn(&orig); |
|---|
| 127 | | - ftrace_generate_kprobe_nop_insn(&new); |
|---|
| 128 | 93 | } else { |
|---|
| 129 | 94 | /* Replace ftrace call with a nop. */ |
|---|
| 130 | 95 | ftrace_generate_call_insn(&orig, rec->ip); |
|---|
| .. | .. |
|---|
| 141 | 106 | { |
|---|
| 142 | 107 | struct ftrace_insn orig, new, old; |
|---|
| 143 | 108 | |
|---|
| 144 | | - if (probe_kernel_read(&old, (void *) rec->ip, sizeof(old))) |
|---|
| 109 | + if (copy_from_kernel_nofault(&old, (void *) rec->ip, sizeof(old))) |
|---|
| 145 | 110 | return -EFAULT; |
|---|
| 146 | | - if (is_kprobe_on_ftrace(&old)) { |
|---|
| 147 | | - /* |
|---|
| 148 | | - * If we find a breakpoint instruction, a kprobe has been |
|---|
| 149 | | - * placed at the beginning of the function. We write the |
|---|
| 150 | | - * constant KPROBE_ON_FTRACE_CALL into the remaining four |
|---|
| 151 | | - * bytes of the original instruction so that the kprobes |
|---|
| 152 | | - * handler can execute a brasl if it reaches this breakpoint. |
|---|
| 153 | | - */ |
|---|
| 154 | | - ftrace_generate_kprobe_nop_insn(&orig); |
|---|
| 155 | | - ftrace_generate_kprobe_call_insn(&new); |
|---|
| 156 | | - } else { |
|---|
| 157 | | - /* Replace nop with an ftrace call. */ |
|---|
| 158 | | - ftrace_generate_nop_insn(&orig); |
|---|
| 159 | | - ftrace_generate_call_insn(&new, rec->ip); |
|---|
| 160 | | - } |
|---|
| 111 | + /* Replace nop with an ftrace call. */ |
|---|
| 112 | + ftrace_generate_nop_insn(&orig); |
|---|
| 113 | + ftrace_generate_call_insn(&new, rec->ip); |
|---|
| 114 | + |
|---|
| 161 | 115 | /* Verify that the to be replaced code matches what we expect. */ |
|---|
| 162 | 116 | if (memcmp(&orig, &old, sizeof(old))) |
|---|
| 163 | 117 | return -EINVAL; |
|---|
| .. | .. |
|---|
| 203 | 157 | * Hook the return address and push it in the stack of return addresses |
|---|
| 204 | 158 | * in current thread info. |
|---|
| 205 | 159 | */ |
|---|
| 206 | | -unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip) |
|---|
| 160 | +unsigned long prepare_ftrace_return(unsigned long ra, unsigned long sp, |
|---|
| 161 | + unsigned long ip) |
|---|
| 207 | 162 | { |
|---|
| 208 | 163 | if (unlikely(ftrace_graph_is_dead())) |
|---|
| 209 | 164 | goto out; |
|---|
| 210 | 165 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) |
|---|
| 211 | 166 | goto out; |
|---|
| 212 | 167 | ip -= MCOUNT_INSN_SIZE; |
|---|
| 213 | | - if (!function_graph_enter(parent, ip, 0, NULL)) |
|---|
| 214 | | - parent = (unsigned long) return_to_handler; |
|---|
| 168 | + if (!function_graph_enter(ra, ip, 0, (void *) sp)) |
|---|
| 169 | + ra = (unsigned long) return_to_handler; |
|---|
| 215 | 170 | out: |
|---|
| 216 | | - return parent; |
|---|
| 171 | + return ra; |
|---|
| 217 | 172 | } |
|---|
| 218 | 173 | NOKPROBE_SYMBOL(prepare_ftrace_return); |
|---|
| 219 | 174 | |
|---|
| .. | .. |
|---|
| 242 | 197 | } |
|---|
| 243 | 198 | |
|---|
| 244 | 199 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
|---|
| 200 | + |
|---|
| 201 | +#ifdef CONFIG_KPROBES_ON_FTRACE |
|---|
| 202 | +void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, |
|---|
| 203 | + struct ftrace_ops *ops, struct pt_regs *regs) |
|---|
| 204 | +{ |
|---|
| 205 | + struct kprobe_ctlblk *kcb; |
|---|
| 206 | + struct kprobe *p = get_kprobe((kprobe_opcode_t *)ip); |
|---|
| 207 | + |
|---|
| 208 | + if (unlikely(!p) || kprobe_disabled(p)) |
|---|
| 209 | + return; |
|---|
| 210 | + |
|---|
| 211 | + if (kprobe_running()) { |
|---|
| 212 | + kprobes_inc_nmissed_count(p); |
|---|
| 213 | + return; |
|---|
| 214 | + } |
|---|
| 215 | + |
|---|
| 216 | + __this_cpu_write(current_kprobe, p); |
|---|
| 217 | + |
|---|
| 218 | + kcb = get_kprobe_ctlblk(); |
|---|
| 219 | + kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
|---|
| 220 | + |
|---|
| 221 | + instruction_pointer_set(regs, ip); |
|---|
| 222 | + |
|---|
| 223 | + if (!p->pre_handler || !p->pre_handler(p, regs)) { |
|---|
| 224 | + |
|---|
| 225 | + instruction_pointer_set(regs, ip + MCOUNT_INSN_SIZE); |
|---|
| 226 | + |
|---|
| 227 | + if (unlikely(p->post_handler)) { |
|---|
| 228 | + kcb->kprobe_status = KPROBE_HIT_SSDONE; |
|---|
| 229 | + p->post_handler(p, regs, 0); |
|---|
| 230 | + } |
|---|
| 231 | + } |
|---|
| 232 | + __this_cpu_write(current_kprobe, NULL); |
|---|
| 233 | +} |
|---|
| 234 | +NOKPROBE_SYMBOL(kprobe_ftrace_handler); |
|---|
| 235 | + |
|---|
| 236 | +int arch_prepare_kprobe_ftrace(struct kprobe *p) |
|---|
| 237 | +{ |
|---|
| 238 | + p->ainsn.insn = NULL; |
|---|
| 239 | + return 0; |
|---|
| 240 | +} |
|---|
| 241 | +#endif |
|---|