.. | .. |
---|
7 | 7 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> |
---|
8 | 8 | * |
---|
9 | 9 | * future possible enhancements: |
---|
10 | | - * - add CONFIG_DYNAMIC_FTRACE |
---|
11 | 10 | * - add CONFIG_STACK_TRACER |
---|
12 | 11 | */ |
---|
13 | 12 | |
---|
14 | 13 | #include <linux/init.h> |
---|
15 | 14 | #include <linux/ftrace.h> |
---|
| 15 | +#include <linux/uaccess.h> |
---|
| 16 | +#include <linux/kprobes.h> |
---|
| 17 | +#include <linux/ptrace.h> |
---|
16 | 18 | |
---|
17 | 19 | #include <asm/assembly.h> |
---|
18 | 20 | #include <asm/sections.h> |
---|
19 | 21 | #include <asm/ftrace.h> |
---|
| 22 | +#include <asm/patch.h> |
---|
20 | 23 | |
---|
21 | | - |
---|
22 | | -#define __hot __attribute__ ((__section__ (".text.hot"))) |
---|
| 24 | +#define __hot __section(".text.hot") |
---|
23 | 25 | |
---|
24 | 26 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
---|
25 | 27 | /* |
---|
.. | .. |
---|
48 | 50 | |
---|
49 | 51 | void notrace __hot ftrace_function_trampoline(unsigned long parent, |
---|
50 | 52 | unsigned long self_addr, |
---|
51 | | - unsigned long org_sp_gr3) |
---|
| 53 | + unsigned long org_sp_gr3, |
---|
| 54 | + struct pt_regs *regs) |
---|
52 | 55 | { |
---|
53 | | - extern ftrace_func_t ftrace_trace_function; /* depends on CONFIG_DYNAMIC_FTRACE */ |
---|
54 | | - extern int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace); |
---|
| 56 | +#ifndef CONFIG_DYNAMIC_FTRACE |
---|
| 57 | + extern ftrace_func_t ftrace_trace_function; |
---|
| 58 | +#endif |
---|
| 59 | + extern struct ftrace_ops *function_trace_op; |
---|
55 | 60 | |
---|
56 | | - if (ftrace_trace_function != ftrace_stub) { |
---|
57 | | - /* struct ftrace_ops *op, struct pt_regs *regs); */ |
---|
58 | | - ftrace_trace_function(parent, self_addr, NULL, NULL); |
---|
59 | | - return; |
---|
60 | | - } |
---|
| 61 | + if (function_trace_op->flags & FTRACE_OPS_FL_ENABLED && |
---|
| 62 | + ftrace_trace_function != ftrace_stub) |
---|
| 63 | + ftrace_trace_function(self_addr, parent, |
---|
| 64 | + function_trace_op, regs); |
---|
61 | 65 | |
---|
62 | 66 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
---|
63 | | - if (ftrace_graph_return != (trace_func_graph_ret_t) ftrace_stub || |
---|
64 | | - ftrace_graph_entry != ftrace_graph_entry_stub) { |
---|
| 67 | + if (dereference_function_descriptor(ftrace_graph_return) != |
---|
| 68 | + dereference_function_descriptor(ftrace_stub) || |
---|
| 69 | + ftrace_graph_entry != ftrace_graph_entry_stub) { |
---|
65 | 70 | unsigned long *parent_rp; |
---|
66 | 71 | |
---|
67 | 72 | /* calculate pointer to %rp in stack */ |
---|
.. | .. |
---|
76 | 81 | #endif |
---|
77 | 82 | } |
---|
78 | 83 | |
---|
| 84 | +#ifdef CONFIG_FUNCTION_GRAPH_TRACER |
---|
| 85 | +int ftrace_enable_ftrace_graph_caller(void) |
---|
| 86 | +{ |
---|
| 87 | + return 0; |
---|
| 88 | +} |
---|
| 89 | + |
---|
| 90 | +int ftrace_disable_ftrace_graph_caller(void) |
---|
| 91 | +{ |
---|
| 92 | + return 0; |
---|
| 93 | +} |
---|
| 94 | +#endif |
---|
| 95 | + |
---|
| 96 | +#ifdef CONFIG_DYNAMIC_FTRACE |
---|
| 97 | + |
---|
| 98 | +int __init ftrace_dyn_arch_init(void) |
---|
| 99 | +{ |
---|
| 100 | + return 0; |
---|
| 101 | +} |
---|
| 102 | +int ftrace_update_ftrace_func(ftrace_func_t func) |
---|
| 103 | +{ |
---|
| 104 | + return 0; |
---|
| 105 | +} |
---|
| 106 | + |
---|
| 107 | +int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, |
---|
| 108 | + unsigned long addr) |
---|
| 109 | +{ |
---|
| 110 | + return 0; |
---|
| 111 | +} |
---|
| 112 | + |
---|
| 113 | +unsigned long ftrace_call_adjust(unsigned long addr) |
---|
| 114 | +{ |
---|
| 115 | + return addr+(FTRACE_PATCHABLE_FUNCTION_SIZE-1)*4; |
---|
| 116 | +} |
---|
| 117 | + |
---|
| 118 | +int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) |
---|
| 119 | +{ |
---|
| 120 | + u32 insn[FTRACE_PATCHABLE_FUNCTION_SIZE]; |
---|
| 121 | + u32 *tramp; |
---|
| 122 | + int size, ret, i; |
---|
| 123 | + void *ip; |
---|
| 124 | + |
---|
| 125 | +#ifdef CONFIG_64BIT |
---|
| 126 | + unsigned long addr2 = |
---|
| 127 | + (unsigned long)dereference_function_descriptor((void *)addr); |
---|
| 128 | + |
---|
| 129 | + u32 ftrace_trampoline[] = { |
---|
| 130 | + 0x73c10208, /* std,ma r1,100(sp) */ |
---|
| 131 | + 0x0c2110c1, /* ldd -10(r1),r1 */ |
---|
| 132 | + 0xe820d002, /* bve,n (r1) */ |
---|
| 133 | + addr2 >> 32, |
---|
| 134 | + addr2 & 0xffffffff, |
---|
| 135 | + 0xe83f1fd7, /* b,l,n .-14,r1 */ |
---|
| 136 | + }; |
---|
| 137 | + |
---|
| 138 | + u32 ftrace_trampoline_unaligned[] = { |
---|
| 139 | + addr2 >> 32, |
---|
| 140 | + addr2 & 0xffffffff, |
---|
| 141 | + 0x37de0200, /* ldo 100(sp),sp */ |
---|
| 142 | + 0x73c13e01, /* std r1,-100(sp) */ |
---|
| 143 | + 0x34213ff9, /* ldo -4(r1),r1 */ |
---|
| 144 | + 0x50213fc1, /* ldd -20(r1),r1 */ |
---|
| 145 | + 0xe820d002, /* bve,n (r1) */ |
---|
| 146 | + 0xe83f1fcf, /* b,l,n .-20,r1 */ |
---|
| 147 | + }; |
---|
| 148 | + |
---|
| 149 | + BUILD_BUG_ON(ARRAY_SIZE(ftrace_trampoline_unaligned) > |
---|
| 150 | + FTRACE_PATCHABLE_FUNCTION_SIZE); |
---|
| 151 | +#else |
---|
| 152 | + u32 ftrace_trampoline[] = { |
---|
| 153 | + (u32)addr, |
---|
| 154 | + 0x6fc10080, /* stw,ma r1,40(sp) */ |
---|
| 155 | + 0x48213fd1, /* ldw -18(r1),r1 */ |
---|
| 156 | + 0xe820c002, /* bv,n r0(r1) */ |
---|
| 157 | + 0xe83f1fdf, /* b,l,n .-c,r1 */ |
---|
| 158 | + }; |
---|
| 159 | +#endif |
---|
| 160 | + |
---|
| 161 | + BUILD_BUG_ON(ARRAY_SIZE(ftrace_trampoline) > |
---|
| 162 | + FTRACE_PATCHABLE_FUNCTION_SIZE); |
---|
| 163 | + |
---|
| 164 | + size = sizeof(ftrace_trampoline); |
---|
| 165 | + tramp = ftrace_trampoline; |
---|
| 166 | + |
---|
| 167 | +#ifdef CONFIG_64BIT |
---|
| 168 | + if (rec->ip & 0x4) { |
---|
| 169 | + size = sizeof(ftrace_trampoline_unaligned); |
---|
| 170 | + tramp = ftrace_trampoline_unaligned; |
---|
| 171 | + } |
---|
| 172 | +#endif |
---|
| 173 | + |
---|
| 174 | + ip = (void *)(rec->ip + 4 - size); |
---|
| 175 | + |
---|
| 176 | + ret = copy_from_kernel_nofault(insn, ip, size); |
---|
| 177 | + if (ret) |
---|
| 178 | + return ret; |
---|
| 179 | + |
---|
| 180 | + for (i = 0; i < size / 4; i++) { |
---|
| 181 | + if (insn[i] != INSN_NOP) |
---|
| 182 | + return -EINVAL; |
---|
| 183 | + } |
---|
| 184 | + |
---|
| 185 | + __patch_text_multiple(ip, tramp, size); |
---|
| 186 | + return 0; |
---|
| 187 | +} |
---|
| 188 | + |
---|
| 189 | +int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, |
---|
| 190 | + unsigned long addr) |
---|
| 191 | +{ |
---|
| 192 | + u32 insn[FTRACE_PATCHABLE_FUNCTION_SIZE]; |
---|
| 193 | + int i; |
---|
| 194 | + |
---|
| 195 | + for (i = 0; i < ARRAY_SIZE(insn); i++) |
---|
| 196 | + insn[i] = INSN_NOP; |
---|
| 197 | + |
---|
| 198 | + __patch_text((void *)rec->ip, INSN_NOP); |
---|
| 199 | + __patch_text_multiple((void *)rec->ip + 4 - sizeof(insn), |
---|
| 200 | + insn, sizeof(insn)-4); |
---|
| 201 | + return 0; |
---|
| 202 | +} |
---|
| 203 | +#endif |
---|
| 204 | + |
---|
| 205 | +#ifdef CONFIG_KPROBES_ON_FTRACE |
---|
| 206 | +void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, |
---|
| 207 | + struct ftrace_ops *ops, struct pt_regs *regs) |
---|
| 208 | +{ |
---|
| 209 | + struct kprobe_ctlblk *kcb; |
---|
| 210 | + struct kprobe *p = get_kprobe((kprobe_opcode_t *)ip); |
---|
| 211 | + |
---|
| 212 | + if (unlikely(!p) || kprobe_disabled(p)) |
---|
| 213 | + return; |
---|
| 214 | + |
---|
| 215 | + if (kprobe_running()) { |
---|
| 216 | + kprobes_inc_nmissed_count(p); |
---|
| 217 | + return; |
---|
| 218 | + } |
---|
| 219 | + |
---|
| 220 | + __this_cpu_write(current_kprobe, p); |
---|
| 221 | + |
---|
| 222 | + kcb = get_kprobe_ctlblk(); |
---|
| 223 | + kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
---|
| 224 | + |
---|
| 225 | + regs->iaoq[0] = ip; |
---|
| 226 | + regs->iaoq[1] = ip + 4; |
---|
| 227 | + |
---|
| 228 | + if (!p->pre_handler || !p->pre_handler(p, regs)) { |
---|
| 229 | + regs->iaoq[0] = ip + 4; |
---|
| 230 | + regs->iaoq[1] = ip + 8; |
---|
| 231 | + |
---|
| 232 | + if (unlikely(p->post_handler)) { |
---|
| 233 | + kcb->kprobe_status = KPROBE_HIT_SSDONE; |
---|
| 234 | + p->post_handler(p, regs, 0); |
---|
| 235 | + } |
---|
| 236 | + } |
---|
| 237 | + __this_cpu_write(current_kprobe, NULL); |
---|
| 238 | +} |
---|
| 239 | +NOKPROBE_SYMBOL(kprobe_ftrace_handler); |
---|
| 240 | + |
---|
| 241 | +int arch_prepare_kprobe_ftrace(struct kprobe *p) |
---|
| 242 | +{ |
---|
| 243 | + p->ainsn.insn = NULL; |
---|
| 244 | + return 0; |
---|
| 245 | +} |
---|
| 246 | +#endif |
---|