.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-or-later |
---|
1 | 2 | /* |
---|
2 | 3 | * Kernel Probes (KProbes) |
---|
3 | | - * |
---|
4 | | - * This program is free software; you can redistribute it and/or modify |
---|
5 | | - * it under the terms of the GNU General Public License as published by |
---|
6 | | - * the Free Software Foundation; either version 2 of the License, or |
---|
7 | | - * (at your option) any later version. |
---|
8 | | - * |
---|
9 | | - * This program is distributed in the hope that it will be useful, |
---|
10 | | - * but WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
11 | | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
---|
12 | | - * GNU General Public License for more details. |
---|
13 | | - * |
---|
14 | | - * You should have received a copy of the GNU General Public License |
---|
15 | | - * along with this program; if not, write to the Free Software |
---|
16 | | - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
---|
17 | 4 | * |
---|
18 | 5 | * Copyright (C) IBM Corporation, 2002, 2004 |
---|
19 | 6 | * |
---|
.. | .. |
---|
36 | 23 | #include <asm/cacheflush.h> |
---|
37 | 24 | #include <asm/sstep.h> |
---|
38 | 25 | #include <asm/sections.h> |
---|
| 26 | +#include <asm/inst.h> |
---|
39 | 27 | #include <linux/uaccess.h> |
---|
40 | 28 | |
---|
41 | 29 | DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; |
---|
.. | .. |
---|
118 | 106 | int arch_prepare_kprobe(struct kprobe *p) |
---|
119 | 107 | { |
---|
120 | 108 | int ret = 0; |
---|
121 | | - kprobe_opcode_t insn = *p->addr; |
---|
| 109 | + struct kprobe *prev; |
---|
| 110 | + struct ppc_inst insn = ppc_inst_read((struct ppc_inst *)p->addr); |
---|
122 | 111 | |
---|
123 | 112 | if ((unsigned long)p->addr & 0x03) { |
---|
124 | 113 | printk("Attempt to register kprobe at an unaligned address\n"); |
---|
125 | 114 | ret = -EINVAL; |
---|
126 | 115 | } else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) { |
---|
127 | 116 | printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n"); |
---|
| 117 | + ret = -EINVAL; |
---|
| 118 | + } else if ((unsigned long)p->addr & ~PAGE_MASK && |
---|
| 119 | + ppc_inst_prefixed(ppc_inst_read((struct ppc_inst *)(p->addr - 1)))) { |
---|
| 120 | + printk("Cannot register a kprobe on the second word of prefixed instruction\n"); |
---|
| 121 | + ret = -EINVAL; |
---|
| 122 | + } |
---|
| 123 | + preempt_disable(); |
---|
| 124 | + prev = get_kprobe(p->addr - 1); |
---|
| 125 | + preempt_enable_no_resched(); |
---|
| 126 | + if (prev && |
---|
| 127 | + ppc_inst_prefixed(ppc_inst_read((struct ppc_inst *)prev->ainsn.insn))) { |
---|
| 128 | + printk("Cannot register a kprobe on the second word of prefixed instruction\n"); |
---|
128 | 129 | ret = -EINVAL; |
---|
129 | 130 | } |
---|
130 | 131 | |
---|
.. | .. |
---|
137 | 138 | } |
---|
138 | 139 | |
---|
139 | 140 | if (!ret) { |
---|
140 | | - memcpy(p->ainsn.insn, p->addr, |
---|
141 | | - MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); |
---|
142 | | - p->opcode = *p->addr; |
---|
143 | | - flush_icache_range((unsigned long)p->ainsn.insn, |
---|
144 | | - (unsigned long)p->ainsn.insn + sizeof(kprobe_opcode_t)); |
---|
| 141 | + patch_instruction((struct ppc_inst *)p->ainsn.insn, insn); |
---|
| 142 | + p->opcode = ppc_inst_val(insn); |
---|
145 | 143 | } |
---|
146 | 144 | |
---|
147 | 145 | p->ainsn.boostable = 0; |
---|
.. | .. |
---|
151 | 149 | |
---|
152 | 150 | void arch_arm_kprobe(struct kprobe *p) |
---|
153 | 151 | { |
---|
154 | | - patch_instruction(p->addr, BREAKPOINT_INSTRUCTION); |
---|
| 152 | + patch_instruction((struct ppc_inst *)p->addr, ppc_inst(BREAKPOINT_INSTRUCTION)); |
---|
155 | 153 | } |
---|
156 | 154 | NOKPROBE_SYMBOL(arch_arm_kprobe); |
---|
157 | 155 | |
---|
158 | 156 | void arch_disarm_kprobe(struct kprobe *p) |
---|
159 | 157 | { |
---|
160 | | - patch_instruction(p->addr, p->opcode); |
---|
| 158 | + patch_instruction((struct ppc_inst *)p->addr, ppc_inst(p->opcode)); |
---|
161 | 159 | } |
---|
162 | 160 | NOKPROBE_SYMBOL(arch_disarm_kprobe); |
---|
163 | 161 | |
---|
.. | .. |
---|
220 | 218 | void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) |
---|
221 | 219 | { |
---|
222 | 220 | ri->ret_addr = (kprobe_opcode_t *)regs->link; |
---|
| 221 | + ri->fp = NULL; |
---|
223 | 222 | |
---|
224 | 223 | /* Replace the return addr with trampoline addr */ |
---|
225 | 224 | regs->link = (unsigned long)kretprobe_trampoline; |
---|
.. | .. |
---|
229 | 228 | static int try_to_emulate(struct kprobe *p, struct pt_regs *regs) |
---|
230 | 229 | { |
---|
231 | 230 | int ret; |
---|
232 | | - unsigned int insn = *p->ainsn.insn; |
---|
| 231 | + struct ppc_inst insn = ppc_inst_read((struct ppc_inst *)p->ainsn.insn); |
---|
233 | 232 | |
---|
234 | 233 | /* regs->nip is also adjusted if emulate_step returns 1 */ |
---|
235 | 234 | ret = emulate_step(regs, insn); |
---|
.. | .. |
---|
246 | 245 | * So, we should never get here... but, its still |
---|
247 | 246 | * good to catch them, just in case... |
---|
248 | 247 | */ |
---|
249 | | - printk("Can't step on instruction %x\n", insn); |
---|
| 248 | + printk("Can't step on instruction %s\n", ppc_inst_as_str(insn)); |
---|
250 | 249 | BUG(); |
---|
251 | 250 | } else { |
---|
252 | 251 | /* |
---|
.. | .. |
---|
288 | 287 | preempt_disable(); |
---|
289 | 288 | kcb = get_kprobe_ctlblk(); |
---|
290 | 289 | |
---|
291 | | - /* Check we're not actually recursing */ |
---|
292 | | - if (kprobe_running()) { |
---|
293 | | - p = get_kprobe(addr); |
---|
294 | | - if (p) { |
---|
295 | | - kprobe_opcode_t insn = *p->ainsn.insn; |
---|
296 | | - if (kcb->kprobe_status == KPROBE_HIT_SS && |
---|
297 | | - is_trap(insn)) { |
---|
298 | | - /* Turn off 'trace' bits */ |
---|
299 | | - regs->msr &= ~MSR_SINGLESTEP; |
---|
300 | | - regs->msr |= kcb->kprobe_saved_msr; |
---|
301 | | - goto no_kprobe; |
---|
302 | | - } |
---|
303 | | - /* We have reentered the kprobe_handler(), since |
---|
304 | | - * another probe was hit while within the handler. |
---|
305 | | - * We here save the original kprobes variables and |
---|
306 | | - * just single step on the instruction of the new probe |
---|
307 | | - * without calling any user handlers. |
---|
308 | | - */ |
---|
309 | | - save_previous_kprobe(kcb); |
---|
310 | | - set_current_kprobe(p, regs, kcb); |
---|
311 | | - kprobes_inc_nmissed_count(p); |
---|
312 | | - kcb->kprobe_status = KPROBE_REENTER; |
---|
313 | | - if (p->ainsn.boostable >= 0) { |
---|
314 | | - ret = try_to_emulate(p, regs); |
---|
315 | | - |
---|
316 | | - if (ret > 0) { |
---|
317 | | - restore_previous_kprobe(kcb); |
---|
318 | | - preempt_enable_no_resched(); |
---|
319 | | - return 1; |
---|
320 | | - } |
---|
321 | | - } |
---|
322 | | - prepare_singlestep(p, regs); |
---|
323 | | - return 1; |
---|
324 | | - } else if (*addr != BREAKPOINT_INSTRUCTION) { |
---|
325 | | - /* If trap variant, then it belongs not to us */ |
---|
326 | | - kprobe_opcode_t cur_insn = *addr; |
---|
327 | | - |
---|
328 | | - if (is_trap(cur_insn)) |
---|
329 | | - goto no_kprobe; |
---|
330 | | - /* The breakpoint instruction was removed by |
---|
331 | | - * another cpu right after we hit, no further |
---|
332 | | - * handling of this interrupt is appropriate |
---|
333 | | - */ |
---|
334 | | - ret = 1; |
---|
335 | | - } |
---|
336 | | - goto no_kprobe; |
---|
337 | | - } |
---|
338 | | - |
---|
339 | 290 | p = get_kprobe(addr); |
---|
340 | 291 | if (!p) { |
---|
341 | | - if (*addr != BREAKPOINT_INSTRUCTION) { |
---|
| 292 | + unsigned int instr; |
---|
| 293 | + |
---|
| 294 | + if (get_kernel_nofault(instr, addr)) |
---|
| 295 | + goto no_kprobe; |
---|
| 296 | + |
---|
| 297 | + if (instr != BREAKPOINT_INSTRUCTION) { |
---|
342 | 298 | /* |
---|
343 | 299 | * PowerPC has multiple variants of the "trap" |
---|
344 | 300 | * instruction. If the current instruction is a |
---|
345 | 301 | * trap variant, it could belong to someone else |
---|
346 | 302 | */ |
---|
347 | | - kprobe_opcode_t cur_insn = *addr; |
---|
348 | | - if (is_trap(cur_insn)) |
---|
| 303 | + if (is_trap(instr)) |
---|
349 | 304 | goto no_kprobe; |
---|
350 | 305 | /* |
---|
351 | 306 | * The breakpoint instruction was removed right |
---|
.. | .. |
---|
358 | 313 | } |
---|
359 | 314 | /* Not one of ours: let kernel handle it */ |
---|
360 | 315 | goto no_kprobe; |
---|
| 316 | + } |
---|
| 317 | + |
---|
| 318 | + /* Check we're not actually recursing */ |
---|
| 319 | + if (kprobe_running()) { |
---|
| 320 | + kprobe_opcode_t insn = *p->ainsn.insn; |
---|
| 321 | + if (kcb->kprobe_status == KPROBE_HIT_SS && is_trap(insn)) { |
---|
| 322 | + /* Turn off 'trace' bits */ |
---|
| 323 | + regs->msr &= ~MSR_SINGLESTEP; |
---|
| 324 | + regs->msr |= kcb->kprobe_saved_msr; |
---|
| 325 | + goto no_kprobe; |
---|
| 326 | + } |
---|
| 327 | + |
---|
| 328 | + /* |
---|
| 329 | + * We have reentered the kprobe_handler(), since another probe |
---|
| 330 | + * was hit while within the handler. We here save the original |
---|
| 331 | + * kprobes variables and just single step on the instruction of |
---|
| 332 | + * the new probe without calling any user handlers. |
---|
| 333 | + */ |
---|
| 334 | + save_previous_kprobe(kcb); |
---|
| 335 | + set_current_kprobe(p, regs, kcb); |
---|
| 336 | + kprobes_inc_nmissed_count(p); |
---|
| 337 | + kcb->kprobe_status = KPROBE_REENTER; |
---|
| 338 | + if (p->ainsn.boostable >= 0) { |
---|
| 339 | + ret = try_to_emulate(p, regs); |
---|
| 340 | + |
---|
| 341 | + if (ret > 0) { |
---|
| 342 | + restore_previous_kprobe(kcb); |
---|
| 343 | + preempt_enable_no_resched(); |
---|
| 344 | + return 1; |
---|
| 345 | + } |
---|
| 346 | + } |
---|
| 347 | + prepare_singlestep(p, regs); |
---|
| 348 | + return 1; |
---|
361 | 349 | } |
---|
362 | 350 | |
---|
363 | 351 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
---|
.. | .. |
---|
410 | 398 | */ |
---|
411 | 399 | static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) |
---|
412 | 400 | { |
---|
413 | | - struct kretprobe_instance *ri = NULL; |
---|
414 | | - struct hlist_head *head, empty_rp; |
---|
415 | | - struct hlist_node *tmp; |
---|
416 | | - unsigned long flags, orig_ret_address = 0; |
---|
417 | | - unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline; |
---|
| 401 | + unsigned long orig_ret_address; |
---|
418 | 402 | |
---|
419 | | - INIT_HLIST_HEAD(&empty_rp); |
---|
420 | | - kretprobe_hash_lock(current, &head, &flags); |
---|
421 | | - |
---|
422 | | - /* |
---|
423 | | - * It is possible to have multiple instances associated with a given |
---|
424 | | - * task either because an multiple functions in the call path |
---|
425 | | - * have a return probe installed on them, and/or more than one return |
---|
426 | | - * return probe was registered for a target function. |
---|
427 | | - * |
---|
428 | | - * We can handle this because: |
---|
429 | | - * - instances are always inserted at the head of the list |
---|
430 | | - * - when multiple return probes are registered for the same |
---|
431 | | - * function, the first instance's ret_addr will point to the |
---|
432 | | - * real return address, and all the rest will point to |
---|
433 | | - * kretprobe_trampoline |
---|
434 | | - */ |
---|
435 | | - hlist_for_each_entry_safe(ri, tmp, head, hlist) { |
---|
436 | | - if (ri->task != current) |
---|
437 | | - /* another task is sharing our hash bucket */ |
---|
438 | | - continue; |
---|
439 | | - |
---|
440 | | - if (ri->rp && ri->rp->handler) |
---|
441 | | - ri->rp->handler(ri, regs); |
---|
442 | | - |
---|
443 | | - orig_ret_address = (unsigned long)ri->ret_addr; |
---|
444 | | - recycle_rp_inst(ri, &empty_rp); |
---|
445 | | - |
---|
446 | | - if (orig_ret_address != trampoline_address) |
---|
447 | | - /* |
---|
448 | | - * This is the real return address. Any other |
---|
449 | | - * instances associated with this task are for |
---|
450 | | - * other calls deeper on the call stack |
---|
451 | | - */ |
---|
452 | | - break; |
---|
453 | | - } |
---|
454 | | - |
---|
455 | | - kretprobe_assert(ri, orig_ret_address, trampoline_address); |
---|
456 | | - |
---|
| 403 | + orig_ret_address = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL); |
---|
457 | 404 | /* |
---|
458 | 405 | * We get here through one of two paths: |
---|
459 | 406 | * 1. by taking a trap -> kprobe_handler() -> here |
---|
.. | .. |
---|
472 | 419 | regs->nip = orig_ret_address - 4; |
---|
473 | 420 | regs->link = orig_ret_address; |
---|
474 | 421 | |
---|
475 | | - kretprobe_hash_unlock(current, &flags); |
---|
476 | | - |
---|
477 | | - hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { |
---|
478 | | - hlist_del(&ri->hlist); |
---|
479 | | - kfree(ri); |
---|
480 | | - } |
---|
481 | | - |
---|
482 | 422 | return 0; |
---|
483 | 423 | } |
---|
484 | 424 | NOKPROBE_SYMBOL(trampoline_probe_handler); |
---|
.. | .. |
---|
493 | 433 | */ |
---|
494 | 434 | int kprobe_post_handler(struct pt_regs *regs) |
---|
495 | 435 | { |
---|
| 436 | + int len; |
---|
496 | 437 | struct kprobe *cur = kprobe_running(); |
---|
497 | 438 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
---|
498 | 439 | |
---|
499 | 440 | if (!cur || user_mode(regs)) |
---|
500 | 441 | return 0; |
---|
501 | 442 | |
---|
| 443 | + len = ppc_inst_len(ppc_inst_read((struct ppc_inst *)cur->ainsn.insn)); |
---|
502 | 444 | /* make sure we got here for instruction we have a kprobe on */ |
---|
503 | | - if (((unsigned long)cur->ainsn.insn + 4) != regs->nip) |
---|
| 445 | + if (((unsigned long)cur->ainsn.insn + len) != regs->nip) |
---|
504 | 446 | return 0; |
---|
505 | 447 | |
---|
506 | 448 | if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { |
---|
.. | .. |
---|
509 | 451 | } |
---|
510 | 452 | |
---|
511 | 453 | /* Adjust nip to after the single-stepped instruction */ |
---|
512 | | - regs->nip = (unsigned long)cur->addr + 4; |
---|
| 454 | + regs->nip = (unsigned long)cur->addr + len; |
---|
513 | 455 | regs->msr |= kcb->kprobe_saved_msr; |
---|
514 | 456 | |
---|
515 | 457 | /*Restore back the original saved kprobes variables and continue. */ |
---|