forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/arch/x86/kernel/kprobes/core.c
....@@ -1,19 +1,6 @@
1
+// SPDX-License-Identifier: GPL-2.0-or-later
12 /*
23  *  Kernel Probes (KProbes)
3
- *
4
- * This program is free software; you can redistribute it and/or modify
5
- * it under the terms of the GNU General Public License as published by
6
- * the Free Software Foundation; either version 2 of the License, or
7
- * (at your option) any later version.
8
- *
9
- * This program is distributed in the hope that it will be useful,
10
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
- * GNU General Public License for more details.
13
- *
14
- * You should have received a copy of the GNU General Public License
15
- * along with this program; if not, write to the Free Software
16
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
174  *
185  * Copyright (C) IBM Corporation, 2002, 2004
196  *
....@@ -46,18 +33,20 @@
4633 #include <linux/hardirq.h>
4734 #include <linux/preempt.h>
4835 #include <linux/sched/debug.h>
36
+#include <linux/perf_event.h>
4937 #include <linux/extable.h>
5038 #include <linux/kdebug.h>
5139 #include <linux/kallsyms.h>
5240 #include <linux/ftrace.h>
53
-#include <linux/frame.h>
5441 #include <linux/kasan.h>
5542 #include <linux/moduleloader.h>
43
+#include <linux/objtool.h>
44
+#include <linux/vmalloc.h>
45
+#include <linux/pgtable.h>
5646 
5747 #include <asm/text-patching.h>
5848 #include <asm/cacheflush.h>
5949 #include <asm/desc.h>
60
-#include <asm/pgtable.h>
6150 #include <linux/uaccess.h>
6251 #include <asm/alternative.h>
6352 #include <asm/insn.h>
....@@ -69,7 +58,7 @@
6958 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
7059 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
7160 
72
-#define stack_addr(regs) ((unsigned long *)kernel_stack_pointer(regs))
61
+#define stack_addr(regs) ((unsigned long *)regs->sp)
7362 
7463 #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
7564 	(((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) |   \
....@@ -132,14 +121,14 @@
132121 /* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
133122 void synthesize_reljump(void *dest, void *from, void *to)
134123 {
135
-	__synthesize_relative_insn(dest, from, to, RELATIVEJUMP_OPCODE);
124
+	__synthesize_relative_insn(dest, from, to, JMP32_INSN_OPCODE);
136125 }
137126 NOKPROBE_SYMBOL(synthesize_reljump);
138127 
139128 /* Insert a call instruction at address 'from', which calls address 'to'.*/
140129 void synthesize_relcall(void *dest, void *from, void *to)
141130 {
142
-	__synthesize_relative_insn(dest, from, to, RELATIVECALL_OPCODE);
131
+	__synthesize_relative_insn(dest, from, to, CALL_INSN_OPCODE);
143132 }
144133 NOKPROBE_SYMBOL(synthesize_relcall);
145134 
....@@ -262,7 +251,7 @@
262251 	 * Fortunately, we know that the original code is the ideal 5-byte
263252 	 * long NOP.
264253 	 */
265
-	if (probe_kernel_read(buf, (void *)addr,
254
+	if (copy_from_kernel_nofault(buf, (void *)addr,
266255 		MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
267256 		return 0UL;
268257 
....@@ -321,7 +310,7 @@
321310 		 * Another debugging subsystem might insert this breakpoint.
322311 		 * In that case, we can't recover it.
323312 		 */
324
-		if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION)
313
+		if (insn.opcode.bytes[0] == INT3_INSN_OPCODE)
325314 			return 0;
326315 		addr += insn.length;
327316 	}
....@@ -365,14 +354,19 @@
365354 		return 0;
366355 
367356 	/* This can access kernel text if given address is not recovered */
368
-	if (probe_kernel_read(dest, (void *)recovered_insn, MAX_INSN_SIZE))
357
+	if (copy_from_kernel_nofault(dest, (void *)recovered_insn,
358
+			MAX_INSN_SIZE))
369359 		return 0;
370360 
371361 	kernel_insn_init(insn, dest, MAX_INSN_SIZE);
372362 	insn_get_length(insn);
373363 
364
+	/* We can not probe force emulate prefixed instruction */
365
+	if (insn_has_emulate_prefix(insn))
366
+		return 0;
367
+
374368 	/* Another subsystem puts a breakpoint, failed to recover */
375
-	if (insn->opcode.bytes[0] == BREAKPOINT_INSTRUCTION)
369
+	if (insn->opcode.bytes[0] == INT3_INSN_OPCODE)
376370 		return 0;
377371 
378372 	/* We should not singlestep on the exception masking instructions */
....@@ -416,14 +410,14 @@
416410 	int len = insn->length;
417411 
418412 	if (can_boost(insn, p->addr) &&
419
-	    MAX_INSN_SIZE - len >= RELATIVEJUMP_SIZE) {
413
+	    MAX_INSN_SIZE - len >= JMP32_INSN_SIZE) {
420414 		/*
421415 		 * These instructions can be executed directly if it
422416 		 * jumps back to correct address.
423417 		 */
424418 		synthesize_reljump(buf + len, p->ainsn.insn + len,
425419 				   p->addr + insn->length);
426
-		len += RELATIVEJUMP_SIZE;
420
+		len += JMP32_INSN_SIZE;
427421 		p->ainsn.boostable = true;
428422 	} else {
429423 		p->ainsn.boostable = false;
....@@ -441,6 +435,7 @@
441435 	if (!page)
442436 		return NULL;
443437 
438
+	set_vm_flush_reset_perms(page);
444439 	/*
445440 	 * First make the page read-only, and only then make it executable to
446441 	 * prevent it from being W+X in between.
....@@ -459,12 +454,6 @@
459454 /* Recover page to RW mode before releasing it */
460455 void free_insn_page(void *page)
461456 {
462
-	/*
463
-	 * First make the page non-executable, and only then make it writable to
464
-	 * prevent it from being W+X in between.
465
-	 */
466
-	set_memory_nx((unsigned long)page, 1);
467
-	set_memory_rw((unsigned long)page, 1);
468457 	module_memfree(page);
469458 }
470459 
....@@ -490,6 +479,9 @@
490479 
491480 	/* Also, displacement change doesn't affect the first byte */
492481 	p->opcode = buf[0];
482
+
483
+	p->ainsn.tp_len = len;
484
+	perf_event_text_poke(p->ainsn.insn, NULL, 0, buf, len);
493485 
494486 	/* OK, write back the instruction(s) into ROX insn buffer */
495487 	text_poke(p->ainsn.insn, buf, len);
....@@ -522,17 +514,28 @@
522514 
523515 void arch_arm_kprobe(struct kprobe *p)
524516 {
525
-	text_poke(p->addr, ((unsigned char []){BREAKPOINT_INSTRUCTION}), 1);
517
+	u8 int3 = INT3_INSN_OPCODE;
518
+
519
+	text_poke(p->addr, &int3, 1);
520
+	text_poke_sync();
521
+	perf_event_text_poke(p->addr, &p->opcode, 1, &int3, 1);
526522 }
527523 
528524 void arch_disarm_kprobe(struct kprobe *p)
529525 {
526
+	u8 int3 = INT3_INSN_OPCODE;
527
+
528
+	perf_event_text_poke(p->addr, &int3, 1, &p->opcode, 1);
530529 	text_poke(p->addr, &p->opcode, 1);
530
+	text_poke_sync();
531531 }
532532 
533533 void arch_remove_kprobe(struct kprobe *p)
534534 {
535535 	if (p->ainsn.insn) {
536
+		/* Record the perf event before freeing the slot */
537
+		perf_event_text_poke(p->ainsn.insn, p->ainsn.insn,
538
+				     p->ainsn.tp_len, NULL, 0);
536539 		free_insn_slot(p->ainsn.insn, p->ainsn.boostable);
537540 		p->ainsn.insn = NULL;
538541 	}
....@@ -605,7 +608,7 @@
605608 	if (setup_detour_execution(p, regs, reenter))
606609 		return;
607610 
608
-#if !defined(CONFIG_PREEMPT)
611
+#if !defined(CONFIG_PREEMPTION)
609612 	if (p->ainsn.boostable && !p->post_handler) {
610613 		/* Boost up -- we can execute copied instructions directly */
611614 		if (!reenter)
....@@ -630,7 +633,7 @@
630633 	regs->flags |= X86_EFLAGS_TF;
631634 	regs->flags &= ~X86_EFLAGS_IF;
632635 	/* single step inline if the instruction is an int3 */
633
-	if (p->opcode == BREAKPOINT_INSTRUCTION)
636
+	if (p->opcode == INT3_INSN_OPCODE)
634637 		regs->ip = (unsigned long)p->addr;
635638 	else
636639 		regs->ip = (unsigned long)p->ainsn.insn;
....@@ -716,7 +719,7 @@
716719 				reset_current_kprobe();
717720 			return 1;
718721 		}
719
-	} else if (*addr != BREAKPOINT_INSTRUCTION) {
722
+	} else if (*addr != INT3_INSN_OPCODE) {
720723 		/*
721724 		 * The breakpoint instruction was removed right
722725 		 * after we hit it.  Another cpu has removed
....@@ -739,160 +742,53 @@
739742  * calls trampoline_handler() runs, which calls the kretprobe's handler.
740743  */
741744 asm(
745
+	".text\n"
742746 	".global kretprobe_trampoline\n"
743747 	".type kretprobe_trampoline, @function\n"
744748 	"kretprobe_trampoline:\n"
745
-#ifdef CONFIG_X86_64
746749 	/* We don't bother saving the ss register */
750
+#ifdef CONFIG_X86_64
747751 	"	pushq %rsp\n"
748752 	"	pushfq\n"
749753 	SAVE_REGS_STRING
750754 	"	movq %rsp, %rdi\n"
751755 	"	call trampoline_handler\n"
752756 	/* Replace saved sp with true return address. */
753
-	"	movq %rax, 152(%rsp)\n"
757
+	"	movq %rax, 19*8(%rsp)\n"
754758 	RESTORE_REGS_STRING
755759 	"	popfq\n"
756760 #else
757
-	"	pushf\n"
761
+	"	pushl %esp\n"
762
+	"	pushfl\n"
758763 	SAVE_REGS_STRING
759764 	"	movl %esp, %eax\n"
760765 	"	call trampoline_handler\n"
761
-	/* Move flags to cs */
762
-	"	movl 56(%esp), %edx\n"
763
-	"	movl %edx, 52(%esp)\n"
764
-	/* Replace saved flags with true return address. */
765
-	"	movl %eax, 56(%esp)\n"
766
+	/* Replace saved sp with true return address. */
767
+	"	movl %eax, 15*4(%esp)\n"
766768 	RESTORE_REGS_STRING
767
-	"	popf\n"
769
+	"	popfl\n"
768770 #endif
769
-	"	ret\n"
771
+	ASM_RET
770772 	".size kretprobe_trampoline, .-kretprobe_trampoline\n"
771773 );
772774 NOKPROBE_SYMBOL(kretprobe_trampoline);
773775 STACK_FRAME_NON_STANDARD(kretprobe_trampoline);
774776 
777
+
775778 /*
776779  * Called from kretprobe_trampoline
777780  */
778
-__visible __used void *trampoline_handler(struct pt_regs *regs)
781
+__used __visible void *trampoline_handler(struct pt_regs *regs)
779782 {
780
-	struct kretprobe_instance *ri = NULL;
781
-	struct hlist_head *head, empty_rp;
782
-	struct hlist_node *tmp;
783
-	unsigned long flags, orig_ret_address = 0;
784
-	unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
785
-	kprobe_opcode_t *correct_ret_addr = NULL;
786
-	void *frame_pointer;
787
-	bool skipped = false;
788
-
789
-	/*
790
-	 * Set a dummy kprobe for avoiding kretprobe recursion.
791
-	 * Since kretprobe never run in kprobe handler, kprobe must not
792
-	 * be running at this point.
793
-	 */
794
-	kprobe_busy_begin();
795
-
796
-	INIT_HLIST_HEAD(&empty_rp);
797
-	kretprobe_hash_lock(current, &head, &flags);
798783 	/* fixup registers */
799
-#ifdef CONFIG_X86_64
800784 	regs->cs = __KERNEL_CS;
801
-	/* On x86-64, we use pt_regs->sp for return address holder. */
802
-	frame_pointer = &regs->sp;
803
-#else
804
-	regs->cs = __KERNEL_CS | get_kernel_rpl();
785
+#ifdef CONFIG_X86_32
805786 	regs->gs = 0;
806
-	/* On x86-32, we use pt_regs->flags for return address holder. */
807
-	frame_pointer = &regs->flags;
808787 #endif
809
-	regs->ip = trampoline_address;
788
+	regs->ip = (unsigned long)&kretprobe_trampoline;
810789 	regs->orig_ax = ~0UL;
811790 
812
-	/*
813
-	 * It is possible to have multiple instances associated with a given
814
-	 * task either because multiple functions in the call path have
815
-	 * return probes installed on them, and/or more than one
816
-	 * return probe was registered for a target function.
817
-	 *
818
-	 * We can handle this because:
819
-	 *     - instances are always pushed into the head of the list
820
-	 *     - when multiple return probes are registered for the same
821
-	 *	 function, the (chronologically) first instance's ret_addr
822
-	 *	 will be the real return address, and all the rest will
823
-	 *	 point to kretprobe_trampoline.
824
-	 */
825
-	hlist_for_each_entry(ri, head, hlist) {
826
-		if (ri->task != current)
827
-			/* another task is sharing our hash bucket */
828
-			continue;
829
-		/*
830
-		 * Return probes must be pushed on this hash list correct
831
-		 * order (same as return order) so that it can be poped
832
-		 * correctly. However, if we find it is pushed it incorrect
833
-		 * order, this means we find a function which should not be
834
-		 * probed, because the wrong order entry is pushed on the
835
-		 * path of processing other kretprobe itself.
836
-		 */
837
-		if (ri->fp != frame_pointer) {
838
-			if (!skipped)
839
-				pr_warn("kretprobe is stacked incorrectly. Trying to fixup.\n");
840
-			skipped = true;
841
-			continue;
842
-		}
843
-
844
-		orig_ret_address = (unsigned long)ri->ret_addr;
845
-		if (skipped)
846
-			pr_warn("%ps must be blacklisted because of incorrect kretprobe order\n",
847
-				ri->rp->kp.addr);
848
-
849
-		if (orig_ret_address != trampoline_address)
850
-			/*
851
-			 * This is the real return address. Any other
852
-			 * instances associated with this task are for
853
-			 * other calls deeper on the call stack
854
-			 */
855
-			break;
856
-	}
857
-
858
-	kretprobe_assert(ri, orig_ret_address, trampoline_address);
859
-
860
-	correct_ret_addr = ri->ret_addr;
861
-	hlist_for_each_entry_safe(ri, tmp, head, hlist) {
862
-		if (ri->task != current)
863
-			/* another task is sharing our hash bucket */
864
-			continue;
865
-		if (ri->fp != frame_pointer)
866
-			continue;
867
-
868
-		orig_ret_address = (unsigned long)ri->ret_addr;
869
-		if (ri->rp && ri->rp->handler) {
870
-			__this_cpu_write(current_kprobe, &ri->rp->kp);
871
-			ri->ret_addr = correct_ret_addr;
872
-			ri->rp->handler(ri, regs);
873
-			__this_cpu_write(current_kprobe, &kprobe_busy);
874
-		}
875
-
876
-		recycle_rp_inst(ri, &empty_rp);
877
-
878
-		if (orig_ret_address != trampoline_address)
879
-			/*
880
-			 * This is the real return address. Any other
881
-			 * instances associated with this task are for
882
-			 * other calls deeper on the call stack
883
-			 */
884
-			break;
885
-	}
886
-
887
-	kretprobe_hash_unlock(current, &flags);
888
-
889
-	kprobe_busy_end();
890
-
891
-	hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
892
-		hlist_del(&ri->hlist);
893
-		kfree(ri);
894
-	}
895
-	return (void *)orig_ret_address;
791
+	return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, &regs->sp);
896792 }
897793 NOKPROBE_SYMBOL(trampoline_handler);
898794 
....@@ -1082,65 +978,11 @@
1082978 		 */
1083979 		if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
1084980 			return 1;
1085
-
1086
-		/*
1087
-		 * In case the user-specified fault handler returned
1088
-		 * zero, try to fix up.
1089
-		 */
1090
-		if (fixup_exception(regs, trapnr))
1091
-			return 1;
1092
-
1093
-		/*
1094
-		 * fixup routine could not handle it,
1095
-		 * Let do_page_fault() fix it.
1096
-		 */
1097981 	}
1098982 
1099983 	return 0;
1100984 }
1101985 NOKPROBE_SYMBOL(kprobe_fault_handler);
1102
-
1103
-/*
1104
- * Wrapper routine for handling exceptions.
1105
- */
1106
-int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val,
1107
-			     void *data)
1108
-{
1109
-	struct die_args *args = data;
1110
-	int ret = NOTIFY_DONE;
1111
-
1112
-	if (args->regs && user_mode(args->regs))
1113
-		return ret;
1114
-
1115
-	if (val == DIE_GPF) {
1116
-		/*
1117
-		 * To be potentially processing a kprobe fault and to
1118
-		 * trust the result from kprobe_running(), we have
1119
-		 * be non-preemptible.
1120
-		 */
1121
-		if (!preemptible() && kprobe_running() &&
1122
-		    kprobe_fault_handler(args->regs, args->trapnr))
1123
-			ret = NOTIFY_STOP;
1124
-	}
1125
-	return ret;
1126
-}
1127
-NOKPROBE_SYMBOL(kprobe_exceptions_notify);
1128
-
1129
-bool arch_within_kprobe_blacklist(unsigned long addr)
1130
-{
1131
-	bool is_in_entry_trampoline_section = false;
1132
-
1133
-#ifdef CONFIG_X86_64
1134
-	is_in_entry_trampoline_section =
1135
-		(addr >= (unsigned long)__entry_trampoline_start &&
1136
-		 addr < (unsigned long)__entry_trampoline_end);
1137
-#endif
1138
-	return  (addr >= (unsigned long)__kprobes_text_start &&
1139
-		 addr < (unsigned long)__kprobes_text_end) ||
1140
-		(addr >= (unsigned long)__entry_text_start &&
1141
-		 addr < (unsigned long)__entry_text_end) ||
1142
-		is_in_entry_trampoline_section;
1143
-}
1144986 
1145987 int __init arch_populate_kprobe_blacklist(void)
1146988 {