forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 e636c8d336489bf3eed5878299e6cc045bbad077 
 kernel/kernel/kprobes.c
....@@ -1,20 +1,7 @@
1
+// SPDX-License-Identifier: GPL-2.0-or-later
12 /*
23  *  Kernel Probes (KProbes)
34  *  kernel/kprobes.c
4
- *
5
- * This program is free software; you can redistribute it and/or modify
6
- * it under the terms of the GNU General Public License as published by
7
- * the Free Software Foundation; either version 2 of the License, or
8
- * (at your option) any later version.
9
- *
10
- * This program is distributed in the hope that it will be useful,
11
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
12
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13
- * GNU General Public License for more details.
14
- *
15
- * You should have received a copy of the GNU General Public License
16
- * along with this program; if not, write to the Free Software
17
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
185  *
196  * Copyright (C) IBM Corporation, 2002, 2004
207  *
....@@ -48,6 +35,8 @@
4835 #include <linux/ftrace.h>
4936 #include <linux/cpu.h>
5037 #include <linux/jump_label.h>
38
+#include <linux/perf_event.h>
39
+#include <linux/static_call.h>
5140 
5241 #include <asm/sections.h>
5342 #include <asm/cacheflush.h>
....@@ -59,6 +48,11 @@
5948 
6049 
6150 static int kprobes_initialized;
51
+/* kprobe_table can be accessed by
52
+ * - Normal hlist traversal and RCU add/del under kprobe_mutex is held.
53
+ * Or
54
+ * - RCU hlist traversal under disabling preempt (breakpoint handlers)
55
+ */
6256 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
6357 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
6458 
....@@ -131,6 +125,7 @@
131125 	.mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),
132126 	.alloc = alloc_insn_page,
133127 	.free = free_insn_page,
128
+	.sym = KPROBE_INSN_PAGE_SYM,
134129 	.pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
135130 	.insn_size = MAX_INSN_SIZE,
136131 	.nr_garbage = 0,
....@@ -196,6 +191,10 @@
196191 	kip->cache = c;
197192 	list_add_rcu(&kip->list, &c->pages);
198193 	slot = kip->insns;
194
+
195
+	/* Record the perf ksymbol register event after adding the page */
196
+	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, (unsigned long)kip->insns,
197
+			   PAGE_SIZE, false, c->sym);
199198 out:
200199 	mutex_unlock(&c->mutex);
201200 	return slot;
....@@ -214,6 +213,13 @@
214213 		 * next time somebody inserts a probe.
215214 		 */
216215 		if (!list_is_singular(&kip->list)) {
216
+			/*
217
+			 * Record perf ksymbol unregister event before removing
218
+			 * the page.
219
+			 */
220
+			perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
221
+					   (unsigned long)kip->insns, PAGE_SIZE, true,
222
+					   kip->cache->sym);
217223 			list_del_rcu(&kip->list);
218224 			synchronize_rcu();
219225 			kip->cache->free(kip->insns);
....@@ -229,7 +235,7 @@
229235 	struct kprobe_insn_page *kip, *next;
230236 
231237 	/* Ensure no-one is interrupted on the garbages */
232
-	synchronize_sched();
238
+	synchronize_rcu();
233239 
234240 	list_for_each_entry_safe(kip, next, &c->pages, list) {
235241 		int i;
....@@ -303,12 +309,34 @@
303309 	return ret;
304310 }
305311 
312
+int kprobe_cache_get_kallsym(struct kprobe_insn_cache *c, unsigned int *symnum,
313
+			     unsigned long *value, char *type, char *sym)
314
+{
315
+	struct kprobe_insn_page *kip;
316
+	int ret = -ERANGE;
317
+
318
+	rcu_read_lock();
319
+	list_for_each_entry_rcu(kip, &c->pages, list) {
320
+		if ((*symnum)--)
321
+			continue;
322
+		strlcpy(sym, c->sym, KSYM_NAME_LEN);
323
+		*type = 't';
324
+		*value = (unsigned long)kip->insns;
325
+		ret = 0;
326
+		break;
327
+	}
328
+	rcu_read_unlock();
329
+
330
+	return ret;
331
+}
332
+
306333 #ifdef CONFIG_OPTPROBES
307334 /* For optimized_kprobe buffer */
308335 struct kprobe_insn_cache kprobe_optinsn_slots = {
309336 	.mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),
310337 	.alloc = alloc_insn_page,
311338 	.free = free_insn_page,
339
+	.sym = KPROBE_OPTINSN_PAGE_SYM,
312340 	.pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
313341 	/* .insn_size is initialized later */
314342 	.nr_garbage = 0,
....@@ -339,7 +367,8 @@
339367 	struct kprobe *p;
340368 
341369 	head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
342
-	hlist_for_each_entry_rcu(p, head, hlist) {
370
+	hlist_for_each_entry_rcu(p, head, hlist,
371
+				 lockdep_is_held(&kprobe_mutex)) {
343372 		if (p->addr == addr)
344373 			return p;
345374 	}
....@@ -418,8 +447,8 @@
418447 	return 0;
419448 }
420449 
421
-/* Return true(!0) if the kprobe is disarmed. Note: p must be on hash list */
422
-static inline int kprobe_disarmed(struct kprobe *p)
450
+/* Return true if the kprobe is disarmed. Note: p must be on hash list */
451
+bool kprobe_disarmed(struct kprobe *p)
423452 {
424453 	struct optimized_kprobe *op;
425454 
....@@ -570,8 +599,6 @@
570599 	mutex_lock(&kprobe_mutex);
571600 	cpus_read_lock();
572601 	mutex_lock(&text_mutex);
573
-	/* Lock modules while optimizing kprobes */
574
-	mutex_lock(&module_mutex);
575602 
576603 	/*
577604 	 * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
....@@ -596,7 +623,6 @@
596623 	/* Step 4: Free cleaned kprobes after quiesence period */
597624 	do_free_cleaned_kprobes();
598625 
599
-	mutex_unlock(&module_mutex);
600626 	mutex_unlock(&text_mutex);
601627 	cpus_read_unlock();
602628 
....@@ -626,7 +652,7 @@
626652 	mutex_unlock(&kprobe_mutex);
627653 }
628654 
629
-static bool optprobe_queued_unopt(struct optimized_kprobe *op)
655
+bool optprobe_queued_unopt(struct optimized_kprobe *op)
630656 {
631657 	struct optimized_kprobe *_op;
632658 
....@@ -682,8 +708,6 @@
682708 	lockdep_assert_cpus_held();
683709 	arch_unoptimize_kprobe(op);
684710 	op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
685
-	if (kprobe_disabled(&op->kp))
686
-		arch_disarm_kprobe(&op->kp);
687711 }
688712 
689713 /* Unoptimize a kprobe if p is optimized */
....@@ -732,7 +756,6 @@
732756 {
733757 	struct optimized_kprobe *op;
734758 
735
-	BUG_ON(!kprobe_unused(ap));
736759 	/*
737760 	 * Unused kprobe MUST be on the way of delayed unoptimizing (means
738761 	 * there is still a relative jump) and disabled.
....@@ -848,7 +871,6 @@
848871 	cpus_read_unlock();
849872 }
850873 
851
-#ifdef CONFIG_SYSCTL
852874 static void optimize_all_kprobes(void)
853875 {
854876 	struct hlist_head *head;
....@@ -864,7 +886,7 @@
864886 	kprobes_allow_optimization = true;
865887 	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
866888 		head = &kprobe_table[i];
867
-		hlist_for_each_entry_rcu(p, head, hlist)
889
+		hlist_for_each_entry(p, head, hlist)
868890 			if (!kprobe_disabled(p))
869891 				optimize_kprobe(p);
870892 	}
....@@ -874,6 +896,7 @@
874896 	mutex_unlock(&kprobe_mutex);
875897 }
876898 
899
+#ifdef CONFIG_SYSCTL
877900 static void unoptimize_all_kprobes(void)
878901 {
879902 	struct hlist_head *head;
....@@ -891,7 +914,7 @@
891914 	kprobes_allow_optimization = false;
892915 	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
893916 		head = &kprobe_table[i];
894
-		hlist_for_each_entry_rcu(p, head, hlist) {
917
+		hlist_for_each_entry(p, head, hlist) {
895918 			if (!kprobe_disabled(p))
896919 				unoptimize_kprobe(p, false);
897920 		}
....@@ -907,7 +930,7 @@
907930 static DEFINE_MUTEX(kprobe_sysctl_mutex);
908931 int sysctl_kprobes_optimization;
909932 int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
910
-				      void __user *buffer, size_t *length,
933
+				      void *buffer, size_t *length,
911934 				      loff_t *ppos)
912935 {
913936 	int ret;
....@@ -998,8 +1021,15 @@
9981021 #ifdef CONFIG_KPROBES_ON_FTRACE
9991022 static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
10001023 	.func = kprobe_ftrace_handler,
1024
+	.flags = FTRACE_OPS_FL_SAVE_REGS,
1025
+};
1026
+
1027
+static struct ftrace_ops kprobe_ipmodify_ops __read_mostly = {
1028
+	.func = kprobe_ftrace_handler,
10011029 	.flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY,
10021030 };
1031
+
1032
+static int kprobe_ipmodify_enabled;
10031033 static int kprobe_ftrace_enabled;
10041034 
10051035 /* Must ensure p->addr is really on ftrace */
....@@ -1012,57 +1042,74 @@
10121042 }
10131043 
10141044 /* Caller must lock kprobe_mutex */
1015
-static int arm_kprobe_ftrace(struct kprobe *p)
1045
+static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
1046
+			       int *cnt)
10161047 {
10171048 	int ret = 0;
10181049 
1019
-	ret = ftrace_set_filter_ip(&kprobe_ftrace_ops,
1020
-				   (unsigned long)p->addr, 0, 0);
1050
+	ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0);
10211051 	if (ret) {
10221052 		pr_debug("Failed to arm kprobe-ftrace at %pS (%d)\n",
10231053 			 p->addr, ret);
10241054 		return ret;
10251055 	}
10261056 
1027
-	if (kprobe_ftrace_enabled == 0) {
1028
-		ret = register_ftrace_function(&kprobe_ftrace_ops);
1057
+	if (*cnt == 0) {
1058
+		ret = register_ftrace_function(ops);
10291059 		if (ret) {
10301060 			pr_debug("Failed to init kprobe-ftrace (%d)\n", ret);
10311061 			goto err_ftrace;
10321062 		}
10331063 	}
10341064 
1035
-	kprobe_ftrace_enabled++;
1065
+	(*cnt)++;
10361066 	return ret;
10371067 
10381068 err_ftrace:
10391069 	/*
1040
-	 * Note: Since kprobe_ftrace_ops has IPMODIFY set, and ftrace requires a
1041
-	 * non-empty filter_hash for IPMODIFY ops, we're safe from an accidental
1042
-	 * empty filter_hash which would undesirably trace all functions.
1070
+	 * At this point, sinec ops is not registered, we should be sefe from
1071
+	 * registering empty filter.
10431072 	 */
1044
-	ftrace_set_filter_ip(&kprobe_ftrace_ops, (unsigned long)p->addr, 1, 0);
1073
+	ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
10451074 	return ret;
10461075 }
10471076 
1077
+static int arm_kprobe_ftrace(struct kprobe *p)
1078
+{
1079
+	bool ipmodify = (p->post_handler != NULL);
1080
+
1081
+	return __arm_kprobe_ftrace(p,
1082
+		ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1083
+		ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1084
+}
1085
+
10481086 /* Caller must lock kprobe_mutex */
1049
-static int disarm_kprobe_ftrace(struct kprobe *p)
1087
+static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
1088
+				  int *cnt)
10501089 {
10511090 	int ret = 0;
10521091 
1053
-	if (kprobe_ftrace_enabled == 1) {
1054
-		ret = unregister_ftrace_function(&kprobe_ftrace_ops);
1092
+	if (*cnt == 1) {
1093
+		ret = unregister_ftrace_function(ops);
10551094 		if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (%d)\n", ret))
10561095 			return ret;
10571096 	}
10581097 
1059
-	kprobe_ftrace_enabled--;
1098
+	(*cnt)--;
10601099 
1061
-	ret = ftrace_set_filter_ip(&kprobe_ftrace_ops,
1062
-			   (unsigned long)p->addr, 1, 0);
1100
+	ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
10631101 	WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (%d)\n",
10641102 		  p->addr, ret);
10651103 	return ret;
1104
+}
1105
+
1106
+static int disarm_kprobe_ftrace(struct kprobe *p)
1107
+{
1108
+	bool ipmodify = (p->post_handler != NULL);
1109
+
1110
+	return __disarm_kprobe_ftrace(p,
1111
+		ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1112
+		ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
10661113 }
10671114 #else	/* !CONFIG_KPROBES_ON_FTRACE */
10681115 static inline int prepare_kprobe(struct kprobe *p)
....@@ -1177,8 +1224,7 @@
11771224 }
11781225 NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
11791226 
1180
-void recycle_rp_inst(struct kretprobe_instance *ri,
1181
-		     struct hlist_head *head)
1227
+static void recycle_rp_inst(struct kretprobe_instance *ri)
11821228 {
11831229 	struct kretprobe *rp = ri->rp;
11841230 
....@@ -1190,12 +1236,11 @@
11901236 		hlist_add_head(&ri->hlist, &rp->free_instances);
11911237 		raw_spin_unlock(&rp->lock);
11921238 	} else
1193
-		/* Unregistering */
1194
-		hlist_add_head(&ri->hlist, head);
1239
+		kfree_rcu(ri, rcu);
11951240 }
11961241 NOKPROBE_SYMBOL(recycle_rp_inst);
11971242 
1198
-void kretprobe_hash_lock(struct task_struct *tsk,
1243
+static void kretprobe_hash_lock(struct task_struct *tsk,
11991244 			 struct hlist_head **head, unsigned long *flags)
12001245 __acquires(hlist_lock)
12011246 {
....@@ -1204,7 +1249,13 @@
12041249 
12051250 	*head = &kretprobe_inst_table[hash];
12061251 	hlist_lock = kretprobe_table_lock_ptr(hash);
1207
-	raw_spin_lock_irqsave(hlist_lock, *flags);
1252
+	/*
1253
+	 * Nested is a workaround that will soon not be needed.
1254
+	 * There's other protections that make sure the same lock
1255
+	 * is not taken on the same CPU that lockdep is unaware of.
1256
+	 * Differentiate when it is taken in NMI context.
1257
+	 */
1258
+	raw_spin_lock_irqsave_nested(hlist_lock, *flags, !!in_nmi());
12081259 }
12091260 NOKPROBE_SYMBOL(kretprobe_hash_lock);
12101261 
....@@ -1213,11 +1264,17 @@
12131264 __acquires(hlist_lock)
12141265 {
12151266 	raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1216
-	raw_spin_lock_irqsave(hlist_lock, *flags);
1267
+	/*
1268
+	 * Nested is a workaround that will soon not be needed.
1269
+	 * There's other protections that make sure the same lock
1270
+	 * is not taken on the same CPU that lockdep is unaware of.
1271
+	 * Differentiate when it is taken in NMI context.
1272
+	 */
1273
+	raw_spin_lock_irqsave_nested(hlist_lock, *flags, !!in_nmi());
12171274 }
12181275 NOKPROBE_SYMBOL(kretprobe_table_lock);
12191276 
1220
-void kretprobe_hash_unlock(struct task_struct *tsk,
1277
+static void kretprobe_hash_unlock(struct task_struct *tsk,
12211278 			   unsigned long *flags)
12221279 __releases(hlist_lock)
12231280 {
....@@ -1238,7 +1295,7 @@
12381295 }
12391296 NOKPROBE_SYMBOL(kretprobe_table_unlock);
12401297 
1241
-struct kprobe kprobe_busy = {
1298
+static struct kprobe kprobe_busy = {
12421299 	.addr = (void *) get_kprobe,
12431300 };
12441301 
....@@ -1267,7 +1324,7 @@
12671324 void kprobe_flush_task(struct task_struct *tk)
12681325 {
12691326 	struct kretprobe_instance *ri;
1270
-	struct hlist_head *head, empty_rp;
1327
+	struct hlist_head *head;
12711328 	struct hlist_node *tmp;
12721329 	unsigned long hash, flags = 0;
12731330 
....@@ -1277,19 +1334,14 @@
12771334 
12781335 	kprobe_busy_begin();
12791336 
1280
-	INIT_HLIST_HEAD(&empty_rp);
12811337 	hash = hash_ptr(tk, KPROBE_HASH_BITS);
12821338 	head = &kretprobe_inst_table[hash];
12831339 	kretprobe_table_lock(hash, &flags);
12841340 	hlist_for_each_entry_safe(ri, tmp, head, hlist) {
12851341 		if (ri->task == tk)
1286
-			recycle_rp_inst(ri, &empty_rp);
1342
+			recycle_rp_inst(ri);
12871343 	}
12881344 	kretprobe_table_unlock(hash, &flags);
1289
-	hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
1290
-		hlist_del(&ri->hlist);
1291
-		kfree(ri);
1292
-	}
12931345 
12941346 	kprobe_busy_end();
12951347 }
....@@ -1313,7 +1365,8 @@
13131365 	struct hlist_node *next;
13141366 	struct hlist_head *head;
13151367 
1316
-	/* No race here */
1368
+	/* To avoid recursive kretprobe by NMI, set kprobe busy here */
1369
+	kprobe_busy_begin();
13171370 	for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
13181371 		kretprobe_table_lock(hash, &flags);
13191372 		head = &kretprobe_inst_table[hash];
....@@ -1323,6 +1376,8 @@
13231376 		}
13241377 		kretprobe_table_unlock(hash, &flags);
13251378 	}
1379
+	kprobe_busy_end();
1380
+
13261381 	free_rp_inst(rp);
13271382 }
13281383 NOKPROBE_SYMBOL(cleanup_rp_inst);
....@@ -1330,8 +1385,6 @@
13301385 /* Add the new probe to ap->list */
13311386 static int add_new_kprobe(struct kprobe *ap, struct kprobe *p)
13321387 {
1333
-	BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
1334
-
13351388 	if (p->post_handler)
13361389 		unoptimize_kprobe(ap, true);	/* Fall back to normal kprobe */
13371390 
....@@ -1440,7 +1493,7 @@
14401493 			if (ret) {
14411494 				ap->flags |= KPROBE_FLAG_DISABLED;
14421495 				list_del_rcu(&p->list);
1443
-				synchronize_sched();
1496
+				synchronize_rcu();
14441497 			}
14451498 		}
14461499 	}
....@@ -1454,7 +1507,7 @@
14541507 	       addr < (unsigned long)__kprobes_text_end;
14551508 }
14561509 
1457
-bool within_kprobe_blacklist(unsigned long addr)
1510
+static bool __within_kprobe_blacklist(unsigned long addr)
14581511 {
14591512 	struct kprobe_blacklist_entry *ent;
14601513 
....@@ -1468,7 +1521,26 @@
14681521 		if (addr >= ent->start_addr && addr < ent->end_addr)
14691522 			return true;
14701523 	}
1524
+	return false;
1525
+}
14711526 
1527
+bool within_kprobe_blacklist(unsigned long addr)
1528
+{
1529
+	char symname[KSYM_NAME_LEN], *p;
1530
+
1531
+	if (__within_kprobe_blacklist(addr))
1532
+		return true;
1533
+
1534
+	/* Check if the address is on a suffixed-symbol */
1535
+	if (!lookup_symbol_name(addr, symname)) {
1536
+		p = strchr(symname, '.');
1537
+		if (!p)
1538
+			return false;
1539
+		*p = '\0';
1540
+		addr = (unsigned long)kprobe_lookup_name(symname, 0);
1541
+		if (addr)
1542
+			return __within_kprobe_blacklist(addr);
1543
+	}
14721544 	return false;
14731545 }
14741546 
....@@ -1508,12 +1580,14 @@
15081580 {
15091581 	struct kprobe *ap, *list_p;
15101582 
1583
+	lockdep_assert_held(&kprobe_mutex);
1584
+
15111585 	ap = get_kprobe(p->addr);
15121586 	if (unlikely(!ap))
15131587 		return NULL;
15141588 
15151589 	if (p != ap) {
1516
-		list_for_each_entry_rcu(list_p, &ap->list, list)
1590
+		list_for_each_entry(list_p, &ap->list, list)
15171591 			if (list_p == p)
15181592 			/* kprobe p is a valid probe */
15191593 				goto valid;
....@@ -1554,6 +1628,17 @@
15541628 	return 0;
15551629 }
15561630 
1631
+static bool is_cfi_preamble_symbol(unsigned long addr)
1632
+{
1633
+	char symbuf[KSYM_NAME_LEN];
1634
+
1635
+	if (lookup_symbol_name(addr, symbuf))
1636
+		return false;
1637
+
1638
+	return str_has_prefix("__cfi_", symbuf) ||
1639
+		str_has_prefix("__pfx_", symbuf);
1640
+}
1641
+
15571642 static int check_kprobe_address_safe(struct kprobe *p,
15581643 				     struct module **probed_mod)
15591644 {
....@@ -1566,10 +1651,14 @@
15661651 	preempt_disable();
15671652 
15681653 	/* Ensure it is not in reserved area nor out of text */
1569
-	if (!kernel_text_address((unsigned long) p->addr) ||
1654
+	if (!(core_kernel_text((unsigned long) p->addr) ||
1655
+	    is_module_text_address((unsigned long) p->addr)) ||
1656
+	    in_gate_area_no_mm((unsigned long) p->addr) ||
15701657 	    within_kprobe_blacklist((unsigned long) p->addr) ||
15711658 	    jump_label_text_reserved(p->addr, p->addr) ||
1572
-	    find_bug((unsigned long)p->addr)) {
1659
+	    static_call_text_reserved(p->addr, p->addr) ||
1660
+	    find_bug((unsigned long)p->addr) ||
1661
+	    is_cfi_preamble_symbol((unsigned long)p->addr)) {
15731662 		ret = -EINVAL;
15741663 		goto out;
15751664 	}
....@@ -1656,7 +1745,7 @@
16561745 		ret = arm_kprobe(p);
16571746 		if (ret) {
16581747 			hlist_del_rcu(&p->hlist);
1659
-			synchronize_sched();
1748
+			synchronize_rcu();
16601749 			goto out;
16611750 		}
16621751 	}
....@@ -1678,7 +1767,9 @@
16781767 {
16791768 	struct kprobe *kp;
16801769 
1681
-	list_for_each_entry_rcu(kp, &ap->list, list)
1770
+	lockdep_assert_held(&kprobe_mutex);
1771
+
1772
+	list_for_each_entry(kp, &ap->list, list)
16821773 		if (!kprobe_disabled(kp))
16831774 			/*
16841775 			 * There is an active probe on the list.
....@@ -1708,11 +1799,12 @@
17081799 		/* Try to disarm and disable this/parent probe */
17091800 		if (p == orig_p || aggr_kprobe_disabled(orig_p)) {
17101801 			/*
1711
-			 * If kprobes_all_disarmed is set, orig_p
1712
-			 * should have already been disarmed, so
1713
-			 * skip unneed disarming process.
1802
+			 * Don't be lazy here.  Even if 'kprobes_all_disarmed'
1803
+			 * is false, 'orig_p' might not have been armed yet.
1804
+			 * Note arm_all_kprobes() __tries__ to arm all kprobes
1805
+			 * on the best effort basis.
17141806 			 */
1715
-			if (!kprobes_all_disarmed) {
1807
+			if (!kprobes_all_disarmed && !kprobe_disabled(orig_p)) {
17161808 				ret = disarm_kprobe(orig_p, true);
17171809 				if (ret) {
17181810 					p->flags &= ~KPROBE_FLAG_DISABLED;
....@@ -1757,11 +1849,17 @@
17571849 	else {
17581850 		/* If disabling probe has special handlers, update aggrprobe */
17591851 		if (p->post_handler && !kprobe_gone(p)) {
1760
-			list_for_each_entry_rcu(list_p, &ap->list, list) {
1852
+			list_for_each_entry(list_p, &ap->list, list) {
17611853 				if ((list_p != p) && (list_p->post_handler))
17621854 					goto noclean;
17631855 			}
1764
-			ap->post_handler = NULL;
1856
+			/*
1857
+			 * For the kprobe-on-ftrace case, we keep the
1858
+			 * post_handler setting to identify this aggrprobe
1859
+			 * armed with kprobe_ipmodify_ops.
1860
+			 */
1861
+			if (!kprobe_ftrace(ap))
1862
+				ap->post_handler = NULL;
17651863 		}
17661864 noclean:
17671865 		/*
....@@ -1779,7 +1877,6 @@
17791877 	return 0;
17801878 
17811879 disarmed:
1782
-	BUG_ON(!kprobe_disarmed(ap));
17831880 	hlist_del_rcu(&ap->hlist);
17841881 	return 0;
17851882 }
....@@ -1836,7 +1933,7 @@
18361933 			kps[i]->addr = NULL;
18371934 	mutex_unlock(&kprobe_mutex);
18381935 
1839
-	synchronize_sched();
1936
+	synchronize_rcu();
18401937 	for (i = 0; i < num; i++)
18411938 		if (kps[i]->addr)
18421939 			__unregister_kprobe_bottom(kps[i]);
....@@ -1861,6 +1958,97 @@
18611958 }
18621959 
18631960 #ifdef CONFIG_KRETPROBES
1961
+
1962
+unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
1963
+					     void *trampoline_address,
1964
+					     void *frame_pointer)
1965
+{
1966
+	struct kretprobe_instance *ri = NULL, *last = NULL;
1967
+	struct hlist_head *head;
1968
+	struct hlist_node *tmp;
1969
+	unsigned long flags;
1970
+	kprobe_opcode_t *correct_ret_addr = NULL;
1971
+	bool skipped = false;
1972
+
1973
+	kretprobe_hash_lock(current, &head, &flags);
1974
+
1975
+	/*
1976
+	 * It is possible to have multiple instances associated with a given
1977
+	 * task either because multiple functions in the call path have
1978
+	 * return probes installed on them, and/or more than one
1979
+	 * return probe was registered for a target function.
1980
+	 *
1981
+	 * We can handle this because:
1982
+	 *     - instances are always pushed into the head of the list
1983
+	 *     - when multiple return probes are registered for the same
1984
+	 *	 function, the (chronologically) first instance's ret_addr
1985
+	 *	 will be the real return address, and all the rest will
1986
+	 *	 point to kretprobe_trampoline.
1987
+	 */
1988
+	hlist_for_each_entry(ri, head, hlist) {
1989
+		if (ri->task != current)
1990
+			/* another task is sharing our hash bucket */
1991
+			continue;
1992
+		/*
1993
+		 * Return probes must be pushed on this hash list correct
1994
+		 * order (same as return order) so that it can be popped
1995
+		 * correctly. However, if we find it is pushed it incorrect
1996
+		 * order, this means we find a function which should not be
1997
+		 * probed, because the wrong order entry is pushed on the
1998
+		 * path of processing other kretprobe itself.
1999
+		 */
2000
+		if (ri->fp != frame_pointer) {
2001
+			if (!skipped)
2002
+				pr_warn("kretprobe is stacked incorrectly. Trying to fixup.\n");
2003
+			skipped = true;
2004
+			continue;
2005
+		}
2006
+
2007
+		correct_ret_addr = ri->ret_addr;
2008
+		if (skipped)
2009
+			pr_warn("%ps must be blacklisted because of incorrect kretprobe order\n",
2010
+				ri->rp->kp.addr);
2011
+
2012
+		if (correct_ret_addr != trampoline_address)
2013
+			/*
2014
+			 * This is the real return address. Any other
2015
+			 * instances associated with this task are for
2016
+			 * other calls deeper on the call stack
2017
+			 */
2018
+			break;
2019
+	}
2020
+
2021
+	BUG_ON(!correct_ret_addr || (correct_ret_addr == trampoline_address));
2022
+	last = ri;
2023
+
2024
+	hlist_for_each_entry_safe(ri, tmp, head, hlist) {
2025
+		if (ri->task != current)
2026
+			/* another task is sharing our hash bucket */
2027
+			continue;
2028
+		if (ri->fp != frame_pointer)
2029
+			continue;
2030
+
2031
+		if (ri->rp && ri->rp->handler) {
2032
+			struct kprobe *prev = kprobe_running();
2033
+
2034
+			__this_cpu_write(current_kprobe, &ri->rp->kp);
2035
+			ri->ret_addr = correct_ret_addr;
2036
+			ri->rp->handler(ri, regs);
2037
+			__this_cpu_write(current_kprobe, prev);
2038
+		}
2039
+
2040
+		recycle_rp_inst(ri);
2041
+
2042
+		if (ri == last)
2043
+			break;
2044
+	}
2045
+
2046
+	kretprobe_hash_unlock(current, &flags);
2047
+
2048
+	return (unsigned long)correct_ret_addr;
2049
+}
2050
+NOKPROBE_SYMBOL(__kretprobe_trampoline_handler)
2051
+
18642052 /*
18652053  * This kprobe pre_handler is registered with every kretprobe. When probe
18662054  * hits it will set up the return probe.
....@@ -1871,20 +2059,14 @@
18712059 	unsigned long hash, flags = 0;
18722060 	struct kretprobe_instance *ri;
18732061 
1874
-	/*
1875
-	 * To avoid deadlocks, prohibit return probing in NMI contexts,
1876
-	 * just skip the probe and increase the (inexact) 'nmissed'
1877
-	 * statistical counter, so that the user is informed that
1878
-	 * something happened:
1879
-	 */
1880
-	if (unlikely(in_nmi())) {
1881
-		rp->nmissed++;
1882
-		return 0;
1883
-	}
1884
-
18852062 	/* TODO: consider to only swap the RA after the last pre_handler fired */
18862063 	hash = hash_ptr(current, KPROBE_HASH_BITS);
1887
-	raw_spin_lock_irqsave(&rp->lock, flags);
2064
+	/*
2065
+	 * Nested is a workaround that will soon not be needed.
2066
+	 * There's other protections that make sure the same lock
2067
+	 * is not taken on the same CPU that lockdep is unaware of.
2068
+	 */
2069
+	raw_spin_lock_irqsave_nested(&rp->lock, flags, 1);
18882070 	if (!hlist_empty(&rp->free_instances)) {
18892071 		ri = hlist_entry(rp->free_instances.first,
18902072 				struct kretprobe_instance, hlist);
....@@ -1895,7 +2077,7 @@
18952077 		ri->task = current;
18962078 
18972079 		if (rp->entry_handler && rp->entry_handler(ri, regs)) {
1898
-			raw_spin_lock_irqsave(&rp->lock, flags);
2080
+			raw_spin_lock_irqsave_nested(&rp->lock, flags, 1);
18992081 			hlist_add_head(&ri->hlist, &rp->free_instances);
19002082 			raw_spin_unlock_irqrestore(&rp->lock, flags);
19012083 			return 0;
....@@ -1985,7 +2167,7 @@
19852167 
19862168 	/* Pre-allocate memory for max kretprobe instances */
19872169 	if (rp->maxactive <= 0) {
1988
-#ifdef CONFIG_PREEMPT
2170
+#ifdef CONFIG_PREEMPTION
19892171 		rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
19902172 #else
19912173 		rp->maxactive = num_possible_cpus();
....@@ -2049,7 +2231,7 @@
20492231 			rps[i]->kp.addr = NULL;
20502232 	mutex_unlock(&kprobe_mutex);
20512233 
2052
-	synchronize_sched();
2234
+	synchronize_rcu();
20532235 	for (i = 0; i < num; i++) {
20542236 		if (rps[i]->kp.addr) {
20552237 			__unregister_kprobe_bottom(&rps[i]->kp);
....@@ -2095,6 +2277,8 @@
20952277 {
20962278 	struct kprobe *kp;
20972279 
2280
+	lockdep_assert_held(&kprobe_mutex);
2281
+
20982282 	if (WARN_ON_ONCE(kprobe_gone(p)))
20992283 		return;
21002284 
....@@ -2104,7 +2288,7 @@
21042288 		 * If this is an aggr_kprobe, we have to list all the
21052289 		 * chained probes and mark them GONE.
21062290 		 */
2107
-		list_for_each_entry_rcu(kp, &p->list, list)
2291
+		list_for_each_entry(kp, &p->list, list)
21082292 			kp->flags |= KPROBE_FLAG_GONE;
21092293 		p->post_handler = NULL;
21102294 		kill_optimized_kprobe(p);
....@@ -2169,8 +2353,11 @@
21692353 	if (!kprobes_all_disarmed && kprobe_disabled(p)) {
21702354 		p->flags &= ~KPROBE_FLAG_DISABLED;
21712355 		ret = arm_kprobe(p);
2172
-		if (ret)
2356
+		if (ret) {
21732357 			p->flags |= KPROBE_FLAG_DISABLED;
2358
+			if (p != kp)
2359
+				kp->flags |= KPROBE_FLAG_DISABLED;
2360
+		}
21742361 	}
21752362 out:
21762363 	mutex_unlock(&kprobe_mutex);
....@@ -2223,6 +2410,46 @@
22232410 	return 0;
22242411 }
22252412 
2413
+/* Remove all symbols in given area from kprobe blacklist */
2414
+static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end)
2415
+{
2416
+	struct kprobe_blacklist_entry *ent, *n;
2417
+
2418
+	list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) {
2419
+		if (ent->start_addr < start || ent->start_addr >= end)
2420
+			continue;
2421
+		list_del(&ent->list);
2422
+		kfree(ent);
2423
+	}
2424
+}
2425
+
2426
+static void kprobe_remove_ksym_blacklist(unsigned long entry)
2427
+{
2428
+	kprobe_remove_area_blacklist(entry, entry + 1);
2429
+}
2430
+
2431
+int __weak arch_kprobe_get_kallsym(unsigned int *symnum, unsigned long *value,
2432
+				   char *type, char *sym)
2433
+{
2434
+	return -ERANGE;
2435
+}
2436
+
2437
+int kprobe_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
2438
+		       char *sym)
2439
+{
2440
+#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
2441
+	if (!kprobe_cache_get_kallsym(&kprobe_insn_slots, &symnum, value, type, sym))
2442
+		return 0;
2443
+#ifdef CONFIG_OPTPROBES
2444
+	if (!kprobe_cache_get_kallsym(&kprobe_optinsn_slots, &symnum, value, type, sym))
2445
+		return 0;
2446
+#endif
2447
+#endif
2448
+	if (!arch_kprobe_get_kallsym(&symnum, value, type, sym))
2449
+		return 0;
2450
+	return -ERANGE;
2451
+}
2452
+
22262453 int __init __weak arch_populate_kprobe_blacklist(void)
22272454 {
22282455 	return 0;
....@@ -2255,8 +2482,60 @@
22552482 	/* Symbols in __kprobes_text are blacklisted */
22562483 	ret = kprobe_add_area_blacklist((unsigned long)__kprobes_text_start,
22572484 					(unsigned long)__kprobes_text_end);
2485
+	if (ret)
2486
+		return ret;
2487
+
2488
+	/* Symbols in noinstr section are blacklisted */
2489
+	ret = kprobe_add_area_blacklist((unsigned long)__noinstr_text_start,
2490
+					(unsigned long)__noinstr_text_end);
22582491 
22592492 	return ret ? : arch_populate_kprobe_blacklist();
2493
+}
2494
+
2495
+static void add_module_kprobe_blacklist(struct module *mod)
2496
+{
2497
+	unsigned long start, end;
2498
+	int i;
2499
+
2500
+	if (mod->kprobe_blacklist) {
2501
+		for (i = 0; i < mod->num_kprobe_blacklist; i++)
2502
+			kprobe_add_ksym_blacklist(mod->kprobe_blacklist[i]);
2503
+	}
2504
+
2505
+	start = (unsigned long)mod->kprobes_text_start;
2506
+	if (start) {
2507
+		end = start + mod->kprobes_text_size;
2508
+		kprobe_add_area_blacklist(start, end);
2509
+	}
2510
+
2511
+	start = (unsigned long)mod->noinstr_text_start;
2512
+	if (start) {
2513
+		end = start + mod->noinstr_text_size;
2514
+		kprobe_add_area_blacklist(start, end);
2515
+	}
2516
+}
2517
+
2518
+static void remove_module_kprobe_blacklist(struct module *mod)
2519
+{
2520
+	unsigned long start, end;
2521
+	int i;
2522
+
2523
+	if (mod->kprobe_blacklist) {
2524
+		for (i = 0; i < mod->num_kprobe_blacklist; i++)
2525
+			kprobe_remove_ksym_blacklist(mod->kprobe_blacklist[i]);
2526
+	}
2527
+
2528
+	start = (unsigned long)mod->kprobes_text_start;
2529
+	if (start) {
2530
+		end = start + mod->kprobes_text_size;
2531
+		kprobe_remove_area_blacklist(start, end);
2532
+	}
2533
+
2534
+	start = (unsigned long)mod->noinstr_text_start;
2535
+	if (start) {
2536
+		end = start + mod->noinstr_text_size;
2537
+		kprobe_remove_area_blacklist(start, end);
2538
+	}
22602539 }
22612540 
22622541 /* Module notifier call back, checking kprobes on the module */
....@@ -2269,6 +2548,11 @@
22692548 	unsigned int i;
22702549 	int checkcore = (val == MODULE_STATE_GOING);
22712550 
2551
+	if (val == MODULE_STATE_COMING) {
2552
+		mutex_lock(&kprobe_mutex);
2553
+		add_module_kprobe_blacklist(mod);
2554
+		mutex_unlock(&kprobe_mutex);
2555
+	}
22722556 	if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
22732557 		return NOTIFY_DONE;
22742558 
....@@ -2281,7 +2565,7 @@
22812565 	mutex_lock(&kprobe_mutex);
22822566 	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
22832567 		head = &kprobe_table[i];
2284
-		hlist_for_each_entry_rcu(p, head, hlist) {
2568
+		hlist_for_each_entry(p, head, hlist) {
22852569 			if (kprobe_gone(p))
22862570 				continue;
22872571 
....@@ -2303,6 +2587,8 @@
23032587 			}
23042588 		}
23052589 	}
2590
+	if (val == MODULE_STATE_GOING)
2591
+		remove_module_kprobe_blacklist(mod);
23062592 	mutex_unlock(&kprobe_mutex);
23072593 	return NOTIFY_DONE;
23082594 }
....@@ -2315,6 +2601,28 @@
23152601 /* Markers of _kprobe_blacklist section */
23162602 extern unsigned long __start_kprobe_blacklist[];
23172603 extern unsigned long __stop_kprobe_blacklist[];
2604
+
2605
+void kprobe_free_init_mem(void)
2606
+{
2607
+	void *start = (void *)(&__init_begin);
2608
+	void *end = (void *)(&__init_end);
2609
+	struct hlist_head *head;
2610
+	struct kprobe *p;
2611
+	int i;
2612
+
2613
+	mutex_lock(&kprobe_mutex);
2614
+
2615
+	/* Kill all kprobes on initmem */
2616
+	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2617
+		head = &kprobe_table[i];
2618
+		hlist_for_each_entry(p, head, hlist) {
2619
+			if (start <= (void *)p->addr && (void *)p->addr < end)
2620
+				kill_kprobe(p);
2621
+		}
2622
+	}
2623
+
2624
+	mutex_unlock(&kprobe_mutex);
2625
+}
23182626 
23192627 static int __init init_kprobes(void)
23202628 {
....@@ -2346,17 +2654,13 @@
23462654 		}
23472655 	}
23482656 
2349
-#if defined(CONFIG_OPTPROBES)
2350
-#if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
2351
-	/* Init kprobe_optinsn_slots */
2352
-	kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2353
-#endif
2354
-	/* By default, kprobes can be optimized */
2355
-	kprobes_allow_optimization = true;
2356
-#endif
2357
-
23582657 	/* By default, kprobes are armed */
23592658 	kprobes_all_disarmed = false;
2659
+
2660
+#if defined(CONFIG_OPTPROBES) && defined(__ARCH_WANT_KPROBES_INSN_SLOT)
2661
+	/* Init kprobe_optinsn_slots for allocation */
2662
+	kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2663
+#endif
23602664 
23612665 	err = arch_init_kprobes();
23622666 	if (!err)
....@@ -2370,6 +2674,22 @@
23702674 		init_test_probes();
23712675 	return err;
23722676 }
2677
+early_initcall(init_kprobes);
2678
+
2679
+#if defined(CONFIG_OPTPROBES)
2680
+static int __init init_optprobes(void)
2681
+{
2682
+	/*
2683
+	 * Enable kprobe optimization - this kicks the optimizer which
2684
+	 * depends on synchronize_rcu_tasks() and ksoftirqd, that is
2685
+	 * not spawned in early initcall. So delay the optimization.
2686
+	 */
2687
+	optimize_all_kprobes();
2688
+
2689
+	return 0;
2690
+}
2691
+subsys_initcall(init_optprobes);
2692
+#endif
23732693 
23742694 #ifdef CONFIG_DEBUG_FS
23752695 static void report_probe(struct seq_file *pi, struct kprobe *p,
....@@ -2445,28 +2765,19 @@
24452765 	return 0;
24462766 }
24472767 
2448
-static const struct seq_operations kprobes_seq_ops = {
2768
+static const struct seq_operations kprobes_sops = {
24492769 	.start = kprobe_seq_start,
24502770 	.next  = kprobe_seq_next,
24512771 	.stop  = kprobe_seq_stop,
24522772 	.show  = show_kprobe_addr
24532773 };
24542774 
2455
-static int kprobes_open(struct inode *inode, struct file *filp)
2456
-{
2457
-	return seq_open(filp, &kprobes_seq_ops);
2458
-}
2459
-
2460
-static const struct file_operations debugfs_kprobes_operations = {
2461
-	.open           = kprobes_open,
2462
-	.read           = seq_read,
2463
-	.llseek         = seq_lseek,
2464
-	.release        = seq_release,
2465
-};
2775
+DEFINE_SEQ_ATTRIBUTE(kprobes);
24662776 
24672777 /* kprobes/blacklist -- shows which functions can not be probed */
24682778 static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos)
24692779 {
2780
+	mutex_lock(&kprobe_mutex);
24702781 	return seq_list_start(&kprobe_blacklist, *pos);
24712782 }
24722783 
....@@ -2493,24 +2804,18 @@
24932804 	return 0;
24942805 }
24952806 
2496
-static const struct seq_operations kprobe_blacklist_seq_ops = {
2497
-	.start = kprobe_blacklist_seq_start,
2498
-	.next  = kprobe_blacklist_seq_next,
2499
-	.stop  = kprobe_seq_stop,	/* Reuse void function */
2500
-	.show  = kprobe_blacklist_seq_show,
2501
-};
2502
-
2503
-static int kprobe_blacklist_open(struct inode *inode, struct file *filp)
2807
+static void kprobe_blacklist_seq_stop(struct seq_file *f, void *v)
25042808 {
2505
-	return seq_open(filp, &kprobe_blacklist_seq_ops);
2809
+	mutex_unlock(&kprobe_mutex);
25062810 }
25072811 
2508
-static const struct file_operations debugfs_kprobe_blacklist_ops = {
2509
-	.open           = kprobe_blacklist_open,
2510
-	.read           = seq_read,
2511
-	.llseek         = seq_lseek,
2512
-	.release        = seq_release,
2812
+static const struct seq_operations kprobe_blacklist_sops = {
2813
+	.start = kprobe_blacklist_seq_start,
2814
+	.next  = kprobe_blacklist_seq_next,
2815
+	.stop  = kprobe_blacklist_seq_stop,
2816
+	.show  = kprobe_blacklist_seq_show,
25132817 };
2818
+DEFINE_SEQ_ATTRIBUTE(kprobe_blacklist);
25142819 
25152820 static int arm_all_kprobes(void)
25162821 {
....@@ -2535,7 +2840,7 @@
25352840 	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
25362841 		head = &kprobe_table[i];
25372842 		/* Arm all kprobes on a best-effort basis */
2538
-		hlist_for_each_entry_rcu(p, head, hlist) {
2843
+		hlist_for_each_entry(p, head, hlist) {
25392844 			if (!kprobe_disabled(p)) {
25402845 				err = arm_kprobe(p);
25412846 				if (err)  {
....@@ -2578,7 +2883,7 @@
25782883 	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
25792884 		head = &kprobe_table[i];
25802885 		/* Disarm all kprobes on a best-effort basis */
2581
-		hlist_for_each_entry_rcu(p, head, hlist) {
2886
+		hlist_for_each_entry(p, head, hlist) {
25822887 			if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) {
25832888 				err = disarm_kprobe(p, false);
25842889 				if (err) {
....@@ -2664,36 +2969,19 @@
26642969 
26652970 static int __init debugfs_kprobe_init(void)
26662971 {
2667
-	struct dentry *dir, *file;
2668
-	unsigned int value = 1;
2972
+	struct dentry *dir;
26692973 
26702974 	dir = debugfs_create_dir("kprobes", NULL);
2671
-	if (!dir)
2672
-		return -ENOMEM;
26732975 
2674
-	file = debugfs_create_file("list", 0400, dir, NULL,
2675
-				&debugfs_kprobes_operations);
2676
-	if (!file)
2677
-		goto error;
2976
+	debugfs_create_file("list", 0400, dir, NULL, &kprobes_fops);
26782977 
2679
-	file = debugfs_create_file("enabled", 0600, dir,
2680
-					&value, &fops_kp);
2681
-	if (!file)
2682
-		goto error;
2978
+	debugfs_create_file("enabled", 0600, dir, NULL, &fops_kp);
26832979 
2684
-	file = debugfs_create_file("blacklist", 0400, dir, NULL,
2685
-				&debugfs_kprobe_blacklist_ops);
2686
-	if (!file)
2687
-		goto error;
2980
+	debugfs_create_file("blacklist", 0400, dir, NULL,
2981
+			    &kprobe_blacklist_fops);
26882982 
26892983 	return 0;
2690
-
2691
-error:
2692
-	debugfs_remove(dir);
2693
-	return -ENOMEM;
26942984 }
26952985 
26962986 late_initcall(debugfs_kprobe_init);
26972987 #endif /* CONFIG_DEBUG_FS */
2698
-
2699
-module_init(init_kprobes);