forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-10 23fa18eaa71266feff7ba8d83022d9e1cc83c65a 
 kernel/arch/arm64/kernel/ftrace.c
....@@ -1,12 +1,9 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * arch/arm64/kernel/ftrace.c
34  *
45  * Copyright (C) 2013 Linaro Limited
56  * Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
6
- *
7
- * This program is free software; you can redistribute it and/or modify
8
- * it under the terms of the GNU General Public License version 2 as
9
- * published by the Free Software Foundation.
107  */
118 
129 #include <linux/ftrace.h>
....@@ -65,6 +62,80 @@
6562 	return ftrace_modify_code(pc, 0, new, false);
6663 }
6764 
65
+static struct plt_entry *get_ftrace_plt(struct module *mod, unsigned long addr)
66
+{
67
+#ifdef CONFIG_ARM64_MODULE_PLTS
68
+	struct plt_entry *plt = mod->arch.ftrace_trampolines;
69
+
70
+	if (addr == FTRACE_ADDR)
71
+		return &plt[FTRACE_PLT_IDX];
72
+	if (addr == FTRACE_REGS_ADDR &&
73
+	    IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
74
+		return &plt[FTRACE_REGS_PLT_IDX];
75
+#endif
76
+	return NULL;
77
+}
78
+
79
+/*
80
+ * Find the address the callsite must branch to in order to reach '*addr'.
81
+ *
82
+ * Due to the limited range of 'BL' instructions, modules may be placed too far
83
+ * away to branch directly and must use a PLT.
84
+ *
85
+ * Returns true when '*addr' contains a reachable target address, or has been
86
+ * modified to contain a PLT address. Returns false otherwise.
87
+ */
88
+static bool ftrace_find_callable_addr(struct dyn_ftrace *rec,
89
+				      struct module *mod,
90
+				      unsigned long *addr)
91
+{
92
+	unsigned long pc = rec->ip;
93
+	long offset = (long)*addr - (long)pc;
94
+	struct plt_entry *plt;
95
+
96
+	/*
97
+	 * When the target is within range of the 'BL' instruction, use 'addr'
98
+	 * as-is and branch to that directly.
99
+	 */
100
+	if (offset >= -SZ_128M && offset < SZ_128M)
101
+		return true;
102
+
103
+	/*
104
+	 * When the target is outside of the range of a 'BL' instruction, we
105
+	 * must use a PLT to reach it. We can only place PLTs for modules, and
106
+	 * only when module PLT support is built-in.
107
+	 */
108
+	if (!IS_ENABLED(CONFIG_ARM64_MODULE_PLTS))
109
+		return false;
110
+
111
+	/*
112
+	 * 'mod' is only set at module load time, but if we end up
113
+	 * dealing with an out-of-range condition, we can assume it
114
+	 * is due to a module being loaded far away from the kernel.
115
+	 *
116
+	 * NOTE: __module_text_address() must be called with preemption
117
+	 * disabled, but we can rely on ftrace_lock to ensure that 'mod'
118
+	 * retains its validity throughout the remainder of this code.
119
+	 */
120
+	if (!mod) {
121
+		preempt_disable();
122
+		mod = __module_text_address(pc);
123
+		preempt_enable();
124
+	}
125
+
126
+	if (WARN_ON(!mod))
127
+		return false;
128
+
129
+	plt = get_ftrace_plt(mod, *addr);
130
+	if (!plt) {
131
+		pr_err("ftrace: no module PLT for %ps\n", (void *)*addr);
132
+		return false;
133
+	}
134
+
135
+	*addr = (unsigned long)plt;
136
+	return true;
137
+}
138
+
68139 /*
69140  * Turn on the call to ftrace_caller() in instrumented function
70141  */
....@@ -72,75 +143,69 @@
72143 {
73144 	unsigned long pc = rec->ip;
74145 	u32 old, new;
75
-	long offset = (long)pc - (long)addr;
76146 
77
-	if (offset < -SZ_128M || offset >= SZ_128M) {
78
-#ifdef CONFIG_ARM64_MODULE_PLTS
79
-		struct plt_entry trampoline, *dst;
80
-		struct module *mod;
81
-
82
-		/*
83
-		 * On kernels that support module PLTs, the offset between the
84
-		 * branch instruction and its target may legally exceed the
85
-		 * range of an ordinary relative 'bl' opcode. In this case, we
86
-		 * need to branch via a trampoline in the module.
87
-		 *
88
-		 * NOTE: __module_text_address() must be called with preemption
89
-		 * disabled, but we can rely on ftrace_lock to ensure that 'mod'
90
-		 * retains its validity throughout the remainder of this code.
91
-		 */
92
-		preempt_disable();
93
-		mod = __module_text_address(pc);
94
-		preempt_enable();
95
-
96
-		if (WARN_ON(!mod))
97
-			return -EINVAL;
98
-
99
-		/*
100
-		 * There is only one ftrace trampoline per module. For now,
101
-		 * this is not a problem since on arm64, all dynamic ftrace
102
-		 * invocations are routed via ftrace_caller(). This will need
103
-		 * to be revisited if support for multiple ftrace entry points
104
-		 * is added in the future, but for now, the pr_err() below
105
-		 * deals with a theoretical issue only.
106
-		 */
107
-		dst = mod->arch.ftrace_trampoline;
108
-		trampoline = get_plt_entry(addr);
109
-		if (!plt_entries_equal(dst, &trampoline)) {
110
-			if (!plt_entries_equal(dst, &(struct plt_entry){})) {
111
-				pr_err("ftrace: far branches to multiple entry points unsupported inside a single module\n");
112
-				return -EINVAL;
113
-			}
114
-
115
-			/* point the trampoline to our ftrace entry point */
116
-			module_disable_ro(mod);
117
-			*dst = trampoline;
118
-			module_enable_ro(mod, true);
119
-
120
-			/*
121
-			 * Ensure updated trampoline is visible to instruction
122
-			 * fetch before we patch in the branch. Although the
123
-			 * architecture doesn't require an IPI in this case,
124
-			 * Neoverse-N1 erratum #1542419 does require one
125
-			 * if the TLB maintenance in module_enable_ro() is
126
-			 * skipped due to rodata_enabled. It doesn't seem worth
127
-			 * it to make it conditional given that this is
128
-			 * certainly not a fast-path.
129
-			 */
130
-			flush_icache_range((unsigned long)&dst[0],
131
-					   (unsigned long)&dst[1]);
132
-		}
133
-		addr = (unsigned long)dst;
134
-#else /* CONFIG_ARM64_MODULE_PLTS */
147
+	if (!ftrace_find_callable_addr(rec, NULL, &addr))
135148 		return -EINVAL;
136
-#endif /* CONFIG_ARM64_MODULE_PLTS */
137
-	}
138149 
139150 	old = aarch64_insn_gen_nop();
140151 	new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
141152 
142153 	return ftrace_modify_code(pc, old, new, true);
143154 }
155
+
156
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
157
+int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
158
+			unsigned long addr)
159
+{
160
+	unsigned long pc = rec->ip;
161
+	u32 old, new;
162
+
163
+	if (!ftrace_find_callable_addr(rec, NULL, &old_addr))
164
+		return -EINVAL;
165
+	if (!ftrace_find_callable_addr(rec, NULL, &addr))
166
+		return -EINVAL;
167
+
168
+	old = aarch64_insn_gen_branch_imm(pc, old_addr,
169
+					  AARCH64_INSN_BRANCH_LINK);
170
+	new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
171
+
172
+	return ftrace_modify_code(pc, old, new, true);
173
+}
174
+
175
+/*
176
+ * The compiler has inserted two NOPs before the regular function prologue.
177
+ * All instrumented functions follow the AAPCS, so x0-x8 and x19-x30 are live,
178
+ * and x9-x18 are free for our use.
179
+ *
180
+ * At runtime we want to be able to swing a single NOP <-> BL to enable or
181
+ * disable the ftrace call. The BL requires us to save the original LR value,
182
+ * so here we insert a <MOV X9, LR> over the first NOP so the instructions
183
+ * before the regular prologue are:
184
+ *
185
+ * | Compiled | Disabled   | Enabled    |
186
+ * +----------+------------+------------+
187
+ * | NOP      | MOV X9, LR | MOV X9, LR |
188
+ * | NOP      | NOP        | BL <entry> |
189
+ *
190
+ * The LR value will be recovered by ftrace_regs_entry, and restored into LR
191
+ * before returning to the regular function prologue. When a function is not
192
+ * being traced, the MOV is not harmful given x9 is not live per the AAPCS.
193
+ *
194
+ * Note: ftrace_process_locs() has pre-adjusted rec->ip to be the address of
195
+ * the BL.
196
+ */
197
+int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
198
+{
199
+	unsigned long pc = rec->ip - AARCH64_INSN_SIZE;
200
+	u32 old, new;
201
+
202
+	old = aarch64_insn_gen_nop();
203
+	new = aarch64_insn_gen_move_reg(AARCH64_INSN_REG_9,
204
+					AARCH64_INSN_REG_LR,
205
+					AARCH64_INSN_VARIANT_64BIT);
206
+	return ftrace_modify_code(pc, old, new, true);
207
+}
208
+#endif
144209 
145210 /*
146211  * Turn off the call to ftrace_caller() in instrumented function
....@@ -149,59 +214,35 @@
149214 		    unsigned long addr)
150215 {
151216 	unsigned long pc = rec->ip;
152
-	bool validate = true;
153217 	u32 old = 0, new;
154
-	long offset = (long)pc - (long)addr;
155
-
156
-	if (offset < -SZ_128M || offset >= SZ_128M) {
157
-#ifdef CONFIG_ARM64_MODULE_PLTS
158
-		u32 replaced;
159
-
160
-		/*
161
-		 * 'mod' is only set at module load time, but if we end up
162
-		 * dealing with an out-of-range condition, we can assume it
163
-		 * is due to a module being loaded far away from the kernel.
164
-		 */
165
-		if (!mod) {
166
-			preempt_disable();
167
-			mod = __module_text_address(pc);
168
-			preempt_enable();
169
-
170
-			if (WARN_ON(!mod))
171
-				return -EINVAL;
172
-		}
173
-
174
-		/*
175
-		 * The instruction we are about to patch may be a branch and
176
-		 * link instruction that was redirected via a PLT entry. In
177
-		 * this case, the normal validation will fail, but we can at
178
-		 * least check that we are dealing with a branch and link
179
-		 * instruction that points into the right module.
180
-		 */
181
-		if (aarch64_insn_read((void *)pc, &replaced))
182
-			return -EFAULT;
183
-
184
-		if (!aarch64_insn_is_bl(replaced) ||
185
-		    !within_module(pc + aarch64_get_branch_offset(replaced),
186
-				   mod))
187
-			return -EINVAL;
188
-
189
-		validate = false;
190
-#else /* CONFIG_ARM64_MODULE_PLTS */
191
-		return -EINVAL;
192
-#endif /* CONFIG_ARM64_MODULE_PLTS */
193
-	} else {
194
-		old = aarch64_insn_gen_branch_imm(pc, addr,
195
-						  AARCH64_INSN_BRANCH_LINK);
196
-	}
197218 
198219 	new = aarch64_insn_gen_nop();
199220 
200
-	return ftrace_modify_code(pc, old, new, validate);
221
+	/*
222
+	 * When using mcount, callsites in modules may have been initalized to
223
+	 * call an arbitrary module PLT (which redirects to the _mcount stub)
224
+	 * rather than the ftrace PLT we'll use at runtime (which redirects to
225
+	 * the ftrace trampoline). We can ignore the old PLT when initializing
226
+	 * the callsite.
227
+	 *
228
+	 * Note: 'mod' is only set at module load time.
229
+	 */
230
+	if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) &&
231
+	    IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && mod) {
232
+		return aarch64_insn_patch_text_nosync((void *)pc, new);
233
+	}
234
+
235
+	if (!ftrace_find_callable_addr(rec, mod, &addr))
236
+		return -EINVAL;
237
+
238
+	old = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
239
+
240
+	return ftrace_modify_code(pc, old, new, true);
201241 }
202242 
203243 void arch_ftrace_update_code(int command)
204244 {
245
+	command |= FTRACE_MAY_SLEEP;
205246 	ftrace_modify_all_code(command);
206247 }
207248 
....@@ -220,7 +261,7 @@
220261  *
221262  * Note that @frame_pointer is used only for sanity check later.
222263  */
223
-void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
264
+void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
224265 			   unsigned long frame_pointer)
225266 {
226267 	unsigned long return_hooker = (unsigned long)&return_to_handler;