.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-or-later |
---|
1 | 2 | /* |
---|
2 | 3 | * Performance counter callchain support - powerpc architecture code |
---|
3 | 4 | * |
---|
4 | 5 | * Copyright © 2009 Paul Mackerras, IBM Corporation. |
---|
5 | | - * |
---|
6 | | - * This program is free software; you can redistribute it and/or |
---|
7 | | - * modify it under the terms of the GNU General Public License |
---|
8 | | - * as published by the Free Software Foundation; either version |
---|
9 | | - * 2 of the License, or (at your option) any later version. |
---|
10 | 6 | */ |
---|
11 | 7 | #include <linux/kernel.h> |
---|
12 | 8 | #include <linux/sched.h> |
---|
.. | .. |
---|
15 | 11 | #include <linux/uaccess.h> |
---|
16 | 12 | #include <linux/mm.h> |
---|
17 | 13 | #include <asm/ptrace.h> |
---|
18 | | -#include <asm/pgtable.h> |
---|
19 | 14 | #include <asm/sigcontext.h> |
---|
20 | 15 | #include <asm/ucontext.h> |
---|
21 | 16 | #include <asm/vdso.h> |
---|
22 | | -#ifdef CONFIG_PPC64 |
---|
23 | | -#include "../kernel/ppc32.h" |
---|
24 | | -#endif |
---|
25 | 17 | #include <asm/pte-walk.h> |
---|
26 | 18 | |
---|
| 19 | +#include "callchain.h" |
---|
27 | 20 | |
---|
28 | 21 | /* |
---|
29 | 22 | * Is sp valid as the address of the next kernel stack frame after prev_sp? |
---|
.. | .. |
---|
68 | 61 | next_sp = fp[0]; |
---|
69 | 62 | |
---|
70 | 63 | if (next_sp == sp + STACK_INT_FRAME_SIZE && |
---|
| 64 | + validate_sp(sp, current, STACK_INT_FRAME_SIZE) && |
---|
71 | 65 | fp[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) { |
---|
72 | 66 | /* |
---|
73 | 67 | * This looks like an interrupt frame for an |
---|
.. | .. |
---|
106 | 100 | } |
---|
107 | 101 | } |
---|
108 | 102 | |
---|
109 | | -#ifdef CONFIG_PPC64 |
---|
110 | | -/* |
---|
111 | | - * On 64-bit we don't want to invoke hash_page on user addresses from |
---|
112 | | - * interrupt context, so if the access faults, we read the page tables |
---|
113 | | - * to find which page (if any) is mapped and access it directly. |
---|
114 | | - */ |
---|
115 | | -static int read_user_stack_slow(void __user *ptr, void *buf, int nb) |
---|
116 | | -{ |
---|
117 | | - int ret = -EFAULT; |
---|
118 | | - pgd_t *pgdir; |
---|
119 | | - pte_t *ptep, pte; |
---|
120 | | - unsigned shift; |
---|
121 | | - unsigned long addr = (unsigned long) ptr; |
---|
122 | | - unsigned long offset; |
---|
123 | | - unsigned long pfn, flags; |
---|
124 | | - void *kaddr; |
---|
125 | | - |
---|
126 | | - pgdir = current->mm->pgd; |
---|
127 | | - if (!pgdir) |
---|
128 | | - return -EFAULT; |
---|
129 | | - |
---|
130 | | - local_irq_save(flags); |
---|
131 | | - ptep = find_current_mm_pte(pgdir, addr, NULL, &shift); |
---|
132 | | - if (!ptep) |
---|
133 | | - goto err_out; |
---|
134 | | - if (!shift) |
---|
135 | | - shift = PAGE_SHIFT; |
---|
136 | | - |
---|
137 | | - /* align address to page boundary */ |
---|
138 | | - offset = addr & ((1UL << shift) - 1); |
---|
139 | | - |
---|
140 | | - pte = READ_ONCE(*ptep); |
---|
141 | | - if (!pte_present(pte) || !pte_user(pte)) |
---|
142 | | - goto err_out; |
---|
143 | | - pfn = pte_pfn(pte); |
---|
144 | | - if (!page_is_ram(pfn)) |
---|
145 | | - goto err_out; |
---|
146 | | - |
---|
147 | | - /* no highmem to worry about here */ |
---|
148 | | - kaddr = pfn_to_kaddr(pfn); |
---|
149 | | - memcpy(buf, kaddr + offset, nb); |
---|
150 | | - ret = 0; |
---|
151 | | -err_out: |
---|
152 | | - local_irq_restore(flags); |
---|
153 | | - return ret; |
---|
154 | | -} |
---|
155 | | - |
---|
156 | | -static int read_user_stack_64(unsigned long __user *ptr, unsigned long *ret) |
---|
157 | | -{ |
---|
158 | | - if ((unsigned long)ptr > TASK_SIZE - sizeof(unsigned long) || |
---|
159 | | - ((unsigned long)ptr & 7)) |
---|
160 | | - return -EFAULT; |
---|
161 | | - |
---|
162 | | - pagefault_disable(); |
---|
163 | | - if (!__get_user_inatomic(*ret, ptr)) { |
---|
164 | | - pagefault_enable(); |
---|
165 | | - return 0; |
---|
166 | | - } |
---|
167 | | - pagefault_enable(); |
---|
168 | | - |
---|
169 | | - return read_user_stack_slow(ptr, ret, 8); |
---|
170 | | -} |
---|
171 | | - |
---|
172 | | -static int read_user_stack_32(unsigned int __user *ptr, unsigned int *ret) |
---|
173 | | -{ |
---|
174 | | - if ((unsigned long)ptr > TASK_SIZE - sizeof(unsigned int) || |
---|
175 | | - ((unsigned long)ptr & 3)) |
---|
176 | | - return -EFAULT; |
---|
177 | | - |
---|
178 | | - pagefault_disable(); |
---|
179 | | - if (!__get_user_inatomic(*ret, ptr)) { |
---|
180 | | - pagefault_enable(); |
---|
181 | | - return 0; |
---|
182 | | - } |
---|
183 | | - pagefault_enable(); |
---|
184 | | - |
---|
185 | | - return read_user_stack_slow(ptr, ret, 4); |
---|
186 | | -} |
---|
187 | | - |
---|
188 | | -static inline int valid_user_sp(unsigned long sp, int is_64) |
---|
189 | | -{ |
---|
190 | | - if (!sp || (sp & 7) || sp > (is_64 ? TASK_SIZE : 0x100000000UL) - 32) |
---|
191 | | - return 0; |
---|
192 | | - return 1; |
---|
193 | | -} |
---|
194 | | - |
---|
195 | | -/* |
---|
196 | | - * 64-bit user processes use the same stack frame for RT and non-RT signals. |
---|
197 | | - */ |
---|
198 | | -struct signal_frame_64 { |
---|
199 | | - char dummy[__SIGNAL_FRAMESIZE]; |
---|
200 | | - struct ucontext uc; |
---|
201 | | - unsigned long unused[2]; |
---|
202 | | - unsigned int tramp[6]; |
---|
203 | | - struct siginfo *pinfo; |
---|
204 | | - void *puc; |
---|
205 | | - struct siginfo info; |
---|
206 | | - char abigap[288]; |
---|
207 | | -}; |
---|
208 | | - |
---|
209 | | -static int is_sigreturn_64_address(unsigned long nip, unsigned long fp) |
---|
210 | | -{ |
---|
211 | | - if (nip == fp + offsetof(struct signal_frame_64, tramp)) |
---|
212 | | - return 1; |
---|
213 | | - if (vdso64_rt_sigtramp && current->mm->context.vdso_base && |
---|
214 | | - nip == current->mm->context.vdso_base + vdso64_rt_sigtramp) |
---|
215 | | - return 1; |
---|
216 | | - return 0; |
---|
217 | | -} |
---|
218 | | - |
---|
219 | | -/* |
---|
220 | | - * Do some sanity checking on the signal frame pointed to by sp. |
---|
221 | | - * We check the pinfo and puc pointers in the frame. |
---|
222 | | - */ |
---|
223 | | -static int sane_signal_64_frame(unsigned long sp) |
---|
224 | | -{ |
---|
225 | | - struct signal_frame_64 __user *sf; |
---|
226 | | - unsigned long pinfo, puc; |
---|
227 | | - |
---|
228 | | - sf = (struct signal_frame_64 __user *) sp; |
---|
229 | | - if (read_user_stack_64((unsigned long __user *) &sf->pinfo, &pinfo) || |
---|
230 | | - read_user_stack_64((unsigned long __user *) &sf->puc, &puc)) |
---|
231 | | - return 0; |
---|
232 | | - return pinfo == (unsigned long) &sf->info && |
---|
233 | | - puc == (unsigned long) &sf->uc; |
---|
234 | | -} |
---|
235 | | - |
---|
236 | | -static void perf_callchain_user_64(struct perf_callchain_entry_ctx *entry, |
---|
237 | | - struct pt_regs *regs) |
---|
238 | | -{ |
---|
239 | | - unsigned long sp, next_sp; |
---|
240 | | - unsigned long next_ip; |
---|
241 | | - unsigned long lr; |
---|
242 | | - long level = 0; |
---|
243 | | - struct signal_frame_64 __user *sigframe; |
---|
244 | | - unsigned long __user *fp, *uregs; |
---|
245 | | - |
---|
246 | | - next_ip = perf_instruction_pointer(regs); |
---|
247 | | - lr = regs->link; |
---|
248 | | - sp = regs->gpr[1]; |
---|
249 | | - perf_callchain_store(entry, next_ip); |
---|
250 | | - |
---|
251 | | - while (entry->nr < entry->max_stack) { |
---|
252 | | - fp = (unsigned long __user *) sp; |
---|
253 | | - if (!valid_user_sp(sp, 1) || read_user_stack_64(fp, &next_sp)) |
---|
254 | | - return; |
---|
255 | | - if (level > 0 && read_user_stack_64(&fp[2], &next_ip)) |
---|
256 | | - return; |
---|
257 | | - |
---|
258 | | - /* |
---|
259 | | - * Note: the next_sp - sp >= signal frame size check |
---|
260 | | - * is true when next_sp < sp, which can happen when |
---|
261 | | - * transitioning from an alternate signal stack to the |
---|
262 | | - * normal stack. |
---|
263 | | - */ |
---|
264 | | - if (next_sp - sp >= sizeof(struct signal_frame_64) && |
---|
265 | | - (is_sigreturn_64_address(next_ip, sp) || |
---|
266 | | - (level <= 1 && is_sigreturn_64_address(lr, sp))) && |
---|
267 | | - sane_signal_64_frame(sp)) { |
---|
268 | | - /* |
---|
269 | | - * This looks like an signal frame |
---|
270 | | - */ |
---|
271 | | - sigframe = (struct signal_frame_64 __user *) sp; |
---|
272 | | - uregs = sigframe->uc.uc_mcontext.gp_regs; |
---|
273 | | - if (read_user_stack_64(&uregs[PT_NIP], &next_ip) || |
---|
274 | | - read_user_stack_64(&uregs[PT_LNK], &lr) || |
---|
275 | | - read_user_stack_64(&uregs[PT_R1], &sp)) |
---|
276 | | - return; |
---|
277 | | - level = 0; |
---|
278 | | - perf_callchain_store_context(entry, PERF_CONTEXT_USER); |
---|
279 | | - perf_callchain_store(entry, next_ip); |
---|
280 | | - continue; |
---|
281 | | - } |
---|
282 | | - |
---|
283 | | - if (level == 0) |
---|
284 | | - next_ip = lr; |
---|
285 | | - perf_callchain_store(entry, next_ip); |
---|
286 | | - ++level; |
---|
287 | | - sp = next_sp; |
---|
288 | | - } |
---|
289 | | -} |
---|
290 | | - |
---|
291 | | -static inline int current_is_64bit(void) |
---|
292 | | -{ |
---|
293 | | - /* |
---|
294 | | - * We can't use test_thread_flag() here because we may be on an |
---|
295 | | - * interrupt stack, and the thread flags don't get copied over |
---|
296 | | - * from the thread_info on the main stack to the interrupt stack. |
---|
297 | | - */ |
---|
298 | | - return !test_ti_thread_flag(task_thread_info(current), TIF_32BIT); |
---|
299 | | -} |
---|
300 | | - |
---|
301 | | -#else /* CONFIG_PPC64 */ |
---|
302 | | -/* |
---|
303 | | - * On 32-bit we just access the address and let hash_page create a |
---|
304 | | - * HPTE if necessary, so there is no need to fall back to reading |
---|
305 | | - * the page tables. Since this is called at interrupt level, |
---|
306 | | - * do_page_fault() won't treat a DSI as a page fault. |
---|
307 | | - */ |
---|
308 | | -static int read_user_stack_32(unsigned int __user *ptr, unsigned int *ret) |
---|
309 | | -{ |
---|
310 | | - int rc; |
---|
311 | | - |
---|
312 | | - if ((unsigned long)ptr > TASK_SIZE - sizeof(unsigned int) || |
---|
313 | | - ((unsigned long)ptr & 3)) |
---|
314 | | - return -EFAULT; |
---|
315 | | - |
---|
316 | | - pagefault_disable(); |
---|
317 | | - rc = __get_user_inatomic(*ret, ptr); |
---|
318 | | - pagefault_enable(); |
---|
319 | | - |
---|
320 | | - return rc; |
---|
321 | | -} |
---|
322 | | - |
---|
323 | | -static inline void perf_callchain_user_64(struct perf_callchain_entry_ctx *entry, |
---|
324 | | - struct pt_regs *regs) |
---|
325 | | -{ |
---|
326 | | -} |
---|
327 | | - |
---|
328 | | -static inline int current_is_64bit(void) |
---|
329 | | -{ |
---|
330 | | - return 0; |
---|
331 | | -} |
---|
332 | | - |
---|
333 | | -static inline int valid_user_sp(unsigned long sp, int is_64) |
---|
334 | | -{ |
---|
335 | | - if (!sp || (sp & 7) || sp > TASK_SIZE - 32) |
---|
336 | | - return 0; |
---|
337 | | - return 1; |
---|
338 | | -} |
---|
339 | | - |
---|
340 | | -#define __SIGNAL_FRAMESIZE32 __SIGNAL_FRAMESIZE |
---|
341 | | -#define sigcontext32 sigcontext |
---|
342 | | -#define mcontext32 mcontext |
---|
343 | | -#define ucontext32 ucontext |
---|
344 | | -#define compat_siginfo_t struct siginfo |
---|
345 | | - |
---|
346 | | -#endif /* CONFIG_PPC64 */ |
---|
347 | | - |
---|
348 | | -/* |
---|
349 | | - * Layout for non-RT signal frames |
---|
350 | | - */ |
---|
351 | | -struct signal_frame_32 { |
---|
352 | | - char dummy[__SIGNAL_FRAMESIZE32]; |
---|
353 | | - struct sigcontext32 sctx; |
---|
354 | | - struct mcontext32 mctx; |
---|
355 | | - int abigap[56]; |
---|
356 | | -}; |
---|
357 | | - |
---|
358 | | -/* |
---|
359 | | - * Layout for RT signal frames |
---|
360 | | - */ |
---|
361 | | -struct rt_signal_frame_32 { |
---|
362 | | - char dummy[__SIGNAL_FRAMESIZE32 + 16]; |
---|
363 | | - compat_siginfo_t info; |
---|
364 | | - struct ucontext32 uc; |
---|
365 | | - int abigap[56]; |
---|
366 | | -}; |
---|
367 | | - |
---|
368 | | -static int is_sigreturn_32_address(unsigned int nip, unsigned int fp) |
---|
369 | | -{ |
---|
370 | | - if (nip == fp + offsetof(struct signal_frame_32, mctx.mc_pad)) |
---|
371 | | - return 1; |
---|
372 | | - if (vdso32_sigtramp && current->mm->context.vdso_base && |
---|
373 | | - nip == current->mm->context.vdso_base + vdso32_sigtramp) |
---|
374 | | - return 1; |
---|
375 | | - return 0; |
---|
376 | | -} |
---|
377 | | - |
---|
378 | | -static int is_rt_sigreturn_32_address(unsigned int nip, unsigned int fp) |
---|
379 | | -{ |
---|
380 | | - if (nip == fp + offsetof(struct rt_signal_frame_32, |
---|
381 | | - uc.uc_mcontext.mc_pad)) |
---|
382 | | - return 1; |
---|
383 | | - if (vdso32_rt_sigtramp && current->mm->context.vdso_base && |
---|
384 | | - nip == current->mm->context.vdso_base + vdso32_rt_sigtramp) |
---|
385 | | - return 1; |
---|
386 | | - return 0; |
---|
387 | | -} |
---|
388 | | - |
---|
389 | | -static int sane_signal_32_frame(unsigned int sp) |
---|
390 | | -{ |
---|
391 | | - struct signal_frame_32 __user *sf; |
---|
392 | | - unsigned int regs; |
---|
393 | | - |
---|
394 | | - sf = (struct signal_frame_32 __user *) (unsigned long) sp; |
---|
395 | | - if (read_user_stack_32((unsigned int __user *) &sf->sctx.regs, ®s)) |
---|
396 | | - return 0; |
---|
397 | | - return regs == (unsigned long) &sf->mctx; |
---|
398 | | -} |
---|
399 | | - |
---|
400 | | -static int sane_rt_signal_32_frame(unsigned int sp) |
---|
401 | | -{ |
---|
402 | | - struct rt_signal_frame_32 __user *sf; |
---|
403 | | - unsigned int regs; |
---|
404 | | - |
---|
405 | | - sf = (struct rt_signal_frame_32 __user *) (unsigned long) sp; |
---|
406 | | - if (read_user_stack_32((unsigned int __user *) &sf->uc.uc_regs, ®s)) |
---|
407 | | - return 0; |
---|
408 | | - return regs == (unsigned long) &sf->uc.uc_mcontext; |
---|
409 | | -} |
---|
410 | | - |
---|
411 | | -static unsigned int __user *signal_frame_32_regs(unsigned int sp, |
---|
412 | | - unsigned int next_sp, unsigned int next_ip) |
---|
413 | | -{ |
---|
414 | | - struct mcontext32 __user *mctx = NULL; |
---|
415 | | - struct signal_frame_32 __user *sf; |
---|
416 | | - struct rt_signal_frame_32 __user *rt_sf; |
---|
417 | | - |
---|
418 | | - /* |
---|
419 | | - * Note: the next_sp - sp >= signal frame size check |
---|
420 | | - * is true when next_sp < sp, for example, when |
---|
421 | | - * transitioning from an alternate signal stack to the |
---|
422 | | - * normal stack. |
---|
423 | | - */ |
---|
424 | | - if (next_sp - sp >= sizeof(struct signal_frame_32) && |
---|
425 | | - is_sigreturn_32_address(next_ip, sp) && |
---|
426 | | - sane_signal_32_frame(sp)) { |
---|
427 | | - sf = (struct signal_frame_32 __user *) (unsigned long) sp; |
---|
428 | | - mctx = &sf->mctx; |
---|
429 | | - } |
---|
430 | | - |
---|
431 | | - if (!mctx && next_sp - sp >= sizeof(struct rt_signal_frame_32) && |
---|
432 | | - is_rt_sigreturn_32_address(next_ip, sp) && |
---|
433 | | - sane_rt_signal_32_frame(sp)) { |
---|
434 | | - rt_sf = (struct rt_signal_frame_32 __user *) (unsigned long) sp; |
---|
435 | | - mctx = &rt_sf->uc.uc_mcontext; |
---|
436 | | - } |
---|
437 | | - |
---|
438 | | - if (!mctx) |
---|
439 | | - return NULL; |
---|
440 | | - return mctx->mc_gregs; |
---|
441 | | -} |
---|
442 | | - |
---|
443 | | -static void perf_callchain_user_32(struct perf_callchain_entry_ctx *entry, |
---|
444 | | - struct pt_regs *regs) |
---|
445 | | -{ |
---|
446 | | - unsigned int sp, next_sp; |
---|
447 | | - unsigned int next_ip; |
---|
448 | | - unsigned int lr; |
---|
449 | | - long level = 0; |
---|
450 | | - unsigned int __user *fp, *uregs; |
---|
451 | | - |
---|
452 | | - next_ip = perf_instruction_pointer(regs); |
---|
453 | | - lr = regs->link; |
---|
454 | | - sp = regs->gpr[1]; |
---|
455 | | - perf_callchain_store(entry, next_ip); |
---|
456 | | - |
---|
457 | | - while (entry->nr < entry->max_stack) { |
---|
458 | | - fp = (unsigned int __user *) (unsigned long) sp; |
---|
459 | | - if (!valid_user_sp(sp, 0) || read_user_stack_32(fp, &next_sp)) |
---|
460 | | - return; |
---|
461 | | - if (level > 0 && read_user_stack_32(&fp[1], &next_ip)) |
---|
462 | | - return; |
---|
463 | | - |
---|
464 | | - uregs = signal_frame_32_regs(sp, next_sp, next_ip); |
---|
465 | | - if (!uregs && level <= 1) |
---|
466 | | - uregs = signal_frame_32_regs(sp, next_sp, lr); |
---|
467 | | - if (uregs) { |
---|
468 | | - /* |
---|
469 | | - * This looks like an signal frame, so restart |
---|
470 | | - * the stack trace with the values in it. |
---|
471 | | - */ |
---|
472 | | - if (read_user_stack_32(&uregs[PT_NIP], &next_ip) || |
---|
473 | | - read_user_stack_32(&uregs[PT_LNK], &lr) || |
---|
474 | | - read_user_stack_32(&uregs[PT_R1], &sp)) |
---|
475 | | - return; |
---|
476 | | - level = 0; |
---|
477 | | - perf_callchain_store_context(entry, PERF_CONTEXT_USER); |
---|
478 | | - perf_callchain_store(entry, next_ip); |
---|
479 | | - continue; |
---|
480 | | - } |
---|
481 | | - |
---|
482 | | - if (level == 0) |
---|
483 | | - next_ip = lr; |
---|
484 | | - perf_callchain_store(entry, next_ip); |
---|
485 | | - ++level; |
---|
486 | | - sp = next_sp; |
---|
487 | | - } |
---|
488 | | -} |
---|
489 | | - |
---|
490 | 103 | void |
---|
491 | 104 | perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) |
---|
492 | 105 | { |
---|
493 | | - if (current_is_64bit()) |
---|
| 106 | + if (!is_32bit_task()) |
---|
494 | 107 | perf_callchain_user_64(entry, regs); |
---|
495 | 108 | else |
---|
496 | 109 | perf_callchain_user_32(entry, regs); |
---|