.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* |
---|
2 | 3 | * Generic stack depot for storing stack traces. |
---|
3 | 4 | * |
---|
.. | .. |
---|
16 | 17 | * Copyright (C) 2016 Google, Inc. |
---|
17 | 18 | * |
---|
18 | 19 | * Based on code by Dmitry Chernenkov. |
---|
19 | | - * |
---|
20 | | - * This program is free software; you can redistribute it and/or |
---|
21 | | - * modify it under the terms of the GNU General Public License |
---|
22 | | - * version 2 as published by the Free Software Foundation. |
---|
23 | | - * |
---|
24 | | - * This program is distributed in the hope that it will be useful, but |
---|
25 | | - * WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
26 | | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
---|
27 | | - * General Public License for more details. |
---|
28 | | - * |
---|
29 | 20 | */ |
---|
30 | 21 | |
---|
31 | 22 | #include <linux/gfp.h> |
---|
| 23 | +#include <linux/interrupt.h> |
---|
32 | 24 | #include <linux/jhash.h> |
---|
33 | 25 | #include <linux/kernel.h> |
---|
34 | 26 | #include <linux/mm.h> |
---|
.. | .. |
---|
39 | 31 | #include <linux/stackdepot.h> |
---|
40 | 32 | #include <linux/string.h> |
---|
41 | 33 | #include <linux/types.h> |
---|
| 34 | +#include <linux/memblock.h> |
---|
42 | 35 | |
---|
43 | 36 | #define DEPOT_STACK_BITS (sizeof(depot_stack_handle_t) * 8) |
---|
44 | 37 | |
---|
.. | .. |
---|
70 | 63 | u32 hash; /* Hash in the hastable */ |
---|
71 | 64 | u32 size; /* Number of frames in the stack */ |
---|
72 | 65 | union handle_parts handle; |
---|
73 | | - unsigned long entries[1]; /* Variable-sized array of entries. */ |
---|
| 66 | + unsigned long entries[]; /* Variable-sized array of entries. */ |
---|
74 | 67 | }; |
---|
75 | 68 | |
---|
76 | 69 | static void *stack_slabs[STACK_ALLOC_MAX_SLABS]; |
---|
.. | .. |
---|
101 | 94 | } |
---|
102 | 95 | /* |
---|
103 | 96 | * This smp_store_release pairs with smp_load_acquire() from |
---|
104 | | - * |next_slab_inited| above and in depot_save_stack(). |
---|
| 97 | + * |next_slab_inited| above and in stack_depot_save(). |
---|
105 | 98 | */ |
---|
106 | 99 | smp_store_release(&next_slab_inited, 1); |
---|
107 | 100 | } |
---|
.. | .. |
---|
112 | 105 | static struct stack_record *depot_alloc_stack(unsigned long *entries, int size, |
---|
113 | 106 | u32 hash, void **prealloc, gfp_t alloc_flags) |
---|
114 | 107 | { |
---|
115 | | - int required_size = offsetof(struct stack_record, entries) + |
---|
116 | | - sizeof(unsigned long) * size; |
---|
117 | 108 | struct stack_record *stack; |
---|
| 109 | + size_t required_size = struct_size(stack, entries, size); |
---|
118 | 110 | |
---|
119 | 111 | required_size = ALIGN(required_size, 1 << STACK_ALLOC_ALIGN); |
---|
120 | 112 | |
---|
.. | .. |
---|
127 | 119 | depot_offset = 0; |
---|
128 | 120 | /* |
---|
129 | 121 | * smp_store_release() here pairs with smp_load_acquire() from |
---|
130 | | - * |next_slab_inited| in depot_save_stack() and |
---|
| 122 | + * |next_slab_inited| in stack_depot_save() and |
---|
131 | 123 | * init_stack_slab(). |
---|
132 | 124 | */ |
---|
133 | 125 | if (depot_index + 1 < STACK_ALLOC_MAX_SLABS) |
---|
.. | .. |
---|
144 | 136 | stack->handle.slabindex = depot_index; |
---|
145 | 137 | stack->handle.offset = depot_offset >> STACK_ALLOC_ALIGN; |
---|
146 | 138 | stack->handle.valid = 1; |
---|
147 | | - memcpy(stack->entries, entries, size * sizeof(unsigned long)); |
---|
| 139 | + memcpy(stack->entries, entries, flex_array_size(stack, entries, size)); |
---|
148 | 140 | depot_offset += required_size; |
---|
149 | 141 | |
---|
150 | 142 | return stack; |
---|
151 | 143 | } |
---|
152 | 144 | |
---|
153 | | -#define STACK_HASH_ORDER 20 |
---|
154 | | -#define STACK_HASH_SIZE (1L << STACK_HASH_ORDER) |
---|
| 145 | +#define STACK_HASH_SIZE (1L << CONFIG_STACK_HASH_ORDER) |
---|
155 | 146 | #define STACK_HASH_MASK (STACK_HASH_SIZE - 1) |
---|
156 | 147 | #define STACK_HASH_SEED 0x9747b28c |
---|
157 | 148 | |
---|
158 | | -static struct stack_record *stack_table[STACK_HASH_SIZE] = { |
---|
159 | | - [0 ... STACK_HASH_SIZE - 1] = NULL |
---|
160 | | -}; |
---|
| 149 | +static bool stack_depot_disable; |
---|
| 150 | +static struct stack_record **stack_table; |
---|
| 151 | + |
---|
| 152 | +static int __init is_stack_depot_disabled(char *str) |
---|
| 153 | +{ |
---|
| 154 | + int ret; |
---|
| 155 | + |
---|
| 156 | + ret = kstrtobool(str, &stack_depot_disable); |
---|
| 157 | + if (!ret && stack_depot_disable) { |
---|
| 158 | + pr_info("Stack Depot is disabled\n"); |
---|
| 159 | + stack_table = NULL; |
---|
| 160 | + } |
---|
| 161 | + return 0; |
---|
| 162 | +} |
---|
| 163 | +early_param("stack_depot_disable", is_stack_depot_disabled); |
---|
| 164 | + |
---|
| 165 | +int __init stack_depot_init(void) |
---|
| 166 | +{ |
---|
| 167 | + if (!stack_depot_disable) { |
---|
| 168 | + size_t size = (STACK_HASH_SIZE * sizeof(struct stack_record *)); |
---|
| 169 | + int i; |
---|
| 170 | + |
---|
| 171 | + stack_table = memblock_alloc(size, size); |
---|
| 172 | + for (i = 0; i < STACK_HASH_SIZE; i++) |
---|
| 173 | + stack_table[i] = NULL; |
---|
| 174 | + } |
---|
| 175 | + return 0; |
---|
| 176 | +} |
---|
161 | 177 | |
---|
162 | 178 | /* Calculate hash for a stack */ |
---|
163 | 179 | static inline u32 hash_stack(unsigned long *entries, unsigned int size) |
---|
164 | 180 | { |
---|
165 | 181 | return jhash2((u32 *)entries, |
---|
166 | | - size * sizeof(unsigned long) / sizeof(u32), |
---|
167 | | - STACK_HASH_SEED); |
---|
| 182 | + array_size(size, sizeof(*entries)) / sizeof(u32), |
---|
| 183 | + STACK_HASH_SEED); |
---|
168 | 184 | } |
---|
169 | 185 | |
---|
170 | 186 | /* Use our own, non-instrumented version of memcmp(). |
---|
.. | .. |
---|
198 | 214 | return NULL; |
---|
199 | 215 | } |
---|
200 | 216 | |
---|
201 | | -void depot_fetch_stack(depot_stack_handle_t handle, struct stack_trace *trace) |
---|
| 217 | +/** |
---|
| 218 | + * stack_depot_fetch - Fetch stack entries from a depot |
---|
| 219 | + * |
---|
| 220 | + * @handle: Stack depot handle which was returned from |
---|
| 221 | + * stack_depot_save(). |
---|
| 222 | + * @entries: Pointer to store the entries address |
---|
| 223 | + * |
---|
| 224 | + * Return: The number of trace entries for this depot. |
---|
| 225 | + */ |
---|
| 226 | +unsigned int stack_depot_fetch(depot_stack_handle_t handle, |
---|
| 227 | + unsigned long **entries) |
---|
202 | 228 | { |
---|
203 | 229 | union handle_parts parts = { .handle = handle }; |
---|
204 | | - void *slab = stack_slabs[parts.slabindex]; |
---|
| 230 | + void *slab; |
---|
205 | 231 | size_t offset = parts.offset << STACK_ALLOC_ALIGN; |
---|
206 | | - struct stack_record *stack = slab + offset; |
---|
| 232 | + struct stack_record *stack; |
---|
207 | 233 | |
---|
208 | | - trace->nr_entries = trace->max_entries = stack->size; |
---|
209 | | - trace->entries = stack->entries; |
---|
210 | | - trace->skip = 0; |
---|
| 234 | + *entries = NULL; |
---|
| 235 | + if (parts.slabindex > depot_index) { |
---|
| 236 | + WARN(1, "slab index %d out of bounds (%d) for stack id %08x\n", |
---|
| 237 | + parts.slabindex, depot_index, handle); |
---|
| 238 | + return 0; |
---|
| 239 | + } |
---|
| 240 | + slab = stack_slabs[parts.slabindex]; |
---|
| 241 | + if (!slab) |
---|
| 242 | + return 0; |
---|
| 243 | + stack = slab + offset; |
---|
| 244 | + |
---|
| 245 | + *entries = stack->entries; |
---|
| 246 | + return stack->size; |
---|
211 | 247 | } |
---|
212 | | -EXPORT_SYMBOL_GPL(depot_fetch_stack); |
---|
| 248 | +EXPORT_SYMBOL_GPL(stack_depot_fetch); |
---|
213 | 249 | |
---|
214 | 250 | /** |
---|
215 | | - * depot_save_stack - save stack in a stack depot. |
---|
216 | | - * @trace - the stacktrace to save. |
---|
217 | | - * @alloc_flags - flags for allocating additional memory if required. |
---|
| 251 | + * stack_depot_save - Save a stack trace from an array |
---|
218 | 252 | * |
---|
219 | | - * Returns the handle of the stack struct stored in depot. |
---|
| 253 | + * @entries: Pointer to storage array |
---|
| 254 | + * @nr_entries: Size of the storage array |
---|
| 255 | + * @alloc_flags: Allocation gfp flags |
---|
| 256 | + * |
---|
| 257 | + * Return: The handle of the stack struct stored in depot |
---|
220 | 258 | */ |
---|
221 | | -depot_stack_handle_t depot_save_stack(struct stack_trace *trace, |
---|
222 | | - gfp_t alloc_flags) |
---|
| 259 | +depot_stack_handle_t stack_depot_save(unsigned long *entries, |
---|
| 260 | + unsigned int nr_entries, |
---|
| 261 | + gfp_t alloc_flags) |
---|
223 | 262 | { |
---|
224 | | - u32 hash; |
---|
225 | | - depot_stack_handle_t retval = 0; |
---|
226 | 263 | struct stack_record *found = NULL, **bucket; |
---|
227 | | - unsigned long flags; |
---|
| 264 | + depot_stack_handle_t retval = 0; |
---|
228 | 265 | struct page *page = NULL; |
---|
229 | 266 | void *prealloc = NULL; |
---|
| 267 | + unsigned long flags; |
---|
| 268 | + u32 hash; |
---|
230 | 269 | |
---|
231 | | - if (unlikely(trace->nr_entries == 0)) |
---|
| 270 | + if (unlikely(nr_entries == 0) || stack_depot_disable) |
---|
232 | 271 | goto fast_exit; |
---|
233 | 272 | |
---|
234 | | - hash = hash_stack(trace->entries, trace->nr_entries); |
---|
| 273 | + hash = hash_stack(entries, nr_entries); |
---|
235 | 274 | bucket = &stack_table[hash & STACK_HASH_MASK]; |
---|
236 | 275 | |
---|
237 | 276 | /* |
---|
.. | .. |
---|
239 | 278 | * The smp_load_acquire() here pairs with smp_store_release() to |
---|
240 | 279 | * |bucket| below. |
---|
241 | 280 | */ |
---|
242 | | - found = find_stack(smp_load_acquire(bucket), trace->entries, |
---|
243 | | - trace->nr_entries, hash); |
---|
| 281 | + found = find_stack(smp_load_acquire(bucket), entries, |
---|
| 282 | + nr_entries, hash); |
---|
244 | 283 | if (found) |
---|
245 | 284 | goto exit; |
---|
246 | 285 | |
---|
.. | .. |
---|
268 | 307 | |
---|
269 | 308 | raw_spin_lock_irqsave(&depot_lock, flags); |
---|
270 | 309 | |
---|
271 | | - found = find_stack(*bucket, trace->entries, trace->nr_entries, hash); |
---|
| 310 | + found = find_stack(*bucket, entries, nr_entries, hash); |
---|
272 | 311 | if (!found) { |
---|
273 | 312 | struct stack_record *new = |
---|
274 | | - depot_alloc_stack(trace->entries, trace->nr_entries, |
---|
| 313 | + depot_alloc_stack(entries, nr_entries, |
---|
275 | 314 | hash, &prealloc, alloc_flags); |
---|
276 | 315 | if (new) { |
---|
277 | 316 | new->next = *bucket; |
---|
.. | .. |
---|
301 | 340 | fast_exit: |
---|
302 | 341 | return retval; |
---|
303 | 342 | } |
---|
304 | | -EXPORT_SYMBOL_GPL(depot_save_stack); |
---|
| 343 | +EXPORT_SYMBOL_GPL(stack_depot_save); |
---|
| 344 | + |
---|
| 345 | +static inline int in_irqentry_text(unsigned long ptr) |
---|
| 346 | +{ |
---|
| 347 | + return (ptr >= (unsigned long)&__irqentry_text_start && |
---|
| 348 | + ptr < (unsigned long)&__irqentry_text_end) || |
---|
| 349 | + (ptr >= (unsigned long)&__softirqentry_text_start && |
---|
| 350 | + ptr < (unsigned long)&__softirqentry_text_end); |
---|
| 351 | +} |
---|
| 352 | + |
---|
| 353 | +unsigned int filter_irq_stacks(unsigned long *entries, |
---|
| 354 | + unsigned int nr_entries) |
---|
| 355 | +{ |
---|
| 356 | + unsigned int i; |
---|
| 357 | + |
---|
| 358 | + for (i = 0; i < nr_entries; i++) { |
---|
| 359 | + if (in_irqentry_text(entries[i])) { |
---|
| 360 | + /* Include the irqentry function into the stack. */ |
---|
| 361 | + return i + 1; |
---|
| 362 | + } |
---|
| 363 | + } |
---|
| 364 | + return nr_entries; |
---|
| 365 | +} |
---|
| 366 | +EXPORT_SYMBOL_GPL(filter_irq_stacks); |
---|