forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/mm/kasan/kasan.h
....@@ -3,28 +3,71 @@
33 #define __MM_KASAN_KASAN_H
44 
55 #include <linux/kasan.h>
6
+#include <linux/kasan-tags.h>
7
+#include <linux/kfence.h>
68 #include <linux/stackdepot.h>
79 
8
-#define KASAN_SHADOW_SCALE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT)
9
-#define KASAN_SHADOW_MASK       (KASAN_SHADOW_SCALE_SIZE - 1)
10
+#ifdef CONFIG_KASAN_HW_TAGS
1011 
11
-#define KASAN_TAG_KERNEL	0xFF /* native kernel pointers tag */
12
-#define KASAN_TAG_INVALID	0xFE /* inaccessible memory tag */
13
-#define KASAN_TAG_MAX		0xFD /* maximum value for random tags */
12
+#include <linux/static_key.h>
13
+#include "../slab.h"
14
+
15
+DECLARE_STATIC_KEY_FALSE(kasan_flag_stacktrace);
16
+extern bool kasan_flag_async __ro_after_init;
17
+
18
+static inline bool kasan_stack_collection_enabled(void)
19
+{
20
+	return static_branch_unlikely(&kasan_flag_stacktrace);
21
+}
22
+
23
+static inline bool kasan_async_mode_enabled(void)
24
+{
25
+	return kasan_flag_async;
26
+}
27
+#else
28
+
29
+static inline bool kasan_stack_collection_enabled(void)
30
+{
31
+	return true;
32
+}
33
+
34
+static inline bool kasan_async_mode_enabled(void)
35
+{
36
+	return false;
37
+}
38
+
39
+#endif
40
+
41
+extern bool kasan_flag_panic __ro_after_init;
42
+extern bool kasan_flag_async __ro_after_init;
43
+
44
+#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
45
+#define KASAN_GRANULE_SIZE	(1UL << KASAN_SHADOW_SCALE_SHIFT)
46
+#else
47
+#include <asm/mte-kasan.h>
48
+#define KASAN_GRANULE_SIZE	MTE_GRANULE_SIZE
49
+#endif
50
+
51
+#define KASAN_GRANULE_MASK	(KASAN_GRANULE_SIZE - 1)
52
+
53
+#define KASAN_MEMORY_PER_SHADOW_PAGE	(KASAN_GRANULE_SIZE << PAGE_SHIFT)
1454 
1555 #ifdef CONFIG_KASAN_GENERIC
1656 #define KASAN_FREE_PAGE         0xFF  /* page was freed */
1757 #define KASAN_PAGE_REDZONE      0xFE  /* redzone for kmalloc_large allocations */
1858 #define KASAN_KMALLOC_REDZONE   0xFC  /* redzone inside slub object */
1959 #define KASAN_KMALLOC_FREE      0xFB  /* object was freed (kmem_cache_free/kfree) */
60
+#define KASAN_KMALLOC_FREETRACK 0xFA  /* object was freed and has free track set */
2061 #else
2162 #define KASAN_FREE_PAGE         KASAN_TAG_INVALID
2263 #define KASAN_PAGE_REDZONE      KASAN_TAG_INVALID
2364 #define KASAN_KMALLOC_REDZONE   KASAN_TAG_INVALID
2465 #define KASAN_KMALLOC_FREE      KASAN_TAG_INVALID
66
+#define KASAN_KMALLOC_FREETRACK KASAN_TAG_INVALID
2567 #endif
2668 
27
-#define KASAN_GLOBAL_REDZONE    0xFA  /* redzone for global variable */
69
+#define KASAN_GLOBAL_REDZONE    0xF9  /* redzone for global variable */
70
+#define KASAN_VMALLOC_INVALID   0xF8  /* unallocated space in vmapped page */
2871 
2972 /*
3073  * Stack redzone shadow values
....@@ -43,10 +86,22 @@
4386 
4487 #define KASAN_ALLOCA_REDZONE_SIZE	32
4588 
89
+/*
90
+ * Stack frame marker (compiler ABI).
91
+ */
92
+#define KASAN_CURRENT_STACK_FRAME_MAGIC 0x41B58AB3
93
+
4694 /* Don't break randconfig/all*config builds */
4795 #ifndef KASAN_ABI_VERSION
4896 #define KASAN_ABI_VERSION 1
4997 #endif
98
+
99
+/* Metadata layout customization. */
100
+#define META_BYTES_PER_BLOCK 1
101
+#define META_BLOCKS_PER_ROW 16
102
+#define META_BYTES_PER_ROW (META_BLOCKS_PER_ROW * META_BYTES_PER_BLOCK)
103
+#define META_MEM_BYTES_PER_ROW (META_BYTES_PER_ROW * KASAN_GRANULE_SIZE)
104
+#define META_ROWS_AROUND_ADDR 2
50105 
51106 struct kasan_access_info {
52107 	const void *access_addr;
....@@ -90,25 +145,59 @@
90145 	depot_stack_handle_t stack;
91146 };
92147 
148
+#ifdef CONFIG_KASAN_SW_TAGS_IDENTIFY
149
+#define KASAN_NR_FREE_STACKS 5
150
+#else
151
+#define KASAN_NR_FREE_STACKS 1
152
+#endif
153
+
93154 struct kasan_alloc_meta {
94155 	struct kasan_track alloc_track;
95
-	struct kasan_track free_track;
156
+#ifdef CONFIG_KASAN_GENERIC
157
+	/*
158
+	 * The auxiliary stack is stored into struct kasan_alloc_meta.
159
+	 * The free stack is stored into struct kasan_free_meta.
160
+	 */
161
+	depot_stack_handle_t aux_stack[2];
162
+#else
163
+	struct kasan_track free_track[KASAN_NR_FREE_STACKS];
164
+#endif
165
+#ifdef CONFIG_KASAN_SW_TAGS_IDENTIFY
166
+	u8 free_pointer_tag[KASAN_NR_FREE_STACKS];
167
+	u8 free_track_idx;
168
+#endif
96169 };
97170 
98171 struct qlist_node {
99172 	struct qlist_node *next;
100173 };
174
+
175
+/*
176
+ * Generic mode either stores free meta in the object itself or in the redzone
177
+ * after the object. In the former case free meta offset is 0, in the latter
178
+ * case it has some sane value smaller than INT_MAX. Use INT_MAX as free meta
179
+ * offset when free meta isn't present.
180
+ */
181
+#define KASAN_NO_FREE_META INT_MAX
182
+
101183 struct kasan_free_meta {
184
+#ifdef CONFIG_KASAN_GENERIC
102185 	/* This field is used while the object is in the quarantine.
103186 	 * Otherwise it might be used for the allocator freelist.
104187 	 */
105188 	struct qlist_node quarantine_link;
189
+	struct kasan_track free_track;
190
+#endif
106191 };
107192 
108
-struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache,
109
-					const void *object);
110
-struct kasan_free_meta *get_free_info(struct kmem_cache *cache,
111
-					const void *object);
193
+struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,
194
+						const void *object);
195
+#ifdef CONFIG_KASAN_GENERIC
196
+struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,
197
+						const void *object);
198
+#endif
199
+
200
+#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
112201 
113202 static inline const void *kasan_shadow_to_mem(const void *shadow_addr)
114203 {
....@@ -116,50 +205,68 @@
116205 		<< KASAN_SHADOW_SCALE_SHIFT);
117206 }
118207 
119
-static inline bool addr_has_shadow(const void *addr)
208
+static inline bool addr_has_metadata(const void *addr)
120209 {
121210 	return (addr >= kasan_shadow_to_mem((void *)KASAN_SHADOW_START));
122211 }
123212 
124
-void kasan_poison_shadow(const void *address, size_t size, u8 value);
125
-
126
-void check_memory_region(unsigned long addr, size_t size, bool write,
213
+/**
214
+ * kasan_check_range - Check memory region, and report if invalid access.
215
+ * @addr: the accessed address
216
+ * @size: the accessed size
217
+ * @write: true if access is a write access
218
+ * @ret_ip: return address
219
+ * @return: true if access was valid, false if invalid
220
+ */
221
+bool kasan_check_range(unsigned long addr, size_t size, bool write,
127222 				unsigned long ret_ip);
128223 
129
-void *find_first_bad_addr(void *addr, size_t size);
130
-const char *get_bug_type(struct kasan_access_info *info);
224
+#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
131225 
132
-void kasan_report(unsigned long addr, size_t size,
226
+static inline bool addr_has_metadata(const void *addr)
227
+{
228
+	return (is_vmalloc_addr(addr) || virt_addr_valid(addr));
229
+}
230
+
231
+#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
232
+
233
+#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
234
+void kasan_print_tags(u8 addr_tag, const void *addr);
235
+#else
236
+static inline void kasan_print_tags(u8 addr_tag, const void *addr) { }
237
+#endif
238
+
239
+void *kasan_find_first_bad_addr(void *addr, size_t size);
240
+const char *kasan_get_bug_type(struct kasan_access_info *info);
241
+void kasan_metadata_fetch_row(char *buffer, void *row);
242
+
243
+#if defined(CONFIG_KASAN_GENERIC) && defined(CONFIG_KASAN_STACK)
244
+void kasan_print_address_stack_frame(const void *addr);
245
+#else
246
+static inline void kasan_print_address_stack_frame(const void *addr) { }
247
+#endif
248
+
249
+bool kasan_report(unsigned long addr, size_t size,
133250 		bool is_write, unsigned long ip);
134251 void kasan_report_invalid_free(void *object, unsigned long ip);
135252 
253
+struct page *kasan_addr_to_page(const void *addr);
254
+
255
+depot_stack_handle_t kasan_save_stack(gfp_t flags);
256
+void kasan_set_track(struct kasan_track *track, gfp_t flags);
257
+void kasan_set_free_info(struct kmem_cache *cache, void *object, u8 tag);
258
+struct kasan_track *kasan_get_free_track(struct kmem_cache *cache,
259
+				void *object, u8 tag);
260
+
136261 #if defined(CONFIG_KASAN_GENERIC) && \
137262 	(defined(CONFIG_SLAB) || defined(CONFIG_SLUB))
138
-void quarantine_put(struct kasan_free_meta *info, struct kmem_cache *cache);
139
-void quarantine_reduce(void);
140
-void quarantine_remove_cache(struct kmem_cache *cache);
263
+bool kasan_quarantine_put(struct kmem_cache *cache, void *object);
264
+void kasan_quarantine_reduce(void);
265
+void kasan_quarantine_remove_cache(struct kmem_cache *cache);
141266 #else
142
-static inline void quarantine_put(struct kasan_free_meta *info,
143
-				struct kmem_cache *cache) { }
144
-static inline void quarantine_reduce(void) { }
145
-static inline void quarantine_remove_cache(struct kmem_cache *cache) { }
146
-#endif
147
-
148
-#ifdef CONFIG_KASAN_SW_TAGS
149
-
150
-void print_tags(u8 addr_tag, const void *addr);
151
-
152
-u8 random_tag(void);
153
-
154
-#else
155
-
156
-static inline void print_tags(u8 addr_tag, const void *addr) { }
157
-
158
-static inline u8 random_tag(void)
159
-{
160
-	return 0;
161
-}
162
-
267
+static inline bool kasan_quarantine_put(struct kmem_cache *cache, void *object) { return false; }
268
+static inline void kasan_quarantine_reduce(void) { }
269
+static inline void kasan_quarantine_remove_cache(struct kmem_cache *cache) { }
163270 #endif
164271 
165272 #ifndef arch_kasan_set_tag
....@@ -168,16 +275,178 @@
168275 	return addr;
169276 }
170277 #endif
171
-#ifndef arch_kasan_reset_tag
172
-#define arch_kasan_reset_tag(addr)	((void *)(addr))
173
-#endif
174278 #ifndef arch_kasan_get_tag
175279 #define arch_kasan_get_tag(addr)	0
176280 #endif
177281 
178282 #define set_tag(addr, tag)	((void *)arch_kasan_set_tag((addr), (tag)))
179
-#define reset_tag(addr)		((void *)arch_kasan_reset_tag(addr))
180283 #define get_tag(addr)		arch_kasan_get_tag(addr)
284
+
285
+#ifdef CONFIG_KASAN_HW_TAGS
286
+
287
+#ifndef arch_enable_tagging_sync
288
+#define arch_enable_tagging_sync()
289
+#endif
290
+#ifndef arch_enable_tagging_async
291
+#define arch_enable_tagging_async()
292
+#endif
293
+#ifndef arch_set_tagging_report_once
294
+#define arch_set_tagging_report_once(state)
295
+#endif
296
+#ifndef arch_force_async_tag_fault
297
+#define arch_force_async_tag_fault()
298
+#endif
299
+#ifndef arch_get_random_tag
300
+#define arch_get_random_tag()	(0xFF)
301
+#endif
302
+#ifndef arch_get_mem_tag
303
+#define arch_get_mem_tag(addr)	(0xFF)
304
+#endif
305
+#ifndef arch_set_mem_tag_range
306
+#define arch_set_mem_tag_range(addr, size, tag, init) ((void *)(addr))
307
+#endif
308
+
309
+#define hw_enable_tagging_sync()		arch_enable_tagging_sync()
310
+#define hw_enable_tagging_async()		arch_enable_tagging_async()
311
+#define hw_set_tagging_report_once(state)	arch_set_tagging_report_once(state)
312
+#define hw_force_async_tag_fault()		arch_force_async_tag_fault()
313
+#define hw_get_random_tag()			arch_get_random_tag()
314
+#define hw_get_mem_tag(addr)			arch_get_mem_tag(addr)
315
+#define hw_set_mem_tag_range(addr, size, tag, init) \
316
+			arch_set_mem_tag_range((addr), (size), (tag), (init))
317
+
318
+#else /* CONFIG_KASAN_HW_TAGS */
319
+
320
+#define hw_enable_tagging_sync()
321
+#define hw_enable_tagging_async()
322
+#define hw_set_tagging_report_once(state)
323
+
324
+#endif /* CONFIG_KASAN_HW_TAGS */
325
+
326
+#if defined(CONFIG_KASAN_HW_TAGS) && IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
327
+
328
+void kasan_set_tagging_report_once(bool state);
329
+void kasan_enable_tagging_sync(void);
330
+void kasan_force_async_fault(void);
331
+
332
+#else /* CONFIG_KASAN_HW_TAGS || CONFIG_KASAN_KUNIT_TEST */
333
+
334
+static inline void kasan_set_tagging_report_once(bool state) { }
335
+static inline void kasan_enable_tagging_sync(void) { }
336
+static inline void kasan_force_async_fault(void) { }
337
+
338
+#endif /* CONFIG_KASAN_HW_TAGS || CONFIG_KASAN_KUNIT_TEST */
339
+
340
+#ifdef CONFIG_KASAN_SW_TAGS
341
+u8 kasan_random_tag(void);
342
+#elif defined(CONFIG_KASAN_HW_TAGS)
343
+static inline u8 kasan_random_tag(void) { return hw_get_random_tag(); }
344
+#else
345
+static inline u8 kasan_random_tag(void) { return 0; }
346
+#endif
347
+
348
+#ifdef CONFIG_KASAN_HW_TAGS
349
+
350
+static inline void kasan_poison(const void *addr, size_t size, u8 value, bool init)
351
+{
352
+	addr = kasan_reset_tag(addr);
353
+
354
+	/* Skip KFENCE memory if called explicitly outside of sl*b. */
355
+	if (is_kfence_address(addr))
356
+		return;
357
+
358
+	if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK))
359
+		return;
360
+	if (WARN_ON(size & KASAN_GRANULE_MASK))
361
+		return;
362
+
363
+	hw_set_mem_tag_range((void *)addr, size, value, init);
364
+}
365
+
366
+static inline void kasan_unpoison(const void *addr, size_t size, bool init)
367
+{
368
+	u8 tag = get_tag(addr);
369
+
370
+	addr = kasan_reset_tag(addr);
371
+
372
+	/* Skip KFENCE memory if called explicitly outside of sl*b. */
373
+	if (is_kfence_address(addr))
374
+		return;
375
+
376
+	if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK))
377
+		return;
378
+	/*
379
+	 * Explicitly initialize the memory with the precise object size to
380
+	 * avoid overwriting the SLAB redzone. This disables initialization in
381
+	 * the arch code and may thus lead to performance penalty. The penalty
382
+	 * is accepted since SLAB redzones aren't enabled in production builds.
383
+	 */
384
+	if (__slub_debug_enabled() &&
385
+	    init && ((unsigned long)size & KASAN_GRANULE_MASK)) {
386
+		init = false;
387
+		memzero_explicit((void *)addr, size);
388
+	}
389
+	size = round_up(size, KASAN_GRANULE_SIZE);
390
+
391
+	hw_set_mem_tag_range((void *)addr, size, tag, init);
392
+}
393
+
394
+static inline bool kasan_byte_accessible(const void *addr)
395
+{
396
+	u8 ptr_tag = get_tag(addr);
397
+	u8 mem_tag = hw_get_mem_tag((void *)addr);
398
+
399
+	return ptr_tag == KASAN_TAG_KERNEL || ptr_tag == mem_tag;
400
+}
401
+
402
+#else /* CONFIG_KASAN_HW_TAGS */
403
+
404
+/**
405
+ * kasan_poison - mark the memory range as unaccessible
406
+ * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE
407
+ * @size - range size, must be aligned to KASAN_GRANULE_SIZE
408
+ * @value - value that's written to metadata for the range
409
+ * @init - whether to initialize the memory range (only for hardware tag-based)
410
+ *
411
+ * The size gets aligned to KASAN_GRANULE_SIZE before marking the range.
412
+ */
413
+void kasan_poison(const void *addr, size_t size, u8 value, bool init);
414
+
415
+/**
416
+ * kasan_unpoison - mark the memory range as accessible
417
+ * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE
418
+ * @size - range size, can be unaligned
419
+ * @init - whether to initialize the memory range (only for hardware tag-based)
420
+ *
421
+ * For the tag-based modes, the @size gets aligned to KASAN_GRANULE_SIZE before
422
+ * marking the range.
423
+ * For the generic mode, the last granule of the memory range gets partially
424
+ * unpoisoned based on the @size.
425
+ */
426
+void kasan_unpoison(const void *addr, size_t size, bool init);
427
+
428
+bool kasan_byte_accessible(const void *addr);
429
+
430
+#endif /* CONFIG_KASAN_HW_TAGS */
431
+
432
+#ifdef CONFIG_KASAN_GENERIC
433
+
434
+/**
435
+ * kasan_poison_last_granule - mark the last granule of the memory range as
436
+ * unaccessible
437
+ * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE
438
+ * @size - range size
439
+ *
440
+ * This function is only available for the generic mode, as it's the only mode
441
+ * that has partially poisoned memory granules.
442
+ */
443
+void kasan_poison_last_granule(const void *address, size_t size);
444
+
445
+#else /* CONFIG_KASAN_GENERIC */
446
+
447
+static inline void kasan_poison_last_granule(const void *address, size_t size) { }
448
+
449
+#endif /* CONFIG_KASAN_GENERIC */
181450 
182451 /*
183452  * Exported functions for interfaces called from assembly or from generated
....@@ -186,8 +455,6 @@
186455 asmlinkage void kasan_unpoison_task_stack_below(const void *watermark);
187456 void __asan_register_globals(struct kasan_global *globals, size_t size);
188457 void __asan_unregister_globals(struct kasan_global *globals, size_t size);
189
-void __asan_loadN(unsigned long addr, size_t size);
190
-void __asan_storeN(unsigned long addr, size_t size);
191458 void __asan_handle_no_return(void);
192459 void __asan_alloca_poison(unsigned long addr, size_t size);
193460 void __asan_allocas_unpoison(const void *stack_top, const void *stack_bottom);
....@@ -202,6 +469,8 @@
202469 void __asan_store8(unsigned long addr);
203470 void __asan_load16(unsigned long addr);
204471 void __asan_store16(unsigned long addr);
472
+void __asan_loadN(unsigned long addr, size_t size);
473
+void __asan_storeN(unsigned long addr, size_t size);
205474 
206475 void __asan_load1_noabort(unsigned long addr);
207476 void __asan_store1_noabort(unsigned long addr);
....@@ -213,6 +482,21 @@
213482 void __asan_store8_noabort(unsigned long addr);
214483 void __asan_load16_noabort(unsigned long addr);
215484 void __asan_store16_noabort(unsigned long addr);
485
+void __asan_loadN_noabort(unsigned long addr, size_t size);
486
+void __asan_storeN_noabort(unsigned long addr, size_t size);
487
+
488
+void __asan_report_load1_noabort(unsigned long addr);
489
+void __asan_report_store1_noabort(unsigned long addr);
490
+void __asan_report_load2_noabort(unsigned long addr);
491
+void __asan_report_store2_noabort(unsigned long addr);
492
+void __asan_report_load4_noabort(unsigned long addr);
493
+void __asan_report_store4_noabort(unsigned long addr);
494
+void __asan_report_load8_noabort(unsigned long addr);
495
+void __asan_report_store8_noabort(unsigned long addr);
496
+void __asan_report_load16_noabort(unsigned long addr);
497
+void __asan_report_store16_noabort(unsigned long addr);
498
+void __asan_report_load_n_noabort(unsigned long addr, size_t size);
499
+void __asan_report_store_n_noabort(unsigned long addr, size_t size);
216500 
217501 void __asan_set_shadow_00(const void *addr, size_t size);
218502 void __asan_set_shadow_f1(const void *addr, size_t size);
....@@ -221,4 +505,19 @@
221505 void __asan_set_shadow_f5(const void *addr, size_t size);
222506 void __asan_set_shadow_f8(const void *addr, size_t size);
223507 
508
+void __hwasan_load1_noabort(unsigned long addr);
509
+void __hwasan_store1_noabort(unsigned long addr);
510
+void __hwasan_load2_noabort(unsigned long addr);
511
+void __hwasan_store2_noabort(unsigned long addr);
512
+void __hwasan_load4_noabort(unsigned long addr);
513
+void __hwasan_store4_noabort(unsigned long addr);
514
+void __hwasan_load8_noabort(unsigned long addr);
515
+void __hwasan_store8_noabort(unsigned long addr);
516
+void __hwasan_load16_noabort(unsigned long addr);
517
+void __hwasan_store16_noabort(unsigned long addr);
518
+void __hwasan_loadN_noabort(unsigned long addr, size_t size);
519
+void __hwasan_storeN_noabort(unsigned long addr, size_t size);
520
+
521
+void __hwasan_tag_memory(unsigned long addr, u8 tag, unsigned long size);
522
+
224523 #endif