.. | .. |
---|
1 | 1 | // SPDX-License-Identifier: GPL-2.0 |
---|
2 | 2 | /* |
---|
3 | | - * This file contains some kasan initialization code. |
---|
| 3 | + * This file contains KASAN shadow initialization code. |
---|
4 | 4 | * |
---|
5 | 5 | * Copyright (c) 2015 Samsung Electronics Co., Ltd. |
---|
6 | 6 | * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com> |
---|
7 | | - * |
---|
8 | | - * This program is free software; you can redistribute it and/or modify |
---|
9 | | - * it under the terms of the GNU General Public License version 2 as |
---|
10 | | - * published by the Free Software Foundation. |
---|
11 | | - * |
---|
12 | 7 | */ |
---|
13 | 8 | |
---|
14 | | -#include <linux/bootmem.h> |
---|
| 9 | +#include <linux/memblock.h> |
---|
15 | 10 | #include <linux/init.h> |
---|
16 | 11 | #include <linux/kasan.h> |
---|
17 | 12 | #include <linux/kernel.h> |
---|
18 | | -#include <linux/memblock.h> |
---|
19 | 13 | #include <linux/mm.h> |
---|
20 | 14 | #include <linux/pfn.h> |
---|
21 | 15 | #include <linux/slab.h> |
---|
.. | .. |
---|
70 | 64 | return false; |
---|
71 | 65 | } |
---|
72 | 66 | #endif |
---|
73 | | -pte_t kasan_early_shadow_pte[PTRS_PER_PTE] __page_aligned_bss; |
---|
| 67 | +pte_t kasan_early_shadow_pte[PTRS_PER_PTE + PTE_HWTABLE_PTRS] |
---|
| 68 | + __page_aligned_bss; |
---|
74 | 69 | |
---|
75 | 70 | static inline bool kasan_pte_table(pmd_t pmd) |
---|
76 | 71 | { |
---|
.. | .. |
---|
84 | 79 | |
---|
85 | 80 | static __init void *early_alloc(size_t size, int node) |
---|
86 | 81 | { |
---|
87 | | - return memblock_virt_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS), |
---|
88 | | - BOOTMEM_ALLOC_ACCESSIBLE, node); |
---|
| 82 | + void *ptr = memblock_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS), |
---|
| 83 | + MEMBLOCK_ALLOC_ACCESSIBLE, node); |
---|
| 84 | + |
---|
| 85 | + if (!ptr) |
---|
| 86 | + panic("%s: Failed to allocate %zu bytes align=%zx nid=%d from=%llx\n", |
---|
| 87 | + __func__, size, size, node, (u64)__pa(MAX_DMA_ADDRESS)); |
---|
| 88 | + |
---|
| 89 | + return ptr; |
---|
89 | 90 | } |
---|
90 | 91 | |
---|
91 | 92 | static void __ref zero_pte_populate(pmd_t *pmd, unsigned long addr, |
---|
.. | .. |
---|
124 | 125 | pte_t *p; |
---|
125 | 126 | |
---|
126 | 127 | if (slab_is_available()) |
---|
127 | | - p = pte_alloc_one_kernel(&init_mm, addr); |
---|
| 128 | + p = pte_alloc_one_kernel(&init_mm); |
---|
128 | 129 | else |
---|
129 | 130 | p = early_alloc(PAGE_SIZE, NUMA_NO_NODE); |
---|
130 | 131 | if (!p) |
---|
.. | .. |
---|
245 | 246 | * 3,2 - level page tables where we don't have |
---|
246 | 247 | * puds,pmds, so pgd_populate(), pud_populate() |
---|
247 | 248 | * is noops. |
---|
248 | | - * |
---|
249 | | - * The ifndef is required to avoid build breakage. |
---|
250 | | - * |
---|
251 | | - * With 5level-fixup.h, pgd_populate() is not nop and |
---|
252 | | - * we reference kasan_early_shadow_p4d. It's not defined |
---|
253 | | - * unless 5-level paging enabled. |
---|
254 | | - * |
---|
255 | | - * The ifndef can be dropped once all KASAN-enabled |
---|
256 | | - * architectures will switch to pgtable-nop4d.h. |
---|
257 | 249 | */ |
---|
258 | | -#ifndef __ARCH_HAS_5LEVEL_HACK |
---|
259 | 250 | pgd_populate(&init_mm, pgd, |
---|
260 | 251 | lm_alias(kasan_early_shadow_p4d)); |
---|
261 | | -#endif |
---|
262 | 252 | p4d = p4d_offset(pgd, addr); |
---|
263 | 253 | p4d_populate(&init_mm, p4d, |
---|
264 | 254 | lm_alias(kasan_early_shadow_pud)); |
---|
.. | .. |
---|
455 | 445 | addr = (unsigned long)kasan_mem_to_shadow(start); |
---|
456 | 446 | end = addr + (size >> KASAN_SHADOW_SCALE_SHIFT); |
---|
457 | 447 | |
---|
458 | | - if (WARN_ON((unsigned long)start % |
---|
459 | | - (KASAN_SHADOW_SCALE_SIZE * PAGE_SIZE)) || |
---|
460 | | - WARN_ON(size % (KASAN_SHADOW_SCALE_SIZE * PAGE_SIZE))) |
---|
| 448 | + if (WARN_ON((unsigned long)start % KASAN_MEMORY_PER_SHADOW_PAGE) || |
---|
| 449 | + WARN_ON(size % KASAN_MEMORY_PER_SHADOW_PAGE)) |
---|
461 | 450 | return; |
---|
462 | 451 | |
---|
463 | 452 | for (; addr < end; addr = next) { |
---|
.. | .. |
---|
491 | 480 | shadow_start = kasan_mem_to_shadow(start); |
---|
492 | 481 | shadow_end = shadow_start + (size >> KASAN_SHADOW_SCALE_SHIFT); |
---|
493 | 482 | |
---|
494 | | - if (WARN_ON((unsigned long)start % |
---|
495 | | - (KASAN_SHADOW_SCALE_SIZE * PAGE_SIZE)) || |
---|
496 | | - WARN_ON(size % (KASAN_SHADOW_SCALE_SIZE * PAGE_SIZE))) |
---|
| 483 | + if (WARN_ON((unsigned long)start % KASAN_MEMORY_PER_SHADOW_PAGE) || |
---|
| 484 | + WARN_ON(size % KASAN_MEMORY_PER_SHADOW_PAGE)) |
---|
497 | 485 | return -EINVAL; |
---|
498 | 486 | |
---|
499 | 487 | ret = kasan_populate_early_shadow(shadow_start, shadow_end); |
---|