.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-or-later |
---|
1 | 2 | /* |
---|
2 | 3 | * PowerPC version |
---|
3 | 4 | * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) |
---|
.. | .. |
---|
11 | 12 | * |
---|
12 | 13 | * Dave Engebretsen <engebret@us.ibm.com> |
---|
13 | 14 | * Rework for PPC64 port. |
---|
14 | | - * |
---|
15 | | - * This program is free software; you can redistribute it and/or |
---|
16 | | - * modify it under the terms of the GNU General Public License |
---|
17 | | - * as published by the Free Software Foundation; either version |
---|
18 | | - * 2 of the License, or (at your option) any later version. |
---|
19 | | - * |
---|
20 | 15 | */ |
---|
21 | 16 | |
---|
22 | 17 | #undef DEBUG |
---|
23 | 18 | |
---|
24 | 19 | #include <linux/string.h> |
---|
| 20 | +#include <linux/pgtable.h> |
---|
25 | 21 | #include <asm/pgalloc.h> |
---|
26 | | -#include <asm/pgtable.h> |
---|
| 22 | +#include <asm/kup.h> |
---|
27 | 23 | |
---|
28 | | -static void pgd_ctor(void *addr) |
---|
| 24 | +phys_addr_t memstart_addr __ro_after_init = (phys_addr_t)~0ull; |
---|
| 25 | +EXPORT_SYMBOL_GPL(memstart_addr); |
---|
| 26 | +phys_addr_t kernstart_addr __ro_after_init; |
---|
| 27 | +EXPORT_SYMBOL_GPL(kernstart_addr); |
---|
| 28 | +unsigned long kernstart_virt_addr __ro_after_init = KERNELBASE; |
---|
| 29 | +EXPORT_SYMBOL_GPL(kernstart_virt_addr); |
---|
| 30 | + |
---|
| 31 | +static bool disable_kuep = !IS_ENABLED(CONFIG_PPC_KUEP); |
---|
| 32 | +static bool disable_kuap = !IS_ENABLED(CONFIG_PPC_KUAP); |
---|
| 33 | + |
---|
| 34 | +static int __init parse_nosmep(char *p) |
---|
29 | 35 | { |
---|
30 | | - memset(addr, 0, PGD_TABLE_SIZE); |
---|
| 36 | + disable_kuep = true; |
---|
| 37 | + pr_warn("Disabling Kernel Userspace Execution Prevention\n"); |
---|
| 38 | + return 0; |
---|
| 39 | +} |
---|
| 40 | +early_param("nosmep", parse_nosmep); |
---|
| 41 | + |
---|
| 42 | +static int __init parse_nosmap(char *p) |
---|
| 43 | +{ |
---|
| 44 | + disable_kuap = true; |
---|
| 45 | + pr_warn("Disabling Kernel Userspace Access Protection\n"); |
---|
| 46 | + return 0; |
---|
| 47 | +} |
---|
| 48 | +early_param("nosmap", parse_nosmap); |
---|
| 49 | + |
---|
| 50 | +void __ref setup_kup(void) |
---|
| 51 | +{ |
---|
| 52 | + setup_kuep(disable_kuep); |
---|
| 53 | + setup_kuap(disable_kuap); |
---|
31 | 54 | } |
---|
32 | 55 | |
---|
33 | | -static void pud_ctor(void *addr) |
---|
34 | | -{ |
---|
35 | | - memset(addr, 0, PUD_TABLE_SIZE); |
---|
| 56 | +#define CTOR(shift) static void ctor_##shift(void *addr) \ |
---|
| 57 | +{ \ |
---|
| 58 | + memset(addr, 0, sizeof(void *) << (shift)); \ |
---|
36 | 59 | } |
---|
37 | 60 | |
---|
38 | | -static void pmd_ctor(void *addr) |
---|
| 61 | +CTOR(0); CTOR(1); CTOR(2); CTOR(3); CTOR(4); CTOR(5); CTOR(6); CTOR(7); |
---|
| 62 | +CTOR(8); CTOR(9); CTOR(10); CTOR(11); CTOR(12); CTOR(13); CTOR(14); CTOR(15); |
---|
| 63 | + |
---|
| 64 | +static inline void (*ctor(int shift))(void *) |
---|
39 | 65 | { |
---|
40 | | - memset(addr, 0, PMD_TABLE_SIZE); |
---|
| 66 | + BUILD_BUG_ON(MAX_PGTABLE_INDEX_SIZE != 15); |
---|
| 67 | + |
---|
| 68 | + switch (shift) { |
---|
| 69 | + case 0: return ctor_0; |
---|
| 70 | + case 1: return ctor_1; |
---|
| 71 | + case 2: return ctor_2; |
---|
| 72 | + case 3: return ctor_3; |
---|
| 73 | + case 4: return ctor_4; |
---|
| 74 | + case 5: return ctor_5; |
---|
| 75 | + case 6: return ctor_6; |
---|
| 76 | + case 7: return ctor_7; |
---|
| 77 | + case 8: return ctor_8; |
---|
| 78 | + case 9: return ctor_9; |
---|
| 79 | + case 10: return ctor_10; |
---|
| 80 | + case 11: return ctor_11; |
---|
| 81 | + case 12: return ctor_12; |
---|
| 82 | + case 13: return ctor_13; |
---|
| 83 | + case 14: return ctor_14; |
---|
| 84 | + case 15: return ctor_15; |
---|
| 85 | + } |
---|
| 86 | + return NULL; |
---|
41 | 87 | } |
---|
42 | 88 | |
---|
43 | | -struct kmem_cache *pgtable_cache[MAX_PGTABLE_INDEX_SIZE]; |
---|
| 89 | +struct kmem_cache *pgtable_cache[MAX_PGTABLE_INDEX_SIZE + 1]; |
---|
44 | 90 | EXPORT_SYMBOL_GPL(pgtable_cache); /* used by kvm_hv module */ |
---|
45 | 91 | |
---|
46 | 92 | /* |
---|
.. | .. |
---|
50 | 96 | * everything else. Caches created by this function are used for all |
---|
51 | 97 | * the higher level pagetables, and for hugepage pagetables. |
---|
52 | 98 | */ |
---|
53 | | -void pgtable_cache_add(unsigned shift, void (*ctor)(void *)) |
---|
| 99 | +void pgtable_cache_add(unsigned int shift) |
---|
54 | 100 | { |
---|
55 | 101 | char *name; |
---|
56 | 102 | unsigned long table_size = sizeof(void *) << shift; |
---|
.. | .. |
---|
71 | 117 | * moment, gcc doesn't seem to recognize is_power_of_2 as a |
---|
72 | 118 | * constant expression, so so much for that. */ |
---|
73 | 119 | BUG_ON(!is_power_of_2(minalign)); |
---|
74 | | - BUG_ON((shift < 1) || (shift > MAX_PGTABLE_INDEX_SIZE)); |
---|
| 120 | + BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE); |
---|
75 | 121 | |
---|
76 | 122 | if (PGT_CACHE(shift)) |
---|
77 | 123 | return; /* Already have a cache of this size */ |
---|
78 | 124 | |
---|
79 | 125 | align = max_t(unsigned long, align, minalign); |
---|
80 | 126 | name = kasprintf(GFP_KERNEL, "pgtable-2^%d", shift); |
---|
81 | | - new = kmem_cache_create(name, table_size, align, 0, ctor); |
---|
| 127 | + new = kmem_cache_create(name, table_size, align, 0, ctor(shift)); |
---|
82 | 128 | if (!new) |
---|
83 | 129 | panic("Could not allocate pgtable cache for order %d", shift); |
---|
84 | 130 | |
---|
85 | 131 | kfree(name); |
---|
86 | | - pgtable_cache[shift - 1] = new; |
---|
| 132 | + pgtable_cache[shift] = new; |
---|
87 | 133 | |
---|
88 | 134 | pr_debug("Allocated pgtable cache for order %d\n", shift); |
---|
89 | 135 | } |
---|
.. | .. |
---|
91 | 137 | |
---|
92 | 138 | void pgtable_cache_init(void) |
---|
93 | 139 | { |
---|
94 | | - pgtable_cache_add(PGD_INDEX_SIZE, pgd_ctor); |
---|
| 140 | + pgtable_cache_add(PGD_INDEX_SIZE); |
---|
95 | 141 | |
---|
96 | | - if (PMD_CACHE_INDEX && !PGT_CACHE(PMD_CACHE_INDEX)) |
---|
97 | | - pgtable_cache_add(PMD_CACHE_INDEX, pmd_ctor); |
---|
| 142 | + if (PMD_CACHE_INDEX) |
---|
| 143 | + pgtable_cache_add(PMD_CACHE_INDEX); |
---|
98 | 144 | /* |
---|
99 | 145 | * In all current configs, when the PUD index exists it's the |
---|
100 | 146 | * same size as either the pgd or pmd index except with THP enabled |
---|
101 | 147 | * on book3s 64 |
---|
102 | 148 | */ |
---|
103 | | - if (PUD_CACHE_INDEX && !PGT_CACHE(PUD_CACHE_INDEX)) |
---|
104 | | - pgtable_cache_add(PUD_CACHE_INDEX, pud_ctor); |
---|
| 149 | + if (PUD_CACHE_INDEX) |
---|
| 150 | + pgtable_cache_add(PUD_CACHE_INDEX); |
---|
105 | 151 | } |
---|