.. | .. |
---|
4 | 4 | #include <linux/percpu.h> |
---|
5 | 5 | #include <linux/kallsyms.h> |
---|
6 | 6 | #include <linux/kcore.h> |
---|
| 7 | +#include <linux/pgtable.h> |
---|
7 | 8 | |
---|
8 | 9 | #include <asm/cpu_entry_area.h> |
---|
9 | | -#include <asm/pgtable.h> |
---|
10 | 10 | #include <asm/fixmap.h> |
---|
11 | 11 | #include <asm/desc.h> |
---|
12 | 12 | |
---|
13 | 13 | static DEFINE_PER_CPU_PAGE_ALIGNED(struct entry_stack_page, entry_stack_storage); |
---|
14 | 14 | |
---|
15 | 15 | #ifdef CONFIG_X86_64 |
---|
16 | | -static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks |
---|
17 | | - [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); |
---|
18 | | -static DEFINE_PER_CPU(struct kcore_list, kcore_entry_trampoline); |
---|
| 16 | +static DEFINE_PER_CPU_PAGE_ALIGNED(struct exception_stacks, exception_stacks); |
---|
| 17 | +DEFINE_PER_CPU(struct cea_exception_stacks*, cea_exception_stacks); |
---|
19 | 18 | #endif |
---|
20 | 19 | |
---|
21 | | -struct cpu_entry_area *get_cpu_entry_area(int cpu) |
---|
| 20 | +#ifdef CONFIG_X86_32 |
---|
| 21 | +DECLARE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack); |
---|
| 22 | +#endif |
---|
| 23 | + |
---|
| 24 | +/* Is called from entry code, so must be noinstr */ |
---|
| 25 | +noinstr struct cpu_entry_area *get_cpu_entry_area(int cpu) |
---|
22 | 26 | { |
---|
23 | 27 | unsigned long va = CPU_ENTRY_AREA_PER_CPU + cpu * CPU_ENTRY_AREA_SIZE; |
---|
24 | 28 | BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE != 0); |
---|
.. | .. |
---|
53 | 57 | cea_set_pte(cea_vaddr, per_cpu_ptr_to_phys(ptr), prot); |
---|
54 | 58 | } |
---|
55 | 59 | |
---|
56 | | -static void percpu_setup_debug_store(int cpu) |
---|
| 60 | +static void __init percpu_setup_debug_store(unsigned int cpu) |
---|
57 | 61 | { |
---|
58 | 62 | #ifdef CONFIG_CPU_SUP_INTEL |
---|
59 | | - int npages; |
---|
| 63 | + unsigned int npages; |
---|
60 | 64 | void *cea; |
---|
61 | 65 | |
---|
62 | 66 | if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) |
---|
.. | .. |
---|
79 | 83 | #endif |
---|
80 | 84 | } |
---|
81 | 85 | |
---|
82 | | -/* Setup the fixmap mappings only once per-processor */ |
---|
83 | | -static void __init setup_cpu_entry_area(int cpu) |
---|
84 | | -{ |
---|
85 | 86 | #ifdef CONFIG_X86_64 |
---|
86 | | - extern char _entry_trampoline[]; |
---|
87 | 87 | |
---|
| 88 | +#define cea_map_stack(name) do { \ |
---|
| 89 | + npages = sizeof(estacks->name## _stack) / PAGE_SIZE; \ |
---|
| 90 | + cea_map_percpu_pages(cea->estacks.name## _stack, \ |
---|
| 91 | + estacks->name## _stack, npages, PAGE_KERNEL); \ |
---|
| 92 | + } while (0) |
---|
| 93 | + |
---|
| 94 | +static void __init percpu_setup_exception_stacks(unsigned int cpu) |
---|
| 95 | +{ |
---|
| 96 | + struct exception_stacks *estacks = per_cpu_ptr(&exception_stacks, cpu); |
---|
| 97 | + struct cpu_entry_area *cea = get_cpu_entry_area(cpu); |
---|
| 98 | + unsigned int npages; |
---|
| 99 | + |
---|
| 100 | + BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0); |
---|
| 101 | + |
---|
| 102 | + per_cpu(cea_exception_stacks, cpu) = &cea->estacks; |
---|
| 103 | + |
---|
| 104 | + /* |
---|
| 105 | + * The exceptions stack mappings in the per cpu area are protected |
---|
| 106 | + * by guard pages so each stack must be mapped separately. DB2 is |
---|
| 107 | + * not mapped; it just exists to catch triple nesting of #DB. |
---|
| 108 | + */ |
---|
| 109 | + cea_map_stack(DF); |
---|
| 110 | + cea_map_stack(NMI); |
---|
| 111 | + cea_map_stack(DB); |
---|
| 112 | + cea_map_stack(MCE); |
---|
| 113 | + |
---|
| 114 | + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { |
---|
| 115 | + if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) { |
---|
| 116 | + cea_map_stack(VC); |
---|
| 117 | + cea_map_stack(VC2); |
---|
| 118 | + } |
---|
| 119 | + } |
---|
| 120 | +} |
---|
| 121 | +#else |
---|
| 122 | +static inline void percpu_setup_exception_stacks(unsigned int cpu) |
---|
| 123 | +{ |
---|
| 124 | + struct cpu_entry_area *cea = get_cpu_entry_area(cpu); |
---|
| 125 | + |
---|
| 126 | + cea_map_percpu_pages(&cea->doublefault_stack, |
---|
| 127 | + &per_cpu(doublefault_stack, cpu), 1, PAGE_KERNEL); |
---|
| 128 | +} |
---|
| 129 | +#endif |
---|
| 130 | + |
---|
| 131 | +/* Setup the fixmap mappings only once per-processor */ |
---|
| 132 | +static void __init setup_cpu_entry_area(unsigned int cpu) |
---|
| 133 | +{ |
---|
| 134 | + struct cpu_entry_area *cea = get_cpu_entry_area(cpu); |
---|
| 135 | +#ifdef CONFIG_X86_64 |
---|
88 | 136 | /* On 64-bit systems, we use a read-only fixmap GDT and TSS. */ |
---|
89 | 137 | pgprot_t gdt_prot = PAGE_KERNEL_RO; |
---|
90 | 138 | pgprot_t tss_prot = PAGE_KERNEL_RO; |
---|
.. | .. |
---|
104 | 152 | pgprot_t tss_prot = PAGE_KERNEL; |
---|
105 | 153 | #endif |
---|
106 | 154 | |
---|
107 | | - cea_set_pte(&get_cpu_entry_area(cpu)->gdt, get_cpu_gdt_paddr(cpu), |
---|
108 | | - gdt_prot); |
---|
| 155 | + cea_set_pte(&cea->gdt, get_cpu_gdt_paddr(cpu), gdt_prot); |
---|
109 | 156 | |
---|
110 | | - cea_map_percpu_pages(&get_cpu_entry_area(cpu)->entry_stack_page, |
---|
| 157 | + cea_map_percpu_pages(&cea->entry_stack_page, |
---|
111 | 158 | per_cpu_ptr(&entry_stack_storage, cpu), 1, |
---|
112 | 159 | PAGE_KERNEL); |
---|
113 | 160 | |
---|
.. | .. |
---|
131 | 178 | BUILD_BUG_ON((offsetof(struct tss_struct, x86_tss) ^ |
---|
132 | 179 | offsetofend(struct tss_struct, x86_tss)) & PAGE_MASK); |
---|
133 | 180 | BUILD_BUG_ON(sizeof(struct tss_struct) % PAGE_SIZE != 0); |
---|
134 | | - cea_map_percpu_pages(&get_cpu_entry_area(cpu)->tss, |
---|
135 | | - &per_cpu(cpu_tss_rw, cpu), |
---|
| 181 | + /* |
---|
| 182 | + * VMX changes the host TR limit to 0x67 after a VM exit. This is |
---|
| 183 | + * okay, since 0x67 covers the size of struct x86_hw_tss. Make sure |
---|
| 184 | + * that this is correct. |
---|
| 185 | + */ |
---|
| 186 | + BUILD_BUG_ON(offsetof(struct tss_struct, x86_tss) != 0); |
---|
| 187 | + BUILD_BUG_ON(sizeof(struct x86_hw_tss) != 0x68); |
---|
| 188 | + |
---|
| 189 | + cea_map_percpu_pages(&cea->tss, &per_cpu(cpu_tss_rw, cpu), |
---|
136 | 190 | sizeof(struct tss_struct) / PAGE_SIZE, tss_prot); |
---|
137 | 191 | |
---|
138 | 192 | #ifdef CONFIG_X86_32 |
---|
139 | | - per_cpu(cpu_entry_area, cpu) = get_cpu_entry_area(cpu); |
---|
| 193 | + per_cpu(cpu_entry_area, cpu) = cea; |
---|
140 | 194 | #endif |
---|
141 | 195 | |
---|
142 | | -#ifdef CONFIG_X86_64 |
---|
143 | | - BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0); |
---|
144 | | - BUILD_BUG_ON(sizeof(exception_stacks) != |
---|
145 | | - sizeof(((struct cpu_entry_area *)0)->exception_stacks)); |
---|
146 | | - cea_map_percpu_pages(&get_cpu_entry_area(cpu)->exception_stacks, |
---|
147 | | - &per_cpu(exception_stacks, cpu), |
---|
148 | | - sizeof(exception_stacks) / PAGE_SIZE, PAGE_KERNEL); |
---|
| 196 | + percpu_setup_exception_stacks(cpu); |
---|
149 | 197 | |
---|
150 | | - cea_set_pte(&get_cpu_entry_area(cpu)->entry_trampoline, |
---|
151 | | - __pa_symbol(_entry_trampoline), PAGE_KERNEL_RX); |
---|
152 | | - /* |
---|
153 | | - * The cpu_entry_area alias addresses are not in the kernel binary |
---|
154 | | - * so they do not show up in /proc/kcore normally. This adds entries |
---|
155 | | - * for them manually. |
---|
156 | | - */ |
---|
157 | | - kclist_add_remap(&per_cpu(kcore_entry_trampoline, cpu), |
---|
158 | | - _entry_trampoline, |
---|
159 | | - &get_cpu_entry_area(cpu)->entry_trampoline, PAGE_SIZE); |
---|
160 | | -#endif |
---|
161 | 198 | percpu_setup_debug_store(cpu); |
---|
162 | 199 | } |
---|
163 | | - |
---|
164 | | -#ifdef CONFIG_X86_64 |
---|
165 | | -int arch_get_kallsym(unsigned int symnum, unsigned long *value, char *type, |
---|
166 | | - char *name) |
---|
167 | | -{ |
---|
168 | | - unsigned int cpu, ncpu = 0; |
---|
169 | | - |
---|
170 | | - if (symnum >= num_possible_cpus()) |
---|
171 | | - return -EINVAL; |
---|
172 | | - |
---|
173 | | - for_each_possible_cpu(cpu) { |
---|
174 | | - if (ncpu++ >= symnum) |
---|
175 | | - break; |
---|
176 | | - } |
---|
177 | | - |
---|
178 | | - *value = (unsigned long)&get_cpu_entry_area(cpu)->entry_trampoline; |
---|
179 | | - *type = 't'; |
---|
180 | | - strlcpy(name, "__entry_SYSCALL_64_trampoline", KSYM_NAME_LEN); |
---|
181 | | - |
---|
182 | | - return 0; |
---|
183 | | -} |
---|
184 | | -#endif |
---|
185 | 200 | |
---|
186 | 201 | static __init void setup_cpu_entry_area_ptes(void) |
---|
187 | 202 | { |
---|
188 | 203 | #ifdef CONFIG_X86_32 |
---|
189 | 204 | unsigned long start, end; |
---|
190 | 205 | |
---|
191 | | - BUILD_BUG_ON(CPU_ENTRY_AREA_PAGES * PAGE_SIZE < CPU_ENTRY_AREA_MAP_SIZE); |
---|
| 206 | + /* The +1 is for the readonly IDT: */ |
---|
| 207 | + BUILD_BUG_ON((CPU_ENTRY_AREA_PAGES+1)*PAGE_SIZE != CPU_ENTRY_AREA_MAP_SIZE); |
---|
| 208 | + BUILD_BUG_ON(CPU_ENTRY_AREA_TOTAL_SIZE != CPU_ENTRY_AREA_MAP_SIZE); |
---|
192 | 209 | BUG_ON(CPU_ENTRY_AREA_BASE & ~PMD_MASK); |
---|
193 | 210 | |
---|
194 | 211 | start = CPU_ENTRY_AREA_BASE; |
---|