hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d
kernel/arch/x86/mm/cpu_entry_area.c
....@@ -4,21 +4,25 @@
44 #include <linux/percpu.h>
55 #include <linux/kallsyms.h>
66 #include <linux/kcore.h>
7
+#include <linux/pgtable.h>
78
89 #include <asm/cpu_entry_area.h>
9
-#include <asm/pgtable.h>
1010 #include <asm/fixmap.h>
1111 #include <asm/desc.h>
1212
1313 static DEFINE_PER_CPU_PAGE_ALIGNED(struct entry_stack_page, entry_stack_storage);
1414
1515 #ifdef CONFIG_X86_64
16
-static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
17
- [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
18
-static DEFINE_PER_CPU(struct kcore_list, kcore_entry_trampoline);
16
+static DEFINE_PER_CPU_PAGE_ALIGNED(struct exception_stacks, exception_stacks);
17
+DEFINE_PER_CPU(struct cea_exception_stacks*, cea_exception_stacks);
1918 #endif
2019
21
-struct cpu_entry_area *get_cpu_entry_area(int cpu)
20
+#ifdef CONFIG_X86_32
21
+DECLARE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack);
22
+#endif
23
+
24
+/* Is called from entry code, so must be noinstr */
25
+noinstr struct cpu_entry_area *get_cpu_entry_area(int cpu)
2226 {
2327 unsigned long va = CPU_ENTRY_AREA_PER_CPU + cpu * CPU_ENTRY_AREA_SIZE;
2428 BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE != 0);
....@@ -53,10 +57,10 @@
5357 cea_set_pte(cea_vaddr, per_cpu_ptr_to_phys(ptr), prot);
5458 }
5559
56
-static void percpu_setup_debug_store(int cpu)
60
+static void __init percpu_setup_debug_store(unsigned int cpu)
5761 {
5862 #ifdef CONFIG_CPU_SUP_INTEL
59
- int npages;
63
+ unsigned int npages;
6064 void *cea;
6165
6266 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
....@@ -79,12 +83,56 @@
7983 #endif
8084 }
8185
82
-/* Setup the fixmap mappings only once per-processor */
83
-static void __init setup_cpu_entry_area(int cpu)
84
-{
8586 #ifdef CONFIG_X86_64
86
- extern char _entry_trampoline[];
8787
88
+#define cea_map_stack(name) do { \
89
+ npages = sizeof(estacks->name## _stack) / PAGE_SIZE; \
90
+ cea_map_percpu_pages(cea->estacks.name## _stack, \
91
+ estacks->name## _stack, npages, PAGE_KERNEL); \
92
+ } while (0)
93
+
94
+static void __init percpu_setup_exception_stacks(unsigned int cpu)
95
+{
96
+ struct exception_stacks *estacks = per_cpu_ptr(&exception_stacks, cpu);
97
+ struct cpu_entry_area *cea = get_cpu_entry_area(cpu);
98
+ unsigned int npages;
99
+
100
+ BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0);
101
+
102
+ per_cpu(cea_exception_stacks, cpu) = &cea->estacks;
103
+
104
+ /*
105
+ * The exceptions stack mappings in the per cpu area are protected
106
+ * by guard pages so each stack must be mapped separately. DB2 is
107
+ * not mapped; it just exists to catch triple nesting of #DB.
108
+ */
109
+ cea_map_stack(DF);
110
+ cea_map_stack(NMI);
111
+ cea_map_stack(DB);
112
+ cea_map_stack(MCE);
113
+
114
+ if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {
115
+ if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) {
116
+ cea_map_stack(VC);
117
+ cea_map_stack(VC2);
118
+ }
119
+ }
120
+}
121
+#else
122
+static inline void percpu_setup_exception_stacks(unsigned int cpu)
123
+{
124
+ struct cpu_entry_area *cea = get_cpu_entry_area(cpu);
125
+
126
+ cea_map_percpu_pages(&cea->doublefault_stack,
127
+ &per_cpu(doublefault_stack, cpu), 1, PAGE_KERNEL);
128
+}
129
+#endif
130
+
131
+/* Setup the fixmap mappings only once per-processor */
132
+static void __init setup_cpu_entry_area(unsigned int cpu)
133
+{
134
+ struct cpu_entry_area *cea = get_cpu_entry_area(cpu);
135
+#ifdef CONFIG_X86_64
88136 /* On 64-bit systems, we use a read-only fixmap GDT and TSS. */
89137 pgprot_t gdt_prot = PAGE_KERNEL_RO;
90138 pgprot_t tss_prot = PAGE_KERNEL_RO;
....@@ -104,10 +152,9 @@
104152 pgprot_t tss_prot = PAGE_KERNEL;
105153 #endif
106154
107
- cea_set_pte(&get_cpu_entry_area(cpu)->gdt, get_cpu_gdt_paddr(cpu),
108
- gdt_prot);
155
+ cea_set_pte(&cea->gdt, get_cpu_gdt_paddr(cpu), gdt_prot);
109156
110
- cea_map_percpu_pages(&get_cpu_entry_area(cpu)->entry_stack_page,
157
+ cea_map_percpu_pages(&cea->entry_stack_page,
111158 per_cpu_ptr(&entry_stack_storage, cpu), 1,
112159 PAGE_KERNEL);
113160
....@@ -131,64 +178,34 @@
131178 BUILD_BUG_ON((offsetof(struct tss_struct, x86_tss) ^
132179 offsetofend(struct tss_struct, x86_tss)) & PAGE_MASK);
133180 BUILD_BUG_ON(sizeof(struct tss_struct) % PAGE_SIZE != 0);
134
- cea_map_percpu_pages(&get_cpu_entry_area(cpu)->tss,
135
- &per_cpu(cpu_tss_rw, cpu),
181
+ /*
182
+ * VMX changes the host TR limit to 0x67 after a VM exit. This is
183
+ * okay, since 0x67 covers the size of struct x86_hw_tss. Make sure
184
+ * that this is correct.
185
+ */
186
+ BUILD_BUG_ON(offsetof(struct tss_struct, x86_tss) != 0);
187
+ BUILD_BUG_ON(sizeof(struct x86_hw_tss) != 0x68);
188
+
189
+ cea_map_percpu_pages(&cea->tss, &per_cpu(cpu_tss_rw, cpu),
136190 sizeof(struct tss_struct) / PAGE_SIZE, tss_prot);
137191
138192 #ifdef CONFIG_X86_32
139
- per_cpu(cpu_entry_area, cpu) = get_cpu_entry_area(cpu);
193
+ per_cpu(cpu_entry_area, cpu) = cea;
140194 #endif
141195
142
-#ifdef CONFIG_X86_64
143
- BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0);
144
- BUILD_BUG_ON(sizeof(exception_stacks) !=
145
- sizeof(((struct cpu_entry_area *)0)->exception_stacks));
146
- cea_map_percpu_pages(&get_cpu_entry_area(cpu)->exception_stacks,
147
- &per_cpu(exception_stacks, cpu),
148
- sizeof(exception_stacks) / PAGE_SIZE, PAGE_KERNEL);
196
+ percpu_setup_exception_stacks(cpu);
149197
150
- cea_set_pte(&get_cpu_entry_area(cpu)->entry_trampoline,
151
- __pa_symbol(_entry_trampoline), PAGE_KERNEL_RX);
152
- /*
153
- * The cpu_entry_area alias addresses are not in the kernel binary
154
- * so they do not show up in /proc/kcore normally. This adds entries
155
- * for them manually.
156
- */
157
- kclist_add_remap(&per_cpu(kcore_entry_trampoline, cpu),
158
- _entry_trampoline,
159
- &get_cpu_entry_area(cpu)->entry_trampoline, PAGE_SIZE);
160
-#endif
161198 percpu_setup_debug_store(cpu);
162199 }
163
-
164
-#ifdef CONFIG_X86_64
165
-int arch_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
166
- char *name)
167
-{
168
- unsigned int cpu, ncpu = 0;
169
-
170
- if (symnum >= num_possible_cpus())
171
- return -EINVAL;
172
-
173
- for_each_possible_cpu(cpu) {
174
- if (ncpu++ >= symnum)
175
- break;
176
- }
177
-
178
- *value = (unsigned long)&get_cpu_entry_area(cpu)->entry_trampoline;
179
- *type = 't';
180
- strlcpy(name, "__entry_SYSCALL_64_trampoline", KSYM_NAME_LEN);
181
-
182
- return 0;
183
-}
184
-#endif
185200
186201 static __init void setup_cpu_entry_area_ptes(void)
187202 {
188203 #ifdef CONFIG_X86_32
189204 unsigned long start, end;
190205
191
- BUILD_BUG_ON(CPU_ENTRY_AREA_PAGES * PAGE_SIZE < CPU_ENTRY_AREA_MAP_SIZE);
206
+ /* The +1 is for the readonly IDT: */
207
+ BUILD_BUG_ON((CPU_ENTRY_AREA_PAGES+1)*PAGE_SIZE != CPU_ENTRY_AREA_MAP_SIZE);
208
+ BUILD_BUG_ON(CPU_ENTRY_AREA_TOTAL_SIZE != CPU_ENTRY_AREA_MAP_SIZE);
192209 BUG_ON(CPU_ENTRY_AREA_BASE & ~PMD_MASK);
193210
194211 start = CPU_ENTRY_AREA_BASE;