forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-10-09 244b2c5ca8b14627e4a17755e5922221e121c771 
 kernel/arch/x86/mm/cpu_entry_area.c
....@@ -4,21 +4,25 @@
44 #include <linux/percpu.h>
55 #include <linux/kallsyms.h>
66 #include <linux/kcore.h>
7
+#include <linux/pgtable.h>
78 
89 #include <asm/cpu_entry_area.h>
9
-#include <asm/pgtable.h>
1010 #include <asm/fixmap.h>
1111 #include <asm/desc.h>
1212 
1313 static DEFINE_PER_CPU_PAGE_ALIGNED(struct entry_stack_page, entry_stack_storage);
1414 
1515 #ifdef CONFIG_X86_64
16
-static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
17
-	[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
18
-static DEFINE_PER_CPU(struct kcore_list, kcore_entry_trampoline);
16
+static DEFINE_PER_CPU_PAGE_ALIGNED(struct exception_stacks, exception_stacks);
17
+DEFINE_PER_CPU(struct cea_exception_stacks*, cea_exception_stacks);
1918 #endif
2019 
21
-struct cpu_entry_area *get_cpu_entry_area(int cpu)
20
+#ifdef CONFIG_X86_32
21
+DECLARE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack);
22
+#endif
23
+
24
+/* Is called from entry code, so must be noinstr */
25
+noinstr struct cpu_entry_area *get_cpu_entry_area(int cpu)
2226 {
2327 	unsigned long va = CPU_ENTRY_AREA_PER_CPU + cpu * CPU_ENTRY_AREA_SIZE;
2428 	BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE != 0);
....@@ -53,10 +57,10 @@
5357 		cea_set_pte(cea_vaddr, per_cpu_ptr_to_phys(ptr), prot);
5458 }
5559 
56
-static void percpu_setup_debug_store(int cpu)
60
+static void __init percpu_setup_debug_store(unsigned int cpu)
5761 {
5862 #ifdef CONFIG_CPU_SUP_INTEL
59
-	int npages;
63
+	unsigned int npages;
6064 	void *cea;
6165 
6266 	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
....@@ -79,12 +83,56 @@
7983 #endif
8084 }
8185 
82
-/* Setup the fixmap mappings only once per-processor */
83
-static void __init setup_cpu_entry_area(int cpu)
84
-{
8586 #ifdef CONFIG_X86_64
86
-	extern char _entry_trampoline[];
8787 
88
+#define cea_map_stack(name) do {					\
89
+	npages = sizeof(estacks->name## _stack) / PAGE_SIZE;		\
90
+	cea_map_percpu_pages(cea->estacks.name## _stack,		\
91
+			estacks->name## _stack, npages, PAGE_KERNEL);	\
92
+	} while (0)
93
+
94
+static void __init percpu_setup_exception_stacks(unsigned int cpu)
95
+{
96
+	struct exception_stacks *estacks = per_cpu_ptr(&exception_stacks, cpu);
97
+	struct cpu_entry_area *cea = get_cpu_entry_area(cpu);
98
+	unsigned int npages;
99
+
100
+	BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0);
101
+
102
+	per_cpu(cea_exception_stacks, cpu) = &cea->estacks;
103
+
104
+	/*
105
+	 * The exceptions stack mappings in the per cpu area are protected
106
+	 * by guard pages so each stack must be mapped separately. DB2 is
107
+	 * not mapped; it just exists to catch triple nesting of #DB.
108
+	 */
109
+	cea_map_stack(DF);
110
+	cea_map_stack(NMI);
111
+	cea_map_stack(DB);
112
+	cea_map_stack(MCE);
113
+
114
+	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {
115
+		if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) {
116
+			cea_map_stack(VC);
117
+			cea_map_stack(VC2);
118
+		}
119
+	}
120
+}
121
+#else
122
+static inline void percpu_setup_exception_stacks(unsigned int cpu)
123
+{
124
+	struct cpu_entry_area *cea = get_cpu_entry_area(cpu);
125
+
126
+	cea_map_percpu_pages(&cea->doublefault_stack,
127
+			     &per_cpu(doublefault_stack, cpu), 1, PAGE_KERNEL);
128
+}
129
+#endif
130
+
131
+/* Setup the fixmap mappings only once per-processor */
132
+static void __init setup_cpu_entry_area(unsigned int cpu)
133
+{
134
+	struct cpu_entry_area *cea = get_cpu_entry_area(cpu);
135
+#ifdef CONFIG_X86_64
88136 	/* On 64-bit systems, we use a read-only fixmap GDT and TSS. */
89137 	pgprot_t gdt_prot = PAGE_KERNEL_RO;
90138 	pgprot_t tss_prot = PAGE_KERNEL_RO;
....@@ -104,10 +152,9 @@
104152 	pgprot_t tss_prot = PAGE_KERNEL;
105153 #endif
106154 
107
-	cea_set_pte(&get_cpu_entry_area(cpu)->gdt, get_cpu_gdt_paddr(cpu),
108
-		    gdt_prot);
155
+	cea_set_pte(&cea->gdt, get_cpu_gdt_paddr(cpu), gdt_prot);
109156 
110
-	cea_map_percpu_pages(&get_cpu_entry_area(cpu)->entry_stack_page,
157
+	cea_map_percpu_pages(&cea->entry_stack_page,
111158 			     per_cpu_ptr(&entry_stack_storage, cpu), 1,
112159 			     PAGE_KERNEL);
113160 
....@@ -131,64 +178,34 @@
131178 	BUILD_BUG_ON((offsetof(struct tss_struct, x86_tss) ^
132179 		      offsetofend(struct tss_struct, x86_tss)) & PAGE_MASK);
133180 	BUILD_BUG_ON(sizeof(struct tss_struct) % PAGE_SIZE != 0);
134
-	cea_map_percpu_pages(&get_cpu_entry_area(cpu)->tss,
135
-			     &per_cpu(cpu_tss_rw, cpu),
181
+	/*
182
+	 * VMX changes the host TR limit to 0x67 after a VM exit. This is
183
+	 * okay, since 0x67 covers the size of struct x86_hw_tss. Make sure
184
+	 * that this is correct.
185
+	 */
186
+	BUILD_BUG_ON(offsetof(struct tss_struct, x86_tss) != 0);
187
+	BUILD_BUG_ON(sizeof(struct x86_hw_tss) != 0x68);
188
+
189
+	cea_map_percpu_pages(&cea->tss, &per_cpu(cpu_tss_rw, cpu),
136190 			     sizeof(struct tss_struct) / PAGE_SIZE, tss_prot);
137191 
138192 #ifdef CONFIG_X86_32
139
-	per_cpu(cpu_entry_area, cpu) = get_cpu_entry_area(cpu);
193
+	per_cpu(cpu_entry_area, cpu) = cea;
140194 #endif
141195 
142
-#ifdef CONFIG_X86_64
143
-	BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0);
144
-	BUILD_BUG_ON(sizeof(exception_stacks) !=
145
-		     sizeof(((struct cpu_entry_area *)0)->exception_stacks));
146
-	cea_map_percpu_pages(&get_cpu_entry_area(cpu)->exception_stacks,
147
-			     &per_cpu(exception_stacks, cpu),
148
-			     sizeof(exception_stacks) / PAGE_SIZE, PAGE_KERNEL);
196
+	percpu_setup_exception_stacks(cpu);
149197 
150
-	cea_set_pte(&get_cpu_entry_area(cpu)->entry_trampoline,
151
-		     __pa_symbol(_entry_trampoline), PAGE_KERNEL_RX);
152
-	/*
153
-	 * The cpu_entry_area alias addresses are not in the kernel binary
154
-	 * so they do not show up in /proc/kcore normally.  This adds entries
155
-	 * for them manually.
156
-	 */
157
-	kclist_add_remap(&per_cpu(kcore_entry_trampoline, cpu),
158
-			 _entry_trampoline,
159
-			 &get_cpu_entry_area(cpu)->entry_trampoline, PAGE_SIZE);
160
-#endif
161198 	percpu_setup_debug_store(cpu);
162199 }
163
-
164
-#ifdef CONFIG_X86_64
165
-int arch_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
166
-		     char *name)
167
-{
168
-	unsigned int cpu, ncpu = 0;
169
-
170
-	if (symnum >= num_possible_cpus())
171
-		return -EINVAL;
172
-
173
-	for_each_possible_cpu(cpu) {
174
-		if (ncpu++ >= symnum)
175
-			break;
176
-	}
177
-
178
-	*value = (unsigned long)&get_cpu_entry_area(cpu)->entry_trampoline;
179
-	*type = 't';
180
-	strlcpy(name, "__entry_SYSCALL_64_trampoline", KSYM_NAME_LEN);
181
-
182
-	return 0;
183
-}
184
-#endif
185200 
186201 static __init void setup_cpu_entry_area_ptes(void)
187202 {
188203 #ifdef CONFIG_X86_32
189204 	unsigned long start, end;
190205 
191
-	BUILD_BUG_ON(CPU_ENTRY_AREA_PAGES * PAGE_SIZE < CPU_ENTRY_AREA_MAP_SIZE);
206
+	/* The +1 is for the readonly IDT: */
207
+	BUILD_BUG_ON((CPU_ENTRY_AREA_PAGES+1)*PAGE_SIZE != CPU_ENTRY_AREA_MAP_SIZE);
208
+	BUILD_BUG_ON(CPU_ENTRY_AREA_TOTAL_SIZE != CPU_ENTRY_AREA_MAP_SIZE);
192209 	BUG_ON(CPU_ENTRY_AREA_BASE & ~PMD_MASK);
193210 
194211 	start = CPU_ENTRY_AREA_BASE;