blame | history | patch | commit | commitdiff
hc
2023-12-06 08f87f769b595151be1afeff53e144f543faa614 
 kernel/arch/x86/kernel/idt.c
....@@ -1,21 +1,15 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Interrupt descriptor table related code
3
- *
4
- * This file is licensed under the GPL V2
54  */
65 #include <linux/interrupt.h>
76 
7
+#include <asm/cpu_entry_area.h>
8
+#include <asm/set_memory.h>
89 #include <asm/traps.h>
910 #include <asm/proto.h>
1011 #include <asm/desc.h>
1112 #include <asm/hw_irq.h>
12
-
13
-struct idt_data {
14
-	unsigned int	vector;
15
-	unsigned int	segment;
16
-	struct idt_bits	bits;
17
-	const void	*addr;
18
-};
1913 
2014 #define DPL0		0x0
2115 #define DPL3		0x3
....@@ -41,27 +35,34 @@
4135 #define SYSG(_vector, _addr)				\
4236 	G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS)
4337 
44
-/* Interrupt gate with interrupt stack */
38
+/*
39
+ * Interrupt gate with interrupt stack. The _ist index is the index in
40
+ * the tss.ist[] array, but for the descriptor it needs to start at 1.
41
+ */
4542 #define ISTG(_vector, _addr, _ist)			\
46
-	G(_vector, _addr, _ist, GATE_INTERRUPT, DPL0, __KERNEL_CS)
47
-
48
-/* System interrupt gate with interrupt stack */
49
-#define SISTG(_vector, _addr, _ist)			\
50
-	G(_vector, _addr, _ist, GATE_INTERRUPT, DPL3, __KERNEL_CS)
43
+	G(_vector, _addr, _ist + 1, GATE_INTERRUPT, DPL0, __KERNEL_CS)
5144 
5245 /* Task gate */
5346 #define TSKG(_vector, _gdt)				\
5447 	G(_vector, NULL, DEFAULT_STACK, GATE_TASK, DPL0, _gdt << 3)
48
+
49
+#define IDT_TABLE_SIZE		(IDT_ENTRIES * sizeof(gate_desc))
50
+
51
+static bool idt_setup_done __initdata;
5552 
5653 /*
5754  * Early traps running on the DEFAULT_STACK because the other interrupt
5855  * stacks work only after cpu_init().
5956  */
6057 static const __initconst struct idt_data early_idts[] = {
61
-	INTG(X86_TRAP_DB,		debug),
62
-	SYSG(X86_TRAP_BP,		int3),
58
+	INTG(X86_TRAP_DB,		asm_exc_debug),
59
+	SYSG(X86_TRAP_BP,		asm_exc_int3),
60
+
6361 #ifdef CONFIG_X86_32
64
-	INTG(X86_TRAP_PF,		page_fault),
62
+	/*
63
+	 * Not possible on 64-bit. See idt_setup_early_pf() for details.
64
+	 */
65
+	INTG(X86_TRAP_PF,		asm_exc_page_fault),
6566 #endif
6667 };
6768 
....@@ -72,33 +73,33 @@
7273  * set up TSS.
7374  */
7475 static const __initconst struct idt_data def_idts[] = {
75
-	INTG(X86_TRAP_DE,		divide_error),
76
-	INTG(X86_TRAP_NMI,		nmi),
77
-	INTG(X86_TRAP_BR,		bounds),
78
-	INTG(X86_TRAP_UD,		invalid_op),
79
-	INTG(X86_TRAP_NM,		device_not_available),
80
-	INTG(X86_TRAP_OLD_MF,		coprocessor_segment_overrun),
81
-	INTG(X86_TRAP_TS,		invalid_TSS),
82
-	INTG(X86_TRAP_NP,		segment_not_present),
83
-	INTG(X86_TRAP_SS,		stack_segment),
84
-	INTG(X86_TRAP_GP,		general_protection),
85
-	INTG(X86_TRAP_SPURIOUS,		spurious_interrupt_bug),
86
-	INTG(X86_TRAP_MF,		coprocessor_error),
87
-	INTG(X86_TRAP_AC,		alignment_check),
88
-	INTG(X86_TRAP_XF,		simd_coprocessor_error),
76
+	INTG(X86_TRAP_DE,		asm_exc_divide_error),
77
+	INTG(X86_TRAP_NMI,		asm_exc_nmi),
78
+	INTG(X86_TRAP_BR,		asm_exc_bounds),
79
+	INTG(X86_TRAP_UD,		asm_exc_invalid_op),
80
+	INTG(X86_TRAP_NM,		asm_exc_device_not_available),
81
+	INTG(X86_TRAP_OLD_MF,		asm_exc_coproc_segment_overrun),
82
+	INTG(X86_TRAP_TS,		asm_exc_invalid_tss),
83
+	INTG(X86_TRAP_NP,		asm_exc_segment_not_present),
84
+	INTG(X86_TRAP_SS,		asm_exc_stack_segment),
85
+	INTG(X86_TRAP_GP,		asm_exc_general_protection),
86
+	INTG(X86_TRAP_SPURIOUS,		asm_exc_spurious_interrupt_bug),
87
+	INTG(X86_TRAP_MF,		asm_exc_coprocessor_error),
88
+	INTG(X86_TRAP_AC,		asm_exc_alignment_check),
89
+	INTG(X86_TRAP_XF,		asm_exc_simd_coprocessor_error),
8990 
9091 #ifdef CONFIG_X86_32
9192 	TSKG(X86_TRAP_DF,		GDT_ENTRY_DOUBLEFAULT_TSS),
9293 #else
93
-	INTG(X86_TRAP_DF,		double_fault),
94
+	INTG(X86_TRAP_DF,		asm_exc_double_fault),
9495 #endif
95
-	INTG(X86_TRAP_DB,		debug),
96
+	INTG(X86_TRAP_DB,		asm_exc_debug),
9697 
9798 #ifdef CONFIG_X86_MCE
98
-	INTG(X86_TRAP_MC,		&machine_check),
99
+	INTG(X86_TRAP_MC,		asm_exc_machine_check),
99100 #endif
100101 
101
-	SYSG(X86_TRAP_OF,		overflow),
102
+	SYSG(X86_TRAP_OF,		asm_exc_overflow),
102103 #if defined(CONFIG_IA32_EMULATION)
103104 	SYSG(IA32_SYSCALL_VECTOR,	entry_INT80_compat),
104105 #elif defined(CONFIG_X86_32)
....@@ -111,112 +112,63 @@
111112  */
112113 static const __initconst struct idt_data apic_idts[] = {
113114 #ifdef CONFIG_SMP
114
-	INTG(RESCHEDULE_VECTOR,		reschedule_interrupt),
115
-	INTG(CALL_FUNCTION_VECTOR,	call_function_interrupt),
116
-	INTG(CALL_FUNCTION_SINGLE_VECTOR, call_function_single_interrupt),
117
-	INTG(IRQ_MOVE_CLEANUP_VECTOR,	irq_move_cleanup_interrupt),
118
-	INTG(REBOOT_VECTOR,		reboot_interrupt),
115
+	INTG(RESCHEDULE_VECTOR,			asm_sysvec_reschedule_ipi),
116
+	INTG(CALL_FUNCTION_VECTOR,		asm_sysvec_call_function),
117
+	INTG(CALL_FUNCTION_SINGLE_VECTOR,	asm_sysvec_call_function_single),
118
+	INTG(IRQ_MOVE_CLEANUP_VECTOR,		asm_sysvec_irq_move_cleanup),
119
+	INTG(REBOOT_VECTOR,			asm_sysvec_reboot),
119120 #endif
120121 
121122 #ifdef CONFIG_X86_THERMAL_VECTOR
122
-	INTG(THERMAL_APIC_VECTOR,	thermal_interrupt),
123
+	INTG(THERMAL_APIC_VECTOR,		asm_sysvec_thermal),
123124 #endif
124125 
125126 #ifdef CONFIG_X86_MCE_THRESHOLD
126
-	INTG(THRESHOLD_APIC_VECTOR,	threshold_interrupt),
127
+	INTG(THRESHOLD_APIC_VECTOR,		asm_sysvec_threshold),
127128 #endif
128129 
129130 #ifdef CONFIG_X86_MCE_AMD
130
-	INTG(DEFERRED_ERROR_VECTOR,	deferred_error_interrupt),
131
+	INTG(DEFERRED_ERROR_VECTOR,		asm_sysvec_deferred_error),
131132 #endif
132133 
133134 #ifdef CONFIG_X86_LOCAL_APIC
134
-	INTG(LOCAL_TIMER_VECTOR,	apic_timer_interrupt),
135
-	INTG(X86_PLATFORM_IPI_VECTOR,	x86_platform_ipi),
135
+	INTG(LOCAL_TIMER_VECTOR,		asm_sysvec_apic_timer_interrupt),
136
+	INTG(X86_PLATFORM_IPI_VECTOR,		asm_sysvec_x86_platform_ipi),
136137 # ifdef CONFIG_HAVE_KVM
137
-	INTG(POSTED_INTR_VECTOR,	kvm_posted_intr_ipi),
138
-	INTG(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi),
139
-	INTG(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi),
138
+	INTG(POSTED_INTR_VECTOR,		asm_sysvec_kvm_posted_intr_ipi),
139
+	INTG(POSTED_INTR_WAKEUP_VECTOR,		asm_sysvec_kvm_posted_intr_wakeup_ipi),
140
+	INTG(POSTED_INTR_NESTED_VECTOR,		asm_sysvec_kvm_posted_intr_nested_ipi),
140141 # endif
141142 # ifdef CONFIG_IRQ_WORK
142
-	INTG(IRQ_WORK_VECTOR,		irq_work_interrupt),
143
+	INTG(IRQ_WORK_VECTOR,			asm_sysvec_irq_work),
143144 # endif
144
-#ifdef CONFIG_X86_UV
145
-	INTG(UV_BAU_MESSAGE,		uv_bau_message_intr1),
146
-#endif
147
-	INTG(SPURIOUS_APIC_VECTOR,	spurious_interrupt),
148
-	INTG(ERROR_APIC_VECTOR,		error_interrupt),
145
+	INTG(SPURIOUS_APIC_VECTOR,		asm_sysvec_spurious_apic_interrupt),
146
+	INTG(ERROR_APIC_VECTOR,			asm_sysvec_error_interrupt),
149147 #endif
150148 };
151149 
152
-#ifdef CONFIG_X86_64
153
-/*
154
- * Early traps running on the DEFAULT_STACK because the other interrupt
155
- * stacks work only after cpu_init().
156
- */
157
-static const __initconst struct idt_data early_pf_idts[] = {
158
-	INTG(X86_TRAP_PF,		page_fault),
159
-};
150
+/* Must be page-aligned because the real IDT is used in the cpu entry area */
151
+static gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss;
160152 
161
-/*
162
- * Override for the debug_idt. Same as the default, but with interrupt
163
- * stack set to DEFAULT_STACK (0). Required for NMI trap handling.
164
- */
165
-static const __initconst struct idt_data dbg_idts[] = {
166
-	INTG(X86_TRAP_DB,	debug),
167
-};
168
-#endif
169
-
170
-/* Must be page-aligned because the real IDT is used in a fixmap. */
171
-gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss;
172
-
173
-struct desc_ptr idt_descr __ro_after_init = {
174
-	.size		= (IDT_ENTRIES * 2 * sizeof(unsigned long)) - 1,
153
+static struct desc_ptr idt_descr __ro_after_init = {
154
+	.size		= IDT_TABLE_SIZE - 1,
175155 	.address	= (unsigned long) idt_table,
176156 };
177157 
178
-#ifdef CONFIG_X86_64
179
-/* No need to be aligned, but done to keep all IDTs defined the same way. */
180
-gate_desc debug_idt_table[IDT_ENTRIES] __page_aligned_bss;
181
-
182
-/*
183
- * The exceptions which use Interrupt stacks. They are setup after
184
- * cpu_init() when the TSS has been initialized.
185
- */
186
-static const __initconst struct idt_data ist_idts[] = {
187
-	ISTG(X86_TRAP_DB,	debug,		DEBUG_STACK),
188
-	ISTG(X86_TRAP_NMI,	nmi,		NMI_STACK),
189
-	ISTG(X86_TRAP_DF,	double_fault,	DOUBLEFAULT_STACK),
190
-#ifdef CONFIG_X86_MCE
191
-	ISTG(X86_TRAP_MC,	&machine_check,	MCE_STACK),
192
-#endif
193
-};
194
-
195
-/*
196
- * Override for the debug_idt. Same as the default, but with interrupt
197
- * stack set to DEFAULT_STACK (0). Required for NMI trap handling.
198
- */
199
-const struct desc_ptr debug_idt_descr = {
200
-	.size		= IDT_ENTRIES * 16 - 1,
201
-	.address	= (unsigned long) debug_idt_table,
202
-};
203
-#endif
204
-
205
-static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d)
158
+void load_current_idt(void)
206159 {
207
-	unsigned long addr = (unsigned long) d->addr;
208
-
209
-	gate->offset_low	= (u16) addr;
210
-	gate->segment		= (u16) d->segment;
211
-	gate->bits		= d->bits;
212
-	gate->offset_middle	= (u16) (addr >> 16);
213
-#ifdef CONFIG_X86_64
214
-	gate->offset_high	= (u32) (addr >> 32);
215
-	gate->reserved		= 0;
216
-#endif
160
+	lockdep_assert_irqs_disabled();
161
+	load_idt(&idt_descr);
217162 }
218163 
219
-static void
164
+#ifdef CONFIG_X86_F00F_BUG
165
+bool idt_is_f00f_address(unsigned long address)
166
+{
167
+	return ((address - idt_descr.address) >> 3) == 6;
168
+}
169
+#endif
170
+
171
+static __init void
220172 idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sys)
221173 {
222174 	gate_desc desc;
....@@ -229,18 +181,11 @@
229181 	}
230182 }
231183 
232
-static void set_intr_gate(unsigned int n, const void *addr)
184
+static __init void set_intr_gate(unsigned int n, const void *addr)
233185 {
234186 	struct idt_data data;
235187 
236
-	BUG_ON(n > 0xFF);
237
-
238
-	memset(&data, 0, sizeof(data));
239
-	data.vector	= n;
240
-	data.addr	= addr;
241
-	data.segment	= __KERNEL_CS;
242
-	data.bits.type	= GATE_INTERRUPT;
243
-	data.bits.p	= 1;
188
+	init_idt_data(&data, n, addr);
244189 
245190 	idt_setup_from_table(idt_table, &data, 1, false);
246191 }
....@@ -268,6 +213,30 @@
268213 }
269214 
270215 #ifdef CONFIG_X86_64
216
+/*
217
+ * Early traps running on the DEFAULT_STACK because the other interrupt
218
+ * stacks work only after cpu_init().
219
+ */
220
+static const __initconst struct idt_data early_pf_idts[] = {
221
+	INTG(X86_TRAP_PF,		asm_exc_page_fault),
222
+};
223
+
224
+/*
225
+ * The exceptions which use Interrupt stacks. They are setup after
226
+ * cpu_init() when the TSS has been initialized.
227
+ */
228
+static const __initconst struct idt_data ist_idts[] = {
229
+	ISTG(X86_TRAP_DB,	asm_exc_debug,			IST_INDEX_DB),
230
+	ISTG(X86_TRAP_NMI,	asm_exc_nmi,			IST_INDEX_NMI),
231
+	ISTG(X86_TRAP_DF,	asm_exc_double_fault,		IST_INDEX_DF),
232
+#ifdef CONFIG_X86_MCE
233
+	ISTG(X86_TRAP_MC,	asm_exc_machine_check,		IST_INDEX_MCE),
234
+#endif
235
+#ifdef CONFIG_AMD_MEM_ENCRYPT
236
+	ISTG(X86_TRAP_VC,	asm_exc_vmm_communication,	IST_INDEX_VC),
237
+#endif
238
+};
239
+
271240 /**
272241  * idt_setup_early_pf - Initialize the idt table with early pagefault handler
273242  *
....@@ -275,8 +244,10 @@
275244  * cpu_init() is invoked and sets up TSS. The IST variant is installed
276245  * after that.
277246  *
278
- * FIXME: Why is 32bit and 64bit installing the PF handler at different
279
- * places in the early setup code?
247
+ * Note, that X86_64 cannot install the real #PF handler in
248
+ * idt_setup_early_traps() because the memory intialization needs the #PF
249
+ * handler from the early_idt_handler_array to initialize the early page
250
+ * tables.
280251  */
281252 void __init idt_setup_early_pf(void)
282253 {
....@@ -291,17 +262,20 @@
291262 {
292263 	idt_setup_from_table(idt_table, ist_idts, ARRAY_SIZE(ist_idts), true);
293264 }
294
-
295
-/**
296
- * idt_setup_debugidt_traps - Initialize the debug idt table with debug traps
297
- */
298
-void __init idt_setup_debugidt_traps(void)
299
-{
300
-	memcpy(&debug_idt_table, &idt_table, IDT_ENTRIES * 16);
301
-
302
-	idt_setup_from_table(debug_idt_table, dbg_idts, ARRAY_SIZE(dbg_idts), false);
303
-}
304265 #endif
266
+
267
+static void __init idt_map_in_cea(void)
268
+{
269
+	/*
270
+	 * Set the IDT descriptor to a fixed read-only location in the cpu
271
+	 * entry area, so that the "sidt" instruction will not leak the
272
+	 * location of the kernel, and to defend the IDT against arbitrary
273
+	 * memory write vulnerabilities.
274
+	 */
275
+	cea_set_pte(CPU_ENTRY_AREA_RO_IDT_VADDR, __pa_symbol(idt_table),
276
+		    PAGE_KERNEL_RO);
277
+	idt_descr.address = CPU_ENTRY_AREA_RO_IDT;
278
+}
305279 
306280 /**
307281  * idt_setup_apic_and_irq_gates - Setup APIC/SMP and normal interrupt gates
....@@ -329,6 +303,14 @@
329303 		set_intr_gate(i, entry);
330304 	}
331305 #endif
306
+	/* Map IDT into CPU entry area and reload it. */
307
+	idt_map_in_cea();
308
+	load_idt(&idt_descr);
309
+
310
+	/* Make the IDT table read only */
311
+	set_memory_ro((unsigned long)&idt_table, 1);
312
+
313
+	idt_setup_done = true;
332314 }
333315 
334316 /**
....@@ -358,16 +340,14 @@
358340 	load_idt(&idt);
359341 }
360342 
361
-void __init update_intr_gate(unsigned int n, const void *addr)
343
+void __init alloc_intr_gate(unsigned int n, const void *addr)
362344 {
363
-	if (WARN_ON_ONCE(!test_bit(n, system_vectors)))
345
+	if (WARN_ON(n < FIRST_SYSTEM_VECTOR))
364346 		return;
365
-	set_intr_gate(n, addr);
366
-}
367347 
368
-void alloc_intr_gate(unsigned int n, const void *addr)
369
-{
370
-	BUG_ON(n < FIRST_SYSTEM_VECTOR);
371
-	if (!test_and_set_bit(n, system_vectors))
348
+	if (WARN_ON(idt_setup_done))
349
+		return;
350
+
351
+	if (!WARN_ON(test_and_set_bit(n, system_vectors)))
372352 		set_intr_gate(n, addr);
373353 }