mk-rootfs.sh

hc

2023-12-11 072de836f53be56a70cecf70b43ae43b7ce17376

 kernel/arch/arm64/kernel/sdei.c
..
..
@@ -2,13 +2,16 @@
2
2
 // Copyright (C) 2017 Arm Ltd.
3
3
 #define pr_fmt(fmt) "sdei: " fmt
4
4
 
5
+#include <linux/arm-smccc.h>
5
6
 #include <linux/arm_sdei.h>
6
7
 #include <linux/hardirq.h>
7
8
 #include <linux/irqflags.h>
8
9
 #include <linux/sched/task_stack.h>
10
+#include <linux/scs.h>
9
11
 #include <linux/uaccess.h>
10
12
 
11
13
 #include <asm/alternative.h>
14
+#include <asm/exception.h>
12
15
 #include <asm/kprobes.h>
13
16
 #include <asm/mmu.h>
14
17
 #include <asm/ptrace.h>
..
..
@@ -36,6 +39,14 @@
36
39
 DEFINE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
37
40
 #endif
38
41
 
42
+DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
43
+DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);
44
+
45
+#ifdef CONFIG_SHADOW_CALL_STACK
46
+DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
47
+DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);
48
+#endif
49
+
39
50
 static void _free_sdei_stack(unsigned long * __percpu *ptr, int cpu)
40
51
 {
41
52
 	unsigned long *p;
..
..
@@ -50,6 +61,9 @@
50
61
 static void free_sdei_stacks(void)
51
62
 {
52
63
 	int cpu;
64
+
65
+	if (!IS_ENABLED(CONFIG_VMAP_STACK))
66
+		return;
53
67
 
54
68
 	for_each_possible_cpu(cpu) {
55
69
 		_free_sdei_stack(&sdei_stack_normal_ptr, cpu);
..
..
@@ -74,6 +88,9 @@
74
88
 	int cpu;
75
89
 	int err = 0;
76
90
 
91
+	if (!IS_ENABLED(CONFIG_VMAP_STACK))
92
+		return 0;
93
+
77
94
 	for_each_possible_cpu(cpu) {
78
95
 		err = _init_sdei_stack(&sdei_stack_normal_ptr, cpu);
79
96
 		if (err)
..
..
@@ -89,24 +106,68 @@
89
106
 	return err;
90
107
 }
91
108
 
109
+static void _free_sdei_scs(unsigned long * __percpu *ptr, int cpu)
110
+{
111
+	void *s;
112
+
113
+	s = per_cpu(*ptr, cpu);
114
+	if (s) {
115
+		per_cpu(*ptr, cpu) = NULL;
116
+		scs_free(s);
117
+	}
118
+}
119
+
120
+static void free_sdei_scs(void)
121
+{
122
+	int cpu;
123
+
124
+	for_each_possible_cpu(cpu) {
125
+		_free_sdei_scs(&sdei_shadow_call_stack_normal_ptr, cpu);
126
+		_free_sdei_scs(&sdei_shadow_call_stack_critical_ptr, cpu);
127
+	}
128
+}
129
+
130
+static int _init_sdei_scs(unsigned long * __percpu *ptr, int cpu)
131
+{
132
+	void *s;
133
+
134
+	s = scs_alloc(cpu_to_node(cpu));
135
+	if (!s)
136
+		return -ENOMEM;
137
+	per_cpu(*ptr, cpu) = s;
138
+
139
+	return 0;
140
+}
141
+
142
+static int init_sdei_scs(void)
143
+{
144
+	int cpu;
145
+	int err = 0;
146
+
147
+	if (!IS_ENABLED(CONFIG_SHADOW_CALL_STACK))
148
+		return 0;
149
+
150
+	for_each_possible_cpu(cpu) {
151
+		err = _init_sdei_scs(&sdei_shadow_call_stack_normal_ptr, cpu);
152
+		if (err)
153
+			break;
154
+		err = _init_sdei_scs(&sdei_shadow_call_stack_critical_ptr, cpu);
155
+		if (err)
156
+			break;
157
+	}
158
+
159
+	if (err)
160
+		free_sdei_scs();
161
+
162
+	return err;
163
+}
164
+
92
165
 static bool on_sdei_normal_stack(unsigned long sp, struct stack_info *info)
93
166
 {
94
167
 	unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
95
168
 	unsigned long high = low + SDEI_STACK_SIZE;
96
169
 
97
-	if (!low)
98
-		return false;
99
-
100
-	if (sp < low || sp >= high)
101
-		return false;
102
-
103
-	if (info) {
104
-		info->low = low;
105
-		info->high = high;
106
-		info->type = STACK_TYPE_SDEI_NORMAL;
107
-	}
108
-
109
-	return true;
170
+	return on_stack(sp, low, high, STACK_TYPE_SDEI_NORMAL, info);
110
171
 }
111
172
 
112
173
 static bool on_sdei_critical_stack(unsigned long sp, struct stack_info *info)
..
..
@@ -114,19 +175,7 @@
114
175
 	unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
115
176
 	unsigned long high = low + SDEI_STACK_SIZE;
116
177
 
117
-	if (!low)
118
-		return false;
119
-
120
-	if (sp < low || sp >= high)
121
-		return false;
122
-
123
-	if (info) {
124
-		info->low = low;
125
-		info->high = high;
126
-		info->type = STACK_TYPE_SDEI_CRITICAL;
127
-	}
128
-
129
-	return true;
178
+	return on_stack(sp, low, high, STACK_TYPE_SDEI_CRITICAL, info);
130
179
 }
131
180
 
132
181
 bool _on_sdei_stack(unsigned long sp, struct stack_info *info)
..
..
@@ -153,15 +202,16 @@
153
202
 	 */
154
203
 	if (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) {
155
204
 		pr_err("Not supported on this hardware/boot configuration\n");
156
-		return 0;
205
+		goto out_err;
157
206
 	}
158
207
 
159
-	if (IS_ENABLED(CONFIG_VMAP_STACK)) {
160
-		if (init_sdei_stacks())
161
-			return 0;
162
-	}
208
+	if (init_sdei_stacks())
209
+		goto out_err;
163
210
 
164
-	sdei_exit_mode = (conduit == CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
211
+	if (init_sdei_scs())
212
+		goto out_err_free_stacks;
213
+
214
+	sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
165
215
 
166
216
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
167
217
 	if (arm64_kernel_unmapped_at_el0()) {
..
..
@@ -174,6 +224,10 @@
174
224
 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
175
225
 		return (unsigned long)__sdei_asm_handler;
176
226
 
227
+out_err_free_stacks:
228
+	free_sdei_stacks();
229
+out_err:
230
+	return 0;
177
231
 }
178
232
 
179
233
 /*
..
..
@@ -203,7 +257,7 @@
203
257
 
204
258
 	/*
205
259
 	 * We didn't take an exception to get here, set PAN. UAO will be cleared
206
-	 * by sdei_event_handler()s set_fs(USER_DS) call.
260
+	 * by sdei_event_handler()s force_uaccess_begin() call.
207
261
 	 */
208
262
 	__uaccess_enable_hw_pan();
209
263
 
..
..
@@ -246,26 +300,16 @@
246
300
 }
247
301
 
248
302
 
249
-asmlinkage __kprobes notrace unsigned long
303
+asmlinkage noinstr unsigned long
250
304
 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
251
305
 {
252
306
 	unsigned long ret;
253
-	bool do_nmi_exit = false;
254
307
 
255
-	/*
256
-	 * nmi_enter() deals with printk() re-entrance and use of RCU when
257
-	 * RCU believed this CPU was idle. Because critical events can
258
-	 * interrupt normal events, we may already be in_nmi().
259
-	 */
260
-	if (!in_nmi()) {
261
-		nmi_enter();
262
-		do_nmi_exit = true;
263
-	}
308
+	arm64_enter_nmi(regs);
264
309
 
265
310
 	ret = _sdei_handler(regs, arg);
266
311
 
267
-	if (do_nmi_exit)
268
-		nmi_exit();
312
+	arm64_exit_nmi(regs);
269
313
 
270
314
 	return ret;
271
315
 }