forked from ~ljy/RK3588_XEN

blame | history | patch | commit | commitdiff
hc
2024-11-15 a46a1ad097419aeea7350987dd95230f50d90392 
 kernel/arch/arm64/kernel/fpsimd.c
....@@ -169,6 +169,42 @@
169169 	WARN_ON(busy);
170170 }
171171 
172
+static void __put_cpu_fpsimd_context(void)
173
+{
174
+	bool busy = __this_cpu_xchg(fpsimd_context_busy, false);
175
+
176
+	WARN_ON(!busy); /* No matching get_cpu_fpsimd_context()? */
177
+}
178
+
179
+#ifdef CONFIG_DOVETAIL
180
+
181
+#define get_cpu_fpsimd_context(__flags)			\
182
+	do {						\
183
+		(__flags) = hard_preempt_disable();	\
184
+		__get_cpu_fpsimd_context();		\
185
+	} while (0)
186
+
187
+#define put_cpu_fpsimd_context(__flags)		\
188
+	do {					\
189
+		__put_cpu_fpsimd_context();	\
190
+		hard_preempt_enable(__flags);	\
191
+	} while (0)
192
+
193
+void fpsimd_restore_current_oob(void)
194
+{
195
+	/*
196
+	 * Restore the fpsimd context for the current task as it
197
+	 * resumes from dovetail_context_switch(), which always happen
198
+	 * on the out-of-band stage. Skip this for kernel threads
199
+	 * which have no such context but always bear
200
+	 * TIF_FOREIGN_FPSTATE.
201
+	 */
202
+	if (current->mm)
203
+		fpsimd_restore_current_state();
204
+}
205
+
206
+#else
207
+
172208 /*
173209  * Claim ownership of the CPU FPSIMD context for use by the calling context.
174210  *
....@@ -178,19 +214,12 @@
178214  * The double-underscore version must only be called if you know the task
179215  * can't be preempted.
180216  */
181
-static void get_cpu_fpsimd_context(void)
182
-{
183
-	local_bh_disable();
184
-	__get_cpu_fpsimd_context();
185
-}
186
-
187
-static void __put_cpu_fpsimd_context(void)
188
-{
189
-	bool busy = __this_cpu_xchg(fpsimd_context_busy, false);
190
-
191
-	WARN_ON(!busy); /* No matching get_cpu_fpsimd_context()? */
192
-}
193
-
217
+#define get_cpu_fpsimd_context(__flags)			\
218
+	do {						\
219
+		preempt_disable();			\
220
+		__get_cpu_fpsimd_context();		\
221
+		(void)(__flags);			\
222
+	} while (0)
194223 /*
195224  * Release the CPU FPSIMD context.
196225  *
....@@ -198,12 +227,14 @@
198227  * previously called, with no call to put_cpu_fpsimd_context() in the
199228  * meantime.
200229  */
201
-static void put_cpu_fpsimd_context(void)
202
-{
203
-	__put_cpu_fpsimd_context();
204
-	local_bh_enable();
205
-}
230
+#define put_cpu_fpsimd_context(__flags)			\
231
+	do {						\
232
+		__put_cpu_fpsimd_context();		\
233
+		preempt_enable();			\
234
+		(void)(__flags);			\
235
+	} while (0)
206236 
237
+#endif	/* !CONFIG_DOVETAIL */
207238 static bool have_cpu_fpsimd_context(void)
208239 {
209240 	return !preemptible() && __this_cpu_read(fpsimd_context_busy);
....@@ -283,7 +314,7 @@
283314 static void task_fpsimd_load(void)
284315 {
285316 	WARN_ON(!system_supports_fpsimd());
286
-	WARN_ON(!have_cpu_fpsimd_context());
317
+	WARN_ON(!hard_irqs_disabled() && !have_cpu_fpsimd_context());
287318 
288319 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
289320 		sve_load_state(sve_pffr(&current->thread),
....@@ -297,14 +328,14 @@
297328  * Ensure FPSIMD/SVE storage in memory for the loaded context is up to
298329  * date with respect to the CPU registers.
299330  */
300
-static void fpsimd_save(void)
331
+static void __fpsimd_save(void)
301332 {
302333 	struct fpsimd_last_state_struct const *last =
303334 		this_cpu_ptr(&fpsimd_last_state);
304335 	/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
305336 
306337 	WARN_ON(!system_supports_fpsimd());
307
-	WARN_ON(!have_cpu_fpsimd_context());
338
+	WARN_ON(!hard_irqs_disabled() && !have_cpu_fpsimd_context());
308339 
309340 	if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
310341 		if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
....@@ -324,6 +355,15 @@
324355 		} else
325356 			fpsimd_save_state(last->st);
326357 	}
358
+}
359
+
360
+void fpsimd_save(void)
361
+{
362
+	unsigned long flags;
363
+
364
+	flags = hard_cond_local_irq_save();
365
+	__fpsimd_save();
366
+	hard_cond_local_irq_restore(flags);
327367 }
328368 
329369 /*
....@@ -444,7 +484,7 @@
444484  * task->thread.uw.fpsimd_state must be up to date before calling this
445485  * function.
446486  */
447
-static void fpsimd_to_sve(struct task_struct *task)
487
+static void _fpsimd_to_sve(struct task_struct *task)
448488 {
449489 	unsigned int vq;
450490 	void *sst = task->thread.sve_state;
....@@ -455,6 +495,15 @@
455495 
456496 	vq = sve_vq_from_vl(task->thread.sve_vl);
457497 	__fpsimd_to_sve(sst, fst, vq);
498
+}
499
+
500
+static void fpsimd_to_sve(struct task_struct *task)
501
+{
502
+	unsigned long flags;
503
+
504
+	flags = hard_cond_local_irq_save();
505
+	_fpsimd_to_sve(task);
506
+	hard_cond_local_irq_restore(flags);
458507 }
459508 
460509 /*
....@@ -475,15 +524,20 @@
475524 	struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
476525 	unsigned int i;
477526 	__uint128_t const *p;
527
+	unsigned long flags;
478528 
479529 	if (!system_supports_sve())
480530 		return;
531
+
532
+	flags = hard_cond_local_irq_save();
481533 
482534 	vq = sve_vq_from_vl(task->thread.sve_vl);
483535 	for (i = 0; i < SVE_NUM_ZREGS; ++i) {
484536 		p = (__uint128_t const *)ZREG(sst, vq, i);
485537 		fst->vregs[i] = arm64_le128_to_cpu(*p);
486538 	}
539
+
540
+	hard_cond_local_irq_restore(flags);
487541 }
488542 
489543 #ifdef CONFIG_ARM64_SVE
....@@ -584,6 +638,8 @@
584638 int sve_set_vector_length(struct task_struct *task,
585639 			  unsigned long vl, unsigned long flags)
586640 {
641
+	unsigned long irqflags = 0;
642
+
587643 	if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
588644 				     PR_SVE_SET_VL_ONEXEC))
589645 		return -EINVAL;
....@@ -621,9 +677,9 @@
621677 	 * non-SVE thread.
622678 	 */
623679 	if (task == current) {
624
-		get_cpu_fpsimd_context();
680
+		get_cpu_fpsimd_context(irqflags);
625681 
626
-		fpsimd_save();
682
+		__fpsimd_save();
627683 	}
628684 
629685 	fpsimd_flush_task_state(task);
....@@ -631,7 +687,7 @@
631687 		sve_to_fpsimd(task);
632688 
633689 	if (task == current)
634
-		put_cpu_fpsimd_context();
690
+		put_cpu_fpsimd_context(irqflags);
635691 
636692 	/*
637693 	 * Force reallocation of task SVE state to the correct size
....@@ -936,17 +992,21 @@
936992  */
937993 void do_sve_acc(unsigned int esr, struct pt_regs *regs)
938994 {
995
+	unsigned long flags;
996
+
997
+	mark_trap_entry(ARM64_TRAP_SVE, regs);
998
+
939999 	/* Even if we chose not to use SVE, the hardware could still trap: */
9401000 	if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
9411001 		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
942
-		return;
1002
+		goto out;
9431003 	}
9441004 
9451005 	sve_alloc(current);
9461006 
947
-	get_cpu_fpsimd_context();
1007
+	get_cpu_fpsimd_context(flags);
9481008 
949
-	fpsimd_save();
1009
+	__fpsimd_save();
9501010 
9511011 	/* Force ret_to_user to reload the registers: */
9521012 	fpsimd_flush_task_state(current);
....@@ -955,7 +1015,9 @@
9551015 	if (test_and_set_thread_flag(TIF_SVE))
9561016 		WARN_ON(1); /* SVE access shouldn't have trapped */
9571017 
958
-	put_cpu_fpsimd_context();
1018
+	put_cpu_fpsimd_context(flags);
1019
+out:
1020
+	mark_trap_exit(ARM64_TRAP_SVE, regs);
9591021 }
9601022 
9611023 /*
....@@ -974,6 +1036,9 @@
9741036 {
9751037 	unsigned int si_code = FPE_FLTUNK;
9761038 
1039
+	if (!mark_cond_trap_entry(ARM64_TRAP_FPE, regs))
1040
+		return;
1041
+
9771042 	if (esr & ESR_ELx_FP_EXC_TFV) {
9781043 		if (esr & FPEXC_IOF)
9791044 			si_code = FPE_FLTINV;
....@@ -990,19 +1055,24 @@
9901055 	send_sig_fault(SIGFPE, si_code,
9911056 		       (void __user *)instruction_pointer(regs),
9921057 		       current);
1058
+
1059
+	mark_trap_exit(ARM64_TRAP_FPE, regs);
9931060 }
9941061 
9951062 void fpsimd_thread_switch(struct task_struct *next)
9961063 {
9971064 	bool wrong_task, wrong_cpu;
1065
+	unsigned long flags;
9981066 
9991067 	if (!system_supports_fpsimd())
10001068 		return;
10011069 
1070
+	flags = hard_cond_local_irq_save();
1071
+
10021072 	__get_cpu_fpsimd_context();
10031073 
10041074 	/* Save unsaved fpsimd state, if any: */
1005
-	fpsimd_save();
1075
+	__fpsimd_save();
10061076 
10071077 	/*
10081078 	 * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
....@@ -1017,16 +1087,19 @@
10171087 			       wrong_task || wrong_cpu);
10181088 
10191089 	__put_cpu_fpsimd_context();
1090
+
1091
+	hard_cond_local_irq_restore(flags);
10201092 }
10211093 
10221094 void fpsimd_flush_thread(void)
10231095 {
10241096 	int vl, supported_vl;
1097
+	unsigned long flags;
10251098 
10261099 	if (!system_supports_fpsimd())
10271100 		return;
10281101 
1029
-	get_cpu_fpsimd_context();
1102
+	get_cpu_fpsimd_context(flags);
10301103 
10311104 	fpsimd_flush_task_state(current);
10321105 	memset(&current->thread.uw.fpsimd_state, 0,
....@@ -1067,7 +1140,7 @@
10671140 			current->thread.sve_vl_onexec = 0;
10681141 	}
10691142 
1070
-	put_cpu_fpsimd_context();
1143
+	put_cpu_fpsimd_context(flags);
10711144 }
10721145 
10731146 /*
....@@ -1076,12 +1149,14 @@
10761149  */
10771150 void fpsimd_preserve_current_state(void)
10781151 {
1152
+	unsigned long flags;
1153
+
10791154 	if (!system_supports_fpsimd())
10801155 		return;
10811156 
1082
-	get_cpu_fpsimd_context();
1083
-	fpsimd_save();
1084
-	put_cpu_fpsimd_context();
1157
+	get_cpu_fpsimd_context(flags);
1158
+	__fpsimd_save();
1159
+	put_cpu_fpsimd_context(flags);
10851160 }
10861161 
10871162 /*
....@@ -1123,7 +1198,7 @@
11231198 	}
11241199 }
11251200 
1126
-void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
1201
+static void __fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
11271202 			      unsigned int sve_vl)
11281203 {
11291204 	struct fpsimd_last_state_struct *last =
....@@ -1137,6 +1212,18 @@
11371212 	last->sve_vl = sve_vl;
11381213 }
11391214 
1215
+void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
1216
+			      unsigned int sve_vl)
1217
+{
1218
+	unsigned long flags;
1219
+
1220
+	WARN_ON(!in_softirq() && !irqs_disabled());
1221
+
1222
+	flags = hard_cond_local_irq_save();
1223
+	__fpsimd_bind_state_to_cpu(st, sve_state, sve_vl);
1224
+	hard_cond_local_irq_restore(flags);
1225
+}
1226
+
11401227 /*
11411228  * Load the userland FPSIMD state of 'current' from memory, but only if the
11421229  * FPSIMD state already held in the registers is /not/ the most recent FPSIMD
....@@ -1144,6 +1231,8 @@
11441231  */
11451232 void fpsimd_restore_current_state(void)
11461233 {
1234
+	unsigned long flags;
1235
+
11471236 	/*
11481237 	 * For the tasks that were created before we detected the absence of
11491238 	 * FP/SIMD, the TIF_FOREIGN_FPSTATE could be set via fpsimd_thread_switch(),
....@@ -1158,14 +1247,14 @@
11581247 		return;
11591248 	}
11601249 
1161
-	get_cpu_fpsimd_context();
1250
+	get_cpu_fpsimd_context(flags);
11621251 
11631252 	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
11641253 		task_fpsimd_load();
11651254 		fpsimd_bind_task_to_cpu();
11661255 	}
11671256 
1168
-	put_cpu_fpsimd_context();
1257
+	put_cpu_fpsimd_context(flags);
11691258 }
11701259 
11711260 /*
....@@ -1175,21 +1264,23 @@
11751264  */
11761265 void fpsimd_update_current_state(struct user_fpsimd_state const *state)
11771266 {
1267
+	unsigned long flags;
1268
+
11781269 	if (WARN_ON(!system_supports_fpsimd()))
11791270 		return;
11801271 
1181
-	get_cpu_fpsimd_context();
1272
+	get_cpu_fpsimd_context(flags);
11821273 
11831274 	current->thread.uw.fpsimd_state = *state;
11841275 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
1185
-		fpsimd_to_sve(current);
1276
+		_fpsimd_to_sve(current);
11861277 
11871278 	task_fpsimd_load();
11881279 	fpsimd_bind_task_to_cpu();
11891280 
11901281 	clear_thread_flag(TIF_FOREIGN_FPSTATE);
11911282 
1192
-	put_cpu_fpsimd_context();
1283
+	put_cpu_fpsimd_context(flags);
11931284 }
11941285 
11951286 /*
....@@ -1239,9 +1330,9 @@
12391330 {
12401331 	if (!system_supports_fpsimd())
12411332 		return;
1242
-	WARN_ON(preemptible());
1333
+	WARN_ON(!hard_irqs_disabled() && preemptible());
12431334 	__get_cpu_fpsimd_context();
1244
-	fpsimd_save();
1335
+	__fpsimd_save();
12451336 	fpsimd_flush_cpu_state();
12461337 	__put_cpu_fpsimd_context();
12471338 }
....@@ -1267,18 +1358,23 @@
12671358  */
12681359 void kernel_neon_begin(void)
12691360 {
1361
+	unsigned long flags;
1362
+
12701363 	if (WARN_ON(!system_supports_fpsimd()))
12711364 		return;
12721365 
12731366 	BUG_ON(!may_use_simd());
12741367 
1275
-	get_cpu_fpsimd_context();
1368
+	get_cpu_fpsimd_context(flags);
12761369 
12771370 	/* Save unsaved fpsimd state, if any: */
1278
-	fpsimd_save();
1371
+	__fpsimd_save();
12791372 
12801373 	/* Invalidate any task state remaining in the fpsimd regs: */
12811374 	fpsimd_flush_cpu_state();
1375
+
1376
+	if (dovetailing())
1377
+		hard_cond_local_irq_restore(flags);
12821378 }
12831379 EXPORT_SYMBOL(kernel_neon_begin);
12841380 
....@@ -1293,10 +1389,12 @@
12931389  */
12941390 void kernel_neon_end(void)
12951391 {
1392
+	unsigned long flags = hard_local_save_flags();
1393
+
12961394 	if (!system_supports_fpsimd())
12971395 		return;
12981396 
1299
-	put_cpu_fpsimd_context();
1397
+	put_cpu_fpsimd_context(flags);
13001398 }
13011399 EXPORT_SYMBOL(kernel_neon_end);
13021400 
....@@ -1386,9 +1484,13 @@
13861484 static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
13871485 				  unsigned long cmd, void *v)
13881486 {
1487
+	unsigned long flags;
1488
+
13891489 	switch (cmd) {
13901490 	case CPU_PM_ENTER:
1491
+		flags = hard_cond_local_irq_save();
13911492 		fpsimd_save_and_flush_cpu_state();
1493
+		hard_cond_local_irq_restore(flags);
13921494 		break;
13931495 	case CPU_PM_EXIT:
13941496 		break;