forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-10 10ebd8556b7990499c896a550e3d416b444211e6 
 kernel/arch/arm64/kernel/fpsimd.c
....@@ -1,27 +1,18 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * FP/SIMD context switching and fault handling
34  *
45  * Copyright (C) 2012 ARM Ltd.
56  * Author: Catalin Marinas <catalin.marinas@arm.com>
6
- *
7
- * This program is free software; you can redistribute it and/or modify
8
- * it under the terms of the GNU General Public License version 2 as
9
- * published by the Free Software Foundation.
10
- *
11
- * This program is distributed in the hope that it will be useful,
12
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
- * GNU General Public License for more details.
15
- *
16
- * You should have received a copy of the GNU General Public License
17
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
187  */
198 
209 #include <linux/bitmap.h>
10
+#include <linux/bitops.h>
2111 #include <linux/bottom_half.h>
2212 #include <linux/bug.h>
2313 #include <linux/cache.h>
2414 #include <linux/compat.h>
15
+#include <linux/compiler.h>
2516 #include <linux/cpu.h>
2617 #include <linux/cpu_pm.h>
2718 #include <linux/kernel.h>
....@@ -38,16 +29,20 @@
3829 #include <linux/slab.h>
3930 #include <linux/stddef.h>
4031 #include <linux/sysctl.h>
32
+#include <linux/swab.h>
4133 
4234 #include <asm/esr.h>
35
+#include <asm/exception.h>
4336 #include <asm/fpsimd.h>
4437 #include <asm/cpufeature.h>
4538 #include <asm/cputype.h>
39
+#include <asm/neon.h>
4640 #include <asm/processor.h>
4741 #include <asm/simd.h>
4842 #include <asm/sigcontext.h>
4943 #include <asm/sysreg.h>
5044 #include <asm/traps.h>
45
+#include <asm/virt.h>
5146 
5247 #define FPEXC_IOF	(1 << 0)
5348 #define FPEXC_DZF	(1 << 1)
....@@ -90,7 +85,8 @@
9085  * To prevent this from racing with the manipulation of the task's FPSIMD state
9186  * from task context and thereby corrupting the state, it is necessary to
9287  * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE
93
- * flag with local_bh_disable() unless softirqs are already masked.
88
+ * flag with {, __}get_cpu_fpsimd_context(). This will still allow softirqs to
89
+ * run but prevent them to use FPSIMD.
9490  *
9591  * For a certain task, the sequence may look something like this:
9692  * - the task gets scheduled in; if both the task's fpsimd_cpu field
....@@ -119,28 +115,99 @@
119115  */
120116 struct fpsimd_last_state_struct {
121117 	struct user_fpsimd_state *st;
118
+	void *sve_state;
119
+	unsigned int sve_vl;
122120 };
123121 
124122 static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
125123 
126124 /* Default VL for tasks that don't set it explicitly: */
127
-static int sve_default_vl = -1;
125
+static int __sve_default_vl = -1;
126
+
127
+static int get_sve_default_vl(void)
128
+{
129
+	return READ_ONCE(__sve_default_vl);
130
+}
128131 
129132 #ifdef CONFIG_ARM64_SVE
130133 
134
+static void set_sve_default_vl(int val)
135
+{
136
+	WRITE_ONCE(__sve_default_vl, val);
137
+}
138
+
131139 /* Maximum supported vector length across all CPUs (initially poisoned) */
132140 int __ro_after_init sve_max_vl = SVE_VL_MIN;
133
-/* Set of available vector lengths, as vq_to_bit(vq): */
134
-static __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
141
+int __ro_after_init sve_max_virtualisable_vl = SVE_VL_MIN;
142
+
143
+/*
144
+ * Set of available vector lengths,
145
+ * where length vq encoded as bit __vq_to_bit(vq):
146
+ */
147
+__ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
148
+/* Set of vector lengths present on at least one cpu: */
149
+static __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
150
+
135151 static void __percpu *efi_sve_state;
136152 
137153 #else /* ! CONFIG_ARM64_SVE */
138154 
139155 /* Dummy declaration for code that will be optimised out: */
140156 extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
157
+extern __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
141158 extern void __percpu *efi_sve_state;
142159 
143160 #endif /* ! CONFIG_ARM64_SVE */
161
+
162
+DEFINE_PER_CPU(bool, fpsimd_context_busy);
163
+EXPORT_PER_CPU_SYMBOL(fpsimd_context_busy);
164
+
165
+static void __get_cpu_fpsimd_context(void)
166
+{
167
+	bool busy = __this_cpu_xchg(fpsimd_context_busy, true);
168
+
169
+	WARN_ON(busy);
170
+}
171
+
172
+/*
173
+ * Claim ownership of the CPU FPSIMD context for use by the calling context.
174
+ *
175
+ * The caller may freely manipulate the FPSIMD context metadata until
176
+ * put_cpu_fpsimd_context() is called.
177
+ *
178
+ * The double-underscore version must only be called if you know the task
179
+ * can't be preempted.
180
+ */
181
+static void get_cpu_fpsimd_context(void)
182
+{
183
+	local_bh_disable();
184
+	__get_cpu_fpsimd_context();
185
+}
186
+
187
+static void __put_cpu_fpsimd_context(void)
188
+{
189
+	bool busy = __this_cpu_xchg(fpsimd_context_busy, false);
190
+
191
+	WARN_ON(!busy); /* No matching get_cpu_fpsimd_context()? */
192
+}
193
+
194
+/*
195
+ * Release the CPU FPSIMD context.
196
+ *
197
+ * Must be called from a context in which get_cpu_fpsimd_context() was
198
+ * previously called, with no call to put_cpu_fpsimd_context() in the
199
+ * meantime.
200
+ */
201
+static void put_cpu_fpsimd_context(void)
202
+{
203
+	__put_cpu_fpsimd_context();
204
+	local_bh_enable();
205
+}
206
+
207
+static bool have_cpu_fpsimd_context(void)
208
+{
209
+	return !preemptible() && __this_cpu_read(fpsimd_context_busy);
210
+}
144211 
145212 /*
146213  * Call __sve_free() directly only if you know task can't be scheduled
....@@ -212,13 +279,11 @@
212279  * This function should be called only when the FPSIMD/SVE state in
213280  * thread_struct is known to be up to date, when preparing to enter
214281  * userspace.
215
- *
216
- * Softirqs (and preemption) must be disabled.
217282  */
218283 static void task_fpsimd_load(void)
219284 {
220
-	WARN_ON(!in_softirq() && !irqs_disabled());
221285 	WARN_ON(!system_supports_fpsimd());
286
+	WARN_ON(!have_cpu_fpsimd_context());
222287 
223288 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
224289 		sve_load_state(sve_pffr(&current->thread),
....@@ -231,52 +296,34 @@
231296 /*
232297  * Ensure FPSIMD/SVE storage in memory for the loaded context is up to
233298  * date with respect to the CPU registers.
234
- *
235
- * Softirqs (and preemption) must be disabled.
236299  */
237
-void fpsimd_save(void)
300
+static void fpsimd_save(void)
238301 {
239
-	struct user_fpsimd_state *st = __this_cpu_read(fpsimd_last_state.st);
302
+	struct fpsimd_last_state_struct const *last =
303
+		this_cpu_ptr(&fpsimd_last_state);
240304 	/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
241305 
242306 	WARN_ON(!system_supports_fpsimd());
243
-	WARN_ON(!in_softirq() && !irqs_disabled());
307
+	WARN_ON(!have_cpu_fpsimd_context());
244308 
245309 	if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
246310 		if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
247
-			if (WARN_ON(sve_get_vl() != current->thread.sve_vl)) {
311
+			if (WARN_ON(sve_get_vl() != last->sve_vl)) {
248312 				/*
249313 				 * Can't save the user regs, so current would
250314 				 * re-enter user with corrupt state.
251315 				 * There's no way to recover, so kill it:
252316 				 */
253
-				force_signal_inject(SIGKILL, SI_KERNEL, 0);
317
+				force_signal_inject(SIGKILL, SI_KERNEL, 0, 0);
254318 				return;
255319 			}
256320 
257
-			sve_save_state(sve_pffr(&current->thread), &st->fpsr);
321
+			sve_save_state((char *)last->sve_state +
322
+						sve_ffr_offset(last->sve_vl),
323
+				       &last->st->fpsr);
258324 		} else
259
-			fpsimd_save_state(st);
325
+			fpsimd_save_state(last->st);
260326 	}
261
-}
262
-
263
-/*
264
- * Helpers to translate bit indices in sve_vq_map to VQ values (and
265
- * vice versa).  This allows find_next_bit() to be used to find the
266
- * _maximum_ VQ not exceeding a certain value.
267
- */
268
-
269
-static unsigned int vq_to_bit(unsigned int vq)
270
-{
271
-	return SVE_VQ_MAX - vq;
272
-}
273
-
274
-static unsigned int bit_to_vq(unsigned int bit)
275
-{
276
-	if (WARN_ON(bit >= SVE_VQ_MAX))
277
-		bit = SVE_VQ_MAX - 1;
278
-
279
-	return SVE_VQ_MAX - bit;
280327 }
281328 
282329 /*
....@@ -300,18 +347,17 @@
300347 		vl = max_vl;
301348 
302349 	bit = find_next_bit(sve_vq_map, SVE_VQ_MAX,
303
-			    vq_to_bit(sve_vq_from_vl(vl)));
304
-	return sve_vl_from_vq(bit_to_vq(bit));
350
+			    __vq_to_bit(sve_vq_from_vl(vl)));
351
+	return sve_vl_from_vq(__bit_to_vq(bit));
305352 }
306353 
307354 #if defined(CONFIG_ARM64_SVE) && defined(CONFIG_SYSCTL)
308355 
309356 static int sve_proc_do_default_vl(struct ctl_table *table, int write,
310
-				  void __user *buffer, size_t *lenp,
311
-				  loff_t *ppos)
357
+				  void *buffer, size_t *lenp, loff_t *ppos)
312358 {
313359 	int ret;
314
-	int vl = sve_default_vl;
360
+	int vl = get_sve_default_vl();
315361 	struct ctl_table tmp_table = {
316362 		.data = &vl,
317363 		.maxlen = sizeof(vl),
....@@ -328,7 +374,7 @@
328374 	if (!sve_vl_valid(vl))
329375 		return -EINVAL;
330376 
331
-	sve_default_vl = find_supported_vector_length(vl);
377
+	set_sve_default_vl(find_supported_vector_length(vl));
332378 	return 0;
333379 }
334380 
....@@ -357,12 +403,42 @@
357403 #define ZREG(sve_state, vq, n) ((char *)(sve_state) +		\
358404 	(SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
359405 
406
+#ifdef CONFIG_CPU_BIG_ENDIAN
407
+static __uint128_t arm64_cpu_to_le128(__uint128_t x)
408
+{
409
+	u64 a = swab64(x);
410
+	u64 b = swab64(x >> 64);
411
+
412
+	return ((__uint128_t)a << 64) | b;
413
+}
414
+#else
415
+static __uint128_t arm64_cpu_to_le128(__uint128_t x)
416
+{
417
+	return x;
418
+}
419
+#endif
420
+
421
+#define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x)
422
+
423
+static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
424
+			    unsigned int vq)
425
+{
426
+	unsigned int i;
427
+	__uint128_t *p;
428
+
429
+	for (i = 0; i < SVE_NUM_ZREGS; ++i) {
430
+		p = (__uint128_t *)ZREG(sst, vq, i);
431
+		*p = arm64_cpu_to_le128(fst->vregs[i]);
432
+	}
433
+}
434
+
360435 /*
361436  * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to
362437  * task->thread.sve_state.
363438  *
364439  * Task can be a non-runnable task, or current.  In the latter case,
365
- * softirqs (and preemption) must be disabled.
440
+ * the caller must have ownership of the cpu FPSIMD context before calling
441
+ * this function.
366442  * task->thread.sve_state must point to at least sve_state_size(task)
367443  * bytes of allocated kernel memory.
368444  * task->thread.uw.fpsimd_state must be up to date before calling this
....@@ -373,15 +449,12 @@
373449 	unsigned int vq;
374450 	void *sst = task->thread.sve_state;
375451 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
376
-	unsigned int i;
377452 
378453 	if (!system_supports_sve())
379454 		return;
380455 
381456 	vq = sve_vq_from_vl(task->thread.sve_vl);
382
-	for (i = 0; i < 32; ++i)
383
-		memcpy(ZREG(sst, vq, i), &fst->vregs[i],
384
-		       sizeof(fst->vregs[i]));
457
+	__fpsimd_to_sve(sst, fst, vq);
385458 }
386459 
387460 /*
....@@ -389,7 +462,8 @@
389462  * task->thread.uw.fpsimd_state.
390463  *
391464  * Task can be a non-runnable task, or current.  In the latter case,
392
- * softirqs (and preemption) must be disabled.
465
+ * the caller must have ownership of the cpu FPSIMD context before calling
466
+ * this function.
393467  * task->thread.sve_state must point to at least sve_state_size(task)
394468  * bytes of allocated kernel memory.
395469  * task->thread.sve_state must be up to date before calling this function.
....@@ -400,14 +474,16 @@
400474 	void const *sst = task->thread.sve_state;
401475 	struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
402476 	unsigned int i;
477
+	__uint128_t const *p;
403478 
404479 	if (!system_supports_sve())
405480 		return;
406481 
407482 	vq = sve_vq_from_vl(task->thread.sve_vl);
408
-	for (i = 0; i < 32; ++i)
409
-		memcpy(&fst->vregs[i], ZREG(sst, vq, i),
410
-		       sizeof(fst->vregs[i]));
483
+	for (i = 0; i < SVE_NUM_ZREGS; ++i) {
484
+		p = (__uint128_t const *)ZREG(sst, vq, i);
485
+		fst->vregs[i] = arm64_le128_to_cpu(*p);
486
+	}
411487 }
412488 
413489 #ifdef CONFIG_ARM64_SVE
....@@ -495,7 +571,6 @@
495571 	unsigned int vq;
496572 	void *sst = task->thread.sve_state;
497573 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
498
-	unsigned int i;
499574 
500575 	if (!test_tsk_thread_flag(task, TIF_SVE))
501576 		return;
....@@ -503,10 +578,7 @@
503578 	vq = sve_vq_from_vl(task->thread.sve_vl);
504579 
505580 	memset(sst, 0, SVE_SIG_REGS_SIZE(vq));
506
-
507
-	for (i = 0; i < 32; ++i)
508
-		memcpy(ZREG(sst, vq, i), &fst->vregs[i],
509
-		       sizeof(fst->vregs[i]));
581
+	__fpsimd_to_sve(sst, fst, vq);
510582 }
511583 
512584 int sve_set_vector_length(struct task_struct *task,
....@@ -549,10 +621,9 @@
549621 	 * non-SVE thread.
550622 	 */
551623 	if (task == current) {
552
-		local_bh_disable();
624
+		get_cpu_fpsimd_context();
553625 
554626 		fpsimd_save();
555
-		set_thread_flag(TIF_FOREIGN_FPSTATE);
556627 	}
557628 
558629 	fpsimd_flush_task_state(task);
....@@ -560,7 +631,7 @@
560631 		sve_to_fpsimd(task);
561632 
562633 	if (task == current)
563
-		local_bh_enable();
634
+		put_cpu_fpsimd_context();
564635 
565636 	/*
566637 	 * Force reallocation of task SVE state to the correct size
....@@ -607,7 +678,7 @@
607678 	vl = arg & PR_SVE_VL_LEN_MASK;
608679 	flags = arg & ~vl;
609680 
610
-	if (!system_supports_sve())
681
+	if (!system_supports_sve() || is_compat_task())
611682 		return -EINVAL;
612683 
613684 	ret = sve_set_vector_length(current, vl, flags);
....@@ -620,17 +691,11 @@
620691 /* PR_SVE_GET_VL */
621692 int sve_get_current_vl(void)
622693 {
623
-	if (!system_supports_sve())
694
+	if (!system_supports_sve() || is_compat_task())
624695 		return -EINVAL;
625696 
626697 	return sve_prctl_status(0);
627698 }
628
-
629
-/*
630
- * Bitmap for temporary storage of the per-CPU set of supported vector lengths
631
- * during secondary boot.
632
- */
633
-static DECLARE_BITMAP(sve_secondary_vq_map, SVE_VQ_MAX);
634699 
635700 static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))
636701 {
....@@ -646,40 +711,82 @@
646711 		write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1); /* self-syncing */
647712 		vl = sve_get_vl();
648713 		vq = sve_vq_from_vl(vl); /* skip intervening lengths */
649
-		set_bit(vq_to_bit(vq), map);
714
+		set_bit(__vq_to_bit(vq), map);
650715 	}
651716 }
652717 
718
+/*
719
+ * Initialise the set of known supported VQs for the boot CPU.
720
+ * This is called during kernel boot, before secondary CPUs are brought up.
721
+ */
653722 void __init sve_init_vq_map(void)
654723 {
655724 	sve_probe_vqs(sve_vq_map);
725
+	bitmap_copy(sve_vq_partial_map, sve_vq_map, SVE_VQ_MAX);
656726 }
657727 
658728 /*
659729  * If we haven't committed to the set of supported VQs yet, filter out
660730  * those not supported by the current CPU.
731
+ * This function is called during the bring-up of early secondary CPUs only.
661732  */
662733 void sve_update_vq_map(void)
663734 {
664
-	sve_probe_vqs(sve_secondary_vq_map);
665
-	bitmap_and(sve_vq_map, sve_vq_map, sve_secondary_vq_map, SVE_VQ_MAX);
735
+	DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
736
+
737
+	sve_probe_vqs(tmp_map);
738
+	bitmap_and(sve_vq_map, sve_vq_map, tmp_map, SVE_VQ_MAX);
739
+	bitmap_or(sve_vq_partial_map, sve_vq_partial_map, tmp_map, SVE_VQ_MAX);
666740 }
667741 
668
-/* Check whether the current CPU supports all VQs in the committed set */
742
+/*
743
+ * Check whether the current CPU supports all VQs in the committed set.
744
+ * This function is called during the bring-up of late secondary CPUs only.
745
+ */
669746 int sve_verify_vq_map(void)
670747 {
671
-	int ret = 0;
748
+	DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
749
+	unsigned long b;
672750 
673
-	sve_probe_vqs(sve_secondary_vq_map);
674
-	bitmap_andnot(sve_secondary_vq_map, sve_vq_map, sve_secondary_vq_map,
675
-		      SVE_VQ_MAX);
676
-	if (!bitmap_empty(sve_secondary_vq_map, SVE_VQ_MAX)) {
751
+	sve_probe_vqs(tmp_map);
752
+
753
+	bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
754
+	if (bitmap_intersects(tmp_map, sve_vq_map, SVE_VQ_MAX)) {
677755 		pr_warn("SVE: cpu%d: Required vector length(s) missing\n",
678756 			smp_processor_id());
679
-		ret = -EINVAL;
757
+		return -EINVAL;
680758 	}
681759 
682
-	return ret;
760
+	if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available())
761
+		return 0;
762
+
763
+	/*
764
+	 * For KVM, it is necessary to ensure that this CPU doesn't
765
+	 * support any vector length that guests may have probed as
766
+	 * unsupported.
767
+	 */
768
+
769
+	/* Recover the set of supported VQs: */
770
+	bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
771
+	/* Find VQs supported that are not globally supported: */
772
+	bitmap_andnot(tmp_map, tmp_map, sve_vq_map, SVE_VQ_MAX);
773
+
774
+	/* Find the lowest such VQ, if any: */
775
+	b = find_last_bit(tmp_map, SVE_VQ_MAX);
776
+	if (b >= SVE_VQ_MAX)
777
+		return 0; /* no mismatches */
778
+
779
+	/*
780
+	 * Mismatches above sve_max_virtualisable_vl are fine, since
781
+	 * no guest is allowed to configure ZCR_EL2.LEN to exceed this:
782
+	 */
783
+	if (sve_vl_from_vq(__bit_to_vq(b)) <= sve_max_virtualisable_vl) {
784
+		pr_warn("SVE: cpu%d: Unsupported vector length(s) present\n",
785
+			smp_processor_id());
786
+		return -EINVAL;
787
+	}
788
+
789
+	return 0;
683790 }
684791 
685792 static void __init sve_efi_setup(void)
....@@ -746,6 +853,8 @@
746853 void __init sve_setup(void)
747854 {
748855 	u64 zcr;
856
+	DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
857
+	unsigned long b;
749858 
750859 	if (!system_supports_sve())
751860 		return;
....@@ -755,8 +864,8 @@
755864 	 * so sve_vq_map must have at least SVE_VQ_MIN set.
756865 	 * If something went wrong, at least try to patch it up:
757866 	 */
758
-	if (WARN_ON(!test_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
759
-		set_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map);
867
+	if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
868
+		set_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map);
760869 
761870 	zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
762871 	sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1);
....@@ -772,12 +881,32 @@
772881 	 * For the default VL, pick the maximum supported value <= 64.
773882 	 * VL == 64 is guaranteed not to grow the signal frame.
774883 	 */
775
-	sve_default_vl = find_supported_vector_length(64);
884
+	set_sve_default_vl(find_supported_vector_length(64));
885
+
886
+	bitmap_andnot(tmp_map, sve_vq_partial_map, sve_vq_map,
887
+		      SVE_VQ_MAX);
888
+
889
+	b = find_last_bit(tmp_map, SVE_VQ_MAX);
890
+	if (b >= SVE_VQ_MAX)
891
+		/* No non-virtualisable VLs found */
892
+		sve_max_virtualisable_vl = SVE_VQ_MAX;
893
+	else if (WARN_ON(b == SVE_VQ_MAX - 1))
894
+		/* No virtualisable VLs?  This is architecturally forbidden. */
895
+		sve_max_virtualisable_vl = SVE_VQ_MIN;
896
+	else /* b + 1 < SVE_VQ_MAX */
897
+		sve_max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1));
898
+
899
+	if (sve_max_virtualisable_vl > sve_max_vl)
900
+		sve_max_virtualisable_vl = sve_max_vl;
776901 
777902 	pr_info("SVE: maximum available vector length %u bytes per vector\n",
778903 		sve_max_vl);
779904 	pr_info("SVE: default vector length %u bytes per vector\n",
780
-		sve_default_vl);
905
+		get_sve_default_vl());
906
+
907
+	/* KVM decides whether to support mismatched systems. Just warn here: */
908
+	if (sve_max_virtualisable_vl < sve_max_vl)
909
+		pr_warn("SVE: unvirtualisable vector lengths present\n");
781910 
782911 	sve_efi_setup();
783912 }
....@@ -801,39 +930,38 @@
801930  * the SVE access trap will be disabled the next time this task
802931  * reaches ret_to_user.
803932  *
804
- * TIF_SVE should be clear on entry: otherwise, task_fpsimd_load()
933
+ * TIF_SVE should be clear on entry: otherwise, fpsimd_restore_current_state()
805934  * would have disabled the SVE access trap for userspace during
806935  * ret_to_user, making an SVE access trap impossible in that case.
807936  */
808
-asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
937
+void do_sve_acc(unsigned int esr, struct pt_regs *regs)
809938 {
810939 	/* Even if we chose not to use SVE, the hardware could still trap: */
811940 	if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
812
-		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
941
+		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
813942 		return;
814943 	}
815944 
816945 	sve_alloc(current);
817946 
818
-	local_bh_disable();
947
+	get_cpu_fpsimd_context();
819948 
820949 	fpsimd_save();
821
-	fpsimd_to_sve(current);
822950 
823951 	/* Force ret_to_user to reload the registers: */
824952 	fpsimd_flush_task_state(current);
825
-	set_thread_flag(TIF_FOREIGN_FPSTATE);
826953 
954
+	fpsimd_to_sve(current);
827955 	if (test_and_set_thread_flag(TIF_SVE))
828956 		WARN_ON(1); /* SVE access shouldn't have trapped */
829957 
830
-	local_bh_enable();
958
+	put_cpu_fpsimd_context();
831959 }
832960 
833961 /*
834962  * Trapped FP/ASIMD access.
835963  */
836
-asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
964
+void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
837965 {
838966 	/* TODO: implement lazy context saving/restoring */
839967 	WARN_ON(1);
....@@ -842,9 +970,8 @@
842970 /*
843971  * Raise a SIGFPE for the current process.
844972  */
845
-asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
973
+void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
846974 {
847
-	siginfo_t info;
848975 	unsigned int si_code = FPE_FLTUNK;
849976 
850977 	if (esr & ESR_ELx_FP_EXC_TFV) {
....@@ -860,12 +987,9 @@
860987 			si_code = FPE_FLTRES;
861988 	}
862989 
863
-	clear_siginfo(&info);
864
-	info.si_signo = SIGFPE;
865
-	info.si_code = si_code;
866
-	info.si_addr = (void __user *)instruction_pointer(regs);
867
-
868
-	send_sig_info(SIGFPE, &info, current);
990
+	send_sig_fault(SIGFPE, si_code,
991
+		       (void __user *)instruction_pointer(regs),
992
+		       current);
869993 }
870994 
871995 void fpsimd_thread_switch(struct task_struct *next)
....@@ -874,6 +998,8 @@
874998 
875999 	if (!system_supports_fpsimd())
8761000 		return;
1001
+
1002
+	__get_cpu_fpsimd_context();
8771003 
8781004 	/* Save unsaved fpsimd state, if any: */
8791005 	fpsimd_save();
....@@ -889,6 +1015,8 @@
8891015 
8901016 	update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
8911017 			       wrong_task || wrong_cpu);
1018
+
1019
+	__put_cpu_fpsimd_context();
8921020 }
8931021 
8941022 void fpsimd_flush_thread(void)
....@@ -898,11 +1026,11 @@
8981026 	if (!system_supports_fpsimd())
8991027 		return;
9001028 
901
-	local_bh_disable();
1029
+	get_cpu_fpsimd_context();
9021030 
1031
+	fpsimd_flush_task_state(current);
9031032 	memset(&current->thread.uw.fpsimd_state, 0,
9041033 	       sizeof(current->thread.uw.fpsimd_state));
905
-	fpsimd_flush_task_state(current);
9061034 
9071035 	if (system_supports_sve()) {
9081036 		clear_thread_flag(TIF_SVE);
....@@ -914,13 +1042,13 @@
9141042 		 * vector length configured: no kernel task can become a user
9151043 		 * task without an exec and hence a call to this function.
9161044 		 * By the time the first call to this function is made, all
917
-		 * early hardware probing is complete, so sve_default_vl
1045
+		 * early hardware probing is complete, so __sve_default_vl
9181046 		 * should be valid.
9191047 		 * If a bug causes this to go wrong, we make some noise and
9201048 		 * try to fudge thread.sve_vl to a safe value here.
9211049 		 */
9221050 		vl = current->thread.sve_vl_onexec ?
923
-			current->thread.sve_vl_onexec : sve_default_vl;
1051
+			current->thread.sve_vl_onexec : get_sve_default_vl();
9241052 
9251053 		if (WARN_ON(!sve_vl_valid(vl)))
9261054 			vl = SVE_VL_MIN;
....@@ -939,9 +1067,7 @@
9391067 			current->thread.sve_vl_onexec = 0;
9401068 	}
9411069 
942
-	set_thread_flag(TIF_FOREIGN_FPSTATE);
943
-
944
-	local_bh_enable();
1070
+	put_cpu_fpsimd_context();
9451071 }
9461072 
9471073 /*
....@@ -953,9 +1079,9 @@
9531079 	if (!system_supports_fpsimd())
9541080 		return;
9551081 
956
-	local_bh_disable();
1082
+	get_cpu_fpsimd_context();
9571083 	fpsimd_save();
958
-	local_bh_enable();
1084
+	put_cpu_fpsimd_context();
9591085 }
9601086 
9611087 /*
....@@ -972,7 +1098,8 @@
9721098 
9731099 /*
9741100  * Associate current's FPSIMD context with this cpu
975
- * Preemption must be disabled when calling this function.
1101
+ * The caller must have ownership of the cpu FPSIMD context before calling
1102
+ * this function.
9761103  */
9771104 void fpsimd_bind_task_to_cpu(void)
9781105 {
....@@ -981,6 +1108,8 @@
9811108 
9821109 	WARN_ON(!system_supports_fpsimd());
9831110 	last->st = &current->thread.uw.fpsimd_state;
1111
+	last->sve_state = current->thread.sve_state;
1112
+	last->sve_vl = current->thread.sve_vl;
9841113 	current->thread.fpsimd_cpu = smp_processor_id();
9851114 
9861115 	if (system_supports_sve()) {
....@@ -994,7 +1123,8 @@
9941123 	}
9951124 }
9961125 
997
-void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st)
1126
+void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
1127
+			      unsigned int sve_vl)
9981128 {
9991129 	struct fpsimd_last_state_struct *last =
10001130 		this_cpu_ptr(&fpsimd_last_state);
....@@ -1003,6 +1133,8 @@
10031133 	WARN_ON(!in_softirq() && !irqs_disabled());
10041134 
10051135 	last->st = st;
1136
+	last->sve_state = sve_state;
1137
+	last->sve_vl = sve_vl;
10061138 }
10071139 
10081140 /*
....@@ -1026,14 +1158,14 @@
10261158 		return;
10271159 	}
10281160 
1029
-	local_bh_disable();
1161
+	get_cpu_fpsimd_context();
10301162 
10311163 	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
10321164 		task_fpsimd_load();
10331165 		fpsimd_bind_task_to_cpu();
10341166 	}
10351167 
1036
-	local_bh_enable();
1168
+	put_cpu_fpsimd_context();
10371169 }
10381170 
10391171 /*
....@@ -1046,7 +1178,7 @@
10461178 	if (WARN_ON(!system_supports_fpsimd()))
10471179 		return;
10481180 
1049
-	local_bh_disable();
1181
+	get_cpu_fpsimd_context();
10501182 
10511183 	current->thread.uw.fpsimd_state = *state;
10521184 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
....@@ -1057,28 +1189,64 @@
10571189 
10581190 	clear_thread_flag(TIF_FOREIGN_FPSTATE);
10591191 
1060
-	local_bh_enable();
1192
+	put_cpu_fpsimd_context();
10611193 }
10621194 
10631195 /*
10641196  * Invalidate live CPU copies of task t's FPSIMD state
1197
+ *
1198
+ * This function may be called with preemption enabled.  The barrier()
1199
+ * ensures that the assignment to fpsimd_cpu is visible to any
1200
+ * preemption/softirq that could race with set_tsk_thread_flag(), so
1201
+ * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared.
1202
+ *
1203
+ * The final barrier ensures that TIF_FOREIGN_FPSTATE is seen set by any
1204
+ * subsequent code.
10651205  */
10661206 void fpsimd_flush_task_state(struct task_struct *t)
10671207 {
10681208 	t->thread.fpsimd_cpu = NR_CPUS;
1209
+	/*
1210
+	 * If we don't support fpsimd, bail out after we have
1211
+	 * reset the fpsimd_cpu for this task and clear the
1212
+	 * FPSTATE.
1213
+	 */
1214
+	if (!system_supports_fpsimd())
1215
+		return;
1216
+	barrier();
1217
+	set_tsk_thread_flag(t, TIF_FOREIGN_FPSTATE);
1218
+
1219
+	barrier();
10691220 }
10701221 
1071
-void fpsimd_flush_cpu_state(void)
1222
+/*
1223
+ * Invalidate any task's FPSIMD state that is present on this cpu.
1224
+ * The FPSIMD context should be acquired with get_cpu_fpsimd_context()
1225
+ * before calling this function.
1226
+ */
1227
+static void fpsimd_flush_cpu_state(void)
10721228 {
10731229 	WARN_ON(!system_supports_fpsimd());
10741230 	__this_cpu_write(fpsimd_last_state.st, NULL);
10751231 	set_thread_flag(TIF_FOREIGN_FPSTATE);
10761232 }
10771233 
1078
-#ifdef CONFIG_KERNEL_MODE_NEON
1234
+/*
1235
+ * Save the FPSIMD state to memory and invalidate cpu view.
1236
+ * This function must be called with preemption disabled.
1237
+ */
1238
+void fpsimd_save_and_flush_cpu_state(void)
1239
+{
1240
+	if (!system_supports_fpsimd())
1241
+		return;
1242
+	WARN_ON(preemptible());
1243
+	__get_cpu_fpsimd_context();
1244
+	fpsimd_save();
1245
+	fpsimd_flush_cpu_state();
1246
+	__put_cpu_fpsimd_context();
1247
+}
10791248 
1080
-DEFINE_PER_CPU(bool, kernel_neon_busy);
1081
-EXPORT_PER_CPU_SYMBOL(kernel_neon_busy);
1249
+#ifdef CONFIG_KERNEL_MODE_NEON
10821250 
10831251 /*
10841252  * Kernel-side NEON support functions
....@@ -1104,19 +1272,13 @@
11041272 
11051273 	BUG_ON(!may_use_simd());
11061274 
1107
-	local_bh_disable();
1108
-
1109
-	__this_cpu_write(kernel_neon_busy, true);
1275
+	get_cpu_fpsimd_context();
11101276 
11111277 	/* Save unsaved fpsimd state, if any: */
11121278 	fpsimd_save();
11131279 
11141280 	/* Invalidate any task state remaining in the fpsimd regs: */
11151281 	fpsimd_flush_cpu_state();
1116
-
1117
-	preempt_disable();
1118
-
1119
-	local_bh_enable();
11201282 }
11211283 EXPORT_SYMBOL(kernel_neon_begin);
11221284 
....@@ -1131,15 +1293,10 @@
11311293  */
11321294 void kernel_neon_end(void)
11331295 {
1134
-	bool busy;
1135
-
11361296 	if (!system_supports_fpsimd())
11371297 		return;
11381298 
1139
-	busy = __this_cpu_xchg(kernel_neon_busy, false);
1140
-	WARN_ON(!busy);	/* No matching kernel_neon_begin()? */
1141
-
1142
-	preempt_enable();
1299
+	put_cpu_fpsimd_context();
11431300 }
11441301 EXPORT_SYMBOL(kernel_neon_end);
11451302 
....@@ -1231,8 +1388,7 @@
12311388 {
12321389 	switch (cmd) {
12331390 	case CPU_PM_ENTER:
1234
-		fpsimd_save();
1235
-		fpsimd_flush_cpu_state();
1391
+		fpsimd_save_and_flush_cpu_state();
12361392 		break;
12371393 	case CPU_PM_EXIT:
12381394 		break;
....@@ -1278,14 +1434,14 @@
12781434  */
12791435 static int __init fpsimd_init(void)
12801436 {
1281
-	if (elf_hwcap & HWCAP_FP) {
1437
+	if (cpu_have_named_feature(FP)) {
12821438 		fpsimd_pm_init();
12831439 		fpsimd_hotplug_init();
12841440 	} else {
12851441 		pr_notice("Floating-point is not implemented\n");
12861442 	}
12871443 
1288
-	if (!(elf_hwcap & HWCAP_ASIMD))
1444
+	if (!cpu_have_named_feature(ASIMD))
12891445 		pr_notice("Advanced SIMD is not implemented\n");
12901446 
12911447 	return sve_sysctl_init();