forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/arch/arm64/kernel/fpsimd.c
....@@ -1,27 +1,18 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * FP/SIMD context switching and fault handling
34  *
45  * Copyright (C) 2012 ARM Ltd.
56  * Author: Catalin Marinas <catalin.marinas@arm.com>
6
- *
7
- * This program is free software; you can redistribute it and/or modify
8
- * it under the terms of the GNU General Public License version 2 as
9
- * published by the Free Software Foundation.
10
- *
11
- * This program is distributed in the hope that it will be useful,
12
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
- * GNU General Public License for more details.
15
- *
16
- * You should have received a copy of the GNU General Public License
17
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
187  */
198 
209 #include <linux/bitmap.h>
10
+#include <linux/bitops.h>
2111 #include <linux/bottom_half.h>
2212 #include <linux/bug.h>
2313 #include <linux/cache.h>
2414 #include <linux/compat.h>
15
+#include <linux/compiler.h>
2516 #include <linux/cpu.h>
2617 #include <linux/cpu_pm.h>
2718 #include <linux/kernel.h>
....@@ -38,16 +29,20 @@
3829 #include <linux/slab.h>
3930 #include <linux/stddef.h>
4031 #include <linux/sysctl.h>
32
+#include <linux/swab.h>
4133 
4234 #include <asm/esr.h>
35
+#include <asm/exception.h>
4336 #include <asm/fpsimd.h>
4437 #include <asm/cpufeature.h>
4538 #include <asm/cputype.h>
39
+#include <asm/neon.h>
4640 #include <asm/processor.h>
4741 #include <asm/simd.h>
4842 #include <asm/sigcontext.h>
4943 #include <asm/sysreg.h>
5044 #include <asm/traps.h>
45
+#include <asm/virt.h>
5146 
5247 #define FPEXC_IOF	(1 << 0)
5348 #define FPEXC_DZF	(1 << 1)
....@@ -90,7 +85,8 @@
9085  * To prevent this from racing with the manipulation of the task's FPSIMD state
9186  * from task context and thereby corrupting the state, it is necessary to
9287  * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE
93
- * flag with local_bh_disable() unless softirqs are already masked.
88
+ * flag with {, __}get_cpu_fpsimd_context(). This will still allow softirqs to
89
+ * run but prevent them to use FPSIMD.
9490  *
9591  * For a certain task, the sequence may look something like this:
9692  * - the task gets scheduled in; if both the task's fpsimd_cpu field
....@@ -119,44 +115,114 @@
119115  */
120116 struct fpsimd_last_state_struct {
121117 	struct user_fpsimd_state *st;
118
+	void *sve_state;
119
+	unsigned int sve_vl;
122120 };
123121 
124122 static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
125123 
126124 /* Default VL for tasks that don't set it explicitly: */
127
-static int sve_default_vl = -1;
125
+static int __sve_default_vl = -1;
126
+
127
+static int get_sve_default_vl(void)
128
+{
129
+	return READ_ONCE(__sve_default_vl);
130
+}
128131 
129132 #ifdef CONFIG_ARM64_SVE
130133 
134
+static void set_sve_default_vl(int val)
135
+{
136
+	WRITE_ONCE(__sve_default_vl, val);
137
+}
138
+
131139 /* Maximum supported vector length across all CPUs (initially poisoned) */
132140 int __ro_after_init sve_max_vl = SVE_VL_MIN;
133
-/* Set of available vector lengths, as vq_to_bit(vq): */
134
-static __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
141
+int __ro_after_init sve_max_virtualisable_vl = SVE_VL_MIN;
142
+
143
+/*
144
+ * Set of available vector lengths,
145
+ * where length vq encoded as bit __vq_to_bit(vq):
146
+ */
147
+__ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
148
+/* Set of vector lengths present on at least one cpu: */
149
+static __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
150
+
135151 static void __percpu *efi_sve_state;
136152 
137153 #else /* ! CONFIG_ARM64_SVE */
138154 
139155 /* Dummy declaration for code that will be optimised out: */
140156 extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
157
+extern __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
141158 extern void __percpu *efi_sve_state;
142159 
143160 #endif /* ! CONFIG_ARM64_SVE */
144161 
145
-/*
146
- * Call __sve_free() directly only if you know task can't be scheduled
147
- * or preempted.
148
- */
149
-static void __sve_free(struct task_struct *task)
162
+DEFINE_PER_CPU(bool, fpsimd_context_busy);
163
+EXPORT_PER_CPU_SYMBOL(fpsimd_context_busy);
164
+
165
+static void __get_cpu_fpsimd_context(void)
150166 {
151
-	kfree(task->thread.sve_state);
152
-	task->thread.sve_state = NULL;
167
+	bool busy = __this_cpu_xchg(fpsimd_context_busy, true);
168
+
169
+	WARN_ON(busy);
153170 }
154171 
155
-static void sve_free(struct task_struct *task)
172
+/*
173
+ * Claim ownership of the CPU FPSIMD context for use by the calling context.
174
+ *
175
+ * The caller may freely manipulate the FPSIMD context metadata until
176
+ * put_cpu_fpsimd_context() is called.
177
+ *
178
+ * The double-underscore version must only be called if you know the task
179
+ * can't be preempted.
180
+ */
181
+static void get_cpu_fpsimd_context(void)
156182 {
183
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
184
+		local_bh_disable();
185
+	else
186
+		preempt_disable();
187
+	__get_cpu_fpsimd_context();
188
+}
189
+
190
+static void __put_cpu_fpsimd_context(void)
191
+{
192
+	bool busy = __this_cpu_xchg(fpsimd_context_busy, false);
193
+
194
+	WARN_ON(!busy); /* No matching get_cpu_fpsimd_context()? */
195
+}
196
+
197
+/*
198
+ * Release the CPU FPSIMD context.
199
+ *
200
+ * Must be called from a context in which get_cpu_fpsimd_context() was
201
+ * previously called, with no call to put_cpu_fpsimd_context() in the
202
+ * meantime.
203
+ */
204
+static void put_cpu_fpsimd_context(void)
205
+{
206
+	__put_cpu_fpsimd_context();
207
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
208
+		local_bh_enable();
209
+	else
210
+		preempt_enable();
211
+}
212
+
213
+static bool have_cpu_fpsimd_context(void)
214
+{
215
+	return !preemptible() && __this_cpu_read(fpsimd_context_busy);
216
+}
217
+
218
+static void *sve_free_atomic(struct task_struct *task)
219
+{
220
+	void *sve_state = task->thread.sve_state;
221
+
157222 	WARN_ON(test_tsk_thread_flag(task, TIF_SVE));
158223 
159
-	__sve_free(task);
224
+	task->thread.sve_state = NULL;
225
+	return sve_state;
160226 }
161227 
162228 /*
....@@ -212,13 +278,11 @@
212278  * This function should be called only when the FPSIMD/SVE state in
213279  * thread_struct is known to be up to date, when preparing to enter
214280  * userspace.
215
- *
216
- * Softirqs (and preemption) must be disabled.
217281  */
218282 static void task_fpsimd_load(void)
219283 {
220
-	WARN_ON(!in_softirq() && !irqs_disabled());
221284 	WARN_ON(!system_supports_fpsimd());
285
+	WARN_ON(!have_cpu_fpsimd_context());
222286 
223287 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
224288 		sve_load_state(sve_pffr(&current->thread),
....@@ -231,52 +295,34 @@
231295 /*
232296  * Ensure FPSIMD/SVE storage in memory for the loaded context is up to
233297  * date with respect to the CPU registers.
234
- *
235
- * Softirqs (and preemption) must be disabled.
236298  */
237
-void fpsimd_save(void)
299
+static void fpsimd_save(void)
238300 {
239
-	struct user_fpsimd_state *st = __this_cpu_read(fpsimd_last_state.st);
301
+	struct fpsimd_last_state_struct const *last =
302
+		this_cpu_ptr(&fpsimd_last_state);
240303 	/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
241304 
242305 	WARN_ON(!system_supports_fpsimd());
243
-	WARN_ON(!in_softirq() && !irqs_disabled());
306
+	WARN_ON(!have_cpu_fpsimd_context());
244307 
245308 	if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
246309 		if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
247
-			if (WARN_ON(sve_get_vl() != current->thread.sve_vl)) {
310
+			if (WARN_ON(sve_get_vl() != last->sve_vl)) {
248311 				/*
249312 				 * Can't save the user regs, so current would
250313 				 * re-enter user with corrupt state.
251314 				 * There's no way to recover, so kill it:
252315 				 */
253
-				force_signal_inject(SIGKILL, SI_KERNEL, 0);
316
+				force_signal_inject(SIGKILL, SI_KERNEL, 0, 0);
254317 				return;
255318 			}
256319 
257
-			sve_save_state(sve_pffr(&current->thread), &st->fpsr);
320
+			sve_save_state((char *)last->sve_state +
321
+						sve_ffr_offset(last->sve_vl),
322
+				       &last->st->fpsr);
258323 		} else
259
-			fpsimd_save_state(st);
324
+			fpsimd_save_state(last->st);
260325 	}
261
-}
262
-
263
-/*
264
- * Helpers to translate bit indices in sve_vq_map to VQ values (and
265
- * vice versa).  This allows find_next_bit() to be used to find the
266
- * _maximum_ VQ not exceeding a certain value.
267
- */
268
-
269
-static unsigned int vq_to_bit(unsigned int vq)
270
-{
271
-	return SVE_VQ_MAX - vq;
272
-}
273
-
274
-static unsigned int bit_to_vq(unsigned int bit)
275
-{
276
-	if (WARN_ON(bit >= SVE_VQ_MAX))
277
-		bit = SVE_VQ_MAX - 1;
278
-
279
-	return SVE_VQ_MAX - bit;
280326 }
281327 
282328 /*
....@@ -300,18 +346,17 @@
300346 		vl = max_vl;
301347 
302348 	bit = find_next_bit(sve_vq_map, SVE_VQ_MAX,
303
-			    vq_to_bit(sve_vq_from_vl(vl)));
304
-	return sve_vl_from_vq(bit_to_vq(bit));
349
+			    __vq_to_bit(sve_vq_from_vl(vl)));
350
+	return sve_vl_from_vq(__bit_to_vq(bit));
305351 }
306352 
307353 #if defined(CONFIG_ARM64_SVE) && defined(CONFIG_SYSCTL)
308354 
309355 static int sve_proc_do_default_vl(struct ctl_table *table, int write,
310
-				  void __user *buffer, size_t *lenp,
311
-				  loff_t *ppos)
356
+				  void *buffer, size_t *lenp, loff_t *ppos)
312357 {
313358 	int ret;
314
-	int vl = sve_default_vl;
359
+	int vl = get_sve_default_vl();
315360 	struct ctl_table tmp_table = {
316361 		.data = &vl,
317362 		.maxlen = sizeof(vl),
....@@ -328,7 +373,7 @@
328373 	if (!sve_vl_valid(vl))
329374 		return -EINVAL;
330375 
331
-	sve_default_vl = find_supported_vector_length(vl);
376
+	set_sve_default_vl(find_supported_vector_length(vl));
332377 	return 0;
333378 }
334379 
....@@ -357,12 +402,42 @@
357402 #define ZREG(sve_state, vq, n) ((char *)(sve_state) +		\
358403 	(SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
359404 
405
+#ifdef CONFIG_CPU_BIG_ENDIAN
406
+static __uint128_t arm64_cpu_to_le128(__uint128_t x)
407
+{
408
+	u64 a = swab64(x);
409
+	u64 b = swab64(x >> 64);
410
+
411
+	return ((__uint128_t)a << 64) | b;
412
+}
413
+#else
414
+static __uint128_t arm64_cpu_to_le128(__uint128_t x)
415
+{
416
+	return x;
417
+}
418
+#endif
419
+
420
+#define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x)
421
+
422
+static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
423
+			    unsigned int vq)
424
+{
425
+	unsigned int i;
426
+	__uint128_t *p;
427
+
428
+	for (i = 0; i < SVE_NUM_ZREGS; ++i) {
429
+		p = (__uint128_t *)ZREG(sst, vq, i);
430
+		*p = arm64_cpu_to_le128(fst->vregs[i]);
431
+	}
432
+}
433
+
360434 /*
361435  * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to
362436  * task->thread.sve_state.
363437  *
364438  * Task can be a non-runnable task, or current.  In the latter case,
365
- * softirqs (and preemption) must be disabled.
439
+ * the caller must have ownership of the cpu FPSIMD context before calling
440
+ * this function.
366441  * task->thread.sve_state must point to at least sve_state_size(task)
367442  * bytes of allocated kernel memory.
368443  * task->thread.uw.fpsimd_state must be up to date before calling this
....@@ -373,15 +448,12 @@
373448 	unsigned int vq;
374449 	void *sst = task->thread.sve_state;
375450 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
376
-	unsigned int i;
377451 
378452 	if (!system_supports_sve())
379453 		return;
380454 
381455 	vq = sve_vq_from_vl(task->thread.sve_vl);
382
-	for (i = 0; i < 32; ++i)
383
-		memcpy(ZREG(sst, vq, i), &fst->vregs[i],
384
-		       sizeof(fst->vregs[i]));
456
+	__fpsimd_to_sve(sst, fst, vq);
385457 }
386458 
387459 /*
....@@ -389,7 +461,8 @@
389461  * task->thread.uw.fpsimd_state.
390462  *
391463  * Task can be a non-runnable task, or current.  In the latter case,
392
- * softirqs (and preemption) must be disabled.
464
+ * the caller must have ownership of the cpu FPSIMD context before calling
465
+ * this function.
393466  * task->thread.sve_state must point to at least sve_state_size(task)
394467  * bytes of allocated kernel memory.
395468  * task->thread.sve_state must be up to date before calling this function.
....@@ -400,14 +473,16 @@
400473 	void const *sst = task->thread.sve_state;
401474 	struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
402475 	unsigned int i;
476
+	__uint128_t const *p;
403477 
404478 	if (!system_supports_sve())
405479 		return;
406480 
407481 	vq = sve_vq_from_vl(task->thread.sve_vl);
408
-	for (i = 0; i < 32; ++i)
409
-		memcpy(&fst->vregs[i], ZREG(sst, vq, i),
410
-		       sizeof(fst->vregs[i]));
482
+	for (i = 0; i < SVE_NUM_ZREGS; ++i) {
483
+		p = (__uint128_t const *)ZREG(sst, vq, i);
484
+		fst->vregs[i] = arm64_le128_to_cpu(*p);
485
+	}
411486 }
412487 
413488 #ifdef CONFIG_ARM64_SVE
....@@ -495,7 +570,6 @@
495570 	unsigned int vq;
496571 	void *sst = task->thread.sve_state;
497572 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
498
-	unsigned int i;
499573 
500574 	if (!test_tsk_thread_flag(task, TIF_SVE))
501575 		return;
....@@ -503,15 +577,13 @@
503577 	vq = sve_vq_from_vl(task->thread.sve_vl);
504578 
505579 	memset(sst, 0, SVE_SIG_REGS_SIZE(vq));
506
-
507
-	for (i = 0; i < 32; ++i)
508
-		memcpy(ZREG(sst, vq, i), &fst->vregs[i],
509
-		       sizeof(fst->vregs[i]));
580
+	__fpsimd_to_sve(sst, fst, vq);
510581 }
511582 
512583 int sve_set_vector_length(struct task_struct *task,
513584 			  unsigned long vl, unsigned long flags)
514585 {
586
+	void *mem = NULL;
515587 	if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
516588 				     PR_SVE_SET_VL_ONEXEC))
517589 		return -EINVAL;
....@@ -549,10 +621,9 @@
549621 	 * non-SVE thread.
550622 	 */
551623 	if (task == current) {
552
-		local_bh_disable();
624
+		get_cpu_fpsimd_context();
553625 
554626 		fpsimd_save();
555
-		set_thread_flag(TIF_FOREIGN_FPSTATE);
556627 	}
557628 
558629 	fpsimd_flush_task_state(task);
....@@ -560,15 +631,16 @@
560631 		sve_to_fpsimd(task);
561632 
562633 	if (task == current)
563
-		local_bh_enable();
634
+		put_cpu_fpsimd_context();
564635 
565636 	/*
566637 	 * Force reallocation of task SVE state to the correct size
567638 	 * on next use:
568639 	 */
569
-	sve_free(task);
640
+	mem = sve_free_atomic(task);
570641 
571642 	task->thread.sve_vl = vl;
643
+	kfree(mem);
572644 
573645 out:
574646 	update_tsk_thread_flag(task, TIF_SVE_VL_INHERIT,
....@@ -607,7 +679,7 @@
607679 	vl = arg & PR_SVE_VL_LEN_MASK;
608680 	flags = arg & ~vl;
609681 
610
-	if (!system_supports_sve())
682
+	if (!system_supports_sve() || is_compat_task())
611683 		return -EINVAL;
612684 
613685 	ret = sve_set_vector_length(current, vl, flags);
....@@ -620,17 +692,11 @@
620692 /* PR_SVE_GET_VL */
621693 int sve_get_current_vl(void)
622694 {
623
-	if (!system_supports_sve())
695
+	if (!system_supports_sve() || is_compat_task())
624696 		return -EINVAL;
625697 
626698 	return sve_prctl_status(0);
627699 }
628
-
629
-/*
630
- * Bitmap for temporary storage of the per-CPU set of supported vector lengths
631
- * during secondary boot.
632
- */
633
-static DECLARE_BITMAP(sve_secondary_vq_map, SVE_VQ_MAX);
634700 
635701 static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))
636702 {
....@@ -646,40 +712,82 @@
646712 		write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1); /* self-syncing */
647713 		vl = sve_get_vl();
648714 		vq = sve_vq_from_vl(vl); /* skip intervening lengths */
649
-		set_bit(vq_to_bit(vq), map);
715
+		set_bit(__vq_to_bit(vq), map);
650716 	}
651717 }
652718 
719
+/*
720
+ * Initialise the set of known supported VQs for the boot CPU.
721
+ * This is called during kernel boot, before secondary CPUs are brought up.
722
+ */
653723 void __init sve_init_vq_map(void)
654724 {
655725 	sve_probe_vqs(sve_vq_map);
726
+	bitmap_copy(sve_vq_partial_map, sve_vq_map, SVE_VQ_MAX);
656727 }
657728 
658729 /*
659730  * If we haven't committed to the set of supported VQs yet, filter out
660731  * those not supported by the current CPU.
732
+ * This function is called during the bring-up of early secondary CPUs only.
661733  */
662734 void sve_update_vq_map(void)
663735 {
664
-	sve_probe_vqs(sve_secondary_vq_map);
665
-	bitmap_and(sve_vq_map, sve_vq_map, sve_secondary_vq_map, SVE_VQ_MAX);
736
+	DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
737
+
738
+	sve_probe_vqs(tmp_map);
739
+	bitmap_and(sve_vq_map, sve_vq_map, tmp_map, SVE_VQ_MAX);
740
+	bitmap_or(sve_vq_partial_map, sve_vq_partial_map, tmp_map, SVE_VQ_MAX);
666741 }
667742 
668
-/* Check whether the current CPU supports all VQs in the committed set */
743
+/*
744
+ * Check whether the current CPU supports all VQs in the committed set.
745
+ * This function is called during the bring-up of late secondary CPUs only.
746
+ */
669747 int sve_verify_vq_map(void)
670748 {
671
-	int ret = 0;
749
+	DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
750
+	unsigned long b;
672751 
673
-	sve_probe_vqs(sve_secondary_vq_map);
674
-	bitmap_andnot(sve_secondary_vq_map, sve_vq_map, sve_secondary_vq_map,
675
-		      SVE_VQ_MAX);
676
-	if (!bitmap_empty(sve_secondary_vq_map, SVE_VQ_MAX)) {
752
+	sve_probe_vqs(tmp_map);
753
+
754
+	bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
755
+	if (bitmap_intersects(tmp_map, sve_vq_map, SVE_VQ_MAX)) {
677756 		pr_warn("SVE: cpu%d: Required vector length(s) missing\n",
678757 			smp_processor_id());
679
-		ret = -EINVAL;
758
+		return -EINVAL;
680759 	}
681760 
682
-	return ret;
761
+	if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available())
762
+		return 0;
763
+
764
+	/*
765
+	 * For KVM, it is necessary to ensure that this CPU doesn't
766
+	 * support any vector length that guests may have probed as
767
+	 * unsupported.
768
+	 */
769
+
770
+	/* Recover the set of supported VQs: */
771
+	bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
772
+	/* Find VQs supported that are not globally supported: */
773
+	bitmap_andnot(tmp_map, tmp_map, sve_vq_map, SVE_VQ_MAX);
774
+
775
+	/* Find the lowest such VQ, if any: */
776
+	b = find_last_bit(tmp_map, SVE_VQ_MAX);
777
+	if (b >= SVE_VQ_MAX)
778
+		return 0; /* no mismatches */
779
+
780
+	/*
781
+	 * Mismatches above sve_max_virtualisable_vl are fine, since
782
+	 * no guest is allowed to configure ZCR_EL2.LEN to exceed this:
783
+	 */
784
+	if (sve_vl_from_vq(__bit_to_vq(b)) <= sve_max_virtualisable_vl) {
785
+		pr_warn("SVE: cpu%d: Unsupported vector length(s) present\n",
786
+			smp_processor_id());
787
+		return -EINVAL;
788
+	}
789
+
790
+	return 0;
683791 }
684792 
685793 static void __init sve_efi_setup(void)
....@@ -746,6 +854,8 @@
746854 void __init sve_setup(void)
747855 {
748856 	u64 zcr;
857
+	DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
858
+	unsigned long b;
749859 
750860 	if (!system_supports_sve())
751861 		return;
....@@ -755,8 +865,8 @@
755865 	 * so sve_vq_map must have at least SVE_VQ_MIN set.
756866 	 * If something went wrong, at least try to patch it up:
757867 	 */
758
-	if (WARN_ON(!test_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
759
-		set_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map);
868
+	if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
869
+		set_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map);
760870 
761871 	zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
762872 	sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1);
....@@ -772,12 +882,32 @@
772882 	 * For the default VL, pick the maximum supported value <= 64.
773883 	 * VL == 64 is guaranteed not to grow the signal frame.
774884 	 */
775
-	sve_default_vl = find_supported_vector_length(64);
885
+	set_sve_default_vl(find_supported_vector_length(64));
886
+
887
+	bitmap_andnot(tmp_map, sve_vq_partial_map, sve_vq_map,
888
+		      SVE_VQ_MAX);
889
+
890
+	b = find_last_bit(tmp_map, SVE_VQ_MAX);
891
+	if (b >= SVE_VQ_MAX)
892
+		/* No non-virtualisable VLs found */
893
+		sve_max_virtualisable_vl = SVE_VQ_MAX;
894
+	else if (WARN_ON(b == SVE_VQ_MAX - 1))
895
+		/* No virtualisable VLs?  This is architecturally forbidden. */
896
+		sve_max_virtualisable_vl = SVE_VQ_MIN;
897
+	else /* b + 1 < SVE_VQ_MAX */
898
+		sve_max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1));
899
+
900
+	if (sve_max_virtualisable_vl > sve_max_vl)
901
+		sve_max_virtualisable_vl = sve_max_vl;
776902 
777903 	pr_info("SVE: maximum available vector length %u bytes per vector\n",
778904 		sve_max_vl);
779905 	pr_info("SVE: default vector length %u bytes per vector\n",
780
-		sve_default_vl);
906
+		get_sve_default_vl());
907
+
908
+	/* KVM decides whether to support mismatched systems. Just warn here: */
909
+	if (sve_max_virtualisable_vl < sve_max_vl)
910
+		pr_warn("SVE: unvirtualisable vector lengths present\n");
781911 
782912 	sve_efi_setup();
783913 }
....@@ -788,7 +918,9 @@
788918  */
789919 void fpsimd_release_task(struct task_struct *dead_task)
790920 {
791
-	__sve_free(dead_task);
921
+	void *mem = NULL;
922
+	mem = sve_free_atomic(dead_task);
923
+	kfree(mem);
792924 }
793925 
794926 #endif /* CONFIG_ARM64_SVE */
....@@ -801,39 +933,38 @@
801933  * the SVE access trap will be disabled the next time this task
802934  * reaches ret_to_user.
803935  *
804
- * TIF_SVE should be clear on entry: otherwise, task_fpsimd_load()
936
+ * TIF_SVE should be clear on entry: otherwise, fpsimd_restore_current_state()
805937  * would have disabled the SVE access trap for userspace during
806938  * ret_to_user, making an SVE access trap impossible in that case.
807939  */
808
-asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
940
+void do_sve_acc(unsigned int esr, struct pt_regs *regs)
809941 {
810942 	/* Even if we chose not to use SVE, the hardware could still trap: */
811943 	if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
812
-		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
944
+		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
813945 		return;
814946 	}
815947 
816948 	sve_alloc(current);
817949 
818
-	local_bh_disable();
950
+	get_cpu_fpsimd_context();
819951 
820952 	fpsimd_save();
821
-	fpsimd_to_sve(current);
822953 
823954 	/* Force ret_to_user to reload the registers: */
824955 	fpsimd_flush_task_state(current);
825
-	set_thread_flag(TIF_FOREIGN_FPSTATE);
826956 
957
+	fpsimd_to_sve(current);
827958 	if (test_and_set_thread_flag(TIF_SVE))
828959 		WARN_ON(1); /* SVE access shouldn't have trapped */
829960 
830
-	local_bh_enable();
961
+	put_cpu_fpsimd_context();
831962 }
832963 
833964 /*
834965  * Trapped FP/ASIMD access.
835966  */
836
-asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
967
+void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
837968 {
838969 	/* TODO: implement lazy context saving/restoring */
839970 	WARN_ON(1);
....@@ -842,9 +973,8 @@
842973 /*
843974  * Raise a SIGFPE for the current process.
844975  */
845
-asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
976
+void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
846977 {
847
-	siginfo_t info;
848978 	unsigned int si_code = FPE_FLTUNK;
849979 
850980 	if (esr & ESR_ELx_FP_EXC_TFV) {
....@@ -860,12 +990,9 @@
860990 			si_code = FPE_FLTRES;
861991 	}
862992 
863
-	clear_siginfo(&info);
864
-	info.si_signo = SIGFPE;
865
-	info.si_code = si_code;
866
-	info.si_addr = (void __user *)instruction_pointer(regs);
867
-
868
-	send_sig_info(SIGFPE, &info, current);
993
+	send_sig_fault(SIGFPE, si_code,
994
+		       (void __user *)instruction_pointer(regs),
995
+		       current);
869996 }
870997 
871998 void fpsimd_thread_switch(struct task_struct *next)
....@@ -874,6 +1001,8 @@
8741001 
8751002 	if (!system_supports_fpsimd())
8761003 		return;
1004
+
1005
+	__get_cpu_fpsimd_context();
8771006 
8781007 	/* Save unsaved fpsimd state, if any: */
8791008 	fpsimd_save();
....@@ -889,24 +1018,27 @@
8891018 
8901019 	update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
8911020 			       wrong_task || wrong_cpu);
1021
+
1022
+	__put_cpu_fpsimd_context();
8921023 }
8931024 
8941025 void fpsimd_flush_thread(void)
8951026 {
8961027 	int vl, supported_vl;
1028
+	void *mem = NULL;
8971029 
8981030 	if (!system_supports_fpsimd())
8991031 		return;
9001032 
901
-	local_bh_disable();
1033
+	get_cpu_fpsimd_context();
9021034 
1035
+	fpsimd_flush_task_state(current);
9031036 	memset(&current->thread.uw.fpsimd_state, 0,
9041037 	       sizeof(current->thread.uw.fpsimd_state));
905
-	fpsimd_flush_task_state(current);
9061038 
9071039 	if (system_supports_sve()) {
9081040 		clear_thread_flag(TIF_SVE);
909
-		sve_free(current);
1041
+		mem = sve_free_atomic(current);
9101042 
9111043 		/*
9121044 		 * Reset the task vector length as required.
....@@ -914,13 +1046,13 @@
9141046 		 * vector length configured: no kernel task can become a user
9151047 		 * task without an exec and hence a call to this function.
9161048 		 * By the time the first call to this function is made, all
917
-		 * early hardware probing is complete, so sve_default_vl
1049
+		 * early hardware probing is complete, so __sve_default_vl
9181050 		 * should be valid.
9191051 		 * If a bug causes this to go wrong, we make some noise and
9201052 		 * try to fudge thread.sve_vl to a safe value here.
9211053 		 */
9221054 		vl = current->thread.sve_vl_onexec ?
923
-			current->thread.sve_vl_onexec : sve_default_vl;
1055
+			current->thread.sve_vl_onexec : get_sve_default_vl();
9241056 
9251057 		if (WARN_ON(!sve_vl_valid(vl)))
9261058 			vl = SVE_VL_MIN;
....@@ -939,9 +1071,8 @@
9391071 			current->thread.sve_vl_onexec = 0;
9401072 	}
9411073 
942
-	set_thread_flag(TIF_FOREIGN_FPSTATE);
943
-
944
-	local_bh_enable();
1074
+	put_cpu_fpsimd_context();
1075
+	kfree(mem);
9451076 }
9461077 
9471078 /*
....@@ -953,9 +1084,9 @@
9531084 	if (!system_supports_fpsimd())
9541085 		return;
9551086 
956
-	local_bh_disable();
1087
+	get_cpu_fpsimd_context();
9571088 	fpsimd_save();
958
-	local_bh_enable();
1089
+	put_cpu_fpsimd_context();
9591090 }
9601091 
9611092 /*
....@@ -972,7 +1103,8 @@
9721103 
9731104 /*
9741105  * Associate current's FPSIMD context with this cpu
975
- * Preemption must be disabled when calling this function.
1106
+ * The caller must have ownership of the cpu FPSIMD context before calling
1107
+ * this function.
9761108  */
9771109 void fpsimd_bind_task_to_cpu(void)
9781110 {
....@@ -981,6 +1113,8 @@
9811113 
9821114 	WARN_ON(!system_supports_fpsimd());
9831115 	last->st = &current->thread.uw.fpsimd_state;
1116
+	last->sve_state = current->thread.sve_state;
1117
+	last->sve_vl = current->thread.sve_vl;
9841118 	current->thread.fpsimd_cpu = smp_processor_id();
9851119 
9861120 	if (system_supports_sve()) {
....@@ -994,7 +1128,8 @@
9941128 	}
9951129 }
9961130 
997
-void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st)
1131
+void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
1132
+			      unsigned int sve_vl)
9981133 {
9991134 	struct fpsimd_last_state_struct *last =
10001135 		this_cpu_ptr(&fpsimd_last_state);
....@@ -1003,6 +1138,8 @@
10031138 	WARN_ON(!in_softirq() && !irqs_disabled());
10041139 
10051140 	last->st = st;
1141
+	last->sve_state = sve_state;
1142
+	last->sve_vl = sve_vl;
10061143 }
10071144 
10081145 /*
....@@ -1026,14 +1163,14 @@
10261163 		return;
10271164 	}
10281165 
1029
-	local_bh_disable();
1166
+	get_cpu_fpsimd_context();
10301167 
10311168 	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
10321169 		task_fpsimd_load();
10331170 		fpsimd_bind_task_to_cpu();
10341171 	}
10351172 
1036
-	local_bh_enable();
1173
+	put_cpu_fpsimd_context();
10371174 }
10381175 
10391176 /*
....@@ -1046,7 +1183,7 @@
10461183 	if (WARN_ON(!system_supports_fpsimd()))
10471184 		return;
10481185 
1049
-	local_bh_disable();
1186
+	get_cpu_fpsimd_context();
10501187 
10511188 	current->thread.uw.fpsimd_state = *state;
10521189 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
....@@ -1057,28 +1194,64 @@
10571194 
10581195 	clear_thread_flag(TIF_FOREIGN_FPSTATE);
10591196 
1060
-	local_bh_enable();
1197
+	put_cpu_fpsimd_context();
10611198 }
10621199 
10631200 /*
10641201  * Invalidate live CPU copies of task t's FPSIMD state
1202
+ *
1203
+ * This function may be called with preemption enabled.  The barrier()
1204
+ * ensures that the assignment to fpsimd_cpu is visible to any
1205
+ * preemption/softirq that could race with set_tsk_thread_flag(), so
1206
+ * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared.
1207
+ *
1208
+ * The final barrier ensures that TIF_FOREIGN_FPSTATE is seen set by any
1209
+ * subsequent code.
10651210  */
10661211 void fpsimd_flush_task_state(struct task_struct *t)
10671212 {
10681213 	t->thread.fpsimd_cpu = NR_CPUS;
1214
+	/*
1215
+	 * If we don't support fpsimd, bail out after we have
1216
+	 * reset the fpsimd_cpu for this task and clear the
1217
+	 * FPSTATE.
1218
+	 */
1219
+	if (!system_supports_fpsimd())
1220
+		return;
1221
+	barrier();
1222
+	set_tsk_thread_flag(t, TIF_FOREIGN_FPSTATE);
1223
+
1224
+	barrier();
10691225 }
10701226 
1071
-void fpsimd_flush_cpu_state(void)
1227
+/*
1228
+ * Invalidate any task's FPSIMD state that is present on this cpu.
1229
+ * The FPSIMD context should be acquired with get_cpu_fpsimd_context()
1230
+ * before calling this function.
1231
+ */
1232
+static void fpsimd_flush_cpu_state(void)
10721233 {
10731234 	WARN_ON(!system_supports_fpsimd());
10741235 	__this_cpu_write(fpsimd_last_state.st, NULL);
10751236 	set_thread_flag(TIF_FOREIGN_FPSTATE);
10761237 }
10771238 
1078
-#ifdef CONFIG_KERNEL_MODE_NEON
1239
+/*
1240
+ * Save the FPSIMD state to memory and invalidate cpu view.
1241
+ * This function must be called with preemption disabled.
1242
+ */
1243
+void fpsimd_save_and_flush_cpu_state(void)
1244
+{
1245
+	if (!system_supports_fpsimd())
1246
+		return;
1247
+	WARN_ON(preemptible());
1248
+	__get_cpu_fpsimd_context();
1249
+	fpsimd_save();
1250
+	fpsimd_flush_cpu_state();
1251
+	__put_cpu_fpsimd_context();
1252
+}
10791253 
1080
-DEFINE_PER_CPU(bool, kernel_neon_busy);
1081
-EXPORT_PER_CPU_SYMBOL(kernel_neon_busy);
1254
+#ifdef CONFIG_KERNEL_MODE_NEON
10821255 
10831256 /*
10841257  * Kernel-side NEON support functions
....@@ -1104,19 +1277,13 @@
11041277 
11051278 	BUG_ON(!may_use_simd());
11061279 
1107
-	local_bh_disable();
1108
-
1109
-	__this_cpu_write(kernel_neon_busy, true);
1280
+	get_cpu_fpsimd_context();
11101281 
11111282 	/* Save unsaved fpsimd state, if any: */
11121283 	fpsimd_save();
11131284 
11141285 	/* Invalidate any task state remaining in the fpsimd regs: */
11151286 	fpsimd_flush_cpu_state();
1116
-
1117
-	preempt_disable();
1118
-
1119
-	local_bh_enable();
11201287 }
11211288 EXPORT_SYMBOL(kernel_neon_begin);
11221289 
....@@ -1131,15 +1298,10 @@
11311298  */
11321299 void kernel_neon_end(void)
11331300 {
1134
-	bool busy;
1135
-
11361301 	if (!system_supports_fpsimd())
11371302 		return;
11381303 
1139
-	busy = __this_cpu_xchg(kernel_neon_busy, false);
1140
-	WARN_ON(!busy);	/* No matching kernel_neon_begin()? */
1141
-
1142
-	preempt_enable();
1304
+	put_cpu_fpsimd_context();
11431305 }
11441306 EXPORT_SYMBOL(kernel_neon_end);
11451307 
....@@ -1231,8 +1393,7 @@
12311393 {
12321394 	switch (cmd) {
12331395 	case CPU_PM_ENTER:
1234
-		fpsimd_save();
1235
-		fpsimd_flush_cpu_state();
1396
+		fpsimd_save_and_flush_cpu_state();
12361397 		break;
12371398 	case CPU_PM_EXIT:
12381399 		break;
....@@ -1278,14 +1439,14 @@
12781439  */
12791440 static int __init fpsimd_init(void)
12801441 {
1281
-	if (elf_hwcap & HWCAP_FP) {
1442
+	if (cpu_have_named_feature(FP)) {
12821443 		fpsimd_pm_init();
12831444 		fpsimd_hotplug_init();
12841445 	} else {
12851446 		pr_notice("Floating-point is not implemented\n");
12861447 	}
12871448 
1288
-	if (!(elf_hwcap & HWCAP_ASIMD))
1449
+	if (!cpu_have_named_feature(ASIMD))
12891450 		pr_notice("Advanced SIMD is not implemented\n");
12901451 
12911452 	return sve_sysctl_init();