forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-10 10ebd8556b7990499c896a550e3d416b444211e6 
 kernel/arch/arm64/kernel/fpsimd.c
....@@ -1,27 +1,18 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * FP/SIMD context switching and fault handling
34  *
45  * Copyright (C) 2012 ARM Ltd.
56  * Author: Catalin Marinas <catalin.marinas@arm.com>
6
- *
7
- * This program is free software; you can redistribute it and/or modify
8
- * it under the terms of the GNU General Public License version 2 as
9
- * published by the Free Software Foundation.
10
- *
11
- * This program is distributed in the hope that it will be useful,
12
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
- * GNU General Public License for more details.
15
- *
16
- * You should have received a copy of the GNU General Public License
17
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
187  */
198 
209 #include <linux/bitmap.h>
10
+#include <linux/bitops.h>
2111 #include <linux/bottom_half.h>
2212 #include <linux/bug.h>
2313 #include <linux/cache.h>
2414 #include <linux/compat.h>
15
+#include <linux/compiler.h>
2516 #include <linux/cpu.h>
2617 #include <linux/cpu_pm.h>
2718 #include <linux/kernel.h>
....@@ -38,16 +29,20 @@
3829 #include <linux/slab.h>
3930 #include <linux/stddef.h>
4031 #include <linux/sysctl.h>
32
+#include <linux/swab.h>
4133 
4234 #include <asm/esr.h>
35
+#include <asm/exception.h>
4336 #include <asm/fpsimd.h>
4437 #include <asm/cpufeature.h>
4538 #include <asm/cputype.h>
39
+#include <asm/neon.h>
4640 #include <asm/processor.h>
4741 #include <asm/simd.h>
4842 #include <asm/sigcontext.h>
4943 #include <asm/sysreg.h>
5044 #include <asm/traps.h>
45
+#include <asm/virt.h>
5146 
5247 #define FPEXC_IOF	(1 << 0)
5348 #define FPEXC_DZF	(1 << 1)
....@@ -90,7 +85,8 @@
9085  * To prevent this from racing with the manipulation of the task's FPSIMD state
9186  * from task context and thereby corrupting the state, it is necessary to
9287  * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE
93
- * flag with local_bh_disable() unless softirqs are already masked.
88
+ * flag with {, __}get_cpu_fpsimd_context(). This will still allow softirqs to
89
+ * run but prevent them to use FPSIMD.
9490  *
9591  * For a certain task, the sequence may look something like this:
9692  * - the task gets scheduled in; if both the task's fpsimd_cpu field
....@@ -119,28 +115,99 @@
119115  */
120116 struct fpsimd_last_state_struct {
121117 	struct user_fpsimd_state *st;
118
+	void *sve_state;
119
+	unsigned int sve_vl;
122120 };
123121 
124122 static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
125123 
126124 /* Default VL for tasks that don't set it explicitly: */
127
-static int sve_default_vl = -1;
125
+static int __sve_default_vl = -1;
126
+
127
+static int get_sve_default_vl(void)
128
+{
129
+	return READ_ONCE(__sve_default_vl);
130
+}
128131 
129132 #ifdef CONFIG_ARM64_SVE
130133 
134
+static void set_sve_default_vl(int val)
135
+{
136
+	WRITE_ONCE(__sve_default_vl, val);
137
+}
138
+
131139 /* Maximum supported vector length across all CPUs (initially poisoned) */
132140 int __ro_after_init sve_max_vl = SVE_VL_MIN;
133
-/* Set of available vector lengths, as vq_to_bit(vq): */
134
-static __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
141
+int __ro_after_init sve_max_virtualisable_vl = SVE_VL_MIN;
142
+
143
+/*
144
+ * Set of available vector lengths,
145
+ * where length vq encoded as bit __vq_to_bit(vq):
146
+ */
147
+__ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
148
+/* Set of vector lengths present on at least one cpu: */
149
+static __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
150
+
135151 static void __percpu *efi_sve_state;
136152 
137153 #else /* ! CONFIG_ARM64_SVE */
138154 
139155 /* Dummy declaration for code that will be optimised out: */
140156 extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
157
+extern __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
141158 extern void __percpu *efi_sve_state;
142159 
143160 #endif /* ! CONFIG_ARM64_SVE */
161
+
162
+DEFINE_PER_CPU(bool, fpsimd_context_busy);
163
+EXPORT_PER_CPU_SYMBOL(fpsimd_context_busy);
164
+
165
+static void __get_cpu_fpsimd_context(void)
166
+{
167
+	bool busy = __this_cpu_xchg(fpsimd_context_busy, true);
168
+
169
+	WARN_ON(busy);
170
+}
171
+
172
+/*
173
+ * Claim ownership of the CPU FPSIMD context for use by the calling context.
174
+ *
175
+ * The caller may freely manipulate the FPSIMD context metadata until
176
+ * put_cpu_fpsimd_context() is called.
177
+ *
178
+ * The double-underscore version must only be called if you know the task
179
+ * can't be preempted.
180
+ */
181
+static void get_cpu_fpsimd_context(void)
182
+{
183
+	local_bh_disable();
184
+	__get_cpu_fpsimd_context();
185
+}
186
+
187
+static void __put_cpu_fpsimd_context(void)
188
+{
189
+	bool busy = __this_cpu_xchg(fpsimd_context_busy, false);
190
+
191
+	WARN_ON(!busy); /* No matching get_cpu_fpsimd_context()? */
192
+}
193
+
194
+/*
195
+ * Release the CPU FPSIMD context.
196
+ *
197
+ * Must be called from a context in which get_cpu_fpsimd_context() was
198
+ * previously called, with no call to put_cpu_fpsimd_context() in the
199
+ * meantime.
200
+ */
201
+static void put_cpu_fpsimd_context(void)
202
+{
203
+	__put_cpu_fpsimd_context();
204
+	local_bh_enable();
205
+}
206
+
207
+static bool have_cpu_fpsimd_context(void)
208
+{
209
+	return !preemptible() && __this_cpu_read(fpsimd_context_busy);
210
+}
144211 
145212 /*
146213  * Call __sve_free() directly only if you know task can't be scheduled
....@@ -157,16 +224,6 @@
157224 	WARN_ON(test_tsk_thread_flag(task, TIF_SVE));
158225 
159226 	__sve_free(task);
160
-}
161
-
162
-static void *sve_free_atomic(struct task_struct *task)
163
-{
164
-	void *sve_state = task->thread.sve_state;
165
-
166
-	WARN_ON(test_tsk_thread_flag(task, TIF_SVE));
167
-
168
-	task->thread.sve_state = NULL;
169
-	return sve_state;
170227 }
171228 
172229 /*
....@@ -222,13 +279,11 @@
222279  * This function should be called only when the FPSIMD/SVE state in
223280  * thread_struct is known to be up to date, when preparing to enter
224281  * userspace.
225
- *
226
- * Softirqs (and preemption) must be disabled.
227282  */
228283 static void task_fpsimd_load(void)
229284 {
230
-	WARN_ON(!in_softirq() && !irqs_disabled());
231285 	WARN_ON(!system_supports_fpsimd());
286
+	WARN_ON(!have_cpu_fpsimd_context());
232287 
233288 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
234289 		sve_load_state(sve_pffr(&current->thread),
....@@ -241,52 +296,34 @@
241296 /*
242297  * Ensure FPSIMD/SVE storage in memory for the loaded context is up to
243298  * date with respect to the CPU registers.
244
- *
245
- * Softirqs (and preemption) must be disabled.
246299  */
247
-void fpsimd_save(void)
300
+static void fpsimd_save(void)
248301 {
249
-	struct user_fpsimd_state *st = __this_cpu_read(fpsimd_last_state.st);
302
+	struct fpsimd_last_state_struct const *last =
303
+		this_cpu_ptr(&fpsimd_last_state);
250304 	/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
251305 
252306 	WARN_ON(!system_supports_fpsimd());
253
-	WARN_ON(!in_softirq() && !irqs_disabled());
307
+	WARN_ON(!have_cpu_fpsimd_context());
254308 
255309 	if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
256310 		if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
257
-			if (WARN_ON(sve_get_vl() != current->thread.sve_vl)) {
311
+			if (WARN_ON(sve_get_vl() != last->sve_vl)) {
258312 				/*
259313 				 * Can't save the user regs, so current would
260314 				 * re-enter user with corrupt state.
261315 				 * There's no way to recover, so kill it:
262316 				 */
263
-				force_signal_inject(SIGKILL, SI_KERNEL, 0);
317
+				force_signal_inject(SIGKILL, SI_KERNEL, 0, 0);
264318 				return;
265319 			}
266320 
267
-			sve_save_state(sve_pffr(&current->thread), &st->fpsr);
321
+			sve_save_state((char *)last->sve_state +
322
+						sve_ffr_offset(last->sve_vl),
323
+				       &last->st->fpsr);
268324 		} else
269
-			fpsimd_save_state(st);
325
+			fpsimd_save_state(last->st);
270326 	}
271
-}
272
-
273
-/*
274
- * Helpers to translate bit indices in sve_vq_map to VQ values (and
275
- * vice versa).  This allows find_next_bit() to be used to find the
276
- * _maximum_ VQ not exceeding a certain value.
277
- */
278
-
279
-static unsigned int vq_to_bit(unsigned int vq)
280
-{
281
-	return SVE_VQ_MAX - vq;
282
-}
283
-
284
-static unsigned int bit_to_vq(unsigned int bit)
285
-{
286
-	if (WARN_ON(bit >= SVE_VQ_MAX))
287
-		bit = SVE_VQ_MAX - 1;
288
-
289
-	return SVE_VQ_MAX - bit;
290327 }
291328 
292329 /*
....@@ -310,18 +347,17 @@
310347 		vl = max_vl;
311348 
312349 	bit = find_next_bit(sve_vq_map, SVE_VQ_MAX,
313
-			    vq_to_bit(sve_vq_from_vl(vl)));
314
-	return sve_vl_from_vq(bit_to_vq(bit));
350
+			    __vq_to_bit(sve_vq_from_vl(vl)));
351
+	return sve_vl_from_vq(__bit_to_vq(bit));
315352 }
316353 
317354 #if defined(CONFIG_ARM64_SVE) && defined(CONFIG_SYSCTL)
318355 
319356 static int sve_proc_do_default_vl(struct ctl_table *table, int write,
320
-				  void __user *buffer, size_t *lenp,
321
-				  loff_t *ppos)
357
+				  void *buffer, size_t *lenp, loff_t *ppos)
322358 {
323359 	int ret;
324
-	int vl = sve_default_vl;
360
+	int vl = get_sve_default_vl();
325361 	struct ctl_table tmp_table = {
326362 		.data = &vl,
327363 		.maxlen = sizeof(vl),
....@@ -338,7 +374,7 @@
338374 	if (!sve_vl_valid(vl))
339375 		return -EINVAL;
340376 
341
-	sve_default_vl = find_supported_vector_length(vl);
377
+	set_sve_default_vl(find_supported_vector_length(vl));
342378 	return 0;
343379 }
344380 
....@@ -367,12 +403,42 @@
367403 #define ZREG(sve_state, vq, n) ((char *)(sve_state) +		\
368404 	(SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
369405 
406
+#ifdef CONFIG_CPU_BIG_ENDIAN
407
+static __uint128_t arm64_cpu_to_le128(__uint128_t x)
408
+{
409
+	u64 a = swab64(x);
410
+	u64 b = swab64(x >> 64);
411
+
412
+	return ((__uint128_t)a << 64) | b;
413
+}
414
+#else
415
+static __uint128_t arm64_cpu_to_le128(__uint128_t x)
416
+{
417
+	return x;
418
+}
419
+#endif
420
+
421
+#define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x)
422
+
423
+static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
424
+			    unsigned int vq)
425
+{
426
+	unsigned int i;
427
+	__uint128_t *p;
428
+
429
+	for (i = 0; i < SVE_NUM_ZREGS; ++i) {
430
+		p = (__uint128_t *)ZREG(sst, vq, i);
431
+		*p = arm64_cpu_to_le128(fst->vregs[i]);
432
+	}
433
+}
434
+
370435 /*
371436  * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to
372437  * task->thread.sve_state.
373438  *
374439  * Task can be a non-runnable task, or current.  In the latter case,
375
- * softirqs (and preemption) must be disabled.
440
+ * the caller must have ownership of the cpu FPSIMD context before calling
441
+ * this function.
376442  * task->thread.sve_state must point to at least sve_state_size(task)
377443  * bytes of allocated kernel memory.
378444  * task->thread.uw.fpsimd_state must be up to date before calling this
....@@ -383,15 +449,12 @@
383449 	unsigned int vq;
384450 	void *sst = task->thread.sve_state;
385451 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
386
-	unsigned int i;
387452 
388453 	if (!system_supports_sve())
389454 		return;
390455 
391456 	vq = sve_vq_from_vl(task->thread.sve_vl);
392
-	for (i = 0; i < 32; ++i)
393
-		memcpy(ZREG(sst, vq, i), &fst->vregs[i],
394
-		       sizeof(fst->vregs[i]));
457
+	__fpsimd_to_sve(sst, fst, vq);
395458 }
396459 
397460 /*
....@@ -399,7 +462,8 @@
399462  * task->thread.uw.fpsimd_state.
400463  *
401464  * Task can be a non-runnable task, or current.  In the latter case,
402
- * softirqs (and preemption) must be disabled.
465
+ * the caller must have ownership of the cpu FPSIMD context before calling
466
+ * this function.
403467  * task->thread.sve_state must point to at least sve_state_size(task)
404468  * bytes of allocated kernel memory.
405469  * task->thread.sve_state must be up to date before calling this function.
....@@ -410,14 +474,16 @@
410474 	void const *sst = task->thread.sve_state;
411475 	struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
412476 	unsigned int i;
477
+	__uint128_t const *p;
413478 
414479 	if (!system_supports_sve())
415480 		return;
416481 
417482 	vq = sve_vq_from_vl(task->thread.sve_vl);
418
-	for (i = 0; i < 32; ++i)
419
-		memcpy(&fst->vregs[i], ZREG(sst, vq, i),
420
-		       sizeof(fst->vregs[i]));
483
+	for (i = 0; i < SVE_NUM_ZREGS; ++i) {
484
+		p = (__uint128_t const *)ZREG(sst, vq, i);
485
+		fst->vregs[i] = arm64_le128_to_cpu(*p);
486
+	}
421487 }
422488 
423489 #ifdef CONFIG_ARM64_SVE
....@@ -505,7 +571,6 @@
505571 	unsigned int vq;
506572 	void *sst = task->thread.sve_state;
507573 	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
508
-	unsigned int i;
509574 
510575 	if (!test_tsk_thread_flag(task, TIF_SVE))
511576 		return;
....@@ -513,10 +578,7 @@
513578 	vq = sve_vq_from_vl(task->thread.sve_vl);
514579 
515580 	memset(sst, 0, SVE_SIG_REGS_SIZE(vq));
516
-
517
-	for (i = 0; i < 32; ++i)
518
-		memcpy(ZREG(sst, vq, i), &fst->vregs[i],
519
-		       sizeof(fst->vregs[i]));
581
+	__fpsimd_to_sve(sst, fst, vq);
520582 }
521583 
522584 int sve_set_vector_length(struct task_struct *task,
....@@ -559,21 +621,17 @@
559621 	 * non-SVE thread.
560622 	 */
561623 	if (task == current) {
562
-		preempt_disable();
563
-		local_bh_disable();
624
+		get_cpu_fpsimd_context();
564625 
565626 		fpsimd_save();
566
-		set_thread_flag(TIF_FOREIGN_FPSTATE);
567627 	}
568628 
569629 	fpsimd_flush_task_state(task);
570630 	if (test_and_clear_tsk_thread_flag(task, TIF_SVE))
571631 		sve_to_fpsimd(task);
572632 
573
-	if (task == current) {
574
-		local_bh_enable();
575
-		preempt_enable();
576
-	}
633
+	if (task == current)
634
+		put_cpu_fpsimd_context();
577635 
578636 	/*
579637 	 * Force reallocation of task SVE state to the correct size
....@@ -620,7 +678,7 @@
620678 	vl = arg & PR_SVE_VL_LEN_MASK;
621679 	flags = arg & ~vl;
622680 
623
-	if (!system_supports_sve())
681
+	if (!system_supports_sve() || is_compat_task())
624682 		return -EINVAL;
625683 
626684 	ret = sve_set_vector_length(current, vl, flags);
....@@ -633,17 +691,11 @@
633691 /* PR_SVE_GET_VL */
634692 int sve_get_current_vl(void)
635693 {
636
-	if (!system_supports_sve())
694
+	if (!system_supports_sve() || is_compat_task())
637695 		return -EINVAL;
638696 
639697 	return sve_prctl_status(0);
640698 }
641
-
642
-/*
643
- * Bitmap for temporary storage of the per-CPU set of supported vector lengths
644
- * during secondary boot.
645
- */
646
-static DECLARE_BITMAP(sve_secondary_vq_map, SVE_VQ_MAX);
647699 
648700 static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))
649701 {
....@@ -659,40 +711,82 @@
659711 		write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1); /* self-syncing */
660712 		vl = sve_get_vl();
661713 		vq = sve_vq_from_vl(vl); /* skip intervening lengths */
662
-		set_bit(vq_to_bit(vq), map);
714
+		set_bit(__vq_to_bit(vq), map);
663715 	}
664716 }
665717 
718
+/*
719
+ * Initialise the set of known supported VQs for the boot CPU.
720
+ * This is called during kernel boot, before secondary CPUs are brought up.
721
+ */
666722 void __init sve_init_vq_map(void)
667723 {
668724 	sve_probe_vqs(sve_vq_map);
725
+	bitmap_copy(sve_vq_partial_map, sve_vq_map, SVE_VQ_MAX);
669726 }
670727 
671728 /*
672729  * If we haven't committed to the set of supported VQs yet, filter out
673730  * those not supported by the current CPU.
731
+ * This function is called during the bring-up of early secondary CPUs only.
674732  */
675733 void sve_update_vq_map(void)
676734 {
677
-	sve_probe_vqs(sve_secondary_vq_map);
678
-	bitmap_and(sve_vq_map, sve_vq_map, sve_secondary_vq_map, SVE_VQ_MAX);
735
+	DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
736
+
737
+	sve_probe_vqs(tmp_map);
738
+	bitmap_and(sve_vq_map, sve_vq_map, tmp_map, SVE_VQ_MAX);
739
+	bitmap_or(sve_vq_partial_map, sve_vq_partial_map, tmp_map, SVE_VQ_MAX);
679740 }
680741 
681
-/* Check whether the current CPU supports all VQs in the committed set */
742
+/*
743
+ * Check whether the current CPU supports all VQs in the committed set.
744
+ * This function is called during the bring-up of late secondary CPUs only.
745
+ */
682746 int sve_verify_vq_map(void)
683747 {
684
-	int ret = 0;
748
+	DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
749
+	unsigned long b;
685750 
686
-	sve_probe_vqs(sve_secondary_vq_map);
687
-	bitmap_andnot(sve_secondary_vq_map, sve_vq_map, sve_secondary_vq_map,
688
-		      SVE_VQ_MAX);
689
-	if (!bitmap_empty(sve_secondary_vq_map, SVE_VQ_MAX)) {
751
+	sve_probe_vqs(tmp_map);
752
+
753
+	bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
754
+	if (bitmap_intersects(tmp_map, sve_vq_map, SVE_VQ_MAX)) {
690755 		pr_warn("SVE: cpu%d: Required vector length(s) missing\n",
691756 			smp_processor_id());
692
-		ret = -EINVAL;
757
+		return -EINVAL;
693758 	}
694759 
695
-	return ret;
760
+	if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available())
761
+		return 0;
762
+
763
+	/*
764
+	 * For KVM, it is necessary to ensure that this CPU doesn't
765
+	 * support any vector length that guests may have probed as
766
+	 * unsupported.
767
+	 */
768
+
769
+	/* Recover the set of supported VQs: */
770
+	bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
771
+	/* Find VQs supported that are not globally supported: */
772
+	bitmap_andnot(tmp_map, tmp_map, sve_vq_map, SVE_VQ_MAX);
773
+
774
+	/* Find the lowest such VQ, if any: */
775
+	b = find_last_bit(tmp_map, SVE_VQ_MAX);
776
+	if (b >= SVE_VQ_MAX)
777
+		return 0; /* no mismatches */
778
+
779
+	/*
780
+	 * Mismatches above sve_max_virtualisable_vl are fine, since
781
+	 * no guest is allowed to configure ZCR_EL2.LEN to exceed this:
782
+	 */
783
+	if (sve_vl_from_vq(__bit_to_vq(b)) <= sve_max_virtualisable_vl) {
784
+		pr_warn("SVE: cpu%d: Unsupported vector length(s) present\n",
785
+			smp_processor_id());
786
+		return -EINVAL;
787
+	}
788
+
789
+	return 0;
696790 }
697791 
698792 static void __init sve_efi_setup(void)
....@@ -759,6 +853,8 @@
759853 void __init sve_setup(void)
760854 {
761855 	u64 zcr;
856
+	DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
857
+	unsigned long b;
762858 
763859 	if (!system_supports_sve())
764860 		return;
....@@ -768,8 +864,8 @@
768864 	 * so sve_vq_map must have at least SVE_VQ_MIN set.
769865 	 * If something went wrong, at least try to patch it up:
770866 	 */
771
-	if (WARN_ON(!test_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
772
-		set_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map);
867
+	if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
868
+		set_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map);
773869 
774870 	zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
775871 	sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1);
....@@ -785,12 +881,32 @@
785881 	 * For the default VL, pick the maximum supported value <= 64.
786882 	 * VL == 64 is guaranteed not to grow the signal frame.
787883 	 */
788
-	sve_default_vl = find_supported_vector_length(64);
884
+	set_sve_default_vl(find_supported_vector_length(64));
885
+
886
+	bitmap_andnot(tmp_map, sve_vq_partial_map, sve_vq_map,
887
+		      SVE_VQ_MAX);
888
+
889
+	b = find_last_bit(tmp_map, SVE_VQ_MAX);
890
+	if (b >= SVE_VQ_MAX)
891
+		/* No non-virtualisable VLs found */
892
+		sve_max_virtualisable_vl = SVE_VQ_MAX;
893
+	else if (WARN_ON(b == SVE_VQ_MAX - 1))
894
+		/* No virtualisable VLs?  This is architecturally forbidden. */
895
+		sve_max_virtualisable_vl = SVE_VQ_MIN;
896
+	else /* b + 1 < SVE_VQ_MAX */
897
+		sve_max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1));
898
+
899
+	if (sve_max_virtualisable_vl > sve_max_vl)
900
+		sve_max_virtualisable_vl = sve_max_vl;
789901 
790902 	pr_info("SVE: maximum available vector length %u bytes per vector\n",
791903 		sve_max_vl);
792904 	pr_info("SVE: default vector length %u bytes per vector\n",
793
-		sve_default_vl);
905
+		get_sve_default_vl());
906
+
907
+	/* KVM decides whether to support mismatched systems. Just warn here: */
908
+	if (sve_max_virtualisable_vl < sve_max_vl)
909
+		pr_warn("SVE: unvirtualisable vector lengths present\n");
794910 
795911 	sve_efi_setup();
796912 }
....@@ -814,41 +930,38 @@
814930  * the SVE access trap will be disabled the next time this task
815931  * reaches ret_to_user.
816932  *
817
- * TIF_SVE should be clear on entry: otherwise, task_fpsimd_load()
933
+ * TIF_SVE should be clear on entry: otherwise, fpsimd_restore_current_state()
818934  * would have disabled the SVE access trap for userspace during
819935  * ret_to_user, making an SVE access trap impossible in that case.
820936  */
821
-asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
937
+void do_sve_acc(unsigned int esr, struct pt_regs *regs)
822938 {
823939 	/* Even if we chose not to use SVE, the hardware could still trap: */
824940 	if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
825
-		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
941
+		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
826942 		return;
827943 	}
828944 
829945 	sve_alloc(current);
830946 
831
-	preempt_disable();
832
-	local_bh_disable();
947
+	get_cpu_fpsimd_context();
833948 
834949 	fpsimd_save();
835
-	fpsimd_to_sve(current);
836950 
837951 	/* Force ret_to_user to reload the registers: */
838952 	fpsimd_flush_task_state(current);
839
-	set_thread_flag(TIF_FOREIGN_FPSTATE);
840953 
954
+	fpsimd_to_sve(current);
841955 	if (test_and_set_thread_flag(TIF_SVE))
842956 		WARN_ON(1); /* SVE access shouldn't have trapped */
843957 
844
-	local_bh_enable();
845
-	preempt_enable();
958
+	put_cpu_fpsimd_context();
846959 }
847960 
848961 /*
849962  * Trapped FP/ASIMD access.
850963  */
851
-asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
964
+void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
852965 {
853966 	/* TODO: implement lazy context saving/restoring */
854967 	WARN_ON(1);
....@@ -857,9 +970,8 @@
857970 /*
858971  * Raise a SIGFPE for the current process.
859972  */
860
-asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
973
+void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
861974 {
862
-	siginfo_t info;
863975 	unsigned int si_code = FPE_FLTUNK;
864976 
865977 	if (esr & ESR_ELx_FP_EXC_TFV) {
....@@ -875,12 +987,9 @@
875987 			si_code = FPE_FLTRES;
876988 	}
877989 
878
-	clear_siginfo(&info);
879
-	info.si_signo = SIGFPE;
880
-	info.si_code = si_code;
881
-	info.si_addr = (void __user *)instruction_pointer(regs);
882
-
883
-	send_sig_info(SIGFPE, &info, current);
990
+	send_sig_fault(SIGFPE, si_code,
991
+		       (void __user *)instruction_pointer(regs),
992
+		       current);
884993 }
885994 
886995 void fpsimd_thread_switch(struct task_struct *next)
....@@ -889,6 +998,8 @@
889998 
890999 	if (!system_supports_fpsimd())
8911000 		return;
1001
+
1002
+	__get_cpu_fpsimd_context();
8921003 
8931004 	/* Save unsaved fpsimd state, if any: */
8941005 	fpsimd_save();
....@@ -904,26 +1015,26 @@
9041015 
9051016 	update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
9061017 			       wrong_task || wrong_cpu);
1018
+
1019
+	__put_cpu_fpsimd_context();
9071020 }
9081021 
9091022 void fpsimd_flush_thread(void)
9101023 {
9111024 	int vl, supported_vl;
912
-	void *mem = NULL;
9131025 
9141026 	if (!system_supports_fpsimd())
9151027 		return;
9161028 
917
-	preempt_disable();
918
-	local_bh_disable();
1029
+	get_cpu_fpsimd_context();
9191030 
1031
+	fpsimd_flush_task_state(current);
9201032 	memset(&current->thread.uw.fpsimd_state, 0,
9211033 	       sizeof(current->thread.uw.fpsimd_state));
922
-	fpsimd_flush_task_state(current);
9231034 
9241035 	if (system_supports_sve()) {
9251036 		clear_thread_flag(TIF_SVE);
926
-		mem = sve_free_atomic(current);
1037
+		sve_free(current);
9271038 
9281039 		/*
9291040 		 * Reset the task vector length as required.
....@@ -931,13 +1042,13 @@
9311042 		 * vector length configured: no kernel task can become a user
9321043 		 * task without an exec and hence a call to this function.
9331044 		 * By the time the first call to this function is made, all
934
-		 * early hardware probing is complete, so sve_default_vl
1045
+		 * early hardware probing is complete, so __sve_default_vl
9351046 		 * should be valid.
9361047 		 * If a bug causes this to go wrong, we make some noise and
9371048 		 * try to fudge thread.sve_vl to a safe value here.
9381049 		 */
9391050 		vl = current->thread.sve_vl_onexec ?
940
-			current->thread.sve_vl_onexec : sve_default_vl;
1051
+			current->thread.sve_vl_onexec : get_sve_default_vl();
9411052 
9421053 		if (WARN_ON(!sve_vl_valid(vl)))
9431054 			vl = SVE_VL_MIN;
....@@ -956,11 +1067,7 @@
9561067 			current->thread.sve_vl_onexec = 0;
9571068 	}
9581069 
959
-	set_thread_flag(TIF_FOREIGN_FPSTATE);
960
-
961
-	local_bh_enable();
962
-	preempt_enable();
963
-	kfree(mem);
1070
+	put_cpu_fpsimd_context();
9641071 }
9651072 
9661073 /*
....@@ -972,11 +1079,9 @@
9721079 	if (!system_supports_fpsimd())
9731080 		return;
9741081 
975
-	preempt_disable();
976
-	local_bh_disable();
1082
+	get_cpu_fpsimd_context();
9771083 	fpsimd_save();
978
-	local_bh_enable();
979
-	preempt_enable();
1084
+	put_cpu_fpsimd_context();
9801085 }
9811086 
9821087 /*
....@@ -993,7 +1098,8 @@
9931098 
9941099 /*
9951100  * Associate current's FPSIMD context with this cpu
996
- * Preemption must be disabled when calling this function.
1101
+ * The caller must have ownership of the cpu FPSIMD context before calling
1102
+ * this function.
9971103  */
9981104 void fpsimd_bind_task_to_cpu(void)
9991105 {
....@@ -1002,6 +1108,8 @@
10021108 
10031109 	WARN_ON(!system_supports_fpsimd());
10041110 	last->st = &current->thread.uw.fpsimd_state;
1111
+	last->sve_state = current->thread.sve_state;
1112
+	last->sve_vl = current->thread.sve_vl;
10051113 	current->thread.fpsimd_cpu = smp_processor_id();
10061114 
10071115 	if (system_supports_sve()) {
....@@ -1015,7 +1123,8 @@
10151123 	}
10161124 }
10171125 
1018
-void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st)
1126
+void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
1127
+			      unsigned int sve_vl)
10191128 {
10201129 	struct fpsimd_last_state_struct *last =
10211130 		this_cpu_ptr(&fpsimd_last_state);
....@@ -1024,6 +1133,8 @@
10241133 	WARN_ON(!in_softirq() && !irqs_disabled());
10251134 
10261135 	last->st = st;
1136
+	last->sve_state = sve_state;
1137
+	last->sve_vl = sve_vl;
10271138 }
10281139 
10291140 /*
....@@ -1047,16 +1158,14 @@
10471158 		return;
10481159 	}
10491160 
1050
-	preempt_disable();
1051
-	local_bh_disable();
1161
+	get_cpu_fpsimd_context();
10521162 
10531163 	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
10541164 		task_fpsimd_load();
10551165 		fpsimd_bind_task_to_cpu();
10561166 	}
10571167 
1058
-	local_bh_enable();
1059
-	preempt_enable();
1168
+	put_cpu_fpsimd_context();
10601169 }
10611170 
10621171 /*
....@@ -1069,8 +1178,7 @@
10691178 	if (WARN_ON(!system_supports_fpsimd()))
10701179 		return;
10711180 
1072
-	preempt_disable();
1073
-	local_bh_disable();
1181
+	get_cpu_fpsimd_context();
10741182 
10751183 	current->thread.uw.fpsimd_state = *state;
10761184 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
....@@ -1081,29 +1189,64 @@
10811189 
10821190 	clear_thread_flag(TIF_FOREIGN_FPSTATE);
10831191 
1084
-	local_bh_enable();
1085
-	preempt_enable();
1192
+	put_cpu_fpsimd_context();
10861193 }
10871194 
10881195 /*
10891196  * Invalidate live CPU copies of task t's FPSIMD state
1197
+ *
1198
+ * This function may be called with preemption enabled.  The barrier()
1199
+ * ensures that the assignment to fpsimd_cpu is visible to any
1200
+ * preemption/softirq that could race with set_tsk_thread_flag(), so
1201
+ * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared.
1202
+ *
1203
+ * The final barrier ensures that TIF_FOREIGN_FPSTATE is seen set by any
1204
+ * subsequent code.
10901205  */
10911206 void fpsimd_flush_task_state(struct task_struct *t)
10921207 {
10931208 	t->thread.fpsimd_cpu = NR_CPUS;
1209
+	/*
1210
+	 * If we don't support fpsimd, bail out after we have
1211
+	 * reset the fpsimd_cpu for this task and clear the
1212
+	 * FPSTATE.
1213
+	 */
1214
+	if (!system_supports_fpsimd())
1215
+		return;
1216
+	barrier();
1217
+	set_tsk_thread_flag(t, TIF_FOREIGN_FPSTATE);
1218
+
1219
+	barrier();
10941220 }
10951221 
1096
-void fpsimd_flush_cpu_state(void)
1222
+/*
1223
+ * Invalidate any task's FPSIMD state that is present on this cpu.
1224
+ * The FPSIMD context should be acquired with get_cpu_fpsimd_context()
1225
+ * before calling this function.
1226
+ */
1227
+static void fpsimd_flush_cpu_state(void)
10971228 {
10981229 	WARN_ON(!system_supports_fpsimd());
10991230 	__this_cpu_write(fpsimd_last_state.st, NULL);
11001231 	set_thread_flag(TIF_FOREIGN_FPSTATE);
11011232 }
11021233 
1103
-#ifdef CONFIG_KERNEL_MODE_NEON
1234
+/*
1235
+ * Save the FPSIMD state to memory and invalidate cpu view.
1236
+ * This function must be called with preemption disabled.
1237
+ */
1238
+void fpsimd_save_and_flush_cpu_state(void)
1239
+{
1240
+	if (!system_supports_fpsimd())
1241
+		return;
1242
+	WARN_ON(preemptible());
1243
+	__get_cpu_fpsimd_context();
1244
+	fpsimd_save();
1245
+	fpsimd_flush_cpu_state();
1246
+	__put_cpu_fpsimd_context();
1247
+}
11041248 
1105
-DEFINE_PER_CPU(bool, kernel_neon_busy);
1106
-EXPORT_PER_CPU_SYMBOL(kernel_neon_busy);
1249
+#ifdef CONFIG_KERNEL_MODE_NEON
11071250 
11081251 /*
11091252  * Kernel-side NEON support functions
....@@ -1129,21 +1272,13 @@
11291272 
11301273 	BUG_ON(!may_use_simd());
11311274 
1132
-	preempt_disable();
1133
-	local_bh_disable();
1134
-
1135
-	__this_cpu_write(kernel_neon_busy, true);
1275
+	get_cpu_fpsimd_context();
11361276 
11371277 	/* Save unsaved fpsimd state, if any: */
11381278 	fpsimd_save();
11391279 
11401280 	/* Invalidate any task state remaining in the fpsimd regs: */
11411281 	fpsimd_flush_cpu_state();
1142
-
1143
-	preempt_disable();
1144
-
1145
-	local_bh_enable();
1146
-	preempt_enable();
11471282 }
11481283 EXPORT_SYMBOL(kernel_neon_begin);
11491284 
....@@ -1158,15 +1293,10 @@
11581293  */
11591294 void kernel_neon_end(void)
11601295 {
1161
-	bool busy;
1162
-
11631296 	if (!system_supports_fpsimd())
11641297 		return;
11651298 
1166
-	busy = __this_cpu_xchg(kernel_neon_busy, false);
1167
-	WARN_ON(!busy);	/* No matching kernel_neon_begin()? */
1168
-
1169
-	preempt_enable();
1299
+	put_cpu_fpsimd_context();
11701300 }
11711301 EXPORT_SYMBOL(kernel_neon_end);
11721302 
....@@ -1258,8 +1388,7 @@
12581388 {
12591389 	switch (cmd) {
12601390 	case CPU_PM_ENTER:
1261
-		fpsimd_save();
1262
-		fpsimd_flush_cpu_state();
1391
+		fpsimd_save_and_flush_cpu_state();
12631392 		break;
12641393 	case CPU_PM_EXIT:
12651394 		break;
....@@ -1305,14 +1434,14 @@
13051434  */
13061435 static int __init fpsimd_init(void)
13071436 {
1308
-	if (elf_hwcap & HWCAP_FP) {
1437
+	if (cpu_have_named_feature(FP)) {
13091438 		fpsimd_pm_init();
13101439 		fpsimd_hotplug_init();
13111440 	} else {
13121441 		pr_notice("Floating-point is not implemented\n");
13131442 	}
13141443 
1315
-	if (!(elf_hwcap & HWCAP_ASIMD))
1444
+	if (!cpu_have_named_feature(ASIMD))
13161445 		pr_notice("Advanced SIMD is not implemented\n");
13171446 
13181447 	return sve_sysctl_init();