forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/arch/x86/kernel/fpu/xstate.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * xsave/xrstor support.
34  *
....@@ -7,6 +8,8 @@
78 #include <linux/cpu.h>
89 #include <linux/mman.h>
910 #include <linux/pkeys.h>
11
+#include <linux/seq_file.h>
12
+#include <linux/proc_fs.h>
1013 
1114 #include <asm/fpu/api.h>
1215 #include <asm/fpu/internal.h>
....@@ -34,6 +37,7 @@
3437 	"AVX-512 ZMM_Hi256"		,
3538 	"Processor Trace (unused)"	,
3639 	"Protection Keys User registers",
40
+	"PASID state",
3741 	"unknown xstate feature"	,
3842 };
3943 
....@@ -48,16 +52,19 @@
4852 	X86_FEATURE_AVX512F,
4953 	X86_FEATURE_INTEL_PT,
5054 	X86_FEATURE_PKU,
55
+	X86_FEATURE_ENQCMD,
5156 };
5257 
5358 /*
54
- * Mask of xstate features supported by the CPU and the kernel:
59
+ * This represents the full set of bits that should ever be set in a kernel
60
+ * XSAVE buffer, both supervisor and user xstates.
5561  */
56
-u64 xfeatures_mask __read_mostly;
62
+u64 xfeatures_mask_all __read_mostly;
5763 
5864 static unsigned int xstate_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1};
5965 static unsigned int xstate_sizes[XFEATURE_MAX]   = { [ 0 ... XFEATURE_MAX - 1] = -1};
60
-static unsigned int xstate_comp_offsets[sizeof(xfeatures_mask)*8];
66
+static unsigned int xstate_comp_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1};
67
+static unsigned int xstate_supervisor_only_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1};
6168 
6269 /*
6370  * The XSAVE area of kernel can be in standard or compacted format;
....@@ -67,22 +74,13 @@
6774 unsigned int fpu_user_xstate_size;
6875 
6976 /*
70
- * Clear all of the X86_FEATURE_* bits that are unavailable
71
- * when the CPU has no XSAVE support.
72
- */
73
-void fpu__xstate_clear_all_cpu_caps(void)
74
-{
75
-	setup_clear_cpu_cap(X86_FEATURE_XSAVE);
76
-}
77
-
78
-/*
7977  * Return whether the system supports a given xfeature.
8078  *
8179  * Also return the name of the (most advanced) feature that the caller requested:
8280  */
8381 int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name)
8482 {
85
-	u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask;
83
+	u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask_all;
8684 
8785 	if (unlikely(feature_name)) {
8886 		long xfeature_idx, max_idx;
....@@ -113,25 +111,17 @@
113111 }
114112 EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
115113 
116
-static int xfeature_is_supervisor(int xfeature_nr)
114
+static bool xfeature_is_supervisor(int xfeature_nr)
117115 {
118116 	/*
119
-	 * We currently do not support supervisor states, but if
120
-	 * we did, we could find out like this.
121
-	 *
122
-	 * SDM says: If state component 'i' is a user state component,
123
-	 * ECX[0] return 0; if state component i is a supervisor
124
-	 * state component, ECX[0] returns 1.
117
+	 * Extended State Enumeration Sub-leaves (EAX = 0DH, ECX = n, n > 1)
118
+	 * returns ECX[0] set to (1) for a supervisor state, and cleared (0)
119
+	 * for a user state.
125120 	 */
126121 	u32 eax, ebx, ecx, edx;
127122 
128123 	cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
129
-	return !!(ecx & 1);
130
-}
131
-
132
-static int xfeature_is_user(int xfeature_nr)
133
-{
134
-	return !xfeature_is_supervisor(xfeature_nr);
124
+	return ecx & 1;
135125 }
136126 
137127 /*
....@@ -164,7 +154,7 @@
164154 	 * None of the feature bits are in init state. So nothing else
165155 	 * to do for us, as the memory layout is up to date.
166156 	 */
167
-	if ((xfeatures & xfeatures_mask) == xfeatures_mask)
157
+	if ((xfeatures & xfeatures_mask_all) == xfeatures_mask_all)
168158 		return;
169159 
170160 	/*
....@@ -191,7 +181,7 @@
191181 	 * in a special way already:
192182 	 */
193183 	feature_bit = 0x2;
194
-	xfeatures = (xfeatures_mask & ~xfeatures) >> 2;
184
+	xfeatures = (xfeatures_mask_user() & ~xfeatures) >> 2;
195185 
196186 	/*
197187 	 * Update all the remaining memory layouts according to their
....@@ -219,30 +209,41 @@
219209  */
220210 void fpu__init_cpu_xstate(void)
221211 {
222
-	if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask)
212
+	u64 unsup_bits;
213
+
214
+	if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask_all)
223215 		return;
224216 	/*
225
-	 * Make it clear that XSAVES supervisor states are not yet
226
-	 * implemented should anyone expect it to work by changing
227
-	 * bits in XFEATURE_MASK_* macros and XCR0.
217
+	 * Unsupported supervisor xstates should not be found in
218
+	 * the xfeatures mask.
228219 	 */
229
-	WARN_ONCE((xfeatures_mask & XFEATURE_MASK_SUPERVISOR),
230
-		"x86/fpu: XSAVES supervisor states are not yet implemented.\n");
220
+	unsup_bits = xfeatures_mask_all & XFEATURE_MASK_SUPERVISOR_UNSUPPORTED;
221
+	WARN_ONCE(unsup_bits, "x86/fpu: Found unsupported supervisor xstates: 0x%llx\n",
222
+		  unsup_bits);
231223 
232
-	xfeatures_mask &= ~XFEATURE_MASK_SUPERVISOR;
224
+	xfeatures_mask_all &= ~XFEATURE_MASK_SUPERVISOR_UNSUPPORTED;
233225 
234226 	cr4_set_bits(X86_CR4_OSXSAVE);
235
-	xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask);
227
+
228
+	/*
229
+	 * XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features
230
+	 * managed by XSAVE{C, OPT, S} and XRSTOR{S}.  Only XSAVE user
231
+	 * states can be set here.
232
+	 */
233
+	xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_user());
234
+
235
+	/*
236
+	 * MSR_IA32_XSS sets supervisor states managed by XSAVES.
237
+	 */
238
+	if (boot_cpu_has(X86_FEATURE_XSAVES)) {
239
+		wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() |
240
+				     xfeatures_mask_dynamic());
241
+	}
236242 }
237243 
238
-/*
239
- * Note that in the future we will likely need a pair of
240
- * functions here: one for user xstates and the other for
241
- * system xstates.  For now, they are the same.
242
- */
243
-static int xfeature_enabled(enum xfeature xfeature)
244
+static bool xfeature_enabled(enum xfeature xfeature)
244245 {
245
-	return !!(xfeatures_mask & (1UL << xfeature));
246
+	return xfeatures_mask_all & BIT_ULL(xfeature);
246247 }
247248 
248249 /*
....@@ -260,10 +261,13 @@
260261 	 * in the fixed offsets in the xsave area in either compacted form
261262 	 * or standard form.
262263 	 */
263
-	xstate_offsets[0] = 0;
264
-	xstate_sizes[0] = offsetof(struct fxregs_state, xmm_space);
265
-	xstate_offsets[1] = xstate_sizes[0];
266
-	xstate_sizes[1] = FIELD_SIZEOF(struct fxregs_state, xmm_space);
264
+	xstate_offsets[XFEATURE_FP]	= 0;
265
+	xstate_sizes[XFEATURE_FP]	= offsetof(struct fxregs_state,
266
+						   xmm_space);
267
+
268
+	xstate_offsets[XFEATURE_SSE]	= xstate_sizes[XFEATURE_FP];
269
+	xstate_sizes[XFEATURE_SSE]	= sizeof_field(struct fxregs_state,
270
+						       xmm_space);
267271 
268272 	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
269273 		if (!xfeature_enabled(i))
....@@ -271,21 +275,25 @@
271275 
272276 		cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
273277 
274
-		/*
275
-		 * If an xfeature is supervisor state, the offset
276
-		 * in EBX is invalid. We leave it to -1.
277
-		 */
278
-		if (xfeature_is_user(i))
279
-			xstate_offsets[i] = ebx;
280
-
281278 		xstate_sizes[i] = eax;
279
+
282280 		/*
283
-		 * In our xstate size checks, we assume that the
284
-		 * highest-numbered xstate feature has the
285
-		 * highest offset in the buffer.  Ensure it does.
281
+		 * If an xfeature is supervisor state, the offset in EBX is
282
+		 * invalid, leave it to -1.
283
+		 */
284
+		if (xfeature_is_supervisor(i))
285
+			continue;
286
+
287
+		xstate_offsets[i] = ebx;
288
+
289
+		/*
290
+		 * In our xstate size checks, we assume that the highest-numbered
291
+		 * xstate feature has the highest offset in the buffer.  Ensure
292
+		 * it does.
286293 		 */
287294 		WARN_ONCE(last_good_offset > xstate_offsets[i],
288
-			"x86/fpu: misordered xstate at %d\n", last_good_offset);
295
+			  "x86/fpu: misordered xstate at %d\n", last_good_offset);
296
+
289297 		last_good_offset = xstate_offsets[i];
290298 	}
291299 }
....@@ -312,6 +320,7 @@
312320 	print_xstate_feature(XFEATURE_MASK_ZMM_Hi256);
313321 	print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
314322 	print_xstate_feature(XFEATURE_MASK_PKRU);
323
+	print_xstate_feature(XFEATURE_MASK_PASID);
315324 }
316325 
317326 /*
....@@ -332,6 +341,13 @@
332341 	u32 eax, ebx, ecx, edx;
333342 
334343 	CHECK_XFEATURE(xfeature_nr);
344
+
345
+	if (!xfeature_enabled(xfeature_nr)) {
346
+		WARN_ONCE(1, "Checking alignment of disabled xfeature %d\n",
347
+			  xfeature_nr);
348
+		return 0;
349
+	}
350
+
335351 	cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
336352 	/*
337353 	 * The value returned by ECX[1] indicates the alignment
....@@ -344,11 +360,11 @@
344360 /*
345361  * This function sets up offsets and sizes of all extended states in
346362  * xsave area. This supports both standard format and compacted format
347
- * of the xsave aread.
363
+ * of the xsave area.
348364  */
349
-static void __init setup_xstate_comp(void)
365
+static void __init setup_xstate_comp_offsets(void)
350366 {
351
-	unsigned int xstate_comp_sizes[sizeof(xfeatures_mask)*8];
367
+	unsigned int next_offset;
352368 	int i;
353369 
354370 	/*
....@@ -356,36 +372,56 @@
356372 	 * in the fixed offsets in the xsave area in either compacted form
357373 	 * or standard form.
358374 	 */
359
-	xstate_comp_offsets[0] = 0;
360
-	xstate_comp_offsets[1] = offsetof(struct fxregs_state, xmm_space);
375
+	xstate_comp_offsets[XFEATURE_FP] = 0;
376
+	xstate_comp_offsets[XFEATURE_SSE] = offsetof(struct fxregs_state,
377
+						     xmm_space);
361378 
362379 	if (!boot_cpu_has(X86_FEATURE_XSAVES)) {
363380 		for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
364
-			if (xfeature_enabled(i)) {
381
+			if (xfeature_enabled(i))
365382 				xstate_comp_offsets[i] = xstate_offsets[i];
366
-				xstate_comp_sizes[i] = xstate_sizes[i];
367
-			}
368383 		}
369384 		return;
370385 	}
371386 
372
-	xstate_comp_offsets[FIRST_EXTENDED_XFEATURE] =
373
-		FXSAVE_SIZE + XSAVE_HDR_SIZE;
387
+	next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE;
374388 
375389 	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
376
-		if (xfeature_enabled(i))
377
-			xstate_comp_sizes[i] = xstate_sizes[i];
378
-		else
379
-			xstate_comp_sizes[i] = 0;
390
+		if (!xfeature_enabled(i))
391
+			continue;
380392 
381
-		if (i > FIRST_EXTENDED_XFEATURE) {
382
-			xstate_comp_offsets[i] = xstate_comp_offsets[i-1]
383
-					+ xstate_comp_sizes[i-1];
393
+		if (xfeature_is_aligned(i))
394
+			next_offset = ALIGN(next_offset, 64);
384395 
385
-			if (xfeature_is_aligned(i))
386
-				xstate_comp_offsets[i] =
387
-					ALIGN(xstate_comp_offsets[i], 64);
388
-		}
396
+		xstate_comp_offsets[i] = next_offset;
397
+		next_offset += xstate_sizes[i];
398
+	}
399
+}
400
+
401
+/*
402
+ * Setup offsets of a supervisor-state-only XSAVES buffer:
403
+ *
404
+ * The offsets stored in xstate_comp_offsets[] only work for one specific
405
+ * value of the Requested Feature BitMap (RFBM).  In cases where a different
406
+ * RFBM value is used, a different set of offsets is required.  This set of
407
+ * offsets is for when RFBM=xfeatures_mask_supervisor().
408
+ */
409
+static void __init setup_supervisor_only_offsets(void)
410
+{
411
+	unsigned int next_offset;
412
+	int i;
413
+
414
+	next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE;
415
+
416
+	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
417
+		if (!xfeature_enabled(i) || !xfeature_is_supervisor(i))
418
+			continue;
419
+
420
+		if (xfeature_is_aligned(i))
421
+			next_offset = ALIGN(next_offset, 64);
422
+
423
+		xstate_supervisor_only_offsets[i] = next_offset;
424
+		next_offset += xstate_sizes[i];
389425 	}
390426 }
391427 
....@@ -420,7 +456,8 @@
420456 	 XFEATURE_MASK_Hi16_ZMM	 |		\
421457 	 XFEATURE_MASK_PKRU |			\
422458 	 XFEATURE_MASK_BNDREGS |		\
423
-	 XFEATURE_MASK_BNDCSR)
459
+	 XFEATURE_MASK_BNDCSR |			\
460
+	 XFEATURE_MASK_PASID)
424461 
425462 /*
426463  * setup the xstate image representing the init state
....@@ -429,7 +466,9 @@
429466 {
430467 	static int on_boot_cpu __initdata = 1;
431468 
432
-	BUILD_BUG_ON(XCNTXT_MASK != XFEATURES_INIT_FPSTATE_HANDLED);
469
+	BUILD_BUG_ON((XFEATURE_MASK_USER_SUPPORTED |
470
+		      XFEATURE_MASK_SUPERVISOR_SUPPORTED) !=
471
+		     XFEATURES_INIT_FPSTATE_HANDLED);
433472 
434473 	WARN_ON_FPU(!on_boot_cpu);
435474 	on_boot_cpu = 0;
....@@ -441,7 +480,8 @@
441480 	print_xstate_features();
442481 
443482 	if (boot_cpu_has(X86_FEATURE_XSAVES))
444
-		init_fpstate.xsave.header.xcomp_bv = (u64)1 << 63 | xfeatures_mask;
483
+		init_fpstate.xsave.header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT |
484
+						     xfeatures_mask_all;
445485 
446486 	/*
447487 	 * Init all the features state with header.xfeatures being 0x0
....@@ -476,7 +516,7 @@
476516 	 * format. Checking a supervisor state's uncompacted offset is
477517 	 * an error.
478518 	 */
479
-	if (XFEATURE_MASK_SUPERVISOR & (1 << xfeature_nr)) {
519
+	if (XFEATURE_MASK_SUPERVISOR_ALL & BIT_ULL(xfeature_nr)) {
480520 		WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr);
481521 		return -1;
482522 	}
....@@ -486,7 +526,7 @@
486526 	return ebx;
487527 }
488528 
489
-static int xfeature_size(int xfeature_nr)
529
+int xfeature_size(int xfeature_nr)
490530 {
491531 	u32 eax, ebx, ecx, edx;
492532 
....@@ -510,10 +550,10 @@
510550 }
511551 
512552 /* Validate an xstate header supplied by userspace (ptrace or sigreturn) */
513
-int validate_xstate_header(const struct xstate_header *hdr)
553
+int validate_user_xstate_header(const struct xstate_header *hdr)
514554 {
515555 	/* No unknown or supervisor features may be set */
516
-	if (hdr->xfeatures & (~xfeatures_mask | XFEATURE_MASK_SUPERVISOR))
556
+	if (hdr->xfeatures & ~xfeatures_mask_user())
517557 		return -EINVAL;
518558 
519559 	/* Userspace must use the uncompacted format */
....@@ -590,6 +630,7 @@
590630 	XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state);
591631 	XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM,  struct avx_512_hi16_state);
592632 	XCHECK_SZ(sz, nr, XFEATURE_PKRU,      struct pkru_state);
633
+	XCHECK_SZ(sz, nr, XFEATURE_PASID,     struct ia32_pasid_state);
593634 
594635 	/*
595636 	 * Make *SURE* to add any feature numbers in below if
....@@ -598,7 +639,8 @@
598639 	 */
599640 	if ((nr < XFEATURE_YMM) ||
600641 	    (nr >= XFEATURE_MAX) ||
601
-	    (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR)) {
642
+	    (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) ||
643
+	    ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_LBR))) {
602644 		WARN_ONCE(1, "no structure for xstate: %d\n", nr);
603645 		XSTATE_WARN_ON(1);
604646 	}
....@@ -608,6 +650,10 @@
608650  * This essentially double-checks what the cpu told us about
609651  * how large the XSAVE buffer needs to be.  We are recalculating
610652  * it to be safe.
653
+ *
654
+ * Dynamic XSAVE features allocate their own buffers and are not
655
+ * covered by these checks. Only the size of the buffer for task->fpu
656
+ * is checked here.
611657  */
612658 static void do_extra_xstate_size_checks(void)
613659 {
....@@ -648,15 +694,12 @@
648694 
649695 
650696 /*
651
- * Get total size of enabled xstates in XCR0/xfeatures_mask.
697
+ * Get total size of enabled xstates in XCR0 | IA32_XSS.
652698  *
653699  * Note the SDM's wording here.  "sub-function 0" only enumerates
654700  * the size of the *user* states.  If we use it to size a buffer
655701  * that we use 'XSAVES' on, we could potentially overflow the
656702  * buffer because 'XSAVES' saves system states too.
657
- *
658
- * Note that we do not currently set any bits on IA32_XSS so
659
- * 'XCR0 | IA32_XSS == XCR0' for now.
660703  */
661704 static unsigned int __init get_xsaves_size(void)
662705 {
....@@ -671,6 +714,33 @@
671714 	 */
672715 	cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
673716 	return ebx;
717
+}
718
+
719
+/*
720
+ * Get the total size of the enabled xstates without the dynamic supervisor
721
+ * features.
722
+ */
723
+static unsigned int __init get_xsaves_size_no_dynamic(void)
724
+{
725
+	u64 mask = xfeatures_mask_dynamic();
726
+	unsigned int size;
727
+
728
+	if (!mask)
729
+		return get_xsaves_size();
730
+
731
+	/* Disable dynamic features. */
732
+	wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor());
733
+
734
+	/*
735
+	 * Ask the hardware what size is required of the buffer.
736
+	 * This is the size required for the task->fpu buffer.
737
+	 */
738
+	size = get_xsaves_size();
739
+
740
+	/* Re-enable dynamic features so XSAVES will work on them again. */
741
+	wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask);
742
+
743
+	return size;
674744 }
675745 
676746 static unsigned int __init get_xsave_size(void)
....@@ -701,7 +771,7 @@
701771 	return false;
702772 }
703773 
704
-static int init_xstate_size(void)
774
+static int __init init_xstate_size(void)
705775 {
706776 	/* Recompute the context size for enabled features: */
707777 	unsigned int possible_xstate_size;
....@@ -710,7 +780,7 @@
710780 	xsave_size = get_xsave_size();
711781 
712782 	if (boot_cpu_has(X86_FEATURE_XSAVES))
713
-		possible_xstate_size = get_xsaves_size();
783
+		possible_xstate_size = get_xsaves_size_no_dynamic();
714784 	else
715785 		possible_xstate_size = xsave_size;
716786 
....@@ -738,9 +808,9 @@
738808  */
739809 static void fpu__init_disable_system_xstate(void)
740810 {
741
-	xfeatures_mask = 0;
811
+	xfeatures_mask_all = 0;
742812 	cr4_clear_bits(X86_CR4_OSXSAVE);
743
-	fpu__xstate_clear_all_cpu_caps();
813
+	setup_clear_cpu_cap(X86_FEATURE_XSAVE);
744814 }
745815 
746816 /*
....@@ -773,16 +843,26 @@
773843 		return;
774844 	}
775845 
846
+	/*
847
+	 * Find user xstates supported by the processor.
848
+	 */
776849 	cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
777
-	xfeatures_mask = eax + ((u64)edx << 32);
850
+	xfeatures_mask_all = eax + ((u64)edx << 32);
778851 
779
-	if ((xfeatures_mask & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) {
852
+	/*
853
+	 * Find supervisor xstates supported by the processor.
854
+	 */
855
+	cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
856
+	xfeatures_mask_all |= ecx + ((u64)edx << 32);
857
+
858
+	if ((xfeatures_mask_user() & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) {
780859 		/*
781860 		 * This indicates that something really unexpected happened
782861 		 * with the enumeration.  Disable XSAVE and try to continue
783862 		 * booting without it.  This is too early to BUG().
784863 		 */
785
-		pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", xfeatures_mask);
864
+		pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n",
865
+		       xfeatures_mask_all);
786866 		goto out_disable;
787867 	}
788868 
....@@ -791,10 +871,10 @@
791871 	 */
792872 	for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) {
793873 		if (!boot_cpu_has(xsave_cpuid_features[i]))
794
-			xfeatures_mask &= ~BIT(i);
874
+			xfeatures_mask_all &= ~BIT_ULL(i);
795875 	}
796876 
797
-	xfeatures_mask &= fpu__get_supported_xfeatures_mask();
877
+	xfeatures_mask_all &= fpu__get_supported_xfeatures_mask();
798878 
799879 	/* Enable xstate instructions to be able to continue with initialization: */
800880 	fpu__init_cpu_xstate();
....@@ -806,15 +886,24 @@
806886 	 * Update info used for ptrace frames; use standard-format size and no
807887 	 * supervisor xstates:
808888 	 */
809
-	update_regset_xstate_info(fpu_user_xstate_size,	xfeatures_mask & ~XFEATURE_MASK_SUPERVISOR);
889
+	update_regset_xstate_info(fpu_user_xstate_size, xfeatures_mask_user());
810890 
811891 	fpu__init_prepare_fx_sw_frame();
812892 	setup_init_fpu_buf();
813
-	setup_xstate_comp();
893
+	setup_xstate_comp_offsets();
894
+	setup_supervisor_only_offsets();
895
+
896
+	/*
897
+	 * CPU capabilities initialization runs before FPU init. So
898
+	 * X86_FEATURE_OSXSAVE is not set. Now that XSAVE is completely
899
+	 * functional, set the feature bit so depending code works.
900
+	 */
901
+	setup_force_cpu_cap(X86_FEATURE_OSXSAVE);
902
+
814903 	print_xstate_offset_size();
815904 
816905 	pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n",
817
-		xfeatures_mask,
906
+		xfeatures_mask_all,
818907 		fpu_kernel_xstate_size,
819908 		boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard");
820909 	return;
....@@ -833,26 +922,31 @@
833922 	 * Restore XCR0 on xsave capable CPUs:
834923 	 */
835924 	if (boot_cpu_has(X86_FEATURE_XSAVE))
836
-		xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask);
925
+		xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_user());
926
+
927
+	/*
928
+	 * Restore IA32_XSS. The same CPUID bit enumerates support
929
+	 * of XSAVES and MSR_IA32_XSS.
930
+	 */
931
+	if (boot_cpu_has(X86_FEATURE_XSAVES)) {
932
+		wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor()  |
933
+				     xfeatures_mask_dynamic());
934
+	}
837935 }
838936 
839937 /*
840
- * Given an xstate feature mask, calculate where in the xsave
938
+ * Given an xstate feature nr, calculate where in the xsave
841939  * buffer the state is.  Callers should ensure that the buffer
842940  * is valid.
843
- *
844
- * Note: does not work for compacted buffers.
845941  */
846
-void *__raw_xsave_addr(struct xregs_state *xsave, int xstate_feature_mask)
942
+static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
847943 {
848
-	int feature_nr = fls64(xstate_feature_mask) - 1;
849
-
850
-	if (!xfeature_enabled(feature_nr)) {
944
+	if (!xfeature_enabled(xfeature_nr)) {
851945 		WARN_ON_FPU(1);
852946 		return NULL;
853947 	}
854948 
855
-	return (void *)xsave + xstate_comp_offsets[feature_nr];
949
+	return (void *)xsave + xstate_comp_offsets[xfeature_nr];
856950 }
857951 /*
858952  * Given the xsave area and a state inside, this function returns the
....@@ -866,13 +960,13 @@
866960  *
867961  * Inputs:
868962  *	xstate: the thread's storage area for all FPU data
869
- *	xstate_feature: state which is defined in xsave.h (e.g.
870
- *	XFEATURE_MASK_FP, XFEATURE_MASK_SSE, etc...)
963
+ *	xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP,
964
+ *	XFEATURE_SSE, etc...)
871965  * Output:
872966  *	address of the state in the xsave area, or NULL if the
873967  *	field is not present in the xsave buffer.
874968  */
875
-void *get_xsave_addr(struct xregs_state *xsave, int xstate_feature)
969
+void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
876970 {
877971 	/*
878972 	 * Do we even *have* xsave state?
....@@ -882,14 +976,13 @@
882976 
883977 	/*
884978 	 * We should not ever be requesting features that we
885
-	 * have not enabled.  Remember that pcntxt_mask is
886
-	 * what we write to the XCR0 register.
979
+	 * have not enabled.
887980 	 */
888
-	WARN_ONCE(!(xfeatures_mask & xstate_feature),
981
+	WARN_ONCE(!(xfeatures_mask_all & BIT_ULL(xfeature_nr)),
889982 		  "get of unsupported state");
890983 	/*
891984 	 * This assumes the last 'xsave*' instruction to
892
-	 * have requested that 'xstate_feature' be saved.
985
+	 * have requested that 'xfeature_nr' be saved.
893986 	 * If it did not, we might be seeing and old value
894987 	 * of the field in the buffer.
895988 	 *
....@@ -898,10 +991,10 @@
898991 	 * or because the "init optimization" caused it
899992 	 * to not be saved.
900993 	 */
901
-	if (!(xsave->header.xfeatures & xstate_feature))
994
+	if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr)))
902995 		return NULL;
903996 
904
-	return __raw_xsave_addr(xsave, xstate_feature);
997
+	return __raw_xsave_addr(xsave, xfeature_nr);
905998 }
906999 EXPORT_SYMBOL_GPL(get_xsave_addr);
9071000 
....@@ -916,25 +1009,23 @@
9161009  * Note that this only works on the current task.
9171010  *
9181011  * Inputs:
919
- *	@xsave_state: state which is defined in xsave.h (e.g. XFEATURE_MASK_FP,
920
- *	XFEATURE_MASK_SSE, etc...)
1012
+ *	@xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP,
1013
+ *	XFEATURE_SSE, etc...)
9211014  * Output:
9221015  *	address of the state in the xsave area or NULL if the state
9231016  *	is not present or is in its 'init state'.
9241017  */
925
-const void *get_xsave_field_ptr(int xsave_state)
1018
+const void *get_xsave_field_ptr(int xfeature_nr)
9261019 {
9271020 	struct fpu *fpu = &current->thread.fpu;
9281021 
929
-	if (!fpu->initialized)
930
-		return NULL;
9311022 	/*
9321023 	 * fpu__save() takes the CPU's xstate registers
9331024 	 * and saves them off to the 'fpu memory buffer.
9341025 	 */
9351026 	fpu__save(fpu);
9361027 
937
-	return get_xsave_addr(&fpu->state.xsave, xsave_state);
1028
+	return get_xsave_addr(&fpu->state.xsave, xfeature_nr);
9381029 }
9391030 
9401031 #ifdef CONFIG_ARCH_HAS_PKEYS
....@@ -1001,32 +1092,10 @@
10011092 	return true;
10021093 }
10031094 
1004
-static void fill_gap(unsigned to, void **kbuf, unsigned *pos, unsigned *count)
1095
+static void copy_feature(bool from_xstate, struct membuf *to, void *xstate,
1096
+			 void *init_xstate, unsigned int size)
10051097 {
1006
-	if (*pos < to) {
1007
-		unsigned size = to - *pos;
1008
-
1009
-		if (size > *count)
1010
-			size = *count;
1011
-		memcpy(*kbuf, (void *)&init_fpstate.xsave + *pos, size);
1012
-		*kbuf += size;
1013
-		*pos += size;
1014
-		*count -= size;
1015
-	}
1016
-}
1017
-
1018
-static void copy_part(unsigned offset, unsigned size, void *from,
1019
-			void **kbuf, unsigned *pos, unsigned *count)
1020
-{
1021
-	fill_gap(offset, kbuf, pos, count);
1022
-	if (size > *count)
1023
-		size = *count;
1024
-	if (size) {
1025
-		memcpy(*kbuf, from, size);
1026
-		*kbuf += size;
1027
-		*pos += size;
1028
-		*count -= size;
1029
-	}
1098
+	membuf_write(to, from_xstate ? xstate : init_xstate, size);
10301099 }
10311100 
10321101 /*
....@@ -1036,154 +1105,83 @@
10361105  * It supports partial copy but pos always starts from zero. This is called
10371106  * from xstateregs_get() and there we check the CPU has XSAVES.
10381107  */
1039
-int copy_xstate_to_kernel(void *kbuf, struct xregs_state *xsave, unsigned int offset_start, unsigned int size_total)
1108
+void copy_xstate_to_kernel(struct membuf to, struct xregs_state *xsave)
10401109 {
1110
+	const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr);
1111
+	struct xregs_state *xinit = &init_fpstate.xsave;
10411112 	struct xstate_header header;
1042
-	const unsigned off_mxcsr = offsetof(struct fxregs_state, mxcsr);
1043
-	unsigned count = size_total;
1113
+	unsigned int zerofrom;
10441114 	int i;
10451115 
10461116 	/*
1047
-	 * Currently copy_regset_to_user() starts from pos 0:
1048
-	 */
1049
-	if (unlikely(offset_start != 0))
1050
-		return -EFAULT;
1051
-
1052
-	/*
10531117 	 * The destination is a ptrace buffer; we put in only user xstates:
10541118 	 */
10551119 	memset(&header, 0, sizeof(header));
10561120 	header.xfeatures = xsave->header.xfeatures;
1057
-	header.xfeatures &= ~XFEATURE_MASK_SUPERVISOR;
1121
+	header.xfeatures &= xfeatures_mask_user();
10581122 
1059
-	if (header.xfeatures & XFEATURE_MASK_FP)
1060
-		copy_part(0, off_mxcsr,
1061
-			  &xsave->i387, &kbuf, &offset_start, &count);
1062
-	if (header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM))
1063
-		copy_part(off_mxcsr, MXCSR_AND_FLAGS_SIZE,
1064
-			  &xsave->i387.mxcsr, &kbuf, &offset_start, &count);
1065
-	if (header.xfeatures & XFEATURE_MASK_FP)
1066
-		copy_part(offsetof(struct fxregs_state, st_space), 128,
1067
-			  &xsave->i387.st_space, &kbuf, &offset_start, &count);
1068
-	if (header.xfeatures & XFEATURE_MASK_SSE)
1069
-		copy_part(xstate_offsets[XFEATURE_SSE], 256,
1070
-			  &xsave->i387.xmm_space, &kbuf, &offset_start, &count);
1071
-	/*
1072
-	 * Fill xsave->i387.sw_reserved value for ptrace frame:
1073
-	 */
1074
-	copy_part(offsetof(struct fxregs_state, sw_reserved), 48,
1075
-		  xstate_fx_sw_bytes, &kbuf, &offset_start, &count);
1076
-	/*
1077
-	 * Copy xregs_state->header:
1078
-	 */
1079
-	copy_part(offsetof(struct xregs_state, header), sizeof(header),
1080
-		  &header, &kbuf, &offset_start, &count);
1123
+	/* Copy FP state up to MXCSR */
1124
+	copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387,
1125
+		     &xinit->i387, off_mxcsr);
1126
+
1127
+	/* Copy MXCSR when SSE or YMM are set in the feature mask */
1128
+	copy_feature(header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM),
1129
+		     &to, &xsave->i387.mxcsr, &xinit->i387.mxcsr,
1130
+		     MXCSR_AND_FLAGS_SIZE);
1131
+
1132
+	/* Copy the remaining FP state */
1133
+	copy_feature(header.xfeatures & XFEATURE_MASK_FP,
1134
+		     &to, &xsave->i387.st_space, &xinit->i387.st_space,
1135
+		     sizeof(xsave->i387.st_space));
1136
+
1137
+	/* Copy the SSE state - shared with YMM, but independently managed */
1138
+	copy_feature(header.xfeatures & XFEATURE_MASK_SSE,
1139
+		     &to, &xsave->i387.xmm_space, &xinit->i387.xmm_space,
1140
+		     sizeof(xsave->i387.xmm_space));
1141
+
1142
+	/* Zero the padding area */
1143
+	membuf_zero(&to, sizeof(xsave->i387.padding));
1144
+
1145
+	/* Copy xsave->i387.sw_reserved */
1146
+	membuf_write(&to, xstate_fx_sw_bytes, sizeof(xsave->i387.sw_reserved));
1147
+
1148
+	/* Copy the user space relevant state of @xsave->header */
1149
+	membuf_write(&to, &header, sizeof(header));
1150
+
1151
+	zerofrom = offsetof(struct xregs_state, extended_state_area);
10811152 
10821153 	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
10831154 		/*
1084
-		 * Copy only in-use xstates:
1155
+		 * The ptrace buffer is in non-compacted XSAVE format.
1156
+		 * In non-compacted format disabled features still occupy
1157
+		 * state space, but there is no state to copy from in the
1158
+		 * compacted init_fpstate. The gap tracking will zero this
1159
+		 * later.
10851160 		 */
1086
-		if ((header.xfeatures >> i) & 1) {
1087
-			void *src = __raw_xsave_addr(xsave, 1 << i);
1161
+		if (!(xfeatures_mask_user() & BIT_ULL(i)))
1162
+			continue;
10881163 
1089
-			copy_part(xstate_offsets[i], xstate_sizes[i],
1090
-				  src, &kbuf, &offset_start, &count);
1091
-		}
1092
-
1093
-	}
1094
-	fill_gap(size_total, &kbuf, &offset_start, &count);
1095
-
1096
-	return 0;
1097
-}
1098
-
1099
-static inline int
1100
-__copy_xstate_to_user(void __user *ubuf, const void *data, unsigned int offset, unsigned int size, unsigned int size_total)
1101
-{
1102
-	if (!size)
1103
-		return 0;
1104
-
1105
-	if (offset < size_total) {
1106
-		unsigned int copy = min(size, size_total - offset);
1107
-
1108
-		if (__copy_to_user(ubuf + offset, data, copy))
1109
-			return -EFAULT;
1110
-	}
1111
-	return 0;
1112
-}
1113
-
1114
-/*
1115
- * Convert from kernel XSAVES compacted format to standard format and copy
1116
- * to a user-space buffer. It supports partial copy but pos always starts from
1117
- * zero. This is called from xstateregs_get() and there we check the CPU
1118
- * has XSAVES.
1119
- */
1120
-int copy_xstate_to_user(void __user *ubuf, struct xregs_state *xsave, unsigned int offset_start, unsigned int size_total)
1121
-{
1122
-	unsigned int offset, size;
1123
-	int ret, i;
1124
-	struct xstate_header header;
1125
-
1126
-	/*
1127
-	 * Currently copy_regset_to_user() starts from pos 0:
1128
-	 */
1129
-	if (unlikely(offset_start != 0))
1130
-		return -EFAULT;
1131
-
1132
-	/*
1133
-	 * The destination is a ptrace buffer; we put in only user xstates:
1134
-	 */
1135
-	memset(&header, 0, sizeof(header));
1136
-	header.xfeatures = xsave->header.xfeatures;
1137
-	header.xfeatures &= ~XFEATURE_MASK_SUPERVISOR;
1138
-
1139
-	/*
1140
-	 * Copy xregs_state->header:
1141
-	 */
1142
-	offset = offsetof(struct xregs_state, header);
1143
-	size = sizeof(header);
1144
-
1145
-	ret = __copy_xstate_to_user(ubuf, &header, offset, size, size_total);
1146
-	if (ret)
1147
-		return ret;
1148
-
1149
-	for (i = 0; i < XFEATURE_MAX; i++) {
11501164 		/*
1151
-		 * Copy only in-use xstates:
1165
+		 * If there was a feature or alignment gap, zero the space
1166
+		 * in the destination buffer.
11521167 		 */
1153
-		if ((header.xfeatures >> i) & 1) {
1154
-			void *src = __raw_xsave_addr(xsave, 1 << i);
1168
+		if (zerofrom < xstate_offsets[i])
1169
+			membuf_zero(&to, xstate_offsets[i] - zerofrom);
11551170 
1156
-			offset = xstate_offsets[i];
1157
-			size = xstate_sizes[i];
1171
+		copy_feature(header.xfeatures & BIT_ULL(i), &to,
1172
+			     __raw_xsave_addr(xsave, i),
1173
+			     __raw_xsave_addr(xinit, i),
1174
+			     xstate_sizes[i]);
11581175 
1159
-			/* The next component has to fit fully into the output buffer: */
1160
-			if (offset + size > size_total)
1161
-				break;
1162
-
1163
-			ret = __copy_xstate_to_user(ubuf, src, offset, size, size_total);
1164
-			if (ret)
1165
-				return ret;
1166
-		}
1167
-
1176
+		/*
1177
+		 * Keep track of the last copied state in the non-compacted
1178
+		 * target buffer for gap zeroing.
1179
+		 */
1180
+		zerofrom = xstate_offsets[i] + xstate_sizes[i];
11681181 	}
11691182 
1170
-	if (xfeatures_mxcsr_quirk(header.xfeatures)) {
1171
-		offset = offsetof(struct fxregs_state, mxcsr);
1172
-		size = MXCSR_AND_FLAGS_SIZE;
1173
-		__copy_xstate_to_user(ubuf, &xsave->i387.mxcsr, offset, size, size_total);
1174
-	}
1175
-
1176
-	/*
1177
-	 * Fill xsave->i387.sw_reserved value for ptrace frame:
1178
-	 */
1179
-	offset = offsetof(struct fxregs_state, sw_reserved);
1180
-	size = sizeof(xstate_fx_sw_bytes);
1181
-
1182
-	ret = __copy_xstate_to_user(ubuf, xstate_fx_sw_bytes, offset, size, size_total);
1183
-	if (ret)
1184
-		return ret;
1185
-
1186
-	return 0;
1183
+	if (to.left)
1184
+		membuf_zero(&to, to.left);
11871185 }
11881186 
11891187 /*
....@@ -1201,14 +1199,14 @@
12011199 
12021200 	memcpy(&hdr, kbuf + offset, size);
12031201 
1204
-	if (validate_xstate_header(&hdr))
1202
+	if (validate_user_xstate_header(&hdr))
12051203 		return -EINVAL;
12061204 
12071205 	for (i = 0; i < XFEATURE_MAX; i++) {
12081206 		u64 mask = ((u64)1 << i);
12091207 
12101208 		if (hdr.xfeatures & mask) {
1211
-			void *dst = __raw_xsave_addr(xsave, 1 << i);
1209
+			void *dst = __raw_xsave_addr(xsave, i);
12121210 
12131211 			offset = xstate_offsets[i];
12141212 			size = xstate_sizes[i];
....@@ -1227,7 +1225,7 @@
12271225 	 * The state that came in from userspace was user-state only.
12281226 	 * Mask all the user states out of 'xfeatures':
12291227 	 */
1230
-	xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR;
1228
+	xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL;
12311229 
12321230 	/*
12331231 	 * Add back in the features that came in from userspace:
....@@ -1255,14 +1253,14 @@
12551253 	if (__copy_from_user(&hdr, ubuf + offset, size))
12561254 		return -EFAULT;
12571255 
1258
-	if (validate_xstate_header(&hdr))
1256
+	if (validate_user_xstate_header(&hdr))
12591257 		return -EINVAL;
12601258 
12611259 	for (i = 0; i < XFEATURE_MAX; i++) {
12621260 		u64 mask = ((u64)1 << i);
12631261 
12641262 		if (hdr.xfeatures & mask) {
1265
-			void *dst = __raw_xsave_addr(xsave, 1 << i);
1263
+			void *dst = __raw_xsave_addr(xsave, i);
12661264 
12671265 			offset = xstate_offsets[i];
12681266 			size = xstate_sizes[i];
....@@ -1283,7 +1281,7 @@
12831281 	 * The state that came in from userspace was user-state only.
12841282 	 * Mask all the user states out of 'xfeatures':
12851283 	 */
1286
-	xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR;
1284
+	xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL;
12871285 
12881286 	/*
12891287 	 * Add back in the features that came in from userspace:
....@@ -1292,3 +1290,175 @@
12921290 
12931291 	return 0;
12941292 }
1293
+
1294
+/*
1295
+ * Save only supervisor states to the kernel buffer.  This blows away all
1296
+ * old states, and is intended to be used only in __fpu__restore_sig(), where
1297
+ * user states are restored from the user buffer.
1298
+ */
1299
+void copy_supervisor_to_kernel(struct xregs_state *xstate)
1300
+{
1301
+	struct xstate_header *header;
1302
+	u64 max_bit, min_bit;
1303
+	u32 lmask, hmask;
1304
+	int err, i;
1305
+
1306
+	if (WARN_ON(!boot_cpu_has(X86_FEATURE_XSAVES)))
1307
+		return;
1308
+
1309
+	if (!xfeatures_mask_supervisor())
1310
+		return;
1311
+
1312
+	max_bit = __fls(xfeatures_mask_supervisor());
1313
+	min_bit = __ffs(xfeatures_mask_supervisor());
1314
+
1315
+	lmask = xfeatures_mask_supervisor();
1316
+	hmask = xfeatures_mask_supervisor() >> 32;
1317
+	XSTATE_OP(XSAVES, xstate, lmask, hmask, err);
1318
+
1319
+	/* We should never fault when copying to a kernel buffer: */
1320
+	if (WARN_ON_FPU(err))
1321
+		return;
1322
+
1323
+	/*
1324
+	 * At this point, the buffer has only supervisor states and must be
1325
+	 * converted back to normal kernel format.
1326
+	 */
1327
+	header = &xstate->header;
1328
+	header->xcomp_bv |= xfeatures_mask_all;
1329
+
1330
+	/*
1331
+	 * This only moves states up in the buffer.  Start with
1332
+	 * the last state and move backwards so that states are
1333
+	 * not overwritten until after they are moved.  Note:
1334
+	 * memmove() allows overlapping src/dst buffers.
1335
+	 */
1336
+	for (i = max_bit; i >= min_bit; i--) {
1337
+		u8 *xbuf = (u8 *)xstate;
1338
+
1339
+		if (!((header->xfeatures >> i) & 1))
1340
+			continue;
1341
+
1342
+		/* Move xfeature 'i' into its normal location */
1343
+		memmove(xbuf + xstate_comp_offsets[i],
1344
+			xbuf + xstate_supervisor_only_offsets[i],
1345
+			xstate_sizes[i]);
1346
+	}
1347
+}
1348
+
1349
+/**
1350
+ * copy_dynamic_supervisor_to_kernel() - Save dynamic supervisor states to
1351
+ *                                       an xsave area
1352
+ * @xstate: A pointer to an xsave area
1353
+ * @mask: Represent the dynamic supervisor features saved into the xsave area
1354
+ *
1355
+ * Only the dynamic supervisor states sets in the mask are saved into the xsave
1356
+ * area (See the comment in XFEATURE_MASK_DYNAMIC for the details of dynamic
1357
+ * supervisor feature). Besides the dynamic supervisor states, the legacy
1358
+ * region and XSAVE header are also saved into the xsave area. The supervisor
1359
+ * features in the XFEATURE_MASK_SUPERVISOR_SUPPORTED and
1360
+ * XFEATURE_MASK_SUPERVISOR_UNSUPPORTED are not saved.
1361
+ *
1362
+ * The xsave area must be 64-bytes aligned.
1363
+ */
1364
+void copy_dynamic_supervisor_to_kernel(struct xregs_state *xstate, u64 mask)
1365
+{
1366
+	u64 dynamic_mask = xfeatures_mask_dynamic() & mask;
1367
+	u32 lmask, hmask;
1368
+	int err;
1369
+
1370
+	if (WARN_ON_FPU(!boot_cpu_has(X86_FEATURE_XSAVES)))
1371
+		return;
1372
+
1373
+	if (WARN_ON_FPU(!dynamic_mask))
1374
+		return;
1375
+
1376
+	lmask = dynamic_mask;
1377
+	hmask = dynamic_mask >> 32;
1378
+
1379
+	XSTATE_OP(XSAVES, xstate, lmask, hmask, err);
1380
+
1381
+	/* Should never fault when copying to a kernel buffer */
1382
+	WARN_ON_FPU(err);
1383
+}
1384
+
1385
+/**
1386
+ * copy_kernel_to_dynamic_supervisor() - Restore dynamic supervisor states from
1387
+ *                                       an xsave area
1388
+ * @xstate: A pointer to an xsave area
1389
+ * @mask: Represent the dynamic supervisor features restored from the xsave area
1390
+ *
1391
+ * Only the dynamic supervisor states sets in the mask are restored from the
1392
+ * xsave area (See the comment in XFEATURE_MASK_DYNAMIC for the details of
1393
+ * dynamic supervisor feature). Besides the dynamic supervisor states, the
1394
+ * legacy region and XSAVE header are also restored from the xsave area. The
1395
+ * supervisor features in the XFEATURE_MASK_SUPERVISOR_SUPPORTED and
1396
+ * XFEATURE_MASK_SUPERVISOR_UNSUPPORTED are not restored.
1397
+ *
1398
+ * The xsave area must be 64-bytes aligned.
1399
+ */
1400
+void copy_kernel_to_dynamic_supervisor(struct xregs_state *xstate, u64 mask)
1401
+{
1402
+	u64 dynamic_mask = xfeatures_mask_dynamic() & mask;
1403
+	u32 lmask, hmask;
1404
+	int err;
1405
+
1406
+	if (WARN_ON_FPU(!boot_cpu_has(X86_FEATURE_XSAVES)))
1407
+		return;
1408
+
1409
+	if (WARN_ON_FPU(!dynamic_mask))
1410
+		return;
1411
+
1412
+	lmask = dynamic_mask;
1413
+	hmask = dynamic_mask >> 32;
1414
+
1415
+	XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
1416
+
1417
+	/* Should never fault when copying from a kernel buffer */
1418
+	WARN_ON_FPU(err);
1419
+}
1420
+
1421
+#ifdef CONFIG_PROC_PID_ARCH_STATUS
1422
+/*
1423
+ * Report the amount of time elapsed in millisecond since last AVX512
1424
+ * use in the task.
1425
+ */
1426
+static void avx512_status(struct seq_file *m, struct task_struct *task)
1427
+{
1428
+	unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp);
1429
+	long delta;
1430
+
1431
+	if (!timestamp) {
1432
+		/*
1433
+		 * Report -1 if no AVX512 usage
1434
+		 */
1435
+		delta = -1;
1436
+	} else {
1437
+		delta = (long)(jiffies - timestamp);
1438
+		/*
1439
+		 * Cap to LONG_MAX if time difference > LONG_MAX
1440
+		 */
1441
+		if (delta < 0)
1442
+			delta = LONG_MAX;
1443
+		delta = jiffies_to_msecs(delta);
1444
+	}
1445
+
1446
+	seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta);
1447
+	seq_putc(m, '\n');
1448
+}
1449
+
1450
+/*
1451
+ * Report architecture specific information
1452
+ */
1453
+int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns,
1454
+			struct pid *pid, struct task_struct *task)
1455
+{
1456
+	/*
1457
+	 * Report AVX512 state if the processor and build option supported.
1458
+	 */
1459
+	if (cpu_feature_enabled(X86_FEATURE_AVX512F))
1460
+		avx512_status(m, task);
1461
+
1462
+	return 0;
1463
+}
1464
+#endif /* CONFIG_PROC_PID_ARCH_STATUS */