~hc/RK356X_SDK_RELEASE.git

..	..	@@ -9,12 +9,13 @@
9	9	* - Andrew D. Balsa (code cleanup).
10	10	*/
11	11	#include <linux/init.h>
12		-#include <linux/utsname.h>
13	12	#include <linux/cpu.h>
14	13	#include <linux/module.h>
15	14	#include <linux/nospec.h>
16	15	#include <linux/prctl.h>
17	16	#include <linux/sched/smt.h>
	17	+#include <linux/pgtable.h>
	18	+#include <linux/bpf.h>
18	19
19	20	#include <asm/spec-ctrl.h>
20	21	#include <asm/cmdline.h>
..	..	@@ -25,34 +26,74 @@
25	26	#include <asm/msr.h>
26	27	#include <asm/vmx.h>
27	28	#include <asm/paravirt.h>
28		-#include <asm/alternative.h>
29		-#include <asm/pgtable.h>
30		-#include <asm/set_memory.h>
31	29	#include <asm/intel-family.h>
32	30	#include <asm/e820/api.h>
33	31	#include <asm/hypervisor.h>
	32	+#include <asm/tlbflush.h>
34	33
35	34	#include "cpu.h"
36	35
37	36	static void __init spectre_v1_select_mitigation(void);
38	37	static void __init spectre_v2_select_mitigation(void);
	38	+static void __init retbleed_select_mitigation(void);
	39	+static void __init spectre_v2_user_select_mitigation(void);
39	40	static void __init ssb_select_mitigation(void);
40	41	static void __init l1tf_select_mitigation(void);
41	42	static void __init mds_select_mitigation(void);
42		-static void __init mds_print_mitigation(void);
	43	+static void __init md_clear_update_mitigation(void);
	44	+static void __init md_clear_select_mitigation(void);
43	45	static void __init taa_select_mitigation(void);
	46	+static void __init mmio_select_mitigation(void);
44	47	static void __init srbds_select_mitigation(void);
	48	+static void __init gds_select_mitigation(void);
	49	+static void __init srso_select_mitigation(void);
45	50
46		-/* The base value of the SPEC_CTRL MSR that always has to be preserved. */
	51	+/* The base value of the SPEC_CTRL MSR without task-specific bits set */
47	52	u64 x86_spec_ctrl_base;
48	53	EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
	54	+
	55	+/* The current value of the SPEC_CTRL MSR with task-specific bits set */
	56	+DEFINE_PER_CPU(u64, x86_spec_ctrl_current);
	57	+EXPORT_SYMBOL_GPL(x86_spec_ctrl_current);
	58	+
	59	+u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB;
	60	+EXPORT_SYMBOL_GPL(x86_pred_cmd);
	61	+
49	62	static DEFINE_MUTEX(spec_ctrl_mutex);
50	63
	64	+void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk;
	65	+
	66	+/* Update SPEC_CTRL MSR and its cached copy unconditionally */
	67	+static void update_spec_ctrl(u64 val)
	68	+{
	69	+ this_cpu_write(x86_spec_ctrl_current, val);
	70	+ wrmsrl(MSR_IA32_SPEC_CTRL, val);
	71	+}
	72	+
51	73	/*
52		- * The vendor and possibly platform specific bits which can be modified in
53		- * x86_spec_ctrl_base.
	74	+ * Keep track of the SPEC_CTRL MSR value for the current task, which may differ
	75	+ * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update().
54	76	*/
55		-static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
	77	+void update_spec_ctrl_cond(u64 val)
	78	+{
	79	+ if (this_cpu_read(x86_spec_ctrl_current) == val)
	80	+ return;
	81	+
	82	+ this_cpu_write(x86_spec_ctrl_current, val);
	83	+
	84	+ /*
	85	+ * When KERNEL_IBRS this MSR is written on return-to-user, unless
	86	+ * forced the update can be delayed until that time.
	87	+ */
	88	+ if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS))
	89	+ wrmsrl(MSR_IA32_SPEC_CTRL, val);
	90	+}
	91	+
	92	+u64 spec_ctrl_current(void)
	93	+{
	94	+ return this_cpu_read(x86_spec_ctrl_current);
	95	+}
	96	+EXPORT_SYMBOL_GPL(spec_ctrl_current);
56	97
57	98	/*
58	99	* AMD specific MSR info for Speculative Store Bypass control.
..	..	@@ -75,107 +116,67 @@
75	116	DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
76	117	EXPORT_SYMBOL_GPL(mds_idle_clear);
77	118
78		-void __init check_bugs(void)
	119	+/* Controls CPU Fill buffer clear before KVM guest MMIO accesses */
	120	+DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear);
	121	+EXPORT_SYMBOL_GPL(mmio_stale_data_clear);
	122	+
	123	+void __init cpu_select_mitigations(void)
79	124	{
80		- identify_boot_cpu();
81		-
82		- /*
83		- * identify_boot_cpu() initialized SMT support information, let the
84		- * core code know.
85		- */
86		- cpu_smt_check_topology();
87		-
88		- if (!IS_ENABLED(CONFIG_SMP)) {
89		- pr_info("CPU: ");
90		- print_cpu_info(&boot_cpu_data);
91		- }
92		-
93	125	/*
94	126	* Read the SPEC_CTRL MSR to account for reserved bits which may
95	127	* have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
96	128	* init code as it is not enumerated and depends on the family.
97	129	*/
98		- if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
	130	+ if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) {
99	131	rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
100	132
101		- /* Allow STIBP in MSR_SPEC_CTRL if supported */
102		- if (boot_cpu_has(X86_FEATURE_STIBP))
103		- x86_spec_ctrl_mask \|= SPEC_CTRL_STIBP;
	133	+ /*
	134	+ * Previously running kernel (kexec), may have some controls
	135	+ * turned ON. Clear them and let the mitigations setup below
	136	+ * rediscover them based on configuration.
	137	+ */
	138	+ x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK;
	139	+ }
104	140
105	141	/* Select the proper CPU mitigations before patching alternatives: */
106	142	spectre_v1_select_mitigation();
107	143	spectre_v2_select_mitigation();
	144	+ /*
	145	+ * retbleed_select_mitigation() relies on the state set by
	146	+ * spectre_v2_select_mitigation(); specifically it wants to know about
	147	+ * spectre_v2=ibrs.
	148	+ */
	149	+ retbleed_select_mitigation();
	150	+ /*
	151	+ * spectre_v2_user_select_mitigation() relies on the state set by
	152	+ * retbleed_select_mitigation(); specifically the STIBP selection is
	153	+ * forced for UNRET or IBPB.
	154	+ */
	155	+ spectre_v2_user_select_mitigation();
108	156	ssb_select_mitigation();
109	157	l1tf_select_mitigation();
110		- mds_select_mitigation();
111		- taa_select_mitigation();
	158	+ md_clear_select_mitigation();
112	159	srbds_select_mitigation();
113	160
114	161	/*
115		- * As MDS and TAA mitigations are inter-related, print MDS
116		- * mitigation until after TAA mitigation selection is done.
	162	+ * srso_select_mitigation() depends and must run after
	163	+ * retbleed_select_mitigation().
117	164	*/
118		- mds_print_mitigation();
119		-
120		- arch_smt_update();
121		-
122		-#ifdef CONFIG_X86_32
123		- /*
124		- * Check whether we are able to run this kernel safely on SMP.
125		- *
126		- * - i386 is no longer supported.
127		- * - In order to run on anything without a TSC, we need to be
128		- * compiled for a i486.
129		- */
130		- if (boot_cpu_data.x86 < 4)
131		- panic("Kernel requires i486+ for 'invlpg' and other features");
132		-
133		- init_utsname()->machine[1] =
134		- '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
135		- alternative_instructions();
136		-
137		- fpu__init_check_bugs();
138		-#else /* CONFIG_X86_64 */
139		- alternative_instructions();
140		-
141		- /*
142		- * Make sure the first 2MB area is not mapped by huge pages
143		- * There are typically fixed size MTRRs in there and overlapping
144		- * MTRRs into large pages causes slow downs.
145		- *
146		- * Right now we don't do that with gbpages because there seems
147		- * very little benefit for that case.
148		- */
149		- if (!direct_gbpages)
150		- set_memory_4k((unsigned long)__va(0), 1);
151		-#endif
	165	+ srso_select_mitigation();
	166	+ gds_select_mitigation();
152	167	}
153	168
	169	+/*
	170	+ * NOTE: For VMX, this function is not called in the vmexit path.
	171	+ * It uses vmx_spec_ctrl_restore_host() instead.
	172	+ */
154	173	void
155	174	x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
156	175	{
157		- u64 msrval, guestval, hostval = x86_spec_ctrl_base;
	176	+ u64 msrval, guestval = guest_spec_ctrl, hostval = spec_ctrl_current();
158	177	struct thread_info *ti = current_thread_info();
159	178
160		- /* Is MSR_SPEC_CTRL implemented ? */
161	179	if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
162		- /*
163		- * Restrict guest_spec_ctrl to supported values. Clear the
164		- * modifiable bits in the host base value and or the
165		- * modifiable bits from the guest value.
166		- */
167		- guestval = hostval & ~x86_spec_ctrl_mask;
168		- guestval \|= guest_spec_ctrl & x86_spec_ctrl_mask;
169		-
170		- /* SSBD controlled in MSR_SPEC_CTRL */
171		- if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) \|\|
172		- static_cpu_has(X86_FEATURE_AMD_SSBD))
173		- hostval \|= ssbd_tif_to_spec_ctrl(ti->flags);
174		-
175		- /* Conditional STIBP enabled? */
176		- if (static_branch_unlikely(&switch_to_cond_stibp))
177		- hostval \|= stibp_tif_to_spec_ctrl(ti->flags);
178		-
179	180	if (hostval != guestval) {
180	181	msrval = setguest ? guestval : hostval;
181	182	wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
..	..	@@ -256,14 +257,6 @@
256	257	}
257	258	}
258	259
259		-static void __init mds_print_mitigation(void)
260		-{
261		- if (!boot_cpu_has_bug(X86_BUG_MDS) \|\| cpu_mitigations_off())
262		- return;
263		-
264		- pr_info("%s\n", mds_strings[mds_mitigation]);
265		-}
266		-
267	260	static int __init mds_cmdline(char *str)
268	261	{
269	262	if (!boot_cpu_has_bug(X86_BUG_MDS))
..	..	@@ -288,6 +281,13 @@
288	281	#undef pr_fmt
289	282	#define pr_fmt(fmt) "TAA: " fmt
290	283
	284	+enum taa_mitigations {
	285	+ TAA_MITIGATION_OFF,
	286	+ TAA_MITIGATION_UCODE_NEEDED,
	287	+ TAA_MITIGATION_VERW,
	288	+ TAA_MITIGATION_TSX_DISABLED,
	289	+};
	290	+
291	291	/* Default mitigation for TAA-affected CPUs */
292	292	static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW;
293	293	static bool taa_nosmt __ro_after_init;
..	..	@@ -311,7 +311,7 @@
311	311	/* TSX previously disabled by tsx=off */
312	312	if (!boot_cpu_has(X86_FEATURE_RTM)) {
313	313	taa_mitigation = TAA_MITIGATION_TSX_DISABLED;
314		- goto out;
	314	+ return;
315	315	}
316	316
317	317	if (cpu_mitigations_off()) {
..	..	@@ -325,7 +325,7 @@
325	325	*/
326	326	if (taa_mitigation == TAA_MITIGATION_OFF &&
327	327	mds_mitigation == MDS_MITIGATION_OFF)
328		- goto out;
	328	+ return;
329	329
330	330	if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
331	331	taa_mitigation = TAA_MITIGATION_VERW;
..	..	@@ -357,18 +357,6 @@
357	357
358	358	if (taa_nosmt \|\| cpu_mitigations_auto_nosmt())
359	359	cpu_smt_disable(false);
360		-
361		- /*
362		- * Update MDS mitigation, if necessary, as the mds_user_clear is
363		- * now enabled for TAA mitigation.
364		- */
365		- if (mds_mitigation == MDS_MITIGATION_OFF &&
366		- boot_cpu_has_bug(X86_BUG_MDS)) {
367		- mds_mitigation = MDS_MITIGATION_FULL;
368		- mds_select_mitigation();
369		- }
370		-out:
371		- pr_info("%s\n", taa_strings[taa_mitigation]);
372	360	}
373	361
374	362	static int __init tsx_async_abort_parse_cmdline(char *str)
..	..	@@ -391,6 +379,154 @@
391	379	return 0;
392	380	}
393	381	early_param("tsx_async_abort", tsx_async_abort_parse_cmdline);
	382	+
	383	+#undef pr_fmt
	384	+#define pr_fmt(fmt) "MMIO Stale Data: " fmt
	385	+
	386	+enum mmio_mitigations {
	387	+ MMIO_MITIGATION_OFF,
	388	+ MMIO_MITIGATION_UCODE_NEEDED,
	389	+ MMIO_MITIGATION_VERW,
	390	+};
	391	+
	392	+/* Default mitigation for Processor MMIO Stale Data vulnerabilities */
	393	+static enum mmio_mitigations mmio_mitigation __ro_after_init = MMIO_MITIGATION_VERW;
	394	+static bool mmio_nosmt __ro_after_init = false;
	395	+
	396	+static const char * const mmio_strings[] = {
	397	+ [MMIO_MITIGATION_OFF] = "Vulnerable",
	398	+ [MMIO_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode",
	399	+ [MMIO_MITIGATION_VERW] = "Mitigation: Clear CPU buffers",
	400	+};
	401	+
	402	+static void __init mmio_select_mitigation(void)
	403	+{
	404	+ u64 ia32_cap;
	405	+
	406	+ if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) \|\|
	407	+ boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN) \|\|
	408	+ cpu_mitigations_off()) {
	409	+ mmio_mitigation = MMIO_MITIGATION_OFF;
	410	+ return;
	411	+ }
	412	+
	413	+ if (mmio_mitigation == MMIO_MITIGATION_OFF)
	414	+ return;
	415	+
	416	+ ia32_cap = x86_read_arch_cap_msr();
	417	+
	418	+ /*
	419	+ * Enable CPU buffer clear mitigation for host and VMM, if also affected
	420	+ * by MDS or TAA. Otherwise, enable mitigation for VMM only.
	421	+ */
	422	+ if (boot_cpu_has_bug(X86_BUG_MDS) \|\| (boot_cpu_has_bug(X86_BUG_TAA) &&
	423	+ boot_cpu_has(X86_FEATURE_RTM)))
	424	+ static_branch_enable(&mds_user_clear);
	425	+ else
	426	+ static_branch_enable(&mmio_stale_data_clear);
	427	+
	428	+ /*
	429	+ * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can
	430	+ * be propagated to uncore buffers, clearing the Fill buffers on idle
	431	+ * is required irrespective of SMT state.
	432	+ */
	433	+ if (!(ia32_cap & ARCH_CAP_FBSDP_NO))
	434	+ static_branch_enable(&mds_idle_clear);
	435	+
	436	+ /*
	437	+ * Check if the system has the right microcode.
	438	+ *
	439	+ * CPU Fill buffer clear mitigation is enumerated by either an explicit
	440	+ * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS
	441	+ * affected systems.
	442	+ */
	443	+ if ((ia32_cap & ARCH_CAP_FB_CLEAR) \|\|
	444	+ (boot_cpu_has(X86_FEATURE_MD_CLEAR) &&
	445	+ boot_cpu_has(X86_FEATURE_FLUSH_L1D) &&
	446	+ !(ia32_cap & ARCH_CAP_MDS_NO)))
	447	+ mmio_mitigation = MMIO_MITIGATION_VERW;
	448	+ else
	449	+ mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED;
	450	+
	451	+ if (mmio_nosmt \|\| cpu_mitigations_auto_nosmt())
	452	+ cpu_smt_disable(false);
	453	+}
	454	+
	455	+static int __init mmio_stale_data_parse_cmdline(char *str)
	456	+{
	457	+ if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
	458	+ return 0;
	459	+
	460	+ if (!str)
	461	+ return -EINVAL;
	462	+
	463	+ if (!strcmp(str, "off")) {
	464	+ mmio_mitigation = MMIO_MITIGATION_OFF;
	465	+ } else if (!strcmp(str, "full")) {
	466	+ mmio_mitigation = MMIO_MITIGATION_VERW;
	467	+ } else if (!strcmp(str, "full,nosmt")) {
	468	+ mmio_mitigation = MMIO_MITIGATION_VERW;
	469	+ mmio_nosmt = true;
	470	+ }
	471	+
	472	+ return 0;
	473	+}
	474	+early_param("mmio_stale_data", mmio_stale_data_parse_cmdline);
	475	+
	476	+#undef pr_fmt
	477	+#define pr_fmt(fmt) "" fmt
	478	+
	479	+static void __init md_clear_update_mitigation(void)
	480	+{
	481	+ if (cpu_mitigations_off())
	482	+ return;
	483	+
	484	+ if (!static_key_enabled(&mds_user_clear))
	485	+ goto out;
	486	+
	487	+ /*
	488	+ * mds_user_clear is now enabled. Update MDS, TAA and MMIO Stale Data
	489	+ * mitigation, if necessary.
	490	+ */
	491	+ if (mds_mitigation == MDS_MITIGATION_OFF &&
	492	+ boot_cpu_has_bug(X86_BUG_MDS)) {
	493	+ mds_mitigation = MDS_MITIGATION_FULL;
	494	+ mds_select_mitigation();
	495	+ }
	496	+ if (taa_mitigation == TAA_MITIGATION_OFF &&
	497	+ boot_cpu_has_bug(X86_BUG_TAA)) {
	498	+ taa_mitigation = TAA_MITIGATION_VERW;
	499	+ taa_select_mitigation();
	500	+ }
	501	+ if (mmio_mitigation == MMIO_MITIGATION_OFF &&
	502	+ boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) {
	503	+ mmio_mitigation = MMIO_MITIGATION_VERW;
	504	+ mmio_select_mitigation();
	505	+ }
	506	+out:
	507	+ if (boot_cpu_has_bug(X86_BUG_MDS))
	508	+ pr_info("MDS: %s\n", mds_strings[mds_mitigation]);
	509	+ if (boot_cpu_has_bug(X86_BUG_TAA))
	510	+ pr_info("TAA: %s\n", taa_strings[taa_mitigation]);
	511	+ if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
	512	+ pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]);
	513	+ else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
	514	+ pr_info("MMIO Stale Data: Unknown: No mitigations\n");
	515	+}
	516	+
	517	+static void __init md_clear_select_mitigation(void)
	518	+{
	519	+ mds_select_mitigation();
	520	+ taa_select_mitigation();
	521	+ mmio_select_mitigation();
	522	+
	523	+ /*
	524	+ * As MDS, TAA and MMIO Stale Data mitigations are inter-related, update
	525	+ * and print their mitigation after MDS, TAA and MMIO Stale Data
	526	+ * mitigation selection is done.
	527	+ */
	528	+ md_clear_update_mitigation();
	529	+}
394	530
395	531	#undef pr_fmt
396	532	#define pr_fmt(fmt) "SRBDS: " fmt
..	..	@@ -453,11 +589,13 @@
453	589	return;
454	590
455	591	/*
456		- * Check to see if this is one of the MDS_NO systems supporting
457		- * TSX that are only exposed to SRBDS when TSX is enabled.
	592	+ * Check to see if this is one of the MDS_NO systems supporting TSX that
	593	+ * are only exposed to SRBDS when TSX is enabled or when CPU is affected
	594	+ * by Processor MMIO Stale Data vulnerability.
458	595	*/
459	596	ia32_cap = x86_read_arch_cap_msr();
460		- if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM))
	597	+ if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) &&
	598	+ !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
461	599	srbds_mitigation = SRBDS_MITIGATION_TSX_OFF;
462	600	else if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
463	601	srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR;
..	..	@@ -482,6 +620,149 @@
482	620	return 0;
483	621	}
484	622	early_param("srbds", srbds_parse_cmdline);
	623	+
	624	+#undef pr_fmt
	625	+#define pr_fmt(fmt) "GDS: " fmt
	626	+
	627	+enum gds_mitigations {
	628	+ GDS_MITIGATION_OFF,
	629	+ GDS_MITIGATION_UCODE_NEEDED,
	630	+ GDS_MITIGATION_FORCE,
	631	+ GDS_MITIGATION_FULL,
	632	+ GDS_MITIGATION_FULL_LOCKED,
	633	+ GDS_MITIGATION_HYPERVISOR,
	634	+};
	635	+
	636	+#if IS_ENABLED(CONFIG_GDS_FORCE_MITIGATION)
	637	+static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FORCE;
	638	+#else
	639	+static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FULL;
	640	+#endif
	641	+
	642	+static const char * const gds_strings[] = {
	643	+ [GDS_MITIGATION_OFF] = "Vulnerable",
	644	+ [GDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode",
	645	+ [GDS_MITIGATION_FORCE] = "Mitigation: AVX disabled, no microcode",
	646	+ [GDS_MITIGATION_FULL] = "Mitigation: Microcode",
	647	+ [GDS_MITIGATION_FULL_LOCKED] = "Mitigation: Microcode (locked)",
	648	+ [GDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status",
	649	+};
	650	+
	651	+bool gds_ucode_mitigated(void)
	652	+{
	653	+ return (gds_mitigation == GDS_MITIGATION_FULL \|\|
	654	+ gds_mitigation == GDS_MITIGATION_FULL_LOCKED);
	655	+}
	656	+EXPORT_SYMBOL_GPL(gds_ucode_mitigated);
	657	+
	658	+void update_gds_msr(void)
	659	+{
	660	+ u64 mcu_ctrl_after;
	661	+ u64 mcu_ctrl;
	662	+
	663	+ switch (gds_mitigation) {
	664	+ case GDS_MITIGATION_OFF:
	665	+ rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
	666	+ mcu_ctrl \|= GDS_MITG_DIS;
	667	+ break;
	668	+ case GDS_MITIGATION_FULL_LOCKED:
	669	+ /*
	670	+ * The LOCKED state comes from the boot CPU. APs might not have
	671	+ * the same state. Make sure the mitigation is enabled on all
	672	+ * CPUs.
	673	+ */
	674	+ case GDS_MITIGATION_FULL:
	675	+ rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
	676	+ mcu_ctrl &= ~GDS_MITG_DIS;
	677	+ break;
	678	+ case GDS_MITIGATION_FORCE:
	679	+ case GDS_MITIGATION_UCODE_NEEDED:
	680	+ case GDS_MITIGATION_HYPERVISOR:
	681	+ return;
	682	+ };
	683	+
	684	+ wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
	685	+
	686	+ /*
	687	+ * Check to make sure that the WRMSR value was not ignored. Writes to
	688	+ * GDS_MITG_DIS will be ignored if this processor is locked but the boot
	689	+ * processor was not.
	690	+ */
	691	+ rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after);
	692	+ WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after);
	693	+}
	694	+
	695	+static void __init gds_select_mitigation(void)
	696	+{
	697	+ u64 mcu_ctrl;
	698	+
	699	+ if (!boot_cpu_has_bug(X86_BUG_GDS))
	700	+ return;
	701	+
	702	+ if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
	703	+ gds_mitigation = GDS_MITIGATION_HYPERVISOR;
	704	+ goto out;
	705	+ }
	706	+
	707	+ if (cpu_mitigations_off())
	708	+ gds_mitigation = GDS_MITIGATION_OFF;
	709	+ /* Will verify below that mitigation _can_ be disabled */
	710	+
	711	+ /* No microcode */
	712	+ if (!(x86_read_arch_cap_msr() & ARCH_CAP_GDS_CTRL)) {
	713	+ if (gds_mitigation == GDS_MITIGATION_FORCE) {
	714	+ /*
	715	+ * This only needs to be done on the boot CPU so do it
	716	+ * here rather than in update_gds_msr()
	717	+ */
	718	+ setup_clear_cpu_cap(X86_FEATURE_AVX);
	719	+ pr_warn("Microcode update needed! Disabling AVX as mitigation.\n");
	720	+ } else {
	721	+ gds_mitigation = GDS_MITIGATION_UCODE_NEEDED;
	722	+ }
	723	+ goto out;
	724	+ }
	725	+
	726	+ /* Microcode has mitigation, use it */
	727	+ if (gds_mitigation == GDS_MITIGATION_FORCE)
	728	+ gds_mitigation = GDS_MITIGATION_FULL;
	729	+
	730	+ rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
	731	+ if (mcu_ctrl & GDS_MITG_LOCKED) {
	732	+ if (gds_mitigation == GDS_MITIGATION_OFF)
	733	+ pr_warn("Mitigation locked. Disable failed.\n");
	734	+
	735	+ /*
	736	+ * The mitigation is selected from the boot CPU. All other CPUs
	737	+ * _should_ have the same state. If the boot CPU isn't locked
	738	+ * but others are then update_gds_msr() will WARN() of the state
	739	+ * mismatch. If the boot CPU is locked update_gds_msr() will
	740	+ * ensure the other CPUs have the mitigation enabled.
	741	+ */
	742	+ gds_mitigation = GDS_MITIGATION_FULL_LOCKED;
	743	+ }
	744	+
	745	+ update_gds_msr();
	746	+out:
	747	+ pr_info("%s\n", gds_strings[gds_mitigation]);
	748	+}
	749	+
	750	+static int __init gds_parse_cmdline(char *str)
	751	+{
	752	+ if (!str)
	753	+ return -EINVAL;
	754	+
	755	+ if (!boot_cpu_has_bug(X86_BUG_GDS))
	756	+ return 0;
	757	+
	758	+ if (!strcmp(str, "off"))
	759	+ gds_mitigation = GDS_MITIGATION_OFF;
	760	+ else if (!strcmp(str, "force"))
	761	+ gds_mitigation = GDS_MITIGATION_FORCE;
	762	+
	763	+ return 0;
	764	+}
	765	+early_param("gather_data_sampling", gds_parse_cmdline);
485	766
486	767	#undef pr_fmt
487	768	#define pr_fmt(fmt) "Spectre V1 : " fmt
..	..	@@ -536,14 +817,12 @@
536	817	* If FSGSBASE is enabled, the user can put a kernel address in
537	818	* GS, in which case SMAP provides no protection.
538	819	*
539		- * [ NOTE: Don't check for X86_FEATURE_FSGSBASE until the
540		- * FSGSBASE enablement patches have been merged. ]
541		- *
542	820	* If FSGSBASE is disabled, the user can only put a user space
543	821	* address in GS. That makes an attack harder, but still
544	822	* possible if there's no SMAP protection.
545	823	*/
546		- if (!smap_works_speculatively()) {
	824	+ if (boot_cpu_has(X86_FEATURE_FSGSBASE) \|\|
	825	+ !smap_works_speculatively()) {
547	826	/*
548	827	* Mitigation can be provided from SWAPGS itself or
549	828	* PTI as the CR3 write in the Meltdown mitigation
..	..	@@ -575,11 +854,182 @@
575	854	}
576	855	early_param("nospectre_v1", nospectre_v1_cmdline);
577	856
578		-#undef pr_fmt
579		-#define pr_fmt(fmt) "Spectre V2 : " fmt
580		-
581	857	static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
582	858	SPECTRE_V2_NONE;
	859	+
	860	+#undef pr_fmt
	861	+#define pr_fmt(fmt) "RETBleed: " fmt
	862	+
	863	+enum retbleed_mitigation {
	864	+ RETBLEED_MITIGATION_NONE,
	865	+ RETBLEED_MITIGATION_UNRET,
	866	+ RETBLEED_MITIGATION_IBPB,
	867	+ RETBLEED_MITIGATION_IBRS,
	868	+ RETBLEED_MITIGATION_EIBRS,
	869	+};
	870	+
	871	+enum retbleed_mitigation_cmd {
	872	+ RETBLEED_CMD_OFF,
	873	+ RETBLEED_CMD_AUTO,
	874	+ RETBLEED_CMD_UNRET,
	875	+ RETBLEED_CMD_IBPB,
	876	+};
	877	+
	878	+const char * const retbleed_strings[] = {
	879	+ [RETBLEED_MITIGATION_NONE] = "Vulnerable",
	880	+ [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk",
	881	+ [RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB",
	882	+ [RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS",
	883	+ [RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS",
	884	+};
	885	+
	886	+static enum retbleed_mitigation retbleed_mitigation __ro_after_init =
	887	+ RETBLEED_MITIGATION_NONE;
	888	+static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init =
	889	+ RETBLEED_CMD_AUTO;
	890	+
	891	+static int __ro_after_init retbleed_nosmt = false;
	892	+
	893	+static int __init retbleed_parse_cmdline(char *str)
	894	+{
	895	+ if (!str)
	896	+ return -EINVAL;
	897	+
	898	+ while (str) {
	899	+ char *next = strchr(str, ',');
	900	+ if (next) {
	901	+ *next = 0;
	902	+ next++;
	903	+ }
	904	+
	905	+ if (!strcmp(str, "off")) {
	906	+ retbleed_cmd = RETBLEED_CMD_OFF;
	907	+ } else if (!strcmp(str, "auto")) {
	908	+ retbleed_cmd = RETBLEED_CMD_AUTO;
	909	+ } else if (!strcmp(str, "unret")) {
	910	+ retbleed_cmd = RETBLEED_CMD_UNRET;
	911	+ } else if (!strcmp(str, "ibpb")) {
	912	+ retbleed_cmd = RETBLEED_CMD_IBPB;
	913	+ } else if (!strcmp(str, "nosmt")) {
	914	+ retbleed_nosmt = true;
	915	+ } else {
	916	+ pr_err("Ignoring unknown retbleed option (%s).", str);
	917	+ }
	918	+
	919	+ str = next;
	920	+ }
	921	+
	922	+ return 0;
	923	+}
	924	+early_param("retbleed", retbleed_parse_cmdline);
	925	+
	926	+#define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n"
	927	+#define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n"
	928	+
	929	+static void __init retbleed_select_mitigation(void)
	930	+{
	931	+ bool mitigate_smt = false;
	932	+
	933	+ if (!boot_cpu_has_bug(X86_BUG_RETBLEED) \|\| cpu_mitigations_off())
	934	+ return;
	935	+
	936	+ switch (retbleed_cmd) {
	937	+ case RETBLEED_CMD_OFF:
	938	+ return;
	939	+
	940	+ case RETBLEED_CMD_UNRET:
	941	+ if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY)) {
	942	+ retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
	943	+ } else {
	944	+ pr_err("WARNING: kernel not compiled with CPU_UNRET_ENTRY.\n");
	945	+ goto do_cmd_auto;
	946	+ }
	947	+ break;
	948	+
	949	+ case RETBLEED_CMD_IBPB:
	950	+ if (!boot_cpu_has(X86_FEATURE_IBPB)) {
	951	+ pr_err("WARNING: CPU does not support IBPB.\n");
	952	+ goto do_cmd_auto;
	953	+ } else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) {
	954	+ retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
	955	+ } else {
	956	+ pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n");
	957	+ goto do_cmd_auto;
	958	+ }
	959	+ break;
	960	+
	961	+do_cmd_auto:
	962	+ case RETBLEED_CMD_AUTO:
	963	+ default:
	964	+ if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD \|\|
	965	+ boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
	966	+ if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY))
	967	+ retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
	968	+ else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY) && boot_cpu_has(X86_FEATURE_IBPB))
	969	+ retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
	970	+ }
	971	+
	972	+ /*
	973	+ * The Intel mitigation (IBRS or eIBRS) was already selected in
	974	+ * spectre_v2_select_mitigation(). 'retbleed_mitigation' will
	975	+ * be set accordingly below.
	976	+ */
	977	+
	978	+ break;
	979	+ }
	980	+
	981	+ switch (retbleed_mitigation) {
	982	+ case RETBLEED_MITIGATION_UNRET:
	983	+ setup_force_cpu_cap(X86_FEATURE_RETHUNK);
	984	+ setup_force_cpu_cap(X86_FEATURE_UNRET);
	985	+
	986	+ if (IS_ENABLED(CONFIG_RETHUNK))
	987	+ x86_return_thunk = retbleed_return_thunk;
	988	+
	989	+ if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
	990	+ boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
	991	+ pr_err(RETBLEED_UNTRAIN_MSG);
	992	+
	993	+ mitigate_smt = true;
	994	+ break;
	995	+
	996	+ case RETBLEED_MITIGATION_IBPB:
	997	+ setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
	998	+ mitigate_smt = true;
	999	+ break;
	1000	+
	1001	+ default:
	1002	+ break;
	1003	+ }
	1004	+
	1005	+ if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) &&
	1006	+ (retbleed_nosmt \|\| cpu_mitigations_auto_nosmt()))
	1007	+ cpu_smt_disable(false);
	1008	+
	1009	+ /*
	1010	+ * Let IBRS trump all on Intel without affecting the effects of the
	1011	+ * retbleed= cmdline option.
	1012	+ */
	1013	+ if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
	1014	+ switch (spectre_v2_enabled) {
	1015	+ case SPECTRE_V2_IBRS:
	1016	+ retbleed_mitigation = RETBLEED_MITIGATION_IBRS;
	1017	+ break;
	1018	+ case SPECTRE_V2_EIBRS:
	1019	+ case SPECTRE_V2_EIBRS_RETPOLINE:
	1020	+ case SPECTRE_V2_EIBRS_LFENCE:
	1021	+ retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;
	1022	+ break;
	1023	+ default:
	1024	+ pr_err(RETBLEED_INTEL_MSG);
	1025	+ }
	1026	+ }
	1027	+
	1028	+ pr_info("%s\n", retbleed_strings[retbleed_mitigation]);
	1029	+}
	1030	+
	1031	+#undef pr_fmt
	1032	+#define pr_fmt(fmt) "Spectre V2 : " fmt
583	1033
584	1034	static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init =
585	1035	SPECTRE_V2_USER_NONE;
..	..	@@ -607,6 +1057,33 @@
607	1057	static inline const char *spectre_v2_module_string(void) { return ""; }
608	1058	#endif
609	1059
	1060	+#define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
	1061	+#define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n"
	1062	+#define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n"
	1063	+#define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n"
	1064	+
	1065	+#ifdef CONFIG_BPF_SYSCALL
	1066	+void unpriv_ebpf_notify(int new_state)
	1067	+{
	1068	+ if (new_state)
	1069	+ return;
	1070	+
	1071	+ /* Unprivileged eBPF is enabled */
	1072	+
	1073	+ switch (spectre_v2_enabled) {
	1074	+ case SPECTRE_V2_EIBRS:
	1075	+ pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
	1076	+ break;
	1077	+ case SPECTRE_V2_EIBRS_LFENCE:
	1078	+ if (sched_smt_active())
	1079	+ pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
	1080	+ break;
	1081	+ default:
	1082	+ break;
	1083	+ }
	1084	+}
	1085	+#endif
	1086	+
610	1087	static inline bool match_option(const char arg, int arglen, const char opt)
611	1088	{
612	1089	int len = strlen(opt);
..	..	@@ -621,7 +1098,11 @@
621	1098	SPECTRE_V2_CMD_FORCE,
622	1099	SPECTRE_V2_CMD_RETPOLINE,
623	1100	SPECTRE_V2_CMD_RETPOLINE_GENERIC,
624		- SPECTRE_V2_CMD_RETPOLINE_AMD,
	1101	+ SPECTRE_V2_CMD_RETPOLINE_LFENCE,
	1102	+ SPECTRE_V2_CMD_EIBRS,
	1103	+ SPECTRE_V2_CMD_EIBRS_RETPOLINE,
	1104	+ SPECTRE_V2_CMD_EIBRS_LFENCE,
	1105	+ SPECTRE_V2_CMD_IBRS,
625	1106	};
626	1107
627	1108	enum spectre_v2_user_cmd {
..	..	@@ -662,13 +1143,15 @@
662	1143	pr_info("spectre_v2_user=%s forced on command line.\n", reason);
663	1144	}
664	1145
	1146	+static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd;
	1147	+
665	1148	static enum spectre_v2_user_cmd __init
666		-spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
	1149	+spectre_v2_parse_user_cmdline(void)
667	1150	{
668	1151	char arg[20];
669	1152	int ret, i;
670	1153
671		- switch (v2_cmd) {
	1154	+ switch (spectre_v2_cmd) {
672	1155	case SPECTRE_V2_CMD_NONE:
673	1156	return SPECTRE_V2_USER_CMD_NONE;
674	1157	case SPECTRE_V2_CMD_FORCE:
..	..	@@ -694,8 +1177,20 @@
694	1177	return SPECTRE_V2_USER_CMD_AUTO;
695	1178	}
696	1179
	1180	+static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode)
	1181	+{
	1182	+ return mode == SPECTRE_V2_EIBRS \|\|
	1183	+ mode == SPECTRE_V2_EIBRS_RETPOLINE \|\|
	1184	+ mode == SPECTRE_V2_EIBRS_LFENCE;
	1185	+}
	1186	+
	1187	+static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode)
	1188	+{
	1189	+ return spectre_v2_in_eibrs_mode(mode) \|\| mode == SPECTRE_V2_IBRS;
	1190	+}
	1191	+
697	1192	static void __init
698		-spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
	1193	+spectre_v2_user_select_mitigation(void)
699	1194	{
700	1195	enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
701	1196	bool smt_possible = IS_ENABLED(CONFIG_SMP);
..	..	@@ -708,7 +1203,7 @@
708	1203	cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
709	1204	smt_possible = false;
710	1205
711		- cmd = spectre_v2_parse_user_cmdline(v2_cmd);
	1206	+ cmd = spectre_v2_parse_user_cmdline();
712	1207	switch (cmd) {
713	1208	case SPECTRE_V2_USER_CMD_NONE:
714	1209	goto set_mode;
..	..	@@ -756,10 +1251,19 @@
756	1251	}
757	1252
758	1253	/*
759		- * If enhanced IBRS is enabled or SMT impossible, STIBP is not
760		- * required.
	1254	+ * If no STIBP, enhanced IBRS is enabled, or SMT impossible, STIBP
	1255	+ * is not required.
	1256	+ *
	1257	+ * Enhanced IBRS also protects against cross-thread branch target
	1258	+ * injection in user-mode as the IBRS bit remains always set which
	1259	+ * implicitly enables cross-thread protections. However, in legacy IBRS
	1260	+ * mode, the IBRS bit is set only on kernel entry and cleared on return
	1261	+ * to userspace. This disables the implicit cross-thread protection,
	1262	+ * so allow for STIBP to be selected in that case.
761	1263	*/
762		- if (!smt_possible \|\| spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
	1264	+ if (!boot_cpu_has(X86_FEATURE_STIBP) \|\|
	1265	+ !smt_possible \|\|
	1266	+ spectre_v2_in_eibrs_mode(spectre_v2_enabled))
763	1267	return;
764	1268
765	1269	/*
..	..	@@ -771,11 +1275,13 @@
771	1275	boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
772	1276	mode = SPECTRE_V2_USER_STRICT_PREFERRED;
773	1277
774		- /*
775		- * If STIBP is not available, clear the STIBP mode.
776		- */
777		- if (!boot_cpu_has(X86_FEATURE_STIBP))
778		- mode = SPECTRE_V2_USER_NONE;
	1278	+ if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET \|\|
	1279	+ retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
	1280	+ if (mode != SPECTRE_V2_USER_STRICT &&
	1281	+ mode != SPECTRE_V2_USER_STRICT_PREFERRED)
	1282	+ pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n");
	1283	+ mode = SPECTRE_V2_USER_STRICT_PREFERRED;
	1284	+ }
779	1285
780	1286	spectre_v2_user_stibp = mode;
781	1287
..	..	@@ -785,9 +1291,12 @@
785	1291
786	1292	static const char * const spectre_v2_strings[] = {
787	1293	[SPECTRE_V2_NONE] = "Vulnerable",
788		- [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline",
789		- [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline",
790		- [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS",
	1294	+ [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines",
	1295	+ [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE",
	1296	+ [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced IBRS",
	1297	+ [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced IBRS + LFENCE",
	1298	+ [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced IBRS + Retpolines",
	1299	+ [SPECTRE_V2_IBRS] = "Mitigation: IBRS",
791	1300	};
792	1301
793	1302	static const struct {
..	..	@@ -798,9 +1307,14 @@
798	1307	{ "off", SPECTRE_V2_CMD_NONE, false },
799	1308	{ "on", SPECTRE_V2_CMD_FORCE, true },
800	1309	{ "retpoline", SPECTRE_V2_CMD_RETPOLINE, false },
801		- { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD, false },
	1310	+ { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false },
	1311	+ { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false },
802	1312	{ "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
	1313	+ { "eibrs", SPECTRE_V2_CMD_EIBRS, false },
	1314	+ { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false },
	1315	+ { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false },
803	1316	{ "auto", SPECTRE_V2_CMD_AUTO, false },
	1317	+ { "ibrs", SPECTRE_V2_CMD_IBRS, false },
804	1318	};
805	1319
806	1320	static void __init spec_v2_print_cond(const char *reason, bool secure)
..	..	@@ -836,22 +1350,133 @@
836	1350	}
837	1351
838	1352	if ((cmd == SPECTRE_V2_CMD_RETPOLINE \|\|
839		- cmd == SPECTRE_V2_CMD_RETPOLINE_AMD \|\|
840		- cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
	1353	+ cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE \|\|
	1354	+ cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC \|\|
	1355	+ cmd == SPECTRE_V2_CMD_EIBRS_LFENCE \|\|
	1356	+ cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
841	1357	!IS_ENABLED(CONFIG_RETPOLINE)) {
842		- pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
	1358	+ pr_err("%s selected but not compiled in. Switching to AUTO select\n",
	1359	+ mitigation_options[i].option);
843	1360	return SPECTRE_V2_CMD_AUTO;
844	1361	}
845	1362
846		- if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
847		- boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
848		- pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
	1363	+ if ((cmd == SPECTRE_V2_CMD_EIBRS \|\|
	1364	+ cmd == SPECTRE_V2_CMD_EIBRS_LFENCE \|\|
	1365	+ cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
	1366	+ !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
	1367	+ pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",
	1368	+ mitigation_options[i].option);
	1369	+ return SPECTRE_V2_CMD_AUTO;
	1370	+ }
	1371	+
	1372	+ if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE \|\|
	1373	+ cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
	1374	+ !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
	1375	+ pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
	1376	+ mitigation_options[i].option);
	1377	+ return SPECTRE_V2_CMD_AUTO;
	1378	+ }
	1379	+
	1380	+ if (cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_CPU_IBRS_ENTRY)) {
	1381	+ pr_err("%s selected but not compiled in. Switching to AUTO select\n",
	1382	+ mitigation_options[i].option);
	1383	+ return SPECTRE_V2_CMD_AUTO;
	1384	+ }
	1385	+
	1386	+ if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
	1387	+ pr_err("%s selected but not Intel CPU. Switching to AUTO select\n",
	1388	+ mitigation_options[i].option);
	1389	+ return SPECTRE_V2_CMD_AUTO;
	1390	+ }
	1391	+
	1392	+ if (cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) {
	1393	+ pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n",
	1394	+ mitigation_options[i].option);
	1395	+ return SPECTRE_V2_CMD_AUTO;
	1396	+ }
	1397	+
	1398	+ if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_has(X86_FEATURE_XENPV)) {
	1399	+ pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n",
	1400	+ mitigation_options[i].option);
849	1401	return SPECTRE_V2_CMD_AUTO;
850	1402	}
851	1403
852	1404	spec_v2_print_cond(mitigation_options[i].option,
853	1405	mitigation_options[i].secure);
854	1406	return cmd;
	1407	+}
	1408	+
	1409	+static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)
	1410	+{
	1411	+ if (!IS_ENABLED(CONFIG_RETPOLINE)) {
	1412	+ pr_err("Kernel not compiled with retpoline; no mitigation available!");
	1413	+ return SPECTRE_V2_NONE;
	1414	+ }
	1415	+
	1416	+ return SPECTRE_V2_RETPOLINE;
	1417	+}
	1418	+
	1419	+/* Disable in-kernel use of non-RSB RET predictors */
	1420	+static void __init spec_ctrl_disable_kernel_rrsba(void)
	1421	+{
	1422	+ u64 ia32_cap;
	1423	+
	1424	+ if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL))
	1425	+ return;
	1426	+
	1427	+ ia32_cap = x86_read_arch_cap_msr();
	1428	+
	1429	+ if (ia32_cap & ARCH_CAP_RRSBA) {
	1430	+ x86_spec_ctrl_base \|= SPEC_CTRL_RRSBA_DIS_S;
	1431	+ update_spec_ctrl(x86_spec_ctrl_base);
	1432	+ }
	1433	+}
	1434	+
	1435	+static void __init spectre_v2_determine_rsb_fill_type_at_vmexit(enum spectre_v2_mitigation mode)
	1436	+{
	1437	+ /*
	1438	+ * Similar to context switches, there are two types of RSB attacks
	1439	+ * after VM exit:
	1440	+ *
	1441	+ * 1) RSB underflow
	1442	+ *
	1443	+ * 2) Poisoned RSB entry
	1444	+ *
	1445	+ * When retpoline is enabled, both are mitigated by filling/clearing
	1446	+ * the RSB.
	1447	+ *
	1448	+ * When IBRS is enabled, while #1 would be mitigated by the IBRS branch
	1449	+ * prediction isolation protections, RSB still needs to be cleared
	1450	+ * because of #2. Note that SMEP provides no protection here, unlike
	1451	+ * user-space-poisoned RSB entries.
	1452	+ *
	1453	+ * eIBRS should protect against RSB poisoning, but if the EIBRS_PBRSB
	1454	+ * bug is present then a LITE version of RSB protection is required,
	1455	+ * just a single call needs to retire before a RET is executed.
	1456	+ */
	1457	+ switch (mode) {
	1458	+ case SPECTRE_V2_NONE:
	1459	+ return;
	1460	+
	1461	+ case SPECTRE_V2_EIBRS_LFENCE:
	1462	+ case SPECTRE_V2_EIBRS:
	1463	+ if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {
	1464	+ setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE);
	1465	+ pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n");
	1466	+ }
	1467	+ return;
	1468	+
	1469	+ case SPECTRE_V2_EIBRS_RETPOLINE:
	1470	+ case SPECTRE_V2_RETPOLINE:
	1471	+ case SPECTRE_V2_LFENCE:
	1472	+ case SPECTRE_V2_IBRS:
	1473	+ setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT);
	1474	+ pr_info("Spectre v2 / SpectreRSB : Filling RSB on VMEXIT\n");
	1475	+ return;
	1476	+ }
	1477	+
	1478	+ pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation at VM exit");
	1479	+ dump_stack();
855	1480	}
856	1481
857	1482	static void __init spectre_v2_select_mitigation(void)
..	..	@@ -874,85 +1499,172 @@
874	1499	case SPECTRE_V2_CMD_FORCE:
875	1500	case SPECTRE_V2_CMD_AUTO:
876	1501	if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
877		- mode = SPECTRE_V2_IBRS_ENHANCED;
878		- /* Force it so VMEXIT will restore correctly */
879		- x86_spec_ctrl_base \|= SPEC_CTRL_IBRS;
880		- wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
881		- goto specv2_set_mode;
	1502	+ mode = SPECTRE_V2_EIBRS;
	1503	+ break;
882	1504	}
883		- if (IS_ENABLED(CONFIG_RETPOLINE))
884		- goto retpoline_auto;
	1505	+
	1506	+ if (IS_ENABLED(CONFIG_CPU_IBRS_ENTRY) &&
	1507	+ boot_cpu_has_bug(X86_BUG_RETBLEED) &&
	1508	+ retbleed_cmd != RETBLEED_CMD_OFF &&
	1509	+ boot_cpu_has(X86_FEATURE_IBRS) &&
	1510	+ boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
	1511	+ mode = SPECTRE_V2_IBRS;
	1512	+ break;
	1513	+ }
	1514	+
	1515	+ mode = spectre_v2_select_retpoline();
885	1516	break;
886		- case SPECTRE_V2_CMD_RETPOLINE_AMD:
887		- if (IS_ENABLED(CONFIG_RETPOLINE))
888		- goto retpoline_amd;
	1517	+
	1518	+ case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
	1519	+ pr_err(SPECTRE_V2_LFENCE_MSG);
	1520	+ mode = SPECTRE_V2_LFENCE;
889	1521	break;
	1522	+
890	1523	case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
891		- if (IS_ENABLED(CONFIG_RETPOLINE))
892		- goto retpoline_generic;
	1524	+ mode = SPECTRE_V2_RETPOLINE;
893	1525	break;
	1526	+
894	1527	case SPECTRE_V2_CMD_RETPOLINE:
895		- if (IS_ENABLED(CONFIG_RETPOLINE))
896		- goto retpoline_auto;
	1528	+ mode = spectre_v2_select_retpoline();
	1529	+ break;
	1530	+
	1531	+ case SPECTRE_V2_CMD_IBRS:
	1532	+ mode = SPECTRE_V2_IBRS;
	1533	+ break;
	1534	+
	1535	+ case SPECTRE_V2_CMD_EIBRS:
	1536	+ mode = SPECTRE_V2_EIBRS;
	1537	+ break;
	1538	+
	1539	+ case SPECTRE_V2_CMD_EIBRS_LFENCE:
	1540	+ mode = SPECTRE_V2_EIBRS_LFENCE;
	1541	+ break;
	1542	+
	1543	+ case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
	1544	+ mode = SPECTRE_V2_EIBRS_RETPOLINE;
897	1545	break;
898	1546	}
899		- pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
900		- return;
901	1547
902		-retpoline_auto:
903		- if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
904		- retpoline_amd:
905		- if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
906		- pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
907		- goto retpoline_generic;
908		- }
909		- mode = SPECTRE_V2_RETPOLINE_AMD;
910		- setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
911		- setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
912		- } else {
913		- retpoline_generic:
914		- mode = SPECTRE_V2_RETPOLINE_GENERIC;
915		- setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
	1548	+ if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
	1549	+ pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
	1550	+
	1551	+ if (spectre_v2_in_ibrs_mode(mode)) {
	1552	+ x86_spec_ctrl_base \|= SPEC_CTRL_IBRS;
	1553	+ update_spec_ctrl(x86_spec_ctrl_base);
916	1554	}
917	1555
918		-specv2_set_mode:
	1556	+ switch (mode) {
	1557	+ case SPECTRE_V2_NONE:
	1558	+ case SPECTRE_V2_EIBRS:
	1559	+ break;
	1560	+
	1561	+ case SPECTRE_V2_IBRS:
	1562	+ setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS);
	1563	+ if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED))
	1564	+ pr_warn(SPECTRE_V2_IBRS_PERF_MSG);
	1565	+ break;
	1566	+
	1567	+ case SPECTRE_V2_LFENCE:
	1568	+ case SPECTRE_V2_EIBRS_LFENCE:
	1569	+ setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
	1570	+ fallthrough;
	1571	+
	1572	+ case SPECTRE_V2_RETPOLINE:
	1573	+ case SPECTRE_V2_EIBRS_RETPOLINE:
	1574	+ setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
	1575	+ break;
	1576	+ }
	1577	+
	1578	+ /*
	1579	+ * Disable alternate RSB predictions in kernel when indirect CALLs and
	1580	+ * JMPs gets protection against BHI and Intramode-BTI, but RET
	1581	+ * prediction from a non-RSB predictor is still a risk.
	1582	+ */
	1583	+ if (mode == SPECTRE_V2_EIBRS_LFENCE \|\|
	1584	+ mode == SPECTRE_V2_EIBRS_RETPOLINE \|\|
	1585	+ mode == SPECTRE_V2_RETPOLINE)
	1586	+ spec_ctrl_disable_kernel_rrsba();
	1587	+
919	1588	spectre_v2_enabled = mode;
920	1589	pr_info("%s\n", spectre_v2_strings[mode]);
921	1590
922	1591	/*
923		- * If spectre v2 protection has been enabled, unconditionally fill
924		- * RSB during a context switch; this protects against two independent
925		- * issues:
	1592	+ * If Spectre v2 protection has been enabled, fill the RSB during a
	1593	+ * context switch. In general there are two types of RSB attacks
	1594	+ * across context switches, for which the CALLs/RETs may be unbalanced.
926	1595	*
927		- * - RSB underflow (and switch to BTB) on Skylake+
928		- * - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
	1596	+ * 1) RSB underflow
	1597	+ *
	1598	+ * Some Intel parts have "bottomless RSB". When the RSB is empty,
	1599	+ * speculated return targets may come from the branch predictor,
	1600	+ * which could have a user-poisoned BTB or BHB entry.
	1601	+ *
	1602	+ * AMD has it even worse: all returns are speculated from the BTB,
	1603	+ * regardless of the state of the RSB.
	1604	+ *
	1605	+ * When IBRS or eIBRS is enabled, the "user -> kernel" attack
	1606	+ * scenario is mitigated by the IBRS branch prediction isolation
	1607	+ * properties, so the RSB buffer filling wouldn't be necessary to
	1608	+ * protect against this type of attack.
	1609	+ *
	1610	+ * The "user -> user" attack scenario is mitigated by RSB filling.
	1611	+ *
	1612	+ * 2) Poisoned RSB entry
	1613	+ *
	1614	+ * If the 'next' in-kernel return stack is shorter than 'prev',
	1615	+ * 'next' could be tricked into speculating with a user-poisoned RSB
	1616	+ * entry.
	1617	+ *
	1618	+ * The "user -> kernel" attack scenario is mitigated by SMEP and
	1619	+ * eIBRS.
	1620	+ *
	1621	+ * The "user -> user" scenario, also known as SpectreBHB, requires
	1622	+ * RSB clearing.
	1623	+ *
	1624	+ * So to mitigate all cases, unconditionally fill RSB on context
	1625	+ * switches.
	1626	+ *
	1627	+ * FIXME: Is this pointless for retbleed-affected AMD?
929	1628	*/
930	1629	setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
931	1630	pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
932	1631
	1632	+ spectre_v2_determine_rsb_fill_type_at_vmexit(mode);
	1633	+
933	1634	/*
934		- * Retpoline means the kernel is safe because it has no indirect
935		- * branches. Enhanced IBRS protects firmware too, so, enable restricted
936		- * speculation around firmware calls only when Enhanced IBRS isn't
937		- * supported.
	1635	+ * Retpoline protects the kernel, but doesn't protect firmware. IBRS
	1636	+ * and Enhanced IBRS protect firmware too, so enable IBRS around
	1637	+ * firmware calls only when IBRS / Enhanced IBRS aren't otherwise
	1638	+ * enabled.
938	1639	*
939	1640	* Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
940	1641	* the user might select retpoline on the kernel command line and if
941	1642	* the CPU supports Enhanced IBRS, kernel might un-intentionally not
942	1643	* enable IBRS around firmware calls.
943	1644	*/
944		- if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
	1645	+ if (boot_cpu_has_bug(X86_BUG_RETBLEED) &&
	1646	+ boot_cpu_has(X86_FEATURE_IBPB) &&
	1647	+ (boot_cpu_data.x86_vendor == X86_VENDOR_AMD \|\|
	1648	+ boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) {
	1649	+
	1650	+ if (retbleed_cmd != RETBLEED_CMD_IBPB) {
	1651	+ setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW);
	1652	+ pr_info("Enabling Speculation Barrier for firmware calls\n");
	1653	+ }
	1654	+
	1655	+ } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) {
945	1656	setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
946	1657	pr_info("Enabling Restricted Speculation for firmware calls\n");
947	1658	}
948	1659
949	1660	/* Set up IBPB and STIBP depending on the general spectre V2 command */
950		- spectre_v2_user_select_mitigation(cmd);
	1661	+ spectre_v2_cmd = cmd;
951	1662	}
952	1663
953	1664	static void update_stibp_msr(void * __unused)
954	1665	{
955		- wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
	1666	+ u64 val = spec_ctrl_current() \| (x86_spec_ctrl_base & SPEC_CTRL_STIBP);
	1667	+ update_spec_ctrl(val);
956	1668	}
957	1669
958	1670	/* Update x86_spec_ctrl_base in case SMT state changed. */
..	..	@@ -987,6 +1699,8 @@
987	1699	/* Update the static key controlling the MDS CPU buffer clear in idle */
988	1700	static void update_mds_branch_idle(void)
989	1701	{
	1702	+ u64 ia32_cap = x86_read_arch_cap_msr();
	1703	+
990	1704	/*
991	1705	* Enable the idle clearing if SMT is active on CPUs which are
992	1706	* affected only by MSBDS and not any other MDS variant.
..	..	@@ -998,18 +1712,25 @@
998	1712	if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
999	1713	return;
1000	1714
1001		- if (sched_smt_active())
	1715	+ if (sched_smt_active()) {
1002	1716	static_branch_enable(&mds_idle_clear);
1003		- else
	1717	+ } else if (mmio_mitigation == MMIO_MITIGATION_OFF \|\|
	1718	+ (ia32_cap & ARCH_CAP_FBSDP_NO)) {
1004	1719	static_branch_disable(&mds_idle_clear);
	1720	+ }
1005	1721	}
1006	1722
1007	1723	#define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
1008	1724	#define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
	1725	+#define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n"
1009	1726
1010		-void arch_smt_update(void)
	1727	+void cpu_bugs_smt_update(void)
1011	1728	{
1012	1729	mutex_lock(&spec_ctrl_mutex);
	1730	+
	1731	+ if (sched_smt_active() && unprivileged_ebpf_enabled() &&
	1732	+ spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
	1733	+ pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
1013	1734
1014	1735	switch (spectre_v2_user_stibp) {
1015	1736	case SPECTRE_V2_USER_NONE:
..	..	@@ -1043,6 +1764,16 @@
1043	1764	break;
1044	1765	case TAA_MITIGATION_TSX_DISABLED:
1045	1766	case TAA_MITIGATION_OFF:
	1767	+ break;
	1768	+ }
	1769	+
	1770	+ switch (mmio_mitigation) {
	1771	+ case MMIO_MITIGATION_VERW:
	1772	+ case MMIO_MITIGATION_UCODE_NEEDED:
	1773	+ if (sched_smt_active())
	1774	+ pr_warn_once(MMIO_MSG_SMT);
	1775	+ break;
	1776	+ case MMIO_MITIGATION_OFF:
1046	1777	break;
1047	1778	}
1048	1779
..	..	@@ -1150,16 +1881,6 @@
1150	1881	}
1151	1882
1152	1883	/*
1153		- * If SSBD is controlled by the SPEC_CTRL MSR, then set the proper
1154		- * bit in the mask to allow guests to use the mitigation even in the
1155		- * case where the host does not enable it.
1156		- */
1157		- if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) \|\|
1158		- static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1159		- x86_spec_ctrl_mask \|= SPEC_CTRL_SSBD;
1160		- }
1161		-
1162		- /*
1163	1884	* We have three CPU feature flags that are in play here:
1164	1885	* - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
1165	1886	* - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
..	..	@@ -1176,7 +1897,7 @@
1176	1897	x86_amd_ssb_disable();
1177	1898	} else {
1178	1899	x86_spec_ctrl_base \|= SPEC_CTRL_SSBD;
1179		- wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
	1900	+ update_spec_ctrl(x86_spec_ctrl_base);
1180	1901	}
1181	1902	}
1182	1903
..	..	@@ -1223,15 +1944,25 @@
1223	1944	if (task_spec_ssb_force_disable(task))
1224	1945	return -EPERM;
1225	1946	task_clear_spec_ssb_disable(task);
	1947	+ task_clear_spec_ssb_noexec(task);
1226	1948	task_update_spec_tif(task);
1227	1949	break;
1228	1950	case PR_SPEC_DISABLE:
1229	1951	task_set_spec_ssb_disable(task);
	1952	+ task_clear_spec_ssb_noexec(task);
1230	1953	task_update_spec_tif(task);
1231	1954	break;
1232	1955	case PR_SPEC_FORCE_DISABLE:
1233	1956	task_set_spec_ssb_disable(task);
1234	1957	task_set_spec_ssb_force_disable(task);
	1958	+ task_clear_spec_ssb_noexec(task);
	1959	+ task_update_spec_tif(task);
	1960	+ break;
	1961	+ case PR_SPEC_DISABLE_NOEXEC:
	1962	+ if (task_spec_ssb_force_disable(task))
	1963	+ return -EPERM;
	1964	+ task_set_spec_ssb_disable(task);
	1965	+ task_set_spec_ssb_noexec(task);
1235	1966	task_update_spec_tif(task);
1236	1967	break;
1237	1968	default:
..	..	@@ -1295,6 +2026,8 @@
1295	2026	if (ctrl == PR_SPEC_FORCE_DISABLE)
1296	2027	task_set_spec_ib_force_disable(task);
1297	2028	task_update_spec_tif(task);
	2029	+ if (task == current)
	2030	+ indirect_branch_prediction_barrier();
1298	2031	break;
1299	2032	default:
1300	2033	return -ERANGE;
..	..	@@ -1335,6 +2068,8 @@
1335	2068	case SPEC_STORE_BYPASS_PRCTL:
1336	2069	if (task_spec_ssb_force_disable(task))
1337	2070	return PR_SPEC_PRCTL \| PR_SPEC_FORCE_DISABLE;
	2071	+ if (task_spec_ssb_noexec(task))
	2072	+ return PR_SPEC_PRCTL \| PR_SPEC_DISABLE_NOEXEC;
1338	2073	if (task_spec_ssb_disable(task))
1339	2074	return PR_SPEC_PRCTL \| PR_SPEC_DISABLE;
1340	2075	return PR_SPEC_PRCTL \| PR_SPEC_ENABLE;
..	..	@@ -1382,7 +2117,7 @@
1382	2117	void x86_spec_ctrl_setup_ap(void)
1383	2118	{
1384	2119	if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
1385		- wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
	2120	+ update_spec_ctrl(x86_spec_ctrl_base);
1386	2121
1387	2122	if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
1388	2123	x86_amd_ssb_disable();
..	..	@@ -1426,15 +2161,15 @@
1426	2161	case INTEL_FAM6_WESTMERE:
1427	2162	case INTEL_FAM6_SANDYBRIDGE:
1428	2163	case INTEL_FAM6_IVYBRIDGE:
1429		- case INTEL_FAM6_HASWELL_CORE:
1430		- case INTEL_FAM6_HASWELL_ULT:
1431		- case INTEL_FAM6_HASWELL_GT3E:
1432		- case INTEL_FAM6_BROADWELL_CORE:
1433		- case INTEL_FAM6_BROADWELL_GT3E:
1434		- case INTEL_FAM6_SKYLAKE_MOBILE:
1435		- case INTEL_FAM6_SKYLAKE_DESKTOP:
1436		- case INTEL_FAM6_KABYLAKE_MOBILE:
1437		- case INTEL_FAM6_KABYLAKE_DESKTOP:
	2164	+ case INTEL_FAM6_HASWELL:
	2165	+ case INTEL_FAM6_HASWELL_L:
	2166	+ case INTEL_FAM6_HASWELL_G:
	2167	+ case INTEL_FAM6_BROADWELL:
	2168	+ case INTEL_FAM6_BROADWELL_G:
	2169	+ case INTEL_FAM6_SKYLAKE_L:
	2170	+ case INTEL_FAM6_SKYLAKE:
	2171	+ case INTEL_FAM6_KABYLAKE_L:
	2172	+ case INTEL_FAM6_KABYLAKE:
1438	2173	if (c->x86_cache_bits < 44)
1439	2174	c->x86_cache_bits = 44;
1440	2175	break;
..	..	@@ -1514,6 +2249,170 @@
1514	2249	early_param("l1tf", l1tf_cmdline);
1515	2250
1516	2251	#undef pr_fmt
	2252	+#define pr_fmt(fmt) "Speculative Return Stack Overflow: " fmt
	2253	+
	2254	+enum srso_mitigation {
	2255	+ SRSO_MITIGATION_NONE,
	2256	+ SRSO_MITIGATION_MICROCODE,
	2257	+ SRSO_MITIGATION_SAFE_RET,
	2258	+ SRSO_MITIGATION_IBPB,
	2259	+ SRSO_MITIGATION_IBPB_ON_VMEXIT,
	2260	+};
	2261	+
	2262	+enum srso_mitigation_cmd {
	2263	+ SRSO_CMD_OFF,
	2264	+ SRSO_CMD_MICROCODE,
	2265	+ SRSO_CMD_SAFE_RET,
	2266	+ SRSO_CMD_IBPB,
	2267	+ SRSO_CMD_IBPB_ON_VMEXIT,
	2268	+};
	2269	+
	2270	+static const char * const srso_strings[] = {
	2271	+ [SRSO_MITIGATION_NONE] = "Vulnerable",
	2272	+ [SRSO_MITIGATION_MICROCODE] = "Mitigation: microcode",
	2273	+ [SRSO_MITIGATION_SAFE_RET] = "Mitigation: safe RET",
	2274	+ [SRSO_MITIGATION_IBPB] = "Mitigation: IBPB",
	2275	+ [SRSO_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT only"
	2276	+};
	2277	+
	2278	+static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_NONE;
	2279	+static enum srso_mitigation_cmd srso_cmd __ro_after_init = SRSO_CMD_SAFE_RET;
	2280	+
	2281	+static int __init srso_parse_cmdline(char *str)
	2282	+{
	2283	+ if (!str)
	2284	+ return -EINVAL;
	2285	+
	2286	+ if (!strcmp(str, "off"))
	2287	+ srso_cmd = SRSO_CMD_OFF;
	2288	+ else if (!strcmp(str, "microcode"))
	2289	+ srso_cmd = SRSO_CMD_MICROCODE;
	2290	+ else if (!strcmp(str, "safe-ret"))
	2291	+ srso_cmd = SRSO_CMD_SAFE_RET;
	2292	+ else if (!strcmp(str, "ibpb"))
	2293	+ srso_cmd = SRSO_CMD_IBPB;
	2294	+ else if (!strcmp(str, "ibpb-vmexit"))
	2295	+ srso_cmd = SRSO_CMD_IBPB_ON_VMEXIT;
	2296	+ else
	2297	+ pr_err("Ignoring unknown SRSO option (%s).", str);
	2298	+
	2299	+ return 0;
	2300	+}
	2301	+early_param("spec_rstack_overflow", srso_parse_cmdline);
	2302	+
	2303	+#define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options."
	2304	+
	2305	+static void __init srso_select_mitigation(void)
	2306	+{
	2307	+ bool has_microcode;
	2308	+
	2309	+ if (!boot_cpu_has_bug(X86_BUG_SRSO) \|\| cpu_mitigations_off())
	2310	+ goto pred_cmd;
	2311	+
	2312	+ /*
	2313	+ * The first check is for the kernel running as a guest in order
	2314	+ * for guests to verify whether IBPB is a viable mitigation.
	2315	+ */
	2316	+ has_microcode = boot_cpu_has(X86_FEATURE_IBPB_BRTYPE) \|\| cpu_has_ibpb_brtype_microcode();
	2317	+ if (!has_microcode) {
	2318	+ pr_warn("IBPB-extending microcode not applied!\n");
	2319	+ pr_warn(SRSO_NOTICE);
	2320	+ } else {
	2321	+ /*
	2322	+ * Enable the synthetic (even if in a real CPUID leaf)
	2323	+ * flags for guests.
	2324	+ */
	2325	+ setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE);
	2326	+
	2327	+ /*
	2328	+ * Zen1/2 with SMT off aren't vulnerable after the right
	2329	+ * IBPB microcode has been applied.
	2330	+ */
	2331	+ if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) {
	2332	+ setup_force_cpu_cap(X86_FEATURE_SRSO_NO);
	2333	+ return;
	2334	+ }
	2335	+ }
	2336	+
	2337	+ if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
	2338	+ if (has_microcode) {
	2339	+ pr_err("Retbleed IBPB mitigation enabled, using same for SRSO\n");
	2340	+ srso_mitigation = SRSO_MITIGATION_IBPB;
	2341	+ goto pred_cmd;
	2342	+ }
	2343	+ }
	2344	+
	2345	+ switch (srso_cmd) {
	2346	+ case SRSO_CMD_OFF:
	2347	+ goto pred_cmd;
	2348	+
	2349	+ case SRSO_CMD_MICROCODE:
	2350	+ if (has_microcode) {
	2351	+ srso_mitigation = SRSO_MITIGATION_MICROCODE;
	2352	+ pr_warn(SRSO_NOTICE);
	2353	+ }
	2354	+ break;
	2355	+
	2356	+ case SRSO_CMD_SAFE_RET:
	2357	+ if (IS_ENABLED(CONFIG_CPU_SRSO)) {
	2358	+ /*
	2359	+ * Enable the return thunk for generated code
	2360	+ * like ftrace, static_call, etc.
	2361	+ */
	2362	+ setup_force_cpu_cap(X86_FEATURE_RETHUNK);
	2363	+ setup_force_cpu_cap(X86_FEATURE_UNRET);
	2364	+
	2365	+ if (boot_cpu_data.x86 == 0x19) {
	2366	+ setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS);
	2367	+ x86_return_thunk = srso_alias_return_thunk;
	2368	+ } else {
	2369	+ setup_force_cpu_cap(X86_FEATURE_SRSO);
	2370	+ x86_return_thunk = srso_return_thunk;
	2371	+ }
	2372	+ srso_mitigation = SRSO_MITIGATION_SAFE_RET;
	2373	+ } else {
	2374	+ pr_err("WARNING: kernel not compiled with CPU_SRSO.\n");
	2375	+ goto pred_cmd;
	2376	+ }
	2377	+ break;
	2378	+
	2379	+ case SRSO_CMD_IBPB:
	2380	+ if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) {
	2381	+ if (has_microcode) {
	2382	+ setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
	2383	+ srso_mitigation = SRSO_MITIGATION_IBPB;
	2384	+ }
	2385	+ } else {
	2386	+ pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n");
	2387	+ goto pred_cmd;
	2388	+ }
	2389	+ break;
	2390	+
	2391	+ case SRSO_CMD_IBPB_ON_VMEXIT:
	2392	+ if (IS_ENABLED(CONFIG_CPU_SRSO)) {
	2393	+ if (!boot_cpu_has(X86_FEATURE_ENTRY_IBPB) && has_microcode) {
	2394	+ setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
	2395	+ srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT;
	2396	+ }
	2397	+ } else {
	2398	+ pr_err("WARNING: kernel not compiled with CPU_SRSO.\n");
	2399	+ goto pred_cmd;
	2400	+ }
	2401	+ break;
	2402	+
	2403	+ default:
	2404	+ break;
	2405	+ }
	2406	+
	2407	+ pr_info("%s%s\n", srso_strings[srso_mitigation], (has_microcode ? "" : ", no microcode"));
	2408	+
	2409	+pred_cmd:
	2410	+ if ((boot_cpu_has(X86_FEATURE_SRSO_NO) \|\| srso_cmd == SRSO_CMD_OFF) &&
	2411	+ boot_cpu_has(X86_FEATURE_SBPB))
	2412	+ x86_pred_cmd = PRED_CMD_SBPB;
	2413	+}
	2414	+
	2415	+#undef pr_fmt
1517	2416	#define pr_fmt(fmt) fmt
1518	2417
1519	2418	#ifdef CONFIG_SYSFS
..	..	@@ -1549,7 +2448,12 @@
1549	2448
1550	2449	static ssize_t itlb_multihit_show_state(char *buf)
1551	2450	{
1552		- if (itlb_multihit_kvm_mitigation)
	2451	+ if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) \|\|
	2452	+ !boot_cpu_has(X86_FEATURE_VMX))
	2453	+ return sprintf(buf, "KVM: Mitigation: VMX unsupported\n");
	2454	+ else if (!(cr4_read_shadow() & X86_CR4_VMXE))
	2455	+ return sprintf(buf, "KVM: Mitigation: VMX disabled\n");
	2456	+ else if (itlb_multihit_kvm_mitigation)
1553	2457	return sprintf(buf, "KVM: Mitigation: Split huge pages\n");
1554	2458	else
1555	2459	return sprintf(buf, "KVM: Vulnerable\n");
..	..	@@ -1598,9 +2502,26 @@
1598	2502	sched_smt_active() ? "vulnerable" : "disabled");
1599	2503	}
1600	2504
	2505	+static ssize_t mmio_stale_data_show_state(char *buf)
	2506	+{
	2507	+ if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
	2508	+ return sysfs_emit(buf, "Unknown: No mitigations\n");
	2509	+
	2510	+ if (mmio_mitigation == MMIO_MITIGATION_OFF)
	2511	+ return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]);
	2512	+
	2513	+ if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
	2514	+ return sysfs_emit(buf, "%s; SMT Host state unknown\n",
	2515	+ mmio_strings[mmio_mitigation]);
	2516	+ }
	2517	+
	2518	+ return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation],
	2519	+ sched_smt_active() ? "vulnerable" : "disabled");
	2520	+}
	2521	+
1601	2522	static char *stibp_state(void)
1602	2523	{
1603		- if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
	2524	+ if (spectre_v2_in_eibrs_mode(spectre_v2_enabled))
1604	2525	return "";
1605	2526
1606	2527	switch (spectre_v2_user_stibp) {
..	..	@@ -1630,9 +2551,78 @@
1630	2551	return "";
1631	2552	}
1632	2553
	2554	+static char *pbrsb_eibrs_state(void)
	2555	+{
	2556	+ if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {
	2557	+ if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) \|\|
	2558	+ boot_cpu_has(X86_FEATURE_RSB_VMEXIT))
	2559	+ return ", PBRSB-eIBRS: SW sequence";
	2560	+ else
	2561	+ return ", PBRSB-eIBRS: Vulnerable";
	2562	+ } else {
	2563	+ return ", PBRSB-eIBRS: Not affected";
	2564	+ }
	2565	+}
	2566	+
	2567	+static ssize_t spectre_v2_show_state(char *buf)
	2568	+{
	2569	+ if (spectre_v2_enabled == SPECTRE_V2_LFENCE)
	2570	+ return sprintf(buf, "Vulnerable: LFENCE\n");
	2571	+
	2572	+ if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
	2573	+ return sprintf(buf, "Vulnerable: eIBRS with unprivileged eBPF\n");
	2574	+
	2575	+ if (sched_smt_active() && unprivileged_ebpf_enabled() &&
	2576	+ spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
	2577	+ return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
	2578	+
	2579	+ return sprintf(buf, "%s%s%s%s%s%s%s\n",
	2580	+ spectre_v2_strings[spectre_v2_enabled],
	2581	+ ibpb_state(),
	2582	+ boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
	2583	+ stibp_state(),
	2584	+ boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
	2585	+ pbrsb_eibrs_state(),
	2586	+ spectre_v2_module_string());
	2587	+}
	2588	+
1633	2589	static ssize_t srbds_show_state(char *buf)
1634	2590	{
1635	2591	return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]);
	2592	+}
	2593	+
	2594	+static ssize_t retbleed_show_state(char *buf)
	2595	+{
	2596	+ if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET \|\|
	2597	+ retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
	2598	+ if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
	2599	+ boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
	2600	+ return sprintf(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n");
	2601	+
	2602	+ return sprintf(buf, "%s; SMT %s\n",
	2603	+ retbleed_strings[retbleed_mitigation],
	2604	+ !sched_smt_active() ? "disabled" :
	2605	+ spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT \|\|
	2606	+ spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ?
	2607	+ "enabled with STIBP protection" : "vulnerable");
	2608	+ }
	2609	+
	2610	+ return sprintf(buf, "%s\n", retbleed_strings[retbleed_mitigation]);
	2611	+}
	2612	+
	2613	+static ssize_t gds_show_state(char *buf)
	2614	+{
	2615	+ return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]);
	2616	+}
	2617	+
	2618	+static ssize_t srso_show_state(char *buf)
	2619	+{
	2620	+ if (boot_cpu_has(X86_FEATURE_SRSO_NO))
	2621	+ return sysfs_emit(buf, "Mitigation: SMT disabled\n");
	2622	+
	2623	+ return sysfs_emit(buf, "%s%s\n",
	2624	+ srso_strings[srso_mitigation],
	2625	+ boot_cpu_has(X86_FEATURE_IBPB_BRTYPE) ? "" : ", no microcode");
1636	2626	}
1637	2627
1638	2628	static ssize_t cpu_show_common(struct device dev, struct device_attribute attr,
..	..	@@ -1655,12 +2645,7 @@
1655	2645	return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
1656	2646
1657	2647	case X86_BUG_SPECTRE_V2:
1658		- return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
1659		- ibpb_state(),
1660		- boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
1661		- stibp_state(),
1662		- boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
1663		- spectre_v2_module_string());
	2648	+ return spectre_v2_show_state(buf);
1664	2649
1665	2650	case X86_BUG_SPEC_STORE_BYPASS:
1666	2651	return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
..	..	@@ -1681,6 +2666,19 @@
1681	2666
1682	2667	case X86_BUG_SRBDS:
1683	2668	return srbds_show_state(buf);
	2669	+
	2670	+ case X86_BUG_MMIO_STALE_DATA:
	2671	+ case X86_BUG_MMIO_UNKNOWN:
	2672	+ return mmio_stale_data_show_state(buf);
	2673	+
	2674	+ case X86_BUG_RETBLEED:
	2675	+ return retbleed_show_state(buf);
	2676	+
	2677	+ case X86_BUG_GDS:
	2678	+ return gds_show_state(buf);
	2679	+
	2680	+ case X86_BUG_SRSO:
	2681	+ return srso_show_state(buf);
1684	2682
1685	2683	default:
1686	2684	break;
..	..	@@ -1733,4 +2731,27 @@
1733	2731	{
1734	2732	return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS);
1735	2733	}
	2734	+
	2735	+ssize_t cpu_show_mmio_stale_data(struct device dev, struct device_attribute attr, char *buf)
	2736	+{
	2737	+ if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
	2738	+ return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_UNKNOWN);
	2739	+ else
	2740	+ return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA);
	2741	+}
	2742	+
	2743	+ssize_t cpu_show_retbleed(struct device dev, struct device_attribute attr, char *buf)
	2744	+{
	2745	+ return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED);
	2746	+}
	2747	+
	2748	+ssize_t cpu_show_gds(struct device dev, struct device_attribute attr, char *buf)
	2749	+{
	2750	+ return cpu_show_common(dev, attr, buf, X86_BUG_GDS);
	2751	+}
	2752	+
	2753	+ssize_t cpu_show_spec_rstack_overflow(struct device dev, struct device_attribute attr, char *buf)
	2754	+{
	2755	+ return cpu_show_common(dev, attr, buf, X86_BUG_SRSO);
	2756	+}
1736	2757	#endif