forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-01-03 2f7c68cb55ecb7331f2381deb497c27155f32faf 
 kernel/arch/x86/kernel/cpu/amd.c
....@@ -28,17 +28,89 @@
2828 
2929 #include "cpu.h"
3030 
31
-static const int amd_erratum_383[];
32
-static const int amd_erratum_400[];
33
-static const int amd_erratum_1054[];
34
-static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum);
35
-
3631 /*
3732  * nodes_per_socket: Stores the number of nodes per socket.
3833  * Refer to Fam15h Models 00-0fh BKDG - CPUID Fn8000_001E_ECX
3934  * Node Identifiers[10:8]
4035  */
4136 static u32 nodes_per_socket = 1;
37
+
38
+/*
39
+ * AMD errata checking
40
+ *
41
+ * Errata are defined as arrays of ints using the AMD_LEGACY_ERRATUM() or
42
+ * AMD_OSVW_ERRATUM() macros. The latter is intended for newer errata that
43
+ * have an OSVW id assigned, which it takes as first argument. Both take a
44
+ * variable number of family-specific model-stepping ranges created by
45
+ * AMD_MODEL_RANGE().
46
+ *
47
+ * Example:
48
+ *
49
+ * const int amd_erratum_319[] =
50
+ *	AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0x4, 0x2),
51
+ *			   AMD_MODEL_RANGE(0x10, 0x8, 0x0, 0x8, 0x0),
52
+ *			   AMD_MODEL_RANGE(0x10, 0x9, 0x0, 0x9, 0x0));
53
+ */
54
+
55
+#define AMD_LEGACY_ERRATUM(...)		{ -1, __VA_ARGS__, 0 }
56
+#define AMD_OSVW_ERRATUM(osvw_id, ...)	{ osvw_id, __VA_ARGS__, 0 }
57
+#define AMD_MODEL_RANGE(f, m_start, s_start, m_end, s_end) \
58
+	((f << 24) | (m_start << 16) | (s_start << 12) | (m_end << 4) | (s_end))
59
+#define AMD_MODEL_RANGE_FAMILY(range)	(((range) >> 24) & 0xff)
60
+#define AMD_MODEL_RANGE_START(range)	(((range) >> 12) & 0xfff)
61
+#define AMD_MODEL_RANGE_END(range)	((range) & 0xfff)
62
+
63
+static const int amd_erratum_400[] =
64
+	AMD_OSVW_ERRATUM(1, AMD_MODEL_RANGE(0xf, 0x41, 0x2, 0xff, 0xf),
65
+			    AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0xff, 0xf));
66
+
67
+static const int amd_erratum_383[] =
68
+	AMD_OSVW_ERRATUM(3, AMD_MODEL_RANGE(0x10, 0, 0, 0xff, 0xf));
69
+
70
+/* #1054: Instructions Retired Performance Counter May Be Inaccurate */
71
+static const int amd_erratum_1054[] =
72
+	AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x17, 0, 0, 0x2f, 0xf));
73
+
74
+static const int amd_zenbleed[] =
75
+	AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x17, 0x30, 0x0, 0x4f, 0xf),
76
+			   AMD_MODEL_RANGE(0x17, 0x60, 0x0, 0x7f, 0xf),
77
+			   AMD_MODEL_RANGE(0x17, 0x90, 0x0, 0x91, 0xf),
78
+			   AMD_MODEL_RANGE(0x17, 0xa0, 0x0, 0xaf, 0xf));
79
+
80
+static const int amd_div0[] =
81
+	AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x17, 0x00, 0x0, 0x2f, 0xf),
82
+			   AMD_MODEL_RANGE(0x17, 0x50, 0x0, 0x5f, 0xf));
83
+
84
+static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum)
85
+{
86
+	int osvw_id = *erratum++;
87
+	u32 range;
88
+	u32 ms;
89
+
90
+	if (osvw_id >= 0 && osvw_id < 65536 &&
91
+	    cpu_has(cpu, X86_FEATURE_OSVW)) {
92
+		u64 osvw_len;
93
+
94
+		rdmsrl(MSR_AMD64_OSVW_ID_LENGTH, osvw_len);
95
+		if (osvw_id < osvw_len) {
96
+			u64 osvw_bits;
97
+
98
+			rdmsrl(MSR_AMD64_OSVW_STATUS + (osvw_id >> 6),
99
+			    osvw_bits);
100
+			return osvw_bits & (1ULL << (osvw_id & 0x3f));
101
+		}
102
+	}
103
+
104
+	/* OSVW unavailable or ID unknown, match family-model-stepping range */
105
+	ms = (cpu->x86_model << 4) | cpu->x86_stepping;
106
+	while ((range = *erratum++))
107
+		if ((cpu->x86 == AMD_MODEL_RANGE_FAMILY(range)) &&
108
+		    (ms >= AMD_MODEL_RANGE_START(range)) &&
109
+		    (ms <= AMD_MODEL_RANGE_END(range)))
110
+			return true;
111
+
112
+	return false;
113
+}
42114 
43115 static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
44116 {
....@@ -932,6 +1004,15 @@
9321004 		}
9331005 	}
9341006 #endif
1007
+	/*
1008
+	 * Work around Erratum 1386.  The XSAVES instruction malfunctions in
1009
+	 * certain circumstances on Zen1/2 uarch, and not all parts have had
1010
+	 * updated microcode at the time of writing (March 2023).
1011
+	 *
1012
+	 * Affected parts all have no supervisor XSAVE states, meaning that
1013
+	 * the XSAVEC instruction (which works fine) is equivalent.
1014
+	 */
1015
+	clear_cpu_cap(c, X86_FEATURE_XSAVES);
9351016 }
9361017 
9371018 static void init_amd_zn(struct cpuinfo_x86 *c)
....@@ -956,6 +1037,47 @@
9561037 		 */
9571038 		if (c->x86 == 0x19 && !cpu_has(c, X86_FEATURE_BTC_NO))
9581039 			set_cpu_cap(c, X86_FEATURE_BTC_NO);
1040
+	}
1041
+}
1042
+
1043
+static bool cpu_has_zenbleed_microcode(void)
1044
+{
1045
+	u32 good_rev = 0;
1046
+
1047
+	switch (boot_cpu_data.x86_model) {
1048
+	case 0x30 ... 0x3f: good_rev = 0x0830107a; break;
1049
+	case 0x60 ... 0x67: good_rev = 0x0860010b; break;
1050
+	case 0x68 ... 0x6f: good_rev = 0x08608105; break;
1051
+	case 0x70 ... 0x7f: good_rev = 0x08701032; break;
1052
+	case 0xa0 ... 0xaf: good_rev = 0x08a00008; break;
1053
+
1054
+	default:
1055
+		return false;
1056
+		break;
1057
+	}
1058
+
1059
+	if (boot_cpu_data.microcode < good_rev)
1060
+		return false;
1061
+
1062
+	return true;
1063
+}
1064
+
1065
+static void zenbleed_check(struct cpuinfo_x86 *c)
1066
+{
1067
+	if (!cpu_has_amd_erratum(c, amd_zenbleed))
1068
+		return;
1069
+
1070
+	if (cpu_has(c, X86_FEATURE_HYPERVISOR))
1071
+		return;
1072
+
1073
+	if (!cpu_has(c, X86_FEATURE_AVX))
1074
+		return;
1075
+
1076
+	if (!cpu_has_zenbleed_microcode()) {
1077
+		pr_notice_once("Zenbleed: please update your microcode for the most optimal fix\n");
1078
+		msr_set_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT);
1079
+	} else {
1080
+		msr_clear_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT);
9591081 	}
9601082 }
9611083 
....@@ -1049,6 +1171,13 @@
10491171 		msr_set_bit(MSR_K7_HWCR, MSR_K7_HWCR_IRPERF_EN_BIT);
10501172 
10511173 	check_null_seg_clears_base(c);
1174
+
1175
+	zenbleed_check(c);
1176
+
1177
+	if (cpu_has_amd_erratum(c, amd_div0)) {
1178
+		pr_notice_once("AMD Zen1 DIV0 bug detected. Disable SMT for full protection.\n");
1179
+		setup_force_cpu_bug(X86_BUG_DIV0);
1180
+	}
10521181 }
10531182 
10541183 #ifdef CONFIG_X86_32
....@@ -1144,73 +1273,6 @@
11441273 
11451274 cpu_dev_register(amd_cpu_dev);
11461275 
1147
-/*
1148
- * AMD errata checking
1149
- *
1150
- * Errata are defined as arrays of ints using the AMD_LEGACY_ERRATUM() or
1151
- * AMD_OSVW_ERRATUM() macros. The latter is intended for newer errata that
1152
- * have an OSVW id assigned, which it takes as first argument. Both take a
1153
- * variable number of family-specific model-stepping ranges created by
1154
- * AMD_MODEL_RANGE().
1155
- *
1156
- * Example:
1157
- *
1158
- * const int amd_erratum_319[] =
1159
- *	AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0x4, 0x2),
1160
- *			   AMD_MODEL_RANGE(0x10, 0x8, 0x0, 0x8, 0x0),
1161
- *			   AMD_MODEL_RANGE(0x10, 0x9, 0x0, 0x9, 0x0));
1162
- */
1163
-
1164
-#define AMD_LEGACY_ERRATUM(...)		{ -1, __VA_ARGS__, 0 }
1165
-#define AMD_OSVW_ERRATUM(osvw_id, ...)	{ osvw_id, __VA_ARGS__, 0 }
1166
-#define AMD_MODEL_RANGE(f, m_start, s_start, m_end, s_end) \
1167
-	((f << 24) | (m_start << 16) | (s_start << 12) | (m_end << 4) | (s_end))
1168
-#define AMD_MODEL_RANGE_FAMILY(range)	(((range) >> 24) & 0xff)
1169
-#define AMD_MODEL_RANGE_START(range)	(((range) >> 12) & 0xfff)
1170
-#define AMD_MODEL_RANGE_END(range)	((range) & 0xfff)
1171
-
1172
-static const int amd_erratum_400[] =
1173
-	AMD_OSVW_ERRATUM(1, AMD_MODEL_RANGE(0xf, 0x41, 0x2, 0xff, 0xf),
1174
-			    AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0xff, 0xf));
1175
-
1176
-static const int amd_erratum_383[] =
1177
-	AMD_OSVW_ERRATUM(3, AMD_MODEL_RANGE(0x10, 0, 0, 0xff, 0xf));
1178
-
1179
-/* #1054: Instructions Retired Performance Counter May Be Inaccurate */
1180
-static const int amd_erratum_1054[] =
1181
-	AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x17, 0, 0, 0x2f, 0xf));
1182
-
1183
-static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum)
1184
-{
1185
-	int osvw_id = *erratum++;
1186
-	u32 range;
1187
-	u32 ms;
1188
-
1189
-	if (osvw_id >= 0 && osvw_id < 65536 &&
1190
-	    cpu_has(cpu, X86_FEATURE_OSVW)) {
1191
-		u64 osvw_len;
1192
-
1193
-		rdmsrl(MSR_AMD64_OSVW_ID_LENGTH, osvw_len);
1194
-		if (osvw_id < osvw_len) {
1195
-			u64 osvw_bits;
1196
-
1197
-			rdmsrl(MSR_AMD64_OSVW_STATUS + (osvw_id >> 6),
1198
-			    osvw_bits);
1199
-			return osvw_bits & (1ULL << (osvw_id & 0x3f));
1200
-		}
1201
-	}
1202
-
1203
-	/* OSVW unavailable or ID unknown, match family-model-stepping range */
1204
-	ms = (cpu->x86_model << 4) | cpu->x86_stepping;
1205
-	while ((range = *erratum++))
1206
-		if ((cpu->x86 == AMD_MODEL_RANGE_FAMILY(range)) &&
1207
-		    (ms >= AMD_MODEL_RANGE_START(range)) &&
1208
-		    (ms <= AMD_MODEL_RANGE_END(range)))
1209
-			return true;
1210
-
1211
-	return false;
1212
-}
1213
-
12141276 void set_dr_addr_mask(unsigned long mask, int dr)
12151277 {
12161278 	if (!boot_cpu_has(X86_FEATURE_BPEXT))
....@@ -1229,3 +1291,45 @@
12291291 		break;
12301292 	}
12311293 }
1294
+
1295
+bool cpu_has_ibpb_brtype_microcode(void)
1296
+{
1297
+	switch (boot_cpu_data.x86) {
1298
+	/* Zen1/2 IBPB flushes branch type predictions too. */
1299
+	case 0x17:
1300
+		return boot_cpu_has(X86_FEATURE_AMD_IBPB);
1301
+	case 0x19:
1302
+		/* Poke the MSR bit on Zen3/4 to check its presence. */
1303
+		if (!wrmsrl_safe(MSR_IA32_PRED_CMD, PRED_CMD_SBPB)) {
1304
+			setup_force_cpu_cap(X86_FEATURE_SBPB);
1305
+			return true;
1306
+		} else {
1307
+			return false;
1308
+		}
1309
+	default:
1310
+		return false;
1311
+	}
1312
+}
1313
+
1314
+static void zenbleed_check_cpu(void *unused)
1315
+{
1316
+	struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());
1317
+
1318
+	zenbleed_check(c);
1319
+}
1320
+
1321
+void amd_check_microcode(void)
1322
+{
1323
+	on_each_cpu(zenbleed_check_cpu, NULL, 1);
1324
+}
1325
+
1326
+/*
1327
+ * Issue a DIV 0/1 insn to clear any division data from previous DIV
1328
+ * operations.
1329
+ */
1330
+void noinstr amd_clear_divider(void)
1331
+{
1332
+	asm volatile(ALTERNATIVE("", "div %2\n\t", X86_BUG_DIV0)
1333
+		     :: "a" (0), "d" (0), "r" (1));
1334
+}
1335
+EXPORT_SYMBOL_GPL(amd_clear_divider);