forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 95099d4622f8cb224d94e314c7a8e0df60b13f87 
 kernel/arch/x86/kernel/cpu/amd.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 #include <linux/export.h>
23 #include <linux/bitops.h>
34 #include <linux/elf.h>
....@@ -7,14 +8,18 @@
78 #include <linux/sched.h>
89 #include <linux/sched/clock.h>
910 #include <linux/random.h>
11
+#include <linux/topology.h>
1012 #include <asm/processor.h>
1113 #include <asm/apic.h>
1214 #include <asm/cacheinfo.h>
1315 #include <asm/cpu.h>
1416 #include <asm/spec-ctrl.h>
1517 #include <asm/smp.h>
18
+#include <asm/numa.h>
1619 #include <asm/pci-direct.h>
1720 #include <asm/delay.h>
21
+#include <asm/debugreg.h>
22
+#include <asm/resctrl.h>
1823 
1924 #ifdef CONFIG_X86_64
2025 # include <asm/mmconfig.h>
....@@ -82,11 +87,14 @@
8287  *	performance at the same time..
8388  */
8489 
90
+#ifdef CONFIG_X86_32
8591 extern __visible void vide(void);
86
-__asm__(".globl vide\n"
92
+__asm__(".text\n"
93
+	".globl vide\n"
8794 	".type vide, @function\n"
8895 	".align 4\n"
8996 	"vide: ret\n");
97
+#endif
9098 
9199 static void init_amd_k5(struct cpuinfo_x86 *c)
92100 {
....@@ -314,13 +322,6 @@
314322 	c->cpu_core_id %= cus_per_node;
315323 }
316324 
317
-
318
-static void amd_get_topology_early(struct cpuinfo_x86 *c)
319
-{
320
-	if (cpu_has(c, X86_FEATURE_TOPOEXT))
321
-		smp_num_siblings = ((cpuid_ebx(0x8000001e) >> 8) & 0xff) + 1;
322
-}
323
-
324325 /*
325326  * Fixup core topology information for
326327  * (1) AMD multi-node processors
....@@ -329,7 +330,6 @@
329330  */
330331 static void amd_get_topology(struct cpuinfo_x86 *c)
331332 {
332
-	u8 node_id;
333333 	int cpu = smp_processor_id();
334334 
335335 	/* get information required for multi-node processors */
....@@ -339,7 +339,7 @@
339339 
340340 		cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
341341 
342
-		node_id  = ecx & 0xff;
342
+		c->cpu_die_id  = ecx & 0xff;
343343 
344344 		if (c->x86 == 0x15)
345345 			c->cu_id = ebx & 0xff;
....@@ -359,15 +359,15 @@
359359 		if (!err)
360360 			c->x86_coreid_bits = get_count_order(c->x86_max_cores);
361361 
362
-		cacheinfo_amd_init_llc_id(c, cpu, node_id);
362
+		cacheinfo_amd_init_llc_id(c, cpu);
363363 
364364 	} else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) {
365365 		u64 value;
366366 
367367 		rdmsrl(MSR_FAM10H_NODE_ID, value);
368
-		node_id = value & 7;
368
+		c->cpu_die_id = value & 7;
369369 
370
-		per_cpu(cpu_llc_id, cpu) = node_id;
370
+		per_cpu(cpu_llc_id, cpu) = c->cpu_die_id;
371371 	} else
372372 		return;
373373 
....@@ -392,7 +392,36 @@
392392 	/* Convert the initial APIC ID into the socket ID */
393393 	c->phys_proc_id = c->initial_apicid >> bits;
394394 	/* use socket ID also for last level cache */
395
-	per_cpu(cpu_llc_id, cpu) = c->phys_proc_id;
395
+	per_cpu(cpu_llc_id, cpu) = c->cpu_die_id = c->phys_proc_id;
396
+}
397
+
398
+static void amd_detect_ppin(struct cpuinfo_x86 *c)
399
+{
400
+	unsigned long long val;
401
+
402
+	if (!cpu_has(c, X86_FEATURE_AMD_PPIN))
403
+		return;
404
+
405
+	/* When PPIN is defined in CPUID, still need to check PPIN_CTL MSR */
406
+	if (rdmsrl_safe(MSR_AMD_PPIN_CTL, &val))
407
+		goto clear_ppin;
408
+
409
+	/* PPIN is locked in disabled mode, clear feature bit */
410
+	if ((val & 3UL) == 1UL)
411
+		goto clear_ppin;
412
+
413
+	/* If PPIN is disabled, try to enable it */
414
+	if (!(val & 2UL)) {
415
+		wrmsrl_safe(MSR_AMD_PPIN_CTL,  val | 2UL);
416
+		rdmsrl_safe(MSR_AMD_PPIN_CTL, &val);
417
+	}
418
+
419
+	/* If PPIN_EN bit is 1, return from here; otherwise fall through */
420
+	if (val & 2UL)
421
+		return;
422
+
423
+clear_ppin:
424
+	clear_cpu_cap(c, X86_FEATURE_AMD_PPIN);
396425 }
397426 
398427 u16 amd_get_nb_id(int cpu)
....@@ -540,12 +569,12 @@
540569 		u32 ecx;
541570 
542571 		ecx = cpuid_ecx(0x8000001e);
543
-		nodes_per_socket = ((ecx >> 8) & 7) + 1;
572
+		__max_die_per_package = nodes_per_socket = ((ecx >> 8) & 7) + 1;
544573 	} else if (boot_cpu_has(X86_FEATURE_NODEID_MSR)) {
545574 		u64 value;
546575 
547576 		rdmsrl(MSR_FAM10H_NODE_ID, value);
548
-		nodes_per_socket = ((value >> 3) & 7) + 1;
577
+		__max_die_per_package = nodes_per_socket = ((value >> 3) & 7) + 1;
549578 	}
550579 
551580 	if (!boot_cpu_has(X86_FEATURE_AMD_SSBD) &&
....@@ -569,6 +598,8 @@
569598 			x86_amd_ls_cfg_ssbd_mask = 1ULL << bit;
570599 		}
571600 	}
601
+
602
+	resctrl_cpu_detect(c);
572603 }
573604 
574605 static void early_detect_mem_encrypt(struct cpuinfo_x86 *c)
....@@ -582,7 +613,7 @@
582613 	 *	      If BIOS has not enabled SME then don't advertise the
583614 	 *	      SME feature (set in scattered.c).
584615 	 *   For SEV: If BIOS has not enabled SEV then don't advertise the
585
-	 *            SEV feature (set in scattered.c).
616
+	 *            SEV and SEV_ES feature (set in scattered.c).
586617 	 *
587618 	 *   In all cases, since support for SME and SEV requires long mode,
588619 	 *   don't advertise the feature under CONFIG_X86_32.
....@@ -613,6 +644,7 @@
613644 		setup_clear_cpu_cap(X86_FEATURE_SME);
614645 clear_sev:
615646 		setup_clear_cpu_cap(X86_FEATURE_SEV);
647
+		setup_clear_cpu_cap(X86_FEATURE_SEV_ES);
616648 	}
617649 }
618650 
....@@ -712,7 +744,8 @@
712744 		}
713745 	}
714746 
715
-	amd_get_topology_early(c);
747
+	if (cpu_has(c, X86_FEATURE_TOPOEXT))
748
+		smp_num_siblings = ((cpuid_ebx(0x8000001e) >> 8) & 0xff) + 1;
716749 }
717750 
718751 static void init_amd_k8(struct cpuinfo_x86 *c)
....@@ -788,8 +821,6 @@
788821 	if (cpu_has_amd_erratum(c, amd_erratum_383))
789822 		set_cpu_bug(c, X86_BUG_AMD_TLB_MMATCH);
790823 }
791
-
792
-#define MSR_AMD64_DE_CFG	0xC0011029
793824 
794825 static void init_amd_ln(struct cpuinfo_x86 *c)
795826 {
....@@ -881,16 +912,51 @@
881912 	clear_rdrand_cpuid_bit(c);
882913 }
883914 
915
+void init_spectral_chicken(struct cpuinfo_x86 *c)
916
+{
917
+#ifdef CONFIG_CPU_UNRET_ENTRY
918
+	u64 value;
919
+
920
+	/*
921
+	 * On Zen2 we offer this chicken (bit) on the altar of Speculation.
922
+	 *
923
+	 * This suppresses speculation from the middle of a basic block, i.e. it
924
+	 * suppresses non-branch predictions.
925
+	 *
926
+	 * We use STIBP as a heuristic to filter out Zen2 from the rest of F17H
927
+	 */
928
+	if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && cpu_has(c, X86_FEATURE_AMD_STIBP)) {
929
+		if (!rdmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, &value)) {
930
+			value |= MSR_ZEN2_SPECTRAL_CHICKEN_BIT;
931
+			wrmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, value);
932
+		}
933
+	}
934
+#endif
935
+}
936
+
884937 static void init_amd_zn(struct cpuinfo_x86 *c)
885938 {
886939 	set_cpu_cap(c, X86_FEATURE_ZEN);
887940 
888
-	/*
889
-	 * Fix erratum 1076: CPB feature bit not being set in CPUID.
890
-	 * Always set it, except when running under a hypervisor.
891
-	 */
892
-	if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && !cpu_has(c, X86_FEATURE_CPB))
893
-		set_cpu_cap(c, X86_FEATURE_CPB);
941
+#ifdef CONFIG_NUMA
942
+	node_reclaim_distance = 32;
943
+#endif
944
+
945
+	/* Fix up CPUID bits, but only if not virtualised. */
946
+	if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) {
947
+
948
+		/* Erratum 1076: CPB feature bit not being set in CPUID. */
949
+		if (!cpu_has(c, X86_FEATURE_CPB))
950
+			set_cpu_cap(c, X86_FEATURE_CPB);
951
+
952
+		/*
953
+		 * Zen3 (Fam19 model < 0x10) parts are not susceptible to
954
+		 * Branch Type Confusion, but predate the allocation of the
955
+		 * BTC_NO bit.
956
+		 */
957
+		if (c->x86 == 0x19 && !cpu_has(c, X86_FEATURE_BTC_NO))
958
+			set_cpu_cap(c, X86_FEATURE_BTC_NO);
959
+	}
894960 }
895961 
896962 static void init_amd(struct cpuinfo_x86 *c)
....@@ -922,7 +988,9 @@
922988 	case 0x12: init_amd_ln(c); break;
923989 	case 0x15: init_amd_bd(c); break;
924990 	case 0x16: init_amd_jg(c); break;
925
-	case 0x17: init_amd_zn(c); break;
991
+	case 0x17: init_spectral_chicken(c);
992
+		   fallthrough;
993
+	case 0x19: init_amd_zn(c); break;
926994 	}
927995 
928996 	/*
....@@ -937,36 +1005,22 @@
9371005 	amd_detect_cmp(c);
9381006 	amd_get_topology(c);
9391007 	srat_detect_node(c);
1008
+	amd_detect_ppin(c);
9401009 
9411010 	init_amd_cacheinfo(c);
9421011 
9431012 	if (cpu_has(c, X86_FEATURE_XMM2)) {
944
-		unsigned long long val;
945
-		int ret;
946
-
9471013 		/*
948
-		 * A serializing LFENCE has less overhead than MFENCE, so
949
-		 * use it for execution serialization.  On families which
1014
+		 * Use LFENCE for execution serialization.  On families which
9501015 		 * don't have that MSR, LFENCE is already serializing.
9511016 		 * msr_set_bit() uses the safe accessors, too, even if the MSR
9521017 		 * is not present.
9531018 		 */
954
-		msr_set_bit(MSR_F10H_DECFG,
955
-			    MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT);
1019
+		msr_set_bit(MSR_AMD64_DE_CFG,
1020
+			    MSR_AMD64_DE_CFG_LFENCE_SERIALIZE_BIT);
9561021 
957
-		/*
958
-		 * Verify that the MSR write was successful (could be running
959
-		 * under a hypervisor) and only then assume that LFENCE is
960
-		 * serializing.
961
-		 */
962
-		ret = rdmsrl_safe(MSR_F10H_DECFG, &val);
963
-		if (!ret && (val & MSR_F10H_DECFG_LFENCE_SERIALIZE)) {
964
-			/* A serializing LFENCE stops RDTSC speculation */
965
-			set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
966
-		} else {
967
-			/* MFENCE stops RDTSC speculation */
968
-			set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
969
-		}
1022
+		/* A serializing LFENCE stops RDTSC speculation */
1023
+		set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
9701024 	}
9711025 
9721026 	/*