forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-13 9d77db3c730780c8ef5ccd4b66403ff5675cfe4e 
 kernel/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
....@@ -26,6 +26,7 @@
2626 #include "kfd_priv.h"
2727 #include "kfd_topology.h"
2828 #include "kfd_iommu.h"
29
+#include "amdgpu_amdkfd.h"
2930 
3031 /* GPU Processor ID base for dGPUs for which VCRAT needs to be created.
3132  * GPU processor ID are expressed with Bit[31]=1.
....@@ -132,9 +133,14 @@
132133 #define fiji_cache_info  carrizo_cache_info
133134 #define polaris10_cache_info carrizo_cache_info
134135 #define polaris11_cache_info carrizo_cache_info
136
+#define polaris12_cache_info carrizo_cache_info
137
+#define vegam_cache_info carrizo_cache_info
135138 /* TODO - check & update Vega10 cache details */
136139 #define vega10_cache_info carrizo_cache_info
137140 #define raven_cache_info carrizo_cache_info
141
+#define renoir_cache_info carrizo_cache_info
142
+/* TODO - check & update Navi10 cache details */
143
+#define navi10_cache_info carrizo_cache_info
138144 
139145 static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
140146 		struct crat_subtype_computeunit *cu)
....@@ -346,15 +352,15 @@
346352 					struct list_head *device_list)
347353 {
348354 	struct kfd_iolink_properties *props = NULL, *props2;
349
-	struct kfd_topology_device *dev, *cpu_dev;
355
+	struct kfd_topology_device *dev, *to_dev;
350356 	uint32_t id_from;
351357 	uint32_t id_to;
352358 
353359 	id_from = iolink->proximity_domain_from;
354360 	id_to = iolink->proximity_domain_to;
355361 
356
-	pr_debug("Found IO link entry in CRAT table with id_from=%d\n",
357
-			id_from);
362
+	pr_debug("Found IO link entry in CRAT table with id_from=%d, id_to %d\n",
363
+			id_from, id_to);
358364 	list_for_each_entry(dev, device_list, list) {
359365 		if (id_from == dev->proximity_domain) {
360366 			props = kfd_alloc_struct(props);
....@@ -369,6 +375,8 @@
369375 
370376 			if (props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS)
371377 				props->weight = 20;
378
+			else if (props->iolink_type == CRAT_IOLINK_TYPE_XGMI)
379
+				props->weight = 15 * iolink->num_hops_xgmi;
372380 			else
373381 				props->weight = node_distance(id_from, id_to);
374382 
....@@ -389,20 +397,23 @@
389397 	/* CPU topology is created before GPUs are detected, so CPU->GPU
390398 	 * links are not built at that time. If a PCIe type is discovered, it
391399 	 * means a GPU is detected and we are adding GPU->CPU to the topology.
392
-	 * At this time, also add the corresponded CPU->GPU link.
400
+	 * At this time, also add the corresponded CPU->GPU link if GPU
401
+	 * is large bar.
402
+	 * For xGMI, we only added the link with one direction in the crat
403
+	 * table, add corresponded reversed direction link now.
393404 	 */
394
-	if (props && props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS) {
395
-		cpu_dev = kfd_topology_device_by_proximity_domain(id_to);
396
-		if (!cpu_dev)
405
+	if (props && (iolink->flags & CRAT_IOLINK_FLAGS_BI_DIRECTIONAL)) {
406
+		to_dev = kfd_topology_device_by_proximity_domain(id_to);
407
+		if (!to_dev)
397408 			return -ENODEV;
398409 		/* same everything but the other direction */
399410 		props2 = kmemdup(props, sizeof(*props2), GFP_KERNEL);
400411 		props2->node_from = id_to;
401412 		props2->node_to = id_from;
402413 		props2->kobj = NULL;
403
-		cpu_dev->io_link_count++;
404
-		cpu_dev->node_props.io_links_count++;
405
-		list_add_tail(&props2->list, &cpu_dev->io_link_props);
414
+		to_dev->io_link_count++;
415
+		to_dev->node_props.io_links_count++;
416
+		list_add_tail(&props2->list, &to_dev->io_link_props);
406417 	}
407418 
408419 	return 0;
....@@ -491,7 +502,7 @@
491502 	num_nodes = crat_table->num_domains;
492503 	image_len = crat_table->length;
493504 
494
-	pr_info("Parsing CRAT table with %d nodes\n", num_nodes);
505
+	pr_debug("Parsing CRAT table with %d nodes\n", num_nodes);
495506 
496507 	for (node_id = 0; node_id < num_nodes; node_id++) {
497508 		top_dev = kfd_create_topology_device(device_list);
....@@ -641,13 +652,36 @@
641652 		pcache_info = polaris11_cache_info;
642653 		num_of_cache_types = ARRAY_SIZE(polaris11_cache_info);
643654 		break;
655
+	case CHIP_POLARIS12:
656
+		pcache_info = polaris12_cache_info;
657
+		num_of_cache_types = ARRAY_SIZE(polaris12_cache_info);
658
+		break;
659
+	case CHIP_VEGAM:
660
+		pcache_info = vegam_cache_info;
661
+		num_of_cache_types = ARRAY_SIZE(vegam_cache_info);
662
+		break;
644663 	case CHIP_VEGA10:
664
+	case CHIP_VEGA12:
665
+	case CHIP_VEGA20:
666
+	case CHIP_ARCTURUS:
645667 		pcache_info = vega10_cache_info;
646668 		num_of_cache_types = ARRAY_SIZE(vega10_cache_info);
647669 		break;
648670 	case CHIP_RAVEN:
649671 		pcache_info = raven_cache_info;
650672 		num_of_cache_types = ARRAY_SIZE(raven_cache_info);
673
+		break;
674
+	case CHIP_RENOIR:
675
+		pcache_info = renoir_cache_info;
676
+		num_of_cache_types = ARRAY_SIZE(renoir_cache_info);
677
+		break;
678
+	case CHIP_NAVI10:
679
+	case CHIP_NAVI12:
680
+	case CHIP_NAVI14:
681
+	case CHIP_SIENNA_CICHLID:
682
+	case CHIP_NAVY_FLOUNDER:
683
+		pcache_info = navi10_cache_info;
684
+		num_of_cache_types = ARRAY_SIZE(navi10_cache_info);
651685 		break;
652686 	default:
653687 		return -EINVAL;
....@@ -678,7 +712,7 @@
678712 						pcache_info,
679713 						cu_info,
680714 						mem_available,
681
-						cu_info->cu_bitmap[i][j],
715
+						cu_info->cu_bitmap[i % 4][j + i / 4],
682716 						ct,
683717 						cu_processor_id,
684718 						k);
....@@ -708,6 +742,22 @@
708742 	return 0;
709743 }
710744 
745
+static bool kfd_ignore_crat(void)
746
+{
747
+	bool ret;
748
+
749
+	if (ignore_crat)
750
+		return true;
751
+
752
+#ifndef KFD_SUPPORT_IOMMU_V2
753
+	ret = true;
754
+#else
755
+	ret = false;
756
+#endif
757
+
758
+	return ret;
759
+}
760
+
711761 /*
712762  * kfd_create_crat_image_acpi - Allocates memory for CRAT image and
713763  * copies CRAT from ACPI (if available).
....@@ -724,6 +774,7 @@
724774 	struct acpi_table_header *crat_table;
725775 	acpi_status status;
726776 	void *pcrat_image;
777
+	int rc = 0;
727778 
728779 	if (!crat_image)
729780 		return -EINVAL;
....@@ -733,7 +784,7 @@
733784 	/* Fetch the CRAT table from ACPI */
734785 	status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
735786 	if (status == AE_NOT_FOUND) {
736
-		pr_warn("CRAT table not found\n");
787
+		pr_info("CRAT table not found\n");
737788 		return -ENODATA;
738789 	} else if (ACPI_FAILURE(status)) {
739790 		const char *err = acpi_format_exception(status);
....@@ -742,31 +793,32 @@
742793 		return -EINVAL;
743794 	}
744795 
745
-	if (ignore_crat) {
796
+	if (kfd_ignore_crat()) {
746797 		pr_info("CRAT table disabled by module option\n");
747798 		return -ENODATA;
748799 	}
749800 
750
-	pcrat_image = kmalloc(crat_table->length, GFP_KERNEL);
751
-	if (!pcrat_image)
752
-		return -ENOMEM;
801
+	pcrat_image = kvmalloc(crat_table->length, GFP_KERNEL);
802
+	if (!pcrat_image) {
803
+		rc = -ENOMEM;
804
+		goto out;
805
+	}
753806 
754807 	memcpy(pcrat_image, crat_table, crat_table->length);
755
-
756808 	*crat_image = pcrat_image;
757809 	*size = crat_table->length;
758
-
759
-	return 0;
810
+out:
811
+	acpi_put_table(crat_table);
812
+	return rc;
760813 }
761814 
762815 /* Memory required to create Virtual CRAT.
763816  * Since there is no easy way to predict the amount of memory required, the
764
- * following amount are allocated for CPU and GPU Virtual CRAT. This is
817
+ * following amount is allocated for GPU Virtual CRAT. This is
765818  * expected to cover all known conditions. But to be safe additional check
766819  * is put in the code to ensure we don't overwrite.
767820  */
768
-#define VCRAT_SIZE_FOR_CPU	(2 * PAGE_SIZE)
769
-#define VCRAT_SIZE_FOR_GPU	(3 * PAGE_SIZE)
821
+#define VCRAT_SIZE_FOR_GPU	(4 * PAGE_SIZE)
770822 
771823 /* kfd_fill_cu_for_cpu - Fill in Compute info for the given CPU NUMA node
772824  *
....@@ -842,7 +894,7 @@
842894 	 */
843895 	pgdat = NODE_DATA(numa_node_id);
844896 	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
845
-		mem_in_bytes += pgdat->node_zones[zone_type].managed_pages;
897
+		mem_in_bytes += zone_managed_pages(&pgdat->node_zones[zone_type]);
846898 	mem_in_bytes <<= PAGE_SHIFT;
847899 
848900 	sub_type_hdr->length_low = lower_32_bits(mem_in_bytes);
....@@ -852,6 +904,7 @@
852904 	return 0;
853905 }
854906 
907
+#ifdef CONFIG_X86_64
855908 static int kfd_fill_iolink_info_for_cpu(int numa_node_id, int *avail_size,
856909 				uint32_t *num_entries,
857910 				struct crat_subtype_iolink *sub_type_hdr)
....@@ -894,6 +947,7 @@
894947 
895948 	return 0;
896949 }
950
+#endif
897951 
898952 /* kfd_create_vcrat_image_cpu - Create Virtual CRAT for CPU
899953  *
....@@ -909,10 +963,12 @@
909963 	struct crat_subtype_generic *sub_type_hdr;
910964 	int avail_size = *size;
911965 	int numa_node_id;
966
+#ifdef CONFIG_X86_64
912967 	uint32_t entries = 0;
968
+#endif
913969 	int ret = 0;
914970 
915
-	if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_CPU)
971
+	if (!pcrat_image)
916972 		return -EINVAL;
917973 
918974 	/* Fill in CRAT Header.
....@@ -936,6 +992,7 @@
936992 				CRAT_OEMID_LENGTH);
937993 		memcpy(crat_table->oem_table_id, acpi_table->oem_table_id,
938994 				CRAT_OEMTABLEID_LENGTH);
995
+		acpi_put_table(acpi_table);
939996 	}
940997 	crat_table->total_entries = 0;
941998 	crat_table->num_domains = 0;
....@@ -971,16 +1028,23 @@
9711028 			sub_type_hdr->length);
9721029 
9731030 		/* Fill in Subtype: IO Link */
1031
+#ifdef CONFIG_X86_64
9741032 		ret = kfd_fill_iolink_info_for_cpu(numa_node_id, &avail_size,
9751033 				&entries,
9761034 				(struct crat_subtype_iolink *)sub_type_hdr);
9771035 		if (ret < 0)
9781036 			return ret;
979
-		crat_table->length += (sub_type_hdr->length * entries);
980
-		crat_table->total_entries += entries;
9811037 
982
-		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
983
-				sub_type_hdr->length * entries);
1038
+		if (entries) {
1039
+			crat_table->length += (sub_type_hdr->length * entries);
1040
+			crat_table->total_entries += entries;
1041
+
1042
+			sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1043
+					sub_type_hdr->length * entries);
1044
+		}
1045
+#else
1046
+		pr_info("IO link not available for non x86 platforms\n");
1047
+#endif
9841048 
9851049 		crat_table->num_domains++;
9861050 	}
....@@ -1037,7 +1101,7 @@
10371101  *
10381102  *	Return 0 if successful else return -ve value
10391103  */
1040
-static int kfd_fill_gpu_direct_io_link(int *avail_size,
1104
+static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
10411105 			struct kfd_dev *kdev,
10421106 			struct crat_subtype_iolink *sub_type_hdr,
10431107 			uint32_t proximity_domain)
....@@ -1052,6 +1116,8 @@
10521116 	sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
10531117 	sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
10541118 	sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
1119
+	if (kfd_dev_is_large_bar(kdev))
1120
+		sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
10551121 
10561122 	/* Fill in IOLINK subtype.
10571123 	 * TODO: Fill-in other fields of iolink subtype
....@@ -1069,6 +1135,32 @@
10691135 	return 0;
10701136 }
10711137 
1138
+static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
1139
+			struct kfd_dev *kdev,
1140
+			struct kfd_dev *peer_kdev,
1141
+			struct crat_subtype_iolink *sub_type_hdr,
1142
+			uint32_t proximity_domain_from,
1143
+			uint32_t proximity_domain_to)
1144
+{
1145
+	*avail_size -= sizeof(struct crat_subtype_iolink);
1146
+	if (*avail_size < 0)
1147
+		return -ENOMEM;
1148
+
1149
+	memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
1150
+
1151
+	sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
1152
+	sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
1153
+	sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED |
1154
+			       CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
1155
+
1156
+	sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
1157
+	sub_type_hdr->proximity_domain_from = proximity_domain_from;
1158
+	sub_type_hdr->proximity_domain_to = proximity_domain_to;
1159
+	sub_type_hdr->num_hops_xgmi =
1160
+		amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd, peer_kdev->kgd);
1161
+	return 0;
1162
+}
1163
+
10721164 /* kfd_create_vcrat_image_gpu - Create Virtual CRAT for CPU
10731165  *
10741166  *	@pcrat_image: Fill in VCRAT for GPU
....@@ -1081,14 +1173,16 @@
10811173 {
10821174 	struct crat_header *crat_table = (struct crat_header *)pcrat_image;
10831175 	struct crat_subtype_generic *sub_type_hdr;
1176
+	struct kfd_local_mem_info local_mem_info;
1177
+	struct kfd_topology_device *peer_dev;
10841178 	struct crat_subtype_computeunit *cu;
10851179 	struct kfd_cu_info cu_info;
10861180 	int avail_size = *size;
10871181 	uint32_t total_num_of_cu;
10881182 	int num_of_cache_entries = 0;
10891183 	int cache_mem_filled = 0;
1184
+	uint32_t nid = 0;
10901185 	int ret = 0;
1091
-	struct kfd_local_mem_info local_mem_info;
10921186 
10931187 	if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_GPU)
10941188 		return -EINVAL;
....@@ -1128,7 +1222,7 @@
11281222 	cu->flags |= CRAT_CU_FLAGS_GPU_PRESENT;
11291223 	cu->proximity_domain = proximity_domain;
11301224 
1131
-	kdev->kfd2kgd->get_cu_info(kdev->kgd, &cu_info);
1225
+	amdgpu_amdkfd_get_cu_info(kdev->kgd, &cu_info);
11321226 	cu->num_simd_per_cu = cu_info.simd_per_cu;
11331227 	cu->num_simd_cores = cu_info.simd_per_cu * cu_info.cu_active_number;
11341228 	cu->max_waves_simd = cu_info.max_waves_per_simd;
....@@ -1159,7 +1253,7 @@
11591253 	 * report the total FB size (public+private) as a single
11601254 	 * private heap.
11611255 	 */
1162
-	kdev->kfd2kgd->get_local_mem_info(kdev->kgd, &local_mem_info);
1256
+	amdgpu_amdkfd_get_local_mem_info(kdev->kgd, &local_mem_info);
11631257 	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
11641258 			sub_type_hdr->length);
11651259 
....@@ -1212,7 +1306,7 @@
12121306 	 */
12131307 	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
12141308 		cache_mem_filled);
1215
-	ret = kfd_fill_gpu_direct_io_link(&avail_size, kdev,
1309
+	ret = kfd_fill_gpu_direct_io_link_to_cpu(&avail_size, kdev,
12161310 		(struct crat_subtype_iolink *)sub_type_hdr, proximity_domain);
12171311 
12181312 	if (ret < 0)
....@@ -1221,6 +1315,35 @@
12211315 	crat_table->length += sub_type_hdr->length;
12221316 	crat_table->total_entries++;
12231317 
1318
+
1319
+	/* Fill in Subtype: IO_LINKS
1320
+	 * Direct links from GPU to other GPUs through xGMI.
1321
+	 * We will loop GPUs that already be processed (with lower value
1322
+	 * of proximity_domain), add the link for the GPUs with same
1323
+	 * hive id (from this GPU to other GPU) . The reversed iolink
1324
+	 * (from other GPU to this GPU) will be added
1325
+	 * in kfd_parse_subtype_iolink.
1326
+	 */
1327
+	if (kdev->hive_id) {
1328
+		for (nid = 0; nid < proximity_domain; ++nid) {
1329
+			peer_dev = kfd_topology_device_by_proximity_domain(nid);
1330
+			if (!peer_dev->gpu)
1331
+				continue;
1332
+			if (peer_dev->gpu->hive_id != kdev->hive_id)
1333
+				continue;
1334
+			sub_type_hdr = (typeof(sub_type_hdr))(
1335
+				(char *)sub_type_hdr +
1336
+				sizeof(struct crat_subtype_iolink));
1337
+			ret = kfd_fill_gpu_xgmi_link_to_gpu(
1338
+				&avail_size, kdev, peer_dev->gpu,
1339
+				(struct crat_subtype_iolink *)sub_type_hdr,
1340
+				proximity_domain, nid);
1341
+			if (ret < 0)
1342
+				return ret;
1343
+			crat_table->length += sub_type_hdr->length;
1344
+			crat_table->total_entries++;
1345
+		}
1346
+	}
12241347 	*size = crat_table->length;
12251348 	pr_info("Virtual CRAT table created for GPU\n");
12261349 
....@@ -1249,30 +1372,37 @@
12491372 				  uint32_t proximity_domain)
12501373 {
12511374 	void *pcrat_image = NULL;
1252
-	int ret = 0;
1375
+	int ret = 0, num_nodes;
1376
+	size_t dyn_size;
12531377 
12541378 	if (!crat_image)
12551379 		return -EINVAL;
12561380 
12571381 	*crat_image = NULL;
12581382 
1259
-	/* Allocate one VCRAT_SIZE_FOR_CPU for CPU virtual CRAT image and
1260
-	 * VCRAT_SIZE_FOR_GPU for GPU virtual CRAT image. This should cover
1261
-	 * all the current conditions. A check is put not to overwrite beyond
1262
-	 * allocated size
1383
+	/* Allocate the CPU Virtual CRAT size based on the number of online
1384
+	 * nodes. Allocate VCRAT_SIZE_FOR_GPU for GPU virtual CRAT image.
1385
+	 * This should cover all the current conditions. A check is put not
1386
+	 * to overwrite beyond allocated size for GPUs
12631387 	 */
12641388 	switch (flags) {
12651389 	case COMPUTE_UNIT_CPU:
1266
-		pcrat_image = kmalloc(VCRAT_SIZE_FOR_CPU, GFP_KERNEL);
1390
+		num_nodes = num_online_nodes();
1391
+		dyn_size = sizeof(struct crat_header) +
1392
+			num_nodes * (sizeof(struct crat_subtype_computeunit) +
1393
+			sizeof(struct crat_subtype_memory) +
1394
+			(num_nodes - 1) * sizeof(struct crat_subtype_iolink));
1395
+		pcrat_image = kvmalloc(dyn_size, GFP_KERNEL);
12671396 		if (!pcrat_image)
12681397 			return -ENOMEM;
1269
-		*size = VCRAT_SIZE_FOR_CPU;
1398
+		*size = dyn_size;
1399
+		pr_debug("CRAT size is %ld", dyn_size);
12701400 		ret = kfd_create_vcrat_image_cpu(pcrat_image, size);
12711401 		break;
12721402 	case COMPUTE_UNIT_GPU:
12731403 		if (!kdev)
12741404 			return -EINVAL;
1275
-		pcrat_image = kmalloc(VCRAT_SIZE_FOR_GPU, GFP_KERNEL);
1405
+		pcrat_image = kvmalloc(VCRAT_SIZE_FOR_GPU, GFP_KERNEL);
12761406 		if (!pcrat_image)
12771407 			return -ENOMEM;
12781408 		*size = VCRAT_SIZE_FOR_GPU;
....@@ -1291,7 +1421,7 @@
12911421 	if (!ret)
12921422 		*crat_image = pcrat_image;
12931423 	else
1294
-		kfree(pcrat_image);
1424
+		kvfree(pcrat_image);
12951425 
12961426 	return ret;
12971427 }
....@@ -1304,5 +1434,5 @@
13041434  */
13051435 void kfd_destroy_crat_image(void *crat_image)
13061436 {
1307
-	kfree(crat_image);
1437
+	kvfree(crat_image);
13081438 }