forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
....@@ -20,13 +20,17 @@
2020  * OTHER DEALINGS IN THE SOFTWARE.
2121  *
2222  */
23
+
2324 #include <linux/firmware.h>
24
-#include <drm/drmP.h>
25
+#include <linux/module.h>
26
+#include <linux/pci.h>
27
+
2528 #include <drm/drm_cache.h>
2629 #include "amdgpu.h"
2730 #include "gmc_v8_0.h"
2831 #include "amdgpu_ucode.h"
2932 #include "amdgpu_amdkfd.h"
33
+#include "amdgpu_gem.h"
3034 
3135 #include "gmc/gmc_8_1_d.h"
3236 #include "gmc/gmc_8_1_sh_mask.h"
....@@ -288,7 +292,7 @@
288292  *
289293  * @adev: amdgpu_device pointer
290294  *
291
- * Load the GDDR MC ucode into the hw (CIK).
295
+ * Load the GDDR MC ucode into the hw (VI).
292296  * Returns 0 on success, error on failure.
293297  */
294298 static int gmc_v8_0_tonga_mc_load_microcode(struct amdgpu_device *adev)
....@@ -432,8 +436,8 @@
432436 		base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
433437 	base <<= 24;
434438 
435
-	amdgpu_device_vram_location(adev, &adev->gmc, base);
436
-	amdgpu_device_gart_location(adev, mc);
439
+	amdgpu_gmc_vram_location(adev, mc, base);
440
+	amdgpu_gmc_gart_location(adev, mc);
437441 }
438442 
439443 /**
....@@ -442,7 +446,7 @@
442446  * @adev: amdgpu_device pointer
443447  *
444448  * Set the location of vram, gart, and AGP in the GPU's
445
- * physical address space (CIK).
449
+ * physical address space (VI).
446450  */
447451 static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
448452 {
....@@ -514,7 +518,7 @@
514518  * @adev: amdgpu_device pointer
515519  *
516520  * Look up the amount of vram, vram width, and decide how to place
517
- * vram and gart within the GPU's physical address space (CIK).
521
+ * vram and gart within the GPU's physical address space (VI).
518522  * Returns 0 for success.
519523  */
520524 static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
....@@ -623,6 +627,39 @@
623627 	return 0;
624628 }
625629 
630
+/**
631
+ * gmc_v8_0_flush_gpu_tlb_pasid - tlb flush via pasid
632
+ *
633
+ * @adev: amdgpu_device pointer
634
+ * @pasid: pasid to be flush
635
+ *
636
+ * Flush the TLB for the requested pasid.
637
+ */
638
+static int gmc_v8_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
639
+					uint16_t pasid, uint32_t flush_type,
640
+					bool all_hub)
641
+{
642
+	int vmid;
643
+	unsigned int tmp;
644
+
645
+	if (amdgpu_in_reset(adev))
646
+		return -EIO;
647
+
648
+	for (vmid = 1; vmid < 16; vmid++) {
649
+
650
+		tmp = RREG32(mmATC_VMID0_PASID_MAPPING + vmid);
651
+		if ((tmp & ATC_VMID0_PASID_MAPPING__VALID_MASK) &&
652
+			(tmp & ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid) {
653
+			WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
654
+			RREG32(mmVM_INVALIDATE_RESPONSE);
655
+			break;
656
+		}
657
+	}
658
+
659
+	return 0;
660
+
661
+}
662
+
626663 /*
627664  * GART
628665  * VMID 0 is the physical GPU addresses as used by the kernel.
....@@ -636,10 +673,10 @@
636673  * @adev: amdgpu_device pointer
637674  * @vmid: vm instance to flush
638675  *
639
- * Flush the TLB for the requested page table (CIK).
676
+ * Flush the TLB for the requested page table (VI).
640677  */
641
-static void gmc_v8_0_flush_gpu_tlb(struct amdgpu_device *adev,
642
-					uint32_t vmid)
678
+static void gmc_v8_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
679
+					uint32_t vmhub, uint32_t flush_type)
643680 {
644681 	/* bits 0-15 are the VM contexts0-15 */
645682 	WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
....@@ -668,72 +705,40 @@
668705 	amdgpu_ring_emit_wreg(ring, mmIH_VMID_0_LUT + vmid, pasid);
669706 }
670707 
671
-/**
672
- * gmc_v8_0_set_pte_pde - update the page tables using MMIO
708
+/*
709
+ * PTE format on VI:
710
+ * 63:40 reserved
711
+ * 39:12 4k physical page base address
712
+ * 11:7 fragment
713
+ * 6 write
714
+ * 5 read
715
+ * 4 exe
716
+ * 3 reserved
717
+ * 2 snooped
718
+ * 1 system
719
+ * 0 valid
673720  *
674
- * @adev: amdgpu_device pointer
675
- * @cpu_pt_addr: cpu address of the page table
676
- * @gpu_page_idx: entry in the page table to update
677
- * @addr: dst addr to write into pte/pde
678
- * @flags: access flags
679
- *
680
- * Update the page tables using the CPU.
721
+ * PDE format on VI:
722
+ * 63:59 block fragment size
723
+ * 58:40 reserved
724
+ * 39:1 physical base address of PTE
725
+ * bits 5:1 must be 0.
726
+ * 0 valid
681727  */
682
-static int gmc_v8_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
683
-				uint32_t gpu_page_idx, uint64_t addr,
684
-				uint64_t flags)
685
-{
686
-	void __iomem *ptr = (void *)cpu_pt_addr;
687
-	uint64_t value;
688
-
689
-	/*
690
-	 * PTE format on VI:
691
-	 * 63:40 reserved
692
-	 * 39:12 4k physical page base address
693
-	 * 11:7 fragment
694
-	 * 6 write
695
-	 * 5 read
696
-	 * 4 exe
697
-	 * 3 reserved
698
-	 * 2 snooped
699
-	 * 1 system
700
-	 * 0 valid
701
-	 *
702
-	 * PDE format on VI:
703
-	 * 63:59 block fragment size
704
-	 * 58:40 reserved
705
-	 * 39:1 physical base address of PTE
706
-	 * bits 5:1 must be 0.
707
-	 * 0 valid
708
-	 */
709
-	value = addr & 0x000000FFFFFFF000ULL;
710
-	value |= flags;
711
-	writeq(value, ptr + (gpu_page_idx * 8));
712
-
713
-	return 0;
714
-}
715
-
716
-static uint64_t gmc_v8_0_get_vm_pte_flags(struct amdgpu_device *adev,
717
-					  uint32_t flags)
718
-{
719
-	uint64_t pte_flag = 0;
720
-
721
-	if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
722
-		pte_flag |= AMDGPU_PTE_EXECUTABLE;
723
-	if (flags & AMDGPU_VM_PAGE_READABLE)
724
-		pte_flag |= AMDGPU_PTE_READABLE;
725
-	if (flags & AMDGPU_VM_PAGE_WRITEABLE)
726
-		pte_flag |= AMDGPU_PTE_WRITEABLE;
727
-	if (flags & AMDGPU_VM_PAGE_PRT)
728
-		pte_flag |= AMDGPU_PTE_PRT;
729
-
730
-	return pte_flag;
731
-}
732728 
733729 static void gmc_v8_0_get_vm_pde(struct amdgpu_device *adev, int level,
734730 				uint64_t *addr, uint64_t *flags)
735731 {
736732 	BUG_ON(*addr & 0xFFFFFF0000000FFFULL);
733
+}
734
+
735
+static void gmc_v8_0_get_vm_pte(struct amdgpu_device *adev,
736
+				struct amdgpu_bo_va_mapping *mapping,
737
+				uint64_t *flags)
738
+{
739
+	*flags &= ~AMDGPU_PTE_EXECUTABLE;
740
+	*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
741
+	*flags &= ~AMDGPU_PTE_PRT;
737742 }
738743 
739744 /**
....@@ -830,21 +835,25 @@
830835  * This sets up the TLBs, programs the page tables for VMID0,
831836  * sets up the hw for VMIDs 1-15 which are allocated on
832837  * demand, and sets up the global locations for the LDS, GDS,
833
- * and GPUVM for FSA64 clients (CIK).
838
+ * and GPUVM for FSA64 clients (VI).
834839  * Returns 0 for success, errors for failure.
835840  */
836841 static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
837842 {
843
+	uint64_t table_addr;
838844 	int r, i;
839845 	u32 tmp, field;
840846 
841
-	if (adev->gart.robj == NULL) {
847
+	if (adev->gart.bo == NULL) {
842848 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
843849 		return -EINVAL;
844850 	}
845851 	r = amdgpu_gart_table_vram_pin(adev);
846852 	if (r)
847853 		return r;
854
+
855
+	table_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
856
+
848857 	/* Setup TLB control */
849858 	tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
850859 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
....@@ -892,7 +901,7 @@
892901 	/* setup context0 */
893902 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12);
894903 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12);
895
-	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
904
+	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, table_addr >> 12);
896905 	WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
897906 			(u32)(adev->dummy_page_addr >> 12));
898907 	WREG32(mmVM_CONTEXT0_CNTL2, 0);
....@@ -916,10 +925,10 @@
916925 	for (i = 1; i < 16; i++) {
917926 		if (i < 8)
918927 			WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
919
-			       adev->gart.table_addr >> 12);
928
+			       table_addr >> 12);
920929 		else
921930 			WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
922
-			       adev->gart.table_addr >> 12);
931
+			       table_addr >> 12);
923932 	}
924933 
925934 	/* enable context1-15 */
....@@ -944,10 +953,10 @@
944953 	else
945954 		gmc_v8_0_set_fault_enable_default(adev, true);
946955 
947
-	gmc_v8_0_flush_gpu_tlb(adev, 0);
956
+	gmc_v8_0_flush_gpu_tlb(adev, 0, 0, 0);
948957 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
949958 		 (unsigned)(adev->gmc.gart_size >> 20),
950
-		 (unsigned long long)adev->gart.table_addr);
959
+		 (unsigned long long)table_addr);
951960 	adev->gart.ready = true;
952961 	return 0;
953962 }
....@@ -956,7 +965,7 @@
956965 {
957966 	int r;
958967 
959
-	if (adev->gart.robj) {
968
+	if (adev->gart.bo) {
960969 		WARN(1, "R600 PCIE GART already initialized\n");
961970 		return 0;
962971 	}
....@@ -974,7 +983,7 @@
974983  *
975984  * @adev: amdgpu_device pointer
976985  *
977
- * This disables all VM page table (CIK).
986
+ * This disables all VM page table (VI).
978987  */
979988 static void gmc_v8_0_gart_disable(struct amdgpu_device *adev)
980989 {
....@@ -1003,8 +1012,9 @@
10031012  * @adev: amdgpu_device pointer
10041013  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
10051014  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
1015
+ * @mc_client: VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT register value
10061016  *
1007
- * Print human readable fault information (CIK).
1017
+ * Print human readable fault information (VI).
10081018  */
10091019 static void gmc_v8_0_vm_decode_fault(struct amdgpu_device *adev, u32 status,
10101020 				     u32 addr, u32 mc_client, unsigned pasid)
....@@ -1084,16 +1094,14 @@
10841094 	unsigned size;
10851095 
10861096 	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1087
-		size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */
1097
+		size = AMDGPU_VBIOS_VGA_ALLOCATION;
10881098 	} else {
10891099 		u32 viewport = RREG32(mmVIEWPORT_SIZE);
10901100 		size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
10911101 			REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) *
10921102 			4);
10931103 	}
1094
-	/* return 0 if the pre-OS buffer uses up most of vram */
1095
-	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))
1096
-		return 0;
1104
+
10971105 	return size;
10981106 }
10991107 
....@@ -1102,8 +1110,9 @@
11021110 static int gmc_v8_0_sw_init(void *handle)
11031111 {
11041112 	int r;
1105
-	int dma_bits;
11061113 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1114
+
1115
+	adev->num_vmhubs = 1;
11071116 
11081117 	if (adev->flags & AMD_IS_APU) {
11091118 		adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
....@@ -1119,11 +1128,11 @@
11191128 		adev->gmc.vram_type = gmc_v8_0_convert_vram_type(tmp);
11201129 	}
11211130 
1122
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_PAGE_INV_FAULT, &adev->gmc.vm_fault);
1131
+	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_PAGE_INV_FAULT, &adev->gmc.vm_fault);
11231132 	if (r)
11241133 		return r;
11251134 
1126
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_MEM_PROT_FAULT, &adev->gmc.vm_fault);
1135
+	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_MEM_PROT_FAULT, &adev->gmc.vm_fault);
11271136 	if (r)
11281137 		return r;
11291138 
....@@ -1139,25 +1148,12 @@
11391148 	 */
11401149 	adev->gmc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
11411150 
1142
-	/* set DMA mask + need_dma32 flags.
1143
-	 * PCIE - can handle 40-bits.
1144
-	 * IGP - can handle 40-bits
1145
-	 * PCI - dma32 for legacy pci gart, 40 bits on newer asics
1146
-	 */
1147
-	adev->need_dma32 = false;
1148
-	dma_bits = adev->need_dma32 ? 32 : 40;
1149
-	r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
1151
+	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(40));
11501152 	if (r) {
1151
-		adev->need_dma32 = true;
1152
-		dma_bits = 32;
1153
-		pr_warn("amdgpu: No suitable DMA available\n");
1153
+		pr_warn("No suitable DMA available\n");
1154
+		return r;
11541155 	}
1155
-	r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
1156
-	if (r) {
1157
-		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
1158
-		pr_warn("amdgpu: No coherent DMA available\n");
1159
-	}
1160
-	adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);
1156
+	adev->need_swiotlb = drm_need_swiotlb(40);
11611157 
11621158 	r = gmc_v8_0_init_microcode(adev);
11631159 	if (r) {
....@@ -1169,7 +1165,7 @@
11691165 	if (r)
11701166 		return r;
11711167 
1172
-	adev->gmc.stolen_size = gmc_v8_0_get_vbios_fb_size(adev);
1168
+	amdgpu_gmc_get_vbios_allocations(adev);
11731169 
11741170 	/* Memory manager */
11751171 	r = amdgpu_bo_init(adev);
....@@ -1186,7 +1182,7 @@
11861182 	 * amdgpu graphics/compute will use VMIDs 1-7
11871183 	 * amdkfd will use VMIDs 8-15
11881184 	 */
1189
-	adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
1185
+	adev->vm_manager.first_kfd_vmid = 8;
11901186 	amdgpu_vm_manager_init(adev);
11911187 
11921188 	/* base offset of vram pages */
....@@ -1473,8 +1469,9 @@
14731469 		gmc_v8_0_set_fault_enable_default(adev, false);
14741470 
14751471 	if (printk_ratelimit()) {
1476
-		struct amdgpu_task_info task_info = { 0 };
1472
+		struct amdgpu_task_info task_info;
14771473 
1474
+		memset(&task_info, 0, sizeof(struct amdgpu_task_info));
14781475 		amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
14791476 
14801477 		dev_err(adev->dev, "GPU fault detected: %d 0x%08x for process %s pid %d thread %s pid %d\n",
....@@ -1742,12 +1739,13 @@
17421739 
17431740 static const struct amdgpu_gmc_funcs gmc_v8_0_gmc_funcs = {
17441741 	.flush_gpu_tlb = gmc_v8_0_flush_gpu_tlb,
1742
+	.flush_gpu_tlb_pasid = gmc_v8_0_flush_gpu_tlb_pasid,
17451743 	.emit_flush_gpu_tlb = gmc_v8_0_emit_flush_gpu_tlb,
17461744 	.emit_pasid_mapping = gmc_v8_0_emit_pasid_mapping,
1747
-	.set_pte_pde = gmc_v8_0_set_pte_pde,
17481745 	.set_prt = gmc_v8_0_set_prt,
1749
-	.get_vm_pte_flags = gmc_v8_0_get_vm_pte_flags,
1750
-	.get_vm_pde = gmc_v8_0_get_vm_pde
1746
+	.get_vm_pde = gmc_v8_0_get_vm_pde,
1747
+	.get_vm_pte = gmc_v8_0_get_vm_pte,
1748
+	.get_vbios_fb_size = gmc_v8_0_get_vbios_fb_size,
17511749 };
17521750 
17531751 static const struct amdgpu_irq_src_funcs gmc_v8_0_irq_funcs = {
....@@ -1757,8 +1755,7 @@
17571755 
17581756 static void gmc_v8_0_set_gmc_funcs(struct amdgpu_device *adev)
17591757 {
1760
-	if (adev->gmc.gmc_funcs == NULL)
1761
-		adev->gmc.gmc_funcs = &gmc_v8_0_gmc_funcs;
1758
+	adev->gmc.gmc_funcs = &gmc_v8_0_gmc_funcs;
17621759 }
17631760 
17641761 static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev)