forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-01-04 1543e317f1da31b75942316931e8f491a8920811 
 kernel/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
....@@ -20,8 +20,11 @@
2020  * OTHER DEALINGS IN THE SOFTWARE.
2121  *
2222  */
23
+
2324 #include <linux/firmware.h>
24
-#include <drm/drmP.h>
25
+#include <linux/module.h>
26
+#include <linux/pci.h>
27
+
2528 #include <drm/drm_cache.h>
2629 #include "amdgpu.h"
2730 #include "cikd.h"
....@@ -29,6 +32,7 @@
2932 #include "gmc_v7_0.h"
3033 #include "amdgpu_ucode.h"
3134 #include "amdgpu_amdkfd.h"
35
+#include "amdgpu_gem.h"
3236 
3337 #include "bif/bif_4_1_d.h"
3438 #include "bif/bif_4_1_sh_mask.h"
....@@ -241,8 +245,8 @@
241245 	u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
242246 	base <<= 24;
243247 
244
-	amdgpu_device_vram_location(adev, &adev->gmc, base);
245
-	amdgpu_device_gart_location(adev, mc);
248
+	amdgpu_gmc_vram_location(adev, mc, base);
249
+	amdgpu_gmc_gart_location(adev, mc);
246250 }
247251 
248252 /**
....@@ -377,7 +381,8 @@
377381 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
378382 
379383 #ifdef CONFIG_X86_64
380
-	if (adev->flags & AMD_IS_APU) {
384
+	if (adev->flags & AMD_IS_APU &&
385
+	    adev->gmc.real_vram_size > adev->gmc.aper_size) {
381386 		adev->gmc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
382387 		adev->gmc.aper_size = adev->gmc.real_vram_size;
383388 	}
....@@ -414,6 +419,38 @@
414419 	return 0;
415420 }
416421 
422
+/**
423
+ * gmc_v7_0_flush_gpu_tlb_pasid - tlb flush via pasid
424
+ *
425
+ * @adev: amdgpu_device pointer
426
+ * @pasid: pasid to be flush
427
+ *
428
+ * Flush the TLB for the requested pasid.
429
+ */
430
+static int gmc_v7_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
431
+					uint16_t pasid, uint32_t flush_type,
432
+					bool all_hub)
433
+{
434
+	int vmid;
435
+	unsigned int tmp;
436
+
437
+	if (amdgpu_in_reset(adev))
438
+		return -EIO;
439
+
440
+	for (vmid = 1; vmid < 16; vmid++) {
441
+
442
+		tmp = RREG32(mmATC_VMID0_PASID_MAPPING + vmid);
443
+		if ((tmp & ATC_VMID0_PASID_MAPPING__VALID_MASK) &&
444
+			(tmp & ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid) {
445
+			WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
446
+			RREG32(mmVM_INVALIDATE_RESPONSE);
447
+			break;
448
+		}
449
+	}
450
+
451
+	return 0;
452
+}
453
+
417454 /*
418455  * GART
419456  * VMID 0 is the physical GPU addresses as used by the kernel.
....@@ -429,7 +466,8 @@
429466  *
430467  * Flush the TLB for the requested page table (CIK).
431468  */
432
-static void gmc_v7_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid)
469
+static void gmc_v7_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
470
+					uint32_t vmhub, uint32_t flush_type)
433471 {
434472 	/* bits 0-15 are the VM contexts0-15 */
435473 	WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
....@@ -458,50 +496,18 @@
458496 	amdgpu_ring_emit_wreg(ring, mmIH_VMID_0_LUT + vmid, pasid);
459497 }
460498 
461
-/**
462
- * gmc_v7_0_set_pte_pde - update the page tables using MMIO
463
- *
464
- * @adev: amdgpu_device pointer
465
- * @cpu_pt_addr: cpu address of the page table
466
- * @gpu_page_idx: entry in the page table to update
467
- * @addr: dst addr to write into pte/pde
468
- * @flags: access flags
469
- *
470
- * Update the page tables using the CPU.
471
- */
472
-static int gmc_v7_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
473
-				 uint32_t gpu_page_idx, uint64_t addr,
474
-				 uint64_t flags)
475
-{
476
-	void __iomem *ptr = (void *)cpu_pt_addr;
477
-	uint64_t value;
478
-
479
-	value = addr & 0xFFFFFFFFFFFFF000ULL;
480
-	value |= flags;
481
-	writeq(value, ptr + (gpu_page_idx * 8));
482
-
483
-	return 0;
484
-}
485
-
486
-static uint64_t gmc_v7_0_get_vm_pte_flags(struct amdgpu_device *adev,
487
-					  uint32_t flags)
488
-{
489
-	uint64_t pte_flag = 0;
490
-
491
-	if (flags & AMDGPU_VM_PAGE_READABLE)
492
-		pte_flag |= AMDGPU_PTE_READABLE;
493
-	if (flags & AMDGPU_VM_PAGE_WRITEABLE)
494
-		pte_flag |= AMDGPU_PTE_WRITEABLE;
495
-	if (flags & AMDGPU_VM_PAGE_PRT)
496
-		pte_flag |= AMDGPU_PTE_PRT;
497
-
498
-	return pte_flag;
499
-}
500
-
501499 static void gmc_v7_0_get_vm_pde(struct amdgpu_device *adev, int level,
502500 				uint64_t *addr, uint64_t *flags)
503501 {
504502 	BUG_ON(*addr & 0xFFFFFF0000000FFFULL);
503
+}
504
+
505
+static void gmc_v7_0_get_vm_pte(struct amdgpu_device *adev,
506
+				struct amdgpu_bo_va_mapping *mapping,
507
+				uint64_t *flags)
508
+{
509
+	*flags &= ~AMDGPU_PTE_EXECUTABLE;
510
+	*flags &= ~AMDGPU_PTE_PRT;
505511 }
506512 
507513 /**
....@@ -601,16 +607,20 @@
601607  */
602608 static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)
603609 {
610
+	uint64_t table_addr;
604611 	int r, i;
605612 	u32 tmp, field;
606613 
607
-	if (adev->gart.robj == NULL) {
614
+	if (adev->gart.bo == NULL) {
608615 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
609616 		return -EINVAL;
610617 	}
611618 	r = amdgpu_gart_table_vram_pin(adev);
612619 	if (r)
613620 		return r;
621
+
622
+	table_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
623
+
614624 	/* Setup TLB control */
615625 	tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
616626 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
....@@ -642,7 +652,7 @@
642652 	/* setup context0 */
643653 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12);
644654 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12);
645
-	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
655
+	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, table_addr >> 12);
646656 	WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
647657 			(u32)(adev->dummy_page_addr >> 12));
648658 	WREG32(mmVM_CONTEXT0_CNTL2, 0);
....@@ -666,10 +676,10 @@
666676 	for (i = 1; i < 16; i++) {
667677 		if (i < 8)
668678 			WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
669
-			       adev->gart.table_addr >> 12);
679
+			       table_addr >> 12);
670680 		else
671681 			WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
672
-			       adev->gart.table_addr >> 12);
682
+			       table_addr >> 12);
673683 	}
674684 
675685 	/* enable context1-15 */
....@@ -693,10 +703,10 @@
693703 		WREG32(mmCHUB_CONTROL, tmp);
694704 	}
695705 
696
-	gmc_v7_0_flush_gpu_tlb(adev, 0);
706
+	gmc_v7_0_flush_gpu_tlb(adev, 0, 0, 0);
697707 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
698708 		 (unsigned)(adev->gmc.gart_size >> 20),
699
-		 (unsigned long long)adev->gart.table_addr);
709
+		 (unsigned long long)table_addr);
700710 	adev->gart.ready = true;
701711 	return 0;
702712 }
....@@ -705,7 +715,7 @@
705715 {
706716 	int r;
707717 
708
-	if (adev->gart.robj) {
718
+	if (adev->gart.bo) {
709719 		WARN(1, "R600 PCIE GART already initialized\n");
710720 		return 0;
711721 	}
....@@ -752,6 +762,7 @@
752762  * @adev: amdgpu_device pointer
753763  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
754764  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
765
+ * @mc_client: VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT register value
755766  *
756767  * Print human readable fault information (CIK).
757768  */
....@@ -959,24 +970,23 @@
959970 	unsigned size;
960971 
961972 	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
962
-		size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */
973
+		size = AMDGPU_VBIOS_VGA_ALLOCATION;
963974 	} else {
964975 		u32 viewport = RREG32(mmVIEWPORT_SIZE);
965976 		size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
966977 			REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) *
967978 			4);
968979 	}
969
-	/* return 0 if the pre-OS buffer uses up most of vram */
970
-	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))
971
-		return 0;
980
+
972981 	return size;
973982 }
974983 
975984 static int gmc_v7_0_sw_init(void *handle)
976985 {
977986 	int r;
978
-	int dma_bits;
979987 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
988
+
989
+	adev->num_vmhubs = 1;
980990 
981991 	if (adev->flags & AMD_IS_APU) {
982992 		adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
....@@ -986,11 +996,11 @@
986996 		adev->gmc.vram_type = gmc_v7_0_convert_vram_type(tmp);
987997 	}
988998 
989
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_PAGE_INV_FAULT, &adev->gmc.vm_fault);
999
+	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_PAGE_INV_FAULT, &adev->gmc.vm_fault);
9901000 	if (r)
9911001 		return r;
9921002 
993
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_MEM_PROT_FAULT, &adev->gmc.vm_fault);
1003
+	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_MEM_PROT_FAULT, &adev->gmc.vm_fault);
9941004 	if (r)
9951005 		return r;
9961006 
....@@ -1006,25 +1016,12 @@
10061016 	 */
10071017 	adev->gmc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
10081018 
1009
-	/* set DMA mask + need_dma32 flags.
1010
-	 * PCIE - can handle 40-bits.
1011
-	 * IGP - can handle 40-bits
1012
-	 * PCI - dma32 for legacy pci gart, 40 bits on newer asics
1013
-	 */
1014
-	adev->need_dma32 = false;
1015
-	dma_bits = adev->need_dma32 ? 32 : 40;
1016
-	r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
1019
+	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(40));
10171020 	if (r) {
1018
-		adev->need_dma32 = true;
1019
-		dma_bits = 32;
1020
-		pr_warn("amdgpu: No suitable DMA available\n");
1021
+		pr_warn("No suitable DMA available\n");
1022
+		return r;
10211023 	}
1022
-	r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
1023
-	if (r) {
1024
-		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
1025
-		pr_warn("amdgpu: No coherent DMA available\n");
1026
-	}
1027
-	adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);
1024
+	adev->need_swiotlb = drm_need_swiotlb(40);
10281025 
10291026 	r = gmc_v7_0_init_microcode(adev);
10301027 	if (r) {
....@@ -1036,7 +1033,7 @@
10361033 	if (r)
10371034 		return r;
10381035 
1039
-	adev->gmc.stolen_size = gmc_v7_0_get_vbios_fb_size(adev);
1036
+	amdgpu_gmc_get_vbios_allocations(adev);
10401037 
10411038 	/* Memory manager */
10421039 	r = amdgpu_bo_init(adev);
....@@ -1053,7 +1050,7 @@
10531050 	 * amdgpu graphics/compute will use VMIDs 1-7
10541051 	 * amdkfd will use VMIDs 8-15
10551052 	 */
1056
-	adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
1053
+	adev->vm_manager.first_kfd_vmid = 8;
10571054 	amdgpu_vm_manager_init(adev);
10581055 
10591056 	/* base offset of vram pages */
....@@ -1368,12 +1365,13 @@
13681365 
13691366 static const struct amdgpu_gmc_funcs gmc_v7_0_gmc_funcs = {
13701367 	.flush_gpu_tlb = gmc_v7_0_flush_gpu_tlb,
1368
+	.flush_gpu_tlb_pasid = gmc_v7_0_flush_gpu_tlb_pasid,
13711369 	.emit_flush_gpu_tlb = gmc_v7_0_emit_flush_gpu_tlb,
13721370 	.emit_pasid_mapping = gmc_v7_0_emit_pasid_mapping,
1373
-	.set_pte_pde = gmc_v7_0_set_pte_pde,
13741371 	.set_prt = gmc_v7_0_set_prt,
1375
-	.get_vm_pte_flags = gmc_v7_0_get_vm_pte_flags,
1376
-	.get_vm_pde = gmc_v7_0_get_vm_pde
1372
+	.get_vm_pde = gmc_v7_0_get_vm_pde,
1373
+	.get_vm_pte = gmc_v7_0_get_vm_pte,
1374
+	.get_vbios_fb_size = gmc_v7_0_get_vbios_fb_size,
13771375 };
13781376 
13791377 static const struct amdgpu_irq_src_funcs gmc_v7_0_irq_funcs = {
....@@ -1383,8 +1381,7 @@
13831381 
13841382 static void gmc_v7_0_set_gmc_funcs(struct amdgpu_device *adev)
13851383 {
1386
-	if (adev->gmc.gmc_funcs == NULL)
1387
-		adev->gmc.gmc_funcs = &gmc_v7_0_gmc_funcs;
1384
+	adev->gmc.gmc_funcs = &gmc_v7_0_gmc_funcs;
13881385 }
13891386 
13901387 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev)