.. | .. |
---|
20 | 20 | * OTHER DEALINGS IN THE SOFTWARE. |
---|
21 | 21 | * |
---|
22 | 22 | */ |
---|
| 23 | + |
---|
23 | 24 | #include <linux/firmware.h> |
---|
24 | | -#include <drm/drmP.h> |
---|
| 25 | +#include <linux/module.h> |
---|
| 26 | +#include <linux/pci.h> |
---|
| 27 | + |
---|
25 | 28 | #include <drm/drm_cache.h> |
---|
26 | 29 | #include "amdgpu.h" |
---|
27 | 30 | #include "gmc_v6_0.h" |
---|
28 | 31 | #include "amdgpu_ucode.h" |
---|
| 32 | +#include "amdgpu_gem.h" |
---|
29 | 33 | |
---|
30 | 34 | #include "bif/bif_3_0_d.h" |
---|
31 | 35 | #include "bif/bif_3_0_sh_mask.h" |
---|
.. | .. |
---|
56 | 60 | #define MC_SEQ_MISC0__MT__GDDR5 0x50000000 |
---|
57 | 61 | #define MC_SEQ_MISC0__MT__HBM 0x60000000 |
---|
58 | 62 | #define MC_SEQ_MISC0__MT__DDR3 0xB0000000 |
---|
59 | | - |
---|
60 | | - |
---|
61 | | -static const u32 crtc_offsets[6] = |
---|
62 | | -{ |
---|
63 | | - SI_CRTC0_REGISTER_OFFSET, |
---|
64 | | - SI_CRTC1_REGISTER_OFFSET, |
---|
65 | | - SI_CRTC2_REGISTER_OFFSET, |
---|
66 | | - SI_CRTC3_REGISTER_OFFSET, |
---|
67 | | - SI_CRTC4_REGISTER_OFFSET, |
---|
68 | | - SI_CRTC5_REGISTER_OFFSET |
---|
69 | | -}; |
---|
70 | 63 | |
---|
71 | 64 | static void gmc_v6_0_mc_stop(struct amdgpu_device *adev) |
---|
72 | 65 | { |
---|
.. | .. |
---|
224 | 217 | u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF; |
---|
225 | 218 | base <<= 24; |
---|
226 | 219 | |
---|
227 | | - amdgpu_device_vram_location(adev, &adev->gmc, base); |
---|
228 | | - amdgpu_device_gart_location(adev, mc); |
---|
| 220 | + amdgpu_gmc_vram_location(adev, mc, base); |
---|
| 221 | + amdgpu_gmc_gart_location(adev, mc); |
---|
229 | 222 | } |
---|
230 | 223 | |
---|
231 | 224 | static void gmc_v6_0_mc_program(struct amdgpu_device *adev) |
---|
.. | .. |
---|
358 | 351 | return 0; |
---|
359 | 352 | } |
---|
360 | 353 | |
---|
361 | | -static void gmc_v6_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid) |
---|
| 354 | +static void gmc_v6_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, |
---|
| 355 | + uint32_t vmhub, uint32_t flush_type) |
---|
362 | 356 | { |
---|
363 | 357 | WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); |
---|
364 | 358 | } |
---|
.. | .. |
---|
381 | 375 | return pd_addr; |
---|
382 | 376 | } |
---|
383 | 377 | |
---|
384 | | -static int gmc_v6_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, |
---|
385 | | - uint32_t gpu_page_idx, uint64_t addr, |
---|
386 | | - uint64_t flags) |
---|
387 | | -{ |
---|
388 | | - void __iomem *ptr = (void *)cpu_pt_addr; |
---|
389 | | - uint64_t value; |
---|
390 | | - |
---|
391 | | - value = addr & 0xFFFFFFFFFFFFF000ULL; |
---|
392 | | - value |= flags; |
---|
393 | | - writeq(value, ptr + (gpu_page_idx * 8)); |
---|
394 | | - |
---|
395 | | - return 0; |
---|
396 | | -} |
---|
397 | | - |
---|
398 | | -static uint64_t gmc_v6_0_get_vm_pte_flags(struct amdgpu_device *adev, |
---|
399 | | - uint32_t flags) |
---|
400 | | -{ |
---|
401 | | - uint64_t pte_flag = 0; |
---|
402 | | - |
---|
403 | | - if (flags & AMDGPU_VM_PAGE_READABLE) |
---|
404 | | - pte_flag |= AMDGPU_PTE_READABLE; |
---|
405 | | - if (flags & AMDGPU_VM_PAGE_WRITEABLE) |
---|
406 | | - pte_flag |= AMDGPU_PTE_WRITEABLE; |
---|
407 | | - if (flags & AMDGPU_VM_PAGE_PRT) |
---|
408 | | - pte_flag |= AMDGPU_PTE_PRT; |
---|
409 | | - |
---|
410 | | - return pte_flag; |
---|
411 | | -} |
---|
412 | | - |
---|
413 | 378 | static void gmc_v6_0_get_vm_pde(struct amdgpu_device *adev, int level, |
---|
414 | 379 | uint64_t *addr, uint64_t *flags) |
---|
415 | 380 | { |
---|
416 | 381 | BUG_ON(*addr & 0xFFFFFF0000000FFFULL); |
---|
| 382 | +} |
---|
| 383 | + |
---|
| 384 | +static void gmc_v6_0_get_vm_pte(struct amdgpu_device *adev, |
---|
| 385 | + struct amdgpu_bo_va_mapping *mapping, |
---|
| 386 | + uint64_t *flags) |
---|
| 387 | +{ |
---|
| 388 | + *flags &= ~AMDGPU_PTE_EXECUTABLE; |
---|
| 389 | + *flags &= ~AMDGPU_PTE_PRT; |
---|
417 | 390 | } |
---|
418 | 391 | |
---|
419 | 392 | static void gmc_v6_0_set_fault_enable_default(struct amdgpu_device *adev, |
---|
.. | .. |
---|
494 | 467 | |
---|
495 | 468 | static int gmc_v6_0_gart_enable(struct amdgpu_device *adev) |
---|
496 | 469 | { |
---|
| 470 | + uint64_t table_addr; |
---|
497 | 471 | int r, i; |
---|
498 | 472 | u32 field; |
---|
499 | 473 | |
---|
500 | | - if (adev->gart.robj == NULL) { |
---|
| 474 | + if (adev->gart.bo == NULL) { |
---|
501 | 475 | dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); |
---|
502 | 476 | return -EINVAL; |
---|
503 | 477 | } |
---|
504 | 478 | r = amdgpu_gart_table_vram_pin(adev); |
---|
505 | 479 | if (r) |
---|
506 | 480 | return r; |
---|
| 481 | + |
---|
| 482 | + table_addr = amdgpu_bo_gpu_offset(adev->gart.bo); |
---|
| 483 | + |
---|
507 | 484 | /* Setup TLB control */ |
---|
508 | 485 | WREG32(mmMC_VM_MX_L1_TLB_CNTL, |
---|
509 | 486 | (0xA << 7) | |
---|
.. | .. |
---|
532 | 509 | /* setup context0 */ |
---|
533 | 510 | WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12); |
---|
534 | 511 | WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12); |
---|
535 | | - WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12); |
---|
| 512 | + WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, table_addr >> 12); |
---|
536 | 513 | WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, |
---|
537 | 514 | (u32)(adev->dummy_page_addr >> 12)); |
---|
538 | 515 | WREG32(mmVM_CONTEXT0_CNTL2, 0); |
---|
.. | .. |
---|
556 | 533 | for (i = 1; i < 16; i++) { |
---|
557 | 534 | if (i < 8) |
---|
558 | 535 | WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i, |
---|
559 | | - adev->gart.table_addr >> 12); |
---|
| 536 | + table_addr >> 12); |
---|
560 | 537 | else |
---|
561 | 538 | WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8, |
---|
562 | | - adev->gart.table_addr >> 12); |
---|
| 539 | + table_addr >> 12); |
---|
563 | 540 | } |
---|
564 | 541 | |
---|
565 | 542 | /* enable context1-15 */ |
---|
.. | .. |
---|
576 | 553 | else |
---|
577 | 554 | gmc_v6_0_set_fault_enable_default(adev, true); |
---|
578 | 555 | |
---|
579 | | - gmc_v6_0_flush_gpu_tlb(adev, 0); |
---|
| 556 | + gmc_v6_0_flush_gpu_tlb(adev, 0, 0, 0); |
---|
580 | 557 | dev_info(adev->dev, "PCIE GART of %uM enabled (table at 0x%016llX).\n", |
---|
581 | 558 | (unsigned)(adev->gmc.gart_size >> 20), |
---|
582 | | - (unsigned long long)adev->gart.table_addr); |
---|
| 559 | + (unsigned long long)table_addr); |
---|
583 | 560 | adev->gart.ready = true; |
---|
584 | 561 | return 0; |
---|
585 | 562 | } |
---|
.. | .. |
---|
588 | 565 | { |
---|
589 | 566 | int r; |
---|
590 | 567 | |
---|
591 | | - if (adev->gart.robj) { |
---|
| 568 | + if (adev->gart.bo) { |
---|
592 | 569 | dev_warn(adev->dev, "gmc_v6_0 PCIE GART already initialized\n"); |
---|
593 | 570 | return 0; |
---|
594 | 571 | } |
---|
.. | .. |
---|
828 | 805 | unsigned size; |
---|
829 | 806 | |
---|
830 | 807 | if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { |
---|
831 | | - size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */ |
---|
| 808 | + size = AMDGPU_VBIOS_VGA_ALLOCATION; |
---|
832 | 809 | } else { |
---|
833 | 810 | u32 viewport = RREG32(mmVIEWPORT_SIZE); |
---|
834 | 811 | size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) * |
---|
835 | 812 | REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) * |
---|
836 | 813 | 4); |
---|
837 | 814 | } |
---|
838 | | - /* return 0 if the pre-OS buffer uses up most of vram */ |
---|
839 | | - if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) |
---|
840 | | - return 0; |
---|
841 | 815 | return size; |
---|
842 | 816 | } |
---|
843 | 817 | |
---|
844 | 818 | static int gmc_v6_0_sw_init(void *handle) |
---|
845 | 819 | { |
---|
846 | 820 | int r; |
---|
847 | | - int dma_bits; |
---|
848 | 821 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
---|
| 822 | + |
---|
| 823 | + adev->num_vmhubs = 1; |
---|
849 | 824 | |
---|
850 | 825 | if (adev->flags & AMD_IS_APU) { |
---|
851 | 826 | adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; |
---|
.. | .. |
---|
855 | 830 | adev->gmc.vram_type = gmc_v6_0_convert_vram_type(tmp); |
---|
856 | 831 | } |
---|
857 | 832 | |
---|
858 | | - r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 146, &adev->gmc.vm_fault); |
---|
| 833 | + r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 146, &adev->gmc.vm_fault); |
---|
859 | 834 | if (r) |
---|
860 | 835 | return r; |
---|
861 | 836 | |
---|
862 | | - r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 147, &adev->gmc.vm_fault); |
---|
| 837 | + r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 147, &adev->gmc.vm_fault); |
---|
863 | 838 | if (r) |
---|
864 | 839 | return r; |
---|
865 | 840 | |
---|
.. | .. |
---|
867 | 842 | |
---|
868 | 843 | adev->gmc.mc_mask = 0xffffffffffULL; |
---|
869 | 844 | |
---|
870 | | - adev->need_dma32 = false; |
---|
871 | | - dma_bits = adev->need_dma32 ? 32 : 40; |
---|
872 | | - r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits)); |
---|
| 845 | + r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(40)); |
---|
873 | 846 | if (r) { |
---|
874 | | - adev->need_dma32 = true; |
---|
875 | | - dma_bits = 32; |
---|
876 | | - dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n"); |
---|
| 847 | + dev_warn(adev->dev, "No suitable DMA available.\n"); |
---|
| 848 | + return r; |
---|
877 | 849 | } |
---|
878 | | - r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits)); |
---|
879 | | - if (r) { |
---|
880 | | - pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32)); |
---|
881 | | - dev_warn(adev->dev, "amdgpu: No coherent DMA available.\n"); |
---|
882 | | - } |
---|
883 | | - adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits); |
---|
| 850 | + adev->need_swiotlb = drm_need_swiotlb(40); |
---|
884 | 851 | |
---|
885 | 852 | r = gmc_v6_0_init_microcode(adev); |
---|
886 | 853 | if (r) { |
---|
.. | .. |
---|
892 | 859 | if (r) |
---|
893 | 860 | return r; |
---|
894 | 861 | |
---|
895 | | - adev->gmc.stolen_size = gmc_v6_0_get_vbios_fb_size(adev); |
---|
| 862 | + amdgpu_gmc_get_vbios_allocations(adev); |
---|
896 | 863 | |
---|
897 | 864 | r = amdgpu_bo_init(adev); |
---|
898 | 865 | if (r) |
---|
.. | .. |
---|
908 | 875 | * amdgpu graphics/compute will use VMIDs 1-7 |
---|
909 | 876 | * amdkfd will use VMIDs 8-15 |
---|
910 | 877 | */ |
---|
911 | | - adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS; |
---|
| 878 | + adev->vm_manager.first_kfd_vmid = 8; |
---|
912 | 879 | amdgpu_vm_manager_init(adev); |
---|
913 | 880 | |
---|
914 | 881 | /* base offset of vram pages */ |
---|
.. | .. |
---|
1163 | 1130 | static const struct amdgpu_gmc_funcs gmc_v6_0_gmc_funcs = { |
---|
1164 | 1131 | .flush_gpu_tlb = gmc_v6_0_flush_gpu_tlb, |
---|
1165 | 1132 | .emit_flush_gpu_tlb = gmc_v6_0_emit_flush_gpu_tlb, |
---|
1166 | | - .set_pte_pde = gmc_v6_0_set_pte_pde, |
---|
1167 | 1133 | .set_prt = gmc_v6_0_set_prt, |
---|
1168 | 1134 | .get_vm_pde = gmc_v6_0_get_vm_pde, |
---|
1169 | | - .get_vm_pte_flags = gmc_v6_0_get_vm_pte_flags |
---|
| 1135 | + .get_vm_pte = gmc_v6_0_get_vm_pte, |
---|
| 1136 | + .get_vbios_fb_size = gmc_v6_0_get_vbios_fb_size, |
---|
1170 | 1137 | }; |
---|
1171 | 1138 | |
---|
1172 | 1139 | static const struct amdgpu_irq_src_funcs gmc_v6_0_irq_funcs = { |
---|
.. | .. |
---|
1176 | 1143 | |
---|
1177 | 1144 | static void gmc_v6_0_set_gmc_funcs(struct amdgpu_device *adev) |
---|
1178 | 1145 | { |
---|
1179 | | - if (adev->gmc.gmc_funcs == NULL) |
---|
1180 | | - adev->gmc.gmc_funcs = &gmc_v6_0_gmc_funcs; |
---|
| 1146 | + adev->gmc.gmc_funcs = &gmc_v6_0_gmc_funcs; |
---|
1181 | 1147 | } |
---|
1182 | 1148 | |
---|
1183 | 1149 | static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev) |
---|