From d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d Mon Sep 17 00:00:00 2001
From: hc <hc@nodka.com>
Date: Mon, 11 Dec 2023 02:45:28 +0000
Subject: [PATCH] add boot partition size
---
kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 280 ++++++++++++++++++++++++++++++++-----------------------
1 files changed, 164 insertions(+), 116 deletions(-)
diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 98d20ef..6aa9fd9 100644
--- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -26,11 +26,17 @@
* Jerome Glisse
*/
#include <linux/ktime.h>
+#include <linux/module.h>
#include <linux/pagemap.h>
-#include <drm/drmP.h>
+#include <linux/pci.h>
+#include <linux/dma-buf.h>
+
#include <drm/amdgpu_drm.h>
+#include <drm/drm_debugfs.h>
+
#include "amdgpu.h"
#include "amdgpu_display.h"
+#include "amdgpu_xgmi.h"
void amdgpu_gem_object_free(struct drm_gem_object *gobj)
{
@@ -45,7 +51,7 @@
int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
int alignment, u32 initial_domain,
u64 flags, enum ttm_bo_type type,
- struct reservation_object *resv,
+ struct dma_resv *resv,
struct drm_gem_object **obj)
{
struct amdgpu_bo *bo;
@@ -54,44 +60,26 @@
memset(&bp, 0, sizeof(bp));
*obj = NULL;
- /* At least align on page size */
- if (alignment < PAGE_SIZE) {
- alignment = PAGE_SIZE;
- }
bp.size = size;
bp.byte_align = alignment;
bp.type = type;
bp.resv = resv;
bp.preferred_domain = initial_domain;
-retry:
bp.flags = flags;
bp.domain = initial_domain;
r = amdgpu_bo_create(adev, &bp, &bo);
- if (r) {
- if (r != -ERESTARTSYS) {
- if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
- flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
- goto retry;
- }
-
- if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
- initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
- goto retry;
- }
- DRM_DEBUG("Failed to allocate GEM object (%ld, %d, %u, %d)\n",
- size, initial_domain, alignment, r);
- }
+ if (r)
return r;
- }
- *obj = &bo->gem_base;
+
+ *obj = &bo->tbo.base;
return 0;
}
void amdgpu_gem_force_release(struct amdgpu_device *adev)
{
- struct drm_device *ddev = adev->ddev;
+ struct drm_device *ddev = adev_to_drm(adev);
struct drm_file *file;
mutex_lock(&ddev->filelist_mutex);
@@ -104,7 +92,7 @@
spin_lock(&file->table_lock);
idr_for_each_entry(&file->object_idr, gobj, handle) {
WARN_ONCE(1, "And also active allocations!\n");
- drm_gem_object_put_unlocked(gobj);
+ drm_gem_object_put(gobj);
}
idr_destroy(&file->object_idr);
spin_unlock(&file->table_lock);
@@ -133,7 +121,7 @@
return -EPERM;
if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID &&
- abo->tbo.resv != vm->root.base.bo->tbo.resv)
+ abo->tbo.base.resv != vm->root.base.bo->tbo.base.resv)
return -EPERM;
r = amdgpu_bo_reserve(abo, false);
@@ -160,16 +148,17 @@
struct amdgpu_bo_list_entry vm_pd;
struct list_head list, duplicates;
+ struct dma_fence *fence = NULL;
struct ttm_validate_buffer tv;
struct ww_acquire_ctx ticket;
struct amdgpu_bo_va *bo_va;
- int r;
+ long r;
INIT_LIST_HEAD(&list);
INIT_LIST_HEAD(&duplicates);
tv.bo = &bo->tbo;
- tv.shared = true;
+ tv.num_shared = 2;
list_add(&tv.head, &list);
amdgpu_vm_get_pd_bo(vm, &list, &vm_pd);
@@ -177,28 +166,34 @@
r = ttm_eu_reserve_buffers(&ticket, &list, false, &duplicates);
if (r) {
dev_err(adev->dev, "leaking bo va because "
- "we fail to reserve bo (%d)\n", r);
+ "we fail to reserve bo (%ld)\n", r);
return;
}
bo_va = amdgpu_vm_bo_find(vm, bo);
- if (bo_va && --bo_va->ref_count == 0) {
- amdgpu_vm_bo_rmv(adev, bo_va);
+ if (!bo_va || --bo_va->ref_count)
+ goto out_unlock;
- if (amdgpu_vm_ready(vm)) {
- struct dma_fence *fence = NULL;
+ amdgpu_vm_bo_rmv(adev, bo_va);
+ if (!amdgpu_vm_ready(vm))
+ goto out_unlock;
- r = amdgpu_vm_clear_freed(adev, vm, &fence);
- if (unlikely(r)) {
- dev_err(adev->dev, "failed to clear page "
- "tables on GEM object close (%d)\n", r);
- }
-
- if (fence) {
- amdgpu_bo_fence(bo, fence, true);
- dma_fence_put(fence);
- }
- }
+ fence = dma_resv_get_excl(bo->tbo.base.resv);
+ if (fence) {
+ amdgpu_bo_fence(bo, fence, true);
+ fence = NULL;
}
+
+ r = amdgpu_vm_clear_freed(adev, vm, &fence);
+ if (r || !fence)
+ goto out_unlock;
+
+ amdgpu_bo_fence(bo, fence, true);
+ dma_fence_put(fence);
+
+out_unlock:
+ if (unlikely(r < 0))
+ dev_err(adev->dev, "failed to clear page "
+ "tables on GEM object close (%ld)\n", r);
ttm_eu_backoff_reservation(&ticket, &list);
}
@@ -208,15 +203,15 @@
int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
struct drm_file *filp)
{
- struct amdgpu_device *adev = dev->dev_private;
+ struct amdgpu_device *adev = drm_to_adev(dev);
struct amdgpu_fpriv *fpriv = filp->driver_priv;
struct amdgpu_vm *vm = &fpriv->vm;
union drm_amdgpu_gem_create *args = data;
uint64_t flags = args->in.domain_flags;
uint64_t size = args->in.bo_size;
- struct reservation_object *resv = NULL;
+ struct dma_resv *resv = NULL;
struct drm_gem_object *gobj;
- uint32_t handle;
+ uint32_t handle, initial_domain;
int r;
/* reject invalid gem flags */
@@ -225,13 +220,19 @@
AMDGPU_GEM_CREATE_CPU_GTT_USWC |
AMDGPU_GEM_CREATE_VRAM_CLEARED |
AMDGPU_GEM_CREATE_VM_ALWAYS_VALID |
- AMDGPU_GEM_CREATE_EXPLICIT_SYNC))
+ AMDGPU_GEM_CREATE_EXPLICIT_SYNC |
+ AMDGPU_GEM_CREATE_ENCRYPTED))
return -EINVAL;
/* reject invalid gem domains */
if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK)
return -EINVAL;
+
+ if (!amdgpu_is_tmz(adev) && (flags & AMDGPU_GEM_CREATE_ENCRYPTED)) {
+ DRM_NOTE_ONCE("Cannot allocate secure buffer since TMZ is disabled\n");
+ return -EINVAL;
+ }
/* create a gem object to contain this object in */
if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
@@ -244,28 +245,38 @@
return -EINVAL;
}
flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
- if (args->in.domains == AMDGPU_GEM_DOMAIN_GDS)
- size = size << AMDGPU_GDS_SHIFT;
- else if (args->in.domains == AMDGPU_GEM_DOMAIN_GWS)
- size = size << AMDGPU_GWS_SHIFT;
- else if (args->in.domains == AMDGPU_GEM_DOMAIN_OA)
- size = size << AMDGPU_OA_SHIFT;
- else
- return -EINVAL;
}
- size = roundup(size, PAGE_SIZE);
if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
r = amdgpu_bo_reserve(vm->root.base.bo, false);
if (r)
return r;
- resv = vm->root.base.bo->tbo.resv;
+ resv = vm->root.base.bo->tbo.base.resv;
}
+retry:
+ initial_domain = (u32)(0xffffffff & args->in.domains);
r = amdgpu_gem_object_create(adev, size, args->in.alignment,
- (u32)(0xffffffff & args->in.domains),
+ initial_domain,
flags, ttm_bo_type_device, resv, &gobj);
+ if (r) {
+ if (r != -ERESTARTSYS) {
+ if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
+ flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
+ goto retry;
+ }
+
+ if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
+ initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
+ goto retry;
+ }
+ DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n",
+ size, initial_domain, args->in.alignment, r);
+ }
+ return r;
+ }
+
if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
if (!r) {
struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
@@ -279,7 +290,7 @@
r = drm_gem_handle_create(filp, gobj, &handle);
/* drop reference from allocate - handle holds it now */
- drm_gem_object_put_unlocked(gobj);
+ drm_gem_object_put(gobj);
if (r)
return r;
@@ -292,7 +303,7 @@
struct drm_file *filp)
{
struct ttm_operation_ctx ctx = { true, false };
- struct amdgpu_device *adev = dev->dev_private;
+ struct amdgpu_device *adev = drm_to_adev(dev);
struct drm_amdgpu_gem_userptr *args = data;
struct drm_gem_object *gobj;
struct amdgpu_bo *bo;
@@ -326,47 +337,42 @@
bo = gem_to_amdgpu_bo(gobj);
bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
- r = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, args->addr, args->flags);
+ r = amdgpu_ttm_tt_set_userptr(&bo->tbo, args->addr, args->flags);
if (r)
goto release_object;
- if (args->flags & AMDGPU_GEM_USERPTR_REGISTER) {
- r = amdgpu_mn_register(bo, args->addr);
- if (r)
- goto release_object;
- }
+ r = amdgpu_mn_register(bo, args->addr);
+ if (r)
+ goto release_object;
if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
- r = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
- bo->tbo.ttm->pages);
+ r = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);
if (r)
goto release_object;
r = amdgpu_bo_reserve(bo, true);
if (r)
- goto free_pages;
+ goto user_pages_done;
amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
amdgpu_bo_unreserve(bo);
if (r)
- goto free_pages;
+ goto user_pages_done;
}
r = drm_gem_handle_create(filp, gobj, &handle);
- /* drop reference from allocate - handle holds it now */
- drm_gem_object_put_unlocked(gobj);
if (r)
- return r;
+ goto user_pages_done;
args->handle = handle;
- return 0;
-free_pages:
- release_pages(bo->tbo.ttm->pages, bo->tbo.ttm->num_pages);
+user_pages_done:
+ if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE)
+ amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
release_object:
- drm_gem_object_put_unlocked(gobj);
+ drm_gem_object_put(gobj);
return r;
}
@@ -385,11 +391,11 @@
robj = gem_to_amdgpu_bo(gobj);
if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) ||
(robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
- drm_gem_object_put_unlocked(gobj);
+ drm_gem_object_put(gobj);
return -EPERM;
}
*offset_p = amdgpu_bo_mmap_offset(robj);
- drm_gem_object_put_unlocked(gobj);
+ drm_gem_object_put(gobj);
return 0;
}
@@ -446,7 +452,7 @@
return -ENOENT;
}
robj = gem_to_amdgpu_bo(gobj);
- ret = reservation_object_wait_timeout_rcu(robj->tbo.resv, true, true,
+ ret = dma_resv_wait_timeout_rcu(robj->tbo.base.resv, true, true,
timeout);
/* ret == 0 means not signaled,
@@ -459,7 +465,7 @@
} else
r = ret;
- drm_gem_object_put_unlocked(gobj);
+ drm_gem_object_put(gobj);
return r;
}
@@ -502,7 +508,7 @@
unreserve:
amdgpu_bo_unreserve(robj);
out:
- drm_gem_object_put_unlocked(gobj);
+ drm_gem_object_put(gobj);
return r;
}
@@ -538,11 +544,39 @@
goto error;
}
- r = amdgpu_vm_update_directories(adev, vm);
+ r = amdgpu_vm_update_pdes(adev, vm, false);
error:
if (r && r != -ERESTARTSYS)
DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
+}
+
+/**
+ * amdgpu_gem_va_map_flags - map GEM UAPI flags into hardware flags
+ *
+ * @adev: amdgpu_device pointer
+ * @flags: GEM UAPI flags
+ *
+ * Returns the GEM UAPI flags mapped into hardware for the ASIC.
+ */
+uint64_t amdgpu_gem_va_map_flags(struct amdgpu_device *adev, uint32_t flags)
+{
+ uint64_t pte_flag = 0;
+
+ if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
+ pte_flag |= AMDGPU_PTE_EXECUTABLE;
+ if (flags & AMDGPU_VM_PAGE_READABLE)
+ pte_flag |= AMDGPU_PTE_READABLE;
+ if (flags & AMDGPU_VM_PAGE_WRITEABLE)
+ pte_flag |= AMDGPU_PTE_WRITEABLE;
+ if (flags & AMDGPU_VM_PAGE_PRT)
+ pte_flag |= AMDGPU_PTE_PRT;
+
+ if (adev->gmc.gmc_funcs->map_mtype)
+ pte_flag |= amdgpu_gmc_map_mtype(adev,
+ flags & AMDGPU_VM_MTYPE_MASK);
+
+ return pte_flag;
}
int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
@@ -556,7 +590,7 @@
struct drm_amdgpu_gem_va *args = data;
struct drm_gem_object *gobj;
- struct amdgpu_device *adev = dev->dev_private;
+ struct amdgpu_device *adev = drm_to_adev(dev);
struct amdgpu_fpriv *fpriv = filp->driver_priv;
struct amdgpu_bo *abo;
struct amdgpu_bo_va *bo_va;
@@ -575,16 +609,16 @@
return -EINVAL;
}
- if (args->va_address >= AMDGPU_VA_HOLE_START &&
- args->va_address < AMDGPU_VA_HOLE_END) {
+ if (args->va_address >= AMDGPU_GMC_HOLE_START &&
+ args->va_address < AMDGPU_GMC_HOLE_END) {
dev_dbg(&dev->pdev->dev,
"va_address 0x%LX is in VA hole 0x%LX-0x%LX\n",
- args->va_address, AMDGPU_VA_HOLE_START,
- AMDGPU_VA_HOLE_END);
+ args->va_address, AMDGPU_GMC_HOLE_START,
+ AMDGPU_GMC_HOLE_END);
return -EINVAL;
}
- args->va_address &= AMDGPU_VA_HOLE_MASK;
+ args->va_address &= AMDGPU_GMC_HOLE_MASK;
vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
vm_size -= AMDGPU_VA_RESERVED_SIZE;
@@ -622,7 +656,10 @@
return -ENOENT;
abo = gem_to_amdgpu_bo(gobj);
tv.bo = &abo->tbo;
- tv.shared = !!(abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID);
+ if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
+ tv.num_shared = 1;
+ else
+ tv.num_shared = 0;
list_add(&tv.head, &list);
} else {
gobj = NULL;
@@ -649,12 +686,7 @@
switch (args->operation) {
case AMDGPU_VA_OP_MAP:
- r = amdgpu_vm_alloc_pts(adev, bo_va->base.vm, args->va_address,
- args->map_size);
- if (r)
- goto error_backoff;
-
- va_flags = amdgpu_gmc_get_pte_flags(adev, args->flags);
+ va_flags = amdgpu_gem_va_map_flags(adev, args->flags);
r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,
args->offset_in_bo, args->map_size,
va_flags);
@@ -669,12 +701,7 @@
args->map_size);
break;
case AMDGPU_VA_OP_REPLACE:
- r = amdgpu_vm_alloc_pts(adev, bo_va->base.vm, args->va_address,
- args->map_size);
- if (r)
- goto error_backoff;
-
- va_flags = amdgpu_gmc_get_pte_flags(adev, args->flags);
+ va_flags = amdgpu_gem_va_map_flags(adev, args->flags);
r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,
args->offset_in_bo, args->map_size,
va_flags);
@@ -690,16 +717,17 @@
ttm_eu_backoff_reservation(&ticket, &list);
error_unref:
- drm_gem_object_put_unlocked(gobj);
+ drm_gem_object_put(gobj);
return r;
}
int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
struct drm_file *filp)
{
- struct amdgpu_device *adev = dev->dev_private;
+ struct amdgpu_device *adev = drm_to_adev(dev);
struct drm_amdgpu_gem_op *args = data;
struct drm_gem_object *gobj;
+ struct amdgpu_vm_bo_base *base;
struct amdgpu_bo *robj;
int r;
@@ -718,7 +746,7 @@
struct drm_amdgpu_gem_create_in info;
void __user *out = u64_to_user_ptr(args->value);
- info.bo_size = robj->gem_base.size;
+ info.bo_size = robj->tbo.base.size;
info.alignment = robj->tbo.mem.page_alignment << PAGE_SHIFT;
info.domains = robj->preferred_domains;
info.domain_flags = robj->flags;
@@ -738,6 +766,15 @@
amdgpu_bo_unreserve(robj);
break;
}
+ for (base = robj->vm_bo; base; base = base->next)
+ if (amdgpu_xgmi_same_hive(amdgpu_ttm_adev(robj->tbo.bdev),
+ amdgpu_ttm_adev(base->vm->root.base.bo->tbo.bdev))) {
+ r = -EINVAL;
+ amdgpu_bo_unreserve(robj);
+ goto out;
+ }
+
+
robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
AMDGPU_GEM_DOMAIN_GTT |
AMDGPU_GEM_DOMAIN_CPU);
@@ -756,7 +793,7 @@
}
out:
- drm_gem_object_put_unlocked(gobj);
+ drm_gem_object_put(gobj);
return r;
}
@@ -764,27 +801,36 @@
struct drm_device *dev,
struct drm_mode_create_dumb *args)
{
- struct amdgpu_device *adev = dev->dev_private;
+ struct amdgpu_device *adev = drm_to_adev(dev);
struct drm_gem_object *gobj;
uint32_t handle;
+ u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+ AMDGPU_GEM_CREATE_CPU_GTT_USWC;
u32 domain;
int r;
+
+ /*
+ * The buffer returned from this function should be cleared, but
+ * it can only be done if the ring is enabled or we'll fail to
+ * create the buffer.
+ */
+ if (adev->mman.buffer_funcs_enabled)
+ flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
args->pitch = amdgpu_align_pitch(adev, args->width,
DIV_ROUND_UP(args->bpp, 8), 0);
args->size = (u64)args->pitch * args->height;
args->size = ALIGN(args->size, PAGE_SIZE);
domain = amdgpu_bo_get_preferred_pin_domain(adev,
- amdgpu_display_supported_domains(adev));
- r = amdgpu_gem_object_create(adev, args->size, 0, domain,
- AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ amdgpu_display_supported_domains(adev, flags));
+ r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,
ttm_bo_type_device, NULL, &gobj);
if (r)
return -ENOMEM;
r = drm_gem_handle_create(file_priv, gobj, &handle);
/* drop reference from allocate - handle holds it now */
- drm_gem_object_put_unlocked(gobj);
+ drm_gem_object_put(gobj);
if (r) {
return r;
}
@@ -831,11 +877,12 @@
if (pin_count)
seq_printf(m, " pin count %d", pin_count);
- dma_buf = READ_ONCE(bo->gem_base.dma_buf);
- attachment = READ_ONCE(bo->gem_base.import_attach);
+ dma_buf = READ_ONCE(bo->tbo.base.dma_buf);
+ attachment = READ_ONCE(bo->tbo.base.import_attach);
if (attachment)
- seq_printf(m, " imported from %p", dma_buf);
+ seq_printf(m, " imported from %p%s", dma_buf,
+ attachment->peer2peer ? " P2P" : "");
else if (dma_buf)
seq_printf(m, " exported as %p", dma_buf);
@@ -896,7 +943,8 @@
int amdgpu_debugfs_gem_init(struct amdgpu_device *adev)
{
#if defined(CONFIG_DEBUG_FS)
- return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_gem_list, 1);
+ return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_gem_list,
+ ARRAY_SIZE(amdgpu_debugfs_gem_list));
#endif
return 0;
}
--
Gitblit v1.6.2