From 6778948f9de86c3cfaf36725a7c87dcff9ba247f Mon Sep 17 00:00:00 2001
From: hc <hc@nodka.com>
Date: Mon, 11 Dec 2023 08:20:59 +0000
Subject: [PATCH] kernel_5.10 no rt
---
kernel/drivers/gpu/drm/amd/amdkfd/kfd_process.c | 894 +++++++++++++++++++++++++++++++++++++++++++++++++++-------
1 files changed, 777 insertions(+), 117 deletions(-)
diff --git a/kernel/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/kernel/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 4694386..d243e60 100644
--- a/kernel/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/kernel/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -25,12 +25,16 @@
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/task.h>
+#include <linux/mmu_context.h>
#include <linux/slab.h>
#include <linux/amd-iommu.h>
#include <linux/notifier.h>
#include <linux/compat.h>
#include <linux/mman.h>
#include <linux/file.h>
+#include <linux/pm_runtime.h>
+#include "amdgpu_amdkfd.h"
+#include "amdgpu.h"
struct mm_struct;
@@ -61,12 +65,548 @@
static struct kfd_process *find_process(const struct task_struct *thread);
static void kfd_process_ref_release(struct kref *ref);
-static struct kfd_process *create_process(const struct task_struct *thread,
- struct file *filep);
+static struct kfd_process *create_process(const struct task_struct *thread);
+static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep);
static void evict_process_worker(struct work_struct *work);
static void restore_process_worker(struct work_struct *work);
+struct kfd_procfs_tree {
+ struct kobject *kobj;
+};
+
+static struct kfd_procfs_tree procfs;
+
+/*
+ * Structure for SDMA activity tracking
+ */
+struct kfd_sdma_activity_handler_workarea {
+ struct work_struct sdma_activity_work;
+ struct kfd_process_device *pdd;
+ uint64_t sdma_activity_counter;
+};
+
+struct temp_sdma_queue_list {
+ uint64_t __user *rptr;
+ uint64_t sdma_val;
+ unsigned int queue_id;
+ struct list_head list;
+};
+
+static void kfd_sdma_activity_worker(struct work_struct *work)
+{
+ struct kfd_sdma_activity_handler_workarea *workarea;
+ struct kfd_process_device *pdd;
+ uint64_t val;
+ struct mm_struct *mm;
+ struct queue *q;
+ struct qcm_process_device *qpd;
+ struct device_queue_manager *dqm;
+ int ret = 0;
+ struct temp_sdma_queue_list sdma_q_list;
+ struct temp_sdma_queue_list *sdma_q, *next;
+
+ workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
+ sdma_activity_work);
+ if (!workarea)
+ return;
+
+ pdd = workarea->pdd;
+ if (!pdd)
+ return;
+ dqm = pdd->dev->dqm;
+ qpd = &pdd->qpd;
+ if (!dqm || !qpd)
+ return;
+ /*
+ * Total SDMA activity is current SDMA activity + past SDMA activity
+ * Past SDMA count is stored in pdd.
+ * To get the current activity counters for all active SDMA queues,
+ * we loop over all SDMA queues and get their counts from user-space.
+ *
+ * We cannot call get_user() with dqm_lock held as it can cause
+ * a circular lock dependency situation. To read the SDMA stats,
+ * we need to do the following:
+ *
+ * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
+ * with dqm_lock/dqm_unlock().
+ * 2. Call get_user() for each node in temporary list without dqm_lock.
+ * Save the SDMA count for each node and also add the count to the total
+ * SDMA count counter.
+ * Its possible, during this step, a few SDMA queue nodes got deleted
+ * from the qpd->queues_list.
+ * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
+ * If any node got deleted, its SDMA count would be captured in the sdma
+ * past activity counter. So subtract the SDMA counter stored in step 2
+ * for this node from the total SDMA count.
+ */
+ INIT_LIST_HEAD(&sdma_q_list.list);
+
+ /*
+ * Create the temp list of all SDMA queues
+ */
+ dqm_lock(dqm);
+
+ list_for_each_entry(q, &qpd->queues_list, list) {
+ if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
+ (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
+ continue;
+
+ sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
+ if (!sdma_q) {
+ dqm_unlock(dqm);
+ goto cleanup;
+ }
+
+ INIT_LIST_HEAD(&sdma_q->list);
+ sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
+ sdma_q->queue_id = q->properties.queue_id;
+ list_add_tail(&sdma_q->list, &sdma_q_list.list);
+ }
+
+ /*
+ * If the temp list is empty, then no SDMA queues nodes were found in
+ * qpd->queues_list. Return the past activity count as the total sdma
+ * count
+ */
+ if (list_empty(&sdma_q_list.list)) {
+ workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
+ dqm_unlock(dqm);
+ return;
+ }
+
+ dqm_unlock(dqm);
+
+ /*
+ * Get the usage count for each SDMA queue in temp_list.
+ */
+ mm = get_task_mm(pdd->process->lead_thread);
+ if (!mm)
+ goto cleanup;
+
+ kthread_use_mm(mm);
+
+ list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
+ val = 0;
+ ret = read_sdma_queue_counter(sdma_q->rptr, &val);
+ if (ret) {
+ pr_debug("Failed to read SDMA queue active counter for queue id: %d",
+ sdma_q->queue_id);
+ } else {
+ sdma_q->sdma_val = val;
+ workarea->sdma_activity_counter += val;
+ }
+ }
+
+ kthread_unuse_mm(mm);
+ mmput(mm);
+
+ /*
+ * Do a second iteration over qpd_queues_list to check if any SDMA
+ * nodes got deleted while fetching SDMA counter.
+ */
+ dqm_lock(dqm);
+
+ workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
+
+ list_for_each_entry(q, &qpd->queues_list, list) {
+ if (list_empty(&sdma_q_list.list))
+ break;
+
+ if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
+ (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
+ continue;
+
+ list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
+ if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
+ (sdma_q->queue_id == q->properties.queue_id)) {
+ list_del(&sdma_q->list);
+ kfree(sdma_q);
+ break;
+ }
+ }
+ }
+
+ dqm_unlock(dqm);
+
+ /*
+ * If temp list is not empty, it implies some queues got deleted
+ * from qpd->queues_list during SDMA usage read. Subtract the SDMA
+ * count for each node from the total SDMA count.
+ */
+ list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
+ workarea->sdma_activity_counter -= sdma_q->sdma_val;
+ list_del(&sdma_q->list);
+ kfree(sdma_q);
+ }
+
+ return;
+
+cleanup:
+ list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
+ list_del(&sdma_q->list);
+ kfree(sdma_q);
+ }
+}
+
+/**
+ * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
+ * by current process. Translates acquired wave count into number of compute units
+ * that are occupied.
+ *
+ * @atr: Handle of attribute that allows reporting of wave count. The attribute
+ * handle encapsulates GPU device it is associated with, thereby allowing collection
+ * of waves in flight, etc
+ *
+ * @buffer: Handle of user provided buffer updated with wave count
+ *
+ * Return: Number of bytes written to user buffer or an error value
+ */
+static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
+{
+ int cu_cnt;
+ int wave_cnt;
+ int max_waves_per_cu;
+ struct kfd_dev *dev = NULL;
+ struct kfd_process *proc = NULL;
+ struct kfd_process_device *pdd = NULL;
+
+ pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
+ dev = pdd->dev;
+ if (dev->kfd2kgd->get_cu_occupancy == NULL)
+ return -EINVAL;
+
+ cu_cnt = 0;
+ proc = pdd->process;
+ if (pdd->qpd.queue_count == 0) {
+ pr_debug("Gpu-Id: %d has no active queues for process %d\n",
+ dev->id, proc->pasid);
+ return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+ }
+
+ /* Collect wave count from device if it supports */
+ wave_cnt = 0;
+ max_waves_per_cu = 0;
+ dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
+ &max_waves_per_cu);
+
+ /* Translate wave count to number of compute units */
+ cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
+ return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+}
+
+static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
+ char *buffer)
+{
+ if (strcmp(attr->name, "pasid") == 0) {
+ struct kfd_process *p = container_of(attr, struct kfd_process,
+ attr_pasid);
+
+ return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid);
+ } else if (strncmp(attr->name, "vram_", 5) == 0) {
+ struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
+ attr_vram);
+ return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage));
+ } else if (strncmp(attr->name, "sdma_", 5) == 0) {
+ struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
+ attr_sdma);
+ struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
+
+ INIT_WORK(&sdma_activity_work_handler.sdma_activity_work,
+ kfd_sdma_activity_worker);
+
+ sdma_activity_work_handler.pdd = pdd;
+ sdma_activity_work_handler.sdma_activity_counter = 0;
+
+ schedule_work(&sdma_activity_work_handler.sdma_activity_work);
+
+ flush_work(&sdma_activity_work_handler.sdma_activity_work);
+
+ return snprintf(buffer, PAGE_SIZE, "%llu\n",
+ (sdma_activity_work_handler.sdma_activity_counter)/
+ SDMA_ACTIVITY_DIVISOR);
+ } else {
+ pr_err("Invalid attribute");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void kfd_procfs_kobj_release(struct kobject *kobj)
+{
+ kfree(kobj);
+}
+
+static const struct sysfs_ops kfd_procfs_ops = {
+ .show = kfd_procfs_show,
+};
+
+static struct kobj_type procfs_type = {
+ .release = kfd_procfs_kobj_release,
+ .sysfs_ops = &kfd_procfs_ops,
+};
+
+void kfd_procfs_init(void)
+{
+ int ret = 0;
+
+ procfs.kobj = kfd_alloc_struct(procfs.kobj);
+ if (!procfs.kobj)
+ return;
+
+ ret = kobject_init_and_add(procfs.kobj, &procfs_type,
+ &kfd_device->kobj, "proc");
+ if (ret) {
+ pr_warn("Could not create procfs proc folder");
+ /* If we fail to create the procfs, clean up */
+ kfd_procfs_shutdown();
+ }
+}
+
+void kfd_procfs_shutdown(void)
+{
+ if (procfs.kobj) {
+ kobject_del(procfs.kobj);
+ kobject_put(procfs.kobj);
+ procfs.kobj = NULL;
+ }
+}
+
+static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
+ struct attribute *attr, char *buffer)
+{
+ struct queue *q = container_of(kobj, struct queue, kobj);
+
+ if (!strcmp(attr->name, "size"))
+ return snprintf(buffer, PAGE_SIZE, "%llu",
+ q->properties.queue_size);
+ else if (!strcmp(attr->name, "type"))
+ return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
+ else if (!strcmp(attr->name, "gpuid"))
+ return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
+ else
+ pr_err("Invalid attribute");
+
+ return 0;
+}
+
+static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
+ struct attribute *attr, char *buffer)
+{
+ if (strcmp(attr->name, "evicted_ms") == 0) {
+ struct kfd_process_device *pdd = container_of(attr,
+ struct kfd_process_device,
+ attr_evict);
+ uint64_t evict_jiffies;
+
+ evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
+
+ return snprintf(buffer,
+ PAGE_SIZE,
+ "%llu\n",
+ jiffies64_to_msecs(evict_jiffies));
+
+ /* Sysfs handle that gets CU occupancy is per device */
+ } else if (strcmp(attr->name, "cu_occupancy") == 0) {
+ return kfd_get_cu_occupancy(attr, buffer);
+ } else {
+ pr_err("Invalid attribute");
+ }
+
+ return 0;
+}
+
+static struct attribute attr_queue_size = {
+ .name = "size",
+ .mode = KFD_SYSFS_FILE_MODE
+};
+
+static struct attribute attr_queue_type = {
+ .name = "type",
+ .mode = KFD_SYSFS_FILE_MODE
+};
+
+static struct attribute attr_queue_gpuid = {
+ .name = "gpuid",
+ .mode = KFD_SYSFS_FILE_MODE
+};
+
+static struct attribute *procfs_queue_attrs[] = {
+ &attr_queue_size,
+ &attr_queue_type,
+ &attr_queue_gpuid,
+ NULL
+};
+
+static const struct sysfs_ops procfs_queue_ops = {
+ .show = kfd_procfs_queue_show,
+};
+
+static struct kobj_type procfs_queue_type = {
+ .sysfs_ops = &procfs_queue_ops,
+ .default_attrs = procfs_queue_attrs,
+};
+
+static const struct sysfs_ops procfs_stats_ops = {
+ .show = kfd_procfs_stats_show,
+};
+
+static struct kobj_type procfs_stats_type = {
+ .sysfs_ops = &procfs_stats_ops,
+ .release = kfd_procfs_kobj_release,
+};
+
+int kfd_procfs_add_queue(struct queue *q)
+{
+ struct kfd_process *proc;
+ int ret;
+
+ if (!q || !q->process)
+ return -EINVAL;
+ proc = q->process;
+
+ /* Create proc/<pid>/queues/<queue id> folder */
+ if (!proc->kobj_queues)
+ return -EFAULT;
+ ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
+ proc->kobj_queues, "%u", q->properties.queue_id);
+ if (ret < 0) {
+ pr_warn("Creating proc/<pid>/queues/%u failed",
+ q->properties.queue_id);
+ kobject_put(&q->kobj);
+ return ret;
+ }
+
+ return 0;
+}
+
+static int kfd_sysfs_create_file(struct kfd_process *p, struct attribute *attr,
+ char *name)
+{
+ int ret = 0;
+
+ if (!p || !attr || !name)
+ return -EINVAL;
+
+ attr->name = name;
+ attr->mode = KFD_SYSFS_FILE_MODE;
+ sysfs_attr_init(attr);
+
+ ret = sysfs_create_file(p->kobj, attr);
+
+ return ret;
+}
+
+static int kfd_procfs_add_sysfs_stats(struct kfd_process *p)
+{
+ int ret = 0;
+ struct kfd_process_device *pdd;
+ char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
+
+ if (!p)
+ return -EINVAL;
+
+ if (!p->kobj)
+ return -EFAULT;
+
+ /*
+ * Create sysfs files for each GPU:
+ * - proc/<pid>/stats_<gpuid>/
+ * - proc/<pid>/stats_<gpuid>/evicted_ms
+ * - proc/<pid>/stats_<gpuid>/cu_occupancy
+ */
+ list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+ struct kobject *kobj_stats;
+
+ snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
+ "stats_%u", pdd->dev->id);
+ kobj_stats = kfd_alloc_struct(kobj_stats);
+ if (!kobj_stats)
+ return -ENOMEM;
+
+ ret = kobject_init_and_add(kobj_stats,
+ &procfs_stats_type,
+ p->kobj,
+ stats_dir_filename);
+
+ if (ret) {
+ pr_warn("Creating KFD proc/stats_%s folder failed",
+ stats_dir_filename);
+ kobject_put(kobj_stats);
+ goto err;
+ }
+
+ pdd->kobj_stats = kobj_stats;
+ pdd->attr_evict.name = "evicted_ms";
+ pdd->attr_evict.mode = KFD_SYSFS_FILE_MODE;
+ sysfs_attr_init(&pdd->attr_evict);
+ ret = sysfs_create_file(kobj_stats, &pdd->attr_evict);
+ if (ret)
+ pr_warn("Creating eviction stats for gpuid %d failed",
+ (int)pdd->dev->id);
+
+ /* Add sysfs file to report compute unit occupancy */
+ if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) {
+ pdd->attr_cu_occupancy.name = "cu_occupancy";
+ pdd->attr_cu_occupancy.mode = KFD_SYSFS_FILE_MODE;
+ sysfs_attr_init(&pdd->attr_cu_occupancy);
+ ret = sysfs_create_file(kobj_stats,
+ &pdd->attr_cu_occupancy);
+ if (ret)
+ pr_warn("Creating %s failed for gpuid: %d",
+ pdd->attr_cu_occupancy.name,
+ (int)pdd->dev->id);
+ }
+ }
+err:
+ return ret;
+}
+
+
+static int kfd_procfs_add_sysfs_files(struct kfd_process *p)
+{
+ int ret = 0;
+ struct kfd_process_device *pdd;
+
+ if (!p)
+ return -EINVAL;
+
+ if (!p->kobj)
+ return -EFAULT;
+
+ /*
+ * Create sysfs files for each GPU:
+ * - proc/<pid>/vram_<gpuid>
+ * - proc/<pid>/sdma_<gpuid>
+ */
+ list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+ snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
+ pdd->dev->id);
+ ret = kfd_sysfs_create_file(p, &pdd->attr_vram, pdd->vram_filename);
+ if (ret)
+ pr_warn("Creating vram usage for gpu id %d failed",
+ (int)pdd->dev->id);
+
+ snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
+ pdd->dev->id);
+ ret = kfd_sysfs_create_file(p, &pdd->attr_sdma, pdd->sdma_filename);
+ if (ret)
+ pr_warn("Creating sdma usage for gpu id %d failed",
+ (int)pdd->dev->id);
+ }
+
+ return ret;
+}
+
+void kfd_procfs_del_queue(struct queue *q)
+{
+ if (!q)
+ return;
+
+ kobject_del(&q->kobj);
+ kobject_put(&q->kobj);
+}
int kfd_process_create_wq(void)
{
@@ -100,8 +640,8 @@
{
struct kfd_dev *dev = pdd->dev;
- dev->kfd2kgd->unmap_memory_to_gpu(dev->kgd, mem, pdd->vm);
- dev->kfd2kgd->free_memory_of_gpu(dev->kgd, mem);
+ amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->vm);
+ amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, NULL);
}
/* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
@@ -119,16 +659,16 @@
int handle;
int err;
- err = kdev->kfd2kgd->alloc_memory_of_gpu(kdev->kgd, gpu_va, size,
+ err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size,
pdd->vm, &mem, NULL, flags);
if (err)
goto err_alloc_mem;
- err = kdev->kfd2kgd->map_memory_to_gpu(kdev->kgd, mem, pdd->vm);
+ err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, pdd->vm);
if (err)
goto err_map_mem;
- err = kdev->kfd2kgd->sync_memory(kdev->kgd, mem, true);
+ err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->kgd, mem, true);
if (err) {
pr_debug("Sync memory failed, wait interrupted by user signal\n");
goto sync_memory_failed;
@@ -147,7 +687,7 @@
}
if (kptr) {
- err = kdev->kfd2kgd->map_gtt_bo_to_kernel(kdev->kgd,
+ err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->kgd,
(struct kgd_mem *)mem, kptr, NULL);
if (err) {
pr_debug("Map GTT BO to kernel failed\n");
@@ -165,7 +705,7 @@
return err;
err_map_mem:
- kdev->kfd2kgd->free_memory_of_gpu(kdev->kgd, mem);
+ amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, NULL);
err_alloc_mem:
*kptr = NULL;
return err;
@@ -180,10 +720,10 @@
static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
{
struct qcm_process_device *qpd = &pdd->qpd;
- uint32_t flags = ALLOC_MEM_FLAGS_GTT |
- ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
- ALLOC_MEM_FLAGS_WRITABLE |
- ALLOC_MEM_FLAGS_EXECUTABLE;
+ uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
+ KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
+ KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
+ KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
void *kaddr;
int ret;
@@ -205,6 +745,7 @@
{
struct kfd_process *process;
struct task_struct *thread = current;
+ int ret;
if (!thread->mm)
return ERR_PTR(-EINVAL);
@@ -222,11 +763,62 @@
/* A prior open of /dev/kfd could have already created the process. */
process = find_process(thread);
- if (process)
+ if (process) {
pr_debug("Process already found\n");
- else
- process = create_process(thread, filep);
+ } else {
+ process = create_process(thread);
+ if (IS_ERR(process))
+ goto out;
+ ret = kfd_process_init_cwsr_apu(process, filep);
+ if (ret) {
+ process = ERR_PTR(ret);
+ goto out;
+ }
+
+ if (!procfs.kobj)
+ goto out;
+
+ process->kobj = kfd_alloc_struct(process->kobj);
+ if (!process->kobj) {
+ pr_warn("Creating procfs kobject failed");
+ goto out;
+ }
+ ret = kobject_init_and_add(process->kobj, &procfs_type,
+ procfs.kobj, "%d",
+ (int)process->lead_thread->pid);
+ if (ret) {
+ pr_warn("Creating procfs pid directory failed");
+ kobject_put(process->kobj);
+ goto out;
+ }
+
+ process->attr_pasid.name = "pasid";
+ process->attr_pasid.mode = KFD_SYSFS_FILE_MODE;
+ sysfs_attr_init(&process->attr_pasid);
+ ret = sysfs_create_file(process->kobj, &process->attr_pasid);
+ if (ret)
+ pr_warn("Creating pasid for pid %d failed",
+ (int)process->lead_thread->pid);
+
+ process->kobj_queues = kobject_create_and_add("queues",
+ process->kobj);
+ if (!process->kobj_queues)
+ pr_warn("Creating KFD proc/queues folder failed");
+
+ ret = kfd_procfs_add_sysfs_stats(process);
+ if (ret)
+ pr_warn("Creating sysfs stats dir for pid %d failed",
+ (int)process->lead_thread->pid);
+
+ ret = kfd_procfs_add_sysfs_files(process);
+ if (ret)
+ pr_warn("Creating sysfs usage file for pid %d failed",
+ (int)process->lead_thread->pid);
+ }
+out:
+ if (!IS_ERR(process))
+ kref_get(&process->ref);
mutex_unlock(&kfd_processes_mutex);
return process;
@@ -296,11 +888,11 @@
per_device_list) {
if (!peer_pdd->vm)
continue;
- peer_pdd->dev->kfd2kgd->unmap_memory_to_gpu(
+ amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
peer_pdd->dev->kgd, mem, peer_pdd->vm);
}
- pdd->dev->kfd2kgd->free_memory_of_gpu(pdd->dev->kgd, mem);
+ amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem, NULL);
kfd_process_device_remove_obj_handle(pdd, id);
}
}
@@ -319,13 +911,16 @@
list_for_each_entry_safe(pdd, temp, &p->per_device_data,
per_device_list) {
- pr_debug("Releasing pdd (topology id %d) for process (pasid %d)\n",
+ pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n",
pdd->dev->id, p->pasid);
- if (pdd->drm_file)
+ if (pdd->drm_file) {
+ amdgpu_amdkfd_gpuvm_release_process_vm(
+ pdd->dev->kgd, pdd->vm);
fput(pdd->drm_file);
+ }
else if (pdd->vm)
- pdd->dev->kfd2kgd->destroy_process_vm(
+ amdgpu_amdkfd_gpuvm_destroy_process_vm(
pdd->dev->kgd, pdd->vm);
list_del(&pdd->per_device_list);
@@ -336,6 +931,18 @@
kfree(pdd->qpd.doorbell_bitmap);
idr_destroy(&pdd->alloc_idr);
+
+ kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index);
+
+ /*
+ * before destroying pdd, make sure to report availability
+ * for auto suspend
+ */
+ if (pdd->runtime_inuse) {
+ pm_runtime_mark_last_busy(pdd->dev->ddev->dev);
+ pm_runtime_put_autosuspend(pdd->dev->ddev->dev);
+ pdd->runtime_inuse = false;
+ }
kfree(pdd);
}
@@ -350,6 +957,32 @@
{
struct kfd_process *p = container_of(work, struct kfd_process,
release_work);
+ struct kfd_process_device *pdd;
+
+ /* Remove the procfs files */
+ if (p->kobj) {
+ sysfs_remove_file(p->kobj, &p->attr_pasid);
+ kobject_del(p->kobj_queues);
+ kobject_put(p->kobj_queues);
+ p->kobj_queues = NULL;
+
+ list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+ sysfs_remove_file(p->kobj, &pdd->attr_vram);
+ sysfs_remove_file(p->kobj, &pdd->attr_sdma);
+
+ sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);
+ if (pdd->dev->kfd2kgd->get_cu_occupancy)
+ sysfs_remove_file(pdd->kobj_stats,
+ &pdd->attr_cu_occupancy);
+ kobject_del(pdd->kobj_stats);
+ kobject_put(pdd->kobj_stats);
+ pdd->kobj_stats = NULL;
+ }
+
+ kobject_del(p->kobj);
+ kobject_put(p->kobj);
+ p->kobj = NULL;
+ }
kfd_iommu_unbind_process(p);
@@ -361,8 +994,6 @@
kfd_event_free_process(p);
kfd_pasid_free(p->pasid);
- kfd_free_process_doorbells(p);
-
mutex_destroy(&p->mutex);
put_task_struct(p->lead_thread);
@@ -378,11 +1009,9 @@
queue_work(kfd_process_wq, &p->release_work);
}
-static void kfd_process_destroy_delayed(struct rcu_head *rcu)
+static void kfd_process_free_notifier(struct mmu_notifier *mn)
{
- struct kfd_process *p = container_of(rcu, struct kfd_process, rcu);
-
- kfd_unref_process(p);
+ kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
}
static void kfd_process_notifier_release(struct mmu_notifier *mn,
@@ -431,15 +1060,20 @@
/* Indicate to other users that MM is no longer valid */
p->mm = NULL;
+ /* Signal the eviction fence after user mode queues are
+ * destroyed. This allows any BOs to be freed without
+ * triggering pointless evictions or waiting for fences.
+ */
+ dma_fence_signal(p->ef);
mutex_unlock(&p->mutex);
- mmu_notifier_unregister_no_release(&p->mmu_notifier, mm);
- mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
+ mmu_notifier_put(&p->mmu_notifier);
}
static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
.release = kfd_process_notifier_release,
+ .free_notifier = kfd_process_free_notifier,
};
static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
@@ -454,8 +1088,7 @@
if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
continue;
- offset = (KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id))
- << PAGE_SHIFT;
+ offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
MAP_SHARED, offset);
@@ -483,8 +1116,9 @@
{
struct kfd_dev *dev = pdd->dev;
struct qcm_process_device *qpd = &pdd->qpd;
- uint32_t flags = ALLOC_MEM_FLAGS_GTT |
- ALLOC_MEM_FLAGS_NO_SUBSTITUTE | ALLOC_MEM_FLAGS_EXECUTABLE;
+ uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
+ | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
+ | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
void *kaddr;
int ret;
@@ -509,81 +1143,64 @@
return 0;
}
-static struct kfd_process *create_process(const struct task_struct *thread,
- struct file *filep)
+/*
+ * On return the kfd_process is fully operational and will be freed when the
+ * mm is released
+ */
+static struct kfd_process *create_process(const struct task_struct *thread)
{
struct kfd_process *process;
int err = -ENOMEM;
process = kzalloc(sizeof(*process), GFP_KERNEL);
-
if (!process)
goto err_alloc_process;
+
+ kref_init(&process->ref);
+ mutex_init(&process->mutex);
+ process->mm = thread->mm;
+ process->lead_thread = thread->group_leader;
+ INIT_LIST_HEAD(&process->per_device_data);
+ INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
+ INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
+ process->last_restore_timestamp = get_jiffies_64();
+ kfd_event_init_process(process);
+ process->is_32bit_user_mode = in_compat_syscall();
process->pasid = kfd_pasid_alloc();
if (process->pasid == 0)
goto err_alloc_pasid;
-
- if (kfd_alloc_process_doorbells(process) < 0)
- goto err_alloc_doorbells;
-
- kref_init(&process->ref);
-
- mutex_init(&process->mutex);
-
- process->mm = thread->mm;
-
- /* register notifier */
- process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
- err = mmu_notifier_register(&process->mmu_notifier, process->mm);
- if (err)
- goto err_mmu_notifier;
-
- hash_add_rcu(kfd_processes_table, &process->kfd_processes,
- (uintptr_t)process->mm);
-
- process->lead_thread = thread->group_leader;
- get_task_struct(process->lead_thread);
-
- INIT_LIST_HEAD(&process->per_device_data);
-
- kfd_event_init_process(process);
err = pqm_init(&process->pqm, process);
if (err != 0)
goto err_process_pqm_init;
/* init process apertures*/
- process->is_32bit_user_mode = in_compat_syscall();
err = kfd_init_apertures(process);
if (err != 0)
goto err_init_apertures;
- INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
- INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
- process->last_restore_timestamp = get_jiffies_64();
-
- err = kfd_process_init_cwsr_apu(process, filep);
+ /* Must be last, have to use release destruction after this */
+ process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
+ err = mmu_notifier_register(&process->mmu_notifier, process->mm);
if (err)
- goto err_init_cwsr;
+ goto err_register_notifier;
+
+ get_task_struct(process->lead_thread);
+ hash_add_rcu(kfd_processes_table, &process->kfd_processes,
+ (uintptr_t)process->mm);
return process;
-err_init_cwsr:
+err_register_notifier:
kfd_process_free_outstanding_kfd_bos(process);
kfd_process_destroy_pdds(process);
err_init_apertures:
pqm_uninit(&process->pqm);
err_process_pqm_init:
- hash_del_rcu(&process->kfd_processes);
- synchronize_rcu();
- mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
-err_mmu_notifier:
- mutex_destroy(&process->mutex);
- kfd_free_process_doorbells(process);
-err_alloc_doorbells:
kfd_pasid_free(process->pasid);
err_alloc_pasid:
+ mutex_destroy(&process->mutex);
kfree(process);
err_alloc_process:
return ERR_PTR(err);
@@ -593,6 +1210,8 @@
struct kfd_dev *dev)
{
unsigned int i;
+ int range_start = dev->shared_resources.non_cp_doorbells_start;
+ int range_end = dev->shared_resources.non_cp_doorbells_end;
if (!KFD_IS_SOC15(dev->device_info->asic_family))
return 0;
@@ -603,13 +1222,19 @@
if (!qpd->doorbell_bitmap)
return -ENOMEM;
- /* Mask out any reserved doorbells */
- for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS; i++)
- if ((dev->shared_resources.reserved_doorbell_mask & i) ==
- dev->shared_resources.reserved_doorbell_val) {
+ /* Mask out doorbells reserved for SDMA, IH, and VCN on SOC15. */
+ pr_debug("reserved doorbell 0x%03x - 0x%03x\n", range_start, range_end);
+ pr_debug("reserved doorbell 0x%03x - 0x%03x\n",
+ range_start + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
+ range_end + KFD_QUEUE_DOORBELL_MIRROR_OFFSET);
+
+ for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS / 2; i++) {
+ if (i >= range_start && i <= range_end) {
set_bit(i, qpd->doorbell_bitmap);
- pr_debug("reserved doorbell 0x%03x\n", i);
+ set_bit(i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
+ qpd->doorbell_bitmap);
}
+ }
return 0;
}
@@ -635,10 +1260,14 @@
if (!pdd)
return NULL;
+ if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {
+ pr_err("Failed to alloc doorbell for pdd\n");
+ goto err_free_pdd;
+ }
+
if (init_doorbell_bitmap(&pdd->qpd, dev)) {
pr_err("Failed to init doorbell for process\n");
- kfree(pdd);
- return NULL;
+ goto err_free_pdd;
}
pdd->dev = dev;
@@ -647,15 +1276,24 @@
pdd->qpd.dqm = dev->dqm;
pdd->qpd.pqm = &p->pqm;
pdd->qpd.evicted = 0;
+ pdd->qpd.mapped_gws_queue = false;
pdd->process = p;
pdd->bound = PDD_UNBOUND;
pdd->already_dequeued = false;
+ pdd->runtime_inuse = false;
+ pdd->vram_usage = 0;
+ pdd->sdma_past_activity_counter = 0;
+ atomic64_set(&pdd->evict_duration_counter, 0);
list_add(&pdd->per_device_list, &p->per_device_data);
/* Init idr used for memory handle translation */
idr_init(&pdd->alloc_idr);
return pdd;
+
+err_free_pdd:
+ kfree(pdd);
+ return NULL;
}
/**
@@ -686,16 +1324,18 @@
dev = pdd->dev;
if (drm_file)
- ret = dev->kfd2kgd->acquire_process_vm(
- dev->kgd, drm_file,
+ ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(
+ dev->kgd, drm_file, p->pasid,
&pdd->vm, &p->kgd_process_info, &p->ef);
else
- ret = dev->kfd2kgd->create_process_vm(
- dev->kgd, &pdd->vm, &p->kgd_process_info, &p->ef);
+ ret = amdgpu_amdkfd_gpuvm_create_process_vm(dev->kgd, p->pasid,
+ &pdd->vm, &p->kgd_process_info, &p->ef);
if (ret) {
pr_err("Failed to create process VM object\n");
return ret;
}
+
+ amdgpu_vm_set_task_info(pdd->vm);
ret = kfd_process_device_reserve_ib_mem(pdd);
if (ret)
@@ -712,7 +1352,7 @@
err_reserve_ib_mem:
kfd_process_device_free_bos(pdd);
if (!drm_file)
- dev->kfd2kgd->destroy_process_vm(dev->kgd, pdd->vm);
+ amdgpu_amdkfd_gpuvm_destroy_process_vm(dev->kgd, pdd->vm);
pdd->vm = NULL;
return ret;
@@ -737,15 +1377,43 @@
return ERR_PTR(-ENOMEM);
}
+ /*
+ * signal runtime-pm system to auto resume and prevent
+ * further runtime suspend once device pdd is created until
+ * pdd is destroyed.
+ */
+ if (!pdd->runtime_inuse) {
+ err = pm_runtime_get_sync(dev->ddev->dev);
+ if (err < 0) {
+ pm_runtime_put_autosuspend(dev->ddev->dev);
+ return ERR_PTR(err);
+ }
+ }
+
err = kfd_iommu_bind_process_to_device(pdd);
if (err)
- return ERR_PTR(err);
+ goto out;
err = kfd_process_device_init_vm(pdd, NULL);
if (err)
- return ERR_PTR(err);
+ goto out;
+
+ /*
+ * make sure that runtime_usage counter is incremented just once
+ * per pdd
+ */
+ pdd->runtime_inuse = true;
return pdd;
+
+out:
+ /* balance runpm reference count and exit with error */
+ if (!pdd->runtime_inuse) {
+ pm_runtime_mark_last_busy(dev->ddev->dev);
+ pm_runtime_put_autosuspend(dev->ddev->dev);
+ }
+
+ return ERR_PTR(err);
}
struct kfd_process_device *kfd_get_first_process_device_data(
@@ -802,7 +1470,7 @@
}
/* This increments the process->ref counter. */
-struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
+struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)
{
struct kfd_process *p, *ret_p = NULL;
unsigned int temp;
@@ -838,7 +1506,7 @@
return p;
}
-/* process_evict_queues - Evict all user queues of a process
+/* kfd_process_evict_queues - Evict all user queues of a process
*
* Eviction is reference-counted per process-device. This means multiple
* evictions from different sources can be nested safely.
@@ -878,7 +1546,7 @@
return r;
}
-/* process_restore_queues - Restore all user queues of a process */
+/* kfd_process_restore_queues - Restore all user queues of a process */
int kfd_process_restore_queues(struct kfd_process *p)
{
struct kfd_process_device *pdd;
@@ -920,7 +1588,7 @@
*/
flush_delayed_work(&p->restore_work);
- pr_debug("Started evicting pasid %d\n", p->pasid);
+ pr_debug("Started evicting pasid 0x%x\n", p->pasid);
ret = kfd_process_evict_queues(p);
if (!ret) {
dma_fence_signal(p->ef);
@@ -929,16 +1597,15 @@
queue_delayed_work(kfd_restore_wq, &p->restore_work,
msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
- pr_debug("Finished evicting pasid %d\n", p->pasid);
+ pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
} else
- pr_err("Failed to evict queues of pasid %d\n", p->pasid);
+ pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid);
}
static void restore_process_worker(struct work_struct *work)
{
struct delayed_work *dwork;
struct kfd_process *p;
- struct kfd_process_device *pdd;
int ret = 0;
dwork = to_delayed_work(work);
@@ -947,17 +1614,7 @@
* lifetime of this thread, kfd_process p will be valid
*/
p = container_of(dwork, struct kfd_process, restore_work);
-
- /* Call restore_process_bos on the first KGD device. This function
- * takes care of restoring the whole process including other devices.
- * Restore can fail if enough memory is not available. If so,
- * reschedule again.
- */
- pdd = list_first_entry(&p->per_device_data,
- struct kfd_process_device,
- per_device_list);
-
- pr_debug("Started restoring pasid %d\n", p->pasid);
+ pr_debug("Started restoring pasid 0x%x\n", p->pasid);
/* Setting last_restore_timestamp before successful restoration.
* Otherwise this would have to be set by KGD (restore_process_bos)
@@ -970,10 +1627,10 @@
*/
p->last_restore_timestamp = get_jiffies_64();
- ret = pdd->dev->kfd2kgd->restore_process_bos(p->kgd_process_info,
+ ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info,
&p->ef);
if (ret) {
- pr_debug("Failed to restore BOs of pasid %d, retry after %d ms\n",
+ pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",
p->pasid, PROCESS_BACK_OFF_TIME_MS);
ret = queue_delayed_work(kfd_restore_wq, &p->restore_work,
msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
@@ -983,9 +1640,9 @@
ret = kfd_process_restore_queues(p);
if (!ret)
- pr_debug("Finished restoring pasid %d\n", p->pasid);
+ pr_debug("Finished restoring pasid 0x%x\n", p->pasid);
else
- pr_err("Failed to restore queues of pasid %d\n", p->pasid);
+ pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid);
}
void kfd_suspend_all_processes(void)
@@ -994,12 +1651,13 @@
unsigned int temp;
int idx = srcu_read_lock(&kfd_processes_srcu);
+ WARN(debug_evictions, "Evicting all processes");
hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
cancel_delayed_work_sync(&p->eviction_work);
cancel_delayed_work_sync(&p->restore_work);
if (kfd_process_evict_queues(p))
- pr_err("Failed to suspend process %d\n", p->pasid);
+ pr_err("Failed to suspend process 0x%x\n", p->pasid);
dma_fence_signal(p->ef);
dma_fence_put(p->ef);
p->ef = NULL;
@@ -1058,16 +1716,17 @@
void kfd_flush_tlb(struct kfd_process_device *pdd)
{
struct kfd_dev *dev = pdd->dev;
- const struct kfd2kgd_calls *f2g = dev->kfd2kgd;
if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
/* Nothing to flush until a VMID is assigned, which
* only happens when the first queue is created.
*/
if (pdd->qpd.vmid)
- f2g->invalidate_tlbs_vmid(dev->kgd, pdd->qpd.vmid);
+ amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->kgd,
+ pdd->qpd.vmid);
} else {
- f2g->invalidate_tlbs(dev->kgd, pdd->process->pasid);
+ amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->kgd,
+ pdd->process->pasid);
}
}
@@ -1082,7 +1741,7 @@
int idx = srcu_read_lock(&kfd_processes_srcu);
hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
- seq_printf(m, "Process %d PASID %d:\n",
+ seq_printf(m, "Process %d PASID 0x%x:\n",
p->lead_thread->tgid, p->pasid);
mutex_lock(&p->mutex);
@@ -1099,3 +1758,4 @@
}
#endif
+
--
Gitblit v1.6.2