From 102a0743326a03cd1a1202ceda21e175b7d3575c Mon Sep 17 00:00:00 2001
From: hc <hc@nodka.com>
Date: Tue, 20 Feb 2024 01:20:52 +0000
Subject: [PATCH] add new system file
---
kernel/drivers/rknpu/rknpu_drv.c | 917 +++++++++-----------------------------------------------
1 files changed, 153 insertions(+), 764 deletions(-)
diff --git a/kernel/drivers/rknpu/rknpu_drv.c b/kernel/drivers/rknpu/rknpu_drv.c
index eb31273..1df364f 100644
--- a/kernel/drivers/rknpu/rknpu_drv.c
+++ b/kernel/drivers/rknpu/rknpu_drv.c
@@ -31,19 +31,11 @@
#include <linux/clk-provider.h>
#include <linux/pm_domain.h>
#include <linux/pm_runtime.h>
-#include <linux/devfreq_cooling.h>
#include <linux/regmap.h>
-#include <linux/dma-iommu.h>
#include <linux/of_address.h>
#ifndef FPGA_PLATFORM
#include <soc/rockchip/rockchip_iommu.h>
-#include <soc/rockchip/rockchip_opp_select.h>
-#include <soc/rockchip/rockchip_system_monitor.h>
-#include <soc/rockchip/rockchip_ipa.h>
-#ifdef CONFIG_PM_DEVFREQ
-#include <../drivers/devfreq/governor.h>
-#endif
#endif
#include "rknpu_ioctl.h"
@@ -51,6 +43,7 @@
#include "rknpu_fence.h"
#include "rknpu_drv.h"
#include "rknpu_gem.h"
+#include "rknpu_devfreq.h"
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
#include <drm/drm_device.h>
@@ -118,7 +111,9 @@
.num_irqs = ARRAY_SIZE(rknpu_irqs),
.num_resets = ARRAY_SIZE(rknpu_resets),
.nbuf_phyaddr = 0,
- .nbuf_size = 0
+ .nbuf_size = 0,
+ .max_submit_number = (1 << 12) - 1,
+ .core_mask = 0x1,
};
static const struct rknpu_config rk3588_rknpu_config = {
@@ -136,7 +131,29 @@
.num_irqs = ARRAY_SIZE(rk3588_npu_irqs),
.num_resets = ARRAY_SIZE(rk3588_npu_resets),
.nbuf_phyaddr = 0,
- .nbuf_size = 0
+ .nbuf_size = 0,
+ .max_submit_number = (1 << 12) - 1,
+ .core_mask = 0x7,
+};
+
+static const struct rknpu_config rk3583_rknpu_config = {
+ .bw_priority_addr = 0x0,
+ .bw_priority_length = 0x0,
+ .dma_mask = DMA_BIT_MASK(40),
+ .pc_data_amount_scale = 2,
+ .pc_task_number_bits = 12,
+ .pc_task_number_mask = 0xfff,
+ .pc_task_status_offset = 0x3c,
+ .pc_dma_ctrl = 0,
+ .bw_enable = 0,
+ .irqs = rk3588_npu_irqs,
+ .resets = rk3588_npu_resets,
+ .num_irqs = 2,
+ .num_resets = 2,
+ .nbuf_phyaddr = 0,
+ .nbuf_size = 0,
+ .max_submit_number = (1 << 12) - 1,
+ .core_mask = 0x3,
};
static const struct rknpu_config rv1106_rknpu_config = {
@@ -154,7 +171,9 @@
.num_irqs = ARRAY_SIZE(rknpu_irqs),
.num_resets = ARRAY_SIZE(rknpu_resets),
.nbuf_phyaddr = 0,
- .nbuf_size = 0
+ .nbuf_size = 0,
+ .max_submit_number = (1 << 16) - 1,
+ .core_mask = 0x1,
};
static const struct rknpu_config rk3562_rknpu_config = {
@@ -172,7 +191,9 @@
.num_irqs = ARRAY_SIZE(rknpu_irqs),
.num_resets = ARRAY_SIZE(rknpu_resets),
.nbuf_phyaddr = 0xfe400000,
- .nbuf_size = 256 * 1024
+ .nbuf_size = 256 * 1024,
+ .max_submit_number = (1 << 16) - 1,
+ .core_mask = 0x1,
};
/* driver probe and init */
@@ -225,7 +246,6 @@
{
int ret = 0;
- cancel_delayed_work(&rknpu_dev->power_off_work);
mutex_lock(&rknpu_dev->power_lock);
if (atomic_inc_return(&rknpu_dev->power_refcount) == 1)
ret = rknpu_power_on(rknpu_dev);
@@ -248,6 +268,9 @@
static int rknpu_power_put_delay(struct rknpu_device *rknpu_dev)
{
+ if (rknpu_dev->power_put_delay == 0)
+ return rknpu_power_put(rknpu_dev);
+
mutex_lock(&rknpu_dev->power_lock);
if (atomic_read(&rknpu_dev->power_refcount) == 1)
queue_delayed_work(
@@ -256,6 +279,7 @@
else
atomic_dec_if_positive(&rknpu_dev->power_refcount);
mutex_unlock(&rknpu_dev->power_lock);
+
return 0;
}
@@ -489,11 +513,13 @@
#endif
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
+#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
static const struct vm_operations_struct rknpu_gem_vm_ops = {
.fault = rknpu_gem_fault,
.open = drm_gem_vm_open,
.close = drm_gem_vm_close,
};
+#endif
static int rknpu_action_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv)
@@ -535,6 +561,9 @@
DRM_RENDER_ALLOW),
};
+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
+DEFINE_DRM_GEM_FOPS(rknpu_drm_driver_fops);
+#else
static const struct file_operations rknpu_drm_driver_fops = {
.owner = THIS_MODULE,
.open = drm_open,
@@ -548,6 +577,7 @@
.release = drm_release,
.llseek = noop_llseek,
};
+#endif
static struct drm_driver rknpu_drm_driver = {
#if KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE
@@ -555,28 +585,34 @@
#else
.driver_features = DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER,
#endif
+#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
.gem_free_object_unlocked = rknpu_gem_free_object,
.gem_vm_ops = &rknpu_gem_vm_ops,
+ .dumb_destroy = drm_gem_dumb_destroy,
+ .gem_prime_export = drm_gem_prime_export,
+ .gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table,
+ .gem_prime_vmap = rknpu_gem_prime_vmap,
+ .gem_prime_vunmap = rknpu_gem_prime_vunmap,
+#endif
.dumb_create = rknpu_gem_dumb_create,
#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
.dumb_map_offset = rknpu_gem_dumb_map_offset,
#else
.dumb_map_offset = drm_gem_dumb_map_offset,
#endif
- .dumb_destroy = drm_gem_dumb_destroy,
.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
- .gem_prime_export = drm_gem_prime_export,
#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
.gem_prime_import = rknpu_gem_prime_import,
#else
.gem_prime_import = drm_gem_prime_import,
#endif
- .gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table,
.gem_prime_import_sg_table = rknpu_gem_prime_import_sg_table,
- .gem_prime_vmap = rknpu_gem_prime_vmap,
- .gem_prime_vunmap = rknpu_gem_prime_vunmap,
+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
+ .gem_prime_mmap = drm_gem_prime_mmap,
+#else
.gem_prime_mmap = rknpu_gem_prime_mmap,
+#endif
.ioctls = rknpu_ioctls,
.num_ioctls = ARRAY_SIZE(rknpu_ioctls),
.fops = &rknpu_drm_driver_fops,
@@ -596,7 +632,7 @@
container_of(timer, struct rknpu_device, timer);
struct rknpu_subcore_data *subcore_data = NULL;
struct rknpu_job *job = NULL;
- ktime_t now = ktime_get();
+ ktime_t now;
unsigned long flags;
int i;
@@ -607,14 +643,16 @@
job = subcore_data->job;
if (job) {
+ now = ktime_get();
subcore_data->timer.busy_time +=
- ktime_us_delta(now, job->hw_recoder_time);
- job->hw_recoder_time = ktime_get();
+ ktime_sub(now, job->hw_recoder_time);
+ job->hw_recoder_time = now;
}
- subcore_data->timer.busy_time_record =
+ subcore_data->timer.total_busy_time =
subcore_data->timer.busy_time;
subcore_data->timer.busy_time = 0;
+
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
}
@@ -661,6 +699,42 @@
}
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
+static int drm_fake_dev_register(struct rknpu_device *rknpu_dev)
+{
+ const struct platform_device_info rknpu_dev_info = {
+ .name = "rknpu_dev",
+ .id = PLATFORM_DEVID_AUTO,
+ .dma_mask = rknpu_dev->config->dma_mask,
+ };
+ struct platform_device *pdev = NULL;
+ int ret = -EINVAL;
+
+ pdev = platform_device_register_full(&rknpu_dev_info);
+ if (pdev) {
+ ret = of_dma_configure(&pdev->dev, NULL, true);
+ if (ret) {
+ platform_device_unregister(pdev);
+ pdev = NULL;
+ }
+ }
+
+ rknpu_dev->fake_dev = pdev ? &pdev->dev : NULL;
+
+ return ret;
+}
+
+static void drm_fake_dev_unregister(struct rknpu_device *rknpu_dev)
+{
+ struct platform_device *pdev = NULL;
+
+ if (!rknpu_dev->fake_dev)
+ return;
+
+ pdev = to_platform_device(rknpu_dev->fake_dev);
+
+ platform_device_unregister(pdev);
+}
+
static int rknpu_drm_probe(struct rknpu_device *rknpu_dev)
{
struct device *dev = rknpu_dev->dev;
@@ -679,6 +753,8 @@
drm_dev->dev_private = rknpu_dev;
rknpu_dev->drm_dev = drm_dev;
+ drm_fake_dev_register(rknpu_dev);
+
return 0;
err_free_drm:
@@ -694,6 +770,8 @@
static void rknpu_drm_remove(struct rknpu_device *rknpu_dev)
{
struct drm_device *drm_dev = rknpu_dev->drm_dev;
+
+ drm_fake_dev_unregister(rknpu_dev);
drm_dev_unregister(drm_dev);
@@ -742,9 +820,7 @@
}
#ifndef FPGA_PLATFORM
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
- rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
-#endif
+ rknpu_devfreq_lock(rknpu_dev);
#endif
if (rknpu_dev->multiple_domains) {
@@ -803,9 +879,7 @@
out:
#ifndef FPGA_PLATFORM
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
- rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
-#endif
+ rknpu_devfreq_unlock(rknpu_dev);
#endif
return ret;
@@ -819,9 +893,7 @@
int ret;
bool val;
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
- rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
-#endif
+ rknpu_devfreq_lock(rknpu_dev);
#endif
pm_runtime_put_sync(dev);
@@ -843,10 +915,7 @@
if (ret) {
LOG_DEV_ERROR(dev, "iommu still enabled\n");
pm_runtime_get_sync(dev);
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
- rockchip_monitor_volt_adjust_unlock(
- rknpu_dev->mdev_info);
-#endif
+ rknpu_devfreq_unlock(rknpu_dev);
return ret;
}
#else
@@ -862,9 +931,7 @@
}
#ifndef FPGA_PLATFORM
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
- rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
-#endif
+ rknpu_devfreq_unlock(rknpu_dev);
#endif
clk_bulk_disable_unprepare(rknpu_dev->num_clks, rknpu_dev->clks);
@@ -879,686 +946,6 @@
return 0;
}
-
-#ifndef FPGA_PLATFORM
-static struct monitor_dev_profile npu_mdevp = {
- .type = MONITOR_TYPE_DEV,
- .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
- .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
- .update_volt = rockchip_monitor_check_rate_volt,
-#endif
-};
-
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
-static int npu_opp_helper(struct dev_pm_set_opp_data *data)
-{
- struct device *dev = data->dev;
- struct dev_pm_opp_supply *old_supply_vdd = &data->old_opp.supplies[0];
- struct dev_pm_opp_supply *old_supply_mem = &data->old_opp.supplies[1];
- struct dev_pm_opp_supply *new_supply_vdd = &data->new_opp.supplies[0];
- struct dev_pm_opp_supply *new_supply_mem = &data->new_opp.supplies[1];
- struct regulator *vdd_reg = data->regulators[0];
- struct regulator *mem_reg = data->regulators[1];
- struct clk *clk = data->clk;
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
- struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
- unsigned long old_freq = data->old_opp.rate;
- unsigned long new_freq = data->new_opp.rate;
- bool is_set_rm = true;
- bool is_set_clk = true;
- u32 target_rm = UINT_MAX;
- int ret = 0;
-
- if (!pm_runtime_active(dev)) {
- is_set_rm = false;
- if (opp_info->scmi_clk)
- is_set_clk = false;
- }
-
- ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
- if (ret < 0) {
- LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
- return ret;
- }
- rockchip_get_read_margin(dev, opp_info, new_supply_vdd->u_volt,
- &target_rm);
-
- /* Change frequency */
- LOG_DEV_DEBUG(dev, "switching OPP: %lu Hz --> %lu Hz\n", old_freq,
- new_freq);
- /* Scaling up? Scale voltage before frequency */
- if (new_freq >= old_freq) {
- rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
- new_freq, true, is_set_clk);
- ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
- INT_MAX);
- if (ret) {
- LOG_DEV_ERROR(dev,
- "failed to set volt %lu uV for mem reg\n",
- new_supply_mem->u_volt);
- goto restore_voltage;
- }
- ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
- INT_MAX);
- if (ret) {
- LOG_DEV_ERROR(dev,
- "failed to set volt %lu uV for vdd reg\n",
- new_supply_vdd->u_volt);
- goto restore_voltage;
- }
- rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
- if (is_set_clk && clk_set_rate(clk, new_freq)) {
- ret = -EINVAL;
- LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
- goto restore_rm;
- }
- /* Scaling down? Scale voltage after frequency */
- } else {
- rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
- new_freq, false, is_set_clk);
- rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
- if (is_set_clk && clk_set_rate(clk, new_freq)) {
- ret = -EINVAL;
- LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
- goto restore_rm;
- }
- ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
- INT_MAX);
- if (ret) {
- LOG_DEV_ERROR(dev,
- "failed to set volt %lu uV for vdd reg\n",
- new_supply_vdd->u_volt);
- goto restore_freq;
- }
- ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
- INT_MAX);
- if (ret) {
- LOG_DEV_ERROR(dev,
- "failed to set volt %lu uV for mem reg\n",
- new_supply_mem->u_volt);
- goto restore_freq;
- }
- }
-
- clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
-
- return 0;
-
-restore_freq:
- if (is_set_clk && clk_set_rate(clk, old_freq))
- LOG_DEV_ERROR(dev, "failed to restore old-freq %lu Hz\n",
- old_freq);
-restore_rm:
- rockchip_get_read_margin(dev, opp_info, old_supply_vdd->u_volt,
- &target_rm);
- rockchip_set_read_margin(dev, opp_info, opp_info->current_rm,
- is_set_rm);
-restore_voltage:
- regulator_set_voltage(mem_reg, old_supply_mem->u_volt, INT_MAX);
- regulator_set_voltage(vdd_reg, old_supply_vdd->u_volt, INT_MAX);
- clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
-
- return ret;
-}
-
-static int npu_devfreq_target(struct device *dev, unsigned long *freq,
- u32 flags)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
- struct dev_pm_opp *opp;
- unsigned long opp_volt;
- int ret = 0;
-
- if (!npu_mdevp.is_checked)
- return -EINVAL;
-
- opp = devfreq_recommended_opp(dev, freq, flags);
- if (IS_ERR(opp))
- return PTR_ERR(opp);
- opp_volt = dev_pm_opp_get_voltage(opp);
- dev_pm_opp_put(opp);
-
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
- rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
-#endif
- ret = dev_pm_opp_set_rate(dev, *freq);
- if (!ret) {
- rknpu_dev->current_freq = *freq;
- if (rknpu_dev->devfreq)
- rknpu_dev->devfreq->last_status.current_frequency =
- *freq;
- rknpu_dev->current_volt = opp_volt;
- LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n",
- rknpu_dev->current_freq, rknpu_dev->current_volt);
- }
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
- rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
-#endif
-
- return ret;
-}
-
-#else
-
-static int npu_devfreq_target(struct device *dev, unsigned long *target_freq,
- u32 flags)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
- struct dev_pm_opp *opp = NULL;
- unsigned long freq = *target_freq;
- unsigned long old_freq = rknpu_dev->current_freq;
- unsigned long volt, old_volt = rknpu_dev->current_volt;
- int ret = -EINVAL;
-
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
- rcu_read_lock();
-#endif
-
- opp = devfreq_recommended_opp(dev, &freq, flags);
- if (IS_ERR(opp)) {
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
- rcu_read_unlock();
-#endif
- LOG_DEV_ERROR(dev, "failed to get opp (%ld)\n", PTR_ERR(opp));
- return PTR_ERR(opp);
- }
- volt = dev_pm_opp_get_voltage(opp);
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
- rcu_read_unlock();
-#endif
-
- /*
- * Only update if there is a change of frequency
- */
- if (old_freq == freq) {
- *target_freq = freq;
- if (old_volt == volt)
- return 0;
- ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
- return ret;
- }
- rknpu_dev->current_volt = volt;
- return 0;
- }
-
- if (rknpu_dev->vdd && old_volt != volt && old_freq < freq) {
- ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to increase volt %lu\n",
- volt);
- return ret;
- }
- }
- LOG_DEV_DEBUG(dev, "%luHz %luuV -> %luHz %luuV\n", old_freq, old_volt,
- freq, volt);
- ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
- return ret;
- }
- *target_freq = freq;
- rknpu_dev->current_freq = freq;
-
- if (rknpu_dev->devfreq)
- rknpu_dev->devfreq->last_status.current_frequency = freq;
-
- if (rknpu_dev->vdd && old_volt != volt && old_freq > freq) {
- ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to decrease volt %lu\n",
- volt);
- return ret;
- }
- }
- rknpu_dev->current_volt = volt;
-
- LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n",
- rknpu_dev->current_freq, rknpu_dev->current_volt);
-
- return ret;
-}
-#endif
-
-static int npu_devfreq_get_dev_status(struct device *dev,
- struct devfreq_dev_status *stat)
-{
- return 0;
-}
-
-static int npu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
-
- *freq = rknpu_dev->current_freq;
-
- return 0;
-}
-
-static struct devfreq_dev_profile npu_devfreq_profile = {
- .polling_ms = 50,
- .target = npu_devfreq_target,
- .get_dev_status = npu_devfreq_get_dev_status,
- .get_cur_freq = npu_devfreq_get_cur_freq,
-};
-
-#ifdef CONFIG_PM_DEVFREQ
-static int devfreq_rknpu_ondemand_func(struct devfreq *df, unsigned long *freq)
-{
- struct rknpu_device *rknpu_dev = df->data;
-
- if (rknpu_dev)
- *freq = rknpu_dev->ondemand_freq;
- else
- *freq = df->previous_freq;
-
- return 0;
-}
-
-static int devfreq_rknpu_ondemand_handler(struct devfreq *devfreq,
- unsigned int event, void *data)
-{
- return 0;
-}
-
-static struct devfreq_governor devfreq_rknpu_ondemand = {
- .name = "rknpu_ondemand",
- .get_target_freq = devfreq_rknpu_ondemand_func,
- .event_handler = devfreq_rknpu_ondemand_handler,
-};
-#endif
-
-static unsigned long npu_get_static_power(struct devfreq *devfreq,
- unsigned long voltage)
-{
- struct device *dev = devfreq->dev.parent;
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
-
- if (!rknpu_dev->model_data)
- return 0;
-
- return rockchip_ipa_get_static_power(rknpu_dev->model_data, voltage);
-}
-
-static struct devfreq_cooling_power npu_cooling_power = {
- .get_static_power = &npu_get_static_power,
-};
-
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
-static int rk3588_npu_get_soc_info(struct device *dev, struct device_node *np,
- int *bin, int *process)
-{
- int ret = 0;
- u8 value = 0;
-
- if (!bin)
- return 0;
-
- if (of_property_match_string(np, "nvmem-cell-names",
- "specification_serial_number") >= 0) {
- ret = rockchip_nvmem_cell_read_u8(
- np, "specification_serial_number", &value);
- if (ret) {
- LOG_DEV_ERROR(
- dev,
- "Failed to get specification_serial_number\n");
- return ret;
- }
- /* RK3588M */
- if (value == 0xd)
- *bin = 1;
- /* RK3588J */
- else if (value == 0xa)
- *bin = 2;
- }
- if (*bin < 0)
- *bin = 0;
- LOG_DEV_INFO(dev, "bin=%d\n", *bin);
-
- return ret;
-}
-
-static int rk3588_npu_set_soc_info(struct device *dev, struct device_node *np,
- int bin, int process, int volt_sel)
-{
- struct opp_table *opp_table;
- u32 supported_hw[2];
-
- if (volt_sel < 0)
- return 0;
- if (bin < 0)
- bin = 0;
-
- if (!of_property_read_bool(np, "rockchip,supported-hw"))
- return 0;
-
- /* SoC Version */
- supported_hw[0] = BIT(bin);
- /* Speed Grade */
- supported_hw[1] = BIT(volt_sel);
- opp_table = dev_pm_opp_set_supported_hw(dev, supported_hw, 2);
- if (IS_ERR(opp_table)) {
- LOG_DEV_ERROR(dev, "failed to set supported opp\n");
- return PTR_ERR(opp_table);
- }
-
- return 0;
-}
-
-static int rk3588_npu_set_read_margin(struct device *dev,
- struct rockchip_opp_info *opp_info,
- u32 rm)
-{
- u32 offset = 0, val = 0;
- int i, ret = 0;
-
- if (!opp_info->grf || !opp_info->volt_rm_tbl)
- return 0;
-
- if (rm == opp_info->current_rm || rm == UINT_MAX)
- return 0;
-
- LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
-
- for (i = 0; i < 3; i++) {
- ret = regmap_read(opp_info->grf, offset, &val);
- if (ret < 0) {
- LOG_DEV_ERROR(dev, "failed to get rm from 0x%x\n",
- offset);
- return ret;
- }
- val &= ~0x1c;
- regmap_write(opp_info->grf, offset, val | (rm << 2));
- offset += 4;
- }
- opp_info->current_rm = rm;
-
- return 0;
-}
-
-static const struct rockchip_opp_data rk3588_npu_opp_data = {
- .get_soc_info = rk3588_npu_get_soc_info,
- .set_soc_info = rk3588_npu_set_soc_info,
- .set_read_margin = rk3588_npu_set_read_margin,
-};
-
-static const struct of_device_id rockchip_npu_of_match[] = {
- {
- .compatible = "rockchip,rk3588",
- .data = (void *)&rk3588_npu_opp_data,
- },
- {},
-};
-
-static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
-{
- struct device *dev = rknpu_dev->dev;
- struct devfreq_dev_profile *dp = &npu_devfreq_profile;
- struct dev_pm_opp *opp;
- struct opp_table *reg_table = NULL;
- struct opp_table *opp_table = NULL;
- const char *const reg_names[] = { "rknpu", "mem" };
- int ret = -EINVAL;
-
- if (of_find_property(dev->of_node, "rknpu-supply", NULL) &&
- of_find_property(dev->of_node, "mem-supply", NULL)) {
- reg_table = dev_pm_opp_set_regulators(dev, reg_names, 2);
- if (IS_ERR(reg_table))
- return PTR_ERR(reg_table);
- opp_table =
- dev_pm_opp_register_set_opp_helper(dev, npu_opp_helper);
- if (IS_ERR(opp_table)) {
- dev_pm_opp_put_regulators(reg_table);
- return PTR_ERR(opp_table);
- }
- } else {
- reg_table = dev_pm_opp_set_regulators(dev, reg_names, 1);
- if (IS_ERR(reg_table))
- return PTR_ERR(reg_table);
- }
-
- rockchip_get_opp_data(rockchip_npu_of_match, &rknpu_dev->opp_info);
- ret = rockchip_init_opp_table(dev, &rknpu_dev->opp_info, "npu_leakage",
- "rknpu");
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
- return ret;
- }
-
- rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
-
- opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0);
- if (IS_ERR(opp)) {
- ret = PTR_ERR(opp);
- goto err_remove_table;
- }
- dev_pm_opp_put(opp);
- dp->initial_freq = rknpu_dev->current_freq;
-
-#ifdef CONFIG_PM_DEVFREQ
- ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
- goto err_remove_table;
- }
-#endif
-
- rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
- (void *)rknpu_dev);
- if (IS_ERR(rknpu_dev->devfreq)) {
- LOG_DEV_ERROR(dev, "failed to add devfreq\n");
- ret = PTR_ERR(rknpu_dev->devfreq);
- goto err_remove_governor;
- }
- devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq);
-
- rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq;
- rknpu_dev->devfreq->last_status.total_time = 1;
- rknpu_dev->devfreq->last_status.busy_time = 1;
-
- npu_mdevp.data = rknpu_dev->devfreq;
- npu_mdevp.opp_info = &rknpu_dev->opp_info;
- rknpu_dev->mdev_info =
- rockchip_system_monitor_register(dev, &npu_mdevp);
- if (IS_ERR(rknpu_dev->mdev_info)) {
- LOG_DEV_DEBUG(dev, "without system monitor\n");
- rknpu_dev->mdev_info = NULL;
- npu_mdevp.is_checked = true;
- }
- rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
- rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
-
- of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
- (u32 *)&npu_cooling_power.dyn_power_coeff);
- rknpu_dev->model_data =
- rockchip_ipa_power_model_init(dev, "npu_leakage");
- if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
- rknpu_dev->model_data = NULL;
- LOG_DEV_ERROR(dev, "failed to initialize power model\n");
- } else if (rknpu_dev->model_data->dynamic_coefficient) {
- npu_cooling_power.dyn_power_coeff =
- rknpu_dev->model_data->dynamic_coefficient;
- }
- if (!npu_cooling_power.dyn_power_coeff) {
- LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
- goto out;
- }
-
- rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
- dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
- if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
- LOG_DEV_ERROR(dev, "failed to register cooling device\n");
-
-out:
- return 0;
-
-err_remove_governor:
-#ifdef CONFIG_PM_DEVFREQ
- devfreq_remove_governor(&devfreq_rknpu_ondemand);
-#endif
-err_remove_table:
- dev_pm_opp_of_remove_table(dev);
-
- rknpu_dev->devfreq = NULL;
-
- return ret;
-}
-
-#else
-
-static int npu_devfreq_adjust_current_freq_volt(struct device *dev,
- struct rknpu_device *rknpu_dev)
-{
- unsigned long volt, old_freq, freq;
- struct dev_pm_opp *opp = NULL;
- int ret = -EINVAL;
-
- old_freq = clk_get_rate(rknpu_dev->clks[0].clk);
- freq = old_freq;
-
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
- rcu_read_lock();
-#endif
-
- opp = devfreq_recommended_opp(dev, &freq, 0);
- volt = dev_pm_opp_get_voltage(opp);
-
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
- rcu_read_unlock();
-#endif
-
- if (freq >= old_freq && rknpu_dev->vdd) {
- ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
- return ret;
- }
- }
- LOG_DEV_DEBUG(dev, "adjust current freq=%luHz, volt=%luuV\n", freq,
- volt);
- ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
- return ret;
- }
- if (freq < old_freq && rknpu_dev->vdd) {
- ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
- return ret;
- }
- }
- rknpu_dev->current_freq = freq;
- rknpu_dev->current_volt = volt;
-
- return 0;
-}
-
-static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
-{
- struct device *dev = rknpu_dev->dev;
- struct devfreq_dev_profile *dp = &npu_devfreq_profile;
- int ret = -EINVAL;
-
- ret = rockchip_init_opp_table(dev, NULL, "npu_leakage", "rknpu");
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
- return ret;
- }
-
- ret = npu_devfreq_adjust_current_freq_volt(dev, rknpu_dev);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to adjust current freq volt\n");
- goto err_remove_table;
- }
- dp->initial_freq = rknpu_dev->current_freq;
-
-#ifdef CONFIG_PM_DEVFREQ
- ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
- goto err_remove_table;
- }
-#endif
-
- rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
- (void *)rknpu_dev);
- if (IS_ERR(rknpu_dev->devfreq)) {
- LOG_DEV_ERROR(dev, "failed to add devfreq\n");
- ret = PTR_ERR(rknpu_dev->devfreq);
- goto err_remove_governor;
- }
- devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq);
-
- rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq;
- rknpu_dev->devfreq->last_status.total_time = 1;
- rknpu_dev->devfreq->last_status.busy_time = 1;
-
- npu_mdevp.data = rknpu_dev->devfreq;
- rknpu_dev->mdev_info =
- rockchip_system_monitor_register(dev, &npu_mdevp);
- if (IS_ERR(rknpu_dev->mdev_info)) {
- LOG_DEV_DEBUG(dev, "without system monitor\n");
- rknpu_dev->mdev_info = NULL;
- }
- rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
- rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
-
- of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
- (u32 *)&npu_cooling_power.dyn_power_coeff);
- rknpu_dev->model_data =
- rockchip_ipa_power_model_init(dev, "npu_leakage");
- if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
- rknpu_dev->model_data = NULL;
- LOG_DEV_ERROR(dev, "failed to initialize power model\n");
- } else if (rknpu_dev->model_data->dynamic_coefficient) {
- npu_cooling_power.dyn_power_coeff =
- rknpu_dev->model_data->dynamic_coefficient;
- }
-
- if (!npu_cooling_power.dyn_power_coeff) {
- LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
- goto out;
- }
-
- rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
- dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
- if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
- LOG_DEV_ERROR(dev, "failed to register cooling device\n");
-
-out:
- return 0;
-
-err_remove_governor:
-#ifdef CONFIG_PM_DEVFREQ
- devfreq_remove_governor(&devfreq_rknpu_ondemand);
-#endif
-err_remove_table:
- dev_pm_opp_of_remove_table(dev);
-
- rknpu_dev->devfreq = NULL;
-
- return ret;
-}
-#endif
-
-static int rknpu_devfreq_remove(struct rknpu_device *rknpu_dev)
-{
- if (rknpu_dev->devfreq) {
- devfreq_unregister_opp_notifier(rknpu_dev->dev,
- rknpu_dev->devfreq);
- dev_pm_opp_of_remove_table(rknpu_dev->dev);
-#ifdef CONFIG_PM_DEVFREQ
- devfreq_remove_governor(&devfreq_rknpu_ondemand);
-#endif
- }
-
- return 0;
-}
-
-#endif
static int rknpu_register_irq(struct platform_device *pdev,
struct rknpu_device *rknpu_dev)
@@ -1685,6 +1072,28 @@
return 0;
}
+static int rknpu_get_invalid_core_mask(struct device *dev)
+{
+ int ret = 0;
+ u8 invalid_core_mask = 0;
+
+ if (of_property_match_string(dev->of_node, "nvmem-cell-names",
+ "cores") >= 0) {
+ ret = rockchip_nvmem_cell_read_u8(dev->of_node, "cores",
+ &invalid_core_mask);
+ /* The default valid npu cores for RK3583 are core0 and core1 */
+ invalid_core_mask |= RKNPU_CORE2_MASK;
+ if (ret) {
+ LOG_DEV_ERROR(
+ dev,
+ "failed to get specification_serial_number\n");
+ return invalid_core_mask;
+ }
+ }
+
+ return (int)invalid_core_mask;
+}
+
static int rknpu_probe(struct platform_device *pdev)
{
struct resource *res = NULL;
@@ -1716,6 +1125,22 @@
if (!config)
return -EINVAL;
+ if (match->data == (void *)&rk3588_rknpu_config) {
+ int invalid_core_mask = rknpu_get_invalid_core_mask(dev);
+ /* The default valid npu cores for RK3583 are core0 and core1 */
+ if (invalid_core_mask & RKNPU_CORE2_MASK) {
+ if ((invalid_core_mask & RKNPU_CORE0_MASK) ||
+ (invalid_core_mask & RKNPU_CORE1_MASK)) {
+ LOG_DEV_ERROR(
+ dev,
+ "rknpu core invalid, invalid core mask: %#x\n",
+ invalid_core_mask);
+ return -ENODEV;
+ }
+ config = &rk3583_rknpu_config;
+ }
+ }
+
rknpu_dev->config = config;
rknpu_dev->dev = dev;
@@ -1744,11 +1169,6 @@
}
#ifndef FPGA_PLATFORM
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
- if (strstr(__clk_get_name(rknpu_dev->clks[0].clk), "scmi"))
- rknpu_dev->opp_info.scmi_clk = rknpu_dev->clks[0].clk;
-#endif
-
rknpu_dev->vdd = devm_regulator_get_optional(dev, "rknpu");
if (IS_ERR(rknpu_dev->vdd)) {
if (PTR_ERR(rknpu_dev->vdd) != -ENODEV) {
@@ -1875,9 +1295,11 @@
virt_dev = dev_pm_domain_attach_by_name(dev, "npu1");
if (!IS_ERR(virt_dev))
rknpu_dev->genpd_dev_npu1 = virt_dev;
- virt_dev = dev_pm_domain_attach_by_name(dev, "npu2");
- if (!IS_ERR(virt_dev))
- rknpu_dev->genpd_dev_npu2 = virt_dev;
+ if (config->num_irqs > 2) {
+ virt_dev = dev_pm_domain_attach_by_name(dev, "npu2");
+ if (!IS_ERR(virt_dev))
+ rknpu_dev->genpd_dev_npu2 = virt_dev;
+ }
rknpu_dev->multiple_domains = true;
}
@@ -1995,47 +1417,14 @@
}
#ifndef FPGA_PLATFORM
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
static int rknpu_runtime_suspend(struct device *dev)
{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
- struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
-
- if (opp_info->scmi_clk) {
- if (clk_set_rate(opp_info->scmi_clk, POWER_DOWN_FREQ))
- LOG_DEV_ERROR(dev, "failed to restore clk rate\n");
- }
- opp_info->current_rm = UINT_MAX;
-
- return 0;
+ return rknpu_devfreq_runtime_suspend(dev);
}
static int rknpu_runtime_resume(struct device *dev)
{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
- struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
- int ret = 0;
-
- if (!rknpu_dev->current_freq || !rknpu_dev->current_volt)
- return 0;
-
- ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
- return ret;
- }
-
- if (opp_info->data && opp_info->data->set_read_margin)
- opp_info->data->set_read_margin(dev, opp_info,
- opp_info->target_rm);
- if (opp_info->scmi_clk) {
- if (clk_set_rate(opp_info->scmi_clk, rknpu_dev->current_freq))
- LOG_DEV_ERROR(dev, "failed to set power down rate\n");
- }
-
- clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
-
- return ret;
+ return rknpu_devfreq_runtime_resume(dev);
}
static const struct dev_pm_ops rknpu_pm_ops = {
@@ -2045,7 +1434,6 @@
NULL)
};
#endif
-#endif
static struct platform_driver rknpu_driver = {
.probe = rknpu_probe,
@@ -2054,9 +1442,7 @@
.owner = THIS_MODULE,
.name = "RKNPU",
#ifndef FPGA_PLATFORM
-#if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
.pm = &rknpu_pm_ops,
-#endif
#endif
.of_match_table = of_match_ptr(rknpu_of_match),
},
@@ -2081,3 +1467,6 @@
MODULE_LICENSE("GPL v2");
MODULE_VERSION(RKNPU_GET_DRV_VERSION_STRING(DRIVER_MAJOR, DRIVER_MINOR,
DRIVER_PATCHLEVEL));
+#if KERNEL_VERSION(5, 16, 0) < LINUX_VERSION_CODE
+MODULE_IMPORT_NS(DMA_BUF);
+#endif
--
Gitblit v1.6.2