~hc/RK356X_SDK_RELEASE.git

..	..	@@ -31,19 +31,11 @@
31	31	#include <linux/clk-provider.h>
32	32	#include <linux/pm_domain.h>
33	33	#include <linux/pm_runtime.h>
34		-#include <linux/devfreq_cooling.h>
35	34	#include <linux/regmap.h>
36		-#include <linux/dma-iommu.h>
37	35	#include <linux/of_address.h>
38	36
39	37	#ifndef FPGA_PLATFORM
40	38	#include <soc/rockchip/rockchip_iommu.h>
41		-#include <soc/rockchip/rockchip_opp_select.h>
42		-#include <soc/rockchip/rockchip_system_monitor.h>
43		-#include <soc/rockchip/rockchip_ipa.h>
44		-#ifdef CONFIG_PM_DEVFREQ
45		-#include <../drivers/devfreq/governor.h>
46		-#endif
47	39	#endif
48	40
49	41	#include "rknpu_ioctl.h"
..	..	@@ -51,6 +43,7 @@
51	43	#include "rknpu_fence.h"
52	44	#include "rknpu_drv.h"
53	45	#include "rknpu_gem.h"
	46	+#include "rknpu_devfreq.h"
54	47
55	48	#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
56	49	#include <drm/drm_device.h>
..	..	@@ -79,36 +72,29 @@
79	72	MODULE_PARM_DESC(bypass_soft_reset,
80	73	"bypass RKNPU soft reset if set it to 1, disabled by default");
81	74
82		-struct npu_irqs_data {
	75	+struct rknpu_irqs_data {
83	76	const char *name;
84	77	irqreturn_t (irq_hdl)(int irq, void ctx);
85	78	};
86	79
87		-static const struct npu_irqs_data rk356x_npu_irqs[] = {
	80	+static const struct rknpu_irqs_data rknpu_irqs[] = {
88	81	{ "npu_irq", rknpu_core0_irq_handler }
89	82	};
90	83
91		-static const struct npu_irqs_data rk3588_npu_irqs[] = {
	84	+static const struct rknpu_irqs_data rk3588_npu_irqs[] = {
92	85	{ "npu0_irq", rknpu_core0_irq_handler },
93	86	{ "npu1_irq", rknpu_core1_irq_handler },
94	87	{ "npu2_irq", rknpu_core2_irq_handler }
95	88	};
96	89
97		-static const struct npu_irqs_data rv110x_npu_irqs[] = {
98		- { "npu_irq", rknpu_core0_irq_handler }
99		-};
	90	+static const struct rknpu_reset_data rknpu_resets[] = { { "srst_a",
	91	+ "srst_h" } };
100	92
101		-static const struct npu_reset_data rk356x_npu_resets[] = { { "srst_a",
102		- "srst_h" } };
103		-
104		-static const struct npu_reset_data rk3588_npu_resets[] = {
	93	+static const struct rknpu_reset_data rk3588_npu_resets[] = {
105	94	{ "srst_a0", "srst_h0" },
106	95	{ "srst_a1", "srst_h1" },
107	96	{ "srst_a2", "srst_h2" }
108	97	};
109		-
110		-static const struct npu_reset_data rv110x_npu_resets[] = { { "srst_a",
111		- "srst_h" } };
112	98
113	99	static const struct rknpu_config rk356x_rknpu_config = {
114	100	.bw_priority_addr = 0xfe180008,
..	..	@@ -117,11 +103,17 @@
117	103	.pc_data_amount_scale = 1,
118	104	.pc_task_number_bits = 12,
119	105	.pc_task_number_mask = 0xfff,
	106	+ .pc_task_status_offset = 0x3c,
	107	+ .pc_dma_ctrl = 0,
120	108	.bw_enable = 1,
121		- .irqs = rk356x_npu_irqs,
122		- .resets = rk356x_npu_resets,
123		- .num_irqs = ARRAY_SIZE(rk356x_npu_irqs),
124		- .num_resets = ARRAY_SIZE(rk356x_npu_resets)
	109	+ .irqs = rknpu_irqs,
	110	+ .resets = rknpu_resets,
	111	+ .num_irqs = ARRAY_SIZE(rknpu_irqs),
	112	+ .num_resets = ARRAY_SIZE(rknpu_resets),
	113	+ .nbuf_phyaddr = 0,
	114	+ .nbuf_size = 0,
	115	+ .max_submit_number = (1 << 12) - 1,
	116	+ .core_mask = 0x1,
125	117	};
126	118
127	119	static const struct rknpu_config rk3588_rknpu_config = {
..	..	@@ -131,11 +123,37 @@
131	123	.pc_data_amount_scale = 2,
132	124	.pc_task_number_bits = 12,
133	125	.pc_task_number_mask = 0xfff,
	126	+ .pc_task_status_offset = 0x3c,
	127	+ .pc_dma_ctrl = 0,
134	128	.bw_enable = 0,
135	129	.irqs = rk3588_npu_irqs,
136	130	.resets = rk3588_npu_resets,
137	131	.num_irqs = ARRAY_SIZE(rk3588_npu_irqs),
138		- .num_resets = ARRAY_SIZE(rk3588_npu_resets)
	132	+ .num_resets = ARRAY_SIZE(rk3588_npu_resets),
	133	+ .nbuf_phyaddr = 0,
	134	+ .nbuf_size = 0,
	135	+ .max_submit_number = (1 << 12) - 1,
	136	+ .core_mask = 0x7,
	137	+};
	138	+
	139	+static const struct rknpu_config rk3583_rknpu_config = {
	140	+ .bw_priority_addr = 0x0,
	141	+ .bw_priority_length = 0x0,
	142	+ .dma_mask = DMA_BIT_MASK(40),
	143	+ .pc_data_amount_scale = 2,
	144	+ .pc_task_number_bits = 12,
	145	+ .pc_task_number_mask = 0xfff,
	146	+ .pc_task_status_offset = 0x3c,
	147	+ .pc_dma_ctrl = 0,
	148	+ .bw_enable = 0,
	149	+ .irqs = rk3588_npu_irqs,
	150	+ .resets = rk3588_npu_resets,
	151	+ .num_irqs = 2,
	152	+ .num_resets = 2,
	153	+ .nbuf_phyaddr = 0,
	154	+ .nbuf_size = 0,
	155	+ .max_submit_number = (1 << 12) - 1,
	156	+ .core_mask = 0x3,
139	157	};
140	158
141	159	static const struct rknpu_config rv1106_rknpu_config = {
..	..	@@ -145,11 +163,37 @@
145	163	.pc_data_amount_scale = 2,
146	164	.pc_task_number_bits = 16,
147	165	.pc_task_number_mask = 0xffff,
	166	+ .pc_task_status_offset = 0x3c,
	167	+ .pc_dma_ctrl = 0,
148	168	.bw_enable = 1,
149		- .irqs = rv110x_npu_irqs,
150		- .resets = rv110x_npu_resets,
151		- .num_irqs = ARRAY_SIZE(rv110x_npu_irqs),
152		- .num_resets = ARRAY_SIZE(rv110x_npu_resets)
	169	+ .irqs = rknpu_irqs,
	170	+ .resets = rknpu_resets,
	171	+ .num_irqs = ARRAY_SIZE(rknpu_irqs),
	172	+ .num_resets = ARRAY_SIZE(rknpu_resets),
	173	+ .nbuf_phyaddr = 0,
	174	+ .nbuf_size = 0,
	175	+ .max_submit_number = (1 << 16) - 1,
	176	+ .core_mask = 0x1,
	177	+};
	178	+
	179	+static const struct rknpu_config rk3562_rknpu_config = {
	180	+ .bw_priority_addr = 0x0,
	181	+ .bw_priority_length = 0x0,
	182	+ .dma_mask = DMA_BIT_MASK(40),
	183	+ .pc_data_amount_scale = 2,
	184	+ .pc_task_number_bits = 16,
	185	+ .pc_task_number_mask = 0xffff,
	186	+ .pc_task_status_offset = 0x48,
	187	+ .pc_dma_ctrl = 1,
	188	+ .bw_enable = 1,
	189	+ .irqs = rknpu_irqs,
	190	+ .resets = rknpu_resets,
	191	+ .num_irqs = ARRAY_SIZE(rknpu_irqs),
	192	+ .num_resets = ARRAY_SIZE(rknpu_resets),
	193	+ .nbuf_phyaddr = 0xfe400000,
	194	+ .nbuf_size = 256 * 1024,
	195	+ .max_submit_number = (1 << 16) - 1,
	196	+ .core_mask = 0x1,
153	197	};
154	198
155	199	/* driver probe and init */
..	..	@@ -169,6 +213,10 @@
169	213	{
170	214	.compatible = "rockchip,rv1106-rknpu",
171	215	.data = &rv1106_rknpu_config,
	216	+ },
	217	+ {
	218	+ .compatible = "rockchip,rk3562-rknpu",
	219	+ .data = &rk3562_rknpu_config,
172	220	},
173	221	{},
174	222	};
..	..	@@ -198,7 +246,6 @@
198	246	{
199	247	int ret = 0;
200	248
201		- cancel_delayed_work(&rknpu_dev->power_off_work);
202	249	mutex_lock(&rknpu_dev->power_lock);
203	250	if (atomic_inc_return(&rknpu_dev->power_refcount) == 1)
204	251	ret = rknpu_power_on(rknpu_dev);
..	..	@@ -221,6 +268,9 @@
221	268
222	269	static int rknpu_power_put_delay(struct rknpu_device *rknpu_dev)
223	270	{
	271	+ if (rknpu_dev->power_put_delay == 0)
	272	+ return rknpu_power_put(rknpu_dev);
	273	+
224	274	mutex_lock(&rknpu_dev->power_lock);
225	275	if (atomic_read(&rknpu_dev->power_refcount) == 1)
226	276	queue_delayed_work(
..	..	@@ -229,6 +279,7 @@
229	279	else
230	280	atomic_dec_if_positive(&rknpu_dev->power_refcount);
231	281	mutex_unlock(&rknpu_dev->power_lock);
	282	+
232	283	return 0;
233	284	}
234	285
..	..	@@ -245,13 +296,17 @@
245	296	ret = rknpu_get_drv_version(&args->value);
246	297	break;
247	298	case RKNPU_GET_FREQ:
	299	+#ifndef FPGA_PLATFORM
248	300	args->value = clk_get_rate(rknpu_dev->clks[0].clk);
	301	+#endif
249	302	ret = 0;
250	303	break;
251	304	case RKNPU_SET_FREQ:
252	305	break;
253	306	case RKNPU_GET_VOLT:
	307	+#ifndef FPGA_PLATFORM
254	308	args->value = regulator_get_voltage(rknpu_dev->vdd);
	309	+#endif
255	310	ret = 0;
256	311	break;
257	312	case RKNPU_SET_VOLT:
..	..	@@ -330,11 +385,56 @@
330	385	#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
331	386	static int rknpu_open(struct inode inode, struct file file)
332	387	{
	388	+ struct rknpu_device *rknpu_dev =
	389	+ container_of(file->private_data, struct rknpu_device, miscdev);
	390	+ struct rknpu_session *session = NULL;
	391	+
	392	+ session = kzalloc(sizeof(*session), GFP_KERNEL);
	393	+ if (!session) {
	394	+ LOG_ERROR("rknpu session alloc failed\n");
	395	+ return -ENOMEM;
	396	+ }
	397	+
	398	+ session->rknpu_dev = rknpu_dev;
	399	+ INIT_LIST_HEAD(&session->list);
	400	+
	401	+ file->private_data = (void *)session;
	402	+
333	403	return nonseekable_open(inode, file);
334	404	}
335	405
336	406	static int rknpu_release(struct inode inode, struct file file)
337	407	{
	408	+ struct rknpu_mem_object *entry;
	409	+ struct rknpu_session *session = file->private_data;
	410	+ struct rknpu_device *rknpu_dev = session->rknpu_dev;
	411	+ LIST_HEAD(local_list);
	412	+
	413	+ spin_lock(&rknpu_dev->lock);
	414	+ list_replace_init(&session->list, &local_list);
	415	+ file->private_data = NULL;
	416	+ spin_unlock(&rknpu_dev->lock);
	417	+
	418	+ while (!list_empty(&local_list)) {
	419	+ entry = list_first_entry(&local_list, struct rknpu_mem_object,
	420	+ head);
	421	+
	422	+ LOG_DEBUG(
	423	+ "Fd close free rknpu_obj: %#llx, rknpu_obj->dma_addr: %#llx\n",
	424	+ (__u64)(uintptr_t)entry, (__u64)entry->dma_addr);
	425	+
	426	+ vunmap(entry->kv_addr);
	427	+ entry->kv_addr = NULL;
	428	+
	429	+ if (!entry->owner)
	430	+ dma_buf_put(entry->dmabuf);
	431	+
	432	+ list_del(&entry->head);
	433	+ kfree(entry);
	434	+ }
	435	+
	436	+ kfree(session);
	437	+
338	438	return 0;
339	439	}
340	440
..	..	@@ -366,8 +466,12 @@
366	466	static long rknpu_ioctl(struct file *file, uint32_t cmd, unsigned long arg)
367	467	{
368	468	long ret = -EINVAL;
369		- struct rknpu_device *rknpu_dev =
370		- container_of(file->private_data, struct rknpu_device, miscdev);
	469	+ struct rknpu_device *rknpu_dev = NULL;
	470	+
	471	+ if (!file->private_data)
	472	+ return -EINVAL;
	473	+
	474	+ rknpu_dev = ((struct rknpu_session *)file->private_data)->rknpu_dev;
371	475
372	476	rknpu_power_get(rknpu_dev);
373	477
..	..	@@ -379,12 +483,12 @@
379	483	ret = rknpu_submit_ioctl(rknpu_dev, arg);
380	484	break;
381	485	case IOCTL_RKNPU_MEM_CREATE:
382		- ret = rknpu_mem_create_ioctl(rknpu_dev, arg);
	486	+ ret = rknpu_mem_create_ioctl(rknpu_dev, arg, file);
383	487	break;
384	488	case RKNPU_MEM_MAP:
385	489	break;
386	490	case IOCTL_RKNPU_MEM_DESTROY:
387		- ret = rknpu_mem_destroy_ioctl(rknpu_dev, arg);
	491	+ ret = rknpu_mem_destroy_ioctl(rknpu_dev, arg, file);
388	492	break;
389	493	case IOCTL_RKNPU_MEM_SYNC:
390	494	ret = rknpu_mem_sync_ioctl(rknpu_dev, arg);
..	..	@@ -409,11 +513,13 @@
409	513	#endif
410	514
411	515	#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
	516	+#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
412	517	static const struct vm_operations_struct rknpu_gem_vm_ops = {
413	518	.fault = rknpu_gem_fault,
414	519	.open = drm_gem_vm_open,
415	520	.close = drm_gem_vm_close,
416	521	};
	522	+#endif
417	523
418	524	static int rknpu_action_ioctl(struct drm_device dev, void data,
419	525	struct drm_file *file_priv)
..	..	@@ -455,6 +561,9 @@
455	561	DRM_RENDER_ALLOW),
456	562	};
457	563
	564	+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
	565	+DEFINE_DRM_GEM_FOPS(rknpu_drm_driver_fops);
	566	+#else
458	567	static const struct file_operations rknpu_drm_driver_fops = {
459	568	.owner = THIS_MODULE,
460	569	.open = drm_open,
..	..	@@ -468,6 +577,7 @@
468	577	.release = drm_release,
469	578	.llseek = noop_llseek,
470	579	};
	580	+#endif
471	581
472	582	static struct drm_driver rknpu_drm_driver = {
473	583	#if KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE
..	..	@@ -475,28 +585,34 @@
475	585	#else
476	586	.driver_features = DRIVER_GEM \| DRIVER_PRIME \| DRIVER_RENDER,
477	587	#endif
	588	+#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
478	589	.gem_free_object_unlocked = rknpu_gem_free_object,
479	590	.gem_vm_ops = &rknpu_gem_vm_ops,
	591	+ .dumb_destroy = drm_gem_dumb_destroy,
	592	+ .gem_prime_export = drm_gem_prime_export,
	593	+ .gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table,
	594	+ .gem_prime_vmap = rknpu_gem_prime_vmap,
	595	+ .gem_prime_vunmap = rknpu_gem_prime_vunmap,
	596	+#endif
480	597	.dumb_create = rknpu_gem_dumb_create,
481	598	#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
482	599	.dumb_map_offset = rknpu_gem_dumb_map_offset,
483	600	#else
484	601	.dumb_map_offset = drm_gem_dumb_map_offset,
485	602	#endif
486		- .dumb_destroy = drm_gem_dumb_destroy,
487	603	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
488	604	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
489		- .gem_prime_export = drm_gem_prime_export,
490	605	#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
491	606	.gem_prime_import = rknpu_gem_prime_import,
492	607	#else
493	608	.gem_prime_import = drm_gem_prime_import,
494	609	#endif
495		- .gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table,
496	610	.gem_prime_import_sg_table = rknpu_gem_prime_import_sg_table,
497		- .gem_prime_vmap = rknpu_gem_prime_vmap,
498		- .gem_prime_vunmap = rknpu_gem_prime_vunmap,
	611	+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
	612	+ .gem_prime_mmap = drm_gem_prime_mmap,
	613	+#else
499	614	.gem_prime_mmap = rknpu_gem_prime_mmap,
	615	+#endif
500	616	.ioctls = rknpu_ioctls,
501	617	.num_ioctls = ARRAY_SIZE(rknpu_ioctls),
502	618	.fops = &rknpu_drm_driver_fops,
..	..	@@ -516,7 +632,7 @@
516	632	container_of(timer, struct rknpu_device, timer);
517	633	struct rknpu_subcore_data *subcore_data = NULL;
518	634	struct rknpu_job *job = NULL;
519		- ktime_t now = ktime_get();
	635	+ ktime_t now;
520	636	unsigned long flags;
521	637	int i;
522	638
..	..	@@ -527,14 +643,16 @@
527	643
528	644	job = subcore_data->job;
529	645	if (job) {
	646	+ now = ktime_get();
530	647	subcore_data->timer.busy_time +=
531		- ktime_us_delta(now, job->hw_recoder_time);
532		- job->hw_recoder_time = ktime_get();
	648	+ ktime_sub(now, job->hw_recoder_time);
	649	+ job->hw_recoder_time = now;
533	650	}
534	651
535		- subcore_data->timer.busy_time_record =
	652	+ subcore_data->timer.total_busy_time =
536	653	subcore_data->timer.busy_time;
537	654	subcore_data->timer.busy_time = 0;
	655	+
538	656	spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
539	657	}
540	658
..	..	@@ -581,6 +699,42 @@
581	699	}
582	700
583	701	#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
	702	+static int drm_fake_dev_register(struct rknpu_device *rknpu_dev)
	703	+{
	704	+ const struct platform_device_info rknpu_dev_info = {
	705	+ .name = "rknpu_dev",
	706	+ .id = PLATFORM_DEVID_AUTO,
	707	+ .dma_mask = rknpu_dev->config->dma_mask,
	708	+ };
	709	+ struct platform_device *pdev = NULL;
	710	+ int ret = -EINVAL;
	711	+
	712	+ pdev = platform_device_register_full(&rknpu_dev_info);
	713	+ if (pdev) {
	714	+ ret = of_dma_configure(&pdev->dev, NULL, true);
	715	+ if (ret) {
	716	+ platform_device_unregister(pdev);
	717	+ pdev = NULL;
	718	+ }
	719	+ }
	720	+
	721	+ rknpu_dev->fake_dev = pdev ? &pdev->dev : NULL;
	722	+
	723	+ return ret;
	724	+}
	725	+
	726	+static void drm_fake_dev_unregister(struct rknpu_device *rknpu_dev)
	727	+{
	728	+ struct platform_device *pdev = NULL;
	729	+
	730	+ if (!rknpu_dev->fake_dev)
	731	+ return;
	732	+
	733	+ pdev = to_platform_device(rknpu_dev->fake_dev);
	734	+
	735	+ platform_device_unregister(pdev);
	736	+}
	737	+
584	738	static int rknpu_drm_probe(struct rknpu_device *rknpu_dev)
585	739	{
586	740	struct device *dev = rknpu_dev->dev;
..	..	@@ -599,6 +753,8 @@
599	753	drm_dev->dev_private = rknpu_dev;
600	754	rknpu_dev->drm_dev = drm_dev;
601	755
	756	+ drm_fake_dev_register(rknpu_dev);
	757	+
602	758	return 0;
603	759
604	760	err_free_drm:
..	..	@@ -614,6 +770,8 @@
614	770	static void rknpu_drm_remove(struct rknpu_device *rknpu_dev)
615	771	{
616	772	struct drm_device *drm_dev = rknpu_dev->drm_dev;
	773	+
	774	+ drm_fake_dev_unregister(rknpu_dev);
617	775
618	776	drm_dev_unregister(drm_dev);
619	777
..	..	@@ -662,9 +820,7 @@
662	820	}
663	821
664	822	#ifndef FPGA_PLATFORM
665		-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
666		- rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
667		-#endif
	823	+ rknpu_devfreq_lock(rknpu_dev);
668	824	#endif
669	825
670	826	if (rknpu_dev->multiple_domains) {
..	..	@@ -723,9 +879,7 @@
723	879
724	880	out:
725	881	#ifndef FPGA_PLATFORM
726		-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
727		- rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
728		-#endif
	882	+ rknpu_devfreq_unlock(rknpu_dev);
729	883	#endif
730	884
731	885	return ret;
..	..	@@ -739,9 +893,7 @@
739	893	int ret;
740	894	bool val;
741	895
742		-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
743		- rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
744		-#endif
	896	+ rknpu_devfreq_lock(rknpu_dev);
745	897	#endif
746	898
747	899	pm_runtime_put_sync(dev);
..	..	@@ -763,10 +915,7 @@
763	915	if (ret) {
764	916	LOG_DEV_ERROR(dev, "iommu still enabled\n");
765	917	pm_runtime_get_sync(dev);
766		-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
767		- rockchip_monitor_volt_adjust_unlock(
768		- rknpu_dev->mdev_info);
769		-#endif
	918	+ rknpu_devfreq_unlock(rknpu_dev);
770	919	return ret;
771	920	}
772	921	#else
..	..	@@ -782,9 +931,7 @@
782	931	}
783	932
784	933	#ifndef FPGA_PLATFORM
785		-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
786		- rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
787		-#endif
	934	+ rknpu_devfreq_unlock(rknpu_dev);
788	935	#endif
789	936
790	937	clk_bulk_disable_unprepare(rknpu_dev->num_clks, rknpu_dev->clks);
..	..	@@ -799,624 +946,6 @@
799	946
800	947	return 0;
801	948	}
802		-
803		-#ifndef FPGA_PLATFORM
804		-static struct monitor_dev_profile npu_mdevp = {
805		- .type = MONITOR_TPYE_DEV,
806		- .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
807		- .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
808		-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
809		- .update_volt = rockchip_monitor_check_rate_volt,
810		-#endif
811		-};
812		-
813		-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
814		-static int npu_opp_helper(struct dev_pm_set_opp_data *data)
815		-{
816		- struct device *dev = data->dev;
817		- struct dev_pm_opp_supply *old_supply_vdd = &data->old_opp.supplies[0];
818		- struct dev_pm_opp_supply *old_supply_mem = &data->old_opp.supplies[1];
819		- struct dev_pm_opp_supply *new_supply_vdd = &data->new_opp.supplies[0];
820		- struct dev_pm_opp_supply *new_supply_mem = &data->new_opp.supplies[1];
821		- struct regulator *vdd_reg = data->regulators[0];
822		- struct regulator *mem_reg = data->regulators[1];
823		- struct clk *clk = data->clk;
824		- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
825		- struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
826		- unsigned long old_freq = data->old_opp.rate;
827		- unsigned long new_freq = data->new_opp.rate;
828		- bool is_set_rm = true;
829		- bool is_set_clk = true;
830		- u32 target_rm = UINT_MAX;
831		- int ret = 0;
832		-
833		- if (!pm_runtime_active(dev)) {
834		- is_set_rm = false;
835		- if (opp_info->scmi_clk)
836		- is_set_clk = false;
837		- }
838		-
839		- ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
840		- if (ret < 0) {
841		- LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
842		- return ret;
843		- }
844		- rockchip_get_read_margin(dev, opp_info, new_supply_vdd->u_volt,
845		- &target_rm);
846		-
847		- /* Change frequency */
848		- LOG_DEV_DEBUG(dev, "switching OPP: %lu Hz --> %lu Hz\n", old_freq,
849		- new_freq);
850		- /* Scaling up? Scale voltage before frequency */
851		- if (new_freq >= old_freq) {
852		- rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
853		- new_freq, true, is_set_clk);
854		- ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
855		- INT_MAX);
856		- if (ret) {
857		- LOG_DEV_ERROR(dev,
858		- "failed to set volt %lu uV for mem reg\n",
859		- new_supply_mem->u_volt);
860		- goto restore_voltage;
861		- }
862		- ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
863		- INT_MAX);
864		- if (ret) {
865		- LOG_DEV_ERROR(dev,
866		- "failed to set volt %lu uV for vdd reg\n",
867		- new_supply_vdd->u_volt);
868		- goto restore_voltage;
869		- }
870		- rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
871		- if (is_set_clk && clk_set_rate(clk, new_freq)) {
872		- ret = -EINVAL;
873		- LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
874		- goto restore_rm;
875		- }
876		- /* Scaling down? Scale voltage after frequency */
877		- } else {
878		- rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
879		- new_freq, false, is_set_clk);
880		- rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
881		- if (is_set_clk && clk_set_rate(clk, new_freq)) {
882		- ret = -EINVAL;
883		- LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
884		- goto restore_rm;
885		- }
886		- ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
887		- INT_MAX);
888		- if (ret) {
889		- LOG_DEV_ERROR(dev,
890		- "failed to set volt %lu uV for vdd reg\n",
891		- new_supply_vdd->u_volt);
892		- goto restore_freq;
893		- }
894		- ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
895		- INT_MAX);
896		- if (ret) {
897		- LOG_DEV_ERROR(dev,
898		- "failed to set volt %lu uV for mem reg\n",
899		- new_supply_mem->u_volt);
900		- goto restore_freq;
901		- }
902		- }
903		-
904		- clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
905		-
906		- return 0;
907		-
908		-restore_freq:
909		- if (is_set_clk && clk_set_rate(clk, old_freq))
910		- LOG_DEV_ERROR(dev, "failed to restore old-freq %lu Hz\n",
911		- old_freq);
912		-restore_rm:
913		- rockchip_get_read_margin(dev, opp_info, old_supply_vdd->u_volt,
914		- &target_rm);
915		- rockchip_set_read_margin(dev, opp_info, opp_info->current_rm,
916		- is_set_rm);
917		-restore_voltage:
918		- regulator_set_voltage(mem_reg, old_supply_mem->u_volt, INT_MAX);
919		- regulator_set_voltage(vdd_reg, old_supply_vdd->u_volt, INT_MAX);
920		- clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
921		-
922		- return ret;
923		-}
924		-
925		-static int npu_devfreq_target(struct device dev, unsigned long freq,
926		- u32 flags)
927		-{
928		- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
929		- struct dev_pm_opp *opp;
930		- unsigned long opp_volt;
931		- int ret = 0;
932		-
933		- if (!npu_mdevp.is_checked)
934		- return -EINVAL;
935		-
936		- opp = devfreq_recommended_opp(dev, freq, flags);
937		- if (IS_ERR(opp))
938		- return PTR_ERR(opp);
939		- opp_volt = dev_pm_opp_get_voltage(opp);
940		- dev_pm_opp_put(opp);
941		-
942		-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
943		- rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
944		-#endif
945		- ret = dev_pm_opp_set_rate(dev, *freq);
946		- if (!ret) {
947		- rknpu_dev->current_freq = *freq;
948		- if (rknpu_dev->devfreq)
949		- rknpu_dev->devfreq->last_status.current_frequency =
950		- *freq;
951		- rknpu_dev->current_volt = opp_volt;
952		- LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n",
953		- rknpu_dev->current_freq, rknpu_dev->current_volt);
954		- }
955		-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
956		- rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
957		-#endif
958		-
959		- return ret;
960		-}
961		-
962		-#else
963		-
964		-static int npu_devfreq_target(struct device dev, unsigned long target_freq,
965		- u32 flags)
966		-{
967		- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
968		- struct dev_pm_opp *opp = NULL;
969		- unsigned long freq = *target_freq;
970		- unsigned long old_freq = rknpu_dev->current_freq;
971		- unsigned long volt, old_volt = rknpu_dev->current_volt;
972		- int ret = -EINVAL;
973		-
974		-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
975		- rcu_read_lock();
976		-#endif
977		-
978		- opp = devfreq_recommended_opp(dev, &freq, flags);
979		- if (IS_ERR(opp)) {
980		-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
981		- rcu_read_unlock();
982		-#endif
983		- LOG_DEV_ERROR(dev, "failed to get opp (%ld)\n", PTR_ERR(opp));
984		- return PTR_ERR(opp);
985		- }
986		- volt = dev_pm_opp_get_voltage(opp);
987		-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
988		- rcu_read_unlock();
989		-#endif
990		-
991		- /*
992		- * Only update if there is a change of frequency
993		- */
994		- if (old_freq == freq) {
995		- *target_freq = freq;
996		- if (old_volt == volt)
997		- return 0;
998		- ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
999		- if (ret) {
1000		- LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
1001		- return ret;
1002		- }
1003		- rknpu_dev->current_volt = volt;
1004		- return 0;
1005		- }
1006		-
1007		- if (rknpu_dev->vdd && old_volt != volt && old_freq < freq) {
1008		- ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1009		- if (ret) {
1010		- LOG_DEV_ERROR(dev, "failed to increase volt %lu\n",
1011		- volt);
1012		- return ret;
1013		- }
1014		- }
1015		- LOG_DEV_DEBUG(dev, "%luHz %luuV -> %luHz %luuV\n", old_freq, old_volt,
1016		- freq, volt);
1017		- ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
1018		- if (ret) {
1019		- LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
1020		- return ret;
1021		- }
1022		- *target_freq = freq;
1023		- rknpu_dev->current_freq = freq;
1024		-
1025		- if (rknpu_dev->devfreq)
1026		- rknpu_dev->devfreq->last_status.current_frequency = freq;
1027		-
1028		- if (rknpu_dev->vdd && old_volt != volt && old_freq > freq) {
1029		- ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1030		- if (ret) {
1031		- LOG_DEV_ERROR(dev, "failed to decrease volt %lu\n",
1032		- volt);
1033		- return ret;
1034		- }
1035		- }
1036		- rknpu_dev->current_volt = volt;
1037		-
1038		- LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n",
1039		- rknpu_dev->current_freq, rknpu_dev->current_volt);
1040		-
1041		- return ret;
1042		-}
1043		-#endif
1044		-
1045		-static int npu_devfreq_get_dev_status(struct device *dev,
1046		- struct devfreq_dev_status *stat)
1047		-{
1048		- return 0;
1049		-}
1050		-
1051		-static int npu_devfreq_get_cur_freq(struct device dev, unsigned long freq)
1052		-{
1053		- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1054		-
1055		- *freq = rknpu_dev->current_freq;
1056		-
1057		- return 0;
1058		-}
1059		-
1060		-static struct devfreq_dev_profile npu_devfreq_profile = {
1061		- .polling_ms = 50,
1062		- .target = npu_devfreq_target,
1063		- .get_dev_status = npu_devfreq_get_dev_status,
1064		- .get_cur_freq = npu_devfreq_get_cur_freq,
1065		-};
1066		-
1067		-#ifdef CONFIG_PM_DEVFREQ
1068		-static int devfreq_rknpu_ondemand_func(struct devfreq df, unsigned long freq)
1069		-{
1070		- struct rknpu_device *rknpu_dev = df->data;
1071		-
1072		- if (rknpu_dev)
1073		- *freq = rknpu_dev->ondemand_freq;
1074		- else
1075		- *freq = df->previous_freq;
1076		-
1077		- return 0;
1078		-}
1079		-
1080		-static int devfreq_rknpu_ondemand_handler(struct devfreq *devfreq,
1081		- unsigned int event, void *data)
1082		-{
1083		- return 0;
1084		-}
1085		-
1086		-static struct devfreq_governor devfreq_rknpu_ondemand = {
1087		- .name = "rknpu_ondemand",
1088		- .get_target_freq = devfreq_rknpu_ondemand_func,
1089		- .event_handler = devfreq_rknpu_ondemand_handler,
1090		-};
1091		-#endif
1092		-
1093		-static unsigned long npu_get_static_power(struct devfreq *devfreq,
1094		- unsigned long voltage)
1095		-{
1096		- struct device *dev = devfreq->dev.parent;
1097		- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1098		-
1099		- if (!rknpu_dev->model_data)
1100		- return 0;
1101		-
1102		- return rockchip_ipa_get_static_power(rknpu_dev->model_data, voltage);
1103		-}
1104		-
1105		-static struct devfreq_cooling_power npu_cooling_power = {
1106		- .get_static_power = &npu_get_static_power,
1107		-};
1108		-
1109		-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
1110		-static int rk3588_npu_set_read_margin(struct device *dev,
1111		- struct rockchip_opp_info *opp_info,
1112		- u32 rm)
1113		-{
1114		- u32 offset = 0, val = 0;
1115		- int i, ret = 0;
1116		-
1117		- if (!opp_info->grf \|\| !opp_info->volt_rm_tbl)
1118		- return 0;
1119		-
1120		- if (rm == opp_info->current_rm \|\| rm == UINT_MAX)
1121		- return 0;
1122		-
1123		- LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
1124		-
1125		- for (i = 0; i < 3; i++) {
1126		- ret = regmap_read(opp_info->grf, offset, &val);
1127		- if (ret < 0) {
1128		- LOG_DEV_ERROR(dev, "failed to get rm from 0x%x\n",
1129		- offset);
1130		- return ret;
1131		- }
1132		- val &= ~0x1c;
1133		- regmap_write(opp_info->grf, offset, val \| (rm << 2));
1134		- offset += 4;
1135		- }
1136		- opp_info->current_rm = rm;
1137		-
1138		- return 0;
1139		-}
1140		-
1141		-static const struct rockchip_opp_data rk3588_npu_opp_data = {
1142		- .set_read_margin = rk3588_npu_set_read_margin,
1143		-};
1144		-
1145		-static const struct of_device_id rockchip_npu_of_match[] = {
1146		- {
1147		- .compatible = "rockchip,rk3588",
1148		- .data = (void *)&rk3588_npu_opp_data,
1149		- },
1150		- {},
1151		-};
1152		-
1153		-static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
1154		-{
1155		- struct device *dev = rknpu_dev->dev;
1156		- struct devfreq_dev_profile *dp = &npu_devfreq_profile;
1157		- struct dev_pm_opp *opp;
1158		- struct opp_table *reg_table = NULL;
1159		- struct opp_table *opp_table = NULL;
1160		- const char *const reg_names[] = { "rknpu", "mem" };
1161		- int ret = -EINVAL;
1162		-
1163		- if (of_find_property(dev->of_node, "rknpu-supply", NULL) &&
1164		- of_find_property(dev->of_node, "mem-supply", NULL)) {
1165		- reg_table = dev_pm_opp_set_regulators(dev, reg_names, 2);
1166		- if (IS_ERR(reg_table))
1167		- return PTR_ERR(reg_table);
1168		- opp_table =
1169		- dev_pm_opp_register_set_opp_helper(dev, npu_opp_helper);
1170		- if (IS_ERR(opp_table)) {
1171		- dev_pm_opp_put_regulators(reg_table);
1172		- return PTR_ERR(opp_table);
1173		- }
1174		- } else {
1175		- reg_table = dev_pm_opp_set_regulators(dev, reg_names, 1);
1176		- if (IS_ERR(reg_table))
1177		- return PTR_ERR(reg_table);
1178		- }
1179		-
1180		- rockchip_get_opp_data(rockchip_npu_of_match, &rknpu_dev->opp_info);
1181		- ret = rockchip_init_opp_table(dev, &rknpu_dev->opp_info, "npu_leakage",
1182		- "rknpu");
1183		- if (ret) {
1184		- LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
1185		- return ret;
1186		- }
1187		-
1188		- rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1189		-
1190		- opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0);
1191		- if (IS_ERR(opp)) {
1192		- ret = PTR_ERR(opp);
1193		- goto err_remove_table;
1194		- }
1195		- dev_pm_opp_put(opp);
1196		- dp->initial_freq = rknpu_dev->current_freq;
1197		-
1198		-#ifdef CONFIG_PM_DEVFREQ
1199		- ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
1200		- if (ret) {
1201		- LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
1202		- goto err_remove_table;
1203		- }
1204		-#endif
1205		-
1206		- rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
1207		- (void *)rknpu_dev);
1208		- if (IS_ERR(rknpu_dev->devfreq)) {
1209		- LOG_DEV_ERROR(dev, "failed to add devfreq\n");
1210		- ret = PTR_ERR(rknpu_dev->devfreq);
1211		- goto err_remove_governor;
1212		- }
1213		- devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq);
1214		-
1215		- rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq;
1216		- rknpu_dev->devfreq->last_status.total_time = 1;
1217		- rknpu_dev->devfreq->last_status.busy_time = 1;
1218		-
1219		- npu_mdevp.data = rknpu_dev->devfreq;
1220		- npu_mdevp.opp_info = &rknpu_dev->opp_info;
1221		- rknpu_dev->mdev_info =
1222		- rockchip_system_monitor_register(dev, &npu_mdevp);
1223		- if (IS_ERR(rknpu_dev->mdev_info)) {
1224		- LOG_DEV_DEBUG(dev, "without system monitor\n");
1225		- rknpu_dev->mdev_info = NULL;
1226		- npu_mdevp.is_checked = true;
1227		- }
1228		- rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1229		- rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
1230		-
1231		- of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
1232		- (u32 *)&npu_cooling_power.dyn_power_coeff);
1233		- rknpu_dev->model_data =
1234		- rockchip_ipa_power_model_init(dev, "npu_leakage");
1235		- if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
1236		- rknpu_dev->model_data = NULL;
1237		- LOG_DEV_ERROR(dev, "failed to initialize power model\n");
1238		- } else if (rknpu_dev->model_data->dynamic_coefficient) {
1239		- npu_cooling_power.dyn_power_coeff =
1240		- rknpu_dev->model_data->dynamic_coefficient;
1241		- }
1242		- if (!npu_cooling_power.dyn_power_coeff) {
1243		- LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
1244		- goto out;
1245		- }
1246		-
1247		- rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
1248		- dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
1249		- if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
1250		- LOG_DEV_ERROR(dev, "failed to register cooling device\n");
1251		-
1252		-out:
1253		- return 0;
1254		-
1255		-err_remove_governor:
1256		-#ifdef CONFIG_PM_DEVFREQ
1257		- devfreq_remove_governor(&devfreq_rknpu_ondemand);
1258		-#endif
1259		-err_remove_table:
1260		- dev_pm_opp_of_remove_table(dev);
1261		-
1262		- rknpu_dev->devfreq = NULL;
1263		-
1264		- return ret;
1265		-}
1266		-
1267		-#else
1268		-
1269		-static int npu_devfreq_adjust_current_freq_volt(struct device *dev,
1270		- struct rknpu_device *rknpu_dev)
1271		-{
1272		- unsigned long volt, old_freq, freq;
1273		- struct dev_pm_opp *opp = NULL;
1274		- int ret = -EINVAL;
1275		-
1276		- old_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1277		- freq = old_freq;
1278		-
1279		-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
1280		- rcu_read_lock();
1281		-#endif
1282		-
1283		- opp = devfreq_recommended_opp(dev, &freq, 0);
1284		- volt = dev_pm_opp_get_voltage(opp);
1285		-
1286		-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
1287		- rcu_read_unlock();
1288		-#endif
1289		-
1290		- if (freq >= old_freq && rknpu_dev->vdd) {
1291		- ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1292		- if (ret) {
1293		- LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
1294		- return ret;
1295		- }
1296		- }
1297		- LOG_DEV_DEBUG(dev, "adjust current freq=%luHz, volt=%luuV\n", freq,
1298		- volt);
1299		- ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
1300		- if (ret) {
1301		- LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
1302		- return ret;
1303		- }
1304		- if (freq < old_freq && rknpu_dev->vdd) {
1305		- ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1306		- if (ret) {
1307		- LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
1308		- return ret;
1309		- }
1310		- }
1311		- rknpu_dev->current_freq = freq;
1312		- rknpu_dev->current_volt = volt;
1313		-
1314		- return 0;
1315		-}
1316		-
1317		-static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
1318		-{
1319		- struct device *dev = rknpu_dev->dev;
1320		- struct devfreq_dev_profile *dp = &npu_devfreq_profile;
1321		- int ret = -EINVAL;
1322		-
1323		- ret = rockchip_init_opp_table(dev, NULL, "npu_leakage", "rknpu");
1324		- if (ret) {
1325		- LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
1326		- return ret;
1327		- }
1328		-
1329		- ret = npu_devfreq_adjust_current_freq_volt(dev, rknpu_dev);
1330		- if (ret) {
1331		- LOG_DEV_ERROR(dev, "failed to adjust current freq volt\n");
1332		- goto err_remove_table;
1333		- }
1334		- dp->initial_freq = rknpu_dev->current_freq;
1335		-
1336		-#ifdef CONFIG_PM_DEVFREQ
1337		- ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
1338		- if (ret) {
1339		- LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
1340		- goto err_remove_table;
1341		- }
1342		-#endif
1343		-
1344		- rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
1345		- (void *)rknpu_dev);
1346		- if (IS_ERR(rknpu_dev->devfreq)) {
1347		- LOG_DEV_ERROR(dev, "failed to add devfreq\n");
1348		- ret = PTR_ERR(rknpu_dev->devfreq);
1349		- goto err_remove_governor;
1350		- }
1351		- devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq);
1352		-
1353		- rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq;
1354		- rknpu_dev->devfreq->last_status.total_time = 1;
1355		- rknpu_dev->devfreq->last_status.busy_time = 1;
1356		-
1357		- npu_mdevp.data = rknpu_dev->devfreq;
1358		- rknpu_dev->mdev_info =
1359		- rockchip_system_monitor_register(dev, &npu_mdevp);
1360		- if (IS_ERR(rknpu_dev->mdev_info)) {
1361		- LOG_DEV_DEBUG(dev, "without system monitor\n");
1362		- rknpu_dev->mdev_info = NULL;
1363		- }
1364		- rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1365		- rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
1366		-
1367		- of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
1368		- (u32 *)&npu_cooling_power.dyn_power_coeff);
1369		- rknpu_dev->model_data =
1370		- rockchip_ipa_power_model_init(dev, "npu_leakage");
1371		- if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
1372		- rknpu_dev->model_data = NULL;
1373		- LOG_DEV_ERROR(dev, "failed to initialize power model\n");
1374		- } else if (rknpu_dev->model_data->dynamic_coefficient) {
1375		- npu_cooling_power.dyn_power_coeff =
1376		- rknpu_dev->model_data->dynamic_coefficient;
1377		- }
1378		-
1379		- if (!npu_cooling_power.dyn_power_coeff) {
1380		- LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
1381		- goto out;
1382		- }
1383		-
1384		- rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
1385		- dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
1386		- if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
1387		- LOG_DEV_ERROR(dev, "failed to register cooling device\n");
1388		-
1389		-out:
1390		- return 0;
1391		-
1392		-err_remove_governor:
1393		-#ifdef CONFIG_PM_DEVFREQ
1394		- devfreq_remove_governor(&devfreq_rknpu_ondemand);
1395		-#endif
1396		-err_remove_table:
1397		- dev_pm_opp_of_remove_table(dev);
1398		-
1399		- rknpu_dev->devfreq = NULL;
1400		-
1401		- return ret;
1402		-}
1403		-#endif
1404		-
1405		-static int rknpu_devfreq_remove(struct rknpu_device *rknpu_dev)
1406		-{
1407		- if (rknpu_dev->devfreq) {
1408		- devfreq_unregister_opp_notifier(rknpu_dev->dev,
1409		- rknpu_dev->devfreq);
1410		- dev_pm_opp_of_remove_table(rknpu_dev->dev);
1411		-#ifdef CONFIG_PM_DEVFREQ
1412		- devfreq_remove_governor(&devfreq_rknpu_ondemand);
1413		-#endif
1414		- }
1415		-
1416		- return 0;
1417		-}
1418		-
1419		-#endif
1420	949
1421	950	static int rknpu_register_irq(struct platform_device *pdev,
1422	951	struct rknpu_device *rknpu_dev)
..	..	@@ -1518,6 +1047,53 @@
1518	1047	return 0;
1519	1048	}
1520	1049
	1050	+static int rknpu_find_nbuf_resource(struct rknpu_device *rknpu_dev)
	1051	+{
	1052	+ struct device *dev = rknpu_dev->dev;
	1053	+
	1054	+ if (rknpu_dev->config->nbuf_size == 0)
	1055	+ return -EINVAL;
	1056	+
	1057	+ rknpu_dev->nbuf_start = rknpu_dev->config->nbuf_phyaddr;
	1058	+ rknpu_dev->nbuf_size = rknpu_dev->config->nbuf_size;
	1059	+ rknpu_dev->nbuf_base_io =
	1060	+ devm_ioremap(dev, rknpu_dev->nbuf_start, rknpu_dev->nbuf_size);
	1061	+ if (IS_ERR(rknpu_dev->nbuf_base_io)) {
	1062	+ LOG_DEV_ERROR(dev, "failed to remap nbuf base io!\n");
	1063	+ rknpu_dev->nbuf_base_io = NULL;
	1064	+ }
	1065	+
	1066	+ rknpu_dev->nbuf_end = rknpu_dev->nbuf_start + rknpu_dev->nbuf_size;
	1067	+
	1068	+ LOG_DEV_INFO(dev, "nbuf region: [%pa, %pa), nbuf size: %#x\n",
	1069	+ &rknpu_dev->nbuf_start, &rknpu_dev->nbuf_end,
	1070	+ rknpu_dev->nbuf_size);
	1071	+
	1072	+ return 0;
	1073	+}
	1074	+
	1075	+static int rknpu_get_invalid_core_mask(struct device *dev)
	1076	+{
	1077	+ int ret = 0;
	1078	+ u8 invalid_core_mask = 0;
	1079	+
	1080	+ if (of_property_match_string(dev->of_node, "nvmem-cell-names",
	1081	+ "cores") >= 0) {
	1082	+ ret = rockchip_nvmem_cell_read_u8(dev->of_node, "cores",
	1083	+ &invalid_core_mask);
	1084	+ /* The default valid npu cores for RK3583 are core0 and core1 */
	1085	+ invalid_core_mask \|= RKNPU_CORE2_MASK;
	1086	+ if (ret) {
	1087	+ LOG_DEV_ERROR(
	1088	+ dev,
	1089	+ "failed to get specification_serial_number\n");
	1090	+ return invalid_core_mask;
	1091	+ }
	1092	+ }
	1093	+
	1094	+ return (int)invalid_core_mask;
	1095	+}
	1096	+
1521	1097	static int rknpu_probe(struct platform_device *pdev)
1522	1098	{
1523	1099	struct resource *res = NULL;
..	..	@@ -1549,6 +1125,22 @@
1549	1125	if (!config)
1550	1126	return -EINVAL;
1551	1127
	1128	+ if (match->data == (void *)&rk3588_rknpu_config) {
	1129	+ int invalid_core_mask = rknpu_get_invalid_core_mask(dev);
	1130	+ /* The default valid npu cores for RK3583 are core0 and core1 */
	1131	+ if (invalid_core_mask & RKNPU_CORE2_MASK) {
	1132	+ if ((invalid_core_mask & RKNPU_CORE0_MASK) \|\|
	1133	+ (invalid_core_mask & RKNPU_CORE1_MASK)) {
	1134	+ LOG_DEV_ERROR(
	1135	+ dev,
	1136	+ "rknpu core invalid, invalid core mask: %#x\n",
	1137	+ invalid_core_mask);
	1138	+ return -ENODEV;
	1139	+ }
	1140	+ config = &rk3583_rknpu_config;
	1141	+ }
	1142	+ }
	1143	+
1552	1144	rknpu_dev->config = config;
1553	1145	rknpu_dev->dev = dev;
1554	1146
..	..	@@ -1577,11 +1169,6 @@
1577	1169	}
1578	1170
1579	1171	#ifndef FPGA_PLATFORM
1580		-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
1581		- if (strstr(__clk_get_name(rknpu_dev->clks[0].clk), "scmi"))
1582		- rknpu_dev->opp_info.scmi_clk = rknpu_dev->clks[0].clk;
1583		-#endif
1584		-
1585	1172	rknpu_dev->vdd = devm_regulator_get_optional(dev, "rknpu");
1586	1173	if (IS_ERR(rknpu_dev->vdd)) {
1587	1174	if (PTR_ERR(rknpu_dev->vdd) != -ENODEV) {
..	..	@@ -1708,9 +1295,11 @@
1708	1295	virt_dev = dev_pm_domain_attach_by_name(dev, "npu1");
1709	1296	if (!IS_ERR(virt_dev))
1710	1297	rknpu_dev->genpd_dev_npu1 = virt_dev;
1711		- virt_dev = dev_pm_domain_attach_by_name(dev, "npu2");
1712		- if (!IS_ERR(virt_dev))
1713		- rknpu_dev->genpd_dev_npu2 = virt_dev;
	1298	+ if (config->num_irqs > 2) {
	1299	+ virt_dev = dev_pm_domain_attach_by_name(dev, "npu2");
	1300	+ if (!IS_ERR(virt_dev))
	1301	+ rknpu_dev->genpd_dev_npu2 = virt_dev;
	1302	+ }
1714	1303	rknpu_dev->multiple_domains = true;
1715	1304	}
1716	1305
..	..	@@ -1734,7 +1323,8 @@
1734	1323	INIT_DEFERRABLE_WORK(&rknpu_dev->power_off_work,
1735	1324	rknpu_power_off_delay_work);
1736	1325
1737		- if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) {
	1326	+ if (IS_ENABLED(CONFIG_NO_GKI) &&
	1327	+ IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) {
1738	1328	if (!rknpu_find_sram_resource(rknpu_dev)) {
1739	1329	ret = rknpu_mm_create(rknpu_dev->sram_size, PAGE_SIZE,
1740	1330	&rknpu_dev->sram_mm);
..	..	@@ -1744,6 +1334,10 @@
1744	1334	LOG_DEV_WARN(dev, "could not find sram resource!\n");
1745	1335	}
1746	1336	}
	1337	+
	1338	+ if (IS_ENABLED(CONFIG_NO_GKI) && rknpu_dev->iommu_en &&
	1339	+ rknpu_dev->config->nbuf_size > 0)
	1340	+ rknpu_find_nbuf_resource(rknpu_dev);
1747	1341
1748	1342	rknpu_power_off(rknpu_dev);
1749	1343	atomic_set(&rknpu_dev->power_refcount, 0);
..	..	@@ -1823,47 +1417,14 @@
1823	1417	}
1824	1418
1825	1419	#ifndef FPGA_PLATFORM
1826		-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
1827	1420	static int rknpu_runtime_suspend(struct device *dev)
1828	1421	{
1829		- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1830		- struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
1831		-
1832		- if (opp_info->scmi_clk) {
1833		- if (clk_set_rate(opp_info->scmi_clk, POWER_DOWN_FREQ))
1834		- LOG_DEV_ERROR(dev, "failed to restore clk rate\n");
1835		- }
1836		- opp_info->current_rm = UINT_MAX;
1837		-
1838		- return 0;
	1422	+ return rknpu_devfreq_runtime_suspend(dev);
1839	1423	}
1840	1424
1841	1425	static int rknpu_runtime_resume(struct device *dev)
1842	1426	{
1843		- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1844		- struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
1845		- int ret = 0;
1846		-
1847		- if (!rknpu_dev->current_freq \|\| !rknpu_dev->current_volt)
1848		- return 0;
1849		-
1850		- ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
1851		- if (ret) {
1852		- LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
1853		- return ret;
1854		- }
1855		-
1856		- if (opp_info->data && opp_info->data->set_read_margin)
1857		- opp_info->data->set_read_margin(dev, opp_info,
1858		- opp_info->target_rm);
1859		- if (opp_info->scmi_clk) {
1860		- if (clk_set_rate(opp_info->scmi_clk, rknpu_dev->current_freq))
1861		- LOG_DEV_ERROR(dev, "failed to set power down rate\n");
1862		- }
1863		-
1864		- clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
1865		-
1866		- return ret;
	1427	+ return rknpu_devfreq_runtime_resume(dev);
1867	1428	}
1868	1429
1869	1430	static const struct dev_pm_ops rknpu_pm_ops = {
..	..	@@ -1873,7 +1434,6 @@
1873	1434	NULL)
1874	1435	};
1875	1436	#endif
1876		-#endif
1877	1437
1878	1438	static struct platform_driver rknpu_driver = {
1879	1439	.probe = rknpu_probe,
..	..	@@ -1882,9 +1442,7 @@
1882	1442	.owner = THIS_MODULE,
1883	1443	.name = "RKNPU",
1884	1444	#ifndef FPGA_PLATFORM
1885		-#if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
1886	1445	.pm = &rknpu_pm_ops,
1887		-#endif
1888	1446	#endif
1889	1447	.of_match_table = of_match_ptr(rknpu_of_match),
1890	1448	},
..	..	@@ -1909,3 +1467,6 @@
1909	1467	MODULE_LICENSE("GPL v2");
1910	1468	MODULE_VERSION(RKNPU_GET_DRV_VERSION_STRING(DRIVER_MAJOR, DRIVER_MINOR,
1911	1469	DRIVER_PATCHLEVEL));
	1470	+#if KERNEL_VERSION(5, 16, 0) < LINUX_VERSION_CODE
	1471	+MODULE_IMPORT_NS(DMA_BUF);
	1472	+#endif