forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/drivers/rknpu/rknpu_drv.c
....@@ -31,19 +31,11 @@
3131 #include <linux/clk-provider.h>
3232 #include <linux/pm_domain.h>
3333 #include <linux/pm_runtime.h>
34
-#include <linux/devfreq_cooling.h>
3534 #include <linux/regmap.h>
36
-#include <linux/dma-iommu.h>
3735 #include <linux/of_address.h>
3836 
3937 #ifndef FPGA_PLATFORM
4038 #include <soc/rockchip/rockchip_iommu.h>
41
-#include <soc/rockchip/rockchip_opp_select.h>
42
-#include <soc/rockchip/rockchip_system_monitor.h>
43
-#include <soc/rockchip/rockchip_ipa.h>
44
-#ifdef CONFIG_PM_DEVFREQ
45
-#include <../drivers/devfreq/governor.h>
46
-#endif
4739 #endif
4840 
4941 #include "rknpu_ioctl.h"
....@@ -51,6 +43,7 @@
5143 #include "rknpu_fence.h"
5244 #include "rknpu_drv.h"
5345 #include "rknpu_gem.h"
46
+#include "rknpu_devfreq.h"
5447 
5548 #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
5649 #include <drm/drm_device.h>
....@@ -79,36 +72,29 @@
7972 MODULE_PARM_DESC(bypass_soft_reset,
8073 		 "bypass RKNPU soft reset if set it to 1, disabled by default");
8174 
82
-struct npu_irqs_data {
75
+struct rknpu_irqs_data {
8376 	const char *name;
8477 	irqreturn_t (*irq_hdl)(int irq, void *ctx);
8578 };
8679 
87
-static const struct npu_irqs_data rk356x_npu_irqs[] = {
80
+static const struct rknpu_irqs_data rknpu_irqs[] = {
8881 	{ "npu_irq", rknpu_core0_irq_handler }
8982 };
9083 
91
-static const struct npu_irqs_data rk3588_npu_irqs[] = {
84
+static const struct rknpu_irqs_data rk3588_npu_irqs[] = {
9285 	{ "npu0_irq", rknpu_core0_irq_handler },
9386 	{ "npu1_irq", rknpu_core1_irq_handler },
9487 	{ "npu2_irq", rknpu_core2_irq_handler }
9588 };
9689 
97
-static const struct npu_irqs_data rv110x_npu_irqs[] = {
98
-	{ "npu_irq", rknpu_core0_irq_handler }
99
-};
90
+static const struct rknpu_reset_data rknpu_resets[] = { { "srst_a",
91
+							  "srst_h" } };
10092 
101
-static const struct npu_reset_data rk356x_npu_resets[] = { { "srst_a",
102
-							     "srst_h" } };
103
-
104
-static const struct npu_reset_data rk3588_npu_resets[] = {
93
+static const struct rknpu_reset_data rk3588_npu_resets[] = {
10594 	{ "srst_a0", "srst_h0" },
10695 	{ "srst_a1", "srst_h1" },
10796 	{ "srst_a2", "srst_h2" }
10897 };
109
-
110
-static const struct npu_reset_data rv110x_npu_resets[] = { { "srst_a",
111
-							     "srst_h" } };
11298 
11399 static const struct rknpu_config rk356x_rknpu_config = {
114100 	.bw_priority_addr = 0xfe180008,
....@@ -117,11 +103,17 @@
117103 	.pc_data_amount_scale = 1,
118104 	.pc_task_number_bits = 12,
119105 	.pc_task_number_mask = 0xfff,
106
+	.pc_task_status_offset = 0x3c,
107
+	.pc_dma_ctrl = 0,
120108 	.bw_enable = 1,
121
-	.irqs = rk356x_npu_irqs,
122
-	.resets = rk356x_npu_resets,
123
-	.num_irqs = ARRAY_SIZE(rk356x_npu_irqs),
124
-	.num_resets = ARRAY_SIZE(rk356x_npu_resets)
109
+	.irqs = rknpu_irqs,
110
+	.resets = rknpu_resets,
111
+	.num_irqs = ARRAY_SIZE(rknpu_irqs),
112
+	.num_resets = ARRAY_SIZE(rknpu_resets),
113
+	.nbuf_phyaddr = 0,
114
+	.nbuf_size = 0,
115
+	.max_submit_number = (1 << 12) - 1,
116
+	.core_mask = 0x1,
125117 };
126118 
127119 static const struct rknpu_config rk3588_rknpu_config = {
....@@ -131,11 +123,37 @@
131123 	.pc_data_amount_scale = 2,
132124 	.pc_task_number_bits = 12,
133125 	.pc_task_number_mask = 0xfff,
126
+	.pc_task_status_offset = 0x3c,
127
+	.pc_dma_ctrl = 0,
134128 	.bw_enable = 0,
135129 	.irqs = rk3588_npu_irqs,
136130 	.resets = rk3588_npu_resets,
137131 	.num_irqs = ARRAY_SIZE(rk3588_npu_irqs),
138
-	.num_resets = ARRAY_SIZE(rk3588_npu_resets)
132
+	.num_resets = ARRAY_SIZE(rk3588_npu_resets),
133
+	.nbuf_phyaddr = 0,
134
+	.nbuf_size = 0,
135
+	.max_submit_number = (1 << 12) - 1,
136
+	.core_mask = 0x7,
137
+};
138
+
139
+static const struct rknpu_config rk3583_rknpu_config = {
140
+	.bw_priority_addr = 0x0,
141
+	.bw_priority_length = 0x0,
142
+	.dma_mask = DMA_BIT_MASK(40),
143
+	.pc_data_amount_scale = 2,
144
+	.pc_task_number_bits = 12,
145
+	.pc_task_number_mask = 0xfff,
146
+	.pc_task_status_offset = 0x3c,
147
+	.pc_dma_ctrl = 0,
148
+	.bw_enable = 0,
149
+	.irqs = rk3588_npu_irqs,
150
+	.resets = rk3588_npu_resets,
151
+	.num_irqs = 2,
152
+	.num_resets = 2,
153
+	.nbuf_phyaddr = 0,
154
+	.nbuf_size = 0,
155
+	.max_submit_number = (1 << 12) - 1,
156
+	.core_mask = 0x3,
139157 };
140158 
141159 static const struct rknpu_config rv1106_rknpu_config = {
....@@ -145,11 +163,37 @@
145163 	.pc_data_amount_scale = 2,
146164 	.pc_task_number_bits = 16,
147165 	.pc_task_number_mask = 0xffff,
166
+	.pc_task_status_offset = 0x3c,
167
+	.pc_dma_ctrl = 0,
148168 	.bw_enable = 1,
149
-	.irqs = rv110x_npu_irqs,
150
-	.resets = rv110x_npu_resets,
151
-	.num_irqs = ARRAY_SIZE(rv110x_npu_irqs),
152
-	.num_resets = ARRAY_SIZE(rv110x_npu_resets)
169
+	.irqs = rknpu_irqs,
170
+	.resets = rknpu_resets,
171
+	.num_irqs = ARRAY_SIZE(rknpu_irqs),
172
+	.num_resets = ARRAY_SIZE(rknpu_resets),
173
+	.nbuf_phyaddr = 0,
174
+	.nbuf_size = 0,
175
+	.max_submit_number = (1 << 16) - 1,
176
+	.core_mask = 0x1,
177
+};
178
+
179
+static const struct rknpu_config rk3562_rknpu_config = {
180
+	.bw_priority_addr = 0x0,
181
+	.bw_priority_length = 0x0,
182
+	.dma_mask = DMA_BIT_MASK(40),
183
+	.pc_data_amount_scale = 2,
184
+	.pc_task_number_bits = 16,
185
+	.pc_task_number_mask = 0xffff,
186
+	.pc_task_status_offset = 0x48,
187
+	.pc_dma_ctrl = 1,
188
+	.bw_enable = 1,
189
+	.irqs = rknpu_irqs,
190
+	.resets = rknpu_resets,
191
+	.num_irqs = ARRAY_SIZE(rknpu_irqs),
192
+	.num_resets = ARRAY_SIZE(rknpu_resets),
193
+	.nbuf_phyaddr = 0xfe400000,
194
+	.nbuf_size = 256 * 1024,
195
+	.max_submit_number = (1 << 16) - 1,
196
+	.core_mask = 0x1,
153197 };
154198 
155199 /* driver probe and init */
....@@ -169,6 +213,10 @@
169213 	{
170214 		.compatible = "rockchip,rv1106-rknpu",
171215 		.data = &rv1106_rknpu_config,
216
+	},
217
+	{
218
+		.compatible = "rockchip,rk3562-rknpu",
219
+		.data = &rk3562_rknpu_config,
172220 	},
173221 	{},
174222 };
....@@ -198,7 +246,6 @@
198246 {
199247 	int ret = 0;
200248 
201
-	cancel_delayed_work(&rknpu_dev->power_off_work);
202249 	mutex_lock(&rknpu_dev->power_lock);
203250 	if (atomic_inc_return(&rknpu_dev->power_refcount) == 1)
204251 		ret = rknpu_power_on(rknpu_dev);
....@@ -221,6 +268,9 @@
221268 
222269 static int rknpu_power_put_delay(struct rknpu_device *rknpu_dev)
223270 {
271
+	if (rknpu_dev->power_put_delay == 0)
272
+		return rknpu_power_put(rknpu_dev);
273
+
224274 	mutex_lock(&rknpu_dev->power_lock);
225275 	if (atomic_read(&rknpu_dev->power_refcount) == 1)
226276 		queue_delayed_work(
....@@ -229,6 +279,7 @@
229279 	else
230280 		atomic_dec_if_positive(&rknpu_dev->power_refcount);
231281 	mutex_unlock(&rknpu_dev->power_lock);
282
+
232283 	return 0;
233284 }
234285 
....@@ -245,13 +296,17 @@
245296 		ret = rknpu_get_drv_version(&args->value);
246297 		break;
247298 	case RKNPU_GET_FREQ:
299
+#ifndef FPGA_PLATFORM
248300 		args->value = clk_get_rate(rknpu_dev->clks[0].clk);
301
+#endif
249302 		ret = 0;
250303 		break;
251304 	case RKNPU_SET_FREQ:
252305 		break;
253306 	case RKNPU_GET_VOLT:
307
+#ifndef FPGA_PLATFORM
254308 		args->value = regulator_get_voltage(rknpu_dev->vdd);
309
+#endif
255310 		ret = 0;
256311 		break;
257312 	case RKNPU_SET_VOLT:
....@@ -330,11 +385,56 @@
330385 #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
331386 static int rknpu_open(struct inode *inode, struct file *file)
332387 {
388
+	struct rknpu_device *rknpu_dev =
389
+		container_of(file->private_data, struct rknpu_device, miscdev);
390
+	struct rknpu_session *session = NULL;
391
+
392
+	session = kzalloc(sizeof(*session), GFP_KERNEL);
393
+	if (!session) {
394
+		LOG_ERROR("rknpu session alloc failed\n");
395
+		return -ENOMEM;
396
+	}
397
+
398
+	session->rknpu_dev = rknpu_dev;
399
+	INIT_LIST_HEAD(&session->list);
400
+
401
+	file->private_data = (void *)session;
402
+
333403 	return nonseekable_open(inode, file);
334404 }
335405 
336406 static int rknpu_release(struct inode *inode, struct file *file)
337407 {
408
+	struct rknpu_mem_object *entry;
409
+	struct rknpu_session *session = file->private_data;
410
+	struct rknpu_device *rknpu_dev = session->rknpu_dev;
411
+	LIST_HEAD(local_list);
412
+
413
+	spin_lock(&rknpu_dev->lock);
414
+	list_replace_init(&session->list, &local_list);
415
+	file->private_data = NULL;
416
+	spin_unlock(&rknpu_dev->lock);
417
+
418
+	while (!list_empty(&local_list)) {
419
+		entry = list_first_entry(&local_list, struct rknpu_mem_object,
420
+					 head);
421
+
422
+		LOG_DEBUG(
423
+			"Fd close free rknpu_obj: %#llx, rknpu_obj->dma_addr: %#llx\n",
424
+			(__u64)(uintptr_t)entry, (__u64)entry->dma_addr);
425
+
426
+		vunmap(entry->kv_addr);
427
+		entry->kv_addr = NULL;
428
+
429
+		if (!entry->owner)
430
+			dma_buf_put(entry->dmabuf);
431
+
432
+		list_del(&entry->head);
433
+		kfree(entry);
434
+	}
435
+
436
+	kfree(session);
437
+
338438 	return 0;
339439 }
340440 
....@@ -366,8 +466,12 @@
366466 static long rknpu_ioctl(struct file *file, uint32_t cmd, unsigned long arg)
367467 {
368468 	long ret = -EINVAL;
369
-	struct rknpu_device *rknpu_dev =
370
-		container_of(file->private_data, struct rknpu_device, miscdev);
469
+	struct rknpu_device *rknpu_dev = NULL;
470
+
471
+	if (!file->private_data)
472
+		return -EINVAL;
473
+
474
+	rknpu_dev = ((struct rknpu_session *)file->private_data)->rknpu_dev;
371475 
372476 	rknpu_power_get(rknpu_dev);
373477 
....@@ -379,12 +483,12 @@
379483 		ret = rknpu_submit_ioctl(rknpu_dev, arg);
380484 		break;
381485 	case IOCTL_RKNPU_MEM_CREATE:
382
-		ret = rknpu_mem_create_ioctl(rknpu_dev, arg);
486
+		ret = rknpu_mem_create_ioctl(rknpu_dev, arg, file);
383487 		break;
384488 	case RKNPU_MEM_MAP:
385489 		break;
386490 	case IOCTL_RKNPU_MEM_DESTROY:
387
-		ret = rknpu_mem_destroy_ioctl(rknpu_dev, arg);
491
+		ret = rknpu_mem_destroy_ioctl(rknpu_dev, arg, file);
388492 		break;
389493 	case IOCTL_RKNPU_MEM_SYNC:
390494 		ret = rknpu_mem_sync_ioctl(rknpu_dev, arg);
....@@ -409,11 +513,13 @@
409513 #endif
410514 
411515 #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
516
+#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
412517 static const struct vm_operations_struct rknpu_gem_vm_ops = {
413518 	.fault = rknpu_gem_fault,
414519 	.open = drm_gem_vm_open,
415520 	.close = drm_gem_vm_close,
416521 };
522
+#endif
417523 
418524 static int rknpu_action_ioctl(struct drm_device *dev, void *data,
419525 			      struct drm_file *file_priv)
....@@ -455,6 +561,9 @@
455561 			  DRM_RENDER_ALLOW),
456562 };
457563 
564
+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
565
+DEFINE_DRM_GEM_FOPS(rknpu_drm_driver_fops);
566
+#else
458567 static const struct file_operations rknpu_drm_driver_fops = {
459568 	.owner = THIS_MODULE,
460569 	.open = drm_open,
....@@ -468,6 +577,7 @@
468577 	.release = drm_release,
469578 	.llseek = noop_llseek,
470579 };
580
+#endif
471581 
472582 static struct drm_driver rknpu_drm_driver = {
473583 #if KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE
....@@ -475,28 +585,34 @@
475585 #else
476586 	.driver_features = DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER,
477587 #endif
588
+#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
478589 	.gem_free_object_unlocked = rknpu_gem_free_object,
479590 	.gem_vm_ops = &rknpu_gem_vm_ops,
591
+	.dumb_destroy = drm_gem_dumb_destroy,
592
+	.gem_prime_export = drm_gem_prime_export,
593
+	.gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table,
594
+	.gem_prime_vmap = rknpu_gem_prime_vmap,
595
+	.gem_prime_vunmap = rknpu_gem_prime_vunmap,
596
+#endif
480597 	.dumb_create = rknpu_gem_dumb_create,
481598 #if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
482599 	.dumb_map_offset = rknpu_gem_dumb_map_offset,
483600 #else
484601 	.dumb_map_offset = drm_gem_dumb_map_offset,
485602 #endif
486
-	.dumb_destroy = drm_gem_dumb_destroy,
487603 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
488604 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
489
-	.gem_prime_export = drm_gem_prime_export,
490605 #if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
491606 	.gem_prime_import = rknpu_gem_prime_import,
492607 #else
493608 	.gem_prime_import = drm_gem_prime_import,
494609 #endif
495
-	.gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table,
496610 	.gem_prime_import_sg_table = rknpu_gem_prime_import_sg_table,
497
-	.gem_prime_vmap = rknpu_gem_prime_vmap,
498
-	.gem_prime_vunmap = rknpu_gem_prime_vunmap,
611
+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
612
+	.gem_prime_mmap = drm_gem_prime_mmap,
613
+#else
499614 	.gem_prime_mmap = rknpu_gem_prime_mmap,
615
+#endif
500616 	.ioctls = rknpu_ioctls,
501617 	.num_ioctls = ARRAY_SIZE(rknpu_ioctls),
502618 	.fops = &rknpu_drm_driver_fops,
....@@ -516,7 +632,7 @@
516632 		container_of(timer, struct rknpu_device, timer);
517633 	struct rknpu_subcore_data *subcore_data = NULL;
518634 	struct rknpu_job *job = NULL;
519
-	ktime_t now = ktime_get();
635
+	ktime_t now;
520636 	unsigned long flags;
521637 	int i;
522638 
....@@ -527,14 +643,16 @@
527643 
528644 		job = subcore_data->job;
529645 		if (job) {
646
+			now = ktime_get();
530647 			subcore_data->timer.busy_time +=
531
-				ktime_us_delta(now, job->hw_recoder_time);
532
-			job->hw_recoder_time = ktime_get();
648
+				ktime_sub(now, job->hw_recoder_time);
649
+			job->hw_recoder_time = now;
533650 		}
534651 
535
-		subcore_data->timer.busy_time_record =
652
+		subcore_data->timer.total_busy_time =
536653 			subcore_data->timer.busy_time;
537654 		subcore_data->timer.busy_time = 0;
655
+
538656 		spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
539657 	}
540658 
....@@ -581,6 +699,42 @@
581699 }
582700 
583701 #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
702
+static int drm_fake_dev_register(struct rknpu_device *rknpu_dev)
703
+{
704
+	const struct platform_device_info rknpu_dev_info = {
705
+		.name = "rknpu_dev",
706
+		.id = PLATFORM_DEVID_AUTO,
707
+		.dma_mask = rknpu_dev->config->dma_mask,
708
+	};
709
+	struct platform_device *pdev = NULL;
710
+	int ret = -EINVAL;
711
+
712
+	pdev = platform_device_register_full(&rknpu_dev_info);
713
+	if (pdev) {
714
+		ret = of_dma_configure(&pdev->dev, NULL, true);
715
+		if (ret) {
716
+			platform_device_unregister(pdev);
717
+			pdev = NULL;
718
+		}
719
+	}
720
+
721
+	rknpu_dev->fake_dev = pdev ? &pdev->dev : NULL;
722
+
723
+	return ret;
724
+}
725
+
726
+static void drm_fake_dev_unregister(struct rknpu_device *rknpu_dev)
727
+{
728
+	struct platform_device *pdev = NULL;
729
+
730
+	if (!rknpu_dev->fake_dev)
731
+		return;
732
+
733
+	pdev = to_platform_device(rknpu_dev->fake_dev);
734
+
735
+	platform_device_unregister(pdev);
736
+}
737
+
584738 static int rknpu_drm_probe(struct rknpu_device *rknpu_dev)
585739 {
586740 	struct device *dev = rknpu_dev->dev;
....@@ -599,6 +753,8 @@
599753 	drm_dev->dev_private = rknpu_dev;
600754 	rknpu_dev->drm_dev = drm_dev;
601755 
756
+	drm_fake_dev_register(rknpu_dev);
757
+
602758 	return 0;
603759 
604760 err_free_drm:
....@@ -614,6 +770,8 @@
614770 static void rknpu_drm_remove(struct rknpu_device *rknpu_dev)
615771 {
616772 	struct drm_device *drm_dev = rknpu_dev->drm_dev;
773
+
774
+	drm_fake_dev_unregister(rknpu_dev);
617775 
618776 	drm_dev_unregister(drm_dev);
619777 
....@@ -662,9 +820,7 @@
662820 	}
663821 
664822 #ifndef FPGA_PLATFORM
665
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
666
-	rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
667
-#endif
823
+	rknpu_devfreq_lock(rknpu_dev);
668824 #endif
669825 
670826 	if (rknpu_dev->multiple_domains) {
....@@ -723,9 +879,7 @@
723879 
724880 out:
725881 #ifndef FPGA_PLATFORM
726
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
727
-	rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
728
-#endif
882
+	rknpu_devfreq_unlock(rknpu_dev);
729883 #endif
730884 
731885 	return ret;
....@@ -739,9 +893,7 @@
739893 	int ret;
740894 	bool val;
741895 
742
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
743
-	rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
744
-#endif
896
+	rknpu_devfreq_lock(rknpu_dev);
745897 #endif
746898 
747899 	pm_runtime_put_sync(dev);
....@@ -763,10 +915,7 @@
763915 		if (ret) {
764916 			LOG_DEV_ERROR(dev, "iommu still enabled\n");
765917 			pm_runtime_get_sync(dev);
766
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
767
-			rockchip_monitor_volt_adjust_unlock(
768
-				rknpu_dev->mdev_info);
769
-#endif
918
+			rknpu_devfreq_unlock(rknpu_dev);
770919 			return ret;
771920 		}
772921 #else
....@@ -782,9 +931,7 @@
782931 	}
783932 
784933 #ifndef FPGA_PLATFORM
785
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
786
-	rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
787
-#endif
934
+	rknpu_devfreq_unlock(rknpu_dev);
788935 #endif
789936 
790937 	clk_bulk_disable_unprepare(rknpu_dev->num_clks, rknpu_dev->clks);
....@@ -799,624 +946,6 @@
799946 
800947 	return 0;
801948 }
802
-
803
-#ifndef FPGA_PLATFORM
804
-static struct monitor_dev_profile npu_mdevp = {
805
-	.type = MONITOR_TPYE_DEV,
806
-	.low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
807
-	.high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
808
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
809
-	.update_volt = rockchip_monitor_check_rate_volt,
810
-#endif
811
-};
812
-
813
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
814
-static int npu_opp_helper(struct dev_pm_set_opp_data *data)
815
-{
816
-	struct device *dev = data->dev;
817
-	struct dev_pm_opp_supply *old_supply_vdd = &data->old_opp.supplies[0];
818
-	struct dev_pm_opp_supply *old_supply_mem = &data->old_opp.supplies[1];
819
-	struct dev_pm_opp_supply *new_supply_vdd = &data->new_opp.supplies[0];
820
-	struct dev_pm_opp_supply *new_supply_mem = &data->new_opp.supplies[1];
821
-	struct regulator *vdd_reg = data->regulators[0];
822
-	struct regulator *mem_reg = data->regulators[1];
823
-	struct clk *clk = data->clk;
824
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
825
-	struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
826
-	unsigned long old_freq = data->old_opp.rate;
827
-	unsigned long new_freq = data->new_opp.rate;
828
-	bool is_set_rm = true;
829
-	bool is_set_clk = true;
830
-	u32 target_rm = UINT_MAX;
831
-	int ret = 0;
832
-
833
-	if (!pm_runtime_active(dev)) {
834
-		is_set_rm = false;
835
-		if (opp_info->scmi_clk)
836
-			is_set_clk = false;
837
-	}
838
-
839
-	ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
840
-	if (ret < 0) {
841
-		LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
842
-		return ret;
843
-	}
844
-	rockchip_get_read_margin(dev, opp_info, new_supply_vdd->u_volt,
845
-				 &target_rm);
846
-
847
-	/* Change frequency */
848
-	LOG_DEV_DEBUG(dev, "switching OPP: %lu Hz --> %lu Hz\n", old_freq,
849
-		      new_freq);
850
-	/* Scaling up? Scale voltage before frequency */
851
-	if (new_freq >= old_freq) {
852
-		rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
853
-					       new_freq, true, is_set_clk);
854
-		ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
855
-					    INT_MAX);
856
-		if (ret) {
857
-			LOG_DEV_ERROR(dev,
858
-				      "failed to set volt %lu uV for mem reg\n",
859
-				      new_supply_mem->u_volt);
860
-			goto restore_voltage;
861
-		}
862
-		ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
863
-					    INT_MAX);
864
-		if (ret) {
865
-			LOG_DEV_ERROR(dev,
866
-				      "failed to set volt %lu uV for vdd reg\n",
867
-				      new_supply_vdd->u_volt);
868
-			goto restore_voltage;
869
-		}
870
-		rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
871
-		if (is_set_clk && clk_set_rate(clk, new_freq)) {
872
-			ret = -EINVAL;
873
-			LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
874
-			goto restore_rm;
875
-		}
876
-		/* Scaling down? Scale voltage after frequency */
877
-	} else {
878
-		rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
879
-					       new_freq, false, is_set_clk);
880
-		rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
881
-		if (is_set_clk && clk_set_rate(clk, new_freq)) {
882
-			ret = -EINVAL;
883
-			LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
884
-			goto restore_rm;
885
-		}
886
-		ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
887
-					    INT_MAX);
888
-		if (ret) {
889
-			LOG_DEV_ERROR(dev,
890
-				      "failed to set volt %lu uV for vdd reg\n",
891
-				      new_supply_vdd->u_volt);
892
-			goto restore_freq;
893
-		}
894
-		ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
895
-					    INT_MAX);
896
-		if (ret) {
897
-			LOG_DEV_ERROR(dev,
898
-				      "failed to set volt %lu uV for mem reg\n",
899
-				      new_supply_mem->u_volt);
900
-			goto restore_freq;
901
-		}
902
-	}
903
-
904
-	clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
905
-
906
-	return 0;
907
-
908
-restore_freq:
909
-	if (is_set_clk && clk_set_rate(clk, old_freq))
910
-		LOG_DEV_ERROR(dev, "failed to restore old-freq %lu Hz\n",
911
-			      old_freq);
912
-restore_rm:
913
-	rockchip_get_read_margin(dev, opp_info, old_supply_vdd->u_volt,
914
-				 &target_rm);
915
-	rockchip_set_read_margin(dev, opp_info, opp_info->current_rm,
916
-				 is_set_rm);
917
-restore_voltage:
918
-	regulator_set_voltage(mem_reg, old_supply_mem->u_volt, INT_MAX);
919
-	regulator_set_voltage(vdd_reg, old_supply_vdd->u_volt, INT_MAX);
920
-	clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
921
-
922
-	return ret;
923
-}
924
-
925
-static int npu_devfreq_target(struct device *dev, unsigned long *freq,
926
-			      u32 flags)
927
-{
928
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
929
-	struct dev_pm_opp *opp;
930
-	unsigned long opp_volt;
931
-	int ret = 0;
932
-
933
-	if (!npu_mdevp.is_checked)
934
-		return -EINVAL;
935
-
936
-	opp = devfreq_recommended_opp(dev, freq, flags);
937
-	if (IS_ERR(opp))
938
-		return PTR_ERR(opp);
939
-	opp_volt = dev_pm_opp_get_voltage(opp);
940
-	dev_pm_opp_put(opp);
941
-
942
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
943
-	rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
944
-#endif
945
-	ret = dev_pm_opp_set_rate(dev, *freq);
946
-	if (!ret) {
947
-		rknpu_dev->current_freq = *freq;
948
-		if (rknpu_dev->devfreq)
949
-			rknpu_dev->devfreq->last_status.current_frequency =
950
-				*freq;
951
-		rknpu_dev->current_volt = opp_volt;
952
-		LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n",
953
-			     rknpu_dev->current_freq, rknpu_dev->current_volt);
954
-	}
955
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
956
-	rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
957
-#endif
958
-
959
-	return ret;
960
-}
961
-
962
-#else
963
-
964
-static int npu_devfreq_target(struct device *dev, unsigned long *target_freq,
965
-			      u32 flags)
966
-{
967
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
968
-	struct dev_pm_opp *opp = NULL;
969
-	unsigned long freq = *target_freq;
970
-	unsigned long old_freq = rknpu_dev->current_freq;
971
-	unsigned long volt, old_volt = rknpu_dev->current_volt;
972
-	int ret = -EINVAL;
973
-
974
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
975
-	rcu_read_lock();
976
-#endif
977
-
978
-	opp = devfreq_recommended_opp(dev, &freq, flags);
979
-	if (IS_ERR(opp)) {
980
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
981
-		rcu_read_unlock();
982
-#endif
983
-		LOG_DEV_ERROR(dev, "failed to get opp (%ld)\n", PTR_ERR(opp));
984
-		return PTR_ERR(opp);
985
-	}
986
-	volt = dev_pm_opp_get_voltage(opp);
987
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
988
-	rcu_read_unlock();
989
-#endif
990
-
991
-	/*
992
-	 * Only update if there is a change of frequency
993
-	 */
994
-	if (old_freq == freq) {
995
-		*target_freq = freq;
996
-		if (old_volt == volt)
997
-			return 0;
998
-		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
999
-		if (ret) {
1000
-			LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
1001
-			return ret;
1002
-		}
1003
-		rknpu_dev->current_volt = volt;
1004
-		return 0;
1005
-	}
1006
-
1007
-	if (rknpu_dev->vdd && old_volt != volt && old_freq < freq) {
1008
-		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1009
-		if (ret) {
1010
-			LOG_DEV_ERROR(dev, "failed to increase volt %lu\n",
1011
-				      volt);
1012
-			return ret;
1013
-		}
1014
-	}
1015
-	LOG_DEV_DEBUG(dev, "%luHz %luuV -> %luHz %luuV\n", old_freq, old_volt,
1016
-		      freq, volt);
1017
-	ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
1018
-	if (ret) {
1019
-		LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
1020
-		return ret;
1021
-	}
1022
-	*target_freq = freq;
1023
-	rknpu_dev->current_freq = freq;
1024
-
1025
-	if (rknpu_dev->devfreq)
1026
-		rknpu_dev->devfreq->last_status.current_frequency = freq;
1027
-
1028
-	if (rknpu_dev->vdd && old_volt != volt && old_freq > freq) {
1029
-		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1030
-		if (ret) {
1031
-			LOG_DEV_ERROR(dev, "failed to decrease volt %lu\n",
1032
-				      volt);
1033
-			return ret;
1034
-		}
1035
-	}
1036
-	rknpu_dev->current_volt = volt;
1037
-
1038
-	LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n",
1039
-		     rknpu_dev->current_freq, rknpu_dev->current_volt);
1040
-
1041
-	return ret;
1042
-}
1043
-#endif
1044
-
1045
-static int npu_devfreq_get_dev_status(struct device *dev,
1046
-				      struct devfreq_dev_status *stat)
1047
-{
1048
-	return 0;
1049
-}
1050
-
1051
-static int npu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
1052
-{
1053
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1054
-
1055
-	*freq = rknpu_dev->current_freq;
1056
-
1057
-	return 0;
1058
-}
1059
-
1060
-static struct devfreq_dev_profile npu_devfreq_profile = {
1061
-	.polling_ms = 50,
1062
-	.target = npu_devfreq_target,
1063
-	.get_dev_status = npu_devfreq_get_dev_status,
1064
-	.get_cur_freq = npu_devfreq_get_cur_freq,
1065
-};
1066
-
1067
-#ifdef CONFIG_PM_DEVFREQ
1068
-static int devfreq_rknpu_ondemand_func(struct devfreq *df, unsigned long *freq)
1069
-{
1070
-	struct rknpu_device *rknpu_dev = df->data;
1071
-
1072
-	if (rknpu_dev)
1073
-		*freq = rknpu_dev->ondemand_freq;
1074
-	else
1075
-		*freq = df->previous_freq;
1076
-
1077
-	return 0;
1078
-}
1079
-
1080
-static int devfreq_rknpu_ondemand_handler(struct devfreq *devfreq,
1081
-					  unsigned int event, void *data)
1082
-{
1083
-	return 0;
1084
-}
1085
-
1086
-static struct devfreq_governor devfreq_rknpu_ondemand = {
1087
-	.name = "rknpu_ondemand",
1088
-	.get_target_freq = devfreq_rknpu_ondemand_func,
1089
-	.event_handler = devfreq_rknpu_ondemand_handler,
1090
-};
1091
-#endif
1092
-
1093
-static unsigned long npu_get_static_power(struct devfreq *devfreq,
1094
-					  unsigned long voltage)
1095
-{
1096
-	struct device *dev = devfreq->dev.parent;
1097
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1098
-
1099
-	if (!rknpu_dev->model_data)
1100
-		return 0;
1101
-
1102
-	return rockchip_ipa_get_static_power(rknpu_dev->model_data, voltage);
1103
-}
1104
-
1105
-static struct devfreq_cooling_power npu_cooling_power = {
1106
-	.get_static_power = &npu_get_static_power,
1107
-};
1108
-
1109
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
1110
-static int rk3588_npu_set_read_margin(struct device *dev,
1111
-				      struct rockchip_opp_info *opp_info,
1112
-				      u32 rm)
1113
-{
1114
-	u32 offset = 0, val = 0;
1115
-	int i, ret = 0;
1116
-
1117
-	if (!opp_info->grf || !opp_info->volt_rm_tbl)
1118
-		return 0;
1119
-
1120
-	if (rm == opp_info->current_rm || rm == UINT_MAX)
1121
-		return 0;
1122
-
1123
-	LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
1124
-
1125
-	for (i = 0; i < 3; i++) {
1126
-		ret = regmap_read(opp_info->grf, offset, &val);
1127
-		if (ret < 0) {
1128
-			LOG_DEV_ERROR(dev, "failed to get rm from 0x%x\n",
1129
-				      offset);
1130
-			return ret;
1131
-		}
1132
-		val &= ~0x1c;
1133
-		regmap_write(opp_info->grf, offset, val | (rm << 2));
1134
-		offset += 4;
1135
-	}
1136
-	opp_info->current_rm = rm;
1137
-
1138
-	return 0;
1139
-}
1140
-
1141
-static const struct rockchip_opp_data rk3588_npu_opp_data = {
1142
-	.set_read_margin = rk3588_npu_set_read_margin,
1143
-};
1144
-
1145
-static const struct of_device_id rockchip_npu_of_match[] = {
1146
-	{
1147
-		.compatible = "rockchip,rk3588",
1148
-		.data = (void *)&rk3588_npu_opp_data,
1149
-	},
1150
-	{},
1151
-};
1152
-
1153
-static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
1154
-{
1155
-	struct device *dev = rknpu_dev->dev;
1156
-	struct devfreq_dev_profile *dp = &npu_devfreq_profile;
1157
-	struct dev_pm_opp *opp;
1158
-	struct opp_table *reg_table = NULL;
1159
-	struct opp_table *opp_table = NULL;
1160
-	const char *const reg_names[] = { "rknpu", "mem" };
1161
-	int ret = -EINVAL;
1162
-
1163
-	if (of_find_property(dev->of_node, "rknpu-supply", NULL) &&
1164
-	    of_find_property(dev->of_node, "mem-supply", NULL)) {
1165
-		reg_table = dev_pm_opp_set_regulators(dev, reg_names, 2);
1166
-		if (IS_ERR(reg_table))
1167
-			return PTR_ERR(reg_table);
1168
-		opp_table =
1169
-			dev_pm_opp_register_set_opp_helper(dev, npu_opp_helper);
1170
-		if (IS_ERR(opp_table)) {
1171
-			dev_pm_opp_put_regulators(reg_table);
1172
-			return PTR_ERR(opp_table);
1173
-		}
1174
-	} else {
1175
-		reg_table = dev_pm_opp_set_regulators(dev, reg_names, 1);
1176
-		if (IS_ERR(reg_table))
1177
-			return PTR_ERR(reg_table);
1178
-	}
1179
-
1180
-	rockchip_get_opp_data(rockchip_npu_of_match, &rknpu_dev->opp_info);
1181
-	ret = rockchip_init_opp_table(dev, &rknpu_dev->opp_info, "npu_leakage",
1182
-				      "rknpu");
1183
-	if (ret) {
1184
-		LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
1185
-		return ret;
1186
-	}
1187
-
1188
-	rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1189
-
1190
-	opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0);
1191
-	if (IS_ERR(opp)) {
1192
-		ret = PTR_ERR(opp);
1193
-		goto err_remove_table;
1194
-	}
1195
-	dev_pm_opp_put(opp);
1196
-	dp->initial_freq = rknpu_dev->current_freq;
1197
-
1198
-#ifdef CONFIG_PM_DEVFREQ
1199
-	ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
1200
-	if (ret) {
1201
-		LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
1202
-		goto err_remove_table;
1203
-	}
1204
-#endif
1205
-
1206
-	rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
1207
-						     (void *)rknpu_dev);
1208
-	if (IS_ERR(rknpu_dev->devfreq)) {
1209
-		LOG_DEV_ERROR(dev, "failed to add devfreq\n");
1210
-		ret = PTR_ERR(rknpu_dev->devfreq);
1211
-		goto err_remove_governor;
1212
-	}
1213
-	devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq);
1214
-
1215
-	rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq;
1216
-	rknpu_dev->devfreq->last_status.total_time = 1;
1217
-	rknpu_dev->devfreq->last_status.busy_time = 1;
1218
-
1219
-	npu_mdevp.data = rknpu_dev->devfreq;
1220
-	npu_mdevp.opp_info = &rknpu_dev->opp_info;
1221
-	rknpu_dev->mdev_info =
1222
-		rockchip_system_monitor_register(dev, &npu_mdevp);
1223
-	if (IS_ERR(rknpu_dev->mdev_info)) {
1224
-		LOG_DEV_DEBUG(dev, "without system monitor\n");
1225
-		rknpu_dev->mdev_info = NULL;
1226
-		npu_mdevp.is_checked = true;
1227
-	}
1228
-	rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1229
-	rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
1230
-
1231
-	of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
1232
-			     (u32 *)&npu_cooling_power.dyn_power_coeff);
1233
-	rknpu_dev->model_data =
1234
-		rockchip_ipa_power_model_init(dev, "npu_leakage");
1235
-	if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
1236
-		rknpu_dev->model_data = NULL;
1237
-		LOG_DEV_ERROR(dev, "failed to initialize power model\n");
1238
-	} else if (rknpu_dev->model_data->dynamic_coefficient) {
1239
-		npu_cooling_power.dyn_power_coeff =
1240
-			rknpu_dev->model_data->dynamic_coefficient;
1241
-	}
1242
-	if (!npu_cooling_power.dyn_power_coeff) {
1243
-		LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
1244
-		goto out;
1245
-	}
1246
-
1247
-	rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
1248
-		dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
1249
-	if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
1250
-		LOG_DEV_ERROR(dev, "failed to register cooling device\n");
1251
-
1252
-out:
1253
-	return 0;
1254
-
1255
-err_remove_governor:
1256
-#ifdef CONFIG_PM_DEVFREQ
1257
-	devfreq_remove_governor(&devfreq_rknpu_ondemand);
1258
-#endif
1259
-err_remove_table:
1260
-	dev_pm_opp_of_remove_table(dev);
1261
-
1262
-	rknpu_dev->devfreq = NULL;
1263
-
1264
-	return ret;
1265
-}
1266
-
1267
-#else
1268
-
1269
-static int npu_devfreq_adjust_current_freq_volt(struct device *dev,
1270
-						struct rknpu_device *rknpu_dev)
1271
-{
1272
-	unsigned long volt, old_freq, freq;
1273
-	struct dev_pm_opp *opp = NULL;
1274
-	int ret = -EINVAL;
1275
-
1276
-	old_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1277
-	freq = old_freq;
1278
-
1279
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
1280
-	rcu_read_lock();
1281
-#endif
1282
-
1283
-	opp = devfreq_recommended_opp(dev, &freq, 0);
1284
-	volt = dev_pm_opp_get_voltage(opp);
1285
-
1286
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
1287
-	rcu_read_unlock();
1288
-#endif
1289
-
1290
-	if (freq >= old_freq && rknpu_dev->vdd) {
1291
-		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1292
-		if (ret) {
1293
-			LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
1294
-			return ret;
1295
-		}
1296
-	}
1297
-	LOG_DEV_DEBUG(dev, "adjust current freq=%luHz, volt=%luuV\n", freq,
1298
-		      volt);
1299
-	ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
1300
-	if (ret) {
1301
-		LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
1302
-		return ret;
1303
-	}
1304
-	if (freq < old_freq && rknpu_dev->vdd) {
1305
-		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1306
-		if (ret) {
1307
-			LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
1308
-			return ret;
1309
-		}
1310
-	}
1311
-	rknpu_dev->current_freq = freq;
1312
-	rknpu_dev->current_volt = volt;
1313
-
1314
-	return 0;
1315
-}
1316
-
1317
-static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
1318
-{
1319
-	struct device *dev = rknpu_dev->dev;
1320
-	struct devfreq_dev_profile *dp = &npu_devfreq_profile;
1321
-	int ret = -EINVAL;
1322
-
1323
-	ret = rockchip_init_opp_table(dev, NULL, "npu_leakage", "rknpu");
1324
-	if (ret) {
1325
-		LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
1326
-		return ret;
1327
-	}
1328
-
1329
-	ret = npu_devfreq_adjust_current_freq_volt(dev, rknpu_dev);
1330
-	if (ret) {
1331
-		LOG_DEV_ERROR(dev, "failed to adjust current freq volt\n");
1332
-		goto err_remove_table;
1333
-	}
1334
-	dp->initial_freq = rknpu_dev->current_freq;
1335
-
1336
-#ifdef CONFIG_PM_DEVFREQ
1337
-	ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
1338
-	if (ret) {
1339
-		LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
1340
-		goto err_remove_table;
1341
-	}
1342
-#endif
1343
-
1344
-	rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
1345
-						     (void *)rknpu_dev);
1346
-	if (IS_ERR(rknpu_dev->devfreq)) {
1347
-		LOG_DEV_ERROR(dev, "failed to add devfreq\n");
1348
-		ret = PTR_ERR(rknpu_dev->devfreq);
1349
-		goto err_remove_governor;
1350
-	}
1351
-	devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq);
1352
-
1353
-	rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq;
1354
-	rknpu_dev->devfreq->last_status.total_time = 1;
1355
-	rknpu_dev->devfreq->last_status.busy_time = 1;
1356
-
1357
-	npu_mdevp.data = rknpu_dev->devfreq;
1358
-	rknpu_dev->mdev_info =
1359
-		rockchip_system_monitor_register(dev, &npu_mdevp);
1360
-	if (IS_ERR(rknpu_dev->mdev_info)) {
1361
-		LOG_DEV_DEBUG(dev, "without system monitor\n");
1362
-		rknpu_dev->mdev_info = NULL;
1363
-	}
1364
-	rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1365
-	rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
1366
-
1367
-	of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
1368
-			     (u32 *)&npu_cooling_power.dyn_power_coeff);
1369
-	rknpu_dev->model_data =
1370
-		rockchip_ipa_power_model_init(dev, "npu_leakage");
1371
-	if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
1372
-		rknpu_dev->model_data = NULL;
1373
-		LOG_DEV_ERROR(dev, "failed to initialize power model\n");
1374
-	} else if (rknpu_dev->model_data->dynamic_coefficient) {
1375
-		npu_cooling_power.dyn_power_coeff =
1376
-			rknpu_dev->model_data->dynamic_coefficient;
1377
-	}
1378
-
1379
-	if (!npu_cooling_power.dyn_power_coeff) {
1380
-		LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
1381
-		goto out;
1382
-	}
1383
-
1384
-	rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
1385
-		dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
1386
-	if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
1387
-		LOG_DEV_ERROR(dev, "failed to register cooling device\n");
1388
-
1389
-out:
1390
-	return 0;
1391
-
1392
-err_remove_governor:
1393
-#ifdef CONFIG_PM_DEVFREQ
1394
-	devfreq_remove_governor(&devfreq_rknpu_ondemand);
1395
-#endif
1396
-err_remove_table:
1397
-	dev_pm_opp_of_remove_table(dev);
1398
-
1399
-	rknpu_dev->devfreq = NULL;
1400
-
1401
-	return ret;
1402
-}
1403
-#endif
1404
-
1405
-static int rknpu_devfreq_remove(struct rknpu_device *rknpu_dev)
1406
-{
1407
-	if (rknpu_dev->devfreq) {
1408
-		devfreq_unregister_opp_notifier(rknpu_dev->dev,
1409
-						rknpu_dev->devfreq);
1410
-		dev_pm_opp_of_remove_table(rknpu_dev->dev);
1411
-#ifdef CONFIG_PM_DEVFREQ
1412
-		devfreq_remove_governor(&devfreq_rknpu_ondemand);
1413
-#endif
1414
-	}
1415
-
1416
-	return 0;
1417
-}
1418
-
1419
-#endif
1420949 
1421950 static int rknpu_register_irq(struct platform_device *pdev,
1422951 			      struct rknpu_device *rknpu_dev)
....@@ -1518,6 +1047,53 @@
15181047 	return 0;
15191048 }
15201049 
1050
+static int rknpu_find_nbuf_resource(struct rknpu_device *rknpu_dev)
1051
+{
1052
+	struct device *dev = rknpu_dev->dev;
1053
+
1054
+	if (rknpu_dev->config->nbuf_size == 0)
1055
+		return -EINVAL;
1056
+
1057
+	rknpu_dev->nbuf_start = rknpu_dev->config->nbuf_phyaddr;
1058
+	rknpu_dev->nbuf_size = rknpu_dev->config->nbuf_size;
1059
+	rknpu_dev->nbuf_base_io =
1060
+		devm_ioremap(dev, rknpu_dev->nbuf_start, rknpu_dev->nbuf_size);
1061
+	if (IS_ERR(rknpu_dev->nbuf_base_io)) {
1062
+		LOG_DEV_ERROR(dev, "failed to remap nbuf base io!\n");
1063
+		rknpu_dev->nbuf_base_io = NULL;
1064
+	}
1065
+
1066
+	rknpu_dev->nbuf_end = rknpu_dev->nbuf_start + rknpu_dev->nbuf_size;
1067
+
1068
+	LOG_DEV_INFO(dev, "nbuf region: [%pa, %pa), nbuf size: %#x\n",
1069
+		     &rknpu_dev->nbuf_start, &rknpu_dev->nbuf_end,
1070
+		     rknpu_dev->nbuf_size);
1071
+
1072
+	return 0;
1073
+}
1074
+
1075
+static int rknpu_get_invalid_core_mask(struct device *dev)
1076
+{
1077
+	int ret = 0;
1078
+	u8 invalid_core_mask = 0;
1079
+
1080
+	if (of_property_match_string(dev->of_node, "nvmem-cell-names",
1081
+				     "cores") >= 0) {
1082
+		ret = rockchip_nvmem_cell_read_u8(dev->of_node, "cores",
1083
+						  &invalid_core_mask);
1084
+		/* The default valid npu cores for RK3583 are core0 and core1 */
1085
+		invalid_core_mask |= RKNPU_CORE2_MASK;
1086
+		if (ret) {
1087
+			LOG_DEV_ERROR(
1088
+				dev,
1089
+				"failed to get specification_serial_number\n");
1090
+			return invalid_core_mask;
1091
+		}
1092
+	}
1093
+
1094
+	return (int)invalid_core_mask;
1095
+}
1096
+
15211097 static int rknpu_probe(struct platform_device *pdev)
15221098 {
15231099 	struct resource *res = NULL;
....@@ -1549,6 +1125,22 @@
15491125 	if (!config)
15501126 		return -EINVAL;
15511127 
1128
+	if (match->data == (void *)&rk3588_rknpu_config) {
1129
+		int invalid_core_mask = rknpu_get_invalid_core_mask(dev);
1130
+		/* The default valid npu cores for RK3583 are core0 and core1 */
1131
+		if (invalid_core_mask & RKNPU_CORE2_MASK) {
1132
+			if ((invalid_core_mask & RKNPU_CORE0_MASK) ||
1133
+			    (invalid_core_mask & RKNPU_CORE1_MASK)) {
1134
+				LOG_DEV_ERROR(
1135
+					dev,
1136
+					"rknpu core invalid, invalid core mask: %#x\n",
1137
+					invalid_core_mask);
1138
+				return -ENODEV;
1139
+			}
1140
+			config = &rk3583_rknpu_config;
1141
+		}
1142
+	}
1143
+
15521144 	rknpu_dev->config = config;
15531145 	rknpu_dev->dev = dev;
15541146 
....@@ -1577,11 +1169,6 @@
15771169 	}
15781170 
15791171 #ifndef FPGA_PLATFORM
1580
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
1581
-	if (strstr(__clk_get_name(rknpu_dev->clks[0].clk), "scmi"))
1582
-		rknpu_dev->opp_info.scmi_clk = rknpu_dev->clks[0].clk;
1583
-#endif
1584
-
15851172 	rknpu_dev->vdd = devm_regulator_get_optional(dev, "rknpu");
15861173 	if (IS_ERR(rknpu_dev->vdd)) {
15871174 		if (PTR_ERR(rknpu_dev->vdd) != -ENODEV) {
....@@ -1708,9 +1295,11 @@
17081295 		virt_dev = dev_pm_domain_attach_by_name(dev, "npu1");
17091296 		if (!IS_ERR(virt_dev))
17101297 			rknpu_dev->genpd_dev_npu1 = virt_dev;
1711
-		virt_dev = dev_pm_domain_attach_by_name(dev, "npu2");
1712
-		if (!IS_ERR(virt_dev))
1713
-			rknpu_dev->genpd_dev_npu2 = virt_dev;
1298
+		if (config->num_irqs > 2) {
1299
+			virt_dev = dev_pm_domain_attach_by_name(dev, "npu2");
1300
+			if (!IS_ERR(virt_dev))
1301
+				rknpu_dev->genpd_dev_npu2 = virt_dev;
1302
+		}
17141303 		rknpu_dev->multiple_domains = true;
17151304 	}
17161305 
....@@ -1734,7 +1323,8 @@
17341323 	INIT_DEFERRABLE_WORK(&rknpu_dev->power_off_work,
17351324 			     rknpu_power_off_delay_work);
17361325 
1737
-	if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) {
1326
+	if (IS_ENABLED(CONFIG_NO_GKI) &&
1327
+	    IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) {
17381328 		if (!rknpu_find_sram_resource(rknpu_dev)) {
17391329 			ret = rknpu_mm_create(rknpu_dev->sram_size, PAGE_SIZE,
17401330 					      &rknpu_dev->sram_mm);
....@@ -1744,6 +1334,10 @@
17441334 			LOG_DEV_WARN(dev, "could not find sram resource!\n");
17451335 		}
17461336 	}
1337
+
1338
+	if (IS_ENABLED(CONFIG_NO_GKI) && rknpu_dev->iommu_en &&
1339
+	    rknpu_dev->config->nbuf_size > 0)
1340
+		rknpu_find_nbuf_resource(rknpu_dev);
17471341 
17481342 	rknpu_power_off(rknpu_dev);
17491343 	atomic_set(&rknpu_dev->power_refcount, 0);
....@@ -1823,47 +1417,14 @@
18231417 }
18241418 
18251419 #ifndef FPGA_PLATFORM
1826
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
18271420 static int rknpu_runtime_suspend(struct device *dev)
18281421 {
1829
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1830
-	struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
1831
-
1832
-	if (opp_info->scmi_clk) {
1833
-		if (clk_set_rate(opp_info->scmi_clk, POWER_DOWN_FREQ))
1834
-			LOG_DEV_ERROR(dev, "failed to restore clk rate\n");
1835
-	}
1836
-	opp_info->current_rm = UINT_MAX;
1837
-
1838
-	return 0;
1422
+	return rknpu_devfreq_runtime_suspend(dev);
18391423 }
18401424 
18411425 static int rknpu_runtime_resume(struct device *dev)
18421426 {
1843
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1844
-	struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
1845
-	int ret = 0;
1846
-
1847
-	if (!rknpu_dev->current_freq || !rknpu_dev->current_volt)
1848
-		return 0;
1849
-
1850
-	ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
1851
-	if (ret) {
1852
-		LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
1853
-		return ret;
1854
-	}
1855
-
1856
-	if (opp_info->data && opp_info->data->set_read_margin)
1857
-		opp_info->data->set_read_margin(dev, opp_info,
1858
-						opp_info->target_rm);
1859
-	if (opp_info->scmi_clk) {
1860
-		if (clk_set_rate(opp_info->scmi_clk, rknpu_dev->current_freq))
1861
-			LOG_DEV_ERROR(dev, "failed to set power down rate\n");
1862
-	}
1863
-
1864
-	clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
1865
-
1866
-	return ret;
1427
+	return rknpu_devfreq_runtime_resume(dev);
18671428 }
18681429 
18691430 static const struct dev_pm_ops rknpu_pm_ops = {
....@@ -1873,7 +1434,6 @@
18731434 				   NULL)
18741435 };
18751436 #endif
1876
-#endif
18771437 
18781438 static struct platform_driver rknpu_driver = {
18791439 	.probe = rknpu_probe,
....@@ -1882,9 +1442,7 @@
18821442 		.owner = THIS_MODULE,
18831443 		.name = "RKNPU",
18841444 #ifndef FPGA_PLATFORM
1885
-#if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
18861445 		.pm = &rknpu_pm_ops,
1887
-#endif
18881446 #endif
18891447 		.of_match_table = of_match_ptr(rknpu_of_match),
18901448 	},
....@@ -1909,3 +1467,6 @@
19091467 MODULE_LICENSE("GPL v2");
19101468 MODULE_VERSION(RKNPU_GET_DRV_VERSION_STRING(DRIVER_MAJOR, DRIVER_MINOR,
19111469 					    DRIVER_PATCHLEVEL));
1470
+#if KERNEL_VERSION(5, 16, 0) < LINUX_VERSION_CODE
1471
+MODULE_IMPORT_NS(DMA_BUF);
1472
+#endif