forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/drivers/rknpu/rknpu_drv.c
....@@ -31,17 +31,11 @@
3131 #include <linux/clk-provider.h>
3232 #include <linux/pm_domain.h>
3333 #include <linux/pm_runtime.h>
34
-#include <linux/devfreq_cooling.h>
3534 #include <linux/regmap.h>
36
-#include <linux/dma-iommu.h>
3735 #include <linux/of_address.h>
3836 
3937 #ifndef FPGA_PLATFORM
4038 #include <soc/rockchip/rockchip_iommu.h>
41
-#include <soc/rockchip/rockchip_opp_select.h>
42
-#include <soc/rockchip/rockchip_system_monitor.h>
43
-#include <soc/rockchip/rockchip_ipa.h>
44
-#include <../drivers/devfreq/governor.h>
4539 #endif
4640 
4741 #include "rknpu_ioctl.h"
....@@ -49,6 +43,7 @@
4943 #include "rknpu_fence.h"
5044 #include "rknpu_drv.h"
5145 #include "rknpu_gem.h"
46
+#include "rknpu_devfreq.h"
5247 
5348 #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
5449 #include <drm/drm_device.h>
....@@ -77,36 +72,29 @@
7772 MODULE_PARM_DESC(bypass_soft_reset,
7873 		 "bypass RKNPU soft reset if set it to 1, disabled by default");
7974 
80
-struct npu_irqs_data {
75
+struct rknpu_irqs_data {
8176 	const char *name;
8277 	irqreturn_t (*irq_hdl)(int irq, void *ctx);
8378 };
8479 
85
-static const struct npu_irqs_data rk356x_npu_irqs[] = {
80
+static const struct rknpu_irqs_data rknpu_irqs[] = {
8681 	{ "npu_irq", rknpu_core0_irq_handler }
8782 };
8883 
89
-static const struct npu_irqs_data rk3588_npu_irqs[] = {
84
+static const struct rknpu_irqs_data rk3588_npu_irqs[] = {
9085 	{ "npu0_irq", rknpu_core0_irq_handler },
9186 	{ "npu1_irq", rknpu_core1_irq_handler },
9287 	{ "npu2_irq", rknpu_core2_irq_handler }
9388 };
9489 
95
-static const struct npu_irqs_data rv110x_npu_irqs[] = {
96
-	{ "npu_irq", rknpu_core0_irq_handler }
97
-};
90
+static const struct rknpu_reset_data rknpu_resets[] = { { "srst_a",
91
+							  "srst_h" } };
9892 
99
-static const struct npu_reset_data rk356x_npu_resets[] = { { "srst_a",
100
-							     "srst_h" } };
101
-
102
-static const struct npu_reset_data rk3588_npu_resets[] = {
93
+static const struct rknpu_reset_data rk3588_npu_resets[] = {
10394 	{ "srst_a0", "srst_h0" },
10495 	{ "srst_a1", "srst_h1" },
10596 	{ "srst_a2", "srst_h2" }
10697 };
107
-
108
-static const struct npu_reset_data rv110x_npu_resets[] = { { "srst_a",
109
-							     "srst_h" } };
11098 
11199 static const struct rknpu_config rk356x_rknpu_config = {
112100 	.bw_priority_addr = 0xfe180008,
....@@ -115,11 +103,17 @@
115103 	.pc_data_amount_scale = 1,
116104 	.pc_task_number_bits = 12,
117105 	.pc_task_number_mask = 0xfff,
106
+	.pc_task_status_offset = 0x3c,
107
+	.pc_dma_ctrl = 0,
118108 	.bw_enable = 1,
119
-	.irqs = rk356x_npu_irqs,
120
-	.resets = rk356x_npu_resets,
121
-	.num_irqs = ARRAY_SIZE(rk356x_npu_irqs),
122
-	.num_resets = ARRAY_SIZE(rk356x_npu_resets)
109
+	.irqs = rknpu_irqs,
110
+	.resets = rknpu_resets,
111
+	.num_irqs = ARRAY_SIZE(rknpu_irqs),
112
+	.num_resets = ARRAY_SIZE(rknpu_resets),
113
+	.nbuf_phyaddr = 0,
114
+	.nbuf_size = 0,
115
+	.max_submit_number = (1 << 12) - 1,
116
+	.core_mask = 0x1,
123117 };
124118 
125119 static const struct rknpu_config rk3588_rknpu_config = {
....@@ -129,11 +123,37 @@
129123 	.pc_data_amount_scale = 2,
130124 	.pc_task_number_bits = 12,
131125 	.pc_task_number_mask = 0xfff,
126
+	.pc_task_status_offset = 0x3c,
127
+	.pc_dma_ctrl = 0,
132128 	.bw_enable = 0,
133129 	.irqs = rk3588_npu_irqs,
134130 	.resets = rk3588_npu_resets,
135131 	.num_irqs = ARRAY_SIZE(rk3588_npu_irqs),
136
-	.num_resets = ARRAY_SIZE(rk3588_npu_resets)
132
+	.num_resets = ARRAY_SIZE(rk3588_npu_resets),
133
+	.nbuf_phyaddr = 0,
134
+	.nbuf_size = 0,
135
+	.max_submit_number = (1 << 12) - 1,
136
+	.core_mask = 0x7,
137
+};
138
+
139
+static const struct rknpu_config rk3583_rknpu_config = {
140
+	.bw_priority_addr = 0x0,
141
+	.bw_priority_length = 0x0,
142
+	.dma_mask = DMA_BIT_MASK(40),
143
+	.pc_data_amount_scale = 2,
144
+	.pc_task_number_bits = 12,
145
+	.pc_task_number_mask = 0xfff,
146
+	.pc_task_status_offset = 0x3c,
147
+	.pc_dma_ctrl = 0,
148
+	.bw_enable = 0,
149
+	.irqs = rk3588_npu_irqs,
150
+	.resets = rk3588_npu_resets,
151
+	.num_irqs = 2,
152
+	.num_resets = 2,
153
+	.nbuf_phyaddr = 0,
154
+	.nbuf_size = 0,
155
+	.max_submit_number = (1 << 12) - 1,
156
+	.core_mask = 0x3,
137157 };
138158 
139159 static const struct rknpu_config rv1106_rknpu_config = {
....@@ -143,11 +163,37 @@
143163 	.pc_data_amount_scale = 2,
144164 	.pc_task_number_bits = 16,
145165 	.pc_task_number_mask = 0xffff,
166
+	.pc_task_status_offset = 0x3c,
167
+	.pc_dma_ctrl = 0,
146168 	.bw_enable = 1,
147
-	.irqs = rv110x_npu_irqs,
148
-	.resets = rv110x_npu_resets,
149
-	.num_irqs = ARRAY_SIZE(rv110x_npu_irqs),
150
-	.num_resets = ARRAY_SIZE(rv110x_npu_resets)
169
+	.irqs = rknpu_irqs,
170
+	.resets = rknpu_resets,
171
+	.num_irqs = ARRAY_SIZE(rknpu_irqs),
172
+	.num_resets = ARRAY_SIZE(rknpu_resets),
173
+	.nbuf_phyaddr = 0,
174
+	.nbuf_size = 0,
175
+	.max_submit_number = (1 << 16) - 1,
176
+	.core_mask = 0x1,
177
+};
178
+
179
+static const struct rknpu_config rk3562_rknpu_config = {
180
+	.bw_priority_addr = 0x0,
181
+	.bw_priority_length = 0x0,
182
+	.dma_mask = DMA_BIT_MASK(40),
183
+	.pc_data_amount_scale = 2,
184
+	.pc_task_number_bits = 16,
185
+	.pc_task_number_mask = 0xffff,
186
+	.pc_task_status_offset = 0x48,
187
+	.pc_dma_ctrl = 1,
188
+	.bw_enable = 1,
189
+	.irqs = rknpu_irqs,
190
+	.resets = rknpu_resets,
191
+	.num_irqs = ARRAY_SIZE(rknpu_irqs),
192
+	.num_resets = ARRAY_SIZE(rknpu_resets),
193
+	.nbuf_phyaddr = 0xfe400000,
194
+	.nbuf_size = 256 * 1024,
195
+	.max_submit_number = (1 << 16) - 1,
196
+	.core_mask = 0x1,
151197 };
152198 
153199 /* driver probe and init */
....@@ -167,6 +213,10 @@
167213 	{
168214 		.compatible = "rockchip,rv1106-rknpu",
169215 		.data = &rv1106_rknpu_config,
216
+	},
217
+	{
218
+		.compatible = "rockchip,rk3562-rknpu",
219
+		.data = &rk3562_rknpu_config,
170220 	},
171221 	{},
172222 };
....@@ -196,7 +246,6 @@
196246 {
197247 	int ret = 0;
198248 
199
-	cancel_delayed_work(&rknpu_dev->power_off_work);
200249 	mutex_lock(&rknpu_dev->power_lock);
201250 	if (atomic_inc_return(&rknpu_dev->power_refcount) == 1)
202251 		ret = rknpu_power_on(rknpu_dev);
....@@ -219,6 +268,9 @@
219268 
220269 static int rknpu_power_put_delay(struct rknpu_device *rknpu_dev)
221270 {
271
+	if (rknpu_dev->power_put_delay == 0)
272
+		return rknpu_power_put(rknpu_dev);
273
+
222274 	mutex_lock(&rknpu_dev->power_lock);
223275 	if (atomic_read(&rknpu_dev->power_refcount) == 1)
224276 		queue_delayed_work(
....@@ -227,6 +279,7 @@
227279 	else
228280 		atomic_dec_if_positive(&rknpu_dev->power_refcount);
229281 	mutex_unlock(&rknpu_dev->power_lock);
282
+
230283 	return 0;
231284 }
232285 
....@@ -243,13 +296,17 @@
243296 		ret = rknpu_get_drv_version(&args->value);
244297 		break;
245298 	case RKNPU_GET_FREQ:
299
+#ifndef FPGA_PLATFORM
246300 		args->value = clk_get_rate(rknpu_dev->clks[0].clk);
301
+#endif
247302 		ret = 0;
248303 		break;
249304 	case RKNPU_SET_FREQ:
250305 		break;
251306 	case RKNPU_GET_VOLT:
307
+#ifndef FPGA_PLATFORM
252308 		args->value = regulator_get_voltage(rknpu_dev->vdd);
309
+#endif
253310 		ret = 0;
254311 		break;
255312 	case RKNPU_SET_VOLT:
....@@ -328,11 +385,56 @@
328385 #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
329386 static int rknpu_open(struct inode *inode, struct file *file)
330387 {
388
+	struct rknpu_device *rknpu_dev =
389
+		container_of(file->private_data, struct rknpu_device, miscdev);
390
+	struct rknpu_session *session = NULL;
391
+
392
+	session = kzalloc(sizeof(*session), GFP_KERNEL);
393
+	if (!session) {
394
+		LOG_ERROR("rknpu session alloc failed\n");
395
+		return -ENOMEM;
396
+	}
397
+
398
+	session->rknpu_dev = rknpu_dev;
399
+	INIT_LIST_HEAD(&session->list);
400
+
401
+	file->private_data = (void *)session;
402
+
331403 	return nonseekable_open(inode, file);
332404 }
333405 
334406 static int rknpu_release(struct inode *inode, struct file *file)
335407 {
408
+	struct rknpu_mem_object *entry;
409
+	struct rknpu_session *session = file->private_data;
410
+	struct rknpu_device *rknpu_dev = session->rknpu_dev;
411
+	LIST_HEAD(local_list);
412
+
413
+	spin_lock(&rknpu_dev->lock);
414
+	list_replace_init(&session->list, &local_list);
415
+	file->private_data = NULL;
416
+	spin_unlock(&rknpu_dev->lock);
417
+
418
+	while (!list_empty(&local_list)) {
419
+		entry = list_first_entry(&local_list, struct rknpu_mem_object,
420
+					 head);
421
+
422
+		LOG_DEBUG(
423
+			"Fd close free rknpu_obj: %#llx, rknpu_obj->dma_addr: %#llx\n",
424
+			(__u64)(uintptr_t)entry, (__u64)entry->dma_addr);
425
+
426
+		vunmap(entry->kv_addr);
427
+		entry->kv_addr = NULL;
428
+
429
+		if (!entry->owner)
430
+			dma_buf_put(entry->dmabuf);
431
+
432
+		list_del(&entry->head);
433
+		kfree(entry);
434
+	}
435
+
436
+	kfree(session);
437
+
336438 	return 0;
337439 }
338440 
....@@ -364,8 +466,12 @@
364466 static long rknpu_ioctl(struct file *file, uint32_t cmd, unsigned long arg)
365467 {
366468 	long ret = -EINVAL;
367
-	struct rknpu_device *rknpu_dev =
368
-		container_of(file->private_data, struct rknpu_device, miscdev);
469
+	struct rknpu_device *rknpu_dev = NULL;
470
+
471
+	if (!file->private_data)
472
+		return -EINVAL;
473
+
474
+	rknpu_dev = ((struct rknpu_session *)file->private_data)->rknpu_dev;
369475 
370476 	rknpu_power_get(rknpu_dev);
371477 
....@@ -377,12 +483,12 @@
377483 		ret = rknpu_submit_ioctl(rknpu_dev, arg);
378484 		break;
379485 	case IOCTL_RKNPU_MEM_CREATE:
380
-		ret = rknpu_mem_create_ioctl(rknpu_dev, arg);
486
+		ret = rknpu_mem_create_ioctl(rknpu_dev, arg, file);
381487 		break;
382488 	case RKNPU_MEM_MAP:
383489 		break;
384490 	case IOCTL_RKNPU_MEM_DESTROY:
385
-		ret = rknpu_mem_destroy_ioctl(rknpu_dev, arg);
491
+		ret = rknpu_mem_destroy_ioctl(rknpu_dev, arg, file);
386492 		break;
387493 	case IOCTL_RKNPU_MEM_SYNC:
388494 		ret = rknpu_mem_sync_ioctl(rknpu_dev, arg);
....@@ -407,11 +513,13 @@
407513 #endif
408514 
409515 #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
516
+#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
410517 static const struct vm_operations_struct rknpu_gem_vm_ops = {
411518 	.fault = rknpu_gem_fault,
412519 	.open = drm_gem_vm_open,
413520 	.close = drm_gem_vm_close,
414521 };
522
+#endif
415523 
416524 static int rknpu_action_ioctl(struct drm_device *dev, void *data,
417525 			      struct drm_file *file_priv)
....@@ -453,6 +561,9 @@
453561 			  DRM_RENDER_ALLOW),
454562 };
455563 
564
+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
565
+DEFINE_DRM_GEM_FOPS(rknpu_drm_driver_fops);
566
+#else
456567 static const struct file_operations rknpu_drm_driver_fops = {
457568 	.owner = THIS_MODULE,
458569 	.open = drm_open,
....@@ -466,6 +577,7 @@
466577 	.release = drm_release,
467578 	.llseek = noop_llseek,
468579 };
580
+#endif
469581 
470582 static struct drm_driver rknpu_drm_driver = {
471583 #if KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE
....@@ -473,28 +585,34 @@
473585 #else
474586 	.driver_features = DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER,
475587 #endif
588
+#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
476589 	.gem_free_object_unlocked = rknpu_gem_free_object,
477590 	.gem_vm_ops = &rknpu_gem_vm_ops,
591
+	.dumb_destroy = drm_gem_dumb_destroy,
592
+	.gem_prime_export = drm_gem_prime_export,
593
+	.gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table,
594
+	.gem_prime_vmap = rknpu_gem_prime_vmap,
595
+	.gem_prime_vunmap = rknpu_gem_prime_vunmap,
596
+#endif
478597 	.dumb_create = rknpu_gem_dumb_create,
479598 #if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
480599 	.dumb_map_offset = rknpu_gem_dumb_map_offset,
481600 #else
482601 	.dumb_map_offset = drm_gem_dumb_map_offset,
483602 #endif
484
-	.dumb_destroy = drm_gem_dumb_destroy,
485603 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
486604 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
487
-	.gem_prime_export = drm_gem_prime_export,
488605 #if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
489606 	.gem_prime_import = rknpu_gem_prime_import,
490607 #else
491608 	.gem_prime_import = drm_gem_prime_import,
492609 #endif
493
-	.gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table,
494610 	.gem_prime_import_sg_table = rknpu_gem_prime_import_sg_table,
495
-	.gem_prime_vmap = rknpu_gem_prime_vmap,
496
-	.gem_prime_vunmap = rknpu_gem_prime_vunmap,
611
+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
612
+	.gem_prime_mmap = drm_gem_prime_mmap,
613
+#else
497614 	.gem_prime_mmap = rknpu_gem_prime_mmap,
615
+#endif
498616 	.ioctls = rknpu_ioctls,
499617 	.num_ioctls = ARRAY_SIZE(rknpu_ioctls),
500618 	.fops = &rknpu_drm_driver_fops,
....@@ -514,7 +632,7 @@
514632 		container_of(timer, struct rknpu_device, timer);
515633 	struct rknpu_subcore_data *subcore_data = NULL;
516634 	struct rknpu_job *job = NULL;
517
-	ktime_t now = ktime_get();
635
+	ktime_t now;
518636 	unsigned long flags;
519637 	int i;
520638 
....@@ -525,14 +643,16 @@
525643 
526644 		job = subcore_data->job;
527645 		if (job) {
646
+			now = ktime_get();
528647 			subcore_data->timer.busy_time +=
529
-				ktime_us_delta(now, job->hw_recoder_time);
530
-			job->hw_recoder_time = ktime_get();
648
+				ktime_sub(now, job->hw_recoder_time);
649
+			job->hw_recoder_time = now;
531650 		}
532651 
533
-		subcore_data->timer.busy_time_record =
652
+		subcore_data->timer.total_busy_time =
534653 			subcore_data->timer.busy_time;
535654 		subcore_data->timer.busy_time = 0;
655
+
536656 		spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
537657 	}
538658 
....@@ -579,6 +699,42 @@
579699 }
580700 
581701 #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
702
+static int drm_fake_dev_register(struct rknpu_device *rknpu_dev)
703
+{
704
+	const struct platform_device_info rknpu_dev_info = {
705
+		.name = "rknpu_dev",
706
+		.id = PLATFORM_DEVID_AUTO,
707
+		.dma_mask = rknpu_dev->config->dma_mask,
708
+	};
709
+	struct platform_device *pdev = NULL;
710
+	int ret = -EINVAL;
711
+
712
+	pdev = platform_device_register_full(&rknpu_dev_info);
713
+	if (pdev) {
714
+		ret = of_dma_configure(&pdev->dev, NULL, true);
715
+		if (ret) {
716
+			platform_device_unregister(pdev);
717
+			pdev = NULL;
718
+		}
719
+	}
720
+
721
+	rknpu_dev->fake_dev = pdev ? &pdev->dev : NULL;
722
+
723
+	return ret;
724
+}
725
+
726
+static void drm_fake_dev_unregister(struct rknpu_device *rknpu_dev)
727
+{
728
+	struct platform_device *pdev = NULL;
729
+
730
+	if (!rknpu_dev->fake_dev)
731
+		return;
732
+
733
+	pdev = to_platform_device(rknpu_dev->fake_dev);
734
+
735
+	platform_device_unregister(pdev);
736
+}
737
+
582738 static int rknpu_drm_probe(struct rknpu_device *rknpu_dev)
583739 {
584740 	struct device *dev = rknpu_dev->dev;
....@@ -597,6 +753,8 @@
597753 	drm_dev->dev_private = rknpu_dev;
598754 	rknpu_dev->drm_dev = drm_dev;
599755 
756
+	drm_fake_dev_register(rknpu_dev);
757
+
600758 	return 0;
601759 
602760 err_free_drm:
....@@ -612,6 +770,8 @@
612770 static void rknpu_drm_remove(struct rknpu_device *rknpu_dev)
613771 {
614772 	struct drm_device *drm_dev = rknpu_dev->drm_dev;
773
+
774
+	drm_fake_dev_unregister(rknpu_dev);
615775 
616776 	drm_dev_unregister(drm_dev);
617777 
....@@ -660,9 +820,7 @@
660820 	}
661821 
662822 #ifndef FPGA_PLATFORM
663
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
664
-	rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
665
-#endif
823
+	rknpu_devfreq_lock(rknpu_dev);
666824 #endif
667825 
668826 	if (rknpu_dev->multiple_domains) {
....@@ -721,9 +879,7 @@
721879 
722880 out:
723881 #ifndef FPGA_PLATFORM
724
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
725
-	rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
726
-#endif
882
+	rknpu_devfreq_unlock(rknpu_dev);
727883 #endif
728884 
729885 	return ret;
....@@ -737,9 +893,7 @@
737893 	int ret;
738894 	bool val;
739895 
740
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
741
-	rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
742
-#endif
896
+	rknpu_devfreq_lock(rknpu_dev);
743897 #endif
744898 
745899 	pm_runtime_put_sync(dev);
....@@ -761,10 +915,7 @@
761915 		if (ret) {
762916 			LOG_DEV_ERROR(dev, "iommu still enabled\n");
763917 			pm_runtime_get_sync(dev);
764
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
765
-			rockchip_monitor_volt_adjust_unlock(
766
-				rknpu_dev->mdev_info);
767
-#endif
918
+			rknpu_devfreq_unlock(rknpu_dev);
768919 			return ret;
769920 		}
770921 #else
....@@ -780,9 +931,7 @@
780931 	}
781932 
782933 #ifndef FPGA_PLATFORM
783
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
784
-	rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
785
-#endif
934
+	rknpu_devfreq_unlock(rknpu_dev);
786935 #endif
787936 
788937 	clk_bulk_disable_unprepare(rknpu_dev->num_clks, rknpu_dev->clks);
....@@ -797,612 +946,6 @@
797946 
798947 	return 0;
799948 }
800
-
801
-#ifndef FPGA_PLATFORM
802
-static struct monitor_dev_profile npu_mdevp = {
803
-	.type = MONITOR_TPYE_DEV,
804
-	.low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
805
-	.high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
806
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
807
-	.update_volt = rockchip_monitor_check_rate_volt,
808
-#endif
809
-};
810
-
811
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
812
-static int npu_opp_helper(struct dev_pm_set_opp_data *data)
813
-{
814
-	struct device *dev = data->dev;
815
-	struct dev_pm_opp_supply *old_supply_vdd = &data->old_opp.supplies[0];
816
-	struct dev_pm_opp_supply *old_supply_mem = &data->old_opp.supplies[1];
817
-	struct dev_pm_opp_supply *new_supply_vdd = &data->new_opp.supplies[0];
818
-	struct dev_pm_opp_supply *new_supply_mem = &data->new_opp.supplies[1];
819
-	struct regulator *vdd_reg = data->regulators[0];
820
-	struct regulator *mem_reg = data->regulators[1];
821
-	struct clk *clk = data->clk;
822
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
823
-	struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
824
-	unsigned long old_freq = data->old_opp.rate;
825
-	unsigned long new_freq = data->new_opp.rate;
826
-	bool is_set_rm = true;
827
-	bool is_set_clk = true;
828
-	u32 target_rm = UINT_MAX;
829
-	int ret = 0;
830
-
831
-	if (!pm_runtime_active(dev)) {
832
-		is_set_rm = false;
833
-		if (opp_info->scmi_clk)
834
-			is_set_clk = false;
835
-	}
836
-
837
-	ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
838
-	if (ret < 0) {
839
-		LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
840
-		return ret;
841
-	}
842
-	rockchip_get_read_margin(dev, opp_info, new_supply_vdd->u_volt,
843
-				 &target_rm);
844
-
845
-	/* Change frequency */
846
-	LOG_DEV_DEBUG(dev, "switching OPP: %lu Hz --> %lu Hz\n", old_freq,
847
-		      new_freq);
848
-	/* Scaling up? Scale voltage before frequency */
849
-	if (new_freq >= old_freq) {
850
-		rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
851
-					       new_freq, true, is_set_clk);
852
-		ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
853
-					    INT_MAX);
854
-		if (ret) {
855
-			LOG_DEV_ERROR(dev,
856
-				      "failed to set volt %lu uV for mem reg\n",
857
-				      new_supply_mem->u_volt);
858
-			goto restore_voltage;
859
-		}
860
-		ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
861
-					    INT_MAX);
862
-		if (ret) {
863
-			LOG_DEV_ERROR(dev,
864
-				      "failed to set volt %lu uV for vdd reg\n",
865
-				      new_supply_vdd->u_volt);
866
-			goto restore_voltage;
867
-		}
868
-		rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
869
-		if (is_set_clk && clk_set_rate(clk, new_freq)) {
870
-			ret = -EINVAL;
871
-			LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
872
-			goto restore_rm;
873
-		}
874
-		/* Scaling down? Scale voltage after frequency */
875
-	} else {
876
-		rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
877
-					       new_freq, false, is_set_clk);
878
-		rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
879
-		if (is_set_clk && clk_set_rate(clk, new_freq)) {
880
-			ret = -EINVAL;
881
-			LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
882
-			goto restore_rm;
883
-		}
884
-		ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
885
-					    INT_MAX);
886
-		if (ret) {
887
-			LOG_DEV_ERROR(dev,
888
-				      "failed to set volt %lu uV for vdd reg\n",
889
-				      new_supply_vdd->u_volt);
890
-			goto restore_freq;
891
-		}
892
-		ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
893
-					    INT_MAX);
894
-		if (ret) {
895
-			LOG_DEV_ERROR(dev,
896
-				      "failed to set volt %lu uV for mem reg\n",
897
-				      new_supply_mem->u_volt);
898
-			goto restore_freq;
899
-		}
900
-	}
901
-
902
-	clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
903
-
904
-	return 0;
905
-
906
-restore_freq:
907
-	if (is_set_clk && clk_set_rate(clk, old_freq))
908
-		LOG_DEV_ERROR(dev, "failed to restore old-freq %lu Hz\n",
909
-			      old_freq);
910
-restore_rm:
911
-	rockchip_get_read_margin(dev, opp_info, old_supply_vdd->u_volt,
912
-				 &target_rm);
913
-	rockchip_set_read_margin(dev, opp_info, opp_info->current_rm,
914
-				 is_set_rm);
915
-restore_voltage:
916
-	regulator_set_voltage(mem_reg, old_supply_mem->u_volt, INT_MAX);
917
-	regulator_set_voltage(vdd_reg, old_supply_vdd->u_volt, INT_MAX);
918
-	clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
919
-
920
-	return ret;
921
-}
922
-
923
-static int npu_devfreq_target(struct device *dev, unsigned long *freq,
924
-			      u32 flags)
925
-{
926
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
927
-	struct dev_pm_opp *opp;
928
-	unsigned long opp_volt;
929
-	int ret = 0;
930
-
931
-	if (!npu_mdevp.is_checked)
932
-		return -EINVAL;
933
-
934
-	opp = devfreq_recommended_opp(dev, freq, flags);
935
-	if (IS_ERR(opp))
936
-		return PTR_ERR(opp);
937
-	opp_volt = dev_pm_opp_get_voltage(opp);
938
-	dev_pm_opp_put(opp);
939
-
940
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
941
-	rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
942
-#endif
943
-	ret = dev_pm_opp_set_rate(dev, *freq);
944
-	if (!ret) {
945
-		rknpu_dev->current_freq = *freq;
946
-		if (rknpu_dev->devfreq)
947
-			rknpu_dev->devfreq->last_status.current_frequency =
948
-				*freq;
949
-		rknpu_dev->current_volt = opp_volt;
950
-		LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n",
951
-			     rknpu_dev->current_freq, rknpu_dev->current_volt);
952
-	}
953
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
954
-	rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
955
-#endif
956
-
957
-	return ret;
958
-}
959
-
960
-#else
961
-
962
-static int npu_devfreq_target(struct device *dev, unsigned long *target_freq,
963
-			      u32 flags)
964
-{
965
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
966
-	struct dev_pm_opp *opp = NULL;
967
-	unsigned long freq = *target_freq;
968
-	unsigned long old_freq = rknpu_dev->current_freq;
969
-	unsigned long volt, old_volt = rknpu_dev->current_volt;
970
-	int ret = -EINVAL;
971
-
972
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
973
-	rcu_read_lock();
974
-#endif
975
-
976
-	opp = devfreq_recommended_opp(dev, &freq, flags);
977
-	if (IS_ERR(opp)) {
978
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
979
-		rcu_read_unlock();
980
-#endif
981
-		LOG_DEV_ERROR(dev, "failed to get opp (%ld)\n", PTR_ERR(opp));
982
-		return PTR_ERR(opp);
983
-	}
984
-	volt = dev_pm_opp_get_voltage(opp);
985
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
986
-	rcu_read_unlock();
987
-#endif
988
-
989
-	/*
990
-	 * Only update if there is a change of frequency
991
-	 */
992
-	if (old_freq == freq) {
993
-		*target_freq = freq;
994
-		if (old_volt == volt)
995
-			return 0;
996
-		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
997
-		if (ret) {
998
-			LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
999
-			return ret;
1000
-		}
1001
-		rknpu_dev->current_volt = volt;
1002
-		return 0;
1003
-	}
1004
-
1005
-	if (rknpu_dev->vdd && old_volt != volt && old_freq < freq) {
1006
-		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1007
-		if (ret) {
1008
-			LOG_DEV_ERROR(dev, "failed to increase volt %lu\n",
1009
-				      volt);
1010
-			return ret;
1011
-		}
1012
-	}
1013
-	LOG_DEV_DEBUG(dev, "%luHz %luuV -> %luHz %luuV\n", old_freq, old_volt,
1014
-		      freq, volt);
1015
-	ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
1016
-	if (ret) {
1017
-		LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
1018
-		return ret;
1019
-	}
1020
-	*target_freq = freq;
1021
-	rknpu_dev->current_freq = freq;
1022
-
1023
-	if (rknpu_dev->devfreq)
1024
-		rknpu_dev->devfreq->last_status.current_frequency = freq;
1025
-
1026
-	if (rknpu_dev->vdd && old_volt != volt && old_freq > freq) {
1027
-		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1028
-		if (ret) {
1029
-			LOG_DEV_ERROR(dev, "failed to decrease volt %lu\n",
1030
-				      volt);
1031
-			return ret;
1032
-		}
1033
-	}
1034
-	rknpu_dev->current_volt = volt;
1035
-
1036
-	LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n",
1037
-		     rknpu_dev->current_freq, rknpu_dev->current_volt);
1038
-
1039
-	return ret;
1040
-}
1041
-#endif
1042
-
1043
-static int npu_devfreq_get_dev_status(struct device *dev,
1044
-				      struct devfreq_dev_status *stat)
1045
-{
1046
-	return 0;
1047
-}
1048
-
1049
-static int npu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
1050
-{
1051
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1052
-
1053
-	*freq = rknpu_dev->current_freq;
1054
-
1055
-	return 0;
1056
-}
1057
-
1058
-static struct devfreq_dev_profile npu_devfreq_profile = {
1059
-	.polling_ms = 50,
1060
-	.target = npu_devfreq_target,
1061
-	.get_dev_status = npu_devfreq_get_dev_status,
1062
-	.get_cur_freq = npu_devfreq_get_cur_freq,
1063
-};
1064
-
1065
-static int devfreq_rknpu_ondemand_func(struct devfreq *df, unsigned long *freq)
1066
-{
1067
-	struct rknpu_device *rknpu_dev = df->data;
1068
-
1069
-	if (rknpu_dev)
1070
-		*freq = rknpu_dev->ondemand_freq;
1071
-	else
1072
-		*freq = df->previous_freq;
1073
-
1074
-	return 0;
1075
-}
1076
-
1077
-static int devfreq_rknpu_ondemand_handler(struct devfreq *devfreq,
1078
-					  unsigned int event, void *data)
1079
-{
1080
-	return 0;
1081
-}
1082
-
1083
-static struct devfreq_governor devfreq_rknpu_ondemand = {
1084
-	.name = "rknpu_ondemand",
1085
-	.get_target_freq = devfreq_rknpu_ondemand_func,
1086
-	.event_handler = devfreq_rknpu_ondemand_handler,
1087
-};
1088
-
1089
-static unsigned long npu_get_static_power(struct devfreq *devfreq,
1090
-					  unsigned long voltage)
1091
-{
1092
-	struct device *dev = devfreq->dev.parent;
1093
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1094
-
1095
-	if (!rknpu_dev->model_data)
1096
-		return 0;
1097
-
1098
-	return rockchip_ipa_get_static_power(rknpu_dev->model_data, voltage);
1099
-}
1100
-
1101
-static struct devfreq_cooling_power npu_cooling_power = {
1102
-	.get_static_power = &npu_get_static_power,
1103
-};
1104
-
1105
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
1106
-static int rk3588_npu_set_read_margin(struct device *dev,
1107
-				      struct rockchip_opp_info *opp_info,
1108
-				      u32 rm)
1109
-{
1110
-	u32 offset = 0, val = 0;
1111
-	int i, ret = 0;
1112
-
1113
-	if (!opp_info->grf || !opp_info->volt_rm_tbl)
1114
-		return 0;
1115
-
1116
-	if (rm == opp_info->current_rm || rm == UINT_MAX)
1117
-		return 0;
1118
-
1119
-	LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
1120
-
1121
-	for (i = 0; i < 3; i++) {
1122
-		ret = regmap_read(opp_info->grf, offset, &val);
1123
-		if (ret < 0) {
1124
-			LOG_DEV_ERROR(dev, "failed to get rm from 0x%x\n",
1125
-				      offset);
1126
-			return ret;
1127
-		}
1128
-		val &= ~0x1c;
1129
-		regmap_write(opp_info->grf, offset, val | (rm << 2));
1130
-		offset += 4;
1131
-	}
1132
-	opp_info->current_rm = rm;
1133
-
1134
-	return 0;
1135
-}
1136
-
1137
-static const struct rockchip_opp_data rk3588_npu_opp_data = {
1138
-	.set_read_margin = rk3588_npu_set_read_margin,
1139
-};
1140
-
1141
-static const struct of_device_id rockchip_npu_of_match[] = {
1142
-	{
1143
-		.compatible = "rockchip,rk3588",
1144
-		.data = (void *)&rk3588_npu_opp_data,
1145
-	},
1146
-	{},
1147
-};
1148
-
1149
-static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
1150
-{
1151
-	struct device *dev = rknpu_dev->dev;
1152
-	struct devfreq_dev_profile *dp = &npu_devfreq_profile;
1153
-	struct dev_pm_opp *opp;
1154
-	struct opp_table *reg_table = NULL;
1155
-	struct opp_table *opp_table = NULL;
1156
-	const char *const reg_names[] = { "rknpu", "mem" };
1157
-	int ret = -EINVAL;
1158
-
1159
-	if (of_find_property(dev->of_node, "rknpu-supply", NULL) &&
1160
-	    of_find_property(dev->of_node, "mem-supply", NULL)) {
1161
-		reg_table = dev_pm_opp_set_regulators(dev, reg_names, 2);
1162
-		if (IS_ERR(reg_table))
1163
-			return PTR_ERR(reg_table);
1164
-		opp_table =
1165
-			dev_pm_opp_register_set_opp_helper(dev, npu_opp_helper);
1166
-		if (IS_ERR(opp_table)) {
1167
-			dev_pm_opp_put_regulators(reg_table);
1168
-			return PTR_ERR(opp_table);
1169
-		}
1170
-	} else {
1171
-		reg_table = dev_pm_opp_set_regulators(dev, reg_names, 1);
1172
-		if (IS_ERR(reg_table))
1173
-			return PTR_ERR(reg_table);
1174
-	}
1175
-
1176
-	rockchip_get_opp_data(rockchip_npu_of_match, &rknpu_dev->opp_info);
1177
-	ret = rockchip_init_opp_table(dev, &rknpu_dev->opp_info, "npu_leakage",
1178
-				      "rknpu");
1179
-	if (ret) {
1180
-		LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
1181
-		return ret;
1182
-	}
1183
-
1184
-	rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1185
-
1186
-	opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0);
1187
-	if (IS_ERR(opp)) {
1188
-		ret = PTR_ERR(opp);
1189
-		goto err_remove_table;
1190
-	}
1191
-	dev_pm_opp_put(opp);
1192
-	dp->initial_freq = rknpu_dev->current_freq;
1193
-
1194
-	ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
1195
-	if (ret) {
1196
-		LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
1197
-		goto err_remove_table;
1198
-	}
1199
-
1200
-	rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
1201
-						     (void *)rknpu_dev);
1202
-	if (IS_ERR(rknpu_dev->devfreq)) {
1203
-		LOG_DEV_ERROR(dev, "failed to add devfreq\n");
1204
-		ret = PTR_ERR(rknpu_dev->devfreq);
1205
-		goto err_remove_governor;
1206
-	}
1207
-	devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq);
1208
-
1209
-	rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq;
1210
-	rknpu_dev->devfreq->last_status.total_time = 1;
1211
-	rknpu_dev->devfreq->last_status.busy_time = 1;
1212
-
1213
-	npu_mdevp.data = rknpu_dev->devfreq;
1214
-	npu_mdevp.opp_info = &rknpu_dev->opp_info;
1215
-	rknpu_dev->mdev_info =
1216
-		rockchip_system_monitor_register(dev, &npu_mdevp);
1217
-	if (IS_ERR(rknpu_dev->mdev_info)) {
1218
-		LOG_DEV_DEBUG(dev, "without system monitor\n");
1219
-		rknpu_dev->mdev_info = NULL;
1220
-		npu_mdevp.is_checked = true;
1221
-	}
1222
-	rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1223
-	rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
1224
-
1225
-	of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
1226
-			     (u32 *)&npu_cooling_power.dyn_power_coeff);
1227
-	rknpu_dev->model_data =
1228
-		rockchip_ipa_power_model_init(dev, "npu_leakage");
1229
-	if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
1230
-		rknpu_dev->model_data = NULL;
1231
-		LOG_DEV_ERROR(dev, "failed to initialize power model\n");
1232
-	} else if (rknpu_dev->model_data->dynamic_coefficient) {
1233
-		npu_cooling_power.dyn_power_coeff =
1234
-			rknpu_dev->model_data->dynamic_coefficient;
1235
-	}
1236
-	if (!npu_cooling_power.dyn_power_coeff) {
1237
-		LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
1238
-		goto out;
1239
-	}
1240
-
1241
-	rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
1242
-		dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
1243
-	if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
1244
-		LOG_DEV_ERROR(dev, "failed to register cooling device\n");
1245
-
1246
-out:
1247
-	return 0;
1248
-
1249
-err_remove_governor:
1250
-	devfreq_remove_governor(&devfreq_rknpu_ondemand);
1251
-err_remove_table:
1252
-	dev_pm_opp_of_remove_table(dev);
1253
-
1254
-	rknpu_dev->devfreq = NULL;
1255
-
1256
-	return ret;
1257
-}
1258
-
1259
-#else
1260
-
1261
-static int npu_devfreq_adjust_current_freq_volt(struct device *dev,
1262
-						struct rknpu_device *rknpu_dev)
1263
-{
1264
-	unsigned long volt, old_freq, freq;
1265
-	struct dev_pm_opp *opp = NULL;
1266
-	int ret = -EINVAL;
1267
-
1268
-	old_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1269
-	freq = old_freq;
1270
-
1271
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
1272
-	rcu_read_lock();
1273
-#endif
1274
-
1275
-	opp = devfreq_recommended_opp(dev, &freq, 0);
1276
-	volt = dev_pm_opp_get_voltage(opp);
1277
-
1278
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
1279
-	rcu_read_unlock();
1280
-#endif
1281
-
1282
-	if (freq >= old_freq && rknpu_dev->vdd) {
1283
-		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1284
-		if (ret) {
1285
-			LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
1286
-			return ret;
1287
-		}
1288
-	}
1289
-	LOG_DEV_DEBUG(dev, "adjust current freq=%luHz, volt=%luuV\n", freq,
1290
-		      volt);
1291
-	ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
1292
-	if (ret) {
1293
-		LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
1294
-		return ret;
1295
-	}
1296
-	if (freq < old_freq && rknpu_dev->vdd) {
1297
-		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1298
-		if (ret) {
1299
-			LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
1300
-			return ret;
1301
-		}
1302
-	}
1303
-	rknpu_dev->current_freq = freq;
1304
-	rknpu_dev->current_volt = volt;
1305
-
1306
-	return 0;
1307
-}
1308
-
1309
-static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
1310
-{
1311
-	struct device *dev = rknpu_dev->dev;
1312
-	struct devfreq_dev_profile *dp = &npu_devfreq_profile;
1313
-	int ret = -EINVAL;
1314
-
1315
-	ret = rockchip_init_opp_table(dev, NULL, "npu_leakage", "rknpu");
1316
-	if (ret) {
1317
-		LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
1318
-		return ret;
1319
-	}
1320
-
1321
-	ret = npu_devfreq_adjust_current_freq_volt(dev, rknpu_dev);
1322
-	if (ret) {
1323
-		LOG_DEV_ERROR(dev, "failed to adjust current freq volt\n");
1324
-		goto err_remove_table;
1325
-	}
1326
-	dp->initial_freq = rknpu_dev->current_freq;
1327
-
1328
-	ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
1329
-	if (ret) {
1330
-		LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
1331
-		goto err_remove_table;
1332
-	}
1333
-
1334
-	rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
1335
-						     (void *)rknpu_dev);
1336
-	if (IS_ERR(rknpu_dev->devfreq)) {
1337
-		LOG_DEV_ERROR(dev, "failed to add devfreq\n");
1338
-		ret = PTR_ERR(rknpu_dev->devfreq);
1339
-		goto err_remove_governor;
1340
-	}
1341
-	devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq);
1342
-
1343
-	rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq;
1344
-	rknpu_dev->devfreq->last_status.total_time = 1;
1345
-	rknpu_dev->devfreq->last_status.busy_time = 1;
1346
-
1347
-	npu_mdevp.data = rknpu_dev->devfreq;
1348
-	rknpu_dev->mdev_info =
1349
-		rockchip_system_monitor_register(dev, &npu_mdevp);
1350
-	if (IS_ERR(rknpu_dev->mdev_info)) {
1351
-		LOG_DEV_DEBUG(dev, "without system monitor\n");
1352
-		rknpu_dev->mdev_info = NULL;
1353
-	}
1354
-	rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1355
-	rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
1356
-
1357
-	of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
1358
-			     (u32 *)&npu_cooling_power.dyn_power_coeff);
1359
-	rknpu_dev->model_data =
1360
-		rockchip_ipa_power_model_init(dev, "npu_leakage");
1361
-	if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
1362
-		rknpu_dev->model_data = NULL;
1363
-		LOG_DEV_ERROR(dev, "failed to initialize power model\n");
1364
-	} else if (rknpu_dev->model_data->dynamic_coefficient) {
1365
-		npu_cooling_power.dyn_power_coeff =
1366
-			rknpu_dev->model_data->dynamic_coefficient;
1367
-	}
1368
-
1369
-	if (!npu_cooling_power.dyn_power_coeff) {
1370
-		LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
1371
-		goto out;
1372
-	}
1373
-
1374
-	rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
1375
-		dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
1376
-	if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
1377
-		LOG_DEV_ERROR(dev, "failed to register cooling device\n");
1378
-
1379
-out:
1380
-	return 0;
1381
-
1382
-err_remove_governor:
1383
-	devfreq_remove_governor(&devfreq_rknpu_ondemand);
1384
-err_remove_table:
1385
-	dev_pm_opp_of_remove_table(dev);
1386
-
1387
-	rknpu_dev->devfreq = NULL;
1388
-
1389
-	return ret;
1390
-}
1391
-#endif
1392
-
1393
-static int rknpu_devfreq_remove(struct rknpu_device *rknpu_dev)
1394
-{
1395
-	if (rknpu_dev->devfreq) {
1396
-		devfreq_unregister_opp_notifier(rknpu_dev->dev,
1397
-						rknpu_dev->devfreq);
1398
-		dev_pm_opp_of_remove_table(rknpu_dev->dev);
1399
-		devfreq_remove_governor(&devfreq_rknpu_ondemand);
1400
-	}
1401
-
1402
-	return 0;
1403
-}
1404
-
1405
-#endif
1406949 
1407950 static int rknpu_register_irq(struct platform_device *pdev,
1408951 			      struct rknpu_device *rknpu_dev)
....@@ -1504,6 +1047,53 @@
15041047 	return 0;
15051048 }
15061049 
1050
+static int rknpu_find_nbuf_resource(struct rknpu_device *rknpu_dev)
1051
+{
1052
+	struct device *dev = rknpu_dev->dev;
1053
+
1054
+	if (rknpu_dev->config->nbuf_size == 0)
1055
+		return -EINVAL;
1056
+
1057
+	rknpu_dev->nbuf_start = rknpu_dev->config->nbuf_phyaddr;
1058
+	rknpu_dev->nbuf_size = rknpu_dev->config->nbuf_size;
1059
+	rknpu_dev->nbuf_base_io =
1060
+		devm_ioremap(dev, rknpu_dev->nbuf_start, rknpu_dev->nbuf_size);
1061
+	if (IS_ERR(rknpu_dev->nbuf_base_io)) {
1062
+		LOG_DEV_ERROR(dev, "failed to remap nbuf base io!\n");
1063
+		rknpu_dev->nbuf_base_io = NULL;
1064
+	}
1065
+
1066
+	rknpu_dev->nbuf_end = rknpu_dev->nbuf_start + rknpu_dev->nbuf_size;
1067
+
1068
+	LOG_DEV_INFO(dev, "nbuf region: [%pa, %pa), nbuf size: %#x\n",
1069
+		     &rknpu_dev->nbuf_start, &rknpu_dev->nbuf_end,
1070
+		     rknpu_dev->nbuf_size);
1071
+
1072
+	return 0;
1073
+}
1074
+
1075
+static int rknpu_get_invalid_core_mask(struct device *dev)
1076
+{
1077
+	int ret = 0;
1078
+	u8 invalid_core_mask = 0;
1079
+
1080
+	if (of_property_match_string(dev->of_node, "nvmem-cell-names",
1081
+				     "cores") >= 0) {
1082
+		ret = rockchip_nvmem_cell_read_u8(dev->of_node, "cores",
1083
+						  &invalid_core_mask);
1084
+		/* The default valid npu cores for RK3583 are core0 and core1 */
1085
+		invalid_core_mask |= RKNPU_CORE2_MASK;
1086
+		if (ret) {
1087
+			LOG_DEV_ERROR(
1088
+				dev,
1089
+				"failed to get specification_serial_number\n");
1090
+			return invalid_core_mask;
1091
+		}
1092
+	}
1093
+
1094
+	return (int)invalid_core_mask;
1095
+}
1096
+
15071097 static int rknpu_probe(struct platform_device *pdev)
15081098 {
15091099 	struct resource *res = NULL;
....@@ -1535,6 +1125,22 @@
15351125 	if (!config)
15361126 		return -EINVAL;
15371127 
1128
+	if (match->data == (void *)&rk3588_rknpu_config) {
1129
+		int invalid_core_mask = rknpu_get_invalid_core_mask(dev);
1130
+		/* The default valid npu cores for RK3583 are core0 and core1 */
1131
+		if (invalid_core_mask & RKNPU_CORE2_MASK) {
1132
+			if ((invalid_core_mask & RKNPU_CORE0_MASK) ||
1133
+			    (invalid_core_mask & RKNPU_CORE1_MASK)) {
1134
+				LOG_DEV_ERROR(
1135
+					dev,
1136
+					"rknpu core invalid, invalid core mask: %#x\n",
1137
+					invalid_core_mask);
1138
+				return -ENODEV;
1139
+			}
1140
+			config = &rk3583_rknpu_config;
1141
+		}
1142
+	}
1143
+
15381144 	rknpu_dev->config = config;
15391145 	rknpu_dev->dev = dev;
15401146 
....@@ -1563,11 +1169,6 @@
15631169 	}
15641170 
15651171 #ifndef FPGA_PLATFORM
1566
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
1567
-	if (strstr(__clk_get_name(rknpu_dev->clks[0].clk), "scmi"))
1568
-		rknpu_dev->opp_info.scmi_clk = rknpu_dev->clks[0].clk;
1569
-#endif
1570
-
15711172 	rknpu_dev->vdd = devm_regulator_get_optional(dev, "rknpu");
15721173 	if (IS_ERR(rknpu_dev->vdd)) {
15731174 		if (PTR_ERR(rknpu_dev->vdd) != -ENODEV) {
....@@ -1694,9 +1295,11 @@
16941295 		virt_dev = dev_pm_domain_attach_by_name(dev, "npu1");
16951296 		if (!IS_ERR(virt_dev))
16961297 			rknpu_dev->genpd_dev_npu1 = virt_dev;
1697
-		virt_dev = dev_pm_domain_attach_by_name(dev, "npu2");
1698
-		if (!IS_ERR(virt_dev))
1699
-			rknpu_dev->genpd_dev_npu2 = virt_dev;
1298
+		if (config->num_irqs > 2) {
1299
+			virt_dev = dev_pm_domain_attach_by_name(dev, "npu2");
1300
+			if (!IS_ERR(virt_dev))
1301
+				rknpu_dev->genpd_dev_npu2 = virt_dev;
1302
+		}
17001303 		rknpu_dev->multiple_domains = true;
17011304 	}
17021305 
....@@ -1720,7 +1323,8 @@
17201323 	INIT_DEFERRABLE_WORK(&rknpu_dev->power_off_work,
17211324 			     rknpu_power_off_delay_work);
17221325 
1723
-	if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) {
1326
+	if (IS_ENABLED(CONFIG_NO_GKI) &&
1327
+	    IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) {
17241328 		if (!rknpu_find_sram_resource(rknpu_dev)) {
17251329 			ret = rknpu_mm_create(rknpu_dev->sram_size, PAGE_SIZE,
17261330 					      &rknpu_dev->sram_mm);
....@@ -1730,6 +1334,10 @@
17301334 			LOG_DEV_WARN(dev, "could not find sram resource!\n");
17311335 		}
17321336 	}
1337
+
1338
+	if (IS_ENABLED(CONFIG_NO_GKI) && rknpu_dev->iommu_en &&
1339
+	    rknpu_dev->config->nbuf_size > 0)
1340
+		rknpu_find_nbuf_resource(rknpu_dev);
17331341 
17341342 	rknpu_power_off(rknpu_dev);
17351343 	atomic_set(&rknpu_dev->power_refcount, 0);
....@@ -1809,47 +1417,14 @@
18091417 }
18101418 
18111419 #ifndef FPGA_PLATFORM
1812
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
18131420 static int rknpu_runtime_suspend(struct device *dev)
18141421 {
1815
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1816
-	struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
1817
-
1818
-	if (opp_info->scmi_clk) {
1819
-		if (clk_set_rate(opp_info->scmi_clk, POWER_DOWN_FREQ))
1820
-			LOG_DEV_ERROR(dev, "failed to restore clk rate\n");
1821
-	}
1822
-	opp_info->current_rm = UINT_MAX;
1823
-
1824
-	return 0;
1422
+	return rknpu_devfreq_runtime_suspend(dev);
18251423 }
18261424 
18271425 static int rknpu_runtime_resume(struct device *dev)
18281426 {
1829
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1830
-	struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
1831
-	int ret = 0;
1832
-
1833
-	if (!rknpu_dev->current_freq || !rknpu_dev->current_volt)
1834
-		return 0;
1835
-
1836
-	ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
1837
-	if (ret) {
1838
-		LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
1839
-		return ret;
1840
-	}
1841
-
1842
-	if (opp_info->data && opp_info->data->set_read_margin)
1843
-		opp_info->data->set_read_margin(dev, opp_info,
1844
-						opp_info->target_rm);
1845
-	if (opp_info->scmi_clk) {
1846
-		if (clk_set_rate(opp_info->scmi_clk, rknpu_dev->current_freq))
1847
-			LOG_DEV_ERROR(dev, "failed to set power down rate\n");
1848
-	}
1849
-
1850
-	clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
1851
-
1852
-	return ret;
1427
+	return rknpu_devfreq_runtime_resume(dev);
18531428 }
18541429 
18551430 static const struct dev_pm_ops rknpu_pm_ops = {
....@@ -1859,7 +1434,6 @@
18591434 				   NULL)
18601435 };
18611436 #endif
1862
-#endif
18631437 
18641438 static struct platform_driver rknpu_driver = {
18651439 	.probe = rknpu_probe,
....@@ -1868,9 +1442,7 @@
18681442 		.owner = THIS_MODULE,
18691443 		.name = "RKNPU",
18701444 #ifndef FPGA_PLATFORM
1871
-#if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
18721445 		.pm = &rknpu_pm_ops,
1873
-#endif
18741446 #endif
18751447 		.of_match_table = of_match_ptr(rknpu_of_match),
18761448 	},
....@@ -1895,3 +1467,6 @@
18951467 MODULE_LICENSE("GPL v2");
18961468 MODULE_VERSION(RKNPU_GET_DRV_VERSION_STRING(DRIVER_MAJOR, DRIVER_MINOR,
18971469 					    DRIVER_PATCHLEVEL));
1470
+#if KERNEL_VERSION(5, 16, 0) < LINUX_VERSION_CODE
1471
+MODULE_IMPORT_NS(DMA_BUF);
1472
+#endif