forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 b22da3d8526a935aa31e086e63f60ff3246cb61c 
 kernel/drivers/rknpu/rknpu_drv.c
....@@ -41,7 +41,9 @@
4141 #include <soc/rockchip/rockchip_opp_select.h>
4242 #include <soc/rockchip/rockchip_system_monitor.h>
4343 #include <soc/rockchip/rockchip_ipa.h>
44
+#ifdef CONFIG_PM_DEVFREQ
4445 #include <../drivers/devfreq/governor.h>
46
+#endif
4547 #endif
4648 
4749 #include "rknpu_ioctl.h"
....@@ -77,36 +79,29 @@
7779 MODULE_PARM_DESC(bypass_soft_reset,
7880 		 "bypass RKNPU soft reset if set it to 1, disabled by default");
7981 
80
-struct npu_irqs_data {
82
+struct rknpu_irqs_data {
8183 	const char *name;
8284 	irqreturn_t (*irq_hdl)(int irq, void *ctx);
8385 };
8486 
85
-static const struct npu_irqs_data rk356x_npu_irqs[] = {
87
+static const struct rknpu_irqs_data rknpu_irqs[] = {
8688 	{ "npu_irq", rknpu_core0_irq_handler }
8789 };
8890 
89
-static const struct npu_irqs_data rk3588_npu_irqs[] = {
91
+static const struct rknpu_irqs_data rk3588_npu_irqs[] = {
9092 	{ "npu0_irq", rknpu_core0_irq_handler },
9193 	{ "npu1_irq", rknpu_core1_irq_handler },
9294 	{ "npu2_irq", rknpu_core2_irq_handler }
9395 };
9496 
95
-static const struct npu_irqs_data rv110x_npu_irqs[] = {
96
-	{ "npu_irq", rknpu_core0_irq_handler }
97
-};
97
+static const struct rknpu_reset_data rknpu_resets[] = { { "srst_a",
98
+							  "srst_h" } };
9899 
99
-static const struct npu_reset_data rk356x_npu_resets[] = { { "srst_a",
100
-							     "srst_h" } };
101
-
102
-static const struct npu_reset_data rk3588_npu_resets[] = {
100
+static const struct rknpu_reset_data rk3588_npu_resets[] = {
103101 	{ "srst_a0", "srst_h0" },
104102 	{ "srst_a1", "srst_h1" },
105103 	{ "srst_a2", "srst_h2" }
106104 };
107
-
108
-static const struct npu_reset_data rv110x_npu_resets[] = { { "srst_a",
109
-							     "srst_h" } };
110105 
111106 static const struct rknpu_config rk356x_rknpu_config = {
112107 	.bw_priority_addr = 0xfe180008,
....@@ -115,11 +110,15 @@
115110 	.pc_data_amount_scale = 1,
116111 	.pc_task_number_bits = 12,
117112 	.pc_task_number_mask = 0xfff,
113
+	.pc_task_status_offset = 0x3c,
114
+	.pc_dma_ctrl = 0,
118115 	.bw_enable = 1,
119
-	.irqs = rk356x_npu_irqs,
120
-	.resets = rk356x_npu_resets,
121
-	.num_irqs = ARRAY_SIZE(rk356x_npu_irqs),
122
-	.num_resets = ARRAY_SIZE(rk356x_npu_resets)
116
+	.irqs = rknpu_irqs,
117
+	.resets = rknpu_resets,
118
+	.num_irqs = ARRAY_SIZE(rknpu_irqs),
119
+	.num_resets = ARRAY_SIZE(rknpu_resets),
120
+	.nbuf_phyaddr = 0,
121
+	.nbuf_size = 0
123122 };
124123 
125124 static const struct rknpu_config rk3588_rknpu_config = {
....@@ -129,11 +128,15 @@
129128 	.pc_data_amount_scale = 2,
130129 	.pc_task_number_bits = 12,
131130 	.pc_task_number_mask = 0xfff,
131
+	.pc_task_status_offset = 0x3c,
132
+	.pc_dma_ctrl = 0,
132133 	.bw_enable = 0,
133134 	.irqs = rk3588_npu_irqs,
134135 	.resets = rk3588_npu_resets,
135136 	.num_irqs = ARRAY_SIZE(rk3588_npu_irqs),
136
-	.num_resets = ARRAY_SIZE(rk3588_npu_resets)
137
+	.num_resets = ARRAY_SIZE(rk3588_npu_resets),
138
+	.nbuf_phyaddr = 0,
139
+	.nbuf_size = 0
137140 };
138141 
139142 static const struct rknpu_config rv1106_rknpu_config = {
....@@ -143,11 +146,33 @@
143146 	.pc_data_amount_scale = 2,
144147 	.pc_task_number_bits = 16,
145148 	.pc_task_number_mask = 0xffff,
149
+	.pc_task_status_offset = 0x3c,
150
+	.pc_dma_ctrl = 0,
146151 	.bw_enable = 1,
147
-	.irqs = rv110x_npu_irqs,
148
-	.resets = rv110x_npu_resets,
149
-	.num_irqs = ARRAY_SIZE(rv110x_npu_irqs),
150
-	.num_resets = ARRAY_SIZE(rv110x_npu_resets)
152
+	.irqs = rknpu_irqs,
153
+	.resets = rknpu_resets,
154
+	.num_irqs = ARRAY_SIZE(rknpu_irqs),
155
+	.num_resets = ARRAY_SIZE(rknpu_resets),
156
+	.nbuf_phyaddr = 0,
157
+	.nbuf_size = 0
158
+};
159
+
160
+static const struct rknpu_config rk3562_rknpu_config = {
161
+	.bw_priority_addr = 0x0,
162
+	.bw_priority_length = 0x0,
163
+	.dma_mask = DMA_BIT_MASK(40),
164
+	.pc_data_amount_scale = 2,
165
+	.pc_task_number_bits = 16,
166
+	.pc_task_number_mask = 0xffff,
167
+	.pc_task_status_offset = 0x48,
168
+	.pc_dma_ctrl = 1,
169
+	.bw_enable = 1,
170
+	.irqs = rknpu_irqs,
171
+	.resets = rknpu_resets,
172
+	.num_irqs = ARRAY_SIZE(rknpu_irqs),
173
+	.num_resets = ARRAY_SIZE(rknpu_resets),
174
+	.nbuf_phyaddr = 0xfe400000,
175
+	.nbuf_size = 256 * 1024
151176 };
152177 
153178 /* driver probe and init */
....@@ -167,6 +192,10 @@
167192 	{
168193 		.compatible = "rockchip,rv1106-rknpu",
169194 		.data = &rv1106_rknpu_config,
195
+	},
196
+	{
197
+		.compatible = "rockchip,rk3562-rknpu",
198
+		.data = &rk3562_rknpu_config,
170199 	},
171200 	{},
172201 };
....@@ -243,13 +272,17 @@
243272 		ret = rknpu_get_drv_version(&args->value);
244273 		break;
245274 	case RKNPU_GET_FREQ:
275
+#ifndef FPGA_PLATFORM
246276 		args->value = clk_get_rate(rknpu_dev->clks[0].clk);
277
+#endif
247278 		ret = 0;
248279 		break;
249280 	case RKNPU_SET_FREQ:
250281 		break;
251282 	case RKNPU_GET_VOLT:
283
+#ifndef FPGA_PLATFORM
252284 		args->value = regulator_get_voltage(rknpu_dev->vdd);
285
+#endif
253286 		ret = 0;
254287 		break;
255288 	case RKNPU_SET_VOLT:
....@@ -328,11 +361,56 @@
328361 #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
329362 static int rknpu_open(struct inode *inode, struct file *file)
330363 {
364
+	struct rknpu_device *rknpu_dev =
365
+		container_of(file->private_data, struct rknpu_device, miscdev);
366
+	struct rknpu_session *session = NULL;
367
+
368
+	session = kzalloc(sizeof(*session), GFP_KERNEL);
369
+	if (!session) {
370
+		LOG_ERROR("rknpu session alloc failed\n");
371
+		return -ENOMEM;
372
+	}
373
+
374
+	session->rknpu_dev = rknpu_dev;
375
+	INIT_LIST_HEAD(&session->list);
376
+
377
+	file->private_data = (void *)session;
378
+
331379 	return nonseekable_open(inode, file);
332380 }
333381 
334382 static int rknpu_release(struct inode *inode, struct file *file)
335383 {
384
+	struct rknpu_mem_object *entry;
385
+	struct rknpu_session *session = file->private_data;
386
+	struct rknpu_device *rknpu_dev = session->rknpu_dev;
387
+	LIST_HEAD(local_list);
388
+
389
+	spin_lock(&rknpu_dev->lock);
390
+	list_replace_init(&session->list, &local_list);
391
+	file->private_data = NULL;
392
+	spin_unlock(&rknpu_dev->lock);
393
+
394
+	while (!list_empty(&local_list)) {
395
+		entry = list_first_entry(&local_list, struct rknpu_mem_object,
396
+					 head);
397
+
398
+		LOG_DEBUG(
399
+			"Fd close free rknpu_obj: %#llx, rknpu_obj->dma_addr: %#llx\n",
400
+			(__u64)(uintptr_t)entry, (__u64)entry->dma_addr);
401
+
402
+		vunmap(entry->kv_addr);
403
+		entry->kv_addr = NULL;
404
+
405
+		if (!entry->owner)
406
+			dma_buf_put(entry->dmabuf);
407
+
408
+		list_del(&entry->head);
409
+		kfree(entry);
410
+	}
411
+
412
+	kfree(session);
413
+
336414 	return 0;
337415 }
338416 
....@@ -364,8 +442,12 @@
364442 static long rknpu_ioctl(struct file *file, uint32_t cmd, unsigned long arg)
365443 {
366444 	long ret = -EINVAL;
367
-	struct rknpu_device *rknpu_dev =
368
-		container_of(file->private_data, struct rknpu_device, miscdev);
445
+	struct rknpu_device *rknpu_dev = NULL;
446
+
447
+	if (!file->private_data)
448
+		return -EINVAL;
449
+
450
+	rknpu_dev = ((struct rknpu_session *)file->private_data)->rknpu_dev;
369451 
370452 	rknpu_power_get(rknpu_dev);
371453 
....@@ -377,12 +459,12 @@
377459 		ret = rknpu_submit_ioctl(rknpu_dev, arg);
378460 		break;
379461 	case IOCTL_RKNPU_MEM_CREATE:
380
-		ret = rknpu_mem_create_ioctl(rknpu_dev, arg);
462
+		ret = rknpu_mem_create_ioctl(rknpu_dev, arg, file);
381463 		break;
382464 	case RKNPU_MEM_MAP:
383465 		break;
384466 	case IOCTL_RKNPU_MEM_DESTROY:
385
-		ret = rknpu_mem_destroy_ioctl(rknpu_dev, arg);
467
+		ret = rknpu_mem_destroy_ioctl(rknpu_dev, arg, file);
386468 		break;
387469 	case IOCTL_RKNPU_MEM_SYNC:
388470 		ret = rknpu_mem_sync_ioctl(rknpu_dev, arg);
....@@ -800,7 +882,7 @@
800882 
801883 #ifndef FPGA_PLATFORM
802884 static struct monitor_dev_profile npu_mdevp = {
803
-	.type = MONITOR_TPYE_DEV,
885
+	.type = MONITOR_TYPE_DEV,
804886 	.low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
805887 	.high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
806888 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
....@@ -1062,6 +1144,7 @@
10621144 	.get_cur_freq = npu_devfreq_get_cur_freq,
10631145 };
10641146 
1147
+#ifdef CONFIG_PM_DEVFREQ
10651148 static int devfreq_rknpu_ondemand_func(struct devfreq *df, unsigned long *freq)
10661149 {
10671150 	struct rknpu_device *rknpu_dev = df->data;
....@@ -1085,6 +1168,7 @@
10851168 	.get_target_freq = devfreq_rknpu_ondemand_func,
10861169 	.event_handler = devfreq_rknpu_ondemand_handler,
10871170 };
1171
+#endif
10881172 
10891173 static unsigned long npu_get_static_power(struct devfreq *devfreq,
10901174 					  unsigned long voltage)
....@@ -1103,6 +1187,66 @@
11031187 };
11041188 
11051189 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
1190
+static int rk3588_npu_get_soc_info(struct device *dev, struct device_node *np,
1191
+				   int *bin, int *process)
1192
+{
1193
+	int ret = 0;
1194
+	u8 value = 0;
1195
+
1196
+	if (!bin)
1197
+		return 0;
1198
+
1199
+	if (of_property_match_string(np, "nvmem-cell-names",
1200
+				     "specification_serial_number") >= 0) {
1201
+		ret = rockchip_nvmem_cell_read_u8(
1202
+			np, "specification_serial_number", &value);
1203
+		if (ret) {
1204
+			LOG_DEV_ERROR(
1205
+				dev,
1206
+				"Failed to get specification_serial_number\n");
1207
+			return ret;
1208
+		}
1209
+		/* RK3588M */
1210
+		if (value == 0xd)
1211
+			*bin = 1;
1212
+		/* RK3588J */
1213
+		else if (value == 0xa)
1214
+			*bin = 2;
1215
+	}
1216
+	if (*bin < 0)
1217
+		*bin = 0;
1218
+	LOG_DEV_INFO(dev, "bin=%d\n", *bin);
1219
+
1220
+	return ret;
1221
+}
1222
+
1223
+static int rk3588_npu_set_soc_info(struct device *dev, struct device_node *np,
1224
+				   int bin, int process, int volt_sel)
1225
+{
1226
+	struct opp_table *opp_table;
1227
+	u32 supported_hw[2];
1228
+
1229
+	if (volt_sel < 0)
1230
+		return 0;
1231
+	if (bin < 0)
1232
+		bin = 0;
1233
+
1234
+	if (!of_property_read_bool(np, "rockchip,supported-hw"))
1235
+		return 0;
1236
+
1237
+	/* SoC Version */
1238
+	supported_hw[0] = BIT(bin);
1239
+	/* Speed Grade */
1240
+	supported_hw[1] = BIT(volt_sel);
1241
+	opp_table = dev_pm_opp_set_supported_hw(dev, supported_hw, 2);
1242
+	if (IS_ERR(opp_table)) {
1243
+		LOG_DEV_ERROR(dev, "failed to set supported opp\n");
1244
+		return PTR_ERR(opp_table);
1245
+	}
1246
+
1247
+	return 0;
1248
+}
1249
+
11061250 static int rk3588_npu_set_read_margin(struct device *dev,
11071251 				      struct rockchip_opp_info *opp_info,
11081252 				      u32 rm)
....@@ -1135,6 +1279,8 @@
11351279 }
11361280 
11371281 static const struct rockchip_opp_data rk3588_npu_opp_data = {
1282
+	.get_soc_info = rk3588_npu_get_soc_info,
1283
+	.set_soc_info = rk3588_npu_set_soc_info,
11381284 	.set_read_margin = rk3588_npu_set_read_margin,
11391285 };
11401286 
....@@ -1191,11 +1337,13 @@
11911337 	dev_pm_opp_put(opp);
11921338 	dp->initial_freq = rknpu_dev->current_freq;
11931339 
1340
+#ifdef CONFIG_PM_DEVFREQ
11941341 	ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
11951342 	if (ret) {
11961343 		LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
11971344 		goto err_remove_table;
11981345 	}
1346
+#endif
11991347 
12001348 	rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
12011349 						     (void *)rknpu_dev);
....@@ -1247,7 +1395,9 @@
12471395 	return 0;
12481396 
12491397 err_remove_governor:
1398
+#ifdef CONFIG_PM_DEVFREQ
12501399 	devfreq_remove_governor(&devfreq_rknpu_ondemand);
1400
+#endif
12511401 err_remove_table:
12521402 	dev_pm_opp_of_remove_table(dev);
12531403 
....@@ -1325,11 +1475,13 @@
13251475 	}
13261476 	dp->initial_freq = rknpu_dev->current_freq;
13271477 
1478
+#ifdef CONFIG_PM_DEVFREQ
13281479 	ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
13291480 	if (ret) {
13301481 		LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
13311482 		goto err_remove_table;
13321483 	}
1484
+#endif
13331485 
13341486 	rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
13351487 						     (void *)rknpu_dev);
....@@ -1380,7 +1532,9 @@
13801532 	return 0;
13811533 
13821534 err_remove_governor:
1535
+#ifdef CONFIG_PM_DEVFREQ
13831536 	devfreq_remove_governor(&devfreq_rknpu_ondemand);
1537
+#endif
13841538 err_remove_table:
13851539 	dev_pm_opp_of_remove_table(dev);
13861540 
....@@ -1396,7 +1550,9 @@
13961550 		devfreq_unregister_opp_notifier(rknpu_dev->dev,
13971551 						rknpu_dev->devfreq);
13981552 		dev_pm_opp_of_remove_table(rknpu_dev->dev);
1553
+#ifdef CONFIG_PM_DEVFREQ
13991554 		devfreq_remove_governor(&devfreq_rknpu_ondemand);
1555
+#endif
14001556 	}
14011557 
14021558 	return 0;
....@@ -1500,6 +1656,31 @@
15001656 	LOG_DEV_INFO(dev, "sram region: [%pa, %pa), sram size: %#x\n",
15011657 		     &rknpu_dev->sram_start, &rknpu_dev->sram_end,
15021658 		     rknpu_dev->sram_size);
1659
+
1660
+	return 0;
1661
+}
1662
+
1663
+static int rknpu_find_nbuf_resource(struct rknpu_device *rknpu_dev)
1664
+{
1665
+	struct device *dev = rknpu_dev->dev;
1666
+
1667
+	if (rknpu_dev->config->nbuf_size == 0)
1668
+		return -EINVAL;
1669
+
1670
+	rknpu_dev->nbuf_start = rknpu_dev->config->nbuf_phyaddr;
1671
+	rknpu_dev->nbuf_size = rknpu_dev->config->nbuf_size;
1672
+	rknpu_dev->nbuf_base_io =
1673
+		devm_ioremap(dev, rknpu_dev->nbuf_start, rknpu_dev->nbuf_size);
1674
+	if (IS_ERR(rknpu_dev->nbuf_base_io)) {
1675
+		LOG_DEV_ERROR(dev, "failed to remap nbuf base io!\n");
1676
+		rknpu_dev->nbuf_base_io = NULL;
1677
+	}
1678
+
1679
+	rknpu_dev->nbuf_end = rknpu_dev->nbuf_start + rknpu_dev->nbuf_size;
1680
+
1681
+	LOG_DEV_INFO(dev, "nbuf region: [%pa, %pa), nbuf size: %#x\n",
1682
+		     &rknpu_dev->nbuf_start, &rknpu_dev->nbuf_end,
1683
+		     rknpu_dev->nbuf_size);
15031684 
15041685 	return 0;
15051686 }
....@@ -1720,7 +1901,8 @@
17201901 	INIT_DEFERRABLE_WORK(&rknpu_dev->power_off_work,
17211902 			     rknpu_power_off_delay_work);
17221903 
1723
-	if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) {
1904
+	if (IS_ENABLED(CONFIG_NO_GKI) &&
1905
+	    IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) {
17241906 		if (!rknpu_find_sram_resource(rknpu_dev)) {
17251907 			ret = rknpu_mm_create(rknpu_dev->sram_size, PAGE_SIZE,
17261908 					      &rknpu_dev->sram_mm);
....@@ -1731,6 +1913,10 @@
17311913 		}
17321914 	}
17331915 
1916
+	if (IS_ENABLED(CONFIG_NO_GKI) && rknpu_dev->iommu_en &&
1917
+	    rknpu_dev->config->nbuf_size > 0)
1918
+		rknpu_find_nbuf_resource(rknpu_dev);
1919
+
17341920 	rknpu_power_off(rknpu_dev);
17351921 	atomic_set(&rknpu_dev->power_refcount, 0);
17361922 	atomic_set(&rknpu_dev->cmdline_power_refcount, 0);