~hc/RK356X_SDK_RELEASE.git

..	..	@@ -33,13 +33,17 @@
33	33	#include <linux/time.h>
34	34	#include <linux/mm.h>
35	35	#include <linux/mman.h>
	36	+#include <linux/dma-buf.h>
36	37	#include <asm/processor.h>
37	38	#include "kfd_priv.h"
38	39	#include "kfd_device_queue_manager.h"
39	40	#include "kfd_dbgmgr.h"
	41	+#include "amdgpu_amdkfd.h"
	42	+#include "kfd_smi_events.h"
40	43
41	44	static long kfd_ioctl(struct file *, unsigned int, unsigned long);
42	45	static int kfd_open(struct inode , struct file );
	46	+static int kfd_release(struct inode , struct file );
43	47	static int kfd_mmap(struct file , struct vm_area_struct );
44	48
45	49	static const char kfd_dev_name[] = "kfd";
..	..	@@ -47,8 +51,9 @@
47	51	static const struct file_operations kfd_fops = {
48	52	.owner = THIS_MODULE,
49	53	.unlocked_ioctl = kfd_ioctl,
50		- .compat_ioctl = kfd_ioctl,
	54	+ .compat_ioctl = compat_ptr_ioctl,
51	55	.open = kfd_open,
	56	+ .release = kfd_release,
52	57	.mmap = kfd_mmap,
53	58	};
54	59
..	..	@@ -92,6 +97,7 @@
92	97	device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0));
93	98	class_destroy(kfd_class);
94	99	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
	100	+ kfd_device = NULL;
95	101	}
96	102
97	103	struct device *kfd_chardev(void)
..	..	@@ -122,11 +128,28 @@
122	128	if (IS_ERR(process))
123	129	return PTR_ERR(process);
124	130
125		- if (kfd_is_locked())
	131	+ if (kfd_is_locked()) {
	132	+ dev_dbg(kfd_device, "kfd is locked!\n"
	133	+ "process %d unreferenced", process->pasid);
	134	+ kfd_unref_process(process);
126	135	return -EAGAIN;
	136	+ }
	137	+
	138	+ /* filep now owns the reference returned by kfd_create_process */
	139	+ filep->private_data = process;
127	140
128	141	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
129	142	process->pasid, process->is_32bit_user_mode);
	143	+
	144	+ return 0;
	145	+}
	146	+
	147	+static int kfd_release(struct inode inode, struct file filep)
	148	+{
	149	+ struct kfd_process *process = filep->private_data;
	150	+
	151	+ if (process)
	152	+ kfd_unref_process(process);
130	153
131	154	return 0;
132	155	}
..	..	@@ -156,8 +179,7 @@
156	179	}
157	180
158	181	if ((args->ring_base_address) &&
159		- (!access_ok(VERIFY_WRITE,
160		- (const void __user *) args->ring_base_address,
	182	+ (!access_ok((const void __user *) args->ring_base_address,
161	183	sizeof(uint64_t)))) {
162	184	pr_err("Can't access ring base address\n");
163	185	return -EFAULT;
..	..	@@ -168,37 +190,34 @@
168	190	return -EINVAL;
169	191	}
170	192
171		- if (!access_ok(VERIFY_WRITE,
172		- (const void __user *) args->read_pointer_address,
	193	+ if (!access_ok((const void __user *) args->read_pointer_address,
173	194	sizeof(uint32_t))) {
174	195	pr_err("Can't access read pointer\n");
175	196	return -EFAULT;
176	197	}
177	198
178		- if (!access_ok(VERIFY_WRITE,
179		- (const void __user *) args->write_pointer_address,
	199	+ if (!access_ok((const void __user *) args->write_pointer_address,
180	200	sizeof(uint32_t))) {
181	201	pr_err("Can't access write pointer\n");
182	202	return -EFAULT;
183	203	}
184	204
185	205	if (args->eop_buffer_address &&
186		- !access_ok(VERIFY_WRITE,
187		- (const void __user *) args->eop_buffer_address,
	206	+ !access_ok((const void __user *) args->eop_buffer_address,
188	207	sizeof(uint32_t))) {
189	208	pr_debug("Can't access eop buffer");
190	209	return -EFAULT;
191	210	}
192	211
193	212	if (args->ctx_save_restore_address &&
194		- !access_ok(VERIFY_WRITE,
195		- (const void __user *) args->ctx_save_restore_address,
	213	+ !access_ok((const void __user *) args->ctx_save_restore_address,
196	214	sizeof(uint32_t))) {
197	215	pr_debug("Can't access ctx save restore buffer");
198	216	return -EFAULT;
199	217	}
200	218
201	219	q_properties->is_interop = false;
	220	+ q_properties->is_gws = false;
202	221	q_properties->queue_percent = args->queue_percentage;
203	222	q_properties->priority = args->queue_priority;
204	223	q_properties->queue_address = args->ring_base_address;
..	..	@@ -216,6 +235,8 @@
216	235	q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
217	236	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
218	237	q_properties->type = KFD_QUEUE_TYPE_SDMA;
	238	+ else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI)
	239	+ q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI;
219	240	else
220	241	return -ENOTSUPP;
221	242
..	..	@@ -259,6 +280,7 @@
259	280	unsigned int queue_id;
260	281	struct kfd_process_device *pdd;
261	282	struct queue_properties q_properties;
	283	+ uint32_t doorbell_offset_in_process = 0;
262	284
263	285	memset(&q_properties, 0, sizeof(struct queue_properties));
264	286
..	..	@@ -283,11 +305,12 @@
283	305	goto err_bind_process;
284	306	}
285	307
286		- pr_debug("Creating queue for PASID %d on gpu 0x%x\n",
	308	+ pr_debug("Creating queue for PASID 0x%x on gpu 0x%x\n",
287	309	p->pasid,
288	310	dev->id);
289	311
290		- err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id);
	312	+ err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id,
	313	+ &doorbell_offset_in_process);
291	314	if (err != 0)
292	315	goto err_create_queue;
293	316
..	..	@@ -297,14 +320,11 @@
297	320	/* Return gpu_id as doorbell offset for mmap usage */
298	321	args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
299	322	args->doorbell_offset \|= KFD_MMAP_GPU_ID(args->gpu_id);
300		- args->doorbell_offset <<= PAGE_SHIFT;
301	323	if (KFD_IS_SOC15(dev->device_info->asic_family))
302		- /* On SOC15 ASICs, doorbell allocation must be
303		- * per-device, and independent from the per-process
304		- * queue_id. Return the doorbell offset within the
305		- * doorbell aperture to user mode.
	324	+ /* On SOC15 ASICs, include the doorbell offset within the
	325	+ * process doorbell frame, which is 2 pages.
306	326	*/
307		- args->doorbell_offset \|= q_properties.doorbell_off;
	327	+ args->doorbell_offset \|= doorbell_offset_in_process;
308	328
309	329	mutex_unlock(&p->mutex);
310	330
..	..	@@ -333,7 +353,7 @@
333	353	int retval;
334	354	struct kfd_ioctl_destroy_queue_args *args = data;
335	355
336		- pr_debug("Destroying queue id %d for pasid %d\n",
	356	+ pr_debug("Destroying queue id %d for pasid 0x%x\n",
337	357	args->queue_id,
338	358	p->pasid);
339	359
..	..	@@ -363,8 +383,7 @@
363	383	}
364	384
365	385	if ((args->ring_base_address) &&
366		- (!access_ok(VERIFY_WRITE,
367		- (const void __user *) args->ring_base_address,
	386	+ (!access_ok((const void __user *) args->ring_base_address,
368	387	sizeof(uint64_t)))) {
369	388	pr_err("Can't access ring base address\n");
370	389	return -EFAULT;
..	..	@@ -380,7 +399,7 @@
380	399	properties.queue_percent = args->queue_percentage;
381	400	properties.priority = args->queue_priority;
382	401
383		- pr_debug("Updating queue id %d for pasid %d\n",
	402	+ pr_debug("Updating queue id %d for pasid 0x%x\n",
384	403	args->queue_id, p->pasid);
385	404
386	405	mutex_lock(&p->mutex);
..	..	@@ -447,6 +466,24 @@
447	466	return retval;
448	467	}
449	468
	469	+static int kfd_ioctl_get_queue_wave_state(struct file *filep,
	470	+ struct kfd_process p, void data)
	471	+{
	472	+ struct kfd_ioctl_get_queue_wave_state_args *args = data;
	473	+ int r;
	474	+
	475	+ mutex_lock(&p->mutex);
	476	+
	477	+ r = pqm_get_wave_state(&p->pqm, args->queue_id,
	478	+ (void __user *)args->ctl_stack_address,
	479	+ &args->ctl_stack_used_size,
	480	+ &args->save_area_used_size);
	481	+
	482	+ mutex_unlock(&p->mutex);
	483	+
	484	+ return r;
	485	+}
	486	+
450	487	static int kfd_ioctl_set_memory_policy(struct file *filep,
451	488	struct kfd_process p, void data)
452	489	{
..	..	@@ -508,7 +545,7 @@
508	545	struct kfd_process_device *pdd;
509	546
510	547	dev = kfd_device_by_id(args->gpu_id);
511		- if (dev == NULL)
	548	+ if (!dev)
512	549	return -EINVAL;
513	550
514	551	mutex_lock(&p->mutex);
..	..	@@ -816,15 +853,14 @@
816	853	dev = kfd_device_by_id(args->gpu_id);
817	854	if (dev)
818	855	/* Reading GPU clock counter from KGD */
819		- args->gpu_clock_counter =
820		- dev->kfd2kgd->get_gpu_clock_counter(dev->kgd);
	856	+ args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(dev->kgd);
821	857	else
822	858	/* Node without GPU resource */
823	859	args->gpu_clock_counter = 0;
824	860
825	861	/* No access to rdtsc. Using raw monotonic time */
826	862	args->cpu_clock_counter = ktime_get_raw_ns();
827		- args->system_clock_counter = ktime_get_boot_ns();
	863	+ args->system_clock_counter = ktime_get_boottime_ns();
828	864
829	865	/* Since the counter is in nano-seconds we use 1GHz frequency */
830	866	args->system_clock_freq = 1000000000;
..	..	@@ -840,7 +876,7 @@
840	876	struct kfd_process_device_apertures *pAperture;
841	877	struct kfd_process_device *pdd;
842	878
843		- dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid);
	879	+ dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
844	880
845	881	args->num_of_nodes = 0;
846	882
..	..	@@ -898,7 +934,7 @@
898	934	uint32_t nodes = 0;
899	935	int ret;
900	936
901		- dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid);
	937	+ dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
902	938
903	939	if (args->num_of_nodes == 0) {
904	940	/* Return number of nodes, so that user space can alloacate
..	..	@@ -1024,7 +1060,7 @@
1024	1060	}
1025	1061	mutex_unlock(&p->mutex);
1026	1062
1027		- err = kfd->kfd2kgd->map_gtt_bo_to_kernel(kfd->kgd,
	1063	+ err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kfd->kgd,
1028	1064	mem, &kern_addr, &size);
1029	1065	if (err) {
1030	1066	pr_err("Failed to map event page to kernel\n");
..	..	@@ -1113,7 +1149,7 @@
1113	1149	mutex_unlock(&p->mutex);
1114	1150
1115	1151	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
1116		- pdd->qpd.vmid != 0)
	1152	+ pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va)
1117	1153	dev->kfd2kgd->set_scratch_backing_va(
1118	1154	dev->kgd, args->va_addr, pdd->qpd.vmid);
1119	1155
..	..	@@ -1136,7 +1172,7 @@
1136	1172	if (!dev)
1137	1173	return -EINVAL;
1138	1174
1139		- dev->kfd2kgd->get_tile_config(dev->kgd, &config);
	1175	+ amdgpu_amdkfd_get_tile_config(dev->kgd, &config);
1140	1176
1141	1177	args->gb_addr_config = config.gb_addr_config;
1142	1178	args->num_banks = config.num_banks;
..	..	@@ -1210,7 +1246,7 @@
1210	1246	return ret;
1211	1247	}
1212	1248
1213		-static bool kfd_dev_is_large_bar(struct kfd_dev *dev)
	1249	+bool kfd_dev_is_large_bar(struct kfd_dev *dev)
1214	1250	{
1215	1251	struct kfd_local_mem_info mem_info;
1216	1252
..	..	@@ -1219,10 +1255,10 @@
1219	1255	return true;
1220	1256	}
1221	1257
1222		- if (dev->device_info->needs_iommu_device)
	1258	+ if (dev->use_iommu_v2)
1223	1259	return false;
1224	1260
1225		- dev->kfd2kgd->get_local_mem_info(dev->kgd, &mem_info);
	1261	+ amdgpu_amdkfd_get_local_mem_info(dev->kgd, &mem_info);
1226	1262	if (mem_info.local_mem_size_private == 0 &&
1227	1263	mem_info.local_mem_size_public > 0)
1228	1264	return true;
..	..	@@ -1263,7 +1299,25 @@
1263	1299	goto err_unlock;
1264	1300	}
1265	1301
1266		- err = dev->kfd2kgd->alloc_memory_of_gpu(
	1302	+ if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
	1303	+ if (args->size != kfd_doorbell_process_slice(dev)) {
	1304	+ err = -EINVAL;
	1305	+ goto err_unlock;
	1306	+ }
	1307	+ offset = kfd_get_process_doorbells(pdd);
	1308	+ } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
	1309	+ if (args->size != PAGE_SIZE) {
	1310	+ err = -EINVAL;
	1311	+ goto err_unlock;
	1312	+ }
	1313	+ offset = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
	1314	+ if (!offset) {
	1315	+ err = -ENOMEM;
	1316	+ goto err_unlock;
	1317	+ }
	1318	+ }
	1319	+
	1320	+ err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1267	1321	dev->kgd, args->va_addr, args->size,
1268	1322	pdd->vm, (struct kgd_mem **) &mem, &offset,
1269	1323	flags);
..	..	@@ -1277,15 +1331,26 @@
1277	1331	goto err_free;
1278	1332	}
1279	1333
	1334	+ /* Update the VRAM usage count */
	1335	+ if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
	1336	+ WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + args->size);
	1337	+
1280	1338	mutex_unlock(&p->mutex);
1281	1339
1282	1340	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1283	1341	args->mmap_offset = offset;
1284	1342
	1343	+ /* MMIO is mapped through kfd device
	1344	+ * Generate a kfd mmap offset
	1345	+ */
	1346	+ if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
	1347	+ args->mmap_offset = KFD_MMAP_TYPE_MMIO
	1348	+ \| KFD_MMAP_GPU_ID(args->gpu_id);
	1349	+
1285	1350	return 0;
1286	1351
1287	1352	err_free:
1288		- dev->kfd2kgd->free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem);
	1353	+ amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem, NULL);
1289	1354	err_unlock:
1290	1355	mutex_unlock(&p->mutex);
1291	1356	return err;
..	..	@@ -1299,6 +1364,7 @@
1299	1364	void *mem;
1300	1365	struct kfd_dev *dev;
1301	1366	int ret;
	1367	+ uint64_t size = 0;
1302	1368
1303	1369	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1304	1370	if (!dev)
..	..	@@ -1320,7 +1386,8 @@
1320	1386	goto err_unlock;
1321	1387	}
1322	1388
1323		- ret = dev->kfd2kgd->free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem);
	1389	+ ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd,
	1390	+ (struct kgd_mem *)mem, &size);
1324	1391
1325	1392	/* If freeing the buffer failed, leave the handle in place for
1326	1393	* clean-up during process tear-down.
..	..	@@ -1328,6 +1395,8 @@
1328	1395	if (!ret)
1329	1396	kfd_process_device_remove_obj_handle(
1330	1397	pdd, GET_IDR_HANDLE(args->handle));
	1398	+
	1399	+ WRITE_ONCE(pdd->vram_usage, pdd->vram_usage - size);
1331	1400
1332	1401	err_unlock:
1333	1402	mutex_unlock(&p->mutex);
..	..	@@ -1400,7 +1469,7 @@
1400	1469	err = PTR_ERR(peer_pdd);
1401	1470	goto get_mem_obj_from_handle_failed;
1402	1471	}
1403		- err = peer->kfd2kgd->map_memory_to_gpu(
	1472	+ err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1404	1473	peer->kgd, (struct kgd_mem *)mem, peer_pdd->vm);
1405	1474	if (err) {
1406	1475	pr_err("Failed to map to gpu %d/%d\n",
..	..	@@ -1412,7 +1481,7 @@
1412	1481
1413	1482	mutex_unlock(&p->mutex);
1414	1483
1415		- err = dev->kfd2kgd->sync_memory(dev->kgd, (struct kgd_mem *) mem, true);
	1484	+ err = amdgpu_amdkfd_gpuvm_sync_memory(dev->kgd, (struct kgd_mem *) mem, true);
1416	1485	if (err) {
1417	1486	pr_debug("Sync memory failed, wait interrupted by user signal\n");
1418	1487	goto sync_memory_failed;
..	..	@@ -1507,7 +1576,7 @@
1507	1576	err = -ENODEV;
1508	1577	goto get_mem_obj_from_handle_failed;
1509	1578	}
1510		- err = dev->kfd2kgd->unmap_memory_to_gpu(
	1579	+ err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1511	1580	peer->kgd, (struct kgd_mem *)mem, peer_pdd->vm);
1512	1581	if (err) {
1513	1582	pr_err("Failed to unmap from gpu %d/%d\n",
..	..	@@ -1529,6 +1598,170 @@
1529	1598	copy_from_user_failed:
1530	1599	kfree(devices_arr);
1531	1600	return err;
	1601	+}
	1602	+
	1603	+static int kfd_ioctl_alloc_queue_gws(struct file *filep,
	1604	+ struct kfd_process p, void data)
	1605	+{
	1606	+ int retval;
	1607	+ struct kfd_ioctl_alloc_queue_gws_args *args = data;
	1608	+ struct queue *q;
	1609	+ struct kfd_dev *dev;
	1610	+
	1611	+ mutex_lock(&p->mutex);
	1612	+ q = pqm_get_user_queue(&p->pqm, args->queue_id);
	1613	+
	1614	+ if (q) {
	1615	+ dev = q->device;
	1616	+ } else {
	1617	+ retval = -EINVAL;
	1618	+ goto out_unlock;
	1619	+ }
	1620	+
	1621	+ if (!dev->gws) {
	1622	+ retval = -ENODEV;
	1623	+ goto out_unlock;
	1624	+ }
	1625	+
	1626	+ if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
	1627	+ retval = -ENODEV;
	1628	+ goto out_unlock;
	1629	+ }
	1630	+
	1631	+ retval = pqm_set_gws(&p->pqm, args->queue_id, args->num_gws ? dev->gws : NULL);
	1632	+ mutex_unlock(&p->mutex);
	1633	+
	1634	+ args->first_gws = 0;
	1635	+ return retval;
	1636	+
	1637	+out_unlock:
	1638	+ mutex_unlock(&p->mutex);
	1639	+ return retval;
	1640	+}
	1641	+
	1642	+static int kfd_ioctl_get_dmabuf_info(struct file *filep,
	1643	+ struct kfd_process p, void data)
	1644	+{
	1645	+ struct kfd_ioctl_get_dmabuf_info_args *args = data;
	1646	+ struct kfd_dev *dev = NULL;
	1647	+ struct kgd_dev *dma_buf_kgd;
	1648	+ void *metadata_buffer = NULL;
	1649	+ uint32_t flags;
	1650	+ unsigned int i;
	1651	+ int r;
	1652	+
	1653	+ /* Find a KFD GPU device that supports the get_dmabuf_info query */
	1654	+ for (i = 0; kfd_topology_enum_kfd_devices(i, &dev) == 0; i++)
	1655	+ if (dev)
	1656	+ break;
	1657	+ if (!dev)
	1658	+ return -EINVAL;
	1659	+
	1660	+ if (args->metadata_ptr) {
	1661	+ metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL);
	1662	+ if (!metadata_buffer)
	1663	+ return -ENOMEM;
	1664	+ }
	1665	+
	1666	+ /* Get dmabuf info from KGD */
	1667	+ r = amdgpu_amdkfd_get_dmabuf_info(dev->kgd, args->dmabuf_fd,
	1668	+ &dma_buf_kgd, &args->size,
	1669	+ metadata_buffer, args->metadata_size,
	1670	+ &args->metadata_size, &flags);
	1671	+ if (r)
	1672	+ goto exit;
	1673	+
	1674	+ /* Reverse-lookup gpu_id from kgd pointer */
	1675	+ dev = kfd_device_by_kgd(dma_buf_kgd);
	1676	+ if (!dev) {
	1677	+ r = -EINVAL;
	1678	+ goto exit;
	1679	+ }
	1680	+ args->gpu_id = dev->id;
	1681	+ args->flags = flags;
	1682	+
	1683	+ /* Copy metadata buffer to user mode */
	1684	+ if (metadata_buffer) {
	1685	+ r = copy_to_user((void __user *)args->metadata_ptr,
	1686	+ metadata_buffer, args->metadata_size);
	1687	+ if (r != 0)
	1688	+ r = -EFAULT;
	1689	+ }
	1690	+
	1691	+exit:
	1692	+ kfree(metadata_buffer);
	1693	+
	1694	+ return r;
	1695	+}
	1696	+
	1697	+static int kfd_ioctl_import_dmabuf(struct file *filep,
	1698	+ struct kfd_process p, void data)
	1699	+{
	1700	+ struct kfd_ioctl_import_dmabuf_args *args = data;
	1701	+ struct kfd_process_device *pdd;
	1702	+ struct dma_buf *dmabuf;
	1703	+ struct kfd_dev *dev;
	1704	+ int idr_handle;
	1705	+ uint64_t size;
	1706	+ void *mem;
	1707	+ int r;
	1708	+
	1709	+ dev = kfd_device_by_id(args->gpu_id);
	1710	+ if (!dev)
	1711	+ return -EINVAL;
	1712	+
	1713	+ dmabuf = dma_buf_get(args->dmabuf_fd);
	1714	+ if (IS_ERR(dmabuf))
	1715	+ return PTR_ERR(dmabuf);
	1716	+
	1717	+ mutex_lock(&p->mutex);
	1718	+
	1719	+ pdd = kfd_bind_process_to_device(dev, p);
	1720	+ if (IS_ERR(pdd)) {
	1721	+ r = PTR_ERR(pdd);
	1722	+ goto err_unlock;
	1723	+ }
	1724	+
	1725	+ r = amdgpu_amdkfd_gpuvm_import_dmabuf(dev->kgd, dmabuf,
	1726	+ args->va_addr, pdd->vm,
	1727	+ (struct kgd_mem **)&mem, &size,
	1728	+ NULL);
	1729	+ if (r)
	1730	+ goto err_unlock;
	1731	+
	1732	+ idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
	1733	+ if (idr_handle < 0) {
	1734	+ r = -EFAULT;
	1735	+ goto err_free;
	1736	+ }
	1737	+
	1738	+ mutex_unlock(&p->mutex);
	1739	+ dma_buf_put(dmabuf);
	1740	+
	1741	+ args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
	1742	+
	1743	+ return 0;
	1744	+
	1745	+err_free:
	1746	+ amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem, NULL);
	1747	+err_unlock:
	1748	+ mutex_unlock(&p->mutex);
	1749	+ dma_buf_put(dmabuf);
	1750	+ return r;
	1751	+}
	1752	+
	1753	+/* Handle requests for watching SMI events */
	1754	+static int kfd_ioctl_smi_events(struct file *filep,
	1755	+ struct kfd_process p, void data)
	1756	+{
	1757	+ struct kfd_ioctl_smi_events_args *args = data;
	1758	+ struct kfd_dev *dev;
	1759	+
	1760	+ dev = kfd_device_by_id(args->gpuid);
	1761	+ if (!dev)
	1762	+ return -EINVAL;
	1763	+
	1764	+ return kfd_smi_event_open(dev, &args->anon_fd);
1532	1765	}
1533	1766
1534	1767	#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
..	..	@@ -1615,6 +1848,20 @@
1615	1848	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
1616	1849	kfd_ioctl_set_cu_mask, 0),
1617	1850
	1851	+ AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE,
	1852	+ kfd_ioctl_get_queue_wave_state, 0),
	1853	+
	1854	+ AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
	1855	+ kfd_ioctl_get_dmabuf_info, 0),
	1856	+
	1857	+ AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
	1858	+ kfd_ioctl_import_dmabuf, 0),
	1859	+
	1860	+ AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS,
	1861	+ kfd_ioctl_alloc_queue_gws, 0),
	1862	+
	1863	+ AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
	1864	+ kfd_ioctl_smi_events, 0),
1618	1865	};
1619	1866
1620	1867	#define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
..	..	@@ -1647,11 +1894,16 @@
1647	1894	} else
1648	1895	goto err_i1;
1649	1896
1650		- dev_dbg(kfd_device, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, nr, arg);
	1897	+ dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg);
1651	1898
1652		- process = kfd_get_process(current);
1653		- if (IS_ERR(process)) {
1654		- dev_dbg(kfd_device, "no process\n");
	1899	+ /* Get the process struct from the filep. Only the process
	1900	+ * that opened /dev/kfd can use the file descriptor. Child
	1901	+ * processes need to create their own KFD device context.
	1902	+ */
	1903	+ process = filep->private_data;
	1904	+ if (process->lead_thread != current->group_leader) {
	1905	+ dev_dbg(kfd_device, "Using KFD FD in wrong process\n");
	1906	+ retcode = -EBADF;
1655	1907	goto err_i1;
1656	1908	}
1657	1909
..	..	@@ -1702,29 +1954,62 @@
1702	1954	kfree(kdata);
1703	1955
1704	1956	if (retcode)
1705		- dev_dbg(kfd_device, "ret = %d\n", retcode);
	1957	+ dev_dbg(kfd_device, "ioctl cmd (#0x%x), arg 0x%lx, ret = %d\n",
	1958	+ nr, arg, retcode);
1706	1959
1707	1960	return retcode;
1708	1961	}
	1962	+
	1963	+static int kfd_mmio_mmap(struct kfd_dev dev, struct kfd_process process,
	1964	+ struct vm_area_struct *vma)
	1965	+{
	1966	+ phys_addr_t address;
	1967	+ int ret;
	1968	+
	1969	+ if (vma->vm_end - vma->vm_start != PAGE_SIZE)
	1970	+ return -EINVAL;
	1971	+
	1972	+ address = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
	1973	+
	1974	+ vma->vm_flags \|= VM_IO \| VM_DONTCOPY \| VM_DONTEXPAND \| VM_NORESERVE \|
	1975	+ VM_DONTDUMP \| VM_PFNMAP;
	1976	+
	1977	+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
	1978	+
	1979	+ pr_debug("pasid 0x%x mapping mmio page\n"
	1980	+ " target user address == 0x%08llX\n"
	1981	+ " physical address == 0x%08llX\n"
	1982	+ " vm_flags == 0x%04lX\n"
	1983	+ " size == 0x%04lX\n",
	1984	+ process->pasid, (unsigned long long) vma->vm_start,
	1985	+ address, vma->vm_flags, PAGE_SIZE);
	1986	+
	1987	+ ret = io_remap_pfn_range(vma,
	1988	+ vma->vm_start,
	1989	+ address >> PAGE_SHIFT,
	1990	+ PAGE_SIZE,
	1991	+ vma->vm_page_prot);
	1992	+ return ret;
	1993	+}
	1994	+
1709	1995
1710	1996	static int kfd_mmap(struct file filp, struct vm_area_struct vma)
1711	1997	{
1712	1998	struct kfd_process *process;
1713	1999	struct kfd_dev *dev = NULL;
1714		- unsigned long vm_pgoff;
	2000	+ unsigned long mmap_offset;
1715	2001	unsigned int gpu_id;
1716	2002
1717	2003	process = kfd_get_process(current);
1718	2004	if (IS_ERR(process))
1719	2005	return PTR_ERR(process);
1720	2006
1721		- vm_pgoff = vma->vm_pgoff;
1722		- vma->vm_pgoff = KFD_MMAP_OFFSET_VALUE_GET(vm_pgoff);
1723		- gpu_id = KFD_MMAP_GPU_ID_GET(vm_pgoff);
	2007	+ mmap_offset = vma->vm_pgoff << PAGE_SHIFT;
	2008	+ gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset);
1724	2009	if (gpu_id)
1725	2010	dev = kfd_device_by_id(gpu_id);
1726	2011
1727		- switch (vm_pgoff & KFD_MMAP_TYPE_MASK) {
	2012	+ switch (mmap_offset & KFD_MMAP_TYPE_MASK) {
1728	2013	case KFD_MMAP_TYPE_DOORBELL:
1729	2014	if (!dev)
1730	2015	return -ENODEV;
..	..	@@ -1737,6 +2022,10 @@
1737	2022	if (!dev)
1738	2023	return -ENODEV;
1739	2024	return kfd_reserved_mem_mmap(dev, process, vma);
	2025	+ case KFD_MMAP_TYPE_MMIO:
	2026	+ if (!dev)
	2027	+ return -ENODEV;
	2028	+ return kfd_mmio_mmap(dev, process, vma);
1740	2029	}
1741	2030
1742	2031	return -EFAULT;