forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-06 08f87f769b595151be1afeff53e144f543faa614 
 kernel/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c
....@@ -1,7 +1,7 @@
11 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
22 /*
33  *
4
- * (C) COPYRIGHT 2011-2021 ARM Limited. All rights reserved.
4
+ * (C) COPYRIGHT 2011-2022 ARM Limited. All rights reserved.
55  *
66  * This program is free software and is provided to you under the terms of the
77  * GNU General Public License version 2 as published by the Free Software
....@@ -23,7 +23,7 @@
2323 
2424 #include <linux/dma-buf.h>
2525 #include <asm/cacheflush.h>
26
-#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
26
+#if IS_ENABLED(CONFIG_SYNC_FILE)
2727 #include <mali_kbase_sync.h>
2828 #endif
2929 #include <linux/dma-mapping.h>
....@@ -73,7 +73,7 @@
7373 	/* Record the start time of this atom so we could cancel it at
7474 	 * the right time.
7575 	 */
76
-	katom->start_timestamp = ktime_get();
76
+	katom->start_timestamp = ktime_get_raw();
7777 
7878 	/* Add the atom to the waiting list before the timer is
7979 	 * (re)started to make sure that it gets processed.
....@@ -95,7 +95,8 @@
9595 	unsigned char *mapped_evt;
9696 	struct kbase_vmap_struct map;
9797 
98
-	mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map);
98
+	mapped_evt = kbase_vmap_prot(kctx, evt, sizeof(*mapped_evt),
99
+				     KBASE_REG_CPU_RD, &map);
99100 	if (!mapped_evt)
100101 		return -EFAULT;
101102 
....@@ -116,7 +117,8 @@
116117 	    (new_status != BASE_JD_SOFT_EVENT_RESET))
117118 		return -EINVAL;
118119 
119
-	mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map);
120
+	mapped_evt = kbase_vmap_prot(kctx, evt, sizeof(*mapped_evt),
121
+				     KBASE_REG_CPU_WR, &map);
120122 	if (!mapped_evt)
121123 		return -EFAULT;
122124 
....@@ -202,7 +204,7 @@
202204 	return 0;
203205 }
204206 
205
-#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
207
+#if IS_ENABLED(CONFIG_SYNC_FILE)
206208 /* Called by the explicit fence mechanism when a fence wait has completed */
207209 void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom)
208210 {
....@@ -211,7 +213,7 @@
211213 	mutex_lock(&kctx->jctx.lock);
212214 	kbasep_remove_waiting_soft_job(katom);
213215 	kbase_finish_soft_job(katom);
214
-	if (jd_done_nolock(katom, NULL))
216
+	if (kbase_jd_done_nolock(katom, true))
215217 		kbase_js_sched_all(kctx->kbdev);
216218 	mutex_unlock(&kctx->jctx.lock);
217219 }
....@@ -225,7 +227,7 @@
225227 	int resched;
226228 
227229 	mutex_lock(&kctx->jctx.lock);
228
-	resched = jd_done_nolock(katom, NULL);
230
+	resched = kbase_jd_done_nolock(katom, true);
229231 	mutex_unlock(&kctx->jctx.lock);
230232 
231233 	if (resched)
....@@ -303,7 +305,7 @@
303305 						 info.fence,
304306 						 info.name,
305307 						 kbase_sync_status_string(info.status));
306
-				 }
308
+				}
307309 			}
308310 
309311 			kbase_fence_debug_check_atom(dep);
....@@ -386,7 +388,7 @@
386388 			soft_job_timeout);
387389 	u32 timeout_ms = (u32)atomic_read(
388390 			&kctx->kbdev->js_data.soft_job_timeout_ms);
389
-	ktime_t cur_time = ktime_get();
391
+	ktime_t cur_time = ktime_get_raw();
390392 	bool restarting = false;
391393 	unsigned long lflags;
392394 	struct list_head *entry, *tmp;
....@@ -496,10 +498,11 @@
496498 static void kbasep_soft_event_cancel_job(struct kbase_jd_atom *katom)
497499 {
498500 	katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
499
-	if (jd_done_nolock(katom, NULL))
501
+	if (kbase_jd_done_nolock(katom, true))
500502 		kbase_js_sched_all(katom->kctx->kbdev);
501503 }
502504 
505
+#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST
503506 static void kbase_debug_copy_finish(struct kbase_jd_atom *katom)
504507 {
505508 	struct kbase_debug_copy_buffer *buffers = katom->softjob_data;
....@@ -671,8 +674,8 @@
671674 		case KBASE_MEM_TYPE_IMPORTED_USER_BUF:
672675 		{
673676 			struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc;
674
-			unsigned long nr_pages =
675
-				alloc->imported.user_buf.nr_pages;
677
+			const unsigned long nr_pages = alloc->imported.user_buf.nr_pages;
678
+			const unsigned long start = alloc->imported.user_buf.address;
676679 
677680 			if (alloc->imported.user_buf.mm != current->mm) {
678681 				ret = -EINVAL;
....@@ -684,11 +687,9 @@
684687 				ret = -ENOMEM;
685688 				goto out_unlock;
686689 			}
687
-
688
-			ret = get_user_pages_fast(
689
-					alloc->imported.user_buf.address,
690
-					nr_pages, 0,
691
-					buffers[i].extres_pages);
690
+			kbase_gpu_vm_unlock(katom->kctx);
691
+			ret = get_user_pages_fast(start, nr_pages, 0, buffers[i].extres_pages);
692
+			kbase_gpu_vm_lock(katom->kctx);
692693 			if (ret != nr_pages) {
693694 				/* Adjust number of pages, so that we only
694695 				 * attempt to release pages in the array that we
....@@ -726,7 +727,6 @@
726727 
727728 	return ret;
728729 }
729
-#endif /* !MALI_USE_CSF */
730730 
731731 #if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE
732732 static void *dma_buf_kmap_page(struct kbase_mem_phy_alloc *gpu_alloc,
....@@ -758,8 +758,18 @@
758758 }
759759 #endif
760760 
761
-int kbase_mem_copy_from_extres(struct kbase_context *kctx,
762
-		struct kbase_debug_copy_buffer *buf_data)
761
+/**
762
+ * kbase_mem_copy_from_extres() - Copy from external resources.
763
+ *
764
+ * @kctx:	kbase context within which the copying is to take place.
765
+ * @buf_data:	Pointer to the information about external resources:
766
+ *		pages pertaining to the external resource, number of
767
+ *		pages to copy.
768
+ *
769
+ * Return:      0 on success, error code otherwise.
770
+ */
771
+static int kbase_mem_copy_from_extres(struct kbase_context *kctx,
772
+				      struct kbase_debug_copy_buffer *buf_data)
763773 {
764774 	unsigned int i;
765775 	unsigned int target_page_nr = 0;
....@@ -808,11 +818,7 @@
808818 
809819 		dma_to_copy = min(dma_buf->size,
810820 			(size_t)(buf_data->nr_extres_pages * PAGE_SIZE));
811
-		ret = dma_buf_begin_cpu_access(dma_buf,
812
-#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && !defined(CONFIG_CHROMEOS)
813
-					       0, dma_to_copy,
814
-#endif
815
-					       DMA_FROM_DEVICE);
821
+		ret = dma_buf_begin_cpu_access(dma_buf, DMA_FROM_DEVICE);
816822 		if (ret)
817823 			goto out_unlock;
818824 
....@@ -839,11 +845,7 @@
839845 					break;
840846 			}
841847 		}
842
-		dma_buf_end_cpu_access(dma_buf,
843
-#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && !defined(CONFIG_CHROMEOS)
844
-				       0, dma_to_copy,
845
-#endif
846
-				       DMA_FROM_DEVICE);
848
+		dma_buf_end_cpu_access(dma_buf, DMA_FROM_DEVICE);
847849 		break;
848850 	}
849851 	default:
....@@ -854,7 +856,6 @@
854856 	return ret;
855857 }
856858 
857
-#if !MALI_USE_CSF
858859 static int kbase_debug_copy(struct kbase_jd_atom *katom)
859860 {
860861 	struct kbase_debug_copy_buffer *buffers = katom->softjob_data;
....@@ -872,6 +873,7 @@
872873 
873874 	return 0;
874875 }
876
+#endif /* IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST */
875877 #endif /* !MALI_USE_CSF */
876878 
877879 #define KBASEP_JIT_ALLOC_GPU_ADDR_ALIGNMENT ((u32)0x7)
....@@ -931,26 +933,6 @@
931933 
932934 #if !MALI_USE_CSF
933935 
934
-/*
935
- * Sizes of user data to copy for each just-in-time memory interface version
936
- *
937
- * In interface version 2 onwards this is the same as the struct size, allowing
938
- * copying of arrays of structures from userspace.
939
- *
940
- * In interface version 1 the structure size was variable, and hence arrays of
941
- * structures cannot be supported easily, and were not a feature present in
942
- * version 1 anyway.
943
- */
944
-static const size_t jit_info_copy_size_for_jit_version[] = {
945
-	/* in jit_version 1, the structure did not have any end padding, hence
946
-	 * it could be a different size on 32 and 64-bit clients. We therefore
947
-	 * do not copy past the last member
948
-	 */
949
-	[1] = offsetofend(struct base_jit_alloc_info_10_2, id),
950
-	[2] = sizeof(struct base_jit_alloc_info_11_5),
951
-	[3] = sizeof(struct base_jit_alloc_info)
952
-};
953
-
954936 static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom)
955937 {
956938 	__user u8 *data = (__user u8 *)(uintptr_t) katom->jc;
....@@ -960,18 +942,18 @@
960942 	u32 count;
961943 	int ret;
962944 	u32 i;
963
-	size_t jit_info_user_copy_size;
964945 
965
-	WARN_ON(kctx->jit_version >=
966
-		ARRAY_SIZE(jit_info_copy_size_for_jit_version));
967
-	jit_info_user_copy_size =
968
-			jit_info_copy_size_for_jit_version[kctx->jit_version];
969
-	WARN_ON(jit_info_user_copy_size > sizeof(*info));
946
+	if (!kbase_mem_allow_alloc(kctx)) {
947
+		dev_dbg(kbdev->dev, "Invalid attempt to allocate JIT memory by %s/%d for ctx %d_%d",
948
+			current->comm, current->pid, kctx->tgid, kctx->id);
949
+		ret = -EINVAL;
950
+		goto fail;
951
+	}
970952 
971953 	/* For backwards compatibility, and to prevent reading more than 1 jit
972954 	 * info struct on jit version 1
973955 	 */
974
-	if (katom->nr_extres == 0 || kctx->jit_version == 1)
956
+	if (katom->nr_extres == 0)
975957 		katom->nr_extres = 1;
976958 	count = katom->nr_extres;
977959 
....@@ -991,17 +973,11 @@
991973 
992974 	katom->softjob_data = info;
993975 
994
-	for (i = 0; i < count; i++, info++, data += jit_info_user_copy_size) {
995
-		if (copy_from_user(info, data, jit_info_user_copy_size) != 0) {
976
+	for (i = 0; i < count; i++, info++, data += sizeof(*info)) {
977
+		if (copy_from_user(info, data, sizeof(*info)) != 0) {
996978 			ret = -EINVAL;
997979 			goto free_info;
998980 		}
999
-		/* Clear any remaining bytes when user struct is smaller than
1000
-		 * kernel struct. For jit version 1, this also clears the
1001
-		 * padding bytes
1002
-		 */
1003
-		memset(((u8 *)info) + jit_info_user_copy_size, 0,
1004
-				sizeof(*info) - jit_info_user_copy_size);
1005981 
1006982 		ret = kbasep_jit_alloc_validate(kctx, info);
1007983 		if (ret)
....@@ -1203,8 +1179,8 @@
12031179 		 * Write the address of the JIT allocation to the user provided
12041180 		 * GPU allocation.
12051181 		 */
1206
-		ptr = kbase_vmap(kctx, info->gpu_alloc_addr, sizeof(*ptr),
1207
-				&mapping);
1182
+		ptr = kbase_vmap_prot(kctx, info->gpu_alloc_addr, sizeof(*ptr),
1183
+				KBASE_REG_CPU_WR, &mapping);
12081184 		if (!ptr) {
12091185 			/*
12101186 			 * Leave the allocations "live" as the JIT free atom
....@@ -1353,7 +1329,7 @@
13531329 
13541330 	mutex_lock(&kctx->jctx.lock);
13551331 	kbase_finish_soft_job(katom);
1356
-	resched = jd_done_nolock(katom, NULL);
1332
+	resched = kbase_jd_done_nolock(katom, true);
13571333 	mutex_unlock(&kctx->jctx.lock);
13581334 
13591335 	if (resched)
....@@ -1393,9 +1369,8 @@
13931369 	lockdep_assert_held(&kctx->jctx.lock);
13941370 
13951371 	ids = kbase_jit_free_get_ids(katom);
1396
-	if (WARN_ON(ids == NULL)) {
1372
+	if (WARN_ON(ids == NULL))
13971373 		return;
1398
-	}
13991374 
14001375 	/* Remove this atom from the jit_atoms_head list */
14011376 	list_del(&katom->jit_node);
....@@ -1483,10 +1458,11 @@
14831458 			if (!kbase_sticky_resource_acquire(katom->kctx,
14841459 					gpu_addr))
14851460 				goto failed_loop;
1486
-		} else
1461
+		} else {
14871462 			if (!kbase_sticky_resource_release_force(katom->kctx, NULL,
14881463 					gpu_addr))
14891464 				failed = true;
1465
+		}
14901466 	}
14911467 
14921468 	/*
....@@ -1546,7 +1522,7 @@
15461522 		ret = kbase_dump_cpu_gpu_time(katom);
15471523 		break;
15481524 
1549
-#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1525
+#if IS_ENABLED(CONFIG_SYNC_FILE)
15501526 	case BASE_JD_REQ_SOFT_FENCE_TRIGGER:
15511527 		katom->event_code = kbase_sync_fence_out_trigger(katom,
15521528 				katom->event_code == BASE_JD_EVENT_DONE ?
....@@ -1575,6 +1551,7 @@
15751551 	case BASE_JD_REQ_SOFT_EVENT_RESET:
15761552 		kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_RESET);
15771553 		break;
1554
+#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST
15781555 	case BASE_JD_REQ_SOFT_DEBUG_COPY:
15791556 	{
15801557 		int res = kbase_debug_copy(katom);
....@@ -1583,6 +1560,7 @@
15831560 			katom->event_code = BASE_JD_EVENT_JOB_INVALID;
15841561 		break;
15851562 	}
1563
+#endif /* IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST */
15861564 	case BASE_JD_REQ_SOFT_JIT_ALLOC:
15871565 		ret = kbase_jit_allocate_process(katom);
15881566 		break;
....@@ -1606,7 +1584,7 @@
16061584 void kbase_cancel_soft_job(struct kbase_jd_atom *katom)
16071585 {
16081586 	switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) {
1609
-#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1587
+#if IS_ENABLED(CONFIG_SYNC_FILE)
16101588 	case BASE_JD_REQ_SOFT_FENCE_WAIT:
16111589 		kbase_sync_fence_in_cancel_wait(katom);
16121590 		break;
....@@ -1629,7 +1607,7 @@
16291607 				return -EINVAL;
16301608 		}
16311609 		break;
1632
-#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1610
+#if IS_ENABLED(CONFIG_SYNC_FILE)
16331611 	case BASE_JD_REQ_SOFT_FENCE_TRIGGER:
16341612 		{
16351613 			struct base_fence fence;
....@@ -1649,7 +1627,12 @@
16491627 			if (copy_to_user((__user void *)(uintptr_t)katom->jc,
16501628 					 &fence, sizeof(fence)) != 0) {
16511629 				kbase_sync_fence_out_remove(katom);
1652
-				kbase_sync_fence_close_fd(fd);
1630
+				/* fd should have been closed here, but there's
1631
+				 * no good way of doing that. Since
1632
+				 * copy_to_user() very rarely fails, and the fd
1633
+				 * will get closed on process termination this
1634
+				 * won't be a problem.
1635
+				 */
16531636 				fence.basep.fd = -EINVAL;
16541637 				return -EINVAL;
16551638 			}
....@@ -1670,20 +1653,9 @@
16701653 							  fence.basep.fd);
16711654 			if (ret < 0)
16721655 				return ret;
1673
-
1674
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
1675
-			/*
1676
-			 * Set KCTX_NO_IMPLICIT_FENCE in the context the first
1677
-			 * time a soft fence wait job is observed. This will
1678
-			 * prevent the implicit dma-buf fence to conflict with
1679
-			 * the Android native sync fences.
1680
-			 */
1681
-			if (!kbase_ctx_flag(katom->kctx, KCTX_NO_IMPLICIT_SYNC))
1682
-				kbase_ctx_flag_set(katom->kctx, KCTX_NO_IMPLICIT_SYNC);
1683
-#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */
16841656 		}
16851657 		break;
1686
-#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */
1658
+#endif /* CONFIG_SYNC_FILE */
16871659 	case BASE_JD_REQ_SOFT_JIT_ALLOC:
16881660 		return kbase_jit_allocate_prepare(katom);
16891661 	case BASE_JD_REQ_SOFT_JIT_FREE:
....@@ -1694,8 +1666,10 @@
16941666 		if (katom->jc == 0)
16951667 			return -EINVAL;
16961668 		break;
1669
+#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST
16971670 	case BASE_JD_REQ_SOFT_DEBUG_COPY:
16981671 		return kbase_debug_copy_prepare(katom);
1672
+#endif /* IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST */
16991673 	case BASE_JD_REQ_SOFT_EXT_RES_MAP:
17001674 		return kbase_ext_res_prepare(katom);
17011675 	case BASE_JD_REQ_SOFT_EXT_RES_UNMAP:
....@@ -1716,7 +1690,7 @@
17161690 	case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME:
17171691 		/* Nothing to do */
17181692 		break;
1719
-#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
1693
+#if IS_ENABLED(CONFIG_SYNC_FILE)
17201694 	case BASE_JD_REQ_SOFT_FENCE_TRIGGER:
17211695 		/* If fence has not yet been signaled, do it now */
17221696 		kbase_sync_fence_out_trigger(katom, katom->event_code ==
....@@ -1726,10 +1700,12 @@
17261700 		/* Release katom's reference to fence object */
17271701 		kbase_sync_fence_in_remove(katom);
17281702 		break;
1729
-#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */
1703
+#endif /* CONFIG_SYNC_FILE */
1704
+#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST
17301705 	case BASE_JD_REQ_SOFT_DEBUG_COPY:
17311706 		kbase_debug_copy_finish(katom);
17321707 		break;
1708
+#endif /* IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST */
17331709 	case BASE_JD_REQ_SOFT_JIT_ALLOC:
17341710 		kbase_jit_allocate_finish(katom);
17351711 		break;
....@@ -1780,7 +1756,7 @@
17801756 
17811757 		if (kbase_process_soft_job(katom_iter) == 0) {
17821758 			kbase_finish_soft_job(katom_iter);
1783
-			resched |= jd_done_nolock(katom_iter, NULL);
1759
+			resched |= kbase_jd_done_nolock(katom_iter, true);
17841760 #ifdef CONFIG_MALI_ARBITER_SUPPORT
17851761 			atomic_dec(&kbdev->pm.gpu_users_waiting);
17861762 #endif /* CONFIG_MALI_ARBITER_SUPPORT */