forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-13 9d77db3c730780c8ef5ccd4b66403ff5675cfe4e 
 kernel/drivers/gpu/drm/msm/msm_gem_submit.c
....@@ -1,25 +1,21 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Copyright (C) 2013 Red Hat
34  * Author: Rob Clark <robdclark@gmail.com>
4
- *
5
- * This program is free software; you can redistribute it and/or modify it
6
- * under the terms of the GNU General Public License version 2 as published by
7
- * the Free Software Foundation.
8
- *
9
- * This program is distributed in the hope that it will be useful, but WITHOUT
10
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12
- * more details.
13
- *
14
- * You should have received a copy of the GNU General Public License along with
15
- * this program.  If not, see <http://www.gnu.org/licenses/>.
165  */
176 
7
+#include <linux/file.h>
188 #include <linux/sync_file.h>
9
+#include <linux/uaccess.h>
10
+
11
+#include <drm/drm_drv.h>
12
+#include <drm/drm_file.h>
13
+#include <drm/drm_syncobj.h>
1914 
2015 #include "msm_drv.h"
2116 #include "msm_gpu.h"
2217 #include "msm_gem.h"
18
+#include "msm_gpu_trace.h"
2319 
2420 /*
2521  * Cmdstream submission:
....@@ -31,12 +27,13 @@
3127 #define BO_PINNED   0x2000
3228 
3329 static struct msm_gem_submit *submit_create(struct drm_device *dev,
34
-		struct msm_gpu *gpu, struct msm_gpu_submitqueue *queue,
35
-		uint32_t nr_bos, uint32_t nr_cmds)
30
+		struct msm_gpu *gpu,
31
+		struct msm_gpu_submitqueue *queue, uint32_t nr_bos,
32
+		uint32_t nr_cmds)
3633 {
3734 	struct msm_gem_submit *submit;
38
-	uint64_t sz = sizeof(*submit) + ((u64)nr_bos * sizeof(submit->bos[0])) +
39
-		((u64)nr_cmds * sizeof(submit->cmd[0]));
35
+	uint64_t sz = struct_size(submit, bos, nr_bos) +
36
+				  ((u64)nr_cmds * sizeof(submit->cmd[0]));
4037 
4138 	if (sz > SIZE_MAX)
4239 		return NULL;
....@@ -46,9 +43,9 @@
4643 		return NULL;
4744 
4845 	submit->dev = dev;
46
+	submit->aspace = queue->ctx->aspace;
4947 	submit->gpu = gpu;
5048 	submit->fence = NULL;
51
-	submit->pid = get_pid(task_pid(current));
5249 	submit->cmd = (void *)&submit->bos[nr_bos];
5350 	submit->queue = queue;
5451 	submit->ring = gpu->rb[queue->prio];
....@@ -59,7 +56,6 @@
5956 
6057 	INIT_LIST_HEAD(&submit->node);
6158 	INIT_LIST_HEAD(&submit->bo_list);
62
-	ww_acquire_init(&submit->ticket, &reservation_ww_class);
6359 
6460 	return submit;
6561 }
....@@ -74,27 +70,14 @@
7470 	kfree(submit);
7571 }
7672 
77
-static inline unsigned long __must_check
78
-copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
79
-{
80
-	if (access_ok(VERIFY_READ, from, n))
81
-		return __copy_from_user_inatomic(to, from, n);
82
-	return -EFAULT;
83
-}
84
-
8573 static int submit_lookup_objects(struct msm_gem_submit *submit,
8674 		struct drm_msm_gem_submit *args, struct drm_file *file)
8775 {
8876 	unsigned i;
8977 	int ret = 0;
9078 
91
-	spin_lock(&file->table_lock);
92
-	pagefault_disable();
93
-
9479 	for (i = 0; i < args->nr_bos; i++) {
9580 		struct drm_msm_gem_submit_bo submit_bo;
96
-		struct drm_gem_object *obj;
97
-		struct msm_gem_object *msm_obj;
9881 		void __user *userptr =
9982 			u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
10083 
....@@ -103,34 +86,41 @@
10386 		 */
10487 		submit->bos[i].flags = 0;
10588 
106
-		if (copy_from_user_inatomic(&submit_bo, userptr, sizeof(submit_bo))) {
107
-			pagefault_enable();
108
-			spin_unlock(&file->table_lock);
109
-			if (copy_from_user(&submit_bo, userptr, sizeof(submit_bo))) {
110
-				ret = -EFAULT;
111
-				goto out;
112
-			}
113
-			spin_lock(&file->table_lock);
114
-			pagefault_disable();
89
+		if (copy_from_user(&submit_bo, userptr, sizeof(submit_bo))) {
90
+			ret = -EFAULT;
91
+			i = 0;
92
+			goto out;
11593 		}
94
+
95
+/* at least one of READ and/or WRITE flags should be set: */
96
+#define MANDATORY_FLAGS (MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE)
11697 
11798 		if ((submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) ||
118
-			!(submit_bo.flags & MSM_SUBMIT_BO_FLAGS)) {
99
+			!(submit_bo.flags & MANDATORY_FLAGS)) {
119100 			DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
120101 			ret = -EINVAL;
121
-			goto out_unlock;
102
+			i = 0;
103
+			goto out;
122104 		}
123105 
106
+		submit->bos[i].handle = submit_bo.handle;
124107 		submit->bos[i].flags = submit_bo.flags;
125108 		/* in validate_objects() we figure out if this is true: */
126109 		submit->bos[i].iova  = submit_bo.presumed;
110
+	}
111
+
112
+	spin_lock(&file->table_lock);
113
+
114
+	for (i = 0; i < args->nr_bos; i++) {
115
+		struct drm_gem_object *obj;
116
+		struct msm_gem_object *msm_obj;
127117 
128118 		/* normally use drm_gem_object_lookup(), but for bulk lookup
129119 		 * all under single table_lock just hit object_idr directly:
130120 		 */
131
-		obj = idr_find(&file->object_idr, submit_bo.handle);
121
+		obj = idr_find(&file->object_idr, submit->bos[i].handle);
132122 		if (!obj) {
133
-			DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i);
123
+			DRM_ERROR("invalid handle %u at index %u\n", submit->bos[i].handle, i);
134124 			ret = -EINVAL;
135125 			goto out_unlock;
136126 		}
....@@ -139,12 +129,12 @@
139129 
140130 		if (!list_empty(&msm_obj->submit_entry)) {
141131 			DRM_ERROR("handle %u at index %u already on submit list\n",
142
-					submit_bo.handle, i);
132
+					submit->bos[i].handle, i);
143133 			ret = -EINVAL;
144134 			goto out_unlock;
145135 		}
146136 
147
-		drm_gem_object_reference(obj);
137
+		drm_gem_object_get(obj);
148138 
149139 		submit->bos[i].obj = msm_obj;
150140 
....@@ -152,7 +142,6 @@
152142 	}
153143 
154144 out_unlock:
155
-	pagefault_enable();
156145 	spin_unlock(&file->table_lock);
157146 
158147 out:
....@@ -167,10 +156,10 @@
167156 	struct msm_gem_object *msm_obj = submit->bos[i].obj;
168157 
169158 	if (submit->bos[i].flags & BO_PINNED)
170
-		msm_gem_put_iova(&msm_obj->base, submit->gpu->aspace);
159
+		msm_gem_unpin_iova(&msm_obj->base, submit->aspace);
171160 
172161 	if (submit->bos[i].flags & BO_LOCKED)
173
-		ww_mutex_unlock(&msm_obj->resv->lock);
162
+		dma_resv_unlock(msm_obj->base.resv);
174163 
175164 	if (backoff && !(submit->bos[i].flags & BO_VALID))
176165 		submit->bos[i].iova = 0;
....@@ -193,8 +182,8 @@
193182 		contended = i;
194183 
195184 		if (!(submit->bos[i].flags & BO_LOCKED)) {
196
-			ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock,
197
-					&submit->ticket);
185
+			ret = dma_resv_lock_interruptible(msm_obj->base.resv,
186
+							  &submit->ticket);
198187 			if (ret)
199188 				goto fail;
200189 			submit->bos[i].flags |= BO_LOCKED;
....@@ -215,8 +204,8 @@
215204 	if (ret == -EDEADLK) {
216205 		struct msm_gem_object *msm_obj = submit->bos[contended].obj;
217206 		/* we lost out in a seqno race, lock and retry.. */
218
-		ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock,
219
-				&submit->ticket);
207
+		ret = dma_resv_lock_slow_interruptible(msm_obj->base.resv,
208
+						       &submit->ticket);
220209 		if (!ret) {
221210 			submit->bos[contended].flags |= BO_LOCKED;
222211 			slow_locked = contended;
....@@ -241,7 +230,8 @@
241230 			 * strange place to call it.  OTOH this is a
242231 			 * convenient can-fail point to hook it in.
243232 			 */
244
-			ret = reservation_object_reserve_shared(msm_obj->resv);
233
+			ret = dma_resv_reserve_shared(msm_obj->base.resv,
234
+								1);
245235 			if (ret)
246236 				return ret;
247237 		}
....@@ -269,8 +259,8 @@
269259 		uint64_t iova;
270260 
271261 		/* if locking succeeded, pin bo: */
272
-		ret = msm_gem_get_iova(&msm_obj->base,
273
-				submit->gpu->aspace, &iova);
262
+		ret = msm_gem_get_and_pin_iova(&msm_obj->base,
263
+				submit->aspace, &iova);
274264 
275265 		if (ret)
276266 			break;
....@@ -316,6 +306,9 @@
316306 	uint32_t i, last_offset = 0;
317307 	uint32_t *ptr;
318308 	int ret = 0;
309
+
310
+	if (!nr_relocs)
311
+		return 0;
319312 
320313 	if (offset % 4) {
321314 		DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
....@@ -396,15 +389,193 @@
396389 		struct msm_gem_object *msm_obj = submit->bos[i].obj;
397390 		submit_unlock_unpin_bo(submit, i, false);
398391 		list_del_init(&msm_obj->submit_entry);
399
-		drm_gem_object_unreference(&msm_obj->base);
392
+		drm_gem_object_put_locked(&msm_obj->base);
393
+	}
394
+}
395
+
396
+
397
+struct msm_submit_post_dep {
398
+	struct drm_syncobj *syncobj;
399
+	uint64_t point;
400
+	struct dma_fence_chain *chain;
401
+};
402
+
403
+static struct drm_syncobj **msm_wait_deps(struct drm_device *dev,
404
+                                          struct drm_file *file,
405
+                                          uint64_t in_syncobjs_addr,
406
+                                          uint32_t nr_in_syncobjs,
407
+                                          size_t syncobj_stride,
408
+                                          struct msm_ringbuffer *ring)
409
+{
410
+	struct drm_syncobj **syncobjs = NULL;
411
+	struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
412
+	int ret = 0;
413
+	uint32_t i, j;
414
+
415
+	syncobjs = kcalloc(nr_in_syncobjs, sizeof(*syncobjs),
416
+	                   GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
417
+	if (!syncobjs)
418
+		return ERR_PTR(-ENOMEM);
419
+
420
+	for (i = 0; i < nr_in_syncobjs; ++i) {
421
+		uint64_t address = in_syncobjs_addr + i * syncobj_stride;
422
+		struct dma_fence *fence;
423
+
424
+		if (copy_from_user(&syncobj_desc,
425
+			           u64_to_user_ptr(address),
426
+			           min(syncobj_stride, sizeof(syncobj_desc)))) {
427
+			ret = -EFAULT;
428
+			break;
429
+		}
430
+
431
+		if (syncobj_desc.point &&
432
+		    !drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) {
433
+			ret = -EOPNOTSUPP;
434
+			break;
435
+		}
436
+
437
+		if (syncobj_desc.flags & ~MSM_SUBMIT_SYNCOBJ_FLAGS) {
438
+			ret = -EINVAL;
439
+			break;
440
+		}
441
+
442
+		ret = drm_syncobj_find_fence(file, syncobj_desc.handle,
443
+		                             syncobj_desc.point, 0, &fence);
444
+		if (ret)
445
+			break;
446
+
447
+		if (!dma_fence_match_context(fence, ring->fctx->context))
448
+			ret = dma_fence_wait(fence, true);
449
+
450
+		dma_fence_put(fence);
451
+		if (ret)
452
+			break;
453
+
454
+		if (syncobj_desc.flags & MSM_SUBMIT_SYNCOBJ_RESET) {
455
+			syncobjs[i] =
456
+				drm_syncobj_find(file, syncobj_desc.handle);
457
+			if (!syncobjs[i]) {
458
+				ret = -EINVAL;
459
+				break;
460
+			}
461
+		}
400462 	}
401463 
402
-	ww_acquire_fini(&submit->ticket);
464
+	if (ret) {
465
+		for (j = 0; j <= i; ++j) {
466
+			if (syncobjs[j])
467
+				drm_syncobj_put(syncobjs[j]);
468
+		}
469
+		kfree(syncobjs);
470
+		return ERR_PTR(ret);
471
+	}
472
+	return syncobjs;
473
+}
474
+
475
+static void msm_reset_syncobjs(struct drm_syncobj **syncobjs,
476
+                               uint32_t nr_syncobjs)
477
+{
478
+	uint32_t i;
479
+
480
+	for (i = 0; syncobjs && i < nr_syncobjs; ++i) {
481
+		if (syncobjs[i])
482
+			drm_syncobj_replace_fence(syncobjs[i], NULL);
483
+	}
484
+}
485
+
486
+static struct msm_submit_post_dep *msm_parse_post_deps(struct drm_device *dev,
487
+                                                       struct drm_file *file,
488
+                                                       uint64_t syncobjs_addr,
489
+                                                       uint32_t nr_syncobjs,
490
+                                                       size_t syncobj_stride)
491
+{
492
+	struct msm_submit_post_dep *post_deps;
493
+	struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
494
+	int ret = 0;
495
+	uint32_t i, j;
496
+
497
+	post_deps = kcalloc(nr_syncobjs, sizeof(*post_deps),
498
+			    GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
499
+	if (!post_deps)
500
+		return ERR_PTR(-ENOMEM);
501
+
502
+	for (i = 0; i < nr_syncobjs; ++i) {
503
+		uint64_t address = syncobjs_addr + i * syncobj_stride;
504
+
505
+		if (copy_from_user(&syncobj_desc,
506
+			           u64_to_user_ptr(address),
507
+			           min(syncobj_stride, sizeof(syncobj_desc)))) {
508
+			ret = -EFAULT;
509
+			break;
510
+		}
511
+
512
+		post_deps[i].point = syncobj_desc.point;
513
+
514
+		if (syncobj_desc.flags) {
515
+			ret = -EINVAL;
516
+			break;
517
+		}
518
+
519
+		if (syncobj_desc.point) {
520
+			if (!drm_core_check_feature(dev,
521
+			                            DRIVER_SYNCOBJ_TIMELINE)) {
522
+				ret = -EOPNOTSUPP;
523
+				break;
524
+			}
525
+
526
+			post_deps[i].chain =
527
+				kmalloc(sizeof(*post_deps[i].chain),
528
+				        GFP_KERNEL);
529
+			if (!post_deps[i].chain) {
530
+				ret = -ENOMEM;
531
+				break;
532
+			}
533
+		}
534
+
535
+		post_deps[i].syncobj =
536
+			drm_syncobj_find(file, syncobj_desc.handle);
537
+		if (!post_deps[i].syncobj) {
538
+			ret = -EINVAL;
539
+			break;
540
+		}
541
+	}
542
+
543
+	if (ret) {
544
+		for (j = 0; j <= i; ++j) {
545
+			kfree(post_deps[j].chain);
546
+			if (post_deps[j].syncobj)
547
+				drm_syncobj_put(post_deps[j].syncobj);
548
+		}
549
+
550
+		kfree(post_deps);
551
+		return ERR_PTR(ret);
552
+	}
553
+
554
+	return post_deps;
555
+}
556
+
557
+static void msm_process_post_deps(struct msm_submit_post_dep *post_deps,
558
+                                  uint32_t count, struct dma_fence *fence)
559
+{
560
+	uint32_t i;
561
+
562
+	for (i = 0; post_deps && i < count; ++i) {
563
+		if (post_deps[i].chain) {
564
+			drm_syncobj_add_point(post_deps[i].syncobj,
565
+			                      post_deps[i].chain,
566
+			                      fence, post_deps[i].point);
567
+			post_deps[i].chain = NULL;
568
+		} else {
569
+			drm_syncobj_replace_fence(post_deps[i].syncobj,
570
+			                          fence);
571
+		}
572
+	}
403573 }
404574 
405575 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
406576 		struct drm_file *file)
407577 {
578
+	static atomic_t ident = ATOMIC_INIT(0);
408579 	struct msm_drm_private *priv = dev->dev_private;
409580 	struct drm_msm_gem_submit *args = data;
410581 	struct msm_file_private *ctx = file->driver_priv;
....@@ -413,12 +584,18 @@
413584 	struct sync_file *sync_file = NULL;
414585 	struct msm_gpu_submitqueue *queue;
415586 	struct msm_ringbuffer *ring;
587
+	struct msm_submit_post_dep *post_deps = NULL;
588
+	struct drm_syncobj **syncobjs_to_reset = NULL;
416589 	int out_fence_fd = -1;
590
+	struct pid *pid = get_pid(task_pid(current));
591
+	bool has_ww_ticket = false;
417592 	unsigned i;
418
-	int ret;
419
-
593
+	int ret, submitid;
420594 	if (!gpu)
421595 		return -ENXIO;
596
+
597
+	if (args->pad)
598
+		return -EINVAL;
422599 
423600 	/* for now, we just have 3d pipe.. eventually this would need to
424601 	 * be more clever to dispatch to appropriate gpu module:
....@@ -439,7 +616,12 @@
439616 	if (!queue)
440617 		return -ENOENT;
441618 
619
+	/* Get a unique identifier for the submission for logging purposes */
620
+	submitid = atomic_inc_return(&ident) - 1;
621
+
442622 	ring = gpu->rb[queue->prio];
623
+	trace_msm_gpu_submit(pid_nr(pid), ring->id, submitid,
624
+		args->nr_bos, args->nr_cmds);
443625 
444626 	if (args->flags & MSM_SUBMIT_FENCE_FD_IN) {
445627 		struct dma_fence *in_fence;
....@@ -462,9 +644,29 @@
462644 			return ret;
463645 	}
464646 
647
+	if (args->flags & MSM_SUBMIT_SYNCOBJ_IN) {
648
+		syncobjs_to_reset = msm_wait_deps(dev, file,
649
+		                                  args->in_syncobjs,
650
+		                                  args->nr_in_syncobjs,
651
+		                                  args->syncobj_stride, ring);
652
+		if (IS_ERR(syncobjs_to_reset))
653
+			return PTR_ERR(syncobjs_to_reset);
654
+	}
655
+
656
+	if (args->flags & MSM_SUBMIT_SYNCOBJ_OUT) {
657
+		post_deps = msm_parse_post_deps(dev, file,
658
+		                                args->out_syncobjs,
659
+		                                args->nr_out_syncobjs,
660
+		                                args->syncobj_stride);
661
+		if (IS_ERR(post_deps)) {
662
+			ret = PTR_ERR(post_deps);
663
+			goto out_post_unlock;
664
+		}
665
+	}
666
+
465667 	ret = mutex_lock_interruptible(&dev->struct_mutex);
466668 	if (ret)
467
-		return ret;
669
+		goto out_post_unlock;
468670 
469671 	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
470672 		out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
....@@ -474,11 +676,15 @@
474676 		}
475677 	}
476678 
477
-	submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds);
679
+	submit = submit_create(dev, gpu, queue, args->nr_bos,
680
+		args->nr_cmds);
478681 	if (!submit) {
479682 		ret = -ENOMEM;
480683 		goto out_unlock;
481684 	}
685
+
686
+	submit->pid = pid;
687
+	submit->ident = submitid;
482688 
483689 	if (args->flags & MSM_SUBMIT_SUDO)
484690 		submit->in_rb = true;
....@@ -487,6 +693,9 @@
487693 	if (ret)
488694 		goto out;
489695 
696
+	/* copy_*_user while holding a ww ticket upsets lockdep */
697
+	ww_acquire_init(&submit->ticket, &reservation_ww_class);
698
+	has_ww_ticket = true;
490699 	ret = submit_lock_objects(submit);
491700 	if (ret)
492701 		goto out;
....@@ -575,7 +784,7 @@
575784 		}
576785 	}
577786 
578
-	msm_gpu_submit(gpu, submit, ctx);
787
+	msm_gpu_submit(gpu, submit);
579788 
580789 	args->fence = submit->fence->seqno;
581790 
....@@ -584,13 +793,38 @@
584793 		args->fence_fd = out_fence_fd;
585794 	}
586795 
796
+	msm_reset_syncobjs(syncobjs_to_reset, args->nr_in_syncobjs);
797
+	msm_process_post_deps(post_deps, args->nr_out_syncobjs,
798
+	                      submit->fence);
799
+
800
+
587801 out:
588802 	submit_cleanup(submit);
803
+	if (has_ww_ticket)
804
+		ww_acquire_fini(&submit->ticket);
589805 	if (ret)
590806 		msm_gem_submit_free(submit);
591807 out_unlock:
592808 	if (ret && (out_fence_fd >= 0))
593809 		put_unused_fd(out_fence_fd);
594810 	mutex_unlock(&dev->struct_mutex);
811
+
812
+out_post_unlock:
813
+	if (!IS_ERR_OR_NULL(post_deps)) {
814
+		for (i = 0; i < args->nr_out_syncobjs; ++i) {
815
+			kfree(post_deps[i].chain);
816
+			drm_syncobj_put(post_deps[i].syncobj);
817
+		}
818
+		kfree(post_deps);
819
+	}
820
+
821
+	if (!IS_ERR_OR_NULL(syncobjs_to_reset)) {
822
+		for (i = 0; i < args->nr_in_syncobjs; ++i) {
823
+			if (syncobjs_to_reset[i])
824
+				drm_syncobj_put(syncobjs_to_reset[i]);
825
+		}
826
+		kfree(syncobjs_to_reset);
827
+	}
828
+
595829 	return ret;
596830 }