forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-11-23 7d07b3ae8ddad407913c5301877e694430a3263f 
 kernel/drivers/video/rockchip/rga3/rga_job.c
....@@ -14,74 +14,28 @@
1414 #include "rga_iommu.h"
1515 #include "rga_debugger.h"
1616 
17
-struct rga_job *
18
-rga_scheduler_get_pending_job_list(struct rga_scheduler_t *scheduler)
19
-{
20
-	unsigned long flags;
21
-	struct rga_job *job;
22
-
23
-	spin_lock_irqsave(&scheduler->irq_lock, flags);
24
-
25
-	job = list_first_entry_or_null(&scheduler->todo_list,
26
-		struct rga_job, head);
27
-
28
-	spin_unlock_irqrestore(&scheduler->irq_lock, flags);
29
-
30
-	return job;
31
-}
32
-
33
-struct rga_job *
34
-rga_scheduler_get_running_job(struct rga_scheduler_t *scheduler)
35
-{
36
-	unsigned long flags;
37
-	struct rga_job *job;
38
-
39
-	spin_lock_irqsave(&scheduler->irq_lock, flags);
40
-
41
-	job = scheduler->running_job;
42
-
43
-	spin_unlock_irqrestore(&scheduler->irq_lock, flags);
44
-
45
-	return job;
46
-}
47
-
48
-struct rga_scheduler_t *rga_job_get_scheduler(struct rga_job *job)
49
-{
50
-	return job->scheduler;
51
-}
52
-
5317 static void rga_job_free(struct rga_job *job)
5418 {
5519 	free_page((unsigned long)job);
5620 }
5721 
58
-void rga_job_session_destroy(struct rga_session *session)
22
+static void rga_job_kref_release(struct kref *ref)
5923 {
60
-	struct rga_scheduler_t *scheduler = NULL;
61
-	struct rga_job *job_pos, *job_q;
62
-	int i;
24
+	struct rga_job *job;
6325 
64
-	unsigned long flags;
26
+	job = container_of(ref, struct rga_job, refcount);
6527 
66
-	for (i = 0; i < rga_drvdata->num_of_scheduler; i++) {
67
-		scheduler = rga_drvdata->scheduler[i];
28
+	rga_job_free(job);
29
+}
6830 
69
-		spin_lock_irqsave(&scheduler->irq_lock, flags);
31
+static int rga_job_put(struct rga_job *job)
32
+{
33
+	return kref_put(&job->refcount, rga_job_kref_release);
34
+}
7035 
71
-		list_for_each_entry_safe(job_pos, job_q, &scheduler->todo_list, head) {
72
-			if (session == job_pos->session) {
73
-				list_del(&job_pos->head);
74
-
75
-				spin_unlock_irqrestore(&scheduler->irq_lock, flags);
76
-
77
-				rga_job_free(job_pos);
78
-
79
-				spin_lock_irqsave(&scheduler->irq_lock, flags);
80
-			}
81
-		}
82
-
83
-		spin_unlock_irqrestore(&scheduler->irq_lock, flags);
84
-	}
36
+static void rga_job_get(struct rga_job *job)
37
+{
38
+	kref_get(&job->refcount);
8539 }
8640 
8741 static int rga_job_cleanup(struct rga_job *job)
....@@ -90,7 +44,7 @@
9044 		pr_err("(pid:%d) job clean use time = %lld\n", job->pid,
9145 			ktime_us_delta(ktime_get(), job->timestamp));
9246 
93
-	rga_job_free(job);
47
+	rga_job_put(job);
9448 
9549 	return 0;
9650 }
....@@ -165,6 +119,7 @@
165119 		return NULL;
166120 
167121 	INIT_LIST_HEAD(&job->head);
122
+	kref_init(&job->refcount);
168123 
169124 	job->timestamp = ktime_get();
170125 	job->pid = current->pid;
....@@ -232,16 +187,18 @@
232187 		return ret;
233188 	}
234189 
190
+	set_bit(RGA_JOB_STATE_RUNNING, &job->state);
191
+
235192 	/* for debug */
236193 	if (DEBUGGER_EN(MSG))
237194 		rga_job_dump_info(job);
238195 
239196 	return ret;
240
-
241197 }
242198 
243
-static void rga_job_next(struct rga_scheduler_t *scheduler)
199
+void rga_job_next(struct rga_scheduler_t *scheduler)
244200 {
201
+	int ret;
245202 	struct rga_job *job = NULL;
246203 	unsigned long flags;
247204 
....@@ -261,51 +218,33 @@
261218 	scheduler->job_count--;
262219 
263220 	scheduler->running_job = job;
221
+	set_bit(RGA_JOB_STATE_PREPARE, &job->state);
222
+	rga_job_get(job);
264223 
265224 	spin_unlock_irqrestore(&scheduler->irq_lock, flags);
266225 
267
-	job->ret = rga_job_run(job, scheduler);
226
+	ret = rga_job_run(job, scheduler);
268227 	/* If some error before hw run */
269
-	if (job->ret < 0) {
270
-		pr_err("some error on rga_job_run before hw start, %s(%d)\n",
271
-			__func__, __LINE__);
228
+	if (ret < 0) {
229
+		pr_err("some error on rga_job_run before hw start, %s(%d)\n", __func__, __LINE__);
272230 
273231 		spin_lock_irqsave(&scheduler->irq_lock, flags);
274232 
275233 		scheduler->running_job = NULL;
234
+		rga_job_put(job);
276235 
277236 		spin_unlock_irqrestore(&scheduler->irq_lock, flags);
278237 
238
+		job->ret = ret;
279239 		rga_request_release_signal(scheduler, job);
280240 
281241 		goto next_job;
282242 	}
243
+
244
+	rga_job_put(job);
283245 }
284246 
285
-static void rga_job_finish_and_next(struct rga_scheduler_t *scheduler,
286
-		struct rga_job *job, int ret)
287
-{
288
-	ktime_t now;
289
-
290
-	job->ret = ret;
291
-
292
-	if (DEBUGGER_EN(TIME)) {
293
-		now = ktime_get();
294
-		pr_info("hw use time = %lld\n", ktime_us_delta(now, job->hw_running_time));
295
-		pr_info("(pid:%d) job done use time = %lld\n", job->pid,
296
-			ktime_us_delta(now, job->timestamp));
297
-	}
298
-
299
-	rga_mm_unmap_job_info(job);
300
-
301
-	rga_request_release_signal(scheduler, job);
302
-
303
-	rga_job_next(scheduler);
304
-
305
-	rga_power_disable(scheduler);
306
-}
307
-
308
-void rga_job_done(struct rga_scheduler_t *scheduler, int ret)
247
+struct rga_job *rga_job_done(struct rga_scheduler_t *scheduler)
309248 {
310249 	struct rga_job *job;
311250 	unsigned long flags;
....@@ -314,16 +253,34 @@
314253 	spin_lock_irqsave(&scheduler->irq_lock, flags);
315254 
316255 	job = scheduler->running_job;
256
+	if (job == NULL) {
257
+		pr_err("core[0x%x] running job has been cleanup.\n", scheduler->core);
258
+
259
+		spin_unlock_irqrestore(&scheduler->irq_lock, flags);
260
+		return NULL;
261
+	}
317262 	scheduler->running_job = NULL;
318263 
319264 	scheduler->timer.busy_time += ktime_us_delta(now, job->hw_recoder_time);
265
+	set_bit(RGA_JOB_STATE_DONE, &job->state);
320266 
321267 	spin_unlock_irqrestore(&scheduler->irq_lock, flags);
268
+
269
+	if (scheduler->ops->read_back_reg)
270
+		scheduler->ops->read_back_reg(job, scheduler);
322271 
323272 	if (DEBUGGER_EN(DUMP_IMAGE))
324273 		rga_dump_job_image(job);
325274 
326
-	rga_job_finish_and_next(scheduler, job, ret);
275
+	if (DEBUGGER_EN(TIME)) {
276
+		pr_info("hw use time = %lld\n", ktime_us_delta(now, job->hw_running_time));
277
+		pr_info("(pid:%d) job done use time = %lld\n", job->pid,
278
+			ktime_us_delta(now, job->timestamp));
279
+	}
280
+
281
+	rga_mm_unmap_job_info(job);
282
+
283
+	return job;
327284 }
328285 
329286 static void rga_job_scheduler_timeout_clean(struct rga_scheduler_t *scheduler)
....@@ -391,13 +348,20 @@
391348 	}
392349 
393350 	scheduler->job_count++;
351
+	set_bit(RGA_JOB_STATE_PENDING, &job->state);
394352 
395353 	spin_unlock_irqrestore(&scheduler->irq_lock, flags);
396354 }
397355 
398356 static struct rga_scheduler_t *rga_job_schedule(struct rga_job *job)
399357 {
358
+	int i;
400359 	struct rga_scheduler_t *scheduler = NULL;
360
+
361
+	for (i = 0; i < rga_drvdata->num_of_scheduler; i++) {
362
+		scheduler = rga_drvdata->scheduler[i];
363
+		rga_job_scheduler_timeout_clean(scheduler);
364
+	}
401365 
402366 	if (rga_drvdata->num_of_scheduler > 1) {
403367 		job->core = rga_job_assign(job);
....@@ -411,14 +375,12 @@
411375 		job->scheduler = rga_drvdata->scheduler[0];
412376 	}
413377 
414
-	scheduler = rga_job_get_scheduler(job);
378
+	scheduler = job->scheduler;
415379 	if (scheduler == NULL) {
416380 		pr_err("failed to get scheduler, %s(%d)\n", __func__, __LINE__);
417381 		job->ret = -EFAULT;
418382 		return NULL;
419383 	}
420
-
421
-	rga_job_scheduler_timeout_clean(scheduler);
422384 
423385 	return scheduler;
424386 }
....@@ -557,26 +519,13 @@
557519 	request->current_mm = NULL;
558520 }
559521 
560
-static int rga_request_alloc_release_fence(struct dma_fence **release_fence)
561
-{
562
-	struct dma_fence *fence;
563
-
564
-	fence = rga_dma_fence_alloc();
565
-	if (IS_ERR(fence)) {
566
-		pr_err("Can not alloc release fence!\n");
567
-		return IS_ERR(fence);
568
-	}
569
-
570
-	*release_fence = fence;
571
-
572
-	return rga_dma_fence_get_fd(fence);
573
-}
574
-
575
-static int rga_request_add_acquire_fence_callback(int acquire_fence_fd, void *private,
522
+static int rga_request_add_acquire_fence_callback(int acquire_fence_fd,
523
+						  struct rga_request *request,
576524 						  dma_fence_func_t cb_func)
577525 {
578526 	int ret;
579527 	struct dma_fence *acquire_fence = NULL;
528
+	struct rga_pending_request_manager *request_manager = rga_drvdata->pend_request_manager;
580529 
581530 	if (DEBUGGER_EN(MSG))
582531 		pr_info("acquire_fence_fd = %d", acquire_fence_fd);
....@@ -588,19 +537,38 @@
588537 		return -EINVAL;
589538 	}
590539 	/* close acquire fence fd */
540
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)
541
+	close_fd(acquire_fence_fd);
542
+#else
591543 	ksys_close(acquire_fence_fd);
544
+#endif
592545 
593546 	ret = rga_dma_fence_get_status(acquire_fence);
594
-	if (ret == 0) {
595
-		ret = rga_dma_fence_add_callback(acquire_fence, cb_func, private);
596
-		if (ret < 0) {
597
-			if (ret == -ENOENT)
598
-				return 1;
547
+	if (ret < 0) {
548
+		pr_err("%s: Current acquire fence unexpectedly has error status before signal\n",
549
+		       __func__);
550
+		return ret;
551
+	} else if (ret > 0) {
552
+		/* has been signaled */
553
+		return ret;
554
+	}
599555 
556
+	/*
557
+	 * Ensure that the request will not be free early when
558
+	 * the callback is called.
559
+	 */
560
+	mutex_lock(&request_manager->lock);
561
+	rga_request_get(request);
562
+	mutex_unlock(&request_manager->lock);
563
+
564
+	ret = rga_dma_fence_add_callback(acquire_fence, cb_func, (void *)request);
565
+	if (ret < 0) {
566
+		if (ret != -ENOENT)
600567 			pr_err("%s: failed to add fence callback\n", __func__);
601
-			return ret;
602
-		}
603
-	} else {
568
+
569
+		mutex_lock(&request_manager->lock);
570
+		rga_request_put(request);
571
+		mutex_unlock(&request_manager->lock);
604572 		return ret;
605573 	}
606574 
....@@ -742,6 +710,70 @@
742710 	mutex_unlock(&request_manager->lock);
743711 }
744712 
713
+void rga_request_session_destroy_abort(struct rga_session *session)
714
+{
715
+	int request_id;
716
+	struct rga_request *request;
717
+	struct rga_pending_request_manager *request_manager;
718
+
719
+	request_manager = rga_drvdata->pend_request_manager;
720
+	if (request_manager == NULL) {
721
+		pr_err("rga_pending_request_manager is null!\n");
722
+		return;
723
+	}
724
+
725
+	mutex_lock(&request_manager->lock);
726
+
727
+	idr_for_each_entry(&request_manager->request_idr, request, request_id) {
728
+		if (session == request->session) {
729
+			pr_err("[tgid:%d pid:%d] destroy request[%d] when the user exits",
730
+			       session->tgid, current->pid, request->id);
731
+			rga_request_put(request);
732
+		}
733
+	}
734
+
735
+	mutex_unlock(&request_manager->lock);
736
+}
737
+
738
+static int rga_request_timeout_query_state(struct rga_request *request)
739
+{
740
+	int i;
741
+	unsigned long flags;
742
+	struct rga_scheduler_t *scheduler = NULL;
743
+	struct rga_job *job = NULL;
744
+
745
+	for (i = 0; i < rga_drvdata->num_of_scheduler; i++) {
746
+		scheduler = rga_drvdata->scheduler[i];
747
+
748
+		spin_lock_irqsave(&scheduler->irq_lock, flags);
749
+
750
+		if (scheduler->running_job) {
751
+			job = scheduler->running_job;
752
+			if (request->id == job->request_id) {
753
+				if (test_bit(RGA_JOB_STATE_DONE, &job->state) &&
754
+				    test_bit(RGA_JOB_STATE_FINISH, &job->state)) {
755
+					spin_unlock_irqrestore(&scheduler->irq_lock, flags);
756
+					return request->ret;
757
+				} else if (!test_bit(RGA_JOB_STATE_DONE, &job->state) &&
758
+					   test_bit(RGA_JOB_STATE_FINISH, &job->state)) {
759
+					spin_unlock_irqrestore(&scheduler->irq_lock, flags);
760
+					pr_err("hardware has finished, but the software has timeout!\n");
761
+					return -EBUSY;
762
+				} else if (!test_bit(RGA_JOB_STATE_DONE, &job->state) &&
763
+					   !test_bit(RGA_JOB_STATE_FINISH, &job->state)) {
764
+					spin_unlock_irqrestore(&scheduler->irq_lock, flags);
765
+					pr_err("hardware has timeout.\n");
766
+					return -EBUSY;
767
+				}
768
+			}
769
+		}
770
+
771
+		spin_unlock_irqrestore(&scheduler->irq_lock, flags);
772
+	}
773
+
774
+	return request->ret;
775
+}
776
+
745777 static int rga_request_wait(struct rga_request *request)
746778 {
747779 	int left_time;
....@@ -752,8 +784,7 @@
752784 
753785 	switch (left_time) {
754786 	case 0:
755
-		pr_err("%s timeout", __func__);
756
-		ret = -EBUSY;
787
+		ret = rga_request_timeout_query_state(request);
757788 		goto err_request_abort;
758789 	case -ERESTARTSYS:
759790 		ret = -ERESTARTSYS;
....@@ -800,9 +831,15 @@
800831 						  struct dma_fence_cb *_waiter)
801832 {
802833 	struct rga_fence_waiter *waiter = (struct rga_fence_waiter *)_waiter;
834
+	struct rga_request *request = (struct rga_request *)waiter->private;
835
+	struct rga_pending_request_manager *request_manager = rga_drvdata->pend_request_manager;
803836 
804
-	if (rga_request_commit((struct rga_request *)waiter->private))
837
+	if (rga_request_commit(request))
805838 		pr_err("rga request commit failed!\n");
839
+
840
+	mutex_lock(&request_manager->lock);
841
+	rga_request_put(request);
842
+	mutex_unlock(&request_manager->lock);
806843 
807844 	kfree(waiter);
808845 }
....@@ -832,8 +869,6 @@
832869 	rga_request_get(request);
833870 	mutex_unlock(&request_manager->lock);
834871 
835
-	rga_job_cleanup(job);
836
-
837872 	spin_lock_irqsave(&request->lock, flags);
838873 
839874 	if (job->ret < 0) {
....@@ -847,6 +882,8 @@
847882 	finished_count = request->finished_task_count;
848883 
849884 	spin_unlock_irqrestore(&request->lock, flags);
885
+
886
+	rga_job_cleanup(job);
850887 
851888 	if ((failed_count + finished_count) >= request->task_count) {
852889 		spin_lock_irqsave(&request->lock, flags);
....@@ -1003,18 +1040,21 @@
10031040 {
10041041 	int ret = 0;
10051042 	unsigned long flags;
1043
+	struct dma_fence *release_fence;
10061044 
10071045 	spin_lock_irqsave(&request->lock, flags);
10081046 
10091047 	if (request->is_running) {
1010
-		pr_err("can not re-config when request is running");
10111048 		spin_unlock_irqrestore(&request->lock, flags);
1049
+
1050
+		pr_err("can not re-config when request is running\n");
10121051 		return -EFAULT;
10131052 	}
10141053 
10151054 	if (request->task_list == NULL) {
1016
-		pr_err("can not find task list from id[%d]", request->id);
10171055 		spin_unlock_irqrestore(&request->lock, flags);
1056
+
1057
+		pr_err("can not find task list from id[%d]\n", request->id);
10181058 		return -EINVAL;
10191059 	}
10201060 
....@@ -1026,46 +1066,71 @@
10261066 
10271067 	rga_request_get_current_mm(request);
10281068 
1069
+	/* Unlock after ensuring that the current request will not be resubmitted. */
10291070 	spin_unlock_irqrestore(&request->lock, flags);
10301071 
10311072 	if (request->sync_mode == RGA_BLIT_ASYNC) {
1032
-		ret = rga_request_alloc_release_fence(&request->release_fence);
1033
-		if (ret < 0) {
1034
-			pr_err("Failed to alloc release fence fd!\n");
1035
-			return ret;
1073
+		release_fence = rga_dma_fence_alloc();
1074
+		if (IS_ERR(release_fence)) {
1075
+			pr_err("Can not alloc release fence!\n");
1076
+			ret = IS_ERR(release_fence);
1077
+			goto error_put_current_mm;
10361078 		}
1037
-		request->release_fence_fd = ret;
1079
+		request->release_fence = release_fence;
10381080 
10391081 		if (request->acquire_fence_fd > 0) {
10401082 			ret = rga_request_add_acquire_fence_callback(
1041
-				request->acquire_fence_fd,
1042
-				(void *)request,
1083
+				request->acquire_fence_fd, request,
10431084 				rga_request_acquire_fence_signaled_cb);
10441085 			if (ret == 0) {
1045
-				return ret;
1046
-			} else if (ret == 1) {
1086
+				/* acquire fence active */
1087
+				goto export_release_fence_fd;
1088
+			} else if (ret > 0) {
1089
+				/* acquire fence has been signaled */
10471090 				goto request_commit;
10481091 			} else {
10491092 				pr_err("Failed to add callback with acquire fence fd[%d]!\n",
10501093 				       request->acquire_fence_fd);
1051
-				goto error_release_fence_put;
1094
+				goto err_put_release_fence;
10521095 			}
10531096 		}
1054
-
10551097 	}
10561098 
10571099 request_commit:
10581100 	ret = rga_request_commit(request);
10591101 	if (ret < 0) {
10601102 		pr_err("rga request commit failed!\n");
1061
-		goto error_release_fence_put;
1103
+		goto err_put_release_fence;
1104
+	}
1105
+
1106
+export_release_fence_fd:
1107
+	if (request->release_fence != NULL) {
1108
+		ret = rga_dma_fence_get_fd(request->release_fence);
1109
+		if (ret < 0) {
1110
+			pr_err("Failed to alloc release fence fd!\n");
1111
+			rga_request_release_abort(request, ret);
1112
+			return ret;
1113
+		}
1114
+
1115
+		request->release_fence_fd = ret;
10621116 	}
10631117 
10641118 	return 0;
10651119 
1066
-error_release_fence_put:
1067
-	rga_dma_fence_put(request->release_fence);
1068
-	request->release_fence = NULL;
1120
+err_put_release_fence:
1121
+	if (request->release_fence != NULL) {
1122
+		rga_dma_fence_put(request->release_fence);
1123
+		request->release_fence = NULL;
1124
+	}
1125
+
1126
+error_put_current_mm:
1127
+	spin_lock_irqsave(&request->lock, flags);
1128
+
1129
+	rga_request_put_current_mm(request);
1130
+	request->is_running = false;
1131
+
1132
+	spin_unlock_irqrestore(&request->lock, flags);
1133
+
10691134 	return ret;
10701135 }
10711136 
....@@ -1159,10 +1224,11 @@
11591224 	request = container_of(ref, struct rga_request, refcount);
11601225 
11611226 	if (rga_dma_fence_get_status(request->release_fence) == 0)
1162
-		rga_dma_fence_signal(request->release_fence, -EEXIST);
1227
+		rga_dma_fence_signal(request->release_fence, -EFAULT);
11631228 
11641229 	spin_lock_irqsave(&request->lock, flags);
11651230 
1231
+	rga_request_put_current_mm(request);
11661232 	rga_dma_fence_put(request->release_fence);
11671233 
11681234 	if (!request->is_running || request->is_done) {
....@@ -1188,6 +1254,7 @@
11881254 
11891255 int rga_request_alloc(uint32_t flags, struct rga_session *session)
11901256 {
1257
+	int new_id;
11911258 	struct rga_pending_request_manager *request_manager;
11921259 	struct rga_request *request;
11931260 
....@@ -1218,17 +1285,17 @@
12181285 	mutex_lock(&request_manager->lock);
12191286 
12201287 	idr_preload(GFP_KERNEL);
1221
-	request->id = idr_alloc(&request_manager->request_idr, request, 1, 0, GFP_KERNEL);
1288
+	new_id = idr_alloc_cyclic(&request_manager->request_idr, request, 1, 0, GFP_NOWAIT);
12221289 	idr_preload_end();
1223
-
1224
-	if (request->id <= 0) {
1225
-		pr_err("alloc request_id failed!\n");
1290
+	if (new_id < 0) {
1291
+		pr_err("request alloc id failed!\n");
12261292 
12271293 		mutex_unlock(&request_manager->lock);
12281294 		kfree(request);
1229
-		return -EFAULT;
1295
+		return new_id;
12301296 	}
12311297 
1298
+	request->id = new_id;
12321299 	request_manager->request_count++;
12331300 
12341301 	mutex_unlock(&request_manager->lock);