forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 95099d4622f8cb224d94e314c7a8e0df60b13f87 
 kernel/drivers/video/rockchip/rga3/rga_job.c
....@@ -14,74 +14,28 @@
1414 #include "rga_iommu.h"
1515 #include "rga_debugger.h"
1616 
17
-struct rga_job *
18
-rga_scheduler_get_pending_job_list(struct rga_scheduler_t *scheduler)
19
-{
20
-	unsigned long flags;
21
-	struct rga_job *job;
22
-
23
-	spin_lock_irqsave(&scheduler->irq_lock, flags);
24
-
25
-	job = list_first_entry_or_null(&scheduler->todo_list,
26
-		struct rga_job, head);
27
-
28
-	spin_unlock_irqrestore(&scheduler->irq_lock, flags);
29
-
30
-	return job;
31
-}
32
-
33
-struct rga_job *
34
-rga_scheduler_get_running_job(struct rga_scheduler_t *scheduler)
35
-{
36
-	unsigned long flags;
37
-	struct rga_job *job;
38
-
39
-	spin_lock_irqsave(&scheduler->irq_lock, flags);
40
-
41
-	job = scheduler->running_job;
42
-
43
-	spin_unlock_irqrestore(&scheduler->irq_lock, flags);
44
-
45
-	return job;
46
-}
47
-
48
-struct rga_scheduler_t *rga_job_get_scheduler(struct rga_job *job)
49
-{
50
-	return job->scheduler;
51
-}
52
-
5317 static void rga_job_free(struct rga_job *job)
5418 {
5519 	free_page((unsigned long)job);
5620 }
5721 
58
-void rga_job_session_destroy(struct rga_session *session)
22
+static void rga_job_kref_release(struct kref *ref)
5923 {
60
-	struct rga_scheduler_t *scheduler = NULL;
61
-	struct rga_job *job_pos, *job_q;
62
-	int i;
24
+	struct rga_job *job;
6325 
64
-	unsigned long flags;
26
+	job = container_of(ref, struct rga_job, refcount);
6527 
66
-	for (i = 0; i < rga_drvdata->num_of_scheduler; i++) {
67
-		scheduler = rga_drvdata->scheduler[i];
28
+	rga_job_free(job);
29
+}
6830 
69
-		spin_lock_irqsave(&scheduler->irq_lock, flags);
31
+static int rga_job_put(struct rga_job *job)
32
+{
33
+	return kref_put(&job->refcount, rga_job_kref_release);
34
+}
7035 
71
-		list_for_each_entry_safe(job_pos, job_q, &scheduler->todo_list, head) {
72
-			if (session == job_pos->session) {
73
-				list_del(&job_pos->head);
74
-
75
-				spin_unlock_irqrestore(&scheduler->irq_lock, flags);
76
-
77
-				rga_job_free(job_pos);
78
-
79
-				spin_lock_irqsave(&scheduler->irq_lock, flags);
80
-			}
81
-		}
82
-
83
-		spin_unlock_irqrestore(&scheduler->irq_lock, flags);
84
-	}
36
+static void rga_job_get(struct rga_job *job)
37
+{
38
+	kref_get(&job->refcount);
8539 }
8640 
8741 static int rga_job_cleanup(struct rga_job *job)
....@@ -90,7 +44,7 @@
9044 		pr_err("(pid:%d) job clean use time = %lld\n", job->pid,
9145 			ktime_us_delta(ktime_get(), job->timestamp));
9246 
93
-	rga_job_free(job);
47
+	rga_job_put(job);
9448 
9549 	return 0;
9650 }
....@@ -165,6 +119,7 @@
165119 		return NULL;
166120 
167121 	INIT_LIST_HEAD(&job->head);
122
+	kref_init(&job->refcount);
168123 
169124 	job->timestamp = ktime_get();
170125 	job->pid = current->pid;
....@@ -232,16 +187,18 @@
232187 		return ret;
233188 	}
234189 
190
+	set_bit(RGA_JOB_STATE_RUNNING, &job->state);
191
+
235192 	/* for debug */
236193 	if (DEBUGGER_EN(MSG))
237194 		rga_job_dump_info(job);
238195 
239196 	return ret;
240
-
241197 }
242198 
243
-static void rga_job_next(struct rga_scheduler_t *scheduler)
199
+void rga_job_next(struct rga_scheduler_t *scheduler)
244200 {
201
+	int ret;
245202 	struct rga_job *job = NULL;
246203 	unsigned long flags;
247204 
....@@ -261,51 +218,33 @@
261218 	scheduler->job_count--;
262219 
263220 	scheduler->running_job = job;
221
+	set_bit(RGA_JOB_STATE_PREPARE, &job->state);
222
+	rga_job_get(job);
264223 
265224 	spin_unlock_irqrestore(&scheduler->irq_lock, flags);
266225 
267
-	job->ret = rga_job_run(job, scheduler);
226
+	ret = rga_job_run(job, scheduler);
268227 	/* If some error before hw run */
269
-	if (job->ret < 0) {
270
-		pr_err("some error on rga_job_run before hw start, %s(%d)\n",
271
-			__func__, __LINE__);
228
+	if (ret < 0) {
229
+		pr_err("some error on rga_job_run before hw start, %s(%d)\n", __func__, __LINE__);
272230 
273231 		spin_lock_irqsave(&scheduler->irq_lock, flags);
274232 
275233 		scheduler->running_job = NULL;
234
+		rga_job_put(job);
276235 
277236 		spin_unlock_irqrestore(&scheduler->irq_lock, flags);
278237 
238
+		job->ret = ret;
279239 		rga_request_release_signal(scheduler, job);
280240 
281241 		goto next_job;
282242 	}
243
+
244
+	rga_job_put(job);
283245 }
284246 
285
-static void rga_job_finish_and_next(struct rga_scheduler_t *scheduler,
286
-		struct rga_job *job, int ret)
287
-{
288
-	ktime_t now;
289
-
290
-	job->ret = ret;
291
-
292
-	if (DEBUGGER_EN(TIME)) {
293
-		now = ktime_get();
294
-		pr_info("hw use time = %lld\n", ktime_us_delta(now, job->hw_running_time));
295
-		pr_info("(pid:%d) job done use time = %lld\n", job->pid,
296
-			ktime_us_delta(now, job->timestamp));
297
-	}
298
-
299
-	rga_mm_unmap_job_info(job);
300
-
301
-	rga_request_release_signal(scheduler, job);
302
-
303
-	rga_job_next(scheduler);
304
-
305
-	rga_power_disable(scheduler);
306
-}
307
-
308
-void rga_job_done(struct rga_scheduler_t *scheduler, int ret)
247
+struct rga_job *rga_job_done(struct rga_scheduler_t *scheduler)
309248 {
310249 	struct rga_job *job;
311250 	unsigned long flags;
....@@ -314,16 +253,34 @@
314253 	spin_lock_irqsave(&scheduler->irq_lock, flags);
315254 
316255 	job = scheduler->running_job;
256
+	if (job == NULL) {
257
+		pr_err("core[0x%x] running job has been cleanup.\n", scheduler->core);
258
+
259
+		spin_unlock_irqrestore(&scheduler->irq_lock, flags);
260
+		return NULL;
261
+	}
317262 	scheduler->running_job = NULL;
318263 
319264 	scheduler->timer.busy_time += ktime_us_delta(now, job->hw_recoder_time);
265
+	set_bit(RGA_JOB_STATE_DONE, &job->state);
320266 
321267 	spin_unlock_irqrestore(&scheduler->irq_lock, flags);
268
+
269
+	if (scheduler->ops->read_back_reg)
270
+		scheduler->ops->read_back_reg(job, scheduler);
322271 
323272 	if (DEBUGGER_EN(DUMP_IMAGE))
324273 		rga_dump_job_image(job);
325274 
326
-	rga_job_finish_and_next(scheduler, job, ret);
275
+	if (DEBUGGER_EN(TIME)) {
276
+		pr_info("hw use time = %lld\n", ktime_us_delta(now, job->hw_running_time));
277
+		pr_info("(pid:%d) job done use time = %lld\n", job->pid,
278
+			ktime_us_delta(now, job->timestamp));
279
+	}
280
+
281
+	rga_mm_unmap_job_info(job);
282
+
283
+	return job;
327284 }
328285 
329286 static void rga_job_scheduler_timeout_clean(struct rga_scheduler_t *scheduler)
....@@ -391,13 +348,20 @@
391348 	}
392349 
393350 	scheduler->job_count++;
351
+	set_bit(RGA_JOB_STATE_PENDING, &job->state);
394352 
395353 	spin_unlock_irqrestore(&scheduler->irq_lock, flags);
396354 }
397355 
398356 static struct rga_scheduler_t *rga_job_schedule(struct rga_job *job)
399357 {
358
+	int i;
400359 	struct rga_scheduler_t *scheduler = NULL;
360
+
361
+	for (i = 0; i < rga_drvdata->num_of_scheduler; i++) {
362
+		scheduler = rga_drvdata->scheduler[i];
363
+		rga_job_scheduler_timeout_clean(scheduler);
364
+	}
401365 
402366 	if (rga_drvdata->num_of_scheduler > 1) {
403367 		job->core = rga_job_assign(job);
....@@ -411,14 +375,12 @@
411375 		job->scheduler = rga_drvdata->scheduler[0];
412376 	}
413377 
414
-	scheduler = rga_job_get_scheduler(job);
378
+	scheduler = job->scheduler;
415379 	if (scheduler == NULL) {
416380 		pr_err("failed to get scheduler, %s(%d)\n", __func__, __LINE__);
417381 		job->ret = -EFAULT;
418382 		return NULL;
419383 	}
420
-
421
-	rga_job_scheduler_timeout_clean(scheduler);
422384 
423385 	return scheduler;
424386 }
....@@ -557,26 +519,13 @@
557519 	request->current_mm = NULL;
558520 }
559521 
560
-static int rga_request_alloc_release_fence(struct dma_fence **release_fence)
561
-{
562
-	struct dma_fence *fence;
563
-
564
-	fence = rga_dma_fence_alloc();
565
-	if (IS_ERR(fence)) {
566
-		pr_err("Can not alloc release fence!\n");
567
-		return IS_ERR(fence);
568
-	}
569
-
570
-	*release_fence = fence;
571
-
572
-	return rga_dma_fence_get_fd(fence);
573
-}
574
-
575
-static int rga_request_add_acquire_fence_callback(int acquire_fence_fd, void *private,
522
+static int rga_request_add_acquire_fence_callback(int acquire_fence_fd,
523
+						  struct rga_request *request,
576524 						  dma_fence_func_t cb_func)
577525 {
578526 	int ret;
579527 	struct dma_fence *acquire_fence = NULL;
528
+	struct rga_pending_request_manager *request_manager = rga_drvdata->pend_request_manager;
580529 
581530 	if (DEBUGGER_EN(MSG))
582531 		pr_info("acquire_fence_fd = %d", acquire_fence_fd);
....@@ -591,16 +540,31 @@
591540 	ksys_close(acquire_fence_fd);
592541 
593542 	ret = rga_dma_fence_get_status(acquire_fence);
594
-	if (ret == 0) {
595
-		ret = rga_dma_fence_add_callback(acquire_fence, cb_func, private);
596
-		if (ret < 0) {
597
-			if (ret == -ENOENT)
598
-				return 1;
543
+	if (ret < 0) {
544
+		pr_err("%s: Current acquire fence unexpectedly has error status before signal\n",
545
+		       __func__);
546
+		return ret;
547
+	} else if (ret > 0) {
548
+		/* has been signaled */
549
+		return ret;
550
+	}
599551 
552
+	/*
553
+	 * Ensure that the request will not be free early when
554
+	 * the callback is called.
555
+	 */
556
+	mutex_lock(&request_manager->lock);
557
+	rga_request_get(request);
558
+	mutex_unlock(&request_manager->lock);
559
+
560
+	ret = rga_dma_fence_add_callback(acquire_fence, cb_func, (void *)request);
561
+	if (ret < 0) {
562
+		if (ret != -ENOENT)
600563 			pr_err("%s: failed to add fence callback\n", __func__);
601
-			return ret;
602
-		}
603
-	} else {
564
+
565
+		mutex_lock(&request_manager->lock);
566
+		rga_request_put(request);
567
+		mutex_unlock(&request_manager->lock);
604568 		return ret;
605569 	}
606570 
....@@ -742,6 +706,70 @@
742706 	mutex_unlock(&request_manager->lock);
743707 }
744708 
709
+void rga_request_session_destroy_abort(struct rga_session *session)
710
+{
711
+	int request_id;
712
+	struct rga_request *request;
713
+	struct rga_pending_request_manager *request_manager;
714
+
715
+	request_manager = rga_drvdata->pend_request_manager;
716
+	if (request_manager == NULL) {
717
+		pr_err("rga_pending_request_manager is null!\n");
718
+		return;
719
+	}
720
+
721
+	mutex_lock(&request_manager->lock);
722
+
723
+	idr_for_each_entry(&request_manager->request_idr, request, request_id) {
724
+		if (session == request->session) {
725
+			pr_err("[tgid:%d pid:%d] destroy request[%d] when the user exits",
726
+			       session->tgid, current->pid, request->id);
727
+			rga_request_put(request);
728
+		}
729
+	}
730
+
731
+	mutex_unlock(&request_manager->lock);
732
+}
733
+
734
+static int rga_request_timeout_query_state(struct rga_request *request)
735
+{
736
+	int i;
737
+	unsigned long flags;
738
+	struct rga_scheduler_t *scheduler = NULL;
739
+	struct rga_job *job = NULL;
740
+
741
+	for (i = 0; i < rga_drvdata->num_of_scheduler; i++) {
742
+		scheduler = rga_drvdata->scheduler[i];
743
+
744
+		spin_lock_irqsave(&scheduler->irq_lock, flags);
745
+
746
+		if (scheduler->running_job) {
747
+			job = scheduler->running_job;
748
+			if (request->id == job->request_id) {
749
+				if (test_bit(RGA_JOB_STATE_DONE, &job->state) &&
750
+				    test_bit(RGA_JOB_STATE_FINISH, &job->state)) {
751
+					spin_unlock_irqrestore(&scheduler->irq_lock, flags);
752
+					return request->ret;
753
+				} else if (!test_bit(RGA_JOB_STATE_DONE, &job->state) &&
754
+					   test_bit(RGA_JOB_STATE_FINISH, &job->state)) {
755
+					spin_unlock_irqrestore(&scheduler->irq_lock, flags);
756
+					pr_err("hardware has finished, but the software has timeout!\n");
757
+					return -EBUSY;
758
+				} else if (!test_bit(RGA_JOB_STATE_DONE, &job->state) &&
759
+					   !test_bit(RGA_JOB_STATE_FINISH, &job->state)) {
760
+					spin_unlock_irqrestore(&scheduler->irq_lock, flags);
761
+					pr_err("hardware has timeout.\n");
762
+					return -EBUSY;
763
+				}
764
+			}
765
+		}
766
+
767
+		spin_unlock_irqrestore(&scheduler->irq_lock, flags);
768
+	}
769
+
770
+	return request->ret;
771
+}
772
+
745773 static int rga_request_wait(struct rga_request *request)
746774 {
747775 	int left_time;
....@@ -752,8 +780,7 @@
752780 
753781 	switch (left_time) {
754782 	case 0:
755
-		pr_err("%s timeout", __func__);
756
-		ret = -EBUSY;
783
+		ret = rga_request_timeout_query_state(request);
757784 		goto err_request_abort;
758785 	case -ERESTARTSYS:
759786 		ret = -ERESTARTSYS;
....@@ -800,9 +827,15 @@
800827 						  struct dma_fence_cb *_waiter)
801828 {
802829 	struct rga_fence_waiter *waiter = (struct rga_fence_waiter *)_waiter;
830
+	struct rga_request *request = (struct rga_request *)waiter->private;
831
+	struct rga_pending_request_manager *request_manager = rga_drvdata->pend_request_manager;
803832 
804
-	if (rga_request_commit((struct rga_request *)waiter->private))
833
+	if (rga_request_commit(request))
805834 		pr_err("rga request commit failed!\n");
835
+
836
+	mutex_lock(&request_manager->lock);
837
+	rga_request_put(request);
838
+	mutex_unlock(&request_manager->lock);
806839 
807840 	kfree(waiter);
808841 }
....@@ -832,8 +865,6 @@
832865 	rga_request_get(request);
833866 	mutex_unlock(&request_manager->lock);
834867 
835
-	rga_job_cleanup(job);
836
-
837868 	spin_lock_irqsave(&request->lock, flags);
838869 
839870 	if (job->ret < 0) {
....@@ -847,6 +878,8 @@
847878 	finished_count = request->finished_task_count;
848879 
849880 	spin_unlock_irqrestore(&request->lock, flags);
881
+
882
+	rga_job_cleanup(job);
850883 
851884 	if ((failed_count + finished_count) >= request->task_count) {
852885 		spin_lock_irqsave(&request->lock, flags);
....@@ -1003,18 +1036,21 @@
10031036 {
10041037 	int ret = 0;
10051038 	unsigned long flags;
1039
+	struct dma_fence *release_fence;
10061040 
10071041 	spin_lock_irqsave(&request->lock, flags);
10081042 
10091043 	if (request->is_running) {
1010
-		pr_err("can not re-config when request is running");
10111044 		spin_unlock_irqrestore(&request->lock, flags);
1045
+
1046
+		pr_err("can not re-config when request is running\n");
10121047 		return -EFAULT;
10131048 	}
10141049 
10151050 	if (request->task_list == NULL) {
1016
-		pr_err("can not find task list from id[%d]", request->id);
10171051 		spin_unlock_irqrestore(&request->lock, flags);
1052
+
1053
+		pr_err("can not find task list from id[%d]\n", request->id);
10181054 		return -EINVAL;
10191055 	}
10201056 
....@@ -1026,46 +1062,71 @@
10261062 
10271063 	rga_request_get_current_mm(request);
10281064 
1065
+	/* Unlock after ensuring that the current request will not be resubmitted. */
10291066 	spin_unlock_irqrestore(&request->lock, flags);
10301067 
10311068 	if (request->sync_mode == RGA_BLIT_ASYNC) {
1032
-		ret = rga_request_alloc_release_fence(&request->release_fence);
1033
-		if (ret < 0) {
1034
-			pr_err("Failed to alloc release fence fd!\n");
1035
-			return ret;
1069
+		release_fence = rga_dma_fence_alloc();
1070
+		if (IS_ERR(release_fence)) {
1071
+			pr_err("Can not alloc release fence!\n");
1072
+			ret = IS_ERR(release_fence);
1073
+			goto error_put_current_mm;
10361074 		}
1037
-		request->release_fence_fd = ret;
1075
+		request->release_fence = release_fence;
10381076 
10391077 		if (request->acquire_fence_fd > 0) {
10401078 			ret = rga_request_add_acquire_fence_callback(
1041
-				request->acquire_fence_fd,
1042
-				(void *)request,
1079
+				request->acquire_fence_fd, request,
10431080 				rga_request_acquire_fence_signaled_cb);
10441081 			if (ret == 0) {
1045
-				return ret;
1046
-			} else if (ret == 1) {
1082
+				/* acquire fence active */
1083
+				goto export_release_fence_fd;
1084
+			} else if (ret > 0) {
1085
+				/* acquire fence has been signaled */
10471086 				goto request_commit;
10481087 			} else {
10491088 				pr_err("Failed to add callback with acquire fence fd[%d]!\n",
10501089 				       request->acquire_fence_fd);
1051
-				goto error_release_fence_put;
1090
+				goto err_put_release_fence;
10521091 			}
10531092 		}
1054
-
10551093 	}
10561094 
10571095 request_commit:
10581096 	ret = rga_request_commit(request);
10591097 	if (ret < 0) {
10601098 		pr_err("rga request commit failed!\n");
1061
-		goto error_release_fence_put;
1099
+		goto err_put_release_fence;
1100
+	}
1101
+
1102
+export_release_fence_fd:
1103
+	if (request->release_fence != NULL) {
1104
+		ret = rga_dma_fence_get_fd(request->release_fence);
1105
+		if (ret < 0) {
1106
+			pr_err("Failed to alloc release fence fd!\n");
1107
+			rga_request_release_abort(request, ret);
1108
+			return ret;
1109
+		}
1110
+
1111
+		request->release_fence_fd = ret;
10621112 	}
10631113 
10641114 	return 0;
10651115 
1066
-error_release_fence_put:
1067
-	rga_dma_fence_put(request->release_fence);
1068
-	request->release_fence = NULL;
1116
+err_put_release_fence:
1117
+	if (request->release_fence != NULL) {
1118
+		rga_dma_fence_put(request->release_fence);
1119
+		request->release_fence = NULL;
1120
+	}
1121
+
1122
+error_put_current_mm:
1123
+	spin_lock_irqsave(&request->lock, flags);
1124
+
1125
+	rga_request_put_current_mm(request);
1126
+	request->is_running = false;
1127
+
1128
+	spin_unlock_irqrestore(&request->lock, flags);
1129
+
10691130 	return ret;
10701131 }
10711132 
....@@ -1159,10 +1220,11 @@
11591220 	request = container_of(ref, struct rga_request, refcount);
11601221 
11611222 	if (rga_dma_fence_get_status(request->release_fence) == 0)
1162
-		rga_dma_fence_signal(request->release_fence, -EEXIST);
1223
+		rga_dma_fence_signal(request->release_fence, -EFAULT);
11631224 
11641225 	spin_lock_irqsave(&request->lock, flags);
11651226 
1227
+	rga_request_put_current_mm(request);
11661228 	rga_dma_fence_put(request->release_fence);
11671229 
11681230 	if (!request->is_running || request->is_done) {
....@@ -1188,6 +1250,7 @@
11881250 
11891251 int rga_request_alloc(uint32_t flags, struct rga_session *session)
11901252 {
1253
+	int new_id;
11911254 	struct rga_pending_request_manager *request_manager;
11921255 	struct rga_request *request;
11931256 
....@@ -1218,17 +1281,17 @@
12181281 	mutex_lock(&request_manager->lock);
12191282 
12201283 	idr_preload(GFP_KERNEL);
1221
-	request->id = idr_alloc(&request_manager->request_idr, request, 1, 0, GFP_KERNEL);
1284
+	new_id = idr_alloc_cyclic(&request_manager->request_idr, request, 1, 0, GFP_NOWAIT);
12221285 	idr_preload_end();
1223
-
1224
-	if (request->id <= 0) {
1225
-		pr_err("alloc request_id failed!\n");
1286
+	if (new_id < 0) {
1287
+		pr_err("request alloc id failed!\n");
12261288 
12271289 		mutex_unlock(&request_manager->lock);
12281290 		kfree(request);
1229
-		return -EFAULT;
1291
+		return new_id;
12301292 	}
12311293 
1294
+	request->id = new_id;
12321295 	request_manager->request_count++;
12331296 
12341297 	mutex_unlock(&request_manager->lock);