| .. | .. |
|---|
| 13 | 13 | #include "rga_mm.h" |
|---|
| 14 | 14 | #include "rga_iommu.h" |
|---|
| 15 | 15 | #include "rga_debugger.h" |
|---|
| 16 | +#include "rga_common.h" |
|---|
| 16 | 17 | |
|---|
| 17 | 18 | static void rga_job_free(struct rga_job *job) |
|---|
| 18 | 19 | { |
|---|
| .. | .. |
|---|
| 40 | 41 | |
|---|
| 41 | 42 | static int rga_job_cleanup(struct rga_job *job) |
|---|
| 42 | 43 | { |
|---|
| 43 | | - if (DEBUGGER_EN(TIME)) |
|---|
| 44 | | - pr_err("(pid:%d) job clean use time = %lld\n", job->pid, |
|---|
| 45 | | - ktime_us_delta(ktime_get(), job->timestamp)); |
|---|
| 46 | | - |
|---|
| 47 | 44 | rga_job_put(job); |
|---|
| 45 | + |
|---|
| 46 | + if (DEBUGGER_EN(TIME)) |
|---|
| 47 | + pr_info("request[%d], job cleanup total cost time %lld us\n", |
|---|
| 48 | + job->request_id, |
|---|
| 49 | + ktime_us_delta(ktime_get(), job->timestamp)); |
|---|
| 48 | 50 | |
|---|
| 49 | 51 | return 0; |
|---|
| 50 | 52 | } |
|---|
| .. | .. |
|---|
| 272 | 274 | if (DEBUGGER_EN(DUMP_IMAGE)) |
|---|
| 273 | 275 | rga_dump_job_image(job); |
|---|
| 274 | 276 | |
|---|
| 275 | | - if (DEBUGGER_EN(TIME)) { |
|---|
| 276 | | - pr_info("hw use time = %lld\n", ktime_us_delta(now, job->hw_running_time)); |
|---|
| 277 | | - pr_info("(pid:%d) job done use time = %lld\n", job->pid, |
|---|
| 278 | | - ktime_us_delta(now, job->timestamp)); |
|---|
| 279 | | - } |
|---|
| 277 | + if (DEBUGGER_EN(TIME)) |
|---|
| 278 | + pr_info("request[%d], hardware[%s] cost time %lld us\n", |
|---|
| 279 | + job->request_id, |
|---|
| 280 | + rga_get_core_name(scheduler->core), |
|---|
| 281 | + ktime_us_delta(now, job->hw_running_time)); |
|---|
| 280 | 282 | |
|---|
| 281 | 283 | rga_mm_unmap_job_info(job); |
|---|
| 282 | 284 | |
|---|
| .. | .. |
|---|
| 492 | 494 | return false; |
|---|
| 493 | 495 | } |
|---|
| 494 | 496 | |
|---|
| 495 | | -static int rga_request_get_current_mm(struct rga_request *request) |
|---|
| 497 | +static struct mm_struct *rga_request_get_current_mm(struct rga_request *request) |
|---|
| 496 | 498 | { |
|---|
| 497 | 499 | int i; |
|---|
| 498 | 500 | |
|---|
| .. | .. |
|---|
| 500 | 502 | if (rga_is_need_current_mm(&(request->task_list[i]))) { |
|---|
| 501 | 503 | mmgrab(current->mm); |
|---|
| 502 | 504 | mmget(current->mm); |
|---|
| 503 | | - request->current_mm = current->mm; |
|---|
| 504 | 505 | |
|---|
| 505 | | - break; |
|---|
| 506 | + return current->mm; |
|---|
| 506 | 507 | } |
|---|
| 507 | 508 | } |
|---|
| 508 | 509 | |
|---|
| 509 | | - return 0; |
|---|
| 510 | + return NULL; |
|---|
| 510 | 511 | } |
|---|
| 511 | 512 | |
|---|
| 512 | | -static void rga_request_put_current_mm(struct rga_request *request) |
|---|
| 513 | +static void rga_request_put_current_mm(struct mm_struct *mm) |
|---|
| 513 | 514 | { |
|---|
| 514 | | - if (request->current_mm == NULL) |
|---|
| 515 | + if (mm == NULL) |
|---|
| 515 | 516 | return; |
|---|
| 516 | 517 | |
|---|
| 517 | | - mmput(request->current_mm); |
|---|
| 518 | | - mmdrop(request->current_mm); |
|---|
| 519 | | - request->current_mm = NULL; |
|---|
| 518 | + mmput(mm); |
|---|
| 519 | + mmdrop(mm); |
|---|
| 520 | 520 | } |
|---|
| 521 | 521 | |
|---|
| 522 | 522 | static int rga_request_add_acquire_fence_callback(int acquire_fence_fd, |
|---|
| .. | .. |
|---|
| 693 | 693 | static void rga_request_release_abort(struct rga_request *request, int err_code) |
|---|
| 694 | 694 | { |
|---|
| 695 | 695 | unsigned long flags; |
|---|
| 696 | + struct mm_struct *current_mm; |
|---|
| 696 | 697 | struct rga_pending_request_manager *request_manager = rga_drvdata->pend_request_manager; |
|---|
| 697 | 698 | |
|---|
| 698 | 699 | if (rga_request_scheduler_job_abort(request) > 0) |
|---|
| .. | .. |
|---|
| 707 | 708 | |
|---|
| 708 | 709 | request->is_running = false; |
|---|
| 709 | 710 | request->is_done = false; |
|---|
| 710 | | - |
|---|
| 711 | | - rga_request_put_current_mm(request); |
|---|
| 711 | + current_mm = request->current_mm; |
|---|
| 712 | + request->current_mm = NULL; |
|---|
| 712 | 713 | |
|---|
| 713 | 714 | spin_unlock_irqrestore(&request->lock, flags); |
|---|
| 715 | + |
|---|
| 716 | + rga_request_put_current_mm(current_mm); |
|---|
| 714 | 717 | |
|---|
| 715 | 718 | rga_dma_fence_signal(request->release_fence, err_code); |
|---|
| 716 | 719 | |
|---|
| .. | .. |
|---|
| 820 | 823 | struct rga_job *job; |
|---|
| 821 | 824 | |
|---|
| 822 | 825 | for (i = 0; i < request->task_count; i++) { |
|---|
| 823 | | - job = rga_job_commit(&(request->task_list[i]), request); |
|---|
| 826 | + struct rga_req *req = &(request->task_list[i]); |
|---|
| 827 | + |
|---|
| 828 | + if (DEBUGGER_EN(MSG)) { |
|---|
| 829 | + pr_info("commit request[%d] task[%d]:\n", request->id, i); |
|---|
| 830 | + rga_cmd_print_debug_info(req); |
|---|
| 831 | + } |
|---|
| 832 | + |
|---|
| 833 | + job = rga_job_commit(req, request); |
|---|
| 824 | 834 | if (IS_ERR(job)) { |
|---|
| 825 | 835 | pr_err("request[%d] task[%d] job_commit failed.\n", request->id, i); |
|---|
| 826 | 836 | rga_request_release_abort(request, PTR_ERR(job)); |
|---|
| .. | .. |
|---|
| 841 | 851 | static void rga_request_acquire_fence_signaled_cb(struct dma_fence *fence, |
|---|
| 842 | 852 | struct dma_fence_cb *_waiter) |
|---|
| 843 | 853 | { |
|---|
| 854 | + int ret; |
|---|
| 855 | + unsigned long flags; |
|---|
| 856 | + struct mm_struct *current_mm; |
|---|
| 844 | 857 | struct rga_fence_waiter *waiter = (struct rga_fence_waiter *)_waiter; |
|---|
| 845 | 858 | struct rga_request *request = (struct rga_request *)waiter->private; |
|---|
| 846 | 859 | struct rga_pending_request_manager *request_manager = rga_drvdata->pend_request_manager; |
|---|
| 847 | 860 | |
|---|
| 848 | | - if (rga_request_commit(request)) |
|---|
| 849 | | - pr_err("rga request[%d] commit failed!\n", request->id); |
|---|
| 861 | + ret = rga_request_commit(request); |
|---|
| 862 | + if (ret < 0) { |
|---|
| 863 | + pr_err("acquire_fence callback: rga request[%d] commit failed!\n", request->id); |
|---|
| 864 | + |
|---|
| 865 | + spin_lock_irqsave(&request->lock, flags); |
|---|
| 866 | + |
|---|
| 867 | + request->is_running = false; |
|---|
| 868 | + current_mm = request->current_mm; |
|---|
| 869 | + request->current_mm = NULL; |
|---|
| 870 | + |
|---|
| 871 | + spin_unlock_irqrestore(&request->lock, flags); |
|---|
| 872 | + |
|---|
| 873 | + rga_request_put_current_mm(current_mm); |
|---|
| 874 | + |
|---|
| 875 | + /* |
|---|
| 876 | + * Since the callback is called while holding &dma_fence.lock, |
|---|
| 877 | + * the _locked API is used here. |
|---|
| 878 | + */ |
|---|
| 879 | + if (dma_fence_get_status_locked(request->release_fence) == 0) |
|---|
| 880 | + dma_fence_signal_locked(request->release_fence); |
|---|
| 881 | + } |
|---|
| 850 | 882 | |
|---|
| 851 | 883 | mutex_lock(&request_manager->lock); |
|---|
| 852 | 884 | rga_request_put(request); |
|---|
| .. | .. |
|---|
| 859 | 891 | { |
|---|
| 860 | 892 | struct rga_pending_request_manager *request_manager; |
|---|
| 861 | 893 | struct rga_request *request; |
|---|
| 894 | + struct mm_struct *current_mm; |
|---|
| 862 | 895 | int finished_count, failed_count; |
|---|
| 863 | 896 | bool is_finished = false; |
|---|
| 864 | 897 | unsigned long flags; |
|---|
| .. | .. |
|---|
| 895 | 928 | |
|---|
| 896 | 929 | spin_unlock_irqrestore(&request->lock, flags); |
|---|
| 897 | 930 | |
|---|
| 898 | | - rga_job_cleanup(job); |
|---|
| 899 | | - |
|---|
| 900 | 931 | if ((failed_count + finished_count) >= request->task_count) { |
|---|
| 901 | 932 | spin_lock_irqsave(&request->lock, flags); |
|---|
| 902 | 933 | |
|---|
| 903 | 934 | request->is_running = false; |
|---|
| 904 | 935 | request->is_done = true; |
|---|
| 905 | | - |
|---|
| 906 | | - rga_request_put_current_mm(request); |
|---|
| 936 | + current_mm = request->current_mm; |
|---|
| 937 | + request->current_mm = NULL; |
|---|
| 907 | 938 | |
|---|
| 908 | 939 | spin_unlock_irqrestore(&request->lock, flags); |
|---|
| 940 | + |
|---|
| 941 | + rga_request_put_current_mm(current_mm); |
|---|
| 909 | 942 | |
|---|
| 910 | 943 | rga_dma_fence_signal(request->release_fence, request->ret); |
|---|
| 911 | 944 | |
|---|
| .. | .. |
|---|
| 929 | 962 | rga_request_put(request); |
|---|
| 930 | 963 | |
|---|
| 931 | 964 | mutex_unlock(&request_manager->lock); |
|---|
| 965 | + |
|---|
| 966 | + if (DEBUGGER_EN(TIME)) |
|---|
| 967 | + pr_info("request[%d], job done total cost time %lld us\n", |
|---|
| 968 | + job->request_id, |
|---|
| 969 | + ktime_us_delta(ktime_get(), job->timestamp)); |
|---|
| 970 | + |
|---|
| 971 | + rga_job_cleanup(job); |
|---|
| 932 | 972 | |
|---|
| 933 | 973 | return 0; |
|---|
| 934 | 974 | } |
|---|
| .. | .. |
|---|
| 1059 | 1099 | int ret = 0; |
|---|
| 1060 | 1100 | unsigned long flags; |
|---|
| 1061 | 1101 | struct dma_fence *release_fence; |
|---|
| 1102 | + struct mm_struct *current_mm; |
|---|
| 1103 | + |
|---|
| 1104 | + current_mm = rga_request_get_current_mm(request); |
|---|
| 1062 | 1105 | |
|---|
| 1063 | 1106 | spin_lock_irqsave(&request->lock, flags); |
|---|
| 1064 | 1107 | |
|---|
| .. | .. |
|---|
| 1066 | 1109 | spin_unlock_irqrestore(&request->lock, flags); |
|---|
| 1067 | 1110 | |
|---|
| 1068 | 1111 | pr_err("can not re-config when request is running\n"); |
|---|
| 1069 | | - return -EFAULT; |
|---|
| 1112 | + ret = -EFAULT; |
|---|
| 1113 | + goto err_put_current_mm; |
|---|
| 1070 | 1114 | } |
|---|
| 1071 | 1115 | |
|---|
| 1072 | 1116 | if (request->task_list == NULL) { |
|---|
| 1073 | 1117 | spin_unlock_irqrestore(&request->lock, flags); |
|---|
| 1074 | 1118 | |
|---|
| 1075 | 1119 | pr_err("can not find task list from id[%d]\n", request->id); |
|---|
| 1076 | | - return -EINVAL; |
|---|
| 1120 | + ret = -EINVAL; |
|---|
| 1121 | + goto err_put_current_mm; |
|---|
| 1077 | 1122 | } |
|---|
| 1078 | 1123 | |
|---|
| 1079 | 1124 | /* Reset */ |
|---|
| .. | .. |
|---|
| 1081 | 1126 | request->is_done = false; |
|---|
| 1082 | 1127 | request->finished_task_count = 0; |
|---|
| 1083 | 1128 | request->failed_task_count = 0; |
|---|
| 1084 | | - |
|---|
| 1085 | | - rga_request_get_current_mm(request); |
|---|
| 1129 | + request->current_mm = current_mm; |
|---|
| 1086 | 1130 | |
|---|
| 1087 | 1131 | /* Unlock after ensuring that the current request will not be resubmitted. */ |
|---|
| 1088 | 1132 | spin_unlock_irqrestore(&request->lock, flags); |
|---|
| .. | .. |
|---|
| 1092 | 1136 | if (IS_ERR(release_fence)) { |
|---|
| 1093 | 1137 | pr_err("Can not alloc release fence!\n"); |
|---|
| 1094 | 1138 | ret = IS_ERR(release_fence); |
|---|
| 1095 | | - goto error_put_current_mm; |
|---|
| 1139 | + goto err_reset_request; |
|---|
| 1096 | 1140 | } |
|---|
| 1097 | 1141 | request->release_fence = release_fence; |
|---|
| 1098 | 1142 | |
|---|
| .. | .. |
|---|
| 1141 | 1185 | request->release_fence = NULL; |
|---|
| 1142 | 1186 | } |
|---|
| 1143 | 1187 | |
|---|
| 1144 | | -error_put_current_mm: |
|---|
| 1188 | +err_reset_request: |
|---|
| 1145 | 1189 | spin_lock_irqsave(&request->lock, flags); |
|---|
| 1146 | 1190 | |
|---|
| 1147 | | - rga_request_put_current_mm(request); |
|---|
| 1191 | + request->current_mm = NULL; |
|---|
| 1148 | 1192 | request->is_running = false; |
|---|
| 1149 | 1193 | |
|---|
| 1150 | 1194 | spin_unlock_irqrestore(&request->lock, flags); |
|---|
| 1195 | + |
|---|
| 1196 | +err_put_current_mm: |
|---|
| 1197 | + rga_request_put_current_mm(current_mm); |
|---|
| 1151 | 1198 | |
|---|
| 1152 | 1199 | return ret; |
|---|
| 1153 | 1200 | } |
|---|
| .. | .. |
|---|
| 1237 | 1284 | static void rga_request_kref_release(struct kref *ref) |
|---|
| 1238 | 1285 | { |
|---|
| 1239 | 1286 | struct rga_request *request; |
|---|
| 1287 | + struct mm_struct *current_mm; |
|---|
| 1240 | 1288 | unsigned long flags; |
|---|
| 1241 | 1289 | |
|---|
| 1242 | 1290 | request = container_of(ref, struct rga_request, refcount); |
|---|
| .. | .. |
|---|
| 1246 | 1294 | |
|---|
| 1247 | 1295 | spin_lock_irqsave(&request->lock, flags); |
|---|
| 1248 | 1296 | |
|---|
| 1249 | | - rga_request_put_current_mm(request); |
|---|
| 1250 | 1297 | rga_dma_fence_put(request->release_fence); |
|---|
| 1298 | + current_mm = request->current_mm; |
|---|
| 1299 | + request->current_mm = NULL; |
|---|
| 1251 | 1300 | |
|---|
| 1252 | 1301 | if (!request->is_running || request->is_done) { |
|---|
| 1253 | 1302 | spin_unlock_irqrestore(&request->lock, flags); |
|---|
| 1303 | + |
|---|
| 1304 | + rga_request_put_current_mm(current_mm); |
|---|
| 1305 | + |
|---|
| 1254 | 1306 | goto free_request; |
|---|
| 1255 | 1307 | } |
|---|
| 1256 | 1308 | |
|---|
| 1257 | 1309 | spin_unlock_irqrestore(&request->lock, flags); |
|---|
| 1258 | 1310 | |
|---|
| 1311 | + rga_request_put_current_mm(current_mm); |
|---|
| 1312 | + |
|---|
| 1259 | 1313 | rga_request_scheduler_job_abort(request); |
|---|
| 1260 | 1314 | |
|---|
| 1261 | 1315 | free_request: |
|---|