forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-06 08f87f769b595151be1afeff53e144f543faa614 
 kernel/drivers/gpu/arm/bifrost/mali_kbase_jd.c
....@@ -1,7 +1,7 @@
11 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
22 /*
33  *
4
- * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
4
+ * (C) COPYRIGHT 2010-2022 ARM Limited. All rights reserved.
55  *
66  * This program is free software and is provided to you under the terms of the
77  * GNU General Public License version 2 as published by the Free Software
....@@ -28,6 +28,11 @@
2828 #include <linux/version.h>
2929 #include <linux/ratelimit.h>
3030 #include <linux/priority_control_manager.h>
31
+#if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE
32
+#include <linux/sched/signal.h>
33
+#else
34
+#include <linux/signal.h>
35
+#endif
3136 
3237 #include <mali_kbase_jm.h>
3338 #include <mali_kbase_kinstr_jm.h>
....@@ -35,12 +40,9 @@
3540 #include <tl/mali_kbase_tracepoints.h>
3641 #include <mali_linux_trace.h>
3742 
38
-#include "mali_kbase_dma_fence.h"
3943 #include <mali_kbase_cs_experimental.h>
4044 
4145 #include <mali_kbase_caps.h>
42
-
43
-#define beenthere(kctx, f, a...)  dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
4446 
4547 /* Return whether katom will run on the GPU or not. Currently only soft jobs and
4648  * dependency-only atoms do not run on the GPU
....@@ -76,6 +78,7 @@
7678 	kbase_kinstr_jm_atom_complete(katom);
7779 	dev_dbg(katom->kctx->kbdev->dev, "Atom %pK status to completed\n",
7880 		(void *)katom);
81
+	KBASE_TLSTREAM_TL_JD_ATOM_COMPLETE(katom->kctx->kbdev, katom);
7982 }
8083 
8184 /* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs
....@@ -83,7 +86,7 @@
8386  * Returns whether the JS needs a reschedule.
8487  *
8588  * Note that the caller must also check the atom status and
86
- * if it is KBASE_JD_ATOM_STATE_COMPLETED must call jd_done_nolock
89
+ * if it is KBASE_JD_ATOM_STATE_COMPLETED must call kbase_jd_done_nolock
8790  */
8891 static bool jd_run_atom(struct kbase_jd_atom *katom)
8992 {
....@@ -139,11 +142,17 @@
139142 		/* katom dep complete, attempt to run it */
140143 		bool resched = false;
141144 
145
+		KBASE_TLSTREAM_TL_RUN_ATOM_START(
146
+			katom->kctx->kbdev, katom,
147
+			kbase_jd_atom_id(katom->kctx, katom));
142148 		resched = jd_run_atom(katom);
149
+		KBASE_TLSTREAM_TL_RUN_ATOM_END(katom->kctx->kbdev, katom,
150
+						  kbase_jd_atom_id(katom->kctx,
151
+								   katom));
143152 
144153 		if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) {
145154 			/* The atom has already finished */
146
-			resched |= jd_done_nolock(katom, NULL);
155
+			resched |= kbase_jd_done_nolock(katom, true);
147156 		}
148157 
149158 		if (resched)
....@@ -153,25 +162,12 @@
153162 
154163 void kbase_jd_free_external_resources(struct kbase_jd_atom *katom)
155164 {
156
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
157
-	/* Flush dma-fence workqueue to ensure that any callbacks that may have
158
-	 * been queued are done before continuing.
159
-	 * Any successfully completed atom would have had all it's callbacks
160
-	 * completed before the atom was run, so only flush for failed atoms.
161
-	 */
162
-	if (katom->event_code != BASE_JD_EVENT_DONE)
163
-		flush_workqueue(katom->kctx->dma_fence.wq);
164
-#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */
165165 }
166166 
167167 static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom)
168168 {
169169 	KBASE_DEBUG_ASSERT(katom);
170170 	KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES);
171
-
172
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
173
-	kbase_dma_fence_signal(katom);
174
-#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */
175171 
176172 	kbase_gpu_vm_lock(katom->kctx);
177173 	/* only roll back if extres is non-NULL */
....@@ -180,13 +176,7 @@
180176 
181177 		res_no = katom->nr_extres;
182178 		while (res_no-- > 0) {
183
-			struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc;
184
-			struct kbase_va_region *reg;
185
-
186
-			reg = kbase_region_tracker_find_region_base_address(
187
-					katom->kctx,
188
-					katom->extres[res_no].gpu_address);
189
-			kbase_unmap_external_resource(katom->kctx, reg, alloc);
179
+			kbase_unmap_external_resource(katom->kctx, katom->extres[res_no]);
190180 		}
191181 		kfree(katom->extres);
192182 		katom->extres = NULL;
....@@ -202,26 +192,8 @@
202192 
203193 static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom *user_atom)
204194 {
205
-	int err_ret_val = -EINVAL;
195
+	int err = -EINVAL;
206196 	u32 res_no;
207
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
208
-	struct kbase_dma_fence_resv_info info = {
209
-		.resv_objs = NULL,
210
-		.dma_fence_resv_count = 0,
211
-		.dma_fence_excl_bitmap = NULL
212
-	};
213
-#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)
214
-	/*
215
-	 * When both dma-buf fence and Android native sync is enabled, we
216
-	 * disable dma-buf fence for contexts that are using Android native
217
-	 * fences.
218
-	 */
219
-	const bool implicit_sync = !kbase_ctx_flag(katom->kctx,
220
-						   KCTX_NO_IMPLICIT_SYNC);
221
-#else /* CONFIG_SYNC || CONFIG_SYNC_FILE*/
222
-	const bool implicit_sync = true;
223
-#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */
224
-#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */
225197 	struct base_external_resource *input_extres;
226198 
227199 	KBASE_DEBUG_ASSERT(katom);
....@@ -235,47 +207,18 @@
235207 	if (!katom->extres)
236208 		return -ENOMEM;
237209 
238
-	/* copy user buffer to the end of our real buffer.
239
-	 * Make sure the struct sizes haven't changed in a way
240
-	 * we don't support
241
-	 */
242
-	BUILD_BUG_ON(sizeof(*input_extres) > sizeof(*katom->extres));
243
-	input_extres = (struct base_external_resource *)
244
-			(((unsigned char *)katom->extres) +
245
-			(sizeof(*katom->extres) - sizeof(*input_extres)) *
246
-			katom->nr_extres);
210
+	input_extres = kmalloc_array(katom->nr_extres, sizeof(*input_extres), GFP_KERNEL);
211
+	if (!input_extres) {
212
+		err = -ENOMEM;
213
+		goto failed_input_alloc;
214
+	}
247215 
248216 	if (copy_from_user(input_extres,
249217 			get_compat_pointer(katom->kctx, user_atom->extres_list),
250218 			sizeof(*input_extres) * katom->nr_extres) != 0) {
251
-		err_ret_val = -EINVAL;
252
-		goto early_err_out;
219
+		err = -EINVAL;
220
+		goto failed_input_copy;
253221 	}
254
-
255
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
256
-	if (implicit_sync) {
257
-		info.resv_objs =
258
-			kmalloc_array(katom->nr_extres,
259
-#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE)
260
-				      sizeof(struct reservation_object *),
261
-#else
262
-				      sizeof(struct dma_resv *),
263
-#endif
264
-				      GFP_KERNEL);
265
-		if (!info.resv_objs) {
266
-			err_ret_val = -ENOMEM;
267
-			goto early_err_out;
268
-		}
269
-
270
-		info.dma_fence_excl_bitmap =
271
-				kcalloc(BITS_TO_LONGS(katom->nr_extres),
272
-					sizeof(unsigned long), GFP_KERNEL);
273
-		if (!info.dma_fence_excl_bitmap) {
274
-			err_ret_val = -ENOMEM;
275
-			goto early_err_out;
276
-		}
277
-	}
278
-#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */
279222 
280223 	/* Take the processes mmap lock */
281224 	down_read(kbase_mem_get_process_mmap_lock());
....@@ -283,19 +226,13 @@
283226 	/* need to keep the GPU VM locked while we set up UMM buffers */
284227 	kbase_gpu_vm_lock(katom->kctx);
285228 	for (res_no = 0; res_no < katom->nr_extres; res_no++) {
286
-		struct base_external_resource *res = &input_extres[res_no];
229
+		struct base_external_resource *user_res = &input_extres[res_no];
287230 		struct kbase_va_region *reg;
288
-		struct kbase_mem_phy_alloc *alloc;
289
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
290
-		bool exclusive;
291
-		exclusive = (res->ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE)
292
-				? true : false;
293
-#endif
231
+
294232 		reg = kbase_region_tracker_find_region_enclosing_address(
295
-				katom->kctx,
296
-				res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE);
233
+			katom->kctx, user_res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE);
297234 		/* did we find a matching region object? */
298
-		if (kbase_is_region_invalid_or_free(reg)) {
235
+		if (unlikely(kbase_is_region_invalid_or_free(reg))) {
299236 			/* roll back */
300237 			goto failed_loop;
301238 		}
....@@ -305,36 +242,11 @@
305242 			katom->atom_flags |= KBASE_KATOM_FLAG_PROTECTED;
306243 		}
307244 
308
-		alloc = kbase_map_external_resource(katom->kctx, reg,
309
-				current->mm);
310
-		if (!alloc) {
311
-			err_ret_val = -EINVAL;
245
+		err = kbase_map_external_resource(katom->kctx, reg, current->mm);
246
+		if (err)
312247 			goto failed_loop;
313
-		}
314248 
315
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
316
-		if (implicit_sync &&
317
-		    reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) {
318
-#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE)
319
-			struct reservation_object *resv;
320
-#else
321
-			struct dma_resv *resv;
322
-#endif
323
-			resv = reg->gpu_alloc->imported.umm.dma_buf->resv;
324
-			if (resv)
325
-				kbase_dma_fence_add_reservation(resv, &info,
326
-								exclusive);
327
-		}
328
-#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */
329
-
330
-		/* finish with updating out array with the data we found */
331
-		/* NOTE: It is important that this is the last thing we do (or
332
-		 * at least not before the first write) as we overwrite elements
333
-		 * as we loop and could be overwriting ourself, so no writes
334
-		 * until the last read for an element.
335
-		 */
336
-		katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */
337
-		katom->extres[res_no].alloc = alloc;
249
+		katom->extres[res_no] = reg;
338250 	}
339251 	/* successfully parsed the extres array */
340252 	/* drop the vm lock now */
....@@ -343,57 +255,33 @@
343255 	/* Release the processes mmap lock */
344256 	up_read(kbase_mem_get_process_mmap_lock());
345257 
346
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
347
-	if (implicit_sync) {
348
-		if (info.dma_fence_resv_count) {
349
-			int ret;
350
-
351
-			ret = kbase_dma_fence_wait(katom, &info);
352
-			if (ret < 0)
353
-				goto failed_dma_fence_setup;
354
-		}
355
-
356
-		kfree(info.resv_objs);
357
-		kfree(info.dma_fence_excl_bitmap);
358
-	}
359
-#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */
258
+	/* Free the buffer holding data from userspace */
259
+	kfree(input_extres);
360260 
361261 	/* all done OK */
362262 	return 0;
363263 
364264 /* error handling section */
365
-
366
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
367
-failed_dma_fence_setup:
368
-	/* Lock the processes mmap lock */
369
-	down_read(kbase_mem_get_process_mmap_lock());
370
-
371
-	/* lock before we unmap */
372
-	kbase_gpu_vm_lock(katom->kctx);
373
-#endif
374
-
375
- failed_loop:
376
-	/* undo the loop work */
265
+failed_loop:
266
+	/* undo the loop work. We are guaranteed to have access to the VA region
267
+	 * as we hold a reference to it until it's unmapped
268
+	 */
377269 	while (res_no-- > 0) {
378
-		struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc;
270
+		struct kbase_va_region *reg = katom->extres[res_no];
379271 
380
-		kbase_unmap_external_resource(katom->kctx, NULL, alloc);
272
+		kbase_unmap_external_resource(katom->kctx, reg);
381273 	}
382274 	kbase_gpu_vm_unlock(katom->kctx);
383275 
384276 	/* Release the processes mmap lock */
385277 	up_read(kbase_mem_get_process_mmap_lock());
386278 
387
- early_err_out:
279
+failed_input_copy:
280
+	kfree(input_extres);
281
+failed_input_alloc:
388282 	kfree(katom->extres);
389283 	katom->extres = NULL;
390
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
391
-	if (implicit_sync) {
392
-		kfree(info.resv_objs);
393
-		kfree(info.dma_fence_excl_bitmap);
394
-	}
395
-#endif
396
-	return err_ret_val;
284
+	return err;
397285 }
398286 
399287 static inline void jd_resolve_dep(struct list_head *out_list,
....@@ -416,10 +304,6 @@
416304 
417305 		if (katom->event_code != BASE_JD_EVENT_DONE &&
418306 			(dep_type != BASE_JD_DEP_TYPE_ORDER)) {
419
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
420
-			kbase_dma_fence_cancel_callbacks(dep_atom);
421
-#endif
422
-
423307 			dep_atom->event_code = katom->event_code;
424308 			KBASE_DEBUG_ASSERT(dep_atom->status !=
425309 						KBASE_JD_ATOM_STATE_UNUSED);
....@@ -433,35 +317,8 @@
433317 				(IS_GPU_ATOM(dep_atom) && !ctx_is_dying &&
434318 				!dep_atom->will_fail_event_code &&
435319 				!other_dep_atom->will_fail_event_code))) {
436
-			bool dep_satisfied = true;
437
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
438
-			int dep_count;
439
-
440
-			dep_count = kbase_fence_dep_count_read(dep_atom);
441
-			if (likely(dep_count == -1)) {
442
-				dep_satisfied = true;
443
-			} else {
444
-				/*
445
-				 * There are either still active callbacks, or
446
-				 * all fences for this @dep_atom has signaled,
447
-				 * but the worker that will queue the atom has
448
-				 * not yet run.
449
-				 *
450
-				 * Wait for the fences to signal and the fence
451
-				 * worker to run and handle @dep_atom. If
452
-				 * @dep_atom was completed due to error on
453
-				 * @katom, then the fence worker will pick up
454
-				 * the complete status and error code set on
455
-				 * @dep_atom above.
456
-				 */
457
-				dep_satisfied = false;
458
-			}
459
-#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */
460
-
461
-			if (dep_satisfied) {
462
-				dep_atom->in_jd_list = true;
463
-				list_add_tail(&dep_atom->jd_item, out_list);
464
-			}
320
+			dep_atom->in_jd_list = true;
321
+			list_add_tail(&dep_atom->jd_item, out_list);
465322 		}
466323 	}
467324 }
....@@ -520,33 +377,8 @@
520377 						dep_atom->dep[0].atom);
521378 				bool dep1_valid = is_dep_valid(
522379 						dep_atom->dep[1].atom);
523
-				bool dep_satisfied = true;
524
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
525
-				int dep_count;
526380 
527
-				dep_count = kbase_fence_dep_count_read(
528
-								dep_atom);
529
-				if (likely(dep_count == -1)) {
530
-					dep_satisfied = true;
531
-				} else {
532
-				/*
533
-				 * There are either still active callbacks, or
534
-				 * all fences for this @dep_atom has signaled,
535
-				 * but the worker that will queue the atom has
536
-				 * not yet run.
537
-				 *
538
-				 * Wait for the fences to signal and the fence
539
-				 * worker to run and handle @dep_atom. If
540
-				 * @dep_atom was completed due to error on
541
-				 * @katom, then the fence worker will pick up
542
-				 * the complete status and error code set on
543
-				 * @dep_atom above.
544
-				 */
545
-					dep_satisfied = false;
546
-				}
547
-#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */
548
-
549
-				if (dep0_valid && dep1_valid && dep_satisfied) {
381
+				if (dep0_valid && dep1_valid) {
550382 					dep_atom->in_jd_list = true;
551383 					list_add(&dep_atom->jd_item, out_list);
552384 				}
....@@ -612,8 +444,8 @@
612444 		else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP)
613445 			size_to_read = sizeof(u64[COUNT]);
614446 
615
-		ptr = kbase_vmap(kctx, reg->heap_info_gpu_addr, size_to_read,
616
-				&mapping);
447
+		ptr = kbase_vmap_prot(kctx, reg->heap_info_gpu_addr, size_to_read,
448
+				KBASE_REG_CPU_RD, &mapping);
617449 
618450 		if (!ptr) {
619451 			dev_warn(kctx->kbdev->dev,
....@@ -697,23 +529,17 @@
697529 }
698530 #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
699531 
700
-/*
701
- * Perform the necessary handling of an atom that has finished running
702
- * on the GPU.
703
- *
704
- * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller
705
- * is responsible for calling kbase_finish_soft_job *before* calling this function.
706
- *
707
- * The caller must hold the kbase_jd_context.lock.
708
- */
709
-bool jd_done_nolock(struct kbase_jd_atom *katom,
710
-		struct list_head *completed_jobs_ctx)
532
+bool kbase_jd_done_nolock(struct kbase_jd_atom *katom, bool post_immediately)
711533 {
712534 	struct kbase_context *kctx = katom->kctx;
713535 	struct list_head completed_jobs;
714536 	struct list_head runnable_jobs;
715537 	bool need_to_try_schedule_context = false;
716538 	int i;
539
+
540
+	lockdep_assert_held(&kctx->jctx.lock);
541
+
542
+	KBASE_TLSTREAM_TL_JD_DONE_NO_LOCK_START(kctx->kbdev, katom);
717543 
718544 	INIT_LIST_HEAD(&completed_jobs);
719545 	INIT_LIST_HEAD(&runnable_jobs);
....@@ -736,6 +562,7 @@
736562 	}
737563 
738564 	jd_mark_atom_complete(katom);
565
+
739566 	list_add_tail(&katom->jd_item, &completed_jobs);
740567 
741568 	while (!list_empty(&completed_jobs)) {
....@@ -767,7 +594,13 @@
767594 
768595 			if (node->status != KBASE_JD_ATOM_STATE_COMPLETED &&
769596 					!kbase_ctx_flag(kctx, KCTX_DYING)) {
597
+				KBASE_TLSTREAM_TL_RUN_ATOM_START(
598
+					kctx->kbdev, node,
599
+					kbase_jd_atom_id(kctx, node));
770600 				need_to_try_schedule_context |= jd_run_atom(node);
601
+				KBASE_TLSTREAM_TL_RUN_ATOM_END(
602
+					kctx->kbdev, node,
603
+					kbase_jd_atom_id(kctx, node));
771604 			} else {
772605 				node->event_code = katom->event_code;
773606 
....@@ -795,10 +628,10 @@
795628 		 * is in a disjoint state (ie. being reset).
796629 		 */
797630 		kbase_disjoint_event_potential(kctx->kbdev);
798
-		if (completed_jobs_ctx)
799
-			list_add_tail(&katom->jd_item, completed_jobs_ctx);
800
-		else
631
+		if (post_immediately && list_empty(&kctx->completed_jobs))
801632 			kbase_event_post(kctx, katom);
633
+		else
634
+			list_add_tail(&katom->jd_item, &kctx->completed_jobs);
802635 
803636 		/* Decrement and check the TOTAL number of jobs. This includes
804637 		 * those not tracked by the scheduler: 'not ready to run' and
....@@ -811,11 +644,11 @@
811644 			 */
812645 			wake_up(&kctx->jctx.zero_jobs_wait);
813646 	}
814
-
647
+	KBASE_TLSTREAM_TL_JD_DONE_NO_LOCK_END(kctx->kbdev, katom);
815648 	return need_to_try_schedule_context;
816649 }
817650 
818
-KBASE_EXPORT_TEST_API(jd_done_nolock);
651
+KBASE_EXPORT_TEST_API(kbase_jd_done_nolock);
819652 
820653 #if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
821654 enum {
....@@ -923,7 +756,6 @@
923756 	katom->jobslot = user_atom->jobslot;
924757 	katom->seq_nr = user_atom->seq_nr;
925758 	katom->atom_flags = 0;
926
-	katom->retry_count = 0;
927759 	katom->need_cache_flush_cores_retained = 0;
928760 	katom->pre_dep = NULL;
929761 	katom->post_dep = NULL;
....@@ -957,14 +789,11 @@
957789 
958790 	INIT_LIST_HEAD(&katom->queue);
959791 	INIT_LIST_HEAD(&katom->jd_item);
960
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
961
-	kbase_fence_dep_count_set(katom, -1);
962
-#endif
963792 
964793 	/* Don't do anything if there is a mess up with dependencies.
965
-	   This is done in a separate cycle to check both the dependencies at ones, otherwise
966
-	   it will be extra complexity to deal with 1st dependency ( just added to the list )
967
-	   if only the 2nd one has invalid config.
794
+	 * This is done in a separate cycle to check both the dependencies at ones, otherwise
795
+	 * it will be extra complexity to deal with 1st dependency ( just added to the list )
796
+	 * if only the 2nd one has invalid config.
968797 	 */
969798 	for (i = 0; i < 2; i++) {
970799 		int dep_atom_number = user_atom->pre_dep[i].atom_id;
....@@ -984,8 +813,7 @@
984813 				 * dependencies.
985814 				 */
986815 				jd_trace_atom_submit(kctx, katom, NULL);
987
-
988
-				return jd_done_nolock(katom, NULL);
816
+				return kbase_jd_done_nolock(katom, true);
989817 			}
990818 		}
991819 	}
....@@ -1049,8 +877,7 @@
1049877 				if (err >= 0)
1050878 					kbase_finish_soft_job(katom);
1051879 			}
1052
-
1053
-			return jd_done_nolock(katom, NULL);
880
+			return kbase_jd_done_nolock(katom, true);
1054881 		}
1055882 
1056883 		katom->will_fail_event_code = katom->event_code;
....@@ -1076,7 +903,7 @@
1076903 	/* Create a new atom. */
1077904 	jd_trace_atom_submit(kctx, katom, &katom->sched_priority);
1078905 
1079
-#if !MALI_INCREMENTAL_RENDERING
906
+#if !MALI_INCREMENTAL_RENDERING_JM
1080907 	/* Reject atoms for incremental rendering if not supported */
1081908 	if (katom->core_req &
1082909 	(BASE_JD_REQ_START_RENDERPASS|BASE_JD_REQ_END_RENDERPASS)) {
....@@ -1084,9 +911,9 @@
1084911 			"Rejecting atom with unsupported core_req 0x%x\n",
1085912 			katom->core_req);
1086913 		katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1087
-		return jd_done_nolock(katom, NULL);
914
+		return kbase_jd_done_nolock(katom, true);
1088915 	}
1089
-#endif /* !MALI_INCREMENTAL_RENDERING */
916
+#endif /* !MALI_INCREMENTAL_RENDERING_JM */
1090917 
1091918 	if (katom->core_req & BASE_JD_REQ_END_RENDERPASS) {
1092919 		WARN_ON(katom->jc != 0);
....@@ -1098,7 +925,7 @@
1098925 		 */
1099926 		dev_err(kctx->kbdev->dev, "Rejecting atom with jc = NULL\n");
1100927 		katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1101
-		return jd_done_nolock(katom, NULL);
928
+		return kbase_jd_done_nolock(katom, true);
1102929 	}
1103930 
1104931 	/* Reject atoms with an invalid device_nr */
....@@ -1108,7 +935,7 @@
1108935 				"Rejecting atom with invalid device_nr %d\n",
1109936 				katom->device_nr);
1110937 		katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1111
-		return jd_done_nolock(katom, NULL);
938
+		return kbase_jd_done_nolock(katom, true);
1112939 	}
1113940 
1114941 	/* Reject atoms with invalid core requirements */
....@@ -1118,7 +945,7 @@
1118945 				"Rejecting atom with invalid core requirements\n");
1119946 		katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1120947 		katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE;
1121
-		return jd_done_nolock(katom, NULL);
948
+		return kbase_jd_done_nolock(katom, true);
1122949 	}
1123950 
1124951 	/* Reject soft-job atom of certain types from accessing external resources */
....@@ -1129,7 +956,7 @@
1129956 		dev_err(kctx->kbdev->dev,
1130957 				"Rejecting soft-job atom accessing external resources\n");
1131958 		katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1132
-		return jd_done_nolock(katom, NULL);
959
+		return kbase_jd_done_nolock(katom, true);
1133960 	}
1134961 
1135962 	if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) {
....@@ -1137,7 +964,7 @@
1137964 		if (kbase_jd_pre_external_resources(katom, user_atom) != 0) {
1138965 			/* setup failed (no access, bad resource, unknown resource types, etc.) */
1139966 			katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1140
-			return jd_done_nolock(katom, NULL);
967
+			return kbase_jd_done_nolock(katom, true);
1141968 		}
1142969 	}
1143970 
....@@ -1148,7 +975,7 @@
1148975 		 * JIT IDs - atom is invalid.
1149976 		 */
1150977 		katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1151
-		return jd_done_nolock(katom, NULL);
978
+		return kbase_jd_done_nolock(katom, true);
1152979 	}
1153980 #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
1154981 
....@@ -1162,13 +989,13 @@
1162989 	if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) {
1163990 		if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) {
1164991 			katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1165
-			return jd_done_nolock(katom, NULL);
992
+			return kbase_jd_done_nolock(katom, true);
1166993 		}
1167994 	} else {
1168995 		/* Soft-job */
1169996 		if (kbase_prepare_soft_job(katom) != 0) {
1170997 			katom->event_code = BASE_JD_EVENT_JOB_INVALID;
1171
-			return jd_done_nolock(katom, NULL);
998
+			return kbase_jd_done_nolock(katom, true);
1172999 		}
11731000 	}
11741001 
....@@ -1181,16 +1008,10 @@
11811008 	if (queued && !IS_GPU_ATOM(katom))
11821009 		return false;
11831010 
1184
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
1185
-	if (kbase_fence_dep_count_read(katom) != -1)
1186
-		return false;
1187
-
1188
-#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */
1189
-
11901011 	if (katom->core_req & BASE_JD_REQ_SOFT_JOB) {
11911012 		if (kbase_process_soft_job(katom) == 0) {
11921013 			kbase_finish_soft_job(katom);
1193
-			return jd_done_nolock(katom, NULL);
1014
+			return kbase_jd_done_nolock(katom, true);
11941015 		}
11951016 		return false;
11961017 	}
....@@ -1220,7 +1041,7 @@
12201041 	}
12211042 
12221043 	/* This is a pure dependency. Resolve it immediately */
1223
-	return jd_done_nolock(katom, NULL);
1044
+	return kbase_jd_done_nolock(katom, true);
12241045 }
12251046 
12261047 int kbase_jd_submit(struct kbase_context *kctx,
....@@ -1235,15 +1056,13 @@
12351056 	u32 latest_flush;
12361057 
12371058 	bool jd_atom_is_v2 = (stride == sizeof(struct base_jd_atom_v2) ||
1238
-	                      stride == offsetof(struct base_jd_atom_v2, renderpass_id));
1059
+		stride == offsetof(struct base_jd_atom_v2, renderpass_id));
12391060 
12401061 	/*
12411062 	 * kbase_jd_submit isn't expected to fail and so all errors with the
12421063 	 * jobs are reported by immediately failing them (through event system)
12431064 	 */
12441065 	kbdev = kctx->kbdev;
1245
-
1246
-	beenthere(kctx, "%s", "Enter");
12471066 
12481067 	if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) {
12491068 		dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it\n");
....@@ -1260,18 +1079,26 @@
12601079 		return -EINVAL;
12611080 	}
12621081 
1082
+	if (nr_atoms > BASE_JD_ATOM_COUNT) {
1083
+		dev_dbg(kbdev->dev, "Invalid attempt to submit %u atoms at once for kctx %d_%d",
1084
+			nr_atoms, kctx->tgid, kctx->id);
1085
+		return -EINVAL;
1086
+	}
1087
+
12631088 	/* All atoms submitted in this call have the same flush ID */
12641089 	latest_flush = kbase_backend_get_current_flush_id(kbdev);
12651090 
12661091 	for (i = 0; i < nr_atoms; i++) {
1267
-		struct base_jd_atom user_atom;
1092
+		struct base_jd_atom user_atom = {
1093
+			.seq_nr = 0,
1094
+		};
12681095 		struct base_jd_fragment user_jc_incr;
12691096 		struct kbase_jd_atom *katom;
12701097 
12711098 		if (unlikely(jd_atom_is_v2)) {
12721099 			if (copy_from_user(&user_atom.jc, user_addr, sizeof(struct base_jd_atom_v2)) != 0) {
12731100 				dev_dbg(kbdev->dev,
1274
-					"Invalid atom address %p passed to job_submit\n",
1101
+					"Invalid atom address %pK passed to job_submit\n",
12751102 					user_addr);
12761103 				err = -EFAULT;
12771104 				break;
....@@ -1282,7 +1109,7 @@
12821109 		} else {
12831110 			if (copy_from_user(&user_atom, user_addr, stride) != 0) {
12841111 				dev_dbg(kbdev->dev,
1285
-					"Invalid atom address %p passed to job_submit\n",
1112
+					"Invalid atom address %pK passed to job_submit\n",
12861113 					user_addr);
12871114 				err = -EFAULT;
12881115 				break;
....@@ -1378,16 +1205,22 @@
13781205 			}
13791206 			mutex_lock(&jctx->lock);
13801207 		}
1381
-
1208
+		KBASE_TLSTREAM_TL_JD_SUBMIT_ATOM_START(kbdev, katom);
13821209 		need_to_try_schedule_context |= jd_submit_atom(kctx, &user_atom,
13831210 			&user_jc_incr, katom);
1384
-
1211
+		KBASE_TLSTREAM_TL_JD_SUBMIT_ATOM_END(kbdev, katom);
13851212 		/* Register a completed job as a disjoint event when the GPU is in a disjoint state
13861213 		 * (ie. being reset).
13871214 		 */
13881215 		kbase_disjoint_event_potential(kbdev);
13891216 
13901217 		mutex_unlock(&jctx->lock);
1218
+		if (fatal_signal_pending(current)) {
1219
+			dev_dbg(kbdev->dev, "Fatal signal pending for kctx %d_%d",
1220
+				kctx->tgid, kctx->id);
1221
+			/* We're being killed so the result code doesn't really matter  */
1222
+			return 0;
1223
+		}
13911224 	}
13921225 
13931226 	if (need_to_try_schedule_context)
....@@ -1463,11 +1296,13 @@
14631296 	}
14641297 
14651298 	if ((katom->event_code != BASE_JD_EVENT_DONE) &&
1466
-			(!kbase_ctx_flag(katom->kctx, KCTX_DYING)))
1467
-		dev_err(kbdev->dev,
1468
-			"t6xx: GPU fault 0x%02lx from job slot %d\n",
1469
-					(unsigned long)katom->event_code,
1470
-								katom->slot_nr);
1299
+			(!kbase_ctx_flag(katom->kctx, KCTX_DYING))) {
1300
+		if (!kbase_is_quick_reset_enabled(kbdev))
1301
+			dev_err(kbdev->dev,
1302
+				"t6xx: GPU fault 0x%02lx from job slot %d\n",
1303
+						(unsigned long)katom->event_code,
1304
+									katom->slot_nr);
1305
+	}
14711306 
14721307 	/* Retain state before the katom disappears */
14731308 	kbasep_js_atom_retained_state_copy(&katom_retained_state, katom);
....@@ -1479,9 +1314,8 @@
14791314 	kbasep_js_remove_job(kbdev, kctx, katom);
14801315 	mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
14811316 	mutex_unlock(&js_devdata->queue_mutex);
1482
-	katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF;
1483
-	/* jd_done_nolock() requires the jsctx_mutex lock to be dropped */
1484
-	jd_done_nolock(katom, &kctx->completed_jobs);
1317
+	/* kbase_jd_done_nolock() requires the jsctx_mutex lock to be dropped */
1318
+	kbase_jd_done_nolock(katom, false);
14851319 
14861320 	/* katom may have been freed now, do not use! */
14871321 
....@@ -1498,22 +1332,23 @@
14981332 		 * drop our reference. But do not call kbase_jm_idle_ctx(), as
14991333 		 * the context is active and fast-starting is allowed.
15001334 		 *
1501
-		 * If an atom has been fast-started then kctx->atoms_pulled will
1502
-		 * be non-zero but KCTX_ACTIVE will still be false (as the
1503
-		 * previous pm reference has been inherited). Do NOT drop our
1504
-		 * reference, as it has been re-used, and leave the context as
1505
-		 * active.
1335
+		 * If an atom has been fast-started then
1336
+		 * kbase_jsctx_atoms_pulled(kctx) will return non-zero but
1337
+		 * KCTX_ACTIVE will still be false (as the previous pm
1338
+		 * reference has been inherited). Do NOT drop our reference, as
1339
+		 * it has been re-used, and leave the context as active.
15061340 		 *
1507
-		 * If no new atoms have been started then KCTX_ACTIVE will still
1508
-		 * be false and atoms_pulled will be zero, so drop the reference
1509
-		 * and call kbase_jm_idle_ctx().
1341
+		 * If no new atoms have been started then KCTX_ACTIVE will
1342
+		 * still be false and kbase_jsctx_atoms_pulled(kctx) will
1343
+		 * return zero, so drop the reference and call
1344
+		 * kbase_jm_idle_ctx().
15101345 		 *
15111346 		 * As the checks are done under both the queue_mutex and
15121347 		 * hwaccess_lock is should be impossible for this to race
15131348 		 * with the scheduler code.
15141349 		 */
15151350 		if (kbase_ctx_flag(kctx, KCTX_ACTIVE) ||
1516
-		    !atomic_read(&kctx->atoms_pulled)) {
1351
+		    !kbase_jsctx_atoms_pulled(kctx)) {
15171352 			/* Calling kbase_jm_idle_ctx() here will ensure that
15181353 			 * atoms are not fast-started when we drop the
15191354 			 * hwaccess_lock. This is not performed if
....@@ -1546,7 +1381,7 @@
15461381 	kbase_js_sched_all(kbdev);
15471382 
15481383 	if (!atomic_dec_return(&kctx->work_count)) {
1549
-		/* If worker now idle then post all events that jd_done_nolock()
1384
+		/* If worker now idle then post all events that kbase_jd_done_nolock()
15501385 		 * has queued
15511386 		 */
15521387 		mutex_lock(&jctx->lock);
....@@ -1595,6 +1430,7 @@
15951430 	bool need_to_try_schedule_context;
15961431 	bool attr_state_changed;
15971432 	struct kbase_device *kbdev;
1433
+	CSTD_UNUSED(need_to_try_schedule_context);
15981434 
15991435 	/* Soft jobs should never reach this function */
16001436 	KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0);
....@@ -1620,7 +1456,7 @@
16201456 
16211457 	mutex_lock(&jctx->lock);
16221458 
1623
-	need_to_try_schedule_context = jd_done_nolock(katom, NULL);
1459
+	need_to_try_schedule_context = kbase_jd_done_nolock(katom, true);
16241460 	/* Because we're zapping, we're not adding any more jobs to this ctx, so no need to
16251461 	 * schedule the context. There's also no need for the jsctx_mutex to have been taken
16261462 	 * around this too.
....@@ -1663,6 +1499,8 @@
16631499 	KBASE_DEBUG_ASSERT(kctx);
16641500 	kbdev = kctx->kbdev;
16651501 	KBASE_DEBUG_ASSERT(kbdev);
1502
+
1503
+	lockdep_assert_held(&kbdev->hwaccess_lock);
16661504 
16671505 	if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT)
16681506 		katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
....@@ -1740,19 +1578,7 @@
17401578 		kbase_cancel_soft_job(katom);
17411579 	}
17421580 
1743
-
1744
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
1745
-	kbase_dma_fence_cancel_all_atoms(kctx);
1746
-#endif
1747
-
17481581 	mutex_unlock(&kctx->jctx.lock);
1749
-
1750
-#ifdef CONFIG_MALI_BIFROST_DMA_FENCE
1751
-	/* Flush dma-fence workqueue to ensure that any callbacks that may have
1752
-	 * been queued are done before continuing.
1753
-	 */
1754
-	flush_workqueue(kctx->dma_fence.wq);
1755
-#endif
17561582 
17571583 #if IS_ENABLED(CONFIG_DEBUG_FS)
17581584 	kbase_debug_job_fault_kctx_unblock(kctx);
....@@ -1790,11 +1616,10 @@
17901616 		kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID;
17911617 		kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED;
17921618 
1793
-#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE)
1619
+#if IS_ENABLED(CONFIG_SYNC_FILE)
17941620 		kctx->jctx.atoms[i].dma_fence.context =
17951621 						dma_fence_context_alloc(1);
17961622 		atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0);
1797
-		INIT_LIST_HEAD(&kctx->jctx.atoms[i].dma_fence.callbacks);
17981623 #endif
17991624 	}
18001625